diff --git a/python/example.py b/python/example.py deleted file mode 100644 index a4c2a88..0000000 --- a/python/example.py +++ /dev/null @@ -1,137 +0,0 @@ - -#-----------------------------------------------------------------------------# - -import raw_eater -import raw_meat -import pyarrow as pa -import pyarrow.parquet as pq -from pathlib import Path - -fileobj1 = Path("samples/datasetA/").rglob("*.raw") -rawlist1 = [str(fl) for fl in fileobj1] - -fileobj2 = Path("samples/datasetB/").rglob("*.raw") -rawlist2 = [str(fl) for fl in fileobj2] - -rawlist = rawlist1 #[rawlist1[0],rawlist1[4],rawlist2[0],rawlist2[6]] -for fil in rawlist2 : - rawlist.append(fil) -rawlist.append("./README.md") - -print("") -print(rawlist) -print() - -#-----------------------------------------------------------------------------# - -# alternatively create "empty" instance of "raw_eater" and set file names -eatraw = raw_eater.raweater() -# eatraw.set_file("../smp/pressure_Vacuum.raw".encode()) - -# convert every single listed file -for rf in rawlist : - - print("converting " + str(rf) + "...\n" + 90*("-") + "\n") - - # setup instance of "raw_eater" and trigger conversion - # eatraw = raw_eater.raweater(rf.encode()) - # eatraw = raw_meat.rawmerger(rf.encode()) - - # use global instance of "raw_eater" to set file and perform decoding - eatraw.set_file(rf.encode()) - try : - eatraw.do_conversion() - except RuntimeError as e : - print("conversion failed: " + str(e)) - - # check validity of file format - if eatraw.validity() : - - # show channel name and its unit - entity = eatraw.channel_name().decode(encoding='UTF-8',errors='ignore') - unit = eatraw.unit().decode(encoding='UTF-8',errors='ignore') - print("\nentity: " + str(entity)) - print("unit: " + str(unit) + "\n") - - # obtain extracted data - xt = eatraw.get_time() - yt = eatraw.get_channel() - - # show excerpt of data - print("time (length: " + str(len(xt)) + ") \n" - + str(xt[:10]) + "\n...\n" + str(xt[-10:]) + "\n") - yttrunc = [round(y,4) for y in yt] - print(str(entity) + " (length: " + str(len(yttrunc)) + ") \n" - + str(yttrunc[:10]) + "\n...\n" + str(yttrunc[-10:]) + "\n") - - outname = rf.split('/')[-1].replace('raw','csv') - - print("write output to : " + outname) - eatraw.write_table(("output/"+outname).encode(),ord(' ')) - - else : - - print("\nerror: invalid/corrupt .raw file") - - print("\n") - -#-----------------------------------------------------------------------------# - -print("convert and merge channels " + "\n" + 90*("-") + "\n") - -# setup new instance to merge channels -eatmea = raw_meat.rawmerger(''.encode()) #rawlist[0].encode()) - -# add every single channel/file in list -for rf in rawlist : - print("\nadding channel " + str(rf)) - try : - succ = eatmea.add_channel(rf.encode()) - print("\nrecent time series: length: " + str(len(eatmea.get_time_series())) + "\n") - except RuntimeError as e : - print("failed to add channel: " + str(e)) - -# show summary of successfully merged channels -print("\nmerged channels:\n") - -# write merged table to .csv output -eatmea.write_table_all('output/allchannels.csv'.encode(),ord(',')) - -# get number of successfully merged channels and their names (+units) -numch = eatmea.get_num_channels() -chnames = [chnm.decode(encoding='UTF-8',errors='ignore') for chnm in eatmea.get_channel_names()] -print("number of channels: " + str(numch)) -print("channel names: " + str(chnames)) - -# obtain final time series -timse = eatmea.get_time_series() -print("\nfinal time series:\nlength:" + str(len(timse)) + "\n") - -# get time unit and prepend column name -chnames.insert(0,"Time ["+str(eatmea.time_unit().decode(encoding='UTF-8',errors='ignore'))+"]") - -# prepare list of pyarrow arrays -pyarrs = [] -pyarrs.append(pa.array(timse)) - -for i in range(0,numch) : - print("\n" + str(i) + " " + str(chnames[i])) - dat = eatmea.get_channel_by_index(i) - print("length: " + str(len(dat))) - pyarrs.append(pa.array(dat)) -print("") -# print("\npyarrow arrays\n" + str(pyarrs)) - -# create pyarrow table from data -pyarwtab = pa.Table.from_arrays(pyarrs,chnames) -print("\n" + 60*"-" + "\n" + str(pyarwtab) + "\n") - -# write pyarrow table to .parquet file with compression -pq.write_table(pyarwtab,'output/allchannels.parquet',compression='BROTLI') # compression='BROTLI', 'SNAPPY') - -# try to read and decode the .parquet file -df = pq.read_table('output/allchannels.parquet') -print(df.to_pandas()) -# df.to_pandas().to_csv('allchannels.csv',index=False,encoding='utf-8',sep=",") - -#-----------------------------------------------------------------------------# diff --git a/python/numpy_to_pyarrow.py b/python/numpy_to_pyarrow.py deleted file mode 100644 index 483d4c4..0000000 --- a/python/numpy_to_pyarrow.py +++ /dev/null @@ -1,24 +0,0 @@ - -import pyarrow as pa -import numpy as np -import pyarrow.parquet as pq - -db = pa.array(np.linspace(10,50,6)) -print(db) -da = pa.array(np.linspace(0,5,6)) -print(db) - -filenam = 'pyarrow_testtab.parquet' - -patab = pa.Table.from_arrays([da,db],['entity A [unitA]','entity B [unitB]']) -print(patab) - -# pq.write_table(patab,filenam,compression='BROTLI') -pq.write_table(patab,filenam,compression='SNAPPY') - -df = pq.read_table(filenam) -print(df) -print(df.to_pandas()) - -#import readline -#readline.write_history_file('generate_pyarrow_table_and_write_parquet.py') diff --git a/python/usage_ext.py b/python/usage_ext.py new file mode 100644 index 0000000..cba97f7 --- /dev/null +++ b/python/usage_ext.py @@ -0,0 +1,50 @@ + +import imc_termite +import json +import os +import datetime + +# declare and initialize instance of "imctermite" by passing a raw-file +try : + imcraw = imc_termite.imctermite(b"samples/sampleB.raw") +except RuntimeError as e : + raise Exception("failed to load/parse raw-file: " + str(e)) + +# obtain list of channels as list of dictionaries (without data) +channels = imcraw.get_channels(False) +print(json.dumps(channels,indent=4, sort_keys=False)) + +# obtain all channels (including full data) +channelsdata = imcraw.get_channels(True) + +# everything that follows is an example that specifically makes use only of +# the first (index = 0) channel ... +idx = 0 + +if len(channelsdata) > 0 : + + # get first channel's data + chnydata = channelsdata[idx]['ydata'] + chnxdata = channelsdata[idx]['xdata'] + print("xdata: " + str(len(chnxdata))) + print("ydata: " + str(len(chnydata))) + + # extract trigger-time + trigtim = datetime.datetime.fromisoformat(channels[idx]["trigger-time"]) + print(trigtim) + + # file output of data with absolute timestamp in 1st column + filname = os.path.join("./",channelsdata[idx]['name']+".csv") + print("writing output into " + filname) + with open(filname,'w') as fout : + # include column header + fout.write( str(channelsdata[idx]['xname']) + '[' + str(channelsdata[idx]['xunit']) + "]" + + "," + + str(channelsdata[idx]['yname']) + '[' + str(channelsdata[idx]['yunit']) + "]" + + "\n" ) + # add data (introduce time shift according to trigger-time) + for row in range(0,len(chnxdata)) : + fout.write( str( (trigtim + datetime.timedelta(seconds=chnxdata[row])).isoformat() ) + + "," + + str( chnydata[row]) + + "\n" )