lib/imc_channel.hpp

@@ -402,7 +402,7 @@ namespace imc
     std::string get_info(int width = 20)
     {
       // prepare printable trigger-time
-      std::time_t tt = std::chrono::system_clock::to_time_t(trigger_time_);
+      //std::time_t tt = std::chrono::system_clock::to_time_t(trigger_time_);
       std::time_t att = std::chrono::system_clock::to_time_t(absolute_trigger_time_);
 
       std::stringstream ss;
@@ -411,7 +411,7 @@ namespace imc
         <<std::setw(width)<<std::left<<"comment:"<<comment_<<"\n"
         <<std::setw(width)<<std::left<<"origin:"<<origin_<<"\n"
         <<std::setw(width)<<std::left<<"description:"<<text_<<"\n"
-        <<std::setw(width)<<std::left<<"trigger-time-nt:"<<std::put_time(std::localtime(&tt),"%FT%T")<<"\n"
+        //<<std::setw(width)<<std::left<<"trigger-time:"<<std::put_time(std::localtime(&tt),"%FT%T")<<"\n"
         <<std::setw(width)<<std::left<<"trigger-time:"<<std::put_time(std::localtime(&att),"%FT%T")<<"\n"
         <<std::setw(width)<<std::left<<"language-code:"<<language_code_<<"\n"
         <<std::setw(width)<<std::left<<"codepage:"<<codepage_<<"\n"
@@ -421,7 +421,7 @@ namespace imc
         <<std::setw(width)<<std::left<<"significant bits:"<<signbits_<<"\n"
         <<std::setw(width)<<std::left<<"buffer-offset:"<<buffer_offset_<<"\n"
         <<std::setw(width)<<std::left<<"buffer-size:"<<buffer_size_<<"\n"
-        <<std::setw(width)<<std::left<<"add-time:"<<addtime_<<"\n"
+        //<<std::setw(width)<<std::left<<"add-time:"<<addtime_<<"\n"
         <<std::setw(width)<<std::left<<"xname:"<<xname_<<"\n"
         <<std::setw(width)<<std::left<<"xunit:"<<xunit_<<"\n"
         <<std::setw(width)<<std::left<<"xstepwidth:"<<xstepwidth_<<"\n"
@@ -440,7 +440,7 @@ namespace imc
     std::string get_json(bool include_data = false)
     {
       // prepare printable trigger-time
-      std::time_t tt = std::chrono::system_clock::to_time_t(trigger_time_);
+      //std::time_t tt = std::chrono::system_clock::to_time_t(trigger_time_);
       std::time_t att = std::chrono::system_clock::to_time_t(absolute_trigger_time_);
 
       std::stringstream ss;
@@ -449,14 +449,12 @@ namespace imc
              <<"\",\"comment\":\""<<comment_
              <<"\",\"origin\":\""<<origin_
              <<"\",\"description\":\""<<text_
-             <<"\",\"trigger-time-nt\":\""<<std::put_time(std::localtime(&tt),"%FT%T")
              <<"\",\"trigger-time\":\""<<std::put_time(std::localtime(&att),"%FT%T")
              <<"\",\"language-code\":\""<<language_code_
              <<"\",\"codepage\":\""<<codepage_
              <<"\",\"yname\":\""<<yname_
              <<"\",\"yunit\":\""<<yunit_
              <<"\",\"significantbits\":\""<<signbits_
-             <<"\",\"addtime\":\""<<addtime_
              <<"\",\"xname\":\""<<xname_
              <<"\",\"xunit\":\""<<xunit_
              <<"\",\"xstepwidth\":\""<<xstepwidth_

python/example.py

@@ -0,0 +1,137 @@
+
+#-----------------------------------------------------------------------------#
+
+import raw_eater
+import raw_meat
+import pyarrow as pa
+import pyarrow.parquet as pq
+from pathlib import Path
+
+fileobj1 = Path("samples/datasetA/").rglob("*.raw")
+rawlist1 = [str(fl) for fl in fileobj1]
+
+fileobj2 = Path("samples/datasetB/").rglob("*.raw")
+rawlist2 = [str(fl) for fl in fileobj2]
+
+rawlist = rawlist1 #[rawlist1[0],rawlist1[4],rawlist2[0],rawlist2[6]]
+for fil in rawlist2 :
+    rawlist.append(fil)
+rawlist.append("./README.md")
+
+print("")
+print(rawlist)
+print()
+
+#-----------------------------------------------------------------------------#
+
+# alternatively create "empty" instance of "raw_eater" and set file names
+eatraw = raw_eater.raweater()
+# eatraw.set_file("../smp/pressure_Vacuum.raw".encode())
+
+# convert every single listed file
+for rf in rawlist :
+
+    print("converting " + str(rf) + "...\n" + 90*("-") + "\n")
+
+    # setup instance of "raw_eater" and trigger conversion
+    # eatraw = raw_eater.raweater(rf.encode())
+    # eatraw = raw_meat.rawmerger(rf.encode())
+
+    # use global instance of "raw_eater" to set file and perform decoding
+    eatraw.set_file(rf.encode())
+    try :
+        eatraw.do_conversion()
+    except RuntimeError as e :
+        print("conversion failed: " + str(e))
+
+    # check validity of file format
+    if eatraw.validity() :
+
+        # show channel name and its unit
+        entity = eatraw.channel_name().decode(encoding='UTF-8',errors='ignore')
+        unit = eatraw.unit().decode(encoding='UTF-8',errors='ignore')
+        print("\nentity: " + str(entity))
+        print("unit:   " + str(unit) + "\n")
+
+        # obtain extracted data
+        xt = eatraw.get_time()
+        yt = eatraw.get_channel()
+
+        # show excerpt of data
+        print("time (length: " + str(len(xt)) + ") \n"
+             + str(xt[:10]) + "\n...\n" + str(xt[-10:]) + "\n")
+        yttrunc = [round(y,4) for y in yt]
+        print(str(entity) + " (length: " + str(len(yttrunc)) + ") \n"
+            + str(yttrunc[:10]) + "\n...\n" + str(yttrunc[-10:]) + "\n")
+
+        outname = rf.split('/')[-1].replace('raw','csv')
+
+        print("write output to : " + outname)
+        eatraw.write_table(("output/"+outname).encode(),ord(' '))
+
+    else :
+
+        print("\nerror: invalid/corrupt .raw file")
+
+    print("\n")
+
+#-----------------------------------------------------------------------------#
+
+print("convert and merge channels " + "\n" + 90*("-") + "\n")
+
+# setup new instance to merge channels
+eatmea = raw_meat.rawmerger(''.encode()) #rawlist[0].encode())
+
+# add every single channel/file in list
+for rf in rawlist :
+    print("\nadding channel " + str(rf))
+    try :
+        succ = eatmea.add_channel(rf.encode())
+        print("\nrecent time series: length: " + str(len(eatmea.get_time_series())) + "\n")
+    except RuntimeError as e :
+        print("failed to add channel: " + str(e))
+
+# show summary of successfully merged channels
+print("\nmerged channels:\n")
+
+# write merged table to .csv output
+eatmea.write_table_all('output/allchannels.csv'.encode(),ord(','))
+
+# get number of successfully merged channels and their names (+units)
+numch = eatmea.get_num_channels()
+chnames = [chnm.decode(encoding='UTF-8',errors='ignore') for chnm in eatmea.get_channel_names()]
+print("number of channels: " + str(numch))
+print("channel names: " + str(chnames))
+
+# obtain final time series
+timse = eatmea.get_time_series()
+print("\nfinal time series:\nlength:" + str(len(timse)) + "\n")
+
+# get time unit and prepend column name
+chnames.insert(0,"Time ["+str(eatmea.time_unit().decode(encoding='UTF-8',errors='ignore'))+"]")
+
+# prepare list of pyarrow arrays
+pyarrs = []
+pyarrs.append(pa.array(timse))
+
+for i in range(0,numch) :
+    print("\n" + str(i) + "  " + str(chnames[i]))
+    dat = eatmea.get_channel_by_index(i)
+    print("length: " + str(len(dat)))
+    pyarrs.append(pa.array(dat))
+print("")
+# print("\npyarrow arrays\n" + str(pyarrs))
+
+# create pyarrow table from data
+pyarwtab = pa.Table.from_arrays(pyarrs,chnames)
+print("\n" + 60*"-" + "\n" + str(pyarwtab) + "\n")
+
+# write pyarrow table to .parquet file with compression
+pq.write_table(pyarwtab,'output/allchannels.parquet',compression='BROTLI')  # compression='BROTLI', 'SNAPPY')
+
+# try to read and decode the .parquet file
+df = pq.read_table('output/allchannels.parquet')
+print(df.to_pandas())
+# df.to_pandas().to_csv('allchannels.csv',index=False,encoding='utf-8',sep=",")
+
+#-----------------------------------------------------------------------------#

python/numpy_to_pyarrow.py

@@ -0,0 +1,24 @@
+
+import pyarrow as pa
+import numpy as np
+import pyarrow.parquet as pq
+
+db = pa.array(np.linspace(10,50,6))
+print(db)
+da = pa.array(np.linspace(0,5,6))
+print(db)
+
+filenam = 'pyarrow_testtab.parquet'
+
+patab = pa.Table.from_arrays([da,db],['entity A [unitA]','entity B [unitB]'])
+print(patab)
+
+# pq.write_table(patab,filenam,compression='BROTLI')
+pq.write_table(patab,filenam,compression='SNAPPY')
+
+df = pq.read_table(filenam)
+print(df)
+print(df.to_pandas())
+
+#import readline
+#readline.write_history_file('generate_pyarrow_table_and_write_parquet.py')

python/usage_ext.py

@@ -1,50 +0,0 @@
-
-import imc_termite
-import json
-import os
-import datetime
-
-# declare and initialize instance of "imctermite" by passing a raw-file
-try :
-    imcraw = imc_termite.imctermite(b"samples/sampleB.raw")
-except RuntimeError as e :
-    raise Exception("failed to load/parse raw-file: " + str(e))
-
-# obtain list of channels as list of dictionaries (without data)
-channels = imcraw.get_channels(False)
-print(json.dumps(channels,indent=4, sort_keys=False))
-
-# obtain all channels (including full data)
-channelsdata = imcraw.get_channels(True)
-
-# everything that follows is an example that specifically makes use only of
-# the first (index = 0) channel ...
-idx = 0
-
-if len(channelsdata) > 0 :
-
-    # get first channel's data
-    chnydata = channelsdata[idx]['ydata']
-    chnxdata = channelsdata[idx]['xdata']
-    print("xdata: " + str(len(chnxdata)))
-    print("ydata: " + str(len(chnydata)))
-
-    # extract trigger-time
-    trigtim = datetime.datetime.fromisoformat(channels[idx]["trigger-time"])
-    print(trigtim)
-
-    # file output of data with absolute timestamp in 1st column
-    filname = os.path.join("./",channelsdata[idx]['name']+".csv")
-    print("writing output into " + filname)
-    with open(filname,'w') as fout :
-        # include column header
-        fout.write( str(channelsdata[idx]['xname']) + '[' + str(channelsdata[idx]['xunit']) + "]"
-                  + ","
-                  + str(channelsdata[idx]['yname']) + '[' + str(channelsdata[idx]['yunit']) + "]"
-                  + "\n" )
-        # add data (introduce time shift according to trigger-time)
-        for row in range(0,len(chnxdata)) :
-            fout.write( str( (trigtim + datetime.timedelta(seconds=chnxdata[row])).isoformat() )
-                      + ","
-                      + str( chnydata[row])
-                      + "\n" )