- cython: raw_meat: add/adjust getter method

- implement full app example in pyt/example.py
	- merges time series of all consistent channels
	- extract them into pyarrow table
	- save pyarrow table as compressed parquet file

cyt/raw_meat.pxd

@@ -22,6 +22,14 @@ cdef extern from "../lib/rawmerge.hpp":
         vector[double] get_time()
         vector[double] get_data()
         # dump all data to .csv
-        void write_table(const char*,char delimiter)
+        void write_table(const char*,char)
         # add channel and try to merge it
         bool add_channel(string)
+        # get total number of (added) channels
+        int get_num_channels()
+        # get list of channel names
+        vector[string] get_channel_names()
+        # get data of particular channel
+        vector[double] get_channel(int)
+        # get total merged time series
+        vector[double] get_time_series()

cyt/raw_meat.pyx

@@ -41,3 +41,15 @@ cdef class rawmerger:
 
     def add_channel(self, string rawfile):
         return self.rawit.add_channel(rawfile)
+
+    def get_num_channels(self):
+        return self.rawit.get_num_channels()
+
+    def get_channel_names(self):
+        return self.rawit.get_channel_names()
+
+    def get_channel_by_index(self, int chidx):
+        return self.rawit.get_channel(chidx)
+
+    def get_time_series(self):
+        return self.rawit.get_time_series()

lib/rawmerge.hpp

@@ -39,7 +39,7 @@ public:
   raw_merger(std::string rawfile): raw_eater(rawfile)
   {
     // add first initial channel
-    this->add_channel(rawfile);
+    //this->add_channel(rawfile);
   }
 
   raw_merger(): raw_eater()

makefile

@@ -63,6 +63,7 @@ build : setup_raw_eater.py cyt/raw_eater.pyx cyt/raw_eater.pxd $(LIB)raweat.hpp
 	python3 setup_raw_meat.py build_ext --inplace
 	cp raw_eater.cpython-*.so pyt/
 	cp raw_meat.cpython-*.so pyt/
+	rm *.so
 
 py_install: setup_raw_eater.py cyt/raw_eater.pyx cyt/raw_eater.pxd $(LIB)raweat.hpp \
             setup_raw_meat.py cyt/raw_meat.pyx cyt/raw_meat.pxd $(LIB)rawmerge.hpp

pyt/example.py

@@ -1,29 +1,42 @@
 
+#-----------------------------------------------------------------------------#
+
 import raw_eater
 import raw_meat
+import pyarrow as pa
+import pyarrow.parquet as pq
 
-rawlist = [ "smp/VehicleSpeed_HS.raw",
+rawlist = [ "smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePedalActiveQF_HS.raw",
+            "smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePressure_HS.raw",
+            "smp/Rangerover_Evoque_F-RR534_2019-05-07/EngineSpeed_HS.raw",
+            "smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_FL.raw",
+            "smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_RL.raw",
+            "smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_Vacuum.raw",
+            "smp/VehicleSpeed_HS.raw",
             "smp/Rangerover_Evoque_F-RR534_2019-05-07/ABS_A_Port1.raw",
             "./pyt/example.py",
-            "smp/Rangerover_Evoque_F-RR534_2019-05-07/LateralAcceleration_HS.raw" ]
+            "smp/Rangerover_Evoque_F-RR534_2019-05-07/LateralAcceleration_HS.raw",
+            "smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw" ]
 
 print("")
 
+#-----------------------------------------------------------------------------#
+
 # convert every single listed file
 for rf in rawlist :
 
     print("converting " + str(rf) + "...\n" + 90*("-") + "\n")
 
     # setup instance of "raw_eater" and trigger conversion
-    # eatraw = raw_eater.raweater(rf.encode())
+    eatraw = raw_eater.raweater(rf.encode())
-    eatraw = raw_meat.rawmerger(rf.encode())
+    # eatraw = raw_meat.rawmerger(rf.encode())
 
     # check validity of file format
     if eatraw.validity() :
 
         # show channel name and its unit
         entity = eatraw.channel_name().decode()
-        unit = eatraw.unit().decode()
+        unit = eatraw.unit().decode(encoding='UTF-8',errors='ignore')
         print("\nentity: " + str(entity))
         print("unit:   " + str(unit) + "\n")
 
@@ -40,10 +53,57 @@ for rf in rawlist :
 
         outname = rf.split('/')[-1].replace('raw','csv')
 
-        eatraw.write_table(outname.encode(),ord(' '))
+        print("write output to : " + outname)
+        eatraw.write_table((outname).encode(),ord(' '))
 
     else :
 
         print("\nerror: invalid/corrupt .raw file")
 
     print("\n")
+
+#-----------------------------------------------------------------------------#
+
+# setup new instance to merge channels
+eatmea = raw_meat.rawmerger(rawlist[0].encode())
+
+for rf in rawlist :
+    print("\nadding channel " + str(rf))
+    eatmea.add_channel(rf.encode())
+
+print("\nmerged channels:\n")
+print("number of channels: " + str(eatmea.get_num_channels()))
+print("channel names: " + str(eatmea.get_channel_names()))
+
+numch = eatmea.get_num_channels()
+chnames = eatmea.get_channel_names()
+
+timse = eatmea.get_time_series()
+print("\nfinal time series:\nlength:" + str(len(timse)) + "\n")
+
+# get time unit and prepend column name
+chnames.insert(0,"Time ["+str(eatmea.time_unit())+"]")
+
+# prepare list of pyarrow arrays
+pyarrs = []
+pyarrs.append(pa.array(timse))
+
+for i in range(0,numch) :
+    print("\n" + str(i) + "  " + str(chnames[i]))
+    dat = eatmea.get_channel_by_index(i)
+    print("length: " + str(len(dat)))
+    pyarrs.append(pa.array(dat))
+
+# print("\npyarrow arrays\n" + str(pyarrs))
+
+# prepare pyarrow table from data
+pyarwtab = pa.Table.from_arrays(pyarrs,chnames)
+print(pyarwtab)
+
+pq.write_table(pyarwtab,'allchannels.parquet',compression='BROTLI')  # compression='BROTLI', 'SNAPPY')
+
+df = pq.read_table('allchannels.parquet')
+print(df)
+print(df.to_pandas())
+
+#-----------------------------------------------------------------------------#