diff --git a/cyt/raw_meat.pxd b/cyt/raw_meat.pxd index 2d97185..deed773 100644 --- a/cyt/raw_meat.pxd +++ b/cyt/raw_meat.pxd @@ -22,6 +22,14 @@ cdef extern from "../lib/rawmerge.hpp": vector[double] get_time() vector[double] get_data() # dump all data to .csv - void write_table(const char*,char delimiter) + void write_table(const char*,char) # add channel and try to merge it bool add_channel(string) + # get total number of (added) channels + int get_num_channels() + # get list of channel names + vector[string] get_channel_names() + # get data of particular channel + vector[double] get_channel(int) + # get total merged time series + vector[double] get_time_series() diff --git a/cyt/raw_meat.pyx b/cyt/raw_meat.pyx index 32499ce..c1b140b 100644 --- a/cyt/raw_meat.pyx +++ b/cyt/raw_meat.pyx @@ -41,3 +41,15 @@ cdef class rawmerger: def add_channel(self, string rawfile): return self.rawit.add_channel(rawfile) + + def get_num_channels(self): + return self.rawit.get_num_channels() + + def get_channel_names(self): + return self.rawit.get_channel_names() + + def get_channel_by_index(self, int chidx): + return self.rawit.get_channel(chidx) + + def get_time_series(self): + return self.rawit.get_time_series() diff --git a/lib/rawmerge.hpp b/lib/rawmerge.hpp index 900470a..6be56c1 100644 --- a/lib/rawmerge.hpp +++ b/lib/rawmerge.hpp @@ -39,7 +39,7 @@ public: raw_merger(std::string rawfile): raw_eater(rawfile) { // add first initial channel - this->add_channel(rawfile); + //this->add_channel(rawfile); } raw_merger(): raw_eater() diff --git a/makefile b/makefile index 08a91ad..b525589 100644 --- a/makefile +++ b/makefile @@ -63,6 +63,7 @@ build : setup_raw_eater.py cyt/raw_eater.pyx cyt/raw_eater.pxd $(LIB)raweat.hpp python3 setup_raw_meat.py build_ext --inplace cp raw_eater.cpython-*.so pyt/ cp raw_meat.cpython-*.so pyt/ + rm *.so py_install: setup_raw_eater.py cyt/raw_eater.pyx cyt/raw_eater.pxd $(LIB)raweat.hpp \ setup_raw_meat.py cyt/raw_meat.pyx cyt/raw_meat.pxd $(LIB)rawmerge.hpp diff --git a/pyt/example.py b/pyt/example.py index d3cd827..3e0300d 100644 --- a/pyt/example.py +++ b/pyt/example.py @@ -1,29 +1,42 @@ +#-----------------------------------------------------------------------------# + import raw_eater import raw_meat +import pyarrow as pa +import pyarrow.parquet as pq -rawlist = [ "smp/VehicleSpeed_HS.raw", +rawlist = [ "smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePedalActiveQF_HS.raw", + "smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePressure_HS.raw", + "smp/Rangerover_Evoque_F-RR534_2019-05-07/EngineSpeed_HS.raw", + "smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_FL.raw", + "smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_RL.raw", + "smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_Vacuum.raw", + "smp/VehicleSpeed_HS.raw", "smp/Rangerover_Evoque_F-RR534_2019-05-07/ABS_A_Port1.raw", "./pyt/example.py", - "smp/Rangerover_Evoque_F-RR534_2019-05-07/LateralAcceleration_HS.raw" ] + "smp/Rangerover_Evoque_F-RR534_2019-05-07/LateralAcceleration_HS.raw", + "smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw" ] print("") +#-----------------------------------------------------------------------------# + # convert every single listed file for rf in rawlist : print("converting " + str(rf) + "...\n" + 90*("-") + "\n") # setup instance of "raw_eater" and trigger conversion - # eatraw = raw_eater.raweater(rf.encode()) - eatraw = raw_meat.rawmerger(rf.encode()) + eatraw = raw_eater.raweater(rf.encode()) + # eatraw = raw_meat.rawmerger(rf.encode()) # check validity of file format if eatraw.validity() : # show channel name and its unit entity = eatraw.channel_name().decode() - unit = eatraw.unit().decode() + unit = eatraw.unit().decode(encoding='UTF-8',errors='ignore') print("\nentity: " + str(entity)) print("unit: " + str(unit) + "\n") @@ -40,10 +53,57 @@ for rf in rawlist : outname = rf.split('/')[-1].replace('raw','csv') - eatraw.write_table(outname.encode(),ord(' ')) + print("write output to : " + outname) + eatraw.write_table((outname).encode(),ord(' ')) else : print("\nerror: invalid/corrupt .raw file") print("\n") + +#-----------------------------------------------------------------------------# + +# setup new instance to merge channels +eatmea = raw_meat.rawmerger(rawlist[0].encode()) + +for rf in rawlist : + print("\nadding channel " + str(rf)) + eatmea.add_channel(rf.encode()) + +print("\nmerged channels:\n") +print("number of channels: " + str(eatmea.get_num_channels())) +print("channel names: " + str(eatmea.get_channel_names())) + +numch = eatmea.get_num_channels() +chnames = eatmea.get_channel_names() + +timse = eatmea.get_time_series() +print("\nfinal time series:\nlength:" + str(len(timse)) + "\n") + +# get time unit and prepend column name +chnames.insert(0,"Time ["+str(eatmea.time_unit())+"]") + +# prepare list of pyarrow arrays +pyarrs = [] +pyarrs.append(pa.array(timse)) + +for i in range(0,numch) : + print("\n" + str(i) + " " + str(chnames[i])) + dat = eatmea.get_channel_by_index(i) + print("length: " + str(len(dat))) + pyarrs.append(pa.array(dat)) + +# print("\npyarrow arrays\n" + str(pyarrs)) + +# prepare pyarrow table from data +pyarwtab = pa.Table.from_arrays(pyarrs,chnames) +print(pyarwtab) + +pq.write_table(pyarwtab,'allchannels.parquet',compression='BROTLI') # compression='BROTLI', 'SNAPPY') + +df = pq.read_table('allchannels.parquet') +print(df) +print(df.to_pandas()) + +#-----------------------------------------------------------------------------#