diff --git a/lib/raweat.hpp b/lib/raweat.hpp index 5cfea1a..dec0289 100644 --- a/lib/raweat.hpp +++ b/lib/raweat.hpp @@ -251,7 +251,7 @@ public: // check validity of format // assert ( totalmarksize > 0 && "didn't find any predefined marker => probably not a valid .raw-file" ); - if ( totalmarksize < 100 ) valid_ = false; + valid_ = ( totalmarksize < 100 ) ? false : true; } // display content of found markers diff --git a/lib/rawmerge.hpp b/lib/rawmerge.hpp index 6be56c1..d802372 100644 --- a/lib/rawmerge.hpp +++ b/lib/rawmerge.hpp @@ -79,13 +79,13 @@ public: } // add a single channel and its associated time series - bool add_channel(std::string rawfile, bool log = false) + bool add_channel(std::string rawfile, bool showlog = false) { // set raw file and perform conversion - this->set_file(rawfile,false); + this->set_file(rawfile,showlog); // show channel name, unit, timestep, time unit, etc. - if ( log ) + if ( showlog && this->get_valid() ) { std::cout<get_name()<<" ["<get_unit()<<"]"<<"\n"; std::cout<<"Time ["<get_temp_unit()<<"]"<<"\n"; @@ -98,7 +98,7 @@ public: // add first/initial time series (and channel data) if ( this->get_valid() && timeseries_.size() == 0 && channels_.size() == 0 ) { - std::cout<<"adding initial channel "<timeseries_ = this->get_time(); @@ -114,9 +114,9 @@ public: return true; } - else + else if ( this->get_valid() ) { - std::cout<<"adding next channel "<get_temp_unit() == this->temp_unit_ ) @@ -163,11 +163,19 @@ public: { // refuse to merge due to different temporal units std::cerr<<"rawmerge: add_channel '"<get_temp_unit()<<"' versus '"<temp_unit_<<"'\n"; return false; } } + else + { + // provided file does not feature a valid .raw format + std::cerr<<"rawmerge: add_channel '"<& result_timeseries, // resulting timeseries ... std::vector>& result_channels, // ...and associated (n+1) channels double placeholder = std::numeric_limits::quiet_NaN(), - bool showlog = true) + bool showlog = false) { if ( showlog ) { @@ -221,11 +229,19 @@ public: // process all time steps in both time series while ( idxCur < current_timeseries.size() || idxNew < new_timeseries.size() ) { + if ( showlog ) std::cout<<"idxCur "< new_timeseries[idxNew] + 1.0e-10 - || idxCur == current_timeseries.size() ) + else if ( idxNew < new_timeseries.size() && + ( current_timeseries[idxCur] > new_timeseries[idxNew] + 1.0e-10 + || idxCur == current_timeseries.size() ) ) { + if ( showlog ) std::cout<<"push_back B\n"; + // insert placeholders for all exisiting channels... for ( unsigned long int ch = 0; ch < numchannels; ch++ ) { @@ -257,6 +276,8 @@ public: // ...points in time of both timeseries match... else { + if ( showlog ) std::cout<<"push_back C\n"; + // add ALL, i.e. both current and new data to result for ( unsigned long int ch = 0; ch < numchannels; ch++ ) { diff --git a/pyt/example.py b/pyt/example.py index 3e0300d..6c5b82d 100644 --- a/pyt/example.py +++ b/pyt/example.py @@ -6,17 +6,19 @@ import raw_meat import pyarrow as pa import pyarrow.parquet as pq -rawlist = [ "smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePedalActiveQF_HS.raw", - "smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePressure_HS.raw", - "smp/Rangerover_Evoque_F-RR534_2019-05-07/EngineSpeed_HS.raw", - "smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_FL.raw", +rawlist = [ + "smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePedalActiveQF_HS.raw", + "smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePressure_HS.raw", + "smp/Rangerover_Evoque_F-RR534_2019-05-07/EngineSpeed_HS.raw", + "smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_FL.raw", "smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_RL.raw", "smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_Vacuum.raw", "smp/VehicleSpeed_HS.raw", "smp/Rangerover_Evoque_F-RR534_2019-05-07/ABS_A_Port1.raw", "./pyt/example.py", "smp/Rangerover_Evoque_F-RR534_2019-05-07/LateralAcceleration_HS.raw", - "smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw" ] + "smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw" + ] print("") @@ -35,7 +37,7 @@ for rf in rawlist : if eatraw.validity() : # show channel name and its unit - entity = eatraw.channel_name().decode() + entity = eatraw.channel_name().decode(encoding='UTF-8',errors='ignore') unit = eatraw.unit().decode(encoding='UTF-8',errors='ignore') print("\nentity: " + str(entity)) print("unit: " + str(unit) + "\n") @@ -64,25 +66,35 @@ for rf in rawlist : #-----------------------------------------------------------------------------# +print("convert and merge channels " + "\n" + 90*("-") + "\n") + # setup new instance to merge channels eatmea = raw_meat.rawmerger(rawlist[0].encode()) +# add every single channel/file in list for rf in rawlist : print("\nadding channel " + str(rf)) - eatmea.add_channel(rf.encode()) + succ = eatmea.add_channel(rf.encode()) + if succ : + print("\nrecent time series: length: " + str(len(eatmea.get_time_series())) + "\n") + else : + print("\nfailed to add channel\n") +# show summary of successfully merged channels print("\nmerged channels:\n") -print("number of channels: " + str(eatmea.get_num_channels())) -print("channel names: " + str(eatmea.get_channel_names())) +# get number of successfully merged channels and their names (+units) numch = eatmea.get_num_channels() -chnames = eatmea.get_channel_names() +chnames = [chnm.decode(encoding='UTF-8',errors='ignore') for chnm in eatmea.get_channel_names()] +print("number of channels: " + str(numch)) +print("channel names: " + str(chnames)) +# obtain final time series timse = eatmea.get_time_series() print("\nfinal time series:\nlength:" + str(len(timse)) + "\n") # get time unit and prepend column name -chnames.insert(0,"Time ["+str(eatmea.time_unit())+"]") +chnames.insert(0,"Time ["+str(eatmea.time_unit().decode(encoding='UTF-8',errors='ignore'))+"]") # prepare list of pyarrow arrays pyarrs = [] @@ -93,17 +105,19 @@ for i in range(0,numch) : dat = eatmea.get_channel_by_index(i) print("length: " + str(len(dat))) pyarrs.append(pa.array(dat)) - +print("") # print("\npyarrow arrays\n" + str(pyarrs)) -# prepare pyarrow table from data +# create pyarrow table from data pyarwtab = pa.Table.from_arrays(pyarrs,chnames) -print(pyarwtab) +print("\n" + 60*"-" + "\n" + str(pyarwtab) + "\n") +# write pyarrow table to .parquet file with compression pq.write_table(pyarwtab,'allchannels.parquet',compression='BROTLI') # compression='BROTLI', 'SNAPPY') +# try to read and decode the .parquet file df = pq.read_table('allchannels.parquet') -print(df) print(df.to_pandas()) +df.to_pandas().to_csv('allchannels.csv',index=False,encoding='utf-8',sep=",") #-----------------------------------------------------------------------------# diff --git a/src/main.cpp b/src/main.cpp index f7aa51b..e76826d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -30,14 +30,14 @@ int main(int argc, char* argv[]) std::string rawfile(argv[1]); // declare instance of "raw_eater" - // raw_eater eatraw(rawfile,true); + raw_eater eatraw(rawfile,true); // declare instance of "raw_merger" - raw_merger eatmea; - eatmea.add_channel(rawfile,true); - eatmea.add_channel("smp/VehicleSpeed_HS.raw",true); - eatmea.add_channel("smp/VehicleSpeed_HS.raw",true); - eatmea.add_channel("smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw",true); +// raw_merger eatmea; +// eatmea.add_channel(rawfile,true); +// eatmea.add_channel("smp/VehicleSpeed_HS.raw",true); +// eatmea.add_channel("smp/VehicleSpeed_HS.raw",true); +// eatmea.add_channel("smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw",true); //eatraw.show_markers(); @@ -65,10 +65,13 @@ int main(int argc, char* argv[]) // for ( unsigned long int i = 0; i < 10; i++ ) std::cout<