fix bugs in lib/rawmerge.hpp: merge_channels(...) and improve example.py

- raweat.hpp: find_markers(): properly (re)adjust valid_ flag when doing multiple conversion
- rawmerge.hpp: improve logging, merge_channels(): add condition to avoid range error in vector
- example.py: obtain return flag from add_channel(), write csv output from parquet table
- main.cpp: return to pure raw_eater test version

lib/raweat.hpp

@@ -251,7 +251,7 @@ public:
 
     // check validity of format
     // assert ( totalmarksize > 0 && "didn't find any predefined marker => probably not a valid .raw-file" );
-    if ( totalmarksize < 100 ) valid_ = false;
+    valid_ = ( totalmarksize < 100 ) ? false : true;
   }
 
   // display content of found markers

lib/rawmerge.hpp

@@ -79,13 +79,13 @@ public:
   }
 
   // add a single channel and its associated time series
-  bool add_channel(std::string rawfile, bool log = false)
+  bool add_channel(std::string rawfile, bool showlog = false)
   {
     // set raw file and perform conversion
-    this->set_file(rawfile,false);
+    this->set_file(rawfile,showlog);
 
     // show channel name, unit, timestep, time unit, etc.
-    if ( log )
+    if ( showlog && this->get_valid() )
     {
       std::cout<<this->get_name()<<" ["<<this->get_unit()<<"]"<<"\n";
       std::cout<<"Time ["<<this->get_temp_unit()<<"]"<<"\n";
@@ -98,7 +98,7 @@ public:
     // add first/initial time series (and channel data)
     if ( this->get_valid() && timeseries_.size() == 0 && channels_.size() == 0 )
     {
-      std::cout<<"adding initial channel "<<rawfile<<"\n\n";
+      if ( showlog ) std::cout<<"adding initial channel "<<rawfile<<"\n\n";
 
       // insert timeseries and its unit
       this->timeseries_ = this->get_time();
@@ -114,9 +114,9 @@ public:
 
       return true;
     }
-    else
+    else if ( this->get_valid() )
     {
-      std::cout<<"adding next channel "<<rawfile<<"\n\n";
+      if ( showlog ) std::cout<<"adding next channel "<<rawfile<<"\n\n";
 
       // check consistency of temporal unit
       if ( this->get_temp_unit() == this->temp_unit_ )
@@ -163,11 +163,19 @@ public:
       {
         // refuse to merge due to different temporal units
         std::cerr<<"rawmerge: add_channel '"<<rawfile
-                 <<"' : inconsistent time units\n";
+                 <<"' : inconsistent time units: '"
+                 <<this->get_temp_unit()<<"' versus '"<<this->temp_unit_<<"'\n";
 
         return false;
       }
     }
+    else
+    {
+      // provided file does not feature a valid .raw format
+      std::cerr<<"rawmerge: add_channel '"<<rawfile<<"' : invalid .raw file\n";
+
+      return false;
+    }
   }
 
   // merge new channel and associated time series with exisiting channels and
@@ -179,7 +187,7 @@ public:
                       std::vector<double>& result_timeseries,              // resulting timeseries ...
                       std::vector<std::vector<double>>& result_channels,   // ...and associated (n+1) channels
                       double placeholder = std::numeric_limits<double>::quiet_NaN(),
-                      bool showlog = true)
+                      bool showlog = false)
   {
     if ( showlog )
     {
@@ -221,11 +229,19 @@ public:
     // process all time steps in both time series
     while ( idxCur < current_timeseries.size() || idxNew < new_timeseries.size() )
     {
+      if ( showlog ) std::cout<<"idxCur "<<std::setw(6)<<idxCur
+                              <<std::setw(20)<<std::right<<current_timeseries[idxCur]<<"\n"
+                              <<"idxNew "<<std::setw(6)<<idxNew
+                              <<std::setw(20)<<std::right<<new_timeseries[idxNew]<<"\n";
+
       // if point in time of "current_timeseries" is BEFORE time of "new_timeseries"
       // or "new_timeseries" is depleted
-      if ( current_timeseries[idxCur] + 1.0e-10 < new_timeseries[idxNew]
+      if ( idxCur < current_timeseries.size() &&
-        || idxNew == new_timeseries.size() )
+         ( current_timeseries[idxCur] + 1.0e-10 < new_timeseries[idxNew]
+        || idxNew == new_timeseries.size() ) )
       {
+        if ( showlog ) std::cout<<"push_back A\n";
+
         // keep current data as it is ...
         for ( unsigned long int ch = 0; ch < numchannels; ch++ )
         {
@@ -239,9 +255,12 @@ public:
         idxCur++;
       }
       // ...just reversed...
-      else if ( current_timeseries[idxCur] > new_timeseries[idxNew] + 1.0e-10
+      else if ( idxNew < new_timeseries.size() &&
-             || idxCur == current_timeseries.size() )
+              ( current_timeseries[idxCur] > new_timeseries[idxNew] + 1.0e-10
+             || idxCur == current_timeseries.size() ) )
       {
+        if ( showlog ) std::cout<<"push_back B\n";
+
         // insert placeholders for all exisiting channels...
         for ( unsigned long int ch = 0; ch < numchannels; ch++ )
         {
@@ -257,6 +276,8 @@ public:
       // ...points in time of both timeseries match...
       else
       {
+        if ( showlog ) std::cout<<"push_back C\n";
+
         // add ALL, i.e. both current and new data to result
         for ( unsigned long int ch = 0; ch < numchannels; ch++ )
         {

pyt/example.py

@@ -6,17 +6,19 @@ import raw_meat
 import pyarrow as pa
 import pyarrow.parquet as pq
 
-rawlist = [ "smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePedalActiveQF_HS.raw",
+rawlist = [
-            "smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePressure_HS.raw",
+            "smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePedalActiveQF_HS.raw",
-            "smp/Rangerover_Evoque_F-RR534_2019-05-07/EngineSpeed_HS.raw",
+             "smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePressure_HS.raw",
-            "smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_FL.raw",
+             "smp/Rangerover_Evoque_F-RR534_2019-05-07/EngineSpeed_HS.raw",
+             "smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_FL.raw",
             "smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_RL.raw",
             "smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_Vacuum.raw",
             "smp/VehicleSpeed_HS.raw",
             "smp/Rangerover_Evoque_F-RR534_2019-05-07/ABS_A_Port1.raw",
             "./pyt/example.py",
             "smp/Rangerover_Evoque_F-RR534_2019-05-07/LateralAcceleration_HS.raw",
-            "smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw" ]
+            "smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw"
+          ]
 
 print("")
 
@@ -35,7 +37,7 @@ for rf in rawlist :
     if eatraw.validity() :
 
         # show channel name and its unit
-        entity = eatraw.channel_name().decode()
+        entity = eatraw.channel_name().decode(encoding='UTF-8',errors='ignore')
         unit = eatraw.unit().decode(encoding='UTF-8',errors='ignore')
         print("\nentity: " + str(entity))
         print("unit:   " + str(unit) + "\n")
@@ -64,25 +66,35 @@ for rf in rawlist :
 
 #-----------------------------------------------------------------------------#
 
+print("convert and merge channels " + "\n" + 90*("-") + "\n")
+
 # setup new instance to merge channels
 eatmea = raw_meat.rawmerger(rawlist[0].encode())
 
+# add every single channel/file in list
 for rf in rawlist :
     print("\nadding channel " + str(rf))
-    eatmea.add_channel(rf.encode())
+    succ = eatmea.add_channel(rf.encode())
+    if succ :
+        print("\nrecent time series: length: " + str(len(eatmea.get_time_series())) + "\n")
+    else :
+        print("\nfailed to add channel\n")
 
+# show summary of successfully merged channels
 print("\nmerged channels:\n")
-print("number of channels: " + str(eatmea.get_num_channels()))
-print("channel names: " + str(eatmea.get_channel_names()))
 
+# get number of successfully merged channels and their names (+units)
 numch = eatmea.get_num_channels()
-chnames = eatmea.get_channel_names()
+chnames = [chnm.decode(encoding='UTF-8',errors='ignore') for chnm in eatmea.get_channel_names()]
+print("number of channels: " + str(numch))
+print("channel names: " + str(chnames))
 
+# obtain final time series
 timse = eatmea.get_time_series()
 print("\nfinal time series:\nlength:" + str(len(timse)) + "\n")
 
 # get time unit and prepend column name
-chnames.insert(0,"Time ["+str(eatmea.time_unit())+"]")
+chnames.insert(0,"Time ["+str(eatmea.time_unit().decode(encoding='UTF-8',errors='ignore'))+"]")
 
 # prepare list of pyarrow arrays
 pyarrs = []
@@ -93,17 +105,19 @@ for i in range(0,numch) :
     dat = eatmea.get_channel_by_index(i)
     print("length: " + str(len(dat)))
     pyarrs.append(pa.array(dat))
-
+print("")
 # print("\npyarrow arrays\n" + str(pyarrs))
 
-# prepare pyarrow table from data
+# create pyarrow table from data
 pyarwtab = pa.Table.from_arrays(pyarrs,chnames)
-print(pyarwtab)
+print("\n" + 60*"-" + "\n" + str(pyarwtab) + "\n")
 
+# write pyarrow table to .parquet file with compression
 pq.write_table(pyarwtab,'allchannels.parquet',compression='BROTLI')  # compression='BROTLI', 'SNAPPY')
 
+# try to read and decode the .parquet file
 df = pq.read_table('allchannels.parquet')
-print(df)
 print(df.to_pandas())
+df.to_pandas().to_csv('allchannels.csv',index=False,encoding='utf-8',sep=",")
 
 #-----------------------------------------------------------------------------#

src/main.cpp

@@ -30,14 +30,14 @@ int main(int argc, char* argv[])
   std::string rawfile(argv[1]);
 
   // declare instance of "raw_eater"
-  // raw_eater eatraw(rawfile,true);
+  raw_eater eatraw(rawfile,true);
 
   // declare instance of "raw_merger"
-  raw_merger eatmea;
+//  raw_merger eatmea;
-  eatmea.add_channel(rawfile,true);
+//  eatmea.add_channel(rawfile,true);
-  eatmea.add_channel("smp/VehicleSpeed_HS.raw",true);
+//  eatmea.add_channel("smp/VehicleSpeed_HS.raw",true);
-  eatmea.add_channel("smp/VehicleSpeed_HS.raw",true);
+//  eatmea.add_channel("smp/VehicleSpeed_HS.raw",true);
-  eatmea.add_channel("smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw",true);
+//  eatmea.add_channel("smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw",true);
 
   //eatraw.show_markers();
 
@@ -65,10 +65,13 @@ int main(int argc, char* argv[])
 //  for ( unsigned long int i = 0; i < 10; i++ ) std::cout<<mydata[i]<<"\n";
 
   // write data in csv-file
-  // eatraw.write_table(std::string(argv[2]));
+  if ( eatraw.get_valid() )
-  // eatraw.write_table(std::string(argv[2]),' ');
+  {
-  // eatmea.write_table(std::string(argv[2]));
+    // eatraw.write_table(std::string(argv[2]));
-  eatmea.write_table(std::string(argv[2]),' ');
+    eatraw.write_table(std::string(argv[2]),' ');
+    // eatmea.write_table(std::string(argv[2]));
+    // eatmea.write_table(std::string(argv[2]),' ');
+  }
 
   return 0;
 }