* fix channel dependent buffer offset, issue #15

* add python example multichannel.py

README.md

@@ -31,6 +31,8 @@ On the [Record Evolution Platform](https://www.record-evolution.de/en/home-en/),
 
 ## File format
 
+[Warning: Take a look at [this issue](https://github.com/RecordEvolution/IMCtermite/issues/14) when reading this section regarding the file format.]
+
 A data file of the _IMC Bus Format_ type with the extension _.raw_ is a _mixed text/binary
 file_ featuring a set of markers (keys) that indicate the start of various blocks
 of data that provide meta information and the actual measurement data. Every single
@@ -152,8 +154,9 @@ python3 -m pip install IMCtermite
 
 which provides binary wheels for multiple architectures on _Windows_ and _Linux_
 and most _Python 3.x_ distributions. However, if your platform/architecture is
-not supported you can still compile the source distribution yourself, which 
+not supported you can still compile the source distribution yourself, which
-requires _python3_setuptools_ and _gcc version >= 10.2.0_.
+requires _python3_setuptools_ and an up-to-date compiler supporting C++11
+standard (e.g. _gcc version >= 10.2.0_).
 
 ## Usage
 
@@ -189,23 +192,23 @@ options `imctermite sample-data.raw -b -c -s '|'`.
 
 ### Python
 
-Given the `imctermite` module is available, we can import it and declare an instance
+Given the `IMCtermite` module is available, we can import it and declare an instance
 of it by passing a _raw_ file to the constructor:
 
 ```Python
-import imc_termite
+import IMCtermite
 
-imcraw = imc_termite.imctermite(b"sample/sampleA.raw")
+imcraw = IMCtermite.imctermite(b"sample/sampleA.raw")
 ```
 
 An example of how to create an instance and obtain the list of channels is:
 
 ```Python
-import imc_termite
+import IMCtermite
 
 # declare and initialize instance of "imctermite" by passing a raw-file
 try :
-    imcraw = imc_termite.imctermite(b"samples/sampleA.raw")
+    imcraw = IMCtermite.imctermite(b"samples/sampleA.raw")
 except RuntimeError as e :
     print("failed to load/parse raw-file: " + str(e))
 

lib/imc_channel.hpp

@@ -318,16 +318,16 @@ namespace imc
     // convert buffer to actual datatype
     void convert_buffer()
     {
-      // TODO no clue how/if/when to handle buffer offset/mask/subsequent_bytes
-      // etc. and whatever that shit is!
       std::vector<imc::parameter> prms = blocks_->at(chnenv_.CSuuid_).get_parameters();
       if ( prms.size() < 4)
       {
         throw std::runtime_error("CS block is invalid and features to few parameters");
       }
+
+      // extract (channel dependent) part of buffer
       unsigned long int buffstrt = prms[3].begin();
-      std::vector<unsigned char> CSbuffer( buffer_->begin()+buffstrt+1,
+      std::vector<unsigned char> CSbuffer( buffer_->begin()+buffstrt+buffer_offset_+1,
-                                           buffer_->begin()+buffstrt+buffer_size_+1 );
+                                           buffer_->begin()+buffstrt+buffer_offset_+buffer_size_+1 );
 
       // determine number of values in buffer
       unsigned long int num_values = (unsigned long int)(CSbuffer.size()/(signbits_/8));
@@ -455,12 +455,12 @@ namespace imc
              <<"\",\"trigger-time\":\""<<std::put_time(std::localtime(&att),"%FT%T")
              <<"\",\"language-code\":\""<<language_code_
              <<"\",\"codepage\":\""<<codepage_
-             <<"\",\"yname\":\""<<yname_
+             <<"\",\"yname\":\""<<prepjsonstr(yname_)
-             <<"\",\"yunit\":\""<<yunit_
+             <<"\",\"yunit\":\""<<prepjsonstr(yunit_)
              <<"\",\"significantbits\":\""<<signbits_
              <<"\",\"addtime\":\""<<addtime_
-             <<"\",\"xname\":\""<<xname_
+             <<"\",\"xname\":\""<<prepjsonstr(xname_)
-             <<"\",\"xunit\":\""<<xunit_
+             <<"\",\"xunit\":\""<<prepjsonstr(xunit_)
              <<"\",\"xstepwidth\":\""<<xstepwidth_
              <<"\",\"xoffset\":\""<<xoffset_
              <<"\",\"group\":{"<<"\"index\":\""<<group_index_
@@ -477,6 +477,25 @@ namespace imc
       return ss.str();
     }
 
+    // prepare string value for usage in JSON dump
+    std::string prepjsonstr(std::string value)
+    {
+      std::stringstream ss;
+      ss<<quoted(value);
+      return strip_quotes(ss.str());
+    }
+
+    // remove any leading or trailing double quotes
+    std::string strip_quotes(std::string astring)
+    {
+      // head
+      if ( astring.front() == '"' ) astring.erase(astring.begin()+0);
+      // tail
+      if ( astring.back() == '"' ) astring.erase(astring.end()-1);
+
+      return astring;
+    }
+
     // print channel
     void print(std::string filename, const char sep = ',', int width = 25, int yprec = 9)
     {

lib/imc_key.hpp

@@ -84,6 +84,7 @@ namespace imc
     // noncritical keys
     key(false,"NO","origin of data",1),
     key(false,"NT","timestamp of trigger",1),
+    key(false,"NT","timestamp of trigger",2),
     key(false,"ND","(color) display properties",1),
     key(false,"NU","user defined key",1),
     key(false,"Np","property of channel",1),

lib/imc_raw.hpp

@@ -236,13 +236,27 @@ namespace imc
             // provide UUID for channel
             chnenv.uuid_ = chnenv.CNuuid_;
 
+            // for multichannel data there may be multiple channels referring to
+            // the same (final) CS block (in contrast to what the IMC software
+            // documentation seems to suggest) resulting in all channels missing
+            // a CS block except for the very last
+            if ( chnenv.CSuuid_.empty() ) {
+              for ( imc::block blkCS: rawblocks_ ) {
+                if ( blkCS.get_key().name_ == "CS"
+                  && blkCS.get_begin() > (unsigned long int)stol(chnenv.uuid_) ) {
+                  chnenv.CSuuid_ = blkCS.get_uuid();
+                }
+              }
+            }
+
             // create channel object and add it to the map of channels
             channels_.insert( std::pair<std::string,imc::channel>
               (chnenv.CNuuid_,imc::channel(chnenv,&mapblocks_,&buffer_))
             );
 
             // reset channel uuid
-            chnenv.CNuuid_.clear();
+            chnenv.reset();
+            //chnenv.CNuuid_.clear();
           }
         }
 
@@ -254,7 +268,6 @@ namespace imc
       }
     }
 
-
   public:
 
     // provide buffer size

makefile

@@ -3,7 +3,7 @@
 SHELL := /bin/bash
 
 # name of executable and CLI tool
-EXE = IMCtermite
+EXE = imctermite
 
 # directory names
 SRC = src/
@@ -31,7 +31,7 @@ INST := /usr/local/bin
 #-----------------------------------------------------------------------------#
 # C++ and CLI tool
 
-# build exectuable
+# build executable
 $(EXE): check-tags $(GVSN) main.o
 	$(CC) $(OPT) main.o -o $@
 

python/IMCtermite.pyx

@@ -20,8 +20,8 @@ cdef class imctermite:
     self.cppimc.set_file(rawfile)
 
   # get JSON list of channels
-  def get_channels(self, bool data):
+  def get_channels(self, bool include_data):
-    chnlst = self.cppimc.get_channels(True,data)
+    chnlst = self.cppimc.get_channels(True,include_data)
     chnlstjn = [jn.loads(chn.decode(errors="ignore")) for chn in chnlst]
     return chnlstjn
 

python/VERSION

@@ -1 +1 @@
-2.0.0
+2.0.6

python/examples/multichannel.py

@@ -0,0 +1,24 @@
+
+import IMCtermite
+import pandas
+
+if __name__ == "__main__" :
+
+    imctm = IMCtermite.imctermite(b"Measurement.raw")
+
+    chns = imctm.get_channels(True)
+
+    df = pandas.DataFrame()
+    
+    xcol = "time ["+chns[0]['xunit']+"]"    
+    df[xcol] = pandas.Series(chns[0]['xdata'])
+
+    for idx,chn in enumerate(chns) :
+        #xcol = (chn['xname'] if chn['xname'] != '' else "x_"+str(idx))+" ["+chn['xunit']+"]"
+        #df[xcol] = pandas.Series(chn['xdata'])
+        ycol = chn['yname']+" ["+chn['yunit']+"]"
+        df[ycol] = pandas.Series(chn['ydata'])
+
+    print(df)
+    df.to_csv("Measurement.csv",header=True,sep='\t',index=False)
+