include (absolute) trigger-time in channel output => issue #10

* executable --help: provide full repository URL

README.md

@@ -1,13 +1,11 @@
 
-<p align="center">
-  <a href="https://record-evolution.de/reswarm">
-    <img
-      alt="imctermite.svg"
-      src="assets/imctermite.svg"
-      width="400"
-    />
-  </a>
-</p>
+[![Total alerts](https://img.shields.io/lgtm/alerts/g/RecordEvolution/IMCtermite.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/RecordEvolution/IMCtermite/alerts/)
+[![Language grade: C/C++](https://img.shields.io/lgtm/grade/cpp/g/RecordEvolution/IMCtermite.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/RecordEvolution/IMCtermite/context:cpp)
+[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/RecordEvolution/IMCtermite.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/RecordEvolution/IMCtermite/context:python)
+[![LICENSE](https://img.shields.io/github/license/RecordEvolution/IMCtermite)](https://img.shields.io/github/license/RecordEvolution/IMCtermite)
+[![STARS](https://img.shields.io/github/stars/RecordEvolution/IMCtermite)](https://img.shields.io/github/stars/RecordEvolution/IMCtermite)
+
+# IMCtermite
 
 _IMCtermite_ provides access to the proprietary data format
 _IMC Bus Format_ with the file extension _.raw_ introduced and developed by
@@ -173,14 +171,18 @@ Options:
  -c, --listchannels      list channels
  -b, --listblocks        list IMC key-blocks
  -d, --output            output directory to print channels
+ -s, --delimiter         csv delimiter/separator char for output
  -h, --help              show this help message
  -v, --version           display version
 ```
 
 For instance, to show a list of all channels included in `sample-data.raw`, you
 do `imctermite sample-data.raw --listchannels`. No output files are
-written by default. Output files are written only when an existing (!) directory is provided as argument to
-the `--output` option.
+written by default. Output files are written only when an existing (!) directory
+is provided as argument to the `--output` option. By default, every output file
+is written using a `,` delimiter. You may provide any custom separator with the
+option `--delimiter`. For example, in order to use `|`, the binary is called with
+options `imctermite sample-data.raw -b -c -s '|'`.
 
 ### Python
 
@@ -219,3 +221,5 @@ in the Python folder.
 - https://www.imc-tm.de/produkte/messtechnik-hardware/imc-cronosflex/ueberblick/
 - https://cython.readthedocs.io/en/latest/src/userguide/wrapping_CPlusPlus.html
 - https://github.com/Apollo3zehn/ImcFamosFile
+- https://apollo3zehn.github.io/ImcFamosFile/api/ImcFamosFile.FamosFileKeyType.html
+- https://pypi.org/help/#apitoken

cython/imc_termite.pxd

@@ -14,6 +14,7 @@ cdef extern from "imc_raw.hpp" namespace "imc":
     void set_file(string rawfile) except +
     # get JSON list of channels
     vector[string] get_channels(bool json, bool data) except +
-    # print all channels
-    void print_channels(string outputdir) except +
+    # print single channel/all channels
+    void print_channel(string channeluuid, string outputdir, char delimiter) except +
+    void print_channels(string outputdir, char delimiter) except +
     void print_table(string outputfile) except +

cython/py_imc_termite.pyx

@@ -22,17 +22,19 @@ cdef class imctermite:
   # get JSON list of channels
   def get_channels(self, bool data):
     chnlst = self.cpp_imc.get_channels(True,data)
-    chnlstjn = [jn.loads(chn.decode(errors="ignore"),parse_float=decimal.Decimal) for chn in chnlst]
+    chnlstjn = [jn.loads(chn.decode(errors="ignore")) for chn in chnlst]
     return chnlstjn
 
-  # print channels
-  def print_channels(self, string outputdir):
-    self.cpp_imc.print_channels(outputdir)
+  # print single channel/all channels
+  def print_channel(self, string channeluuid, string outputfile, char delimiter):
+    self.cpp_imc.print_channel(channeluuid,outputfile,delimiter)
+  def print_channels(self, string outputdir, char delimiter):
+    self.cpp_imc.print_channels(outputdir,delimiter)
 
   # print table including channels
   def print_table(self, string outputfile):
     chnlst = self.cpp_imc.get_channels(True,True)
-    chnlstjn = [jn.loads(chn.decode(errors="ignore"),parse_float=decimal.Decimal) for chn in chnlst]
+    chnlstjn = [jn.loads(chn.decode(errors="ignore")) for chn in chnlst]
     with open(outputfile.decode(),'w') as fout:
       for chn in chnlstjn:
         fout.write('#' +str(chn['xname']).rjust(19)+str(chn['yname']).rjust(20)+'\n')

cython/setup.py

@@ -15,7 +15,7 @@ extensions = Extension(
 
 setup(
     name="imc_termite",
-    version='1.2.3',
+    version='1.2.8',
     description='IMCtermite cython extension',
     author='Record Evolution GmbH',
     author_email='mario.fink@record-evolution.de',

lib/imc_block.hpp

@@ -63,7 +63,7 @@ namespace imc
         std::cout<<"WARNING: invalid length parameter in "<<thekey_.name_<<"-block "
                  <<"(block-end:"<<end_<<",buffer-size:"<<buffer_->size()<<")"
                  <<" => resetting block-end to buffer-size\n";
-        end_ = buffer_->size();
+        end_ = (unsigned long int)(buffer_->size());
       }
 
       try {

lib/imc_channel.hpp

@@ -3,9 +3,12 @@
 #ifndef IMCCHANNEL
 #define IMCCHANNEL
 
-#include <sstream>
 #include "imc_datatype.hpp"
 #include "imc_conversion.hpp"
+#include <sstream>
+#include <math.h>
+#include <chrono>
+#include <ctime>
 
 //---------------------------------------------------------------------------//
 
@@ -20,7 +23,7 @@ namespace imc
     // collect affiliate blocks for a single channel
     std::string CBuuid_, CGuuid_, CCuuid_, CNuuid_;
     std::string CDuuid_, CTuuid_, Cbuuid_, CPuuid_, CRuuid_, CSuuid_;
-    std::string NTuuid_, NOuuid_;
+    std::string NTuuid_, NOuuid_, NLuuid_;
 
     // reset all members
     void reset()
@@ -38,6 +41,7 @@ namespace imc
       CSuuid_.clear();
       NTuuid_.clear();
       NOuuid_.clear();
+      NLuuid_.clear();
     }
 
     // get info
@@ -58,7 +62,8 @@ namespace imc
         <<std::setw(width)<<std::left<<"CSuuid:"<<CSuuid_<<"\n"
         //
         <<std::setw(width)<<std::left<<"NTuuid:"<<NTuuid_<<"\n"
-        <<std::setw(width)<<std::left<<"NOuuid:"<<NOuuid_<<"\n";
+        <<std::setw(width)<<std::left<<"NOuuid:"<<NOuuid_<<"\n"
+        <<std::setw(width)<<std::left<<"NLuuid:"<<NLuuid_<<"\n";
       return ss.str();
     }
 
@@ -79,32 +84,55 @@ namespace imc
              <<"\",\"CSuuid\":\""<<CSuuid_
              <<"\",\"NTuuid\":\""<<NTuuid_
              <<"\",\"NOuuid\":\""<<NOuuid_
+             <<"\",\"NLuuid\":\""<<NLuuid_
              <<"\"}";
       return ss.str();
     }
   };
 
+  // adjust stream object
+  void customize_stream(std::ostream& stout, int prec, bool fixed)
+  {
+    if ( fixed )
+    {
+      stout<<std::setprecision(prec)<<std::fixed;
+    }
+    else
+    {
+      stout<<std::setprecision(prec);
+    }
+  }
+
   // given a list of numeric objects, join it into a string
   template<typename dt>
-  std::string joinvec(std::vector<dt> myvec, unsigned long int limit = 10)
+  std::string joinvec(std::vector<dt> myvec, unsigned long int limit = 10, int prec = 10, bool fixed = true)
   {
-    // include entire list for limit = - 1
+    // include entire list for limit = 0
     limit = (limit == 0) ? myvec.size() : limit;
 
     std::stringstream ss;
     ss<<"[";
     if ( myvec.size() <= limit )
     {
-      for ( dt el: myvec ) ss<<std::setprecision(10)<<el<<",";
+      for ( dt el: myvec )
+      {
+        customize_stream(ss,prec,fixed);
+        ss<<el<<",";
+      }
     }
     else
     {
       unsigned long int heals = (unsigned long int)(limit/2.);
-      for ( unsigned long int i = 0; i < heals; i++ ) ss<<myvec[i]<<",";
+      for ( unsigned long int i = 0; i < heals; i++ )
+      {
+        customize_stream(ss,prec,fixed);
+        ss<<myvec[i]<<",";
+      }
       ss<<"...";
       for ( unsigned long int i = myvec.size()-heals; i < myvec.size(); i++ )
       {
-        ss<<std::setprecision(10)<<myvec[i]<<",";
+        customize_stream(ss,prec,fixed);
+        ss<<myvec[i]<<",";
       }
     }
     std::string sumstr = ss.str();
@@ -126,14 +154,19 @@ namespace imc
     std::string uuid_;
     std::string name_, comment_;
     std::string origin_, origin_comment_, text_;
+    std::chrono::system_clock::time_point trigger_time_, absolute_trigger_time_;
+    double trigger_time_frac_secs_;
+    std::string language_code_, codepage_;
     std::string yname_, yunit_;
     std::string xname_, xunit_;
     double xstepwidth_, xoffset_;
+    int xprec_;
 
     // buffer and data
     int signbits_, num_bytes_;
     // unsigned long int byte_offset_;
     unsigned long int buffer_offset_, buffer_size_;
+    long int addtime_;
     int datatp_;
     imc::datatype dattyp_;
     std::vector<imc::datatype> ydata_;
@@ -147,7 +180,7 @@ namespace imc
     std::string group_uuid_, group_name_, group_comment_;
 
     // constructor takes channel's block environment
-    channel(channel_env chnenv, std::map<std::string,imc::block>* blocks,
+    channel(channel_env &chnenv, std::map<std::string,imc::block>* blocks,
                                  std::vector<unsigned char>* buffer):
       chnenv_(chnenv), blocks_(blocks), buffer_(buffer),
       factor_(1.), offset_(0.),
@@ -185,6 +218,9 @@ namespace imc
         xunit_ = blocks_->at(chnenv_.CDuuid_).get_parameter(prms[5]);
         // TODO
         // xname_ = std::string("time");
+
+        // find appropriate precision for "xdata_" by means of "xstepwidth_"
+        xprec_ = (xstepwidth_ > 0 ) ? (int)ceil(fabs(log10(xstepwidth_))) : 10;
       }
 
       // extract associated CP data
@@ -204,6 +240,7 @@ namespace imc
         buffer_offset_ = std::stoul(blocks_->at(chnenv_.Cbuuid_).get_parameter(prms[6]));
         buffer_size_ = std::stoul(blocks_->at(chnenv_.Cbuuid_).get_parameter(prms[7]));
         xoffset_ = std::stod(blocks_->at(chnenv_.Cbuuid_).get_parameter(prms[11]));
+        addtime_ = (long int)std::stod(blocks_->at(chnenv_.Cbuuid_).get_parameter(prms[12]));
       }
 
       // extract associated CR data
@@ -233,8 +270,48 @@ namespace imc
         origin_comment_ = blocks_->at(chnenv_.NOuuid_).get_parameter(prms[6]);
       }
 
+      // extract associated NL data
+      // codepage:
+      // - http://www.iana.org/assignments/character-sets/character-sets.xhtml
+      // - https://de.wikipedia.org/wiki/Zeichensatztabelle
+      // language-code:
+      // - https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/a9eac961-e77d-41a6-90a5-ce1a8b0cdb9c?redirectedfrom=MSDN
+      if ( blocks_->count(chnenv_.NLuuid_) == 1 )
+      {
+        prms = blocks_->at(chnenv_.NLuuid_).get_parameters();
+        codepage_ = blocks_->at(chnenv_.NLuuid_).get_parameter(prms[2]);
+        language_code_ = blocks_->at(chnenv_.NLuuid_).get_parameter(prms[3]);
+      }
+
+      // obtain NT data
+      // - https://en.cppreference.com/w/cpp/chrono/c/tm
+      // - https://en.cppreference.com/w/cpp/io/manip/put_time
+      if ( blocks_->count(chnenv_.NTuuid_) == 1 )
+      {
+        prms = blocks_->at(chnenv_.NTuuid_).get_parameters();
+        //std::tm tm{};
+        std::tm tms = std::tm();
+        tms.tm_mday = std::stoi(blocks_->at(chnenv_.NTuuid_).get_parameter(prms[2]));
+        tms.tm_mon = std::stoi(blocks_->at(chnenv_.NTuuid_).get_parameter(prms[3])) - 1;
+        tms.tm_year = std::stoi(blocks_->at(chnenv_.NTuuid_).get_parameter(prms[4])) - 1900;
+        tms.tm_hour = std::stoi(blocks_->at(chnenv_.NTuuid_).get_parameter(prms[5]));
+        tms.tm_min = std::stoi(blocks_->at(chnenv_.NTuuid_).get_parameter(prms[6]));
+        double secs = std::stold(blocks_->at(chnenv_.NTuuid_).get_parameter(prms[7]));
+        double secs_int;
+        trigger_time_frac_secs_ = modf(secs,&secs_int);
+        tms.tm_sec = (int)secs_int;
+
+        // generate std::chrono::system_clock::time_point type
+        std::time_t ts = std::mktime(&tms);
+        trigger_time_ = std::chrono::system_clock::from_time_t(ts);
+      }
+
       // start converting binary buffer to imc::datatype
       if ( !chnenv_.CSuuid_.empty() ) convert_buffer();
+
+      // calculate absolute trigger-time
+      absolute_trigger_time_ = trigger_time_ + std::chrono::seconds(addtime_);
+      //                                       + std::chrono::nanoseconds((long int)(trigger_time_frac_secs_*1.e9));
     }
 
     // convert buffer to actual datatype
@@ -252,7 +329,7 @@ namespace imc
                                            buffer_->begin()+buffstrt+buffer_size_+1 );
 
       // determine number of values in buffer
-      unsigned long int num_values = CSbuffer.size()/(signbits_/8);
+      unsigned long int num_values = (unsigned long int)(CSbuffer.size()/(signbits_/8));
       if ( num_values*(signbits_/8) != CSbuffer.size() )
       {
         throw std::runtime_error("CSbuffer and significant bits of datatype don't match");
@@ -324,18 +401,27 @@ namespace imc
     // get info string
     std::string get_info(int width = 20)
     {
+      // prepare printable trigger-time
+      //std::time_t tt = std::chrono::system_clock::to_time_t(trigger_time_);
+      std::time_t att = std::chrono::system_clock::to_time_t(absolute_trigger_time_);
+
       std::stringstream ss;
       ss<<std::setw(width)<<std::left<<"uuid:"<<uuid_<<"\n"
         <<std::setw(width)<<std::left<<"name:"<<name_<<"\n"
         <<std::setw(width)<<std::left<<"comment:"<<comment_<<"\n"
         <<std::setw(width)<<std::left<<"origin:"<<origin_<<"\n"
         <<std::setw(width)<<std::left<<"description:"<<text_<<"\n"
+        //<<std::setw(width)<<std::left<<"trigger-time:"<<std::put_time(std::localtime(&tt),"%FT%T")<<"\n"
+        <<std::setw(width)<<std::left<<"trigger-time:"<<std::put_time(std::localtime(&att),"%FT%T")<<"\n"
+        <<std::setw(width)<<std::left<<"language-code:"<<language_code_<<"\n"
+        <<std::setw(width)<<std::left<<"codepage:"<<codepage_<<"\n"
         <<std::setw(width)<<std::left<<"yname:"<<yname_<<"\n"
         <<std::setw(width)<<std::left<<"yunit:"<<yunit_<<"\n"
         <<std::setw(width)<<std::left<<"datatype:"<<datatp_<<"\n"
         <<std::setw(width)<<std::left<<"significant bits:"<<signbits_<<"\n"
         <<std::setw(width)<<std::left<<"buffer-offset:"<<buffer_offset_<<"\n"
         <<std::setw(width)<<std::left<<"buffer-size:"<<buffer_size_<<"\n"
+        //<<std::setw(width)<<std::left<<"add-time:"<<addtime_<<"\n"
         <<std::setw(width)<<std::left<<"xname:"<<xname_<<"\n"
         <<std::setw(width)<<std::left<<"xunit:"<<xunit_<<"\n"
         <<std::setw(width)<<std::left<<"xstepwidth:"<<xstepwidth_<<"\n"
@@ -344,8 +430,8 @@ namespace imc
         <<std::setw(width)<<std::left<<"offset:"<<offset_<<"\n"
         <<std::setw(width)<<std::left<<"group:"<<"("<<group_index_<<","<<group_name_
                                                     <<","<<group_comment_<<")"<<"\n"
-        <<std::setw(width)<<std::left<<"ydata:"<<imc::joinvec<imc::datatype>(ydata_)<<"\n"
-        <<std::setw(width)<<std::left<<"xdata:"<<imc::joinvec<double>(xdata_)<<"\n";
+        <<std::setw(width)<<std::left<<"ydata:"<<imc::joinvec<imc::datatype>(ydata_,6,9,true)<<"\n"
+        <<std::setw(width)<<std::left<<"xdata:"<<imc::joinvec<double>(xdata_,6,xprec_,true)<<"\n";
         // <<std::setw(width)<<std::left<<"aff. blocks:"<<chnenv_.get_json()<<"\n";
       return ss.str();
     }
@@ -353,12 +439,19 @@ namespace imc
     // provide JSON string of metadata
     std::string get_json(bool include_data = false)
     {
+      // prepare printable trigger-time
+      //std::time_t tt = std::chrono::system_clock::to_time_t(trigger_time_);
+      std::time_t att = std::chrono::system_clock::to_time_t(absolute_trigger_time_);
+
       std::stringstream ss;
       ss<<"{"<<"\"uuid\":\""<<uuid_
              <<"\",\"name\":\""<<name_
              <<"\",\"comment\":\""<<comment_
              <<"\",\"origin\":\""<<origin_
              <<"\",\"description\":\""<<text_
+             <<"\",\"trigger-time\":\""<<std::put_time(std::localtime(&att),"%FT%T")
+             <<"\",\"language-code\":\""<<language_code_
+             <<"\",\"codepage\":\""<<codepage_
              <<"\",\"yname\":\""<<yname_
              <<"\",\"yunit\":\""<<yunit_
              <<"\",\"significantbits\":\""<<signbits_
@@ -371,8 +464,8 @@ namespace imc
                                <<"\",\"comment\":\""<<group_comment_<<"\""<<"}";
       if ( include_data )
       {
-        ss<<",\"ydata\":"<<imc::joinvec<imc::datatype>(ydata_,0)
-          <<",\"xdata\":"<<imc::joinvec<double>(xdata_,0);
+        ss<<",\"ydata\":"<<imc::joinvec<imc::datatype>(ydata_,0,9,true)
+          <<",\"xdata\":"<<imc::joinvec<double>(xdata_,0,xprec_,true);
       }
       // ss<<"\",\"aff. blocks\":\""<<chnenv_.get_json()
       ss<<"}";
@@ -381,7 +474,7 @@ namespace imc
     }
 
     // print channel
-    void print(std::string filename, const char sep = ' ', int width = 25)
+    void print(std::string filename, const char sep = ',', int width = 25, int yprec = 9)
     {
       std::ofstream fou(filename);
 
@@ -402,13 +495,16 @@ namespace imc
       {
         if ( sep == ' ' )
         {
-          fou<<std::setprecision(9)<<std::fixed
+          fou<<std::setprecision(xprec_)<<std::fixed
              <<std::setw(width)<<std::left<<xdata_[i]
+             <<std::setprecision(yprec)<<std::fixed
              <<std::setw(width)<<std::left<<ydata_[i]<<"\n";
         }
         else
         {
-          fou<<xdata_[i]<<sep<<ydata_[i]<<"\n";
+          fou<<std::setprecision(xprec_)<<std::fixed<<xdata_[i]
+             <<sep
+             <<std::setprecision(yprec)<<std::fixed<<ydata_[i]<<"\n";
         }
       }
 

lib/imc_conversion.hpp

@@ -1,6 +1,6 @@
 //---------------------------------------------------------------------------//
 
-#ifndef IMCCONVRSION
+#ifndef IMCCONVERSION
 #define IMCCONVERSION
 
 #include <vector>

lib/imc_datatype.hpp

@@ -48,7 +48,7 @@ namespace imc
   public:
     datatype(): ubyte_(0), sbyte_(0),
                 ushort_(0), sshort_(0),
-                ulint_(0.0), slint_(0.0),
+                ulint_(0), slint_(0),
                 sfloat_(0.0), sdouble_(0.0),
                 sdigital_(0),
                 dtidx_(0) { };
@@ -61,15 +61,30 @@ namespace imc
     datatype(imc_Slongint num): slint_(num), dtidx_(5) {};
     datatype(imc_float num): sfloat_(num), dtidx_(6) {};
     datatype(imc_double num): ubyte_(0), sbyte_(0), ushort_(0), sshort_(0),
-                              ulint_(0.0), slint_(0.0), sfloat_(0.0), sdouble_(num),
+                              ulint_(0), slint_(0), sfloat_(0.0), sdouble_(num),
                               sdigital_(0), dtidx_(7) {};
     datatype(imc_digital num): ubyte_(0), sbyte_(0), ushort_(0), sshort_(0),
-                               ulint_(0.0), slint_(0.0), sfloat_(0.0), sdouble_(num),
+                               ulint_(0), slint_(0), sfloat_(0.0), sdouble_(num),
                                sdigital_(num), dtidx_(10) {};
 
     // identify type
     short int& dtype() { return dtidx_; }
 
+    // copy constructor
+    datatype(const datatype &num)
+    {
+      this->ubyte_ = num.ubyte_;
+      this->sbyte_ = num.sbyte_;
+      this->ushort_ = num.ushort_;
+      this->sshort_ = num.sshort_;
+      this->ulint_ = num.ulint_;
+      this->slint_ = num.slint_;
+      this->sfloat_ = num.sfloat_;
+      this->sdouble_ = num.sdouble_;
+      this->sdigital_ = num.sdigital_;
+      this->dtidx_ = num.dtidx_;
+    }
+
     // overall assignment operator
     datatype& operator=(const datatype &num)
     {

lib/imc_key.hpp

@@ -87,7 +87,8 @@ namespace imc
     key(false,"ND","(color) display properties",1),
     key(false,"NU","user defined key",1),
     key(false,"Np","property of channel",1),
-    key(false,"NE","extraction rule for BUS channels",1)
+    key(false,"NE","extraction rule for BUS channels",1),
+    key(false,"NL","language info and code page",1)
 
   };
 

lib/imc_object.hpp

@@ -500,22 +500,22 @@ namespace imc
       minute_ = std::stoi( get_parameter(buffer,&parameters[6]) );
       second_ = std::stod( get_parameter(buffer,&parameters[7]) );
 
-      time_t rawtime;
-      struct tm* ts;
-      time(&rawtime);
-      ts = localtime(&rawtime);
-      ts->tm_mday = day_;
-      ts->tm_mon = month_-1;
-      ts->tm_year = year_-1900;
-      ts->tm_hour = hour_;
-      ts->tm_min = minute_;
-      ts->tm_sec = (int)second_;
-      timestamp_ = asctime(ts);
-      // timestamp_ = std::to_string(year_) + std::string("-") + std::to_string(month_)
-      //                                    + std::string("-") + std::to_string(day_)
-      //                                    + std::string("T") + std::to_string(hour_)
-      //                                    + std::string(":") + std::to_string(minute_)
-      //                                    + std::string(":") + std::to_string(second_);
+      //time_t rawtime;
+      //struct tm ts;
+      //time(&rawtime);
+      //localtime_r(&rawtime,&ts);
+      //ts.tm_mday = day_;
+      //ts.tm_mon = month_-1;
+      //ts.tm_year = year_-1900;
+      //ts.tm_hour = hour_;
+      //ts.tm_min = minute_;
+      //ts.tm_sec = (int)second_;
+      //asctime_r(&ts,&timestamp_[0]);
+      timestamp_ = std::to_string(year_) + std::string("-") + std::to_string(month_)
+                                         + std::string("-") + std::to_string(day_)
+                                         + std::string("T") + std::to_string(hour_)
+                                         + std::string(":") + std::to_string(minute_)
+                                         + std::string(":") + std::to_string(second_);
     }
 
     // get info string

lib/imc_raw.hpp

@@ -123,20 +123,20 @@ namespace imc
                   leng.push_back((char)*(it+pos));
                   pos++;
                 }
-                unsigned long length = std::stoul(leng);
+                unsigned long int length = std::stoul(leng);
 
                 // declare and initialize corresponding key and block
                 // imc::key bkey( *(it+1)==imc::key_crit_ , newkey,
                 //                imc::keys.at(newkey).description_, version );
-                imc::block blk(itkey,it-buffer_.begin(),
-                                     it-buffer_.begin()+pos+1+length,
+                imc::block blk(itkey,(unsigned long int)(it-buffer_.begin()),
+                                     (unsigned long int)(it-buffer_.begin()+pos+1+length),
                                      raw_file_, &buffer_);
 
                 // add block to list
                 rawblocks_.push_back(blk);
 
                 // skip the remaining block according to its length
-                if ( it-buffer_.begin()+length < buffer_.size() )
+                if ( (unsigned long int)(it-buffer_.begin()+length) < (unsigned long int)(buffer_.size()) )
                 {
                   std::advance(it,length);
                 }
@@ -174,7 +174,7 @@ namespace imc
     // check consistency of blocks
     void check_consistency()
     {
-      for ( unsigned long int b = 0; b < this->rawblocks_.size()-1; b++ )
+      for ( unsigned long int b = 0; b < this->rawblocks_.size()-1 && this->rawblocks_.size() > 0; b++ )
       {
         if ( this->rawblocks_[b].get_end() >= this->rawblocks_[b+1].get_begin() )
         {
@@ -217,6 +217,7 @@ namespace imc
         else if ( blk.get_key().name_ == "CS" ) chnenv.CSuuid_ = blk.get_uuid();
         else if ( blk.get_key().name_ == "NT" ) chnenv.NTuuid_ = blk.get_uuid();
         else if ( blk.get_key().name_ == "NO" ) chnenv.NOuuid_ = blk.get_uuid();
+        else if ( blk.get_key().name_ == "NL" ) chnenv.NLuuid_ = blk.get_uuid();
 
         // check for currently associated channel
         if ( !chnenv.CNuuid_.empty() )
@@ -252,7 +253,7 @@ namespace imc
     // provide buffer size
     unsigned long int buffer_size()
     {
-      return buffer_.size();
+      return (unsigned long int)buffer_.size();
     }
 
     // get blocks
@@ -300,7 +301,7 @@ namespace imc
     }
 
     // list a particular type of block
-    std::vector<imc::block> list_blocks(imc::key mykey)
+    std::vector<imc::block> list_blocks(const imc::key &mykey)
     {
       std::vector<imc::block> myblocks;
       for ( imc::block blk: this->rawblocks_ )
@@ -332,14 +333,38 @@ namespace imc
       return channels;
     }
 
-    // print all channels in directory
-    void print_channels(std::string output)
+    // print single specific channel
+    void print_channel(std::string channeluuid, std::string outputfile, const char sep)
+    {
+      // check for given parent directory of output file
+      std::filesystem::path pdf = outputfile;
+      if ( !std::filesystem::is_directory(pdf.parent_path()) )
+      {
+        throw std::runtime_error(std::string("required directory does not exist: ")
+                                 + pdf.parent_path().u8string() );
+      }
+
+      // find channel with given name
+      if ( channels_.count(channeluuid) == 1 )
+      {
+        channels_.at(channeluuid).print(outputfile,sep);
+      }
+      else
+      {
+        throw std::runtime_error(std::string("channel does not exist:")
+                                 + channeluuid);
+      }
+    }
+
+    // print all channels into given directory
+    void print_channels(std::string output, const char sep)
     {
       // check for given directory
       std::filesystem::path pd = output;
       if ( !std::filesystem::is_directory(pd) )
       {
-        throw std::runtime_error("given directory does not exist");
+        throw std::runtime_error(std::string("given directory does not exist: ")
+                                 + output);
       }
 
       for ( std::map<std::string,imc::channel>::iterator it = channels_.begin();
@@ -352,7 +377,7 @@ namespace imc
         std::filesystem::path pf = pd / filenam;
 
         // and print the channel
-        it->second.print(pf);
+        it->second.print(pf.u8string(),sep);
       }
     }
 

makefile

@@ -53,6 +53,12 @@ cpp-clean :
 	rm -vf $(EXE)
 	rm -vf *.o
 
+#-----------------------------------------------------------------------------#
+# linter and code check
+
+check-code:
+	cppcheck --enable=all -I lib/ src/main.cpp
+
 #-----------------------------------------------------------------------------#
 # check version consistency of git tags and version string in package.json
 
@@ -87,7 +93,7 @@ cython-install : check-vtag $(CTAG) $(CYT)setup.py $(CYT)imc_termite.pxd $(CYT)p
 cython-clean :
 	rm -vf imc_termite.cpython-*.so
 	rm -vf $(PYT)imc_termite.cpython-*.so
-	rm -rv build/
+	rm -rvf build/
 
 #-----------------------------------------------------------------------------#
 # pip

parquet/parquet/generate_make.py

@@ -1,6 +1,5 @@
 #-----------------------------------------------------------------------------#
 
-import glob
 from pathlib import Path
 
 # find source files

pip/makefile

@@ -11,6 +11,9 @@ sdist: ../cython/py_imc_termite.pyx ../cython/imc_termite.pxd ../cython/py_imc_t
 	cp -v ../LICENSE ./
 	python3 setup.py sdist
 
+# authentication:
+# - username: __token__
+# - password: <token value including pypi-prefix>
 upload:
 	python3 -m twine upload dist/$(shell ls -t dist/ | head -n1)
 

pip/setup.py

@@ -1,13 +1,24 @@
 
 from setuptools import setup, Extension
-import os
+import sys
+
+print("building on platform: "+sys.platform)
+
+if sys.platform == "linux" or sys.platform == "darwin" :
+	cmpargs = ['-std=c++17','-Wno-unused-variable']
+	lnkargs = ['-std=c++17','-Wno-unused-variable']
+elif sys.platform == "win32" :
+	cmpargs = ['/EHsc','/std:c++17']
+	lnkargs = []
+else :
+	raise RuntimeError("unknown platform")
 
 with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
 setup(
     name="IMCtermite",
-    version="1.2.3",
+    version="1.2.8",
     author="Record Evolution GmbH",
     author_email="mario.fink@record-evolution.de",
     maintainer="Record Evolution GmbH",
@@ -31,7 +42,8 @@ setup(
                            # include_dirs=["3rdparty/pugixml/","lib/"],
                            # depends=["../lib/tdm_termite.hpp"]
                            language='c++',
-                           extra_compile_args=['-std=c++17','-Wno-unused-variable'],
-                           extra_link_args=['-std=c++17'],
-                           )],
+                           extra_compile_args=cmpargs,
+                           extra_link_args=lnkargs,
+                           )
+                ],
 )

python/example.py

@@ -5,19 +5,18 @@ import raw_eater
 import raw_meat
 import pyarrow as pa
 import pyarrow.parquet as pq
-import glob
 from pathlib import Path
 
-fileobj1 = Path("smp/Rangerover_Evoque_F-RR534_2019-05-07/").rglob("*.raw")
+fileobj1 = Path("samples/datasetA/").rglob("*.raw")
 rawlist1 = [str(fl) for fl in fileobj1]
 
-fileobj2 = Path("smp/Mercedes_E-Klasse-2019-08-08/").rglob("*.raw")
+fileobj2 = Path("samples/datasetB/").rglob("*.raw")
 rawlist2 = [str(fl) for fl in fileobj2]
 
 rawlist = rawlist1 #[rawlist1[0],rawlist1[4],rawlist2[0],rawlist2[6]]
 for fil in rawlist2 :
     rawlist.append(fil)
-rawlist.append("smp/pressure_Vacuum.asc")
+rawlist.append("./README.md")
 
 print("")
 print(rawlist)

python/usage.py

@@ -1,10 +1,11 @@
 
 import imc_termite
 import json
+import os
 
 # declare and initialize instance of "imctermite" by passing a raw-file
 try :
-    imcraw = imc_termite.imctermite(b"samples/sampleA.raw")
+    imcraw = imc_termite.imctermite(b"samples/exampleB.raw")
 except RuntimeError as e :
     raise Exception("failed to load/parse raw-file: " + str(e))
 
@@ -22,7 +23,16 @@ print(len(chnydata))
 print(len(chnxdata))
 
 # print the channels into a specific directory
-imcraw.print_channels(b"./")
+imcraw.print_channels(b"./data",ord(','))
+
+# print all channels separately
+idx = 0
+for chn in channels :
+    print(str(idx)+" : "+chn['name']+" : "+chn['uuid'])
+    filname = os.path.join("./data",str(idx) + "_" + chn['name']+".csv")
+    print(filname)
+    imcraw.print_channel(chn['uuid'].encode(),filname.encode(),ord(','))
+    idx = idx + 1
 
 # print all channels in single file
-imcraw.print_table(b"./allchannels.csv")
+# imcraw.print_table(b"./data/allchannels.csv")

python/usage_adv.py

@@ -1,9 +1,7 @@
 
 import imc_termite
 import json
-import glob
 import os
-from pathlib import Path
 
 # list files in sample directory
 # fileobj1 = Path("samples/").rglob("*.raw")

shell/check_markers.sh

@@ -1,12 +1,50 @@
 ##!/bin/bash/
 
-dir=$1
+fildir=$1
 
-#ls ${dir} | while read fn; do echo $fn; cat ${dir}$fn | grep -a "|[A-Z][A-Z]," -o | wc -l; done;
+if [ -z "${fildir}" ]; then
+  echo "CLI argument missing: provide file or directory" >&2
+  exit 1
+fi
 
-#ls ${dir} | while read fn; do echo $fn; cat ${dir}$fn | grep -a "|[A-Z][A-Z]," -o; done;
+list-markers()
+{
+  arg="$1"
+  if [ -z "${arg}" ]; then
+    echo "list-markers: missing file argument" >&2
+    exit 1
+  else
+    if [ -d "${arg}" ]; then
+      echo "list-markers: file argument is a directory" >&2
+      exit 1
+    fi
+  fi
   
-#ls ${dir} | while read fn; do echo $fn; cat ${dir}$fn | xxd | head -n10 | tail -n3; done;
+  echo -e "\n${arg}\n"
+  mrks=$(cat ${arg} | grep -a "|[A-Z][a-zA-Z]," -o)
+  mrksnum=$(echo "${mrks}" | wc -l)
+  echo -e "(${mrksnum})\n${mrks}"
+}
+
+if [ -f "${fildir}" ]; then
+
+  echo "analyzing single file ${fildir} ..."
+  list-markers "${fildir}"
+
+elif [ -d "${fildir}" ]; then
+
+  echo "analyzing all *.raw files in directory ${fildir} ..."
+  lsfls=$(ls ${fildir}/*.raw | sed 's/\/\//\//g')
+  echo -e "\n${lsfls}"
+
+  for fl in ${lsfls}; do
+    list-markers "${fl}"
+  done
+
+else
+
+  echo "does not exist: ${fildir}" >&2
+  exit 1
+
+fi
 
-ls ${dir} | while read fn; do echo $fn; cat ${dir}$fn | grep -a "|[A-Z][a-zA-Z]," -o | wc -l; done;
-ls ${dir} | while read fn; do echo $fn; cat ${dir}$fn | grep -a "|[A-Z][a-zA-Z]," -o; done;

src/main.cpp

@@ -32,25 +32,37 @@ optkeys parse_args(int argc, char* argv[], bool list_args = false)
   // parse all CLI arguments
   for ( int i = 1; i < argc; i++ )
   {
-    if ( std::string(argv[i]) == std::string("--showmeta")
-      || std::string(argv[i]) == std::string("-m") )
-    {
-      prsdkeys.insert(std::pair<std::string,std::string>("showmeta",argv[i]));
-    }
+    // if ( std::string(argv[i]) == std::string("--showmeta")
+    //   || std::string(argv[i]) == std::string("-m") )
+    // {
+    //   prsdkeys.insert(std::pair<std::string,std::string>("showmeta",argv[i]));
+    // }
     // else if ( std::string(argv[i]) == std::string("--listgroups")
     //        || std::string(argv[i]) == std::string("-g") )
     // {
     //   prsdkeys.insert(std::pair<std::string,std::string>("listgroups",argv[i]));
     // }
-    else if ( std::string(argv[i]) == std::string("--listchannels")
+    if ( std::string(argv[i]) == std::string("--listchannels")
            || std::string(argv[i]) == std::string("-c") )
     {
       prsdkeys.insert(std::pair<std::string,std::string>("listchannels",argv[i]));
+      if ( i+1 < argc ) {
+        if ( argv[i+1][0] != '-' ) {
+          std::cerr<<"option --list-channels does not take any argument\n";
+          prsdkeys.insert(std::pair<std::string,std::string>("invalid","channels"));
+        }
+      }
     }
     else if ( std::string(argv[i]) == std::string("--listblocks")
            || std::string(argv[i]) == std::string("-b") )
     {
       prsdkeys.insert(std::pair<std::string,std::string>("listblocks",argv[i]));
+      if ( i+1 < argc ) {
+        if ( argv[i+1][0] != '-' ) {
+          std::cerr<<"option --list-blocks does not take any argument\n";
+          prsdkeys.insert(std::pair<std::string,std::string>("invalid","listblocks"));
+        }
+      }
     }
     else if ( std::string(argv[i]) == std::string("--output")
            || std::string(argv[i]) == std::string("-d") )
@@ -66,6 +78,20 @@ optkeys parse_args(int argc, char* argv[], bool list_args = false)
         i = i + 1;
       }
     }
+    else if ( std::string(argv[i]) == std::string("--delimiter")
+           || std::string(argv[i]) == std::string("-s") )
+    {
+      if ( i+1 == argc || argv[i+1][0] == '-' )
+      {
+        std::cerr<<"invalid or missing --delimiter argument\n";
+        prsdkeys.insert(std::pair<std::string,std::string>("invalid","delimiter"));
+      }
+      else
+      {
+        prsdkeys.insert(std::pair<std::string,std::string>("delimiter",argv[i+1]));
+        i = i + 1;
+      }
+    }
     else if ( std::string(argv[i]) == std::string("--help")
            || std::string(argv[i]) == std::string("-h") )
     {
@@ -84,13 +110,13 @@ optkeys parse_args(int argc, char* argv[], bool list_args = false)
         std::cerr<<"invalid or unkown argument: "<<argv[i]<<"\n";
         prsdkeys.insert(std::pair<std::string,std::string>("invalid",argv[i]));
       }
-      // or missing filenames
-      else if ( std::string(argv[i]).find(".raw") == std::string::npos )
-      {
-        std::cerr<<"doesn't look like a .raw file (make sure to include extension in filename!): "
-                 <<argv[i]<<"\n";
-        prsdkeys.insert(std::pair<std::string,std::string>("invalid",argv[i]));
-      }
+      // // or missing filenames
+      // else if ( std::string(argv[i]).find(".raw") == std::string::npos )
+      // {
+      //   std::cerr<<"doesn't look like a .raw file (make sure to include extension in filename!): "
+      //            <<argv[i]<<"\n";
+      //   prsdkeys.insert(std::pair<std::string,std::string>("invalid",argv[i]));
+      // }
     }
   }
 
@@ -107,7 +133,7 @@ void show_version()
 void show_usage()
 {
   std::cout<<"\n"
-           <<"imctermite ["<<gittag<<"-g"<<githash<<"] (github.com/RecordEvolution/IMCtermite.git)"
+           <<"imctermite ["<<gittag<<"-g"<<githash<<"] (https://github.com/RecordEvolution/IMCtermite.git)"
            <<"\n\n"
            <<"Decode IMC raw files and dump data as *.csv"
            <<"\n\n"
@@ -121,9 +147,13 @@ void show_usage()
            <<" -c, --listchannels      list channels\n"
            <<" -b, --listblocks        list IMC key-blocks\n"
            <<" -d, --output            output directory to print channels\n"
+           <<" -s, --delimiter         csv delimiter/separator char for output\n"
            <<" -h, --help              show this help message \n"
            <<" -v, --version           display version\n"
-           <<"\n";
+           <<"\n"
+           <<"Example:"
+           <<" $ ./imctermite sample/data_A.raw -c -b -d ./data -s ','"
+           <<"\n\n";
 }
 
 //---------------------------------------------------------------------------//
@@ -178,6 +208,13 @@ int main(int argc, char* argv[])
       return 1;
     }
 
+    // catch invalid or empty ".raw" file
+    if ( imcraw.blocks().size() == 0 )
+    {
+      std::cerr<<"this appears to be an empty/invalid '.raw' file since no blocks were found"<<"\n";
+      return 1;
+    }
+
     // list blocks
     if ( cfgopts.count("listblocks") == 1 )
     {
@@ -202,7 +239,22 @@ int main(int argc, char* argv[])
     if ( cfgopts.count("output") == 1 )
     {
       try {
-        imcraw.print_channels(cfgopts.at("output"));
+        // check and use desired delimiter
+        char delim;
+        if ( cfgopts.count("delimiter") == 1 )
+        {
+          if ( cfgopts.at("delimiter").size() > 1 )
+          {
+            throw std::runtime_error("invalid delimiter comprised of more than a single char");
+          }
+          delim = cfgopts.at("delimiter")[0];
+        }
+        // use comma by default
+        else
+        {
+          delim = ',';
+        }
+        imcraw.print_channels(cfgopts.at("output"),delim);
       } catch (const std::exception& e) {
         std::cerr<<"failed to print channels for "<<rawfile<<": "<<e.what()<<"\n";
       }