include origin comment of NO block

* https://github.com/badges/shields/issues/8733
* https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/adding-a-workflow-status-badge

.github/workflows/pypi-deploy.yml

@@ -1,9 +1,9 @@
 
-name: Build Python Wheels
+name: CI Build Wheel
 
 on:
   push:
-    #branches: [master]
+    # branches: [master]
     tags: ["v[0-9]+.[0-9]+.[0-9]+"]
 
 jobs:

README.md

@@ -1,15 +1,13 @@
 
-[![Total alerts](https://img.shields.io/lgtm/alerts/g/RecordEvolution/IMCtermite.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/RecordEvolution/IMCtermite/alerts/)
-[![Language grade: C/C++](https://img.shields.io/lgtm/grade/cpp/g/RecordEvolution/IMCtermite.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/RecordEvolution/IMCtermite/context:cpp)
-[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/RecordEvolution/IMCtermite.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/RecordEvolution/IMCtermite/context:python)
 [![LICENSE](https://img.shields.io/github/license/RecordEvolution/IMCtermite)](https://img.shields.io/github/license/RecordEvolution/IMCtermite)
 [![STARS](https://img.shields.io/github/stars/RecordEvolution/IMCtermite)](https://img.shields.io/github/stars/RecordEvolution/IMCtermite)
-![Build Python Wheels](https://github.com/RecordEvolution/IMCtermite/actions/workflows/pypi-deploy.yml/badge.svg)
+![CI Build Wheel](https://github.com/RecordEvolution/IMCtermite/actions/workflows/pypi-deploy.yml/badge.svg?branch=&event=push)
+[![PYPI](https://img.shields.io/pypi/v/IMCtermite.svg)](https://pypi.org/project/IMCtermite/)
 
 # IMCtermite
 
 _IMCtermite_ provides access to the proprietary data format
-_IMC Bus Format_ with the file extension _.raw_ introduced and developed by
+_IMC2 Data Format_ with the file extension _.raw_ (or .dat) introduced and developed by
 [imc Test & Measurement GmbH](https://www.imc-tm.de/). This data format is
 employed i.a. by the measurement hardware
 [imc CRONOSflex](https://www.imc-tm.de/produkte/messtechnik-hardware/imc-cronosflex/ueberblick/)
@@ -20,7 +18,9 @@ for measurement data control and analysis. Thanks to the integrated Python modul
 the extracted measurement data can be stored in any open-source file format
 accessible by Python like i.a. _csv_, _json_ or _parquet_.
 
-On the [Record Evolution Platform](https://www.record-evolution.de/en/home-en/), the library can be used both as a command line tool for interactive usage and as a Python module to integrate the _.raw_  format into any ETL workflow.   
+On the [Record Evolution Platform](https://www.record-evolution.de/en/home-en/),
+the library can be used both as a command line tool for interactive usage and as a
+Python module to integrate the _.raw_  format into any ETL workflow.
 
 ## Overview
 
@@ -31,12 +31,11 @@ On the [Record Evolution Platform](https://www.record-evolution.de/en/home-en/),
 
 ## File format
 
-[Warning: Take a look at [this issue](https://github.com/RecordEvolution/IMCtermite/issues/14) when reading this section regarding the file format.]
+A file of the _IMC2 Data Format_ type with extension _.raw_ (or .dat) is a _mixed text/binary
-
-A data file of the _IMC Bus Format_ type with the extension _.raw_ is a _mixed text/binary
 file_ featuring a set of markers (keys) that indicate the start of various blocks
 of data that provide meta information and the actual measurement data. Every single
-marker is introduced by the character `"|" = 0x 7c` followed by two uppercase letters that characterize the type of marker. Each block is further divided into several
+marker is introduced by the character `"|" = 0x 7c` followed by two uppercase letters that
+characterize the type of marker. Each block is further divided into several
 parameters separated by commata `"," = 0x 2c` and terminated by a semicolon
 `";" = 0x 3b`. For instance, the header - first 600 bytes - of a raw file may
 look like this (in UTF-8 encoding):
@@ -225,6 +224,8 @@ can be found in the `python/examples` folder.
 
 - https://www.imc-tm.de/produkte/messtechnik-software/imc-famos/funktionen/im-und-export/
 - https://www.imc-tm.de/produkte/messtechnik-hardware/imc-cronosflex/ueberblick/
+- https://www.imc-tm.de/download-center/produkt-downloads/imc-famos/handbuecher
+- https://www.imc-tm.de/fileadmin/Public/Downloads/Manuals/imc_FAMOS/imcGemeinsameKomponenten.pdf
 - https://cython.readthedocs.io/en/latest/src/userguide/wrapping_CPlusPlus.html
 - https://github.com/Apollo3zehn/ImcFamosFile
 - https://apollo3zehn.github.io/ImcFamosFile/api/ImcFamosFile.FamosFileKeyType.html
@@ -233,3 +234,6 @@ can be found in the `python/examples` folder.
 - https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 - https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml
 - https://cibuildwheel.readthedocs.io/en/stable/deliver-to-pypi/
+- https://www.gnu.org/software/libiconv/
+- https://vcpkg.io/en/packages.html
+- https://vcpkg.io/en/getting-started

lib/imc_channel.hpp

@@ -9,6 +9,12 @@
 #include <math.h>
 #include <chrono>
 #include <ctime>
+#include <time.h>
+#if defined(__linux__) || defined(__APPLE__)
+#include <iconv.h>
+#elif defined(__WIN32__) || defined(_WIN32)
+#define timegm _mkgmtime
+#endif
 
 //---------------------------------------------------------------------------//
 
@@ -142,6 +148,87 @@ namespace imc
     return sumstr;
   }
 
+  #if defined(__linux__) || defined(__APPLE__)
+  // convert encoding of any descriptions, channel-names, units etc.
+  class iconverter
+  {
+    std::string in_enc_, out_enc_;
+    iconv_t cd_;
+    size_t out_buffer_size_;
+
+    public:
+
+      iconverter(std::string in_enc, std::string out_enc, size_t out_buffer_size = 1024) :
+        in_enc_(in_enc), out_enc_(out_enc), out_buffer_size_(out_buffer_size)
+      {
+        // allocate descriptor for character set conversion
+        // (https://man7.org/linux/man-pages/man3/iconv_open.3.html)
+        cd_ = iconv_open(out_enc.c_str(), in_enc.c_str());
+
+        if ( (iconv_t)-1 == cd_ )
+        {
+          if ( errno == EINVAL )
+          {
+            std::string errmsg = std::string("The encoding conversion from ") + in_enc
+              + std::string(" to ") + out_enc + std::string(" is not supported by the implementation.");
+            throw std::runtime_error(errmsg);
+          }
+        }
+      }
+
+      void convert(std::string &astring)
+      {
+        if ( astring.empty() ) return;
+
+        std::vector<char> in_buffer(astring.begin(),astring.end());
+        char *inbuf = &in_buffer[0];
+        size_t inbytes = in_buffer.size();
+
+        std::vector<char> out_buffer(out_buffer_size_);
+        char *outbuf = &out_buffer[0];
+        size_t outbytes = out_buffer.size();
+
+        // perform character set conversion
+        // ( - https://man7.org/linux/man-pages/man3/iconv.3.html
+        //   - https://www.ibm.com/docs/en/zos/2.2.0?topic=functions-iconv-code-conversion )
+        while ( inbytes > 0 )
+        {
+          size_t res = iconv(cd_,&inbuf,&inbytes,&outbuf,&outbytes);
+
+          if ( (size_t)-1 == res )
+          {
+            std::string errmsg;
+            if ( errno == EILSEQ )
+            {
+              errmsg = std::string("An invalid multibyte sequence is encountered in the input.");
+              throw std::runtime_error(errmsg);
+            }
+            else if ( errno == EINVAL )
+            {
+              errmsg = std::string("An incomplete multibyte sequence is encountered in the input")
+                     + std::string(" and the input byte sequence terminates after it.");
+            }
+            else if ( errno == E2BIG )
+            {
+              errmsg = std::string("The output buffer has no more room for the next converted character.");
+            }
+            throw std::runtime_error(errmsg);
+          }
+        }
+
+        std::string outstring(out_buffer.begin(),out_buffer.end()-outbytes);
+        astring = outstring;
+      }
+  };
+  #elif defined(__WIN32__) || defined(_WIN32)
+  class iconverter
+  {
+    public:
+      iconverter(std::string in_enc, std::string out_enc, size_t out_buffer_size = 1024) {}
+      void convert(std::string &astring) {}
+  };
+  #endif
+
   // channel
   struct channel
   {
@@ -301,9 +388,12 @@ namespace imc
         double secs_int;
         trigger_time_frac_secs_ = modf((double)secs,&secs_int);
         tms.tm_sec = (int)secs_int;
+        //tms.tm_isdst = -1;
 
         // generate std::chrono::system_clock::time_point type
-        std::time_t ts = std::mktime(&tms);
+        // ( - https://www.gnu.org/software/libc/manual/html_node/Broken_002ddown-Time.html
+        //   - https://man7.org/linux/man-pages/man3/tzset.3.html )
+        std::time_t ts = timegm(&tms); //std::mktime(&tms);
         trigger_time_ = std::chrono::system_clock::from_time_t(ts);
       }
 
@@ -313,21 +403,24 @@ namespace imc
       // calculate absolute trigger-time
       absolute_trigger_time_ = trigger_time_ + std::chrono::seconds(addtime_);
       //                                       + std::chrono::nanoseconds((long int)(trigger_time_frac_secs_*1.e9));
+
+      // convert any non-UTF-8 codepage to UTF-8
+      convert_encoding();
     }
 
     // convert buffer to actual datatype
     void convert_buffer()
     {
-      // TODO no clue how/if/when to handle buffer offset/mask/subsequent_bytes
-      // etc. and whatever that shit is!
       std::vector<imc::parameter> prms = blocks_->at(chnenv_.CSuuid_).get_parameters();
       if ( prms.size() < 4)
       {
         throw std::runtime_error("CS block is invalid and features to few parameters");
       }
+
+      // extract (channel dependent) part of buffer
       unsigned long int buffstrt = prms[3].begin();
-      std::vector<unsigned char> CSbuffer( buffer_->begin()+buffstrt+1,
+      std::vector<unsigned char> CSbuffer( buffer_->begin()+buffstrt+buffer_offset_+1,
-                                           buffer_->begin()+buffstrt+buffer_size_+1 );
+                                           buffer_->begin()+buffstrt+buffer_offset_+buffer_size_+1 );
 
       // determine number of values in buffer
       unsigned long int num_values = (unsigned long int)(CSbuffer.size()/(signbits_/8));
@@ -400,6 +493,33 @@ namespace imc
       }
     }
 
+    // convert any description, units etc. to UTF-8 (by default)
+    void convert_encoding()
+    {
+      if ( !codepage_.empty() )
+      {
+        // construct iconv-compatible name for respective codepage
+        std::string cpn = std::string("CP") + codepage_;
+
+        // set up converter
+        std::string utf = std::string("UTF-8");
+        iconverter conv(cpn,utf);
+
+        conv.convert(name_);
+        conv.convert(comment_);
+        conv.convert(origin_);
+        conv.convert(origin_comment_);
+        conv.convert(text_);
+        conv.convert(language_code_);
+        conv.convert(yname_);
+        conv.convert(yunit_);
+        conv.convert(xname_);
+        conv.convert(xunit_);
+        conv.convert(group_name_);
+        conv.convert(group_comment_);
+      }
+    }
+
     // get info string
     std::string get_info(int width = 20)
     {
@@ -412,9 +532,10 @@ namespace imc
         <<std::setw(width)<<std::left<<"name:"<<name_<<"\n"
         <<std::setw(width)<<std::left<<"comment:"<<comment_<<"\n"
         <<std::setw(width)<<std::left<<"origin:"<<origin_<<"\n"
+        <<std::setw(width)<<std::left<<"origin-comment:"<<origin_comment_<<"\n"
         <<std::setw(width)<<std::left<<"description:"<<text_<<"\n"
-        <<std::setw(width)<<std::left<<"trigger-time-nt:"<<std::put_time(std::localtime(&tt),"%FT%T")<<"\n"
+        <<std::setw(width)<<std::left<<"trigger-time-nt:"<<std::put_time(std::gmtime(&tt),"%FT%T")<<"\n"
-        <<std::setw(width)<<std::left<<"trigger-time:"<<std::put_time(std::localtime(&att),"%FT%T")<<"\n"
+        <<std::setw(width)<<std::left<<"trigger-time:"<<std::put_time(std::gmtime(&att),"%FT%T")<<"\n"
         <<std::setw(width)<<std::left<<"language-code:"<<language_code_<<"\n"
         <<std::setw(width)<<std::left<<"codepage:"<<codepage_<<"\n"
         <<std::setw(width)<<std::left<<"yname:"<<yname_<<"\n"
@@ -450,9 +571,10 @@ namespace imc
              <<"\",\"name\":\""<<name_
              <<"\",\"comment\":\""<<comment_
              <<"\",\"origin\":\""<<origin_
+             <<"\",\"origin-comment\":\""<<origin_comment_
              <<"\",\"description\":\""<<text_
-             <<"\",\"trigger-time-nt\":\""<<std::put_time(std::localtime(&tt),"%FT%T")
+             <<"\",\"trigger-time-nt\":\""<<std::put_time(std::gmtime(&tt),"%FT%T")
-             <<"\",\"trigger-time\":\""<<std::put_time(std::localtime(&att),"%FT%T")
+             <<"\",\"trigger-time\":\""<<std::put_time(std::gmtime(&att),"%FT%T")
              <<"\",\"language-code\":\""<<language_code_
              <<"\",\"codepage\":\""<<codepage_
              <<"\",\"yname\":\""<<prepjsonstr(yname_)

makefile

@@ -10,8 +10,11 @@ SRC = src/
 LIB = lib/
 PYT = python/
 
-# list headers
+# list headers and include directories
 HPP = $(wildcard $(LIB)/*.hpp)
+IPP = $(shell find $(LIB) -type f -name '*.hpp')
+KIB = $(shell find $(LIB) -type d)
+MIB = $(foreach dir,$(KIB),-I $(dir))
 
 # choose compiler and its options
 CC = g++ -std=c++17
@@ -36,12 +39,12 @@ $(EXE): check-tags $(GVSN) main.o
 	$(CC) $(OPT) main.o -o $@
 
 # build main.cpp and include git version/commit tag
-main.o: src/main.cpp $(HPP)
+main.o: src/main.cpp $(IPP)
 	@cp $< $<.cpp
 	@sed -i 's/TAGSTRING/$(GTAG)/g' $<.cpp
 	@sed -i 's/HASHSTRING/$(GHSH)/g' $<.cpp
 	@sed -i 's/TIMESTAMPSTRING/$(TMS)/g' $<.cpp
-	$(CC) -c $(OPT) -I $(LIB) $<.cpp -o $@
+	$(CC) -c $(OPT) $(MIB) $<.cpp -o $@
 	@rm $<.cpp
 
 install: $(EXE)

python/IMCtermite.pyx

@@ -5,6 +5,16 @@ from IMCtermite cimport cppimctermite
 
 import json as jn
 import decimal
+import platform
+
+# auxiliary function for codepage conversion
+def get_codepage(chn) :
+    if platform == 'Windows' :
+        chndec = jn.loads(chn.decode(errors="ignore"))
+        chncdp = chndec["codepage"]
+        return 'utf-8' if chncdp is None else chncdp
+    else :
+        return 'utf-8'
 
 cdef class imctermite:
 
@@ -22,7 +32,7 @@ cdef class imctermite:
   # get JSON list of channels
   def get_channels(self, bool include_data):
     chnlst = self.cppimc.get_channels(True,include_data)
-    chnlstjn = [jn.loads(chn.decode(errors="ignore")) for chn in chnlst]
+    chnlstjn = [jn.loads(chn.decode(get_codepage(chn),errors="ignore")) for chn in chnlst]
     return chnlstjn
 
   # print single channel/all channels

python/MANIFEST.in

@@ -2,3 +2,4 @@ include lib/*.hpp
 include *.cpp
 include *.pyx
 include *.pxd
+include VERSION

python/VERSION

@@ -1 +1 @@
-2.0.6
+2.0.20

python/examples/multichannel.py

@@ -0,0 +1,43 @@
+
+import IMCtermite
+import pandas
+import datetime
+
+def add_trigger_time(trigger_time, add_time) :
+    trgts = datetime.datetime.strptime(trigger_time,'%Y-%m-%dT%H:%M:%S')
+    dt = datetime.timedelta(seconds=add_time)
+    return (trgts + dt).strftime('%Y-%m-%dT%H:%M:%S:%f')
+
+if __name__ == "__main__" :
+
+    # read file and extract data
+    imctm = IMCtermite.imctermite(b"Measurement.raw")
+    chns = imctm.get_channels(True)
+    
+    # prepare abscissa
+    xcol = "time ["+chns[0]['xunit']+"]"
+    #xcol = "timestamp"
+    xsts = [add_trigger_time(chns[0]['trigger-time'],tm) for tm in chns[0]['xdata']]
+
+    # sort channels
+    chnnms = sorted([chn['name'] for chn in chns], reverse=False)
+    chnsdict = {}
+    for chn in chns :
+        chnsdict[chn['name']] = chn
+
+    # construct dataframe
+    df = pandas.DataFrame()
+    df[xcol] = pandas.Series(chns[0]['xdata'])
+    #df[xcol] = pandas.Series(xsts)
+    #for idx,chn in enumerate(chns) :
+    for chnnm in chnnms :
+        chn = chnsdict[chnnm]
+        #xcol = (chn['xname'] if chn['xname'] != '' else "x_"+str(idx))+" ["+chn['xunit']+"]"
+        #df[xcol] = pandas.Series(chn['xdata'])
+        ycol = chn['yname']+" ["+chn['yunit']+"]"
+        df[ycol] = pandas.Series(chn['ydata'])
+
+    # show entire dataframe and write file
+    print(df)
+    df.to_csv("Measurement.csv",header=True,sep='\t',index=False)
+

samples/exampleB.raw

@@ -8,7 +8,7 @@
 |CR,1,49,1,3.921568627450980E-2,0.000000000000000E+0,1,1,V;
 |ND,1,50,-1,-1,-1,0.000000000000000E+0,1.000000000000000E+1;|CN,1,17,1,0,0,6,kanal2,0,;
 |CS,1,8,1,leclass="Code Example" translate="true">
-|NO,1,37,1,9,                   imc-FAMOS,20,Erzeugt:E.Mustermann; |CG,1,5,2,2,2;
+|NO,1,37,1,9,imc-FAMOS,20,Erzeugt:E.Mustermann; |CG,1,5,2,2,2;
 |CD,1,31,1.000000000000000E-1,1,0,,0,0,0;|NT,1,27,10, 6,1993,19,18,20.0000000;
 |CC,1,3,1,1;|CP,1,16,1,4,7,32,0,0,1,0;|Cb,1,40,1,0,1,1,0,16,0,16,1,0.000000000000E+0,0,;
 |CR,1,11,0,0,0,1,1,V;|CN,1,20,0,0,0,9,E06_6_121,0,;|CC,1,3,2,1;|CP,1,16,2,4,5,32,0,0,1,0;