install libiconv in github workflow for matrix.os windows-2019

* convert to UTF-8 for any non-empty codepage: fix buffer string
  conversion

.github/setup.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-cat ../README.md | grep '^# IMCtermite' -A 50000 > ../python/README.md
-cp -r ../lib ../python/
-cp -v ../LICENSE ../python/

.github/workflows/pypi-deploy.yml

@@ -3,7 +3,8 @@ name: Build Python Wheels
 
 on:
   push:
-    branches: [master]
+    #branches: [master]
+    tags: ["v[0-9]+.[0-9]+.[0-9]+"]
 
 jobs:
 
@@ -11,10 +12,14 @@ jobs:
     name: Prepare environment for wheel builds
     runs-on: ubuntu-20.04
     steps:
-      - name: Prepare files
+      - uses: actions/checkout@v2
-        run: |
+      - name: Prepare wheel build
-          chmod +x "${GITHUB_WORKSPACE}/.github/setup.sh"
+        run: make -C python/ setup
-          "${GITHUB_WORKSPACE}/.github/setup.sh"
+      - name: Store wheel configuration files
+        uses: actions/upload-artifact@v2
+        with:
+          name: wheel-config
+          path: python/
 
   build_wheels:
     name: Build binary wheels on ${{ matrix.os }}
@@ -22,20 +27,39 @@ jobs:
     needs: [build_setup]
     strategy:
       matrix:
-        os: [ubuntu-20.04, windows-2019, macOS-10.15]
+        os: [ubuntu-20.04, windows-2019]
-        arch: [auto32, auto64, aarch64, ppc64le, s390x]
+        arch: [auto32, auto64, aarch64]
 
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
       - name: Install cibuildwheel
         run: python -m pip install cibuildwheel==2.1.2
-        working-directory: python/
+      - name: Get wheel configuration files
+        uses: actions/download-artifact@v2
+        with:
+          name: wheel-config
+          path: python/
+      - name: Install iconv for Windows
+        if: matrix.os == 'windows-2019'
+        steps:
+          - name: Get vcpkg repo
+            run: git clone https://github.com/Microsoft/vcpkg.git
+          - name: Install vcpkg
+            run: .\vcpkg\bootstrap-vcpkg.bat
+            working-directory: vcpkg/
+          - name: Install required package libiconv
+            run: vcpkg install libiconv
       - name: Build wheels
         run: python -m cibuildwheel --output-dir wheelhouse
         working-directory: python/
         # env:
         #   CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* cp310-*
+      - name: Store binary wheels
+        uses: actions/upload-artifact@v2
+        with:
+          name: binary-wheels
+          path: python/wheelhouse/*.whl
 
   build_sdist:
     name: Build source distribution
@@ -45,9 +69,20 @@ jobs:
       - uses: actions/checkout@v2
       - name: Install cython
         run: python -m pip install cython==0.29.24
+      - name: Get wheel configuration files
+        uses: actions/download-artifact@v2
+        with:
+          name: wheel-config
+          path: python/
       - name: Build sdist
         run: python setup.py sdist
         working-directory: python/
+      - name: Store source wheels
+        uses: actions/upload-artifact@v2
+        with:
+          name: source-wheels
+          path: python/dist/*.tar.gz
+
 
   upload_pypi:
     name: Upload binary wheels to PyPI
@@ -55,6 +90,17 @@ jobs:
     needs: [build_wheels, build_sdist]
 
     steps:
+      - name: Get source wheels
+        uses: actions/download-artifact@v2
+        with:
+          name: source-wheels
+          path: dist/
+      - name: Get binary wheels
+        uses: actions/download-artifact@v2
+        with:
+          name: binary-wheels
+          path: dist/
+
       - uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__

.gitignore

@@ -34,6 +34,7 @@ pip/dist/
 pip/build/
 
 python/README.md
+python/README.rst
 python/LICENSE
 python/build
 python/*.egg-info

README.md

@@ -31,6 +31,8 @@ On the [Record Evolution Platform](https://www.record-evolution.de/en/home-en/),
 
 ## File format
 
+[Warning: Take a look at [this issue](https://github.com/RecordEvolution/IMCtermite/issues/14) when reading this section regarding the file format.]
+
 A data file of the _IMC Bus Format_ type with the extension _.raw_ is a _mixed text/binary
 file_ featuring a set of markers (keys) that indicate the start of various blocks
 of data that provide meta information and the actual measurement data. Every single
@@ -150,8 +152,11 @@ To install the latest version simply do
 python3 -m pip install IMCtermite
 ```
 
-Note, that _python3_setuptools_ and _gcc version >= 10.2.0_ are required to
+which provides binary wheels for multiple architectures on _Windows_ and _Linux_
-successfully install and use it.
+and most _Python 3.x_ distributions. However, if your platform/architecture is
+not supported you can still compile the source distribution yourself, which
+requires _python3_setuptools_ and an up-to-date compiler supporting C++11
+standard (e.g. _gcc version >= 10.2.0_).
 
 ## Usage
 
@@ -187,23 +192,23 @@ options `imctermite sample-data.raw -b -c -s '|'`.
 
 ### Python
 
-Given the `imctermite` module is available, we can import it and declare an instance
+Given the `IMCtermite` module is available, we can import it and declare an instance
 of it by passing a _raw_ file to the constructor:
 
 ```Python
-import imc_termite
+import IMCtermite
 
-imcraw = imc_termite.imctermite(b"sample/sampleA.raw")
+imcraw = IMCtermite.imctermite(b"sample/sampleA.raw")
 ```
 
 An example of how to create an instance and obtain the list of channels is:
 
 ```Python
-import imc_termite
+import IMCtermite
 
 # declare and initialize instance of "imctermite" by passing a raw-file
 try :
-    imcraw = imc_termite.imctermite(b"samples/sampleA.raw")
+    imcraw = IMCtermite.imctermite(b"samples/sampleA.raw")
 except RuntimeError as e :
     print("failed to load/parse raw-file: " + str(e))
 

lib/imc_channel.hpp

@@ -9,6 +9,8 @@
 #include <math.h>
 #include <chrono>
 #include <ctime>
+#include <time.h>
+#include <iconv.h>
 
 //---------------------------------------------------------------------------//
 
@@ -142,6 +144,78 @@ namespace imc
     return sumstr;
   }
 
+  // convert encoding of any descriptions, channel-names, units etc.
+  class iconverter
+  {
+      std::string in_enc_, out_enc_;
+      iconv_t cd_;
+      size_t out_buffer_size_;
+
+    public:
+
+      iconverter(std::string in_enc, std::string out_enc, size_t out_buffer_size = 1024) :
+        in_enc_(in_enc), out_enc_(out_enc), out_buffer_size_(out_buffer_size)
+      {
+        // allocate descriptor for character set conversion
+        // (https://man7.org/linux/man-pages/man3/iconv_open.3.html)
+        cd_ = iconv_open(out_enc.c_str(), in_enc.c_str());
+
+        if ( (iconv_t)-1 == cd_ )
+        {
+          if ( errno == EINVAL )
+          {
+            std::string errmsg = std::string("The encoding conversion from ") + in_enc
+              + std::string(" to ") + out_enc + std::string(" is not supported by the implementation.");
+            throw std::runtime_error(errmsg);
+          }
+        }
+      }
+
+      void convert(std::string &astring)
+      {
+        if ( astring.empty() ) return;
+
+        std::vector<char> in_buffer(astring.begin(),astring.end());
+        char *inbuf = &in_buffer[0];
+        size_t inbytes = in_buffer.size();
+
+        std::vector<char> out_buffer(out_buffer_size_);
+        char *outbuf = &out_buffer[0];
+        size_t outbytes = out_buffer.size();
+
+        // perform character set conversion
+        // ( - https://man7.org/linux/man-pages/man3/iconv.3.html
+        //   - https://www.ibm.com/docs/en/zos/2.2.0?topic=functions-iconv-code-conversion )
+        while ( inbytes > 0 )
+        {
+          size_t res = iconv(cd_,&inbuf,&inbytes,&outbuf,&outbytes);
+
+          if ( (size_t)-1 == res )
+          {
+            std::string errmsg;
+            if ( errno == EILSEQ )
+            {
+              errmsg = std::string("An invalid multibyte sequence is encountered in the input.");
+              throw std::runtime_error(errmsg);
+            }
+            else if ( errno == EINVAL )
+            {
+              errmsg = std::string("An incomplete multibyte sequence is encountered in the input")
+                     + std::string(" and the input byte sequence terminates after it.");
+            }
+            else if ( errno == E2BIG )
+            {
+              errmsg = std::string("The output buffer has no more room for the next converted character.");
+            }
+            throw std::runtime_error(errmsg);
+          }
+        }
+
+        std::string outstring(out_buffer.begin(),out_buffer.end()-outbytes);
+        astring = outstring;
+      }
+  };
+
   // channel
   struct channel
   {
@@ -301,9 +375,12 @@ namespace imc
         double secs_int;
         trigger_time_frac_secs_ = modf((double)secs,&secs_int);
         tms.tm_sec = (int)secs_int;
+        //tms.tm_isdst = -1;
 
         // generate std::chrono::system_clock::time_point type
-        std::time_t ts = std::mktime(&tms);
+        // ( - https://www.gnu.org/software/libc/manual/html_node/Broken_002ddown-Time.html
+        //   - https://man7.org/linux/man-pages/man3/tzset.3.html )
+        std::time_t ts = timegm(&tms); //std::mktime(&tms);
         trigger_time_ = std::chrono::system_clock::from_time_t(ts);
       }
 
@@ -313,21 +390,24 @@ namespace imc
       // calculate absolute trigger-time
       absolute_trigger_time_ = trigger_time_ + std::chrono::seconds(addtime_);
       //                                       + std::chrono::nanoseconds((long int)(trigger_time_frac_secs_*1.e9));
+
+      // convert any non-UTF-8 codepage to UTF-8
+      convert_encoding();
     }
 
     // convert buffer to actual datatype
     void convert_buffer()
     {
-      // TODO no clue how/if/when to handle buffer offset/mask/subsequent_bytes
-      // etc. and whatever that shit is!
       std::vector<imc::parameter> prms = blocks_->at(chnenv_.CSuuid_).get_parameters();
       if ( prms.size() < 4)
       {
         throw std::runtime_error("CS block is invalid and features to few parameters");
       }
+
+      // extract (channel dependent) part of buffer
       unsigned long int buffstrt = prms[3].begin();
-      std::vector<unsigned char> CSbuffer( buffer_->begin()+buffstrt+1,
+      std::vector<unsigned char> CSbuffer( buffer_->begin()+buffstrt+buffer_offset_+1,
-                                           buffer_->begin()+buffstrt+buffer_size_+1 );
+                                           buffer_->begin()+buffstrt+buffer_offset_+buffer_size_+1 );
 
       // determine number of values in buffer
       unsigned long int num_values = (unsigned long int)(CSbuffer.size()/(signbits_/8));
@@ -400,6 +480,33 @@ namespace imc
       }
     }
 
+    // convert any description, units etc. to UTF-8 (by default)
+    void convert_encoding()
+    {
+      if ( !codepage_.empty() )
+      {
+        // construct iconv-compatible name for respective codepage
+        std::string cpn = std::string("CP") + codepage_;
+
+        // set up converter
+        std::string utf = std::string("UTF-8");
+        iconverter conv(cpn,utf);
+
+        conv.convert(name_);
+        conv.convert(comment_);
+        conv.convert(origin_);
+        conv.convert(origin_comment_);
+        conv.convert(text_);
+        conv.convert(language_code_);
+        conv.convert(yname_);
+        conv.convert(yunit_);
+        conv.convert(xname_);
+        conv.convert(xunit_);
+        conv.convert(group_name_);
+        conv.convert(group_comment_);
+      }
+    }
+
     // get info string
     std::string get_info(int width = 20)
     {
@@ -413,8 +520,8 @@ namespace imc
         <<std::setw(width)<<std::left<<"comment:"<<comment_<<"\n"
         <<std::setw(width)<<std::left<<"origin:"<<origin_<<"\n"
         <<std::setw(width)<<std::left<<"description:"<<text_<<"\n"
-        <<std::setw(width)<<std::left<<"trigger-time-nt:"<<std::put_time(std::localtime(&tt),"%FT%T")<<"\n"
+        <<std::setw(width)<<std::left<<"trigger-time-nt:"<<std::put_time(std::gmtime(&tt),"%FT%T")<<"\n"
-        <<std::setw(width)<<std::left<<"trigger-time:"<<std::put_time(std::localtime(&att),"%FT%T")<<"\n"
+        <<std::setw(width)<<std::left<<"trigger-time:"<<std::put_time(std::gmtime(&att),"%FT%T")<<"\n"
         <<std::setw(width)<<std::left<<"language-code:"<<language_code_<<"\n"
         <<std::setw(width)<<std::left<<"codepage:"<<codepage_<<"\n"
         <<std::setw(width)<<std::left<<"yname:"<<yname_<<"\n"
@@ -451,16 +558,16 @@ namespace imc
              <<"\",\"comment\":\""<<comment_
              <<"\",\"origin\":\""<<origin_
              <<"\",\"description\":\""<<text_
-             <<"\",\"trigger-time-nt\":\""<<std::put_time(std::localtime(&tt),"%FT%T")
+             <<"\",\"trigger-time-nt\":\""<<std::put_time(std::gmtime(&tt),"%FT%T")
-             <<"\",\"trigger-time\":\""<<std::put_time(std::localtime(&att),"%FT%T")
+             <<"\",\"trigger-time\":\""<<std::put_time(std::gmtime(&att),"%FT%T")
              <<"\",\"language-code\":\""<<language_code_
              <<"\",\"codepage\":\""<<codepage_
-             <<"\",\"yname\":\""<<yname_
+             <<"\",\"yname\":\""<<prepjsonstr(yname_)
-             <<"\",\"yunit\":\""<<yunit_
+             <<"\",\"yunit\":\""<<prepjsonstr(yunit_)
              <<"\",\"significantbits\":\""<<signbits_
              <<"\",\"addtime\":\""<<addtime_
-             <<"\",\"xname\":\""<<xname_
+             <<"\",\"xname\":\""<<prepjsonstr(xname_)
-             <<"\",\"xunit\":\""<<xunit_
+             <<"\",\"xunit\":\""<<prepjsonstr(xunit_)
              <<"\",\"xstepwidth\":\""<<xstepwidth_
              <<"\",\"xoffset\":\""<<xoffset_
              <<"\",\"group\":{"<<"\"index\":\""<<group_index_
@@ -477,6 +584,25 @@ namespace imc
       return ss.str();
     }
 
+    // prepare string value for usage in JSON dump
+    std::string prepjsonstr(std::string value)
+    {
+      std::stringstream ss;
+      ss<<quoted(value);
+      return strip_quotes(ss.str());
+    }
+
+    // remove any leading or trailing double quotes
+    std::string strip_quotes(std::string astring)
+    {
+      // head
+      if ( astring.front() == '"' ) astring.erase(astring.begin()+0);
+      // tail
+      if ( astring.back() == '"' ) astring.erase(astring.end()-1);
+
+      return astring;
+    }
+
     // print channel
     void print(std::string filename, const char sep = ',', int width = 25, int yprec = 9)
     {

lib/imc_key.hpp

@@ -84,6 +84,7 @@ namespace imc
     // noncritical keys
     key(false,"NO","origin of data",1),
     key(false,"NT","timestamp of trigger",1),
+    key(false,"NT","timestamp of trigger",2),
     key(false,"ND","(color) display properties",1),
     key(false,"NU","user defined key",1),
     key(false,"Np","property of channel",1),

lib/imc_raw.hpp

@@ -236,13 +236,27 @@ namespace imc
             // provide UUID for channel
             chnenv.uuid_ = chnenv.CNuuid_;
 
+            // for multichannel data there may be multiple channels referring to
+            // the same (final) CS block (in contrast to what the IMC software
+            // documentation seems to suggest) resulting in all channels missing
+            // a CS block except for the very last
+            if ( chnenv.CSuuid_.empty() ) {
+              for ( imc::block blkCS: rawblocks_ ) {
+                if ( blkCS.get_key().name_ == "CS"
+                  && blkCS.get_begin() > (unsigned long int)stol(chnenv.uuid_) ) {
+                  chnenv.CSuuid_ = blkCS.get_uuid();
+                }
+              }
+            }
+
             // create channel object and add it to the map of channels
             channels_.insert( std::pair<std::string,imc::channel>
               (chnenv.CNuuid_,imc::channel(chnenv,&mapblocks_,&buffer_))
             );
 
             // reset channel uuid
-            chnenv.CNuuid_.clear();
+            chnenv.reset();
+            //chnenv.CNuuid_.clear();
           }
         }
 
@@ -254,7 +268,6 @@ namespace imc
       }
     }
 
-
   public:
 
     // provide buffer size

makefile

@@ -3,15 +3,18 @@
 SHELL := /bin/bash
 
 # name of executable and CLI tool
-EXE = IMCtermite
+EXE = imctermite
 
 # directory names
 SRC = src/
 LIB = lib/
 PYT = python/
 
-# list headers
+# list headers and include directories
 HPP = $(wildcard $(LIB)/*.hpp)
+IPP = $(shell find $(LIB) -type f -name '*.hpp')
+KIB = $(shell find $(LIB) -type d)
+MIB = $(foreach dir,$(KIB),-I $(dir))
 
 # choose compiler and its options
 CC = g++ -std=c++17
@@ -31,17 +34,17 @@ INST := /usr/local/bin
 #-----------------------------------------------------------------------------#
 # C++ and CLI tool
 
-# build exectuable
+# build executable
 $(EXE): check-tags $(GVSN) main.o
 	$(CC) $(OPT) main.o -o $@
 
 # build main.cpp and include git version/commit tag
-main.o: src/main.cpp $(HPP)
+main.o: src/main.cpp $(IPP)
 	@cp $< $<.cpp
 	@sed -i 's/TAGSTRING/$(GTAG)/g' $<.cpp
 	@sed -i 's/HASHSTRING/$(GHSH)/g' $<.cpp
 	@sed -i 's/TIMESTAMPSTRING/$(TMS)/g' $<.cpp
-	$(CC) -c $(OPT) -I $(LIB) $<.cpp -o $@
+	$(CC) -c $(OPT) $(MIB) $<.cpp -o $@
 	@rm $<.cpp
 
 install: $(EXE)
@@ -91,6 +94,9 @@ python-clean:
 	make -C python/ clean
 	rm -vf IMCtermite*.so
 
+python-test:
+	PYTHONPATH=./ python python/examples/usage.py
+
 #-----------------------------------------------------------------------------#
 # clean
 

python/IMCtermite.pyx

@@ -20,8 +20,8 @@ cdef class imctermite:
     self.cppimc.set_file(rawfile)
 
   # get JSON list of channels
-  def get_channels(self, bool data):
+  def get_channels(self, bool include_data):
-    chnlst = self.cppimc.get_channels(True,data)
+    chnlst = self.cppimc.get_channels(True,include_data)
     chnlstjn = [jn.loads(chn.decode(errors="ignore")) for chn in chnlst]
     return chnlstjn
 

python/MANIFEST.in

@@ -2,3 +2,4 @@ include lib/*.hpp
 include *.cpp
 include *.pyx
 include *.pxd
+include VERSION

python/VERSION

@@ -1 +1 @@
-2.0.0
+2.0.9

python/examples/multichannel.py

@@ -0,0 +1,43 @@
+
+import IMCtermite
+import pandas
+import datetime
+
+def add_trigger_time(trigger_time, add_time) :
+    trgts = datetime.datetime.strptime(trigger_time,'%Y-%m-%dT%H:%M:%S')
+    dt = datetime.timedelta(seconds=add_time)
+    return (trgts + dt).strftime('%Y-%m-%dT%H:%M:%S:%f')
+
+if __name__ == "__main__" :
+
+    # read file and extract data
+    imctm = IMCtermite.imctermite(b"Measurement.raw")
+    chns = imctm.get_channels(True)
+    
+    # prepare abscissa
+    xcol = "time ["+chns[0]['xunit']+"]"
+    #xcol = "timestamp"
+    xsts = [add_trigger_time(chns[0]['trigger-time'],tm) for tm in chns[0]['xdata']]
+
+    # sort channels
+    chnnms = sorted([chn['name'] for chn in chns], reverse=False)
+    chnsdict = {}
+    for chn in chns :
+        chnsdict[chn['name']] = chn
+
+    # construct dataframe
+    df = pandas.DataFrame()
+    df[xcol] = pandas.Series(chns[0]['xdata'])
+    #df[xcol] = pandas.Series(xsts)
+    #for idx,chn in enumerate(chns) :
+    for chnnm in chnnms :
+        chn = chnsdict[chnnm]
+        #xcol = (chn['xname'] if chn['xname'] != '' else "x_"+str(idx))+" ["+chn['xunit']+"]"
+        #df[xcol] = pandas.Series(chn['xdata'])
+        ycol = chn['yname']+" ["+chn['yunit']+"]"
+        df[ycol] = pandas.Series(chn['ydata'])
+
+    # show entire dataframe and write file
+    print(df)
+    df.to_csv("Measurement.csv",header=True,sep='\t',index=False)
+

python/examples/usage.py

@@ -21,18 +21,18 @@ if len(channelsdata) > 0 :
 
 print(len(chnydata))
 print(len(chnxdata))
+print()
 
 # print the channels into a specific directory
-imcraw.print_channels(b"./data",ord(','))
+imcraw.print_channels(b"/tmp/",ord(','))
 
 # print all channels separately
-idx = 0
+for i,chn in enumerate(channels) :
-for chn in channels :
+    print(str(i)+" : "+chn['name']+" : "+chn['uuid'])
-    print(str(idx)+" : "+chn['name']+" : "+chn['uuid'])
+    filname = os.path.join("/tmp/",str(i) + "_" + chn['name']+".csv")
-    filname = os.path.join("./data",str(idx) + "_" + chn['name']+".csv")
     print(filname)
     imcraw.print_channel(chn['uuid'].encode(),filname.encode(),ord(','))
-    idx = idx + 1
 
 # print all channels in single file
-# imcraw.print_table(b"./data/allchannels.csv")
+imcraw.print_table(b"/tmp/allchannels.csv")
+

python/makefile

@@ -1,11 +1,13 @@
 
 setup:
 	cat ../README.md | grep '^# IMCtermite' -A 50000 > ./README.md
+	#pandoc -f markdown -t rst -o README.rst README.md
+	#python -m rstvalidator README.rst
 	cp -r ../lib ./
 	cp -v ../LICENSE ./
 
 setup-clean:
-	rm -vf README.md LICENSE
+	rm -vf README.md README.rst LICENSE
 	rm -rf lib/
 
 build: setup
@@ -16,9 +18,11 @@ build-inplace: setup
 
 build-sdist: setup
 	python setup.py sdist
+	python -m twine check dist/*
 
 build-bdist: setup
 	python setup.py bdist
+	python -m twine check dist/*
 
 build-clean:
 	python setup.py clean --all

python/setup.cfg

@@ -1,8 +1,10 @@
 
 [metadata]
 name = IMCtermite
-description = Enables extraction of measurement data from binary files with extension 'raw' used by proprietary software imcFAMOS/imcSTUDIO and facilitates its storage in open source file formats
+description = Enables extraction of measurement data from binary files with extension 'raw' used by proprietary software imcFAMOS and imcSTUDIO and facilitates its storage in open source file formats
 long_description = file: README.md
+# long_description_content_type = text/x-rst
+long_description_content_type = text/markdown
 version = file: VERSION
 author = Record Evolution GmbH
 author_email = mario.fink@record-evolution.de
@@ -12,8 +14,8 @@ license = MIT License
 license_files = LICENSE
 keywords = IMC, raw, imcFAMOS, imcSTUDIO, imcCRONOS
 classifiers =
-  Programming Language :: Python :: 3,
+  Programming Language :: Python :: 3
-  License :: OSI Approved :: MIT License,
+  License :: OSI Approved :: MIT License
   Operating System :: OS Independent
   Topic :: Scientific/Engineering
   Topic :: Software Development :: Libraries :: Python Modules

python/setup.py

@@ -1,6 +1,5 @@
 from setuptools import Extension, setup
 from Cython.Build import cythonize
-import os
 import sys
 
 print("building on platform: "+sys.platform)