README.md: fix typo

* imc_object: asc_time, localtime: make threadsafe
* imc_datatype: satisfy 'rule of two'
* python: remove all unused imports

Dockerfile

@@ -4,8 +4,9 @@ FROM debian:bullseye-20210111
 USER root
 
 RUN apt-get update && apt-get install -y \
-    build-essential git \
+    build-essential git vim \
     python3 python3-pip
+RUN python3 -m pip install cython
 
 RUN g++ -v
 
@@ -15,6 +16,6 @@ COPY ./ /IMCtermite/
 RUN cd /IMCtermite && ls -lh && make install && ls -lh /usr/local/bin/imctermite
 
 # install Python module
-# RUN cd /tdm_ripper && ls -lh && make cython-requirements && make cython-install
+RUN cd /IMCtermite && ls -lh && make cython-install
 
-CMD ["sleep","inifity"]
+CMD ["sleep","infinity"]

README.md

@@ -1,13 +1,11 @@
 
-<p align="center">
+[![Total alerts](https://img.shields.io/lgtm/alerts/g/RecordEvolution/IMCtermite.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/RecordEvolution/IMCtermite/alerts/)
-  <a href="https://record-evolution.de/reswarm">
+[![Language grade: C/C++](https://img.shields.io/lgtm/grade/cpp/g/RecordEvolution/IMCtermite.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/RecordEvolution/IMCtermite/context:cpp)
-    <img
+[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/RecordEvolution/IMCtermite.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/RecordEvolution/IMCtermite/context:python)
-      alt="imctermite.svg"
+[![LICENSE](https://img.shields.io/github/license/RecordEvolution/IMCtermite)](https://img.shields.io/github/license/RecordEvolution/IMCtermite)
-      src="assets/imctermite.svg"
+[![STARS](https://img.shields.io/github/stars/RecordEvolution/IMCtermite)](https://img.shields.io/github/stars/RecordEvolution/IMCtermite)
-      width="400"
+
-    />
+# IMCtermite
-  </a>
-</p>
 
 _IMCtermite_ provides access to the proprietary data format
 _IMC Bus Format_ with the file extension _.raw_ introduced and developed by
@@ -219,3 +217,5 @@ in the Python folder.
 - https://www.imc-tm.de/produkte/messtechnik-hardware/imc-cronosflex/ueberblick/
 - https://cython.readthedocs.io/en/latest/src/userguide/wrapping_CPlusPlus.html
 - https://github.com/Apollo3zehn/ImcFamosFile
+- https://pypi.org/help/#apitoken
+

cython/imc_termite.pxd

@@ -16,3 +16,4 @@ cdef extern from "imc_raw.hpp" namespace "imc":
     vector[string] get_channels(bool json, bool data) except +
     # print all channels
     void print_channels(string outputdir) except +
+    void print_table(string outputfile) except +

cython/py_imc_termite.pyx

@@ -3,6 +3,7 @@
 from imc_termite cimport imc_termite
 
 import json as jn
+import decimal
 # import numpy as np
 
 cdef class imctermite:
@@ -21,9 +22,21 @@ cdef class imctermite:
   # get JSON list of channels
   def get_channels(self, bool data):
     chnlst = self.cpp_imc.get_channels(True,data)
-    chnlstjn = [jn.loads(chn.decode()) for chn in chnlst]
+    chnlstjn = [jn.loads(chn.decode(errors="ignore")) for chn in chnlst]
     return chnlstjn
 
-  # print a channels
+  # print channels
   def print_channels(self, string outputdir):
     self.cpp_imc.print_channels(outputdir)
+
+  # print table including channels
+  def print_table(self, string outputfile):
+    chnlst = self.cpp_imc.get_channels(True,True)
+    chnlstjn = [jn.loads(chn.decode(errors="ignore")) for chn in chnlst]
+    with open(outputfile.decode(),'w') as fout:
+      for chn in chnlstjn:
+        fout.write('#' +str(chn['xname']).rjust(19)+str(chn['yname']).rjust(20)+'\n')
+        fout.write('#'+str(chn['xunit']).rjust(19)+str(chn['yunit']).rjust(20)+'\n')
+        for n in range(0,len(chn['ydata'])):
+          fout.write(str(chn['xdata'][n]).rjust(20)+
+                     str(chn['ydata'][n]).rjust(20)+'\n')

cython/setup.py

@@ -14,11 +14,11 @@ extensions = Extension(
 )
 
 setup(
-    version='1.2.2',
+    name="imc_termite",
+    version='1.2.4',
     description='IMCtermite cython extension',
     author='Record Evolution GmbH',
     author_email='mario.fink@record-evolution.de',
     url='https://github.com/RecordEvolution/IMCtermite.git',
-    name="imc_termite",
     ext_modules=cythonize(extensions,force=True)
 )

lib/imc_channel.hpp

@@ -95,7 +95,7 @@ namespace imc
     ss<<"[";
     if ( myvec.size() <= limit )
     {
-      for ( dt el: myvec ) ss<<el<<",";
+      for ( dt el: myvec ) ss<<std::setprecision(10)<<el<<",";
     }
     else
     {
@@ -104,7 +104,7 @@ namespace imc
       ss<<"...";
       for ( unsigned long int i = myvec.size()-heals; i < myvec.size(); i++ )
       {
-        ss<<myvec[i]<<",";
+        ss<<std::setprecision(10)<<myvec[i]<<",";
       }
     }
     std::string sumstr = ss.str();
@@ -147,9 +147,11 @@ namespace imc
     std::string group_uuid_, group_name_, group_comment_;
 
     // constructor takes channel's block environment
-    channel(channel_env chnenv, std::map<std::string,imc::block>* blocks,
+    channel(channel_env &chnenv, std::map<std::string,imc::block>* blocks,
-                                std::vector<unsigned char>* buffer):
+                                 std::vector<unsigned char>* buffer):
-      chnenv_(chnenv), blocks_(blocks), buffer_(buffer), group_index_(-1)
+      chnenv_(chnenv), blocks_(blocks), buffer_(buffer),
+      factor_(1.), offset_(0.),
+      group_index_(-1)
     {
       // declare list of block parameters
       std::vector<imc::parameter> prms;
@@ -292,6 +294,11 @@ namespace imc
       {
         imc::convert_data_to_type<imc_double>(CSbuffer,ydata_);
       }
+      // ...
+      else if ( datatp_ == 11 )
+      {
+        imc::convert_data_to_type<imc_digital>(CSbuffer,ydata_);
+      }
       else
       {
         throw std::runtime_error(std::string("unsupported/unknown datatype") + std::to_string(datatp_));

lib/imc_conversion.hpp

@@ -17,7 +17,11 @@ namespace imc
     // check number of elements of type "datatype" in buffer
     if ( subbuffer.size() != channel.size()*sizeof(datatype) )
     {
-      throw std::runtime_error("size mismatch between subbuffer and datatype");
+      throw std::runtime_error( std::string("size mismatch between subbuffer (")
+                              + std::to_string(subbuffer.size())
+                              + std::string(") and datatype (")
+                              + std::to_string(channel.size()) + std::string("*")
+                              + std::to_string(sizeof(datatype)) + std::string(")") );
     }
 
     // extract every single number of type "datatype" from buffer
@@ -36,6 +40,8 @@ namespace imc
       // save number in channel
       channel[i] = df;
     }
+
+    // for ( auto el: channel ) std::cout<<el<<"\n";
   }
 
 }

lib/imc_datatype.hpp

@@ -10,31 +10,47 @@ namespace imc
   // map datatype to machine's datatypes
   typedef unsigned char imc_Ubyte;
   typedef signed char imc_Sbyte;
+  //
   typedef unsigned short imc_Ushort;
   typedef signed short imc_Sshort;
+  //
+  // e.g. ARM Cortex-A72 armv7l gcc version 10.2.0 (Ubuntu 10.2.0-13ubuntu1)
+  // #ifdef __arm__
   typedef unsigned long int imc_Ulongint;
   typedef signed long int imc_Slongint;
+  // e.g. Intel(R) Core(TM) i7-3520M CPU @ 2.90GHz x86_64 gcc version 10.2.0 (Ubuntu 10.2.0-13ubuntu1)
+  // #ifdef i386 __i386 __i386__
+  // typedef unsigned int imc_Ulongint;
+  // typedef signed int imc_Slongint;
+  //
   typedef float imc_float;
   typedef double imc_double;
-  // TODO remaining types are not yet supported
+  //
+  // TODO not all remaining types are supported yet
+  // typedef <whatever that is ->... > "imc Devices Transitional Recording"
+  // typedf <sometimestamptype>  "Timestamp Ascii"
+  typedef char16_t imc_digital;
+  // typedef <  > imc_sixbyte "6byte unsigned long"
 
   class datatype
   {
   protected:
-    imc_Ubyte ubyte_;     // 0
+    imc_Ubyte ubyte_;      // 0
-    imc_Sbyte sbyte_;     // 1
+    imc_Sbyte sbyte_;      // 1
-    imc_Ushort ushort_;   // 2
+    imc_Ushort ushort_;    // 2
-    imc_Sshort sshort_;   // 3
+    imc_Sshort sshort_;    // 3
-    imc_Ulongint ulint_;  // 4
+    imc_Ulongint ulint_;   // 4
-    imc_Slongint slint_;  // 5
+    imc_Slongint slint_;   // 5
-    imc_float sfloat_;    // 6
+    imc_float sfloat_;     // 6
-    imc_double sdouble_;  // 7
+    imc_double sdouble_;   // 7
-    short int dtidx_;     // \in \{0,...,7\}
+    imc_digital sdigital_; // 10
+    short int dtidx_;      // \in \{0,...,7,10\}
   public:
     datatype(): ubyte_(0), sbyte_(0),
                 ushort_(0), sshort_(0),
                 ulint_(0.0), slint_(0.0),
                 sfloat_(0.0), sdouble_(0.0),
+                sdigital_(0),
                 dtidx_(0) { };
     // every supported datatype gets its own constructor
     datatype(imc_Ubyte num): ubyte_(num), dtidx_(0) {};
@@ -46,11 +62,29 @@ namespace imc
     datatype(imc_float num): sfloat_(num), dtidx_(6) {};
     datatype(imc_double num): ubyte_(0), sbyte_(0), ushort_(0), sshort_(0),
                               ulint_(0.0), slint_(0.0), sfloat_(0.0), sdouble_(num),
-                              dtidx_(7) {};
+                              sdigital_(0), dtidx_(7) {};
+    datatype(imc_digital num): ubyte_(0), sbyte_(0), ushort_(0), sshort_(0),
+                               ulint_(0.0), slint_(0.0), sfloat_(0.0), sdouble_(num),
+                               sdigital_(num), dtidx_(10) {};
 
     // identify type
     short int& dtype() { return dtidx_; }
 
+    // copy constructor
+    datatype(const datatype &num)
+    {
+      this->ubyte_ = num.ubyte_;
+      this->sbyte_ = num.sbyte_;
+      this->ushort_ = num.ushort_;
+      this->sshort_ = num.sshort_;
+      this->ulint_ = num.ulint_;
+      this->slint_ = num.slint_;
+      this->sfloat_ = num.sfloat_;
+      this->sdouble_ = num.sdouble_;
+      this->sdigital_ = num.sdigital_;
+      this->dtidx_ = num.dtidx_;
+    }
+
     // overall assignment operator
     datatype& operator=(const datatype &num)
     {
@@ -64,6 +98,7 @@ namespace imc
         this->slint_ = num.slint_;
         this->sfloat_ = num.sfloat_;
         this->sdouble_ = num.sdouble_;
+        this->sdigital_ = num.sdigital_;
         this->dtidx_ = num.dtidx_;
       }
 
@@ -119,6 +154,12 @@ namespace imc
       this->dtidx_ = 7;
       return *this;
     }
+    datatype& operator=(const imc_digital &num)
+    {
+      this->sdigital_ = num;
+      this->dtidx_ = 10;
+      return *this;
+    }
 
     // obtain number as double
     double as_double()
@@ -132,6 +173,7 @@ namespace imc
       else if ( dtidx_ == 5 ) num = (double)slint_;
       else if ( dtidx_ == 6 ) num = (double)sfloat_;
       else if ( dtidx_ == 7 ) num = (double)sdouble_;
+      else if ( dtidx_ == 10 ) num = static_cast<double>(sdigital_);
       return num;
     }
 
@@ -146,6 +188,7 @@ namespace imc
       else if ( num.dtidx_ == 5 ) out<<num.slint_;
       else if ( num.dtidx_ == 6 ) out<<num.sfloat_;
       else if ( num.dtidx_ == 7 ) out<<num.sdouble_;
+      else if ( num.dtidx_ == 10 ) out<<static_cast<double>(num.sdigital_);
       return out;
     }
 

lib/imc_object.hpp

@@ -501,16 +501,16 @@ namespace imc
       second_ = std::stod( get_parameter(buffer,&parameters[7]) );
 
       time_t rawtime;
-      struct tm* ts;
+      struct tm ts;
       time(&rawtime);
-      ts = localtime(&rawtime);
+      localtime_r(&rawtime,&ts);
-      ts->tm_mday = day_;
+      ts.tm_mday = day_;
-      ts->tm_mon = month_-1;
+      ts.tm_mon = month_-1;
-      ts->tm_year = year_-1900;
+      ts.tm_year = year_-1900;
-      ts->tm_hour = hour_;
+      ts.tm_hour = hour_;
-      ts->tm_min = minute_;
+      ts.tm_min = minute_;
-      ts->tm_sec = (int)second_;
+      ts.tm_sec = (int)second_;
-      timestamp_ = asctime(ts);
+      asctime_r(&ts,&timestamp_[0]);
       // timestamp_ = std::to_string(year_) + std::string("-") + std::to_string(month_)
       //                                    + std::string("-") + std::to_string(day_)
       //                                    + std::string("T") + std::to_string(hour_)

lib/imc_raw.hpp

@@ -7,7 +7,7 @@
 #include <filesystem>
 #include <iostream>
 
-#include "hexshow.hpp"
+// #include "hexshow.hpp"
 #include "imc_key.hpp"
 #include "imc_block.hpp"
 #include "imc_datatype.hpp"
@@ -174,7 +174,7 @@ namespace imc
     // check consistency of blocks
     void check_consistency()
     {
-      for ( unsigned long int b = 0; b < this->rawblocks_.size()-1; b++ )
+      for ( unsigned long int b = 0; b < this->rawblocks_.size()-1 && this->rawblocks_.size() > 0; b++ )
       {
         if ( this->rawblocks_[b].get_end() >= this->rawblocks_[b+1].get_begin() )
         {
@@ -300,7 +300,7 @@ namespace imc
     }
 
     // list a particular type of block
-    std::vector<imc::block> list_blocks(imc::key mykey)
+    std::vector<imc::block> list_blocks(const imc::key &mykey)
     {
       std::vector<imc::block> myblocks;
       for ( imc::block blk: this->rawblocks_ )

makefile

@@ -53,6 +53,12 @@ cpp-clean :
 	rm -vf $(EXE)
 	rm -vf *.o
 
+#-----------------------------------------------------------------------------#
+# linter and code check
+
+check-code:
+	cppcheck --enable=all -I lib/ src/main.cpp
+
 #-----------------------------------------------------------------------------#
 # check version consistency of git tags and version string in package.json
 

parquet/parquet/generate_make.py

@@ -1,6 +1,5 @@
 #-----------------------------------------------------------------------------#
 
-import glob
 from pathlib import Path
 
 # find source files

pip/makefile

@@ -11,6 +11,9 @@ sdist: ../cython/py_imc_termite.pyx ../cython/imc_termite.pxd ../cython/py_imc_t
 	cp -v ../LICENSE ./
 	python3 setup.py sdist
 
+# authentication:
+# - username: __token__
+# - password: <token value including pypi-prefix>
 upload:
 	python3 -m twine upload dist/$(shell ls -t dist/ | head -n1)
 

pip/setup.py

@@ -1,13 +1,12 @@
 
 from setuptools import setup, Extension
-import os
 
 with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
 setup(
     name="IMCtermite",
-    version="1.2.2",
+    version="1.2.4",
     author="Record Evolution GmbH",
     author_email="mario.fink@record-evolution.de",
     maintainer="Record Evolution GmbH",

python/example.py

@@ -5,7 +5,6 @@ import raw_eater
 import raw_meat
 import pyarrow as pa
 import pyarrow.parquet as pq
-import glob
 from pathlib import Path
 
 fileobj1 = Path("smp/Rangerover_Evoque_F-RR534_2019-05-07/").rglob("*.raw")

python/usage.py

@@ -6,7 +6,7 @@ import json
 try :
     imcraw = imc_termite.imctermite(b"samples/sampleA.raw")
 except RuntimeError as e :
-    print("failed to load/parse raw-file: " + str(e))
+    raise Exception("failed to load/parse raw-file: " + str(e))
 
 # obtain list of channels as list of dictionaries (without data)
 channels = imcraw.get_channels(False)
@@ -23,3 +23,6 @@ print(len(chnxdata))
 
 # print the channels into a specific directory
 imcraw.print_channels(b"./")
+
+# print all channels in single file
+imcraw.print_table(b"./allchannels.csv")

python/usage_adv.py

@@ -0,0 +1,30 @@
+
+import imc_termite
+import json
+import os
+
+# list files in sample directory
+# fileobj1 = Path("samples/").rglob("*.raw")
+# rawlist1 = [str(fl) for fl in fileobj1]
+rawlist1 = ["samples/datasetB/datasetB_29.raw"]
+print(rawlist1)
+
+for fl in rawlist1:
+
+    print("converting " + str(fl) + " : " + str(os.path.basename(fl)) )
+
+    # declare and initialize instance of "imctermite" by passing a raw-file
+    try :
+        imcraw = imc_termite.imctermite(fl.encode())
+    except RuntimeError as e :
+        raise Exception("failed to load/parse raw-file: " + str(e))
+
+    # obtain list of channels as list of dictionaries (without data)
+    channels = imcraw.get_channels(False)
+    print(json.dumps(channels,indent=4, sort_keys=False))
+
+    # print the channels into a specific directory
+    imcraw.print_channels(b"./")
+
+    # print all channels in single file
+    imcraw.print_table(("./"+str(os.path.basename(fl).split('.')[0])+"_allchannels.csv").encode())

src/main.cpp

@@ -178,6 +178,13 @@ int main(int argc, char* argv[])
       return 1;
     }
 
+    // catch invalid or empty ".raw" file
+    if ( imcraw.blocks().size() == 0 )
+    {
+      std::cerr<<"this appears to be an empty/invalid '.raw' file since no blocks were found"<<"\n";
+      return 1;
+    }
+
     // list blocks
     if ( cfgopts.count("listblocks") == 1 )
     {