bump version 1.2.12

* makefile for pip: extract proper section of README.md
* bump version 1.2.11

.gitignore

@@ -15,6 +15,7 @@ cython/*.cpp
 
 *.log
 *.so
+*.pyd
 *.o
 *.csv
 *.parquet
@@ -29,3 +30,4 @@ pip/README.md
 pip/LICENSE
 pip/*egg-info
 pip/dist/
+pip/build/

cython/setup.py

@@ -1,6 +1,28 @@
 from distutils.core import setup
 from distutils.extension import Extension
 from Cython.Build import cythonize
+import os
+import sys
+
+print("building on platform: "+sys.platform)
+os.system("git tag -l --sort=version:refname > gittags.log")
+with open ("gittags.log","r") as gt:
+    taglst = gt.readlines()
+os.remove("gittags.log")
+if len(taglst) > 0 :
+    version = taglst[-1].replace('\n','').replace('v','')
+else:
+    version = 'unkown'
+print("building version: "+version)
+
+if sys.platform == "linux" or sys.platform == "darwin" :
+	cmpargs = ['-std=c++17','-Wno-unused-variable']
+	lnkargs = ['-std=c++17','-Wno-unused-variable']
+elif sys.platform == "win32" :
+	cmpargs = ['/EHsc','/std:c++17']
+	lnkargs = []
+else :
+	raise RuntimeError("unknown platform")
 
 extensions = Extension(
     name="imc_termite",
@@ -9,13 +31,13 @@ extensions = Extension(
     # library_dirs=["lib"],
     include_dirs=["lib"],
     language='c++',
-    extra_compile_args=['-std=c++17','-Wno-unused-variable'],
+    extra_compile_args=cmpargs,
-    extra_link_args=['-std=c++17'],
+    extra_link_args=lnkargs,
 )
 
 setup(
-    name="imc_termite",
+    name='imc_termite',
-    version='1.2.6',
+    version='1.2.12',
     description='IMCtermite cython extension',
     author='Record Evolution GmbH',
     author_email='mario.fink@record-evolution.de',

lib/imc_block.hpp

@@ -63,7 +63,7 @@ namespace imc
         std::cout<<"WARNING: invalid length parameter in "<<thekey_.name_<<"-block "
                  <<"(block-end:"<<end_<<",buffer-size:"<<buffer_->size()<<")"
                  <<" => resetting block-end to buffer-size\n";
-        end_ = buffer_->size();
+        end_ = (unsigned long int)(buffer_->size());
       }
 
       try {

lib/imc_channel.hpp

@@ -3,9 +3,12 @@
 #ifndef IMCCHANNEL
 #define IMCCHANNEL
 
-#include <sstream>
 #include "imc_datatype.hpp"
 #include "imc_conversion.hpp"
+#include <sstream>
+#include <math.h>
+#include <chrono>
+#include <ctime>
 
 //---------------------------------------------------------------------------//
 
@@ -87,27 +90,50 @@ namespace imc
     }
   };
 
+  // adjust stream object
+  void customize_stream(std::ostream& stout, int prec, bool fixed)
+  {
+    if ( fixed )
+    {
+      stout<<std::setprecision(prec)<<std::fixed;
+    }
+    else
+    {
+      stout<<std::setprecision(prec);
+    }
+  }
+
   // given a list of numeric objects, join it into a string
   template<typename dt>
-  std::string joinvec(std::vector<dt> myvec, unsigned long int limit = 10)
+  std::string joinvec(std::vector<dt> myvec, unsigned long int limit = 10, int prec = 10, bool fixed = true)
   {
-    // include entire list for limit = - 1
+    // include entire list for limit = 0
-    limit = (limit == 0) ? myvec.size() : limit;
+    unsigned long int myvecsize = (unsigned long int)myvec.size();
+    limit = (limit == 0) ? myvecsize : limit;
 
     std::stringstream ss;
     ss<<"[";
     if ( myvec.size() <= limit )
     {
-      for ( dt el: myvec ) ss<<std::setprecision(10)<<el<<",";
+      for ( dt el: myvec )
+      {
+        customize_stream(ss,prec,fixed);
+        ss<<el<<",";
+      }
     }
     else
     {
-      unsigned long int heals = (unsigned long int)(limit/2.);
+      unsigned long int heals = limit/2;
-      for ( unsigned long int i = 0; i < heals; i++ ) ss<<myvec[i]<<",";
+      for ( unsigned long int i = 0; i < heals; i++ )
-      ss<<"...";
-      for ( unsigned long int i = myvec.size()-heals; i < myvec.size(); i++ )
       {
-        ss<<std::setprecision(10)<<myvec[i]<<",";
+        customize_stream(ss,prec,fixed);
+        ss<<myvec[i]<<",";
+      }
+      ss<<"...";
+      for ( unsigned long int i = myvecsize-heals; i < myvecsize; i++ )
+      {
+        customize_stream(ss,prec,fixed);
+        ss<<myvec[i]<<",";
       }
     }
     std::string sumstr = ss.str();
@@ -129,15 +155,19 @@ namespace imc
     std::string uuid_;
     std::string name_, comment_;
     std::string origin_, origin_comment_, text_;
+    std::chrono::system_clock::time_point trigger_time_, absolute_trigger_time_;
+    double trigger_time_frac_secs_;
     std::string language_code_, codepage_;
     std::string yname_, yunit_;
     std::string xname_, xunit_;
     double xstepwidth_, xoffset_;
+    int xprec_;
 
     // buffer and data
     int signbits_, num_bytes_;
     // unsigned long int byte_offset_;
     unsigned long int buffer_offset_, buffer_size_;
+    long int addtime_;
     int datatp_;
     imc::datatype dattyp_;
     std::vector<imc::datatype> ydata_;
@@ -189,6 +219,9 @@ namespace imc
         xunit_ = blocks_->at(chnenv_.CDuuid_).get_parameter(prms[5]);
         // TODO
         // xname_ = std::string("time");
+
+        // find appropriate precision for "xdata_" by means of "xstepwidth_"
+        xprec_ = (xstepwidth_ > 0 ) ? (int)ceil(fabs(log10(xstepwidth_))) : 10;
       }
 
       // extract associated CP data
@@ -208,6 +241,7 @@ namespace imc
         buffer_offset_ = std::stoul(blocks_->at(chnenv_.Cbuuid_).get_parameter(prms[6]));
         buffer_size_ = std::stoul(blocks_->at(chnenv_.Cbuuid_).get_parameter(prms[7]));
         xoffset_ = std::stod(blocks_->at(chnenv_.Cbuuid_).get_parameter(prms[11]));
+        addtime_ = (long int)std::stod(blocks_->at(chnenv_.Cbuuid_).get_parameter(prms[12]));
       }
 
       // extract associated CR data
@@ -250,8 +284,35 @@ namespace imc
         language_code_ = blocks_->at(chnenv_.NLuuid_).get_parameter(prms[3]);
       }
 
+      // obtain NT data
+      // - https://en.cppreference.com/w/cpp/chrono/c/tm
+      // - https://en.cppreference.com/w/cpp/io/manip/put_time
+      if ( blocks_->count(chnenv_.NTuuid_) == 1 )
+      {
+        prms = blocks_->at(chnenv_.NTuuid_).get_parameters();
+        //std::tm tm{};
+        std::tm tms = std::tm();
+        tms.tm_mday = std::stoi(blocks_->at(chnenv_.NTuuid_).get_parameter(prms[2]));
+        tms.tm_mon = std::stoi(blocks_->at(chnenv_.NTuuid_).get_parameter(prms[3])) - 1;
+        tms.tm_year = std::stoi(blocks_->at(chnenv_.NTuuid_).get_parameter(prms[4])) - 1900;
+        tms.tm_hour = std::stoi(blocks_->at(chnenv_.NTuuid_).get_parameter(prms[5]));
+        tms.tm_min = std::stoi(blocks_->at(chnenv_.NTuuid_).get_parameter(prms[6]));
+        long double secs = std::stold(blocks_->at(chnenv_.NTuuid_).get_parameter(prms[7]));
+        double secs_int;
+        trigger_time_frac_secs_ = modf((double)secs,&secs_int);
+        tms.tm_sec = (int)secs_int;
+
+        // generate std::chrono::system_clock::time_point type
+        std::time_t ts = std::mktime(&tms);
+        trigger_time_ = std::chrono::system_clock::from_time_t(ts);
+      }
+
       // start converting binary buffer to imc::datatype
       if ( !chnenv_.CSuuid_.empty() ) convert_buffer();
+
+      // calculate absolute trigger-time
+      absolute_trigger_time_ = trigger_time_ + std::chrono::seconds(addtime_);
+      //                                       + std::chrono::nanoseconds((long int)(trigger_time_frac_secs_*1.e9));
     }
 
     // convert buffer to actual datatype
@@ -269,7 +330,7 @@ namespace imc
                                            buffer_->begin()+buffstrt+buffer_size_+1 );
 
       // determine number of values in buffer
-      unsigned long int num_values = CSbuffer.size()/(signbits_/8);
+      unsigned long int num_values = (unsigned long int)(CSbuffer.size()/(signbits_/8));
       if ( num_values*(signbits_/8) != CSbuffer.size() )
       {
         throw std::runtime_error("CSbuffer and significant bits of datatype don't match");
@@ -324,7 +385,7 @@ namespace imc
       // fill xdata_
       for ( unsigned long int i = 0; i < num_values; i++ )
       {
-        xdata_.push_back(xoffset_+i*xstepwidth_);
+        xdata_.push_back(xoffset_+(double)i*xstepwidth_);
       }
 
       // employ data transformation
@@ -332,8 +393,9 @@ namespace imc
       {
         for ( imc::datatype& el: ydata_ )
         {
-          // std::cout<<"value:"<<el.as_double()<<"\n";
+          //std::cout<<"value:"<<el.as_double()<<"\n";
-          el = imc::datatype(el.as_double()*factor_ + offset_);
+          double fact = ( factor_ == 0.0 ) ? 1.0 : factor_;
+          el = imc::datatype(el.as_double()*fact + offset_);
         }
       }
     }
@@ -341,12 +403,18 @@ namespace imc
     // get info string
     std::string get_info(int width = 20)
     {
+      // prepare printable trigger-time
+      std::time_t tt = std::chrono::system_clock::to_time_t(trigger_time_);
+      std::time_t att = std::chrono::system_clock::to_time_t(absolute_trigger_time_);
+
       std::stringstream ss;
       ss<<std::setw(width)<<std::left<<"uuid:"<<uuid_<<"\n"
         <<std::setw(width)<<std::left<<"name:"<<name_<<"\n"
         <<std::setw(width)<<std::left<<"comment:"<<comment_<<"\n"
         <<std::setw(width)<<std::left<<"origin:"<<origin_<<"\n"
         <<std::setw(width)<<std::left<<"description:"<<text_<<"\n"
+        <<std::setw(width)<<std::left<<"trigger-time-nt:"<<std::put_time(std::localtime(&tt),"%FT%T")<<"\n"
+        <<std::setw(width)<<std::left<<"trigger-time:"<<std::put_time(std::localtime(&att),"%FT%T")<<"\n"
         <<std::setw(width)<<std::left<<"language-code:"<<language_code_<<"\n"
         <<std::setw(width)<<std::left<<"codepage:"<<codepage_<<"\n"
         <<std::setw(width)<<std::left<<"yname:"<<yname_<<"\n"
@@ -355,6 +423,7 @@ namespace imc
         <<std::setw(width)<<std::left<<"significant bits:"<<signbits_<<"\n"
         <<std::setw(width)<<std::left<<"buffer-offset:"<<buffer_offset_<<"\n"
         <<std::setw(width)<<std::left<<"buffer-size:"<<buffer_size_<<"\n"
+        <<std::setw(width)<<std::left<<"add-time:"<<addtime_<<"\n"
         <<std::setw(width)<<std::left<<"xname:"<<xname_<<"\n"
         <<std::setw(width)<<std::left<<"xunit:"<<xunit_<<"\n"
         <<std::setw(width)<<std::left<<"xstepwidth:"<<xstepwidth_<<"\n"
@@ -363,8 +432,8 @@ namespace imc
         <<std::setw(width)<<std::left<<"offset:"<<offset_<<"\n"
         <<std::setw(width)<<std::left<<"group:"<<"("<<group_index_<<","<<group_name_
                                                     <<","<<group_comment_<<")"<<"\n"
-        <<std::setw(width)<<std::left<<"ydata:"<<imc::joinvec<imc::datatype>(ydata_)<<"\n"
+        <<std::setw(width)<<std::left<<"ydata:"<<imc::joinvec<imc::datatype>(ydata_,6,9,true)<<"\n"
-        <<std::setw(width)<<std::left<<"xdata:"<<imc::joinvec<double>(xdata_)<<"\n";
+        <<std::setw(width)<<std::left<<"xdata:"<<imc::joinvec<double>(xdata_,6,xprec_,true)<<"\n";
         // <<std::setw(width)<<std::left<<"aff. blocks:"<<chnenv_.get_json()<<"\n";
       return ss.str();
     }
@@ -372,17 +441,24 @@ namespace imc
     // provide JSON string of metadata
     std::string get_json(bool include_data = false)
     {
+      // prepare printable trigger-time
+      std::time_t tt = std::chrono::system_clock::to_time_t(trigger_time_);
+      std::time_t att = std::chrono::system_clock::to_time_t(absolute_trigger_time_);
+
       std::stringstream ss;
       ss<<"{"<<"\"uuid\":\""<<uuid_
              <<"\",\"name\":\""<<name_
              <<"\",\"comment\":\""<<comment_
              <<"\",\"origin\":\""<<origin_
              <<"\",\"description\":\""<<text_
+             <<"\",\"trigger-time-nt\":\""<<std::put_time(std::localtime(&tt),"%FT%T")
+             <<"\",\"trigger-time\":\""<<std::put_time(std::localtime(&att),"%FT%T")
              <<"\",\"language-code\":\""<<language_code_
              <<"\",\"codepage\":\""<<codepage_
              <<"\",\"yname\":\""<<yname_
              <<"\",\"yunit\":\""<<yunit_
              <<"\",\"significantbits\":\""<<signbits_
+             <<"\",\"addtime\":\""<<addtime_
              <<"\",\"xname\":\""<<xname_
              <<"\",\"xunit\":\""<<xunit_
              <<"\",\"xstepwidth\":\""<<xstepwidth_
@@ -392,8 +468,8 @@ namespace imc
                                <<"\",\"comment\":\""<<group_comment_<<"\""<<"}";
       if ( include_data )
       {
-        ss<<",\"ydata\":"<<imc::joinvec<imc::datatype>(ydata_,0)
+        ss<<",\"ydata\":"<<imc::joinvec<imc::datatype>(ydata_,0,9,true)
-          <<",\"xdata\":"<<imc::joinvec<double>(xdata_,0);
+          <<",\"xdata\":"<<imc::joinvec<double>(xdata_,0,xprec_,true);
       }
       // ss<<"\",\"aff. blocks\":\""<<chnenv_.get_json()
       ss<<"}";
@@ -402,7 +478,7 @@ namespace imc
     }
 
     // print channel
-    void print(std::string filename, const char sep = ',', int width = 25)
+    void print(std::string filename, const char sep = ',', int width = 25, int yprec = 9)
     {
       std::ofstream fou(filename);
 
@@ -423,13 +499,16 @@ namespace imc
       {
         if ( sep == ' ' )
         {
-          fou<<std::setprecision(9)<<std::fixed
+          fou<<std::setprecision(xprec_)<<std::fixed
              <<std::setw(width)<<std::left<<xdata_[i]
+             <<std::setprecision(yprec)<<std::fixed
              <<std::setw(width)<<std::left<<ydata_[i]<<"\n";
         }
         else
         {
-          fou<<xdata_[i]<<sep<<ydata_[i]<<"\n";
+          fou<<std::setprecision(xprec_)<<std::fixed<<xdata_[i]
+             <<sep
+             <<std::setprecision(yprec)<<std::fixed<<ydata_[i]<<"\n";
         }
       }
 

lib/imc_conversion.hpp

@@ -1,6 +1,6 @@
 //---------------------------------------------------------------------------//
 
-#ifndef IMCCONVRSION
+#ifndef IMCCONVERSION
 #define IMCCONVERSION
 
 #include <vector>

lib/imc_datatype.hpp

@@ -48,7 +48,7 @@ namespace imc
   public:
     datatype(): ubyte_(0), sbyte_(0),
                 ushort_(0), sshort_(0),
-                ulint_(0.0), slint_(0.0),
+                ulint_(0), slint_(0),
                 sfloat_(0.0), sdouble_(0.0),
                 sdigital_(0),
                 dtidx_(0) { };
@@ -61,10 +61,10 @@ namespace imc
     datatype(imc_Slongint num): slint_(num), dtidx_(5) {};
     datatype(imc_float num): sfloat_(num), dtidx_(6) {};
     datatype(imc_double num): ubyte_(0), sbyte_(0), ushort_(0), sshort_(0),
-                              ulint_(0.0), slint_(0.0), sfloat_(0.0), sdouble_(num),
+                              ulint_(0), slint_(0), sfloat_(0.0), sdouble_(num),
                               sdigital_(0), dtidx_(7) {};
     datatype(imc_digital num): ubyte_(0), sbyte_(0), ushort_(0), sshort_(0),
-                               ulint_(0.0), slint_(0.0), sfloat_(0.0), sdouble_(num),
+                               ulint_(0), slint_(0), sfloat_(0.0), sdouble_(num),
                                sdigital_(num), dtidx_(10) {};
 
     // identify type

lib/imc_object.hpp

@@ -500,22 +500,22 @@ namespace imc
       minute_ = std::stoi( get_parameter(buffer,&parameters[6]) );
       second_ = std::stod( get_parameter(buffer,&parameters[7]) );
 
-      time_t rawtime;
+      //time_t rawtime;
-      struct tm ts;
+      //struct tm ts;
-      time(&rawtime);
+      //time(&rawtime);
-      localtime_r(&rawtime,&ts);
+      //localtime_r(&rawtime,&ts);
-      ts.tm_mday = day_;
+      //ts.tm_mday = day_;
-      ts.tm_mon = month_-1;
+      //ts.tm_mon = month_-1;
-      ts.tm_year = year_-1900;
+      //ts.tm_year = year_-1900;
-      ts.tm_hour = hour_;
+      //ts.tm_hour = hour_;
-      ts.tm_min = minute_;
+      //ts.tm_min = minute_;
-      ts.tm_sec = (int)second_;
+      //ts.tm_sec = (int)second_;
-      asctime_r(&ts,&timestamp_[0]);
+      //asctime_r(&ts,&timestamp_[0]);
-      // timestamp_ = std::to_string(year_) + std::string("-") + std::to_string(month_)
+      timestamp_ = std::to_string(year_) + std::string("-") + std::to_string(month_)
-      //                                    + std::string("-") + std::to_string(day_)
+                                         + std::string("-") + std::to_string(day_)
-      //                                    + std::string("T") + std::to_string(hour_)
+                                         + std::string("T") + std::to_string(hour_)
-      //                                    + std::string(":") + std::to_string(minute_)
+                                         + std::string(":") + std::to_string(minute_)
-      //                                    + std::string(":") + std::to_string(second_);
+                                         + std::string(":") + std::to_string(second_);
     }
 
     // get info string

lib/imc_raw.hpp

@@ -123,20 +123,20 @@ namespace imc
                   leng.push_back((char)*(it+pos));
                   pos++;
                 }
-                unsigned long length = std::stoul(leng);
+                unsigned long int length = std::stoul(leng);
 
                 // declare and initialize corresponding key and block
                 // imc::key bkey( *(it+1)==imc::key_crit_ , newkey,
                 //                imc::keys.at(newkey).description_, version );
-                imc::block blk(itkey,it-buffer_.begin(),
+                imc::block blk(itkey,(unsigned long int)(it-buffer_.begin()),
-                                     it-buffer_.begin()+pos+1+length,
+                                     (unsigned long int)(it-buffer_.begin()+pos+1+length),
                                      raw_file_, &buffer_);
 
                 // add block to list
                 rawblocks_.push_back(blk);
 
                 // skip the remaining block according to its length
-                if ( it-buffer_.begin()+length < buffer_.size() )
+                if ( (unsigned long int)(it-buffer_.begin()+length) < (unsigned long int)(buffer_.size()) )
                 {
                   std::advance(it,length);
                 }
@@ -253,7 +253,7 @@ namespace imc
     // provide buffer size
     unsigned long int buffer_size()
     {
-      return buffer_.size();
+      return (unsigned long int)buffer_.size();
     }
 
     // get blocks
@@ -377,7 +377,7 @@ namespace imc
         std::filesystem::path pf = pd / filenam;
 
         // and print the channel
-        it->second.print(pf,sep);
+        it->second.print(pf.u8string(),sep);
       }
     }
 

makefile

@@ -17,13 +17,13 @@ HPP = $(wildcard $(LIB)/*.hpp)
 # choose compiler and its options
 CC = g++ -std=c++17
 #OPT = -O3 -Wall -mavx -mno-tbm -mf16c -mno-f16c
-OPT = -O3 -Wall -Werror -Wunused-variable -Wsign-compare
+OPT = -O3 -Wall -Wconversion -Wpedantic -Werror -Wunused-variable -Wsign-compare
 
 # determine git version/commit and release tag
-GTAG := $(shell git tag | tail -n1)
+GTAG := $(shell git tag -l --sort=version:refname | tail -n1)
 GHSH := $(shell git rev-parse HEAD | head -c8)
-RTAG := v$(shell cat pip/setup.py | grep version | grep -oP "([0-9]\.){2}[0-9]")
+RTAG := v$(shell cat pip/setup.py | grep version | grep -oP "([0-9]\.){2}[0-9]{1,2}")
-CTAG := v$(shell cat cython/setup.py | grep version | grep -oP "([0-9]\.){2}[0-9]")
+CTAG := v$(shell cat cython/setup.py | grep version | grep -oP "([0-9]\.){2}[0-9]{1,2}")
 
 # define install location
 INST := /usr/local/bin
@@ -99,7 +99,7 @@ cython-clean :
 # pip
 
 pip-release: check-vtag $(RTAG) cython-build
-	cd ./pip/ && make publish
+	cd ./pip/ && make publish-source
 
 #-----------------------------------------------------------------------------#
 # clean

pip/makefile

@@ -2,15 +2,23 @@
 
 SHELL := /bin/bash
 
-publish: sdist upload
+publish-source: sdist upload
+publish-binary: bdist upload
 
-sdist: ../cython/py_imc_termite.pyx ../cython/imc_termite.pxd ../cython/py_imc_termite.cpp
+prepdist: ../cython/py_imc_termite.pyx ../cython/imc_termite.pxd ../cython/py_imc_termite.cpp
 	cp -v $? ./
 	cp -v $(shell ls ../lib/imc_*.hpp) ./
-	tail -n 212 ../README.md > ./README.md
+	cat ../README.md | grep '^# IMCtermite' -A 50000 > ./README.md
 	cp -v ../LICENSE ./
+
+sdist: prepdist
 	python3 setup.py sdist
 
+# TODO use manylinux wheel to avoid ERROR "unsupported platform tag 'linux_x86_64'"
+# see: - https://github.com/pypa/manylinux
+bdist: prepdist
+	python3 setup.py bdist_wheel
+
 # authentication:
 # - username: __token__
 # - password: <token value including pypi-prefix>

pip/setup.py

@@ -1,12 +1,24 @@
 
 from setuptools import setup, Extension
+import sys
+
+print("building on platform: "+sys.platform)
+
+if sys.platform == "linux" or sys.platform == "darwin" :
+	cmpargs = ['-std=c++17','-Wno-unused-variable']
+	lnkargs = ['-std=c++17','-Wno-unused-variable']
+elif sys.platform == "win32" :
+	cmpargs = ['/EHsc','/std:c++17']
+	lnkargs = []
+else :
+	raise RuntimeError("unknown platform")
 
 with open("README.md", "r", encoding="utf-8") as fh:
     long_description = fh.read()
 
 setup(
     name="IMCtermite",
-    version="1.2.6",
+    version="1.2.12",
     author="Record Evolution GmbH",
     author_email="mario.fink@record-evolution.de",
     maintainer="Record Evolution GmbH",
@@ -30,7 +42,8 @@ setup(
                            # include_dirs=["3rdparty/pugixml/","lib/"],
                            # depends=["../lib/tdm_termite.hpp"]
                            language='c++',
-                           extra_compile_args=['-std=c++17','-Wno-unused-variable'],
+                           extra_compile_args=cmpargs,
-                           extra_link_args=['-std=c++17'],
+                           extra_link_args=lnkargs,
-                           )],
+                           )
+                ],
 )

python/example.py

@@ -1,137 +0,0 @@
-
-#-----------------------------------------------------------------------------#
-
-import raw_eater
-import raw_meat
-import pyarrow as pa
-import pyarrow.parquet as pq
-from pathlib import Path
-
-fileobj1 = Path("samples/datasetA/").rglob("*.raw")
-rawlist1 = [str(fl) for fl in fileobj1]
-
-fileobj2 = Path("samples/datasetB/").rglob("*.raw")
-rawlist2 = [str(fl) for fl in fileobj2]
-
-rawlist = rawlist1 #[rawlist1[0],rawlist1[4],rawlist2[0],rawlist2[6]]
-for fil in rawlist2 :
-    rawlist.append(fil)
-rawlist.append("./README.md")
-
-print("")
-print(rawlist)
-print()
-
-#-----------------------------------------------------------------------------#
-
-# alternatively create "empty" instance of "raw_eater" and set file names
-eatraw = raw_eater.raweater()
-# eatraw.set_file("../smp/pressure_Vacuum.raw".encode())
-
-# convert every single listed file
-for rf in rawlist :
-
-    print("converting " + str(rf) + "...\n" + 90*("-") + "\n")
-
-    # setup instance of "raw_eater" and trigger conversion
-    # eatraw = raw_eater.raweater(rf.encode())
-    # eatraw = raw_meat.rawmerger(rf.encode())
-
-    # use global instance of "raw_eater" to set file and perform decoding
-    eatraw.set_file(rf.encode())
-    try :
-        eatraw.do_conversion()
-    except RuntimeError as e :
-        print("conversion failed: " + str(e))
-
-    # check validity of file format
-    if eatraw.validity() :
-
-        # show channel name and its unit
-        entity = eatraw.channel_name().decode(encoding='UTF-8',errors='ignore')
-        unit = eatraw.unit().decode(encoding='UTF-8',errors='ignore')
-        print("\nentity: " + str(entity))
-        print("unit:   " + str(unit) + "\n")
-
-        # obtain extracted data
-        xt = eatraw.get_time()
-        yt = eatraw.get_channel()
-
-        # show excerpt of data
-        print("time (length: " + str(len(xt)) + ") \n"
-             + str(xt[:10]) + "\n...\n" + str(xt[-10:]) + "\n")
-        yttrunc = [round(y,4) for y in yt]
-        print(str(entity) + " (length: " + str(len(yttrunc)) + ") \n"
-            + str(yttrunc[:10]) + "\n...\n" + str(yttrunc[-10:]) + "\n")
-
-        outname = rf.split('/')[-1].replace('raw','csv')
-
-        print("write output to : " + outname)
-        eatraw.write_table(("output/"+outname).encode(),ord(' '))
-
-    else :
-
-        print("\nerror: invalid/corrupt .raw file")
-
-    print("\n")
-
-#-----------------------------------------------------------------------------#
-
-print("convert and merge channels " + "\n" + 90*("-") + "\n")
-
-# setup new instance to merge channels
-eatmea = raw_meat.rawmerger(''.encode()) #rawlist[0].encode())
-
-# add every single channel/file in list
-for rf in rawlist :
-    print("\nadding channel " + str(rf))
-    try :
-        succ = eatmea.add_channel(rf.encode())
-        print("\nrecent time series: length: " + str(len(eatmea.get_time_series())) + "\n")
-    except RuntimeError as e :
-        print("failed to add channel: " + str(e))
-
-# show summary of successfully merged channels
-print("\nmerged channels:\n")
-
-# write merged table to .csv output
-eatmea.write_table_all('output/allchannels.csv'.encode(),ord(','))
-
-# get number of successfully merged channels and their names (+units)
-numch = eatmea.get_num_channels()
-chnames = [chnm.decode(encoding='UTF-8',errors='ignore') for chnm in eatmea.get_channel_names()]
-print("number of channels: " + str(numch))
-print("channel names: " + str(chnames))
-
-# obtain final time series
-timse = eatmea.get_time_series()
-print("\nfinal time series:\nlength:" + str(len(timse)) + "\n")
-
-# get time unit and prepend column name
-chnames.insert(0,"Time ["+str(eatmea.time_unit().decode(encoding='UTF-8',errors='ignore'))+"]")
-
-# prepare list of pyarrow arrays
-pyarrs = []
-pyarrs.append(pa.array(timse))
-
-for i in range(0,numch) :
-    print("\n" + str(i) + "  " + str(chnames[i]))
-    dat = eatmea.get_channel_by_index(i)
-    print("length: " + str(len(dat)))
-    pyarrs.append(pa.array(dat))
-print("")
-# print("\npyarrow arrays\n" + str(pyarrs))
-
-# create pyarrow table from data
-pyarwtab = pa.Table.from_arrays(pyarrs,chnames)
-print("\n" + 60*"-" + "\n" + str(pyarwtab) + "\n")
-
-# write pyarrow table to .parquet file with compression
-pq.write_table(pyarwtab,'output/allchannels.parquet',compression='BROTLI')  # compression='BROTLI', 'SNAPPY')
-
-# try to read and decode the .parquet file
-df = pq.read_table('output/allchannels.parquet')
-print(df.to_pandas())
-# df.to_pandas().to_csv('allchannels.csv',index=False,encoding='utf-8',sep=",")
-
-#-----------------------------------------------------------------------------#

python/numpy_to_pyarrow.py

@@ -1,24 +0,0 @@
-
-import pyarrow as pa
-import numpy as np
-import pyarrow.parquet as pq
-
-db = pa.array(np.linspace(10,50,6))
-print(db)
-da = pa.array(np.linspace(0,5,6))
-print(db)
-
-filenam = 'pyarrow_testtab.parquet'
-
-patab = pa.Table.from_arrays([da,db],['entity A [unitA]','entity B [unitB]'])
-print(patab)
-
-# pq.write_table(patab,filenam,compression='BROTLI')
-pq.write_table(patab,filenam,compression='SNAPPY')
-
-df = pq.read_table(filenam)
-print(df)
-print(df.to_pandas())
-
-#import readline
-#readline.write_history_file('generate_pyarrow_table_and_write_parquet.py')

python/usage_ext.py

@@ -0,0 +1,50 @@
+
+import imc_termite
+import json
+import os
+import datetime
+
+# declare and initialize instance of "imctermite" by passing a raw-file
+try :
+    imcraw = imc_termite.imctermite(b"samples/sampleB.raw")
+except RuntimeError as e :
+    raise Exception("failed to load/parse raw-file: " + str(e))
+
+# obtain list of channels as list of dictionaries (without data)
+channels = imcraw.get_channels(False)
+print(json.dumps(channels,indent=4, sort_keys=False))
+
+# obtain all channels (including full data)
+channelsdata = imcraw.get_channels(True)
+
+# everything that follows is an example that specifically makes use only of
+# the first (index = 0) channel ...
+idx = 0
+
+if len(channelsdata) > 0 :
+
+    # get first channel's data
+    chnydata = channelsdata[idx]['ydata']
+    chnxdata = channelsdata[idx]['xdata']
+    print("xdata: " + str(len(chnxdata)))
+    print("ydata: " + str(len(chnydata)))
+
+    # extract trigger-time
+    trigtim = datetime.datetime.fromisoformat(channels[idx]["trigger-time"])
+    print(trigtim)
+
+    # file output of data with absolute timestamp in 1st column
+    filname = os.path.join("./",channelsdata[idx]['name']+".csv")
+    print("writing output into " + filname)
+    with open(filname,'w') as fout :
+        # include column header
+        fout.write( str(channelsdata[idx]['xname']) + '[' + str(channelsdata[idx]['xunit']) + "]"
+                  + ","
+                  + str(channelsdata[idx]['yname']) + '[' + str(channelsdata[idx]['yunit']) + "]"
+                  + "\n" )
+        # add data (introduce time shift according to trigger-time)
+        for row in range(0,len(chnxdata)) :
+            fout.write( str( (trigtim + datetime.timedelta(seconds=chnxdata[row])).isoformat() )
+                      + ","
+                      + str( chnydata[row])
+                      + "\n" )

shell/check_markers.sh

@@ -1,12 +1,50 @@
 ##!/bin/bash/
 
-dir=$1
+fildir=$1
 
-#ls ${dir} | while read fn; do echo $fn; cat ${dir}$fn | grep -a "|[A-Z][A-Z]," -o | wc -l; done;
+if [ -z "${fildir}" ]; then
+  echo "CLI argument missing: provide file or directory" >&2
+  exit 1
+fi
 
-#ls ${dir} | while read fn; do echo $fn; cat ${dir}$fn | grep -a "|[A-Z][A-Z]," -o; done;
+list-markers()
+{
+  arg="$1"
+  if [ -z "${arg}" ]; then
+    echo "list-markers: missing file argument" >&2
+    exit 1
+  else
+    if [ -d "${arg}" ]; then
+      echo "list-markers: file argument is a directory" >&2
+      exit 1
+    fi
+  fi
   
-#ls ${dir} | while read fn; do echo $fn; cat ${dir}$fn | xxd | head -n10 | tail -n3; done;
+  echo -e "\n${arg}\n"
+  mrks=$(cat ${arg} | grep -a "|[A-Z][a-zA-Z]," -o)
+  mrksnum=$(echo "${mrks}" | wc -l)
+  echo -e "(${mrksnum})\n${mrks}"
+}
+
+if [ -f "${fildir}" ]; then
+
+  echo "analyzing single file ${fildir} ..."
+  list-markers "${fildir}"
+
+elif [ -d "${fildir}" ]; then
+
+  echo "analyzing all *.raw files in directory ${fildir} ..."
+  lsfls=$(ls ${fildir}/*.raw | sed 's/\/\//\//g')
+  echo -e "\n${lsfls}"
+
+  for fl in ${lsfls}; do
+    list-markers "${fl}"
+  done
+
+else
+
+  echo "does not exist: ${fildir}" >&2
+  exit 1
+
+fi
 
-ls ${dir} | while read fn; do echo $fn; cat ${dir}$fn | grep -a "|[A-Z][a-zA-Z]," -o | wc -l; done;
-ls ${dir} | while read fn; do echo $fn; cat ${dir}$fn | grep -a "|[A-Z][a-zA-Z]," -o; done;

src/main.cpp

@@ -133,7 +133,7 @@ void show_version()
 void show_usage()
 {
   std::cout<<"\n"
-           <<"imctermite ["<<gittag<<"-g"<<githash<<"] (github.com/RecordEvolution/IMCtermite.git)"
+           <<"imctermite ["<<gittag<<"-g"<<githash<<"] (https://github.com/RecordEvolution/IMCtermite.git)"
            <<"\n\n"
            <<"Decode IMC raw files and dump data as *.csv"
            <<"\n\n"