Compare commits

...

18 Commits

Author SHA1 Message Date
fdd107fbb3 README.md: fix typo 2021-05-06 18:11:40 +02:00
cd4ce55e7e badge starts 2021-05-06 16:01:07 +02:00
27683cbbd6 license badge 2021-05-06 15:59:43 +02:00
25a51215a0 adjust README.md header 2021-05-06 15:50:38 +02:00
36c02089aa clean up README.md header 2021-05-06 15:47:31 +02:00
cd75689d32 lgtm code stats 2021-05-06 11:46:43 +02:00
57c7f6021b makefile: cppcheck target 2021-05-05 15:58:10 +02:00
83922c343f * imc_channel, imc_raw: optimize with pass by reference
* imc_object: asc_time, localtime: make threadsafe
* imc_datatype: satisfy 'rule of two'
* python: remove all unused imports
2021-05-05 13:28:11 +02:00
72378877ec add some pipy release documentation 2021-05-02 19:57:13 +02:00
e7094d0125 * py_imc_termite.pyx: remove json loads parse_float option
* imc_raw.hpp: catch empty/invalid file to avoid seg fault
2021-05-02 19:34:08 +02:00
16a77ecf1e bump version 1.2.4 for pip/cython 2021-04-30 11:10:41 +02:00
798fc22d7a imc_conversion.hpp: more comprehensive error message, imc_datatype: imc_Ulongint,imc_Slongint typedef for x86_64 vs. arm 2021-04-27 12:42:24 +02:00
2b2e69f0e5 * add support for datatype 11 = 2-byte-word digital
* usage.py: raise proper exception
* usage_adv.py: use different example file
2021-04-27 11:38:43 +02:00
Mario Fink
8708d2d008 * cython/py_imc_termite.pyx: json loads take care of floats, fix column assignment
* imc_channel.hpp: fix floating precision for get_channel()
* extend/include advanced usage in examples
2021-04-26 19:41:56 +02:00
9feadb50c1 py_imc_termite.pyx: fix special character encoding 2021-04-16 09:37:58 +02:00
dbeb1b862a py_imc_termite.pyx: print_table: encoding, xdata 2021-04-15 23:51:44 +02:00
bda799832c fix Dockerfile, cython: print_table to print all channels in single table 2021-04-15 20:17:02 +02:00
97c2195f98 ignore hexshow.hpp, version 1.2.3 2021-03-05 09:19:06 +01:00
18 changed files with 167 additions and 50 deletions

View File

@@ -4,8 +4,9 @@ FROM debian:bullseye-20210111
USER root
RUN apt-get update && apt-get install -y \
build-essential git \
build-essential git vim \
python3 python3-pip
RUN python3 -m pip install cython
RUN g++ -v
@@ -15,6 +16,6 @@ COPY ./ /IMCtermite/
RUN cd /IMCtermite && ls -lh && make install && ls -lh /usr/local/bin/imctermite
# install Python module
# RUN cd /tdm_ripper && ls -lh && make cython-requirements && make cython-install
RUN cd /IMCtermite && ls -lh && make cython-install
CMD ["sleep","inifity"]
CMD ["sleep","infinity"]

View File

@@ -1,13 +1,11 @@
<p align="center">
<a href="https://record-evolution.de/reswarm">
<img
alt="imctermite.svg"
src="assets/imctermite.svg"
width="400"
/>
</a>
</p>
[![Total alerts](https://img.shields.io/lgtm/alerts/g/RecordEvolution/IMCtermite.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/RecordEvolution/IMCtermite/alerts/)
[![Language grade: C/C++](https://img.shields.io/lgtm/grade/cpp/g/RecordEvolution/IMCtermite.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/RecordEvolution/IMCtermite/context:cpp)
[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/RecordEvolution/IMCtermite.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/RecordEvolution/IMCtermite/context:python)
[![LICENSE](https://img.shields.io/github/license/RecordEvolution/IMCtermite)](https://img.shields.io/github/license/RecordEvolution/IMCtermite)
[![STARS](https://img.shields.io/github/stars/RecordEvolution/IMCtermite)](https://img.shields.io/github/stars/RecordEvolution/IMCtermite)
# IMCtermite
_IMCtermite_ provides access to the proprietary data format
_IMC Bus Format_ with the file extension _.raw_ introduced and developed by
@@ -219,3 +217,5 @@ in the Python folder.
- https://www.imc-tm.de/produkte/messtechnik-hardware/imc-cronosflex/ueberblick/
- https://cython.readthedocs.io/en/latest/src/userguide/wrapping_CPlusPlus.html
- https://github.com/Apollo3zehn/ImcFamosFile
- https://pypi.org/help/#apitoken

View File

@@ -16,3 +16,4 @@ cdef extern from "imc_raw.hpp" namespace "imc":
vector[string] get_channels(bool json, bool data) except +
# print all channels
void print_channels(string outputdir) except +
void print_table(string outputfile) except +

View File

@@ -3,6 +3,7 @@
from imc_termite cimport imc_termite
import json as jn
import decimal
# import numpy as np
cdef class imctermite:
@@ -21,9 +22,21 @@ cdef class imctermite:
# get JSON list of channels
def get_channels(self, bool data):
chnlst = self.cpp_imc.get_channels(True,data)
chnlstjn = [jn.loads(chn.decode()) for chn in chnlst]
chnlstjn = [jn.loads(chn.decode(errors="ignore")) for chn in chnlst]
return chnlstjn
# print a channels
# print channels
def print_channels(self, string outputdir):
self.cpp_imc.print_channels(outputdir)
# print table including channels
def print_table(self, string outputfile):
chnlst = self.cpp_imc.get_channels(True,True)
chnlstjn = [jn.loads(chn.decode(errors="ignore")) for chn in chnlst]
with open(outputfile.decode(),'w') as fout:
for chn in chnlstjn:
fout.write('#' +str(chn['xname']).rjust(19)+str(chn['yname']).rjust(20)+'\n')
fout.write('#'+str(chn['xunit']).rjust(19)+str(chn['yunit']).rjust(20)+'\n')
for n in range(0,len(chn['ydata'])):
fout.write(str(chn['xdata'][n]).rjust(20)+
str(chn['ydata'][n]).rjust(20)+'\n')

View File

@@ -14,11 +14,11 @@ extensions = Extension(
)
setup(
version='1.2.2',
name="imc_termite",
version='1.2.4',
description='IMCtermite cython extension',
author='Record Evolution GmbH',
author_email='mario.fink@record-evolution.de',
url='https://github.com/RecordEvolution/IMCtermite.git',
name="imc_termite",
ext_modules=cythonize(extensions,force=True)
)

View File

@@ -95,7 +95,7 @@ namespace imc
ss<<"[";
if ( myvec.size() <= limit )
{
for ( dt el: myvec ) ss<<el<<",";
for ( dt el: myvec ) ss<<std::setprecision(10)<<el<<",";
}
else
{
@@ -104,7 +104,7 @@ namespace imc
ss<<"...";
for ( unsigned long int i = myvec.size()-heals; i < myvec.size(); i++ )
{
ss<<myvec[i]<<",";
ss<<std::setprecision(10)<<myvec[i]<<",";
}
}
std::string sumstr = ss.str();
@@ -147,9 +147,11 @@ namespace imc
std::string group_uuid_, group_name_, group_comment_;
// constructor takes channel's block environment
channel(channel_env chnenv, std::map<std::string,imc::block>* blocks,
std::vector<unsigned char>* buffer):
chnenv_(chnenv), blocks_(blocks), buffer_(buffer), group_index_(-1)
channel(channel_env &chnenv, std::map<std::string,imc::block>* blocks,
std::vector<unsigned char>* buffer):
chnenv_(chnenv), blocks_(blocks), buffer_(buffer),
factor_(1.), offset_(0.),
group_index_(-1)
{
// declare list of block parameters
std::vector<imc::parameter> prms;
@@ -292,6 +294,11 @@ namespace imc
{
imc::convert_data_to_type<imc_double>(CSbuffer,ydata_);
}
// ...
else if ( datatp_ == 11 )
{
imc::convert_data_to_type<imc_digital>(CSbuffer,ydata_);
}
else
{
throw std::runtime_error(std::string("unsupported/unknown datatype") + std::to_string(datatp_));

View File

@@ -17,7 +17,11 @@ namespace imc
// check number of elements of type "datatype" in buffer
if ( subbuffer.size() != channel.size()*sizeof(datatype) )
{
throw std::runtime_error("size mismatch between subbuffer and datatype");
throw std::runtime_error( std::string("size mismatch between subbuffer (")
+ std::to_string(subbuffer.size())
+ std::string(") and datatype (")
+ std::to_string(channel.size()) + std::string("*")
+ std::to_string(sizeof(datatype)) + std::string(")") );
}
// extract every single number of type "datatype" from buffer
@@ -36,6 +40,8 @@ namespace imc
// save number in channel
channel[i] = df;
}
// for ( auto el: channel ) std::cout<<el<<"\n";
}
}

View File

@@ -10,31 +10,47 @@ namespace imc
// map datatype to machine's datatypes
typedef unsigned char imc_Ubyte;
typedef signed char imc_Sbyte;
//
typedef unsigned short imc_Ushort;
typedef signed short imc_Sshort;
//
// e.g. ARM Cortex-A72 armv7l gcc version 10.2.0 (Ubuntu 10.2.0-13ubuntu1)
// #ifdef __arm__
typedef unsigned long int imc_Ulongint;
typedef signed long int imc_Slongint;
// e.g. Intel(R) Core(TM) i7-3520M CPU @ 2.90GHz x86_64 gcc version 10.2.0 (Ubuntu 10.2.0-13ubuntu1)
// #ifdef i386 __i386 __i386__
// typedef unsigned int imc_Ulongint;
// typedef signed int imc_Slongint;
//
typedef float imc_float;
typedef double imc_double;
// TODO remaining types are not yet supported
//
// TODO not all remaining types are supported yet
// typedef <whatever that is ->... > "imc Devices Transitional Recording"
// typedf <sometimestamptype> "Timestamp Ascii"
typedef char16_t imc_digital;
// typedef < > imc_sixbyte "6byte unsigned long"
class datatype
{
protected:
imc_Ubyte ubyte_; // 0
imc_Sbyte sbyte_; // 1
imc_Ushort ushort_; // 2
imc_Sshort sshort_; // 3
imc_Ulongint ulint_; // 4
imc_Slongint slint_; // 5
imc_float sfloat_; // 6
imc_double sdouble_; // 7
short int dtidx_; // \in \{0,...,7\}
imc_Ubyte ubyte_; // 0
imc_Sbyte sbyte_; // 1
imc_Ushort ushort_; // 2
imc_Sshort sshort_; // 3
imc_Ulongint ulint_; // 4
imc_Slongint slint_; // 5
imc_float sfloat_; // 6
imc_double sdouble_; // 7
imc_digital sdigital_; // 10
short int dtidx_; // \in \{0,...,7,10\}
public:
datatype(): ubyte_(0), sbyte_(0),
ushort_(0), sshort_(0),
ulint_(0.0), slint_(0.0),
sfloat_(0.0), sdouble_(0.0),
sdigital_(0),
dtidx_(0) { };
// every supported datatype gets its own constructor
datatype(imc_Ubyte num): ubyte_(num), dtidx_(0) {};
@@ -46,11 +62,29 @@ namespace imc
datatype(imc_float num): sfloat_(num), dtidx_(6) {};
datatype(imc_double num): ubyte_(0), sbyte_(0), ushort_(0), sshort_(0),
ulint_(0.0), slint_(0.0), sfloat_(0.0), sdouble_(num),
dtidx_(7) {};
sdigital_(0), dtidx_(7) {};
datatype(imc_digital num): ubyte_(0), sbyte_(0), ushort_(0), sshort_(0),
ulint_(0.0), slint_(0.0), sfloat_(0.0), sdouble_(num),
sdigital_(num), dtidx_(10) {};
// identify type
short int& dtype() { return dtidx_; }
// copy constructor
datatype(const datatype &num)
{
this->ubyte_ = num.ubyte_;
this->sbyte_ = num.sbyte_;
this->ushort_ = num.ushort_;
this->sshort_ = num.sshort_;
this->ulint_ = num.ulint_;
this->slint_ = num.slint_;
this->sfloat_ = num.sfloat_;
this->sdouble_ = num.sdouble_;
this->sdigital_ = num.sdigital_;
this->dtidx_ = num.dtidx_;
}
// overall assignment operator
datatype& operator=(const datatype &num)
{
@@ -64,6 +98,7 @@ namespace imc
this->slint_ = num.slint_;
this->sfloat_ = num.sfloat_;
this->sdouble_ = num.sdouble_;
this->sdigital_ = num.sdigital_;
this->dtidx_ = num.dtidx_;
}
@@ -119,6 +154,12 @@ namespace imc
this->dtidx_ = 7;
return *this;
}
datatype& operator=(const imc_digital &num)
{
this->sdigital_ = num;
this->dtidx_ = 10;
return *this;
}
// obtain number as double
double as_double()
@@ -132,6 +173,7 @@ namespace imc
else if ( dtidx_ == 5 ) num = (double)slint_;
else if ( dtidx_ == 6 ) num = (double)sfloat_;
else if ( dtidx_ == 7 ) num = (double)sdouble_;
else if ( dtidx_ == 10 ) num = static_cast<double>(sdigital_);
return num;
}
@@ -146,6 +188,7 @@ namespace imc
else if ( num.dtidx_ == 5 ) out<<num.slint_;
else if ( num.dtidx_ == 6 ) out<<num.sfloat_;
else if ( num.dtidx_ == 7 ) out<<num.sdouble_;
else if ( num.dtidx_ == 10 ) out<<static_cast<double>(num.sdigital_);
return out;
}

View File

@@ -501,16 +501,16 @@ namespace imc
second_ = std::stod( get_parameter(buffer,&parameters[7]) );
time_t rawtime;
struct tm* ts;
struct tm ts;
time(&rawtime);
ts = localtime(&rawtime);
ts->tm_mday = day_;
ts->tm_mon = month_-1;
ts->tm_year = year_-1900;
ts->tm_hour = hour_;
ts->tm_min = minute_;
ts->tm_sec = (int)second_;
timestamp_ = asctime(ts);
localtime_r(&rawtime,&ts);
ts.tm_mday = day_;
ts.tm_mon = month_-1;
ts.tm_year = year_-1900;
ts.tm_hour = hour_;
ts.tm_min = minute_;
ts.tm_sec = (int)second_;
asctime_r(&ts,&timestamp_[0]);
// timestamp_ = std::to_string(year_) + std::string("-") + std::to_string(month_)
// + std::string("-") + std::to_string(day_)
// + std::string("T") + std::to_string(hour_)

View File

@@ -7,7 +7,7 @@
#include <filesystem>
#include <iostream>
#include "hexshow.hpp"
// #include "hexshow.hpp"
#include "imc_key.hpp"
#include "imc_block.hpp"
#include "imc_datatype.hpp"
@@ -174,7 +174,7 @@ namespace imc
// check consistency of blocks
void check_consistency()
{
for ( unsigned long int b = 0; b < this->rawblocks_.size()-1; b++ )
for ( unsigned long int b = 0; b < this->rawblocks_.size()-1 && this->rawblocks_.size() > 0; b++ )
{
if ( this->rawblocks_[b].get_end() >= this->rawblocks_[b+1].get_begin() )
{
@@ -300,7 +300,7 @@ namespace imc
}
// list a particular type of block
std::vector<imc::block> list_blocks(imc::key mykey)
std::vector<imc::block> list_blocks(const imc::key &mykey)
{
std::vector<imc::block> myblocks;
for ( imc::block blk: this->rawblocks_ )

View File

@@ -53,6 +53,12 @@ cpp-clean :
rm -vf $(EXE)
rm -vf *.o
#-----------------------------------------------------------------------------#
# linter and code check
check-code:
cppcheck --enable=all -I lib/ src/main.cpp
#-----------------------------------------------------------------------------#
# check version consistency of git tags and version string in package.json

View File

@@ -1,6 +1,5 @@
#-----------------------------------------------------------------------------#
import glob
from pathlib import Path
# find source files

View File

@@ -11,6 +11,9 @@ sdist: ../cython/py_imc_termite.pyx ../cython/imc_termite.pxd ../cython/py_imc_t
cp -v ../LICENSE ./
python3 setup.py sdist
# authentication:
# - username: __token__
# - password: <token value including pypi-prefix>
upload:
python3 -m twine upload dist/$(shell ls -t dist/ | head -n1)

View File

@@ -1,13 +1,12 @@
from setuptools import setup, Extension
import os
with open("README.md", "r", encoding="utf-8") as fh:
long_description = fh.read()
setup(
name="IMCtermite",
version="1.2.2",
version="1.2.4",
author="Record Evolution GmbH",
author_email="mario.fink@record-evolution.de",
maintainer="Record Evolution GmbH",

View File

@@ -5,7 +5,6 @@ import raw_eater
import raw_meat
import pyarrow as pa
import pyarrow.parquet as pq
import glob
from pathlib import Path
fileobj1 = Path("smp/Rangerover_Evoque_F-RR534_2019-05-07/").rglob("*.raw")

View File

@@ -6,7 +6,7 @@ import json
try :
imcraw = imc_termite.imctermite(b"samples/sampleA.raw")
except RuntimeError as e :
print("failed to load/parse raw-file: " + str(e))
raise Exception("failed to load/parse raw-file: " + str(e))
# obtain list of channels as list of dictionaries (without data)
channels = imcraw.get_channels(False)
@@ -23,3 +23,6 @@ print(len(chnxdata))
# print the channels into a specific directory
imcraw.print_channels(b"./")
# print all channels in single file
imcraw.print_table(b"./allchannels.csv")

30
python/usage_adv.py Normal file
View File

@@ -0,0 +1,30 @@
import imc_termite
import json
import os
# list files in sample directory
# fileobj1 = Path("samples/").rglob("*.raw")
# rawlist1 = [str(fl) for fl in fileobj1]
rawlist1 = ["samples/datasetB/datasetB_29.raw"]
print(rawlist1)
for fl in rawlist1:
print("converting " + str(fl) + " : " + str(os.path.basename(fl)) )
# declare and initialize instance of "imctermite" by passing a raw-file
try :
imcraw = imc_termite.imctermite(fl.encode())
except RuntimeError as e :
raise Exception("failed to load/parse raw-file: " + str(e))
# obtain list of channels as list of dictionaries (without data)
channels = imcraw.get_channels(False)
print(json.dumps(channels,indent=4, sort_keys=False))
# print the channels into a specific directory
imcraw.print_channels(b"./")
# print all channels in single file
imcraw.print_table(("./"+str(os.path.basename(fl).split('.')[0])+"_allchannels.csv").encode())

View File

@@ -178,6 +178,13 @@ int main(int argc, char* argv[])
return 1;
}
// catch invalid or empty ".raw" file
if ( imcraw.blocks().size() == 0 )
{
std::cerr<<"this appears to be an empty/invalid '.raw' file since no blocks were found"<<"\n";
return 1;
}
// list blocks
if ( cfgopts.count("listblocks") == 1 )
{