full imc raw format revision

This commit is contained in:
Mario Fink 2021-02-05 19:45:33 +01:00
parent 09011b50ba
commit 584243f466
8 changed files with 736 additions and 1 deletions

View File

@ -79,7 +79,7 @@ are defined
|--------|-----------------------------------------------------------------------------------------------------|
| CF | format version and processor |
| CK | start of group of keys, no. parameters = 3, indicates (in)correct closure of the measurement series |
| CB | defines a group of cannels |
| CB | defines a group of channels |
| CT | text definition including group association index |
| CG | introduces group of components corresponding to CC keys |
| CD1,2 | old/new version of abscissa description |

View File

@ -4,6 +4,7 @@
#define HEXSHOW
#include <sstream>
#include <assert.h>
//---------------------------------------------------------------------------//

113
lib/imc_block.hpp Normal file
View File

@ -0,0 +1,113 @@
//---------------------------------------------------------------------------//
#ifndef IMCBLOCK
#define IMCBLOCK
#include <iomanip>
#include <map>
#include <string>
#include <sstream>
#include <vector>
//---------------------------------------------------------------------------//
namespace imc
{
// define "magic bytes" announcing start/end of blocks and separation within
const unsigned char ch_bgn_ = 0x7c, ch_end_ = 0x3b, ch_sep_ = 0x2c;
// single parameter (in a block) is determined by offset of its first/last byte
class parameter
{
// offset of first/last byte of parameter
unsigned long int begin_, end_;
public:
parameter(unsigned long int begin, unsigned long int end):
begin_(begin), end_(end)
{
if ( end_ <= begin_ )
{
throw std::logic_error("parameter: offset of first byte larger than last byte's offset");
}
}
// access members
unsigned long int& begin() { return begin_; }
unsigned long int& end() { return end_; }
};
// define properties of a raw file block
class block
{
// associated key
key thekey_;
// offset (in byte) of first (=ch_bgn_) and last byte (=ch_end_) of block
// w.r.t. to entire raw file
unsigned long int begin_, end_;
// name and buffer of associated raw file
std::string raw_file_;
const std::vector<unsigned char>* buffer_;
// offset of first/last byte of parameters in block (separated by ch_sep_)
// w.r.t. to first byte of block (=0)
std::vector<parameter> parameters_;
public:
// constructor
block(key thekey, unsigned long int begin, unsigned long int end,
std::string raw_file, std::vector<unsigned char>* buffer):
thekey_(thekey)
{
if ( keys.count(thekey.name_) != 1 ) throw std::logic_error("unknown key");
begin_ = begin;
end_ = end;
if ( end_ <= begin_ )
{
throw std::logic_error("block: offset of first byte larger than last byte's offset");
}
raw_file_ = raw_file;
buffer_ = buffer;
}
// access members
imc::key& get_key() { return thekey_; }
unsigned long int& get_begin() { return begin_; }
unsigned long int& get_end() { return end_; }
// get parameter
std::vector<unsigned char> extract_parameter(parameter& param)
{
// check parameter w.r.t. to block
if ( param.begin() < begin_ || param.end() > end_ )
{
throw std::logic_error("inconsistent parameter offsets");
}
std::vector<unsigned char> parambuff(buffer_->begin()+begin_+param.begin(),
buffer_->begin()+begin_+param.end());
return parambuff;
}
// get info string
std::string get_info(int width = 20)
{
std::stringstream ss;
ss<<std::setw(width)<<std::left<<"block-key:"<<thekey_.name_<<"\n"
<<std::setw(width)<<std::left<<"begin:"<<begin_<<"\n"
<<std::setw(width)<<std::left<<"end:"<<end_<<"\n"
<<std::setw(width)<<std::left<<"rawfile:"<<raw_file_<<"\n"
<<std::setw(width)<<std::left<<"buffersize:"<<buffer_->size()<<"\n";
return ss.str();
}
};
}
#endif
//---------------------------------------------------------------------------//

89
lib/imc_key.hpp Normal file
View File

@ -0,0 +1,89 @@
//---------------------------------------------------------------------------//
#ifndef IMCKEY
#define IMCKEY
#include <iomanip>
#include <map>
#include <string>
#include <sstream>
#include <vector>
//---------------------------------------------------------------------------//
namespace imc
{
// define properties of marker/key
struct key
{
// (non)critical key
bool critical_;
// name of key (two uppercase letters)
std::string name_;
// short description
std::string description_;
// version of marker
int version_;
// constructor
key(bool critical, std::string name,
std::string description = std::string(""), int version = 1)
{
critical_ = critical;
if ( name.size() != 2 ) throw std::logic_error("invalid key name");
name_ = name;
description_ = description;
version_ = version;
}
// get info string
std::string get_info(int width = 20)
{
std::stringstream ss;
ss<<std::setw(width)<<std::left<<"critical:"<<(critical_?"yes":"no")<<"\n"
<<std::setw(width)<<std::left<<"name:"<<name_<<"\n"
<<std::setw(width)<<std::left<<"decription:"<<description_<<"\n"
<<std::setw(width)<<std::left<<"version:"<<version_<<"\n";
return ss.str();
}
};
// define (non)critial markers/keys
std::map<std::string,key> keys = {
// critical keys
{"CF", key(true,"CF","format version and processor",1)},
{"CK", key(true,"CK","start of group of keys",1)},
{"CB", key(true,"CB","group of channels",1)},
{"CT", key(true,"CT","text definition",1)},
{"CG", key(true,"CG","group of components",1)},
{"CD", key(true,"CD","abscissa description",1)},
{"CD", key(true,"CD","abscissa description",2)},
{"CZ", key(true,"CZ","scaling of z-axis",1)},
{"CC", key(true,"CC","start of component",1)},
{"CP", key(true,"CP","buffer, datatype and samples of component ",1)},
{"Cb", key(true,"Cb","buffer description",1)},
{"CR", key(true,"CR","permissible range of values in component",1)},
{"CN", key(true,"CN","name and comment of channel",1)},
{"CS", key(true,"CS","raw binary data",1)},
{"CI", key(true,"CI","single numerical value",1)},
{"Ca", key(true,"Ca","add reference key",1)},
// noncritical keys
{"NO", key(false,"NO","origin of data",1)},
{"NT", key(false,"NT","timestamp of trigger",1)},
{"ND", key(false,"ND","(color) display properties",1)},
{"NU", key(false,"NU","user defined key",1)},
{"Np", key(false,"Np","property of channel",1)},
{"NE", key(false,"NE","extraction rule for BUS channels",1)}
};
}
#endif
//---------------------------------------------------------------------------//

135
lib/imc_raw.hpp Normal file
View File

@ -0,0 +1,135 @@
//---------------------------------------------------------------------------//
#ifndef IMCRAW
#define IMCRAW
#include <fstream>
// #include <filesystem>
#include "hexshow.hpp"
#include "imc_key.hpp"
#include "imc_block.hpp"
//---------------------------------------------------------------------------//
namespace imc
{
class raw
{
// (path of) raw-file and its basename
std::string raw_file_, file_name_;
// buffer of raw-file
std::vector<unsigned char> buffer_;
// list of imc-blocks
std::vector<block> rawblocks_;
public:
// constructor
raw() {};
raw(std::string raw_file): raw_file_(raw_file) { this->parse(); };
// provide new raw-file
void set_file(std::string raw_file)
{
raw_file_ = raw_file;
this->parse();
}
// list all blocks in buffer
void parse()
{
// open file and put data in buffer
try {
std::ifstream fin(raw_file_.c_str(),std::ifstream::binary);
if ( !fin.good() ) throw std::runtime_error("failed to open file");
std::vector<unsigned char> buffer((std::istreambuf_iterator<char>(fin)),
(std::istreambuf_iterator<char>()));
buffer_ = buffer;
fin.close();
} catch ( const std::exception& e ) {
throw std::runtime_error(
std::string("failed to open raw-file and stream data in buffer: ") + e.what()
);
}
// start parsing raw-blocks in buffer
for ( std::vector<unsigned char>::iterator it=buffer_.begin();
it!=buffer_.end(); ++it )
{
// check for "magic byte"
if ( *it == ch_bgn_ )
{
// check for (non)critical key
if ( *(it+1) == 0x43 || *(it+1) == 0x4e )
{
// compose entire key
std::string newkey = { (char)*(it+1), (char)*(it+2) };
// check for known keys
if ( keys.count(newkey) == 1 )
{
// expecting ch_sep_ after key
if ( *(it+3) == ch_sep_ )
{
// extract key version
std::string vers("");
unsigned long int pos = 4;
while ( *(it+pos) != ch_sep_ )
{
vers.push_back((char)*(it+pos));
pos++;
}
int version = std::stoi(vers);
// get block length
std::string leng("");
pos++;
while ( *(it+pos) != ch_sep_ )
{
leng.push_back((char)*(it+pos));
pos++;
}
unsigned long length = std::stoul(leng);
// declare corresponding key and block
imc::key bkey( (*(it+1)-0x43)==0, newkey,
imc::keys.at(newkey).description_, version );
imc::block blk(bkey,it-buffer_.begin(),
it-buffer_.begin()+8+length,
raw_file_, &buffer_);
// add block to list
rawblocks_.push_back(blk);
}
else
{
throw std::runtime_error( std::string("invalid block or corrupt buffer at byte: ")
+ std::to_string(it-buffer_.begin()) );
}
}
else
{
throw std::runtime_error(std::string("unknown IMC key: ") + newkey);
}
}
}
}
}
// get blocks
std::vector<imc::block>& blocks()
{
return rawblocks_;
}
};
}
#endif
//---------------------------------------------------------------------------//

112
newmakefile Normal file
View File

@ -0,0 +1,112 @@
#-----------------------------------------------------------------------------#
# name of executable and CLI tool
EXE = imctermite
# directory names
SRC = src/
LIB = lib/
CYT = cyt/
# list headers
HPP = $(wildcard $(LIB)/*.hpp)
# choose compiler and its options
CC = g++ -std=c++17
#OPT = -O3 -Wall -mavx -mno-tbm -mf16c -mno-f16c
OPT = -O3 -Wall -Werror -Wunused-variable -Wsign-compare
# determine git version/commit tag
GTAG := $(shell git tag | tail -n1)
GHSH := $(shell git rev-parse HEAD | head -c8)
# define install location
INST := /usr/local/bin
#-----------------------------------------------------------------------------#
# C++ and CLI tool
# build exectuable
$(EXE) : main.o
$(CC) $(OPT) $^ -o $@
# build main.cpp and include git version/commit tag
main.o : src/main.cpp $(HPP)
@cp $< $<.cpp
@sed -i 's/TAGSTRING/$(GTAG)/g' $<.cpp
@sed -i 's/HASHSTRING/$(GHSH)/g' $<.cpp
$(CC) -c $(OPT) -I $(LIB) $<.cpp -o $@
@rm $<.cpp
install : $(EXE)
cp $< $(INST)/
uninstall : $(INST)/$(EXE)
rm $<
clean :
rm -vf $(EXE)
rm -v *.o
# # build CLI executable
# $(EXE) : $(SRC)main.cpp $(LIB)raweat.hpp $(LIB)hexshow.hpp $(LIB)rawmerge.hpp output
# $(CCC) $(OPT) $< -o $@
# # development version
# eatdev : $(SRC)main_dev.cpp $(LIB)raweat.hpp
# $(CCC) $(OPT) $< -o $@
#
# # build target for conversion set of .raw files
# eatall : $(SRC)eatall.cpp $(LIB)raweat.hpp
# $(CCC) $(OPT) $< -o $@
# check existence of name of executable globally
# chexe:=$(shell command -v $(EXE))
#
# # install executable if name does not exist yet
# install : $(EXE)
# ifeq ($(chexe),)
# cp $(EXE) /usr/local/bin/
# else
# @echo "executable with name already exists! choose different name!"
# @exit 1
# endif
# # uninstall
# uninstall :
# rm /usr/local/bin/$(EXE)
#-----------------------------------------------------------------------------#
# Python
#
# # build python module
# py : $(CYT)setup_raw_eater.py $(CYT)raw_eater.pyx $(CYT)raw_eater.pxd $(LIB)raweat.hpp \
# $(CYT)setup_raw_meat.py $(CYT)raw_meat.pyx $(CYT)raw_meat.pxd $(LIB)rawmerge.hpp \
# output
# python3 $(CYT)setup_raw_eater.py build_ext --inplace
# python3 $(CYT)setup_raw_meat.py build_ext --inplace
# cp raw_eater.cpython-*.so pyt/
# cp raw_meat.cpython-*.so pyt/
# rm *.so
#
# py-install: $(CYT)setup_raw_eater.py $(CYT)raw_eater.pyx $(CYT)raw_eater.pxd $(LIB)raweat.hpp \
# $(CYT)setup_raw_meat.py $(CYT)raw_meat.pyx $(CYT)raw_meat.pxd $(LIB)rawmerge.hpp
# python3 $(CYT)setup_raw_eater.py install --record files_raw_eater.txt
# python3 $(CYT)setup_raw_meat.py install --record files_raw_meat.txt
#
# py-clean :
# rm -f raw_eater.cpython-*.so
# rm -f pyt/raw_eater.cpython-*.so
# rm -f $(CYT)raw_eater.cpp
# rm -f raw_meat.cpython-*.so
# rm -f pyt/raw_meat.cpython-*.so
# rm -f $(CYT)raw_meat.cpp
# rm -rf build/
# rm -f *.txt
# rm -rf output/
#
# # prepare directory for test output
# output :
# mkdir -pv output/
#-----------------------------------------------------------------------------#

193
src/newmain.cpp Normal file
View File

@ -0,0 +1,193 @@
//---------------------------------------------------------------------------//
#include <iostream>
#include <filesystem>
#include "imc_key.hpp"
#include "imc_block.hpp"
#include "imc_raw.hpp"
//---------------------------------------------------------------------------//
const std::string gittag("TAGSTRING");
const std::string githash("HASHSTRING");
//---------------------------------------------------------------------------//
// define type of key-value map object
typedef std::map<std::string,std::string> optkeys;
optkeys parse_args(int argc, char* argv[], bool list_args = false)
{
if ( list_args )
{
std::cout<<"number of CLI-arguments: "<<argc<<"\n";
for ( int i = 0; i < argc; i++ ) std::cout<<i<<": "<<argv[i]<<"\n";
}
// declare empty key-value object
optkeys prsdkeys;
// parse all CLI arguments
for ( int i = 1; i < argc; i++ )
{
if ( std::string(argv[i]) == std::string("--showmeta")
|| std::string(argv[i]) == std::string("-m") )
{
prsdkeys.insert(std::pair<std::string,std::string>("showmeta",argv[i]));
}
else if ( std::string(argv[i]) == std::string("--listgroups")
|| std::string(argv[i]) == std::string("-g") )
{
prsdkeys.insert(std::pair<std::string,std::string>("listgroups",argv[i]));
}
else if ( std::string(argv[i]) == std::string("--listchannels")
|| std::string(argv[i]) == std::string("-c") )
{
prsdkeys.insert(std::pair<std::string,std::string>("listchannels",argv[i]));
}
else if ( std::string(argv[i]) == std::string("--listblocks")
|| std::string(argv[i]) == std::string("-b") )
{
prsdkeys.insert(std::pair<std::string,std::string>("listblocks",argv[i]));
}
else if ( std::string(argv[i]) == std::string("--filename")
|| std::string(argv[i]) == std::string("-f") )
{
if ( i+1 == argc || argv[i+1][0] == '-' )
{
std::cerr<<"invalid or missing --filename argument\n";
prsdkeys.insert(std::pair<std::string,std::string>("invalid","filename"));
}
else
{
prsdkeys.insert(std::pair<std::string,std::string>("filename",argv[i+1]));
i = i + 1;
}
}
else if ( std::string(argv[i]) == std::string("--help")
|| std::string(argv[i]) == std::string("-h") )
{
prsdkeys.insert(std::pair<std::string,std::string>("help",argv[i]));
}
else if ( std::string(argv[i]) == std::string("--version")
|| std::string(argv[i]) == std::string("-v") )
{
prsdkeys.insert(std::pair<std::string,std::string>("version",argv[i]));
}
else
{
// check for invalid arguments
if ( argv[i][0] == '-' )
{
std::cerr<<"invalid or unkown argument: "<<argv[i]<<"\n";
prsdkeys.insert(std::pair<std::string,std::string>("invalid",argv[i]));
}
// or missing filenames
else if ( std::string(argv[i]).find(".raw") == std::string::npos )
{
std::cerr<<"doesn't look like a .raw file (make sure to include extension in filename!): "
<<argv[i]<<"\n";
prsdkeys.insert(std::pair<std::string,std::string>("invalid",argv[i]));
}
}
}
return prsdkeys;
}
//---------------------------------------------------------------------------//
void show_version()
{
std::cout<<"imctermite ["<<gittag<<"-g"<<githash<<"]"<<"\n";
}
void show_usage()
{
std::cout<<"\n"
<<"imctermite ["<<gittag<<"-g"<<githash<<"] (github.com/RecordEvolution/IMCtermite.git)"
<<"\n\n"
<<"Decode IMC raw files and dump data as *.csv"
<<"\n\n"
<<"Usage:\n\n"
<<" imctermite <raw-file> [options]"
<<"\n\n"
<<"Options:"
<<"\n\n"
<<" -m, --showmeta show meta information about IMC dataset\n"
<<" -g, --listgroups list channelgroups\n"
<<" -c, --listchannels list channels\n"
<<" -b, --listblocks list IMC key-blocks\n"
<<"- f, --filename filename for csv output\n"
<<" -h, --help show this help message \n"
<<" -v, --version display version\n"
<<"\n";
}
//---------------------------------------------------------------------------//
int main(int argc, char* argv[])
{
// parse CLI arguments
optkeys cfgopts = parse_args(argc,argv);
// exit on any invalid arguments
if ( cfgopts.count("invalid") > 0 ) return 1;
if ( cfgopts.count("version") == 1 )
{
show_version();
}
else if ( cfgopts.count("help") == 1 )
{
show_usage();
}
else
{
// check for at least one file argument
if ( cfgopts.size() == (unsigned int)argc-1 )
{
std::cerr<<"no .raw file given => check --help for usage\n";
return 1;
}
std::string rawfile(argv[1]);
// check existence of file
std::filesystem::path rawpath = rawfile;
if ( !std::filesystem::exists(rawpath) )
{
std::cerr<<"file does not exist: "<<rawfile<<"\n";
return 1;
}
// initialize "imc::raw" instance
imc::raw imcraw;
try {
imcraw.set_file(rawfile);
} catch (const std::exception& e ) {
std::cerr<<"failed to open and parse raw-file: "<<e.what()<<"\n";
return 1;
}
// list blocks
for ( imc::block blk: imcraw.blocks() )
{
std::cout<<blk.get_info()<<"\n";
}
}
// for ( std::map<std::string,imc::key>::iterator it = imc::keys.begin();
// it != imc::keys.end(); ++it )
// {
// std::cout<<it->second.get_info()<<"\n";
// }
//
// std::vector<unsigned char> buff;
// imc::block blk(imc::keys.at("CF"),0,100,"data.raw",&buff);
// std::cout<<blk.get_info()<<"\n";
return 0;
}
//---------------------------------------------------------------------------//

92
src/raweater.cpp Normal file
View File

@ -0,0 +1,92 @@
//---------------------------------------------------------------------------//
#include <iomanip>
#include <iostream>
#include "../lib/raweat.hpp"
#include "../lib/rawmerge.hpp"
//---------------------------------------------------------------------------//
int main(int argc, char* argv[])
{
if ( argc != 3 ) //or ( argc == 1 && std::string(argv[1]) == str::string("--help") )
{
std::cout<<"\n"<<"Usage: ./eatit INPUTFILE OUTPUTFILE"<<"\n"
<<"Convert any file in imc-format .raw to plain text .csv"<<"\n"
<<"Example: ./eatit Druck_THZ_DK.raw druck_thz_dk.csv"<<"\n\n";
return 0;
}
// get name/path of file from CLI argument
std::string rawfile(argv[1]);
// declare instance of 'raw_eater' (and decode data)
raw_eater eatraw; //rawfile,false);
// set file to be decoded
eatraw.set_file(rawfile,true);
// do conversion
eatraw.setup_and_conversion(false);
// eatraw.show_markers();
// std::cout<<"\n";
// std::map<std::string,std::vector<unsigned char>> marks = eatraw.get_markers();
// for ( auto mrk: marks )
// {
// // get data
// std::vector<unsigned char> dat = eatraw.get_marker_data(mrk.first);
//
// // print marker name, length and data
// std::cout<<mrk.first<<" : "<<dat.size()<<'\n';
// std::cout<<std::setfill('-')<<std::setw(96)<<'\n'<<std::setfill(' ');
// eatraw.show_hex(dat,32,512);
// std::cout<<"\n";
//
// std::vector<std::string> segvec = eatraw.get_segment(mrk.first);
// std::cout<<"number of elements in segment: "<<segvec.size()<<"\n\n";
// //for ( auto el: segvec ) std::cout<<el<<"\n";
// }
// get array of encoded data
// std::vector<double> maindata = eatraw.get_data();
// std::cout<<"\nsize of data array: "<<maindata.size()<<"\n\n";
// for ( unsigned long int i = 0; i < 10; i++ ) std::cout<<mydata[i]<<"\n";
// write data in csv-file
if ( eatraw.get_valid() )
{
eatraw.write_table(std::string(argv[2]),' ');
}
else
{
std::cout<<"=> invalid/corrupt .raw file\n";
}
//---------------------------------------------------------------------------//
// declare instance of 'raw_merger'
raw_merger eatmea;
// add channels
eatmea.add_channel(rawfile,false);
eatmea.add_channel("smp/VehicleSpeed_HS.raw",false);
// eatmea.add_channel("smp/Mercedes_E-Klasse-2019-08-08/Flex_StWhl_AnglSpd.raw",false);
// eatmea.add_channel("smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw",false);
// write merged table
if ( eatmea.get_num_channels() > 0 )
{
eatmea.write_table_all("output/allchannels.csv",' ');
}
else
{
std::cout<<"no valid channel added\n";
}
return 0;
}
//---------------------------------------------------------------------------//