Compare commits

..

No commits in common. "master" and "v0.2" have entirely different histories.
master ... v0.2

32 changed files with 297 additions and 1507 deletions

View File

@ -1,100 +0,0 @@
name: CI Build Wheel
on:
push:
tags: ["v[0-9]+.[0-9]+.[0-9]+"]
jobs:
build_setup:
name: Prepare environment for wheel builds
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v2
- name: Prepare wheel build
run: make -C python/ setup
- name: Store wheel configuration files
uses: actions/upload-artifact@v4.6.0
with:
name: wheel-config
path: python/
- name: Display files
run: ls -lR
build_wheels:
name: Build binary wheels on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
needs: [build_setup]
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- name: Install cibuildwheel
run: python -m pip install cibuildwheel==2.1.2
- name: Get wheel configuration files
uses: actions/download-artifact@v4.1.7
with:
name: wheel-config
path: python/
- name: Build wheels
run: python -m cibuildwheel --output-dir wheelhouse
working-directory: python/
- name: Store binary wheels
uses: actions/upload-artifact@v4.6.0
with:
name: binary-wheels-${{matrix.os}}-${{ strategy.job-index }}
path: python/wheelhouse/*.whl
build_sdist:
name: Build source distribution
runs-on: ubuntu-24.04
needs: [build_setup]
steps:
- uses: actions/checkout@v2
- name: Install cython
run: python -m pip install cython==0.29.24
- name: Get wheel configuration files
uses: actions/download-artifact@v4.1.7
with:
name: wheel-config
path: python/
- name: Build sdist
run: python setup.py sdist
working-directory: python/
- name: Store source wheels
uses: actions/upload-artifact@v4.6.0
with:
name: source-wheels
path: python/dist/*.tar.gz
- name: Display files
run: ls -lR
upload_pypi:
name: Upload wheels to PyPI
runs-on: ubuntu-24.04
needs: [build_wheels, build_sdist]
steps:
- name: Get source wheels
uses: actions/download-artifact@v4.1.7
with:
name: source-wheels
path: dist/
- name: Get binary wheels
uses: actions/download-artifact@v4.1.7
with:
path: dist/
pattern: binary-wheels-*
merge-multiple: true
- name: Display files
run: ls -lR
- uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
password: ${{ secrets.TDMTERMITE_GITHUB_WORKFLOW_PYPI_API_TOKEN }}

19
.gitignore vendored
View File

@ -11,22 +11,3 @@ tdmripper
*.log *.log
tdmreaper tdmreaper
cython/*.cpp cython/*.cpp
tdmtermite
dist/
*.egg-info/
output/
monitor-process.sh
tdmtest
pip/*.hpp
pip/*.md
pip/*.cpp
pip/*.pyx
pip/*.pxd
pip/LICENSE
python/3rdparty/
python/LICENSE
python/README.md
python/TDMtermite.cpp
python/lib/

View File

@ -1,30 +1,20 @@
FROM debian:bullseye FROM debian:bullseye-20210111
# install requirements USER root
RUN apt-get update && apt-get upgrade -y && apt-get install -y \
build-essential \ RUN apt-get update && apt-get install -y \
g++ make git \ build-essential git \
python3 python3-pip python3 python3-pip
# check compiler and current user RUN g++ -v
RUN g++ -v && whoami
# use /home as working directory COPY ./ /tdm_ripper/
WORKDIR /home
# get the public TDMtermite repository
RUN git clone https://github.com/RecordEvolution/TDMtermite.git
# install CLI tool # install CLI tool
RUN cd ./TDMtermite && ls -lh && make install && ls -lh /usr/local/bin/tdmtermite RUN cd /tdm_ripper && ls -lh && make install && ls -lh /usr/local/bin/tdmreaper
# install Python module # install Python module
RUN cd ./TDMtermite && ls -lh && make cython-requirements && make cython-list && make cython-install RUN cd /tdm_ripper && ls -lh && make cython-requirements && make cython-install
# create directory for data exchange
#RUN [ "/bin/bash", "-c", "mkdir -pv data/{input,output}" ]
RUN mkdir -pv data
CMD ["sleep","infinity"]
CMD ["sleep","inifity"]

184
README.md
View File

@ -1,41 +1,35 @@
[![LICENSE](https://img.shields.io/github/license/RecordEvolution/TDMtermite)](https://img.shields.io/github/license/RecordEvolution/TDMtermite) <p align="center">
[![STARS](https://img.shields.io/github/stars/RecordEvolution/TDMtermite)](https://img.shields.io/github/stars/RecordEvolution/TDMtermite) <a href="https://github.com/RecordEvolution/tdm_ripper.git">
![CI Build Wheel](https://github.com/RecordEvolution/TDMtermite/actions/workflows/pypi-deploy.yml/badge.svg?branch=&event=push) <img
[![PYPI](https://img.shields.io/pypi/v/TDMtermite.svg)](https://pypi.org/project/tdmtermite/) alt="tdmreaper.svg"
src="assets/tdmreaper.svg"
width="400"
/>
</a>
</p>
# TDMtermite The _tdm_reaper_ is a C++ based library that decodes (encodes) the proprietary
file format _TDM/TDX_ for measurement data, which relies upon the
_TDMtermite_ is a C++ based library that decodes the proprietary _technical data management_ data model. The TDM format was introduced by
file format _TDM/TDX_ for measurement data. First introduced by [National Instruments](https://www.ni.com) and is employed by
[National Instruments](https://www.ni.com), the TDM format relies on the
_technical data management_ data model and is employed by
[LabVIEW](https://www.ni.com/de-de/shop/labview.html), LabWindows™/CVI™, [LabVIEW](https://www.ni.com/de-de/shop/labview.html), LabWindows™/CVI™,
Measurement Studio, SignalExpress, and [DIAdem](https://www.ni.com/de-de/shop/data-acquisition-and-control/application-software-for-data-acquisition-and-control-category/what-is-diadem.html). Measurement Studio, SignalExpress, and [DIAdem](https://www.ni.com/de-de/shop/data-acquisition-and-control/application-software-for-data-acquisition-and-control-category/what-is-diadem.html).
The [Record Evolution Platform](https://www.record-evolution.de/en/home-en/) uses TDMtermite to integrate measurement data into ETL processes. The TDMtermite library is available both as a command line tool and as a Python module. The Python module of TDMtermite enables data scientists to conveniently include TDM formats in their existing data pipelines by providing access to both raw data and metadata in terms of native Python objects. ## Data Format
## Overview Datasets encoded in the TDM/TDX format come along in pairs comprised of a
.tdm (header) and a .tdx (data) file. While the .tdm file is a human-readable
* [TDM file format](#Dataformat) file providing meta information about the data set, the .tdx is a binary
* [Build and Installation](#Installation)
* [Usage and Examples](#Usage)
* [References](#References)
## Dataformat
Datasets encoded in the TDM/TDX format come in pairs comprised of a
.tdm (header) file and a .tdx (data) file. While the .tdm file is a human-readable
file providing meta information about the dataset, the .tdx file is a binary file
containing the actual data. The .tdm based on the _technical data management_ containing the actual data. The .tdm based on the _technical data management_
model is an XML file. It describes what data the .tdx file contains and how model is an XML file and basically describes what data the .tdx contains and how
to read it. The to read it. The
[TDM data model](https://www.ni.com/de-de/support/documentation/supplemental/10/ni-tdm-data-model.html) [TDM data model](https://www.ni.com/de-de/support/documentation/supplemental/10/ni-tdm-data-model.html)
structures the data hierarchically with respect to _file_, _(channel)_ _groups_ and structures the data hierarchically with respect to _file_, (channel) _groups_ and
_channels_. The file-level XML may contain any number of (channel) groups, each _channels_. The file level XML may contain any number of (channel) groups each
of which is made up of an arbitrary number of channels. Thus, the XML tree in of which is made up of an arbitrary number of channels. Thus, the XML tree in
the [TDM header file](https://zone.ni.com/reference/de-XX/help/370858P-0113/tdmdatamodel/tdmdatamodel/tdm_headerfile/) the [TDM header file](https://zone.ni.com/reference/de-XX/help/370858P-0113/tdmdatamodel/tdmdatamodel/tdm_headerfile/)
looks like this: looks basically like this:
```xml ```xml
<?xml version="1.0" encoding="UTF-8" standalone="no" ?> <?xml version="1.0" encoding="UTF-8" standalone="no" ?>
@ -66,14 +60,14 @@ looks like this:
</usi:tdm> </usi:tdm>
``` ```
The XML tree is comprised of _four_ main XML elements: `usi:documentation`, `usi:model`, and is comprised of _four_ main XML elements: `usi:documentation`, `usi:model`,
`usi:include` and `usi:data`. The element `usi:include` references the data file `usi:include` and `usi:data`. The element `usi:include` references the data file
`example.tdx` and reveals one of _two_ possible orderings of the mass data (.tdx): `example.tdx` and reveals one of _two_ possible orderings of the mass data (.tdx):
1. either _channel-wise_ (`<block>`) - all values of a specific channel follow subsequently 1. either _channel wise_ (`<block>`) - all values of a specific channel follow subsequently -
1. or _block-wise_ (`<block_bm>`) - all values of a specific measurement time follow subsequently. 1. or _block wise_ (`<block_bm>`) - all values of a specific measurement time follow subsequently -
The supported _numerical data types_ are: ordering. The supported _numerical data types_ are
| datatype | channel datatype | numeric | value sequence | size | description | | datatype | channel datatype | numeric | value sequence | size | description |
|-------------|------------------|---------|-----------------|-------|-------------------------| |-------------|------------------|---------|-----------------|-------|-------------------------|
@ -86,10 +80,10 @@ The supported _numerical data types_ are:
| eFloat64Usi | DT_DOUBLE | 7 | double_sequence | 8byte | 64 Bit double | | eFloat64Usi | DT_DOUBLE | 7 | double_sequence | 8byte | 64 Bit double |
| eStringUsi | DT_STRING | 1 | string_sequence | | text | | eStringUsi | DT_STRING | 1 | string_sequence | | text |
The XML element `<usi:data>` is comprised of _five_ different types of The XML element `<usi:data>` is basically comprised of _five_ different types of
elements that are `<tdm_root>`, `<tdm_channelgroup>`, `<tdm_channel>`, `<localcolumn>` elements that are `<tdm_root>`, `<tdm_channelgroup>`, `<tdm_channel>`, `<localcolumn>`
and `<submatrix>`. The root element `<tdm_root>` describes the general properties and `<submatrix>`. The root element `<tdm_root>` describes the general properties
of the dataset and lists the _ids_ of all channel groups that belong to of the dataset and lists the _id's_ of all channel groups that belong to
the dataset. The element `<tdm_channelgroup>` divides the _channels_ into groups the dataset. The element `<tdm_channelgroup>` divides the _channels_ into groups
and has a unique _id_ that is referenced by its root element. The `<channels>` and has a unique _id_ that is referenced by its root element. The `<channels>`
element in `<tdm_channelgroup>` lists the unique ids of all channels that belong element in `<tdm_channelgroup>` lists the unique ids of all channels that belong
@ -109,7 +103,7 @@ actual data including its datatype. The remaining element types are
</localcolumn> </localcolumn>
``` ```
with a unique id, the `<measurement_quantity>` referring to one specific channel, with a unique id, the `<measurement_quantity>` refering to one specific channel,
the `<submatrix>` and its id respectively, the type of representation in the `<submatrix>` and its id respectively, the type of representation in
`<sequence_representation>` - being one of _explicit_, _implicit linear_ or `<sequence_representation>` - being one of _explicit_, _implicit linear_ or
_rawlinear_ - and the `<values>` element, which refers to one _value sequence_, _rawlinear_ - and the `<values>` element, which refers to one _value sequence_,
@ -124,121 +118,107 @@ and the element `<submatrix>`
</submatrix> </submatrix>
``` ```
that references the channel group in `<measurement>` to which it belongs and provides that references the channel group in `<measurement>` it belongs to and provides
the _number of rows_ in the channels listed in `<local_columns>`. the _number of rows_ in the channels listed in `<local_columns>`.
## Installation ## Installation
The library can be used both as a _CLI_-based tool and as a _Python_ module. The library can be used both as a _CLI_ based tool and as a _Python_ module.
### CLI tool ### CLI tool
To install the CLI tool _TDMtermite_, do To install the CLI tool _tdmreaper_ do
```Shell ```Shell
make install make install
``` ```
which uses `/usr/local/bin` as an installation directory. On _macOSX_, please first which uses `/usr/local/bin` as installation directory. On _macOSX_ please first
build the binary locally with `make` and install it in your preferred location. build the binary locally with `make` and install it in your preferred location.
### Python ### Python
In order to build a _Python module_ from the _C++_ code base, the In order to build a _Python module_ from the _C++_ code base the
[Cython](https://cython.readthedocs.io/en/latest/index.html) package must be [Cython](https://cython.readthedocs.io/en/latest/index.html) package must be
available. It may be installed via `python3 -m pip install cython` . available, which may be installed via `python3 -m pip install cython` .
The [Numpy](https://numpy.org) package is recommended Furthermore, the [Numpy](https://numpy.org) package is recommended to be able
to pass arrays of data from the C++ kernel to Python. The _makefile_ provides to pass arrays of data from the C++ kernel to Python. The _makefile_ provides
the target `make cython-requirements` to install all required Python modules. the target `make cython-requirements` to install all required Python modules.
Finally, to build the Python extension _tdm_termite_ locally or install Finally, to build the Python extension _tdm_reaper_ either locally or install
it, the targets `make cython-build` and `make cython-install` are provided. it the targets `make cython-build` and `make cython-install` are provided.
To install the Python module on the system, simply do Hence, to install the Python module on the system simply do
```Shell ```Shell
make cython-requirements make cython-requirements
make cython-install make cython-install
``` ```
which makes the module available for import by `import tdm_termite` . that makes the module available to be imported as `import tdm_reaper` .
#### Installation with pip
The package is also available via the [Python Package Index](https://pypi.org) at
[TDMtermite](https://pypi.org/project/tdmtermite/). To install the latest version simply do
```Shell
python3 -m pip install tdmtermite
```
##### Unix
Note, that _python3_setuptools_ and _gcc version >= 10.2.0_ are required to
successfully install and use it.
## Usage ## Usage
### CLI tool ### CLI tool
The usage of the CLI tool is sufficiently clarified by its help message displayed The usage of the CLI tool is sufficiently clarified by its help message displayed
by `tdmtermite --help`. To extract the data decoded in the pair of by `tdmreaper --help`. For instance, to extract the data decoded in the pair of
files `samples/SineData.tdm` and `samples/SineData.tdx` into the directory files `samples/SineData.tdm` and `samples/SineData.tdx` into the directory
`/home/jack/data/`: `/home/jack/data/`:
```Shell ```Shell
tdmtermite samples/SineData.tdm samples/SineData.tdx --output /home/jack/data tdmreaper samples/SineData.tdm samples/SineData.tdx --output /home/jack/data
``` ```
The tool can also be used to list the available objects in the TDM dataset, which The tool can also be used to list the available objects in the TDM dataset, which
are i.a. _channels_, _channelgroups_ and TDX _blocks_. To list are i.a. _channels_, _channelgroups_ and TDX _blocks_. For instance, to list
all channels and channelgroups (without writing any file output): all channels and channelgroups (without writing any file output):
```Shell ```Shell
tdmtermite samples/SineData.tdm samples/SineData.tdx --listgroups --listchannels tdmreaper samples/SineData.tdm samples/SineData.tdx --listgroups --listchannels
``` ```
The user may also submit a _filenaming rule_ to control the names of the files the The user may also submit a _filenaming rule_ to control the names of the files the
channel(group)s are written to. To this end, the _magic flags_ `%G` `%g`, `%C` channel(-group)s are written to. To this end, the _magic flags_ `%G` `%g`, `%C`
and `%c` representing the group id, group name, channel index and channel name and `%c` representing the group id, group name, channel index and channel name
are defined. The default filenaming option is: are defined. The default filenaming option is
```Shell ```Shell
tdmtermite samples/SineData.tdm samples/SineData.tdx --output /home/jack/data --filenames channelgroup_%G.csv tdmreaper samples/SineData.tdm samples/SineData.tdx --output /home/jack/data --filenames channelgroup_%G.csv
``` ```
This makes the tool write _all channels_ grouped into files according to their which makes the tool write _all channels_ grouped into files according to their
group association, while all channelgroup filenames obey the pattern `channelgroup_%G.csv`, group association, while all channelgroup filenames obey the pattern `channelgroup_%G.csv`
with `%G` being replaced by the group id. The filenaming rule also enables the user with `%G` being replaced by the group id. The filenaming rule also enables the user
to extract only a single channel(group) by providing a particular channel(group) to extract only a single channel(group) by providing a particular channel(-group)
id in the filenaming flag. For example, id in the filenaming flag. For example,
```Shell ```Shell
tdmtermite samples/SineData.tdm samples/SineData.tdx --output /home/jack/data -f channel_usi16_%c.csv --includemeta tdmreaper samples/SineData.tdm samples/SineData.tdx --output /home/jack/data -f channel_usi16_%c.csv --includemeta
``` ```
This will write the single channel with the id `usi16` to the file will write the single channel with id `usi16` to the file
`/home/jack/data/channel_usi16_A4.csv`, including its meta-data as a file header. `/home/jack/data/channel_usi16_A4.csv` including its meta-data as a file header.
### Python ### Python
To be able to use the Python module _tdm_termite_, it first has to be built locally To be able to use the Python module _tdm_reaper_ it first has to be build locally
or installed on the system. In the Python interpreter, simply do: or installed on the system. In the Python interpreter simply do:
```Python ```Python
import tdmtermite import tdm_reaper
``` ```
This will import the module. The TDM files are provided by creating an instance of to import the module. The TDM files are provided by creating an instance of
the _tdmtermite_ class: the _tdm_reaper_ class:
```Python ```Python
# create 'tdmtermite' instance object # create 'tdm_reaper' instance object
try : try :
jack = tdmtermite.tdmtermite(b'samples/SineData.tdm',b'samples/SineData.tdx') jack = tdm_reaper.tdmreaper(b'samples/SineData.tdm',b'samples/SineData.tdx')
except RuntimeError as e: except RuntimeError as e:
print("failed to load/decode TDM files: " + str(e)) print("failed to load/decode TDM files: " + str(e))
``` ```
After initializing the _tdmtermite_ object, it can be used to extract any of the After initializing the _tdm_reaper_ object it can be used to extract any of the
available data. For instance, to list the included channelgroups and channels: available data. For instance, to list the included channelgroups and channels:
```Python ```Python
@ -250,16 +230,16 @@ grpids = jack.get_channelgroup_ids()
chnids = jack.get_channel_ids() chnids = jack.get_channel_ids()
``` ```
As a use case, we have a look at listing the ids of all channelgroups and printing As a use case, we have look at listing the ids of all channelgroups and printing
their data to separate files: their data to separate files:
```Python ```Python
import tdmtermite import tdm_reaper
import re import re
# create 'tdmtermite' instance object # create 'tdm_reaper' instance object
try : try :
jack = tdmtermite.tdmtermite(b'samples/SineData.tdm',b'samples/SineData.tdx') jack = tdm_reaper.tdmreaper(b'samples/SineData.tdm',b'samples/SineData.tdx')
except RuntimeError as e : except RuntimeError as e :
print("failed to load/decode TDM files: " + str(e)) print("failed to load/decode TDM files: " + str(e))
@ -287,21 +267,17 @@ for grp in grpids :
print("failed to print channelgroup: " + str(grp) + " : " + str(e)) print("failed to print channelgroup: " + str(grp) + " : " + str(e))
``` ```
For details, see this [extensive example](python/usage.py) For a full example including more details see [python/usage.py](python/usage.py)
and the absolute minimal example [minimal usage](python/minimal.py). In order and the absolute minimal example [minimal usage](python/minimal.py). In order
to simply extract all data of the TDM datatset and dump it to files in a given to simply extract all data of the TDM datatset and dump it to files in a given
(existing!) directory, do (existing!) directory, do
```Python ```Python
import tdmtermite import tdm_reaper
jack = tdmtermite.tdmtermite(b'samples/SineData.tdm',b'samples/SineData.tdx') jack = tdm_reaper.tdmreaper(b'samples/SineData.tdm',b'samples/SineData.tdx')
jack.write_all(b"./my_tdm_data_directory/") jack.write_all(b"./my_tdm_data/")
``` ```
The interface allows you to construct customized file/column headers from any
meta-data and provide these headers for usage in file output (see this
[example](python/custom.py)).
## References ## References
### TDM ### TDM
@ -321,30 +297,6 @@ meta-data and provide these headers for usage in file output (see this
### Implementation ### Implementation
- https://en.cppreference.com/w/
- https://pugixml.org/ - https://pugixml.org/
- https://github.com/zeux/pugixml - https://github.com/zeux/pugixml
- https://cython.readthedocs.io/en/latest/src/userguide/wrapping_CPlusPlus.html - https://cython.readthedocs.io/en/latest/src/userguide/wrapping_CPlusPlus.html
### Packaging
#### Documentation
- https://packaging.python.org/tutorials/packaging-projects/
- https://setuptools.readthedocs.io/en/latest/userguide/declarative_config.html
- https://test.pypi.org/account/register/
- https://github.com/pypa/auditwheel
- https://github.com/pypa/python-manylinux-demo
- https://github.com/pypa/manylinux
#### C/C++ Extensions
- https://docs.python.org/3/extending/building.html
#### Articles
- https://martinsosic.com/development/2016/02/08/wrapping-c-library-as-python-module.html
- https://malramsay.com/post/perils-of-packaging/
- https://github.com/neuronsimulator/nrn/issues/329
- https://levelup.gitconnected.com/how-to-deploy-a-cython-package-to-pypi-8217a6581f09
- https://medium.com/swlh/distributing-python-packages-protected-with-cython-40fc29d84caf

View File

@ -1,55 +0,0 @@
<html>
<head>
<title>TDMtermite</title>
<style>
</style>
</head>
<body>
<p align="center">
<a href="https://github.com/RecordEvolution/TDMtermite.git">
<img alt="tdmtermite.svg"
src="assets/tdmtermite.svg"
width="400"
/>
</a>
</p>
<div style="width: 100%; display: block; margin-left: 28%; margin-right: 28%;">
<div style="width: 100%; overflow: hidden;">
<div style="margin: 5px; float: left;">
<a href="https://lgtm.com/projects/g/RecordEvolution/TDMtermite/alerts/">
<img alt="Total alerts"
src="https://img.shields.io/lgtm/alerts/g/RecordEvolution/TDMtermite.svg?logo=lgtm&logoWidth=18"/>
</a>
</div>
<div style="margin: 5px; float: left;">
<a href="https://lgtm.com/projects/g/RecordEvolution/TDMtermite/context:cpp">
<img alt="Language grade: C/C++"
src="https://img.shields.io/lgtm/grade/cpp/g/RecordEvolution/TDMtermite.svg?logo=lgtm&logoWidth=18"/>
</a>
</div>
<div style="margin: 5px; float: left;">
<a href="https://lgtm.com/projects/g/RecordEvolution/TDMtermite/context:python">
<img alt="Language grade: Python"
src="https://img.shields.io/lgtm/grade/python/g/RecordEvolution/TDMtermite.svg?logo=lgtm&logoWidth=18"/>
</a>
</div>
</div>
</div>
</body>
</html>

View File

@ -1,100 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
viewBox="0 0 366.72 77.08"
version="1.1"
id="svg3945"
width="366.72"
height="77.080002"
sodipodi:docname="tdmtermite.svg"
inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)">
<sodipodi:namedview
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1"
objecttolerance="10"
gridtolerance="10"
guidetolerance="10"
inkscape:pageopacity="0"
inkscape:pageshadow="2"
inkscape:window-width="1920"
inkscape:window-height="1168"
id="namedview19"
showgrid="false"
inkscape:zoom="3.8766824"
inkscape:cx="199.46638"
inkscape:cy="38.540001"
inkscape:window-x="2048"
inkscape:window-y="0"
inkscape:window-maximized="1"
inkscape:current-layer="svg3945" />
<metadata
id="metadata3951">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title>flasher</dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<defs
id="defs3949" />
<title
id="title3916">flasher</title>
<g
id="logog">
<path
id="path138"
d="m 32.86,2 -13,7.5 v 0 h -0.05 v 0 l -0.48,0.28 c -4.27,2.46 -5.68,11.38 -6.06,14.75 L 36.2,11.33 c 0.39,-0.19 7.6,-3.69 13.57,-3.69 h 0.14 L 40.13,2 a 8.15,8.15 0 0 0 -7.27,0"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path142"
d="M 5.68,17.69 A 8.2,8.2 0 0 0 2,24 v 15.78 c 0,4.9 7,10.48 9.75,12.46 V 25.77 c 0,-0.44 0.6,-8.55 3.65,-13.72 z"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path146"
d="m 12.1,54.12 v 0 C 11.74,53.88 5,49.41 2,44.24 v 11.14 a 8.2,8.2 0 0 0 3.64,6.3 l 13.5,7.79 c 4.28,2.46 12.7,-0.77 15.81,-2.12 z"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path150"
d="m 36.79,68 c -0.4,0.19 -7.71,3.75 -13.71,3.69 l 9.78,5.64 a 8.15,8.15 0 0 0 7.27,0 l 13.51,-7.8 c 4.27,-2.46 5.68,-11.39 6.06,-14.75 z"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path154"
d="M 61.2,27.13 V 53.6 c 0,0.44 -0.6,8.55 -3.65,13.72 l 9.77,-5.64 A 8.2,8.2 0 0 0 71,55.38 V 39.59 c 0,-4.94 -7,-10.5 -9.75,-12.46"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path158"
d="M 67.31,17.69 53.81,9.9 C 49.53,7.44 41.11,10.67 38,12 l 22.85,13.23 v 0 a 43.43,43.43 0 0 1 5.7,4.51 24,24 0 0 1 4.45,5.35 V 24 a 8.2,8.2 0 0 0 -3.64,-6.3"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
</g>
<g
id="re" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;font-size:40px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#364d5c;fill-opacity:1;stroke:none"
x="74.101189"
y="54.47554"
id="text3955"><tspan
id="tspan3953"
x="74.101189"
y="54.47554"
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:44px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#364d5c;fill-opacity:1"><tspan
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold'"
id="tspan86">TDMtermite</tspan> </tspan></text>
</svg>

Before

Width:  |  Height:  |  Size: 4.0 KiB

View File

@ -1,52 +1,51 @@
# distutils: language = c++ # distutils: language = c++
# cython: language_level = 3
from tdmtermite cimport cpptdmtermite
from tdm_reaper cimport tdm_reaper
import json as jn import json as jn
# import numpy as np
cdef class tdmtermite: cdef class tdmreaper:
# C++ instance of class => stack allocated (requires nullary constructor!) # C++ instance of class => stack allocated (requires nullary constructor!)
cdef cpptdmtermite cpptdm cdef tdm_reaper cpp_tdm
# constructor # constructor
def __cinit__(self, string tdmfile, string tdxfile): def __cinit__(self, string tdmfile, string tdxfile):
self.cpptdm = cpptdmtermite(tdmfile,tdxfile) self.cpp_tdm = tdm_reaper(tdmfile,tdxfile)
# provide TDM files # provide TDM files
def submit_files(self,string tdmfile, string tdxfile): def submit_files(self,string tdmfile, string tdxfile):
self.cpptdm.submit_files(tdmfile,tdxfile) self.cpp_tdm.submit_files(tdmfile,tdxfile)
# get list of channel(-group) ids # get list of channel(-group) ids
def get_channelgroup_ids(self): def get_channelgroup_ids(self):
return self.cpptdm.get_channelgroup_ids() return self.cpp_tdm.get_channelgroup_ids()
def get_channel_ids(self): def get_channel_ids(self):
return self.cpptdm.get_channel_ids() return self.cpp_tdm.get_channel_ids()
# get data of specific channel # get data of specific channel
def get_channel(self, string id): def get_channel(self, string id):
return self.cpptdm.get_channel_as_double(id) return self.cpp_tdm.get_channel_as_double(id)
# get meta-data of channel(-group) (as dictionary) # get meta-data of channel(-group) (as dictionary)
def get_channelgroup_info(self, string id): def get_channelgroup_info(self, string id):
grpstr = self.cpptdm.get_channelgroup_info(id) grpstr = self.cpp_tdm.get_channelgroup_info(id)
return jn.loads(grpstr.decode()) return jn.loads(grpstr.decode())
def get_channel_info(self, string id): def get_channel_info(self, string id):
chnstr = self.cpptdm.get_channel_info(id) chnstr = self.cpp_tdm.get_channel_info(id)
return jn.loads(chnstr.decode()) return jn.loads(chnstr.decode())
# print a channel(-group) # print a channel(-group)
def print_channelgroup(self, string id, const char* filename, bool include_meta, def print_channelgroup(self, string id, const char* filename,
char delimiter, string column_header): bool include_meta, char delimiter):
self.cpptdm.print_group(id,filename,include_meta,delimiter,column_header) self.cpp_tdm.print_group(id,filename,include_meta,delimiter)
def print_channel(self, string id, const char* filename, def print_channel(self, string id, const char* filename,
bool include_meta): bool include_meta):
self.cpptdm.print_channel(id,filename,include_meta) self.cpp_tdm.print_channel(id,filename,include_meta)
# print all data grouped by channelgroups # print all data grouped by channelgroups
def write_all(self, string outputdir) : def write_all(self, string outputdir) :
grpids = self.cpptdm.get_channelgroup_ids() grpids = self.cpp_tdm.get_channelgroup_ids()
for id in grpids : for id in grpids :
grpfile = outputdir.decode() + "/channelgroup_" + id.decode() + ".csv" grpfile = outputdir.decode() + "/channelgroup_" + id.decode() + ".csv"
self.cpptdm.print_group(id,grpfile.encode(),True,ord(','),"".encode()) self.cpp_tdm.print_group(id,grpfile.encode(),True,ord(','))

1
cython/requirements.txt Normal file
View File

@ -0,0 +1 @@
Cython==0.29.21

24
cython/setup.py Normal file
View File

@ -0,0 +1,24 @@
from distutils.core import setup
from distutils.extension import Extension
from Cython.Build import cythonize
extensions = Extension(
name="tdm_reaper",
sources=["cython/py_tdm_reaper.pyx"],
# libraries=[""],
# library_dirs=["lib"],
include_dirs=["lib","pugixml"],
language='c++',
extra_compile_args=['-std=c++17','-Wno-unused-variable'],
extra_link_args=['-std=c++17'],
)
setup(
version='0.1',
description='TDMReaper cython extension',
author='Record Evolution GmbH',
author_email='mario.fink@record-evolution.de',
url='https://github.com/RecordEvolution/tdm_ripper.git',
name="tdm_reaper",
ext_modules=cythonize(extensions)
)

View File

@ -1,35 +1,28 @@
# cython: language_level = 3
# use some C++ STL libraries # use some C++ STL libraries
from libcpp.string cimport string from libcpp.string cimport string
from libcpp.vector cimport vector from libcpp.vector cimport vector
from libcpp cimport bool from libcpp cimport bool
cdef extern from "lib/tdm_termite.cpp": cdef extern from "tdm_reaper.cpp":
pass pass
cdef extern from "lib/tdm_termite.hpp": cdef extern from "tdm_reaper.hpp":
cdef cppclass tdm_reaper:
cdef cppclass cpptdmtermite "tdm_termite":
# constructor(s) # constructor(s)
cpptdmtermite() except + tdm_reaper() except +
cpptdmtermite(string tdmfile, string tdxfile) except + tdm_reaper(string tdmfile, string tdxfile) except +
# provide TDM files # provide TDM files
void submit_files(string tdmfile, string tdxfile) except+ void submit_files(string tdmfile, string tdxfile) except+
# get list of channel(-group) ids # get list of channel(-group) ids
vector[string] get_channelgroup_ids() except+ vector[string] get_channelgroup_ids() except+
vector[string] get_channel_ids() except+ vector[string] get_channel_ids() except+
# get data of specific channel # get data of specific channel
vector[double] get_channel_as_double(string id) except+ vector[double] get_channel_as_double(string id) except+
# get meta-data # get meta-data
string get_channelgroup_info(string id) except+ string get_channelgroup_info(string id) except+
string get_channel_info(string id) except+ string get_channel_info(string id) except+
# print a channel(-group) # print a channel(-group)
void print_group(string id, const char* filename, bool include_meta, void print_group(string id, const char* filename, bool include_meta, char delimiter) except+
char delimiter, string column_header) except+
void print_channel(string id, const char* filename, bool include_meta) except+ void print_channel(string id, const char* filename, bool include_meta) except+

View File

@ -14,11 +14,9 @@ typedef unsigned short int eUInt16Usi;
typedef unsigned int eUInt32Usi; typedef unsigned int eUInt32Usi;
typedef float eFloat32Usi; typedef float eFloat32Usi;
typedef double eFloat64Usi; typedef double eFloat64Usi;
typedef char eStringUsi;
class tdmdatatype class tdmdatatype
{ {
protected: protected:
eInt16Usi sint16_; // 0 eInt16Usi sint16_; // 0
eInt32Usi sint32_; // 1 eInt32Usi sint32_; // 1
@ -27,13 +25,11 @@ protected:
eUInt32Usi uint32_; // 4 eUInt32Usi uint32_; // 4
eFloat32Usi float32_; // 5 eFloat32Usi float32_; // 5
eFloat64Usi float64_; // 6 eFloat64Usi float64_; // 6
eStringUsi string_; // 7 short int dtidx_; // \in \{0,...,6\}
short int dtidx_; // \in \{0,...,7\}
public: public:
tdmdatatype(): sint16_(0), sint32_(0), tdmdatatype(): sint16_(0), sint32_(0),
uint8_(0), uint16_(0), uint32_(0), uint8_(0), uint16_(0), uint32_(0),
float32_(0.0), float64_(0.0), string_(0), float32_(0.0), float64_(0.0),
dtidx_(0) { }; dtidx_(0) { };
// every supported datatype gets its own constructor // every supported datatype gets its own constructor
tdmdatatype(eInt16Usi num): sint16_(num), dtidx_(0) {}; tdmdatatype(eInt16Usi num): sint16_(num), dtidx_(0) {};
@ -43,25 +39,10 @@ public:
tdmdatatype(eUInt32Usi num): uint32_(num), dtidx_(4) {}; tdmdatatype(eUInt32Usi num): uint32_(num), dtidx_(4) {};
tdmdatatype(eFloat32Usi num): float32_(num), dtidx_(5) {}; tdmdatatype(eFloat32Usi num): float32_(num), dtidx_(5) {};
tdmdatatype(eFloat64Usi num): float64_(num), dtidx_(6) {}; tdmdatatype(eFloat64Usi num): float64_(num), dtidx_(6) {};
tdmdatatype(eStringUsi num): string_(num), dtidx_(7) {};
// identify type // identify type
short int& dtype() { return dtidx_; } short int& dtype() { return dtidx_; }
// copy constructor
tdmdatatype(const tdmdatatype &num)
{
this->sint16_ = num.sint16_;
this->sint32_ = num.sint32_;
this->uint8_ = num.uint8_;
this->uint16_ = num.uint16_;
this->uint32_ = num.uint32_;
this->float32_ = num.float32_;
this->float64_ = num.float64_;
this->string_ = num.string_;
this->dtidx_ = num.dtidx_;
}
// overall assignment operator // overall assignment operator
tdmdatatype& operator=(const tdmdatatype &num) tdmdatatype& operator=(const tdmdatatype &num)
{ {
@ -74,11 +55,9 @@ public:
this->uint32_ = num.uint32_; this->uint32_ = num.uint32_;
this->float32_ = num.float32_; this->float32_ = num.float32_;
this->float64_ = num.float64_; this->float64_ = num.float64_;
this->string_ = num.string_;
this->dtidx_ = num.dtidx_;
} }
return *this; return *this;
} }
// implement assignment operator for individual datatypes // implement assignment operator for individual datatypes
@ -124,12 +103,6 @@ public:
this->dtidx_ = 6; this->dtidx_ = 6;
return *this; return *this;
} }
tdmdatatype& operator=(const eStringUsi &num)
{
this->string_ = num;
this->dtidx_ = 7;
return *this;
}
// obtain number as double // obtain number as double
double as_double() double as_double()
@ -142,7 +115,6 @@ public:
else if ( dtidx_ == 4 ) num = (double)uint32_; else if ( dtidx_ == 4 ) num = (double)uint32_;
else if ( dtidx_ == 5 ) num = (double)float32_; else if ( dtidx_ == 5 ) num = (double)float32_;
else if ( dtidx_ == 6 ) num = (double)float64_; else if ( dtidx_ == 6 ) num = (double)float64_;
else if ( dtidx_ == 7 ) num = (double)(int)string_;
return num; return num;
} }
@ -156,7 +128,6 @@ public:
else if ( num.dtidx_ == 4 ) out<<num.uint32_; else if ( num.dtidx_ == 4 ) out<<num.uint32_;
else if ( num.dtidx_ == 5 ) out<<num.float32_; else if ( num.dtidx_ == 5 ) out<<num.float32_;
else if ( num.dtidx_ == 6 ) out<<num.float64_; else if ( num.dtidx_ == 6 ) out<<num.float64_;
else if ( num.dtidx_ == 7 ) out<<num.string_;
return out; return out;
} }
@ -345,7 +316,7 @@ const std::vector<tdm_datatype> tdm_datatypes = {
{"eFloat32Usi","DT_FLOAT",3,"float_sequence",4,"32 bit float"}, {"eFloat32Usi","DT_FLOAT",3,"float_sequence",4,"32 bit float"},
{"eFloat64Usi","DT_DOUBLE",7,"double_sequence",8,"64 bit double"}, {"eFloat64Usi","DT_DOUBLE",7,"double_sequence",8,"64 bit double"},
{"eStringUsi","DT_STRING",1,"string_sequence",1,"text"} // {"eStringUsi","DT_STRING",1,"string_sequence",0,"text"}
}; };

View File

@ -1,97 +1,32 @@
// -------------------------------------------------------------------------- // // -------------------------------------------------------------------------- //
#include "tdm_termite.hpp" #include "tdm_reaper.hpp"
// -------------------------------------------------------------------------- // // -------------------------------------------------------------------------- //
tdm_termite::tdm_termite() tdm_reaper::tdm_reaper()
{ {
tdx_ifstream_ = new std::ifstream;
} }
tdm_termite::tdm_termite(std::string tdmfile, std::string tdxfile, bool showlog): tdm_reaper::tdm_reaper(std::string tdmfile, std::string tdxfile, bool showlog):
tdmfile_(tdmfile), tdxfile_(tdxfile) tdmfile_(tdmfile), tdxfile_(tdxfile)
{ {
tdx_ifstream_ = new std::ifstream;
// start processing tdm data model // start processing tdm data model
this->process_tdm(showlog); this->process_tdm(showlog);
} }
tdm_termite::~tdm_termite() void tdm_reaper::submit_files(std::string tdmfile, std::string tdxfile, bool showlog)
{
// close tdx-file stream and free memory
if ( tdx_ifstream_->is_open() ) tdx_ifstream_->close();
delete tdx_ifstream_;
}
tdm_termite::tdm_termite(const tdm_termite& other):
tdmfile_(other.tdmfile_), tdxfile_(other.tdxfile_), csvfile_(other.csvfile_),
endianness_(other.endianness_), machine_endianness_(other.machine_endianness_),
meta_data_(other.meta_data_), tdmdt_name_(other.tdmdt_name_),
tdmdt_chan_(other.tdmdt_chan_),
tdx_blocks_(other.tdx_blocks_), tdmroot_(other.tdmroot_),
tdmchannelgroups_(other.tdmchannelgroups_), tdmchannels_(other.tdmchannels_),
tdmchannels_data_(other.tdmchannels_data_), submatrices_(other.submatrices_),
localcolumns_(other.localcolumns_), tdxbuffer_(other.tdxbuffer_)
{
tdx_ifstream_ = new std::ifstream;
if ( other.tdx_ifstream_->is_open() )
{
tdx_ifstream_->open(tdxfile_.c_str(),std::ifstream::binary);
tdx_ifstream_->seekg(other.tdx_ifstream_->tellg());
}
}
tdm_termite& tdm_termite::operator=(const tdm_termite& other)
{
if ( this != &other )
{
tdmfile_ = other.tdmfile_;
tdxfile_ = other.tdxfile_;
csvfile_ = other.csvfile_;
endianness_ = other.endianness_;
machine_endianness_ = other.machine_endianness_;
meta_data_ = other.meta_data_;
tdmdt_name_ = other.tdmdt_name_;
tdmdt_chan_= other.tdmdt_chan_;
tdx_blocks_ = other.tdx_blocks_;
tdmroot_ = other.tdmroot_;
tdmchannelgroups_ = other.tdmchannelgroups_;
tdmchannels_ = other.tdmchannels_;
tdmchannels_data_ = other.tdmchannels_data_;
submatrices_ = other.submatrices_;
localcolumns_ = other.localcolumns_;
tdxbuffer_ = other.tdxbuffer_;
if ( other.tdx_ifstream_->is_open() )
{
tdx_ifstream_->open(tdxfile_.c_str(),std::ifstream::binary);
tdx_ifstream_->seekg(other.tdx_ifstream_->tellg());
}
}
return *this;
}
void tdm_termite::submit_files(std::string tdmfile, std::string tdxfile, bool showlog)
{ {
// save files // save files
tdmfile_ = tdmfile; tdmfile_ = tdmfile;
tdxfile_ = tdxfile; tdxfile_ = tdxfile;
// reset and clear all members
tdx_blocks_.clear();
tdmchannelgroups_.clear();
tdmchannels_.clear();
submatrices_.clear();
localcolumns_.clear();
tdxbuffer_.clear();
// start processing tdm data model // start processing tdm data model
this->process_tdm(showlog); this->process_tdm(showlog);
} }
void tdm_termite::process_tdm(bool showlog) void tdm_reaper::process_tdm(bool showlog)
{ {
// check both tdm, tdx files // check both tdm, tdx files
std::filesystem::path ptdm(tdmfile_), ptdx(tdxfile_); std::filesystem::path ptdm(tdmfile_), ptdx(tdxfile_);
@ -163,62 +98,37 @@ void tdm_termite::process_tdm(bool showlog)
this->process_localcolumns(showlog,xml_doc); this->process_localcolumns(showlog,xml_doc);
// open .tdx and stream all binary data into buffer // open .tdx and stream all binary data into buffer
// try {
// std::ifstream fin(tdxfile_.c_str(),std::ifstream::binary);
// // if ( !fin.good() ) std::cerr<<"failed to open .tdx-file\n";
//
// std::vector<unsigned char> tdxbuf((std::istreambuf_iterator<char>(fin)),
// (std::istreambuf_iterator<char>()));
// tdxbuffer_ = tdxbuf;
//
// if ( showlog ) std::cout<<"size of .tdx buffer (bytes): "<<tdxbuffer_.size()<<"\n\n";
//
// // close .tdx file
// fin.close();
// } catch (const std::exception& e ) {
// throw std::runtime_error( std::string("failed to open .tdx and stream data to buffer: ")
// + e.what() );
// }
try { try {
tdx_ifstream_->open(tdxfile_.c_str(),std::ifstream::binary); std::ifstream fin(tdxfile_.c_str(),std::ifstream::binary);
} catch (const std::exception& e) { // if ( !fin.good() ) std::cerr<<"failed to open .tdx-file\n";
throw std::runtime_error( std::string("failed to open .tdx file in ifstream: ")
std::vector<unsigned char> tdxbuf((std::istreambuf_iterator<char>(fin)),
(std::istreambuf_iterator<char>()));
tdxbuffer_ = tdxbuf;
if ( showlog ) std::cout<<"size of .tdx buffer (bytes): "<<tdxbuffer_.size()<<"\n\n";
// close .tdx file
fin.close();
} catch (const std::exception& e ) {
throw std::runtime_error( std::string("failed to open .tdx and stream data to buffer: ")
+ e.what() ); + e.what() );
} }
} }
void tdm_termite::process_include(bool showlog, pugi::xml_document& xml_doc) void tdm_reaper::process_include(bool showlog, pugi::xml_document& xml_doc)
{ {
// get XML node // get XML node
pugi::xml_node tdmincl = xml_doc.child("usi:tdm").child("usi:include"); pugi::xml_node tdmincl = xml_doc.child("usi:tdm").child("usi:include");
// check endianness // check endianness
std::string endianness(tdmincl.child("file").attribute("byteOrder").value()); std::string endianness(tdmincl.child("file").attribute("byteOrder").value());
// endianness_ = endianness.compare("littleEndian") == 0 ? true : false; endianness_ = endianness.compare("littleEndian") == 0 ? true : false;
if ( endianness.compare("littleEndian") == 0 )
{
endianness_ = true;
}
else if ( endianness.compare("bigEndian") == 0 )
{
endianness_ = false;
}
else
{
throw std::runtime_error(std::string("unsupported endianness: ") + endianness);
}
// obtain machine's endianness // obtain machine's endianness
machine_endianness_ = this->detect_endianness(); int num = 1;
// if ( machine_endianness_ != endianness_ ) machine_endianness_ = ( *(char*)&num == 1 );
// { if ( machine_endianness_ != endianness_ ) throw std::runtime_error("endianness mismatch");
// std::stringstream ss;
// ss<<"endianness mismatch: "<<"TDM = "<<(endianness_?"little":"big")
// <<" , "
// <<"Arch = "<<(machine_endianness_?"little":"big");
// // std::cout<<ss.str()<<"\n";
// // throw std::runtime_error(ss.str());
// }
// list block of massdata // list block of massdata
for (pugi::xml_node anode: tdmincl.child("file").children()) for (pugi::xml_node anode: tdmincl.child("file").children())
@ -260,7 +170,7 @@ void tdm_termite::process_include(bool showlog, pugi::xml_document& xml_doc)
if ( showlog ) std::cout<<"number of blocks: "<<tdx_blocks_.size()<<"\n\n"; if ( showlog ) std::cout<<"number of blocks: "<<tdx_blocks_.size()<<"\n\n";
} }
void tdm_termite::process_root(bool showlog, pugi::xml_document& xml_doc) void tdm_reaper::process_root(bool showlog, pugi::xml_document& xml_doc)
{ {
// get XML node // get XML node
pugi::xml_node tdmdataroot = xml_doc.child("usi:tdm").child("usi:data") pugi::xml_node tdmdataroot = xml_doc.child("usi:tdm").child("usi:data")
@ -280,7 +190,7 @@ void tdm_termite::process_root(bool showlog, pugi::xml_document& xml_doc)
if ( showlog ) std::cout<<tdmroot_.get_info()<<"\n"; if ( showlog ) std::cout<<tdmroot_.get_info()<<"\n";
} }
void tdm_termite::process_channelgroups(bool showlog, pugi::xml_document& xml_doc) void tdm_reaper::process_channelgroups(bool showlog, pugi::xml_document& xml_doc)
{ {
// get XML node <usi:data> // get XML node <usi:data>
pugi::xml_node tdmdata = xml_doc.child("usi:tdm").child("usi:data"); pugi::xml_node tdmdata = xml_doc.child("usi:tdm").child("usi:data");
@ -318,7 +228,7 @@ void tdm_termite::process_channelgroups(bool showlog, pugi::xml_document& xml_do
if ( showlog ) std::cout<<"number of channelgroups: "<<tdmchannelgroups_.size()<<"\n\n"; if ( showlog ) std::cout<<"number of channelgroups: "<<tdmchannelgroups_.size()<<"\n\n";
} }
void tdm_termite::process_channels(bool showlog, pugi::xml_document& xml_doc) void tdm_reaper::process_channels(bool showlog, pugi::xml_document& xml_doc)
{ {
// get XML node <usi:data> // get XML node <usi:data>
pugi::xml_node tdmdata = xml_doc.child("usi:tdm").child("usi:data"); pugi::xml_node tdmdata = xml_doc.child("usi:tdm").child("usi:data");
@ -362,7 +272,7 @@ void tdm_termite::process_channels(bool showlog, pugi::xml_document& xml_doc)
if ( showlog ) std::cout<<"number of channels: "<<tdmchannels_.size()<<"\n\n"; if ( showlog ) std::cout<<"number of channels: "<<tdmchannels_.size()<<"\n\n";
} }
void tdm_termite::process_submatrices(bool showlog, pugi::xml_document& xml_doc) void tdm_reaper::process_submatrices(bool showlog, pugi::xml_document& xml_doc)
{ {
// get XML node <usi:data> // get XML node <usi:data>
pugi::xml_node tdmdata = xml_doc.child("usi:tdm").child("usi:data"); pugi::xml_node tdmdata = xml_doc.child("usi:tdm").child("usi:data");
@ -401,7 +311,7 @@ void tdm_termite::process_submatrices(bool showlog, pugi::xml_document& xml_doc)
if ( showlog ) std::cout<<"number of submatrices: "<<submatrices_.size()<<"\n\n"; if ( showlog ) std::cout<<"number of submatrices: "<<submatrices_.size()<<"\n\n";
} }
void tdm_termite::process_localcolumns(bool showlog, pugi::xml_document& xml_doc) void tdm_reaper::process_localcolumns(bool showlog, pugi::xml_document& xml_doc)
{ {
// get XML node <usi:data> // get XML node <usi:data>
pugi::xml_node tdmdata = xml_doc.child("usi:tdm").child("usi:data"); pugi::xml_node tdmdata = xml_doc.child("usi:tdm").child("usi:data");
@ -464,14 +374,9 @@ void tdm_termite::process_localcolumns(bool showlog, pugi::xml_document& xml_doc
{ {
locc.values_ = vl.at(0); locc.values_ = vl.at(0);
} }
else if ( vl.size() == 0 )
{
//std::cerr<<"localcolumn ("<<locc.id_<<","<<locc.name_<<") misses any value-ids"<<"\n";
locc.values_ = "none";
}
else else
{ {
throw std::logic_error("localcolumn with multiple values id(s)"); throw std::logic_error("localcolumn with out/multiple values id(s)");
} }
// add external id referring to block in <usi:include> // add external id referring to block in <usi:include>
@ -484,7 +389,7 @@ void tdm_termite::process_localcolumns(bool showlog, pugi::xml_document& xml_doc
{ {
throw std::runtime_error(std::string("measurement_quantity: ") throw std::runtime_error(std::string("measurement_quantity: ")
+ locc.measurement_quantity_ + locc.measurement_quantity_
+ std::string(" is missing/ambiguous") ); + std::string(" is ambiguous") );
} }
std::string dt = tdmchannels_.at(locc.measurement_quantity_).datatype_; std::string dt = tdmchannels_.at(locc.measurement_quantity_).datatype_;
std::string sequence_type; std::string sequence_type;
@ -506,10 +411,8 @@ void tdm_termite::process_localcolumns(bool showlog, pugi::xml_document& xml_doc
if ( locc.external_id_.empty() ) if ( locc.external_id_.empty() )
{ {
//throw std::logic_error( std::string("no external id found for ") throw std::logic_error( std::string("no external id found for ")
// + sequence_type + std::string(" with ") + locc.values_ ); + sequence_type + std::string(" with ") + locc.values_ );
//std::cerr<<"no external id found for "<<sequence_type<<" with "<<locc.values_<<"\n";
locc.external_id_ = "none";
} }
} }
@ -524,7 +427,7 @@ void tdm_termite::process_localcolumns(bool showlog, pugi::xml_document& xml_doc
// -------------------------------------------------------------------------- // // -------------------------------------------------------------------------- //
std::string tdm_termite::get_channel_overview(format chformatter) std::string tdm_reaper::get_channel_overview(format chformatter)
{ {
// summarize all output in single string // summarize all output in single string
std::string channels_summary; std::string channels_summary;
@ -534,8 +437,8 @@ std::string tdm_termite::get_channel_overview(format chformatter)
// compose header // compose header
chformatter.set_header(true); chformatter.set_header(true);
//tdm_channelgroup grp; tdm_channelgroup grp;
//channels_summary += grp.get_info(chformatter); channels_summary += grp.get_info(chformatter);
tdm_channel chn; tdm_channel chn;
channels_summary += chn.get_info(chformatter); channels_summary += chn.get_info(chformatter);
std::string rule; // = std::string("#"); std::string rule; // = std::string("#");
@ -552,9 +455,8 @@ std::string tdm_termite::get_channel_overview(format chformatter)
it!=tdmchannels_.end(); ++it) it!=tdmchannels_.end(); ++it)
{ {
// get corresponding group // get corresponding group
// tdm_channelgroup grp = tdmchannelgroups_.at(it->second.group_); tdm_channelgroup grp = tdmchannelgroups_.at(it->second.group_);
// channels_summary += grp.get_info(chformatter); channels_summary += grp.get_info(chformatter);
// ...and actual channel // ...and actual channel
channels_summary += it->second.get_info(chformatter); channels_summary += it->second.get_info(chformatter);
channels_summary += std::string("\n"); channels_summary += std::string("\n");
@ -564,7 +466,7 @@ std::string tdm_termite::get_channel_overview(format chformatter)
} }
template<typename tdmelement> template<typename tdmelement>
std::string tdm_termite::get_overview(format formatter) std::string tdm_reaper::get_overview(format formatter)
{ {
// summarize all output in single string // summarize all output in single string
std::string summary; std::string summary;
@ -590,12 +492,12 @@ std::string tdm_termite::get_overview(format formatter)
return summary; return summary;
} }
template std::string tdm_termite::get_overview<tdm_channelgroup>(format formatter); template std::string tdm_reaper::get_overview<tdm_channelgroup>(format formatter);
template std::string tdm_termite::get_overview<submatrix>(format formatter); template std::string tdm_reaper::get_overview<submatrix>(format formatter);
template std::string tdm_termite::get_overview<localcolumn>(format formatter); template std::string tdm_reaper::get_overview<localcolumn>(format formatter);
template std::string tdm_termite::get_overview<block>(format formatter); template std::string tdm_reaper::get_overview<block>(format formatter);
void tdm_termite::summarize_member(tdm_channelgroup &chp, std::string& summary, format& formatter) void tdm_reaper::summarize_member(tdm_channelgroup chp, std::string& summary, format& formatter)
{ {
for ( std::map<std::string,tdm_channelgroup>::iterator it=this->tdmchannelgroups_.begin(); for ( std::map<std::string,tdm_channelgroup>::iterator it=this->tdmchannelgroups_.begin();
it!=this->tdmchannelgroups_.end(); ++it) it!=this->tdmchannelgroups_.end(); ++it)
@ -605,7 +507,7 @@ void tdm_termite::summarize_member(tdm_channelgroup &chp, std::string& summary,
} }
} }
void tdm_termite::summarize_member(submatrix &sbm, std::string& summary, format& formatter) void tdm_reaper::summarize_member(submatrix sbm, std::string& summary, format& formatter)
{ {
for ( std::map<std::string,submatrix>::iterator it=this->submatrices_.begin(); for ( std::map<std::string,submatrix>::iterator it=this->submatrices_.begin();
it!=this->submatrices_.end(); ++it) it!=this->submatrices_.end(); ++it)
@ -615,7 +517,7 @@ void tdm_termite::summarize_member(submatrix &sbm, std::string& summary, format&
} }
} }
void tdm_termite::summarize_member(localcolumn &lcc, std::string& summary, format& formatter) void tdm_reaper::summarize_member(localcolumn lcc, std::string& summary, format& formatter)
{ {
for ( std::map<std::string,localcolumn>::iterator it=this->localcolumns_.begin(); for ( std::map<std::string,localcolumn>::iterator it=this->localcolumns_.begin();
it!=this->localcolumns_.end(); ++it) it!=this->localcolumns_.end(); ++it)
@ -625,7 +527,7 @@ void tdm_termite::summarize_member(localcolumn &lcc, std::string& summary, forma
} }
} }
void tdm_termite::summarize_member(block &blk, std::string& summary, format& formatter) void tdm_reaper::summarize_member(block blk, std::string& summary, format& formatter)
{ {
for ( std::map<std::string,block>::iterator it=this->tdx_blocks_.begin(); for ( std::map<std::string,block>::iterator it=this->tdx_blocks_.begin();
it!=this->tdx_blocks_.end(); ++it) it!=this->tdx_blocks_.end(); ++it)
@ -638,7 +540,7 @@ void tdm_termite::summarize_member(block &blk, std::string& summary, format& for
// -------------------------------------------------------------------------- // // -------------------------------------------------------------------------- //
// extract channel by id // extract channel by id
std::vector<tdmdatatype> tdm_termite::get_channel(std::string& id) std::vector<tdmdatatype> tdm_reaper::get_channel(std::string& id)
{ {
// check for existence of required channel id (=key) // check for existence of required channel id (=key)
if ( tdmchannels_.count(id) == 1 ) if ( tdmchannels_.count(id) == 1 )
@ -646,7 +548,7 @@ std::vector<tdmdatatype> tdm_termite::get_channel(std::string& id)
// retrieve full channel info // retrieve full channel info
tdm_channel chn = tdmchannels_.at(id); tdm_channel chn = tdmchannels_.at(id);
// extract (first) "localcolumn" for channel TODO there should only be a single!! local_column!! // extract (first) "localcolumn" for channel
if ( chn.local_columns_.size() != 1 ) if ( chn.local_columns_.size() != 1 )
{ {
throw std::runtime_error(std::string("invalid local_columns_ of channel: ") + id); throw std::runtime_error(std::string("invalid local_columns_ of channel: ") + id);
@ -663,28 +565,8 @@ std::vector<tdmdatatype> tdm_termite::get_channel(std::string& id)
} }
// use "values" id to map to external block // use "values" id to map to external block
if ( loccol.external_id_ == "none" )
{
//throw std::runtime_error(std::string("missing external_id in local_column ")+loccol.id_);
//std::cerr<<"missing external_id in local_column "<<loccol.id_<<"\n";
return std::vector<tdmdatatype>(0);
}
block blk = tdx_blocks_.at(loccol.external_id_); block blk = tdx_blocks_.at(loccol.external_id_);
// find corresponding submatrix
if ( submatrices_.count(loccol.submatrix_) != 1 )
{
throw std::runtime_error(std::string("no associated submatrix for localcolumn found: ") + loccol.id_);
}
submatrix subm = submatrices_.at(loccol.submatrix_);
if ( subm.number_of_rows_ != blk.length_ )
{
std::stringstream ss;
ss<<"number of rows in submatrix "<<subm.id_<<" ("<<subm.number_of_rows_<<") "
<<" does not agree with length of associated block "<<blk.id_<<" ("<<blk.length_<<")";
throw std::runtime_error(ss.str());
}
// declare vector of appropriate length // declare vector of appropriate length
std::vector<tdmdatatype> datavec(blk.length_); std::vector<tdmdatatype> datavec(blk.length_);
@ -694,20 +576,10 @@ std::vector<tdmdatatype> tdm_termite::get_channel(std::string& id)
// declare buffer covering the required range of "tdxbuffer_" // declare buffer covering the required range of "tdxbuffer_"
// (consider both channel-wise and block-wise ordering) // (consider both channel-wise and block-wise ordering)
unsigned long int strtidx = blk.block_offset_*blk.block_size_ unsigned long int strtidx = blk.block_offset_*blk.block_size_
+ blk.byte_offset_; + blk.byte_offset_,
// fnshidx = strtidx + blk.length_*dtyp.size_; fnshidx = strtidx + blk.length_*dtyp.size_;
// std::vector<unsigned char> tdxblk( tdxbuffer_.begin()+strtidx, std::vector<unsigned char> tdxblk( tdxbuffer_.begin()+strtidx,
// tdxbuffer_.begin()+fnshidx ); tdxbuffer_.begin()+fnshidx );
char* blkbuf = new char[blk.length_*dtyp.size_];
try {
tdx_ifstream_->seekg(strtidx);
tdx_ifstream_->read(blkbuf,blk.length_*dtyp.size_);
} catch (const std::exception& e) {
throw std::runtime_error( std::string("failed to read block from tdx_ifstream_: ")
+ e.what() );
}
std::vector<unsigned char> tdxblk(blkbuf,blkbuf+blk.length_*dtyp.size_);
delete []blkbuf;
// distinguish numeric datatypes included in "tdmdatatype" // distinguish numeric datatypes included in "tdmdatatype"
if ( blk.value_type_ == std::string("eInt16Usi") ) if ( blk.value_type_ == std::string("eInt16Usi") )
@ -738,10 +610,6 @@ std::vector<tdmdatatype> tdm_termite::get_channel(std::string& id)
{ {
this->convert_data_to_type<eFloat64Usi>(tdxblk,datavec); this->convert_data_to_type<eFloat64Usi>(tdxblk,datavec);
} }
else if ( blk.value_type_ == std::string("eStringUsi") )
{
this->convert_data_to_type<eStringUsi>(tdxblk,datavec);
}
else else
{ {
throw std::runtime_error(std::string("unsupported/unknown datatype") + blk.value_type_); throw std::runtime_error(std::string("unsupported/unknown datatype") + blk.value_type_);
@ -777,7 +645,7 @@ std::vector<tdmdatatype> tdm_termite::get_channel(std::string& id)
// -------------------------------------------------------------------------- // // -------------------------------------------------------------------------- //
void tdm_termite::print_channel(std::string &id, const char* filename, bool include_meta) void tdm_reaper::print_channel(std::string &id, const char* filename, bool include_meta)
{ {
// check required path // check required path
this->check_filename_path(filename); this->check_filename_path(filename);
@ -822,8 +690,7 @@ void tdm_termite::print_channel(std::string &id, const char* filename, bool incl
} }
} }
void tdm_termite::print_group(std::string &id, const char* filename, bool include_meta, void tdm_reaper::print_group(std::string &id, const char* filename, bool include_meta, char sep)
char sep, std::string column_header)
{ {
// check required path // check required path
this->check_filename_path(filename); this->check_filename_path(filename);
@ -849,7 +716,6 @@ void tdm_termite::print_group(std::string &id, const char* filename, bool includ
int width = 25; int width = 25;
// file header
if ( include_meta ) if ( include_meta )
{ {
// group meta data // group meta data
@ -903,34 +769,10 @@ void tdm_termite::print_group(std::string &id, const char* filename, bool includ
for ( std::string chn: chngrp.channels_ ) for ( std::string chn: chngrp.channels_ )
{ {
std::vector<tdmdatatype> chndat = this->get_channel(chn); std::vector<tdmdatatype> chndat = this->get_channel(chn);
if ( chndat.size() > maxrows ) maxrows = (unsigned int)chndat.size(); if ( chndat.size() > maxrows ) maxrows = chndat.size();
allchns.push_back(chndat); allchns.push_back(chndat);
} }
// provide column header (with channel ids)
if ( column_header.empty() )
{
for ( std::string chn: chngrp.channels_ )
{
// use given csv separator token
if ( sep == ' ' )
{
fou<<std::setw(width)<<std::left<<chn;
}
else
{
fou<<chn;
}
if ( chn != chngrp.channels_.back() && sep != ' ' ) fou<<sep;
}
}
else
{
fou<<column_header;
}
fou<<"\n";
// write data
for ( unsigned int row = 0; row < maxrows; row++ ) for ( unsigned int row = 0; row < maxrows; row++ )
{ {
for ( unsigned int chi = 0; chi < chngrp.channels_.size(); chi++ ) for ( unsigned int chi = 0; chi < chngrp.channels_.size(); chi++ )
@ -947,8 +789,19 @@ void tdm_termite::print_group(std::string &id, const char* filename, bool includ
fou<<allchns.at(chi).at(row); fou<<allchns.at(chi).at(row);
} }
} }
else
{
if ( sep == ' ' )
{
fou<<std::setw(width)<<std::left<<"";
}
else
{
fou<<sep;
}
}
if ( chi+1 < chngrp.channels_.size() && sep != ' ' ) fou<<sep; if ( chi+1 < chngrp.channels_.size() ) fou<<sep;
} }
fou<<"\n"; fou<<"\n";
} }
@ -958,7 +811,7 @@ void tdm_termite::print_group(std::string &id, const char* filename, bool includ
} }
} }
void tdm_termite::check_filename_path(const char* filename) void tdm_reaper::check_filename_path(const char* filename)
{ {
// declare filesystem path instance from filename // declare filesystem path instance from filename
std::filesystem::path pt(filename); std::filesystem::path pt(filename);
@ -968,13 +821,13 @@ void tdm_termite::check_filename_path(const char* filename)
if ( !std::filesystem::is_directory(pt) ) if ( !std::filesystem::is_directory(pt) )
{ {
throw std::runtime_error( std::string("directory does not exist: ") + pt.u8string() ); throw std::runtime_error(std::string("directory does not exist: ") + pt.c_str() );
} }
} }
// -------------------------------------------------------------------------- // // -------------------------------------------------------------------------- //
void tdm_termite::check_local_datatypes() void tdm_reaper::check_local_datatypes()
{ {
std::cout<<"\nmachine's C++ datatypes:\n"; std::cout<<"\nmachine's C++ datatypes:\n";
std::cout<<std::setw(25)<<std::left<<"char:" std::cout<<std::setw(25)<<std::left<<"char:"
@ -1005,7 +858,7 @@ void tdm_termite::check_local_datatypes()
<<std::setw(5)<<std::left<<sizeof(long double)<<"byte(s)\n\n"; <<std::setw(5)<<std::left<<sizeof(long double)<<"byte(s)\n\n";
} }
void tdm_termite::check_datatype_consistency() void tdm_reaper::check_datatype_consistency()
{ {
// check datatype consistency, i.e. "local" representation of datatypes // check datatype consistency, i.e. "local" representation of datatypes
for ( tdm_datatype el: tdm_datatypes ) for ( tdm_datatype el: tdm_datatypes )
@ -1038,10 +891,6 @@ void tdm_termite::check_datatype_consistency()
{ {
if ( el.size_ != sizeof(eFloat64Usi) ) throw std::logic_error("invalid representation of eFloat64Usi"); if ( el.size_ != sizeof(eFloat64Usi) ) throw std::logic_error("invalid representation of eFloat64Usi");
} }
else if ( el.name_ == "eStringUsi" )
{
if ( el.size_ != sizeof(eStringUsi) ) throw std::logic_error("invalid representation of eStringUsi");
}
else else
{ {
throw std::logic_error("missing datatype validation"); throw std::logic_error("missing datatype validation");
@ -1052,7 +901,7 @@ void tdm_termite::check_datatype_consistency()
// -------------------------------------------------------------------------- // // -------------------------------------------------------------------------- //
template<typename datatype> template<typename datatype>
void tdm_termite::convert_data_to_type(std::vector<unsigned char> &buffer, void tdm_reaper::convert_data_to_type(std::vector<unsigned char> &buffer,
std::vector<tdmdatatype> &channel) std::vector<tdmdatatype> &channel)
{ {
// check number of elements of type "datatype" in buffer // check number of elements of type "datatype" in buffer
@ -1071,15 +920,7 @@ void tdm_termite::convert_data_to_type(std::vector<unsigned char> &buffer,
for ( unsigned long int j = 0; j < sizeof(datatype); j++ ) for ( unsigned long int j = 0; j < sizeof(datatype); j++ )
{ {
// matching byte order between TDM/TDX and machine's architecture ? dfcast[j] = (int)buffer[i*sizeof(datatype)+j];
if ( machine_endianness_ == endianness_ )
{
dfcast[j] = (int)buffer[i*sizeof(datatype)+j];
}
else
{
dfcast[j] = (int)buffer[(i+1)*sizeof(datatype)-(j+1)];
}
} }
// save number in channel // save number in channel

View File

@ -1,7 +1,7 @@
// ------------------------------------------------------------------------- // // ------------------------------------------------------------------------- //
#ifndef TDM_TERMITE #ifndef TDM_REAPER
#define TDM_TERMITE #define TDM_REAPER
#include <iostream> #include <iostream>
#include <fstream> #include <fstream>
@ -24,7 +24,7 @@
// -------------------------------------------------------------------------- // // -------------------------------------------------------------------------- //
class tdm_termite class tdm_reaper
{ {
// .tdm and .tdx paths/filenames // .tdm and .tdx paths/filenames
std::string tdmfile_; std::string tdmfile_;
@ -39,13 +39,13 @@ class tdm_termite
// tdm meta-data // tdm meta-data
tdm_meta meta_data_; tdm_meta meta_data_;
// blocks of data in .tdx file
std::map<std::string,block> tdx_blocks_;
// resconstruct "tdm_datatype.hpp: tdm_datatypes" as map to quickly map // resconstruct "tdm_datatype.hpp: tdm_datatypes" as map to quickly map
// "valueType"/"channel_datatype" to full datatype // "valueType"/"channel_datatype" to full datatype
std::map<std::string,tdm_datatype> tdmdt_name_, tdmdt_chan_; std::map<std::string,tdm_datatype> tdmdt_name_, tdmdt_chan_;
// blocks of data in .tdx file
std::map<std::string,block> tdx_blocks_;
// tdm root // tdm root
tdm_root tdmroot_; tdm_root tdmroot_;
@ -54,38 +54,13 @@ class tdm_termite
// tdm channels // tdm channels
std::map<std::string,tdm_channel> tdmchannels_; std::map<std::string,tdm_channel> tdmchannels_;
std::map<std::string,std::vector<tdm_datatype>> tdmchannels_data_;
// submatrices and local_columns // submatrices and local_columns
std::map<std::string,submatrix> submatrices_; std::map<std::string,submatrix> submatrices_;
std::map<std::string,localcolumn> localcolumns_; std::map<std::string,localcolumn> localcolumns_;
// binary data container/file stream // binary data container
std::vector<unsigned char> tdxbuffer_; std::vector<unsigned char> tdxbuffer_;
std::ifstream *tdx_ifstream_;
// find machine's endianness at runtime
// detect machine endianness (C++20 !!)
// if ( std::endian::native == std::endian::little )
// {
// machine_endianness_ = true;
// }
// else if ( std::endian::native == std::endian::big )
// {
// machine_endianness_ = false;
// }
// else
// {
// throw std::runtime_error("mixed endianness architecture is not supported");
// }
bool detect_endianness()
{
// int num = 1;
// machine_endianness_ = ( *(char*)&num == 1 );
std::uint32_t num = 0x11223344;
uint8_t* dfc = reinterpret_cast<uint8_t*>(&num);
return ( dfc[0] == 0x44 );
}
// extract list of identifiers from e.g. "#xpointer(id("usi12") id("usi13"))" // extract list of identifiers from e.g. "#xpointer(id("usi12") id("usi13"))"
std::vector<std::string> extract_ids(std::string idstring) std::vector<std::string> extract_ids(std::string idstring)
@ -139,15 +114,12 @@ class tdm_termite
public: public:
// encoding // encoding
// tdm_termite(std::vector<std::string> csvfile); // tdm_reaper(std::vector<std::string> csvfile);
// decoding // decoding
tdm_termite(); tdm_reaper();
tdm_termite(std::string tdmfile, std::string tdxfile = std::string(""), tdm_reaper(std::string tdmfile, std::string tdxfile = std::string(""),
bool showlog = false); bool showlog = false);
~tdm_termite();
tdm_termite(const tdm_termite& other);
tdm_termite& operator=(const tdm_termite& other);
// provide (tdm,tdx) files // provide (tdm,tdx) files
void submit_files(std::string tdmfile, std::string tdxfile = std::string(""), void submit_files(std::string tdmfile, std::string tdxfile = std::string(""),
@ -207,10 +179,10 @@ public:
template<typename tdmelement> template<typename tdmelement>
std::string get_overview(format formatter); std::string get_overview(format formatter);
private: private:
void summarize_member(tdm_channelgroup &chp, std::string& summary, format& formatter); void summarize_member(tdm_channelgroup chp, std::string& summary, format& formatter);
void summarize_member(submatrix &sbm, std::string& summary, format& formatter); void summarize_member(submatrix sbm, std::string& summary, format& formatter);
void summarize_member(localcolumn &lcc, std::string& summary, format& formatter); void summarize_member(localcolumn lcc, std::string& summary, format& formatter);
void summarize_member(block &blk, std::string& summary, format& formatter); void summarize_member(block blk, std::string& summary, format& formatter);
public: public:
// get list of channelgroup ids // get list of channelgroup ids
@ -266,8 +238,7 @@ public:
// dump a single channel/entire group (identified by id) to file // dump a single channel/entire group (identified by id) to file
void print_channel(std::string &id, const char* filename, bool include_meta = true); void print_channel(std::string &id, const char* filename, bool include_meta = true);
void print_group(std::string &id, const char* filename, bool include_meta = true, void print_group(std::string &id, const char* filename, bool include_meta = true, char sep = ' ');
char sep = ' ', std::string column_header = std::string(""));
void check_filename_path(const char* filename); void check_filename_path(const char* filename);
// check machine's datatypes // check machine's datatypes

132
makefile
View File

@ -1,108 +1,62 @@
# --------------------------------------------------------------------------- # # --------------------------------------------------------------------------- #
SHELL := /bin/bash
# declare name of executable # declare name of executable
EXE = tdmtermite EXE = tdmreaper
# sources and headers # sources and headers
SRC := tdm_termite SRC := tdm_reaper
HPP = $(wildcard lib/*.hpp) HPP = $(wildcard lib/*.hpp)
# compiler and C++ standard # compiler and C++ standard
CC = g++ -std=c++17 CC = g++ -std=c++17
# compiler options and optimization flags # compiler options and optimization flags
OPT = -O3 -Wall -Wconversion -Wpedantic -Wunused-variable -Wsign-compare OPT = -O3 -Wall -Werror -Wunused-variable -Wsign-compare
# include 3rd party libraries paths # include library path
LIBB := 3rdparty/ LIB = pugixml/
LIB := $(foreach dir,$(shell ls $(LIBB)),-I $(LIBB)$(dir))
# determine git version/commit tag # determine git version/commit tag
GTAG := $(shell git tag -l --sort=version:refname | tail -n1 | sed "s/$^v//g") GTAG := $(shell git tag | head -n1)
GHSH := $(shell git rev-parse HEAD | head -c8) GHSH := $(shell git rev-parse HEAD | head -c8)
GVSN := $(shell cat python/VERSION | tr -d ' \n')
# current timestamp
TMS = $(shell date +%Y%m%dT%H%M%S)
# define install location # define install location
INST := /usr/local/bin INST := /usr/local/bin
# platform and current working directory # platform
OST := $(shell uname) OST := $(shell uname)
CWD := $(shell pwd)
# --------------------------------------------------------------------------- # # --------------------------------------------------------------------------- #
# C++ and CLI tool # CLI tool
# check tags and build executable $(EXE) : main.o $(SRC).o
exec: check-tags $(GVSN) $(EXE)
# build executable
$(EXE): $(SRC).o main.o
$(CC) $(OPT) $^ -o $@ $(CC) $(OPT) $^ -o $@
$(SRC).o : lib/$(SRC).cpp lib/$(SRC).hpp $(HPP) install : $(EXE)
$(CC) -c $(OPT) $(LIB) $< -o $@
# build main.cpp object file and include git version/commit tag
main.o: src/main.cpp lib/$(SRC).hpp $(HPP)
@cp $< $<.cpp
@if [ $(OST) = "Linux" ]; then\
sed -i 's/TAGSTRING/$(GTAG)/g' $<.cpp; \
sed -i 's/HASHSTRING/$(GHSH)/g' $<.cpp; \
sed -i 's/TIMESTAMPSTRING/$(TMS)/g' $<.cpp; \
fi
@if [ $(OST) = "Darwin" ]; then\
sed -i '' 's/TAGSTRING/$(GTAG)/g' $<.cpp; \
sed -i '' 's/HASHSTRING/$(GHSH)/g' $<.cpp; \
sed -i '' 's/TIMESTAMPSTRING/$(TMS)/g' $<.cpp; \
fi
$(CC) -c $(OPT) $(LIB) -I lib/ $<.cpp -o $@
@rm $<.cpp
install: $(EXE)
cp $< $(INST)/ cp $< $(INST)/
uninstall : $(INST)/$(EXE) uninstall : $(INST)/$(EXE)
rm $< rm $<
tdmtest : tdmtest.o # build main.cpp object file and include git version/commit tag
$(CC) $(OPT) $^ -o $@ main.o : src/main.cpp lib/$(SRC).hpp $(HPP)
@cp $< $<.cpp
@if [ $(OST) = "Linux" ]; then\
sed -i 's/TAGSTRING/$(GTAG)/g' $<.cpp; \
sed -i 's/HASHSTRING/$(GHSH)/g' $<.cpp; \
fi
@if [ $(OST) = "Darwin" ]; then\
sed -i '' 's/TAGSTRING/$(GTAG)/g' $<.cpp; \
sed -i '' 's/HASHSTRING/$(GHSH)/g' $<.cpp; \
fi
$(CC) -c $(OPT) -I $(LIB) -I lib/ $<.cpp -o $@
@rm $<.cpp
tdmtest.o : src/test.cpp lib/$(SRC).hpp $(HPP) $(SRC).o : lib/$(SRC).cpp lib/$(SRC).hpp $(HPP)
$(CC) -c $(OPT) $(LIB) -I lib/ $< -o $@ $(CC) -c $(OPT) -I $(LIB) $< -o $@
cpp-clean : clean-cpp :
rm -vf $(EXE) rm -f $(EXE) *.o src/main.cpp.cpp
rm -vf *.o src/main.cpp.cpp tdmtest
#-----------------------------------------------------------------------------#
# versions
$(GTAG):
@echo "consistent versions check successful: building $(GTAG)"
check-tags:
@echo "latest git tag: $(GTAG)"
@echo "latest git hash: $(GHSH)"
@echo "python version: $(GVSN)"
# --------------------------------------------------------------------------- #
# docker
docker-build:
docker build . --tag tdmtermite:latest
docker-run:
mkdir -pv data/{input,output}
docker run -it --rm --volume $(CWD)/data:/home/data tdmtermite:latest /bin/bash
docker-clean:
rm -rv data
docker image remove tdmtermite
# --------------------------------------------------------------------------- # # --------------------------------------------------------------------------- #
# check process # check process
@ -112,25 +66,31 @@ checkps :
@ps aux | grep $(EXE) | grep -v "grep" @ps aux | grep $(EXE) | grep -v "grep"
# --------------------------------------------------------------------------- # # --------------------------------------------------------------------------- #
# python # python/cython module
python-build: check-tags $(GVSN) cython-requirements: cython/requirements.txt
make -C python/ build-inplace python3 -m pip install -r $<
cp python/TDMtermite*.so ./ -v
python-install: check-tags $(GVSN) cython-help : cython/setup.py
make -C python/ install python3 $< --help
python-clean: cython-list : cython/setup.py
make -C python/ clean python3 $< --name --description --author --author-email --url
rm -vf TDMtermite*.so
python-test: cython-build : cython/setup.py cython/tdm_reaper.pxd cython/py_tdm_reaper.pyx $(HPP) lib/tdm_reaper.cpp
PYTHONPATH=./ python python/examples/usage.py python3 $< build_ext --inplace
cp -v tdm_reaper.cpython-*.so python/
cython-install : cython/setup.py cython/tdm_reaper.pxd cython/py_tdm_reaper.pyx $(HPP) lib/tdm_reaper.cpp
python3 $< install
clean-cython :
rm -vf cython/py_tdm_reaper.cpp
rm -vf tdm_reaper.cpython-*.so python/tdm_reaper.cpython-*.so
rm -rf build
# --------------------------------------------------------------------------- # # --------------------------------------------------------------------------- #
# clean
clean : cpp-clean python-clean clean : clean-cpp clean-cython
# --------------------------------------------------------------------------- # # --------------------------------------------------------------------------- #

View File

@ -1,4 +0,0 @@
include lib/*.hpp
include *.cpp
include *.pyx
include *.pxd

View File

@ -1 +0,0 @@
2.1.3

View File

@ -1,50 +0,0 @@
import tdmtermite
import json
import re
# create 'tdm_termite' instance object
try :
jack = tdmtermite.tdmtermite(b'samples/SineData.tdm',b'samples/SineData.tdx')
except RuntimeError as e :
print("failed to load/decode TDM files: " + str(e))
# list ids of channelgroups
grpids = jack.get_channelgroup_ids()
# iterate through group-ids
for grp in grpids[0:2] :
# obtain meta data of channelgroups
grpinfo = jack.get_channelgroup_info(grp)
print( json.dumps(grpinfo,sort_keys=False,indent=4) )
# get channel meta-data and compose header
column_header = ""
chns = grpinfo['channels'].split(' ')
for chn in chns :
# obtain channel's meta-data as dictionary (JSON)
chninfo = jack.get_channel_info(chn.encode())
# print( json.dumps(chninfo,sort_keys=False,indent=4) )
# choose e.g. channel-id and name to compose column header
chnhead = ( str(chninfo['channel-id']) + "("
+ str(chninfo['name']).replace(',',' ') + ")" )
# append to full header
column_header = column_header + chnhead + ","
# remove last comma
column_header = column_header[0:-1]
print("column_header:\n"+column_header+"\n")
# write channelgroup to file
try :
grpname = re.sub('[^A-Za-z0-9]','',grpinfo['name'])
grpfile = str("./channelgroup_") + str(grp.decode()) + "_" + str(grpname) + str(".csv")
jack.print_channelgroup(grp, # id of group to be written
grpfile.encode(), # filename
False, # no meta-data header
ord(','), # csv separator character
column_header.encode() # provide column header
)
except RuntimeError as e :
print("failed to print channelgroup: " + str(grp) + " : " + str(e))

View File

@ -1,20 +0,0 @@
import tdmtermite
# create 'tdm_termite' instance object
try :
jack = tdmtermite.tdmtermite(b'samples/SineData.tdm',b'samples/SineData.tdx')
# list ids of channelgroups
grpids = jack.get_channelgroup_ids()
# iterate through groups
for grp in grpids :
# write channelgroup to file (prepend './' for local directory!!)
grpfile = "./channelgroup_" + str(grp.decode()) + ".csv"
jack.print_channelgroup(grp, # id of group to be written
grpfile.encode(), # filename
False, # include meta-data fileheader?
ord(','), # csv separator character
b"" # (empty=default) column header
)
except RuntimeError as e :
print("failed to load/decode/write TDM files: " + str(e))

View File

@ -1,60 +0,0 @@
setup:
cat ../README.md | grep '^# TDMtermite' -A 50000 > ./README.md
#pandoc -f markdown -t rst -o README.rst README.md
#python -m rstvalidator README.rst
cp -r ../lib ./
cp -r ../3rdparty ./
cp -v ../LICENSE ./
setup-clean:
rm -vf README.md README.rst LICENSE
rm -rf lib/ 3rdparty/
build: setup
python setup.py build
build-inplace: setup
python setup.py build_ext --inplace
build-install: setup
python setup.py install
build-sdist: setup
python setup.py sdist
python -m twine check dist/*
build-bdist: setup
python setup.py bdist
python -m twine check dist/*
build-clean:
python setup.py clean --all
rm -vf tdmtermite*.so tdmtermite*.cpp
rm -rvf dist/ tdmtermite.egg-info/
cibuildwheel-build: setup
cibuildwheel --platform linux
cibuildwheel-clean:
rm -rvf wheelhouse/
pypi-audit:
auditwheel repair $(shell find dist/ -name "*-linux_x86_64.whl")
# username: __token__
# password: API-token including "pypi-"
pypi-upload-test:
python -m twine upload --repository testpypi dist/$(shell ls -t dist/ | head -n1)
pypi-install-test:
python -m pip install --index-url https://test.pypi.org/simple --no-deps TDMtermite-RecordEvolution
# python3 -m pip install -i https://test.pypi.org/simple/ TDMtermite-RecordEvolution==0.5
pypi-upload:
python -m twine upload dist/$(shell ls -t dist/ | head -n1)
clean: setup build-clean cibuildwheel-clean setup-clean
run-example:
PYTHONPATH=$(pwd) python examples/usage.py

15
python/minimal.py Normal file
View File

@ -0,0 +1,15 @@
import tdm_reaper
# create 'tdm_reaper' instance object
try :
jack = tdm_reaper.tdmreaper(b'samples/SineData.tdm',b'samples/SineData.tdx')
# list ids of channelgroups
grpids = jack.get_channelgroup_ids()
# iterate through groups
for grp in grpids :
# write channelgroup to file (prepend './' for local directory!!)
grpfile = "./channelgroup_" + str(grp.decode()) + ".csv"
jack.print_channelgroup(grp,grpfile.encode(),True,ord(' '))
except RuntimeError as e :
print("failed to load/decode/write TDM files: " + str(e))

View File

@ -1,6 +0,0 @@
[build-system]
requires = ["setuptools", "wheel","Cython"]
build-backend = "setuptools.build_meta"
[tool.cibuildwheel]
before-all = ""

View File

@ -1,23 +0,0 @@
[metadata]
name = tdmtermite
description = Extract and read data from National Instruments LabVIEW tdx/tdm files and export them as csv files
long_description = file: README.md
# long_description_content_type = text/x-rst
long_description_content_type = text/markdown
version = file: VERSION
author = Record Evolution GmbH
author_email = mario.fink@record-evolution.de
maintainer = Record Evolution GmbH
url= https://github.com/RecordEvolution/TDMtermite.git
license = MIT License
license_files = LICENSE
keywords = TDM, TDX, National Instruments, DIAdem, LabVIEW, Measurement Studio, SignalExpress
classifiers =
Programming Language :: Python :: 3
License :: OSI Approved :: MIT License
Operating System :: OS Independent
Topic :: Scientific/Engineering
Topic :: Software Development :: Libraries :: Python Modules
[options]

View File

@ -1,23 +0,0 @@
from setuptools import Extension, setup
from Cython.Build import cythonize
import sys
print("building on platform: "+sys.platform)
cmpArgs = {
"linux": ['-std=c++17','-Wno-unused-variable'],
"darwin": ['-std=c++17','-Wno-unused-variable'],
"win32": ['/EHsc','/std:c++17']
}
extension = Extension(
"tdmtermite",
language='c++',
sources=["tdmtermite.pyx"],
include_dirs=["lib","3rdparty/pugixml"],
extra_compile_args=cmpArgs[sys.platform]
)
setup(
ext_modules=cythonize(extension,language_level=3)
)

View File

@ -1,12 +1,12 @@
import tdmtermite import tdm_reaper
# import numpy as np # import numpy as np
import json import json
import re import re
# create 'tdm_termite' instance object # create 'tdm_reaper' instance object
try : try :
jack = tdmtermite.tdmtermite(b'samples/SineData.tdm',b'samples/SineData.tdx') jack = tdm_reaper.tdmreaper(b'samples/SineData.tdm',b'samples/SineData.tdx')
except RuntimeError as e : except RuntimeError as e :
print("failed to load/decode TDM files: " + str(e)) print("failed to load/decode TDM files: " + str(e))
@ -24,8 +24,8 @@ for grp in grpids[0:2] :
# write channelgroup to file # write channelgroup to file
try : try :
grpname = re.sub('[^A-Za-z0-9]','',grpinfo['name']) grpname = re.sub('[^A-Za-z0-9]','',grpinfo['name'])
grpfile = "./channelgroup_" + str(grp) + "_" + str(grpname) + ".csv" grpfile = "channelgroup_" + str(grp) + "_" + str(grpname) + ".csv"
jack.print_channelgroup(grp.encode(),grpfile.encode(),True,ord(' '),b'') jack.print_channelgroup(grp.encode(),grpfile.encode(),True,ord(' '))
except RuntimeError as e : except RuntimeError as e :
print("failed to print channelgroup: " + str(grp) + " : " + str(e)) print("failed to print channelgroup: " + str(grp) + " : " + str(e))
@ -49,7 +49,7 @@ for chn in chnids[0:2] :
print(str(chndata[0:6]) + " ...") print(str(chndata[0:6]) + " ...")
# write channel to file # write channel to file
chnfile = "./channel_" + str(chn) + ".csv" chnfile = "channel_" + str(chn) + ".csv"
try : try :
jack.print_channel(chn.encode(),chnfile.encode(),True) jack.print_channel(chn.encode(),chnfile.encode(),True)
except RuntimeError as e : except RuntimeError as e :

View File

@ -1,369 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
<usi:tdm xmlns:usi="http://www.ni.com/Schemas/USI/1_0" version="1.0">
<usi:documentation>
<usi:exporter>National Instruments USI</usi:exporter>
<usi:exporterVersion>1.5</usi:exporterVersion>
</usi:documentation>
<usi:model modelName="National Instruments USI generated meta file" modelVersion="1.0">
<usi:include nsUri="http://www.ni.com/DataModels/USI/TDM/1_0"/>
</usi:model>
<usi:include>
<file byteOrder="bigEndian" url="SineData-be.tdx">
<block byteOffset="0" id="inc0" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="8000" id="inc1" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="16000" id="inc2" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="24000" id="inc3" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="32000" id="inc4" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="40000" id="inc5" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="48000" id="inc6" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="56000" id="inc7" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="64000" id="inc8" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="72000" id="inc9" length="1000" valueType="eFloat64Usi"/>
</file>
</usi:include>
<usi:data>
<double_sequence id="usi1">
<values external="inc0"/>
</double_sequence>
<double_sequence id="usi2">
<values external="inc1"/>
</double_sequence>
<double_sequence id="usi3">
<values external="inc2"/>
</double_sequence>
<double_sequence id="usi4">
<values external="inc3"/>
</double_sequence>
<double_sequence id="usi5">
<values external="inc4"/>
</double_sequence>
<double_sequence id="usi6">
<values external="inc5"/>
</double_sequence>
<double_sequence id="usi7">
<values external="inc6"/>
</double_sequence>
<double_sequence id="usi8">
<values external="inc7"/>
</double_sequence>
<double_sequence id="usi9">
<values external="inc8"/>
</double_sequence>
<double_sequence id="usi10">
<values external="inc9"/>
</double_sequence>
<tdm_root id="usi11">
<name>SineData.TDM</name>
<description>Sine signals of various amplitudes and frequencies.</description>
<title>SineData</title>
<author>National Instruments</author>
<datetime>2008-05-06T17:20:12.65074539184570313</datetime>
<channelgroups>#xpointer(id("usi12") id("usi13"))</channelgroups>
</tdm_root>
<tdm_channelgroup id="usi12">
<name>Amplitudes</name>
<description>Sine Signals of various amplitudes.</description>
<root>#xpointer(id("usi11"))</root>
<instance_attributes>
<double_attribute name="Frequency">1</double_attribute>
</instance_attributes>
<channels>#xpointer(id("usi14") id("usi15") id("usi16") id("usi17") id("usi18"))</channels>
<submatrices>#xpointer(id("usi24") id("usi25") id("usi26") id("usi27") id("usi28"))</submatrices>
</tdm_channelgroup>
<tdm_channelgroup id="usi13">
<name>Frequencies</name>
<description>Sine signals of various frequencies.</description>
<root>#xpointer(id("usi11"))</root>
<instance_attributes>
<double_attribute name="Amplitude">1</double_attribute>
</instance_attributes>
<channels>#xpointer(id("usi19") id("usi20") id("usi21") id("usi22") id("usi23"))</channels>
<submatrices>#xpointer(id("usi29") id("usi30") id("usi31") id("usi32") id("usi33"))</submatrices>
</tdm_channelgroup>
<tdm_channel id="usi14">
<name>A = 1</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999997146387718</minimum>
<maximum>0.999999682931835</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">0</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi34"))</local_columns>
</tdm_channel>
<tdm_channel id="usi15">
<name>A = 2</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-1.99999429277544</minimum>
<maximum>1.99999936586367</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">1</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi35"))</local_columns>
</tdm_channel>
<tdm_channel id="usi16">
<name>A = 4</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-3.99998858555087</minimum>
<maximum>3.99999873172734</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">2</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi36"))</local_columns>
</tdm_channel>
<tdm_channel id="usi17">
<name>A = 8</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-7.99997717110174</minimum>
<maximum>7.99999746345468</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">3</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi37"))</local_columns>
</tdm_channel>
<tdm_channel id="usi18">
<name>A = 16</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-15.9999543422035</minimum>
<maximum>15.9999949269094</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">4</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi38"))</local_columns>
</tdm_channel>
<tdm_channel id="usi19">
<name>F = 1</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999997146387718</minimum>
<maximum>0.999999682931835</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">0</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi39"))</local_columns>
</tdm_channel>
<tdm_channel id="usi20">
<name>F = 2</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999999230697499</minimum>
<maximum>0.999995986891472</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">1</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi40"))</local_columns>
</tdm_channel>
<tdm_channel id="usi21">
<name>F = 4</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999999230697499</minimum>
<maximum>0.99994907791452</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">2</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi41"))</local_columns>
</tdm_channel>
<tdm_channel id="usi22">
<name>F = 8</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999999230697499</minimum>
<maximum>0.999996490345607</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">3</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi42"))</local_columns>
</tdm_channel>
<tdm_channel id="usi23">
<name>F = 16</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999999230697499</minimum>
<maximum>0.999993076284592</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">4</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi43"))</local_columns>
</tdm_channel>
<submatrix id="usi24">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi34"))</local_columns>
</submatrix>
<submatrix id="usi25">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi35"))</local_columns>
</submatrix>
<submatrix id="usi26">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi36"))</local_columns>
</submatrix>
<submatrix id="usi27">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi37"))</local_columns>
</submatrix>
<submatrix id="usi28">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi38"))</local_columns>
</submatrix>
<submatrix id="usi29">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi39"))</local_columns>
</submatrix>
<submatrix id="usi30">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi40"))</local_columns>
</submatrix>
<submatrix id="usi31">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi41"))</local_columns>
</submatrix>
<submatrix id="usi32">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi42"))</local_columns>
</submatrix>
<submatrix id="usi33">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi43"))</local_columns>
</submatrix>
<localcolumn id="usi34">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi14"))</measurement_quantity>
<submatrix>#xpointer(id("usi24"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi1"))</values>
</localcolumn>
<localcolumn id="usi35">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi15"))</measurement_quantity>
<submatrix>#xpointer(id("usi25"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi2"))</values>
</localcolumn>
<localcolumn id="usi36">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi16"))</measurement_quantity>
<submatrix>#xpointer(id("usi26"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi3"))</values>
</localcolumn>
<localcolumn id="usi37">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi17"))</measurement_quantity>
<submatrix>#xpointer(id("usi27"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi4"))</values>
</localcolumn>
<localcolumn id="usi38">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi18"))</measurement_quantity>
<submatrix>#xpointer(id("usi28"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi5"))</values>
</localcolumn>
<localcolumn id="usi39">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi19"))</measurement_quantity>
<submatrix>#xpointer(id("usi29"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi6"))</values>
</localcolumn>
<localcolumn id="usi40">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi20"))</measurement_quantity>
<submatrix>#xpointer(id("usi30"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi7"))</values>
</localcolumn>
<localcolumn id="usi41">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi21"))</measurement_quantity>
<submatrix>#xpointer(id("usi31"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi8"))</values>
</localcolumn>
<localcolumn id="usi42">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi22"))</measurement_quantity>
<submatrix>#xpointer(id("usi32"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi9"))</values>
</localcolumn>
<localcolumn id="usi43">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi23"))</measurement_quantity>
<submatrix>#xpointer(id("usi33"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi10"))</values>
</localcolumn>
</usi:data>
</usi:tdm>

Binary file not shown.

View File

@ -1,63 +0,0 @@
#include <iostream>
#include <fstream>
#include <iterator>
#include <vector>
int main(int argc, char* argv[])
{
for ( int i = 0; i < argc; i++ )
{
std::cout<<argv[i]<<"\n";
}
if ( argc < 3 )
{
std::cout<<"missing file argument\n";
return 1;
}
std::ifstream fin(argv[1],std::ifstream::binary);
std::vector<unsigned char> tdxbuf((std::istreambuf_iterator<char>(fin)),
(std::istreambuf_iterator<char>()));
fin.close();
std::cout<<"length of buffer: "<<tdxbuf.size()<<"\n";
unsigned long int dtsize = 8;
if ( tdxbuf.size()%dtsize != 0 )
{
std::cout<<"mismatch between datatype size and length of buffer\n";
return 1;
}
unsigned long int nums = tdxbuf.size()/dtsize;
std::cout<<"number of entities: "<<nums<<"\n";
std::vector<unsigned char> tdxbufrev(tdxbuf.size());
for ( unsigned long int i = 0; i < nums; i++ )
{
for ( unsigned long int j = 0; j < dtsize; j++ )
{
tdxbufrev[i*dtsize+j] = tdxbuf[(i+1)*dtsize-(j+1)];
}
}
std::ofstream fou(argv[2],std::ifstream::binary);
for ( unsigned char ch: tdxbufrev)
{
fou<<ch;
}
fou.close();
return 0;
}

View File

@ -1,6 +1,6 @@
// ------------------------------------------------------------------------- // // ------------------------------------------------------------------------- //
#include "tdm_termite.hpp" #include "tdm_reaper.hpp"
#include <filesystem> #include <filesystem>
#include <regex> #include <regex>
@ -11,17 +11,16 @@
const std::string gittag("TAGSTRING"); const std::string gittag("TAGSTRING");
const std::string githash("HASHSTRING"); const std::string githash("HASHSTRING");
const std::string timestamp("TIMESTAMPSTRING");
void show_usage() void show_usage()
{ {
std::cout<<"\n" std::cout<<"\n"
<<"tdmtermite ["<<gittag<<"-g"<<githash<<"-"<<timestamp<<"] (https://github.com/RecordEvolution/TDMtermite.git)" <<"tdmreaper ["<<gittag<<"-g"<<githash<<"] (github.com/RecordEvolution/tdm_ripper.git)"
<<"\n\n" <<"\n\n"
<<"Decode TDM/TDX files and dump data as *.csv" <<"Decode TDM/TDX files and dump data as *.csv"
<<"\n\n" <<"\n\n"
<<"Usage:\n\n" <<"Usage:\n\n"
<<" tdmtermite <tdm-file> <tdx-file> [options]" <<" tdmreaper <tdm-file> <tdx-file> [options]"
<<"\n\n" <<"\n\n"
<<"Options:" <<"Options:"
<<"\n\n" <<"\n\n"
@ -56,7 +55,7 @@ void show_usage()
typedef std::map<std::string,std::string> optkeys; typedef std::map<std::string,std::string> optkeys;
const std::string argmsg = std::string("both .tdm and .tdx file must be provided!"); const std::string argmsg = std::string("both .tdm and .tdx file must be provided!");
const std::string arguse = std::string("see $ tdmtermite --help for usage"); const std::string arguse = std::string("see $ tdmreaper --help for usage");
optkeys parse_args(int argc, char* argv[], bool showargs = false) optkeys parse_args(int argc, char* argv[], bool showargs = false)
{ {
@ -79,7 +78,7 @@ optkeys parse_args(int argc, char* argv[], bool showargs = false)
else if ( std::string(argv[1]) == std::string("--version") else if ( std::string(argv[1]) == std::string("--version")
|| std::string(argv[1]) == std::string("-v") ) || std::string(argv[1]) == std::string("-v") )
{ {
std::cout<<"tdmtermite "<<gittag<<"-g"<<githash<<"-"<<timestamp<<"\n"; std::cout<<"tdmreaper "<<gittag<<"-g"<<githash<<"\n";
} }
else else
{ {
@ -271,8 +270,8 @@ int main(int argc, char* argv[])
std::string csvsep = cfgopts.count("csvsep") == 1 ? cfgopts.at("csvsep") std::string csvsep = cfgopts.count("csvsep") == 1 ? cfgopts.at("csvsep")
: std::string(","); : std::string(",");
// declare and initialize tdm_termite instance // declare and initialize tdm_reaper instance
tdm_termite jack; tdm_reaper jack;
try { try {
jack.submit_files(cfgopts.at("tdm"),cfgopts.at("tdx"),false); jack.submit_files(cfgopts.at("tdm"),cfgopts.at("tdx"),false);
} catch (const std::exception& e) { } catch (const std::exception& e) {
@ -285,13 +284,13 @@ int main(int argc, char* argv[])
if ( showroot ) std::cout<<"\n"<<jack.get_root().get_info()<<"\n"; if ( showroot ) std::cout<<"\n"<<jack.get_root().get_info()<<"\n";
// get complete channel(-group) overview // get complete channel(-group) overview
format grpformatter(16,false,false,' '); format grpformatter(26,false,false,' ');
if (listgroups) std::cout<<"\n"<<jack.get_overview<tdm_channelgroup>(grpformatter)<<"\n"; if (listgroups) std::cout<<"\n"<<jack.get_overview<tdm_channelgroup>(grpformatter)<<"\n";
format chformatter(16,false,false,' '); format chformatter(14,false,false,' ');
if (listchannels) std::cout<<"\n"<<jack.get_channel_overview(chformatter)<<"\n"; if (listchannels) std::cout<<"\n"<<jack.get_channel_overview(chformatter)<<"\n";
// get complete submatrix/localcolumns overview // get complete submatrix/localcolumns overview
format formatter(16,false,false,' '); format formatter(18,false,false,' ');
if (listblocks) std::cout<<jack.get_overview<block>(formatter)<<"\n"; if (listblocks) std::cout<<jack.get_overview<block>(formatter)<<"\n";
if (listsubmatrices) std::cout<<jack.get_overview<submatrix>(formatter)<<"\n"; if (listsubmatrices) std::cout<<jack.get_overview<submatrix>(formatter)<<"\n";
if (listlocalcolumns) std::cout<<jack.get_overview<localcolumn>(formatter)<<"\n"; if (listlocalcolumns) std::cout<<jack.get_overview<localcolumn>(formatter)<<"\n";

View File

@ -1,33 +0,0 @@
// ------------------------------------------------------------------------- //
#include "tdm_termite.hpp"
#include <filesystem>
#include <regex>
#include <thread>
#include <chrono>
// ------------------------------------------------------------------------- //
int main(int argc, char* argv[])
{
std::string tdmfile(argv[1]);
std::string tdxfile(argv[2]);
std::cout<<tdmfile<<"\n"<<tdxfile<<"\n";
pugi::xml_document xml_doc;
pugi::xml_parse_result xml_result;
// load XML document from stream
std::ifstream fin(tdmfile.c_str());
xml_result = xml_doc.load(fin);
fin.close();
std::cout<<"\nloading "<<tdmfile<<": "<<xml_result.description()<<"\n";
std::cout<<"encoding: "<<(pugi::xml_encoding)xml_result.encoding<<"\n\n";
std::this_thread::sleep_for(std::chrono::milliseconds(4000));
return 0;
}