Compare commits

..

No commits in common. "master" and "v0.1" have entirely different histories.
master ... v0.1

46 changed files with 1460 additions and 4735 deletions

View File

@ -1,100 +0,0 @@
name: CI Build Wheel
on:
push:
tags: ["v[0-9]+.[0-9]+.[0-9]+"]
jobs:
build_setup:
name: Prepare environment for wheel builds
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v2
- name: Prepare wheel build
run: make -C python/ setup
- name: Store wheel configuration files
uses: actions/upload-artifact@v4.6.0
with:
name: wheel-config
path: python/
- name: Display files
run: ls -lR
build_wheels:
name: Build binary wheels on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
needs: [build_setup]
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- name: Install cibuildwheel
run: python -m pip install cibuildwheel==2.1.2
- name: Get wheel configuration files
uses: actions/download-artifact@v4.1.7
with:
name: wheel-config
path: python/
- name: Build wheels
run: python -m cibuildwheel --output-dir wheelhouse
working-directory: python/
- name: Store binary wheels
uses: actions/upload-artifact@v4.6.0
with:
name: binary-wheels-${{matrix.os}}-${{ strategy.job-index }}
path: python/wheelhouse/*.whl
build_sdist:
name: Build source distribution
runs-on: ubuntu-24.04
needs: [build_setup]
steps:
- uses: actions/checkout@v2
- name: Install cython
run: python -m pip install cython==0.29.24
- name: Get wheel configuration files
uses: actions/download-artifact@v4.1.7
with:
name: wheel-config
path: python/
- name: Build sdist
run: python setup.py sdist
working-directory: python/
- name: Store source wheels
uses: actions/upload-artifact@v4.6.0
with:
name: source-wheels
path: python/dist/*.tar.gz
- name: Display files
run: ls -lR
upload_pypi:
name: Upload wheels to PyPI
runs-on: ubuntu-24.04
needs: [build_wheels, build_sdist]
steps:
- name: Get source wheels
uses: actions/download-artifact@v4.1.7
with:
name: source-wheels
path: dist/
- name: Get binary wheels
uses: actions/download-artifact@v4.1.7
with:
path: dist/
pattern: binary-wheels-*
merge-multiple: true
- name: Display files
run: ls -lR
- uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
password: ${{ secrets.TDMTERMITE_GITHUB_WORKFLOW_PYPI_API_TOKEN }}

24
.gitignore vendored
View File

@ -6,27 +6,3 @@ build/
*.a *.a
*.dat *.dat
data/ data/
tdmripper
*.csv
*.log
tdmreaper
cython/*.cpp
tdmtermite
dist/
*.egg-info/
output/
monitor-process.sh
tdmtest
pip/*.hpp
pip/*.md
pip/*.cpp
pip/*.pyx
pip/*.pxd
pip/LICENSE
python/3rdparty/
python/LICENSE
python/README.md
python/TDMtermite.cpp
python/lib/

View File

@ -1,30 +0,0 @@
FROM debian:bullseye
# install requirements
RUN apt-get update && apt-get upgrade -y && apt-get install -y \
build-essential \
g++ make git \
python3 python3-pip
# check compiler and current user
RUN g++ -v && whoami
# use /home as working directory
WORKDIR /home
# get the public TDMtermite repository
RUN git clone https://github.com/RecordEvolution/TDMtermite.git
# install CLI tool
RUN cd ./TDMtermite && ls -lh && make install && ls -lh /usr/local/bin/tdmtermite
# install Python module
RUN cd ./TDMtermite && ls -lh && make cython-requirements && make cython-list && make cython-install
# create directory for data exchange
#RUN [ "/bin/bash", "-c", "mkdir -pv data/{input,output}" ]
RUN mkdir -pv data
CMD ["sleep","infinity"]

399
README.md
View File

@ -1,350 +1,115 @@
# tdm_ripper
[![LICENSE](https://img.shields.io/github/license/RecordEvolution/TDMtermite)](https://img.shields.io/github/license/RecordEvolution/TDMtermite) The tdm_ripper provides convenient access to the TDM/TDMS data format employed by
[![STARS](https://img.shields.io/github/stars/RecordEvolution/TDMtermite)](https://img.shields.io/github/stars/RecordEvolution/TDMtermite) National Instruments LabView and DIAdem.
![CI Build Wheel](https://github.com/RecordEvolution/TDMtermite/actions/workflows/pypi-deploy.yml/badge.svg?branch=&event=push)
[![PYPI](https://img.shields.io/pypi/v/TDMtermite.svg)](https://pypi.org/project/tdmtermite/)
# TDMtermite ## Data Format
_TDMtermite_ is a C++ based library that decodes the proprietary Datasets encoded in the TDM/TDMS format come along in pairs comprised of a
file format _TDM/TDX_ for measurement data. First introduced by .tdm and .tdx file. While the .tdm file is a human-readable document providing
[National Instruments](https://www.ni.com), the TDM format relies on the meta information about the dataset, the .tdx is a binary containing the actual data.
_technical data management_ data model and is employed by The .tdm is represented in XML format and basically reveals what data the .tdx
[LabVIEW](https://www.ni.com/de-de/shop/labview.html), LabWindows™/CVI™, contains and how to read it. The XML tree is usually made up of several groups,
Measurement Studio, SignalExpress, and [DIAdem](https://www.ni.com/de-de/shop/data-acquisition-and-control/application-software-for-data-acquisition-and-control-category/what-is-diadem.html). each containing an arbitrary amount of channels.
The [Record Evolution Platform](https://www.record-evolution.de/en/home-en/) uses TDMtermite to integrate measurement data into ETL processes. The TDMtermite library is available both as a command line tool and as a Python module. The Python module of TDMtermite enables data scientists to conveniently include TDM formats in their existing data pipelines by providing access to both raw data and metadata in terms of native Python objects.
## Overview
* [TDM file format](#Dataformat)
* [Build and Installation](#Installation)
* [Usage and Examples](#Usage)
* [References](#References)
## Dataformat
Datasets encoded in the TDM/TDX format come in pairs comprised of a
.tdm (header) file and a .tdx (data) file. While the .tdm file is a human-readable
file providing meta information about the dataset, the .tdx file is a binary file
containing the actual data. The .tdm based on the _technical data management_
model is an XML file. It describes what data the .tdx file contains and how
to read it. The
[TDM data model](https://www.ni.com/de-de/support/documentation/supplemental/10/ni-tdm-data-model.html)
structures the data hierarchically with respect to _file_, _(channel)_ _groups_ and
_channels_. The file-level XML may contain any number of (channel) groups, each
of which is made up of an arbitrary number of channels. Thus, the XML tree in
the [TDM header file](https://zone.ni.com/reference/de-XX/help/370858P-0113/tdmdatamodel/tdmdatamodel/tdm_headerfile/)
looks like this:
```xml
<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
<usi:tdm xmlns:usi="http://www.ni.com/Schemas/USI/1_0" version="1.0">
<usi:documentation>
<usi:exporter>National Instruments USI</usi:exporter>
<usi:exporterVersion>1.5</usi:exporterVersion>
</usi:documentation>
<usi:model modelName="National Instruments USI generated meta file" modelVersion="1.0">
<usi:include nsUri="http://www.ni.com/DataModels/USI/TDM/1_0"/>
</usi:model>
<usi:include>
<file byteOrder="littleEndian" url="example.tdx">
...
<block byteOffset="0" id="inc0" length="1000" valueType="eFloat64Usi"/>
...
<block_bm id="inc4" blockOffset="100" blockSize="7" byteOffset="0" length="4" valueType="eInt8Usi"/>
...
</usi:include>
<usi:data>
...
</usi:data>
</usi:tdm>
```
The XML tree is comprised of _four_ main XML elements: `usi:documentation`, `usi:model`,
`usi:include` and `usi:data`. The element `usi:include` references the data file
`example.tdx` and reveals one of _two_ possible orderings of the mass data (.tdx):
1. either _channel-wise_ (`<block>`) - all values of a specific channel follow subsequently
1. or _block-wise_ (`<block_bm>`) - all values of a specific measurement time follow subsequently.
The supported _numerical data types_ are:
| datatype | channel datatype | numeric | value sequence | size | description |
|-------------|------------------|---------|-----------------|-------|-------------------------|
| eInt16Usi | DT_SHORT | 2 | short_sequence | 2byte | signed 16 bit integer |
| eInt32Usi | DT_LONG | 6 | long_sequence | 4byte | signed 32 bit integer |
| eUInt8Usi | DT_BYTE | 5 | byte_sequence | 1byte | unsigned 8 bit integer |
| eUInt16Usi | DT_SHORT | 2 | short_sequence | 2byte | unsigned 16 bit integer |
| eUInt32Usi | DT_LONG | 6 | long_sequence | 4byte | unsigned 32 bit integer |
| eFloat32Usi | DT_FLOAT | 3 | float_sequence | 4byte | 32 bit float |
| eFloat64Usi | DT_DOUBLE | 7 | double_sequence | 8byte | 64 Bit double |
| eStringUsi | DT_STRING | 1 | string_sequence | | text |
The XML element `<usi:data>` is comprised of _five_ different types of
elements that are `<tdm_root>`, `<tdm_channelgroup>`, `<tdm_channel>`, `<localcolumn>`
and `<submatrix>`. The root element `<tdm_root>` describes the general properties
of the dataset and lists the _ids_ of all channel groups that belong to
the dataset. The element `<tdm_channelgroup>` divides the _channels_ into groups
and has a unique _id_ that is referenced by its root element. The `<channels>`
element in `<tdm_channelgroup>` lists the unique ids of all channels that belong
to that group. Finally, the element `<tdm_channel>` describes a single column of
actual data including its datatype. The remaining element types are
`<localcolumn>`
```xml
<localcolumn id="usiXY">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usiAB"))</measurement_quantity>
<submatrix>#xpointer(id("usiMN"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation> ... </sequence_representation>
<values>#xpointer(id("usiZ"))</values>
</localcolumn>
```
with a unique id, the `<measurement_quantity>` referring to one specific channel,
the `<submatrix>` and its id respectively, the type of representation in
`<sequence_representation>` - being one of _explicit_, _implicit linear_ or
_rawlinear_ - and the `<values>` element, which refers to one _value sequence_,
and the element `<submatrix>`
```xml
<submatrix id="usiXX">
<name>Untitled</name>
<measurement>#xpointer(id("usiUV"))</measurement>
<number_of_rows>N</number_of_rows>
<local_columns>#xpointer(id("usiMN"))</local_columns>
</submatrix>
```
that references the channel group in `<measurement>` to which it belongs and provides
the _number of rows_ in the channels listed in `<local_columns>`.
## Installation ## Installation
The library can be used both as a _CLI_-based tool and as a _Python_ module. The makefile provides targets for using the library both as native C++ extension
and as Python module. The package supports usage on Linux and MacOSX.
### CLI tool The tdm_ripper module is built on these platforms by
To install the CLI tool _TDMtermite_, do
```Shell ```Shell
# Linux
pip install Cython
make install make install
``` ```
which uses `/usr/local/bin` as an installation directory. On _macOSX_, please first and
build the binary locally with `make` and install it in your preferred location.
### Python
In order to build a _Python module_ from the _C++_ code base, the
[Cython](https://cython.readthedocs.io/en/latest/index.html) package must be
available. It may be installed via `python3 -m pip install cython` .
The [Numpy](https://numpy.org) package is recommended
to pass arrays of data from the C++ kernel to Python. The _makefile_ provides
the target `make cython-requirements` to install all required Python modules.
Finally, to build the Python extension _tdm_termite_ locally or install
it, the targets `make cython-build` and `make cython-install` are provided.
To install the Python module on the system, simply do
```Shell ```Shell
make cython-requirements # macOS
make cython-install pip install Cython
make install_osx
``` ```
which makes the module available for import by `import tdm_termite` .
#### Installation with pip
The package is also available via the [Python Package Index](https://pypi.org) at
[TDMtermite](https://pypi.org/project/tdmtermite/). To install the latest version simply do
```Shell
python3 -m pip install tdmtermite
```
##### Unix
Note, that _python3_setuptools_ and _gcc version >= 10.2.0_ are required to
successfully install and use it.
## Usage ## Usage
### CLI tool Although the package is built upon a C++ core, which decodes the data, it may be
used as a Python module, as well, by interfacing the C++ library with a Cython
wrapper.
The usage of the CLI tool is sufficiently clarified by its help message displayed ### C++ core
by `tdmtermite --help`. To extract the data decoded in the pair of
files `samples/SineData.tdm` and `samples/SineData.tdx` into the directory
`/home/jack/data/`:
```Shell - In order to parse the XML tree of the .tdm file, the library employs pugixml:
tdmtermite samples/SineData.tdm samples/SineData.tdx --output /home/jack/data https://pugixml.org/ and https://github.com/zeux/pugixml
``` - The package currently supports the following datatypes:
- eInt8Usi: 8 byte
- eInt16Usi: 16 byte
- eInt32Usi: 32 byte
- eInt64Usi: 64 byte
- eUInt8Usi: 8 byte
- eUInt16Usi: 16 byte
- eUInt32Usi: 32 byte
- eUInt64Usi: 64 byte
- eFloat32Usi: 32 byte
- eFloat64Usi: 64 byte
- The core of the library takes care of the decoding by reinterpretation of the
binary in the buffer as the required datatype implemented by
The tool can also be used to list the available objects in the TDM dataset, which ```C++
are i.a. _channels_, _channelgroups_ and TDX _blocks_. To list uint8_t *dfcast = reinterpret_cast<uint8_t*>(&df);
all channels and channelgroups (without writing any file output):
```Shell for ( int i = 0; i < (int)sizeof(double); i++ )
tdmtermite samples/SineData.tdm samples/SineData.tdx --listgroups --listchannels {
``` dfcast[i] = (int)bych[i];
}
```
The user may also submit a _filenaming rule_ to control the names of the files the where for instance df is the resulting float and bych contains the binary
channel(group)s are written to. To this end, the _magic flags_ `%G` `%g`, `%C` data as an array of chars.
and `%c` representing the group id, group name, channel index and channel name - main.cpp contains an example of how the C++ library might be used to provide
are defined. The default filenaming option is: the channels and groups of the dataset. It is simply build by
```Shell ```Shell
tdmtermite samples/SineData.tdm samples/SineData.tdx --output /home/jack/data --filenames channelgroup_%G.csv make
``` ```
This makes the tool write _all channels_ grouped into files according to their - extract_all.cpp takes the .tdm, the .tdx file and some output directory as arguments
group association, while all channelgroup filenames obey the pattern `channelgroup_%G.csv`, to provide all given information in .csv format without any logging. To build:
with `%G` being replaced by the group id. The filenaming rule also enables the user
to extract only a single channel(group) by providing a particular channel(group)
id in the filenaming flag. For example,
```Shell ```Shell
tdmtermite samples/SineData.tdm samples/SineData.tdx --output /home/jack/data -f channel_usi16_%c.csv --includemeta make extall
``` ```
This will write the single channel with the id `usi16` to the file For instance, the executable accepts the following arguments:
`/home/jack/data/channel_usi16_A4.csv`, including its meta-data as a file header.
### Python ```Shell
./extract_all samples/SineData.tdm samples/SineData.tdx data/
```
To be able to use the Python module _tdm_termite_, it first has to be built locally ### Python module
or installed on the system. In the Python interpreter, simply do:
```Python - The library may also be used as a Python module and supports the use of
import tdmtermite group channels in NumPy arrays as shown in example.py .
``` - To extract all available information and data in the TDM files without any
further interaction, the use of extract_all.py is recommended. To exhibit the
required arguments:
This will import the module. The TDM files are provided by creating an instance of ```Shell
the _tdmtermite_ class: python extract_all.py --help
```
- The same functionality may be obtained from an existing python script by
importing the tdm_ripper module and calling the extract_all function. For
instance
```Python ```Python
# create 'tdmtermite' instance object import tdm_ripper as td
try :
jack = tdmtermite.tdmtermite(b'samples/SineData.tdm',b'samples/SineData.tdx')
except RuntimeError as e:
print("failed to load/decode TDM files: " + str(e))
```
After initializing the _tdmtermite_ object, it can be used to extract any of the files = td.extract_all(b"samples/SineData.tdm",b"samples/SineData.tdx",b"data/",b"my_tdm")
available data. For instance, to list the included channelgroups and channels: ```
```Python where the arguments "data/" and "my_tdm" are optional. "data/" specifies the
# list ids of channelgroups directory where all .csv output is dumped while "my_tdm" represents a name
grpids = jack.get_channelgroup_ids() prefix for all csv. files.
Note, that all string arguments must be converted to bytes before passing to
the argument list by prepending "b".
# list ids of channels
chnids = jack.get_channel_ids()
```
As a use case, we have a look at listing the ids of all channelgroups and printing
their data to separate files:
```Python
import tdmtermite
import re
# create 'tdmtermite' instance object
try :
jack = tdmtermite.tdmtermite(b'samples/SineData.tdm',b'samples/SineData.tdx')
except RuntimeError as e :
print("failed to load/decode TDM files: " + str(e))
# list ids of channelgroups
grpids = jack.get_channelgroup_ids()
grpids = [x.decode() for x in grpids]
print("list of channelgroups: ",grpids)
for grp in grpids :
# obtain meta data of channelgroups
grpinfo = jack.get_channelgroup_info(grp.encode())
print( json.dumps(grpinfo,sort_keys=False,indent=4) )
# write this channelgroup to file
try :
grpname = re.sub('[^A-Za-z0-9]','',grpinfo['name'])
grpfile = "channelgroup_" + str(grp) + "_" + str(grpname) + ".csv"
jack.print_channelgroup(grp.encode(), # id of group to be printed
grpfile.encode(), # filename
True, # include metadata as fileheader
ord(' ') # delimiter char
)
except RuntimeError as e :
print("failed to print channelgroup: " + str(grp) + " : " + str(e))
```
For details, see this [extensive example](python/usage.py)
and the absolute minimal example [minimal usage](python/minimal.py). In order
to simply extract all data of the TDM datatset and dump it to files in a given
(existing!) directory, do
```Python
import tdmtermite
jack = tdmtermite.tdmtermite(b'samples/SineData.tdm',b'samples/SineData.tdx')
jack.write_all(b"./my_tdm_data_directory/")
```
The interface allows you to construct customized file/column headers from any
meta-data and provide these headers for usage in file output (see this
[example](python/custom.py)).
## References
### TDM
- https://www.ni.com/de-de/support/documentation/supplemental/10/ni-tdm-data-model.html
- https://zone.ni.com/reference/en-XX/help/371361R-01/lvconcepts/fileio_tdms_model/
- https://zone.ni.com/reference/en-XX/help/371361R-01/lvhowto/ni_test_data_exchange/
- https://www.ni.com/de-de/support/documentation/supplemental/06/the-ni-tdms-file-format.html
- https://zone.ni.com/reference/de-XX/help/370858P-0113/tdmdatamodel/tdmdatamodel/tdm_headerfile/
- https://www.ni.com/content/dam/web/product-documentation/c_dll_tdm.zip
### IEEE Standard and datatypes
- https://en.wikipedia.org/wiki/IEEE_754
- https://www.ias.ac.in/public/Volumes/reso/021/01/0011-0030.pdf
- https://en.cppreference.com/w/cpp/language/types
### Implementation
- https://en.cppreference.com/w/
- https://pugixml.org/
- https://github.com/zeux/pugixml
- https://cython.readthedocs.io/en/latest/src/userguide/wrapping_CPlusPlus.html
### Packaging
#### Documentation
- https://packaging.python.org/tutorials/packaging-projects/
- https://setuptools.readthedocs.io/en/latest/userguide/declarative_config.html
- https://test.pypi.org/account/register/
- https://github.com/pypa/auditwheel
- https://github.com/pypa/python-manylinux-demo
- https://github.com/pypa/manylinux
#### C/C++ Extensions
- https://docs.python.org/3/extending/building.html
#### Articles
- https://martinsosic.com/development/2016/02/08/wrapping-c-library-as-python-module.html
- https://malramsay.com/post/perils-of-packaging/
- https://github.com/neuronsimulator/nrn/issues/329
- https://levelup.gitconnected.com/how-to-deploy-a-cython-package-to-pypi-8217a6581f09
- https://medium.com/swlh/distributing-python-packages-protected-with-cython-40fc29d84caf

View File

@ -1,55 +0,0 @@
<html>
<head>
<title>TDMtermite</title>
<style>
</style>
</head>
<body>
<p align="center">
<a href="https://github.com/RecordEvolution/TDMtermite.git">
<img alt="tdmtermite.svg"
src="assets/tdmtermite.svg"
width="400"
/>
</a>
</p>
<div style="width: 100%; display: block; margin-left: 28%; margin-right: 28%;">
<div style="width: 100%; overflow: hidden;">
<div style="margin: 5px; float: left;">
<a href="https://lgtm.com/projects/g/RecordEvolution/TDMtermite/alerts/">
<img alt="Total alerts"
src="https://img.shields.io/lgtm/alerts/g/RecordEvolution/TDMtermite.svg?logo=lgtm&logoWidth=18"/>
</a>
</div>
<div style="margin: 5px; float: left;">
<a href="https://lgtm.com/projects/g/RecordEvolution/TDMtermite/context:cpp">
<img alt="Language grade: C/C++"
src="https://img.shields.io/lgtm/grade/cpp/g/RecordEvolution/TDMtermite.svg?logo=lgtm&logoWidth=18"/>
</a>
</div>
<div style="margin: 5px; float: left;">
<a href="https://lgtm.com/projects/g/RecordEvolution/TDMtermite/context:python">
<img alt="Language grade: Python"
src="https://img.shields.io/lgtm/grade/python/g/RecordEvolution/TDMtermite.svg?logo=lgtm&logoWidth=18"/>
</a>
</div>
</div>
</div>
</body>
</html>

View File

@ -1,60 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
version="1.1" baseProfile="full"
width="1200px" height="800px"
viewBox="0 0 1200 800">
<title>TDM data model</title>
<desc>Graphical respresentation of the TDM data model</desc>
<style>
.rered {
fill: #e62828;
}
.reblue {
fill: #334d5c;
}
.rebrown {
fill: #c7ae7f;
}
.regreen {
fill: #49a078;
}
.reorange {
fill: #f9ae1e;
}
.regunmetal {
fill: #233543;
}
.relightgrey {
fill: #98a6ac;
}
.header {
font-family:sans-serif;
font-size: 26px;
font-weight: normal;
fill: #33bb78;
stroke: #33bb78;
stroke-width: 1;
}
</style>
<!-- tdm_root -->
<rect x="400" y="200" width="400" height="100" fill="white" stroke="black"/>
<text class="header" id="tdmroot" x="600" y="240">tdm_root</text>
<!-- <path d="M 174.5,126.0 L 175.0,127.5 173.5,128.0 174.5,126.0 Z M 100,100 C 100,200 200,100 200,200 Q 50,150,100,100 L 200,100 z" /> -->
<!-- <line x1="0" y1="200" x2="700" y2="200" stroke="black" stroke-width="20px"/> -->
<!-- Das Rechteck -->
<!-- <rect x="100" y="100" width="500" height="200" fill="white" stroke="black" stroke-width="20px"/> -->
<!-- Der Schleifer -->
<!-- <line x1="180" y1="370" x2="500" y2="50" stroke="black" stroke-width="15px"/> -->
<!-- Die Pfeilspitze -->
<!-- <polygon points="585 0 525 25 585 50" transform="rotate(135 525 25)" stroke="black" stroke-width="4px" fill="blue"/> -->
<!-- <circle class="reblue" cx="50" cy="50" r="40" stroke="black"/> -->
</svg>

Before

Width:  |  Height:  |  Size: 1.6 KiB

View File

@ -1,80 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 347.72 77.08"
version="1.1"
id="svg3945"
width="347.72"
height="77.080002">
<metadata
id="metadata3951">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title>flasher</dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<defs
id="defs3949" />
<title
id="title3916">flasher</title>
<g
id="logog">
<path
id="path138"
d="m 32.86,2 -13,7.5 v 0 h -0.05 v 0 l -0.48,0.28 c -4.27,2.46 -5.68,11.38 -6.06,14.75 L 36.2,11.33 c 0.39,-0.19 7.6,-3.69 13.57,-3.69 h 0.14 L 40.13,2 a 8.15,8.15 0 0 0 -7.27,0"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path142"
d="M 5.68,17.69 A 8.2,8.2 0 0 0 2,24 v 15.78 c 0,4.9 7,10.48 9.75,12.46 V 25.77 c 0,-0.44 0.6,-8.55 3.65,-13.72 z"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path146"
d="m 12.1,54.12 v 0 C 11.74,53.88 5,49.41 2,44.24 v 11.14 a 8.2,8.2 0 0 0 3.64,6.3 l 13.5,7.79 c 4.28,2.46 12.7,-0.77 15.81,-2.12 z"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path150"
d="m 36.79,68 c -0.4,0.19 -7.71,3.75 -13.71,3.69 l 9.78,5.64 a 8.15,8.15 0 0 0 7.27,0 l 13.51,-7.8 c 4.27,-2.46 5.68,-11.39 6.06,-14.75 z"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path154"
d="M 61.2,27.13 V 53.6 c 0,0.44 -0.6,8.55 -3.65,13.72 l 9.77,-5.64 A 8.2,8.2 0 0 0 71,55.38 V 39.59 c 0,-4.94 -7,-10.5 -9.75,-12.46"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path158"
d="M 67.31,17.69 53.81,9.9 C 49.53,7.44 41.11,10.67 38,12 l 22.85,13.23 v 0 a 43.43,43.43 0 0 1 5.7,4.51 24,24 0 0 1 4.45,5.35 V 24 a 8.2,8.2 0 0 0 -3.64,-6.3"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
</g>
<g
id="re" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;font-size:40px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#364d5c;fill-opacity:1;stroke:none"
x="74.101189"
y="54.47554"
id="text3955"><tspan
id="tspan3953"
x="74.101189"
y="54.47554"
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:44px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#364d5c;fill-opacity:1"><tspan
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold'"
id="tspan86">TDM</tspan><tspan
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:sans-serif;-inkscape-font-specification:sans-serif"
id="tspan3845"><tspan
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold'"
id="tspan150">RE</tspan>aper</tspan> </tspan></text>
</svg>

Before

Width:  |  Height:  |  Size: 3.6 KiB

View File

@ -1,78 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 328.72 77.08"
version="1.1"
id="svg3945"
width="328.72"
height="77.080002">
<metadata
id="metadata3951">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title>flasher</dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<defs
id="defs3949" />
<title
id="title3916">flasher</title>
<g
id="logog">
<path
id="path138"
d="m 32.86,2 -13,7.5 v 0 h -0.05 v 0 l -0.48,0.28 c -4.27,2.46 -5.68,11.38 -6.06,14.75 L 36.2,11.33 c 0.39,-0.19 7.6,-3.69 13.57,-3.69 h 0.14 L 40.13,2 a 8.15,8.15 0 0 0 -7.27,0"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path142"
d="M 5.68,17.69 A 8.2,8.2 0 0 0 2,24 v 15.78 c 0,4.9 7,10.48 9.75,12.46 V 25.77 c 0,-0.44 0.6,-8.55 3.65,-13.72 z"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path146"
d="m 12.1,54.12 v 0 C 11.74,53.88 5,49.41 2,44.24 v 11.14 a 8.2,8.2 0 0 0 3.64,6.3 l 13.5,7.79 c 4.28,2.46 12.7,-0.77 15.81,-2.12 z"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path150"
d="m 36.79,68 c -0.4,0.19 -7.71,3.75 -13.71,3.69 l 9.78,5.64 a 8.15,8.15 0 0 0 7.27,0 l 13.51,-7.8 c 4.27,-2.46 5.68,-11.39 6.06,-14.75 z"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path154"
d="M 61.2,27.13 V 53.6 c 0,0.44 -0.6,8.55 -3.65,13.72 l 9.77,-5.64 A 8.2,8.2 0 0 0 71,55.38 V 39.59 c 0,-4.94 -7,-10.5 -9.75,-12.46"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path158"
d="M 67.31,17.69 53.81,9.9 C 49.53,7.44 41.11,10.67 38,12 l 22.85,13.23 v 0 a 43.43,43.43 0 0 1 5.7,4.51 24,24 0 0 1 4.45,5.35 V 24 a 8.2,8.2 0 0 0 -3.64,-6.3"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
</g>
<g
id="re" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;font-size:40px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#364d5c;fill-opacity:1;stroke:none"
x="74.101189"
y="54.47554"
id="text3955"><tspan
id="tspan3953"
x="74.101189"
y="54.47554"
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:44px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#364d5c;fill-opacity:1"><tspan
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold'"
id="tspan86">TDM</tspan><tspan
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:sans-serif;-inkscape-font-specification:sans-serif"
id="tspan3845">Ripper</tspan> </tspan></text>
</svg>

Before

Width:  |  Height:  |  Size: 3.4 KiB

View File

@ -1,100 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
viewBox="0 0 366.72 77.08"
version="1.1"
id="svg3945"
width="366.72"
height="77.080002"
sodipodi:docname="tdmtermite.svg"
inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)">
<sodipodi:namedview
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1"
objecttolerance="10"
gridtolerance="10"
guidetolerance="10"
inkscape:pageopacity="0"
inkscape:pageshadow="2"
inkscape:window-width="1920"
inkscape:window-height="1168"
id="namedview19"
showgrid="false"
inkscape:zoom="3.8766824"
inkscape:cx="199.46638"
inkscape:cy="38.540001"
inkscape:window-x="2048"
inkscape:window-y="0"
inkscape:window-maximized="1"
inkscape:current-layer="svg3945" />
<metadata
id="metadata3951">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title>flasher</dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<defs
id="defs3949" />
<title
id="title3916">flasher</title>
<g
id="logog">
<path
id="path138"
d="m 32.86,2 -13,7.5 v 0 h -0.05 v 0 l -0.48,0.28 c -4.27,2.46 -5.68,11.38 -6.06,14.75 L 36.2,11.33 c 0.39,-0.19 7.6,-3.69 13.57,-3.69 h 0.14 L 40.13,2 a 8.15,8.15 0 0 0 -7.27,0"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path142"
d="M 5.68,17.69 A 8.2,8.2 0 0 0 2,24 v 15.78 c 0,4.9 7,10.48 9.75,12.46 V 25.77 c 0,-0.44 0.6,-8.55 3.65,-13.72 z"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path146"
d="m 12.1,54.12 v 0 C 11.74,53.88 5,49.41 2,44.24 v 11.14 a 8.2,8.2 0 0 0 3.64,6.3 l 13.5,7.79 c 4.28,2.46 12.7,-0.77 15.81,-2.12 z"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path150"
d="m 36.79,68 c -0.4,0.19 -7.71,3.75 -13.71,3.69 l 9.78,5.64 a 8.15,8.15 0 0 0 7.27,0 l 13.51,-7.8 c 4.27,-2.46 5.68,-11.39 6.06,-14.75 z"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path154"
d="M 61.2,27.13 V 53.6 c 0,0.44 -0.6,8.55 -3.65,13.72 l 9.77,-5.64 A 8.2,8.2 0 0 0 71,55.38 V 39.59 c 0,-4.94 -7,-10.5 -9.75,-12.46"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
<path
id="path158"
d="M 67.31,17.69 53.81,9.9 C 49.53,7.44 41.11,10.67 38,12 l 22.85,13.23 v 0 a 43.43,43.43 0 0 1 5.7,4.51 24,24 0 0 1 4.45,5.35 V 24 a 8.2,8.2 0 0 0 -3.64,-6.3"
transform="translate(-2.04,-1.15)"
style="fill:#364d5c" />
</g>
<g
id="re" />
<text
xml:space="preserve"
style="font-style:normal;font-weight:normal;font-size:40px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#364d5c;fill-opacity:1;stroke:none"
x="74.101189"
y="54.47554"
id="text3955"><tspan
id="tspan3953"
x="74.101189"
y="54.47554"
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:44px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#364d5c;fill-opacity:1"><tspan
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold'"
id="tspan86">TDMtermite</tspan> </tspan></text>
</svg>

Before

Width:  |  Height:  |  Size: 4.0 KiB

56
example.py Normal file
View File

@ -0,0 +1,56 @@
import tdm_ripper
import numpy as np
import matplotlib.pyplot as plt
tdmpatha = b"samples/SineData.tdm"
tdxpatha = b"samples/SineData.tdx"
tdmpathb = b"/Users/mariofink/Downloads/CONTI_HBS_samples/config_id_1/001_Test_DIAdem/Messung.tdm"
tdxpathb = b"/Users/mariofink/Downloads/CONTI_HBS_samples/config_id_1/001_Test_DIAdem/Messung.tdx"
tdmpathchz = tdmpathb
tdxpathchz = tdxpathb
# create instance of ripper class
# RP = tdm_ripper.pytdmripper(tdmpath)
RP = tdm_ripper.pytdmripper(tdmpathchz,tdxpathchz)
# provide overview of available channels
RP.show_channels()
print(RP.num_channels())
print(RP.num_groups())
for i in range(0,RP.num_groups()):
print(str(i+1).rjust(10)+str(RP.no_channels(i)).rjust(10))
# print particular channel to file
RP.print_channel(1,b"SineData_extract.dat")
# show some meta information
print(RP.meta_info(b"SMP_Name").decode('utf-8'))
print(RP.meta_info(b"Location").decode('utf-8'))
print('\n')
RP.print_meta(b"meta_information.dat")
# extract channel and return it to numpy array
# channels = RP.get_channel(1)
# Nlen = len(channels)
# channels = np.append(channels,RP.get_channel(2))
# channels = np.append(channels,RP.get_channel(3))
# channels = np.append(channels,RP.get_channel(4))
# channels = np.append(channels,RP.get_channel(5))
# channels = np.append(channels,RP.get_channel(6))
# channels = np.append(channels,RP.get_channel(7))
# channels = np.append(channels,RP.get_channel(8))
# print(channels.shape)
# print("\n\n")
# print(channels[0:40])
#
# x = np.linspace(0,Nlen,Nlen)
# plt.plot(x,channels[0:Nlen])
# plt.plot(x,channels[Nlen:2*Nlen])
# plt.plot(x,channels[2*Nlen:3*Nlen])
#
# plt.grid()
# plt.show()

62
extract_all.py Normal file
View File

@ -0,0 +1,62 @@
import tdm_ripper
import numpy as np
import argparse
import re
parser = argparse.ArgumentParser(description='provide path of both .tdm and corresponding .tdx file')
parser.add_argument('tdm_file',type=str,help='path of .tdm file')
parser.add_argument('tdx_file',type=str,help='path of .tdx file')
#parser.add_argument('--tdx_file',type=str,help='path of .tdx file',default='')
parser.add_argument('--out_directory',type=str,help='choose directory where to write data',default='./')
parser.add_argument('--prefix_name',type=str,help='provide dataset name used as filename prefix',default='')
args = parser.parse_args()
#print(args)
# process arguments
tdmpath = args.tdm_file #"samples/SineData.tdm"
tdxpath = args.tdx_file #"samples/SineData.tdx"
outdirx = args.out_directory
fprefix = args.prefix_name
# if no prefix is given, .tdm filename will be used
if fprefix == '' :
fprefix = tdmpath.rstrip('.tdm').split('/')[-1]
# TODO better use os.path !!
#print(fprefix)
# create instance of ripper class
RP = tdm_ripper.pytdmripper(tdmpath.encode('utf-8'),tdxpath.encode('utf-8'))
# provide overview over available groups and channels
#RP.show_groups()
#RP.show_channels()
# obtain number of available groups and channels
numgr = RP.num_groups()
numch = RP.num_channels()
# dump all meta information
RP.print_meta((outdirx+fprefix+'.csv').encode('utf-8'))
# dump all available groups and channels
for g in range(0,numgr):
numgrch = RP.no_channels(g)
for c in range(0,numgrch):
#print(str(g).rjust(10)+str(c).rjust(10))
# print(str(RP.channel_length(g,c)))
# print(RP.channel_name(g,c))
#print(RP.channel(g,c))
# obtained overall channel id
chid = RP.obtain_channel_id(g,c)
# get group's and channel's name
gname = RP.group_name(g)
cname = RP.channel_name(g,c)
#print(gname.rjust(30)+cname.rjust(30))
# use regular expression replacement to sanitize group and channel names
gname = re.sub('[!@#$%^&*()-+= ,]','',gname)
cname = re.sub('[!@#$%^&*()-+= ,]','',cname)
# generate filename
fichan = fprefix + '_' + str(g+1) + '_' + str(c+1) + '_' + gname + '_' + cname + '.csv'
# print channel
RP.print_channel(chid,(outdirx+fichan).encode('utf-8'))

14
lib/makefile Normal file
View File

@ -0,0 +1,14 @@
CC = gcc -std=c++11 -stdlib=libc++
CPPFLAGS = -O3 -Wall -Werror
LIB = ../pugixml/
libtdmripper.a : tdm_ripper.o
ar rcs $@ $^
tdm_ripper.o : tdm_ripper.cpp tdm_ripper.hpp
$(CC) -c $(CPPFLAGS) -I $(LIB) $< -o $@
clean :
rm -f *.o *.a

View File

@ -1,343 +0,0 @@
// ------------------------------------------------------------------------- //
/*
for reference of the tdm data model, see
https://zone.ni.com/reference/de-XX/help/370858P-0113/tdmdatamodel/tdmdatamodel/tdm_headerfile/
https://zone.ni.com/reference/de-XX/help/370858P-0113/tdmdatamodel/tdmdatamodel/tdm_datamodel/
*/
#ifndef TDM_DATAMODEL
#define TDM_DATAMODEL
#include <iostream>
#include <iterator>
#include <vector>
#include <chrono>
#include <sstream>
#include "tdm_format.hpp"
// -------------------------------------------------------------------------- //
// tdm meta data
struct tdm_meta {
// usi:documentation
std::string docu_expo_, docu_expover_;
// usi:model
std::string model_name_, model_version_;
std::string model_include_uri_;
// usi:include
std::string byte_order_; // little versus big endian
std::string file_url_; // path/URL of corresponding .tdx file
const std::string get_info() { return get_info(defformat); }
const std::string get_info(format& formatter)
{
formatter.set_columns({ std::make_pair("exporter",docu_expo_),
std::make_pair("exporterVersion",docu_expover_),
std::make_pair("modelName",model_name_),
std::make_pair("modelVersion",model_version_),
std::make_pair("modelnsURI",model_include_uri_),
std::make_pair("byteOrder",byte_order_),
std::make_pair("fileURL",file_url_) } );
return formatter.get_info();
}
};
// -------------------------------------------------------------------------- //
// block of data
struct block {
std::string id_;
unsigned long int byte_offset_;
unsigned long int length_;
unsigned long int block_offset_, block_size_;
std::string value_type_;
block () {
id_ = std::string("");
byte_offset_ = 0;
length_ = 0;
block_offset_ = 0;
block_size_ = 0;
value_type_ = std::string("");
}
const std::string get_info() { return get_info(defformat); }
const std::string get_info(format& formatter)
{
formatter.set_columns({ std::make_pair("block-id",id_),
std::make_pair("byteOffset",std::to_string(byte_offset_)),
std::make_pair("length",std::to_string(length_)),
std::make_pair("blockOffset",std::to_string(block_offset_)),
std::make_pair("blockSize",std::to_string(block_size_)),
std::make_pair("valueType",value_type_) });
return formatter.get_info();
}
};
// -------------------------------------------------------------------------- //
// tdm_root
struct tdm_root {
std::string id_;
std::string name_;
std::string description_;
std::string title_;
std::string author_;
std::string timestamp_;
// std::chrono::time_point timepoint_; // from string 2008-05-06T17:20:12.65074539184570313
// std::stringstream ss;
// ss<<"2008-05-06T17:20:12.65074539184570313";
// std::cout<<ss.str()<<"\n";
//
// std::chrono::time_point start = std::chrono::high_resolution_clock::now();
// std::time_t tt = std::chrono::system_clock::to_time_t(start);
// // https://en.cppreference.com/w/cpp/io/manip/put_time
// std::cout<<std::put_time(std::localtime(&tt),"%Y-%m-%dT%H:%M:%S")<<"\n";// "%F %T")<<"\n";
//
// // std::tm ts;
// // // https://en.cppreference.com/w/cpp/io/manip/get_time
// // std::get_time(&ts, "%Y-%m-%dT%H:%M:%S");
// // auto tp = std::chrono::system_clock::from_time_t(std::mktime(&ts));
// // std::time_t tt = std::chrono::system_clock::to_time_t(tp);
// // // std::cout<<tt.strftime("%Y")<<"\n";
// // std::cout<<ctime(&tt)<<"\n";
std::vector<std::string> channelgroups_;
const std::string get_info() { return get_info(defformat); }
const std::string get_info(format& formatter)
{
formatter.set_columns({ std::make_pair("root-id",id_),
std::make_pair("name",name_),
std::make_pair("description",description_),
std::make_pair("title",title_),
std::make_pair("author",author_),
std::make_pair("timestamp",timestamp_),
std::make_pair("channelgroups",join_strings(channelgroups_)) });
return formatter.get_info();
}
};
// -------------------------------------------------------------------------- //
// tdm_channelgroup
// https://zone.ni.com/reference/de-XX/help/370858P-0113/tdmdatamodel/tdmdatamodel/tdm_metadata_chngroup/
struct tdm_channelgroup {
std::string id_;
std::string name_;
std::string description_;
std::string root_;
std::vector<std::string> channels_; // referenced by id
std::vector<std::string> submatrices_;
const std::string get_info() { return get_info(defformat); }
const std::string get_info(format& formatter)
{
formatter.set_columns({ std::make_pair("group-id",id_),
std::make_pair("name",name_),
std::make_pair("description",description_),
std::make_pair("root",root_),
std::make_pair("channels",join_strings(channels_)),
std::make_pair("submatrices",join_strings(submatrices_)) });
return formatter.get_info();
}
const std::string get_json()
{
std::stringstream ss;
ss<<"{"<<"\"group-id\":\""<<id_
<<"\",\"name\":\""<<name_
<<"\",\"description\":\""<<description_
<<"\",\"root\":\""<<root_
<<"\",\"channels\":\""<<join_strings(channels_)<<"\"}";
return ss.str();
}
};
// -------------------------------------------------------------------------- //
// tdm_channel
// https://zone.ni.com/reference/de-XX/help/370858P-0113/tdmdatamodel/tdmdatamodel/tdm_metadata_channel/
// waveform channel type
enum class wf_time_pref_type {
absolute,
relative
};
// additional elements for wave form channels (encoded as attributes in
// <instance_attributes> of <tdm_channel>)
// https://zone.ni.com/reference/de-XX/help/370858P-0113/tdmdatamodel/tdmdatamodel/tdm_tdxdata_waveform/
struct waveform_channel {
std::string wf_xname_;
std::string wf_xunit_string_;
std::string wf_start_time_;
double wf_start_offset_;
double wf_increment_;
unsigned long wf_samples_;
wf_time_pref_type wf_time_pref;
};
struct tdm_channel {
std::string id_;
std::string name_;
std::string description_;
std::string unit_string_;
std::string datatype_;
double minimum_, maximum_;
std::string group_;
std::vector<std::string> local_columns_;
// TODO
waveform_channel wf_channel_;
const std::string get_info() { return get_info(defformat); }
const std::string get_info(format& formatter)
{
formatter.set_columns({ std::make_pair("channel-id",id_),
std::make_pair("name",name_),
std::make_pair("description",description_),
std::make_pair("unit_string",unit_string_),
std::make_pair("datatype",datatype_),
std::make_pair("minimum",std::to_string(minimum_)),
std::make_pair("maximum",std::to_string(maximum_)),
std::make_pair("group",group_),
std::make_pair("local_columns",join_strings(local_columns_)) });
return formatter.get_info();
}
const std::string get_json()
{
std::stringstream ss;
ss<<"{"<<"\"channel-id\":\""<<id_
<<"\",\"name\":\""<<name_
<<"\",\"description\":\""<<description_
<<"\",\"unit_string\":\""<<unit_string_
<<"\",\"datatype\":\""<<datatype_
<<"\",\"minimum\":\""<<minimum_
<<"\",\"maximum\":\""<<maximum_
<<"\",\"group\":\""<<group_<<"\"}";
return ss.str();
}
};
// -------------------------------------------------------------------------- //
// submatrix
// https://zone.ni.com/reference/de-XX/help/370858P-0113/tdmdatamodel/tdmdatamodel/tdm_tdxdata_submatrix/
struct submatrix {
std::string id_;
std::string name_;
std::string description_;
std::string measurement_; // -> tdm_channelgroup id
std::vector<std::string> local_columns_; // -> list of type "localcolumn"
unsigned long int number_of_rows_; // -> number of values in channels
const std::string get_info() { return get_info(defformat); }
const std::string get_info(format& formatter)
{
formatter.set_columns({ std::make_pair("submatrix-id",id_),
std::make_pair("name",name_),
std::make_pair("description",description_),
std::make_pair("measurement",measurement_),
std::make_pair("local_columns",join_strings(local_columns_)),
std::make_pair("number_of_rows",std::to_string(number_of_rows_)) });
return formatter.get_info();
}
};
// -------------------------------------------------------------------------- //
// localcolumn
enum class representation {
explicit_, // !! explicit is C++ keyword!!
implicit_linear_, // datatype is always DT_DOUBLE, no <value_sequence> for implicit_linear_!!
raw_linear_ // datatype is always DT_DOUBLE
};
// https://zone.ni.com/reference/de-XX/help/370858P-0113/tdmdatamodel/tdmdatamodel/tdm_tdxdata_localcolumn/
struct localcolumn {
std::string id_;
std::string name_;
std::string description_;
std::string measurement_quantity_; // -> tdm_channel
std::string submatrix_;
unsigned long int global_flag_;
unsigned long int independent_;
double minimum_, maximum_;
// representation sequence_representation_;
std::string sequence_representation_;
std::vector<double> generation_parameters_; // { offset, factor }
std::string values_; // -> refers to usi:data -> _sequence
std::string external_id_;
localcolumn () {
id_ = std::string("");
name_ = std::string("");
description_ = std::string("");
measurement_quantity_ = std::string("");
submatrix_ = std::string("");
global_flag_ = 15;
independent_ = 0;
minimum_ = 0.0;
maximum_ = 0.0;
sequence_representation_ = std::string("explicit");
generation_parameters_ = { 0.0, 1.0 };
values_ = std::string("");
external_id_ = std::string("");
}
const std::string get_info() { return get_info(defformat); }
const std::string get_info(format& formatter)
{
formatter.set_columns({ std::make_pair("localcolumn-id",id_),
std::make_pair("name",name_),
std::make_pair("description",description_),
std::make_pair("measurement_quantity",measurement_quantity_),
std::make_pair("submatrix_",submatrix_),
std::make_pair("minimum",std::to_string(minimum_)),
std::make_pair("maximum",std::to_string(maximum_)),
std::make_pair("sequence_representation",sequence_representation_),
std::make_pair("generation_parameters",join<double>(generation_parameters_)),
std::make_pair("values",values_),
std::make_pair("external",external_id_) });
return formatter.get_info();
}
};
#endif
// -------------------------------------------------------------------------- //

View File

@ -1,354 +0,0 @@
// -------------------------------------------------------------------------- //
#ifndef TDMDATATYPE
#define TDMDATATYPE
// https://zone.ni.com/reference/de-XX/help/370858P-0113/tdmdatamodel/tdmdatamodel/tdm_header_tdx_data/
// !!!! define mapping of locally supported datatypes to tdm datatypes
// !!!! this is where the magic happens !!!
typedef short int eInt16Usi;
typedef int eInt32Usi;
typedef unsigned char eUInt8Usi;
typedef unsigned short int eUInt16Usi;
typedef unsigned int eUInt32Usi;
typedef float eFloat32Usi;
typedef double eFloat64Usi;
typedef char eStringUsi;
class tdmdatatype
{
protected:
eInt16Usi sint16_; // 0
eInt32Usi sint32_; // 1
eUInt8Usi uint8_; // 2
eUInt16Usi uint16_; // 3
eUInt32Usi uint32_; // 4
eFloat32Usi float32_; // 5
eFloat64Usi float64_; // 6
eStringUsi string_; // 7
short int dtidx_; // \in \{0,...,7\}
public:
tdmdatatype(): sint16_(0), sint32_(0),
uint8_(0), uint16_(0), uint32_(0),
float32_(0.0), float64_(0.0), string_(0),
dtidx_(0) { };
// every supported datatype gets its own constructor
tdmdatatype(eInt16Usi num): sint16_(num), dtidx_(0) {};
tdmdatatype(eInt32Usi num): sint32_(num), dtidx_(1) {};
tdmdatatype(eUInt8Usi num): uint8_(num), dtidx_(2) {};
tdmdatatype(eUInt16Usi num): uint16_(num), dtidx_(3) {};
tdmdatatype(eUInt32Usi num): uint32_(num), dtidx_(4) {};
tdmdatatype(eFloat32Usi num): float32_(num), dtidx_(5) {};
tdmdatatype(eFloat64Usi num): float64_(num), dtidx_(6) {};
tdmdatatype(eStringUsi num): string_(num), dtidx_(7) {};
// identify type
short int& dtype() { return dtidx_; }
// copy constructor
tdmdatatype(const tdmdatatype &num)
{
this->sint16_ = num.sint16_;
this->sint32_ = num.sint32_;
this->uint8_ = num.uint8_;
this->uint16_ = num.uint16_;
this->uint32_ = num.uint32_;
this->float32_ = num.float32_;
this->float64_ = num.float64_;
this->string_ = num.string_;
this->dtidx_ = num.dtidx_;
}
// overall assignment operator
tdmdatatype& operator=(const tdmdatatype &num)
{
if ( this != &num )
{
this->sint16_ = num.sint16_;
this->sint32_ = num.sint32_;
this->uint8_ = num.uint8_;
this->uint16_ = num.uint16_;
this->uint32_ = num.uint32_;
this->float32_ = num.float32_;
this->float64_ = num.float64_;
this->string_ = num.string_;
this->dtidx_ = num.dtidx_;
}
return *this;
}
// implement assignment operator for individual datatypes
tdmdatatype& operator=(const eInt16Usi &num)
{
this->sint16_ = num;
this->dtidx_ = 0;
return *this;
}
tdmdatatype& operator=(const eInt32Usi &num)
{
this->sint32_ = num;
this->dtidx_ = 1;
return *this;
}
tdmdatatype& operator=(const eUInt8Usi &num)
{
this->uint8_ = num;
this->dtidx_ = 2;
return *this;
}
tdmdatatype& operator=(const eUInt16Usi &num)
{
this->uint16_ = num;
this->dtidx_ = 3;
return *this;
}
tdmdatatype& operator=(const eUInt32Usi &num)
{
this->uint32_ = num;
this->dtidx_ = 4;
return *this;
}
tdmdatatype& operator=(const eFloat32Usi &num)
{
this->float32_ = num;
this->dtidx_ = 5;
return *this;
}
tdmdatatype& operator=(const eFloat64Usi &num)
{
this->float64_ = num;
this->dtidx_ = 6;
return *this;
}
tdmdatatype& operator=(const eStringUsi &num)
{
this->string_ = num;
this->dtidx_ = 7;
return *this;
}
// obtain number as double
double as_double()
{
double num = 0.0;
if ( dtidx_ == 0 ) num = (double)sint16_;
else if ( dtidx_ == 1 ) num = (double)sint32_;
else if ( dtidx_ == 2 ) num = (double)uint8_;
else if ( dtidx_ == 3 ) num = (double)uint16_;
else if ( dtidx_ == 4 ) num = (double)uint32_;
else if ( dtidx_ == 5 ) num = (double)float32_;
else if ( dtidx_ == 6 ) num = (double)float64_;
else if ( dtidx_ == 7 ) num = (double)(int)string_;
return num;
}
// define custom stream operator to print the correct type
friend std::ostream& operator<<(std::ostream& out, const tdmdatatype& num)
{
if ( num.dtidx_ == 0 ) out<<num.sint16_;
else if ( num.dtidx_ == 1 ) out<<num.sint32_;
else if ( num.dtidx_ == 2 ) out<<num.uint8_;
else if ( num.dtidx_ == 3 ) out<<num.uint16_;
else if ( num.dtidx_ == 4 ) out<<num.uint32_;
else if ( num.dtidx_ == 5 ) out<<num.float32_;
else if ( num.dtidx_ == 6 ) out<<num.float64_;
else if ( num.dtidx_ == 7 ) out<<num.string_;
return out;
}
};
// // base class for all tdm datatypes
// class tdmdatatype
// {
// protected:
// short int sint16_;
// int sint32_;
// unsigned char uint8_;
// unsigned short int uint16_;
// unsigned int uint32_;
// float float32_;
// double float64_;
// public:
// tdmdatatype(): sint16_(0), sint32_(0),
// uint8_(0), uint16_(0), uint32_(0),
// float32_(0.0), float64_(0.0) {};
// virtual ~tdmdatatype() = default;
// friend std::ostream& operator<<(std::ostream& out, const tdmdatatype& num)
// {
// return num.print(out);
// }
// virtual std::ostream& print(std::ostream& out) const
// {
// out<<"tdmdatatype";
// return out;
// }
// };
// class eInt16Usi: public tdmdatatype
// {
// public:
// eInt16Usi() { }
// eInt16Usi(short int num) { sint16_ = num; }
// // eInt16Usi& operator=(const eInt16Usi &num)
// // {
// // // self-assignment check
// // if ( this != &num)
// // {
// // this->sint16_ = num.sint16_;
// // }
// // return *this;
// // }
// friend std::ostream& operator<<(std::ostream& out, const eInt16Usi& num)
// {
// return num.print(out);
// }
// std::ostream& print(std::ostream& out) const override
// {
// out<<sint16_;
// return out;
// }
// };
//
// class eInt32Usi: public tdmdatatype
// {
// public:
// eInt32Usi() { }
// eInt32Usi(int num) { sint32_ = num; }
// friend std::ostream& operator<<(std::ostream& out, const eInt32Usi& num)
// {
// return num.print(out);
// }
// std::ostream& print(std::ostream& out) const override
// {
// out<<sint32_;
// return out;
// }
// };
//
// class eUInt8Usi: public tdmdatatype
// {
// public:
// eUInt8Usi() { }
// eUInt8Usi(int num) { uint8_ = num; }
// friend std::ostream& operator<<(std::ostream& out, const eUInt8Usi& num)
// {
// return num.print(out);
// }
// std::ostream& print(std::ostream& out) const override
// {
// out<<uint8_;
// return out;
// }
// };
//
// class eUInt16Usi: public tdmdatatype
// {
// public:
// eUInt16Usi() { }
// eUInt16Usi(int num) { uint16_ = num; }
// friend std::ostream& operator<<(std::ostream& out, const eUInt16Usi& num)
// {
// return num.print(out);
// }
// std::ostream& print(std::ostream& out) const override
// {
// out<<uint16_;
// return out;
// }
// };
//
// class eUInt32Usi: public tdmdatatype
// {
// public:
// eUInt32Usi() { }
// eUInt32Usi(int num) { uint32_ = num; }
// friend std::ostream& operator<<(std::ostream& out, const eUInt32Usi& num)
// {
// return num.print(out);
// }
// std::ostream& print(std::ostream& out) const override
// {
// out<<uint32_;
// return out;
// }
// };
//
// class eFloat32Usi: public tdmdatatype
// {
// public:
// eFloat32Usi() { }
// eFloat32Usi(int num) { float32_ = num; }
// friend std::ostream& operator<<(std::ostream& out, const eFloat32Usi& num)
// {
// return num.print(out);
// }
// std::ostream& print(std::ostream& out) const override
// {
// out<<float32_;
// return out;
// }
// };
//
// class eFloat64Usi: public tdmdatatype
// {
// public:
// eFloat64Usi() { }
// eFloat64Usi(int num) { float64_ = num; }
// friend std::ostream& operator<<(std::ostream& out, const eFloat64Usi& num)
// {
// return num.print(out);
// }
// std::ostream& print(std::ostream& out) const override
// {
// out<<float64_;
// return out;
// }
// };
struct tdm_datatype {
std::string name_;
std::string channel_datatype_;
int numeric_;
std::string value_sequence_;
unsigned int size_;
std::string description_;
const std::string get_info() { return get_info(defformat); }
const std::string get_info(format& formatter)
{
formatter.set_columns( { std::make_pair("name",name_),
std::make_pair("channel_datatype",channel_datatype_),
std::make_pair("name",name_),
std::make_pair("value_sequence",value_sequence_),
std::make_pair("size",std::to_string(size_)),
std::make_pair("description",description_) } );
return formatter.get_info();
}
};
const std::vector<tdm_datatype> tdm_datatypes = {
{"eInt16Usi","DT_SHORT",2,"short_sequence",2,"signed 16 bit integer"},
{"eInt32Usi","DT_LONG",6,"long_sequence",4,"signed 32 bit integer"},
{"eUInt8Usi","DT_BYTE",5,"byte_sequence",1,"unsigned 8 bit integer"},
{"eUInt16Usi","DT_SHORT",2,"short_sequence",2,"unsigned 16 bit integer"},
{"eUInt32Usi","DT_LONG",6,"long_sequence",4,"unsigned 32 bit integer"},
{"eFloat32Usi","DT_FLOAT",3,"float_sequence",4,"32 bit float"},
{"eFloat64Usi","DT_DOUBLE",7,"double_sequence",8,"64 bit double"},
{"eStringUsi","DT_STRING",1,"string_sequence",1,"text"}
};
#endif
// -------------------------------------------------------------------------- //

View File

@ -1,119 +0,0 @@
// -------------------------------------------------------------------------- //
#ifndef TDMFORMAT
#define TDMFORMAT
// format output of info strings for terminal and file table
class format
{
private:
unsigned int width_;
bool tabular_;
bool header_;
char sep_;
std::vector<std::pair<std::string,std::string>> columns_;
public:
format(int width = 25, bool tabular = false, bool header = false, char sep = ' '):
width_(width), tabular_(tabular), header_(header), sep_(sep)
{
}
void set_width(int width)
{
width_ = width;
}
void set_tabular(bool tabular)
{
tabular_ = tabular;
}
void set_header(bool header)
{
header_ = header;
}
void set_sep(char sep)
{
sep_ = sep;
}
void set_columns(std::vector<std::pair<std::string,std::string>> columns)
{
columns_ = columns;
}
std::string get_info()
{
std::stringstream ss;
for ( std::vector<std::pair<std::string,std::string>>::iterator it = columns_.begin();
it != columns_.end(); ++it )
{
if ( tabular_ )
{
// header or body of table
std::string entry = header_? it->first : it->second;
// make broad aligned columns for human reader
if ( sep_ == ' ' )
{
entry = entry.size() > width_-2 ? entry.substr(0,width_-2) : entry;
// if ( it == columns_.begin() && !header_ ) ss<<" ";
ss<<std::setw(width_)<<std::left<<entry;
}
// make compressed csv like columns
else
{
ss<<entry;
if ( std::next(it,1) != columns_.end() ) ss<<sep_;
}
}
else
{
ss<<std::setw(width_)<<std::left<<(it->first+std::string(":"))<<it->second<<"\n";
}
}
return ss.str();
}
};
// define default formatter
static format defformat(25,false,false,',');
// join a list of strings
static std::string join_strings(std::vector<std::string> &thestring, const char* sep = " ")
{
std::string joined;
for ( std::vector<std::string>::iterator it = thestring.begin();
it != thestring.end(); ++it )
{
joined += std::next(it,1) != thestring.end() ? ( *it + std::string(sep) ) : *it;
}
return joined;
}
// join a list of numbers
template<class numtype>
static std::string join(std::vector<numtype> &thevec, const char* sep = " ")
{
std::string joined;
for ( unsigned int i = 0; i < thevec.size(); i++ )
{
joined += std::to_string(thevec.at(i));
if ( i+1 < thevec.size() ) joined += std::string(sep);
}
return joined;
}
#endif
// -------------------------------------------------------------------------- //

566
lib/tdm_ripper.cpp Normal file
View File

@ -0,0 +1,566 @@
#include "tdm_ripper.hpp"
tdm_ripper::tdm_ripper(std::string tdmfile, std::string tdxfile,
bool suppress_status, bool neglect_empty_groups):
tdmfile_(tdmfile), tdxfile_(tdxfile), suppress_status_(suppress_status),
neglect_empty_groups_(neglect_empty_groups), num_empty_groups_(0),
num_channels_(0), num_groups_(0), channel_id_(0), inc_id_(0), units_(0),
channel_name_(0), group_id_(0), group_name_(0),
num_channels_group_(0), channels_group_(0), channel_ext_(0), minmax_(0),
byteoffset_(0), length_(0), type_(0), external_id_(0)
{
datatypes_ = {
{"eInt8Usi",8},
{"eInt16Usi",16},
{"eInt32Usi",32},
{"eInt64Usi",64},
{"eUInt8Usi",8},
{"eUInt16Usi",16},
{"eUInt32Usi",32},
{"eUInt64Usi",64},
{"eFloat32Usi",32},
{"eFloat64Usi",64}
};
// make sure the provided file is a .tdm file
assert( tdmfile_.compare("") != 0 && "please provide a valid .tdm file" );
std::string::size_type idx;
idx = tdmfile_.find_last_of(".");
assert( idx != std::string::npos && "there's no file extension at all - .tdm is required" );
assert( tdmfile_.substr(tdmfile_.find_last_of(".")+1).compare("tdm") == 0 && "it's not a .tdm file" );
// setup of xml-parser
xml_result_ = xml_doc_.load_file(tdmfile_.c_str());
if ( !suppress_status_ )
{
std::cout<<"\nloading and parsing file: "<<xml_result_.description()<<"\n";
std::cout<<"\nencoding: "<<(pugi::xml_encoding)xml_result_.encoding<<"\n\n";
}
pugi::xml_node subtreeincl = xml_doc_.child("usi:tdm").child("usi:include");
if ( !suppress_status_ )
{
std::cout<<"file modified: "<<xml_doc_.child("usi:tdm").child("usi:data")
.child("tdm_root").child_value("datetime")<<"\n\n";
}
// obtain corresponding .tdx filename given in .tdm file
if ( tdxfile_.compare("") == 0 )
{
tdxfile_ = tdmfile_.substr(0, tdmfile_.find_last_of("\\/"))
+std::string("/")+subtreeincl.child("file").attribute("url").value();
}
// obtain endianness specified in .tdm file
std::string endianness(subtreeincl.child("file").attribute("byteOrder").value());
endianness_ = endianness.compare("littleEndian") == 0 ? true : false;
// obtain machine's endianess
int num = 1;
machine_endianness_ = ( *(char*)&num == 1 );
assert( machine_endianness_ == endianness_ );
if ( !suppress_status_ )
{
std::cout<<"required .tdx-file is '"<<tdxfile_<<"'\n\n";
}
parse_structure();
// open .tdx and stream all binary data into vector
std::ifstream fin(tdxfile_.c_str(),std::ifstream::binary);
assert( fin.good() && "failed to open .tdx-file" );
if ( !fin.good() ) std::cout<<"failed to open .tdx-file\n\n";
// assert( errno == 0 );
// std::cout<<"error code "<<strerror(errno)<<"\n\n";
std::vector<unsigned char> tdxbuf((std::istreambuf_iterator<char>(fin)),
(std::istreambuf_iterator<char>()));
tdxbuf_ = tdxbuf;
if ( !suppress_status_ )
{
std::cout<<"number of bytes in binary file: "<<tdxbuf_.size()<<"\n\n";
}
}
void tdm_ripper::parse_structure()
{
// get node with channel and endianess information
pugi::xml_node subtreefile = xml_doc_.child("usi:tdm").child("usi:include").child("file");
for (pugi::xml_node anode: subtreefile.children())
{
// count overall number of channels
num_channels_++;
// get byteoffset of channel
byteoffset_.push_back(atoi(anode.attribute("byteOffset").value()));
// get length of channel
length_.push_back(atoi(anode.attribute("length").value()));
// find datatype of channel
type_.push_back(anode.attribute("valueType").value());
// external id of channel
external_id_.push_back(anode.attribute("id").value());
}
// get node with channels and groups
pugi::xml_node subtreedata = xml_doc_.child("usi:tdm").child("usi:data");
// extract basic information about available groups
int groupcount = 0;
for (pugi::xml_node anode: subtreedata.children())
{
// get meta-info contained in tdm_root element
if ( std::string(anode.name()).compare("tdm_root") == 0 )
{
// preliminiary: extract some hard-coded information tags only
root_info_.insert(std::pair<std::string,std::string>("name",anode.child_value("name")));
root_info_.insert(std::pair<std::string,std::string>("description",anode.child_value("description")));
root_info_.insert(std::pair<std::string,std::string>("title",anode.child_value("title")));
root_info_.insert(std::pair<std::string,std::string>("author",anode.child_value("author")));
}
if ( std::string(anode.name()).compare("tdm_channelgroup") == 0 )
{
groupcount++;
for ( pugi::xml_node mnode: anode.child("instance_attributes").children() )
{
// preliminiary fix for Conti-TDM files since values are one arbitrary tree level above
bool pretdmfix = ( std::string(mnode.child_value()).compare("") == 0 ) ? false : true;
if ( pretdmfix )
{
meta_info_.insert(std::pair<std::string,std::string>(mnode.attribute("name").value(),mnode.child_value()));
}
else
{
meta_info_.insert(std::pair<std::string,std::string>(mnode.attribute("name").value(),mnode.child_value("s")));
}
}
int numchann = count_occ_string(anode.child_value("channels"),"id");
if ( numchann > 0 || !neglect_empty_groups_ )
{
num_groups_++;
group_id_.push_back(anode.attribute("id").value());
group_name_.push_back(anode.child_value("name"));
num_channels_group_.push_back(numchann);
// get time-stamp
pugi::xml_node insatt = anode.child("instance_attributes");
std::pair<std::string,std::string> startstop;
for ( pugi::xml_node bnode: insatt.children() )
{
assert( std::string(bnode.name()).compare("double_attribute") == 0 );
if ( std::string(bnode.attribute("name").value()).compare("Starttime") == 0 )
{
startstop.first = bnode.child_value();
}
else if ( std::string(bnode.attribute("name").value()).compare("Stoptime") == 0 )
{
startstop.second = bnode.child_value();
}
else
{
startstop.first = "";
startstop.second = "";
}
}
group_timestamp_.push_back(startstop);
}
if ( numchann == 0 ) num_empty_groups_++;
}
}
// obtain list of xpointers and ids to assign channels
for (pugi::xml_node anode: subtreedata.children())
{
if ( std::string(anode.name()).compare("tdm_channel") == 0 )
{
std::string id(anode.attribute("id").value());
std::string val = get_str_between(anode.child_value("local_columns"),"\"","\"");
xml_local_columns_.insert(std::pair<std::string,std::string>(id,val));
}
if ( std::string(anode.name()).compare("localcolumn") == 0 )
{
std::string id(anode.attribute("id").value());
std::string val = get_str_between(anode.child_value("values"),"\"","\"");
xml_values_.insert(std::pair<std::string,std::string>(id,val));
}
if ( std::string(anode.name()).compare("double_sequence") == 0
|| std::string(anode.name()).compare("long_sequence") == 0 )
{
std::string id(anode.attribute("id").value());
std::string val = anode.child("values").attribute("external").value();
xml_double_sequence_.insert(std::pair<std::string,std::string>(id,val));
}
}
if ( !suppress_status_ )
{
std::cout<<"number of pairs in\n";
std::cout<<std::setw(25)<<std::left<<"xml_local_columns_:"<<xml_local_columns_.size()<<"\n";
std::cout<<std::setw(25)<<std::left<<"xml_values_:"<<xml_values_.size()<<"\n";
std::cout<<std::setw(25)<<std::left<<"xml_double_sequence_:"<<xml_double_sequence_.size()<<"\n";
std::cout<<std::right<<"\n\n";
std::cout<<"meta-info snippets "<<meta_info_.size()<<"\n\n";
}
// extract basic information about available channels
// int prog = 0;
for (pugi::xml_node anode: subtreedata.children())
{
if ( std::string(anode.name()).compare("tdm_channel") == 0 )
{
// prog++;
// std::cout<<"processing channel "<<prog<<"\n";
channel_id_.push_back(anode.attribute("id").value());
channel_name_.push_back(anode.child_value("name"));
std::string groupid(anode.child_value("group"));
for ( int g = 0; g < num_groups_; g++ )
{
if ( groupid.find(group_id_[g]) != std::string::npos ) channels_group_.push_back(g);
}
// obtain measurement unit of channel
units_.push_back(anode.child_value("unit_string"));
if ( (*(units_.end()-1)).compare("°C") == 0 ) (*(units_.end()-1)) = "deg. Celsius";
// obtain minimum/maximum of channel
std::pair<double,double> minmaxchan(atof(anode.child_value("minimum")),
atof(anode.child_value("maximum")));
minmax_.push_back(minmaxchan);
// get correct assignment of channels to byteoffset, length and datatype
std::string locolvalext;
locolvalext = xml_double_sequence_[xml_values_[xml_local_columns_[anode.attribute("id").value()]]];
// save external id of channel and get corresponding channel index
inc_id_.push_back(locolvalext);
int extid = 1;
for ( int i = 0; i < (int)external_id_.size(); i++ )
{
if ( external_id_[i].compare(locolvalext) == 0 ) extid = i;
}
channel_ext_.push_back(extid);
}
}
// std::string keyinit("usi23258");
// std::cout<<"xml test "<<xml_double_sequence_[xml_values_[xml_local_columns_[keyinit]]]<<"\n\n";
// for ( auto el: minmax_ ) std::cout<<el.first<<" "<<el.second<<"\n";
// std::cout<<"\n\n";
// check consistency of number of channel-groups
int numgroups = count_occ_string(subtreedata.child("tdm_root").child_value("channelgroups"),"id");
assert( (neglect_empty_groups_ && numgroups == num_groups_+num_empty_groups_)
|| (!neglect_empty_groups_ && numgroups == num_groups_) );
// check consistency of number of channels
assert( num_channels_ == (int)channel_id_.size()
&& num_channels_ == (int)channel_name_.size()
&& num_channels_ == (int)channels_group_.size() );
}
void tdm_ripper::list_channels(std::ostream& gout, int width, int maxshow)
{
gout<<std::setw(width)<<"index";
gout<<std::setw(width)<<"id";
gout<<std::setw(width)<<"inc_id";
gout<<std::setw(2*width)<<"name";
gout<<std::setw(width)<<"offset";
gout<<std::setw(width)<<"length";
gout<<std::setw(width)<<"datatype";
gout<<std::setw(width)<<"unit";
gout<<std::setw(width)<<"minimum";
gout<<std::setw(width)<<"maximum";
gout<<std::setw(width)<<"group";
gout<<std::setw(width)<<"group id";
gout<<std::setw(width)<<"group name";
gout<<std::setw(width)<<"num channels";
gout<<"\n";
gout<<std::setfill('-')<<std::setw(15*width+1)<<"\n";
gout<<std::setfill(' ');
for ( int i = 0; i < num_channels_ && i < maxshow; i++ )
{
gout<<std::setw(width)<<i+1;
gout<<std::setw(width)<<channel_id_[i];
gout<<std::setw(width)<<inc_id_[i];
gout<<std::setw(2*width)<<channel_name_[i];
gout<<std::setw(width)<<byteoffset_[channel_ext_[i]];
gout<<std::setw(width)<<length_[channel_ext_[i]];
gout<<std::setw(width)<<type_[channel_ext_[i]];
gout<<std::setw(width)<<units_[i];
gout<<std::setw(width)<<minmax_[i].first;
gout<<std::setw(width)<<minmax_[i].second;
gout<<std::setw(width)<<channels_group_[i];
gout<<std::setw(width)<<group_id_[channels_group_[i]];
gout<<std::setw(width)<<group_name_[channels_group_[i]];
gout<<std::setw(width)<<num_channels_group_[channels_group_[i]];
gout<<"\n";
}
gout<<"\n\n";
if ( num_channels_ > 3*maxshow )
{
for ( int i = num_channels_-maxshow; i < num_channels_; i++ )
{
gout<<std::setw(width)<<i+1;
gout<<std::setw(width)<<channel_id_[i];
gout<<std::setw(width)<<inc_id_[i];
gout<<std::setw(2*width)<<channel_name_[i];
gout<<std::setw(width)<<byteoffset_[channel_ext_[i]];
gout<<std::setw(width)<<length_[channel_ext_[i]];
gout<<std::setw(width)<<type_[channel_ext_[i]];
gout<<std::setw(width)<<units_[i];
gout<<std::setw(width)<<minmax_[i].first;
gout<<std::setw(width)<<minmax_[i].second;
gout<<std::setw(width)<<channels_group_[i];
gout<<std::setw(width)<<group_id_[channels_group_[i]];
gout<<std::setw(width)<<group_name_[channels_group_[i]];
gout<<std::setw(width)<<num_channels_group_[channels_group_[i]];
gout<<"\n";
}
gout<<"\n\n";
}
}
void tdm_ripper::list_groups(std::ostream& gout, int width, int maxshow)
{
// if present, show timestamps
bool showts = ( group_timestamp_[0].first.compare("") != 0 );
gout<<std::setw(width)<<"group";
gout<<std::setw(width)<<"group id";
gout<<std::setw(width)<<"group name";
gout<<std::setw(width)<<"num channels";
if ( showts ) gout<<std::setw(2*width)<<"start time";
if ( showts ) gout<<std::setw(2*width)<<"stop time";
gout<<"\n";
gout<<std::setfill('-')<<(showts?std::setw(8*width+1):std::setw(4*width+1))<<"\n";
gout<<std::setfill(' ');
for ( int i = 0; i < num_groups_ && i < maxshow; i++ )
{
gout<<std::setw(width)<<i+1;
gout<<std::setw(width)<<group_id_[i];
gout<<std::setw(width)<<group_name_[i];
gout<<std::setw(width)<<num_channels_group_[i];
if ( showts ) gout<<std::setw(2*width)<<time_stamp(i,true);
if ( showts ) gout<<std::setw(2*width)<<time_stamp(i,false);
gout<<"\n";
}
gout<<"\n\n";
if ( num_groups_ > 3*maxshow )
{
for ( int i = num_groups_-maxshow; i < num_groups_; i++ )
{
gout<<std::setw(width)<<i+1;
gout<<std::setw(width)<<group_id_[i];
gout<<std::setw(width)<<group_name_[i];
gout<<std::setw(width)<<num_channels_group_[i];
if ( showts ) gout<<std::setw(2*width)<<time_stamp(i,true);
if ( showts ) gout<<std::setw(2*width)<<time_stamp(i,false);
gout<<"\n";
}
gout<<"\n\n";
}
}
void tdm_ripper::show_structure()
{
int width = 25;
std::cout<<"second level tree elements:\n";
for ( pugi::xml_node child: xml_doc_.child("usi:tdm").children())
{
std::cout<<child.name()<<"\n";
}
std::cout<<"\n\n";
pugi::xml_node subtreeincl = xml_doc_.child("usi:tdm").child("usi:include");
// most important information in .tdm file
// - byteOffset provides the starting position of particular channel
// - length is the number of e.g. double (=8byte) value in that channel
std::cout<<"file properties:\n\n";
for (pugi::xml_node anode: subtreeincl.children("file"))
{
for (pugi::xml_attribute attr: anode.attributes())
{
std::cout<<" "<<attr.name()<<" = "<<attr.value()<<" ";
}
std::cout<<"\n\n";
int iter = 0;
for (pugi::xml_node child: anode.children())
{
if ( iter < 100 )
{
std::cout<<std::right;
std::cout<<std::setw(width)<<iter;
std::cout<<std::setw(width)<<child.name();
std::cout<<std::setw(width)<<child.value();
for (pugi::xml_attribute attr: child.attributes())
{
std::cout<<std::right<<attr.name()<<" = "<<std::setw(width)<<std::left<<attr.value()<<" ";
if ( std::string(attr.name()).compare("valueType") == 0 )
{
std::cout<<"number of bytes = "<<datatypes_[attr.value()]/CHAR_BIT;
}
}
std::cout<<"\n";
}
iter++;
}
}
std::cout<<"\n\n";
}
void tdm_ripper::list_datatypes()
{
// show datatype size on machine
int width = 30;
std::cout<<std::setw(width)<<"size of short int: "<<sizeof(short int)<<"\n";
std::cout<<std::setw(width)<<"size of int: "<<sizeof(int)<<"\n";
std::cout<<std::setw(width)<<"size of long int: "<<sizeof(long int)<<"\n";
std::cout<<std::setw(width)<<"size of unsigned short int: "<<sizeof(unsigned short int)<<"\n";
std::cout<<std::setw(width)<<"size of unsigned int: "<<sizeof(unsigned int)<<"\n";
std::cout<<std::setw(width)<<"size of unsigned long int: "<<sizeof(unsigned long int)<<"\n\n";
std::cout<<std::setw(width)<<"size of float: "<<sizeof(float)<<"\n";
std::cout<<std::setw(width)<<"size of double: "<<sizeof(double)<<"\n";
std::cout<<std::setw(width)<<"size of long double: "<<sizeof(long double)<<"\n\n";
}
// convert array of chars to integer
int tdm_ripper::convert_int(std::vector<unsigned char> bych)
{
assert( bych.size() == sizeof(int) );
assert( endianness_ );
int df = 0.0;
uint8_t *dfcast = reinterpret_cast<uint8_t*>(&df);
for ( int i = 0; i < (int)sizeof(int); i++ )
{
dfcast[i] = (int)bych[i];
}
return df;
}
// disassemble single integer into array of chars
std::vector<unsigned char> tdm_ripper::convert_int(int df)
{
assert( endianness_ );
std::vector<unsigned char> bych((int)sizeof(int));
uint8_t *dfcast = reinterpret_cast<uint8_t*>(&df);
for ( int i = 0; i < (int)sizeof(int); i++ )
{
bych[i] = (int)dfcast[i];
}
return bych;
}
// convert array of chars to floating point double
double tdm_ripper::convert_double(std::vector<unsigned char> bych)
{
assert( bych.size() == sizeof(double) );
assert( endianness_ );
// check for IEEE754 floating point standard
assert( std::numeric_limits<double>::is_iec559 );
double df = 0.0;
uint8_t *dfcast = reinterpret_cast<uint8_t*>(&df);
for ( int i = 0; i < (int)sizeof(double); i++ )
{
dfcast[i] = (int)bych[i];
}
return df;
}
std::vector<double> tdm_ripper::convert_channel(int channelid)
{
// obtain offset, length of channel and size of datatype
int byteoffset = byteoffset_[channelid];
int length = length_[channelid];
int typesize = datatypes_[type_[channelid]]/CHAR_BIT;
// declare resulting array
std::vector<double> chann(length);
for ( int i = 0; i < length; i++ )
{
std::vector<unsigned char> cseg(tdxbuf_.begin()+byteoffset+i*typesize,
tdxbuf_.begin()+byteoffset+(i+1)*typesize);
if ( type_[channelid].compare("eInt32Usi") == 0 )
{
chann[i] = convert_int(cseg);
}
else if ( type_[channelid].compare("eFloat64Usi") == 0 )
{
chann[i] = convert_double(cseg);
}
else
{
assert( false && "datatype not supported!" );
}
}
return chann;
}
std::vector<double> tdm_ripper::get_channel(int channelid)
{
assert( channelid >= 0 && channelid < num_channels_ && "please provide valid channel id" );
std::vector<double> chann = convert_channel(channel_ext_[channelid]);
// check if converted value is within expected range
for ( int i = 0; i < (int)chann.size(); i++ )
{
assert( chann[i] >= minmax_[channelid].first - 1.0e-6
&& chann[i] <= minmax_[channelid].second + 1.0e-6 );
}
return chann;
}
void tdm_ripper::print_channel(int channelid, const char* filename, int width)
{
assert( channelid >= 0 && channelid < num_channels_ && "please provide valid channel id" );
std::ofstream fout(filename);
std::vector<double> channdat = get_channel(channelid);
for ( auto el: channdat ) fout<<std::setw(width)<<el<<"\n";
fout.close();
}

360
lib/tdm_ripper.hpp Normal file
View File

@ -0,0 +1,360 @@
#ifndef TDM_RIPPER
#define TDM_RIPPER
#include <iostream>
#include <fstream>
#include <iterator>
#include <vector>
#include <iomanip>
#include <stdlib.h>
#include <assert.h>
#include <map>
#include <numeric>
#include <algorithm>
#include "../pugixml/pugixml.hpp"
class tdm_ripper
{
// .tdm and .tdx filenames
std::string tdmfile_;
std::string tdxfile_;
bool suppress_status_;
// endianness (true = little, false = big)
bool endianness_, machine_endianness_;
// evtl. neglect groups with no actual channels
bool neglect_empty_groups_;
int num_empty_groups_;
// number/names/ids of channels, channelgroups and channels's assignment to groups
int num_channels_, num_groups_;
std::vector<std::string> channel_id_, inc_id_, units_, channel_name_;
std::vector<std::string> group_id_, group_name_;
std::vector<std::pair<std::string,std::string>> group_timestamp_;
std::vector<int> num_channels_group_;
std::vector<int> channels_group_;
std::vector<int> channel_ext_;
// minimum/maximum value in particular channel (is provided in .tdm file as float)
std::vector<std::pair<double,double>> minmax_;
// use xpointers and ids to assign channels to byteoffsets
std::map<std::string,std::string> xml_local_columns_, xml_values_, xml_double_sequence_;
// byteoffset, length and datatype of channels
std::vector<int> byteoffset_;
std::vector<int> length_;
std::vector<std::string> type_;
std::vector<std::string> external_id_;
// mapping of NI datatype to size (in bytes) of type
std::map<std::string, int> datatypes_;
// xml parser
pugi::xml_document xml_doc_;
pugi::xml_parse_result xml_result_;
// .tdm-file eventually contains some meta information (about measurement)
std::map<std::string,std::string> root_info_;
std::map<std::string,std::string> meta_info_;
// binary data container
std::vector<unsigned char> tdxbuf_;
public:
tdm_ripper(std::string tdmfile, std::string tdxfile = "",
bool suppress_status = true, bool neglect_empty_groups = true);
void parse_structure();
void list_channels(std::ostream& gout = std::cout, int width = 15, int maxshow = 50);
void list_groups(std::ostream& gout = std::cout, int width = 15, int maxshow = 50);
void show_structure();
// count number of occurences of substring in string
int count_occ_string(std::string s, std::string sub)
{
int num_occs = 0;
std::string::size_type pos = 0;
while ( ( pos = s.find(sub,pos) ) != std::string::npos )
{
num_occs++;
pos += sub.length();
}
return num_occs;
}
// obtain substring of 'entirestr' in between starting and stopping delimiter
std::string get_str_between(std::string entirestr, std::string startlim, std::string stoplim)
{
std::size_t apos = entirestr.find(startlim);
std::size_t bpos = entirestr.find_last_of(stoplim);
assert( apos != std::string::npos && bpos != std::string::npos );
return entirestr.substr(apos+startlim.length(),bpos-(apos+startlim.length()));
}
void print_hash_local(const char* filename, int width = 20)
{
std::ofstream fout(filename);
std::map<std::string,std::string>::iterator it;
int count = 0;
for ( it = xml_local_columns_.begin(); it != xml_local_columns_.end(); it++ )
{
count++;
fout<<std::setw(width)<<count;
fout<<std::setw(width)<<it->first;
fout<<std::setw(width)<<it->second;
fout<<"\n";
}
fout.close();
}
void print_hash_values(const char* filename, int width = 20)
{
std::ofstream fout(filename);
std::map<std::string,std::string>::iterator it;
int count = 0;
for ( it = xml_values_.begin(); it != xml_values_.end(); it++ )
{
count++;
fout<<std::setw(width)<<count;
fout<<std::setw(width)<<it->first;
fout<<std::setw(width)<<it->second;
fout<<"\n";
}
fout.close();
}
void print_hash_double(const char* filename, int width = 20)
{
std::ofstream fout(filename);
std::map<std::string,std::string>::iterator it;
int count = 0;
for ( it = xml_double_sequence_.begin(); it != xml_double_sequence_.end(); it++ )
{
count++;
fout<<std::setw(width)<<count;
fout<<std::setw(width)<<it->first;
fout<<std::setw(width)<<it->second;
fout<<"\n";
}
fout.close();
}
void print_extid(const char* filename, int width = 20)
{
std::ofstream fout(filename);
int count = 0;
for ( auto extid: channel_ext_ )
{
count++;
fout<<std::setw(width)<<count;
fout<<std::setw(width)<<extid;
fout<<"\n";
}
fout.close();
}
// provide number of channels and group
const int& num_channels()
{
return num_channels_;
}
const int& num_groups()
{
return num_groups_;
}
// get number of channels in specific group
const int& no_channels(int groupid)
{
assert( groupid >= 0 && groupid < num_groups_ );
return num_channels_group_[groupid];
}
const std::string& channel_name(int channelid)
{
assert( channelid >= 0 && channelid < num_channels_ );
return channel_name_[channelid];
}
// obtain overall channel id from combined group and group-specific channel id
int obtain_channel_id(int groupid, int channelid)
{
assert( groupid >= 0 && groupid < num_groups_ );
assert( channelid >= 0 && channelid < num_channels_group_[groupid] );
// find cummulative number of channels
int numsum = 0;
for ( int i = 0; i < groupid; i++ )
{
numsum += num_channels_group_[i];
}
assert( (numsum + channelid) > 0 && (numsum + channelid) <= num_channels_ );
return numsum+channelid;
}
const std::string& channel_name(int groupid, int channelid)
{
return channel_name_[obtain_channel_id(groupid,channelid)];
}
const std::string& group_name(int groupid)
{
assert( groupid >= 0 && groupid < num_groups_ );
return group_name_[groupid];
}
const std::string& channel_unit(int groupid, int channelid)
{
return units_[obtain_channel_id(groupid,channelid)];
}
int channel_exists(int groupid, std::string channel_name)
{
assert( groupid >= 0 && groupid < num_groups_ );
int channelid = -1;
for ( int i = 0; i < num_channels_group_[groupid]; i++)
{
if ( comparestrings(channel_name_[obtain_channel_id(groupid,i)],channel_name) )
{
channelid = i;
}
}
return channelid;
}
bool comparestrings(std::string s1, std::string s2, bool case_sensitive = false)
{
if ( case_sensitive )
{
return ( s1.compare(s2) == 0 );
}
else
{
std::transform( s1.begin(), s1.end(), s1.begin(), ::tolower);
std::transform( s2.begin(), s2.end(), s2.begin(), ::tolower);
return ( s1.compare(s2) == 0 );
}
}
// get time-stamp of channel-group in .tdm file given in unix format
static std::string unix_timestamp(std::string unixts)
{
// average year of Gregorian calender
const double avgdaysofyear = 365.0 + 1./4 - 1./100 + 1./400
- 8./24561; // gauge timestamp according to DIADEM result
// convert string to long int = number of seconds since 0000/01/01 00:00
long int ts = atol(unixts.c_str());
assert( ts >= 0 );
// use STL to convert timestamp (epoch usually starts on 01.01.1970)
std::time_t tstime = ts - 1970*avgdaysofyear*86400;
// get rid of linebreak character and return the result
return strtok(std::ctime(&tstime),"\n");
}
std::string time_stamp(int groupid, bool startstop = true)
{
assert( groupid >= 0 && groupid < num_groups_ );
return startstop ? unix_timestamp(group_timestamp_[groupid].first)
: unix_timestamp(group_timestamp_[groupid].second);
}
void list_datatypes();
// convert array of chars to single integer or floating point double
int convert_int(std::vector<unsigned char> bych);
double convert_double(std::vector<unsigned char> bych);
// disassemble single integer or double into array of chars
std::vector<unsigned char> convert_int(int number);
std::vector<unsigned char> convert_double(double number);
// convert entire channel, i.e. expert of .tdx binary file
// std::vector<double> convert_channel(int byteoffset, int length, int typesize);
std::vector<double> convert_channel(int channelid);
// obtain channel from overall channel id...
std::vector<double> get_channel(int channelid);
// ...or from group id and group-specific channel id
std::vector<double> channel(int groupid, int channelid)
{
return get_channel(obtain_channel_id(groupid,channelid));
}
int channel_length(int groupid, int channelid)
{
return length_[channel_ext_[obtain_channel_id(groupid,channelid)]];
}
double get_min(int groupid, int channelid)
{
return minmax_[obtain_channel_id(groupid,channelid)].first;
}
double get_max(int groupid, int channelid)
{
return minmax_[obtain_channel_id(groupid,channelid)].second;
}
void print_channel(int channelid, const char* filename, int width = 15);
// obtain any meta information about .tdm-file if available
std::string get_meta(std::string attribute_name)
{
// check if key "attribute_name" actually exits
std::map<std::string,std::string>::iterator positer = meta_info_.find(attribute_name);
bool ispresent = ( positer == meta_info_.end() ) ? false : true;
return ispresent ? meta_info_[attribute_name] : "key does not exist";
}
// prepare meta information file including all available meta-data
void print_meta(const char* filename, std::string sep = ",")
{
// open file
std::ofstream fout(filename);
for ( const auto& it : root_info_ )
{
fout<<it.first<<sep<<it.second<<"\n";
}
fout<<sep<<"\n";
for ( const auto& it : meta_info_ )
{
fout<<it.first<<sep<<it.second<<"\n";
}
// close down file
fout.close();
}
// TODO add elements/methods to build .tdm and write .tdx files for your own data
// by constructing xml document tree and write data to binary .tdx
// void set_channels(std::vector<std::string> channels);
// void set_groups(std::vector<std::string> groups);
// void set_assigment(std::vector<int> assignment);
// void set_channel(int i, std::vector<double> data);
};
#endif

File diff suppressed because it is too large Load Diff

View File

@ -1,290 +0,0 @@
// ------------------------------------------------------------------------- //
#ifndef TDM_TERMITE
#define TDM_TERMITE
#include <iostream>
#include <fstream>
#include <iterator>
#include <vector>
#include <iomanip>
#include <stdlib.h>
#include <assert.h>
#include <map>
#include <numeric>
#include <algorithm>
#include <chrono>
#include <sstream>
#include <filesystem>
#include <regex>
#include "pugixml.hpp"
#include "tdm_datamodel.hpp"
#include "tdm_datatype.hpp"
// -------------------------------------------------------------------------- //
class tdm_termite
{
// .tdm and .tdx paths/filenames
std::string tdmfile_;
std::string tdxfile_;
// set of .csv files (encoding mode)
std::vector<std::string> csvfile_;
// endianness (true = little, false = big)
bool endianness_, machine_endianness_;
// tdm meta-data
tdm_meta meta_data_;
// resconstruct "tdm_datatype.hpp: tdm_datatypes" as map to quickly map
// "valueType"/"channel_datatype" to full datatype
std::map<std::string,tdm_datatype> tdmdt_name_, tdmdt_chan_;
// blocks of data in .tdx file
std::map<std::string,block> tdx_blocks_;
// tdm root
tdm_root tdmroot_;
// tdm channelgroups
std::map<std::string,tdm_channelgroup> tdmchannelgroups_;
// tdm channels
std::map<std::string,tdm_channel> tdmchannels_;
std::map<std::string,std::vector<tdm_datatype>> tdmchannels_data_;
// submatrices and local_columns
std::map<std::string,submatrix> submatrices_;
std::map<std::string,localcolumn> localcolumns_;
// binary data container/file stream
std::vector<unsigned char> tdxbuffer_;
std::ifstream *tdx_ifstream_;
// find machine's endianness at runtime
// detect machine endianness (C++20 !!)
// if ( std::endian::native == std::endian::little )
// {
// machine_endianness_ = true;
// }
// else if ( std::endian::native == std::endian::big )
// {
// machine_endianness_ = false;
// }
// else
// {
// throw std::runtime_error("mixed endianness architecture is not supported");
// }
bool detect_endianness()
{
// int num = 1;
// machine_endianness_ = ( *(char*)&num == 1 );
std::uint32_t num = 0x11223344;
uint8_t* dfc = reinterpret_cast<uint8_t*>(&num);
return ( dfc[0] == 0x44 );
}
// extract list of identifiers from e.g. "#xpointer(id("usi12") id("usi13"))"
std::vector<std::string> extract_ids(std::string idstring)
{
// collect group identifiers by means of regex pattern "usi[0-9]+"
std::regex regid("(usi[0-9]+)");
// declare match instance and regex iterator (to find ALL matches)
std::smatch usi_match;
std::sregex_iterator pos(idstring.begin(), idstring.end(), regid);
std::sregex_iterator end;
// iterate through all matches
std::vector<std::string> listofids;
for ( ; pos != end; ++pos) listofids.push_back(pos->str());
return listofids;
}
// split string into substrings by delimiting string
std::vector<std::string> split(std::string fullstring, std::string delstr)
{
// declare array of resulting strings
std::vector<std::string> splitstrings(0);
// parse input string for substring
while ( fullstring.find(delstr) != std::string::npos )
{
// find first occurence of delimiting string in 'mystring'
std::size_t delpos = fullstring.find(delstr);
// extract substring
std::string stringel = fullstring.substr(0,delpos);
// append first word to array
if ( !stringel.empty() )
{
splitstrings.push_back(stringel);
}
// remove first word from 'fullstring'
fullstring = fullstring.substr(delpos+delstr.size(),fullstring.size());
}
// append last word to array
splitstrings.push_back(fullstring);
return splitstrings;
}
public:
// encoding
// tdm_termite(std::vector<std::string> csvfile);
// decoding
tdm_termite();
tdm_termite(std::string tdmfile, std::string tdxfile = std::string(""),
bool showlog = false);
~tdm_termite();
tdm_termite(const tdm_termite& other);
tdm_termite& operator=(const tdm_termite& other);
// provide (tdm,tdx) files
void submit_files(std::string tdmfile, std::string tdxfile = std::string(""),
bool showlog = false);
// process TDM data model in tdm file
void process_tdm(bool showlog);
// process <usi:include> element
void process_include(bool showlog, pugi::xml_document& xml_doc);
// extract tdm_root
void process_root(bool showlog, pugi::xml_document& xml_doc);
// process/list all channels and groups
void process_channelgroups(bool showlog, pugi::xml_document& xml_doc);
void process_channels(bool showlog, pugi::xml_document& xml_doc);
// process submatrices and localcolumns
void process_submatrices(bool showlog, pugi::xml_document& xml_doc);
void process_localcolumns(bool showlog, pugi::xml_document& xml_doc);
// get meta-data
tdm_meta get_meta()
{
return meta_data_;
}
// get root element
tdm_root get_root()
{
return tdmroot_;
}
// get channel/channelgroup meta object
tdm_channel& channel(std::string channelid)
{
if ( tdmchannels_.count(channelid) == 1 ) {
return tdmchannels_.at(channelid);
} else {
throw std::runtime_error(std::string("channel does not exist: ") + channelid);
}
}
tdm_channelgroup& channelgroup(std::string groupid)
{
if ( tdmchannelgroups_.count(groupid) == 1 ) {
return tdmchannelgroups_.at(groupid);
} else {
throw std::runtime_error(std::string("channelgroup does not exist: ") + groupid);
}
}
// get full channel(group) overview
std::string get_channel_overview(format chformatter);
// get block/submatrix/localcolumn overview
template<typename tdmelement>
std::string get_overview(format formatter);
private:
void summarize_member(tdm_channelgroup &chp, std::string& summary, format& formatter);
void summarize_member(submatrix &sbm, std::string& summary, format& formatter);
void summarize_member(localcolumn &lcc, std::string& summary, format& formatter);
void summarize_member(block &blk, std::string& summary, format& formatter);
public:
// get list of channelgroup ids
std::vector<std::string> get_channelgroup_ids()
{
std::vector<std::string> channelgroup_ids;
for (std::map<std::string,tdm_channelgroup>::iterator it=tdmchannelgroups_.begin();
it!=tdmchannelgroups_.end(); ++it) channelgroup_ids.push_back(it->first);
return channelgroup_ids;
}
// get list of channel ids
std::vector<std::string> get_channel_ids()
{
std::vector<std::string> channel_ids;
for (std::map<std::string,tdm_channel>::iterator it=tdmchannels_.begin();
it!=tdmchannels_.end(); ++it) channel_ids.push_back(it->first);
return channel_ids;
}
// extract channel by id
std::vector<tdmdatatype> get_channel(std::string& id);
// additional methods for Cython/Python compatibiliy
//
// extract and return any channel as datatype double
std::vector<double> get_channel_as_double(std::string id)
{
std::vector<tdmdatatype> tdmchn = this->get_channel(id);
std::vector<double> chn;
for ( tdmdatatype el: tdmchn ) chn.push_back(el.as_double());
return chn;
}
// get channel(-group) meta-data
std::string get_channelgroup_info(std::string id)
{
if ( tdmchannelgroups_.count(id) == 1 ) {
return tdmchannelgroups_.at(id).get_json();
} else {
throw std::runtime_error(std::string("channelgroup does not exist: ") + id);
}
}
std::string get_channel_info(std::string id)
{
if ( tdmchannels_.count(id) == 1 ) {
return tdmchannels_.at(id).get_json();
} else {
throw std::runtime_error(std::string("channel does not exist: ") + id);
}
}
// dump a single channel/entire group (identified by id) to file
void print_channel(std::string &id, const char* filename, bool include_meta = true);
void print_group(std::string &id, const char* filename, bool include_meta = true,
char sep = ' ', std::string column_header = std::string(""));
void check_filename_path(const char* filename);
// check machine's datatypes
// https://en.cppreference.com/w/cpp/language/types
void check_local_datatypes();
private:
template<typename datatype>
void convert_data_to_type(std::vector<unsigned char> &buffer,
std::vector<tdmdatatype> &channel);
// check consistency of mapped datatypes between C++ and TDM datatypes
void check_datatype_consistency();
};
#endif
// -------------------------------------------------------------------------- //

70
main.cpp Normal file
View File

@ -0,0 +1,70 @@
#include "lib/tdm_ripper.hpp"
int main(int argc, char* argv[])
{
// path of filename provided ?
assert( argc > 1 && "please provide a filename and path" );
std::cout<<"number of CLI-arguments: "<<argc<<"\n";
for ( int i = 0; i < argc; i++ ) std::cout<<std::setw(5)<<i<<": "<<argv[i]<<"\n";
// declare and initialize tdm_ripper
assert( argc == 3 );
tdm_ripper ripper(argv[1],argv[2],false); //,"samples/SineData.tdx",false);
// ripper.list_datatypes();
// ripper.show_structure();
ripper.print_hash_local("data/hash_table_xml_local.dat");
ripper.print_hash_values("data/hash_table_xml_value.dat");
ripper.print_hash_double("data/hash_table_xml_double.dat");
ripper.print_extid("data/channel_ext_id.dat");
ripper.list_groups();
std::ofstream gout("data/list_of_groups.dat");
ripper.list_groups(gout);
gout.close();
ripper.list_channels();
std::ofstream fout("data/list_of_channels.dat");
ripper.list_channels(fout);
fout.close();
// long int nsa = 6.3636349745e10; // expected result: 22.07.2016, 19:49:05
// long int nsb = 6.3636350456e10;
// std::string ts = std::to_string(nsa);
// std::cout<<ripper.unix_timestamp(ts)<<"\n";
std::cout<<"number of channels "<<ripper.num_channels()<<"\n";
std::cout<<"number of groups "<<ripper.num_groups()<<"\n\n";
// obtain some specific meta information tags
std::cout<<"\n"<<ripper.get_meta("SMP_Name")<<"\n";
std::cout<<ripper.get_meta("SMP_Aufbau_Nr")<<"\n";
std::cout<<ripper.get_meta("SMP_Type")<<"\n";
std::cout<<ripper.get_meta("Location")<<"\n\n";
// print all meta information
ripper.print_meta("data/meta_info.csv");
// for ( int i = 0; i < ripper.num_groups(); i++ )
// {
// std::cout<<std::setw(10)<<i+1<<std::setw(10)<<ripper.no_channels(i+1)<<"\n";
// for ( int j = 0; j < ripper.no_channels(i+1); j++ )
// {
// std::cout<<std::setw(10)<<i+1<<std::setw(10)<<j+1<<std::setw(30)<<ripper.channel_name(i+1,j+1)<<"\n";
// }
// }
// for ( int i = 3; i < 10; i++ )
for ( int i = 0; i < ripper.num_channels(); i++ )
// for ( int i = 11880; i < ripper.num_channels(); i++ )
{
ripper.print_channel(i,("data/channel_"+std::to_string(i+1)+"_"
+ripper.channel_name(i)+".dat").c_str());
}
return 0;
}

157
makefile
View File

@ -1,136 +1,47 @@
# --------------------------------------------------------------------------- #
SHELL := /bin/bash
# declare name of executable EXE = tdm_parser
EXE = tdmtermite CC = g++ -std=c++11
# -stdlib=libc++
CPPFLAGS = -O3 -Wall -Werror -Wunused-variable -Wsign-compare
LIB = pugixml/
# sources and headers $(EXE) : main.o tdm_ripper.o
SRC := tdm_termite $(CC) $(CPPFLAGS) $^ -o $@
HPP = $(wildcard lib/*.hpp)
# compiler and C++ standard main.o : main.cpp
CC = g++ -std=c++17 $(CC) -c $(CPPFLAGS) -I $(LIB) $< -o $@
# compiler options and optimization flags tdm_ripper.o : lib/tdm_ripper.cpp lib/tdm_ripper.hpp
OPT = -O3 -Wall -Wconversion -Wpedantic -Wunused-variable -Wsign-compare $(CC) -c $(CPPFLAGS) -I $(LIB) $< -o $@
# include 3rd party libraries paths extall : extract_all.o tdm_ripper.o
LIBB := 3rdparty/ $(CC) $(CPPFLAGS) $^ -o extract_all
LIB := $(foreach dir,$(shell ls $(LIBB)),-I $(LIBB)$(dir))
# determine git version/commit tag extract_all.o : extract_all.cpp
GTAG := $(shell git tag -l --sort=version:refname | tail -n1 | sed "s/$^v//g") $(CC) -c $(CPPFLAGS) -I $(LIB) $< -o $@
GHSH := $(shell git rev-parse HEAD | head -c8)
GVSN := $(shell cat python/VERSION | tr -d ' \n')
# current timestamp clean :
TMS = $(shell date +%Y%m%dT%H%M%S) rm -f $(EXE) *.o
rm -f *.dat
rm -f extract_all
rm -f data/*.dat
rm -f data/*.csv
# define install location pylib : setup.py pytdm_ripper.pyx tdm_ripper.pxd tdm_ripper.o
INST := /usr/local/bin python3 setup.py build_ext --inplace
# platform and current working directory install : setup.py pytdm_ripper.pyx tdm_ripper.pxd lib/tdm_ripper.cpp lib/tdm_ripper.hpp
OST := $(shell uname) python3 setup.py install
CWD := $(shell pwd)
# --------------------------------------------------------------------------- # install_osx : setup_osx.py pytdm_ripper.pyx tdm_ripper.pxd lib/tdm_ripper.cpp lib/tdm_ripper.hpp
# C++ and CLI tool python3 setup_osx.py install
# check tags and build executable lib/libtdmripper.a :
exec: check-tags $(GVSN) $(EXE) make -C lib libtdmripper.a
# build executable clean-lib :
$(EXE): $(SRC).o main.o rm -f lib/*.o lib/*.a
$(CC) $(OPT) $^ -o $@ rm -f -r build
rm -f pytdm_ripper.cpp
$(SRC).o : lib/$(SRC).cpp lib/$(SRC).hpp $(HPP) rm -f *.so
$(CC) -c $(OPT) $(LIB) $< -o $@
# build main.cpp object file and include git version/commit tag
main.o: src/main.cpp lib/$(SRC).hpp $(HPP)
@cp $< $<.cpp
@if [ $(OST) = "Linux" ]; then\
sed -i 's/TAGSTRING/$(GTAG)/g' $<.cpp; \
sed -i 's/HASHSTRING/$(GHSH)/g' $<.cpp; \
sed -i 's/TIMESTAMPSTRING/$(TMS)/g' $<.cpp; \
fi
@if [ $(OST) = "Darwin" ]; then\
sed -i '' 's/TAGSTRING/$(GTAG)/g' $<.cpp; \
sed -i '' 's/HASHSTRING/$(GHSH)/g' $<.cpp; \
sed -i '' 's/TIMESTAMPSTRING/$(TMS)/g' $<.cpp; \
fi
$(CC) -c $(OPT) $(LIB) -I lib/ $<.cpp -o $@
@rm $<.cpp
install: $(EXE)
cp $< $(INST)/
uninstall : $(INST)/$(EXE)
rm $<
tdmtest : tdmtest.o
$(CC) $(OPT) $^ -o $@
tdmtest.o : src/test.cpp lib/$(SRC).hpp $(HPP)
$(CC) -c $(OPT) $(LIB) -I lib/ $< -o $@
cpp-clean :
rm -vf $(EXE)
rm -vf *.o src/main.cpp.cpp tdmtest
#-----------------------------------------------------------------------------#
# versions
$(GTAG):
@echo "consistent versions check successful: building $(GTAG)"
check-tags:
@echo "latest git tag: $(GTAG)"
@echo "latest git hash: $(GHSH)"
@echo "python version: $(GVSN)"
# --------------------------------------------------------------------------- #
# docker
docker-build:
docker build . --tag tdmtermite:latest
docker-run:
mkdir -pv data/{input,output}
docker run -it --rm --volume $(CWD)/data:/home/data tdmtermite:latest /bin/bash
docker-clean:
rm -rv data
docker image remove tdmtermite
# --------------------------------------------------------------------------- #
# check process
checkps :
@ps aux | head -n1
@ps aux | grep $(EXE) | grep -v "grep"
# --------------------------------------------------------------------------- #
# python
python-build: check-tags $(GVSN)
make -C python/ build-inplace
cp python/TDMtermite*.so ./ -v
python-install: check-tags $(GVSN)
make -C python/ install
python-clean:
make -C python/ clean
rm -vf TDMtermite*.so
python-test:
PYTHONPATH=./ python python/examples/usage.py
# --------------------------------------------------------------------------- #
# clean
clean : cpp-clean python-clean
# --------------------------------------------------------------------------- #

142
pytdm_ripper.pyx Normal file
View File

@ -0,0 +1,142 @@
from tdm_ripper cimport tdm_ripper
import numpy as np
import re
cdef class pytdmripper:
# pointer to C++ instance (since there's no nullary constructor)
cdef tdm_ripper *cripp
def __cinit__(self, string tdmfile, string tdxfile = b""):
self.cripp = new tdm_ripper(tdmfile,tdxfile)
def __dealloc__(self):
del self.cripp
def show_channels(self):
self.cripp.list_channels()
def show_groups(self):
self.cripp.list_groups()
def num_channels(self):
return self.cripp.num_channels()
def no_channels(self, int groupid):
assert (groupid >= 0 and groupid < self.cripp.num_groups()), "index of group must be in [0,n-1]"
return self.cripp.no_channels(groupid)
def num_groups(self):
return self.cripp.num_groups()
def no_channel_groups(self):
return self.cripp.num_groups()
def channel_name(self,int groupid,int channelid):
return self.cripp.channel_name(groupid,channelid).decode('utf-8')
def group_name(self,int groupid):
return self.cripp.group_name(groupid).decode('utf-8')
def channel_unit(self,int groupid,int channelid):
return (self.cripp.channel_unit(groupid,channelid))
def channel_exists(self,int groupid, string channelname):
return self.cripp.channel_exists(groupid,channelname)
def obtain_channel_id(self,int groupid, int channelid):
return self.cripp.obtain_channel_id(groupid,channelid)
def get_channel(self, int channelid):
return np.asarray(self.cripp.get_channel(channelid))
def channel(self,int groupid,int channelid):
return self.cripp.channel(groupid,channelid)
def channel_length(self,int groupid, int channelid):
return self.cripp.channel_length(groupid,channelid)
def time_stamp(self,int groupid, bool startstop):
return self.cripp.time_stamp(groupid,startstop)
def get_min(self,int groupid, int channelid):
return self.cripp.get_min(groupid,channelid)
def get_max(self,int groupid, int channelid):
return self.cripp.get_max(groupid,channelid)
def print_channel(self, int channelid, const char* filename):
self.cripp.print_channel(channelid,filename)
def meta_info(self, string attribute_name):
return self.cripp.get_meta(attribute_name)
def print_meta(self, const char* filename):
self.cripp.print_meta(filename)
def close(self):
dummy = ""
#=============================================================================#
def extract_all(string tdmfile, string tdxfile, string outdirx = b"./", string prfxnam = b""):
"""
Python function extracting all available data from .tdm and .tdx to .csv dump
Args:
tdmfile: path and filename of .tdm file
tdxfile: path and filename of associated .tdx file
outdirx: directory where all .csv output is dumped
prfxnam: optionally specify a name prefix for all .csv files
Return:
None
"""
# TODO preliminary: assume utf-8
encoding = 'utf-8'
# set up instance of Cython ripper
td = pytdmripper(tdmfile,tdxfile)
# if no name prefix is given, .tdm filename will be used
prfx = ""
if prfxnam.decode(encoding) == prfx :
prfx = tdmfile.decode(encoding).rstrip('.tdm').split('/')[-1]
else:
prfx = prfxnam.decode(encoding)
# obtain number of available groups and channels
numgr = td.num_groups()
numch = td.num_channels()
# generate list of all files produced
filelist = []
# dump all meta information
metafile = outdirx.decode(encoding)+prfx+'.csv'
td.print_meta(metafile.encode(encoding))
filelist.append(metafile.lstrip("./"))
# dump all available groups and channels
for g in range(0,numgr):
numgrch = td.no_channels(g)
for c in range(0,numgrch):
# obtained overall channel id
chid = td.obtain_channel_id(g,c)
# get group's and channel's name
gname = td.group_name(g)
cname = td.channel_name(g,c)
# use regular expression replacement to sanitize group and channel names
gname = re.sub('[!@#$%^&*()-+= ,]','',gname)
cname = re.sub('[!@#$%^&*()-+= ,]','',cname)
# generate filename
fichan = prfx + '_' + str(g+1) + '_' + str(c+1) + '_' + gname + '_' + cname + '.csv'
# print channel
td.print_channel(chid,(outdirx.decode(encoding)+fichan).encode(encoding))
# append filename to list
filelist.append(fichan)
# return list of all files
return filelist

View File

@ -1,4 +0,0 @@
include lib/*.hpp
include *.cpp
include *.pyx
include *.pxd

View File

@ -1 +0,0 @@
2.1.3

View File

@ -1,50 +0,0 @@
import tdmtermite
import json
import re
# create 'tdm_termite' instance object
try :
jack = tdmtermite.tdmtermite(b'samples/SineData.tdm',b'samples/SineData.tdx')
except RuntimeError as e :
print("failed to load/decode TDM files: " + str(e))
# list ids of channelgroups
grpids = jack.get_channelgroup_ids()
# iterate through group-ids
for grp in grpids[0:2] :
# obtain meta data of channelgroups
grpinfo = jack.get_channelgroup_info(grp)
print( json.dumps(grpinfo,sort_keys=False,indent=4) )
# get channel meta-data and compose header
column_header = ""
chns = grpinfo['channels'].split(' ')
for chn in chns :
# obtain channel's meta-data as dictionary (JSON)
chninfo = jack.get_channel_info(chn.encode())
# print( json.dumps(chninfo,sort_keys=False,indent=4) )
# choose e.g. channel-id and name to compose column header
chnhead = ( str(chninfo['channel-id']) + "("
+ str(chninfo['name']).replace(',',' ') + ")" )
# append to full header
column_header = column_header + chnhead + ","
# remove last comma
column_header = column_header[0:-1]
print("column_header:\n"+column_header+"\n")
# write channelgroup to file
try :
grpname = re.sub('[^A-Za-z0-9]','',grpinfo['name'])
grpfile = str("./channelgroup_") + str(grp.decode()) + "_" + str(grpname) + str(".csv")
jack.print_channelgroup(grp, # id of group to be written
grpfile.encode(), # filename
False, # no meta-data header
ord(','), # csv separator character
column_header.encode() # provide column header
)
except RuntimeError as e :
print("failed to print channelgroup: " + str(grp) + " : " + str(e))

View File

@ -1,20 +0,0 @@
import tdmtermite
# create 'tdm_termite' instance object
try :
jack = tdmtermite.tdmtermite(b'samples/SineData.tdm',b'samples/SineData.tdx')
# list ids of channelgroups
grpids = jack.get_channelgroup_ids()
# iterate through groups
for grp in grpids :
# write channelgroup to file (prepend './' for local directory!!)
grpfile = "./channelgroup_" + str(grp.decode()) + ".csv"
jack.print_channelgroup(grp, # id of group to be written
grpfile.encode(), # filename
False, # include meta-data fileheader?
ord(','), # csv separator character
b"" # (empty=default) column header
)
except RuntimeError as e :
print("failed to load/decode/write TDM files: " + str(e))

View File

@ -1,56 +0,0 @@
import tdmtermite
# import numpy as np
import json
import re
# create 'tdm_termite' instance object
try :
jack = tdmtermite.tdmtermite(b'samples/SineData.tdm',b'samples/SineData.tdx')
except RuntimeError as e :
print("failed to load/decode TDM files: " + str(e))
# list ids of channelgroups
grpids = jack.get_channelgroup_ids()
grpids = [x.decode() for x in grpids]
print("list of channelgroups: ",grpids)
for grp in grpids[0:2] :
# obtain meta data of channelgroups
grpinfo = jack.get_channelgroup_info(grp.encode())
print( json.dumps(grpinfo,sort_keys=False,indent=4) )
# write channelgroup to file
try :
grpname = re.sub('[^A-Za-z0-9]','',grpinfo['name'])
grpfile = "./channelgroup_" + str(grp) + "_" + str(grpname) + ".csv"
jack.print_channelgroup(grp.encode(),grpfile.encode(),True,ord(' '),b'')
except RuntimeError as e :
print("failed to print channelgroup: " + str(grp) + " : " + str(e))
# list ids of channels
chnids = jack.get_channel_ids()
chnids = [x.decode() for x in chnids]
print("list of channels: ",chnids)
for chn in chnids[0:2] :
# obtain meta-data
chninfo = jack.get_channel_info(chn.encode())
print( json.dumps(chninfo,sort_keys=False,indent=4) )
# channel data
try :
chndata = jack.get_channel(chn.encode())
except RuntimeError as e :
print("failed to extract channel: " + str(chn) + " : " + str(e))
print(str(chndata[0:6]) + " ...")
# write channel to file
chnfile = "./channel_" + str(chn) + ".csv"
try :
jack.print_channel(chn.encode(),chnfile.encode(),True)
except RuntimeError as e :
print("failed to print channel: " + str(chn) + " : " + str(e))

View File

@ -1,60 +0,0 @@
setup:
cat ../README.md | grep '^# TDMtermite' -A 50000 > ./README.md
#pandoc -f markdown -t rst -o README.rst README.md
#python -m rstvalidator README.rst
cp -r ../lib ./
cp -r ../3rdparty ./
cp -v ../LICENSE ./
setup-clean:
rm -vf README.md README.rst LICENSE
rm -rf lib/ 3rdparty/
build: setup
python setup.py build
build-inplace: setup
python setup.py build_ext --inplace
build-install: setup
python setup.py install
build-sdist: setup
python setup.py sdist
python -m twine check dist/*
build-bdist: setup
python setup.py bdist
python -m twine check dist/*
build-clean:
python setup.py clean --all
rm -vf tdmtermite*.so tdmtermite*.cpp
rm -rvf dist/ tdmtermite.egg-info/
cibuildwheel-build: setup
cibuildwheel --platform linux
cibuildwheel-clean:
rm -rvf wheelhouse/
pypi-audit:
auditwheel repair $(shell find dist/ -name "*-linux_x86_64.whl")
# username: __token__
# password: API-token including "pypi-"
pypi-upload-test:
python -m twine upload --repository testpypi dist/$(shell ls -t dist/ | head -n1)
pypi-install-test:
python -m pip install --index-url https://test.pypi.org/simple --no-deps TDMtermite-RecordEvolution
# python3 -m pip install -i https://test.pypi.org/simple/ TDMtermite-RecordEvolution==0.5
pypi-upload:
python -m twine upload dist/$(shell ls -t dist/ | head -n1)
clean: setup build-clean cibuildwheel-clean setup-clean
run-example:
PYTHONPATH=$(pwd) python examples/usage.py

View File

@ -1,6 +0,0 @@
[build-system]
requires = ["setuptools", "wheel","Cython"]
build-backend = "setuptools.build_meta"
[tool.cibuildwheel]
before-all = ""

View File

@ -1,23 +0,0 @@
[metadata]
name = tdmtermite
description = Extract and read data from National Instruments LabVIEW tdx/tdm files and export them as csv files
long_description = file: README.md
# long_description_content_type = text/x-rst
long_description_content_type = text/markdown
version = file: VERSION
author = Record Evolution GmbH
author_email = mario.fink@record-evolution.de
maintainer = Record Evolution GmbH
url= https://github.com/RecordEvolution/TDMtermite.git
license = MIT License
license_files = LICENSE
keywords = TDM, TDX, National Instruments, DIAdem, LabVIEW, Measurement Studio, SignalExpress
classifiers =
Programming Language :: Python :: 3
License :: OSI Approved :: MIT License
Operating System :: OS Independent
Topic :: Scientific/Engineering
Topic :: Software Development :: Libraries :: Python Modules
[options]

View File

@ -1,23 +0,0 @@
from setuptools import Extension, setup
from Cython.Build import cythonize
import sys
print("building on platform: "+sys.platform)
cmpArgs = {
"linux": ['-std=c++17','-Wno-unused-variable'],
"darwin": ['-std=c++17','-Wno-unused-variable'],
"win32": ['/EHsc','/std:c++17']
}
extension = Extension(
"tdmtermite",
language='c++',
sources=["tdmtermite.pyx"],
include_dirs=["lib","3rdparty/pugixml"],
extra_compile_args=cmpArgs[sys.platform]
)
setup(
ext_modules=cythonize(extension,language_level=3)
)

View File

@ -1,35 +0,0 @@
# use some C++ STL libraries
from libcpp.string cimport string
from libcpp.vector cimport vector
from libcpp cimport bool
cdef extern from "lib/tdm_termite.cpp":
pass
cdef extern from "lib/tdm_termite.hpp":
cdef cppclass cpptdmtermite "tdm_termite":
# constructor(s)
cpptdmtermite() except +
cpptdmtermite(string tdmfile, string tdxfile) except +
# provide TDM files
void submit_files(string tdmfile, string tdxfile) except+
# get list of channel(-group) ids
vector[string] get_channelgroup_ids() except+
vector[string] get_channel_ids() except+
# get data of specific channel
vector[double] get_channel_as_double(string id) except+
# get meta-data
string get_channelgroup_info(string id) except+
string get_channel_info(string id) except+
# print a channel(-group)
void print_group(string id, const char* filename, bool include_meta,
char delimiter, string column_header) except+
void print_channel(string id, const char* filename, bool include_meta) except+

View File

@ -1,52 +0,0 @@
# distutils: language = c++
# cython: language_level = 3
from tdmtermite cimport cpptdmtermite
import json as jn
cdef class tdmtermite:
# C++ instance of class => stack allocated (requires nullary constructor!)
cdef cpptdmtermite cpptdm
# constructor
def __cinit__(self, string tdmfile, string tdxfile):
self.cpptdm = cpptdmtermite(tdmfile,tdxfile)
# provide TDM files
def submit_files(self,string tdmfile, string tdxfile):
self.cpptdm.submit_files(tdmfile,tdxfile)
# get list of channel(-group) ids
def get_channelgroup_ids(self):
return self.cpptdm.get_channelgroup_ids()
def get_channel_ids(self):
return self.cpptdm.get_channel_ids()
# get data of specific channel
def get_channel(self, string id):
return self.cpptdm.get_channel_as_double(id)
# get meta-data of channel(-group) (as dictionary)
def get_channelgroup_info(self, string id):
grpstr = self.cpptdm.get_channelgroup_info(id)
return jn.loads(grpstr.decode())
def get_channel_info(self, string id):
chnstr = self.cpptdm.get_channel_info(id)
return jn.loads(chnstr.decode())
# print a channel(-group)
def print_channelgroup(self, string id, const char* filename, bool include_meta,
char delimiter, string column_header):
self.cpptdm.print_group(id,filename,include_meta,delimiter,column_header)
def print_channel(self, string id, const char* filename,
bool include_meta):
self.cpptdm.print_channel(id,filename,include_meta)
# print all data grouped by channelgroups
def write_all(self, string outputdir) :
grpids = self.cpptdm.get_channelgroup_ids()
for id in grpids :
grpfile = outputdir.decode() + "/channelgroup_" + id.decode() + ".csv"
self.cpptdm.print_group(id,grpfile.encode(),True,ord(','),"".encode())

View File

@ -1,369 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
<usi:tdm xmlns:usi="http://www.ni.com/Schemas/USI/1_0" version="1.0">
<usi:documentation>
<usi:exporter>National Instruments USI</usi:exporter>
<usi:exporterVersion>1.5</usi:exporterVersion>
</usi:documentation>
<usi:model modelName="National Instruments USI generated meta file" modelVersion="1.0">
<usi:include nsUri="http://www.ni.com/DataModels/USI/TDM/1_0"/>
</usi:model>
<usi:include>
<file byteOrder="bigEndian" url="SineData-be.tdx">
<block byteOffset="0" id="inc0" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="8000" id="inc1" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="16000" id="inc2" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="24000" id="inc3" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="32000" id="inc4" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="40000" id="inc5" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="48000" id="inc6" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="56000" id="inc7" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="64000" id="inc8" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="72000" id="inc9" length="1000" valueType="eFloat64Usi"/>
</file>
</usi:include>
<usi:data>
<double_sequence id="usi1">
<values external="inc0"/>
</double_sequence>
<double_sequence id="usi2">
<values external="inc1"/>
</double_sequence>
<double_sequence id="usi3">
<values external="inc2"/>
</double_sequence>
<double_sequence id="usi4">
<values external="inc3"/>
</double_sequence>
<double_sequence id="usi5">
<values external="inc4"/>
</double_sequence>
<double_sequence id="usi6">
<values external="inc5"/>
</double_sequence>
<double_sequence id="usi7">
<values external="inc6"/>
</double_sequence>
<double_sequence id="usi8">
<values external="inc7"/>
</double_sequence>
<double_sequence id="usi9">
<values external="inc8"/>
</double_sequence>
<double_sequence id="usi10">
<values external="inc9"/>
</double_sequence>
<tdm_root id="usi11">
<name>SineData.TDM</name>
<description>Sine signals of various amplitudes and frequencies.</description>
<title>SineData</title>
<author>National Instruments</author>
<datetime>2008-05-06T17:20:12.65074539184570313</datetime>
<channelgroups>#xpointer(id("usi12") id("usi13"))</channelgroups>
</tdm_root>
<tdm_channelgroup id="usi12">
<name>Amplitudes</name>
<description>Sine Signals of various amplitudes.</description>
<root>#xpointer(id("usi11"))</root>
<instance_attributes>
<double_attribute name="Frequency">1</double_attribute>
</instance_attributes>
<channels>#xpointer(id("usi14") id("usi15") id("usi16") id("usi17") id("usi18"))</channels>
<submatrices>#xpointer(id("usi24") id("usi25") id("usi26") id("usi27") id("usi28"))</submatrices>
</tdm_channelgroup>
<tdm_channelgroup id="usi13">
<name>Frequencies</name>
<description>Sine signals of various frequencies.</description>
<root>#xpointer(id("usi11"))</root>
<instance_attributes>
<double_attribute name="Amplitude">1</double_attribute>
</instance_attributes>
<channels>#xpointer(id("usi19") id("usi20") id("usi21") id("usi22") id("usi23"))</channels>
<submatrices>#xpointer(id("usi29") id("usi30") id("usi31") id("usi32") id("usi33"))</submatrices>
</tdm_channelgroup>
<tdm_channel id="usi14">
<name>A = 1</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999997146387718</minimum>
<maximum>0.999999682931835</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">0</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi34"))</local_columns>
</tdm_channel>
<tdm_channel id="usi15">
<name>A = 2</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-1.99999429277544</minimum>
<maximum>1.99999936586367</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">1</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi35"))</local_columns>
</tdm_channel>
<tdm_channel id="usi16">
<name>A = 4</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-3.99998858555087</minimum>
<maximum>3.99999873172734</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">2</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi36"))</local_columns>
</tdm_channel>
<tdm_channel id="usi17">
<name>A = 8</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-7.99997717110174</minimum>
<maximum>7.99999746345468</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">3</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi37"))</local_columns>
</tdm_channel>
<tdm_channel id="usi18">
<name>A = 16</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-15.9999543422035</minimum>
<maximum>15.9999949269094</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">4</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi38"))</local_columns>
</tdm_channel>
<tdm_channel id="usi19">
<name>F = 1</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999997146387718</minimum>
<maximum>0.999999682931835</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">0</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi39"))</local_columns>
</tdm_channel>
<tdm_channel id="usi20">
<name>F = 2</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999999230697499</minimum>
<maximum>0.999995986891472</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">1</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi40"))</local_columns>
</tdm_channel>
<tdm_channel id="usi21">
<name>F = 4</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999999230697499</minimum>
<maximum>0.99994907791452</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">2</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi41"))</local_columns>
</tdm_channel>
<tdm_channel id="usi22">
<name>F = 8</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999999230697499</minimum>
<maximum>0.999996490345607</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">3</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi42"))</local_columns>
</tdm_channel>
<tdm_channel id="usi23">
<name>F = 16</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999999230697499</minimum>
<maximum>0.999993076284592</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">4</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi43"))</local_columns>
</tdm_channel>
<submatrix id="usi24">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi34"))</local_columns>
</submatrix>
<submatrix id="usi25">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi35"))</local_columns>
</submatrix>
<submatrix id="usi26">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi36"))</local_columns>
</submatrix>
<submatrix id="usi27">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi37"))</local_columns>
</submatrix>
<submatrix id="usi28">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi38"))</local_columns>
</submatrix>
<submatrix id="usi29">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi39"))</local_columns>
</submatrix>
<submatrix id="usi30">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi40"))</local_columns>
</submatrix>
<submatrix id="usi31">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi41"))</local_columns>
</submatrix>
<submatrix id="usi32">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi42"))</local_columns>
</submatrix>
<submatrix id="usi33">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi43"))</local_columns>
</submatrix>
<localcolumn id="usi34">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi14"))</measurement_quantity>
<submatrix>#xpointer(id("usi24"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi1"))</values>
</localcolumn>
<localcolumn id="usi35">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi15"))</measurement_quantity>
<submatrix>#xpointer(id("usi25"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi2"))</values>
</localcolumn>
<localcolumn id="usi36">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi16"))</measurement_quantity>
<submatrix>#xpointer(id("usi26"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi3"))</values>
</localcolumn>
<localcolumn id="usi37">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi17"))</measurement_quantity>
<submatrix>#xpointer(id("usi27"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi4"))</values>
</localcolumn>
<localcolumn id="usi38">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi18"))</measurement_quantity>
<submatrix>#xpointer(id("usi28"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi5"))</values>
</localcolumn>
<localcolumn id="usi39">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi19"))</measurement_quantity>
<submatrix>#xpointer(id("usi29"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi6"))</values>
</localcolumn>
<localcolumn id="usi40">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi20"))</measurement_quantity>
<submatrix>#xpointer(id("usi30"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi7"))</values>
</localcolumn>
<localcolumn id="usi41">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi21"))</measurement_quantity>
<submatrix>#xpointer(id("usi31"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi8"))</values>
</localcolumn>
<localcolumn id="usi42">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi22"))</measurement_quantity>
<submatrix>#xpointer(id("usi32"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi9"))</values>
</localcolumn>
<localcolumn id="usi43">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi23"))</measurement_quantity>
<submatrix>#xpointer(id("usi33"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi10"))</values>
</localcolumn>
</usi:data>
</usi:tdm>

Binary file not shown.

View File

@ -1,377 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
<usi:tdm xmlns:usi="http://www.ni.com/Schemas/USI/1_0" version="1.0">
<usi:documentation>
<usi:exporter>National Instruments USI</usi:exporter>
<usi:exporterVersion>1.5</usi:exporterVersion>
</usi:documentation>
<usi:model modelName="National Instruments USI generated meta file" modelVersion="1.0">
<usi:include nsUri="http://www.ni.com/DataModels/USI/TDM/1_0"/>
</usi:model>
<usi:include>
<file byteOrder="littleEndian" url="SineData.tdx">
<block byteOffset="0" id="inc0" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="8000" id="inc1" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="16000" id="inc2" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="24000" id="inc3" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="32000" id="inc4" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="40000" id="inc5" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="48000" id="inc6" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="56000" id="inc7" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="64000" id="inc8" length="1000" valueType="eFloat64Usi"/>
<block byteOffset="72000" id="inc9" length="1000" valueType="eFloat64Usi"/>
</file>
</usi:include>
<usi:data>
<double_sequence id="usi1">
<values external="inc0"/>
</double_sequence>
<double_sequence id="usi2">
<values external="inc1"/>
</double_sequence>
<double_sequence id="usi3">
<values external="inc2"/>
</double_sequence>
<double_sequence id="usi4">
<values external="inc3"/>
</double_sequence>
<double_sequence id="usi5">
<values external="inc4"/>
</double_sequence>
<double_sequence id="usi6">
<values external="inc5"/>
</double_sequence>
<double_sequence id="usi7">
<values external="inc6"/>
</double_sequence>
<double_sequence id="usi8">
<values external="inc7"/>
</double_sequence>
<double_sequence id="usi9">
<values external="inc8"/>
</double_sequence>
<double_sequence id="usi10">
<values external="inc9"/>
</double_sequence>
<tdm_root id="usi11">
<name>SineData.TDM</name>
<description>Sine signals of various amplitudes and frequencies.</description>
<title>SineData</title>
<author>National Instruments</author>
<datetime>2008-05-06T17:20:12.65074539184570313</datetime>
<channelgroups>#xpointer(id("usi12") id("usi13"))</channelgroups>
</tdm_root>
<tdm_channelgroup id="usi12">
<name>Amplitudes</name>
<description>Sine Signals of various amplitudes.</description>
<root>#xpointer(id("usi11"))</root>
<instance_attributes>
<double_attribute name="Frequency">1</double_attribute>
</instance_attributes>
<channels>#xpointer(id("usi14") id("usi15") id("usi16") id("usi17") id("usi18"))</channels>
<submatrices>#xpointer(id("usi24") id("usi25") id("usi26") id("usi27") id("usi28"))</submatrices>
</tdm_channelgroup>
<tdm_channelgroup id="usi13">
<name>Frequencies</name>
<description>Sine signals of various frequencies.</description>
<root>#xpointer(id("usi11"))</root>
<instance_attributes>
<double_attribute name="Amplitude">1</double_attribute>
</instance_attributes>
<channels>#xpointer(id("usi19") id("usi20") id("usi21") id("usi22") id("usi23"))</channels>
<submatrices>#xpointer(id("usi29") id("usi30") id("usi31") id("usi32") id("usi33"))</submatrices>
</tdm_channelgroup>
<tdm_channel id="usi14">
<name>A = 1</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999997146387718</minimum>
<maximum>0.999999682931835</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">0</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi34"))</local_columns>
</tdm_channel>
<tdm_channel id="usi15">
<name>A = 2</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-1.99999429277544</minimum>
<maximum>1.99999936586367</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">1</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi35"))</local_columns>
</tdm_channel>
<tdm_channel id="usi16">
<name>A = 4</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-3.99998858555087</minimum>
<maximum>3.99999873172734</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">2</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi36"))</local_columns>
</tdm_channel>
<tdm_channel id="usi17">
<name>A = 8</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-7.99997717110174</minimum>
<maximum>7.99999746345468</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">3</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi37"))</local_columns>
</tdm_channel>
<tdm_channel id="usi18">
<name>A = 16</name>
<group>#xpointer(id("usi12"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-15.9999543422035</minimum>
<maximum>15.9999949269094</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">4</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi38"))</local_columns>
</tdm_channel>
<tdm_channel id="usi19">
<name>F = 1</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999997146387718</minimum>
<maximum>0.999999682931835</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">0</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi39"))</local_columns>
</tdm_channel>
<tdm_channel id="usi20">
<name>F = 2</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999999230697499</minimum>
<maximum>0.999995986891472</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">1</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi40"))</local_columns>
</tdm_channel>
<tdm_channel id="usi21">
<name>F = 4</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999999230697499</minimum>
<maximum>0.99994907791452</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">2</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi41"))</local_columns>
</tdm_channel>
<tdm_channel id="usi22">
<name>F = 8</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999999230697499</minimum>
<maximum>0.999996490345607</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">3</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi42"))</local_columns>
</tdm_channel>
<tdm_channel id="usi23">
<name>F = 16</name>
<group>#xpointer(id("usi13"))</group>
<datatype>DT_DOUBLE</datatype>
<minimum>-0.999999230697499</minimum>
<maximum>0.999993076284592</maximum>
<instance_attributes>
<long_attribute name="NI_ArrayColumn">4</long_attribute>
<long_attribute name="NI_ChannelLength">1000</long_attribute>
<long_attribute name="NI_DataType">10</long_attribute>
</instance_attributes>
<local_columns>#xpointer(id("usi43"))</local_columns>
</tdm_channel>
<submatrix id="usi24">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi34"))</local_columns>
</submatrix>
<submatrix id="usi25">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi35"))</local_columns>
</submatrix>
<submatrix id="usi26">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi36"))</local_columns>
</submatrix>
<submatrix id="usi27">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi37"))</local_columns>
</submatrix>
<submatrix id="usi28">
<name>Untitled</name>
<measurement>#xpointer(id("usi12"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi38"))</local_columns>
</submatrix>
<submatrix id="usi29">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi39"))</local_columns>
</submatrix>
<submatrix id="usi30">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi40"))</local_columns>
</submatrix>
<submatrix id="usi31">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi41"))</local_columns>
</submatrix>
<submatrix id="usi32">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi42"))</local_columns>
</submatrix>
<submatrix id="usi33">
<name>Untitled</name>
<measurement>#xpointer(id("usi13"))</measurement>
<number_of_rows>1000</number_of_rows>
<local_columns>#xpointer(id("usi43"))</local_columns>
</submatrix>
<localcolumn id="usi34">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi14"))</measurement_quantity>
<submatrix>#xpointer(id("usi24"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>implicit_linear</sequence_representation>
<generation_parameters>-0.5 1.2</generation_parameters>
<values>#xpointer(id("usi1"))</values>
</localcolumn>
<localcolumn id="usi35">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi15"))</measurement_quantity>
<submatrix>#xpointer(id("usi25"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>implicit_linear</sequence_representation>
<generation_parameters>-0.5 1.2</generation_parameters>
<values>#xpointer(id("usi2"))</values>
</localcolumn>
<localcolumn id="usi36">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi16"))</measurement_quantity>
<submatrix>#xpointer(id("usi26"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>implicit_linear</sequence_representation>
<generation_parameters>-0.5 1.2</generation_parameters>
<values>#xpointer(id("usi3"))</values>
</localcolumn>
<localcolumn id="usi37">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi17"))</measurement_quantity>
<submatrix>#xpointer(id("usi27"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>implicit_linear</sequence_representation>
<generation_parameters>-0.5 1.2</generation_parameters>
<values>#xpointer(id("usi4"))</values>
</localcolumn>
<localcolumn id="usi38">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi18"))</measurement_quantity>
<submatrix>#xpointer(id("usi28"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>implicit_linear</sequence_representation>
<generation_parameters>-0.5 1.2</generation_parameters>
<values>#xpointer(id("usi5"))</values>
</localcolumn>
<localcolumn id="usi39">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi19"))</measurement_quantity>
<submatrix>#xpointer(id("usi29"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>implicit_linear</sequence_representation>
<generation_parameters>-0.5 1.2</generation_parameters>
<values>#xpointer(id("usi6"))</values>
</localcolumn>
<localcolumn id="usi40">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi20"))</measurement_quantity>
<submatrix>#xpointer(id("usi30"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>implicit_linear</sequence_representation>
<generation_parameters>-0.5 1.2</generation_parameters>
<values>#xpointer(id("usi7"))</values>
</localcolumn>
<localcolumn id="usi41">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi21"))</measurement_quantity>
<submatrix>#xpointer(id("usi31"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>implicit_linear</sequence_representation>
<generation_parameters>-0.0 1.0</generation_parameters>
<values>#xpointer(id("usi8"))</values>
</localcolumn>
<localcolumn id="usi42">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi22"))</measurement_quantity>
<submatrix>#xpointer(id("usi32"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>explicit</sequence_representation>
<values>#xpointer(id("usi9"))</values>
</localcolumn>
<localcolumn id="usi43">
<name>Untitled</name>
<measurement_quantity>#xpointer(id("usi23"))</measurement_quantity>
<submatrix>#xpointer(id("usi33"))</submatrix>
<global_flag>15</global_flag>
<independent>0</independent>
<sequence_representation>implicit_linear</sequence_representation>
<values>#xpointer(id("usi10"))</values>
</localcolumn>
</usi:data>
</usi:tdm>

View File

@ -1,63 +0,0 @@
#include <iostream>
#include <fstream>
#include <iterator>
#include <vector>
int main(int argc, char* argv[])
{
for ( int i = 0; i < argc; i++ )
{
std::cout<<argv[i]<<"\n";
}
if ( argc < 3 )
{
std::cout<<"missing file argument\n";
return 1;
}
std::ifstream fin(argv[1],std::ifstream::binary);
std::vector<unsigned char> tdxbuf((std::istreambuf_iterator<char>(fin)),
(std::istreambuf_iterator<char>()));
fin.close();
std::cout<<"length of buffer: "<<tdxbuf.size()<<"\n";
unsigned long int dtsize = 8;
if ( tdxbuf.size()%dtsize != 0 )
{
std::cout<<"mismatch between datatype size and length of buffer\n";
return 1;
}
unsigned long int nums = tdxbuf.size()/dtsize;
std::cout<<"number of entities: "<<nums<<"\n";
std::vector<unsigned char> tdxbufrev(tdxbuf.size());
for ( unsigned long int i = 0; i < nums; i++ )
{
for ( unsigned long int j = 0; j < dtsize; j++ )
{
tdxbufrev[i*dtsize+j] = tdxbuf[(i+1)*dtsize-(j+1)];
}
}
std::ofstream fou(argv[2],std::ifstream::binary);
for ( unsigned char ch: tdxbufrev)
{
fou<<ch;
}
fou.close();
return 0;
}

19
setup.py Normal file
View File

@ -0,0 +1,19 @@
from distutils.core import setup
from distutils.extension import Extension
from Cython.Build import cythonize
extensions = Extension(
name="tdm_ripper",
sources=["pytdm_ripper.pyx"],
# libraries=[""],
library_dirs=["lib"],
include_dirs=["lib"],
language='c++',
extra_compile_args=['-std=c++11','-Wno-unused-variable'],
extra_link_args=['-std=c++11'],
)
setup(
name="tdm_ripper",
ext_modules=cythonize(extensions)
)

19
setup_osx.py Normal file
View File

@ -0,0 +1,19 @@
from distutils.core import setup
from distutils.extension import Extension
from Cython.Build import cythonize
extensions = Extension(
name="tdm_ripper",
sources=["pytdm_ripper.pyx"],
# libraries=[""],
library_dirs=["lib"],
include_dirs=["lib"],
language='c++',
extra_compile_args=['-stdlib=libc++','-std=c++11','-Wno-unused-variable'],
extra_link_args=['-stdlib=libc++','-std=c++11'],
)
setup(
name="tdm_ripper",
ext_modules=cythonize(extensions)
)

View File

@ -1,400 +0,0 @@
// ------------------------------------------------------------------------- //
#include "tdm_termite.hpp"
#include <filesystem>
#include <regex>
#include <thread>
#include <chrono>
// ------------------------------------------------------------------------- //
const std::string gittag("TAGSTRING");
const std::string githash("HASHSTRING");
const std::string timestamp("TIMESTAMPSTRING");
void show_usage()
{
std::cout<<"\n"
<<"tdmtermite ["<<gittag<<"-g"<<githash<<"-"<<timestamp<<"] (https://github.com/RecordEvolution/TDMtermite.git)"
<<"\n\n"
<<"Decode TDM/TDX files and dump data as *.csv"
<<"\n\n"
<<"Usage:\n\n"
<<" tdmtermite <tdm-file> <tdx-file> [options]"
<<"\n\n"
<<"Options:"
<<"\n\n"
<<" -m, --showmeta show meta information about tdm dataset\n"
<<" -r, --showroot show root data of dataset\n"
<<" -g, --listgroups list channelgroups\n"
<<" -c, --listchannels list channels\n"
<<" -b, --listblocks list TDX blocks\n"
<<" -u, --listsubmatrices list submatrices\n"
<<" -l, --listlocalcolumns list localcolumns\n"
// <<"options with arguments must be provided as '--option arg' or '-o arg'\n"
<<" -d, --output (existing) output directory (default: no default)\n"
<<" -f, --filenames If an output directory is provided, all channels\n"
<<" of the dataset will be written to file(s).\n"
<<" The filenaming rule using %C (channel index), %c (channel name),\n"
<<" %G (group index), %g (group name) \n"
<<" determines whether the channels are collected in files according\n"
<<" to their channelgroups (as long as the filenaming rule includes\n"
<<" %G or %g) or written in separate files. For instance, to obtain\n"
<<" files with only one channel each use '--f channel_%C_%c.csv'.\n"
<<" (default: '--filenames channelgroup_%G.csv' )\n"
<<" -s, --csvsep separator character used in .csv files (default is comma '--csvsep ,')\n"
<<" -i, --includemeta include channel(-group) meta-data into files\n"
<<" -h, --help show this help message \n"
<<" -v, --version display version\n"
<<"\n";
}
// ------------------------------------------------------------------------- //
// define type of key-value map object
typedef std::map<std::string,std::string> optkeys;
const std::string argmsg = std::string("both .tdm and .tdx file must be provided!");
const std::string arguse = std::string("see $ tdmtermite --help for usage");
optkeys parse_args(int argc, char* argv[], bool showargs = false)
{
if ( showargs )
{
std::cout<<"number of CLI-arguments: "<<argc<<"\n";
for ( int i = 0; i < argc; i++ ) std::cout<<i<<": "<<argv[i]<<"\n";
}
// declare empty key-value object
optkeys prsdkeys;
if ( argc == 2 )
{
if ( std::string(argv[1]) == std::string("--help")
|| std::string(argv[1]) == std::string("-h") )
{
show_usage();
}
else if ( std::string(argv[1]) == std::string("--version")
|| std::string(argv[1]) == std::string("-v") )
{
std::cout<<"tdmtermite "<<gittag<<"-g"<<githash<<"-"<<timestamp<<"\n";
}
else
{
std::cerr<<argmsg + std::string("\n") + arguse + std::string("\n");
prsdkeys.insert(std::pair<std::string,std::string>("invalidoption",argv[1]));
return prsdkeys;
}
}
else if ( argc > 2 )
{
// tdm file
if ( std::string(argv[1]).find(std::string(".tdm")) != std::string::npos )
{
prsdkeys.insert(std::pair<std::string,std::string>("tdm",argv[1]));
}
else
{
std::string tdmerr = std::string(argv[1])
+ std::string(" does not look like a .tdm file")
+ std::string(", evtl. add file extension *.tdm")
+ std::string("\n") + arguse;
std::cerr<<tdmerr + std::string("\n");
prsdkeys.insert(std::pair<std::string,std::string>("invalidoption",argv[1]));
return prsdkeys;
}
// tdx file
if ( std::string(argv[2]).find(std::string(".tdx")) != std::string::npos )
{
prsdkeys.insert(std::pair<std::string,std::string>("tdx",argv[2]));
}
else
{
std::string tdxerr = std::string(argv[2])
+ std::string(" does not look like a .tdx file")
+ std::string(", evtl. add file extension *.tdx")
+ std::string("\n") + arguse;
std::cerr<<tdxerr + std::string("\n");
prsdkeys.insert(std::pair<std::string,std::string>("invalidoption",argv[2]));
return prsdkeys;
}
// options (in any order)
for ( int i = 3; i < argc; i++ )
{
if ( std::string(argv[i]) == std::string("--showmeta")
|| std::string(argv[i]) == std::string("-m") )
{
prsdkeys.insert(std::pair<std::string,std::string>("showmeta","showmeta"));
}
else if ( std::string(argv[i]) == std::string("--showroot")
|| std::string(argv[i]) == std::string("-r") )
{
prsdkeys.insert(std::pair<std::string,std::string>("showroot","showroot"));
}
else if ( std::string(argv[i]) == std::string("--listgroups")
|| std::string(argv[i]) == std::string("-g") )
{
prsdkeys.insert(std::pair<std::string,std::string>("listgroups","listgroups"));
}
else if ( std::string(argv[i]) == std::string("--listchannels")
|| std::string(argv[i]) == std::string("-c") )
{
prsdkeys.insert(std::pair<std::string,std::string>("listchannels","listchannels"));
}
else if ( std::string(argv[i]) == std::string("--listblocks")
|| std::string(argv[i]) == std::string("-b") )
{
prsdkeys.insert(std::pair<std::string,std::string>("listblocks","listblocks"));
}
else if ( std::string(argv[i]) == std::string("--listsubmatrices")
|| std::string(argv[i]) == std::string("-u") )
{
prsdkeys.insert(std::pair<std::string,std::string>("listsubmatrices","listsubmatrices"));
}
else if ( std::string(argv[i]) == std::string("--listlocalcolumns")
|| std::string(argv[i]) == std::string("-l") )
{
prsdkeys.insert(std::pair<std::string,std::string>("listlocalcolumns","listlocalcolumns"));
}
//
else if ( std::string(argv[i]) == std::string("--output")
|| std::string(argv[i]) == std::string("-d") )
{
if ( argc > i+1 )
{
prsdkeys.insert(std::pair<std::string,std::string>("output",argv[i+1]));
i += 1;
}
else
{
std::cerr<<"missing argument for "<<argv[i]<<"\n";
prsdkeys.insert(std::pair<std::string,std::string>("invalidoption",""));
}
}
else if ( std::string(argv[i]) == std::string("--filenames")
|| std::string(argv[i]) == std::string("-f") )
{
if ( argc > i+1 )
{
prsdkeys.insert(std::pair<std::string,std::string>("filenames",argv[i+1]));
i += 1;
}
else
{
std::cerr<<"missing argument for "<<argv[i]<<"\n";
prsdkeys.insert(std::pair<std::string,std::string>("invalidoption",""));
}
}
else if ( std::string(argv[i]) == std::string("--csvsep")
|| std::string(argv[i]) == std::string("-s") )
{
if ( argc > i+1 )
{
if ( std::string(argv[i+1]).size() == 1 )
{
prsdkeys.insert(std::pair<std::string,std::string>("csvsep",argv[i+1]));
i += 1;
}
else
{
std::cerr<<"invalid argument of --csvsep\n";
prsdkeys.insert(std::pair<std::string,std::string>("invalidoption",argv[i+1]));
}
}
else
{
std::cerr<<"missing argument for "<<argv[i]<<"\n";
prsdkeys.insert(std::pair<std::string,std::string>("invalidoption",""));
}
}
//
else if ( std::string(argv[i]) == std::string("--includemeta")
|| std::string(argv[i]) == std::string("-i") )
{
prsdkeys.insert(std::pair<std::string,std::string>("includemeta","includemeta"));
}
else
{
std::string argerr = std::string("unkown option '") + argv[i] + std::string("'");
std::cerr<<argerr + std::string("\n") + arguse + std::string("\n");
prsdkeys.insert(std::pair<std::string,std::string>("invalidoption",argv[i]));
return prsdkeys;
}
}
}
else
{
std::cerr<<argmsg + std::string("\n") + arguse + std::string("\n");
}
return prsdkeys;
}
// ------------------------------------------------------------------------- //
int main(int argc, char* argv[])
{
// parse CLI arguments
optkeys cfgopts = parse_args(argc,argv);
// show all CLI arguments
if ( false )
{
for ( optkeys::iterator it=cfgopts.begin(); it!=cfgopts.end(); ++it )
{
std::cout<<std::setw(20)<<std::left<<it->first<<it->second<<"\n";
}
}
// exit on any invalid option
if ( cfgopts.count("tdm") == 1 && cfgopts.count("tdx") == 1
&& cfgopts.count("invalidoption") == 0 )
{
// get options
bool showmeta = cfgopts.count("showmeta") == 1 ? true : false;
bool showroot = cfgopts.count("showroot") == 1 ? true : false;
bool listgroups = cfgopts.count("listgroups") == 1 ? true : false;
bool listchannels = cfgopts.count("listchannels") == 1 ? true : false;
bool listblocks = cfgopts.count("listblocks") == 1 ? true : false;
bool listsubmatrices = cfgopts.count("listsubmatrices") == 1 ? true : false;
bool listlocalcolumns = cfgopts.count("listlocalcolumns") == 1 ? true : false;
bool includemeta = cfgopts.count("includemeta") == 1 ? true : false;
// set required option values
std::string output = cfgopts.count("output") == 1 ? cfgopts.at("output")
: std::string("");
std::string files = cfgopts.count("filenames") == 1 ? cfgopts.at("filenames")
: std::string("channelgroup_%G.csv");
std::string csvsep = cfgopts.count("csvsep") == 1 ? cfgopts.at("csvsep")
: std::string(",");
// declare and initialize tdm_termite instance
tdm_termite jack;
try {
jack.submit_files(cfgopts.at("tdm"),cfgopts.at("tdx"),false);
} catch (const std::exception& e) {
throw std::runtime_error( std::string("failed to load/parse tdm/tdx files: ")
+ e.what() );
}
// show some meta data of the dataset
if ( showmeta ) std::cout<<"\n"<<jack.get_meta().get_info()<<"\n";
if ( showroot ) std::cout<<"\n"<<jack.get_root().get_info()<<"\n";
// get complete channel(-group) overview
format grpformatter(16,false,false,' ');
if (listgroups) std::cout<<"\n"<<jack.get_overview<tdm_channelgroup>(grpformatter)<<"\n";
format chformatter(16,false,false,' ');
if (listchannels) std::cout<<"\n"<<jack.get_channel_overview(chformatter)<<"\n";
// get complete submatrix/localcolumns overview
format formatter(16,false,false,' ');
if (listblocks) std::cout<<jack.get_overview<block>(formatter)<<"\n";
if (listsubmatrices) std::cout<<jack.get_overview<submatrix>(formatter)<<"\n";
if (listlocalcolumns) std::cout<<jack.get_overview<localcolumn>(formatter)<<"\n";
// print data to files
if ( !output.empty() )
{
// declare filesystem path
std::filesystem::path pd = output;
// check for given directory
if ( std::filesystem::is_directory(pd) )
{
// obtain channelgroup ids
std::vector<std::string> chgrids = jack.get_channelgroup_ids();
// check for any existing group-id in filenames rule
std::string slctgrp("");
for ( auto id: chgrids ) if ( files.find(id) != std::string::npos ) slctgrp = id;
// obtain list of channel ids
std::vector<std::string> chids = jack.get_channel_ids();
// check for any existing channel-id in filenames rule
std::string slctchn("");
for ( auto id: chids ) if ( files.find(id) != std::string::npos ) slctchn = id;
// write channels in files grouped by channelgroups
if ( files.find("%G") != std::string::npos || files.find("%g") != std::string::npos
|| !slctgrp.empty() )
{
// iterate through channelgroup ids
for ( auto id: chgrids )
{
// write all channelgroups or single chosen one
if ( slctgrp.empty() || slctgrp == id )
{
// get and sanitize group name
tdm_channelgroup tdmgrp = jack.channelgroup(id);
std::string grpnm = tdmgrp.name_;
std::regex regg("([^A-Za-z0-9])");
std::string grpname = std::regex_replace(grpnm,regg,"");
// construct file name according to filenaming rule
std::string filenm = files;
filenm = std::regex_replace(filenm,std::regex("\\%G"),tdmgrp.id_);
filenm = std::regex_replace(filenm,std::regex("\\%g"),grpname);
// concat paths
std::filesystem::path outfile = pd / filenm;
// write entire channelgroup to file
jack.print_group(id,outfile.c_str(),includemeta,csvsep.at(0));
}
}
}
// ...or write channels separately
else
{
// iterate through channel ids
for ( auto id: chids )
{
// write all channelgroups or single chosen one
if ( slctchn.empty() || slctchn == id )
{
// get and sanitize channel name
tdm_channel tdmchn = jack.channel(id);
std::string chnnm = tdmchn.name_;
std::regex regg("([^A-Za-z0-9])");
std::string chnname = std::regex_replace(chnnm,regg,"");
// construct file name according to filenaming rule
std::string filenm = files;
filenm = std::regex_replace(filenm,std::regex("\\%C"),tdmchn.id_);
filenm = std::regex_replace(filenm,std::regex("\\%c"),chnname);
// concat paths
std::filesystem::path outfile = pd / filenm;
// write entire channelgroup to file
jack.print_channel(id,outfile.c_str(),includemeta);
}
}
}
}
else
{
std::cerr<<std::string("directory '") + output
+ std::string("' does not exist") + std::string("\n");
}
}
else
{
if ( cfgopts.count("showmeta") == 0 && cfgopts.count("showroot") == 0
&& cfgopts.count("listgroups") == 0 && cfgopts.count("listchannels") == 0
&& cfgopts.count("listblocks") == 0 && cfgopts.count("listsubmatrices") == 0
&& cfgopts.count("listlocalcolumns") == 0 ) std::cerr<<"no output directory given\n";
}
}
// std::this_thread::sleep_for(std::chrono::milliseconds(10000));
return 0;
}
// ------------------------------------------------------------------------- //

View File

@ -1,33 +0,0 @@
// ------------------------------------------------------------------------- //
#include "tdm_termite.hpp"
#include <filesystem>
#include <regex>
#include <thread>
#include <chrono>
// ------------------------------------------------------------------------- //
int main(int argc, char* argv[])
{
std::string tdmfile(argv[1]);
std::string tdxfile(argv[2]);
std::cout<<tdmfile<<"\n"<<tdxfile<<"\n";
pugi::xml_document xml_doc;
pugi::xml_parse_result xml_result;
// load XML document from stream
std::ifstream fin(tdmfile.c_str());
xml_result = xml_doc.load(fin);
fin.close();
std::cout<<"\nloading "<<tdmfile<<": "<<xml_result.description()<<"\n";
std::cout<<"encoding: "<<(pugi::xml_encoding)xml_result.encoding<<"\n\n";
std::this_thread::sleep_for(std::chrono::milliseconds(4000));
return 0;
}

36
tdm_ripper.pxd Normal file
View File

@ -0,0 +1,36 @@
# cython: language_level = 3
# distutils: language = c++
# use some C++ STL libraries
from libcpp.string cimport string
from libcpp.vector cimport vector
from libcpp cimport bool
cdef extern from "tdm_ripper.cpp":
pass
cdef extern from "tdm_ripper.hpp":
cdef cppclass tdm_ripper:
tdm_ripper(string,string) except +
void list_channels()
void list_groups()
int num_channels()
int no_channels(int)
int num_groups()
int no_channel_groups()
string channel_name(int,int)
string group_name(int)
string channel_unit(int,int)
int channel_exists(int,string)
int obtain_channel_id(int,int)
vector[double] get_channel(int)
int channel_length(int,int)
string time_stamp(int,bool)
double get_min(int,int)
double get_max(int,int)
vector[double] channel(int,int)
void print_channel(int,const char*)
string get_meta(string attribute_name)
void print_meta(const char*)
# dummy method for compatibility
void close()