inital commit

.gitignore

@@ -0,0 +1 @@
+tdm_parser

README.md

@@ -0,0 +1 @@
+# tdm_ripper

lib/makefile

@@ -0,0 +1,14 @@
+
+
+CC = gcc -std=c++11 -stdlib=libc++
+CPPFLAGS = -O3 -Wall -Werror
+LIB = ../pugixml/
+
+libtdmripper.a : tdm_ripper.o
+	ar rcs $@ $^
+
+tdm_ripper.o : tdm_ripper.cpp tdm_ripper.hpp
+		$(CC) -c $(CPPFLAGS) -I $(LIB) $< -o $@
+
+clean :
+	rm -f *.o *.a

lib/tdm_ripper.cpp

@@ -0,0 +1,312 @@
+
+#include "tdm_ripper.hpp"
+
+tdm_ripper::tdm_ripper():
+  tdmfile_(""), tdxfile_(""), filesprovided_(false),
+  num_channels_(0), num_groups_(0),
+  channel_id_(0), group_id_(0), channel_name_(0), group_name_(0),
+  num_channels_group_(0), channels_group_(0),
+  byteoffset_(0), length_(0), type_(0)
+{
+
+}
+
+tdm_ripper::tdm_ripper(std::string tdmfile, std::string tdxfile):
+  tdmfile_(tdmfile), tdxfile_(tdxfile), filesprovided_(true),
+  num_channels_(0), num_groups_(0),
+  channel_id_(0), group_id_(0), channel_name_(0), group_name_(0),
+  num_channels_group_(0), channels_group_(0),
+  byteoffset_(0), length_(0), type_(0)
+{
+  setup();
+}
+
+void tdm_ripper::setfiles(std::string tdmfile, std::string tdxfile)
+{
+  tdmfile_ = tdmfile;
+  tdxfile_ = tdxfile;
+  filesprovided_ = true;
+  setup();
+}
+
+void tdm_ripper::setup()
+{
+  assert( filesprovided_ );
+
+  // TODO directly provide the C datatype to be used!!
+  datatypes_ = {
+    {"eInt8Usi",8},
+    {"eInt16Usi",16},
+    {"eInt32Usi",32},
+    {"eInt64Usi",64},
+    {"eUInt8Usi",8},
+    {"eUInt16Usi",16},
+    {"eUInt32Usi",32},
+    {"eUInt64Usi",64},
+    {"eFloat32Usi",32},
+    {"eFloat64Usi",64}
+  };
+
+  // make sure the provided file is a .tdm file
+  assert( tdmfile_.compare("") != 0 && "please provide a valid .tdm file" );
+  std::string::size_type idx;
+  idx = tdmfile_.find_last_of(".");
+  assert( idx != std::string::npos && "there's no file extension at all - .tdm is required" );
+  assert( tdmfile_.substr(tdmfile_.find_last_of(".")+1).compare("tdm") == 0 && "it's not a .tdm file" );
+
+  // setup of xml-parser
+  xml_result_ = xml_doc_.load_file(tdmfile_.c_str());
+  std::cout<<"\nloading and parsing file: "<<xml_result_.description()<<"\n";
+  std::cout<<"\nencoding: "<<(pugi::xml_encoding)xml_result_.encoding<<"\n\n";
+
+  pugi::xml_node subtreeincl = xml_doc_.child("usi:tdm").child("usi:include");
+
+  std::cout<<"file modified: "<<xml_doc_.child("usi:tdm").child("usi:data")
+                        .child("tdm_root").child_value("datetime")<<"\n\n";
+
+  // obtain corresponding .tdx filename given in .tdm file
+  if ( tdxfile_.compare("") == 0 )
+  {
+    tdxfile_ = tdmfile_.substr(0, tdmfile_.find_last_of("\\/"))
+                  +std::string("/")+subtreeincl.child("file").attribute("url").value();
+  }
+
+  // obtain endianness specified in .tdm file
+  std::string endianness(subtreeincl.child("file").attribute("byteOrder").value());
+  endianness_ = endianness.compare("littleEndian") == 0 ? true : false;
+
+  // obtain machine's endianess
+  int num = 1;
+  machine_endianness_ = ( *(char*)&num == 1 );
+  assert( machine_endianness_ == endianness_ );
+
+  std::cout<<"required .tdx-file is '"<<tdxfile_<<"'\n\n";
+
+  parse_structure();
+
+  // open .tdx and stream all binary data into vector
+  std::ifstream fin(tdxfile_.c_str(),std::ifstream::binary);
+  std::vector<unsigned char> tdxbuf((std::istreambuf_iterator<char>(fin)),
+                                    (std::istreambuf_iterator<char>()));
+  tdxbuf_ = tdxbuf;
+
+  std::cout<<"number of bytes in binary file: "<<tdxbuf_.size()<<"\n\n";
+}
+
+void tdm_ripper::parse_structure()
+{
+  // get node with channel and endianess information
+  pugi::xml_node subtreefile = xml_doc_.child("usi:tdm").child("usi:include").child("file");
+
+  for (pugi::xml_node anode: subtreefile.children())
+  {
+    // count overall number of channels
+    num_channels_++;
+
+    // get byteoffset of channel
+    byteoffset_.push_back(atoi(anode.attribute("byteOffset").value()));
+
+    // get length of channel
+    length_.push_back(atoi(anode.attribute("length").value()));
+
+    // find datatype of channel
+    type_.push_back(anode.attribute("valueType").value());
+  }
+
+  // get node with channels and groups
+  pugi::xml_node subtreedata = xml_doc_.child("usi:tdm").child("usi:data");
+
+  // extract basic information about available groups
+  for (pugi::xml_node anode: subtreedata.children())
+  {
+    if ( std::string(anode.name()).compare("tdm_channelgroup") == 0 )
+    {
+      num_groups_++;
+      group_id_.push_back(anode.attribute("id").value());
+      group_name_.push_back(anode.child_value("name"));
+      int numchann = count_occ_string(anode.child_value("channels"),"id");
+      num_channels_group_.push_back(numchann);
+    }
+  }
+
+  // extract basic information about available channels
+  for (pugi::xml_node anode: subtreedata.children())
+  {
+    if ( std::string(anode.name()).compare("tdm_channel") == 0 )
+    {
+      channel_id_.push_back(anode.attribute("id").value());
+      channel_name_.push_back(anode.child_value("name"));
+      std::string groupid(anode.child_value("group"));
+      for ( int g = 0; g < num_groups_; g++ )
+      {
+        if ( groupid.find(group_id_[g]) != std::string::npos ) channels_group_.push_back(g+1);
+      }
+    }
+  }
+
+  // check consistency of number of channelgroups
+  int numgroups = count_occ_string(subtreedata.child("tdm_root").child_value("channelgroups"),"id");
+  assert( numgroups == num_groups_ );
+
+  // check consistency of number of channels
+  assert( num_channels_ == (int)channel_id_.size()
+       && num_channels_ == (int)channel_name_.size()
+       && num_channels_ == (int)channels_group_.size() );
+}
+
+void tdm_ripper::show_channels(int width, int maxshow)
+{
+  std::cout<<std::setw(width)<<"index";
+  std::cout<<std::setw(width)<<"id";
+  std::cout<<std::setw(2*width)<<"name";
+  std::cout<<std::setw(width)<<"offset";
+  std::cout<<std::setw(width)<<"length";
+  std::cout<<std::setw(width)<<"datatype";
+  std::cout<<std::setw(width)<<"group";
+  std::cout<<std::setw(width)<<"group id";
+  std::cout<<std::setw(width)<<"group name";
+  std::cout<<std::setw(width)<<"num channels";
+  std::cout<<"\n";
+  std::cout<<std::setfill('-')<<std::setw(11*width+1)<<"\n";
+  std::cout<<std::setfill(' ');
+
+  for ( int i = 0; i < num_channels_ && i < maxshow; i++ )
+  {
+    std::cout<<std::setw(width)<<i+1;
+    std::cout<<std::setw(width)<<channel_id_[i];
+    std::cout<<std::setw(2*width)<<channel_name_[i];
+    std::cout<<std::setw(width)<<byteoffset_[i];
+    std::cout<<std::setw(width)<<length_[i];
+    std::cout<<std::setw(width)<<type_[i];
+    std::cout<<std::setw(width)<<channels_group_[i];
+    std::cout<<std::setw(width)<<group_id_[channels_group_[i]-1];
+    std::cout<<std::setw(width)<<group_name_[channels_group_[i]-1];
+    std::cout<<std::setw(width)<<num_channels_group_[channels_group_[i]-1];
+    std::cout<<"\n";
+  }
+  std::cout<<"\n\n";
+
+  if ( num_channels_ > maxshow )
+  {
+    for ( int i = num_channels_-maxshow; i < num_channels_; i++ )
+    {
+      std::cout<<std::setw(width)<<i+1;
+      std::cout<<std::setw(width)<<channel_id_[i];
+      std::cout<<std::setw(2*width)<<channel_name_[i];
+      std::cout<<std::setw(width)<<byteoffset_[i];
+      std::cout<<std::setw(width)<<length_[i];
+      std::cout<<std::setw(width)<<type_[i];
+      std::cout<<std::setw(width)<<channels_group_[i];
+      std::cout<<std::setw(width)<<group_id_[channels_group_[i]-1];
+      std::cout<<std::setw(width)<<group_name_[channels_group_[i]-1];
+      std::cout<<std::setw(width)<<num_channels_group_[channels_group_[i]-1];
+      std::cout<<"\n";
+    }
+    std::cout<<"\n\n";
+  }
+}
+
+void tdm_ripper::show_structure()
+{
+  int width = 25;
+
+  std::cout<<"second level tree elements:\n";
+  for ( pugi::xml_node child: xml_doc_.child("usi:tdm").children())
+  {
+    std::cout<<child.name()<<"\n";
+  }
+  std::cout<<"\n\n";
+
+  pugi::xml_node subtreeincl = xml_doc_.child("usi:tdm").child("usi:include");
+
+  // most important information in .tdm file
+  // - byteOffset provides the starting position of particular channel
+  // - length     is the number of e.g. double (=8byte) value in that channel
+  std::cout<<"file properties:\n\n";
+  for (pugi::xml_node anode: subtreeincl.children("file"))
+  {
+    for (pugi::xml_attribute attr: anode.attributes())
+    {
+      std::cout<<" "<<attr.name()<<" = "<<attr.value()<<"   ";
+    }
+    std::cout<<"\n\n";
+
+    int iter = 0;
+    for (pugi::xml_node child: anode.children())
+    {
+      if ( iter < 100 )
+      {
+        std::cout<<std::right;
+        std::cout<<std::setw(width)<<iter;
+        std::cout<<std::setw(width)<<child.name();
+        std::cout<<std::setw(width)<<child.value();
+
+        for (pugi::xml_attribute attr: child.attributes())
+        {
+          std::cout<<std::right<<attr.name()<<" = "<<std::setw(width)<<std::left<<attr.value()<<"   ";
+
+          if ( std::string(attr.name()).compare("valueType") == 0 )
+          {
+            std::cout<<"number of bytes = "<<datatypes_[attr.value()]/CHAR_BIT;
+          }
+        }
+        std::cout<<"\n";
+      }
+
+      iter++;
+    }
+  }
+  std::cout<<"\n\n";
+}
+
+// convert array of chars to floating point double
+double tdm_ripper::convert_float(std::vector<unsigned char> bych)
+{
+  assert( bych.size() == sizeof(double) );
+  assert( endianness_ );
+
+  // check for IEEE754 floating point standard
+  assert( std::numeric_limits<double>::is_iec559 );
+
+  double df = 0.0;
+
+  uint8_t *dfcast = reinterpret_cast<uint8_t*>(&df);
+
+  for ( int i = 0; i < (int)sizeof(double); i++ )
+  {
+    dfcast[i] = (int)bych[i];
+  }
+
+  return df;
+}
+
+std::vector<double> tdm_ripper::convert_channel(int byteoffset, int length, int typesize)
+{
+  std::vector<double> chann(length);
+
+  for ( int i = 0; i < length; i++ )
+  {
+    std::vector<unsigned char> cseg(tdxbuf_.begin()+byteoffset+i*typesize,
+                                    tdxbuf_.begin()+byteoffset+(i+1)*typesize);
+    chann[i] = convert_float(cseg);
+  }
+
+  return chann;
+}
+
+std::vector<double> tdm_ripper::get_channel(int channelid)
+{
+  assert( channelid > 0 && channelid <= num_channels_ && "please provide valid channel id" );
+
+  return convert_channel(byteoffset_[channelid-1],length_[channelid-1],
+                              datatypes_[type_[channelid-1]]/CHAR_BIT);
+}
+
+void tdm_ripper::print_channel(int channelid, const char* filename, int width)
+{
+  std::ofstream fout(filename);
+
+  std::vector<double> channdat = get_channel(channelid);
+  for ( auto el: channdat ) fout<<std::setw(width)<<el<<"\n";
+  fout.close();
+}

lib/tdm_ripper.hpp

@@ -0,0 +1,97 @@
+
+#ifndef TDM_RIPPER
+#define TDM_RIPPER
+
+#include <iostream>
+#include <fstream>
+#include <iterator>
+#include <vector>
+#include <iomanip>
+#include <stdlib.h>
+#include <assert.h>
+#include <map>
+#include "../pugixml/pugixml.hpp"
+
+class tdm_ripper
+{
+  // .tdm and .tdx filenames
+  std::string tdmfile_;
+  std::string tdxfile_;
+  bool filesprovided_;
+
+  // endianness (true = little, false = big)
+  bool endianness_, machine_endianness_;
+
+  // number/names/ids of channels, channelgroups and channels's assignment to groups
+  int num_channels_, num_groups_;
+  std::vector<std::string> channel_id_, group_id_, channel_name_, group_name_;
+  std::vector<int> num_channels_group_;
+  std::vector<int> channels_group_;
+
+  // byteoffset, length and datatype of channels
+  std::vector<int> byteoffset_;
+  std::vector<int> length_;
+  std::vector<std::string> type_;
+
+  // mapping of NI datatype to size (in bytes) of type
+  std::map<std::string, int> datatypes_;
+
+  // xml parser
+  pugi::xml_document xml_doc_;
+  pugi::xml_parse_result xml_result_;
+
+  // binary data container
+  std::vector<unsigned char> tdxbuf_;
+
+public:
+
+  tdm_ripper();
+  tdm_ripper(std::string tdmfile, std::string tdxfile = "");
+
+  void setfiles(std::string tdmfile, std::string tdxfile = "");
+  void setup();
+
+  void parse_structure();
+
+  void show_channels(int width = 15, int maxshow = 300);
+
+  void show_structure();
+
+  // count number of occurences of substring in string
+  int count_occ_string(std::string s, std::string sub)
+  {
+    int num_occs = 0;
+    std::string::size_type pos = 0;
+
+    while ( ( pos = s.find(sub,pos) ) != std::string::npos )
+    {
+      num_occs++;
+      pos += sub.length();
+    }
+
+    return num_occs;
+  }
+
+  // provide number of channels and group
+  const int& num_channels()
+  {
+    return num_channels_;
+  }
+  const int& num_groups()
+  {
+    return num_groups_;
+  }
+
+  // convert array of chars to single floating point double
+  double convert_float(std::vector<unsigned char> bych);
+
+  // convert entire channel, i.e. expert of .tdx binary file
+  std::vector<double> convert_channel(int byteoffset, int length, int typesize);
+
+  std::vector<double> get_channel(int channelid);
+
+  void print_channel(int channelid, const char* filename, int width = 15);
+
+};
+
+#endif

main.cpp

@@ -0,0 +1,29 @@
+
+
+#include "lib/tdm_ripper.hpp"
+
+int main(int argc, char* argv[])
+{
+  // path of filename provided ?
+  assert( argc > 1 && "please provide a filename and path" );
+
+  // declare and initialize tdm_ripper
+  tdm_ripper ripper(argv[1]);
+
+  ripper.show_channels();
+  // ripper.show_structure();
+
+  std::cout<<"number of channels "<<ripper.num_channels()<<"\n\n";
+  std::cout<<"number of groups "<<ripper.num_groups()<<"\n\n";
+
+  // std::vector<double> channA = ripper.get_channel(9);
+  // for ( auto el: channA ) std::cout<<el<<"\n";
+  // std::cout<<"\n\n";
+
+  for ( int i = 0; i < 20; i++ )
+  {
+    ripper.print_channel(i+1,("channel_"+std::to_string(i+1)+".dat").c_str());
+  }
+
+  return 0;
+}

makefile

@@ -0,0 +1,33 @@
+
+
+EXE = tdm_parser
+CC = g++ -std=c++11 -stdlib=libc++
+CPPFLAGS = -O3 -Wall -Werror -Wunused-variable -Wsign-compare
+LIB = pugixml/
+
+$(EXE) : main.o tdm_ripper.o
+	$(CC) $(CPPFLAGS) $^ -o $@
+
+main.o : main.cpp
+	$(CC) -c $(CPPFLAGS) -I $(LIB) $< -o $@
+
+tdm_ripper.o : lib/tdm_ripper.cpp lib/tdm_ripper.hpp
+		$(CC) -c $(CPPFLAGS) -I $(LIB) $< -o $@
+
+clean :
+	rm -f $(EXE) *.o
+
+newlib :
+	g++ -bundle -undefined dynamic_lookup -L/anaconda3/lib -arch x86_64 -L/anaconda3/lib -arch x86_64 -arch x86_64 build/temp.macosx-10.7-x86_64-3.7/pytdm_ripper.o -Llib -o /Users/mariofink/Desktop/tdm_tdx/tdm_ripper/pytdm_ripper.cpython-37m-darwin.so
+
+pylib : setup.py pytdm_ripper.pyx tdm_ripper.pxd
+	python3 setup.py build_ext --inplace
+
+lib/libtdmripper.a :
+	make -C lib libtdmripper.a
+
+clean-lib :
+	rm -f lib/*.o lib/*.a
+	rm -f -r build
+	rm -f tdm_ripper.c tdm_ripper.cpp
+	rm -f *.so

pugixml/pugiconfig.hpp

@@ -0,0 +1,74 @@
+/**
+ * pugixml parser - version 1.9
+ * --------------------------------------------------------
+ * Copyright (C) 2006-2019, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com)
+ * Report bugs and download new versions at https://pugixml.org/
+ *
+ * This library is distributed under the MIT License. See notice at the end
+ * of this file.
+ *
+ * This work is based on the pugxml parser, which is:
+ * Copyright (C) 2003, by Kristen Wegner (kristen@tima.net)
+ */
+
+#ifndef HEADER_PUGICONFIG_HPP
+#define HEADER_PUGICONFIG_HPP
+
+// Uncomment this to enable wchar_t mode
+// #define PUGIXML_WCHAR_MODE
+
+// Uncomment this to enable compact mode
+// #define PUGIXML_COMPACT
+
+// Uncomment this to disable XPath
+// #define PUGIXML_NO_XPATH
+
+// Uncomment this to disable STL
+// #define PUGIXML_NO_STL
+
+// Uncomment this to disable exceptions
+// #define PUGIXML_NO_EXCEPTIONS
+
+// Set this to control attributes for public classes/functions, i.e.:
+// #define PUGIXML_API __declspec(dllexport) // to export all public symbols from DLL
+// #define PUGIXML_CLASS __declspec(dllimport) // to import all classes from DLL
+// #define PUGIXML_FUNCTION __fastcall // to set calling conventions to all public functions to fastcall
+// In absence of PUGIXML_CLASS/PUGIXML_FUNCTION definitions PUGIXML_API is used instead
+
+// Tune these constants to adjust memory-related behavior
+// #define PUGIXML_MEMORY_PAGE_SIZE 32768
+// #define PUGIXML_MEMORY_OUTPUT_STACK 10240
+// #define PUGIXML_MEMORY_XPATH_PAGE_SIZE 4096
+
+// Uncomment this to switch to header-only version
+#define PUGIXML_HEADER_ONLY
+
+// Uncomment this to enable long long support
+// #define PUGIXML_HAS_LONG_LONG
+
+#endif
+
+/**
+ * Copyright (c) 2006-2019 Arseny Kapoulkine
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */

pytdm_ripper.pyx

@@ -0,0 +1,18 @@
+# cython: language_level=3
+# distutils: language = c++
+## distutils: sources = lib/tdm_ripper.cpp
+
+# from libcpp.string cimport string
+# from libcpp.vector cimport vector
+
+from tdm_ripper cimport tdm_ripper
+
+cdef class pytdmripper:
+    # C++ instance
+    cdef tdm_ripper cripp
+
+    def __cinit__(self, string tdmfile, string tdxfile):
+        self.cripp = tdm_ripper(tdmfile,tdxfile)
+
+    def setfiles(self, string tdmfile, string tdxfile):
+        self.cripp(tdmfile,tdxfile)

setup.py

@@ -0,0 +1,19 @@
+from distutils.core import setup
+from distutils.extension import Extension
+from Cython.Build import cythonize
+
+extensions = Extension(
+    name="tdm_ripper",
+    sources=["pytdm_ripper.pyx"], #,"lib/tdm_ripper.cpp"],
+    # libraries=["tdm_ripper"],
+    library_dirs=["lib"],
+    include_dirs=["lib"],
+    language='c++',
+    extra_compile_args=['-stdlib=libc++','-std=c++11'],
+    extra_link_args=['-std=c++11'],
+)
+
+setup(
+    name="tdm_ripper",
+    ext_modules=cythonize(extensions)
+)

tdm_ripper.pxd

@@ -0,0 +1,14 @@
+
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+
+# use only in absence of # distutils: sources = Rectangle.cpp in .pyx
+cdef extern from "tdm_ripper.cpp":
+    pass
+
+cdef extern from "tdm_ripper.hpp":
+    cdef cppclass tdm_ripper:
+        tdm_ripper() except +
+        tdm_ripper(string, string) except +
+        void setfiles(string, string)
+        void setup()