mario/TDMtermite
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

keep ifstream object for tdx file instead of reading entire file in buffer

Browse Source
This commit is contained in:
Mario Fink 2021-05-03 18:20:15 +02:00
parent 3443d2dafa
commit c4f2a5ff5d
3 changed files with 44 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -14,3 +14,5 @@ cython/*.cpp
tdmtermite
dist/
*.egg-info/
output/
monitor-process.sh

56
lib/tdm_termite.cpp
View File

@ -16,6 +16,12 @@ tdm_termite::tdm_termite(std::string tdmfile, std::string tdxfile, bool showlog)
this->process_tdm(showlog);
}
tdm_termite::~tdm_termite()
{
// close tdx-file stream
tdx_ifstream_.close();
}
void tdm_termite::submit_files(std::string tdmfile, std::string tdxfile, bool showlog)
{
// save files
@ -106,20 +112,26 @@ void tdm_termite::process_tdm(bool showlog)
this->process_localcolumns(showlog,xml_doc);
// open .tdx and stream all binary data into buffer
// try {
// std::ifstream fin(tdxfile_.c_str(),std::ifstream::binary);
// // if ( !fin.good() ) std::cerr<<"failed to open .tdx-file\n";
//
// std::vector<unsigned char> tdxbuf((std::istreambuf_iterator<char>(fin)),
// (std::istreambuf_iterator<char>()));
// tdxbuffer_ = tdxbuf;
//
// if ( showlog ) std::cout<<"size of .tdx buffer (bytes): "<<tdxbuffer_.size()<<"\n\n";
//
// // close .tdx file
// fin.close();
// } catch (const std::exception& e ) {
// throw std::runtime_error( std::string("failed to open .tdx and stream data to buffer: ")
// + e.what() );
// }
try {
std::ifstream fin(tdxfile_.c_str(),std::ifstream::binary);
// if ( !fin.good() ) std::cerr<<"failed to open .tdx-file\n";
std::vector<unsigned char> tdxbuf((std::istreambuf_iterator<char>(fin)),
(std::istreambuf_iterator<char>()));
tdxbuffer_ = tdxbuf;
if ( showlog ) std::cout<<"size of .tdx buffer (bytes): "<<tdxbuffer_.size()<<"\n\n";
// close .tdx file
fin.close();
} catch (const std::exception& e ) {
throw std::runtime_error( std::string("failed to open .tdx and stream data to buffer: ")
tdx_ifstream_.open(tdxfile_.c_str(),std::ifstream::binary);
} catch (const std::exception& e) {
throw std::runtime_error( std::string("failed to open .tdx file in ifstream: ")
+ e.what() );
}
}
@ -584,10 +596,20 @@ std::vector<tdmdatatype> tdm_termite::get_channel(std::string& id)
// declare buffer covering the required range of "tdxbuffer_"
// (consider both channel-wise and block-wise ordering)
unsigned long int strtidx = blk.block_offset_*blk.block_size_
+ blk.byte_offset_,
fnshidx = strtidx + blk.length_*dtyp.size_;
std::vector<unsigned char> tdxblk( tdxbuffer_.begin()+strtidx,
tdxbuffer_.begin()+fnshidx );
+ blk.byte_offset_;
// fnshidx = strtidx + blk.length_*dtyp.size_;
// std::vector<unsigned char> tdxblk( tdxbuffer_.begin()+strtidx,
// tdxbuffer_.begin()+fnshidx );
char* blkbuf = new char[blk.length_*dtyp.size_];
try {
tdx_ifstream_.seekg(strtidx);
tdx_ifstream_.read(blkbuf,blk.length_*dtyp.size_);
} catch (const std::exception& e) {
throw std::runtime_error( std::string("failed to read block from tdx_ifstream_: ")
+ e.what() );
}
std::vector<unsigned char> tdxblk(blkbuf,blkbuf+blk.length_*dtyp.size_);
delete []blkbuf;
// distinguish numeric datatypes included in "tdmdatatype"
if ( blk.value_type_ == std::string("eInt16Usi") )

4
lib/tdm_termite.hpp
View File

@ -60,8 +60,9 @@ class tdm_termite
std::map<std::string,submatrix> submatrices_;
std::map<std::string,localcolumn> localcolumns_;
// binary data container
// binary data container/file stream
std::vector<unsigned char> tdxbuffer_;
std::ifstream tdx_ifstream_;
// extract list of identifiers from e.g. "#xpointer(id("usi12") id("usi13"))"
std::vector<std::string> extract_ids(std::string idstring)
@ -121,6 +122,7 @@ public:
tdm_termite();
tdm_termite(std::string tdmfile, std::string tdxfile = std::string(""),
bool showlog = false);
~tdm_termite();
// provide (tdm,tdx) files
void submit_files(std::string tdmfile, std::string tdxfile = std::string(""),