mario/IMCtermite
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

read and parse all segment marker elements and start distinction between datatype based on header info

Browse Source
This commit is contained in:
Mario Fink 2020-02-12 11:54:25 +00:00
parent 56438bb967
commit 68378ef8fa
4 changed files with 758 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

600
csv/VehicleSpeed_HS.csv
View File

@ -0,0 +1,600 @@
5.94
5.93
5.92
5.9
5.9
5.89
5.89
5.87
5.84
5.83
5.81
5.79
5.78
5.76
5.75
5.74
5.74
5.73
5.73
5.72
5.71
5.71
5.69
5.69
5.68
5.66
5.63
5.62
5.6
5.58
5.57
5.55
5.53
5.51
5.5
5.47
5.45
5.42
5.4
5.38
5.35
5.31
5.28
5.27
5.24
5.19
5.16
5.14
5.11
5.06
5.02
5
4.97
4.92
4.88
4.86
4.82
4.77
4.74
4.7
4.66
4.62
4.58
4.54
4.48
4.43
4.37
4.31
4.24
4.18
4.13
4.06
3.98
3.91
3.83
3.77
3.72
3.64
3.57
3.51
3.44
3.39
3.33
3.28
3.24
3.17
3.11
3.06
3.01
2.96
2.92
2.88
2.83
2.79
2.75
2.71
2.68
2.64
2.61
2.59
2.56
2.55
2.52
2.51
2.49
2.48
2.46
2.46
2.45
2.44
2.43
2.42
2.41
2.4
2.41
2.39
2.38
2.35
2.31
2.26
2.21
2.16
2.1
2.05
2.01
1.94
1.91
1.84
1.77
1.73
1.65
1.59
1.58
1.55
1.52
1.53
1.52
1.5
1.49
1.47
1.47
1.47
1.47
1.48
1.48
1.48
1.48
1.48
1.46
1.44
1.44
1.43
1.41
1.41
1.4
1.38
1.37
1.36
1.35
1.33
1.32
1.31
1.28
1.27
1.26
1.24
1.22
1.21
1.21
1.19
1.17
1.16
1.16
1.13
1.12
1.12
1.11
1.09
1.08
1.07
1.07
1.05
1.04
1.03
1.02
1.01
0.99
0.98
0.96
0.95
0.94
0.9
0.87
0.84
0.8
0.79
0.78
0.75
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
0
1 5.94
1 5.94
2 5.93
3 5.92
4 5.9
5 5.9
6 5.89
7 5.89
8 5.87
9 5.84
10 5.83
11 5.81
12 5.79
13 5.78
14 5.76
15 5.75
16 5.74
17 5.74
18 5.73
19 5.73
20 5.72
21 5.71
22 5.71
23 5.69
24 5.69
25 5.68
26 5.66
27 5.63
28 5.62
29 5.6
30 5.58
31 5.57
32 5.55
33 5.53
34 5.51
35 5.5
36 5.47
37 5.45
38 5.42
39 5.4
40 5.38
41 5.35
42 5.31
43 5.28
44 5.27
45 5.24
46 5.19
47 5.16
48 5.14
49 5.11
50 5.06
51 5.02
52 5
53 4.97
54 4.92
55 4.88
56 4.86
57 4.82
58 4.77
59 4.74
60 4.7
61 4.66
62 4.62
63 4.58
64 4.54
65 4.48
66 4.43
67 4.37
68 4.31
69 4.24
70 4.18
71 4.13
72 4.06
73 3.98
74 3.91
75 3.83
76 3.77
77 3.72
78 3.64
79 3.57
80 3.51
81 3.44
82 3.39
83 3.33
84 3.28
85 3.24
86 3.17
87 3.11
88 3.06
89 3.01
90 2.96
91 2.92
92 2.88
93 2.83
94 2.79
95 2.75
96 2.71
97 2.68
98 2.64
99 2.61
100 2.59
101 2.56
102 2.55
103 2.52
104 2.51
105 2.49
106 2.48
107 2.46
108 2.46
109 2.45
110 2.44
111 2.43
112 2.42
113 2.41
114 2.4
115 2.41
116 2.39
117 2.38
118 2.35
119 2.31
120 2.26
121 2.21
122 2.16
123 2.1
124 2.05
125 2.01
126 1.94
127 1.91
128 1.84
129 1.77
130 1.73
131 1.65
132 1.59
133 1.58
134 1.55
135 1.52
136 1.53
137 1.52
138 1.5
139 1.49
140 1.47
141 1.47
142 1.47
143 1.47
144 1.48
145 1.48
146 1.48
147 1.48
148 1.48
149 1.46
150 1.44
151 1.44
152 1.43
153 1.41
154 1.41
155 1.4
156 1.38
157 1.37
158 1.36
159 1.35
160 1.33
161 1.32
162 1.31
163 1.28
164 1.27
165 1.26
166 1.24
167 1.22
168 1.21
169 1.21
170 1.19
171 1.17
172 1.16
173 1.16
174 1.13
175 1.12
176 1.12
177 1.11
178 1.09
179 1.08
180 1.07
181 1.07
182 1.05
183 1.04
184 1.03
185 1.02
186 1.01
187 0.99
188 0.98
189 0.96
190 0.95
191 0.94
192 0.9
193 0.87
194 0.84
195 0.8
196 0.79
197 0.78
198 0.75
199 0
200 0
201 0
202 0
203 0
204 0
205 0
206 0
207 0
208 0
209 0
210 0
211 0
212 0
213 0
214 0
215 0
216 0
217 0
218 0
219 0
220 0
221 0
222 0
223 0
224 0
225 0
226 0
227 0
228 0
229 0
230 0
231 0
232 0
233 0
234 0
235 0
236 0
237 0
238 0
239 0
240 0
241 0
242 0
243 0
244 0
245 0
246 0
247 0
248 0
249 0
250 0
251 0
252 0
253 0
254 0
255 0
256 0
257 0
258 0
259 0
260 0
261 0
262 0
263 0
264 0
265 0
266 0
267 0
268 0
269 0
270 0
271 0
272 0
273 0
274 0
275 0
276 0
277 0
278 0
279 0
280 0
281 0
282 0
283 0
284 0
285 0
286 0
287 0
288 0
289 0
290 0
291 0
292 0
293 0
294 0
295 0
296 0
297 0
298 0
299 0
300 0
301 0
302 0
303 0
304 0
305 0
306 0
307 0
308 0
309 0
310 0
311 0
312 0
313 0
314 0
315 0
316 0
317 0
318 0
319 0
320 0
321 0
322 0
323 0
324 0
325 0
326 0
327 0
328 0
329 0
330 0
331 0
332 0
333 0
334 0
335 0
336 0
337 0
338 0
339 0
340 0
341 0
342 0
343 0
344 0
345 0
346 0
347 0
348 0
349 0
350 0
351 0
352 0
353 0
354 0
355 0
356 0
357 0
358 0
359 0
360 0
361 0
362 0
363 0
364 0
365 0
366 0
367 0
368 0
369 0
370 0
371 0
372 0
373 0
374 0
375 0
376 0
377 0
378 0
379 0
380 0
381 0
382 0
383 0
384 0
385 0
386 0
387 0
388 0
389 0
390 0
391 0
392 0
393 0
394 0
395 0
396 0
397 0
398 0
399 0
400 0
401 0
402 0
403 0
404 0
405 0
406 0
407 0
408 0
409 0
410 0
411 0
412 0
413 0
414 0
415 0
416 0
417 0
418 0
419 0
420 0
421 0
422 0
423 0
424 0
425 0
426 0
427 0
428 0
429 0
430 0
431 0
432 0
433 0
434 0
435 0
436 0
437 0
438 0
439 0
440 0
441 0
442 0
443 0
444 0
445 0
446 0
447 0
448 0
449 0
450 0
451 0
452 0
453 0
454 0
455 0
456 0
457 0
458 0
459 0
460 0
461 0
462 0
463 0
464 0
465 0
466 0
467 0
468 0
469 0
470 0
471 0
472 0
473 0
474 0
475 0
476 0
477 0
478 0
479 0
480 0
481 0
482 0
483 0
484 0
485 0
486 0
487 0
488 0
489 0
490 0
491 0
492 0
493 0
494 0
495 0
496 0
497 0
498 0
499 0
500 0
501 0
502 0
503 0
504 0
505 0
506 0
507 0
508 0
509 0
510 0
511 0
512 0
513 0
514 0
515 0
516 0
517 0
518 0
519 0
520 0
521 0
522 0
523 0
524 0
525 0
526 0
527 0
528 0
529 0
530 0
531 0
532 0
533 0
534 0
535 0
536 0
537 0
538 0
539 0
540 0
541 0
542 0
543 0
544 0
545 0
546 0
547 0
548 0
549 0
550 0
551 0
552 0
553 0
554 0
555 0
556 0
557 0
558 0
559 0
560 0
561 0
562 0
563 0
564 0
565 0
566 0
567 0
568 0
569 0
570 0
571 0
572 0
573 0
574 0
575 0
576 0
577 0
578 0
579 0
580 0
581 0
582 0
583 0
584 0
585 0
586 0
587 0
588 0
589 0
590 0
591 0
592 0
593 0
594 0
595 0
596 0
597 0
598 0
599 0
600 0

BIN
eatit
View File

Binary file not shown.

11
src/main.cpp
View File

@ -28,6 +28,9 @@ int main(int argc, char* argv[])
// determine position of markers in buffer and read data
eatraw.find_markers();
// split data segments into elements
eatraw.split_segments();
std::cout<<"\n";
std::map<std::string,std::vector<unsigned char>> marks = eatraw.get_markers();
for ( auto mrk: marks )
@ -40,12 +43,14 @@ int main(int argc, char* argv[])
std::cout<<std::setfill('-')<<std::setw(96)<<'\n'<<std::setfill(' ');
eatraw.show_hex(dat,32,512);
std::cout<<"\n";
std::vector<std::string> segvec = eatraw.get_segment(mrk.first);
std::cout<<"number of elements in segment: "<<segvec.size()<<"\n\n";
//for ( auto el: segvec ) std::cout<<el<<"\n";
}
// convert unsigned char data in buffer to desired data type
eatraw.convert_data_16_bit_decimal();
//eatraw.convert_data_16_bit_float();
//eatraw.convert_data_32_bit_float();
eatraw.convert_data();
// get array of encoded data
std::vector<double> maindata = eatraw.get_data();

240
src/raweat.hpp
View File

@ -53,12 +53,13 @@ private:
// data sections corresponding to markers
std::map<std::string,std::vector<unsigned char>> datasec_;
// split segments into arrays of simple number/string element
std::map<std::string,std::vector<std::string>> segments_;
// length of data array
unsigned long int datsize_;
// TODO preliminary: for now, we assume 32/64 bit ? floats in all data
// index in buffer of datasec_["datas marker"] where actual data starts
unsigned long int datstartidx_;
std::vector<double> datmes_;
public:
@ -116,12 +117,14 @@ public:
std::cout<<std::dec;
}
//---------------------------------------------------------------------------//
// find predefined markers in data buffer
void find_markers()
{
for (std::pair<std::string,std::vector<unsigned char>> mrk : markers_ )
{
assert( mrk.second.size() > 0 && "please don't defined any empty marker" );
assert( mrk.second.size() > 0 && "please don't define any empty marker" );
// find marker's byte sequence in buffer
for ( unsigned long int idx = 0; idx < rawdata_.size(); idx++ )
@ -150,10 +153,7 @@ public:
}
else
{
// make sure the data marker is actually the last and extends until end of file
//assert( TODO && "data marker doesn't appear to be the very last");
// that's the data itself
// data marker is actually assumed to be the last and should extend until end of file
for ( unsigned long int didx = idx; didx < rawdata_.size()-1; didx++ )
{
markseq.push_back(rawdata_[didx]);
@ -161,21 +161,6 @@ public:
// obtain length of data segment
datsize_ = markseq.size();
// find starting index (supposed to be after fourth comma = 0x2c)
int countcomma = 0;
for ( unsigned long int buidx = 0; buidx < datsize_; buidx++ )
{
// count number of comma chars in head of data segment
if ( markseq[buidx] == 0x2c ) countcomma++;
// save position following fourth comma
if ( countcomma == 4 )
{
datstartidx_ = buidx + 1;
break;
}
}
}
// save segment corresponding to marker
@ -198,6 +183,143 @@ public:
return datasec_[marker];
}
// split data segments into arrays
void split_segments()
{
// split segments of all markers
for (std::pair<std::string,std::vector<unsigned char>> mrk : markers_ )
{
// declare empty array for this segment and auxiliary string
std::vector<std::string> segvec;
std::string elstr("");
// only start collecting after first comma in segment
bool parse = false;
// count number of commata
long int commcount = 0;
// parse data segment
for ( unsigned char el: datasec_[mrk.first] )
{
// note that data segment of "datas marker" may contain any number of 0x2c's
if ( ( el != 0x2c && parse ) || ( mrk.first == "datas marker" && commcount > 2 ) )
{
elstr.push_back(el);
}
else if ( el == 0x2c && parse )
{
// comma marks end of element of segment: save string and reset it
segvec.push_back(elstr);
elstr = std::string("");
commcount++;
}
else
{
// enable parsing after first comma
if ( el == 0x2c ) parse = true;
}
}
// include last element
segvec.push_back(elstr);
// save array of elements
segments_.insert(std::pair<std::string,std::vector<std::string>>(mrk.first,segvec));;
}
}
//---------------------------------------------------------------------------//
// convert actual measurement data
void convert_data()
{
// by convention, the actual data is the 4th element
std::string datstr = segments_["datas marker"][3];
std::vector<unsigned char> datbuf(datstr.begin(),datstr.end());
// retrieve datatype from segment
int typesize = std::stoi(segments_["datyp marker"][5]);
if ( typesize == 32 ) convert_data_32_bit_float(datbuf);
if ( false ) convert_data_16_bit_float();
if ( typesize == 16 ) convert_data_16_bit_decimal(datbuf);
}
// convert single precision 32bit floating point numbers
void convert_data_32_bit_float(std::vector<unsigned char> &datbuf)
{
// check size of buffer assuming size of single precision float is 4 byte
assert ( datbuf.size()%4 == 0 && "length of buffer is not a multiple of 4" );
// get number of single precision floats in buffer
unsigned long int totnumfl = datbuf.size()/(int)sizeof(float);
for ( unsigned long int numfl = 0; numfl < totnumfl; numfl++ )
{
// assuming 4 byte (32bit) float
float num = 0.0;
uint8_t* pnum = reinterpret_cast<uint8_t*>(&num);
// parse all 4 bytes of the number
for ( int byi = 0; byi < (int)sizeof(float); byi++ )
{
// TODO what's the byte order (little/big endian) in the file??
// for now, we just don't care...
pnum[byi] = (int)datbuf[(unsigned long int)(numfl*sizeof(float)+byi)];
}
// add number of array
datmes_.push_back((double)num);
}
}
// convert half-precision (16bit) floating point numbers
void convert_data_16_bit_float()
{
assert ( (datsize_-28)%2 == 0 && "length of buffer is not a multiple of 2" );
unsigned long int totnumby = (datsize_-28)/2;
for ( unsigned long int by = 0; by < totnumby; by++ )
{
// declare single (16bit) floating point number
half_float::half hfl;
// reinterpret bytes in buffer as memory of floating point number
uint8_t* pnum = reinterpret_cast<uint8_t*>(&hfl);
for ( int i = 0; i < (int)sizeof(half_float::half); i++ )
{
pnum[i] = (int)datasec_["datas marker"][(unsigned long int)(28+by*sizeof(half_float::half)+i)];
}
// add number to array
datmes_.push_back((double)hfl);
}
}
// convert 16bit "decimal-encoding" floating point numbers
void convert_data_16_bit_decimal(std::vector<unsigned char> &datbuf)
{
assert ( datbuf.size()%2 == 0 && "length of data is not a multiple of 2" );
// encoding parameters
double shift = -128.;
double scale = 1.0/100.;
double offse = 0.0;
for ( unsigned long int idx = 0; idx < datbuf.size()-1; idx += 2 )
{
// convert to float
datmes_.push_back(
(double)( (int)(datbuf[idx])*1. + ( (int)(datbuf[idx+1])*1. + shift )*256. )*scale + offse
);
}
}
//---------------------------------------------------------------------------//
// show hex dump
void show_hex(std::vector<unsigned char> &datavec, int width = 32, unsigned long int maxchars = 512)
{
@ -238,80 +360,18 @@ public:
std::cout<<std::dec;
}
// convert actual measurement data
void convert_data_32_bit_float()
{
assert ( (datsize_-28)%4 == 0 && "length of buffer is not a multiple of 4" );
unsigned long int totnumfl = (datsize_-28)/(int)sizeof(float);
for ( unsigned long int numfl = 0; numfl < totnumfl; numfl++ )
{
// assuming 4 byte (32bit) float
float num = 0.0;
uint8_t* pnum = reinterpret_cast<uint8_t*>(&num);
for ( int byi = 0; byi < (int)sizeof(float); byi++ )
{
// TODO what's the byte order in the file??
// for now, we just don't care...
pnum[byi] = (int)datasec_["datas marker"][(unsigned long int)(28+numfl*sizeof(float)+byi)];
}
// add number of array
datmes_.push_back((double)num);
}
}
// convert half-precision (16bit) floating point numbers
void convert_data_16_bit_float()
{
assert ( (datsize_-28)%2 == 0 && "length of buffer is not a multiple of 2" );
unsigned long int totnumby = (datsize_-28)/2;
for ( unsigned long int by = 0; by < totnumby; by++ )
{
// declare single (16bit) floating point number
half_float::half hfl;
// reinterpret bytes in buffer as memory of floating point number
uint8_t* pnum = reinterpret_cast<uint8_t*>(&hfl);
for ( int i = 0; i < (int)sizeof(half_float::half); i++ )
{
pnum[i] = (int)datasec_["datas marker"][(unsigned long int)(28+by*sizeof(half_float::half)+i)];
}
// add number to array
datmes_.push_back((double)hfl);
}
}
// convert 16bit "decimal-encoding" floating point numbers
void convert_data_16_bit_decimal()
{
assert ( (datsize_-datstartidx_)%2 == 0 && "length of data is not a multiple of 2" );
double flstp = 0.04395;
// parse bytes in data buffer
unsigned long int totnumby = (datsize_-datstartidx_)/2;
for ( unsigned long int by = 0; by < totnumby; by++ )
{
// retrieve set of two subsequent bytes
std::vector<uint8_t> pnum;
for ( int i = 0; i < 2; i++ ) pnum.push_back(datasec_["datas marker"][(unsigned long int)(datstartidx_+by*2+i)]);
// convert to double
//datmes_.push_back((double)( (((int)pnum[0]-128)*256 + (int)pnum[1])/100.0 ));
datmes_.push_back((double)( (((int)pnum[1]-128)*256. + (int)pnum[0])/100.0 ));
//datmes_.push_back( (double) ( ( (int)pnum[0] + (int)pnum[1]*256 )*flstp ) );
}
}
// get data array encoded as floats/doubles
std::vector<double>& get_data()
{
return datmes_;
}
// get segment's array of elements
std::vector<std::string> get_segment(std::string marker)
{
return segments_[marker];
}
// write data to csv-like file
void write_data(std::string filename, int precision = 9)
{