fix bugs in lib/rawmerge.hpp: merge_channels(...) and improve example.py
- raweat.hpp: find_markers(): properly (re)adjust valid_ flag when doing multiple conversion - rawmerge.hpp: improve logging, merge_channels(): add condition to avoid range error in vector - example.py: obtain return flag from add_channel(), write csv output from parquet table - main.cpp: return to pure raw_eater test version
This commit is contained in:
Mario Fink
parent
8d92bce9f0
commit
0bd96a1426
@ -251,7 +251,7 @@ public:
|
||||
|
||||
// check validity of format
|
||||
// assert ( totalmarksize > 0 && "didn't find any predefined marker => probably not a valid .raw-file" );
|
||||
if ( totalmarksize < 100 ) valid_ = false;
|
||||
valid_ = ( totalmarksize < 100 ) ? false : true;
|
||||
}
|
||||
|
||||
// display content of found markers
|
||||
|
||||
@ -79,13 +79,13 @@ public:
|
||||
}
|
||||
|
||||
// add a single channel and its associated time series
|
||||
bool add_channel(std::string rawfile, bool log = false)
|
||||
bool add_channel(std::string rawfile, bool showlog = false)
|
||||
{
|
||||
// set raw file and perform conversion
|
||||
this->set_file(rawfile,false);
|
||||
this->set_file(rawfile,showlog);
|
||||
|
||||
// show channel name, unit, timestep, time unit, etc.
|
||||
if ( log )
|
||||
if ( showlog && this->get_valid() )
|
||||
{
|
||||
std::cout<<this->get_name()<<" ["<<this->get_unit()<<"]"<<"\n";
|
||||
std::cout<<"Time ["<<this->get_temp_unit()<<"]"<<"\n";
|
||||
@ -98,7 +98,7 @@ public:
|
||||
// add first/initial time series (and channel data)
|
||||
if ( this->get_valid() && timeseries_.size() == 0 && channels_.size() == 0 )
|
||||
{
|
||||
std::cout<<"adding initial channel "<<rawfile<<"\n\n";
|
||||
if ( showlog ) std::cout<<"adding initial channel "<<rawfile<<"\n\n";
|
||||
|
||||
// insert timeseries and its unit
|
||||
this->timeseries_ = this->get_time();
|
||||
@ -114,9 +114,9 @@ public:
|
||||
|
||||
return true;
|
||||
}
|
||||
else
|
||||
else if ( this->get_valid() )
|
||||
{
|
||||
std::cout<<"adding next channel "<<rawfile<<"\n\n";
|
||||
if ( showlog ) std::cout<<"adding next channel "<<rawfile<<"\n\n";
|
||||
|
||||
// check consistency of temporal unit
|
||||
if ( this->get_temp_unit() == this->temp_unit_ )
|
||||
@ -163,11 +163,19 @@ public:
|
||||
{
|
||||
// refuse to merge due to different temporal units
|
||||
std::cerr<<"rawmerge: add_channel '"<<rawfile
|
||||
<<"' : inconsistent time units\n";
|
||||
<<"' : inconsistent time units: '"
|
||||
<<this->get_temp_unit()<<"' versus '"<<this->temp_unit_<<"'\n";
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// provided file does not feature a valid .raw format
|
||||
std::cerr<<"rawmerge: add_channel '"<<rawfile<<"' : invalid .raw file\n";
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// merge new channel and associated time series with exisiting channels and
|
||||
@ -179,7 +187,7 @@ public:
|
||||
std::vector<double>& result_timeseries, // resulting timeseries ...
|
||||
std::vector<std::vector<double>>& result_channels, // ...and associated (n+1) channels
|
||||
double placeholder = std::numeric_limits<double>::quiet_NaN(),
|
||||
bool showlog = true)
|
||||
bool showlog = false)
|
||||
{
|
||||
if ( showlog )
|
||||
{
|
||||
@ -221,11 +229,19 @@ public:
|
||||
// process all time steps in both time series
|
||||
while ( idxCur < current_timeseries.size() || idxNew < new_timeseries.size() )
|
||||
{
|
||||
if ( showlog ) std::cout<<"idxCur "<<std::setw(6)<<idxCur
|
||||
<<std::setw(20)<<std::right<<current_timeseries[idxCur]<<"\n"
|
||||
<<"idxNew "<<std::setw(6)<<idxNew
|
||||
<<std::setw(20)<<std::right<<new_timeseries[idxNew]<<"\n";
|
||||
|
||||
// if point in time of "current_timeseries" is BEFORE time of "new_timeseries"
|
||||
// or "new_timeseries" is depleted
|
||||
if ( current_timeseries[idxCur] + 1.0e-10 < new_timeseries[idxNew]
|
||||
|| idxNew == new_timeseries.size() )
|
||||
if ( idxCur < current_timeseries.size() &&
|
||||
( current_timeseries[idxCur] + 1.0e-10 < new_timeseries[idxNew]
|
||||
|| idxNew == new_timeseries.size() ) )
|
||||
{
|
||||
if ( showlog ) std::cout<<"push_back A\n";
|
||||
|
||||
// keep current data as it is ...
|
||||
for ( unsigned long int ch = 0; ch < numchannels; ch++ )
|
||||
{
|
||||
@ -239,9 +255,12 @@ public:
|
||||
idxCur++;
|
||||
}
|
||||
// ...just reversed...
|
||||
else if ( current_timeseries[idxCur] > new_timeseries[idxNew] + 1.0e-10
|
||||
|| idxCur == current_timeseries.size() )
|
||||
else if ( idxNew < new_timeseries.size() &&
|
||||
( current_timeseries[idxCur] > new_timeseries[idxNew] + 1.0e-10
|
||||
|| idxCur == current_timeseries.size() ) )
|
||||
{
|
||||
if ( showlog ) std::cout<<"push_back B\n";
|
||||
|
||||
// insert placeholders for all exisiting channels...
|
||||
for ( unsigned long int ch = 0; ch < numchannels; ch++ )
|
||||
{
|
||||
@ -257,6 +276,8 @@ public:
|
||||
// ...points in time of both timeseries match...
|
||||
else
|
||||
{
|
||||
if ( showlog ) std::cout<<"push_back C\n";
|
||||
|
||||
// add ALL, i.e. both current and new data to result
|
||||
for ( unsigned long int ch = 0; ch < numchannels; ch++ )
|
||||
{
|
||||
|
||||
@ -6,7 +6,8 @@ import raw_meat
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
rawlist = [ "smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePedalActiveQF_HS.raw",
|
||||
rawlist = [
|
||||
"smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePedalActiveQF_HS.raw",
|
||||
"smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePressure_HS.raw",
|
||||
"smp/Rangerover_Evoque_F-RR534_2019-05-07/EngineSpeed_HS.raw",
|
||||
"smp/Rangerover_Evoque_F-RR534_2019-05-07/pressure_FL.raw",
|
||||
@ -16,7 +17,8 @@ rawlist = [ "smp/Rangerover_Evoque_F-RR534_2019-05-07/BrakePedalActiveQF_HS.raw"
|
||||
"smp/Rangerover_Evoque_F-RR534_2019-05-07/ABS_A_Port1.raw",
|
||||
"./pyt/example.py",
|
||||
"smp/Rangerover_Evoque_F-RR534_2019-05-07/LateralAcceleration_HS.raw",
|
||||
"smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw" ]
|
||||
"smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw"
|
||||
]
|
||||
|
||||
print("")
|
||||
|
||||
@ -35,7 +37,7 @@ for rf in rawlist :
|
||||
if eatraw.validity() :
|
||||
|
||||
# show channel name and its unit
|
||||
entity = eatraw.channel_name().decode()
|
||||
entity = eatraw.channel_name().decode(encoding='UTF-8',errors='ignore')
|
||||
unit = eatraw.unit().decode(encoding='UTF-8',errors='ignore')
|
||||
print("\nentity: " + str(entity))
|
||||
print("unit: " + str(unit) + "\n")
|
||||
@ -64,25 +66,35 @@ for rf in rawlist :
|
||||
|
||||
#-----------------------------------------------------------------------------#
|
||||
|
||||
print("convert and merge channels " + "\n" + 90*("-") + "\n")
|
||||
|
||||
# setup new instance to merge channels
|
||||
eatmea = raw_meat.rawmerger(rawlist[0].encode())
|
||||
|
||||
# add every single channel/file in list
|
||||
for rf in rawlist :
|
||||
print("\nadding channel " + str(rf))
|
||||
eatmea.add_channel(rf.encode())
|
||||
succ = eatmea.add_channel(rf.encode())
|
||||
if succ :
|
||||
print("\nrecent time series: length: " + str(len(eatmea.get_time_series())) + "\n")
|
||||
else :
|
||||
print("\nfailed to add channel\n")
|
||||
|
||||
# show summary of successfully merged channels
|
||||
print("\nmerged channels:\n")
|
||||
print("number of channels: " + str(eatmea.get_num_channels()))
|
||||
print("channel names: " + str(eatmea.get_channel_names()))
|
||||
|
||||
# get number of successfully merged channels and their names (+units)
|
||||
numch = eatmea.get_num_channels()
|
||||
chnames = eatmea.get_channel_names()
|
||||
chnames = [chnm.decode(encoding='UTF-8',errors='ignore') for chnm in eatmea.get_channel_names()]
|
||||
print("number of channels: " + str(numch))
|
||||
print("channel names: " + str(chnames))
|
||||
|
||||
# obtain final time series
|
||||
timse = eatmea.get_time_series()
|
||||
print("\nfinal time series:\nlength:" + str(len(timse)) + "\n")
|
||||
|
||||
# get time unit and prepend column name
|
||||
chnames.insert(0,"Time ["+str(eatmea.time_unit())+"]")
|
||||
chnames.insert(0,"Time ["+str(eatmea.time_unit().decode(encoding='UTF-8',errors='ignore'))+"]")
|
||||
|
||||
# prepare list of pyarrow arrays
|
||||
pyarrs = []
|
||||
@ -93,17 +105,19 @@ for i in range(0,numch) :
|
||||
dat = eatmea.get_channel_by_index(i)
|
||||
print("length: " + str(len(dat)))
|
||||
pyarrs.append(pa.array(dat))
|
||||
|
||||
print("")
|
||||
# print("\npyarrow arrays\n" + str(pyarrs))
|
||||
|
||||
# prepare pyarrow table from data
|
||||
# create pyarrow table from data
|
||||
pyarwtab = pa.Table.from_arrays(pyarrs,chnames)
|
||||
print(pyarwtab)
|
||||
print("\n" + 60*"-" + "\n" + str(pyarwtab) + "\n")
|
||||
|
||||
# write pyarrow table to .parquet file with compression
|
||||
pq.write_table(pyarwtab,'allchannels.parquet',compression='BROTLI') # compression='BROTLI', 'SNAPPY')
|
||||
|
||||
# try to read and decode the .parquet file
|
||||
df = pq.read_table('allchannels.parquet')
|
||||
print(df)
|
||||
print(df.to_pandas())
|
||||
df.to_pandas().to_csv('allchannels.csv',index=False,encoding='utf-8',sep=",")
|
||||
|
||||
#-----------------------------------------------------------------------------#
|
||||
|
||||
19
src/main.cpp
19
src/main.cpp
@ -30,14 +30,14 @@ int main(int argc, char* argv[])
|
||||
std::string rawfile(argv[1]);
|
||||
|
||||
// declare instance of "raw_eater"
|
||||
// raw_eater eatraw(rawfile,true);
|
||||
raw_eater eatraw(rawfile,true);
|
||||
|
||||
// declare instance of "raw_merger"
|
||||
raw_merger eatmea;
|
||||
eatmea.add_channel(rawfile,true);
|
||||
eatmea.add_channel("smp/VehicleSpeed_HS.raw",true);
|
||||
eatmea.add_channel("smp/VehicleSpeed_HS.raw",true);
|
||||
eatmea.add_channel("smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw",true);
|
||||
// raw_merger eatmea;
|
||||
// eatmea.add_channel(rawfile,true);
|
||||
// eatmea.add_channel("smp/VehicleSpeed_HS.raw",true);
|
||||
// eatmea.add_channel("smp/VehicleSpeed_HS.raw",true);
|
||||
// eatmea.add_channel("smp/Rangerover_Evoque_F-RR534_2019-05-07/Temp_Disc_FR.raw",true);
|
||||
|
||||
//eatraw.show_markers();
|
||||
|
||||
@ -65,10 +65,13 @@ int main(int argc, char* argv[])
|
||||
// for ( unsigned long int i = 0; i < 10; i++ ) std::cout<<mydata[i]<<"\n";
|
||||
|
||||
// write data in csv-file
|
||||
if ( eatraw.get_valid() )
|
||||
{
|
||||
// eatraw.write_table(std::string(argv[2]));
|
||||
// eatraw.write_table(std::string(argv[2]),' ');
|
||||
eatraw.write_table(std::string(argv[2]),' ');
|
||||
// eatmea.write_table(std::string(argv[2]));
|
||||
eatmea.write_table(std::string(argv[2]),' ');
|
||||
// eatmea.write_table(std::string(argv[2]),' ');
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user