delete deprecated python example, add new python example dealing with absolute time stamp according to trigger-time
This commit is contained in:
Mario Fink
parent
b42a170650
commit
1a381f01b7
@ -1,137 +0,0 @@
|
|||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
import raw_eater
|
|
||||||
import raw_meat
|
|
||||||
import pyarrow as pa
|
|
||||||
import pyarrow.parquet as pq
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
fileobj1 = Path("samples/datasetA/").rglob("*.raw")
|
|
||||||
rawlist1 = [str(fl) for fl in fileobj1]
|
|
||||||
|
|
||||||
fileobj2 = Path("samples/datasetB/").rglob("*.raw")
|
|
||||||
rawlist2 = [str(fl) for fl in fileobj2]
|
|
||||||
|
|
||||||
rawlist = rawlist1 #[rawlist1[0],rawlist1[4],rawlist2[0],rawlist2[6]]
|
|
||||||
for fil in rawlist2 :
|
|
||||||
rawlist.append(fil)
|
|
||||||
rawlist.append("./README.md")
|
|
||||||
|
|
||||||
print("")
|
|
||||||
print(rawlist)
|
|
||||||
print()
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
# alternatively create "empty" instance of "raw_eater" and set file names
|
|
||||||
eatraw = raw_eater.raweater()
|
|
||||||
# eatraw.set_file("../smp/pressure_Vacuum.raw".encode())
|
|
||||||
|
|
||||||
# convert every single listed file
|
|
||||||
for rf in rawlist :
|
|
||||||
|
|
||||||
print("converting " + str(rf) + "...\n" + 90*("-") + "\n")
|
|
||||||
|
|
||||||
# setup instance of "raw_eater" and trigger conversion
|
|
||||||
# eatraw = raw_eater.raweater(rf.encode())
|
|
||||||
# eatraw = raw_meat.rawmerger(rf.encode())
|
|
||||||
|
|
||||||
# use global instance of "raw_eater" to set file and perform decoding
|
|
||||||
eatraw.set_file(rf.encode())
|
|
||||||
try :
|
|
||||||
eatraw.do_conversion()
|
|
||||||
except RuntimeError as e :
|
|
||||||
print("conversion failed: " + str(e))
|
|
||||||
|
|
||||||
# check validity of file format
|
|
||||||
if eatraw.validity() :
|
|
||||||
|
|
||||||
# show channel name and its unit
|
|
||||||
entity = eatraw.channel_name().decode(encoding='UTF-8',errors='ignore')
|
|
||||||
unit = eatraw.unit().decode(encoding='UTF-8',errors='ignore')
|
|
||||||
print("\nentity: " + str(entity))
|
|
||||||
print("unit: " + str(unit) + "\n")
|
|
||||||
|
|
||||||
# obtain extracted data
|
|
||||||
xt = eatraw.get_time()
|
|
||||||
yt = eatraw.get_channel()
|
|
||||||
|
|
||||||
# show excerpt of data
|
|
||||||
print("time (length: " + str(len(xt)) + ") \n"
|
|
||||||
+ str(xt[:10]) + "\n...\n" + str(xt[-10:]) + "\n")
|
|
||||||
yttrunc = [round(y,4) for y in yt]
|
|
||||||
print(str(entity) + " (length: " + str(len(yttrunc)) + ") \n"
|
|
||||||
+ str(yttrunc[:10]) + "\n...\n" + str(yttrunc[-10:]) + "\n")
|
|
||||||
|
|
||||||
outname = rf.split('/')[-1].replace('raw','csv')
|
|
||||||
|
|
||||||
print("write output to : " + outname)
|
|
||||||
eatraw.write_table(("output/"+outname).encode(),ord(' '))
|
|
||||||
|
|
||||||
else :
|
|
||||||
|
|
||||||
print("\nerror: invalid/corrupt .raw file")
|
|
||||||
|
|
||||||
print("\n")
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
print("convert and merge channels " + "\n" + 90*("-") + "\n")
|
|
||||||
|
|
||||||
# setup new instance to merge channels
|
|
||||||
eatmea = raw_meat.rawmerger(''.encode()) #rawlist[0].encode())
|
|
||||||
|
|
||||||
# add every single channel/file in list
|
|
||||||
for rf in rawlist :
|
|
||||||
print("\nadding channel " + str(rf))
|
|
||||||
try :
|
|
||||||
succ = eatmea.add_channel(rf.encode())
|
|
||||||
print("\nrecent time series: length: " + str(len(eatmea.get_time_series())) + "\n")
|
|
||||||
except RuntimeError as e :
|
|
||||||
print("failed to add channel: " + str(e))
|
|
||||||
|
|
||||||
# show summary of successfully merged channels
|
|
||||||
print("\nmerged channels:\n")
|
|
||||||
|
|
||||||
# write merged table to .csv output
|
|
||||||
eatmea.write_table_all('output/allchannels.csv'.encode(),ord(','))
|
|
||||||
|
|
||||||
# get number of successfully merged channels and their names (+units)
|
|
||||||
numch = eatmea.get_num_channels()
|
|
||||||
chnames = [chnm.decode(encoding='UTF-8',errors='ignore') for chnm in eatmea.get_channel_names()]
|
|
||||||
print("number of channels: " + str(numch))
|
|
||||||
print("channel names: " + str(chnames))
|
|
||||||
|
|
||||||
# obtain final time series
|
|
||||||
timse = eatmea.get_time_series()
|
|
||||||
print("\nfinal time series:\nlength:" + str(len(timse)) + "\n")
|
|
||||||
|
|
||||||
# get time unit and prepend column name
|
|
||||||
chnames.insert(0,"Time ["+str(eatmea.time_unit().decode(encoding='UTF-8',errors='ignore'))+"]")
|
|
||||||
|
|
||||||
# prepare list of pyarrow arrays
|
|
||||||
pyarrs = []
|
|
||||||
pyarrs.append(pa.array(timse))
|
|
||||||
|
|
||||||
for i in range(0,numch) :
|
|
||||||
print("\n" + str(i) + " " + str(chnames[i]))
|
|
||||||
dat = eatmea.get_channel_by_index(i)
|
|
||||||
print("length: " + str(len(dat)))
|
|
||||||
pyarrs.append(pa.array(dat))
|
|
||||||
print("")
|
|
||||||
# print("\npyarrow arrays\n" + str(pyarrs))
|
|
||||||
|
|
||||||
# create pyarrow table from data
|
|
||||||
pyarwtab = pa.Table.from_arrays(pyarrs,chnames)
|
|
||||||
print("\n" + 60*"-" + "\n" + str(pyarwtab) + "\n")
|
|
||||||
|
|
||||||
# write pyarrow table to .parquet file with compression
|
|
||||||
pq.write_table(pyarwtab,'output/allchannels.parquet',compression='BROTLI') # compression='BROTLI', 'SNAPPY')
|
|
||||||
|
|
||||||
# try to read and decode the .parquet file
|
|
||||||
df = pq.read_table('output/allchannels.parquet')
|
|
||||||
print(df.to_pandas())
|
|
||||||
# df.to_pandas().to_csv('allchannels.csv',index=False,encoding='utf-8',sep=",")
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
@ -1,24 +0,0 @@
|
|||||||
|
|
||||||
import pyarrow as pa
|
|
||||||
import numpy as np
|
|
||||||
import pyarrow.parquet as pq
|
|
||||||
|
|
||||||
db = pa.array(np.linspace(10,50,6))
|
|
||||||
print(db)
|
|
||||||
da = pa.array(np.linspace(0,5,6))
|
|
||||||
print(db)
|
|
||||||
|
|
||||||
filenam = 'pyarrow_testtab.parquet'
|
|
||||||
|
|
||||||
patab = pa.Table.from_arrays([da,db],['entity A [unitA]','entity B [unitB]'])
|
|
||||||
print(patab)
|
|
||||||
|
|
||||||
# pq.write_table(patab,filenam,compression='BROTLI')
|
|
||||||
pq.write_table(patab,filenam,compression='SNAPPY')
|
|
||||||
|
|
||||||
df = pq.read_table(filenam)
|
|
||||||
print(df)
|
|
||||||
print(df.to_pandas())
|
|
||||||
|
|
||||||
#import readline
|
|
||||||
#readline.write_history_file('generate_pyarrow_table_and_write_parquet.py')
|
|
||||||
50
python/usage_ext.py
Normal file
50
python/usage_ext.py
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
|
||||||
|
import imc_termite
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
# declare and initialize instance of "imctermite" by passing a raw-file
|
||||||
|
try :
|
||||||
|
imcraw = imc_termite.imctermite(b"samples/sampleB.raw")
|
||||||
|
except RuntimeError as e :
|
||||||
|
raise Exception("failed to load/parse raw-file: " + str(e))
|
||||||
|
|
||||||
|
# obtain list of channels as list of dictionaries (without data)
|
||||||
|
channels = imcraw.get_channels(False)
|
||||||
|
print(json.dumps(channels,indent=4, sort_keys=False))
|
||||||
|
|
||||||
|
# obtain all channels (including full data)
|
||||||
|
channelsdata = imcraw.get_channels(True)
|
||||||
|
|
||||||
|
# everything that follows is an example that specifically makes use only of
|
||||||
|
# the first (index = 0) channel ...
|
||||||
|
idx = 0
|
||||||
|
|
||||||
|
if len(channelsdata) > 0 :
|
||||||
|
|
||||||
|
# get first channel's data
|
||||||
|
chnydata = channelsdata[idx]['ydata']
|
||||||
|
chnxdata = channelsdata[idx]['xdata']
|
||||||
|
print("xdata: " + str(len(chnxdata)))
|
||||||
|
print("ydata: " + str(len(chnydata)))
|
||||||
|
|
||||||
|
# extract trigger-time
|
||||||
|
trigtim = datetime.datetime.fromisoformat(channels[idx]["trigger-time"])
|
||||||
|
print(trigtim)
|
||||||
|
|
||||||
|
# file output of data with absolute timestamp in 1st column
|
||||||
|
filname = os.path.join("./",channelsdata[idx]['name']+".csv")
|
||||||
|
print("writing output into " + filname)
|
||||||
|
with open(filname,'w') as fout :
|
||||||
|
# include column header
|
||||||
|
fout.write( str(channelsdata[idx]['xname']) + '[' + str(channelsdata[idx]['xunit']) + "]"
|
||||||
|
+ ","
|
||||||
|
+ str(channelsdata[idx]['yname']) + '[' + str(channelsdata[idx]['yunit']) + "]"
|
||||||
|
+ "\n" )
|
||||||
|
# add data (introduce time shift according to trigger-time)
|
||||||
|
for row in range(0,len(chnxdata)) :
|
||||||
|
fout.write( str( (trigtim + datetime.timedelta(seconds=chnxdata[row])).isoformat() )
|
||||||
|
+ ","
|
||||||
|
+ str( chnydata[row])
|
||||||
|
+ "\n" )
|
||||||
Loading…
x
Reference in New Issue
Block a user