* clean up repository: remove cython/ pip/ parquet/ assets/
* update python module MANIFEST.in to include sources
This commit is contained in:
parent
e6315ee186
commit
2d654d2d2a
@ -1,111 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
|
||||||
<svg
|
|
||||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
|
||||||
xmlns:cc="http://creativecommons.org/ns#"
|
|
||||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
|
||||||
xmlns:svg="http://www.w3.org/2000/svg"
|
|
||||||
xmlns="http://www.w3.org/2000/svg"
|
|
||||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
|
||||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
|
||||||
height="78.080002"
|
|
||||||
width="325.72"
|
|
||||||
id="svg3945"
|
|
||||||
version="1.1"
|
|
||||||
viewBox="0 0 325.72 78.08"
|
|
||||||
sodipodi:docname="imctermite.svg"
|
|
||||||
inkscape:version="1.0.1 (3bc2e813f5, 2020-09-07)">
|
|
||||||
<sodipodi:namedview
|
|
||||||
pagecolor="#ffffff"
|
|
||||||
bordercolor="#666666"
|
|
||||||
borderopacity="1"
|
|
||||||
objecttolerance="10"
|
|
||||||
gridtolerance="10"
|
|
||||||
guidetolerance="10"
|
|
||||||
inkscape:pageopacity="0"
|
|
||||||
inkscape:pageshadow="2"
|
|
||||||
inkscape:window-width="2048"
|
|
||||||
inkscape:window-height="1088"
|
|
||||||
id="namedview18"
|
|
||||||
showgrid="false"
|
|
||||||
inkscape:zoom="2.5577614"
|
|
||||||
inkscape:cx="138.13369"
|
|
||||||
inkscape:cy="-27.086877"
|
|
||||||
inkscape:window-x="0"
|
|
||||||
inkscape:window-y="32"
|
|
||||||
inkscape:window-maximized="1"
|
|
||||||
inkscape:current-layer="svg3945"
|
|
||||||
inkscape:document-rotation="0" />
|
|
||||||
<metadata
|
|
||||||
id="metadata3951">
|
|
||||||
<rdf:RDF>
|
|
||||||
<cc:Work
|
|
||||||
rdf:about="">
|
|
||||||
<dc:format>image/svg+xml</dc:format>
|
|
||||||
<dc:type
|
|
||||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
|
||||||
<dc:title>flasher</dc:title>
|
|
||||||
</cc:Work>
|
|
||||||
</rdf:RDF>
|
|
||||||
</metadata>
|
|
||||||
<defs
|
|
||||||
id="defs3949" />
|
|
||||||
<title
|
|
||||||
id="title3916">flasher</title>
|
|
||||||
<g
|
|
||||||
id="logog"
|
|
||||||
transform="translate(0,0.99981694)">
|
|
||||||
<path
|
|
||||||
style="fill:#364d5c"
|
|
||||||
transform="translate(-2.04,-1.15)"
|
|
||||||
d="m 32.86,2 -13,7.5 v 0 h -0.05 v 0 l -0.48,0.28 c -4.27,2.46 -5.68,11.38 -6.06,14.75 L 36.2,11.33 c 0.39,-0.19 7.6,-3.69 13.57,-3.69 h 0.14 L 40.13,2 a 8.15,8.15 0 0 0 -7.27,0"
|
|
||||||
id="path138"
|
|
||||||
inkscape:connector-curvature="0" />
|
|
||||||
<path
|
|
||||||
style="fill:#364d5c"
|
|
||||||
transform="translate(-2.04,-1.15)"
|
|
||||||
d="M 5.68,17.69 A 8.2,8.2 0 0 0 2,24 v 15.78 c 0,4.9 7,10.48 9.75,12.46 V 25.77 c 0,-0.44 0.6,-8.55 3.65,-13.72 z"
|
|
||||||
id="path142"
|
|
||||||
inkscape:connector-curvature="0" />
|
|
||||||
<path
|
|
||||||
style="fill:#364d5c"
|
|
||||||
transform="translate(-2.04,-1.15)"
|
|
||||||
d="m 12.1,54.12 v 0 C 11.74,53.88 5,49.41 2,44.24 v 11.14 a 8.2,8.2 0 0 0 3.64,6.3 l 13.5,7.79 c 4.28,2.46 12.7,-0.77 15.81,-2.12 z"
|
|
||||||
id="path146"
|
|
||||||
inkscape:connector-curvature="0" />
|
|
||||||
<path
|
|
||||||
style="fill:#364d5c"
|
|
||||||
transform="translate(-2.04,-1.15)"
|
|
||||||
d="m 36.79,68 c -0.4,0.19 -7.71,3.75 -13.71,3.69 l 9.78,5.64 a 8.15,8.15 0 0 0 7.27,0 l 13.51,-7.8 c 4.27,-2.46 5.68,-11.39 6.06,-14.75 z"
|
|
||||||
id="path150"
|
|
||||||
inkscape:connector-curvature="0" />
|
|
||||||
<path
|
|
||||||
style="fill:#364d5c"
|
|
||||||
transform="translate(-2.04,-1.15)"
|
|
||||||
d="M 61.2,27.13 V 53.6 c 0,0.44 -0.6,8.55 -3.65,13.72 l 9.77,-5.64 A 8.2,8.2 0 0 0 71,55.38 V 39.59 c 0,-4.94 -7,-10.5 -9.75,-12.46"
|
|
||||||
id="path154"
|
|
||||||
inkscape:connector-curvature="0" />
|
|
||||||
<path
|
|
||||||
style="fill:#364d5c"
|
|
||||||
transform="translate(-2.04,-1.15)"
|
|
||||||
d="M 67.31,17.69 53.81,9.9 C 49.53,7.44 41.11,10.67 38,12 l 22.85,13.23 v 0 a 43.43,43.43 0 0 1 5.7,4.51 24,24 0 0 1 4.45,5.35 V 24 a 8.2,8.2 0 0 0 -3.64,-6.3"
|
|
||||||
id="path158"
|
|
||||||
inkscape:connector-curvature="0" />
|
|
||||||
</g>
|
|
||||||
<g
|
|
||||||
id="re"
|
|
||||||
transform="translate(0,0.99981694)" />
|
|
||||||
<text
|
|
||||||
id="text3955"
|
|
||||||
y="55.47554"
|
|
||||||
x="74.101189"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:40px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#364d5c;fill-opacity:1;stroke:none"
|
|
||||||
xml:space="preserve"><tspan
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:44px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#364d5c;fill-opacity:1"
|
|
||||||
y="55.47554"
|
|
||||||
x="74.101189"
|
|
||||||
id="tspan3953"><tspan
|
|
||||||
id="tspan24"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold'">IMC</tspan><tspan
|
|
||||||
id="tspan3845"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:sans-serif;-inkscape-font-specification:sans-serif">termite</tspan> </tspan></text>
|
|
||||||
</svg>
|
|
||||||
|
Before Width: | Height: | Size: 4.5 KiB |
@ -1,112 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
|
||||||
<svg
|
|
||||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
|
||||||
xmlns:cc="http://creativecommons.org/ns#"
|
|
||||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
|
||||||
xmlns:svg="http://www.w3.org/2000/svg"
|
|
||||||
xmlns="http://www.w3.org/2000/svg"
|
|
||||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
|
||||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
|
||||||
height="78.080002"
|
|
||||||
width="290.72"
|
|
||||||
id="svg3945"
|
|
||||||
version="1.1"
|
|
||||||
viewBox="0 0 290.72 78.08"
|
|
||||||
sodipodi:docname="raweater.svg"
|
|
||||||
inkscape:version="0.92.5 (2060ec1f9f, 2020-04-08)">
|
|
||||||
<sodipodi:namedview
|
|
||||||
pagecolor="#ffffff"
|
|
||||||
bordercolor="#666666"
|
|
||||||
borderopacity="1"
|
|
||||||
objecttolerance="10"
|
|
||||||
gridtolerance="10"
|
|
||||||
guidetolerance="10"
|
|
||||||
inkscape:pageopacity="0"
|
|
||||||
inkscape:pageshadow="2"
|
|
||||||
inkscape:window-width="1360"
|
|
||||||
inkscape:window-height="704"
|
|
||||||
id="namedview18"
|
|
||||||
showgrid="false"
|
|
||||||
inkscape:zoom="0.90430522"
|
|
||||||
inkscape:cx="191.86"
|
|
||||||
inkscape:cy="38.540001"
|
|
||||||
inkscape:window-x="0"
|
|
||||||
inkscape:window-y="27"
|
|
||||||
inkscape:window-maximized="1"
|
|
||||||
inkscape:current-layer="svg3945" />
|
|
||||||
<metadata
|
|
||||||
id="metadata3951">
|
|
||||||
<rdf:RDF>
|
|
||||||
<cc:Work
|
|
||||||
rdf:about="">
|
|
||||||
<dc:format>image/svg+xml</dc:format>
|
|
||||||
<dc:type
|
|
||||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
|
||||||
<dc:title>flasher</dc:title>
|
|
||||||
</cc:Work>
|
|
||||||
</rdf:RDF>
|
|
||||||
</metadata>
|
|
||||||
<defs
|
|
||||||
id="defs3949" />
|
|
||||||
<title
|
|
||||||
id="title3916">flasher</title>
|
|
||||||
<g
|
|
||||||
id="logog"
|
|
||||||
transform="translate(0,0.99981694)">
|
|
||||||
<path
|
|
||||||
style="fill:#364d5c"
|
|
||||||
transform="translate(-2.04,-1.15)"
|
|
||||||
d="m 32.86,2 -13,7.5 v 0 h -0.05 v 0 l -0.48,0.28 c -4.27,2.46 -5.68,11.38 -6.06,14.75 L 36.2,11.33 c 0.39,-0.19 7.6,-3.69 13.57,-3.69 h 0.14 L 40.13,2 a 8.15,8.15 0 0 0 -7.27,0"
|
|
||||||
id="path138"
|
|
||||||
inkscape:connector-curvature="0" />
|
|
||||||
<path
|
|
||||||
style="fill:#364d5c"
|
|
||||||
transform="translate(-2.04,-1.15)"
|
|
||||||
d="M 5.68,17.69 A 8.2,8.2 0 0 0 2,24 v 15.78 c 0,4.9 7,10.48 9.75,12.46 V 25.77 c 0,-0.44 0.6,-8.55 3.65,-13.72 z"
|
|
||||||
id="path142"
|
|
||||||
inkscape:connector-curvature="0" />
|
|
||||||
<path
|
|
||||||
style="fill:#364d5c"
|
|
||||||
transform="translate(-2.04,-1.15)"
|
|
||||||
d="m 12.1,54.12 v 0 C 11.74,53.88 5,49.41 2,44.24 v 11.14 a 8.2,8.2 0 0 0 3.64,6.3 l 13.5,7.79 c 4.28,2.46 12.7,-0.77 15.81,-2.12 z"
|
|
||||||
id="path146"
|
|
||||||
inkscape:connector-curvature="0" />
|
|
||||||
<path
|
|
||||||
style="fill:#364d5c"
|
|
||||||
transform="translate(-2.04,-1.15)"
|
|
||||||
d="m 36.79,68 c -0.4,0.19 -7.71,3.75 -13.71,3.69 l 9.78,5.64 a 8.15,8.15 0 0 0 7.27,0 l 13.51,-7.8 c 4.27,-2.46 5.68,-11.39 6.06,-14.75 z"
|
|
||||||
id="path150"
|
|
||||||
inkscape:connector-curvature="0" />
|
|
||||||
<path
|
|
||||||
style="fill:#364d5c"
|
|
||||||
transform="translate(-2.04,-1.15)"
|
|
||||||
d="M 61.2,27.13 V 53.6 c 0,0.44 -0.6,8.55 -3.65,13.72 l 9.77,-5.64 A 8.2,8.2 0 0 0 71,55.38 V 39.59 c 0,-4.94 -7,-10.5 -9.75,-12.46"
|
|
||||||
id="path154"
|
|
||||||
inkscape:connector-curvature="0" />
|
|
||||||
<path
|
|
||||||
style="fill:#364d5c"
|
|
||||||
transform="translate(-2.04,-1.15)"
|
|
||||||
d="M 67.31,17.69 53.81,9.9 C 49.53,7.44 41.11,10.67 38,12 l 22.85,13.23 v 0 a 43.43,43.43 0 0 1 5.7,4.51 24,24 0 0 1 4.45,5.35 V 24 a 8.2,8.2 0 0 0 -3.64,-6.3"
|
|
||||||
id="path158"
|
|
||||||
inkscape:connector-curvature="0" />
|
|
||||||
</g>
|
|
||||||
<g
|
|
||||||
id="re"
|
|
||||||
transform="translate(0,0.99981694)" />
|
|
||||||
<text
|
|
||||||
id="text3955"
|
|
||||||
y="55.47554"
|
|
||||||
x="74.101189"
|
|
||||||
style="font-style:normal;font-weight:normal;font-size:40px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#364d5c;fill-opacity:1;stroke:none"
|
|
||||||
xml:space="preserve"><tspan
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:44px;font-family:sans-serif;-inkscape-font-specification:'sans-serif, Bold';font-variant-ligatures:normal;font-variant-caps:normal;font-variant-numeric:normal;font-feature-settings:normal;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#364d5c;fill-opacity:1"
|
|
||||||
y="55.47554"
|
|
||||||
x="74.101189"
|
|
||||||
id="tspan3953"><tspan
|
|
||||||
id="tspan24"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold'">R</tspan><tspan
|
|
||||||
id="tspan3845"
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-family:sans-serif;-inkscape-font-specification:sans-serif">aw<tspan
|
|
||||||
style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold'"
|
|
||||||
id="tspan22">E</tspan>ater</tspan> </tspan></text>
|
|
||||||
</svg>
|
|
||||||
|
Before Width: | Height: | Size: 4.6 KiB |
@ -1,20 +0,0 @@
|
|||||||
# cython: language_level = 3
|
|
||||||
|
|
||||||
# use some C++ STL libraries
|
|
||||||
from libcpp.string cimport string
|
|
||||||
from libcpp.vector cimport vector
|
|
||||||
from libcpp cimport bool
|
|
||||||
|
|
||||||
cdef extern from "imc_raw.hpp" namespace "imc":
|
|
||||||
cdef cppclass imc_termite "imc::raw":
|
|
||||||
# constructor(s)
|
|
||||||
imc_termite() except +
|
|
||||||
imc_termite(string rawfile) except +
|
|
||||||
# provide raw file
|
|
||||||
void set_file(string rawfile) except +
|
|
||||||
# get JSON list of channels
|
|
||||||
vector[string] get_channels(bool json, bool data) except +
|
|
||||||
# print single channel/all channels
|
|
||||||
void print_channel(string channeluuid, string outputdir, char delimiter) except +
|
|
||||||
void print_channels(string outputdir, char delimiter) except +
|
|
||||||
void print_table(string outputfile) except +
|
|
||||||
@ -1,44 +0,0 @@
|
|||||||
# distutils: language = c++
|
|
||||||
|
|
||||||
from imc_termite cimport imc_termite
|
|
||||||
|
|
||||||
import json as jn
|
|
||||||
import decimal
|
|
||||||
# import numpy as np
|
|
||||||
|
|
||||||
cdef class imctermite:
|
|
||||||
|
|
||||||
# C++ instance of class => stack allocated (requires nullary constructor!)
|
|
||||||
cdef imc_termite cpp_imc
|
|
||||||
|
|
||||||
# constructor
|
|
||||||
def __cinit__(self, string rawfile):
|
|
||||||
self.cpp_imc = imc_termite(rawfile)
|
|
||||||
|
|
||||||
# provide raw file
|
|
||||||
def submit_file(self,string rawfile):
|
|
||||||
self.cpp_imc.set_file(rawfile)
|
|
||||||
|
|
||||||
# get JSON list of channels
|
|
||||||
def get_channels(self, bool data):
|
|
||||||
chnlst = self.cpp_imc.get_channels(True,data)
|
|
||||||
chnlstjn = [jn.loads(chn.decode(errors="ignore")) for chn in chnlst]
|
|
||||||
return chnlstjn
|
|
||||||
|
|
||||||
# print single channel/all channels
|
|
||||||
def print_channel(self, string channeluuid, string outputfile, char delimiter):
|
|
||||||
self.cpp_imc.print_channel(channeluuid,outputfile,delimiter)
|
|
||||||
def print_channels(self, string outputdir, char delimiter):
|
|
||||||
self.cpp_imc.print_channels(outputdir,delimiter)
|
|
||||||
|
|
||||||
# print table including channels
|
|
||||||
def print_table(self, string outputfile):
|
|
||||||
chnlst = self.cpp_imc.get_channels(True,True)
|
|
||||||
chnlstjn = [jn.loads(chn.decode(errors="ignore")) for chn in chnlst]
|
|
||||||
with open(outputfile.decode(),'w') as fout:
|
|
||||||
for chn in chnlstjn:
|
|
||||||
fout.write('#' +str(chn['xname']).rjust(19)+str(chn['yname']).rjust(20)+'\n')
|
|
||||||
fout.write('#'+str(chn['xunit']).rjust(19)+str(chn['yunit']).rjust(20)+'\n')
|
|
||||||
for n in range(0,len(chn['ydata'])):
|
|
||||||
fout.write(str(chn['xdata'][n]).rjust(20)+
|
|
||||||
str(chn['ydata'][n]).rjust(20)+'\n')
|
|
||||||
@ -1,41 +0,0 @@
|
|||||||
# cython: language_level = 3
|
|
||||||
# distutils: language = c++
|
|
||||||
|
|
||||||
# use some C++ STL libraries
|
|
||||||
from libcpp.string cimport string
|
|
||||||
from libcpp.vector cimport vector
|
|
||||||
from libcpp cimport bool
|
|
||||||
|
|
||||||
# to include implemenation/definition file
|
|
||||||
#cdef extern from "raweat.cpp":
|
|
||||||
# pass
|
|
||||||
|
|
||||||
# these method names have to match the C definitions of the methods!!
|
|
||||||
#
|
|
||||||
# for how to overload the constructor see
|
|
||||||
# https://cython.readthedocs.io/en/latest/src/userguide/wrapping_CPlusPlus.html
|
|
||||||
# and propagating exceptions from C++ to Python
|
|
||||||
# http://docs.cython.org/en/latest/src/userguide/wrapping_CPlusPlus.html#exceptions
|
|
||||||
|
|
||||||
cdef extern from "../lib/raweat.hpp":
|
|
||||||
cdef cppclass raw_eater:
|
|
||||||
# constructor(s)
|
|
||||||
raw_eater() except +
|
|
||||||
raw_eater(string) except +
|
|
||||||
# set new file for decoding
|
|
||||||
void set_file(string)
|
|
||||||
# perform conversion (pass any C++ exceptions to Python)
|
|
||||||
void setup_and_conversion() except +
|
|
||||||
# get validity of data format
|
|
||||||
bool get_valid()
|
|
||||||
# get channel name and unit
|
|
||||||
string get_name()
|
|
||||||
string get_unit()
|
|
||||||
# get time step and time unit
|
|
||||||
double get_dt()
|
|
||||||
string get_temp_unit()
|
|
||||||
# get data array of time and measured quantity's channel
|
|
||||||
vector[double] get_time()
|
|
||||||
vector[double] get_data()
|
|
||||||
# dump all data to .csv
|
|
||||||
void write_table(const char*,char delimiter)
|
|
||||||
@ -1,58 +0,0 @@
|
|||||||
|
|
||||||
from raw_eater cimport raweater
|
|
||||||
import numpy as np
|
|
||||||
import re
|
|
||||||
import os
|
|
||||||
|
|
||||||
cdef class raweater:
|
|
||||||
|
|
||||||
# C++ instance of class => stack allocated (requires nullary constructor!)
|
|
||||||
cdef raw_eater rawit
|
|
||||||
|
|
||||||
# pointer to C++ instance (if there's no nullary constructor)
|
|
||||||
# cdef raw_eater *rawit
|
|
||||||
|
|
||||||
def __cinit__(self, string rawfile = b''):
|
|
||||||
if rawfile.decode() == "":
|
|
||||||
self.rawit = raw_eater()
|
|
||||||
# self.rawit = new raw_eater()
|
|
||||||
else:
|
|
||||||
if not os.path.isfile(rawfile) :
|
|
||||||
raise ValueError("'" + str(rawfile) + "' does not exist")
|
|
||||||
self.rawit = raw_eater(rawfile)
|
|
||||||
# self.rawit = new raw_eater(rawfile)
|
|
||||||
|
|
||||||
# def __dealloc__(self):
|
|
||||||
# del self.rawit
|
|
||||||
|
|
||||||
def set_file(self, string rawfile):
|
|
||||||
if not os.path.isfile(rawfile) :
|
|
||||||
raise ValueError("'" + str(rawfile) + "' does not exist")
|
|
||||||
self.rawit.set_file(rawfile)
|
|
||||||
|
|
||||||
def do_conversion(self):
|
|
||||||
self.rawit.setup_and_conversion()
|
|
||||||
|
|
||||||
def validity(self):
|
|
||||||
return self.rawit.get_valid()
|
|
||||||
|
|
||||||
def channel_name(self):
|
|
||||||
return self.rawit.get_name()
|
|
||||||
|
|
||||||
def unit(self):
|
|
||||||
return self.rawit.get_unit()
|
|
||||||
|
|
||||||
def dt(self):
|
|
||||||
return self.rawit.get_dt()
|
|
||||||
|
|
||||||
def time_unit(self):
|
|
||||||
return self.rawit.get_temp_unit()
|
|
||||||
|
|
||||||
def get_time(self):
|
|
||||||
return self.rawit.get_time()
|
|
||||||
|
|
||||||
def get_channel(self):
|
|
||||||
return self.rawit.get_data()
|
|
||||||
|
|
||||||
def write_table(self, const char* csvfile, char delimiter):
|
|
||||||
self.rawit.write_table(csvfile,delimiter)
|
|
||||||
@ -1,37 +0,0 @@
|
|||||||
# cython: language_level = 3
|
|
||||||
# distutils: language = c++
|
|
||||||
|
|
||||||
# use some C++ STL libraries
|
|
||||||
from libcpp.string cimport string
|
|
||||||
from libcpp.vector cimport vector
|
|
||||||
from libcpp cimport bool
|
|
||||||
|
|
||||||
# these method names have to match the C++ definitions of the methods!!
|
|
||||||
cdef extern from "../lib/rawmerge.hpp":
|
|
||||||
cdef cppclass raw_merger:
|
|
||||||
raw_merger(string) except +
|
|
||||||
# get validity of data format
|
|
||||||
bool get_valid()
|
|
||||||
# get channel name and unit
|
|
||||||
string get_name()
|
|
||||||
string get_unit()
|
|
||||||
# get time step and time unit
|
|
||||||
double get_dt()
|
|
||||||
string get_temp_unit()
|
|
||||||
# get data array of time and measured quantity's channel
|
|
||||||
vector[double] get_time()
|
|
||||||
vector[double] get_data()
|
|
||||||
# dump all data to .csv
|
|
||||||
void write_table(const char*,char)
|
|
||||||
# add channel and try to merge it (pass C++ exceptions to Python)
|
|
||||||
bool add_channel(string) except +
|
|
||||||
# get total number of (added) channels
|
|
||||||
int get_num_channels()
|
|
||||||
# get list of channel names
|
|
||||||
vector[string] get_channel_names()
|
|
||||||
# get data of particular channel
|
|
||||||
vector[double] get_channel(int)
|
|
||||||
# get total merged time series
|
|
||||||
vector[double] get_time_series()
|
|
||||||
# dump all channels to .csv
|
|
||||||
void write_table_all(const char*,char)
|
|
||||||
@ -1,58 +0,0 @@
|
|||||||
|
|
||||||
# from <raw_meat> has to match name of .pxd file and cimport name of class defined in .pxd
|
|
||||||
from raw_meat cimport raw_merger
|
|
||||||
import numpy as np
|
|
||||||
import re
|
|
||||||
|
|
||||||
cdef class rawmerger:
|
|
||||||
|
|
||||||
# pointer to C++ instance (since there's no nullary constructor)
|
|
||||||
cdef raw_merger *rawit
|
|
||||||
|
|
||||||
def __cinit__(self, string rawfile):
|
|
||||||
self.rawit = new raw_merger(rawfile)
|
|
||||||
|
|
||||||
def __dealloc__(self):
|
|
||||||
del self.rawit
|
|
||||||
|
|
||||||
def validity(self):
|
|
||||||
return self.rawit.get_valid()
|
|
||||||
|
|
||||||
def channel_name(self):
|
|
||||||
return self.rawit.get_name()
|
|
||||||
|
|
||||||
def unit(self):
|
|
||||||
return self.rawit.get_unit()
|
|
||||||
|
|
||||||
def dt(self):
|
|
||||||
return self.rawit.get_dt()
|
|
||||||
|
|
||||||
def time_unit(self):
|
|
||||||
return self.rawit.get_temp_unit()
|
|
||||||
|
|
||||||
def get_time(self):
|
|
||||||
return self.rawit.get_time()
|
|
||||||
|
|
||||||
def get_channel(self):
|
|
||||||
return self.rawit.get_data()
|
|
||||||
|
|
||||||
def write_table(self, const char* csvfile, char delimiter):
|
|
||||||
return self.rawit.write_table(csvfile,delimiter)
|
|
||||||
|
|
||||||
def add_channel(self, string rawfile):
|
|
||||||
return self.rawit.add_channel(rawfile)
|
|
||||||
|
|
||||||
def get_num_channels(self):
|
|
||||||
return self.rawit.get_num_channels()
|
|
||||||
|
|
||||||
def get_channel_names(self):
|
|
||||||
return self.rawit.get_channel_names()
|
|
||||||
|
|
||||||
def get_channel_by_index(self, int chidx):
|
|
||||||
return self.rawit.get_channel(chidx)
|
|
||||||
|
|
||||||
def get_time_series(self):
|
|
||||||
return self.rawit.get_time_series()
|
|
||||||
|
|
||||||
def write_table_all(self, const char* csvfile, char delimiter):
|
|
||||||
return self.rawit.write_table_all(csvfile,delimiter)
|
|
||||||
@ -1,46 +0,0 @@
|
|||||||
from distutils.core import setup
|
|
||||||
from distutils.extension import Extension
|
|
||||||
from Cython.Build import cythonize
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
print("building on platform: "+sys.platform)
|
|
||||||
os.system("git tag -l --sort=version:refname > gittags.log")
|
|
||||||
with open ("gittags.log","r") as gt:
|
|
||||||
taglst = gt.readlines()
|
|
||||||
os.remove("gittags.log")
|
|
||||||
if len(taglst) > 0 :
|
|
||||||
version = taglst[-1].replace('\n','').replace('v','')
|
|
||||||
else:
|
|
||||||
version = 'unkown'
|
|
||||||
print("building version: "+version)
|
|
||||||
|
|
||||||
if sys.platform == "linux" or sys.platform == "darwin" :
|
|
||||||
cmpargs = ['-std=c++17','-Wno-unused-variable']
|
|
||||||
lnkargs = ['-std=c++17','-Wno-unused-variable']
|
|
||||||
elif sys.platform == "win32" :
|
|
||||||
cmpargs = ['/EHsc','/std:c++17']
|
|
||||||
lnkargs = []
|
|
||||||
else :
|
|
||||||
raise RuntimeError("unknown platform")
|
|
||||||
|
|
||||||
extensions = Extension(
|
|
||||||
name="imc_termite",
|
|
||||||
sources=["cython/py_imc_termite.pyx"],
|
|
||||||
# libraries=[""],
|
|
||||||
# library_dirs=["lib"],
|
|
||||||
include_dirs=["lib"],
|
|
||||||
language='c++',
|
|
||||||
extra_compile_args=cmpargs,
|
|
||||||
extra_link_args=lnkargs,
|
|
||||||
)
|
|
||||||
|
|
||||||
setup(
|
|
||||||
name='imc_termite',
|
|
||||||
version='1.2.12',
|
|
||||||
description='IMCtermite cython extension',
|
|
||||||
author='Record Evolution GmbH',
|
|
||||||
author_email='mario.fink@record-evolution.de',
|
|
||||||
url='https://github.com/RecordEvolution/IMCtermite.git',
|
|
||||||
ext_modules=cythonize(extensions,force=True)
|
|
||||||
)
|
|
||||||
@ -1,20 +0,0 @@
|
|||||||
from distutils.core import setup
|
|
||||||
from distutils.extension import Extension
|
|
||||||
from Cython.Build import cythonize
|
|
||||||
|
|
||||||
extensions = Extension(
|
|
||||||
name="raw_eater",
|
|
||||||
sources=["cython/raw_eater.pyx"],
|
|
||||||
# libraries=[""],
|
|
||||||
library_dirs=["src"],
|
|
||||||
include_dirs=["src"],
|
|
||||||
language='c++',
|
|
||||||
extra_compile_args=['-std=c++11','-Wno-unused-variable'],
|
|
||||||
extra_link_args=['-std=c++11'],
|
|
||||||
#extra_objects=["lib/parquet/libarrow.so.200.0.0"],
|
|
||||||
)
|
|
||||||
|
|
||||||
setup(
|
|
||||||
name="raw_eater",
|
|
||||||
ext_modules=cythonize(extensions)
|
|
||||||
)
|
|
||||||
@ -1,20 +0,0 @@
|
|||||||
from distutils.core import setup
|
|
||||||
from distutils.extension import Extension
|
|
||||||
from Cython.Build import cythonize
|
|
||||||
|
|
||||||
extensions = Extension(
|
|
||||||
name="raw_meat",
|
|
||||||
sources=["cython/raw_meat.pyx"],
|
|
||||||
# libraries=[""],
|
|
||||||
library_dirs=["src"],
|
|
||||||
include_dirs=["src"],
|
|
||||||
language='c++',
|
|
||||||
extra_compile_args=['-std=c++11','-Wno-unused-variable'],
|
|
||||||
extra_link_args=['-std=c++11'],
|
|
||||||
#extra_objects=["lib/parquet/libarrow.so.200.0.0"],
|
|
||||||
)
|
|
||||||
|
|
||||||
setup(
|
|
||||||
name="raw_meat",
|
|
||||||
ext_modules=cythonize(extensions)
|
|
||||||
)
|
|
||||||
@ -1,133 +0,0 @@
|
|||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import os
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='List all source dependencies')
|
|
||||||
#parser.add_argument('pathToRepo',type=str,help='path of source repository')
|
|
||||||
parser.add_argument('mainSource',type=str,help='main source file')
|
|
||||||
parser.add_argument('depFile',type=str,help='file listing all dependencies')
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
libpaths = ["/home/mario/Desktop/arrow/cpp/src/",
|
|
||||||
"/home/mario/Desktop/arrow/cpp/thrift_ep-install/include/",
|
|
||||||
"/home/mario/Desktop/arrow/cpp/boost_ep-prefix/src/boost_ep/"]
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
def find_dependencies(srcfile, recdepth, cdeplist) :
|
|
||||||
"""
|
|
||||||
Given a source file and its dependencies in the given repository path
|
|
||||||
list all further dependencies recursively
|
|
||||||
|
|
||||||
Args:
|
|
||||||
srcfile (string): path/name of source file
|
|
||||||
recdepth (integer): current recursion depth
|
|
||||||
cdeplist (list): current list of dependencies
|
|
||||||
Return:
|
|
||||||
deps (list): list of source files in repository, the source file depends on
|
|
||||||
"""
|
|
||||||
# define indentation to visual recursion
|
|
||||||
indent = recdepth*(" ")
|
|
||||||
|
|
||||||
print("\n" + indent + "find_dependencies:"
|
|
||||||
+ "\n" + indent + "1: " + srcfile
|
|
||||||
+ "\n" + indent + "2: " + str(recdepth)
|
|
||||||
+ "\n" + indent + "3: " + str(len(cdeplist)) + "\n")
|
|
||||||
|
|
||||||
# show dependencies so far
|
|
||||||
#print(cdeplist)
|
|
||||||
|
|
||||||
# generate dependencies by means of g++
|
|
||||||
libdeps = (" -I ").join(libpaths)
|
|
||||||
cmd = "g++ -c -MMD " + srcfile + " -I " + libdeps
|
|
||||||
print(indent + cmd )
|
|
||||||
os.system(cmd)
|
|
||||||
|
|
||||||
# open dependency file and extract list of sources
|
|
||||||
basename = srcfile.split('/')[-1].split('.')[0]
|
|
||||||
depfile = basename + '.d'
|
|
||||||
print(indent + "reading dependency file " + depfile)
|
|
||||||
with open(depfile,'r') as fin :
|
|
||||||
depslist = fin.readlines()
|
|
||||||
|
|
||||||
# delete dependencies and object files
|
|
||||||
os.system("rm " + basename + ".d")
|
|
||||||
os.system("rm " + basename + ".o")
|
|
||||||
|
|
||||||
# remove first line
|
|
||||||
depslist = depslist[1:]
|
|
||||||
|
|
||||||
# delete leading space and trailing backslash
|
|
||||||
depslistcl = [dep.lstrip(' ').rstrip(' \\\n') for dep in depslist]
|
|
||||||
|
|
||||||
# collect dependencies
|
|
||||||
newdeps = []
|
|
||||||
|
|
||||||
# check all dependencies recursively and collect further dependencies
|
|
||||||
count = 0
|
|
||||||
for dep in depslistcl :
|
|
||||||
# append source itself to list
|
|
||||||
if dep not in cdeplist :
|
|
||||||
print(indent + "adding dependency " + dep)
|
|
||||||
newdeps.append(dep)
|
|
||||||
count = count + 1
|
|
||||||
print(indent + "=> added " + str(count) + "/" + str(len(depslistcl)) )
|
|
||||||
|
|
||||||
# check recursion depth
|
|
||||||
if recdepth < 20 :
|
|
||||||
# check all dependencies of every single dependency
|
|
||||||
for dep in depslistcl :
|
|
||||||
# try to find corresponding *.cc, (*.cpp) file
|
|
||||||
depcc = dep.split('.')[0] + '.cc'
|
|
||||||
print(indent + "checking for " + depcc)
|
|
||||||
if os.path.exists(depcc) :
|
|
||||||
if depcc not in cdeplist and depcc not in newdeps :
|
|
||||||
# add file itself as dependency
|
|
||||||
newdeps.append(depcc)
|
|
||||||
# find dependencies of single source
|
|
||||||
newrecdeps = find_dependencies(depcc,recdepth+1,cdeplist+newdeps)
|
|
||||||
# append to list
|
|
||||||
for el in newrecdeps :
|
|
||||||
if el not in newdeps :
|
|
||||||
newdeps.append(el)
|
|
||||||
else :
|
|
||||||
print(indent + "already in list")
|
|
||||||
else :
|
|
||||||
print(indent + "does not exist")
|
|
||||||
|
|
||||||
print("\n")
|
|
||||||
|
|
||||||
# provide list of dependencies
|
|
||||||
return newdeps
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
if __name__== "__main__":
|
|
||||||
|
|
||||||
print("\nCLI arguments:\n" + str(args) + "\n")
|
|
||||||
|
|
||||||
# collect list of dependencies
|
|
||||||
deps = []
|
|
||||||
|
|
||||||
# start recursion with given source file
|
|
||||||
deps = find_dependencies(args.mainSource,0,[])
|
|
||||||
|
|
||||||
print("\nfinal list of dependencies: (" + str(len(deps)) + ")\n")
|
|
||||||
print(deps)
|
|
||||||
print("\n")
|
|
||||||
|
|
||||||
# remove any duplicates
|
|
||||||
depsuni = set(deps)
|
|
||||||
print("\nfinal set of dependencies: (" + str(len(depsuni)) + ")\n")
|
|
||||||
print(depsuni)
|
|
||||||
print("\n")
|
|
||||||
|
|
||||||
# write list of dependencies
|
|
||||||
with open(args.depFile,'w') as fout :
|
|
||||||
for el in depsuni :
|
|
||||||
fout.write(str(el) + '\n')
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
@ -1,22 +0,0 @@
|
|||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
# find source files
|
|
||||||
srcpaths = Path("src/").rglob('*.cc')
|
|
||||||
deps =[ str(path) for path in srcpaths ]
|
|
||||||
|
|
||||||
print(deps)
|
|
||||||
|
|
||||||
with open('makefileobj','w') as fout :
|
|
||||||
for el in deps :
|
|
||||||
basnam = el.split('/')[-1]
|
|
||||||
print(str(el) + " : " + str(basnam) + " : " + str(basnam.split('.')[1]))
|
|
||||||
if basnam.split('.')[1] == 'cc' :
|
|
||||||
objfile = 'bin/' + basnam.replace('.cc','.o')
|
|
||||||
fout.write(objfile + " : " + el + "\n")
|
|
||||||
fout.write("\t" + "$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@\n")
|
|
||||||
fout.write("\n")
|
|
||||||
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
@ -1,356 +0,0 @@
|
|||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
CPP := g++ -std=c++14
|
|
||||||
CPPFLAGS := -Woverflow -Wpedantic -Wextra -Waddress -Waligned-new -Walloc-zero
|
|
||||||
|
|
||||||
SRC := src/
|
|
||||||
BIN := bin/
|
|
||||||
|
|
||||||
LIBS := -I src/src/ -I src/thrift_ep-install/include/ -I src/boost_ep-prefix/src/boost_ep/
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
# prepare source
|
|
||||||
#
|
|
||||||
# before: $ cd arrow/cpp/ and compile relevant sources by
|
|
||||||
# $ cmake . -D ARROW_PARQUET=ON -D PARQUET_BUILD_EXAMPLES=ON -D ARROW_WITH_SNAPPY=ON
|
|
||||||
# $ cmake .. -D ARROW_PARQUET=ON ARROW_BUILD_EXAMPLES=ON
|
|
||||||
|
|
||||||
lib :
|
|
||||||
cmake . -D ARROW_WITH_BROTLI=ON -D ARROW_WITH_BZ2=ON -D ARROW_WITH_LZ4=ON -D ARROW_WITH_SNAPPY=ON -D ARROW_WITH_ZLIB=ON -D ARROW_PARQUET=ON -D ARROW_PYTHON=ON
|
|
||||||
|
|
||||||
# cp-src : deps.log
|
|
||||||
# ./src_copy.sh
|
|
||||||
|
|
||||||
deps.log :
|
|
||||||
python3 generate_deps.py reader-writer.cc $@
|
|
||||||
|
|
||||||
SRC := $(shell find $(SRC) -name '*.cc')
|
|
||||||
|
|
||||||
# OBJ := $(apprefix obj/, $(SRC:%.cc=%.o))
|
|
||||||
OBJ := $(addprefix $(BIN),$(notdir $(SRC:%.cc=%.o)))
|
|
||||||
|
|
||||||
check :
|
|
||||||
@echo $(SRC)
|
|
||||||
@echo $(OBJ)
|
|
||||||
|
|
||||||
# vpath %.cc src/
|
|
||||||
|
|
||||||
reader-writer-example : reader-writer.cc $(OBJ) bin/utilmemory.o
|
|
||||||
$(CPP) $(CPPFLAGS) $< $(LIBS) -o $@ $(OBJ) bin/utilmemory.o
|
|
||||||
|
|
||||||
# $(OBJ) : $(SRC)
|
|
||||||
# $(CPP) $(OPT) -c $< -o $@ -I src/src/
|
|
||||||
#
|
|
||||||
# $(BIN)%.o : $(SRC)
|
|
||||||
# $(CPP) $(OPT) -c $< -I src/src/ -o $@
|
|
||||||
|
|
||||||
clean-obj :
|
|
||||||
rm -f $(OBJ)
|
|
||||||
|
|
||||||
# => do build with cmake like here
|
|
||||||
# https://arrow.apache.org/docs/developers/python.html#build-and-test
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
bin/type.o : src/src/arrow/type.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/result.o : src/src/arrow/result.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder.o : src/src/arrow/builder.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/tensor.o : src/src/arrow/tensor.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/table.o : src/src/arrow/table.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/extension_type.o : src/src/arrow/extension_type.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/device.o : src/src/arrow/device.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/memory_pool.o : src/src/arrow/memory_pool.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/datum.o : src/src/arrow/datum.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/record_batch.o : src/src/arrow/record_batch.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/compare.o : src/src/arrow/compare.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/visitor.o : src/src/arrow/visitor.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/chunked_array.o : src/src/arrow/chunked_array.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/status.o : src/src/arrow/status.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/pretty_print.o : src/src/arrow/pretty_print.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/sparse_tensor.o : src/src/arrow/sparse_tensor.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/buffer.o : src/src/arrow/buffer.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/scalar.o : src/src/arrow/scalar.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/string.o : src/src/arrow/util/string.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/utilmemory.o : src/src/arrow/util/memory.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/future.o : src/src/arrow/util/future.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/iterator.o : src/src/arrow/util/iterator.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/compression.o : src/src/arrow/util/compression.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/utf8.o : src/src/arrow/util/utf8.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/time.o : src/src/arrow/util/time.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/cpu_info.o : src/src/arrow/util/cpu_info.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/thread_pool.o : src/src/arrow/util/thread_pool.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/bit_util.o : src/src/arrow/util/bit_util.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/logging.o : src/src/arrow/util/logging.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/basic_decimal.o : src/src/arrow/util/basic_decimal.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/decimal.o : src/src/arrow/util/decimal.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/bit_block_counter.o : src/src/arrow/util/bit_block_counter.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/key_value_metadata.o : src/src/arrow/util/key_value_metadata.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/int_util.o : src/src/arrow/util/int_util.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/io_util.o : src/src/arrow/util/io_util.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/bitmap_ops.o : src/src/arrow/util/bitmap_ops.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/bitmap_builders.o : src/src/arrow/util/bitmap_builders.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/bit_run_reader.o : src/src/arrow/util/bit_run_reader.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/value_parsing.o : src/src/arrow/util/value_parsing.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/string_builder.o : src/src/arrow/util/string_builder.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/formatting.o : src/src/arrow/util/formatting.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/array_primitive.o : src/src/arrow/array/array_primitive.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/array_dict.o : src/src/arrow/array/array_dict.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_binary.o : src/src/arrow/array/builder_binary.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_union.o : src/src/arrow/array/builder_union.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/concatenate.o : src/src/arrow/array/concatenate.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/array_nested.o : src/src/arrow/array/array_nested.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/array_decimal.o : src/src/arrow/array/array_decimal.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_primitive.o : src/src/arrow/array/builder_primitive.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/data.o : src/src/arrow/array/data.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/diff.o : src/src/arrow/array/diff.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_nested.o : src/src/arrow/array/builder_nested.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_decimal.o : src/src/arrow/array/builder_decimal.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_dict.o : src/src/arrow/array/builder_dict.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/array_binary.o : src/src/arrow/array/array_binary.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_adaptive.o : src/src/arrow/array/builder_adaptive.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/array_base.o : src/src/arrow/array/array_base.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/validate.o : src/src/arrow/array/validate.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_base.o : src/src/arrow/array/builder_base.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/util.o : src/src/arrow/array/util.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/caching.o : src/src/arrow/io/caching.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/memory.o : src/src/arrow/io/memory.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/interfaces.o : src/src/arrow/io/interfaces.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/buffered.o : src/src/arrow/io/buffered.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/file.o : src/src/arrow/io/file.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/strtod.o : src/src/arrow/vendored/double-conversion/strtod.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/bignum.o : src/src/arrow/vendored/double-conversion/bignum.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/fixed-dtoa.o : src/src/arrow/vendored/double-conversion/fixed-dtoa.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/fast-dtoa.o : src/src/arrow/vendored/double-conversion/fast-dtoa.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/diy-fp.o : src/src/arrow/vendored/double-conversion/diy-fp.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/double-conversion.o : src/src/arrow/vendored/double-conversion/double-conversion.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/bignum-dtoa.o : src/src/arrow/vendored/double-conversion/bignum-dtoa.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/cached-powers.o : src/src/arrow/vendored/double-conversion/cached-powers.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/api_aggregate.o : src/src/arrow/compute/api_aggregate.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/exec.o : src/src/arrow/compute/exec.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/kernel.o : src/src/arrow/compute/kernel.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/registry.o : src/src/arrow/compute/registry.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/function.o : src/src/arrow/compute/function.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/cast.o : src/src/arrow/compute/cast.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/api_vector.o : src/src/arrow/compute/api_vector.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/api_scalar.o : src/src/arrow/compute/api_scalar.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/codegen_internal.o : src/src/arrow/compute/kernels/codegen_internal.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/column_scanner.o : src/src/parquet/column_scanner.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/statistics.o : src/src/parquet/statistics.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/internal_file_decryptor.o : src/src/parquet/internal_file_decryptor.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/column_writer.o : src/src/parquet/column_writer.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/encryption.o : src/src/parquet/encryption.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/file_reader.o : src/src/parquet/file_reader.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/properties.o : src/src/parquet/properties.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/encryption_internal.o : src/src/parquet/encryption_internal.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/internal_file_encryptor.o : src/src/parquet/internal_file_encryptor.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/types.o : src/src/parquet/types.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/encoding.o : src/src/parquet/encoding.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/metadata.o : src/src/parquet/metadata.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/printer.o : src/src/parquet/printer.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/level_conversion.o : src/src/parquet/level_conversion.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/deprecated_io.o : src/src/parquet/deprecated_io.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/file_writer.o : src/src/parquet/file_writer.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/schema.o : src/src/parquet/schema.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/platform.o : src/src/parquet/platform.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/column_reader.o : src/src/parquet/column_reader.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
@ -1,96 +0,0 @@
|
|||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
PARQUETDIR := /home/mario/Desktop/Record_Evolution/parquet-cpp
|
|
||||||
ARROWDIR := /home/mario/Desktop/Record_Evolution/arrow/cpp/src
|
|
||||||
|
|
||||||
CPP := g++ -std=c++14
|
|
||||||
OPT := -Wall -Woverflow -Wpedantic -Wextra -Waddress -Waligned-new -Walloc-zero
|
|
||||||
|
|
||||||
prepare : collect_parquet modify_parquet collect_arrow modify_arrow
|
|
||||||
|
|
||||||
collect_parquet :
|
|
||||||
cp -r $(PARQUETDIR)/src/parquet ./
|
|
||||||
cp $(PARQUETDIR)/examples/low-level-api/reader_writer.h ./
|
|
||||||
cp $(PARQUETDIR)/examples/low-level-api/reader-writer.cc ./
|
|
||||||
|
|
||||||
modify_parquet :
|
|
||||||
cp parquet/parquet_version.h.in parquet/parquet_version.h
|
|
||||||
sed -i 's/ReadableFileInterface/ReadWriteFileInterface/g' parquet/util/memory.h
|
|
||||||
sed -i 's/ReadableFileInterface/ReadWriteFileInterface/g' parquet/file_reader.h
|
|
||||||
sed -i 's/arrow::Codec/arrow::util::Codec/g' parquet/util/memory.h
|
|
||||||
sed -i 's/valid_bits_writer/valid_bits_offset/g' parquet/column_reader.h
|
|
||||||
|
|
||||||
collect_arrow :
|
|
||||||
cp -r $(ARROWDIR)/arrow ./
|
|
||||||
|
|
||||||
modify_arrow :
|
|
||||||
cp arrow/util/bit_util.h arrow/util/bit-util.h
|
|
||||||
|
|
||||||
collect_test :
|
|
||||||
cp $(PARQUETDIR)/examples/low-level-api/reader-writer.cc ./
|
|
||||||
|
|
||||||
subst :
|
|
||||||
sed -i 's/#include \"arrow\//\/\/#include \"arrow/g' parquet/properties.h
|
|
||||||
|
|
||||||
test :
|
|
||||||
$(CPP) $(OPT) -I$(PWD) reader-writer.cc
|
|
||||||
|
|
||||||
clean :
|
|
||||||
rm -r parquet/ arrow/
|
|
||||||
rm reader-writer.cc reader_writer.h
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
# choose shell
|
|
||||||
SHELL:=/bin/bash
|
|
||||||
|
|
||||||
SRC = reader-writer
|
|
||||||
|
|
||||||
# specify path of cloned directory
|
|
||||||
ARROWGIT := /home/mario/Desktop/Record_Evolution/arrow
|
|
||||||
|
|
||||||
filewriter : parquet/file_writer.cc
|
|
||||||
$(CPP) -c $(OPT) $<
|
|
||||||
|
|
||||||
# build executable (and generate dependency file)
|
|
||||||
readwrite : reader-writer.cc
|
|
||||||
$(CPP) $(OPT) -MMD $< -I ./
|
|
||||||
|
|
||||||
# generate dependency file
|
|
||||||
$(SRC).d : $(SRC).cc
|
|
||||||
$(CPP) -c -MMD $< -I ./ -I $(ARROWGIT)/cpp/src/
|
|
||||||
|
|
||||||
# extract source dependencies
|
|
||||||
extract-dep : $(SRC).d
|
|
||||||
@# extract relevant dependencies
|
|
||||||
cat $< | sed 's/ /\n/g' | awk 'NF' | grep -v '\\' | grep '\/' > deps.log
|
|
||||||
cat deps.log | sed ':a;N;$!ba;s/\n/ /g' > headers.log
|
|
||||||
cat headers.log | sed 's/.h$$/.cc/g' > sources.log
|
|
||||||
@# copy required sources
|
|
||||||
mkdir -p temp/
|
|
||||||
cp --parents `cat headers.log` temp/
|
|
||||||
cp --parents `cat sources.log` temp/ 2>/dev/null
|
|
||||||
mv temp$(ARROWGIT)/cpp/src/* ./
|
|
||||||
rm -r temp
|
|
||||||
|
|
||||||
clean-dep :
|
|
||||||
rm -f deps.log headers.log sources.log $(SRC).d
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
# only use more recent and up to date repository arrow.git
|
|
||||||
|
|
||||||
# build arrow shared/static libraries
|
|
||||||
build :
|
|
||||||
cd arrow/cpp
|
|
||||||
# cmake -LA to show all options
|
|
||||||
cmake . -D ARROW_PARQUET=ON #ARROW_ARMV8_ARCH=armv8-a
|
|
||||||
make
|
|
||||||
|
|
||||||
example :
|
|
||||||
cd arrow/cpp/examples/parquet/low-level-api/
|
|
||||||
g++ reader-writer.cc -I. -I../../../src/ -L../../../../cpp/build/release/ -larrow -lparquet
|
|
||||||
|
|
||||||
# set environment variable LD_LIBRARY_PATH=../../../../cpp/build/release/ before launching executable
|
|
||||||
|
|
||||||
|
|
||||||
#------------------------------------------------------------------------------------#
|
|
||||||
@ -1,303 +0,0 @@
|
|||||||
bin/type.o : src/src/arrow/type.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/result.o : src/src/arrow/result.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder.o : src/src/arrow/builder.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/tensor.o : src/src/arrow/tensor.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/table.o : src/src/arrow/table.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/extension_type.o : src/src/arrow/extension_type.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/device.o : src/src/arrow/device.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/memory_pool.o : src/src/arrow/memory_pool.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/datum.o : src/src/arrow/datum.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/record_batch.o : src/src/arrow/record_batch.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/compare.o : src/src/arrow/compare.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/visitor.o : src/src/arrow/visitor.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/chunked_array.o : src/src/arrow/chunked_array.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/status.o : src/src/arrow/status.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/pretty_print.o : src/src/arrow/pretty_print.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/sparse_tensor.o : src/src/arrow/sparse_tensor.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/buffer.o : src/src/arrow/buffer.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/scalar.o : src/src/arrow/scalar.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/string.o : src/src/arrow/util/string.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/memory.o : src/src/arrow/util/memory.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/future.o : src/src/arrow/util/future.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/iterator.o : src/src/arrow/util/iterator.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/compression.o : src/src/arrow/util/compression.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/utf8.o : src/src/arrow/util/utf8.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/time.o : src/src/arrow/util/time.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/cpu_info.o : src/src/arrow/util/cpu_info.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/thread_pool.o : src/src/arrow/util/thread_pool.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/bit_util.o : src/src/arrow/util/bit_util.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/logging.o : src/src/arrow/util/logging.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/basic_decimal.o : src/src/arrow/util/basic_decimal.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/decimal.o : src/src/arrow/util/decimal.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/bit_block_counter.o : src/src/arrow/util/bit_block_counter.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/key_value_metadata.o : src/src/arrow/util/key_value_metadata.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/int_util.o : src/src/arrow/util/int_util.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/io_util.o : src/src/arrow/util/io_util.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/bitmap_ops.o : src/src/arrow/util/bitmap_ops.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/bitmap_builders.o : src/src/arrow/util/bitmap_builders.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/bit_run_reader.o : src/src/arrow/util/bit_run_reader.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/value_parsing.o : src/src/arrow/util/value_parsing.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/string_builder.o : src/src/arrow/util/string_builder.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/formatting.o : src/src/arrow/util/formatting.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/array_primitive.o : src/src/arrow/array/array_primitive.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/array_dict.o : src/src/arrow/array/array_dict.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_binary.o : src/src/arrow/array/builder_binary.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_union.o : src/src/arrow/array/builder_union.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/concatenate.o : src/src/arrow/array/concatenate.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/array_nested.o : src/src/arrow/array/array_nested.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/array_decimal.o : src/src/arrow/array/array_decimal.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_primitive.o : src/src/arrow/array/builder_primitive.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/data.o : src/src/arrow/array/data.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/diff.o : src/src/arrow/array/diff.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_nested.o : src/src/arrow/array/builder_nested.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_decimal.o : src/src/arrow/array/builder_decimal.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_dict.o : src/src/arrow/array/builder_dict.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/array_binary.o : src/src/arrow/array/array_binary.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_adaptive.o : src/src/arrow/array/builder_adaptive.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/array_base.o : src/src/arrow/array/array_base.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/validate.o : src/src/arrow/array/validate.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/builder_base.o : src/src/arrow/array/builder_base.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/util.o : src/src/arrow/array/util.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/caching.o : src/src/arrow/io/caching.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/memory.o : src/src/arrow/io/memory.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/interfaces.o : src/src/arrow/io/interfaces.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/buffered.o : src/src/arrow/io/buffered.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/file.o : src/src/arrow/io/file.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/strtod.o : src/src/arrow/vendored/double-conversion/strtod.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/bignum.o : src/src/arrow/vendored/double-conversion/bignum.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/fixed-dtoa.o : src/src/arrow/vendored/double-conversion/fixed-dtoa.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/fast-dtoa.o : src/src/arrow/vendored/double-conversion/fast-dtoa.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/diy-fp.o : src/src/arrow/vendored/double-conversion/diy-fp.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/double-conversion.o : src/src/arrow/vendored/double-conversion/double-conversion.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/bignum-dtoa.o : src/src/arrow/vendored/double-conversion/bignum-dtoa.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/cached-powers.o : src/src/arrow/vendored/double-conversion/cached-powers.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/api_aggregate.o : src/src/arrow/compute/api_aggregate.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/exec.o : src/src/arrow/compute/exec.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/kernel.o : src/src/arrow/compute/kernel.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/registry.o : src/src/arrow/compute/registry.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/function.o : src/src/arrow/compute/function.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/cast.o : src/src/arrow/compute/cast.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/api_vector.o : src/src/arrow/compute/api_vector.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/api_scalar.o : src/src/arrow/compute/api_scalar.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/codegen_internal.o : src/src/arrow/compute/kernels/codegen_internal.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/column_scanner.o : src/src/parquet/column_scanner.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/statistics.o : src/src/parquet/statistics.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/internal_file_decryptor.o : src/src/parquet/internal_file_decryptor.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/column_writer.o : src/src/parquet/column_writer.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/encryption.o : src/src/parquet/encryption.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/file_reader.o : src/src/parquet/file_reader.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/properties.o : src/src/parquet/properties.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/encryption_internal.o : src/src/parquet/encryption_internal.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/internal_file_encryptor.o : src/src/parquet/internal_file_encryptor.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/types.o : src/src/parquet/types.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/encoding.o : src/src/parquet/encoding.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/metadata.o : src/src/parquet/metadata.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/printer.o : src/src/parquet/printer.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/level_conversion.o : src/src/parquet/level_conversion.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/deprecated_io.o : src/src/parquet/deprecated_io.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/file_writer.o : src/src/parquet/file_writer.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/schema.o : src/src/parquet/schema.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/platform.o : src/src/parquet/platform.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
bin/column_reader.o : src/src/parquet/column_reader.cc
|
|
||||||
$(CPP) $(CPPFLAGS) -c $< $(LIBS) -o $@
|
|
||||||
|
|
||||||
@ -1,413 +0,0 @@
|
|||||||
// Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
// or more contributor license agreements. See the NOTICE file
|
|
||||||
// distributed with this work for additional information
|
|
||||||
// regarding copyright ownership. The ASF licenses this file
|
|
||||||
// to you under the Apache License, Version 2.0 (the
|
|
||||||
// "License"); you may not use this file except in compliance
|
|
||||||
// with the License. You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing,
|
|
||||||
// software distributed under the License is distributed on an
|
|
||||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
||||||
// KIND, either express or implied. See the License for the
|
|
||||||
// specific language governing permissions and limitations
|
|
||||||
// under the License.
|
|
||||||
|
|
||||||
#include <cassert>
|
|
||||||
#include <fstream>
|
|
||||||
#include <iostream>
|
|
||||||
#include <memory>
|
|
||||||
|
|
||||||
#include "reader_writer.h"
|
|
||||||
|
|
||||||
/*
|
|
||||||
* This example describes writing and reading Parquet Files in C++ and serves as a
|
|
||||||
* reference to the API.
|
|
||||||
* The file contains all the physical data types supported by Parquet.
|
|
||||||
* This example uses the RowGroupWriter API that supports writing RowGroups optimized for
|
|
||||||
*memory consumption
|
|
||||||
**/
|
|
||||||
|
|
||||||
/* Parquet is a structured columnar file format
|
|
||||||
* Parquet File = "Parquet data" + "Parquet Metadata"
|
|
||||||
* "Parquet data" is simply a vector of RowGroups. Each RowGroup is a batch of rows in a
|
|
||||||
* columnar layout
|
|
||||||
* "Parquet Metadata" contains the "file schema" and attributes of the RowGroups and their
|
|
||||||
* Columns
|
|
||||||
* "file schema" is a tree where each node is either a primitive type (leaf nodes) or a
|
|
||||||
* complex (nested) type (internal nodes)
|
|
||||||
* For specific details, please refer the format here:
|
|
||||||
* https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
|
|
||||||
**/
|
|
||||||
|
|
||||||
constexpr int NUM_ROWS_PER_ROW_GROUP = 500;
|
|
||||||
const char PARQUET_FILENAME[] = "parquet_cpp_example.parquet";
|
|
||||||
|
|
||||||
int main(int argc, char** argv) {
|
|
||||||
/**********************************************************************************
|
|
||||||
PARQUET WRITER EXAMPLE
|
|
||||||
**********************************************************************************/
|
|
||||||
// parquet::REQUIRED fields do not need definition and repetition level values
|
|
||||||
// parquet::OPTIONAL fields require only definition level values
|
|
||||||
// parquet::REPEATED fields require both definition and repetition level values
|
|
||||||
try {
|
|
||||||
// Create a local file output stream instance.
|
|
||||||
using FileClass = ::arrow::io::FileOutputStream;
|
|
||||||
std::shared_ptr<FileClass> out_file;
|
|
||||||
PARQUET_ASSIGN_OR_THROW(out_file, FileClass::Open(PARQUET_FILENAME));
|
|
||||||
|
|
||||||
// Setup the parquet schema
|
|
||||||
std::shared_ptr<GroupNode> schema = SetupSchema();
|
|
||||||
|
|
||||||
// Add writer properties
|
|
||||||
parquet::WriterProperties::Builder builder;
|
|
||||||
builder.compression(parquet::Compression::UNCOMPRESSED);
|
|
||||||
std::shared_ptr<parquet::WriterProperties> props = builder.build();
|
|
||||||
|
|
||||||
// Create a ParquetFileWriter instance
|
|
||||||
std::shared_ptr<parquet::ParquetFileWriter> file_writer =
|
|
||||||
parquet::ParquetFileWriter::Open(out_file, schema, props);
|
|
||||||
|
|
||||||
// Append a RowGroup with a specific number of rows.
|
|
||||||
parquet::RowGroupWriter* rg_writer = file_writer->AppendRowGroup();
|
|
||||||
|
|
||||||
// Write the Bool column
|
|
||||||
parquet::BoolWriter* bool_writer =
|
|
||||||
static_cast<parquet::BoolWriter*>(rg_writer->NextColumn());
|
|
||||||
for (int i = 0; i < NUM_ROWS_PER_ROW_GROUP; i++) {
|
|
||||||
bool value = ((i % 2) == 0) ? true : false;
|
|
||||||
bool_writer->WriteBatch(1, nullptr, nullptr, &value);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write the Int32 column
|
|
||||||
parquet::Int32Writer* int32_writer =
|
|
||||||
static_cast<parquet::Int32Writer*>(rg_writer->NextColumn());
|
|
||||||
for (int i = 0; i < NUM_ROWS_PER_ROW_GROUP; i++) {
|
|
||||||
int32_t value = i;
|
|
||||||
int32_writer->WriteBatch(1, nullptr, nullptr, &value);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write the Int64 column. Each row has repeats twice.
|
|
||||||
parquet::Int64Writer* int64_writer =
|
|
||||||
static_cast<parquet::Int64Writer*>(rg_writer->NextColumn());
|
|
||||||
for (int i = 0; i < 2 * NUM_ROWS_PER_ROW_GROUP; i++) {
|
|
||||||
int64_t value = i * 1000 * 1000;
|
|
||||||
value *= 1000 * 1000;
|
|
||||||
int16_t definition_level = 1;
|
|
||||||
int16_t repetition_level = 0;
|
|
||||||
if ((i % 2) == 0) {
|
|
||||||
repetition_level = 1; // start of a new record
|
|
||||||
}
|
|
||||||
int64_writer->WriteBatch(1, &definition_level, &repetition_level, &value);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write the INT96 column.
|
|
||||||
parquet::Int96Writer* int96_writer =
|
|
||||||
static_cast<parquet::Int96Writer*>(rg_writer->NextColumn());
|
|
||||||
for (int i = 0; i < NUM_ROWS_PER_ROW_GROUP; i++) {
|
|
||||||
parquet::Int96 value;
|
|
||||||
value.value[0] = i;
|
|
||||||
value.value[1] = i + 1;
|
|
||||||
value.value[2] = i + 2;
|
|
||||||
int96_writer->WriteBatch(1, nullptr, nullptr, &value);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write the Float column
|
|
||||||
parquet::FloatWriter* float_writer =
|
|
||||||
static_cast<parquet::FloatWriter*>(rg_writer->NextColumn());
|
|
||||||
for (int i = 0; i < NUM_ROWS_PER_ROW_GROUP; i++) {
|
|
||||||
float value = static_cast<float>(i) * 1.1f;
|
|
||||||
float_writer->WriteBatch(1, nullptr, nullptr, &value);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write the Double column
|
|
||||||
parquet::DoubleWriter* double_writer =
|
|
||||||
static_cast<parquet::DoubleWriter*>(rg_writer->NextColumn());
|
|
||||||
for (int i = 0; i < NUM_ROWS_PER_ROW_GROUP; i++) {
|
|
||||||
double value = i * 1.1111111;
|
|
||||||
double_writer->WriteBatch(1, nullptr, nullptr, &value);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write the ByteArray column. Make every alternate values NULL
|
|
||||||
parquet::ByteArrayWriter* ba_writer =
|
|
||||||
static_cast<parquet::ByteArrayWriter*>(rg_writer->NextColumn());
|
|
||||||
for (int i = 0; i < NUM_ROWS_PER_ROW_GROUP; i++) {
|
|
||||||
parquet::ByteArray value;
|
|
||||||
char hello[FIXED_LENGTH] = "parquet";
|
|
||||||
hello[7] = static_cast<char>(static_cast<int>('0') + i / 100);
|
|
||||||
hello[8] = static_cast<char>(static_cast<int>('0') + (i / 10) % 10);
|
|
||||||
hello[9] = static_cast<char>(static_cast<int>('0') + i % 10);
|
|
||||||
if (i % 2 == 0) {
|
|
||||||
int16_t definition_level = 1;
|
|
||||||
value.ptr = reinterpret_cast<const uint8_t*>(&hello[0]);
|
|
||||||
value.len = FIXED_LENGTH;
|
|
||||||
ba_writer->WriteBatch(1, &definition_level, nullptr, &value);
|
|
||||||
} else {
|
|
||||||
int16_t definition_level = 0;
|
|
||||||
ba_writer->WriteBatch(1, &definition_level, nullptr, nullptr);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write the FixedLengthByteArray column
|
|
||||||
parquet::FixedLenByteArrayWriter* flba_writer =
|
|
||||||
static_cast<parquet::FixedLenByteArrayWriter*>(rg_writer->NextColumn());
|
|
||||||
for (int i = 0; i < NUM_ROWS_PER_ROW_GROUP; i++) {
|
|
||||||
parquet::FixedLenByteArray value;
|
|
||||||
char v = static_cast<char>(i);
|
|
||||||
char flba[FIXED_LENGTH] = {v, v, v, v, v, v, v, v, v, v};
|
|
||||||
value.ptr = reinterpret_cast<const uint8_t*>(&flba[0]);
|
|
||||||
|
|
||||||
flba_writer->WriteBatch(1, nullptr, nullptr, &value);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close the ParquetFileWriter
|
|
||||||
file_writer->Close();
|
|
||||||
|
|
||||||
// Write the bytes to file
|
|
||||||
DCHECK(out_file->Close().ok());
|
|
||||||
} catch (const std::exception& e) {
|
|
||||||
std::cerr << "Parquet write error: " << e.what() << std::endl;
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**********************************************************************************
|
|
||||||
PARQUET READER EXAMPLE
|
|
||||||
**********************************************************************************/
|
|
||||||
|
|
||||||
try {
|
|
||||||
// Create a ParquetReader instance
|
|
||||||
std::unique_ptr<parquet::ParquetFileReader> parquet_reader =
|
|
||||||
parquet::ParquetFileReader::OpenFile(PARQUET_FILENAME, false);
|
|
||||||
|
|
||||||
// Get the File MetaData
|
|
||||||
std::shared_ptr<parquet::FileMetaData> file_metadata = parquet_reader->metadata();
|
|
||||||
|
|
||||||
// Get the number of RowGroups
|
|
||||||
int num_row_groups = file_metadata->num_row_groups();
|
|
||||||
assert(num_row_groups == 1);
|
|
||||||
|
|
||||||
// Get the number of Columns
|
|
||||||
int num_columns = file_metadata->num_columns();
|
|
||||||
assert(num_columns == 8);
|
|
||||||
|
|
||||||
// Iterate over all the RowGroups in the file
|
|
||||||
for (int r = 0; r < num_row_groups; ++r) {
|
|
||||||
// Get the RowGroup Reader
|
|
||||||
std::shared_ptr<parquet::RowGroupReader> row_group_reader =
|
|
||||||
parquet_reader->RowGroup(r);
|
|
||||||
|
|
||||||
int64_t values_read = 0;
|
|
||||||
int64_t rows_read = 0;
|
|
||||||
int16_t definition_level;
|
|
||||||
int16_t repetition_level;
|
|
||||||
int i;
|
|
||||||
std::shared_ptr<parquet::ColumnReader> column_reader;
|
|
||||||
|
|
||||||
ARROW_UNUSED(rows_read); // prevent warning in release build
|
|
||||||
|
|
||||||
// Get the Column Reader for the boolean column
|
|
||||||
column_reader = row_group_reader->Column(0);
|
|
||||||
parquet::BoolReader* bool_reader =
|
|
||||||
static_cast<parquet::BoolReader*>(column_reader.get());
|
|
||||||
|
|
||||||
// Read all the rows in the column
|
|
||||||
i = 0;
|
|
||||||
while (bool_reader->HasNext()) {
|
|
||||||
bool value;
|
|
||||||
// Read one value at a time. The number of rows read is returned. values_read
|
|
||||||
// contains the number of non-null rows
|
|
||||||
rows_read = bool_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
|
|
||||||
// Ensure only one value is read
|
|
||||||
assert(rows_read == 1);
|
|
||||||
// There are no NULL values in the rows written
|
|
||||||
assert(values_read == 1);
|
|
||||||
// Verify the value written
|
|
||||||
bool expected_value = ((i % 2) == 0) ? true : false;
|
|
||||||
assert(value == expected_value);
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the Column Reader for the Int32 column
|
|
||||||
column_reader = row_group_reader->Column(1);
|
|
||||||
parquet::Int32Reader* int32_reader =
|
|
||||||
static_cast<parquet::Int32Reader*>(column_reader.get());
|
|
||||||
// Read all the rows in the column
|
|
||||||
i = 0;
|
|
||||||
while (int32_reader->HasNext()) {
|
|
||||||
int32_t value;
|
|
||||||
// Read one value at a time. The number of rows read is returned. values_read
|
|
||||||
// contains the number of non-null rows
|
|
||||||
rows_read = int32_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
|
|
||||||
// Ensure only one value is read
|
|
||||||
assert(rows_read == 1);
|
|
||||||
// There are no NULL values in the rows written
|
|
||||||
assert(values_read == 1);
|
|
||||||
// Verify the value written
|
|
||||||
assert(value == i);
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the Column Reader for the Int64 column
|
|
||||||
column_reader = row_group_reader->Column(2);
|
|
||||||
parquet::Int64Reader* int64_reader =
|
|
||||||
static_cast<parquet::Int64Reader*>(column_reader.get());
|
|
||||||
// Read all the rows in the column
|
|
||||||
i = 0;
|
|
||||||
while (int64_reader->HasNext()) {
|
|
||||||
int64_t value;
|
|
||||||
// Read one value at a time. The number of rows read is returned. values_read
|
|
||||||
// contains the number of non-null rows
|
|
||||||
rows_read = int64_reader->ReadBatch(1, &definition_level, &repetition_level,
|
|
||||||
&value, &values_read);
|
|
||||||
// Ensure only one value is read
|
|
||||||
assert(rows_read == 1);
|
|
||||||
// There are no NULL values in the rows written
|
|
||||||
assert(values_read == 1);
|
|
||||||
// Verify the value written
|
|
||||||
int64_t expected_value = i * 1000 * 1000;
|
|
||||||
expected_value *= 1000 * 1000;
|
|
||||||
assert(value == expected_value);
|
|
||||||
if ((i % 2) == 0) {
|
|
||||||
assert(repetition_level == 1);
|
|
||||||
} else {
|
|
||||||
assert(repetition_level == 0);
|
|
||||||
}
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the Column Reader for the Int96 column
|
|
||||||
column_reader = row_group_reader->Column(3);
|
|
||||||
parquet::Int96Reader* int96_reader =
|
|
||||||
static_cast<parquet::Int96Reader*>(column_reader.get());
|
|
||||||
// Read all the rows in the column
|
|
||||||
i = 0;
|
|
||||||
while (int96_reader->HasNext()) {
|
|
||||||
parquet::Int96 value;
|
|
||||||
// Read one value at a time. The number of rows read is returned. values_read
|
|
||||||
// contains the number of non-null rows
|
|
||||||
rows_read = int96_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
|
|
||||||
// Ensure only one value is read
|
|
||||||
assert(rows_read == 1);
|
|
||||||
// There are no NULL values in the rows written
|
|
||||||
assert(values_read == 1);
|
|
||||||
// Verify the value written
|
|
||||||
parquet::Int96 expected_value;
|
|
||||||
ARROW_UNUSED(expected_value); // prevent warning in release build
|
|
||||||
expected_value.value[0] = i;
|
|
||||||
expected_value.value[1] = i + 1;
|
|
||||||
expected_value.value[2] = i + 2;
|
|
||||||
for (int j = 0; j < 3; j++) {
|
|
||||||
assert(value.value[j] == expected_value.value[j]);
|
|
||||||
}
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the Column Reader for the Float column
|
|
||||||
column_reader = row_group_reader->Column(4);
|
|
||||||
parquet::FloatReader* float_reader =
|
|
||||||
static_cast<parquet::FloatReader*>(column_reader.get());
|
|
||||||
// Read all the rows in the column
|
|
||||||
i = 0;
|
|
||||||
while (float_reader->HasNext()) {
|
|
||||||
float value;
|
|
||||||
// Read one value at a time. The number of rows read is returned. values_read
|
|
||||||
// contains the number of non-null rows
|
|
||||||
rows_read = float_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
|
|
||||||
// Ensure only one value is read
|
|
||||||
assert(rows_read == 1);
|
|
||||||
// There are no NULL values in the rows written
|
|
||||||
assert(values_read == 1);
|
|
||||||
// Verify the value written
|
|
||||||
float expected_value = static_cast<float>(i) * 1.1f;
|
|
||||||
assert(value == expected_value);
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the Column Reader for the Double column
|
|
||||||
column_reader = row_group_reader->Column(5);
|
|
||||||
parquet::DoubleReader* double_reader =
|
|
||||||
static_cast<parquet::DoubleReader*>(column_reader.get());
|
|
||||||
// Read all the rows in the column
|
|
||||||
i = 0;
|
|
||||||
while (double_reader->HasNext()) {
|
|
||||||
double value;
|
|
||||||
// Read one value at a time. The number of rows read is returned. values_read
|
|
||||||
// contains the number of non-null rows
|
|
||||||
rows_read = double_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
|
|
||||||
// Ensure only one value is read
|
|
||||||
assert(rows_read == 1);
|
|
||||||
// There are no NULL values in the rows written
|
|
||||||
assert(values_read == 1);
|
|
||||||
// Verify the value written
|
|
||||||
double expected_value = i * 1.1111111;
|
|
||||||
assert(value == expected_value);
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the Column Reader for the ByteArray column
|
|
||||||
column_reader = row_group_reader->Column(6);
|
|
||||||
parquet::ByteArrayReader* ba_reader =
|
|
||||||
static_cast<parquet::ByteArrayReader*>(column_reader.get());
|
|
||||||
// Read all the rows in the column
|
|
||||||
i = 0;
|
|
||||||
while (ba_reader->HasNext()) {
|
|
||||||
parquet::ByteArray value;
|
|
||||||
// Read one value at a time. The number of rows read is returned. values_read
|
|
||||||
// contains the number of non-null rows
|
|
||||||
rows_read =
|
|
||||||
ba_reader->ReadBatch(1, &definition_level, nullptr, &value, &values_read);
|
|
||||||
// Ensure only one value is read
|
|
||||||
assert(rows_read == 1);
|
|
||||||
// Verify the value written
|
|
||||||
char expected_value[FIXED_LENGTH] = "parquet";
|
|
||||||
ARROW_UNUSED(expected_value); // prevent warning in release build
|
|
||||||
expected_value[7] = static_cast<char>('0' + i / 100);
|
|
||||||
expected_value[8] = static_cast<char>('0' + (i / 10) % 10);
|
|
||||||
expected_value[9] = static_cast<char>('0' + i % 10);
|
|
||||||
if (i % 2 == 0) { // only alternate values exist
|
|
||||||
// There are no NULL values in the rows written
|
|
||||||
assert(values_read == 1);
|
|
||||||
assert(value.len == FIXED_LENGTH);
|
|
||||||
assert(memcmp(value.ptr, &expected_value[0], FIXED_LENGTH) == 0);
|
|
||||||
assert(definition_level == 1);
|
|
||||||
} else {
|
|
||||||
// There are NULL values in the rows written
|
|
||||||
assert(values_read == 0);
|
|
||||||
assert(definition_level == 0);
|
|
||||||
}
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the Column Reader for the FixedLengthByteArray column
|
|
||||||
column_reader = row_group_reader->Column(7);
|
|
||||||
parquet::FixedLenByteArrayReader* flba_reader =
|
|
||||||
static_cast<parquet::FixedLenByteArrayReader*>(column_reader.get());
|
|
||||||
// Read all the rows in the column
|
|
||||||
i = 0;
|
|
||||||
while (flba_reader->HasNext()) {
|
|
||||||
parquet::FixedLenByteArray value;
|
|
||||||
// Read one value at a time. The number of rows read is returned. values_read
|
|
||||||
// contains the number of non-null rows
|
|
||||||
rows_read = flba_reader->ReadBatch(1, nullptr, nullptr, &value, &values_read);
|
|
||||||
// Ensure only one value is read
|
|
||||||
assert(rows_read == 1);
|
|
||||||
// There are no NULL values in the rows written
|
|
||||||
assert(values_read == 1);
|
|
||||||
// Verify the value written
|
|
||||||
char v = static_cast<char>(i);
|
|
||||||
char expected_value[FIXED_LENGTH] = {v, v, v, v, v, v, v, v, v, v};
|
|
||||||
assert(memcmp(value.ptr, &expected_value[0], FIXED_LENGTH) == 0);
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (const std::exception& e) {
|
|
||||||
std::cerr << "Parquet read error: " << e.what() << std::endl;
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::cout << "Parquet Writing and Reading Complete" << std::endl;
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
@ -1,71 +0,0 @@
|
|||||||
// Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
// or more contributor license agreements. See the NOTICE file
|
|
||||||
// distributed with this work for additional information
|
|
||||||
// regarding copyright ownership. The ASF licenses this file
|
|
||||||
// to you under the Apache License, Version 2.0 (the
|
|
||||||
// "License"); you may not use this file except in compliance
|
|
||||||
// with the License. You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing,
|
|
||||||
// software distributed under the License is distributed on an
|
|
||||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
||||||
// KIND, either express or implied. See the License for the
|
|
||||||
// specific language governing permissions and limitations
|
|
||||||
// under the License.
|
|
||||||
|
|
||||||
#include <arrow/io/file.h>
|
|
||||||
#include <arrow/util/logging.h>
|
|
||||||
|
|
||||||
#include <parquet/api/reader.h>
|
|
||||||
#include <parquet/api/writer.h>
|
|
||||||
|
|
||||||
using parquet::ConvertedType;
|
|
||||||
using parquet::Repetition;
|
|
||||||
using parquet::Type;
|
|
||||||
using parquet::schema::GroupNode;
|
|
||||||
using parquet::schema::PrimitiveNode;
|
|
||||||
|
|
||||||
constexpr int FIXED_LENGTH = 10;
|
|
||||||
|
|
||||||
static std::shared_ptr<GroupNode> SetupSchema() {
|
|
||||||
parquet::schema::NodeVector fields;
|
|
||||||
// Create a primitive node named 'boolean_field' with type:BOOLEAN,
|
|
||||||
// repetition:REQUIRED
|
|
||||||
fields.push_back(PrimitiveNode::Make("boolean_field", Repetition::REQUIRED,
|
|
||||||
Type::BOOLEAN, ConvertedType::NONE));
|
|
||||||
|
|
||||||
// Create a primitive node named 'int32_field' with type:INT32, repetition:REQUIRED,
|
|
||||||
// logical type:TIME_MILLIS
|
|
||||||
fields.push_back(PrimitiveNode::Make("int32_field", Repetition::REQUIRED, Type::INT32,
|
|
||||||
ConvertedType::TIME_MILLIS));
|
|
||||||
|
|
||||||
// Create a primitive node named 'int64_field' with type:INT64, repetition:REPEATED
|
|
||||||
fields.push_back(PrimitiveNode::Make("int64_field", Repetition::REPEATED, Type::INT64,
|
|
||||||
ConvertedType::NONE));
|
|
||||||
|
|
||||||
fields.push_back(PrimitiveNode::Make("int96_field", Repetition::REQUIRED, Type::INT96,
|
|
||||||
ConvertedType::NONE));
|
|
||||||
|
|
||||||
fields.push_back(PrimitiveNode::Make("float_field", Repetition::REQUIRED, Type::FLOAT,
|
|
||||||
ConvertedType::NONE));
|
|
||||||
|
|
||||||
fields.push_back(PrimitiveNode::Make("double_field", Repetition::REQUIRED, Type::DOUBLE,
|
|
||||||
ConvertedType::NONE));
|
|
||||||
|
|
||||||
// Create a primitive node named 'ba_field' with type:BYTE_ARRAY, repetition:OPTIONAL
|
|
||||||
fields.push_back(PrimitiveNode::Make("ba_field", Repetition::OPTIONAL, Type::BYTE_ARRAY,
|
|
||||||
ConvertedType::NONE));
|
|
||||||
|
|
||||||
// Create a primitive node named 'flba_field' with type:FIXED_LEN_BYTE_ARRAY,
|
|
||||||
// repetition:REQUIRED, field_length = FIXED_LENGTH
|
|
||||||
fields.push_back(PrimitiveNode::Make("flba_field", Repetition::REQUIRED,
|
|
||||||
Type::FIXED_LEN_BYTE_ARRAY, ConvertedType::NONE,
|
|
||||||
FIXED_LENGTH));
|
|
||||||
|
|
||||||
// Create a GroupNode named 'schema' using the primitive nodes defined above
|
|
||||||
// This GroupNode is the root node of the schema tree
|
|
||||||
return std::static_pointer_cast<GroupNode>(
|
|
||||||
GroupNode::Make("schema", Repetition::REQUIRED, fields));
|
|
||||||
}
|
|
||||||
@ -1,6 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
mkdir src
|
|
||||||
cat deps.log | while read f; do cp --parents $f src/; done;
|
|
||||||
mv src/home/mario/Desktop/arrow/cpp/* src/
|
|
||||||
rm -r src/home/
|
|
||||||
@ -1,171 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
# NOTE: before starting to extract the minimal required sources and dependencies
|
|
||||||
# run
|
|
||||||
# $ cd cpp/
|
|
||||||
# $ cmake -D ARROW_PARQUET=ON
|
|
||||||
# in the arrow repository
|
|
||||||
|
|
||||||
# provide
|
|
||||||
# - local path of clone of https://github.com/apache/arrow.git
|
|
||||||
# - name/path of main .hpp file of cython extension
|
|
||||||
repo="$1"
|
|
||||||
main="$2"
|
|
||||||
depf="$3"
|
|
||||||
|
|
||||||
# check CLI arguments
|
|
||||||
if [ -z "$repo" ] || [ -z "$main" ] || [ -z "$depf" ]; then
|
|
||||||
echo "please provide..."
|
|
||||||
echo "1. local path of arrow repository"
|
|
||||||
echo "2. name of main .hpp/.cpp"
|
|
||||||
echo "3. desired name of dependency file"
|
|
||||||
echo -e "example:\n./setup-sources.sh /home/mario/Desktop/Record_Evolution/arrow/ reader-writer.cc deps.log"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo -e "extracting sources from/for \n1: ${repo}\n2: ${main}\n3: ${depf}\n"
|
|
||||||
|
|
||||||
# make sure the dependency file is empty
|
|
||||||
rm -f ${depf}
|
|
||||||
touch ${depf}
|
|
||||||
|
|
||||||
# define maximal recursion depth
|
|
||||||
maxdep=8
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
# define function to list dependencies of source file in repository recursively
|
|
||||||
listDependencies()
|
|
||||||
{
|
|
||||||
rep="$1"
|
|
||||||
src="$2"
|
|
||||||
dep="$3"
|
|
||||||
rec="$4"
|
|
||||||
|
|
||||||
echo -e "\nstarting 'listDependencies()' for\n1. ${rep}\n2. ${src}\n3. ${dep}\n4. ${rec}"
|
|
||||||
|
|
||||||
# generate dependency file (and remove resulting object file)
|
|
||||||
echo -e "g++ -c -MMD ${src} -I ${rep}cpp/src/\n"
|
|
||||||
g++ -c -MMD ${src} -I ${rep}cpp/src/
|
|
||||||
|
|
||||||
# derive name of dependency and object files
|
|
||||||
depf=$(basename ${src} | sed 's/.cc/.d/g')
|
|
||||||
objf=$(basename ${src} | sed 's/.cc/.o/g')
|
|
||||||
rm ${objf}
|
|
||||||
|
|
||||||
# list dependencies by
|
|
||||||
# 1. removing header
|
|
||||||
# 2. remove source itself
|
|
||||||
# 3. delete leading spaces
|
|
||||||
# 4. delete trailing backslashs
|
|
||||||
# 5. remove empty lines
|
|
||||||
cat ${depf} | grep ${rep} | grep -v ${src} | tr -d "^ " | tr -d "\\\\" | awk 'NF' > listdep.log
|
|
||||||
# rm ${depf}
|
|
||||||
|
|
||||||
while IFS= read -r fs
|
|
||||||
do
|
|
||||||
echo "$fs"
|
|
||||||
# check if dependency is already in the list
|
|
||||||
if grep -Fxq "$fs" "$dep"
|
|
||||||
then
|
|
||||||
echo "dep exist"
|
|
||||||
else
|
|
||||||
echo "dep does not exist yet => adding it"
|
|
||||||
# add dependency to list
|
|
||||||
echo "$fs" >> ${dep}
|
|
||||||
# check for corresponding source file
|
|
||||||
fssourc=$(echo ${fs} | sed 's/.h$/.cc/g' | sed 's/.hpp$/.cpp/g')
|
|
||||||
echo ${fssourc}
|
|
||||||
if [ -f "$fssourc" ]
|
|
||||||
then
|
|
||||||
echo "source file exists"
|
|
||||||
# list nested dependencies
|
|
||||||
if [ "$rec" -lt "$maxdep" ]
|
|
||||||
then
|
|
||||||
# increment recursion depth
|
|
||||||
recinc=$(($rec+1))
|
|
||||||
# call recursion
|
|
||||||
listDependencies ${rep} ${fssourc} ${dep} ${recinc}
|
|
||||||
else
|
|
||||||
echo "maximal recursion depth exceeded"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "source file does not exist"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
echo ""
|
|
||||||
done < listdep.log
|
|
||||||
|
|
||||||
# cat listdep.log | while read fs
|
|
||||||
# do
|
|
||||||
# echo $fs
|
|
||||||
# # check if dependency is already in the list
|
|
||||||
# inlist=$(cat listdep.log | grep ${fs} | wc -l)
|
|
||||||
# echo ${inlist}
|
|
||||||
# # check for any corresponding source files
|
|
||||||
# # if [ -f ]
|
|
||||||
# done
|
|
||||||
}
|
|
||||||
|
|
||||||
#-----------------------------------------------------------------------------#
|
|
||||||
|
|
||||||
# call function to list dependencies (recursively)
|
|
||||||
listDependencies ${repo} ${main} ${depf} 0
|
|
||||||
|
|
||||||
# # generate dependency file (and remove resulting object file)
|
|
||||||
# echo -e "generate dependencies:\ng++ -c -MMD ${main} -I ./ -I ${repo}cpp/src/\n"
|
|
||||||
# g++ -c -MMD ${main} -I ${repo}cpp/src/
|
|
||||||
# rm $(echo ${main} | sed 's/.cc/.o/g')
|
|
||||||
#
|
|
||||||
# # derive name of dependency file
|
|
||||||
# dep=$(echo ${main} | sed 's/.cc/.d/g')
|
|
||||||
#
|
|
||||||
# if [ -f "$dep" ]; then
|
|
||||||
#
|
|
||||||
# # list dependencies
|
|
||||||
# cat ${dep} | sed 's/ /\n/g' | awk 'NF' | grep -v '\\' | grep '\/' > deps.log
|
|
||||||
#
|
|
||||||
# # extract list of headers
|
|
||||||
# cat deps.log | sed ':a;N;$!ba;s/\n/ /g' > deps-headers.log
|
|
||||||
# echo "list of required headers ($(cat deps.log | wc -l))"
|
|
||||||
# cat deps-headers.log
|
|
||||||
# echo ""
|
|
||||||
#
|
|
||||||
# # imply list of sources
|
|
||||||
# cat deps.log | sed 's/.h$/.cc/g' | sed 's/.hpp$/.cpp/g' > sources_raw.log
|
|
||||||
# cat sources_raw.log | while read f
|
|
||||||
# do
|
|
||||||
# if [ -f "$f" ]; then
|
|
||||||
# echo $f >> sources_check.log
|
|
||||||
# fi
|
|
||||||
# done
|
|
||||||
# cat sources_check.log | sed ':a;N;$!ba;s/\n/ /g' > deps-sources.log
|
|
||||||
# echo "list of required sources ($(cat sources_check.log | wc -l))"
|
|
||||||
# cat deps-sources.log
|
|
||||||
# echo ""
|
|
||||||
#
|
|
||||||
# # remove all temporary files
|
|
||||||
# rm ${dep} deps.log
|
|
||||||
# rm sources_raw.log sources_check.log
|
|
||||||
#
|
|
||||||
# # copy required headers and sources
|
|
||||||
# echo -e "copy required headers and sources"
|
|
||||||
# mkdir temp/
|
|
||||||
# cp --parents `cat deps-headers.log` temp/
|
|
||||||
# cp --parents `cat deps-sources.log` temp/
|
|
||||||
# mv temp${repo}cpp/src/* ./
|
|
||||||
# rm -r temp
|
|
||||||
#
|
|
||||||
# # remove dependencies
|
|
||||||
# #rm deps-headers.log deps-sources.log
|
|
||||||
#
|
|
||||||
# # show files
|
|
||||||
# ls -lh
|
|
||||||
#
|
|
||||||
# else
|
|
||||||
#
|
|
||||||
# echo -e "\nERROR: failed to generate dependency file\n"
|
|
||||||
#
|
|
||||||
# fi
|
|
||||||
@ -1,26 +0,0 @@
|
|||||||
|
|
||||||
FROM ubuntu:19.10
|
|
||||||
|
|
||||||
RUN apt-get update -y && apt-get install -y \
|
|
||||||
apt-utils \
|
|
||||||
git g++ \
|
|
||||||
make cmake \
|
|
||||||
pkg-config \
|
|
||||||
#build-essentials \
|
|
||||||
python3 \
|
|
||||||
python3-setuptools \
|
|
||||||
cython3 \
|
|
||||||
python3-numpy
|
|
||||||
|
|
||||||
RUN git clone https://github.com/apache/arrow.git --single-branch --depth=1
|
|
||||||
|
|
||||||
COPY . ./
|
|
||||||
|
|
||||||
RUN chmod u+x ./build_arrow_cpp.sh
|
|
||||||
RUN chmod u+x ./build_arrow_python.sh
|
|
||||||
RUN ./build_arrow_cpp.sh
|
|
||||||
RUN ./build_arrow_python.sh
|
|
||||||
|
|
||||||
#RUN chmod u+x ./build_arrow.sh
|
|
||||||
#CMD ["./build_arrow.sh"]
|
|
||||||
CMD ["sleep 1d"]
|
|
||||||
@ -1,5 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
docker build . --tag=pyarrowbuild:latest
|
|
||||||
|
|
||||||
docker run -it pyarrowbuild:latest /bin/bash
|
|
||||||
@ -1,65 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
sleep infinity
|
|
||||||
|
|
||||||
startts=$(date)
|
|
||||||
echo "starting build process at ${startts}..."
|
|
||||||
|
|
||||||
echo -e "\nhome directory is..."
|
|
||||||
pwd
|
|
||||||
|
|
||||||
echo -e "\ncloning apache/arrow..."
|
|
||||||
git clone https://github.com/apache/arrow.git --single-branch --depth=1
|
|
||||||
|
|
||||||
echo -e "\nls -lh /\n"
|
|
||||||
ls -lh /
|
|
||||||
|
|
||||||
echo -e "\nls -lh arrow/\n"
|
|
||||||
ls -lh arrow/
|
|
||||||
|
|
||||||
echo -e "\nls -lh arrow/python/\n"
|
|
||||||
ls -lh arrow/python
|
|
||||||
|
|
||||||
|
|
||||||
mkdir arrow/cpp/build
|
|
||||||
pushd arrow/cpp/build
|
|
||||||
|
|
||||||
cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
|
|
||||||
-DCMAKE_INSTALL_LIBDIR=lib \
|
|
||||||
-DARROW_WITH_BZ2=ON \
|
|
||||||
-DARROW_WITH_ZLIB=ON \
|
|
||||||
-DARROW_WITH_ZSTD=ON \
|
|
||||||
-DARROW_WITH_LZ4=ON \
|
|
||||||
-DARROW_WITH_SNAPPY=ON \
|
|
||||||
-DARROW_WITH_BROTLI=ON \
|
|
||||||
-DARROW_PARQUET=ON \
|
|
||||||
-DARROW_PYTHON=ON \
|
|
||||||
-DARROW_BUILD_TESTS=OFF \
|
|
||||||
-DARROW_WITH_HDFS=OFF \
|
|
||||||
..
|
|
||||||
|
|
||||||
make -j4
|
|
||||||
make install
|
|
||||||
popd
|
|
||||||
|
|
||||||
#cython --version
|
|
||||||
cython3 --version
|
|
||||||
|
|
||||||
pushd arrow/python
|
|
||||||
export ARROW_LIB_DIR=/lib/
|
|
||||||
export PYARROW_WITH_PARQUET=1
|
|
||||||
export PYARROW_WITH_CUDA=0
|
|
||||||
export PYARROW_WITH_FlIGHT=0
|
|
||||||
export PYARROW_WITH_DATASET=0
|
|
||||||
export PYARROW_WITH_ORC=0
|
|
||||||
export PYARROW_WITH_PLASMA=0
|
|
||||||
export PYARROW_WITH_S3FS=0
|
|
||||||
export PYARROW_WITH_HDFS=0
|
|
||||||
export PYARROW_WITH_GANDIVA=0
|
|
||||||
python3 setup.py build_ext --inplace
|
|
||||||
popd
|
|
||||||
|
|
||||||
echo " started build process at ${startts} ..."
|
|
||||||
finishts=$(date)
|
|
||||||
echo "finishing build process at ${finishts}..."
|
|
||||||
|
|
||||||
@ -1,23 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
mkdir arrow/cpp/build
|
|
||||||
pushd arrow/cpp/build
|
|
||||||
|
|
||||||
cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
|
|
||||||
-DCMAKE_INSTALL_LIBDIR=lib \
|
|
||||||
-DARROW_WITH_BZ2=ON \
|
|
||||||
-DARROW_WITH_ZLIB=ON \
|
|
||||||
-DARROW_WITH_ZSTD=ON \
|
|
||||||
-DARROW_WITH_LZ4=ON \
|
|
||||||
-DARROW_WITH_SNAPPY=ON \
|
|
||||||
-DARROW_WITH_BROTLI=ON \
|
|
||||||
-DARROW_PARQUET=ON \
|
|
||||||
-DARROW_PYTHON=ON \
|
|
||||||
-DARROW_BUILD_TESTS=OFF \
|
|
||||||
-DARROW_WITH_HDFS=OFF \
|
|
||||||
-DARROW_WITH_IPC=OFF \
|
|
||||||
..
|
|
||||||
|
|
||||||
make -j4
|
|
||||||
make install
|
|
||||||
popd
|
|
||||||
@ -1,15 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
pushd arrow/python
|
|
||||||
export PYARROW_WITH_PARQUET=1
|
|
||||||
export PYARROW_WITH_CUDA=0
|
|
||||||
export PYARROW_WITH_FlIGHT=0
|
|
||||||
export PYARROW_WITH_DATASET=0
|
|
||||||
export PYARROW_WITH_ORC=0
|
|
||||||
export PYARROW_WITH_PLASMA=0
|
|
||||||
export PYARROW_WITH_S3FS=0
|
|
||||||
export PYARROW_WITH_HDFS=0
|
|
||||||
export PYARROW_WITH_GANDIVA=0
|
|
||||||
# python3 setup.py build_ext --inplace
|
|
||||||
python3 setup.py install
|
|
||||||
popd
|
|
||||||
@ -1,23 +0,0 @@
|
|||||||
|
|
||||||
build :
|
|
||||||
docker build . --tag pyarrowbuild
|
|
||||||
|
|
||||||
run :
|
|
||||||
docker run -it pyarrowbuild:latest
|
|
||||||
|
|
||||||
run-bash :
|
|
||||||
docker run -it --volume=$(pwd)/build:/home pyarrowbuild:latest /bin/bash
|
|
||||||
|
|
||||||
run-volume :
|
|
||||||
docker run -it -v /home/pirate/pyarrow/build/:/arrow/python/ pyarrowbuild:latest
|
|
||||||
#sudo docker run -it --volume=$(pwd)/build:/home ubuntu:latest /bin/bash
|
|
||||||
|
|
||||||
rm-container :
|
|
||||||
cont=$(docker ps -a | tail -n 26 | awk '{print $NF}' | sed ':a;N;$!ba;s/\n/ /g')
|
|
||||||
echo ${cont}
|
|
||||||
docker rm ${cont}
|
|
||||||
|
|
||||||
rm-image :
|
|
||||||
img=$(docker image ls --quiet | sed ':a;N;$!ba;s/\n/ /g')
|
|
||||||
docker image rm ${img}
|
|
||||||
|
|
||||||
@ -1,18 +0,0 @@
|
|||||||
|
|
||||||
import pyarrow.parquet as pq
|
|
||||||
import pyarrow.csv as pv
|
|
||||||
|
|
||||||
csvfile = 'pressureVacuum.csv'
|
|
||||||
|
|
||||||
tb = pv.read_csv(csvfile,parse_options=pv.ParseOptions(delimiter=','))
|
|
||||||
|
|
||||||
print(tb)
|
|
||||||
|
|
||||||
parquetfile = 'pressureVacuum.parquet'
|
|
||||||
|
|
||||||
pq.write_table(tb,parquetfile,compression='BROTLI')
|
|
||||||
# {'NONE', 'SNAPPY', 'GZIP', 'LZO', 'BROTLI', 'LZ4', 'ZSTD'}
|
|
||||||
|
|
||||||
df = pq.read_table(parquetfile,columns=None)
|
|
||||||
|
|
||||||
print(df)
|
|
||||||
@ -1,8 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
if [ -z "$1" ]
|
|
||||||
then
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
scp $1 pirate@mf-pi-40:/home/pirate/pyarrow/
|
|
||||||
@ -1,4 +0,0 @@
|
|||||||
include *.hpp
|
|
||||||
include *.cpp
|
|
||||||
include *.pyx
|
|
||||||
include *.pxd
|
|
||||||
@ -1,4 +0,0 @@
|
|||||||
[build-system]
|
|
||||||
requires = [
|
|
||||||
"setuptools"
|
|
||||||
]
|
|
||||||
37
pip/makefile
37
pip/makefile
@ -1,37 +0,0 @@
|
|||||||
# --------------------------------------------------------------------------- #
|
|
||||||
|
|
||||||
SHELL := /bin/bash
|
|
||||||
|
|
||||||
publish-source: sdist upload
|
|
||||||
publish-binary: bdist upload
|
|
||||||
|
|
||||||
prepdist: ../cython/py_imc_termite.pyx ../cython/imc_termite.pxd ../cython/py_imc_termite.cpp
|
|
||||||
cp -v $? ./
|
|
||||||
cp -v $(shell ls ../lib/imc_*.hpp) ./
|
|
||||||
cat ../README.md | grep '^# IMCtermite' -A 50000 > ./README.md
|
|
||||||
cp -v ../LICENSE ./
|
|
||||||
|
|
||||||
sdist: prepdist
|
|
||||||
python3 setup.py sdist
|
|
||||||
|
|
||||||
# TODO use manylinux wheel to avoid ERROR "unsupported platform tag 'linux_x86_64'"
|
|
||||||
# see: - https://github.com/pypa/manylinux
|
|
||||||
bdist: prepdist
|
|
||||||
python3 setup.py bdist_wheel
|
|
||||||
|
|
||||||
# authentication:
|
|
||||||
# - username: __token__
|
|
||||||
# - password: <token value including pypi-prefix>
|
|
||||||
upload:
|
|
||||||
python3 -m twine upload dist/$(shell ls -t dist/ | head -n1)
|
|
||||||
|
|
||||||
clean:
|
|
||||||
rm -rvf dist/
|
|
||||||
rm -rvf *.egg-info
|
|
||||||
rm -rvf build/
|
|
||||||
rm -rvf cython/
|
|
||||||
rm -vf *.pyx *.pxd
|
|
||||||
rm -vf *.cpp *.c *.hpp
|
|
||||||
rm -vf README.md LICENSE
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------- #
|
|
||||||
49
pip/setup.py
49
pip/setup.py
@ -1,49 +0,0 @@
|
|||||||
|
|
||||||
from setuptools import setup, Extension
|
|
||||||
import sys
|
|
||||||
|
|
||||||
print("building on platform: "+sys.platform)
|
|
||||||
|
|
||||||
if sys.platform == "linux" or sys.platform == "darwin" :
|
|
||||||
cmpargs = ['-std=c++17','-Wno-unused-variable']
|
|
||||||
lnkargs = ['-std=c++17','-Wno-unused-variable']
|
|
||||||
elif sys.platform == "win32" :
|
|
||||||
cmpargs = ['/EHsc','/std:c++17']
|
|
||||||
lnkargs = []
|
|
||||||
else :
|
|
||||||
raise RuntimeError("unknown platform")
|
|
||||||
|
|
||||||
with open("README.md", "r", encoding="utf-8") as fh:
|
|
||||||
long_description = fh.read()
|
|
||||||
|
|
||||||
setup(
|
|
||||||
name="IMCtermite",
|
|
||||||
version="1.2.12",
|
|
||||||
author="Record Evolution GmbH",
|
|
||||||
author_email="mario.fink@record-evolution.de",
|
|
||||||
maintainer="Record Evolution GmbH",
|
|
||||||
license="MIT",
|
|
||||||
description="Enables extraction of measurement data from binary files with extension 'raw' used by proprietary software imcFAMOS/imcSTUDIO and facilitates its storage in open source file formats",
|
|
||||||
keywords="IMC raw imcFAMOS imcSTUDIO imcCRONOS",
|
|
||||||
long_description=long_description,
|
|
||||||
long_description_content_type="text/markdown",
|
|
||||||
url="https://github.com/RecordEvolution/IMCtermite.git",
|
|
||||||
project_urls={
|
|
||||||
"Bug Tracker": "https://github.com/RecordEvolution/IMCtermite/issues",
|
|
||||||
},
|
|
||||||
classifiers=[
|
|
||||||
"Programming Language :: Python :: 3",
|
|
||||||
"License :: OSI Approved :: MIT License",
|
|
||||||
"Operating System :: OS Independent",
|
|
||||||
],
|
|
||||||
ext_modules=[Extension("imc_termite",
|
|
||||||
["py_imc_termite.cpp"],
|
|
||||||
# libraries_dirs=["cython/"],
|
|
||||||
# include_dirs=["3rdparty/pugixml/","lib/"],
|
|
||||||
# depends=["../lib/tdm_termite.hpp"]
|
|
||||||
language='c++',
|
|
||||||
extra_compile_args=cmpargs,
|
|
||||||
extra_link_args=lnkargs,
|
|
||||||
)
|
|
||||||
],
|
|
||||||
)
|
|
||||||
@ -0,0 +1,4 @@
|
|||||||
|
include *.hpp
|
||||||
|
include *.cpp
|
||||||
|
include *.pyx
|
||||||
|
include *.pxd
|
||||||
Loading…
x
Reference in New Issue
Block a user