diff --git a/go.mod b/go.mod index 4758fe00..e363ba63 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,8 @@ require ( github.com/tidwall/btree v1.6.0 github.com/tidwall/gjson v1.10.2 github.com/tidwall/wal v1.1.7 + github.com/xitongsys/parquet-go v1.6.2 + github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 github.com/zbiljic/go-filelock v0.0.0-20170914061330-1dbf7103ab7d golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb modernc.org/sqlite v1.26.0 @@ -24,6 +26,8 @@ require ( require ( github.com/DataDog/zstd v1.4.5 // indirect + github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect + github.com/apache/thrift v0.14.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -58,6 +62,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/onsi/gomega v1.20.0 // indirect github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect + github.com/pierrec/lz4/v4 v4.1.8 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect @@ -80,6 +85,7 @@ require ( golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/tools v0.6.0 // indirect + golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/uint128 v1.2.0 // indirect diff --git a/go.sum b/go.sum index 89f72143..b6394ebd 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,11 @@ github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= +github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 h1:byKBBF2CKWBjjA4J1ZL2JXttJULvWSl50LegTyRZ728= +github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= +github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.14.2 h1:hY4rAyg7Eqbb27GB6gkhUKrRAuc8xRjlNtJq+LseKeY= +github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -133,6 +138,7 @@ github.com/ashanbrown/forbidigo v1.3.0/go.mod h1:vVW7PEdqEFqapJe95xHkTfB1+XvZXBF github.com/ashanbrown/makezero v1.1.1/go.mod h1:i1bJLCRSCHOcOa9Y6MyF2FTfMZMFdHvxKHxgO5Z1axI= github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.36.30/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= @@ -216,6 +222,7 @@ github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= github.com/confio/ics23/go v0.9.0 h1:cWs+wdbS2KRPZezoaaj+qBleXgUk5WOQFMP3CQFGTr4= github.com/confio/ics23/go v0.9.0/go.mod h1:4LPZ2NYqnYIVRklaozjNR1FScgDJ2s5Xrp+e/mYVRak= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= @@ -442,6 +449,7 @@ github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= github.com/google/certificate-transparency-go v1.1.1/go.mod h1:FDKqPvSXawb2ecErVRrD+nfy23RCzyl7eqVCEmlT1Zs= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -559,6 +567,7 @@ github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -601,6 +610,7 @@ github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbk github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a/go.mod h1:Zi/ZFkEqFHTm7qkjyNJjaWH4LQA9LQhGJyF0lTYGpxw= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -615,6 +625,7 @@ github.com/jhump/protoreflect v1.12.1-0.20220417024638-438db461d753/go.mod h1:Jy github.com/jingyugao/rowserrcheck v1.1.1/go.mod h1:4yvlZSDb3IyDTUZJUmpZfm2Hwok+Dtp+nu2qOq+er9c= github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= @@ -655,7 +666,9 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= @@ -852,6 +865,7 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -866,7 +880,10 @@ github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 h1:hDSdbBuw3Lefr6 github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1 h1:VGcrWe3yk6o+t7BdVNy5UDPWa4OZuDWtE1W1ZbS7Kyw= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4/v4 v4.1.8 h1:ieHkV+i2BRzngO4Wd/3HGowuZStgq6QkPsD1eolNAO4= +github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= @@ -1000,6 +1017,7 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= @@ -1033,6 +1051,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -1106,6 +1125,12 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= +github.com/xitongsys/parquet-go v1.6.2 h1:MhCaXii4eqceKPu9BwrjLqyK10oX9WF+xGhwvwbw7xM= +github.com/xitongsys/parquet-go v1.6.2/go.mod h1:IulAQyalCm0rPiZVNnCgm/PCL64X2tdSVGMQ/UeKqWA= +github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= +github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 h1:a742S4V5A15F93smuVxA60LQWsrCnN8bKeWDBARU1/k= +github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk= @@ -1176,6 +1201,7 @@ go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1617,6 +1643,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= @@ -1832,6 +1859,11 @@ gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/R gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= diff --git a/ss/parquet/backend.go b/ss/parquet/backend.go new file mode 100644 index 00000000..d68887ee --- /dev/null +++ b/ss/parquet/backend.go @@ -0,0 +1,112 @@ +package parquet + +import ( + "fmt" + + errorutils "github.com/sei-protocol/sei-db/common/errors" + "github.com/sei-protocol/sei-db/proto" + "github.com/sei-protocol/sei-db/ss/types" +) + +// ParquetBackend is a wrapper over an underlying StateStore that also writes +// changes to Parquet files partitioned by version and owner for fast analytics/debugging. +type ParquetBackend struct { + underlying types.StateStore + writer *Writer +} + +func NewParquetBackend(underlying types.StateStore, writer *Writer) (*ParquetBackend, error) { + if underlying == nil { + return nil, fmt.Errorf("underlying store cannot be nil") + } + if writer == nil { + return nil, fmt.Errorf("parquet writer cannot be nil") + } + return &ParquetBackend{underlying: underlying, writer: writer}, nil +} + +func (p *ParquetBackend) Close() error { + var err error + if p.writer != nil { + err = p.writer.Close() + } + if p.underlying != nil { + if errU := p.underlying.Close(); err == nil { + err = errU + } else if errU != nil { + err = errorutils.Join(err, errU) + } + } + return err +} + +// Delegated read APIs +func (p *ParquetBackend) Get(storeKey string, version int64, key []byte) ([]byte, error) { + return p.underlying.Get(storeKey, version, key) +} +func (p *ParquetBackend) Has(storeKey string, version int64, key []byte) (bool, error) { + return p.underlying.Has(storeKey, version, key) +} +func (p *ParquetBackend) Iterator(storeKey string, version int64, start, end []byte) (types.DBIterator, error) { + return p.underlying.Iterator(storeKey, version, start, end) +} +func (p *ParquetBackend) ReverseIterator(storeKey string, version int64, start, end []byte) (types.DBIterator, error) { + return p.underlying.ReverseIterator(storeKey, version, start, end) +} +func (p *ParquetBackend) RawIterate(storeKey string, fn func([]byte, []byte, int64) bool) (bool, error) { + return p.underlying.RawIterate(storeKey, fn) +} +func (p *ParquetBackend) GetLatestVersion() (int64, error) { return p.underlying.GetLatestVersion() } +func (p *ParquetBackend) SetLatestVersion(version int64) error { return p.underlying.SetLatestVersion(version) } +func (p *ParquetBackend) GetEarliestVersion() (int64, error) { return p.underlying.GetEarliestVersion() } +func (p *ParquetBackend) SetEarliestVersion(version int64, ignoreVersion bool) error { + return p.underlying.SetEarliestVersion(version, ignoreVersion) +} +func (p *ParquetBackend) GetLatestMigratedKey() ([]byte, error) { return p.underlying.GetLatestMigratedKey() } +func (p *ParquetBackend) SetLatestMigratedKey(key []byte) error { return p.underlying.SetLatestMigratedKey(key) } +func (p *ParquetBackend) GetLatestMigratedModule() (string, error) { + return p.underlying.GetLatestMigratedModule() +} +func (p *ParquetBackend) SetLatestMigratedModule(module string) error { + return p.underlying.SetLatestMigratedModule(module) +} +func (p *ParquetBackend) WriteBlockRangeHash(storeKey string, beginBlockRange, endBlockRange int64, hash []byte) error { + return p.underlying.WriteBlockRangeHash(storeKey, beginBlockRange, endBlockRange, hash) +} +func (p *ParquetBackend) DeleteKeysAtVersion(module string, version int64) error { + return p.underlying.DeleteKeysAtVersion(module, version) +} + +// Mutating APIs with Parquet side-writes +func (p *ParquetBackend) ApplyChangeset(version int64, cs *proto.NamedChangeSet) error { + if err := p.underlying.ApplyChangeset(version, cs); err != nil { + return err + } + // Best-effort Parquet write after underlying commit + if err := p.writer.WriteChangeSet(version, cs); err != nil { + // Do not fail core path; surface error for logging upstream + return errorutils.Join(nil, fmt.Errorf("parquet write failed: %w", err)) + } + return nil +} + +func (p *ParquetBackend) ApplyChangesetAsync(version int64, changesets []*proto.NamedChangeSet) error { + // Fire parquet writes immediately as best-effort + for _, cs := range changesets { + _ = p.writer.WriteChangeSet(version, cs) + } + return p.underlying.ApplyChangesetAsync(version, changesets) +} + +func (p *ParquetBackend) Import(version int64, ch <-chan types.SnapshotNode) error { + // Delegate import to underlying. Parquet writes for full import are optional; skip for speed. + return p.underlying.Import(version, ch) +} + +func (p *ParquetBackend) RawImport(ch <-chan types.RawSnapshotNode) error { + // Delegate raw import to underlying. + return p.underlying.RawImport(ch) +} + +func (p *ParquetBackend) Prune(version int64) error { return p.underlying.Prune(version) } + diff --git a/ss/parquet/writer.go b/ss/parquet/writer.go new file mode 100644 index 00000000..37821197 --- /dev/null +++ b/ss/parquet/writer.go @@ -0,0 +1,137 @@ +package parquet + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/sei-protocol/sei-db/proto" + "github.com/xitongsys/parquet-go/parquet" + "github.com/xitongsys/parquet-go-source/local" + "github.com/xitongsys/parquet-go/writer" +) + +// OwnerExtractor determines the owner string for a KV pair. +// Implementations should be deterministic and stable across versions. +type OwnerExtractor func(storeKey string, key []byte, value []byte) string + +// Writer writes change rows to partitioned Parquet files using HDFS-style layout. +type Writer struct { + baseDir string + ownerExtractor OwnerExtractor +} + +func NewWriter(baseDir string, extractor OwnerExtractor) (*Writer, error) { + if baseDir == "" { + return nil, fmt.Errorf("baseDir cannot be empty") + } + if extractor == nil { + extractor = func(storeKey string, key []byte, value []byte) string { return "" } + } + if err := os.MkdirAll(baseDir, os.ModePerm); err != nil { + return nil, err + } + return &Writer{baseDir: baseDir, ownerExtractor: extractor}, nil +} + +// ChangeRow defines the schema for Parquet rows. +type ChangeRow struct { + Version int64 `parquet:"name=version, type=INT64"` + StoreKey string `parquet:"name=store_key, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` + Key []byte `parquet:"name=key, type=BYTE_ARRAY"` + Value []byte `parquet:"name=value, type=BYTE_ARRAY"` + Tombstone bool `parquet:"name=tombstone, type=BOOLEAN"` + Owner string `parquet:"name=owner, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` +} + +// WriteChangeSet writes the provided changes into Parquet files partitioned by version and owner. +func (w *Writer) WriteChangeSet(version int64, cs *proto.NamedChangeSet) error { + // Keep version >= 1 in parity with MVCC backends + if version == 0 { + version = 1 + } + if cs == nil || cs.Changeset.Pairs == nil || len(cs.Changeset.Pairs) == 0 { + return nil + } + + // Group rows by owner + grouped := map[string][]*ChangeRow{} + for _, kv := range cs.Changeset.Pairs { + tomb := kv.Value == nil + val := kv.Value + if tomb { + val = []byte{} + } + owner := strings.TrimSpace(w.ownerExtractor(cs.Name, kv.Key, kv.Value)) + row := &ChangeRow{ + Version: version, + StoreKey: cs.Name, + Key: kv.Key, + Value: val, + Tombstone: tomb, + Owner: owner, + } + grouped[owner] = append(grouped[owner], row) + } + + for owner, rows := range grouped { + if err := w.writePartition(version, owner, rows); err != nil { + return err + } + } + return nil +} + +func (w *Writer) writePartition(version int64, owner string, rows []*ChangeRow) error { + // Build HDFS-style partition path: version=/owner= + partDir := filepath.Join(w.baseDir, fmt.Sprintf("version=%d", version)) + ownerValue := owner + if ownerValue == "" { + ownerValue = "_" + } + partDir = filepath.Join(partDir, fmt.Sprintf("owner=%s", ownerValue)) + if err := os.MkdirAll(partDir, os.ModePerm); err != nil { + return err + } + + // Unique file name + suf := randomHex(8) + filePath := filepath.Join(partDir, fmt.Sprintf("part-%s.parquet", suf)) + + fw, err := local.NewLocalFileWriter(filePath) + if err != nil { + return fmt.Errorf("open parquet file: %w", err) + } + defer fw.Close() + + pw, err := writer.NewParquetWriter(fw, new(ChangeRow), 2) + if err != nil { + return fmt.Errorf("create parquet writer: %w", err) + } + pw.RowGroupSize = 8 * 1024 * 1024 // 8MB row groups by default + pw.CompressionType = parquet.CompressionCodec_SNAPPY + + for _, r := range rows { + if err := pw.Write(r); err != nil { + _ = pw.WriteStop() + return fmt.Errorf("write parquet row: %w", err) + } + } + if err := pw.WriteStop(); err != nil { + return fmt.Errorf("finalize parquet: %w", err) + } + return nil +} + +func (w *Writer) Close() error { return nil } + +// randomHex returns a random hex string of n bytes length. +func randomHex(n int) string { + b := make([]byte, n) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + diff --git a/ss/parquet_init.go b/ss/parquet_init.go new file mode 100644 index 00000000..39ebc663 --- /dev/null +++ b/ss/parquet_init.go @@ -0,0 +1,38 @@ +package ss + +import ( + "path/filepath" + + "github.com/sei-protocol/sei-db/common/utils" + "github.com/sei-protocol/sei-db/config" + parquetbackend "github.com/sei-protocol/sei-db/ss/parquet" + "github.com/sei-protocol/sei-db/ss/pebbledb" + "github.com/sei-protocol/sei-db/ss/types" +) + +func init() { + initializer := func(dir string, configs config.StateStoreConfig) (types.StateStore, error) { + // Underlying KV store (Pebble) + kvDir := utils.GetStateStorePath(dir, "pebbledb") + parquetDir := utils.GetStateStorePath(dir, "parquet_data") + if configs.DBDirectory != "" { + kvDir = filepath.Join(configs.DBDirectory, "pebbledb") + parquetDir = filepath.Join(configs.DBDirectory, "parquet_data") + } + + kv, err := pebbledb.New(kvDir, configs) + if err != nil { + return nil, err + } + + writer, err := parquetbackend.NewWriter(parquetDir, nil) + if err != nil { + _ = kv.Close() + return nil, err + } + + return parquetbackend.NewParquetBackend(kv, writer) + } + RegisterBackend(ParquetBackend, initializer) +} + diff --git a/ss/store.go b/ss/store.go index a472c867..c2db87a0 100644 --- a/ss/store.go +++ b/ss/store.go @@ -24,6 +24,9 @@ const ( // SQLiteBackend represents sqlite SQLiteBackend BackendType = "sqlite" + + // ParquetBackend represents a Parquet-writer wrapper around a KV backend + ParquetBackend BackendType = "parquet" ) type BackendInitializer func(dir string, config config.StateStoreConfig) (types.StateStore, error)