From 1062e68353a4cc406a0321bdfcabfe6fddaf6aad Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 5 Jan 2024 10:08:00 +0530 Subject: [PATCH 1/2] fix: migration for removing old materialized columns --- migration-0.38/Dockerfile | 43 +++++++ migration-0.38/README.md | 3 + migration-0.38/go.mod | 19 +++ migration-0.38/go.sum | 134 ++++++++++++++++++++ migration-0.38/main.go | 252 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 451 insertions(+) create mode 100644 migration-0.38/Dockerfile create mode 100644 migration-0.38/README.md create mode 100644 migration-0.38/go.mod create mode 100644 migration-0.38/go.sum create mode 100644 migration-0.38/main.go diff --git a/migration-0.38/Dockerfile b/migration-0.38/Dockerfile new file mode 100644 index 0000000..beb8be6 --- /dev/null +++ b/migration-0.38/Dockerfile @@ -0,0 +1,43 @@ +FROM golang:1.18-alpine AS builder + +RUN apk --update add \ + util-linux-dev \ + build-base \ + musl-dev + +ARG TARGETPLATFORM + +ENV CGO_ENABLED=0 +ENV GOPATH=/go + +RUN export GOOS=$(echo ${TARGETPLATFORM} | cut -d / -f1) && \ + export GOARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) + +# Prepare and enter src directory +WORKDIR /go/src/github.com/signoz/migrate + +# Cache dependencies +ADD go.mod . +ADD go.sum . +RUN go mod download -x + +# Add the sources and proceed with build +ADD . . +RUN go build -o ./bin/migrate -a -ldflags "-linkmode external -extldflags '-static' -s -w" +RUN chmod +x ./bin/migrate + + +# use a minimal alpine image +FROM alpine:3.7 + +# Add Maintainer Info +LABEL maintainer="signoz" + +# add ca-certificates in case you need them +RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/* + +# copy the binary from builder +COPY --from=builder /go/src/github.com/signoz/migrate/bin/migrate /usr/local/bin/migrate + +# run the binary +ENTRYPOINT ["/usr/local/bin/migrate"] \ No newline at end of file diff --git a/migration-0.38/README.md b/migration-0.38/README.md new file mode 100644 index 0000000..eb4e5a1 --- /dev/null +++ b/migration-0.38/README.md @@ -0,0 +1,3 @@ + + +This migration removes the duplicate underscore materialized columns. \ No newline at end of file diff --git a/migration-0.38/go.mod b/migration-0.38/go.mod new file mode 100644 index 0000000..5a338c5 --- /dev/null +++ b/migration-0.38/go.mod @@ -0,0 +1,19 @@ +module go.signoz.io/migrate-0.11 + +go 1.17 + +require ( + github.com/ClickHouse/clickhouse-go/v2 v2.2.0 + go.uber.org/zap v1.21.0 +) + +require ( + github.com/google/uuid v1.3.0 // indirect + github.com/paulmach/orb v0.7.1 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/shopspring/decimal v1.3.1 // indirect + go.opentelemetry.io/otel v1.8.0 // indirect + go.opentelemetry.io/otel/trace v1.8.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.8.0 // indirect +) diff --git a/migration-0.38/go.sum b/migration-0.38/go.sum new file mode 100644 index 0000000..2b6678b --- /dev/null +++ b/migration-0.38/go.sum @@ -0,0 +1,134 @@ +github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= +github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/ClickHouse/clickhouse-go/v2 v2.2.0 h1:dj00TDKY+xwuTJdbpspCSmTLFyWzRJerTHwaBxut1C0= +github.com/ClickHouse/clickhouse-go/v2 v2.2.0/go.mod h1:8f2XZUi7XoeU+uPIytSi1cvx8fmJxi7vIgqpvYTF1+o= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= +github.com/paulmach/orb v0.7.1 h1:Zha++Z5OX/l168sqHK3k4z18LDvr+YAO/VjK0ReQ9rU= +github.com/paulmach/orb v0.7.1/go.mod h1:FWRlTgl88VI1RBx/MkrwWDRhQ96ctqMCh8boXhmqB/A= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q= +github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= +github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= +go.opentelemetry.io/otel v1.8.0 h1:zcvBFizPbpa1q7FehvFiHbQwGzmPILebO0tyqIR5Djg= +go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= +go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= +go.opentelemetry.io/otel/trace v1.8.0 h1:cSy0DF9eGI5WIfNwZ1q2iUyGj00tGzP24dE1lOlHrfY= +go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/migration-0.38/main.go b/migration-0.38/main.go new file mode 100644 index 0000000..bca846f --- /dev/null +++ b/migration-0.38/main.go @@ -0,0 +1,252 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "regexp" + "strings" + "time" + + clickhouse "github.com/ClickHouse/clickhouse-go/v2" + "go.uber.org/zap" +) + +func connect(host string, port string, userName string, password string) (clickhouse.Conn, error) { + var ( + ctx = context.Background() + conn, err = clickhouse.Open(&clickhouse.Options{ + Addr: []string{fmt.Sprintf("%s:%s", host, port)}, + Auth: clickhouse.Auth{ + Database: "default", + Username: userName, + Password: password, + }, + //Debug: true, + }) + ) + if err != nil { + return nil, err + } + if err := conn.Ping(ctx); err != nil { + if exception, ok := err.(*clickhouse.Exception); ok { + zap.S().Info(fmt.Sprintf("Catch exception [%d] %s \n%s", exception.Code, exception.Message, exception.StackTrace)) + } + return nil, err + } + return conn, nil +} + +type LogField struct { + Name string `json:"name" ch:"name"` + DataType string `json:"dataType" ch:"datatype"` + Type string `json:"type"` + IndexType string `json:"indexType"` +} + +type ShowCreateTableStatement struct { + Statement string `json:"statement" ch:"statement"` +} + +const ( + attribute = "attribute" + resource = "resource" +) + +// remove this after sometime +func getFieldsToBeDeleted(fields []LogField, tablestatement string) []LogField { + lookup := map[string]LogField{} + for _, v := range fields { + lookup[v.Name+v.DataType] = v + } + + updatedFields := []LogField{} + for k := range lookup { + if strings.Contains(k, ".") { + // only capture ones which has a materialized columns + if uVal, ok := lookup[strings.ReplaceAll(k, ".", "_")]; ok { + colname := fmt.Sprintf("%s_%s_%s", strings.ToLower(uVal.Type), strings.ToLower(uVal.DataType), uVal.Name) + if hasMaterializedColumn(tablestatement, colname, uVal.DataType) { + // we will delete the underscore field which is not required + updatedFields = append(updatedFields, uVal) + } + } + + } + } + return updatedFields +} + +func GetFields(conn clickhouse.Conn) ([]LogField, error) { + ctx := context.Background() + // get attribute keys + attributes := []LogField{} + query := "SELECT DISTINCT name, datatype from signoz_logs.distributed_logs_attribute_keys group by name, datatype" + err := conn.Select(ctx, &attributes, query) + if err != nil { + zap.S().Error(fmt.Errorf("error while getting attribute keys. Err=%v", err)) + return nil, err + } + + // get resource keys + resources := []LogField{} + query = "SELECT DISTINCT name, datatype from signoz_logs.distributed_logs_resource_keys group by name, datatype" + err = conn.Select(ctx, &resources, query) + if err != nil { + zap.S().Error(fmt.Errorf("error while getting resource keys. Err=%v", err)) + return nil, err + } + + for i := 0; i < len(attributes); i++ { + attributes[i].Type = attribute + } + + for i := 0; i < len(resources); i++ { + resources[i].Type = resource + } + + statement, err := GetTableStatement(conn) + if err != nil { + zap.S().Error(fmt.Errorf("error while table statement. Err=%v", err)) + return nil, err + } + + attributes = getFieldsToBeDeleted(attributes, statement) + resources = getFieldsToBeDeleted(resources, statement) + + attributes = append(attributes, resources...) + + return attributes, nil +} + +func GetTableStatement(conn clickhouse.Conn) (string, error) { + ctx := context.Background() + statements := []ShowCreateTableStatement{} + query := "SHOW CREATE TABLE signoz_logs.logs" + err := conn.Select(ctx, &statements, query) + if err != nil { + zap.S().Error(fmt.Errorf("error while getting logs table statement. Err=%v", err)) + return "", err + } + return statements[0].Statement, nil +} + +func hasMaterializedColumn(tableStatement, field, dataType string) bool { + regex := fmt.Sprintf("`%s` (?i)(%s) MATERIALIZED", field, dataType) + res, err := regexp.MatchString(regex, tableStatement) + if err != nil { + zap.S().Error(fmt.Errorf("error while matching regex. Err=%v", err)) + return false + } + + if !res { + // try checking for default as well + regex := fmt.Sprintf("`%s` (?i)(%s) DEFAULT", field, dataType) + res, err = regexp.MatchString(regex, tableStatement) + if err != nil { + zap.S().Error(fmt.Errorf("error while matching regex. Err=%v", err)) + return false + } + } + return res +} + +func TruncateKeyTable(conn clickhouse.Conn) error { + query := "truncate table signoz_logs.logs_attribute_keys on cluster cluster;" + err := conn.Exec(context.Background(), query) + if err != nil { + zap.S().Error(fmt.Errorf("error truncating tag attributes table, Err=%v", err)) + return err + } + + query = "truncate table signoz_logs.logs_resource_keys on cluster cluster;" + err = conn.Exec(context.Background(), query) + if err != nil { + zap.S().Error(fmt.Errorf("error truncating tag attributes table, Err=%v", err)) + return err + } + return nil +} + +func removeUnderscoreMaterializedColumnsIndex(conn clickhouse.Conn, fields []LogField) error { + ctx := context.Background() + for _, field := range fields { + // columns name is __ + colname := fmt.Sprintf("%s_%s_%s", strings.ToLower(field.Type), strings.ToLower(field.DataType), strings.ReplaceAll(field.Name, ".", "_")) + + for _, table := range []string{"logs", "distributed_logs"} { + // drop column + zap.S().Info(fmt.Sprintf("creating materialized for: %s i.e %s", field.Name, colname)) + query := fmt.Sprintf("ALTER TABLE signoz_logs.%s on cluster cluster DROP COLUMN IF EXISTS %s", table, colname) + err := conn.Exec(context.Background(), query) + if err != nil { + zap.S().Error(fmt.Errorf("error while creating materialized column on logs table. Err=%v", err)) + } + + // drop exists column + query = fmt.Sprintf("ALTER TABLE signoz_logs.%s ON CLUSTER cluster DROP COLUMN IF EXISTS %s_exists", + table, + colname, + ) + err = conn.Exec(ctx, query) + if err != nil { + zap.S().Error(fmt.Errorf("error while creating exists column on logs table. Err=%v", err)) + return err + } + } + + // delete the index + zap.S().Info(fmt.Sprintf("Create index: %s_idx", colname)) + query := fmt.Sprintf("ALTER TABLE signoz_logs.logs on cluster cluster DROP INDEX IF EXISTS %s_idx", colname) + err := conn.Exec(context.Background(), query) + if err != nil { + zap.S().Error(fmt.Errorf("error while renaming index. Err=%v", err)) + return err + } + + } + return nil +} + +func main() { + logger, _ := zap.NewDevelopment() + defer logger.Sync() + undo := zap.ReplaceGlobals(logger) + defer undo() + + start := time.Now() + hostFlag := flag.String("host", "127.0.0.1", "clickhouse host") + portFlag := flag.String("port", "9000", "clickhouse port") + userNameFlag := flag.String("userName", "default", "clickhouse username") + passwordFlag := flag.String("password", "", "clickhouse password") + flag.Parse() + zap.S().Debug(fmt.Sprintf("Params: %s %s %s", *hostFlag, *portFlag, *userNameFlag)) + + conn, err := connect(*hostFlag, *portFlag, *userNameFlag, *passwordFlag) + if err != nil { + zap.S().Fatal("Error while connecting to clickhouse", zap.Error(err)) + } + + // get fields which has both dot and underscore and has materialized + fields, err := GetFields(conn) + if err != nil { + zap.S().Fatal("Error while getting fields", zap.Error(err)) + os.Exit(1) + } + + // remove the materialized columns and indexes for underscore ones + err = removeUnderscoreMaterializedColumnsIndex(conn, fields) + if err != nil { + zap.S().Fatal("Error while renaming materialized columns", zap.Error(err)) + os.Exit(1) + } + + // truncate the attribute_keys and resource attribute_keys table + err = TruncateKeyTable(conn) + if err != nil { + zap.S().Fatal("Error while truncating attribute keys table", zap.Error(err)) + os.Exit(1) + } + zap.S().Info(fmt.Sprintf("Completed migration in: %s", time.Since(start))) +} From 690a2a328bd812720f9239486ee44775e92c0c74 Mon Sep 17 00:00:00 2001 From: nityanandagohain Date: Fri, 5 Jan 2024 17:06:28 +0530 Subject: [PATCH 2/2] fix: migration fixed --- migration-0.38/main.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/migration-0.38/main.go b/migration-0.38/main.go index bca846f..86e4f36 100644 --- a/migration-0.38/main.go +++ b/migration-0.38/main.go @@ -175,13 +175,22 @@ func removeUnderscoreMaterializedColumnsIndex(conn clickhouse.Conn, fields []Log // columns name is __ colname := fmt.Sprintf("%s_%s_%s", strings.ToLower(field.Type), strings.ToLower(field.DataType), strings.ReplaceAll(field.Name, ".", "_")) + // delete the index + zap.S().Info(fmt.Sprintf("Deleting index: %s_idx", colname)) + query := fmt.Sprintf("ALTER TABLE signoz_logs.logs on cluster cluster DROP INDEX IF EXISTS %s_idx", colname) + err := conn.Exec(context.Background(), query) + if err != nil { + zap.S().Error(fmt.Errorf("error while deleting index. Err=%v", err)) + return err + } + + zap.S().Info(fmt.Sprintf("deleting materialized for: %s i.e %s and %s_exists", field.Name, colname, colname)) for _, table := range []string{"logs", "distributed_logs"} { // drop column - zap.S().Info(fmt.Sprintf("creating materialized for: %s i.e %s", field.Name, colname)) query := fmt.Sprintf("ALTER TABLE signoz_logs.%s on cluster cluster DROP COLUMN IF EXISTS %s", table, colname) err := conn.Exec(context.Background(), query) if err != nil { - zap.S().Error(fmt.Errorf("error while creating materialized column on logs table. Err=%v", err)) + zap.S().Error(fmt.Errorf("error while deleting materialized column on logs table. Err=%v", err)) } // drop exists column @@ -191,20 +200,11 @@ func removeUnderscoreMaterializedColumnsIndex(conn clickhouse.Conn, fields []Log ) err = conn.Exec(ctx, query) if err != nil { - zap.S().Error(fmt.Errorf("error while creating exists column on logs table. Err=%v", err)) + zap.S().Error(fmt.Errorf("error while deleting exists column on logs table. Err=%v", err)) return err } } - // delete the index - zap.S().Info(fmt.Sprintf("Create index: %s_idx", colname)) - query := fmt.Sprintf("ALTER TABLE signoz_logs.logs on cluster cluster DROP INDEX IF EXISTS %s_idx", colname) - err := conn.Exec(context.Background(), query) - if err != nil { - zap.S().Error(fmt.Errorf("error while renaming index. Err=%v", err)) - return err - } - } return nil }