diff --git a/Makefile b/Makefile index b0cba777..4b7f9c4e 100644 --- a/Makefile +++ b/Makefile @@ -124,3 +124,8 @@ $(CONTROLLER_GEN): $(LOCALBIN) kform: $(KFORM) ## Download kform locally if necessary. $(KFORM): $(LOCALBIN) test -s $(LOCALBIN)/kform || GOBIN=$(LOCALBIN) go install github.com/kform-dev/kform/cmd/kform@$(KFORM_VERSION) + +.PHONY: goreleaser-nightly +goreleaser-nightly: + go install github.com/goreleaser/goreleaser/v2@latest + goreleaser release --clean -f .goreleaser.nightlies.yml --skip=validate \ No newline at end of file diff --git a/go.mod b/go.mod index e7de5c8c..8c315407 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/sdcio/config-server -go 1.23.3 +go 1.23.4 + +toolchain go1.23.5 replace sigs.k8s.io/structured-merge-diff/v4 => sigs.k8s.io/structured-merge-diff/v4 v4.4.1 @@ -24,16 +26,16 @@ require ( github.com/prometheus/client_model v0.6.1 github.com/prometheus/prometheus v0.300.1 github.com/sdcio/data-server v0.0.51 - github.com/sdcio/sdc-protos v0.0.39 + github.com/sdcio/sdc-protos v0.0.40 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.10.0 - go.opentelemetry.io/otel v1.32.0 + go.opentelemetry.io/otel v1.34.0 go.starlark.net v0.0.0-20250205221240-492d3672b3f4 go.uber.org/zap v1.27.0 golang.org/x/mod v0.22.0 golang.org/x/sync v0.10.0 - google.golang.org/grpc v1.70.0 - google.golang.org/protobuf v1.36.5 + google.golang.org/grpc v1.71.1 + google.golang.org/protobuf v1.36.6 k8s.io/api v0.31.4 k8s.io/apimachinery v0.31.4 k8s.io/apiserver v0.31.4 @@ -49,7 +51,7 @@ require ( require ( bitbucket.org/creachadair/stringset v0.0.14 // indirect - cloud.google.com/go/compute/metadata v0.5.2 // indirect + cloud.google.com/go/compute/metadata v0.6.0 // indirect dario.cat/mergo v1.0.0 // indirect github.com/AlekSi/pointer v1.2.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect @@ -87,7 +89,7 @@ require ( github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-redis/redis/v8 v8.11.5 // indirect - github.com/golang/glog v1.2.3 // indirect + github.com/golang/glog v1.2.4 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/cel-go v0.21.0 // indirect @@ -134,29 +136,30 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.5.16 // indirect go.etcd.io/etcd/client/v3 v3.5.16 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.30.0 // indirect - go.opentelemetry.io/otel/metric v1.32.0 // indirect - go.opentelemetry.io/otel/sdk v1.32.0 // indirect - go.opentelemetry.io/otel/trace v1.32.0 // indirect + go.opentelemetry.io/otel/metric v1.34.0 // indirect + go.opentelemetry.io/otel/sdk v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.34.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect - golang.org/x/crypto v0.30.0 // indirect + golang.org/x/crypto v0.32.0 // indirect golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect - golang.org/x/net v0.32.0 // indirect - golang.org/x/oauth2 v0.24.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.27.0 // indirect + golang.org/x/net v0.34.0 // indirect + golang.org/x/oauth2 v0.25.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/term v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.8.0 // indirect golang.org/x/tools v0.27.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect diff --git a/go.sum b/go.sum index d47c339a..c1166247 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ bitbucket.org/creachadair/stringset v0.0.14 h1:t1ejQyf8utS4GZV/4fM+1gvYucggZkfhb+tMobDxYOE= bitbucket.org/creachadair/stringset v0.0.14/go.mod h1:Ej8fsr6rQvmeMDf6CCWMWGb14H9mz8kmDgPPTdiVT0w= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= -cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/AlekSi/pointer v1.2.0 h1:glcy/gc4h8HnG2Z3ZECSzZ1IX1x2JxRVuDzaJwQE0+w= @@ -116,8 +116,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.2.3 h1:oDTdz9f5VGVVNGu/Q7UXKWYsD0873HXLHdJUNBsSEKM= -github.com/golang/glog v1.2.3/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= +github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -272,15 +272,15 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prometheus/prometheus v0.300.1 h1:9KKcTTq80gkzmXW0Et/QCFSrBPgmwiS3Hlcxc6o8KlM= github.com/prometheus/prometheus v0.300.1/go.mod h1:gtTPY/XVyCdqqnjA3NzDMb0/nc5H9hOu1RMame+gHyM= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sdcio/data-server v0.0.51 h1:PBoUkLld/jw35N1XJ+ODqB8ZFDZOTOJ5EWrK0DtMEwU= github.com/sdcio/data-server v0.0.51/go.mod h1:pxkCNGvJyHZUkZkeymwJmTKQHu4xWRErkHU6XoYQIfE= github.com/sdcio/schema-server v0.0.28 h1:Tj3AI0WA5zrT2Tm8SGW0g9i+dQOWUsIWFQC/IWN/8Ps= github.com/sdcio/schema-server v0.0.28/go.mod h1:uwX0+giyy7Y/cgX7mVf6GojCvE/XSEZomzhLaBzPYQg= -github.com/sdcio/sdc-protos v0.0.39 h1:GDpiUcMmOnHDSCD92t3brZ315QfmkN4yBiFdaZJv5B0= -github.com/sdcio/sdc-protos v0.0.39/go.mod h1:uG2jk1oV3iQ6WRypxJA+vssPqtfZchQzGwAXueV9z6A= +github.com/sdcio/sdc-protos v0.0.40 h1:uexHKpoG6Td+JAg5dUlNegpOs8KYH0bsSf/tB2ACk3o= +github.com/sdcio/sdc-protos v0.0.40/go.mod h1:pNoWbFaK4s478bLxQBEhjNTHJQWyEmXdhxJWMZFY7Sw= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -337,24 +337,26 @@ go.etcd.io/etcd/server/v3 v3.5.13 h1:V6KG+yMfMSqWt+lGnhFpP5z5dRUj1BDRJ5k1fQ9DFok go.etcd.io/etcd/server/v3 v3.5.13/go.mod h1:K/8nbsGupHqmr5MkgaZpLlH1QdX1pcNQLAkODy44XcQ= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM= -go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= -go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 h1:lsInsfvhVIfOI6qHVyysXMNDnjO9Npvl7tlDPJFBVd4= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0/go.mod h1:KQsVNh4OjgjTG0G6EiNi1jVpnaeeKsKMRwbLN+f1+8M= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.30.0 h1:m0yTiGDLUvVYaTFbAvCkVYIYcvwKt3G7OLoN77NUs/8= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.30.0/go.mod h1:wBQbT4UekBfegL2nx0Xk1vBcnzyBPsIVm9hRG4fYcr4= -go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= -go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= -go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= -go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= -go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= -go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= -go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= -go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.starlark.net v0.0.0-20250205221240-492d3672b3f4 h1:eBP+boBfJoGU3irqbxGTcTlKcbNwJCOdbmsnDq56nak= @@ -374,8 +376,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= @@ -403,11 +405,11 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= -golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= +golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -434,15 +436,15 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -479,17 +481,17 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= -google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a h1:OAiGFfOiA0v9MRYsSidp3ubZaBnteRUyn3xB2ZQ5G/E= -google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a/go.mod h1:jehYqy3+AhJU9ve55aNOaSml7wUXjF9x6z2LcCfpAhY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a h1:hgh8P4EuoxpsuKMXX/To36nOFD7vixReXgn8lPGnt+o= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422 h1:GVIKPyP/kLIyVOgOnTwFOrvQaQUzOzGMCxgFUOEmm24= +google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422/go.mod h1:b6h1vNKhxaSoEI+5jc3PJUCustfli/mRab7295pY7rw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= -google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= +google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= +google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -499,8 +501,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/pkg/git/git.go b/pkg/git/git.go index e6c15076..39c23b74 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -29,44 +29,36 @@ import ( "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/client" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/storage/memory" "github.com/henderiw/logger/log" sdcerrors "github.com/sdcio/config-server/pkg/errors" - "github.com/sdcio/config-server/pkg/git/auth" - "k8s.io/apimachinery/pkg/types" ) type GoGit struct { - gitRepo GitRepo - r *gogit.Repository - credentialResolver auth.CredentialResolver - secret types.NamespacedName - // credential contains the information needed to authenticate against - // a git repository. - credential auth.Credential - ProxyURL *url.URL + gitRepo GitRepoSpec + r *gogit.Repository } -// make sure GoGit satisfies the Git interface. -var _ Git = (*GoGit)(nil) - -func NewGoGit(gitRepo GitRepo, secret types.NamespacedName, credentialResolver auth.CredentialResolver) *GoGit { - return &GoGit{ - gitRepo: gitRepo, - credentialResolver: credentialResolver, - secret: secret, +func NewGoGit(gitRepo GitRepoSpec) *GoGit { + gg := &GoGit{ + gitRepo: gitRepo, + } + if proxy := gitRepo.GetProxy(); proxy != nil { + gg.setProxy(proxy) } + return gg } -func (g *GoGit) SetProxy(p string) error { - var err error - g.ProxyURL, err = url.Parse(p) - if err != nil { - return err +func (g *GoGit) setProxy(proxyUrl *url.URL) error { + customClient := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyURL(proxyUrl), + }, } + client.InstallProtocol("https", githttp.NewClient(customClient)) + client.InstallProtocol("http", githttp.NewClient(customClient)) return nil } @@ -86,6 +78,7 @@ func (g *GoGit) Clone(ctx context.Context) error { } func (g *GoGit) getDefaultBranch(ctx context.Context) (string, error) { + var err error rem := gogit.NewRemote(memory.NewStorage(), &config.RemoteConfig{ Name: "origin", URLs: []string{g.gitRepo.GetCloneURL().String()}, @@ -93,13 +86,11 @@ func (g *GoGit) getDefaultBranch(ctx context.Context) (string, error) { // We can then use every Remote functions to retrieve wanted information var refs []*plumbing.Reference - if err := g.doGitWithAuth(ctx, func(auth transport.AuthMethod) error { - var err error - refs, err = rem.List(&gogit.ListOptions{ - Auth: auth, - }) - return err - }); err != nil { + + refs, err = rem.List(&gogit.ListOptions{ + Auth: g.gitRepo.GetAuth(), + }) + if err != nil { return "", &sdcerrors.UnrecoverableError{Message: "cannot get default branch", WrappedError: err} } @@ -129,17 +120,6 @@ func (g *GoGit) cloneExistingRepo(ctx context.Context) error { log := log.FromContext(ctx) log.Info("loading git", "repo", g.gitRepo.GetLocalPath()) - // if the ProxyURL is set, use custom transport as per https://github.com/go-git/go-git/blob/master/_examples/custom_http/main.go - if g.ProxyURL != nil { - customClient := &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyURL(g.ProxyURL), - }, - } - client.InstallProtocol("https", githttp.NewClient(customClient)) - client.InstallProtocol("http", githttp.NewClient(customClient)) - } - // open the existing repo err = g.openRepo(ctx) if err != nil { @@ -166,44 +146,43 @@ func (g *GoGit) cloneExistingRepo(ctx context.Context) error { // get the branch or tag or figure out the default branch main / master / sth. else. var refRemoteName plumbing.ReferenceName var refName plumbing.ReferenceName - branch := g.gitRepo.GetBranch() - tag := g.gitRepo.GetTag() - if branch != "" { - refName = plumbing.NewBranchReferenceName(branch) - refRemoteName = plumbing.NewRemoteReferenceName("origin", branch) - } else if tag != "" { - refRemoteName = plumbing.NewTagReferenceName(tag) - refName = plumbing.NewTagReferenceName(tag) - } else { + + refKind, ref := g.gitRepo.GetGitRef() + + switch refKind { + case GitRefKindTag: + refName = plumbing.NewTagReferenceName(ref) + refRemoteName = plumbing.NewRemoteReferenceName("origin", ref) + case GitRefKindBranch: + refName = plumbing.NewBranchReferenceName(ref) + refRemoteName = plumbing.NewRemoteReferenceName("origin", ref) + default: log.Debug("default branch not set. determining it") - branch, err = g.getDefaultBranch(ctx) + defaultBranch, err := g.getDefaultBranch(ctx) if err != nil { return err } - refRemoteName = plumbing.NewRemoteReferenceName("origin", branch) - refName = plumbing.NewBranchReferenceName(branch) - log.Debug("default", "branch", branch) + refRemoteName = plumbing.NewRemoteReferenceName("origin", defaultBranch) + refName = plumbing.NewBranchReferenceName(defaultBranch) + log.Debug("default", "branch", defaultBranch) } refSpec := config.RefSpec(fmt.Sprintf("+%s:%s", refName, refRemoteName)) log.Debug("fetching latest repo data") // execute the fetch - err = g.doGitWithAuth(ctx, func(auth transport.AuthMethod) error { - return g.r.FetchContext(ctx, &gogit.FetchOptions{ - Depth: 1, - Auth: auth, - Force: true, - Prune: true, - RefSpecs: []config.RefSpec{ - refSpec, - }, - }) + err = g.r.FetchContext(ctx, &gogit.FetchOptions{ + Depth: 1, + Auth: g.gitRepo.GetAuth(), + Force: true, + Prune: true, + RefSpecs: []config.RefSpec{ + refSpec, + }, }) switch { case errors.Is(err, gogit.NoErrAlreadyUpToDate): err = nil - } - if err != nil { + case err != nil: return &sdcerrors.UnrecoverableError{Message: "cannot perform fetch", WrappedError: err} } @@ -233,134 +212,70 @@ func (g *GoGit) cloneExistingRepo(ctx context.Context) error { return nil } -func (g *GoGit) fetchNonExistingBranch(ctx context.Context, branch string) error { - // init the remote - remote, err := g.r.Remote("origin") - if err != nil { - return &sdcerrors.UnrecoverableError{Message: "cannot get remote from repo", WrappedError: err} - } - - // build the RefSpec, that wires the remote to the local branch - localRef := plumbing.NewBranchReferenceName(branch) - remoteRef := plumbing.NewRemoteReferenceName("origin", branch) - refSpec := config.RefSpec(fmt.Sprintf("+%s:%s", localRef, remoteRef)) - - // execute the fetch - err = g.doGitWithAuth(ctx, func(auth transport.AuthMethod) error { - return remote.Fetch(&gogit.FetchOptions{ - Depth: 1, - RefSpecs: []config.RefSpec{refSpec}, - Auth: auth, - }) - }) - switch { - case err == nil, errors.Is(err, gogit.NoErrAlreadyUpToDate): - default: - return &sdcerrors.UnrecoverableError{Message: "cannot fetch repo for branch that does not exist", WrappedError: err} - } - - // make sure the branch is also showing up in .git/config - err = g.r.CreateBranch(&config.Branch{ - Name: branch, - Remote: "origin", - Merge: localRef, - }) - - return &sdcerrors.UnrecoverableError{Message: "cannot create branch", WrappedError: err} -} +// func (g *GoGit) fetchNonExistingBranch(ctx context.Context, branch string) error { +// // init the remote +// remote, err := g.r.Remote("origin") +// if err != nil { +// return &sdcerrors.UnrecoverableError{Message: "cannot get remote from repo", WrappedError: err} +// } + +// // build the RefSpec, that wires the remote to the local branch +// localRef := plumbing.NewBranchReferenceName(branch) +// remoteRef := plumbing.NewRemoteReferenceName("origin", branch) +// refSpec := config.RefSpec(fmt.Sprintf("+%s:%s", localRef, remoteRef)) + +// // execute the fetch +// err = remote.Fetch(&gogit.FetchOptions{ +// Depth: 1, +// RefSpecs: []config.RefSpec{refSpec}, +// Auth: g.gitRepo.GetAuth(), +// }) + +// switch { +// case err == nil, errors.Is(err, gogit.NoErrAlreadyUpToDate): +// default: +// return &sdcerrors.UnrecoverableError{Message: "cannot fetch repo for branch that does not exist", WrappedError: err} +// } + +// // make sure the branch is also showing up in .git/config +// err = g.r.CreateBranch(&config.Branch{ +// Name: branch, +// Remote: "origin", +// Merge: localRef, +// }) + +// return &sdcerrors.UnrecoverableError{Message: "cannot create branch", WrappedError: err} +// } func (g *GoGit) cloneNonExisting(ctx context.Context) error { var err error - // if the ProxyURL is set, use custom transport as per https://github.com/go-git/go-git/blob/master/_examples/custom_http/main.go - if g.ProxyURL != nil { - customClient := &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyURL(g.ProxyURL), - }, - } - client.InstallProtocol("https", githttp.NewClient(customClient)) - client.InstallProtocol("http", githttp.NewClient(customClient)) - } // init clone options co := &gogit.CloneOptions{ Depth: 1, URL: g.gitRepo.GetCloneURL().String(), SingleBranch: true, + Auth: g.gitRepo.GetAuth(), } - // set branch reference if set - if g.gitRepo.GetBranch() != "" { - co.ReferenceName = plumbing.NewBranchReferenceName(g.gitRepo.GetBranch()) - } else if g.gitRepo.GetTag() != "" { - co.ReferenceName = plumbing.NewTagReferenceName(g.gitRepo.GetTag()) - } else { + refKind, ref := g.gitRepo.GetGitRef() + + switch refKind { + case GitRefKindBranch: + co.ReferenceName = plumbing.NewBranchReferenceName(ref) + case GitRefKindTag: + co.ReferenceName = plumbing.NewTagReferenceName(ref) + // case GitRefKindHash: + default: branchName, err := g.getDefaultBranch(ctx) if err != nil { return err } co.ReferenceName = plumbing.NewBranchReferenceName(branchName) } - + co.Auth = g.gitRepo.GetAuth() // perform clone - return g.doGitWithAuth(ctx, func(auth transport.AuthMethod) error { - co.Auth = auth - g.r, err = gogit.PlainClone(g.gitRepo.GetLocalPath(), false, co) - return err - }) -} - -type Git interface { - // Clone takes the given GitRepo reference and clones the repo - // with its internal implementation. - Clone(ctx context.Context) error -} - -// doGitWithAuth fetches auth information for git and provides it -// to the provided function which performs the operation against a git repo. -func (g *GoGit) doGitWithAuth(ctx context.Context, op func(transport.AuthMethod) error) error { - log := log.FromContext(ctx) - auth, err := g.getAuthMethod(ctx, false) - if err != nil { - return err - } - err = op(auth) - if err != nil { - if !errors.Is(err, transport.ErrAuthenticationRequired) || !errors.Is(err, transport.ErrAuthorizationFailed) { - return &sdcerrors.UnrecoverableError{Message: "authentication failed", WrappedError: err} - } - log.Info("Authentication failed. Trying to refresh credentials") - // TODO: Consider having some kind of backoff here. - auth, err := g.getAuthMethod(ctx, true) - if err != nil { - return err - } - err = op(auth) - if err != nil { - return &sdcerrors.UnrecoverableError{Message: "authentication failed", WrappedError: err} - } - } - return nil -} - -// getAuthMethod fetches the credentials for authenticating to git. It caches the -// credentials between calls and refresh credentials when the tokens have expired. -func (g *GoGit) getAuthMethod(ctx context.Context, forceRefresh bool) (transport.AuthMethod, error) { - // If no secret is provided, we try without any auth. - log := log.FromContext(ctx) - log.Info("getAuthMethod", "secret", g.secret, "credential", g.credential) - if g.secret.Name == "" { - return nil, nil - } - - if g.credential == nil || !g.credential.Valid() || forceRefresh { - if cred, err := g.credentialResolver.ResolveCredential(ctx, g.secret); err != nil { - return nil, &sdcerrors.UnrecoverableError{Message: "cannot obtain credentials", WrappedError: err} - } else { - g.credential = cred - } - } - - return g.credential.ToAuthMethod(), nil + g.r, err = gogit.PlainClone(g.gitRepo.GetLocalPath(), false, co) + return err } func (g *GoGit) EnsureCommit(ctx context.Context, commitHash string) (string, error) { @@ -397,15 +312,14 @@ func (g *GoGit) fetchCommit(ctx context.Context, commitHash string) error { log := log.FromContext(ctx) log.Info("Fetching commit", "commit", commitHash) - err := g.doGitWithAuth(ctx, func(auth transport.AuthMethod) error { - return g.r.FetchContext(ctx, &gogit.FetchOptions{ - RefSpecs: []config.RefSpec{config.RefSpec(fmt.Sprintf("+refs/*:refs/*"))}, // Fetch all refs - Depth: 0, - Auth: auth, - Force: true, - Prune: true, - }) + err := g.r.FetchContext(ctx, &gogit.FetchOptions{ + RefSpecs: []config.RefSpec{config.RefSpec(fmt.Sprintf("+refs/*:refs/*"))}, // Fetch all refs + Depth: 0, + Auth: g.gitRepo.GetAuth(), + Force: true, + Prune: true, }) + if err != nil && !errors.Is(err, gogit.NoErrAlreadyUpToDate) { return &sdcerrors.UnrecoverableError{Message: "Failed to fetch commit", WrappedError: err} } diff --git a/pkg/git/refKind.go b/pkg/git/refKind.go new file mode 100644 index 00000000..80d383e0 --- /dev/null +++ b/pkg/git/refKind.go @@ -0,0 +1,9 @@ +package git + +type GitRefKind string + +const ( + GitRefKindTag GitRefKind = "tag" + GitRefKindBranch GitRefKind = "branch" + GitRefKindHash GitRefKind = "hash" +) diff --git a/pkg/git/repo.go b/pkg/git/repo.go index 67d402ab..363a3d3c 100644 --- a/pkg/git/repo.go +++ b/pkg/git/repo.go @@ -17,85 +17,88 @@ limitations under the License. package git import ( - "fmt" "net/url" - sdcerrors "github.com/sdcio/config-server/pkg/errors" + "github.com/go-git/go-git/v5/plumbing/transport" ) // GitRepoStruct is a struct that contains all the fields // required for a GitRepo instance. type GitRepoStruct struct { - // CloneURL is the URL that will be used for cloning the repo - CloneURL *url.URL - RepositoryName string - // GitBranch is the referenced Git branch name. - GitBranch string - // Tag name - either tag or branch should be set. - Tag string - LocalRepoPath string + // cloneURL is the URL that will be used for cloning the repo + cloneURL *url.URL + repoAuth transport.AuthMethod + repositoryName string + ref string + refKind GitRefKind + localRepoPath string + proxy *url.URL } // GetName returns the repository name. func (u *GitRepoStruct) GetName() string { - return u.RepositoryName + return u.repositoryName } func (u *GitRepoStruct) GetLocalPath() string { - return u.LocalRepoPath -} - -// GetBranch returns the referenced Git branch name. -// the empty string is returned otherwise. -func (u *GitRepoStruct) GetBranch() string { - return u.GitBranch + return u.localRepoPath } // GetCloneURL returns the CloneURL of the repository. func (u *GitRepoStruct) GetCloneURL() *url.URL { - return u.CloneURL + return u.cloneURL +} + +func (u *GitRepoStruct) GetGitRef() (GitRefKind, string) { + return u.refKind, u.ref +} + +func (u *GitRepoStruct) SetGitRef(refKind GitRefKind, ref string) { + u.ref = ref + u.refKind = refKind +} + +func (u *GitRepoStruct) SetLocalPath(localPath string) { + u.localRepoPath = localPath } -func (u *GitRepoStruct) GetTag() string { - return u.Tag +func (u *GitRepoStruct) SetProxy(proxy *url.URL) { + u.proxy = proxy } -func (u *GitRepoStruct) SetTag(t string) { - u.Tag = t +func (u *GitRepoStruct) GetProxy() *url.URL { + return u.proxy } -func (u *GitRepoStruct) SetBranch(b string) { - u.GitBranch = b +func (u *GitRepoStruct) SetAuth(auth transport.AuthMethod) { + u.repoAuth = auth } -func (u *GitRepoStruct) SetLocalPath(p string) { - u.LocalRepoPath = p +func (u *GitRepoStruct) GetAuth() transport.AuthMethod { + return u.repoAuth } -type GitRepo interface { +type GitRepoSpec interface { GetName() string GetCloneURL() *url.URL - GetBranch() string + SetGitRef(refKind GitRefKind, ref string) + GetGitRef() (GitRefKind, string) GetLocalPath() string - GetTag() string - SetTag(string) - SetBranch(string) SetLocalPath(string) + SetProxy(*url.URL) + GetProxy() *url.URL + SetAuth(transport.AuthMethod) + GetAuth() transport.AuthMethod } -// NewRepo parses the given git urlPath and returns an interface +// NewRepoSpec parses the given git urlPath and returns an interface // that is backed by Github or Gitlab repo implementations. -func NewRepo(urlPath string) (GitRepo, error) { - var r GitRepo +func NewRepoSpec(urlPath *url.URL) (GitRepoSpec, error) { + var r GitRepoSpec var err error - u, err := url.Parse(urlPath) - if err != nil { - return nil, &sdcerrors.UnrecoverableError{Message: fmt.Sprintf("cannot parse urlPath: %s", urlPath), WrappedError: err} - } - r = &GitRepoStruct{ - CloneURL: u, + cloneURL: urlPath, } return r, err diff --git a/pkg/reconcilers/all/all.go b/pkg/reconcilers/all/all.go index 2145ab70..342447a3 100644 --- a/pkg/reconcilers/all/all.go +++ b/pkg/reconcilers/all/all.go @@ -20,11 +20,11 @@ import ( _ "github.com/sdcio/config-server/pkg/reconcilers/config" _ "github.com/sdcio/config-server/pkg/reconcilers/configset" _ "github.com/sdcio/config-server/pkg/reconcilers/discoveryrule" + _ "github.com/sdcio/config-server/pkg/reconcilers/rollout" _ "github.com/sdcio/config-server/pkg/reconcilers/schema" + _ "github.com/sdcio/config-server/pkg/reconcilers/subscription" _ "github.com/sdcio/config-server/pkg/reconcilers/target" _ "github.com/sdcio/config-server/pkg/reconcilers/targetconfigserver" _ "github.com/sdcio/config-server/pkg/reconcilers/targetdatastore" - _ "github.com/sdcio/config-server/pkg/reconcilers/subscription" _ "github.com/sdcio/config-server/pkg/reconcilers/workspace" - _ "github.com/sdcio/config-server/pkg/reconcilers/rollout" ) diff --git a/pkg/reconcilers/schema/reconciler.go b/pkg/reconcilers/schema/reconciler.go index 0da4660a..19aa3ea1 100644 --- a/pkg/reconcilers/schema/reconciler.go +++ b/pkg/reconcilers/schema/reconciler.go @@ -20,7 +20,8 @@ import ( "context" "errors" "fmt" - "path/filepath" + "net/url" + "path" "reflect" "github.com/henderiw/apiserver-store/pkg/storebackend" @@ -29,18 +30,20 @@ import ( condv1alpha1 "github.com/sdcio/config-server/apis/condition/v1alpha1" invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" sdcerrors "github.com/sdcio/config-server/pkg/errors" + "github.com/sdcio/config-server/pkg/git" "github.com/sdcio/config-server/pkg/git/auth/secret" "github.com/sdcio/config-server/pkg/reconcilers" "github.com/sdcio/config-server/pkg/reconcilers/ctrlconfig" "github.com/sdcio/config-server/pkg/reconcilers/eventhandler" "github.com/sdcio/config-server/pkg/reconcilers/resource" - schemaloader "github.com/sdcio/config-server/pkg/schema" + yangSchema "github.com/sdcio/config-server/pkg/schema" sdcctx "github.com/sdcio/config-server/pkg/sdc/ctx" ssclient "github.com/sdcio/config-server/pkg/sdc/schemaserver/client" sdcpb "github.com/sdcio/sdc-protos/sdcpb" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -85,13 +88,7 @@ func (r *reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, c i r.finalizer = resource.NewAPIFinalizer(mgr.GetClient(), finalizer, reconcilerName) // initializes the directory r.schemaBasePath = cfg.SchemaDir - r.schemaLoader, err = schemaloader.NewLoader( - filepath.Join(r.schemaBasePath, "tmp"), - r.schemaBasePath, - secret.NewCredentialResolver(mgr.GetClient(), []secret.Resolver{ - secret.NewBasicAuthResolver(), - }), - ) + r.schemaLoader, err = yangSchema.NewSchemaLoader(path.Join(r.schemaBasePath, "tmp"), r.schemaBasePath, r.schemaclient, yangSchema.NewSchemaUploaderLocal(r.schemaclient)) if err != nil { return nil, pkgerrors.Wrap(err, "cannot initialize schemaloader") } @@ -108,7 +105,7 @@ type reconciler struct { client.Client finalizer *resource.APIFinalizer - schemaLoader *schemaloader.Loader + schemaLoader *yangSchema.SchemaLoader schemaclient ssclient.Client schemaBasePath string recorder record.EventRecorder @@ -144,7 +141,7 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } // delete the reference from disk - if err := r.schemaLoader.DelRef(ctx, spec.GetKey()); err != nil { + if err := r.schemaLoader.RemoveSchema(ctx, yangSchema.NewSchemaID(spec.Provider, spec.Version)); err != nil { return r.handleError(ctx, schemaOrig, "cannot delete reference", err) } // remove the finalizer @@ -166,23 +163,21 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return r.handleError(ctx, schemaOrig, "schema server not ready", nil) } - // we just insert the schema again - r.schemaLoader.AddRef(ctx, schema) - _, dirExists, err := r.schemaLoader.GetRef(ctx, spec.GetKey()) - if err != nil { - return r.handleError(ctx, schemaOrig, "cannot get schema reference", err) - } - - if !dirExists { + if !r.schemaLoader.SchemaExists(yangSchema.NewSchemaID(spec.Provider, spec.Version)) { // we set the loading condition to know loading started schema.SetConditions(invv1alpha1.Loading()) if err := r.Status().Update(ctx, schema); err != nil { // we always retry when status fails -> optimistic concurrency return r.handleError(ctx, schemaOrig, "cannot update status", err) } - r.recorder.Eventf(schema, corev1.EventTypeNormal, - "schema", "loading") - if err := r.schemaLoader.Load(ctx, spec.GetKey()); err != nil { + r.recorder.Eventf(schema, corev1.EventTypeNormal, "schema", "loading") + // we just insert the schema again + schemaDef, err := schemaSpecToSchemaDefinition(ctx, schema, r.Client) + if err != nil { + return r.handleError(ctx, schemaOrig, "error converting from api type to internal", nil) + } + + if err = r.schemaLoader.AddSchema(ctx, schemaDef); err != nil { return r.handleError(ctx, schemaOrig, "cannot load schema", err) } } @@ -191,37 +186,89 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu Schema: spec.GetSchema(), }) if err != nil { - // schema does not exists in schema-server -> create it - if _, err := r.schemaclient.CreateSchema(ctx, &sdcpb.CreateSchemaRequest{ - Schema: spec.GetSchema(), - File: spec.GetNewSchemaBase(r.schemaBasePath).Models, - Directory: spec.GetNewSchemaBase(r.schemaBasePath).Includes, - Exclude: spec.GetNewSchemaBase(r.schemaBasePath).Excludes, - }); err != nil { - return r.handleError(ctx, schemaOrig, "cannot create schema", err) - } - return r.handleSuccess(ctx, schemaOrig) + return r.handleError(ctx, schemaOrig, "get schema detail returned with error", err) } if rsp == nil || rsp.Schema == nil { return r.handleError(ctx, schemaOrig, "get schema detail response w/o a response or schems", nil) } - switch rsp.Schema.Status { - case sdcpb.SchemaStatus_FAILED: - if _, err := r.schemaclient.CreateSchema(ctx, &sdcpb.CreateSchemaRequest{ - Schema: spec.GetSchema(), - File: spec.GetNewSchemaBase(r.schemaBasePath).Models, - Directory: spec.GetNewSchemaBase(r.schemaBasePath).Includes, - Exclude: spec.GetNewSchemaBase(r.schemaBasePath).Excludes, - }); err != nil { - return r.handleError(ctx, schemaOrig, "cannot create schema", err) + // switch rsp.Schema.Status { + // case sdcpb.SchemaStatus_FAILED: + // if _, err := r.schemaclient.CreateSchema(ctx, &sdcpb.CreateSchemaRequest{ + // Schema: spec.GetSchema(), + // File: spec.GetNewSchemaBase(r.schemaBasePath).Models, + // Directory: spec.GetNewSchemaBase(r.schemaBasePath).Includes, + // Exclude: spec.GetNewSchemaBase(r.schemaBasePath).Excludes, + // }); err != nil { + // return r.handleError(ctx, schemaOrig, "cannot create schema", err) + // } + // return r.handleSuccess(ctx, schemaOrig) + // case sdcpb.SchemaStatus_RELOADING, sdcpb.SchemaStatus_INITIALIZING: + // return r.handleError(ctx, schemaOrig, fmt.Sprintf("schema %s", rsp.Schema.Status), nil) + // default: // OK case + // return r.handleSuccess(ctx, schemaOrig) + // } + return r.handleSuccess(ctx, schemaOrig) +} + +func schemaSpecToSchemaDefinition(ctx context.Context, schema *invv1alpha1.Schema, client client.Reader) (*yangSchema.SchemaDefinition, error) { + sid := yangSchema.NewSchemaID(schema.Spec.Provider, schema.Spec.Version) + sd := yangSchema.NewSchemaDefinition(sid) + + for _, r := range schema.Spec.Repositories { + repoUrl, err := url.Parse(r.RepositoryURL) + if err != nil { + return nil, err + } + + dirs := make([]*yangSchema.SrcDstPath, 0, len(r.Dirs)) + + for _, d := range r.Dirs { + dirs = append(dirs, yangSchema.NewSrcDstPath(d.Src, d.Dst)) + } + + refKind, err := convertRefKind(r.Kind) + if err != nil { + return nil, err } - return r.handleSuccess(ctx, schemaOrig) - case sdcpb.SchemaStatus_RELOADING, sdcpb.SchemaStatus_INITIALIZING: - return r.handleError(ctx, schemaOrig, fmt.Sprintf("schema %s", rsp.Schema.Status), nil) - default: // OK case - return r.handleSuccess(ctx, schemaOrig) + + rs := yangSchema.NewRepositorySpec(repoUrl, refKind, r.Ref, yangSchema.NewSchemaLoadSpec(r.Schema.Models, r.Schema.Includes, r.Schema.Excludes), dirs) + + // set proxy for the repo + if r.Proxy != nil && r.Proxy.URL != "" { + purl, err := url.Parse(r.Proxy.URL) + if err != nil { + return nil, err + } + rs.SetProxy(purl) + } + + if r.Credentials != "" { + cred, err := secret.NewCredentialResolver(client, []secret.Resolver{ + secret.NewBasicAuthResolver(), + }).ResolveCredential(ctx, types.NamespacedName{Namespace: schema.Namespace, Name: r.Credentials}) + if err != nil { + return nil, err + } + rs.SetCredentials(cred.ToAuthMethod()) + } + + sd.AddRepositorySpec(rs) + } + + return sd, nil +} + +func convertRefKind(btc invv1alpha1.BranchTagKind) (git.GitRefKind, error) { + switch btc { + case invv1alpha1.BranchTagKindBranch: + return git.GitRefKindBranch, nil + case invv1alpha1.BranchTagKindTag: + return git.GitRefKindTag, nil + case invv1alpha1.BranchTagKindHash: + return git.GitRefKindHash, nil } + return "", fmt.Errorf("unknown BranchTagKind") } func (r *reconciler) handleSuccess(ctx context.Context, schema *invv1alpha1.Schema) (ctrl.Result, error) { diff --git a/pkg/reconcilers/target/reconciler.go b/pkg/reconcilers/target/reconciler.go index 4c9c8ad7..a7337152 100644 --- a/pkg/reconcilers/target/reconciler.go +++ b/pkg/reconcilers/target/reconciler.go @@ -117,7 +117,7 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu pkgerrors.Wrap(r.handleError(ctx, targetOrig, "tcxt does not exist", err), errUpdateStatus) } - resp, err := tctx.GetDataStore(ctx, &sdcpb.GetDataStoreRequest{Name: targetKey.String()}) + resp, err := tctx.GetDataStore(ctx, &sdcpb.GetDataStoreRequest{DatastoreName: targetKey.String()}) if err != nil { if errs := r.targetStore.UpdateWithKeyFn(ctx, targetKey, func(ctx context.Context, tctx *sdctarget.Context) *sdctarget.Context { if tctx != nil { diff --git a/pkg/reconcilers/targetdatastore/reconciler.go b/pkg/reconcilers/targetdatastore/reconciler.go index f18c8b4f..4cfc110a 100644 --- a/pkg/reconcilers/targetdatastore/reconciler.go +++ b/pkg/reconcilers/targetdatastore/reconciler.go @@ -412,7 +412,7 @@ func (r *reconciler) getTargetStatus(ctx context.Context, cr *invv1alpha1.Target } // timeout to wait for target connection in the data-server since it is just created time.Sleep(1 * time.Second) - resp, err := tctx.GetDataStore(ctx, &sdcpb.GetDataStoreRequest{Name: targetKey.String()}) + resp, err := tctx.GetDataStore(ctx, &sdcpb.GetDataStoreRequest{DatastoreName: targetKey.String()}) if err != nil { log.Error("cannot get target from the datastore", "key", targetKey.String(), "error", err) return false, err @@ -463,7 +463,7 @@ func (r *reconciler) updateDataStoreTargetReady(ctx context.Context, target *inv return changed, nil, err } // get the datastore from the dataserver - getRsp, err := tctx.GetDataStore(ctx, &sdcpb.GetDataStoreRequest{Name: targetKey.String()}) + getRsp, err := tctx.GetDataStore(ctx, &sdcpb.GetDataStoreRequest{DatastoreName: targetKey.String()}) if err != nil { // datastore does not exist or dataserver is unhealthy if !strings.Contains(err.Error(), "unknown datastore") { @@ -517,14 +517,14 @@ func (r *reconciler) hasDataStoreChanged( ) bool { log := log.FromContext(ctx) log.Debug("hasDataStoreChanged", - "name", fmt.Sprintf("%s/%s", req.Name, rsp.Name), + "name", fmt.Sprintf("%s/%s", req.GetDatastoreName(), rsp.GetDatastoreName()), "schema Name", fmt.Sprintf("%s/%s", req.Schema.Name, rsp.Schema.Name), "schema Vendor", fmt.Sprintf("%s/%s", req.Schema.Vendor, rsp.Schema.Vendor), "schema Version", fmt.Sprintf("%s/%s", req.Schema.Version, rsp.Schema.Version), "target Type", fmt.Sprintf("%s/%s", req.Target.Type, rsp.Target.Type), "target Address", fmt.Sprintf("%s/%s", req.Target.Address, rsp.Target.Address), ) - if req.Name != rsp.Name { + if req.GetDatastoreName() != rsp.GetDatastoreName() { return true } @@ -668,7 +668,7 @@ func (r *reconciler) getCreateDataStoreRequest(ctx context.Context, target *invv } req := &sdcpb.CreateDataStoreRequest{ - Name: storebackend.KeyFromNSN(types.NamespacedName{Namespace: target.Namespace, Name: target.Name}).String(), + DatastoreName: storebackend.KeyFromNSN(types.NamespacedName{Namespace: target.Namespace, Name: target.Name}).String(), Target: &sdcpb.Target{ Type: string(connProfile.Spec.Protocol), Address: target.Spec.Address, diff --git a/pkg/schema/gitdownloader.go b/pkg/schema/gitdownloader.go deleted file mode 100644 index 41602206..00000000 --- a/pkg/schema/gitdownloader.go +++ /dev/null @@ -1,94 +0,0 @@ -/* -Copyright 2024 Nokia. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package schema - -import ( - "context" - "path" - - "github.com/henderiw/logger/log" - invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" - "github.com/sdcio/config-server/pkg/git" - "github.com/sdcio/config-server/pkg/git/auth" - "k8s.io/apimachinery/pkg/types" -) - -type gitDownloader struct { - providerDownloader -} - -func newGitDownloader(destDir, namespace string, schemaRepo *invv1alpha1.SchemaSpecRepository, credentialResolver auth.CredentialResolver) *gitDownloader { - return &gitDownloader{ - providerDownloader{ - destDir: destDir, - namespace: namespace, - schemaRepo: schemaRepo, - credentialResolver: credentialResolver, - }, - } -} - -func (l *gitDownloader) Download(ctx context.Context) error { - log := log.FromContext(ctx) - - repo, err := git.NewRepo(l.schemaRepo.RepositoryURL) - if err != nil { - return err - } - - repoPath := path.Join(l.destDir, repo.GetCloneURL().Path) - repo.SetLocalPath(repoPath) - - // set branch or tag - if l.schemaRepo.Kind == invv1alpha1.BranchTagKindBranch { - repo.SetBranch(l.schemaRepo.Ref) - } else { - // set the git tag that we're after - // if both branch and tag are the empty string - // the git impl will retrieve the default branch - repo.SetTag(l.schemaRepo.Ref) - } - - // init the actual git instance - goGit := git.NewGoGit(repo, - types.NamespacedName{ - Namespace: l.namespace, - Name: l.schemaRepo.Credentials}, - l.credentialResolver, - ) - - log.Info("cloning", "from", repo.GetCloneURL(), "to", repo.GetLocalPath()) - - if l.schemaRepo.Proxy != nil && l.schemaRepo.Proxy.URL != "" { - err = goGit.SetProxy(l.schemaRepo.Proxy.URL) - if err != nil { - return err - } - log.Debug("SetProxy", "proxy", l.schemaRepo.Proxy.URL) - } - - return goGit.Clone(ctx) -} - -func (l *gitDownloader) LocalPath(urlPath string) (string, error) { - repo, err := git.NewRepo(urlPath) - if err != nil { - return "", err - } - - return path.Join(l.destDir, repo.GetCloneURL().Path), nil -} diff --git a/pkg/schema/loader.go b/pkg/schema/loader.go deleted file mode 100644 index 32cca288..00000000 --- a/pkg/schema/loader.go +++ /dev/null @@ -1,211 +0,0 @@ -/* -Copyright 2024 Nokia. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package schema - -import ( - "context" - "fmt" - "path" - "sync" - - "github.com/henderiw/logger/log" - "github.com/otiai10/copy" - invv1alpha1 "github.com/sdcio/config-server/apis/inv/v1alpha1" - sdcerrors "github.com/sdcio/config-server/pkg/errors" - "github.com/sdcio/config-server/pkg/git/auth" - "github.com/sdcio/config-server/pkg/utils" -) - -type Loader struct { - tmpDir string - schemaDir string - credentialResolver auth.CredentialResolver - - //schemas contains the Schema Reference indexed by Provider.Version key - sm sync.RWMutex - schemas map[string]*invv1alpha1.Schema - // repo manager allocates sempahores to ensure no concurrent downloads from the same schema - repoMgr *RepoMgr -} - -type downloadable interface { - Download(ctx context.Context) error - LocalPath(urlPath string) (string, error) -} - -type providerDownloader struct { - destDir string - namespace string - schemaRepo *invv1alpha1.SchemaSpecRepository - credentialResolver auth.CredentialResolver -} - -func NewLoader(tmpDir string, schemaDir string, credentialResolver auth.CredentialResolver) (*Loader, error) { - var err error - - if !utils.DirExists(tmpDir) { - err = utils.CreateDirectory(tmpDir, 0766) - if err != nil { - return nil, err - } - } - - if !utils.DirExists(schemaDir) { - err = utils.CreateDirectory(schemaDir, 0766) - if err != nil { - return nil, err - } - } - - return &Loader{ - tmpDir: tmpDir, - schemaDir: schemaDir, - schemas: map[string]*invv1alpha1.Schema{}, - credentialResolver: credentialResolver, - repoMgr: NewRepoMgr(), - }, nil -} - -// AddRef overwrites the provider schema version -// The schemaRef is immutable -func (r *Loader) AddRef(ctx context.Context, schema *invv1alpha1.Schema) { - r.sm.Lock() - defer r.sm.Unlock() - r.schemas[schema.Spec.GetKey()] = schema -} - -// DelRef deletes the provider schema version -func (r *Loader) DelRef(ctx context.Context, key string) error { - schema, dirExists, err := r.GetRef(ctx, key) - if err != nil { - // ref does not exist -> we dont return an error - return nil - } - if dirExists { - if err := utils.RemoveDirectory(schema.Spec.GetBasePath(r.schemaDir)); err != nil { - return err - } - } - r.del(key) - return nil -} - -func (r *Loader) del(key string) { - r.sm.Lock() - defer r.sm.Unlock() - delete(r.schemas, key) -} - -// GetRef return an error if the ref does not exist -// If the ref exists the ref is retrieved with an indication if the base provider schema version dir exists -func (r *Loader) GetRef(ctx context.Context, key string) (*invv1alpha1.Schema, bool, error) { - schema, exists := r.get(key) - if !exists { - return nil, false, fmt.Errorf("no repository reference registered for key %q", key) - } - baseRefPath := schema.Spec.GetBasePath(r.schemaDir) - - return schema, utils.DirExists(baseRefPath), nil -} - -func (r *Loader) get(key string) (*invv1alpha1.Schema, bool) { - r.sm.RLock() - defer r.sm.RUnlock() - schema, exists := r.schemas[key] - return schema, exists -} - -func (r *Loader) Load(ctx context.Context, key string) error { - schema, _, err := r.GetRef(ctx, key) - if err != nil { - return err - } - - for _, schemaRepo := range schema.Spec.Repositories { - r.download(ctx, schema, schemaRepo) - } - - return nil - -} - -func (r *Loader) download(ctx context.Context, schema *invv1alpha1.Schema, schemaRepo *invv1alpha1.SchemaSpecRepository) error { - log := log.FromContext(ctx) - - - // for now we only use git, but in the future we can extend this to use other downloaders e.g. OCI/... - var downloader downloadable - switch { - default: - downloader = newGitDownloader(r.tmpDir, schema.Namespace, schemaRepo, r.credentialResolver) - } - - if downloader == nil { - return &sdcerrors.UnrecoverableError{ - Message: "could not detect repository type", - WrappedError: fmt.Errorf("no provider found for schema %q", schema.GetName()), - } - } - - sem := r.repoMgr.GetOrAdd(schemaRepo.RepositoryURL) - // Attempt to acquire the semaphore - if err := sem.Acquire(ctx, 1); err != nil { - return fmt.Errorf("failed to acquire semaphore for %s: %w", schemaRepo.RepositoryURL, err) - } - defer sem.Release(1) - - err := downloader.Download(ctx) - if err != nil { - return err - } - - localPath, err := downloader.LocalPath(schemaRepo.RepositoryURL) - if err != nil { - return err - } - providerVersionBasePath := schema.Spec.GetBasePath(r.schemaDir) - - // copy data to correct destination - if len(schemaRepo.Dirs) == 0 { - schemaRepo.Dirs = []invv1alpha1.SrcDstPath{{Src: ".", Dst: "."}} - } - for i, dir := range schemaRepo.Dirs { - // build the source path - src := path.Join(localPath, dir.Src) - // check path is still within the base schema folder - // -> prevent escaping the folder - err := utils.ErrNotIsSubfolder(localPath, src) - if err != nil { - return err - } - // build dst path - dst := path.Join(providerVersionBasePath, dir.Dst) - // check path is still within the base schema folder - // -> prevent escaping the folder - err = utils.ErrNotIsSubfolder(providerVersionBasePath, dst) - if err != nil { - return err - } - - log.Info("copying", "index", fmt.Sprintf("%d, %d", i+1, len(schemaRepo.Dirs)), "from", src, "to", dst) - err = copy.Copy(src, dst) - if err != nil { - return err - } - } - return nil -} diff --git a/pkg/schema/repomgr.go b/pkg/schema/repomgr.go deleted file mode 100644 index 01a3c3fb..00000000 --- a/pkg/schema/repomgr.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2024 Nokia. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package schema - -import ( - "sync" - - "golang.org/x/sync/semaphore" -) - -type RepoMgr struct { - m sync.RWMutex - repos map[string]*semaphore.Weighted -} - -// NewRepoMgr initializes a new RepoMgr -func NewRepoMgr() *RepoMgr { - return &RepoMgr{ - repos: make(map[string]*semaphore.Weighted), - } -} - -// getOrAdd returns the semaphore for a repository, adding it if necessary -func (r *RepoMgr) GetOrAdd(url string) *semaphore.Weighted { - r.m.Lock() - defer r.m.Unlock() - - // Check if the semaphore already exists - if sem, exists := r.repos[url]; exists { - return sem - } - - // Create and store a new semaphore - sem := semaphore.NewWeighted(1) - r.repos[url] = sem - return sem -} - -// get retrieves the semaphore for a repository -func (r *RepoMgr) get(url string) *semaphore.Weighted { - r.m.RLock() - defer r.m.RUnlock() - return r.repos[url] -} diff --git a/pkg/schema/schema_loader.go b/pkg/schema/schema_loader.go new file mode 100644 index 00000000..581cfd93 --- /dev/null +++ b/pkg/schema/schema_loader.go @@ -0,0 +1,291 @@ +package schema + +import ( + "context" + "fmt" + "net/url" + "os" + "path" + "sync" + + "github.com/go-git/go-git/v5/plumbing/transport" + logger "github.com/henderiw/logger/log" + "github.com/otiai10/copy" + "github.com/sdcio/config-server/pkg/git" + "github.com/sdcio/config-server/pkg/utils" + sdcpb "github.com/sdcio/sdc-protos/sdcpb" +) + +type SchemaLoader struct { + tmpDir string + schemaDir string + schemas map[SchemaIDHash]*SchemaDefinition + schemasMutex *sync.Mutex + schemaServerClient sdcpb.SchemaServerClient + schemaUploader SchemaServerConnector +} + +func NewSchemaLoader(tmpDir string, schemaDir string, ssc sdcpb.SchemaServerClient, schemaUploader SchemaServerConnector) (*SchemaLoader, error) { + var err error + + // create the temp dir if not existing + if !utils.DirExists(tmpDir) { + err = utils.CreateDirectory(tmpDir, 0766) + if err != nil { + return nil, err + } + } + + // create the schema dir if not existing + if !utils.DirExists(schemaDir) { + err = utils.CreateDirectory(schemaDir, 0766) + if err != nil { + return nil, err + } + } + + return &SchemaLoader{ + tmpDir: tmpDir, + schemaDir: schemaDir, + schemaServerClient: ssc, + schemaUploader: schemaUploader, + schemas: map[SchemaIDHash]*SchemaDefinition{}, + schemasMutex: &sync.Mutex{}, + }, nil +} + +func (s *SchemaLoader) SchemaExists(sid *SchemaID) bool { + s.schemasMutex.Lock() + defer s.schemasMutex.Unlock() + _, exists := s.schemas[sid.Hash()] + return exists +} + +func (s *SchemaLoader) AddSchema(ctx context.Context, sd *SchemaDefinition) error { + + relSchemaDstPath := path.Join(sd.provider, sd.version) + absSchemaDstPath := path.Join(s.schemaDir, sd.provider, sd.version) + + // if directory exists, clear it + if utils.DirExists(absSchemaDstPath) { + err := os.RemoveAll(absSchemaDstPath) + if err != nil { + return err + } + } + + // create directory + err := utils.CreateDirectory(absSchemaDstPath, 0755) + if err != nil { + return err + } + + // iterate over repositories + for _, repoSpec := range sd.repositorySpecs { + gitRepoSpec, err := git.NewRepoSpec(repoSpec.url) + if err != nil { + return err + } + + repoPath := path.Join(s.tmpDir, repoSpec.url.Path) + gitRepoSpec.SetLocalPath(repoPath) + gitRepoSpec.SetGitRef(repoSpec.refKind, repoSpec.ref) + + // init gogit instance + gogit := git.NewGoGit(gitRepoSpec) + + // perform clone + err = gogit.Clone(ctx) + if err != nil { + return err + } + + // copy data from repo to schema dir + err = s.copyDirs(ctx, repoPath, absSchemaDstPath, repoSpec.dirs) + if err != nil { + return err + } + } + + // collect the LoadSpecs of all the repo definitions + schemaOverallLoadSpec := NewSchemaLoadSpec(nil, nil, nil) + for _, x := range sd.repositorySpecs { + schemaOverallLoadSpec.Join(x.schemaLoadSpec) + } + + // add the relative path portion based on the s.schemaDir + schemaOverallLoadSpec.AddPrefixDir(relSchemaDstPath) + + // upload the schema to schema server + err = s.schemaUploader.Upload(ctx, relSchemaDstPath, sd.SchemaID, schemaOverallLoadSpec) + if err != nil { + return err + } + + s.schemasMutex.Lock() + s.schemas[sd.Hash()] = sd + s.schemasMutex.Unlock() + + return nil +} + +func (s *SchemaLoader) copyDirs(ctx context.Context, repoPath string, schemaDstPath string, sdp []*SrcDstPath) error { + + log := logger.FromContext(ctx) + + for i, dir := range sdp { + // build the source path + src := path.Join(repoPath, dir.Src) + // check path is still within the base schema folder + // -> prevent escaping the folder + err := utils.ErrNotIsSubfolder(repoPath, src) + if err != nil { + return err + } + // build dst path + dst := path.Join(schemaDstPath, dir.Dst) + // check path is still within the base schema folder + // -> prevent escaping the folder + err = utils.ErrNotIsSubfolder(schemaDstPath, dst) + if err != nil { + return err + } + + log.Info("copying", "index", fmt.Sprintf("%d, %d", i+1, len(sdp)), "from", src, "to", dst) + err = copy.Copy(src, dst) + if err != nil { + return err + } + } + + return nil +} + +func (s *SchemaLoader) RemoveSchema(ctx context.Context, schema *SchemaID) error { + _, err := s.schemaServerClient.DeleteSchema(ctx, &sdcpb.DeleteSchemaRequest{Schema: schema.ToSdcpbSchema()}) + return err +} + +type SchemaDefinition struct { + SchemaID + repositorySpecs []*RepositorySpec +} + +func NewSchemaDefinition(id *SchemaID) *SchemaDefinition { + return &SchemaDefinition{ + SchemaID: *id, + repositorySpecs: []*RepositorySpec{}, + } +} + +func (sd *SchemaDefinition) AddRepositorySpec(rs *RepositorySpec) { + sd.repositorySpecs = append(sd.repositorySpecs, rs) +} + +type RepositorySpec struct { + url *url.URL + credentials transport.AuthMethod + // Ref defines the branch or tag of the repository corresponding to the + // provider schema version + ref string + refKind git.GitRefKind + // Dirs defines the list of directories that identified the provider schema in src/dst pairs + // relative within the repository + dirs []*SrcDstPath + schemaLoadSpec *SchemaLoadSpec + proxy *url.URL +} + +func NewRepositorySpec(url *url.URL, refKind git.GitRefKind, ref string, schemaLoadSpec *SchemaLoadSpec, dirs []*SrcDstPath) *RepositorySpec { + return &RepositorySpec{ + url: url, + ref: ref, + refKind: refKind, + dirs: dirs, + schemaLoadSpec: schemaLoadSpec, + } +} + +func (rs *RepositorySpec) SetProxy(proxyUrl *url.URL) { + rs.proxy = proxyUrl +} + +func (rs *RepositorySpec) SetCredentials(cred transport.AuthMethod) { + rs.credentials = cred +} + +// SrcDstPath provide a src/dst pair for the loader to download the schema from a specific src +// in the repository to a given destination in the schema server +type SrcDstPath struct { + // Src is the relative directory in the repository URL + Src string + // Dst is the relative directory in the schema server + Dst string +} + +func NewSrcDstPath(src string, dst string) *SrcDstPath { + return &SrcDstPath{ + Src: src, + Dst: dst, + } +} + +type SchemaID struct { + provider string + version string +} + +func NewSchemaID(provider, version string) *SchemaID { + return &SchemaID{ + provider: provider, + version: version, + } +} + +func (s *SchemaID) Hash() SchemaIDHash { + return SchemaIDHash(fmt.Sprintf("%s|%s", s.provider, s.version)) +} + +func (s *SchemaID) ToSdcpbSchema() *sdcpb.Schema { + return &sdcpb.Schema{ + Vendor: s.provider, + Version: s.version, + } +} + +type SchemaIDHash string + +type SchemaLoadSpec struct { + // Models defines the list of files/directories to be used as a model + Models []string `json:"models,omitempty" yaml:"models,omitempty" protobuf:"bytes,1,rep,name=models"` + // Excludes defines the list of files/directories to be excluded + Includes []string `json:"includes,omitempty" yaml:"includes,omitempty" protobuf:"bytes,2,rep,name=includes"` + // Excludes defines the list of files/directories to be excluded + Excludes []string `json:"excludes,omitempty" yaml:"excludes,omitempty" protobuf:"bytes,3,rep,name=excludes"` +} + +func NewSchemaLoadSpec(models, includes, excludes []string) *SchemaLoadSpec { + return &SchemaLoadSpec{ + Models: models, + Includes: includes, + Excludes: excludes, + } +} + +func (s *SchemaLoadSpec) Join(other *SchemaLoadSpec) { + if other == nil { + return + } + s.Models = append(s.Models, other.Models...) + s.Includes = append(s.Includes, other.Includes...) + s.Excludes = append(s.Excludes, other.Excludes...) +} + +func (s *SchemaLoadSpec) AddPrefixDir(prefix string) { + for idx, x := range s.Models { + s.Models[idx] = path.Join(prefix, x) + } + for idx, x := range s.Includes { + s.Includes[idx] = path.Join(prefix, x) + } +} diff --git a/pkg/schema/schema_server_connector.go b/pkg/schema/schema_server_connector.go new file mode 100644 index 00000000..358d79fc --- /dev/null +++ b/pkg/schema/schema_server_connector.go @@ -0,0 +1,10 @@ +package schema + +import ( + "context" +) + +type SchemaServerConnector interface { + Upload(ctx context.Context, schemaFolder string, schema SchemaID, schemaLoadSpec *SchemaLoadSpec) error + Remove(ctx context.Context, schema SchemaID) error +} diff --git a/pkg/schema/schema_server_connector_local.go b/pkg/schema/schema_server_connector_local.go new file mode 100644 index 00000000..1e5243db --- /dev/null +++ b/pkg/schema/schema_server_connector_local.go @@ -0,0 +1,42 @@ +package schema + +import ( + "context" + + sdcpb "github.com/sdcio/sdc-protos/sdcpb" +) + +type SchemaServerConnectorLocal struct { + scc sdcpb.SchemaServerClient +} + +var _ SchemaServerConnector = (*SchemaServerConnectorLocal)(nil) + +func NewSchemaUploaderLocal(scc sdcpb.SchemaServerClient) *SchemaServerConnectorLocal { + return &SchemaServerConnectorLocal{ + scc: scc, + } +} + +func (s *SchemaServerConnectorLocal) Upload(ctx context.Context, schemaFolder string, schema SchemaID, schemaLoadSpec *SchemaLoadSpec) error { + _, err := s.scc.CreateSchema(ctx, &sdcpb.CreateSchemaRequest{ + Schema: &sdcpb.Schema{ + Vendor: schema.provider, + Version: schema.version, + }, + File: schemaLoadSpec.Models, + Directory: schemaLoadSpec.Includes, + Exclude: schemaLoadSpec.Excludes, + }) + if err != nil { + return err + } + return nil +} + +func (s *SchemaServerConnectorLocal) Remove(ctx context.Context, schema SchemaID) error { + _, err := s.scc.DeleteSchema(ctx, &sdcpb.DeleteSchemaRequest{ + Schema: schema.ToSdcpbSchema(), + }) + return err +} diff --git a/pkg/schema/schema_server_connector_remote.go b/pkg/schema/schema_server_connector_remote.go new file mode 100644 index 00000000..846038cc --- /dev/null +++ b/pkg/schema/schema_server_connector_remote.go @@ -0,0 +1,80 @@ +package schema + +import ( + "context" + + sdcpb "github.com/sdcio/sdc-protos/sdcpb" +) + +type SchemaServerConnectorRemote struct { + scc sdcpb.SchemaServerClient +} + +var _ SchemaServerConnector = (*SchemaServerConnectorRemote)(nil) + +func NewSchemaUploaderRemote() *SchemaServerConnectorRemote { + return &SchemaServerConnectorRemote{} +} + +func (s *SchemaServerConnectorRemote) Upload(ctx context.Context, schemaFolder string, schema SchemaID, schemaLoadSpec *SchemaLoadSpec) error { + uploadClient, err := s.scc.UploadSchema(ctx) + if err != nil { + return err + } + + err = uploadClient.Send( + &sdcpb.UploadSchemaRequest{ + Upload: &sdcpb.UploadSchemaRequest_CreateSchema{ + CreateSchema: &sdcpb.CreateSchemaRequest{ + Schema: &sdcpb.Schema{ + Vendor: schema.provider, + Version: schema.version, + }, + File: schemaLoadSpec.Models, // ATTENTION: not sure what kind of path needs to be provided here + Directory: schemaLoadSpec.Includes, // ATTENTION: not sure what kind of path needs to be provided here + Exclude: schemaLoadSpec.Excludes, // ATTENTION: not sure what kind of path needs to be provided here + }, + }, + }, + ) + if err != nil { + return err + } + + panic("SchemaServerConnectorRemote not implemented yet.") + + // // for files in folders do + + // err = uploadClient.Send(&sdcpb.UploadSchemaRequest{ + // Upload: &sdcpb.UploadSchemaRequest_SchemaFile{ + // SchemaFile: &sdcpb.UploadSchemaFile{ + // FileName: "foo", + // FileType: sdcpb.UploadSchemaFile_MODULE, + // Contents: []byte{}, + // Hash: &sdcpb.Hash{ + // Method: sdcpb.Hash_MD5, + // Hash: []byte{}, + // }, + + // }, + // }, + // }) + + // // finally end schema creation + // err = uploadClient.Send(&sdcpb.UploadSchemaRequest{ + // Upload: &sdcpb.UploadSchemaRequest_Finalize{ + // Finalize: &sdcpb.UploadSchemaFinalize{}, + // }, + // }) + + // // close the upload client ... not sure if this is correct + // err = uploadClient.CloseSend() + // if err != nil { + // return err + // } + // return nil +} + +func (s *SchemaServerConnectorRemote) Remove(ctx context.Context, schema SchemaID) error { + panic("SchemaServerConnectorRemote not implemented yet.") +} diff --git a/pkg/sdc/dataserver/client/client.go b/pkg/sdc/dataserver/client/client.go index e90991e9..e1811a37 100644 --- a/pkg/sdc/dataserver/client/client.go +++ b/pkg/sdc/dataserver/client/client.go @@ -134,45 +134,14 @@ func (r *client) DeleteDataStore(ctx context.Context, in *sdcpb.DeleteDataStoreR return r.dsclient.DeleteDataStore(ctx, in, opts...) } -func (r *client) Commit(ctx context.Context, in *sdcpb.CommitRequest, opts ...grpc.CallOption) (*sdcpb.CommitResponse, error) { - return r.dsclient.Commit(ctx, in, opts...) -} - -func (r *client) Rebase(ctx context.Context, in *sdcpb.RebaseRequest, opts ...grpc.CallOption) (*sdcpb.RebaseResponse, error) { - return r.dsclient.Rebase(ctx, in, opts...) -} - -func (r *client) Discard(ctx context.Context, in *sdcpb.DiscardRequest, opts ...grpc.CallOption) (*sdcpb.DiscardResponse, error) { - return r.dsclient.Discard(ctx, in, opts...) -} - -func (r *client) GetData(ctx context.Context, in *sdcpb.GetDataRequest, opts ...grpc.CallOption) (sdcpb.DataServer_GetDataClient, error) { - return r.dsclient.GetData(ctx, in, opts...) -} - //func (r *client) SetData(ctx context.Context, in *sdcpb.SetDataRequest, opts ...grpc.CallOption) (*sdcpb.SetDataResponse, error) { // return r.dsclient.SetData(ctx, in, opts...) //} -func (r *client) Diff(ctx context.Context, in *sdcpb.DiffRequest, opts ...grpc.CallOption) (*sdcpb.DiffResponse, error) { - return r.dsclient.Diff(ctx, in, opts...) -} - -func (r *client) Subscribe(ctx context.Context, in *sdcpb.SubscribeRequest, opts ...grpc.CallOption) (sdcpb.DataServer_SubscribeClient, error) { - return r.dsclient.Subscribe(ctx, in, opts...) -} - -func (r *client) Watch(ctx context.Context, in *sdcpb.WatchRequest, opts ...grpc.CallOption) (sdcpb.DataServer_WatchClient, error) { - return r.dsclient.Watch(ctx, in, opts...) -} - func (r *client) GetIntent(ctx context.Context, in *sdcpb.GetIntentRequest, opts ...grpc.CallOption) (*sdcpb.GetIntentResponse, error) { return r.dsclient.GetIntent(ctx, in, opts...) } -func (r *client) SetIntent(ctx context.Context, in *sdcpb.SetIntentRequest, opts ...grpc.CallOption) (*sdcpb.SetIntentResponse, error) { - return r.dsclient.SetIntent(ctx, in, opts...) -} func (r *client) ListIntent(ctx context.Context, in *sdcpb.ListIntentRequest, opts ...grpc.CallOption) (*sdcpb.ListIntentResponse, error) { return r.dsclient.ListIntent(ctx, in, opts...) } diff --git a/pkg/sdc/dataserver/client/fake_client.go b/pkg/sdc/dataserver/client/fake_client.go index 1e6af91a..c01d1cb6 100644 --- a/pkg/sdc/dataserver/client/fake_client.go +++ b/pkg/sdc/dataserver/client/fake_client.go @@ -55,46 +55,10 @@ func (r *fakeclient) DeleteDataStore(ctx context.Context, in *sdcpb.DeleteDataSt return &sdcpb.DeleteDataStoreResponse{}, nil } -func (r *fakeclient) Commit(ctx context.Context, in *sdcpb.CommitRequest, opts ...grpc.CallOption) (*sdcpb.CommitResponse, error) { - return &sdcpb.CommitResponse{}, nil -} - -func (r *fakeclient) Rebase(ctx context.Context, in *sdcpb.RebaseRequest, opts ...grpc.CallOption) (*sdcpb.RebaseResponse, error) { - return &sdcpb.RebaseResponse{}, nil -} - -func (r *fakeclient) Discard(ctx context.Context, in *sdcpb.DiscardRequest, opts ...grpc.CallOption) (*sdcpb.DiscardResponse, error) { - return &sdcpb.DiscardResponse{}, nil -} - -func (r *fakeclient) GetData(ctx context.Context, in *sdcpb.GetDataRequest, opts ...grpc.CallOption) (sdcpb.DataServer_GetDataClient, error) { - return nil, nil -} - -func (r *fakeclient) SetData(ctx context.Context, in *sdcpb.SetDataRequest, opts ...grpc.CallOption) (*sdcpb.SetDataResponse, error) { - return &sdcpb.SetDataResponse{}, nil -} - -func (r *fakeclient) Diff(ctx context.Context, in *sdcpb.DiffRequest, opts ...grpc.CallOption) (*sdcpb.DiffResponse, error) { - return &sdcpb.DiffResponse{}, nil -} - -func (r *fakeclient) Subscribe(ctx context.Context, in *sdcpb.SubscribeRequest, opts ...grpc.CallOption) (sdcpb.DataServer_SubscribeClient, error) { - return nil, nil -} - -func (r *fakeclient) Watch(ctx context.Context, in *sdcpb.WatchRequest, opts ...grpc.CallOption) (sdcpb.DataServer_WatchClient, error) { - return nil, nil -} - func (r *fakeclient) GetIntent(ctx context.Context, in *sdcpb.GetIntentRequest, opts ...grpc.CallOption) (*sdcpb.GetIntentResponse, error) { return &sdcpb.GetIntentResponse{}, nil } -func (r *fakeclient) SetIntent(ctx context.Context, in *sdcpb.SetIntentRequest, opts ...grpc.CallOption) (*sdcpb.SetIntentResponse, error) { - return &sdcpb.SetIntentResponse{}, nil -} - func (r *fakeclient) TransactionSet(ctx context.Context, in *sdcpb.TransactionSetRequest, opts ...grpc.CallOption) (*sdcpb.TransactionSetResponse, error) { return &sdcpb.TransactionSetResponse{}, nil } diff --git a/pkg/target/context.go b/pkg/target/context.go index 5856ad04..f04e6149 100644 --- a/pkg/target/context.go +++ b/pkg/target/context.go @@ -20,7 +20,6 @@ import ( "context" errors "errors" "fmt" - "io" "strings" "sync" @@ -145,7 +144,7 @@ func (r *Context) CreateDS(ctx context.Context, datastoreReq *sdcpb.CreateDataSt r.setDatastoreReq(datastoreReq) // will also create the deviation watcher and collector r.SetReady(ctx) - + // The collector is not started when a datastore is created but when a subscription is received. log.Debug("create datastore succeeded", "resp", prototext.Format(rsp)) return nil @@ -246,22 +245,22 @@ func (r *Context) getIntentUpdate(ctx context.Context, key storebackend.Key, con return update, nil } - func (r *Context) TransactionSet(ctx context.Context, req *sdcpb.TransactionSetRequest) (string, error) { - rsp, err := r.dsclient.TransactionSet(ctx, req) - msg, err := r.processTransactionResponse(ctx, rsp, err) - if err != nil { - return msg, err - } - // Assumption: if no error this succeeded, if error this is providing the error code and the info can be - // retrieved from the individual intents - if _, err := r.dsclient.TransactionConfirm(ctx, &sdcpb.TransactionConfirmRequest{ - DatastoreName: req.DatastoreName, - TransactionId: req.TransactionId, - }); err != nil { - return msg, err - } - return msg, nil +func (r *Context) TransactionSet(ctx context.Context, req *sdcpb.TransactionSetRequest) (string, error) { + rsp, err := r.dsclient.TransactionSet(ctx, req) + msg, err := r.processTransactionResponse(ctx, rsp, err) + if err != nil { + return msg, err } + // Assumption: if no error this succeeded, if error this is providing the error code and the info can be + // retrieved from the individual intents + if _, err := r.dsclient.TransactionConfirm(ctx, &sdcpb.TransactionConfirmRequest{ + DatastoreName: req.DatastoreName, + TransactionId: req.TransactionId, + }); err != nil { + return msg, err + } + return msg, nil +} func (r *Context) SetIntent(ctx context.Context, key storebackend.Key, config *config.Config, dryRun bool) (string, error) { log := log.FromContext(ctx).With("target", key.String(), "intent", getGVKNSN(config)) @@ -415,7 +414,7 @@ func (r *Context) Cancel(ctx context.Context, key storebackend.Key, transactionI return err } -// processTransactionResponse returns the warnings as a string and aggregates the errors in a single error and classifies them +// processTransactionResponse returns the warnings as a string and aggregates the errors in a single error and classifies them // as recoverable or non recoverable. func (r *Context) processTransactionResponse(ctx context.Context, rsp *sdcpb.TransactionSetResponse, rsperr error) (string, error) { log := log.FromContext(ctx) @@ -464,44 +463,17 @@ func (r *Context) GetData(ctx context.Context, key storebackend.Key) (*config.Ru if !r.IsReady() { return nil, fmt.Errorf("target context not ready") } - path, err := utils.ParsePath("/") - if err != nil { - return nil, fmt.Errorf("create data failed for target %s, path %s invalid", key.String(), "/") - } - stream, err := r.dsclient.GetData(ctx, &sdcpb.GetDataRequest{ - Name: key.String(), - Datastore: &sdcpb.DataStore{Type: sdcpb.Type_MAIN}, - Path: []*sdcpb.Path{path}, - DataType: sdcpb.DataType_CONFIG, - Encoding: sdcpb.Encoding_JSON, + rsp, err := r.dsclient.GetIntent(ctx, &sdcpb.GetIntentRequest{ + DatastoreName: key.String(), + Intent: "running", + Format: sdcpb.Format_Intent_Format_JSON, }) if err != nil { log.Error("get data failed", "error", err.Error()) return nil, err } - var b []byte - for { - rsp, err := stream.Recv() - if err != nil { - if err == io.EOF { - break - } - return nil, err - } - - if len(rsp.GetNotification()) == 1 { - if len(rsp.GetNotification()[0].GetUpdate()) == 1 { - b = rsp.GetNotification()[0].GetUpdate()[0].GetValue().GetJsonVal() - } else { - log.Debug("get data", "updates", len(rsp.GetNotification()[0].GetUpdate())) - } - } else { - log.Debug("get data", "notifications", len(rsp.GetNotification())) - } - } - return config.BuildRunningConfig( metav1.ObjectMeta{ Name: key.Name, @@ -510,7 +482,7 @@ func (r *Context) GetData(ctx context.Context, key storebackend.Key) (*config.Ru config.RunningConfigSpec{}, config.RunningConfigStatus{ Value: runtime.RawExtension{ - Raw: b, + Raw: rsp.GetBlob(), }, }, ), nil diff --git a/pkg/workspace/loader.go b/pkg/workspace/loader.go index 01fe62b1..fd851450 100644 --- a/pkg/workspace/loader.go +++ b/pkg/workspace/loader.go @@ -18,6 +18,7 @@ package workspace import ( "context" + "net/url" "path" "github.com/henderiw/logger/log" @@ -48,22 +49,39 @@ func NewLoader(workspaceDir string, credentialResolver auth.CredentialResolver) func (r *Loader) EnsureCommit(ctx context.Context, workspace *invv1alpha1.Workspace) (string, error) { log := log.FromContext(ctx) - repo, err := git.NewRepo(workspace.Spec.RepositoryURL) + repoUrl, err := url.Parse(workspace.Spec.RepositoryURL) + if err != nil { + return "", err + } + + repo, err := git.NewRepoSpec(repoUrl) if err != nil { return "", err } repoPath := path.Join(r.workspaceDir, repo.GetCloneURL().Path) repo.SetLocalPath(repoPath) - // init the actual git instance - goGit := git.NewGoGit(repo, - types.NamespacedName{ + if workspace.Spec.Credentials != "" { + cred, err := r.credentialResolver.ResolveCredential(ctx, types.NamespacedName{ Namespace: workspace.Namespace, - Name: workspace.Spec.Credentials}, - r.credentialResolver, - ) + Name: workspace.Spec.Credentials, + }) + if err != nil { + return "", err + } + repo.SetAuth(cred.ToAuthMethod()) + } + + // init the actual git instance + goGit := git.NewGoGit(repo) if workspace.Spec.Proxy != nil && workspace.Spec.Proxy.URL != "" { - err = goGit.SetProxy(workspace.Spec.Proxy.URL) + + proxyUrl, err := url.Parse(workspace.Spec.Proxy.URL) + if err != nil { + return "", err + } + + repo.SetProxy(proxyUrl) if err != nil { return "", err } diff --git a/pkg/workspace/reader.go b/pkg/workspace/reader.go index b0fcd7e3..615615a2 100644 --- a/pkg/workspace/reader.go +++ b/pkg/workspace/reader.go @@ -19,13 +19,13 @@ package workspace import ( "context" "fmt" + "net/url" "os" "path" "path/filepath" "github.com/henderiw/apiserver-store/pkg/storebackend" memstore "github.com/henderiw/apiserver-store/pkg/storebackend/memory" - "github.com/henderiw/logger/log" "github.com/sdcio/config-server/apis/config" configapi "github.com/sdcio/config-server/apis/config" configv1alpha1 "github.com/sdcio/config-server/apis/config/v1alpha1" @@ -49,29 +49,41 @@ func NewReader(workspaceDir string, credentialResolver auth.CredentialResolver) } func (r *Reader) GetConfigs(ctx context.Context, rollout *invv1alpha1.Rollout) (storebackend.Storer[storebackend.Storer[*config.Config]], error) { - log := log.FromContext(ctx) + // log := log.FromContext(ctx) - repo, err := git.NewRepo(rollout.Spec.RepositoryURL) + repoUrl, err := url.Parse(rollout.Spec.RepositoryURL) + if err != nil { + return nil, err + } + + repo, err := git.NewRepoSpec(repoUrl) if err != nil { return nil, err } repoPath := path.Join(r.workspaceDir, repo.GetCloneURL().Path) repo.SetLocalPath(repoPath) - // init the actual git instance - goGit := git.NewGoGit(repo, - types.NamespacedName{ - Namespace: rollout.Namespace, - Name: rollout.Spec.Credentials}, - r.credentialResolver, - ) if rollout.Spec.Proxy != nil && rollout.Spec.Proxy.URL != "" { - err = goGit.SetProxy(rollout.Spec.Proxy.URL) + proxyUrl, err := url.Parse(rollout.Spec.Proxy.URL) if err != nil { return nil, err } - log.Debug("SetProxy", "proxy", rollout.Spec.Proxy.URL) + repo.SetProxy(proxyUrl) } + + if rollout.Spec.Credentials != "" { + cred, err := r.credentialResolver.ResolveCredential(ctx, types.NamespacedName{ + Namespace: rollout.Namespace, + Name: rollout.Spec.Credentials}) + if err != nil { + return nil, err + } + repo.SetAuth(cred.ToAuthMethod()) + } + + // init the actual git instance + goGit := git.NewGoGit(repo) + if err := goGit.CheckoutCommit(ctx, rollout.Spec.Ref); err != nil { return nil, err }