Skip to content

Commit 287333f

Browse files
committed
Implemented X-Disable-Versioning header to disable versioning on EOS
This commit introduces a new header, `X-Disable-Versioning`, that is available on PUT requests. This header will disable versioning for this file save if the storage backend is EOS. Additionally, this commit introduces two unit tests related to this functionality: - TestDisableVersioningLeadsToCorrectQueryParams: test whether disabling versioning leads to correct query parameteers for the EOS HTTP / GRPC client - TestDisableVersioningHeaderPassedAlong: test whether a header passed to the initial endpoint is propagated to the actual upload endpoint Finally, this commit also fixes a bug that was already present, where the app was not passed along in the case of token-based authz
1 parent 979f8e6 commit 287333f

File tree

13 files changed

+219
-12
lines changed

13 files changed

+219
-12
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Enhancement: Add HTTP header to disable versioning on EOS
2+
3+
This enhancement introduces a new header, `X-Disable-Versioning`, on PUT requests. EOS will not version this file save whenever this header is set with a truthy value.
4+
See also: https://github.com/cs3org/reva/pull/4864

go.mod

+2-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ require (
1414
github.com/coreos/go-oidc/v3 v3.9.0
1515
github.com/creasty/defaults v1.7.0
1616
github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e
17-
github.com/cs3org/go-cs3apis v0.0.0-20240802083356-d617314e1795
17+
github.com/cs3org/go-cs3apis v0.0.0-20240927085705-d50e291cbf4c
1818
github.com/dgraph-io/ristretto v0.1.1
1919
github.com/dolthub/go-mysql-server v0.14.0
2020
github.com/gdexlab/go-render v1.0.1
@@ -94,6 +94,7 @@ require (
9494
github.com/hashicorp/golang-lru v1.0.2 // indirect
9595
github.com/huandu/xstrings v1.4.0 // indirect
9696
github.com/imdario/mergo v0.3.16 // indirect
97+
github.com/jarcoal/httpmock v1.3.1 // indirect
9798
github.com/klauspost/compress v1.17.7 // indirect
9899
github.com/leodido/go-urn v1.4.0 // indirect
99100
github.com/lestrrat-go/strftime v1.0.4 // indirect

go.sum

+6
Original file line numberDiff line numberDiff line change
@@ -895,6 +895,10 @@ github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJff
895895
github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4=
896896
github.com/cs3org/go-cs3apis v0.0.0-20240802083356-d617314e1795 h1:8WkweBxMQ1W6IhcK0X3eWY+aQCjEktGwVt/4KLrtOZ8=
897897
github.com/cs3org/go-cs3apis v0.0.0-20240802083356-d617314e1795/go.mod h1:yyP8PRo0EZou3nSH7H4qjlzQwaydPeIRNgX50npQHpE=
898+
github.com/cs3org/go-cs3apis v0.0.0-20240906084627-d1b1d7653d75 h1:Gcs8Y6T5/rLsUIq8vRymNbxDUpzHvqhVHT0i/LkrjAo=
899+
github.com/cs3org/go-cs3apis v0.0.0-20240906084627-d1b1d7653d75/go.mod h1:yyP8PRo0EZou3nSH7H4qjlzQwaydPeIRNgX50npQHpE=
900+
github.com/cs3org/go-cs3apis v0.0.0-20240927085705-d50e291cbf4c h1:91oR7NL5bBvwHj00a/1aTePzOBheIjeNGqBWzG6try0=
901+
github.com/cs3org/go-cs3apis v0.0.0-20240927085705-d50e291cbf4c/go.mod h1:DedpcqXl193qF/08Y04IO0PpxyyMu8+GrkD6kWK2MEQ=
898902
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
899903
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
900904
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -1202,6 +1206,8 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
12021206
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
12031207
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
12041208
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
1209+
github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
1210+
github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
12051211
github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo=
12061212
github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag=
12071213
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=

internal/http/services/owncloud/ocdav/put.go

+8
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,14 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ
278278
httpReq.Header.Set(HeaderLockHolder, lockholder)
279279
}
280280

281+
// Propagate X-Disable-Versioning header
282+
// Used to disable versioning for applications that do not expect this behaviour
283+
// See reva#4855 for more info
284+
disableVersioning, err := strconv.ParseBool(r.Header.Get(HeaderDisableVersioning))
285+
if err == nil && disableVersioning {
286+
httpReq.Header.Set(HeaderDisableVersioning, strconv.FormatBool(true))
287+
}
288+
281289
httpRes, err := s.client.Do(httpReq)
282290
if err != nil {
283291
log.Error().Err(err).Msg("error doing PUT request to data service")
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
// Copyright 2018-2024 CERN
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
//
15+
// In applying this license, CERN does not waive the privileges and immunities
16+
// granted to it by virtue of its status as an Intergovernmental Organization
17+
// or submit itself to any jurisdiction.
18+
19+
package ocdav
20+
21+
import (
22+
"context"
23+
"net/http"
24+
"net/http/httptest"
25+
"strconv"
26+
"strings"
27+
"testing"
28+
29+
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
30+
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
31+
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
32+
mockgateway "github.com/cs3org/go-cs3apis/mocks/github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
33+
"github.com/cs3org/reva/pkg/httpclient"
34+
"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
35+
"github.com/rs/zerolog"
36+
"github.com/stretchr/testify/mock"
37+
)
38+
39+
// Test that when calls come in to the PUT endpoint with a X-Disable-Versioning header,
40+
// this header is propagated to the actual upload endpoint
41+
func TestDisableVersioningHeaderPassedAlong(t *testing.T) {
42+
43+
gatewayAPIEndpoint := "my-api-endpoint"
44+
incomingPath := "http://my-reva.com/myfile.txt"
45+
input := "Hello world!"
46+
47+
// create HTTP request
48+
request := httptest.NewRequest(http.MethodPut, incomingPath, strings.NewReader(input))
49+
request.Header.Add(HeaderContentLength, strconv.Itoa(len(input)))
50+
request.Header.Add(HeaderDisableVersioning, "true")
51+
52+
// Create fake HTTP server for upload endpoint
53+
// Here we will check whether the header was correctly set
54+
calls := 0
55+
w := httptest.NewRecorder()
56+
mockServerUpload := httptest.NewServer(
57+
http.HandlerFunc(
58+
func(w http.ResponseWriter, r *http.Request) {
59+
if header := r.Header.Get(HeaderDisableVersioning); header == "" {
60+
t.Errorf("expected header %s but header was not set", HeaderDisableVersioning)
61+
}
62+
calls++
63+
},
64+
),
65+
)
66+
endpointPath := mockServerUpload.URL
67+
68+
// Set up mocked GatewayAPIClient
69+
gatewayClient := mockgateway.NewMockGatewayAPIClient(t)
70+
gatewayClient.On("Stat", mock.Anything, mock.Anything).Return(&provider.StatResponse{Status: &rpc.Status{Code: rpc.Code_CODE_NOT_FOUND}}, nil)
71+
gatewayClient.On("InitiateFileUpload", mock.Anything, mock.Anything).Return(&gateway.InitiateFileUploadResponse{
72+
Status: &rpc.Status{Code: rpc.Code_CODE_OK},
73+
Protocols: []*gateway.FileUploadProtocol{
74+
{Protocol: "simple", UploadEndpoint: endpointPath, Token: "my-secret-token"},
75+
}}, nil)
76+
pool.RegisterGatewayServiceClient(gatewayClient, gatewayAPIEndpoint)
77+
78+
// Set up OCDAV Service
79+
service := svc{
80+
c: &Config{
81+
GatewaySvc: gatewayAPIEndpoint,
82+
},
83+
client: httpclient.New(),
84+
}
85+
ref := provider.Reference{}
86+
87+
// Do the actual call
88+
service.handlePut(context.Background(), w, request, &ref, zerolog.Logger{})
89+
90+
// If no connection was made to the upload endpoint, something is also wrong
91+
if calls == 0 {
92+
t.Errorf("Upload endpoint was not called. ")
93+
}
94+
}

internal/http/services/owncloud/ocdav/webdav.go

+1
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ const (
7878
HeaderTransferAuth = "TransferHeaderAuthorization"
7979
HeaderLockID = "X-Lock-Id"
8080
HeaderLockHolder = "X-Lock-Holder"
81+
HeaderDisableVersioning = "X-Disable-Versioning"
8182
)
8283

8384
// WebDavHandler implements a dav endpoint.

pkg/eosclient/eosbinary/eosbinary.go

+12-4
Original file line numberDiff line numberDiff line change
@@ -723,7 +723,7 @@ func (c *Client) Read(ctx context.Context, auth eosclient.Authorization, path st
723723
}
724724

725725
// Write writes a stream to the mgm.
726-
func (c *Client) Write(ctx context.Context, auth eosclient.Authorization, path string, stream io.ReadCloser, app string) error {
726+
func (c *Client) Write(ctx context.Context, auth eosclient.Authorization, path string, stream io.ReadCloser, app string, disableVersioning bool) error {
727727
fd, err := os.CreateTemp(c.opt.CacheDirectory, "eoswrite-")
728728
if err != nil {
729729
return err
@@ -736,19 +736,27 @@ func (c *Client) Write(ctx context.Context, auth eosclient.Authorization, path s
736736
if err != nil {
737737
return err
738738
}
739-
return c.writeFile(ctx, auth, path, fd.Name(), app)
739+
return c.writeFile(ctx, auth, path, fd.Name(), app, disableVersioning)
740740
}
741741

742742
// WriteFile writes an existing file to the mgm.
743-
func (c *Client) writeFile(ctx context.Context, auth eosclient.Authorization, path, source, app string) error {
743+
func (c *Client) writeFile(ctx context.Context, auth eosclient.Authorization, path, source, app string, disableVersioning bool) error {
744744
xrdPath := fmt.Sprintf("%s//%s", c.opt.URL, path)
745745
args := []string{"--nopbar", "--silent", "-f", source, xrdPath}
746746

747+
options := fmt.Sprintf("-ODeos.app=%s", app)
748+
if disableVersioning {
749+
options += "&eos.versioning=0"
750+
}
751+
747752
if auth.Token != "" {
748753
args[4] += "?authz=" + auth.Token
749754
} else if auth.Role.UID != "" && auth.Role.GID != "" {
750-
args = append(args, fmt.Sprintf("-ODeos.ruid=%s&eos.rgid=%s&eos.app=%s", auth.Role.UID, auth.Role.GID, app))
755+
options += fmt.Sprintf("&eos.ruid=%s&eos.rgid=%s", auth.Role.UID, auth.Role.GID)
756+
} else {
757+
return errors.New("No authentication provided")
751758
}
759+
args = append(args, options)
752760

753761
_, _, err := c.executeXRDCopy(ctx, args)
754762
return err

pkg/eosclient/eosclient.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ type EOSClient interface {
5151
Rename(ctx context.Context, auth Authorization, oldPath, newPath string) error
5252
List(ctx context.Context, auth Authorization, path string) ([]*FileInfo, error)
5353
Read(ctx context.Context, auth Authorization, path string) (io.ReadCloser, error)
54-
Write(ctx context.Context, auth Authorization, path string, stream io.ReadCloser, app string) error
54+
Write(ctx context.Context, auth Authorization, path string, stream io.ReadCloser, app string, disableVersioning bool) error
5555
ListDeletedEntries(ctx context.Context, auth Authorization, maxentries int, from, to time.Time) ([]*DeletedEntry, error)
5656
RestoreDeletedEntry(ctx context.Context, auth Authorization, key string) error
5757
PurgeDeletedEntries(ctx context.Context, auth Authorization) error

pkg/eosclient/eosgrpc/eosgrpc.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -1321,7 +1321,7 @@ func (c *Client) Read(ctx context.Context, auth eosclient.Authorization, path st
13211321

13221322
// Write writes a file to the mgm
13231323
// Somehow the same considerations as Read apply.
1324-
func (c *Client) Write(ctx context.Context, auth eosclient.Authorization, path string, stream io.ReadCloser, app string) error {
1324+
func (c *Client) Write(ctx context.Context, auth eosclient.Authorization, path string, stream io.ReadCloser, app string, disableVersioning bool) error {
13251325
log := appctx.GetLogger(ctx)
13261326
log.Info().Str("func", "Write").Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("path", path).Msg("")
13271327
var length int64
@@ -1354,10 +1354,10 @@ func (c *Client) Write(ctx context.Context, auth eosclient.Authorization, path s
13541354
defer wfd.Close()
13551355
defer os.RemoveAll(fd.Name())
13561356

1357-
return c.httpcl.PUTFile(ctx, u.Username, auth, path, wfd, length, app)
1357+
return c.httpcl.PUTFile(ctx, u.Username, auth, path, wfd, length, app, disableVersioning)
13581358
}
13591359

1360-
return c.httpcl.PUTFile(ctx, u.Username, auth, path, stream, length, app)
1360+
return c.httpcl.PUTFile(ctx, u.Username, auth, path, stream, length, app, disableVersioning)
13611361
}
13621362

13631363
// ListDeletedEntries returns a list of the deleted entries.

pkg/eosclient/eosgrpc/eoshttp.go

+17-2
Original file line numberDiff line numberDiff line change
@@ -352,12 +352,27 @@ func (c *EOSHTTPClient) GETFile(ctx context.Context, remoteuser string, auth eos
352352
}
353353

354354
// PUTFile does an entire PUT to upload a full file, taking the data from a stream.
355-
func (c *EOSHTTPClient) PUTFile(ctx context.Context, remoteuser string, auth eosclient.Authorization, urlpath string, stream io.ReadCloser, length int64, app string) error {
355+
func (c *EOSHTTPClient) PUTFile(ctx context.Context, remoteuser string, auth eosclient.Authorization, urlpath string, stream io.ReadCloser, length int64, app string, disableVersioning bool) error {
356356
log := appctx.GetLogger(ctx)
357357
log.Info().Str("func", "PUTFile").Str("remoteuser", remoteuser).Str("uid,gid", auth.Role.UID+","+auth.Role.GID).Str("path", urlpath).Int64("length", length).Str("app", app).Msg("")
358358

359359
// Now send the req and see what happens
360-
finalurl, err := c.buildFullURL(urlpath, auth)
360+
tempUrl, err := c.buildFullURL(urlpath, auth)
361+
if err != nil {
362+
return err
363+
}
364+
base, err := url.Parse(tempUrl)
365+
if err != nil {
366+
return errtypes.PermissionDenied("Could not parse url " + urlpath)
367+
}
368+
queryValues := base.Query()
369+
370+
if disableVersioning {
371+
queryValues.Add("eos.versioning", strconv.Itoa(0))
372+
}
373+
base.RawQuery = queryValues.Encode()
374+
finalurl := base.String()
375+
361376
if err != nil {
362377
log.Error().Str("func", "PUTFile").Str("url", finalurl).Str("err", err.Error()).Msg("can't create request")
363378
return err

pkg/eosclient/eosgrpc/eoshttp_test.go

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package eosgrpc
2+
3+
import (
4+
"context"
5+
"io"
6+
"net/http"
7+
"net/http/httptest"
8+
"strconv"
9+
"strings"
10+
"testing"
11+
12+
"github.com/cs3org/reva/pkg/eosclient"
13+
)
14+
15+
// Test that, when the PUTFile method is called with disableVersioning
16+
// set to true, the url for the EOS endpoint contains the right query param
17+
func TestDisableVersioningLeadsToCorrectQueryParams(t *testing.T) {
18+
19+
stream := io.NopCloser(strings.NewReader("Hello world!"))
20+
length := int64(12)
21+
app := "my-app"
22+
urlpath := "/my-file.txt?queryparam=1"
23+
token := "my-secret-token"
24+
25+
// Create fake HTTP server that acts as the EOS endpoint
26+
calls := 0
27+
mockServerUpload := httptest.NewServer(
28+
http.HandlerFunc(
29+
func(w http.ResponseWriter, r *http.Request) {
30+
calls++
31+
queryValues := r.URL.Query()
32+
if queryValues.Get("eos.versioning") == "" {
33+
t.Errorf("Query parameter eos.versioning not set")
34+
}
35+
if q := queryValues.Get("eos.versioning"); q != strconv.Itoa(0) {
36+
t.Errorf("Query parameter eos.versioning set to wrong value; got %s, expected 0", q)
37+
}
38+
},
39+
),
40+
)
41+
42+
// Create EOS HTTP Client
43+
// TODO: right now, expects files to be on the FS
44+
client, err := NewEOSHTTPClient(&HTTPOptions{
45+
BaseURL: mockServerUpload.URL,
46+
})
47+
if err != nil {
48+
t.Errorf("Failed to construct client: %s", err.Error())
49+
}
50+
51+
// Test actual PUTFile call
52+
client.PUTFile(context.Background(), "remote-user", eosclient.Authorization{
53+
Token: token}, urlpath, stream, length, app, true)
54+
55+
// If no connection was made to the EOS endpoint, something is wrong
56+
if calls == 0 {
57+
t.Errorf("EOS endpoint was not called. ")
58+
}
59+
}

pkg/rhttp/datatx/manager/simple/simple.go

+4
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
8484
metadata["lockholder"] = lockholder
8585
}
8686

87+
if disableVersioning := r.Header.Get(ocdav.HeaderDisableVersioning); disableVersioning != "" {
88+
metadata["disableVersioning"] = disableVersioning
89+
}
90+
8791
err := fs.Upload(ctx, ref, r.Body, metadata)
8892
switch v := err.(type) {
8993
case nil:

pkg/storage/utils/eosfs/upload.go

+8-1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"io"
2424
"os"
2525
"path"
26+
"strconv"
2627

2728
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
2829
"github.com/cs3org/reva/pkg/errtypes"
@@ -86,7 +87,13 @@ func (fs *eosfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadC
8687
// if we have a lock context, the app for EOS must match the lock holder
8788
app = fs.EncodeAppName(app)
8889
}
89-
return fs.c.Write(ctx, auth, fn, r, app)
90+
91+
disableVersioning, err := strconv.ParseBool(metadata["disableVersioning"])
92+
if err != nil {
93+
disableVersioning = false
94+
}
95+
96+
return fs.c.Write(ctx, auth, fn, r, app, disableVersioning)
9097
}
9198

9299
func (fs *eosfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) {

0 commit comments

Comments
 (0)