From d5f31c5a9e539ba6f7cc8de928ad15d7adb3c709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20Compagnon?= Date: Tue, 3 Jun 2025 22:21:58 +0200 Subject: [PATCH 1/5] Add WebDAV storage backend --- README.md | 21 +++++++- cmd/cmd.go | 32 +++++++++++++ go.mod | 1 + go.sum | 2 + server/storage/webdav.go | 101 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 server/storage/webdav.go diff --git a/README.md b/README.md index 8a11a3c1..c99af8a3 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Easy and fast file sharing from the command-line. This code contains the server with everything you need to create your own instance. -Transfer.sh currently supports the s3 (Amazon S3), gdrive (Google Drive), storj (Storj) providers, and local file system (local). +Transfer.sh currently supports the s3 (Amazon S3), gdrive (Google Drive), storj (Storj), webdav and local file system (local). ## Disclaimer @@ -113,7 +113,7 @@ proxy-path | path prefix when service is run behind a proxy proxy-port | port of the proxy when the service is run behind a proxy | | PROXY_PORT | email-contact | email contact for the front end | | EMAIL_CONTACT | ga-key | google analytics key for the front end | | GA_KEY | -provider | which storage provider to use | (s3, storj, gdrive or local) | +provider | which storage provider to use | (s3, storj, gdrive, webdav or local) | uservoice-key | user voice key for the front end | | USERVOICE_KEY | aws-access-key | aws access key | | AWS_ACCESS_KEY | aws-secret-key | aws access key | | AWS_SECRET_KEY | @@ -124,6 +124,9 @@ s3-no-multipart | disables s3 multipart upload s3-path-style | Forces path style URLs, required for Minio. | false | S3_PATH_STYLE | storj-access | Access for the project | | STORJ_ACCESS | storj-bucket | Bucket to use within the project | | STORJ_BUCKET | +webdav-url | url of webdav server | | WEBDAV_URL | +webdav-username | username for webdav | | WEBDAV_USERNAME | +webdav-password | password for webdav | | WEBDAV_PASSWORD | basedir | path storage for local/gdrive provider | | BASEDIR | gdrive-client-json-filepath | path to oauth client json config for gdrive provider | | GDRIVE_CLIENT_JSON_FILEPATH | gdrive-local-config-path | path to store local transfer.sh config cache for gdrive provider | | GDRIVE_LOCAL_CONFIG_PATH | @@ -271,6 +274,20 @@ You need to create an OAuth Client id from console.cloud.google.com, download th ```go run main.go --provider gdrive --basedir /tmp/ --gdrive-client-json-filepath /[credential_dir] --gdrive-local-config-path [directory_to_save_config] ``` +## WebDAV Usage + +For WebDAV you need to specify the following options: +- provider `--provider webdav` +- webdav-url _(either via flag or environment variable `WEBDAV_URL`)_ +- webdav-username _(either via flag or environment variable `WEBDAV_USERNAME`)_ +- webdav-password _(either via flag or environment variable `WEBDAV_PASSWORD`)_ +- basedir + +Example: +``` +go run main.go --provider webdav --basedir /remote/path --webdav-url https://dav.example.com --webdav-username user --webdav-password pass +``` + ## Shell functions ### Bash, ash and zsh (multiple files uploaded as zip archive) diff --git a/cmd/cmd.go b/cmd/cmd.go index 3c1eece4..d80600b0 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -195,6 +195,24 @@ var globalFlags = []cli.Flag{ Value: "", EnvVars: []string{"STORJ_BUCKET"}, }, + &cli.StringFlag{ + Name: "webdav-url", + Usage: "WebDAV server URL", + Value: "", + EnvVars: []string{"WEBDAV_URL"}, + }, + &cli.StringFlag{ + Name: "webdav-username", + Usage: "WebDAV username", + Value: "", + EnvVars: []string{"WEBDAV_USERNAME"}, + }, + &cli.StringFlag{ + Name: "webdav-password", + Usage: "WebDAV password", + Value: "", + EnvVars: []string{"WEBDAV_PASSWORD"}, + }, &cli.IntFlag{ Name: "rate-limit", Usage: "requests per minute", @@ -519,6 +537,20 @@ func New() *Cmd { } else { options = append(options, server.UseStorage(store)) } + case "webdav": + if url := c.String("webdav-url"); url == "" { + return errors.New("webdav-url not set") + } else if user := c.String("webdav-username"); user == "" { + return errors.New("webdav-username not set") + } else if password := c.String("webdav-password"); password == "" { + return errors.New("webdav-password not set") + } else if basedir := c.String("basedir"); basedir == "" { + return errors.New("basedir not set") + } else if store, err := storage.NewWebDAVStorage(url, basedir, user, password, logger); err != nil { + return err + } else { + options = append(options, server.UseStorage(store)) + } case "local": if v := c.String("basedir"); v == "" { return errors.New("basedir not set.") diff --git a/go.mod b/go.mod index b08650b7..b7b40c54 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/microcosm-cc/bluemonday v1.0.23 github.com/russross/blackfriday/v2 v2.1.0 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e + github.com/studio-b12/gowebdav v0.10.0 github.com/tg123/go-htpasswd v1.2.1 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce github.com/urfave/cli/v2 v2.25.3 diff --git a/go.sum b/go.sum index 109c1174..a054837f 100644 --- a/go.sum +++ b/go.sum @@ -218,6 +218,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/studio-b12/gowebdav v0.10.0 h1:Yewz8FFiadcGEu4hxS/AAJQlHelndqln1bns3hcJIYc= +github.com/studio-b12/gowebdav v0.10.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/tg123/go-htpasswd v1.2.1 h1:i4wfsX1KvvkyoMiHZzjS0VzbAPWfxzI8INcZAKtutoU= github.com/tg123/go-htpasswd v1.2.1/go.mod h1:erHp1B86KXdwQf1X5ZrLb7erXZnWueEQezb2dql4q58= github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce h1:fb190+cK2Xz/dvi9Hv8eCYJYvIGUTN2/KLq1pT6CjEc= diff --git a/server/storage/webdav.go b/server/storage/webdav.go new file mode 100644 index 00000000..1f48bd65 --- /dev/null +++ b/server/storage/webdav.go @@ -0,0 +1,101 @@ +package storage + +import ( + "context" + "io" + "log" + "os" + "path" + "time" + + "github.com/studio-b12/gowebdav" +) + +// WebDAVStorage is a storage backed by a WebDAV server +// basePath is the root directory within the WebDAV server +// where files will be stored. +type WebDAVStorage struct { + client *gowebdav.Client + basePath string + logger *log.Logger +} + +// NewWebDAVStorage creates a new WebDAVStorage +func NewWebDAVStorage(url, basePath, username, password string, logger *log.Logger) (*WebDAVStorage, error) { + c := gowebdav.NewClient(url, username, password) + if err := c.Connect(); err != nil { + return nil, err + } + return &WebDAVStorage{client: c, basePath: basePath, logger: logger}, nil +} + +// Type returns the storage type +func (s *WebDAVStorage) Type() string { return "webdav" } + +func (s *WebDAVStorage) fullPath(token, filename string) string { + return path.Join(s.basePath, token, filename) +} + +// Head retrieves content length of a file from storage +func (s *WebDAVStorage) Head(_ context.Context, token, filename string) (uint64, error) { + fi, err := s.client.Stat(s.fullPath(token, filename)) + if err != nil { + return 0, err + } + return uint64(fi.Size()), nil +} + +// Get retrieves a file from storage +func (s *WebDAVStorage) Get(_ context.Context, token, filename string, rng *Range) (io.ReadCloser, uint64, error) { + p := s.fullPath(token, filename) + var rc io.ReadCloser + var err error + if rng != nil { + rc, err = s.client.ReadStreamRange(p, int64(rng.Start), int64(rng.Limit)) + } else { + rc, err = s.client.ReadStream(p) + } + if err != nil { + return nil, 0, err + } + fi, err := s.client.Stat(p) + if err != nil { + rc.Close() + return nil, 0, err + } + size := uint64(fi.Size()) + if rng != nil { + size = rng.AcceptLength(size) + } + return rc, size, nil +} + +// Delete removes a file from storage +func (s *WebDAVStorage) Delete(_ context.Context, token, filename string) error { + return s.client.Remove(s.fullPath(token, filename)) +} + +// Purge cleans up the storage (noop for webdav) +func (s *WebDAVStorage) Purge(context.Context, time.Duration) error { return nil } + +// Put saves a file on storage +func (s *WebDAVStorage) Put(_ context.Context, token, filename string, reader io.Reader, _ string, _ uint64) error { + dir := path.Join(s.basePath, token) + if err := s.client.MkdirAll(dir, 0755); err != nil { + return err + } + return s.client.WriteStream(s.fullPath(token, filename), reader, 0644) +} + +// IsNotExist indicates if a file doesn't exist on storage +func (s *WebDAVStorage) IsNotExist(err error) bool { + if err == nil { + return false + } + if _, ok := err.(*os.PathError); ok { + return true + } + return false +} + +func (s *WebDAVStorage) IsRangeSupported() bool { return true } From f71eca69dcfa17906ed5b9bc4264f9af861da93c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20Compagnon?= Date: Wed, 4 Jun 2025 00:24:49 +0200 Subject: [PATCH 2/5] Add error logging for WebDAV storage --- server/storage/webdav.go | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/server/storage/webdav.go b/server/storage/webdav.go index 1f48bd65..38c44335 100644 --- a/server/storage/webdav.go +++ b/server/storage/webdav.go @@ -24,6 +24,9 @@ type WebDAVStorage struct { func NewWebDAVStorage(url, basePath, username, password string, logger *log.Logger) (*WebDAVStorage, error) { c := gowebdav.NewClient(url, username, password) if err := c.Connect(); err != nil { + if logger != nil { + logger.Printf("webdav connect error: %v", err) + } return nil, err } return &WebDAVStorage{client: c, basePath: basePath, logger: logger}, nil @@ -40,6 +43,9 @@ func (s *WebDAVStorage) fullPath(token, filename string) string { func (s *WebDAVStorage) Head(_ context.Context, token, filename string) (uint64, error) { fi, err := s.client.Stat(s.fullPath(token, filename)) if err != nil { + if s.logger != nil { + s.logger.Printf("webdav head %s/%s error: %v", token, filename, err) + } return 0, err } return uint64(fi.Size()), nil @@ -56,11 +62,17 @@ func (s *WebDAVStorage) Get(_ context.Context, token, filename string, rng *Rang rc, err = s.client.ReadStream(p) } if err != nil { + if s.logger != nil { + s.logger.Printf("webdav get %s/%s error: %v", token, filename, err) + } return nil, 0, err } fi, err := s.client.Stat(p) if err != nil { rc.Close() + if s.logger != nil { + s.logger.Printf("webdav stat %s/%s error: %v", token, filename, err) + } return nil, 0, err } size := uint64(fi.Size()) @@ -72,7 +84,13 @@ func (s *WebDAVStorage) Get(_ context.Context, token, filename string, rng *Rang // Delete removes a file from storage func (s *WebDAVStorage) Delete(_ context.Context, token, filename string) error { - return s.client.Remove(s.fullPath(token, filename)) + if err := s.client.Remove(s.fullPath(token, filename)); err != nil { + if s.logger != nil { + s.logger.Printf("webdav delete %s/%s error: %v", token, filename, err) + } + return err + } + return nil } // Purge cleans up the storage (noop for webdav) @@ -82,9 +100,18 @@ func (s *WebDAVStorage) Purge(context.Context, time.Duration) error { return nil func (s *WebDAVStorage) Put(_ context.Context, token, filename string, reader io.Reader, _ string, _ uint64) error { dir := path.Join(s.basePath, token) if err := s.client.MkdirAll(dir, 0755); err != nil { + if s.logger != nil { + s.logger.Printf("webdav mkdir %s error: %v", dir, err) + } + return err + } + if err := s.client.WriteStream(s.fullPath(token, filename), reader, 0644); err != nil { + if s.logger != nil { + s.logger.Printf("webdav put %s/%s error: %v", token, filename, err) + } return err } - return s.client.WriteStream(s.fullPath(token, filename), reader, 0644) + return nil } // IsNotExist indicates if a file doesn't exist on storage From c2caa17de641c90129e71f3494f31bcf77c7a349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20Compagnon?= Date: Wed, 4 Jun 2025 00:54:40 +0200 Subject: [PATCH 3/5] Add success logs for WebDAV storage --- server/storage/webdav.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/server/storage/webdav.go b/server/storage/webdav.go index 38c44335..c8b7a3bf 100644 --- a/server/storage/webdav.go +++ b/server/storage/webdav.go @@ -29,6 +29,9 @@ func NewWebDAVStorage(url, basePath, username, password string, logger *log.Logg } return nil, err } + if logger != nil { + logger.Printf("webdav connected to %s", url) + } return &WebDAVStorage{client: c, basePath: basePath, logger: logger}, nil } @@ -48,6 +51,9 @@ func (s *WebDAVStorage) Head(_ context.Context, token, filename string) (uint64, } return 0, err } + if s.logger != nil { + s.logger.Printf("webdav head %s/%s ok", token, filename) + } return uint64(fi.Size()), nil } @@ -79,6 +85,9 @@ func (s *WebDAVStorage) Get(_ context.Context, token, filename string, rng *Rang if rng != nil { size = rng.AcceptLength(size) } + if s.logger != nil { + s.logger.Printf("webdav get %s/%s ok", token, filename) + } return rc, size, nil } @@ -90,6 +99,9 @@ func (s *WebDAVStorage) Delete(_ context.Context, token, filename string) error } return err } + if s.logger != nil { + s.logger.Printf("webdav delete %s/%s ok", token, filename) + } return nil } @@ -105,12 +117,18 @@ func (s *WebDAVStorage) Put(_ context.Context, token, filename string, reader io } return err } + if s.logger != nil { + s.logger.Printf("webdav mkdir %s ok", dir) + } if err := s.client.WriteStream(s.fullPath(token, filename), reader, 0644); err != nil { if s.logger != nil { s.logger.Printf("webdav put %s/%s error: %v", token, filename, err) } return err } + if s.logger != nil { + s.logger.Printf("webdav put %s/%s ok", token, filename) + } return nil } From 3743668ee9f9dd7a295c66e8bfb386ac43366200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20Compagnon?= Date: Wed, 4 Jun 2025 01:06:34 +0200 Subject: [PATCH 4/5] fix: handle close error in webdav --- server/storage/webdav.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/storage/webdav.go b/server/storage/webdav.go index c8b7a3bf..aa5d8040 100644 --- a/server/storage/webdav.go +++ b/server/storage/webdav.go @@ -75,7 +75,9 @@ func (s *WebDAVStorage) Get(_ context.Context, token, filename string, rng *Rang } fi, err := s.client.Stat(p) if err != nil { - rc.Close() + if cerr := rc.Close(); cerr != nil && s.logger != nil { + s.logger.Printf("webdav close %s/%s error: %v", token, filename, cerr) + } if s.logger != nil { s.logger.Printf("webdav stat %s/%s error: %v", token, filename, err) } From d26cf884d1c9c57791e58cf10f3db33f31032ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20Compagnon?= Date: Wed, 4 Jun 2025 01:23:53 +0200 Subject: [PATCH 5/5] chore(ci): update GitHub Actions versions in release workflow Updated actions/setup-go to v5, actions/upload-artifact to v4, and softprops/action-gh-release to v2 in .github/workflows/release.yml. --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee0ed8ff..aa868b68 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -105,7 +105,7 @@ jobs: echo "ASSET_NAME=$_NAME" >> $GITHUB_ENV - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: ^1.18 @@ -153,14 +153,14 @@ jobs: mv build_assets transfersh-${GITHUB_REF##*/}-${ASSET_NAME} - name: Upload files to Artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: transfersh-${{ steps.get_filename.outputs.GIT_TAG }}-${{ steps.get_filename.outputs.ASSET_NAME }} path: | ./transfersh-${{ steps.get_filename.outputs.GIT_TAG }}-${{ steps.get_filename.outputs.ASSET_NAME }}/* - name: Upload binaries to release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 if: github.event_name == 'release' with: files: |