diff --git a/cback/http/cback.go b/cback/http/cback.go new file mode 100644 index 0000000..03a9696 --- /dev/null +++ b/cback/http/cback.go @@ -0,0 +1,334 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package cback + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "strconv" + "text/template" + "time" + + "github.com/Masterminds/sprig" + cbackfs "github.com/cernbox/reva-plugins/cback/storage" + cback "github.com/cernbox/reva-plugins/cback/utils" + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + storage "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/pkg/rhttp/global" + "github.com/cs3org/reva/pkg/sharedconf" + "github.com/go-chi/chi/v5" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + "github.com/rs/zerolog" +) + +func init() { + reva.RegisterPlugin(svc{}) +} + +const webdavPrefix = "/remote.php/dav/files/" + +type config struct { + Prefix string `mapstructure:"prefix"` + Token string `mapstructure:"token"` + URL string `mapstructure:"url"` + Insecure bool `mapstructure:"insecure"` + Timeout int `mapstructure:"timeout"` + GatewaySvc string `mapstructure:"gatewaysvc"` + StorageID string `mapstructure:"storage_id"` + TemplateToStorage string `mapstructure:"template_to_storage"` + TemplateToCback string `mapstructure:"template_to_cback"` +} + +type svc struct { + config *config + router *chi.Mux + client *cback.Client + gw gateway.GatewayAPIClient + tplStorage *template.Template + tplCback *template.Template +} + +func (svc) RevaPlugin() reva.PluginInfo { + return reva.PluginInfo{ + ID: "http.services.cback", + New: New, + } +} + +// New returns a new cback http service. +func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + return nil, errors.Wrap(err, "cback: error decodinf config") + } + + c.init() + + gw, err := pool.GetGatewayServiceClient(pool.Endpoint(c.GatewaySvc)) + if err != nil { + return nil, errors.Wrap(err, "cback: error getting gateway client") + } + + tplStorage, err := template.New("tpl_storage").Funcs(sprig.TxtFuncMap()).Parse(c.TemplateToStorage) + if err != nil { + return nil, errors.Wrap(err, "cback: error creating template") + } + + tplCback, err := template.New("tpl_cback").Funcs(sprig.TxtFuncMap()).Parse(c.TemplateToCback) + if err != nil { + return nil, errors.Wrap(err, "cback: error creating template") + } + + r := chi.NewRouter() + s := &svc{ + config: c, + gw: gw, + router: r, + client: cback.New(&cback.Config{ + URL: c.URL, + Token: c.Token, + Timeout: c.Timeout, + }), + tplStorage: tplStorage, + tplCback: tplCback, + } + + s.initRouter() + + return s, nil +} + +// Close cleanup the cback http service. +func (s *svc) Close() error { + return nil +} + +func (c *config) init() { + if c.Prefix == "" { + c.Prefix = "cback" + } + if c.TemplateToStorage == "" { + c.TemplateToStorage = "{{.}}" + } + if c.TemplateToCback == "" { + c.TemplateToCback = "{{.}}" + } + c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) +} + +func (s *svc) Prefix() string { + return s.config.Prefix +} + +func (s *svc) Unprotected() []string { + return nil +} + +func (s *svc) initRouter() { + s.router.Get("/restores", s.getRestores) + s.router.Get("/restores/{id}", s.getRestoreByID) + s.router.Post("/restores", s.createRestore) + + s.router.Get("/backups", s.getBackups) +} + +type restoreOut struct { + ID int `json:"id"` + Path string `json:"path"` + Destination string `json:"destination"` + Status int `json:"status"` + Created time.Time `json:"created"` +} + +func (s *svc) convertToRestoureOut(r *cback.Restore) *restoreOut { + dest, _ := getPath(r.Destionation, s.tplStorage) + return &restoreOut{ + ID: r.ID, + Path: r.Pattern, + Destination: dest, + Status: r.Status, + Created: r.Created.Time, + } +} + +func (s *svc) createRestore(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + user, ok := appctx.ContextGetUser(ctx) + if !ok { + http.Error(w, "user not authenticated", http.StatusUnauthorized) + return + } + + path := r.URL.Query().Get("path") + if path == "" { + http.Error(w, "missing path", http.StatusBadRequest) + return + } + + stat, err := s.gw.Stat(ctx, &storage.StatRequest{ + Ref: &storage.Reference{ + Path: path, + }, + }) + + switch { + case err != nil: + http.Error(w, err.Error(), http.StatusInternalServerError) + return + case stat.Status.Code == rpc.Code_CODE_NOT_FOUND: + http.Error(w, stat.Status.Message, http.StatusNotFound) + return + case stat.Status.Code != rpc.Code_CODE_OK: + http.Error(w, stat.Status.Message, http.StatusInternalServerError) + return + } + + if stat.Info.Id == nil || stat.Info.Id.StorageId != s.config.StorageID { + http.Error(w, fmt.Sprintf("path not belonging to %s storage driver", s.config.StorageID), http.StatusBadRequest) + return + } + + path, snapshotID, backupID, ok := cbackfs.GetBackupInfo(stat.Info.Id) + if !ok { + http.Error(w, "cannot restore the given path", http.StatusBadRequest) + return + } + + restore, err := s.client.NewRestore(ctx, user.Username, backupID, s.cbackPath(path), snapshotID, true) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + s.writeJSON(w, s.convertToRestoureOut(restore)) +} + +func must[T any](v T, err error) T { + if err != nil { + panic(err) + } + return v +} + +func (s *svc) cbackPath(p string) string { + return must(getPath(p, s.tplCback)) +} + +func (s *svc) getRestores(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + user, ok := appctx.ContextGetUser(ctx) + if !ok { + http.Error(w, "user not authenticated", http.StatusUnauthorized) + return + } + + list, err := s.client.ListRestores(ctx, user.Username) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + res := make([]*restoreOut, 0, len(list)) + for _, r := range list { + res = append(res, s.convertToRestoureOut(r)) + } + + s.writeJSON(w, res) +} + +func (s *svc) writeJSON(w http.ResponseWriter, r any) { + w.WriteHeader(http.StatusOK) + w.Header().Add("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(r) +} + +func (s *svc) getRestoreByID(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + user, ok := appctx.ContextGetUser(ctx) + if !ok { + http.Error(w, "user not authenticated", http.StatusUnauthorized) + return + } + + id := chi.URLParam(r, "id") + restoreID, err := strconv.ParseInt(id, 10, 32) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + restore, err := s.client.GetRestore(ctx, user.Username, int(restoreID)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + s.writeJSON(w, s.convertToRestoureOut(restore)) +} + +func getPath(p string, tpl *template.Template) (string, error) { + var b bytes.Buffer + if err := tpl.Execute(&b, p); err != nil { + return "", err + } + return b.String(), nil +} + +func (s *svc) getBackups(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + user, ok := appctx.ContextGetUser(ctx) + if !ok { + http.Error(w, "user not authenticated", http.StatusUnauthorized) + return + } + + list, err := s.client.ListBackups(ctx, user.Username) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + paths := make([]string, 0, len(list)) + for _, b := range list { + d, err := getPath(b.Source, s.tplStorage) + if err != nil { + continue + } + paths = append(paths, d) + } + + s.writeJSON(w, paths) +} + +func (s *svc) Handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.router.ServeHTTP(w, r) + }) +} diff --git a/cback/storage/cache.go b/cback/storage/cache.go new file mode 100644 index 0000000..4e5450a --- /dev/null +++ b/cback/storage/cache.go @@ -0,0 +1,88 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package cbackfs + +import ( + "context" + "fmt" + "time" + + "github.com/cernbox/reva-plugins/cback/utils" +) + +func (f *fs) listBackups(ctx context.Context, username string) ([]*utils.Backup, error) { + key := "backups:" + username + if d, err := f.cache.Get(key); err == nil { + return d.([]*utils.Backup), nil + } + backups, err := f.client.ListBackups(ctx, username) + if err != nil { + return nil, err + } + for _, b := range backups { + b.Source = convertTemplate(b.Source, f.tplStorage) + } + _ = f.cache.SetWithExpire(key, backups, time.Duration(f.conf.Expiration)*time.Second) + return backups, nil +} + +func (f *fs) stat(ctx context.Context, username string, id int, snapshot, path string) (*utils.Resource, error) { + key := fmt.Sprintf("stat:%s:%d:%s:%s", username, id, snapshot, path) + if s, err := f.cache.Get(key); err == nil { + return s.(*utils.Resource), nil + } + s, err := f.client.Stat(ctx, username, id, snapshot, path, true) + if err != nil { + return nil, err + } + _ = f.cache.SetWithExpire(key, s, time.Duration(f.conf.Expiration)*time.Second) + return s, nil +} + +func (f *fs) listFolder(ctx context.Context, username string, id int, snapshot, path string) ([]*utils.Resource, error) { + key := fmt.Sprintf("list:%s:%d:%s:%s", username, id, snapshot, path) + if l, err := f.cache.Get(key); err == nil { + return l.([]*utils.Resource), nil + } + path = convertTemplate(path, f.tplCback) + l, err := f.client.ListFolder(ctx, username, id, snapshot, path, true) + if err != nil { + return nil, err + } + _ = f.cache.SetWithExpire(key, l, time.Duration(f.conf.Expiration)*time.Second) + return l, nil +} + +func (f *fs) listSnapshots(ctx context.Context, username string, id int) ([]*utils.Snapshot, error) { + key := fmt.Sprintf("snapshots:%s:%d", username, id) + if l, err := f.cache.Get(key); err == nil { + return l.([]*utils.Snapshot), nil + } + l, err := f.client.ListSnapshots(ctx, username, id) + if err != nil { + return nil, err + } + for _, snap := range l { + // truncate the time according to the given format + t, _ := time.Parse(f.conf.TimestampFormat, snap.Time.Format(f.conf.TimestampFormat)) + snap.Time = utils.CBackTime{Time: t} + } + _ = f.cache.SetWithExpire(key, l, time.Duration(f.conf.Expiration)*time.Second) + return l, nil +} diff --git a/cback/storage/cback.go b/cback/storage/cback.go new file mode 100644 index 0000000..cffe478 --- /dev/null +++ b/cback/storage/cback.go @@ -0,0 +1,568 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package cbackfs + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io" + "net/url" + "path/filepath" + "strconv" + "strings" + "text/template" + "time" + + "github.com/Masterminds/sprig" + "github.com/bluele/gcache" + "github.com/cernbox/reva-plugins/cback/utils" + cback "github.com/cernbox/reva-plugins/cback/utils" + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/mime" + "github.com/cs3org/reva/pkg/storage" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +type fs struct { + conf *Config + client *utils.Client + cache gcache.Cache + tplStorage *template.Template + tplCback *template.Template +} + +func init() { + reva.RegisterPlugin(fs{}) +} + +// New returns an implementation to the storage.FS interface that expose +// the snapshots stored in cback. +func New(m map[string]interface{}) (storage.FS, error) { + c := &Config{} + if err := mapstructure.Decode(m, c); err != nil { + return nil, errors.Wrap(err, "cback: error decoding config") + } + c.init() + + tplStorage, err := template.New("tpl_storage").Funcs(sprig.TxtFuncMap()).Parse(c.TemplateToStorage) + if err != nil { + return nil, errors.Wrap(err, "cback: error creating template") + } + + tplCback, err := template.New("tpl_cback").Funcs(sprig.TxtFuncMap()).Parse(c.TemplateToCback) + if err != nil { + return nil, errors.Wrap(err, "cback: error creating template") + } + + client := utils.New( + &utils.Config{ + URL: c.APIURL, + Token: c.Token, + Timeout: c.Timeout, + }, + ) + + return &fs{ + conf: c, + client: client, + cache: gcache.New(c.Size).LRU().Build(), + tplStorage: tplStorage, + tplCback: tplCback, + }, nil +} + +func split(path string, backups []*cback.Backup) (string, string, string, int, bool) { + for _, b := range backups { + if strings.HasPrefix(path, b.Source) { + // the path could be in this form: + // // + // snap_id and path are optional + rel, _ := filepath.Rel(b.Source, path) + if rel == "." { + // both snap_id and path were not provided + return b.Source, "", "", b.ID, true + } + split := strings.SplitN(rel, "/", 2) + + var snap, p string + snap = split[0] + if len(split) == 2 { + p = split[1] + } + return b.Source, snap, p, b.ID, true + } + } + return "", "", "", 0, false +} + +func (fs) RevaPlugin() reva.PluginInfo { + return reva.PluginInfo{ + ID: "grpc.services.storageprovider.drivers.cback", + New: New, + } +} + +func (f *fs) convertToResourceInfo(r *utils.Resource, path string, resID, parentID *provider.ResourceId, owner *user.UserId) *provider.ResourceInfo { + rtype := provider.ResourceType_RESOURCE_TYPE_FILE + perms := permFile + if r.IsDir() { + rtype = provider.ResourceType_RESOURCE_TYPE_CONTAINER + perms = permDir + } + + return &provider.ResourceInfo{ + Type: rtype, + Id: resID, + Checksum: &provider.ResourceChecksum{ + Type: provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_UNSET, + }, + Etag: strconv.FormatUint(uint64(r.CTime), 10), + MimeType: mime.Detect(r.IsDir(), path), + Mtime: &types.Timestamp{ + Seconds: uint64(r.CTime), + }, + Path: path, + PermissionSet: perms, + Size: r.Size, + Owner: owner, + ParentId: parentID, + } +} + +func encodeBackupInResourceID(backupID int, snapshotID, source, path string) *provider.ResourceId { + id := fmt.Sprintf("%d#%s#%s#%s", backupID, snapshotID, source, path) + opaque := base64.StdEncoding.EncodeToString([]byte(id)) + return &provider.ResourceId{ + StorageId: "cback", + OpaqueId: opaque, + } +} + +// return b.Source, snap, p, b.ID, true. +func decodeResourceID(r *provider.ResourceId) (string, string, string, int, bool) { + if r == nil { + return "", "", "", 0, false + } + data, err := base64.StdEncoding.DecodeString(r.OpaqueId) + if err != nil { + return "", "", "", 0, false + } + split := strings.SplitN(string(data), "#", 4) + if len(split) != 4 { + return "", "", "", 0, false + } + backupID, err := strconv.ParseInt(split[0], 10, 64) + if err != nil { + return "", "", "", 0, false + } + return split[2], split[1], split[3], int(backupID), true +} + +// GetBackupInfo returns a tuple path, snapshot, backup id from a resource id. +func GetBackupInfo(r *provider.ResourceId) (string, string, int, bool) { + source, snap, path, id, ok := decodeResourceID(r) + return filepath.Join(source, path), snap, id, ok +} + +func (f *fs) placeholderResourceInfo(path string, owner *user.UserId, mtime *types.Timestamp, resID *provider.ResourceId) *provider.ResourceInfo { + if mtime == nil { + mtime = &types.Timestamp{ + Seconds: 0, + } + } + if resID == nil { + resID = &provider.ResourceId{ + StorageId: "cback", + OpaqueId: path, + } + } + return &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_CONTAINER, + Id: resID, + Checksum: &provider.ResourceChecksum{ + Type: provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_UNSET, + }, + Etag: "", + MimeType: mime.Detect(true, path), + Mtime: mtime, + Path: path, + PermissionSet: permDir, + Size: 0, + Owner: owner, + } +} + +func hasPrefix(lst, prefix []string) bool { + for i, p := range prefix { + if lst[i] != p { + return false + } + } + return true +} + +func (f *fs) isParentOfBackup(path string, backups []*utils.Backup) bool { + pathSplit := []string{""} + if path != "/" { + pathSplit = strings.Split(path, "/") + } + for _, b := range backups { + backupSplit := strings.Split(b.Source, "/") + if hasPrefix(backupSplit, pathSplit) { + return true + } + } + return false +} + +func (f *fs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) { + user, ok := appctx.ContextGetUser(ctx) + if !ok { + return nil, errtypes.UserRequired("cback: user not found in context") + } + + var ( + source, snapshot, path string + id int + ) + + backups, err := f.listBackups(ctx, user.Username) + if err != nil { + return nil, errors.Wrapf(err, "cback: error listing backups") + } + + if ref.ResourceId != nil { + source, snapshot, path, id, ok = decodeResourceID(ref.ResourceId) + if ref.Path != "" { + path = filepath.Join(path, ref.Path) + } + } else { + source, snapshot, path, id, ok = split(ref.Path, backups) + source = convertTemplate(source, f.tplCback) + } + + if ok { + if snapshot != "" && path != "" { + // the path from the user is something like /eos/home-g/gdelmont//rest/of/path + // in this case the method has to return the stat of the file /eos/home-g/gdelmont/rest/of/path + // in the snapshot + res, err := f.stat(ctx, user.Username, id, snapshot, filepath.Join(source, path)) + if err != nil { + return nil, err + } + return f.convertToResourceInfo( + res, + filepath.Join(source, snapshot, path), + encodeBackupInResourceID(id, snapshot, source, path), + encodeBackupInResourceID(id, snapshot, source, filepath.Dir(path)), + user.Id, + ), nil + } else if snapshot != "" && path == "" { + // the path from the user is something like /eos/home-g/gdelmont/ + snap, err := f.getSnapshot(ctx, user.Username, id, snapshot) + if err != nil { + return nil, errors.Wrap(err, "cback: error getting snapshot") + } + return f.placeholderResourceInfo(filepath.Join(source, snapshot), user.Id, timeToTimestamp(snap.Time.Time), encodeBackupInResourceID(id, snapshot, source, "")), nil + } + // the path from the user is something like /eos/home-g/gdelmont + return f.placeholderResourceInfo(source, user.Id, nil, nil), nil + } + + // the path is not one of the backup. There is a situation in which + // the user's path is a parent folder of some of the backups + + if f.isParentOfBackup(source, backups) { + return f.placeholderResourceInfo(source, user.Id, nil, nil), nil + } + + return nil, errtypes.NotFound(fmt.Sprintf("path %s does not exist", source)) +} + +func timeToTimestamp(t time.Time) *types.Timestamp { + return &types.Timestamp{ + Seconds: uint64(t.Unix()), + } +} + +func (f *fs) getSnapshot(ctx context.Context, username string, backupID int, timestamp string) (*cback.Snapshot, error) { + snapshots, err := f.listSnapshots(ctx, username, backupID) + if err != nil { + return nil, err + } + t, err := time.Parse(f.conf.TimestampFormat, timestamp) + if err != nil { + return nil, err + } + for _, snap := range snapshots { + if snap.Time.Equal(t) { + return snap, nil + } + } + return nil, errtypes.NotFound(fmt.Sprintf("snapshot %s from backup %d not found", timestamp, backupID)) +} + +func (f *fs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) ([]*provider.ResourceInfo, error) { + user, ok := appctx.ContextGetUser(ctx) + if !ok { + return nil, errtypes.UserRequired("cback: user not found in context") + } + + backups, err := f.listBackups(ctx, user.Username) + if err != nil { + return nil, errors.Wrapf(err, "cback: error listing backups") + } + + source, snapshot, path, id, ok := split(ref.Path, backups) + if ok { + if snapshot != "" { + // the path from the user is something like /eos/home-g/gdelmont//(rest/of/path) + // in this case the method has to return the content of the folder /eos/home-g/gdelmont/(rest/of/path) + // in the snapshot + content, err := f.listFolder(ctx, user.Username, id, snapshot, filepath.Join(source, path)) + if err != nil { + return nil, err + } + res := make([]*provider.ResourceInfo, 0, len(content)) + parentID := encodeBackupInResourceID(id, snapshot, source, path) + for _, info := range content { + base := filepath.Base(info.Name) + res = append(res, f.convertToResourceInfo( + info, + filepath.Join(source, snapshot, path, base), + encodeBackupInResourceID(id, snapshot, source, filepath.Join(path, base)), + parentID, + user.Id, + )) + } + return res, nil + } + // the path from the user is something like /eos/home-g/gdelmont + // the method needs to return the list of snapshots as folders + snapshots, err := f.listSnapshots(ctx, user.Username, id) + if err != nil { + return nil, err + } + res := make([]*provider.ResourceInfo, 0, len(snapshots)) + for _, s := range snapshots { + snapTime := s.Time.Format(f.conf.TimestampFormat) + res = append(res, f.placeholderResourceInfo(filepath.Join(source, snapTime), user.Id, timeToTimestamp(s.Time.Time), encodeBackupInResourceID(id, snapTime, source, ""))) + } + return res, nil + } + + // the path is not one of the backup. Can happen that the + // user's path is a parent folder of some of the backups + resSet := make(map[string]struct{}) // used to discard duplicates + var resources []*provider.ResourceInfo + + sourceSplit := []string{""} + if ref.Path != "/" { + sourceSplit = strings.Split(ref.Path, "/") + } + for _, b := range backups { + backupSplit := strings.Split(b.Source, "/") + if hasPrefix(backupSplit, sourceSplit) { + base := backupSplit[len(sourceSplit)] + path := filepath.Join(ref.Path, base) + + if _, ok := resSet[path]; !ok { + resources = append(resources, f.placeholderResourceInfo(path, user.Id, nil, nil)) + resSet[path] = struct{}{} + } + } + } + + if len(resources) != 0 { + return resources, nil + } + + return nil, errtypes.NotFound(fmt.Sprintf("path %s does not exist", ref.Path)) +} + +func (f *fs) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { + user, ok := appctx.ContextGetUser(ctx) + if !ok { + return nil, errtypes.UserRequired("cback: user not found in context") + } + + stat, err := f.GetMD(ctx, ref, nil) + if err != nil { + return nil, errors.Wrap(err, "cback: error statting resource") + } + + if stat.Type != provider.ResourceType_RESOURCE_TYPE_FILE { + return nil, errtypes.BadRequest("cback: can only download files") + } + + source, snapshot, path, id, ok := decodeResourceID(stat.Id) + if !ok { + return nil, errtypes.BadRequest("cback: can only download files") + } + source = convertTemplate(source, f.tplCback) + return f.client.Download(ctx, user.Username, id, snapshot, filepath.Join(source, path), true) +} + +func convertTemplate(s string, t *template.Template) string { + var b bytes.Buffer + if err := t.Execute(&b, s); err != nil { + panic(errors.Wrap(err, "error executing template")) + } + return b.String() +} + +func (f *fs) GetHome(ctx context.Context) (string, error) { + return "", errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) CreateHome(ctx context.Context) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) CreateDir(ctx context.Context, ref *provider.Reference) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) TouchFile(ctx context.Context, ref *provider.Reference) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) Delete(ctx context.Context, ref *provider.Reference) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) Move(ctx context.Context, oldRef, newRef *provider.Reference) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) ListRevisions(ctx context.Context, ref *provider.Reference) ([]*provider.FileVersion, error) { + return nil, errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) DownloadRevision(ctx context.Context, ref *provider.Reference, key string) (io.ReadCloser, error) { + return nil, errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) RestoreRevision(ctx context.Context, ref *provider.Reference, key string) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) { + return "", errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) DenyGrant(ctx context.Context, ref *provider.Reference, g *provider.Grantee) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) ListGrants(ctx context.Context, ref *provider.Reference) ([]*provider.Grant, error) { + return nil, errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) GetQuota(ctx context.Context, ref *provider.Reference) (uint64, uint64, error) { + return 0, 0, errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) CreateReference(ctx context.Context, path string, targetURI *url.URL) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) Shutdown(ctx context.Context) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) EmptyRecycle(ctx context.Context) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) { + return nil, errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) ListRecycle(ctx context.Context, basePath, key, relativePath string) ([]*provider.RecycleItem, error) { + return nil, errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) RestoreRecycleItem(ctx context.Context, basePath, key, relativePath string, restoreRef *provider.Reference) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) PurgeRecycleItem(ctx context.Context, basePath, key, relativePath string) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { + return nil, errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) UpdateStorageSpace(ctx context.Context, req *provider.UpdateStorageSpaceRequest) (*provider.UpdateStorageSpaceResponse, error) { + return nil, errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) SetLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) { + return nil, errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) RefreshLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock, existingLockID string) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) Unlock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { + return errtypes.NotSupported("Operation Not Permitted") +} + +func (f *fs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { + return nil, errtypes.NotSupported("Operation Not Permitted") +} diff --git a/cback/storage/config.go b/cback/storage/config.go new file mode 100644 index 0000000..1f4d430 --- /dev/null +++ b/cback/storage/config.go @@ -0,0 +1,100 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package cbackfs + +import provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + +// Config for the cback driver. +type Config struct { + Token string `mapstructure:"token"` + APIURL string `mapstructure:"api_url"` + Insecure bool `mapstructure:"insecure"` + Timeout int `mapstructure:"timeout"` + Size int `mapstructure:"size"` + Expiration int `mapstructure:"expiration"` + TemplateToStorage string `mapstructure:"template_to_storage"` + TemplateToCback string `mapstructure:"template_to_cback"` + TimestampFormat string `mapstructure:"timestamp_format"` +} + +func (c *Config) init() { + if c.Size == 0 { + c.Size = 1_000_000 + } + + if c.Expiration == 0 { + c.Expiration = 300 + } + + if c.TemplateToCback == "" { + c.TemplateToCback = "{{ . }}" + } + + if c.TemplateToStorage == "" { + c.TemplateToStorage = "{{ . }}" + } + + if c.TimestampFormat == "" { + c.TimestampFormat = "2006-01-02T15:04:05Z07:00" + } +} + +var permDir = &provider.ResourcePermissions{ + AddGrant: false, + CreateContainer: false, + Delete: false, + GetPath: true, + GetQuota: true, + InitiateFileDownload: true, + InitiateFileUpload: false, + ListGrants: true, + ListContainer: true, + ListFileVersions: true, + ListRecycle: false, + Move: false, + RemoveGrant: false, + PurgeRecycle: false, + RestoreFileVersion: false, + RestoreRecycleItem: false, + Stat: true, + UpdateGrant: false, + DenyGrant: false, +} + +var permFile = &provider.ResourcePermissions{ + AddGrant: false, + CreateContainer: false, + Delete: false, + GetPath: true, + GetQuota: true, + InitiateFileDownload: true, + InitiateFileUpload: false, + ListGrants: true, + ListContainer: false, + ListFileVersions: true, + ListRecycle: false, + Move: false, + RemoveGrant: false, + PurgeRecycle: false, + RestoreFileVersion: false, + RestoreRecycleItem: false, + Stat: true, + UpdateGrant: false, + DenyGrant: false, +} diff --git a/cback/utils/cback.go b/cback/utils/cback.go new file mode 100644 index 0000000..cc902d7 --- /dev/null +++ b/cback/utils/cback.go @@ -0,0 +1,252 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package utils + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/httpclient" + "github.com/pkg/errors" +) + +// Config is the config used by the cback client. +type Config struct { + URL string + Token string + Timeout int +} + +// Client is the client to connect to cback. +type Client struct { + c *Config + client *httpclient.Client +} + +// New creates a new cback client. +func New(c *Config) *Client { + return &Client{ + c: c, + client: httpclient.New( + httpclient.Timeout(time.Duration(c.Timeout)), + ), + } +} + +func (c *Client) doHTTPRequest(ctx context.Context, username, reqType, endpoint string, body io.Reader) (io.ReadCloser, error) { + url := c.c.URL + endpoint + req, err := http.NewRequestWithContext(ctx, reqType, url, body) + if err != nil { + return nil, errors.Wrapf(err, "error creationg http %s request to %s", reqType, url) + } + + req.SetBasicAuth(username, c.c.Token) + + if body != nil { + req.Header.Add("Content-Type", "application/json") + } + + req.Header.Add("accept", `application/json`) + + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + switch resp.StatusCode { + case http.StatusNotFound: + return nil, errtypes.NotFound("cback: resource not found") + case http.StatusForbidden: + return nil, errtypes.PermissionDenied("cback: user has no permissions to get the backup") + case http.StatusBadRequest: + return nil, errtypes.BadRequest("") + default: + return nil, errtypes.InternalError("cback: internal server error: " + resp.Status) + } + } + + return resp.Body, nil +} + +// ListBackups gets all the backups of a user. +func (c *Client) ListBackups(ctx context.Context, username string) ([]*Backup, error) { + body, err := c.doHTTPRequest(ctx, username, http.MethodGet, "/backups/", nil) + if err != nil { + return nil, errors.Wrap(err, "cback: error listing backups for user "+username) + } + defer body.Close() + + var backups []*Backup + + if err := json.NewDecoder(body).Decode(&backups); err != nil { + return nil, errors.Wrap(err, "cback: error decoding response body for backups' list") + } + + return backups, nil +} + +// ListSnapshots gets all the snapshots of a backup. +func (c *Client) ListSnapshots(ctx context.Context, username string, backupID int) ([]*Snapshot, error) { + endpoint := fmt.Sprintf("/backups/%d/snapshots", backupID) + body, err := c.doHTTPRequest(ctx, username, http.MethodGet, endpoint, nil) + if err != nil { + return nil, errors.Wrapf(err, "cback: error listing snapshots for backup %d", backupID) + } + defer body.Close() + + var snapshots []*Snapshot + + if err := json.NewDecoder(body).Decode(&snapshots); err != nil { + return nil, errors.Wrap(err, "cbacK: error decoding response body for snapshots' list") + } + + return snapshots, nil +} + +// Stat gets the info of a resource stored in cback. +func (c *Client) Stat(ctx context.Context, username string, backupID int, snapshotID, path string, isTimestamp bool) (*Resource, error) { + endpoint := fmt.Sprintf("/backups/%d/snapshots/%s/%s", backupID, snapshotID, path) + if isTimestamp { + endpoint += "?timestamp=true" + } + body, err := c.doHTTPRequest(ctx, username, http.MethodOptions, endpoint, nil) + if err != nil { + return nil, errors.Wrapf(err, "cback: error statting %s in snapshot %s in backup %d", path, snapshotID, backupID) + } + defer body.Close() + + var res *Resource + + if err := json.NewDecoder(body).Decode(&res); err != nil { + return nil, errors.Wrap(err, "cback: error decoding response body") + } + + return res, nil +} + +// ListFolder gets the content of a folder stored in cback. +func (c *Client) ListFolder(ctx context.Context, username string, backupID int, snapshotID, path string, isTimestamp bool) ([]*Resource, error) { + endpoint := fmt.Sprintf("/backups/%d/snapshots/%s/%s?content=true", backupID, snapshotID, path) + if isTimestamp { + endpoint += "×tamp=true" + } + body, err := c.doHTTPRequest(ctx, username, http.MethodOptions, endpoint, nil) + if err != nil { + return nil, errors.Wrapf(err, "cback: error statting %s in snapshot %s in backup %d", path, snapshotID, backupID) + } + defer body.Close() + + var res []*Resource + + if err := json.NewDecoder(body).Decode(&res); err != nil { + return nil, errors.Wrap(err, "cback: error decoding response body") + } + + return res, nil +} + +// Download gets the content of a file stored in cback. +func (c *Client) Download(ctx context.Context, username string, backupID int, snapshotID, path string, isTimestamp bool) (io.ReadCloser, error) { + endpoint := fmt.Sprintf("/backups/%d/snapshots/%s/%s", backupID, snapshotID, path) + if isTimestamp { + endpoint += "?timestamp=true" + } + return c.doHTTPRequest(ctx, username, http.MethodGet, endpoint, nil) +} + +// ListRestores gets the list of restore jobs created by the user. +func (c *Client) ListRestores(ctx context.Context, username string) ([]*Restore, error) { + body, err := c.doHTTPRequest(ctx, username, http.MethodGet, "/restores/", nil) + if err != nil { + return nil, errors.Wrap(err, "cback: error getting restores") + } + defer body.Close() + + var res []*Restore + + if err := json.NewDecoder(body).Decode(&res); err != nil { + return nil, errors.Wrap(err, "cback: error decoding response body") + } + + return res, nil +} + +// GetRestore get the info of a restore job. +func (c *Client) GetRestore(ctx context.Context, username string, restoreID int) (*Restore, error) { + endpoint := fmt.Sprintf("/restores/%d", restoreID) + body, err := c.doHTTPRequest(ctx, username, http.MethodGet, endpoint, nil) + if err != nil { + return nil, errors.Wrap(err, "cback: error getting restores") + } + defer body.Close() + + var res *Restore + + if err := json.NewDecoder(body).Decode(&res); err != nil { + return nil, errors.Wrap(err, "cback: error decoding response body") + } + + return res, nil +} + +type newRestoreRequest struct { + BackupID int `json:"backup_id"` + Pattern string `json:"pattern,omitempty"` + Date string `json:"date,omitempty"` + Snapshot string `json:"snapshot"` +} + +// NewRestore creates a new restore job in cback. +func (c *Client) NewRestore(ctx context.Context, username string, backupID int, pattern, snapshotID string, timestamp bool) (*Restore, error) { + r := newRestoreRequest{ + BackupID: backupID, + Pattern: pattern, + } + if timestamp { + r.Date = snapshotID + } else { + r.Snapshot = snapshotID + } + + req, err := json.Marshal(r) + if err != nil { + return nil, errors.Wrap(err, "cback: error marshaling new restore request") + } + + body, err := c.doHTTPRequest(ctx, username, http.MethodPost, "/restores/", bytes.NewReader(req)) + if err != nil { + return nil, errors.Wrap(err, "cback: error getting restores") + } + defer body.Close() + + var res *Restore + + if err := json.NewDecoder(body).Decode(&res); err != nil { + return nil, errors.Wrap(err, "cback: error decoding response body") + } + + return res, nil +} diff --git a/cback/utils/structs.go b/cback/utils/structs.go new file mode 100644 index 0000000..e7ed610 --- /dev/null +++ b/cback/utils/structs.go @@ -0,0 +1,101 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package utils + +import ( + "encoding/json" + "strings" + "time" +) + +// Group is the group in cback. +type Group struct { + ID int `json:"id"` + Name string `json:"name"` +} + +// Backup represents the metadata information of a backuo job. +type Backup struct { + ID int `json:"id"` + Group Group `json:"group"` + Repository string `json:"repository"` + Username string `json:"username"` + Name string `json:"name"` + Source string `json:"source"` +} + +// Snapshot represents the metadata information of a snapshot in a backup. +type Snapshot struct { + ID string `json:"id"` + Time CBackTime `json:"time"` + Paths []string `json:"paths"` +} + +// Resource represents the metadata information of a file stored in cback. +type Resource struct { + Name string `json:"name"` + Type string `json:"type"` + Mode uint64 `json:"mode"` + MTime float64 `json:"mtime"` + ATime float64 `json:"atime"` + CTime float64 `json:"ctime"` + Inode uint64 `json:"inode"` + Size uint64 `json:"size"` +} + +// Restore represents the metadata information of a restore job. +type Restore struct { + ID int `json:"id"` + BackupID int `json:"backup_id"` + SnapshotID string `json:"snapshot"` + Destionation string `json:"destination"` + Pattern string `json:"pattern"` + Status int `json:"status"` + Created CBackTime `json:"created"` +} + +type CBackTime struct{ time.Time } + +func (c CBackTime) MarshalJSON() ([]byte, error) { + return json.Marshal(c.Time) +} + +func (c *CBackTime) UnmarshalJSON(b []byte) error { + s := strings.Trim(string(b), "\"") + t, err := time.Parse("2006-01-02T15:04:05", s) + if err != nil { + // fall back to the default unmarshaler for date0time + if err := json.Unmarshal(b, &t); err != nil { + return err + } + + } + *c = CBackTime{t} + return nil +} + +// IsDir returns true if the resoure is a directory. +func (r *Resource) IsDir() bool { + return r.Type == "dir" +} + +// IsFile returns true if the resoure is a file. +func (r *Resource) IsFile() bool { + return r.Type == "file" +}