diff --git a/Manual.md b/Manual.md index cd19a84b..be49fd50 100644 --- a/Manual.md +++ b/Manual.md @@ -272,6 +272,10 @@ NAME: USAGE: clickhouse-backup restore_remote [--schema] [--data] [-t, --tables=.] [-m, --restore-database-mapping=:[,<...>]] [--tm, --restore-table-mapping=:[,<...>]] [--partitions=] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] [--named-collections] [--resumable] [--skip-empty-tables] +DESCRIPTION: + A remote backup with ClickHouse Cloud / native BACKUP layout (.backup without metadata.json, `cloud` in `list remote`) is restored via restore_cloud, only with remote_storage s3 or azblob and without mapping, --schema, --data, --rbac, --configs, --named-collections, --resume and --streaming options + https://github.com/Altinity/clickhouse-backup/issues/1574 + OPTIONS: --table string, --tables string, -t string Download and restore objects which matched with table name patterns, separated by comma, allow ? and * as wildcard --restore-database-mapping string, -m string [ --restore-database-mapping string, -m string ] Define the rule to restore data. For the database not defined in this struct, the program will not deal with it. @@ -324,13 +328,14 @@ DESCRIPTION: Credentials and defaults are taken from the s3 config section (also works for GCS via s3->endpoint=https://storage.googleapis.com with HMAC keys), or from the azblob config section when --container / --azblob-restore-url is passed or general->remote_storage is azblob When s3->assume_role_arn is set, the manifest is read and RESTORE ... FROM S3(..., extra_credentials(role_arn='...')) is executed with the assumed AWS IAM role, the static keys only sign the STS AssumeRole call (requires ClickHouse 25.8+); without any static keys the STS AssumeRole call is signed by the ambient AWS identity instead: shared credentials file / IRSA / EC2-ECS instance profile for the manifest reads, and the ClickHouse server's own environment for the RESTORE statement + For an incremental backup the base backups chain is resolved from of each .backup inside the same bucket/container, RESTORE gets SETTINGS base_backup with the configured credentials https://github.com/Altinity/clickhouse-backup/issues/1508 OPTIONS: --bucket string S3 bucket with the ClickHouse Cloud backup, overrides s3->bucket from config --region string AWS region of the bucket, overrides s3->region from config --endpoint string Custom S3 endpoint (MinIO, etc.), overrides s3->endpoint from config - --base-prefix string S3 key prefix of the base backup, for incremental backups with use_base files + --base-prefix string Key prefix of the base backup for incremental backups, by default the chain of base backups is resolved from of the .backup manifest, use it when the backups were copied from another bucket/prefix --s3-restore-url string URL passed to RESTORE ... FROM S3('...'), default https://s3..amazonaws.com// --container string AzureBlobStorage container with the ClickHouse Cloud backup, overrides azblob->container from config and switches the source to AzureBlobStorage --azblob-restore-url string Blob endpoint passed to RESTORE ... FROM AzureBlobStorage(...), e.g. http://azurite:10000/devstoreaccount1, when it differs from azblob config section, switches the source to AzureBlobStorage diff --git a/ReadMe.md b/ReadMe.md index 07f1ac06..a9146d47 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -786,6 +786,8 @@ Note: this operation is asynchronous, so the API will return once the operation Download and restore data from remote backup: `curl -s localhost:7171/backup/restore_remote/ -X POST | jq .` +A backup with ClickHouse Cloud / native `BACKUP ... TO S3/AzureBlobStorage` layout (`.backup` without `metadata.json`, shown as `cloud` in `/backup/list/remote`) is restored the same way as `POST /backup/restore_cloud`, see the `restore_remote` CLI command description. + - Optional boolean query argument `dry-run` or `dry_run` works the same as the `--dry-run` CLI argument (show the number of tables and the data size which would be downloaded and restored, without downloading and restoring). A dry-run request executes synchronously, is allowed even when another operation is in progress regardless of `api.allow_parallel`, and returns the report as JSON in the response body instead of the asynchronous acknowledgement. - Optional string query argument `table` works the same as the `--table value` CLI argument. - Optional string query argument `partitions` works the same as the `--partitions value` CLI argument. @@ -1152,6 +1154,10 @@ NAME: USAGE: clickhouse-backup restore_remote [--schema] [--data] [-t, --tables=.
] [-m, --restore-database-mapping=:[,<...>]] [--tm, --restore-table-mapping=:[,<...>]] [--partitions=] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] [--named-collections] [--resumable] [--skip-empty-tables] +DESCRIPTION: + A remote backup with ClickHouse Cloud / native BACKUP layout (.backup without metadata.json, `cloud` in `list remote`) is restored via restore_cloud, only with remote_storage s3 or azblob and without mapping, --schema, --data, --rbac, --configs, --named-collections, --resume and --streaming options + https://github.com/Altinity/clickhouse-backup/issues/1574 + OPTIONS: --table string, --tables string, -t string Download and restore objects which matched with table name patterns, separated by comma, allow ? and * as wildcard --restore-database-mapping string, -m string [ --restore-database-mapping string, -m string ] Define the rule to restore data. For the database not defined in this struct, the program will not deal with it. @@ -1204,13 +1210,14 @@ DESCRIPTION: Credentials and defaults are taken from the s3 config section (also works for GCS via s3->endpoint=https://storage.googleapis.com with HMAC keys), or from the azblob config section when --container / --azblob-restore-url is passed or general->remote_storage is azblob When s3->assume_role_arn is set, the manifest is read and RESTORE ... FROM S3(..., extra_credentials(role_arn='...')) is executed with the assumed AWS IAM role, the static keys only sign the STS AssumeRole call (requires ClickHouse 25.8+); without any static keys the STS AssumeRole call is signed by the ambient AWS identity instead: shared credentials file / IRSA / EC2-ECS instance profile for the manifest reads, and the ClickHouse server's own environment for the RESTORE statement + For an incremental backup the base backups chain is resolved from of each .backup inside the same bucket/container, RESTORE gets SETTINGS base_backup with the configured credentials https://github.com/Altinity/clickhouse-backup/issues/1508 OPTIONS: --bucket string S3 bucket with the ClickHouse Cloud backup, overrides s3->bucket from config --region string AWS region of the bucket, overrides s3->region from config --endpoint string Custom S3 endpoint (MinIO, etc.), overrides s3->endpoint from config - --base-prefix string S3 key prefix of the base backup, for incremental backups with use_base files + --base-prefix string Key prefix of the base backup for incremental backups, by default the chain of base backups is resolved from of the .backup manifest, use it when the backups were copied from another bucket/prefix --s3-restore-url string URL passed to RESTORE ... FROM S3('...'), default https://s3..amazonaws.com// --container string AzureBlobStorage container with the ClickHouse Cloud backup, overrides azblob->container from config and switches the source to AzureBlobStorage --azblob-restore-url string Blob endpoint passed to RESTORE ... FROM AzureBlobStorage(...), e.g. http://azurite:10000/devstoreaccount1, when it differs from azblob config section, switches the source to AzureBlobStorage diff --git a/cmd/clickhouse-backup/main.go b/cmd/clickhouse-backup/main.go index 90f4ee71..2a902354 100644 --- a/cmd/clickhouse-backup/main.go +++ b/cmd/clickhouse-backup/main.go @@ -746,6 +746,8 @@ func newRootCommand() *cli.Command { Name: "restore_remote", Usage: "Download and restore", UsageText: "clickhouse-backup restore_remote [--schema] [--data] [-t, --tables=.
] [-m, --restore-database-mapping=:[,<...>]] [--tm, --restore-table-mapping=:[,<...>]] [--partitions=] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] [--named-collections] [--resumable] [--skip-empty-tables] ", + Description: "A remote backup with ClickHouse Cloud / native BACKUP layout (.backup without metadata.json, `cloud` in `list remote`) is restored via restore_cloud, only with remote_storage s3 or azblob and without mapping, --schema, --data, --rbac, --configs, --named-collections, --resume and --streaming options\n" + + " https://github.com/Altinity/clickhouse-backup/issues/1574", Action: func(ctx context.Context, c *cli.Command) error { b := backup.NewBackuper(config.GetConfigFromCli(c)) b.DryRun = c.Bool("dry-run") @@ -909,6 +911,7 @@ func newRootCommand() *cli.Command { " Credentials and defaults are taken from the s3 config section (also works for GCS via s3->endpoint=https://storage.googleapis.com with HMAC keys), or from the azblob config section when --container / --azblob-restore-url is passed or general->remote_storage is azblob\n" + " When s3->assume_role_arn is set, the manifest is read and RESTORE ... FROM S3(..., extra_credentials(role_arn='...')) is executed with the assumed AWS IAM role, the static keys only sign the STS AssumeRole call (requires ClickHouse 25.8+);\n" + " without any static keys the STS AssumeRole call is signed by the ambient AWS identity instead: shared credentials file / IRSA / EC2-ECS instance profile for the manifest reads, and the ClickHouse server's own environment for the RESTORE statement\n" + + " For an incremental backup the base backups chain is resolved from of each .backup inside the same bucket/container, RESTORE gets SETTINGS base_backup with the configured credentials\n" + " https://github.com/Altinity/clickhouse-backup/issues/1508", Action: func(ctx context.Context, c *cli.Command) error { if c.Args().First() == "" { @@ -956,7 +959,7 @@ func newRootCommand() *cli.Command { &cli.StringFlag{ Name: "base-prefix", Hidden: false, - Usage: "S3 key prefix of the base backup, for incremental backups with use_base files", + Usage: "Key prefix of the base backup for incremental backups, by default the chain of base backups is resolved from of the .backup manifest, use it when the backups were copied from another bucket/prefix", }, &cli.StringFlag{ Name: "s3-restore-url", diff --git a/pkg/backup/download.go b/pkg/backup/download.go index e699636f..f15030a2 100644 --- a/pkg/backup/download.go +++ b/pkg/backup/download.go @@ -246,6 +246,9 @@ func (b *Backuper) downloadRemoteBackupInfo(ctx context.Context, backupName, tab if !found { return storage.Backup{}, nil, nil, errors.Errorf("'%s' is not found on remote storage", backupName) } + if remoteBackup.DataFormat == storage.CloudBackupDataFormat { + return storage.Backup{}, nil, nil, errors.Errorf("'%s' has ClickHouse Cloud / native BACKUP layout (.backup without metadata.json), it can't be downloaded, use `restore_remote %s` or `restore_cloud`", backupName, backupName) + } // Download file manifest for Walk-free restore (falls back gracefully if not present) backupManifest := b.dst.DownloadManifest(ctx, backupName) diff --git a/pkg/backup/restore_cloud.go b/pkg/backup/restore_cloud.go index 11a7a8a9..56ad3e27 100644 --- a/pkg/backup/restore_cloud.go +++ b/pkg/backup/restore_cloud.go @@ -78,6 +78,9 @@ type cloudManifestFile struct { type cloudBackupManifest struct { XMLName xml.Name `xml:"config"` + UUID string `xml:"uuid"` + BaseBackup string `xml:"base_backup"` // contains credentials, see storage.CloudBackupHeader + BaseBackupUUID string `xml:"base_backup_uuid"` DataFileNameGenerator string `xml:"data_file_name_generator"` DataFileNamePrefixLength int `xml:"data_file_name_prefix_length"` Files []cloudManifestFile `xml:"contents>file"` @@ -218,7 +221,7 @@ type RestoreCloudOptions struct { Region string Endpoint string Container string // azblob container, switches the source to AzureBlobStorage - BasePrefix string // prefix of the base backup for incremental backups with use_base + BasePrefix string // prefix of the base backup for incremental backups with use_base, overrides the first `` of the .backup manifest S3RestoreURL string // URL passed to RESTORE ... FROM S3('...'), default https://s3..amazonaws.com// AzblobRestoreURL string // blob endpoint passed to RESTORE ... FROM AzureBlobStorage(...), e.g. http://azurite:10000/devstoreaccount1, switches the source to AzureBlobStorage TablePattern string @@ -237,11 +240,14 @@ type cloudSource struct { reader interface { GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error) } - isNotFound func(error) bool - restoreLocation string // FROM clause of the RESTORE statement, contains credentials - secrets []string // credentials to redact from logs and errors - label string // human-readable source for logs, e.g. s3://bucket - close func(ctx context.Context) + isNotFound func(error) bool + // restoreLocation returns the FROM / base_backup clause of the RESTORE statement for a key prefix, contains credentials + restoreLocation func(prefix string) string + secrets []string // credentials to redact from logs and errors + label string // human-readable source for logs, e.g. s3://bucket + isAzblob bool + bucketOrContainer string + close func(ctx context.Context) } // connectCloudSourceS3 - S3 and any S3-compatible endpoint (GCS with HMAC keys, MinIO) @@ -279,25 +285,31 @@ func (b *Backuper) connectCloudSourceS3(ctx context.Context, opts *RestoreCloudO if err := s3Client.Connect(ctx); err != nil { return nil, errors.Wrap(err, "can't connect to s3") } - restoreURL := strings.TrimSuffix(opts.S3RestoreURL, "/") - if restoreURL == "" { + // --s3-restore-url points to the backup itself, a base backup prefix replaces its trailing prefix + customRestoreURL := strings.TrimSuffix(opts.S3RestoreURL, "/") + restoreURL := func(p string) string { + if customRestoreURL != "" && p == prefix { + return customRestoreURL + } + if customRestoreURL != "" && strings.HasSuffix(customRestoreURL, "/"+prefix) { + return strings.TrimSuffix(customRestoreURL, prefix) + p + } if s3cfg.Endpoint != "" { - restoreURL = fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(s3cfg.Endpoint, "/"), s3cfg.Bucket, prefix) - } else { - restoreURL = fmt.Sprintf("https://s3.%s.amazonaws.com/%s/%s", s3cfg.Region, s3cfg.Bucket, prefix) + return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(s3cfg.Endpoint, "/"), s3cfg.Bucket, p) } + return fmt.Sprintf("https://s3.%s.amazonaws.com/%s/%s", s3cfg.Region, s3cfg.Bucket, p) } // s3->assume_role_arn: the RESTORE reads the bucket with the assumed role's permissions, // the static keys (when present) only sign the STS AssumeRole call, without them the // ClickHouse server's ambient identity signs it (same semantics as the manifest reads above) - var restoreLocation string - switch { - case s3cfg.AssumeRoleARN != "" && accessKey == "": - restoreLocation = fmt.Sprintf("S3('%s', extra_credentials(role_arn = '%s'))", restoreURL, s3cfg.AssumeRoleARN) - case s3cfg.AssumeRoleARN != "": - restoreLocation = fmt.Sprintf("S3('%s', '%s', '%s', extra_credentials(role_arn = '%s'))", restoreURL, accessKey, secretKey, s3cfg.AssumeRoleARN) - default: - restoreLocation = fmt.Sprintf("S3('%s', '%s', '%s')", restoreURL, accessKey, secretKey) + restoreLocation := func(p string) string { + switch { + case s3cfg.AssumeRoleARN != "" && accessKey == "": + return fmt.Sprintf("S3('%s', extra_credentials(role_arn = '%s'))", restoreURL(p), s3cfg.AssumeRoleARN) + case s3cfg.AssumeRoleARN != "": + return fmt.Sprintf("S3('%s', '%s', '%s', extra_credentials(role_arn = '%s'))", restoreURL(p), accessKey, secretKey, s3cfg.AssumeRoleARN) + } + return fmt.Sprintf("S3('%s', '%s', '%s')", restoreURL(p), accessKey, secretKey) } return &cloudSource{ reader: s3Client, @@ -305,9 +317,10 @@ func (b *Backuper) connectCloudSourceS3(ctx context.Context, opts *RestoreCloudO var noSuchKey *s3types.NoSuchKey return stderrors.As(err, &noSuchKey) }, - restoreLocation: restoreLocation, - secrets: []string{accessKey, secretKey}, - label: fmt.Sprintf("s3://%s", s3cfg.Bucket), + restoreLocation: restoreLocation, + secrets: []string{accessKey, secretKey}, + label: fmt.Sprintf("s3://%s", s3cfg.Bucket), + bucketOrContainer: s3cfg.Bucket, close: func(ctx context.Context) { if closeErr := s3Client.Close(ctx); closeErr != nil { log.Warn().Msgf("can't close S3 connection: %v", closeErr) @@ -355,9 +368,13 @@ func (b *Backuper) connectCloudSourceAzblob(ctx context.Context, opts *RestoreCl isNotFound: func(err error) bool { return bloberror.HasCode(err, bloberror.BlobNotFound) }, - restoreLocation: fmt.Sprintf("AzureBlobStorage('%s', '%s', '%s/')", connectionString, azcfg.Container, prefix), - secrets: []string{azcfg.AccountKey}, - label: fmt.Sprintf("azblob://%s", azcfg.Container), + restoreLocation: func(p string) string { + return fmt.Sprintf("AzureBlobStorage('%s', '%s', '%s/')", connectionString, azcfg.Container, p) + }, + secrets: []string{azcfg.AccountKey}, + label: fmt.Sprintf("azblob://%s", azcfg.Container), + isAzblob: true, + bucketOrContainer: azcfg.Container, close: func(ctx context.Context) { if closeErr := azClient.Close(ctx); closeErr != nil { log.Warn().Msgf("can't close AzureBlob connection: %v", closeErr) @@ -424,6 +441,16 @@ func (b *Backuper) RestoreCloud(opts RestoreCloudOptions, commandId int) error { return err } log.Info().Msgf("manifest generator=%s prefix_length=%d files=%d", manifest.DataFileNameGenerator, manifest.DataFileNamePrefixLength, len(manifest.Files)) + basePrefixes, err := b.resolveCloudBaseChain(ctx, source, manifest, basePrefix) + if err != nil { + return errors.Errorf("%s", restoreCloudRedact(err.Error(), source.secrets...)) + } + // RESTORE takes the base backup location with credentials from ``, it is replaced with the + // resolved location readable with the configured credentials, deeper bases are still read from their own .backup + restoreSettings := "allow_different_database_def=1, allow_different_table_def=1" + if len(basePrefixes) > 0 { + restoreSettings = "base_backup = " + source.restoreLocation(basePrefixes[0]) + ", " + restoreSettings + } // group manifest entries: database DDL, table DDL and logical data bytes per table, // `BACKUP ... ON CLUSTER` prefixes every name with shards//replicas// @@ -526,7 +553,7 @@ func (b *Backuper) RestoreCloud(opts RestoreCloudOptions, commandId int) error { log.Info().Msgf("skip empty %s", f.Name) continue } - ddl, fetchErr := b.fetchCloudBlob(ctx, source, manifest, f, prefix, basePrefix) + ddl, fetchErr := b.fetchCloudBlob(ctx, source, manifest, f, prefix, basePrefixes) if fetchErr != nil { if handledErr := handleError(fmt.Sprintf("fetch %s.%s", database, table), fetchErr); handledErr != nil { return handledErr @@ -549,7 +576,7 @@ func (b *Backuper) RestoreCloud(opts RestoreCloudOptions, commandId int) error { } log.Info().Msgf("######## database %s ########", database) if f, exists := dbDDLFiles[database]; exists && f.Size > 0 { - ddl, fetchErr := b.fetchCloudBlob(ctx, source, manifest, f, prefix, basePrefix) + ddl, fetchErr := b.fetchCloudBlob(ctx, source, manifest, f, prefix, basePrefixes) if fetchErr == nil { fetchErr = b.restoreCloudExec(ctx, injectCloudOnCluster(rewriteCloudSchema(ddl, "database", "", ""), onClusterSQL), fmt.Sprintf("database %s", database), source.secrets) } @@ -589,8 +616,8 @@ func (b *Backuper) RestoreCloud(opts RestoreCloudOptions, commandId int) error { } if restoreErr == nil { restoreSQL := fmt.Sprintf( - "RESTORE TABLE %s.%s%s%s FROM %s SETTINGS allow_different_database_def=1, allow_different_table_def=1", - cloudQuoteIdent(database), cloudQuoteIdent(t.table), t.partitionsSQL, onClusterSQL, source.restoreLocation, + "RESTORE TABLE %s.%s%s%s FROM %s SETTINGS %s", + cloudQuoteIdent(database), cloudQuoteIdent(t.table), t.partitionsSQL, onClusterSQL, source.restoreLocation(prefix), restoreSettings, ) restoreErr = b.restoreCloudExec(tablesCtx, restoreSQL, fmt.Sprintf("RESTORE TABLE %s", label), source.secrets) } @@ -693,16 +720,12 @@ func (b *Backuper) cloudRestorePartitionsSQL(ctx context.Context, database, tabl } // fetchCloudBlob downloads a manifest entry, resolving its blob key and falling back between the -// backup prefix and the base backup prefix (incremental backups store unchanged files in the base) -func (b *Backuper) fetchCloudBlob(ctx context.Context, source *cloudSource, manifest *cloudBackupManifest, f *cloudManifestFile, prefix, basePrefix string) (string, error) { +// backup prefix and the base backups chain prefixes (incremental backups store unchanged files in the base) +func (b *Backuper) fetchCloudBlob(ctx context.Context, source *cloudSource, manifest *cloudBackupManifest, f *cloudManifestFile, prefix string, basePrefixes []string) (string, error) { blob := manifest.blobKey(f) - prefixes := []string{prefix} - if basePrefix != "" { - if f.UseBase { - prefixes = []string{basePrefix, prefix} - } else { - prefixes = []string{prefix, basePrefix} - } + prefixes := append([]string{prefix}, basePrefixes...) + if f.UseBase && len(basePrefixes) > 0 { + prefixes = append(append([]string{}, basePrefixes...), prefix) } var lastErr error for _, p := range prefixes { @@ -739,3 +762,134 @@ func (b *Backuper) restoreCloudExec(ctx context.Context, sql, what string, secre } return nil } + +// remoteCloudBackupPrefix returns the key prefix of backupName inside the bucket/container when the remote backup has +// ClickHouse Cloud / native BACKUP layout (`.backup` without metadata.json), empty string for a regular backup, +// https://github.com/Altinity/clickhouse-backup/issues/1574 +func (b *Backuper) remoteCloudBackupPrefix(ctx context.Context, backupName string) (string, error) { + if b.cfg.General.RemoteStorage == "none" || b.cfg.General.RemoteStorage == "custom" { + return "", nil + } + if !b.ch.IsOpen { + if err := b.ch.Connect(); err != nil { + return "", errors.Wrap(err, "can't connect to clickhouse") + } + defer b.ch.Close() + } + // NewBackupDestination resolves macros in s3->path / azblob->path of b.cfg in place + bd, err := storage.NewBackupDestination(ctx, b.cfg, b.ch, "") + if err != nil { + return "", errors.Wrap(err, "NewBackupDestination") + } + if err = bd.Connect(ctx); err != nil { + return "", errors.Wrap(err, "bd.Connect") + } + defer func() { + if closeErr := bd.Close(ctx); closeErr != nil { + log.Warn().Msgf("can't close BackupDestination error: %v", closeErr) + } + }() + backupList, err := bd.BackupList(ctx, true, backupName) + if err != nil { + return "", errors.Wrap(err, "bd.BackupList") + } + if len(backupList) != 1 || backupList[0].DataFormat != storage.CloudBackupDataFormat { + return "", nil + } + switch b.cfg.General.RemoteStorage { + case "s3": + return path.Join(b.cfg.S3.Path, backupName), nil + case "azblob": + return path.Join(b.cfg.AzureBlob.Path, backupName), nil + } + return "", errors.Errorf("'%s' has ClickHouse Cloud / native BACKUP layout, restore_remote supports it only with `remote_storage: s3` or `remote_storage: azblob`, got `%s`, use restore_cloud with s3 or azblob config section", backupName, b.cfg.General.RemoteStorage) +} + +// cloudUnsupportedRestoreRemoteOptions - restore_remote options which restore_cloud can't apply, restore fails +// instead of silently ignoring them +func cloudUnsupportedRestoreRemoteOptions(databaseMapping, tableMapping, skipProjections []string, schemaOnly, dataOnly, ignoreDependencies, restoreRBAC, rbacOnly, restoreConfigs, configsOnly, restoreNamedCollections, namedCollectionsOnly, resume, schemaAsAttach, replicatedCopyToDetached, hardlinkExistsFiles, streaming bool) []string { + unsupported := make([]string, 0) + for _, option := range []struct { + name string + set bool + }{ + {"--restore-database-mapping", len(databaseMapping) > 0}, + {"--restore-table-mapping", len(tableMapping) > 0}, + {"--skip-projections", len(skipProjections) > 0}, + {"--schema", schemaOnly}, + {"--data", dataOnly}, + {"--ignore-dependencies", ignoreDependencies}, + {"--rbac", restoreRBAC}, + {"--rbac-only", rbacOnly}, + {"--configs", restoreConfigs}, + {"--configs-only", configsOnly}, + {"--named-collections", restoreNamedCollections}, + {"--named-collections-only", namedCollectionsOnly}, + {"--resume", resume}, + {"--restore-schema-as-attach", schemaAsAttach}, + {"--replicated-copy-to-detached", replicatedCopyToDetached}, + {"--hardlink-exists-files", hardlinkExistsFiles}, + {"--streaming", streaming}, + } { + if option.set { + unsupported = append(unsupported, option.name) + } + } + return unsupported +} + +// resolveCloudBaseChain returns the key prefixes of the base backups chain of an incremental backup, the nearest base +// first, each base is resolved from `` of the previous `.backup` and verified by ``, +// basePrefix (--base-prefix) overrides the first base, e.g. when the backups were copied from another bucket +func (b *Backuper) resolveCloudBaseChain(ctx context.Context, source *cloudSource, manifest *cloudBackupManifest, basePrefix string) ([]string, error) { + // a manifest without keeps the behavior of --base-prefix before base auto-detection + if manifest.BaseBackup == "" { + if basePrefix != "" { + return []string{basePrefix}, nil + } + return nil, nil + } + chain := make([]string, 0) + seen := map[string]struct{}{} + baseBackup, baseUUID := manifest.BaseBackup, manifest.BaseBackupUUID + for baseBackup != "" { + loc, err := storage.ParseCloudBackupLocation(baseBackup) + if err != nil { + return nil, err + } + source.secrets = append(source.secrets, loc.Secrets...) + p := basePrefix + if len(chain) > 0 || p == "" { + key, found := loc.KeyIn(source.bucketOrContainer) + if !found || loc.IsAzureBlob() != source.isAzblob { + if len(chain) == 0 { + return nil, errors.Errorf("base backup %s is not stored in %s, pass --base-prefix when the backups were copied there", loc.String(), source.label) + } + log.Warn().Msgf("base backup %s of %s is not stored in %s, RESTORE reads it with the location and credentials from %s/.backup", loc.String(), chain[len(chain)-1], source.label, chain[len(chain)-1]) + break + } + p = key + } + if _, cycle := seen[p]; cycle { + return nil, errors.Errorf("base backups chain has a cycle at %s", p) + } + seen[p] = struct{}{} + manifestKey := path.Join(p, ".backup") + reader, err := source.reader.GetFileReaderAbsolute(ctx, manifestKey) + if err != nil { + return nil, errors.Wrapf(err, "can't read base backup %s/%s", source.label, manifestKey) + } + header, err := storage.ReadCloudBackupHeader(reader) + _ = reader.Close() + if err != nil { + return nil, errors.Wrapf(err, "can't parse base backup %s/%s", source.label, manifestKey) + } + if baseUUID != "" && header.UUID != baseUUID { + return nil, errors.Errorf("base backup %s/%s has uuid %s, expected %s", source.label, p, header.UUID, baseUUID) + } + log.Info().Msgf("base backup %s/%s uuid=%s", source.label, p, header.UUID) + chain = append(chain, p) + baseBackup, baseUUID = header.BaseBackup, header.BaseBackupUUID + } + return chain, nil +} diff --git a/pkg/backup/restore_cloud_test.go b/pkg/backup/restore_cloud_test.go index 28fe7a35..2ac1e2df 100644 --- a/pkg/backup/restore_cloud_test.go +++ b/pkg/backup/restore_cloud_test.go @@ -2,6 +2,8 @@ package backup import ( "context" + "errors" + "io" "strings" "testing" @@ -248,3 +250,118 @@ func TestCloudLogicalNamesShardPrefix(t *testing.T) { assert.Equal(t, "default", db) assert.Equal(t, "", table) } + +func TestCloudUnsupportedRestoreRemoteOptions(t *testing.T) { + assert.Empty(t, cloudUnsupportedRestoreRemoteOptions(nil, nil, nil, false, false, false, false, false, false, false, false, false, false, false, false, false, false)) + assert.Equal(t, + []string{"--restore-table-mapping", "--schema", "--streaming"}, + cloudUnsupportedRestoreRemoteOptions(nil, []string{"t1:t2"}, nil, true, false, false, false, false, false, false, false, false, false, false, false, false, true), + ) +} + +type memCloudReader map[string]string + +var errMemCloudNotFound = errors.New("not found") + +func (m memCloudReader) GetFileReaderAbsolute(_ context.Context, key string) (io.ReadCloser, error) { + content, exists := m[key] + if !exists { + return nil, errMemCloudNotFound + } + return io.NopCloser(strings.NewReader(content)), nil +} + +func testCloudHeaderXML(uuid, baseURL, baseUUID string) string { + base := "" + if baseURL != "" { + base = "S3('" + baseURL + "', 'BASEKEY', 'BASESECRET')" + baseUUID + "" + } + return "1" + uuid + "" + base + "" +} + +func TestResolveCloudBaseChain(t *testing.T) { + b := &Backuper{} + ctx := context.Background() + newSource := func(files memCloudReader) *cloudSource { + return &cloudSource{ + reader: files, + isNotFound: func(err error) bool { return errors.Is(err, errMemCloudNotFound) }, + label: "s3://bucket", + bucketOrContainer: "bucket", + } + } + files := memCloudReader{ + "backups/incr2/.backup": testCloudHeaderXML("u3", "https://s3.us-east-1.amazonaws.com/bucket/backups/incr1", "u2"), + "backups/incr1/.backup": testCloudHeaderXML("u2", "https://bucket.s3.us-east-1.amazonaws.com/backups/full", "u1"), + "backups/full/.backup": testCloudHeaderXML("u1", "", ""), + "copy/incr1/.backup": testCloudHeaderXML("u2", "https://s3.us-east-1.amazonaws.com/bucket/backups/full", "u1"), + } + parse := func(key string) *cloudBackupManifest { + m, err := parseCloudManifest(strings.NewReader(files[key])) + require.NoError(t, err) + return m + } + + source := newSource(files) + chain, err := b.resolveCloudBaseChain(ctx, source, parse("backups/incr2/.backup"), "") + require.NoError(t, err) + assert.Equal(t, []string{"backups/incr1", "backups/full"}, chain) + // credentials of are redacted from errors and logs + assert.Contains(t, source.secrets, "BASESECRET") + + // --base-prefix overrides the first base only + chain, err = b.resolveCloudBaseChain(ctx, newSource(files), parse("backups/incr2/.backup"), "copy/incr1") + require.NoError(t, err) + assert.Equal(t, []string{"copy/incr1", "backups/full"}, chain) + + // full backup, legacy --base-prefix without + chain, err = b.resolveCloudBaseChain(ctx, newSource(files), parse("backups/full/.backup"), "") + require.NoError(t, err) + assert.Empty(t, chain) + chain, err = b.resolveCloudBaseChain(ctx, newSource(files), parse("backups/full/.backup"), "legacy/base") + require.NoError(t, err) + assert.Equal(t, []string{"legacy/base"}, chain) + + // uuid mismatch + mismatch := parse("backups/incr2/.backup") + mismatch.BaseBackupUUID = "other" + _, err = b.resolveCloudBaseChain(ctx, newSource(files), mismatch, "") + assert.ErrorContains(t, err, "expected other") + + // base in another bucket + other, err := parseCloudManifest(strings.NewReader(testCloudHeaderXML("u9", "https://s3.us-east-1.amazonaws.com/other/full", "u1"))) + require.NoError(t, err) + _, err = b.resolveCloudBaseChain(ctx, newSource(files), other, "") + assert.ErrorContains(t, err, "pass --base-prefix") + assert.NotContains(t, err.Error(), "BASESECRET") + + // missing base + missing, err := parseCloudManifest(strings.NewReader(testCloudHeaderXML("u9", "https://s3.us-east-1.amazonaws.com/bucket/missing", "u1"))) + require.NoError(t, err) + _, err = b.resolveCloudBaseChain(ctx, newSource(files), missing, "") + assert.ErrorContains(t, err, "can't read base backup s3://bucket/missing/.backup") +} + +func TestFetchCloudBlobBaseChain(t *testing.T) { + b := &Backuper{} + manifest := &cloudBackupManifest{DataFileNameGenerator: "FirstFileName"} + source := &cloudSource{ + reader: memCloudReader{ + "incr/metadata/default/new.sql": "CREATE TABLE default.new", + "full/metadata/default/old.sql": "CREATE TABLE default.old", + "incr/metadata/default/stale.sql": "stale", + "base/metadata/default/stale.sql": "CREATE TABLE default.stale", + }, + isNotFound: func(err error) bool { return errors.Is(err, errMemCloudNotFound) }, + } + ddl, err := b.fetchCloudBlob(context.Background(), source, manifest, &cloudManifestFile{Name: "metadata/default/new.sql"}, "incr", []string{"base", "full"}) + require.NoError(t, err) + assert.Equal(t, "CREATE TABLE default.new", ddl) + // use_base looks into the chain first, the deepest base last + ddl, err = b.fetchCloudBlob(context.Background(), source, manifest, &cloudManifestFile{Name: "metadata/default/old.sql", UseBase: true}, "incr", []string{"base", "full"}) + require.NoError(t, err) + assert.Equal(t, "CREATE TABLE default.old", ddl) + ddl, err = b.fetchCloudBlob(context.Background(), source, manifest, &cloudManifestFile{Name: "metadata/default/stale.sql", UseBase: true}, "incr", []string{"base", "full"}) + require.NoError(t, err) + assert.Equal(t, "CREATE TABLE default.stale", ddl) +} diff --git a/pkg/backup/restore_remote.go b/pkg/backup/restore_remote.go index 02ced8f9..e6be6b3c 100644 --- a/pkg/backup/restore_remote.go +++ b/pkg/backup/restore_remote.go @@ -25,6 +25,32 @@ import ( ) func (b *Backuper) RestoreFromRemote(backupName, tablePattern string, databaseMapping, tableMapping, partitions, skipProjections []string, schemaOnly, dataOnly, dropExists, ignoreDependencies, restoreRBAC, rbacOnly, restoreConfigs, configsOnly, restoreNamedCollections, namedCollectionsOnly, resume, schemaAsAttach, replicatedCopyToDetached, skipEmptyTables, hardlinkExistsFiles, streaming bool, version string, commandId int) error { + // ClickHouse Cloud / native BACKUP layout is restored via restore_cloud, https://github.com/Altinity/clickhouse-backup/issues/1574 + if !b.EmbeddedOnClusterWorker { + ctx, cancel, ctxErr := status.Current.GetContextWithCancel(commandId) + if ctxErr != nil { + return pkgerrors.Wrap(ctxErr, "RestoreFromRemote GetContextWithCancel") + } + cloudPrefix, detectErr := b.remoteCloudBackupPrefix(ctx, backupName) + cancel() + if detectErr != nil { + return pkgerrors.Wrap(detectErr, "RestoreFromRemote remoteCloudBackupPrefix") + } + if cloudPrefix != "" { + if unsupported := cloudUnsupportedRestoreRemoteOptions(databaseMapping, tableMapping, skipProjections, schemaOnly, dataOnly, ignoreDependencies, restoreRBAC, rbacOnly, restoreConfigs, configsOnly, restoreNamedCollections, namedCollectionsOnly, resume, schemaAsAttach, replicatedCopyToDetached, hardlinkExistsFiles, streaming); len(unsupported) > 0 { + return pkgerrors.Errorf("'%s' has ClickHouse Cloud / native BACKUP layout, restore_remote switches to restore_cloud which doesn't support %s", backupName, strings.Join(unsupported, ", ")) + } + log.Info().Str("backup", backupName).Str("prefix", cloudPrefix).Msg("backup has ClickHouse Cloud / native BACKUP layout (.backup without metadata.json), switching to restore_cloud") + return b.RestoreCloud(RestoreCloudOptions{ + Prefix: cloudPrefix, + TablePattern: tablePattern, + Partitions: partitions, + RestoreOnCluster: b.cfg.General.RestoreSchemaOnCluster, + SkipEmptyTables: skipEmptyTables, + Drop: dropExists, + }, commandId) + } + } // don't need to create pid separately because we combine Download+Restore defer pidlock.RemovePidFile(backupName) // the worker nodes run `restore_remote --embedded-on-cluster-worker` below, Restore must not start `restore` on them diff --git a/pkg/backup/upload.go b/pkg/backup/upload.go index 7301ccc2..76a881a4 100644 --- a/pkg/backup/upload.go +++ b/pkg/backup/upload.go @@ -496,6 +496,15 @@ func (b *Backuper) RemoveOldBackupsRemote(ctx context.Context) error { if err != nil { return errors.Wrap(err, "BackupList") } + // native BACKUP (ClickHouse Cloud) backups are not created by clickhouse-backup, retention never deletes them, + // https://github.com/Altinity/clickhouse-backup/issues/1574 + ownBackups := backupList[:0] + for _, backup := range backupList { + if backup.DataFormat != storage.CloudBackupDataFormat { + ownBackups = append(ownBackups, backup) + } + } + backupList = ownBackups if b.cfg.General.RebaseBeforeRemoveOldRemote { backupList = b.rebaseRequiredLiveBackups(ctx, backupList) } diff --git a/pkg/storage/cloud_backup.go b/pkg/storage/cloud_backup.go new file mode 100644 index 00000000..30974c21 --- /dev/null +++ b/pkg/storage/cloud_backup.go @@ -0,0 +1,217 @@ +package storage + +import ( + "context" + "encoding/xml" + "io" + "path" + "strconv" + "strings" + "time" + + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" +) + +// CloudBackupDataFormat - DataFormat of a remote backup without metadata.json but with the `.backup` manifest, +// created by native `BACKUP ... TO S3 / AzureBlobStorage` (ClickHouse Cloud), restore_remote switches to restore_cloud for it, +// https://github.com/Altinity/clickhouse-backup/issues/1574 +const CloudBackupDataFormat = "cloud" + +// CloudBackupHeader - fields of the `.backup` manifest written before +type CloudBackupHeader struct { + Timestamp time.Time + UUID string + // BaseBackup - `` of an incremental backup, BackupInfo::toString() of ClickHouse keeps the credentials, + // never log it as is, use ParseCloudBackupLocation + BaseBackup string + BaseBackupUUID string +} + +// cloudBackupSummary - aggregates of the `.backup` manifest, the manifest has no totals, +// ClickHouse computes them in BackupImpl::writeBackupMetadata and keeps them in system.backups only +type cloudBackupSummary struct { + CloudBackupHeader + // DataSize - bytes stored under this backup prefix, same formula as size_of_entries in BackupImpl::writeBackupMetadata + DataSize uint64 +} + +// ReadCloudBackupHeader streams the `.backup` XML until , so the file list of a big manifest is not downloaded +func ReadCloudBackupHeader(r io.Reader) (*CloudBackupHeader, error) { + summary, err := parseCloudBackup(r, true) + if err != nil { + return nil, err + } + return &summary.CloudBackupHeader, nil +} + +func parseCloudBackupSummary(r io.Reader) (*cloudBackupSummary, error) { + return parseCloudBackup(r, false) +} + +// parseCloudBackup streams the `.backup` XML token by token, the manifest has one per logical file +// of the backup and can be hundreds of MB, so the file list is never kept in memory +func parseCloudBackup(r io.Reader, headerOnly bool) (*cloudBackupSummary, error) { + summary := &cloudBackupSummary{} + decoder := xml.NewDecoder(r) + deduplicateFiles, checksumGenerator := true, false + var text strings.Builder + var fileName, dataFile string + var size, baseSize uint64 + var useBase bool + parseUint := func(s string) (uint64, error) { + return strconv.ParseUint(strings.TrimSpace(s), 10, 64) + } + for { + token, err := decoder.Token() + if err == io.EOF { + return summary, nil + } + if err != nil { + return nil, errors.Wrap(err, "can't parse .backup manifest") + } + switch t := token.(type) { + case xml.StartElement: + text.Reset() + if headerOnly && t.Name.Local == "contents" { + return summary, nil + } + if t.Name.Local == "file" { + fileName, dataFile, size, baseSize, useBase = "", "", 0, 0, false + } + case xml.CharData: + text.Write(t) + case xml.EndElement: + value := strings.TrimSpace(text.String()) + text.Reset() + switch t.Name.Local { + case "timestamp": + if ts, parseErr := time.Parse(time.DateTime, value); parseErr == nil { + summary.Timestamp = ts + } + case "uuid": + summary.UUID = value + case "base_backup": + summary.BaseBackup = value + case "base_backup_uuid": + summary.BaseBackupUUID = value + case "deduplicate_files": + deduplicateFiles = value != "0" && !strings.EqualFold(value, "false") + case "data_file_name_generator": + checksumGenerator = strings.EqualFold(value, "checksum") + case "name": + fileName = value + case "data_file": + dataFile = value + case "use_base": + useBase = strings.EqualFold(value, "true") + case "size": + if size, err = parseUint(value); err != nil { + return nil, errors.Wrapf(err, "bad %q in .backup manifest", value) + } + case "base_size": + if baseSize, err = parseUint(value); err != nil { + return nil, errors.Wrapf(err, "bad %q in .backup manifest", value) + } + case "file": + // is written only when it differs from + if useBase && baseSize == 0 { + baseSize = size + } + // a deduplicated file of the FirstFileName generator points to another file which holds the bytes, + // the checksum generator always writes = the checksum-named blob + hasEntry := !deduplicateFiles || (size > 0 && size != baseSize && (dataFile == "" || dataFile == fileName || checksumGenerator)) + if hasEntry && size > baseSize { + summary.DataSize += size - baseSize + } + } + } + } +} + +// readCloudBackupMetadata detects the native BACKUP layout for a backup without metadata.json, +// returns nil when `/.backup` does not exist, the second result is false when +// the manifest could not be read and the entry shall not be cached +func (bd *BackupDestination) readCloudBackupMetadata(ctx context.Context, backupName string) (*Backup, bool) { + manifestKey := path.Join(backupName, ".backup") + mf, err := bd.StatFile(ctx, manifestKey) + if err != nil { + return nil, false + } + cloudBackup := &Backup{ + BackupMetadata: metadata.BackupMetadata{ + BackupName: backupName, + DataFormat: CloudBackupDataFormat, + CreationDate: mf.LastModified(), + MetadataSize: uint64(mf.Size()), + }, + UploadDate: mf.LastModified(), + } + // the backup is not marked as broken on a read error, clean_remote_broken would delete a backup which clickhouse-backup did not create + r, err := bd.GetFileReader(ctx, manifestKey) + if err != nil { + log.Warn().Err(err).Str("backup", backupName).Msg("can't open .backup, data size is unknown") + return cloudBackup, false + } + summary, err := parseCloudBackupSummary(r) + _ = r.Close() + if err != nil { + log.Warn().Err(err).Str("backup", backupName).Msg("can't read .backup, data size is unknown") + return cloudBackup, false + } + if !summary.Timestamp.IsZero() { + cloudBackup.CreationDate = summary.Timestamp + } + cloudBackup.DataSize = summary.DataSize + if summary.BaseBackup != "" { + bucketOrContainer, remotePath := bd.cloudBucketAndPath() + cloudBackup.RequiredBackup, cloudBackup.Tags = cloudBaseBackupName(summary.BaseBackup, bucketOrContainer, remotePath) + } + return cloudBackup, true +} + +// cloudBucketAndPath - bucket/container and path of the remote storage, to map `` to a backup name +func (bd *BackupDestination) cloudBucketAndPath() (string, string) { + switch rs := bd.RemoteStorage.(type) { + case *S3: + return rs.Config.Bucket, rs.Config.Path + case *GCS: + return rs.Config.Bucket, rs.Config.Path + case *AzureBlob: + return rs.Config.Container, rs.Config.Path + } + return "", "" +} + +// cloudBaseBackupName maps `` to the name of a backup under the same remote path (RequiredBackup), +// a base stored elsewhere is described in tags without credentials +func cloudBaseBackupName(baseBackup, bucketOrContainer, remotePath string) (string, string) { + loc, err := ParseCloudBackupLocation(baseBackup) + if err != nil { + log.Warn().Err(err).Msg("can't parse in .backup") + return "", "base=unknown" + } + if key, ok := loc.KeyIn(bucketOrContainer); ok { + remotePath = strings.Trim(remotePath, "/") + name, found := strings.CutPrefix(key, remotePath+"/") + if remotePath == "" { + name, found = key, true + } + if found && !strings.Contains(name, "/") { + return name, "" + } + } + return "", "base=" + loc.String() +} + +// isStaleCloudCacheEntry - embedded `create_remote` writes `.backup` via BACKUP ... TO S3 before metadata.json is uploaded, +// a `list` in between caches the backup as cloud, so a cached cloud entry is dropped once metadata.json appears +func (bd *BackupDestination) isStaleCloudCacheEntry(ctx context.Context, cached Backup) bool { + if cached.DataFormat != CloudBackupDataFormat { + return false + } + _, err := bd.StatFile(ctx, path.Join(cached.BackupName, "metadata.json")) + return err == nil +} diff --git a/pkg/storage/cloud_backup_location.go b/pkg/storage/cloud_backup_location.go new file mode 100644 index 00000000..aab1aa89 --- /dev/null +++ b/pkg/storage/cloud_backup_location.go @@ -0,0 +1,202 @@ +package storage + +import ( + "net/url" + "path" + "regexp" + "strings" + + "github.com/pkg/errors" +) + +// CloudBackupLocation - `` of a native BACKUP, e.g. S3('url','key','secret'), +// S3('url', extra_credentials(role_arn='...')) or AzureBlobStorage('connection_string','container','blob_path'), +// see BackupInfo::toAST in ClickHouse +type CloudBackupLocation struct { + Engine string + // Args - positional string literal arguments + Args []string + // KV - key=value arguments, e.g. url='...' + KV map[string]string + // Secrets - credentials found in the arguments, to redact them from logs and errors + Secrets []string +} + +var cloudAccountKeyRE = regexp.MustCompile(`(?i)AccountKey=([^;]+)`) + +// ParseCloudBackupLocation parses BackupInfo::toString() output, identifiers (named collections) and nested +// functions (extra_credentials) are skipped +func ParseCloudBackupLocation(s string) (*CloudBackupLocation, error) { + s = strings.TrimSpace(s) + open := strings.IndexByte(s, '(') + if open <= 0 || !strings.HasSuffix(s, ")") { + return nil, errors.New("can't parse , expected Engine(...)") + } + loc := &CloudBackupLocation{Engine: strings.TrimSpace(s[:open]), KV: map[string]string{}} + for _, arg := range splitCloudBackupArgs(s[open+1:len(s)-1], ',') { + arg = strings.TrimSpace(arg) + if strings.HasPrefix(arg, "'") { + value, err := unquoteCloudBackupArg(arg) + if err != nil { + return nil, err + } + loc.Args = append(loc.Args, value) + continue + } + if kv := splitCloudBackupArgs(arg, '='); len(kv) == 2 && strings.HasPrefix(strings.TrimSpace(kv[1]), "'") { + value, err := unquoteCloudBackupArg(strings.TrimSpace(kv[1])) + if err != nil { + return nil, err + } + loc.KV[strings.ToLower(strings.TrimSpace(kv[0]))] = value + } + } + // everything except the location itself may hold credentials + locationArgs := 1 + if loc.IsAzureBlob() { + locationArgs = 3 + for _, v := range append(append([]string{}, loc.Args...), loc.KV["connection_string"]) { + if m := cloudAccountKeyRE.FindStringSubmatch(v); m != nil { + loc.Secrets = append(loc.Secrets, m[1]) + } + } + } + for i, v := range loc.Args { + if i >= locationArgs && v != "" { + loc.Secrets = append(loc.Secrets, v) + } + } + for k, v := range loc.KV { + if k != "url" && k != "container" && k != "blob_path" && k != "connection_string" && v != "" { + loc.Secrets = append(loc.Secrets, v) + } + } + return loc, nil +} + +func (loc *CloudBackupLocation) IsAzureBlob() bool { + return strings.EqualFold(loc.Engine, "AzureBlobStorage") +} + +func (loc *CloudBackupLocation) arg(key string, pos int) string { + if v, ok := loc.KV[key]; ok { + return v + } + if pos < len(loc.Args) { + return loc.Args[pos] + } + return "" +} + +// KeyIn returns the key prefix of the backup inside bucket (S3) or container (AzureBlobStorage), +// false when the backup is stored in another bucket/container or the location is unknown +func (loc *CloudBackupLocation) KeyIn(bucketOrContainer string) (string, bool) { + if bucketOrContainer == "" { + return "", false + } + if loc.IsAzureBlob() { + if loc.arg("container", 1) != bucketOrContainer { + return "", false + } + key := strings.Trim(loc.arg("blob_path", 2), "/") + return key, key != "" + } + if !strings.EqualFold(loc.Engine, "S3") { + return "", false + } + u, err := url.Parse(loc.arg("url", 0)) + if err != nil || u.Host == "" { + return "", false + } + var key string + switch { + // s3://bucket/key, gs://bucket/key + case u.Scheme != "http" && u.Scheme != "https": + if u.Host != bucketOrContainer { + return "", false + } + key = u.Path + // virtual-hosted style https://bucket.s3.region.amazonaws.com/key + case strings.HasPrefix(u.Host, bucketOrContainer+"."): + key = u.Path + // path style https://s3.region.amazonaws.com/bucket/key, https://storage.googleapis.com/bucket/key, http://minio:9000/bucket/key + case strings.HasPrefix(u.Path, "/"+bucketOrContainer+"/"): + key = strings.TrimPrefix(u.Path, "/"+bucketOrContainer) + default: + return "", false + } + key = strings.Trim(key, "/") + return key, key != "" +} + +// String - the location without credentials, safe for logs and `list` output +func (loc *CloudBackupLocation) String() string { + if loc.IsAzureBlob() { + return "azblob://" + path.Join(loc.arg("container", 1), loc.arg("blob_path", 2)) + } + u, err := url.Parse(loc.arg("url", 0)) + if err != nil { + return loc.Engine + "(?)" + } + u.User = nil + u.RawQuery = "" + return u.String() +} + +// splitCloudBackupArgs splits by sep outside of quotes and parentheses +func splitCloudBackupArgs(s string, sep byte) []string { + parts := make([]string, 0) + depth, start := 0, 0 + inQuote := false + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case inQuote && c == '\\': + i++ + case c == '\'': + inQuote = !inQuote + case inQuote: + case c == '(': + depth++ + case c == ')': + depth-- + case c == sep && depth == 0: + parts = append(parts, s[start:i]) + start = i + 1 + } + } + return append(parts, s[start:]) +} + +// unquoteCloudBackupArg reverts writeQuotedString of ClickHouse: backslash escapes and doubled quotes +func unquoteCloudBackupArg(s string) (string, error) { + if len(s) < 2 || s[0] != '\'' || s[len(s)-1] != '\'' { + return "", errors.New("can't parse , bad string literal") + } + s = s[1 : len(s)-1] + var out strings.Builder + for i := 0; i < len(s); i++ { + c := s[i] + if c == '\\' && i+1 < len(s) { + i++ + switch s[i] { + case 'n': + out.WriteByte('\n') + case 't': + out.WriteByte('\t') + case 'r': + out.WriteByte('\r') + case '0': + out.WriteByte(0) + default: + out.WriteByte(s[i]) + } + continue + } + if c == '\'' && i+1 < len(s) && s[i+1] == '\'' { + i++ + } + out.WriteByte(c) + } + return out.String(), nil +} diff --git a/pkg/storage/cloud_backup_test.go b/pkg/storage/cloud_backup_test.go new file mode 100644 index 00000000..390386e7 --- /dev/null +++ b/pkg/storage/cloud_backup_test.go @@ -0,0 +1,271 @@ +package storage + +import ( + "bytes" + "context" + "io" + "sort" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// FirstFileName generator: metadata files are stored as is, the second hits.bin is deduplicated +// into the first one via , the use_base file is stored in the base backup (only the appended bytes here) +const testCloudBackupFirstFileNameXML = `11` + + `2026-09-20 12:34:568b6c9b4e-0000-0000-0000-000000000000` + + `S3('https://s3.us-east-1.amazonaws.com/bucket/base')8b6c9b4e-0000-0000-0000-000000000001` + + `` + + `metadata/default.sql77abcdef0123456789abcdef0123456789` + + `metadata/default/hits.sql2560123456789abcdef0123456789abcdef` + + `data/default/hits/all_1_1_0/hits.bin100011111111111111111111111111111111` + + `data/default/hits/all_2_2_0/hits.bin100011111111111111111111111111111111data/default/hits/all_1_1_0/hits.bin` + + `data/default/hits/all_0_0_0/hits.bin500022222222222222222222222222222222true` + + `data/default/hits/all_0_0_0/count.txt30033333333333333333333333333333333true10044444444444444444444444444444444` + + `metadata/default/empty.sql0` + + `` + +// Checksum generator (ClickHouse Cloud): every stored file has = the checksum-named blob +const testCloudBackupChecksumXML = `11` + + `2026-09-21 01:02:038b6c9b4e-0000-0000-0000-000000000002` + + `checksum` + + `` + + `metadata/default/hits.sql2560123456789abcdef0123456789abcdef012/3456789abcdef0123456789abcdef` + + `data/default/hits/all_0_0_0/data.packed4096fedcba9876543210fedcba9876543210fed/cba9876543210fedcba9876543210` + + `` + +func TestParseCloudBackupSummary(t *testing.T) { + summary, err := parseCloudBackupSummary(strings.NewReader(testCloudBackupFirstFileNameXML)) + require.NoError(t, err) + assert.Equal(t, time.Date(2026, 9, 20, 12, 34, 56, 0, time.UTC), summary.Timestamp) + // 77 + 256 + 1000 (deduplicated copy skipped) + 0 (whole file in base) + 200 (appended to base) + assert.Equal(t, uint64(77+256+1000+200), summary.DataSize) + + summary, err = parseCloudBackupSummary(strings.NewReader(testCloudBackupChecksumXML)) + require.NoError(t, err) + assert.Equal(t, time.Date(2026, 9, 21, 1, 2, 3, 0, time.UTC), summary.Timestamp) + assert.Equal(t, uint64(256+4096), summary.DataSize) + + // deduplicate_files=0 stores every file under its own name + noDedup := strings.Replace(testCloudBackupFirstFileNameXML, "1", "0", 1) + summary, err = parseCloudBackupSummary(strings.NewReader(noDedup)) + require.NoError(t, err) + assert.Equal(t, uint64(77+256+1000+1000+200), summary.DataSize) + + _, err = parseCloudBackupSummary(strings.NewReader(`abc`)) + assert.Error(t, err) + _, err = parseCloudBackupSummary(strings.NewReader(``)) + assert.Error(t, err) +} + +type memRemoteFile struct { + name string + size int64 + lastModified time.Time +} + +func (f memRemoteFile) Size() int64 { return f.size } +func (f memRemoteFile) Name() string { return f.name } +func (f memRemoteFile) LastModified() time.Time { return f.lastModified } + +// memRemoteStorage - in-memory RemoteStorage, only the methods used by BackupList are implemented +type memRemoteStorage struct { + RemoteStorage + files map[string]string + lastModified time.Time +} + +func (m *memRemoteStorage) Kind() string { return "mem-cloud-backup-test" } + +func (m *memRemoteStorage) StatFile(_ context.Context, key string) (RemoteFile, error) { + content, exists := m.files[key] + if !exists { + return nil, ErrNotFound + } + return memRemoteFile{name: key, size: int64(len(content)), lastModified: m.lastModified}, nil +} + +func (m *memRemoteStorage) GetFileReader(_ context.Context, key string) (io.ReadCloser, error) { + content, exists := m.files[key] + if !exists { + return nil, ErrNotFound + } + return io.NopCloser(bytes.NewBufferString(content)), nil +} + +// Walk - non-recursive listing of the top-level "directories" under prefix +func (m *memRemoteStorage) Walk(ctx context.Context, prefix string, _ bool, fn func(context.Context, RemoteFile) error) error { + prefix = strings.TrimPrefix(prefix, "/") + dirs := map[string]struct{}{} + for key := range m.files { + if !strings.HasPrefix(key, prefix) { + continue + } + dirs[strings.SplitN(strings.TrimPrefix(key, prefix), "/", 2)[0]+"/"] = struct{}{} + } + names := make([]string, 0, len(dirs)) + for name := range dirs { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if err := fn(ctx, memRemoteFile{name: name, lastModified: m.lastModified}); err != nil { + return err + } + } + return nil +} + +func TestBackupListDetectsCloudBackup(t *testing.T) { + // the metadata cache lives in os.TempDir + t.Setenv("TMPDIR", t.TempDir()) + lastModified := time.Date(2026, 9, 22, 0, 0, 0, 0, time.UTC) + bd := &BackupDestination{RemoteStorage: &memRemoteStorage{ + lastModified: lastModified, + files: map[string]string{ + // regular clickhouse-backup backup + "regular/metadata.json": `{"backup_name":"regular","data_format":"tar","data_size":10}`, + // embedded clickhouse-backup backup carries its own .backup, metadata.json wins + "embedded/metadata.json": `{"backup_name":"embedded","data_format":"embedded","data_size":20}`, + "embedded/.backup": testCloudBackupChecksumXML, + // native BACKUP ... TO S3 (ClickHouse Cloud) + "cloud/.backup": testCloudBackupChecksumXML, + "cloud/012/3456789abcdef0123456789abcdef": "CREATE TABLE ...", + // neither metadata.json nor .backup + "broken/shadow/default/t/part.tar": "x", + }, + }} + ctx := context.Background() + + backups, err := bd.BackupList(ctx, true, "") + require.NoError(t, err) + byName := map[string]Backup{} + for _, b := range backups { + byName[b.BackupName] = b + } + require.Len(t, byName, 4) + assert.Equal(t, "tar", byName["regular"].DataFormat) + assert.Equal(t, "embedded", byName["embedded"].DataFormat) + assert.Empty(t, byName["embedded"].Broken) + + cloud := byName["cloud"] + assert.Empty(t, cloud.Broken) + assert.Equal(t, CloudBackupDataFormat, cloud.DataFormat) + assert.Equal(t, uint64(256+4096), cloud.DataSize) + assert.Equal(t, uint64(len(testCloudBackupChecksumXML)), cloud.MetadataSize) + assert.Equal(t, time.Date(2026, 9, 21, 1, 2, 3, 0, time.UTC), cloud.CreationDate) + assert.Equal(t, lastModified, cloud.UploadDate) + + assert.Equal(t, "broken (can't stat metadata.json)", byName["broken"].Broken) + + // fast path used by restore_remote / download, cache is removed to exercise the direct read + require.NoError(t, bd.writeMetadataCacheFile(ctx, map[string]Backup{})) + backups, err = bd.BackupList(ctx, true, "cloud") + require.NoError(t, err) + require.Len(t, backups, 1) + assert.Equal(t, CloudBackupDataFormat, backups[0].DataFormat) + assert.Equal(t, uint64(256+4096), backups[0].DataSize) + + // metadata.json uploaded after a list cached the backup as cloud (embedded create_remote in progress) + bd.RemoteStorage.(*memRemoteStorage).files["cloud/metadata.json"] = `{"backup_name":"cloud","data_format":"embedded","data_size":30}` + backups, err = bd.BackupList(ctx, true, "cloud") + require.NoError(t, err) + require.Len(t, backups, 1) + assert.Equal(t, "embedded", backups[0].DataFormat) + backups, err = bd.BackupList(ctx, true, "") + require.NoError(t, err) + for _, b := range backups { + assert.NotEqual(t, CloudBackupDataFormat, b.DataFormat, b.BackupName) + } + + backups, err = bd.BackupList(ctx, true, "broken") + require.NoError(t, err) + require.Len(t, backups, 1) + assert.Equal(t, "broken (can't stat metadata.json)", backups[0].Broken) +} + +func TestParseCloudBackupLocation(t *testing.T) { + testCases := []struct { + name, baseBackup, bucket, expectedKey, expectedString string + expectedFound bool + expectedSecrets []string + }{ + {"s3 path style", `S3('https://s3.us-east-1.amazonaws.com/bucket/path/base', 'AKIAKEY', 'se\'cret')`, "bucket", "path/base", "https://s3.us-east-1.amazonaws.com/bucket/path/base", true, []string{"AKIAKEY", "se'cret"}}, + {"s3 virtual hosted", `S3('https://bucket.s3.us-east-1.amazonaws.com/path/base/', 'KEYX', 'SECRETX')`, "bucket", "path/base", "https://bucket.s3.us-east-1.amazonaws.com/path/base/", true, []string{"KEYX", "SECRETX"}}, + {"gcs", `S3('https://storage.googleapis.com/bucket/base','KEYX','SECRETX')`, "bucket", "base", "https://storage.googleapis.com/bucket/base", true, []string{"KEYX", "SECRETX"}}, + {"s3 scheme", `S3('s3://bucket/path/base')`, "bucket", "path/base", "s3://bucket/path/base", true, nil}, + {"iam role", `S3('https://s3.us-west-2.amazonaws.com/bucket/base', extra_credentials(role_arn = 'arn:aws:iam::1:role/r'))`, "bucket", "base", "https://s3.us-west-2.amazonaws.com/bucket/base", true, nil}, + {"key value url", `S3(url = 'http://minio:9000/bucket/base', access_key_id = 'KEYX', secret_access_key = 'SECRETX')`, "bucket", "base", "http://minio:9000/bucket/base", true, []string{"KEYX", "SECRETX"}}, + {"another bucket", `S3('https://s3.us-east-1.amazonaws.com/other/base','KEYX','SECRETX')`, "bucket", "", "https://s3.us-east-1.amazonaws.com/other/base", false, []string{"KEYX", "SECRETX"}}, + {"bucket name prefix only", `S3('https://s3.us-east-1.amazonaws.com/bucket2/base')`, "bucket", "", "https://s3.us-east-1.amazonaws.com/bucket2/base", false, nil}, + {"azblob", `AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=acc;AccountKey=azkey==;BlobEndpoint=https://acc.blob.core.windows.net;', 'container', 'path/base/')`, "container", "path/base", "azblob://container/path/base", true, []string{"azkey=="}}, + {"azblob account args", `AzureBlobStorage('https://acc.blob.core.windows.net', 'container', 'base', 'acc', 'azkey')`, "other", "", "azblob://container/base", false, []string{"acc", "azkey"}}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + loc, err := ParseCloudBackupLocation(tc.baseBackup) + require.NoError(t, err) + key, found := loc.KeyIn(tc.bucket) + assert.Equal(t, tc.expectedFound, found) + assert.Equal(t, tc.expectedKey, key) + assert.Equal(t, tc.expectedString, loc.String()) + assert.ElementsMatch(t, tc.expectedSecrets, loc.Secrets) + for _, secret := range loc.Secrets { + assert.NotContains(t, loc.String(), secret) + } + }) + } + _, err := ParseCloudBackupLocation("S3") + assert.Error(t, err) + _, err = ParseCloudBackupLocation("S3('unterminated)") + assert.Error(t, err) +} + +func TestCloudBaseBackupName(t *testing.T) { + base := func(u string) string { return "S3('" + u + "', 'KEY', 'SECRET')" } + name, tags := cloudBaseBackupName(base("https://s3.us-east-1.amazonaws.com/bucket/backups/full"), "bucket", "/backups/") + assert.Equal(t, "full", name) + assert.Empty(t, tags) + name, _ = cloudBaseBackupName(base("https://bucket.s3.us-east-1.amazonaws.com/full"), "bucket", "") + assert.Equal(t, "full", name) + // nested under another path of the same bucket + name, tags = cloudBaseBackupName(base("https://s3.us-east-1.amazonaws.com/bucket/other/full"), "bucket", "backups") + assert.Empty(t, name) + assert.Equal(t, "base=https://s3.us-east-1.amazonaws.com/bucket/other/full", tags) + name, tags = cloudBaseBackupName(base("https://s3.us-east-1.amazonaws.com/bucket/backups/deep/full"), "bucket", "backups") + assert.Empty(t, name) + assert.NotContains(t, tags, "SECRET") + name, tags = cloudBaseBackupName(base("https://s3.us-east-1.amazonaws.com/other/full"), "bucket", "") + assert.Empty(t, name) + assert.Equal(t, "base=https://s3.us-east-1.amazonaws.com/other/full", tags) + name, tags = cloudBaseBackupName("garbage", "bucket", "") + assert.Empty(t, name) + assert.Equal(t, "base=unknown", tags) +} + +func TestBackupListCloudBaseBackup(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + // the XML escaping of is decoded, memRemoteStorage has no bucket, so the base is described in tags + bd := &BackupDestination{RemoteStorage: &memRemoteStorage{files: map[string]string{ + "incremental/.backup": `12026-09-23 00:00:00u2` + + `S3('https://s3.us-east-1.amazonaws.com/bucket/full', 'KEY', 'SECRET')u1` + + `metadata/default/t.sql10true`, + }}} + backups, err := bd.BackupList(context.Background(), true, "") + require.NoError(t, err) + require.Len(t, backups, 1) + assert.Equal(t, CloudBackupDataFormat, backups[0].DataFormat) + assert.Equal(t, "base=https://s3.us-east-1.amazonaws.com/bucket/full", backups[0].Tags) +} + +func TestReadCloudBackupHeader(t *testing.T) { + // the header reader stops at , a truncated file list is not an error + header, err := ReadCloudBackupHeader(strings.NewReader(`u2S3('u')u1x`)) + require.NoError(t, err) + assert.Equal(t, "u2", header.UUID) + assert.Equal(t, "S3('u')", header.BaseBackup) + assert.Equal(t, "u1", header.BaseBackupUUID) +} diff --git a/pkg/storage/general.go b/pkg/storage/general.go index 2d29ce24..e35ba2fd 100644 --- a/pkg/storage/general.go +++ b/pkg/storage/general.go @@ -323,7 +323,7 @@ func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, if loadErr != nil && !os.IsNotExist(loadErr) { return nil, errors.Wrap(loadErr, "BackupList loadMetadataCache") } - if cached, ok := listCache[parseMetadataOnly]; ok && cached.Broken == "" { + if cached, ok := listCache[parseMetadataOnly]; ok && cached.Broken == "" && !bd.isStaleCloudCacheEntry(ctx, cached) { log.Debug().Str("backup", parseMetadataOnly).Msg("BackupList: using on-disk metadata cache") return []Backup{cached}, nil } @@ -332,6 +332,19 @@ func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, return nil, errors.Wrap(err, "BackupList readBackupMetadataDirect") } if backup == nil { + // no metadata.json, native BACKUP layout (ClickHouse Cloud) has `.backup` instead, https://github.com/Altinity/clickhouse-backup/issues/1574 + if cloudBackup, complete := bd.readCloudBackupMetadata(ctx, parseMetadataOnly); cloudBackup != nil { + if complete { + if listCache == nil { + listCache = map[string]Backup{} + } + listCache[parseMetadataOnly] = *cloudBackup + if writeErr := bd.writeMetadataCacheFile(ctx, listCache); writeErr != nil { + log.Warn().Err(writeErr).Msg("BackupList writeMetadataCacheFile (fast path)") + } + } + return []Backup{*cloudBackup}, nil + } // metadata.json not found — check if the backup prefix has any // content at all. Walk with recursive=false returns top-level // entries only; if we get at least one the folder exists and the @@ -395,12 +408,23 @@ func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, } return nil } - if cachedMetadata, isCached := listCache[backupName]; isCached { + if cachedMetadata, isCached := listCache[backupName]; isCached && !bd.isStaleCloudCacheEntry(ctx, cachedMetadata) { result = append(result, cachedMetadata) return nil } mf, err := bd.StatFile(ctx, path.Join(o.Name(), "metadata.json")) if err != nil { + // native BACKUP layout (ClickHouse Cloud) has `.backup` instead of metadata.json, https://github.com/Altinity/clickhouse-backup/issues/1574 + if errors.Is(err, ErrNotFound) { + if cloudBackup, complete := bd.readCloudBackupMetadata(ctx, backupName); cloudBackup != nil { + if complete { + listCache[backupName] = *cloudBackup + cacheMiss = true + } + result = append(result, *cloudBackup) + return nil + } + } brokenBackup := Backup{ metadata.BackupMetadata{ BackupName: backupName, diff --git a/test/integration/restoreCloud_test.go b/test/integration/restoreCloud_test.go index 4205f89b..149ed5a7 100644 --- a/test/integration/restoreCloud_test.go +++ b/test/integration/restoreCloud_test.go @@ -327,3 +327,160 @@ azblob: Timeout: "1m", }}) } + +// TestRestoreRemoteCloudAutoDetect - a real ClickHouse Cloud backup exported via `BACKUP ... TO S3` into s3->path +// next to a regular backup is detected by layout (`.backup` without metadata.json), +// https://github.com/Altinity/clickhouse-backup/issues/1574: +// `list remote` shows it as `cloud` instead of broken, `clean_remote_broken` and backups_to_keep_remote retention keep it, +// `download` refuses it, `restore_remote` (CLI and POST /backup/restore_remote) switches to restore_cloud. +func TestRestoreRemoteCloudAutoDetect(t *testing.T) { + if os.Getenv("QA_AWS_CLOUD_ENDPOINT") == "" || os.Getenv("QA_AWS_CLOUD_BUCKET") == "" || os.Getenv("QA_AWS_CLOUD_ACCESS_KEY") == "" { + t.Skip("QA_AWS_CLOUD_ENDPOINT, QA_AWS_CLOUD_BUCKET or QA_AWS_CLOUD_ACCESS_KEY is empty, TestRestoreRemoteCloudAutoDetect will skip") + } + realCloudPackedVersionGate(t) + r := require.New(t) + bucket, region := os.Getenv("QA_AWS_CLOUD_BUCKET"), getEnvDefault("QA_AWS_CLOUD_REGION", "us-west-2") + accessKey, secretKey := os.Getenv("QA_AWS_CLOUD_ACCESS_KEY"), os.Getenv("QA_AWS_CLOUD_SECRET_KEY") + id := rand.Int31() + table := fmt.Sprintf("test_restore_remote_cloud_%d", id) + regularTable := fmt.Sprintf("test_restore_remote_regular_%d", id) + cloudBackupName := fmt.Sprintf("cloud_backup_%d", id) + regularBackupName := fmt.Sprintf("regular_backup_%d", id) + // GITHUB_RUN_ID isolates parallel CI jobs which share the bucket + remotePath := fmt.Sprintf("restore_cloud_e2e/autodetect_%s_%d", os.Getenv("GITHUB_RUN_ID"), id) + + cloudQuery(r, fmt.Sprintf("CREATE TABLE default.%s (id UInt64) PARTITION BY id %% 4 ORDER BY id", table)) + defer cloudQuery(r, fmt.Sprintf("DROP TABLE IF EXISTS default.%s", table)) + cloudQuery(r, fmt.Sprintf("INSERT INTO default.%s SELECT number FROM numbers(10000)", table)) + cloudQuery(r, fmt.Sprintf( + "BACKUP TABLE default.%s TO S3('https://s3.%s.amazonaws.com/%s/%s/%s','%s','%s')", + table, region, bucket, remotePath, cloudBackupName, accessKey, secretKey, + ), accessKey, secretKey) + defer deleteCloudBackup(r, &storage.S3{Config: &config.S3Config{ + AccessKey: accessKey, SecretKey: secretKey, Bucket: bucket, Region: region, + }, Concurrency: 1}, remotePath) + + env, _ := NewTestEnvironment(t) + defer env.Cleanup(t, r) + env.connectWithWait(t, r, 500*time.Millisecond, 1*time.Second, 1*time.Minute) + + env.queryWithNoError(t, r, fmt.Sprintf("DROP TABLE IF EXISTS default.%s SYNC", table)) + defer env.queryWithNoError(t, r, fmt.Sprintf("DROP TABLE IF EXISTS default.%s SYNC", table)) + env.queryWithNoError(t, r, fmt.Sprintf("CREATE TABLE default.%s (id UInt64) ENGINE=MergeTree ORDER BY id", regularTable)) + defer env.queryWithNoError(t, r, fmt.Sprintf("DROP TABLE IF EXISTS default.%s SYNC", regularTable)) + env.queryWithNoError(t, r, fmt.Sprintf("INSERT INTO default.%s SELECT number FROM numbers(100)", regularTable)) + + configYAML := `general: + remote_storage: s3 + backups_to_keep_remote: 1 +` + cloudTestClickHouseYAML + ` +s3: + access_key: ${QA_AWS_CLOUD_ACCESS_KEY} + secret_key: ${QA_AWS_CLOUD_SECRET_KEY} + bucket: ${QA_AWS_CLOUD_BUCKET} + region: ${QA_AWS_CLOUD_REGION:-us-west-2} + path: ` + remotePath + configFile := "/etc/clickhouse-backup/config-cloud-autodetect.yml" + env.DockerExecNoError(r, "clickhouse", "bash", "-ce", "cat > "+configFile+" <>"+serverLog) + defer func() { + r.NoError(env.DockerExec("clickhouse", "pkill", "-n", "-f", "clickhouse-backup")) + }() + env.DockerExecNoError(r, "clickhouse", "bash", "-ce", "for i in $(seq 1 30); do wget -q -O - http://localhost:7171/backup/status >/dev/null 2>&1 && exit 0; sleep 1; done; echo 'API server did not start'; cat "+serverLog+"; exit 1") + // the server lists remote backups for metrics at startup, POST returns 423 until it finishes + env.DockerExecNoError(r, "clickhouse", "bash", "-ce", "for i in $(seq 1 60); do wget -q -O - http://localhost:7171/backup/status | grep -q 'in progress' || exit 0; sleep 1; done; echo 'startup operations are still in progress'; exit 1") + apiOut, err := env.DockerExecOut("clickhouse", "bash", "-ce", "wget -q -O - --content-on-error --post-data='' http://localhost:7171/backup/restore_remote/"+cloudBackupName+" || (cat "+serverLog+"; exit 1)") + r.NoError(err, "POST /backup/restore_remote output: %s", apiOut) + r.Contains(apiOut, "acknowledged") + env.DockerExecNoError(r, "clickhouse", "bash", "-ce", "for i in $(seq 1 60); do wget -q -O - http://localhost:7171/backup/status | grep -q 'in progress' || exit 0; sleep 1; done; echo 'restore_remote is still in progress'; exit 1") + statusOut, err := env.DockerExecOut("clickhouse", "bash", "-ce", "wget -q -O - http://localhost:7171/backup/status") + r.NoError(err) + r.Contains(statusOut, `"status":"success"`, "unexpected /backup/status: %s", statusOut) + checkCloudRestored(env, r, table, 10000) + + // incremental backup on top of the cloud backup: the base is resolved from of .backup, no --base-prefix + incrementalBackupName := fmt.Sprintf("cloud_incremental_%d", id) + backupURL := func(name string) string { + return fmt.Sprintf("S3('https://s3.%s.amazonaws.com/%s/%s/%s','%s','%s')", region, bucket, remotePath, name, accessKey, secretKey) + } + cloudQuery(r, fmt.Sprintf("INSERT INTO default.%s SELECT number FROM numbers(10000, 5000)", table)) + cloudQuery(r, fmt.Sprintf("BACKUP TABLE default.%s TO %s SETTINGS base_backup = %s", table, backupURL(incrementalBackupName), backupURL(cloudBackupName)), accessKey, secretKey) + out, err = chb("list", "remote") + r.NoError(err, "list remote output: %s", out) + incrementalLine := "" + for _, line := range strings.Split(out, "\n") { + if strings.HasPrefix(line, incrementalBackupName+" ") { + incrementalLine = line + } + } + r.Contains(incrementalLine, "+"+cloudBackupName, "incremental cloud backup shall require the base: %s", out) + r.NotContains(out, secretKey) + // the base is protected by the required_backup guard + out, err = chb("delete", "remote", cloudBackupName) + r.Error(err, "delete remote of the base shall fail: %s", out) + r.Contains(out, "is required by remote backup(s) "+incrementalBackupName) + + env.queryWithNoError(t, r, fmt.Sprintf("DROP TABLE IF EXISTS default.%s SYNC", table)) + out, err = chb("restore_remote", incrementalBackupName) + r.NoError(err, "restore_remote of the incremental backup output: %s", out) + r.Contains(out, "base backup s3://"+bucket+"/"+remotePath+"/"+cloudBackupName) + r.Contains(out, fmt.Sprintf("base_backup = S3('https://s3.%s.amazonaws.com/%s/%s/%s'", region, bucket, remotePath, cloudBackupName)) + checkCloudRestored(env, r, table, 15000) + + out, err = chb("restore_cloud", "--drop", remotePath+"/"+incrementalBackupName) + r.NoError(err, "restore_cloud of the incremental backup output: %s", out) + checkCloudRestored(env, r, table, 15000) +}