Skip to content

Commit 36fc213

Browse files
authored
Merge pull request #1667 from Altinity/issue_1663_cross_disk_reuse
Reuse parts moved between local disks in --diff-from-remote, fix #1663
2 parents 7982a44 + c365ec0 commit 36fc213

11 files changed

Lines changed: 276 additions & 49 deletions

File tree

‎pkg/backup/create.go‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1242,7 +1242,7 @@ func (b *Backuper) AddTableToLocalBackup(ctx context.Context, backupName string,
12421242
}
12431243
var diffTableMetadata metadata.TableMetadata
12441244
if tablesDiffFromRemote != nil {
1245-
diffTableMetadata = tablesDiffFromRemote[metadata.TableTitle{Database: table.Database, Table: table.Name}]
1245+
diffTableMetadata = diffTableMetadataForDisk(tablesDiffFromRemote[metadata.TableTitle{Database: table.Database, Table: table.Name}], disk, diskList)
12461246
}
12471247
// If partitionsIdsMap is not empty, only parts in this partition will back up.
12481248
start := time.Now()
@@ -1413,6 +1413,23 @@ func (b *Backuper) AddTableToLocalBackup(ctx context.Context, backupName string,
14131413
return disksToPartsMap, realSize, objectDiskSize, checksums, hashOfAllFiles, brokenParts, nil
14141414
}
14151415

1416+
// diffTableMetadataForDisk - diff source parts which can be referenced as Required from disk, parts on other
1417+
// disks are added under disk.Name when both disks are compatible, see isCrossDiskReuseCompatible and
1418+
// https://github.com/Altinity/clickhouse-backup/issues/1663
1419+
func diffTableMetadataForDisk(diffTable metadata.TableMetadata, disk clickhouse.Disk, diskList []clickhouse.Disk) metadata.TableMetadata {
1420+
if len(diffTable.Parts) == 0 {
1421+
return diffTable
1422+
}
1423+
diskParts := append([]metadata.Part{}, diffTable.Parts[disk.Name]...)
1424+
for i, d := range diskList {
1425+
if d.Name != disk.Name && isCrossDiskReuseCompatible(&diskList[i], &disk) {
1426+
diskParts = append(diskParts, diffTable.Parts[d.Name]...)
1427+
}
1428+
}
1429+
diffTable.Parts = map[string][]metadata.Part{disk.Name: diskParts}
1430+
return diffTable
1431+
}
1432+
14161433
// listShadowParts walks a frozen shadow directory and lists data-part directories, mirroring the
14171434
// depth-4 part detection (and --partitions filter) in filesystemhelper.MoveShadowToBackup. It is a
14181435
// best-effort list used to attribute broken parts when a shadow move fails and

‎pkg/backup/download.go‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1971,7 +1971,7 @@ func (b *Backuper) downloadDiffParts(ctx context.Context, remoteBackup metadata.
19711971
return nil
19721972
}
19731973
}
1974-
tableRemoteFiles, findErr := b.findDiffBackupFilesRemote(downloadDiffCtx, remoteBackup, requiredBackup, requiredTable, table, diskForDownload, partForDownload)
1974+
tableRemoteFiles, findErr := b.findDiffBackupFilesRemote(downloadDiffCtx, remoteBackup, requiredBackup, requiredTable, table, disks, diskForDownload, partForDownload)
19751975
if findErr != nil {
19761976
return errors.Wrap(findErr, "findDiffBackupFilesRemote")
19771977
}
@@ -2118,7 +2118,7 @@ func (b *Backuper) checkNewPath(newPath string, part metadata.Part) error {
21182118
return nil
21192119
}
21202120

2121-
func (b *Backuper) findDiffBackupFilesRemote(ctx context.Context, backup metadata.BackupMetadata, requiredBackup *metadata.BackupMetadata, requiredTable *metadata.TableMetadata, table metadata.TableMetadata, disk string, part metadata.Part) (map[string]string, error) {
2121+
func (b *Backuper) findDiffBackupFilesRemote(ctx context.Context, backup metadata.BackupMetadata, requiredBackup *metadata.BackupMetadata, requiredTable *metadata.TableMetadata, table metadata.TableMetadata, disks []clickhouse.Disk, disk string, part metadata.Part) (map[string]string, error) {
21222122
log.Debug().Fields(map[string]interface{}{"database": table.Database, "table": table.Table, "part": part.Name, "logger": "findDiffBackupFilesRemote"}).Msg("start")
21232123
var err error
21242124
if requiredBackup == nil {
@@ -2136,7 +2136,7 @@ func (b *Backuper) findDiffBackupFilesRemote(ctx context.Context, backup metadat
21362136
}
21372137

21382138
// recursive find if part in RequiredBackup also Required
2139-
tableRemoteFiles, found, err := b.findDiffRecursive(ctx, requiredBackup, table, requiredTable, part, disk)
2139+
tableRemoteFiles, found, err := b.findDiffRecursive(ctx, requiredBackup, table, requiredTable, disks, part, disk)
21402140
if found {
21412141
return tableRemoteFiles, nil
21422142
}
@@ -2150,7 +2150,7 @@ func (b *Backuper) findDiffBackupFilesRemote(ctx context.Context, backup metadat
21502150

21512151
// try to find part on other disks
21522152
for requiredDisk := range requiredBackup.Disks {
2153-
if requiredDisk != disk {
2153+
if requiredDisk != disk && !b.isCrossDiskDownloadIncompatible(disks, requiredBackup, requiredDisk, disk) {
21542154
tableRemoteFiles, err, found = b.findDiffOnePart(ctx, requiredBackup, table, disk, requiredDisk, part)
21552155
if found {
21562156
return tableRemoteFiles, nil
@@ -2160,6 +2160,9 @@ func (b *Backuper) findDiffBackupFilesRemote(ctx context.Context, backup metadat
21602160
// find one or multiple big files, disk_X.tar files by part.Name
21612161
tableRemoteFiles = make(map[string]string)
21622162
for requiredDisk, requiredParts := range requiredTable.Parts {
2163+
if requiredDisk != disk && b.isCrossDiskDownloadIncompatible(disks, requiredBackup, requiredDisk, disk) {
2164+
continue
2165+
}
21632166
for _, requiredPart := range requiredParts {
21642167
if part.Name == requiredPart.Name {
21652168
localTableDir := path.Join(b.DiskToPathMap[disk], "backup", requiredBackup.BackupName, "shadow", common.TablePathEncode(table.Database), common.TablePathEncode(table.Table), disk)
@@ -2177,7 +2180,23 @@ func (b *Backuper) findDiffBackupFilesRemote(ctx context.Context, backup metadat
21772180
return nil, errors.Errorf("%s.%s %s not found on %s and all required backups sequence", table.Database, table.Table, part.Name, requiredBackup.BackupName)
21782181
}
21792182

2180-
func (b *Backuper) findDiffRecursive(ctx context.Context, requiredBackup *metadata.BackupMetadata, table metadata.TableMetadata, requiredTable *metadata.TableMetadata, part metadata.Part, disk string) (map[string]string, bool, error) {
2183+
// isCrossDiskDownloadIncompatible - a part stored on requiredDisk of the required backup can't be downloaded
2184+
// into disk when only one of them is a plain local disk, files differ: object metadata or ciphertext against
2185+
// data files, see https://github.com/Altinity/clickhouse-backup/issues/1663. A required disk absent in
2186+
// system.disks (rebalance) is described by the required backup disk_types, unknown type is not filtered
2187+
func (b *Backuper) isCrossDiskDownloadIncompatible(disks []clickhouse.Disk, requiredBackup *metadata.BackupMetadata, requiredDisk, disk string) bool {
2188+
currentDisk := b.findDiskByName(disks, disk)
2189+
requiredDiskInfo := b.findDiskByName(disks, requiredDisk)
2190+
if requiredDiskInfo == nil && requiredBackup.DiskTypes[requiredDisk] != "" {
2191+
requiredDiskInfo = &clickhouse.Disk{Name: requiredDisk, Type: requiredBackup.DiskTypes[requiredDisk]}
2192+
}
2193+
if currentDisk == nil || requiredDiskInfo == nil {
2194+
return false
2195+
}
2196+
return isPlainLocalDisk(currentDisk) != isPlainLocalDisk(requiredDiskInfo)
2197+
}
2198+
2199+
func (b *Backuper) findDiffRecursive(ctx context.Context, requiredBackup *metadata.BackupMetadata, table metadata.TableMetadata, requiredTable *metadata.TableMetadata, disks []clickhouse.Disk, part metadata.Part, disk string) (map[string]string, bool, error) {
21812200
log.Debug().Fields(map[string]interface{}{"database": table.Database, "table": table.Table, "part": part.Name, "logger": "findDiffRecursive"}).Msg("start")
21822201
found := false
21832202
for _, requiredParts := range requiredTable.Parts {
@@ -2186,7 +2205,7 @@ func (b *Backuper) findDiffRecursive(ctx context.Context, requiredBackup *metada
21862205
found = true
21872206
if requiredPart.Required {
21882207
// pass nil, nil to fetch the next level of the backup chain (requiredBackup.RequiredBackup)
2189-
tableRemoteFiles, err := b.findDiffBackupFilesRemote(ctx, *requiredBackup, nil, nil, table, disk, part)
2208+
tableRemoteFiles, err := b.findDiffBackupFilesRemote(ctx, *requiredBackup, nil, nil, table, disks, disk, part)
21902209
if err != nil {
21912210
found = false
21922211
log.Warn().Msgf("try find %s.%s %s recursive return err: %v", table.Database, table.Table, part.Name, err)

‎pkg/backup/download_test.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -552,3 +552,25 @@ func TestSkipMissingPartNoSuchUpload(t *testing.T) {
552552
}
553553
}
554554
}
555+
556+
// https://github.com/Altinity/clickhouse-backup/issues/1663
557+
func TestIsCrossDiskDownloadIncompatible(t *testing.T) {
558+
b := &Backuper{}
559+
disks := []clickhouse.Disk{{Name: "default", Type: "local"}, {Name: "hdd1", Type: "local"}, {Name: "disk_s3", Type: "s3"}, {Name: "disk_s3_2", Type: "s3"}, {Name: "hdd3_encrypted", Type: "local", IsEncrypted: 1}}
560+
requiredBackup := &metadata.BackupMetadata{DiskTypes: map[string]string{"removed_local": "local", "removed_s3": "s3"}}
561+
for _, tc := range []struct {
562+
requiredDisk, disk string
563+
incompatible bool
564+
}{
565+
{"hdd1", "default", false},
566+
{"disk_s3", "default", true},
567+
{"default", "disk_s3", true},
568+
{"hdd3_encrypted", "default", true},
569+
{"disk_s3", "disk_s3_2", false},
570+
{"removed_local", "default", false},
571+
{"removed_s3", "default", true},
572+
{"removed_unknown_type", "default", false},
573+
} {
574+
require.Equal(t, tc.incompatible, b.isCrossDiskDownloadIncompatible(disks, requiredBackup, tc.requiredDisk, tc.disk), "%s -> %s", tc.requiredDisk, tc.disk)
575+
}
576+
}

‎pkg/backup/restore.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2869,6 +2869,11 @@ func (b *Backuper) findRequiredPartLocal(ctx context.Context, backupName string,
28692869
if requiredPart.Required {
28702870
return b.findRequiredPartLocal(ctx, requiredBackup.RequiredBackup, *requiredTable, requiredDisk, requiredPart, activeDisk, dbAndTableDir, diskMap, disks)
28712871
}
2872+
// a part moved between disks is referenced from another disk, hardlink can't cross filesystems,
2873+
// download it from remote, see https://github.com/Altinity/clickhouse-backup/issues/1663
2874+
if requiredActiveDisk != activeDisk {
2875+
return "", false, nil
2876+
}
28722877
diskPath, exists := diskMap[requiredActiveDisk]
28732878
if !exists {
28742879
return "", false, errors.Errorf("disk %s not found in diskMap", requiredActiveDisk)

‎pkg/backup/upload.go‎

Lines changed: 55 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ func (b *Backuper) Upload(backupName string, deleteSource bool, diffFrom, diffFr
105105
Database: table.Database,
106106
Table: table.Table,
107107
}]; diffExists {
108-
b.markDuplicatedParts(backupMetadata, &diffTable, table, checkLocalPart)
108+
b.markDuplicatedParts(backupMetadata, &diffTable, table, disks, checkLocalPart)
109109
}
110110
tableDataSize, tableParts, unknownParts := b.estimateUploadDataSize(table, disks)
111111
report.DataSize += tableDataSize
@@ -334,7 +334,7 @@ func (b *Backuper) uploadOneTable(ctx context.Context, backupName string, delete
334334
Database: table.Database,
335335
Table: table.Table,
336336
}]; diffExists {
337-
b.markDuplicatedParts(backupMetadata, &diffTable, table, checkLocalPart)
337+
b.markDuplicatedParts(backupMetadata, &diffTable, table, disks, checkLocalPart)
338338
}
339339
}
340340
log.Info().Fields(map[string]interface{}{
@@ -1086,55 +1086,72 @@ func (b *Backuper) uploadTableMetadataEmbedded(ctx context.Context, backupName s
10861086
return info.Size(), nil
10871087
}
10881088

1089-
func (b *Backuper) markDuplicatedParts(backup *metadata.BackupMetadata, existsTable *metadata.TableMetadata, newTable *metadata.TableMetadata, checkLocal bool) {
1089+
func (b *Backuper) markDuplicatedParts(backup *metadata.BackupMetadata, existsTable *metadata.TableMetadata, newTable *metadata.TableMetadata, disks []clickhouse.Disk, checkLocal bool) {
1090+
// part name -> disk in the diff backup, a part lives on exactly one disk
1091+
existsPartDisks := map[string]string{}
1092+
for disk, parts := range existsTable.Parts {
1093+
for _, p := range parts {
1094+
existsPartDisks[p.Name] = disk
1095+
}
1096+
}
10901097
for disk, newParts := range newTable.Parts {
1091-
if _, diskExists := existsTable.Parts[disk]; diskExists {
1092-
if len(existsTable.Parts[disk]) == 0 {
1098+
for i := range newParts {
1099+
name := newParts[i].Name
1100+
existsDisk, partExists := existsPartDisks[name]
1101+
if !partExists {
10931102
continue
10941103
}
1095-
existsPartsMap := common.EmptyMap{}
1096-
for _, p := range existsTable.Parts[disk] {
1097-
existsPartsMap[p.Name] = struct{}{}
1104+
// a part moved between disks (TTL TO VOLUME, MOVE PARTITION) keeps name and hash_of_all_files,
1105+
// reuse it only when both disks store the same bytes in the backup, checkLocal compares inodes
1106+
// which never match across disks, see https://github.com/Altinity/clickhouse-backup/issues/1663
1107+
if existsDisk != disk && (checkLocal || !isCrossDiskReuseCompatible(b.findDiskByName(disks, existsDisk), b.findDiskByName(disks, disk))) {
1108+
continue
10981109
}
1099-
for i := range newParts {
1100-
name := newParts[i].Name
1101-
if _, partExists := existsPartsMap[name]; !partExists {
1110+
// Name matched in the diff backup — verify content fingerprint
1111+
// before treating the part as a duplicate. Prefer
1112+
// hash_of_all_files; fall back to legacy checksums; if neither
1113+
// side has any fingerprint, accept the name match.
1114+
// See https://github.com/Altinity/clickhouse-backup/issues/1307
1115+
if diffHash, ok := existsTable.HashOfAllFiles[name]; ok {
1116+
newHash := newTable.HashOfAllFiles[name]
1117+
if newHash == "" || diffHash != newHash {
1118+
log.Debug().Msgf("part %s.%s/%s name matched diff backup %s but hash_of_all_files mismatch (%q vs %q), will be re-uploaded", existsTable.Database, existsTable.Table, name, backup.RequiredBackup, diffHash, newHash)
11021119
continue
11031120
}
1104-
// Name matched in the diff backup — verify content fingerprint
1105-
// before treating the part as a duplicate. Prefer
1106-
// hash_of_all_files; fall back to legacy checksums; if neither
1107-
// side has any fingerprint, accept the name match.
1108-
// See https://github.com/Altinity/clickhouse-backup/issues/1307
1109-
if diffHash, ok := existsTable.HashOfAllFiles[name]; ok {
1110-
newHash := newTable.HashOfAllFiles[name]
1111-
if newHash == "" || diffHash != newHash {
1112-
log.Debug().Msgf("part %s.%s/%s name matched diff backup %s but hash_of_all_files mismatch (%q vs %q), will be re-uploaded", existsTable.Database, existsTable.Table, name, backup.RequiredBackup, diffHash, newHash)
1113-
continue
1114-
}
1115-
} else if diffCksum, ok := existsTable.Checksums[name]; ok {
1116-
newCksum, newOk := newTable.Checksums[name]
1117-
if !newOk || diffCksum != newCksum {
1118-
log.Debug().Msgf("part %s.%s/%s name matched diff backup %s but checksums mismatch (%d vs %d), will be re-uploaded", existsTable.Database, existsTable.Table, name, backup.RequiredBackup, diffCksum, newCksum)
1119-
continue
1120-
}
1121+
} else if diffCksum, ok := existsTable.Checksums[name]; ok {
1122+
newCksum, newOk := newTable.Checksums[name]
1123+
if !newOk || diffCksum != newCksum {
1124+
log.Debug().Msgf("part %s.%s/%s name matched diff backup %s but checksums mismatch (%d vs %d), will be re-uploaded", existsTable.Database, existsTable.Table, name, backup.RequiredBackup, diffCksum, newCksum)
1125+
continue
11211126
}
1122-
if checkLocal {
1123-
dbAndTablePath := path.Join(common.TablePathEncode(existsTable.Database), common.TablePathEncode(existsTable.Table))
1124-
existsPath := path.Join(b.DiskToPathMap[disk], "backup", backup.RequiredBackup, "shadow", dbAndTablePath, disk, name)
1125-
newPath := path.Join(b.DiskToPathMap[disk], "backup", backup.BackupName, "shadow", dbAndTablePath, disk, name)
1126-
1127-
if err := filesystemhelper.IsDuplicatedParts(existsPath, newPath); err != nil {
1128-
log.Debug().Msgf("part '%s' and '%s' must be the same: %v", existsPath, newPath, err)
1129-
continue
1130-
}
1127+
}
1128+
if checkLocal {
1129+
dbAndTablePath := path.Join(common.TablePathEncode(existsTable.Database), common.TablePathEncode(existsTable.Table))
1130+
existsPath := path.Join(b.DiskToPathMap[disk], "backup", backup.RequiredBackup, "shadow", dbAndTablePath, disk, name)
1131+
newPath := path.Join(b.DiskToPathMap[disk], "backup", backup.BackupName, "shadow", dbAndTablePath, disk, name)
1132+
1133+
if err := filesystemhelper.IsDuplicatedParts(existsPath, newPath); err != nil {
1134+
log.Debug().Msgf("part '%s' and '%s' must be the same: %v", existsPath, newPath, err)
1135+
continue
11311136
}
1132-
newParts[i].Required = true
11331137
}
1138+
newParts[i].Required = true
11341139
}
11351140
}
11361141
}
11371142

1143+
// isCrossDiskReuseCompatible - a part moved between two disks can be referenced from the diff backup
1144+
// only when both disks keep plain part files: object disks keep object metadata files instead of data,
1145+
// encrypted disks keep ciphertext, while hash_of_all_files is the same for all of them
1146+
func isCrossDiskReuseCompatible(existsDisk, newDisk *clickhouse.Disk) bool {
1147+
return existsDisk != nil && newDisk != nil && isPlainLocalDisk(existsDisk) && isPlainLocalDisk(newDisk)
1148+
}
1149+
1150+
// isPlainLocalDisk - 22.9+ reports the underlying type for encrypted disks, so is_encrypted is checked too
1151+
func isPlainLocalDisk(disk *clickhouse.Disk) bool {
1152+
return disk.Type == "local" && disk.IsEncrypted == 0
1153+
}
1154+
11381155
func (b *Backuper) ReadBackupMetadataLocal(ctx context.Context, backupName string) (*metadata.BackupMetadata, error) {
11391156
var backupMetadataBody []byte
11401157
var err error

‎pkg/backup/upload_test.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"runtime"
88
"testing"
99

10+
"github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse"
1011
"github.com/Altinity/clickhouse-backup/v2/pkg/config"
1112
"github.com/Altinity/clickhouse-backup/v2/pkg/metadata"
1213

@@ -132,3 +133,37 @@ func TestNewPostUploadError(t *testing.T) {
132133
assert.Contains(t, err.Error(), stepErr.Error())
133134
}
134135
}
136+
137+
// https://github.com/Altinity/clickhouse-backup/issues/1663
138+
func TestMarkDuplicatedPartsMovedBetweenDisks(t *testing.T) {
139+
b := &Backuper{}
140+
backup := &metadata.BackupMetadata{RequiredBackup: "full"}
141+
disks := []clickhouse.Disk{{Name: "default", Type: "local"}, {Name: "hdd1", Type: "local"}, {Name: "disk_s3", Type: "s3"}, {Name: "hdd3_encrypted", Type: "local", IsEncrypted: 1}}
142+
existsTable := &metadata.TableMetadata{
143+
Parts: map[string][]metadata.Part{"default": {{Name: "0_1_1_0"}, {Name: "1_2_2_0"}, {Name: "2_3_3_0"}, {Name: "3_4_4_0"}, {Name: "4_5_5_0"}}},
144+
HashOfAllFiles: map[string]string{"0_1_1_0": "h0", "1_2_2_0": "h1", "2_3_3_0": "h2", "3_4_4_0": "h3", "4_5_5_0": "h4"},
145+
}
146+
newTable := &metadata.TableMetadata{
147+
Parts: map[string][]metadata.Part{
148+
"default": {{Name: "0_1_1_0"}},
149+
"hdd1": {{Name: "1_2_2_0"}, {Name: "4_5_5_0"}},
150+
"disk_s3": {{Name: "2_3_3_0"}},
151+
"hdd3_encrypted": {{Name: "3_4_4_0"}},
152+
},
153+
HashOfAllFiles: map[string]string{"0_1_1_0": "h0", "1_2_2_0": "h1", "2_3_3_0": "h2", "3_4_4_0": "h3", "4_5_5_0": "changed"},
154+
}
155+
b.markDuplicatedParts(backup, existsTable, newTable, disks, false)
156+
required := map[string]bool{}
157+
for _, parts := range newTable.Parts {
158+
for _, p := range parts {
159+
required[p.Name] = p.Required
160+
}
161+
}
162+
require.Equal(t, map[string]bool{
163+
"0_1_1_0": true, // same disk
164+
"1_2_2_0": true, // local -> local
165+
"2_3_3_0": false, // local -> s3
166+
"3_4_4_0": false, // local -> encrypted
167+
"4_5_5_0": false, // local -> local, content changed
168+
}, required)
169+
}

0 commit comments

Comments
 (0)