|
8 | 8 | "github.com/cockroachdb/errors" |
9 | 9 | "github.com/cockroachdb/pebble/internal/base" |
10 | 10 | "github.com/cockroachdb/pebble/internal/manifest" |
| 11 | + "github.com/cockroachdb/pebble/metrics" |
| 12 | + "github.com/cockroachdb/pebble/objstorage" |
11 | 13 | ) |
12 | 14 |
|
13 | 15 | // EstimateDiskUsage returns the estimated filesystem space used in bytes for |
@@ -48,71 +50,115 @@ func (d *DB) EstimateDiskUsageByBackingType( |
48 | 50 | readState := d.loadReadState() |
49 | 51 | defer readState.unref() |
50 | 52 |
|
51 | | - sizes := d.fileSizeAnnotator.VersionRangeAnnotation(readState.current, bounds) |
52 | | - return sizes.totalSize, sizes.remoteSize, sizes.externalSize, nil |
| 53 | + sizes := d.tableDiskUsageAnnotator.VersionRangeAnnotation(readState.current, bounds) |
| 54 | + externalSize = sizes.External.TotalBytes() |
| 55 | + remoteSize = externalSize + sizes.Shared.TotalBytes() |
| 56 | + totalSize = remoteSize + sizes.Local.TotalBytes() |
| 57 | + return totalSize, remoteSize, externalSize, nil |
53 | 58 | } |
54 | 59 |
|
55 | | -// fileSizeByBacking contains the estimated file size for LSM data within some |
56 | | -// bounds. It is broken down by backing type. The file size refers to both the |
57 | | -// sstable size and an estimate of the referenced blob sizes. |
58 | | -type fileSizeByBacking struct { |
59 | | - // totalSize is the estimated size of all files for the given bounds. |
60 | | - totalSize uint64 |
61 | | - // remoteSize is the estimated size of remote files for the given bounds. |
62 | | - remoteSize uint64 |
63 | | - // externalSize is the estimated size of external files for the given bounds. |
64 | | - externalSize uint64 |
| 60 | +// TableUsageByPlacement contains space usage information for tables, broken |
| 61 | +// down by where they are stored. |
| 62 | +// |
| 63 | +// Depending on context, this can refer to all tables in the LSM, all tables on |
| 64 | +// a level, or tables within some specified bounds (in the latter case, for |
| 65 | +// tables overlapping the bounds, the usage is a best-effort estimation). |
| 66 | +type TableUsageByPlacement struct { |
| 67 | + metrics.ByPlacement[TableDiskUsage] |
65 | 68 | } |
66 | 69 |
|
67 | | -func (d *DB) singleFileSizeByBacking( |
68 | | - fileSize uint64, t *manifest.TableMetadata, |
69 | | -) (_ fileSizeByBacking, ok bool) { |
70 | | - res := fileSizeByBacking{ |
71 | | - totalSize: fileSize, |
72 | | - } |
| 70 | +// Accumulate adds the rhs counts and sizes to the receiver. |
| 71 | +func (u *TableUsageByPlacement) Accumulate(rhs TableUsageByPlacement) { |
| 72 | + u.Local.Accumulate(rhs.Local) |
| 73 | + u.Shared.Accumulate(rhs.Shared) |
| 74 | + u.External.Accumulate(rhs.External) |
| 75 | +} |
| 76 | + |
| 77 | +// TableDiskUsage contains space usage information for a set of sstables. |
| 78 | +type TableDiskUsage struct { |
| 79 | + // Physical contains the count and total size of physical tables in the set. |
| 80 | + Physical metrics.CountAndSize |
| 81 | + |
| 82 | + // Virtual contains the count and total estimated referenced bytes of virtual |
| 83 | + // tables in the set. |
| 84 | + Virtual metrics.CountAndSize |
| 85 | + |
| 86 | + // ReferencedBytes contains the total estimated size of values stored in blob |
| 87 | + // files referenced by tables in this set (either physical or virtual). |
| 88 | + ReferencedBytes uint64 |
| 89 | +} |
| 90 | + |
| 91 | +// TotalBytes returns the sum of all the byte fields. |
| 92 | +func (u TableDiskUsage) TotalBytes() uint64 { |
| 93 | + return u.Physical.Bytes + u.Virtual.Bytes + u.ReferencedBytes |
| 94 | +} |
| 95 | + |
| 96 | +// Accumulate adds the rhs counts and sizes to the receiver. |
| 97 | +func (u *TableDiskUsage) Accumulate(rhs TableDiskUsage) { |
| 98 | + u.Physical.Accumulate(rhs.Physical) |
| 99 | + u.Virtual.Accumulate(rhs.Virtual) |
| 100 | + u.ReferencedBytes += rhs.ReferencedBytes |
| 101 | +} |
73 | 102 |
|
74 | | - objMeta, err := d.objProvider.Lookup(base.FileTypeTable, t.TableBacking.DiskFileNum) |
75 | | - if err != nil { |
76 | | - return res, false |
| 103 | +func (d *DB) singleTableDiskUsage( |
| 104 | + fileSize uint64, referencedSize uint64, fileNum base.DiskFileNum, isVirtual bool, |
| 105 | +) TableUsageByPlacement { |
| 106 | + u := TableDiskUsage{ |
| 107 | + ReferencedBytes: referencedSize, |
77 | 108 | } |
78 | | - if objMeta.IsRemote() { |
79 | | - res.remoteSize += fileSize |
80 | | - if objMeta.IsExternal() { |
81 | | - res.externalSize += fileSize |
82 | | - } |
| 109 | + if isVirtual { |
| 110 | + u.Virtual.Inc(fileSize) |
| 111 | + } else { |
| 112 | + u.Physical.Inc(fileSize) |
83 | 113 | } |
84 | | - return res, true |
| 114 | + placement := objstorage.Placement(d.objProvider, base.FileTypeTable, fileNum) |
| 115 | + var res TableUsageByPlacement |
| 116 | + res.Set(placement, u) |
| 117 | + return res |
85 | 118 | } |
86 | 119 |
|
87 | | -var fileSizeAnnotatorIdx = manifest.NewTableAnnotationIdx() |
| 120 | +var tableDiskUsageAnnotatorIdx = manifest.NewTableAnnotationIdx() |
88 | 121 |
|
89 | | -// makeFileSizeAnnotator returns an annotator that computes the storage size of |
90 | | -// files. When applicable, this includes both the sstable size and the size of |
91 | | -// any referenced blob files. |
92 | | -func (d *DB) makeFileSizeAnnotator() manifest.TableAnnotator[fileSizeByBacking] { |
93 | | - return manifest.MakeTableAnnotator[fileSizeByBacking]( |
94 | | - fileSizeAnnotatorIdx, |
95 | | - manifest.TableAnnotatorFuncs[fileSizeByBacking]{ |
96 | | - Merge: func(dst *fileSizeByBacking, src fileSizeByBacking) { |
97 | | - dst.totalSize += src.totalSize |
98 | | - dst.remoteSize += src.remoteSize |
99 | | - dst.externalSize += src.externalSize |
100 | | - }, |
101 | | - Table: func(f *manifest.TableMetadata) (v fileSizeByBacking, cacheOK bool) { |
102 | | - return d.singleFileSizeByBacking(f.Size+f.EstimatedReferenceSize(), f) |
| 122 | +// makeTableDiskSpaceUsageAnnotator returns an annotator that computes the |
| 123 | +// storage size of files. When applicable, this includes both the sstable size |
| 124 | +// and the size of any referenced blob files. |
| 125 | +func (d *DB) makeTableDiskSpaceUsageAnnotator() manifest.TableAnnotator[TableUsageByPlacement] { |
| 126 | + return manifest.MakeTableAnnotator[TableUsageByPlacement]( |
| 127 | + tableDiskUsageAnnotatorIdx, |
| 128 | + manifest.TableAnnotatorFuncs[TableUsageByPlacement]{ |
| 129 | + Merge: (*TableUsageByPlacement).Accumulate, |
| 130 | + Table: func(f *manifest.TableMetadata) (v TableUsageByPlacement, cacheOK bool) { |
| 131 | + return d.singleTableDiskUsage(f.Size, f.EstimatedReferenceSize(), f.TableBacking.DiskFileNum, f.Virtual), true |
103 | 132 | }, |
104 | | - PartialOverlap: func(f *manifest.TableMetadata, bounds base.UserKeyBounds) fileSizeByBacking { |
| 133 | + PartialOverlap: func(f *manifest.TableMetadata, bounds base.UserKeyBounds) TableUsageByPlacement { |
105 | 134 | overlappingFileSize, err := d.fileCache.estimateSize(f, bounds.Start, bounds.End.Key) |
106 | 135 | if err != nil { |
107 | | - return fileSizeByBacking{} |
| 136 | + return TableUsageByPlacement{} |
108 | 137 | } |
109 | 138 | overlapFraction := float64(overlappingFileSize) / float64(f.Size) |
110 | 139 | // Scale the blob reference size proportionally to the file |
111 | 140 | // overlap from the bounds to approximate only the blob |
112 | 141 | // references that overlap with the requested bounds. |
113 | | - size := overlappingFileSize + uint64(float64(f.EstimatedReferenceSize())*overlapFraction) |
114 | | - res, _ := d.singleFileSizeByBacking(size, f) |
115 | | - return res |
| 142 | + referencedSize := uint64(float64(f.EstimatedReferenceSize()) * overlapFraction) |
| 143 | + return d.singleTableDiskUsage(overlappingFileSize, referencedSize, f.TableBacking.DiskFileNum, f.Virtual) |
| 144 | + }, |
| 145 | + }) |
| 146 | +} |
| 147 | + |
| 148 | +var blobFileDiskUsageAnnotatorIdx = manifest.NewBlobAnnotationIdx() |
| 149 | + |
| 150 | +// makeDiskSpaceUsageAnnotator returns an annotator that computes the storage size of |
| 151 | +// files. When applicable, this includes both the sstable size and the size of |
| 152 | +// any referenced blob files. |
| 153 | +func (d *DB) makeBlobFileDiskSpaceUsageAnnotator() manifest.BlobFileAnnotator[metrics.CountAndSizeByPlacement] { |
| 154 | + return manifest.MakeBlobFileAnnotator[metrics.CountAndSizeByPlacement]( |
| 155 | + blobFileDiskUsageAnnotatorIdx, |
| 156 | + manifest.BlobFileAnnotatorFuncs[metrics.CountAndSizeByPlacement]{ |
| 157 | + Merge: (*metrics.CountAndSizeByPlacement).Accumulate, |
| 158 | + BlobFile: func(m manifest.BlobFileMetadata) (res metrics.CountAndSizeByPlacement, cacheOK bool) { |
| 159 | + placement := objstorage.Placement(d.objProvider, base.FileTypeBlob, m.Physical.FileNum) |
| 160 | + res.Ptr(placement).Inc(m.Physical.Size) |
| 161 | + return res, true |
116 | 162 | }, |
117 | 163 | }) |
118 | 164 | } |
0 commit comments