Skip to content

HDDS-14118. Support OBS keys and additional flags in ContainerToKeyMapping tool #9599

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sarvekshayr wants to merge 3 commits into
base: master
Choose a base branch
from
+174 −60
Open

HDDS-14118. Support OBS keys and additional flags in ContainerToKeyMapping tool #9599

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
54c06b8
HDDS-14118. Support OBS keys and additional flags in ContainerToKeyMa…
sarvekshayr Jan 7, 2026
8a0bcaf
Fix javadoc
sarvekshayr Jan 16, 2026
3c1cd17
Skip prepareDirIdTree if onlyFileNames is true and optimise
sarvekshayr Jan 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 84 additions & 38 deletions hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/om/ContainerToKeyMapping.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,12 @@

/**
* Tool to map full key paths that use the specified containers.
* Note: Currently only processes FSO layout buckets.
* Supports both FSO (File System Optimized) and OBS (Object Store) bucket layouts.
*/
@CommandLine.Command(
name = "container-key-mapping",
aliases = "ckm",
description = "Maps full key paths that use the specified containers. " +
"Note: A container can have both FSO and OBS keys. Currently this tool processes only FSO keys")
description = "Maps full key paths that use the specified containers.")
public class ContainerToKeyMapping extends AbstractSubcommand implements Callable<Void> {
private static final String DIRTREE_DB_NAME = "omdirtree.db";
private static final String DIRTREE_TABLE_NAME = "dirTreeTable";
Expand All @@ -80,22 +79,26 @@ public class ContainerToKeyMapping extends AbstractSubcommand implements Callabl
description = "Comma separated Container IDs")
private String containers;

@CommandLine.Option(names = {"--onlyFileNames"},
defaultValue = "false",
description = "Only display file names without full path")
private boolean onlyFileNames;

private DBStore omDbStore;
private Table<String, OmVolumeArgs> volumeTable;
private Table<String, OmBucketInfo> bucketTable;
private Table<String, OmDirectoryInfo> directoryTable;
private Table<String, OmKeyInfo> fileTable;
private Table<String, OmKeyInfo> keyTable;
private DBStore dirTreeDbStore;
private Table<Long, String> dirTreeTable;
// Cache volume IDs to avoid repeated lookups
private final Map<String, Long> volumeCache = new HashMap<>();
private ConfigurationSource conf;

// TODO: Add support to OBS keys (HDDS-14118)
@Override
public Void call() throws Exception {
err().println("Note: A container can have both FSO and OBS keys. Currently this tool processes only FSO keys");


Copy link
Contributor

@priyeshkaratha priyeshkaratha Jan 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit - avoid this new line

String dbPath = parent.getDbPath();
// Parse container IDs
Set<Long> containerIDs = Arrays.stream(containers.split(","))
Expand All @@ -122,9 +125,9 @@ public Void call() throws Exception {
bucketTable = OMDBDefinition.BUCKET_TABLE_DEF.getTable(omDbStore, CacheType.NO_CACHE);
directoryTable = OMDBDefinition.DIRECTORY_TABLE_DEF.getTable(omDbStore, CacheType.NO_CACHE);
fileTable = OMDBDefinition.FILE_TABLE_DEF.getTable(omDbStore, CacheType.NO_CACHE);
keyTable = OMDBDefinition.KEY_TABLE_DEF.getTable(omDbStore, CacheType.NO_CACHE);

openDirTreeDB(dbPath);
retrieve(writer, containerIDs);
retrieve(dbPath, writer, containerIDs);
} catch (Exception e) {
err().println("Failed to open RocksDB: " + e);
throw e;
Expand Down Expand Up @@ -163,59 +166,102 @@ private void closeDirTreeDB(String dbPath) throws IOException {
}
}

private void retrieve(PrintWriter writer, Set<Long> containerIds) {
// Build dir tree
private void retrieve(String dbPath, PrintWriter writer, Set<Long> containerIds) {
Map<Long, Pair<Long, String>> bucketVolMap = new HashMap<>();
try {
prepareDirIdTree(bucketVolMap);
} catch (Exception e) {
err().println("Exception occurred reading directory Table, " + e);
return;
// Build dir tree for FSO keys only if we need full paths
if (!onlyFileNames) {
try {
openDirTreeDB(dbPath);
prepareDirIdTree(bucketVolMap);
} catch (Exception e) {
err().println("Exception occurred reading directory Table, " + e);
return;
}
}

// Map to collect keys per container
Map<Long, List<String>> containerToKeysMap = new HashMap<>();
// Track unreferenced keys count per container
// Track unreferenced keys count per container (FSO only)
Map<Long, Long> unreferencedCountMap = new HashMap<>();
for (Long containerId : containerIds) {
containerToKeysMap.put(containerId, new ArrayList<>());
unreferencedCountMap.put(containerId, 0L);
}

// Iterate file table and filter for container
try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> fileIterator =
// Process FSO keys (fileTable)
processFSOKeys(containerIds, containerToKeysMap, unreferencedCountMap, bucketVolMap);

// Process OBS keys (keyTable)
processOBSKeys(containerIds, containerToKeysMap);

jsonOutput(writer, containerToKeysMap, unreferencedCountMap);
}
Comment on lines +191 to +198
Copy link
Contributor

@priyeshkaratha priyeshkaratha Jan 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit - please remove unnecessary new lines in the code. I can see after calling every method a new line is added.


private void processFSOKeys(Set<Long> containerIds, Map<Long, List<String>> containerToKeysMap,
Map<Long, Long> unreferencedCountMap, Map<Long, Pair<Long, String>> bucketVolMap) {
try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> fileIterator =
fileTable.iterator()) {

while (fileIterator.hasNext()) {
Table.KeyValue<String, OmKeyInfo> entry = fileIterator.next();
OmKeyInfo keyInfo = entry.getValue();

// Find which containers this key uses
Set<Long> keyContainers = new HashSet<>();
keyInfo.getKeyLocationVersions().forEach(
e -> e.getLocationList().forEach(
blk -> {
long cid = blk.getBlockID().getContainerID();
if (containerIds.contains(cid)) {
keyContainers.add(cid);
}
}));
Set<Long> keyContainers = getKeyContainers(keyInfo, containerIds);

if (!keyContainers.isEmpty()) {
// Reconstruct full path
String fullPath = reconstructFullPath(keyInfo, bucketVolMap, unreferencedCountMap, keyContainers);
if (fullPath != null) {
// For FSO keys, reconstruct the full path
// Or extract just the key name if onlyFileNames is true
String keyPath = onlyFileNames ? keyInfo.getKeyName() :
reconstructFullPath(keyInfo, bucketVolMap, unreferencedCountMap, keyContainers);
if (keyPath != null) {
for (Long containerId : keyContainers) {
containerToKeysMap.get(containerId).add(fullPath);
containerToKeysMap.get(containerId).add(keyPath);
}
}
}
}
} catch (Exception e) {
err().println("Exception occurred reading file Table, " + e);
return;
err().println("Exception occurred reading fileTable (FSO keys), " + e);
}
jsonOutput(writer, containerToKeysMap, unreferencedCountMap);
}

private void processOBSKeys(Set<Long> containerIds, Map<Long, List<String>> containerToKeysMap) {
try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> keyIterator =
keyTable.iterator()) {

while (keyIterator.hasNext()) {
Table.KeyValue<String, OmKeyInfo> entry = keyIterator.next();
OmKeyInfo keyInfo = entry.getValue();

// Find which containers this key uses
Set<Long> keyContainers = getKeyContainers(keyInfo, containerIds);

if (!keyContainers.isEmpty()) {
// For OBS keys, use the database key directly (already in /volume/bucket/key format)
// Or extract just the key name if onlyFileNames is true
String keyPath = onlyFileNames ? keyInfo.getKeyName() : entry.getKey();
for (Long containerId : keyContainers) {
containerToKeysMap.get(containerId).add(keyPath);
}
}
}
} catch (Exception e) {
err().println("Exception occurred reading keyTable (OBS keys), " + e);
}
}

private Set<Long> getKeyContainers(OmKeyInfo keyInfo, Set<Long> targetContainerIds) {
Set<Long> keyContainers = new HashSet<>();
keyInfo.getKeyLocationVersions().forEach(
e -> e.getLocationList().forEach(
blk -> {
long cid = blk.getBlockID().getContainerID();
if (targetContainerIds.contains(cid)) {
keyContainers.add(cid);
}
}));
return keyContainers;
}

private void prepareDirIdTree(Map<Long, Pair<Long, String>> bucketVolMap) throws Exception {
Expand Down Expand Up @@ -263,19 +309,19 @@ private String reconstructFullPath(OmKeyInfo keyInfo, Map<Long, Pair<Long, Strin
sb.insert(0, nameParentPair.getValue() + OM_KEY_PREFIX);
prvParent = nameParentPair.getKey();
if (null == prvParent) {
return sb.toString();
return OM_KEY_PREFIX + sb;
}
continue;
}

// Check dir tree
Pair<Long, String> nameParentPair = getFromDirTree(prvParent);
if (nameParentPair == null) {
// Parent not found - increment unreferenced count for all containers this key uses
// If parent is not found, mark the key as unreferenced and increment its count
for (Long containerId : keyContainers) {
unreferencedCountMap.put(containerId, unreferencedCountMap.get(containerId) + 1);
}
break;
return "[unreferenced] " + keyInfo.getKeyName();
}
sb.insert(0, nameParentPair.getValue() + OM_KEY_PREFIX);
prvParent = nameParentPair.getKey();
Expand Down Expand Up @@ -342,7 +388,7 @@ private void jsonOutput(PrintWriter writer, Map<Long, List<String>> containerToK
}

containerNode.set("keys", keysArray);
containerNode.put("numOfKeys", entry.getValue().size());
containerNode.put("totalKeys", entry.getValue().size()); // includes unreferenced keys

// Add unreferenced count if > 0
long unreferencedCount = unreferencedCountMap.get(containerId);
Expand Down
Loading