HDDS-13183. Create Cluster Capacity page UI.

hadoop-ozone/recon/pom.xml

@@ -102,6 +102,10 @@
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-compress</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-csv</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/DataNodeMetricsService.java

@@ -24,6 +24,7 @@
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -271,6 +272,11 @@ private void handleCompletedFuture(
   private void updateFinalState(CollectionContext context) {
     // Update shared state atomically
     synchronized (this) {
+      // Sort by pendingBlockSize in descending order so highest values appear first
+      context.results.sort(
+          Comparator.comparingLong(DatanodePendingDeletionMetrics::getPendingBlockSize)
+              .reversed()
+      );
       pendingDeletionList = context.results;
       totalPendingDeletion = context.totalPending;
       totalNodesQueried = context.totalQueried;
@@ -294,16 +300,23 @@ private void resetState() {
     totalNodesFailed = 0;
   }
 
-  public DataNodeMetricsServiceResponse getCollectedMetrics() {
+  public DataNodeMetricsServiceResponse getCollectedMetrics(int limit) {
     startTask();
     if (currentStatus == MetricCollectionStatus.FINISHED) {
-      return DataNodeMetricsServiceResponse.newBuilder()
+      DataNodeMetricsServiceResponse.Builder dnMetricsBuilder = DataNodeMetricsServiceResponse.newBuilder();
+      dnMetricsBuilder
           .setStatus(currentStatus)
-          .setPendingDeletion(pendingDeletionList)
           .setTotalPendingDeletionSize(totalPendingDeletion)
           .setTotalNodesQueried(totalNodesQueried)
-          .setTotalNodeQueryFailures(totalNodesFailed)
-          .build();
+          .setTotalNodeQueryFailures(totalNodesFailed);
+
+      if (limit < 0) {
+        return dnMetricsBuilder.setPendingDeletion(pendingDeletionList).build();
+      } else {
+        return dnMetricsBuilder.setPendingDeletion(
+            pendingDeletionList.subList(0, Math.min(limit, pendingDeletionList.size())
+        )).build();
+      }
     }
     return DataNodeMetricsServiceResponse.newBuilder()
         .setStatus(currentStatus)

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/PendingDeletionEndpoint.java

@@ -17,16 +17,23 @@
 
 package org.apache.hadoop.ozone.recon.api;
 
+import java.io.BufferedWriter;
+import java.io.OutputStreamWriter;
 import java.util.Map;
 import javax.inject.Inject;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
 import org.apache.hadoop.ozone.recon.api.types.DataNodeMetricsServiceResponse;
+import org.apache.hadoop.ozone.recon.api.types.DatanodePendingDeletionMetrics;
 import org.apache.hadoop.ozone.recon.api.types.ScmPendingDeletion;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -56,15 +63,20 @@ public PendingDeletionEndpoint(
   }
 
   @GET
-  public Response getPendingDeletionByComponent(@QueryParam("component") String component) {
+  public Response getPendingDeletionByComponent(
+      @QueryParam("component")
+      String component,
+      @QueryParam("limit")
+      int limit
+  ) {
     if (component == null || component.isEmpty()) {
       return Response.status(Response.Status.BAD_REQUEST)
           .entity("component query parameter is required").build();
     }
     final String normalizedComponent = component.trim().toLowerCase();
     switch (normalizedComponent) {
     case "dn":
-      return handleDataNodeMetrics();
+      return handleDataNodeMetrics(limit);
     case "scm":
       return handleScmPendingDeletion();
     case "om":
@@ -75,8 +87,57 @@ public Response getPendingDeletionByComponent(@QueryParam("component") String co
     }
   }
 
-  private Response handleDataNodeMetrics() {
-    DataNodeMetricsServiceResponse response = dataNodeMetricsService.getCollectedMetrics();
+  @GET
+  @Path("/download")
+  public Response downloadPendingDeleteData() {
+    DataNodeMetricsServiceResponse dnMetricsResponse = dataNodeMetricsService.getCollectedMetrics(-1);
+
+    if (dnMetricsResponse.getStatus() != DataNodeMetricsService.MetricCollectionStatus.FINISHED) {
+      return Response.status(Response.Status.ACCEPTED)
+          .entity(dnMetricsResponse)
+          .type("application/json")
+          .build();
+    }
+
+    if (null == dnMetricsResponse.getPendingDeletionPerDataNode()) {
+      return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
+          .entity("Metrics data is missing despite FINISHED status.")
+          .type("text/plain")
+          .build();
+    }
+
+    StreamingOutput stream = output -> {
+      CSVFormat format = CSVFormat.DEFAULT.builder()
+          .setHeader("HostName", "Datanode UUID", "Pending Block Size (bytes)").build();
+      try (CSVPrinter csvPrinter = new CSVPrinter(
+          new BufferedWriter(new OutputStreamWriter(output)), format)) {
+        for (DatanodePendingDeletionMetrics metric : dnMetricsResponse.getPendingDeletionPerDataNode()) {
+          csvPrinter.printRecord(
+              metric.getHostName(),
+              metric.getDatanodeUuid(),
+              metric.getPendingBlockSize()
+          );
+        }
+        csvPrinter.flush();
+      } catch (Exception e) {
+        LOG.error("Failed to stream CSV", e);
+        throw new WebApplicationException("Failed to generate CSV", e);
+      }
+    };
+
+    return Response.status(Response.Status.ACCEPTED)
+        .entity(stream)
+        .type("text/csv")
+        .header("Content-Disposition", "attachment; filename=\"pending_deletion_stats.csv\"")
+        .build();
+  }
+
+  private Response handleDataNodeMetrics(int limit) {
+    if (limit < 1) {
+      return Response.status(Response.Status.BAD_REQUEST)
+          .entity("Limit query parameter must be at-least 1").build();
+    }
+    DataNodeMetricsServiceResponse response = dataNodeMetricsService.getCollectedMetrics(limit);
     if (response.getStatus() == DataNodeMetricsService.MetricCollectionStatus.FINISHED) {
       return Response.ok(response).build();
     } else {

hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json

@@ -6866,5 +6866,146 @@
     "selectedRowKeys": [
       "b5907812-a5f2-11ea-bb37-0242ac130011"
     ]
+  },
+  "utilization": {
+    "globalStorage": {
+        "totalUsedSpace": 30679040,
+        "totalFreeSpace": 539776978944,
+        "totalCapacity": 879519597390
+    },
+    "globalNamespace": {
+        "totalUsedSpace": 12349932,
+        "totalKeys": 1576
+    },
+    "usedSpaceBreakdown": {
+        "openKeyBytes": 19255266,
+        "committedKeyBytes": 1249923,
+        "preAllocatedContainerBytes": 1022024
+    },
+    "dataNodeUsage": [
+        {
+            "datanodeUuid": "1bf314dc-3eba-4774-9dbc-6d957cc7670b",
+            "hostName": "ozone-datanode-7.ozone_default",
+            "capacity": 125645656770,
+            "used": 4382720,
+            "remaining": 77110996992,
+            "committed": 0,
+            "minimumFreeSpace": 104857600,
+            "reserved": 12565822
+        },
+        {
+            "datanodeUuid": "8adead67-85a6-4ba9-942f-cd313f1472f9",
+            "hostName": "ozone-datanode-6.ozone_default",
+            "capacity": 125645656770,
+            "used": 4382720,
+            "remaining": 77110996992,
+            "committed": 0,
+            "minimumFreeSpace": 104857600,
+            "reserved": 12565822
+        },
+        {
+            "datanodeUuid": "953b91f4-c03a-45d8-9fbe-142887531cb1",
+            "hostName": "ozone-datanode-5.ozone_default",
+            "capacity": 125645656770,
+            "used": 4382720,
+            "remaining": 77110996992,
+            "committed": 0,
+            "minimumFreeSpace": 104857600,
+            "reserved": 12565822
+        },
+        {
+            "datanodeUuid": "44617070-a300-48af-a0e3-117167b008b6",
+            "hostName": "ozone-datanode-2.ozone_default",
+            "capacity": 125645656770,
+            "used": 4382720,
+            "remaining": 77110996992,
+            "committed": 0,
+            "minimumFreeSpace": 104857600,
+            "reserved": 12565822
+        },
+        {
+            "datanodeUuid": "d3ade292-6ec1-47b2-bd9e-045c1acc2c29",
+            "hostName": "ozone-datanode-1.ozone_default",
+            "capacity": 125645656770,
+            "used": 4382720,
+            "remaining": 77110996992,
+            "committed": 0,
+            "minimumFreeSpace": 104857600,
+            "reserved": 12565822
+        },
+        {
+            "datanodeUuid": "149a1640-e600-4e98-b62d-397804059f0e",
+            "hostName": "ozone-datanode-3.ozone_default",
+            "capacity": 125645656770,
+            "used": 4382720,
+            "remaining": 77110996992,
+            "committed": 0,
+            "minimumFreeSpace": 104857600,
+            "reserved": 12565822
+        },
+        {
+            "datanodeUuid": "17e2562a-dae8-416f-a749-7d4e8a0a781e",
+            "hostName": "ozone-datanode-4.ozone_default",
+            "capacity": 125645656770,
+            "used": 4382720,
+            "remaining": 77110996992,
+            "committed": 0,
+            "minimumFreeSpace": 104857600,
+            "reserved": 12565822
+        }
+    ]
+  },
+  "pendingDeletionDN": {
+    "status": "FINISHED",
+    "totalPendingDeletionSize": 12203,
+    "pendingDeletionPerDataNode": [
+      {
+        "hostName": "ozone-datanode-5.ozone_default",
+        "datanodeUuid": "953b91f4-c03a-45d8-9fbe-142887531cb1",
+        "pendingBlockSize": 1200
+      },
+      {
+        "hostName": "ozone-datanode-3.ozone_default",
+        "datanodeUuid": "149a1640-e600-4e98-b62d-397804059f0e",
+        "pendingBlockSize": 803
+      },
+      {
+        "hostName": "ozone-datanode-4.ozone_default",
+        "datanodeUuid": "17e2562a-dae8-416f-a749-7d4e8a0a781e",
+        "pendingBlockSize": -1
+      },
+      {
+        "hostName": "ozone-datanode-7.ozone_default",
+        "datanodeUuid": "1bf314dc-3eba-4774-9dbc-6d957cc7670b",
+        "pendingBlockSize": 2200
+      },
+      {
+        "hostName": "ozone-datanode-1.ozone_default",
+        "datanodeUuid": "d3ade292-6ec1-47b2-bd9e-045c1acc2c29",
+        "pendingBlockSize": -1
+      },
+      {
+        "hostName": "ozone-datanode-2.ozone_default",
+        "datanodeUuid": "44617070-a300-48af-a0e3-117167b008b6",
+        "pendingBlockSize": 300
+      },
+      {
+        "hostName": "ozone-datanode-6.ozone_default",
+        "datanodeUuid": "8adead67-85a6-4ba9-942f-cd313f1472f9",
+        "pendingBlockSize": 2730
+      }
+    ],
+    "totalNodesQueried": 7,
+    "totalNodeQueriesFailed": 2
+  },
+  "pendingDeletionOM": {
+    "totalSize": 240430,
+    "pendingDirectorySize": 120040,
+    "pendingKeySize": 120390
+  },
+  "pendingDeletionSCM": {
+    "totalBlocksize": 24030,
+    "totalReplicatedBlockSize": 120040,
+    "totalBlocksCount": 120390
   }
 }

hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json

@@ -53,5 +53,10 @@
   "/keys/deletePending/dirs?limit=*": "/dirdeletePending",
   "/datanodes/decommission/info": "/decommissioninfo",
   "/datanodes/decommission/info/datanode?uuid=*": "/DatanodesDecommissionInfo",
-  "/datanodes/remove": "/datanodesRemove"
+  "/datanodes/remove": "/datanodesRemove",
+
+  "/storageDistribution": "/utilization",
+  "/pendingDeletion?component=dn": "/pendingDeletionDN",
+  "/pendingDeletion?component=om": "/pendingDeletionOM",
+  "/pendingDeletion?component=scm": "/pendingDeletionSCM"
 }

hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/utils/themeIcons.tsx

@@ -98,3 +98,30 @@ export class ReplicationIcon extends React.PureComponent<IReplicationIconProps>
     return icon;
   }
 }
+
+interface IGraphLegendIconProps {
+  color: string;
+  height?: number;
+};
+export class GraphLegendIcon extends React.PureComponent<IGraphLegendIconProps> {
+  render() {
+    const { color, height = 14 } = this.props;
+
+    return (
+      <svg
+      width="18"
+      height={height}
+      viewBox={`0 0 18 ${height}`}
+      xmlns="http://www.w3.org/2000/svg"
+      style={{ display: 'inline-block', verticalAlign: 'middle' }} // Optional: helps with alignment
+    >
+      <circle
+        cx="6"
+        cy="6"
+        r="6" 
+        fill={color} // Use the color prop for the fill
+      />
+    </svg>
+    )
+  }
+};