XRP. Added HA setup tests and modified node dashboard

Author: vlasonfa

lib/xrp/README.md

@@ -2,7 +2,7 @@
 
 |          Contributed by          |
 |:--------------------------------:|
-| Pedro Aceves <br/>acevespa@amazon.com |
+| [Pedro Aceves](https://github.com/acevesp)|
 
 XRP node deployment on AWS. All nodes are configure as ["Stock Servers"](https://xrpl.org/docs/infrastructure/configuration/server-modes/run-rippled-as-a-stock-server)
 
@@ -27,7 +27,7 @@ XRP node deployment on AWS. All nodes are configure as ["Stock Servers"](https:/
 ## Well-Architected
 
 <details>
-<summary>Review the for pros and cons of this solution.</summary>
+<summary>Review pros and cons of this solution.</summary>
 
 ### Well-Architected Checklist
 
@@ -50,7 +50,7 @@ This is the Well-Architected checklist for XRP nodes implementation of the AWS B
 |                         | Resource monitoring               | How are workload resources monitored?                                            | Resources are being monitored using Amazon CloudWatch dashboards. Amazon CloudWatch custom metrics are being pushed via CloudWatch Agent.  |
 | Performance efficiency  | Compute selection                 | How is compute solution selected?                                                | Compute solution is selected based on best price-performance.  |
 |                         | Storage selection                 | How is storage solution selected?                                                | Storage solution is selected based on best price-performance. |
-| Operational excellence  | Workload health                   | How is health of workload determined?                                            | Health of workload is determined via AWS Application Load Balancer Target Group Health Checks, on port 8545.  |
+| Operational excellence  | Workload health                   | How is health of workload determined?                                            | Health of workload is determined via AWS Application Load Balancer Target Group Health Checks, on port 6005.  |
 | Sustainability          | Hardware & services               | Select most efficient hardware for your workload                                 | Amazon EC2 R7a instances support the Sustainability Pillar of the AWS Well-Architected Framework by offering memory optimization that enables more efficient resource utilization, potentially reducing overall energy consumption and hardware requirements for data-intensive workloads.  |
 
 </details>
@@ -59,7 +59,7 @@ This is the Well-Architected checklist for XRP nodes implementation of the AWS B
 
 ### Open AWS CloudShell
 
-To begin, ensure you login to your AWS account with permissions to create and modify resources in IAM, EC2, EBS, VPC, S3, KMS, and Secrets Manager.
+To begin, ensure you login to your AWS account with permissions to create and modify resources in IAM, EC2, EBS, VPC, S3, and KMS.
 
 From the AWS Management Console, open the [AWS CloudShell](https://docs.aws.amazon.com/cloudshell/latest/userguide/welcome.html), a web-based shell environment. If unfamiliar, review the [2-minute YouTube video](https://youtu.be/fz4rbjRaiQM) for an overview and check out [CloudShell with VPC environment](https://docs.aws.amazon.com/cloudshell/latest/userguide/creating-vpc-environment.html) that we'll use to test nodes API from internal IP address space.
 
@@ -93,7 +93,7 @@ cd lib/xrp
 cp ./sample-configs/.env-sample-testnet .env
 nano .env
 ```
-> **NOTE:** *You can find more examples inside `sample-configs` *
+> **NOTE:** *You can find more examples inside `sample-configs`*
 
 
 4. Deploy common components such as IAM role:
@@ -194,13 +194,13 @@ pwd
 # Make sure you are in aws-blockchain-node-runners/lib/xrp
 
 # Destroy HA Nodes
-cdk destroy XRP-ha-nodes
+npx cdk destroy XRP-ha-nodes
 
 # Destroy Single Node
-cdk destroy XRP-single-node
+npx cdk destroy XRP-single-node
 
 # Delete all common components like IAM role and Security Group
-cdk destroy XRP-common
+npx cdk destroy XRP-common
 ```
 
 ### FAQ

lib/xrp/lib/assets/user-data/check_xrp_sequence.sh

@@ -26,7 +26,8 @@ set -euo pipefail
 MAX_RETRIES=3
 RETRY_DELAY=5
 NAMESPACE="CWAgent"
-METRIC_NAME="XRP_Sequence"
+CURRENT_METRIC_NAME="XRP_Current_Sequence"
+DELTA_METRIC_NAME="XRP_Delta_Sequence"
 LOCKFILE="/tmp/check_xrp_sequence.lock"
 LOCK_FD=200
 
@@ -115,6 +116,29 @@ get_current_sequence() {
     local retry_count=0
     local seq
 
+    while [[ ${retry_count} -lt ${MAX_RETRIES} ]]; do
+        if seq=$(curl -s -f -H 'Content-Type: application/json' \
+            -d '{"method":"ledger_current","params":[{}]}' \
+            http://localhost:5005/ | \
+            jq -e '.result.ledger_current_index // 0'); then
+            if [[ "${seq}" != "0" ]]; then
+                echo "${seq}"
+                return 0
+            fi
+        fi
+        log_warning "Failed to get sequence, attempt $((retry_count + 1))/${MAX_RETRIES}"
+        retry_count=$((retry_count + 1))
+        sleep ${RETRY_DELAY}
+    done
+
+    log_error "Failed to get current sequence after ${MAX_RETRIES} attempts"
+    return 1
+}
+
+get_validated_sequence() {
+    local retry_count=0
+    local seq
+
     while [[ ${retry_count} -lt ${MAX_RETRIES} ]]; do
         if seq=$(curl -s -f -H 'Content-Type: application/json' \
             -d '{"method":"server_info","params":[{}]}' \
@@ -137,12 +161,13 @@ get_current_sequence() {
 # Function to send metric to CloudWatch with retries
 send_to_cloudwatch() {
     local sequence=$1
+    local metric_name=$2
     local retry_count=0
 
     while [[ ${retry_count} -lt ${MAX_RETRIES} ]]; do
         if aws cloudwatch put-metric-data \
             --namespace "${NAMESPACE}" \
-            --metric-name "${METRIC_NAME}" \
+            --metric-name "${metric_name}" \
             --value "${sequence}" \
             --region "${REGION}" \
             --dimensions "InstanceId=${INSTANCE_ID}" \
@@ -192,14 +217,26 @@ main() {
     fi
 
     # Get current sequence
-    if ! sequence=$(get_current_sequence); then
+    if ! current_sequence=$(get_current_sequence); then
         return 1
     fi
 
-    log_info "Retrieved sequence: ${sequence}"
+        # Get current sequence
+    if ! validated_sequence=$(get_validated_sequence); then
+        return 1
+    fi
+
+    log_info "Retrieved current sequence: ${current_sequence}"
+    log_info "Retrieved validated sequence: ${validated_sequence}"
+
+    # Send to CloudWatch
+    if ! send_to_cloudwatch "${current_sequence}" "${CURRENT_METRIC_NAME}"; then
+        return 1
+    fi
 
     # Send to CloudWatch
-    if ! send_to_cloudwatch "${sequence}"; then
+    delta_sequence=$((current_sequence - validated_sequence))
+    if ! send_to_cloudwatch "${delta_sequence}" "${DELTA_METRIC_NAME}"; then
         return 1
     fi
 

lib/xrp/lib/constructs/node-cw-dashboard.ts

@@ -1,7 +1,7 @@
 export const SingleNodeCWDashboardJSON = {
     "widgets": [
         {
-            "height": 6,
+            "height": 4,
             "width": 8,
             "y": 0,
             "x": 0,
@@ -18,23 +18,23 @@ export const SingleNodeCWDashboardJSON = {
                 },
                 "region": "${REGION}",
                 "metrics": [
-                    [ "AWS/EC2", "CPUUtilization","InstanceId", "${INSTANCE_ID}", {"label": "${INSTANCE_ID}-${INSTANCE_NAME}" } ]
+                    [ "AWS/EC2", "CPUUtilization", "InstanceId", "${INSTANCE_ID}", { "label": "${INSTANCE_ID}-${INSTANCE_NAME}" } ]
                 ],
                 "title": "CPU utilization (%)"
             }
         },
         {
-            "height": 6,
+            "height": 4,
             "width": 8,
             "y": 0,
             "x": 8,
             "type": "metric",
             "properties": {
                 "metrics": [
                     [ { "expression": "m7/PERIOD(m7)", "label": "Read", "id": "e7" } ],
-                    [ "CWAgent", "diskio_reads","InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m7", "visible": false, "stat": "Sum", "period": 60 } ],
+                    [ "CWAgent", "diskio_reads", "InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m7", "visible": false, "stat": "Sum", "period": 60 } ],
                     [ { "expression": "m8/PERIOD(m8)", "label": "Write", "id": "e8" } ],
-                    [ "CWAgent", "diskio_writes","InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m8", "visible": false, "stat": "Sum", "period": 60 } ]
+                    [ "CWAgent", "diskio_writes", "InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m8", "visible": false, "stat": "Sum", "period": 60 } ]
                 ],
                 "view": "timeSeries",
                 "stacked": false,
@@ -45,31 +45,32 @@ export const SingleNodeCWDashboardJSON = {
             }
         },
         {
-            "height": 6,
+            "height": 4,
             "width": 8,
             "y": 0,
             "x": 16,
             "type": "metric",
             "properties": {
+                "metrics": [
+                    [ "CWAgent", "XRP_Current_Sequence", "InstanceId", "${INSTANCE_ID}", { "label": "${INSTANCE_ID}-${INSTANCE_NAME}", "region": "${REGION}" } ]
+                ],
                 "sparkline": false,
-                "view": "singleValue",
+                "view": "timeSeries",
                 "region": "${REGION}",
                 "stacked": false,
                 "singleValueFullPrecision": true,
                 "liveData": true,
                 "setPeriodToTimeRange": false,
                 "trend": true,
-                "metrics": [
-                    [ "CWAgent", "XRP_Sequence","InstanceId", "${INSTANCE_ID}", {"label": "${INSTANCE_ID}-${INSTANCE_NAME}" } ]
-                ],
-                "title": "XRP Sequence"
+                "title": "XRP Current Sequence",
+                "period": 300
             }
         },
         {
-            "height": 6,
+            "height": 4,
             "width": 8,
-            "y": 6,
-            "x": 0,
+            "y": 12,
+            "x": 16,
             "type": "metric",
             "properties": {
                 "view": "timeSeries",
@@ -83,16 +84,16 @@ export const SingleNodeCWDashboardJSON = {
                 },
                 "region": "${REGION}",
                 "metrics": [
-                    [ "AWS/EC2", "NetworkIn","InstanceId", "${INSTANCE_ID}", {"label": "${INSTANCE_ID}-${INSTANCE_NAME}" } ]
+                    [ "AWS/EC2", "NetworkIn", "InstanceId", "${INSTANCE_ID}", { "label": "${INSTANCE_ID}-${INSTANCE_NAME}" } ]
                 ],
                 "title": "Network in (bytes)"
             }
         },
         {
-            "height": 6,
+            "height": 4,
             "width": 8,
-            "y": 6,
-            "x": 8,
+            "y": 4,
+            "x": 0,
             "type": "metric",
             "properties": {
                 "view": "timeSeries",
@@ -101,16 +102,16 @@ export const SingleNodeCWDashboardJSON = {
                 "stat": "Average",
                 "period": 300,
                 "metrics": [
-                    [ "CWAgent", "cpu_usage_iowait","InstanceId", "${INSTANCE_ID}", {"label": "${INSTANCE_ID}-${INSTANCE_NAME}" } ]
+                    [ "CWAgent", "cpu_usage_iowait", "InstanceId", "${INSTANCE_ID}", { "label": "${INSTANCE_ID}-${INSTANCE_NAME}" } ]
                 ],
                 "title": "CPU Usage IO wait (%)"
             }
         },
         {
-            "height": 6,
+            "height": 4,
             "width": 8,
-            "y": 6,
-            "x": 16,
+            "y": 4,
+            "x": 8,
             "type": "metric",
             "properties": {
                 "view": "timeSeries",
@@ -125,20 +126,20 @@ export const SingleNodeCWDashboardJSON = {
                 "region": "${REGION}",
                 "metrics": [
                     [ { "expression": "IF(m7_2 !=0, (m7_1 / m7_2), 0)", "label": "Read", "id": "e7" } ],
-                    [ "CWAgent", "diskio_read_time","InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m7_1", "visible": false, "stat": "Sum", "period": 60 } ],
-                    [ "CWAgent", "diskio_reads","InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m7_2", "visible": false, "stat": "Sum", "period": 60 } ],
+                    [ "CWAgent", "diskio_read_time", "InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m7_1", "visible": false, "stat": "Sum", "period": 60 } ],
+                    [ "CWAgent", "diskio_reads", "InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m7_2", "visible": false, "stat": "Sum", "period": 60 } ],
                     [ { "expression": "IF(m7_4 !=0, (m7_3 / m7_4), 0)", "label": "Write", "id": "e8" } ],
-                    [ "CWAgent", "diskio_write_time","InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m7_3", "visible": false, "stat": "Sum", "period": 60 } ],
-                    [ "CWAgent", "diskio_writes","InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m7_4", "visible": false, "stat": "Sum", "period": 60 } ]
+                    [ "CWAgent", "diskio_write_time", "InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m7_3", "visible": false, "stat": "Sum", "period": 60 } ],
+                    [ "CWAgent", "diskio_writes", "InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m7_4", "visible": false, "stat": "Sum", "period": 60 } ]
                 ],
                 "title": "nvme1n1 Volume Read/Write latency (ms/op)"
             }
         },
         {
-            "height": 6,
+            "height": 4,
             "width": 8,
-            "y": 12,
-            "x": 0,
+            "y": 8,
+            "x": 16,
             "type": "metric",
             "properties": {
                 "view": "timeSeries",
@@ -152,16 +153,16 @@ export const SingleNodeCWDashboardJSON = {
                 },
                 "region": "${REGION}",
                 "metrics": [
-                    [ "AWS/EC2", "NetworkOut","InstanceId", "${INSTANCE_ID}", {"label": "${INSTANCE_ID}-${INSTANCE_NAME}" } ]
+                    [ "AWS/EC2", "NetworkOut", "InstanceId", "${INSTANCE_ID}", { "label": "${INSTANCE_ID}-${INSTANCE_NAME}" } ]
                 ],
                 "title": "Network out (bytes)"
             }
         },
         {
-            "height": 6,
+            "height": 4,
             "width": 8,
-            "y": 12,
-            "x": 8,
+            "y": 8,
+            "x": 0,
             "type": "metric",
             "properties": {
                 "view": "timeSeries",
@@ -170,23 +171,23 @@ export const SingleNodeCWDashboardJSON = {
                 "stat": "Average",
                 "period": 300,
                 "metrics": [
-                    [ "CWAgent", "mem_used_percent","InstanceId", "${INSTANCE_ID}", {"label": "${INSTANCE_ID}-${INSTANCE_NAME}" } ]
+                    [ "CWAgent", "mem_used_percent", "InstanceId", "${INSTANCE_ID}", { "label": "${INSTANCE_ID}-${INSTANCE_NAME}" } ]
                 ],
                 "title": "Mem Used (%)"
             }
         },
         {
-            "height": 6,
+            "height": 4,
             "width": 8,
-            "y": 12,
-            "x": 16,
+            "y": 8,
+            "x": 8,
             "type": "metric",
             "properties": {
                 "metrics": [
-                    [ { "expression": "m2/PERIOD(m2)", "label": "Read", "id": "e2", "period": 60, "region": "us-east-1" } ],
-                    [ "CWAgent", "diskio_read_bytes","InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m2", "stat": "Sum", "visible": false, "period": 60 } ],
-                    [ { "expression": "m3/PERIOD(m3)", "label": "Write", "id": "e3", "period": 60, "region": "us-east-1" } ],
-                    [ "CWAgent", "diskio_write_bytes","InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m3", "stat": "Sum", "visible": false, "period": 60 } ]
+                    [ { "expression": "m2/PERIOD(m2)", "label": "Read", "id": "e2", "period": 60, "region": "${REGION}" } ],
+                    [ "CWAgent", "diskio_read_bytes", "InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m2", "stat": "Sum", "visible": false, "period": 60 } ],
+                    [ { "expression": "m3/PERIOD(m3)", "label": "Write", "id": "e3", "period": 60, "region": "${REGION}" } ],
+                    [ "CWAgent", "diskio_write_bytes", "InstanceId", "${INSTANCE_ID}", "name", "nvme1n1", { "id": "m3", "stat": "Sum", "visible": false, "period": 60 } ]
                 ],
                 "view": "timeSeries",
                 "stacked": false,
@@ -197,22 +198,40 @@ export const SingleNodeCWDashboardJSON = {
             }
         },
         {
-            "height": 6,
+            "height": 4,
             "width": 8,
-            "y": 18,
-            "x": 0,
+            "y": 12,
+            "x": 8,
             "type": "metric",
             "properties": {
                 "metrics": [
-                    [ "CWAgent", "disk_used_percent","InstanceId", "${INSTANCE_ID}", "device", "nvme1n1", "path", "/var/lib/rippled", "fstype", "xfs", { "region": "${REGION}", "label": "/var/lib/rippled" } ]
+                    [ "CWAgent", "disk_used_percent", "InstanceId", "${INSTANCE_ID}", "device", "nvme1n1", "path", "/var/lib/rippled", "fstype", "xfs", { "region": "${REGION}", "label": "/var/lib/rippled" } ]
                 ],
                 "sparkline": true,
                 "view": "singleValue",
                 "region": "${REGION}",
                 "title": "nvme1n1 Disk Used (%)",
                 "period": 60,
-                "stat": "Average"
+                "stat": "Maximum"
+            }
+        },
+        {
+            "type": "metric",
+            "x": 16,
+            "y": 4,
+            "width": 8,
+            "height": 4,
+            "properties": {
+                "metrics": [
+                    [ "CWAgent", "XRP_Delta_Sequence", "InstanceId", "${INSTANCE_ID}", { "region": "${REGION}", "label": "XRP Current - Validated Sequence" } ]
+                ],
+                "view": "timeSeries",
+                "stacked": false,
+                "region": "${REGION}",
+                "period": 300,
+                "stat": "Maximum",
+                "title": "XRP Current - Validated Sequence"
             }
         }
     ]
-}
+}