From 8e4d15fd9c81ebcb51fa3ab15732dc4c514f3c43 Mon Sep 17 00:00:00 2001 From: schmikei Date: Tue, 4 Nov 2025 16:50:00 -0500 Subject: [PATCH 1/3] wip hbase sample application --- sample-apps/apache-hbase/.CI_BYPASS | 0 sample-apps/apache-hbase/Makefile | 50 +++ sample-apps/apache-hbase/README.md | 225 ++++++++++ sample-apps/apache-hbase/cloud-init.yaml | 396 ++++++++++++++++++ .../jinja/templates/cloud-init-template.yaml | 214 ++++++++++ .../install-jmx-prometheus-exporter.sh | 35 ++ .../jinja/templates/scripts/install.sh | 148 +++++++ .../jinja/variables/cloud-init.yaml | 7 + 8 files changed, 1075 insertions(+) create mode 100644 sample-apps/apache-hbase/.CI_BYPASS create mode 100644 sample-apps/apache-hbase/Makefile create mode 100644 sample-apps/apache-hbase/README.md create mode 100644 sample-apps/apache-hbase/cloud-init.yaml create mode 100644 sample-apps/apache-hbase/jinja/templates/cloud-init-template.yaml create mode 100644 sample-apps/apache-hbase/jinja/templates/scripts/install-jmx-prometheus-exporter.sh create mode 100644 sample-apps/apache-hbase/jinja/templates/scripts/install.sh create mode 100644 sample-apps/apache-hbase/jinja/variables/cloud-init.yaml diff --git a/sample-apps/apache-hbase/.CI_BYPASS b/sample-apps/apache-hbase/.CI_BYPASS new file mode 100644 index 00000000..e69de29b diff --git a/sample-apps/apache-hbase/Makefile b/sample-apps/apache-hbase/Makefile new file mode 100644 index 00000000..98c1216d --- /dev/null +++ b/sample-apps/apache-hbase/Makefile @@ -0,0 +1,50 @@ +VM_NAME ?= apache-hbase-sample-app +CONFIG_FILE_DIR := jinja/variables +CONFIG_FILE := $(CONFIG_FILE_DIR)/cloud-init.yaml +LOKI_INSTANCE ?= your-loki-instance:3100 +PROMETHEUS_INSTANCE ?= your-prometheus-instance:9090 + +include ../Makefile + +.PHONY: run run-ci stop exporter fetch-prometheus-metrics render-config launch-vm clean defaultconfig system-status master-metrics regionserver-metrics + +run: launch-vm + @echo "VM $(VM_NAME) launched and configured." + +run-ci: clean defaultconfig launch-vm + @echo "Running in CI mode" + +stop: + @multipass stop $(VM_NAME) --force + @multipass delete $(VM_NAME) + @multipass purge + +render-config: + @docker run --rm -v $(shell pwd)/jinja/templates:/templates -v $(shell pwd)/jinja/variables:/variables dinutac/jinja2docker:latest /templates/cloud-init-template.yaml /variables/cloud-init.yaml --format=yaml > cloud-init.yaml + +launch-vm: render-config + @multipass launch -n $(VM_NAME) --memory 8G --disk 10G --cloud-init cloud-init.yaml + +clean: + @rm -f cloud-init.yaml + @rm -rf $(CONFIG_FILE_DIR) + +defaultconfig: + @mkdir -p $(CONFIG_FILE_DIR) + @echo "interval: \"10s\"" >> jinja/variables/cloud-init.yaml + @echo "loki_url: http://$(LOKI_INSTANCE)/loki/api/v1/push" >> jinja/variables/cloud-init.yaml + @echo "loki_user: your_loki_username" >> jinja/variables/cloud-init.yaml + @echo "loki_pass: your_loki_password" >> jinja/variables/cloud-init.yaml + @echo "prom_url: http://$(PROMETHEUS_INSTANCE)/api/v1/push" >> jinja/variables/cloud-init.yaml + @echo "prom_user: your_prometheus_username" >> jinja/variables/cloud-init.yaml + @echo "prom_pass: your_prometheus_password" >> jinja/variables/cloud-init.yaml + + +system-status: + @multipass exec $(VM_NAME) -- sudo systemctl status hbase + +master-metrics: + @multipass exec $(VM_NAME) -- curl -s http://localhost:8888/metrics || echo "HBase Master metrics not available" + +regionserver-metrics: + @multipass exec $(VM_NAME) -- curl -s http://localhost:8889/metrics || echo "HBase RegionServer metrics not available" diff --git a/sample-apps/apache-hbase/README.md b/sample-apps/apache-hbase/README.md new file mode 100644 index 00000000..6645f6b4 --- /dev/null +++ b/sample-apps/apache-hbase/README.md @@ -0,0 +1,225 @@ +# Apache HBase sample app + +This sample application creates an Ubuntu VM integrated with Alloy for metric and log collection. This sample app utilizes cloud-init and Make commands to facilitate the setup, configuration, and monitoring of [Apache HBase](https://hbase.apache.org/) using the [JMX Prometheus Exporter](https://github.com/prometheus/jmx_exporter). + +## Prerequisites + +Before you begin, ensure you have the following installed: + +- [Multipass](https://multipass.run/) +- Docker (for rendering the cloud-init configuration) +- Git (for cloning the repository) + +## Quick Start for new users + +To get started with the sample app, follow these steps: + +1. **Clone the repository**: + ```sh + git clone https://github.com/grafana/integration-sample-apps.git + cd integration-sample-apps/sample-apps/apache-hbase + ``` + +2. **Set up default config**: + Execute `make defaultconfig` to create a template file with default configuration variables. Modify `jinja/variables/cloud-init.yaml` to connect Alloy to an external Prometheus compatible TSDB and/or Loki server. + +3. **Render cloud-init configuration**: + Run `make render-config` to generate the `cloud-init.yaml` file based on your configuration. + +4. **Create and set up VM**: + Use `make run` to start the Apache HBase sample app. + +5. **Stop and clean Up**: + Use `make stop` to clean up the VM and `make clean` to remove temporary files. + +## Make commands + +- `make defaultconfig`: Initializes the configuration file with default values for cloud-init templates. +- `make render-config`: Generates the `cloud-init.yaml` configuration file using the defined variables. +- `make run`: Creates the Apache HBase sample app VM with HBase installed and configured. +- `make run-ci`: Runs in CI mode by cleaning, setting up default config, and launching the VM. +- `make stop`: Stops and removes the VM, then purges multipass resources. +- `make clean`: Removes generated configuration files and temporary resources. +- `make master-metrics`: Fetches metrics from the HBase Master JMX endpoint. +- `make regionserver-metrics`: Fetches metrics from the HBase RegionServer JMX endpoint. +- `make system-status`: Shows the status of the HBase systemd service. + +## Default configuration variables + +- `prom_pass`: Your Prometheus password. +- `prom_user`: Your Prometheus username. +- `prom_url`: URL for Prometheus push endpoint (e.g., `http://your-prometheus-instance:9090/api/v1/push`). +- `loki_url`: URL for Loki push endpoint (e.g., `http://your-loki-instance:3100/loki/api/v1/push`). +- `loki_user`: Your Loki username. +- `loki_pass`: Your Loki password. +- `interval`: Metrics collection interval (default: `10s`). +- `hbase_version`: Version of Apache HBase to install (default: `2.5.10`). +- `jmx_exporter_version`: Version of JMX Prometheus Exporter to use (default: `1.3.0`). + +You can edit these variables in `jinja/variables/cloud-init.yaml` before rendering the configuration. + +## HBase configuration + +The sample app installs and configures Apache HBase in standalone mode: + +- **HBase Version**: 2.5.10 (configurable via `hbase_version` variable) +- **Java**: OpenJDK 11 +- **Mode**: Standalone (single-node with embedded ZooKeeper) +- **Storage Configuration**: + - Root directory: `/opt/volume/hbase` + - ZooKeeper data: `/opt/volume/hbase/zookeeper` +- **Web UI Ports**: + - Master Web UI: port 16010 + - RegionServer Web UI: port 16030 +- **JMX Monitoring**: + - HBase Master: port 8888 + - HBase RegionServer: port 8889 + - JMX Prometheus Exporter version: 1.3.0 + +## Validating services + +### Alloy +- **Check service status**: Confirm that Alloy is running. + ```bash + systemctl status alloy.service + ``` +- **Review configuration**: Verify the configuration in `/etc/alloy/config.alloy` is correct. +- **Check logs**: Review Alloy logs for any connectivity or configuration issues. + ```bash + journalctl -u alloy.service + ``` + +### Apache HBase +- **Check HBase service status**: Confirm that HBase is running. + ```bash + systemctl status hbase.service + ``` +- **Check HBase shell**: Access the HBase shell to verify it's working. + ```bash + sudo -u hbase /opt/hbase/bin/hbase shell + ``` + Then try commands like: + ``` + status + list + exit + ``` +- **Check Master Web UI**: Access the HBase Master web interface. + ```bash + curl localhost:16010 + ``` +- **Check RegionServer Web UI**: Access the HBase RegionServer web interface. + ```bash + curl localhost:16030 + ``` +- **Check HBase logs**: Review HBase logs for any issues. + ```bash + sudo -u hbase tail -f /opt/hbase/logs/hbase-hbase-master-*.log + sudo -u hbase tail -f /opt/hbase/logs/hbase-hbase-regionserver-*.log + ``` + +### JMX Prometheus Exporter +- **Check Master metrics**: Verify Master metrics are exposed. + ```bash + curl localhost:8888/metrics + ``` +- **Check RegionServer metrics**: Verify RegionServer metrics are exposed. + ```bash + curl localhost:8889/metrics + ``` + +### Load Generator +- **Check load generator status**: Confirm that the load generator is running. + ```bash + systemctl status hbase-loadgen.service + ``` +- **Check load generator logs**: View the load generator activity. + ```bash + journalctl -u hbase-loadgen.service -f + ``` + +## Troubleshooting + +For debugging and troubleshooting, you can access the VM directly using: +```bash +multipass shell apache-hbase-sample-app +``` + +Common troubleshooting steps: + +1. **HBase won't start**: Check Java installation and JAVA_HOME + ```bash + java -version + echo $JAVA_HOME + ``` + +2. **Metrics not available**: Verify JMX exporter is loaded + ```bash + sudo -u hbase ps aux | grep jmx_prometheus + ``` + +3. **Permission issues**: Check file ownership + ```bash + ls -la /opt/hbase + ls -la /opt/volume/hbase + ``` + +4. **Port conflicts**: Verify ports are not in use + ```bash + sudo netstat -tlnp | grep -E '(8888|8889|16010|16030)' + ``` + +## Updating Versions + +### Updating the Apache HBase Version + +To target a different version of HBase, simply replace the `hbase_version` (default=`2.5.10`) within your variables with your desired version. + +### Updating the JMX Prometheus Exporter Version + +To target a different version of the JMX Prometheus Exporter, simply update the `jmx_exporter_version` (default=`1.3.0`) with your desired version. + +Example `jinja/variables/cloud-init.yaml`: +```yaml +interval: "10s" +loki_url: http://your-loki-instance:3100/loki/api/v1/push +loki_user: your_loki_username +loki_pass: your_loki_password +prom_url: http://your-prometheus-instance:9090/api/v1/push +prom_user: your_prometheus_username +prom_pass: your_prometheus_password +hbase_version: 2.5.10 +jmx_exporter_version: 1.3.0 +``` + +## Testing HBase + +Once the VM is running, you can test HBase functionality: + +1. **Access HBase shell**: + ```bash + multipass exec apache-hbase-sample-app -- sudo -u hbase /opt/hbase/bin/hbase shell + ``` + +2. **Create a test table**: + ``` + create 'test', 'cf' + ``` + +3. **Insert data**: + ``` + put 'test', 'row1', 'cf:a', 'value1' + put 'test', 'row2', 'cf:b', 'value2' + ``` + +4. **Scan the table**: + ``` + scan 'test' + ``` + +5. **Exit the shell**: + ``` + exit + ``` + +The included load generator automatically creates a `test_table` and periodically inserts data to generate metrics. diff --git a/sample-apps/apache-hbase/cloud-init.yaml b/sample-apps/apache-hbase/cloud-init.yaml new file mode 100644 index 00000000..d546d4f5 --- /dev/null +++ b/sample-apps/apache-hbase/cloud-init.yaml @@ -0,0 +1,396 @@ +# jinja/templates/cloud-init-template.yaml +# Cloud-init configuration for setting up Alloy and required Apache HBase sample-app + +apt: + sources: + grafana: + source: deb https://apt.grafana.com stable main + keyid: 963FA27710458545 + keyserver: https://apt.grafana.com/gpg.key + +packages: + - git + - gpg + - alloy + +write_files: + # Alloy configuration + - owner: root:root + path: /etc/alloy/config.alloy + content: | + prometheus.exporter.self "alloy_check" { } + + discovery.relabel "alloy_check" { + targets = prometheus.exporter.self.alloy_check.targets + + rule { + target_label = "instance" + replacement = constants.hostname + } + + rule { + target_label = "alloy_hostname" + replacement = constants.hostname + } + + rule { + target_label = "job" + replacement = "integrations/alloy-check" + } + } + + prometheus.scrape "alloy_check" { + targets = discovery.relabel.alloy_check.output + forward_to = [prometheus.relabel.alloy_check.receiver] + + scrape_interval = "60s" + } + + prometheus.relabel "alloy_check" { + forward_to = [prometheus.remote_write.metrics_service.receiver] + + rule { + source_labels = ["__name__"] + regex = "(prometheus_target_sync_length_seconds_sum|prometheus_target_scrapes_.*|prometheus_target_interval.*|prometheus_sd_discovered_targets|alloy_build.*|prometheus_remote_write_wal_samples_appended_total|process_start_time_seconds)" + action = "keep" + } + } + + prometheus.remote_write "metrics_service" { + endpoint { + url = "http://your-prometheus-instance:9090/api/v1/push" + + basic_auth { + username = "your_prometheus_username" + password = "your_prometheus_password" + } + } + } + + loki.write "grafana_cloud_loki" { + endpoint { + url = "http://your-loki-instance:3100/loki/api/v1/push" + + basic_auth { + username = "your_loki_username" + password = "your_loki_password" + } + } + } + + prometheus.scrape "metrics_integrations_integrations_apache_hbase" { + targets = [{ + __address__ = "localhost:8888", + hbase_cluster = "sample_hbase_cluster", + instance = constants.hostname, + },{ + __address__ = "localhost:8889", + hbase_cluster = "sample_hbase_cluster", + instance = constants.hostname, + }] + forward_to = [prometheus.remote_write.metrics_service.receiver] + scrape_interval = "10s" + job_name = "integrations/apache-hbase" + } + + local.file_match "hbase_logs" { + path_targets = [{ + __path__ = "/opt/hbase/logs/*.log", + }] + } + + loki.source.file "hbase_logs" { + targets = local.file_match.hbase_logs.targets + forward_to = [loki.process.hbase_logs.receiver] + } + + loki.process "hbase_logs" { + stage.static_labels { + values = { + job = "integrations/apache-hbase", + hbase_cluster = "sample_hbase_cluster", + instance = constants.hostname, + } + } + + forward_to = [loki.write.grafana_cloud_loki.receiver] + } + + # HBase installation script + - owner: root:root + path: /home/ubuntu/install_hbase.sh + permissions: '0755' + content: | + #!/bin/bash + + + + set -e + + sudo apt-get update + sudo apt-get install -y openjdk-11-jdk-headless + + sudo mkdir -p /opt/hbase/logs \ + /opt/volume/hbase + + echo "Downloading HBase 3.0.0-beta-1..." + HBASE_URL="https://archive.apache.org/dist/hbase/3.0.0-beta-1/hbase-3.0.0-beta-1-bin.tar.gz" + sudo curl -L --progress-bar --retry 3 --retry-delay 5 "$HBASE_URL" -o /home/ubuntu/hbase.tar.gz + + sudo mkdir -p /opt/hbase + sudo tar -xzf /home/ubuntu/hbase.tar.gz -C /opt/hbase --strip 1 + + ARCH=$(uname -m) + if [ "$ARCH" == "x86_64" ]; then + JAVA_HOME="/usr/lib/jvm/java-11-openjdk-amd64" + else + JAVA_HOME="/usr/lib/jvm/java-11-openjdk-arm64" + fi + + sudo cat < /opt/hbase/conf/hbase-site.xml << 'EOF' + + + + + hbase.rootdir + file:///opt/volume/hbase + + + hbase.zookeeper.property.dataDir + /opt/volume/hbase/zookeeper + + + hbase.unsafe.stream.capability.enforce + false + + + hbase.cluster.distributed + true + + + hbase.zookeeper.quorum + localhost + + + hbase.zookeeper.property.clientPort + 2181 + + + EOF + + # Create regionservers file to ensure RegionServer starts on localhost + echo "localhost" | sudo tee /opt/hbase/conf/regionservers > /dev/null + + # Disable SSH requirement by configuring HBase to use local process spawning + sudo cat >> /opt/hbase/conf/hbase-env.sh << 'EOFENV' + + # Use local process spawning instead of SSH for single-node setup + export HBASE_MANAGES_ZK=true + EOFENV + + # Configure HBase environment to use JMX exporter + # Note: JMX exporter should be installed separately via install-jmx-prometheus-exporter.sh + JMX_EXPORTER_VERSION=1.3.0 + JMX_EXPORTER_DIR="/opt/jmx-prometheus-exporter" + JMX_EXPORTER_JAR="$JMX_EXPORTER_DIR/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" + JMX_MASTER_CONFIG="$JMX_EXPORTER_DIR/apache-hbase-master-jmx-config.yml" + JMX_REGIONSERVER_CONFIG="$JMX_EXPORTER_DIR/apache-hbase-regionserver-jmx-config.yml" + JMX_MASTER_PORT=8888 + JMX_REGIONSERVER_PORT=8889 + + sudo cat >> /opt/hbase/conf/hbase-env.sh << EOFENV + + # JMX Prometheus Exporter configuration + # Master JMX exporter endpoint: http://localhost:${JMX_MASTER_PORT}/metrics + export HBASE_MASTER_OPTS="\$HBASE_MASTER_OPTS -javaagent:${JMX_EXPORTER_JAR}=${JMX_MASTER_PORT}:${JMX_MASTER_CONFIG}" + + # RegionServer JMX exporter endpoint: http://localhost:${JMX_REGIONSERVER_PORT}/metrics + export HBASE_REGIONSERVER_OPTS="\$HBASE_REGIONSERVER_OPTS -javaagent:${JMX_EXPORTER_JAR}=${JMX_REGIONSERVER_PORT}:${JMX_REGIONSERVER_CONFIG}" + EOFENV + + # Create hbase user + sudo useradd -r -s /bin/bash -d /opt/hbase hbase || true + sudo chown -R hbase:hbase /opt/hbase + sudo chown -R hbase:hbase /opt/volume/hbase + # Ensure logs directory is writable for PID files + sudo chmod 755 /opt/hbase/logs + + # Create systemd service for HBase (starts ZooKeeper, Master, and RegionServer) + sudo tee /etc/systemd/system/hbase.service > /dev/null << EOF + [Unit] + Description=Apache HBase + After=network.target + + [Service] + Type=forking + User=hbase + Group=hbase + EnvironmentFile=/opt/hbase/hbase.env + Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${JAVA_HOME}/bin:/opt/hbase/bin" + ExecStart=/bin/bash -c 'source /opt/hbase/env && /opt/hbase/bin/hbase-daemon.sh start zookeeper && sleep 5 && /opt/hbase/bin/start-hbase.sh' + ExecStop=/bin/bash -c 'source /opt/hbase/env && /opt/hbase/bin/stop-hbase.sh && /opt/hbase/bin/hbase-daemon.sh stop zookeeper' + Restart=on-failure + RestartSec=10 + + [Install] + WantedBy=multi-user.target + EOF + + # Reload systemd and enable HBase service + sudo systemctl daemon-reload + sudo systemctl enable hbase.service + + # Start HBase (don't fail if it doesn't start immediately - it may take time) + sudo systemctl start hbase.service || echo "HBase service start returned non-zero, but continuing..." + + # Wait a moment for HBase to initialize + sleep 5 + + # Check if HBase is actually running + if sudo systemctl is-active --quiet hbase.service; then + echo "HBase service is running successfully!" + else + echo "WARNING: HBase service may not be fully started yet. Check logs with: sudo journalctl -u hbase.service" + fi + + echo "HBase installation completed!" + echo "HBase Master Web UI: http://localhost:16010" + echo "HBase Master JMX Metrics: http://localhost:8888/metrics" + echo "HBase RegionServer Web UI: http://localhost:16030" + echo "HBase RegionServer JMX Metrics: http://localhost:8889/metrics" + + + # JMX Prometheus Exporter installation script + - owner: root:root + path: /home/ubuntu/install-jmx-prometheus-exporter.sh + permissions: '0755' + content: | + #!/bin/bash + + + + # JMX Prometheus Exporter configuration + JMX_EXPORTER_VERSION=1.3.0 + JMX_EXPORTER_DIR="/opt/jmx-prometheus-exporter" + JMX_EXPORTER_JAR="$JMX_EXPORTER_DIR/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" + JMX_MASTER_CONFIG_FILE="$JMX_EXPORTER_DIR/apache-hbase-master-jmx-config.yml" + JMX_REGIONSERVER_CONFIG_FILE="$JMX_EXPORTER_DIR/apache-hbase-regionserver-jmx-config.yml" + + sudo mkdir -p "$JMX_EXPORTER_DIR" + + echo "Downloading JMX Prometheus Java Agent version $JMX_EXPORTER_VERSION..." + sudo wget -O "$JMX_EXPORTER_JAR" \ + "https://github.com/prometheus/jmx_exporter/releases/download/${JMX_EXPORTER_VERSION}/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" + + # Create JMX exporter config for HBase Master + sudo cat > "$JMX_MASTER_CONFIG_FILE" <<'EOF' + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + - pattern: ".*" + EOF + + # Create JMX exporter config for HBase RegionServer + sudo cat > "$JMX_REGIONSERVER_CONFIG_FILE" <<'EOF' + lowercaseOutputName: true + lowercaseOutputLabelNames: true + rules: + - pattern: ".*" + EOF + + echo "JMX Prometheus Exporter installation completed!" + echo "JAR location: $JMX_EXPORTER_JAR" + echo "Master config: $JMX_MASTER_CONFIG_FILE" + echo "RegionServer config: $JMX_REGIONSERVER_CONFIG_FILE" + + + # Load generator script + - owner: root:root + path: /home/ubuntu/loadgen.sh + permissions: '0755' + content: | + #!/bin/bash + + # Wait for HBase to be fully ready + sleep 60 + + # Create a test table and insert data periodically + while true; do + echo "Creating test table and inserting data..." + /opt/hbase/bin/hbase shell << 'EOFHBASE' || true + + # Create table if it doesn't exist + if !exists('test_table') + create 'test_table', 'cf' + end + + # Insert some test data + put 'test_table', 'row1', 'cf:col1', 'value1' + put 'test_table', 'row2', 'cf:col1', 'value2' + put 'test_table', 'row3', 'cf:col1', 'value3' + + # Scan table + scan 'test_table', {LIMIT => 5} + + exit + EOFHBASE + + echo "Load generation cycle complete. Sleeping for 60 seconds..." + sleep 60 + done + + # Systemd service for load generator + - owner: root:root + path: /etc/systemd/system/hbase-loadgen.service + content: | + [Unit] + Description=HBase Load Generator + After=hbase.service + Requires=hbase.service + + [Service] + Type=simple + User=root + WorkingDirectory=/home/ubuntu + ExecStart=/home/ubuntu/loadgen.sh + Restart=always + RestartSec=10 + + [Install] + WantedBy=multi-user.target + +runcmd: + # General setup + - sudo apt-get update + - sudo chmod +x /home/ubuntu/install_hbase.sh + - sudo chmod +x /home/ubuntu/install-jmx-prometheus-exporter.sh + + # Install JMX Prometheus Exporter (must be done before HBase installation) + - sudo /home/ubuntu/install-jmx-prometheus-exporter.sh > /home/ubuntu/install-jmx-prometheus-exporter.log 2>&1 + + # Install HBase (redirect output to log file for debugging) + - nohup sudo /home/ubuntu/install_hbase.sh > /home/ubuntu/hbase-install.log 2>&1 || echo "HBase installation script completed with warnings or errors, check /home/ubuntu/hbase-install.log" & + + # Configure Alloy + - sudo systemctl enable alloy.service + - sudo systemctl start alloy.service + + # Enable and start load generator + - sudo systemctl enable hbase-loadgen.service + - sudo systemctl start hbase-loadgen.service diff --git a/sample-apps/apache-hbase/jinja/templates/cloud-init-template.yaml b/sample-apps/apache-hbase/jinja/templates/cloud-init-template.yaml new file mode 100644 index 00000000..b1d09fdd --- /dev/null +++ b/sample-apps/apache-hbase/jinja/templates/cloud-init-template.yaml @@ -0,0 +1,214 @@ +# jinja/templates/cloud-init-template.yaml +# Cloud-init configuration for setting up Alloy and required Apache HBase sample-app + +apt: + sources: + grafana: + source: deb https://apt.grafana.com stable main + keyid: 963FA27710458545 + keyserver: https://apt.grafana.com/gpg.key + +packages: + - git + - gpg + - alloy + +write_files: + # Alloy configuration + - owner: root:root + path: /etc/alloy/config.alloy + content: | + prometheus.exporter.self "alloy_check" { } + + discovery.relabel "alloy_check" { + targets = prometheus.exporter.self.alloy_check.targets + + rule { + target_label = "instance" + replacement = constants.hostname + } + + rule { + target_label = "alloy_hostname" + replacement = constants.hostname + } + + rule { + target_label = "job" + replacement = "integrations/alloy-check" + } + } + + prometheus.scrape "alloy_check" { + targets = discovery.relabel.alloy_check.output + forward_to = [prometheus.relabel.alloy_check.receiver] + + scrape_interval = "60s" + } + + prometheus.relabel "alloy_check" { + forward_to = [prometheus.remote_write.metrics_service.receiver] + + rule { + source_labels = ["__name__"] + regex = "(prometheus_target_sync_length_seconds_sum|prometheus_target_scrapes_.*|prometheus_target_interval.*|prometheus_sd_discovered_targets|alloy_build.*|prometheus_remote_write_wal_samples_appended_total|process_start_time_seconds)" + action = "keep" + } + } + + prometheus.remote_write "metrics_service" { + endpoint { + url = "{{ prom_url }}" + + {% if prom_user and prom_pass -%} + basic_auth { + username = "{{ prom_user }}" + password = "{{ prom_pass }}" + } + {%- endif %} + } + } + + loki.write "grafana_cloud_loki" { + endpoint { + url = "{{ loki_url }}" + + {% if loki_user and loki_pass -%} + basic_auth { + username = "{{ loki_user }}" + password = "{{ loki_pass }}" + } + {%- endif %} + } + } + + prometheus.scrape "metrics_integrations_integrations_apache_hbase" { + targets = [{ + __address__ = "localhost:8888", + hbase_cluster = "sample_hbase_cluster", + instance = constants.hostname, + },{ + __address__ = "localhost:8889", + hbase_cluster = "sample_hbase_cluster", + instance = constants.hostname, + }] + forward_to = [prometheus.remote_write.metrics_service.receiver] + scrape_interval = "{{ interval }}" + job_name = "integrations/apache-hbase" + } + + local.file_match "hbase_logs" { + path_targets = [{ + __path__ = "/opt/hbase/logs/*.log", + }] + } + + loki.source.file "hbase_logs" { + targets = local.file_match.hbase_logs.targets + forward_to = [loki.process.hbase_logs.receiver] + } + + loki.process "hbase_logs" { + stage.static_labels { + values = { + job = "integrations/apache-hbase", + hbase_cluster = "sample_hbase_cluster", + instance = constants.hostname, + } + } + + forward_to = [loki.write.grafana_cloud_loki.receiver] + } + + # HBase installation script + - owner: root:root + path: /home/ubuntu/install_hbase.sh + permissions: '0755' + content: | + {% filter indent(6) %} + {%- include 'scripts/install.sh' -%} + {% endfilter %} + + # JMX Prometheus Exporter installation script + - owner: root:root + path: /home/ubuntu/install-jmx-prometheus-exporter.sh + permissions: '0755' + content: | + {% filter indent(6) %} + {%- include 'scripts/install-jmx-prometheus-exporter.sh' -%} + {% endfilter %} + + # Load generator script + - owner: root:root + path: /home/ubuntu/loadgen.sh + permissions: '0755' + content: | + #!/bin/bash + + # Wait for HBase to be fully ready + sleep 60 + + # Create a test table and insert data periodically + while true; do + echo "Creating test table and inserting data..." + /opt/hbase/bin/hbase shell << 'EOFHBASE' || true + + # Create table if it doesn't exist + if !exists('test_table') + create 'test_table', 'cf' + end + + # Insert some test data + put 'test_table', 'row1', 'cf:col1', 'value1' + put 'test_table', 'row2', 'cf:col1', 'value2' + put 'test_table', 'row3', 'cf:col1', 'value3' + + # Scan table + scan 'test_table', {LIMIT => 5} + + exit + EOFHBASE + + echo "Load generation cycle complete. Sleeping for 60 seconds..." + sleep 60 + done + + # Systemd service for load generator + - owner: root:root + path: /etc/systemd/system/hbase-loadgen.service + content: | + [Unit] + Description=HBase Load Generator + After=hbase.service + Requires=hbase.service + + [Service] + Type=simple + User=root + WorkingDirectory=/home/ubuntu + ExecStart=/home/ubuntu/loadgen.sh + Restart=always + RestartSec=10 + + [Install] + WantedBy=multi-user.target + +runcmd: + # General setup + - sudo apt-get update + - sudo chmod +x /home/ubuntu/install_hbase.sh + - sudo chmod +x /home/ubuntu/install-jmx-prometheus-exporter.sh + + # Install JMX Prometheus Exporter (must be done before HBase installation) + - sudo /home/ubuntu/install-jmx-prometheus-exporter.sh > /home/ubuntu/install-jmx-prometheus-exporter.log 2>&1 + + # Install HBase (redirect output to log file for debugging) + - nohup sudo /home/ubuntu/install_hbase.sh > /home/ubuntu/hbase-install.log 2>&1 || echo "HBase installation script completed with warnings or errors, check /home/ubuntu/hbase-install.log" & + + # Configure Alloy + - sudo systemctl enable alloy.service + - sudo systemctl start alloy.service + + # Enable and start load generator + - sudo systemctl enable hbase-loadgen.service + - sudo systemctl start hbase-loadgen.service diff --git a/sample-apps/apache-hbase/jinja/templates/scripts/install-jmx-prometheus-exporter.sh b/sample-apps/apache-hbase/jinja/templates/scripts/install-jmx-prometheus-exporter.sh new file mode 100644 index 00000000..6cff64a0 --- /dev/null +++ b/sample-apps/apache-hbase/jinja/templates/scripts/install-jmx-prometheus-exporter.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +{% set jmx_exporter_version = jmx_exporter_version | default("1.3.0") %} + +# JMX Prometheus Exporter configuration +JMX_EXPORTER_VERSION={{ jmx_exporter_version }} +JMX_EXPORTER_DIR="/opt/jmx-prometheus-exporter" +JMX_EXPORTER_JAR="$JMX_EXPORTER_DIR/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" +JMX_MASTER_CONFIG_FILE="$JMX_EXPORTER_DIR/apache-hbase-master-jmx-config.yml" +JMX_REGIONSERVER_CONFIG_FILE="$JMX_EXPORTER_DIR/apache-hbase-regionserver-jmx-config.yml" + +sudo mkdir -p "$JMX_EXPORTER_DIR" + +echo "Downloading JMX Prometheus Java Agent version $JMX_EXPORTER_VERSION..." +sudo wget -O "$JMX_EXPORTER_JAR" \ + "https://github.com/prometheus/jmx_exporter/releases/download/${JMX_EXPORTER_VERSION}/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" + +sudo cat > "$JMX_MASTER_CONFIG_FILE" <<'EOF' +lowercaseOutputName: true +lowercaseOutputLabelNames: true +rules: + - pattern: ".*" +EOF + +sudo cat > "$JMX_REGIONSERVER_CONFIG_FILE" <<'EOF' +lowercaseOutputName: true +lowercaseOutputLabelNames: true +rules: + - pattern: ".*" +EOF + +echo "JMX Prometheus Exporter installation completed!" +echo "JAR location: $JMX_EXPORTER_JAR" +echo "Master config: $JMX_MASTER_CONFIG_FILE" +echo "RegionServer config: $JMX_REGIONSERVER_CONFIG_FILE" diff --git a/sample-apps/apache-hbase/jinja/templates/scripts/install.sh b/sample-apps/apache-hbase/jinja/templates/scripts/install.sh new file mode 100644 index 00000000..420e6e71 --- /dev/null +++ b/sample-apps/apache-hbase/jinja/templates/scripts/install.sh @@ -0,0 +1,148 @@ +#!/bin/bash + +# Integration only claims support for HBase 3.0+ +{% set hbase_version = hbase_version | default("3.0.0-beta-1") %} + +set -e + +sudo apt-get update +sudo apt-get install -y openjdk-11-jdk-headless + +sudo mkdir -p /opt/hbase/logs \ + /opt/volume/hbase + +echo "Downloading HBase {{ hbase_version }}..." +HBASE_URL="https://archive.apache.org/dist/hbase/{{ hbase_version }}/hbase-{{ hbase_version }}-bin.tar.gz" +sudo curl -L --progress-bar --retry 3 --retry-delay 5 "$HBASE_URL" -o /home/ubuntu/hbase.tar.gz + +sudo mkdir -p /opt/hbase +sudo tar -xzf /home/ubuntu/hbase.tar.gz -C /opt/hbase --strip 1 + +ARCH=$(uname -m) +if [ "$ARCH" == "x86_64" ]; then + JAVA_HOME="/usr/lib/jvm/java-11-openjdk-amd64" +else + JAVA_HOME="/usr/lib/jvm/java-11-openjdk-arm64" +fi + +sudo cat < /opt/hbase/conf/hbase-site.xml << 'EOF' + + + + + hbase.rootdir + file:///opt/volume/hbase + + + hbase.zookeeper.property.dataDir + /opt/volume/hbase/zookeeper + + + hbase.unsafe.stream.capability.enforce + false + + + hbase.cluster.distributed + true + + + hbase.zookeeper.quorum + localhost + + + hbase.zookeeper.property.clientPort + 2181 + + +EOF + +# Create regionservers file to ensure RegionServer starts on localhost +echo "localhost" | sudo tee /opt/hbase/conf/regionservers > /dev/null + +# Disable SSH requirement by configuring HBase to use local process spawning +sudo cat >> /opt/hbase/conf/hbase-env.sh << 'EOFENV' + +# Use local process spawning instead of SSH for single-node setup +export HBASE_MANAGES_ZK=true +EOFENV + +# Configure HBase environment to use JMX exporter +# Note: JMX exporter should be installed separately via install-jmx-prometheus-exporter.sh +JMX_EXPORTER_VERSION={{ jmx_exporter_version | default("1.3.0") }} +JMX_EXPORTER_DIR="/opt/jmx-prometheus-exporter" +JMX_EXPORTER_JAR="$JMX_EXPORTER_DIR/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" +JMX_MASTER_CONFIG="$JMX_EXPORTER_DIR/apache-hbase-master-jmx-config.yml" +JMX_REGIONSERVER_CONFIG="$JMX_EXPORTER_DIR/apache-hbase-regionserver-jmx-config.yml" +JMX_MASTER_PORT=8888 +JMX_REGIONSERVER_PORT=8889 + +sudo cat >> /opt/hbase/conf/hbase-env.sh << EOFENV + +# JMX Prometheus Exporter configuration +# Master JMX exporter endpoint: http://localhost:${JMX_MASTER_PORT}/metrics +export HBASE_MASTER_OPTS="\$HBASE_MASTER_OPTS -javaagent:${JMX_EXPORTER_JAR}=${JMX_MASTER_PORT}:${JMX_MASTER_CONFIG}" + +# RegionServer JMX exporter endpoint: http://localhost:${JMX_REGIONSERVER_PORT}/metrics +export HBASE_REGIONSERVER_OPTS="\$HBASE_REGIONSERVER_OPTS -javaagent:${JMX_EXPORTER_JAR}=${JMX_REGIONSERVER_PORT}:${JMX_REGIONSERVER_CONFIG}" +EOFENV + +# Create hbase user +sudo useradd -r -s /bin/bash -d /opt/hbase hbase || true +sudo chown -R hbase:hbase /opt/hbase +sudo chown -R hbase:hbase /opt/volume/hbase +# Ensure logs directory is writable for PID files +sudo chmod 755 /opt/hbase/logs + +# Create systemd service for HBase (starts ZooKeeper, Master, and RegionServer) +sudo tee /etc/systemd/system/hbase.service > /dev/null << EOF +[Unit] +Description=Apache HBase +After=network.target + +[Service] +Type=forking +User=hbase +Group=hbase +EnvironmentFile=/opt/hbase/hbase.env +Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${JAVA_HOME}/bin:/opt/hbase/bin" +ExecStart=/bin/bash -c 'source /opt/hbase/env && /opt/hbase/bin/hbase-daemon.sh start zookeeper && sleep 5 && /opt/hbase/bin/start-hbase.sh' +ExecStop=/bin/bash -c 'source /opt/hbase/env && /opt/hbase/bin/stop-hbase.sh && /opt/hbase/bin/hbase-daemon.sh stop zookeeper' +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target +EOF +sudo systemctl daemon-reload +sudo systemctl enable hbase.service + +sudo systemctl start hbase.service || echo "HBase service start returned non-zero, but continuing..." + +# Wait a moment for HBase to initialize +sleep 5 + +if sudo systemctl is-active --quiet hbase.service; then + echo "HBase service is running successfully!" +else + echo "WARNING: HBase service may not be fully started yet. Check logs with: sudo journalctl -u hbase.service" +fi + +echo "HBase installation completed!" +echo "HBase Master Web UI: http://localhost:16010" +echo "HBase Master JMX Metrics: http://localhost:8888/metrics" +echo "HBase RegionServer Web UI: http://localhost:16030" +echo "HBase RegionServer JMX Metrics: http://localhost:8889/metrics" diff --git a/sample-apps/apache-hbase/jinja/variables/cloud-init.yaml b/sample-apps/apache-hbase/jinja/variables/cloud-init.yaml new file mode 100644 index 00000000..2c9f313f --- /dev/null +++ b/sample-apps/apache-hbase/jinja/variables/cloud-init.yaml @@ -0,0 +1,7 @@ +interval: "10s" +loki_url: http://your-loki-instance:3100/loki/api/v1/push +loki_user: your_loki_username +loki_pass: your_loki_password +prom_url: http://your-prometheus-instance:9090/api/v1/push +prom_user: your_prometheus_username +prom_pass: your_prometheus_password From 943aefe0ae32855fb6d0ea4d13f74c513b6149f2 Mon Sep 17 00:00:00 2001 From: schmikei Date: Thu, 6 Nov 2025 10:31:47 -0500 Subject: [PATCH 2/3] update to use /prometheus endpoint --- sample-apps/apache-hbase/Makefile | 3 + sample-apps/apache-hbase/README.md | 40 ++++--- sample-apps/apache-hbase/cloud-init.yaml | 103 +++--------------- .../jinja/templates/cloud-init-template.yaml | 38 +++---- .../install-jmx-prometheus-exporter.sh | 35 ------ .../jinja/templates/scripts/install.sh | 19 ---- .../jinja/variables/cloud-init.yaml | 4 +- 7 files changed, 55 insertions(+), 187 deletions(-) delete mode 100644 sample-apps/apache-hbase/jinja/templates/scripts/install-jmx-prometheus-exporter.sh diff --git a/sample-apps/apache-hbase/Makefile b/sample-apps/apache-hbase/Makefile index 98c1216d..e4235e9a 100644 --- a/sample-apps/apache-hbase/Makefile +++ b/sample-apps/apache-hbase/Makefile @@ -48,3 +48,6 @@ master-metrics: regionserver-metrics: @multipass exec $(VM_NAME) -- curl -s http://localhost:8889/metrics || echo "HBase RegionServer metrics not available" + +tail-install: + @multipass exec $(VM_NAME) -- tail -f /home/ubuntu/hbase-install.log \ No newline at end of file diff --git a/sample-apps/apache-hbase/README.md b/sample-apps/apache-hbase/README.md index 6645f6b4..42717129 100644 --- a/sample-apps/apache-hbase/README.md +++ b/sample-apps/apache-hbase/README.md @@ -1,6 +1,8 @@ # Apache HBase sample app -This sample application creates an Ubuntu VM integrated with Alloy for metric and log collection. This sample app utilizes cloud-init and Make commands to facilitate the setup, configuration, and monitoring of [Apache HBase](https://hbase.apache.org/) using the [JMX Prometheus Exporter](https://github.com/prometheus/jmx_exporter). +> Note: this sample application takes a really long time to setup as the install for HBase takes near 20 minutes on good days. Feel free to run a `make tail-install` to track install progress. The install is triggered in the background so expect a long time for the metrics to show up. + +This sample application creates an Ubuntu VM integrated with Alloy for metric and log collection. It uses cloud-init and Make commands to ease setup, configuration, and monitoring of [Apache HBase](https://hbase.apache.org/), collecting HBase metrics and logs natively from HBase’s built-in `/prometheus` endpoints. ## Prerequisites @@ -40,8 +42,8 @@ To get started with the sample app, follow these steps: - `make run-ci`: Runs in CI mode by cleaning, setting up default config, and launching the VM. - `make stop`: Stops and removes the VM, then purges multipass resources. - `make clean`: Removes generated configuration files and temporary resources. -- `make master-metrics`: Fetches metrics from the HBase Master JMX endpoint. -- `make regionserver-metrics`: Fetches metrics from the HBase RegionServer JMX endpoint. +- `make master-metrics`: Fetches metrics from the HBase Master's `/prometheus` endpoint. +- `make regionserver-metrics`: Fetches metrics from the HBase RegionServer's `/prometheus` endpoint. - `make system-status`: Shows the status of the HBase systemd service. ## Default configuration variables @@ -54,7 +56,6 @@ To get started with the sample app, follow these steps: - `loki_pass`: Your Loki password. - `interval`: Metrics collection interval (default: `10s`). - `hbase_version`: Version of Apache HBase to install (default: `2.5.10`). -- `jmx_exporter_version`: Version of JMX Prometheus Exporter to use (default: `1.3.0`). You can edit these variables in `jinja/variables/cloud-init.yaml` before rendering the configuration. @@ -71,10 +72,11 @@ The sample app installs and configures Apache HBase in standalone mode: - **Web UI Ports**: - Master Web UI: port 16010 - RegionServer Web UI: port 16030 -- **JMX Monitoring**: - - HBase Master: port 8888 - - HBase RegionServer: port 8889 - - JMX Prometheus Exporter version: 1.3.0 +- **Prometheus Metrics Endpoints**: + - HBase Master: `http://localhost:16010/prometheus` + - HBase RegionServer: `http://localhost:16030/prometheus` + +No JMX exporter or sidecar is needed: Alloy natively scrapes metrics from these `/prometheus` endpoints exposed by HBase. ## Validating services @@ -118,14 +120,14 @@ The sample app installs and configures Apache HBase in standalone mode: sudo -u hbase tail -f /opt/hbase/logs/hbase-hbase-regionserver-*.log ``` -### JMX Prometheus Exporter -- **Check Master metrics**: Verify Master metrics are exposed. +### Metrics endpoints +- **Check Master metrics**: Verify metrics are exposed by HBase Master at the `/prometheus` endpoint. ```bash - curl localhost:8888/metrics + curl localhost:16010/prometheus ``` -- **Check RegionServer metrics**: Verify RegionServer metrics are exposed. +- **Check RegionServer metrics**: Verify metrics are exposed by the RegionServer at the `/prometheus` endpoint. ```bash - curl localhost:8889/metrics + curl localhost:16030/prometheus ``` ### Load Generator @@ -153,9 +155,10 @@ Common troubleshooting steps: echo $JAVA_HOME ``` -2. **Metrics not available**: Verify JMX exporter is loaded +2. **Metrics not available**: Ensure the `/prometheus` endpoints are enabled and reachable. Try: ```bash - sudo -u hbase ps aux | grep jmx_prometheus + curl localhost:16010/prometheus + curl localhost:16030/prometheus ``` 3. **Permission issues**: Check file ownership @@ -166,7 +169,7 @@ Common troubleshooting steps: 4. **Port conflicts**: Verify ports are not in use ```bash - sudo netstat -tlnp | grep -E '(8888|8889|16010|16030)' + sudo netstat -tlnp | grep -E '(16010|16030)' ``` ## Updating Versions @@ -175,10 +178,6 @@ Common troubleshooting steps: To target a different version of HBase, simply replace the `hbase_version` (default=`2.5.10`) within your variables with your desired version. -### Updating the JMX Prometheus Exporter Version - -To target a different version of the JMX Prometheus Exporter, simply update the `jmx_exporter_version` (default=`1.3.0`) with your desired version. - Example `jinja/variables/cloud-init.yaml`: ```yaml interval: "10s" @@ -189,7 +188,6 @@ prom_url: http://your-prometheus-instance:9090/api/v1/push prom_user: your_prometheus_username prom_pass: your_prometheus_password hbase_version: 2.5.10 -jmx_exporter_version: 1.3.0 ``` ## Testing HBase diff --git a/sample-apps/apache-hbase/cloud-init.yaml b/sample-apps/apache-hbase/cloud-init.yaml index d546d4f5..bfdf0611 100644 --- a/sample-apps/apache-hbase/cloud-init.yaml +++ b/sample-apps/apache-hbase/cloud-init.yaml @@ -58,8 +58,7 @@ write_files: prometheus.remote_write "metrics_service" { endpoint { - url = "http://your-prometheus-instance:9090/api/v1/push" - + url = "http://192.168.64.3:9009/api/v1/push" basic_auth { username = "your_prometheus_username" password = "your_prometheus_password" @@ -69,8 +68,7 @@ write_files: loki.write "grafana_cloud_loki" { endpoint { - url = "http://your-loki-instance:3100/loki/api/v1/push" - + url = "http://192.168.64.3:3100/loki/api/v1/push" basic_auth { username = "your_loki_username" password = "your_loki_password" @@ -79,15 +77,20 @@ write_files: } prometheus.scrape "metrics_integrations_integrations_apache_hbase" { - targets = [{ - __address__ = "localhost:8888", - hbase_cluster = "sample_hbase_cluster", - instance = constants.hostname, - },{ - __address__ = "localhost:8889", - hbase_cluster = "sample_hbase_cluster", - instance = constants.hostname, - }] + targets = [ + { + __address__ = "localhost:16010", + metrics_path = "/prometheus", + hbase_cluster = "sample_hbase_cluster", + instance = constants.hostname, + }, + { + __address__ = "localhost:16030", + metrics_path = "/prometheus", + hbase_cluster = "sample_hbase_cluster", + instance = constants.hostname, + }, + ] forward_to = [prometheus.remote_write.metrics_service.receiver] scrape_interval = "10s" job_name = "integrations/apache-hbase" @@ -123,6 +126,7 @@ write_files: content: | #!/bin/bash + # Integration only claims support for HBase 3.0+ set -e @@ -154,7 +158,6 @@ write_files: export PATH=$PATH:/opt/hbase/bin EOF - # Create systemd-compatible environment file (without export statements) sudo cat < /opt/hbase/conf/hbase-site.xml << 'EOF' @@ -204,25 +206,6 @@ write_files: export HBASE_MANAGES_ZK=true EOFENV - # Configure HBase environment to use JMX exporter - # Note: JMX exporter should be installed separately via install-jmx-prometheus-exporter.sh - JMX_EXPORTER_VERSION=1.3.0 - JMX_EXPORTER_DIR="/opt/jmx-prometheus-exporter" - JMX_EXPORTER_JAR="$JMX_EXPORTER_DIR/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" - JMX_MASTER_CONFIG="$JMX_EXPORTER_DIR/apache-hbase-master-jmx-config.yml" - JMX_REGIONSERVER_CONFIG="$JMX_EXPORTER_DIR/apache-hbase-regionserver-jmx-config.yml" - JMX_MASTER_PORT=8888 - JMX_REGIONSERVER_PORT=8889 - - sudo cat >> /opt/hbase/conf/hbase-env.sh << EOFENV - - # JMX Prometheus Exporter configuration - # Master JMX exporter endpoint: http://localhost:${JMX_MASTER_PORT}/metrics - export HBASE_MASTER_OPTS="\$HBASE_MASTER_OPTS -javaagent:${JMX_EXPORTER_JAR}=${JMX_MASTER_PORT}:${JMX_MASTER_CONFIG}" - - # RegionServer JMX exporter endpoint: http://localhost:${JMX_REGIONSERVER_PORT}/metrics - export HBASE_REGIONSERVER_OPTS="\$HBASE_REGIONSERVER_OPTS -javaagent:${JMX_EXPORTER_JAR}=${JMX_REGIONSERVER_PORT}:${JMX_REGIONSERVER_CONFIG}" - EOFENV # Create hbase user sudo useradd -r -s /bin/bash -d /opt/hbase hbase || true @@ -251,18 +234,14 @@ write_files: [Install] WantedBy=multi-user.target EOF - - # Reload systemd and enable HBase service sudo systemctl daemon-reload sudo systemctl enable hbase.service - # Start HBase (don't fail if it doesn't start immediately - it may take time) sudo systemctl start hbase.service || echo "HBase service start returned non-zero, but continuing..." # Wait a moment for HBase to initialize sleep 5 - # Check if HBase is actually running if sudo systemctl is-active --quiet hbase.service; then echo "HBase service is running successfully!" else @@ -276,50 +255,6 @@ write_files: echo "HBase RegionServer JMX Metrics: http://localhost:8889/metrics" - # JMX Prometheus Exporter installation script - - owner: root:root - path: /home/ubuntu/install-jmx-prometheus-exporter.sh - permissions: '0755' - content: | - #!/bin/bash - - - - # JMX Prometheus Exporter configuration - JMX_EXPORTER_VERSION=1.3.0 - JMX_EXPORTER_DIR="/opt/jmx-prometheus-exporter" - JMX_EXPORTER_JAR="$JMX_EXPORTER_DIR/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" - JMX_MASTER_CONFIG_FILE="$JMX_EXPORTER_DIR/apache-hbase-master-jmx-config.yml" - JMX_REGIONSERVER_CONFIG_FILE="$JMX_EXPORTER_DIR/apache-hbase-regionserver-jmx-config.yml" - - sudo mkdir -p "$JMX_EXPORTER_DIR" - - echo "Downloading JMX Prometheus Java Agent version $JMX_EXPORTER_VERSION..." - sudo wget -O "$JMX_EXPORTER_JAR" \ - "https://github.com/prometheus/jmx_exporter/releases/download/${JMX_EXPORTER_VERSION}/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" - - # Create JMX exporter config for HBase Master - sudo cat > "$JMX_MASTER_CONFIG_FILE" <<'EOF' - lowercaseOutputName: true - lowercaseOutputLabelNames: true - rules: - - pattern: ".*" - EOF - - # Create JMX exporter config for HBase RegionServer - sudo cat > "$JMX_REGIONSERVER_CONFIG_FILE" <<'EOF' - lowercaseOutputName: true - lowercaseOutputLabelNames: true - rules: - - pattern: ".*" - EOF - - echo "JMX Prometheus Exporter installation completed!" - echo "JAR location: $JMX_EXPORTER_JAR" - echo "Master config: $JMX_MASTER_CONFIG_FILE" - echo "RegionServer config: $JMX_REGIONSERVER_CONFIG_FILE" - - # Load generator script - owner: root:root path: /home/ubuntu/loadgen.sh @@ -379,10 +314,6 @@ runcmd: # General setup - sudo apt-get update - sudo chmod +x /home/ubuntu/install_hbase.sh - - sudo chmod +x /home/ubuntu/install-jmx-prometheus-exporter.sh - - # Install JMX Prometheus Exporter (must be done before HBase installation) - - sudo /home/ubuntu/install-jmx-prometheus-exporter.sh > /home/ubuntu/install-jmx-prometheus-exporter.log 2>&1 # Install HBase (redirect output to log file for debugging) - nohup sudo /home/ubuntu/install_hbase.sh > /home/ubuntu/hbase-install.log 2>&1 || echo "HBase installation script completed with warnings or errors, check /home/ubuntu/hbase-install.log" & diff --git a/sample-apps/apache-hbase/jinja/templates/cloud-init-template.yaml b/sample-apps/apache-hbase/jinja/templates/cloud-init-template.yaml index b1d09fdd..0a308251 100644 --- a/sample-apps/apache-hbase/jinja/templates/cloud-init-template.yaml +++ b/sample-apps/apache-hbase/jinja/templates/cloud-init-template.yaml @@ -59,7 +59,6 @@ write_files: prometheus.remote_write "metrics_service" { endpoint { url = "{{ prom_url }}" - {% if prom_user and prom_pass -%} basic_auth { username = "{{ prom_user }}" @@ -72,7 +71,6 @@ write_files: loki.write "grafana_cloud_loki" { endpoint { url = "{{ loki_url }}" - {% if loki_user and loki_pass -%} basic_auth { username = "{{ loki_user }}" @@ -83,15 +81,20 @@ write_files: } prometheus.scrape "metrics_integrations_integrations_apache_hbase" { - targets = [{ - __address__ = "localhost:8888", - hbase_cluster = "sample_hbase_cluster", - instance = constants.hostname, - },{ - __address__ = "localhost:8889", - hbase_cluster = "sample_hbase_cluster", - instance = constants.hostname, - }] + targets = [ + { + __address__ = "localhost:16010", + metrics_path = "/prometheus", + hbase_cluster = "sample_hbase_cluster", + instance = constants.hostname, + }, + { + __address__ = "localhost:16030", + metrics_path = "/prometheus", + hbase_cluster = "sample_hbase_cluster", + instance = constants.hostname, + }, + ] forward_to = [prometheus.remote_write.metrics_service.receiver] scrape_interval = "{{ interval }}" job_name = "integrations/apache-hbase" @@ -129,15 +132,6 @@ write_files: {%- include 'scripts/install.sh' -%} {% endfilter %} - # JMX Prometheus Exporter installation script - - owner: root:root - path: /home/ubuntu/install-jmx-prometheus-exporter.sh - permissions: '0755' - content: | - {% filter indent(6) %} - {%- include 'scripts/install-jmx-prometheus-exporter.sh' -%} - {% endfilter %} - # Load generator script - owner: root:root path: /home/ubuntu/loadgen.sh @@ -197,10 +191,6 @@ runcmd: # General setup - sudo apt-get update - sudo chmod +x /home/ubuntu/install_hbase.sh - - sudo chmod +x /home/ubuntu/install-jmx-prometheus-exporter.sh - - # Install JMX Prometheus Exporter (must be done before HBase installation) - - sudo /home/ubuntu/install-jmx-prometheus-exporter.sh > /home/ubuntu/install-jmx-prometheus-exporter.log 2>&1 # Install HBase (redirect output to log file for debugging) - nohup sudo /home/ubuntu/install_hbase.sh > /home/ubuntu/hbase-install.log 2>&1 || echo "HBase installation script completed with warnings or errors, check /home/ubuntu/hbase-install.log" & diff --git a/sample-apps/apache-hbase/jinja/templates/scripts/install-jmx-prometheus-exporter.sh b/sample-apps/apache-hbase/jinja/templates/scripts/install-jmx-prometheus-exporter.sh deleted file mode 100644 index 6cff64a0..00000000 --- a/sample-apps/apache-hbase/jinja/templates/scripts/install-jmx-prometheus-exporter.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -{% set jmx_exporter_version = jmx_exporter_version | default("1.3.0") %} - -# JMX Prometheus Exporter configuration -JMX_EXPORTER_VERSION={{ jmx_exporter_version }} -JMX_EXPORTER_DIR="/opt/jmx-prometheus-exporter" -JMX_EXPORTER_JAR="$JMX_EXPORTER_DIR/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" -JMX_MASTER_CONFIG_FILE="$JMX_EXPORTER_DIR/apache-hbase-master-jmx-config.yml" -JMX_REGIONSERVER_CONFIG_FILE="$JMX_EXPORTER_DIR/apache-hbase-regionserver-jmx-config.yml" - -sudo mkdir -p "$JMX_EXPORTER_DIR" - -echo "Downloading JMX Prometheus Java Agent version $JMX_EXPORTER_VERSION..." -sudo wget -O "$JMX_EXPORTER_JAR" \ - "https://github.com/prometheus/jmx_exporter/releases/download/${JMX_EXPORTER_VERSION}/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" - -sudo cat > "$JMX_MASTER_CONFIG_FILE" <<'EOF' -lowercaseOutputName: true -lowercaseOutputLabelNames: true -rules: - - pattern: ".*" -EOF - -sudo cat > "$JMX_REGIONSERVER_CONFIG_FILE" <<'EOF' -lowercaseOutputName: true -lowercaseOutputLabelNames: true -rules: - - pattern: ".*" -EOF - -echo "JMX Prometheus Exporter installation completed!" -echo "JAR location: $JMX_EXPORTER_JAR" -echo "Master config: $JMX_MASTER_CONFIG_FILE" -echo "RegionServer config: $JMX_REGIONSERVER_CONFIG_FILE" diff --git a/sample-apps/apache-hbase/jinja/templates/scripts/install.sh b/sample-apps/apache-hbase/jinja/templates/scripts/install.sh index 420e6e71..bed610f7 100644 --- a/sample-apps/apache-hbase/jinja/templates/scripts/install.sh +++ b/sample-apps/apache-hbase/jinja/templates/scripts/install.sh @@ -80,25 +80,6 @@ sudo cat >> /opt/hbase/conf/hbase-env.sh << 'EOFENV' export HBASE_MANAGES_ZK=true EOFENV -# Configure HBase environment to use JMX exporter -# Note: JMX exporter should be installed separately via install-jmx-prometheus-exporter.sh -JMX_EXPORTER_VERSION={{ jmx_exporter_version | default("1.3.0") }} -JMX_EXPORTER_DIR="/opt/jmx-prometheus-exporter" -JMX_EXPORTER_JAR="$JMX_EXPORTER_DIR/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" -JMX_MASTER_CONFIG="$JMX_EXPORTER_DIR/apache-hbase-master-jmx-config.yml" -JMX_REGIONSERVER_CONFIG="$JMX_EXPORTER_DIR/apache-hbase-regionserver-jmx-config.yml" -JMX_MASTER_PORT=8888 -JMX_REGIONSERVER_PORT=8889 - -sudo cat >> /opt/hbase/conf/hbase-env.sh << EOFENV - -# JMX Prometheus Exporter configuration -# Master JMX exporter endpoint: http://localhost:${JMX_MASTER_PORT}/metrics -export HBASE_MASTER_OPTS="\$HBASE_MASTER_OPTS -javaagent:${JMX_EXPORTER_JAR}=${JMX_MASTER_PORT}:${JMX_MASTER_CONFIG}" - -# RegionServer JMX exporter endpoint: http://localhost:${JMX_REGIONSERVER_PORT}/metrics -export HBASE_REGIONSERVER_OPTS="\$HBASE_REGIONSERVER_OPTS -javaagent:${JMX_EXPORTER_JAR}=${JMX_REGIONSERVER_PORT}:${JMX_REGIONSERVER_CONFIG}" -EOFENV # Create hbase user sudo useradd -r -s /bin/bash -d /opt/hbase hbase || true diff --git a/sample-apps/apache-hbase/jinja/variables/cloud-init.yaml b/sample-apps/apache-hbase/jinja/variables/cloud-init.yaml index 2c9f313f..4f387bb9 100644 --- a/sample-apps/apache-hbase/jinja/variables/cloud-init.yaml +++ b/sample-apps/apache-hbase/jinja/variables/cloud-init.yaml @@ -1,7 +1,7 @@ interval: "10s" -loki_url: http://your-loki-instance:3100/loki/api/v1/push +loki_url: http://192.168.64.3:3100/loki/api/v1/push loki_user: your_loki_username loki_pass: your_loki_password -prom_url: http://your-prometheus-instance:9090/api/v1/push +prom_url: http://192.168.64.3:9009/api/v1/push prom_user: your_prometheus_username prom_pass: your_prometheus_password From 84bdccf330171f3b3b223ac78c5187ec5b860189 Mon Sep 17 00:00:00 2001 From: schmikei Date: Thu, 6 Nov 2025 14:29:18 -0500 Subject: [PATCH 3/3] fix metrics_path --- .../apache-hbase/jinja/templates/cloud-init-template.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sample-apps/apache-hbase/jinja/templates/cloud-init-template.yaml b/sample-apps/apache-hbase/jinja/templates/cloud-init-template.yaml index 0a308251..ff038e7b 100644 --- a/sample-apps/apache-hbase/jinja/templates/cloud-init-template.yaml +++ b/sample-apps/apache-hbase/jinja/templates/cloud-init-template.yaml @@ -84,19 +84,18 @@ write_files: targets = [ { __address__ = "localhost:16010", - metrics_path = "/prometheus", hbase_cluster = "sample_hbase_cluster", instance = constants.hostname, }, { __address__ = "localhost:16030", - metrics_path = "/prometheus", hbase_cluster = "sample_hbase_cluster", instance = constants.hostname, }, ] forward_to = [prometheus.remote_write.metrics_service.receiver] scrape_interval = "{{ interval }}" + metrics_path = "/prometheus", job_name = "integrations/apache-hbase" }