diff --git a/README.md b/README.md index 6bf89d3..301a564 100644 --- a/README.md +++ b/README.md @@ -63,21 +63,22 @@ The `hub-server` will automatically log the detected version at the `INFO` level #### Environment Variables -| Variable | Description | Default | -|----------|-------------|---------| -| `TZ` | Timezone (e.g., `Australia/Brisbane`) | `Australia/Brisbane` | -| `LOG_LEVEL` | Logging verbosity (`DEBUG`, `INFO`, `WARN`, `ERROR`) | `INFO` | -| `MAINS_VOLTAGE` | Local mains voltage (e.g., `230` for AU/UK, `120` for US) | `230` | -| `POWER_FACTOR` | Power factor (H1/H2 use `0.6`, H3 uses `1.0`) | `0.6` | -| `HISTORY_RETENTION_MONTHS` | How many months of data to keep (`0` = keep everything) | `0` | -| `MQTT_ENABLED` | Enable or disable MQTT publishing (`true`/`false`) | `false` | -| `MQTT_BROKER` | IP address or hostname of your MQTT broker | `10.0.0.220` | -| `MQTT_PORT` | Port for your MQTT broker | `1883` | -| `MQTT_USER` | Username for MQTT broker | `None` | -| `MQTT_PASS` | Password for MQTT broker | `None` | -| `HA_DISCOVERY` | Enable Home Assistant MQTT Discovery (`true`/`false`) | `false` | -| `ENERGY_MONTHLY_RESET` | Reset cumulative energy in HA each month (`true`/`false`) | `false` | -| `DEVICE_URL` | Link to device management (e.g., Portainer or NAS UI) | `.../powermeter_hub_server` | +| Variable | Description | Default | +|----------------------------|-----------------------------------------------------------|-----------------------------| +| `TZ` | Timezone (e.g., `Australia/Brisbane`) | `Australia/Brisbane` | +| `LOG_LEVEL` | Logging verbosity (`DEBUG`, `INFO`, `WARN`, `ERROR`) | `INFO` | +| `TELEMETRY_ENABLED` | Record unknown packets to help aid reverse engineering. | `true` | +| `MAINS_VOLTAGE` | Local mains voltage (e.g., `230` for AU/UK, `120` for US) | `230` | +| `POWER_FACTOR` | Power factor (H1/H2 use `0.6`, H3 uses `1.0`) | `0.6` | +| `HISTORY_RETENTION_MONTHS` | How many months of data to keep (`0` = keep everything) | `0` | +| `MQTT_ENABLED` | Enable or disable MQTT publishing (`true`/`false`) | `false` | +| `MQTT_BROKER` | IP address or hostname of your MQTT broker | `10.0.0.220` | +| `MQTT_PORT` | Port for your MQTT broker | `1883` | +| `MQTT_USER` | Username for MQTT broker | `None` | +| `MQTT_PASS` | Password for MQTT broker | `None` | +| `HA_DISCOVERY` | Enable Home Assistant MQTT Discovery (`true`/`false`) | `false` | +| `ENERGY_MONTHLY_RESET` | Reset cumulative energy in HA each month (`true`/`false`) | `false` | +| `DEVICE_URL` | Link to device management (e.g., Portainer or NAS UI) | `.../powermeter_hub_server` | ### 3. Run the Services diff --git a/docker-compose.yml b/docker-compose.yml index 9102b8b..4938abb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,9 @@ services: TZ: Australia/Brisbane # Logging LOG_LEVEL: INFO + # Telemetry, unknown packets are recorded to further reverse engineering. + # You may disable this by setting TELEMETRY_ENABLED to false + TELEMETRY_ENABLED: true # Common voltages AU,UK = 230 | US = 120 https://en.wikipedia.org/wiki/Mains_electricity_by_country MAINS_VOLTAGE: 230 # H1 and H2 devices used a power factor of 0.6, H3 uses 1.0 diff --git a/hub-server/config.py b/hub-server/config.py index e18b78a..291b895 100644 --- a/hub-server/config.py +++ b/hub-server/config.py @@ -9,6 +9,11 @@ # Logging level, values are DEBUG, INFO, WARN, ERROR, CRITICAL LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper() +# Telemetry, unknown packets are recorded to further reverse engineering. +# You may disable this by setting TELEMETRY_ENABLED to false +TELEMETRY_ENABLED = os.getenv("TELEMETRY_ENABLED", "true").lower() in ("true", "1", "yes", "on") +TELEMETRY_URL = os.getenv("TELEMETRY_URL", "https://docs.google.com/forms/d/e/1FAIpQLSf4fOvc65H3a9ZbgZ8e8NGvq8tYPOs0bBn5aAqWrugeg5Jqsw/formResponse") + # SQL timeout in seconds SQLITE_TIMEOUT = float(os.getenv("SQLITE_TIMEOUT", "5.0")) SQLITE_RETRIES = int(os.getenv("SQLITE_RETRIES", "5")) diff --git a/hub-server/hub_server.py b/hub-server/hub_server.py index 6491d44..eb3677c 100755 --- a/hub-server/hub_server.py +++ b/hub-server/hub_server.py @@ -4,9 +4,11 @@ This server emulates the sensornet.info API endpoints for an Efergy hub, logging incoming sensor data to a sqlite database. """ +import json import logging import socket import sys +import requests from http.server import HTTPServer, SimpleHTTPRequestHandler from typing import Type from urllib.parse import urlparse, parse_qs @@ -20,7 +22,7 @@ SERVER_PORT, LOG_LEVEL, MQTT_ENABLED, HA_DISCOVERY, ENERGY_MONTHLY_RESET, HISTORY_RETENTION_MONTHS, SQLITE_TIMEOUT, SQLITE_RETRIES, SQLITE_RETRY_DELAY, POWER_VALUE_TEMPLATE_H1, POWER_UNIT_OF_MEASUREMENT_H1, POWER_VALUE_TEMPLATE_H2, POWER_UNIT_OF_MEASUREMENT_H2, POWER_VALUE_TEMPLATE_H3, POWER_UNIT_OF_MEASUREMENT_H3, ENERGY_VALUE_TEMPLATE, - ENERGY_UNIT_OF_MEASUREMENT + ENERGY_UNIT_OF_MEASUREMENT, TELEMETRY_ENABLED, TELEMETRY_URL ) class EfergyHTTPServer(HTTPServer): @@ -97,24 +99,29 @@ def _handle_unknown_packet(self, post_data_bytes: bytes = None): try: parsed_url = urlparse(self.path) query = parse_qs(parsed_url.query) + method = self.command + path = self.path content_length = int(self.headers.get("Content-Length", 0)) + content_type = self.headers.get("Content-Type", "") + body_utf8 = "No request body present" + body_hex = "" if post_data_bytes is None and content_length > 0: post_data_bytes = self.rfile.read(content_length) logging.warning("========== UNKNOWN PACKET ==========") - logging.warning("Method: %s", self.command) - logging.warning("Raw path: %s", self.path) + logging.warning("Method: %s", method) + logging.warning("Raw path: %s", path) logging.warning("Path: %s", parsed_url.path) logging.warning("Query: %s", query) - logging.warning("Content-Type: %s", self.headers.get("Content-Type")) + logging.warning("Content-Type: %s", content_type) logging.warning("Content-Length: %d", content_length) logging.warning("Headers: %s", dict(self.headers)) if not post_data_bytes: logging.warning("No request body present") else: - utf8 = post_data_bytes.decode("utf-8", "ignore") + body_utf8 = post_data_bytes.decode("utf-8", "ignore") hex_lines = [] for i in range(0, len(post_data_bytes), 16): @@ -123,12 +130,33 @@ def _handle_unknown_packet(self, post_data_bytes: bytes = None): ascii_bytes = "".join(chr(b) if 32 <= b <= 126 else "." for b in chunk) hex_lines.append(f"{i:04X} {hex_bytes:<48} {ascii_bytes}") - logging.warning("Payload UTF8:\n%s", utf8) - logging.warning("Payload HEX:\n%s", "\n".join(hex_lines)) + body_hex = "\n".join(hex_lines) + logging.warning("Payload UTF8:\n%s", body_utf8) + logging.warning("Payload HEX:\n%s", body_hex) logging.warning("====================================") + if TELEMETRY_ENABLED: + payload = { + "entry.712651641": str(method), + "entry.647193072": str(path), + "entry.718643403": str(parsed_url.path), + "entry.628690130": json.dumps(query, ensure_ascii=False), + "entry.744795787": str(content_type), + "entry.2143045526": str(content_length), + "entry.311956051": json.dumps(dict(self.headers), ensure_ascii=False), + "entry.1917117670": body_utf8, + "entry.2020488076": body_hex, + } + + logging.warning("Submitting telemetry for unknown packet") + response = requests.post(TELEMETRY_URL, data=payload, timeout=5) + if response.status_code == 200: + logging.warning("Telemetry submission successful") + else: + logging.warning(f"Failed to submit Telemetry: {response.status_code}") + logging.warning("====================================") except Exception: logging.exception("Error in unknown HTTP handler") if not self.wfile.closed: @@ -255,7 +283,7 @@ def process_sensor_data(self, post_data_bytes: bytes, hub_version: str, firmware parsed_results = parse_sensor_payload(post_data_bytes, hub_version, firmware_version) if hub_version.upper() not in self.server.detected_versions: - logging.info(f"Detected Efergy Hub version: {hub_version.upper()}") + logging.info(f"Detected Efergy Hub version: {hub_version.upper()} - {firmware_version}") self.server.detected_versions.add(hub_version.upper()) for data in parsed_results: @@ -342,6 +370,7 @@ def run_server(database: Database, host: str = '0.0.0.0', port: int = 5000): logging.info(f" Python: {sys.version.split()[0]}") logging.info(f" Port: {SERVER_PORT}") logging.info(f" Logging level: {LOG_LEVEL}") + logging.info(f" Telemetry enabled: {TELEMETRY_ENABLED}") logging.info(f" MQTT: {'enabled' if MQTT_ENABLED else 'disabled'}") logging.info(f" HA discovery: {'enabled' if HA_DISCOVERY else 'disabled'}") logging.info(f" Monthly reset: {ENERGY_MONTHLY_RESET}") diff --git a/hub-server/requirements-dev.txt b/hub-server/requirements-dev.txt index daa4fe4..3728a68 100644 --- a/hub-server/requirements-dev.txt +++ b/hub-server/requirements-dev.txt @@ -1,3 +1,4 @@ paho-mqtt >= 2.1.0 pytest >= 8.4.1 -pytest-cov >= 7.0.0 \ No newline at end of file +pytest-cov >= 7.0.0 +requests >= 2.32.5 \ No newline at end of file diff --git a/hub-server/requirements.txt b/hub-server/requirements.txt index 949447f..f5ce654 100644 --- a/hub-server/requirements.txt +++ b/hub-server/requirements.txt @@ -1 +1,2 @@ -paho-mqtt >= 2.1.0 \ No newline at end of file +paho-mqtt >= 2.1.0 +requests >= 2.32.5 \ No newline at end of file diff --git a/hub-server/tests/conftest.py b/hub-server/tests/conftest.py new file mode 100644 index 0000000..3aba5f8 --- /dev/null +++ b/hub-server/tests/conftest.py @@ -0,0 +1,5 @@ +import os + +# Set environment variables for tests before any other modules are imported +os.environ["TELEMETRY_ENABLED"] = "false" +os.environ["TELEMETRY_URL"] = "" \ No newline at end of file