diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 4e063163..e5cb8f73 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -427,7 +427,33 @@ create_backup_to_file(data_dir, dest_dir) ## Driver Architecture -Modem drivers live in `app/drivers/` and implement the `ModemDriver` base class: +Modem drivers live in `app/drivers/` and implement the `ModemDriver` base +class. A driver owns device orchestration: network requests, endpoint and +firmware selection, authentication, sessions, retries, TLS, and cryptography. +It fetches a raw modem payload and delegates normalization to one of the pure, +explicit profiles in `app/drivers/formats/`. + +The dependency direction is one-way: + +```text +collector -> concrete driver -> named format profile -> parser primitives/types + -> transport helpers (driver code only) +``` + +Format modules never depend on a concrete driver, session, request client, +Flask, clocks, randomness, or cryptography. They return an immutable +`ParseResult(value, diagnostics)`. Diagnostics contain only the finite safe +fields `family`, `profile`, `code`, `direction`, `row`, `index`, and `field`. +Public driver methods unwrap the result and retain the established +`DocsisData`/channel-list contracts. + +`app/drivers/format_compat.py` is a finite compatibility boundary for legacy +warning messages. `app/drivers/arris_html.py` remains an import-compatible shim +for the established bonded 8/7-column parser. Existing private parser seams +that are covered by integrations remain one-statement delegations; format +grammar is not implemented in concrete drivers. + +The base interface is: ```python class ModemDriver(ABC): @@ -446,27 +472,42 @@ class ModemDriver(ABC): def get_connection_info(self) -> dict: ... ``` -### Supported Drivers - -| Driver | Module | Hardware | Auth | -|--------|--------|----------|------| -| `fritzbox` | `fritzbox.py` | AVM FRITZ!Box | SID-based (data.lua) | -| `tc4400` | `tc4400.py` | Technicolor TC4400 | SNMP | -| `ultrahub7` | `ultrahub7.py` | Vodafone Ultra Hub 7 | Session cookie | -| `cm3500` | `cm3500.py` | Arris CM3500B | Form POST (IP-based session) | -| `connectbox` | `connectbox.py` | Unitymedia Connect Box (CH7465) | Session cookie | -| `vodafone_station` | `vodafone_station.py` | CGA6444VF, CGA4322DE, TG3442DE | Auto-detected (see below) | -| `cm1000` | `cm1000.py` | Netgear CM1000 | HTTP Basic or local Genie form | -| `cm3000` | `cm3000.py` | Netgear CM3000 | HTTP Basic Auth | -| `surfboard` | `surfboard.py` | Arris SURFboard S33/S34/SB8200 | HNAP1 HMAC-SHA256 | -| `sb6183` | `sb6183.py` | Arris SB6183 | None (HTTP status pages) | -| `cm8200` | `cm8200.py` | Arris Touchstone CM8200A | Base64 query string | -| `hitron_coda_4680` | `hitron_coda_4680.py` | Hitron CODA-4680 | Form POST (`/1/Device/Users/Login`) | -| `generic` | `generic.py` | Generic Router (no DOCSIS) | None | - -### Driver Registry (`app/drivers/__init__.py`) - -Drivers are loaded by name via `load_driver(modem_type, url, user, password)`. The registry maps type strings to fully qualified class paths for lazy importing. +### Exact driver-to-profile matrix + +Every registered concrete class exposes a non-empty immutable +`FORMAT_FAMILIES` tuple. Registry aliases appear together in the first column; +there are 21 keys, 20 concrete classes, and 22 explicit profiles. + +| Registry key(s) | Concrete class | Format profile(s) | Cohesive module / entrypoint | +|---|---|---|---| +| `cgm4981` | `CGM4981Driver` | `cgm4981_columnar_html` | `html_columnar.parse_cgm4981_columnar_html` | +| `ch7465`, `ch7465_play` | `CH7465Driver` | `ch7465_xml` | `xml_payloads.parse_ch7465_xml` | +| `cm1000` | `CM1000Driver` | `cm1000_html_table`, `cm1000_javascript` | `html_rows.parse_cm1000_html_table`; `javascript.parse_cm1000_javascript` | +| `cm3000` | `CM3000Driver` | `cm3000_javascript` | `javascript.parse_cm3000_javascript` | +| `cm3500` | `CM3500Driver` | `cm3500_html` | `html_rows.parse_cm3500_html` | +| `cm8200` | `CM8200Driver` | `arris_html` | `html_rows.parse_arris_html` | +| `f3896lg` | `F3896LGDriver` | `f3896lg_rest_json` | `sagemcom.parse_f3896lg_rest_json` | +| `fritzbox` | `FritzBoxDriver` | `fritzbox_data_lua` | `fritzbox.parse_fritzbox_data_lua` | +| `generic` | `GenericDriver` | `generic_no_docsis` | `boundaries.parse_generic_no_docsis` | +| `hitron` | `HitronDriver` | `hitron_coda56_json` | `hitron.parse_hitron_coda56_json` | +| `hitron_coda_4680` | `HitronCoda4680Driver` | `hitron_coda4680_json` | `hitron.parse_hitron_coda4680_json` | +| `sagemcom` | `SagemcomDriver` | `sagemcom_xmo_json` | `sagemcom.parse_sagemcom_xmo_json` | +| `sb6141` | `SB6141Driver` | `sb6141_transposed_html` | `html_transposed.parse_sb6141_transposed_html` | +| `sb6183` | `SB6183Driver` | `sb6183_html` | `html_rows.parse_sb6183_html` | +| `sb6190` | `SB6190Driver` | `sb6190_html` | `html_rows.parse_sb6190_html` | +| `sercom_dm1000` | `SercomDM1000Driver` | `sercom_dm1000_json` | `sercom.parse_sercom_dm1000_json` | +| `surfboard` | `SurfboardDriver` | `arris_html`, `surfboard_hnap` | `html_rows.parse_arris_html`; `surfboard.parse_surfboard_hnap` | +| `tc4400` | `TC4400Driver` | `tc4400_html` | `html_rows.parse_tc4400_html` | +| `ultrahub7` | `UltraHub7Driver` | `ultrahub7_json` | `vodafone.parse_ultrahub7_json` | +| `vodafone_station` | `VodafoneStationDriver` | `vodafone_station_cga_json`, `vodafone_station_tg_embedded_json` | `vodafone.parse_vodafone_station_cga_json`; `vodafone.parse_vodafone_station_tg_embedded_json` | + +### Driver Registry (`app/drivers/registry.py`) + +Drivers are loaded by name through `load_driver(modem_type, url, user, +password)`. The registry maps type strings to fully qualified class paths for +lazy importing. `ch7465` and `ch7465_play` intentionally resolve to the same +class; the registry applies the Play firmware selection without creating a +second parser profile. ### Extension module state diff --git a/app/drivers/arris_html.py b/app/drivers/arris_html.py index 092f2a3d..24e590c2 100644 --- a/app/drivers/arris_html.py +++ b/app/drivers/arris_html.py @@ -1,229 +1,39 @@ -"""Shared Arris HTML channel-table parser for DOCSight. - -Parses the ``/cmconnectionstatus.html`` status page used by Arris cable -modems (CM8200A, SB8200 HTML fallback, and similar) into the standard -DOCSight channel data format. - -The page contains two HTML tables: -- "Downstream Bonded Channels" (8 columns) -- "Upstream Bonded Channels" (7 columns) - -DOCSIS version is inferred from modulation / channel type: -- DS: modulation "Other" = OFDM (3.1), anything else = SC-QAM (3.0) -- US: "OFDM" in type without "SC-QAM" = OFDMA (3.1), else SC-QAM (3.0) -""" +"""Compatibility shim for the established shared Arris bonded-table parser.""" from __future__ import annotations -import logging - -from bs4 import BeautifulSoup, Tag - -from ..types import DocsisDataFritz, RawChannel - -log = logging.getLogger("docsis.arris_html") - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - -def parse_arris_channel_tables(html: str) -> DocsisDataFritz: - """Parse Arris modem status page HTML into DOCSight channel format. - - Returns:: - - {"channelDs": {"docsis30": [...], "docsis31": [...]}, - "channelUs": {"docsis30": [...], "docsis31": [...]}} - """ - soup = BeautifulSoup(html, "html.parser") - ds_table, us_table = _find_channel_tables(soup) - - ds30, ds31 = _parse_downstream(ds_table) - us30, us31 = _parse_upstream(us_table) - - return { - "channelDs": {"docsis30": ds30, "docsis31": ds31}, - "channelUs": {"docsis30": us30, "docsis31": us31}, - } - - -# --------------------------------------------------------------------------- -# Table discovery -# --------------------------------------------------------------------------- - -def _find_channel_tables(soup: BeautifulSoup) -> tuple: - """Find downstream and upstream channel tables by header text. - - Returns ``(ds_table, us_table)`` where either may be ``None``. - """ - ds_table = None - us_table = None - - for table in soup.find_all("table"): - header = table.find("tr") - if not header: - continue - text = header.get_text(strip=True).lower() - if "downstream bonded" in text: - ds_table = table - elif "upstream bonded" in text: - us_table = table - - return ds_table, us_table - - -# --------------------------------------------------------------------------- -# Row classification -# --------------------------------------------------------------------------- - -def _is_header_row(row: Tag) -> bool: - """True if *row* is a table title or column-header row (not data).""" - if row.find("th"): - return True - if row.find("strong"): - return True - return False - - -# --------------------------------------------------------------------------- -# Downstream parser -# --------------------------------------------------------------------------- - -def _parse_downstream(table) -> tuple: - """Parse downstream table into ``(docsis30, docsis31)`` channel lists. - - Expected 8 columns per data row: - Channel ID | Lock Status | Modulation | Frequency | - Power | SNR/MER | Corrected | Uncorrectables - """ - ds30: list[dict] = [] - ds31: list[dict] = [] - if not table: - return ds30, ds31 - - for row in table.find_all("tr"): - if _is_header_row(row): - continue - cells = [td.get_text(strip=True) for td in row.find_all("td")] - if len(cells) < 8: - continue - - lock_status = cells[1] - if lock_status != "Locked": - continue - - try: - channel_id = int(cells[0]) - modulation = cells[2] - frequency = _parse_freq_hz(cells[3]) - power = _parse_value(cells[4]) - snr = _parse_value(cells[5]) - corrected = int(cells[6]) - uncorrectables = int(cells[7]) - - channel: RawChannel = { - "channelID": channel_id, - "frequency": frequency, - "powerLevel": power, - "modulation": modulation, - "corrErrors": corrected, - "nonCorrErrors": uncorrectables, - } - - if modulation == "Other": - # OFDM channel (DOCSIS 3.1) - channel["type"] = "OFDM" - channel["mer"] = snr - channel["mse"] = None - ds31.append(channel) - else: - # SC-QAM channel (DOCSIS 3.0) - channel["mer"] = snr - channel["mse"] = -snr if snr is not None else None - ds30.append(channel) - except (ValueError, TypeError, IndexError) as e: - log.warning("Failed to parse DS row: %s", e) - - return ds30, ds31 - - -# --------------------------------------------------------------------------- -# Upstream parser -# --------------------------------------------------------------------------- - -def _parse_upstream(table) -> tuple: - """Parse upstream table into ``(docsis30, docsis31)`` channel lists. - - Expected 7 columns per data row: - Channel | Channel ID | Lock Status | US Channel Type | - Frequency | Width | Power - """ - us30: list[dict] = [] - us31: list[dict] = [] - if not table: - return us30, us31 - - for row in table.find_all("tr"): - if _is_header_row(row): - continue - cells = [td.get_text(strip=True) for td in row.find_all("td")] - if len(cells) < 7: - continue - - lock_status = cells[2] - if lock_status != "Locked": - continue +from .formats.html_rows import ( + _arris_tables as _find_channel_tables, + _optional_value as _parse_value, + parse_arris_downstream, + parse_arris_html, + parse_arris_upstream, +) +from .formats.primitives import hz_to_mhz as _parse_freq_hz - try: - channel_id = int(cells[1]) - channel_type = cells[3] - frequency = _parse_freq_hz(cells[4]) - power = _parse_value(cells[6]) - channel: RawChannel = { - "channelID": channel_id, - "frequency": frequency, - "powerLevel": power, - "modulation": channel_type, - } +def parse_arris_channel_tables(html: str): + return parse_arris_html(html).value - if "OFDM" in channel_type and "SC-QAM" not in channel_type: - # OFDMA channel (DOCSIS 3.1) - channel["type"] = "OFDMA" - channel["multiplex"] = "" - us31.append(channel) - else: - # SC-QAM channel (DOCSIS 3.0) - channel["multiplex"] = "SC-QAM" - us30.append(channel) - except (ValueError, TypeError, IndexError) as e: - log.warning("Failed to parse US row: %s", e) - return us30, us31 +def _parse_downstream(table): + return parse_arris_downstream(table).value -# --------------------------------------------------------------------------- -# Value helpers -# --------------------------------------------------------------------------- +def _parse_upstream(table): + return parse_arris_upstream(table).value -def _parse_freq_hz(freq_str: str) -> str: - """Convert ``'795000000 Hz'`` to ``'795 MHz'``.""" - from .utils import hz_to_mhz - return hz_to_mhz(freq_str) +def _is_header_row(row): + return bool(row.find("th") or row.find("strong")) -def _parse_value(val_str: str): - """Parse ``'8.2 dBmV'`` or ``'43.0 dB'`` to float. - Note: Returns None (not 0.0) for empty/unparseable input, unlike - parse_number(). This preserves arris_html's existing behaviour where - None signals "value not present" vs 0.0 for "value is zero". - """ - if not val_str: - return None - parts = val_str.strip().split() - try: - return float(parts[0]) - except (ValueError, IndexError): - return None +__all__ = [ + "parse_arris_channel_tables", + "_find_channel_tables", + "_is_header_row", + "_parse_downstream", + "_parse_freq_hz", + "_parse_upstream", + "_parse_value", +] diff --git a/app/drivers/cgm4981.py b/app/drivers/cgm4981.py index 027227c2..16dff029 100644 --- a/app/drivers/cgm4981.py +++ b/app/drivers/cgm4981.py @@ -40,11 +40,19 @@ import requests from .base import ModemDriver -from ..docsis_utils import parse_qam_order -from ..types import ConnectionInfo, DeviceInfo, DocsisData, RawChannel +from ..types import ConnectionInfo, DeviceInfo, DocsisData +from .formats.html_columnar import ( + _float, + _modulation, + build_cgm4981_downstream, + build_cgm4981_upstream, + parse_cgm4981_columnar_html, +) log = logging.getLogger("docsis.driver.cgm4981") +__all__ = ["CGM4981Driver", "_float", "_modulation"] + # --------------------------------------------------------------------------- # Constants # --------------------------------------------------------------------------- @@ -58,121 +66,6 @@ # Cookie name that confirms a valid session. _SESSION_COOKIE = "DUKSID" -# HTML markers used to split the page into three sections. -_MARKER_DS = ">Downstream<" -_MARKER_US = ">Upstream<" -_MARKER_ERR = "CM Error Codewords" - -# Compiled patterns used throughout parsing. -_RE_TR = re.compile(r"]*>(.*?)", re.DOTALL | re.IGNORECASE) -_RE_TH = re.compile(r"]*>(.*?)", re.DOTALL | re.IGNORECASE) -_RE_NETWIDTH = re.compile(r']*class="netWidth"[^>]*>(.*?)', re.DOTALL) -_RE_STRIP = re.compile(r"<[^>]+>") -_RE_NUMBER = re.compile(r"-?\d+\.?\d*") - - -# --------------------------------------------------------------------------- -# Parsing helpers -# --------------------------------------------------------------------------- - -def _text(html: str) -> str: - """Strip HTML tags and return plain text.""" - return _RE_STRIP.sub("", html).strip() - - -def _float(raw: str) -> float | None: - """Return first float found in a string, or None if absent.""" - m = _RE_NUMBER.search(raw.strip()) - return float(m.group()) if m else None - - -def _freq_mhz(raw: str) -> str: - """Normalise a frequency string to 'NNN MHz'. - - Handles: - - '189 MHz' → '189 MHz' - - '300000000' → '300 MHz' (raw Hz integer from OFDM row) - - '950000000' → '950 MHz' - - '17 MHz' → '17 MHz' (extra whitespace) - """ - raw = raw.strip() - if re.search(r"[Mm][Hh][Zz]", raw): - num = _RE_NUMBER.search(raw) - if num: - mhz = float(num.group()) - return f"{int(mhz) if mhz == int(mhz) else mhz} MHz" - m = _RE_NUMBER.search(raw) - if m: - val = float(m.group()) - if val > 1_000_000: - mhz = val / 1_000_000 - return f"{int(mhz) if mhz == int(mhz) else mhz} MHz" - return f"{int(val) if val == int(val) else val} MHz" - return raw - - -def _modulation(raw: str) -> str: - """Normalise modulation string for DOCSight's analyser. - - Maps common CGM4981 values: - '256 QAM' → '256QAM' - 'OFDM' → 'OFDM' - 'OFDMA' → 'OFDMA' - 'QAM' → 'QAM' (upstream SC-QAM without order) - """ - up = raw.strip().upper() - if "OFDMA" in up: - return "OFDMA" - if "OFDM" in up: - return "OFDM" - # '256 QAM' → '256QAM', '64 QAM' → '64QAM', etc. - qam_order = parse_qam_order(up) - if qam_order is not None: - return f"{qam_order}QAM" - if "QAM" in up: - return "QAM" - return raw.strip() - - -def _section_rows(html: str) -> dict[str, list[str]]: - """Parse all ```` rows in an HTML *section* into {label: [values]}. - - Each data row is expected to have: - - A ```` element whose text is the row label. - - One or more ```` cells each containing a - ``
VALUE
`` element. - - Labels are unique within a section (the caller is responsible for - passing only one table's HTML, not the whole page). - """ - rows: dict[str, list[str]] = {} - for tr_m in _RE_TR.finditer(html): - tr = tr_m.group(1) - th_m = _RE_TH.search(tr) - if not th_m: - continue - label = _text(th_m.group(1)) - if not label: - continue - values = [_text(v) for v in _RE_NETWIDTH.findall(tr)] - if values: - rows[label] = values - return rows - - -def _split_sections(html: str) -> tuple[str, str, str]: - """Return (ds_html, us_html, err_html) by finding section markers.""" - ds_idx = html.find(_MARKER_DS) - us_idx = html.find(_MARKER_US) - err_idx = html.find(_MARKER_ERR) - - ds_html = html[ds_idx:us_idx] if ds_idx >= 0 and us_idx > ds_idx else "" - us_html = html[us_idx:err_idx] if us_idx >= 0 and err_idx > us_idx else "" - err_html = html[err_idx:] if err_idx >= 0 else "" - - return ds_html, us_html, err_html - - # --------------------------------------------------------------------------- # Driver # --------------------------------------------------------------------------- @@ -186,6 +79,8 @@ class CGM4981Driver(ModemDriver): table on the same page. """ + FORMAT_FAMILIES = ("cgm4981_columnar_html",) + def __init__(self, url: str, user: str, password: str) -> None: super().__init__(url, user, password) self._session: requests.Session = requests.Session() @@ -217,34 +112,7 @@ def login(self) -> None: def get_docsis_data(self) -> DocsisData: """Return parsed downstream and upstream channel data.""" - html = self._fetch_status_page() - ds_html, us_html, err_html = _split_sections(html) - - if not ds_html: - log.warning("CGM4981: Downstream section not found in status page") - if not us_html: - log.warning("CGM4981: Upstream section not found in status page") - - ds_rows = _section_rows(ds_html) - us_rows = _section_rows(us_html) - err_rows = _section_rows(err_html) - - ds_channels = self._build_ds_channels(ds_rows, err_rows) - us_channels = self._build_us_channels(us_rows) - - ds30 = [ch for ch in ds_channels if ch.get("modulation") != "OFDM"] - ds31 = [ch for ch in ds_channels if ch.get("modulation") == "OFDM"] - us30 = [ch for ch in us_channels if ch.get("modulation") != "OFDMA"] - us31 = [ch for ch in us_channels if ch.get("modulation") == "OFDMA"] - - log.debug( - "CGM4981 parsed: DS SC-QAM=%d OFDM=%d | US SC-QAM=%d OFDMA=%d", - len(ds30), len(ds31), len(us30), len(us31), - ) - return { - "channelDs": {"docsis30": ds30, "docsis31": ds31}, - "channelUs": {"docsis30": us30, "docsis31": us31}, - } + return parse_cgm4981_columnar_html(self._fetch_status_page()).value def get_device_info(self) -> DeviceInfo: """Return model, firmware version, and uptime from the status page.""" @@ -338,103 +206,8 @@ def _build_ds_channels( ds_rows: dict[str, list[str]], err_rows: dict[str, list[str]], ) -> list[dict]: - """Build downstream channel list from the DS section and error table. - - Error counts from the "CM Error Codewords" table are indexed by - channel ID to ensure correct alignment even if row order varies. - """ - ch_ids = ds_rows.get("Channel ID", []) - locks = ds_rows.get("Lock Status", []) - freqs = ds_rows.get("Frequency", []) - snrs = ds_rows.get("SNR", []) - powers = ds_rows.get("Power Level", []) - mods = ds_rows.get("Modulation", []) - - if not ch_ids: - log.warning("CGM4981: no DS Channel ID row found") - return [] - - # Build a {channel_id: (corr, uncorr)} map from the error table. - err_ch_ids = err_rows.get("Channel ID", []) - err_corr = err_rows.get("Correctable Codewords", []) - err_uncorr = err_rows.get("Uncorrectable Codewords", []) - err_map: dict[str, tuple[int, int]] = {} - for i, cid in enumerate(err_ch_ids): - corr = int(err_corr[i]) if i < len(err_corr) and err_corr[i].lstrip("-").isdigit() else 0 - uncorr = int(err_uncorr[i]) if i < len(err_uncorr) and err_uncorr[i].lstrip("-").isdigit() else 0 - err_map[cid] = (corr, uncorr) - - channels = [] - for i, cid in enumerate(ch_ids): - lock = locks[i] if i < len(locks) else "" - if lock.lower() != "locked": - continue - try: - mod = _modulation(mods[i] if i < len(mods) else "") - freq = _freq_mhz(freqs[i] if i < len(freqs) else "") - snr = _float(snrs[i] if i < len(snrs) else "") - pwr = _float(powers[i] if i < len(powers) else "") - corr, uncorr = err_map.get(cid, (0, 0)) - - ch: RawChannel = { - "channelID": int(cid), - "frequency": freq, - "powerLevel": pwr, - "mer": snr, - "mse": -snr if snr else None, - "modulation": mod, - "corrErrors": corr, - "nonCorrErrors": uncorr, - } - if mod == "OFDM": - ch["type"] = "OFDM" - - channels.append(ch) - except (ValueError, IndexError) as exc: - log.warning("CGM4981 DS channel %s parse error: %s", cid, exc) - - return channels + return build_cgm4981_downstream(ds_rows, err_rows).value @staticmethod def _build_us_channels(us_rows: dict[str, list[str]]) -> list[dict]: - """Build upstream channel list from the US section only.""" - ch_ids = us_rows.get("Channel ID", []) - locks = us_rows.get("Lock Status", []) - freqs = us_rows.get("Frequency", []) - powers = us_rows.get("Power Level", []) - mods = us_rows.get("Modulation", []) - types = us_rows.get("Channel Type",[]) - - if not ch_ids: - log.warning("CGM4981: no US Channel ID row found") - return [] - - channels = [] - for i, cid in enumerate(ch_ids): - lock = locks[i] if i < len(locks) else "" - if lock.lower() != "locked": - continue - try: - raw_mod = mods[i] if i < len(mods) else "" - raw_type = types[i] if i < len(types) else "" - mod = _modulation(raw_mod) - # OFDMA upstream: modulation field says 'OFDMA' directly. - # ATDMA SC-QAM: modulation field says 'QAM' (no order on this device). - freq = _freq_mhz(freqs[i] if i < len(freqs) else "") - pwr = _float(powers[i] if i < len(powers) else "") - - ch: RawChannel = { - "channelID": int(cid), - "frequency": freq, - "powerLevel": pwr, - "modulation": mod, - "multiplex": raw_type.upper() or mod, - } - if mod == "OFDMA": - ch["type"] = "OFDMA" - - channels.append(ch) - except (ValueError, IndexError) as exc: - log.warning("CGM4981 US channel %s parse error: %s", cid, exc) - - return channels + return build_cgm4981_upstream(us_rows).value diff --git a/app/drivers/ch7465.py b/app/drivers/ch7465.py index 84cfc338..abaac96e 100644 --- a/app/drivers/ch7465.py +++ b/app/drivers/ch7465.py @@ -17,7 +17,8 @@ import weakref from enum import Enum from .base import ModemDriver -from .utils import normalize_modulation +from .formats.primitives import normalize_modulation +from .formats.xml_payloads import parse_ch7465_xml from ..types import DocsisData, DeviceInfo, ConnectionInfo log = logging.getLogger("docsis.driver.ch7465") @@ -50,6 +51,8 @@ class CH7465Driver(ModemDriver): DOCSIS data is fetched via XML API endpoints. """ + FORMAT_FAMILIES = ("ch7465_xml",) + def __init__( self, url: str, @@ -141,62 +144,13 @@ def login(self) -> None: def get_docsis_data(self) -> DocsisData: """Query DOCSIS channel data.""" - result = { - "docsis": "3.0", - "downstream": [], - "upstream": [], - } - - # Downstream channels - xml = self._get_data(Query.DOWNSTREAM_TABLE) - root = ET.fromstring(xml) - for channel in root.findall("downstream"): - # Map to FritzBox-compatible format for analyzer - item = { - "channelID": int(channel.find("chid").text), - "frequency": _node_text(channel.find("freq")), - "powerLevel": float(_node_text(channel.find("pow"), "0")), - } - mer = _node_text(channel.find("RxMER")) - modulation = self._normalize_modulation(_node_text(channel.find("mod"))) - pre_rs = _node_text(channel.find("PreRs")) - post_rs = _node_text(channel.find("PostRs")) - if mer: - item["mer"] = float(mer) - item["mse"] = -float(mer) - if modulation: - item["modulation"] = modulation - if pre_rs: - item["corrErrors"] = int(pre_rs) - if post_rs: - item["nonCorrErrors"] = int(post_rs) - result["downstream"].append(item) - - # Upstream channels - xml = self._get_data(Query.UPSTREAM_TABLE) - root = ET.fromstring(xml) - for channel in root.findall("upstream"): - # Map to FritzBox-compatible format for analyzer - item = { - "channelID": int(channel.find("usid").text), - "frequency": _node_text(channel.find("freq")), - "powerLevel": float(_node_text(channel.find("power"), "0")), - } - modulation = self._normalize_modulation(_node_text(channel.find("mod"))) - messageType = _node_text(channel.find("messageType")) - multiplex = { - "2": "tdma", # "1.0" - "29": "atdma", # "2.0" - "35": "atdma", # "3.0" - }.get(messageType, messageType) - if modulation: - item["modulation"] = modulation - if multiplex: - item["multiplex"] = multiplex - # TODO: estimate "latency" from modulation, "srate", "t1Timeouts", .., "t4Timeouts" - result["upstream"].append(item) - - return result + parsed = parse_ch7465_xml( + self._get_data(Query.DOWNSTREAM_TABLE), + self._get_data(Query.UPSTREAM_TABLE), + ) + if parsed.value is None: + raise ValueError("invalid CH7465 channel XML") + return parsed.value def get_device_info(self) -> DeviceInfo: """Try to get CH7465 model info.""" @@ -319,9 +273,4 @@ def _get_login_fail_count(self) -> int: @staticmethod def _normalize_modulation(modulation: str) -> str: - """Normalize modulation string to analyzer format. - - Input: "256qam", "64qam", "qpsk", .. - Output: "256QAM", "64QAM", "QPSK", .. - """ return normalize_modulation(modulation) diff --git a/app/drivers/cm1000.py b/app/drivers/cm1000.py index f89ffcea..cb577295 100644 --- a/app/drivers/cm1000.py +++ b/app/drivers/cm1000.py @@ -15,9 +15,17 @@ import requests from bs4 import BeautifulSoup -from ..types import ConnectionInfo, DeviceInfo, DocsisData, RawChannel +from ..types import ConnectionInfo, DeviceInfo, DocsisData from .base import ModemDriver -from .utils import hz_to_mhz, normalize_modulation +from .formats.html_rows import ( + parse_cm1000_downstream_table, + parse_cm1000_upstream_table, +) +from .formats.javascript import ( + extract_cm1000_tag_value_list, + parse_cm1000_downstream_tag_values, + parse_cm1000_upstream_tag_values, +) log = logging.getLogger("docsis.driver.cm1000") @@ -85,6 +93,8 @@ class CM1000Driver(ModemDriver): """Driver for the Netgear CM1000 DOCSIS 3.1 cable modem.""" + FORMAT_FAMILIES = ("cm1000_html_table", "cm1000_javascript") + def __init__(self, url: str, user: str, password: str): super().__init__(url.rstrip("/"), user, password) self._session = self._new_session() @@ -143,20 +153,20 @@ def get_docsis_data(self) -> DocsisData: html = self._fetch_status_page() soup = BeautifulSoup(html, "html.parser") - ds_tag_values = self._extract_tag_value_list(html, _DS_TAG_VALUE_FUNCTION) + ds_tag_values = extract_cm1000_tag_value_list(html, _DS_TAG_VALUE_FUNCTION) if ds_tag_values is not None: - ds30 = self._parse_downstream_tag_values(ds_tag_values) + ds30 = parse_cm1000_downstream_tag_values(ds_tag_values).value else: - ds30 = self._parse_downstream_table(soup, "dsTable", docsis31=False) + ds30 = parse_cm1000_downstream_table(soup, "dsTable", docsis31=False).value - us_tag_values = self._extract_tag_value_list(html, _US_TAG_VALUE_FUNCTION) + us_tag_values = extract_cm1000_tag_value_list(html, _US_TAG_VALUE_FUNCTION) if us_tag_values is not None: - us30 = self._parse_upstream_tag_values(us_tag_values) + us30 = parse_cm1000_upstream_tag_values(us_tag_values).value else: - us30 = self._parse_upstream_table(soup, "usTable", docsis31=False) + us30 = parse_cm1000_upstream_table(soup, "usTable", docsis31=False).value - ds31 = self._parse_downstream_table(soup, "d31dsTable", docsis31=True) - us31 = self._parse_upstream_table(soup, "d31usTable", docsis31=True) + ds31 = parse_cm1000_downstream_table(soup, "d31dsTable", docsis31=True).value + us31 = parse_cm1000_upstream_table(soup, "d31usTable", docsis31=True).value if not any((ds30, us30, ds31, us31)): log.warning("CM1000 parsed 0 locked channels from DocsisStatus.asp") @@ -230,7 +240,7 @@ def _is_status_page(html: str) -> bool: if any(soup.find("table", id=table_id) is not None for table_id in _TABLE_IDS): return True return any( - CM1000Driver._extract_tag_value_list(html, function_name) is not None + extract_cm1000_tag_value_list(html, function_name) is not None for function_name in (_DS_TAG_VALUE_FUNCTION, _US_TAG_VALUE_FUNCTION) ) @@ -242,393 +252,3 @@ def _ensure_status_page(html: str) -> None: raise RuntimeError( "CM1000 authentication failed: modem did not return DocsisStatus.asp" ) - - @staticmethod - def _normalize_header(value: str) -> str: - return _HEADER_RE.sub("", value.lower()) - - @classmethod - def _table_rows(cls, soup: BeautifulSoup, table_id: str) -> list[dict[str, str]]: - table = soup.find("table", id=table_id) - if table is None: - return [] - - headers: list[str] = [] - result: list[dict[str, str]] = [] - for row in table.find_all("tr"): - cells = row.find_all(["th", "td"], recursive=False) - if not cells: - cells = row.find_all(["th", "td"]) - values = [cell.get_text(" ", strip=True) for cell in cells] - if not values: - continue - - normalized = [cls._normalize_header(value) for value in values] - if not headers and cls._looks_like_header(normalized): - headers = normalized - continue - - if headers: - # Ignore malformed/placeholder rows rather than shifting cells. - if len(values) < len(headers): - continue - result.append(dict(zip(headers, values))) - else: - result.append({str(index): value for index, value in enumerate(values)}) - return result - - @staticmethod - def _looks_like_header(values: list[str]) -> bool: - known = set().union(*_ALIASES.values()) - return "channel" in values and any(value in known for value in values[1:]) - - @staticmethod - def _get(row: dict[str, str], field: str, default: str = "") -> str: - for alias in _ALIASES[field]: - if alias in row: - return row[alias] - return default - - @classmethod - def _is_locked(cls, row: dict[str, str]) -> bool: - status = cls._get(row, "lock") - if not status and "1" in row: - status = row["1"] - return status.strip().lower() == "locked" - - @classmethod - def _channel_id(cls, row: dict[str, str]) -> int | None: - value = cls._get(row, "channel_id") - if not value: - value = cls._get(row, "channel") or row.get("0", "") - return cls._parse_int(value) - - @classmethod - def _parse_downstream_tag_values(cls, raw: str) -> list[RawChannel]: - """Parse seven-field DOCSIS 3.0 downstream JavaScript rows.""" - rows = cls._split_tag_value_rows(raw) - if rows is None: - return [] - - result: list[RawChannel] = [] - for row in rows: - if row[1].strip().lower() != "locked": - continue - - channel_id = cls._parse_int(row[3]) - if channel_id is None: - continue - snr = cls._parse_float(row[6]) - modulation = normalize_modulation(row[2]) - channel: RawChannel = { - "channelID": channel_id, - "frequency": hz_to_mhz(row[4]), - "powerLevel": cls._parse_float(row[5]), - "mer": snr, - "mse": -snr if snr is not None else None, - "modulation": modulation, - "corrErrors": None, - "nonCorrErrors": None, - } - symbol_rate = _ANNEX_B_DOWNSTREAM_SYMBOL_RATES.get(modulation) - if symbol_rate is not None: - channel["symbolRate"] = symbol_rate - result.append(channel) - return result - - @classmethod - def _parse_upstream_tag_values(cls, raw: str) -> list[RawChannel]: - """Parse seven-field DOCSIS 3.0 upstream JavaScript rows.""" - rows = cls._split_tag_value_rows(raw) - if rows is None: - return [] - - result: list[RawChannel] = [] - for row in rows: - if row[1].strip().lower() != "locked": - continue - - channel_id = cls._parse_int(row[3]) - if channel_id is None: - continue - modulation = normalize_modulation(row[2]) - channel: RawChannel = { - "channelID": channel_id, - "frequency": hz_to_mhz(row[5]), - "powerLevel": cls._parse_float(row[6]), - "modulation": modulation, - "multiplex": modulation, - } - symbol_rate = cls._parse_int(row[4]) - if symbol_rate is not None: - channel["symbolRate"] = symbol_rate - result.append(channel) - return result - - @staticmethod - def _extract_function_body(html: str, function_name: str) -> str | None: - """Return the body of a named no-argument JavaScript function.""" - for match in _FUNCTION_START_RE.finditer(html): - if match.group("name") != function_name: - continue - - body_start = match.end() - depth = 1 - index = body_start - while index < len(html) and depth: - if html[index] == "{": - depth += 1 - elif html[index] == "}": - depth -= 1 - index += 1 - - if depth == 0: - return html[body_start : index - 1] - return None - return None - - @staticmethod - def _strip_javascript_comments(source: str) -> str: - """Remove JavaScript comments while preserving quoted string contents.""" - result: list[str] = [] - index = 0 - quote: str | None = None - - while index < len(source): - char = source[index] - if quote is not None: - result.append(char) - if char == "\\" and index + 1 < len(source): - index += 1 - result.append(source[index]) - elif char == quote: - quote = None - index += 1 - continue - - if char in {"'", '"'}: - quote = char - result.append(char) - index += 1 - continue - - if source.startswith("//", index): - newline = source.find("\n", index + 2) - if newline == -1: - break - result.append("\n") - index = newline + 1 - continue - - if source.startswith("/*", index): - comment_end = source.find("*/", index + 2) - if comment_end == -1: - break - result.append(" ") - index = comment_end + 2 - continue - - result.append(char) - index += 1 - - return "".join(result) - - @classmethod - def _extract_tag_value_list(cls, html: str, function_name: str) -> str | None: - """Extract the live, possibly concatenated tagValueList assignment.""" - body = cls._extract_function_body(html, function_name) - if body is None: - return None - - body = cls._strip_javascript_comments(body) - assignment = _TAG_VALUE_ASSIGNMENT_RE.search(body) - if assignment is None: - return None - - literals: list[str] = [] - for match in _STRING_LITERAL_RE.finditer(assignment.group("value")): - value = match.group("single") - if value is None: - value = match.group("double") - literals.append( - value.replace(r"\'", "'") - .replace(r'\"', '"') - .replace(r"\\", "\\") - ) - if not literals: - return None - - payload = "".join(literals) - return payload if cls._split_tag_value_rows(payload) is not None else None - - @staticmethod - def _split_tag_value_rows(raw: str) -> list[list[str]] | None: - """Validate and split a leading-count tagValueList into seven-field rows.""" - parts = raw.split("|") - count_prefix = parts[0].strip() - if not count_prefix.isdecimal(): - return None - - row_count = int(count_prefix) - values = parts[1:] - if values and not values[-1]: - values.pop() - if len(values) != row_count * _TAG_VALUE_FIELDS_PER_ROW: - return None - - return [ - values[index : index + _TAG_VALUE_FIELDS_PER_ROW] - for index in range(0, len(values), _TAG_VALUE_FIELDS_PER_ROW) - ] - - @classmethod - def _parse_downstream_table( - cls, soup: BeautifulSoup, table_id: str, *, docsis31: bool - ) -> list[RawChannel]: - result: list[RawChannel] = [] - for row in cls._table_rows(soup, table_id): - if not cls._is_locked(row): - continue - - positional = not any(key.isalpha() for key in row) - if positional: - row = cls._map_downstream_positional(row) - - channel_id = cls._channel_id(row) - if channel_id is None: - continue - frequency = cls._get(row, "frequency") - power = cls._parse_float(cls._get(row, "power")) - snr = cls._parse_float(cls._get(row, "snr")) - modulation_raw = cls._get(row, "modulation") - corr = cls._parse_int(cls._get(row, "corr")) - uncorr = cls._parse_int(cls._get(row, "uncorr")) - - if docsis31: - channel: RawChannel = { - "channelID": channel_id, - "type": "OFDM", - "frequency": hz_to_mhz(frequency), - "powerLevel": power, - "mer": snr, - "mse": None, - "modulation": "OFDM", - "corrErrors": corr, - "nonCorrErrors": uncorr, - } - else: - modulation = normalize_modulation(modulation_raw) - channel = { - "channelID": channel_id, - "frequency": hz_to_mhz(frequency), - "powerLevel": power, - "mer": snr, - "mse": -snr if snr is not None else None, - "modulation": modulation, - "corrErrors": corr, - "nonCorrErrors": uncorr, - } - symbol_rate = _ANNEX_B_DOWNSTREAM_SYMBOL_RATES.get(modulation) - if symbol_rate is not None: - channel["symbolRate"] = symbol_rate - result.append(channel) - return result - - @classmethod - def _parse_upstream_table( - cls, soup: BeautifulSoup, table_id: str, *, docsis31: bool - ) -> list[RawChannel]: - result: list[RawChannel] = [] - for row in cls._table_rows(soup, table_id): - if not cls._is_locked(row): - continue - - positional = not any(key.isalpha() for key in row) - if positional: - row = cls._map_upstream_positional(row) - - channel_id = cls._channel_id(row) - if channel_id is None: - continue - frequency = cls._get(row, "frequency") - power = cls._parse_float(cls._get(row, "power")) - modulation_raw = cls._get(row, "modulation") - modulation = normalize_modulation(modulation_raw) - - if docsis31: - channel: RawChannel = { - "channelID": channel_id, - "type": "OFDMA", - "frequency": hz_to_mhz(frequency), - "powerLevel": power, - "modulation": "OFDMA", - "multiplex": "", - } - else: - channel = { - "channelID": channel_id, - "frequency": hz_to_mhz(frequency), - "powerLevel": power, - "modulation": modulation, - "multiplex": modulation, - } - symbol_rate = cls._parse_int(cls._get(row, "symbol_rate")) - if symbol_rate is not None: - channel["symbolRate"] = symbol_rate - result.append(channel) - return result - - @classmethod - def _map_downstream_positional(cls, row: dict[str, str]) -> dict[str, str]: - values = [row[str(index)] for index in range(len(row))] - if len(values) < 9: - return row - mapped = { - "channel": values[0], - "lockstatus": values[1], - "modulation": values[2], - "channelid": values[3], - "frequency": values[4], - "power": values[5], - "snr": values[6], - } - # Ten/eleven-column layouts include Unerrored before Correctables. - if len(values) >= 10: - mapped["correctables"] = values[-2] - mapped["uncorrectables"] = values[-1] - else: - mapped["correctables"] = values[7] - mapped["uncorrectables"] = values[8] - return mapped - - @staticmethod - def _map_upstream_positional(row: dict[str, str]) -> dict[str, str]: - values = [row[str(index)] for index in range(len(row))] - if len(values) < 6: - return row - mapped = { - "channel": values[0], - "lockstatus": values[1], - "modulation": values[2], - "channelid": values[3], - } - if len(values) >= 7: - mapped["symbolrate"] = values[4] - mapped["frequency"] = values[5] - mapped["power"] = values[6] - else: - mapped["frequency"] = values[4] - mapped["power"] = values[5] - return mapped - - @staticmethod - def _parse_float(value: str) -> float | None: - if not value: - return None - match = _NUMBER_RE.search(value.replace(",", "")) - return float(match.group(0)) if match else None - - @classmethod - def _parse_int(cls, value: str) -> int | None: - number = cls._parse_float(value) - return int(number) if number is not None else None diff --git a/app/drivers/cm3000.py b/app/drivers/cm3000.py index fbb83a17..21800cc2 100644 --- a/app/drivers/cm3000.py +++ b/app/drivers/cm3000.py @@ -24,6 +24,16 @@ import requests from .base import ModemDriver +from .formats.javascript import ( + extract_cm3000_tag_value_list, + normalize_cm3000_modulation, + parse_cm3000_ds_ofdm, + parse_cm3000_ds_qam, + parse_cm3000_us_atdma, + parse_cm3000_us_ofdma, + split_cm3000_channels, +) +from .formats.primitives import hz_to_mhz, parse_number from ..types import DocsisData, DeviceInfo, ConnectionInfo, RawChannel log = logging.getLogger("docsis.driver.cm3000") @@ -70,6 +80,8 @@ class CM3000Driver(ModemDriver): DOCSIS data is extracted from JavaScript variables on /DocsisStatus.htm. """ + FORMAT_FAMILIES = ("cm3000_javascript",) + def __init__(self, url: str, user: str, password: str): super().__init__(url, user, password) self._session = requests.Session() @@ -353,238 +365,48 @@ def _log_status_page_diagnostics(html: str, context: str) -> None: diag["has_channel_data"], ) - # -- Channel parsers -- + # Compatibility parser seams; each delegates to the pure profile. def _parse_ds_qam(self, html: str) -> list[RawChannel]: - """Parse downstream SC-QAM channels from InitDsTableTagValue(). - - Per channel (9 fields): - num | lock | modulation | channelID | frequency | power | snr | corrErrors | uncorrErrors - """ - raw = self._extract_tag_value_list(html, "InitDsTableTagValue") - if not raw: - return [] - - channels = self._split_channels(raw, _DS_QAM_FIELDS) - result = [] - for ch in channels: - if ch[1] != "Locked": - continue - try: - result.append({ - "channelID": int(ch[3]), - "frequency": self._hz_to_mhz(ch[4]), - "powerLevel": float(ch[5]), - "mer": float(ch[6]), - "mse": -float(ch[6]), - "modulation": self._normalize_modulation(ch[2]), - "corrErrors": int(ch[7]), - "nonCorrErrors": int(ch[8]), - }) - except (ValueError, IndexError) as e: - log.warning("Failed to parse CM3000 DS QAM channel: %s", e) - return result + return parse_cm3000_ds_qam(html).value def _parse_us_atdma(self, html: str) -> list[RawChannel]: - """Parse upstream ATDMA channels from InitUsTableTagValue(). - - Per channel (7 fields): - num | lock | type | channelID | symbolRate | frequency | power - """ - raw = self._extract_tag_value_list(html, "InitUsTableTagValue") - if not raw: - return [] - - channels = self._split_channels(raw, _US_ATDMA_FIELDS) - result = [] - for ch in channels: - if ch[1] != "Locked": - continue - try: - result.append({ - "channelID": int(ch[3]), - "frequency": self._hz_to_mhz(ch[5]), - "powerLevel": self._parse_number(ch[6]), - "modulation": self._normalize_modulation(ch[2]), - "multiplex": ch[2].upper() if ch[2] else "", - }) - except (ValueError, IndexError) as e: - log.warning("Failed to parse CM3000 US ATDMA channel: %s", e) - return result + return parse_cm3000_us_atdma(html).value def _parse_ds_ofdm(self, html: str) -> list[RawChannel]: - """Parse downstream OFDM channels from InitDsOfdmTableTagValue(). - - Per channel (11 fields): - num | lock | profiles | channelID | frequency | power | snr | subcarriers | corrErrors | uncorrErrors | unknown - """ - raw = self._extract_tag_value_list(html, "InitDsOfdmTableTagValue") - if not raw: - return [] - - channels = self._split_channels(raw, _DS_OFDM_FIELDS) - result = [] - for ch in channels: - if ch[1] != "Locked": - continue - try: - result.append({ - "channelID": int(ch[3]), - "type": "OFDM", - "frequency": self._hz_to_mhz(ch[4]), - "powerLevel": self._parse_number(ch[5]), - "mer": self._parse_number(ch[6]), - "mse": None, - "corrErrors": int(ch[8]), - "nonCorrErrors": int(ch[9]), - }) - except (ValueError, IndexError) as e: - log.warning("Failed to parse CM3000 DS OFDM channel: %s", e) - return result + return parse_cm3000_ds_ofdm(html).value def _parse_us_ofdma(self, html: str) -> list[RawChannel]: - """Parse upstream OFDMA channels from InitUsOfdmaTableTagValue(). - - Per channel (6 fields): - num | lock | profiles | channelID | frequency | power - """ - raw = self._extract_tag_value_list(html, "InitUsOfdmaTableTagValue") - if not raw: - return [] - - channels = self._split_channels(raw, _US_OFDMA_FIELDS) - result = [] - for ch in channels: - if ch[1] != "Locked": - continue - try: - result.append({ - "channelID": int(ch[3]), - "type": "OFDMA", - "frequency": self._hz_to_mhz(ch[4]), - "powerLevel": self._parse_number(ch[5]), - "modulation": "OFDMA", - "multiplex": "", - }) - except (ValueError, IndexError) as e: - log.warning("Failed to parse CM3000 US OFDMA channel: %s", e) - return result - - # -- Value parsers -- + return parse_cm3000_us_ofdma(html).value @staticmethod def _extract_tag_value_list(html: str, function_name: str) -> str | None: - """Extract the live tagValueList payload from a firmware JS function. - - CM3000 firmware variants use different quoting styles and may build - the string across multiple concatenated literals. We extract the full - function body, remove block comments, and then join the string - literals from the live tagValueList assignment. - """ - body = CM3000Driver._extract_function_body(html, function_name) - if not body: - return None - - body = _RE_BLOCK_COMMENT.sub("", body) - assign_idx = body.find("var tagValueList") - if assign_idx == -1: - return None - - assign_expr = body[assign_idx:] - assign_expr = assign_expr.split("=", 1) - if len(assign_expr) != 2: - return None - - assign_expr = assign_expr[1] - return_idx = assign_expr.find("return tagValueList.split") - if return_idx != -1: - assign_expr = assign_expr[:return_idx] - assign_expr = assign_expr.strip().rstrip(";").strip() - - singles = _RE_SINGLE_QUOTED.findall(assign_expr) - doubles = _RE_DOUBLE_QUOTED.findall(assign_expr) - literals = singles + doubles - if not literals: - return None - - return "".join(bytes(value, "utf-8").decode("unicode_escape") for value in literals) - - @staticmethod - def _extract_function_body(html: str, function_name: str) -> str | None: - """Return the body text for a named JavaScript function.""" - for match in _RE_FUNCTION_START.finditer(html): - if match.group("name") != function_name: - continue - - body_start = match.end() - depth = 1 - idx = body_start - while idx < len(html) and depth > 0: - char = html[idx] - if char == "{": - depth += 1 - elif char == "}": - depth -= 1 - idx += 1 - - if depth == 0: - return html[body_start : idx - 1] - return None - return None + return extract_cm3000_tag_value_list(html, function_name) @staticmethod def _split_channels(raw: str, fields_per_channel: int) -> list[list[str]]: - """Split a pipe-delimited tagValueList into per-channel field lists. - - The first value is the channel count, followed by repeating groups - of ``fields_per_channel`` fields. - """ - parts = raw.split("|") - # First element is the count -- skip it - data = parts[1:] - # Remove trailing empty element from trailing pipe - if data and data[-1] == "": - data = data[:-1] - - channels = [] - for i in range(0, len(data), fields_per_channel): - chunk = data[i : i + fields_per_channel] - if len(chunk) == fields_per_channel: - channels.append(chunk) - return channels + return split_cm3000_channels(raw, fields_per_channel) @staticmethod def _hz_to_mhz(freq_str: str) -> str: - from .utils import hz_to_mhz return hz_to_mhz(freq_str) @staticmethod def _parse_number(value: str) -> float: - from .utils import parse_number return parse_number(value) @staticmethod def _normalize_modulation(mod: str) -> str: - """Normalize modulation string. - - 'QAM256' -> 'QAM256' - 'ATDMA' -> 'ATDMA' - We preserve the original format since the CM3500 driver does the same. - """ - return mod.strip() if mod else "" + return normalize_cm3000_modulation(mod) @staticmethod def _parse_uptime(uptime_str: str) -> int | None: - """Parse uptime string to seconds. - - '23 days 09:26:24' -> 2020784 - """ - m = re.match(r"(\d+)\s+days?\s+(\d+):(\d+):(\d+)", uptime_str.strip()) - if m: - return ( - int(m.group(1)) * 86400 - + int(m.group(2)) * 3600 - + int(m.group(3)) * 60 - + int(m.group(4)) - ) - return None + match = re.match(r"(\d+)\s+days?\s+(\d+):(\d+):(\d+)", uptime_str.strip()) + if not match: + return None + return ( + int(match.group(1)) * 86400 + + int(match.group(2)) * 3600 + + int(match.group(3)) * 60 + + int(match.group(4)) + ) diff --git a/app/drivers/cm3500.py b/app/drivers/cm3500.py index c8027ee9..0ae5011c 100644 --- a/app/drivers/cm3500.py +++ b/app/drivers/cm3500.py @@ -22,6 +22,16 @@ from bs4 import BeautifulSoup from .base import ModemDriver +from .formats.html_rows import ( + find_cm3500_sections, + format_cm3500_frequency, + parse_cm3500_ds_ofdm, + parse_cm3500_ds_qam, + parse_cm3500_html, + parse_cm3500_us_ofdm, + parse_cm3500_us_qam, +) +from .formats.primitives import parse_number from ..types import DocsisData, DeviceInfo, ConnectionInfo, RawChannel log = logging.getLogger("docsis.driver.cm3500") @@ -34,6 +44,8 @@ class CM3500Driver(ModemDriver): DOCSIS data is scraped from HTML tables. """ + FORMAT_FAMILIES = ("cm3500_html",) + def __init__(self, url: str, user: str, password: str): # CM3500B requires HTTPS; upgrade silently if user provided HTTP if url.startswith("http://"): @@ -76,27 +88,7 @@ def get_docsis_data(self) -> DocsisData: Returns pre-split format so the analyzer correctly labels QAM channels as DOCSIS 3.0 and OFDM/OFDMA channels as 3.1. """ - soup = self._fetch_status_page() - sections = self._find_table_sections(soup) - - ds30 = [] - ds31 = [] - if "downstream qam" in sections: - ds30.extend(self._parse_ds_qam(sections["downstream qam"])) - if "downstream ofdm" in sections: - ds31.extend(self._parse_ds_ofdm(sections["downstream ofdm"])) - - us30 = [] - us31 = [] - if "upstream qam" in sections: - us30.extend(self._parse_us_qam(sections["upstream qam"])) - if "upstream ofdm" in sections: - us31.extend(self._parse_us_ofdm(sections["upstream ofdm"])) - - return { - "channelDs": {"docsis30": ds30, "docsis31": ds31}, - "channelUs": {"docsis30": us30, "docsis31": us31}, - } + return parse_cm3500_html(self._fetch_status_page()).value def get_device_info(self) -> DeviceInfo: """Retrieve device info from status page.""" @@ -207,180 +199,30 @@ def _fetch_status_page(self) -> BeautifulSoup: return BeautifulSoup(r.text, "html.parser") def _find_table_sections(self, soup) -> dict[str, object]: - """Map

heading text to the following element.""" - sections = {} - for h4 in soup.find_all("h4"): - heading = h4.get_text(strip=True).lower() - table = h4.find_next_sibling("table") - if table: - sections[heading] = table - return sections + return find_cm3500_sections(soup) # -- Downstream parsers -- def _parse_ds_qam(self, table) -> list[RawChannel]: - """Parse Downstream QAM table. - - Columns: (label), DCID, Freq, Power, SNR, Modulation, Octets, - Correcteds, Uncorrectables - """ - rows = table.find_all("tr") - if len(rows) < 2: - return [] - - result = [] - for row in rows[1:]: - cells = [td.get_text(strip=True) for td in row.find_all("td")] - if len(cells) < 9: - continue - try: - result.append({ - "channelID": int(self._parse_number(cells[1])), - "frequency": self._format_freq(cells[2]), - "powerLevel": self._parse_number(cells[3]), - "mse": -self._parse_number(cells[4]) if cells[4] else None, - "mer": self._parse_number(cells[4]) if cells[4] else None, - "modulation": cells[5], - "corrErrors": int(self._parse_number(cells[7])), - "nonCorrErrors": int(self._parse_number(cells[8])), - }) - except (ValueError, TypeError, IndexError) as e: - log.warning("Failed to parse CM3500 DS QAM row: %s", e) - return result + return parse_cm3500_ds_qam(table).value def _parse_ds_ofdm(self, table) -> list[RawChannel]: - """Parse Downstream OFDM table. - - Columns: (label), FFT Type, Channel Width(MHz), # Active Subcarriers, - First Active Subcarrier(MHz), Last Active Subcarrier(MHz), - Average RxMER: Pilot, PLC, Data - """ - rows = table.find("tbody") - if not rows: - return [] - data_rows = rows.find_all("tr") - if not data_rows: - return [] - - result = [] - chan_id = 200 - for row in data_rows: - cells = [td.get_text(strip=True) for td in row.find_all("td")] - if len(cells) < 8: - continue - label = cells[0].lower() - if "downstream" not in label: - continue - try: - first_freq = self._parse_number(cells[4]) - last_freq = self._parse_number(cells[5]) - mer_data = self._parse_number(cells[8]) if len(cells) > 8 else self._parse_number(cells[7]) - - result.append({ - "channelID": chan_id, - "type": "OFDM", - "frequency": f"{int(first_freq)}-{int(last_freq)} MHz", - "powerLevel": None, - "mer": mer_data, - "mse": None, - "corrErrors": None, - "nonCorrErrors": None, - }) - chan_id += 1 - except (ValueError, TypeError, IndexError) as e: - log.warning("Failed to parse CM3500 DS OFDM row: %s", e) - return result + return parse_cm3500_ds_ofdm(table).value # -- Upstream parsers -- def _parse_us_qam(self, table) -> list[RawChannel]: - """Parse Upstream QAM table. - - Columns: (label), UCID, Freq, Power, Channel Type, Symbol Rate, Modulation - """ - rows = table.find_all("tr") - if len(rows) < 2: - return [] - - result = [] - for row in rows[1:]: - cells = [td.get_text(strip=True) for td in row.find_all("td")] - if len(cells) < 7: - continue - try: - channel_type = cells[4] - multiplex = "" - if "ATDMA" in channel_type.upper(): - multiplex = "ATDMA" - elif "TDMA" in channel_type.upper(): - multiplex = "TDMA" - - result.append({ - "channelID": int(self._parse_number(cells[1])), - "frequency": self._format_freq(cells[2]), - "powerLevel": self._parse_number(cells[3]), - "modulation": cells[6], - "multiplex": multiplex, - }) - except (ValueError, TypeError, IndexError) as e: - log.warning("Failed to parse CM3500 US QAM row: %s", e) - return result + return parse_cm3500_us_qam(table).value def _parse_us_ofdm(self, table) -> list[RawChannel]: - """Parse Upstream OFDM table. - - Columns: (label), FFT Type, Channel Width(MHz), # Active Subcarriers, - First Active Subcarrier, Last Active Subcarrier, - Lower Frequency(MHz), Upper Frequency(MHz), - Tx Power(dBmV) - """ - rows = table.find("tbody") - if not rows: - return [] - data_rows = rows.find_all("tr") - - result = [] - chan_id = 200 - for row in data_rows: - cells = [td.get_text(strip=True) for td in row.find_all("td")] - if len(cells) < 9: - continue - label = cells[0].lower() - if "upstream" not in label: - continue - try: - first_freq = self._parse_number(cells[6]) - last_freq = self._parse_number(cells[7]) - power = self._parse_number(cells[8]) - - result.append({ - "channelID": chan_id, - "type": "OFDMA", - "frequency": f"{int(first_freq)}-{int(last_freq)} MHz", - "powerLevel": power, - "modulation": "OFDMA", - "multiplex": "", - }) - chan_id += 1 - except (ValueError, TypeError, IndexError) as e: - log.warning("Failed to parse CM3500 US OFDM row: %s", e) - return result + return parse_cm3500_us_ofdm(table).value # -- Value parsers -- @staticmethod def _parse_number(value: str) -> float: - from .utils import parse_number return parse_number(value) @staticmethod def _format_freq(freq_str: str) -> str: - """Normalize frequency string to 'NNN MHz' format (always integer).""" - if not freq_str: - return "" - parts = freq_str.strip().split() - try: - mhz = float(parts[0]) - return f"{int(mhz)} MHz" - except (ValueError, IndexError): - return freq_str + return format_cm3500_frequency(freq_str) diff --git a/app/drivers/cm8200.py b/app/drivers/cm8200.py index 9e7e7d69..cab5bf37 100644 --- a/app/drivers/cm8200.py +++ b/app/drivers/cm8200.py @@ -48,6 +48,8 @@ class CM8200Driver(ModemDriver): hand-crafted Cookie header (malformed Set-Cookie workaround). """ + FORMAT_FAMILIES = ("arris_html",) + def __init__(self, url: str, user: str, password: str): if url.startswith("http://"): url = "https://" + url[len("http://"):] @@ -226,4 +228,3 @@ def _fetch_status_page(self) -> BeautifulSoup: self._check_lockout() self._credential_auth() return BeautifulSoup(self._status_html, "html.parser") - diff --git a/app/drivers/f3896lg.py b/app/drivers/f3896lg.py index 68429564..e24212ec 100644 --- a/app/drivers/f3896lg.py +++ b/app/drivers/f3896lg.py @@ -34,6 +34,8 @@ from ..types import ConnectionInfo, DeviceInfo, DocsisData, RawChannel from .base import ModemDriver +from .formats.sagemcom import parse_f3896lg_downstream, parse_f3896lg_upstream +from .format_compat import unwrap_f3896lg log = logging.getLogger("docsis.driver.f3896lg") @@ -43,6 +45,8 @@ class F3896LGDriver(ModemDriver): """Virgin Media Hub 5 / Sagemcom F3896LG (Liberty Global REST API).""" + FORMAT_FAMILIES = ("f3896lg_rest_json",) + def __init__(self, url: str, user: str, password: str): super().__init__(url.rstrip("/"), user, password) self._session = requests.Session() @@ -138,133 +142,10 @@ def get_connection_info(self) -> ConnectionInfo: log.warning("F3896LG connection info fetch failed: %s", e) return out - # -- parsing -- + # Compatibility parser seams. def _parse_downstream(self, channels: list[dict]) -> tuple[list[RawChannel], list[RawChannel]]: - ds30: list[RawChannel] = [] - ds31: list[RawChannel] = [] - for ch in channels: - if not isinstance(ch, dict): - continue - if not ch.get("lockStatus", False): - continue - channel_type = ch.get("channelType") - if channel_type not in {"sc_qam", "ofdm"}: - log.debug("Skipping unknown downstream channel type %r", channel_type) - continue - try: - mer = ch.get("rxMer") - if channel_type == "ofdm": - try: - power = self._unscale(ch.get("power")) - except (ValueError, TypeError): - log.warning("Invalid F3896LG OFDM power %r; using no power", ch.get("power")) - power = None - try: - mer = self._unscale(mer) - except (ValueError, TypeError): - log.warning("Invalid F3896LG OFDM rxMer %r; using no MER", mer) - mer = None - if mer == 0: - mer = None - profile_modulation = self._modulation(ch.get("modulation", "")) - # firstActiveSubcarrier is an index; without a subcarrier-zero/base - # frequency from the API, the channel frequency remains unknown. - channel: RawChannel = { - "channelID": ch.get("channelId", 0), - "type": "OFDM", - "frequency": "", - "powerLevel": power, - "mer": mer, - "mse": None, - "modulation": "OFDM", - "corrErrors": ch.get("correctedErrors"), - "nonCorrErrors": ch.get("uncorrectedErrors"), - } - if profile_modulation: - channel["profile_modulation"] = profile_modulation - ds31.append(channel) - else: - snr = ch.get("snr") or mer - ds30.append({ - "channelID": ch.get("channelId", 0), - "frequency": self._hz_to_mhz(ch.get("frequency")), - "powerLevel": ch.get("power"), - "mer": snr, - "mse": -snr if snr else None, - "modulation": self._modulation(ch.get("modulation", "")), - "corrErrors": ch.get("correctedErrors"), - "nonCorrErrors": ch.get("uncorrectedErrors"), - }) - except (ValueError, TypeError) as e: - log.warning("Failed to parse F3896LG DS channel %s: %s", ch, e) - return ds30, ds31 + return unwrap_f3896lg(parse_f3896lg_downstream(channels), channels, "downstream", log) def _parse_upstream(self, channels: list[dict]) -> tuple[list[RawChannel], list[RawChannel]]: - us30: list[RawChannel] = [] - us31: list[RawChannel] = [] - for ch in channels: - if not isinstance(ch, dict): - continue - if not ch.get("lockStatus", False): - continue - channel_type = ch.get("channelType") - if channel_type not in {"atdma", "ofdma"}: - log.debug("Skipping unknown upstream channel type %r", channel_type) - continue - try: - if channel_type == "ofdma": - try: - power = self._unscale(ch.get("power")) - except (ValueError, TypeError): - log.warning("Invalid F3896LG OFDMA power %r; using no power", ch.get("power")) - power = None - profile_modulation = self._modulation(ch.get("modulation", "")) - # firstActiveSubcarrier is an index; without a subcarrier-zero/base - # frequency from the API, the channel frequency remains unknown. - channel: RawChannel = { - "channelID": ch.get("channelId", 0), - "type": "OFDMA", - "frequency": "", - "powerLevel": power, - "modulation": "OFDMA", - "multiplex": "", - } - if profile_modulation: - channel["profile_modulation"] = profile_modulation - us31.append(channel) - else: - us30.append({ - "channelID": ch.get("channelId", 0), - "frequency": self._hz_to_mhz(ch.get("frequency")), - "powerLevel": ch.get("power"), - "modulation": self._modulation(ch.get("modulation", "")), - "multiplex": str(ch.get("channelType", "")).upper(), - "symbolRate": ch.get("symbolRate"), - }) - except (ValueError, TypeError) as e: - log.warning("Failed to parse F3896LG US channel %s: %s", ch, e) - return us30, us31 - - # -- helpers -- - - @staticmethod - def _hz_to_mhz(freq_hz) -> str: - if not freq_hz: - return "" - return f"{float(freq_hz) / 1_000_000:g} MHz" - - @staticmethod - def _unscale(power) -> float | None: - """OFDM/OFDMA power is reported x10 on this firmware.""" - if power is None: - return None - return float(power) / 10.0 - - @staticmethod - def _modulation(raw: str) -> str: - """qam_256 -> 256QAM (matches other drivers' display convention).""" - raw = (raw or "").lower() - if raw.startswith("qam_"): - return f"{raw[4:]}QAM" - return raw.upper() + return unwrap_f3896lg(parse_f3896lg_upstream(channels), channels, "upstream", log) diff --git a/app/drivers/format_compat.py b/app/drivers/format_compat.py new file mode 100644 index 00000000..8557d70a --- /dev/null +++ b/app/drivers/format_compat.py @@ -0,0 +1,32 @@ +"""Finite legacy logging adapters for driver-private parser seams.""" + +from __future__ import annotations + +from typing import Any + +from .formats.contract import ParseResult + + +def unwrap_f3896lg(result: ParseResult, rows: list[dict[str, Any]], direction: str, logger): + for issue in result.diagnostics: + row = rows[issue.index] if issue.index is not None and issue.index < len(rows) else {} + if issue.code == "unknown_channel_type": + logger.debug("Skipping unknown %s channel type %r", direction, row.get("channelType")) + elif issue.field == "rxMer": + logger.warning("Invalid F3896LG OFDM rxMer %r; using no MER", row.get("rxMer")) + elif issue.field == "power": + lane = "OFDM" if direction == "downstream" else "OFDMA" + logger.warning("Invalid F3896LG %s power %r; using no power", lane, row.get("power")) + return result.value + + +def unwrap_hitron(result: ParseResult, logger): + if any(issue.code == "missing_field" and issue.field == "repPower1_6" for issue in result.diagnostics): + logger.warning("Hitron CODA-56 OFDMA row missing repPower1_6; leaving power unsupported") + return result.value + + +def unwrap_sercom(result: ParseResult, logger): + if any(issue.code == "missing_field" and issue.field == "rep power1_6" for issue in result.diagnostics): + logger.warning("Sercom DM1000 OFDMA row missing rep power1_6; leaving power unsupported") + return result.value diff --git a/app/drivers/formats/__init__.py b/app/drivers/formats/__init__.py new file mode 100644 index 00000000..9e8cda79 --- /dev/null +++ b/app/drivers/formats/__init__.py @@ -0,0 +1,35 @@ +"""Pure, explicit modem payload format profiles.""" + +from __future__ import annotations + +from types import MappingProxyType + +from .contract import ParseDiagnostic, ParseResult + + +FORMAT_PROFILE_MODULES = MappingProxyType({ + "arris_html": "app.drivers.formats.html_rows", + "cgm4981_columnar_html": "app.drivers.formats.html_columnar", + "ch7465_xml": "app.drivers.formats.xml_payloads", + "cm1000_html_table": "app.drivers.formats.html_rows", + "cm1000_javascript": "app.drivers.formats.javascript", + "cm3000_javascript": "app.drivers.formats.javascript", + "cm3500_html": "app.drivers.formats.html_rows", + "f3896lg_rest_json": "app.drivers.formats.sagemcom", + "fritzbox_data_lua": "app.drivers.formats.fritzbox", + "generic_no_docsis": "app.drivers.formats.boundaries", + "hitron_coda4680_json": "app.drivers.formats.hitron", + "hitron_coda56_json": "app.drivers.formats.hitron", + "sagemcom_xmo_json": "app.drivers.formats.sagemcom", + "sb6141_transposed_html": "app.drivers.formats.html_transposed", + "sb6183_html": "app.drivers.formats.html_rows", + "sb6190_html": "app.drivers.formats.html_rows", + "sercom_dm1000_json": "app.drivers.formats.sercom", + "surfboard_hnap": "app.drivers.formats.surfboard", + "tc4400_html": "app.drivers.formats.html_rows", + "ultrahub7_json": "app.drivers.formats.vodafone", + "vodafone_station_cga_json": "app.drivers.formats.vodafone", + "vodafone_station_tg_embedded_json": "app.drivers.formats.vodafone", +}) + +__all__ = ["FORMAT_PROFILE_MODULES", "ParseDiagnostic", "ParseResult"] diff --git a/app/drivers/formats/boundaries.py b/app/drivers/formats/boundaries.py new file mode 100644 index 00000000..957fa781 --- /dev/null +++ b/app/drivers/formats/boundaries.py @@ -0,0 +1,10 @@ +"""Explicit non-parser boundaries registered alongside DOCSIS profiles.""" + +from __future__ import annotations + +from ...types import DocsisDataFritz +from .contract import ParseResult, docsis_split + + +def parse_generic_no_docsis() -> ParseResult[DocsisDataFritz]: + return ParseResult(docsis_split([], [], [], [])) diff --git a/app/drivers/formats/contract.py b/app/drivers/formats/contract.py new file mode 100644 index 00000000..33a6ea18 --- /dev/null +++ b/app/drivers/formats/contract.py @@ -0,0 +1,80 @@ +"""Immutable result contract for pure modem payload parsers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Generic, TypeVar + +from ...types import DocsisDataFritz, RawChannel + +T = TypeVar("T") + + +@dataclass(frozen=True, slots=True) +class ParseDiagnostic: + """Stable parser metadata that cannot contain payload or transport data.""" + + family: str + profile: str + code: str + direction: str | None = None + row: int | None = None + index: int | None = None + field: str | None = None + + +@dataclass(frozen=True, slots=True) +class ParseResult(Generic[T]): + """A normalized value plus immutable, payload-safe diagnostics.""" + + value: T + diagnostics: tuple[ParseDiagnostic, ...] = () + + +def docsis_split( + ds30: list[RawChannel], + ds31: list[RawChannel], + us30: list[RawChannel], + us31: list[RawChannel], +) -> DocsisDataFritz: + """Build the common four-lane result without hiding lane semantics.""" + return { + "channelDs": {"docsis30": ds30, "docsis31": ds31}, + "channelUs": {"docsis30": us30, "docsis31": us31}, + } + + +def docsis_result( + ds30: ParseResult[list[RawChannel]], + ds31: ParseResult[list[RawChannel]], + us30: ParseResult[list[RawChannel]], + us31: ParseResult[list[RawChannel]], +) -> ParseResult[DocsisDataFritz]: + """Combine four explicit lane results and their safe diagnostics.""" + results = (ds30, ds31, us30, us31) + return ParseResult( + docsis_split(ds30.value, ds31.value, us30.value, us31.value), + tuple(issue for result in results for issue in result.diagnostics), + ) + + +def diagnostic( + profile: str, + code: str, + *, + family: str, + direction: str | None = None, + row: int | None = None, + index: int | None = None, + field: str | None = None, +) -> ParseDiagnostic: + """Construct a diagnostic while keeping its finite schema explicit.""" + return ParseDiagnostic( + family=family, + profile=profile, + code=code, + direction=direction, + row=row, + index=index, + field=field, + ) diff --git a/app/drivers/formats/fritzbox.py b/app/drivers/formats/fritzbox.py new file mode 100644 index 00000000..73f569c4 --- /dev/null +++ b/app/drivers/formats/fritzbox.py @@ -0,0 +1,29 @@ +"""Pure normalization boundary for pre-normalized FritzBox data.lua payloads.""" + +from __future__ import annotations + +from copy import deepcopy +from typing import Any, cast + +from ...types import DocsisData +from .contract import ParseDiagnostic, ParseResult, diagnostic + + +_UPSTREAM_31_POWER_OFFSET = 6.0 + + +def parse_fritzbox_data_lua(payload: dict[str, Any] | None) -> ParseResult[DocsisData]: + """Copy the pre-normalized boundary and apply the existing US 3.1 correction.""" + value = cast(DocsisData, deepcopy(payload) if isinstance(payload, dict) else {}) + diagnostics: list[ParseDiagnostic] = [] + upstream = value.get("channelUs", {}).get("docsis31", []) + for index, channel in enumerate(upstream): + try: + raw = float(channel.get("powerLevel", 0)) + channel["powerLevel"] = str(round(raw + _UPSTREAM_31_POWER_OFFSET, 1)) + except (TypeError, ValueError): + diagnostics.append(diagnostic( + "fritzbox_data_lua", "invalid_field", family="fritzbox", + direction="upstream", index=index, field="powerLevel", + )) + return ParseResult(value, tuple(diagnostics)) diff --git a/app/drivers/formats/hitron.py b/app/drivers/formats/hitron.py new file mode 100644 index 00000000..389c7e17 --- /dev/null +++ b/app/drivers/formats/hitron.py @@ -0,0 +1,226 @@ +"""Pure structured-payload profiles for incompatible Hitron CODA APIs.""" + +from __future__ import annotations + +from collections.abc import Callable +from types import MappingProxyType +from typing import Any + +from ...types import DocsisDataFritz, RawChannel +from .contract import ParseDiagnostic, ParseResult, diagnostic, docsis_result +from .primitives import hz_to_mhz, normalize_modulation, parse_optional_finite_float + + +_CODA56_MODULATION = MappingProxyType({ + 0: "16QAM", 1: "64QAM", 2: "256QAM", 3: "1024QAM", + 4: "32QAM", 5: "128QAM", 6: "QPSK", +}) + + +def _invalid(profile: str, direction: str, index: int, field: str | None = None) -> ParseDiagnostic: + return diagnostic( + profile, "invalid_row", family="hitron", direction=direction, + index=index, field=field, + ) + + +def _parse_rows( + rows: list[Any], + profile: str, + direction: str, + build: Callable[[dict[str, Any]], RawChannel], + active: Callable[[dict[str, Any]], bool] | None = None, +) -> ParseResult[list[RawChannel]]: + channels: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, row in enumerate(rows): + if not isinstance(row, dict): + diagnostics.append(_invalid(profile, direction, index)) + continue + if active is not None and not active(row): + continue + try: + channels.append(build(row)) + except (ValueError, KeyError, TypeError): + diagnostics.append(_invalid(profile, direction, index)) + return ParseResult(channels, tuple(diagnostics)) + + +def _is_plc_locked(row: dict[str, Any]) -> bool: + return str(row.get("plclock", "")).strip().upper() == "YES" + + +def _coda56_ds_scqam(row: dict[str, Any]) -> RawChannel: + code = int(row.get("modulation", -1)) + snr = float(row["snr"]) + return { + "channelID": int(row["channelId"]), "frequency": hz_to_mhz(row["frequency"]), + "powerLevel": float(row["signalStrength"]), + "modulation": _CODA56_MODULATION.get(code, f"Unknown({code})"), + "mer": snr, "mse": -snr, + "corrErrors": int(row["correcteds"]), "nonCorrErrors": int(row["uncorrect"]), + } + + +def _coda56_us_scqam(row: dict[str, Any]) -> RawChannel: + return { + "channelID": int(row["channelId"]), "frequency": hz_to_mhz(row["frequency"]), + "powerLevel": float(row["signalStrength"]), + "modulation": row.get("modtype", ""), "multiplex": row.get("scdmaMode", ""), + } + + +def _coda56_ds_ofdm(row: dict[str, Any]) -> RawChannel: + return { + "channelID": int(row["receive"]), "type": "OFDM", + "frequency": hz_to_mhz(row.get("Subcarr0freqFreq", "0")), + "powerLevel": float(row["plcpower"]), "modulation": "OFDM", + "mer": float(row["SNR"]), "mse": None, + "corrErrors": int(row["correcteds"]), "nonCorrErrors": int(row["uncorrect"]), + } + + +def parse_coda56_ds_scqam(rows: list[dict[str, Any]]) -> ParseResult[list[RawChannel]]: + return _parse_rows(rows, "hitron_coda56_json", "downstream", _coda56_ds_scqam) + + +def parse_coda56_us_scqam(rows: list[dict[str, Any]]) -> ParseResult[list[RawChannel]]: + return _parse_rows(rows, "hitron_coda56_json", "upstream", _coda56_us_scqam) + + +def parse_coda56_ds_ofdm(rows: list[dict[str, Any]]) -> ParseResult[list[RawChannel]]: + return _parse_rows( + rows, + "hitron_coda56_json", + "downstream", + _coda56_ds_ofdm, + _is_plc_locked, + ) + + +def _ofdma_power(row: dict[str, Any], profile: str, index: int) -> tuple[float | None, ParseDiagnostic | None]: + if "repPower1_6" not in row: + return None, diagnostic( + profile, "missing_field", family="hitron", direction="upstream", + index=index, field="repPower1_6", + ) + return parse_optional_finite_float(row.get("repPower1_6")), None + + +def parse_hitron_ofdma_power(row: dict[str, Any]) -> float | None: + return parse_optional_finite_float(row.get("repPower1_6")) if "repPower1_6" in row else None + + +def parse_coda56_us_ofdma(rows: list[dict[str, Any]]) -> ParseResult[list[RawChannel]]: + channels: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, row in enumerate(rows): + if str(row.get("state", "")).strip().upper() != "OPERATE": + continue + try: + power, issue = _ofdma_power(row, "hitron_coda56_json", index) + if issue: + diagnostics.append(issue) + channels.append({ + "channelID": int(row["uschindex"]), "type": "OFDMA", + "frequency": hz_to_mhz(row.get("frequency", "0")), + "powerLevel": power, "modulation": "OFDMA", "multiplex": "", + }) + except (ValueError, KeyError, TypeError): + diagnostics.append(_invalid("hitron_coda56_json", "upstream", index)) + return ParseResult(channels, tuple(diagnostics)) + + +def parse_hitron_coda56_json( + payloads: dict[str, list[dict[str, Any]]] | None, +) -> ParseResult[DocsisDataFritz]: + data = payloads if isinstance(payloads, dict) else {} + return docsis_result( + parse_coda56_ds_scqam(data.get("downstream", [])), + parse_coda56_ds_ofdm(data.get("downstream_ofdm", [])), + parse_coda56_us_scqam(data.get("upstream", [])), + parse_coda56_us_ofdma(data.get("upstream_ofdma", [])), + ) + + +def _coda4680_ds_scqam(row: dict[str, Any]) -> RawChannel: + snr = float(row["snr"]) + return { + "channelID": int(row["channelId"]), "frequency": hz_to_mhz(row["frequency"]), + "powerLevel": float(row["signalStrength"]), + "modulation": normalize_modulation(row.get("modulation", "")), + "mer": snr, "mse": -snr, + "corrErrors": int(row["correcteds"]), "nonCorrErrors": int(row["uncorrect"]), + } + + +def _coda4680_us_scqam(row: dict[str, Any]) -> RawChannel: + modulation = normalize_modulation(row.get("modulationType") or row.get("modtype") or "") + channel: RawChannel = { + "channelID": int(row["channelId"]), "frequency": hz_to_mhz(row["frequency"]), + "powerLevel": float(row["signalStrength"]), "modulation": modulation, + "multiplex": "ATDMA", + } + if row.get("symbolrate") is not None: + channel["symbolRate"] = int(row["symbolrate"]) + return channel + + +def _coda4680_ds_ofdm(row: dict[str, Any]) -> RawChannel: + return { + "channelID": int(row["receive"]), "type": "OFDM", + "frequency": hz_to_mhz(row.get("Subcarr0freqFreq", "")), + "powerLevel": float(row["plcpower"]), "modulation": "OFDM", + "mer": None, "mse": None, "corrErrors": None, "nonCorrErrors": None, + } + + +def parse_coda4680_ds_scqam(rows: list[dict[str, Any]]) -> ParseResult[list[RawChannel]]: + return _parse_rows(rows, "hitron_coda4680_json", "downstream", _coda4680_ds_scqam) + + +def parse_coda4680_us_scqam(rows: list[dict[str, Any]]) -> ParseResult[list[RawChannel]]: + return _parse_rows(rows, "hitron_coda4680_json", "upstream", _coda4680_us_scqam) + + +def parse_coda4680_ds_ofdm(rows: list[dict[str, Any]]) -> ParseResult[list[RawChannel]]: + return _parse_rows( + rows, + "hitron_coda4680_json", + "downstream", + _coda4680_ds_ofdm, + _is_plc_locked, + ) + + +def parse_coda4680_us_ofdma(rows: list[dict[str, Any]]) -> ParseResult[list[RawChannel]]: + channels: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, row in enumerate(rows): + if str(row.get("state", "")).strip().upper() != "OPERATE": + continue + try: + power, issue = _ofdma_power(row, "hitron_coda4680_json", index) + if issue: + diagnostics.append(issue) + channels.append({ + "channelID": int(row["uschindex"]), "type": "OFDMA", + # This API does not expose OFDMA center frequency. + "frequency": "", "powerLevel": power, + "modulation": "OFDMA", "multiplex": "OFDMA", + }) + except (KeyError, TypeError, ValueError): + diagnostics.append(_invalid("hitron_coda4680_json", "upstream", index)) + return ParseResult(channels, tuple(diagnostics)) + + +def parse_hitron_coda4680_json( + payloads: dict[str, dict[str, Any]] | None, +) -> ParseResult[DocsisDataFritz]: + data = payloads if isinstance(payloads, dict) else {} + return docsis_result( + parse_coda4680_ds_scqam(data.get("downstream", {}).get("Freq_List", [])), + parse_coda4680_ds_ofdm(data.get("downstream_ofdm", {}).get("OFDMs_List", [])), + parse_coda4680_us_scqam(data.get("upstream", {}).get("Freq_List", [])), + parse_coda4680_us_ofdma(data.get("upstream_ofdma", {}).get("OFDMAs_List", [])), + ) diff --git a/app/drivers/formats/html_columnar.py b/app/drivers/formats/html_columnar.py new file mode 100644 index 00000000..53d95b9c --- /dev/null +++ b/app/drivers/formats/html_columnar.py @@ -0,0 +1,178 @@ +"""Pure parser for the CGM4981 columnar section-vector profile.""" + +from __future__ import annotations + +import re + +from ...docsis_utils import parse_qam_order +from ...types import DocsisDataFritz, RawChannel +from .contract import ParseDiagnostic, ParseResult, diagnostic + + +_TR = re.compile(r"]*>(.*?)", re.DOTALL | re.IGNORECASE) +_TH = re.compile(r"]*>(.*?)", re.DOTALL | re.IGNORECASE) +_NETWIDTH = re.compile(r']*class="netWidth"[^>]*>(.*?)', re.DOTALL) +_STRIP = re.compile(r"<[^>]+>") +_NUMBER = re.compile(r"-?\d+\.?\d*") + + +def _text(html: str) -> str: + return _STRIP.sub("", html).strip() + + +def _float(raw: str) -> float | None: + match = _NUMBER.search(raw.strip()) + return float(match.group()) if match else None + + +def _frequency(raw: str) -> str: + raw = raw.strip() + if re.search(r"[Mm][Hh][Zz]", raw): + number = _NUMBER.search(raw) + if number: + mhz = float(number.group()) + return f"{int(mhz) if mhz == int(mhz) else mhz} MHz" + match = _NUMBER.search(raw) + if match: + value = float(match.group()) + if value > 1_000_000: + value /= 1_000_000 + return f"{int(value) if value == int(value) else value} MHz" + return raw + + +def _modulation(raw: str) -> str: + upper = raw.strip().upper() + if "OFDMA" in upper: + return "OFDMA" + if "OFDM" in upper: + return "OFDM" + order = parse_qam_order(upper) + if order is not None: + return f"{order}QAM" + if "QAM" in upper: + return "QAM" + return raw.strip() + + +def section_rows(html: str) -> dict[str, list[str]]: + rows: dict[str, list[str]] = {} + for row_match in _TR.finditer(html or ""): + row = row_match.group(1) + heading = _TH.search(row) + if not heading: + continue + label = _text(heading.group(1)) + values = [_text(value) for value in _NETWIDTH.findall(row)] + if label and values: + rows[label] = values + return rows + + +def split_sections(html: str) -> tuple[str, str, str]: + downstream = html.find(">Downstream<") + upstream = html.find(">Upstream<") + errors = html.find("CM Error Codewords") + return ( + html[downstream:upstream] if downstream >= 0 and upstream > downstream else "", + html[upstream:errors] if upstream >= 0 and errors > upstream else "", + html[errors:] if errors >= 0 else "", + ) + + +def build_cgm4981_downstream( + rows: dict[str, list[str]], + error_rows: dict[str, list[str]], +) -> ParseResult[list[RawChannel]]: + channel_ids = rows.get("Channel ID", []) + locks = rows.get("Lock Status", []) + frequencies = rows.get("Frequency", []) + snrs = rows.get("SNR", []) + powers = rows.get("Power Level", []) + modulations = rows.get("Modulation", []) + error_ids = error_rows.get("Channel ID", []) + corrected = error_rows.get("Correctable Codewords", []) + uncorrected = error_rows.get("Uncorrectable Codewords", []) + error_map: dict[str, tuple[int, int]] = {} + for index, channel_id in enumerate(error_ids): + corr = int(corrected[index]) if index < len(corrected) and corrected[index].lstrip("-").isdigit() else 0 + uncorr = int(uncorrected[index]) if index < len(uncorrected) and uncorrected[index].lstrip("-").isdigit() else 0 + error_map[channel_id] = (corr, uncorr) + + result: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, channel_id in enumerate(channel_ids): + if (locks[index] if index < len(locks) else "").lower() != "locked": + continue + try: + modulation = _modulation(modulations[index] if index < len(modulations) else "") + snr = _float(snrs[index] if index < len(snrs) else "") + corr, uncorr = error_map.get(channel_id, (0, 0)) + channel: RawChannel = { + "channelID": int(channel_id), + "frequency": _frequency(frequencies[index] if index < len(frequencies) else ""), + "powerLevel": _float(powers[index] if index < len(powers) else ""), + "mer": snr, + "mse": -snr if snr else None, + "modulation": modulation, + "corrErrors": corr, + "nonCorrErrors": uncorr, + } + if modulation == "OFDM": + channel["type"] = "OFDM" + result.append(channel) + except (ValueError, IndexError): + diagnostics.append(diagnostic( + "cgm4981_columnar_html", "invalid_channel", family="html_columnar", + direction="downstream", index=index, + )) + return ParseResult(result, tuple(diagnostics)) + + +def build_cgm4981_upstream(rows: dict[str, list[str]]) -> ParseResult[list[RawChannel]]: + channel_ids = rows.get("Channel ID", []) + locks = rows.get("Lock Status", []) + frequencies = rows.get("Frequency", []) + powers = rows.get("Power Level", []) + modulations = rows.get("Modulation", []) + channel_types = rows.get("Channel Type", []) + result: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, channel_id in enumerate(channel_ids): + if (locks[index] if index < len(locks) else "").lower() != "locked": + continue + try: + modulation = _modulation(modulations[index] if index < len(modulations) else "") + raw_type = channel_types[index] if index < len(channel_types) else "" + channel: RawChannel = { + "channelID": int(channel_id), + "frequency": _frequency(frequencies[index] if index < len(frequencies) else ""), + "powerLevel": _float(powers[index] if index < len(powers) else ""), + "modulation": modulation, + "multiplex": raw_type.upper() or modulation, + } + if modulation == "OFDMA": + channel["type"] = "OFDMA" + result.append(channel) + except (ValueError, IndexError): + diagnostics.append(diagnostic( + "cgm4981_columnar_html", "invalid_channel", family="html_columnar", + direction="upstream", index=index, + )) + return ParseResult(result, tuple(diagnostics)) + + +def parse_cgm4981_columnar_html(html: str) -> ParseResult[DocsisDataFritz]: + downstream_html, upstream_html, errors_html = split_sections(html or "") + downstream = build_cgm4981_downstream(section_rows(downstream_html), section_rows(errors_html)) + upstream = build_cgm4981_upstream(section_rows(upstream_html)) + return ParseResult({ + "channelDs": { + "docsis30": [item for item in downstream.value if item.get("modulation") != "OFDM"], + "docsis31": [item for item in downstream.value if item.get("modulation") == "OFDM"], + }, + "channelUs": { + "docsis30": [item for item in upstream.value if item.get("modulation") != "OFDMA"], + "docsis31": [item for item in upstream.value if item.get("modulation") == "OFDMA"], + }, + }, downstream.diagnostics + upstream.diagnostics) diff --git a/app/drivers/formats/html_rows.py b/app/drivers/formats/html_rows.py new file mode 100644 index 00000000..421554cb --- /dev/null +++ b/app/drivers/formats/html_rows.py @@ -0,0 +1,676 @@ +"""Explicit parsers for compatible row-oriented modem HTML profiles.""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from types import MappingProxyType +from typing import Any, Literal + +from bs4 import BeautifulSoup, Tag + +from ...types import DocsisDataFritz, RawChannel +from .contract import ParseDiagnostic, ParseResult, diagnostic, docsis_result, docsis_split +from .primitives import ( + hz_to_mhz, + normalize_mhz, + normalize_modulation, + parse_mhz_value, + parse_number, +) + + +_NUMBER_RE = re.compile(r"[-+]?\d+(?:\.\d+)?") +_HEADER_RE = re.compile(r"[^a-z0-9]+") +_ANNEX_B_DOWNSTREAM_SYMBOL_RATES = MappingProxyType({"64QAM": 5057, "256QAM": 5361}) +_CM1000_ALIASES = MappingProxyType({ + "channel": frozenset({"channel", "channelnumber", "channelno"}), + "lock": frozenset({"lockstatus", "status"}), + "modulation": frozenset({ + "modulation", "channeltype", "uschanneltype", "profile", "profiles", + "profileid", "profileids", "profilemodulation", + }), + "channel_id": frozenset({"channelid", "id"}), + "frequency": frozenset({"frequency", "frequencyhz"}), + "power": frozenset({"power", "powerlevel", "powerdbmv"}), + "snr": frozenset({"snr", "mer", "snrmer"}), + "symbol_rate": frozenset({"symbolrate", "symbolrateksymsec"}), + "corr": frozenset({ + "correctables", "correctable", "correctablecodewords", "corrected", + "correctedcodewords", + }), + "uncorr": frozenset({ + "uncorrectables", "uncorrectable", "uncorrectablecodewords", "uncorrected", + "uncorrectedcodewords", + }), +}) + + +def _optional_value(value: str) -> float | None: + if not value: + return None + try: + return float(value.strip().split()[0]) + except (ValueError, IndexError): + return None + + +# Arris bonded 8/7-column profile ----------------------------------------- + +def _arris_tables(soup: BeautifulSoup) -> tuple[Tag | None, Tag | None]: + ds_table = us_table = None + for table in soup.find_all("table"): + header = table.find("tr") + if not header: + continue + text = header.get_text(strip=True).lower() + if "downstream bonded" in text: + ds_table = table + elif "upstream bonded" in text: + us_table = table + return ds_table, us_table + + +def _data_rows(table: Tag | None): + if not table: + return + for row in table.find_all("tr"): + if row.find("th") or row.find("strong"): + continue + yield row + + +def parse_arris_downstream(table: Tag | None) -> ParseResult[tuple[list[RawChannel], list[RawChannel]]]: + ds30: list[RawChannel] = [] + ds31: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for row_index, row in enumerate(_data_rows(table) or ()): + cells = [td.get_text(strip=True) for td in row.find_all("td")] + if len(cells) < 8 or cells[1] != "Locked": + continue + try: + snr = _optional_value(cells[5]) + channel: RawChannel = { + "channelID": int(cells[0]), + "frequency": hz_to_mhz(cells[3]), + "powerLevel": _optional_value(cells[4]), + "modulation": cells[2], + "corrErrors": int(cells[6]), + "nonCorrErrors": int(cells[7]), + } + if cells[2] == "Other": + channel.update({"type": "OFDM", "mer": snr, "mse": None}) + ds31.append(channel) + else: + channel.update({"mer": snr, "mse": -snr if snr is not None else None}) + ds30.append(channel) + except (ValueError, TypeError, IndexError): + diagnostics.append(diagnostic( + "arris_html", "invalid_row", family="html_rows", + direction="downstream", row=row_index, + )) + return ParseResult((ds30, ds31), tuple(diagnostics)) + + +def parse_arris_upstream(table: Tag | None) -> ParseResult[tuple[list[RawChannel], list[RawChannel]]]: + us30: list[RawChannel] = [] + us31: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for row_index, row in enumerate(_data_rows(table) or ()): + cells = [td.get_text(strip=True) for td in row.find_all("td")] + if len(cells) < 7 or cells[2] != "Locked": + continue + try: + channel_type = cells[3] + channel: RawChannel = { + "channelID": int(cells[1]), + "frequency": hz_to_mhz(cells[4]), + "powerLevel": _optional_value(cells[6]), + "modulation": channel_type, + } + if "OFDM" in channel_type and "SC-QAM" not in channel_type: + channel.update({"type": "OFDMA", "multiplex": ""}) + us31.append(channel) + else: + channel["multiplex"] = "SC-QAM" + us30.append(channel) + except (ValueError, TypeError, IndexError): + diagnostics.append(diagnostic( + "arris_html", "invalid_row", family="html_rows", + direction="upstream", row=row_index, + )) + return ParseResult((us30, us31), tuple(diagnostics)) + + +def parse_arris_html(html: str) -> ParseResult[DocsisDataFritz]: + soup = BeautifulSoup(html or "", "html.parser") + ds_table, us_table = _arris_tables(soup) + downstream = parse_arris_downstream(ds_table) + upstream = parse_arris_upstream(us_table) + ds30, ds31 = downstream.value + us30, us31 = upstream.value + return ParseResult( + docsis_split(ds30, ds31, us30, us31), + downstream.diagnostics + upstream.diagnostics, + ) + + +# SB6183 / SB6190 row profiles -------------------------------------------- + + +def _parse_sb_table( + table: Tag | None, + profile: str, + frequency_parser: Callable[[str], str], + direction: Literal["downstream", "upstream"], +) -> ParseResult[list[RawChannel]]: + result: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + if not table: + return ParseResult(result) + for row_index, tr in enumerate(table.find_all("tr")): + cells = [td.get_text(" ", strip=True) for td in tr.find_all("td")] + required = 9 if direction == "downstream" else 7 + if ( + len(cells) < required + or not cells[3].isdigit() + or cells[1].strip().lower() != "locked" + ): + continue + try: + if direction == "downstream": + snr = parse_number(cells[6]) + channel: RawChannel = { + "channelID": int(cells[3]), "frequency": frequency_parser(cells[4]), + "powerLevel": parse_number(cells[5]), "mer": snr, + "mse": -snr if snr else None, "modulation": cells[2], + "corrErrors": int(parse_number(cells[7])), + "nonCorrErrors": int(parse_number(cells[8])), + } + else: + channel = { + "channelID": int(cells[3]), "frequency": frequency_parser(cells[5]), + "powerLevel": parse_number(cells[6]), "modulation": cells[2], + "multiplex": cells[2], + } + result.append(channel) + except (ValueError, TypeError, IndexError): + diagnostics.append(diagnostic( + profile, "invalid_row", family="html_rows", + direction=direction, row=row_index, + )) + return ParseResult(result, tuple(diagnostics)) + + +def parse_sb6183_downstream(table: Tag | None) -> ParseResult[list[RawChannel]]: + return _parse_sb_table(table, "sb6183_html", hz_to_mhz, "downstream") + + +def parse_sb6183_upstream(table: Tag | None) -> ParseResult[list[RawChannel]]: + return _parse_sb_table(table, "sb6183_html", hz_to_mhz, "upstream") + + +def parse_sb6190_downstream(table: Tag | None) -> ParseResult[list[RawChannel]]: + return _parse_sb_table(table, "sb6190_html", normalize_mhz, "downstream") + + +def parse_sb6190_upstream(table: Tag | None) -> ParseResult[list[RawChannel]]: + return _parse_sb_table(table, "sb6190_html", normalize_mhz, "upstream") + + +def _bonded_tables(html: str) -> tuple[Tag | None, Tag | None]: + soup = BeautifulSoup(html or "", "html.parser") + ds_table = us_table = None + for table in soup.find_all("table"): + heading = table.find("th") + text = heading.get_text(" ", strip=True).lower() if heading else "" + if "downstream bonded" in text: + ds_table = table + elif "upstream bonded" in text: + us_table = table + return ds_table, us_table + + +def _parse_sb_html(html: str, profile: str) -> ParseResult[DocsisDataFritz]: + ds_table, us_table = _bonded_tables(html) + if profile == "sb6183_html": + downstream = parse_sb6183_downstream(ds_table) + upstream = parse_sb6183_upstream(us_table) + else: + downstream = parse_sb6190_downstream(ds_table) + upstream = parse_sb6190_upstream(us_table) + return ParseResult( + docsis_split(downstream.value, [], upstream.value, []), + downstream.diagnostics + upstream.diagnostics, + ) + + +def parse_sb6183_html(html: str) -> ParseResult[DocsisDataFritz]: + return _parse_sb_html(html, "sb6183_html") + + +def parse_sb6190_html(html: str) -> ParseResult[DocsisDataFritz]: + return _parse_sb_html(html, "sb6190_html") + + +# CM3500 heading-separated tables ----------------------------------------- + +def find_cm3500_sections(soup: BeautifulSoup) -> dict[str, Tag]: + sections: dict[str, Tag] = {} + for heading in soup.find_all("h4"): + table = heading.find_next_sibling("table") + if table: + sections[heading.get_text(strip=True).lower()] = table + return sections + + +def format_cm3500_frequency(value: str) -> str: + if not value: + return "" + try: + return f"{int(float(value.strip().split()[0]))} MHz" + except (ValueError, IndexError): + return value + + +def _cm3500_rows(table: Tag | None, lane: str) -> ParseResult[list[RawChannel]]: + result: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + if not table: + return ParseResult(result) + if lane in {"ds_ofdm", "us_ofdm"}: + body = table.find("tbody") + rows = body.find_all("tr") if body else [] + else: + rows = table.find_all("tr")[1:] + channel_id = 200 + for row_index, row in enumerate(rows): + cells = [td.get_text(strip=True) for td in row.find_all("td")] + try: + if lane == "ds_qam": + if len(cells) < 9: + continue + result.append({ + "channelID": int(parse_number(cells[1])), + "frequency": format_cm3500_frequency(cells[2]), + "powerLevel": parse_number(cells[3]), + "mse": -parse_number(cells[4]) if cells[4] else None, + "mer": parse_number(cells[4]) if cells[4] else None, + "modulation": cells[5], + "corrErrors": int(parse_number(cells[7])), + "nonCorrErrors": int(parse_number(cells[8])), + }) + elif lane == "ds_ofdm": + if len(cells) < 8 or "downstream" not in cells[0].lower(): + continue + first = parse_number(cells[4]) + last = parse_number(cells[5]) + mer = parse_number(cells[8]) if len(cells) > 8 else parse_number(cells[7]) + result.append({ + "channelID": channel_id, "type": "OFDM", + "frequency": f"{int(first)}-{int(last)} MHz", + "powerLevel": None, "mer": mer, "mse": None, + "corrErrors": None, "nonCorrErrors": None, + }) + channel_id += 1 + elif lane == "us_qam": + if len(cells) < 7: + continue + kind = cells[4].upper() + multiplex = "ATDMA" if "ATDMA" in kind else "TDMA" if "TDMA" in kind else "" + result.append({ + "channelID": int(parse_number(cells[1])), + "frequency": format_cm3500_frequency(cells[2]), + "powerLevel": parse_number(cells[3]), + "modulation": cells[6], "multiplex": multiplex, + }) + else: + if len(cells) < 9 or "upstream" not in cells[0].lower(): + continue + first = parse_number(cells[6]) + last = parse_number(cells[7]) + result.append({ + "channelID": channel_id, "type": "OFDMA", + "frequency": f"{int(first)}-{int(last)} MHz", + "powerLevel": parse_number(cells[8]), + "modulation": "OFDMA", "multiplex": "", + }) + channel_id += 1 + except (ValueError, TypeError, IndexError): + diagnostics.append(diagnostic( + "cm3500_html", "invalid_row", family="html_rows", + direction="downstream" if lane.startswith("ds") else "upstream", + row=row_index, + )) + return ParseResult(result, tuple(diagnostics)) + + +def parse_cm3500_ds_qam(table: Tag | None) -> ParseResult[list[RawChannel]]: + return _cm3500_rows(table, "ds_qam") + + +def parse_cm3500_ds_ofdm(table: Tag | None) -> ParseResult[list[RawChannel]]: + return _cm3500_rows(table, "ds_ofdm") + + +def parse_cm3500_us_qam(table: Tag | None) -> ParseResult[list[RawChannel]]: + return _cm3500_rows(table, "us_qam") + + +def parse_cm3500_us_ofdm(table: Tag | None) -> ParseResult[list[RawChannel]]: + return _cm3500_rows(table, "us_ofdm") + + +def parse_cm3500_html(html: str | BeautifulSoup) -> ParseResult[DocsisDataFritz]: + soup = html if isinstance(html, BeautifulSoup) else BeautifulSoup(html or "", "html.parser") + sections = find_cm3500_sections(soup) + return docsis_result( + parse_cm3500_ds_qam(sections.get("downstream qam")), + parse_cm3500_ds_ofdm(sections.get("downstream ofdm")), + parse_cm3500_us_qam(sections.get("upstream qam")), + parse_cm3500_us_ofdm(sections.get("upstream ofdm")), + ) + + +# TC4400 dynamic-header tables -------------------------------------------- + +def _tc_header_row(rows): + for row in rows: + cells = row.find_all(["th", "td"]) + if cells and any(cell.get("colspan") for cell in cells): + continue + if len(cells) > 3: + return row + return None + + +def _tc_columns(headers: list[str]) -> dict[str, int | None]: + columns = {key: None for key in ( + "channel_id", "lock_status", "modulation", "channel_type", "frequency", + "power", "snr", "corrected", "uncorrected", + )} + for index, header in enumerate(headers): + if "channel" in header and "id" in header: + columns["channel_id"] = index + elif "channel" in header and "index" in header and columns["channel_id"] is None: + columns["channel_id"] = index + elif "lock" in header: + columns["lock_status"] = index + elif "channel" in header and "type" in header: + columns["channel_type"] = index + elif "modulation" in header or "profile" in header: + columns["modulation"] = index + elif "freq" in header: + columns["frequency"] = index + elif any(word in header for word in ("power", "receive", "transmit")): + columns["power"] = index + elif "snr" in header or "mer" in header: + columns["snr"] = index + elif "corrected" in header and "un" not in header: + columns["corrected"] = index + elif "uncorrect" in header: + columns["uncorrected"] = index + columns["channel_id"] = 0 if columns["channel_id"] is None else columns["channel_id"] + columns["lock_status"] = 1 if columns["lock_status"] is None else columns["lock_status"] + if columns["channel_type"] is None and columns["modulation"] is None: + columns["modulation"] = 2 + columns["frequency"] = 3 if columns["frequency"] is None else columns["frequency"] + return columns + + +def _cell(cells: list[str], index: int | None, default: str = "") -> str: + return default if index is None or index >= len(cells) else cells[index] + + +def _parse_tc_table(table: Tag | None, direction: str) -> ParseResult[list[RawChannel]]: + if not table: + return ParseResult([]) + rows = table.find_all("tr") + header = _tc_header_row(rows) + if header is None: + return ParseResult([]) + columns = _tc_columns([cell.get_text(strip=True).lower() for cell in header.find_all(["th", "td"])]) + result: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for row_index, row in enumerate(item for item in rows if item != header): + cells = [td.get_text(strip=True) for td in row.find_all("td")] + if len(cells) < 4 or _cell(cells, columns["lock_status"]).lower() != "locked": + continue + try: + channel_id = _cell(cells, columns["channel_id"], "0") + modulation = normalize_modulation(_cell(cells, columns["modulation"])) + frequency = parse_mhz_value(_cell(cells, columns["frequency"])) + power = parse_number(_cell(cells, columns["power"])) + if direction == "downstream": + channel_type = _cell(cells, columns["channel_type"]) + if channel_type.upper() == "OFDM": + final_type = "OFDM" + elif channel_type.upper() == "SC-QAM": + final_type = modulation or "QAM" + else: + final_type = modulation or "unknown" + snr = parse_number(_cell(cells, columns["snr"])) + result.append({ + "channelID": channel_id, "type": final_type, + "frequency": f"{int(frequency)} MHz" if frequency else "", + "powerLevel": power, + "mse": None if final_type == "OFDM" else (-snr if snr else None), + "mer": snr if snr else None, "latency": 0, + "corrError": int(parse_number(_cell(cells, columns["corrected"]))), + "nonCorrError": int(parse_number(_cell(cells, columns["uncorrected"]))), + }) + else: + result.append({ + "channelID": channel_id, "type": modulation, + "frequency": f"{int(frequency)} MHz" if frequency else "", + "powerLevel": power, "multiplex": "", + }) + except (ValueError, TypeError, IndexError): + diagnostics.append(diagnostic( + "tc4400_html", "invalid_row", family="html_rows", + direction=direction, row=row_index, + )) + return ParseResult(result, tuple(diagnostics)) + + +def parse_tc4400_downstream(table: Tag | None) -> ParseResult[list[RawChannel]]: + return _parse_tc_table(table, "downstream") + + +def parse_tc4400_upstream(table: Tag | None) -> ParseResult[list[RawChannel]]: + return _parse_tc_table(table, "upstream") + + +def parse_tc4400_html(downstream_table: Tag | None, upstream_table: Tag | None) -> ParseResult[dict[str, Any]]: + downstream = parse_tc4400_downstream(downstream_table) + upstream = parse_tc4400_upstream(upstream_table) + return ParseResult( + {"docsis": "3.1", "downstream": downstream.value, "upstream": upstream.value}, + downstream.diagnostics + upstream.diagnostics, + ) + + +# CM1000 server-rendered tables ------------------------------------------- + +def _cm1000_get(row: dict[str, str], field: str, default: str = "") -> str: + for alias in _CM1000_ALIASES[field]: + if alias in row: + return row[alias] + return default + + +def normalize_cm1000_header(value: str) -> str: + return _HEADER_RE.sub("", value.lower()) + + +def cm1000_looks_like_header(values: list[str]) -> bool: + known = set().union(*_CM1000_ALIASES.values()) + return "channel" in values and any(value in known for value in values[1:]) + + +def cm1000_get(row: dict[str, str], field: str, default: str = "") -> str: + return _cm1000_get(row, field, default) + + +def _cm1000_float(value: str) -> float | None: + if not value: + return None + match = _NUMBER_RE.search(value.replace(",", "")) + return float(match.group(0)) if match else None + + +def _cm1000_int(value: str) -> int | None: + number = _cm1000_float(value) + return int(number) if number is not None else None + + +def parse_cm1000_float(value: str) -> float | None: + return _cm1000_float(value) + + +def parse_cm1000_int(value: str) -> int | None: + return _cm1000_int(value) + + +def cm1000_is_locked(row: dict[str, str]) -> bool: + status = _cm1000_get(row, "lock") or row.get("1", "") + return status.strip().lower() == "locked" + + +def cm1000_channel_id(row: dict[str, str]) -> int | None: + return _cm1000_int( + _cm1000_get(row, "channel_id") + or _cm1000_get(row, "channel") + or row.get("0", "") + ) + + +def cm1000_table_rows(soup: BeautifulSoup, table_id: str) -> list[dict[str, str]]: + table = soup.find("table", id=table_id) + if table is None: + return [] + headers: list[str] = [] + result: list[dict[str, str]] = [] + for row in table.find_all("tr"): + cells = row.find_all(["th", "td"], recursive=False) or row.find_all(["th", "td"]) + values = [cell.get_text(" ", strip=True) for cell in cells] + if not values: + continue + normalized = [normalize_cm1000_header(value) for value in values] + if not headers and cm1000_looks_like_header(normalized): + headers = normalized + continue + if headers: + if len(values) < len(headers): + continue + result.append(dict(zip(headers, values))) + else: + result.append({str(index): value for index, value in enumerate(values)}) + return result + + +def _cm1000_positional(row: dict[str, str], direction: str) -> dict[str, str]: + values = [row[str(index)] for index in range(len(row))] + if direction == "downstream": + if len(values) < 9: + return row + mapped = { + "channel": values[0], "lockstatus": values[1], "modulation": values[2], + "channelid": values[3], "frequency": values[4], "power": values[5], + "snr": values[6], + } + mapped["correctables"] = values[-2] if len(values) >= 10 else values[7] + mapped["uncorrectables"] = values[-1] if len(values) >= 10 else values[8] + return mapped + if len(values) < 6: + return row + mapped = { + "channel": values[0], "lockstatus": values[1], "modulation": values[2], + "channelid": values[3], + } + if len(values) >= 7: + mapped.update({"symbolrate": values[4], "frequency": values[5], "power": values[6]}) + else: + mapped.update({"frequency": values[4], "power": values[5]}) + return mapped + + +def map_cm1000_downstream_positional(row: dict[str, str]) -> dict[str, str]: + return _cm1000_positional(row, "downstream") + + +def map_cm1000_upstream_positional(row: dict[str, str]) -> dict[str, str]: + return _cm1000_positional(row, "upstream") + + +def _parse_cm1000_table( + soup: BeautifulSoup, table_id: str, *, direction: str, docsis31: bool, +) -> ParseResult[list[RawChannel]]: + result: list[RawChannel] = [] + for row in cm1000_table_rows(soup, table_id): + status = _cm1000_get(row, "lock") or row.get("1", "") + if status.strip().lower() != "locked": + continue + if not any(key.isalpha() for key in row): + row = _cm1000_positional(row, direction) + channel_id = _cm1000_int( + _cm1000_get(row, "channel_id") or _cm1000_get(row, "channel") or row.get("0", "") + ) + if channel_id is None: + continue + frequency = hz_to_mhz(_cm1000_get(row, "frequency")) + power = _cm1000_float(_cm1000_get(row, "power")) + modulation = normalize_modulation(_cm1000_get(row, "modulation")) + if direction == "downstream": + snr = _cm1000_float(_cm1000_get(row, "snr")) + if docsis31: + channel: RawChannel = { + "channelID": channel_id, "type": "OFDM", "frequency": frequency, + "powerLevel": power, "mer": snr, "mse": None, "modulation": "OFDM", + "corrErrors": _cm1000_int(_cm1000_get(row, "corr")), + "nonCorrErrors": _cm1000_int(_cm1000_get(row, "uncorr")), + } + else: + channel = { + "channelID": channel_id, "frequency": frequency, "powerLevel": power, + "mer": snr, "mse": -snr if snr is not None else None, + "modulation": modulation, + "corrErrors": _cm1000_int(_cm1000_get(row, "corr")), + "nonCorrErrors": _cm1000_int(_cm1000_get(row, "uncorr")), + } + symbol_rate = _ANNEX_B_DOWNSTREAM_SYMBOL_RATES.get(modulation) + if symbol_rate is not None: + channel["symbolRate"] = symbol_rate + elif docsis31: + channel = { + "channelID": channel_id, "type": "OFDMA", "frequency": frequency, + "powerLevel": power, "modulation": "OFDMA", "multiplex": "", + } + else: + channel = { + "channelID": channel_id, "frequency": frequency, "powerLevel": power, + "modulation": modulation, "multiplex": modulation, + } + symbol_rate = _cm1000_int(_cm1000_get(row, "symbol_rate")) + if symbol_rate is not None: + channel["symbolRate"] = symbol_rate + result.append(channel) + return ParseResult(result) + + +def parse_cm1000_downstream_table(soup: BeautifulSoup, table_id: str, *, docsis31: bool) -> ParseResult[list[RawChannel]]: + return _parse_cm1000_table(soup, table_id, direction="downstream", docsis31=docsis31) + + +def parse_cm1000_upstream_table(soup: BeautifulSoup, table_id: str, *, docsis31: bool) -> ParseResult[list[RawChannel]]: + return _parse_cm1000_table(soup, table_id, direction="upstream", docsis31=docsis31) + + +def parse_cm1000_html_table(html: str | BeautifulSoup) -> ParseResult[DocsisDataFritz]: + soup = html if isinstance(html, BeautifulSoup) else BeautifulSoup(html or "", "html.parser") + return docsis_result( + parse_cm1000_downstream_table(soup, "dsTable", docsis31=False), + parse_cm1000_downstream_table(soup, "d31dsTable", docsis31=True), + parse_cm1000_upstream_table(soup, "usTable", docsis31=False), + parse_cm1000_upstream_table(soup, "d31usTable", docsis31=True), + ) diff --git a/app/drivers/formats/html_transposed.py b/app/drivers/formats/html_transposed.py new file mode 100644 index 00000000..c9b6665f --- /dev/null +++ b/app/drivers/formats/html_transposed.py @@ -0,0 +1,138 @@ +"""Pure parser for the SB6141 transposed metric-table profile.""" + +from __future__ import annotations + +import re + +from bs4 import BeautifulSoup, Tag + +from ...types import DocsisDataFritz, RawChannel +from .contract import ParseDiagnostic, ParseResult, diagnostic, docsis_split +from .primitives import hz_to_mhz, parse_number + + +def extract_transposed_rows(table: Tag | None) -> list[tuple[str, list[str]]]: + if not table: + return [] + rows: list[tuple[str, list[str]]] = [] + for row in table.find_all("tr"): + if row.find("th"): + continue + cells = row.find_all("td", recursive=False) + if len(cells) >= 2: + rows.append(( + cells[0].get_text(strip=True), + [cell.get_text(strip=True) for cell in cells[1:]], + )) + return rows + + +def get_row_values(rows: list[tuple[str, list[str]]], keyword: str) -> list[str]: + keyword = keyword.lower() + for label, values in rows: + if keyword in label.lower(): + return values + return [] + + +def extract_upstream_modulation(raw: str) -> str: + if not raw: + return "" + last = "" + for part in re.split(r"[\n\r]+", raw.strip()): + cleaned = re.sub(r"^\[\d+\]\s*", "", part.strip()) + if cleaned: + last = cleaned + return last + + +def parse_sb6141_downstream( + downstream_table: Tag | None, + codeword_table: Tag | None, +) -> ParseResult[list[RawChannel]]: + if not downstream_table: + return ParseResult([]) + rows = extract_transposed_rows(downstream_table) + channel_ids = get_row_values(rows, "channel id") + frequencies = get_row_values(rows, "frequency") + snrs = get_row_values(rows, "signal to noise") + modulations = get_row_values(rows, "modulation") + powers = get_row_values(rows, "power level") + corrected: list[str] = [] + uncorrected: list[str] = [] + if codeword_table: + codewords = extract_transposed_rows(codeword_table) + corrected = get_row_values(codewords, "correctable") + uncorrected = get_row_values(codewords, "uncorrectable") + + result: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, raw_channel_id in enumerate(channel_ids): + try: + snr = parse_number(snrs[index] if index < len(snrs) else "") + result.append({ + "channelID": int(raw_channel_id), + "frequency": hz_to_mhz(frequencies[index] if index < len(frequencies) else ""), + "powerLevel": parse_number(powers[index] if index < len(powers) else ""), + "mer": snr, + "mse": -snr if snr else None, + "modulation": modulations[index].strip() if index < len(modulations) else "", + # Missing codewords deliberately remain zero for this profile. + "corrErrors": int(parse_number(corrected[index])) if index < len(corrected) else 0, + "nonCorrErrors": int(parse_number(uncorrected[index])) if index < len(uncorrected) else 0, + }) + except (ValueError, TypeError, IndexError): + diagnostics.append(diagnostic( + "sb6141_transposed_html", "invalid_channel", family="html_transposed", + direction="downstream", index=index, + )) + return ParseResult(result, tuple(diagnostics)) + + +def parse_sb6141_upstream(upstream_table: Tag | None) -> ParseResult[list[RawChannel]]: + if not upstream_table: + return ParseResult([]) + rows = extract_transposed_rows(upstream_table) + channel_ids = get_row_values(rows, "channel id") + frequencies = get_row_values(rows, "frequency") + powers = get_row_values(rows, "power level") + modulations = get_row_values(rows, "modulation") + result: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, raw_channel_id in enumerate(channel_ids): + try: + result.append({ + "channelID": int(raw_channel_id), + "frequency": hz_to_mhz(frequencies[index] if index < len(frequencies) else ""), + "powerLevel": parse_number(powers[index] if index < len(powers) else ""), + "modulation": extract_upstream_modulation( + modulations[index] if index < len(modulations) else "" + ), + "multiplex": "SC-QAM", + }) + except (ValueError, TypeError, IndexError): + diagnostics.append(diagnostic( + "sb6141_transposed_html", "invalid_channel", family="html_transposed", + direction="upstream", index=index, + )) + return ParseResult(result, tuple(diagnostics)) + + +def parse_sb6141_transposed_html(html: str) -> ParseResult[DocsisDataFritz]: + soup = BeautifulSoup(html or "", "html.parser") + downstream = upstream = codewords = None + for table in soup.find_all("table"): + heading = table.find("th") + text = heading.get_text(" ", strip=True).lower() if heading else "" + if "downstream" in text and "signal" not in text: + downstream = table + elif "upstream" in text: + upstream = table + elif "signal status" in text or "codeword" in text: + codewords = table + ds = parse_sb6141_downstream(downstream, codewords) + us = parse_sb6141_upstream(upstream) + return ParseResult( + docsis_split(ds.value, [], us.value, []), + ds.diagnostics + us.diagnostics, + ) diff --git a/app/drivers/formats/javascript.py b/app/drivers/formats/javascript.py new file mode 100644 index 00000000..8c570344 --- /dev/null +++ b/app/drivers/formats/javascript.py @@ -0,0 +1,293 @@ +"""Explicit JavaScript tag-list profiles for Netgear modem payloads.""" + +from __future__ import annotations + +import re +from types import MappingProxyType +from typing import Literal + +from ...types import DocsisDataFritz, RawChannel +from .contract import ParseDiagnostic, ParseResult, diagnostic, docsis_result, docsis_split +from .primitives import hz_to_mhz, normalize_modulation, parse_number + + +_FUNCTION_START = re.compile(r"function\s+(?P\w+)\s*\(\)\s*\{") +_CM1000_ASSIGNMENT = re.compile(r"\bvar\s+tagValueList\s*=\s*(?P.*?);", re.DOTALL) +_STRING_LITERAL = re.compile( + r"'(?P(?:\\.|[^'\\])*)'|\"(?P(?:\\.|[^\"\\])*)\"", + re.DOTALL, +) +_CM3000_SINGLE = re.compile(r"'([^'\\]*(?:\\.[^'\\]*)*)'", re.DOTALL) +_CM3000_DOUBLE = re.compile(r'"([^"\\]*(?:\\.[^"\\]*)*)"', re.DOTALL) +_BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL) +_NUMBER = re.compile(r"[-+]?\d+(?:\.\d+)?") +_CM1000_FIELDS = 7 +_CM3000_FIELDS = MappingProxyType({ + "InitDsTableTagValue": 9, + "InitUsTableTagValue": 7, + "InitDsOfdmTableTagValue": 11, + "InitUsOfdmaTableTagValue": 6, +}) +_ANNEX_B_DOWNSTREAM_SYMBOL_RATES = MappingProxyType({"64QAM": 5057, "256QAM": 5361}) + + +def extract_function_body(source: str, function_name: str) -> str | None: + """Extract one named function body with balanced braces.""" + for match in _FUNCTION_START.finditer(source or ""): + if match.group("name") != function_name: + continue + start = match.end() + depth = 1 + index = start + while index < len(source) and depth: + if source[index] == "{": + depth += 1 + elif source[index] == "}": + depth -= 1 + index += 1 + return source[start:index - 1] if depth == 0 else None + return None + + +def strip_javascript_comments(source: str) -> str: + """Remove comments while preserving quoted content for the CM1000 grammar.""" + result: list[str] = [] + index = 0 + quote: str | None = None + while index < len(source): + char = source[index] + if quote is not None: + result.append(char) + if char == "\\" and index + 1 < len(source): + index += 1 + result.append(source[index]) + elif char == quote: + quote = None + index += 1 + continue + if char in {"'", '"'}: + quote = char + result.append(char) + index += 1 + continue + if source.startswith("//", index): + newline = source.find("\n", index + 2) + if newline == -1: + break + result.append("\n") + index = newline + 1 + continue + if source.startswith("/*", index): + end = source.find("*/", index + 2) + if end == -1: + break + result.append(" ") + index = end + 2 + continue + result.append(char) + index += 1 + return "".join(result) + + +def split_cm1000_rows(raw: str) -> list[list[str]] | None: + parts = raw.split("|") + if not parts[0].strip().isdecimal(): + return None + row_count = int(parts[0].strip()) + values = parts[1:] + if values and not values[-1]: + values.pop() + if len(values) != row_count * _CM1000_FIELDS: + return None + return [values[index:index + _CM1000_FIELDS] for index in range(0, len(values), _CM1000_FIELDS)] + + +def extract_cm1000_tag_value_list(html: str, function_name: str) -> str | None: + body = extract_function_body(html, function_name) + if body is None: + return None + assignment = _CM1000_ASSIGNMENT.search(strip_javascript_comments(body)) + if assignment is None: + return None + literals: list[str] = [] + for match in _STRING_LITERAL.finditer(assignment.group("value")): + value = match.group("single") + if value is None: + value = match.group("double") + literals.append(value.replace(r"\'", "'").replace(r'\"', '"').replace(r"\\", "\\")) + if not literals: + return None + payload = "".join(literals) + return payload if split_cm1000_rows(payload) is not None else None + + +def _optional_number(value: str) -> float | None: + if not value: + return None + match = _NUMBER.search(value.replace(",", "")) + return float(match.group(0)) if match else None + + +def _parse_cm1000_tag_values( + raw: str, + direction: Literal["downstream", "upstream"], +) -> ParseResult[list[RawChannel]]: + rows = split_cm1000_rows(raw) + if rows is None: + return ParseResult([], (diagnostic( + "cm1000_javascript", "invalid_framing", family="javascript", + direction=direction, + ),)) + result: list[RawChannel] = [] + for row in rows: + if row[1].strip().lower() != "locked": + continue + channel_id = _optional_number(row[3]) + if channel_id is None: + continue + modulation = normalize_modulation(row[2]) + if direction == "downstream": + snr = _optional_number(row[6]) + channel: RawChannel = { + "channelID": int(channel_id), "frequency": hz_to_mhz(row[4]), + "powerLevel": _optional_number(row[5]), "mer": snr, + "mse": -snr if snr is not None else None, "modulation": modulation, + "corrErrors": None, "nonCorrErrors": None, + } + symbol_rate = _ANNEX_B_DOWNSTREAM_SYMBOL_RATES.get(modulation) + else: + channel = { + "channelID": int(channel_id), "frequency": hz_to_mhz(row[5]), + "powerLevel": _optional_number(row[6]), "modulation": modulation, + "multiplex": modulation, + } + number = _optional_number(row[4]) + symbol_rate = int(number) if number is not None else None + if symbol_rate is not None: + channel["symbolRate"] = symbol_rate + result.append(channel) + return ParseResult(result) + + +def parse_cm1000_downstream_tag_values(raw: str) -> ParseResult[list[RawChannel]]: + return _parse_cm1000_tag_values(raw, "downstream") + + +def parse_cm1000_upstream_tag_values(raw: str) -> ParseResult[list[RawChannel]]: + return _parse_cm1000_tag_values(raw, "upstream") + + +def parse_cm1000_javascript(html: str) -> ParseResult[DocsisDataFritz]: + downstream_raw = extract_cm1000_tag_value_list(html, "InitDsTableTagValue") + upstream_raw = extract_cm1000_tag_value_list(html, "InitUsTableTagValue") + downstream = parse_cm1000_downstream_tag_values(downstream_raw) if downstream_raw is not None else ParseResult([]) + upstream = parse_cm1000_upstream_tag_values(upstream_raw) if upstream_raw is not None else ParseResult([]) + return ParseResult( + docsis_split(downstream.value, [], upstream.value, []), + downstream.diagnostics + upstream.diagnostics, + ) + + +def extract_cm3000_tag_value_list(html: str, function_name: str) -> str | None: + body = extract_function_body(html, function_name) + if not body: + return None + body = _BLOCK_COMMENT.sub("", body) + assignment_index = body.find("var tagValueList") + if assignment_index == -1: + return None + parts = body[assignment_index:].split("=", 1) + if len(parts) != 2: + return None + expression = parts[1] + return_index = expression.find("return tagValueList.split") + if return_index != -1: + expression = expression[:return_index] + expression = expression.strip().rstrip(";").strip() + literals = _CM3000_SINGLE.findall(expression) + _CM3000_DOUBLE.findall(expression) + if not literals: + return None + return "".join(bytes(value, "utf-8").decode("unicode_escape") for value in literals) + + +def split_cm3000_channels(raw: str, fields_per_channel: int) -> list[list[str]]: + values = raw.split("|")[1:] + if values and values[-1] == "": + values = values[:-1] + return [ + values[index:index + fields_per_channel] + for index in range(0, len(values), fields_per_channel) + if len(values[index:index + fields_per_channel]) == fields_per_channel + ] + + +def normalize_cm3000_modulation(value: str) -> str: + return value.strip() if value else "" + + +def _parse_cm3000_lane(html: str, function_name: str) -> ParseResult[list[RawChannel]]: + raw = extract_cm3000_tag_value_list(html, function_name) + if not raw: + return ParseResult([]) + result: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + direction = "downstream" if function_name.startswith("InitDs") else "upstream" + for index, channel in enumerate(split_cm3000_channels(raw, _CM3000_FIELDS[function_name])): + if channel[1] != "Locked": + continue + try: + if function_name == "InitDsTableTagValue": + result.append({ + "channelID": int(channel[3]), "frequency": hz_to_mhz(channel[4]), + "powerLevel": float(channel[5]), "mer": float(channel[6]), + "mse": -float(channel[6]), "modulation": normalize_cm3000_modulation(channel[2]), + "corrErrors": int(channel[7]), "nonCorrErrors": int(channel[8]), + }) + elif function_name == "InitUsTableTagValue": + result.append({ + "channelID": int(channel[3]), "frequency": hz_to_mhz(channel[5]), + "powerLevel": parse_number(channel[6]), "modulation": normalize_cm3000_modulation(channel[2]), + "multiplex": channel[2].upper() if channel[2] else "", + }) + elif function_name == "InitDsOfdmTableTagValue": + result.append({ + "channelID": int(channel[3]), "type": "OFDM", + "frequency": hz_to_mhz(channel[4]), "powerLevel": parse_number(channel[5]), + "mer": parse_number(channel[6]), "mse": None, + "corrErrors": int(channel[8]), "nonCorrErrors": int(channel[9]), + }) + else: + result.append({ + "channelID": int(channel[3]), "type": "OFDMA", + "frequency": hz_to_mhz(channel[4]), "powerLevel": parse_number(channel[5]), + "modulation": "OFDMA", "multiplex": "", + }) + except (ValueError, IndexError): + diagnostics.append(diagnostic( + "cm3000_javascript", "invalid_channel", family="javascript", + direction=direction, index=index, + )) + return ParseResult(result, tuple(diagnostics)) + + +def parse_cm3000_ds_qam(html: str) -> ParseResult[list[RawChannel]]: + return _parse_cm3000_lane(html, "InitDsTableTagValue") + + +def parse_cm3000_us_atdma(html: str) -> ParseResult[list[RawChannel]]: + return _parse_cm3000_lane(html, "InitUsTableTagValue") + + +def parse_cm3000_ds_ofdm(html: str) -> ParseResult[list[RawChannel]]: + return _parse_cm3000_lane(html, "InitDsOfdmTableTagValue") + + +def parse_cm3000_us_ofdma(html: str) -> ParseResult[list[RawChannel]]: + return _parse_cm3000_lane(html, "InitUsOfdmaTableTagValue") + + +def parse_cm3000_javascript(html: str) -> ParseResult[DocsisDataFritz]: + return docsis_result( + parse_cm3000_ds_qam(html), parse_cm3000_ds_ofdm(html), + parse_cm3000_us_atdma(html), parse_cm3000_us_ofdma(html), + ) diff --git a/app/drivers/formats/primitives.py b/app/drivers/formats/primitives.py new file mode 100644 index 00000000..ea5ae236 --- /dev/null +++ b/app/drivers/formats/primitives.py @@ -0,0 +1,132 @@ +"""Small pure value primitives shared only where modem semantics agree.""" + +from __future__ import annotations + +import math +import re + + +_MOD_TOKEN_SPLIT = re.compile(r"[\s_\-]+") + + +def parse_number(value: str) -> float: + """Parse the leading number, preserving the established zero fallback.""" + if not value: + return 0.0 + parts = value.strip().split() + try: + return float(parts[0]) + except (ValueError, IndexError): + return 0.0 + + +def parse_optional_finite_float(value: object) -> float | None: + """Parse a finite float and preserve missing/invalid values as unsupported.""" + try: + number = float(str(value).strip()) + except (TypeError, ValueError): + return None + return number if math.isfinite(number) else None + + +def hz_to_mhz(freq: object) -> str: + """Convert numeric or unit-bearing frequency input to the legacy MHz string.""" + if isinstance(freq, (int, float)): + if freq == 0: + return "0 MHz" + mhz = float(freq) / 1_000_000 + if mhz == int(mhz): + return f"{int(mhz)} MHz" + return f"{mhz:.1f} MHz" + + freq_str = str(freq).strip() + if not freq_str: + return "" + parts = freq_str.split() + try: + val = float(parts[0]) + except (ValueError, IndexError): + return freq_str + + unit = parts[1].lower() if len(parts) > 1 else "" + if unit == "hz": + mhz = val / 1_000_000 + elif unit == "khz": + mhz = val / 1_000 + elif unit == "mhz": + mhz = val + elif val > 1_000_000: + mhz = val / 1_000_000 + elif val > 1_000: + mhz = val / 1_000 + else: + mhz = val + + if mhz == int(mhz): + return f"{int(mhz)} MHz" + return f"{mhz:.1f} MHz" + + +def normalize_modulation(modulation: object) -> str: + """Normalize the finite modulation spellings used across compatible profiles.""" + if modulation is None: + return "" + raw = str(modulation).strip() + if not raw: + return "" + mod = _MOD_TOKEN_SPLIT.sub("", raw).lower() + if not mod: + return raw.upper() + if "qpsk" in mod: + return "QPSK" + if "ofdma" in mod: + return "OFDMA" + if "ofdm" in mod: + return "OFDM" + if "atdma" in mod: + return "ATDMA" + if mod == "tdma": + return "TDMA" + if "qam" in mod: + number = mod.replace("qam", "") + if number.isdigit(): + return f"{number}QAM" + return "QAM" if not number else f"{number.upper()}QAM" + return raw.upper() + + +def normalize_mhz(freq_str: str) -> str: + """Normalize an already-MHz value to the established display string.""" + if not freq_str: + return "" + parts = freq_str.strip().split() + try: + mhz = float(parts[0]) + if mhz == int(mhz): + return f"{int(mhz)} MHz" + return f"{mhz:.1f} MHz" + except (ValueError, IndexError): + return freq_str + + +def parse_mhz_value(freq_str: str) -> float: + """Parse Hz/kHz/MHz input to a numeric MHz value with the legacy zero fallback.""" + if not freq_str: + return 0.0 + parts = freq_str.strip().split() + try: + value = float(parts[0]) + except (IndexError, ValueError): + return 0.0 + unit = parts[1].lower() if len(parts) > 1 else "" + if unit == "hz": + return value / 1_000_000 + if unit == "khz": + return value / 1_000 + if unit == "mhz": + return value + if value > 1_000_000: + return value / 1_000_000 + if value > 1_000: + return value / 1_000 + return value diff --git a/app/drivers/formats/sagemcom.py b/app/drivers/formats/sagemcom.py new file mode 100644 index 00000000..59779263 --- /dev/null +++ b/app/drivers/formats/sagemcom.py @@ -0,0 +1,267 @@ +"""Pure profiles for the incompatible Sagemcom XMO and F3896LG REST payloads.""" + +from __future__ import annotations + +from typing import Any + +from ...types import DocsisDataFritz, RawChannel +from .contract import ParseDiagnostic, ParseResult, diagnostic, docsis_split +from .primitives import hz_to_mhz + + +def _sagemcom_frequency(value: object) -> str: + return hz_to_mhz(value) if value else "" + + +def _sagemcom_modulation(value: str) -> str: + if not value: + return "" + stripped = value.strip() + return f"{stripped[3:]}QAM" if stripped.lower().startswith("qam") else stripped + + +def _sagemcom_is_ofdm(modulation: str, bandwidth: int) -> bool: + return bool(bandwidth and bandwidth > 8_000_000) or bool( + modulation and modulation.startswith("256-QAM") + ) + + +def parse_sagemcom_xmo_downstream( + rows: list[Any], +) -> ParseResult[tuple[list[RawChannel], list[RawChannel]]]: + docsis30: list[RawChannel] = [] + docsis31: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, row in enumerate(rows): + if not isinstance(row, dict): + diagnostics.append(diagnostic( + "sagemcom_xmo_json", "invalid_channel", family="sagemcom", + direction="downstream", index=index, + )) + continue + if not row.get("LockStatus", False): + continue + try: + channel_id = row.get("ChannelID", 0) + frequency = _sagemcom_frequency(row.get("Frequency", 0)) + power = row.get("PowerLevel", 0) + snr = row.get("SNR", 0) + modulation = row.get("Modulation", "") + corrected = row.get("CorrectableCodewords", 0) + uncorrected = row.get("UncorrectableCodewords", 0) + if _sagemcom_is_ofdm(modulation, row.get("BandWidth", 0)): + docsis31.append({ + "channelID": channel_id, "type": "OFDM", "frequency": frequency, + "powerLevel": power, "mer": snr, "mse": None, + "corrErrors": corrected, "nonCorrErrors": uncorrected, + }) + else: + docsis30.append({ + "channelID": channel_id, "frequency": frequency, "powerLevel": power, + "mer": snr, "mse": -snr if snr else None, + "modulation": _sagemcom_modulation(modulation), + "corrErrors": corrected, "nonCorrErrors": uncorrected, + }) + except (ValueError, TypeError): + diagnostics.append(diagnostic( + "sagemcom_xmo_json", "invalid_channel", family="sagemcom", + direction="downstream", index=index, + )) + return ParseResult((docsis30, docsis31), tuple(diagnostics)) + + +def parse_sagemcom_xmo_upstream( + rows: list[Any], +) -> ParseResult[tuple[list[RawChannel], list[RawChannel]] | None]: + docsis30: list[RawChannel] = [] + docsis31: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, row in enumerate(rows): + if not isinstance(row, dict): + diagnostics.append(diagnostic( + "sagemcom_xmo_json", "invalid_channel", family="sagemcom", + direction="upstream", index=index, + )) + continue + if not row.get("LockStatus", False): + continue + try: + channel_id = row.get("ChannelID", 0) + frequency = _sagemcom_frequency(row.get("Frequency", 0)) + power = row.get("PowerLevel", 0) + modulation = row.get("Modulation", "") + if not isinstance(modulation, str): + diagnostics.append(diagnostic( + "sagemcom_xmo_json", "invalid_field", family="sagemcom", + direction="upstream", index=index, field="Modulation", + )) + return ParseResult(None, tuple(diagnostics)) + if modulation.lower() == "ofdma": + docsis31.append({ + "channelID": channel_id, "type": "OFDMA", "frequency": frequency, + "powerLevel": power, "modulation": "OFDMA", "multiplex": "", + }) + else: + docsis30.append({ + "channelID": channel_id, "frequency": frequency, "powerLevel": power, + "modulation": modulation.strip().upper() if modulation else "", + "multiplex": modulation.upper() if modulation else "", + }) + except (ValueError, TypeError): + diagnostics.append(diagnostic( + "sagemcom_xmo_json", "invalid_channel", family="sagemcom", + direction="upstream", index=index, + )) + return ParseResult((docsis30, docsis31), tuple(diagnostics)) + + +def parse_sagemcom_xmo_json( + payload: dict[str, list[dict[str, Any]]] | None, +) -> ParseResult[DocsisDataFritz | None]: + data = payload if isinstance(payload, dict) else {} + downstream = parse_sagemcom_xmo_downstream(data.get("downstream", [])) + upstream = parse_sagemcom_xmo_upstream(data.get("upstream", [])) + if upstream.value is None: + return ParseResult(None, downstream.diagnostics + upstream.diagnostics) + return ParseResult( + docsis_split(*downstream.value, *upstream.value), + downstream.diagnostics + upstream.diagnostics, + ) + + +def _f3896_frequency(value: object) -> str: + return "" if not value else f"{float(value) / 1_000_000:g} MHz" + + +def _f3896_unscale(value: object) -> float | None: + return None if value is None else float(value) / 10.0 + + +def _f3896_modulation(value: str) -> str: + raw = (value or "").lower() + return f"{raw[4:]}QAM" if raw.startswith("qam_") else raw.upper() + + +def parse_f3896lg_downstream(rows: list[dict[str, Any]]) -> ParseResult[tuple[list[RawChannel], list[RawChannel]]]: + docsis30: list[RawChannel] = [] + docsis31: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, row in enumerate(rows): + if not isinstance(row, dict) or not row.get("lockStatus", False): + continue + channel_type = row.get("channelType") + if channel_type not in {"sc_qam", "ofdm"}: + diagnostics.append(diagnostic( + "f3896lg_rest_json", "unknown_channel_type", family="sagemcom", + direction="downstream", index=index, field="channelType", + )) + continue + try: + mer = row.get("rxMer") + if channel_type == "ofdm": + try: + power = _f3896_unscale(row.get("power")) + except (ValueError, TypeError): + power = None + diagnostics.append(diagnostic( + "f3896lg_rest_json", "invalid_field", family="sagemcom", + direction="downstream", index=index, field="power", + )) + try: + mer = _f3896_unscale(mer) + except (ValueError, TypeError): + mer = None + diagnostics.append(diagnostic( + "f3896lg_rest_json", "invalid_field", family="sagemcom", + direction="downstream", index=index, field="rxMer", + )) + if mer == 0: + mer = None + channel: RawChannel = { + "channelID": row.get("channelId", 0), "type": "OFDM", "frequency": "", + "powerLevel": power, "mer": mer, "mse": None, "modulation": "OFDM", + "corrErrors": row.get("correctedErrors"), + "nonCorrErrors": row.get("uncorrectedErrors"), + } + profile = _f3896_modulation(row.get("modulation", "")) + if profile: + channel["profile_modulation"] = profile + docsis31.append(channel) + else: + snr = row.get("snr") or mer + docsis30.append({ + "channelID": row.get("channelId", 0), + "frequency": _f3896_frequency(row.get("frequency")), + "powerLevel": row.get("power"), "mer": snr, + "mse": -snr if snr else None, + "modulation": _f3896_modulation(row.get("modulation", "")), + "corrErrors": row.get("correctedErrors"), + "nonCorrErrors": row.get("uncorrectedErrors"), + }) + except (ValueError, TypeError): + diagnostics.append(diagnostic( + "f3896lg_rest_json", "invalid_channel", family="sagemcom", + direction="downstream", index=index, + )) + return ParseResult((docsis30, docsis31), tuple(diagnostics)) + + +def parse_f3896lg_upstream(rows: list[dict[str, Any]]) -> ParseResult[tuple[list[RawChannel], list[RawChannel]]]: + docsis30: list[RawChannel] = [] + docsis31: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, row in enumerate(rows): + if not isinstance(row, dict) or not row.get("lockStatus", False): + continue + channel_type = row.get("channelType") + if channel_type not in {"atdma", "ofdma"}: + diagnostics.append(diagnostic( + "f3896lg_rest_json", "unknown_channel_type", family="sagemcom", + direction="upstream", index=index, field="channelType", + )) + continue + try: + if channel_type == "ofdma": + try: + power = _f3896_unscale(row.get("power")) + except (ValueError, TypeError): + power = None + diagnostics.append(diagnostic( + "f3896lg_rest_json", "invalid_field", family="sagemcom", + direction="upstream", index=index, field="power", + )) + channel: RawChannel = { + "channelID": row.get("channelId", 0), "type": "OFDMA", "frequency": "", + "powerLevel": power, "modulation": "OFDMA", "multiplex": "", + } + profile = _f3896_modulation(row.get("modulation", "")) + if profile: + channel["profile_modulation"] = profile + docsis31.append(channel) + else: + docsis30.append({ + "channelID": row.get("channelId", 0), + "frequency": _f3896_frequency(row.get("frequency")), + "powerLevel": row.get("power"), + "modulation": _f3896_modulation(row.get("modulation", "")), + "multiplex": str(row.get("channelType", "")).upper(), + "symbolRate": row.get("symbolRate"), + }) + except (ValueError, TypeError): + diagnostics.append(diagnostic( + "f3896lg_rest_json", "invalid_channel", family="sagemcom", + direction="upstream", index=index, + )) + return ParseResult((docsis30, docsis31), tuple(diagnostics)) + + +def parse_f3896lg_rest_json( + payload: dict[str, list[dict[str, Any]]] | None, +) -> ParseResult[DocsisDataFritz]: + data = payload if isinstance(payload, dict) else {} + downstream = parse_f3896lg_downstream(data.get("downstream", [])) + upstream = parse_f3896lg_upstream(data.get("upstream", [])) + return ParseResult( + docsis_split(*downstream.value, *upstream.value), + downstream.diagnostics + upstream.diagnostics, + ) diff --git a/app/drivers/formats/sercom.py b/app/drivers/formats/sercom.py new file mode 100644 index 00000000..bd78ac10 --- /dev/null +++ b/app/drivers/formats/sercom.py @@ -0,0 +1,171 @@ +"""Pure indexed-node parser for the Sercom DM1000 payload grammar.""" + +from __future__ import annotations + +import math +from typing import Any + +from ...types import DocsisDataFritz, RawChannel +from .contract import ParseDiagnostic, ParseResult, diagnostic, docsis_result +from .primitives import hz_to_mhz, normalize_modulation, parse_optional_finite_float + + +def _issue(direction: str, index: int, field: str | None = None, code: str = "invalid_row") -> ParseDiagnostic: + return diagnostic( + "sercom_dm1000_json", code, family="sercom", + direction=direction, index=index, field=field, + ) + + +def parse_sercom_ds_scqam(rows: list[dict[str, Any]]) -> ParseResult[list[RawChannel]]: + channels: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, row in enumerate(rows): + try: + modulation = normalize_modulation(row.get("qamD", "")) + if not modulation or modulation in {"QAM_NONE", "NONE"}: + continue + snr = float(row["SNRD"]) + channels.append({ + "channelID": int(row["DCIDD"]), "frequency": hz_to_mhz(row.get("FreqD", "")), + "powerLevel": float(row["PowerD"]), "modulation": modulation, + "mer": snr, "mse": -snr, + "corrErrors": int(row["correctedsD"]), + "nonCorrErrors": int(row["uncorrectedsD"]), + }) + except (KeyError, TypeError, ValueError): + diagnostics.append(_issue("downstream", index)) + return ParseResult(channels, tuple(diagnostics)) + + +def parse_sercom_ds_ofdm(rows: list[dict[str, Any]]) -> ParseResult[list[RawChannel]]: + channels: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, row in enumerate(rows): + if str(row.get("PLC", "")).strip().upper() != "YES": + continue + if str(row.get("MDC1", "")).strip().upper() != "YES": + continue + try: + mer = parse_optional_finite_float(row.get("AV_Data")) + if mer is None: + mer = parse_optional_finite_float(row.get("AV_PLC")) + if mer is None: + continue + channels.append({ + "channelID": int(row["num"]), "type": "OFDM", + "frequency": hz_to_mhz(row.get("OFDMFreq", "")), + "powerLevel": float(row["PLC_power"]), "modulation": "OFDM", + "mer": mer, "mse": None, "corrErrors": None, "nonCorrErrors": None, + }) + except (KeyError, TypeError, ValueError): + diagnostics.append(_issue("downstream", index)) + return ParseResult(channels, tuple(diagnostics)) + + +def _symbol_rate(value: Any) -> int | None: + number = parse_optional_finite_float(value) + return int(round(number * 1000)) if number is not None else None + + +def parse_sercom_us_scqam(rows: list[dict[str, Any]]) -> ParseResult[list[RawChannel]]: + channels: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, row in enumerate(rows): + try: + modulation = normalize_modulation(row.get("modulation", "")) + upstream = str(row.get("upstream", "")).strip() + rate = str(row.get("rate", "")).strip() + power = float(row["rep_power"]) + if ( + not modulation or modulation in {"QAM_NONE", "NONE"} + or upstream in {"", "---"} or not math.isfinite(power) + or rate.lower() == "invalid" + ): + continue + channel: RawChannel = { + "channelID": int(upstream), "frequency": hz_to_mhz(row.get("Freq", "")), + "powerLevel": power, "modulation": modulation, "multiplex": "ATDMA", + } + symbol_rate = _symbol_rate(rate) + if symbol_rate is not None: + channel["symbolRate"] = symbol_rate + channels.append(channel) + except (KeyError, TypeError, ValueError): + diagnostics.append(_issue("upstream", index)) + return ParseResult(channels, tuple(diagnostics)) + + +def _index_sort_key(name: str) -> int: + try: + return int(name.removeprefix("index")) + except ValueError: + return 0 + + +def pivot_sercom_indexed_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + pivot: dict[str, dict[str, Any]] = {} + for row in rows: + name = str(row.get("name") or "").strip() + if not name: + continue + for key, value in row.items(): + if key.startswith("index"): + pivot.setdefault(key, {})[name] = value + return [pivot[key] for key in sorted(pivot, key=_index_sort_key)] + + +def _profile_modulation(value: Any) -> str | None: + try: + bits = int(float(str(value).strip())) + except (TypeError, ValueError): + return None + if bits == 2: + return "QPSK" + if bits <= 0 or bits > 12: + return None + return f"{2 ** bits}QAM" + + +def parse_sercom_us_ofdma(rows: list[dict[str, Any]]) -> ParseResult[list[RawChannel]]: + channels: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, column in enumerate(pivot_sercom_indexed_rows(rows)): + state = str(column.get("STATE", "")).strip().upper() + power_state = str(column.get("Power", "")).strip().upper() + if power_state != "ON" or state in {"", "DISABLED", "OFF"}: + continue + frequency_value = column.get("Center Freq SC0") + frequency = parse_optional_finite_float(frequency_value) + if frequency is None or frequency <= 0: + continue + try: + if "rep power1_6" in column: + power = parse_optional_finite_float(column.get("rep power1_6")) + else: + power = None + diagnostics.append(_issue("upstream", index, "rep power1_6", "missing_field")) + channel: RawChannel = { + "channelID": int(str(column.get("CH", "")).strip()), "type": "OFDMA", + "frequency": hz_to_mhz(frequency_value), "powerLevel": power, + "modulation": "OFDMA", "multiplex": "OFDMA", + } + profile = _profile_modulation(column.get("bit Loading")) + if profile: + channel["profile_modulation"] = profile + channels.append(channel) + except (KeyError, TypeError, ValueError): + diagnostics.append(_issue("upstream", index)) + return ParseResult(channels, tuple(diagnostics)) + + +def parse_sercom_dm1000_json( + payload: dict[str, list[dict[str, Any]]] | None, +) -> ParseResult[DocsisDataFritz]: + data = payload if isinstance(payload, dict) else {} + return docsis_result( + parse_sercom_ds_scqam(data.get("downstream", [])), + parse_sercom_ds_ofdm(data.get("downstream_ofdm", [])), + parse_sercom_us_scqam(data.get("upstream", [])), + parse_sercom_us_ofdma(data.get("upstream_ofdma", [])), + ) diff --git a/app/drivers/formats/surfboard.py b/app/drivers/formats/surfboard.py new file mode 100644 index 00000000..e33b7171 --- /dev/null +++ b/app/drivers/formats/surfboard.py @@ -0,0 +1,99 @@ +"""Pure parser for Surfboard HNAP channel strings.""" + +from __future__ import annotations + +from ...types import DocsisDataFritz, RawChannel +from .contract import ParseDiagnostic, ParseResult, diagnostic, docsis_split +from .primitives import hz_to_mhz + + +_DOWNSTREAM_FIELDS = 9 +_UPSTREAM_FIELDS = 7 + + +def normalize_surfboard_modulation(value: str) -> str: + return value.strip() if value else "" + + +def parse_surfboard_downstream(raw: str) -> ParseResult[tuple[list[RawChannel], list[RawChannel]]]: + docsis30: list[RawChannel] = [] + docsis31: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, entry in enumerate((raw or "").split("|+|")): + if not entry.strip(): + continue + fields = entry.strip().split("^") + if fields and fields[-1] == "": + fields.pop() + if len(fields) < _DOWNSTREAM_FIELDS or fields[1].strip() != "Locked": + continue + try: + modulation = normalize_surfboard_modulation(fields[2]) + channel_id = int(fields[3]) + frequency = hz_to_mhz(int(fields[4])) + power = float(fields[5].strip()) + snr = float(fields[6].strip()) + corrected = int(fields[7]) + uncorrected = int(fields[8]) + if "OFDM" in modulation.upper(): + docsis31.append({ + "channelID": channel_id, "type": "OFDM", "frequency": frequency, + "powerLevel": power, "mer": snr, "mse": None, + "corrErrors": corrected, "nonCorrErrors": uncorrected, + }) + else: + docsis30.append({ + "channelID": channel_id, "frequency": frequency, "powerLevel": power, + "mer": snr, "mse": -snr, "modulation": modulation, + "corrErrors": corrected, "nonCorrErrors": uncorrected, + }) + except (ValueError, IndexError): + diagnostics.append(diagnostic( + "surfboard_hnap", "invalid_channel", family="surfboard", + direction="downstream", index=index, + )) + return ParseResult((docsis30, docsis31), tuple(diagnostics)) + + +def parse_surfboard_upstream(raw: str) -> ParseResult[tuple[list[RawChannel], list[RawChannel]]]: + docsis30: list[RawChannel] = [] + docsis31: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, entry in enumerate((raw or "").split("|+|")): + if not entry.strip(): + continue + fields = entry.strip().split("^") + if fields and fields[-1] == "": + fields.pop() + if len(fields) < _UPSTREAM_FIELDS or fields[1].strip() != "Locked": + continue + try: + channel_type = normalize_surfboard_modulation(fields[2]) + channel_id = int(fields[3]) + frequency = hz_to_mhz(int(fields[5])) + power = float(fields[6].strip()) + if "OFDMA" in channel_type.upper(): + docsis31.append({ + "channelID": channel_id, "type": "OFDMA", "frequency": frequency, + "powerLevel": power, "modulation": "OFDMA", "multiplex": "", + }) + else: + docsis30.append({ + "channelID": channel_id, "frequency": frequency, "powerLevel": power, + "modulation": channel_type, "multiplex": channel_type, + }) + except (ValueError, IndexError): + diagnostics.append(diagnostic( + "surfboard_hnap", "invalid_channel", family="surfboard", + direction="upstream", index=index, + )) + return ParseResult((docsis30, docsis31), tuple(diagnostics)) + + +def parse_surfboard_hnap(downstream_raw: str, upstream_raw: str) -> ParseResult[DocsisDataFritz]: + downstream = parse_surfboard_downstream(downstream_raw) + upstream = parse_surfboard_upstream(upstream_raw) + return ParseResult( + docsis_split(*downstream.value, *upstream.value), + downstream.diagnostics + upstream.diagnostics, + ) diff --git a/app/drivers/formats/vodafone.py b/app/drivers/formats/vodafone.py new file mode 100644 index 00000000..e5dcddd8 --- /dev/null +++ b/app/drivers/formats/vodafone.py @@ -0,0 +1,253 @@ +"""Pure profiles for Vodafone CGA, TG embedded JSON, and Ultra Hub payloads.""" + +from __future__ import annotations + +import json +import re +from typing import Any + +from ...types import DocsisDataFritz, RawChannel +from .contract import ParseDiagnostic, ParseResult, diagnostic, docsis_split +from .primitives import normalize_modulation, parse_number + + +def _ultra_value(value: Any) -> float: + if not value: + return 0.0 + try: + return float(str(value).strip().split()[0]) + except (IndexError, ValueError): + return 0.0 + + +def parse_ultrahub7_downstream(rows: list[dict[str, Any]]) -> ParseResult[list[RawChannel]]: + channels: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, row in enumerate(rows): + try: + channel_id = int(row.get("ChannelID", "0")) + frequency = _ultra_value(row.get("Frequency", "0")) + modulation = str(row.get("Modulation", "") or "").upper().replace("-", "") + power = _ultra_value(row.get("PowerLevel", "0")) + snr = _ultra_value(row.get("SNRLevel", "")) + channels.append({ + "channelID": str(channel_id), "type": modulation, + "frequency": f"{int(frequency)} MHz", "powerLevel": power, + "mer": snr if snr > 0 else None, "mse": None, "latency": 0, + "corrErrors": None, "nonCorrErrors": None, + }) + except (ValueError, TypeError): + diagnostics.append(diagnostic( + "ultrahub7_json", "invalid_channel", family="vodafone", + direction="downstream", index=index, + )) + return ParseResult(channels, tuple(diagnostics)) + + +def parse_ultrahub7_upstream(rows: list[dict[str, Any]]) -> ParseResult[list[RawChannel]]: + channels: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, row in enumerate(rows): + try: + channel_id = int(row.get("ChannelID", "0")) + frequency = _ultra_value(row.get("Frequency", "0")) + modulation = str(row.get("Modulation", "") or "").upper().replace("-", "") + channels.append({ + "channelID": str(channel_id), "type": modulation, + "frequency": f"{int(frequency)} MHz", + "powerLevel": _ultra_value(row.get("PowerLevel", "0")), + "multiplex": "", + }) + except (ValueError, TypeError): + diagnostics.append(diagnostic( + "ultrahub7_json", "invalid_channel", family="vodafone", + direction="upstream", index=index, + )) + return ParseResult(channels, tuple(diagnostics)) + + +def parse_ultrahub7_json( + payload: dict[str, list[dict[str, Any]]] | None, +) -> ParseResult[dict[str, Any]]: + data = payload if isinstance(payload, dict) else {} + downstream = parse_ultrahub7_downstream(data.get("downstream", [])) + upstream = parse_ultrahub7_upstream(data.get("upstream", [])) + return ParseResult({ + "docsis": "3.1", "downstream": downstream.value, "upstream": upstream.value, + }, downstream.diagnostics + upstream.diagnostics) + + +def _cga_frequency(value: Any) -> float: + number = parse_number(value) if isinstance(value, str) else float(value or 0) + return number / 1_000_000 if number > 1_000_000 else number + + +def parse_vodafone_number(value: Any) -> float: + return parse_number(value) if isinstance(value, str) else float(value or 0) + + +def _parse_cga_lane(rows: Any, lane: str) -> ParseResult[list[RawChannel] | None]: + if not isinstance(rows, list): + return ParseResult([]) + result: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, row in enumerate(rows): + if not isinstance(row, dict): + diagnostics.append(diagnostic( + "vodafone_station_cga_json", "invalid_channel", family="vodafone", + direction="downstream" if lane.startswith("ds") else "upstream", index=index, + )) + return ParseResult(None, tuple(diagnostics)) + try: + if lane == "ds30": + snr = abs(parse_number(row.get("SNR", "0"))) + modulation = normalize_modulation(row.get("FFT", "")) + frequency = _cga_frequency(row.get("CentralFrequency", "0")) + result.append({ + "channelID": int(parse_number(row.get("channelid", "0"))), + "type": modulation, "frequency": f"{int(frequency)} MHz" if frequency else "", + "powerLevel": parse_number(row.get("power", "0")), + "mse": -snr if snr else None, "mer": snr if snr else None, + "latency": 0, "corrError": 0, "nonCorrError": 0, + }) + elif lane == "ds31": + snr = abs(parse_number(row.get("SNR_ofdm", "0"))) + frequency = _cga_frequency(row.get("CentralFrequency_ofdm", "0")) + result.append({ + "channelID": int(parse_number(row.get("channelid_ofdm", "0"))), + "type": "OFDM", "frequency": f"{int(frequency)} MHz" if frequency else "", + "powerLevel": parse_number(row.get("power_ofdm", "0")), + "mse": -snr if snr else None, "mer": snr if snr else None, + "latency": 0, "corrError": 0, "nonCorrError": 0, + }) + else: + frequency = _cga_frequency(row.get("CentralFrequency", "0")) + modulation = normalize_modulation(row.get("FFT", "")) + channel: RawChannel = { + "channelID": int(parse_number(row.get("channelidup", "0"))), + "type": "OFDMA" if lane == "us31" else modulation, + "frequency": f"{int(frequency)} MHz" if frequency else "", + "powerLevel": parse_number(row.get("power", "0")), + "multiplex": "", + } + if lane == "us31": + channel["modulation"] = modulation or "OFDMA" + result.append(channel) + except (ValueError, TypeError): + diagnostics.append(diagnostic( + "vodafone_station_cga_json", "invalid_channel", family="vodafone", + direction="downstream" if lane.startswith("ds") else "upstream", index=index, + )) + return ParseResult(result, tuple(diagnostics)) + + +def parse_vodafone_station_cga_json(payload: Any) -> ParseResult[DocsisDataFritz | None]: + data = payload if isinstance(payload, dict) else {} + parsed = ( + _parse_cga_lane(data.get("downstream", []) or [], "ds30"), + _parse_cga_lane(data.get("ofdm_downstream", []) or [], "ds31"), + _parse_cga_lane(data.get("upstream", []) or [], "us30"), + _parse_cga_lane(data.get("ofdma_upstream", []) or [], "us31"), + ) + diagnostics = tuple(item for result in parsed for item in result.diagnostics) + if any(result.value is None for result in parsed): + return ParseResult(None, diagnostics) + return ParseResult( + docsis_split( + parsed[0].value, + parsed[1].value, + parsed[2].value, + parsed[3].value, + ), + diagnostics, + ) + + +def parse_tg_power(value: Any) -> float: + if isinstance(value, (int, float)): + return float(value) + if not value or not isinstance(value, str): + return 0.0 + return parse_number(value.split("/")[0].replace("dBmV", "").replace("dBuV", "").strip()) + + +def parse_tg_frequency(value: Any) -> float: + if isinstance(value, (int, float)): + number = float(value) + return number / 1_000_000 if number > 1_000_000 else number + if not value or not isinstance(value, str): + return 0.0 + if "~" in value: + try: + start, end = value.split("~", 1) + number = (float(start.strip()) + float(end.strip())) / 2 + return number / 1_000_000 if number > 1_000_000 else number + except (ValueError, IndexError): + return 0.0 + number = parse_number(value) + return number / 1_000_000 if number > 1_000_000 else number + + +def parse_vodafone_station_tg_embedded_json(html: str) -> ParseResult[DocsisDataFritz | None]: + downstream_match = re.search(r"json_dsData\s*=\s*(\[.+?\])\s*;", html or "", re.DOTALL) + upstream_match = re.search(r"json_usData\s*=\s*(\[.+?\])\s*;", html or "", re.DOTALL) + if not downstream_match and not upstream_match: + return ParseResult(None, (diagnostic( + "vodafone_station_tg_embedded_json", "missing_data", family="vodafone", + ),)) + try: + downstream_rows = json.loads(downstream_match.group(1)) if downstream_match else [] + upstream_rows = json.loads(upstream_match.group(1)) if upstream_match else [] + except (json.JSONDecodeError, TypeError): + return ParseResult(None, (diagnostic( + "vodafone_station_tg_embedded_json", "invalid_json", family="vodafone", + ),)) + + ds30: list[RawChannel] = [] + ds31: list[RawChannel] = [] + us30: list[RawChannel] = [] + us31: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, row in enumerate(downstream_rows): + try: + channel_type = row.get("ChannelType", "SC-QAM") + frequency = parse_tg_frequency(row.get("Frequency", "0")) + snr = abs(parse_number(row.get("SNRLevel", "0"))) + modulation = normalize_modulation(row.get("Modulation", "")) + ofdm = "OFDM" in channel_type.upper() + if ofdm: + modulation = modulation or "OFDM" + channel: RawChannel = { + "channelID": int(float(row.get("ChannelID", 0))), "type": modulation, + "frequency": f"{frequency:.3f} MHz" if frequency else "", + "powerLevel": parse_tg_power(row.get("PowerLevel", "0")), + "mse": -snr if snr else None, "mer": snr if snr else None, + "latency": 0, "corrError": 0, "nonCorrError": 0, + } + (ds31 if ofdm else ds30).append(channel) + except (ValueError, TypeError, AttributeError): + diagnostics.append(diagnostic( + "vodafone_station_tg_embedded_json", "invalid_channel", family="vodafone", + direction="downstream", index=index, + )) + for index, row in enumerate(upstream_rows): + try: + channel_type = row.get("ChannelType", "SC-QAM") + frequency = parse_tg_frequency(row.get("Frequency", "0")) + modulation = normalize_modulation(row.get("Modulation", "")) + ofdma = "OFDMA" in channel_type.upper() + if ofdma: + modulation = modulation or "OFDMA" + channel = { + "channelID": int(float(row.get("ChannelID", 0))), "type": modulation, + "frequency": f"{frequency:.3f} MHz" if frequency else "", + "powerLevel": parse_tg_power(row.get("PowerLevel", "0")), + "multiplex": "", + } + (us31 if ofdma else us30).append(channel) + except (ValueError, TypeError, AttributeError): + diagnostics.append(diagnostic( + "vodafone_station_tg_embedded_json", "invalid_channel", family="vodafone", + direction="upstream", index=index, + )) + return ParseResult(docsis_split(ds30, ds31, us30, us31), tuple(diagnostics)) diff --git a/app/drivers/formats/xml_payloads.py b/app/drivers/formats/xml_payloads.py new file mode 100644 index 00000000..bb8c36bc --- /dev/null +++ b/app/drivers/formats/xml_payloads.py @@ -0,0 +1,88 @@ +"""Pure parser for the Compal CH7465 downstream/upstream XML profile.""" + +from __future__ import annotations + +import xml.etree.ElementTree as ET + +from ...types import RawChannel +from .contract import ParseDiagnostic, ParseResult, diagnostic +from .primitives import normalize_modulation + + +def _text(node: ET.Element | None, default: str = "") -> str: + return node.text if node is not None and node.text is not None else default + + +def parse_ch7465_xml( + downstream_xml: str | None, + upstream_xml: str | None, +) -> ParseResult[dict | None]: + if downstream_xml is None or upstream_xml is None: + return ParseResult(None, (diagnostic( + "ch7465_xml", "invalid_xml", family="xml_payloads", + ),)) + try: + downstream_root = ET.fromstring(downstream_xml) + upstream_root = ET.fromstring(upstream_xml) + except ET.ParseError: + return ParseResult(None, (diagnostic( + "ch7465_xml", "invalid_xml", family="xml_payloads", + ),)) + + downstream: list[RawChannel] = [] + upstream: list[RawChannel] = [] + diagnostics: list[ParseDiagnostic] = [] + for index, channel in enumerate(downstream_root.findall("downstream")): + try: + item: RawChannel = { + "channelID": int(channel.find("chid").text), + "frequency": _text(channel.find("freq")), + "powerLevel": float(_text(channel.find("pow"), "0")), + } + mer = _text(channel.find("RxMER")) + modulation = normalize_modulation(_text(channel.find("mod"))) + corrected = _text(channel.find("PreRs")) + uncorrected = _text(channel.find("PostRs")) + if mer: + item["mer"] = float(mer) + item["mse"] = -float(mer) + if modulation: + item["modulation"] = modulation + if corrected: + item["corrErrors"] = int(corrected) + if uncorrected: + item["nonCorrErrors"] = int(uncorrected) + downstream.append(item) + except (AttributeError, TypeError, ValueError): + diagnostics.append(diagnostic( + "ch7465_xml", "invalid_channel", family="xml_payloads", + direction="downstream", index=index, + )) + + for index, channel in enumerate(upstream_root.findall("upstream")): + try: + item = { + "channelID": int(channel.find("usid").text), + "frequency": _text(channel.find("freq")), + "powerLevel": float(_text(channel.find("power"), "0")), + } + modulation = normalize_modulation(_text(channel.find("mod"))) + message_type = _text(channel.find("messageType")) + multiplex = {"2": "tdma", "29": "atdma", "35": "atdma"}.get( + message_type, message_type + ) + if modulation: + item["modulation"] = modulation + if multiplex: + item["multiplex"] = multiplex + upstream.append(item) + except (AttributeError, TypeError, ValueError): + diagnostics.append(diagnostic( + "ch7465_xml", "invalid_channel", family="xml_payloads", + direction="upstream", index=index, + )) + + return ParseResult( + {"docsis": "3.0", "downstream": downstream, "upstream": upstream}, + tuple(diagnostics), + ) diff --git a/app/drivers/fritzbox.py b/app/drivers/fritzbox.py index 01c598aa..b42bed0d 100644 --- a/app/drivers/fritzbox.py +++ b/app/drivers/fritzbox.py @@ -5,17 +5,12 @@ import logging from .base import ModemDriver +from .formats.fritzbox import parse_fritzbox_data_lua from ..types import DocsisData, DeviceInfo, ConnectionInfo from .. import fritzbox as fb log = logging.getLogger("docsis.driver.fritzbox") -# Fritz!Box displays DOCSIS 3.1 upstream power 6 dB lower than the actual -# value. We compensate here so that the analyzer can use real VFKD thresholds -# that work identically for every modem. -_FRITZBOX_US31_POWER_OFFSET = 6.0 - - class FritzBoxDriver(ModemDriver): """Driver for AVM FritzBox cable modems. @@ -23,6 +18,8 @@ class FritzBoxDriver(ModemDriver): login() call to avoid session expiry issues. """ + FORMAT_FAMILIES = ("fritzbox_data_lua",) + def __init__(self, url: str, user: str, password: str): super().__init__(url, user, password) self._sid: str | None = None @@ -31,20 +28,11 @@ def login(self) -> None: self._sid = fb.login(self._url, self._user, self._password) def get_docsis_data(self) -> DocsisData: - data = fb.get_docsis_data(self._url, self._sid) - self._compensate_us31_power(data) - return data + return parse_fritzbox_data_lua(fb.get_docsis_data(self._url, self._sid)).value @staticmethod def _compensate_us31_power(data: DocsisData) -> None: - """Add +6 dB to DOCSIS 3.1 upstream power to correct Fritz!Box display bug.""" - us31 = data.get("channelUs", {}).get("docsis31", []) - for ch in us31: - try: - raw = float(ch.get("powerLevel", 0)) - ch["powerLevel"] = str(round(raw + _FRITZBOX_US31_POWER_OFFSET, 1)) - except (TypeError, ValueError): - pass + data.update(parse_fritzbox_data_lua(data).value) def get_device_info(self) -> DeviceInfo: info = fb.get_device_info(self._url, self._sid) diff --git a/app/drivers/generic.py b/app/drivers/generic.py index 451390bc..19bc6be4 100644 --- a/app/drivers/generic.py +++ b/app/drivers/generic.py @@ -3,6 +3,7 @@ from __future__ import annotations from .base import ModemDriver +from .formats.boundaries import parse_generic_no_docsis from ..types import DocsisData, DeviceInfo, ConnectionInfo @@ -13,14 +14,13 @@ class GenericDriver(ModemDriver): BNetzA, Weather, Journal) to work standalone. """ + FORMAT_FAMILIES = ("generic_no_docsis",) + def login(self) -> None: pass def get_docsis_data(self) -> DocsisData: - return { - "channelDs": {"docsis30": [], "docsis31": []}, - "channelUs": {"docsis30": [], "docsis31": []}, - } + return parse_generic_no_docsis().value def get_device_info(self) -> DeviceInfo: return { diff --git a/app/drivers/hitron.py b/app/drivers/hitron.py index ec66b37f..21272b05 100644 --- a/app/drivers/hitron.py +++ b/app/drivers/hitron.py @@ -20,8 +20,18 @@ import requests from .base import ModemDriver +from .formats.hitron import ( + parse_coda56_ds_ofdm, + parse_coda56_ds_scqam, + parse_coda56_us_ofdma, + parse_coda56_us_scqam, + parse_hitron_coda56_json, + parse_hitron_ofdma_power, +) +from .formats.primitives import hz_to_mhz +from .format_compat import unwrap_hitron from ..types import DocsisData, DeviceInfo, ConnectionInfo, RawChannel -from .utils import make_legacy_tls_adapter, parse_optional_finite_float +from .utils import make_legacy_tls_adapter log = logging.getLogger("docsis.driver.hitron") @@ -43,6 +53,8 @@ class HitronDriver(ModemDriver): ASP endpoints with a cache-buster query parameter. """ + FORMAT_FAMILIES = ("hitron_coda56_json",) + def __init__(self, url: str, user: str, password: str): super().__init__(url, user, password) self._session = requests.Session() @@ -68,15 +80,12 @@ def login(self) -> None: def get_docsis_data(self) -> DocsisData: """Retrieve DOCSIS channel data from all four endpoints.""" - ds30 = self._fetch_ds_scqam() - us30 = self._fetch_us_scqam() - ds31 = self._fetch_ds_ofdm() - us31 = self._fetch_us_ofdma() - - return { - "channelDs": {"docsis30": ds30, "docsis31": ds31}, - "channelUs": {"docsis30": us30, "docsis31": us31}, - } + return parse_hitron_coda56_json({ + "downstream": self._fetch_json("/data/dsinfo.asp"), + "upstream": self._fetch_json("/data/usinfo.asp"), + "downstream_ofdm": self._fetch_json("/data/dsofdminfo.asp"), + "upstream_ofdma": self._fetch_json("/data/usofdminfo.asp"), + }).value def get_device_info(self) -> DeviceInfo: """Return static device info (model not available via API).""" @@ -106,93 +115,20 @@ def _fetch_json(self, path: str) -> list[dict[str, str]]: return [] def _fetch_ds_scqam(self) -> list[RawChannel]: - """Parse downstream SC-QAM channels (DOCSIS 3.0).""" - channels = [] - for ch in self._fetch_json("/data/dsinfo.asp"): - try: - mod_code = int(ch.get("modulation", -1)) - modulation = _DS_MODULATION.get(mod_code, f"Unknown({mod_code})") - channels.append({ - "channelID": int(ch["channelId"]), - "frequency": self._hz_to_mhz(ch["frequency"]), - "powerLevel": float(ch["signalStrength"]), - "modulation": modulation, - "mer": float(ch["snr"]), - "mse": -float(ch["snr"]), - "corrErrors": int(ch["correcteds"]), - "nonCorrErrors": int(ch["uncorrect"]), - }) - except (ValueError, KeyError, TypeError) as e: - log.warning("Failed to parse Hitron DS row: %s", e) - return channels + return parse_coda56_ds_scqam(self._fetch_json("/data/dsinfo.asp")).value def _fetch_us_scqam(self) -> list[RawChannel]: - """Parse upstream SC-QAM channels (DOCSIS 3.0).""" - channels = [] - for ch in self._fetch_json("/data/usinfo.asp"): - try: - channels.append({ - "channelID": int(ch["channelId"]), - "frequency": self._hz_to_mhz(ch["frequency"]), - "powerLevel": float(ch["signalStrength"]), - "modulation": ch.get("modtype", ""), - "multiplex": ch.get("scdmaMode", ""), - }) - except (ValueError, KeyError, TypeError) as e: - log.warning("Failed to parse Hitron US row: %s", e) - return channels + return parse_coda56_us_scqam(self._fetch_json("/data/usinfo.asp")).value def _fetch_ds_ofdm(self) -> list[RawChannel]: - """Parse downstream OFDM channels (DOCSIS 3.1).""" - channels = [] - for ch in self._fetch_json("/data/dsofdminfo.asp"): - try: - plc_lock = ch.get("plclock", "").strip().upper() - if plc_lock != "YES": - continue - channels.append({ - "channelID": int(ch["receive"]), - "type": "OFDM", - "frequency": self._hz_to_mhz(ch.get("Subcarr0freqFreq", "0")), - "powerLevel": float(ch["plcpower"]), - "modulation": "OFDM", - "mer": float(ch["SNR"]), - "mse": None, - "corrErrors": int(ch["correcteds"]), - "nonCorrErrors": int(ch["uncorrect"]), - }) - except (ValueError, KeyError, TypeError) as e: - log.warning("Failed to parse Hitron DS OFDM row: %s", e) - return channels + return parse_coda56_ds_ofdm(self._fetch_json("/data/dsofdminfo.asp")).value def _fetch_us_ofdma(self) -> list[RawChannel]: - """Parse upstream OFDMA channels (DOCSIS 3.1).""" - channels = [] - for ch in self._fetch_json("/data/usofdminfo.asp"): - try: - state = ch.get("state", "").strip().upper() - if state != "OPERATE": - continue - channels.append({ - "channelID": int(ch["uschindex"]), - "type": "OFDMA", - "frequency": self._hz_to_mhz(ch.get("frequency", "0")), - "powerLevel": self._ofdma_power_1_6(ch), - "modulation": "OFDMA", - "multiplex": "", - }) - except (ValueError, KeyError, TypeError) as e: - log.warning("Failed to parse Hitron US OFDMA row: %s", e) - return channels + return unwrap_hitron(parse_coda56_us_ofdma(self._fetch_json("/data/usofdminfo.asp")), log) @staticmethod def _ofdma_power_1_6(row: dict[str, str]) -> float | None: - if "repPower1_6" not in row: - log.warning("Hitron CODA-56 OFDMA row missing repPower1_6; leaving power unsupported") - return None - return parse_optional_finite_float(row.get("repPower1_6")) - - # -- Helpers -- + return parse_hitron_ofdma_power(row) @staticmethod def _cache_bust() -> str: @@ -200,5 +136,4 @@ def _cache_bust() -> str: @staticmethod def _hz_to_mhz(hz_str: str) -> str: - from .utils import hz_to_mhz return hz_to_mhz(hz_str) diff --git a/app/drivers/hitron_coda_4680.py b/app/drivers/hitron_coda_4680.py index 90c6cb62..79b9fe40 100644 --- a/app/drivers/hitron_coda_4680.py +++ b/app/drivers/hitron_coda_4680.py @@ -21,7 +21,15 @@ from ..types import ConnectionInfo, DeviceInfo, DocsisData, RawChannel from .base import ModemDriver -from .utils import hz_to_mhz, make_legacy_tls_adapter, normalize_modulation, parse_optional_finite_float +from .formats.hitron import ( + parse_coda4680_ds_ofdm, + parse_coda4680_ds_scqam, + parse_coda4680_us_ofdma, + parse_coda4680_us_scqam, + parse_hitron_coda4680_json, + parse_hitron_ofdma_power, +) +from .utils import make_legacy_tls_adapter log = logging.getLogger("docsis.driver.hitron_coda_4680") @@ -29,6 +37,8 @@ class HitronCoda4680Driver(ModemDriver): """Driver for authenticated Hitron CODA-4680 modem/router UIs.""" + FORMAT_FAMILIES = ("hitron_coda4680_json",) + def __init__(self, url: str, user: str, password: str): super().__init__(url.rstrip("/"), user, password) self._session = requests.Session() @@ -75,16 +85,12 @@ def get_docsis_data(self) -> DocsisData: ds_ofdm = self._fetch_payload("/1/Device/CM/DsOfdm") us_ofdma = self._fetch_payload("/1/Device/CM/UsOfdm") - return { - "channelDs": { - "docsis30": self._parse_ds_scqam(ds_info.get("Freq_List", [])), - "docsis31": self._parse_ds_ofdm(ds_ofdm.get("OFDMs_List", [])), - }, - "channelUs": { - "docsis30": self._parse_us_scqam(us_info.get("Freq_List", [])), - "docsis31": self._parse_us_ofdma(us_ofdma.get("OFDMAs_List", [])), - }, - } + return parse_hitron_coda4680_json({ + "downstream": ds_info, + "upstream": us_info, + "downstream_ofdm": ds_ofdm, + "upstream_ofdma": us_ofdma, + }).value def get_device_info(self) -> DeviceInfo: """Retrieve model and firmware from the version endpoint.""" @@ -197,93 +203,20 @@ def _parse_device_info(payload: dict[str, Any]) -> DeviceInfo: } def _parse_ds_scqam(self, rows: list[dict[str, Any]]) -> list[RawChannel]: - channels: list[RawChannel] = [] - for row in rows: - try: - snr = float(row["snr"]) - channels.append({ - "channelID": int(row["channelId"]), - "frequency": hz_to_mhz(row["frequency"]), - "powerLevel": float(row["signalStrength"]), - "modulation": normalize_modulation(row.get("modulation", "")), - "mer": snr, - "mse": -snr, - "corrErrors": int(row["correcteds"]), - "nonCorrErrors": int(row["uncorrect"]), - }) - except (KeyError, TypeError, ValueError) as exc: - log.warning("Failed to parse Hitron CODA-4680 DS row: %s", exc) - return channels + return parse_coda4680_ds_scqam(rows).value def _parse_us_scqam(self, rows: list[dict[str, Any]]) -> list[RawChannel]: - channels: list[RawChannel] = [] - for row in rows: - try: - modulation = normalize_modulation(row.get("modulationType") or row.get("modtype") or "") - channel: RawChannel = { - "channelID": int(row["channelId"]), - "frequency": hz_to_mhz(row["frequency"]), - "powerLevel": float(row["signalStrength"]), - "modulation": modulation, - # CODA-4680 does not expose scdmaMode in the captured API; - # ATDMA is the SC-QAM upstream lane DOCSight should score. - "multiplex": "ATDMA", - } - symbol_rate = row.get("symbolrate") - if symbol_rate is not None: - channel["symbolRate"] = int(symbol_rate) - channels.append(channel) - except (KeyError, TypeError, ValueError) as exc: - log.warning("Failed to parse Hitron CODA-4680 US row: %s", exc) - return channels + return parse_coda4680_us_scqam(rows).value def _parse_ds_ofdm(self, rows: list[dict[str, Any]]) -> list[RawChannel]: - channels: list[RawChannel] = [] - for row in rows: - try: - if str(row.get("plclock", "")).strip().upper() != "YES": - continue - channels.append({ - "channelID": int(row["receive"]), - "type": "OFDM", - "frequency": hz_to_mhz(row.get("Subcarr0freqFreq", "")), - "powerLevel": float(row["plcpower"]), - "modulation": "OFDM", - "mer": None, - "mse": None, - "corrErrors": None, - "nonCorrErrors": None, - }) - except (KeyError, TypeError, ValueError) as exc: - log.warning("Failed to parse Hitron CODA-4680 DS OFDM row: %s", exc) - return channels + return parse_coda4680_ds_ofdm(rows).value def _parse_us_ofdma(self, rows: list[dict[str, Any]]) -> list[RawChannel]: - channels: list[RawChannel] = [] - for row in rows: - try: - if str(row.get("state", "")).strip().upper() != "OPERATE": - continue - channels.append({ - "channelID": int(row["uschindex"]), - "type": "OFDMA", - # The captured CODA-4680 OFDMA API does not expose center - # frequency. Preserve unsupported as blank instead of 0 MHz. - "frequency": "", - "powerLevel": self._ofdma_power_1_6(row), - "modulation": "OFDMA", - "multiplex": "OFDMA", - }) - except (KeyError, TypeError, ValueError) as exc: - log.warning("Failed to parse Hitron CODA-4680 US OFDMA row: %s", exc) - return channels + return parse_coda4680_us_ofdma(rows).value @staticmethod def _ofdma_power_1_6(row: dict[str, Any]) -> float | None: - if "repPower1_6" not in row: - log.warning("Hitron CODA-4680 OFDMA row missing repPower1_6; leaving power unsupported") - return None - return parse_optional_finite_float(row.get("repPower1_6")) + return parse_hitron_ofdma_power(row) @staticmethod def _parse_rate_kbps(value: Any) -> int: diff --git a/app/drivers/sagemcom.py b/app/drivers/sagemcom.py index 09936562..810b5e50 100644 --- a/app/drivers/sagemcom.py +++ b/app/drivers/sagemcom.py @@ -22,6 +22,13 @@ import requests from .base import ModemDriver +from .formats.sagemcom import ( + _sagemcom_frequency, + _sagemcom_is_ofdm, + _sagemcom_modulation, + parse_sagemcom_xmo_downstream, + parse_sagemcom_xmo_upstream, +) from ..types import ConnectionInfo, DeviceInfo, DocsisData, RawChannel log = logging.getLogger("docsis.driver.sagemcom") @@ -56,6 +63,8 @@ class SagemcomDriver(ModemDriver): Uses XMO JSON-RPC API with SHA-512 digest authentication. """ + FORMAT_FAMILIES = ("sagemcom_xmo_json",) + def __init__(self, url: str, user: str, password: str): super().__init__(url.rstrip("/"), user, password) self._session = requests.Session() @@ -285,126 +294,26 @@ def _json_encode(obj) -> str: import json return json.dumps(obj, separators=(",", ":")) - # -- Channel parsers -- + # Compatibility parser seams. def _parse_downstream(self, channels: list[dict[str, object]]) -> tuple[list[RawChannel], list[RawChannel]]: - ds30 = [] - ds31 = [] - - for ch in channels: - if not ch.get("LockStatus", False): - continue - - try: - channel_id = ch.get("ChannelID", 0) - freq_hz = ch.get("Frequency", 0) - power = ch.get("PowerLevel", 0) - snr = ch.get("SNR", 0) - modulation = ch.get("Modulation", "") - bandwidth = ch.get("BandWidth", 0) - corr = ch.get("CorrectableCodewords", 0) - uncorr = ch.get("UncorrectableCodewords", 0) - - freq_mhz = self._hz_to_mhz(freq_hz) - - if self._is_ofdm_downstream(modulation, bandwidth): - ds31.append({ - "channelID": channel_id, - "type": "OFDM", - "frequency": freq_mhz, - "powerLevel": power, - "mer": snr, - "mse": None, - "corrErrors": corr, - "nonCorrErrors": uncorr, - }) - else: - ds30.append({ - "channelID": channel_id, - "frequency": freq_mhz, - "powerLevel": power, - "mer": snr, - "mse": -snr if snr else None, - "modulation": self._normalize_modulation(modulation), - "corrErrors": corr, - "nonCorrErrors": uncorr, - }) - except (ValueError, TypeError) as e: - log.warning("Failed to parse Sagemcom DS channel: %s", e) - - return ds30, ds31 + return parse_sagemcom_xmo_downstream(channels).value def _parse_upstream(self, channels: list[dict[str, object]]) -> tuple[list[RawChannel], list[RawChannel]]: - us30 = [] - us31 = [] - - for ch in channels: - if not ch.get("LockStatus", False): - continue - - try: - channel_id = ch.get("ChannelID", 0) - freq_hz = ch.get("Frequency", 0) - power = ch.get("PowerLevel", 0) - modulation = ch.get("Modulation", "") - - freq_mhz = self._hz_to_mhz(freq_hz) - - if modulation.lower() == "ofdma": - us31.append({ - "channelID": channel_id, - "type": "OFDMA", - "frequency": freq_mhz, - "powerLevel": power, - "modulation": "OFDMA", - "multiplex": "", - }) - else: - us30.append({ - "channelID": channel_id, - "frequency": freq_mhz, - "powerLevel": power, - "modulation": self._normalize_us_modulation(modulation), - "multiplex": modulation.upper() if modulation else "", - }) - except (ValueError, TypeError) as e: - log.warning("Failed to parse Sagemcom US channel: %s", e) - - return us30, us31 - - # -- Helpers -- + return parse_sagemcom_xmo_upstream(channels).value @staticmethod def _hz_to_mhz(freq_hz) -> str: - """Convert Hz to MHz string. Returns '' for falsy input (0, None, '').""" - if not freq_hz: - return "" - from .utils import hz_to_mhz - return hz_to_mhz(freq_hz) + return _sagemcom_frequency(freq_hz) @staticmethod def _is_ofdm_downstream(modulation: str, bandwidth: int) -> bool: - if bandwidth and bandwidth > 8_000_000: - return True - if modulation and modulation.startswith("256-QAM"): - return True - return False + return _sagemcom_is_ofdm(modulation, bandwidth) @staticmethod - def _normalize_modulation(mod: str) -> str: - if not mod: - return "" - m = mod.strip() - if m.lower().startswith("qam"): - n = m[3:] - return f"{n}QAM" - return m + def _normalize_modulation(modulation: str) -> str: + return _sagemcom_modulation(modulation) @staticmethod - def _normalize_us_modulation(mod: str) -> str: - if not mod: - return "" - m = mod.strip().upper() - if m == "ATDMA": - return "ATDMA" - return m + def _normalize_us_modulation(modulation: str) -> str: + return modulation.strip().upper() if modulation else "" diff --git a/app/drivers/sb6141.py b/app/drivers/sb6141.py index ae9be747..e734682a 100644 --- a/app/drivers/sb6141.py +++ b/app/drivers/sb6141.py @@ -14,12 +14,18 @@ from __future__ import annotations import logging -import re - import requests from bs4 import BeautifulSoup from .base import ModemDriver +from .formats.html_transposed import ( + extract_transposed_rows, + extract_upstream_modulation, + get_row_values, + parse_sb6141_downstream, + parse_sb6141_upstream, +) +from .formats.primitives import hz_to_mhz, parse_number from ..types import DocsisData, DeviceInfo, ConnectionInfo, RawChannel log = logging.getLogger("docsis.driver.sb6141") @@ -36,6 +42,8 @@ class SB6141Driver(ModemDriver): HTML tables where each row is a metric and each column is a channel. """ + FORMAT_FAMILIES = ("sb6141_transposed_html",) + def __init__(self, url: str, user: str, password: str): super().__init__(url, user, password) self._session = requests.Session() @@ -126,158 +134,28 @@ def get_connection_info(self) -> ConnectionInfo: """Standalone modem, no connection info.""" return {} - # -- Transposed table parsers -- - def _parse_downstream(self, ds_table, cw_table) -> list[RawChannel]: - """Parse transposed downstream + codewords tables. - - In the SB6141 tables, each row is a metric and each column is a - channel. The first cell of each row is the metric label. - """ - if not ds_table: - return [] - - ds_rows = self._extract_transposed_rows(ds_table) - channel_ids = self._get_row_values(ds_rows, "channel id") - frequencies = self._get_row_values(ds_rows, "frequency") - snrs = self._get_row_values(ds_rows, "signal to noise") - modulations = self._get_row_values(ds_rows, "modulation") - powers = self._get_row_values(ds_rows, "power level") - - # Get error counts from codewords table - corrected = [] - uncorrected = [] - if cw_table: - cw_rows = self._extract_transposed_rows(cw_table) - corrected = self._get_row_values(cw_rows, "correctable") - uncorrected = self._get_row_values(cw_rows, "uncorrectable") - - num_channels = len(channel_ids) - result = [] - - for i in range(num_channels): - try: - channel_id = int(channel_ids[i]) - freq = self._parse_freq_hz(frequencies[i] if i < len(frequencies) else "") - snr = self._parse_number(snrs[i] if i < len(snrs) else "") - power = self._parse_number(powers[i] if i < len(powers) else "") - mod = modulations[i].strip() if i < len(modulations) else "" - corr = int(self._parse_number(corrected[i])) if i < len(corrected) else 0 - uncorr = int(self._parse_number(uncorrected[i])) if i < len(uncorrected) else 0 - - result.append({ - "channelID": channel_id, - "frequency": freq, - "powerLevel": power, - "mer": snr, - "mse": -snr if snr else None, - "modulation": mod, - "corrErrors": corr, - "nonCorrErrors": uncorr, - }) - except (ValueError, TypeError, IndexError) as e: - log.warning("Failed to parse SB6141 DS channel %d: %s", i, e) - - return result + return parse_sb6141_downstream(ds_table, cw_table).value def _parse_upstream(self, us_table) -> list[RawChannel]: - """Parse transposed upstream table.""" - if not us_table: - return [] - - us_rows = self._extract_transposed_rows(us_table) - channel_ids = self._get_row_values(us_rows, "channel id") - frequencies = self._get_row_values(us_rows, "frequency") - powers = self._get_row_values(us_rows, "power level") - modulations = self._get_row_values(us_rows, "modulation") - - num_channels = len(channel_ids) - result = [] - - for i in range(num_channels): - try: - channel_id = int(channel_ids[i]) - freq = self._parse_freq_hz(frequencies[i] if i < len(frequencies) else "") - power = self._parse_number(powers[i] if i < len(powers) else "") - - # Upstream modulation can have multiple BR-separated entries - # like "[3] QPSK\n[3] 64QAM". Take the last (highest) one. - raw_mod = modulations[i] if i < len(modulations) else "" - mod = self._extract_upstream_modulation(raw_mod) - - result.append({ - "channelID": channel_id, - "frequency": freq, - "powerLevel": power, - "modulation": mod, - "multiplex": "SC-QAM", - }) - except (ValueError, TypeError, IndexError) as e: - log.warning("Failed to parse SB6141 US channel %d: %s", i, e) - - return result - - # -- Table helpers -- + return parse_sb6141_upstream(us_table).value @staticmethod def _extract_transposed_rows(table) -> list[tuple[str, list[str]]]: - """Extract rows from a transposed table. - - Returns list of (label, [values]) tuples, skipping the header row. - TRs may be inside a TBODY element, so we search recursively. - """ - rows = [] - for tr in table.find_all("tr"): - # Skip header rows (contain TH elements) - if tr.find("th"): - continue - cells = tr.find_all("td", recursive=False) - if len(cells) < 2: - continue - label = cells[0].get_text(strip=True) - values = [td.get_text(strip=True) for td in cells[1:]] - rows.append((label, values)) - return rows + return extract_transposed_rows(table) @staticmethod def _get_row_values(rows: list[tuple[str, list[str]]], keyword: str) -> list[str]: - """Find a row by keyword in the label and return its values.""" - keyword = keyword.lower() - for label, values in rows: - if keyword in label.lower(): - return values - return [] + return get_row_values(rows, keyword) @staticmethod def _extract_upstream_modulation(raw: str) -> str: - """Extract modulation from upstream field. - - Input may be "[3] QPSK [3] 64QAM" (BR tags become spaces). - Returns the last/highest modulation without the bracket prefix. - """ - if not raw: - return "" - # Split on common separators and find modulation entries - parts = re.split(r'[\n\r]+', raw.strip()) - last_mod = "" - for part in parts: - part = part.strip() - if not part: - continue - # Remove bracket prefix like "[3] " - cleaned = re.sub(r'^\[\d+\]\s*', '', part) - if cleaned: - last_mod = cleaned - return last_mod - - # -- Value parsers (delegated to shared utils) -- + return extract_upstream_modulation(raw) @staticmethod def _parse_freq_hz(freq_str: str) -> str: - from .utils import hz_to_mhz return hz_to_mhz(freq_str) @staticmethod def _parse_number(val_str: str) -> float: - from .utils import parse_number return parse_number(val_str) diff --git a/app/drivers/sb6183.py b/app/drivers/sb6183.py index d0f683cc..9c4390d1 100644 --- a/app/drivers/sb6183.py +++ b/app/drivers/sb6183.py @@ -14,7 +14,7 @@ from bs4 import BeautifulSoup from .base import ModemDriver -from .utils import hz_to_mhz, parse_number +from .formats.html_rows import parse_sb6183_downstream, parse_sb6183_upstream from ..types import ConnectionInfo, DeviceInfo, DocsisData, RawChannel log = logging.getLogger("docsis.driver.sb6183") @@ -27,6 +27,8 @@ class SB6183Driver(ModemDriver): ``/RgConnect.asp`` and product information from ``/RgSwInfo.asp``. """ + FORMAT_FAMILIES = ("sb6183_html",) + def __init__(self, url: str, user: str, password: str): super().__init__(url.rstrip("/"), user, password) self._session = requests.Session() @@ -109,50 +111,10 @@ def get_connection_info(self) -> ConnectionInfo: return {} def _parse_downstream(self, table) -> list[RawChannel]: - """Parse downstream table where each row is one channel.""" - if not table: - return [] - result: list[RawChannel] = [] - for tr in table.find_all("tr"): - cells = [td.get_text(" ", strip=True) for td in tr.find_all("td")] - if len(cells) < 9 or not cells[3].isdigit() or cells[1].strip().lower() != "locked": - continue - try: - snr = parse_number(cells[6]) - result.append({ - "channelID": int(cells[3]), - "frequency": hz_to_mhz(cells[4]), - "powerLevel": parse_number(cells[5]), - "mer": snr, - "mse": -snr if snr else None, - "modulation": cells[2], - "corrErrors": int(parse_number(cells[7])), - "nonCorrErrors": int(parse_number(cells[8])), - }) - except (ValueError, TypeError, IndexError) as e: - log.warning("Failed to parse SB6183 DS channel: %s", e) - return result + return parse_sb6183_downstream(table).value def _parse_upstream(self, table) -> list[RawChannel]: - """Parse upstream table where each row is one channel.""" - if not table: - return [] - result: list[RawChannel] = [] - for tr in table.find_all("tr"): - cells = [td.get_text(" ", strip=True) for td in tr.find_all("td")] - if len(cells) < 7 or not cells[3].isdigit() or cells[1].strip().lower() != "locked": - continue - try: - result.append({ - "channelID": int(cells[3]), - "frequency": hz_to_mhz(cells[5]), - "powerLevel": parse_number(cells[6]), - "modulation": cells[2], - "multiplex": cells[2], - }) - except (ValueError, TypeError, IndexError) as e: - log.warning("Failed to parse SB6183 US channel: %s", e) - return result + return parse_sb6183_upstream(table).value @staticmethod def _is_status_page(html: str) -> bool: diff --git a/app/drivers/sb6190.py b/app/drivers/sb6190.py index 18f30f0c..f1fa2adb 100644 --- a/app/drivers/sb6190.py +++ b/app/drivers/sb6190.py @@ -17,6 +17,8 @@ from bs4 import BeautifulSoup from .base import ModemDriver +from .formats.html_rows import parse_sb6190_downstream, parse_sb6190_upstream +from .formats.primitives import normalize_mhz, parse_number from ..types import DocsisData, DeviceInfo, ConnectionInfo, RawChannel log = logging.getLogger("docsis.driver.sb6190") @@ -30,6 +32,8 @@ class SB6190Driver(ModemDriver): is scraped from /cgi-bin/status where each table row is one channel. """ + FORMAT_FAMILIES = ("sb6190_html",) + def __init__(self, url, user, password): if url.startswith("http://"): url = "https://" + url[len("http://"):] @@ -124,69 +128,19 @@ def get_connection_info(self) -> ConnectionInfo: # -- Parsers -- def _parse_downstream(self, table) -> list[RawChannel]: - """Parse downstream table: each row = one channel. - - Columns: Channel | Lock Status | Modulation | Channel ID | - Frequency | Power | SNR | Corrected | Uncorrectables - """ - if not table: - return [] - result = [] - for tr in table.find_all("tr"): - cells = [td.get_text(strip=True) for td in tr.find_all("td")] - if len(cells) < 9 or not cells[3].isdigit() or cells[1].strip().lower() != "locked": - continue - try: - snr = self._parse_number(cells[6]) - result.append({ - "channelID": int(cells[3]), - "frequency": self._normalize_mhz(cells[4]), - "powerLevel": self._parse_number(cells[5]), - "mer": snr, - "mse": -snr if snr else None, - "modulation": cells[2], - "corrErrors": int(self._parse_number(cells[7])), - "nonCorrErrors": int(self._parse_number(cells[8])), - }) - except (ValueError, TypeError, IndexError) as e: - log.warning("Failed to parse SB6190 DS channel: %s", e) - return result + return parse_sb6190_downstream(table).value def _parse_upstream(self, table) -> list[RawChannel]: - """Parse upstream table: each row = one channel. - - Columns: Channel | Lock Status | US Channel Type | Channel ID | - Symbol Rate | Frequency | Power - """ - if not table: - return [] - result = [] - for tr in table.find_all("tr"): - cells = [td.get_text(strip=True) for td in tr.find_all("td")] - if len(cells) < 7 or not cells[3].isdigit() or cells[1].strip().lower() != "locked": - continue - try: - result.append({ - "channelID": int(cells[3]), - "frequency": self._normalize_mhz(cells[5]), - "powerLevel": self._parse_number(cells[6]), - "modulation": cells[2], - "multiplex": cells[2], - }) - except (ValueError, TypeError, IndexError) as e: - log.warning("Failed to parse SB6190 US channel: %s", e) - return result + return parse_sb6190_upstream(table).value # -- Value helpers -- @staticmethod def _normalize_mhz(freq_str: str) -> str: - from .utils import normalize_mhz return normalize_mhz(freq_str) @staticmethod def _parse_number(val_str: str) -> float: - from .utils import parse_number return parse_number(val_str) @staticmethod diff --git a/app/drivers/sercom_dm1000.py b/app/drivers/sercom_dm1000.py index 9f5e8c89..09da02e4 100644 --- a/app/drivers/sercom_dm1000.py +++ b/app/drivers/sercom_dm1000.py @@ -14,14 +14,20 @@ import base64 import logging -import math from typing import Any import requests from ..types import ConnectionInfo, DeviceInfo, DocsisData, RawChannel from .base import ModemDriver -from .utils import hz_to_mhz, normalize_modulation +from .formats.sercom import ( + _profile_modulation, + parse_sercom_ds_ofdm, + parse_sercom_ds_scqam, + parse_sercom_us_ofdma, + parse_sercom_us_scqam, +) +from .format_compat import unwrap_sercom log = logging.getLogger("docsis.driver.sercom_dm1000") @@ -38,6 +44,8 @@ class SercomDM1000Driver(ModemDriver): """Driver for authenticated Sercom DM1000 cable modem UIs.""" + FORMAT_FAMILIES = ("sercom_dm1000_json",) + def __init__(self, url: str, user: str, password: str): super().__init__(url.rstrip("/"), user, password) self._session = requests.Session() @@ -386,182 +394,17 @@ def _first_text(payload: dict[str, Any], *keys: str) -> str: return "" def _parse_ds_scqam(self, rows: list[dict[str, Any]]) -> list[RawChannel]: - channels: list[RawChannel] = [] - for row in rows: - try: - modulation = normalize_modulation(row.get("qamD", "")) - if not modulation or modulation in {"QAM_NONE", "NONE"}: - continue - snr = float(row["SNRD"]) - channels.append({ - "channelID": int(row["DCIDD"]), - "frequency": hz_to_mhz(row.get("FreqD", "")), - "powerLevel": float(row["PowerD"]), - "modulation": modulation, - "mer": snr, - # Keep the long-standing DOCSight raw-channel convention: - # drivers expose MSE as the inverse of SNR/MER when the - # modem does not provide a separate MSE counter. - "mse": -snr, - "corrErrors": int(row["correctedsD"]), - "nonCorrErrors": int(row["uncorrectedsD"]), - }) - except (KeyError, TypeError, ValueError) as exc: - log.warning("Failed to parse Sercom DM1000 DS row: %s", exc) - return channels + return parse_sercom_ds_scqam(rows).value def _parse_ds_ofdm(self, rows: list[dict[str, Any]]) -> list[RawChannel]: - channels: list[RawChannel] = [] - for row in rows: - try: - if str(row.get("PLC", "")).strip().upper() != "YES": - continue - if str(row.get("MDC1", "")).strip().upper() != "YES": - continue - mer = self._optional_float(row.get("AV_Data")) - if mer is None: - mer = self._optional_float(row.get("AV_PLC")) - if mer is None: - continue - channels.append({ - "channelID": int(row["num"]), - "type": "OFDM", - "frequency": hz_to_mhz(row.get("OFDMFreq", "")), - "powerLevel": float(row["PLC_power"]), - "modulation": "OFDM", - # The Sercom UI labels these as average OFDM values; use - # data-subcarrier MER first and PLC MER only as a fallback. - "mer": mer, - "mse": None, - "corrErrors": None, - "nonCorrErrors": None, - }) - except (KeyError, TypeError, ValueError) as exc: - log.warning("Failed to parse Sercom DM1000 DS OFDM row: %s", exc) - return channels + return parse_sercom_ds_ofdm(rows).value def _parse_us_scqam(self, rows: list[dict[str, Any]]) -> list[RawChannel]: - channels: list[RawChannel] = [] - for row in rows: - try: - modulation = normalize_modulation(row.get("modulation", "")) - upstream = str(row.get("upstream", "")).strip() - rate_text = str(row.get("rate", "")).strip() - power = float(row["rep_power"]) - if ( - not modulation - or modulation in {"QAM_NONE", "NONE"} - or upstream in {"", "---"} - or not math.isfinite(power) - or rate_text.lower() == "invalid" - ): - continue - channel: RawChannel = { - "channelID": int(upstream), - "frequency": hz_to_mhz(row.get("Freq", "")), - "powerLevel": power, - "modulation": modulation, - "multiplex": "ATDMA", - } - symbol_rate = self._symbol_rate_ksym(rate_text) - if symbol_rate is not None: - channel["symbolRate"] = symbol_rate - channels.append(channel) - except (KeyError, TypeError, ValueError) as exc: - log.warning("Failed to parse Sercom DM1000 US row: %s", exc) - return channels + return parse_sercom_us_scqam(rows).value def _parse_us_ofdma(self, rows: list[dict[str, Any]]) -> list[RawChannel]: - channels: list[RawChannel] = [] - for column in self._pivot_indexed_rows(rows): - try: - state = str(column.get("STATE", "")).strip().upper() - power_state = str(column.get("Power", "")).strip().upper() - # Captured active state was RNG3. Treat any explicit non-disabled - # state as usable because Sercom firmware may report other - # ranging/operational labels while the OFDMA channel is up. - if power_state != "ON" or state in {"", "DISABLED", "OFF"}: - continue - frequency_value = column.get("Center Freq SC0") - frequency_mhz = self._optional_float(frequency_value) - if frequency_mhz is None or frequency_mhz <= 0: - continue - channel: RawChannel = { - "channelID": int(str(column.get("CH", "")).strip()), - "type": "OFDMA", - # The captured label is SC0 rather than a guaranteed center; - # preserve the modem-exposed frequency instead of inventing one. - "frequency": hz_to_mhz(frequency_value), - "powerLevel": self._ofdma_power(column), - "modulation": "OFDMA", - "multiplex": "OFDMA", - } - profile_modulation = self._profile_modulation_from_bits(column.get("bit Loading")) - if profile_modulation: - channel["profile_modulation"] = profile_modulation - channels.append(channel) - except (KeyError, TypeError, ValueError) as exc: - log.warning("Failed to parse Sercom DM1000 US OFDMA row: %s", exc) - return channels - - @staticmethod - def _pivot_indexed_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - indexes: list[str] = [] - pivot: dict[str, dict[str, Any]] = {} - for row in rows: - name = str(row.get("name") or "").strip() - if not name: - continue - for key, value in row.items(): - if not key.startswith("index"): - continue - if key not in pivot: - pivot[key] = {} - indexes.append(key) - pivot[key][name] = value - return [pivot[key] for key in sorted(indexes, key=SercomDM1000Driver._index_sort_key)] - - @staticmethod - def _index_sort_key(index_name: str) -> int: - suffix = index_name.removeprefix("index") - try: - return int(suffix) - except ValueError: - return 0 - - @staticmethod - def _ofdma_power(column: dict[str, Any]) -> float | None: - if "rep power1_6" not in column: - log.warning("Sercom DM1000 OFDMA row missing rep power1_6; leaving power unsupported") - return None - return SercomDM1000Driver._optional_float(column.get("rep power1_6")) - - @staticmethod - def _optional_float(value: Any) -> float | None: - try: - number = float(str(value).strip()) - except (TypeError, ValueError): - return None - return number if math.isfinite(number) else None - - @staticmethod - def _symbol_rate_ksym(value: Any) -> int | None: - try: - number = float(str(value).strip()) - except (TypeError, ValueError): - return None - if not math.isfinite(number): - return None - return int(round(number * 1000)) + return unwrap_sercom(parse_sercom_us_ofdma(rows), log) @staticmethod def _profile_modulation_from_bits(value: Any) -> str | None: - try: - bits = int(float(str(value).strip())) - except (TypeError, ValueError): - return None - if bits == 2: - return "QPSK" - if bits <= 0 or bits > 12: - return None - return f"{2 ** bits}QAM" + return _profile_modulation(value) diff --git a/app/drivers/surfboard.py b/app/drivers/surfboard.py index 8c0d022d..87d21eb7 100644 --- a/app/drivers/surfboard.py +++ b/app/drivers/surfboard.py @@ -34,6 +34,12 @@ from requests.adapters import HTTPAdapter from .base import ModemDriver +from .formats.primitives import hz_to_mhz +from .formats.surfboard import ( + normalize_surfboard_modulation, + parse_surfboard_downstream, + parse_surfboard_upstream, +) from ..types import DocsisData, DeviceInfo, ConnectionInfo log = logging.getLogger("docsis.driver.surfboard") @@ -99,6 +105,8 @@ class SurfboardDriver(ModemDriver): re-authenticates when a request fails or when no session exists yet. """ + FORMAT_FAMILIES = ("arris_html", "surfboard_hnap") + def __init__(self, url: str, user: str, password: str): url = self._normalize_url(url) self._http_fallback_url = "" @@ -814,133 +822,18 @@ def _hnap_post(self, action: str, body: dict[str, str], *, assert last_err is not None raise last_err - # -- Channel parsers -- + # Compatibility parser seams. def _parse_downstream(self, raw: str) -> tuple[list, list]: - """Parse downstream channel string into (docsis30, docsis31) lists.""" - if not raw: - return [], [] - - ds30 = [] - ds31 = [] - - for entry in raw.split("|+|"): - entry = entry.strip() - if not entry: - continue - - fields = entry.split("^") - # Remove trailing empty from trailing "^" - if fields and fields[-1] == "": - fields = fields[:-1] - - if len(fields) < _DS_FIELDS: - continue - - lock = fields[1].strip() - if lock != "Locked": - continue - - try: - modulation = fields[2].strip() - channel_id = int(fields[3]) - freq_hz = int(fields[4]) - power = float(fields[5].strip()) - snr = float(fields[6].strip()) - corr = int(fields[7]) - uncorr = int(fields[8]) - - if "OFDM" in modulation.upper(): - ds31.append({ - "channelID": channel_id, - "type": "OFDM", - "frequency": self._hz_to_mhz(freq_hz), - "powerLevel": power, - "mer": snr, - "mse": None, - "corrErrors": corr, - "nonCorrErrors": uncorr, - }) - else: - ds30.append({ - "channelID": channel_id, - "frequency": self._hz_to_mhz(freq_hz), - "powerLevel": power, - "mer": snr, - "mse": -snr, - "modulation": self._normalize_modulation(modulation), - "corrErrors": corr, - "nonCorrErrors": uncorr, - }) - except (ValueError, IndexError) as e: - log.warning("Failed to parse SURFboard DS channel: %s", e) - - return ds30, ds31 + return parse_surfboard_downstream(raw).value def _parse_upstream(self, raw: str) -> tuple[list, list]: - """Parse upstream channel string into (docsis30, docsis31) lists.""" - if not raw: - return [], [] - - us30 = [] - us31 = [] - - for entry in raw.split("|+|"): - entry = entry.strip() - if not entry: - continue - - fields = entry.split("^") - if fields and fields[-1] == "": - fields = fields[:-1] - - if len(fields) < _US_FIELDS: - continue - - lock = fields[1].strip() - if lock != "Locked": - continue - - try: - ch_type = fields[2].strip() - channel_id = int(fields[3]) - freq_hz = int(fields[5]) - power = float(fields[6].strip()) - - if "OFDMA" in ch_type.upper(): - us31.append({ - "channelID": channel_id, - "type": "OFDMA", - "frequency": self._hz_to_mhz(freq_hz), - "powerLevel": power, - "modulation": "OFDMA", - "multiplex": "", - }) - else: - us30.append({ - "channelID": channel_id, - "frequency": self._hz_to_mhz(freq_hz), - "powerLevel": power, - "modulation": ch_type, - "multiplex": ch_type, - }) - except (ValueError, IndexError) as e: - log.warning("Failed to parse SURFboard US channel: %s", e) - - return us30, us31 - - # -- Value helpers -- + return parse_surfboard_upstream(raw).value @staticmethod def _hz_to_mhz(freq_hz: int) -> str: - from .utils import hz_to_mhz return hz_to_mhz(freq_hz) @staticmethod def _normalize_modulation(mod: str) -> str: - """Normalize modulation string. - - "256QAM" -> "256QAM" - "OFDM PLC" -> "OFDM PLC" - """ - return mod.strip() if mod else "" + return normalize_surfboard_modulation(mod) diff --git a/app/drivers/tc4400.py b/app/drivers/tc4400.py index 5b6f8ace..5889af52 100644 --- a/app/drivers/tc4400.py +++ b/app/drivers/tc4400.py @@ -18,7 +18,14 @@ from bs4 import BeautifulSoup from .base import ModemDriver -from .utils import normalize_modulation +from .formats.html_rows import ( + _cell as format_cell, + _tc_columns, + _tc_header_row, + parse_tc4400_downstream, + parse_tc4400_upstream, +) +from .formats.primitives import normalize_modulation, parse_mhz_value, parse_number from ..types import DocsisData, DeviceInfo, ConnectionInfo, RawChannel log = logging.getLogger("docsis.driver.tc4400") @@ -31,6 +38,8 @@ class TC4400Driver(ModemDriver): HTML tables (no JSON API available). Response time can be ~20s. """ + FORMAT_FAMILIES = ("tc4400_html",) + def __init__(self, url: str, user: str, password: str): super().__init__(url, user, password) self._session = requests.Session() @@ -115,191 +124,21 @@ def get_connection_info(self) -> ConnectionInfo: # ── Parsers ──────────────────────────────────────────────── def _parse_downstream(self, table) -> list[RawChannel]: - """Parse downstream HTML table (SC-QAM + OFDM channels).""" - rows = table.find_all("tr") - if not rows: - return [] - - header_row = self._find_header_row(rows) - if header_row is None: - return [] - - headers = [th.get_text(strip=True).lower() for th in header_row.find_all(["th", "td"])] - col = self._map_columns(headers) - - result = [] - data_rows = [r for r in rows if r != header_row] - for row in data_rows: - cells = [td.get_text(strip=True) for td in row.find_all("td")] - if len(cells) < 4: - continue - - lock = self._cell(cells, col["lock_status"]) - if lock.lower() != "locked": - continue - - try: - channel_id = self._cell(cells, col["channel_id"], "0") - - # Use channel_type (OFDM/SC-QAM) for type, modulation as fallback - channel_type = self._cell(cells, col["channel_type"], "") - modulation = self._normalize_modulation( - self._cell(cells, col["modulation"]) - ) - - # For OFDM channels, channel_type gives us OFDM vs SC-QAM - # For SC-QAM, modulation gives us "256QAM" etc. - if channel_type.upper() in ("OFDM",): - final_type = "OFDM" - elif channel_type.upper() in ("SC-QAM",): - final_type = modulation if modulation else "QAM" - else: - final_type = modulation if modulation else "unknown" - - frequency = self._parse_frequency( - self._cell(cells, col["frequency"]) - ) - power = self._parse_number(self._cell(cells, col["power"])) - snr = self._parse_number(self._cell(cells, col["snr"])) - corr = int(self._parse_number(self._cell(cells, col["corrected"]))) - uncorr = int( - self._parse_number(self._cell(cells, col["uncorrected"])) - ) - - is_ofdm = final_type == "OFDM" - - result.append({ - "channelID": channel_id, - "type": final_type, - "frequency": f"{int(frequency)} MHz" if frequency else "", - "powerLevel": power, - "mse": None if is_ofdm else (-snr if snr else None), - "mer": snr if snr else None, - "latency": 0, - "corrError": corr, - "nonCorrError": uncorr, - }) - except (ValueError, TypeError, IndexError) as e: - log.warning("Failed to parse TC4400 DS row: %s", e) - - return result + return parse_tc4400_downstream(table).value def _parse_upstream(self, table) -> list[RawChannel]: - """Parse upstream HTML table (ATDMA + OFDMA channels).""" - rows = table.find_all("tr") - if not rows: - return [] - - header_row = self._find_header_row(rows) - if header_row is None: - return [] - - headers = [th.get_text(strip=True).lower() for th in header_row.find_all(["th", "td"])] - col = self._map_columns(headers) - - result = [] - data_rows = [r for r in rows if r != header_row] - for row in data_rows: - cells = [td.get_text(strip=True) for td in row.find_all("td")] - if len(cells) < 4: - continue - - lock = self._cell(cells, col["lock_status"]) - if lock.lower() != "locked": - continue - - try: - channel_id = self._cell(cells, col["channel_id"], "0") - modulation = self._normalize_modulation( - self._cell(cells, col["modulation"]) - ) - frequency = self._parse_frequency( - self._cell(cells, col["frequency"]) - ) - power = self._parse_number(self._cell(cells, col["power"])) - - result.append({ - "channelID": channel_id, - "type": modulation, - "frequency": f"{int(frequency)} MHz" if frequency else "", - "powerLevel": power, - "multiplex": "", - }) - except (ValueError, TypeError, IndexError) as e: - log.warning("Failed to parse TC4400 US row: %s", e) - - return result + return parse_tc4400_upstream(table).value @staticmethod def _find_header_row(rows): - """Find the actual header row, skipping title rows with colspan.""" - for row in rows: - cells = row.find_all(["th", "td"]) - if cells and any(cell.get("colspan") for cell in cells): - continue - if cells and len(cells) > 3: - return row - return None + return _tc_header_row(rows) def _map_columns(self, headers: list[str]) -> dict[str, int | None]: - """Map header names to column indices. - - Uses fuzzy matching to handle firmware variations like - "Received Level" vs "Receive Level", "Channel ID" vs "Channel Index", - "Channel Type" vs "Modulation / Profile ID". - """ - col = { - "channel_id": None, - "lock_status": None, - "modulation": None, - "channel_type": None, - "frequency": None, - "power": None, - "snr": None, - "corrected": None, - "uncorrected": None, - } - - for i, h in enumerate(headers): - if "channel" in h and "id" in h: - col["channel_id"] = i - elif "channel" in h and "index" in h and col["channel_id"] is None: - col["channel_id"] = i - elif "lock" in h: - col["lock_status"] = i - elif "channel" in h and "type" in h: - col["channel_type"] = i - elif "modulation" in h or "profile" in h: - col["modulation"] = i - elif "freq" in h: - col["frequency"] = i - elif any(kw in h for kw in ("power", "receive", "transmit")): - col["power"] = i - elif "snr" in h or "mer" in h: - col["snr"] = i - elif "corrected" in h and "un" not in h: - col["corrected"] = i - elif "uncorrect" in h: - col["uncorrected"] = i - - # Positional fallbacks for common TC4400 table layout - if col["channel_id"] is None: - col["channel_id"] = 0 - if col["lock_status"] is None: - col["lock_status"] = 1 - if col["channel_type"] is None and col["modulation"] is None: - col["modulation"] = 2 - if col["frequency"] is None: - col["frequency"] = 3 - - return col + return _tc_columns(headers) @staticmethod def _cell(cells: list[str], index: int | None, default: str = "") -> str: - """Safely get a cell value by index.""" - if index is None or index >= len(cells): - return default - return cells[index] + return format_cell(cells, index, default) def _parse_info_table(self, soup) -> dict[str, str]: """Parse key-value info table from /cmswinfo.html.""" @@ -317,44 +156,12 @@ def _parse_info_table(self, soup) -> dict[str, str]: # ── Value Parsers ────────────────────────────────────────── def _parse_frequency(self, freq_str: str) -> float: - """Parse frequency string to MHz float. - - Handles: "279000000 Hz", "350000 kHz", "279 MHz" - Note: Returns float (MHz), not a string. TC4400 formats - the result itself in _parse_downstream/_parse_upstream. - """ - if not freq_str: - return 0.0 - - parts = freq_str.strip().split() - try: - value = float(parts[0]) - except (IndexError, ValueError): - return 0.0 - - unit = parts[1].lower() if len(parts) > 1 else "" - if unit == "hz": - return value / 1_000_000 - elif unit == "khz": - return value / 1_000 - elif unit == "mhz": - return value - elif value > 1_000_000: - return value / 1_000_000 - elif value > 1_000: - return value / 1_000 - return value + return parse_mhz_value(freq_str) @staticmethod def _parse_number(value: str) -> float: - from .utils import parse_number return parse_number(value) @staticmethod def _normalize_modulation(modulation: str) -> str: - """Normalize modulation string to analyzer format. - - Input: "256QAM", "256-qam", "qam_256", "OFDM", "ATDMA", "OFDMA" - Output: "256QAM", "256QAM", "256QAM", "OFDM", "ATDMA", "OFDMA" - """ return normalize_modulation(modulation) diff --git a/app/drivers/ultrahub7.py b/app/drivers/ultrahub7.py index 19d3d258..8730c225 100644 --- a/app/drivers/ultrahub7.py +++ b/app/drivers/ultrahub7.py @@ -17,6 +17,11 @@ from cryptography.hazmat.primitives.ciphers.aead import AESCCM from .base import ModemDriver +from .formats.vodafone import ( + parse_ultrahub7_downstream, + parse_ultrahub7_json, + parse_ultrahub7_upstream, +) from .utils import pbkdf2_sha256 from ..types import DocsisData, DeviceInfo, ConnectionInfo, RawChannel @@ -30,6 +35,8 @@ class UltraHub7Driver(ModemDriver): DOCSIS data is fetched via clean JSON API endpoints. """ + FORMAT_FAMILIES = ("ultrahub7_json",) + def __init__(self, url: str, user: str, password: str): super().__init__(url, user, password) self._session = requests.Session() # Persistent session for cookie handling @@ -246,14 +253,10 @@ def get_docsis_data(self) -> DocsisData: us_response.raise_for_status() us_data = us_response.json() - downstream = self._parse_downstream_channels(ds_data.get("channels", [])) - upstream = self._parse_upstream_channels(us_data.get("channels", [])) - - return { - "docsis": "3.1", # Ultra Hub 7 is DOCSIS 3.1 - "downstream": downstream, - "upstream": upstream - } + return parse_ultrahub7_json({ + "downstream": ds_data.get("channels", []), + "upstream": us_data.get("channels", []), + }).value except requests.RequestException as e: log.error("Failed to fetch DOCSIS data: %s", e) @@ -279,116 +282,7 @@ def get_connection_info(self) -> ConnectionInfo: return {} def _parse_downstream_channels(self, channels: list[dict[str, str]]) -> list[RawChannel]: - """Parse downstream channel data from Ultra Hub 7 API format.""" - result = [] - - for ch in channels: - try: - channel_id = int(ch.get("ChannelID", "0")) - frequency = self._parse_frequency(ch.get("Frequency", "0")) - modulation = self._normalize_modulation(ch.get("Modulation", "")) - power = self._parse_power(ch.get("PowerLevel", "0")) - snr = self._parse_snr(ch.get("SNRLevel", "")) - - # FritzBox-compatible format expected by analyzer - result.append({ - "channelID": str(channel_id), - "type": modulation, - "frequency": f"{int(frequency)} MHz", - "powerLevel": power, - "mer": snr if snr > 0 else None, # DOCSIS 3.1 uses MER - "mse": None, # Not provided - "latency": 0, - "corrErrors": None, # Not provided by Ultra Hub 7 API - "nonCorrErrors": None # Not provided by Ultra Hub 7 API - }) - - except (ValueError, TypeError) as e: - log.warning("Failed to parse downstream channel %s: %s", ch, e) - continue - - return result + return parse_ultrahub7_downstream(channels).value def _parse_upstream_channels(self, channels: list[dict[str, str]]) -> list[RawChannel]: - """Parse upstream channel data from Ultra Hub 7 API format.""" - result = [] - - for ch in channels: - try: - channel_id = int(ch.get("ChannelID", "0")) - frequency = self._parse_frequency(ch.get("Frequency", "0")) - modulation = self._normalize_modulation(ch.get("Modulation", "")) - power = self._parse_power(ch.get("PowerLevel", "0")) - - # FritzBox-compatible format expected by analyzer - result.append({ - "channelID": str(channel_id), - "type": modulation, - "frequency": f"{int(frequency)} MHz", - "powerLevel": power, - "multiplex": "" # Not relevant for display - }) - - except (ValueError, TypeError) as e: - log.warning("Failed to parse upstream channel %s: %s", ch, e) - continue - - return result - - def _parse_frequency(self, freq_str: str) -> float: - """Parse frequency string to MHz float. - - Handles both single and double spaces: "264 MHz" and "51 MHz" - """ - if not freq_str: - return 0.0 - - try: - parts = freq_str.strip().split() - return float(parts[0]) - except (IndexError, ValueError): - log.warning("Failed to parse frequency: %s", freq_str) - return 0.0 - - def _parse_power(self, power_str: str) -> float: - """Parse power string to dBmV float. - - Format: "15.1 dBmV" → 15.1 - """ - if not power_str: - return 0.0 - - try: - parts = power_str.strip().split() - return float(parts[0]) - except (IndexError, ValueError): - log.warning("Failed to parse power: %s", power_str) - return 0.0 - - def _parse_snr(self, snr_str: str) -> float: - """Parse SNR string to dB float. - - Format: "41.9 dB" → 41.9 - Empty string → 0.0 (upstream channels don't have SNR) - """ - if not snr_str or snr_str.strip() == "": - return 0.0 - - try: - # Split on space and take first part - parts = snr_str.strip().split() - return float(parts[0]) - except (IndexError, ValueError): - log.warning("Failed to parse SNR: %s", snr_str) - return 0.0 - - def _normalize_modulation(self, modulation: str) -> str: - """Normalize modulation string to match analyzer expectations. - - Ultra Hub 7 API returns: "256QAM", "64QAM", "4096QAM", "OFDM", "OFDMA" - Analyzer expects: "256QAM", "64QAM", etc. (uppercase, no hyphens) - """ - if not modulation: - return "" - - return modulation.upper().replace("-", "") + return parse_ultrahub7_upstream(channels).value diff --git a/app/drivers/utils.py b/app/drivers/utils.py index eaa2e084..9c962bdb 100644 --- a/app/drivers/utils.py +++ b/app/drivers/utils.py @@ -7,12 +7,28 @@ import hashlib import logging -import math -import re import ssl from requests.adapters import HTTPAdapter +from .formats.primitives import ( + hz_to_mhz, + normalize_mhz, + normalize_modulation, + parse_number, + parse_optional_finite_float, +) + +__all__ = [ + "hz_to_mhz", + "make_legacy_tls_adapter", + "normalize_mhz", + "normalize_modulation", + "parse_number", + "parse_optional_finite_float", + "pbkdf2_sha256", +] + log = logging.getLogger("docsis.drivers.utils") @@ -28,163 +44,6 @@ def pbkdf2_sha256(key_material: bytes, salt: bytes, *, length: int = 16, iterati return hashlib.pbkdf2_hmac("sha256", key_material, salt, iterations, dklen=length) -# --------------------------------------------------------------------------- -# Value parsing -# --------------------------------------------------------------------------- - -def parse_number(value: str) -> float: - """Parse a numeric value from a string with an optional unit suffix. - - Examples:: - - '43.3 dBmV' -> 43.3 - '-0.32 dBmV' -> -0.32 - '41.8 dB' -> 41.8 - '10.50 dBmV' -> 10.5 - '5.120 Msym/sec' -> 5.12 - '' -> 0.0 - - Duplicated in: cm3000, cm3500, tc4400, sb6141, sb6190, arris_html, - ultrahub7 (_parse_power, _parse_snr, _parse_frequency). - """ - if not value: - return 0.0 - parts = value.strip().split() - try: - return float(parts[0]) - except (ValueError, IndexError): - return 0.0 - - -def parse_optional_finite_float(value) -> float | None: - """Parse a finite float while preserving missing or invalid values as unsupported.""" - try: - number = float(str(value).strip()) - except (TypeError, ValueError): - return None - return number if math.isfinite(number) else None - - -def hz_to_mhz(freq) -> str: - """Convert a frequency value (Hz) to a human-readable MHz string. - - Accepts int, float, or string inputs. - - Examples:: - - 591000000 -> '591 MHz' - 495000000 -> '495 MHz' - 29200000 -> '29.2 MHz' - '795000000 Hz' -> '795 MHz' - '350000 kHz' -> '350 MHz' (string with kHz — handled via parse) - 0 -> '0 MHz' - - Duplicated in: cm3000, cm3500, surfboard, sb6141, sb6190, hitron, - sagemcom, arris_html, cgm4981. - """ - # Numeric input (int or float) - if isinstance(freq, (int, float)): - if freq == 0: - return "0 MHz" - mhz = float(freq) / 1_000_000 - if mhz == int(mhz): - return f"{int(mhz)} MHz" - return f"{mhz:.1f} MHz" - - # String input — parse Hz value and optional unit - freq_str = str(freq).strip() - if not freq_str: - return "" - parts = freq_str.split() - try: - val = float(parts[0]) - except (ValueError, IndexError): - return freq_str - - unit = parts[1].lower() if len(parts) > 1 else "" - if unit == "hz": - mhz = val / 1_000_000 - elif unit == "khz": - mhz = val / 1_000 - elif unit == "mhz": - mhz = val - elif val > 1_000_000: - mhz = val / 1_000_000 - elif val > 1_000: - mhz = val / 1_000 - else: - mhz = val - - if mhz == int(mhz): - return f"{int(mhz)} MHz" - return f"{mhz:.1f} MHz" - - -_MOD_TOKEN_SPLIT = re.compile(r"[\s_\-]+") - - -def normalize_modulation(modulation) -> str: - """Normalise a modulation string to a canonical analyzer label. - - Handles vendor variations like "256QAM" / "256-qam" / "256 qam" / - "qam256" / "qam_256" -> "256QAM", "QPSK" / "qpsk" -> "QPSK", - "OFDM" / "ofdm" -> "OFDM", and similarly for OFDMA / ATDMA / TDMA. - - Unknown non-empty values are returned uppercased and stripped so - downstream code keeps a stable, readable label. - """ - if modulation is None: - return "" - if not isinstance(modulation, str): - modulation = str(modulation) - raw = modulation.strip() - if not raw: - return "" - mod = _MOD_TOKEN_SPLIT.sub("", raw).lower() - if not mod: - return raw.upper() - - if "qpsk" in mod: - return "QPSK" - if "ofdma" in mod: - return "OFDMA" - if "ofdm" in mod: - return "OFDM" - if "atdma" in mod: - return "ATDMA" - if mod == "tdma": - return "TDMA" - if "qam" in mod: - num = mod.replace("qam", "") - if num.isdigit(): - return f"{num}QAM" - return "QAM" if not num else f"{num.upper()}QAM" - return raw.upper() - - -def normalize_mhz(freq_str: str) -> str: - """Normalise a frequency string already in MHz to a clean format. - - Examples:: - - '465.00 MHz' -> '465 MHz' - '17 MHz' -> '17 MHz' - '29.2' -> '29.2 MHz' (no unit) - - Used by sb6190, cm3500. - """ - if not freq_str: - return "" - parts = freq_str.strip().split() - try: - mhz = float(parts[0]) - if mhz == int(mhz): - return f"{int(mhz)} MHz" - return f"{mhz:.1f} MHz" - except (ValueError, IndexError): - return freq_str - - # --------------------------------------------------------------------------- # TLS adapters for modems with legacy/weak certificates # --------------------------------------------------------------------------- diff --git a/app/drivers/vodafone_station.py b/app/drivers/vodafone_station.py index 8a4ba046..6fc0c1d1 100644 --- a/app/drivers/vodafone_station.py +++ b/app/drivers/vodafone_station.py @@ -22,7 +22,15 @@ from cryptography.hazmat.primitives.ciphers.aead import AESCCM from .base import ModemDriver -from .utils import normalize_modulation, pbkdf2_sha256 +from .formats.primitives import normalize_modulation +from .formats.vodafone import ( + parse_tg_frequency, + parse_tg_power, + parse_vodafone_number, + parse_vodafone_station_cga_json, + parse_vodafone_station_tg_embedded_json, +) +from .utils import pbkdf2_sha256 from ..types import DocsisData, DeviceInfo, ConnectionInfo log = logging.getLogger("docsis.driver.vodafone_station") @@ -41,6 +49,11 @@ def _aes_ccm_decrypt_hex(key: bytes, nonce: bytes, encrypted_hex: str, aad: byte class VodafoneStationDriver(ModemDriver): """Driver for Vodafone Station (CommScope/ARRIS TG6442VF/TG3442DE, CommScope/Technicolor CGA6444VF/CGA4322DE).""" + FORMAT_FAMILIES = ( + "vodafone_station_cga_json", + "vodafone_station_tg_embedded_json", + ) + VARIANT_CGA = "cga" # CGA6444VF/CGA4322DE (JSON API + double PBKDF2) VARIANT_TG = "tg" # TG6442VF/TG3442DE (HTML + AES-CCM) @@ -280,122 +293,11 @@ def _get_docsis_cga(self) -> DocsisData: self._invalidate_cga_session() raise RuntimeError(f"CGA DOCSIS data retrieval failed: {e}") - # CGA API wraps channel data inside "data" key data = raw.get("data", raw) - log.debug("CGA DOCSIS response keys: %s", list(data.keys())) - - ds_30 = [] - ds_31 = [] - us_30 = [] - us_31 = [] - - # SC-QAM Downstream channels (DOCSIS 3.0) - for ch in data.get("downstream", []) or []: - try: - channel_id = int(self._parse_number(ch.get("channelid", "0"))) - freq = self._parse_number(ch.get("CentralFrequency", "0")) - power = self._parse_number(ch.get("power", "0")) - snr = self._parse_number(ch.get("SNR", "0")) - if snr < 0: - snr = abs(snr) - modulation = self._normalize_modulation(ch.get("FFT", "")) - - if freq > 1_000_000: - freq = freq / 1_000_000 - - ds_30.append({ - "channelID": channel_id, - "type": modulation, - "frequency": f"{int(freq)} MHz" if freq else "", - "powerLevel": power, - "mse": -snr if snr else None, - "mer": snr if snr else None, - "latency": 0, - "corrError": 0, - "nonCorrError": 0, - }) - except (ValueError, TypeError) as e: - log.warning("Failed to parse CGA DS channel %s: %s", ch, e) - - # OFDM Downstream channels (DOCSIS 3.1) - for ch in data.get("ofdm_downstream", []) or []: - try: - channel_id = int(self._parse_number(ch.get("channelid_ofdm", "0"))) - freq = self._parse_number(ch.get("CentralFrequency_ofdm", "0")) - power = self._parse_number(ch.get("power_ofdm", "0")) - snr = self._parse_number(ch.get("SNR_ofdm", "0")) - if snr < 0: - snr = abs(snr) - - if freq > 1_000_000: - freq = freq / 1_000_000 - - ds_31.append({ - "channelID": channel_id, - "type": "OFDM", - "frequency": f"{int(freq)} MHz" if freq else "", - "powerLevel": power, - "mse": -snr if snr else None, - "mer": snr if snr else None, - "latency": 0, - "corrError": 0, - "nonCorrError": 0, - }) - except (ValueError, TypeError) as e: - log.warning("Failed to parse CGA OFDM DS channel %s: %s", ch, e) - - # SC-QAM Upstream channels (DOCSIS 3.0) - for ch in data.get("upstream", []) or []: - try: - channel_id = int(self._parse_number(ch.get("channelidup", "0"))) - freq = self._parse_number(ch.get("CentralFrequency", "0")) - power = self._parse_number(ch.get("power", "0")) - modulation = self._normalize_modulation(ch.get("FFT", "")) - - if freq > 1_000_000: - freq = freq / 1_000_000 - - us_30.append({ - "channelID": channel_id, - "type": modulation, - "frequency": f"{int(freq)} MHz" if freq else "", - "powerLevel": power, - "multiplex": "", - }) - except (ValueError, TypeError) as e: - log.warning("Failed to parse CGA US channel %s: %s", ch, e) - - # OFDMA Upstream channels (DOCSIS 3.1) - for ch in data.get("ofdma_upstream", []) or []: - try: - channel_id = int(self._parse_number(ch.get("channelidup", "0"))) - freq = self._parse_number(ch.get("CentralFrequency", "0")) - power = self._parse_number(ch.get("power", "0")) - modulation = self._normalize_modulation(ch.get("FFT", "")) or "OFDMA" - - if freq > 1_000_000: - freq = freq / 1_000_000 - - us_31.append({ - "channelID": channel_id, - "type": "OFDMA", - "frequency": f"{int(freq)} MHz" if freq else "", - "powerLevel": power, - "modulation": modulation, - "multiplex": "", - }) - except (ValueError, TypeError) as e: - log.warning("Failed to parse CGA OFDMA US channel %s: %s", ch, e) - - log.debug( - "CGA DOCSIS parsed: %d DS3.0 + %d DS3.1 + %d US3.0 + %d US3.1 channels", - len(ds_30), len(ds_31), len(us_30), len(us_31), - ) - - return { - "channelDs": {"docsis30": ds_30, "docsis31": ds_31}, - "channelUs": {"docsis30": us_30, "docsis31": us_31}, - } + parsed = parse_vodafone_station_cga_json(data) + if parsed.value is None: + raise TypeError("invalid CGA channel payload") + return parsed.value def _get_device_info_cga(self) -> DeviceInfo: """CGA: Retrieve device info from API.""" @@ -679,91 +581,11 @@ def _get_docsis_tg(self) -> DocsisData: self._invalidate_tg_session() raise RuntimeError(f"TG DOCSIS data retrieval failed: {e}") - html = r.text - - # Extract JSON arrays from embedded JS variables - ds_match = re.search(r"json_dsData\s*=\s*(\[.+?\])\s*;", html, re.DOTALL) - us_match = re.search(r"json_usData\s*=\s*(\[.+?\])\s*;", html, re.DOTALL) - - if not ds_match and not us_match: + parsed = parse_vodafone_station_tg_embedded_json(r.text) + if parsed.value is None: self._invalidate_tg_session() - raise RuntimeError( - "TG: Could not extract DOCSIS data from response " - "(json_dsData/json_usData not found)" - ) - - ds_raw = json.loads(ds_match.group(1)) if ds_match else [] - us_raw = json.loads(us_match.group(1)) if us_match else [] - - log.debug("TG DOCSIS: %d DS channels, %d US channels", len(ds_raw), len(us_raw)) - - ds_30 = [] - ds_31 = [] - us_30 = [] - us_31 = [] - - for ch in ds_raw: - try: - channel_id = int(float(ch.get("ChannelID", 0))) - ch_type = ch.get("ChannelType", "SC-QAM") - freq = self._parse_tg_frequency(ch.get("Frequency", "0")) - power = self._parse_tg_power(ch.get("PowerLevel", "0")) - snr = self._parse_number(ch.get("SNRLevel", "0")) - if snr < 0: - snr = abs(snr) - modulation = self._normalize_modulation(ch.get("Modulation", "")) - is_ofdm = "OFDM" in ch_type.upper() - - if is_ofdm: - modulation = modulation or "OFDM" - - ch_dict = { - "channelID": channel_id, - "type": modulation, - "frequency": f"{freq:.3f} MHz" if freq else "", - "powerLevel": power, - "mse": -snr if snr else None, - "mer": snr if snr else None, - "latency": 0, - "corrError": 0, - "nonCorrError": 0, - } - (ds_31 if is_ofdm else ds_30).append(ch_dict) - except (ValueError, TypeError) as e: - log.warning("Failed to parse TG DS channel %s: %s", ch, e) - - for ch in us_raw: - try: - channel_id = int(float(ch.get("ChannelID", 0))) - ch_type = ch.get("ChannelType", "SC-QAM") - freq = self._parse_tg_frequency(ch.get("Frequency", "0")) - power = self._parse_tg_power(ch.get("PowerLevel", "0")) - modulation = self._normalize_modulation(ch.get("Modulation", "")) - is_ofdma = "OFDMA" in ch_type.upper() - - if is_ofdma: - modulation = modulation or "OFDMA" - - ch_dict = { - "channelID": channel_id, - "type": modulation, - "frequency": f"{freq:.3f} MHz" if freq else "", - "powerLevel": power, - "multiplex": "", - } - (us_31 if is_ofdma else us_30).append(ch_dict) - except (ValueError, TypeError) as e: - log.warning("Failed to parse TG US channel %s: %s", ch, e) - - log.debug( - "TG DOCSIS parsed: %d DS3.0 + %d DS3.1 + %d US3.0 + %d US3.1 channels", - len(ds_30), len(ds_31), len(us_30), len(us_31), - ) - - return { - "channelDs": {"docsis30": ds_30, "docsis31": ds_31}, - "channelUs": {"docsis30": us_30, "docsis31": us_31}, - } + raise RuntimeError("TG: invalid embedded DOCSIS payload") + return parsed.value def _tg_docsis_request(self) -> requests.Response: """Make a single TG DOCSIS data request. @@ -859,60 +681,16 @@ def _validate_hex(value: str, name: str) -> None: @staticmethod def _parse_number(value) -> float: - """Parse numeric value from string, handling units and whitespace.""" - if isinstance(value, (int, float)): - return float(value) - if not value or not isinstance(value, str): - return 0.0 - parts = value.strip().split() - try: - return float(parts[0]) - except (IndexError, ValueError): - return 0.0 + return parse_vodafone_number(value) @staticmethod def _parse_tg_power(value) -> float: - """Parse TG power level string like '-1.2 dBmV/1158.8 dBuV'.""" - if isinstance(value, (int, float)): - return float(value) - if not value or not isinstance(value, str): - return 0.0 - # Extract dBmV part (before the slash) - parts = value.split("/") - return VodafoneStationDriver._parse_number( - parts[0].replace("dBmV", "").replace("dBuV", "").strip() - ) + return parse_tg_power(value) @staticmethod def _parse_tg_frequency(value) -> float: - """Parse TG frequency: Hz number or 'start~end' OFDM range. - - Returns frequency in MHz. - """ - if isinstance(value, (int, float)): - freq = float(value) - return freq / 1_000_000 if freq > 1_000_000 else freq - if not value or not isinstance(value, str): - return 0.0 - # OFDM range format: "start~end" (in Hz) - if "~" in value: - parts = value.split("~") - try: - start = float(parts[0].strip()) - end = float(parts[1].strip()) - # Use center frequency, convert to MHz - freq = (start + end) / 2 - return freq / 1_000_000 if freq > 1_000_000 else freq - except (ValueError, IndexError): - return 0.0 - freq = VodafoneStationDriver._parse_number(value) - return freq / 1_000_000 if freq > 1_000_000 else freq + return parse_tg_frequency(value) @staticmethod def _normalize_modulation(modulation: str) -> str: - """Normalize modulation string to analyzer format. - - Input: "256QAM", "64QAM", "OFDM", "4096QAM", "256-qam", "qam_64" - Output: "256QAM", "64QAM", "OFDM", "4096QAM", "256QAM", "64QAM" - """ return normalize_modulation(modulation) diff --git a/scripts/driver_parser_golden.py b/scripts/driver_parser_golden.py new file mode 100644 index 00000000..a4b68a94 --- /dev/null +++ b/scripts/driver_parser_golden.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Generate a deterministic characterization matrix for modem parsers.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import sys +from typing import Any, Iterable + +if __package__ in {None, ""}: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tests.drivers.driver_format_cases import CASES, DriverFormatCase + + +def canonical_bytes(value: Any) -> bytes: + """Serialize with sorted mapping keys while preserving list semantics.""" + return json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + +def canonical_sha256(value: Any) -> str: + return hashlib.sha256(canonical_bytes(value)).hexdigest() + + +def structural_counts(value: Any) -> dict[str, int]: + counts = {"channels": 0, "dicts": 0, "lists": 0, "nulls": 0, "scalars": 0} + + def walk(node: Any) -> None: + if node is None: + counts["nulls"] += 1 + elif isinstance(node, dict): + counts["dicts"] += 1 + for child in node.values(): + walk(child) + elif isinstance(node, list): + counts["lists"] += 1 + for child in node: + walk(child) + else: + counts["scalars"] += 1 + + walk(value) + if isinstance(value, dict): + if "channelDs" in value and "channelUs" in value: + for direction in ("channelDs", "channelUs"): + lanes = value.get(direction, {}) + if isinstance(lanes, dict): + counts["channels"] += sum( + len(items) for items in lanes.values() if isinstance(items, list) + ) + else: + counts["channels"] += sum( + len(value.get(key, [])) + for key in ("downstream", "upstream") + if isinstance(value.get(key), list) + ) + return counts + + +def build_report(source_label: str, cases: Iterable[DriverFormatCase] = CASES) -> dict[str, Any]: + if not source_label: + raise ValueError("source_label must not be empty") + + rows = [] + for case in sorted(cases, key=lambda item: item.case_id): + observation = case.observe() + row: dict[str, Any] = { + "case_id": case.case_id, + "driver": case.driver, + "family": case.family, + "output_sha256": canonical_sha256(observation.output), + "structural_counts": structural_counts(observation.output), + } + if observation.diagnostics: + row["diagnostics_sha256"] = canonical_sha256(observation.diagnostics) + rows.append(row) + + return { + "format": "docsight-driver-parser-golden-v1", + "source_label": source_label, + "cases": rows, + } + + +def write_report(output: Path, source_label: str) -> None: + report = build_report(source_label) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(canonical_bytes(report) + b"\n") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-label", required=True, help="Source/base label embedded verbatim") + parser.add_argument("--output", required=True, type=Path, help="Explicit JSON output path") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + write_report(args.output, args.source_label) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/architecture/test_driver_formats_contract_red.py b/tests/architecture/test_driver_formats_contract_red.py new file mode 100644 index 00000000..53713298 --- /dev/null +++ b/tests/architecture/test_driver_formats_contract_red.py @@ -0,0 +1,134 @@ +"""Representative RED contract for the future parser extraction.""" + +from __future__ import annotations + +import importlib +import importlib.util + +from app.drivers import driver_registry + + +EXPECTED_REGISTRY_KEYS = { + "cgm4981", + "ch7465", + "ch7465_play", + "cm1000", + "cm3000", + "cm3500", + "cm8200", + "f3896lg", + "fritzbox", + "generic", + "hitron", + "hitron_coda_4680", + "sagemcom", + "sb6141", + "sb6183", + "sb6190", + "sercom_dm1000", + "surfboard", + "tc4400", + "ultrahub7", + "vodafone_station", +} + +EXPECTED_CLASS_FAMILIES = { + "app.drivers.cgm4981.CGM4981Driver": ("cgm4981_columnar_html",), + "app.drivers.ch7465.CH7465Driver": ("ch7465_xml",), + "app.drivers.cm1000.CM1000Driver": ("cm1000_html_table", "cm1000_javascript"), + "app.drivers.cm3000.CM3000Driver": ("cm3000_javascript",), + "app.drivers.cm3500.CM3500Driver": ("cm3500_html",), + "app.drivers.cm8200.CM8200Driver": ("arris_html",), + "app.drivers.f3896lg.F3896LGDriver": ("f3896lg_rest_json",), + "app.drivers.fritzbox.FritzBoxDriver": ("fritzbox_data_lua",), + "app.drivers.generic.GenericDriver": ("generic_no_docsis",), + "app.drivers.hitron.HitronDriver": ("hitron_coda56_json",), + "app.drivers.hitron_coda_4680.HitronCoda4680Driver": ("hitron_coda4680_json",), + "app.drivers.sagemcom.SagemcomDriver": ("sagemcom_xmo_json",), + "app.drivers.sb6141.SB6141Driver": ("sb6141_transposed_html",), + "app.drivers.sb6183.SB6183Driver": ("sb6183_html",), + "app.drivers.sb6190.SB6190Driver": ("sb6190_html",), + "app.drivers.sercom_dm1000.SercomDM1000Driver": ("sercom_dm1000_json",), + "app.drivers.surfboard.SurfboardDriver": ("arris_html", "surfboard_hnap"), + "app.drivers.tc4400.TC4400Driver": ("tc4400_html",), + "app.drivers.ultrahub7.UltraHub7Driver": ("ultrahub7_json",), + "app.drivers.vodafone_station.VodafoneStationDriver": ( + "vodafone_station_cga_json", + "vodafone_station_tg_embedded_json", + ), +} + +# Profiles remain explicit while cohesive modules own related grammars. This +# prevents a one-file-per-device reshuffle from passing as consolidation. +EXPECTED_PROFILE_MODULES = { + "arris_html": "app.drivers.formats.html_rows", + "cgm4981_columnar_html": "app.drivers.formats.html_columnar", + "ch7465_xml": "app.drivers.formats.xml_payloads", + "cm1000_html_table": "app.drivers.formats.html_rows", + "cm1000_javascript": "app.drivers.formats.javascript", + "cm3000_javascript": "app.drivers.formats.javascript", + "cm3500_html": "app.drivers.formats.html_rows", + "f3896lg_rest_json": "app.drivers.formats.sagemcom", + "fritzbox_data_lua": "app.drivers.formats.fritzbox", + "generic_no_docsis": "app.drivers.formats.boundaries", + "hitron_coda4680_json": "app.drivers.formats.hitron", + "hitron_coda56_json": "app.drivers.formats.hitron", + "sagemcom_xmo_json": "app.drivers.formats.sagemcom", + "sb6141_transposed_html": "app.drivers.formats.html_transposed", + "sb6183_html": "app.drivers.formats.html_rows", + "sb6190_html": "app.drivers.formats.html_rows", + "sercom_dm1000_json": "app.drivers.formats.sercom", + "surfboard_hnap": "app.drivers.formats.surfboard", + "tc4400_html": "app.drivers.formats.html_rows", + "ultrahub7_json": "app.drivers.formats.vodafone", + "vodafone_station_cga_json": "app.drivers.formats.vodafone", + "vodafone_station_tg_embedded_json": "app.drivers.formats.vodafone", +} + + +def test_formats_package_and_immutable_per_driver_family_metadata_contract_red(): + """Intentionally RED until the production extraction creates this contract.""" + issues = [] + actual_keys = driver_registry.get_all_type_keys() + if actual_keys != EXPECTED_REGISTRY_KEYS: + issues.append( + f"registry keys differ: missing={sorted(EXPECTED_REGISTRY_KEYS - actual_keys)!r} " + f"extra={sorted(actual_keys - EXPECTED_REGISTRY_KEYS)!r}" + ) + + package_exists = importlib.util.find_spec("app.drivers.formats") is not None + if not package_exists: + issues.append("missing package app.drivers.formats") + + paths_by_key = driver_registry._builtin + concrete_paths = set(paths_by_key.values()) + if concrete_paths != set(EXPECTED_CLASS_FAMILIES): + issues.append("concrete registry classes differ from the finite contract") + + # CH7465 and CH7465 Play are registry aliases for one concrete behavior. + if paths_by_key["ch7465"] != paths_by_key["ch7465_play"]: + issues.append("CH7465 alias keys no longer resolve to one concrete class") + + for class_path, expected_families in sorted(EXPECTED_CLASS_FAMILIES.items()): + module_name, class_name = class_path.rsplit(".", 1) + cls = getattr(importlib.import_module(module_name), class_name) + actual_families = getattr(cls, "FORMAT_FAMILIES", None) + if actual_families != expected_families or not isinstance(actual_families, tuple): + issues.append( + f"{class_path}.FORMAT_FAMILIES must be immutable tuple {expected_families!r}; " + f"got {actual_families!r}" + ) + + if package_exists: + formats_package = importlib.import_module("app.drivers.formats") + actual_profile_modules = getattr(formats_package, "FORMAT_PROFILE_MODULES", None) + if actual_profile_modules != EXPECTED_PROFILE_MODULES: + issues.append( + "app.drivers.formats.FORMAT_PROFILE_MODULES must match the finite " + "profile-to-cohesive-module contract" + ) + for module_name in sorted(set(EXPECTED_PROFILE_MODULES.values())): + if importlib.util.find_spec(module_name) is None: + issues.append(f"missing cohesive parser module {module_name}") + + assert issues == [], "future driver formats contract is unmet:\n- " + "\n- ".join(issues) diff --git a/tests/architecture/test_driver_formats_static.py b/tests/architecture/test_driver_formats_static.py new file mode 100644 index 00000000..302df486 --- /dev/null +++ b/tests/architecture/test_driver_formats_static.py @@ -0,0 +1,190 @@ +"""Static dependency, entrypoint, and thin-adapter guards for driver formats.""" + +from __future__ import annotations + +import ast +import importlib +from pathlib import Path + +from app.drivers.formats import FORMAT_PROFILE_MODULES +from tests.architecture.test_driver_formats_contract_red import EXPECTED_CLASS_FAMILIES + + +ROOT = Path(__file__).resolve().parents[2] +FORMATS = ROOT / "app" / "drivers" / "formats" + +FORBIDDEN_IMPORTS = { + "cryptography", "flask", "random", "requests", "secrets", "socket", "ssl", "time", +} + +PROFILE_ENTRYPOINTS = { + "arris_html": "parse_arris_html", + "cgm4981_columnar_html": "parse_cgm4981_columnar_html", + "ch7465_xml": "parse_ch7465_xml", + "cm1000_html_table": "parse_cm1000_html_table", + "cm1000_javascript": "parse_cm1000_javascript", + "cm3000_javascript": "parse_cm3000_javascript", + "cm3500_html": "parse_cm3500_html", + "f3896lg_rest_json": "parse_f3896lg_rest_json", + "fritzbox_data_lua": "parse_fritzbox_data_lua", + "generic_no_docsis": "parse_generic_no_docsis", + "hitron_coda4680_json": "parse_hitron_coda4680_json", + "hitron_coda56_json": "parse_hitron_coda56_json", + "sagemcom_xmo_json": "parse_sagemcom_xmo_json", + "sb6141_transposed_html": "parse_sb6141_transposed_html", + "sb6183_html": "parse_sb6183_html", + "sb6190_html": "parse_sb6190_html", + "sercom_dm1000_json": "parse_sercom_dm1000_json", + "surfboard_hnap": "parse_surfboard_hnap", + "tc4400_html": "parse_tc4400_html", + "ultrahub7_json": "parse_ultrahub7_json", + "vodafone_station_cga_json": "parse_vodafone_station_cga_json", + "vodafone_station_tg_embedded_json": "parse_vodafone_station_tg_embedded_json", +} + +# Only these private methods remain as compatibility seams. Device-info, +# connection-info, auth, and transport parsers are outside the DOCSIS grammar. +COMPATIBILITY_METHODS = { + "CGM4981Driver": {"_build_ds_channels", "_build_us_channels"}, + "CH7465Driver": {"_normalize_modulation"}, + "CM3000Driver": { + "_parse_ds_qam", "_parse_us_atdma", "_parse_ds_ofdm", "_parse_us_ofdma", + "_extract_tag_value_list", "_split_channels", "_hz_to_mhz", "_parse_number", + "_normalize_modulation", + }, + "CM3500Driver": { + "_find_table_sections", "_parse_ds_qam", "_parse_ds_ofdm", "_parse_us_qam", + "_parse_us_ofdm", "_parse_number", "_format_freq", + }, + "F3896LGDriver": {"_parse_downstream", "_parse_upstream"}, + "FritzBoxDriver": {"_compensate_us31_power"}, + "HitronDriver": { + "_fetch_ds_scqam", "_fetch_us_scqam", "_fetch_ds_ofdm", "_fetch_us_ofdma", + "_ofdma_power_1_6", "_hz_to_mhz", + }, + "HitronCoda4680Driver": { + "_parse_ds_scqam", "_parse_us_scqam", "_parse_ds_ofdm", "_parse_us_ofdma", + "_ofdma_power_1_6", + }, + "SagemcomDriver": { + "_parse_downstream", "_parse_upstream", "_hz_to_mhz", "_is_ofdm_downstream", + "_normalize_modulation", "_normalize_us_modulation", + }, + "SB6141Driver": { + "_parse_downstream", "_parse_upstream", "_extract_transposed_rows", + "_get_row_values", "_extract_upstream_modulation", "_parse_freq_hz", "_parse_number", + }, + "SB6183Driver": {"_parse_downstream", "_parse_upstream"}, + "SB6190Driver": {"_parse_downstream", "_parse_upstream", "_normalize_mhz", "_parse_number"}, + "SercomDM1000Driver": { + "_parse_ds_scqam", "_parse_ds_ofdm", "_parse_us_scqam", "_parse_us_ofdma", + "_profile_modulation_from_bits", + }, + "SurfboardDriver": {"_parse_downstream", "_parse_upstream", "_hz_to_mhz", "_normalize_modulation"}, + "TC4400Driver": { + "_parse_downstream", "_parse_upstream", "_find_header_row", "_map_columns", "_cell", + "_parse_frequency", "_parse_number", "_normalize_modulation", + }, + "UltraHub7Driver": {"_parse_downstream_channels", "_parse_upstream_channels"}, + "VodafoneStationDriver": { + "_parse_number", "_parse_tg_power", "_parse_tg_frequency", "_normalize_modulation", + }, +} + +# These parse device metadata rather than normalized DOCSIS channel payloads. +NON_DOCSIS_PARSERS = { + "CM3000Driver": {"_parse_uptime"}, + "CM3500Driver": {"_parse_service_flows"}, + "HitronCoda4680Driver": {"_parse_device_info", "_parse_rate_kbps"}, + "TC4400Driver": {"_parse_info_table"}, +} + + +def test_formats_modules_have_no_transport_auth_or_runtime_dependencies(): + issues = [] + for path in sorted(FORMATS.glob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + imported = [] + if isinstance(node, ast.Import): + imported = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.module: + imported = [node.module] + for name in imported: + if name.split(".", 1)[0] in FORBIDDEN_IMPORTS: + issues.append(f"{path.name}:{node.lineno} imports {name}") + assert issues == [] + + +def test_every_profile_has_one_named_entrypoint_in_its_cohesive_module(): + assert set(PROFILE_ENTRYPOINTS) == set(FORMAT_PROFILE_MODULES) + for profile, function_name in PROFILE_ENTRYPOINTS.items(): + module = importlib.import_module(FORMAT_PROFILE_MODULES[profile]) + assert callable(getattr(module, function_name, None)), (profile, function_name) + + +def test_registry_matrix_is_complete_and_alias_safe_in_both_directions(): + matrix_profiles = { + profile + for profiles in EXPECTED_CLASS_FAMILIES.values() + for profile in profiles + } + assert matrix_profiles == set(FORMAT_PROFILE_MODULES) + assert len(EXPECTED_CLASS_FAMILIES) == 20 + assert len(PROFILE_ENTRYPOINTS) == 22 + + +def test_migrated_private_methods_are_finite_one_statement_delegations(): + discovered: dict[str, set[str]] = {} + issues = [] + for class_path in EXPECTED_CLASS_FAMILIES: + module_name, class_name = class_path.rsplit(".", 1) + path = ROOT / (module_name.replace(".", "/") + ".py") + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + class_node = next( + node for node in tree.body + if isinstance(node, ast.ClassDef) and node.name == class_name + ) + expected = COMPATIBILITY_METHODS.get(class_name, set()) + actual = { + node.name for node in class_node.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name in expected + } + if actual: + discovered[class_name] = actual + for node in class_node.body: + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) or node.name not in expected: + continue + statements = node.body[1:] if ( + node.body and isinstance(node.body[0], ast.Expr) + and isinstance(node.body[0].value, ast.Constant) + and isinstance(node.body[0].value.value, str) + ) else node.body + if len(statements) != 1 or isinstance(statements[0], (ast.For, ast.While, ast.If, ast.Try)): + issues.append(f"{class_name}.{node.name} is not a one-statement delegation") + assert discovered == {name: methods for name, methods in COMPATIBILITY_METHODS.items() if methods} + assert issues == [] + + +def test_concrete_drivers_cannot_add_local_parser_implementations(): + unexpected = [] + for class_path in EXPECTED_CLASS_FAMILIES: + module_name, class_name = class_path.rsplit(".", 1) + path = ROOT / (module_name.replace(".", "/") + ".py") + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + class_node = next( + node for node in tree.body + if isinstance(node, ast.ClassDef) and node.name == class_name + ) + allowed = COMPATIBILITY_METHODS.get(class_name, set()) | NON_DOCSIS_PARSERS.get( + class_name, set() + ) + parser_names = { + node.name for node in class_node.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name.startswith("_parse") + } + unexpected.extend( + f"{class_name}.{name}" for name in sorted(parser_names - allowed) + ) + assert unexpected == [] diff --git a/tests/drivers/driver_format_cases.py b/tests/drivers/driver_format_cases.py new file mode 100644 index 00000000..fd4c689e --- /dev/null +++ b/tests/drivers/driver_format_cases.py @@ -0,0 +1,606 @@ +"""Network-free characterization cases for built-in modem parser formats. + +This module deliberately lives under ``tests/``: it invokes the parser seams +that exist on the source baseline and is also imported by the maintainer golden +matrix script. Raw samples are either captured fixtures already used by the +suite or minimal malformed/empty variants around fields in those fixtures. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +import json +import logging +from pathlib import Path +from typing import Any, Callable, Iterator +from unittest.mock import patch + +from bs4 import BeautifulSoup + +from app.drivers.arris_html import parse_arris_channel_tables +from app.drivers.cgm4981 import CGM4981Driver +from app.drivers.ch7465 import CH7465Driver, Query +from app.drivers.cm1000 import CM1000Driver +from app.drivers.cm3000 import CM3000Driver +from app.drivers.cm3500 import CM3500Driver +from app.drivers.f3896lg import F3896LGDriver +from app.drivers.fritzbox import FritzBoxDriver +from app.drivers.generic import GenericDriver +from app.drivers.hitron import HitronDriver +from app.drivers.hitron_coda_4680 import HitronCoda4680Driver +from app.drivers.sagemcom import SagemcomDriver +from app.drivers.sb6141 import SB6141Driver +from app.drivers.sb6183 import SB6183Driver +from app.drivers.sb6190 import SB6190Driver +from app.drivers.sercom_dm1000 import SercomDM1000Driver +from app.drivers.surfboard import SurfboardDriver +from app.drivers.tc4400 import TC4400Driver +from app.drivers.ultrahub7 import UltraHub7Driver +from app.drivers.vodafone_station import VodafoneStationDriver + + +ROOT = Path(__file__).resolve().parents[2] + +EMPTY_SPLIT = { + "channelDs": {"docsis30": [], "docsis31": []}, + "channelUs": {"docsis30": [], "docsis31": []}, +} +EMPTY_FLAT_31 = {"docsis": "3.1", "downstream": [], "upstream": []} + + +@dataclass(frozen=True) +class CaseObservation: + """One parser observation, keeping diagnostics out of normalized data.""" + + output: Any + diagnostics: tuple[dict[str, str], ...] = () + + +@dataclass(frozen=True) +class DriverFormatCase: + """A stable case-registry entry.""" + + case_id: str + driver: str + family: str + evidence: str + invoke: Callable[[], Any] + expected: Any + + def observe(self) -> CaseObservation: + with _capture_parser_warnings() as warnings: + try: + output = self.invoke() + except Exception as exc: # Characterize an existing malformed seam. + return CaseObservation( + output=None, + diagnostics=tuple(warnings) + + ({"kind": "exception", "type": type(exc).__name__, "message": str(exc)},), + ) + return CaseObservation(output=output, diagnostics=tuple(warnings)) + + +class _WarningCollector(logging.Handler): + def __init__(self) -> None: + super().__init__(level=logging.WARNING) + self.items: list[dict[str, str]] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.items.append( + {"kind": "log", "level": record.levelname, "message": record.getMessage()} + ) + + +@contextmanager +def _capture_parser_warnings() -> Iterator[list[dict[str, str]]]: + handler = _WarningCollector() + root = logging.getLogger() + old_level = root.level + root.addHandler(handler) + if old_level > logging.WARNING: + root.setLevel(logging.WARNING) + try: + yield handler.items + finally: + root.removeHandler(handler) + root.setLevel(old_level) + + +class _Response: + def __init__(self, *, payload: Any = None, text: str = "") -> None: + self._payload = payload + self.text = text + self.status_code = 200 + + def json(self) -> Any: + return self._payload + + def raise_for_status(self) -> None: + return None + + +def _split(ds30=None, ds31=None, us30=None, us31=None) -> dict[str, Any]: + return { + "channelDs": {"docsis30": ds30 or [], "docsis31": ds31 or []}, + "channelUs": {"docsis30": us30 or [], "docsis31": us31 or []}, + } + + +def _flat(downstream=None, upstream=None) -> dict[str, Any]: + return {"docsis": "3.1", "downstream": downstream or [], "upstream": upstream or []} + + +def _fritz(raw: dict[str, Any]) -> Any: + driver = FritzBoxDriver("http://modem.invalid", "user", "password") + response = _Response(payload={"data": raw}) + with patch("app.fritzbox.requests.post", return_value=response): + return driver.get_docsis_data() + + +def _tc(ds_html: str, us_html: str) -> Any: + driver = TC4400Driver("http://modem.invalid", "user", "password") + ds = BeautifulSoup(ds_html, "html.parser").find("table") + us = BeautifulSoup(us_html, "html.parser").find("table") + return _flat(driver._parse_downstream(ds), driver._parse_upstream(us)) + + +def _ultrahub(ds: list[dict[str, Any]], us: list[dict[str, Any]]) -> Any: + driver = UltraHub7Driver("http://modem.invalid", "", "password") + return _flat(driver._parse_downstream_channels(ds), driver._parse_upstream_channels(us)) + + +def _vodafone_cga(payload: Any) -> Any: + driver = VodafoneStationDriver("http://modem.invalid", "admin", "password") + with patch.object(driver, "_cga_request", return_value=_Response(payload={"data": payload})): + return driver._get_docsis_cga() + + +def _vodafone_tg(html: str) -> Any: + driver = VodafoneStationDriver("http://modem.invalid", "admin", "password") + driver._tg_nonce = "captured-boundary" + with patch.object(driver, "_tg_docsis_request", return_value=_Response(text=html)): + return driver._get_docsis_tg() + + +def _ch7465(ds_xml: str, us_xml: str) -> Any: + driver = CH7465Driver("http://modem.invalid", "admin", "password") + + def get_data(query: Query) -> str: + return ds_xml if query is Query.DOWNSTREAM_TABLE else us_xml + + with patch.object(driver, "_get_data", side_effect=get_data): + return driver.get_docsis_data() + + +def _cm3000_html(ds: str, us: str, ds_ofdm: str, us_ofdma: str) -> str: + return f""" + + """ + + +def _cm3000(html: str) -> Any: + driver = CM3000Driver("http://modem.invalid", "admin", "password") + with patch.object(driver, "_fetch_status_page", return_value=html): + return driver.get_docsis_data() + + +def _cm1000(html: str) -> Any: + driver = CM1000Driver("http://modem.invalid", "admin", "password") + with patch.object(driver, "_fetch_status_page", return_value=html): + return driver.get_docsis_data() + + +def _cm3500(html: str) -> Any: + driver = CM3500Driver("http://modem.invalid", "admin", "password") + soup = BeautifulSoup(html, "html.parser") + with patch.object(driver, "_fetch_status_page", return_value=soup): + return driver.get_docsis_data() + + +def _surfboard(ds: str, us: str) -> Any: + driver = SurfboardDriver("https://modem.invalid", "admin", "password") + ds30, ds31 = driver._parse_downstream(ds) + us30, us31 = driver._parse_upstream(us) + return _split(ds30, ds31, us30, us31) + + +def _first_data_table(html: str, marker: str) -> Any: + soup = BeautifulSoup(html, "html.parser") + for table in soup.find_all("table"): + if marker in table.get_text(" ", strip=True).lower(): + data_rows = [ + row for row in table.find_all("tr") + if len(row.find_all("td")) >= 7 and not row.find("strong") + ] + if len(data_rows) > 1: + for row in data_rows[1:]: + row.extract() + return table + return None + + +def _sb6141(signal_html: str) -> Any: + driver = SB6141Driver("http://modem.invalid", "", "") + soup = BeautifulSoup(signal_html, "html.parser") + tables = soup.find_all("table") + ds = us = cw = None + for table in tables: + th = table.find("th") + heading = th.get_text(" ", strip=True).lower() if th else "" + if "downstream" in heading: + ds = table + elif "upstream" in heading: + us = table + elif "signal status" in heading or "codeword" in heading: + cw = table + return _split(driver._parse_downstream(ds, cw), [], driver._parse_upstream(us), []) + + +def _row_html_driver(driver_cls: type, html: str) -> Any: + driver = driver_cls("http://modem.invalid", "admin", "password") + soup = BeautifulSoup(html, "html.parser") + ds = _first_data_table(str(soup), "downstream bonded") + us = _first_data_table(str(soup), "upstream bonded") + return _split(driver._parse_downstream(ds), [], driver._parse_upstream(us), []) + + +def _hitron(payloads: dict[str, list[dict[str, Any]]]) -> Any: + driver = HitronDriver("http://modem.invalid", "", "") + with patch.object(driver, "_fetch_json", side_effect=lambda path: payloads[path]): + return driver.get_docsis_data() + + +def _hitron_4680(payloads: dict[str, dict[str, Any]]) -> Any: + driver = HitronCoda4680Driver("http://modem.invalid", "admin", "password") + with patch.object(driver, "_fetch_payload", side_effect=lambda path: payloads[path]): + return driver.get_docsis_data() + + +def _sagemcom(ds: list[dict[str, Any]], us: list[dict[str, Any]]) -> Any: + driver = SagemcomDriver("http://modem.invalid", "admin", "password") + ds30, ds31 = driver._parse_downstream(ds) + us30, us31 = driver._parse_upstream(us) + return _split(ds30, ds31, us30, us31) + + +def _f3896(ds: list[dict[str, Any]], us: list[dict[str, Any]]) -> Any: + driver = F3896LGDriver("http://modem.invalid", "", "") + ds30, ds31 = driver._parse_downstream(ds) + us30, us31 = driver._parse_upstream(us) + return _split(ds30, ds31, us30, us31) + + +def _sercom( + ds: list[dict[str, Any]], + ds_ofdm: list[dict[str, Any]], + us: list[dict[str, Any]], + us_ofdma: list[dict[str, Any]], +) -> Any: + driver = SercomDM1000Driver("http://modem.invalid", "technician", "password") + return _split( + driver._parse_ds_scqam(ds), + driver._parse_ds_ofdm(ds_ofdm), + driver._parse_us_scqam(us), + driver._parse_us_ofdma(us_ofdma), + ) + + +def _cgm(ds: dict[str, list[str]], us: dict[str, list[str]], errors: dict[str, list[str]]) -> Any: + driver = CGM4981Driver("http://modem.invalid", "admin", "password") + downstream = driver._build_ds_channels(ds, errors) + upstream = driver._build_us_channels(us) + return _split( + [channel for channel in downstream if channel.get("modulation") != "OFDM"], + [channel for channel in downstream if channel.get("modulation") == "OFDM"], + [channel for channel in upstream if channel.get("modulation") != "OFDMA"], + [channel for channel in upstream if channel.get("modulation") == "OFDMA"], + ) + + +# Captured values reused from the existing focused tests, reduced to one row per +# supported lane so the expected structures stay readable. +TC_DS_SUCCESS = """
Channel IDLock StatusChannel TypeModulationFrequencyPowerSNRCorrectedUncorrected
7LockedSC-QAM256QAM591000000 Hz4.6 dBmV36.4 dB330
193LockedOFDMOFDM275600000 Hz3.2 dBmV38 dB91
""" +TC_US_SUCCESS = """
Channel IDLock StatusModulationFrequencyPower
6LockedATDMA25900000 Hz35 dBmV
41LockedOFDMA42000000 Hz37.75 dBmV
""" + +TG_SUCCESS = """""" + +CM3000_SUCCESS = _cm3000_html( + "1|1|Locked|QAM256|7|591000000 Hz|-2.5|40.0|1234|5|", + "1|1|Locked|ATDMA|3|5120 Ksym/sec|29200000 Hz|43.5 dBmV|", + "1|1|Locked|0 ,1 ,2 ,3|193|690000000 Hz|-0.32 dBmV|41.8 dB|388 ~ 3707|9|1|0|", + "1|1|Locked|12 ,13|41|36200000 Hz|36.5 dBmV|", +) + +CM1000_TABLE_SUCCESS = """ +
ChannelLock StatusModulationChannel IDFrequencyPowerSNRCorrectablesUnCorrectables
1Locked256QAM7591000000 Hz0.0 dBmV40 dB05
+
ChannelLock StatusModulationChannel IDFrequencyPower
1LockedATDMA329200000 Hz43.5 dBmV
+
ChannelLock StatusModulationChannel IDFrequencyPowerSNRCorrectablesUnCorrectables
1LockedOFDM193690000000 Hz-0.32 dBmV41.8 dB91
+
ChannelLock StatusModulationChannel IDFrequencyPower
1LockedOFDMA4136200000 Hz36.5 dBmV
+""" + +CM3500_SUCCESS = """ +

Downstream QAM

DCIDFreqPowerSNRModulationOctetsCorrectedsUncorrectables
Downstream 13570 MHz4.7 dBmV38.98 dB256QAM100920
+

Downstream OFDM

Downstream 14K1903800135324474041
+

Upstream QAM

UCIDFreqPowerChannel TypeSymbol RateModulation
Upstream 1930.8 MHz39.5 dBmVDOCSIS2.0 (ATDMA)512064QAM
+

Upstream OFDM

Upstream 02K326407477329.864.842.25
+""" + +ARRIS_SUCCESS = """ +
Downstream Bonded Channels
IDLockModulationFrequencyPowerSNRCorrectedUncorrected
7Locked256QAM591000000 Hz0.0 dBmV40 dB30
193LockedOther690000000 Hz41.8 dB91
+
Upstream Bonded Channels
ChannelIDLockTypeFrequencyWidthPower
13LockedSC-QAM Upstream29200000 Hz640000043.5 dBmV
241LockedOFDM Upstream36200000 Hz4440000036.5 dBmV
+""" + +SB6141_SUCCESS = """ +
Downstream
Channel ID7
Frequency591000000 Hz
Signal to Noise Ratio40.0 dB
Modulation256QAM
Power Level0.0 dBmV
+
Upstream
Channel ID3
Frequency29200000 Hz
Power Level43.5 dBmV
Modulation[3] QPSK +[3] 64QAM
+
Signal Status / Codewords
Correctable3
Uncorrectable1
+""" + + +FRITZ_SUCCESS = { + "channelDs": { + "docsis30": [ + {"channelID": 1, "frequency": "591 MHz", "powerLevel": 0, "mse": None}, + {"channelID": 1, "frequency": "597 MHz", "powerLevel": None}, + {"frequency": "603 MHz", "powerLevel": "1.0"}, + ], + "docsis31": [], + }, + "channelUs": { + "docsis30": [], + "docsis31": [ + {"channelID": 5, "type": "OFDMA", "powerLevel": "37.0"}, + {"channelID": 6, "powerLevel": None}, + ], + }, +} + + +def _cases() -> list[DriverFormatCase]: + cases: list[DriverFormatCase] = [] + + def add(case_id: str, driver: str, family: str, evidence: str, invoke, expected) -> None: + cases.append(DriverFormatCase(case_id, driver, family, evidence, invoke, expected)) + + # Fritz!Box data.lua boundary: input is already normalized by app.fritzbox. + fritz_expected = json.loads(json.dumps(FRITZ_SUCCESS)) + fritz_expected["channelUs"]["docsis31"][0]["powerLevel"] = "43.0" + add("fritzbox_data_lua.success_duplicates_missing_id", "fritzbox", "fritzbox_data_lua", "captured-shape", lambda: _fritz(json.loads(json.dumps(FRITZ_SUCCESS))), fritz_expected) + add("fritzbox_data_lua.empty", "fritzbox", "fritzbox_data_lua", "minimal-empty", lambda: _fritz(json.loads(json.dumps(EMPTY_SPLIT))), EMPTY_SPLIT) + fritz_bad = _split(us31=[{"channelID": 5, "powerLevel": "not-a-number"}]) + add("fritzbox_data_lua.malformed_power", "fritzbox", "fritzbox_data_lua", "minimal-synthetic-malformed", lambda: _fritz(json.loads(json.dumps(fritz_bad))), fritz_bad) + + tc_expected = _flat( + [ + {"channelID": "7", "type": "256QAM", "frequency": "591 MHz", "powerLevel": 4.6, "mse": -36.4, "mer": 36.4, "latency": 0, "corrError": 3, "nonCorrError": 30}, + {"channelID": "193", "type": "OFDM", "frequency": "275 MHz", "powerLevel": 3.2, "mse": None, "mer": 38.0, "latency": 0, "corrError": 9, "nonCorrError": 1}, + ], + [ + {"channelID": "6", "type": "ATDMA", "frequency": "25 MHz", "powerLevel": 35.0, "multiplex": ""}, + {"channelID": "41", "type": "OFDMA", "frequency": "42 MHz", "powerLevel": 37.75, "multiplex": ""}, + ], + ) + add("tc4400_html.success_scqam_ofdm_ofdma", "tc4400", "tc4400_html", "minimal-synthetic-existing-fields", lambda: _tc(TC_DS_SUCCESS, TC_US_SUCCESS), tc_expected) + add("tc4400_html.empty_tables", "tc4400", "tc4400_html", "minimal-empty", lambda: _tc("
", "
"), EMPTY_FLAT_31) + add("tc4400_html.malformed_short_rows", "tc4400", "tc4400_html", "minimal-synthetic-malformed", lambda: _tc("
ABCD
bad
", "
ABCD
bad
"), EMPTY_FLAT_31) + + ultra_ds = [{"ChannelID": "7", "Frequency": "591 MHz", "Modulation": "256QAM", "PowerLevel": "0 dBmV", "SNRLevel": "40 dB"}, {"ChannelID": "193", "Frequency": "690 MHz", "Modulation": "OFDM", "PowerLevel": "-0.32 dBmV", "SNRLevel": "41.8 dB"}] + ultra_us = [{"ChannelID": "3", "Frequency": "29.2 MHz", "Modulation": "ATDMA", "PowerLevel": "43.5 dBmV"}, {"ChannelID": "41", "Frequency": "36.2 MHz", "Modulation": "OFDMA", "PowerLevel": "36.5 dBmV"}] + ultra_expected = _flat( + [{"channelID": "7", "type": "256QAM", "frequency": "591 MHz", "powerLevel": 0.0, "mer": 40.0, "mse": None, "latency": 0, "corrErrors": None, "nonCorrErrors": None}, {"channelID": "193", "type": "OFDM", "frequency": "690 MHz", "powerLevel": -0.32, "mer": 41.8, "mse": None, "latency": 0, "corrErrors": None, "nonCorrErrors": None}], + [{"channelID": "3", "type": "ATDMA", "frequency": "29 MHz", "powerLevel": 43.5, "multiplex": ""}, {"channelID": "41", "type": "OFDMA", "frequency": "36 MHz", "powerLevel": 36.5, "multiplex": ""}], + ) + add("ultrahub7_json.success_scqam_ofdm_ofdma", "ultrahub7", "ultrahub7_json", "minimal-synthetic-existing-fields", lambda: _ultrahub(ultra_ds, ultra_us), ultra_expected) + add("ultrahub7_json.missing_fields_become_zero", "ultrahub7", "ultrahub7_json", "minimal-missing", lambda: _ultrahub([{}], [{}]), _flat([{"channelID": "0", "type": "", "frequency": "0 MHz", "powerLevel": 0.0, "mer": None, "mse": None, "latency": 0, "corrErrors": None, "nonCorrErrors": None}], [{"channelID": "0", "type": "", "frequency": "0 MHz", "powerLevel": 0.0, "multiplex": ""}])) + add("ultrahub7_json.malformed_channel_id", "ultrahub7", "ultrahub7_json", "minimal-synthetic-malformed", lambda: _ultrahub([{"ChannelID": "bad"}], [{"ChannelID": "bad"}]), EMPTY_FLAT_31) + + cga_success = {"downstream": [{"channelid": "7", "CentralFrequency": "591000000", "power": "0", "SNR": "40", "FFT": "256QAM"}], "ofdm_downstream": [{"channelid_ofdm": "193", "CentralFrequency_ofdm": "690000000", "power_ofdm": "-0.32", "SNR_ofdm": "41.8"}], "upstream": [{"channelidup": "3", "CentralFrequency": "29200000", "power": "43.5", "FFT": "64QAM"}], "ofdma_upstream": [{"channelidup": "41", "CentralFrequency": "36200000", "power": "36.5", "FFT": "64-qam", "ChannelType": "OFDMA"}]} + cga_expected = _split( + [{"channelID": 7, "type": "256QAM", "frequency": "591 MHz", "powerLevel": 0.0, "mse": -40.0, "mer": 40.0, "latency": 0, "corrError": 0, "nonCorrError": 0}], + [{"channelID": 193, "type": "OFDM", "frequency": "690 MHz", "powerLevel": -0.32, "mse": -41.8, "mer": 41.8, "latency": 0, "corrError": 0, "nonCorrError": 0}], + [{"channelID": 3, "type": "64QAM", "frequency": "29 MHz", "powerLevel": 43.5, "multiplex": ""}], + [{"channelID": 41, "type": "OFDMA", "frequency": "36 MHz", "powerLevel": 36.5, "modulation": "64QAM", "multiplex": ""}], + ) + add("vodafone_station_cga_json.success_all_lanes", "vodafone_station", "vodafone_station_cga_json", "captured-shape", lambda: _vodafone_cga(cga_success), cga_expected) + add("vodafone_station_cga_json.empty_object", "vodafone_station", "vodafone_station_cga_json", "minimal-empty", lambda: _vodafone_cga({}), EMPTY_SPLIT) + add("vodafone_station_cga_json.malformed_non_object_channel", "vodafone_station", "vodafone_station_cga_json", "minimal-synthetic-malformed", lambda: _vodafone_cga({"downstream": [None]}), None) + + tg_expected = _split( + [{"channelID": 7, "type": "256QAM", "frequency": "591.000 MHz", "powerLevel": -1.2, "mse": -40.0, "mer": 40.0, "latency": 0, "corrError": 0, "nonCorrError": 0}], + [{"channelID": 193, "type": "OFDM", "frequency": "374.275 MHz", "powerLevel": 3.2, "mse": -38.0, "mer": 38.0, "latency": 0, "corrError": 0, "nonCorrError": 0}], + [{"channelID": 6, "type": "64QAM", "frequency": "25.900 MHz", "powerLevel": 35.0, "multiplex": ""}], + [{"channelID": 41, "type": "OFDMA", "frequency": "42.000 MHz", "powerLevel": 37.75, "multiplex": ""}], + ) + add("vodafone_station_tg_embedded_json.success_all_lanes", "vodafone_station", "vodafone_station_tg_embedded_json", "minimal-synthetic-existing-fields", lambda: _vodafone_tg(TG_SUCCESS), tg_expected) + add("vodafone_station_tg_embedded_json.empty_arrays", "vodafone_station", "vodafone_station_tg_embedded_json", "minimal-empty", lambda: _vodafone_tg(""), None) + add("vodafone_station_tg_embedded_json.malformed_json", "vodafone_station", "vodafone_station_tg_embedded_json", "minimal-synthetic-malformed", lambda: _vodafone_tg(""), None) + + ch_ds = "7591 MHz040256qam317597 MHz1" + ch_us = "329 MHz43.564qam35" + ch_expected = {"docsis": "3.0", "downstream": [{"channelID": 7, "frequency": "591 MHz", "powerLevel": 0.0, "mer": 40.0, "mse": -40.0, "modulation": "256QAM", "corrErrors": 3, "nonCorrErrors": 1}, {"channelID": 7, "frequency": "597 MHz", "powerLevel": 1.0}], "upstream": [{"channelID": 3, "frequency": "29 MHz", "powerLevel": 43.5, "modulation": "64QAM", "multiplex": "atdma"}]} + add("ch7465_xml.success_duplicate_ids", "ch7465,ch7465_play", "ch7465_xml", "minimal-synthetic-existing-fields", lambda: _ch7465(ch_ds, ch_us), ch_expected) + add("ch7465_xml.empty_roots", "ch7465,ch7465_play", "ch7465_xml", "minimal-empty", lambda: _ch7465("", ""), {"docsis": "3.0", "downstream": [], "upstream": []}) + add("ch7465_xml.malformed_xml", "ch7465,ch7465_play", "ch7465_xml", "minimal-synthetic-malformed", lambda: _ch7465("", ""), None) + + cm3000_expected = _split( + [{"channelID": 7, "frequency": "591 MHz", "powerLevel": -2.5, "mer": 40.0, "mse": -40.0, "modulation": "QAM256", "corrErrors": 1234, "nonCorrErrors": 5}], + [{"channelID": 193, "type": "OFDM", "frequency": "690 MHz", "powerLevel": -0.32, "mer": 41.8, "mse": None, "corrErrors": 9, "nonCorrErrors": 1}], + [{"channelID": 3, "frequency": "29.2 MHz", "powerLevel": 43.5, "modulation": "ATDMA", "multiplex": "ATDMA"}], + [{"channelID": 41, "type": "OFDMA", "frequency": "36.2 MHz", "powerLevel": 36.5, "modulation": "OFDMA", "multiplex": ""}], + ) + add("cm3000_javascript.success_all_lanes", "cm3000", "cm3000_javascript", "captured-subset", lambda: _cm3000(CM3000_SUCCESS), cm3000_expected) + add("cm3000_javascript.empty_tag_values", "cm3000", "cm3000_javascript", "minimal-empty", lambda: _cm3000(_cm3000_html("", "", "", "")), EMPTY_SPLIT) + add("cm3000_javascript.malformed_numeric", "cm3000", "cm3000_javascript", "minimal-synthetic-malformed", lambda: _cm3000(_cm3000_html("1|1|Locked|QAM256|bad|591000000 Hz|x|y|z|q|", "", "", "")), EMPTY_SPLIT) + + cm1000_js = (ROOT / "tests/fixtures/cm1000/DocsisStatus.asp.html").read_text(encoding="utf-8") + cm1000_js_expected = _split( + [{"channelID": 143, "frequency": "453 MHz", "powerLevel": -2.5, "mer": 48.5, "mse": -48.5, "modulation": "64QAM", "corrErrors": None, "nonCorrErrors": None, "symbolRate": 5057}], + [], + [{"channelID": 1, "frequency": "33 MHz", "powerLevel": 34.8, "modulation": "TDMA", "multiplex": "TDMA", "symbolRate": 2560}], + [], + ) + add("cm1000_javascript.success_captured_fixture", "cm1000", "cm1000_javascript", "captured-disk", lambda: _cm1000(cm1000_js), cm1000_js_expected) + add("cm1000_javascript.empty_tag_values", "cm1000", "cm1000_javascript", "minimal-empty", lambda: _cm1000(""), EMPTY_SPLIT) + add("cm1000_javascript.malformed_row_width", "cm1000", "cm1000_javascript", "minimal-synthetic-malformed", lambda: _cm1000(""), EMPTY_SPLIT) + + cm1000_table_expected = _split( + [{"channelID": 7, "frequency": "591 MHz", "powerLevel": 0.0, "mer": 40.0, "mse": -40.0, "modulation": "256QAM", "corrErrors": 0, "nonCorrErrors": 5, "symbolRate": 5361}], + [{"channelID": 193, "type": "OFDM", "frequency": "690 MHz", "powerLevel": -0.32, "mer": 41.8, "mse": None, "modulation": "OFDM", "corrErrors": 9, "nonCorrErrors": 1}], + [{"channelID": 3, "frequency": "29.2 MHz", "powerLevel": 43.5, "modulation": "ATDMA", "multiplex": "ATDMA"}], + [{"channelID": 41, "type": "OFDMA", "frequency": "36.2 MHz", "powerLevel": 36.5, "modulation": "OFDMA", "multiplex": ""}], + ) + add("cm1000_html_table.success_all_lanes", "cm1000", "cm1000_html_table", "captured-subset", lambda: _cm1000(CM1000_TABLE_SUCCESS), cm1000_table_expected) + add("cm1000_html_table.empty_document", "cm1000", "cm1000_html_table", "minimal-empty", lambda: _cm1000(""), EMPTY_SPLIT) + add("cm1000_html_table.malformed_missing_channel_id", "cm1000", "cm1000_html_table", "minimal-synthetic-malformed", lambda: _cm1000("
Lock StatusFrequencyPowerSNR
Locked591000000040
"), EMPTY_SPLIT) + + cm3500_expected = _split( + [{"channelID": 3, "frequency": "570 MHz", "powerLevel": 4.7, "mse": -38.98, "mer": 38.98, "modulation": "256QAM", "corrErrors": 92, "nonCorrErrors": 0}], + [{"channelID": 200, "type": "OFDM", "frequency": "135-324 MHz", "powerLevel": None, "mer": 41.0, "mse": None, "corrErrors": None, "nonCorrErrors": None}], + [{"channelID": 9, "frequency": "30 MHz", "powerLevel": 39.5, "modulation": "64QAM", "multiplex": "ATDMA"}], + [{"channelID": 200, "type": "OFDMA", "frequency": "29-64 MHz", "powerLevel": 42.25, "modulation": "OFDMA", "multiplex": ""}], + ) + add("cm3500_html.success_all_lanes", "cm3500", "cm3500_html", "captured-subset", lambda: _cm3500(CM3500_SUCCESS), cm3500_expected) + add("cm3500_html.empty_document", "cm3500", "cm3500_html", "minimal-empty", lambda: _cm3500(""), EMPTY_SPLIT) + add("cm3500_html.malformed_numeric", "cm3500", "cm3500_html", "minimal-synthetic-malformed", lambda: _cm3500("

Downstream QAM

abcdefghi
xbadbadbadbadQAM0badbad
"), _split(ds30=[{"channelID": 0, "frequency": "bad", "powerLevel": 0.0, "mse": -0.0, "mer": 0.0, "modulation": "QAM", "corrErrors": 0, "nonCorrErrors": 0}])) + + surf_ds = "1^Locked^256QAM^43^705000000^0.0^40.9^31^0^|+|2^Locked^OFDM PLC^193^957000000^0.1^43.0^9^1^" + surf_us = "1^Locked^SC-QAM^3^6400000^29200000^46.5^|+|2^Locked^OFDMA^41^44400000^36200000^43.8^" + surf_expected = _split( + [{"channelID": 43, "frequency": "705 MHz", "powerLevel": 0.0, "mer": 40.9, "mse": -40.9, "modulation": "256QAM", "corrErrors": 31, "nonCorrErrors": 0}], + [{"channelID": 193, "type": "OFDM", "frequency": "957 MHz", "powerLevel": 0.1, "mer": 43.0, "mse": None, "corrErrors": 9, "nonCorrErrors": 1}], + [{"channelID": 3, "frequency": "29.2 MHz", "powerLevel": 46.5, "modulation": "SC-QAM", "multiplex": "SC-QAM"}], + [{"channelID": 41, "type": "OFDMA", "frequency": "36.2 MHz", "powerLevel": 43.8, "modulation": "OFDMA", "multiplex": ""}], + ) + add("surfboard_hnap.success_all_lanes", "surfboard", "surfboard_hnap", "captured-subset", lambda: _surfboard(surf_ds, surf_us), surf_expected) + add("surfboard_hnap.empty_strings", "surfboard", "surfboard_hnap", "minimal-empty", lambda: _surfboard("", ""), EMPTY_SPLIT) + add("surfboard_hnap.malformed_numeric", "surfboard", "surfboard_hnap", "minimal-synthetic-malformed", lambda: _surfboard("1^Locked^256QAM^bad^bad^bad^bad^bad^bad^", "1^Locked^SC-QAM^bad^0^bad^bad^"), EMPTY_SPLIT) + + arris_expected = _split( + [{"channelID": 7, "frequency": "591 MHz", "powerLevel": 0.0, "modulation": "256QAM", "corrErrors": 3, "nonCorrErrors": 0, "mer": 40.0, "mse": -40.0}], + [{"channelID": 193, "frequency": "690 MHz", "powerLevel": None, "modulation": "Other", "corrErrors": 9, "nonCorrErrors": 1, "type": "OFDM", "mer": 41.8, "mse": None}], + [{"channelID": 3, "frequency": "29.2 MHz", "powerLevel": 43.5, "modulation": "SC-QAM Upstream", "multiplex": "SC-QAM"}], + [{"channelID": 41, "frequency": "36.2 MHz", "powerLevel": 36.5, "modulation": "OFDM Upstream", "type": "OFDMA", "multiplex": ""}], + ) + add("arris_html.success_scqam_ofdm_ofdma", "surfboard,cm8200", "arris_html", "captured-subset", lambda: parse_arris_channel_tables(ARRIS_SUCCESS), arris_expected) + add("arris_html.empty_document", "surfboard,cm8200", "arris_html", "minimal-empty", lambda: parse_arris_channel_tables(""), EMPTY_SPLIT) + arris_bad = ARRIS_SUCCESS.replace("7Locked", "badLocked").replace("3Locked", "badLocked") + add("arris_html.malformed_channel_ids", "surfboard,cm8200", "arris_html", "minimal-synthetic-malformed", lambda: parse_arris_channel_tables(arris_bad), _split(ds31=arris_expected["channelDs"]["docsis31"], us31=arris_expected["channelUs"]["docsis31"])) + + sb6141_expected = _split( + [{"channelID": 7, "frequency": "591 MHz", "powerLevel": 0.0, "mer": 40.0, "mse": -40.0, "modulation": "256QAM", "corrErrors": 3, "nonCorrErrors": 1}], [], + [{"channelID": 3, "frequency": "29.2 MHz", "powerLevel": 43.5, "modulation": "64QAM", "multiplex": "SC-QAM"}], [], + ) + add("sb6141_transposed_html.success", "sb6141", "sb6141_transposed_html", "captured-subset", lambda: _sb6141(SB6141_SUCCESS), sb6141_expected) + add("sb6141_transposed_html.empty_document", "sb6141", "sb6141_transposed_html", "minimal-empty", lambda: _sb6141(""), EMPTY_SPLIT) + add("sb6141_transposed_html.malformed_channel_id", "sb6141", "sb6141_transposed_html", "minimal-synthetic-malformed", lambda: _sb6141(SB6141_SUCCESS.replace("7", "bad", 1).replace("3", "bad", 1)), EMPTY_SPLIT) + + from tests.test_sb6183_driver import SAMPLE_STATUS_HTML as SB6183_HTML + from tests.test_sb6190_driver import SAMPLE_STATUS_HTML as SB6190_HTML + sb6183_success = _split( + ds30=[{"channelID": 6, "frequency": "315 MHz", "powerLevel": 0.7, "mer": 40.2, "mse": -40.2, "modulation": "QAM256", "corrErrors": 29, "nonCorrErrors": 0}], + us30=[{"channelID": 67, "frequency": "30.4 MHz", "powerLevel": 48.3, "modulation": "ATDMA", "multiplex": "ATDMA"}], + ) + sb6190_success = _split( + ds30=[{"channelID": 13, "frequency": "807 MHz", "powerLevel": 10.5, "mer": 40.95, "mse": -40.95, "modulation": "256QAM", "corrErrors": 33, "nonCorrErrors": 0}], + us30=[{"channelID": 1, "frequency": "17.6 MHz", "powerLevel": 35.0, "modulation": "ATDMA", "multiplex": "ATDMA"}], + ) + add("sb6183_html.success_captured_first_rows", "sb6183", "sb6183_html", "captured-disk", lambda: _row_html_driver(SB6183Driver, SB6183_HTML), sb6183_success) + add("sb6183_html.empty_document", "sb6183", "sb6183_html", "minimal-empty", lambda: _row_html_driver(SB6183Driver, ""), EMPTY_SPLIT) + add("sb6183_html.malformed_rows", "sb6183", "sb6183_html", "minimal-synthetic-malformed", lambda: _row_html_driver(SB6183Driver, "
Downstream Bonded
1LockedQAMbadbadbadbadbadbad
"), EMPTY_SPLIT) + add("sb6190_html.success_captured_first_rows", "sb6190", "sb6190_html", "captured-inline", lambda: _row_html_driver(SB6190Driver, SB6190_HTML), sb6190_success) + add("sb6190_html.empty_document", "sb6190", "sb6190_html", "minimal-empty", lambda: _row_html_driver(SB6190Driver, ""), EMPTY_SPLIT) + add("sb6190_html.malformed_rows", "sb6190", "sb6190_html", "minimal-synthetic-malformed", lambda: _row_html_driver(SB6190Driver, "
Downstream Bonded
1LockedQAMbadbadbadbadbadbad
"), EMPTY_SPLIT) + + from tests.test_hitron_driver import DS_OFDM_DATA, DS_SCQAM_DATA, US_OFDMA_DATA, US_SCQAM_DATA + hitron_payload = {"/data/dsinfo.asp": DS_SCQAM_DATA[:1], "/data/usinfo.asp": US_SCQAM_DATA[:1], "/data/dsofdminfo.asp": DS_OFDM_DATA[:1], "/data/usofdminfo.asp": US_OFDMA_DATA[:1]} + hitron_expected = _split( + ds30=[{"channelID": 7, "frequency": "591 MHz", "powerLevel": 4.6, "modulation": "256QAM", "mer": 36.387, "mse": -36.387, "corrErrors": 3, "nonCorrErrors": 30}], + ds31=[{"channelID": 0, "type": "OFDM", "frequency": "275.6 MHz", "powerLevel": 3.200001, "modulation": "OFDM", "mer": 38.0, "mse": None, "corrErrors": 652068075, "nonCorrErrors": 19}], + us30=[{"channelID": 6, "frequency": "25.9 MHz", "powerLevel": 35.0, "modulation": "64QAM", "multiplex": "ATDMA"}], + us31=[{"channelID": 0, "type": "OFDMA", "frequency": "42 MHz", "powerLevel": 37.75, "modulation": "OFDMA", "multiplex": ""}], + ) + add("hitron_coda56_json.success_captured_first_rows", "hitron", "hitron_coda56_json", "captured-inline", lambda: _hitron(hitron_payload), hitron_expected) + add("hitron_coda56_json.empty_arrays", "hitron", "hitron_coda56_json", "minimal-empty", lambda: _hitron({path: [] for path in hitron_payload}), EMPTY_SPLIT) + add("hitron_coda56_json.malformed_missing_required_fields", "hitron", "hitron_coda56_json", "minimal-synthetic-malformed", lambda: _hitron({path: [{}] for path in hitron_payload}), EMPTY_SPLIT) + + from tests.test_hitron_coda_4680_driver import DS_INFO, DS_OFDM, US_INFO, US_OFDMA + h4680_payload = {"/1/Device/CM/DsInfo": {"Freq_List": DS_INFO["Freq_List"][:1]}, "/1/Device/CM/UsInfo": {"Freq_List": US_INFO["Freq_List"][:1]}, "/1/Device/CM/DsOfdm": {"OFDMs_List": DS_OFDM["OFDMs_List"][:1]}, "/1/Device/CM/UsOfdm": {"OFDMAs_List": US_OFDMA["OFDMAs_List"][:1]}} + h4680_expected = _split( + ds30=[{"channelID": 18, "frequency": "663 MHz", "powerLevel": 5.099, "modulation": "256QAM", "mer": 40.946, "mse": -40.946, "corrErrors": 5, "nonCorrErrors": 33}], + ds31=[{"channelID": 0, "type": "OFDM", "frequency": "275.6 MHz", "powerLevel": 3.0, "modulation": "OFDM", "mer": None, "mse": None, "corrErrors": None, "nonCorrErrors": None}], + us30=[{"channelID": 3, "frequency": "32.3 MHz", "powerLevel": 42.77, "modulation": "64QAM", "multiplex": "ATDMA", "symbolRate": 5120}], + us31=[{"channelID": 0, "type": "OFDMA", "frequency": "", "powerLevel": 38.75, "modulation": "OFDMA", "multiplex": "OFDMA"}], + ) + add("hitron_coda4680_json.success_captured_first_rows", "hitron_coda_4680", "hitron_coda4680_json", "captured-inline", lambda: _hitron_4680(h4680_payload), h4680_expected) + add("hitron_coda4680_json.empty_arrays", "hitron_coda_4680", "hitron_coda4680_json", "minimal-empty", lambda: _hitron_4680({"/1/Device/CM/DsInfo": {}, "/1/Device/CM/UsInfo": {}, "/1/Device/CM/DsOfdm": {}, "/1/Device/CM/UsOfdm": {}}), EMPTY_SPLIT) + add("hitron_coda4680_json.malformed_missing_required_fields", "hitron_coda_4680", "hitron_coda4680_json", "minimal-synthetic-malformed", lambda: _hitron_4680({"/1/Device/CM/DsInfo": {"Freq_List": [{}]}, "/1/Device/CM/UsInfo": {"Freq_List": [{}]}, "/1/Device/CM/DsOfdm": {"OFDMs_List": [{"plclock": "YES"}]}, "/1/Device/CM/UsOfdm": {"OFDMAs_List": [{"state": "OPERATE"}]}}), EMPTY_SPLIT) + + sagem_ds = [{"ChannelID": 13, "LockStatus": True, "Frequency": 546000000.0, "SNR": 44.0, "PowerLevel": 0, "Modulation": "Qam256", "BandWidth": 8000000, "CorrectableCodewords": 10, "UncorrectableCodewords": 2}, {"ChannelID": 193, "LockStatus": True, "Frequency": 666000000.0, "SNR": 44.0, "PowerLevel": 7.9, "Modulation": "256-QAM1K-QAM2K-QA", "BandWidth": 128000000, "CorrectableCodewords": 100, "UncorrectableCodewords": 0}] + sagem_us = [{"ChannelID": 1, "LockStatus": True, "Frequency": 38000000.0, "PowerLevel": 41.8, "Modulation": "atdma"}, {"ChannelID": 41, "LockStatus": True, "Frequency": 104800000.0, "PowerLevel": 88.0, "Modulation": "ofdma"}] + sagem_expected = _split( + ds30=[{"channelID": 13, "frequency": "546 MHz", "powerLevel": 0, "mer": 44.0, "mse": -44.0, "modulation": "256QAM", "corrErrors": 10, "nonCorrErrors": 2}], + ds31=[{"channelID": 193, "type": "OFDM", "frequency": "666 MHz", "powerLevel": 7.9, "mer": 44.0, "mse": None, "corrErrors": 100, "nonCorrErrors": 0}], + us30=[{"channelID": 1, "frequency": "38 MHz", "powerLevel": 41.8, "modulation": "ATDMA", "multiplex": "ATDMA"}], + us31=[{"channelID": 41, "type": "OFDMA", "frequency": "104.8 MHz", "powerLevel": 88.0, "modulation": "OFDMA", "multiplex": ""}], + ) + add("sagemcom_xmo_json.success_all_lanes", "sagemcom", "sagemcom_xmo_json", "captured-inline", lambda: _sagemcom(sagem_ds, sagem_us), sagem_expected) + add("sagemcom_xmo_json.empty_arrays", "sagemcom", "sagemcom_xmo_json", "minimal-empty", lambda: _sagemcom([], []), EMPTY_SPLIT) + add("sagemcom_xmo_json.malformed_values", "sagemcom", "sagemcom_xmo_json", "minimal-synthetic-malformed", lambda: _sagemcom([{"LockStatus": True, "BandWidth": "bad"}], [{"LockStatus": True, "Modulation": None}]), None) + + from tests.drivers.f3896lg._data import DOWNSTREAM, UPSTREAM + f_ds = [DOWNSTREAM["downstream"]["channels"][0], DOWNSTREAM["downstream"]["channels"][-1]] + f_us = [UPSTREAM["upstream"]["channels"][0], UPSTREAM["upstream"]["channels"][-1]] + f_expected = _split( + ds30=[{"channelID": 1, "frequency": "411 MHz", "powerLevel": -4.3, "mer": 39, "mse": -39, "modulation": "256QAM", "corrErrors": 26, "nonCorrErrors": 0}], + ds31=[{"channelID": 41, "type": "OFDM", "frequency": "", "powerLevel": -11.8, "mer": None, "mse": None, "modulation": "OFDM", "corrErrors": 1361678039, "nonCorrErrors": 483483438, "profile_modulation": "4096QAM"}], + us30=[{"channelID": 6, "frequency": "49.6 MHz", "powerLevel": 42.5, "modulation": "64QAM", "multiplex": "ATDMA", "symbolRate": 5120}], + us31=[{"channelID": 12, "type": "OFDMA", "frequency": "", "powerLevel": 38.0, "modulation": "OFDMA", "multiplex": "", "profile_modulation": "256QAM"}], + ) + add("f3896lg_rest_json.success_captured_all_lanes", "f3896lg", "f3896lg_rest_json", "captured-inline", lambda: _f3896(f_ds, f_us), f_expected) + add("f3896lg_rest_json.empty_arrays", "f3896lg", "f3896lg_rest_json", "minimal-empty", lambda: _f3896([], []), EMPTY_SPLIT) + add("f3896lg_rest_json.malformed_values", "f3896lg", "f3896lg_rest_json", "minimal-synthetic-malformed", lambda: _f3896([{"lockStatus": True, "channelType": "sc_qam", "frequency": "bad", "snr": 0}], [{"lockStatus": True, "channelType": "ofdma", "power": "bad"}]), _split(us31=[{"channelID": 0, "type": "OFDMA", "frequency": "", "powerLevel": None, "modulation": "OFDMA", "multiplex": ""}])) + + from tests.test_sercom_dm1000_driver import DS_INFO as SER_DS, DS_OFDM as SER_OFDM, US_INFO as SER_US, US_OFDMA as SER_OFDMA + ser_expected = _split( + ds30=[{"channelID": 7, "frequency": "591 MHz", "powerLevel": 8.800003, "modulation": "256QAM", "mer": 38.983261, "mse": -38.983261, "corrErrors": 41, "nonCorrErrors": 2}], + ds31=[{"channelID": 0, "type": "OFDM", "frequency": "275.6 MHz", "powerLevel": 10.300003, "modulation": "OFDM", "mer": 39.0, "mse": None, "corrErrors": None, "nonCorrErrors": None}], + us30=[{"channelID": 5, "frequency": "21.1 MHz", "powerLevel": 35.2603, "modulation": "64QAM", "multiplex": "ATDMA", "symbolRate": 2560}], + us31=[{"channelID": 0, "type": "OFDMA", "frequency": "42 MHz", "powerLevel": 37.25, "modulation": "OFDMA", "multiplex": "OFDMA", "profile_modulation": "256QAM"}], + ) + add("sercom_dm1000_json.success_captured_all_lanes", "sercom_dm1000", "sercom_dm1000_json", "captured-inline", lambda: _sercom(SER_DS["nodes"][:1], SER_OFDM["nodes"][:1], SER_US["nodes"][:1], SER_OFDMA["nodes"]), ser_expected) + add("sercom_dm1000_json.empty_arrays", "sercom_dm1000", "sercom_dm1000_json", "minimal-empty", lambda: _sercom([], [], [], []), EMPTY_SPLIT) + add("sercom_dm1000_json.malformed_missing_required_fields", "sercom_dm1000", "sercom_dm1000_json", "minimal-synthetic-malformed", lambda: _sercom([{"qamD": "256QAM"}], [{"PLC": "YES", "MDC1": "YES", "AV_Data": "40"}], [{"modulation": "64QAM", "upstream": "1", "rate": "2.56"}], [{"name": "CH", "index1": "bad"}, {"name": "Power", "index1": "ON"}, {"name": "STATE", "index1": "RNG3"}, {"name": "Center Freq SC0", "index1": "42"}]), EMPTY_SPLIT) + + cgm_ds = {"Channel ID": ["7", "193"], "Lock Status": ["Locked", "Locked"], "Frequency": ["591 MHz", "690000000"], "SNR": ["40 dB", "41.8 dB"], "Power Level": ["0 dBmV", ""], "Modulation": ["256 QAM", "OFDM"]} + cgm_us = {"Channel ID": ["3", "41"], "Lock Status": ["Locked", "Locked"], "Frequency": ["29 MHz", "42 MHz"], "Power Level": ["43.5 dBmV", "37.75 dBmV"], "Modulation": ["QAM", "OFDMA"], "Channel Type": ["ATDMA", "OFDMA"]} + cgm_err = {"Channel ID": ["7", "193"], "Correctable Codewords": ["3", "9"], "Uncorrectable Codewords": ["0", "1"]} + cgm_expected = _split( + ds30=[{"channelID": 7, "frequency": "591 MHz", "powerLevel": 0.0, "mer": 40.0, "mse": -40.0, "modulation": "256QAM", "corrErrors": 3, "nonCorrErrors": 0}], + ds31=[{"channelID": 193, "frequency": "690 MHz", "powerLevel": None, "mer": 41.8, "mse": -41.8, "modulation": "OFDM", "corrErrors": 9, "nonCorrErrors": 1, "type": "OFDM"}], + us30=[{"channelID": 3, "frequency": "29 MHz", "powerLevel": 43.5, "modulation": "QAM", "multiplex": "ATDMA"}], + us31=[{"channelID": 41, "frequency": "42 MHz", "powerLevel": 37.75, "modulation": "OFDMA", "multiplex": "OFDMA", "type": "OFDMA"}], + ) + add("cgm4981_columnar_html.success_all_lanes", "cgm4981", "cgm4981_columnar_html", "minimal-synthetic-existing-fields", lambda: _cgm(cgm_ds, cgm_us, cgm_err), cgm_expected) + add("cgm4981_columnar_html.empty_rows", "cgm4981", "cgm4981_columnar_html", "minimal-empty", lambda: _cgm({}, {}, {}), EMPTY_SPLIT) + add("cgm4981_columnar_html.malformed_channel_ids", "cgm4981", "cgm4981_columnar_html", "minimal-synthetic-malformed", lambda: _cgm({"Channel ID": ["bad"], "Lock Status": ["Locked"]}, {"Channel ID": ["bad"], "Lock Status": ["Locked"]}, {}), EMPTY_SPLIT) + + generic = GenericDriver("", "", "") + add("generic_no_docsis.success_boundary", "generic", "generic_no_docsis", "no-input-boundary", generic.get_docsis_data, EMPTY_SPLIT) + add("generic_no_docsis.missing_input_not_applicable", "generic", "generic_no_docsis", "no-input-boundary", GenericDriver("", "", "").get_docsis_data, EMPTY_SPLIT) + add("generic_no_docsis.malformed_input_not_applicable", "generic", "generic_no_docsis", "no-input-boundary", GenericDriver("", "", "").get_docsis_data, EMPTY_SPLIT) + + return cases + + +CASES = tuple(sorted(_cases(), key=lambda case: case.case_id)) +CASE_BY_ID = {case.case_id: case for case in CASES} + +if len(CASE_BY_ID) != len(CASES): + raise AssertionError("driver format case IDs must be unique") diff --git a/tests/drivers/test_driver_format_characterization.py b/tests/drivers/test_driver_format_characterization.py new file mode 100644 index 00000000..60fd4721 --- /dev/null +++ b/tests/drivers/test_driver_format_characterization.py @@ -0,0 +1,105 @@ +"""Exact characterization assertions for current modem parser seams.""" + +from __future__ import annotations + +from collections import Counter + +from tests.drivers.driver_format_cases import CASES, CASE_BY_ID + + +EXPECTED_FAMILIES = { + "arris_html", + "cgm4981_columnar_html", + "ch7465_xml", + "cm1000_html_table", + "cm1000_javascript", + "cm3000_javascript", + "cm3500_html", + "f3896lg_rest_json", + "fritzbox_data_lua", + "generic_no_docsis", + "hitron_coda4680_json", + "hitron_coda56_json", + "sagemcom_xmo_json", + "sb6141_transposed_html", + "sb6183_html", + "sb6190_html", + "sercom_dm1000_json", + "surfboard_hnap", + "tc4400_html", + "ultrahub7_json", + "vodafone_station_cga_json", + "vodafone_station_tg_embedded_json", +} + +FAMILIES_WITH_DOCSIS_31_CHANNELS = { + "arris_html", + "cgm4981_columnar_html", + "cm1000_html_table", + "cm3000_javascript", + "cm3500_html", + "f3896lg_rest_json", + "fritzbox_data_lua", + "hitron_coda4680_json", + "hitron_coda56_json", + "sagemcom_xmo_json", + "sercom_dm1000_json", + "surfboard_hnap", + "tc4400_html", + "ultrahub7_json", + "vodafone_station_cga_json", + "vodafone_station_tg_embedded_json", +} + + +def _has_ofdm_or_ofdma(output) -> bool: + if isinstance(output, dict): + if output.get("type") in {"OFDM", "OFDMA"}: + return True + return any(_has_ofdm_or_ofdma(value) for value in output.values()) + if isinstance(output, list): + return any(_has_ofdm_or_ofdma(value) for value in output) + return False + + +def test_case_registry_is_complete_and_has_three_boundary_cases_per_family(): + counts = Counter(case.family for case in CASES) + assert set(counts) == EXPECTED_FAMILIES + assert counts == {family: 3 for family in EXPECTED_FAMILIES} + assert len(CASE_BY_ID) == len(CASES) == 66 + assert list(CASE_BY_ID) == sorted(CASE_BY_ID) + + +def test_each_current_parser_observation_matches_the_frozen_normalized_structure(): + mismatches = {} + for case in CASES: + actual = case.observe().output + if actual != case.expected: + mismatches[case.case_id] = {"expected": case.expected, "actual": actual} + assert mismatches == {} + + +def test_supported_docsis31_families_have_ofdm_or_ofdma_in_success_output(): + covered = { + case.family + for case in CASES + if ".success" in case.case_id and _has_ofdm_or_ofdma(case.expected) + } + assert covered == FAMILIES_WITH_DOCSIS_31_CHANNELS + + +def test_fritzbox_boundary_preserves_order_duplicates_and_missing_channel_ids(): + output = CASE_BY_ID["fritzbox_data_lua.success_duplicates_missing_id"].observe().output + channels = output["channelDs"]["docsis30"] + assert [channel.get("channelID") for channel in channels] == [1, 1, None] + assert len(channels) == 3 + assert channels[0]["powerLevel"] == 0 + assert channels[1]["powerLevel"] is None + assert "channelID" not in channels[2] + + +def test_malformed_diagnostics_are_stable_across_repeated_observation(): + for case in CASES: + if ".malformed" not in case.case_id: + continue + assert case.observe().diagnostics == case.observe().diagnostics diff --git a/tests/drivers/test_driver_parser_golden.py b/tests/drivers/test_driver_parser_golden.py new file mode 100644 index 00000000..f25c7d9f --- /dev/null +++ b/tests/drivers/test_driver_parser_golden.py @@ -0,0 +1,73 @@ +"""Determinism and canonical-byte tests for the golden matrix generator.""" + +from __future__ import annotations + +import json +import math + +import pytest + +from scripts.driver_parser_golden import ( + build_report, + canonical_bytes, + canonical_sha256, + main, +) +from tests.drivers.driver_format_cases import CASES + + +def test_canonical_serialization_sorts_only_mapping_keys_and_preserves_list_order(): + value = {"z": [{"b": 2, "a": 1}, {"a": 0}], "a": "ä"} + assert canonical_bytes(value) == b'{"a":"\xc3\xa4","z":[{"a":1,"b":2},{"a":0}]}' + + +def test_canonical_digest_is_stable_across_mapping_insertion_order(): + left = {"b": 2, "a": [{"d": 4, "c": 3}]} + right = {"a": [{"c": 3, "d": 4}], "b": 2} + assert canonical_bytes(left) == canonical_bytes(right) + assert canonical_sha256(left) == canonical_sha256(right) + + +def test_null_zero_and_missing_keys_have_distinct_canonical_bytes_and_digests(): + values = [{"power": None}, {"power": 0}, {}] + assert len({canonical_bytes(value) for value in values}) == 3 + assert len({canonical_sha256(value) for value in values}) == 3 + + +@pytest.mark.parametrize("value", [math.nan, math.inf, -math.inf]) +def test_canonical_serialization_rejects_non_finite_numbers(value): + with pytest.raises(ValueError, match="JSON compliant"): + canonical_bytes({"value": value}) + + +def test_report_rows_are_sorted_and_contain_only_digests_counts_and_labels(): + report = build_report("c1e4946880a5be8d29d7db92587f987bf426920c") + rows = report["cases"] + assert [row["case_id"] for row in rows] == sorted(case.case_id for case in CASES) + assert all(len(row["output_sha256"]) == 64 for row in rows) + assert all( + set(row) <= { + "case_id", + "driver", + "family", + "output_sha256", + "diagnostics_sha256", + "structural_counts", + } + for row in rows + ) + encoded = canonical_bytes(report) + assert b"modem.invalid" not in encoded + assert b"/tmp/" not in encoded + + +def test_cli_writes_identical_bytes_twice_for_same_source_label(tmp_path): + first = tmp_path / "one.json" + second = tmp_path / "two.json" + label = "source-base-c1e4946" + assert main(["--source-label", label, "--output", str(first)]) == 0 + assert main(["--source-label", label, "--output", str(second)]) == 0 + assert first.read_bytes() == second.read_bytes() + parsed = json.loads(first.read_text(encoding="utf-8")) + assert parsed["source_label"] == label + assert len(parsed["cases"]) == len(CASES) diff --git a/tests/drivers/test_format_profiles.py b/tests/drivers/test_format_profiles.py new file mode 100644 index 00000000..95fec4b6 --- /dev/null +++ b/tests/drivers/test_format_profiles.py @@ -0,0 +1,284 @@ +"""Direct, network-free tests for every explicit pure format profile.""" + +from __future__ import annotations + +from bs4 import BeautifulSoup +import pytest + +from app.drivers.formats.boundaries import parse_generic_no_docsis +from app.drivers.formats.contract import ParseResult +from app.drivers.formats.fritzbox import parse_fritzbox_data_lua +from app.drivers.formats.hitron import ( + parse_hitron_coda4680_json, + parse_hitron_coda56_json, +) +from app.drivers.formats.html_columnar import parse_cgm4981_columnar_html +from app.drivers.formats.html_rows import ( + parse_arris_html, + parse_cm1000_html_table, + parse_cm3500_html, + parse_sb6183_html, + parse_sb6190_html, + parse_tc4400_html, +) +from app.drivers.formats.html_transposed import parse_sb6141_transposed_html +from app.drivers.formats.javascript import ( + parse_cm1000_javascript, + parse_cm3000_javascript, +) +from app.drivers.formats.sagemcom import ( + parse_f3896lg_rest_json, + parse_sagemcom_xmo_json, +) +from app.drivers.formats.sercom import parse_sercom_dm1000_json +from app.drivers.formats.surfboard import parse_surfboard_hnap +from app.drivers.formats.vodafone import ( + parse_ultrahub7_json, + parse_vodafone_station_cga_json, + parse_vodafone_station_tg_embedded_json, +) +from app.drivers.formats.xml_payloads import parse_ch7465_xml +from tests.drivers.driver_format_cases import ( + ARRIS_SUCCESS, + CM1000_TABLE_SUCCESS, + CM3000_SUCCESS, + CM3500_SUCCESS, + FRITZ_SUCCESS, + SB6141_SUCCESS, + TC_DS_SUCCESS, + TC_US_SUCCESS, + TG_SUCCESS, +) +from tests.drivers.f3896lg._data import DOWNSTREAM as F3896_DS, UPSTREAM as F3896_US +from tests.test_hitron_coda_4680_driver import DS_INFO, DS_OFDM, US_INFO, US_OFDMA +from tests.test_hitron_driver import DS_OFDM_DATA, DS_SCQAM_DATA, US_OFDMA_DATA, US_SCQAM_DATA +from tests.test_sb6183_driver import SAMPLE_STATUS_HTML as SB6183_HTML +from tests.test_sb6190_driver import SAMPLE_STATUS_HTML as SB6190_HTML +from tests.test_sercom_dm1000_driver import ( + DS_INFO as SERCOM_DS, + DS_OFDM as SERCOM_OFDM, + US_INFO as SERCOM_US, + US_OFDMA as SERCOM_OFDMA, +) + + +def _tc_success(): + downstream = BeautifulSoup(TC_DS_SUCCESS, "html.parser").find("table") + upstream = BeautifulSoup(TC_US_SUCCESS, "html.parser").find("table") + return parse_tc4400_html(downstream, upstream) + + +def _columnar_success() -> str: + def rows(values): + return "".join( + f'{label}{"".join(f"
{value}
" for value in items)}' + for label, items in values.items() + ) + + downstream = { + "Channel ID": ["7", "193"], "Lock Status": ["Locked", "Locked"], + "Frequency": ["591 MHz", "690000000"], "SNR": ["40", "41.8"], + "Power Level": ["0", ""], "Modulation": ["256 QAM", "OFDM"], + } + upstream = { + "Channel ID": ["3", "41"], "Lock Status": ["Locked", "Locked"], + "Frequency": ["29 MHz", "42 MHz"], "Power Level": ["43.5", "37.75"], + "Modulation": ["QAM", "OFDMA"], "Channel Type": ["ATDMA", "OFDMA"], + } + errors = { + "Channel ID": ["7", "193"], "Correctable Codewords": ["3", "9"], + "Uncorrectable Codewords": ["0", "1"], + } + return f">Downstream<{rows(downstream)}>Upstream<{rows(upstream)}CM Error Codewords{rows(errors)}" + + +SUCCESS_PROFILES = { + "arris_html": lambda: parse_arris_html(ARRIS_SUCCESS), + "cgm4981_columnar_html": lambda: parse_cgm4981_columnar_html(_columnar_success()), + "ch7465_xml": lambda: parse_ch7465_xml( + "7591 MHz0256qam", + "329 MHz43.564qam", + ), + "cm1000_html_table": lambda: parse_cm1000_html_table(CM1000_TABLE_SUCCESS), + "cm1000_javascript": lambda: parse_cm1000_javascript( + open("tests/fixtures/cm1000/DocsisStatus.asp.html", encoding="utf-8").read() + ), + "cm3000_javascript": lambda: parse_cm3000_javascript(CM3000_SUCCESS), + "cm3500_html": lambda: parse_cm3500_html(CM3500_SUCCESS), + "f3896lg_rest_json": lambda: parse_f3896lg_rest_json({ + "downstream": [F3896_DS["downstream"]["channels"][0], F3896_DS["downstream"]["channels"][-1]], + "upstream": [F3896_US["upstream"]["channels"][0], F3896_US["upstream"]["channels"][-1]], + }), + "fritzbox_data_lua": lambda: parse_fritzbox_data_lua(FRITZ_SUCCESS), + "generic_no_docsis": parse_generic_no_docsis, + "hitron_coda4680_json": lambda: parse_hitron_coda4680_json({ + "downstream": DS_INFO, "upstream": US_INFO, + "downstream_ofdm": DS_OFDM, "upstream_ofdma": US_OFDMA, + }), + "hitron_coda56_json": lambda: parse_hitron_coda56_json({ + "downstream": DS_SCQAM_DATA[:1], "upstream": US_SCQAM_DATA[:1], + "downstream_ofdm": DS_OFDM_DATA[:1], "upstream_ofdma": US_OFDMA_DATA[:1], + }), + "sagemcom_xmo_json": lambda: parse_sagemcom_xmo_json({ + "downstream": [{"ChannelID": 193, "LockStatus": True, "Frequency": 666000000, + "SNR": 44.0, "PowerLevel": 7.9, "Modulation": "256-QAM1K-QAM2K-QA", + "BandWidth": 128000000}], + "upstream": [{"ChannelID": 41, "LockStatus": True, "Frequency": 104800000, + "PowerLevel": 38.0, "Modulation": "ofdma"}], + }), + "sb6141_transposed_html": lambda: parse_sb6141_transposed_html(SB6141_SUCCESS), + "sb6183_html": lambda: parse_sb6183_html(SB6183_HTML), + "sb6190_html": lambda: parse_sb6190_html(SB6190_HTML), + "sercom_dm1000_json": lambda: parse_sercom_dm1000_json({ + "downstream": SERCOM_DS["nodes"], "downstream_ofdm": SERCOM_OFDM["nodes"], + "upstream": SERCOM_US["nodes"], "upstream_ofdma": SERCOM_OFDMA["nodes"], + }), + "surfboard_hnap": lambda: parse_surfboard_hnap( + "1^Locked^OFDM PLC^193^957000000^0.1^43.0^9^1^", + "1^Locked^OFDMA^41^44400000^36200000^43.8^", + ), + "tc4400_html": _tc_success, + "ultrahub7_json": lambda: parse_ultrahub7_json({ + "downstream": [{"ChannelID": "193", "Frequency": "690 MHz", "Modulation": "OFDM", "PowerLevel": "0", "SNRLevel": "41.8"}], + "upstream": [{"ChannelID": "41", "Frequency": "36.2 MHz", "Modulation": "OFDMA", "PowerLevel": "36.5"}], + }), + "vodafone_station_cga_json": lambda: parse_vodafone_station_cga_json({ + "ofdm_downstream": [{"channelid_ofdm": "193", "CentralFrequency_ofdm": "690000000", "power_ofdm": "0", "SNR_ofdm": "41.8"}], + "ofdma_upstream": [{"channelidup": "41", "CentralFrequency": "36200000", "power": "36.5"}], + }), + "vodafone_station_tg_embedded_json": lambda: parse_vodafone_station_tg_embedded_json(TG_SUCCESS), +} + + +@pytest.mark.parametrize("profile", sorted(SUCCESS_PROFILES)) +def test_each_explicit_profile_is_directly_callable_without_transport(profile): + result = SUCCESS_PROFILES[profile]() + assert isinstance(result, ParseResult) + assert result.value is not None + + +EMPTY_OR_MISSING_PROFILES = { + "arris_html": lambda: parse_arris_html(""), + "cgm4981_columnar_html": lambda: parse_cgm4981_columnar_html(""), + "ch7465_xml": lambda: parse_ch7465_xml("", ""), + "cm1000_html_table": lambda: parse_cm1000_html_table(""), + "cm1000_javascript": lambda: parse_cm1000_javascript(""), + "cm3000_javascript": lambda: parse_cm3000_javascript(""), + "cm3500_html": lambda: parse_cm3500_html(""), + "f3896lg_rest_json": lambda: parse_f3896lg_rest_json({}), + "fritzbox_data_lua": lambda: parse_fritzbox_data_lua({}), + "generic_no_docsis": parse_generic_no_docsis, + "hitron_coda4680_json": lambda: parse_hitron_coda4680_json({}), + "hitron_coda56_json": lambda: parse_hitron_coda56_json({}), + "sagemcom_xmo_json": lambda: parse_sagemcom_xmo_json({}), + "sb6141_transposed_html": lambda: parse_sb6141_transposed_html(""), + "sb6183_html": lambda: parse_sb6183_html(""), + "sb6190_html": lambda: parse_sb6190_html(""), + "sercom_dm1000_json": lambda: parse_sercom_dm1000_json({}), + "surfboard_hnap": lambda: parse_surfboard_hnap("", ""), + "tc4400_html": lambda: parse_tc4400_html(None, None), + "ultrahub7_json": lambda: parse_ultrahub7_json({}), + "vodafone_station_cga_json": lambda: parse_vodafone_station_cga_json({}), + "vodafone_station_tg_embedded_json": lambda: parse_vodafone_station_tg_embedded_json(""), +} + + +NULL_PROFILES = { + "arris_html": lambda: parse_arris_html(None), + "cgm4981_columnar_html": lambda: parse_cgm4981_columnar_html(None), + "ch7465_xml": lambda: parse_ch7465_xml(None, None), + "cm1000_html_table": lambda: parse_cm1000_html_table(None), + "cm1000_javascript": lambda: parse_cm1000_javascript(None), + "cm3000_javascript": lambda: parse_cm3000_javascript(None), + "cm3500_html": lambda: parse_cm3500_html(None), + "f3896lg_rest_json": lambda: parse_f3896lg_rest_json(None), + "fritzbox_data_lua": lambda: parse_fritzbox_data_lua(None), + "hitron_coda4680_json": lambda: parse_hitron_coda4680_json(None), + "hitron_coda56_json": lambda: parse_hitron_coda56_json(None), + "sagemcom_xmo_json": lambda: parse_sagemcom_xmo_json(None), + "sb6141_transposed_html": lambda: parse_sb6141_transposed_html(None), + "sb6183_html": lambda: parse_sb6183_html(None), + "sb6190_html": lambda: parse_sb6190_html(None), + "sercom_dm1000_json": lambda: parse_sercom_dm1000_json(None), + "surfboard_hnap": lambda: parse_surfboard_hnap(None, None), + "tc4400_html": lambda: parse_tc4400_html(None, None), + "ultrahub7_json": lambda: parse_ultrahub7_json(None), + "vodafone_station_cga_json": lambda: parse_vodafone_station_cga_json(None), + "vodafone_station_tg_embedded_json": lambda: parse_vodafone_station_tg_embedded_json(None), +} + + +def _assert_payload_safe_result(result): + assert isinstance(result, ParseResult) + for issue in result.diagnostics: + assert issue.family and issue.profile and issue.code + assert not hasattr(issue, "message") + + +@pytest.mark.parametrize("profile", sorted(EMPTY_OR_MISSING_PROFILES)) +def test_each_profile_handles_empty_or_missing_payload(profile): + _assert_payload_safe_result(EMPTY_OR_MISSING_PROFILES[profile]()) + + +@pytest.mark.parametrize("profile", sorted(NULL_PROFILES)) +def test_each_payload_profile_handles_null_payload(profile): + _assert_payload_safe_result(NULL_PROFILES[profile]()) + + +MALFORMED_PROFILES = { + "arris_html": lambda: parse_arris_html("
bad
"), + "cgm4981_columnar_html": lambda: parse_cgm4981_columnar_html(">Downstream<Channel ID
bad
"), + "ch7465_xml": lambda: parse_ch7465_xml("", ""), + "cm1000_html_table": lambda: parse_cm1000_html_table("
bad
"), + "cm1000_javascript": lambda: parse_cm1000_javascript("function InitDsTableTagValue(){var tagValueList='1|short';}"), + "cm3000_javascript": lambda: parse_cm3000_javascript("function InitDsTableTagValue(){var tagValueList='1|1|Locked|QAM|bad|bad|bad|bad|bad|';}"), + "cm3500_html": lambda: parse_cm3500_html("

Downstream QAM

bad
"), + "f3896lg_rest_json": lambda: parse_f3896lg_rest_json({"downstream": [], "upstream": [{"lockStatus": True, "channelType": "ofdma", "power": "bad"}]}), + "fritzbox_data_lua": lambda: parse_fritzbox_data_lua({"channelUs": {"docsis31": [{"powerLevel": None}]}}), + "generic_no_docsis": parse_generic_no_docsis, + "hitron_coda4680_json": lambda: parse_hitron_coda4680_json({"downstream": {"Freq_List": [{}]}}), + "hitron_coda56_json": lambda: parse_hitron_coda56_json({"downstream": [{}]}), + "sagemcom_xmo_json": lambda: parse_sagemcom_xmo_json({"downstream": [], "upstream": [{"LockStatus": True, "Modulation": None}]}), + "sb6141_transposed_html": lambda: parse_sb6141_transposed_html(SB6141_SUCCESS.replace("7", "bad", 1)), + "sb6183_html": lambda: parse_sb6183_html("
Downstream Bonded
bad
"), + "sb6190_html": lambda: parse_sb6190_html("
Downstream Bonded
bad
"), + "sercom_dm1000_json": lambda: parse_sercom_dm1000_json({"downstream": [{"qamD": "256QAM"}]}), + "surfboard_hnap": lambda: parse_surfboard_hnap("1^Locked^QAM^bad^bad^bad^bad^bad^bad^", ""), + "tc4400_html": lambda: parse_tc4400_html(None, None), + "ultrahub7_json": lambda: parse_ultrahub7_json({"downstream": [{"ChannelID": "bad"}], "upstream": []}), + "vodafone_station_cga_json": lambda: parse_vodafone_station_cga_json({"downstream": [None]}), + "vodafone_station_tg_embedded_json": lambda: parse_vodafone_station_tg_embedded_json("json_dsData=[{bad}];"), +} + + +@pytest.mark.parametrize("profile", sorted(MALFORMED_PROFILES)) +def test_each_profile_handles_empty_missing_or_malformed_input_without_payload_diagnostics(profile): + result = MALFORMED_PROFILES[profile]() + assert isinstance(result, ParseResult) + for issue in result.diagnostics: + assert issue.family and issue.profile and issue.code + assert not hasattr(issue, "message") + + +def test_locked_active_and_docsis31_status_rules_are_profile_specific(): + unlocked = ARRIS_SUCCESS.replace("Locked", "Not Locked") + assert parse_arris_html(unlocked).value["channelDs"]["docsis30"] == [] + + inactive = parse_hitron_coda56_json({ + "downstream_ofdm": [{"plclock": "NO"}], + "upstream_ofdma": [{"state": "DISABLED"}], + }).value + assert inactive["channelDs"]["docsis31"] == [] + assert inactive["channelUs"]["docsis31"] == [] + + success = parse_arris_html(ARRIS_SUCCESS).value + assert success["channelDs"]["docsis31"][0]["type"] == "OFDM" + assert success["channelUs"]["docsis31"][0]["type"] == "OFDMA" + + +def test_result_and_diagnostics_are_immutable_contract_objects(): + result = parse_ch7465_xml("", "") + with pytest.raises(AttributeError): + result.value = {} + with pytest.raises(AttributeError): + result.diagnostics[0].code = "changed"