diff --git a/meta/columns.json b/meta/columns.json index f4073f6ee5..706dedabc7 100644 --- a/meta/columns.json +++ b/meta/columns.json @@ -54,6 +54,17 @@ "cellType": "arrayPopover", "group": "capabilities" }, + "requestLimits": { + "key": "requestLimits", + "label": "Request limits", + "icon": "lucide:Gauge", + "description": "Documented per-request limits: batchRequests and blockSpan ceilings per method/transport, with a primary source per entry. Blank means not yet verified; plan-dependent limits stay blank until scoped.", + "filter": null, + "sorting": "arrayLength", + "pinning": null, + "cellType": "arrayPopover", + "group": "capabilities" + }, "offer": { "key": "offer", "label": "Offer", diff --git a/tools/csv_to_json.py b/tools/csv_to_json.py index 025fc42870..cc627dd2a2 100644 --- a/tools/csv_to_json.py +++ b/tools/csv_to_json.py @@ -7,6 +7,10 @@ OFFER_REF_PREFIX = "!offer:" +REQUEST_LIMITS_COLUMN = "requestLimits" +REQUEST_LIMIT_METRICS = ("batchRequests", "blockSpan") +REQUEST_LIMIT_TRANSPORTS = ("http", "websocket") + SDK_TBD_FIELDS = ( "latestKnownVersion", "latestKnownReleaseDate", @@ -530,6 +534,71 @@ def resolve_offers( return resolved +def validate_request_limits(items: list, context: str) -> list[str]: + """Validate resolved requestLimits arrays for apis rows (DBIP #3724). + + Rules: + - blank/None is valid (unverified); + - each entry must be an object with exactly the keys + method, transport, metric, maximum, sourceUrl; + - transport must be http|websocket and metric batchRequests|blockSpan; + - maximum must be a positive integer (no 0 sentinel, no "unlimited"); + - method and sourceUrl must be non-empty strings, sourceUrl http(s); + - (method, transport, metric) must be unique within one row. + """ + errors = [] + for idx, item in enumerate(items): + if not isinstance(item, dict): + continue + value = item.get(REQUEST_LIMITS_COLUMN) + if value is None: + continue + slug = item.get("slug") or f"row {idx + 2}" + label = f"{context}: apis '{slug}': {REQUEST_LIMITS_COLUMN}" + if not isinstance(value, list): + errors.append(f"{label} must be a JSON array, got {type(value).__name__}") + continue + seen = set() + for pos, entry in enumerate(value): + where = f"{label}[{pos}]" + if not isinstance(entry, dict): + errors.append(f"{where} must be an object, got {type(entry).__name__}") + continue + expected_keys = {"method", "transport", "metric", "maximum", "sourceUrl"} + keys = set(entry.keys()) + if keys != expected_keys: + missing = sorted(expected_keys - keys) + extra = sorted(keys - expected_keys) + errors.append( + f"{where} keys mismatch (missing: {missing or 'none'}, unexpected: {extra or 'none'})" + ) + continue + method = entry["method"] + if not isinstance(method, str) or not method.strip(): + errors.append(f"{where}.method must be a non-empty string") + transport = entry["transport"] + if transport not in REQUEST_LIMIT_TRANSPORTS: + errors.append(f"{where}.transport must be one of {list(REQUEST_LIMIT_TRANSPORTS)}, got {transport!r}") + metric = entry["metric"] + if metric not in REQUEST_LIMIT_METRICS: + errors.append(f"{where}.metric must be one of {list(REQUEST_LIMIT_METRICS)}, got {metric!r}") + maximum = entry["maximum"] + if not isinstance(maximum, int) or isinstance(maximum, bool) or maximum < 1: + errors.append(f"{where}.maximum must be a positive integer, got {maximum!r}") + source = entry["sourceUrl"] + if ( + not isinstance(source, str) + or not source.strip() + or not source.lower().startswith(("http://", "https://")) + ): + errors.append(f"{where}.sourceUrl must be an http(s) URL") + key = (method, transport, metric) + if key in seen: + errors.append(f"{where} duplicates (method, transport, metric) = {key}") + seen.add(key) + return errors + + def get_schema_version( schema_path: str = "schema.json", fallback: str = "1.0.0" ) -> str: @@ -700,6 +769,15 @@ def main(): column_meta = load_json_file("meta/columns.json") offers_by_category = load_categories_from_folder("references/offers") + canonical_request_limit_errors = validate_request_limits( + offers_by_category.get("apis", []), context="canonical offers" + ) + if canonical_request_limit_errors: + print(f"Validation errors for {REQUEST_LIMITS_COLUMN} in canonical offers:") + for e in canonical_request_limit_errors: + print(e) + exit(1) + # Global listings (apply to every network) global_listings = load_categories_from_folder(all_networks_dir) global_listings_categories = list_categories(folder=all_networks_dir) @@ -773,6 +851,15 @@ def main(): ensure_sdks_tbd_fields(result) + request_limit_errors = validate_request_limits( + result.get("apis", []), context=f"network '{network_name}'" + ) + if request_limit_errors: + print(f"Validation errors for {REQUEST_LIMITS_COLUMN} in network '{network_name}':") + for e in request_limit_errors: + print(e) + exit(1) + result["columns"] = get_column_order( base_categories=list_categories(network_dir), extra_categories=global_listings_categories, diff --git a/tools/schema.json b/tools/schema.json index 345bae747d..96492b698d 100644 --- a/tools/schema.json +++ b/tools/schema.json @@ -98,6 +98,21 @@ "type": "string", "examples": ["Pruned", "Archive"] }, + "requestLimits": { + "type": ["array", "null"], + "items": { + "type": "object", + "additionalProperties": false, + "required": ["method", "transport", "metric", "maximum", "sourceUrl"], + "properties": { + "method": { "type": "string", "minLength": 1 }, + "transport": { "type": "string", "enum": ["http", "websocket"] }, + "metric": { "type": "string", "enum": ["batchRequests", "blockSpan"] }, + "maximum": { "type": "integer", "minimum": 1 }, + "sourceUrl": { "type": "string", "minLength": 1 } + } + } + }, "tag": { "type": ["array", "null"], "items": { "type": "string" } } }, "required": [