From d966d711d99dbee831bf5aaaf44bca20c01740e2 Mon Sep 17 00:00:00 2001 From: Florian Reinle <226492742+fl0rianr@users.noreply.github.com> Date: Mon, 8 Jun 2026 19:57:31 +0200 Subject: [PATCH 1/2] test: check if code is API breaking --- .github/workflows/api_contract.yml | 26 +++ test/api_contract_manifest.json | 74 +++++++ test/test_api_contract.py | 301 +++++++++++++++++++++++++++++ 3 files changed, 401 insertions(+) create mode 100644 .github/workflows/api_contract.yml create mode 100644 test/api_contract_manifest.json create mode 100644 test/test_api_contract.py diff --git a/.github/workflows/api_contract.yml b/.github/workflows/api_contract.yml new file mode 100644 index 000000000..9bd814a61 --- /dev/null +++ b/.github/workflows/api_contract.yml @@ -0,0 +1,26 @@ +name: API Contract + +on: + push: + branches: ["main"] + pull_request: + merge_group: + +permissions: + contents: read + +jobs: + api-contract: + name: Check public API route contract + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + steps: + - uses: actions/checkout@v5 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Verify public API contract + run: python test/test_api_contract.py diff --git a/test/api_contract_manifest.json b/test/api_contract_manifest.json new file mode 100644 index 000000000..7b7035f29 --- /dev/null +++ b/test/api_contract_manifest.json @@ -0,0 +1,74 @@ +{ + "version": 1, + "description": "Public Lemonade API route contract. CI compares this manifest with the C++ route registrations so accidental method or path removals are caught during review.", + "sources": [ + "src/cpp/server/server.cpp", + "src/cpp/server/ollama_api.cpp", + "src/cpp/server/anthropic_api.cpp", + "src/cpp/server/websocket_server.cpp" + ], + "versioned_prefixes": [ + "/api/v0", + "/api/v1", + "/v0", + "/v1" + ], + "http_routes": [ + {"method": "GET", "path": "/live"}, + {"method": "GET", "path": "/metrics"}, + + {"method": "GET", "versioned_path": "health"}, + {"method": "GET", "versioned_path": "models"}, + {"method": "GET", "versioned_path": "models/{param}"}, + {"method": "GET", "versioned_path": "slots"}, + {"method": "GET", "versioned_path": "pull/variants"}, + {"method": "GET", "versioned_path": "downloads"}, + {"method": "GET", "versioned_path": "stats"}, + {"method": "GET", "versioned_path": "system-info"}, + {"method": "GET", "versioned_path": "system-stats"}, + + {"method": "POST", "versioned_path": "chat/completions"}, + {"method": "POST", "versioned_path": "completions"}, + {"method": "POST", "versioned_path": "embeddings"}, + {"method": "POST", "versioned_path": "reranking"}, + {"method": "POST", "versioned_path": "slots/{param}"}, + {"method": "POST", "versioned_path": "tokenize"}, + {"method": "POST", "versioned_path": "audio/transcriptions"}, + {"method": "POST", "versioned_path": "audio/speech"}, + {"method": "POST", "versioned_path": "images/generations"}, + {"method": "POST", "versioned_path": "images/edits"}, + {"method": "POST", "versioned_path": "images/variations"}, + {"method": "POST", "versioned_path": "images/upscale"}, + {"method": "POST", "versioned_path": "responses"}, + {"method": "POST", "versioned_path": "pull"}, + {"method": "POST", "versioned_path": "downloads/control"}, + {"method": "POST", "versioned_path": "load"}, + {"method": "POST", "versioned_path": "unload"}, + {"method": "POST", "versioned_path": "delete"}, + {"method": "POST", "versioned_path": "params"}, + {"method": "POST", "versioned_path": "install"}, + {"method": "POST", "versioned_path": "uninstall"}, + {"method": "POST", "versioned_path": "log-level"}, + + {"method": "POST", "path": "/api/chat"}, + {"method": "POST", "path": "/api/generate"}, + {"method": "GET", "path": "/api/tags"}, + {"method": "POST", "path": "/api/show"}, + {"method": "DELETE", "path": "/api/delete"}, + {"method": "POST", "path": "/api/pull"}, + {"method": "POST", "path": "/api/embed"}, + {"method": "POST", "path": "/api/embeddings"}, + {"method": "GET", "path": "/api/ps"}, + {"method": "GET", "path": "/api/version"}, + {"method": "POST", "path": "/api/create"}, + {"method": "POST", "path": "/api/copy"}, + {"method": "POST", "path": "/api/push"}, + {"method": "POST", "path": "/api/blobs/{param}"}, + + {"method": "POST", "path": "/v1/messages"} + ], + "websocket_routes": [ + {"method": "WS", "path": "/realtime"}, + {"method": "WS", "path": "/logs/stream"} + ] +} diff --git a/test/test_api_contract.py b/test/test_api_contract.py new file mode 100644 index 000000000..188bf584f --- /dev/null +++ b/test/test_api_contract.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +"""Verify Lemonade's public API route contract against C++ route registrations. + +This is intentionally static and dependency-free. It catches accidental breaking +changes early in CI without building the server or downloading models. When a +public API change is intentional, update test/api_contract_manifest.json in the +same PR so reviewers get a clear contract diff. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Iterable, NamedTuple + + +REPO_ROOT = Path(__file__).resolve().parents[1] +MANIFEST_PATH = REPO_ROOT / "test" / "api_contract_manifest.json" + + +class Route(NamedTuple): + method: str + path: str + + +ROUTE_CALL_RE = re.compile( + r"\b(?:web_server|server)\." + r"(?PGet|Post|Delete|Put|Patch)\(\s*" + r"(?PR\"\(.*?\)\"|\"[^\"]+\")" + r"(?P\s*\+\s*endpoint)?", + re.DOTALL, +) + +REGISTER_CALL_RE = re.compile( + r"\b(?Pregister_get|register_post)\(\s*\"(?P[^\"]+)\"" +) + +HELPER_METHODS = { + "register_get": "GET", + "register_post": "POST", +} + +CPP_TO_HTTP_METHOD = { + "Get": "GET", + "Post": "POST", + "Delete": "DELETE", + "Put": "PUT", + "Patch": "PATCH", +} + + +def strip_cpp_comments(source: str) -> str: + """Remove C/C++ comments while preserving strings and line positions.""" + + out: list[str] = [] + i = 0 + n = len(source) + + while i < n: + # Preserve raw strings used for httplib regex routes, e.g. R"(/v1/foo/(.+))". + if source.startswith('R"(', i): + end = source.find(')"', i + 3) + if end == -1: + out.append(source[i:]) + break + out.append(source[i : end + 2]) + i = end + 2 + continue + + char = source[i] + nxt = source[i + 1] if i + 1 < n else "" + + if char == '"': + out.append(char) + i += 1 + while i < n: + out.append(source[i]) + if source[i] == "\\" and i + 1 < n: + i += 1 + out.append(source[i]) + elif source[i] == '"': + i += 1 + break + i += 1 + continue + + if char == "'": + out.append(char) + i += 1 + while i < n: + out.append(source[i]) + if source[i] == "\\" and i + 1 < n: + i += 1 + out.append(source[i]) + elif source[i] == "'": + i += 1 + break + i += 1 + continue + + if char == "/" and nxt == "/": + i += 2 + while i < n and source[i] != "\n": + i += 1 + if i < n: + out.append("\n") + i += 1 + continue + + if char == "/" and nxt == "*": + i += 2 + while i + 1 < n and not (source[i] == "*" and source[i + 1] == "/"): + if source[i] == "\n": + out.append("\n") + i += 1 + i += 2 if i + 1 < n else 0 + continue + + out.append(char) + i += 1 + + return "".join(out) + + +def load_manifest() -> dict: + with MANIFEST_PATH.open(encoding="utf-8") as f: + return json.load(f) + + +def normalize_path(path: str) -> str: + path = path.strip() + path = re.sub(r"\(\.\+\)", "{param}", path) + path = re.sub(r"\(\\d\+\)", "{param}", path) + path = path.replace("//", "/") + if not path.startswith("/"): + path = "/" + path + return path.rstrip("/") or "/" + + +def decode_cpp_literal(literal: str) -> str: + if literal.startswith('R"(') and literal.endswith(')"'): + return literal[3:-2] + if literal.startswith('"') and literal.endswith('"'): + return literal[1:-1] + raise ValueError(f"unsupported C++ string literal: {literal!r}") + + +def find_helper_body(source: str, helper_name: str) -> str: + start = source.find(f"auto {helper_name}") + if start == -1: + return "" + + brace = source.find("{", start) + if brace == -1: + return "" + + depth = 0 + for index in range(brace, len(source)): + char = source[index] + if char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth == 0: + return source[brace : index + 1] + return "" + + +def extract_helper_prefixes(source: str, helper_name: str, cpp_method: str) -> list[str]: + body = find_helper_body(source, helper_name) + if not body: + return [] + + pattern = re.compile( + rf"\bweb_server\.{cpp_method}\(\s*\"(?P[^\"]*)\"\s*\+\s*endpoint\s*,\s*handler" + ) + prefixes = [] + for match in pattern.finditer(body): + prefixes.append(normalize_path(match.group("prefix"))) + return prefixes + + +def extract_routes_from_source(source: str) -> set[Route]: + source = strip_cpp_comments(source) + routes: set[Route] = set() + + get_prefixes = extract_helper_prefixes(source, "register_get", "Get") + post_prefixes = extract_helper_prefixes(source, "register_post", "Post") + + for match in REGISTER_CALL_RE.finditer(source): + helper = match.group("helper") + endpoint = match.group("endpoint") + method = HELPER_METHODS[helper] + prefixes = get_prefixes if helper == "register_get" else post_prefixes + for prefix in prefixes: + routes.add(Route(method, normalize_path(f"{prefix}/{endpoint}"))) + + for match in ROUTE_CALL_RE.finditer(source): + if match.group("concat"): + # These are helper definitions such as web_server.Get("/v1/" + endpoint, handler). + # The expanded helper calls above are the contract-relevant routes. + continue + method = CPP_TO_HTTP_METHOD[match.group("method")] + path = normalize_path(decode_cpp_literal(match.group("literal"))) + routes.add(Route(method, path)) + + return routes + + +def extract_websocket_routes(source: str) -> set[Route]: + source = strip_cpp_comments(source) + return { + Route("WS", normalize_path(path)) + for path in re.findall(r"path\s*==\s*\"([^\"]+)\"", source) + } + + +def expected_routes(manifest: dict) -> set[Route]: + prefixes = manifest["versioned_prefixes"] + routes: set[Route] = set() + + for route in manifest["http_routes"]: + method = route["method"] + if "path" in route: + routes.add(Route(method, normalize_path(route["path"]))) + continue + suffix = route["versioned_path"].lstrip("/") + for prefix in prefixes: + routes.add(Route(method, normalize_path(f"{prefix}/{suffix}"))) + + for route in manifest["websocket_routes"]: + routes.add(Route(route["method"], normalize_path(route["path"]))) + + return routes + + +def implemented_routes(manifest: dict) -> set[Route]: + routes: set[Route] = set() + missing_sources: list[Path] = [] + + for relative_source in manifest["sources"]: + source_path = REPO_ROOT / relative_source + if not source_path.exists(): + missing_sources.append(source_path) + continue + + source = source_path.read_text(encoding="utf-8") + if relative_source.endswith("websocket_server.cpp"): + routes.update(extract_websocket_routes(source)) + else: + routes.update(extract_routes_from_source(source)) + + if missing_sources: + paths = "\n".join(f" - {path.relative_to(REPO_ROOT)}" for path in missing_sources) + raise FileNotFoundError(f"API contract source file(s) missing:\n{paths}") + + return routes + + +def format_routes(routes: Iterable[Route]) -> str: + return "\n".join(f" - {route.method:6} {route.path}" for route in sorted(routes)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dump-implemented", + action="store_true", + help="Print the route registrations discovered in the C++ sources.", + ) + args = parser.parse_args() + + manifest = load_manifest() + expected = expected_routes(manifest) + actual = implemented_routes(manifest) + + if args.dump_implemented: + print(format_routes(actual)) + return 0 + + missing = expected - actual + if missing: + print("Public API contract check failed: expected route(s) are missing from C++ route registrations.") + print() + print(format_routes(missing)) + print() + print( + "If this API change is intentional, update test/api_contract_manifest.json " + "in the same PR so reviewers can evaluate the contract change." + ) + return 1 + + print(f"Public API contract check passed ({len(expected)} expected routes).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From be80d6fc28e838184fc07d8e5de0bdf3ecd90cb6 Mon Sep 17 00:00:00 2001 From: Florian Reinle <226492742+fl0rianr@users.noreply.github.com> Date: Mon, 22 Jun 2026 15:28:36 +0200 Subject: [PATCH 2/2] test: align API contract with current public routes Update the static API contract check to extract WebSocket routes from WebSocketServer::classify_path(), including version-prefixed websocket paths. Extend the manifest to cover the current public route surface: cloud auth, MCP, and version-prefixed websocket endpoints. --- test/api_contract_manifest.json | 10 ++- test/test_api_contract.py | 109 ++++++++++++++++++++++++++++---- 2 files changed, 104 insertions(+), 15 deletions(-) diff --git a/test/api_contract_manifest.json b/test/api_contract_manifest.json index 7b7035f29..199924327 100644 --- a/test/api_contract_manifest.json +++ b/test/api_contract_manifest.json @@ -5,6 +5,7 @@ "src/cpp/server/server.cpp", "src/cpp/server/ollama_api.cpp", "src/cpp/server/anthropic_api.cpp", + "src/cpp/server/mcp_server.cpp", "src/cpp/server/websocket_server.cpp" ], "versioned_prefixes": [ @@ -49,6 +50,8 @@ {"method": "POST", "versioned_path": "install"}, {"method": "POST", "versioned_path": "uninstall"}, {"method": "POST", "versioned_path": "log-level"}, + {"method": "POST", "versioned_path": "cloud/auth"}, + {"method": "DELETE", "versioned_path": "cloud/auth/{param}"}, {"method": "POST", "path": "/api/chat"}, {"method": "POST", "path": "/api/generate"}, @@ -65,10 +68,13 @@ {"method": "POST", "path": "/api/push"}, {"method": "POST", "path": "/api/blobs/{param}"}, - {"method": "POST", "path": "/v1/messages"} + {"method": "POST", "path": "/v1/messages"}, + {"method": "POST", "path": "/mcp"} ], "websocket_routes": [ {"method": "WS", "path": "/realtime"}, - {"method": "WS", "path": "/logs/stream"} + {"method": "WS", "versioned_path": "realtime"}, + {"method": "WS", "path": "/logs/stream"}, + {"method": "WS", "versioned_path": "logs/stream"} ] } diff --git a/test/test_api_contract.py b/test/test_api_contract.py index 188bf584f..8d3949d19 100644 --- a/test/test_api_contract.py +++ b/test/test_api_contract.py @@ -148,8 +148,33 @@ def decode_cpp_literal(literal: str) -> str: raise ValueError(f"unsupported C++ string literal: {literal!r}") -def find_helper_body(source: str, helper_name: str) -> str: - start = source.find(f"auto {helper_name}") +def skip_cpp_string_or_char(source: str, index: int) -> int: + """Return the index just after a C++ string/char literal starting at index.""" + + if source.startswith('R"(', index): + end = source.find(')"', index + 3) + return len(source) if end == -1 else end + 2 + + quote = source[index] + if quote not in {'"', "'"}: + return index + + index += 1 + while index < len(source): + if source[index] == "\\" and index + 1 < len(source): + index += 2 + continue + if source[index] == quote: + return index + 1 + index += 1 + + return index + + +def find_braced_body(source: str, start_token: str) -> str: + """Find the {...} body that follows start_token, ignoring braces in strings.""" + + start = source.find(start_token) if start == -1: return "" @@ -158,7 +183,12 @@ def find_helper_body(source: str, helper_name: str) -> str: return "" depth = 0 - for index in range(brace, len(source)): + index = brace + while index < len(source): + if source.startswith('R"(', index) or source[index] in {'"', "'"}: + index = skip_cpp_string_or_char(source, index) + continue + char = source[index] if char == "{": depth += 1 @@ -166,9 +196,15 @@ def find_helper_body(source: str, helper_name: str) -> str: depth -= 1 if depth == 0: return source[brace : index + 1] + index += 1 + return "" +def find_helper_body(source: str, helper_name: str) -> str: + return find_braced_body(source, f"auto {helper_name}") + + def extract_helper_prefixes(source: str, helper_name: str, cpp_method: str) -> list[str]: body = find_helper_body(source, helper_name) if not body: @@ -210,11 +246,64 @@ def extract_routes_from_source(source: str) -> set[Route]: return routes +def extract_websocket_prefixes(classify_path_body: str) -> list[str]: + prefix_match = re.search( + r"for\s*\([^:]+:\s*\{(?P[^}]+)\}\)", + classify_path_body, + re.DOTALL, + ) + if not prefix_match: + return [] + + return [ + normalize_path(prefix) + for prefix in re.findall(r'"([^"]+)"', prefix_match.group("prefixes")) + ] + + def extract_websocket_routes(source: str) -> set[Route]: source = strip_cpp_comments(source) + body = find_braced_body(source, "WebSocketServer::classify_path") + + if body: + prefixes = extract_websocket_prefixes(body) + websocket_paths = [ + normalize_path(path) + for path in re.findall(r'\bstripped\s*==\s*"([^"]+)"', body) + ] + + # Fallback for older implementations that compare the raw path directly. + if not websocket_paths: + websocket_paths = [ + normalize_path(path) + for path in re.findall(r'\bpath\s*==\s*"([^"]+)"', body) + ] + else: + prefixes = [] + websocket_paths = [ + normalize_path(path) + for path in re.findall(r'\bpath\s*==\s*"([^"]+)"', source) + ] + + routes: set[Route] = set() + for path in websocket_paths: + routes.add(Route("WS", path)) + for prefix in prefixes: + routes.add(Route("WS", normalize_path(f"{prefix}/{path.lstrip('/')}"))) + + return routes + + +def expand_manifest_route(route: dict, prefixes: Iterable[str]) -> set[Route]: + method = route["method"] + + if "path" in route: + return {Route(method, normalize_path(route["path"]))} + + suffix = route["versioned_path"].lstrip("/") return { - Route("WS", normalize_path(path)) - for path in re.findall(r"path\s*==\s*\"([^\"]+)\"", source) + Route(method, normalize_path(f"{prefix}/{suffix}")) + for prefix in prefixes } @@ -223,16 +312,10 @@ def expected_routes(manifest: dict) -> set[Route]: routes: set[Route] = set() for route in manifest["http_routes"]: - method = route["method"] - if "path" in route: - routes.add(Route(method, normalize_path(route["path"]))) - continue - suffix = route["versioned_path"].lstrip("/") - for prefix in prefixes: - routes.add(Route(method, normalize_path(f"{prefix}/{suffix}"))) + routes.update(expand_manifest_route(route, prefixes)) for route in manifest["websocket_routes"]: - routes.add(Route(route["method"], normalize_path(route["path"]))) + routes.update(expand_manifest_route(route, prefixes)) return routes