From d22dbb0ef93ee2b531c0523e20f084b0ce463ea5 Mon Sep 17 00:00:00 2001 From: Hamza Merzic Date: Thu, 16 Jul 2026 13:11:52 +0000 Subject: [PATCH 1/3] Add confined reads from shared Git history --- ARCHITECTURE.md | 2 +- backend/app/routes/storage.py | 117 ++++++++++++++++++++++++++++++++++ backend/tests/test_storage.py | 107 +++++++++++++++++++++++++++++++ 3 files changed, 225 insertions(+), 1 deletion(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 42ce9ff71..a332bfcc6 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -189,7 +189,7 @@ Each module exposes a `router`; registration is in `routes/__init__.py`. | `chats.py` | Chat CRUD + reversible soft-delete with recovery; the chat-load serializer drops tool outputs >4KB to an `output_truncated`/`output_full_len` marker (read-side only — the stored message keeps the full text; blocks ≤4KB or without a message `ts` stay inline), lazy-fetched by `ToolBlock` on expand via `GET /{id}/tool-output?ts=&i=`; also `GET /{id}/agent-context` — read-only inspection of the assembled prompt (system prompt + injected memory / app-context / compaction blocks) | | `chats_stream.py` | `POST /messages` (starts a turn, returns 202) + `GET /stream` (SSE) | | `chat_logs.py` | Gated, redacted chat-log read API for mini-apps | -| `storage.py` | Per-app and shared file storage | +| `storage.py` | Per-app and shared file storage, plus confined immutable blob reads from full commits reachable on a shared repository's `main` branch (`GET /api/storage/shared-git/{repo}?revision=&file=`). The Git route applies the same Memory capability gate, rejects traversal/symlinks/submodules, and never reads the mutable worktree. | | `secrets.py` | Bounded encrypted secret storage scoped to an app; an app can write/delete/check its own values, while only the owner or owner-scoped agent can decrypt them; no cross-app access or listing surface | | `fs.py` | Owner-facing filesystem + git oversight API | | `uploads.py` | Per-chat file upload management | diff --git a/backend/app/routes/storage.py b/backend/app/routes/storage.py index 0d61f214c..b618d53fb 100644 --- a/backend/app/routes/storage.py +++ b/backend/app/routes/storage.py @@ -39,6 +39,7 @@ import os import re import shutil +import subprocess from pathlib import Path from fastapi import APIRouter, Depends, HTTPException, Request, Response @@ -75,6 +76,8 @@ _log = logging.getLogger(__name__) _SAFE_RE = re.compile(r"^[\w.\-\/]+$") +_GIT_COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +_GIT_BLOB_READ_MAX = 8 * 1024 * 1024 _LEVELS = {"none": 0, "read": 1, "write": 2} @@ -471,6 +474,31 @@ def _serve_file(file_path: Path, stored_mime: str | None = None): return PlainTextResponse(text, media_type=(mime or "text/plain")) +def _git_read_env() -> dict[str, str]: + """Minimal deterministic environment for read-only Git object commands.""" + return { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": "/dev/null", + "GIT_NO_REPLACE_OBJECTS": "1", + "HOME": "/nonexistent", + } + + +def _git_read( + repo: Path, *args: str, timeout: int = 10, +) -> subprocess.CompletedProcess: + return subprocess.run( + [ + "git", "--no-pager", f"--git-dir={repo / '.git'}", + f"--work-tree={repo}", *args, + ], + env=_git_read_env(), capture_output=True, timeout=timeout, + ) + + def _is_envelope(body) -> bool: """True iff body is the legacy `{"content": ""}` shape.""" return ( @@ -845,6 +873,95 @@ async def delete_app_file( return Response(status_code=204) +@router.get("/shared-git/{repo:path}") +def read_shared_git_file( + repo: str, + revision: str, + file: str, + principal: Principal = Depends(get_principal), + db: Session = Depends(get_db), +): + """Read one regular file from a pinned commit in a shared-data Git repo. + + This is a generic immutable-read surface, not a Memory-specific router. The + Memory namespace keeps its ordinary live capability gate. The repository and + file are confined below ``/data/shared``; revisions must be full commit SHAs + reachable from ``refs/heads/main``; symlinks, submodules, replacement refs, + hooks and external Git configuration are never consulted. + """ + _require_shared_memory_read(f"{repo}/{file}", principal, db) + if not _GIT_COMMIT_RE.fullmatch(revision): + raise HTTPException(status_code=400, detail="Invalid Git revision.") + if ( + not file + or not _SAFE_RE.fullmatch(file) + or ".." in Path(file).parts + or any(part.startswith(".") for part in Path(file).parts) + ): + raise HTTPException(status_code=400, detail="Invalid Git file path.") + base = Path(get_settings().data_dir) / "shared" + repo_path = _resolve(base, repo) + git_dir = repo_path / ".git" + if ( + repo_path.is_symlink() + or not repo_path.is_dir() + or git_dir.is_symlink() + or not git_dir.is_dir() + ): + raise HTTPException(status_code=404, detail="Git repository not found.") + try: + reachable = _git_read( + repo_path, "merge-base", "--is-ancestor", revision, "refs/heads/main", + ) + if reachable.returncode != 0: + raise HTTPException(status_code=404, detail="Git revision not found.") + entry = _git_read( + repo_path, "ls-tree", "-z", "--full-tree", revision, "--", file, + ) + except (OSError, subprocess.TimeoutExpired): + raise HTTPException(status_code=503, detail="Git storage unavailable.") + if entry.returncode != 0 or not entry.stdout.endswith(b"\0"): + raise HTTPException(status_code=404, detail="Git file not found.") + record = entry.stdout[:-1] + try: + metadata, raw_path = record.split(b"\t", 1) + mode, object_type, object_sha = metadata.decode("ascii").split(" ") + listed_path = raw_path.decode("utf-8") + except (ValueError, UnicodeError): + raise HTTPException(status_code=404, detail="Git file not found.") + if ( + listed_path != file + or mode not in ("100644", "100755") + or object_type != "blob" + or not _GIT_COMMIT_RE.fullmatch(object_sha) + ): + raise HTTPException(status_code=404, detail="Git file not found.") + try: + sized = _git_read(repo_path, "cat-file", "-s", object_sha) + size = int(sized.stdout.strip()) if sized.returncode == 0 else -1 + except (OSError, ValueError, subprocess.TimeoutExpired): + raise HTTPException(status_code=503, detail="Git storage unavailable.") + if size < 0: + raise HTTPException(status_code=404, detail="Git file not found.") + # Unlike an ordinary storage read, ``git cat-file`` is captured from a + # subprocess rather than streamed by FileResponse. Keep that allocation + # comfortably below the general 50 MiB write limit; Memory graphs and notes + # are already bounded far below this ceiling. + if size > _GIT_BLOB_READ_MAX: + raise HTTPException(status_code=413, detail="Stored Git object too large.") + try: + blob = _git_read(repo_path, "cat-file", "blob", object_sha) + except (OSError, subprocess.TimeoutExpired): + raise HTTPException(status_code=503, detail="Git storage unavailable.") + if blob.returncode != 0 or len(blob.stdout) != size: + raise HTTPException(status_code=503, detail="Git storage unavailable.") + media_type = mimetypes.guess_type(file)[0] or "text/plain" + response = Response(content=blob.stdout, media_type=media_type) + response.headers["ETag"] = f'"{revision}-{object_sha}"' + response.headers["Cache-Control"] = "private, max-age=31536000, immutable" + return response + + @router.get("/shared/{path:path}") def read_shared_file( path: str, diff --git a/backend/tests/test_storage.py b/backend/tests/test_storage.py index aa6f2e5c4..50af2ad48 100644 --- a/backend/tests/test_storage.py +++ b/backend/tests/test_storage.py @@ -2,6 +2,7 @@ import json from pathlib import Path +import subprocess import pytest @@ -189,6 +190,112 @@ def test_shared_memory_reads_require_live_declared_contract( ).status_code == 200 +def _memory_git_repo() -> tuple[Path, str]: + repo = Path(get_settings().data_dir) / "shared" / "memory" / "repository" + repo.mkdir(parents=True) + subprocess.run( + ["git", "init", "-b", "main", str(repo)], check=True, capture_output=True, + ) + subprocess.run( + ["git", "-C", str(repo), "config", "user.name", "Memory"], check=True, + ) + subprocess.run( + ["git", "-C", str(repo), "config", "user.email", "memory@mobius.local"], + check=True, + ) + (repo / "graph.json").write_text('{"version":1}\n', encoding="utf-8") + (repo / "notes").mkdir() + (repo / "notes" / "fact.md").write_text("old fact\n", encoding="utf-8") + subprocess.run( + ["git", "-C", str(repo), "add", "graph.json", "notes"], check=True, + ) + subprocess.run( + ["git", "-C", str(repo), "commit", "-m", "initial"], + check=True, capture_output=True, + ) + revision = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], check=True, + capture_output=True, text=True, + ).stdout.strip() + return repo, revision + + +def test_shared_git_read_is_commit_pinned_and_rejects_symlinks(client, auth): + repo, first = _memory_git_repo() + # A dirty working tree cannot affect a commit-addressed read. + (repo / "notes" / "fact.md").write_text( + "unpublished fact\n", encoding="utf-8", + ) + response = client.get( + "/api/storage/shared-git/memory/repository", + params={"revision": first, "file": "notes/fact.md"}, + headers=auth, + ) + assert response.status_code == 200, response.text + assert response.text == "old fact\n" + assert response.headers["cache-control"].endswith("immutable") + + (repo / "notes" / "link.md").symlink_to("fact.md") + subprocess.run(["git", "-C", str(repo), "add", "notes"], check=True) + subprocess.run( + ["git", "-C", str(repo), "commit", "-m", "link"], + check=True, capture_output=True, + ) + second = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], check=True, + capture_output=True, text=True, + ).stdout.strip() + blocked = client.get( + "/api/storage/shared-git/memory/repository", + params={"revision": second, "file": "notes/link.md"}, + headers=auth, + ) + assert blocked.status_code == 404 + traversal = client.get( + "/api/storage/shared-git/memory/repository", + params={"revision": second, "file": "../.git/config"}, + headers=auth, + ) + assert traversal.status_code == 400 + + +def test_shared_git_memory_read_requires_live_declared_contract( + client, owner_token, db, +): + _repo, revision = _memory_git_repo() + app_id = _make_app(client, owner_token) + token = client.post( + "/api/auth/app-token", + json={"app_id": app_id}, + headers={"Authorization": f"Bearer {owner_token}"}, + ).json()["token"] + app_auth = {"Authorization": f"Bearer {token}"} + url = "/api/storage/shared-git/memory/repository" + params = {"revision": revision, "file": "graph.json"} + + assert client.get(url, params=params, headers=app_auth).status_code == 403 + app = db.query(models.App).filter(models.App.id == app_id).one() + app.capability_contract = {"data": {"shared_memory": "read"}} + db.commit() + + allowed = client.get(url, params=params, headers=app_auth) + assert allowed.status_code == 200 + assert allowed.json() == {"version": 1} + + +def test_shared_git_read_caps_buffered_blob(client, auth, monkeypatch): + repo, revision = _memory_git_repo() + monkeypatch.setattr(storage_routes, "_GIT_BLOB_READ_MAX", 4) + + response = client.get( + "/api/storage/shared-git/memory/repository", + params={"revision": revision, "file": "graph.json"}, + headers=auth, + ) + + assert response.status_code == 413 + + def test_put_text_accepts_non_json_content_type(client, auth, owner_token): app_id = _make_app(client, owner_token) From d6babb06c119680892479eb696d329c2ec9f047c Mon Sep 17 00:00:00 2001 From: Hamza Merzic Date: Thu, 16 Jul 2026 14:11:56 +0000 Subject: [PATCH 2/3] test: harden shared Git read boundaries --- backend/tests/test_storage.py | 54 +++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/backend/tests/test_storage.py b/backend/tests/test_storage.py index 50af2ad48..91d17dafe 100644 --- a/backend/tests/test_storage.py +++ b/backend/tests/test_storage.py @@ -296,6 +296,60 @@ def test_shared_git_read_caps_buffered_blob(client, auth, monkeypatch): assert response.status_code == 413 +def test_shared_git_read_requires_full_reachable_commit_and_regular_blob( + client, auth, +): + repo, revision = _memory_git_repo() + url = "/api/storage/shared-git/memory/repository" + + abbreviated = client.get( + url, + params={"revision": revision[:12], "file": "graph.json"}, + headers=auth, + ) + assert abbreviated.status_code == 400 + + tree = subprocess.run( + ["git", "-C", str(repo), "rev-parse", f"{revision}^{{tree}}"], + check=True, capture_output=True, text=True, + ).stdout.strip() + unreachable = subprocess.run( + ["git", "-C", str(repo), "commit-tree", tree, "-m", "unreachable"], + check=True, capture_output=True, text=True, + ).stdout.strip() + hidden = client.get( + url, + params={"revision": unreachable, "file": "graph.json"}, + headers=auth, + ) + assert hidden.status_code == 404 + + subprocess.run( + [ + "git", "-C", str(repo), "update-index", "--add", "--cacheinfo", + f"160000,{revision},notes/nested-repository", + ], + check=True, + ) + subprocess.run( + ["git", "-C", str(repo), "commit", "-m", "gitlink"], + check=True, capture_output=True, + ) + gitlink_revision = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], + check=True, capture_output=True, text=True, + ).stdout.strip() + gitlink = client.get( + url, + params={ + "revision": gitlink_revision, + "file": "notes/nested-repository", + }, + headers=auth, + ) + assert gitlink.status_code == 404 + + def test_put_text_accepts_non_json_content_type(client, auth, owner_token): app_id = _make_app(client, owner_token) From de7cf0fd8d5ec3671f55e0eb12c07109ad4a126b Mon Sep 17 00:00:00 2001 From: Hamza Merzic Date: Thu, 16 Jul 2026 14:11:56 +0000 Subject: [PATCH 3/3] ci: validate Caddyfile with placeholder origins --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6bb538e0f..6ecad1039 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,6 +63,9 @@ jobs: - name: Validate bundled Caddy policy run: >- docker run --rm + -e DOMAIN=localhost + -e FRONTEND_ORIGIN=http://localhost + -e MOBIUS_SERVICE_TANDOOR_ORIGIN=http://tandoor.localhost -v "$PWD/Caddyfile:/etc/caddy/Caddyfile:ro" caddy:2.9-alpine caddy validate --config /etc/caddy/Caddyfile