From 4f97d8339ee54657e27e4c070b69b3fb11d3156d Mon Sep 17 00:00:00 2001 From: Amir Fathi Date: Mon, 14 Sep 2026 02:58:06 +0000 Subject: [PATCH] fix(config): add !rawfile tag to bypass YAML-sniffing on prompt includes !file parses every included file's content as YAML and returns a dict/list whenever the sub-parse succeeds structurally. A Markdown prompt whose prose happens to parse as YAML (a line ending in ':' followed by a '- ' bullet list) then fails prompt/system_prompt validation, and an unrelated prose edit can flip a working file across that boundary. Add !rawfile, which always returns the file's content verbatim. Extract the shared file-reading and cycle-detection logic into _read_included_file so both tags use it; !rawfile skips the YAML sub-parse and never recurses. Fixes #528 --- docs/workflow-syntax.md | 22 +++++- src/conductor/config/loader.py | 29 ++++++- .../fixtures/file_tag/yaml_shaped.md | 3 + tests/test_config/test_file_tag.py | 76 +++++++++++++++++++ 4 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 tests/test_config/fixtures/file_tag/yaml_shaped.md diff --git a/docs/workflow-syntax.md b/docs/workflow-syntax.md index 2ad2037d..63a9b8d8 100644 --- a/docs/workflow-syntax.md +++ b/docs/workflow-syntax.md @@ -2762,6 +2762,24 @@ The content of the referenced file is handled based on its structure: - **YAML dict or list** — If the file content parses as a YAML mapping or sequence, it is returned as structured data (dict or list). This is useful for output schemas, tool lists, or any structured configuration. - **Scalar or non-YAML** — If the file contains a YAML scalar (e.g., a plain string), is not valid YAML, or is a non-YAML format like Markdown, the raw file content is returned as a string. +This detection applies to the file's *structure*, not its extension: a Markdown prompt whose text happens to parse as a YAML mapping (a line ending in `:` followed by a `- ` bulleted list, for example) is returned as a dict, and a string-typed field like `prompt` or `system_prompt` then rejects it. Because this depends on the file's exact content, an unrelated prose edit can flip a working prompt file across the boundary with no config change. Use `!rawfile` (below) for any file whose content must always stay a string regardless of what it happens to contain. + +### Raw Text Includes (`!rawfile`) + +`!rawfile` reads the referenced file and always returns its content verbatim as a string, skipping the YAML-sniffing `!file` does. Use it for `prompt` and `system_prompt` files so their type can never depend on whether the prose happens to be valid YAML: + +```yaml +agents: + - name: reviewer + model: gpt-4 + system_prompt: !rawfile prompts/system.md + prompt: !rawfile prompts/review.md + routes: + - to: $end +``` + +`!rawfile` does not parse nested `!file`/`!rawfile` tags inside the included file; the content is returned exactly as read. It supports the same path resolution, environment variable resolution, and Jinja include search root as a `!file` tag that happened to return a string. + ### Path Resolution File paths are resolved **relative to the directory containing the YAML file** that uses the `!file` tag, not relative to the current working directory. @@ -2882,9 +2900,9 @@ A comprehensive summary of the analysis results. ### Jinja Includes in Prompt Files -When a prompt or system_prompt is loaded via `!file`, the directory of that file becomes the search root for Jinja template loading. This allows statements like `{% include "_shared.md" %}`, `{% import "_macros.md" as m %}`, and `{% extends "_base.md" %}` to resolve relative to the prompt file's directory rather than the workflow's directory or the current working directory. +When a prompt or system_prompt is loaded via `!file` or `!rawfile`, the directory of that file becomes the search root for Jinja template loading. This allows statements like `{% include "_shared.md" %}`, `{% import "_macros.md" as m %}`, and `{% extends "_base.md" %}` to resolve relative to the prompt file's directory rather than the workflow's directory or the current working directory. -Only `prompt: !file` and `system_prompt: !file` support this behavior. Other fields that use `!file` (such as command, stdin, value, schemas, or tool lists) don't have include loader support. Inline prompts defined as plain strings don't support loader-dependent Jinja tags. If you attempt to use them inline, the system raises a template rendering error suggesting you switch to a file-backed prompt: +Only `prompt`/`system_prompt` loaded via `!file` or `!rawfile` support this behavior. Other fields that use `!file`/`!rawfile` (such as command, stdin, value, schemas, or tool lists) don't have include loader support. Inline prompts defined as plain strings don't support loader-dependent Jinja tags. If you attempt to use them inline, the system raises a template rendering error suggesting you switch to a file-backed prompt: ``` Template rendering failed: loader-dependent Jinja constructs ({% include %}, {% import %}, {% extends %}) require a file-backed prompt via prompt: !file ... diff --git a/src/conductor/config/loader.py b/src/conductor/config/loader.py index a5ad2907..a3d03fc4 100644 --- a/src/conductor/config/loader.py +++ b/src/conductor/config/loader.py @@ -121,8 +121,12 @@ class FileTagConstructor(RoundTripConstructor): _base_dir: Path = Path(".") _file_stack: list[str] = [] - def construct_file_tag(self, node: Any) -> Any: - """Resolve a !file tag by reading and optionally parsing the referenced file.""" + def _read_included_file(self, node: Any) -> tuple[str, Path]: + """Resolve a !file/!rawfile path against the current base dir and read it. + + Raises ConfigurationError on a circular reference, a missing file, or + invalid UTF-8. Returns the file's raw text content and its resolved path. + """ path_str = self.construct_scalar(node) cls = type(self) @@ -139,7 +143,6 @@ def construct_file_tag(self, node: Any) -> Any: suggestion="Remove the circular !file reference.", ) - # Read file content try: content = file_path.read_text(encoding="utf-8") except FileNotFoundError as e: @@ -154,9 +157,16 @@ def construct_file_tag(self, node: Any) -> Any: suggestion="Ensure the file is saved as UTF-8 text.", ) from e + return content, file_path + + def construct_file_tag(self, node: Any) -> Any: + """Resolve a !file tag by reading and optionally parsing the referenced file.""" + content, file_path = self._read_included_file(node) + cls = type(self) + # Try to parse as YAML (with nested !file support) saved_base_dir = cls._base_dir - cls._file_stack.append(file_path_str) + cls._file_stack.append(str(file_path)) try: cls._base_dir = file_path.parent sub_yaml = YAML() @@ -173,7 +183,18 @@ def construct_file_tag(self, node: Any) -> Any: cls._base_dir = saved_base_dir cls._file_stack.pop() + def construct_rawfile_tag(self, node: Any) -> Any: + """Resolve a !rawfile tag: always return the file's content verbatim. + + Unlike !file, this never sniffs the content as YAML, so a prompt file + cannot have its type silently flip between a string and a parsed + mapping/list depending on whether its prose happens to parse as YAML. + """ + content, file_path = self._read_included_file(node) + return FileString(content, source_path=file_path) + FileTagConstructor.add_constructor("!file", FileTagConstructor.construct_file_tag) + FileTagConstructor.add_constructor("!rawfile", FileTagConstructor.construct_rawfile_tag) return FileTagConstructor diff --git a/tests/test_config/fixtures/file_tag/yaml_shaped.md b/tests/test_config/fixtures/file_tag/yaml_shaped.md new file mode 100644 index 00000000..4511c16e --- /dev/null +++ b/tests/test_config/fixtures/file_tag/yaml_shaped.md @@ -0,0 +1,3 @@ +Summarize the changelog, keeping only what affects the triage of a dependency update: +- breaking changes and migration notes; +- security fixes. diff --git a/tests/test_config/test_file_tag.py b/tests/test_config/test_file_tag.py index ae3b316f..c3c38732 100644 --- a/tests/test_config/test_file_tag.py +++ b/tests/test_config/test_file_tag.py @@ -331,3 +331,79 @@ def test_load_string_state_reset_after_error(self) -> None: """ config = loader.load_string(valid_yaml) assert config.workflow.name == "after-error" + + +class TestRawFileTag: + """Tests for the !rawfile tag: always returns verbatim string content.""" + + def test_file_tag_trap_on_yaml_shaped_markdown(self) -> None: + """!file on a Markdown file that parses as YAML rejects a str-typed field. + + Documents the trap !rawfile exists to avoid: yaml_shaped.md's prose is a + valid YAML mapping (a line ending in ':' followed by a '- ' bullet list), + so !file returns a dict where system_prompt (typed str) needs a string. + """ + loader = ConfigLoader() + yaml_content = """\ +workflow: + name: file-tag-trap + entry_point: agent1 + +agents: + - name: agent1 + model: gpt-4 + system_prompt: !file yaml_shaped.md + prompt: "Hello" + routes: + - to: $end +""" + with pytest.raises(ConfigurationError, match="valid string"): + loader.load_string( + yaml_content, + source_path=FIXTURES_DIR / "file_tag_trap.yaml", + ) + + def test_rawfile_bypasses_yaml_sniffing(self) -> None: + """!rawfile on the same YAML-shaped Markdown file stays a string.""" + loader = ConfigLoader() + yaml_content = """\ +workflow: + name: rawfile-test + entry_point: agent1 + +agents: + - name: agent1 + model: gpt-4 + system_prompt: !rawfile yaml_shaped.md + prompt: "Hello" + routes: + - to: $end +""" + config = loader.load_string( + yaml_content, + source_path=FIXTURES_DIR / "rawfile_test.yaml", + ) + assert isinstance(config.agents[0].system_prompt, str) + assert "Summarize the changelog" in config.agents[0].system_prompt + + def test_rawfile_missing_file_raises_configuration_error(self) -> None: + """!rawfile shares !file's missing-file error handling.""" + loader = ConfigLoader() + yaml_content = """\ +workflow: + name: rawfile-missing-test + entry_point: agent1 + +agents: + - name: agent1 + model: gpt-4 + system_prompt: !rawfile nonexistent.md + prompt: "Hello" + routes: + - to: $end +""" + with pytest.raises(ConfigurationError, match="File not found"): + loader.load_string( + yaml_content, + source_path=FIXTURES_DIR / "rawfile_missing_test.yaml", + )