From 7ac93d629dac072a303f39f8ebd980ba262f7f08 Mon Sep 17 00:00:00 2001 From: Alexey Medvedev Date: Thu, 13 Mar 2025 10:57:14 +0300 Subject: [PATCH 1/4] fix chaotic: support relative paths in ref: --- chaotic/chaotic/front/ref_resolver.py | 55 ++++++++++++++++++--------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/chaotic/chaotic/front/ref_resolver.py b/chaotic/chaotic/front/ref_resolver.py index 8072e6c0813a..9d3188cbe544 100644 --- a/chaotic/chaotic/front/ref_resolver.py +++ b/chaotic/chaotic/front/ref_resolver.py @@ -1,3 +1,4 @@ +import os import collections from typing import Any from typing import Dict @@ -38,6 +39,18 @@ def do_node(node: str): return sorted_nodes +def normalize_ref(ref: str) -> str: + """ + Normalizes a link by converting relative paths to canonical form. + For example, "../test.yaml#/definitions/test" becomes "test.yaml#/definitions/test". + """ + if '#/' in ref: + file_path, fragment = ref.split('#', 1) + file_path = os.path.normpath(file_path) + return file_path + '#' + fragment + return os.path.normpath(ref) + + class RefResolver: def sort_schemas( self, @@ -53,10 +66,16 @@ def sort_schemas( nodes = set() name = '' + # Let's build a map: normalized value of $ref -> original value of the schema key + norm_to_orig = {} + for key in schemas.schemas.keys(): + norm_to_orig[normalize_ref(key)] = key + def visitor( local_schema: types.Schema, parent: Optional[types.Schema], ) -> None: + nonlocal name if not isinstance(local_schema, types.Ref): return @@ -68,22 +87,23 @@ def visitor( if cur_node.indirect: indirect = True - if cur_node.ref not in schemas.schemas: - ref = external_schemas.schemas.get(cur_node.ref) + norm_ref = normalize_ref(cur_node.ref) + if norm_ref not in norm_to_orig: + ref = external_schemas.schemas.get(norm_ref) if ref: cur_node = ref is_external = True else: - known = '\n'.join([f'- {v}' for v in schemas.schemas.keys()]) - known += '\n'.join([f'- {v}' for v in external_schemas.schemas.keys()]) + known = '\n'.join([f'- {v}' for v in norm_to_orig.keys()]) + known += '\n' + '\n'.join([f'- {v}' for v in external_schemas.schemas.keys()]) raise Exception( - f'$ref to unknown type "{cur_node.ref}", known refs:\n{known}', + f'$ref to unknown type "{norm_ref}", known refs:\n{known}', ) else: - cur_node = schemas.schemas[cur_node.ref] + orig = norm_to_orig[norm_ref] + cur_node = schemas.schemas[orig] if cur_node in seen: - # cycle is detected - # an exception will be raised later in sort_dfs() + # cycle is detected; an exception will be raised later in sort_dfs() break seen.add(cur_node) local_schema.schema = cur_node @@ -91,7 +111,7 @@ def visitor( local_schema.indirect = indirect if isinstance(parent, types.Array): - if name == local_schema.ref: + if name == normalize_ref(local_schema.ref): if indirect: raise error.BaseError( full_filepath=local_schema.source_location().filepath, @@ -111,14 +131,13 @@ def visitor( ) if not indirect: if not is_external: - # print(f'add {name} -> {local_schema.ref}') - edges[name].append(local_schema.ref) + edges[name].append(normalize_ref(local_schema.ref)) else: # skip indirect link pass - # TODO: forbid non-schemas/ $refs - for name, schema_item in schemas.schemas.items(): + for key, schema_item in schemas.schemas.items(): + name = normalize_ref(key) visitor(schema_item, None) schema_item.visit_children(visitor) nodes.add(name) @@ -127,7 +146,8 @@ def visitor( sorted_schemas = types.ResolvedSchemas(schemas={}) for node in sorted_nodes: - sorted_schemas.schemas[node] = schemas.schemas[node] + orig = norm_to_orig[node] + sorted_schemas.schemas[node] = schemas.schemas[orig] return sorted_schemas @classmethod @@ -162,13 +182,14 @@ def sort_json_types( edges = collections.defaultdict(list) for name, value in types.items(): - nodes.append(name.rstrip('/')) + norm_name = normalize_ref(name.rstrip('/')) + nodes.append(norm_name) refs = self._search_refs(value, inside_items=False) for ref in refs: if ref.startswith('#/'): - edges[name.rstrip('/')].append(erase_path_prefix + ref[1:]) + edges[norm_name].append(erase_path_prefix + ref[1:]) sorted_nodes = sort_dfs(set(nodes), edges) - return {key + '/': types[key + '/'] for key in sorted_nodes} + return {key + '/': types[normalize_ref(key) + '/'] for key in sorted_nodes} From e841a5ae16587dc2c61f7afae1fb3a88977cc2c1 Mon Sep 17 00:00:00 2001 From: Alexey Medvedev Date: Thu, 13 Mar 2025 10:57:56 +0300 Subject: [PATCH 2/4] feat chaotic: tests for relative paths in ref --- .../schemas/object/external_ref_with_relative_path.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 chaotic/integration_tests/schemas/object/external_ref_with_relative_path.yaml diff --git a/chaotic/integration_tests/schemas/object/external_ref_with_relative_path.yaml b/chaotic/integration_tests/schemas/object/external_ref_with_relative_path.yaml new file mode 100644 index 000000000000..c124df3b1029 --- /dev/null +++ b/chaotic/integration_tests/schemas/object/external_ref_with_relative_path.yaml @@ -0,0 +1,7 @@ +definitions: + ObjectWithExternalRefRelativePath: + type: object + properties: + empty: + $ref: '../object_empty.yaml#/definitions/ObjectEmpty' + additionalProperties: false \ No newline at end of file From ca879fa5de55ed7482bc924783832002a499ca28 Mon Sep 17 00:00:00 2001 From: Alexey Medvedev Date: Thu, 13 Mar 2025 12:49:56 +0300 Subject: [PATCH 3/4] fix chaotic: error with ref: # --- chaotic/chaotic/front/ref_resolver.py | 62 +++++++++++++-------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/chaotic/chaotic/front/ref_resolver.py b/chaotic/chaotic/front/ref_resolver.py index 9d3188cbe544..be600655b6c4 100644 --- a/chaotic/chaotic/front/ref_resolver.py +++ b/chaotic/chaotic/front/ref_resolver.py @@ -1,10 +1,6 @@ import os import collections -from typing import Any -from typing import Dict -from typing import List -from typing import Optional -from typing import Set +from typing import Any, Dict, List, Optional, Set from chaotic import error from chaotic.front import types @@ -39,14 +35,22 @@ def do_node(node: str): return sorted_nodes -def normalize_ref(ref: str) -> str: +def normalize_ref(ref: str, base: str = None) -> str: """ - Normalizes a link by converting relative paths to canonical form. - For example, "../test.yaml#/definitions/test" becomes "test.yaml#/definitions/test". + Normalizes a link, converting relative paths to canonical form. + If the link contains an anchor ("#/"): + - If the part before '#' is empty and base is specified, then base is used. + - Otherwise, the part of the path is normalized via os.path.normpath. + Examples: + "../role.yaml#/definitions/role" -> "role.yaml#/definitions/role" + "#/definitions/type" (with base="vfull") -> "vfull#/definitions/type" """ if '#/' in ref: file_path, fragment = ref.split('#', 1) - file_path = os.path.normpath(file_path) + if not file_path and base: + file_path = base + else: + file_path = os.path.normpath(file_path) return file_path + '#' + fragment return os.path.normpath(ref) @@ -66,16 +70,10 @@ def sort_schemas( nodes = set() name = '' - # Let's build a map: normalized value of $ref -> original value of the schema key - norm_to_orig = {} - for key in schemas.schemas.keys(): - norm_to_orig[normalize_ref(key)] = key - def visitor( local_schema: types.Schema, parent: Optional[types.Schema], ) -> None: - nonlocal name if not isinstance(local_schema, types.Ref): return @@ -87,23 +85,23 @@ def visitor( if cur_node.indirect: indirect = True - norm_ref = normalize_ref(cur_node.ref) - if norm_ref not in norm_to_orig: + norm_ref = normalize_ref(cur_node.ref, base=name.split('#')[0] if cur_node.ref.startswith('#') else None) + if norm_ref not in schemas.schemas: ref = external_schemas.schemas.get(norm_ref) if ref: cur_node = ref is_external = True else: - known = '\n'.join([f'- {v}' for v in norm_to_orig.keys()]) - known += '\n' + '\n'.join([f'- {v}' for v in external_schemas.schemas.keys()]) + known = '\n'.join([f'- {v}' for v in schemas.schemas.keys()]) + known += '\n'.join([f'- {v}' for v in external_schemas.schemas.keys()]) raise Exception( f'$ref to unknown type "{norm_ref}", known refs:\n{known}', ) else: - orig = norm_to_orig[norm_ref] - cur_node = schemas.schemas[orig] + cur_node = schemas.schemas[norm_ref] if cur_node in seen: - # cycle is detected; an exception will be raised later in sort_dfs() + # cycle is detected + # an exception will be raised later in sort_dfs() break seen.add(cur_node) local_schema.schema = cur_node @@ -111,7 +109,7 @@ def visitor( local_schema.indirect = indirect if isinstance(parent, types.Array): - if name == normalize_ref(local_schema.ref): + if name == normalize_ref(local_schema.ref, base=name.split('#')[0] if local_schema.ref.startswith('#') else None): if indirect: raise error.BaseError( full_filepath=local_schema.source_location().filepath, @@ -131,13 +129,15 @@ def visitor( ) if not indirect: if not is_external: - edges[name].append(normalize_ref(local_schema.ref)) + if local_schema.ref.startswith('#'): + edges[name].append(normalize_ref(local_schema.ref, base=name.split('#')[0])) + else: + edges[name].append(normalize_ref(local_schema.ref)) else: # skip indirect link pass - for key, schema_item in schemas.schemas.items(): - name = normalize_ref(key) + for name, schema_item in schemas.schemas.items(): visitor(schema_item, None) schema_item.visit_children(visitor) nodes.add(name) @@ -146,8 +146,7 @@ def visitor( sorted_schemas = types.ResolvedSchemas(schemas={}) for node in sorted_nodes: - orig = norm_to_orig[node] - sorted_schemas.schemas[node] = schemas.schemas[orig] + sorted_schemas.schemas[node] = schemas.schemas[node] return sorted_schemas @classmethod @@ -182,14 +181,13 @@ def sort_json_types( edges = collections.defaultdict(list) for name, value in types.items(): - norm_name = normalize_ref(name.rstrip('/')) - nodes.append(norm_name) + nodes.append(name.rstrip('/')) refs = self._search_refs(value, inside_items=False) for ref in refs: if ref.startswith('#/'): - edges[norm_name].append(erase_path_prefix + ref[1:]) + edges[name.rstrip('/')].append(erase_path_prefix + ref[1:]) sorted_nodes = sort_dfs(set(nodes), edges) - return {key + '/': types[normalize_ref(key) + '/'] for key in sorted_nodes} + return {key + '/': types[key + '/'] for key in sorted_nodes} From 71214c8873179975959a14c4cf4525ce26a4b144 Mon Sep 17 00:00:00 2001 From: Alexey Medvedev Date: Thu, 13 Mar 2025 17:08:18 +0300 Subject: [PATCH 4/4] feat chaotic: tests for relative path --- chaotic/tests/front/test_ref.py | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/chaotic/tests/front/test_ref.py b/chaotic/tests/front/test_ref.py index 1ab19cadc95e..6fd60752e6ca 100644 --- a/chaotic/tests/front/test_ref.py +++ b/chaotic/tests/front/test_ref.py @@ -201,3 +201,47 @@ def test_cycle(): assert str(exc) == '$ref cycle: vfull#/definitions/type1, vfull#/definitions/type2' else: assert False + +def test_relative_path(): + config = ParserConfig(erase_prefix='') + schemas = [] + + parser = SchemaParser( + config=config, + full_filepath='folder/full', + full_vfilepath='folder/vfull', + ) + parser.parse_schema('/definitions/vfull', {'type': 'integer'}) + schemas.append(parser.parsed_schemas()) + + parser = SchemaParser( + config=config, + full_filepath='folder/obj/full2', + full_vfilepath='folder/obj/vfull2', + ) + parser.parse_schema('/definitions/vfull2', { + 'type': 'object', + 'properties': { + 'test': {'$ref': '../vfull#/definitions/vfull'} + }, + 'additionalProperties': False, + }) + schemas.append(parser.parsed_schemas()) + + rr = ref_resolver.RefResolver() + parsed_schemas = rr.sort_schemas(types.ParsedSchemas.merge(schemas)) + + assert parsed_schemas.schemas == { + 'folder/vfull#/definitions/vfull': Integer(), + 'folder/obj/vfull2#/definitions/vfull2': SchemaObject( + properties={ + 'test': Ref( + ref='folder/obj/../vfull#/definitions/vfull', + schema=Integer(), + indirect=False, + self_ref=False, + ) + }, + additionalProperties=False, + ), + }