diff --git a/docs/changelog/951.bugfix.rst b/docs/changelog/951.bugfix.rst new file mode 100644 index 000000000..33e078f26 --- /dev/null +++ b/docs/changelog/951.bugfix.rst @@ -0,0 +1,3 @@ +Reject a :class:`RELAX NG ` ```` with no name, a reference cycle that never crosses an +``element``, and an ``interleave`` whose branches share an element name or both match text when the schema compiles, and +validate an ambiguous ``choice`` in bounded memory, instead of crashing, hanging, or exhausting memory. diff --git a/docs/explanation/validation.rst b/docs/explanation/validation.rst index 603eb4942..7cb7173d5 100644 --- a/docs/explanation/validation.rst +++ b/docs/explanation/validation.rst @@ -31,8 +31,11 @@ restriction chain. Namespaces resolve from the in-scope ``xmlns`` declarations, compiles to that algebra, and validation takes the *derivative* of the pattern with respect to each start tag, attribute, text run, and end tag: the pattern that remains after consuming one piece of the document. This is what makes ``interleave`` fall out for free -- the derivative of ``interleave(p1, p2)`` over an element is the choice of advancing -either side -- with no backtracking and no combinatorial blow-up, because smart constructors absorb ``notAllowed`` and -``empty`` to keep the residual pattern small. +either side -- with no backtracking. Smart constructors absorb ``notAllowed`` and ``empty``, the residual patterns are +hash-consed per validation, and ``choice`` drops a branch already present, so an ambiguous grammar stays bounded instead +of doubling the residual on each child. The restrictions the specification places on a schema are enforced when it +compiles: a ```` with no name, a reference cycle that never crosses an ``element``, and an ``interleave`` whose +branches compete for an element name or text are rejected with a :class:`ValueError` rather than reached at validation. **************** Why the C core @@ -42,8 +45,8 @@ The datatype and facet layer is where validation spends its time: every leaf val (is ``2020-13-40`` a date?) and then against its constraining facets (``minInclusive``, ``pattern``, ``length``, ...). Doing that in the extension -- over the code-point buffers the parser already produced, with a compact Thompson-NFA matcher for the ``pattern`` facet -- keeps a schema check close to the cost of the parse it follows, rather than a -second pass in Python. The recursion guards that protect the RELAX NG derivative from schemas whose refs recurse without -an element in between live there too, so an adversarial schema fails cleanly instead of overflowing the stack. +second pass in Python. A RELAX NG schema whose refs recurse without an element in between is rejected when it compiles, +so the derivative never reaches such a grammar and an adversarial schema fails cleanly instead of overflowing the stack. Compilation and validation start with an iterative tree-depth scan. A schema or instance nested 400 levels or deeper raises :class:`RecursionError` before a recursive grammar walk starts, including on small worker-thread stacks. diff --git a/docs/reference/validate.rst b/docs/reference/validate.rst index 519e793d4..d97efcf6b 100644 --- a/docs/reference/validate.rst +++ b/docs/reference/validate.rst @@ -19,6 +19,12 @@ above ``m``. The message names the limit and the offset where the pattern reache the repeat counts, or split the pattern into several ``pattern`` facets to stay within the limits. Matching a value takes time linear in its length. +Compiling a RELAX NG schema raises :class:`ValueError` for a grammar the RELAX NG specification forbids: a ```` +with no ``name`` attribute (section 4.10), a reference cycle whose expansion never passes through an ``element`` +(section 4.19), and an ``interleave`` whose branches can match an element with the same name or can both match text +(section 7.4). The message names the offending ``define`` or construct. A legal but ambiguous ``choice`` or +``interleave`` validates in memory bounded by the schema size rather than growing per child element. + .. autoclass:: XMLSchema :members: :inherited-members: diff --git a/src/turbohtml/_c/validate/relaxng.h b/src/turbohtml/_c/validate/relaxng.h index 87319b3e4..53a450fa1 100644 --- a/src/turbohtml/_c/validate/relaxng.h +++ b/src/turbohtml/_c/validate/relaxng.h @@ -51,6 +51,71 @@ struct pattern { Py_ssize_t value_len; }; +/* Per-validation hash-consing of the derivative patterns. James Clark's paper ("Interning + patterns" / "Avoiding exponential blowup") and the jing/MSV implementations the P1-C2 + audit cites (jing PatternInterner, MSV ExpressionPool) make a structurally identical + pattern share one node so choice can drop a duplicate branch by pointer and the residual + stays bounded on an ambiguous but legal schema. Keyed on (type, p1, p2); P_ONEMORE keys + on (type, p1, NULL). Allocated in the per-call arena and never set while compiling. */ +typedef struct patintern { + pattern **slots; + size_t cap; /* power of two, 0 until first grow */ + size_t count; +} patintern; + +static uint64_t pat_key_hash(int type, pattern *p1, pattern *p2) { + uint64_t hash = UINT64_C(1469598103934665603); + hash = (hash ^ (uint64_t)(unsigned)type) * UINT64_C(1099511628211); + hash = (hash ^ (uint64_t)(uintptr_t)p1) * UINT64_C(1099511628211); + hash = (hash ^ (uint64_t)(uintptr_t)p2) * UINT64_C(1099511628211); + return hash; +} + +static pattern *intern_find(const patintern *table, int type, pattern *p1, pattern *p2) { + if (table->cap == 0) { + return NULL; + } + size_t slot = (size_t)pat_key_hash(type, p1, p2) & (table->cap - 1); + while (table->slots[slot] != NULL) { + pattern *node = table->slots[slot]; + if (node->type == type && node->p1 == p1 && node->p2 == p2) { + return node; + } + slot = (slot + 1) & (table->cap - 1); + } + return NULL; +} + +static void intern_insert(th_schema *schema, patintern *table, pattern *node) { + if (table->cap == 0 || table->count * 2 >= table->cap) { + size_t cap = table->cap ? table->cap * 2 : 64; + pattern **slots = arena_alloc(&schema->mem, cap * sizeof(pattern *)); + if (slots == NULL) { /* GCOVR_EXCL_BR_LINE: arena OOM is unforceable */ + return; /* GCOVR_EXCL_LINE: skipping the insert only loses dedup, not correctness */ + } + memset(slots, 0, cap * sizeof(pattern *)); + for (size_t index = 0; index < table->cap; index++) { + pattern *existing = table->slots[index]; + if (existing == NULL) { + continue; + } + size_t slot = (size_t)pat_key_hash(existing->type, existing->p1, existing->p2) & (cap - 1); + while (slots[slot] != NULL) { + slot = (slot + 1) & (cap - 1); + } + slots[slot] = existing; + } + table->slots = slots; + table->cap = cap; + } + size_t slot = (size_t)pat_key_hash(node->type, node->p1, node->p2) & (table->cap - 1); + while (table->slots[slot] != NULL) { + slot = (slot + 1) & (table->cap - 1); + } + table->slots[slot] = node; + table->count++; +} + static pattern *pat_new(th_schema *schema, int type) { pattern *pattern = arena_alloc(&schema->mem, sizeof(*pattern)); if (pattern == NULL) { /* GCOVR_EXCL_BR_LINE: arena OOM is unforceable */ @@ -66,6 +131,12 @@ static pattern *pat_new(th_schema *schema, int type) { } static pattern *pat_binary(th_schema *schema, int type, pattern *p1, pattern *p2) { + if (schema->intern != NULL) { + pattern *found = intern_find(schema->intern, type, p1, p2); + if (found != NULL) { + return found; + } + } pattern *node = pat_new(schema, type); if (node == NULL) { /* GCOVR_EXCL_BR_LINE: arena OOM is unforceable */ return schema->p_notallowed; /* GCOVR_EXCL_LINE */ @@ -75,9 +146,59 @@ static pattern *pat_binary(th_schema *schema, int type, pattern *p1, pattern *p2 if (type != P_AFTER && p1->nullable >= 0 && p2->nullable >= 0) { node->nullable = type == P_CHOICE ? p1->nullable || p2->nullable : p1->nullable && p2->nullable; } + if (schema->intern != NULL) { + intern_insert(schema, schema->intern, node); + } return node; } +/* The leaves of a canonical choice, a right-leaning chain whose every left child is a + non-choice pattern, the leaves sorted by node address and free of duplicates. */ +static Py_ssize_t choice_leaf_count(const pattern *p) { + Py_ssize_t count = 1; + while (p->type == P_CHOICE) { + count++; + p = p->p2; + } + return count; +} + +static void choice_collect(pattern *p, pattern **out, Py_ssize_t *at) { + while (p->type == P_CHOICE) { + out[(*at)++] = p->p1; + p = p->p2; + } + out[(*at)++] = p; +} + +/* Total order on nodes by address, so a canonical choice has a single shape per leaf set. */ +static int pat_ptr_cmp(const void *left, const void *right) { + pattern *left_pat = *(pattern *const *)left; + pattern *right_pat = *(pattern *const *)right; + return (left_pat > right_pat) - (left_pat < right_pat); +} + +/* Merge the leaf sets of two canonical choices, dropping duplicates (Clark's "eliminate + redundant choices"; jing makeChoice / MSV createChoice), then rebuild the canonical form. */ +static pattern *pat_choice_merge(th_schema *schema, pattern *p1, pattern *p2) { + Py_ssize_t total = choice_leaf_count(p1) + choice_leaf_count(p2); + pattern **leaves = arena_alloc(&schema->mem, (size_t)total * sizeof(pattern *)); + if (leaves == NULL) { /* GCOVR_EXCL_BR_LINE: arena OOM is unforceable */ + return pat_binary(schema, P_CHOICE, p1, p2); /* GCOVR_EXCL_LINE */ + } + Py_ssize_t len = 0; + choice_collect(p1, leaves, &len); + choice_collect(p2, leaves, &len); + qsort(leaves, (size_t)total, sizeof(pattern *), pat_ptr_cmp); + pattern *result = leaves[total - 1]; + for (Py_ssize_t index = total - 2; index >= 0; index--) { + if (leaves[index] != leaves[index + 1]) { /* an adjacent equal is a duplicate branch, dropped */ + result = pat_binary(schema, P_CHOICE, leaves[index], result); + } + } + return result; +} + /* Smart constructors: absorb notAllowed / empty so derivatives stay bounded. */ static pattern *pat_choice(th_schema *schema, pattern *p1, pattern *p2) { if (p1->type == P_NOTALLOWED) { @@ -86,7 +207,16 @@ static pattern *pat_choice(th_schema *schema, pattern *p1, pattern *p2) { if (p2->type == P_NOTALLOWED) { return p1; } - return pat_binary(schema, P_CHOICE, p1, p2); + if (schema->intern == NULL) { /* compiling: build the plain binary; interning is a validation-time concern */ + return pat_binary(schema, P_CHOICE, p1, p2); + } + if (p1->type != P_CHOICE && p2->type != P_CHOICE) { /* hot path: two leaves, order them and drop a duplicate */ + if (p1 == p2) { + return p1; + } + return p1 < p2 ? pat_binary(schema, P_CHOICE, p1, p2) : pat_binary(schema, P_CHOICE, p2, p1); + } + return pat_choice_merge(schema, p1, p2); } static pattern *pat_group(th_schema *schema, pattern *p1, pattern *p2) { @@ -131,12 +261,21 @@ static pattern *pat_onemore(th_schema *schema, pattern *p1) { if (p1->type == P_NOTALLOWED) { return schema->p_notallowed; } + if (schema->intern != NULL) { + pattern *found = intern_find(schema->intern, P_ONEMORE, p1, NULL); + if (found != NULL) { + return found; + } + } pattern *node = pat_new(schema, P_ONEMORE); if (node == NULL) { /* GCOVR_EXCL_BR_LINE: arena OOM is unforceable */ return schema->p_notallowed; /* GCOVR_EXCL_LINE */ } node->p1 = p1; node->nullable = p1->nullable; + if (schema->intern != NULL) { + intern_insert(schema, schema->intern, node); + } return node; } @@ -417,6 +556,7 @@ static int rng_datatype_id(th_schema *schema, th_node *node) { } static pattern *rng_build(th_schema *schema, th_node *node); +static int rng_check_interleave_node(th_schema *schema, th_node *interleave); /* Group the pattern children of a container into a single pattern (Empty when none). */ static pattern *rng_build_children(th_schema *schema, th_node *node, th_node *skip) { @@ -489,6 +629,11 @@ static pattern *rng_build(th_schema *schema, th_node *node) { return combined; } if (is_schema_el(schema, node, RNG_NS, "interleave")) { + /* at compile (intern == NULL) this is the short-form root's own 7.4 check; a grammar's reachable + interleaves are checked by the scan instead, so it is not repeated during lazy validation builds */ + if (schema->intern == NULL && rng_check_interleave_node(schema, node) < 0) { + return schema->p_notallowed; + } pattern *combined = schema->p_empty; for (th_node *child = node->first_child; child != NULL; child = child->next_sibling) { if (child->type == TH_NODE_ELEMENT) { @@ -544,6 +689,11 @@ static pattern *rng_build(th_schema *schema, th_node *node) { } if (is_schema_el(schema, node, RNG_NS, "ref")) { const th_node_attr *name = attr_exact(tree, node, "name", 4); + if (name == NULL) { /* 4.10 requires a name; the grammar scan rejects this for a reachable define, + so this guards the short-form root build (never reached during validation) */ + PyErr_SetString(PyExc_ValueError, "RELAX NG is missing the required name attribute"); + return schema->p_notallowed; + } Py_ssize_t index = def_find(&schema->defines, name->value, name->value_len); if (index >= 0) { pattern *node_pat = pat_new(schema, P_REF); @@ -591,7 +741,6 @@ static pattern *rng_resolve(th_schema *schema, int def_index) { /* ---- derivatives ---- */ static int rng_nullable(th_schema *schema, pattern *p) { - /* Reference nullability depends on the active recursion guard. */ if (p->nullable >= 0) { return p->nullable; } @@ -603,16 +752,8 @@ static int rng_nullable(th_schema *schema, pattern *p) { return rng_nullable(schema, p->p1) && rng_nullable(schema, p->p2); case P_ONEMORE: return rng_nullable(schema, p->p1); - default: { /* P_REF is the remaining kind without cached nullability. */ - def_entry *entry = &schema->defines.items[p->def_index]; - if (entry->building) { /* a ref recursive without an element guard is not nullable */ - return 0; - } - entry->building = 1; - int nullable = rng_nullable(schema, rng_resolve(schema, p->def_index)); - entry->building = 0; - return nullable; - } + default: /* P_REF; the compile-time 4.19 check rules out a ref that recurses here without an element */ + return rng_nullable(schema, rng_resolve(schema, p->def_index)); } } @@ -700,16 +841,8 @@ static pattern *rng_text_deriv(th_schema *schema, pattern *p, const Py_UCS4 *val } return rng_nullable(schema, derived) ? schema->p_empty : schema->p_notallowed; } - case P_REF: { - def_entry *entry = &schema->defines.items[p->def_index]; - if (entry->building) { /* a ref recursive without an element guard consumes no text */ - return schema->p_notallowed; - } - entry->building = 1; - pattern *derived = rng_text_deriv(schema, rng_resolve(schema, p->def_index), value, len); - entry->building = 0; - return derived; - } + case P_REF: /* the compile-time 4.19 check rules out a ref that recurses here without an element */ + return rng_text_deriv(schema, rng_resolve(schema, p->def_index), value, len); default: return schema->p_notallowed; } @@ -805,16 +938,8 @@ static pattern *rng_att_deriv(th_schema *schema, pattern *p, const qname *name, case P_ATTRIBUTE: return nc_contains(p->nc, name) && rng_value_match(schema, p->p1, value, len) ? schema->p_empty : schema->p_notallowed; - case P_REF: { - def_entry *entry = &schema->defines.items[p->def_index]; - if (entry->building) { /* a ref recursive without an element guard carries no attribute */ - return schema->p_notallowed; - } - entry->building = 1; - pattern *derived = rng_att_deriv(schema, rng_resolve(schema, p->def_index), name, value, len); - entry->building = 0; - return derived; - } + case P_REF: /* the compile-time 4.19 check rules out a ref that recurses here without an element */ + return rng_att_deriv(schema, rng_resolve(schema, p->def_index), name, value, len); default: return schema->p_notallowed; } @@ -834,16 +959,8 @@ static pattern *rng_start_tag_close(th_schema *schema, pattern *p) { return pat_after(schema, rng_start_tag_close(schema, p->p1), p->p2); case P_ATTRIBUTE: return schema->p_notallowed; - case P_REF: { - def_entry *entry = &schema->defines.items[p->def_index]; - if (entry->building) { /* a recursive content ref carries no unmatched attribute; keep it opaque */ - return p; - } - entry->building = 1; - pattern *closed = rng_start_tag_close(schema, rng_resolve(schema, p->def_index)); - entry->building = 0; - return closed; - } + case P_REF: /* the compile-time 4.19 check rules out a ref that recurses here without an element */ + return rng_start_tag_close(schema, rng_resolve(schema, p->def_index)); default: return p; } @@ -955,6 +1072,205 @@ static pattern *rng_child_element(valctx *ctx, pattern *p, th_node *element) { return ended; } +/* ---- compile-time grammar restriction checks ---- + + A single reachability-following pass (rng_scan, below) enforces three restrictions on the + patterns the validator can actually reach -- a must have a name (spec 4.10), a ref loop + must cross an (4.19), and interleave branches must not compete (7.4). Only reachable + patterns are checked because an unreachable is never built or validated, so it cannot + crash or run away; this also keeps compilation off the cost of a schema's dead definitions. */ + +/* Whether two concrete (NC_NAME) element name classes are the same expanded name. */ +static int nc_name_eq(const nameclass *left, const nameclass *right) { + return u_eq_u(left->local, left->local_len, right->local, right->local_len) && + u_eq_u(left->uri, left->uri_len, right->uri, right->uri_len); +} + +/* The name class of an / pattern node (mirrors rng_build's own choice). */ +static nameclass *rng_pattern_nameclass(th_schema *schema, th_node *node, int is_attr) { + const th_node_attr *name = attr_exact(schema->tree, node, "name", 4); + if (name != NULL) { + return nc_from_qname(schema, node, name->value, name->value_len, is_attr); + } + return rng_build_nameclass(schema, first_element_child(node)); +} + +typedef struct { + nameclass **items; + Py_ssize_t len, cap; + int text; +} ncvec; + +static int ncvec_push(th_schema *schema, ncvec *vec, nameclass *nc) { + if (vec->len == vec->cap) { + Py_ssize_t cap = vec->cap ? vec->cap * 2 : 8; + nameclass **items = arena_alloc(&schema->mem, (size_t)cap * sizeof(nameclass *)); + if (items == NULL) { /* GCOVR_EXCL_BR_LINE: arena OOM is unforceable */ + return -1; /* GCOVR_EXCL_LINE */ + } + if (vec->len > 0) { + memcpy(items, vec->items, (size_t)vec->len * sizeof(nameclass *)); + } + vec->items = items; + vec->cap = cap; + } + vec->items[vec->len++] = nc; + return 0; +} + +/* Collect the concrete element names one interleave branch can match as a child, and whether it + allows text, descending through the pattern combinators but stopping at an (its + content is a new level). An is skipped (its own text content is not interleave + text), a is opaque, and an element whose name class is a wildcard or choice is not a + single concrete name -- both fall to the interning backstop rather than a false rejection. */ +static int rng_branch_nameclasses(th_schema *schema, th_node *node, ncvec *out) { + if (is_schema_el(schema, node, RNG_NS, "element")) { + nameclass *nc = rng_pattern_nameclass(schema, node, 0); + return nc->type == NC_NAME ? ncvec_push(schema, out, nc) : 0; + } + if (is_schema_el(schema, node, RNG_NS, "text")) { + out->text = 1; + return 0; + } + if (is_schema_el(schema, node, RNG_NS, "attribute") || is_schema_el(schema, node, RNG_NS, "ref")) { + return 0; + } + for (th_node *child = node->first_child; child != NULL; child = child->next_sibling) { + if (child->type != TH_NODE_ELEMENT) { + continue; + } + if (rng_branch_nameclasses(schema, child, out) < 0) { /* GCOVR_EXCL_BR_LINE: only ncvec_push arena OOM fails */ + return -1; /* GCOVR_EXCL_LINE */ + } + } + return 0; +} + +/* Enforce the RELAX NG 7.4 restriction on one : two branches must not both match an + element with the same name, and at most one branch may match text. libxml2 ("Element or text + conflicts in interleave") and jing/MSV reject such a schema; without it an ambiguous interleave + drives the derivative into exponential memory. */ +static int rng_check_interleave_node(th_schema *schema, th_node *interleave) { + ncvec seen = {NULL, 0, 0, 0}; + for (th_node *branch = interleave->first_child; branch != NULL; branch = branch->next_sibling) { + if (branch->type != TH_NODE_ELEMENT) { + continue; + } + ncvec here = {NULL, 0, 0, 0}; + if (rng_branch_nameclasses(schema, branch, &here) < 0) { /* GCOVR_EXCL_BR_LINE: unforceable arena OOM */ + return -1; /* GCOVR_EXCL_LINE */ + } + if (here.text && seen.text) { + PyErr_SetString(PyExc_ValueError, "RELAX NG interleave has text in more than one branch"); + return -1; + } + for (Py_ssize_t here_index = 0; here_index < here.len; here_index++) { + for (Py_ssize_t seen_index = 0; seen_index < seen.len; seen_index++) { + if (nc_name_eq(here.items[here_index], seen.items[seen_index])) { + PyErr_SetString(PyExc_ValueError, + "RELAX NG interleave has the same element name in more than one branch"); + return -1; + } + } + } + for (Py_ssize_t here_index = 0; here_index < here.len; here_index++) { + if (ncvec_push(schema, &seen, here.items[here_index]) < 0) { /* GCOVR_EXCL_BR_LINE: arena OOM */ + return -1; /* GCOVR_EXCL_LINE */ + } + } + seen.text |= here.text; + } + return 0; +} + +/* The restriction-relevant kind of a schema element, from a single namespace resolution. */ +enum { RNG_SCAN_OTHER, RNG_SCAN_REF, RNG_SCAN_INTERLEAVE, RNG_SCAN_ELEMENT }; + +static int rng_scan_kind(const th_schema *schema, th_node *node) { + const Py_UCS4 *local, *prefix; + Py_ssize_t local_len = 0, prefix_len = 0; + split_prefix(node->text, node->text_len, &local, &local_len, &prefix, &prefix_len); + int kind; + if (u_eq_ascii(local, local_len, "ref")) { + kind = RNG_SCAN_REF; + } else if (u_eq_ascii(local, local_len, "interleave")) { + kind = RNG_SCAN_INTERLEAVE; + } else if (u_eq_ascii(local, local_len, "element")) { + kind = RNG_SCAN_ELEMENT; + } else { + return RNG_SCAN_OTHER; /* most nodes are not a restriction keyword: skip the namespace resolution */ + } + /* a keyword name matched; resolve the namespace once to confirm it is RELAX NG, not a foreign element */ + qname name = schema_direct_qname(schema, node); + return u_eq_ascii(name.uri, name.uri_len, RNG_NS) ? kind : RNG_SCAN_OTHER; +} + +static int rng_scan_define(th_schema *schema, Py_ssize_t def_index, int depth); +static int rng_scan(th_schema *schema, th_node *container, int depth); + +/* Apply the restrictions to one reachable pattern node and descend. A must have a name and is + followed for the 4.19 loop check; an is checked against 7.4; an opens a new + depth level; anything else descends at the same depth. */ +static int rng_scan_node(th_schema *schema, th_node *node, int depth) { + switch (rng_scan_kind(schema, node)) { + case RNG_SCAN_REF: { + const th_node_attr *name = attr_exact(schema->tree, node, "name", 4); + if (name == NULL) { + PyErr_SetString(PyExc_ValueError, "RELAX NG is missing the required name attribute"); + return -1; + } + Py_ssize_t index = def_find(&schema->defines, name->value, name->value_len); + return index >= 0 ? rng_scan_define(schema, index, depth) : 0; + } + case RNG_SCAN_INTERLEAVE: + return rng_check_interleave_node(schema, node) < 0 ? -1 : rng_scan(schema, node, depth); + case RNG_SCAN_ELEMENT: + return rng_scan(schema, node, depth + 1); + default: + return rng_scan(schema, node, depth); + } +} + +/* One reachability-following pass enforcing all three restrictions on the patterns the validator + can reach: a must have a name (4.10), following it must not re-enter a define still open at + this element depth (4.19), and every reached must satisfy 7.4. An unreachable + is never built, so skipping it cannot hide a crash or a blow-up and keeps dead + definitions off the cost. */ +static int rng_scan(th_schema *schema, th_node *container, int depth) { + for (th_node *child = container->first_child; child != NULL; child = child->next_sibling) { + if (child->type != TH_NODE_ELEMENT) { + continue; + } + if (rng_scan_node(schema, child, depth) < 0) { + return -1; + } + } + return 0; +} + +/* RELAX NG 4.19: expanding a must not require expanding the same ref without an in + between. Stamp each define with the element depth where its expansion begins; meeting it again at + that same depth is a loop. Matches libxml2 ("Detected a cycle in %s references") and jing/MSV. */ +static int rng_scan_define(th_schema *schema, Py_ssize_t def_index, int depth) { + def_entry *entry = &schema->defines.items[def_index]; + if (entry->cycle_depth == -1) { + entry->cycle_depth = depth; + int ret = rng_scan(schema, entry->first, depth); + for (def_part *part = entry->extra; ret == 0 && part != NULL; part = part->next) { + ret = rng_scan(schema, part->node, depth); + } + entry->cycle_depth = -2; + return ret; + } + if (entry->cycle_depth == depth) { + char buffer[256]; + PyErr_Format(PyExc_ValueError, "RELAX NG define '%s' references itself with no element in between", + name_utf8(entry->name, entry->len, buffer, sizeof(buffer))); + return -1; + } + return 0; +} + /* ---- compile & entry ---- */ static int rng_compile(th_schema *schema) { @@ -963,8 +1279,10 @@ static int rng_compile(th_schema *schema) { schema->p_notallowed = pat_new(schema, P_NOTALLOWED); schema->p_text = pat_new(schema, P_TEXT); if (!is_schema_el(schema, schema->root, RNG_NS, "grammar")) { + /* the short form has no defines and so no ref cycles; rng_build carries the name (4.10) and + interleave (7.4) checks inline, avoiding a second walk of the whole pattern */ schema->start = rng_build(schema, schema->root); - return 1; + return PyErr_Occurred() ? 0 : 1; } for (th_node *child = schema->root->first_child; child != NULL; child = child->next_sibling) { if (is_schema_el(schema, child, RNG_NS, "define")) { @@ -983,12 +1301,26 @@ static int rng_compile(th_schema *schema) { PyErr_SetString(PyExc_ValueError, "grammar has no start element"); return 0; } + for (Py_ssize_t index = 0; index < schema->defines.len; index++) { + schema->defines.items[index].cycle_depth = -1; + } + if (rng_scan(schema, start, 0) < 0) { + return 0; + } schema->start = rng_build_children(schema, start, NULL); return 1; } static void rng_validate_root(valctx *ctx, th_node *root) { th_schema *schema = ctx->schema; + patintern *intern = arena_alloc(&schema->mem, sizeof(*intern)); + if (intern == NULL) { /* GCOVR_EXCL_BR_LINE: arena OOM is unforceable */ + ctx->failed = 1; /* GCOVR_EXCL_LINE */ + PyErr_NoMemory(); /* GCOVR_EXCL_LINE */ + return; /* GCOVR_EXCL_LINE */ + } + memset(intern, 0, sizeof(*intern)); + schema->intern = intern; Py_ssize_t before = ctx->error_count; pattern *result = rng_child_element(ctx, schema->start, root); if (!rng_nullable(schema, result) && ctx->error_count == before) { diff --git a/src/turbohtml/_c/validate/schema.c b/src/turbohtml/_c/validate/schema.c index e7c50ed7a..e30540c81 100644 --- a/src/turbohtml/_c/validate/schema.c +++ b/src/turbohtml/_c/validate/schema.c @@ -456,8 +456,8 @@ typedef struct def_entry { Py_ssize_t len; th_node *first; /* first element for this name */ def_part *extra, *last; - pattern *built; /* memoized pattern, NULL until first resolved */ - int building; /* recursion guard */ + pattern *built; /* memoized pattern, NULL until first resolved */ + int cycle_depth; /* compile-time 4.19 cycle check: -1 unvisited, -2 cleared, else the element depth in progress */ } def_entry; typedef struct { @@ -482,6 +482,7 @@ typedef struct th_schema { pattern *start; pattern *p_empty, *p_notallowed, *p_text; def_vec defines; + struct patintern *intern; /* per-validation pattern hash-consing table; NULL while compiling */ /* every schema element node's resolved qname, sorted by node pointer for is_schema_el */ sqname_entry *sqnames; Py_ssize_t sqname_count; diff --git a/tests/validate/test_relaxng.py b/tests/validate/test_relaxng.py index 8fe7ca88b..34a7cd51b 100644 --- a/tests/validate/test_relaxng.py +++ b/tests/validate/test_relaxng.py @@ -537,24 +537,186 @@ def test_rng_interleave_choice_same_name() -> None: assert rng_ok(schema, "y") -def test_rng_forbidden_attribute_recursion_is_guarded() -> None: - # left-recursion through an attribute is forbidden by RELAX NG; the engine must not - # loop forever on it -- it treats the recursive branch as unmatchable. +def test_rng_forbidden_attribute_recursion_is_rejected() -> None: + # left-recursion through an attribute reaches the ref again with no element in between, + # which RELAX NG 4.19 forbids; compilation rejects it like libxml2/jing/MSV. schema = rgrammar( '' '' '' "" ) - assert RelaxNG(schema).validate(parse_xml('')).valid + with pytest.raises(ValueError, match="define 'x' references itself with no element in between"): + RelaxNG(schema) -def test_rng_forbidden_text_recursion_is_guarded() -> None: +def test_rng_forbidden_text_recursion_is_rejected() -> None: schema = rgrammar( '' '' ) - assert RelaxNG(schema).validate(parse_xml("hi")).valid + with pytest.raises(ValueError, match="define 'x' references itself with no element in between"): + RelaxNG(schema) + + +def test_rng_nameless_ref_rejected() -> None: + # a with no name attribute would dereference a null attribute while building; RELAX NG + # 4.10 requires the name, so compilation rejects it instead of crashing (libxml2 XML_RNGP_REF_NO_NAME). + schema = rgrammar('') + with pytest.raises(ValueError, match=" is missing the required name attribute"): + RelaxNG(schema) + + +def test_rng_nameless_ref_rejected_short_form() -> None: + # the short form (no ) is checked during the pattern build, not the ref-graph scan; a + # nameless there is still rejected rather than dereferencing a null attribute. + with pytest.raises(ValueError, match=" is missing the required name attribute"): + RelaxNG(rwrap("")) + + +def test_rng_grammar_interleave_conflict_rejected() -> None: + # an overlapping interleave reached from through the grammar is rejected by the ref-graph + # scan (the short form's interleave check lives in the build instead). + schema = rgrammar( + '' + '' + "" + ) + with pytest.raises(ValueError, match="same element name in more than one branch"): + RelaxNG(schema) + + +def test_rng_self_reference_cycle_rejected() -> None: + # expanding requires expanding it again with no element in between (4.19); this + # looped forever before the compile-time check (lxml/jing/MSV all reject it). + schema = rgrammar('') + with pytest.raises(ValueError, match="define 'a' references itself with no element in between"): + RelaxNG(schema) + + +def test_rng_mutual_reference_cycle_rejected() -> None: + schema = rgrammar( + '' + '' + ) + with pytest.raises(ValueError, match="references itself with no element in between"): + RelaxNG(schema) + + +def test_rng_reference_cycle_through_element_is_allowed() -> None: + # the loop passes through an , so 4.19 permits it and validation still terminates + schema = rgrammar( + '' + '' + '' + ) + assert rng_ok(schema, "") + + +def test_rng_ambiguous_choice_stays_bounded() -> None: + # oneOrMore(choice(a, group(a, a))) is legal but ambiguous: without interning the derivative's + # choice doubled per child and exhausted memory at ~26 children. Interning plus duplicate-branch + # elimination keeps it linear, so 400 children validate at once (lxml accepts it too). + schema = rwrap( + '' + '' + "" + ) + validator = RelaxNG(schema) + assert validator.validate(parse_xml("" + "" * 400 + "")).valid + + +def test_rng_overlapping_interleave_rejected() -> None: + # two interleave branches both matching violate the 4.19 interleave restriction and drove the + # derivative into exponential memory; compilation rejects it ("Element or text conflicts in + # interleave" in libxml2). + schema = rwrap("" + '' * 3 + "") + with pytest.raises(ValueError, match="same element name in more than one branch"): + RelaxNG(schema) + + +def test_rng_interleave_double_text_rejected() -> None: + schema = rwrap("") + with pytest.raises(ValueError, match="text in more than one branch"): + RelaxNG(schema) + + +def test_rng_interleave_wildcard_branch_not_analysed() -> None: + # a branch whose name class is a wildcard (anyName), not a single concrete name, is skipped by + # the 4.19 element-name check; the derivative interning still bounds validation of the schema. + schema = rwrap('') + assert rng_ok(schema, "") + + +def test_rng_interleave_branch_with_whitespace_and_group() -> None: + # a branch that is a with insignificant whitespace between its children: the 7.4 scan + # skips the non-element nodes and still collects the element names (a, b disjoint -> accepted). + schema = rwrap( + '\n \n ' + '' + ) + assert rng_ok(schema, "") + + +def test_rng_interleave_same_local_name_different_namespace() -> None: + # two interleave branches share a local name but sit in different namespaces, so the expanded + # names differ and 7.4 permits them; disjoint by namespace, the schema compiles and validates. + schema = rwrap( + '' + '' + ) + assert rng_ok(schema, '') + + +def test_rng_foreign_ref_not_treated_as_relaxng() -> None: + # an element named "ref" in a non-RELAX NG namespace is not a RELAX NG : the grammar scan + # must not raise the missing-name error for it, and treats it as unmatchable content. + schema = rgrammar('') + assert not RelaxNG(schema).validate(parse_xml("x")).valid + + +def test_rng_nested_interleave_conflict_in_valid_interleave_branch() -> None: + # the outer interleave branches (a, b) are disjoint, so 7.4 passes for it; the conflict sits in + # branch a's element content and is reached only by the scan recursing past the valid interleave. + schema = rwrap( + "" + '' + '' + "" + '' + "" + ) + with pytest.raises(ValueError, match="same element name in more than one branch"): + RelaxNG(schema) + + +def test_rng_nested_interleave_conflict_rejected() -> None: + # the conflicting interleave is nested inside another element, so the 7.4 check reaches it only + # through the recursive descent, not the top-level grammar scan. + schema = rwrap( + '' + '' + "" + ) + with pytest.raises(ValueError, match="same element name in more than one branch"): + RelaxNG(schema) + + +def test_rng_attribute_content_ref_matches_text() -> None: + # the attribute content is a that is text-derived directly (not through an element close), + # so the value is matched by taking the derivative of the resolved reference. + schema = rgrammar( + '' + '' + ) + assert rng_ok(schema, '') + + +def test_rng_unreferenced_cyclic_define_compiles() -> None: + # a self-referential define that nothing reaches from start is never expanded, so like lxml the + # grammar compiles; only cycles reachable from start are rejected. + schema = rgrammar('') + assert rng_ok(schema, "") def test_rng_cdata_and_prefixed_attribute() -> None: @@ -865,7 +1027,6 @@ def test_lxml_rng_reuse_benchmark_output(index: int) -> None: pytest.param("group", "", '', False, id="group-right-required"), pytest.param("interleave", "", "", True, id="interleave-empty"), pytest.param("oneOrMore", "", "", True, id="one-or-more-empty"), - pytest.param("choice", '', "", True, id="recursive-choice-empty"), ], ) def test_rng_reference_nullability(combinator: str, left: str, right: str, *, expected: bool) -> None: