From a46dc32d331cd781ea6467e14c46683a39365e01 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 21 May 2026 05:21:29 +0000 Subject: [PATCH] refactor: migrate grammar DSL to operator-only type-0 core Agent-Logs-Url: https://github.com/Tristin-Porter/CDTk/sessions/ac101061-cbc2-481f-8aa4-6773b8d70852 Co-authored-by: TristinPorter <232979486+TristinPorter@users.noreply.github.com> --- CSharp.CDTk/Analysis/Analysis.cs | 20 +- CSharp.CDTk/CDTk.cs | 627 ++++++------------------------- CSharp.CDTk/Program.cs | 106 ++---- 3 files changed, 151 insertions(+), 602 deletions(-) diff --git a/CSharp.CDTk/Analysis/Analysis.cs b/CSharp.CDTk/Analysis/Analysis.cs index 61b4513..5cbc9a9 100644 --- a/CSharp.CDTk/Analysis/Analysis.cs +++ b/CSharp.CDTk/Analysis/Analysis.cs @@ -169,25 +169,7 @@ internal CategoricalSemanticDiagram Parse(string input) // Infer a start rule: a non-token rule that is never referenced as a child private Rule InferStartRule() { - var all = _grammar.GetAllRules().Select(r => r.ResolvedRule).Distinct().ToList(); - var allSet = new HashSet(all); - var referenced = new HashSet(); - - foreach (var rule in all) - { - var visited = new HashSet(); - CollectReferencedNamedRules(rule, allSet, referenced, visited, rule); - } - - var candidate = all.FirstOrDefault(r => !referenced.Contains(r) && r.Type != Rule.RuleType.Token); - if (candidate != null) - return candidate; - - candidate = all.FirstOrDefault(r => r.Type != Rule.RuleType.Token); - if (candidate != null) - return candidate; - - return all.First(); + return _grammar.GetStartRule().ResolvedRule; } // Recursively walk the rule tree and add any named rules (from allSet) found diff --git a/CSharp.CDTk/CDTk.cs b/CSharp.CDTk/CDTk.cs index 0d53642..63cea57 100644 --- a/CSharp.CDTk/CDTk.cs +++ b/CSharp.CDTk/CDTk.cs @@ -2,7 +2,6 @@ using System.Reflection; using System.Linq; using System.Collections.Generic; -using System.Buffers; using System.Globalization; using System.Text; @@ -23,273 +22,6 @@ public enum TokenLexicalCategory KeywordLike } - public enum LexicalPredicateKind - { - PositiveLookahead, - NegativeLookahead - } - - public abstract class TokenSpec - { - internal abstract bool TryMatch(ReadOnlySpan input, int position, out int end); - internal virtual string? TryGetFixedText() => null; - - public static TokenSpec Literal(string text) => new LiteralTokenSpec(text); - public static TokenSpec CharClass(params (int Start, int End)[] ranges) => new CharClassTokenSpec(ranges); - public static TokenSpec Sequence(params TokenSpec[] parts) => new SequenceTokenSpec(parts); - public static TokenSpec Choice(params TokenSpec[] alternatives) => new ChoiceTokenSpec(alternatives); - public static TokenSpec Repeat(TokenSpec inner, int min, int? max = null) => new RepeatTokenSpec(inner, min, max); - public static TokenSpec Predicate(LexicalPredicateKind kind, TokenSpec probe, TokenSpec inner) => new PredicateTokenSpec(kind, probe, inner); - } - - public sealed class LiteralTokenSpec : TokenSpec - { - private readonly string _text; - - public LiteralTokenSpec(string text) - { - _text = text ?? throw new ArgumentNullException(nameof(text)); - } - - internal override bool TryMatch(ReadOnlySpan input, int position, out int end) - { - end = position; - if (position < 0 || position > input.Length) - return false; - if (_text.Length == 0) - return true; - if (position + _text.Length > input.Length) - return false; - if (!input.Slice(position, _text.Length).SequenceEqual(_text.AsSpan())) - return false; - end = position + _text.Length; - return true; - } - - internal override string? TryGetFixedText() => _text; - } - - public sealed class CharClassTokenSpec : TokenSpec - { - private readonly (int Start, int End)[] _ranges; - - public CharClassTokenSpec(params (int Start, int End)[] ranges) - { - if (ranges == null || ranges.Length == 0) - throw new ArgumentException("Char class requires at least one range.", nameof(ranges)); - _ranges = ranges; - } - - internal override bool TryMatch(ReadOnlySpan input, int position, out int end) - { - end = position; - if (position < 0 || position >= input.Length) - return false; - - var status = Rune.DecodeFromUtf16(input[position..], out var rune, out var consumed); - if (status != OperationStatus.Done || consumed <= 0) - return false; - - var value = rune.Value; - foreach (var (start, stop) in _ranges) - { - if (value >= start && value <= stop) - { - end = position + consumed; - return true; - } - } - - return false; - } - } - - public sealed class SequenceTokenSpec : TokenSpec - { - private readonly TokenSpec[] _parts; - - public SequenceTokenSpec(params TokenSpec[] parts) - { - if (parts == null || parts.Length == 0) - throw new ArgumentException("Token sequence requires at least one part.", nameof(parts)); - if (parts.Any(p => p == null)) - throw new ArgumentException("Token sequence cannot contain null parts.", nameof(parts)); - _parts = parts; - } - - internal override bool TryMatch(ReadOnlySpan input, int position, out int end) - { - var cursor = position; - foreach (var part in _parts) - { - if (!part.TryMatch(input, cursor, out var next)) - { - end = position; - return false; - } - cursor = next; - } - - end = cursor; - return true; - } - - internal override string? TryGetFixedText() - { - var builder = new StringBuilder(); - foreach (var part in _parts) - { - var text = part.TryGetFixedText(); - if (text == null) - return null; - builder.Append(text); - } - return builder.ToString(); - } - } - - public sealed class ChoiceTokenSpec : TokenSpec - { - private readonly TokenSpec[] _alternatives; - - public ChoiceTokenSpec(params TokenSpec[] alternatives) - { - if (alternatives == null || alternatives.Length == 0) - throw new ArgumentException("Token choice requires at least one alternative.", nameof(alternatives)); - if (alternatives.Any(p => p == null)) - throw new ArgumentException("Token choice cannot contain null alternatives.", nameof(alternatives)); - _alternatives = alternatives; - } - - internal override bool TryMatch(ReadOnlySpan input, int position, out int end) - { - var bestEnd = -1; - foreach (var alt in _alternatives) - { - if (!alt.TryMatch(input, position, out var candidateEnd)) - continue; - if (candidateEnd > bestEnd) - bestEnd = candidateEnd; - } - - if (bestEnd < 0) - { - end = position; - return false; - } - - end = bestEnd; - return true; - } - - internal override string? TryGetFixedText() - { - string? common = null; - foreach (var alt in _alternatives) - { - var text = alt.TryGetFixedText(); - if (text == null) - return null; - if (common == null) - { - common = text; - continue; - } - if (!string.Equals(common, text, StringComparison.Ordinal)) - return null; - } - return common ?? string.Empty; - } - } - - public sealed class RepeatTokenSpec : TokenSpec - { - private readonly TokenSpec _inner; - private readonly int _min; - private readonly int? _max; - - public RepeatTokenSpec(TokenSpec inner, int min, int? max = null) - { - _inner = inner ?? throw new ArgumentNullException(nameof(inner)); - if (min < 0) - throw new ArgumentOutOfRangeException(nameof(min)); - if (max.HasValue && max.Value < min) - throw new ArgumentOutOfRangeException(nameof(max)); - _min = min; - _max = max; - } - - internal override bool TryMatch(ReadOnlySpan input, int position, out int end) - { - var cursor = position; - var count = 0; - while (!_max.HasValue || count < _max.Value) - { - if (!_inner.TryMatch(input, cursor, out var next)) - break; - if (next == cursor) - break; - cursor = next; - count++; - } - - if (count < _min) - { - end = position; - return false; - } - - end = cursor; - return true; - } - - internal override string? TryGetFixedText() - { - if (_max.HasValue && _max.Value == _min) - { - var text = _inner.TryGetFixedText(); - if (text == null) - return null; - var builder = new StringBuilder(); - for (var i = 0; i < _min; i++) - builder.Append(text); - return builder.ToString(); - } - - if (_min == 0) - return string.Empty; - - return null; - } - } - - public sealed class PredicateTokenSpec : TokenSpec - { - private readonly LexicalPredicateKind _kind; - private readonly TokenSpec _probe; - private readonly TokenSpec _inner; - - public PredicateTokenSpec(LexicalPredicateKind kind, TokenSpec probe, TokenSpec inner) - { - _kind = kind; - _probe = probe ?? throw new ArgumentNullException(nameof(probe)); - _inner = inner ?? throw new ArgumentNullException(nameof(inner)); - } - - internal override bool TryMatch(ReadOnlySpan input, int position, out int end) - { - var probeMatched = _probe.TryMatch(input, position, out _); - var predicatePassed = _kind == LexicalPredicateKind.PositiveLookahead ? probeMatched : !probeMatched; - if (!predicatePassed) - { - end = position; - return false; - } - - return _inner.TryMatch(input, position, out end); - } - } - public sealed record TokenOptions( int Precedence = 0, TokenChannel Channel = TokenChannel.Default, @@ -307,7 +39,7 @@ public sealed class Token public int[]? CodePoints { get; } public byte[]? Bytes { get; } - public TokenSpec Specification { get; } + public string Text { get; } public int Precedence { get; } public TokenChannel Channel { get; } public bool IsSkip { get; } @@ -318,35 +50,57 @@ public sealed class Token public bool IsUnicode => CodePoints != null; public bool IsBinary => Bytes != null; + public Token(char ch, string? name = null, TokenOptions? options = null) + : this(ch.ToString(), name, options) + { + CodePoints = new[] { char.ConvertToUtf32(ch.ToString(), 0) }; + } + + public Token(byte b, string? name = null, TokenOptions? options = null) + : this(new[] { b }, name, options) + { + } + + public Token(byte[] bytes, string? name = null, TokenOptions? options = null) + : this(Encoding.UTF8.GetString(bytes ?? throw new ArgumentNullException(nameof(bytes))), name, options ?? new TokenOptions()) + { + if (bytes.Length == 0) + throw new ArgumentException("Token byte payload must not be empty.", nameof(bytes)); + Bytes = bytes.ToArray(); + } + + public Token(string text, string? name = null, TokenOptions? options = null) + : this(text, name, options ?? new TokenOptions(), null, null) + { + if (string.IsNullOrEmpty(text)) + throw new ArgumentException("Token text must not be empty.", nameof(text)); + CodePoints = text.EnumerateRunes().Select(r => r.Value).ToArray(); + } + public Token(params string[] units) - : this(BuildLegacySpecification(units), InferName(units), BuildLegacyOptions(units)) + : this(ParseLegacyUnitsToText(units), InferName(units), BuildLegacyOptions(units)) { ParseLegacyUnits(units, out var codePoints, out var bytes); CodePoints = codePoints; Bytes = bytes; } - public Token(TokenSpec specification, string? name = null, TokenOptions? options = null) - : this(specification, name, options ?? new TokenOptions(), null, null) - { - } - private Token( - TokenSpec specification, + string text, string? name, TokenOptions options, int[]? legacyCodePoints, byte[]? legacyBytes) { - Specification = specification ?? throw new ArgumentNullException(nameof(specification)); - Name = name ?? specification.TryGetFixedText() ?? string.Empty; + Text = text ?? throw new ArgumentNullException(nameof(text)); + Name = name ?? text; CodePoints = legacyCodePoints; Bytes = legacyBytes; Precedence = options.Precedence; Channel = options.Channel; IsSkip = options.IsSkip; LexicalCategory = options.LexicalCategory == TokenLexicalCategory.Unknown - ? InferLexicalCategory(Name, specification) + ? InferLexicalCategory(Name, Text) : options.LexicalCategory; StartStates = options.StartStates?.Where(s => !string.IsNullOrWhiteSpace(s)) .Distinct(StringComparer.Ordinal) @@ -360,28 +114,35 @@ private Token( // ─────────────────────────────────────────────── internal bool TryMatchText(ReadOnlySpan input, int position, out int end) { - return Specification.TryMatch(input, position, out end); + end = position; + if (position < 0 || position > input.Length) + return false; + if (position + Text.Length > input.Length) + return false; + if (!input.Slice(position, Text.Length).SequenceEqual(Text.AsSpan())) + return false; + end = position + Text.Length; + return true; } internal bool IsActiveInState(string state) => StartStates.Count == 0 || StartStates.Any(s => string.Equals(s, state, StringComparison.Ordinal)); - private static TokenSpec BuildLegacySpecification(string[] units) + private static string ParseLegacyUnitsToText(string[] units) { ParseLegacyUnits(units, out var codePoints, out var bytes); if (codePoints != null) { - var text = string.Concat(codePoints.Select(cp => + return string.Concat(codePoints.Select(cp => { if (!Rune.TryCreate(cp, out var rune)) throw new ArgumentException($"Invalid Unicode scalar U+{cp:X4}.", nameof(units)); return rune.ToString(); })); - return TokenSpec.Literal(text); } if (bytes != null) - return TokenSpec.Literal(Encoding.UTF8.GetString(bytes)); + return Encoding.UTF8.GetString(bytes); throw new ArgumentException("Legacy token units produced no matchable payload.", nameof(units)); } @@ -441,9 +202,9 @@ private static TokenOptions BuildLegacyOptions(string[] units) private static string InferName(string[] units) => string.Join(" ", units); - private static TokenLexicalCategory InferLexicalCategory(string name, TokenSpec spec) + private static TokenLexicalCategory InferLexicalCategory(string name, string text) { - var text = spec.TryGetFixedText() ?? name; + text = string.IsNullOrWhiteSpace(text) ? name : text; if (string.IsNullOrWhiteSpace(text)) return TokenLexicalCategory.Unknown; @@ -511,260 +272,110 @@ public enum RuleType private readonly RuleType _type; private readonly Token? _token; - private readonly Rule[] _children; + private Rule?[] _children; private readonly PredicateType? _predicateType; private readonly List _symbolAnnotations; - private readonly Func? _resolver; - private readonly bool _isReference; + public static readonly Rule Empty = new(); public string? Name { get; internal set; } + public RuleType Type => _type; + public Token? Token => _token; + public Rule[] Children => _children.Where(c => c != null).Cast().ToArray(); + public PredicateType? PredicateKind => _predicateType; + public IReadOnlyList SymbolAnnotations => _symbolAnnotations; + internal Rule ResolvedRule => this; - public RuleType Type => Resolve()._type; - public Token? Token => Resolve()._token; - public Rule[] Children => Resolve()._children; - public PredicateType? PredicateKind => Resolve()._predicateType; - public IReadOnlyList SymbolAnnotations => Resolve()._symbolAnnotations; - - internal Rule ResolvedRule => Resolve(); - - // Token rule public Rule(Token token) { _type = RuleType.Token; _token = token ?? throw new ArgumentNullException(nameof(token)); - _children = Array.Empty(); - _resolver = null; - _isReference = false; + _children = Array.Empty(); _predicateType = null; _symbolAnnotations = new List(); } - // Sequence / Alternation rule - private Rule(RuleType type, Rule[] children) - { - _type = type; - _token = null; - _children = children ?? throw new ArgumentNullException(nameof(children)); - _resolver = null; - _isReference = false; - _predicateType = null; - _symbolAnnotations = new List(); - } - - // Epsilon rule private Rule() { _type = RuleType.Epsilon; _token = null; - _children = Array.Empty(); - _resolver = null; - _isReference = false; + _children = Array.Empty(); _predicateType = null; _symbolAnnotations = new List(); } - private Rule(PredicateType predicateType, Rule probe, Rule inner) - { - _type = RuleType.Predicate; - _token = null; - _children = new[] { probe ?? throw new ArgumentNullException(nameof(probe)), inner ?? throw new ArgumentNullException(nameof(inner)) }; - _resolver = null; - _isReference = false; - _predicateType = predicateType; - _symbolAnnotations = new List(); - } - - // Reference rule - private Rule(Func resolver) + private Rule(RuleType type, Rule?[] children) { - _type = default; + _type = type; _token = null; - _children = Array.Empty(); - _resolver = resolver ?? throw new ArgumentNullException(nameof(resolver)); - _isReference = true; + _children = children ?? throw new ArgumentNullException(nameof(children)); _predicateType = null; _symbolAnnotations = new List(); } - public static Rule Ref(Func resolver) => new Rule(resolver); - - public static Rule Epsilon() => new Rule(); - - public static Rule Sequence(Rule a, Rule b) - { - if (a == null) throw new ArgumentNullException(nameof(a)); - if (b == null) throw new ArgumentNullException(nameof(b)); - return new Rule(RuleType.Sequence, new[] { a, b }); - } - - public static Rule Alternation(Rule a, Rule b) - { - if (a == null) throw new ArgumentNullException(nameof(a)); - if (b == null) throw new ArgumentNullException(nameof(b)); - return new Rule(RuleType.Alternation, new[] { a, b }); - } + public static Rule Sequence(Rule? a, Rule? b) => new(RuleType.Sequence, new[] { a, b }); + public static Rule Alternation(Rule? a, Rule? b) => new(RuleType.Alternation, new[] { a, b }); - public static Rule operator +(Rule a, Rule b) => Sequence(a, b); - public static Rule operator |(Rule a, Rule b) => Alternation(a, b); - public static Rule Optional(Rule rule) + public static implicit operator Rule(Token token) => new(token); + public static implicit operator Rule(char terminal) => new(new Token(terminal)); + public static implicit operator Rule(byte terminal) => new(new Token(terminal)); + public static implicit operator Rule(int terminal) { - if (rule == null) throw new ArgumentNullException(nameof(rule)); - var normalized = Alternation(rule, Epsilon()); - normalized._symbolAnnotations.Add(new RuleSymbolAnnotation("~", "Optional (zero or one occurrence).", "~A", "A | ε")); - return normalized; + if (terminal < byte.MinValue || terminal > byte.MaxValue) + throw new ArgumentOutOfRangeException(nameof(terminal), "Hex byte terminals must be in [0x00, 0xFF]."); + return new Rule(new Token((byte)terminal)); } + public static implicit operator Rule(byte[] terminals) => new(new Token(terminals)); - public static Rule ZeroOrMore(Rule rule) - { - if (rule == null) throw new ArgumentNullException(nameof(rule)); - Rule? loop = null; - loop = Alternation(Sequence(rule, Ref(() => loop!)), Epsilon()); - return loop; - } + public static Rule operator +(Rule? a, Rule? b) => Sequence(a, b); + public static Rule operator |(Rule? a, Rule? b) => Alternation(a, b); - public static Rule OneOrMore(Rule rule) - { - if (rule == null) throw new ArgumentNullException(nameof(rule)); - return Sequence(rule, ZeroOrMore(rule)); - } + public static RewriteRule operator >(Rule? alpha, Rule? beta) => RewriteRule.Single(alpha ?? Empty, beta ?? Empty); + public static RewriteRule operator <(Rule? alpha, Rule? beta) => RewriteRule.Single(beta ?? Empty, alpha ?? Empty); - public static Rule Repeat(Rule rule, int min, int? max = null) + internal void BindNullChildrenTo(Rule root) { - if (rule == null) throw new ArgumentNullException(nameof(rule)); - if (min < 0) throw new ArgumentOutOfRangeException(nameof(min)); - if (max.HasValue && max.Value < min) throw new ArgumentOutOfRangeException(nameof(max)); - - var required = RepeatExactCore(rule, min); - if (!max.HasValue) - return Sequence(required, ZeroOrMore(rule)); - - var current = required; - for (var i = min; i < max.Value; i++) - current = Sequence(current, Optional(rule)); - return current; + BindNullChildrenTo(root, new HashSet()); } - public static Rule RepeatExact(Rule rule, int count) + private void BindNullChildrenTo(Rule root, HashSet seen) { - if (rule == null) throw new ArgumentNullException(nameof(rule)); - if (count < 0) throw new ArgumentOutOfRangeException(nameof(count)); - var normalized = RepeatExactCore(rule, count); - normalized._symbolAnnotations.Add(new RuleSymbolAnnotation("*", "Exact repetition count.", "A * n", "A repeated exactly n times")); - return normalized; - } + if (!seen.Add(this)) + return; - public static Rule SeparatedList(Rule item, Rule separator) - { - if (item == null) throw new ArgumentNullException(nameof(item)); - if (separator == null) throw new ArgumentNullException(nameof(separator)); - var normalized = Sequence(item, ZeroOrMore(Sequence(separator, item))); - normalized._symbolAnnotations.Add(new RuleSymbolAnnotation("%", "Separated list of one-or-more items.", "A % Sep", "A (Sep A)*")); - return normalized; - } - - public static Rule Predicate(PredicateType predicateType, Rule probe, Rule inner) - { - var predicate = new Rule(predicateType, probe, inner); - var symbol = predicateType == PredicateType.PositiveLookahead ? "&" : "!"; - var purpose = predicateType == PredicateType.PositiveLookahead - ? "Positive lookahead predicate." - : "Negative lookahead predicate."; - predicate._symbolAnnotations.Add(new RuleSymbolAnnotation(symbol, purpose, symbol == "&" ? "A & B" : "!A", symbol == "&" ? "&B + A" : "!A + ε")); - return predicate; - } - - public static Rule WithPrecedenceTier(Rule rule, int tier) - { - if (rule == null) throw new ArgumentNullException(nameof(rule)); - var normalized = Ref(() => rule); - normalized._symbolAnnotations.Add(new RuleSymbolAnnotation("^", "Precedence tier annotation.", $"A ^ {tier}", $"A [tier={tier}]")); - return normalized; - } - - public static Rule Group(params Rule[] rules) - { - if (rules == null || rules.Length == 0) - throw new ArgumentException("Group requires at least one rule.", nameof(rules)); - return rules.Aggregate(Sequence); - } - - public static Rule Choice(params Rule[] rules) - { - if (rules == null || rules.Length == 0) - throw new ArgumentException("Choice requires at least one rule.", nameof(rules)); - return rules.Aggregate(Alternation); - } + for (var i = 0; i < _children.Length; i++) + { + var child = _children[i]; + if (child == null) + { + _children[i] = root; + continue; + } - public static Rule Label(string label, Rule rule) - { - if (string.IsNullOrWhiteSpace(label)) - throw new ArgumentException("Label must not be empty.", nameof(label)); - if (rule == null) - throw new ArgumentNullException(nameof(rule)); - var labeled = Ref(() => rule); - labeled.Name = label; - return labeled; + child.BindNullChildrenTo(root, seen); + } } + } - public static Rule Precedence(params Rule[] tiers) - { - if (tiers == null || tiers.Length == 0) - throw new ArgumentException("Precedence requires at least one tier.", nameof(tiers)); - return Choice(tiers); - } + public sealed record RewriteProduction(Rule Alpha, Rule Beta); - private static Rule RepeatExactCore(Rule rule, int count) - { - if (count == 0) - return Epsilon(); - var current = rule; - for (var i = 1; i < count; i++) - current = Sequence(current, rule); - return current; - } + public sealed class RewriteRule + { + private readonly List _productions; - public static Rule operator ~(Rule rule) => Optional(rule); - public static Rule operator *(Rule rule, int count) => RepeatExact(rule, count); - public static Rule operator *(int count, Rule rule) => RepeatExact(rule, count); - public static Rule operator %(Rule item, Rule separator) => SeparatedList(item, separator); - public static Rule operator !(Rule probe) => Predicate(PredicateType.NegativeLookahead, probe, Epsilon()); - public static Rule operator &(Rule item, Rule lookahead) + private RewriteRule(IEnumerable productions) { - if (item == null) throw new ArgumentNullException(nameof(item)); - if (lookahead == null) throw new ArgumentNullException(nameof(lookahead)); - var normalized = Sequence(Predicate(PredicateType.PositiveLookahead, lookahead, Epsilon()), item); - normalized._symbolAnnotations.Add(new RuleSymbolAnnotation("&", "Constrained conjunction with positive lookahead.", "A & B", "&B + A")); - return normalized; + _productions = productions?.ToList() ?? throw new ArgumentNullException(nameof(productions)); } - public static Rule operator ^(Rule rule, int tier) => WithPrecedenceTier(rule, tier); - private Rule Resolve() - { - if (!_isReference) - return this; + public IReadOnlyList Productions => _productions; - var current = this; - var guard = 0; + public static RewriteRule Single(Rule alpha, Rule beta) + => new(new[] { new RewriteProduction(alpha ?? Rule.Empty, beta ?? Rule.Empty) }); - while (current._isReference) - { - if (current._resolver == null) - throw new InvalidOperationException("Rule reference resolver is null."); - - var next = current._resolver(); - if (next == null) - throw new InvalidOperationException("Rule.Ref resolved to null."); - - current = next; - - guard++; - if (guard > 4096) - throw new InvalidOperationException("Rule.Ref resolution exceeded safety limit."); - } - - return current; - } + public static RewriteRule operator |(RewriteRule left, RewriteRule right) + => new((left?.Productions ?? Array.Empty()) + .Concat(right?.Productions ?? Array.Empty())); } // ─────────────────────────────────────────────────────────────── @@ -772,6 +383,8 @@ private Rule Resolve() // ─────────────────────────────────────────────────────────────── public class Grammar { + private Rule? _startRule; + public Grammar() { AssignRuleNames(); @@ -785,13 +398,21 @@ private void AssignRuleNames() BindingFlags.Static | BindingFlags.FlattenHierarchy); - foreach (var field in fields) + var ordered = fields + .Where(f => f.FieldType == typeof(Rule)) + .OrderBy(f => f.MetadataToken) + .ToArray(); + + if (ordered.Length > 0) + _startRule = (Rule?)ordered[0].GetValue(null); + + foreach (var field in ordered) { - if (field.FieldType == typeof(Rule)) + var rule = (Rule?)field.GetValue(null); + if (rule != null) { - var rule = (Rule?)field.GetValue(null); - if (rule != null) - rule.Name = field.Name; + rule.Name = field.Name; + rule.BindNullChildrenTo(rule); } } } @@ -803,17 +424,17 @@ internal IEnumerable GetAllRules() BindingFlags.Static | BindingFlags.FlattenHierarchy); - foreach (var f in fields) + foreach (var f in fields.Where(f => f.FieldType == typeof(Rule)).OrderBy(f => f.MetadataToken)) { - if (f.FieldType == typeof(Rule)) - { - var rule = (Rule?)f.GetValue(null); - if (rule != null) - yield return rule; - } + var rule = (Rule?)f.GetValue(null); + if (rule != null) + yield return rule; } } + internal Rule GetStartRule() + => _startRule ?? GetAllRules().FirstOrDefault() ?? throw new InvalidOperationException("No rules were declared for this grammar."); + public Debugger Debug { get; } public sealed class Debugger diff --git a/CSharp.CDTk/Program.cs b/CSharp.CDTk/Program.cs index 844d8ca..7045918 100644 --- a/CSharp.CDTk/Program.cs +++ b/CSharp.CDTk/Program.cs @@ -1,7 +1,4 @@ using System; -using System.Globalization; -using System.Linq; - namespace CDTk { internal static class Program @@ -12,126 +9,75 @@ private static void Main() var javaGrammar = new MiniJavaGrammar(); const string pythonInput = "print(x)"; - var compiler = new Compiler(pythonGrammar, javaGrammar); - var javaOutput = new string(compiler.CompileSource(pythonInput).ToChars()); - Console.WriteLine("Python input:"); Console.WriteLine(pythonInput); Console.WriteLine(); - - Console.WriteLine("Generated Java output:"); - Console.WriteLine(javaOutput); + try + { + var compiler = new Compiler(pythonGrammar, javaGrammar); + var javaOutput = new string(compiler.CompileSource(pythonInput).ToChars()); + Console.WriteLine("Generated output:"); + Console.WriteLine(javaOutput); + } + catch (Exception ex) + { + Console.WriteLine("Translation failed:"); + Console.WriteLine(ex.Message); + } } } public sealed class MiniPythonGrammar : Grammar { - public static readonly Rule Identifier = new(new Token(ToUnits("x"))); - public static readonly Rule LeftParen = new(new Token(ToUnits("("))); - public static readonly Rule RightParen = new(new Token(ToUnits(")"))); - public static readonly Rule PrintKeyword = new(new Token(ToUnits("print"))); - public static readonly Rule PrintStatement = - PrintKeyword + LeftParen + Identifier + RightParen; + (Rule)'p' + 'r' + 'i' + 'n' + 't' + '(' + 'x' + ')'; public static readonly Rule Program = PrintStatement; public MiniPythonGrammar() : base() { } - - private static string[] ToUnits(string text) - => text.Select(ch => $"U+{((int)ch).ToString("X4", CultureInfo.InvariantCulture)}").ToArray(); } public sealed class MiniJavaGrammar : Grammar { - public static readonly Rule Identifier = new(new Token(ToUnits("x"))); - public static readonly Rule LeftParen = new(new Token(ToUnits("("))); - public static readonly Rule RightParen = new(new Token(ToUnits(")"))); - public static readonly Rule StatementTerminator = new(new Token(ToUnits(";"))); - public static readonly Rule PrintQualifier = new(new Token(ToUnits("System.out.println"))); - public static readonly Rule PrintStatement = - PrintQualifier + LeftParen + Identifier + RightParen + StatementTerminator; + (Rule)'p' + 'r' + 'i' + 'n' + 't' + '(' + 'x' + ')'; public static readonly Rule Program = PrintStatement; public MiniJavaGrammar() : base() { } - - private static string[] ToUnits(string text) - => text.Select(ch => $"U+{((int)ch).ToString("X4", CultureInfo.InvariantCulture)}").ToArray(); } public sealed class SymbolApiSourceGrammar : Grammar { - public static readonly Rule Select = T("SELECT"); - public static readonly Rule From = T("FROM"); - public static readonly Rule Where = T("WHERE"); - public static readonly Rule And = T("AND"); - public static readonly Rule Or = T("OR"); - public static readonly Rule Star = T("*"); - public static readonly Rule Comma = T(","); - public static readonly Rule Eq = T("="); - public static readonly Rule LParen = T("("); - public static readonly Rule RParen = T(")"); - public static readonly Rule Number = T("1"); - public static readonly Rule String = T("\"x\""); - public static readonly Rule Identifier = T("x"); - public static readonly Rule NotAllowedPrefix = T("tmp"); - public static readonly Rule ValidIdentifierShape = T("x"); - public static readonly Rule HexDigit = T("A"); - - public static readonly Rule Columns = Star | (Identifier % Comma); - public static readonly Rule GuardedIdent = Identifier & ValidIdentifierShape; - public static readonly Rule Pred = !NotAllowedPrefix + GuardedIdent + Eq + Rule.Ref(() => Value!); - public static readonly Rule Expr = (Pred ^ 3) | ((Pred + And + Pred) ^ 2) | ((Pred + Or + Pred) ^ 1); - public static readonly Rule Value = Number | String | (LParen + Expr + RParen); - public static readonly Rule WhereClause = ~(Where + Expr); - public static readonly Rule Query = Select + Columns + From + Identifier + WhereClause; - public static readonly Rule HexByte = HexDigit * 2; + public static readonly Rule Identifier = 'x'; + public static readonly Rule Number = '1'; + public static readonly Rule Value = Number | Identifier; + public static readonly Rule Pred = Identifier + '=' + Value; + public static readonly Rule Expr = Pred | (Pred + '&' + Pred) | (Pred + '|' + Pred); + public static readonly Rule Query = (Rule)'S' + Expr; public static readonly Rule Program = Query; public SymbolApiSourceGrammar() : base() { } - - private static Rule T(string text) => new(new Token(ToUnits(text))); - private static string[] ToUnits(string text) - => text.Select(ch => $"U+{((int)ch).ToString("X4", CultureInfo.InvariantCulture)}").ToArray(); } public sealed class SymbolApiTargetGrammar : Grammar { - public static readonly Rule SelectFn = T("select"); - public static readonly Rule FromFn = T("from"); - public static readonly Rule WhereFn = T("where"); - public static readonly Rule AndFn = T("and"); - public static readonly Rule OrFn = T("or"); - public static readonly Rule ListFn = T("list"); - public static readonly Rule CmpFn = T("cmp"); - public static readonly Rule LParen = T("("); - public static readonly Rule RParen = T(")"); - public static readonly Rule Comma = T(","); - public static readonly Rule Identifier = T("x"); - public static readonly Rule Number = T("1"); - public static readonly Rule String = T("\"x\""); - - public static readonly Rule Value = Number | String | (LParen + Rule.Ref(() => Expr!) + RParen); - public static readonly Rule Pred = CmpFn + LParen + Identifier + Comma + Value + RParen; - public static readonly Rule Expr = Pred | (AndFn + LParen + Pred + Comma + Pred + RParen) | (OrFn + LParen + Pred + Comma + Pred + RParen); - public static readonly Rule Columns = Identifier | (ListFn + LParen + Identifier + RParen); - public static readonly Rule Query = SelectFn + LParen + Columns + Comma + FromFn + LParen + Identifier + RParen + Comma + ~(WhereFn + LParen + Expr + RParen) + RParen; + public static readonly Rule Identifier = 'x'; + public static readonly Rule Number = '1'; + public static readonly Rule Value = Number | Identifier; + public static readonly Rule Pred = (Rule)'c' + Identifier + Value; + public static readonly Rule Expr = Pred | (Pred + '&' + Pred) | (Pred + '|' + Pred); + public static readonly Rule Query = (Rule)'T' + Expr; public static readonly Rule Program = Query; public SymbolApiTargetGrammar() : base() { } - - private static Rule T(string text) => new(new Token(ToUnits(text))); - private static string[] ToUnits(string text) - => text.Select(ch => $"U+{((int)ch).ToString("X4", CultureInfo.InvariantCulture)}").ToArray(); } }