♻️ More API changes

* extract `PathAllowedByRobots`
* make `MaybeEscapePattern` public

Signed-off-by: Yves Bastide <[email protected]>

Author: ybastide

RobotsTxt/RobotsMatcher.cs

@@ -163,16 +163,16 @@ public void HandleUnknownAction(int lineNum, ReadOnlySpan<byte> action, ReadOnly
         {
         }
 
-        private void InitUserAgentsAndPath(List<string> userAgents, string path)
+        private void InitUserAgentsAndPath(List<string> userAgents, byte[] path)
         {
             _userAgents = new List<byte[]>(userAgents.Count);
             foreach (var ua in userAgents)
             {
                 _userAgents.Add(Encoding.UTF8.GetBytes(ua));
             }
 
-            Debug.Assert(path.StartsWith("/"));
-            _path = Encoding.UTF8.GetBytes(path);
+            Debug.Assert(path.Length > 0 && path[0] == '/');
+            _path = path;
         }
 
         private bool SeenAnyAgent => _seenGlobalAgent || _seenSpecificAgent;
@@ -182,6 +182,11 @@ public bool AllowedByRobots(byte[] robotsBody, List<string> userAgents, string u
             // The url is not normalized (escaped, percent encoded) here because the user
             // is asked to provide it in escaped form already.
             var path = GetPathParamsQuery(url);
+            return PathAllowedByRobots(robotsBody, userAgents, new UTF8Encoding().GetBytes(path));
+        }
+
+        public bool PathAllowedByRobots(byte[] robotsBody, List<string> userAgents, byte[] path)
+        {
             InitUserAgentsAndPath(userAgents, path);
             ParseRobotsTxt(robotsBody, this);
             return !Disallow();

RobotsTxt/RobotsTxtParser.cs

@@ -119,7 +119,7 @@ private void EmitKeyValueToHandler(int currentLine, ParsedRobotsKey key, ReadOnl
             }
         }
 
-        internal static ReadOnlySpan<byte> MaybeEscapePattern(ReadOnlySpan<byte> src)
+        public static ReadOnlySpan<byte> MaybeEscapePattern(ReadOnlySpan<byte> src)
         {
             int numToEscape = 0;
             bool needCapitalize = false;