Merge pull request #141 from Gallaecio/fix-fragment_marker_2019

Do not decode # in URL paths during canonicalization

Author: wRAR

tests/test_url.py

@@ -219,6 +219,17 @@ def test_safe_url_port_number(self):
             safe_url_string(u"http://www.example.com:/résumé?q=résumé"),
             "http://www.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9")
 
+    def test_safe_url_string_preserve_nonfragment_hash(self):
+        # don't decode `%23` to `#`
+        self.assertEqual(safe_url_string("http://www.example.com/path/to/%23/foo/bar"),
+                                          "http://www.example.com/path/to/%23/foo/bar")
+        self.assertEqual(safe_url_string("http://www.example.com/path/to/%23/foo/bar#frag"),
+                                          "http://www.example.com/path/to/%23/foo/bar#frag")
+        self.assertEqual(safe_url_string("http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2Fpath%2Fto%2F%23%2Fbar%2Ffoo"),
+                                          "http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2Fpath%2Fto%2F%23%2Fbar%2Ffoo")
+        self.assertEqual(safe_url_string("http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2F%2Fpath%2Fto%2F%23%2Fbar%2Ffoo#frag"),
+                                          "http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2F%2Fpath%2Fto%2F%23%2Fbar%2Ffoo#frag")
+
     def test_safe_download_url(self):
         self.assertEqual(safe_download_url('http://www.example.org'),
                          'http://www.example.org/')
@@ -650,6 +661,21 @@ def test_canonicalize_url_idna_exceptions(self):
             "http://www.{label}.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9".format(
                     label=u"example"*11))
 
+    def test_preserve_nonfragment_hash(self):
+        # don't decode `%23` to `#`
+        self.assertEqual(canonicalize_url("http://www.example.com/path/to/%23/foo/bar"),
+                                          "http://www.example.com/path/to/%23/foo/bar")
+        self.assertEqual(canonicalize_url("http://www.example.com/path/to/%23/foo/bar#frag"),
+                                          "http://www.example.com/path/to/%23/foo/bar")
+        self.assertEqual(canonicalize_url("http://www.example.com/path/to/%23/foo/bar#frag", keep_fragments=True),
+                                          "http://www.example.com/path/to/%23/foo/bar#frag")
+        self.assertEqual(canonicalize_url("http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2Fpath%2Fto%2F%23%2Fbar%2Ffoo"),
+                                          "http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2Fpath%2Fto%2F%23%2Fbar%2Ffoo")
+        self.assertEqual(canonicalize_url("http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2F%2Fpath%2Fto%2F%23%2Fbar%2Ffoo#frag"),
+                                          "http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2F%2Fpath%2Fto%2F%23%2Fbar%2Ffoo")
+        self.assertEqual(canonicalize_url("http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2F%2Fpath%2Fto%2F%23%2Fbar%2Ffoo#frag", keep_fragments=True),
+                                          "http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2F%2Fpath%2Fto%2F%23%2Fbar%2Ffoo#frag")
+
 
 class DataURITests(unittest.TestCase):
 

w3lib/url.py

@@ -33,6 +33,7 @@ def _quote_byte(error):
 EXTRA_SAFE_CHARS = b'|'  # see https://github.com/scrapy/w3lib/pull/25
 
 _safe_chars = RFC3986_RESERVED + RFC3986_UNRESERVED + EXTRA_SAFE_CHARS + b'%'
+_path_safe_chars = _safe_chars.replace(b'#', b'')
 
 _ascii_tab_newline_re = re.compile(r'[\t\n\r]')  # see https://infra.spec.whatwg.org/#ascii-tab-or-newline
 
@@ -74,7 +75,7 @@ def safe_url_string(url, encoding='utf8', path_encoding='utf8', quote_path=True)
 
     # default encoding for path component SHOULD be UTF-8
     if quote_path:
-        path = quote(to_bytes(parts.path, path_encoding), _safe_chars)
+        path = quote(to_bytes(parts.path, path_encoding), _path_safe_chars)
     else:
         path = to_native_str(parts.path)
 
@@ -414,7 +415,7 @@ def _safe_ParseResult(parts, encoding='utf8', path_encoding='utf8'):
         to_native_str(netloc),
 
         # default encoding for path component SHOULD be UTF-8
-        quote(to_bytes(parts.path, path_encoding), _safe_chars),
+        quote(to_bytes(parts.path, path_encoding), _path_safe_chars),
         quote(to_bytes(parts.params, path_encoding), _safe_chars),
 
         # encoding of query and fragment follows page encoding
@@ -502,7 +503,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False,
     # 2. decode percent-encoded sequences in path as UTF-8 (or keep raw bytes)
     #    and percent-encode path again (this normalizes to upper-case %XX)
     uqp = _unquotepath(path)
-    path = quote(uqp, _safe_chars) or '/'
+    path = quote(uqp, _path_safe_chars) or '/'
 
     fragment = '' if not keep_fragments else fragment