Add excluded urls option

* refactor from pull request comments to compile regex
* add exclude test

Author: Jim Priest

pylinkvalidator/models.py

@@ -173,7 +173,7 @@ def should_download(self, url_split):
         url = url_split.geturl()
 
         for exclude_url in self.excluded_urls:
-            if re.search(exclude_url, url):
+            if exclude_url.search(url):
                 return False
 
         for ignored_prefix in self.ignored_prefixes:
@@ -215,7 +215,8 @@ def _parse_config(self):
             self.ignored_prefixes = self.options.ignored_prefixes.split(',')
 
         if self.options.excluded_urls:
-            self.excluded_urls = self.options.excluded_urls.split(',')
+            self.excluded_urls = [re.compile(pattern) for pattern in self.options.excluded_urls.split(',')]
+
 
         if self.options.workers:
             self.worker_size = self.options.workers

pylinkvalidator/tests.py

@@ -331,6 +331,13 @@ def test_run_once(self):
         self.assertEqual(8, len(site.pages))
         self.assertEqual(0, len(site.error_pages))
 
+    def test_exclude(self):
+        site = self._run_crawler_plain(ThreadSiteCrawler, ["--exclude=/sub/"])
+
+        # exclude /sub/ directory = 4 pages linked on the index
+        self.assertEqual(4, len(site.pages))
+        self.assertEqual(0, len(site.error_pages))
+
     def test_depth_0(self):
         site = self._run_crawler_plain(
             ThreadSiteCrawler, ["--depth", "0"], "/depth/root.html")