fix: make the crawler runnable and testable on Windows (deepset-ai#3830)

* fix crawler and try to run CI * more compact expression * try to fix * improve naming regex * revert regex * make test_url compatible wirh Windows * better conditional expression
joesalvati68 · Jan 10, 2023 · be31178 · be31178
1 parent 7f89101
commit be31178
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 4 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -21,7 +21,6 @@ env:
     --ignore=test/pipelines/test_ray.py
     --ignore=test/document_stores/test_knowledge_graph.py
     --ignore=test/nodes/test_audio.py
-    --ignore=test/nodes/test_connector.py
     --ignore=test/nodes/test_summarizer_translation.py
     --ignore=test/nodes/test_summarizer.py
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

diff --git a/haystack/nodes/connector/crawler.py b/haystack/nodes/connector/crawler.py
@@ -100,8 +100,8 @@ def __init__(
         super().__init__()
 
         IN_COLAB = "google.colab" in sys.modules
-        IN_AZUREML = True if os.environ.get("AZUREML_ENVIRONMENT_IMAGE", None) == "True" else False
-        IS_ROOT = True if os.geteuid() == 0 else False
+        IN_AZUREML = os.environ.get("AZUREML_ENVIRONMENT_IMAGE", None) == "True"
+        IS_ROOT = sys.platform not in ["win32", "cygwin"] and os.geteuid() == 0
 
         if webdriver_options is None:
             webdriver_options = ["--headless", "--disable-gpu", "--disable-dev-shm-usage", "--single-process"]

diff --git a/test/nodes/test_connector.py b/test/nodes/test_connector.py
@@ -18,7 +18,7 @@
 
 @pytest.fixture(scope="session")
 def test_url():
-    return f"file://{SAMPLES_PATH.absolute()}/crawler"
+    return (SAMPLES_PATH / "crawler").absolute().as_uri()
 
 
 def content_match(crawler: Crawler, url: str, crawled_page: Path):