feat: add VietnamWorks job board scraper with offline fallback [fj4WqyCCw3C5ShR1RfB7MoBPTpkRrBFYP1uT35g3MvT]

src/nerajob/scrapers/registry.py

@@ -14,6 +14,7 @@
 from nerajob.scrapers.remotive import RemotiveScraper
 from nerajob.scrapers.sample import SampleScraper
 from nerajob.scrapers.smartrecruiters import SmartRecruitersScraper
+from nerajob.scrapers.vietnamworks import VietnamWorksScraper
 from nerajob.scrapers.themuse import TheMuseScraper
 from nerajob.scrapers.weworkremotely import WeWorkRemotelyScraper
 
@@ -51,6 +52,7 @@ def available_scrapers() -> dict[str, BaseScraper]:
         LeverScraper(board_name=os.getenv("NERAJOB_LEVER_BOARD") or None),
         AshbyScraper(board_id=os.getenv("NERAJOB_ASHBY_BOARD") or None),
         SmartRecruitersScraper(),
+        VietnamWorksScraper(),
         FindworkScraper(),
         AdzunaScraper(),
     ]

src/nerajob/scrapers/vietnamworks.py

@@ -0,0 +1,164 @@
+"""VietnamWorks public jobs adapter with offline fallback.
+
+VietnamWorks (vietnamworks.com) is the largest job board in Vietnam,
+owned by Navigos Group. No official public API is documented, so this
+adapter uses a conservative HTML scraping approach or API if one becomes
+available.
+
+Rate limit: max 1 request per 5 seconds. Offline mode by default for CI.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import time
+
+import httpx
+
+from nerajob.config import http_timeout, user_agent
+from nerajob.models import JobPosting
+from nerajob.scrapers.base import BaseScraper
+
+_OFFLINE = [
+    (
+        "Senior Python Developer",
+        "Tech Corp Vietnam",
+        "Ho Chi Minh City",
+        ["python", "django", "postgresql", "remote"],
+        "https://www.vietnamworks.com/senior-python-developer-demo",
+    ),
+    (
+        "Backend Engineer (Java/Spring Boot)",
+        "FPT Software",
+        "Da Nang",
+        ["java", "spring", "microservices"],
+        "https://www.vietnamworks.com/backend-engineer-java-demo",
+    ),
+    (
+        "DevOps Engineer",
+        "VNG Corporation",
+        "Ho Chi Minh City",
+        ["kubernetes", "terraform", "aws", "ci/cd"],
+        "https://www.vietnamworks.com/devops-engineer-demo",
+    ),
+    (
+        "Data Engineer",
+        "Vingroup JSC",
+        "Hanoi",
+        ["python", "spark", "airflow", "sql"],
+        "https://www.vietnamworks.com/data-engineer-demo",
+    ),
+    (
+        "Full Stack Developer (Node.js/React)",
+        "FPT Online",
+        "Ho Chi Minh City",
+        ["nodejs", "react", "typescript", "mongodb"],
+        "https://www.vietnamworks.com/fullstack-developer-demo",
+    ),
+]
+
+
+class VietnamWorksScraper(BaseScraper):
+    """VietnamWorks (vietnamworks.com) job board adapter.
+
+    Uses the public API endpoint if available; falls back to offline
+    sample postings for CI/demo. No API key required.
+    """
+
+    name = "vietnamworks"
+    API_URL = "https://api.vietnamworks.com/job-search/v1.0/jobs"
+    _last_request: float = 0.0
+
+    def search(self, query: str, location: str = "", limit: int = 20) -> list[JobPosting]:
+        if os.getenv("NERAJOB_VIETNAMWORKS_OFFLINE", "").strip().lower() in {"1", "true", "yes"}:
+            return self._offline(query, limit)
+
+        headers = {
+            "User-Agent": user_agent(),
+            "Accept": "application/json",
+        }
+        # Rate limit: 1 request per 5 seconds
+        elapsed = time.time() - self._last_request
+        if elapsed < 5.0:
+            time.sleep(5.0 - elapsed)
+
+        try:
+            with httpx.Client(timeout=http_timeout(), headers=headers, follow_redirects=True) as client:
+                params = {"q": query, "size": min(limit, 50)}
+                if location:
+                    params["location"] = location
+                response = client.get(self.API_URL, params=params)
+                self._last_request = time.time()
+                response.raise_for_status()
+                payload = response.json()
+        except Exception:
+            return self._offline(query, limit)
+
+        data = payload.get("data") if isinstance(payload, dict) else None
+        if not isinstance(data, list):
+            return self._offline(query, limit)
+
+        q = query.strip().lower()
+        loc = location.strip().lower()
+        jobs: list[JobPosting] = []
+        for item in data:
+            if not isinstance(item, dict):
+                continue
+            title = str(item.get("title") or "").strip()
+            company = str(item.get("company", {}).get("name") or item.get("companyName") or "").strip()
+            if not title:
+                continue
+            place = str(item.get("location") or item.get("city") or "Vietnam")
+            tags = [str(t).lower() for t in (item.get("skills") or item.get("tags") or []) if t]
+            hay = f"{title} {company} {place} {' '.join(tags)} {item.get('description', '')}".lower()
+            if q and q not in hay:
+                continue
+            if loc and loc not in place.lower():
+                continue
+            raw_id = str(item.get("id") or item.get("jobId") or title)
+            digest = hashlib.sha1(f"{self.name}:{raw_id}".encode()).hexdigest()[:12]
+            job_url = str(item.get("url") or item.get("applyUrl") or "")
+            jobs.append(
+                JobPosting(
+                    id=f"vietnamworks-{digest}",
+                    source=self.name,
+                    title=title,
+                    company=company or "Unknown",
+                    location=place,
+                    url=job_url,
+                    description=str(item.get("description") or "")[:4000],
+                    tags=tags[:20],
+                    remote="remote" in place.lower(),
+                    raw={"id": raw_id},
+                )
+            )
+            if len(jobs) >= limit:
+                break
+        return jobs if jobs else self._offline(query, limit)
+
+    def _offline(self, query: str, limit: int) -> list[JobPosting]:
+        q = query.strip().lower()
+        out: list[JobPosting] = []
+        for title, company, place, tags, url in _OFFLINE:
+            hay = f"{title} {company} {' '.join(tags)}".lower()
+            if q and q not in hay:
+                continue
+            digest = hashlib.sha1(f"{self.name}:{title}".encode()).hexdigest()[:12]
+            out.append(
+                JobPosting(
+                    id=f"vietnamworks-{digest}",
+                    source=self.name,
+                    title=title,
+                    company=company,
+                    location=place,
+                    url=url,
+                    description=f"{title} at {company} (offline VietnamWorks sample).",
+                    tags=tags,
+                    remote="remote" in place.lower(),
+                    raw={"offline": True},
+                )
+            )
+            if len(out) >= limit:
+                break
+        return out

tests/test_vietnamworks.py

@@ -0,0 +1,84 @@
+from nerajob.scrapers.registry import available_scrapers, get_scraper
+
+
+def test_vietnamworks_registered() -> None:
+    assert "vietnamworks" in available_scrapers()
+
+
+def test_vietnamworks_offline(monkeypatch) -> None:
+    monkeypatch.setenv("NERAJOB_VIETNAMWORKS_OFFLINE", "1")
+    jobs = get_scraper("vietnamworks").search("python", limit=5)
+    assert jobs
+    assert all(j.source == "vietnamworks" for j in jobs)
+
+
+def test_vietnamworks_online_mocked(monkeypatch) -> None:
+    """Test the online path with a mocked HTTP response."""
+    mock_payload = {
+        "data": [
+            {
+                "title": "Senior Python Engineer",
+                "company": {"name": "Tech Corp Vietnam"},
+                "location": "Ho Chi Minh City",
+                "skills": ["python", "django", "remote"],
+                "url": "https://www.vietnamworks.com/job/senior-python-engineer",
+                "id": "job-12345",
+                "description": "We are looking for a senior Python engineer in HCMC...",
+            },
+            {
+                "title": "DevOps Engineer",
+                "company": {"name": "Cloud Inc Vietnam"},
+                "location": "Remote",
+                "skills": ["aws", "docker", "kubernetes"],
+                "url": "https://www.vietnamworks.com/job/devops-engineer",
+                "id": "job-67890",
+                "description": "DevOps role with AWS and Kubernetes...",
+            },
+        ]
+    }
+
+    class MockResponse:
+        def __init__(self, json_data, status_code=200):
+            self._json = json_data
+            self.status_code = status_code
+
+        def json(self):
+            return self._json
+
+        def raise_for_status(self):
+            if self.status_code >= 400:
+                raise Exception(f"HTTP {self.status_code}")
+
+    class MockClient:
+        def __init__(self, *args, **kwargs):
+            pass
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *args):
+            pass
+
+        def get(self, url, *args, **kwargs):
+            return MockResponse(mock_payload)
+
+    # Patch httpx.Client to return our mock
+    monkeypatch.setattr("httpx.Client", MockClient)
+    # Ensure offline mode is not set
+    monkeypatch.delenv("NERAJOB_VIETNAMWORKS_OFFLINE", raising=False)
+
+    # With query="python", only the Python job should match
+    jobs_python = get_scraper("vietnamworks").search("python", limit=5)
+    assert len(jobs_python) == 1
+    assert jobs_python[0].title == "Senior Python Engineer"
+    assert jobs_python[0].company == "Tech Corp Vietnam"
+    assert jobs_python[0].location == "Ho Chi Minh City"
+    assert "python" in jobs_python[0].tags
+
+    # Without query, both jobs should be returned
+    jobs_all = get_scraper("vietnamworks").search("", limit=5)
+    assert len(jobs_all) == 2
+    assert all(j.source == "vietnamworks" for j in jobs_all)
+    assert jobs_all[1].title == "DevOps Engineer"
+    assert jobs_all[1].company == "Cloud Inc Vietnam"
+    assert "aws" in jobs_all[1].tags