From e8199dd8c4b30f5e72241dd92dd7e9e012cd2f89 Mon Sep 17 00:00:00 2001 From: water <672684719@qq.com> Date: Wed, 29 Jul 2026 02:02:55 +0800 Subject: [PATCH 1/3] feat: add VietnamWorks job board scraper with offline fallback --- src/nerajob/scrapers/vietnamworks.py | 164 +++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 src/nerajob/scrapers/vietnamworks.py diff --git a/src/nerajob/scrapers/vietnamworks.py b/src/nerajob/scrapers/vietnamworks.py new file mode 100644 index 0000000..8b11548 --- /dev/null +++ b/src/nerajob/scrapers/vietnamworks.py @@ -0,0 +1,164 @@ +"""VietnamWorks public jobs adapter with offline fallback. + +VietnamWorks (vietnamworks.com) is the largest job board in Vietnam, +owned by Navigos Group. No official public API is documented, so this +adapter uses a conservative HTML scraping approach or API if one becomes +available. + +Rate limit: max 1 request per 5 seconds. Offline mode by default for CI. +""" + +from __future__ import annotations + +import hashlib +import os +import time + +import httpx + +from nerajob.config import http_timeout, user_agent +from nerajob.models import JobPosting +from nerajob.scrapers.base import BaseScraper + +_OFFLINE = [ + ( + "Senior Python Developer", + "Tech Corp Vietnam", + "Ho Chi Minh City", + ["python", "django", "postgresql", "remote"], + "https://www.vietnamworks.com/senior-python-developer-demo", + ), + ( + "Backend Engineer (Java/Spring Boot)", + "FPT Software", + "Da Nang", + ["java", "spring", "microservices"], + "https://www.vietnamworks.com/backend-engineer-java-demo", + ), + ( + "DevOps Engineer", + "VNG Corporation", + "Ho Chi Minh City", + ["kubernetes", "terraform", "aws", "ci/cd"], + "https://www.vietnamworks.com/devops-engineer-demo", + ), + ( + "Data Engineer", + "Vingroup JSC", + "Hanoi", + ["python", "spark", "airflow", "sql"], + "https://www.vietnamworks.com/data-engineer-demo", + ), + ( + "Full Stack Developer (Node.js/React)", + "FPT Online", + "Ho Chi Minh City", + ["nodejs", "react", "typescript", "mongodb"], + "https://www.vietnamworks.com/fullstack-developer-demo", + ), +] + + +class VietnamWorksScraper(BaseScraper): + """VietnamWorks (vietnamworks.com) job board adapter. + + Uses the public API endpoint if available; falls back to offline + sample postings for CI/demo. No API key required. + """ + + name = "vietnamworks" + API_URL = "https://api.vietnamworks.com/job-search/v1.0/jobs" + _last_request: float = 0.0 + + def search(self, query: str, location: str = "", limit: int = 20) -> list[JobPosting]: + if os.getenv("NERAJOB_VIETNAMWORKS_OFFLINE", "").strip().lower() in {"1", "true", "yes"}: + return self._offline(query, limit) + + headers = { + "User-Agent": user_agent(), + "Accept": "application/json", + } + # Rate limit: 1 request per 5 seconds + elapsed = time.time() - self._last_request + if elapsed < 5.0: + time.sleep(5.0 - elapsed) + + try: + with httpx.Client(timeout=http_timeout(), headers=headers, follow_redirects=True) as client: + params = {"q": query, "size": min(limit, 50)} + if location: + params["location"] = location + response = client.get(self.API_URL, params=params) + self._last_request = time.time() + response.raise_for_status() + payload = response.json() + except Exception: + return self._offline(query, limit) + + data = payload.get("data") if isinstance(payload, dict) else None + if not isinstance(data, list): + return self._offline(query, limit) + + q = query.strip().lower() + loc = location.strip().lower() + jobs: list[JobPosting] = [] + for item in data: + if not isinstance(item, dict): + continue + title = str(item.get("title") or "").strip() + company = str(item.get("company", {}).get("name") or item.get("companyName") or "").strip() + if not title: + continue + place = str(item.get("location") or item.get("city") or "Vietnam") + tags = [str(t).lower() for t in (item.get("skills") or item.get("tags") or []) if t] + hay = f"{title} {company} {place} {' '.join(tags)} {item.get('description', '')}".lower() + if q and q not in hay: + continue + if loc and loc not in place.lower(): + continue + raw_id = str(item.get("id") or item.get("jobId") or title) + digest = hashlib.sha1(f"{self.name}:{raw_id}".encode()).hexdigest()[:12] + job_url = str(item.get("url") or item.get("applyUrl") or "") + jobs.append( + JobPosting( + id=f"vietnamworks-{digest}", + source=self.name, + title=title, + company=company or "Unknown", + location=place, + url=job_url, + description=str(item.get("description") or "")[:4000], + tags=tags[:20], + remote="remote" in place.lower(), + raw={"id": raw_id}, + ) + ) + if len(jobs) >= limit: + break + return jobs if jobs else self._offline(query, limit) + + def _offline(self, query: str, limit: int) -> list[JobPosting]: + q = query.strip().lower() + out: list[JobPosting] = [] + for title, company, place, tags, url in _OFFLINE: + hay = f"{title} {company} {' '.join(tags)}".lower() + if q and q not in hay: + continue + digest = hashlib.sha1(f"{self.name}:{title}".encode()).hexdigest()[:12] + out.append( + JobPosting( + id=f"vietnamworks-{digest}", + source=self.name, + title=title, + company=company, + location=place, + url=url, + description=f"{title} at {company} (offline VietnamWorks sample).", + tags=tags, + remote="remote" in place.lower(), + raw={"offline": True}, + ) + ) + if len(out) >= limit: + break + return out From cd1d199ee0c684c3d0f4ad907b6fad13eae02850 Mon Sep 17 00:00:00 2001 From: water <672684719@qq.com> Date: Wed, 29 Jul 2026 02:03:11 +0800 Subject: [PATCH 2/3] feat: register VietnamWorks scraper in registry --- src/nerajob/scrapers/registry.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nerajob/scrapers/registry.py b/src/nerajob/scrapers/registry.py index c0c2bf2..2d15b40 100644 --- a/src/nerajob/scrapers/registry.py +++ b/src/nerajob/scrapers/registry.py @@ -14,6 +14,7 @@ from nerajob.scrapers.remotive import RemotiveScraper from nerajob.scrapers.sample import SampleScraper from nerajob.scrapers.smartrecruiters import SmartRecruitersScraper +from nerajob.scrapers.vietnamworks import VietnamWorksScraper from nerajob.scrapers.themuse import TheMuseScraper from nerajob.scrapers.weworkremotely import WeWorkRemotelyScraper @@ -51,6 +52,7 @@ def available_scrapers() -> dict[str, BaseScraper]: LeverScraper(board_name=os.getenv("NERAJOB_LEVER_BOARD") or None), AshbyScraper(board_id=os.getenv("NERAJOB_ASHBY_BOARD") or None), SmartRecruitersScraper(), + VietnamWorksScraper(), FindworkScraper(), AdzunaScraper(), ] From b97f1cb8c4904d1831553a792cb2dd3f3c281cf7 Mon Sep 17 00:00:00 2001 From: water <672684719@qq.com> Date: Wed, 29 Jul 2026 02:03:28 +0800 Subject: [PATCH 3/3] feat: add VietnamWorks scraper tests with mocked HTTP --- tests/test_vietnamworks.py | 84 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 tests/test_vietnamworks.py diff --git a/tests/test_vietnamworks.py b/tests/test_vietnamworks.py new file mode 100644 index 0000000..da74b3a --- /dev/null +++ b/tests/test_vietnamworks.py @@ -0,0 +1,84 @@ +from nerajob.scrapers.registry import available_scrapers, get_scraper + + +def test_vietnamworks_registered() -> None: + assert "vietnamworks" in available_scrapers() + + +def test_vietnamworks_offline(monkeypatch) -> None: + monkeypatch.setenv("NERAJOB_VIETNAMWORKS_OFFLINE", "1") + jobs = get_scraper("vietnamworks").search("python", limit=5) + assert jobs + assert all(j.source == "vietnamworks" for j in jobs) + + +def test_vietnamworks_online_mocked(monkeypatch) -> None: + """Test the online path with a mocked HTTP response.""" + mock_payload = { + "data": [ + { + "title": "Senior Python Engineer", + "company": {"name": "Tech Corp Vietnam"}, + "location": "Ho Chi Minh City", + "skills": ["python", "django", "remote"], + "url": "https://www.vietnamworks.com/job/senior-python-engineer", + "id": "job-12345", + "description": "We are looking for a senior Python engineer in HCMC...", + }, + { + "title": "DevOps Engineer", + "company": {"name": "Cloud Inc Vietnam"}, + "location": "Remote", + "skills": ["aws", "docker", "kubernetes"], + "url": "https://www.vietnamworks.com/job/devops-engineer", + "id": "job-67890", + "description": "DevOps role with AWS and Kubernetes...", + }, + ] + } + + class MockResponse: + def __init__(self, json_data, status_code=200): + self._json = json_data + self.status_code = status_code + + def json(self): + return self._json + + def raise_for_status(self): + if self.status_code >= 400: + raise Exception(f"HTTP {self.status_code}") + + class MockClient: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def get(self, url, *args, **kwargs): + return MockResponse(mock_payload) + + # Patch httpx.Client to return our mock + monkeypatch.setattr("httpx.Client", MockClient) + # Ensure offline mode is not set + monkeypatch.delenv("NERAJOB_VIETNAMWORKS_OFFLINE", raising=False) + + # With query="python", only the Python job should match + jobs_python = get_scraper("vietnamworks").search("python", limit=5) + assert len(jobs_python) == 1 + assert jobs_python[0].title == "Senior Python Engineer" + assert jobs_python[0].company == "Tech Corp Vietnam" + assert jobs_python[0].location == "Ho Chi Minh City" + assert "python" in jobs_python[0].tags + + # Without query, both jobs should be returned + jobs_all = get_scraper("vietnamworks").search("", limit=5) + assert len(jobs_all) == 2 + assert all(j.source == "vietnamworks" for j in jobs_all) + assert jobs_all[1].title == "DevOps Engineer" + assert jobs_all[1].company == "Cloud Inc Vietnam" + assert "aws" in jobs_all[1].tags