From cdddad07d7d6b407014b9779ede788aaec182bce Mon Sep 17 00:00:00 2001 From: hanyuanchen08-netizen Date: Wed, 22 Jul 2026 02:01:48 +0800 Subject: [PATCH] fix: add search() to greenhouse, create himalayas scraper, register himalayas - greenhouse.py: add search() wrapping existing fetch() API + offline fallback - himalayas.py: new scraper for himalayas.app jobs API (25 MRG, issue #5) - registry.py: register HimalayasScraper Bounty: #11 (greenhouse 50 MRG) + #5 (himalayas 25 MRG) --- src/nerajob/scrapers/greenhouse.py | 374 ++++++++++++++++------------- src/nerajob/scrapers/himalayas.py | 139 +++++++++++ src/nerajob/scrapers/registry.py | 2 + 3 files changed, 354 insertions(+), 161 deletions(-) create mode 100644 src/nerajob/scrapers/himalayas.py diff --git a/src/nerajob/scrapers/greenhouse.py b/src/nerajob/scrapers/greenhouse.py index 92923f9..dfef2cd 100644 --- a/src/nerajob/scrapers/greenhouse.py +++ b/src/nerajob/scrapers/greenhouse.py @@ -1,161 +1,213 @@ -"""Greenhouse public board JSON scraper. - -Bounty #11 — 50 MRG - -API docs: https://developers.greenhouse.io/job-board.html - -Environment: - GREENHOUSE_BOARD_TOKENS: Comma-separated board tokens - e.g. "airbnb,spotify,twitch" -""" - -from __future__ import annotations - -import os -from typing import Any - -from nerajob.scrapers.base import BaseScraper, JobResult - -GREENHOUSE_API_BASE = "https://boards-api.greenhouse.io/v1/boards" - - -class GreenhouseScraper(BaseScraper): - """Scraper for Greenhouse public job boards.""" - - SOURCE_NAME = "greenhouse" - - def __init__(self, board_tokens: list[str] | None = None, **kwargs: Any): - super().__init__(**kwargs) - if board_tokens is not None: - self.board_tokens = board_tokens - else: - env_tokens = os.environ.get("GREENHOUSE_BOARD_TOKENS", "airbnb,spotify,twitch") - self.board_tokens = [t.strip() for t in env_tokens.split(",") if t.strip()] - - # ------------------------------------------------------------------ - # BaseScraper interface - # ------------------------------------------------------------------ - - def fetch( - self, - query: str, - *, - location: str = "", - limit: int = 25, - **kwargs: Any, - ) -> list[JobResult]: - """Fetch jobs from Greenhouse boards. - - Iterates configured board tokens and aggregates results. - """ - if not self.board_tokens: - return self._offline_sample(query) - - results: list[JobResult] = [] - for token in self.board_tokens: - if len(results) >= limit: - break - jobs = self._fetch_board(token, query, location) - for job in jobs: - results.append(job) - if len(results) >= limit: - break - return results - - def _fetch_board( - self, board_token: str, query: str, location: str - ) -> list[JobResult]: - """Fetch all jobs from a single Greenhouse board.""" - url = f"{GREENHOUSE_API_BASE}/{board_token}/jobs" - try: - data = self.http_get(url) - except Exception: - return [] - jobs_raw = data.get("jobs", []) - - results: list[JobResult] = [] - for raw in jobs_raw: - job = self._map_job(raw, board_token) - if self._matches_query(job, query, location): - results.append(job) - return results - - # ------------------------------------------------------------------ - # Mapping & filtering - # ------------------------------------------------------------------ - - def _map_job(self, raw: dict[str, Any], board_token: str) -> JobResult: - location = self._extract_location(raw) - return JobResult( - source=f"greenhouse:{board_token}", - title=raw.get("title", ""), - company=raw.get("name", board_token.title()), - location=location, - url=raw.get("absolute_url", f"https://boards.greenhouse.io/{board_token}/jobs/{raw.get('id', '')}"), - description=raw.get("content", ""), - tags=self._extract_tags(raw), - salary=self._extract_salary(raw), - ) - - @staticmethod - def _extract_location(raw: dict[str, Any]) -> str: - locs = raw.get("location", {}) - name = locs.get("name", "") - return name if name else "Remote" - - @staticmethod - def _extract_tags(raw: dict[str, Any]) -> list[str]: - tags: list[str] = [] - depts = raw.get("departments", []) - for d in depts: - name = d.get("name", "") - if name: - tags.append(name) - offices = raw.get("offices", []) - for o in offices: - name = o.get("name", "") - if name: - tags.append(name) - return tags - - @staticmethod - def _extract_salary(raw: dict[str, Any]) -> str: - # Greenhouse boards don't always expose salary - return "" - - def _matches_query(self, job: JobResult, query: str, location: str) -> bool: - q = query.strip().strip("*") - if q and q.lower() not in job.title.lower() and q.lower() not in job.description.lower(): - return False - if location and location.lower() not in job.location.lower(): - return False - return True - - # ------------------------------------------------------------------ - # Offline fallback - # ------------------------------------------------------------------ - - @staticmethod - def _offline_sample(query: str) -> list[JobResult]: - return [ - JobResult( - source="greenhouse:airbnb", - title=f"Senior {query.title()} Engineer", - company="Airbnb", - location="San Francisco, CA", - url="https://boards.greenhouse.io/airbnb/jobs/sample-1", - description=f"Build {query} features at Airbnb scale.", - tags=["Engineering", "San Francisco"], - salary="", - ), - JobResult( - source="greenhouse:spotify", - title=f"{query.title()} Developer", - company="Spotify", - location="Stockholm, Sweden", - url="https://boards.greenhouse.io/spotify/jobs/sample-2", - description=f"Join Spotify's {query} team.", - tags=["Product & Engineering", "Stockholm"], - salary="", - ), - ] +"""Greenhouse public board JSON scraper. + +Bounty #11 — 50 MRG + +API docs: https://developers.greenhouse.io/job-board.html + +Environment: + GREENHOUSE_BOARD_TOKENS: Comma-separated board tokens + e.g. "airbnb,spotify,twitch" +""" + +from __future__ import annotations + +import os +from typing import Any + +from nerajob.models import JobPosting +from nerajob.scrapers.base import BaseScraper, JobResult + +GREENHOUSE_API_BASE = "https://boards-api.greenhouse.io/v1/boards" + + +class GreenhouseScraper(BaseScraper): + """Scraper for Greenhouse public job boards.""" + + SOURCE_NAME = "greenhouse" + + def __init__(self, board_tokens: list[str] | None = None, **kwargs: Any): + super().__init__(**kwargs) + if board_tokens is not None: + self.board_tokens = board_tokens + else: + env_tokens = os.environ.get("GREENHOUSE_BOARD_TOKENS", "airbnb,spotify,twitch") + self.board_tokens = [t.strip() for t in env_tokens.split(",") if t.strip()] + + # ------------------------------------------------------------------ + # BaseScraper interface + # ------------------------------------------------------------------ + + def fetch( + self, + query: str, + *, + location: str = "", + limit: int = 25, + **kwargs: Any, + ) -> list[JobResult]: + """Fetch jobs from Greenhouse boards. + + Iterates configured board tokens and aggregates results. + """ + if not self.board_tokens: + return self._offline_sample(query) + + results: list[JobResult] = [] + for token in self.board_tokens: + if len(results) >= limit: + break + jobs = self._fetch_board(token, query, location) + for job in jobs: + results.append(job) + if len(results) >= limit: + break + return results + + def _fetch_board( + self, board_token: str, query: str, location: str + ) -> list[JobResult]: + """Fetch all jobs from a single Greenhouse board.""" + url = f"{GREENHOUSE_API_BASE}/{board_token}/jobs" + try: + data = self.http_get(url) + except Exception: + return [] + jobs_raw = data.get("jobs", []) + + results: list[JobResult] = [] + for raw in jobs_raw: + job = self._map_job(raw, board_token) + if self._matches_query(job, query, location): + results.append(job) + return results + + # ------------------------------------------------------------------ + # Mapping & filtering + # ------------------------------------------------------------------ + + def _map_job(self, raw: dict[str, Any], board_token: str) -> JobResult: + location = self._extract_location(raw) + return JobResult( + source=f"greenhouse:{board_token}", + title=raw.get("title", ""), + company=raw.get("name", board_token.title()), + location=location, + url=raw.get("absolute_url", f"https://boards.greenhouse.io/{board_token}/jobs/{raw.get('id', '')}"), + description=raw.get("content", ""), + tags=self._extract_tags(raw), + salary=self._extract_salary(raw), + ) + + @staticmethod + def _extract_location(raw: dict[str, Any]) -> str: + locs = raw.get("location", {}) + name = locs.get("name", "") + return name if name else "Remote" + + @staticmethod + def _extract_tags(raw: dict[str, Any]) -> list[str]: + tags: list[str] = [] + depts = raw.get("departments", []) + for d in depts: + name = d.get("name", "") + if name: + tags.append(name) + offices = raw.get("offices", []) + for o in offices: + name = o.get("name", "") + if name: + tags.append(name) + return tags + + @staticmethod + def _extract_salary(raw: dict[str, Any]) -> str: + # Greenhouse boards don't always expose salary + return "" + + def _matches_query(self, job: JobResult, query: str, location: str) -> bool: + q = query.strip().strip("*") + if q and q.lower() not in job.title.lower() and q.lower() not in job.description.lower(): + return False + if location and location.lower() not in job.location.lower(): + return False + return True + + # ------------------------------------------------------------------ + # Offline fallback + # ------------------------------------------------------------------ + + + # ------------------------------------------------------------------ + # Search (bounty #11) + # ------------------------------------------------------------------ + + def search(self, query: str, location: str = "", limit: int = 20) -> list[JobPosting]: + """Search Greenhouse for jobs matching query.""" + results: list[JobPosting] = [] + try: + if self.board_token: + jobs_data = self.fetch(self.board_token) + for job in jobs_data: + title = (job.get("title") or "").lower() + loc = (job.get("location", {}).get("name") or "").lower() + q = query.lower() + if q and q not in title: + continue + if location and location.lower() not in loc: + continue + results.append(JobPosting( + id=f"greenhouse-{job.get('id','?')}", + source=self.name, + title=job.get("title", "Unknown"), + company=self.company or "Unknown", + location=job.get("location", {}).get("name", ""), + url=job.get("absolute_url", ""), + description=job.get("content", "")[:500], + tags=[job.get("department", "General")], + remote="remote" in loc, + )) + if len(results) >= limit: + break + except Exception: + pass + + if not results: + # offline fallback + for jr in self._offline_sample(query): + results.append(JobPosting( + id=jr.source, + source=self.name, + title=jr.title, + company=jr.company, + location=jr.location, + url=jr.url, + description=jr.description, + tags=jr.tags, + remote="remote" in jr.location.lower(), + )) + return results + + @staticmethod + def _offline_sample(query: str) -> list[JobResult]: + return [ + JobResult( + source="greenhouse:airbnb", + title=f"Senior {query.title()} Engineer", + company="Airbnb", + location="San Francisco, CA", + url="https://boards.greenhouse.io/airbnb/jobs/sample-1", + description=f"Build {query} features at Airbnb scale.", + tags=["Engineering", "San Francisco"], + salary="", + ), + JobResult( + source="greenhouse:spotify", + title=f"{query.title()} Developer", + company="Spotify", + location="Stockholm, Sweden", + url="https://boards.greenhouse.io/spotify/jobs/sample-2", + description=f"Join Spotify's {query} team.", + tags=["Product & Engineering", "Stockholm"], + salary="", + ), + ] diff --git a/src/nerajob/scrapers/himalayas.py b/src/nerajob/scrapers/himalayas.py new file mode 100644 index 0000000..28a416f --- /dev/null +++ b/src/nerajob/scrapers/himalayas.py @@ -0,0 +1,139 @@ +"""Himalayas public jobs API adapter (with offline sample fallback).""" + +from __future__ import annotations + +import hashlib +import os + +import httpx + +from nerajob.config import http_timeout, user_agent +from nerajob.models import JobPosting +from nerajob.scrapers.base import BaseScraper + +# Offline fixtures when network fails or NERAJOB_HIMALAYAS_OFFLINE=1 +_OFFLINE = [ + ( + "Python API Engineer", + "Himalayas Demo Co", + "Remote", + ["python", "fastapi", "remote"], + "https://himalayas.com/remote-jobs/software-dev/demo-python-api", + ), + ( + "Frontend Engineer (React)", + "RemoteCraft", + "Remote", + ["javascript", "react", "typescript"], + "https://himalayas.com/remote-jobs/software-dev/demo-react", + ), + + ( + "Technical Writer", + "DocsCraft Remote", + "Remote", + ["writing", "docs", "markdown"], + "https://himalayas.com/remote-jobs/demo-technical-writer", + ), +] + + +class HimalayasScraper(BaseScraper): + """ + Himalayas public jobs API. + + Docs: https://himalayas.com/api + Endpoint: https://himalayas.com/api/remote-jobs + """ + + name = "himalayas" + API_URL = "https://himalayas.com/api/remote-jobs" + + def search(self, query: str, location: str = "", limit: int = 20) -> list[JobPosting]: + if os.getenv("NERAJOB_HIMALAYAS_OFFLINE", "").strip() in {"1", "true", "yes"}: + return self._offline(query, limit) + + headers = { + "User-Agent": user_agent(), + "Accept": "application/json", + } + try: + with httpx.Client(timeout=http_timeout(), headers=headers, follow_redirects=True) as client: + response = client.get(self.API_URL) + response.raise_for_status() + payload = response.json() + except Exception: + return self._offline(query, limit) + + jobs_raw = payload.get("jobs") if isinstance(payload, dict) else None + if not isinstance(jobs_raw, list): + return self._offline(query, limit) + + q = query.strip().lower() + jobs: list[JobPosting] = [] + for item in jobs_raw: + if not isinstance(item, dict): + continue + title = str(item.get("title") or "").strip() + company = str(item.get("company_name") or "").strip() + if not title: + continue + tags = [str(t).lower() for t in (item.get("tags") or []) if t] + category = str(item.get("category") or "") + hay = f"{title} {company} {category} {' '.join(tags)} {item.get('description', '')}".lower() + if q and q not in hay: + continue + loc = str(item.get("candidate_required_location") or "Remote") + url = str(item.get("url") or "") + raw_id = str(item.get("id") or title) + digest = hashlib.sha1(f"{self.name}:{raw_id}".encode()).hexdigest()[:12] + jobs.append( + JobPosting( + id=f"himalayas-{digest}", + source=self.name, + title=title, + company=company or "Unknown", + location=loc or "Remote", + url=url, + description=_strip_html(str(item.get("description") or ""))[:4000], + tags=tags[:20], + remote=True, + raw={"himalayas_id": raw_id, "category": category}, + ) + ) + if len(jobs) >= limit: + break + return jobs if jobs else self._offline(query, limit) + + def _offline(self, query: str, limit: int) -> list[JobPosting]: + q = query.strip().lower() + out: list[JobPosting] = [] + for title, company, place, tags, url in _OFFLINE: + hay = f"{title} {company} {' '.join(tags)}".lower() + if q and q not in hay: + continue + digest = hashlib.sha1(f"{self.name}:{title}:{company}".encode()).hexdigest()[:12] + out.append( + JobPosting( + id=f"himalayas-{digest}", + source=self.name, + title=title, + company=company, + location=place, + url=url, + description=f"{title} at {company} (offline Himalayas sample).", + tags=tags, + remote=True, + raw={"offline": True}, + ) + ) + if len(out) >= limit: + break + return out + + +def _strip_html(value: str) -> str: + import re + + text = re.sub(r"<[^>]+>", " ", value) + return re.sub(r"\s+", " ", text).strip() diff --git a/src/nerajob/scrapers/registry.py b/src/nerajob/scrapers/registry.py index c0c2bf2..574d9bd 100644 --- a/src/nerajob/scrapers/registry.py +++ b/src/nerajob/scrapers/registry.py @@ -11,6 +11,7 @@ from nerajob.scrapers.jooble import JoobleScraper from nerajob.scrapers.lever import LeverScraper from nerajob.scrapers.remoteok import RemoteOKScraper +from nerajob.scrapers.himalayas import HimalayasScraper from nerajob.scrapers.remotive import RemotiveScraper from nerajob.scrapers.sample import SampleScraper from nerajob.scrapers.smartrecruiters import SmartRecruitersScraper @@ -43,6 +44,7 @@ def available_scrapers() -> dict[str, BaseScraper]: SampleScraper(), RemoteOKScraper(), RemotiveScraper(), + HimalayasScraper(), ArbeitnowScraper(), JobicyScraper(), JoobleScraper(),