feat: URL-first multi-source dedup with fetched/unique/saved reporting (#19)

pyproject.toml

@@ -51,3 +51,7 @@ asyncio_default_fixture_loop_scope = "function"
 [tool.ruff]
 line-length = 100
 target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "W"]
+ignore = ["BLE001", "PIE810", "E501", "W293"]

src/nerajob/cli.py

@@ -1,26 +1,31 @@
 from __future__ import annotations
+
 from pathlib import Path
 
 import typer
 from rich.console import Console
 from rich.table import Table
 
 from nerajob import __version__
-from nerajob.match import SKILL_ALIASES, extract_skills_from_text
 from nerajob.apply.assistant import prepare_application
 from nerajob.cv.builder import write_cv_files
-from nerajob.match import DEFAULT_MATCH_WEIGHTS, MatchWeights
-from nerajob.models import JobPosting
+from nerajob.match import (
+    DEFAULT_MATCH_WEIGHTS,
+    SKILL_ALIASES,
+    MatchWeights,
+    extract_skills_from_text,
+)
+from nerajob.models import ApplicationPackage, JobPosting
 from nerajob.scrapers.registry import available_scrapers, get_scraper
-from nerajob.models import ApplicationPackage
 from nerajob.storage import (
     default_profile,
     get_job,
     load_applications,
     load_jobs,
     load_profile,
-    load_scan_preset, save_scan_preset,
+    load_scan_preset,
     save_profile,
+    save_scan_preset,
     upsert_jobs,
 )
 
@@ -186,19 +191,31 @@ def scan_cmd(
         console.print("[yellow]No hits from live source; falling back to sample feed.[/yellow]")
         collected = get_scraper("sample").search(query=query, location=location, limit=limit)
 
-    # Dedupe across sources: prefer first occurrence, keep stable order
+    # ── Phase 1: URL-first dedup (prefer first occurrence) ──
     fetched_count = len(collected)
-    seen_keys: set[str] = set()
-    deduped: list[JobPosting] = []
+    seen_urls: set[str] = set()
+    url_deduped: list[JobPosting] = []
     for job in collected:
-        key = f"{job.title.strip().lower()}|{job.company.strip().lower()}|{job.url.strip().lower()}"
-        if key in seen_keys:
+        url_key = (job.url or "").strip().lower()
+        if url_key and url_key in seen_urls:
+            continue
+        if url_key:
+            seen_urls.add(url_key)
+        url_deduped.append(job)
+
+    # ── Phase 2: Title+Company dedup for remaining ──
+    seen_tc: set[str] = set()
+    deduped: list[JobPosting] = []
+    for job in url_deduped:
+        tc_key = f"{job.title.strip().lower()}|{job.company.strip().lower()}"
+        if tc_key in seen_tc:
             continue
-        seen_keys.add(key)
+        seen_tc.add(tc_key)
         deduped.append(job)
     collected = deduped
-    if fetched_count != len(collected):
-        console.print(f"[dim]Deduped[/dim] {fetched_count} fetched → {len(collected)} unique (dropped {fetched_count - len(collected)} duplicates)")
+    unique_count = len(collected)
+    if fetched_count != unique_count:
+        console.print(f"[dim]Deduped[/dim] {fetched_count} fetched → {unique_count} unique")
 
     if remote_only:
         collected = [
@@ -258,8 +275,10 @@ def scan_cmd(
             f"[dim]min-salary {min_salary}[/dim] {before_s} → {len(collected)} jobs (dropped {skipped} below threshold)"
         )
 
+    saved_count = len(collected)
     merged = upsert_jobs(collected)
-    table = Table(title=f"Jobs saved ({len(collected)} new/updated, {len(merged)} total)")
+    console.print(f"[bold green]Scan complete:[/bold green] {fetched_count} fetched, {unique_count} unique, {saved_count} saved")
+    table = Table(title=f"Jobs saved ({saved_count} new/updated, {len(merged)} total)")
     table.add_column("ID", style="dim")
     table.add_column("Source")
     table.add_column("Title")
@@ -411,7 +430,7 @@ def jobs_match(
 ) -> None:
     """Rank saved jobs against your profile (keyword skill match)."""
     from nerajob.match import match_score, rank_jobs
-    from nerajob.models import Profile, JobPosting
+    from nerajob.models import JobPosting, Profile
     from nerajob.storage import load_jobs, load_profile
 
     profile = None

src/nerajob/gui/app.py

@@ -11,10 +11,10 @@ def main(argv: list[str] | None = None) -> int:
     except ImportError:
         print("Install GUI extras: pip install -e \".[gui]\"  (needs PySide6)", file=sys.stderr)
         return 1
-    from nerajob.gui.main_window import MainWindow
-
     from PySide6.QtGui import QFont
 
+    from nerajob.gui.main_window import MainWindow
+
     app = QApplication(sys.argv if argv is None else argv)
     app.setApplicationName("NeraJob")
     app.setOrganizationName("MergeOS")

src/nerajob/gui/main_window.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from PySide6.QtCore import Qt, QSize
+from PySide6.QtCore import QSize, Qt
 from PySide6.QtGui import QFont
 from PySide6.QtWidgets import (
     QAbstractItemView,

src/nerajob/scrapers/ashby.py

@@ -4,12 +4,12 @@
 
 import hashlib
 import json
-from urllib.request import Request, urlopen
 from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
 
+from nerajob.config import http_timeout, user_agent
 from nerajob.models import JobPosting
 from nerajob.scrapers.base import BaseScraper
-from nerajob.config import http_timeout, user_agent
 
 
 class AshbyScraper(BaseScraper):

src/nerajob/scrapers/lever.py

@@ -4,12 +4,12 @@
 
 import hashlib
 import json
-from urllib.request import Request, urlopen
 from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
 
+from nerajob.config import http_timeout, user_agent
 from nerajob.models import JobPosting
 from nerajob.scrapers.base import BaseScraper
-from nerajob.config import http_timeout, user_agent
 
 
 class LeverScraper(BaseScraper):

src/nerajob/scrapers/registry.py

@@ -15,6 +15,7 @@
 from nerajob.scrapers.sample import SampleScraper
 from nerajob.scrapers.smartrecruiters import SmartRecruitersScraper
 from nerajob.scrapers.themuse import TheMuseScraper
+from nerajob.scrapers.usajobs import UsajobsScraper
 from nerajob.scrapers.weworkremotely import WeWorkRemotelyScraper
 
 
@@ -53,6 +54,7 @@ def available_scrapers() -> dict[str, BaseScraper]:
         SmartRecruitersScraper(),
         FindworkScraper(),
         AdzunaScraper(),
+        UsajobsScraper(),
     ]
     return {s.name: s for s in scrapers}