Skip to content

Improve scraping on recent releases #472

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kekkokk wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Improve scraping on recent releases #472

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions .env-sample
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,23 @@ TORRENT_CACHE_TTL=2592000 # 30 days
# Set to -1 to never trigger new searches (always use existing cache only).
LIVE_TORRENT_CACHE_TTL=604800 # 7 days

# LIVE_TORRENT_CACHE_TTL_RECENT_RELEASE: Shorter TTL for recently released content.
# Content released within RECENT_RELEASE_DAYS will use this TTL instead of LIVE_TORRENT_CACHE_TTL.
# This allows more frequent refreshes for new releases that may have new torrents appearing.
# Leave empty or set to None to disable (uses LIVE_TORRENT_CACHE_TTL for all content).
LIVE_TORRENT_CACHE_TTL_RECENT_RELEASE= # Optional: 3600 for 1 hour (disabled by default)

# RECENT_RELEASE_DAYS: Days to consider content "recently released".
# Content released within this many days will use LIVE_TORRENT_CACHE_TTL_RECENT_RELEASE.
# Leave empty or set to None to disable.
RECENT_RELEASE_DAYS= # Optional: 7 for 7 days (disabled by default)

# LIVE_TORRENT_CACHE_TTL_NO_DEBRID: Fallback TTL when no torrents are cached on debrid service.
# If release date is unavailable and no torrents are cached on debrid, this shorter TTL is used
# to trigger more frequent searches. This helps find new torrents that may be cached on debrid.
# Leave empty or set to None to disable.
LIVE_TORRENT_CACHE_TTL_NO_DEBRID= # Optional: 3600 for 1 hour (disabled by default)

DEBRID_CACHE_TTL=86400 # 1 day
DEBRID_CACHE_CHECK_RATIO=0.0 # Minimum ratio (0.5 = 5%) of cached torrents/total torrents required to skip re-checking availability on the debrid service.
METRICS_CACHE_TTL=60 # 1 minute
Expand Down
110 changes: 109 additions & 1 deletion comet/api/endpoints/stream.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,36 @@ async def stream(
media_id, id, season, episode
)

# Check release date to determine effective TTL for cache staleness
# Recent releases get shorter TTL to allow more frequent refreshes
release_date = await database.fetch_val(
"""
SELECT release_date FROM digital_release_cache
WHERE media_id = :media_id
""",
{"media_id": media_id},
)

# Calculate effective cache TTL based on release date
used_release_date = False
if (
release_date is not None
and settings.RECENT_RELEASE_DAYS is not None
and settings.RECENT_RELEASE_DAYS > 0
and settings.LIVE_TORRENT_CACHE_TTL_RECENT_RELEASE is not None
and settings.LIVE_TORRENT_CACHE_TTL_RECENT_RELEASE > 0
):
days_since_release = (time.time() - release_date) / 86400
if days_since_release <= settings.RECENT_RELEASE_DAYS:
effective_cache_ttl = settings.LIVE_TORRENT_CACHE_TTL_RECENT_RELEASE
used_release_date = True
else:
effective_cache_ttl = settings.LIVE_TORRENT_CACHE_TTL
used_release_date = True
else:
effective_cache_ttl = settings.LIVE_TORRENT_CACHE_TTL
used_release_date = False

# Quick check for "fresh" cached torrents to decide if we need to re-scrape.
# This does NOT filter what torrents are shown, it only determines if a new search is triggered.
# LIVE_TORRENT_CACHE_TTL controls when cache is considered "stale" and needs refreshing.
Expand All @@ -264,7 +294,7 @@ async def stream(
"media_id": id,
"season": season,
"episode": episode,
"cache_ttl": settings.LIVE_TORRENT_CACHE_TTL,
"cache_ttl": effective_cache_ttl,
"current_time": time.time(),
},
)
Expand Down Expand Up @@ -507,6 +537,84 @@ async def stream(
)
total_count = len(torrent_manager.torrents)

# Fallback: If release date was not available and no torrents are cached on debrid,
# re-evaluate cache staleness with shorter TTL
if (
not used_release_date
and cached_count == 0
and has_cached_results
and settings.LIVE_TORRENT_CACHE_TTL >= 0
and settings.LIVE_TORRENT_CACHE_TTL_NO_DEBRID is not None
and settings.LIVE_TORRENT_CACHE_TTL_NO_DEBRID > 0
):
# Re-check with shorter TTL for no-debrid case
fresh_cached_count_no_debrid = await database.fetch_val(
"""
SELECT COUNT(*)
FROM torrents
WHERE media_id = :media_id
AND ((season IS NOT NULL AND season = CAST(:season as INTEGER)) OR (season IS NULL AND CAST(:season as INTEGER) IS NULL))
AND (episode IS NULL OR episode = CAST(:episode as INTEGER))
AND timestamp + :cache_ttl >= :current_time
""",
{
"media_id": id,
"season": season,
"episode": episode,
"cache_ttl": settings.LIVE_TORRENT_CACHE_TTL_NO_DEBRID,
"current_time": time.time(),
},
)
if fresh_cached_count_no_debrid == 0:
cache_is_stale = True
# If no torrents are cached on debrid, do live scrape (user has 0 results)
# Only if we haven't already scraped in this request
if cached_count == 0 and not needs_scraping and not lock_acquired:
# Acquire lock for live scraping
scrape_lock = DistributedLock(media_id)
lock_acquired = await scrape_lock.acquire()

if lock_acquired:
logger.log(
"SCRAPER",
f"🔄 Cache stale (no debrid-cached torrents) - starting live scrape for {media_id}",
)
needs_scraping = True
# Perform live scraping
try:
await torrent_manager.scrape_torrents()
logger.log(
"SCRAPER",
f"📥 Torrents after global RTN filtering: {len(torrent_manager.torrents)}",
)
# Re-check debrid availability after scraping
await debrid_service_instance.check_existing_availability(
torrent_manager.torrents, season, episode
)
cached_count = sum(
1 for torrent in torrent_manager.torrents.values() if torrent["cached"]
)
total_count = len(torrent_manager.torrents)
finally:
await scrape_lock.release()
lock_acquired = False
else:
# Another instance is scraping, wait for it
logger.log(
"SCRAPER",
f"🔄 Another instance is scraping {media_id}, waiting for results...",
)
await wait_for_scrape_completion(media_id)
# Re-check cache after waiting
await torrent_manager.get_cached_torrents()
await debrid_service_instance.check_existing_availability(
torrent_manager.torrents, season, episode
)
cached_count = sum(
1 for torrent in torrent_manager.torrents.values() if torrent["cached"]
)
total_count = len(torrent_manager.torrents)

if (
(
not has_cached_results
Expand Down
3 changes: 3 additions & 0 deletions comet/core/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ class AppSettings(BaseSettings):
METADATA_CACHE_TTL: Optional[int] = 2592000 # 30 days
TORRENT_CACHE_TTL: Optional[int] = 2592000 # 30 days
LIVE_TORRENT_CACHE_TTL: Optional[int] = 604800 # 7 days
LIVE_TORRENT_CACHE_TTL_RECENT_RELEASE: Optional[int] = None # TTL for recently released content (None = disabled, uses LIVE_TORRENT_CACHE_TTL)
RECENT_RELEASE_DAYS: Optional[int] = None # Days to consider content "recently released" (None = disabled)
LIVE_TORRENT_CACHE_TTL_NO_DEBRID: Optional[int] = None # TTL fallback when no torrents are cached on debrid (None = disabled)
DEBRID_CACHE_TTL: Optional[int] = 86400 # 1 day
METRICS_CACHE_TTL: Optional[int] = 60 # 1 minute
DEBRID_CACHE_CHECK_RATIO: Optional[float] = 0.0 # 0.0 to 1.0
Expand Down