stars.py

#!/usr/bin/env -S uv run
# /// script
# requires-python = ">=3.11,<3.12"
# dependencies = []
# ///

import argparse
import json
import re
import sys
import urllib.parse
from collections import defaultdict
from datetime import datetime, timedelta
import signal
import errno
from pathlib import Path


def default_tree_files():
    """Return the live diary root plus every flat ISO-week tree.

    The weekly companion carries Markdown link selections, while the weekly
    note can carry native Forester links.  Reading both keeps deduplication
    correct as the live root is gradually extracted.
    """

    fixed = [Path("trees/uts-0018.tree"), Path("trees/uts-016E.tree")]
    weekly = sorted(Path("trees").glob("????-W??.tree"))
    return [str(path) for path in dict.fromkeys([*fixed, *weekly])]


def strip_ansi(text):
    ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
    return ansi_escape.sub("", text)


def generate_title_from_url(url):
    """Generate a title from URL for cases where title is missing"""
    # Match Mastodon-like URLs: https://mathstodon.xyz/@tao/114603605315214772
    mastodon_pattern = re.compile(r"https?://([^/]+)/@([^/]+)")
    mastodon_match = mastodon_pattern.match(url)

    if mastodon_match:
        domain = mastodon_match.group(1)
        username = mastodon_match.group(2)
        return f"{username}'s post on {domain}"

    # Default case - use domain name
    try:
        domain = re.match(r"https?://(?:www\.)?([^/]+)", url).group(1)
        return f"Post on {domain}"
    except (AttributeError, IndexError):
        return "Untitled post"


def clean_title(title):
    """Clean a title by replacing newlines and multiple spaces with a single space"""
    if not title:
        return ""
    # Replace newlines with spaces, then collapse multiple spaces into one
    return re.sub(r'\s+', ' ', title.replace('\n', ' ')).strip()


def get_primary_url(entry):
    """Extract the primary URL from an entry, preferring externalURL if available"""
    return entry.get("externalURL") or entry.get("url") or entry.get("uniqueID")


def normalize_url(url):
    """
    Normalize a URL for comparison without altering its content identifiers.
    Only normalizes scheme, hostname case, trailing slashes, and removes fragments.
    """
    if not url:
        return None

    try:
        parsed = urllib.parse.urlparse(url)

        # Normalize components that don't affect content
        normalized = urllib.parse.ParseResult(
            scheme=parsed.scheme.lower(),
            netloc=parsed.netloc.lower(),
            path=parsed.path.rstrip("/")
            or "/",  # Normalize empty paths to / and remove trailing /
            params=parsed.params,
            query=parsed.query,  # Preserve query string as is
            fragment="",  # Remove fragment for deduplication (e.g. #atom-everything)
        )

        return urllib.parse.urlunparse(normalized)
    except Exception:
        return url  # Return original if parsing fails


def is_source_from(url, source):
    """Check if a URL is from a specific source"""
    if not url:
        return False

    if source == "lobste.rs":
        return "lobste.rs" in url
    elif source == "HN":
        return "news.ycombinator.com" in url or "ycombinator.com/item" in url

    return False


def extract_existing_urls_from_tree(tree_content):
    """Extract existing URLs from the tree file content"""
    # Match markdown links with various formats, including verbatim-wrapped URLs
    link_pattern = re.compile(r'(?:\[.*?\]|\\\verb>>>.*?>>>)\((.*?)\)')
    urls = set(link_pattern.findall(tree_content))
    
    # Normalize all extracted URLs for more accurate comparison
    normalized_urls = {normalize_url(url) for url in urls if url}
    return normalized_urls


def needs_verb_wrapping(text):
    """Check if text needs to be wrapped with \verb>>>| and >>>"""
    # Texts containing special characters like % need verb wrapping
    return "%" in text


def format_title_with_link(title, url):
    """Format a title with its link, handling special characters and source prefixes"""
    # Clean URL by removing hashtag fragments (like #atom-everything)
    clean_url = url.split('#')[0] if '#' in url else url
    
    # Add source prefix for simonwillison.net
    if 'simonwillison.net' in url:
        if not title.startswith('Simon Willison | '):
            title = f"Simon Willison | {title}"
    
    formatted_title = f"\\verb>>>|{title}>>>" if needs_verb_wrapping(title) else title
    formatted_url = format_url(clean_url)
    return f"[{formatted_title}]({formatted_url})"


def format_url(url):
    """Format a URL, handling special characters properly"""
    if needs_verb_wrapping(url):
        return f"\\verb>>>|{url}>>>"
    return url


def parse_content_notes_highlights(content):
    """
    Parse content field into notes and highlights following these rules:
    
    Notes:
    - Continuous lines (no empty lines between) that are NOT URLs, markdown URLs, or highlights
    - Multiple notes separated by empty lines
    - Example: "line\nline2\n\nline3" = 2 notes
    
    Highlights:
    - Begin with "> Highlight:" 
    - Continue with "> " prefix on subsequent lines
    - Each new "> Highlight:" starts a new highlight
    - Example: "> Highlight:text\n> more text\n> Highlight:new highlight" = 2 highlights
    
    URLs:
    - Standalone URLs (https://...) or markdown URLs ([title](url))
    - Treated separately from notes and highlights
    """
    if not content:
        return [], [], []
    
    lines = content.split('\n')
    notes = []
    highlights = []
    urls = []
    
    i = 0
    while i < len(lines):
        line = lines[i].strip()
        
        # Skip empty lines at top level
        if not line:
            i += 1
            continue
        
        # Check if this is a URL (standalone or markdown)
        if (line.startswith('http://') or line.startswith('https://') or 
            (line.startswith('[') and '](http' in line)):
            urls.append(line)
            i += 1
            continue
        
        # Check if this is a highlight
        if line.startswith('> Highlight:'):
            highlight_lines = [line[12:].strip()]  # Remove "> Highlight:" prefix
            i += 1
            
            # Collect continuation lines with "> " prefix
            while i < len(lines) and lines[i].strip().startswith('> ') and not lines[i].strip().startswith('> Highlight:'):
                highlight_lines.append(lines[i].strip()[2:])  # Remove "> " prefix
                i += 1
            
            highlights.append('\n'.join(highlight_lines))
            continue
        
        # This must be a note - collect continuous non-empty lines
        note_lines = [line]
        i += 1
        
        # Collect continuation lines until empty line or special content
        while i < len(lines):
            next_line = lines[i].strip()
            
            # Stop at empty line (note boundary)
            if not next_line:
                break
            
            # Stop at URL or highlight
            if (next_line.startswith('http://') or next_line.startswith('https://') or
                (next_line.startswith('[') and '](http' in next_line) or
                next_line.startswith('> Highlight:') or next_line.startswith('> ')):
                break
            
            note_lines.append(next_line)
            i += 1
        
        # Add the note if it has content
        if note_lines and any(line.strip() for line in note_lines):
            notes.append('\n'.join(note_lines))
    
    return notes, highlights, urls

def process_rss_json(input_text, existing_urls=None, deduplicate=True, show_all_sources=False, days_filter=7):
    """Process JSON output from rss2linkwarden.py into Forester format"""
    if existing_urls is None:
        existing_urls = set()
        
    content_by_external_url = {}
    date_groups = defaultdict(list)
    debug_info = {
        "total_entries": 0,
        "entries_with_hn": 0,
        "entries_with_lobsters": 0,
        "entries_with_both": 0,
        "skipped_duplicates": 0,
    }

    # Calculate cutoff date if days_filter is specified
    cutoff_date = None
    if days_filter > 0:
        cutoff_date = datetime.now() - timedelta(days=days_filter)

    for line in input_text.strip().split('\n'):
        if not line.strip():
            continue
            
        try:
            entry = json.loads(line)
            debug_info["total_entries"] += 1
            
            # Get timestamp and check date filter
            timestamp = entry.get("datePublished", 0)
            if timestamp:
                entry_date = datetime.fromtimestamp(float(timestamp))
                if cutoff_date and entry_date < cutoff_date:
                    continue
                date = entry_date.strftime("%Y-%m-%d")
            else:
                date = datetime.now().strftime("%Y-%m-%d")
            
            # Get primary URL and normalize
            primary_url = entry.get("externalURL") or entry.get("url", "")
            if not primary_url:
                continue
                
            normalized_url = normalize_url(primary_url)
            
            # Skip if deduplicating and URL exists
            if deduplicate and normalized_url in existing_urls:
                debug_info["skipped_duplicates"] += 1
                continue
            
            # Get title and tags
            title = entry.get("title", "")
            if not title:
                title = generate_title_from_url(primary_url)
            else:
                title = clean_title(title)
            
            tags = entry.get("tags", "")
            content = entry.get("content", "")
            text_content = entry.get("textContent", "")
            
            # Check if this is a discussion link (re/hn, re/lb, re/rd tags)
            is_discussion = False
            discussion_platform = None
            source_url = None
            
            if tags and ("re/hn" in tags or "re/lb" in tags or "re/rd" in tags):
                is_discussion = True
                if "re/hn" in tags:
                    discussion_platform = "HN"
                elif "re/lb" in tags:
                    discussion_platform = "lobste.rs"
                elif "re/rd" in tags:
                    discussion_platform = "Reddit"
                
                # Extract source URL from content
                source_match = re.search(r'from (https?://[^\s]+)', content)
                if source_match:
                    source_url = source_match.group(1)
            
            if is_discussion and source_url:
                # This is a discussion link - attach it to the source URL
                source_normalized = normalize_url(source_url)
                
                # Find or create entry for source URL
                if source_normalized not in content_by_external_url:
                    # Create placeholder entry for source
                    content_by_external_url[source_normalized] = {
                        "timestamp": float(timestamp) if timestamp else 0,
                        "date": date,
                        "title": generate_title_from_url(source_url),
                        "primary_url": source_url,
                        "sources": {},
                        "tags": "",
                        "content": "",
                        "textContent": "",
                        "folder": "",
                    }
                
                # Add discussion link to sources
                content_by_external_url[source_normalized]["sources"][discussion_platform] = primary_url
                if discussion_platform == "HN":
                    debug_info["entries_with_hn"] += 1
                elif discussion_platform == "lobste.rs":
                    debug_info["entries_with_lobsters"] += 1
                    
            else:
                # This is a main content entry
                if normalized_url not in content_by_external_url:
                    content_by_external_url[normalized_url] = {
                        "timestamp": float(timestamp) if timestamp else 0,
                        "date": date,
                        "title": title,
                        "primary_url": primary_url,
                        "sources": {},
                        "tags": tags,
                        "content": content,
                        "textContent": text_content,
                        "folder": entry.get("folder", ""),
                    }
                elif float(timestamp) > content_by_external_url[normalized_url]["timestamp"]:
                    # Update with newer timestamp but preserve sources
                    existing_sources = content_by_external_url[normalized_url].get("sources", {})
                    content_by_external_url[normalized_url].update({
                        "timestamp": float(timestamp),
                        "date": date,
                        "title": title,
                        "primary_url": primary_url,
                        "tags": tags,
                        "content": content,
                        "textContent": text_content,
                    })
                    content_by_external_url[normalized_url]["sources"] = existing_sources
                    
        except (json.JSONDecodeError, ValueError):
            continue
    
    # Count entries with both sources
    for content_data in content_by_external_url.values():
        sources = content_data.get("sources", {})
        if "lobste.rs" in sources and "HN" in sources:
            debug_info["entries_with_both"] += 1
    
    # Group entries by tags and format them
    for normalized_url, content_data in content_by_external_url.items():
        date = content_data["date"]
        title = content_data["title"]
        primary_url = content_data["primary_url"]
        sources = content_data.get("sources", {})
        tags = content_data.get("tags", "")
        content = content_data.get("content", "")
        
        # Parse tags
        tag_list = []
        if tags:
            tag_list = [t.strip() for t in tags.replace("|", ",").split(",") if t.strip()]
        
        # Format main link with title only
        formatted_title_link = format_title_with_link(title, primary_url)
        
        # Build entry content lines
        entry_content = []
        
        # Add discussion links
        if "HN" in sources:
            hn_url = format_url(sources["HN"])
            entry_content.append(f"        - [On HN]({hn_url})")
        if "lobste.rs" in sources:
            lb_url = format_url(sources["lobste.rs"])
            entry_content.append(f"        - [On lobste.rs]({lb_url})")
        
        # Parse content into notes, highlights, and URLs
        notes, highlights, content_urls = parse_content_notes_highlights(content)
        
        # Add related URLs
        for url in content_urls:
            if url.startswith('[') and '](http' in url:
                url_match = re.search(r'\]\((https?://[^)]+)\)', url)
                if url_match:
                    url = url_match.group(1)
            if url != primary_url and url not in sources.values():
                # Clean hashtag fragments from related URLs
                clean_url = url.split('#')[0] if '#' in url else url
                entry_content.append(f"        - {clean_url}")
        
        # Add notes
        for note in notes:
            if note.strip() and len(note.strip()) > 5:
                for line in note.split('\n'):
                    if line.strip():
                        entry_content.append(f"        - {line.strip()}")
        
        # Add highlights
        if highlights:
            entry_content.append("")
            for highlight in highlights:
                highlight_lines = highlight.split('\n')
                for i, line in enumerate(highlight_lines):
                    if line.strip():
                        entry_content.append(f"        > {line.strip()}")
                        # Add empty line after each highlight line except the last one
                        if i < len(highlight_lines) - 1 and highlight_lines[i + 1].strip():
                            entry_content.append("")
            entry_content.append("")
        
        # Store entry data
        entry_data = {
            'title_link': formatted_title_link,
            'content': entry_content,
            'tags': tag_list,
            'date': date
        }
        
        # Add to date group
        if date not in date_groups:
            date_groups[date] = []
        date_groups[date].append(entry_data)
    
    # Process each date group to organize by shared tag sets
    for date in date_groups:
        entries = date_groups[date]
        
        # Simple greedy grouping: find pairs/groups with most shared tags
        grouped_entries = []
        remaining_entries = entries[:]
        
        while remaining_entries:
            if len(remaining_entries) == 1:
                # Last entry, add it alone
                entry = remaining_entries[0]
                grouped_entries.append({
                    'shared_tags': entry['tags'],
                    'entries': [entry]
                })
                break
            
            # Find the best pair/group starting from first remaining entry
            current_entry = remaining_entries[0]
            current_tags = set(current_entry['tags'])
            
            best_group = None
            best_shared_count = 0
            
            # Only check reasonable tag combinations (1-3 tags max)
            for shared_count in range(min(3, len(current_tags)), 0, -1):
                from itertools import combinations
                for tag_combo in combinations(current_tags, shared_count):
                    tag_set = set(tag_combo)
                    
                    # Find all entries that have these exact tags
                    matching_entries = []
                    for entry in remaining_entries:
                        entry_tags = set(entry['tags'])
                        if tag_set.issubset(entry_tags):
                            matching_entries.append(entry)
                    
                    # Need at least 2 entries to form a group
                    if len(matching_entries) >= 2 and len(tag_set) > best_shared_count:
                        best_group = {
                            'shared_tags': list(tag_combo),
                            'entries': matching_entries
                        }
                        best_shared_count = len(tag_set)
                        break  # Take first good match at this level
                
                if best_group:
                    break  # Found a group, don't try smaller tag counts
            
            if best_group:
                # Use the best group found
                grouped_entries.append(best_group)
                for entry in best_group['entries']:
                    remaining_entries.remove(entry)
            else:
                # No grouping possible, add first entry alone
                entry = remaining_entries[0]
                grouped_entries.append({
                    'shared_tags': entry['tags'],
                    'entries': [entry]
                })
                remaining_entries.remove(entry)
        
        # Format output for this date
        formatted_entries = []
        
        for group in grouped_entries:
            shared_tags = group['shared_tags']
            entries = group['entries']
            
            if not shared_tags:
                # Untagged entries
                for entry in entries:
                    title_line = f"- {entry['title_link']}"
                    formatted_entries.append(title_line)
                    formatted_entries.extend(entry['content'])
            elif len(entries) == 1:
                # Single entry with tags
                entry = entries[0]
                title_line = f"- {entry['title_link']}"
                if entry['tags']:
                    tag_str = " ".join(f"#{t}" for t in entry['tags'])
                    title_line += f" {tag_str}"
                formatted_entries.append(title_line)
                formatted_entries.extend(entry['content'])
            else:
                # Multiple entries sharing tags
                shared_tag_str = " ".join(f"#{t}" for t in shared_tags)
                formatted_entries.append(f"- {shared_tag_str}")
                
                for entry in entries:
                    # Show remaining tags (excluding shared tags)
                    remaining_tags = [t for t in entry['tags'] if t not in shared_tags]
                    title_line = f"    - {entry['title_link']}"
                    if remaining_tags:
                        tag_str = " ".join(f"#{t}" for t in remaining_tags)
                        title_line += f" {tag_str}"
                    formatted_entries.append(title_line)
                    formatted_entries.extend(entry['content'])
        
        date_groups[date] = formatted_entries
    
    # Generate debug comment
    debug_comment = [
        "\\comment{",
        f"Total entries processed: {debug_info['total_entries']}",
        f"Entries with HN links: {debug_info['entries_with_hn']}",
        f"Entries with lobste.rs links: {debug_info['entries_with_lobsters']}",
        f"Entries with both links: {debug_info['entries_with_both']}",
        f"Skipped duplicates: {debug_info['skipped_duplicates']}",
        "}",
    ]
    
    # Generate output
    output = []
    output.extend(debug_comment)
    output.append("")
    
    for date in sorted(date_groups.keys(), reverse=True):
        if not date_groups[date]:
            continue
        output.append(f"\\subtree[{date}]{{\\mdnote{{{date}}}{{")
        output.extend(date_groups[date])
        output.append("}}")
        output.append("")
    
    return "\n".join(output)

def process_stars(input_text, existing_urls=None, deduplicate=True, show_all_sources=False, days_filter=7):
    # Initialize set of existing URLs if provided
    if existing_urls is None:
        existing_urls = set()
    
    # Calculate cutoff date for filtering if days_filter is not -1
    cutoff_timestamp = None
    if days_filter != -1:
        cutoff_date = datetime.now() - timedelta(days=days_filter)
        cutoff_timestamp = cutoff_date.timestamp()
    
    # Group by date
    date_groups = defaultdict(list)

    # Track content by normalized external URLs
    content_by_external_url = {}

    # Debug info to help understand input
    debug_info = {
        "total_entries": 0,
        "entries_with_hn": 0,
        "entries_with_lobsters": 0,
        "entries_with_both": 0,
        "skipped_duplicates": 0,
    }

    # Split input into JSON objects and process each one
    clean_input = strip_ansi(input_text)

    # Skip any lines starting with "Reading" or containing "dependencies"
    content_lines = []
    capturing = False
    for line in clean_input.split("\n"):
        if line.startswith("Reading") or "dependencies" in line:
            continue
        if line.strip().startswith("{"):
            capturing = True
            content_lines = [line]
        elif capturing and line.strip():
            content_lines.append(line)
            if line.strip() == "}":
                try:
                    entry = json.loads("\n".join(content_lines))
                    debug_info["total_entries"] += 1

                    # Convert timestamps to date, using dateArrived as fallback
                    timestamp = entry.get("datePublished") or entry.get("dateArrived")
                    if timestamp is None:
                        continue  # Skip if no timestamp available

                    # Filter by days if specified
                    if cutoff_timestamp is not None and timestamp < cutoff_timestamp:
                        continue  # Skip entries older than the cutoff

                    date = datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d")

                    # Get primary URL (preferring external URL, then url, then uniqueID)
                    primary_url = get_primary_url(entry)
                    if not primary_url:  # Skip if no URL available
                        continue

                    # For deduplication, normalize the URL
                    normalized_url = normalize_url(primary_url)
                    if not normalized_url:
                        continue

                    # Check if URL already exists in the tree file and skip if deduplication is enabled
                    if deduplicate and normalized_url in existing_urls:
                        debug_info["skipped_duplicates"] += 1
                        continue

                    # Handle null title or empty title
                    title = entry.get("title")
                    if not title:  # If title is None or empty string
                        title = generate_title_from_url(primary_url)
                    else:
                        # Clean the title by replacing newlines with spaces
                        title = clean_title(title)

                    # Check if we've seen this content before
                    if normalized_url not in content_by_external_url:
                        content_by_external_url[normalized_url] = {
                            "timestamp": timestamp,
                            "date": date,
                            "title": title,
                            "primary_url": primary_url,
                            "sources": {},
                        }
                    elif timestamp > content_by_external_url[normalized_url]["timestamp"]:
                        # Update with newer timestamp and title
                        content_by_external_url[normalized_url]["timestamp"] = timestamp
                        content_by_external_url[normalized_url]["date"] = date
                        content_by_external_url[normalized_url]["title"] = title
                        content_by_external_url[normalized_url][
                            "primary_url"
                        ] = primary_url

                    # Make sure sources dict exists
                    if "sources" not in content_by_external_url[normalized_url]:
                        content_by_external_url[normalized_url]["sources"] = {}

                    # Record the source information regardless of timestamp
                    # Check all possible URL fields for source detection
                    sources_to_check = [entry.get("url"), entry.get("uniqueID"), entry.get("externalURL")]

                    for url in sources_to_check:
                        if url:
                            if is_source_from(url, "lobste.rs"):
                                content_by_external_url[normalized_url]["sources"][
                                    "lobste.rs"
                                ] = url
                                debug_info["entries_with_lobsters"] += 1
                            elif is_source_from(url, "HN"):
                                content_by_external_url[normalized_url]["sources"][
                                    "HN"
                                ] = url
                                debug_info["entries_with_hn"] += 1

                except json.JSONDecodeError:
                    continue  # Skip invalid JSON
                finally:
                    capturing = False  # Reset capturing state

    # Count entries that have both HN and lobste.rs links
    for normalized_url, content_data in content_by_external_url.items():
        sources = content_data.get("sources", {})
        if "lobste.rs" in sources and "HN" in sources:
            debug_info["entries_with_both"] += 1

    # Add debug info as a comment at the top of the output
    debug_comment = [
        "\\comment{",
        f"Total entries processed: {debug_info['total_entries']}",
        f"Entries with HN links: {debug_info['entries_with_hn']}",
        f"Entries with lobste.rs links: {debug_info['entries_with_lobsters']}",
        f"Entries with both links: {debug_info['entries_with_both']}",
        f"Skipped duplicates: {debug_info['skipped_duplicates']}",
        "}",
    ]

    # Process unique entries and format them
    for normalized_url, content_data in content_by_external_url.items():
        date = content_data["date"]
        title = content_data["title"]
        primary_url = content_data["primary_url"]

        # Format title with special character handling
        formatted_title_link = format_title_with_link(title, primary_url)

        # Start building the link with the primary content - CHANGED: removed "read " from here
        link_parts = [f"- {formatted_title_link}"]

        # Add source references - show when show_all_sources is true, or when there's HN (alone or with lobste.rs)
        # AGENT-NOTE: Source link display logic - only show links when HN is present (alone or with lobste.rs), not for lobste.rs-only
        sources = content_data.get("sources", {})
        
        if show_all_sources:
            # Show all sources when explicitly requested
            for source_name, source_url in sorted(sources.items()):
                # Only add source reference if it's not the same as primary URL
                if normalize_url(source_url) != normalize_url(primary_url):
                    formatted_source_url = format_url(source_url)
                    link_parts.append(f"([on {source_name}]({formatted_source_url}))")
        else:
            # Show source references only when there's HN (alone or with lobste.rs)
            has_hn = "HN" in sources
            
            # Show links if there's HN present
            if has_hn:
                for source_name, source_url in sorted(sources.items()):
                    # Only add source reference if it's not the same as primary URL
                    if normalize_url(source_url) != normalize_url(primary_url):
                        formatted_source_url = format_url(source_url)
                        link_parts.append(f"([on {source_name}]({formatted_source_url}))")

        # Join parts with spaces
        link = " ".join(link_parts)

        # Add to date group
        date_groups[date].append(link)

    # Generate output in chronological order (newest first)
    output = []
    output.extend(debug_comment)
    output.append("")

    for date in sorted(date_groups.keys(), reverse=True):
        # Skip dates with no links
        if not date_groups[date]:
            continue
            
        output.append(f"\\subtree[{date}]{{\\mdnote{{{date}}}{{")
        output.extend(sorted(date_groups[date]))  # Sort links alphabetically
        output.append("}}")
        output.append("")

    return "\n".join(output)


def handle_broken_pipe():
    """Handle broken pipe errors gracefully"""
    # Reset signal handler
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
    
    # Close stdout without causing another error
    try:
        sys.stdout.close()
    except BrokenPipeError:
        pass
    
    # Use os._exit to exit immediately without traceback
    sys.exit(0)


def main():
    # Set up signal handlers for broken pipe
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
    
    parser = argparse.ArgumentParser(description="Process starred items into Forester format")
    parser.add_argument('--no-deduplicate', action='store_true', help='Disable deduplication of existing URLs')
    parser.add_argument('--tree-files', type=str, nargs='+', help='Paths to tree files to extract existing URLs from', default=default_tree_files())
    parser.add_argument('--show-all-sources', action='store_true', help='Show all sources, not just when both lobste.rs and HN are present')
    parser.add_argument('--days', type=int, default=7, help='Show only entries from the last X days (default: 7, use -1 for all days)')
    parser.add_argument('--rss-json', action='store_true', help='Process JSON output from rss2linkwarden.py instead of GitHub stars')
    args = parser.parse_args()

    try:
        # Read input
        input_text = sys.stdin.read()
        
        # Extract existing URLs from tree files if provided
        existing_urls = set()
        if args.tree_files and not args.no_deduplicate:
            for tree_file in args.tree_files:
                try:
                    with open(tree_file, 'r', encoding='utf-8') as f:
                        tree_content = f.read()
                        urls = extract_existing_urls_from_tree(tree_content)
                        existing_urls.update(urls)
                        print(f"Extracted {len(urls)} URLs from {tree_file}", file=sys.stderr)
                except Exception as e:
                    print(f"Warning: Could not read tree file {tree_file}: {e}", file=sys.stderr)
        
        # Process based on input type
        if args.rss_json:
            output = process_rss_json(input_text, existing_urls, not args.no_deduplicate, args.show_all_sources, args.days)
        else:
            output = process_stars(input_text, existing_urls, not args.no_deduplicate, args.show_all_sources, args.days)
        print(output)
    
    except BrokenPipeError:
        handle_broken_pipe()
    except KeyboardInterrupt:
        sys.exit(130)
    except Exception as e:
        print(f"Error: {e}", file=sys.stderr)
        sys.exit(1)


if __name__ == "__main__":
    main()