main.py

import argparse

from douban_movie_scraper.crawler import DoubanMovieCrawler
from douban_movie_scraper.pipeline import DoubanMoviePipeline


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Crawl latest short comments for Douban Movie Top250."
    )
    parser.add_argument("--top-m", type=int, default=5, help="Top m movies to crawl.")
    parser.add_argument("--pages", type=int, default=1, help="Comment pages per movie.")
    parser.add_argument(
        "--output-dir",
        default="data/douban_movie_top250",
        help="Directory for per-movie JSON files.",
    )
    parser.add_argument(
        "--cookie",
        default="",
        help="Douban cookie. You can also fill DOUBAN_COOKIE in config.py.",
    )
    parser.add_argument(
        "--rating-strategy",
        choices=("drop", "zero", "none"),
        default="drop",
        help="How to handle comments without rating.",
    )
    parser.add_argument(
        "--comment-sort",
        default="time",
        help="Douban comment sort, default time for latest comments.",
    )
    parser.add_argument("--sleep", type=float, default=1.0, help="Request interval in seconds.")
    return parser.parse_args()


def main() -> None:
    args = parse_args()
    crawler = DoubanMovieCrawler(cookie=args.cookie, sleep_seconds=args.sleep)
    pipeline = DoubanMoviePipeline(
        output_dir=args.output_dir,
        rating_strategy=args.rating_strategy,
    )

    try:
        movies = crawler.fetch_top_movies(args.top_m)
        saved_paths = []
        for movie in movies:
            movie.comment_list = crawler.fetch_movie_comments(
                movie.movie_id,
                args.pages,
                comment_sort=args.comment_sort,
            )
            movie = pipeline.process_movie(movie)
            saved_paths.append(pipeline.save_movie(movie))
    except RuntimeError as error:
        raise SystemExit(f"crawl_failed={error}") from error

    index_path = pipeline.save_index(movies)
    print(f"crawled_movies={len(movies)}")
    print(f"saved_index={index_path}")
    for path in saved_paths:
        print(f"saved_movie={path}")


if __name__ == "__main__":
    main()