diff --git a/inb4404.py b/inb4404.py index 06bcd34..7fc3821 100755 --- a/inb4404.py +++ b/inb4404.py @@ -36,6 +36,21 @@ def configure_logging(args): else: logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(message)s', datefmt='%I:%M:%S %p') +def save_thread_context(directory, board, thread_id, html_content): + html_path = os.path.join(directory, thread_id + '.html') + json_path = os.path.join(directory, thread_id + '.json') + api_url = 'https://a.4cdn.org/' + board + '/thread/' + thread_id + '.json' + + with open(html_path, 'w', encoding='utf-8') as f: + f.write(html_content) + + try: + thread_json = json.loads(load(api_url).decode('utf-8')) + with open(json_path, 'w', encoding='utf-8') as f: + json.dump(thread_json, f, indent=2, ensure_ascii=False) + except (urllib.error.URLError, urllib.error.HTTPError, http.client.BadStatusLine, http.client.IncompleteRead, http.client.RemoteDisconnected, json.JSONDecodeError) as e: + log.warning('Could not save thread JSON for %s/%s: %s', board, thread_id, e) + def main(): global args parser = argparse.ArgumentParser(description='inb4404') @@ -54,6 +69,8 @@ def main(): parser.add_argument( '--throttle', type=float, default=0.5, help='Delay in seconds between downloads in the same thread') parser.add_argument( '--backoff', type=float, default=0.5, help='Delay in seconds by which throttle should increase on 429') parser.add_argument( '--run-once', action='store_true', help='download images once and exit (do not loop)') + parser.add_argument( '--download-dir', default=None, help='store downloaded files in this directory instead of the default downloads/board/thread path') + parser.add_argument( '--save-thread-context', action='store_true', help='save thread HTML and 4chan API JSON in the download directory') args = parser.parse_args() configure_logging(args) @@ -143,10 +160,16 @@ def download_thread(thread_link, args): regex_result_len = len(regex_result) regex_result_cnt = 1 - directory = os.path.join(workpath, 'downloads', board, thread) + if args.download_dir: + directory = os.path.abspath(os.path.expanduser(args.download_dir)) + else: + directory = os.path.join(workpath, 'downloads', board, thread) if not os.path.exists(directory): os.makedirs(directory) + if args.save_thread_context: + save_thread_context(directory, board, thread_id, html_result) + # Load downloaded tracking if enabled downloaded_set = load_downloaded_list(directory) if args.track_downloaded else None @@ -216,9 +239,10 @@ def download_thread(thread_link, args): log.info('%s %s\'d', thread_link, str(ex2.code)) break continue - except (urllib.error.URLError, http.client.BadStatusLine, http.client.IncompleteRead): - log.fatal(thread_link + ' crashed!') - raise + except (urllib.error.URLError, http.client.BadStatusLine, http.client.IncompleteRead, http.client.RemoteDisconnected) as ex: + log.warning('%s failed with %s; retrying', thread_link, type(ex).__name__) + time.sleep(10) + continue if args.run_once: break