Skip to content

Add download dir and thread context options #65

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Clausinho wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add download dir and thread context options #65

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 28 additions & 4 deletions inb4404.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,21 @@ def configure_logging(args):
else:
logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(message)s', datefmt='%I:%M:%S %p')

def save_thread_context(directory, board, thread_id, html_content):
html_path = os.path.join(directory, thread_id + '.html')
json_path = os.path.join(directory, thread_id + '.json')
api_url = 'https://a.4cdn.org/' + board + '/thread/' + thread_id + '.json'

with open(html_path, 'w', encoding='utf-8') as f:
f.write(html_content)

try:
thread_json = json.loads(load(api_url).decode('utf-8'))
with open(json_path, 'w', encoding='utf-8') as f:
json.dump(thread_json, f, indent=2, ensure_ascii=False)
except (urllib.error.URLError, urllib.error.HTTPError, http.client.BadStatusLine, http.client.IncompleteRead, http.client.RemoteDisconnected, json.JSONDecodeError) as e:
log.warning('Could not save thread JSON for %s/%s: %s', board, thread_id, e)

def main():
global args
parser = argparse.ArgumentParser(description='inb4404')
Expand All @@ -54,6 +69,8 @@ def main():
parser.add_argument( '--throttle', type=float, default=0.5, help='Delay in seconds between downloads in the same thread')
parser.add_argument( '--backoff', type=float, default=0.5, help='Delay in seconds by which throttle should increase on 429')
parser.add_argument( '--run-once', action='store_true', help='download images once and exit (do not loop)')
parser.add_argument( '--download-dir', default=None, help='store downloaded files in this directory instead of the default downloads/board/thread path')
parser.add_argument( '--save-thread-context', action='store_true', help='save thread HTML and 4chan API JSON in the download directory')
args = parser.parse_args()
configure_logging(args)

Expand Down Expand Up @@ -143,10 +160,16 @@ def download_thread(thread_link, args):
regex_result_len = len(regex_result)
regex_result_cnt = 1

directory = os.path.join(workpath, 'downloads', board, thread)
if args.download_dir:
directory = os.path.abspath(os.path.expanduser(args.download_dir))
else:
directory = os.path.join(workpath, 'downloads', board, thread)
if not os.path.exists(directory):
os.makedirs(directory)

if args.save_thread_context:
save_thread_context(directory, board, thread_id, html_result)

# Load downloaded tracking if enabled
downloaded_set = load_downloaded_list(directory) if args.track_downloaded else None

Expand Down Expand Up @@ -216,9 +239,10 @@ def download_thread(thread_link, args):
log.info('%s %s\'d', thread_link, str(ex2.code))
break
continue
except (urllib.error.URLError, http.client.BadStatusLine, http.client.IncompleteRead):
log.fatal(thread_link + ' crashed!')
raise
except (urllib.error.URLError, http.client.BadStatusLine, http.client.IncompleteRead, http.client.RemoteDisconnected) as ex:
log.warning('%s failed with %s; retrying', thread_link, type(ex).__name__)
time.sleep(10)
continue

if args.run_once:
break
Expand Down