v0.4.0: More Algorithms

Author: ProgrammingIncluded

CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## 0.3.1: User README Updates
+
+* Add example README
+* Update README
+* Add scroll algorithm selection
+
 ## 0.3.0: Remove Ads and Average Scrolling
 
 * Add average scrolling to compensate for scroll heights

README.md

@@ -1,15 +1,34 @@
-# Birdwatch: A Twitter Profile Snapshot Tool
 
-Birdwatch snapshots a profile page when given a URL or an exported list of `following` from the official Twitter exporter.
-This script is purely for the purposes of archival use-only.
+<p align="center">
+    <img height="418" src="logo.png" alt="Birdwatch Logo">
+</p>
 
-**Note, without logging in, you can only fetch a few posts from the profile.**
+[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
+
+# Birdwatch: A Twitter Profile Archival Tool
+
+
+Birdwatch snapshots a profile page when given a URL (or an exported `.js` list of `following` from the official Twitter exporter.)
+Supports UTF-8 text JSON files and image snapshots of each Twitter post!
+
+This script is purely for the purposes of archival use only.
+
+![Birdwatch](demo.gif)
+
+## Features
+
+* Stores metadata in json format for each specified twitter profile.
+* Neatly organizes tweets by user and takes a snapshot of each tweet.
+* Marks potential tweets that are self-retweeted.
+* Removes Tweet Ads.
+* Allows for manual login (use at own risk.)
 
 ## Usage
 
 ```bash
+# Install the requirements. Once only.
 python -m pip install -r requirements.py
-python birdwatch.py
+python birdwatch.py --url www.twitter.com/<profile>
 
 # For more help use:
 python birdwatch.py --help
@@ -32,3 +51,85 @@ Birdwatch generates the following in the snapshots folder:
                 9.png
                 tweets.json # Metadata of each screen-capped tweet.
 ```
+
+### Self Boosted Tweet Detection
+
+A self-boosted tweet is a tweet where the original author retweets.
+These types of tweets are marked with `potential_boost` as true in `tweets.json`.
+The script detects these by matching exact meta-datas e.g. duplicate posts.
+
+## Schemas
+
+Assume all data is UTF-8 compliant.
+
+### Input File
+
+These files are what the Twitter exporter should generate (`.js` file) from the users you are following:
+
+```json
+window.* = [
+    {
+        "following": {
+            "accountId": <id>,
+            "userLink": <url>
+        }
+        ...
+    }
+]
+```
+
+You can rename as json or specify via input flags to parse the file. `window.* =` is automatically removed by the script and is default generated by Twitter. However, you can also manually remove it to parse the file as JSON directly.
+
+### tweets.json
+
+```json
+[
+    {
+        "id": int,
+        "tag_text": str,
+        "name": str,
+        "handle" str,
+        "timestamp": str,
+        "tweet_text": str,
+        "retweet_count": str,
+        "like_count": str,
+        "reply_count": str,
+        "potential_boost":  bool
+    }
+]
+```
+
+Invalid string entries will be marked as "NULL".
+
+###  metadata.json
+
+```json
+{
+    "bio": str,
+    "name": str,
+    "username": str,
+    "location": str,
+    "website": str,
+    "join_date": str,
+    "following": str,
+    "followers": str
+}
+```
+
+Invalid string entries will be marked as "NULL".
+
+
+## Troubleshoot
+
+* My scraper terminates early?
+
+It is possible that either your images are taking sometime to load Consider using `-s` to adjust load-time.
+Or your scrolling height is too low / too high. Consider using `--scroll-algorithm` to adjust the type of algorithm
+Then passing in a value to the algorithm `--scroll-value`.
+
+Help has more information as to what `--scroll-value` encodes.
+
+## Future Updates
+
+* Support Running Multiple Sessions to Resume Per-Profile
+* Expand Images and Attachments to Archive Images

birdwatch.py

@@ -26,6 +26,59 @@
 from webdriver_manager.chrome import ChromeDriverManager
 from selenium.webdriver.support.ui import WebDriverWait
 
+LOGO = """
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,'''''''',,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,''',;;:;;:cllollllccc:;,'.',,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,'';:codxkkkkkkkkkkkkkkkkkkxocc:;',,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,;,'';coxkkkkkkkkkkkkkkkkkkkkkkkkkkkkxl:,',,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,'.,:dkkkkkkkkkkkkkkkkkxocdkkkkkkkkkkxdxxd:,',,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,'.';lxkkkkkkkkkolxkkkkkkkkl;:okkkkkkkkxxxddxxo;.',,,,,,,,,,,,,,,,,,
+,,,,,,,,'...,:cdxxxkkkkkkkkx;;xkkkkkkkko;cooxkkkkkkkxdocldxdl,',,,,,,,,,,,,,,,,,
+,,,,,,,,,. .',,;:oxkkkkkkkxl::dkkkkkkkko,cOkcdkxxxkkkxl;,;:lol:,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,''..:oxkkkkxdxkocl,:kkkkkxxkl,:ONklokxodkkkxc,;;;:coc,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,;oxkkkkkkolxx:dk::okkkkxxkl,cONNOccddcokkkl':lcccldc,,,,,,,,,,,,,,,
+,,,,,,,,,,,.:xkxxkkkkxllo:;kkodcoxkkxddc';ccll:..,,'lkkl,:lcclloo,',,,,,,,,,,,,,
+,,,,,,,,,'.,dxxxxkkkkdlxd,':;':c:oxkkc,,.;:,'....;. .cxc,lo:;cloo;.',,,,,,,,,,,,
+,,,,,,,,'.;ooldxxxkkkoll;;c:'.  ,l:cx:.'.;'     'Ox.':;;:oo:;ollol;',,,,,,,,,,,,
+,,,,,,,'..''''cxdxkkko,.lXx.    ;00xo:oKOx'     ;XO'lk:cooo;dNdcoodl,,,;,,,,,,,,
+,,,,,,,,''',,',oc;dkkl..kWo     ;XWWWXNWMNo    .xWOdOocoool;d0oloodxdc,',,,,,,,,
+,,,,,,,,,,,,,,',..;dkl..dWO.    lNMWWWWWWWXl..,dNWWXdcloooc,:lloooodxxdc'.,,,,,,
+,,,,,,,,,,,,,,;,.',,::coONNc   :KWWWWWWWWWWWXKKXNNXdcloooo;,looooooooddxd:.',,,,
+,,,,,,,,,,,,,,,;,.,;'.'kWWWKdokXWWWWNNWWWWWWWWNNNXd:looooc,;clooolcooo::ox:.,;,,
+,,,,,,,,,,,,,,,'.';;;.;0NXXNNWWWWWWWXNWWWWWWWWWXKd:looooc,:c,:doo;.,ll',oo;.,;,,
+,,,,,,,,,,,,''''',;:;..cxk0KXNNNNNXXNNNKOkxollcc,'cooooc,:oc.'odl'.',,.;o;.',,,,
+,,,,,,,,,,,...''.':c;';c;,,:ool:,,,;ldxl,',;,;lc';oool;;:c;...:l,.,;;'',,',,,,,,
+,,,,,,,,,,,,,,,,,',;',::,..,lc'.......;oddkXKd;..;l:,.',,'',;,''.'''...',,,,,,,,
+,,,,,,,,,,,,,,,,,,'...   ..,dxd:,.. .:xkkONWWWO;'clc'.......,;::cll.    .,,,,,,,
+,,,,,,,,,,,,,,,,,,.,ooc'...'O0:'...  ':okOXWWWWl;o;.    ....kOddo0K,    .,,,,,,,
+,,,,,,,,,,,,,,,,'.'xWX0Od, .d:.. ......;dk0kxOO'.'.     ...:xc,;;k0,     ',,,,,,
+,,,,,,,,,,,,,,,. ;xONWXOkl..d:.  ,::,. ,oodd,..    .     .xXX0O00XX;     .;,,,,,
+,,,,,,,,,,,,,,,. ;OOXWXk:'..dk,.  .. .,kXOOO,      .     .kWNWWWWWNc     .,,,,,,
+,,,,,,,,,,,,,,,. .xkOK0k, .'lOkc,'..;oO00xOK,      .      dWWWWWWNWd.    .,,,,,,
+,,,,,,,,,,,,,,,'. ;kkxxko.'xOOOOOOkOOOOOOdOK:. .;;..      :XWWWWNXKx.    .,,,,,,
+,,,,,,,,,,,,,,,,,..;odk00d,'d00KKKK0KKK0OkKKdxxoo, .      .;lccc:;,'....',,,,,,,
+,,,,,,,,,,,,,,,,,,;';oxkxxl..looooooooollll:'..   ''........''',,,;;,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,;,'',.                   ..lXO;,;,;,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,......         ...',,;xKo',,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,;;,,,,,''',,,,,,,,,,;;'',,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,;,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+
+"""
+TITLE = """
+ ____  _         _               _       _     
+|  _ \(_)       | |             | |     | |    
+| |_) |_ _ __ __| |_      ____ _| |_ ___| |__  
+|  _ <| | '__/ _` \ \ /\ / / _` | __/ __| '_ \ 
+| |_) | | | | (_| |\ V  V / (_| | || (__| | | |
+|____/|_|_|  \__,_| \_/\_/ \__,_|\__\___|_| |_|
+                                               
+"""
+
 SCRAPE_N_TWEETS = 20
 IS_DEBUG = False
 
@@ -77,13 +130,26 @@ def remove_elements(driver, elements, remove_parent=True):
     }}
     """.format(",".join(elements)))
 
-def calc_average(lst):
-    if len(lst) < 4:
-        return sum(lst) / len(lst)
-
-    cut_off = int(len(lst) * 0.25)
-    s = sorted(lst)[cut_off:len(lst) - cut_off]
-    return sum(s) / len(s)
+def calc_average(percentage):
+    def _func(lst):
+        if len(lst) < 4:
+            return sum(lst) / len(lst)
+    
+        cut_off = int(len(lst) * percentage)
+        s = sorted(lst)[cut_off:len(lst) - cut_off]
+        return sum(s) / len(s)
+    return _func
+
+def window_average(window):
+    def _func(lst):
+        v = lst[:-min(window, len(lst))]
+        return sum(v) / len(v)
+    return _func
+
+def constant(const):
+    def _func(lst):
+        return const
+    return _func
 
 def remove_ads(driver):
     return driver.execute_script("""
@@ -99,7 +165,7 @@ def remove_ads(driver):
     """)
 
 
-def fetch_html(driver, url, fpath, load_times, force=False, number_posts_to_cap=SCRAPE_N_TWEETS, bio_only=False):
+def fetch_html(driver, url, fpath, load_times, offset_func, force=False, number_posts_to_cap=SCRAPE_N_TWEETS, bio_only=False):
     driver.get(url)
     state = ""
     while state != "complete":
@@ -253,7 +319,7 @@ def fetch_html(driver, url, fpath, load_times, force=False, number_posts_to_cap=
                     break
 
             # Scroll!
-            driver.execute_script("window.scrollTo(0, {});".format(estimated_height + calc_average(height_diffs)))
+            driver.execute_script("window.scrollTo(0, {});".format(estimated_height + offset_func(height_diffs)))
             time.sleep(random.uniform(load_times, load_times + 2))
             new_height = driver.execute_script("return document.body.scrollHeight")
             if new_height == last_height:
@@ -277,8 +343,13 @@ def parse_args():
     parser.add_argument("--posts", "-p", help="Max number of posts to screenshot.", default=SCRAPE_N_TWEETS, type=int)
     parser.add_argument("--bio-only", "-b", help="Only store bio, no snapshots or tweets.", action="store_true")
     parser.add_argument("--debug", help="Print debug output.", action="store_true")
-    parser.add_argument("--login", help="Prompt user login to remove tweet limit..", action="store_true")
+    parser.add_argument("--login", help="Prompt user login to remove limits / default filters. USE AT OWN RISK.", action="store_true")
     parser.add_argument("--scroll-load-time", "-s", help="Number of seconds (float). The higher, the stabler the fetch.", default=5, type=int)
+    parser.add_argument("--scroll-algorithm", help="Type of algorithm to calculate scroll offset.", choices=["percentile", "window", "constant"], default="window")
+    parser.add_argument("--scroll-value", default=5, type=float, help=("Value used by --scroll-algorithm."
+                                                                        "If percentile, percentage of percentile calculated. "
+                                                                        "If window, the size of window average."
+                                                                        "If constant, size of pixel to scroll by."))
 
     group = parser.add_mutually_exclusive_group()
     group.add_argument("--input-json", "-i", help="Input json file", default="input.json")
@@ -290,9 +361,22 @@ def main():
     global IS_DEBUG
     IS_DEBUG = args.debug
 
+    print(LOGO)
+    print(TITLE)
+
     output_folder = "snapshots"
     os.makedirs(output_folder, exist_ok=True)
     extra_args = {"force": args.force, "bio_only": args.bio_only, "load_times": args.scroll_load_time, "number_posts_to_cap": args.posts}
+    f = None
+    if args.scroll_algorithm == "percentile":
+        assert args.scroll_value <= 1.0 and args.scroll_value >= 0.0
+        f = calc_average(args.scroll_value)
+    elif args.scroll_algorithm == "window":
+        f = window_average(args.scroll_value)
+    else:
+        f = constant(args.scroll_value)
+
+    extra_args["offset_func"] = f
 
     driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
     if args.login: