diff --git a/advertools/code_recipes/spider_strategies.py b/advertools/code_recipes/spider_strategies.py index 7d4066d4..45c939ff 100644 --- a/advertools/code_recipes/spider_strategies.py +++ b/advertools/code_recipes/spider_strategies.py @@ -383,6 +383,9 @@ Apple TV 4th Gen AppleTV5,3/9.1.1 Apple TV 5th Gen 4K AppleTV6,2/11.1 Apple TV 6th Gen 4K AppleTV11,1/11.1 +Apple iPhone 11 Mozilla/5.0 (iPhone12,1; U; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/15E148 Safari/602.1 +Apple iPhone 12 Mozilla/5.0 (iPhone13,2; U; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/15E148 Safari/602.1 +Apple iPhone 13 Pro Max Mozilla/5.0 (iPhone14,3; U; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/19A346 Safari/602.1 Apple iPhone 6 Mozilla/5.0 (Apple-iPhone7C2/1202.466; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543 Safari/419.3 Apple iPhone 7 Mozilla/5.0 (iPhone9,3; U; CPU iPhone OS 10_0_1 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A403 Safari/602.1 Apple iPhone 7 Plus Mozilla/5.0 (iPhone9,4; U; CPU iPhone OS 10_0_1 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A403 Safari/602.1 @@ -450,12 +453,6 @@ Xbox One S Mozilla/5.0 (Windows NT 10.0; Win64; x64; XBOX_ONE_ED) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393 Xbox Series X Mozilla/5.0 (Windows NT 10.0; Win64; x64; Xbox; Xbox Series X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.82 Safari/537.36 Edge/20.02 Yahoo! bot Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp) -iPhone 11 Mozilla/5.0 (iPhone12,1; U; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/15E148 Safari/602.1 -iPhone 11 Mozilla/5.0 (iPhone12,1; U; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/15E148 Safari/602.1 -iPhone 12 Mozilla/5.0 (iPhone13,2; U; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/15E148 Safari/602.1 -iPhone 13 Pro Max Mozilla/5.0 (iPhone14,3; U; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/19A346 Safari/602.1 ======================================================== ========================================================================================================================================================================= - - """ # noqa: E501 diff --git a/docs/_build/doctrees/advertools.code_recipes.spider_strategies.doctree b/docs/_build/doctrees/advertools.code_recipes.spider_strategies.doctree index b4c6b31c..871d8752 100644 Binary files a/docs/_build/doctrees/advertools.code_recipes.spider_strategies.doctree and b/docs/_build/doctrees/advertools.code_recipes.spider_strategies.doctree differ diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index 3dd6aee7..887d6a8c 100644 Binary files a/docs/_build/doctrees/environment.pickle and b/docs/_build/doctrees/environment.pickle differ diff --git a/docs/_build/html/advertools.code_recipes.spider_strategies.html b/docs/_build/html/advertools.code_recipes.spider_strategies.html index a9787e94..200ece2a 100644 --- a/docs/_build/html/advertools.code_recipes.spider_strategies.html +++ b/docs/_build/html/advertools.code_recipes.spider_strategies.html @@ -677,219 +677,216 @@

User-agent strings for use in crawling

Apple TV 6th Gen 4K

AppleTV11,1/11.1

-

Apple iPhone 6

+

Apple iPhone 11

+

Mozilla/5.0 (iPhone12,1; U; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/15E148 Safari/602.1

+ +

Apple iPhone 12

+

Mozilla/5.0 (iPhone13,2; U; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/15E148 Safari/602.1

+ +

Apple iPhone 13 Pro Max

+

Mozilla/5.0 (iPhone14,3; U; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/19A346 Safari/602.1

+ +

Apple iPhone 6

Mozilla/5.0 (Apple-iPhone7C2/1202.466; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543 Safari/419.3

-

Apple iPhone 7

+

Apple iPhone 7

Mozilla/5.0 (iPhone9,3; U; CPU iPhone OS 10_0_1 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A403 Safari/602.1

-

Apple iPhone 7 Plus

+

Apple iPhone 7 Plus

Mozilla/5.0 (iPhone9,4; U; CPU iPhone OS 10_0_1 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A403 Safari/602.1

-

Apple iPhone 8

+

Apple iPhone 8

Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) Version/11.0 Mobile/15A5341f Safari/604.1

-

Apple iPhone 8 Plus

+

Apple iPhone 8 Plus

Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A5370a Safari/604.1

-

Apple iPhone SE (3rd generation)

+

Apple iPhone SE (3rd generation)

Mozilla/5.0 (iPhone14,6; U; CPU iPhone OS 15_4 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/19E241 Safari/602.1

-

Apple iPhone X

+

Apple iPhone X

Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1

-

Apple iPhone XR (Safari)

+

Apple iPhone XR (Safari)

Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1

-

Apple iPhone XS (Chrome)

+

Apple iPhone XS (Chrome)

Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/69.0.3497.105 Mobile/15E148 Safari/605.1

-

Apple iPhone XS Max (Firefox)

+

Apple iPhone XS Max (Firefox)

Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/13.2b11866 Mobile/16A366 Safari/605.1.15

-

Bing bot

+

Bing bot

Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)

-

Chrome OS-based laptop using Chrome browser (Chromebook)

+

Chrome OS-based laptop using Chrome browser (Chromebook)

Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36

-

Chromecast

+

Chromecast

Mozilla/5.0 (CrKey armv7l 1.5.16041) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.0 Safari/537.36

-

Google ADT-2

+

Google ADT-2

Dalvik/2.1.0 (Linux; U; Android 9; ADT-2 Build/PTT5.181126.002)

-

Google Nexus Player

+

Google Nexus Player

Dalvik/2.1.0 (Linux; U; Android 6.0.1; Nexus Player Build/MMB29T)

-

Google Pixel

+

Google Pixel

Mozilla/5.0 (Linux; Android 7.1.1; Google Pixel Build/NMF26F; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/54.0.2840.85 Mobile Safari/537.36

-

Google Pixel 2

+

Google Pixel 2

Mozilla/5.0 (Linux; Android 8.0.0; Pixel 2 Build/OPD1.170811.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/59.0.3071.125 Mobile Safari/537.36

-

Google Pixel 3

+

Google Pixel 3

Mozilla/5.0 (Linux; Android 10; Google Pixel 4 Build/QD1A.190821.014.C2; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.108 Mobile Safari/537.36

-

Google Pixel 4

+

Google Pixel 4

Mozilla/5.0 (Linux; Android 10; Google Pixel 4 Build/QD1A.190821.014.C2; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.108 Mobile Safari/537.36

-

Google Pixel 5

+

Google Pixel 5

Mozilla/5.0 (Linux; Android 11; Pixel 5 Build/RQ3A.210805.001.A1; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/92.0.4515.159 Mobile Safari/537.36

-

Google Pixel 6

+

Google Pixel 6

Mozilla/5.0 (Linux; Android 12; Pixel 6 Build/SD1A.210817.023; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/94.0.4606.71 Mobile Safari/537.36

-

Google Pixel C

+

Google Pixel C

Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.98 Safari/537.36

-

Google bot

+

Google bot

Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)

-

HTC Desire 21 Pro 5G

+

HTC Desire 21 Pro 5G

Mozilla/5.0 (Linux; Android 10; HTC Desire 21 pro 5G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.127 Mobile Safari/537.36

-

HTC One M9

+

HTC One M9

Mozilla/5.0 (Linux; Android 6.0; HTC One M9 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.3

-

HTC One X10

+

HTC One X10

Mozilla/5.0 (Linux; Android 6.0; HTC One X10 Build/MRA58K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/61.0.3163.98 Mobile Safari/537.36

-

HTC U20 5G

+

HTC U20 5G

Mozilla/5.0 (Linux; Android 10; Wildfire U20 5G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.136 Mobile Safari/537.36

-

LG G Pad 7.0

+

LG G Pad 7.0

Mozilla/5.0 (Linux; Android 5.0.2; LG-V410/V41020c Build/LRX22G) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/34.0.1847.118 Safari/537.36

-

Lenovo Yoga Tab 11

+

Lenovo Yoga Tab 11

Mozilla/5.0 (Linux; Android 11; Lenovo YT-J706X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36

-

Linux-based PC using a Firefox browser

+

Linux-based PC using a Firefox browser

Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1

-

Mac OS X-based computer using a Safari browser

+

Mac OS X-based computer using a Safari browser

Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9

-

Microsoft Lumia 550

+

Microsoft Lumia 550

Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; RM-1127_16056) AppleWebKit/537.36(KHTML, like Gecko) Chrome/42.0.2311.135 Mobile Safari/537.36 Edge/12.10536

-

Microsoft Lumia 650

+

Microsoft Lumia 650

Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254

-

Microsoft Lumia 950

+

Microsoft Lumia 950

Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; Lumia 950) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Mobile Safari/537.36 Edge/13.1058

-

Minix NEO X5

+

Minix NEO X5

Mozilla/5.0 (Linux; U; Android 4.2.2; he-il; NEO-X5-116A Build/JDQ39) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Safari/534.30

-

Nexus 6P

+

Nexus 6P

Mozilla/5.0 (Linux; Android 6.0.1; Nexus 6P Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.83 Mobile Safari/537.36

-

Nintendo 3DS

+

Nintendo 3DS

Mozilla/5.0 (Nintendo 3DS; U; ; en) Version/1.7412.EU

-

Nintendo Switch

+

Nintendo Switch

Mozilla/5.0 (Nintendo Switch; WifiWebAuthApplet) AppleWebKit/601.6 (KHTML, like Gecko) NF/4.0.0.5.10 NintendoBrowser/5.1.0.13343

-

Nintendo Wii U

+

Nintendo Wii U

Mozilla/5.0 (Nintendo WiiU) AppleWebKit/536.30 (KHTML, like Gecko) NX/3.0.4.2.12 NintendoBrowser/4.3.1.11264.US

-

Nvidia Shield Tablet K1

+

Nvidia Shield Tablet K1

Mozilla/5.0 (Linux; Android 6.0.1; SHIELD Tablet K1 Build/MRA58K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/55.0.2883.91 Safari/537.36

-

Playstation 4

+

Playstation 4

Mozilla/5.0 (PlayStation 4 3.11) AppleWebKit/537.73 (KHTML, like Gecko)

-

Playstation 5

+

Playstation 5

Mozilla/5.0 (PlayStation; PlayStation 5/2.26) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Safari/605.1.15

-

Playstation Vita

+

Playstation Vita

Mozilla/5.0 (PlayStation Vita 3.61) AppleWebKit/537.73 (KHTML, like Gecko) Silk/3.2

-

Roku Ultra

+

Roku Ultra

Roku4640X/DVP-7.70 (297.70E04154A)

-

Samsung Galaxy S10

+

Samsung Galaxy S10

Mozilla/5.0 (Linux; Android 9; SM-G973U Build/PPR1.180610.011) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Mobile Safari/537.36

-

Samsung Galaxy S20

+

Samsung Galaxy S20

Mozilla/5.0 (Linux; Android 10; SM-G980F Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.96 Mobile Safari/537.36

-

Samsung Galaxy S21

+

Samsung Galaxy S21

Mozilla/5.0 (Linux; Android 10; SM-G996U Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Mobile Safari/537.36

-

Samsung Galaxy S22

+

Samsung Galaxy S22

Mozilla/5.0 (Linux; Android 12; SM-S906N Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/80.0.3987.119 Mobile Safari/537.36

-

Samsung Galaxy S6

+

Samsung Galaxy S6

Mozilla/5.0 (Linux; Android 6.0.1; SM-G920V Build/MMB29K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.36

-

Samsung Galaxy S6 Edge Plus

+

Samsung Galaxy S6 Edge Plus

Mozilla/5.0 (Linux; Android 5.1.1; SM-G928X Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.83 Mobile Safari/537.36

-

Samsung Galaxy S7

+

Samsung Galaxy S7

Mozilla/5.0 (Linux; Android 7.0; SM-G930VC Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/58.0.3029.83 Mobile Safari/537.36

-

Samsung Galaxy S7 Edge

+

Samsung Galaxy S7 Edge

Mozilla/5.0 (Linux; Android 6.0.1; SM-G935S Build/MMB29K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/55.0.2883.91 Mobile Safari/537.36

-

Samsung Galaxy S8

+

Samsung Galaxy S8

Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Mobile Safari/537.36

-

Samsung Galaxy S9

+

Samsung Galaxy S9

Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36

-

Samsung Galaxy Tab A

+

Samsung Galaxy Tab A

Mozilla/5.0 (Linux; Android 5.0.2; SAMSUNG SM-T550 Build/LRX22G) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.3 Chrome/38.0.2125.102 Safari/537.36

-

Samsung Galaxy Tab S3

+

Samsung Galaxy Tab S3

Mozilla/5.0 (Linux; Android 7.0; SM-T827R4 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.116 Safari/537.36

-

Samsung Galaxy Tab S8 Ultra

+

Samsung Galaxy Tab S8 Ultra

Mozilla/5.0 (Linux; Android 12; SM-X906C Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/80.0.3987.119 Mobile Safari/537.36

-

Sony Xperia 1

+

Sony Xperia 1

Mozilla/5.0 (Linux; Android 9; J8110 Build/55.0.A.0.552; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36

-

Sony Xperia XZ

+

Sony Xperia XZ

Mozilla/5.0 (Linux; Android 7.1.1; G8231 Build/41.2.A.0.219; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/59.0.3071.125 Mobile Safari/537.36

-

Sony Xperia Z4 Tablet

+

Sony Xperia Z4 Tablet

Mozilla/5.0 (Linux; Android 6.0.1; SGP771 Build/32.2.A.0.253; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.98 Safari/537.36

-

Sony Xperia Z5

+

Sony Xperia Z5

Mozilla/5.0 (Linux; Android 6.0.1; E6653 Build/32.2.A.0.253) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.36

-

Windows 10-based PC using Edge browser

+

Windows 10-based PC using Edge browser

Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246

-

Windows 7-based PC using a Chrome browser

+

Windows 7-based PC using a Chrome browser

Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36

-

Xbox One

+

Xbox One

Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Xbox; Xbox One) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Mobile Safari/537.36 Edge/13.10586

-

Xbox One S

+

Xbox One S

Mozilla/5.0 (Windows NT 10.0; Win64; x64; XBOX_ONE_ED) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393

-

Xbox Series X

+

Xbox Series X

Mozilla/5.0 (Windows NT 10.0; Win64; x64; Xbox; Xbox Series X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.82 Safari/537.36 Edge/20.02

-

Yahoo! bot

+

Yahoo! bot

Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)

-

iPhone 11

-

Mozilla/5.0 (iPhone12,1; U; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/15E148 Safari/602.1

- -

iPhone 11

-

Mozilla/5.0 (iPhone12,1; U; CPU iPhone OS 13_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/15E148 Safari/602.1

- -

iPhone 12

-

Mozilla/5.0 (iPhone13,2; U; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/15E148 Safari/602.1

- -

iPhone 13 Pro Max

-

Mozilla/5.0 (iPhone14,3; U; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/19A346 Safari/602.1

- diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index f5411d46..51348a4b 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"docnames": ["advertools", "advertools.ad_create", "advertools.ad_from_string", "advertools.cli", "advertools.cli.cli", "advertools.code_recipes", "advertools.code_recipes.spider_strategies", "advertools.crawlytics", "advertools.emoji", "advertools.extract", "advertools.header_spider", "advertools.image_spider", "advertools.knowledge_graph", "advertools.kw_generate", "advertools.logs", "advertools.regex", "advertools.reverse_dns_lookup", "advertools.robotstxt", "advertools.serp", "advertools.sitemaps", "advertools.spider", "advertools.stopwords", "advertools.survey", "advertools.twitter", "advertools.url_builders", "advertools.urlytics", "advertools.word_frequency", "advertools.word_tokenize", "advertools.youtube", "include_changelog", "index", "modules", "readme"], "filenames": ["advertools.rst", "advertools.ad_create.rst", "advertools.ad_from_string.rst", "advertools.cli.rst", "advertools.cli.cli.rst", "advertools.code_recipes.rst", "advertools.code_recipes.spider_strategies.rst", "advertools.crawlytics.rst", "advertools.emoji.rst", "advertools.extract.rst", "advertools.header_spider.rst", "advertools.image_spider.rst", "advertools.knowledge_graph.rst", "advertools.kw_generate.rst", "advertools.logs.rst", "advertools.regex.rst", "advertools.reverse_dns_lookup.rst", "advertools.robotstxt.rst", "advertools.serp.rst", "advertools.sitemaps.rst", "advertools.spider.rst", "advertools.stopwords.rst", "advertools.survey.rst", "advertools.twitter.rst", "advertools.url_builders.rst", "advertools.urlytics.rst", "advertools.word_frequency.rst", "advertools.word_tokenize.rst", "advertools.youtube.rst", "include_changelog.rst", "index.rst", "modules.rst", "readme.rst"], "titles": ["advertools package", "Create Ads on a Large Scale", "Create Ads Using Long Descriptive Text (top-down approach)", "advertools.cli module", "advertools Command Line Interface (CLI)", "advertools.code_recipes package", "\ud83d\udd77 SEO Crawling & Scraping: Strategies & Recipes", "Crawling and Scraping Analysis", "Emoji: Extract, Analyze, and Get Insights", "Extract structured entities from text lists", "\ud83d\udd77 Python Status Code Checker with Response Headers", "Image Crawler and Downloader", "Import and Analyze Knowledge Graph Results on a Large Scale", "Generate Keywords for SEM Campaigns", "Log File Analysis", "Regular Expressions for Extracting Structured Entities", "Reverse DNS Lookup in Bulk", "\ud83e\udd16 Analyze and Test robots.txt Files on a Large Scale", "Import Search Engine Results Pages (SERPs) for Google and YouTube", "Download, Parse, and Analyze XML Sitemaps", "\ud83d\udd77 Python SEO Crawler / Spider", "Stopwords in Several Languages", "Survey - share feedback", "Twitter Data API", "URL Builders", "Split, Parse, and Analyze URL Structure", "Text Analysis", "Tokenize Words (N-grams)", "YouTube Data API", "advertools", "advertools", "advertools", "advertools: productivity & analysis tools to scale your online marketing"], "terms": {"cli": [0, 29, 30, 31], "code_recip": [0, 29, 30, 31], "seo": [0, 5, 12, 13, 17, 19, 26, 29, 31], "crawl": [0, 5, 10, 11, 16, 17, 25, 29, 30, 31, 32], "scrape": [0, 5, 14, 20, 29, 30, 31], "strategi": [0, 5, 7, 10, 11, 13, 17, 19, 20, 28, 30], "recip": [0, 5, 20, 29, 30], "how": [0, 2, 5, 7, 8, 9, 10, 13, 17, 19, 20, 23, 25, 26, 29, 30, 31, 32], "list": [0, 1, 2, 5, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 21, 23, 25, 26, 27, 28, 29, 30, 31, 32], "page": [0, 2, 4, 5, 7, 9, 10, 11, 12, 13, 14, 17, 19, 21, 23, 25, 26, 28, 29, 30, 31, 32], "those": [0, 2, 4, 5, 7, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 25, 26, 28, 29, 30, 32], "onli": [0, 1, 4, 5, 7, 9, 10, 11, 14, 15, 17, 18, 19, 20, 23, 24, 26, 28, 29, 30, 32], "mode": [0, 4, 5, 9, 23, 29, 30, 31], "can": [0, 2, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29, 30, 32], "i": [0, 1, 2, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29, 30, 32], "websit": [0, 4, 5, 12, 14, 17, 18, 19, 20, 25, 28, 29, 30, 31, 32], "includ": [0, 4, 5, 7, 9, 11, 13, 14, 17, 18, 19, 20, 23, 25, 27, 28, 29, 30], "its": [0, 1, 4, 5, 7, 8, 10, 12, 13, 14, 19, 20, 23, 25, 26, 28, 30, 32], "sub": [0, 4, 5, 7, 8, 9, 11, 14, 17, 19, 20, 23, 25, 29, 30, 32], "domain": [0, 4, 5, 7, 10, 14, 16, 17, 18, 20, 25, 29, 30], "save": [0, 5, 7, 10, 11, 14, 17, 20, 25, 29, 30, 32], "copi": [0, 5, 30, 32], "log": [0, 5, 16, 18, 19, 20, 25, 28, 31, 32], "my": [0, 1, 5, 9, 20, 26, 30], "audit": [0, 5, 17, 19, 20, 30], "them": [0, 2, 4, 5, 7, 9, 10, 11, 13, 14, 16, 17, 19, 20, 23, 25, 26, 29, 30, 32], "later": [0, 5, 14, 19, 30], "automat": [0, 5, 18, 28, 30, 32], "stop": [0, 4, 5, 20, 26, 29, 30, 32], "base": [0, 2, 5, 8, 10, 11, 15, 18, 20, 23, 26, 28, 29, 30, 32], "certain": [0, 4, 5, 7, 9, 10, 12, 14, 16, 17, 19, 20, 21, 25, 26, 29, 30], "condit": [0, 5, 11, 14, 20, 30], "di": [0, 5, 11, 30], "obei": [0, 5, 11, 14, 30], "robot": [0, 5, 11, 14, 19, 20, 29, 30, 31, 32], "txt": [0, 5, 8, 11, 14, 19, 20, 29, 30, 31, 32], "rule": [0, 5, 10, 11, 14, 17, 20, 25, 30, 32], "do": [0, 2, 4, 5, 7, 9, 10, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 28, 29, 30, 32], "set": [0, 2, 4, 5, 7, 10, 11, 12, 13, 14, 18, 19, 23, 25, 26, 27, 28, 29, 30, 31, 32], "user": [0, 4, 5, 9, 11, 13, 14, 16, 18, 20, 23, 28, 29, 30, 31, 32], "agent": [0, 5, 10, 11, 14, 16, 20, 29, 30, 31, 32], "while": [0, 4, 5, 7, 14, 18, 23, 25, 26, 28, 29, 30, 31], "control": [0, 5, 10, 11, 18, 20, 29, 30, 32], "number": [0, 5, 7, 8, 12, 13, 16, 17, 18, 19, 20, 23, 26, 28, 29, 30, 31, 32], "concurr": [0, 5, 18, 28, 29, 30], "request": [0, 5, 7, 10, 11, 12, 14, 16, 18, 20, 23, 28, 29, 30, 32], "slow": [0, 5, 30], "down": [0, 5, 13, 14, 26, 29, 30, 31, 32], "so": [0, 2, 5, 7, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 30, 32], "don": [0, 1, 4, 5, 7, 9, 10, 14, 18, 20, 25, 26, 29, 30, 32], "t": [0, 1, 4, 5, 7, 9, 10, 13, 14, 18, 20, 25, 26, 27, 29, 30, 32], "hit": [0, 5, 10, 20, 30], "server": [0, 5, 10, 14, 16, 18, 20, 28, 30], "too": [0, 5, 9, 20, 26, 30, 32], "hard": [0, 5, 30], "multipl": [0, 4, 5, 7, 9, 12, 17, 18, 19, 20, 29, 30, 32], "same": [0, 1, 2, 4, 5, 7, 8, 9, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 29, 30, 32], "job": [0, 5, 13, 30, 32], "want": [0, 1, 2, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 30, 32], "follow": [0, 2, 4, 5, 7, 8, 9, 14, 18, 19, 21, 23, 25, 26, 28, 29, 30, 31, 32], "link": [0, 4, 5, 9, 10, 18, 19, 23, 25, 28, 29, 30, 31, 32], "from": [0, 5, 7, 11, 12, 14, 15, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 30, 31, 32], "specifi": [0, 2, 5, 7, 9, 12, 14, 17, 18, 20, 23, 27, 28, 29, 30, 32], "depth": [0, 5, 10, 20, 30], "paus": [0, 5, 30], "resum": [0, 5, 30, 32], "make": [0, 1, 2, 5, 7, 9, 10, 14, 16, 17, 18, 19, 20, 23, 25, 26, 29, 30, 32], "sure": [0, 1, 2, 5, 9, 10, 16, 18, 20, 23, 25, 29, 30], "twice": [0, 5, 26, 29, 30], "us": [0, 1, 5, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 28, 29, 30, 31, 32], "proxi": [0, 5, 16, 30], "chang": [0, 2, 4, 5, 11, 14, 17, 18, 20, 26, 32], "default": [0, 2, 4, 5, 9, 12, 13, 14, 18, 19, 20, 23, 25, 26, 27, 28, 29, 30, 32], "header": [0, 4, 5, 7, 17, 19, 20, 29, 30, 31, 32], "xpath": [0, 4, 5, 29, 30, 31, 32], "express": [0, 4, 5, 7, 8, 9, 11, 13, 14, 17, 18, 20, 26, 29, 30, 31, 32], "custom": [0, 4, 5, 10, 11, 14, 18, 29, 30, 31, 32], "extract": [0, 5, 7, 14, 17, 25, 29, 30, 31, 32], "string": [0, 1, 2, 5, 7, 8, 9, 12, 15, 18, 23, 24, 27, 28, 29, 30], "creat": [0, 6, 7, 8, 10, 11, 12, 13, 18, 23, 25, 28, 29, 30, 31, 32], "ad": [0, 4, 7, 9, 13, 20, 23, 24, 26, 28, 29, 30, 31, 32], "larg": [0, 6, 8, 14, 16, 18, 19, 20, 26, 29, 30, 31, 32], "scale": [0, 16, 29, 30, 31], "long": [0, 1, 4, 7, 9, 14, 16, 17, 18, 23, 25, 28, 29, 30, 31, 32], "descript": [0, 6, 7, 9, 12, 13, 20, 23, 25, 26, 28, 29, 30, 31, 32], "text": [0, 1, 6, 7, 10, 14, 20, 21, 23, 27, 28, 29, 31], "top": [0, 7, 8, 9, 12, 13, 14, 16, 18, 19, 20, 23, 25, 26, 28, 29, 30, 31, 32], "approach": [0, 13, 25, 26, 29, 30, 31], "googl": [0, 4, 6, 14, 16, 17, 25, 28, 29, 30, 31, 32], "facebook": [0, 17, 24, 26, 29, 30, 31], "feed": [0, 28, 29, 30, 31], "instant": [0, 29, 30, 31], "articl": [0, 7, 19, 25, 29, 30, 31, 32], "analysi": [0, 9, 10, 16, 19, 20, 23, 29, 31], "analyz": [0, 10, 18, 20, 21, 23, 26, 29, 30, 31, 32], "imag": [0, 10, 12, 14, 17, 18, 19, 20, 23, 29, 30, 31], "redirect": [0, 4, 14, 20, 29, 30, 31, 32], "handl": [0, 6, 14, 16, 17, 18, 23, 29, 30, 31, 32], "veri": [0, 1, 6, 10, 12, 13, 14, 16, 17, 19, 20, 23, 25, 26, 29, 30, 31, 32], "file": [0, 1, 6, 10, 16, 18, 19, 20, 25, 29, 30, 31, 32], "compress": [0, 14, 25, 29, 30, 31], "explor": [0, 8, 9, 19, 20, 21, 23, 29, 30, 31], "column": [0, 4, 6, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 29, 30, 31], "data": [0, 4, 9, 10, 11, 13, 17, 18, 19, 25, 26, 29, 30, 31, 32], "type": [0, 1, 2, 4, 6, 9, 10, 12, 13, 14, 17, 18, 20, 23, 25, 26, 28, 29, 30, 31, 32], "parquet": [0, 14, 25, 29, 30, 31], "function": [0, 2, 4, 6, 8, 10, 11, 12, 13, 16, 17, 18, 19, 25, 26, 27, 29, 30, 31, 32], "emoji": [0, 20, 26, 29, 30, 31, 32], "get": [0, 1, 2, 6, 7, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29, 30, 31, 32], "insight": [0, 6, 19, 26, 29, 30, 31, 32], "search": [0, 10, 12, 17, 19, 23, 24, 25, 28, 29, 30, 31, 32], "structur": [0, 6, 7, 10, 13, 19, 23, 29, 30, 31, 32], "entiti": [0, 12, 17, 19, 23, 29, 30, 31, 32], "hashtag": [0, 15, 17, 23, 26, 29, 30, 31, 32], "mention": [0, 12, 15, 23, 25, 26, 29, 30, 31, 32], "currenc": [0, 26, 29, 30, 31, 32], "1234567890\u0660\u0661\u0662\u0663\u0664\u0665\u0666\u0667\u0668\u0669\u32ba\ud804\udc5b\ud800\udd0d\ud802\udcaa\u24f2\ud804\udc63\ud800\udd28\ud802\udd1b": [0, 29, 30, 31], "question": [0, 2, 19, 23, 26, 27, 29, 30, 31, 32], "\u0294": [0, 29, 30, 31], "exclam": [0, 27, 29, 30, 31], "python": [0, 4, 8, 21, 23, 29, 30, 31, 32], "statu": [0, 4, 7, 8, 14, 20, 23, 28, 29, 30, 31, 32], "code": [0, 4, 6, 7, 8, 9, 11, 12, 14, 18, 19, 20, 23, 24, 28, 29, 30, 31, 32], "checker": [0, 29, 30, 31], "respons": [0, 4, 7, 11, 12, 14, 17, 18, 19, 20, 23, 28, 29, 30, 31, 32], "crawler": [0, 6, 10, 14, 29, 30, 31, 32], "download": [0, 6, 7, 10, 17, 20, 25, 28, 29, 30, 31, 32], "name": [0, 1, 4, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 19, 20, 23, 24, 25, 26, 28, 29, 30, 31, 32], "import": [0, 1, 2, 6, 7, 8, 9, 10, 11, 13, 14, 16, 17, 19, 20, 21, 23, 25, 26, 27, 28, 29, 30, 31, 32], "knowledg": [0, 28, 29, 30, 31], "graph": [0, 20, 29, 30, 31], "result": [0, 1, 6, 7, 8, 11, 14, 17, 19, 20, 23, 25, 26, 28, 29, 30, 31, 32], "account": [0, 13, 17, 18, 23, 26, 28, 29, 30, 31], "setup": [0, 29, 30, 31], "": [0, 1, 2, 4, 6, 7, 9, 10, 11, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 29, 30, 31, 32], "api": [0, 7, 9, 18, 29, 30, 31, 32], "gener": [0, 6, 9, 14, 17, 20, 24, 25, 26, 28, 29, 30, 31, 32], "keyword": [0, 8, 12, 20, 21, 26, 29, 30, 31, 32], "sem": [0, 20, 29, 31], "campaign": [0, 1, 4, 24, 29, 30, 31], "run": [0, 4, 6, 8, 11, 12, 13, 16, 17, 18, 19, 20, 23, 25, 26, 29, 30, 31, 32], "logs_to_df": [0, 29, 30, 31], "support": [0, 10, 17, 18, 20, 23, 28, 29, 30, 31, 32], "format": [0, 1, 2, 7, 10, 11, 13, 15, 17, 18, 20, 23, 28, 29, 30, 31, 32], "prepar": [0, 8, 19, 29, 30, 31], "pars": [0, 10, 11, 20, 29, 30, 31, 32], "datafram": [0, 6, 7, 8, 9, 11, 12, 13, 17, 18, 19, 20, 23, 25, 26, 29, 30, 31, 32], "regular": [0, 4, 7, 8, 9, 11, 14, 20, 26, 29, 30, 31], "revers": [0, 14, 18, 23, 28, 29, 30, 31], "dn": [0, 14, 29, 30, 31], "lookup": [0, 14, 23, 29, 30, 31], "bulk": [0, 29, 30, 31], "test": [0, 6, 8, 29, 30, 31, 32], "tester": [0, 29, 30, 31], "engin": [0, 10, 12, 13, 14, 17, 19, 29, 30, 31, 32], "serp": [0, 12, 25, 29, 30, 31, 32], "youtub": [0, 12, 26, 29, 30, 31, 32], "xml": [0, 10, 17, 25, 29, 30, 31, 32], "sitemap": [0, 17, 20, 25, 29, 30, 31, 32], "index": [0, 4, 7, 10, 12, 14, 18, 20, 23, 29, 31, 32], "new": [0, 6, 7, 9, 14, 18, 25, 26, 29, 30, 31, 32], "video": [0, 17, 18, 23, 26, 28, 29, 30, 31, 32], "spider": [0, 4, 6, 10, 11, 14, 29, 30, 31, 32], "discoveri": [0, 29, 30, 31], "On": [0, 2, 23, 29, 30, 31], "element": [0, 2, 6, 7, 9, 10, 12, 29, 30, 31, 32], "pre": [0, 14, 26, 29, 30, 31], "determin": [0, 13, 18, 26, 29, 30, 31], "analyt": [0, 25, 26, 29, 30], "consol": [0, 18, 25, 28, 30], "css": [0, 4, 6, 29, 30, 31, 32], "selector": [0, 4, 29, 30, 31, 32], "behavior": [0, 4, 10, 11, 19, 28, 29, 30, 31, 32], "url": [0, 2, 6, 7, 9, 10, 11, 12, 14, 15, 17, 18, 19, 23, 26, 28, 29, 30, 31, 32], "queri": [0, 11, 12, 14, 17, 18, 19, 23, 28, 29, 30, 31, 32], "paramet": [0, 1, 2, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 23, 24, 26, 27, 28, 29, 30, 31, 32], "regex": [0, 6, 7, 8, 9, 11, 14, 15, 26, 29, 30, 32], "pattern": [0, 4, 6, 7, 9, 11, 17, 19, 25, 26, 30], "addit": [0, 2, 6, 9, 11, 14, 16, 18, 23, 26, 28, 29, 30, 31, 32], "stopword": [0, 26, 29, 30, 31, 32], "sever": [0, 6, 7, 10, 14, 18, 23, 26, 28, 29, 30, 31, 32], "languag": [0, 6, 10, 12, 13, 18, 19, 20, 23, 25, 26, 28, 29, 30, 31, 32], "twitter": [0, 6, 9, 10, 17, 20, 24, 29, 30, 31, 32], "authent": [0, 16, 18, 28, 29, 30, 31], "builder": [0, 29, 30, 31], "split": [0, 2, 14, 19, 20, 26, 27, 29, 30, 31, 32], "The": [0, 1, 2, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 26, 27, 28, 29, 30, 31, 32], "path": [0, 6, 7, 10, 11, 14, 17, 19, 20, 29, 30, 31], "directori": [0, 11, 29, 30, 31], "absolut": [0, 16, 17, 25, 29, 30, 31, 32], "weight": [0, 29, 30, 31, 32], "word": [0, 1, 2, 8, 9, 12, 13, 14, 18, 19, 23, 29, 30, 31, 32], "count": [0, 1, 7, 8, 9, 14, 16, 17, 19, 23, 27, 29, 30, 31, 32], "v": [0, 19, 30], "frequenc": [0, 8, 9, 16, 29, 30, 32], "token": [0, 26, 29, 30, 31], "n": [0, 4, 6, 7, 19, 29, 30, 31], "gram": [0, 4, 29, 30, 31], "digit": [0, 32], "market": [0, 12, 13, 18, 20, 24, 29], "product": [0, 1, 2, 12, 13, 18, 20, 25, 26, 29], "tool": [0, 10, 14, 18, 19, 20, 25], "when": [1, 4, 6, 9, 10, 12, 14, 17, 18, 19, 20, 23, 26, 28, 29, 32], "you": [1, 2, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 30, 32], "also": [1, 2, 4, 6, 7, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29, 32], "need": [1, 2, 4, 6, 7, 9, 10, 11, 12, 13, 14, 16, 17, 18, 20, 23, 26, 29, 32], "For": [1, 4, 6, 7, 10, 12, 14, 17, 18, 19, 20, 23, 25, 26, 28], "similar": [1, 7, 9, 20, 25, 28, 29], "categori": [1, 18, 19, 25, 28], "typic": [1, 2, 7, 9, 10, 14, 16, 18, 19, 20, 23, 25, 26, 28, 32], "replac": [1, 14, 19, 25, 29, 32], "latest": [1, 2, 4, 20, 23], "now": [1, 2, 7, 9, 13, 14, 17, 19, 20, 23, 26, 29], "mani": [1, 2, 6, 7, 8, 9, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 28, 32], "time": [1, 2, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 19, 20, 23, 26, 28, 29, 32], "have": [1, 2, 4, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "advertool": [1, 2, 6, 7, 8, 9, 10, 11, 13, 14, 16, 17, 19, 20, 21, 23, 25, 26], "adv": [1, 2, 4, 6, 7, 8, 9, 10, 11, 13, 14, 16, 17, 19, 20, 21, 23, 25, 26], "dubai": 1, "tokyo": 1, "singapor": 1, "ad_creat": [1, 13, 20, 29, 32], "templat": [1, 2, 6, 19], "5": [1, 6, 7, 8, 9, 12, 13, 14, 16, 17, 18, 19, 20, 21, 26, 30], "star": [1, 19], "hotel": [1, 8, 18], "max_len": [1, 4, 13], "30": [1, 2, 6, 14, 17, 18, 19, 30], "fallback": 1, "great": [1, 9, 20, 25, 26, 32], "citi": [1, 12], "In": [1, 2, 6, 7, 9, 11, 12, 14, 16, 17, 18, 19, 20, 23, 25, 26, 28, 29, 32], "an": [1, 2, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 26, 27, 28, 29, 30, 32], "thing": [1, 2, 9, 12, 13, 19, 20, 25, 27, 30, 32], "watch": [1, 19, 23], "out": [1, 9, 12, 14, 17, 18, 20, 23, 25, 26, 28, 32], "sinc": [1, 2, 7, 9, 10, 19, 20, 23, 26], "limit": [1, 2, 12, 17, 18, 20, 23, 28, 32], "each": [1, 2, 4, 7, 8, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29, 32], "slot": [1, 2, 29, 32], "exce": [1, 2], "thi": [1, 2, 4, 6, 7, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 30, 32], "provid": [1, 2, 4, 6, 7, 9, 11, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 28, 29, 32], "case": [1, 2, 4, 6, 7, 8, 9, 10, 11, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 29, 32], "longer": [1, 2, 9, 18, 26, 28, 29], "than": [1, 2, 17, 18, 20, 23, 26, 28, 29, 32], "lisbon": 1, "porto": 1, "algarv": 1, "freixo": 1, "de": [1, 6, 12, 13, 20], "espada": 1, "\u00e0": 1, "cinta": 1, "portug": 1, "capit": [1, 2, 4, 12, 13], "true": [1, 2, 4, 6, 7, 9, 10, 11, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 29], "sourc": [1, 2, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 24, 25, 26, 27, 28], "insert": [1, 25], "place": [1, 6, 8, 9, 11, 12, 23, 29], "within": [1, 2, 13, 18, 20, 23, 26, 27, 28], "str": [1, 2, 7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 24, 25, 26], "A": [1, 2, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "brace": 1, "e": [1, 4, 6, 7, 10, 12, 13, 18, 20, 24, 25, 26, 28, 29], "g": [1, 4, 6, 10, 18, 20, 24, 28, 29], "todai": [1, 8, 9], "int": [1, 7, 9, 11, 12, 13, 16, 18, 19, 23, 26, 29], "maximum": [1, 2, 12, 13, 16, 18, 19, 20, 23, 28], "allow": [1, 2, 4, 10, 14, 17, 18, 20, 23, 25, 28, 29, 32], "length": [1, 2, 10, 19, 23, 25, 26, 27, 29], "full": [1, 6, 7, 8, 10, 11, 13, 19, 20, 23, 26, 29, 32], "bool": [1, 2, 9, 12, 13, 18, 19, 20, 23, 25, 26], "whether": [1, 2, 4, 6, 7, 9, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 29], "return": [1, 2, 4, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 23, 24, 25, 26, 27, 28, 29, 32], "exampl": [1, 2, 4, 6, 7, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "let": [1, 2, 7, 9, 12, 13, 17, 18, 19, 20, 26, 28, 32], "one": [1, 2, 4, 6, 7, 8, 9, 12, 14, 15, 17, 18, 19, 20, 23, 25, 26, 28, 29, 32], "two": [1, 2, 6, 7, 9, 11, 13, 15, 17, 18, 20, 25, 26, 28, 29, 32], "three": [1, 6, 9, 19, 20, 23, 25, 26, 29, 32], "20": [1, 4, 6, 9, 12, 14, 17, 18, 19, 20, 23, 26, 28, 29], "One": [1, 6, 7, 10, 13, 17, 19, 20, 26], "favorit": [1, 9, 23, 26], "car": 1, "toyota": [1, 13], "bmw": [1, 13], "merced": 1, "lamborghini": 1, "28": [1, 17], "keep": [1, 6, 8, 12, 13, 16, 17, 20, 23, 26, 29, 32], "As": [1, 6, 7, 16, 17, 19, 25, 26, 27, 28, 32], "50": [1, 6, 17, 18, 26, 29], "fals": [1, 2, 6, 7, 9, 13, 14, 17, 19, 20, 23, 25, 26, 29], "produc": [1, 14, 18, 19, 32], "error": [1, 4, 6, 14, 16, 18, 20, 28, 29], "someth": [1, 2, 26], "traceback": 1, "most": [1, 2, 6, 7, 9, 10, 13, 14, 18, 19, 20, 23, 26, 28, 29, 32], "recent": [1, 23, 28], "call": [1, 18, 19, 20, 23, 25, 26, 29, 32], "last": [1, 2, 9, 17, 19, 20, 25, 26, 29], "input": [1, 2], "line": [1, 6, 7, 10, 12, 14, 16, 17, 18, 20, 29, 30], "1": [1, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 30], "modul": [1, 9, 20, 26, 29, 30, 31, 32], "26": [1, 6, 7, 9, 10, 14, 20, 24, 30], "valueerror": [1, 29], "should": [1, 4, 6, 9, 10, 11, 12, 17, 18, 20, 23, 26, 28, 32], "char": 1, "about": [2, 6, 7, 8, 9, 10, 11, 14, 16, 17, 18, 19, 20, 23, 25, 26, 28, 29, 30, 32], "your": [2, 4, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 23, 24, 25, 26, 28, 30], "especi": [2, 6, 13, 17, 19, 26], "respect": [2, 6, 7, 14, 20, 23, 26, 29], "land": [2, 13, 32], "ha": [2, 6, 7, 9, 13, 16, 17, 19, 20, 23, 25, 26, 28, 29, 32], "becom": [2, 4, 14, 20, 25, 26], "consider": [2, 26, 32], "platform": [2, 26, 32], "90": [2, 14], "charact": [2, 4, 9, 18, 23, 26, 27, 28, 29], "total": [2, 14, 17, 26], "270": 2, "That": [2, 6, 9, 20, 26, 28], "more": [2, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 29, 32], "enough": [2, 13, 18, 26, 28], "space": [2, 4, 18, 19, 23, 27, 29], "talk": 2, "main": [2, 7, 12, 17, 18, 19, 20, 23, 25, 26, 29, 32], "featur": [2, 11, 18, 19, 23, 28, 29, 32], "util": [2, 23], "all": [2, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 29, 32], "detail": [2, 4, 6, 7, 10, 11, 13, 14, 17, 18, 20, 23, 28, 32], "fit": [2, 28, 29], "correctli": 2, "given": [2, 6, 11, 17, 18, 19, 23, 25, 29], "ar": [2, 4, 6, 7, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 30, 32], "ad_from_str": [2, 13, 20, 29, 32], "doe": [2, 6, 8, 9, 16, 17, 18, 19, 20, 26, 27, 28, 32], "exactli": [2, 7, 8, 28], "divid": [2, 26], "ani": [2, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "remain": [2, 19, 27], "append": [2, 17, 18, 20, 29], "end": [2, 7, 9, 10, 18, 20, 25, 27, 29, 32], "anoth": [2, 7, 10, 13, 14, 19, 25, 26, 27, 32], "benefit": [2, 25, 32], "take": [2, 6, 7, 9, 12, 13, 14, 17, 19, 20, 23, 25, 26, 29, 32], "write": [2, 7, 32], "onc": [2, 4, 6, 7, 12, 13, 14, 18, 19, 20, 23, 25, 26, 28, 30, 32], "easili": [2, 6, 7, 10, 14, 17, 19, 21, 23, 25, 32], "differ": [2, 4, 6, 8, 9, 12, 13, 14, 15, 17, 18, 19, 20, 23, 25, 26, 28, 32], "here": [2, 7, 8, 9, 10, 13, 14, 18, 19, 20, 25, 26, 32], "quick": [2, 7, 17, 19], "overview": [2, 7, 8, 9, 16, 19, 32], "avail": [2, 4, 6, 7, 10, 11, 12, 14, 15, 18, 19, 20, 21, 23, 25, 26, 28, 29, 32], "option": [2, 6, 7, 9, 10, 11, 13, 14, 17, 18, 20, 21, 23, 25, 26, 27, 28, 29, 32], "would": [2, 6, 7, 9, 11, 14, 17, 18, 19, 20, 23, 25, 26, 28], "note": [2, 7, 9, 12, 18, 20, 23, 25, 26, 28, 32], "although": [2, 20, 26, 32], "other": [2, 4, 7, 8, 9, 10, 11, 12, 14, 15, 17, 18, 19, 20, 23, 25, 26, 28, 29, 32], "group": [2, 4, 7, 8, 9, 13, 17, 29, 32], "fewer": 2, "sep": [2, 29], "separ": [2, 4, 7, 9, 14, 18, 20, 23, 25, 28, 29], "which": [2, 4, 6, 7, 9, 10, 11, 12, 14, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29, 32], "none": [2, 4, 7, 8, 9, 10, 11, 12, 14, 17, 18, 20, 23, 24, 25, 26, 28], "whitespac": [2, 26, 27, 29], "els": [2, 12, 23, 26], "sometim": [2, 6, 13, 17, 20, 26, 32], "might": [2, 4, 6, 7, 8, 9, 10, 11, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 30, 32], "hyphen": 2, "leav": [2, 17], "intact": 2, "If": [2, 6, 9, 12, 14, 18, 19, 20, 23, 25, 26, 28, 29, 30], "first": [2, 6, 7, 9, 12, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "letter": [2, 7, 8, 18, 28], "five": [2, 17, 20, 25, 26], "alwai": [2, 23, 26, 29], "six": [2, 26], "ensur": [2, 20], "remaind": [2, 29], "lost": [2, 6], "know": [2, 7, 9, 13, 14, 20, 25, 26, 32], "what": [2, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 19, 20, 23, 25, 26, 32], "miss": [2, 9, 19, 26, 29], "shorter": [2, 28, 29], "still": [2, 9, 11, 18, 19, 23, 25, 26, 28], "consist": [2, 6, 13, 14, 23, 25, 26, 29], "desc_text": 2, "gadget": 2, "onlin": [2, 20], "gx12": 2, "model": [2, 14, 19], "come": [2, 13, 14, 26], "13": [2, 6, 7, 8, 9, 10, 11, 12, 14, 17, 19, 30], "lot": [2, 7, 10, 14, 19, 25, 32], "good": [2, 4, 6, 9, 14, 16, 19, 20, 25, 26], "stuff": [2, 32], "health": [2, 28], "start": [2, 4, 6, 9, 13, 14, 18, 19, 20, 23, 26, 28, 29, 32], "shop": [2, 20], "len": [2, 4], "130": [2, 14, 16, 19], "see": [2, 6, 7, 8, 9, 11, 12, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 32], "scenario": 2, "valu": [2, 4, 6, 10, 12, 14, 18, 19, 20, 23, 25, 26, 28, 29], "extra": [2, 14, 26], "empti": [2, 6, 7, 8, 9, 25, 26, 29], "125": [2, 6, 16], "25": [2, 7, 9, 16, 19, 20, 24, 26, 30], "look": [2, 7, 13, 19, 23, 26], "just": [2, 4, 8, 9, 11, 14, 18, 20, 23, 26, 32], "second": [2, 6, 7, 9, 12, 13, 18, 20, 25, 29, 32], "where": [2, 4, 6, 7, 9, 10, 11, 14, 18, 19, 20, 23, 25, 26, 29, 32], "our": [2, 7, 9, 14, 23, 26], "we": [2, 7, 9, 12, 13, 14, 17, 19, 20, 25, 26, 32], "up": [2, 7, 10, 12, 13, 18, 20, 23, 25, 26, 29, 32], "15": [2, 6, 7, 9, 12, 14, 19, 25, 28], "convert": [2, 7, 14, 20, 25, 29, 32], "restrict": [2, 11, 12, 18, 20, 23, 28, 29], "iter": 2, "integ": [2, 18, 28], "after": [2, 4, 6, 7, 9, 11, 18, 19, 20, 23, 25, 26, 27, 28, 32], "text_ad": 2, "accord": [2, 17, 27], "spec": 2, "short": [2, 18, 26, 28], "wai": [2, 6, 7, 9, 10, 14, 19, 20, 23, 25, 29, 32], "10": [2, 6, 7, 8, 9, 10, 14, 17, 18, 19, 20, 23, 25, 26, 30], "bY": 2, "captial": 2, "To": [2, 11, 13, 17, 20, 26, 28, 30], "instal": [4, 6, 30], "python3": [4, 32], "m": [4, 9, 14, 17, 18, 19, 20, 26, 28, 32], "pip": [4, 6, 30, 32], "acess": 4, "go": [4, 9, 10, 11, 13, 14, 17, 18, 19, 20, 25, 26, 29], "program": [4, 23, 32], "help": [4, 6, 7, 9, 10, 14, 16, 18, 19, 20, 23, 25, 27, 29, 32], "h": [4, 7, 9, 14, 29], "access": [4, 11, 14, 17, 18, 21, 23, 28, 32], "specif": [4, 7, 10, 18, 23, 28, 32], "usag": [4, 7, 8, 9, 20, 23, 32], "web": [4, 10, 11, 12, 18, 19, 20, 21], "local": [4, 18, 23, 28], "machin": [4, 19, 32], "http": [4, 6, 7, 8, 9, 10, 11, 12, 14, 17, 19, 20, 23, 25, 26], "www": [4, 6, 7, 8, 9, 10, 11, 14, 17, 19], "com": [4, 6, 7, 8, 9, 10, 11, 12, 14, 16, 17, 18, 19, 20, 23, 24, 25, 28], "jo": 4, "output": [4, 6, 7, 10, 11, 14, 20, 23, 25, 29], "google_robot": 4, "robotslist": 4, "multi_robot": 4, "posit": [4, 7, 9, 12, 18, 23, 28], "argument": [4, 6, 12, 18, 20, 29], "show": [4, 7, 8, 9, 12, 13, 14, 18, 20, 25, 26, 28, 29], "messag": [4, 14, 29], "exit": [4, 14], "r": 4, "0": [4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 25, 26, 28, 30], "sitemap_url": [4, 19], "recurs": [4, 19, 20, 29], "fetch": [4, 17, 20, 29], "url_list": [4, 6, 10, 14, 20], "custom_set": [4, 6, 10, 11, 14, 20, 29], "output_fil": [4, 6, 7, 10, 14, 17, 20, 25, 29], "filepath": [4, 7, 20], "jl": [4, 6, 7, 10, 14, 17, 20, 29], "modifi": [4, 8, 9, 10, 11, 13, 17, 19, 20, 21, 23, 26], "equal": [4, 14, 23, 25], "sign": [4, 9, 17, 20, 25, 27], "without": [4, 6, 10, 18, 20, 25, 26, 28, 29, 32], "between": [4, 6, 13, 14, 18, 19, 20, 26, 27, 28, 32], "log_fil": [4, 6, 14, 20], "closespider_timeout": [4, 6, 20], "f": [4, 14], "field": [4, 11, 14, 18, 28, 29], "errors_fil": [4, 14], "log_format": [4, 14], "common": [4, 6, 10, 14, 18, 28], "combin": [4, 12, 13, 14, 15, 17, 18, 20, 32], "common_with_vhost": [4, 14], "nginx_error": [4, 14], "apache_error": [4, 14], "special": [4, 9, 11, 19, 20, 25, 29, 32], "instead": [4, 9, 17, 20, 23, 25, 26, 28, 29], "ip_list": [4, 16], "semkw": 4, "exact": [4, 6, 13, 23], "broad": [4, 13], "l": 4, "c": [4, 6, 10, 21], "campaign_nam": [4, 13], "contain": [4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "sell": [4, 13, 19], "per": [4, 6, 7, 8, 9, 17, 18, 19, 23, 26, 32], "match": [4, 7, 8, 9, 11, 12, 13, 14, 18, 20, 23, 28, 29], "max": [4, 6, 10, 20], "3": [4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 30], "adgroup": [4, 13], "order": [4, 7, 9, 11, 12, 13, 18, 20, 28, 29], "matter": [4, 13], "permut": [4, 13], "bui": [4, 13, 17], "arab": [4, 9, 18, 19, 21, 29], "azerbaijani": [4, 21, 29], "bengali": [4, 21, 29], "catalan": [4, 18, 21, 29], "chines": [4, 18, 21, 28, 29], "croatian": [4, 18, 21, 29], "danish": [4, 18, 21, 29], "dutch": [4, 18, 21, 29], "english": [4, 18, 21, 25, 26, 29], "finnish": [4, 18, 21, 29], "french": [4, 18, 21, 29], "german": [4, 18, 21, 29], "greek": [4, 9, 18, 21, 29], "hebrew": [4, 18, 21, 29], "hindi": [4, 19, 21, 29], "hungarian": [4, 18, 21, 29], "indonesian": [4, 18, 21, 29], "irish": [4, 21, 29], "italian": [4, 18, 21, 29], "japanes": [4, 18, 21, 29], "kazakh": [4, 21, 29], "nepali": [4, 21, 29], "norwegian": [4, 18, 21, 29], "persian": [4, 19, 21, 29], "polish": [4, 18, 21, 29], "portugues": [4, 18, 19, 21, 29], "romanian": [4, 18, 21, 29], "russian": [4, 18, 19, 21, 29], "sinhala": [4, 21, 29], "spanish": [4, 9, 18, 21, 29], "swedish": [4, 18, 21, 29], "tagalog": [4, 21, 29], "tamil": [4, 21, 29], "tatar": [4, 21, 29], "telugu": [4, 21, 29], "thai": [4, 21, 29], "turkish": [4, 18, 21, 29, 32], "ukrainian": [4, 21, 29], "urdu": [4, 19, 21, 29], "vietnames": [4, 19, 21, 29], "wordfreq": 4, "number_list": 4, "phrase_len": [4, 26, 27, 29], "text_list": [4, 8, 9, 26, 27], "sentenc": [4, 9, 26], "exclud": [4, 11, 13, 18, 20, 23, 25, 28, 29], "follow_link": [4, 6, 14, 20, 29], "d": [4, 7, 9, 10, 14, 18, 26], "allowed_domain": [4, 6, 20], "param": 4, "exclude_url_param": [4, 20, 29], "include_url_param": [4, 20, 29], "exclude_url_regex": [4, 20, 29], "include_url_regex": [4, 20, 29], "css_selector": [4, 20, 29], "xpath_selector": [4, 20, 29], "encount": [4, 14, 25], "parmet": [4, 20], "rais": [4, 20, 29], "dictionari": [4, 6, 8, 9, 10, 20, 21, 29], "map": [4, 7, 9, 12, 13, 20, 23, 32], "requir": [4, 6, 14, 18, 19, 20, 23, 24, 28, 29], "content": [4, 6, 7, 10, 12, 17, 18, 19, 20, 21, 25, 26, 28, 29, 31], "add": [4, 6, 10, 14, 20, 23, 26], "over": [4, 6, 10, 19, 20, 23, 26, 29], "170": [4, 10, 20], "kind": [4, 10, 20, 29, 32], "pleas": [4, 10, 12, 18, 20, 23, 26, 28], "refer": [4, 7, 10, 12, 14, 18, 20, 23, 25], "doc": [4, 7, 12, 23], "scrapi": [4, 6, 10, 20, 32], "org": [4, 7, 8, 10, 12], "en": [4, 6, 7, 10, 12, 19, 20, 23, 25, 28], "topic": [4, 18, 19, 23, 25, 28, 29], "html": [4, 6, 7, 10, 14, 17, 19, 20, 23, 25, 28, 29], "home": [4, 17, 28], "examl": 4, "example_output": 4, "url_1": [4, 11, 25], "url_2": [4, 11, 25], "url_3": [4, 11], "OR": [4, 18, 23, 28, 30], "process": [4, 6, 7, 10, 14, 16, 19, 25], "000": [4, 18, 23, 27], "closespider_pagecount": [4, 6, 20], "1000": [4, 18, 28], "master": [6, 10, 20], "basic": [6, 10, 13, 16, 19, 25, 32], "probabl": [6, 7, 20, 32], "achiev": [6, 27, 32], "better": [6, 7, 19, 20, 23, 25, 29, 32], "These": [6, 14, 20, 26], "some": [6, 7, 8, 9, 10, 11, 14, 16, 17, 18, 19, 20, 23, 25, 26, 28, 29, 32], "kei": [6, 8, 9, 12, 18, 20, 21, 23, 25, 26, 28, 29], "indic": [6, 10, 17, 18, 20, 23, 25, 28, 29], "simpli": [6, 7, 13, 14, 16, 17, 19, 20, 26], "done": [6, 7, 10, 13, 14, 18, 19, 25, 26, 32], "page_1": 6, "page_2": 6, "page_3": 6, "page_4": 6, "example_crawl_1": 6, "goe": [6, 19], "through": [6, 10, 11, 14, 17, 18, 19, 20, 23, 25, 26, 27, 28, 32], "discov": [6, 14, 20, 25, 32], "find": [6, 8, 9, 18, 20, 23, 25, 26, 28, 32], "exmapl": [6, 7, 10, 20, 25], "won": [6, 9, 26, 32], "solut": [6, 7], "therefor": [6, 18, 26, 28], "origin": [6, 11, 14, 18, 20, 25, 29], "commun": [6, 25, 32], "It": [6, 9, 10, 12, 13, 14, 16, 18, 19, 20, 23, 25, 26, 28, 32], "usual": [6, 12, 16, 23, 25, 26], "check": [6, 7, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 26, 32], "except": [6, 23, 25, 26], "stat": [6, 8, 9, 29], "etc": [6, 7, 8, 9, 12, 13, 14, 15, 18, 19, 20, 21, 25, 26, 27, 29, 32], "pass": [6, 12, 17, 18, 19, 20, 23, 32], "cutom_set": 6, "practic": [6, 14, 16, 20, 32], "give": [6, 7, 9, 10, 13, 16, 18, 20, 23, 25, 26], "extens": [6, 14, 17, 18, 25, 29], "easier": [6, 9, 13, 20, 23, 25, 29, 30, 32], "retreiv": [6, 19, 29], "website_name_crawl_1": 6, "work": [6, 7, 9, 12, 13, 14, 17, 20, 23, 26, 28, 29, 32], "website_name_crawl_2": 6, "There": [6, 7, 9, 10, 14, 18, 19, 20, 23, 25, 29, 32], "few": [6, 7, 9, 10, 11, 14, 19, 20, 25, 26, 32], "trigger": 6, "thei": [6, 7, 9, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "mostli": [6, 7, 20, 25, 26, 32], "closespider_errorcount": [6, 20], "wait": [6, 20], "hour": [6, 23], "finish": 6, "had": [6, 8, 9, 26], "investig": 6, "issu": [6, 7, 29, 32], "closespider_itemcount": [6, 20], "anyth": [6, 23, 26, 29], "item": [6, 7, 11, 13, 18, 20, 23, 25, 28, 29], "h1": [6, 7, 20], "titl": [6, 7, 9, 13, 18, 20, 25, 26, 28, 29, 32], "meta_desc": [6, 7, 20], "been": [6, 19, 23, 26, 28], "exploratori": [6, 20], "techniqu": [6, 26, 32], "thousand": [6, 8, 10, 16, 26, 32], "idea": [6, 7, 9, 13, 20, 26, 32], "mind": [6, 8, 12, 20, 23, 32], "500": [6, 7, 12, 17, 18, 23, 26, 28], "robotstxt_obei": [6, 10, 11], "under": [6, 10, 11, 12, 14, 17, 26, 29], "user_ag": [6, 10, 11, 14, 17, 20], "found": [6, 7, 11, 14, 18, 19, 23], "current": [6, 7, 14, 19, 20, 23, 28, 29, 32], "your_user_ag": 6, "high": [6, 12, 18, 19, 20, 28], "sensit": [6, 13, 23], "autom": [6, 10, 17, 32], "quickli": 6, "block": [6, 14, 17, 19, 20, 29, 32], "ban": 6, "polit": [6, 23, 25, 28], "kill": 6, "concurrent_item": 6, "100": [6, 7, 14, 18, 20, 23, 26], "concurrent_request": 6, "16": [6, 9, 11, 14, 17, 19], "concurrent_requests_per_domain": [6, 20], "8": [6, 7, 8, 9, 10, 11, 12, 14, 16, 17, 19, 20, 23, 26, 30], "concurrent_requests_per_ip": 6, "download_delai": [6, 20], "interv": [6, 32], "befor": [6, 9, 18, 20, 23, 26, 28], "consecut": [6, 20], "400": [6, 17, 26], "75": [6, 7, 8, 9, 14, 16, 17, 20], "depth_limit": [6, 20], "level": [6, 14, 18, 20, 23, 25, 28, 32], "2": [6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 30], "initi": [6, 12, 29], "reason": [6, 9, 10, 14, 17, 20, 26, 29, 32], "why": [6, 7, 14, 17, 26], "mainli": [6, 11, 13, 16, 17, 26, 27], "updat": [6, 19, 29], "site": [6, 7, 13, 17, 18, 19, 20, 25, 26, 28, 29, 32], "alreadi": [6, 26], "big": [6, 13, 14, 25], "hurri": 6, "across": [6, 7, 8, 12, 18, 19, 25, 26, 32], "dai": [6, 9, 18, 23, 26, 28], "emerg": 6, "measur": [6, 18, 28, 29], "connect": [6, 7, 17, 23, 29, 32], "batteri": 6, "left": [6, 9, 28], "off": [6, 18, 26], "extrem": [6, 10, 13, 20, 32], "simpl": [6, 7, 8, 9, 10, 16, 19, 20, 23, 26, 32], "folder": [6, 11], "rerun": 6, "worri": [6, 17], "jobdir": 6, "abov": [6, 11, 12, 20, 23, 25, 26, 29], "happen": [6, 14, 25], "accid": 6, "close": 6, "comput": [6, 14, 20], "manual": 6, "ctrl": 6, "command": [6, 14, 16, 17, 29, 30], "again": [6, 14, 19, 20, 25, 26], "wa": [6, 7, 8, 9, 11, 14, 17, 19, 20, 23, 26, 29], "manag": [6, 7, 10, 18, 23, 28, 32], "But": [6, 20, 26], "doesn": [6, 13, 14, 20, 26], "duplic": [6, 16, 18, 20, 23], "step": [6, 12, 27, 32], "3rd": 6, "parti": [6, 23], "packag": [6, 20, 21, 29, 30, 31, 32], "rotat": 6, "retri": 6, "downloader_middlewar": 6, "rotating_proxy_list_path": 6, "usernam": [6, 28], "password": 6, "ipaddress": 6, "port": [6, 25], "random": [6, 17], "user123": 6, "password123": 6, "12": [6, 7, 9, 11, 14, 17, 19, 20, 23, 30], "34": [6, 12], "56": [6, 13, 14, 16, 17, 19], "78": [6, 17], "1111": 6, "1112": 6, "1113": 6, "1114": 6, "Then": [6, 20, 23], "rotating_proxi": 6, "middlewar": [6, 14], "rotatingproxymiddlewar": 6, "610": 6, "bandetectionmiddlewar": 6, "620": 6, "read": [6, 7, 9, 11, 14, 25, 32], "normal": [6, 14, 21], "being": [6, 18, 19, 20, 23, 25, 26, 28, 29, 32], "crawldf": [6, 7], "pd": [6, 7, 9, 10, 14, 17, 19, 20, 25, 26, 29], "read_json": [6, 7, 10, 17, 20], "filter": [6, 7, 14, 18, 19, 23, 25, 28, 29], "head": [6, 7, 8, 9, 10, 13, 14, 17, 19, 20, 26, 29, 30], "_rotating_proxi": 6, "request_headers_proxi": 6, "author": [6, 18, 20, 23, 25, 28], "proxy_retry_tim": 6, "123": [6, 9], "456": [6, 9], "789": [6, 9], "101": [6, 14], "8893": 6, "b3vzy214dhg6odlld29rmgrsdfgt": 6, "nan": [6, 7, 10, 12, 14, 19, 20, 25, 29], "8894": 6, "8895": 6, "8896": 6, "4": [6, 7, 8, 9, 11, 12, 13, 14, 16, 17, 19, 20, 23, 25, 26, 28, 30], "8897": 6, "easi": [6, 10, 13, 19, 25, 29, 32], "default_request_head": [6, 20, 29], "accept": [6, 10, 18, 20, 28, 29], "encod": [6, 10, 14, 20, 23, 24, 25, 29], "gzip": [6, 10, 20], "deflat": [6, 10, 20], "actual": [6, 12, 16, 19, 20, 25, 26, 28], "were": [6, 7, 14, 18, 19, 23, 26, 28, 29], "request_headers_": [6, 7, 20], "request_headers_accept": [6, 10, 20], "request_headers_us": [6, 10, 20], "suggest": [6, 25, 28], "tag": [6, 7, 10, 17, 18, 19, 20, 28, 29, 32], "meta": [6, 10, 18, 20, 23, 25], "attribut": [6, 7, 18, 20, 23, 26, 29], "viewport": [6, 7, 20, 29], "charset": [6, 7, 10, 20, 29], "h2": [6, 7, 20, 29], "h3": [6, 7, 10, 20], "h4": 6, "h5": 6, "h6": [6, 7, 20], "canon": [6, 7, 10, 20, 29], "rel": [6, 7, 12, 16, 17, 20, 25, 29, 32], "href": [6, 20, 29], "alt_href": [6, 7, 20], "altern": [6, 17, 20, 29], "alt_hreflang": [6, 7, 20], "hreflang": [6, 10, 29], "og_prop": 6, "properti": [6, 7, 18, 20, 28], "og": [6, 7, 20, 29], "who": [6, 9, 17, 18, 19, 23, 26, 28], "opengraph": [6, 10], "og_cont": 6, "twtr_name": 6, "twtr_content": 6, "iframe_src": 6, "ifram": 6, "src": [6, 7, 11, 20, 29], "gtm_script": 6, "script": [6, 10], "googletagmanag": 6, "gtm": 6, "j": [6, 14], "id": [6, 10, 12, 18, 20, 23, 28], "gtm_noscript": 6, "link_rel_rel": 6, "link_rel_href": 6, "link_rel_stylesheet": 6, "stylesheet": 6, "css_link": 6, "nav_links_text": [6, 20], "nav": [6, 20, 29], "anchor": [6, 20], "nav_links_href": 6, "header_links_text": [6, 20], "header_links_href": 6, "footer_links_text": [6, 20], "footer": [6, 20, 29], "footer_links_href": 6, "js_script_src": 6, "javascript": 6, "js_script_text": 6, "script_src": 6, "canonical_par": 6, "parent": [6, 28], "collect": [6, 15, 17, 18, 23, 28, 29], "popular": [6, 14, 23, 28, 32], "amazon": [6, 17, 19, 23], "4k": [6, 9], "fire": [6, 9, 26], "tv": [6, 19, 28], "mozilla": [6, 14, 20], "linux": [6, 14, 29], "android": [6, 14, 19], "aft": 6, "build": [6, 8, 10, 14, 32], "lmy47o": 6, "applewebkit": [6, 14], "537": [6, 14, 17], "36": [6, 14, 17, 20], "khtml": [6, 14], "like": [6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 19, 20, 23, 26, 28, 29, 32], "gecko": [6, 14], "version": [6, 10, 14, 20, 29], "chrome": [6, 12, 14], "41": [6, 19], "99900": 6, "2250": 6, "0242": 6, "safari": [6, 14], "aftwmst22": 6, "9": [6, 7, 8, 9, 10, 12, 14, 17, 19, 20, 26, 30], "ps7233": 6, "wv": 6, "88": 6, "4324": 6, "152": [6, 14], "mobil": [6, 14], "kindl": 6, "u": [6, 7, 13, 14, 18, 19, 20, 26, 28], "528": 6, "screen": [6, 23], "600x800": 6, "x11": [6, 14], "armv7l": 6, "531": 6, "533": [6, 19], "hdx": 6, "7": [6, 7, 8, 9, 12, 14, 16, 17, 19, 23, 26, 30], "kfthwi": 6, "ktu84m": 6, "silk": 6, "47": [6, 14], "79": [6, 14, 17], "2526": 6, "80": [6, 14, 16, 26], "appl": [6, 17, 26], "4th": 6, "gen": 6, "appletv5": 6, "5th": 6, "appletv6": 6, "11": [6, 7, 8, 9, 10, 14, 17, 18, 19, 20, 30], "6th": 6, "appletv11": 6, "iphon": 6, "6": [6, 7, 8, 9, 11, 12, 14, 16, 17, 19, 26, 30], "iphone7c2": 6, "1202": 6, "466": 6, "cpu": [6, 14], "mac": 6, "o": [6, 15], "x": [6, 12, 13, 14, 18, 19, 20, 32], "420": 6, "1a543": 6, "419": 6, "iphone9": 6, "10_0_1": 6, "602": 6, "14a403": 6, "plu": 6, "11_0": 6, "604": 6, "15a5341f": 6, "38": [6, 14, 20], "15a5370a": 6, "se": 6, "iphone14": 6, "15_4": 6, "19e241": 6, "15a372": 6, "xr": 6, "12_0": 6, "605": 6, "15e148": 6, "crio": 6, "69": [6, 14, 20], "3497": 6, "105": [6, 7, 14], "firefox": 6, "fxio": 6, "2b11866": 6, "16a366": 6, "bing": [6, 12], "bot": [6, 14, 16], "compat": [6, 14, 23], "bingbot": [6, 14, 17], "htm": [6, 14], "laptop": 6, "browser": [6, 12], "chromebook": 6, "cro": 6, "x86_64": [6, 14], "8172": 6, "45": [6, 14, 18], "51": [6, 14, 16, 17], "2704": 6, "64": [6, 9, 17], "chromecast": 6, "crkei": 6, "16041": 6, "31": [6, 14, 16, 19, 30], "1650": 6, "adt": 6, "dalvik": 6, "ptt5": 6, "181126": 6, "002": 6, "nexu": [6, 14], "player": [6, 28], "mmb29t": 6, "pixel": [6, 11], "nmf26f": 6, "54": [6, 12, 19], "2840": 6, "85": 6, "opd1": 6, "170811": 6, "59": [6, 13], "3071": 6, "qd1a": 6, "190821": 6, "014": 6, "c2": 6, "3904": 6, "108": [6, 17], "rq3a": 6, "210805": 6, "001": 6, "a1": 6, "92": [6, 14], "4515": 6, "159": [6, 9], "sd1a": 6, "210817": 6, "023": 6, "94": [6, 14], "4606": 6, "71": 6, "nrd90m": 6, "52": [6, 17, 19], "2743": 6, "98": [6, 14], "googlebot": [6, 14, 16, 17], "htc": 6, "desir": [6, 7, 14, 23], "21": [6, 9, 11, 14, 17, 18, 19, 20, 23, 24, 30], "pro": 6, "5g": 6, "4183": 6, "127": 6, "m9": 6, "mra58k": 6, "x10": 6, "61": 6, "3163": 6, "u20": 6, "wildfir": 6, "74": [6, 16], "3729": 6, "136": 6, "lg": 6, "pad": 6, "v410": 6, "v41020c": 6, "lrx22g": 6, "1847": 6, "118": [6, 14], "lenovo": 6, "yoga": 6, "tab": 6, "yt": 6, "j706x": 6, "96": [6, 16], "4664": 6, "pc": 6, "ubuntu": [6, 10], "rv": 6, "20100101": 6, "macintosh": 6, "intel": 6, "10_11_2": 6, "601": [6, 9], "microsoft": 6, "lumia": 6, "550": 6, "window": [6, 14, 20], "phone": [6, 9], "rm": 6, "1127_16056": 6, "42": [6, 14], "2311": 6, "135": [6, 14], "edg": 6, "10536": 6, "650": 6, "1152": 6, "116": 6, "15254": 6, "950": 6, "46": [6, 14, 19], "2486": 6, "1058": 6, "minix": 6, "neo": 6, "x5": 6, "he": [6, 9, 19, 26], "il": 6, "116a": 6, "jdq39": 6, "534": 6, "6p": 6, "mmb29p": [6, 14], "83": [6, 14], "nintendo": 6, "3d": [6, 18, 28], "7412": 6, "eu": 6, "switch": 6, "wifiwebauthapplet": 6, "nf": 6, "nintendobrows": 6, "13343": 6, "wii": 6, "wiiu": 6, "536": [6, 17], "nx": 6, "11264": 6, "nvidia": 6, "shield": 6, "tablet": [6, 19], "k1": 6, "55": [6, 12, 13, 19], "2883": 6, "91": [6, 16], "playstat": 6, "73": [6, 14], "vita": 6, "roku": 6, "ultra": [6, 19], "roku4640x": 6, "dvp": 6, "70": [6, 14, 20], "297": 6, "70e04154a": 6, "samsung": [6, 19], "galaxi": [6, 19], "s10": 6, "sm": [6, 25], "g973u": 6, "ppr1": 6, "180610": 6, "011": 6, "s20": 6, "g980f": 6, "qp1a": 6, "190711": 6, "020": 6, "s21": 6, "g996u": 6, "s22": [6, 19], "s906n": 6, "3987": 6, "119": [6, 14], "s6": 6, "g920v": 6, "mmb29k": 6, "g928x": 6, "lmy47x": 6, "s7": 6, "g930vc": 6, "58": [6, 13, 14, 17], "3029": 6, "g935": 6, "s8": 6, "g892a": 6, "60": [6, 14, 16, 17], "3112": 6, "107": 6, "s9": 6, "g960f": 6, "r16nw": 6, "62": 6, "3202": 6, "84": 6, "t550": 6, "samsungbrows": 6, "2125": 6, "102": 6, "s3": 6, "t827r4": 6, "x906c": 6, "soni": 6, "xperia": 6, "j8110": 6, "552": 6, "3578": 6, "99": [6, 7, 9, 19, 26], "xz": 6, "g8231": 6, "219": 6, "z4": 6, "sgp771": 6, "32": [6, 9, 10, 14, 16, 17, 19, 20], "253": 6, "z5": 6, "e6653": 6, "nt": [6, 14, 20], "win64": [6, 14], "x64": [6, 14], "246": 6, "wow64": 6, "111": 6, "xbox": 6, "10586": 6, "xbox_one_": 6, "14": [6, 7, 9, 10, 11, 12, 14, 17, 19, 30], "14393": 6, "seri": [6, 19, 25], "48": [6, 14, 17, 19], "2564": 6, "82": [6, 16, 20], "02": [6, 10, 17, 19, 30], "yahoo": [6, 23], "slurp": [6, 17], "ysearch": 6, "iphone12": 6, "13_0": 6, "iphone13": 6, "14_0": 6, "15_0": 6, "19a346": 6, "bunch": [7, 20], "gain": [7, 19, 30], "undersand": 7, "technic": [7, 12, 32], "readi": [7, 13, 23], "made": [7, 9, 16, 23, 26], "anayz": 7, "independ": [7, 28, 32], "size": [7, 10, 14, 17, 18, 19, 20, 23, 25, 29], "cours": [7, 10, 13, 26, 32], "togeth": [7, 8, 9, 14, 17, 23, 25, 26, 32], "put": [7, 13, 14, 19, 20, 26], "context": [7, 9, 12, 15], "thought": [7, 12], "describ": [7, 13, 26], "aspect": 7, "yet": [7, 13, 20, 25, 26], "spread": [7, 32], "everi": [7, 14, 17, 19, 20, 23, 26, 28, 32], "turn": [7, 18], "alt": [7, 20, 23, 29], "width": [7, 11, 20, 28, 29], "unpack": [7, 19], "point": [7, 8, 12, 18, 25, 28], "tidi": 7, "form": [7, 9, 12, 26], "distribut": 7, "summar": [7, 8, 9, 14], "panda": [7, 8, 9, 10, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 29, 32], "img_df": 7, "crawlyt": [7, 29], "img_src": [7, 20], "img_alt": [7, 20], "img_load": 7, "img_siz": 7, "img_decod": 7, "img_width": 7, "img_height": 7, "img_bord": 7, "nytim": [7, 19], "vi": 7, "asset": [7, 11], "static": [7, 11, 14, 17], "icon": [7, 18], "morning_144x144": 7, "b12a6923b6ad9102b766352261b1a847": 7, "webp": 7, "morn": [7, 9, 26], "logo": [7, 11], "upshot_144x144": 7, "0b1553ff703bbd07ac8fe73e6d215888": 7, "upshot": 7, "static01": [7, 19], "nyt": [7, 19], "2017": [7, 11, 19, 28], "01": [7, 17, 18, 19, 28, 30], "29": [7, 9, 10, 14, 19, 30], "podcast": [7, 19], "daili": 7, "album": 7, "art": [7, 28], "square320": 7, "v5": 7, "jpg": [7, 11, 19], "qualiti": [7, 10, 11, 18], "auto": [7, 11], "disabl": [7, 18], "upscal": 7, "newslett": 7, "brief": 7, "europ": 7, "email": [7, 12, 24], "500px": 7, "australia": [7, 18], "australialett": 7, "interpret": 7, "sonl": 7, "theinterpret": 7, "section": [7, 17, 18, 28], "world": [7, 19], "middleeast": [7, 19], "2024": [7, 11, 30], "multimedia": [7, 19], "25israel": 7, "hbcz": 7, "thumbwid": 7, "min": [7, 14], "1024px": 7, "205px": 7, "150px": 7, "async": 7, "150": [7, 17], "hama": 7, "icj": 7, "explain": [7, 19, 32], "wjth": 7, "qatar": 7, "israel": 7, "ctbv": 7, "becaus": [7, 12, 14, 16, 17, 20, 23, 25, 26, 32], "particular": [7, 17, 18, 19, 28], "repres": [7, 9, 12, 14, 18, 19, 23, 26], "own": [7, 9, 13, 14, 18, 23, 25, 26, 28, 32], "row": [7, 13, 14, 17, 18, 19, 26], "seen": [7, 14], "repeat": [7, 9, 25, 29], "interest": [7, 9, 10, 11, 12, 13, 14, 17, 19, 20, 26, 30], "variou": [7, 8, 9, 10, 19, 26, 29, 32], "notna": 7, "averag": 7, "mean": [7, 9, 10, 12, 13, 14, 17, 18, 20, 23, 25, 26, 32], "sort_valu": [7, 26], "ascend": [7, 26], "to_fram": 7, "round": 7, "86": 7, "img_srcset": 7, "almost": [7, 14, 26], "height": [7, 11, 20, 28, 29], "immedi": [7, 17, 23], "estim": 7, "plan": [7, 14], "accordingli": 7, "webpag": [7, 18, 28], "understand": [7, 12, 13, 17, 18, 19, 23, 25, 32], "intern": [7, 10, 18, 19, 20, 23], "extern": [7, 10, 14], "summari": [7, 8, 9, 11, 29, 32], "link_df": 7, "internal_url_regex": 7, "nofollow": [7, 20, 29], "skip": [7, 20, 29], "dfp": 7, "advertis": 7, "middl": [7, 29], "east": 7, "suppli": [7, 11, 16, 18, 20, 29], "defin": [7, 12, 14, 18, 26, 28, 29], "realli": [7, 10, 13, 14, 20, 26], "could": [7, 9, 10, 13, 18, 26, 28, 29], "even": [7, 14, 17, 18, 19, 20, 23, 26, 28, 29], "consid": [7, 9, 23, 28, 32], "part": [7, 9, 13, 16, 18, 21, 25, 26, 27, 28, 32], "thu": [7, 10, 20, 26], "frequent": [7, 26], "inform": [7, 8, 10, 12, 14, 16, 18, 19, 20, 23, 25, 26, 28, 29], "present": [7, 18, 23, 29], "redirect_df": 7, "download_lat": [7, 10, 20], "redirect_tim": [7, 10, 20], "301": [7, 20], "220263": 7, "200": [7, 10, 14, 20, 23, 26], "privaci": 7, "polici": [7, 10], "079844": 7, "hc": 7, "10940941449492": 7, "403": 7, "0630789": 7, "13537530305428": 7, "218": 7, "spotlight": 7, "project": [7, 10, 12, 18, 20, 28, 32], "protect": 7, "852014": 7, "225": [7, 14], "regul": 7, "732559": 7, "310": 7, "sahil": 7, "chinoi": 7, "435062": 7, "intermedi": 7, "well": [7, 9, 10, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "latenc": [7, 10], "back": [7, 14, 23, 25, 26, 32], "memori": [7, 14, 20, 25], "imposs": [7, 25], "availablel": 7, "subset": [7, 19], "jsonlin": [7, 10, 20], "jl_subset": [7, 29], "massiv": [7, 14, 16, 25], "reduc": 7, "consumpt": 7, "small": [7, 8, 17, 18, 28], "delet": [7, 29], "old": 7, "crawl_subset": 7, "col1": 7, "col2": 7, "column_regex": 7, "img_": [7, 20], "availab": 7, "img": [7, 11, 20, 29], "jsonld_": [7, 20], "json": [7, 10, 14, 18, 20, 23, 29], "ld": [7, 10, 20, 29], "resp_headers_": [7, 20], "redirect_": [7, 20], "links_": 7, "characterist": 7, "either": [7, 9, 13, 18, 19, 23, 26, 28, 29, 32], "depend": [7, 9, 10, 12, 13, 19], "deal": [7, 32], "highli": [7, 18, 20, 28, 32], "perform": [7, 14, 16, 18, 19, 20, 28], "jl_to_parquet": [7, 29], "much": [7, 19, 20, 26, 29], "smaller": [7, 19, 28, 29], "disk": [7, 14], "power": [7, 9, 10, 20, 32], "effici": [7, 10, 14, 16, 20, 25], "read_parquet": [7, 14, 25], "pydata": 7, "_": [7, 19], "document": [7, 10, 12, 18, 20, 23, 26, 29], "advantag": [7, 14], "select": [7, 11, 14, 18, 20, 23, 28], "parquet_column": [7, 29], "nyt_crawl": 7, "value_count": [7, 14, 19], "215": 7, "doubl": [7, 29], "22": [7, 14, 17, 19, 30], "int64": [7, 14, 17, 19, 29], "struct": 7, "contenturl": [7, 12], "credittext": 7, "caption": [7, 18, 28, 29], "timestamp": [7, 14], "img_summari": 7, "image_df": 7, "chunksiz": 7, "chunk": 7, "jsonld": 7, "df_subset": 7, "jl_filepath": 7, "parquet_filepath": 7, "exist": [7, 8, 9, 13, 14, 23, 29], "parquet_fileapth": 7, "pather": 7, "identifi": [7, 18, 20, 23, 25, 28], "datatyp": 7, "columns_typ": 7, "chain": [7, 20], "inermedi": 7, "minu": [7, 20], "worth": 8, "helper": [8, 27], "aid": [8, 32], "emoji_entri": 8, "unicod": [8, 15], "textual": [8, 9], "v13": [8, 29], "public": [8, 10, 20, 23], "emoji_df": [8, 29], "extract_emoji": [8, 9, 29], "statist": [8, 9, 16, 28, 29, 32], "emoji_search": [8, 29, 32], "choic": 8, "emoji_raw": 8, "develop": [8, 12, 17, 18, 23, 28], "kaggl": [8, 32], "eliasdabba": 8, "whole": [8, 10, 20, 26], "databas": [8, 9, 14, 15, 29, 32], "vegetable_emoji": 8, "veget": 8, "codepoint": 8, "sub_group": 8, "1f951": 8, "fulli": [8, 9, 23], "qualifi": 8, "avocado": 8, "food": [8, 9, 28], "drink": [8, 9, 26], "1f346": 8, "eggplant": 8, "1f954": 8, "potato": 8, "1f955": 8, "carrot": 8, "1f33d": 8, "ear": 8, "corn": 8, "expect": [8, 10, 11, 20], "love_emoji": 8, "love": [8, 23, 26], "1f48c": 8, "smilei": [8, 9], "emot": [8, 9], "1f91f": 8, "gestur": 8, "peopl": [8, 9, 12, 23, 26, 32], "bodi": [8, 9, 10, 20, 23, 32], "hand": [8, 12, 13], "finger": 8, "partial": 8, "1f3fb": 8, "light": [8, 10], "skin": 8, "tone": 8, "1f3fc": 8, "medium": [8, 18, 24, 28], "1f3fd": 8, "1f3fe": 8, "dark": 8, "1f3ff": 8, "1f340": 8, "four": [8, 9, 17, 18, 20, 25, 26, 28], "leaf": 8, "clover": 8, "anim": [8, 9], "natur": [8, 9, 11], "plant": 8, "1f3e9": 8, "travel": [8, 9], "1f94a": 8, "box": [8, 17, 19, 28], "glove": 8, "activ": [8, 9, 12, 18, 19, 28], "sport": [8, 11, 19, 28], "1f9e4": 8, "object": [8, 9, 12, 14, 19, 20, 23, 28, 32], "cloth": 8, "1f1f8": 8, "1f1ee": 8, "flag": [8, 9], "slovenia": 8, "countri": [8, 18, 20, 25, 28, 29, 32], "social": [8, 9, 20, 21, 25, 26], "media": [8, 9, 11, 14, 20, 21, 23, 25, 26], "plai": [8, 12, 18, 19, 28], "around": [8, 9, 26], "sampl": [8, 12, 19, 20], "feel": [8, 9, 26, 29], "basketbal": [8, 28], "footbal": [8, 13, 19, 28, 32], "Not": [8, 23], "emoji_summari": [8, 9], "print": [8, 9, 14, 19, 21, 29], "entri": 8, "insensit": [8, 29], "dog": 8, "1f436": 8, "face": [8, 18], "mammal": 8, "1f415": 8, "1f9ae": 8, "guid": [8, 28], "200d": 8, "1f9ba": 8, "servic": [8, 12, 13, 18, 19], "1f32d": 8, "hot": [8, 9], "blue": [8, 9, 18, 25, 26, 28], "1f499": 8, "heart": 8, "1fad0": 8, "blueberri": 8, "fruit": 8, "1f4d8": 8, "book": [8, 26], "paper": 8, "1f535": 8, "circl": 8, "symbol": [8, 9, 29], "geometr": 8, "1f7e6": 8, "squar": 8, "1f537": 8, "diamond": 8, "1f539": 8, "ones": [8, 9, 10, 14, 16, 19, 20, 26], "post": [8, 9, 21, 23, 25, 26, 30, 32], "am": [8, 26], "grin": 8, "cat": 8, "hello": [8, 9, 17, 20], "dict_kei": [8, 9, 21], "emoji_text": [8, 9], "emoji_flat": [8, 9], "emoji_flat_text": [8, 9], "emoji_count": [8, 9], "emoji_freq": [8, 9], "top_emoji": [8, 9], "top_emoji_text": [8, 9], "top_emoji_group": [8, 9], "top_emoji_sub_group": [8, 9], "yellow": [8, 18], "flat": [8, 9], "number_of_emoji": 8, "smile": 8, "num_post": [8, 9], "num_emoji": [8, 9], "emoji_per_post": [8, 9], "unique_emoji": [8, 9], "infer": 9, "contrast": 9, "compani": [9, 12, 17], "brand": [9, 12, 14, 19], "extract_": [9, 29, 32], "fucntion": 9, "extract_curr": [9, 29], "surround": [9, 29], "abbrevi": 9, "usd": 9, "eur": 9, "jpy": 9, "extract_exclam": [9, 29], "excalam": 9, "mark": [9, 23, 25, 26, 27, 29], "extract_hashtag": [9, 29], "extract_intense_word": [9, 29], "intens": [9, 29], "neg": [9, 13, 23], "looooooovvvve": 9, "extract_ment": [9, 29], "network": [9, 25], "extract_numb": [9, 29], "extract_quest": [9, 29], "extract_url": [9, 29], "extract_word": [9, 29], "arbitrari": [9, 23, 29], "rest": [9, 12, 17], "restaur": 9, "along": [9, 23, 26], "recommend": [9, 14, 20, 23], "hashtag_summari": 9, "hashtags_flat": 9, "hashtag_count": 9, "hashtag_freq": 9, "top_hashtag": 9, "num_hashtag": 9, "hashtags_per_post": 9, "unique_hashtag": 9, "proper": [9, 32], "dataset": [9, 20, 25, 26, 32], "tweet": [9, 21, 23, 26, 32], "read_csv": [9, 26], "csv": [9, 11, 14, 26], "shape": [9, 14, 19], "tweet_text": [9, 26], "followers_count": [9, 26], "aerialmagzc": [9, 26], "penguinnyyyyi": [9, 26], "afraid": [9, 26], "real": [9, 23, 26], "157": [9, 26], "vibe": [9, 26], "offic": [9, 26], "metallica": [9, 26], "boss": [9, 26], "coffe": [9, 26], "break": [9, 19, 23, 26, 29, 32], "theoffic": [9, 26], "co": [9, 26], "u5vdyevvf": [9, 26], "4687": [9, 26], "ann": [9, 26], "sai": [9, 13, 18, 20, 26, 32], "she": [9, 26], "sugar": [9, 26], "hfubv4v3ai": [9, 26], "104": [9, 16, 20, 26], "venti": [9, 26], "ic": [9, 26, 28], "pump": [9, 26], "white": [9, 18, 26, 27], "mocha": [9, 26], "sweet": [9, 26], "cream": [9, 26], "caramel": [9, 26], "drizzl": [9, 26], "shout": [9, 26], "tiktok": [9, 26], "lol": [9, 26], "126": [9, 26], "never": [9, 26], "person": [9, 12, 18, 26], "until": [9, 23, 26], "kid": [9, 26], "cup": [9, 26], "life": [9, 26, 30], "saver": [9, 26], "zo0cnvuigj": [9, 26], "1595": [9, 26], "excit": [9, 26], "next": [9, 13, 20, 26, 28, 32], "chat": [9, 26], "re": [9, 23, 26], "john": [9, 26], "bradford": [9, 26], "lineup": [9, 26], "discuss": [9, 26], "redistrict": [9, 26], "area": [9, 18, 26, 28, 32], "rsvp": [9, 26], "r3ynjjjcug": [9, 26], "join": [9, 20, 26], "meet": [9, 18, 26, 28], "ho4kx7zz24": [9, 26], "kfpdr3hupi": [9, 26], "5004": [9, 26], "paid": [9, 26], "husband": [9, 26], "165": [9, 26], "nippli": [9, 26], "outsid": [9, 14, 18, 26, 28], "side": [9, 14, 26, 27], "sound": [9, 26], "blowjob": [9, 26], "front": [9, 26], "visit": [9, 26], "green": [9, 18, 26], "tea": [9, 26], "hahahahahahaha": [9, 26], "spend": [9, 23, 26, 32], "pamper": [9, 26], "hope": [9, 26, 32], "everyon": [9, 26], "tuesdai": [9, 26], "enjoi": [9, 26], "189": [9, 26], "marvinmilton2": [9, 26], "nearli": [9, 26], "choke": [9, 26], "1160": [9, 26], "2000": 9, "733": 9, "3665": 9, "572": 9, "mention_summari": 9, "mentions_flat": 9, "mention_count": 9, "mention_freq": 9, "top_ment": 9, "num_ment": 9, "1346": 9, "mentions_per_post": 9, "673": 9, "unique_ment": 9, "1132": 9, "zip": [9, 14, 29], "currency_summari": 9, "currency_symbol": 9, "currency_symbols_flat": 9, "currency_symbol_count": 9, "currency_symbol_freq": 9, "top_currency_symbol": 9, "currency_symbol_nam": 9, "surrounding_text": 9, "num_currency_symbol": 9, "37": [9, 18, 19, 28], "currency_symbols_per_post": 9, "0185": 9, "unique_currency_symbol": 9, "sym": 9, "number_summari": 9, "numbers_flat": 9, "number_count": 9, "number_freq": 9, "top_numb": 9, "num_numb": 9, "1727": 9, "numbers_per_post": 9, "8635": 9, "unique_numb": 9, "257": 9, "question_summari": 9, "question_mark": 9, "question_marks_flat": 9, "question_mark_count": 9, "question_mark_freq": 9, "top_question_mark": 9, "question_mark_nam": 9, "question_text": 9, "num_question_mark": 9, "321": [9, 12], "question_marks_per_post": 9, "1605": 9, "unique_question_mark": 9, "ckaiserjr": 9, "perry_ron": 9, "lilguyisback": 9, "okai": 9, "water": 9, "flavor": 9, "think": [9, 20, 25], "ll": [9, 17, 23, 26], "loos": 9, "mayb": [9, 11, 13, 17], "exclamation_summari": 9, "exclamation_mark": 9, "exclamation_marks_flat": 9, "exclamation_mark_count": 9, "exclamation_mark_freq": 9, "top_exclamation_mark": 9, "exclamation_mark_nam": 9, "exclamation_text": 9, "num_exclamation_mark": 9, "563": 9, "exclamation_marks_per_post": 9, "2815": 9, "unique_exclamation_mark": 9, "1149": 9, "5745": 9, "279": 9, "emoji_nam": 9, "72": [9, 14, 17, 28], "49": [9, 12, 14], "210": 9, "97": [9, 16], "67": [9, 20], "33": [9, 12, 17], "key_nam": 9, "kwarg": [9, 10, 11], "singular": 9, "straightforward": [9, 11, 20, 25], "left_char": 9, "right_char": 9, "dict": [9, 10, 11, 14, 20], "number_of_symbol": 9, "bitcoin": 9, "dollar": [9, 27], "pound": 9, "euro": 9, "odai": 9, "ound": 9, "6666666666666667": 9, "written": [9, 17, 18, 19], "said": [9, 17], "No": [9, 18], "6666666666666666": 9, "posts2": 9, "\u0645\u0631\u062d\u0628\u0627": 9, "\u0644\u0627": 9, "\u062a\u0630\u0647\u0628": 9, "hola": 9, "c\u00f3mo": 9, "est\u00e1": 9, "displai": [9, 14, 17, 18, 28], "opposit": 9, "due": [9, 14, 23, 26, 28], "rtl": 9, "invert": 9, "number_of_hashtag": 9, "min_rep": 9, "instanc": [9, 12, 29], "repetit": 9, "looooooveee": 9, "youuuuuuu": 9, "haaatttteee": 9, "youuuuuu": 9, "both": [9, 13, 18, 20, 23, 26, 27, 28, 32], "jenni": 9, "hi": [9, 19, 26], "number_of_ment": 9, "number_separ": 9, "333": 9, "444": 9, "555": 9, "number_of_numb": 9, "ask": [9, 18, 23, 29], "armenian": 9, "\u03c0\u03ce\u03c2": 9, "\u03b5\u03af\u03c3\u03b1\u03b9": 9, "\u0643\u064a\u0641": 9, "\u062d\u0627\u0644\u0643": 9, "did": [9, 19, 26], "notic": [9, 17], "correct": [9, 12, 29], "NOT": [9, 18, 28], "valid": [9, 18, 23, 24, 28], "b": [9, 10, 14], "url_summari": 9, "urls_flat": 9, "url_count": 9, "url_freq": 9, "top_url": 9, "top_domain": 9, "top_tld": 9, "number_of_url": 9, "num_url": 9, "urls_per_post": 9, "unique_url": 9, "words_to_extract": 9, "entire_words_onli": 9, "complet": [9, 13, 18, 23, 25, 26, 28], "words_to_find": 9, "rain": [9, 26], "snow": [9, 26], "noth": [9, 26], "word_summari": 9, "words_flat": 9, "word_count": 9, "word_freq": [9, 26], "top_word": 9, "num_word": 9, "words_per_post": 9, "unique_word": 9, "number_of_word": 9, "occurr": [9, 26], "occur": [9, 19, 20, 23, 26, 28, 29], "train": 9, "relat": [9, 18, 19, 20, 28, 32], "mini": 10, "known": [10, 20, 23, 26, 29], "hood": 10, "simplifi": [10, 18, 28], "interfac": [10, 18, 23, 29, 30, 32], "crawl_head": [10, 29], "assur": 10, "super": [10, 14], "fast": [10, 16, 29], "straight": 10, "forward": 10, "readthedoc": [10, 16, 20, 23], "io": [10, 16, 20, 23], "adver": [10, 14, 20], "dashboardom": 10, "povertydata": 10, "headers_df": 10, "crawl_tim": [10, 20], "download_timeout": [10, 20], "download_slot": [10, 20], "protocol": 10, "resp_headers_cont": [10, 20], "resp_headers_serv": [10, 20], "resp_headers_d": [10, 20], "resp_headers_vari": [10, 20], "redirect_ttl": [10, 20], "redirect_url": [10, 20], "redirect_reason": [10, 20], "resp_headers_x": [10, 20], "amz": 10, "resp_headers_last": [10, 20], "resp_headers_etag": 10, "serv": [10, 20], "backend": [10, 20], "rtd": [10, 20], "method": [10, 14, 18, 20, 23, 28, 29, 30], "resp_headers_referr": 10, "resp_headers_permiss": 10, "resp_headers_strict": [10, 20], "transport": [10, 19, 20], "secur": [10, 20], "resp_headers_cf": [10, 20], "cach": [10, 20], "resp_headers_ag": [10, 20], "resp_headers_expir": [10, 20], "resp_headers_cach": [10, 20], "resp_headers_expect": [10, 20], "ct": [10, 20], "rai": [10, 20], "resp_headers_alt": 10, "svc": 10, "resp_headers_via": 10, "2022": [10, 14, 17, 19, 30], "180": [10, 20], "0270483": 10, "nginx": [10, 20], "18": [10, 11, 14, 19, 30], "fri": 10, "feb": [10, 14], "gmt": [10, 20], "utf": [10, 14, 23], "applic": [10, 12, 14, 18, 20, 23, 28], "xhtml": [10, 20], "q": [10, 18, 23, 28], "rc2": 10, "06442": 10, "13270": 10, "0271282": 10, "cloudflar": [10, 20], "19": [10, 11, 17, 20, 30], "302": [10, 20], "rnkt7myjj7hcnsvbnzg9qdqizefftx9ytz3": 10, "gwnlj8m99yumucgdd6ytm": 10, "ibmo9hrztai": 10, "iyl50": 10, "ee0djx6z511tgx88": 10, "17": [10, 14, 16, 17, 19, 20, 30], "04": [10, 19, 30], "27": [10, 17, 19, 28, 30], "w": [10, 18, 19], "14c904a172315a4922f4d28948b916c2": 10, "proxito": [10, 20], "sendfil": [10, 20], "0710e93d610dd8c3": 10, "subdomain": [10, 20], "referr": [10, 24], "downgrad": 10, "cohort": [10, 32], "ag": [10, 20], "31536000": [10, 20], "includesubdomain": 10, "preload": 10, "1083": 10, "7200": 10, "604800": [10, 20], "report": [10, 16, 20, 25, 26, 29, 32], "uri": [10, 20], "cdn": 10, "cgi": 10, "beacon": 10, "6dba2aae6b424107": 10, "prg": 10, "443": 10, "ma": [10, 20], "86400": 10, "118614": 10, "26837": 10, "gunicorn": 10, "vegur": 10, "tip": 10, "mainten": 10, "task": [10, 13, 26, 27, 32], "continu": [10, 32], "hundr": [10, 20, 26], "period": 10, "basi": 10, "alert": 10, "ye": [10, 12], "ok": 10, "compon": [10, 14, 23, 25, 29], "metatag": 10, "direct": [10, 17], "noindex": 10, "byte": [10, 19, 20, 29], "With": [10, 19, 23, 26, 32], "consum": [10, 23, 25], "bandwidth": 10, "lookout": 10, "jpeg": 10, "png": [10, 11, 19, 20], "class": [10, 11, 20], "headersspid": 10, "arg": [10, 11], "autothrottle_en": [10, 11], "autothrottle_target_concurr": [10, 11], "httperror_allow_al": [10, 11], "errback": 10, "failur": 10, "headers_spid": 10, "start_request": [10, 11], "sine": 10, "speed": [10, 20, 29], "piec": 10, "expens": 10, "Being": 10, "abl": [10, 12, 17, 18, 32], "decis": [10, 19, 32], "optim": [10, 12], "dynam": [10, 20], "crawl_df": [10, 20], "experiment": [11, 29], "crawl_imag": [11, 29], "output_dir": 11, "min_width": 11, "minimum": 11, "avoid": [11, 25], "track": [11, 28, 29, 32], "navig": [11, 23], "elemenst": 11, "min_height": 11, "include_img_regex": 11, "Or": [11, 13, 20, 25], "economi": 11, "summarize_crawled_img": 11, "image_loc": [11, 19], "image_url": 11, "buzzfe": 11, "hannahdobro": 11, "dirti": 11, "littl": [11, 25, 30, 32], "industri": [11, 17, 18, 19, 32], "secret": 11, "tuh": 11, "user_imag": 11, "6r1oxxopc_larg": 11, "downsiz": 11, "120": 11, "03": [11, 17, 19, 30], "fce856744ed8": 11, "buzz": 11, "1303": 11, "1710779249": 11, "gif": 11, "base64": 11, "r0lgodlhaqabaiaaaaaaap": 11, "yh5baeaaaaalaaaaaabaaeaaaibraa7": 11, "245ecfa321e9": 11, "894": 11, "1710779358": 11, "chelseastewart": 11, "josh": 11, "peck": 11, "statement": 11, "drake": 11, "bell": 11, "abus": 11, "claim": [11, 16], "prod": 11, "v2": 11, "5590": 11, "1513102854": 11, "0_larg": 11, "ea6298160040": 11, "1093": 11, "1711048323": 11, "700": 11, "3a": 11, "2a": 11, "ivborw0kggoaaaansuheugaaafqaaaa7camaaadsf118aaaap1bmveuaaadigxpohbk5ewdfghi5fwi8grteghe7eqdmhr7": 11, "vymfddnm5hx334": 11, "py8fhdj5dlvvxnq6zjotzvbg1s8skwaaaacxrstlmav4eo10jnqa8ihfydaaabjuleqvryw93y64rcmbcg4czk5fszdav3f63bdaxfv4qm": 11, "axr96": 11, "wmnj0klhtpib9lcutya8k": 11, "f1rkxqh4kmipzviovwnszequmfjmvlb3": 11, "ysriv8zrqmwha1znqibuuv3jo3cn5fly3qimy2kitajb3": 11, "umlrxrgovgmqtj4hxc69an5hj9pcyyqzfxsavk58tjmntwgv24pw9kpe0fgbioklomczkngleuxlhyiimx": 11, "dt": 11, "xj8sxgocdz6ejcp7jspbqllibivmpewy7as1poez30pvqlaqvjrgeqtlfp1dblpyb0bdd": 11, "oyl2nhr7e34yujtjw6zmc3am": 11, "kxlspoodchrqwiwbxi85q6kc9pnehscmhj0vjgppuac3lwqo": 11, "ourl0aefg76m8izrt6eaaaaasuvork5cyii": 11, "josephlongo": 11, "celeb": 11, "wear": 11, "rewear": 11, "dress": 11, "2021": [11, 17, 19, 30], "06": [11, 19, 30], "a824550933a9": 11, "tomiobaro": 11, "2174": 11, "1622738336": 11, "41_larg": 11, "6634db63f453": 11, "576": [11, 12], "1710855734": 11, "cb8db05df7e7": 11, "1743": 11, "1710855790": 11, "taken": 11, "slug": [11, 19, 23, 25], "slash": 11, "locat": [11, 18, 19, 23, 25, 28, 32], "tabl": [11, 12, 13, 20, 32], "advimagespipelin": 11, "store_uri": 11, "download_func": 11, "imagespipelin": 11, "file_path": 11, "info": [11, 14, 18, 29], "store": [11, 14, 20], "imagespid": 11, "item_pipelin": 11, "image_spid": 11, "imgitem": 11, "start_url": 11, "behaviour": [11, 20], "image_dir": 11, "tha": 11, "rank": [12, 20, 29, 32], "zero": [12, 18, 20, 28], "comparison": [12, 16], "elig": 12, "score": [12, 26], "suitabl": 12, "critic": [12, 18], "clear": [12, 13, 23], "reliabl": 12, "view": [12, 18, 23, 26, 28], "send": [12, 14, 23, 32], "bill": [12, 18], "credenti": [12, 18, 23, 28], "shown": [12, 25], "below": [12, 18, 20, 26, 27, 28], "And": [12, 14], "your_google_developer_kei": 12, "knowledge_graph": [12, 29], "resultscor": 12, "203191": 12, "corpor": 12, "organ": 12, "technologi": [12, 28], "49462": 12, "19142": 12, "gmail": 12, "13251": 12, "7549": 12, "softwareappl": 12, "drive": 12, "6853": 12, "6543": 12, "4312": 12, "multin": 12, "conglomer": 12, "alphabet": [12, 18, 28], "inc": 12, "3395": 12, "1306": 12, "detaileddescript": 12, "articlebodi": 12, "licens": [12, 18, 28], "query_tim": [12, 29], "dtype": [12, 14, 17, 19], "203": [12, 14], "191": 12, "462": 12, "understood": 12, "fall": [12, 18, 23, 28], "inherit": 12, "everyth": [12, 23, 26], "hierarchi": 12, "belong": [12, 16, 19, 23], "funcion": 12, "manner": [12, 25], "aggreg": 12, "fr": [12, 18, 20], "evalu": 12, "3587": 12, "suchmaschinenoptimierung": 12, "lokal": 12, "252": 12, "suchmaschinenmarket": 12, "71756": 12, "5056": 12, "seop": 12, "3313": 12, "seoul": 12, "administrativearea": 12, "hauptstadt": 12, "von": 12, "s\u00fcdkorea": 12, "1509": 12, "yea": 12, "ji": 12, "schauspielerin": 12, "584": 12, "actriz": 12, "posicionamiento": 12, "buscador": 12, "35": [12, 14, 20], "316": 12, "jin": 12, "cantant": 12, "53": [12, 14], "8760": 12, "south": 12, "korea": 12, "1435": 12, "sulli": 12, "korean": [12, 18], "actress": 12, "prefix": [12, 29], "state": [12, 18, 19, 25, 28], "liter": 12, "iso": [12, 18, 23, 28], "639": [12, 18, 23, 28], "schema": 12, "enabl": [12, 18, 29], "substr": 12, "against": [12, 23, 26], "alias": 12, "jung": 12, "jungl": 12, "ho": 12, "kang": 12, "higher": [12, 18, 19, 28, 29], "chanc": 12, "kg_df": 12, "v1": 12, "properli": [13, 14, 18, 19, 25, 28, 29], "right": [13, 14, 18, 19, 25], "research": [13, 17, 32], "tediou": [13, 25], "shift": 13, "oppos": [13, 25, 29], "anywai": [13, 26], "phrase": [13, 18, 23, 26, 27, 29], "barcelona": 13, "guitar": 13, "rio": 13, "janeiro": 13, "trip": 13, "club": [13, 32], "verb": 13, "purchas": 13, "noun": 13, "intent": [13, 32], "price": [13, 20, 25], "offer": [13, 23], "clearli": [13, 25], "aren": 13, "tutori": [13, 20, 32], "certif": 13, "learn": [13, 19, 20, 26, 28, 32], "educ": 13, "fifteen": [13, 26], "twenti": [13, 17, 26], "segment": [13, 23], "target": [13, 28], "shouldn": [13, 14], "difficult": 13, "commerc": [13, 26], "focu": [13, 16, 32], "cheap": 13, "discount": 13, "luxuri": 13, "signifi": 13, "graphic": 13, "design": [13, 17, 32], "career": [13, 17], "vacanc": 13, "kw_gener": [13, 20, 29, 32], "possibl": [13, 14, 17, 18, 26, 32], "upload": [13, 18, 19, 23, 28], "kw_df": 13, "criterion": 13, "label": 13, "sem_campaign": 13, "625": [13, 16], "626": 13, "627": 13, "628": 13, "629": 13, "630": 13, "bottom": [13, 26, 32], "kw_broad": 13, "tutor": 13, "kw_exact": 13, "match_typ": 13, "capitalize_adgroup": [13, 29], "order_matt": 13, "frame": 13, "relev": [13, 18, 23, 24, 28], "final": [13, 19, 25, 28, 29], "keywords_df": 13, "tail": 13, "57": [13, 17, 19], "retain": [13, 25], "integr": [13, 32], "kw_modifi": 13, "kw_neg_broad": 13, "kw_neg_exact": 13, "kw_neg_phras": 13, "kw_phrase": 13, "event": [14, 18, 19, 25, 28], "complex": [14, 23], "ourselv": [14, 26], "pageview": [14, 26], "mai": [14, 17, 18, 20, 23, 26, 28], "session": [14, 18, 23], "characterisit": 14, "usuali": 14, "cater": 14, "rapid": 14, "tl": 14, "dr": 14, "access_log": 14, "log_error": 14, "logs_df": 14, "try": [14, 17, 18, 20, 26, 28, 32], "certainli": 14, "conform": 14, "weren": 14, "went": 14, "wrong": [14, 17], "fix": [14, 20, 29], "temporari": 14, "debug": [14, 18], "howev": [14, 18, 23, 26, 28], "singl": [14, 17, 18, 20, 23], "distinguish": [14, 20], "client": 14, "k": [14, 26], "extend": [14, 23], "effect": [14, 23, 25], "importantli": [14, 25, 32], "datetim": [14, 18, 19, 28, 29], "date": [14, 17, 18, 19, 20, 23, 28, 29, 32], "categor": [14, 25], "storag": [14, 19], "to_datetim": 14, "hostnam": [14, 16, 25], "ip": [14, 16, 18, 20, 29], "address": [14, 16, 20, 29], "reverse_dns_lookup": [14, 16, 29], "resourc": [14, 18, 20, 23, 28, 32], "url_to_df": [14, 19, 25, 29, 32], "famili": [14, 23], "oper": [14, 18, 19, 23, 28, 29], "system": [14, 25, 29], "non": [14, 18, 26, 27, 28, 29, 32], "sample_log": 14, "66": [14, 16, 20], "249": [14, 16], "00": [14, 17, 18, 19, 28], "0000": 14, "1095": 14, "5x": 14, "4758": 14, "109": 14, "237": 14, "103": 14, "39": [14, 16, 17, 19, 20], "env": 14, "404": [14, 20], "209": 14, "81": 14, "4044": 14, "129": 14, "223": 14, "214": 14, "23": [14, 17, 19, 20, 30], "2240": 14, "4430": 14, "68": [14, 17, 20], "77": [14, 17], "192": 14, "241": 14, "211": [14, 16], "176": 14, "login": [14, 17], "zgrab": 14, "stage": 14, "urlyt": 14, "520": [14, 19], "_dash": 14, "suit": 14, "dash": [14, 29], "dash_html_compon": 14, "v2_0_0m1638886228": 14, "154258": 14, "layout": [14, 28], "2547": 14, "ua_pars": 14, "user_agent_pars": 14, "max_column": 14, "adv_log": 14, "adv_error": 14, "y": [14, 18], "z": [14, 19], "host_df": [14, 16], "1210": 14, "745": 14, "sy": 14, "729": 14, "wall": 14, "ip_address": [14, 16, 20], "cum_count": [14, 16], "perc": [14, 16], "cum_perc": [14, 16], "aliaslist": [14, 16], "ipaddrlist": [14, 16], "143": 14, "244": 14, "132": 14, "426": 14, "0701004": 14, "errno": [14, 16], "unknown": [14, 16], "host": [14, 16, 18, 29], "146": [14, 17], "164": 14, "110": 14, "290": [14, 17], "716": 14, "0477209": 14, "117821": 14, "177": 14, "196": 14, "171": 14, "908": 14, "0315945": 14, "149416": 14, "ppp046177196171": 14, "hol": 14, "gr": 14, "addr": [14, 16], "arpa": [14, 16], "185": [14, 16], "173": 14, "182": 14, "1090": 14, "029949": 14, "179365": 14, "226": 14, "1261": 14, "0281389": 14, "207504": 14, "174": 14, "154": 14, "1415": 14, "0253415": 14, "232845": 14, "89": 14, "44": [14, 17], "1545": 14, "0213921": 14, "254237": 14, "ppp089047044105": 14, "1664": 14, "019582": 14, "273819": 14, "234": 14, "113": 14, "1777": 14, "0185947": 14, "292414": 14, "217": 14, "1858": 14, "0133289": 14, "305743": 14, "d9646265": 14, "ziggozakelijk": 14, "nl": 14, "163": 14, "243": [14, 16], "1937": 14, "0129998": 14, "318743": 14, "2014": [14, 19], "0126707": 14, "331414": 14, "194": [14, 16], "179": 14, "2074": 14, "00987329": 14, "341287": 14, "vmi660635": 14, "contaboserv": 14, "net": [14, 19], "137": 14, "2132": 14, "00954418": 14, "350831": 14, "2190": 14, "360375": 14, "tor": 14, "anonym": 14, "appliedprivaci": 14, "adress": [14, 16], "ip_host_dict": 14, "request_url_df": 14, "add_prefix": 14, "request_": 14, "request_url": 14, "request_schem": 14, "request_netloc": 14, "request_path": 14, "request_queri": 14, "request_frag": 14, "request_hostnam": 14, "request_port": 14, "request_dir_1": 14, "request_dir_2": 14, "request_dir_3": 14, "request_dir_4": 14, "request_dir_5": 14, "request_dir_6": 14, "request_dir_7": 14, "request_dir_8": 14, "request_dir_9": 14, "request_dir_10": 14, "request_dir_11": 14, "request_dir_12": 14, "request_dir_13": 14, "request_last_dir": 14, "request_query_index": 14, "request_query_": 14, "request_query_xdebug_session_start": 14, "request_query_funct": 14, "request_query_var": 14, "request_query_fil": 14, "request_query_url": 14, "request_query_a": 14, "request_query_cont": 14, "request_query_wt": 14, "request_query_act": 14, "request_query_usernam": 14, "request_query_psd": 14, "request_query_dn": 14, "request_query_step": 14, "request_query_cmd": 14, "request_query_lang": 14, "request_query_opt": 14, "request_query_folderid": 14, "request_query_input_fil": 14, "request_query_currentset": 14, "request_query_typ": 14, "request_query_next_fil": 14, "request_query_curpath": 14, "request_query_pag": 14, "request_query_id": 14, "request_query_img": 14, "request_query_panel": 14, "request_query_todo": 14, "request_query_cod": 14, "request_query_ref": 14, "request_query_scopenam": 14, "request_query_op": 14, "request_query_control": 14, "request_query_q": 14, "request_query_sb_categori": 14, "request_query_email": 14, "request_query_nam": 14, "request_query_abspath": 14, "request_query_fn": 14, "request_query_thumb": 14, "request_query_nocontinu": 14, "request_query_filepath": 14, "request_query_file_link": 14, "request_query_mypath": 14, "request_query_adapt": 14, "source_fil": 14, "request_query_aam": 14, "request_query_cpabc_calendar_upd": 14, "request_query_term": 14, "request_query_itemid": 14, "request_query_search_kei": 14, "request_query_short": 14, "request_query_titl": 14, "request_query_format": 14, "request_query_findcli": 14, "request_query_v": 14, "request_query_target": 14, "request_query__": 14, "request_query_albid": 14, "request_query_p": 14, "request_query_path": 14, "request_query_mod": 14, "request_query_libpath": 14, "request_query_srt": 14, "request_query_redirect": 14, "request_query_ord": 14, "request_query_item": 14, "request_query_gid": 14, "request_query_rid": 14, "request_query_servic": 14, "request_query_ag": 14, "request_query_typeid": 14, "request_query_dir": 14, "request_query_stockcodeintern": 14, "request_query_sit": 14, "request_query_posit": 14, "request_query_filenam": 14, "referer_url_df": 14, "referer_": 14, "referer_url": 14, "referer_schem": 14, "referer_netloc": 14, "referer_path": 14, "referer_queri": 14, "referer_frag": 14, "referer_hostnam": 14, "referer_port": 14, "referer_dir_1": 14, "referer_dir_2": 14, "referer_dir_3": 14, "referer_last_dir": 14, "ua_df": 14, "json_norm": [14, 29], "ua": 14, "ua_": 14, "ua_str": 14, "ua_famili": 14, "ua_major": 14, "ua_minor": 14, "ua_patch": 14, "ua_o": 14, "major": [14, 17, 19, 23, 32], "minor": [14, 29], "patch": 14, "patch_minor": 14, "ua_devic": 14, "smartphon": [14, 19], "desktop": 14, "concat": [14, 20, 29], "axi": 14, "to_parquet": 14, "adv_logs_fin": 14, "doen": 14, "load": [14, 20, 22, 29], "satisfi": 14, "top_bot": 14, "499": 14, "petalbot": 14, "ahrefsbot": 14, "yandexbot": 14, "linkedinbot": [14, 17], "baiduspid": [14, 17], "dotbot": 14, "twitterbot": [14, 17], "mj12bot": 14, "java": 14, "nutch": 14, "masscan": 14, "facebookbot": 14, "happi": [14, 23], "By": [14, 18, 20, 26, 28], "destin": [14, 25], "stdout": 14, "review": [14, 18, 19, 28], "altogeth": 14, "chose": 14, "crawllogs_to_df": [14, 29], "open": [14, 17, 19, 20, 29], "core": 14, "scraper": 14, "handler": 14, "method_to": 14, "redirect_to": 14, "method_from": 14, "redirect_from": 14, "blocked_url": 14, "logs_file_path": 14, "itself": [14, 20, 25, 26], "und": 14, "crawl_logs_to_df": 14, "crawl_logs_df": 14, "conformig": 14, "chosen": 14, "log_field": 14, "must": [14, 18, 23, 26, 28], "reader": 14, "latin": [14, 28], "regex_raw": 15, "hashtag_raw": 15, "mention_raw": 15, "raw": 15, "share": [15, 20, 26], "compil": 15, "readabl": [15, 17, 20, 32], "annot": 15, "v11": 15, "cookbook": 15, "2nd": 15, "ed": 15, "reilli": 15, "verifi": [16, 19], "pipelin": [16, 32], "pointer": 16, "comand": 16, "375": 16, "mail": 16, "garda": 16, "ir": 16, "875": 16, "shatel": 16, "cumul": [16, 26], "percentag": [16, 26, 29], "attent": 16, "max_work": [16, 19, 29], "equival": [16, 20], "worker": [16, 19], "multi": [16, 18], "though": [17, 18, 20, 23, 26, 28], "tini": 17, "potent": 17, "instruct": [17, 18, 28], "suppos": [17, 25], "mistak": 17, "ideal": [17, 25, 26, 32], "robotstxt_to_df": [17, 29], "etag": [17, 19, 29], "robotstxt_last_modifi": [17, 29], "robotstxt_url": 17, "download_d": [17, 19, 29], "a850165d925db701988daf7ead7492d3": 17, "200689": 17, "disallow": [17, 20], "exec": 17, "obido": 17, "style": [17, 20, 29], "flex": 17, "hp": 17, "mystuff": 17, "147": 17, "gp": 17, "profil": [17, 23], "148": 17, "149": 17, "etaospid": 17, "delai": 17, "ey": 17, "robots_url": 17, "googtwfb": 17, "groupbi": 17, "541": 17, "289": 17, "07": [17, 19, 30], "375724": 17, "howsearchwork": 17, "comment": [17, 20, 28, 29, 32], "nat": [17, 19], "461815": 17, "291": 17, "292": [17, 19], "_escaped_fragment_": 17, "293": 17, "lang": [17, 23], "397": 17, "474456": 17, "398": 17, "prohibit": 17, "unless": [17, 26], "permiss": 17, "399": 17, "conduct": 17, "purpos": 17, "401": 17, "app": [17, 18, 23, 28, 32], "site_scraping_tos_term": 17, "php": 17, "robotstxt_test": [17, 29], "owner": [17, 18, 23, 28], "realiti": 17, "appli": [17, 18, 19, 20], "care": 17, "fb_robot": 17, "951053": 17, "ajax": 17, "pagelet": 17, "pagepostssectionpagelet": 17, "538": [17, 32], "safetycheck": 17, "539": 17, "540": 17, "fb_userag": 17, "drop_dupl": 17, "tolist": [17, 19], "applebot": 17, "discordbot": 17, "facebookexternalhit": 17, "ia_archiv": 17, "msnbot": 17, "naverbot": 17, "pinterestbot": 17, "seznambot": 17, "teoma": 17, "telegrambot": 17, "yandex": 17, "yeti": 17, "quit": [17, 19, 26], "bbc": [17, 19], "urls_to_test": 17, "fb_test": 17, "url_path": 17, "can_fetch": 17, "76": 17, "receiv": [17, 23], "eighti": 17, "denot": 17, "24": [17, 19, 30], "40": [17, 18, 20], "figur": [17, 20, 23, 32], "linkedin": 17, "pinterest": 17, "clue": 17, "robotx": 17, "robotstxt_test_df": 17, "soon": 17, "robotstxt_df": 17, "2020": [17, 19, 20, 30], "09": [17, 19, 20, 30], "702814": 17, "08": [17, 19, 30], "087985": 17, "283": 17, "284": 17, "imgr": 17, "285": 17, "286": 17, "468588": 17, "287": 17, "lose": 17, "patienc": 17, "robots_output_fil": 17, "robotsfiles_df": 17, "fill": 18, "questionnair": 18, "survei": [18, 30], "serp_": [18, 32], "dimens": [18, 28], "serp_goog": [18, 20, 29, 32], "best": [18, 23, 26], "gl": 18, "ca": [18, 26], "uk": 18, "au": 18, "nz": 18, "ten": [18, 26], "450": 18, "snippet": [18, 20, 28, 29], "querytim": [18, 29], "serp_youtub": [18, 29], "At": [18, 28], "enter": [18, 20], "panel": 18, "remov": [18, 20, 23, 26, 27, 28, 29], "entir": [18, 23], "retriev": [18, 19, 23, 28], "programmat": 18, "free": 18, "pai": 18, "cx": 18, "c2coff": 18, "cr": 18, "daterestrict": 18, "exactterm": 18, "excludeterm": 18, "filetyp": 18, "highrang": 18, "hl": [18, 28], "hq": 18, "imgcolortyp": 18, "imgdominantcolor": 18, "imgsiz": 18, "imgtyp": 18, "linksit": 18, "lowrang": 18, "lr": 18, "num": 18, "orterm": 18, "safe": [18, 23], "searchtyp": 18, "sitesearch": 18, "sitesearchfilt": 18, "sort": [18, 21, 25, 26, 28, 29], "tradit": [18, 28], "disabled0": 18, "boolean": [18, 20, 28], "tld": [18, 29], "urlth": 18, "geograph": [18, 28], "addressse": 18, "past": [18, 32], "week": [18, 19, 23, 32], "month": [18, 19, 23], "year": [18, 19, 23], "appear": [18, 19, 20, 23, 26, 29], "center": [18, 28], "crowd": 18, "improv": [18, 26, 29], "geoloc": 18, "boost": 18, "whose": [18, 26], "lead": 18, "particularli": [18, 20], "speak": 18, "unit": [18, 19, 23, 28], "rang": [18, 23], "inclus": [18, 28], "explicitli": [18, 23, 25], "internation": 18, "term": [18, 20, 24, 28, 32], "logic": 18, "AND": 18, "black": [18, 20, 26], "grayscal": 18, "color": [18, 20, 25, 26], "mono": 18, "grai": 18, "domin": 18, "brown": 18, "orang": [18, 26], "pink": 18, "purpl": 18, "red": [18, 25], "teal": 18, "huge": 18, "xlarg": 18, "xxlarg": 18, "clipart": 18, "lineart": 18, "photo": [18, 23], "lang_ja": 18, "lang_ar": 18, "lang_bg": 18, "bulgarian": 18, "lang_ca": 18, "lang_c": 18, "czech": 18, "lang_da": 18, "lang_d": 18, "lang_el": 18, "lang_en": 18, "lang_": 18, "lang_et": 18, "estonian": 18, "lang_fi": 18, "lang_fr": 18, "lang_hr": 18, "lang_hu": 18, "lang_id": 18, "lang_i": 18, "iceland": 18, "lang_it": 18, "lang_iw": 18, "lang_ko": 18, "lang_lt": 18, "lithuanian": 18, "lang_lv": 18, "latvian": 18, "lang_nl": 18, "lang_no": 18, "lang_pl": 18, "lang_pt": 18, "lang_ro": 18, "lang_ru": 18, "lang_sk": 18, "slovak": 18, "lang_sl": 18, "slovenian": 18, "lang_sr": 18, "serbian": 18, "lang_sv": 18, "lang_tr": 18, "lang_zh": 18, "cn": 18, "tw": 18, "least": [18, 26, 28], "cc_publicdomain": 18, "cc_attribut": 18, "cc_sharealik": 18, "cc_noncommerci": 18, "cc_nonderiv": 18, "safeti": 18, "safesearch": [18, 28], "unspecifi": 18, "forth": [18, 23, 28], "ever": [18, 26], "serp_df": 18, "usa": 18, "franc": 18, "your_cx": 18, "your_kei": 18, "prouc": 18, "fligt": 18, "ticket": 18, "focus": 18, "flight": [18, 23], "countryuk": 18, "countryau": 18, "channelid": [18, 28], "channeltyp": [18, 28], "eventtyp": [18, 28], "forcontentown": [18, 28], "fordevelop": [18, 28], "formin": [18, 28], "locationradiu": [18, 28], "maxresult": [18, 28], "onbehalfofcontentown": [18, 28], "pagetoken": [18, 28], "publishedaft": [18, 28], "publishedbefor": [18, 28], "regioncod": [18, 28], "relatedtovideoid": [18, 28], "relevancelanguag": [18, 28], "topicid": [18, 28], "videocapt": [18, 28], "videocategoryid": [18, 28], "videodefinit": [18, 28], "videodimens": [18, 28], "videodur": [18, 28], "videoembedd": [18, 28], "videolicens": [18, 28], "videosynd": [18, 28], "videotyp": [18, 28], "loop": [18, 23, 29], "merg": [18, 20, 23, 29], "associ": [18, 26, 28], "boat": [18, 28], "sail": [18, 28], "similarli": [18, 20, 28], "fish": [18, 28], "pipe": [18, 26, 28], "escap": [18, 28], "sent": [18, 23, 28], "7c": [18, 28], "channel": [18, 28, 29, 32], "constrain": [18, 28], "broadcast": [18, 28], "live": [18, 28, 32], "upcom": [18, 28, 32], "intend": [18, 23, 28], "exclus": [18, 28], "partner": [18, 28], "via": [18, 23, 26, 28], "conjunct": [18, 23, 28], "subsequ": [18, 28], "circular": [18, 28], "metadata": [18, 23, 28, 29], "latitud": [18, 23, 28], "longitud": [18, 23, 28], "coordin": [18, 28], "42307": [18, 28], "122": [18, 28], "08427": [18, 28], "distanc": [18, 28], "float": [18, 28], "km": [18, 23, 28], "ft": [18, 28], "mi": [18, 23, 28], "1500m": [18, 28], "5km": [18, 28], "10000ft": [18, 28], "75mi": [18, 28], "larger": [18, 23, 28], "kilomet": [18, 23, 28], "definit": [18, 28], "cm": [18, 28], "act": [18, 28], "behalf": [18, 28], "individu": [18, 28], "chronolog": [18, 28], "rate": [18, 19, 20, 23, 26, 28], "highest": [18, 28], "lowest": [18, 28], "videocount": [18, 28], "descend": [18, 28], "viewcount": [18, 28], "viewer": [18, 28], "ongo": [18, 28], "nextpagetoken": [18, 28, 29], "prevpagetoken": [18, 28], "rfc": [18, 28], "3339": [18, 28], "1970": [18, 28], "01t00": [18, 28], "00z": [18, 19, 28], "3166": [18, 28], "alpha": [18, 28], "zh": [18, 28], "han": [18, 28], "hant": [18, 28], "standard": [18, 20, 23, 25, 28, 32], "moder": [18, 28], "demot": [18, 28], "strict": [18, 28], "freebas": [18, 28], "comma": [18, 23, 26, 27, 28], "playlist": [18, 28, 29], "closedcapt": [18, 28], "hd": [18, 28], "sd": [18, 28], "playback": [18, 28], "720p": [18, 28], "resolut": [18, 28], "1080p": [18, 28], "regardless": [18, 28, 29], "2d": [18, 28], "durat": [18, 28], "minut": [18, 19, 25, 28], "less": [18, 20, 23, 26, 28], "embed": [18, 23, 28], "embedd": [18, 28], "choos": [18, 20, 28], "attach": [18, 23, 28], "creativ": [18, 28], "creativecommon": [18, 28], "reus": [18, 28], "syndic": [18, 28], "episod": [18, 28], "movi": [18, 19, 23, 28], "set_logging_level": 18, "level_or_nam": 18, "dure": [18, 26], "notset": 18, "warn": 18, "youtube_channel_detail": 18, "channel_id": 18, "assum": [18, 20, 23, 29], "channel_df": 18, "youtube_video_detail": 18, "vid_id": 18, "video_df": 18, "fastest": 19, "easiest": 19, "reveal": 19, "correspond": 19, "publish": [19, 28], "rich": [19, 23], "sitemap_to_df": [19, 20, 29], "loc": 19, "hte": 19, "lastmod": 19, "sitemap_last_modifi": [19, 29], "sitemap_size_mb": [19, 29], "mega": 19, "1mb": 19, "024": [19, 29], "sitmeapindex": 19, "decid": [19, 20, 23], "bbc_sitemap": 19, "archiv": 19, "2009": [19, 20], "090620_as_iraq_explosion_tc2": 19, "e7e15811c65f406f89f89fe10aef29f5": 19, "05": [19, 20, 30], "63124": 19, "461037": 19, "090620_iraq_blast_tc2": 19, "43": 19, "busi": [19, 28], "090622_me_worldbank_tc2": 19, "090624_me_inpictures_brazil_tc2": 19, "090618_tomtest": 19, "090625_sf_tamim_verdict_tc2": 19, "090623_iz_cairo_russia_tc2": 19, "090622_me_egypt_us_tc2": 19, "090624_mz_wimbledon_tc2": 19, "worldnew": 19, "090623_mz_leaders_lifespan_tc2": 19, "49999": 19, "datetime64": 19, "utc": 19, "float64": 19, "set_index": 19, "resampl": 19, "2008": 19, "2287": 19, "47603": 19, "2010": 19, "2011": 19, "2012": 19, "2013": 19, "2015": [19, 21, 23], "2016": [19, 21], "2018": [19, 30], "2019": [19, 30], "freq": 19, "dec": 19, "seem": [19, 20, 26], "compar": [19, 25, 29], "rolling_new": 19, "090628_rn_pakistani_soldiries_ambush": 19, "pakistan": 19, "090421_mqm_speaks_rza": 19, "090723_ae_silwan_tc2": 19, "noticia": 19, "090729_iraquerefenbritsfn": 19, "090623_egitomilitaresfn": 19, "090302_gazaconferenciaml": 19, "090715_hillary_iran_cq": 19, "cultur": 19, "090409_machienhuu_revisit": 19, "090524_paquistaoupdateg": 19, "090629_om_pakistan_report_tc2": 19, "yyyi": [19, 23, 28], "mm": [19, 23, 28], "yymmdd_article_titl": 19, "url_df": 19, "scheme": [19, 25], "netloc": [19, 25], "fragment": [19, 25], "dir_1": [19, 25], "dir_2": [19, 25], "dir_3": [19, 25], "dir_4": 19, "dir_5": 19, "dir_6": 19, "dir_7": 19, "last_dir": [19, 25, 29], "49994": 19, "090831_dalailamataiwan": 19, "49995": 19, "090901_putin_regret_pact": 19, "49996": 19, "090901_tiananmen_movi": 19, "49997": 19, "pictur": [19, 26], "090830_ugc_ddh_sand": 19, "49998": 19, "090901_japecontask": 19, "14022": 19, "10968": 19, "5403": 19, "5068": 19, "mundo": 19, "5065": 19, "3561": 19, "2984": 19, "1677": 19, "turkc": 19, "706": 19, "ukchina": 19, "545": 19, "1506": 19, "2910": 19, "3021": 19, "3250": 19, "2769": 19, "9044": 19, "5050": 19, "4224": 19, "iran": 19, "3682": 19, "2103": 19, "afghanistan": 19, "1959": 19, "1657": 19, "internacion": 19, "1555": 19, "1350": 19, "1293": 19, "india": 19, "1285": 19, "america_latina": 19, "1274": 19, "1204": 19, "cultura_sociedad": 19, "913": 19, "874": 19, "872": 19, "russia": 19, "841": 19, "radio": 19, "769": 19, "scienc": [19, 32], "755": 19, "674": 19, "underscor": 19, "concaten": 19, "meaning": [19, 26], "explod": 19, "rn": 19, "8808": 19, "tc2": 19, "3153": 19, "1534": 19, "973": 19, "obama": 19, "882": 19, "862": 19, "china": 19, "815": 19, "ir88": 19, "727": 19, "683": 19, "si": 19, "640": 19, "np": [19, 29], "638": 19, "afghan": 19, "632": 19, "ka": 19, "565": 19, "556": 19, "iraq": 19, "554": 19, "547": 19, "nh": 19, "cq": 19, "510": 19, "ra": 19, "491": 19, "familiar": 19, "bug": [19, 29], "nyt_new": 19, "gz": [19, 29], "5085": 19, "news_publ": 19, "publication_nam": 19, "publication_languag": 19, "news_publication_d": 19, "news_titl": 19, "news_keyword": 19, "interact": [19, 23, 29, 32], "ottawa": 19, "ohio": 19, "covid": 19, "york": 19, "27t17": 19, "counti": 19, "exposur": 19, "risk": 19, "tracker": 19, "coronaviru": 19, "ncov": 19, "death": 19, "fatal": 19, "diseas": 19, "promo": 19, "1585539358901": 19, "articlelarg": 19, "v274": 19, "0cff645fbb74c21791568b78a888967d": 19, "0774069": 19, "744247": 19, "hopewel": 19, "virginia": 19, "1585539536519": 19, "v271": 19, "butt": 19, "nebraska": 19, "1585539237156": 19, "v281": 19, "stearn": 19, "minnesota": 19, "1585539172701": 19, "v282": 19, "benton": 19, "iowa": 19, "1585539039190": 19, "v286": 19, "5080": 19, "hodgeman": 19, "kansa": 19, "1585539054298": 19, "v285": 19, "f53301c8286f9bf59ef297f0232dcfc1": 19, "914107": 19, "995323": 19, "5081": 19, "miller": 19, "georgia": 19, "1585538956622": 19, "v290": 19, "5082": 19, "elect": 19, "west": 19, "hous": 19, "district": 19, "03t17": 19, "congression": 19, "david": 19, "mckinlei": 19, "natali": 19, "cline": 19, "presidenti": 19, "eln": 19, "race": [19, 28], "1winner": 19, "mckinleyd": 19, "5083": 19, "senat": 19, "susan": 19, "collin": 19, "defeat": 19, "sara": 19, "gideon": 19, "senatewinn": 19, "collinss": 19, "5084": 19, "randolph": 19, "missouri": 19, "1585539206866": 19, "wired_video": 19, "wire": 19, "2955": 19, "video_thumbnail_loc": 19, "video_titl": 19, "video_descript": 19, "video_content_loc": 19, "video_dur": 19, "video_publication_d": 19, "video_expiration_d": 19, "autocomplet": [19, 32], "inverview": 19, "owen": 19, "wilson": 19, "answer": 19, "dwgyu36up6iuz": 19, "cloudfront": 19, "heru80fdn": 19, "c_fill": 19, "d_placeholder_thescen": 19, "fl_progress": 19, "g_face": 19, "h_180": 19, "q_80": 19, "w_320": 19, "v1644595412": 19, "wired_autocomplet": 19, "interview": 19, "internet": 19, "himself": [19, 26], "nose": 19, "ben": 19, "stiller": 19, "anderson": 19, "skateboard": 19, "dp8hsntg6do36": 19, "62067f085577c277dd9acf42": 19, "39687acb": 19, "505b": 19, "4c69": 19, "94f1": 19, "afaa7cb5e636low": 19, "mp4": 19, "645": 19, "11t17": 19, "90b11f47f8b2ab57cb180cbd3c6f06f9": 19, "86199": 19, "841851": 19, "v1644418652": 19, "wired_wir": 19, "julian": 19, "chokkattu": 19, "editor": 19, "walk": 19, "6203cd7b5577c23d19622259": 19, "fe546b9b": 19, "a320": 19, "4883": 19, "9cbd": 19, "0d790f23c36dlow": 19, "184": 19, "10t17": 19, "v1644381627": 19, "wired_first": 19, "debut": 19, "newest": [19, 28], "620345a15577c23d46622256": 19, "d74930cf": 19, "11e1": 19, "466e": 19, "b023": 19, "1d9b91664204low": 19, "373": 19, "09t15": 19, "reinvent": 19, "v1642801328": 19, "wired_reinv": 19, "lab": 19, "aw": 19, "seattl": 19, "seahawk": 19, "win": [19, 26], "teach": 19, "swami": 19, "sivasubramanian": 19, "vp": 19, "ai": 19, "team": 19, "nfl": 19, "captur": [19, 32], "strateg": [19, 32], "619bd9be1d75db41adee6b58": 19, "d4889b15": 19, "4f34": 19, "41b0": 19, "b935": 19, "0c79465a9793low": 19, "09t13": 19, "seth": 19, "rogen": 19, "v1644335726": 19, "wired_seth": 19, "pam": 19, "amp": 19, "tommi": 19, "potteri": 19, "celebr": 19, "christma": 19, "duti": 19, "premier": 19, "februari": [19, 28], "hulu": 19, "march": 19, "6201430a1d75db06ae1f62e8": 19, "488ed635": 19, "91d0": 19, "4281": 19, "9e64": 19, "34be9bf74f00low": 19, "635": 19, "08t17": 19, "2950": 19, "genr": 19, "2951": 19, "2952": 19, "2953": 19, "promot": 19, "2954": 19, "thread": [19, 28, 29], "faster": [19, 29], "attack": 19, "sitemap_df": 19, "changefreq": 19, "prioriti": 19, "customiz": 20, "configur": [20, 28], "flexibl": [20, 32], "simplest": [20, 25], "reachabl": 20, "my_output_fil": 20, "affect": 20, "indepent": 20, "overwrit": [20, 29], "otherwis": [20, 23, 25, 26], "crash": 20, "sitename_crawl_yyyy_mm_dd": 20, "remark": 20, "rquest": 20, "card": [20, 23, 29], "jsonld_1_": 20, "item_a": 20, "item_b": 20, "whichev": [20, 25, 29, 32], "links_url": [20, 29], "links_text": [20, 29], "links_nofollow": [20, 29], "nofllow": 20, "tell": 20, "nav_links_url": 20, "header_links_url": 20, "footer_links_url": 20, "body_text": [20, 29], "p": [20, 29], "span": [20, 29], "li": [20, 29], "amount": [20, 25, 26], "took": 20, "download_timout": 20, "sec": 20, "crossorigin": [20, 29], "ismap": [20, 29], "longdesc": [20, 29], "referrerpolici": [20, 29], "srcset": [20, 29], "usemap": [20, 29], "global": [20, 23, 29], "draggabl": [20, 29], "third": [20, 23, 26], "truncat": 20, "site_crawl": 20, "links_href": 20, "leas": 20, "blob": 20, "resp_headers_access": 20, "request_headers_cooki": 20, "camp": 20, "readm": 20, "kw_": [20, 29, 32], "wed": 20, "720a8581": 20, "501e": 20, "0043": 20, "01a2": 20, "2e77d2": 20, "unlock": 20, "blockblob": 20, "web00007c": 20, "includesubdo": 20, "3600": 20, "ht": 20, "596daca7dbaa7e9": 20, "bud": 20, "02d86a3cea00007e9edb0cf2000000": 20, "xm": 20, "__cfduid": 20, "d76b68d148ddec1efd004": 20, "202": 20, "abil": [20, 28, 29], "4f7bea3b": 20, "701e": 20, "0039": 20, "3f44": 20, "2f1d9f": 20, "web00007h": 20, "596daca9bcab7e9": 20, "02d86a3e0e00007e9edb0d72000000": 20, "98b729fa": 20, "e01": 20, "00bf": 20, "24c3": 20, "2e494d": 20, "596daca9bf26d423": 20, "02d86a3e150000d423322742000000": 20, "submodul": [20, 29, 30, 31], "advertoo": 20, "7a28ef3b": 20, "801e": 20, "00c2": 20, "2ed585": 20, "web000079": 20, "596daca9bddb7ec2": 20, "02d86a3e1300007ec2a808a2000000": 20, "copyright": [20, 21], "eli": 20, "_static": 20, "75911c9e": 20, "201e": 20, "00e6": 20, "34c3": 20, "2e4ccb": 20, "web00007g": 20, "596daca9b91fd437": 20, "02d86a3e140000d437b81532000000": 20, "url_build": 20, "pyt": 20, "d99f2368": 20, "c01e": 20, "006f": 20, "18c3": 20, "2ef5ef": 20, "web00007a": 20, "596dacbbb8afd437": 20, "02d86a494f0000d437b828b2000000": 20, "pyth": 20, "85855c48": 20, "00ce": 20, "13c3": 20, "2e3b74": 20, "596dacbd980bd423": 20, "02d86a4a7f0000d423323b42000000": 20, "ad_": [20, 32], "b0aef497": 20, "004a": 20, "1647": 20, "2f6d5c": 20, "web00007k": 20, "596dacbd980cd423": 20, "02d86a4a7f0000d423209db2000000": 20, "9dfdd38a": 20, "101e": 20, "00a1": 20, "7ec3": 20, "2e93a0": 20, "596dacbd99847ec2": 20, "02d86a4a7f00007ec2a811f2000000": 20, "emo": 20, "2ad504a1": 20, "000b": 20, "03c3": 20, "2e454f": 20, "596dacbd9fb97e9": 20, "02d86a4a7f00007e9edb13a2000000": 20, "ran": 20, "got": 20, "richer": 20, "perspect": [20, 25, 26], "bounc": [20, 26], "traffic": [20, 23, 24], "export": 20, "addition": [20, 23], "pretti": 20, "name_1": 20, "selector_1": 20, "name_2": 20, "selector_2": 20, "reli": 20, "slectorgadget": 20, "selecotr": 20, "tricki": 20, "documentaion": 20, "w3c": 20, "decrib": 20, "sidebar": [20, 29], "toctre": 20, "l1": 20, "attr": 20, "sidebar_link": 20, "sidebar_links_url": 20, "shoe": [20, 26], "model_a": 20, "model_b": 20, "unexpect": 20, "rememb": 20, "granular": 20, "potenti": [20, 23], "region": [20, 23, 28], "regard": [20, 26], "further": [20, 25, 26, 32], "simultan": 20, "lower": 20, "pressur": 20, "deep": 20, "fraction": 20, "strongli": [20, 23], "yourself": [20, 26], "confid": 20, "fine": 20, "explan": 20, "outpuf_fil": 20, "attempt": 20, "product2": 20, "anotherexampl": 20, "anotherexmapl": 20, "author_url": 20, "contributornameid": 20, "spaci": [21, 29], "mine": [21, 26, 27, 28, 32], "footnot": 21, "explosionai": 21, "ug": 21, "haftungsbeschr\u00e4nkt": 21, "gmbh": 21, "matthew": 21, "honnib": 21, "complic": 23, "unnest": 23, "nest": [23, 28], "influenti": 23, "credibl": 23, "signatur": 23, "dashboard": [23, 32], "auth_param": 23, "app_kei": [23, 29], "your_app_kei": 23, "app_secret": [23, 29], "your_app_secret": 23, "set_auth_param": 23, "oauth_token": [23, 29], "oauth_token_secret": [23, 29], "your_oauth_token": 23, "your_oauth_token_secret": 23, "python_tweet": 23, "tweet_mod": 23, "140": 23, "prepend": [23, 25], "tweet_": 23, "user_": 23, "func": 23, "get_application_rate_limit_statu": 23, "consumed_onli": 23, "rate_limit_statu": 23, "get_available_trend": 23, "trend": [23, 26, 32], "get_favorit": 23, "user_id": 23, "screen_nam": 23, "since_id": 23, "max_id": 23, "include_ent": 23, "whom": [23, 26], "greater": 23, "forc": 23, "oldest": 23, "older": 23, "node": 23, "omit": 23, "engag": 23, "get_followers_id": 23, "cursor": 23, "stringify_id": 23, "semi": 23, "caus": [23, 29], "broken": 23, "5000": 23, "guarante": 23, "suspend": 23, "previous_cursor": 23, "next_cursor": 23, "environ": 23, "get_followers_list": 23, "skip_statu": 23, "include_user_ent": 23, "status": 23, "get_friends_id": 23, "friend": 23, "get_friends_list": 23, "get_home_timelin": 23, "trim_us": 23, "exclude_repli": 23, "retweet": [23, 26], "timelin": 23, "numer": [23, 28], "prevent": [23, 29], "repli": [23, 28], "home_timelin": 23, "get_list_memb": 23, "list_id": 23, "owner_screen_nam": 23, "owner_id": 23, "member": 23, "get_list_membership": 23, "filter_to_owned_list": 23, "disambigu": 23, "begin": [23, 27, 29], "membership": 23, "get_list_status": 23, "include_rt": 23, "ON": 23, "varieti": 23, "discreet": 23, "user_ment": 23, "nativ": 23, "stream": [23, 32], "ident": 23, "represent": 23, "get_list_subscrib": 23, "subscrib": [23, 28], "opt": 23, "futur": [23, 28], "truestatus": 23, "get_list_subscript": 23, "obtain": [23, 25, 26], "subscript": [23, 28], "get_mentions_timelin": 23, "mentions_timelin": 23, "get_place_trend": [23, 29], "woeid": 23, "earth": 23, "get_retweeters_id": 23, "get_retweet": 23, "get_supported_languag": 23, "get_user_timelin": 23, "strip": [23, 26, 27], "toward": [23, 26], "maxim": 23, "slice": 23, "user_timelin": 23, "lookup_statu": 23, "include_ext_alt_text": 23, "include_card_uri": 23, "hydrat": 23, "cannot": [23, 26], "null": 23, "pair": 23, "ext_alt_text": 23, "card_uri": 23, "lookup_us": 23, "encourag": 23, "make_datafram": 23, "retweeted_of_m": 23, "retweets_of_m": 23, "geocod": 23, "result_typ": 23, "lat": 23, "lon": 23, "dist": 23, "radiu": 23, "preferenti": 23, "geotag": 23, "mile": 23, "directli": 23, "distinct": 23, "detect": 23, "effort": 23, "ja": 23, "prefer": 23, "mix": [23, 25, 28], "dd": 23, "hate": 23, "beer": 23, "root": 23, "haiku": 23, "interior": 23, "nasa": 23, "astronaut": 23, "puppi": 23, "native_video": 23, "amplifi": 23, "periscop": 23, "vine": 23, "instagram": 23, "twimg": 23, "pic": 23, "hilari": 23, "anywher": [23, 26], "superhero": 23, "scari": 23, "attitud": 23, "search_us": 23, "access_token": 23, "token_typ": 23, "bearer": 23, "oauth_vers": 23, "api_vers": 23, "client_arg": 23, "auth_endpoint": 23, "twython": [23, 29], "starting_out": 23, "show_list": 23, "show_owned_list": 23, "ownership": 23, "url_utm_ga": [24, 29], "utm_sourc": 24, "utm_medium": 24, "utm_campaign": 24, "utm_cont": 24, "utm_term": 24, "utm": [24, 29], "banner": 24, "summer_promo": 24, "20pct_off": 24, "differenti": 24, "728x90": 24, "mpu": 24, "square_bann": 24, "bid": 24, "mysit": 24, "THE": 24, "2anam": 24, "5e": 24, "fairli": [25, 32], "situat": 25, "enhanc": 25, "path_1": 25, "path_2": 25, "frag_1": 25, "frag_2": 25, "path_3": 25, "query_color": 25, "query_pric": 25, "query_s": 25, "elabor": 25, "decod": 25, "self": 25, "explanatori": 25, "blog": 25, "previou": [25, 28, 29], "unalign": 25, "popul": 25, "na": [25, 29], "hash": 25, "query_": 25, "colliss": 25, "unlik": 25, "delimit": [25, 27], "unusu": 25, "product1": 25, "sens": [25, 26], "renam": 25, "inconsist": 25, "topic1": 25, "topic2": 25, "artilc": 25, "yout": 25, "distort": 25, "role": [25, 28], "topic_1": 25, "topic_2": 25, "align": 25, "ouput_fil": 25, "urldf": 25, "corpu": 26, "accomplish": [26, 32], "word_frequ": [26, 27, 29, 32], "sequenc": [26, 32], "dot": [26, 27], "quotat": 26, "whatev": [26, 32], "sale": 26, "quantifi": 26, "bag": 26, "half": 26, "revenu": 26, "million": 26, "hidden": 26, "ppc": 26, "num_list": [26, 29], "experi": 26, "rm_word": 26, "ignor": 26, "possibli": 26, "extra_info": 26, "abs_freq": [26, 29], "wtd_freq": [26, 29], "rel_valu": [26, 29], "essenti": [26, 27, 32], "multipli": 26, "abs_perc": 26, "abs_perc_cum": 26, "wtd_freq_perc": 26, "wtd_freq_perc_cum": 26, "afterward": 26, "alon": 26, "among": 26, "amongst": 26, "anyhow": 26, "anyon": 26, "becam": 26, "beforehand": 26, "behind": 26, "besid": 26, "beyond": 26, "eight": 26, "eleven": 26, "elsewher": 26, "everywher": 26, "fifti": 26, "former": 26, "formerli": 26, "forti": [26, 32], "henc": 26, "her": 26, "hereaft": 26, "herebi": 26, "herein": 26, "hereupon": 26, "herself": 26, "him": 26, "inde": 26, "latter": 26, "latterli": 26, "me": [26, 27], "meanwhil": 26, "moreov": 26, "move": 26, "myself": 26, "neither": 26, "nevertheless": 26, "nine": 26, "nobodi": 26, "noon": 26, "nor": 26, "nowher": 26, "often": 26, "onto": 26, "perhap": 26, "rather": 26, "seriou": 26, "sixti": 26, "somehow": 26, "someon": 26, "somewher": [26, 32], "themselv": [26, 29], "thenc": 26, "thereaft": 26, "therebi": [26, 28], "therein": 26, "thereupon": 26, "throughout": 26, "thru": 26, "twelv": 26, "upon": 26, "whenc": 26, "whenev": 26, "whereaft": 26, "wherea": 26, "wherebi": 26, "wherein": 26, "whereupon": 26, "wherev": [26, 29], "whither": 26, "whoever": 26, "yourselv": 26, "blown": 26, "ngram": 26, "metric": 26, "abs_wtd_df": 26, "banana": 26, "kiwi": 26, "mango": 26, "250": 26, "300": 26, "beat": 26, "text_list2": 26, "222222": 26, "333333": 26, "111111": 26, "266667": 26, "600000": 26, "666667": 26, "200000": 26, "800000": 26, "888889": 26, "133333": 26, "933333": 26, "000000": 26, "066667": 26, "word_token": [27, 29], "trim": 27, "quot": [27, 29], "parenthes": 27, "trail": [27, 29], "insid": 27, "activities_list": 28, "criteria": 28, "child": 28, "quota": 28, "cost": 28, "contentdetail": 28, "uniqu": [28, 29], "deprec": [28, 29], "unsign": 28, "earliest": 28, "8601": 28, "ddthh": 28, "ss": 28, "sz": 28, "captions_list": 28, "videoid": 28, "channel_sections_list": 28, "channelsect": 28, "i18nlanguag": 28, "channels_list": 28, "categoryid": 28, "forusernam": 28, "managedbym": 28, "mysubscrib": 28, "auditdetail": 28, "brandingset": 28, "contentownerdetail": 28, "invideopromot": 28, "topicdetail": 28, "comment_threads_list": 28, "allthreadsrelatedtochannelid": 28, "moderationstatu": 28, "searchterm": 28, "textformat": 28, "commentthread": 28, "heldforreview": 28, "await": 28, "likelyspam": 28, "classifi": 28, "spam": 28, "plaintext": 28, "plain": 28, "comments_list": 28, "parentid": 28, "guide_categories_list": [28, 29], "guidecategori": 28, "i18n_languages_list": 28, "en_u": 28, "i18n_regions_list": 28, "i18nregion": 28, "playlist_items_list": 28, "playlistid": 28, "playlistitem": 28, "resourceid": 28, "playlists_list": 28, "onbehalfofcontentownerchannel": 28, "timecr": 28, "action": [28, 29], "curat": 28, "music": 28, "04rlf": 28, "02mscn": 28, "christian": 28, "0ggq0m": 28, "classic": 28, "01lyv": 28, "02lkt": 28, "electron": 28, "0glt670": 28, "hip": 28, "hop": 28, "05rwpb": 28, "03_d0": 28, "jazz": 28, "028sqc": 28, "asia": 28, "0g293": 28, "america": 28, "064t9": 28, "pop": 28, "06cqb": 28, "regga": 28, "06j6l": 28, "rhythm": 28, "06by7": 28, "rock": 28, "0gywn": 28, "soul": 28, "game": 28, "0bzvm2": 28, "025zzc": 28, "02ntfj": 28, "adventur": 28, "0b1vjn": 28, "casual": 28, "02hygl": 28, "04q1x3q": 28, "puzzl": 28, "01sjng": 28, "0403l3g": 28, "021bp2": 28, "simul": 28, "022dc6": 28, "03hf_rm": 28, "06ntj": 28, "0jm_": 28, "american": 28, "018jz": 28, "basebal": 28, "018w8": 28, "01cgz": 28, "09xp_": 28, "cricket": 28, "02vx4": 28, "037hz": 28, "golf": 28, "03tmr": 28, "hockei": 28, "01h7lh": 28, "martial": 28, "0410tth": 28, "motorsport": 28, "07bs0": 28, "tenni": 28, "07_53": 28, "volleybal": 28, "entertain": 28, "02jjt": 28, "09kqc": 28, "humor": 28, "02vxn": 28, "05qjc": 28, "066wd": 28, "profession": 28, "wrestl": 28, "0f2f9": 28, "lifestyl": 28, "019_rr": 28, "032tl": 28, "fashion": 28, "027x7n": 28, "02wbm": 28, "03glg": 28, "hobbi": 28, "068hy": 28, "pet": 28, "041xxh": 28, "physic": 28, "attract": 28, "beauti": 28, "07c1v": 28, "07bxq": 28, "tourism": 28, "07yv9": 28, "vehicl": 28, "societi": 28, "098wr": 28, "09s1f": 28, "0kt51": 28, "01h6rj": 28, "militari": 28, "05qt0": 28, "06bvp": 28, "religion": 28, "01k8wb": 28, "channelplaylistvideo": 28, "subscriptions_list": 28, "myrecentsubscrib": 28, "forchannelid": 28, "subscribersnippet": 28, "subscription_order_relev": 28, "unread": 28, "video_categories_list": 28, "videocategori": 28, "videos_list": 28, "chart": 28, "myrat": 28, "maxheight": 28, "maxwidth": 28, "filedetail": 28, "livestreamingdetail": 28, "processingdetail": 28, "recordingdetail": 28, "mostpopular": 28, "dislik": 28, "embedhtml": 28, "emb": 28, "appropri": 28, "violat": 28, "8192": 28, "narrow": 28, "subpackag": [29, 30, 31], "log_date_format": 29, "relatedsit": 29, "v15": 29, "contribut": 29, "danielp77": 29, "offlin": 29, "preserv": 29, "autothrottl": 29, "minim": 29, "fillna": 29, "ffill": 29, "andypayn": 29, "newlin": 29, "clarifi": 29, "thebe": 29, "sphinx": 29, "deprac": 29, "skip_url_param": 29, "versatil": 29, "bad": 29, "timeout": 29, "sitemapindex": 29, "mb": 29, "024x1": 29, "anymor": 29, "jsonld_error": 29, "resp_meta_": 29, "preced": 29, "url_redirected_to": 29, "links_frag": 29, "invalid": 29, "stricter": 29, "unifi": 29, "element_1": 29, "element_2": 29, "drop": 29, "slight": 29, "relayout": 29, "clarif": 29, "robotstxt": 29, "cse": 29, "returnd": 29, "sitemap_download": 29, "variabl": 29, "expand": 29, "pagemap": 29, "df": 29, "top_emoji_categori": 29, "top_emoji_sub_categori": 29, "db": 29, "simpler": 29, "__init__": 29, "ve": 29, "punctuat": 29, "pagin": [29, 32], "reflect": 29, "lenght": 29, "rewrit": 29, "_dict_product": 29, "msg": 29, "implement": 29, "repons": 29, "town": 29, "wrap": 29, "pand": 29, "cheat": 29, "sheet": [29, 32], "coverag": 29, "releas": 29, "pypi": 29, "placehold": [29, 32], "feedback": 30, "pip3": 30, "unreleas": 30, "2023": 30, "announc": 32, "scientist": 32, "manipul": 32, "visual": 32, "sophist": 32, "algorithm": 32, "cool": 32, "spent": 32, "wrangl": 32, "stitch": 32, "124": 32, "hopefulli": 32, "pick": 32, "excel": 32, "formula": 32, "unix": 32, "doug": 32, "mcilroi": 32, "univers": 32, "aim": 32, "unrel": 32, "workflow": 32, "practition": 32, "plotli": 32, "librari": 32, "tabular": 32, "kept": 32, "modular": 32, "coder": 32, "promis": 32, "deliveri": 32, "didn": 32, "headlin": 32, "datacamp": 32, "semrush": 32, "comprehens": 32, "render": 32, "creation": 32, "outreach": 32, "built": 32, "megabyt": 32, "monitor": 32, "parser": 32, "notebook": 32, "tackl": 32, "bloomberg": 32, "click": 32, "divers": 32, "3k": 32, "conveni": 32, "introductori": 32, "clean": 32, "131k": 32, "european": 32, "url_": 32, "emoji_": 32, "_to_df": 32}, "objects": {"": [[0, 0, 0, "-", "advertools"]], "advertools": [[1, 0, 0, "-", "ad_create"], [2, 0, 0, "-", "ad_from_string"], [3, 0, 0, "-", "cli"], [5, 0, 0, "-", "code_recipes"], [7, 0, 0, "-", "crawlytics"], [8, 0, 0, "-", "emoji"], [9, 0, 0, "-", "extract"], [10, 0, 0, "-", "header_spider"], [11, 0, 0, "-", "image_spider"], [12, 0, 0, "-", "knowledge_graph"], [13, 0, 0, "-", "kw_generate"], [14, 0, 0, "-", "logs"], [15, 0, 0, "-", "regex"], [16, 0, 0, "-", "reverse_dns_lookup"], [17, 0, 0, "-", "robotstxt"], [18, 0, 0, "-", "serp"], [19, 0, 0, "-", "sitemaps"], [20, 0, 0, "-", "spider"], [21, 0, 0, "-", "stopwords"], [23, 0, 0, "-", "twitter"], [24, 0, 0, "-", "url_builders"], [25, 0, 0, "-", "urlytics"], [26, 0, 0, "-", "word_frequency"], [27, 0, 0, "-", "word_tokenize"], [28, 0, 0, "-", "youtube"]], "advertools.ad_create": [[1, 1, 1, "", "ad_create"]], "advertools.ad_from_string": [[2, 1, 1, "", "ad_from_string"]], "advertools.cli": [[4, 0, 0, "-", "cli"]], "advertools.code_recipes": [[6, 0, 0, "-", "spider_strategies"]], "advertools.crawlytics": [[7, 1, 1, "", "images"], [7, 1, 1, "", "jl_subset"], [7, 1, 1, "", "jl_to_parquet"], [7, 1, 1, "", "links"], [7, 1, 1, "", "parquet_columns"], [7, 1, 1, "", "redirects"]], "advertools.emoji": [[8, 1, 1, "", "emoji_search"], [8, 1, 1, "", "extract_emoji"]], "advertools.extract": [[9, 1, 1, "", "extract"], [9, 1, 1, "", "extract_currency"], [9, 1, 1, "", "extract_exclamations"], [9, 1, 1, "", "extract_hashtags"], [9, 1, 1, "", "extract_intense_words"], [9, 1, 1, "", "extract_mentions"], [9, 1, 1, "", "extract_numbers"], [9, 1, 1, "", "extract_questions"], [9, 1, 1, "", "extract_urls"], [9, 1, 1, "", "extract_words"]], "advertools.header_spider": [[10, 2, 1, "", "HeadersSpider"], [10, 1, 1, "", "crawl_headers"]], "advertools.header_spider.HeadersSpider": [[10, 3, 1, "", "custom_settings"], [10, 4, 1, "", "errback"], [10, 3, 1, "", "name"], [10, 4, 1, "", "parse"], [10, 4, 1, "", "start_requests"]], "advertools.image_spider": [[11, 2, 1, "", "AdvImagesPipeline"], [11, 2, 1, "", "ImageSpider"], [11, 2, 1, "", "ImgItem"], [11, 1, 1, "", "crawl_images"], [11, 1, 1, "", "summarize_crawled_imgs"]], "advertools.image_spider.AdvImagesPipeline": [[11, 4, 1, "", "file_path"]], "advertools.image_spider.ImageSpider": [[11, 3, 1, "", "custom_settings"], [11, 3, 1, "", "include_img_regex"], [11, 3, 1, "", "name"], [11, 4, 1, "", "parse"], [11, 4, 1, "", "start_requests"]], "advertools.image_spider.ImgItem": [[11, 3, 1, "", "fields"]], "advertools.knowledge_graph": [[12, 1, 1, "", "knowledge_graph"]], "advertools.kw_generate": [[13, 1, 1, "", "kw_broad"], [13, 1, 1, "", "kw_exact"], [13, 1, 1, "", "kw_generate"], [13, 1, 1, "", "kw_modified"], [13, 1, 1, "", "kw_neg_broad"], [13, 1, 1, "", "kw_neg_exact"], [13, 1, 1, "", "kw_neg_phrase"], [13, 1, 1, "", "kw_phrase"]], "advertools.logs": [[14, 1, 1, "", "crawllogs_to_df"], [14, 1, 1, "", "logs_to_df"]], "advertools.reverse_dns_lookup": [[16, 1, 1, "", "reverse_dns_lookup"]], "advertools.robotstxt": [[17, 1, 1, "", "robotstxt_test"], [17, 1, 1, "", "robotstxt_to_df"]], "advertools.serp": [[18, 1, 1, "", "serp_goog"], [18, 1, 1, "", "serp_youtube"], [18, 1, 1, "", "set_logging_level"], [18, 1, 1, "", "youtube_channel_details"], [18, 1, 1, "", "youtube_video_details"]], "advertools.sitemaps": [[19, 1, 1, "", "sitemap_to_df"]], "advertools.spider": [[20, 1, 1, "", "crawl"]], "advertools.twitter": [[23, 1, 1, "", "authenticate"], [23, 1, 1, "", "get_application_rate_limit_status"], [23, 1, 1, "", "get_available_trends"], [23, 1, 1, "", "get_favorites"], [23, 1, 1, "", "get_followers_ids"], [23, 1, 1, "", "get_followers_list"], [23, 1, 1, "", "get_friends_ids"], [23, 1, 1, "", "get_friends_list"], [23, 1, 1, "", "get_home_timeline"], [23, 1, 1, "", "get_list_members"], [23, 1, 1, "", "get_list_memberships"], [23, 1, 1, "", "get_list_statuses"], [23, 1, 1, "", "get_list_subscribers"], [23, 1, 1, "", "get_list_subscriptions"], [23, 1, 1, "", "get_mentions_timeline"], [23, 1, 1, "", "get_place_trends"], [23, 1, 1, "", "get_retweeters_ids"], [23, 1, 1, "", "get_retweets"], [23, 1, 1, "", "get_supported_languages"], [23, 1, 1, "", "get_user_timeline"], [23, 1, 1, "", "lookup_status"], [23, 1, 1, "", "lookup_user"], [23, 1, 1, "", "make_dataframe"], [23, 1, 1, "", "retweeted_of_me"], [23, 1, 1, "", "search"], [23, 1, 1, "", "search_users"], [23, 1, 1, "", "set_auth_params"], [23, 1, 1, "", "show_lists"], [23, 1, 1, "", "show_owned_lists"]], "advertools.url_builders": [[24, 1, 1, "", "url_utm_ga"]], "advertools.urlytics": [[25, 1, 1, "", "url_to_df"]], "advertools.word_frequency": [[26, 1, 1, "", "word_frequency"]], "advertools.word_tokenize": [[27, 1, 1, "", "word_tokenize"]], "advertools.youtube": [[28, 1, 1, "", "activities_list"], [28, 1, 1, "", "captions_list"], [28, 1, 1, "", "channel_sections_list"], [28, 1, 1, "", "channels_list"], [28, 1, 1, "", "comment_threads_list"], [28, 1, 1, "", "comments_list"], [28, 1, 1, "", "guide_categories_list"], [28, 1, 1, "", "i18n_languages_list"], [28, 1, 1, "", "i18n_regions_list"], [28, 1, 1, "", "playlist_items_list"], [28, 1, 1, "", "playlists_list"], [28, 1, 1, "", "search"], [28, 1, 1, "", "subscriptions_list"], [28, 1, 1, "", "video_categories_list"], [28, 1, 1, "", "videos_list"]]}, "objtypes": {"0": "py:module", "1": "py:function", "2": "py:class", "3": "py:attribute", "4": "py:method"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "function", "Python function"], "2": ["py", "class", "Python class"], "3": ["py", "attribute", "Python attribute"], "4": ["py", "method", "Python method"]}, "titleterms": {"advertool": [0, 3, 4, 5, 29, 30, 31, 32], "packag": [0, 5], "subpackag": 0, "submodul": [0, 5], "modul": [0, 3, 5, 7], "content": [0, 5, 30, 32], "creat": [1, 2], "ad": [1, 2], "larg": [1, 7, 12, 17, 25], "scale": [1, 12, 17, 32], "us": [2, 4, 6, 12], "long": 2, "descript": 2, "text": [2, 4, 8, 9, 26, 30, 32], "top": 2, "down": [2, 6], "approach": [2, 17, 20, 32], "googl": [2, 12, 18, 20], "facebook": 2, "feed": 2, "instant": 2, "articl": [2, 20], "cli": [3, 4], "command": 4, "line": 4, "interfac": 4, "convert": 4, "robot": [4, 6, 17], "txt": [4, 6, 17], "file": [4, 7, 11, 14, 17], "list": [4, 6, 9, 20], "url": [4, 20, 24, 25], "tabl": [4, 30], "csv": 4, "format": [4, 14], "download": [4, 11, 19], "pars": [4, 14, 19, 25], "save": [4, 6], "an": 4, "xml": [4, 19], "sitemap": [4, 19], "split": [4, 25], "compon": 4, "scheme": 4, "netloc": 4, "path": [4, 25], "queri": [4, 20, 25], "etc": 4, "crawl": [4, 6, 7, 14, 20], "known": 4, "head": 4, "method": 4, "compress": [4, 7], "log": [4, 6, 14, 29, 30], "datafram": [4, 14], "parquet": [4, 7], "perform": 4, "revers": [4, 16], "dn": [4, 16], "lookup": [4, 16], "ip": 4, "address": 4, "gener": [4, 13], "sem": [4, 13, 30, 32], "keyword": [4, 13], "suppli": 4, "product": [4, 30, 32], "intent": 4, "word": [4, 26, 27], "get": [4, 8], "stopword": [4, 21], "select": 4, "languag": [4, 21], "count": [4, 26], "option": 4, "weight": [4, 26], "number": [4, 6, 9, 25], "search": [4, 8, 18, 20], "emoji": [4, 8, 9], "regex": [4, 20], "extract": [4, 6, 8, 9, 15, 20], "structur": [4, 9, 15, 25], "entiti": [4, 9, 15], "from": [4, 6, 8, 9], "hashtag": [4, 9], "mention": [4, 9], "token": [4, 27], "document": 4, "phrase": 4, "tweet": 4, "desir": 4, "length": 4, "seo": [4, 6, 20, 30, 32], "crawler": [4, 11, 20], "code_recip": 5, "scrape": [6, 7], "strategi": 6, "recip": 6, "how": [6, 12, 14], "page": [6, 18, 20], "those": 6, "onli": 6, "mode": [6, 20], "can": 6, "i": 6, "websit": [6, 7], "includ": 6, "its": 6, "sub": 6, "domain": 6, "copi": 6, "my": 6, "audit": 6, "them": 6, "later": 6, "automat": 6, "stop": 6, "base": 6, "certain": 6, "condit": 6, "di": 6, "obei": 6, "rule": 6, "do": 6, "set": [6, 20], "user": [6, 17], "agent": [6, 17], "while": [6, 20], "control": 6, "concurr": 6, "request": 6, "slow": 6, "so": 6, "don": 6, "t": 6, "hit": 6, "server": 6, "too": 6, "hard": 6, "multipl": 6, "same": 6, "job": 6, "want": 6, "follow": [6, 20], "link": [6, 7, 20], "specifi": 6, "depth": 6, "paus": 6, "resum": 6, "make": 6, "sure": 6, "twice": 6, "proxi": 6, "chang": [6, 29, 30], "default": 6, "header": [6, 10], "xpath": [6, 20], "express": [6, 15], "custom": [6, 20], "string": 6, "analysi": [7, 14, 26, 30, 32], "analyz": [7, 8, 12, 14, 17, 19, 25], "imag": [7, 11], "redirect": 7, "handl": 7, "veri": 7, "explor": 7, "column": 7, "data": [7, 14, 20, 23, 28], "type": 7, "function": [7, 9, 14, 20, 23], "insight": 8, "currenc": 9, "1234567890\u0660\u0661\u0662\u0663\u0664\u0665\u0666\u0667\u0668\u0669\u32ba\ud804\udc5b\ud800\udd0d\ud802\udcaa\u24f2\ud804\udc63\ud800\udd28\ud802\udd1b": 9, "question": 9, "\u0294": 9, "exclam": 9, "python": [10, 20], "statu": 10, "code": 10, "checker": 10, "respons": 10, "name": 11, "import": [12, 18], "knowledg": 12, "graph": 12, "result": [12, 18], "account": 12, "setup": 12, "": 12, "api": [12, 23, 28], "campaign": [13, 32], "run": 14, "logs_to_df": 14, "support": 14, "prepar": 14, "regular": [15, 19], "bulk": [16, 17], "test": 17, "tester": 17, "engin": 18, "serp": [18, 20], "youtub": [18, 28], "index": [19, 30], "new": [19, 20], "video": 19, "spider": 20, "discoveri": 20, "On": 20, "element": 20, "pre": 20, "determin": 20, "analyt": 20, "consol": 20, "css": 20, "selector": 20, "behavior": 20, "paramet": [20, 25], "pattern": 20, "addit": 20, "sever": 21, "survei": 22, "share": 22, "feedback": 22, "twitter": 23, "authent": 23, "builder": 24, "The": 25, "directori": 25, "absolut": 26, "v": 26, "frequenc": 26, "n": 27, "gram": 27, "unreleas": 29, "0": 29, "14": 29, "2": 29, "2024": 29, "02": 29, "24": 29, "1": 29, "21": 29, "18": 29, "13": 29, "5": 29, "2023": 29, "08": 29, "22": 29, "4": 29, "07": 29, "26": 29, "3": 29, "06": 29, "27": 29, "2022": 29, "09": 29, "30": 29, "05": 29, "11": 29, "10": 29, "12": 29, "2021": 29, "04": 29, "03": 29, "31": 29, "7": 29, "2020": 29, "6": 29, "25": 29, "23": 29, "9": 29, "19": 29, "8": 29, "2019": 29, "17": 29, "29": 29, "01": 29, "2018": 29, "onlin": [30, 32], "market": [30, 32], "tool": [30, 32], "social": [30, 32], "media": [30, 32], "indic": 30, "your": 32, "instal": 32, "philosophi": 32, "convent": 32}, "envversion": {"sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 60}, "alltitles": {"advertools package": [[0, "advertools-package"]], "Subpackages": [[0, "subpackages"]], "Submodules": [[0, "submodules"], [5, "submodules"]], "Module contents": [[0, "module-advertools"], [5, "module-advertools.code_recipes"]], "Create Ads on a Large Scale": [[1, "create-ads-on-a-large-scale"]], "Create Ads Using Long Descriptive Text (top-down approach)": [[2, "create-ads-using-long-descriptive-text-top-down-approach"]], "Google Text Ads": [[2, "google-text-ads"]], "Facebook Feed Ads": [[2, "facebook-feed-ads"]], "Facebook Instant Article Ad": [[2, "facebook-instant-article-ad"]], "advertools.cli module": [[3, "module-advertools.cli"]], "advertools Command Line Interface (CLI)": [[4, "advertools-command-line-interface-cli"]], "convert a robots.txt file (or list of file URLs) to a table in a CSV format": [[4, "convert-a-robots-txt-file-or-list-of-file-urls-to-a-table-in-a-csv-format"]], "download, parse, and save an XML sitemap to a table in a CSV file": [[4, "download-parse-and-save-an-xml-sitemap-to-a-table-in-a-csv-file"]], "split a list of URLs into their components: scheme, netloc, path, query, etc.": [[4, "split-a-list-of-urls-into-their-components-scheme-netloc-path-query-etc"]], "crawl a list of known URLs using the HEAD method": [[4, "crawl-a-list-of-known-urls-using-the-head-method"]], "parse, compress and convert a log file to a DataFrame in the .parquet format": [[4, "parse-compress-and-convert-a-log-file-to-a-dataframe-in-the-parquet-format"]], "perform a reverse DNS lookup on a list of IP addresses": [[4, "perform-a-reverse-dns-lookup-on-a-list-of-ip-addresses"]], "generate a table of SEM keywords by supplying a list of products and a list of intent words": [[4, "generate-a-table-of-sem-keywords-by-supplying-a-list-of-products-and-a-list-of-intent-words"]], "get stopwords of the selected language": [[4, "get-stopwords-of-the-selected-language"]], "get word counts of a text list optionally weighted by a number list": [[4, "get-word-counts-of-a-text-list-optionally-weighted-by-a-number-list"]], "search for emoji using a regex": [[4, "search-for-emoji-using-a-regex"]], "extract structured entities from a text list; emoji, hashtags, mentions": [[4, "extract-structured-entities-from-a-text-list-emoji-hashtags-mentions"]], "tokenize documents (phrases, keywords, tweets, etc) into token of the desired length": [[4, "tokenize-documents-phrases-keywords-tweets-etc-into-token-of-the-desired-length"]], "SEO crawler": [[4, "seo-crawler"]], "advertools.code_recipes package": [[5, "advertools-code-recipes-package"]], "\ud83d\udd77 SEO Crawling & Scraping: Strategies & Recipes": [[6, "seo-crawling-scraping-strategies-recipes"]], "How to crawl a list of pages, and those pages only (list mode)?": [[6, "how-to-crawl-a-list-of-pages-and-those-pages-only-list-mode"]], "How can I crawl a website including its sub-domains?": [[6, "how-can-i-crawl-a-website-including-its-sub-domains"]], "How can I save a copy of the logs of my crawl for auditing them later?": [[6, "how-can-i-save-a-copy-of-the-logs-of-my-crawl-for-auditing-them-later"]], "How can I automatically stop my crawl based on a certain condition?": [[6, "how-can-i-automatically-stop-my-crawl-based-on-a-certain-condition"]], "How can I (dis)obey robots.txt rules?": [[6, "how-can-i-dis-obey-robots-txt-rules"]], "How do I set my User-agent while crawling?": [[6, "how-do-i-set-my-user-agent-while-crawling"]], "How can I control the number of concurrent requests while crawling?": [[6, "how-can-i-control-the-number-of-concurrent-requests-while-crawling"]], "How can I slow down the crawling so I don't hit the websites' servers too hard?": [[6, "how-can-i-slow-down-the-crawling-so-i-don-t-hit-the-websites-servers-too-hard"]], "How can I set multiple settings to the same crawl job?": [[6, "how-can-i-set-multiple-settings-to-the-same-crawl-job"]], "I want to crawl a list of pages, follow links from those pages, but only to a certain specified depth": [[6, "i-want-to-crawl-a-list-of-pages-follow-links-from-those-pages-but-only-to-a-certain-specified-depth"]], "How do I pause/resume crawling, while making sure I don't crawl the same page twice?": [[6, "how-do-i-pause-resume-crawling-while-making-sure-i-don-t-crawl-the-same-page-twice"]], "How do I use a proxy while crawling?": [[6, "how-do-i-use-a-proxy-while-crawling"]], "How can I change the default request headers?": [[6, "how-can-i-change-the-default-request-headers"]], "XPath expressions for custom extraction": [[6, "xpath-expressions-for-custom-extraction"]], "User-agent strings for use in crawling": [[6, "user-agent-strings-for-use-in-crawling"]], "Crawling and Scraping Analysis": [[7, "module-advertools.crawlytics"]], "Analyzing crawled images": [[7, "analyzing-crawled-images"]], "Analyzing links in a crawled website": [[7, "analyzing-links-in-a-crawled-website"]], "Analyzing the redirects of a crawled website": [[7, "analyzing-the-redirects-of-a-crawled-website"]], "Handling very large crawl files": [[7, "handling-very-large-crawl-files"]], "Compressing large crawl files": [[7, "compressing-large-crawl-files"]], "Exploring the columns and data types of parquet files": [[7, "exploring-the-columns-and-data-types-of-parquet-files"]], "Module functions": [[7, "module-functions"]], "Emoji: Extract, Analyze, and Get Insights": [[8, "emoji-extract-analyze-and-get-insights"]], "Emoji Search": [[8, "emoji-search"]], "Extract Emoji from Text": [[8, "extract-emoji-from-text"]], "Extract structured entities from text lists": [[9, "extract-structured-entities-from-text-lists"]], "Extract Functions": [[9, "extract-functions"]], "Extract #hashtags": [[9, "extract-hashtags"]], "Extract @mentions": [[9, "extract-mentions"]], "Extract Currency $ \u00a2 \u00a3 \u00a4 \u00a5 \u058f \u060b \u20b2 \u20b5 \u20b8 \u20b9\ufe69 \uffe0 \uffe1 \uffe5 \uffe6 \u20ba \u20bb \u20bc \u20bd \u20be \u20bf \ufdfc": [[9, "extract-currency"]], "Extract numbers 1234567890\u0660\u0661\u0662\u0663\u0664\u0665\u0666\u0667\u0668\u0669\u32ba\ud804\udc5b\ud800\udd0d\ud802\udcaa\u24f2\ud804\udc63\ud800\udd28\ud802\udd1b": [[9, "extract-numbers-123456789045"]], "Extract questions ? \u00bf \u037e \u055e \u061f \u1367 \u1945 \u2047 \u2048 \u2049 \u2cfa \u2cfb \u2e2e \ua60f \ua6f7 \ufe16 \ufe56 \uff1f \ud804\udd43 \ud83a\udd5f \u0294 \u203d": [[9, "extract-questions"]], "Extract Exclamations ! \u00a1 \u055c \u07f9 \u1944 \u203c \u2048 \u2049 \ufe15 \ufe57 \uff01 \ud83a\udd5e": [[9, "extract-exclamations"]], "Extract Emoji \ud83d\ude02\ud83d\ude2d\ud83e\udd7a\ud83e\udd23\u2764\ufe0f\u2728\ud83d\ude4f\ud83d\ude0d": [[9, "extract-emoji"]], "\ud83d\udd77 Python Status Code Checker with Response Headers": [[10, "python-status-code-checker-with-response-headers"]], "Image Crawler and Downloader": [[11, "image-crawler-and-downloader"]], "Image file names": [[11, "image-file-names"]], "Import and Analyze Knowledge Graph Results on a Large Scale": [[12, "import-and-analyze-knowledge-graph-results-on-a-large-scale"]], "Account Setup": [[12, "account-setup"]], "How to use Google's Knowledge Graph API": [[12, "how-to-use-google-s-knowledge-graph-api"]], "Generate Keywords for SEM Campaigns": [[13, "generate-keywords-for-sem-campaigns"]], "Log File Analysis": [[14, "log-file-analysis"]], "How to run the logs_to_df() function:": [[14, "how-to-run-the-logs-to-df-function"]], "Supported Log Formats": [[14, "supported-log-formats"]], "Log File Analysis - Data Preparation": [[14, "log-file-analysis-data-preparation"]], "Parse and Analyze Crawl Logs in a Dataframe": [[14, "parse-and-analyze-crawl-logs-in-a-dataframe"]], "Regular Expressions for Extracting Structured Entities": [[15, "regular-expressions-for-extracting-structured-entities"]], "Reverse DNS Lookup in Bulk": [[16, "module-advertools.reverse_dns_lookup"]], "\ud83e\udd16 Analyze and Test robots.txt Files on a Large Scale": [[17, "analyze-and-test-robots-txt-files-on-a-large-scale"]], "Bulk robots.txt Tester": [[17, "bulk-robots-txt-tester"]], "User-agents": [[17, "user-agents"]], "robots.txt Testing Approach": [[17, "robots-txt-testing-approach"]], "Import Search Engine Results Pages (SERPs) for Google and YouTube": [[18, "import-search-engine-results-pages-serps-for-google-and-youtube"]], "Download, Parse, and Analyze XML Sitemaps": [[19, "download-parse-and-analyze-xml-sitemaps"]], "Sitemap Index": [[19, "sitemap-index"]], "Regular XML Sitemaps": [[19, "regular-xml-sitemaps"]], "News Sitemaps": [[19, "news-sitemaps"]], "Video Sitemaps": [[19, "video-sitemaps"]], "\ud83d\udd77 Python SEO Crawler / Spider": [[20, "python-seo-crawler-spider"]], "Discovery Crawling Approach": [[20, "discovery-crawling-approach"]], "Extracted On-Page SEO Elements": [[20, "extracted-on-page-seo-elements"]], "Pre-Determined Crawling Approach (List Mode)": [[20, "pre-determined-crawling-approach-list-mode"]], "SERP Data": [[20, "serp-data"]], "News Articles": [[20, "news-articles"]], "Google Analytics / Google Search Console": [[20, "google-analytics-google-search-console"]], "Custom Extraction with CSS and XPath Selectors": [[20, "custom-extraction-with-css-and-xpath-selectors"]], "Customizing the Crawling Behavior while Following Links": [[20, "customizing-the-crawling-behavior-while-following-links"]], "URL Query Parameters": [[20, "url-query-parameters"]], "URL Regex Patterns": [[20, "url-regex-patterns"]], "Spider Custom Settings and Additional Functionality": [[20, "spider-custom-settings-and-additional-functionality"]], "Stopwords in Several Languages": [[21, "stopwords-in-several-languages"]], "Stopword Languages": [[21, "stopword-languages"]], "Survey - share feedback": [[22, "survey-share-feedback"]], "Twitter Data API": [[23, "twitter-data-api"]], "Authentication": [[23, "authentication"]], "Functions": [[23, "functions"]], "URL Builders": [[24, "url-builders"]], "Split, Parse, and Analyze URL Structure": [[25, "split-parse-and-analyze-url-structure"]], "Query Parameters": [[25, "query-parameters"]], "The URL Path (Directories):": [[25, "the-url-path-directories"]], "Analyzing a large number of URLs": [[25, "analyzing-a-large-number-of-urls"]], "Text Analysis": [[26, "text-analysis"]], "Absolute and Weighted Word Count": [[26, "absolute-and-weighted-word-count"]], "Absolute vs Weighted Frequency": [[26, "absolute-vs-weighted-frequency"]], "Tokenize Words (N-grams)": [[27, "tokenize-words-n-grams"]], "YouTube Data API": [[28, "youtube-data-api"]], "advertools": [[29, "advertools"], [30, "advertools"], [31, "advertools"]], "Change Log - advertools": [[29, "change-log-advertools"]], "(UNRELEASED)": [[29, "unreleased"]], "0.14.2 (2024-02-24)": [[29, "id1"]], "0.14.1 (2024-02-21)": [[29, "id2"]], "0.14.0 (2024-02-18)": [[29, "id3"]], "0.13.5 (2023-08-22)": [[29, "id4"]], "0.13.4 (2023-07-26)": [[29, "id5"]], "0.13.3 (2023-06-27)": [[29, "id6"]], "0.13.2 (2022-09-30)": [[29, "id7"]], "0.13.1 (2022-05-11)": [[29, "id8"]], "0.13.0 (2022-02-10)": [[29, "id9"]], "0.12.3 (2021-11-27)": [[29, "id10"]], "0.12.0,1,2 (2021-11-27)": [[29, "id11"]], "0.11.1 (2021-04-09)": [[29, "id12"]], "0.11.0 (2021-03-31)": [[29, "id13"]], "0.10.7 (2020-09-18)": [[29, "id14"]], "0.10.6 (2020-06-30)": [[29, "id15"]], "0.10.5 (2020-06-14)": [[29, "id16"]], "0.10.4 (2020-06-07)": [[29, "id17"]], "0.10.3 (2020-06-03)": [[29, "id18"]], "0.10.2 (2020-05-25)": [[29, "id19"]], "0.10.1 (2020-05-23)": [[29, "id20"]], "0.10.0 (2020-05-21)": [[29, "id21"]], "0.9.1 (2020-05-19)": [[29, "id22"]], "0.9.0 (2020-04-03)": [[29, "id23"]], "0.8.1 (2020-02-08)": [[29, "id24"]], "0.8.0 (2020-02-02)": [[29, "id25"]], "0.7.3 (2019-04-17)": [[29, "id26"]], "0.7.2 (2019-03-29)": [[29, "id27"]], "0.7.1 (2019-03-26)": [[29, "id28"]], "0.7.0 (2019-03-26)": [[29, "id29"]], "0.6.0 (2019-02-11)": [[29, "id30"]], "0.5.3 (2019-01-31)": [[29, "id31"]], "0.5.2 (2018-12-01)": [[29, "id32"]], "0.5.1 (2018-11-06)": [[29, "id33"]], "0.5.0 (2018-11-04)": [[29, "id34"]], "0.4.1 (2018-10-13)": [[29, "id35"]], "0.4.0 (2018-10-08)": [[29, "id36"]], "0.3.0 (2018-08-14)": [[29, "id37"]], "0.2.0 (2018-07-06)": [[29, "id38"]], "0.1.0 (2018-07-02)": [[29, "id39"]], "Online marketing productivity and analysis tools": [[30, "online-marketing-productivity-and-analysis-tools"]], "SEM": [[30, null]], "SEO": [[30, null], [32, "seo"]], "Text & Content Analysis": [[30, null]], "Social Media": [[30, null], [32, "social-media"]], "Indices and tables": [[30, "indices-and-tables"]], "Index & Change Log": [[30, null]], "advertools: productivity & analysis tools to scale your online marketing": [[32, "advertools-productivity-analysis-tools-to-scale-your-online-marketing"]], "Installation": [[32, "installation"]], "Philosophy/approach": [[32, "philosophy-approach"]], "SEM Campaigns": [[32, "sem-campaigns"]], "Text & Content Analysis (for SEO & Social Media)": [[32, "text-content-analysis-for-seo-social-media"]], "Conventions": [[32, "conventions"]]}, "indexentries": {"advertools": [[0, "module-advertools"]], "module": [[0, "module-advertools"], [1, "module-advertools.ad_create"], [2, "module-advertools.ad_from_string"], [3, "module-advertools.cli"], [4, "module-advertools.cli.cli"], [5, "module-advertools.code_recipes"], [6, "module-advertools.code_recipes.spider_strategies"], [7, "module-advertools.crawlytics"], [8, "module-advertools.emoji"], [9, "module-advertools.extract"], [10, "module-advertools.header_spider"], [11, "module-advertools.image_spider"], [12, "module-advertools.knowledge_graph"], [13, "module-advertools.kw_generate"], [14, "module-advertools.logs"], [15, "module-advertools.regex"], [16, "module-advertools.reverse_dns_lookup"], [17, "module-advertools.robotstxt"], [18, "module-advertools.serp"], [19, "module-advertools.sitemaps"], [20, "module-advertools.spider"], [21, "module-advertools.stopwords"], [23, "module-advertools.twitter"], [24, "module-advertools.url_builders"], [25, "module-advertools.urlytics"], [26, "module-advertools.word_frequency"], [27, "module-advertools.word_tokenize"], [28, "module-advertools.youtube"]], "ad_create() (in module advertools.ad_create)": [[1, "advertools.ad_create.ad_create"]], "advertools.ad_create": [[1, "module-advertools.ad_create"]], "ad_from_string() (in module advertools.ad_from_string)": [[2, "advertools.ad_from_string.ad_from_string"]], "advertools.ad_from_string": [[2, "module-advertools.ad_from_string"]], "capitalize": [[2, "term-capitalize"]], "s": [[2, "term-s"]], "sep": [[2, "term-sep"]], "slots": [[2, "term-slots"]], "advertools.cli": [[3, "module-advertools.cli"]], "advertools.cli.cli": [[4, "module-advertools.cli.cli"]], "advertools.code_recipes": [[5, "module-advertools.code_recipes"]], "advertools.code_recipes.spider_strategies": [[6, "module-advertools.code_recipes.spider_strategies"]], "advertools.crawlytics": [[7, "module-advertools.crawlytics"]], "images() (in module advertools.crawlytics)": [[7, "advertools.crawlytics.images"]], "jl_subset() (in module advertools.crawlytics)": [[7, "advertools.crawlytics.jl_subset"]], "jl_to_parquet() (in module advertools.crawlytics)": [[7, "advertools.crawlytics.jl_to_parquet"]], "links() (in module advertools.crawlytics)": [[7, "advertools.crawlytics.links"]], "parquet_columns() (in module advertools.crawlytics)": [[7, "advertools.crawlytics.parquet_columns"]], "redirects() (in module advertools.crawlytics)": [[7, "advertools.crawlytics.redirects"]], "advertools.emoji": [[8, "module-advertools.emoji"]], "emoji_search() (in module advertools.emoji)": [[8, "advertools.emoji.emoji_search"]], "extract_emoji() (in module advertools.emoji)": [[8, "advertools.emoji.extract_emoji"]], "advertools.extract": [[9, "module-advertools.extract"]], "extract() (in module advertools.extract)": [[9, "advertools.extract.extract"]], "extract_currency() (in module advertools.extract)": [[9, "advertools.extract.extract_currency"]], "extract_exclamations() (in module advertools.extract)": [[9, "advertools.extract.extract_exclamations"]], "extract_hashtags() (in module advertools.extract)": [[9, "advertools.extract.extract_hashtags"]], "extract_intense_words() (in module advertools.extract)": [[9, "advertools.extract.extract_intense_words"]], "extract_mentions() (in module advertools.extract)": [[9, "advertools.extract.extract_mentions"]], "extract_numbers() (in module advertools.extract)": [[9, "advertools.extract.extract_numbers"]], "extract_questions() (in module advertools.extract)": [[9, "advertools.extract.extract_questions"]], "extract_urls() (in module advertools.extract)": [[9, "advertools.extract.extract_urls"]], "extract_words() (in module advertools.extract)": [[9, "advertools.extract.extract_words"]], "headersspider (class in advertools.header_spider)": [[10, "advertools.header_spider.HeadersSpider"]], "advertools.header_spider": [[10, "module-advertools.header_spider"]], "crawl_headers() (in module advertools.header_spider)": [[10, "advertools.header_spider.crawl_headers"]], "custom_settings (headersspider attribute)": [[10, "advertools.header_spider.HeadersSpider.custom_settings"]], "errback() (headersspider method)": [[10, "advertools.header_spider.HeadersSpider.errback"]], "name (headersspider attribute)": [[10, "advertools.header_spider.HeadersSpider.name"]], "parse() (headersspider method)": [[10, "advertools.header_spider.HeadersSpider.parse"]], "start_requests() (headersspider method)": [[10, "advertools.header_spider.HeadersSpider.start_requests"]], "advimagespipeline (class in advertools.image_spider)": [[11, "advertools.image_spider.AdvImagesPipeline"]], "imagespider (class in advertools.image_spider)": [[11, "advertools.image_spider.ImageSpider"]], "imgitem (class in advertools.image_spider)": [[11, "advertools.image_spider.ImgItem"]], "advertools.image_spider": [[11, "module-advertools.image_spider"]], "crawl_images() (in module advertools.image_spider)": [[11, "advertools.image_spider.crawl_images"]], "custom_settings (imagespider attribute)": [[11, "advertools.image_spider.ImageSpider.custom_settings"]], "fields (imgitem attribute)": [[11, "advertools.image_spider.ImgItem.fields"]], "file_path() (advimagespipeline method)": [[11, "advertools.image_spider.AdvImagesPipeline.file_path"]], "include_img_regex (imagespider attribute)": [[11, "advertools.image_spider.ImageSpider.include_img_regex"]], "name (imagespider attribute)": [[11, "advertools.image_spider.ImageSpider.name"]], "parse() (imagespider method)": [[11, "advertools.image_spider.ImageSpider.parse"]], "start_requests() (imagespider method)": [[11, "advertools.image_spider.ImageSpider.start_requests"]], "summarize_crawled_imgs() (in module advertools.image_spider)": [[11, "advertools.image_spider.summarize_crawled_imgs"]], "advertools.knowledge_graph": [[12, "module-advertools.knowledge_graph"]], "knowledge_graph() (in module advertools.knowledge_graph)": [[12, "advertools.knowledge_graph.knowledge_graph"]], "advertools.kw_generate": [[13, "module-advertools.kw_generate"]], "kw_broad() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_broad"]], "kw_exact() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_exact"]], "kw_generate() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_generate"]], "kw_modified() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_modified"]], "kw_neg_broad() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_neg_broad"]], "kw_neg_exact() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_neg_exact"]], "kw_neg_phrase() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_neg_phrase"]], "kw_phrase() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_phrase"]], "advertools.logs": [[14, "module-advertools.logs"]], "crawllogs_to_df() (in module advertools.logs)": [[14, "advertools.logs.crawllogs_to_df"]], "logs_to_df() (in module advertools.logs)": [[14, "advertools.logs.logs_to_df"]], "advertools.regex": [[15, "module-advertools.regex"]], "advertools.reverse_dns_lookup": [[16, "module-advertools.reverse_dns_lookup"]], "reverse_dns_lookup() (in module advertools.reverse_dns_lookup)": [[16, "advertools.reverse_dns_lookup.reverse_dns_lookup"]], "advertools.robotstxt": [[17, "module-advertools.robotstxt"]], "robotstxt_test() (in module advertools.robotstxt)": [[17, "advertools.robotstxt.robotstxt_test"]], "robotstxt_to_df() (in module advertools.robotstxt)": [[17, "advertools.robotstxt.robotstxt_to_df"]], "advertools.serp": [[18, "module-advertools.serp"]], "serp_goog() (in module advertools.serp)": [[18, "advertools.serp.serp_goog"]], "serp_youtube() (in module advertools.serp)": [[18, "advertools.serp.serp_youtube"]], "set_logging_level() (in module advertools.serp)": [[18, "advertools.serp.set_logging_level"]], "youtube_channel_details() (in module advertools.serp)": [[18, "advertools.serp.youtube_channel_details"]], "youtube_video_details() (in module advertools.serp)": [[18, "advertools.serp.youtube_video_details"]], "advertools.sitemaps": [[19, "module-advertools.sitemaps"]], "sitemap_to_df() (in module advertools.sitemaps)": [[19, "advertools.sitemaps.sitemap_to_df"]], "advertools.spider": [[20, "module-advertools.spider"]], "crawl() (in module advertools.spider)": [[20, "advertools.spider.crawl"]], "advertools.stopwords": [[21, "module-advertools.stopwords"]], "advertools.twitter": [[23, "module-advertools.twitter"]], "authenticate() (in module advertools.twitter)": [[23, "advertools.twitter.authenticate"]], "get_application_rate_limit_status() (in module advertools.twitter)": [[23, "advertools.twitter.get_application_rate_limit_status"]], "get_available_trends() (in module advertools.twitter)": [[23, "advertools.twitter.get_available_trends"]], "get_favorites() (in module advertools.twitter)": [[23, "advertools.twitter.get_favorites"]], "get_followers_ids() (in module advertools.twitter)": [[23, "advertools.twitter.get_followers_ids"]], "get_followers_list() (in module advertools.twitter)": [[23, "advertools.twitter.get_followers_list"]], "get_friends_ids() (in module advertools.twitter)": [[23, "advertools.twitter.get_friends_ids"]], "get_friends_list() (in module advertools.twitter)": [[23, "advertools.twitter.get_friends_list"]], "get_home_timeline() (in module advertools.twitter)": [[23, "advertools.twitter.get_home_timeline"]], "get_list_members() (in module advertools.twitter)": [[23, "advertools.twitter.get_list_members"]], "get_list_memberships() (in module advertools.twitter)": [[23, "advertools.twitter.get_list_memberships"]], "get_list_statuses() (in module advertools.twitter)": [[23, "advertools.twitter.get_list_statuses"]], "get_list_subscribers() (in module advertools.twitter)": [[23, "advertools.twitter.get_list_subscribers"]], "get_list_subscriptions() (in module advertools.twitter)": [[23, "advertools.twitter.get_list_subscriptions"]], "get_mentions_timeline() (in module advertools.twitter)": [[23, "advertools.twitter.get_mentions_timeline"]], "get_place_trends() (in module advertools.twitter)": [[23, "advertools.twitter.get_place_trends"]], "get_retweeters_ids() (in module advertools.twitter)": [[23, "advertools.twitter.get_retweeters_ids"]], "get_retweets() (in module advertools.twitter)": [[23, "advertools.twitter.get_retweets"]], "get_supported_languages() (in module advertools.twitter)": [[23, "advertools.twitter.get_supported_languages"]], "get_user_timeline() (in module advertools.twitter)": [[23, "advertools.twitter.get_user_timeline"]], "lookup_status() (in module advertools.twitter)": [[23, "advertools.twitter.lookup_status"]], "lookup_user() (in module advertools.twitter)": [[23, "advertools.twitter.lookup_user"]], "make_dataframe() (in module advertools.twitter)": [[23, "advertools.twitter.make_dataframe"]], "retweeted_of_me() (in module advertools.twitter)": [[23, "advertools.twitter.retweeted_of_me"]], "search() (in module advertools.twitter)": [[23, "advertools.twitter.search"]], "search_users() (in module advertools.twitter)": [[23, "advertools.twitter.search_users"]], "set_auth_params() (in module advertools.twitter)": [[23, "advertools.twitter.set_auth_params"]], "show_lists() (in module advertools.twitter)": [[23, "advertools.twitter.show_lists"]], "show_owned_lists() (in module advertools.twitter)": [[23, "advertools.twitter.show_owned_lists"]], "advertools.url_builders": [[24, "module-advertools.url_builders"]], "url_utm_ga() (in module advertools.url_builders)": [[24, "advertools.url_builders.url_utm_ga"]], "advertools.urlytics": [[25, "module-advertools.urlytics"]], "url_to_df() (in module advertools.urlytics)": [[25, "advertools.urlytics.url_to_df"]], "advertools.word_frequency": [[26, "module-advertools.word_frequency"]], "extra_info": [[26, "term-extra_info"]], "num_list": [[26, "term-num_list"]], "phrase_len": [[26, "term-phrase_len"]], "regex": [[26, "term-regex"]], "rm_words": [[26, "term-rm_words"]], "text_list": [[26, "term-text_list"]], "word_frequency() (in module advertools.word_frequency)": [[26, "advertools.word_frequency.word_frequency"]], "advertools.word_tokenize": [[27, "module-advertools.word_tokenize"]], "word_tokenize() (in module advertools.word_tokenize)": [[27, "advertools.word_tokenize.word_tokenize"]], "activities_list() (in module advertools.youtube)": [[28, "advertools.youtube.activities_list"]], "advertools.youtube": [[28, "module-advertools.youtube"]], "captions_list() (in module advertools.youtube)": [[28, "advertools.youtube.captions_list"]], "channel_sections_list() (in module advertools.youtube)": [[28, "advertools.youtube.channel_sections_list"]], "channels_list() (in module advertools.youtube)": [[28, "advertools.youtube.channels_list"]], "comment_threads_list() (in module advertools.youtube)": [[28, "advertools.youtube.comment_threads_list"]], "comments_list() (in module advertools.youtube)": [[28, "advertools.youtube.comments_list"]], "guide_categories_list() (in module advertools.youtube)": [[28, "advertools.youtube.guide_categories_list"]], "i18n_languages_list() (in module advertools.youtube)": [[28, "advertools.youtube.i18n_languages_list"]], "i18n_regions_list() (in module advertools.youtube)": [[28, "advertools.youtube.i18n_regions_list"]], "playlist_items_list() (in module advertools.youtube)": [[28, "advertools.youtube.playlist_items_list"]], "playlists_list() (in module advertools.youtube)": [[28, "advertools.youtube.playlists_list"]], "search() (in module advertools.youtube)": [[28, "advertools.youtube.search"]], "subscriptions_list() (in module advertools.youtube)": [[28, "advertools.youtube.subscriptions_list"]], "video_categories_list() (in module advertools.youtube)": [[28, "advertools.youtube.video_categories_list"]], "videos_list() (in module advertools.youtube)": [[28, "advertools.youtube.videos_list"]]}}) \ No newline at end of file +Search.setIndex({"docnames": ["advertools", "advertools.ad_create", "advertools.ad_from_string", "advertools.cli", "advertools.cli.cli", "advertools.code_recipes", "advertools.code_recipes.spider_strategies", "advertools.crawlytics", "advertools.emoji", "advertools.extract", "advertools.header_spider", "advertools.image_spider", "advertools.knowledge_graph", "advertools.kw_generate", "advertools.logs", "advertools.regex", "advertools.reverse_dns_lookup", "advertools.robotstxt", "advertools.serp", "advertools.sitemaps", "advertools.spider", "advertools.stopwords", "advertools.survey", "advertools.twitter", "advertools.url_builders", "advertools.urlytics", "advertools.word_frequency", "advertools.word_tokenize", "advertools.youtube", "include_changelog", "index", "modules", "readme"], "filenames": ["advertools.rst", "advertools.ad_create.rst", "advertools.ad_from_string.rst", "advertools.cli.rst", "advertools.cli.cli.rst", "advertools.code_recipes.rst", "advertools.code_recipes.spider_strategies.rst", "advertools.crawlytics.rst", "advertools.emoji.rst", "advertools.extract.rst", "advertools.header_spider.rst", "advertools.image_spider.rst", "advertools.knowledge_graph.rst", "advertools.kw_generate.rst", "advertools.logs.rst", "advertools.regex.rst", "advertools.reverse_dns_lookup.rst", "advertools.robotstxt.rst", "advertools.serp.rst", "advertools.sitemaps.rst", "advertools.spider.rst", "advertools.stopwords.rst", "advertools.survey.rst", "advertools.twitter.rst", "advertools.url_builders.rst", "advertools.urlytics.rst", "advertools.word_frequency.rst", "advertools.word_tokenize.rst", "advertools.youtube.rst", "include_changelog.rst", "index.rst", "modules.rst", "readme.rst"], "titles": ["advertools package", "Create Ads on a Large Scale", "Create Ads Using Long Descriptive Text (top-down approach)", "advertools.cli module", "advertools Command Line Interface (CLI)", "advertools.code_recipes package", "\ud83d\udd77 SEO Crawling & Scraping: Strategies & Recipes", "Crawling and Scraping Analysis", "Emoji: Extract, Analyze, and Get Insights", "Extract structured entities from text lists", "\ud83d\udd77 Python Status Code Checker with Response Headers", "Image Crawler and Downloader", "Import and Analyze Knowledge Graph Results on a Large Scale", "Generate Keywords for SEM Campaigns", "Log File Analysis", "Regular Expressions for Extracting Structured Entities", "Reverse DNS Lookup in Bulk", "\ud83e\udd16 Analyze and Test robots.txt Files on a Large Scale", "Import Search Engine Results Pages (SERPs) for Google and YouTube", "Download, Parse, and Analyze XML Sitemaps", "\ud83d\udd77 Python SEO Crawler / Spider", "Stopwords in Several Languages", "Survey - share feedback", "Twitter Data API", "URL Builders", "Split, Parse, and Analyze URL Structure", "Text Analysis", "Tokenize Words (N-grams)", "YouTube Data API", "advertools", "advertools", "advertools", "advertools: productivity & analysis tools to scale your online marketing"], "terms": {"cli": [0, 29, 30, 31], "code_recip": [0, 29, 30, 31], "seo": [0, 5, 12, 13, 17, 19, 26, 29, 31], "crawl": [0, 5, 10, 11, 16, 17, 25, 29, 30, 31, 32], "scrape": [0, 5, 14, 20, 29, 30, 31], "strategi": [0, 5, 7, 10, 11, 13, 17, 19, 20, 28, 30], "recip": [0, 5, 20, 29, 30], "how": [0, 2, 5, 7, 8, 9, 10, 13, 17, 19, 20, 23, 25, 26, 29, 30, 31, 32], "list": [0, 1, 2, 5, 7, 8, 10, 11, 12, 13, 14, 16, 17, 18, 19, 21, 23, 25, 26, 27, 28, 29, 30, 31, 32], "page": [0, 2, 4, 5, 7, 9, 10, 11, 12, 13, 14, 17, 19, 21, 23, 25, 26, 28, 29, 30, 31, 32], "those": [0, 2, 4, 5, 7, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 25, 26, 28, 29, 30, 32], "onli": [0, 1, 4, 5, 7, 9, 10, 11, 14, 15, 17, 18, 19, 20, 23, 24, 26, 28, 29, 30, 32], "mode": [0, 4, 5, 9, 23, 29, 30, 31], "can": [0, 2, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29, 30, 32], "i": [0, 1, 2, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29, 30, 32], "websit": [0, 4, 5, 12, 14, 17, 18, 19, 20, 25, 28, 29, 30, 31, 32], "includ": [0, 4, 5, 7, 9, 11, 13, 14, 17, 18, 19, 20, 23, 25, 27, 28, 29, 30], "its": [0, 1, 4, 5, 7, 8, 10, 12, 13, 14, 19, 20, 23, 25, 26, 28, 30, 32], "sub": [0, 4, 5, 7, 8, 9, 11, 14, 17, 19, 20, 23, 25, 29, 30, 32], "domain": [0, 4, 5, 7, 10, 14, 16, 17, 18, 20, 25, 29, 30], "save": [0, 5, 7, 10, 11, 14, 17, 20, 25, 29, 30, 32], "copi": [0, 5, 30, 32], "log": [0, 5, 16, 18, 19, 20, 25, 28, 31, 32], "my": [0, 1, 5, 9, 20, 26, 30], "audit": [0, 5, 17, 19, 20, 30], "them": [0, 2, 4, 5, 7, 9, 10, 11, 13, 14, 16, 17, 19, 20, 23, 25, 26, 29, 30, 32], "later": [0, 5, 14, 19, 30], "automat": [0, 5, 18, 28, 30, 32], "stop": [0, 4, 5, 20, 26, 29, 30, 32], "base": [0, 2, 5, 8, 10, 11, 15, 18, 20, 23, 26, 28, 29, 30, 32], "certain": [0, 4, 5, 7, 9, 10, 12, 14, 16, 17, 19, 20, 21, 25, 26, 29, 30], "condit": [0, 5, 11, 14, 20, 30], "di": [0, 5, 11, 30], "obei": [0, 5, 11, 14, 30], "robot": [0, 5, 11, 14, 19, 20, 29, 30, 31, 32], "txt": [0, 5, 8, 11, 14, 19, 20, 29, 30, 31, 32], "rule": [0, 5, 10, 11, 14, 17, 20, 25, 30, 32], "do": [0, 2, 4, 5, 7, 9, 10, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 28, 29, 30, 32], "set": [0, 2, 4, 5, 7, 10, 11, 12, 13, 14, 18, 19, 23, 25, 26, 27, 28, 29, 30, 31, 32], "user": [0, 4, 5, 9, 11, 13, 14, 16, 18, 20, 23, 28, 29, 30, 31, 32], "agent": [0, 5, 10, 11, 14, 16, 20, 29, 30, 31, 32], "while": [0, 4, 5, 7, 14, 18, 23, 25, 26, 28, 29, 30, 31], "control": [0, 5, 10, 11, 18, 20, 29, 30, 32], "number": [0, 5, 7, 8, 12, 13, 16, 17, 18, 19, 20, 23, 26, 28, 29, 30, 31, 32], "concurr": [0, 5, 18, 28, 29, 30], "request": [0, 5, 7, 10, 11, 12, 14, 16, 18, 20, 23, 28, 29, 30, 32], "slow": [0, 5, 30], "down": [0, 5, 13, 14, 26, 29, 30, 31, 32], "so": [0, 2, 5, 7, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 30, 32], "don": [0, 1, 4, 5, 7, 9, 10, 14, 18, 20, 25, 26, 29, 30, 32], "t": [0, 1, 4, 5, 7, 9, 10, 13, 14, 18, 20, 25, 26, 27, 29, 30, 32], "hit": [0, 5, 10, 20, 30], "server": [0, 5, 10, 14, 16, 18, 20, 28, 30], "too": [0, 5, 9, 20, 26, 30, 32], "hard": [0, 5, 30], "multipl": [0, 4, 5, 7, 9, 12, 17, 18, 19, 20, 29, 30, 32], "same": [0, 1, 2, 4, 5, 7, 8, 9, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 29, 30, 32], "job": [0, 5, 13, 30, 32], "want": [0, 1, 2, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 30, 32], "follow": [0, 2, 4, 5, 7, 8, 9, 14, 18, 19, 21, 23, 25, 26, 28, 29, 30, 31, 32], "link": [0, 4, 5, 9, 10, 18, 19, 23, 25, 28, 29, 30, 31, 32], "from": [0, 5, 7, 11, 12, 14, 15, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 30, 31, 32], "specifi": [0, 2, 5, 7, 9, 12, 14, 17, 18, 20, 23, 27, 28, 29, 30, 32], "depth": [0, 5, 10, 20, 30], "paus": [0, 5, 30], "resum": [0, 5, 30, 32], "make": [0, 1, 2, 5, 7, 9, 10, 14, 16, 17, 18, 19, 20, 23, 25, 26, 29, 30, 32], "sure": [0, 1, 2, 5, 9, 10, 16, 18, 20, 23, 25, 29, 30], "twice": [0, 5, 26, 29, 30], "us": [0, 1, 5, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 28, 29, 30, 31, 32], "proxi": [0, 5, 16, 30], "chang": [0, 2, 4, 5, 11, 14, 17, 18, 20, 26, 32], "default": [0, 2, 4, 5, 9, 12, 13, 14, 18, 19, 20, 23, 25, 26, 27, 28, 29, 30, 32], "header": [0, 4, 5, 7, 17, 19, 20, 29, 30, 31, 32], "xpath": [0, 4, 5, 29, 30, 31, 32], "express": [0, 4, 5, 7, 8, 9, 11, 13, 14, 17, 18, 20, 26, 29, 30, 31, 32], "custom": [0, 4, 5, 10, 11, 14, 18, 29, 30, 31, 32], "extract": [0, 5, 7, 14, 17, 25, 29, 30, 31, 32], "string": [0, 1, 2, 5, 7, 8, 9, 12, 15, 18, 23, 24, 27, 28, 29, 30], "creat": [0, 6, 7, 8, 10, 11, 12, 13, 18, 23, 25, 28, 29, 30, 31, 32], "ad": [0, 4, 7, 9, 13, 20, 23, 24, 26, 28, 29, 30, 31, 32], "larg": [0, 6, 8, 14, 16, 18, 19, 20, 26, 29, 30, 31, 32], "scale": [0, 16, 29, 30, 31], "long": [0, 1, 4, 7, 9, 14, 16, 17, 18, 23, 25, 28, 29, 30, 31, 32], "descript": [0, 6, 7, 9, 12, 13, 20, 23, 25, 26, 28, 29, 30, 31, 32], "text": [0, 1, 6, 7, 10, 14, 20, 21, 23, 27, 28, 29, 31], "top": [0, 7, 8, 9, 12, 13, 14, 16, 18, 19, 20, 23, 25, 26, 28, 29, 30, 31, 32], "approach": [0, 13, 25, 26, 29, 30, 31], "googl": [0, 4, 6, 14, 16, 17, 25, 28, 29, 30, 31, 32], "facebook": [0, 17, 24, 26, 29, 30, 31], "feed": [0, 28, 29, 30, 31], "instant": [0, 29, 30, 31], "articl": [0, 7, 19, 25, 29, 30, 31, 32], "analysi": [0, 9, 10, 16, 19, 20, 23, 29, 31], "analyz": [0, 10, 18, 20, 21, 23, 26, 29, 30, 31, 32], "imag": [0, 10, 12, 14, 17, 18, 19, 20, 23, 29, 30, 31], "redirect": [0, 4, 14, 20, 29, 30, 31, 32], "handl": [0, 6, 14, 16, 17, 18, 23, 29, 30, 31, 32], "veri": [0, 1, 6, 10, 12, 13, 14, 16, 17, 19, 20, 23, 25, 26, 29, 30, 31, 32], "file": [0, 1, 6, 10, 16, 18, 19, 20, 25, 29, 30, 31, 32], "compress": [0, 14, 25, 29, 30, 31], "explor": [0, 8, 9, 19, 20, 21, 23, 29, 30, 31], "column": [0, 4, 6, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 29, 30, 31], "data": [0, 4, 9, 10, 11, 13, 17, 18, 19, 25, 26, 29, 30, 31, 32], "type": [0, 1, 2, 4, 6, 9, 10, 12, 13, 14, 17, 18, 20, 23, 25, 26, 28, 29, 30, 31, 32], "parquet": [0, 14, 25, 29, 30, 31], "function": [0, 2, 4, 6, 8, 10, 11, 12, 13, 16, 17, 18, 19, 25, 26, 27, 29, 30, 31, 32], "emoji": [0, 20, 26, 29, 30, 31, 32], "get": [0, 1, 2, 6, 7, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29, 30, 31, 32], "insight": [0, 6, 19, 26, 29, 30, 31, 32], "search": [0, 10, 12, 17, 19, 23, 24, 25, 28, 29, 30, 31, 32], "structur": [0, 6, 7, 10, 13, 19, 23, 29, 30, 31, 32], "entiti": [0, 12, 17, 19, 23, 29, 30, 31, 32], "hashtag": [0, 15, 17, 23, 26, 29, 30, 31, 32], "mention": [0, 12, 15, 23, 25, 26, 29, 30, 31, 32], "currenc": [0, 26, 29, 30, 31, 32], "1234567890\u0660\u0661\u0662\u0663\u0664\u0665\u0666\u0667\u0668\u0669\u32ba\ud804\udc5b\ud800\udd0d\ud802\udcaa\u24f2\ud804\udc63\ud800\udd28\ud802\udd1b": [0, 29, 30, 31], "question": [0, 2, 19, 23, 26, 27, 29, 30, 31, 32], "\u0294": [0, 29, 30, 31], "exclam": [0, 27, 29, 30, 31], "python": [0, 4, 8, 21, 23, 29, 30, 31, 32], "statu": [0, 4, 7, 8, 14, 20, 23, 28, 29, 30, 31, 32], "code": [0, 4, 6, 7, 8, 9, 11, 12, 14, 18, 19, 20, 23, 24, 28, 29, 30, 31, 32], "checker": [0, 29, 30, 31], "respons": [0, 4, 7, 11, 12, 14, 17, 18, 19, 20, 23, 28, 29, 30, 31, 32], "crawler": [0, 6, 10, 14, 29, 30, 31, 32], "download": [0, 6, 7, 10, 17, 20, 25, 28, 29, 30, 31, 32], "name": [0, 1, 4, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 19, 20, 23, 24, 25, 26, 28, 29, 30, 31, 32], "import": [0, 1, 2, 6, 7, 8, 9, 10, 11, 13, 14, 16, 17, 19, 20, 21, 23, 25, 26, 27, 28, 29, 30, 31, 32], "knowledg": [0, 28, 29, 30, 31], "graph": [0, 20, 29, 30, 31], "result": [0, 1, 6, 7, 8, 11, 14, 17, 19, 20, 23, 25, 26, 28, 29, 30, 31, 32], "account": [0, 13, 17, 18, 23, 26, 28, 29, 30, 31], "setup": [0, 29, 30, 31], "": [0, 1, 2, 4, 6, 7, 9, 10, 11, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 29, 30, 31, 32], "api": [0, 7, 9, 18, 29, 30, 31, 32], "gener": [0, 6, 9, 14, 17, 20, 24, 25, 26, 28, 29, 30, 31, 32], "keyword": [0, 8, 12, 20, 21, 26, 29, 30, 31, 32], "sem": [0, 20, 29, 31], "campaign": [0, 1, 4, 24, 29, 30, 31], "run": [0, 4, 6, 8, 11, 12, 13, 16, 17, 18, 19, 20, 23, 25, 26, 29, 30, 31, 32], "logs_to_df": [0, 29, 30, 31], "support": [0, 10, 17, 18, 20, 23, 28, 29, 30, 31, 32], "format": [0, 1, 2, 7, 10, 11, 13, 15, 17, 18, 20, 23, 28, 29, 30, 31, 32], "prepar": [0, 8, 19, 29, 30, 31], "pars": [0, 10, 11, 20, 29, 30, 31, 32], "datafram": [0, 6, 7, 8, 9, 11, 12, 13, 17, 18, 19, 20, 23, 25, 26, 29, 30, 31, 32], "regular": [0, 4, 7, 8, 9, 11, 14, 20, 26, 29, 30, 31], "revers": [0, 14, 18, 23, 28, 29, 30, 31], "dn": [0, 14, 29, 30, 31], "lookup": [0, 14, 23, 29, 30, 31], "bulk": [0, 29, 30, 31], "test": [0, 6, 8, 29, 30, 31, 32], "tester": [0, 29, 30, 31], "engin": [0, 10, 12, 13, 14, 17, 19, 29, 30, 31, 32], "serp": [0, 12, 25, 29, 30, 31, 32], "youtub": [0, 12, 26, 29, 30, 31, 32], "xml": [0, 10, 17, 25, 29, 30, 31, 32], "sitemap": [0, 17, 20, 25, 29, 30, 31, 32], "index": [0, 4, 7, 10, 12, 14, 18, 20, 23, 29, 31, 32], "new": [0, 6, 7, 9, 14, 18, 25, 26, 29, 30, 31, 32], "video": [0, 17, 18, 23, 26, 28, 29, 30, 31, 32], "spider": [0, 4, 6, 10, 11, 14, 29, 30, 31, 32], "discoveri": [0, 29, 30, 31], "On": [0, 2, 23, 29, 30, 31], "element": [0, 2, 6, 7, 9, 10, 12, 29, 30, 31, 32], "pre": [0, 14, 26, 29, 30, 31], "determin": [0, 13, 18, 26, 29, 30, 31], "analyt": [0, 25, 26, 29, 30], "consol": [0, 18, 25, 28, 30], "css": [0, 4, 6, 29, 30, 31, 32], "selector": [0, 4, 29, 30, 31, 32], "behavior": [0, 4, 10, 11, 19, 28, 29, 30, 31, 32], "url": [0, 2, 6, 7, 9, 10, 11, 12, 14, 15, 17, 18, 19, 23, 26, 28, 29, 30, 31, 32], "queri": [0, 11, 12, 14, 17, 18, 19, 23, 28, 29, 30, 31, 32], "paramet": [0, 1, 2, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 23, 24, 26, 27, 28, 29, 30, 31, 32], "regex": [0, 6, 7, 8, 9, 11, 14, 15, 26, 29, 30, 32], "pattern": [0, 4, 6, 7, 9, 11, 17, 19, 25, 26, 30], "addit": [0, 2, 6, 9, 11, 14, 16, 18, 23, 26, 28, 29, 30, 31, 32], "stopword": [0, 26, 29, 30, 31, 32], "sever": [0, 6, 7, 10, 14, 18, 23, 26, 28, 29, 30, 31, 32], "languag": [0, 6, 10, 12, 13, 18, 19, 20, 23, 25, 26, 28, 29, 30, 31, 32], "twitter": [0, 6, 9, 10, 17, 20, 24, 29, 30, 31, 32], "authent": [0, 16, 18, 28, 29, 30, 31], "builder": [0, 29, 30, 31], "split": [0, 2, 14, 19, 20, 26, 27, 29, 30, 31, 32], "The": [0, 1, 2, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 26, 27, 28, 29, 30, 31, 32], "path": [0, 6, 7, 10, 11, 14, 17, 19, 20, 29, 30, 31], "directori": [0, 11, 29, 30, 31], "absolut": [0, 16, 17, 25, 29, 30, 31, 32], "weight": [0, 29, 30, 31, 32], "word": [0, 1, 2, 8, 9, 12, 13, 14, 18, 19, 23, 29, 30, 31, 32], "count": [0, 1, 7, 8, 9, 14, 16, 17, 19, 23, 27, 29, 30, 31, 32], "v": [0, 19, 30], "frequenc": [0, 8, 9, 16, 29, 30, 32], "token": [0, 26, 29, 30, 31], "n": [0, 4, 6, 7, 19, 29, 30, 31], "gram": [0, 4, 29, 30, 31], "digit": [0, 32], "market": [0, 12, 13, 18, 20, 24, 29], "product": [0, 1, 2, 12, 13, 18, 20, 25, 26, 29], "tool": [0, 10, 14, 18, 19, 20, 25], "when": [1, 4, 6, 9, 10, 12, 14, 17, 18, 19, 20, 23, 26, 28, 29, 32], "you": [1, 2, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 30, 32], "also": [1, 2, 4, 6, 7, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29, 32], "need": [1, 2, 4, 6, 7, 9, 10, 11, 12, 13, 14, 16, 17, 18, 20, 23, 26, 29, 32], "For": [1, 4, 6, 7, 10, 12, 14, 17, 18, 19, 20, 23, 25, 26, 28], "similar": [1, 7, 9, 20, 25, 28, 29], "categori": [1, 18, 19, 25, 28], "typic": [1, 2, 7, 9, 10, 14, 16, 18, 19, 20, 23, 25, 26, 28, 32], "replac": [1, 14, 19, 25, 29, 32], "latest": [1, 2, 4, 20, 23], "now": [1, 2, 7, 9, 13, 14, 17, 19, 20, 23, 26, 29], "mani": [1, 2, 6, 7, 8, 9, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 28, 32], "time": [1, 2, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 19, 20, 23, 26, 28, 29, 32], "have": [1, 2, 4, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "advertool": [1, 2, 6, 7, 8, 9, 10, 11, 13, 14, 16, 17, 19, 20, 21, 23, 25, 26], "adv": [1, 2, 4, 6, 7, 8, 9, 10, 11, 13, 14, 16, 17, 19, 20, 21, 23, 25, 26], "dubai": 1, "tokyo": 1, "singapor": 1, "ad_creat": [1, 13, 20, 29, 32], "templat": [1, 2, 6, 19], "5": [1, 6, 7, 8, 9, 12, 13, 14, 16, 17, 18, 19, 20, 21, 26, 30], "star": [1, 19], "hotel": [1, 8, 18], "max_len": [1, 4, 13], "30": [1, 2, 6, 14, 17, 18, 19, 30], "fallback": 1, "great": [1, 9, 20, 25, 26, 32], "citi": [1, 12], "In": [1, 2, 6, 7, 9, 11, 12, 14, 16, 17, 18, 19, 20, 23, 25, 26, 28, 29, 32], "an": [1, 2, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 26, 27, 28, 29, 30, 32], "thing": [1, 2, 9, 12, 13, 19, 20, 25, 27, 30, 32], "watch": [1, 19, 23], "out": [1, 9, 12, 14, 17, 18, 20, 23, 25, 26, 28, 32], "sinc": [1, 2, 7, 9, 10, 19, 20, 23, 26], "limit": [1, 2, 12, 17, 18, 20, 23, 28, 32], "each": [1, 2, 4, 7, 8, 9, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29, 32], "slot": [1, 2, 29, 32], "exce": [1, 2], "thi": [1, 2, 4, 6, 7, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 30, 32], "provid": [1, 2, 4, 6, 7, 9, 11, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 28, 29, 32], "case": [1, 2, 4, 6, 7, 8, 9, 10, 11, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 29, 32], "longer": [1, 2, 9, 18, 26, 28, 29], "than": [1, 2, 17, 18, 20, 23, 26, 28, 29, 32], "lisbon": 1, "porto": 1, "algarv": 1, "freixo": 1, "de": [1, 6, 12, 13, 20], "espada": 1, "\u00e0": 1, "cinta": 1, "portug": 1, "capit": [1, 2, 4, 12, 13], "true": [1, 2, 4, 6, 7, 9, 10, 11, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 29], "sourc": [1, 2, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 24, 25, 26, 27, 28], "insert": [1, 25], "place": [1, 6, 8, 9, 11, 12, 23, 29], "within": [1, 2, 13, 18, 20, 23, 26, 27, 28], "str": [1, 2, 7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 24, 25, 26], "A": [1, 2, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "brace": 1, "e": [1, 4, 6, 7, 10, 12, 13, 18, 20, 24, 25, 26, 28, 29], "g": [1, 4, 6, 10, 18, 20, 24, 28, 29], "todai": [1, 8, 9], "int": [1, 7, 9, 11, 12, 13, 16, 18, 19, 23, 26, 29], "maximum": [1, 2, 12, 13, 16, 18, 19, 20, 23, 28], "allow": [1, 2, 4, 10, 14, 17, 18, 20, 23, 25, 28, 29, 32], "length": [1, 2, 10, 19, 23, 25, 26, 27, 29], "full": [1, 6, 7, 8, 10, 11, 13, 19, 20, 23, 26, 29, 32], "bool": [1, 2, 9, 12, 13, 18, 19, 20, 23, 25, 26], "whether": [1, 2, 4, 6, 7, 9, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 29], "return": [1, 2, 4, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 23, 24, 25, 26, 27, 28, 29, 32], "exampl": [1, 2, 4, 6, 7, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "let": [1, 2, 7, 9, 12, 13, 17, 18, 19, 20, 26, 28, 32], "one": [1, 2, 4, 6, 7, 8, 9, 12, 14, 15, 17, 18, 19, 20, 23, 25, 26, 28, 29, 32], "two": [1, 2, 6, 7, 9, 11, 13, 15, 17, 18, 20, 25, 26, 28, 29, 32], "three": [1, 6, 9, 19, 20, 23, 25, 26, 29, 32], "20": [1, 4, 6, 9, 12, 14, 17, 18, 19, 20, 23, 26, 28, 29], "One": [1, 6, 7, 10, 13, 17, 19, 20, 26], "favorit": [1, 9, 23, 26], "car": 1, "toyota": [1, 13], "bmw": [1, 13], "merced": 1, "lamborghini": 1, "28": [1, 17], "keep": [1, 6, 8, 12, 13, 16, 17, 20, 23, 26, 29, 32], "As": [1, 6, 7, 16, 17, 19, 25, 26, 27, 28, 32], "50": [1, 6, 17, 18, 26, 29], "fals": [1, 2, 6, 7, 9, 13, 14, 17, 19, 20, 23, 25, 26, 29], "produc": [1, 14, 18, 19, 32], "error": [1, 4, 6, 14, 16, 18, 20, 28, 29], "someth": [1, 2, 26], "traceback": 1, "most": [1, 2, 6, 7, 9, 10, 13, 14, 18, 19, 20, 23, 26, 28, 29, 32], "recent": [1, 23, 28], "call": [1, 18, 19, 20, 23, 25, 26, 29, 32], "last": [1, 2, 9, 17, 19, 20, 25, 26, 29], "input": [1, 2], "line": [1, 6, 7, 10, 12, 14, 16, 17, 18, 20, 29, 30], "1": [1, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 30], "modul": [1, 9, 20, 26, 29, 30, 31, 32], "26": [1, 6, 7, 9, 10, 14, 20, 24, 30], "valueerror": [1, 29], "should": [1, 4, 6, 9, 10, 11, 12, 17, 18, 20, 23, 26, 28, 32], "char": 1, "about": [2, 6, 7, 8, 9, 10, 11, 14, 16, 17, 18, 19, 20, 23, 25, 26, 28, 29, 30, 32], "your": [2, 4, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 23, 24, 25, 26, 28, 30], "especi": [2, 6, 13, 17, 19, 26], "respect": [2, 6, 7, 14, 20, 23, 26, 29], "land": [2, 13, 32], "ha": [2, 6, 7, 9, 13, 16, 17, 19, 20, 23, 25, 26, 28, 29, 32], "becom": [2, 4, 14, 20, 25, 26], "consider": [2, 26, 32], "platform": [2, 26, 32], "90": [2, 14], "charact": [2, 4, 9, 18, 23, 26, 27, 28, 29], "total": [2, 14, 17, 26], "270": 2, "That": [2, 6, 9, 20, 26, 28], "more": [2, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 29, 32], "enough": [2, 13, 18, 26, 28], "space": [2, 4, 18, 19, 23, 27, 29], "talk": 2, "main": [2, 7, 12, 17, 18, 19, 20, 23, 25, 26, 29, 32], "featur": [2, 11, 18, 19, 23, 28, 29, 32], "util": [2, 23], "all": [2, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 29, 32], "detail": [2, 4, 6, 7, 10, 11, 13, 14, 17, 18, 20, 23, 28, 32], "fit": [2, 28, 29], "correctli": 2, "given": [2, 6, 11, 17, 18, 19, 23, 25, 29], "ar": [2, 4, 6, 7, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 30, 32], "ad_from_str": [2, 13, 20, 29, 32], "doe": [2, 6, 8, 9, 16, 17, 18, 19, 20, 26, 27, 28, 32], "exactli": [2, 7, 8, 28], "divid": [2, 26], "ani": [2, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "remain": [2, 19, 27], "append": [2, 17, 18, 20, 29], "end": [2, 7, 9, 10, 18, 20, 25, 27, 29, 32], "anoth": [2, 7, 10, 13, 14, 19, 25, 26, 27, 32], "benefit": [2, 25, 32], "take": [2, 6, 7, 9, 12, 13, 14, 17, 19, 20, 23, 25, 26, 29, 32], "write": [2, 7, 32], "onc": [2, 4, 6, 7, 12, 13, 14, 18, 19, 20, 23, 25, 26, 28, 30, 32], "easili": [2, 6, 7, 10, 14, 17, 19, 21, 23, 25, 32], "differ": [2, 4, 6, 8, 9, 12, 13, 14, 15, 17, 18, 19, 20, 23, 25, 26, 28, 32], "here": [2, 7, 8, 9, 10, 13, 14, 18, 19, 20, 25, 26, 32], "quick": [2, 7, 17, 19], "overview": [2, 7, 8, 9, 16, 19, 32], "avail": [2, 4, 6, 7, 10, 11, 12, 14, 15, 18, 19, 20, 21, 23, 25, 26, 28, 29, 32], "option": [2, 6, 7, 9, 10, 11, 13, 14, 17, 18, 20, 21, 23, 25, 26, 27, 28, 29, 32], "would": [2, 6, 7, 9, 11, 14, 17, 18, 19, 20, 23, 25, 26, 28], "note": [2, 7, 9, 12, 18, 20, 23, 25, 26, 28, 32], "although": [2, 20, 26, 32], "other": [2, 4, 7, 8, 9, 10, 11, 12, 14, 15, 17, 18, 19, 20, 23, 25, 26, 28, 29, 32], "group": [2, 4, 7, 8, 9, 13, 17, 29, 32], "fewer": 2, "sep": [2, 29], "separ": [2, 4, 7, 9, 14, 18, 20, 23, 25, 28, 29], "which": [2, 4, 6, 7, 9, 10, 11, 12, 14, 17, 18, 19, 20, 21, 23, 25, 26, 27, 28, 29, 32], "none": [2, 4, 7, 8, 9, 10, 11, 12, 14, 17, 18, 20, 23, 24, 25, 26, 28], "whitespac": [2, 26, 27, 29], "els": [2, 12, 23, 26], "sometim": [2, 6, 13, 17, 20, 26, 32], "might": [2, 4, 6, 7, 8, 9, 10, 11, 13, 14, 17, 18, 19, 20, 23, 25, 26, 28, 30, 32], "hyphen": 2, "leav": [2, 17], "intact": 2, "If": [2, 6, 9, 12, 14, 18, 19, 20, 23, 25, 26, 28, 29, 30], "first": [2, 6, 7, 9, 12, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "letter": [2, 7, 8, 18, 28], "five": [2, 17, 20, 25, 26], "alwai": [2, 23, 26, 29], "six": [2, 26], "ensur": [2, 20], "remaind": [2, 29], "lost": [2, 6], "know": [2, 7, 9, 13, 14, 20, 25, 26, 32], "what": [2, 6, 7, 8, 9, 10, 12, 13, 14, 16, 17, 19, 20, 23, 25, 26, 32], "miss": [2, 9, 19, 26, 29], "shorter": [2, 28, 29], "still": [2, 9, 11, 18, 19, 23, 25, 26, 28], "consist": [2, 6, 13, 14, 23, 25, 26, 29], "desc_text": 2, "gadget": 2, "onlin": [2, 20], "gx12": 2, "model": [2, 14, 19], "come": [2, 13, 14, 26], "13": [2, 6, 7, 8, 9, 10, 11, 12, 14, 17, 19, 30], "lot": [2, 7, 10, 14, 19, 25, 32], "good": [2, 4, 6, 9, 14, 16, 19, 20, 25, 26], "stuff": [2, 32], "health": [2, 28], "start": [2, 4, 6, 9, 13, 14, 18, 19, 20, 23, 26, 28, 29, 32], "shop": [2, 20], "len": [2, 4], "130": [2, 14, 16, 19], "see": [2, 6, 7, 8, 9, 11, 12, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 32], "scenario": 2, "valu": [2, 4, 6, 10, 12, 14, 18, 19, 20, 23, 25, 26, 28, 29], "extra": [2, 14, 26], "empti": [2, 6, 7, 8, 9, 25, 26, 29], "125": [2, 6, 16], "25": [2, 7, 9, 16, 19, 20, 24, 26, 30], "look": [2, 7, 13, 19, 23, 26], "just": [2, 4, 8, 9, 11, 14, 18, 20, 23, 26, 32], "second": [2, 6, 7, 9, 12, 13, 18, 20, 25, 29, 32], "where": [2, 4, 6, 7, 9, 10, 11, 14, 18, 19, 20, 23, 25, 26, 29, 32], "our": [2, 7, 9, 14, 23, 26], "we": [2, 7, 9, 12, 13, 14, 17, 19, 20, 25, 26, 32], "up": [2, 7, 10, 12, 13, 18, 20, 23, 25, 26, 29, 32], "15": [2, 6, 7, 9, 12, 14, 19, 25, 28], "convert": [2, 7, 14, 20, 25, 29, 32], "restrict": [2, 11, 12, 18, 20, 23, 28, 29], "iter": 2, "integ": [2, 18, 28], "after": [2, 4, 6, 7, 9, 11, 18, 19, 20, 23, 25, 26, 27, 28, 32], "text_ad": 2, "accord": [2, 17, 27], "spec": 2, "short": [2, 18, 26, 28], "wai": [2, 6, 7, 9, 10, 14, 19, 20, 23, 25, 29, 32], "10": [2, 6, 7, 8, 9, 10, 14, 17, 18, 19, 20, 23, 25, 26, 30], "bY": 2, "captial": 2, "To": [2, 11, 13, 17, 20, 26, 28, 30], "instal": [4, 6, 30], "python3": [4, 32], "m": [4, 9, 14, 17, 18, 19, 20, 26, 28, 32], "pip": [4, 6, 30, 32], "acess": 4, "go": [4, 9, 10, 11, 13, 14, 17, 18, 19, 20, 25, 26, 29], "program": [4, 23, 32], "help": [4, 6, 7, 9, 10, 14, 16, 18, 19, 20, 23, 25, 27, 29, 32], "h": [4, 7, 9, 14, 29], "access": [4, 11, 14, 17, 18, 21, 23, 28, 32], "specif": [4, 7, 10, 18, 23, 28, 32], "usag": [4, 7, 8, 9, 20, 23, 32], "web": [4, 10, 11, 12, 18, 19, 20, 21], "local": [4, 18, 23, 28], "machin": [4, 19, 32], "http": [4, 6, 7, 8, 9, 10, 11, 12, 14, 17, 19, 20, 23, 25, 26], "www": [4, 6, 7, 8, 9, 10, 11, 14, 17, 19], "com": [4, 6, 7, 8, 9, 10, 11, 12, 14, 16, 17, 18, 19, 20, 23, 24, 25, 28], "jo": 4, "output": [4, 6, 7, 10, 11, 14, 20, 23, 25, 29], "google_robot": 4, "robotslist": 4, "multi_robot": 4, "posit": [4, 7, 9, 12, 18, 23, 28], "argument": [4, 6, 12, 18, 20, 29], "show": [4, 7, 8, 9, 12, 13, 14, 18, 20, 25, 26, 28, 29], "messag": [4, 14, 29], "exit": [4, 14], "r": 4, "0": [4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 25, 26, 28, 30], "sitemap_url": [4, 19], "recurs": [4, 19, 20, 29], "fetch": [4, 17, 20, 29], "url_list": [4, 6, 10, 14, 20], "custom_set": [4, 6, 10, 11, 14, 20, 29], "output_fil": [4, 6, 7, 10, 14, 17, 20, 25, 29], "filepath": [4, 7, 20], "jl": [4, 6, 7, 10, 14, 17, 20, 29], "modifi": [4, 8, 9, 10, 11, 13, 17, 19, 20, 21, 23, 26], "equal": [4, 14, 23, 25], "sign": [4, 9, 17, 20, 25, 27], "without": [4, 6, 10, 18, 20, 25, 26, 28, 29, 32], "between": [4, 6, 13, 14, 18, 19, 20, 26, 27, 28, 32], "log_fil": [4, 6, 14, 20], "closespider_timeout": [4, 6, 20], "f": [4, 14], "field": [4, 11, 14, 18, 28, 29], "errors_fil": [4, 14], "log_format": [4, 14], "common": [4, 6, 10, 14, 18, 28], "combin": [4, 12, 13, 14, 15, 17, 18, 20, 32], "common_with_vhost": [4, 14], "nginx_error": [4, 14], "apache_error": [4, 14], "special": [4, 9, 11, 19, 20, 25, 29, 32], "instead": [4, 9, 17, 20, 23, 25, 26, 28, 29], "ip_list": [4, 16], "semkw": 4, "exact": [4, 6, 13, 23], "broad": [4, 13], "l": 4, "c": [4, 6, 10, 21], "campaign_nam": [4, 13], "contain": [4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "sell": [4, 13, 19], "per": [4, 6, 7, 8, 9, 17, 18, 19, 23, 26, 32], "match": [4, 7, 8, 9, 11, 12, 13, 14, 18, 20, 23, 28, 29], "max": [4, 6, 10, 20], "3": [4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 30], "adgroup": [4, 13], "order": [4, 7, 9, 11, 12, 13, 18, 20, 28, 29], "matter": [4, 13], "permut": [4, 13], "bui": [4, 13, 17], "arab": [4, 9, 18, 19, 21, 29], "azerbaijani": [4, 21, 29], "bengali": [4, 21, 29], "catalan": [4, 18, 21, 29], "chines": [4, 18, 21, 28, 29], "croatian": [4, 18, 21, 29], "danish": [4, 18, 21, 29], "dutch": [4, 18, 21, 29], "english": [4, 18, 21, 25, 26, 29], "finnish": [4, 18, 21, 29], "french": [4, 18, 21, 29], "german": [4, 18, 21, 29], "greek": [4, 9, 18, 21, 29], "hebrew": [4, 18, 21, 29], "hindi": [4, 19, 21, 29], "hungarian": [4, 18, 21, 29], "indonesian": [4, 18, 21, 29], "irish": [4, 21, 29], "italian": [4, 18, 21, 29], "japanes": [4, 18, 21, 29], "kazakh": [4, 21, 29], "nepali": [4, 21, 29], "norwegian": [4, 18, 21, 29], "persian": [4, 19, 21, 29], "polish": [4, 18, 21, 29], "portugues": [4, 18, 19, 21, 29], "romanian": [4, 18, 21, 29], "russian": [4, 18, 19, 21, 29], "sinhala": [4, 21, 29], "spanish": [4, 9, 18, 21, 29], "swedish": [4, 18, 21, 29], "tagalog": [4, 21, 29], "tamil": [4, 21, 29], "tatar": [4, 21, 29], "telugu": [4, 21, 29], "thai": [4, 21, 29], "turkish": [4, 18, 21, 29, 32], "ukrainian": [4, 21, 29], "urdu": [4, 19, 21, 29], "vietnames": [4, 19, 21, 29], "wordfreq": 4, "number_list": 4, "phrase_len": [4, 26, 27, 29], "text_list": [4, 8, 9, 26, 27], "sentenc": [4, 9, 26], "exclud": [4, 11, 13, 18, 20, 23, 25, 28, 29], "follow_link": [4, 6, 14, 20, 29], "d": [4, 7, 9, 10, 14, 18, 26], "allowed_domain": [4, 6, 20], "param": 4, "exclude_url_param": [4, 20, 29], "include_url_param": [4, 20, 29], "exclude_url_regex": [4, 20, 29], "include_url_regex": [4, 20, 29], "css_selector": [4, 20, 29], "xpath_selector": [4, 20, 29], "encount": [4, 14, 25], "parmet": [4, 20], "rais": [4, 20, 29], "dictionari": [4, 6, 8, 9, 10, 20, 21, 29], "map": [4, 7, 9, 12, 13, 20, 23, 32], "requir": [4, 6, 14, 18, 19, 20, 23, 24, 28, 29], "content": [4, 6, 7, 10, 12, 17, 18, 19, 20, 21, 25, 26, 28, 29, 31], "add": [4, 6, 10, 14, 20, 23, 26], "over": [4, 6, 10, 19, 20, 23, 26, 29], "170": [4, 10, 20], "kind": [4, 10, 20, 29, 32], "pleas": [4, 10, 12, 18, 20, 23, 26, 28], "refer": [4, 7, 10, 12, 14, 18, 20, 23, 25], "doc": [4, 7, 12, 23], "scrapi": [4, 6, 10, 20, 32], "org": [4, 7, 8, 10, 12], "en": [4, 6, 7, 10, 12, 19, 20, 23, 25, 28], "topic": [4, 18, 19, 23, 25, 28, 29], "html": [4, 6, 7, 10, 14, 17, 19, 20, 23, 25, 28, 29], "home": [4, 17, 28], "examl": 4, "example_output": 4, "url_1": [4, 11, 25], "url_2": [4, 11, 25], "url_3": [4, 11], "OR": [4, 18, 23, 28, 30], "process": [4, 6, 7, 10, 14, 16, 19, 25], "000": [4, 18, 23, 27], "closespider_pagecount": [4, 6, 20], "1000": [4, 18, 28], "master": [6, 10, 20], "basic": [6, 10, 13, 16, 19, 25, 32], "probabl": [6, 7, 20, 32], "achiev": [6, 27, 32], "better": [6, 7, 19, 20, 23, 25, 29, 32], "These": [6, 14, 20, 26], "some": [6, 7, 8, 9, 10, 11, 14, 16, 17, 18, 19, 20, 23, 25, 26, 28, 29, 32], "kei": [6, 8, 9, 12, 18, 20, 21, 23, 25, 26, 28, 29], "indic": [6, 10, 17, 18, 20, 23, 25, 28, 29], "simpli": [6, 7, 13, 14, 16, 17, 19, 20, 26], "done": [6, 7, 10, 13, 14, 18, 19, 25, 26, 32], "page_1": 6, "page_2": 6, "page_3": 6, "page_4": 6, "example_crawl_1": 6, "goe": [6, 19], "through": [6, 10, 11, 14, 17, 18, 19, 20, 23, 25, 26, 27, 28, 32], "discov": [6, 14, 20, 25, 32], "find": [6, 8, 9, 18, 20, 23, 25, 26, 28, 32], "exmapl": [6, 7, 10, 20, 25], "won": [6, 9, 26, 32], "solut": [6, 7], "therefor": [6, 18, 26, 28], "origin": [6, 11, 14, 18, 20, 25, 29], "commun": [6, 25, 32], "It": [6, 9, 10, 12, 13, 14, 16, 18, 19, 20, 23, 25, 26, 28, 32], "usual": [6, 12, 16, 23, 25, 26], "check": [6, 7, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 26, 32], "except": [6, 23, 25, 26], "stat": [6, 8, 9, 29], "etc": [6, 7, 8, 9, 12, 13, 14, 15, 18, 19, 20, 21, 25, 26, 27, 29, 32], "pass": [6, 12, 17, 18, 19, 20, 23, 32], "cutom_set": 6, "practic": [6, 14, 16, 20, 32], "give": [6, 7, 9, 10, 13, 16, 18, 20, 23, 25, 26], "extens": [6, 14, 17, 18, 25, 29], "easier": [6, 9, 13, 20, 23, 25, 29, 30, 32], "retreiv": [6, 19, 29], "website_name_crawl_1": 6, "work": [6, 7, 9, 12, 13, 14, 17, 20, 23, 26, 28, 29, 32], "website_name_crawl_2": 6, "There": [6, 7, 9, 10, 14, 18, 19, 20, 23, 25, 29, 32], "few": [6, 7, 9, 10, 11, 14, 19, 20, 25, 26, 32], "trigger": 6, "thei": [6, 7, 9, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "mostli": [6, 7, 20, 25, 26, 32], "closespider_errorcount": [6, 20], "wait": [6, 20], "hour": [6, 23], "finish": 6, "had": [6, 8, 9, 26], "investig": 6, "issu": [6, 7, 29, 32], "closespider_itemcount": [6, 20], "anyth": [6, 23, 26, 29], "item": [6, 7, 11, 13, 18, 20, 23, 25, 28, 29], "h1": [6, 7, 20], "titl": [6, 7, 9, 13, 18, 20, 25, 26, 28, 29, 32], "meta_desc": [6, 7, 20], "been": [6, 19, 23, 26, 28], "exploratori": [6, 20], "techniqu": [6, 26, 32], "thousand": [6, 8, 10, 16, 26, 32], "idea": [6, 7, 9, 13, 20, 26, 32], "mind": [6, 8, 12, 20, 23, 32], "500": [6, 7, 12, 17, 18, 23, 26, 28], "robotstxt_obei": [6, 10, 11], "under": [6, 10, 11, 12, 14, 17, 26, 29], "user_ag": [6, 10, 11, 14, 17, 20], "found": [6, 7, 11, 14, 18, 19, 23], "current": [6, 7, 14, 19, 20, 23, 28, 29, 32], "your_user_ag": 6, "high": [6, 12, 18, 19, 20, 28], "sensit": [6, 13, 23], "autom": [6, 10, 17, 32], "quickli": 6, "block": [6, 14, 17, 19, 20, 29, 32], "ban": 6, "polit": [6, 23, 25, 28], "kill": 6, "concurrent_item": 6, "100": [6, 7, 14, 18, 20, 23, 26], "concurrent_request": 6, "16": [6, 9, 11, 14, 17, 19], "concurrent_requests_per_domain": [6, 20], "8": [6, 7, 8, 9, 10, 11, 12, 14, 16, 17, 19, 20, 23, 26, 30], "concurrent_requests_per_ip": 6, "download_delai": [6, 20], "interv": [6, 32], "befor": [6, 9, 18, 20, 23, 26, 28], "consecut": [6, 20], "400": [6, 17, 26], "75": [6, 7, 8, 9, 14, 16, 17, 20], "depth_limit": [6, 20], "level": [6, 14, 18, 20, 23, 25, 28, 32], "2": [6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 30], "initi": [6, 12, 29], "reason": [6, 9, 10, 14, 17, 20, 26, 29, 32], "why": [6, 7, 14, 17, 26], "mainli": [6, 11, 13, 16, 17, 26, 27], "updat": [6, 19, 29], "site": [6, 7, 13, 17, 18, 19, 20, 25, 26, 28, 29, 32], "alreadi": [6, 26], "big": [6, 13, 14, 25], "hurri": 6, "across": [6, 7, 8, 12, 18, 19, 25, 26, 32], "dai": [6, 9, 18, 23, 26, 28], "emerg": 6, "measur": [6, 18, 28, 29], "connect": [6, 7, 17, 23, 29, 32], "batteri": 6, "left": [6, 9, 28], "off": [6, 18, 26], "extrem": [6, 10, 13, 20, 32], "simpl": [6, 7, 8, 9, 10, 16, 19, 20, 23, 26, 32], "folder": [6, 11], "rerun": 6, "worri": [6, 17], "jobdir": 6, "abov": [6, 11, 12, 20, 23, 25, 26, 29], "happen": [6, 14, 25], "accid": 6, "close": 6, "comput": [6, 14, 20], "manual": 6, "ctrl": 6, "command": [6, 14, 16, 17, 29, 30], "again": [6, 14, 19, 20, 25, 26], "wa": [6, 7, 8, 9, 11, 14, 17, 19, 20, 23, 26, 29], "manag": [6, 7, 10, 18, 23, 28, 32], "But": [6, 20, 26], "doesn": [6, 13, 14, 20, 26], "duplic": [6, 16, 18, 20, 23], "step": [6, 12, 27, 32], "3rd": 6, "parti": [6, 23], "packag": [6, 20, 21, 29, 30, 31, 32], "rotat": 6, "retri": 6, "downloader_middlewar": 6, "rotating_proxy_list_path": 6, "usernam": [6, 28], "password": 6, "ipaddress": 6, "port": [6, 25], "random": [6, 17], "user123": 6, "password123": 6, "12": [6, 7, 9, 11, 14, 17, 19, 20, 23, 30], "34": [6, 12], "56": [6, 13, 14, 16, 17, 19], "78": [6, 17], "1111": 6, "1112": 6, "1113": 6, "1114": 6, "Then": [6, 20, 23], "rotating_proxi": 6, "middlewar": [6, 14], "rotatingproxymiddlewar": 6, "610": 6, "bandetectionmiddlewar": 6, "620": 6, "read": [6, 7, 9, 11, 14, 25, 32], "normal": [6, 14, 21], "being": [6, 18, 19, 20, 23, 25, 26, 28, 29, 32], "crawldf": [6, 7], "pd": [6, 7, 9, 10, 14, 17, 19, 20, 25, 26, 29], "read_json": [6, 7, 10, 17, 20], "filter": [6, 7, 14, 18, 19, 23, 25, 28, 29], "head": [6, 7, 8, 9, 10, 13, 14, 17, 19, 20, 26, 29, 30], "_rotating_proxi": 6, "request_headers_proxi": 6, "author": [6, 18, 20, 23, 25, 28], "proxy_retry_tim": 6, "123": [6, 9], "456": [6, 9], "789": [6, 9], "101": [6, 14], "8893": 6, "b3vzy214dhg6odlld29rmgrsdfgt": 6, "nan": [6, 7, 10, 12, 14, 19, 20, 25, 29], "8894": 6, "8895": 6, "8896": 6, "4": [6, 7, 8, 9, 11, 12, 13, 14, 16, 17, 19, 20, 23, 25, 26, 28, 30], "8897": 6, "easi": [6, 10, 13, 19, 25, 29, 32], "default_request_head": [6, 20, 29], "accept": [6, 10, 18, 20, 28, 29], "encod": [6, 10, 14, 20, 23, 24, 25, 29], "gzip": [6, 10, 20], "deflat": [6, 10, 20], "actual": [6, 12, 16, 19, 20, 25, 26, 28], "were": [6, 7, 14, 18, 19, 23, 26, 28, 29], "request_headers_": [6, 7, 20], "request_headers_accept": [6, 10, 20], "request_headers_us": [6, 10, 20], "suggest": [6, 25, 28], "tag": [6, 7, 10, 17, 18, 19, 20, 28, 29, 32], "meta": [6, 10, 18, 20, 23, 25], "attribut": [6, 7, 18, 20, 23, 26, 29], "viewport": [6, 7, 20, 29], "charset": [6, 7, 10, 20, 29], "h2": [6, 7, 20, 29], "h3": [6, 7, 10, 20], "h4": 6, "h5": 6, "h6": [6, 7, 20], "canon": [6, 7, 10, 20, 29], "rel": [6, 7, 12, 16, 17, 20, 25, 29, 32], "href": [6, 20, 29], "alt_href": [6, 7, 20], "altern": [6, 17, 20, 29], "alt_hreflang": [6, 7, 20], "hreflang": [6, 10, 29], "og_prop": 6, "properti": [6, 7, 18, 20, 28], "og": [6, 7, 20, 29], "who": [6, 9, 17, 18, 19, 23, 26, 28], "opengraph": [6, 10], "og_cont": 6, "twtr_name": 6, "twtr_content": 6, "iframe_src": 6, "ifram": 6, "src": [6, 7, 11, 20, 29], "gtm_script": 6, "script": [6, 10], "googletagmanag": 6, "gtm": 6, "j": [6, 14], "id": [6, 10, 12, 18, 20, 23, 28], "gtm_noscript": 6, "link_rel_rel": 6, "link_rel_href": 6, "link_rel_stylesheet": 6, "stylesheet": 6, "css_link": 6, "nav_links_text": [6, 20], "nav": [6, 20, 29], "anchor": [6, 20], "nav_links_href": 6, "header_links_text": [6, 20], "header_links_href": 6, "footer_links_text": [6, 20], "footer": [6, 20, 29], "footer_links_href": 6, "js_script_src": 6, "javascript": 6, "js_script_text": 6, "script_src": 6, "canonical_par": 6, "parent": [6, 28], "collect": [6, 15, 17, 18, 23, 28, 29], "popular": [6, 14, 23, 28, 32], "amazon": [6, 17, 19, 23], "4k": [6, 9], "fire": [6, 9, 26], "tv": [6, 19, 28], "mozilla": [6, 14, 20], "linux": [6, 14, 29], "android": [6, 14, 19], "aft": 6, "build": [6, 8, 10, 14, 32], "lmy47o": 6, "applewebkit": [6, 14], "537": [6, 14, 17], "36": [6, 14, 17, 20], "khtml": [6, 14], "like": [6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 19, 20, 23, 26, 28, 29, 32], "gecko": [6, 14], "version": [6, 10, 14, 20, 29], "chrome": [6, 12, 14], "41": [6, 19], "99900": 6, "2250": 6, "0242": 6, "safari": [6, 14], "aftwmst22": 6, "9": [6, 7, 8, 9, 10, 12, 14, 17, 19, 20, 26, 30], "ps7233": 6, "wv": 6, "88": 6, "4324": 6, "152": [6, 14], "mobil": [6, 14], "kindl": 6, "u": [6, 7, 13, 14, 18, 19, 20, 26, 28], "528": 6, "screen": [6, 23], "600x800": 6, "x11": [6, 14], "armv7l": 6, "531": 6, "533": [6, 19], "hdx": 6, "7": [6, 7, 8, 9, 12, 14, 16, 17, 19, 23, 26, 30], "kfthwi": 6, "ktu84m": 6, "silk": 6, "47": [6, 14], "79": [6, 14, 17], "2526": 6, "80": [6, 14, 16, 26], "appl": [6, 17, 26], "4th": 6, "gen": 6, "appletv5": 6, "5th": 6, "appletv6": 6, "11": [6, 7, 8, 9, 10, 14, 17, 18, 19, 20, 30], "6th": 6, "appletv11": 6, "iphon": 6, "iphone12": 6, "cpu": [6, 14], "o": [6, 15], "13_0": 6, "mac": 6, "x": [6, 12, 13, 14, 18, 19, 20, 32], "602": 6, "15e148": 6, "iphone13": 6, "14_0": 6, "pro": 6, "iphone14": 6, "15_0": 6, "19a346": 6, "6": [6, 7, 8, 9, 11, 12, 14, 16, 17, 19, 26, 30], "iphone7c2": 6, "1202": 6, "466": 6, "420": 6, "1a543": 6, "419": 6, "iphone9": 6, "10_0_1": 6, "14a403": 6, "plu": 6, "11_0": 6, "604": 6, "15a5341f": 6, "38": [6, 14, 20], "15a5370a": 6, "se": 6, "15_4": 6, "19e241": 6, "15a372": 6, "xr": 6, "12_0": 6, "605": 6, "crio": 6, "69": [6, 14, 20], "3497": 6, "105": [6, 7, 14], "firefox": 6, "fxio": 6, "2b11866": 6, "16a366": 6, "bing": [6, 12], "bot": [6, 14, 16], "compat": [6, 14, 23], "bingbot": [6, 14, 17], "htm": [6, 14], "laptop": 6, "browser": [6, 12], "chromebook": 6, "cro": 6, "x86_64": [6, 14], "8172": 6, "45": [6, 14, 18], "51": [6, 14, 16, 17], "2704": 6, "64": [6, 9, 17], "chromecast": 6, "crkei": 6, "16041": 6, "31": [6, 14, 16, 19, 30], "1650": 6, "adt": 6, "dalvik": 6, "ptt5": 6, "181126": 6, "002": 6, "nexu": [6, 14], "player": [6, 28], "mmb29t": 6, "pixel": [6, 11], "nmf26f": 6, "54": [6, 12, 19], "2840": 6, "85": 6, "opd1": 6, "170811": 6, "59": [6, 13], "3071": 6, "qd1a": 6, "190821": 6, "014": 6, "c2": 6, "3904": 6, "108": [6, 17], "rq3a": 6, "210805": 6, "001": 6, "a1": 6, "92": [6, 14], "4515": 6, "159": [6, 9], "sd1a": 6, "210817": 6, "023": 6, "94": [6, 14], "4606": 6, "71": 6, "nrd90m": 6, "52": [6, 17, 19], "2743": 6, "98": [6, 14], "googlebot": [6, 14, 16, 17], "htc": 6, "desir": [6, 7, 14, 23], "21": [6, 9, 11, 14, 17, 18, 19, 20, 23, 24, 30], "5g": 6, "4183": 6, "127": 6, "m9": 6, "mra58k": 6, "x10": 6, "61": 6, "3163": 6, "u20": 6, "wildfir": 6, "74": [6, 16], "3729": 6, "136": 6, "lg": 6, "pad": 6, "v410": 6, "v41020c": 6, "lrx22g": 6, "1847": 6, "118": [6, 14], "lenovo": 6, "yoga": 6, "tab": 6, "yt": 6, "j706x": 6, "96": [6, 16], "4664": 6, "pc": 6, "ubuntu": [6, 10], "rv": 6, "20100101": 6, "macintosh": 6, "intel": 6, "10_11_2": 6, "601": [6, 9], "microsoft": 6, "lumia": 6, "550": 6, "window": [6, 14, 20], "phone": [6, 9], "rm": 6, "1127_16056": 6, "42": [6, 14], "2311": 6, "135": [6, 14], "edg": 6, "10536": 6, "650": 6, "1152": 6, "116": 6, "15254": 6, "950": 6, "46": [6, 14, 19], "2486": 6, "1058": 6, "minix": 6, "neo": 6, "x5": 6, "he": [6, 9, 19, 26], "il": 6, "116a": 6, "jdq39": 6, "534": 6, "6p": 6, "mmb29p": [6, 14], "83": [6, 14], "nintendo": 6, "3d": [6, 18, 28], "7412": 6, "eu": 6, "switch": 6, "wifiwebauthapplet": 6, "nf": 6, "nintendobrows": 6, "13343": 6, "wii": 6, "wiiu": 6, "536": [6, 17], "nx": 6, "11264": 6, "nvidia": 6, "shield": 6, "tablet": [6, 19], "k1": 6, "55": [6, 12, 13, 19], "2883": 6, "91": [6, 16], "playstat": 6, "73": [6, 14], "vita": 6, "roku": 6, "ultra": [6, 19], "roku4640x": 6, "dvp": 6, "70": [6, 14, 20], "297": 6, "70e04154a": 6, "samsung": [6, 19], "galaxi": [6, 19], "s10": 6, "sm": [6, 25], "g973u": 6, "ppr1": 6, "180610": 6, "011": 6, "s20": 6, "g980f": 6, "qp1a": 6, "190711": 6, "020": 6, "s21": 6, "g996u": 6, "s22": [6, 19], "s906n": 6, "3987": 6, "119": [6, 14], "s6": 6, "g920v": 6, "mmb29k": 6, "g928x": 6, "lmy47x": 6, "s7": 6, "g930vc": 6, "58": [6, 13, 14, 17], "3029": 6, "g935": 6, "s8": 6, "g892a": 6, "60": [6, 14, 16, 17], "3112": 6, "107": 6, "s9": 6, "g960f": 6, "r16nw": 6, "62": 6, "3202": 6, "84": 6, "t550": 6, "samsungbrows": 6, "2125": 6, "102": 6, "s3": 6, "t827r4": 6, "x906c": 6, "soni": 6, "xperia": 6, "j8110": 6, "552": 6, "3578": 6, "99": [6, 7, 9, 19, 26], "xz": 6, "g8231": 6, "219": 6, "z4": 6, "sgp771": 6, "32": [6, 9, 10, 14, 16, 17, 19, 20], "253": 6, "z5": 6, "e6653": 6, "nt": [6, 14, 20], "win64": [6, 14], "x64": [6, 14], "246": 6, "wow64": 6, "111": 6, "xbox": 6, "10586": 6, "xbox_one_": 6, "14": [6, 7, 9, 10, 11, 12, 14, 17, 19, 30], "14393": 6, "seri": [6, 19, 25], "48": [6, 14, 17, 19], "2564": 6, "82": [6, 16, 20], "02": [6, 10, 17, 19, 30], "yahoo": [6, 23], "slurp": [6, 17], "ysearch": 6, "bunch": [7, 20], "gain": [7, 19, 30], "undersand": 7, "technic": [7, 12, 32], "readi": [7, 13, 23], "made": [7, 9, 16, 23, 26], "anayz": 7, "independ": [7, 28, 32], "size": [7, 10, 14, 17, 18, 19, 20, 23, 25, 29], "cours": [7, 10, 13, 26, 32], "togeth": [7, 8, 9, 14, 17, 23, 25, 26, 32], "put": [7, 13, 14, 19, 20, 26], "context": [7, 9, 12, 15], "thought": [7, 12], "describ": [7, 13, 26], "aspect": 7, "yet": [7, 13, 20, 25, 26], "spread": [7, 32], "everi": [7, 14, 17, 19, 20, 23, 26, 28, 32], "turn": [7, 18], "alt": [7, 20, 23, 29], "width": [7, 11, 20, 28, 29], "unpack": [7, 19], "point": [7, 8, 12, 18, 25, 28], "tidi": 7, "form": [7, 9, 12, 26], "distribut": 7, "summar": [7, 8, 9, 14], "panda": [7, 8, 9, 10, 12, 13, 14, 17, 18, 19, 20, 23, 25, 26, 29, 32], "img_df": 7, "crawlyt": [7, 29], "img_src": [7, 20], "img_alt": [7, 20], "img_load": 7, "img_siz": 7, "img_decod": 7, "img_width": 7, "img_height": 7, "img_bord": 7, "nytim": [7, 19], "vi": 7, "asset": [7, 11], "static": [7, 11, 14, 17], "icon": [7, 18], "morning_144x144": 7, "b12a6923b6ad9102b766352261b1a847": 7, "webp": 7, "morn": [7, 9, 26], "logo": [7, 11], "upshot_144x144": 7, "0b1553ff703bbd07ac8fe73e6d215888": 7, "upshot": 7, "static01": [7, 19], "nyt": [7, 19], "2017": [7, 11, 19, 28], "01": [7, 17, 18, 19, 28, 30], "29": [7, 9, 10, 14, 19, 30], "podcast": [7, 19], "daili": 7, "album": 7, "art": [7, 28], "square320": 7, "v5": 7, "jpg": [7, 11, 19], "qualiti": [7, 10, 11, 18], "auto": [7, 11], "disabl": [7, 18], "upscal": 7, "newslett": 7, "brief": 7, "europ": 7, "email": [7, 12, 24], "500px": 7, "australia": [7, 18], "australialett": 7, "interpret": 7, "sonl": 7, "theinterpret": 7, "section": [7, 17, 18, 28], "world": [7, 19], "middleeast": [7, 19], "2024": [7, 11, 30], "multimedia": [7, 19], "25israel": 7, "hbcz": 7, "thumbwid": 7, "min": [7, 14], "1024px": 7, "205px": 7, "150px": 7, "async": 7, "150": [7, 17], "hama": 7, "icj": 7, "explain": [7, 19, 32], "wjth": 7, "qatar": 7, "israel": 7, "ctbv": 7, "becaus": [7, 12, 14, 16, 17, 20, 23, 25, 26, 32], "particular": [7, 17, 18, 19, 28], "repres": [7, 9, 12, 14, 18, 19, 23, 26], "own": [7, 9, 13, 14, 18, 23, 25, 26, 28, 32], "row": [7, 13, 14, 17, 18, 19, 26], "seen": [7, 14], "repeat": [7, 9, 25, 29], "interest": [7, 9, 10, 11, 12, 13, 14, 17, 19, 20, 26, 30], "variou": [7, 8, 9, 10, 19, 26, 29, 32], "notna": 7, "averag": 7, "mean": [7, 9, 10, 12, 13, 14, 17, 18, 20, 23, 25, 26, 32], "sort_valu": [7, 26], "ascend": [7, 26], "to_fram": 7, "round": 7, "86": 7, "img_srcset": 7, "almost": [7, 14, 26], "height": [7, 11, 20, 28, 29], "immedi": [7, 17, 23], "estim": 7, "plan": [7, 14], "accordingli": 7, "webpag": [7, 18, 28], "understand": [7, 12, 13, 17, 18, 19, 23, 25, 32], "intern": [7, 10, 18, 19, 20, 23], "extern": [7, 10, 14], "summari": [7, 8, 9, 11, 29, 32], "link_df": 7, "internal_url_regex": 7, "nofollow": [7, 20, 29], "skip": [7, 20, 29], "dfp": 7, "advertis": 7, "middl": [7, 29], "east": 7, "suppli": [7, 11, 16, 18, 20, 29], "defin": [7, 12, 14, 18, 26, 28, 29], "realli": [7, 10, 13, 14, 20, 26], "could": [7, 9, 10, 13, 18, 26, 28, 29], "even": [7, 14, 17, 18, 19, 20, 23, 26, 28, 29], "consid": [7, 9, 23, 28, 32], "part": [7, 9, 13, 16, 18, 21, 25, 26, 27, 28, 32], "thu": [7, 10, 20, 26], "frequent": [7, 26], "inform": [7, 8, 10, 12, 14, 16, 18, 19, 20, 23, 25, 26, 28, 29], "present": [7, 18, 23, 29], "redirect_df": 7, "download_lat": [7, 10, 20], "redirect_tim": [7, 10, 20], "301": [7, 20], "220263": 7, "200": [7, 10, 14, 20, 23, 26], "privaci": 7, "polici": [7, 10], "079844": 7, "hc": 7, "10940941449492": 7, "403": 7, "0630789": 7, "13537530305428": 7, "218": 7, "spotlight": 7, "project": [7, 10, 12, 18, 20, 28, 32], "protect": 7, "852014": 7, "225": [7, 14], "regul": 7, "732559": 7, "310": 7, "sahil": 7, "chinoi": 7, "435062": 7, "intermedi": 7, "well": [7, 9, 10, 13, 14, 16, 17, 18, 19, 20, 23, 25, 26, 27, 28, 29, 32], "latenc": [7, 10], "back": [7, 14, 23, 25, 26, 32], "memori": [7, 14, 20, 25], "imposs": [7, 25], "availablel": 7, "subset": [7, 19], "jsonlin": [7, 10, 20], "jl_subset": [7, 29], "massiv": [7, 14, 16, 25], "reduc": 7, "consumpt": 7, "small": [7, 8, 17, 18, 28], "delet": [7, 29], "old": 7, "crawl_subset": 7, "col1": 7, "col2": 7, "column_regex": 7, "img_": [7, 20], "availab": 7, "img": [7, 11, 20, 29], "jsonld_": [7, 20], "json": [7, 10, 14, 18, 20, 23, 29], "ld": [7, 10, 20, 29], "resp_headers_": [7, 20], "redirect_": [7, 20], "links_": 7, "characterist": 7, "either": [7, 9, 13, 18, 19, 23, 26, 28, 29, 32], "depend": [7, 9, 10, 12, 13, 19], "deal": [7, 32], "highli": [7, 18, 20, 28, 32], "perform": [7, 14, 16, 18, 19, 20, 28], "jl_to_parquet": [7, 29], "much": [7, 19, 20, 26, 29], "smaller": [7, 19, 28, 29], "disk": [7, 14], "power": [7, 9, 10, 20, 32], "effici": [7, 10, 14, 16, 20, 25], "read_parquet": [7, 14, 25], "pydata": 7, "_": [7, 19], "document": [7, 10, 12, 18, 20, 23, 26, 29], "advantag": [7, 14], "select": [7, 11, 14, 18, 20, 23, 28], "parquet_column": [7, 29], "nyt_crawl": 7, "value_count": [7, 14, 19], "215": 7, "doubl": [7, 29], "22": [7, 14, 17, 19, 30], "int64": [7, 14, 17, 19, 29], "struct": 7, "contenturl": [7, 12], "credittext": 7, "caption": [7, 18, 28, 29], "timestamp": [7, 14], "img_summari": 7, "image_df": 7, "chunksiz": 7, "chunk": 7, "jsonld": 7, "df_subset": 7, "jl_filepath": 7, "parquet_filepath": 7, "exist": [7, 8, 9, 13, 14, 23, 29], "parquet_fileapth": 7, "pather": 7, "identifi": [7, 18, 20, 23, 25, 28], "datatyp": 7, "columns_typ": 7, "chain": [7, 20], "inermedi": 7, "minu": [7, 20], "worth": 8, "helper": [8, 27], "aid": [8, 32], "emoji_entri": 8, "unicod": [8, 15], "textual": [8, 9], "v13": [8, 29], "public": [8, 10, 20, 23], "emoji_df": [8, 29], "extract_emoji": [8, 9, 29], "statist": [8, 9, 16, 28, 29, 32], "emoji_search": [8, 29, 32], "choic": 8, "emoji_raw": 8, "develop": [8, 12, 17, 18, 23, 28], "kaggl": [8, 32], "eliasdabba": 8, "whole": [8, 10, 20, 26], "databas": [8, 9, 14, 15, 29, 32], "vegetable_emoji": 8, "veget": 8, "codepoint": 8, "sub_group": 8, "1f951": 8, "fulli": [8, 9, 23], "qualifi": 8, "avocado": 8, "food": [8, 9, 28], "drink": [8, 9, 26], "1f346": 8, "eggplant": 8, "1f954": 8, "potato": 8, "1f955": 8, "carrot": 8, "1f33d": 8, "ear": 8, "corn": 8, "expect": [8, 10, 11, 20], "love_emoji": 8, "love": [8, 23, 26], "1f48c": 8, "smilei": [8, 9], "emot": [8, 9], "1f91f": 8, "gestur": 8, "peopl": [8, 9, 12, 23, 26, 32], "bodi": [8, 9, 10, 20, 23, 32], "hand": [8, 12, 13], "finger": 8, "partial": 8, "1f3fb": 8, "light": [8, 10], "skin": 8, "tone": 8, "1f3fc": 8, "medium": [8, 18, 24, 28], "1f3fd": 8, "1f3fe": 8, "dark": 8, "1f3ff": 8, "1f340": 8, "four": [8, 9, 17, 18, 20, 25, 26, 28], "leaf": 8, "clover": 8, "anim": [8, 9], "natur": [8, 9, 11], "plant": 8, "1f3e9": 8, "travel": [8, 9], "1f94a": 8, "box": [8, 17, 19, 28], "glove": 8, "activ": [8, 9, 12, 18, 19, 28], "sport": [8, 11, 19, 28], "1f9e4": 8, "object": [8, 9, 12, 14, 19, 20, 23, 28, 32], "cloth": 8, "1f1f8": 8, "1f1ee": 8, "flag": [8, 9], "slovenia": 8, "countri": [8, 18, 20, 25, 28, 29, 32], "social": [8, 9, 20, 21, 25, 26], "media": [8, 9, 11, 14, 20, 21, 23, 25, 26], "plai": [8, 12, 18, 19, 28], "around": [8, 9, 26], "sampl": [8, 12, 19, 20], "feel": [8, 9, 26, 29], "basketbal": [8, 28], "footbal": [8, 13, 19, 28, 32], "Not": [8, 23], "emoji_summari": [8, 9], "print": [8, 9, 14, 19, 21, 29], "entri": 8, "insensit": [8, 29], "dog": 8, "1f436": 8, "face": [8, 18], "mammal": 8, "1f415": 8, "1f9ae": 8, "guid": [8, 28], "200d": 8, "1f9ba": 8, "servic": [8, 12, 13, 18, 19], "1f32d": 8, "hot": [8, 9], "blue": [8, 9, 18, 25, 26, 28], "1f499": 8, "heart": 8, "1fad0": 8, "blueberri": 8, "fruit": 8, "1f4d8": 8, "book": [8, 26], "paper": 8, "1f535": 8, "circl": 8, "symbol": [8, 9, 29], "geometr": 8, "1f7e6": 8, "squar": 8, "1f537": 8, "diamond": 8, "1f539": 8, "ones": [8, 9, 10, 14, 16, 19, 20, 26], "post": [8, 9, 21, 23, 25, 26, 30, 32], "am": [8, 26], "grin": 8, "cat": 8, "hello": [8, 9, 17, 20], "dict_kei": [8, 9, 21], "emoji_text": [8, 9], "emoji_flat": [8, 9], "emoji_flat_text": [8, 9], "emoji_count": [8, 9], "emoji_freq": [8, 9], "top_emoji": [8, 9], "top_emoji_text": [8, 9], "top_emoji_group": [8, 9], "top_emoji_sub_group": [8, 9], "yellow": [8, 18], "flat": [8, 9], "number_of_emoji": 8, "smile": 8, "num_post": [8, 9], "num_emoji": [8, 9], "emoji_per_post": [8, 9], "unique_emoji": [8, 9], "infer": 9, "contrast": 9, "compani": [9, 12, 17], "brand": [9, 12, 14, 19], "extract_": [9, 29, 32], "fucntion": 9, "extract_curr": [9, 29], "surround": [9, 29], "abbrevi": 9, "usd": 9, "eur": 9, "jpy": 9, "extract_exclam": [9, 29], "excalam": 9, "mark": [9, 23, 25, 26, 27, 29], "extract_hashtag": [9, 29], "extract_intense_word": [9, 29], "intens": [9, 29], "neg": [9, 13, 23], "looooooovvvve": 9, "extract_ment": [9, 29], "network": [9, 25], "extract_numb": [9, 29], "extract_quest": [9, 29], "extract_url": [9, 29], "extract_word": [9, 29], "arbitrari": [9, 23, 29], "rest": [9, 12, 17], "restaur": 9, "along": [9, 23, 26], "recommend": [9, 14, 20, 23], "hashtag_summari": 9, "hashtags_flat": 9, "hashtag_count": 9, "hashtag_freq": 9, "top_hashtag": 9, "num_hashtag": 9, "hashtags_per_post": 9, "unique_hashtag": 9, "proper": [9, 32], "dataset": [9, 20, 25, 26, 32], "tweet": [9, 21, 23, 26, 32], "read_csv": [9, 26], "csv": [9, 11, 14, 26], "shape": [9, 14, 19], "tweet_text": [9, 26], "followers_count": [9, 26], "aerialmagzc": [9, 26], "penguinnyyyyi": [9, 26], "afraid": [9, 26], "real": [9, 23, 26], "157": [9, 26], "vibe": [9, 26], "offic": [9, 26], "metallica": [9, 26], "boss": [9, 26], "coffe": [9, 26], "break": [9, 19, 23, 26, 29, 32], "theoffic": [9, 26], "co": [9, 26], "u5vdyevvf": [9, 26], "4687": [9, 26], "ann": [9, 26], "sai": [9, 13, 18, 20, 26, 32], "she": [9, 26], "sugar": [9, 26], "hfubv4v3ai": [9, 26], "104": [9, 16, 20, 26], "venti": [9, 26], "ic": [9, 26, 28], "pump": [9, 26], "white": [9, 18, 26, 27], "mocha": [9, 26], "sweet": [9, 26], "cream": [9, 26], "caramel": [9, 26], "drizzl": [9, 26], "shout": [9, 26], "tiktok": [9, 26], "lol": [9, 26], "126": [9, 26], "never": [9, 26], "person": [9, 12, 18, 26], "until": [9, 23, 26], "kid": [9, 26], "cup": [9, 26], "life": [9, 26, 30], "saver": [9, 26], "zo0cnvuigj": [9, 26], "1595": [9, 26], "excit": [9, 26], "next": [9, 13, 20, 26, 28, 32], "chat": [9, 26], "re": [9, 23, 26], "john": [9, 26], "bradford": [9, 26], "lineup": [9, 26], "discuss": [9, 26], "redistrict": [9, 26], "area": [9, 18, 26, 28, 32], "rsvp": [9, 26], "r3ynjjjcug": [9, 26], "join": [9, 20, 26], "meet": [9, 18, 26, 28], "ho4kx7zz24": [9, 26], "kfpdr3hupi": [9, 26], "5004": [9, 26], "paid": [9, 26], "husband": [9, 26], "165": [9, 26], "nippli": [9, 26], "outsid": [9, 14, 18, 26, 28], "side": [9, 14, 26, 27], "sound": [9, 26], "blowjob": [9, 26], "front": [9, 26], "visit": [9, 26], "green": [9, 18, 26], "tea": [9, 26], "hahahahahahaha": [9, 26], "spend": [9, 23, 26, 32], "pamper": [9, 26], "hope": [9, 26, 32], "everyon": [9, 26], "tuesdai": [9, 26], "enjoi": [9, 26], "189": [9, 26], "marvinmilton2": [9, 26], "nearli": [9, 26], "choke": [9, 26], "1160": [9, 26], "2000": 9, "733": 9, "3665": 9, "572": 9, "mention_summari": 9, "mentions_flat": 9, "mention_count": 9, "mention_freq": 9, "top_ment": 9, "num_ment": 9, "1346": 9, "mentions_per_post": 9, "673": 9, "unique_ment": 9, "1132": 9, "zip": [9, 14, 29], "currency_summari": 9, "currency_symbol": 9, "currency_symbols_flat": 9, "currency_symbol_count": 9, "currency_symbol_freq": 9, "top_currency_symbol": 9, "currency_symbol_nam": 9, "surrounding_text": 9, "num_currency_symbol": 9, "37": [9, 18, 19, 28], "currency_symbols_per_post": 9, "0185": 9, "unique_currency_symbol": 9, "sym": 9, "number_summari": 9, "numbers_flat": 9, "number_count": 9, "number_freq": 9, "top_numb": 9, "num_numb": 9, "1727": 9, "numbers_per_post": 9, "8635": 9, "unique_numb": 9, "257": 9, "question_summari": 9, "question_mark": 9, "question_marks_flat": 9, "question_mark_count": 9, "question_mark_freq": 9, "top_question_mark": 9, "question_mark_nam": 9, "question_text": 9, "num_question_mark": 9, "321": [9, 12], "question_marks_per_post": 9, "1605": 9, "unique_question_mark": 9, "ckaiserjr": 9, "perry_ron": 9, "lilguyisback": 9, "okai": 9, "water": 9, "flavor": 9, "think": [9, 20, 25], "ll": [9, 17, 23, 26], "loos": 9, "mayb": [9, 11, 13, 17], "exclamation_summari": 9, "exclamation_mark": 9, "exclamation_marks_flat": 9, "exclamation_mark_count": 9, "exclamation_mark_freq": 9, "top_exclamation_mark": 9, "exclamation_mark_nam": 9, "exclamation_text": 9, "num_exclamation_mark": 9, "563": 9, "exclamation_marks_per_post": 9, "2815": 9, "unique_exclamation_mark": 9, "1149": 9, "5745": 9, "279": 9, "emoji_nam": 9, "72": [9, 14, 17, 28], "49": [9, 12, 14], "210": 9, "97": [9, 16], "67": [9, 20], "33": [9, 12, 17], "key_nam": 9, "kwarg": [9, 10, 11], "singular": 9, "straightforward": [9, 11, 20, 25], "left_char": 9, "right_char": 9, "dict": [9, 10, 11, 14, 20], "number_of_symbol": 9, "bitcoin": 9, "dollar": [9, 27], "pound": 9, "euro": 9, "odai": 9, "ound": 9, "6666666666666667": 9, "written": [9, 17, 18, 19], "said": [9, 17], "No": [9, 18], "6666666666666666": 9, "posts2": 9, "\u0645\u0631\u062d\u0628\u0627": 9, "\u0644\u0627": 9, "\u062a\u0630\u0647\u0628": 9, "hola": 9, "c\u00f3mo": 9, "est\u00e1": 9, "displai": [9, 14, 17, 18, 28], "opposit": 9, "due": [9, 14, 23, 26, 28], "rtl": 9, "invert": 9, "number_of_hashtag": 9, "min_rep": 9, "instanc": [9, 12, 29], "repetit": 9, "looooooveee": 9, "youuuuuuu": 9, "haaatttteee": 9, "youuuuuu": 9, "both": [9, 13, 18, 20, 23, 26, 27, 28, 32], "jenni": 9, "hi": [9, 19, 26], "number_of_ment": 9, "number_separ": 9, "333": 9, "444": 9, "555": 9, "number_of_numb": 9, "ask": [9, 18, 23, 29], "armenian": 9, "\u03c0\u03ce\u03c2": 9, "\u03b5\u03af\u03c3\u03b1\u03b9": 9, "\u0643\u064a\u0641": 9, "\u062d\u0627\u0644\u0643": 9, "did": [9, 19, 26], "notic": [9, 17], "correct": [9, 12, 29], "NOT": [9, 18, 28], "valid": [9, 18, 23, 24, 28], "b": [9, 10, 14], "url_summari": 9, "urls_flat": 9, "url_count": 9, "url_freq": 9, "top_url": 9, "top_domain": 9, "top_tld": 9, "number_of_url": 9, "num_url": 9, "urls_per_post": 9, "unique_url": 9, "words_to_extract": 9, "entire_words_onli": 9, "complet": [9, 13, 18, 23, 25, 26, 28], "words_to_find": 9, "rain": [9, 26], "snow": [9, 26], "noth": [9, 26], "word_summari": 9, "words_flat": 9, "word_count": 9, "word_freq": [9, 26], "top_word": 9, "num_word": 9, "words_per_post": 9, "unique_word": 9, "number_of_word": 9, "occurr": [9, 26], "occur": [9, 19, 20, 23, 26, 28, 29], "train": 9, "relat": [9, 18, 19, 20, 28, 32], "mini": 10, "known": [10, 20, 23, 26, 29], "hood": 10, "simplifi": [10, 18, 28], "interfac": [10, 18, 23, 29, 30, 32], "crawl_head": [10, 29], "assur": 10, "super": [10, 14], "fast": [10, 16, 29], "straight": 10, "forward": 10, "readthedoc": [10, 16, 20, 23], "io": [10, 16, 20, 23], "adver": [10, 14, 20], "dashboardom": 10, "povertydata": 10, "headers_df": 10, "crawl_tim": [10, 20], "download_timeout": [10, 20], "download_slot": [10, 20], "protocol": 10, "resp_headers_cont": [10, 20], "resp_headers_serv": [10, 20], "resp_headers_d": [10, 20], "resp_headers_vari": [10, 20], "redirect_ttl": [10, 20], "redirect_url": [10, 20], "redirect_reason": [10, 20], "resp_headers_x": [10, 20], "amz": 10, "resp_headers_last": [10, 20], "resp_headers_etag": 10, "serv": [10, 20], "backend": [10, 20], "rtd": [10, 20], "method": [10, 14, 18, 20, 23, 28, 29, 30], "resp_headers_referr": 10, "resp_headers_permiss": 10, "resp_headers_strict": [10, 20], "transport": [10, 19, 20], "secur": [10, 20], "resp_headers_cf": [10, 20], "cach": [10, 20], "resp_headers_ag": [10, 20], "resp_headers_expir": [10, 20], "resp_headers_cach": [10, 20], "resp_headers_expect": [10, 20], "ct": [10, 20], "rai": [10, 20], "resp_headers_alt": 10, "svc": 10, "resp_headers_via": 10, "2022": [10, 14, 17, 19, 30], "180": [10, 20], "0270483": 10, "nginx": [10, 20], "18": [10, 11, 14, 19, 30], "fri": 10, "feb": [10, 14], "gmt": [10, 20], "utf": [10, 14, 23], "applic": [10, 12, 14, 18, 20, 23, 28], "xhtml": [10, 20], "q": [10, 18, 23, 28], "rc2": 10, "06442": 10, "13270": 10, "0271282": 10, "cloudflar": [10, 20], "19": [10, 11, 17, 20, 30], "302": [10, 20], "rnkt7myjj7hcnsvbnzg9qdqizefftx9ytz3": 10, "gwnlj8m99yumucgdd6ytm": 10, "ibmo9hrztai": 10, "iyl50": 10, "ee0djx6z511tgx88": 10, "17": [10, 14, 16, 17, 19, 20, 30], "04": [10, 19, 30], "27": [10, 17, 19, 28, 30], "w": [10, 18, 19], "14c904a172315a4922f4d28948b916c2": 10, "proxito": [10, 20], "sendfil": [10, 20], "0710e93d610dd8c3": 10, "subdomain": [10, 20], "referr": [10, 24], "downgrad": 10, "cohort": [10, 32], "ag": [10, 20], "31536000": [10, 20], "includesubdomain": 10, "preload": 10, "1083": 10, "7200": 10, "604800": [10, 20], "report": [10, 16, 20, 25, 26, 29, 32], "uri": [10, 20], "cdn": 10, "cgi": 10, "beacon": 10, "6dba2aae6b424107": 10, "prg": 10, "443": 10, "ma": [10, 20], "86400": 10, "118614": 10, "26837": 10, "gunicorn": 10, "vegur": 10, "tip": 10, "mainten": 10, "task": [10, 13, 26, 27, 32], "continu": [10, 32], "hundr": [10, 20, 26], "period": 10, "basi": 10, "alert": 10, "ye": [10, 12], "ok": 10, "compon": [10, 14, 23, 25, 29], "metatag": 10, "direct": [10, 17], "noindex": 10, "byte": [10, 19, 20, 29], "With": [10, 19, 23, 26, 32], "consum": [10, 23, 25], "bandwidth": 10, "lookout": 10, "jpeg": 10, "png": [10, 11, 19, 20], "class": [10, 11, 20], "headersspid": 10, "arg": [10, 11], "autothrottle_en": [10, 11], "autothrottle_target_concurr": [10, 11], "httperror_allow_al": [10, 11], "errback": 10, "failur": 10, "headers_spid": 10, "start_request": [10, 11], "sine": 10, "speed": [10, 20, 29], "piec": 10, "expens": 10, "Being": 10, "abl": [10, 12, 17, 18, 32], "decis": [10, 19, 32], "optim": [10, 12], "dynam": [10, 20], "crawl_df": [10, 20], "experiment": [11, 29], "crawl_imag": [11, 29], "output_dir": 11, "min_width": 11, "minimum": 11, "avoid": [11, 25], "track": [11, 28, 29, 32], "navig": [11, 23], "elemenst": 11, "min_height": 11, "include_img_regex": 11, "Or": [11, 13, 20, 25], "economi": 11, "summarize_crawled_img": 11, "image_loc": [11, 19], "image_url": 11, "buzzfe": 11, "hannahdobro": 11, "dirti": 11, "littl": [11, 25, 30, 32], "industri": [11, 17, 18, 19, 32], "secret": 11, "tuh": 11, "user_imag": 11, "6r1oxxopc_larg": 11, "downsiz": 11, "120": 11, "03": [11, 17, 19, 30], "fce856744ed8": 11, "buzz": 11, "1303": 11, "1710779249": 11, "gif": 11, "base64": 11, "r0lgodlhaqabaiaaaaaaap": 11, "yh5baeaaaaalaaaaaabaaeaaaibraa7": 11, "245ecfa321e9": 11, "894": 11, "1710779358": 11, "chelseastewart": 11, "josh": 11, "peck": 11, "statement": 11, "drake": 11, "bell": 11, "abus": 11, "claim": [11, 16], "prod": 11, "v2": 11, "5590": 11, "1513102854": 11, "0_larg": 11, "ea6298160040": 11, "1093": 11, "1711048323": 11, "700": 11, "3a": 11, "2a": 11, "ivborw0kggoaaaansuheugaaafqaaaa7camaaadsf118aaaap1bmveuaaadigxpohbk5ewdfghi5fwi8grteghe7eqdmhr7": 11, "vymfddnm5hx334": 11, "py8fhdj5dlvvxnq6zjotzvbg1s8skwaaaacxrstlmav4eo10jnqa8ihfydaaabjuleqvryw93y64rcmbcg4czk5fszdav3f63bdaxfv4qm": 11, "axr96": 11, "wmnj0klhtpib9lcutya8k": 11, "f1rkxqh4kmipzviovwnszequmfjmvlb3": 11, "ysriv8zrqmwha1znqibuuv3jo3cn5fly3qimy2kitajb3": 11, "umlrxrgovgmqtj4hxc69an5hj9pcyyqzfxsavk58tjmntwgv24pw9kpe0fgbioklomczkngleuxlhyiimx": 11, "dt": 11, "xj8sxgocdz6ejcp7jspbqllibivmpewy7as1poez30pvqlaqvjrgeqtlfp1dblpyb0bdd": 11, "oyl2nhr7e34yujtjw6zmc3am": 11, "kxlspoodchrqwiwbxi85q6kc9pnehscmhj0vjgppuac3lwqo": 11, "ourl0aefg76m8izrt6eaaaaasuvork5cyii": 11, "josephlongo": 11, "celeb": 11, "wear": 11, "rewear": 11, "dress": 11, "2021": [11, 17, 19, 30], "06": [11, 19, 30], "a824550933a9": 11, "tomiobaro": 11, "2174": 11, "1622738336": 11, "41_larg": 11, "6634db63f453": 11, "576": [11, 12], "1710855734": 11, "cb8db05df7e7": 11, "1743": 11, "1710855790": 11, "taken": 11, "slug": [11, 19, 23, 25], "slash": 11, "locat": [11, 18, 19, 23, 25, 28, 32], "tabl": [11, 12, 13, 20, 32], "advimagespipelin": 11, "store_uri": 11, "download_func": 11, "imagespipelin": 11, "file_path": 11, "info": [11, 14, 18, 29], "store": [11, 14, 20], "imagespid": 11, "item_pipelin": 11, "image_spid": 11, "imgitem": 11, "start_url": 11, "behaviour": [11, 20], "image_dir": 11, "tha": 11, "rank": [12, 20, 29, 32], "zero": [12, 18, 20, 28], "comparison": [12, 16], "elig": 12, "score": [12, 26], "suitabl": 12, "critic": [12, 18], "clear": [12, 13, 23], "reliabl": 12, "view": [12, 18, 23, 26, 28], "send": [12, 14, 23, 32], "bill": [12, 18], "credenti": [12, 18, 23, 28], "shown": [12, 25], "below": [12, 18, 20, 26, 27, 28], "And": [12, 14], "your_google_developer_kei": 12, "knowledge_graph": [12, 29], "resultscor": 12, "203191": 12, "corpor": 12, "organ": 12, "technologi": [12, 28], "49462": 12, "19142": 12, "gmail": 12, "13251": 12, "7549": 12, "softwareappl": 12, "drive": 12, "6853": 12, "6543": 12, "4312": 12, "multin": 12, "conglomer": 12, "alphabet": [12, 18, 28], "inc": 12, "3395": 12, "1306": 12, "detaileddescript": 12, "articlebodi": 12, "licens": [12, 18, 28], "query_tim": [12, 29], "dtype": [12, 14, 17, 19], "203": [12, 14], "191": 12, "462": 12, "understood": 12, "fall": [12, 18, 23, 28], "inherit": 12, "everyth": [12, 23, 26], "hierarchi": 12, "belong": [12, 16, 19, 23], "funcion": 12, "manner": [12, 25], "aggreg": 12, "fr": [12, 18, 20], "evalu": 12, "3587": 12, "suchmaschinenoptimierung": 12, "lokal": 12, "252": 12, "suchmaschinenmarket": 12, "71756": 12, "5056": 12, "seop": 12, "3313": 12, "seoul": 12, "administrativearea": 12, "hauptstadt": 12, "von": 12, "s\u00fcdkorea": 12, "1509": 12, "yea": 12, "ji": 12, "schauspielerin": 12, "584": 12, "actriz": 12, "posicionamiento": 12, "buscador": 12, "35": [12, 14, 20], "316": 12, "jin": 12, "cantant": 12, "53": [12, 14], "8760": 12, "south": 12, "korea": 12, "1435": 12, "sulli": 12, "korean": [12, 18], "actress": 12, "prefix": [12, 29], "state": [12, 18, 19, 25, 28], "liter": 12, "iso": [12, 18, 23, 28], "639": [12, 18, 23, 28], "schema": 12, "enabl": [12, 18, 29], "substr": 12, "against": [12, 23, 26], "alias": 12, "jung": 12, "jungl": 12, "ho": 12, "kang": 12, "higher": [12, 18, 19, 28, 29], "chanc": 12, "kg_df": 12, "v1": 12, "properli": [13, 14, 18, 19, 25, 28, 29], "right": [13, 14, 18, 19, 25], "research": [13, 17, 32], "tediou": [13, 25], "shift": 13, "oppos": [13, 25, 29], "anywai": [13, 26], "phrase": [13, 18, 23, 26, 27, 29], "barcelona": 13, "guitar": 13, "rio": 13, "janeiro": 13, "trip": 13, "club": [13, 32], "verb": 13, "purchas": 13, "noun": 13, "intent": [13, 32], "price": [13, 20, 25], "offer": [13, 23], "clearli": [13, 25], "aren": 13, "tutori": [13, 20, 32], "certif": 13, "learn": [13, 19, 20, 26, 28, 32], "educ": 13, "fifteen": [13, 26], "twenti": [13, 17, 26], "segment": [13, 23], "target": [13, 28], "shouldn": [13, 14], "difficult": 13, "commerc": [13, 26], "focu": [13, 16, 32], "cheap": 13, "discount": 13, "luxuri": 13, "signifi": 13, "graphic": 13, "design": [13, 17, 32], "career": [13, 17], "vacanc": 13, "kw_gener": [13, 20, 29, 32], "possibl": [13, 14, 17, 18, 26, 32], "upload": [13, 18, 19, 23, 28], "kw_df": 13, "criterion": 13, "label": 13, "sem_campaign": 13, "625": [13, 16], "626": 13, "627": 13, "628": 13, "629": 13, "630": 13, "bottom": [13, 26, 32], "kw_broad": 13, "tutor": 13, "kw_exact": 13, "match_typ": 13, "capitalize_adgroup": [13, 29], "order_matt": 13, "frame": 13, "relev": [13, 18, 23, 24, 28], "final": [13, 19, 25, 28, 29], "keywords_df": 13, "tail": 13, "57": [13, 17, 19], "retain": [13, 25], "integr": [13, 32], "kw_modifi": 13, "kw_neg_broad": 13, "kw_neg_exact": 13, "kw_neg_phras": 13, "kw_phrase": 13, "event": [14, 18, 19, 25, 28], "complex": [14, 23], "ourselv": [14, 26], "pageview": [14, 26], "mai": [14, 17, 18, 20, 23, 26, 28], "session": [14, 18, 23], "characterisit": 14, "usuali": 14, "cater": 14, "rapid": 14, "tl": 14, "dr": 14, "access_log": 14, "log_error": 14, "logs_df": 14, "try": [14, 17, 18, 20, 26, 28, 32], "certainli": 14, "conform": 14, "weren": 14, "went": 14, "wrong": [14, 17], "fix": [14, 20, 29], "temporari": 14, "debug": [14, 18], "howev": [14, 18, 23, 26, 28], "singl": [14, 17, 18, 20, 23], "distinguish": [14, 20], "client": 14, "k": [14, 26], "extend": [14, 23], "effect": [14, 23, 25], "importantli": [14, 25, 32], "datetim": [14, 18, 19, 28, 29], "date": [14, 17, 18, 19, 20, 23, 28, 29, 32], "categor": [14, 25], "storag": [14, 19], "to_datetim": 14, "hostnam": [14, 16, 25], "ip": [14, 16, 18, 20, 29], "address": [14, 16, 20, 29], "reverse_dns_lookup": [14, 16, 29], "resourc": [14, 18, 20, 23, 28, 32], "url_to_df": [14, 19, 25, 29, 32], "famili": [14, 23], "oper": [14, 18, 19, 23, 28, 29], "system": [14, 25, 29], "non": [14, 18, 26, 27, 28, 29, 32], "sample_log": 14, "66": [14, 16, 20], "249": [14, 16], "00": [14, 17, 18, 19, 28], "0000": 14, "1095": 14, "5x": 14, "4758": 14, "109": 14, "237": 14, "103": 14, "39": [14, 16, 17, 19, 20], "env": 14, "404": [14, 20], "209": 14, "81": 14, "4044": 14, "129": 14, "223": 14, "214": 14, "23": [14, 17, 19, 20, 30], "2240": 14, "4430": 14, "68": [14, 17, 20], "77": [14, 17], "192": 14, "241": 14, "211": [14, 16], "176": 14, "login": [14, 17], "zgrab": 14, "stage": 14, "urlyt": 14, "520": [14, 19], "_dash": 14, "suit": 14, "dash": [14, 29], "dash_html_compon": 14, "v2_0_0m1638886228": 14, "154258": 14, "layout": [14, 28], "2547": 14, "ua_pars": 14, "user_agent_pars": 14, "max_column": 14, "adv_log": 14, "adv_error": 14, "y": [14, 18], "z": [14, 19], "host_df": [14, 16], "1210": 14, "745": 14, "sy": 14, "729": 14, "wall": 14, "ip_address": [14, 16, 20], "cum_count": [14, 16], "perc": [14, 16], "cum_perc": [14, 16], "aliaslist": [14, 16], "ipaddrlist": [14, 16], "143": 14, "244": 14, "132": 14, "426": 14, "0701004": 14, "errno": [14, 16], "unknown": [14, 16], "host": [14, 16, 18, 29], "146": [14, 17], "164": 14, "110": 14, "290": [14, 17], "716": 14, "0477209": 14, "117821": 14, "177": 14, "196": 14, "171": 14, "908": 14, "0315945": 14, "149416": 14, "ppp046177196171": 14, "hol": 14, "gr": 14, "addr": [14, 16], "arpa": [14, 16], "185": [14, 16], "173": 14, "182": 14, "1090": 14, "029949": 14, "179365": 14, "226": 14, "1261": 14, "0281389": 14, "207504": 14, "174": 14, "154": 14, "1415": 14, "0253415": 14, "232845": 14, "89": 14, "44": [14, 17], "1545": 14, "0213921": 14, "254237": 14, "ppp089047044105": 14, "1664": 14, "019582": 14, "273819": 14, "234": 14, "113": 14, "1777": 14, "0185947": 14, "292414": 14, "217": 14, "1858": 14, "0133289": 14, "305743": 14, "d9646265": 14, "ziggozakelijk": 14, "nl": 14, "163": 14, "243": [14, 16], "1937": 14, "0129998": 14, "318743": 14, "2014": [14, 19], "0126707": 14, "331414": 14, "194": [14, 16], "179": 14, "2074": 14, "00987329": 14, "341287": 14, "vmi660635": 14, "contaboserv": 14, "net": [14, 19], "137": 14, "2132": 14, "00954418": 14, "350831": 14, "2190": 14, "360375": 14, "tor": 14, "anonym": 14, "appliedprivaci": 14, "adress": [14, 16], "ip_host_dict": 14, "request_url_df": 14, "add_prefix": 14, "request_": 14, "request_url": 14, "request_schem": 14, "request_netloc": 14, "request_path": 14, "request_queri": 14, "request_frag": 14, "request_hostnam": 14, "request_port": 14, "request_dir_1": 14, "request_dir_2": 14, "request_dir_3": 14, "request_dir_4": 14, "request_dir_5": 14, "request_dir_6": 14, "request_dir_7": 14, "request_dir_8": 14, "request_dir_9": 14, "request_dir_10": 14, "request_dir_11": 14, "request_dir_12": 14, "request_dir_13": 14, "request_last_dir": 14, "request_query_index": 14, "request_query_": 14, "request_query_xdebug_session_start": 14, "request_query_funct": 14, "request_query_var": 14, "request_query_fil": 14, "request_query_url": 14, "request_query_a": 14, "request_query_cont": 14, "request_query_wt": 14, "request_query_act": 14, "request_query_usernam": 14, "request_query_psd": 14, "request_query_dn": 14, "request_query_step": 14, "request_query_cmd": 14, "request_query_lang": 14, "request_query_opt": 14, "request_query_folderid": 14, "request_query_input_fil": 14, "request_query_currentset": 14, "request_query_typ": 14, "request_query_next_fil": 14, "request_query_curpath": 14, "request_query_pag": 14, "request_query_id": 14, "request_query_img": 14, "request_query_panel": 14, "request_query_todo": 14, "request_query_cod": 14, "request_query_ref": 14, "request_query_scopenam": 14, "request_query_op": 14, "request_query_control": 14, "request_query_q": 14, "request_query_sb_categori": 14, "request_query_email": 14, "request_query_nam": 14, "request_query_abspath": 14, "request_query_fn": 14, "request_query_thumb": 14, "request_query_nocontinu": 14, "request_query_filepath": 14, "request_query_file_link": 14, "request_query_mypath": 14, "request_query_adapt": 14, "source_fil": 14, "request_query_aam": 14, "request_query_cpabc_calendar_upd": 14, "request_query_term": 14, "request_query_itemid": 14, "request_query_search_kei": 14, "request_query_short": 14, "request_query_titl": 14, "request_query_format": 14, "request_query_findcli": 14, "request_query_v": 14, "request_query_target": 14, "request_query__": 14, "request_query_albid": 14, "request_query_p": 14, "request_query_path": 14, "request_query_mod": 14, "request_query_libpath": 14, "request_query_srt": 14, "request_query_redirect": 14, "request_query_ord": 14, "request_query_item": 14, "request_query_gid": 14, "request_query_rid": 14, "request_query_servic": 14, "request_query_ag": 14, "request_query_typeid": 14, "request_query_dir": 14, "request_query_stockcodeintern": 14, "request_query_sit": 14, "request_query_posit": 14, "request_query_filenam": 14, "referer_url_df": 14, "referer_": 14, "referer_url": 14, "referer_schem": 14, "referer_netloc": 14, "referer_path": 14, "referer_queri": 14, "referer_frag": 14, "referer_hostnam": 14, "referer_port": 14, "referer_dir_1": 14, "referer_dir_2": 14, "referer_dir_3": 14, "referer_last_dir": 14, "ua_df": 14, "json_norm": [14, 29], "ua": 14, "ua_": 14, "ua_str": 14, "ua_famili": 14, "ua_major": 14, "ua_minor": 14, "ua_patch": 14, "ua_o": 14, "major": [14, 17, 19, 23, 32], "minor": [14, 29], "patch": 14, "patch_minor": 14, "ua_devic": 14, "smartphon": [14, 19], "desktop": 14, "concat": [14, 20, 29], "axi": 14, "to_parquet": 14, "adv_logs_fin": 14, "doen": 14, "load": [14, 20, 22, 29], "satisfi": 14, "top_bot": 14, "499": 14, "petalbot": 14, "ahrefsbot": 14, "yandexbot": 14, "linkedinbot": [14, 17], "baiduspid": [14, 17], "dotbot": 14, "twitterbot": [14, 17], "mj12bot": 14, "java": 14, "nutch": 14, "masscan": 14, "facebookbot": 14, "happi": [14, 23], "By": [14, 18, 20, 26, 28], "destin": [14, 25], "stdout": 14, "review": [14, 18, 19, 28], "altogeth": 14, "chose": 14, "crawllogs_to_df": [14, 29], "open": [14, 17, 19, 20, 29], "core": 14, "scraper": 14, "handler": 14, "method_to": 14, "redirect_to": 14, "method_from": 14, "redirect_from": 14, "blocked_url": 14, "logs_file_path": 14, "itself": [14, 20, 25, 26], "und": 14, "crawl_logs_to_df": 14, "crawl_logs_df": 14, "conformig": 14, "chosen": 14, "log_field": 14, "must": [14, 18, 23, 26, 28], "reader": 14, "latin": [14, 28], "regex_raw": 15, "hashtag_raw": 15, "mention_raw": 15, "raw": 15, "share": [15, 20, 26], "compil": 15, "readabl": [15, 17, 20, 32], "annot": 15, "v11": 15, "cookbook": 15, "2nd": 15, "ed": 15, "reilli": 15, "verifi": [16, 19], "pipelin": [16, 32], "pointer": 16, "comand": 16, "375": 16, "mail": 16, "garda": 16, "ir": 16, "875": 16, "shatel": 16, "cumul": [16, 26], "percentag": [16, 26, 29], "attent": 16, "max_work": [16, 19, 29], "equival": [16, 20], "worker": [16, 19], "multi": [16, 18], "though": [17, 18, 20, 23, 26, 28], "tini": 17, "potent": 17, "instruct": [17, 18, 28], "suppos": [17, 25], "mistak": 17, "ideal": [17, 25, 26, 32], "robotstxt_to_df": [17, 29], "etag": [17, 19, 29], "robotstxt_last_modifi": [17, 29], "robotstxt_url": 17, "download_d": [17, 19, 29], "a850165d925db701988daf7ead7492d3": 17, "200689": 17, "disallow": [17, 20], "exec": 17, "obido": 17, "style": [17, 20, 29], "flex": 17, "hp": 17, "mystuff": 17, "147": 17, "gp": 17, "profil": [17, 23], "148": 17, "149": 17, "etaospid": 17, "delai": 17, "ey": 17, "robots_url": 17, "googtwfb": 17, "groupbi": 17, "541": 17, "289": 17, "07": [17, 19, 30], "375724": 17, "howsearchwork": 17, "comment": [17, 20, 28, 29, 32], "nat": [17, 19], "461815": 17, "291": 17, "292": [17, 19], "_escaped_fragment_": 17, "293": 17, "lang": [17, 23], "397": 17, "474456": 17, "398": 17, "prohibit": 17, "unless": [17, 26], "permiss": 17, "399": 17, "conduct": 17, "purpos": 17, "401": 17, "app": [17, 18, 23, 28, 32], "site_scraping_tos_term": 17, "php": 17, "robotstxt_test": [17, 29], "owner": [17, 18, 23, 28], "realiti": 17, "appli": [17, 18, 19, 20], "care": 17, "fb_robot": 17, "951053": 17, "ajax": 17, "pagelet": 17, "pagepostssectionpagelet": 17, "538": [17, 32], "safetycheck": 17, "539": 17, "540": 17, "fb_userag": 17, "drop_dupl": 17, "tolist": [17, 19], "applebot": 17, "discordbot": 17, "facebookexternalhit": 17, "ia_archiv": 17, "msnbot": 17, "naverbot": 17, "pinterestbot": 17, "seznambot": 17, "teoma": 17, "telegrambot": 17, "yandex": 17, "yeti": 17, "quit": [17, 19, 26], "bbc": [17, 19], "urls_to_test": 17, "fb_test": 17, "url_path": 17, "can_fetch": 17, "76": 17, "receiv": [17, 23], "eighti": 17, "denot": 17, "24": [17, 19, 30], "40": [17, 18, 20], "figur": [17, 20, 23, 32], "linkedin": 17, "pinterest": 17, "clue": 17, "robotx": 17, "robotstxt_test_df": 17, "soon": 17, "robotstxt_df": 17, "2020": [17, 19, 20, 30], "09": [17, 19, 20, 30], "702814": 17, "08": [17, 19, 30], "087985": 17, "283": 17, "284": 17, "imgr": 17, "285": 17, "286": 17, "468588": 17, "287": 17, "lose": 17, "patienc": 17, "robots_output_fil": 17, "robotsfiles_df": 17, "fill": 18, "questionnair": 18, "survei": [18, 30], "serp_": [18, 32], "dimens": [18, 28], "serp_goog": [18, 20, 29, 32], "best": [18, 23, 26], "gl": 18, "ca": [18, 26], "uk": 18, "au": 18, "nz": 18, "ten": [18, 26], "450": 18, "snippet": [18, 20, 28, 29], "querytim": [18, 29], "serp_youtub": [18, 29], "At": [18, 28], "enter": [18, 20], "panel": 18, "remov": [18, 20, 23, 26, 27, 28, 29], "entir": [18, 23], "retriev": [18, 19, 23, 28], "programmat": 18, "free": 18, "pai": 18, "cx": 18, "c2coff": 18, "cr": 18, "daterestrict": 18, "exactterm": 18, "excludeterm": 18, "filetyp": 18, "highrang": 18, "hl": [18, 28], "hq": 18, "imgcolortyp": 18, "imgdominantcolor": 18, "imgsiz": 18, "imgtyp": 18, "linksit": 18, "lowrang": 18, "lr": 18, "num": 18, "orterm": 18, "safe": [18, 23], "searchtyp": 18, "sitesearch": 18, "sitesearchfilt": 18, "sort": [18, 21, 25, 26, 28, 29], "tradit": [18, 28], "disabled0": 18, "boolean": [18, 20, 28], "tld": [18, 29], "urlth": 18, "geograph": [18, 28], "addressse": 18, "past": [18, 32], "week": [18, 19, 23, 32], "month": [18, 19, 23], "year": [18, 19, 23], "appear": [18, 19, 20, 23, 26, 29], "center": [18, 28], "crowd": 18, "improv": [18, 26, 29], "geoloc": 18, "boost": 18, "whose": [18, 26], "lead": 18, "particularli": [18, 20], "speak": 18, "unit": [18, 19, 23, 28], "rang": [18, 23], "inclus": [18, 28], "explicitli": [18, 23, 25], "internation": 18, "term": [18, 20, 24, 28, 32], "logic": 18, "AND": 18, "black": [18, 20, 26], "grayscal": 18, "color": [18, 20, 25, 26], "mono": 18, "grai": 18, "domin": 18, "brown": 18, "orang": [18, 26], "pink": 18, "purpl": 18, "red": [18, 25], "teal": 18, "huge": 18, "xlarg": 18, "xxlarg": 18, "clipart": 18, "lineart": 18, "photo": [18, 23], "lang_ja": 18, "lang_ar": 18, "lang_bg": 18, "bulgarian": 18, "lang_ca": 18, "lang_c": 18, "czech": 18, "lang_da": 18, "lang_d": 18, "lang_el": 18, "lang_en": 18, "lang_": 18, "lang_et": 18, "estonian": 18, "lang_fi": 18, "lang_fr": 18, "lang_hr": 18, "lang_hu": 18, "lang_id": 18, "lang_i": 18, "iceland": 18, "lang_it": 18, "lang_iw": 18, "lang_ko": 18, "lang_lt": 18, "lithuanian": 18, "lang_lv": 18, "latvian": 18, "lang_nl": 18, "lang_no": 18, "lang_pl": 18, "lang_pt": 18, "lang_ro": 18, "lang_ru": 18, "lang_sk": 18, "slovak": 18, "lang_sl": 18, "slovenian": 18, "lang_sr": 18, "serbian": 18, "lang_sv": 18, "lang_tr": 18, "lang_zh": 18, "cn": 18, "tw": 18, "least": [18, 26, 28], "cc_publicdomain": 18, "cc_attribut": 18, "cc_sharealik": 18, "cc_noncommerci": 18, "cc_nonderiv": 18, "safeti": 18, "safesearch": [18, 28], "unspecifi": 18, "forth": [18, 23, 28], "ever": [18, 26], "serp_df": 18, "usa": 18, "franc": 18, "your_cx": 18, "your_kei": 18, "prouc": 18, "fligt": 18, "ticket": 18, "focus": 18, "flight": [18, 23], "countryuk": 18, "countryau": 18, "channelid": [18, 28], "channeltyp": [18, 28], "eventtyp": [18, 28], "forcontentown": [18, 28], "fordevelop": [18, 28], "formin": [18, 28], "locationradiu": [18, 28], "maxresult": [18, 28], "onbehalfofcontentown": [18, 28], "pagetoken": [18, 28], "publishedaft": [18, 28], "publishedbefor": [18, 28], "regioncod": [18, 28], "relatedtovideoid": [18, 28], "relevancelanguag": [18, 28], "topicid": [18, 28], "videocapt": [18, 28], "videocategoryid": [18, 28], "videodefinit": [18, 28], "videodimens": [18, 28], "videodur": [18, 28], "videoembedd": [18, 28], "videolicens": [18, 28], "videosynd": [18, 28], "videotyp": [18, 28], "loop": [18, 23, 29], "merg": [18, 20, 23, 29], "associ": [18, 26, 28], "boat": [18, 28], "sail": [18, 28], "similarli": [18, 20, 28], "fish": [18, 28], "pipe": [18, 26, 28], "escap": [18, 28], "sent": [18, 23, 28], "7c": [18, 28], "channel": [18, 28, 29, 32], "constrain": [18, 28], "broadcast": [18, 28], "live": [18, 28, 32], "upcom": [18, 28, 32], "intend": [18, 23, 28], "exclus": [18, 28], "partner": [18, 28], "via": [18, 23, 26, 28], "conjunct": [18, 23, 28], "subsequ": [18, 28], "circular": [18, 28], "metadata": [18, 23, 28, 29], "latitud": [18, 23, 28], "longitud": [18, 23, 28], "coordin": [18, 28], "42307": [18, 28], "122": [18, 28], "08427": [18, 28], "distanc": [18, 28], "float": [18, 28], "km": [18, 23, 28], "ft": [18, 28], "mi": [18, 23, 28], "1500m": [18, 28], "5km": [18, 28], "10000ft": [18, 28], "75mi": [18, 28], "larger": [18, 23, 28], "kilomet": [18, 23, 28], "definit": [18, 28], "cm": [18, 28], "act": [18, 28], "behalf": [18, 28], "individu": [18, 28], "chronolog": [18, 28], "rate": [18, 19, 20, 23, 26, 28], "highest": [18, 28], "lowest": [18, 28], "videocount": [18, 28], "descend": [18, 28], "viewcount": [18, 28], "viewer": [18, 28], "ongo": [18, 28], "nextpagetoken": [18, 28, 29], "prevpagetoken": [18, 28], "rfc": [18, 28], "3339": [18, 28], "1970": [18, 28], "01t00": [18, 28], "00z": [18, 19, 28], "3166": [18, 28], "alpha": [18, 28], "zh": [18, 28], "han": [18, 28], "hant": [18, 28], "standard": [18, 20, 23, 25, 28, 32], "moder": [18, 28], "demot": [18, 28], "strict": [18, 28], "freebas": [18, 28], "comma": [18, 23, 26, 27, 28], "playlist": [18, 28, 29], "closedcapt": [18, 28], "hd": [18, 28], "sd": [18, 28], "playback": [18, 28], "720p": [18, 28], "resolut": [18, 28], "1080p": [18, 28], "regardless": [18, 28, 29], "2d": [18, 28], "durat": [18, 28], "minut": [18, 19, 25, 28], "less": [18, 20, 23, 26, 28], "embed": [18, 23, 28], "embedd": [18, 28], "choos": [18, 20, 28], "attach": [18, 23, 28], "creativ": [18, 28], "creativecommon": [18, 28], "reus": [18, 28], "syndic": [18, 28], "episod": [18, 28], "movi": [18, 19, 23, 28], "set_logging_level": 18, "level_or_nam": 18, "dure": [18, 26], "notset": 18, "warn": 18, "youtube_channel_detail": 18, "channel_id": 18, "assum": [18, 20, 23, 29], "channel_df": 18, "youtube_video_detail": 18, "vid_id": 18, "video_df": 18, "fastest": 19, "easiest": 19, "reveal": 19, "correspond": 19, "publish": [19, 28], "rich": [19, 23], "sitemap_to_df": [19, 20, 29], "loc": 19, "hte": 19, "lastmod": 19, "sitemap_last_modifi": [19, 29], "sitemap_size_mb": [19, 29], "mega": 19, "1mb": 19, "024": [19, 29], "sitmeapindex": 19, "decid": [19, 20, 23], "bbc_sitemap": 19, "archiv": 19, "2009": [19, 20], "090620_as_iraq_explosion_tc2": 19, "e7e15811c65f406f89f89fe10aef29f5": 19, "05": [19, 20, 30], "63124": 19, "461037": 19, "090620_iraq_blast_tc2": 19, "43": 19, "busi": [19, 28], "090622_me_worldbank_tc2": 19, "090624_me_inpictures_brazil_tc2": 19, "090618_tomtest": 19, "090625_sf_tamim_verdict_tc2": 19, "090623_iz_cairo_russia_tc2": 19, "090622_me_egypt_us_tc2": 19, "090624_mz_wimbledon_tc2": 19, "worldnew": 19, "090623_mz_leaders_lifespan_tc2": 19, "49999": 19, "datetime64": 19, "utc": 19, "float64": 19, "set_index": 19, "resampl": 19, "2008": 19, "2287": 19, "47603": 19, "2010": 19, "2011": 19, "2012": 19, "2013": 19, "2015": [19, 21, 23], "2016": [19, 21], "2018": [19, 30], "2019": [19, 30], "freq": 19, "dec": 19, "seem": [19, 20, 26], "compar": [19, 25, 29], "rolling_new": 19, "090628_rn_pakistani_soldiries_ambush": 19, "pakistan": 19, "090421_mqm_speaks_rza": 19, "090723_ae_silwan_tc2": 19, "noticia": 19, "090729_iraquerefenbritsfn": 19, "090623_egitomilitaresfn": 19, "090302_gazaconferenciaml": 19, "090715_hillary_iran_cq": 19, "cultur": 19, "090409_machienhuu_revisit": 19, "090524_paquistaoupdateg": 19, "090629_om_pakistan_report_tc2": 19, "yyyi": [19, 23, 28], "mm": [19, 23, 28], "yymmdd_article_titl": 19, "url_df": 19, "scheme": [19, 25], "netloc": [19, 25], "fragment": [19, 25], "dir_1": [19, 25], "dir_2": [19, 25], "dir_3": [19, 25], "dir_4": 19, "dir_5": 19, "dir_6": 19, "dir_7": 19, "last_dir": [19, 25, 29], "49994": 19, "090831_dalailamataiwan": 19, "49995": 19, "090901_putin_regret_pact": 19, "49996": 19, "090901_tiananmen_movi": 19, "49997": 19, "pictur": [19, 26], "090830_ugc_ddh_sand": 19, "49998": 19, "090901_japecontask": 19, "14022": 19, "10968": 19, "5403": 19, "5068": 19, "mundo": 19, "5065": 19, "3561": 19, "2984": 19, "1677": 19, "turkc": 19, "706": 19, "ukchina": 19, "545": 19, "1506": 19, "2910": 19, "3021": 19, "3250": 19, "2769": 19, "9044": 19, "5050": 19, "4224": 19, "iran": 19, "3682": 19, "2103": 19, "afghanistan": 19, "1959": 19, "1657": 19, "internacion": 19, "1555": 19, "1350": 19, "1293": 19, "india": 19, "1285": 19, "america_latina": 19, "1274": 19, "1204": 19, "cultura_sociedad": 19, "913": 19, "874": 19, "872": 19, "russia": 19, "841": 19, "radio": 19, "769": 19, "scienc": [19, 32], "755": 19, "674": 19, "underscor": 19, "concaten": 19, "meaning": [19, 26], "explod": 19, "rn": 19, "8808": 19, "tc2": 19, "3153": 19, "1534": 19, "973": 19, "obama": 19, "882": 19, "862": 19, "china": 19, "815": 19, "ir88": 19, "727": 19, "683": 19, "si": 19, "640": 19, "np": [19, 29], "638": 19, "afghan": 19, "632": 19, "ka": 19, "565": 19, "556": 19, "iraq": 19, "554": 19, "547": 19, "nh": 19, "cq": 19, "510": 19, "ra": 19, "491": 19, "familiar": 19, "bug": [19, 29], "nyt_new": 19, "gz": [19, 29], "5085": 19, "news_publ": 19, "publication_nam": 19, "publication_languag": 19, "news_publication_d": 19, "news_titl": 19, "news_keyword": 19, "interact": [19, 23, 29, 32], "ottawa": 19, "ohio": 19, "covid": 19, "york": 19, "27t17": 19, "counti": 19, "exposur": 19, "risk": 19, "tracker": 19, "coronaviru": 19, "ncov": 19, "death": 19, "fatal": 19, "diseas": 19, "promo": 19, "1585539358901": 19, "articlelarg": 19, "v274": 19, "0cff645fbb74c21791568b78a888967d": 19, "0774069": 19, "744247": 19, "hopewel": 19, "virginia": 19, "1585539536519": 19, "v271": 19, "butt": 19, "nebraska": 19, "1585539237156": 19, "v281": 19, "stearn": 19, "minnesota": 19, "1585539172701": 19, "v282": 19, "benton": 19, "iowa": 19, "1585539039190": 19, "v286": 19, "5080": 19, "hodgeman": 19, "kansa": 19, "1585539054298": 19, "v285": 19, "f53301c8286f9bf59ef297f0232dcfc1": 19, "914107": 19, "995323": 19, "5081": 19, "miller": 19, "georgia": 19, "1585538956622": 19, "v290": 19, "5082": 19, "elect": 19, "west": 19, "hous": 19, "district": 19, "03t17": 19, "congression": 19, "david": 19, "mckinlei": 19, "natali": 19, "cline": 19, "presidenti": 19, "eln": 19, "race": [19, 28], "1winner": 19, "mckinleyd": 19, "5083": 19, "senat": 19, "susan": 19, "collin": 19, "defeat": 19, "sara": 19, "gideon": 19, "senatewinn": 19, "collinss": 19, "5084": 19, "randolph": 19, "missouri": 19, "1585539206866": 19, "wired_video": 19, "wire": 19, "2955": 19, "video_thumbnail_loc": 19, "video_titl": 19, "video_descript": 19, "video_content_loc": 19, "video_dur": 19, "video_publication_d": 19, "video_expiration_d": 19, "autocomplet": [19, 32], "inverview": 19, "owen": 19, "wilson": 19, "answer": 19, "dwgyu36up6iuz": 19, "cloudfront": 19, "heru80fdn": 19, "c_fill": 19, "d_placeholder_thescen": 19, "fl_progress": 19, "g_face": 19, "h_180": 19, "q_80": 19, "w_320": 19, "v1644595412": 19, "wired_autocomplet": 19, "interview": 19, "internet": 19, "himself": [19, 26], "nose": 19, "ben": 19, "stiller": 19, "anderson": 19, "skateboard": 19, "dp8hsntg6do36": 19, "62067f085577c277dd9acf42": 19, "39687acb": 19, "505b": 19, "4c69": 19, "94f1": 19, "afaa7cb5e636low": 19, "mp4": 19, "645": 19, "11t17": 19, "90b11f47f8b2ab57cb180cbd3c6f06f9": 19, "86199": 19, "841851": 19, "v1644418652": 19, "wired_wir": 19, "julian": 19, "chokkattu": 19, "editor": 19, "walk": 19, "6203cd7b5577c23d19622259": 19, "fe546b9b": 19, "a320": 19, "4883": 19, "9cbd": 19, "0d790f23c36dlow": 19, "184": 19, "10t17": 19, "v1644381627": 19, "wired_first": 19, "debut": 19, "newest": [19, 28], "620345a15577c23d46622256": 19, "d74930cf": 19, "11e1": 19, "466e": 19, "b023": 19, "1d9b91664204low": 19, "373": 19, "09t15": 19, "reinvent": 19, "v1642801328": 19, "wired_reinv": 19, "lab": 19, "aw": 19, "seattl": 19, "seahawk": 19, "win": [19, 26], "teach": 19, "swami": 19, "sivasubramanian": 19, "vp": 19, "ai": 19, "team": 19, "nfl": 19, "captur": [19, 32], "strateg": [19, 32], "619bd9be1d75db41adee6b58": 19, "d4889b15": 19, "4f34": 19, "41b0": 19, "b935": 19, "0c79465a9793low": 19, "09t13": 19, "seth": 19, "rogen": 19, "v1644335726": 19, "wired_seth": 19, "pam": 19, "amp": 19, "tommi": 19, "potteri": 19, "celebr": 19, "christma": 19, "duti": 19, "premier": 19, "februari": [19, 28], "hulu": 19, "march": 19, "6201430a1d75db06ae1f62e8": 19, "488ed635": 19, "91d0": 19, "4281": 19, "9e64": 19, "34be9bf74f00low": 19, "635": 19, "08t17": 19, "2950": 19, "genr": 19, "2951": 19, "2952": 19, "2953": 19, "promot": 19, "2954": 19, "thread": [19, 28, 29], "faster": [19, 29], "attack": 19, "sitemap_df": 19, "changefreq": 19, "prioriti": 19, "customiz": 20, "configur": [20, 28], "flexibl": [20, 32], "simplest": [20, 25], "reachabl": 20, "my_output_fil": 20, "affect": 20, "indepent": 20, "overwrit": [20, 29], "otherwis": [20, 23, 25, 26], "crash": 20, "sitename_crawl_yyyy_mm_dd": 20, "remark": 20, "rquest": 20, "card": [20, 23, 29], "jsonld_1_": 20, "item_a": 20, "item_b": 20, "whichev": [20, 25, 29, 32], "links_url": [20, 29], "links_text": [20, 29], "links_nofollow": [20, 29], "nofllow": 20, "tell": 20, "nav_links_url": 20, "header_links_url": 20, "footer_links_url": 20, "body_text": [20, 29], "p": [20, 29], "span": [20, 29], "li": [20, 29], "amount": [20, 25, 26], "took": 20, "download_timout": 20, "sec": 20, "crossorigin": [20, 29], "ismap": [20, 29], "longdesc": [20, 29], "referrerpolici": [20, 29], "srcset": [20, 29], "usemap": [20, 29], "global": [20, 23, 29], "draggabl": [20, 29], "third": [20, 23, 26], "truncat": 20, "site_crawl": 20, "links_href": 20, "leas": 20, "blob": 20, "resp_headers_access": 20, "request_headers_cooki": 20, "camp": 20, "readm": 20, "kw_": [20, 29, 32], "wed": 20, "720a8581": 20, "501e": 20, "0043": 20, "01a2": 20, "2e77d2": 20, "unlock": 20, "blockblob": 20, "web00007c": 20, "includesubdo": 20, "3600": 20, "ht": 20, "596daca7dbaa7e9": 20, "bud": 20, "02d86a3cea00007e9edb0cf2000000": 20, "xm": 20, "__cfduid": 20, "d76b68d148ddec1efd004": 20, "202": 20, "abil": [20, 28, 29], "4f7bea3b": 20, "701e": 20, "0039": 20, "3f44": 20, "2f1d9f": 20, "web00007h": 20, "596daca9bcab7e9": 20, "02d86a3e0e00007e9edb0d72000000": 20, "98b729fa": 20, "e01": 20, "00bf": 20, "24c3": 20, "2e494d": 20, "596daca9bf26d423": 20, "02d86a3e150000d423322742000000": 20, "submodul": [20, 29, 30, 31], "advertoo": 20, "7a28ef3b": 20, "801e": 20, "00c2": 20, "2ed585": 20, "web000079": 20, "596daca9bddb7ec2": 20, "02d86a3e1300007ec2a808a2000000": 20, "copyright": [20, 21], "eli": 20, "_static": 20, "75911c9e": 20, "201e": 20, "00e6": 20, "34c3": 20, "2e4ccb": 20, "web00007g": 20, "596daca9b91fd437": 20, "02d86a3e140000d437b81532000000": 20, "url_build": 20, "pyt": 20, "d99f2368": 20, "c01e": 20, "006f": 20, "18c3": 20, "2ef5ef": 20, "web00007a": 20, "596dacbbb8afd437": 20, "02d86a494f0000d437b828b2000000": 20, "pyth": 20, "85855c48": 20, "00ce": 20, "13c3": 20, "2e3b74": 20, "596dacbd980bd423": 20, "02d86a4a7f0000d423323b42000000": 20, "ad_": [20, 32], "b0aef497": 20, "004a": 20, "1647": 20, "2f6d5c": 20, "web00007k": 20, "596dacbd980cd423": 20, "02d86a4a7f0000d423209db2000000": 20, "9dfdd38a": 20, "101e": 20, "00a1": 20, "7ec3": 20, "2e93a0": 20, "596dacbd99847ec2": 20, "02d86a4a7f00007ec2a811f2000000": 20, "emo": 20, "2ad504a1": 20, "000b": 20, "03c3": 20, "2e454f": 20, "596dacbd9fb97e9": 20, "02d86a4a7f00007e9edb13a2000000": 20, "ran": 20, "got": 20, "richer": 20, "perspect": [20, 25, 26], "bounc": [20, 26], "traffic": [20, 23, 24], "export": 20, "addition": [20, 23], "pretti": 20, "name_1": 20, "selector_1": 20, "name_2": 20, "selector_2": 20, "reli": 20, "slectorgadget": 20, "selecotr": 20, "tricki": 20, "documentaion": 20, "w3c": 20, "decrib": 20, "sidebar": [20, 29], "toctre": 20, "l1": 20, "attr": 20, "sidebar_link": 20, "sidebar_links_url": 20, "shoe": [20, 26], "model_a": 20, "model_b": 20, "unexpect": 20, "rememb": 20, "granular": 20, "potenti": [20, 23], "region": [20, 23, 28], "regard": [20, 26], "further": [20, 25, 26, 32], "simultan": 20, "lower": 20, "pressur": 20, "deep": 20, "fraction": 20, "strongli": [20, 23], "yourself": [20, 26], "confid": 20, "fine": 20, "explan": 20, "outpuf_fil": 20, "attempt": 20, "product2": 20, "anotherexampl": 20, "anotherexmapl": 20, "author_url": 20, "contributornameid": 20, "spaci": [21, 29], "mine": [21, 26, 27, 28, 32], "footnot": 21, "explosionai": 21, "ug": 21, "haftungsbeschr\u00e4nkt": 21, "gmbh": 21, "matthew": 21, "honnib": 21, "complic": 23, "unnest": 23, "nest": [23, 28], "influenti": 23, "credibl": 23, "signatur": 23, "dashboard": [23, 32], "auth_param": 23, "app_kei": [23, 29], "your_app_kei": 23, "app_secret": [23, 29], "your_app_secret": 23, "set_auth_param": 23, "oauth_token": [23, 29], "oauth_token_secret": [23, 29], "your_oauth_token": 23, "your_oauth_token_secret": 23, "python_tweet": 23, "tweet_mod": 23, "140": 23, "prepend": [23, 25], "tweet_": 23, "user_": 23, "func": 23, "get_application_rate_limit_statu": 23, "consumed_onli": 23, "rate_limit_statu": 23, "get_available_trend": 23, "trend": [23, 26, 32], "get_favorit": 23, "user_id": 23, "screen_nam": 23, "since_id": 23, "max_id": 23, "include_ent": 23, "whom": [23, 26], "greater": 23, "forc": 23, "oldest": 23, "older": 23, "node": 23, "omit": 23, "engag": 23, "get_followers_id": 23, "cursor": 23, "stringify_id": 23, "semi": 23, "caus": [23, 29], "broken": 23, "5000": 23, "guarante": 23, "suspend": 23, "previous_cursor": 23, "next_cursor": 23, "environ": 23, "get_followers_list": 23, "skip_statu": 23, "include_user_ent": 23, "status": 23, "get_friends_id": 23, "friend": 23, "get_friends_list": 23, "get_home_timelin": 23, "trim_us": 23, "exclude_repli": 23, "retweet": [23, 26], "timelin": 23, "numer": [23, 28], "prevent": [23, 29], "repli": [23, 28], "home_timelin": 23, "get_list_memb": 23, "list_id": 23, "owner_screen_nam": 23, "owner_id": 23, "member": 23, "get_list_membership": 23, "filter_to_owned_list": 23, "disambigu": 23, "begin": [23, 27, 29], "membership": 23, "get_list_status": 23, "include_rt": 23, "ON": 23, "varieti": 23, "discreet": 23, "user_ment": 23, "nativ": 23, "stream": [23, 32], "ident": 23, "represent": 23, "get_list_subscrib": 23, "subscrib": [23, 28], "opt": 23, "futur": [23, 28], "truestatus": 23, "get_list_subscript": 23, "obtain": [23, 25, 26], "subscript": [23, 28], "get_mentions_timelin": 23, "mentions_timelin": 23, "get_place_trend": [23, 29], "woeid": 23, "earth": 23, "get_retweeters_id": 23, "get_retweet": 23, "get_supported_languag": 23, "get_user_timelin": 23, "strip": [23, 26, 27], "toward": [23, 26], "maxim": 23, "slice": 23, "user_timelin": 23, "lookup_statu": 23, "include_ext_alt_text": 23, "include_card_uri": 23, "hydrat": 23, "cannot": [23, 26], "null": 23, "pair": 23, "ext_alt_text": 23, "card_uri": 23, "lookup_us": 23, "encourag": 23, "make_datafram": 23, "retweeted_of_m": 23, "retweets_of_m": 23, "geocod": 23, "result_typ": 23, "lat": 23, "lon": 23, "dist": 23, "radiu": 23, "preferenti": 23, "geotag": 23, "mile": 23, "directli": 23, "distinct": 23, "detect": 23, "effort": 23, "ja": 23, "prefer": 23, "mix": [23, 25, 28], "dd": 23, "hate": 23, "beer": 23, "root": 23, "haiku": 23, "interior": 23, "nasa": 23, "astronaut": 23, "puppi": 23, "native_video": 23, "amplifi": 23, "periscop": 23, "vine": 23, "instagram": 23, "twimg": 23, "pic": 23, "hilari": 23, "anywher": [23, 26], "superhero": 23, "scari": 23, "attitud": 23, "search_us": 23, "access_token": 23, "token_typ": 23, "bearer": 23, "oauth_vers": 23, "api_vers": 23, "client_arg": 23, "auth_endpoint": 23, "twython": [23, 29], "starting_out": 23, "show_list": 23, "show_owned_list": 23, "ownership": 23, "url_utm_ga": [24, 29], "utm_sourc": 24, "utm_medium": 24, "utm_campaign": 24, "utm_cont": 24, "utm_term": 24, "utm": [24, 29], "banner": 24, "summer_promo": 24, "20pct_off": 24, "differenti": 24, "728x90": 24, "mpu": 24, "square_bann": 24, "bid": 24, "mysit": 24, "THE": 24, "2anam": 24, "5e": 24, "fairli": [25, 32], "situat": 25, "enhanc": 25, "path_1": 25, "path_2": 25, "frag_1": 25, "frag_2": 25, "path_3": 25, "query_color": 25, "query_pric": 25, "query_s": 25, "elabor": 25, "decod": 25, "self": 25, "explanatori": 25, "blog": 25, "previou": [25, 28, 29], "unalign": 25, "popul": 25, "na": [25, 29], "hash": 25, "query_": 25, "colliss": 25, "unlik": 25, "delimit": [25, 27], "unusu": 25, "product1": 25, "sens": [25, 26], "renam": 25, "inconsist": 25, "topic1": 25, "topic2": 25, "artilc": 25, "yout": 25, "distort": 25, "role": [25, 28], "topic_1": 25, "topic_2": 25, "align": 25, "ouput_fil": 25, "urldf": 25, "corpu": 26, "accomplish": [26, 32], "word_frequ": [26, 27, 29, 32], "sequenc": [26, 32], "dot": [26, 27], "quotat": 26, "whatev": [26, 32], "sale": 26, "quantifi": 26, "bag": 26, "half": 26, "revenu": 26, "million": 26, "hidden": 26, "ppc": 26, "num_list": [26, 29], "experi": 26, "rm_word": 26, "ignor": 26, "possibli": 26, "extra_info": 26, "abs_freq": [26, 29], "wtd_freq": [26, 29], "rel_valu": [26, 29], "essenti": [26, 27, 32], "multipli": 26, "abs_perc": 26, "abs_perc_cum": 26, "wtd_freq_perc": 26, "wtd_freq_perc_cum": 26, "afterward": 26, "alon": 26, "among": 26, "amongst": 26, "anyhow": 26, "anyon": 26, "becam": 26, "beforehand": 26, "behind": 26, "besid": 26, "beyond": 26, "eight": 26, "eleven": 26, "elsewher": 26, "everywher": 26, "fifti": 26, "former": 26, "formerli": 26, "forti": [26, 32], "henc": 26, "her": 26, "hereaft": 26, "herebi": 26, "herein": 26, "hereupon": 26, "herself": 26, "him": 26, "inde": 26, "latter": 26, "latterli": 26, "me": [26, 27], "meanwhil": 26, "moreov": 26, "move": 26, "myself": 26, "neither": 26, "nevertheless": 26, "nine": 26, "nobodi": 26, "noon": 26, "nor": 26, "nowher": 26, "often": 26, "onto": 26, "perhap": 26, "rather": 26, "seriou": 26, "sixti": 26, "somehow": 26, "someon": 26, "somewher": [26, 32], "themselv": [26, 29], "thenc": 26, "thereaft": 26, "therebi": [26, 28], "therein": 26, "thereupon": 26, "throughout": 26, "thru": 26, "twelv": 26, "upon": 26, "whenc": 26, "whenev": 26, "whereaft": 26, "wherea": 26, "wherebi": 26, "wherein": 26, "whereupon": 26, "wherev": [26, 29], "whither": 26, "whoever": 26, "yourselv": 26, "blown": 26, "ngram": 26, "metric": 26, "abs_wtd_df": 26, "banana": 26, "kiwi": 26, "mango": 26, "250": 26, "300": 26, "beat": 26, "text_list2": 26, "222222": 26, "333333": 26, "111111": 26, "266667": 26, "600000": 26, "666667": 26, "200000": 26, "800000": 26, "888889": 26, "133333": 26, "933333": 26, "000000": 26, "066667": 26, "word_token": [27, 29], "trim": 27, "quot": [27, 29], "parenthes": 27, "trail": [27, 29], "insid": 27, "activities_list": 28, "criteria": 28, "child": 28, "quota": 28, "cost": 28, "contentdetail": 28, "uniqu": [28, 29], "deprec": [28, 29], "unsign": 28, "earliest": 28, "8601": 28, "ddthh": 28, "ss": 28, "sz": 28, "captions_list": 28, "videoid": 28, "channel_sections_list": 28, "channelsect": 28, "i18nlanguag": 28, "channels_list": 28, "categoryid": 28, "forusernam": 28, "managedbym": 28, "mysubscrib": 28, "auditdetail": 28, "brandingset": 28, "contentownerdetail": 28, "invideopromot": 28, "topicdetail": 28, "comment_threads_list": 28, "allthreadsrelatedtochannelid": 28, "moderationstatu": 28, "searchterm": 28, "textformat": 28, "commentthread": 28, "heldforreview": 28, "await": 28, "likelyspam": 28, "classifi": 28, "spam": 28, "plaintext": 28, "plain": 28, "comments_list": 28, "parentid": 28, "guide_categories_list": [28, 29], "guidecategori": 28, "i18n_languages_list": 28, "en_u": 28, "i18n_regions_list": 28, "i18nregion": 28, "playlist_items_list": 28, "playlistid": 28, "playlistitem": 28, "resourceid": 28, "playlists_list": 28, "onbehalfofcontentownerchannel": 28, "timecr": 28, "action": [28, 29], "curat": 28, "music": 28, "04rlf": 28, "02mscn": 28, "christian": 28, "0ggq0m": 28, "classic": 28, "01lyv": 28, "02lkt": 28, "electron": 28, "0glt670": 28, "hip": 28, "hop": 28, "05rwpb": 28, "03_d0": 28, "jazz": 28, "028sqc": 28, "asia": 28, "0g293": 28, "america": 28, "064t9": 28, "pop": 28, "06cqb": 28, "regga": 28, "06j6l": 28, "rhythm": 28, "06by7": 28, "rock": 28, "0gywn": 28, "soul": 28, "game": 28, "0bzvm2": 28, "025zzc": 28, "02ntfj": 28, "adventur": 28, "0b1vjn": 28, "casual": 28, "02hygl": 28, "04q1x3q": 28, "puzzl": 28, "01sjng": 28, "0403l3g": 28, "021bp2": 28, "simul": 28, "022dc6": 28, "03hf_rm": 28, "06ntj": 28, "0jm_": 28, "american": 28, "018jz": 28, "basebal": 28, "018w8": 28, "01cgz": 28, "09xp_": 28, "cricket": 28, "02vx4": 28, "037hz": 28, "golf": 28, "03tmr": 28, "hockei": 28, "01h7lh": 28, "martial": 28, "0410tth": 28, "motorsport": 28, "07bs0": 28, "tenni": 28, "07_53": 28, "volleybal": 28, "entertain": 28, "02jjt": 28, "09kqc": 28, "humor": 28, "02vxn": 28, "05qjc": 28, "066wd": 28, "profession": 28, "wrestl": 28, "0f2f9": 28, "lifestyl": 28, "019_rr": 28, "032tl": 28, "fashion": 28, "027x7n": 28, "02wbm": 28, "03glg": 28, "hobbi": 28, "068hy": 28, "pet": 28, "041xxh": 28, "physic": 28, "attract": 28, "beauti": 28, "07c1v": 28, "07bxq": 28, "tourism": 28, "07yv9": 28, "vehicl": 28, "societi": 28, "098wr": 28, "09s1f": 28, "0kt51": 28, "01h6rj": 28, "militari": 28, "05qt0": 28, "06bvp": 28, "religion": 28, "01k8wb": 28, "channelplaylistvideo": 28, "subscriptions_list": 28, "myrecentsubscrib": 28, "forchannelid": 28, "subscribersnippet": 28, "subscription_order_relev": 28, "unread": 28, "video_categories_list": 28, "videocategori": 28, "videos_list": 28, "chart": 28, "myrat": 28, "maxheight": 28, "maxwidth": 28, "filedetail": 28, "livestreamingdetail": 28, "processingdetail": 28, "recordingdetail": 28, "mostpopular": 28, "dislik": 28, "embedhtml": 28, "emb": 28, "appropri": 28, "violat": 28, "8192": 28, "narrow": 28, "subpackag": [29, 30, 31], "log_date_format": 29, "relatedsit": 29, "v15": 29, "contribut": 29, "danielp77": 29, "offlin": 29, "preserv": 29, "autothrottl": 29, "minim": 29, "fillna": 29, "ffill": 29, "andypayn": 29, "newlin": 29, "clarifi": 29, "thebe": 29, "sphinx": 29, "deprac": 29, "skip_url_param": 29, "versatil": 29, "bad": 29, "timeout": 29, "sitemapindex": 29, "mb": 29, "024x1": 29, "anymor": 29, "jsonld_error": 29, "resp_meta_": 29, "preced": 29, "url_redirected_to": 29, "links_frag": 29, "invalid": 29, "stricter": 29, "unifi": 29, "element_1": 29, "element_2": 29, "drop": 29, "slight": 29, "relayout": 29, "clarif": 29, "robotstxt": 29, "cse": 29, "returnd": 29, "sitemap_download": 29, "variabl": 29, "expand": 29, "pagemap": 29, "df": 29, "top_emoji_categori": 29, "top_emoji_sub_categori": 29, "db": 29, "simpler": 29, "__init__": 29, "ve": 29, "punctuat": 29, "pagin": [29, 32], "reflect": 29, "lenght": 29, "rewrit": 29, "_dict_product": 29, "msg": 29, "implement": 29, "repons": 29, "town": 29, "wrap": 29, "pand": 29, "cheat": 29, "sheet": [29, 32], "coverag": 29, "releas": 29, "pypi": 29, "placehold": [29, 32], "feedback": 30, "pip3": 30, "unreleas": 30, "2023": 30, "announc": 32, "scientist": 32, "manipul": 32, "visual": 32, "sophist": 32, "algorithm": 32, "cool": 32, "spent": 32, "wrangl": 32, "stitch": 32, "124": 32, "hopefulli": 32, "pick": 32, "excel": 32, "formula": 32, "unix": 32, "doug": 32, "mcilroi": 32, "univers": 32, "aim": 32, "unrel": 32, "workflow": 32, "practition": 32, "plotli": 32, "librari": 32, "tabular": 32, "kept": 32, "modular": 32, "coder": 32, "promis": 32, "deliveri": 32, "didn": 32, "headlin": 32, "datacamp": 32, "semrush": 32, "comprehens": 32, "render": 32, "creation": 32, "outreach": 32, "built": 32, "megabyt": 32, "monitor": 32, "parser": 32, "notebook": 32, "tackl": 32, "bloomberg": 32, "click": 32, "divers": 32, "3k": 32, "conveni": 32, "introductori": 32, "clean": 32, "131k": 32, "european": 32, "url_": 32, "emoji_": 32, "_to_df": 32}, "objects": {"": [[0, 0, 0, "-", "advertools"]], "advertools": [[1, 0, 0, "-", "ad_create"], [2, 0, 0, "-", "ad_from_string"], [3, 0, 0, "-", "cli"], [5, 0, 0, "-", "code_recipes"], [7, 0, 0, "-", "crawlytics"], [8, 0, 0, "-", "emoji"], [9, 0, 0, "-", "extract"], [10, 0, 0, "-", "header_spider"], [11, 0, 0, "-", "image_spider"], [12, 0, 0, "-", "knowledge_graph"], [13, 0, 0, "-", "kw_generate"], [14, 0, 0, "-", "logs"], [15, 0, 0, "-", "regex"], [16, 0, 0, "-", "reverse_dns_lookup"], [17, 0, 0, "-", "robotstxt"], [18, 0, 0, "-", "serp"], [19, 0, 0, "-", "sitemaps"], [20, 0, 0, "-", "spider"], [21, 0, 0, "-", "stopwords"], [23, 0, 0, "-", "twitter"], [24, 0, 0, "-", "url_builders"], [25, 0, 0, "-", "urlytics"], [26, 0, 0, "-", "word_frequency"], [27, 0, 0, "-", "word_tokenize"], [28, 0, 0, "-", "youtube"]], "advertools.ad_create": [[1, 1, 1, "", "ad_create"]], "advertools.ad_from_string": [[2, 1, 1, "", "ad_from_string"]], "advertools.cli": [[4, 0, 0, "-", "cli"]], "advertools.code_recipes": [[6, 0, 0, "-", "spider_strategies"]], "advertools.crawlytics": [[7, 1, 1, "", "images"], [7, 1, 1, "", "jl_subset"], [7, 1, 1, "", "jl_to_parquet"], [7, 1, 1, "", "links"], [7, 1, 1, "", "parquet_columns"], [7, 1, 1, "", "redirects"]], "advertools.emoji": [[8, 1, 1, "", "emoji_search"], [8, 1, 1, "", "extract_emoji"]], "advertools.extract": [[9, 1, 1, "", "extract"], [9, 1, 1, "", "extract_currency"], [9, 1, 1, "", "extract_exclamations"], [9, 1, 1, "", "extract_hashtags"], [9, 1, 1, "", "extract_intense_words"], [9, 1, 1, "", "extract_mentions"], [9, 1, 1, "", "extract_numbers"], [9, 1, 1, "", "extract_questions"], [9, 1, 1, "", "extract_urls"], [9, 1, 1, "", "extract_words"]], "advertools.header_spider": [[10, 2, 1, "", "HeadersSpider"], [10, 1, 1, "", "crawl_headers"]], "advertools.header_spider.HeadersSpider": [[10, 3, 1, "", "custom_settings"], [10, 4, 1, "", "errback"], [10, 3, 1, "", "name"], [10, 4, 1, "", "parse"], [10, 4, 1, "", "start_requests"]], "advertools.image_spider": [[11, 2, 1, "", "AdvImagesPipeline"], [11, 2, 1, "", "ImageSpider"], [11, 2, 1, "", "ImgItem"], [11, 1, 1, "", "crawl_images"], [11, 1, 1, "", "summarize_crawled_imgs"]], "advertools.image_spider.AdvImagesPipeline": [[11, 4, 1, "", "file_path"]], "advertools.image_spider.ImageSpider": [[11, 3, 1, "", "custom_settings"], [11, 3, 1, "", "include_img_regex"], [11, 3, 1, "", "name"], [11, 4, 1, "", "parse"], [11, 4, 1, "", "start_requests"]], "advertools.image_spider.ImgItem": [[11, 3, 1, "", "fields"]], "advertools.knowledge_graph": [[12, 1, 1, "", "knowledge_graph"]], "advertools.kw_generate": [[13, 1, 1, "", "kw_broad"], [13, 1, 1, "", "kw_exact"], [13, 1, 1, "", "kw_generate"], [13, 1, 1, "", "kw_modified"], [13, 1, 1, "", "kw_neg_broad"], [13, 1, 1, "", "kw_neg_exact"], [13, 1, 1, "", "kw_neg_phrase"], [13, 1, 1, "", "kw_phrase"]], "advertools.logs": [[14, 1, 1, "", "crawllogs_to_df"], [14, 1, 1, "", "logs_to_df"]], "advertools.reverse_dns_lookup": [[16, 1, 1, "", "reverse_dns_lookup"]], "advertools.robotstxt": [[17, 1, 1, "", "robotstxt_test"], [17, 1, 1, "", "robotstxt_to_df"]], "advertools.serp": [[18, 1, 1, "", "serp_goog"], [18, 1, 1, "", "serp_youtube"], [18, 1, 1, "", "set_logging_level"], [18, 1, 1, "", "youtube_channel_details"], [18, 1, 1, "", "youtube_video_details"]], "advertools.sitemaps": [[19, 1, 1, "", "sitemap_to_df"]], "advertools.spider": [[20, 1, 1, "", "crawl"]], "advertools.twitter": [[23, 1, 1, "", "authenticate"], [23, 1, 1, "", "get_application_rate_limit_status"], [23, 1, 1, "", "get_available_trends"], [23, 1, 1, "", "get_favorites"], [23, 1, 1, "", "get_followers_ids"], [23, 1, 1, "", "get_followers_list"], [23, 1, 1, "", "get_friends_ids"], [23, 1, 1, "", "get_friends_list"], [23, 1, 1, "", "get_home_timeline"], [23, 1, 1, "", "get_list_members"], [23, 1, 1, "", "get_list_memberships"], [23, 1, 1, "", "get_list_statuses"], [23, 1, 1, "", "get_list_subscribers"], [23, 1, 1, "", "get_list_subscriptions"], [23, 1, 1, "", "get_mentions_timeline"], [23, 1, 1, "", "get_place_trends"], [23, 1, 1, "", "get_retweeters_ids"], [23, 1, 1, "", "get_retweets"], [23, 1, 1, "", "get_supported_languages"], [23, 1, 1, "", "get_user_timeline"], [23, 1, 1, "", "lookup_status"], [23, 1, 1, "", "lookup_user"], [23, 1, 1, "", "make_dataframe"], [23, 1, 1, "", "retweeted_of_me"], [23, 1, 1, "", "search"], [23, 1, 1, "", "search_users"], [23, 1, 1, "", "set_auth_params"], [23, 1, 1, "", "show_lists"], [23, 1, 1, "", "show_owned_lists"]], "advertools.url_builders": [[24, 1, 1, "", "url_utm_ga"]], "advertools.urlytics": [[25, 1, 1, "", "url_to_df"]], "advertools.word_frequency": [[26, 1, 1, "", "word_frequency"]], "advertools.word_tokenize": [[27, 1, 1, "", "word_tokenize"]], "advertools.youtube": [[28, 1, 1, "", "activities_list"], [28, 1, 1, "", "captions_list"], [28, 1, 1, "", "channel_sections_list"], [28, 1, 1, "", "channels_list"], [28, 1, 1, "", "comment_threads_list"], [28, 1, 1, "", "comments_list"], [28, 1, 1, "", "guide_categories_list"], [28, 1, 1, "", "i18n_languages_list"], [28, 1, 1, "", "i18n_regions_list"], [28, 1, 1, "", "playlist_items_list"], [28, 1, 1, "", "playlists_list"], [28, 1, 1, "", "search"], [28, 1, 1, "", "subscriptions_list"], [28, 1, 1, "", "video_categories_list"], [28, 1, 1, "", "videos_list"]]}, "objtypes": {"0": "py:module", "1": "py:function", "2": "py:class", "3": "py:attribute", "4": "py:method"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "function", "Python function"], "2": ["py", "class", "Python class"], "3": ["py", "attribute", "Python attribute"], "4": ["py", "method", "Python method"]}, "titleterms": {"advertool": [0, 3, 4, 5, 29, 30, 31, 32], "packag": [0, 5], "subpackag": 0, "submodul": [0, 5], "modul": [0, 3, 5, 7], "content": [0, 5, 30, 32], "creat": [1, 2], "ad": [1, 2], "larg": [1, 7, 12, 17, 25], "scale": [1, 12, 17, 32], "us": [2, 4, 6, 12], "long": 2, "descript": 2, "text": [2, 4, 8, 9, 26, 30, 32], "top": 2, "down": [2, 6], "approach": [2, 17, 20, 32], "googl": [2, 12, 18, 20], "facebook": 2, "feed": 2, "instant": 2, "articl": [2, 20], "cli": [3, 4], "command": 4, "line": 4, "interfac": 4, "convert": 4, "robot": [4, 6, 17], "txt": [4, 6, 17], "file": [4, 7, 11, 14, 17], "list": [4, 6, 9, 20], "url": [4, 20, 24, 25], "tabl": [4, 30], "csv": 4, "format": [4, 14], "download": [4, 11, 19], "pars": [4, 14, 19, 25], "save": [4, 6], "an": 4, "xml": [4, 19], "sitemap": [4, 19], "split": [4, 25], "compon": 4, "scheme": 4, "netloc": 4, "path": [4, 25], "queri": [4, 20, 25], "etc": 4, "crawl": [4, 6, 7, 14, 20], "known": 4, "head": 4, "method": 4, "compress": [4, 7], "log": [4, 6, 14, 29, 30], "datafram": [4, 14], "parquet": [4, 7], "perform": 4, "revers": [4, 16], "dn": [4, 16], "lookup": [4, 16], "ip": 4, "address": 4, "gener": [4, 13], "sem": [4, 13, 30, 32], "keyword": [4, 13], "suppli": 4, "product": [4, 30, 32], "intent": 4, "word": [4, 26, 27], "get": [4, 8], "stopword": [4, 21], "select": 4, "languag": [4, 21], "count": [4, 26], "option": 4, "weight": [4, 26], "number": [4, 6, 9, 25], "search": [4, 8, 18, 20], "emoji": [4, 8, 9], "regex": [4, 20], "extract": [4, 6, 8, 9, 15, 20], "structur": [4, 9, 15, 25], "entiti": [4, 9, 15], "from": [4, 6, 8, 9], "hashtag": [4, 9], "mention": [4, 9], "token": [4, 27], "document": 4, "phrase": 4, "tweet": 4, "desir": 4, "length": 4, "seo": [4, 6, 20, 30, 32], "crawler": [4, 11, 20], "code_recip": 5, "scrape": [6, 7], "strategi": 6, "recip": 6, "how": [6, 12, 14], "page": [6, 18, 20], "those": 6, "onli": 6, "mode": [6, 20], "can": 6, "i": 6, "websit": [6, 7], "includ": 6, "its": 6, "sub": 6, "domain": 6, "copi": 6, "my": 6, "audit": 6, "them": 6, "later": 6, "automat": 6, "stop": 6, "base": 6, "certain": 6, "condit": 6, "di": 6, "obei": 6, "rule": 6, "do": 6, "set": [6, 20], "user": [6, 17], "agent": [6, 17], "while": [6, 20], "control": 6, "concurr": 6, "request": 6, "slow": 6, "so": 6, "don": 6, "t": 6, "hit": 6, "server": 6, "too": 6, "hard": 6, "multipl": 6, "same": 6, "job": 6, "want": 6, "follow": [6, 20], "link": [6, 7, 20], "specifi": 6, "depth": 6, "paus": 6, "resum": 6, "make": 6, "sure": 6, "twice": 6, "proxi": 6, "chang": [6, 29, 30], "default": 6, "header": [6, 10], "xpath": [6, 20], "express": [6, 15], "custom": [6, 20], "string": 6, "analysi": [7, 14, 26, 30, 32], "analyz": [7, 8, 12, 14, 17, 19, 25], "imag": [7, 11], "redirect": 7, "handl": 7, "veri": 7, "explor": 7, "column": 7, "data": [7, 14, 20, 23, 28], "type": 7, "function": [7, 9, 14, 20, 23], "insight": 8, "currenc": 9, "1234567890\u0660\u0661\u0662\u0663\u0664\u0665\u0666\u0667\u0668\u0669\u32ba\ud804\udc5b\ud800\udd0d\ud802\udcaa\u24f2\ud804\udc63\ud800\udd28\ud802\udd1b": 9, "question": 9, "\u0294": 9, "exclam": 9, "python": [10, 20], "statu": 10, "code": 10, "checker": 10, "respons": 10, "name": 11, "import": [12, 18], "knowledg": 12, "graph": 12, "result": [12, 18], "account": 12, "setup": 12, "": 12, "api": [12, 23, 28], "campaign": [13, 32], "run": 14, "logs_to_df": 14, "support": 14, "prepar": 14, "regular": [15, 19], "bulk": [16, 17], "test": 17, "tester": 17, "engin": 18, "serp": [18, 20], "youtub": [18, 28], "index": [19, 30], "new": [19, 20], "video": 19, "spider": 20, "discoveri": 20, "On": 20, "element": 20, "pre": 20, "determin": 20, "analyt": 20, "consol": 20, "css": 20, "selector": 20, "behavior": 20, "paramet": [20, 25], "pattern": 20, "addit": 20, "sever": 21, "survei": 22, "share": 22, "feedback": 22, "twitter": 23, "authent": 23, "builder": 24, "The": 25, "directori": 25, "absolut": 26, "v": 26, "frequenc": 26, "n": 27, "gram": 27, "unreleas": 29, "0": 29, "14": 29, "2": 29, "2024": 29, "02": 29, "24": 29, "1": 29, "21": 29, "18": 29, "13": 29, "5": 29, "2023": 29, "08": 29, "22": 29, "4": 29, "07": 29, "26": 29, "3": 29, "06": 29, "27": 29, "2022": 29, "09": 29, "30": 29, "05": 29, "11": 29, "10": 29, "12": 29, "2021": 29, "04": 29, "03": 29, "31": 29, "7": 29, "2020": 29, "6": 29, "25": 29, "23": 29, "9": 29, "19": 29, "8": 29, "2019": 29, "17": 29, "29": 29, "01": 29, "2018": 29, "onlin": [30, 32], "market": [30, 32], "tool": [30, 32], "social": [30, 32], "media": [30, 32], "indic": 30, "your": 32, "instal": 32, "philosophi": 32, "convent": 32}, "envversion": {"sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 60}, "alltitles": {"advertools package": [[0, "advertools-package"]], "Subpackages": [[0, "subpackages"]], "Submodules": [[0, "submodules"], [5, "submodules"]], "Module contents": [[0, "module-advertools"], [5, "module-advertools.code_recipes"]], "Create Ads on a Large Scale": [[1, "create-ads-on-a-large-scale"]], "Create Ads Using Long Descriptive Text (top-down approach)": [[2, "create-ads-using-long-descriptive-text-top-down-approach"]], "Google Text Ads": [[2, "google-text-ads"]], "Facebook Feed Ads": [[2, "facebook-feed-ads"]], "Facebook Instant Article Ad": [[2, "facebook-instant-article-ad"]], "advertools.cli module": [[3, "module-advertools.cli"]], "advertools Command Line Interface (CLI)": [[4, "advertools-command-line-interface-cli"]], "convert a robots.txt file (or list of file URLs) to a table in a CSV format": [[4, "convert-a-robots-txt-file-or-list-of-file-urls-to-a-table-in-a-csv-format"]], "download, parse, and save an XML sitemap to a table in a CSV file": [[4, "download-parse-and-save-an-xml-sitemap-to-a-table-in-a-csv-file"]], "split a list of URLs into their components: scheme, netloc, path, query, etc.": [[4, "split-a-list-of-urls-into-their-components-scheme-netloc-path-query-etc"]], "crawl a list of known URLs using the HEAD method": [[4, "crawl-a-list-of-known-urls-using-the-head-method"]], "parse, compress and convert a log file to a DataFrame in the .parquet format": [[4, "parse-compress-and-convert-a-log-file-to-a-dataframe-in-the-parquet-format"]], "perform a reverse DNS lookup on a list of IP addresses": [[4, "perform-a-reverse-dns-lookup-on-a-list-of-ip-addresses"]], "generate a table of SEM keywords by supplying a list of products and a list of intent words": [[4, "generate-a-table-of-sem-keywords-by-supplying-a-list-of-products-and-a-list-of-intent-words"]], "get stopwords of the selected language": [[4, "get-stopwords-of-the-selected-language"]], "get word counts of a text list optionally weighted by a number list": [[4, "get-word-counts-of-a-text-list-optionally-weighted-by-a-number-list"]], "search for emoji using a regex": [[4, "search-for-emoji-using-a-regex"]], "extract structured entities from a text list; emoji, hashtags, mentions": [[4, "extract-structured-entities-from-a-text-list-emoji-hashtags-mentions"]], "tokenize documents (phrases, keywords, tweets, etc) into token of the desired length": [[4, "tokenize-documents-phrases-keywords-tweets-etc-into-token-of-the-desired-length"]], "SEO crawler": [[4, "seo-crawler"]], "advertools.code_recipes package": [[5, "advertools-code-recipes-package"]], "\ud83d\udd77 SEO Crawling & Scraping: Strategies & Recipes": [[6, "seo-crawling-scraping-strategies-recipes"]], "How to crawl a list of pages, and those pages only (list mode)?": [[6, "how-to-crawl-a-list-of-pages-and-those-pages-only-list-mode"]], "How can I crawl a website including its sub-domains?": [[6, "how-can-i-crawl-a-website-including-its-sub-domains"]], "How can I save a copy of the logs of my crawl for auditing them later?": [[6, "how-can-i-save-a-copy-of-the-logs-of-my-crawl-for-auditing-them-later"]], "How can I automatically stop my crawl based on a certain condition?": [[6, "how-can-i-automatically-stop-my-crawl-based-on-a-certain-condition"]], "How can I (dis)obey robots.txt rules?": [[6, "how-can-i-dis-obey-robots-txt-rules"]], "How do I set my User-agent while crawling?": [[6, "how-do-i-set-my-user-agent-while-crawling"]], "How can I control the number of concurrent requests while crawling?": [[6, "how-can-i-control-the-number-of-concurrent-requests-while-crawling"]], "How can I slow down the crawling so I don't hit the websites' servers too hard?": [[6, "how-can-i-slow-down-the-crawling-so-i-don-t-hit-the-websites-servers-too-hard"]], "How can I set multiple settings to the same crawl job?": [[6, "how-can-i-set-multiple-settings-to-the-same-crawl-job"]], "I want to crawl a list of pages, follow links from those pages, but only to a certain specified depth": [[6, "i-want-to-crawl-a-list-of-pages-follow-links-from-those-pages-but-only-to-a-certain-specified-depth"]], "How do I pause/resume crawling, while making sure I don't crawl the same page twice?": [[6, "how-do-i-pause-resume-crawling-while-making-sure-i-don-t-crawl-the-same-page-twice"]], "How do I use a proxy while crawling?": [[6, "how-do-i-use-a-proxy-while-crawling"]], "How can I change the default request headers?": [[6, "how-can-i-change-the-default-request-headers"]], "XPath expressions for custom extraction": [[6, "xpath-expressions-for-custom-extraction"]], "User-agent strings for use in crawling": [[6, "user-agent-strings-for-use-in-crawling"]], "Crawling and Scraping Analysis": [[7, "module-advertools.crawlytics"]], "Analyzing crawled images": [[7, "analyzing-crawled-images"]], "Analyzing links in a crawled website": [[7, "analyzing-links-in-a-crawled-website"]], "Analyzing the redirects of a crawled website": [[7, "analyzing-the-redirects-of-a-crawled-website"]], "Handling very large crawl files": [[7, "handling-very-large-crawl-files"]], "Compressing large crawl files": [[7, "compressing-large-crawl-files"]], "Exploring the columns and data types of parquet files": [[7, "exploring-the-columns-and-data-types-of-parquet-files"]], "Module functions": [[7, "module-functions"]], "Emoji: Extract, Analyze, and Get Insights": [[8, "emoji-extract-analyze-and-get-insights"]], "Emoji Search": [[8, "emoji-search"]], "Extract Emoji from Text": [[8, "extract-emoji-from-text"]], "Extract structured entities from text lists": [[9, "extract-structured-entities-from-text-lists"]], "Extract Functions": [[9, "extract-functions"]], "Extract #hashtags": [[9, "extract-hashtags"]], "Extract @mentions": [[9, "extract-mentions"]], "Extract Currency $ \u00a2 \u00a3 \u00a4 \u00a5 \u058f \u060b \u20b2 \u20b5 \u20b8 \u20b9\ufe69 \uffe0 \uffe1 \uffe5 \uffe6 \u20ba \u20bb \u20bc \u20bd \u20be \u20bf \ufdfc": [[9, "extract-currency"]], "Extract numbers 1234567890\u0660\u0661\u0662\u0663\u0664\u0665\u0666\u0667\u0668\u0669\u32ba\ud804\udc5b\ud800\udd0d\ud802\udcaa\u24f2\ud804\udc63\ud800\udd28\ud802\udd1b": [[9, "extract-numbers-123456789045"]], "Extract questions ? \u00bf \u037e \u055e \u061f \u1367 \u1945 \u2047 \u2048 \u2049 \u2cfa \u2cfb \u2e2e \ua60f \ua6f7 \ufe16 \ufe56 \uff1f \ud804\udd43 \ud83a\udd5f \u0294 \u203d": [[9, "extract-questions"]], "Extract Exclamations ! \u00a1 \u055c \u07f9 \u1944 \u203c \u2048 \u2049 \ufe15 \ufe57 \uff01 \ud83a\udd5e": [[9, "extract-exclamations"]], "Extract Emoji \ud83d\ude02\ud83d\ude2d\ud83e\udd7a\ud83e\udd23\u2764\ufe0f\u2728\ud83d\ude4f\ud83d\ude0d": [[9, "extract-emoji"]], "\ud83d\udd77 Python Status Code Checker with Response Headers": [[10, "python-status-code-checker-with-response-headers"]], "Image Crawler and Downloader": [[11, "image-crawler-and-downloader"]], "Image file names": [[11, "image-file-names"]], "Import and Analyze Knowledge Graph Results on a Large Scale": [[12, "import-and-analyze-knowledge-graph-results-on-a-large-scale"]], "Account Setup": [[12, "account-setup"]], "How to use Google's Knowledge Graph API": [[12, "how-to-use-google-s-knowledge-graph-api"]], "Generate Keywords for SEM Campaigns": [[13, "generate-keywords-for-sem-campaigns"]], "Log File Analysis": [[14, "log-file-analysis"]], "How to run the logs_to_df() function:": [[14, "how-to-run-the-logs-to-df-function"]], "Supported Log Formats": [[14, "supported-log-formats"]], "Log File Analysis - Data Preparation": [[14, "log-file-analysis-data-preparation"]], "Parse and Analyze Crawl Logs in a Dataframe": [[14, "parse-and-analyze-crawl-logs-in-a-dataframe"]], "Regular Expressions for Extracting Structured Entities": [[15, "regular-expressions-for-extracting-structured-entities"]], "Reverse DNS Lookup in Bulk": [[16, "module-advertools.reverse_dns_lookup"]], "\ud83e\udd16 Analyze and Test robots.txt Files on a Large Scale": [[17, "analyze-and-test-robots-txt-files-on-a-large-scale"]], "Bulk robots.txt Tester": [[17, "bulk-robots-txt-tester"]], "User-agents": [[17, "user-agents"]], "robots.txt Testing Approach": [[17, "robots-txt-testing-approach"]], "Import Search Engine Results Pages (SERPs) for Google and YouTube": [[18, "import-search-engine-results-pages-serps-for-google-and-youtube"]], "Download, Parse, and Analyze XML Sitemaps": [[19, "download-parse-and-analyze-xml-sitemaps"]], "Sitemap Index": [[19, "sitemap-index"]], "Regular XML Sitemaps": [[19, "regular-xml-sitemaps"]], "News Sitemaps": [[19, "news-sitemaps"]], "Video Sitemaps": [[19, "video-sitemaps"]], "\ud83d\udd77 Python SEO Crawler / Spider": [[20, "python-seo-crawler-spider"]], "Discovery Crawling Approach": [[20, "discovery-crawling-approach"]], "Extracted On-Page SEO Elements": [[20, "extracted-on-page-seo-elements"]], "Pre-Determined Crawling Approach (List Mode)": [[20, "pre-determined-crawling-approach-list-mode"]], "SERP Data": [[20, "serp-data"]], "News Articles": [[20, "news-articles"]], "Google Analytics / Google Search Console": [[20, "google-analytics-google-search-console"]], "Custom Extraction with CSS and XPath Selectors": [[20, "custom-extraction-with-css-and-xpath-selectors"]], "Customizing the Crawling Behavior while Following Links": [[20, "customizing-the-crawling-behavior-while-following-links"]], "URL Query Parameters": [[20, "url-query-parameters"]], "URL Regex Patterns": [[20, "url-regex-patterns"]], "Spider Custom Settings and Additional Functionality": [[20, "spider-custom-settings-and-additional-functionality"]], "Stopwords in Several Languages": [[21, "stopwords-in-several-languages"]], "Stopword Languages": [[21, "stopword-languages"]], "Survey - share feedback": [[22, "survey-share-feedback"]], "Twitter Data API": [[23, "twitter-data-api"]], "Authentication": [[23, "authentication"]], "Functions": [[23, "functions"]], "URL Builders": [[24, "url-builders"]], "Split, Parse, and Analyze URL Structure": [[25, "split-parse-and-analyze-url-structure"]], "Query Parameters": [[25, "query-parameters"]], "The URL Path (Directories):": [[25, "the-url-path-directories"]], "Analyzing a large number of URLs": [[25, "analyzing-a-large-number-of-urls"]], "Text Analysis": [[26, "text-analysis"]], "Absolute and Weighted Word Count": [[26, "absolute-and-weighted-word-count"]], "Absolute vs Weighted Frequency": [[26, "absolute-vs-weighted-frequency"]], "Tokenize Words (N-grams)": [[27, "tokenize-words-n-grams"]], "YouTube Data API": [[28, "youtube-data-api"]], "advertools": [[29, "advertools"], [30, "advertools"], [31, "advertools"]], "Change Log - advertools": [[29, "change-log-advertools"]], "(UNRELEASED)": [[29, "unreleased"]], "0.14.2 (2024-02-24)": [[29, "id1"]], "0.14.1 (2024-02-21)": [[29, "id2"]], "0.14.0 (2024-02-18)": [[29, "id3"]], "0.13.5 (2023-08-22)": [[29, "id4"]], "0.13.4 (2023-07-26)": [[29, "id5"]], "0.13.3 (2023-06-27)": [[29, "id6"]], "0.13.2 (2022-09-30)": [[29, "id7"]], "0.13.1 (2022-05-11)": [[29, "id8"]], "0.13.0 (2022-02-10)": [[29, "id9"]], "0.12.3 (2021-11-27)": [[29, "id10"]], "0.12.0,1,2 (2021-11-27)": [[29, "id11"]], "0.11.1 (2021-04-09)": [[29, "id12"]], "0.11.0 (2021-03-31)": [[29, "id13"]], "0.10.7 (2020-09-18)": [[29, "id14"]], "0.10.6 (2020-06-30)": [[29, "id15"]], "0.10.5 (2020-06-14)": [[29, "id16"]], "0.10.4 (2020-06-07)": [[29, "id17"]], "0.10.3 (2020-06-03)": [[29, "id18"]], "0.10.2 (2020-05-25)": [[29, "id19"]], "0.10.1 (2020-05-23)": [[29, "id20"]], "0.10.0 (2020-05-21)": [[29, "id21"]], "0.9.1 (2020-05-19)": [[29, "id22"]], "0.9.0 (2020-04-03)": [[29, "id23"]], "0.8.1 (2020-02-08)": [[29, "id24"]], "0.8.0 (2020-02-02)": [[29, "id25"]], "0.7.3 (2019-04-17)": [[29, "id26"]], "0.7.2 (2019-03-29)": [[29, "id27"]], "0.7.1 (2019-03-26)": [[29, "id28"]], "0.7.0 (2019-03-26)": [[29, "id29"]], "0.6.0 (2019-02-11)": [[29, "id30"]], "0.5.3 (2019-01-31)": [[29, "id31"]], "0.5.2 (2018-12-01)": [[29, "id32"]], "0.5.1 (2018-11-06)": [[29, "id33"]], "0.5.0 (2018-11-04)": [[29, "id34"]], "0.4.1 (2018-10-13)": [[29, "id35"]], "0.4.0 (2018-10-08)": [[29, "id36"]], "0.3.0 (2018-08-14)": [[29, "id37"]], "0.2.0 (2018-07-06)": [[29, "id38"]], "0.1.0 (2018-07-02)": [[29, "id39"]], "Online marketing productivity and analysis tools": [[30, "online-marketing-productivity-and-analysis-tools"]], "SEM": [[30, null]], "SEO": [[30, null], [32, "seo"]], "Text & Content Analysis": [[30, null]], "Social Media": [[30, null], [32, "social-media"]], "Indices and tables": [[30, "indices-and-tables"]], "Index & Change Log": [[30, null]], "advertools: productivity & analysis tools to scale your online marketing": [[32, "advertools-productivity-analysis-tools-to-scale-your-online-marketing"]], "Installation": [[32, "installation"]], "Philosophy/approach": [[32, "philosophy-approach"]], "SEM Campaigns": [[32, "sem-campaigns"]], "Text & Content Analysis (for SEO & Social Media)": [[32, "text-content-analysis-for-seo-social-media"]], "Conventions": [[32, "conventions"]]}, "indexentries": {"advertools": [[0, "module-advertools"]], "module": [[0, "module-advertools"], [1, "module-advertools.ad_create"], [2, "module-advertools.ad_from_string"], [3, "module-advertools.cli"], [4, "module-advertools.cli.cli"], [5, "module-advertools.code_recipes"], [6, "module-advertools.code_recipes.spider_strategies"], [7, "module-advertools.crawlytics"], [8, "module-advertools.emoji"], [9, "module-advertools.extract"], [10, "module-advertools.header_spider"], [11, "module-advertools.image_spider"], [12, "module-advertools.knowledge_graph"], [13, "module-advertools.kw_generate"], [14, "module-advertools.logs"], [15, "module-advertools.regex"], [16, "module-advertools.reverse_dns_lookup"], [17, "module-advertools.robotstxt"], [18, "module-advertools.serp"], [19, "module-advertools.sitemaps"], [20, "module-advertools.spider"], [21, "module-advertools.stopwords"], [23, "module-advertools.twitter"], [24, "module-advertools.url_builders"], [25, "module-advertools.urlytics"], [26, "module-advertools.word_frequency"], [27, "module-advertools.word_tokenize"], [28, "module-advertools.youtube"]], "ad_create() (in module advertools.ad_create)": [[1, "advertools.ad_create.ad_create"]], "advertools.ad_create": [[1, "module-advertools.ad_create"]], "ad_from_string() (in module advertools.ad_from_string)": [[2, "advertools.ad_from_string.ad_from_string"]], "advertools.ad_from_string": [[2, "module-advertools.ad_from_string"]], "capitalize": [[2, "term-capitalize"]], "s": [[2, "term-s"]], "sep": [[2, "term-sep"]], "slots": [[2, "term-slots"]], "advertools.cli": [[3, "module-advertools.cli"]], "advertools.cli.cli": [[4, "module-advertools.cli.cli"]], "advertools.code_recipes": [[5, "module-advertools.code_recipes"]], "advertools.code_recipes.spider_strategies": [[6, "module-advertools.code_recipes.spider_strategies"]], "advertools.crawlytics": [[7, "module-advertools.crawlytics"]], "images() (in module advertools.crawlytics)": [[7, "advertools.crawlytics.images"]], "jl_subset() (in module advertools.crawlytics)": [[7, "advertools.crawlytics.jl_subset"]], "jl_to_parquet() (in module advertools.crawlytics)": [[7, "advertools.crawlytics.jl_to_parquet"]], "links() (in module advertools.crawlytics)": [[7, "advertools.crawlytics.links"]], "parquet_columns() (in module advertools.crawlytics)": [[7, "advertools.crawlytics.parquet_columns"]], "redirects() (in module advertools.crawlytics)": [[7, "advertools.crawlytics.redirects"]], "advertools.emoji": [[8, "module-advertools.emoji"]], "emoji_search() (in module advertools.emoji)": [[8, "advertools.emoji.emoji_search"]], "extract_emoji() (in module advertools.emoji)": [[8, "advertools.emoji.extract_emoji"]], "advertools.extract": [[9, "module-advertools.extract"]], "extract() (in module advertools.extract)": [[9, "advertools.extract.extract"]], "extract_currency() (in module advertools.extract)": [[9, "advertools.extract.extract_currency"]], "extract_exclamations() (in module advertools.extract)": [[9, "advertools.extract.extract_exclamations"]], "extract_hashtags() (in module advertools.extract)": [[9, "advertools.extract.extract_hashtags"]], "extract_intense_words() (in module advertools.extract)": [[9, "advertools.extract.extract_intense_words"]], "extract_mentions() (in module advertools.extract)": [[9, "advertools.extract.extract_mentions"]], "extract_numbers() (in module advertools.extract)": [[9, "advertools.extract.extract_numbers"]], "extract_questions() (in module advertools.extract)": [[9, "advertools.extract.extract_questions"]], "extract_urls() (in module advertools.extract)": [[9, "advertools.extract.extract_urls"]], "extract_words() (in module advertools.extract)": [[9, "advertools.extract.extract_words"]], "headersspider (class in advertools.header_spider)": [[10, "advertools.header_spider.HeadersSpider"]], "advertools.header_spider": [[10, "module-advertools.header_spider"]], "crawl_headers() (in module advertools.header_spider)": [[10, "advertools.header_spider.crawl_headers"]], "custom_settings (headersspider attribute)": [[10, "advertools.header_spider.HeadersSpider.custom_settings"]], "errback() (headersspider method)": [[10, "advertools.header_spider.HeadersSpider.errback"]], "name (headersspider attribute)": [[10, "advertools.header_spider.HeadersSpider.name"]], "parse() (headersspider method)": [[10, "advertools.header_spider.HeadersSpider.parse"]], "start_requests() (headersspider method)": [[10, "advertools.header_spider.HeadersSpider.start_requests"]], "advimagespipeline (class in advertools.image_spider)": [[11, "advertools.image_spider.AdvImagesPipeline"]], "imagespider (class in advertools.image_spider)": [[11, "advertools.image_spider.ImageSpider"]], "imgitem (class in advertools.image_spider)": [[11, "advertools.image_spider.ImgItem"]], "advertools.image_spider": [[11, "module-advertools.image_spider"]], "crawl_images() (in module advertools.image_spider)": [[11, "advertools.image_spider.crawl_images"]], "custom_settings (imagespider attribute)": [[11, "advertools.image_spider.ImageSpider.custom_settings"]], "fields (imgitem attribute)": [[11, "advertools.image_spider.ImgItem.fields"]], "file_path() (advimagespipeline method)": [[11, "advertools.image_spider.AdvImagesPipeline.file_path"]], "include_img_regex (imagespider attribute)": [[11, "advertools.image_spider.ImageSpider.include_img_regex"]], "name (imagespider attribute)": [[11, "advertools.image_spider.ImageSpider.name"]], "parse() (imagespider method)": [[11, "advertools.image_spider.ImageSpider.parse"]], "start_requests() (imagespider method)": [[11, "advertools.image_spider.ImageSpider.start_requests"]], "summarize_crawled_imgs() (in module advertools.image_spider)": [[11, "advertools.image_spider.summarize_crawled_imgs"]], "advertools.knowledge_graph": [[12, "module-advertools.knowledge_graph"]], "knowledge_graph() (in module advertools.knowledge_graph)": [[12, "advertools.knowledge_graph.knowledge_graph"]], "advertools.kw_generate": [[13, "module-advertools.kw_generate"]], "kw_broad() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_broad"]], "kw_exact() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_exact"]], "kw_generate() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_generate"]], "kw_modified() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_modified"]], "kw_neg_broad() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_neg_broad"]], "kw_neg_exact() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_neg_exact"]], "kw_neg_phrase() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_neg_phrase"]], "kw_phrase() (in module advertools.kw_generate)": [[13, "advertools.kw_generate.kw_phrase"]], "advertools.logs": [[14, "module-advertools.logs"]], "crawllogs_to_df() (in module advertools.logs)": [[14, "advertools.logs.crawllogs_to_df"]], "logs_to_df() (in module advertools.logs)": [[14, "advertools.logs.logs_to_df"]], "advertools.regex": [[15, "module-advertools.regex"]], "advertools.reverse_dns_lookup": [[16, "module-advertools.reverse_dns_lookup"]], "reverse_dns_lookup() (in module advertools.reverse_dns_lookup)": [[16, "advertools.reverse_dns_lookup.reverse_dns_lookup"]], "advertools.robotstxt": [[17, "module-advertools.robotstxt"]], "robotstxt_test() (in module advertools.robotstxt)": [[17, "advertools.robotstxt.robotstxt_test"]], "robotstxt_to_df() (in module advertools.robotstxt)": [[17, "advertools.robotstxt.robotstxt_to_df"]], "advertools.serp": [[18, "module-advertools.serp"]], "serp_goog() (in module advertools.serp)": [[18, "advertools.serp.serp_goog"]], "serp_youtube() (in module advertools.serp)": [[18, "advertools.serp.serp_youtube"]], "set_logging_level() (in module advertools.serp)": [[18, "advertools.serp.set_logging_level"]], "youtube_channel_details() (in module advertools.serp)": [[18, "advertools.serp.youtube_channel_details"]], "youtube_video_details() (in module advertools.serp)": [[18, "advertools.serp.youtube_video_details"]], "advertools.sitemaps": [[19, "module-advertools.sitemaps"]], "sitemap_to_df() (in module advertools.sitemaps)": [[19, "advertools.sitemaps.sitemap_to_df"]], "advertools.spider": [[20, "module-advertools.spider"]], "crawl() (in module advertools.spider)": [[20, "advertools.spider.crawl"]], "advertools.stopwords": [[21, "module-advertools.stopwords"]], "advertools.twitter": [[23, "module-advertools.twitter"]], "authenticate() (in module advertools.twitter)": [[23, "advertools.twitter.authenticate"]], "get_application_rate_limit_status() (in module advertools.twitter)": [[23, "advertools.twitter.get_application_rate_limit_status"]], "get_available_trends() (in module advertools.twitter)": [[23, "advertools.twitter.get_available_trends"]], "get_favorites() (in module advertools.twitter)": [[23, "advertools.twitter.get_favorites"]], "get_followers_ids() (in module advertools.twitter)": [[23, "advertools.twitter.get_followers_ids"]], "get_followers_list() (in module advertools.twitter)": [[23, "advertools.twitter.get_followers_list"]], "get_friends_ids() (in module advertools.twitter)": [[23, "advertools.twitter.get_friends_ids"]], "get_friends_list() (in module advertools.twitter)": [[23, "advertools.twitter.get_friends_list"]], "get_home_timeline() (in module advertools.twitter)": [[23, "advertools.twitter.get_home_timeline"]], "get_list_members() (in module advertools.twitter)": [[23, "advertools.twitter.get_list_members"]], "get_list_memberships() (in module advertools.twitter)": [[23, "advertools.twitter.get_list_memberships"]], "get_list_statuses() (in module advertools.twitter)": [[23, "advertools.twitter.get_list_statuses"]], "get_list_subscribers() (in module advertools.twitter)": [[23, "advertools.twitter.get_list_subscribers"]], "get_list_subscriptions() (in module advertools.twitter)": [[23, "advertools.twitter.get_list_subscriptions"]], "get_mentions_timeline() (in module advertools.twitter)": [[23, "advertools.twitter.get_mentions_timeline"]], "get_place_trends() (in module advertools.twitter)": [[23, "advertools.twitter.get_place_trends"]], "get_retweeters_ids() (in module advertools.twitter)": [[23, "advertools.twitter.get_retweeters_ids"]], "get_retweets() (in module advertools.twitter)": [[23, "advertools.twitter.get_retweets"]], "get_supported_languages() (in module advertools.twitter)": [[23, "advertools.twitter.get_supported_languages"]], "get_user_timeline() (in module advertools.twitter)": [[23, "advertools.twitter.get_user_timeline"]], "lookup_status() (in module advertools.twitter)": [[23, "advertools.twitter.lookup_status"]], "lookup_user() (in module advertools.twitter)": [[23, "advertools.twitter.lookup_user"]], "make_dataframe() (in module advertools.twitter)": [[23, "advertools.twitter.make_dataframe"]], "retweeted_of_me() (in module advertools.twitter)": [[23, "advertools.twitter.retweeted_of_me"]], "search() (in module advertools.twitter)": [[23, "advertools.twitter.search"]], "search_users() (in module advertools.twitter)": [[23, "advertools.twitter.search_users"]], "set_auth_params() (in module advertools.twitter)": [[23, "advertools.twitter.set_auth_params"]], "show_lists() (in module advertools.twitter)": [[23, "advertools.twitter.show_lists"]], "show_owned_lists() (in module advertools.twitter)": [[23, "advertools.twitter.show_owned_lists"]], "advertools.url_builders": [[24, "module-advertools.url_builders"]], "url_utm_ga() (in module advertools.url_builders)": [[24, "advertools.url_builders.url_utm_ga"]], "advertools.urlytics": [[25, "module-advertools.urlytics"]], "url_to_df() (in module advertools.urlytics)": [[25, "advertools.urlytics.url_to_df"]], "advertools.word_frequency": [[26, "module-advertools.word_frequency"]], "extra_info": [[26, "term-extra_info"]], "num_list": [[26, "term-num_list"]], "phrase_len": [[26, "term-phrase_len"]], "regex": [[26, "term-regex"]], "rm_words": [[26, "term-rm_words"]], "text_list": [[26, "term-text_list"]], "word_frequency() (in module advertools.word_frequency)": [[26, "advertools.word_frequency.word_frequency"]], "advertools.word_tokenize": [[27, "module-advertools.word_tokenize"]], "word_tokenize() (in module advertools.word_tokenize)": [[27, "advertools.word_tokenize.word_tokenize"]], "activities_list() (in module advertools.youtube)": [[28, "advertools.youtube.activities_list"]], "advertools.youtube": [[28, "module-advertools.youtube"]], "captions_list() (in module advertools.youtube)": [[28, "advertools.youtube.captions_list"]], "channel_sections_list() (in module advertools.youtube)": [[28, "advertools.youtube.channel_sections_list"]], "channels_list() (in module advertools.youtube)": [[28, "advertools.youtube.channels_list"]], "comment_threads_list() (in module advertools.youtube)": [[28, "advertools.youtube.comment_threads_list"]], "comments_list() (in module advertools.youtube)": [[28, "advertools.youtube.comments_list"]], "guide_categories_list() (in module advertools.youtube)": [[28, "advertools.youtube.guide_categories_list"]], "i18n_languages_list() (in module advertools.youtube)": [[28, "advertools.youtube.i18n_languages_list"]], "i18n_regions_list() (in module advertools.youtube)": [[28, "advertools.youtube.i18n_regions_list"]], "playlist_items_list() (in module advertools.youtube)": [[28, "advertools.youtube.playlist_items_list"]], "playlists_list() (in module advertools.youtube)": [[28, "advertools.youtube.playlists_list"]], "search() (in module advertools.youtube)": [[28, "advertools.youtube.search"]], "subscriptions_list() (in module advertools.youtube)": [[28, "advertools.youtube.subscriptions_list"]], "video_categories_list() (in module advertools.youtube)": [[28, "advertools.youtube.video_categories_list"]], "videos_list() (in module advertools.youtube)": [[28, "advertools.youtube.videos_list"]]}}) \ No newline at end of file