From 6af55b577b8d3f9e1009d7ae2187a5b997354680 Mon Sep 17 00:00:00 2001 From: aizazalisyed Date: Mon, 8 Jul 2024 02:02:52 +0500 Subject: [PATCH 1/5] looping through file but causing: RuntimeWarning: Enable tracemalloc to get the object allocation traceback~ --- domains.txt | 4 + theHarvester/__main__.py | 962 ++++++++++++++++++++------------------- 2 files changed, 500 insertions(+), 466 deletions(-) create mode 100644 domains.txt diff --git a/domains.txt b/domains.txt new file mode 100644 index 00000000000..b0c279a18e7 --- /dev/null +++ b/domains.txt @@ -0,0 +1,4 @@ +tesla.com +pieas.com +chichi.com + diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 89e007cf5f3..fa405b3c2c2 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -65,7 +65,7 @@ async def start(rest_args: argparse.Namespace | None = None): parser = argparse.ArgumentParser( description='theHarvester is used to gather open source intelligence (OSINT) on a company or domain.' ) - parser.add_argument('-d', '--domain', help='Company name or domain to search.', required=True) + parser.add_argument('-d', '--domain', help='Company name or domain to search.') parser.add_argument( '-l', '--limit', @@ -155,6 +155,12 @@ async def start(rest_args: argparse.Namespace | None = None): urlscan, virustotal, yahoo, zoomeye""", ) + parser.add_argument( + '--domainlist', + help='File containing a list of domains.', + type=str, + ) + # determines if filename is coming from rest api or user rest_filename = '' # indicates this from the rest API @@ -174,6 +180,26 @@ async def start(rest_args: argparse.Namespace | None = None): args = parser.parse_args() filename = args.filename dnsbrute = (args.dns_brute, False) + + # Check if neither -d/--domain nor --domainlist is provided + if not args.domain and not args.domainlist: + parser.print_help() + print("\nError: Either -d/--domain or --domainlist must be provided.") + sys.exit(1) + + # Initialize domainLists from file if --domainlist is provided + if args.domainlist: + domain_list_file = args.domainlist + try: + with open(domain_list_file, 'r') as f: + domainLists = [line.strip() for line in f.readlines() if line.strip()] + except FileNotFoundError: + print(f"Error: File '{domain_list_file}' not found.") + sys.exit(1) + else: + domainLists = [args.domain] if args.domain else [] + + try: db = stash.StashManager() await db.do_init() @@ -237,7 +263,6 @@ async def start(rest_args: argparse.Namespace | None = None): all_urls: list = [] vhost: list = [] virtual = args.virtual_host - word: str = args.domain.rstrip('\n') takeover_status = args.take_over use_proxy = args.proxies linkedin_people_list_tracker: list = [] @@ -282,536 +307,541 @@ async def store( :param store_interestingurls: whether to store interesting urls :param store_asns: whether to store asns """ - await search_engine.process(use_proxy) if process_param is None else await search_engine.process(process_param, use_proxy) - db_stash = stash.StashManager() - - if source: - print(f'\033[94m[*] Searching {source[0].upper() + source[1:]}. ') - - if store_host: - host_names = list({host for host in await search_engine.get_hostnames() if f'.{word}' in host}) - host_names = list(host_names) - if source != 'hackertarget' and source != 'pentesttools' and source != 'rapiddns': - # If a source is inside this conditional, it means the hosts returned must be resolved to obtain ip - # This should only be checked if --dns-resolve has a wordlist - if dnsresolve is None or len(final_dns_resolver_list) > 0: - # indicates that -r was passed in if dnsresolve is None - full_hosts_checker = hostchecker.Checker(host_names, final_dns_resolver_list) - # If full, this is only getting resolved hosts - ( - resolved_pair, - temp_hosts, - temp_ips, - ) = await full_hosts_checker.check() - all_ip.extend(temp_ips) - full.extend(resolved_pair) - # full.extend(temp_hosts) + + for word in domainLists: + + await search_engine.process(use_proxy) if process_param is None else await search_engine.process(process_param, use_proxy) + db_stash = stash.StashManager() + + if source: + print(f'\033[94m[*] Searching {source[0].upper() + source[1:]}. ') + + if store_host: + host_names = list({host for host in await search_engine.get_hostnames() if f'.{word}' in host}) + host_names = list(host_names) + if source != 'hackertarget' and source != 'pentesttools' and source != 'rapiddns': + # If a source is inside this conditional, it means the hosts returned must be resolved to obtain ip + # This should only be checked if --dns-resolve has a wordlist + if dnsresolve is None or len(final_dns_resolver_list) > 0: + # indicates that -r was passed in if dnsresolve is None + full_hosts_checker = hostchecker.Checker(host_names, final_dns_resolver_list) + # If full, this is only getting resolved hosts + ( + resolved_pair, + temp_hosts, + temp_ips, + ) = await full_hosts_checker.check() + all_ip.extend(temp_ips) + full.extend(resolved_pair) + # full.extend(temp_hosts) + else: + full.extend(host_names) else: full.extend(host_names) + all_hosts.extend(host_names) + await db_stash.store_all(word, all_hosts, 'host', source) + + if store_emails: + email_list = await search_engine.get_emails() + all_emails.extend(email_list) + await db_stash.store_all(word, email_list, 'email', source) + + if store_ip: + ips_list = await search_engine.get_ips() + all_ip.extend(ips_list) + await db_stash.store_all(word, all_ip, 'ip', source) + + if store_results: + email_list, host_names, urls = await search_engine.get_results() + all_emails.extend(email_list) + host_names = list({host for host in host_names if f'.{word}' in host}) + all_urls.extend(urls) + all_hosts.extend(host_names) + await db.store_all(word, all_hosts, 'host', source) + await db.store_all(word, all_emails, 'email', source) + + if store_people: + people_list = await search_engine.get_people() + await db_stash.store_all(word, people_list, 'people', source) + + if store_links: + links = await search_engine.get_links() + linkedin_links_tracker.extend(links) + if len(links) > 0: + await db.store_all(word, links, 'linkedinlinks', engineitem) + + if store_interestingurls: + iurls = await search_engine.get_interestingurls() + interesting_urls.extend(iurls) + if len(iurls) > 0: + await db.store_all(word, iurls, 'interestingurls', engineitem) + + if store_asns: + fasns = await search_engine.get_asns() + total_asns.extend(fasns) + if len(fasns) > 0: + await db.store_all(word, fasns, 'asns', engineitem) + + + for word in domainLists: + stor_lst = [] + if args.source is not None: + if args.source.lower() != 'all': + engines = sorted(set(map(str.strip, args.source.split(',')))) else: - full.extend(host_names) - all_hosts.extend(host_names) - await db_stash.store_all(word, all_hosts, 'host', source) - - if store_emails: - email_list = await search_engine.get_emails() - all_emails.extend(email_list) - await db_stash.store_all(word, email_list, 'email', source) - - if store_ip: - ips_list = await search_engine.get_ips() - all_ip.extend(ips_list) - await db_stash.store_all(word, all_ip, 'ip', source) - - if store_results: - email_list, host_names, urls = await search_engine.get_results() - all_emails.extend(email_list) - host_names = list({host for host in host_names if f'.{word}' in host}) - all_urls.extend(urls) - all_hosts.extend(host_names) - await db.store_all(word, all_hosts, 'host', source) - await db.store_all(word, all_emails, 'email', source) - - if store_people: - people_list = await search_engine.get_people() - await db_stash.store_all(word, people_list, 'people', source) - - if store_links: - links = await search_engine.get_links() - linkedin_links_tracker.extend(links) - if len(links) > 0: - await db.store_all(word, links, 'linkedinlinks', engineitem) - - if store_interestingurls: - iurls = await search_engine.get_interestingurls() - interesting_urls.extend(iurls) - if len(iurls) > 0: - await db.store_all(word, iurls, 'interestingurls', engineitem) - - if store_asns: - fasns = await search_engine.get_asns() - total_asns.extend(fasns) - if len(fasns) > 0: - await db.store_all(word, fasns, 'asns', engineitem) - - stor_lst = [] - if args.source is not None: - if args.source.lower() != 'all': - engines = sorted(set(map(str.strip, args.source.split(',')))) - else: - engines = Core.get_supportedengines() - # Iterate through search engines in order - if set(engines).issubset(Core.get_supportedengines()): - print(f'\n[*] Target: {word} \n') - - for engineitem in engines: - if engineitem == 'anubis': - try: - anubis_search = anubis.SearchAnubis(word) - stor_lst.append(store(anubis_search, engineitem, store_host=True)) - except Exception as e: - print(e) + engines = Core.get_supportedengines() + # Iterate through search engines in order + if set(engines).issubset(Core.get_supportedengines()): + print(f'\n[*] Target: {word} \n') - elif engineitem == 'baidu': - try: - baidu_search = baidusearch.SearchBaidu(word, limit) - stor_lst.append( - store( - baidu_search, - engineitem, - store_host=True, - store_emails=True, - ) - ) - except Exception as e: - print(e) + for engineitem in engines: + if engineitem == 'anubis': + try: + anubis_search = anubis.SearchAnubis(word) + stor_lst.append(store(anubis_search, engineitem, store_host=True)) + except Exception as e: + print(e) - elif engineitem == 'bevigil': - try: - bevigil_search = bevigil.SearchBeVigil(word) - stor_lst.append( - store( - bevigil_search, - engineitem, - store_host=True, - store_interestingurls=True, + elif engineitem == 'baidu': + try: + baidu_search = baidusearch.SearchBaidu(word, limit) + stor_lst.append( + store( + baidu_search, + engineitem, + store_host=True, + store_emails=True, + ) ) - ) - except Exception as e: - print(e) - - elif engineitem == 'binaryedge': - try: - binaryedge_search = binaryedgesearch.SearchBinaryEdge(word, limit) - stor_lst.append(store(binaryedge_search, engineitem, store_host=True)) - except Exception as e: - print(e) + except Exception as e: + print(e) - elif engineitem == 'bing' or engineitem == 'bingapi': - try: - bing_search = bingsearch.SearchBing(word, limit, start) - bingapi = '' - if engineitem == 'bingapi': - bingapi += 'yes' - else: - bingapi += 'no' - stor_lst.append( - store( - bing_search, - 'bing', - process_param=bingapi, - store_host=True, - store_emails=True, + elif engineitem == 'bevigil': + try: + bevigil_search = bevigil.SearchBeVigil(word) + stor_lst.append( + store( + bevigil_search, + engineitem, + store_host=True, + store_interestingurls=True, + ) ) - ) - except Exception as e: - if isinstance(e, MissingKey): + except Exception as e: print(e) - else: + + elif engineitem == 'binaryedge': + try: + binaryedge_search = binaryedgesearch.SearchBinaryEdge(word, limit) + stor_lst.append(store(binaryedge_search, engineitem, store_host=True)) + except Exception as e: print(e) - elif engineitem == 'bufferoverun': - try: - bufferoverun_search = bufferoverun.SearchBufferover(word) - stor_lst.append( - store( - bufferoverun_search, - engineitem, - store_host=True, - store_ip=True, + elif engineitem == 'bing' or engineitem == 'bingapi': + try: + bing_search = bingsearch.SearchBing(word, limit, start) + bingapi = '' + if engineitem == 'bingapi': + bingapi += 'yes' + else: + bingapi += 'no' + stor_lst.append( + store( + bing_search, + 'bing', + process_param=bingapi, + store_host=True, + store_emails=True, + ) ) - ) - except Exception as e: - print(e) + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + print(e) - elif engineitem == 'brave': - try: - brave_search = bravesearch.SearchBrave(word, limit) - stor_lst.append( - store( - brave_search, - engineitem, - store_host=True, - store_emails=True, - ) - ) - except Exception as e: - print(e) - - elif engineitem == 'censys': - try: - censys_search = censysearch.SearchCensys(word, limit) - stor_lst.append( - store( - censys_search, - engineitem, - store_host=True, - store_emails=True, + elif engineitem == 'bufferoverun': + try: + bufferoverun_search = bufferoverun.SearchBufferover(word) + stor_lst.append( + store( + bufferoverun_search, + engineitem, + store_host=True, + store_ip=True, + ) ) - ) - except Exception as e: - if isinstance(e, MissingKey): + except Exception as e: print(e) - elif engineitem == 'certspotter': - try: - certspotter_search = certspottersearch.SearchCertspoter(word) - stor_lst.append(store(certspotter_search, engineitem, None, store_host=True)) - except Exception as e: - print(e) - - elif engineitem == 'criminalip': - try: - criminalip_search = criminalip.SearchCriminalIP(word) - stor_lst.append( - store( - criminalip_search, - engineitem, - store_host=True, - store_ip=True, - store_asns=True, + elif engineitem == 'brave': + try: + brave_search = bravesearch.SearchBrave(word, limit) + stor_lst.append( + store( + brave_search, + engineitem, + store_host=True, + store_emails=True, + ) ) - ) - except Exception as e: - if isinstance(e, MissingKey): + except Exception as e: print(e) - else: - print(f'An excepion has occurred in criminalip: {e}') - elif engineitem == 'crtsh': - try: - crtsh_search = crtsh.SearchCrtsh(word) - stor_lst.append(store(crtsh_search, 'CRTsh', store_host=True)) - except Exception as e: - print(f'[!] A timeout occurred with crtsh, cannot find {args.domain}\n {e}') - - elif engineitem == 'dnsdumpster': - try: - dns_dumpster_search = dnsdumpster.SearchDnsDumpster(word) - stor_lst.append( - store( - dns_dumpster_search, - engineitem, - store_host=True, - store_ip=True, + elif engineitem == 'censys': + try: + censys_search = censysearch.SearchCensys(word, limit) + stor_lst.append( + store( + censys_search, + engineitem, + store_host=True, + store_emails=True, + ) ) - ) - except Exception as e: - print(f'[!] An error occurred with dnsdumpster: {e}') - - elif engineitem == 'duckduckgo': - duckduckgo_search = duckduckgosearch.SearchDuckDuckGo(word, limit) - stor_lst.append( - store( - duckduckgo_search, - engineitem, - store_host=True, - store_emails=True, - ) - ) + except Exception as e: + if isinstance(e, MissingKey): + print(e) - elif engineitem == 'fullhunt': - try: - fullhunt_search = fullhuntsearch.SearchFullHunt(word) - stor_lst.append(store(fullhunt_search, engineitem, store_host=True)) - except Exception as e: - if isinstance(e, MissingKey): + elif engineitem == 'certspotter': + try: + certspotter_search = certspottersearch.SearchCertspoter(word) + stor_lst.append(store(certspotter_search, engineitem, None, store_host=True)) + except Exception as e: print(e) - elif engineitem == 'github-code': - try: - github_search = githubcode.SearchGithubCode(word, limit) - stor_lst.append( - store( - github_search, - engineitem, - store_host=True, - store_emails=True, + elif engineitem == 'criminalip': + try: + criminalip_search = criminalip.SearchCriminalIP(word) + stor_lst.append( + store( + criminalip_search, + engineitem, + store_host=True, + store_ip=True, + store_asns=True, + ) ) - ) - except MissingKey as ex: - print(ex) + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + print(f'An excepion has occurred in criminalip: {e}') - elif engineitem == 'hackertarget': - hackertarget_search = hackertarget.SearchHackerTarget(word) - stor_lst.append(store(hackertarget_search, engineitem, store_host=True)) + elif engineitem == 'crtsh': + try: + crtsh_search = crtsh.SearchCrtsh(word) + stor_lst.append(store(crtsh_search, 'CRTsh', store_host=True)) + except Exception as e: + print(f'[!] A timeout occurred with crtsh, cannot find {args.domain}\n {e}') - elif engineitem == 'hunter': - try: - hunter_search = huntersearch.SearchHunter(word, limit, start) + elif engineitem == 'dnsdumpster': + try: + dns_dumpster_search = dnsdumpster.SearchDnsDumpster(word) + stor_lst.append( + store( + dns_dumpster_search, + engineitem, + store_host=True, + store_ip=True, + ) + ) + except Exception as e: + print(f'[!] An error occurred with dnsdumpster: {e}') + + elif engineitem == 'duckduckgo': + duckduckgo_search = duckduckgosearch.SearchDuckDuckGo(word, limit) stor_lst.append( store( - hunter_search, + duckduckgo_search, engineitem, store_host=True, store_emails=True, ) ) - except Exception as e: - if isinstance(e, MissingKey): - print(e) - elif engineitem == 'hunterhow': - try: - hunterhow_search = searchhunterhow.SearchHunterHow(word) - stor_lst.append(store(hunterhow_search, engineitem, store_host=True)) - except Exception as e: - if isinstance(e, MissingKey): - print(e) - else: - print(f'An exception has occurred in hunterhow search: {e}') + elif engineitem == 'fullhunt': + try: + fullhunt_search = fullhuntsearch.SearchFullHunt(word) + stor_lst.append(store(fullhunt_search, engineitem, store_host=True)) + except Exception as e: + if isinstance(e, MissingKey): + print(e) - elif engineitem == 'intelx': - try: - intelx_search = intelxsearch.SearchIntelx(word) - stor_lst.append( - store( - intelx_search, - engineitem, - store_interestingurls=True, - store_emails=True, + elif engineitem == 'github-code': + try: + github_search = githubcode.SearchGithubCode(word, limit) + stor_lst.append( + store( + github_search, + engineitem, + store_host=True, + store_emails=True, + ) ) - ) - except Exception as e: - if isinstance(e, MissingKey): - print(e) - else: - print(f'An exception has occurred in Intelx search: {e}') + except MissingKey as ex: + print(ex) - elif engineitem == 'netlas': - try: - netlas_search = netlas.SearchNetlas(word) - stor_lst.append( - store( - netlas_search, - engineitem, - store_host=True, - store_ip=True, + elif engineitem == 'hackertarget': + hackertarget_search = hackertarget.SearchHackerTarget(word) + stor_lst.append(store(hackertarget_search, engineitem, store_host=True)) + + elif engineitem == 'hunter': + try: + hunter_search = huntersearch.SearchHunter(word, limit, start) + stor_lst.append( + store( + hunter_search, + engineitem, + store_host=True, + store_emails=True, + ) ) - ) - except Exception as e: - if isinstance(e, MissingKey): - print(e) + except Exception as e: + if isinstance(e, MissingKey): + print(e) - elif engineitem == 'onyphe': - try: - onyphe_search = onyphe.SearchOnyphe(word) - stor_lst.append( - store( - onyphe_search, - engineitem, - store_host=True, - store_ip=True, - store_asns=True, + elif engineitem == 'hunterhow': + try: + hunterhow_search = searchhunterhow.SearchHunterHow(word) + stor_lst.append(store(hunterhow_search, engineitem, store_host=True)) + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + print(f'An exception has occurred in hunterhow search: {e}') + + elif engineitem == 'intelx': + try: + intelx_search = intelxsearch.SearchIntelx(word) + stor_lst.append( + store( + intelx_search, + engineitem, + store_interestingurls=True, + store_emails=True, + ) ) - ) - except Exception as e: - print(e) + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + print(f'An exception has occurred in Intelx search: {e}') - elif engineitem == 'otx': - try: - otxsearch_search = otxsearch.SearchOtx(word) - stor_lst.append( - store( - otxsearch_search, - engineitem, - store_host=True, - store_ip=True, + elif engineitem == 'netlas': + try: + netlas_search = netlas.SearchNetlas(word) + stor_lst.append( + store( + netlas_search, + engineitem, + store_host=True, + store_ip=True, + ) ) - ) - except Exception as e: - print(e) + except Exception as e: + if isinstance(e, MissingKey): + print(e) - elif engineitem == 'pentesttools': - try: - pentesttools_search = pentesttools.SearchPentestTools(word) - stor_lst.append(store(pentesttools_search, engineitem, store_host=True)) - except Exception as e: - if isinstance(e, MissingKey): + elif engineitem == 'onyphe': + try: + onyphe_search = onyphe.SearchOnyphe(word) + stor_lst.append( + store( + onyphe_search, + engineitem, + store_host=True, + store_ip=True, + store_asns=True, + ) + ) + except Exception as e: print(e) - else: - print(f'An exception has occurred in PentestTools search: {e}') - elif engineitem == 'projectdiscovery': - try: - projectdiscovery_search = projectdiscovery.SearchDiscovery(word) - stor_lst.append(store(projectdiscovery_search, engineitem, store_host=True)) - except Exception as e: - if isinstance(e, MissingKey): + elif engineitem == 'otx': + try: + otxsearch_search = otxsearch.SearchOtx(word) + stor_lst.append( + store( + otxsearch_search, + engineitem, + store_host=True, + store_ip=True, + ) + ) + except Exception as e: print(e) - else: - print('An exception has occurred in ProjectDiscovery') - elif engineitem == 'rapiddns': - try: - rapiddns_search = rapiddns.SearchRapidDns(word) - stor_lst.append(store(rapiddns_search, engineitem, store_host=True)) - except Exception as e: - print(e) - - elif engineitem == 'rocketreach': - try: - rocketreach_search = rocketreach.SearchRocketReach(word, limit) - stor_lst.append(store(rocketreach_search, engineitem, store_links=True)) - except Exception as e: - if isinstance(e, MissingKey): + elif engineitem == 'pentesttools': + try: + pentesttools_search = pentesttools.SearchPentestTools(word) + stor_lst.append(store(pentesttools_search, engineitem, store_host=True)) + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + print(f'An exception has occurred in PentestTools search: {e}') + + elif engineitem == 'projectdiscovery': + try: + projectdiscovery_search = projectdiscovery.SearchDiscovery(word) + stor_lst.append(store(projectdiscovery_search, engineitem, store_host=True)) + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + print('An exception has occurred in ProjectDiscovery') + + elif engineitem == 'rapiddns': + try: + rapiddns_search = rapiddns.SearchRapidDns(word) + stor_lst.append(store(rapiddns_search, engineitem, store_host=True)) + except Exception as e: print(e) - else: - print(f'An exception has occurred in RocketReach: {e}') - elif engineitem == 'subdomaincenter': - try: - subdomaincenter_search = subdomaincenter.SubdomainCenter(word) - stor_lst.append(store(subdomaincenter_search, engineitem, store_host=True)) - except Exception as e: - print(e) + elif engineitem == 'rocketreach': + try: + rocketreach_search = rocketreach.SearchRocketReach(word, limit) + stor_lst.append(store(rocketreach_search, engineitem, store_links=True)) + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + print(f'An exception has occurred in RocketReach: {e}') + + elif engineitem == 'subdomaincenter': + try: + subdomaincenter_search = subdomaincenter.SubdomainCenter(word) + stor_lst.append(store(subdomaincenter_search, engineitem, store_host=True)) + except Exception as e: + print(e) - elif engineitem == 'securityTrails': - try: - securitytrails_search = securitytrailssearch.SearchSecuritytrail(word) - stor_lst.append( - store( - securitytrails_search, - engineitem, - store_host=True, - store_ip=True, + elif engineitem == 'securityTrails': + try: + securitytrails_search = securitytrailssearch.SearchSecuritytrail(word) + stor_lst.append( + store( + securitytrails_search, + engineitem, + store_host=True, + store_ip=True, + ) ) - ) - except Exception as e: - if isinstance(e, MissingKey): - print(e) + except Exception as e: + if isinstance(e, MissingKey): + print(e) - elif engineitem == 'sitedossier': - try: - sitedossier_search = sitedossier.SearchSitedossier(word) - stor_lst.append(store(sitedossier_search, engineitem, store_host=True)) - except Exception as e: - print(e) + elif engineitem == 'sitedossier': + try: + sitedossier_search = sitedossier.SearchSitedossier(word) + stor_lst.append(store(sitedossier_search, engineitem, store_host=True)) + except Exception as e: + print(e) - elif engineitem == 'subdomainfinderc99': - try: - subdomainfinderc99_search = subdomainfinderc99.SearchSubdomainfinderc99(word) - stor_lst.append(store(subdomainfinderc99_search, engineitem, store_host=True)) - except Exception as e: - if isinstance(e, MissingKey): + elif engineitem == 'subdomainfinderc99': + try: + subdomainfinderc99_search = subdomainfinderc99.SearchSubdomainfinderc99(word) + stor_lst.append(store(subdomainfinderc99_search, engineitem, store_host=True)) + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + print(f'An exception has occurred in Subdomainfinderc99 search: {e}') + + elif engineitem == 'threatminer': + try: + threatminer_search = threatminer.SearchThreatminer(word) + stor_lst.append( + store( + threatminer_search, + engineitem, + store_host=True, + store_ip=True, + ) + ) + except Exception as e: print(e) - else: - print(f'An exception has occurred in Subdomainfinderc99 search: {e}') - elif engineitem == 'threatminer': - try: - threatminer_search = threatminer.SearchThreatminer(word) - stor_lst.append( - store( - threatminer_search, - engineitem, - store_host=True, - store_ip=True, + elif engineitem == 'tomba': + try: + tomba_search = tombasearch.SearchTomba(word, limit, start) + stor_lst.append( + store( + tomba_search, + engineitem, + store_host=True, + store_emails=True, + ) ) - ) - except Exception as e: - print(e) + except Exception as e: + if isinstance(e, MissingKey): + print(e) - elif engineitem == 'tomba': - try: - tomba_search = tombasearch.SearchTomba(word, limit, start) - stor_lst.append( - store( - tomba_search, - engineitem, - store_host=True, - store_emails=True, + elif engineitem == 'urlscan': + try: + urlscan_search = urlscan.SearchUrlscan(word) + stor_lst.append( + store( + urlscan_search, + engineitem, + store_host=True, + store_ip=True, + store_interestingurls=True, + store_asns=True, + ) ) - ) - except Exception as e: - if isinstance(e, MissingKey): + except Exception as e: print(e) - elif engineitem == 'urlscan': - try: - urlscan_search = urlscan.SearchUrlscan(word) - stor_lst.append( - store( - urlscan_search, - engineitem, - store_host=True, - store_ip=True, - store_interestingurls=True, - store_asns=True, - ) - ) - except Exception as e: - print(e) + elif engineitem == 'virustotal': + try: + virustotal_search = virustotal.SearchVirustotal(word) + stor_lst.append(store(virustotal_search, engineitem, store_host=True)) + except Exception as e: + if isinstance(e, MissingKey): + print(e) - elif engineitem == 'virustotal': - try: - virustotal_search = virustotal.SearchVirustotal(word) - stor_lst.append(store(virustotal_search, engineitem, store_host=True)) - except Exception as e: - if isinstance(e, MissingKey): + elif engineitem == 'yahoo': + try: + yahoo_search = yahoosearch.SearchYahoo(word, limit) + stor_lst.append( + store( + yahoo_search, + engineitem, + store_host=True, + store_emails=True, + ) + ) + except Exception as e: print(e) - elif engineitem == 'yahoo': - try: - yahoo_search = yahoosearch.SearchYahoo(word, limit) - stor_lst.append( - store( - yahoo_search, - engineitem, - store_host=True, - store_emails=True, + elif engineitem == 'zoomeye': + try: + zoomeye_search = zoomeyesearch.SearchZoomEye(word, limit) + stor_lst.append( + store( + zoomeye_search, + engineitem, + store_host=True, + store_emails=True, + store_ip=True, + store_interestingurls=True, + store_asns=True, + ) ) - ) - except Exception as e: - print(e) - - elif engineitem == 'zoomeye': + except Exception as e: + if isinstance(e, MissingKey): + print(e) + else: + if rest_args is not None: try: - zoomeye_search = zoomeyesearch.SearchZoomEye(word, limit) - stor_lst.append( - store( - zoomeye_search, - engineitem, - store_host=True, - store_emails=True, - store_ip=True, - store_interestingurls=True, - store_asns=True, - ) - ) - except Exception as e: - if isinstance(e, MissingKey): - print(e) - else: - if rest_args is not None: - try: - rest_args.dns_brute - except Exception: + rest_args.dns_brute + except Exception: + print('\n[!] Invalid source.\n') + sys.exit(1) + else: print('\n[!] Invalid source.\n') sys.exit(1) - else: - print('\n[!] Invalid source.\n') - sys.exit(1) async def worker(queue): while True: @@ -1069,7 +1099,7 @@ async def handler(lst): # run all the reversing tasks concurrently await asyncio.gather(*__reverse_dns_tasks.values()) - # Display the newly found hosts + # Displ-d/--domainay the newly found hosts print('\n[*] Hosts found after reverse lookup (in target domain):') print('--------------------------------------------------------') for xh in dnsrev: From 14771d51d99c13ac35b7683eb98dd3513e92c9f3 Mon Sep 17 00:00:00 2001 From: aizazalisyed Date: Fri, 19 Jul 2024 23:14:18 +0500 Subject: [PATCH 2/5] getting results for the domains in the file --- domains.txt | 1 + theHarvester/__main__.py | 344 ++++++++++++++++++++------------------- 2 files changed, 177 insertions(+), 168 deletions(-) diff --git a/domains.txt b/domains.txt index b0c279a18e7..a0efe2bae04 100644 --- a/domains.txt +++ b/domains.txt @@ -1,3 +1,4 @@ +x.com tesla.com pieas.com chichi.com diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index fa405b3c2c2..1d9c1ecac89 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -308,83 +308,91 @@ async def store( :param store_asns: whether to store asns """ - for word in domainLists: + # Initialize variables to store results for each domain + # all_hosts = [] + # all_emails = [] + # all_ip = [] + # all_people = [] + # all_links = [] + # all_interestingurls = [] + # all_asns = [] - await search_engine.process(use_proxy) if process_param is None else await search_engine.process(process_param, use_proxy) - db_stash = stash.StashManager() - - if source: - print(f'\033[94m[*] Searching {source[0].upper() + source[1:]}. ') - - if store_host: - host_names = list({host for host in await search_engine.get_hostnames() if f'.{word}' in host}) - host_names = list(host_names) - if source != 'hackertarget' and source != 'pentesttools' and source != 'rapiddns': - # If a source is inside this conditional, it means the hosts returned must be resolved to obtain ip - # This should only be checked if --dns-resolve has a wordlist - if dnsresolve is None or len(final_dns_resolver_list) > 0: - # indicates that -r was passed in if dnsresolve is None - full_hosts_checker = hostchecker.Checker(host_names, final_dns_resolver_list) - # If full, this is only getting resolved hosts - ( - resolved_pair, - temp_hosts, - temp_ips, - ) = await full_hosts_checker.check() - all_ip.extend(temp_ips) - full.extend(resolved_pair) - # full.extend(temp_hosts) - else: - full.extend(host_names) + + await search_engine.process(use_proxy) if process_param is None else await search_engine.process(process_param, use_proxy) + db_stash = stash.StashManager() + + if source: + print(f'\033[94m[*] Searching {source[0].upper() + source[1:]}. ') + + if store_host: + host_names = list({host for host in await search_engine.get_hostnames() if f'.{word}' in host}) + host_names = list(host_names) + if source != 'hackertarget' and source != 'pentesttools' and source != 'rapiddns': + # If a source is inside this conditional, it means the hosts returned must be resolved to obtain ip + # This should only be checked if --dns-resolve has a wordlist + if dnsresolve is None or len(final_dns_resolver_list) > 0: + # indicates that -r was passed in if dnsresolve is None + full_hosts_checker = hostchecker.Checker(host_names, final_dns_resolver_list) + # If full, this is only getting resolved hosts + ( + resolved_pair, + temp_hosts, + temp_ips, + ) = await full_hosts_checker.check() + all_ip.extend(temp_ips) + full.extend(resolved_pair) + # full.extend(temp_hosts) else: full.extend(host_names) - all_hosts.extend(host_names) - await db_stash.store_all(word, all_hosts, 'host', source) - - if store_emails: - email_list = await search_engine.get_emails() - all_emails.extend(email_list) - await db_stash.store_all(word, email_list, 'email', source) - - if store_ip: - ips_list = await search_engine.get_ips() - all_ip.extend(ips_list) - await db_stash.store_all(word, all_ip, 'ip', source) - - if store_results: - email_list, host_names, urls = await search_engine.get_results() - all_emails.extend(email_list) - host_names = list({host for host in host_names if f'.{word}' in host}) - all_urls.extend(urls) - all_hosts.extend(host_names) - await db.store_all(word, all_hosts, 'host', source) - await db.store_all(word, all_emails, 'email', source) - - if store_people: - people_list = await search_engine.get_people() - await db_stash.store_all(word, people_list, 'people', source) - - if store_links: - links = await search_engine.get_links() - linkedin_links_tracker.extend(links) - if len(links) > 0: - await db.store_all(word, links, 'linkedinlinks', engineitem) - - if store_interestingurls: - iurls = await search_engine.get_interestingurls() - interesting_urls.extend(iurls) - if len(iurls) > 0: - await db.store_all(word, iurls, 'interestingurls', engineitem) - - if store_asns: - fasns = await search_engine.get_asns() - total_asns.extend(fasns) - if len(fasns) > 0: - await db.store_all(word, fasns, 'asns', engineitem) + else: + full.extend(host_names) + all_hosts.extend(host_names) + await db_stash.store_all(word, all_hosts, 'host', source) + + if store_emails: + email_list = await search_engine.get_emails() + all_emails.extend(email_list) + await db_stash.store_all(word, email_list, 'email', source) + + if store_ip: + ips_list = await search_engine.get_ips() + all_ip.extend(ips_list) + await db_stash.store_all(word, all_ip, 'ip', source) + + if store_results: + email_list, host_names, urls = await search_engine.get_results() + all_emails.extend(email_list) + host_names = list({host for host in host_names if f'.{word}' in host}) + all_urls.extend(urls) + all_hosts.extend(host_names) + await db.store_all(word, all_hosts, 'host', source) + await db.store_all(word, all_emails, 'email', source) + + if store_people: + people_list = await search_engine.get_people() + await db_stash.store_all(word, people_list, 'people', source) + + if store_links: + links = await search_engine.get_links() + linkedin_links_tracker.extend(links) + if len(links) > 0: + await db.store_all(word, links, 'linkedinlinks', engineitem) + + if store_interestingurls: + iurls = await search_engine.get_interestingurls() + interesting_urls.extend(iurls) + if len(iurls) > 0: + await db.store_all(word, iurls, 'interestingurls', engineitem) + + if store_asns: + fasns = await search_engine.get_asns() + total_asns.extend(fasns) + if len(fasns) > 0: + await db.store_all(word, fasns, 'asns', engineitem) + stor_lst = [] for word in domainLists: - stor_lst = [] if args.source is not None: if args.source.lower() != 'all': engines = sorted(set(map(str.strip, args.source.split(',')))) @@ -398,7 +406,7 @@ async def store( if engineitem == 'anubis': try: anubis_search = anubis.SearchAnubis(word) - stor_lst.append(store(anubis_search, engineitem, store_host=True)) + stor_lst.append(await store(anubis_search, engineitem, store_host=True)) except Exception as e: print(e) @@ -406,7 +414,7 @@ async def store( try: baidu_search = baidusearch.SearchBaidu(word, limit) stor_lst.append( - store( + await store( baidu_search, engineitem, store_host=True, @@ -420,7 +428,7 @@ async def store( try: bevigil_search = bevigil.SearchBeVigil(word) stor_lst.append( - store( + await store( bevigil_search, engineitem, store_host=True, @@ -433,7 +441,7 @@ async def store( elif engineitem == 'binaryedge': try: binaryedge_search = binaryedgesearch.SearchBinaryEdge(word, limit) - stor_lst.append(store(binaryedge_search, engineitem, store_host=True)) + stor_lst.append(await store(binaryedge_search, engineitem, store_host=True)) except Exception as e: print(e) @@ -446,7 +454,7 @@ async def store( else: bingapi += 'no' stor_lst.append( - store( + await store( bing_search, 'bing', process_param=bingapi, @@ -464,7 +472,7 @@ async def store( try: bufferoverun_search = bufferoverun.SearchBufferover(word) stor_lst.append( - store( + await store( bufferoverun_search, engineitem, store_host=True, @@ -478,7 +486,7 @@ async def store( try: brave_search = bravesearch.SearchBrave(word, limit) stor_lst.append( - store( + await store( brave_search, engineitem, store_host=True, @@ -492,7 +500,7 @@ async def store( try: censys_search = censysearch.SearchCensys(word, limit) stor_lst.append( - store( + await store( censys_search, engineitem, store_host=True, @@ -506,7 +514,7 @@ async def store( elif engineitem == 'certspotter': try: certspotter_search = certspottersearch.SearchCertspoter(word) - stor_lst.append(store(certspotter_search, engineitem, None, store_host=True)) + stor_lst.append(await store(certspotter_search, engineitem, None, store_host=True)) except Exception as e: print(e) @@ -514,7 +522,7 @@ async def store( try: criminalip_search = criminalip.SearchCriminalIP(word) stor_lst.append( - store( + await store( criminalip_search, engineitem, store_host=True, @@ -531,7 +539,7 @@ async def store( elif engineitem == 'crtsh': try: crtsh_search = crtsh.SearchCrtsh(word) - stor_lst.append(store(crtsh_search, 'CRTsh', store_host=True)) + stor_lst.append(await store(crtsh_search, 'CRTsh', store_host=True)) except Exception as e: print(f'[!] A timeout occurred with crtsh, cannot find {args.domain}\n {e}') @@ -539,7 +547,7 @@ async def store( try: dns_dumpster_search = dnsdumpster.SearchDnsDumpster(word) stor_lst.append( - store( + await store( dns_dumpster_search, engineitem, store_host=True, @@ -552,7 +560,7 @@ async def store( elif engineitem == 'duckduckgo': duckduckgo_search = duckduckgosearch.SearchDuckDuckGo(word, limit) stor_lst.append( - store( + await store( duckduckgo_search, engineitem, store_host=True, @@ -563,7 +571,7 @@ async def store( elif engineitem == 'fullhunt': try: fullhunt_search = fullhuntsearch.SearchFullHunt(word) - stor_lst.append(store(fullhunt_search, engineitem, store_host=True)) + stor_lst.append(await store(fullhunt_search, engineitem, store_host=True)) except Exception as e: if isinstance(e, MissingKey): print(e) @@ -572,7 +580,7 @@ async def store( try: github_search = githubcode.SearchGithubCode(word, limit) stor_lst.append( - store( + await store( github_search, engineitem, store_host=True, @@ -584,13 +592,13 @@ async def store( elif engineitem == 'hackertarget': hackertarget_search = hackertarget.SearchHackerTarget(word) - stor_lst.append(store(hackertarget_search, engineitem, store_host=True)) + stor_lst.append(await store(hackertarget_search, engineitem, store_host=True)) elif engineitem == 'hunter': try: hunter_search = huntersearch.SearchHunter(word, limit, start) stor_lst.append( - store( + await store( hunter_search, engineitem, store_host=True, @@ -604,7 +612,7 @@ async def store( elif engineitem == 'hunterhow': try: hunterhow_search = searchhunterhow.SearchHunterHow(word) - stor_lst.append(store(hunterhow_search, engineitem, store_host=True)) + stor_lst.append(await store(hunterhow_search, engineitem, store_host=True)) except Exception as e: if isinstance(e, MissingKey): print(e) @@ -615,7 +623,7 @@ async def store( try: intelx_search = intelxsearch.SearchIntelx(word) stor_lst.append( - store( + await store( intelx_search, engineitem, store_interestingurls=True, @@ -632,7 +640,7 @@ async def store( try: netlas_search = netlas.SearchNetlas(word) stor_lst.append( - store( + await store( netlas_search, engineitem, store_host=True, @@ -647,7 +655,7 @@ async def store( try: onyphe_search = onyphe.SearchOnyphe(word) stor_lst.append( - store( + await store( onyphe_search, engineitem, store_host=True, @@ -662,7 +670,7 @@ async def store( try: otxsearch_search = otxsearch.SearchOtx(word) stor_lst.append( - store( + await store( otxsearch_search, engineitem, store_host=True, @@ -675,7 +683,7 @@ async def store( elif engineitem == 'pentesttools': try: pentesttools_search = pentesttools.SearchPentestTools(word) - stor_lst.append(store(pentesttools_search, engineitem, store_host=True)) + stor_lst.append(await store(pentesttools_search, engineitem, store_host=True)) except Exception as e: if isinstance(e, MissingKey): print(e) @@ -685,7 +693,7 @@ async def store( elif engineitem == 'projectdiscovery': try: projectdiscovery_search = projectdiscovery.SearchDiscovery(word) - stor_lst.append(store(projectdiscovery_search, engineitem, store_host=True)) + stor_lst.append(await store(projectdiscovery_search, engineitem, store_host=True)) except Exception as e: if isinstance(e, MissingKey): print(e) @@ -702,7 +710,7 @@ async def store( elif engineitem == 'rocketreach': try: rocketreach_search = rocketreach.SearchRocketReach(word, limit) - stor_lst.append(store(rocketreach_search, engineitem, store_links=True)) + stor_lst.append(await store(rocketreach_search, engineitem, store_links=True)) except Exception as e: if isinstance(e, MissingKey): print(e) @@ -712,7 +720,7 @@ async def store( elif engineitem == 'subdomaincenter': try: subdomaincenter_search = subdomaincenter.SubdomainCenter(word) - stor_lst.append(store(subdomaincenter_search, engineitem, store_host=True)) + stor_lst.append(await store(subdomaincenter_search, engineitem, store_host=True)) except Exception as e: print(e) @@ -720,7 +728,7 @@ async def store( try: securitytrails_search = securitytrailssearch.SearchSecuritytrail(word) stor_lst.append( - store( + await store( securitytrails_search, engineitem, store_host=True, @@ -734,14 +742,14 @@ async def store( elif engineitem == 'sitedossier': try: sitedossier_search = sitedossier.SearchSitedossier(word) - stor_lst.append(store(sitedossier_search, engineitem, store_host=True)) + stor_lst.append(await store(sitedossier_search, engineitem, store_host=True)) except Exception as e: print(e) elif engineitem == 'subdomainfinderc99': try: subdomainfinderc99_search = subdomainfinderc99.SearchSubdomainfinderc99(word) - stor_lst.append(store(subdomainfinderc99_search, engineitem, store_host=True)) + stor_lst.append(await store(subdomainfinderc99_search, engineitem, store_host=True)) except Exception as e: if isinstance(e, MissingKey): print(e) @@ -752,7 +760,7 @@ async def store( try: threatminer_search = threatminer.SearchThreatminer(word) stor_lst.append( - store( + await store( threatminer_search, engineitem, store_host=True, @@ -766,7 +774,7 @@ async def store( try: tomba_search = tombasearch.SearchTomba(word, limit, start) stor_lst.append( - store( + await store( tomba_search, engineitem, store_host=True, @@ -781,7 +789,7 @@ async def store( try: urlscan_search = urlscan.SearchUrlscan(word) stor_lst.append( - store( + await store( urlscan_search, engineitem, store_host=True, @@ -796,7 +804,7 @@ async def store( elif engineitem == 'virustotal': try: virustotal_search = virustotal.SearchVirustotal(word) - stor_lst.append(store(virustotal_search, engineitem, store_host=True)) + stor_lst.append(await store(virustotal_search, engineitem, store_host=True)) except Exception as e: if isinstance(e, MissingKey): print(e) @@ -805,7 +813,7 @@ async def store( try: yahoo_search = yahoosearch.SearchYahoo(word, limit) stor_lst.append( - store( + await store( yahoo_search, engineitem, store_host=True, @@ -819,7 +827,7 @@ async def store( try: zoomeye_search = zoomeyesearch.SearchZoomEye(word, limit) stor_lst.append( - store( + await store( zoomeye_search, engineitem, store_host=True, @@ -843,68 +851,68 @@ async def store( print('\n[!] Invalid source.\n') sys.exit(1) - async def worker(queue): - while True: - # Get a "work item" out of the queue. - stor = await queue.get() - try: - await stor - queue.task_done() - # Notify the queue that the "work item" has been processed. - except Exception: - queue.task_done() - - async def handler(lst): - queue = asyncio.Queue() - for stor_method in lst: - # enqueue the coroutines - queue.put_nowait(stor_method) - # Create three worker tasks to process the queue concurrently. - tasks = [] - for i in range(3): - task = asyncio.create_task(worker(queue)) - tasks.append(task) - - # Wait until the queue is fully processed. - await queue.join() - - # Cancel our worker tasks. - for task in tasks: - task.cancel() - # Wait until all worker tasks are cancelled. - await asyncio.gather(*tasks, return_exceptions=True) - - await handler(lst=stor_lst) - return_ips: list = [] - if rest_args is not None and len(rest_filename) == 0 and rest_args.dns_brute is False: - # Indicates user is using REST api but not wanting output to be saved to a file - # cast to string so Rest API can understand the type - return_ips.extend([str(ip) for ip in sorted([netaddr.IPAddress(ip.strip()) for ip in set(all_ip)])]) - # return list(set(all_emails)), return_ips, full, '', '' - all_hosts = [host.replace('www.', '') for host in all_hosts if host.replace('www.', '') in all_hosts] - all_hosts = list(sorted(set(all_hosts))) - return ( - total_asns, - interesting_urls, - twitter_people_list_tracker, - linkedin_people_list_tracker, - linkedin_links_tracker, - all_urls, - all_ip, - all_emails, - all_hosts, - ) - # Check to see if all_emails and all_hosts are defined. - try: - all_emails - except NameError: - print('\n\n[!] No emails found because all_emails is not defined.\n\n ') - sys.exit(1) - try: - all_hosts - except NameError: - print('\n\n[!] No hosts found because all_hosts is not defined.\n\n ') - sys.exit(1) + async def worker(queue): + while True: + # Get a "work item" out of the queue. + stor = await queue.get() + try: + await stor + queue.task_done() + # Notify the queue that the "work item" has been processed. + except Exception: + queue.task_done() + + async def handler(lst): + queue = asyncio.Queue() + for stor_method in lst: + # enqueue the coroutines + queue.put_nowait(stor_method) + # Create three worker tasks to process the queue concurrently. + tasks = [] + for i in range(3): + task = asyncio.create_task(worker(queue)) + tasks.append(task) + + # Wait until the queue is fully processed. + await queue.join() + + # Cancel our worker tasks. + for task in tasks: + task.cancel() + # Wait until all worker tasks are cancelled. + await asyncio.gather(*tasks, return_exceptions=True) + + await handler(lst=stor_lst) + return_ips: list = [] + if rest_args is not None and len(rest_filename) == 0 and rest_args.dns_brute is False: + # Indicates user is using REST api but not wanting output to be saved to a file + # cast to string so Rest API can understand the type + return_ips.extend([str(ip) for ip in sorted([netaddr.IPAddress(ip.strip()) for ip in set(all_ip)])]) + # return list(set(all_emails)), return_ips, full, '', '' + all_hosts = [host.replace('www.', '') for host in all_hosts if host.replace('www.', '') in all_hosts] + all_hosts = list(sorted(set(all_hosts))) + return ( + total_asns, + interesting_urls, + twitter_people_list_tracker, + linkedin_people_list_tracker, + linkedin_links_tracker, + all_urls, + all_ip, + all_emails, + all_hosts, + ) + # Check to see if all_emails and all_hosts are defined. + try: + all_emails + except NameError: + print('\n\n[!] No emails found because all_emails is not defined.\n\n ') + sys.exit(1) + try: + all_hosts + except NameError: + print('\n\n[!] No hosts found because all_hosts is not defined.\n\n ') + sys.exit(1) # Results if len(total_asns) > 0: From 3c65f3d22c57397c5180a8319ad3ed942056df71 Mon Sep 17 00:00:00 2001 From: aizazalisyed Date: Fri, 19 Jul 2024 23:28:25 +0500 Subject: [PATCH 3/5] [Feature Request] Multiple Domains loaded from file #418 --- theHarvester/__main__.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index 1d9c1ecac89..c897cff6cf8 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -308,16 +308,6 @@ async def store( :param store_asns: whether to store asns """ - # Initialize variables to store results for each domain - # all_hosts = [] - # all_emails = [] - # all_ip = [] - # all_people = [] - # all_links = [] - # all_interestingurls = [] - # all_asns = [] - - await search_engine.process(use_proxy) if process_param is None else await search_engine.process(process_param, use_proxy) db_stash = stash.StashManager() From 6d447146843291fc0a8a2affbebcf6d618a81688 Mon Sep 17 00:00:00 2001 From: aizazalisyed Date: Sat, 20 Jul 2024 21:13:49 +0500 Subject: [PATCH 4/5] fixed ruff lint issue and missing modules --- .gitignore | 1 + domains.txt | 5 ++--- theHarvester/__main__.py | 10 ++++------ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 58f8c14611c..eb5fe0572e3 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ api-keys.yaml .DS_Store .venv .pyre +domains.txt \ No newline at end of file diff --git a/domains.txt b/domains.txt index a0efe2bae04..2fa26c15749 100644 --- a/domains.txt +++ b/domains.txt @@ -1,5 +1,4 @@ -x.com tesla.com -pieas.com -chichi.com + + diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index c897cff6cf8..f49c3af889d 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -180,18 +180,18 @@ async def start(rest_args: argparse.Namespace | None = None): args = parser.parse_args() filename = args.filename dnsbrute = (args.dns_brute, False) - + # Check if neither -d/--domain nor --domainlist is provided if not args.domain and not args.domainlist: parser.print_help() - print("\nError: Either -d/--domain or --domainlist must be provided.") + print('\nError: Either -d/--domain or --domainlist must be provided.') sys.exit(1) # Initialize domainLists from file if --domainlist is provided if args.domainlist: domain_list_file = args.domainlist try: - with open(domain_list_file, 'r') as f: + with open(domain_list_file) as f: domainLists = [line.strip() for line in f.readlines() if line.strip()] except FileNotFoundError: print(f"Error: File '{domain_list_file}' not found.") @@ -199,7 +199,6 @@ async def start(rest_args: argparse.Namespace | None = None): else: domainLists = [args.domain] if args.domain else [] - try: db = stash.StashManager() await db.do_init() @@ -307,7 +306,7 @@ async def store( :param store_interestingurls: whether to store interesting urls :param store_asns: whether to store asns """ - + await search_engine.process(use_proxy) if process_param is None else await search_engine.process(process_param, use_proxy) db_stash = stash.StashManager() @@ -380,7 +379,6 @@ async def store( if len(fasns) > 0: await db.store_all(word, fasns, 'asns', engineitem) - stor_lst = [] for word in domainLists: if args.source is not None: From c79708e55a04a2342a825ff470be9f8b45fe07d0 Mon Sep 17 00:00:00 2001 From: aizazalisyed Date: Sun, 21 Jul 2024 00:21:13 +0500 Subject: [PATCH 5/5] resolved static type checking with mypy --- .gitignore | 3 +- domains.txt | 4 --- theHarvester/__main__.py | 70 ++++++++++++++++++++-------------------- 3 files changed, 36 insertions(+), 41 deletions(-) delete mode 100644 domains.txt diff --git a/.gitignore b/.gitignore index eb5fe0572e3..fdfdaf8ead4 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,4 @@ theHarvester.egg-info api-keys.yaml .DS_Store .venv -.pyre -domains.txt \ No newline at end of file +.pyre \ No newline at end of file diff --git a/domains.txt b/domains.txt deleted file mode 100644 index 2fa26c15749..00000000000 --- a/domains.txt +++ /dev/null @@ -1,4 +0,0 @@ -tesla.com - - - diff --git a/theHarvester/__main__.py b/theHarvester/__main__.py index f49c3af889d..9e12d82297c 100644 --- a/theHarvester/__main__.py +++ b/theHarvester/__main__.py @@ -394,7 +394,7 @@ async def store( if engineitem == 'anubis': try: anubis_search = anubis.SearchAnubis(word) - stor_lst.append(await store(anubis_search, engineitem, store_host=True)) + stor_lst.append(store(anubis_search, engineitem, store_host=True)) except Exception as e: print(e) @@ -402,7 +402,7 @@ async def store( try: baidu_search = baidusearch.SearchBaidu(word, limit) stor_lst.append( - await store( + store( baidu_search, engineitem, store_host=True, @@ -416,7 +416,7 @@ async def store( try: bevigil_search = bevigil.SearchBeVigil(word) stor_lst.append( - await store( + store( bevigil_search, engineitem, store_host=True, @@ -429,7 +429,7 @@ async def store( elif engineitem == 'binaryedge': try: binaryedge_search = binaryedgesearch.SearchBinaryEdge(word, limit) - stor_lst.append(await store(binaryedge_search, engineitem, store_host=True)) + stor_lst.append(store(binaryedge_search, engineitem, store_host=True)) except Exception as e: print(e) @@ -442,7 +442,7 @@ async def store( else: bingapi += 'no' stor_lst.append( - await store( + store( bing_search, 'bing', process_param=bingapi, @@ -460,7 +460,7 @@ async def store( try: bufferoverun_search = bufferoverun.SearchBufferover(word) stor_lst.append( - await store( + store( bufferoverun_search, engineitem, store_host=True, @@ -474,7 +474,7 @@ async def store( try: brave_search = bravesearch.SearchBrave(word, limit) stor_lst.append( - await store( + store( brave_search, engineitem, store_host=True, @@ -488,7 +488,7 @@ async def store( try: censys_search = censysearch.SearchCensys(word, limit) stor_lst.append( - await store( + store( censys_search, engineitem, store_host=True, @@ -502,7 +502,7 @@ async def store( elif engineitem == 'certspotter': try: certspotter_search = certspottersearch.SearchCertspoter(word) - stor_lst.append(await store(certspotter_search, engineitem, None, store_host=True)) + stor_lst.append(store(certspotter_search, engineitem, None, store_host=True)) except Exception as e: print(e) @@ -510,7 +510,7 @@ async def store( try: criminalip_search = criminalip.SearchCriminalIP(word) stor_lst.append( - await store( + store( criminalip_search, engineitem, store_host=True, @@ -527,7 +527,7 @@ async def store( elif engineitem == 'crtsh': try: crtsh_search = crtsh.SearchCrtsh(word) - stor_lst.append(await store(crtsh_search, 'CRTsh', store_host=True)) + stor_lst.append(store(crtsh_search, 'CRTsh', store_host=True)) except Exception as e: print(f'[!] A timeout occurred with crtsh, cannot find {args.domain}\n {e}') @@ -535,7 +535,7 @@ async def store( try: dns_dumpster_search = dnsdumpster.SearchDnsDumpster(word) stor_lst.append( - await store( + store( dns_dumpster_search, engineitem, store_host=True, @@ -548,7 +548,7 @@ async def store( elif engineitem == 'duckduckgo': duckduckgo_search = duckduckgosearch.SearchDuckDuckGo(word, limit) stor_lst.append( - await store( + store( duckduckgo_search, engineitem, store_host=True, @@ -559,7 +559,7 @@ async def store( elif engineitem == 'fullhunt': try: fullhunt_search = fullhuntsearch.SearchFullHunt(word) - stor_lst.append(await store(fullhunt_search, engineitem, store_host=True)) + stor_lst.append(store(fullhunt_search, engineitem, store_host=True)) except Exception as e: if isinstance(e, MissingKey): print(e) @@ -568,7 +568,7 @@ async def store( try: github_search = githubcode.SearchGithubCode(word, limit) stor_lst.append( - await store( + store( github_search, engineitem, store_host=True, @@ -580,13 +580,13 @@ async def store( elif engineitem == 'hackertarget': hackertarget_search = hackertarget.SearchHackerTarget(word) - stor_lst.append(await store(hackertarget_search, engineitem, store_host=True)) + stor_lst.append(store(hackertarget_search, engineitem, store_host=True)) elif engineitem == 'hunter': try: hunter_search = huntersearch.SearchHunter(word, limit, start) stor_lst.append( - await store( + store( hunter_search, engineitem, store_host=True, @@ -600,7 +600,7 @@ async def store( elif engineitem == 'hunterhow': try: hunterhow_search = searchhunterhow.SearchHunterHow(word) - stor_lst.append(await store(hunterhow_search, engineitem, store_host=True)) + stor_lst.append(store(hunterhow_search, engineitem, store_host=True)) except Exception as e: if isinstance(e, MissingKey): print(e) @@ -611,7 +611,7 @@ async def store( try: intelx_search = intelxsearch.SearchIntelx(word) stor_lst.append( - await store( + store( intelx_search, engineitem, store_interestingurls=True, @@ -628,7 +628,7 @@ async def store( try: netlas_search = netlas.SearchNetlas(word) stor_lst.append( - await store( + store( netlas_search, engineitem, store_host=True, @@ -643,7 +643,7 @@ async def store( try: onyphe_search = onyphe.SearchOnyphe(word) stor_lst.append( - await store( + store( onyphe_search, engineitem, store_host=True, @@ -658,7 +658,7 @@ async def store( try: otxsearch_search = otxsearch.SearchOtx(word) stor_lst.append( - await store( + store( otxsearch_search, engineitem, store_host=True, @@ -671,7 +671,7 @@ async def store( elif engineitem == 'pentesttools': try: pentesttools_search = pentesttools.SearchPentestTools(word) - stor_lst.append(await store(pentesttools_search, engineitem, store_host=True)) + stor_lst.append(store(pentesttools_search, engineitem, store_host=True)) except Exception as e: if isinstance(e, MissingKey): print(e) @@ -681,7 +681,7 @@ async def store( elif engineitem == 'projectdiscovery': try: projectdiscovery_search = projectdiscovery.SearchDiscovery(word) - stor_lst.append(await store(projectdiscovery_search, engineitem, store_host=True)) + stor_lst.append(store(projectdiscovery_search, engineitem, store_host=True)) except Exception as e: if isinstance(e, MissingKey): print(e) @@ -698,7 +698,7 @@ async def store( elif engineitem == 'rocketreach': try: rocketreach_search = rocketreach.SearchRocketReach(word, limit) - stor_lst.append(await store(rocketreach_search, engineitem, store_links=True)) + stor_lst.append(store(rocketreach_search, engineitem, store_links=True)) except Exception as e: if isinstance(e, MissingKey): print(e) @@ -708,7 +708,7 @@ async def store( elif engineitem == 'subdomaincenter': try: subdomaincenter_search = subdomaincenter.SubdomainCenter(word) - stor_lst.append(await store(subdomaincenter_search, engineitem, store_host=True)) + stor_lst.append(store(subdomaincenter_search, engineitem, store_host=True)) except Exception as e: print(e) @@ -716,7 +716,7 @@ async def store( try: securitytrails_search = securitytrailssearch.SearchSecuritytrail(word) stor_lst.append( - await store( + store( securitytrails_search, engineitem, store_host=True, @@ -730,14 +730,14 @@ async def store( elif engineitem == 'sitedossier': try: sitedossier_search = sitedossier.SearchSitedossier(word) - stor_lst.append(await store(sitedossier_search, engineitem, store_host=True)) + stor_lst.append(store(sitedossier_search, engineitem, store_host=True)) except Exception as e: print(e) elif engineitem == 'subdomainfinderc99': try: subdomainfinderc99_search = subdomainfinderc99.SearchSubdomainfinderc99(word) - stor_lst.append(await store(subdomainfinderc99_search, engineitem, store_host=True)) + stor_lst.append(store(subdomainfinderc99_search, engineitem, store_host=True)) except Exception as e: if isinstance(e, MissingKey): print(e) @@ -748,7 +748,7 @@ async def store( try: threatminer_search = threatminer.SearchThreatminer(word) stor_lst.append( - await store( + store( threatminer_search, engineitem, store_host=True, @@ -762,7 +762,7 @@ async def store( try: tomba_search = tombasearch.SearchTomba(word, limit, start) stor_lst.append( - await store( + store( tomba_search, engineitem, store_host=True, @@ -777,7 +777,7 @@ async def store( try: urlscan_search = urlscan.SearchUrlscan(word) stor_lst.append( - await store( + store( urlscan_search, engineitem, store_host=True, @@ -792,7 +792,7 @@ async def store( elif engineitem == 'virustotal': try: virustotal_search = virustotal.SearchVirustotal(word) - stor_lst.append(await store(virustotal_search, engineitem, store_host=True)) + stor_lst.append(store(virustotal_search, engineitem, store_host=True)) except Exception as e: if isinstance(e, MissingKey): print(e) @@ -801,7 +801,7 @@ async def store( try: yahoo_search = yahoosearch.SearchYahoo(word, limit) stor_lst.append( - await store( + store( yahoo_search, engineitem, store_host=True, @@ -815,7 +815,7 @@ async def store( try: zoomeye_search = zoomeyesearch.SearchZoomEye(word, limit) stor_lst.append( - await store( + store( zoomeye_search, engineitem, store_host=True,