diff --git a/ahmia/ahmia/items.py b/ahmia/ahmia/items.py index 9dd85d4..5e462c3 100644 --- a/ahmia/ahmia/items.py +++ b/ahmia/ahmia/items.py @@ -23,6 +23,8 @@ class DocumentItem(Item): title = Field(input_processor=MapCompose(remove_control_chars), output_processor=TakeFirst()) meta = Field(input_processor=MapCompose(remove_control_chars), output_processor=TakeFirst()) content = Field(input_processor=MapCompose(remove_control_chars), output_processor=TakeFirst()) + clean_content = Field(input_processor=TakeFirst()) + content_hash = Field(input_processor=TakeFirst()) domain = Field(output_processor=TakeFirst()) content_type = Field(output_processor=TakeFirst()) updated_on = Field(output_processor=TakeFirst()) diff --git a/ahmia/ahmia/pipelines.py b/ahmia/ahmia/pipelines.py index bcd0225..5339dc1 100644 --- a/ahmia/ahmia/pipelines.py +++ b/ahmia/ahmia/pipelines.py @@ -2,6 +2,7 @@ """ Pipelines """ import hashlib import logging +import re from elasticsearch import Elasticsearch from elasticsearch.helpers import bulk from .items import DocumentItem @@ -40,6 +41,68 @@ def process_item(self, item): self.index_item(item) return item # To continue passing items through other pipelines + def preprocess(self, text): + + # 小文字化（大文字・小文字の違いを統一） + text = text.lower() + + # 改行・タブ・複数スペースを1つの空白に統一 + text = re.sub(r'\s+', ' ', text) + + # onionアドレスを削除（サイト固有の識別情報を除去） + text = re.sub( + r'[a-z2-7]{56}\.onion', + '', + text + ) + + # URLを削除（リンク先の違いによる差を除去） + text = re.sub( + r'https?://\S+', + '', + text + ) + + # Bitcoinアドレスを削除（決済情報などサイト固有情報を除去） + text = re.sub( + r'\b(bc1|[13])[a-zA-HJ-NP-Z0-9]{25,62}\b', + '', + text + ) + + # メールアドレスを削除（連絡先情報を除去） + text = re.sub( + r'\S+@\S+\.\S+', + '', + text + ) + + # 本文と無関係なテンプレート由来の脚注・用語集ノイズを除去 + text = re.sub( + r'\*\[[^\]]+\]:\s*[^*]*', + '', + text + ) + + # 連続する特殊文字を削除（Markdown記号などのノイズ除去） + text = re.sub( + r'[*#|_\-]{2,}', + '', + text + ) + + # 再度、余分な空白を整理 + text = re.sub( + r'\s+', + ' ', + text + ) + + # 前後の空白を削除 + text = text.strip() + + return text + def index_item(self, item): """ Items are indexed here. @@ -49,6 +112,18 @@ def index_item(self, item): doc_id = hashlib.sha1(item['url'].encode('utf-8')).hexdigest() if isinstance(item, DocumentItem): + + content = item.get('content', '') + + if content: + # 前処理 + clean_content = self.preprocess(content) + item['clean_content'] = clean_content + # ハッシュ値の生成 + item['content_hash'] = hashlib.sha256( + clean_content.encode('utf-8') + ).hexdigest() + action = { "_index": self.index_name, "_id": doc_id, diff --git a/ahmia/ahmia/settings.py b/ahmia/ahmia/settings.py index ab097aa..ab9b96c 100644 --- a/ahmia/ahmia/settings.py +++ b/ahmia/ahmia/settings.py @@ -93,7 +93,7 @@ } # Tor proxy settings: http://localhost:15000 - http://localhost:15099 -HTTP_PROXY_TOR_PROXIES = [f"http://localhost:150{i:02}" for i in range(0, 100)] +HTTP_PROXY_TOR_PROXIES = [f"http://localhost:150{i:02}" for i in range(0, 50)] def extract_onions_from_url(url, timeout=120): """ Helper function to extract unique onion base URLs """ diff --git a/ahmia/ahmia/spiders/onionspider.py b/ahmia/ahmia/spiders/onionspider.py index 534f101..0c15c13 100644 --- a/ahmia/ahmia/spiders/onionspider.py +++ b/ahmia/ahmia/spiders/onionspider.py @@ -20,7 +20,7 @@ class OnionSpider(CrawlSpider): LARGE_TEXT_BYTES = 2 * 1024 * 1024 MAX_EXTRACTED_LINKS_PER_DOMAIN = 50000 - + # 集めるサイトの条件指定 rules = ( Rule( LinkExtractor( @@ -30,7 +30,7 @@ class OnionSpider(CrawlSpider): "jpg", "jpeg", "mp3", "mp4", "m4a", "ogg", "pdf", "png", "rar", "svg", "tar", "tgz", "webm", "webp", "xz", "zip" ], - unique=True, + unique=True, # 同一リンクは排除 canonicalize=True, ), callback="parse_item", @@ -39,6 +39,7 @@ class OnionSpider(CrawlSpider): ), ) + # 初期設定 def __init__(self, *args, seedlist=None, **kwargs): """ Init """ super().__init__(*args, **kwargs) @@ -47,7 +48,7 @@ def __init__(self, *args, seedlist=None, **kwargs): self.html_converter.ignore_links = True self.html_converter.ignore_images = True - self._extracted_links_per_domain = defaultdict(int) + self._extracted_links_per_domain = defaultdict(int) # リンク内にあったリンク数を保存する変数 self._current_response_host = None if seedlist: @@ -58,6 +59,7 @@ def __init__(self, *args, seedlist=None, **kwargs): self.start_urls = get_project_settings().get("SEEDLIST", []) self.logger.info("Using SEEDLIST from settings.py with %d URLs.", len(self.start_urls)) + # 1ドメインから取得するリンク数の制限 def limit_links_per_domain(self, links): """Limit extracted links to 50,000 per one domain.""" kept = [] @@ -156,6 +158,7 @@ def _safe_html2text(self, response): ) return "" + # 取得したHTMLをparseItemに変換 def parse_item(self, response): """Parse items.""" self._current_response_host = (urlparse(response.url).hostname or "").lower() diff --git a/run.sh b/run.sh old mode 100755 new mode 100644 diff --git a/run_daily.sh b/run_daily.sh old mode 100755 new mode 100644 diff --git a/torfleet/runfleet.sh b/torfleet/runfleet.sh index c8d7ec3..fc250fe 100644 --- a/torfleet/runfleet.sh +++ b/torfleet/runfleet.sh @@ -8,7 +8,7 @@ DATA_DIR="$BASE_DIR/data/tor" TORRC_DIR="$BASE_DIR/torrcs" PRIVOXY_DIR="$BASE_DIR/privoxy_configs" -NUM_INSTANCES=100 +NUM_INSTANCES=50 BASE_SOCKS_PORT=19050 # Tor SOCKS5 BASE_HTTP_PORT=15000 # Privoxy HTTP