From e244329b46eb18ce06667a4e969b96b2ec5c3f5d Mon Sep 17 00:00:00 2001 From: takuto050204 Date: Sun, 19 Jul 2026 15:53:41 +0900 Subject: [PATCH 1/6] first commit --- ahmia/ahmia/spiders/onionspider.py | 9 ++++++--- torfleet/runfleet.sh | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) mode change 100644 => 100755 torfleet/runfleet.sh diff --git a/ahmia/ahmia/spiders/onionspider.py b/ahmia/ahmia/spiders/onionspider.py index 534f101..0c15c13 100644 --- a/ahmia/ahmia/spiders/onionspider.py +++ b/ahmia/ahmia/spiders/onionspider.py @@ -20,7 +20,7 @@ class OnionSpider(CrawlSpider): LARGE_TEXT_BYTES = 2 * 1024 * 1024 MAX_EXTRACTED_LINKS_PER_DOMAIN = 50000 - + # 集めるサイトの条件指定 rules = ( Rule( LinkExtractor( @@ -30,7 +30,7 @@ class OnionSpider(CrawlSpider): "jpg", "jpeg", "mp3", "mp4", "m4a", "ogg", "pdf", "png", "rar", "svg", "tar", "tgz", "webm", "webp", "xz", "zip" ], - unique=True, + unique=True, # 同一リンクは排除 canonicalize=True, ), callback="parse_item", @@ -39,6 +39,7 @@ class OnionSpider(CrawlSpider): ), ) + # 初期設定 def __init__(self, *args, seedlist=None, **kwargs): """ Init """ super().__init__(*args, **kwargs) @@ -47,7 +48,7 @@ def __init__(self, *args, seedlist=None, **kwargs): self.html_converter.ignore_links = True self.html_converter.ignore_images = True - self._extracted_links_per_domain = defaultdict(int) + self._extracted_links_per_domain = defaultdict(int) # リンク内にあったリンク数を保存する変数 self._current_response_host = None if seedlist: @@ -58,6 +59,7 @@ def __init__(self, *args, seedlist=None, **kwargs): self.start_urls = get_project_settings().get("SEEDLIST", []) self.logger.info("Using SEEDLIST from settings.py with %d URLs.", len(self.start_urls)) + # 1ドメインから取得するリンク数の制限 def limit_links_per_domain(self, links): """Limit extracted links to 50,000 per one domain.""" kept = [] @@ -156,6 +158,7 @@ def _safe_html2text(self, response): ) return "" + # 取得したHTMLをparseItemに変換 def parse_item(self, response): """Parse items.""" self._current_response_host = (urlparse(response.url).hostname or "").lower() diff --git a/torfleet/runfleet.sh b/torfleet/runfleet.sh old mode 100644 new mode 100755 index c8d7ec3..6734854 --- a/torfleet/runfleet.sh +++ b/torfleet/runfleet.sh @@ -8,7 +8,7 @@ DATA_DIR="$BASE_DIR/data/tor" TORRC_DIR="$BASE_DIR/torrcs" PRIVOXY_DIR="$BASE_DIR/privoxy_configs" -NUM_INSTANCES=100 +NUM_INSTANCES=5 # 100から5に変更した BASE_SOCKS_PORT=19050 # Tor SOCKS5 BASE_HTTP_PORT=15000 # Privoxy HTTP From 3e49302522ee57e527e2d8aa5982b1a914950feb Mon Sep 17 00:00:00 2001 From: takuto050204 Date: Sun, 26 Jul 2026 11:28:40 +0900 Subject: [PATCH 2/6] =?UTF-8?q?=E3=83=8F=E3=83=83=E3=82=B7=E3=83=A5?= =?UTF-8?q?=E5=80=A4=E3=81=AE=E8=A8=88=E7=AE=97=E5=AE=9F=E8=A3=85=EF=BC=88?= =?UTF-8?q?=E5=B0=8F=E8=A6=8F=E6=A8=A1=E3=82=AF=E3=83=AD=E3=83=BC=E3=83=AB?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ahmia/ahmia/items.py | 1 + ahmia/ahmia/pipelines.py | 9 +++++++++ ahmia/ahmia/settings.py | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ahmia/ahmia/items.py b/ahmia/ahmia/items.py index 9dd85d4..7eaf56a 100644 --- a/ahmia/ahmia/items.py +++ b/ahmia/ahmia/items.py @@ -23,6 +23,7 @@ class DocumentItem(Item): title = Field(input_processor=MapCompose(remove_control_chars), output_processor=TakeFirst()) meta = Field(input_processor=MapCompose(remove_control_chars), output_processor=TakeFirst()) content = Field(input_processor=MapCompose(remove_control_chars), output_processor=TakeFirst()) + content_hash = Field(input_processor=TakeFirst()) domain = Field(output_processor=TakeFirst()) content_type = Field(output_processor=TakeFirst()) updated_on = Field(output_processor=TakeFirst()) diff --git a/ahmia/ahmia/pipelines.py b/ahmia/ahmia/pipelines.py index bcd0225..0034018 100644 --- a/ahmia/ahmia/pipelines.py +++ b/ahmia/ahmia/pipelines.py @@ -49,6 +49,15 @@ def index_item(self, item): doc_id = hashlib.sha1(item['url'].encode('utf-8')).hexdigest() if isinstance(item, DocumentItem): + + # ハッシュ値の生成 + content = item.get('content', '') + + if content: + item['content_hash'] = hashlib.sha256( + content.encode('utf-8') + ).hexdigest() + action = { "_index": self.index_name, "_id": doc_id, diff --git a/ahmia/ahmia/settings.py b/ahmia/ahmia/settings.py index ab097aa..c51ea81 100644 --- a/ahmia/ahmia/settings.py +++ b/ahmia/ahmia/settings.py @@ -93,7 +93,7 @@ } # Tor proxy settings: http://localhost:15000 - http://localhost:15099 -HTTP_PROXY_TOR_PROXIES = [f"http://localhost:150{i:02}" for i in range(0, 100)] +HTTP_PROXY_TOR_PROXIES = [f"http://localhost:150{i:02}" for i in range(0, 5)] # いったん100->5に変更中 def extract_onions_from_url(url, timeout=120): """ Helper function to extract unique onion base URLs """ From 7852d70363b5336f6a3c6f68926ade6274075e67 Mon Sep 17 00:00:00 2001 From: takuto050204 Date: Sun, 26 Jul 2026 16:02:24 +0900 Subject: [PATCH 3/6] =?UTF-8?q?50=E3=81=93=E3=81=AETor=E3=81=A7=E3=82=AF?= =?UTF-8?q?=E3=83=AD=E3=83=BC=E3=83=AB=E3=81=A7=E3=81=8D=E3=82=8B=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ahmia/ahmia/settings.py | 2 +- torfleet/runfleet.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ahmia/ahmia/settings.py b/ahmia/ahmia/settings.py index c51ea81..ab9b96c 100644 --- a/ahmia/ahmia/settings.py +++ b/ahmia/ahmia/settings.py @@ -93,7 +93,7 @@ } # Tor proxy settings: http://localhost:15000 - http://localhost:15099 -HTTP_PROXY_TOR_PROXIES = [f"http://localhost:150{i:02}" for i in range(0, 5)] # いったん100->5に変更中 +HTTP_PROXY_TOR_PROXIES = [f"http://localhost:150{i:02}" for i in range(0, 50)] def extract_onions_from_url(url, timeout=120): """ Helper function to extract unique onion base URLs """ diff --git a/torfleet/runfleet.sh b/torfleet/runfleet.sh index 6734854..fc250fe 100755 --- a/torfleet/runfleet.sh +++ b/torfleet/runfleet.sh @@ -8,7 +8,7 @@ DATA_DIR="$BASE_DIR/data/tor" TORRC_DIR="$BASE_DIR/torrcs" PRIVOXY_DIR="$BASE_DIR/privoxy_configs" -NUM_INSTANCES=5 # 100から5に変更した +NUM_INSTANCES=50 BASE_SOCKS_PORT=19050 # Tor SOCKS5 BASE_HTTP_PORT=15000 # Privoxy HTTP From 8b7318b7403d829967010234b84274b7ffd309ca Mon Sep 17 00:00:00 2001 From: takuto050204 Date: Sun, 26 Jul 2026 19:25:48 +0900 Subject: [PATCH 4/6] =?UTF-8?q?=E3=83=8F=E3=83=83=E3=82=B7=E3=83=A5?= =?UTF-8?q?=E5=80=A4=E7=AE=97=E5=87=BA=E3=81=AE=E5=89=8D=E5=87=A6=E7=90=86?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ahmia/ahmia/items.py | 1 + ahmia/ahmia/pipelines.py | 63 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/ahmia/ahmia/items.py b/ahmia/ahmia/items.py index 7eaf56a..5e462c3 100644 --- a/ahmia/ahmia/items.py +++ b/ahmia/ahmia/items.py @@ -23,6 +23,7 @@ class DocumentItem(Item): title = Field(input_processor=MapCompose(remove_control_chars), output_processor=TakeFirst()) meta = Field(input_processor=MapCompose(remove_control_chars), output_processor=TakeFirst()) content = Field(input_processor=MapCompose(remove_control_chars), output_processor=TakeFirst()) + clean_content = Field(input_processor=TakeFirst()) content_hash = Field(input_processor=TakeFirst()) domain = Field(output_processor=TakeFirst()) content_type = Field(output_processor=TakeFirst()) diff --git a/ahmia/ahmia/pipelines.py b/ahmia/ahmia/pipelines.py index 0034018..060700d 100644 --- a/ahmia/ahmia/pipelines.py +++ b/ahmia/ahmia/pipelines.py @@ -2,6 +2,7 @@ """ Pipelines """ import hashlib import logging +import re from elasticsearch import Elasticsearch from elasticsearch.helpers import bulk from .items import DocumentItem @@ -40,6 +41,61 @@ def process_item(self, item): self.index_item(item) return item # To continue passing items through other pipelines + def preprocess(self, text): + + # 小文字化（大文字・小文字の違いを統一） + text = text.lower() + + # 改行・タブ・複数スペースを1つの空白に統一 + text = re.sub(r'\s+', ' ', text) + + # onionアドレスを削除（サイト固有の識別情報を除去） + text = re.sub( + r'[a-z2-7]{56}\.onion', + '', + text + ) + + # URLを削除（リンク先の違いによる差を除去） + text = re.sub( + r'https?://\S+', + '', + text + ) + + # Bitcoinアドレスを削除（決済情報などサイト固有情報を除去） + text = re.sub( + r'\b(bc1|[13])[a-zA-HJ-NP-Z0-9]{25,62}\b', + '', + text + ) + + # メールアドレスを削除（連絡先情報を除去） + text = re.sub( + r'\S+@\S+\.\S+', + '', + text + ) + + # 連続する特殊文字を削除（Markdown記号などのノイズ除去） + text = re.sub( + r'[*#|_\-]{2,}', + '', + text + ) + + # 再度、余分な空白を整理 + text = re.sub( + r'\s+', + ' ', + text + ) + + # 前後の空白を削除 + text = text.strip() + + return text + def index_item(self, item): """ Items are indexed here. @@ -50,12 +106,15 @@ def index_item(self, item): if isinstance(item, DocumentItem): - # ハッシュ値の生成 content = item.get('content', '') if content: + # 前処理 + clean_content = self.preprocess(content) + item['clean_content'] = clean_content + # ハッシュ値の生成 item['content_hash'] = hashlib.sha256( - content.encode('utf-8') + clean_content.encode('utf-8') ).hexdigest() action = { From f7ed80c9b39978909fa773292be56adcc03897e0 Mon Sep 17 00:00:00 2001 From: ntaku02040204 Date: Fri, 31 Jul 2026 14:37:04 +0900 Subject: [PATCH 5/6] no message --- run.sh | 0 run_daily.sh | 0 torfleet/runfleet.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 run.sh mode change 100755 => 100644 run_daily.sh mode change 100755 => 100644 torfleet/runfleet.sh diff --git a/run.sh b/run.sh old mode 100755 new mode 100644 diff --git a/run_daily.sh b/run_daily.sh old mode 100755 new mode 100644 diff --git a/torfleet/runfleet.sh b/torfleet/runfleet.sh old mode 100755 new mode 100644 From 783224413579d4384cc1c839b1ff31b0b3b2dbce Mon Sep 17 00:00:00 2001 From: ntaku02040204 Date: Sun, 2 Aug 2026 13:21:56 +0900 Subject: [PATCH 6/6] =?UTF-8?q?=E8=84=9A=E6=B3=A8=E3=83=96=E3=83=AD?= =?UTF-8?q?=E3=83=83=E3=82=AF=E3=81=AB=E5=AF=BE=E3=81=99=E3=82=8B=E5=89=8D?= =?UTF-8?q?=E5=87=A6=E7=90=86=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ahmia/ahmia/pipelines.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ahmia/ahmia/pipelines.py b/ahmia/ahmia/pipelines.py index 060700d..5339dc1 100644 --- a/ahmia/ahmia/pipelines.py +++ b/ahmia/ahmia/pipelines.py @@ -77,6 +77,13 @@ def preprocess(self, text): text ) + # 本文と無関係なテンプレート由来の脚注・用語集ノイズを除去 + text = re.sub( + r'\*\[[^\]]+\]:\s*[^*]*', + '', + text + ) + # 連続する特殊文字を削除（Markdown記号などのノイズ除去） text = re.sub( r'[*#|_\-]{2,}',