Dev

ahmia/ahmia/items.py

@@ -23,6 +23,8 @@ class DocumentItem(Item):
     title = Field(input_processor=MapCompose(remove_control_chars), output_processor=TakeFirst())
     meta = Field(input_processor=MapCompose(remove_control_chars), output_processor=TakeFirst())
     content = Field(input_processor=MapCompose(remove_control_chars), output_processor=TakeFirst())
+    clean_content = Field(input_processor=TakeFirst())
+    content_hash = Field(input_processor=TakeFirst())
     domain = Field(output_processor=TakeFirst())
     content_type = Field(output_processor=TakeFirst())
     updated_on = Field(output_processor=TakeFirst())

ahmia/ahmia/pipelines.py

@@ -2,6 +2,7 @@
 """ Pipelines """
 import hashlib
 import logging
+import re
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import bulk
 from .items import DocumentItem
@@ -40,6 +41,68 @@ def process_item(self, item):
         self.index_item(item)
         return item  # To continue passing items through other pipelines
 
+    def preprocess(self, text):
+
+        # 小文字化（大文字・小文字の違いを統一）
+        text = text.lower()
+
+        # 改行・タブ・複数スペースを1つの空白に統一
+        text = re.sub(r'\s+', ' ', text)
+
+        # onionアドレスを削除（サイト固有の識別情報を除去）
+        text = re.sub(
+            r'[a-z2-7]{56}\.onion',
+            '',
+            text
+        )
+
+        # URLを削除（リンク先の違いによる差を除去）
+        text = re.sub(
+            r'https?://\S+',
+            '',
+            text
+        )
+
+        # Bitcoinアドレスを削除（決済情報などサイト固有情報を除去）
+        text = re.sub(
+            r'\b(bc1|[13])[a-zA-HJ-NP-Z0-9]{25,62}\b',
+            '',
+            text
+        )
+
+        # メールアドレスを削除（連絡先情報を除去）
+        text = re.sub(
+            r'\S+@\S+\.\S+',
+            '',
+            text
+        )
+
+        # 本文と無関係なテンプレート由来の脚注・用語集ノイズを除去
+        text = re.sub(
+            r'\*\[[^\]]+\]:\s*[^*]*',
+            '',
+            text
+        )
+
+        # 連続する特殊文字を削除（Markdown記号などのノイズ除去）
+        text = re.sub(
+            r'[*#|_\-]{2,}',
+            '',
+            text
+        )
+
+        # 再度、余分な空白を整理
+        text = re.sub(
+            r'\s+',
+            ' ',
+            text
+        )
+
+        # 前後の空白を削除
+        text = text.strip()
+
+        return text
+
     def index_item(self, item):
         """
         Items are indexed here.
@@ -49,6 +112,18 @@ def index_item(self, item):
         doc_id = hashlib.sha1(item['url'].encode('utf-8')).hexdigest()
 
         if isinstance(item, DocumentItem):
+
+            content = item.get('content', '')
+
+            if content:
+                # 前処理
+                clean_content = self.preprocess(content)
+                item['clean_content'] = clean_content
+                # ハッシュ値の生成
+                item['content_hash'] = hashlib.sha256(
+                    clean_content.encode('utf-8')
+                ).hexdigest()
+
             action = {
                 "_index": self.index_name,
                 "_id": doc_id,

ahmia/ahmia/settings.py

@@ -93,7 +93,7 @@
 }
 
 # Tor proxy settings: http://localhost:15000 - http://localhost:15099
-HTTP_PROXY_TOR_PROXIES = [f"http://localhost:150{i:02}" for i in range(0, 100)]
+HTTP_PROXY_TOR_PROXIES = [f"http://localhost:150{i:02}" for i in range(0, 50)]
 
 def extract_onions_from_url(url, timeout=120):
     """ Helper function to extract unique onion base URLs """

ahmia/ahmia/spiders/onionspider.py

@@ -20,7 +20,7 @@ class OnionSpider(CrawlSpider):
     LARGE_TEXT_BYTES = 2 * 1024 * 1024
 
     MAX_EXTRACTED_LINKS_PER_DOMAIN = 50000
-
+    # 集めるサイトの条件指定
     rules = (
         Rule(
             LinkExtractor(
@@ -30,7 +30,7 @@ class OnionSpider(CrawlSpider):
                     "jpg", "jpeg", "mp3", "mp4", "m4a", "ogg", "pdf", "png", "rar", "svg",
                     "tar", "tgz", "webm", "webp", "xz", "zip"
                 ],
-                unique=True,
+                unique=True, # 同一リンクは排除
                 canonicalize=True,
             ),
             callback="parse_item",
@@ -39,6 +39,7 @@ class OnionSpider(CrawlSpider):
         ),
     )
 
+    # 初期設定
     def __init__(self, *args, seedlist=None, **kwargs):
         """ Init """
         super().__init__(*args, **kwargs)
@@ -47,7 +48,7 @@ def __init__(self, *args, seedlist=None, **kwargs):
         self.html_converter.ignore_links = True
         self.html_converter.ignore_images = True
 
-        self._extracted_links_per_domain = defaultdict(int)
+        self._extracted_links_per_domain = defaultdict(int) # リンク内にあったリンク数を保存する変数
         self._current_response_host = None
 
         if seedlist:
@@ -58,6 +59,7 @@ def __init__(self, *args, seedlist=None, **kwargs):
             self.start_urls = get_project_settings().get("SEEDLIST", [])
             self.logger.info("Using SEEDLIST from settings.py with %d URLs.", len(self.start_urls))
 
+    # 1ドメインから取得するリンク数の制限
     def limit_links_per_domain(self, links):
         """Limit extracted links to 50,000 per one domain."""
         kept = []
@@ -156,6 +158,7 @@ def _safe_html2text(self, response):
             )
             return ""
 
+    # 取得したHTMLをparseItemに変換
     def parse_item(self, response):
         """Parse items."""
         self._current_response_host = (urlparse(response.url).hostname or "").lower()

torfleet/runfleet.sh

@@ -8,7 +8,7 @@ DATA_DIR="$BASE_DIR/data/tor"
 TORRC_DIR="$BASE_DIR/torrcs"
 PRIVOXY_DIR="$BASE_DIR/privoxy_configs"
 
-NUM_INSTANCES=100
+NUM_INSTANCES=50
 BASE_SOCKS_PORT=19050   # Tor SOCKS5
 BASE_HTTP_PORT=15000    # Privoxy HTTP