From f3030d62e57ae9f47191d965ee5b1f03484d239a Mon Sep 17 00:00:00 2001
From: Loyalsoldier <10487845+Loyalsoldier@users.noreply.github.com>
Date: Fri, 9 Aug 2024 09:12:44 +0800
Subject: [PATCH] Feat: get ASN data from MaxMind GeoLite2 ASN CSV instead of
 ripe.net

---
 .github/workflows/build.yml | 14 ++++-----
 asn.csv                     |  7 -----
 asn.sh                      | 23 ---------------
 config.json                 | 59 +++++++++++++++++++++++++++++++++++++
 4 files changed, 65 insertions(+), 38 deletions(-)
 delete mode 100644 asn.csv
 delete mode 100755 asn.sh

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7ba6ab3d2de..fdfc0d7ae7d 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -33,13 +33,9 @@ jobs:
           echo "RELEASE_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV
         shell: bash
 
-      - name: Fetch lists from ripe.net
-        run: |
-          chmod +x asn.sh
-          ./asn.sh
-
       - name: Append more CIDRs
         run: |
+          mkdir -p data
           curl -sSL https://www.gstatic.com/ipranges/goog.json | jq --raw-output '.prefixes[].ipv4Prefix,.prefixes[].ipv6Prefix | select(. != null)' >> data/google
           curl -sSL https://www.gstatic.com/ipranges/cloud.json | jq --raw-output '.prefixes[].ipv4Prefix,.prefixes[].ipv6Prefix | select(. != null)' >> data/google
           curl -sSL https://api.fastly.com/public-ip-list | jq --raw-output '.addresses[],.ipv6_addresses[]' >> data/fastly
@@ -50,9 +46,11 @@ jobs:
           LICENSE_KEY: ${{ secrets.MAXMIND_GEOLITE2_LICENSE }}
         run: |
           curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country-CSV&license_key=${LICENSE_KEY}&suffix=zip" -o GeoLite2-Country-CSV.zip
-          unzip GeoLite2-Country-CSV.zip
-          rm -f GeoLite2-Country-CSV.zip
-          mv GeoLite2* geolite2
+          curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN-CSV&license_key=${LICENSE_KEY}&suffix=zip" -o GeoLite2-ASN-CSV.zip
+          mkdir -p geolite2
+          unzip *.zip
+          cp GeoLite2-Country-CSV_*/*.csv geolite2/
+          cp GeoLite2-ASN-CSV_*/*.csv geolite2/
 
       - name: Build geoip files
         run: |
diff --git a/asn.csv b/asn.csv
deleted file mode 100644
index 2a9ac6f8abc..00000000000
--- a/asn.csv
+++ /dev/null
@@ -1,7 +0,0 @@
-cloudflare,AS395747|AS394536|AS209242|AS203898|AS202623|AS14789|AS139242|AS133877|AS13335|AS132892
-facebook,AS63293|AS54115|AS32934
-fastly,AS54113|AS394192
-google,AS6432|AS55023|AS45566|AS43515|AS41264|AS40873|AS396982|AS395973|AS394699|AS394639|AS394507|AS36987|AS36492|AS36385|AS36384|AS36040|AS36039|AS26910|AS26684|AS22859|AS22577|AS19527|AS19448|AS19425|AS16591|AS16550|AS15169|AS13949|AS139190|AS139070
-netflix,AS55095|AS40027|AS394406|AS2906
-telegram,AS62041|AS62014|AS59930|AS44907|AS211157
-twitter,AS8945|AS63179|AS54888|AS35995|AS13414
diff --git a/asn.sh b/asn.sh
deleted file mode 100755
index c47cb0d2d86..00000000000
--- a/asn.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-input="./asn.csv"
-mkdir -p ./tmp ./data
-
-while IFS= read -r line; do
-  filename=$(echo ${line} | awk -F ',' '{print $1}')
-  IFS='|' read -r -a asns <<<$(echo ${line} | awk -F ',' '{print $2}')
-  file="data/${filename}"
-
-  echo "==================================="
-  echo "Generating ${filename} CIDR list..."
-  rm -rf ${file} && touch ${file}
-  for asn in ${asns[@]}; do
-    url="https://stat.ripe.net/data/ris-prefixes/data.json?list_prefixes=true&types=o&resource=${asn}"
-    echo "-----------------------"
-    echo "Fetching ${asn}..."
-    curl -sL ${url} -o ./tmp/${filename}-${asn}.txt \
-      -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36'
-    jq --raw-output '.data.prefixes.v4.originating[]' ./tmp/${filename}-${asn}.txt | sort -u >>${file}
-    jq --raw-output '.data.prefixes.v6.originating[]' ./tmp/${filename}-${asn}.txt | sort -u >>${file}
-  done
-done <${input}
diff --git a/config.json b/config.json
index 7518b060d68..e75942f08bc 100644
--- a/config.json
+++ b/config.json
@@ -75,6 +75,65 @@
         "inputDir": "./data"
       }
     },
+    {
+      "type": "maxmindGeoLite2ASNCSV",
+      "action": "add",
+      "args": {
+        "ipv4": "./geolite2/GeoLite2-ASN-Blocks-IPv4.csv",
+        "ipv6": "./geolite2/GeoLite2-ASN-Blocks-IPv6.csv",
+        "wantedList": {
+          "cloudflare": [
+            "AS395747",
+            "AS394536",
+            "AS209242",
+            "AS203898",
+            "AS202623",
+            "AS14789",
+            "AS139242",
+            "AS133877",
+            "AS13335",
+            "AS132892"
+          ],
+          "facebook": ["AS63293", "AS54115", "AS32934"],
+          "fastly": ["AS54113", "AS394192"],
+          "google": [
+            "AS6432",
+            "AS55023",
+            "AS45566",
+            "AS43515",
+            "AS41264",
+            "AS40873",
+            "AS396982",
+            "AS395973",
+            "AS394699",
+            "AS394639",
+            "AS394507",
+            "AS36987",
+            "AS36492",
+            "AS36385",
+            "AS36384",
+            "AS36040",
+            "AS36039",
+            "AS26910",
+            "AS26684",
+            "AS22859",
+            "AS22577",
+            "AS19527",
+            "AS19448",
+            "AS19425",
+            "AS16591",
+            "AS16550",
+            "AS15169",
+            "AS13949",
+            "AS139190",
+            "AS139070"
+          ],
+          "netflix": ["AS55095", "AS40027", "AS394406", "AS2906"],
+          "telegram": ["AS62041", "AS62014", "AS59930", "AS44907", "AS211157"],
+          "twitter": ["AS8945", "AS63179", "AS54888", "AS35995", "AS13414"]
+        }
+      }
+    },
     {
       "type": "private",
       "action": "add"