Test of source.csv and csv_import.sh v0.1.1

empty files!!!

---------

Thanks to Jetbrains for Sponsoring My Privacy DNS with their Open Source software licenses.

Their software helps us develop and maintain My Privacy DNS and other project as they made writing code easier.

Author: spirillen

.editorconfig

@@ -0,0 +1,22 @@
+; Check http://editorconfig.org/ for more information's
+
+root = true
+
+[*]
+trim_trailing_whitespace = true
+end_of_line = LF
+insert_final_newline = true
+charset = utf-8
+indent_style = space
+indent_size = 4
+tab_width = 4
+
+[*.md]
+trim_trailing_whitespace = false
+
+[*.{yaml,yml}]
+indent_size = 2
+tab_width = 2
+
+[*.list]
+indent_size = 2

.gitattributes

@@ -14,6 +14,8 @@
 # Should ensure submitted domain.list are encode with LF (\n) on gitlab
 /**/domain.list text eol=lf
 /**/wildcard.list text eol=lf
+/**/domain.csv text eol=lf
+/**/wildcard.csv text eol=lf
 
 # Ensure the non-standard windows understand files ending....
 # PS: Before you comments on this comment... *nix is bar far the most used OS

scripts/csv_import.sh

@@ -46,13 +46,17 @@ while IFS="," read -r name type url; do
     # mkdir -p "${git_dir}/test/data/$name"
 
     mkdir -p "${git_dir}/test/data/"
+    ls -lha "${git_dir}/test/data/"
 
     if [ "$type" == rfc954 ]; then
-        # fetch "$url" | awk '/localhost/{next}; /^(#|$)/{ next }; { if ( $1 ~ /^[0-9]/ ) printf("%s\n",tolower($2)) | "sort -i | uniq -u -i " }' >"${git_dir}/test/data/$name.csv"
-        echo "rfc954"
+        ls -lha "${git_dir}/test/data/"
+        fetch "$url" | awk '/localhost/{next}; /^(#|$)/{ next }; { if ( $1 ~ /^[0-9]/ ) printf("%s\n",tolower($2)) | "sort -i | uniq -u -i " }' > "${git_dir}/test/data/$name.csv"
+        ls -lha "${git_dir}/test/data/"
 
     elif [ "$type" == 'domain' ]; then
-        fetch "$url" | awk '/localhost/{next}; /^(#|$)/{ next }; { printf("%s\n",tolower($1)) | "sort -i | uniq -u -i " }' | more #>"${git_dir}/test/data/$name.csv"
+        ls -lha "${git_dir}/test/data/"
+        fetch "$url" | awk '/localhost/{next}; /^(#|$)/{ next }; { printf("%s\n",tolower($1)) | "sort -i | uniq -u -i " }' > "${git_dir}/test/data/$name.csv"
+        ls -lha "${git_dir}/test/data/"
     fi
     echo "imported $name"
     echo ""

scripts/dependencies.sh

@@ -27,5 +27,5 @@ bash -c "$(curl -sL https://raw.githubusercontent.com/ilikenwf/apt-fast/master/q
 
 apt-fast update -yqq
 #apt-fastdist-upgrade -yqq
-apt-fast install -yqq openssh-client curl wget ldnsutils \
+apt-fast install -yqq openssh-client curl wget dos2unix ldnsutils \
     libdomain-publicsuffix-perl

scripts/import.sh

@@ -329,12 +329,14 @@ echo "Imported $name"
 name="mineNu"
 mkdir -p "${git_dir}/data/$name/"
 c https://hostsfile.mine.nu/hosts0.txt | awk '/localhost/{next}; /^(#|$)/{ next }; { if ( $1 ~ /^[0-9]/ ) printf("%s\n",tolower($2)) | "sort -i | uniq -u -i " }' > "${git_dir}/data/$name/domain.csv"
+dos2unix -f -r -S "${git_dir}/data/$name/domain.csv"
 echo "Imported $name"
 
 # HostsFileOrg
 name="HostsFileOrg"
 mkdir -p "${git_dir}/data/$name/"
 c https://hostsfile.org/Downloads/hosts.txt | awk '/localhost/{next}; /^(#|$)/{ next }; { if ( $1 ~ /^[0-9]/ ) printf("%s\n",tolower($2)) | "sort -i | uniq -u -i " }' > "${git_dir}/data/$name/domain.csv"
+dos2unix -f -r -S "${git_dir}/data/$name/domain.csv"
 echo "Imported $name"
 
 # digitalside | This list is not public available
@@ -379,29 +381,13 @@ mkdir -p "${git_dir}/data/$name/"
 c https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt | awk '/localhost/{next}; /^(#|$)/{ next }; { if ( $1 ~ /^[0-9]/ ) printf("%s\n",tolower($2)) | "sort -i | uniq -u -i " }' > "${git_dir}/data/$name/domain.csv"
 echo "Imported $name"
 
-#
-name=""
+# BarbBlock
+name="BarbBlock"
 mkdir -p "${git_dir}/data/$name/"
-c  | awk '/localhost/{next}; /^(#|$)/{ next }; { if ( $1 ~ /^[0-9]/ ) printf("%s\n",tolower($2)) | "sort -i | uniq -u -i " }' > "${git_dir}/data/$name/domain.csv"
+c https://paulgb.github.io/BarbBlock/blacklists/domain-list.txt | awk '/localhost/{next}; /^(#|$)/{ next }; { if ( $1 ~ /^[0-9]/ ) printf("%s\n",tolower($2)) | "sort -i | uniq -u -i " }' > "${git_dir}/data/$name/domain.csv"
 echo "Imported $name"
 
-#
-name=""
-mkdir -p "${git_dir}/data/$name/"
-c  | awk '/localhost/{next}; /^(#|$)/{ next }; { if ( $1 ~ /^[0-9]/ ) printf("%s\n",tolower($2)) | "sort -i | uniq -u -i " }' > "${git_dir}/data/$name/domain.csv"
-echo "Imported $name"
 
-#
-name=""
-mkdir -p "${git_dir}/data/$name/"
-c  | awk '/localhost/{next}; /^(#|$)/{ next }; { if ( $1 ~ /^[0-9]/ ) printf("%s\n",tolower($2)) | "sort -i | uniq -u -i " }' > "${git_dir}/data/$name/domain.csv"
-echo "Imported $name"
-
-#
-name=""
-mkdir -p "${git_dir}/data/$name/"
-c  | awk '/localhost/{next}; /^(#|$)/{ next }; { if ( $1 ~ /^[0-9]/ ) printf("%s\n",tolower($2)) | "sort -i | uniq -u -i " }' > "${git_dir}/data/$name/domain.csv"
-echo "Imported $name"
 echo ""
 echo ""
 echo "The script ${0}"

scripts/source.csv

@@ -4,3 +4,4 @@ AssoEchap,domain,https://raw.githubusercontent.com/AssoEchap/stalkerware-indicat
 KADhosts_azet12,domain,https://raw.githubusercontent.com/FiltersHeroes/KADhosts/master/KADomains.txt
 phishing-block-list,domain,https://raw.githubusercontent.com/chainapsis/phishing-block-list/main/block-list.txt
 adblock-nocoin-list,rfc954,https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt
+BarbBlock,domain,https://paulgb.github.io/BarbBlock/blacklists/domain-list.txt