Skip to content

Commit

Permalink
Improve README.md output
Browse files Browse the repository at this point in the history
  • Loading branch information
@swiftlyfalling
swiftlyfalling committed Feb 7, 2018
1 parent 4287568 commit 7d0d153
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 19 deletions.
56 changes: 41 additions & 15 deletions buildrepo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,17 @@ def construct_download_link(dl_base_url: str, path_in_repo: str) -> str:
else:
return "{}/{}".format(dl_base_url, path_in_repo)

# Extract the license description / URL from the list data, and return it + the source
def get_license_info(filterlist):
if 'License' in filterlist['Header'] and len(filterlist['Header']['License']) > 0:
# Use the License information extracted from the list file itself
return (filterlist['Header']['License'], 'List Header')
elif 'License' in filterlist and len(filterlist['License']) > 0:
# Use the License information provided to convertlists.py (via lists.json)
return (filterlist['License'], 'Input')
else:
return ("", "No License Information")

# Construct the README.md file
def construct_readme_file(lists_info_array, downloadbaseurl: str, verbosity: int):
lists_details = list()
Expand All @@ -66,6 +77,7 @@ def construct_readme_file(lists_info_array, downloadbaseurl: str, verbosity: int
if verbosity >= 2:
print("\tAdding List: \"{!s}\"".format(filterlist['Title']))
lists_details.append(" - [{}](#{}) (Domains-only)".format(filterlist['Title'], construct_github_anchor_link(filterlist['Title'] + " (Domains-only)")))
lists_details.append("- [License](#license)")
lists_details.append("- [Reporting Conversion Issues](#reporting-conversion-issues)")
lists_details.append("")
lists_details.append(" ")
Expand Down Expand Up @@ -94,14 +106,29 @@ def construct_readme_file(lists_info_array, downloadbaseurl: str, verbosity: int
lists_details.append("")
lists_details.append("# The Lists:")
lists_details.append("")
lists_details.append("| Converted List | Domains | Raw Domain List Link | Last Updated |")
lists_details.append(":- | - | - | - |")
lists_details.append("| Converted List | Domains | Domain List Link | Last Updated | License |")
lists_details.append(":- | - | :-: | - | - |")
for filterlist in lists_info_array:
raw_download_link = construct_download_link(downloadbaseurl, os.path.join("lists/", filterlist['Output Formats']['Just Domains']))
last_updated = ""
if 'Last Modified' in filterlist['Header']:
last_updated = filterlist['Header']['Last Modified']
lists_details.append("| [{}](#{}) | {} | [{}]({}) | {} |".format(filterlist['Title'], construct_github_anchor_link(filterlist['Title'] + " (Domains-only)"), filterlist['Domains Output'], filterlist['Output Formats']['Just Domains'], raw_download_link, last_updated))

license_info, license_source = get_license_info(filterlist)
license_link = ""
if 'License Identifier' in filterlist:
if should_linkify(license_info):
license_link = "[{}]({})".format(filterlist['License Identifier'], license_info)
else:
license_link = filterlist['License Identifier']
elif len(license_info) > 0:
if should_linkify(license_info):
license_link = "[(link)]({})".format(license_info)
else:
license_link = license_info
else:
license_link = "(see source)"
lists_details.append("| [{}](#{}) | {} | [Download]({}) | {} | {} |".format(filterlist['Title'], construct_github_anchor_link(filterlist['Title'] + " (Domains-only)"), filterlist['Domains Output'], raw_download_link, last_updated, license_link))
lists_details.append("")
lists_details.append(" ")
lists_details.append("")
Expand Down Expand Up @@ -138,38 +165,37 @@ def construct_readme_file(lists_info_array, downloadbaseurl: str, verbosity: int
lists_details.append(" ")
lists_details.append("")


lists_details.append("# License:")
lists_details.append("Each converted / modified list file is licensed under the same license as the original list.")
lists_details.append("For more details, see the [LICENSE](LICENSE) file.")
lists_details.append("")
lists_details.append(" ")
lists_details.append("")
lists_details.append("# Reporting Conversion Issues:")
lists_details.append("If you find an issue in the output of the conversion process (i.e. comparing to the original upstream list), please report it over on: https://github.com/justdomains/ci/issues")
lists_details.append("")
lists_details.append("**NOTE: We do not manage the upstream lists themselves, and will not be able to add any new blocks to the lists.**")
lists_details.append("")
lists_details.append("##### Disclaimer:")
lists_details.append("<sup>These files are provided \"AS IS\", without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, arising from, out of or in connection with the files or the use of the files.</sup>")
lists_details.append("")
lists_details.append("<sub>Any and all trademarks are the property of their respective owners.</sub>")
lists_details.append("")

return lists_details


# Construct the LICENSE file
def construct_license_file(lists_info_array, verbosity: int):
lists_licenses = list()
lists_licenses.append("All converted / modified list files are licensed under the same license as the original list.")
lists_licenses.append("")
for filterlist in lists_info_array:
if 'License' in filterlist['Header']:
# Use the License information extracted from the list file itself
lists_licenses.append("[{}]".format(filterlist['Base Output Filename']))
lists_licenses.append("\tLicense: {}".format(filterlist['Header']['License']))
if verbosity >= 2:
print("\tAdding License for List \"{!s}\": \"{!s}\"; source='List Header'".format(filterlist['Title'], filterlist['Header']['License']))
elif 'License' in filterlist:
# Use the License information provided to convertlists.py (via lists.json)
license_info, license_source = get_license_info(filterlist)
if len(license_info) > 0:
lists_licenses.append("[{}]".format(filterlist['Base Output Filename']))
lists_licenses.append("\tLicense: {}".format(filterlist['License']))
lists_licenses.append("\tLicense: {}".format(license_info))
if verbosity >= 2:
print("\tAdding License for List \"{!s}\": \"{!s}\"; source='Input'".format(filterlist['Title'], filterlist['License']))
print("\tAdding License for List \"{!s}\": \"{!s}\"; source='{!s}'".format(filterlist['Title'], license_info, license_source))
elif verbosity >= 1:
print("\tWARNING: No License Information Available for List: \"{!s}\"; skipping".format(filterlist['Title']))

Expand Down
3 changes: 3 additions & 0 deletions convertlists.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#
# Optional dictionary keys/values for each list are:
# "license": used to supply a license URL or description, if no license information can be extracted from the list itself (string)
# "license-identifier": a short license name / title (ex. "GPL3", "MIT")
# "outputfile": the base filename used for both the downloaded original and the converted output file (string) - important if multiple downloaded lists have the same filename
#
# [Example Usage]:
Expand Down Expand Up @@ -575,6 +576,8 @@ def appendtofilename(filename: str, string_to_append: str) -> str:
list_info['Source'] = filterlist['url']
if 'license' in filterlist:
list_info['License'] = filterlist['license']
if 'license-identifier' in filterlist:
list_info['License Identifier'] = filterlist['license-identifier']
list_info['Base Output Filename'] = appendtofilename(basefilename, "-{FORMAT}")
list_info['Header'] = listdetails['Header']
list_info['Conversion'] = listdetails['Conversion']
Expand Down
12 changes: 8 additions & 4 deletions lists.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,29 +4,33 @@
"url": "https://easylist.to/easylist/easylist.txt",
"format": "adbp",
"moreinformation": "https://easylist.to/",
"description": "EasyList is the primary filter list that removes most adverts from international webpages, including unwanted frames, images and objects. It is the most popular list used by many ad blockers and forms the basis of over a dozen combination and supplementary filter lists."
"description": "EasyList is the primary filter list that removes most adverts from international webpages, including unwanted frames, images and objects. It is the most popular list used by many ad blockers and forms the basis of over a dozen combination and supplementary filter lists.",
"license-identifier": "GPL3 / CC BY-SA 3.0"
},
{
"name": "EasyPrivacy",
"url": "https://easylist.to/easylist/easyprivacy.txt",
"format": "adbp",
"moreinformation": "https://easylist.to/",
"description": "EasyPrivacy is an optional supplementary filter list that completely removes all forms of tracking from the internet, including web bugs, tracking scripts and information collectors, thereby protecting your personal data."
"description": "EasyPrivacy is an optional supplementary filter list that completely removes all forms of tracking from the internet, including web bugs, tracking scripts and information collectors, thereby protecting your personal data.",
"license-identifier": "GPL3 / CC BY-SA 3.0"
},
{
"name": "AdGuard Simplified Domain Names Filter",
"url": "https://filters.adtidy.org/extension/chromium/filters/15.txt",
"format": "adbp",
"outputfile": "adguarddns.txt",
"moreinformation": "https://kb.adguard.com/en/general/adguard-ad-filters",
"description": "A filter composed from several other filters (English filter, Social media filter, Spyware filter, Mobile ads filter, EasyList and EasyPrivacy) and simplified specifically to be better compatible with DNS-level ad blocking. This filter is used by AdGuard DNS servers to block ads."
"description": "A filter composed from several other filters (English filter, Social media filter, Spyware filter, Mobile ads filter, EasyList and EasyPrivacy) and simplified specifically to be better compatible with DNS-level ad blocking. This filter is used by AdGuard DNS servers to block ads.",
"license-identifier": "GPL3"
},
{
"name": "NoCoin Filter List",
"url": "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt",
"format": "hosts",
"outputfile": "nocoin.txt",
"moreinformation": "https://github.com/hoshsadiq/adblock-nocoin-list/",
"description": "Blocking Web Browser Bitcoin Mining"
"description": "Blocking Web Browser Bitcoin Mining",
"license-identifier": "MIT"
}
]

0 comments on commit 7d0d153

Please sign in to comment.