|
| 1 | +#!/usr/bin/env python3 |
| 2 | +# -*- coding: utf-8 -*-# |
| 3 | +# |
| 4 | +# PSL linter written in python |
| 5 | +# |
| 6 | +# Copyright 2016 Tim Rühsen (tim dot ruehsen at gmx dot de). All rights reserved. |
| 7 | +# |
| 8 | +# Permission is hereby granted, free of charge, to any person obtaining a |
| 9 | +# copy of this software and associated documentation files (the "Software"), |
| 10 | +# to deal in the Software without restriction, including without limitation |
| 11 | +# the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 12 | +# and/or sell copies of the Software, and to permit persons to whom the |
| 13 | +# Software is furnished to do so, subject to the following conditions: |
| 14 | +# |
| 15 | +# The above copyright notice and this permission notice shall be included in |
| 16 | +# all copies or substantial portions of the Software. |
| 17 | +# |
| 18 | +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 19 | +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 20 | +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 21 | +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 22 | +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| 23 | +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
| 24 | +# DEALINGS IN THE SOFTWARE. |
| 25 | + |
| 26 | +import sys |
| 27 | +import codecs |
| 28 | + |
| 29 | +nline = 0 |
| 30 | +line = "" |
| 31 | +orig_line = "" |
| 32 | +warnings = 0 |
| 33 | +errors = 0 |
| 34 | +skip_order_check = False |
| 35 | + |
| 36 | +def warning(msg): |
| 37 | + global warnings, orig_line, nline |
| 38 | + print('%d: warning: %s%s' % (nline, msg, ": \'" + orig_line + "\'" if orig_line else "")) |
| 39 | + warnings += 1 |
| 40 | + |
| 41 | +def error(msg): |
| 42 | + global errors, orig_line, nline |
| 43 | + print('%d: error: %s%s' % (nline, msg, ": \'" + orig_line + "\'" if orig_line else "")) |
| 44 | + errors += 1 |
| 45 | +# skip_order_check = True |
| 46 | + |
| 47 | +def print_psl(list): |
| 48 | + for domain in list: |
| 49 | + print(".".join(str(label) for label in reversed(domain))) |
| 50 | + |
| 51 | +def psl_key(s): |
| 52 | + if s[0] == '*': |
| 53 | + return 0 |
| 54 | + if s[0] == '!': |
| 55 | + return 1 |
| 56 | + return 2 |
| 57 | + |
| 58 | +def check_order(group): |
| 59 | + """Check the correct order of a domain group""" |
| 60 | + global skip_order_check |
| 61 | + |
| 62 | + try: |
| 63 | + if skip_order_check or len(group) < 2: |
| 64 | + skip_order_check = False |
| 65 | + return |
| 66 | + |
| 67 | + # check if the TLD is the identical within the group |
| 68 | + if any(group[0][0] != labels[0] for labels in group): |
| 69 | + warning('Domain group TLD is not consistent') |
| 70 | + |
| 71 | + # sort by # of labels, label-by-label (labels are in reversed order) |
| 72 | + sorted_group = sorted(group, key = lambda labels: (len(labels), psl_key(labels[-1][0]), labels)) |
| 73 | + |
| 74 | + if group != sorted_group: |
| 75 | + warning('Incorrectly sorted group of domains') |
| 76 | + print(" " + str(group)) |
| 77 | + print(" " + str(sorted_group)) |
| 78 | + print("Correct sorting would be:") |
| 79 | + print_psl(sorted_group) |
| 80 | + |
| 81 | + finally: |
| 82 | + del group[:] |
| 83 | + |
| 84 | + |
| 85 | +def lint_psl(infile): |
| 86 | + """Parses PSL file and performs syntax checking""" |
| 87 | + global orig_line, nline |
| 88 | + |
| 89 | + PSL_FLAG_EXCEPTION = (1<<0) |
| 90 | + PSL_FLAG_WILDCARD = (1<<1) |
| 91 | + PSL_FLAG_ICANN = (1<<2) # entry of ICANN section |
| 92 | + PSL_FLAG_PRIVATE = (1<<3) # entry of PRIVATE section |
| 93 | + PSL_FLAG_PLAIN = (1<<4) #just used for PSL syntax checking |
| 94 | + |
| 95 | + line2number = {} |
| 96 | + line2flag = {} |
| 97 | + group = [] |
| 98 | + section = 0 |
| 99 | + icann_sections = 0 |
| 100 | + private_sections = 0 |
| 101 | + |
| 102 | + lines = [line.strip('\n') for line in infile] |
| 103 | + |
| 104 | + for line in lines: |
| 105 | + nline += 1 |
| 106 | + |
| 107 | + # check for leadind/trailing whitespace |
| 108 | + stripped = line.strip() |
| 109 | + if stripped != line: |
| 110 | + line = line.replace('\t','\\t') |
| 111 | + line = line.replace('\r','^M') |
| 112 | + orig_line = line |
| 113 | + warning('Leading/Trailing whitespace') |
| 114 | + orig_line = line |
| 115 | + line = stripped |
| 116 | + |
| 117 | + # empty line (end of sorted domain group) |
| 118 | + if not line: |
| 119 | + # check_order(group) |
| 120 | + continue |
| 121 | + |
| 122 | + # check for section begin/end |
| 123 | + if line[0:2] == "//": |
| 124 | + # check_order(group) |
| 125 | + |
| 126 | + if section == 0: |
| 127 | + if line == "// ===BEGIN ICANN DOMAINS===": |
| 128 | + section = PSL_FLAG_ICANN |
| 129 | + icann_sections += 1 |
| 130 | + elif line == "// ===BEGIN PRIVATE DOMAINS===": |
| 131 | + section = PSL_FLAG_PRIVATE |
| 132 | + private_sections += 1 |
| 133 | + elif line[3:11] == "===BEGIN": |
| 134 | + error('Unexpected begin of unknown section') |
| 135 | + elif line[3:9] == "===END": |
| 136 | + error('End of section without previous begin') |
| 137 | + elif section == PSL_FLAG_ICANN: |
| 138 | + if line == "// ===END ICANN DOMAINS===": |
| 139 | + section = 0 |
| 140 | + elif line[3:11] == "===BEGIN": |
| 141 | + error('Unexpected begin of section: ') |
| 142 | + elif line[3:9] == "===END": |
| 143 | + error('Unexpected end of section') |
| 144 | + elif section == PSL_FLAG_PRIVATE: |
| 145 | + if line == "// ===END PRIVATE DOMAINS===": |
| 146 | + section = 0 |
| 147 | + elif line[3:11] == "===BEGIN": |
| 148 | + error('Unexpected begin of section') |
| 149 | + elif line[3:9] == "===END": |
| 150 | + error('Unexpected end of section') |
| 151 | + |
| 152 | + continue # processing of comments ends here |
| 153 | + |
| 154 | + # No rule must be outside of a section |
| 155 | + if section == 0: |
| 156 | + error('Rule outside of section') |
| 157 | + |
| 158 | + group.append(list(reversed(line.split('.')))) |
| 159 | + |
| 160 | + # decode UTF-8 input into unicode, needed only for python 2.x |
| 161 | + try: |
| 162 | + if sys.version_info[0] < 3: |
| 163 | + line = line.decode('utf-8') |
| 164 | + else: |
| 165 | + line.encode('utf-8') |
| 166 | + except (UnicodeDecodeError, UnicodeEncodeError): |
| 167 | + orig_line = None |
| 168 | + error('Invalid UTF-8 character') |
| 169 | + continue |
| 170 | + |
| 171 | + # each rule must be lowercase (or more exactly: not uppercase and not titlecase) |
| 172 | + if line != line.lower(): |
| 173 | + error('Rule must be lowercase') |
| 174 | + |
| 175 | + # strip leading wildcards |
| 176 | + flags = section |
| 177 | + # while line[0:2] == '*.': |
| 178 | + if line[0:2] == '*.': |
| 179 | + flags |= PSL_FLAG_WILDCARD |
| 180 | + line = line[2:] |
| 181 | + |
| 182 | + if line[0] == '!': |
| 183 | + flags |= PSL_FLAG_EXCEPTION |
| 184 | + line = line[1:] |
| 185 | + else: |
| 186 | + flags |= PSL_FLAG_PLAIN |
| 187 | + |
| 188 | + # wildcard and exception must not combine |
| 189 | + if flags & PSL_FLAG_WILDCARD and flags & PSL_FLAG_EXCEPTION: |
| 190 | + error('Combination of wildcard and exception') |
| 191 | + continue |
| 192 | + |
| 193 | + labels = line.split('.') |
| 194 | + |
| 195 | + if flags & PSL_FLAG_EXCEPTION and len(labels) > 1: |
| 196 | + domain = ".".join(str(label) for label in labels[1:]) |
| 197 | + if not domain in line2flag: |
| 198 | + error('Exception without previous wildcard') |
| 199 | + elif not line2flag[domain] & PSL_FLAG_WILDCARD: |
| 200 | + error('Exception without previous wildcard') |
| 201 | + |
| 202 | + for label in labels: |
| 203 | + if not label: |
| 204 | + error('Leading/trailing or multiple dot') |
| 205 | + continue |
| 206 | + |
| 207 | + if label[0:4] == 'xn--': |
| 208 | + error('Punycode found') |
| 209 | + continue |
| 210 | + |
| 211 | + if '--' in label: |
| 212 | + error('Double minus found') |
| 213 | + continue |
| 214 | + |
| 215 | + # allowed are a-z,0-9,- and unicode >= 128 (maybe that can be finetuned a bit !?) |
| 216 | + for c in label: |
| 217 | + if not c.isalnum() and c != '-' and ord(c) < 128: |
| 218 | + error('Illegal character') |
| 219 | + break |
| 220 | + |
| 221 | + if line in line2flag: |
| 222 | + '''Found existing entry: |
| 223 | + Combination of exception and plain rule is contradictionary |
| 224 | + !foo.bar + foo.bar |
| 225 | + Doublette, since *.foo.bar implies foo.bar: |
| 226 | + foo.bar + *.foo.bar |
| 227 | + Allowed: |
| 228 | + !foo.bar + *.foo.bar |
| 229 | + ''' |
| 230 | + error('Found doublette/ambiguity (previous line was %d)' % line2number[line]) |
| 231 | + |
| 232 | + line2number[line] = nline |
| 233 | + line2flag[line] = flags |
| 234 | + |
| 235 | + orig_line = None |
| 236 | + |
| 237 | + if section == PSL_FLAG_ICANN: |
| 238 | + error('ICANN section not closed') |
| 239 | + elif section == PSL_FLAG_PRIVATE: |
| 240 | + error('PRIVATE section not closed') |
| 241 | + |
| 242 | + if icann_sections < 1: |
| 243 | + warning('No ICANN section found') |
| 244 | + elif icann_sections > 1: |
| 245 | + warning('%d ICANN sections found' % icann_sections) |
| 246 | + |
| 247 | + if private_sections < 1: |
| 248 | + warning('No PRIVATE section found') |
| 249 | + elif private_sections > 1: |
| 250 | + warning('%d PRIVATE sections found' % private_sections) |
| 251 | + |
| 252 | +def usage(): |
| 253 | + """Prints the usage""" |
| 254 | + print('usage: %s PSLfile' % sys.argv[0]) |
| 255 | + print('or %s - # To read PSL from STDIN' % sys.argv[0]) |
| 256 | + exit(1) |
| 257 | + |
| 258 | + |
| 259 | +def main(): |
| 260 | + """Check syntax of a PSL file""" |
| 261 | + if len(sys.argv) < 2: |
| 262 | + usage() |
| 263 | + |
| 264 | + with sys.stdin if sys.argv[-1] == '-' else open(sys.argv[-1], 'r', encoding='utf-8', errors="surrogateescape") as infile: |
| 265 | + lint_psl(infile) |
| 266 | + |
| 267 | + return errors != 0 |
| 268 | + |
| 269 | + |
| 270 | +if __name__ == '__main__': |
| 271 | + sys.exit(main()) |
0 commit comments