diff --git a/.gitignore b/.gitignore old mode 100755 new mode 100644 index c736d62..57e1011 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ __pycache__ node_modules build -dist cache downloads installs @@ -33,4 +32,4 @@ src/root src/index.html src/module.js src/prejs.js -src/main.c \ No newline at end of file +src/main.c diff --git a/dist-debug/coldbrew.asm.data b/dist-debug/coldbrew.asm.data new file mode 100644 index 0000000..a3aa410 --- /dev/null +++ b/dist-debug/coldbrew.asm.data @@ -0,0 +1,244043 @@ +"""functools.py - Tools for working with functions and callable objects +""" +# Python module wrapper for _functools C module +# to allow utilities written in Python to be added +# to the functools module. +# Written by Nick Coghlan , +# Raymond Hettinger , +# and Ɓukasz Langa . +# Copyright (C) 2006-2013 Python Software Foundation. +# See C source code for _functools credits/copyright + +__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', + 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial', + 'partialmethod', 'singledispatch'] + +try: + from _functools import reduce +except ImportError: + pass +from abc import get_cache_token +from collections import namedtuple +from types import MappingProxyType +from weakref import WeakKeyDictionary +try: + from _thread import RLock +except ImportError: + class RLock: + 'Dummy reentrant lock for builds without threads' + def __enter__(self): pass + def __exit__(self, exctype, excinst, exctb): pass + + +################################################################################ +### update_wrapper() and wraps() decorator +################################################################################ + +# update_wrapper() and wraps() are tools to help write +# wrapper functions that can handle naive introspection + +WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__', + '__annotations__') +WRAPPER_UPDATES = ('__dict__',) +def update_wrapper(wrapper, + wrapped, + assigned = WRAPPER_ASSIGNMENTS, + updated = WRAPPER_UPDATES): + """Update a wrapper function to look like the wrapped function + + wrapper is the function to be updated + wrapped is the original function + assigned is a tuple naming the attributes assigned directly + from the wrapped function to the wrapper function (defaults to + functools.WRAPPER_ASSIGNMENTS) + updated is a tuple naming the attributes of the wrapper that + are updated with the corresponding attribute from the wrapped + function (defaults to functools.WRAPPER_UPDATES) + """ + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + pass + else: + setattr(wrapper, attr, value) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + # Issue #17482: set __wrapped__ last so we don't inadvertently copy it + # from the wrapped function when updating __dict__ + wrapper.__wrapped__ = wrapped + # Return the wrapper so this can be used as a decorator via partial() + return wrapper + +def wraps(wrapped, + assigned = WRAPPER_ASSIGNMENTS, + updated = WRAPPER_UPDATES): + """Decorator factory to apply update_wrapper() to a wrapper function + + Returns a decorator that invokes update_wrapper() with the decorated + function as the wrapper argument and the arguments to wraps() as the + remaining arguments. Default arguments are as for update_wrapper(). + This is a convenience function to simplify applying partial() to + update_wrapper(). + """ + return partial(update_wrapper, wrapped=wrapped, + assigned=assigned, updated=updated) + + +################################################################################ +### total_ordering class decorator +################################################################################ + +# The total ordering functions all invoke the root magic method directly +# rather than using the corresponding operator. This avoids possible +# infinite recursion that could occur when the operator dispatch logic +# detects a NotImplemented result and then calls a reflected method. + +def _gt_from_lt(self, other, NotImplemented=NotImplemented): + 'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).' + op_result = self.__lt__(other) + if op_result is NotImplemented: + return op_result + return not op_result and self != other + +def _le_from_lt(self, other, NotImplemented=NotImplemented): + 'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).' + op_result = self.__lt__(other) + return op_result or self == other + +def _ge_from_lt(self, other, NotImplemented=NotImplemented): + 'Return a >= b. Computed by @total_ordering from (not a < b).' + op_result = self.__lt__(other) + if op_result is NotImplemented: + return op_result + return not op_result + +def _ge_from_le(self, other, NotImplemented=NotImplemented): + 'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).' + op_result = self.__le__(other) + if op_result is NotImplemented: + return op_result + return not op_result or self == other + +def _lt_from_le(self, other, NotImplemented=NotImplemented): + 'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).' + op_result = self.__le__(other) + if op_result is NotImplemented: + return op_result + return op_result and self != other + +def _gt_from_le(self, other, NotImplemented=NotImplemented): + 'Return a > b. Computed by @total_ordering from (not a <= b).' + op_result = self.__le__(other) + if op_result is NotImplemented: + return op_result + return not op_result + +def _lt_from_gt(self, other, NotImplemented=NotImplemented): + 'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).' + op_result = self.__gt__(other) + if op_result is NotImplemented: + return op_result + return not op_result and self != other + +def _ge_from_gt(self, other, NotImplemented=NotImplemented): + 'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).' + op_result = self.__gt__(other) + return op_result or self == other + +def _le_from_gt(self, other, NotImplemented=NotImplemented): + 'Return a <= b. Computed by @total_ordering from (not a > b).' + op_result = self.__gt__(other) + if op_result is NotImplemented: + return op_result + return not op_result + +def _le_from_ge(self, other, NotImplemented=NotImplemented): + 'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).' + op_result = self.__ge__(other) + if op_result is NotImplemented: + return op_result + return not op_result or self == other + +def _gt_from_ge(self, other, NotImplemented=NotImplemented): + 'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).' + op_result = self.__ge__(other) + if op_result is NotImplemented: + return op_result + return op_result and self != other + +def _lt_from_ge(self, other, NotImplemented=NotImplemented): + 'Return a < b. Computed by @total_ordering from (not a >= b).' + op_result = self.__ge__(other) + if op_result is NotImplemented: + return op_result + return not op_result + +_convert = { + '__lt__': [('__gt__', _gt_from_lt), + ('__le__', _le_from_lt), + ('__ge__', _ge_from_lt)], + '__le__': [('__ge__', _ge_from_le), + ('__lt__', _lt_from_le), + ('__gt__', _gt_from_le)], + '__gt__': [('__lt__', _lt_from_gt), + ('__ge__', _ge_from_gt), + ('__le__', _le_from_gt)], + '__ge__': [('__le__', _le_from_ge), + ('__gt__', _gt_from_ge), + ('__lt__', _lt_from_ge)] +} + +def total_ordering(cls): + """Class decorator that fills in missing ordering methods""" + # Find user-defined comparisons (not those inherited from object). + roots = [op for op in _convert if getattr(cls, op, None) is not getattr(object, op, None)] + if not roots: + raise ValueError('must define at least one ordering operation: < > <= >=') + root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__ + for opname, opfunc in _convert[root]: + if opname not in roots: + opfunc.__name__ = opname + setattr(cls, opname, opfunc) + return cls + + +################################################################################ +### cmp_to_key() function converter +################################################################################ + +def cmp_to_key(mycmp): + """Convert a cmp= function into a key= function""" + class K(object): + __slots__ = ['obj'] + def __init__(self, obj): + self.obj = obj + def __lt__(self, other): + return mycmp(self.obj, other.obj) < 0 + def __gt__(self, other): + return mycmp(self.obj, other.obj) > 0 + def __eq__(self, other): + return mycmp(self.obj, other.obj) == 0 + def __le__(self, other): + return mycmp(self.obj, other.obj) <= 0 + def __ge__(self, other): + return mycmp(self.obj, other.obj) >= 0 + __hash__ = None + return K + +try: + from _functools import cmp_to_key +except ImportError: + pass + + +################################################################################ +### partial() argument application +################################################################################ + +# Purely functional, no descriptor behaviour +def partial(func, *args, **keywords): + """New function with partial application of the given arguments + and keywords. + """ + if hasattr(func, 'func'): + args = func.args + args + tmpkw = func.keywords.copy() + tmpkw.update(keywords) + keywords = tmpkw + del tmpkw + func = func.func + + def newfunc(*fargs, **fkeywords): + newkeywords = keywords.copy() + newkeywords.update(fkeywords) + return func(*(args + fargs), **newkeywords) + newfunc.func = func + newfunc.args = args + newfunc.keywords = keywords + return newfunc + +try: + from _functools import partial +except ImportError: + pass + +# Descriptor version +class partialmethod(object): + """Method descriptor with partial application of the given arguments + and keywords. + + Supports wrapping existing descriptors and handles non-descriptor + callables as instance methods. + """ + + def __init__(self, func, *args, **keywords): + if not callable(func) and not hasattr(func, "__get__"): + raise TypeError("{!r} is not callable or a descriptor" + .format(func)) + + # func could be a descriptor like classmethod which isn't callable, + # so we can't inherit from partial (it verifies func is callable) + if isinstance(func, partialmethod): + # flattening is mandatory in order to place cls/self before all + # other arguments + # it's also more efficient since only one function will be called + self.func = func.func + self.args = func.args + args + self.keywords = func.keywords.copy() + self.keywords.update(keywords) + else: + self.func = func + self.args = args + self.keywords = keywords + + def __repr__(self): + args = ", ".join(map(repr, self.args)) + keywords = ", ".join("{}={!r}".format(k, v) + for k, v in self.keywords.items()) + format_string = "{module}.{cls}({func}, {args}, {keywords})" + return format_string.format(module=self.__class__.__module__, + cls=self.__class__.__qualname__, + func=self.func, + args=args, + keywords=keywords) + + def _make_unbound_method(self): + def _method(*args, **keywords): + call_keywords = self.keywords.copy() + call_keywords.update(keywords) + cls_or_self, *rest = args + call_args = (cls_or_self,) + self.args + tuple(rest) + return self.func(*call_args, **call_keywords) + _method.__isabstractmethod__ = self.__isabstractmethod__ + _method._partialmethod = self + return _method + + def __get__(self, obj, cls): + get = getattr(self.func, "__get__", None) + result = None + if get is not None: + new_func = get(obj, cls) + if new_func is not self.func: + # Assume __get__ returning something new indicates the + # creation of an appropriate callable + result = partial(new_func, *self.args, **self.keywords) + try: + result.__self__ = new_func.__self__ + except AttributeError: + pass + if result is None: + # If the underlying descriptor didn't do anything, treat this + # like an instance method + result = self._make_unbound_method().__get__(obj, cls) + return result + + @property + def __isabstractmethod__(self): + return getattr(self.func, "__isabstractmethod__", False) + + +################################################################################ +### LRU Cache function decorator +################################################################################ + +_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) + +class _HashedSeq(list): + """ This class guarantees that hash() will be called no more than once + per element. This is important because the lru_cache() will hash + the key multiple times on a cache miss. + + """ + + __slots__ = 'hashvalue' + + def __init__(self, tup, hash=hash): + self[:] = tup + self.hashvalue = hash(tup) + + def __hash__(self): + return self.hashvalue + +def _make_key(args, kwds, typed, + kwd_mark = (object(),), + fasttypes = {int, str, frozenset, type(None)}, + sorted=sorted, tuple=tuple, type=type, len=len): + """Make a cache key from optionally typed positional and keyword arguments + + The key is constructed in a way that is flat as possible rather than + as a nested structure that would take more memory. + + If there is only a single argument and its data type is known to cache + its hash value, then that argument is returned without a wrapper. This + saves space and improves lookup speed. + + """ + key = args + if kwds: + sorted_items = sorted(kwds.items()) + key += kwd_mark + for item in sorted_items: + key += item + if typed: + key += tuple(type(v) for v in args) + if kwds: + key += tuple(type(v) for k, v in sorted_items) + elif len(key) == 1 and type(key[0]) in fasttypes: + return key[0] + return _HashedSeq(key) + +def lru_cache(maxsize=128, typed=False): + """Least-recently-used cache decorator. + + If *maxsize* is set to None, the LRU features are disabled and the cache + can grow without bound. + + If *typed* is True, arguments of different types will be cached separately. + For example, f(3.0) and f(3) will be treated as distinct calls with + distinct results. + + Arguments to the cached function must be hashable. + + View the cache statistics named tuple (hits, misses, maxsize, currsize) + with f.cache_info(). Clear the cache and statistics with f.cache_clear(). + Access the underlying function with f.__wrapped__. + + See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used + + """ + + # Users should only access the lru_cache through its public API: + # cache_info, cache_clear, and f.__wrapped__ + # The internals of the lru_cache are encapsulated for thread safety and + # to allow the implementation to change (including a possible C version). + + # Early detection of an erroneous call to @lru_cache without any arguments + # resulting in the inner function being passed to maxsize instead of an + # integer or None. + if maxsize is not None and not isinstance(maxsize, int): + raise TypeError('Expected maxsize to be an integer or None') + + def decorating_function(user_function): + wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo) + return update_wrapper(wrapper, user_function) + + return decorating_function + +def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo): + # Constants shared by all lru cache instances: + sentinel = object() # unique object used to signal cache misses + make_key = _make_key # build a key from the function arguments + PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields + + cache = {} + hits = misses = 0 + full = False + cache_get = cache.get # bound method to lookup a key or return None + lock = RLock() # because linkedlist updates aren't threadsafe + root = [] # root of the circular doubly linked list + root[:] = [root, root, None, None] # initialize by pointing to self + + if maxsize == 0: + + def wrapper(*args, **kwds): + # No caching -- just a statistics update after a successful call + nonlocal misses + result = user_function(*args, **kwds) + misses += 1 + return result + + elif maxsize is None: + + def wrapper(*args, **kwds): + # Simple caching without ordering or size limit + nonlocal hits, misses + key = make_key(args, kwds, typed) + result = cache_get(key, sentinel) + if result is not sentinel: + hits += 1 + return result + result = user_function(*args, **kwds) + cache[key] = result + misses += 1 + return result + + else: + + def wrapper(*args, **kwds): + # Size limited caching that tracks accesses by recency + nonlocal root, hits, misses, full + key = make_key(args, kwds, typed) + with lock: + link = cache_get(key) + if link is not None: + # Move the link to the front of the circular queue + link_prev, link_next, _key, result = link + link_prev[NEXT] = link_next + link_next[PREV] = link_prev + last = root[PREV] + last[NEXT] = root[PREV] = link + link[PREV] = last + link[NEXT] = root + hits += 1 + return result + result = user_function(*args, **kwds) + with lock: + if key in cache: + # Getting here means that this same key was added to the + # cache while the lock was released. Since the link + # update is already done, we need only return the + # computed result and update the count of misses. + pass + elif full: + # Use the old root to store the new key and result. + oldroot = root + oldroot[KEY] = key + oldroot[RESULT] = result + # Empty the oldest link and make it the new root. + # Keep a reference to the old key and old result to + # prevent their ref counts from going to zero during the + # update. That will prevent potentially arbitrary object + # clean-up code (i.e. __del__) from running while we're + # still adjusting the links. + root = oldroot[NEXT] + oldkey = root[KEY] + oldresult = root[RESULT] + root[KEY] = root[RESULT] = None + # Now update the cache dictionary. + del cache[oldkey] + # Save the potentially reentrant cache[key] assignment + # for last, after the root and links have been put in + # a consistent state. + cache[key] = oldroot + else: + # Put result in a new link at the front of the queue. + last = root[PREV] + link = [last, root, key, result] + last[NEXT] = root[PREV] = cache[key] = link + full = (len(cache) >= maxsize) + misses += 1 + return result + + def cache_info(): + """Report cache statistics""" + with lock: + return _CacheInfo(hits, misses, maxsize, len(cache)) + + def cache_clear(): + """Clear the cache and cache statistics""" + nonlocal hits, misses, full + with lock: + cache.clear() + root[:] = [root, root, None, None] + hits = misses = 0 + full = False + + wrapper.cache_info = cache_info + wrapper.cache_clear = cache_clear + return wrapper + +try: + from _functools import _lru_cache_wrapper +except ImportError: + pass + + +################################################################################ +### singledispatch() - single-dispatch generic function decorator +################################################################################ + +def _c3_merge(sequences): + """Merges MROs in *sequences* to a single MRO using the C3 algorithm. + + Adapted from http://www.python.org/download/releases/2.3/mro/. + + """ + result = [] + while True: + sequences = [s for s in sequences if s] # purge empty sequences + if not sequences: + return result + for s1 in sequences: # find merge candidates among seq heads + candidate = s1[0] + for s2 in sequences: + if candidate in s2[1:]: + candidate = None + break # reject the current head, it appears later + else: + break + if candidate is None: + raise RuntimeError("Inconsistent hierarchy") + result.append(candidate) + # remove the chosen candidate + for seq in sequences: + if seq[0] == candidate: + del seq[0] + +def _c3_mro(cls, abcs=None): + """Computes the method resolution order using extended C3 linearization. + + If no *abcs* are given, the algorithm works exactly like the built-in C3 + linearization used for method resolution. + + If given, *abcs* is a list of abstract base classes that should be inserted + into the resulting MRO. Unrelated ABCs are ignored and don't end up in the + result. The algorithm inserts ABCs where their functionality is introduced, + i.e. issubclass(cls, abc) returns True for the class itself but returns + False for all its direct base classes. Implicit ABCs for a given class + (either registered or inferred from the presence of a special method like + __len__) are inserted directly after the last ABC explicitly listed in the + MRO of said class. If two implicit ABCs end up next to each other in the + resulting MRO, their ordering depends on the order of types in *abcs*. + + """ + for i, base in enumerate(reversed(cls.__bases__)): + if hasattr(base, '__abstractmethods__'): + boundary = len(cls.__bases__) - i + break # Bases up to the last explicit ABC are considered first. + else: + boundary = 0 + abcs = list(abcs) if abcs else [] + explicit_bases = list(cls.__bases__[:boundary]) + abstract_bases = [] + other_bases = list(cls.__bases__[boundary:]) + for base in abcs: + if issubclass(cls, base) and not any( + issubclass(b, base) for b in cls.__bases__ + ): + # If *cls* is the class that introduces behaviour described by + # an ABC *base*, insert said ABC to its MRO. + abstract_bases.append(base) + for base in abstract_bases: + abcs.remove(base) + explicit_c3_mros = [_c3_mro(base, abcs=abcs) for base in explicit_bases] + abstract_c3_mros = [_c3_mro(base, abcs=abcs) for base in abstract_bases] + other_c3_mros = [_c3_mro(base, abcs=abcs) for base in other_bases] + return _c3_merge( + [[cls]] + + explicit_c3_mros + abstract_c3_mros + other_c3_mros + + [explicit_bases] + [abstract_bases] + [other_bases] + ) + +def _compose_mro(cls, types): + """Calculates the method resolution order for a given class *cls*. + + Includes relevant abstract base classes (with their respective bases) from + the *types* iterable. Uses a modified C3 linearization algorithm. + + """ + bases = set(cls.__mro__) + # Remove entries which are already present in the __mro__ or unrelated. + def is_related(typ): + return (typ not in bases and hasattr(typ, '__mro__') + and issubclass(cls, typ)) + types = [n for n in types if is_related(n)] + # Remove entries which are strict bases of other entries (they will end up + # in the MRO anyway. + def is_strict_base(typ): + for other in types: + if typ != other and typ in other.__mro__: + return True + return False + types = [n for n in types if not is_strict_base(n)] + # Subclasses of the ABCs in *types* which are also implemented by + # *cls* can be used to stabilize ABC ordering. + type_set = set(types) + mro = [] + for typ in types: + found = [] + for sub in typ.__subclasses__(): + if sub not in bases and issubclass(cls, sub): + found.append([s for s in sub.__mro__ if s in type_set]) + if not found: + mro.append(typ) + continue + # Favor subclasses with the biggest number of useful bases + found.sort(key=len, reverse=True) + for sub in found: + for subcls in sub: + if subcls not in mro: + mro.append(subcls) + return _c3_mro(cls, abcs=mro) + +def _find_impl(cls, registry): + """Returns the best matching implementation from *registry* for type *cls*. + + Where there is no registered implementation for a specific type, its method + resolution order is used to find a more generic implementation. + + Note: if *registry* does not contain an implementation for the base + *object* type, this function may return None. + + """ + mro = _compose_mro(cls, registry.keys()) + match = None + for t in mro: + if match is not None: + # If *match* is an implicit ABC but there is another unrelated, + # equally matching implicit ABC, refuse the temptation to guess. + if (t in registry and t not in cls.__mro__ + and match not in cls.__mro__ + and not issubclass(match, t)): + raise RuntimeError("Ambiguous dispatch: {} or {}".format( + match, t)) + break + if t in registry: + match = t + return registry.get(match) + +def singledispatch(func): + """Single-dispatch generic function decorator. + + Transforms a function into a generic function, which can have different + behaviours depending upon the type of its first argument. The decorated + function acts as the default implementation, and additional + implementations can be registered using the register() attribute of the + generic function. + + """ + registry = {} + dispatch_cache = WeakKeyDictionary() + cache_token = None + + def dispatch(cls): + """generic_func.dispatch(cls) -> + + Runs the dispatch algorithm to return the best available implementation + for the given *cls* registered on *generic_func*. + + """ + nonlocal cache_token + if cache_token is not None: + current_token = get_cache_token() + if cache_token != current_token: + dispatch_cache.clear() + cache_token = current_token + try: + impl = dispatch_cache[cls] + except KeyError: + try: + impl = registry[cls] + except KeyError: + impl = _find_impl(cls, registry) + dispatch_cache[cls] = impl + return impl + + def register(cls, func=None): + """generic_func.register(cls, func) -> func + + Registers a new implementation for the given *cls* on a *generic_func*. + + """ + nonlocal cache_token + if func is None: + return lambda f: register(cls, f) + registry[cls] = func + if cache_token is None and hasattr(cls, '__abstractmethods__'): + cache_token = get_cache_token() + dispatch_cache.clear() + return func + + def wrapper(*args, **kw): + return dispatch(args[0].__class__)(*args, **kw) + + registry[object] = func + wrapper.register = register + wrapper.dispatch = dispatch + wrapper.registry = MappingProxyType(registry) + wrapper._clear_cache = dispatch_cache.clear + update_wrapper(wrapper, func) + return wrapper +r"""plistlib.py -- a tool to generate and parse MacOSX .plist files. + +The property list (.plist) file format is a simple XML pickle supporting +basic object types, like dictionaries, lists, numbers and strings. +Usually the top level object is a dictionary. + +To write out a plist file, use the dump(value, file) +function. 'value' is the top level object, 'file' is +a (writable) file object. + +To parse a plist from a file, use the load(file) function, +with a (readable) file object as the only argument. It +returns the top level object (again, usually a dictionary). + +To work with plist data in bytes objects, you can use loads() +and dumps(). + +Values can be strings, integers, floats, booleans, tuples, lists, +dictionaries (but only with string keys), Data, bytes, bytearray, or +datetime.datetime objects. + +Generate Plist example: + + pl = dict( + aString = "Doodah", + aList = ["A", "B", 12, 32.1, [1, 2, 3]], + aFloat = 0.1, + anInt = 728, + aDict = dict( + anotherString = "", + aUnicodeValue = "M\xe4ssig, Ma\xdf", + aTrueValue = True, + aFalseValue = False, + ), + someData = b"", + someMoreData = b"" * 10, + aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())), + ) + with open(fileName, 'wb') as fp: + dump(pl, fp) + +Parse Plist example: + + with open(fileName, 'rb') as fp: + pl = load(fp) + print(pl["aKey"]) +""" +__all__ = [ + "readPlist", "writePlist", "readPlistFromBytes", "writePlistToBytes", + "Plist", "Data", "Dict", "FMT_XML", "FMT_BINARY", + "load", "dump", "loads", "dumps" +] + +import binascii +import codecs +import contextlib +import datetime +import enum +from io import BytesIO +import itertools +import os +import re +import struct +from warnings import warn +from xml.parsers.expat import ParserCreate + + +PlistFormat = enum.Enum('PlistFormat', 'FMT_XML FMT_BINARY', module=__name__) +globals().update(PlistFormat.__members__) + + +# +# +# Deprecated functionality +# +# + + +class _InternalDict(dict): + + # This class is needed while Dict is scheduled for deprecation: + # we only need to warn when a *user* instantiates Dict or when + # the "attribute notation for dict keys" is used. + __slots__ = () + + def __getattr__(self, attr): + try: + value = self[attr] + except KeyError: + raise AttributeError(attr) + warn("Attribute access from plist dicts is deprecated, use d[key] " + "notation instead", DeprecationWarning, 2) + return value + + def __setattr__(self, attr, value): + warn("Attribute access from plist dicts is deprecated, use d[key] " + "notation instead", DeprecationWarning, 2) + self[attr] = value + + def __delattr__(self, attr): + try: + del self[attr] + except KeyError: + raise AttributeError(attr) + warn("Attribute access from plist dicts is deprecated, use d[key] " + "notation instead", DeprecationWarning, 2) + + +class Dict(_InternalDict): + + def __init__(self, **kwargs): + warn("The plistlib.Dict class is deprecated, use builtin dict instead", + DeprecationWarning, 2) + super().__init__(**kwargs) + + +@contextlib.contextmanager +def _maybe_open(pathOrFile, mode): + if isinstance(pathOrFile, str): + with open(pathOrFile, mode) as fp: + yield fp + + else: + yield pathOrFile + + +class Plist(_InternalDict): + """This class has been deprecated. Use dump() and load() + functions instead, together with regular dict objects. + """ + + def __init__(self, **kwargs): + warn("The Plist class is deprecated, use the load() and " + "dump() functions instead", DeprecationWarning, 2) + super().__init__(**kwargs) + + @classmethod + def fromFile(cls, pathOrFile): + """Deprecated. Use the load() function instead.""" + with _maybe_open(pathOrFile, 'rb') as fp: + value = load(fp) + plist = cls() + plist.update(value) + return plist + + def write(self, pathOrFile): + """Deprecated. Use the dump() function instead.""" + with _maybe_open(pathOrFile, 'wb') as fp: + dump(self, fp) + + +def readPlist(pathOrFile): + """ + Read a .plist from a path or file. pathOrFile should either + be a file name, or a readable binary file object. + + This function is deprecated, use load instead. + """ + warn("The readPlist function is deprecated, use load() instead", + DeprecationWarning, 2) + + with _maybe_open(pathOrFile, 'rb') as fp: + return load(fp, fmt=None, use_builtin_types=False, + dict_type=_InternalDict) + +def writePlist(value, pathOrFile): + """ + Write 'value' to a .plist file. 'pathOrFile' may either be a + file name or a (writable) file object. + + This function is deprecated, use dump instead. + """ + warn("The writePlist function is deprecated, use dump() instead", + DeprecationWarning, 2) + with _maybe_open(pathOrFile, 'wb') as fp: + dump(value, fp, fmt=FMT_XML, sort_keys=True, skipkeys=False) + + +def readPlistFromBytes(data): + """ + Read a plist data from a bytes object. Return the root object. + + This function is deprecated, use loads instead. + """ + warn("The readPlistFromBytes function is deprecated, use loads() instead", + DeprecationWarning, 2) + return load(BytesIO(data), fmt=None, use_builtin_types=False, + dict_type=_InternalDict) + + +def writePlistToBytes(value): + """ + Return 'value' as a plist-formatted bytes object. + + This function is deprecated, use dumps instead. + """ + warn("The writePlistToBytes function is deprecated, use dumps() instead", + DeprecationWarning, 2) + f = BytesIO() + dump(value, f, fmt=FMT_XML, sort_keys=True, skipkeys=False) + return f.getvalue() + + +class Data: + """ + Wrapper for binary data. + + This class is deprecated, use a bytes object instead. + """ + + def __init__(self, data): + if not isinstance(data, bytes): + raise TypeError("data must be as bytes") + self.data = data + + @classmethod + def fromBase64(cls, data): + # base64.decodebytes just calls binascii.a2b_base64; + # it seems overkill to use both base64 and binascii. + return cls(_decode_base64(data)) + + def asBase64(self, maxlinelength=76): + return _encode_base64(self.data, maxlinelength) + + def __eq__(self, other): + if isinstance(other, self.__class__): + return self.data == other.data + elif isinstance(other, bytes): + return self.data == other + else: + return NotImplemented + + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, repr(self.data)) + +# +# +# End of deprecated functionality +# +# + + +# +# XML support +# + + +# XML 'header' +PLISTHEADER = b"""\ + + +""" + + +# Regex to find any control chars, except for \t \n and \r +_controlCharPat = re.compile( + r"[\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0b\x0c\x0e\x0f" + r"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f]") + +def _encode_base64(s, maxlinelength=76): + # copied from base64.encodebytes(), with added maxlinelength argument + maxbinsize = (maxlinelength//4)*3 + pieces = [] + for i in range(0, len(s), maxbinsize): + chunk = s[i : i + maxbinsize] + pieces.append(binascii.b2a_base64(chunk)) + return b''.join(pieces) + +def _decode_base64(s): + if isinstance(s, str): + return binascii.a2b_base64(s.encode("utf-8")) + + else: + return binascii.a2b_base64(s) + +# Contents should conform to a subset of ISO 8601 +# (in particular, YYYY '-' MM '-' DD 'T' HH ':' MM ':' SS 'Z'. Smaller units +# may be omitted with # a loss of precision) +_dateParser = re.compile(r"(?P\d\d\d\d)(?:-(?P\d\d)(?:-(?P\d\d)(?:T(?P\d\d)(?::(?P\d\d)(?::(?P\d\d))?)?)?)?)?Z", re.ASCII) + + +def _date_from_string(s): + order = ('year', 'month', 'day', 'hour', 'minute', 'second') + gd = _dateParser.match(s).groupdict() + lst = [] + for key in order: + val = gd[key] + if val is None: + break + lst.append(int(val)) + return datetime.datetime(*lst) + + +def _date_to_string(d): + return '%04d-%02d-%02dT%02d:%02d:%02dZ' % ( + d.year, d.month, d.day, + d.hour, d.minute, d.second + ) + +def _escape(text): + m = _controlCharPat.search(text) + if m is not None: + raise ValueError("strings can't contains control characters; " + "use bytes instead") + text = text.replace("\r\n", "\n") # convert DOS line endings + text = text.replace("\r", "\n") # convert Mac line endings + text = text.replace("&", "&") # escape '&' + text = text.replace("<", "<") # escape '<' + text = text.replace(">", ">") # escape '>' + return text + +class _PlistParser: + def __init__(self, use_builtin_types, dict_type): + self.stack = [] + self.current_key = None + self.root = None + self._use_builtin_types = use_builtin_types + self._dict_type = dict_type + + def parse(self, fileobj): + self.parser = ParserCreate() + self.parser.StartElementHandler = self.handle_begin_element + self.parser.EndElementHandler = self.handle_end_element + self.parser.CharacterDataHandler = self.handle_data + self.parser.ParseFile(fileobj) + return self.root + + def handle_begin_element(self, element, attrs): + self.data = [] + handler = getattr(self, "begin_" + element, None) + if handler is not None: + handler(attrs) + + def handle_end_element(self, element): + handler = getattr(self, "end_" + element, None) + if handler is not None: + handler() + + def handle_data(self, data): + self.data.append(data) + + def add_object(self, value): + if self.current_key is not None: + if not isinstance(self.stack[-1], type({})): + raise ValueError("unexpected element at line %d" % + self.parser.CurrentLineNumber) + self.stack[-1][self.current_key] = value + self.current_key = None + elif not self.stack: + # this is the root object + self.root = value + else: + if not isinstance(self.stack[-1], type([])): + raise ValueError("unexpected element at line %d" % + self.parser.CurrentLineNumber) + self.stack[-1].append(value) + + def get_data(self): + data = ''.join(self.data) + self.data = [] + return data + + # element handlers + + def begin_dict(self, attrs): + d = self._dict_type() + self.add_object(d) + self.stack.append(d) + + def end_dict(self): + if self.current_key: + raise ValueError("missing value for key '%s' at line %d" % + (self.current_key,self.parser.CurrentLineNumber)) + self.stack.pop() + + def end_key(self): + if self.current_key or not isinstance(self.stack[-1], type({})): + raise ValueError("unexpected key at line %d" % + self.parser.CurrentLineNumber) + self.current_key = self.get_data() + + def begin_array(self, attrs): + a = [] + self.add_object(a) + self.stack.append(a) + + def end_array(self): + self.stack.pop() + + def end_true(self): + self.add_object(True) + + def end_false(self): + self.add_object(False) + + def end_integer(self): + self.add_object(int(self.get_data())) + + def end_real(self): + self.add_object(float(self.get_data())) + + def end_string(self): + self.add_object(self.get_data()) + + def end_data(self): + if self._use_builtin_types: + self.add_object(_decode_base64(self.get_data())) + + else: + self.add_object(Data.fromBase64(self.get_data())) + + def end_date(self): + self.add_object(_date_from_string(self.get_data())) + + +class _DumbXMLWriter: + def __init__(self, file, indent_level=0, indent="\t"): + self.file = file + self.stack = [] + self._indent_level = indent_level + self.indent = indent + + def begin_element(self, element): + self.stack.append(element) + self.writeln("<%s>" % element) + self._indent_level += 1 + + def end_element(self, element): + assert self._indent_level > 0 + assert self.stack.pop() == element + self._indent_level -= 1 + self.writeln("" % element) + + def simple_element(self, element, value=None): + if value is not None: + value = _escape(value) + self.writeln("<%s>%s" % (element, value, element)) + + else: + self.writeln("<%s/>" % element) + + def writeln(self, line): + if line: + # plist has fixed encoding of utf-8 + + # XXX: is this test needed? + if isinstance(line, str): + line = line.encode('utf-8') + self.file.write(self._indent_level * self.indent) + self.file.write(line) + self.file.write(b'\n') + + +class _PlistWriter(_DumbXMLWriter): + def __init__( + self, file, indent_level=0, indent=b"\t", writeHeader=1, + sort_keys=True, skipkeys=False): + + if writeHeader: + file.write(PLISTHEADER) + _DumbXMLWriter.__init__(self, file, indent_level, indent) + self._sort_keys = sort_keys + self._skipkeys = skipkeys + + def write(self, value): + self.writeln("") + self.write_value(value) + self.writeln("") + + def write_value(self, value): + if isinstance(value, str): + self.simple_element("string", value) + + elif value is True: + self.simple_element("true") + + elif value is False: + self.simple_element("false") + + elif isinstance(value, int): + if -1 << 63 <= value < 1 << 64: + self.simple_element("integer", "%d" % value) + else: + raise OverflowError(value) + + elif isinstance(value, float): + self.simple_element("real", repr(value)) + + elif isinstance(value, dict): + self.write_dict(value) + + elif isinstance(value, Data): + self.write_data(value) + + elif isinstance(value, (bytes, bytearray)): + self.write_bytes(value) + + elif isinstance(value, datetime.datetime): + self.simple_element("date", _date_to_string(value)) + + elif isinstance(value, (tuple, list)): + self.write_array(value) + + else: + raise TypeError("unsupported type: %s" % type(value)) + + def write_data(self, data): + self.write_bytes(data.data) + + def write_bytes(self, data): + self.begin_element("data") + self._indent_level -= 1 + maxlinelength = max( + 16, + 76 - len(self.indent.replace(b"\t", b" " * 8) * self._indent_level)) + + for line in _encode_base64(data, maxlinelength).split(b"\n"): + if line: + self.writeln(line) + self._indent_level += 1 + self.end_element("data") + + def write_dict(self, d): + if d: + self.begin_element("dict") + if self._sort_keys: + items = sorted(d.items()) + else: + items = d.items() + + for key, value in items: + if not isinstance(key, str): + if self._skipkeys: + continue + raise TypeError("keys must be strings") + self.simple_element("key", key) + self.write_value(value) + self.end_element("dict") + + else: + self.simple_element("dict") + + def write_array(self, array): + if array: + self.begin_element("array") + for value in array: + self.write_value(value) + self.end_element("array") + + else: + self.simple_element("array") + + +def _is_fmt_xml(header): + prefixes = (b'offset... + # TRAILER + self._fp = fp + self._fp.seek(-32, os.SEEK_END) + trailer = self._fp.read(32) + if len(trailer) != 32: + raise InvalidFileException() + ( + offset_size, self._ref_size, num_objects, top_object, + offset_table_offset + ) = struct.unpack('>6xBBQQQ', trailer) + self._fp.seek(offset_table_offset) + self._object_offsets = self._read_ints(num_objects, offset_size) + return self._read_object(self._object_offsets[top_object]) + + except (OSError, IndexError, struct.error): + raise InvalidFileException() + + def _get_size(self, tokenL): + """ return the size of the next object.""" + if tokenL == 0xF: + m = self._fp.read(1)[0] & 0x3 + s = 1 << m + f = '>' + _BINARY_FORMAT[s] + return struct.unpack(f, self._fp.read(s))[0] + + return tokenL + + def _read_ints(self, n, size): + data = self._fp.read(size * n) + if size in _BINARY_FORMAT: + return struct.unpack('>' + _BINARY_FORMAT[size] * n, data) + else: + return tuple(int.from_bytes(data[i: i + size], 'big') + for i in range(0, size * n, size)) + + def _read_refs(self, n): + return self._read_ints(n, self._ref_size) + + def _read_object(self, offset): + """ + read the object at offset. + + May recursively read sub-objects (content of an array/dict/set) + """ + self._fp.seek(offset) + token = self._fp.read(1)[0] + tokenH, tokenL = token & 0xF0, token & 0x0F + + if token == 0x00: + return None + + elif token == 0x08: + return False + + elif token == 0x09: + return True + + # The referenced source code also mentions URL (0x0c, 0x0d) and + # UUID (0x0e), but neither can be generated using the Cocoa libraries. + + elif token == 0x0f: + return b'' + + elif tokenH == 0x10: # int + return int.from_bytes(self._fp.read(1 << tokenL), + 'big', signed=tokenL >= 3) + + elif token == 0x22: # real + return struct.unpack('>f', self._fp.read(4))[0] + + elif token == 0x23: # real + return struct.unpack('>d', self._fp.read(8))[0] + + elif token == 0x33: # date + f = struct.unpack('>d', self._fp.read(8))[0] + # timestamp 0 of binary plists corresponds to 1/1/2001 + # (year of Mac OS X 10.0), instead of 1/1/1970. + return datetime.datetime(2001, 1, 1) + datetime.timedelta(seconds=f) + + elif tokenH == 0x40: # data + s = self._get_size(tokenL) + if self._use_builtin_types: + return self._fp.read(s) + else: + return Data(self._fp.read(s)) + + elif tokenH == 0x50: # ascii string + s = self._get_size(tokenL) + result = self._fp.read(s).decode('ascii') + return result + + elif tokenH == 0x60: # unicode string + s = self._get_size(tokenL) + return self._fp.read(s * 2).decode('utf-16be') + + # tokenH == 0x80 is documented as 'UID' and appears to be used for + # keyed-archiving, not in plists. + + elif tokenH == 0xA0: # array + s = self._get_size(tokenL) + obj_refs = self._read_refs(s) + return [self._read_object(self._object_offsets[x]) + for x in obj_refs] + + # tokenH == 0xB0 is documented as 'ordset', but is not actually + # implemented in the Apple reference code. + + # tokenH == 0xC0 is documented as 'set', but sets cannot be used in + # plists. + + elif tokenH == 0xD0: # dict + s = self._get_size(tokenL) + key_refs = self._read_refs(s) + obj_refs = self._read_refs(s) + result = self._dict_type() + for k, o in zip(key_refs, obj_refs): + result[self._read_object(self._object_offsets[k]) + ] = self._read_object(self._object_offsets[o]) + return result + + raise InvalidFileException() + +def _count_to_size(count): + if count < 1 << 8: + return 1 + + elif count < 1 << 16: + return 2 + + elif count << 1 << 32: + return 4 + + else: + return 8 + +class _BinaryPlistWriter (object): + def __init__(self, fp, sort_keys, skipkeys): + self._fp = fp + self._sort_keys = sort_keys + self._skipkeys = skipkeys + + def write(self, value): + + # Flattened object list: + self._objlist = [] + + # Mappings from object->objectid + # First dict has (type(object), object) as the key, + # second dict is used when object is not hashable and + # has id(object) as the key. + self._objtable = {} + self._objidtable = {} + + # Create list of all objects in the plist + self._flatten(value) + + # Size of object references in serialized containers + # depends on the number of objects in the plist. + num_objects = len(self._objlist) + self._object_offsets = [0]*num_objects + self._ref_size = _count_to_size(num_objects) + + self._ref_format = _BINARY_FORMAT[self._ref_size] + + # Write file header + self._fp.write(b'bplist00') + + # Write object list + for obj in self._objlist: + self._write_object(obj) + + # Write refnum->object offset table + top_object = self._getrefnum(value) + offset_table_offset = self._fp.tell() + offset_size = _count_to_size(offset_table_offset) + offset_format = '>' + _BINARY_FORMAT[offset_size] * num_objects + self._fp.write(struct.pack(offset_format, *self._object_offsets)) + + # Write trailer + sort_version = 0 + trailer = ( + sort_version, offset_size, self._ref_size, num_objects, + top_object, offset_table_offset + ) + self._fp.write(struct.pack('>5xBBBQQQ', *trailer)) + + def _flatten(self, value): + # First check if the object is in the object table, not used for + # containers to ensure that two subcontainers with the same contents + # will be serialized as distinct values. + if isinstance(value, ( + str, int, float, datetime.datetime, bytes, bytearray)): + if (type(value), value) in self._objtable: + return + + elif isinstance(value, Data): + if (type(value.data), value.data) in self._objtable: + return + + # Add to objectreference map + refnum = len(self._objlist) + self._objlist.append(value) + try: + if isinstance(value, Data): + self._objtable[(type(value.data), value.data)] = refnum + else: + self._objtable[(type(value), value)] = refnum + except TypeError: + self._objidtable[id(value)] = refnum + + # And finally recurse into containers + if isinstance(value, dict): + keys = [] + values = [] + items = value.items() + if self._sort_keys: + items = sorted(items) + + for k, v in items: + if not isinstance(k, str): + if self._skipkeys: + continue + raise TypeError("keys must be strings") + keys.append(k) + values.append(v) + + for o in itertools.chain(keys, values): + self._flatten(o) + + elif isinstance(value, (list, tuple)): + for o in value: + self._flatten(o) + + def _getrefnum(self, value): + try: + if isinstance(value, Data): + return self._objtable[(type(value.data), value.data)] + else: + return self._objtable[(type(value), value)] + except TypeError: + return self._objidtable[id(value)] + + def _write_size(self, token, size): + if size < 15: + self._fp.write(struct.pack('>B', token | size)) + + elif size < 1 << 8: + self._fp.write(struct.pack('>BBB', token | 0xF, 0x10, size)) + + elif size < 1 << 16: + self._fp.write(struct.pack('>BBH', token | 0xF, 0x11, size)) + + elif size < 1 << 32: + self._fp.write(struct.pack('>BBL', token | 0xF, 0x12, size)) + + else: + self._fp.write(struct.pack('>BBQ', token | 0xF, 0x13, size)) + + def _write_object(self, value): + ref = self._getrefnum(value) + self._object_offsets[ref] = self._fp.tell() + if value is None: + self._fp.write(b'\x00') + + elif value is False: + self._fp.write(b'\x08') + + elif value is True: + self._fp.write(b'\x09') + + elif isinstance(value, int): + if value < 0: + try: + self._fp.write(struct.pack('>Bq', 0x13, value)) + except struct.error: + raise OverflowError(value) from None + elif value < 1 << 8: + self._fp.write(struct.pack('>BB', 0x10, value)) + elif value < 1 << 16: + self._fp.write(struct.pack('>BH', 0x11, value)) + elif value < 1 << 32: + self._fp.write(struct.pack('>BL', 0x12, value)) + elif value < 1 << 63: + self._fp.write(struct.pack('>BQ', 0x13, value)) + elif value < 1 << 64: + self._fp.write(b'\x14' + value.to_bytes(16, 'big', signed=True)) + else: + raise OverflowError(value) + + elif isinstance(value, float): + self._fp.write(struct.pack('>Bd', 0x23, value)) + + elif isinstance(value, datetime.datetime): + f = (value - datetime.datetime(2001, 1, 1)).total_seconds() + self._fp.write(struct.pack('>Bd', 0x33, f)) + + elif isinstance(value, Data): + self._write_size(0x40, len(value.data)) + self._fp.write(value.data) + + elif isinstance(value, (bytes, bytearray)): + self._write_size(0x40, len(value)) + self._fp.write(value) + + elif isinstance(value, str): + try: + t = value.encode('ascii') + self._write_size(0x50, len(value)) + except UnicodeEncodeError: + t = value.encode('utf-16be') + self._write_size(0x60, len(value)) + + self._fp.write(t) + + elif isinstance(value, (list, tuple)): + refs = [self._getrefnum(o) for o in value] + s = len(refs) + self._write_size(0xA0, s) + self._fp.write(struct.pack('>' + self._ref_format * s, *refs)) + + elif isinstance(value, dict): + keyRefs, valRefs = [], [] + + if self._sort_keys: + rootItems = sorted(value.items()) + else: + rootItems = value.items() + + for k, v in rootItems: + if not isinstance(k, str): + if self._skipkeys: + continue + raise TypeError("keys must be strings") + keyRefs.append(self._getrefnum(k)) + valRefs.append(self._getrefnum(v)) + + s = len(keyRefs) + self._write_size(0xD0, s) + self._fp.write(struct.pack('>' + self._ref_format * s, *keyRefs)) + self._fp.write(struct.pack('>' + self._ref_format * s, *valRefs)) + + else: + raise TypeError(value) + + +def _is_fmt_binary(header): + return header[:8] == b'bplist00' + + +# +# Generic bits +# + +_FORMATS={ + FMT_XML: dict( + detect=_is_fmt_xml, + parser=_PlistParser, + writer=_PlistWriter, + ), + FMT_BINARY: dict( + detect=_is_fmt_binary, + parser=_BinaryPlistParser, + writer=_BinaryPlistWriter, + ) +} + + +def load(fp, *, fmt=None, use_builtin_types=True, dict_type=dict): + """Read a .plist file. 'fp' should be (readable) file object. + Return the unpacked root object (which usually is a dictionary). + """ + if fmt is None: + header = fp.read(32) + fp.seek(0) + for info in _FORMATS.values(): + if info['detect'](header): + P = info['parser'] + break + + else: + raise InvalidFileException() + + else: + P = _FORMATS[fmt]['parser'] + + p = P(use_builtin_types=use_builtin_types, dict_type=dict_type) + return p.parse(fp) + + +def loads(value, *, fmt=None, use_builtin_types=True, dict_type=dict): + """Read a .plist file from a bytes object. + Return the unpacked root object (which usually is a dictionary). + """ + fp = BytesIO(value) + return load( + fp, fmt=fmt, use_builtin_types=use_builtin_types, dict_type=dict_type) + + +def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False): + """Write 'value' to a .plist file. 'fp' should be a (writable) + file object. + """ + if fmt not in _FORMATS: + raise ValueError("Unsupported format: %r"%(fmt,)) + + writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys) + writer.write(value) + + +def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True): + """Return a bytes object with the contents for a .plist file. + """ + fp = BytesIO() + dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys) + return fp.getvalue() +"""Generic socket server classes. + +This module tries to capture the various aspects of defining a server: + +For socket-based servers: + +- address family: + - AF_INET{,6}: IP (Internet Protocol) sockets (default) + - AF_UNIX: Unix domain sockets + - others, e.g. AF_DECNET are conceivable (see +- socket type: + - SOCK_STREAM (reliable stream, e.g. TCP) + - SOCK_DGRAM (datagrams, e.g. UDP) + +For request-based servers (including socket-based): + +- client address verification before further looking at the request + (This is actually a hook for any processing that needs to look + at the request before anything else, e.g. logging) +- how to handle multiple requests: + - synchronous (one request is handled at a time) + - forking (each request is handled by a new process) + - threading (each request is handled by a new thread) + +The classes in this module favor the server type that is simplest to +write: a synchronous TCP/IP server. This is bad class design, but +save some typing. (There's also the issue that a deep class hierarchy +slows down method lookups.) + +There are five classes in an inheritance diagram, four of which represent +synchronous servers of four types: + + +------------+ + | BaseServer | + +------------+ + | + v + +-----------+ +------------------+ + | TCPServer |------->| UnixStreamServer | + +-----------+ +------------------+ + | + v + +-----------+ +--------------------+ + | UDPServer |------->| UnixDatagramServer | + +-----------+ +--------------------+ + +Note that UnixDatagramServer derives from UDPServer, not from +UnixStreamServer -- the only difference between an IP and a Unix +stream server is the address family, which is simply repeated in both +unix server classes. + +Forking and threading versions of each type of server can be created +using the ForkingMixIn and ThreadingMixIn mix-in classes. For +instance, a threading UDP server class is created as follows: + + class ThreadingUDPServer(ThreadingMixIn, UDPServer): pass + +The Mix-in class must come first, since it overrides a method defined +in UDPServer! Setting the various member variables also changes +the behavior of the underlying server mechanism. + +To implement a service, you must derive a class from +BaseRequestHandler and redefine its handle() method. You can then run +various versions of the service by combining one of the server classes +with your request handler class. + +The request handler class must be different for datagram or stream +services. This can be hidden by using the request handler +subclasses StreamRequestHandler or DatagramRequestHandler. + +Of course, you still have to use your head! + +For instance, it makes no sense to use a forking server if the service +contains state in memory that can be modified by requests (since the +modifications in the child process would never reach the initial state +kept in the parent process and passed to each child). In this case, +you can use a threading server, but you will probably have to use +locks to avoid two requests that come in nearly simultaneous to apply +conflicting changes to the server state. + +On the other hand, if you are building e.g. an HTTP server, where all +data is stored externally (e.g. in the file system), a synchronous +class will essentially render the service "deaf" while one request is +being handled -- which may be for a very long time if a client is slow +to read all the data it has requested. Here a threading or forking +server is appropriate. + +In some cases, it may be appropriate to process part of a request +synchronously, but to finish processing in a forked child depending on +the request data. This can be implemented by using a synchronous +server and doing an explicit fork in the request handler class +handle() method. + +Another approach to handling multiple simultaneous requests in an +environment that supports neither threads nor fork (or where these are +too expensive or inappropriate for the service) is to maintain an +explicit table of partially finished requests and to use a selector to +decide which request to work on next (or whether to handle a new +incoming request). This is particularly important for stream services +where each client can potentially be connected for a long time (if +threads or subprocesses cannot be used). + +Future work: +- Standard classes for Sun RPC (which uses either UDP or TCP) +- Standard mix-in classes to implement various authentication + and encryption schemes + +XXX Open problems: +- What to do with out-of-band data? + +BaseServer: +- split generic "request" functionality out into BaseServer class. + Copyright (C) 2000 Luke Kenneth Casson Leighton + + example: read entries from a SQL database (requires overriding + get_request() to return a table entry from the database). + entry is processed by a RequestHandlerClass. + +""" + +# Author of the BaseServer patch: Luke Kenneth Casson Leighton + +__version__ = "0.4" + + +import socket +import selectors +import os +import errno +try: + import threading +except ImportError: + import dummy_threading as threading +from time import monotonic as time + +__all__ = ["BaseServer", "TCPServer", "UDPServer", "ForkingUDPServer", + "ForkingTCPServer", "ThreadingUDPServer", "ThreadingTCPServer", + "BaseRequestHandler", "StreamRequestHandler", + "DatagramRequestHandler", "ThreadingMixIn", "ForkingMixIn"] +if hasattr(socket, "AF_UNIX"): + __all__.extend(["UnixStreamServer","UnixDatagramServer", + "ThreadingUnixStreamServer", + "ThreadingUnixDatagramServer"]) + +# poll/select have the advantage of not requiring any extra file descriptor, +# contrarily to epoll/kqueue (also, they require a single syscall). +if hasattr(selectors, 'PollSelector'): + _ServerSelector = selectors.PollSelector +else: + _ServerSelector = selectors.SelectSelector + + +class BaseServer: + + """Base class for server classes. + + Methods for the caller: + + - __init__(server_address, RequestHandlerClass) + - serve_forever(poll_interval=0.5) + - shutdown() + - handle_request() # if you do not use serve_forever() + - fileno() -> int # for selector + + Methods that may be overridden: + + - server_bind() + - server_activate() + - get_request() -> request, client_address + - handle_timeout() + - verify_request(request, client_address) + - server_close() + - process_request(request, client_address) + - shutdown_request(request) + - close_request(request) + - service_actions() + - handle_error() + + Methods for derived classes: + + - finish_request(request, client_address) + + Class variables that may be overridden by derived classes or + instances: + + - timeout + - address_family + - socket_type + - allow_reuse_address + + Instance variables: + + - RequestHandlerClass + - socket + + """ + + timeout = None + + def __init__(self, server_address, RequestHandlerClass): + """Constructor. May be extended, do not override.""" + self.server_address = server_address + self.RequestHandlerClass = RequestHandlerClass + self.__is_shut_down = threading.Event() + self.__shutdown_request = False + + def server_activate(self): + """Called by constructor to activate the server. + + May be overridden. + + """ + pass + + def serve_forever(self, poll_interval=0.5): + """Handle one request at a time until shutdown. + + Polls for shutdown every poll_interval seconds. Ignores + self.timeout. If you need to do periodic tasks, do them in + another thread. + """ + self.__is_shut_down.clear() + try: + # XXX: Consider using another file descriptor or connecting to the + # socket to wake this up instead of polling. Polling reduces our + # responsiveness to a shutdown request and wastes cpu at all other + # times. + with _ServerSelector() as selector: + selector.register(self, selectors.EVENT_READ) + + while not self.__shutdown_request: + ready = selector.select(poll_interval) + if ready: + self._handle_request_noblock() + + self.service_actions() + finally: + self.__shutdown_request = False + self.__is_shut_down.set() + + def shutdown(self): + """Stops the serve_forever loop. + + Blocks until the loop has finished. This must be called while + serve_forever() is running in another thread, or it will + deadlock. + """ + self.__shutdown_request = True + self.__is_shut_down.wait() + + def service_actions(self): + """Called by the serve_forever() loop. + + May be overridden by a subclass / Mixin to implement any code that + needs to be run during the loop. + """ + pass + + # The distinction between handling, getting, processing and finishing a + # request is fairly arbitrary. Remember: + # + # - handle_request() is the top-level call. It calls selector.select(), + # get_request(), verify_request() and process_request() + # - get_request() is different for stream or datagram sockets + # - process_request() is the place that may fork a new process or create a + # new thread to finish the request + # - finish_request() instantiates the request handler class; this + # constructor will handle the request all by itself + + def handle_request(self): + """Handle one request, possibly blocking. + + Respects self.timeout. + """ + # Support people who used socket.settimeout() to escape + # handle_request before self.timeout was available. + timeout = self.socket.gettimeout() + if timeout is None: + timeout = self.timeout + elif self.timeout is not None: + timeout = min(timeout, self.timeout) + if timeout is not None: + deadline = time() + timeout + + # Wait until a request arrives or the timeout expires - the loop is + # necessary to accommodate early wakeups due to EINTR. + with _ServerSelector() as selector: + selector.register(self, selectors.EVENT_READ) + + while True: + ready = selector.select(timeout) + if ready: + return self._handle_request_noblock() + else: + if timeout is not None: + timeout = deadline - time() + if timeout < 0: + return self.handle_timeout() + + def _handle_request_noblock(self): + """Handle one request, without blocking. + + I assume that selector.select() has returned that the socket is + readable before this function was called, so there should be no risk of + blocking in get_request(). + """ + try: + request, client_address = self.get_request() + except OSError: + return + if self.verify_request(request, client_address): + try: + self.process_request(request, client_address) + except: + self.handle_error(request, client_address) + self.shutdown_request(request) + else: + self.shutdown_request(request) + + def handle_timeout(self): + """Called if no new request arrives within self.timeout. + + Overridden by ForkingMixIn. + """ + pass + + def verify_request(self, request, client_address): + """Verify the request. May be overridden. + + Return True if we should proceed with this request. + + """ + return True + + def process_request(self, request, client_address): + """Call finish_request. + + Overridden by ForkingMixIn and ThreadingMixIn. + + """ + self.finish_request(request, client_address) + self.shutdown_request(request) + + def server_close(self): + """Called to clean-up the server. + + May be overridden. + + """ + pass + + def finish_request(self, request, client_address): + """Finish one request by instantiating RequestHandlerClass.""" + self.RequestHandlerClass(request, client_address, self) + + def shutdown_request(self, request): + """Called to shutdown and close an individual request.""" + self.close_request(request) + + def close_request(self, request): + """Called to clean up an individual request.""" + pass + + def handle_error(self, request, client_address): + """Handle an error gracefully. May be overridden. + + The default is to print a traceback and continue. + + """ + print('-'*40) + print('Exception happened during processing of request from', end=' ') + print(client_address) + import traceback + traceback.print_exc() # XXX But this goes to stderr! + print('-'*40) + + +class TCPServer(BaseServer): + + """Base class for various socket-based server classes. + + Defaults to synchronous IP stream (i.e., TCP). + + Methods for the caller: + + - __init__(server_address, RequestHandlerClass, bind_and_activate=True) + - serve_forever(poll_interval=0.5) + - shutdown() + - handle_request() # if you don't use serve_forever() + - fileno() -> int # for selector + + Methods that may be overridden: + + - server_bind() + - server_activate() + - get_request() -> request, client_address + - handle_timeout() + - verify_request(request, client_address) + - process_request(request, client_address) + - shutdown_request(request) + - close_request(request) + - handle_error() + + Methods for derived classes: + + - finish_request(request, client_address) + + Class variables that may be overridden by derived classes or + instances: + + - timeout + - address_family + - socket_type + - request_queue_size (only for stream sockets) + - allow_reuse_address + + Instance variables: + + - server_address + - RequestHandlerClass + - socket + + """ + + address_family = socket.AF_INET + + socket_type = socket.SOCK_STREAM + + request_queue_size = 5 + + allow_reuse_address = False + + def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True): + """Constructor. May be extended, do not override.""" + BaseServer.__init__(self, server_address, RequestHandlerClass) + self.socket = socket.socket(self.address_family, + self.socket_type) + if bind_and_activate: + try: + self.server_bind() + self.server_activate() + except: + self.server_close() + raise + + def server_bind(self): + """Called by constructor to bind the socket. + + May be overridden. + + """ + if self.allow_reuse_address: + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.socket.bind(self.server_address) + self.server_address = self.socket.getsockname() + + def server_activate(self): + """Called by constructor to activate the server. + + May be overridden. + + """ + self.socket.listen(self.request_queue_size) + + def server_close(self): + """Called to clean-up the server. + + May be overridden. + + """ + self.socket.close() + + def fileno(self): + """Return socket file number. + + Interface required by selector. + + """ + return self.socket.fileno() + + def get_request(self): + """Get the request and client address from the socket. + + May be overridden. + + """ + return self.socket.accept() + + def shutdown_request(self, request): + """Called to shutdown and close an individual request.""" + try: + #explicitly shutdown. socket.close() merely releases + #the socket and waits for GC to perform the actual close. + request.shutdown(socket.SHUT_WR) + except OSError: + pass #some platforms may raise ENOTCONN here + self.close_request(request) + + def close_request(self, request): + """Called to clean up an individual request.""" + request.close() + + +class UDPServer(TCPServer): + + """UDP server class.""" + + allow_reuse_address = False + + socket_type = socket.SOCK_DGRAM + + max_packet_size = 8192 + + def get_request(self): + data, client_addr = self.socket.recvfrom(self.max_packet_size) + return (data, self.socket), client_addr + + def server_activate(self): + # No need to call listen() for UDP. + pass + + def shutdown_request(self, request): + # No need to shutdown anything. + self.close_request(request) + + def close_request(self, request): + # No need to close anything. + pass + +class ForkingMixIn: + + """Mix-in class to handle each request in a new process.""" + + timeout = 300 + active_children = None + max_children = 40 + + def collect_children(self): + """Internal routine to wait for children that have exited.""" + if self.active_children is None: + return + + # If we're above the max number of children, wait and reap them until + # we go back below threshold. Note that we use waitpid(-1) below to be + # able to collect children in size() syscalls instead + # of size(): the downside is that this might reap children + # which we didn't spawn, which is why we only resort to this when we're + # above max_children. + while len(self.active_children) >= self.max_children: + try: + pid, _ = os.waitpid(-1, 0) + self.active_children.discard(pid) + except ChildProcessError: + # we don't have any children, we're done + self.active_children.clear() + except OSError: + break + + # Now reap all defunct children. + for pid in self.active_children.copy(): + try: + pid, _ = os.waitpid(pid, os.WNOHANG) + # if the child hasn't exited yet, pid will be 0 and ignored by + # discard() below + self.active_children.discard(pid) + except ChildProcessError: + # someone else reaped it + self.active_children.discard(pid) + except OSError: + pass + + def handle_timeout(self): + """Wait for zombies after self.timeout seconds of inactivity. + + May be extended, do not override. + """ + self.collect_children() + + def service_actions(self): + """Collect the zombie child processes regularly in the ForkingMixIn. + + service_actions is called in the BaseServer's serve_forver loop. + """ + self.collect_children() + + def process_request(self, request, client_address): + """Fork a new subprocess to process the request.""" + pid = os.fork() + if pid: + # Parent process + if self.active_children is None: + self.active_children = set() + self.active_children.add(pid) + self.close_request(request) + return + else: + # Child process. + # This must never return, hence os._exit()! + try: + self.finish_request(request, client_address) + self.shutdown_request(request) + os._exit(0) + except: + try: + self.handle_error(request, client_address) + self.shutdown_request(request) + finally: + os._exit(1) + + +class ThreadingMixIn: + """Mix-in class to handle each request in a new thread.""" + + # Decides how threads will act upon termination of the + # main process + daemon_threads = False + + def process_request_thread(self, request, client_address): + """Same as in BaseServer but as a thread. + + In addition, exception handling is done here. + + """ + try: + self.finish_request(request, client_address) + self.shutdown_request(request) + except: + self.handle_error(request, client_address) + self.shutdown_request(request) + + def process_request(self, request, client_address): + """Start a new thread to process the request.""" + t = threading.Thread(target = self.process_request_thread, + args = (request, client_address)) + t.daemon = self.daemon_threads + t.start() + + +class ForkingUDPServer(ForkingMixIn, UDPServer): pass +class ForkingTCPServer(ForkingMixIn, TCPServer): pass + +class ThreadingUDPServer(ThreadingMixIn, UDPServer): pass +class ThreadingTCPServer(ThreadingMixIn, TCPServer): pass + +if hasattr(socket, 'AF_UNIX'): + + class UnixStreamServer(TCPServer): + address_family = socket.AF_UNIX + + class UnixDatagramServer(UDPServer): + address_family = socket.AF_UNIX + + class ThreadingUnixStreamServer(ThreadingMixIn, UnixStreamServer): pass + + class ThreadingUnixDatagramServer(ThreadingMixIn, UnixDatagramServer): pass + +class BaseRequestHandler: + + """Base class for request handler classes. + + This class is instantiated for each request to be handled. The + constructor sets the instance variables request, client_address + and server, and then calls the handle() method. To implement a + specific service, all you need to do is to derive a class which + defines a handle() method. + + The handle() method can find the request as self.request, the + client address as self.client_address, and the server (in case it + needs access to per-server information) as self.server. Since a + separate instance is created for each request, the handle() method + can define other arbitrary instance variables. + + """ + + def __init__(self, request, client_address, server): + self.request = request + self.client_address = client_address + self.server = server + self.setup() + try: + self.handle() + finally: + self.finish() + + def setup(self): + pass + + def handle(self): + pass + + def finish(self): + pass + + +# The following two classes make it possible to use the same service +# class for stream or datagram servers. +# Each class sets up these instance variables: +# - rfile: a file object from which receives the request is read +# - wfile: a file object to which the reply is written +# When the handle() method returns, wfile is flushed properly + + +class StreamRequestHandler(BaseRequestHandler): + + """Define self.rfile and self.wfile for stream sockets.""" + + # Default buffer sizes for rfile, wfile. + # We default rfile to buffered because otherwise it could be + # really slow for large data (a getc() call per byte); we make + # wfile unbuffered because (a) often after a write() we want to + # read and we need to flush the line; (b) big writes to unbuffered + # files are typically optimized by stdio even when big reads + # aren't. + rbufsize = -1 + wbufsize = 0 + + # A timeout to apply to the request socket, if not None. + timeout = None + + # Disable nagle algorithm for this socket, if True. + # Use only when wbufsize != 0, to avoid small packets. + disable_nagle_algorithm = False + + def setup(self): + self.connection = self.request + if self.timeout is not None: + self.connection.settimeout(self.timeout) + if self.disable_nagle_algorithm: + self.connection.setsockopt(socket.IPPROTO_TCP, + socket.TCP_NODELAY, True) + self.rfile = self.connection.makefile('rb', self.rbufsize) + self.wfile = self.connection.makefile('wb', self.wbufsize) + + def finish(self): + if not self.wfile.closed: + try: + self.wfile.flush() + except socket.error: + # A final socket error may have occurred here, such as + # the local error ECONNABORTED. + pass + self.wfile.close() + self.rfile.close() + + +class DatagramRequestHandler(BaseRequestHandler): + + """Define self.rfile and self.wfile for datagram sockets.""" + + def setup(self): + from io import BytesIO + self.packet, self.socket = self.request + self.rfile = BytesIO(self.packet) + self.wfile = BytesIO() + + def finish(self): + self.socket.sendto(self.wfile.getvalue(), self.client_address) +"""Thread-local objects. + +(Note that this module provides a Python version of the threading.local + class. Depending on the version of Python you're using, there may be a + faster one available. You should always import the `local` class from + `threading`.) + +Thread-local objects support the management of thread-local data. +If you have data that you want to be local to a thread, simply create +a thread-local object and use its attributes: + + >>> mydata = local() + >>> mydata.number = 42 + >>> mydata.number + 42 + +You can also access the local-object's dictionary: + + >>> mydata.__dict__ + {'number': 42} + >>> mydata.__dict__.setdefault('widgets', []) + [] + >>> mydata.widgets + [] + +What's important about thread-local objects is that their data are +local to a thread. If we access the data in a different thread: + + >>> log = [] + >>> def f(): + ... items = sorted(mydata.__dict__.items()) + ... log.append(items) + ... mydata.number = 11 + ... log.append(mydata.number) + + >>> import threading + >>> thread = threading.Thread(target=f) + >>> thread.start() + >>> thread.join() + >>> log + [[], 11] + +we get different data. Furthermore, changes made in the other thread +don't affect data seen in this thread: + + >>> mydata.number + 42 + +Of course, values you get from a local object, including a __dict__ +attribute, are for whatever thread was current at the time the +attribute was read. For that reason, you generally don't want to save +these values across threads, as they apply only to the thread they +came from. + +You can create custom local objects by subclassing the local class: + + >>> class MyLocal(local): + ... number = 2 + ... initialized = False + ... def __init__(self, **kw): + ... if self.initialized: + ... raise SystemError('__init__ called too many times') + ... self.initialized = True + ... self.__dict__.update(kw) + ... def squared(self): + ... return self.number ** 2 + +This can be useful to support default values, methods and +initialization. Note that if you define an __init__ method, it will be +called each time the local object is used in a separate thread. This +is necessary to initialize each thread's dictionary. + +Now if we create a local object: + + >>> mydata = MyLocal(color='red') + +Now we have a default number: + + >>> mydata.number + 2 + +an initial color: + + >>> mydata.color + 'red' + >>> del mydata.color + +And a method that operates on the data: + + >>> mydata.squared() + 4 + +As before, we can access the data in a separate thread: + + >>> log = [] + >>> thread = threading.Thread(target=f) + >>> thread.start() + >>> thread.join() + >>> log + [[('color', 'red'), ('initialized', True)], 11] + +without affecting this thread's data: + + >>> mydata.number + 2 + >>> mydata.color + Traceback (most recent call last): + ... + AttributeError: 'MyLocal' object has no attribute 'color' + +Note that subclasses can define slots, but they are not thread +local. They are shared across threads: + + >>> class MyLocal(local): + ... __slots__ = 'number' + + >>> mydata = MyLocal() + >>> mydata.number = 42 + >>> mydata.color = 'red' + +So, the separate thread: + + >>> thread = threading.Thread(target=f) + >>> thread.start() + >>> thread.join() + +affects what we see: + + >>> mydata.number + 11 + +>>> del mydata +""" + +from weakref import ref +from contextlib import contextmanager + +__all__ = ["local"] + +# We need to use objects from the threading module, but the threading +# module may also want to use our `local` class, if support for locals +# isn't compiled in to the `thread` module. This creates potential problems +# with circular imports. For that reason, we don't import `threading` +# until the bottom of this file (a hack sufficient to worm around the +# potential problems). Note that all platforms on CPython do have support +# for locals in the `thread` module, and there is no circular import problem +# then, so problems introduced by fiddling the order of imports here won't +# manifest. + +class _localimpl: + """A class managing thread-local dicts""" + __slots__ = 'key', 'dicts', 'localargs', 'locallock', '__weakref__' + + def __init__(self): + # The key used in the Thread objects' attribute dicts. + # We keep it a string for speed but make it unlikely to clash with + # a "real" attribute. + self.key = '_threading_local._localimpl.' + str(id(self)) + # { id(Thread) -> (ref(Thread), thread-local dict) } + self.dicts = {} + + def get_dict(self): + """Return the dict for the current thread. Raises KeyError if none + defined.""" + thread = current_thread() + return self.dicts[id(thread)][1] + + def create_dict(self): + """Create a new dict for the current thread, and return it.""" + localdict = {} + key = self.key + thread = current_thread() + idt = id(thread) + def local_deleted(_, key=key): + # When the localimpl is deleted, remove the thread attribute. + thread = wrthread() + if thread is not None: + del thread.__dict__[key] + def thread_deleted(_, idt=idt): + # When the thread is deleted, remove the local dict. + # Note that this is suboptimal if the thread object gets + # caught in a reference loop. We would like to be called + # as soon as the OS-level thread ends instead. + local = wrlocal() + if local is not None: + dct = local.dicts.pop(idt) + wrlocal = ref(self, local_deleted) + wrthread = ref(thread, thread_deleted) + thread.__dict__[key] = wrlocal + self.dicts[idt] = wrthread, localdict + return localdict + + +@contextmanager +def _patch(self): + impl = object.__getattribute__(self, '_local__impl') + try: + dct = impl.get_dict() + except KeyError: + dct = impl.create_dict() + args, kw = impl.localargs + self.__init__(*args, **kw) + with impl.locallock: + object.__setattr__(self, '__dict__', dct) + yield + + +class local: + __slots__ = '_local__impl', '__dict__' + + def __new__(cls, *args, **kw): + if (args or kw) and (cls.__init__ is object.__init__): + raise TypeError("Initialization arguments are not supported") + self = object.__new__(cls) + impl = _localimpl() + impl.localargs = (args, kw) + impl.locallock = RLock() + object.__setattr__(self, '_local__impl', impl) + # We need to create the thread dict in anticipation of + # __init__ being called, to make sure we don't call it + # again ourselves. + impl.create_dict() + return self + + def __getattribute__(self, name): + with _patch(self): + return object.__getattribute__(self, name) + + def __setattr__(self, name, value): + if name == '__dict__': + raise AttributeError( + "%r object attribute '__dict__' is read-only" + % self.__class__.__name__) + with _patch(self): + return object.__setattr__(self, name, value) + + def __delattr__(self, name): + if name == '__dict__': + raise AttributeError( + "%r object attribute '__dict__' is read-only" + % self.__class__.__name__) + with _patch(self): + return object.__delattr__(self, name) + + +from threading import current_thread, RLock +"""Cache lines from Python source files. + +This is intended to read lines from modules imported -- hence if a filename +is not found, it will look down the module search path for a file by +that name. +""" + +import functools +import sys +import os +import tokenize + +__all__ = ["getline", "clearcache", "checkcache"] + +def getline(filename, lineno, module_globals=None): + lines = getlines(filename, module_globals) + if 1 <= lineno <= len(lines): + return lines[lineno-1] + else: + return '' + + +# The cache + +# The cache. Maps filenames to either a thunk which will provide source code, +# or a tuple (size, mtime, lines, fullname) once loaded. +cache = {} + + +def clearcache(): + """Clear the cache entirely.""" + + global cache + cache = {} + + +def getlines(filename, module_globals=None): + """Get the lines for a Python source file from the cache. + Update the cache if it doesn't contain an entry for this file already.""" + + if filename in cache: + entry = cache[filename] + if len(entry) != 1: + return cache[filename][2] + + try: + return updatecache(filename, module_globals) + except MemoryError: + clearcache() + return [] + + +def checkcache(filename=None): + """Discard cache entries that are out of date. + (This is not checked upon each call!)""" + + if filename is None: + filenames = list(cache.keys()) + else: + if filename in cache: + filenames = [filename] + else: + return + + for filename in filenames: + entry = cache[filename] + if len(entry) == 1: + # lazy cache entry, leave it lazy. + continue + size, mtime, lines, fullname = entry + if mtime is None: + continue # no-op for files loaded via a __loader__ + try: + stat = os.stat(fullname) + except OSError: + del cache[filename] + continue + if size != stat.st_size or mtime != stat.st_mtime: + del cache[filename] + + +def updatecache(filename, module_globals=None): + """Update a cache entry and return its list of lines. + If something's wrong, print a message, discard the cache entry, + and return an empty list.""" + + if filename in cache: + if len(cache[filename]) != 1: + del cache[filename] + if not filename or (filename.startswith('<') and filename.endswith('>')): + return [] + + fullname = filename + try: + stat = os.stat(fullname) + except OSError: + basename = filename + + # Realise a lazy loader based lookup if there is one + # otherwise try to lookup right now. + if lazycache(filename, module_globals): + try: + data = cache[filename][0]() + except (ImportError, OSError): + pass + else: + if data is None: + # No luck, the PEP302 loader cannot find the source + # for this module. + return [] + cache[filename] = ( + len(data), None, + [line+'\n' for line in data.splitlines()], fullname + ) + return cache[filename][2] + + # Try looking through the module search path, which is only useful + # when handling a relative filename. + if os.path.isabs(filename): + return [] + + for dirname in sys.path: + try: + fullname = os.path.join(dirname, basename) + except (TypeError, AttributeError): + # Not sufficiently string-like to do anything useful with. + continue + try: + stat = os.stat(fullname) + break + except OSError: + pass + else: + return [] + try: + with tokenize.open(fullname) as fp: + lines = fp.readlines() + except OSError: + return [] + if lines and not lines[-1].endswith('\n'): + lines[-1] += '\n' + size, mtime = stat.st_size, stat.st_mtime + cache[filename] = size, mtime, lines, fullname + return lines + + +def lazycache(filename, module_globals): + """Seed the cache for filename with module_globals. + + The module loader will be asked for the source only when getlines is + called, not immediately. + + If there is an entry in the cache already, it is not altered. + + :return: True if a lazy load is registered in the cache, + otherwise False. To register such a load a module loader with a + get_source method must be found, the filename must be a cachable + filename, and the filename must not be already cached. + """ + if filename in cache: + if len(cache[filename]) == 1: + return True + else: + return False + if not filename or (filename.startswith('<') and filename.endswith('>')): + return False + # Try for a __loader__, if available + if module_globals and '__loader__' in module_globals: + name = module_globals.get('__name__') + loader = module_globals['__loader__'] + get_source = getattr(loader, 'get_source', None) + + if name and get_source: + get_lines = functools.partial(get_source, name) + cache[filename] = (get_lines,) + return True + return False +r"""Utilities to compile possibly incomplete Python source code. + +This module provides two interfaces, broadly similar to the builtin +function compile(), which take program text, a filename and a 'mode' +and: + +- Return code object if the command is complete and valid +- Return None if the command is incomplete +- Raise SyntaxError, ValueError or OverflowError if the command is a + syntax error (OverflowError and ValueError can be produced by + malformed literals). + +Approach: + +First, check if the source consists entirely of blank lines and +comments; if so, replace it with 'pass', because the built-in +parser doesn't always do the right thing for these. + +Compile three times: as is, with \n, and with \n\n appended. If it +compiles as is, it's complete. If it compiles with one \n appended, +we expect more. If it doesn't compile either way, we compare the +error we get when compiling with \n or \n\n appended. If the errors +are the same, the code is broken. But if the errors are different, we +expect more. Not intuitive; not even guaranteed to hold in future +releases; but this matches the compiler's behavior from Python 1.4 +through 2.2, at least. + +Caveat: + +It is possible (but not likely) that the parser stops parsing with a +successful outcome before reaching the end of the source; in this +case, trailing symbols may be ignored instead of causing an error. +For example, a backslash followed by two newlines may be followed by +arbitrary garbage. This will be fixed once the API for the parser is +better. + +The two interfaces are: + +compile_command(source, filename, symbol): + + Compiles a single command in the manner described above. + +CommandCompiler(): + + Instances of this class have __call__ methods identical in + signature to compile_command; the difference is that if the + instance compiles program text containing a __future__ statement, + the instance 'remembers' and compiles all subsequent program texts + with the statement in force. + +The module also provides another class: + +Compile(): + + Instances of this class act like the built-in function compile, + but with 'memory' in the sense described above. +""" + +import __future__ + +_features = [getattr(__future__, fname) + for fname in __future__.all_feature_names] + +__all__ = ["compile_command", "Compile", "CommandCompiler"] + +PyCF_DONT_IMPLY_DEDENT = 0x200 # Matches pythonrun.h + +def _maybe_compile(compiler, source, filename, symbol): + # Check for source consisting of only blank lines and comments + for line in source.split("\n"): + line = line.strip() + if line and line[0] != '#': + break # Leave it alone + else: + if symbol != "eval": + source = "pass" # Replace it with a 'pass' statement + + err = err1 = err2 = None + code = code1 = code2 = None + + try: + code = compiler(source, filename, symbol) + except SyntaxError as err: + pass + + try: + code1 = compiler(source + "\n", filename, symbol) + except SyntaxError as e: + err1 = e + + try: + code2 = compiler(source + "\n\n", filename, symbol) + except SyntaxError as e: + err2 = e + + if code: + return code + if not code1 and repr(err1) == repr(err2): + raise err1 + +def _compile(source, filename, symbol): + return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT) + +def compile_command(source, filename="", symbol="single"): + r"""Compile a command and determine whether it is incomplete. + + Arguments: + + source -- the source string; may contain \n characters + filename -- optional filename from which source was read; default + "" + symbol -- optional grammar start symbol; "single" (default) or "eval" + + Return value / exceptions raised: + + - Return a code object if the command is complete and valid + - Return None if the command is incomplete + - Raise SyntaxError, ValueError or OverflowError if the command is a + syntax error (OverflowError and ValueError can be produced by + malformed literals). + """ + return _maybe_compile(_compile, source, filename, symbol) + +class Compile: + """Instances of this class behave much like the built-in compile + function, but if one is used to compile text containing a future + statement, it "remembers" and compiles all subsequent program texts + with the statement in force.""" + def __init__(self): + self.flags = PyCF_DONT_IMPLY_DEDENT + + def __call__(self, source, filename, symbol): + codeob = compile(source, filename, symbol, self.flags, 1) + for feature in _features: + if codeob.co_flags & feature.compiler_flag: + self.flags |= feature.compiler_flag + return codeob + +class CommandCompiler: + """Instances of this class have __call__ methods identical in + signature to compile_command; the difference is that if the + instance compiles program text containing a __future__ statement, + the instance 'remembers' and compiles all subsequent program texts + with the statement in force.""" + + def __init__(self,): + self.compiler = Compile() + + def __call__(self, source, filename="", symbol="single"): + r"""Compile a command and determine whether it is incomplete. + + Arguments: + + source -- the source string; may contain \n characters + filename -- optional filename from which source was read; + default "" + symbol -- optional grammar start symbol; "single" (default) or + "eval" + + Return value / exceptions raised: + + - Return a code object if the command is complete and valid + - Return None if the command is incomplete + - Raise SyntaxError, ValueError or OverflowError if the command is a + syntax error (OverflowError and ValueError can be produced by + malformed literals). + """ + return _maybe_compile(self.compiler, source, filename, symbol) +"""Macintosh-specific module for conversion between pathnames and URLs. + +Do not import directly; use urllib instead.""" + +import urllib.parse +import os + +__all__ = ["url2pathname","pathname2url"] + +def url2pathname(pathname): + """OS-specific conversion from a relative URL of the 'file' scheme + to a file system path; not recommended for general use.""" + # + # XXXX The .. handling should be fixed... + # + tp = urllib.parse.splittype(pathname)[0] + if tp and tp != 'file': + raise RuntimeError('Cannot convert non-local URL to pathname') + # Turn starting /// into /, an empty hostname means current host + if pathname[:3] == '///': + pathname = pathname[2:] + elif pathname[:2] == '//': + raise RuntimeError('Cannot convert non-local URL to pathname') + components = pathname.split('/') + # Remove . and embedded .. + i = 0 + while i < len(components): + if components[i] == '.': + del components[i] + elif components[i] == '..' and i > 0 and \ + components[i-1] not in ('', '..'): + del components[i-1:i+1] + i = i-1 + elif components[i] == '' and i > 0 and components[i-1] != '': + del components[i] + else: + i = i+1 + if not components[0]: + # Absolute unix path, don't start with colon + rv = ':'.join(components[1:]) + else: + # relative unix path, start with colon. First replace + # leading .. by empty strings (giving ::file) + i = 0 + while i < len(components) and components[i] == '..': + components[i] = '' + i = i + 1 + rv = ':' + ':'.join(components) + # and finally unquote slashes and other funny characters + return urllib.parse.unquote(rv) + +def pathname2url(pathname): + """OS-specific conversion from a file system path to a relative URL + of the 'file' scheme; not recommended for general use.""" + if '/' in pathname: + raise RuntimeError("Cannot convert pathname containing slashes") + components = pathname.split(':') + # Remove empty first and/or last component + if components[0] == '': + del components[0] + if components[-1] == '': + del components[-1] + # Replace empty string ('::') by .. (will result in '/../' later) + for i in range(len(components)): + if components[i] == '': + components[i] = '..' + # Truncate names longer than 31 bytes + components = map(_pncomp2url, components) + + if os.path.isabs(pathname): + return '/' + '/'.join(components) + else: + return '/'.join(components) + +def _pncomp2url(component): + # We want to quote slashes + return urllib.parse.quote(component[:31], safe='') +#! /usr/bin/env python3 +"""Interfaces for launching and remotely controlling Web browsers.""" +# Maintained by Georg Brandl. + +import os +import shlex +import shutil +import sys +import subprocess + +__all__ = ["Error", "open", "open_new", "open_new_tab", "get", "register"] + +class Error(Exception): + pass + +_browsers = {} # Dictionary of available browser controllers +_tryorder = [] # Preference order of available browsers + +def register(name, klass, instance=None, update_tryorder=1): + """Register a browser connector and, optionally, connection.""" + _browsers[name.lower()] = [klass, instance] + if update_tryorder > 0: + _tryorder.append(name) + elif update_tryorder < 0: + _tryorder.insert(0, name) + +def get(using=None): + """Return a browser launcher instance appropriate for the environment.""" + if using is not None: + alternatives = [using] + else: + alternatives = _tryorder + for browser in alternatives: + if '%s' in browser: + # User gave us a command line, split it into name and args + browser = shlex.split(browser) + if browser[-1] == '&': + return BackgroundBrowser(browser[:-1]) + else: + return GenericBrowser(browser) + else: + # User gave us a browser name or path. + try: + command = _browsers[browser.lower()] + except KeyError: + command = _synthesize(browser) + if command[1] is not None: + return command[1] + elif command[0] is not None: + return command[0]() + raise Error("could not locate runnable browser") + +# Please note: the following definition hides a builtin function. +# It is recommended one does "import webbrowser" and uses webbrowser.open(url) +# instead of "from webbrowser import *". + +def open(url, new=0, autoraise=True): + for name in _tryorder: + browser = get(name) + if browser.open(url, new, autoraise): + return True + return False + +def open_new(url): + return open(url, 1) + +def open_new_tab(url): + return open(url, 2) + + +def _synthesize(browser, update_tryorder=1): + """Attempt to synthesize a controller base on existing controllers. + + This is useful to create a controller when a user specifies a path to + an entry in the BROWSER environment variable -- we can copy a general + controller to operate using a specific installation of the desired + browser in this way. + + If we can't create a controller in this way, or if there is no + executable for the requested browser, return [None, None]. + + """ + cmd = browser.split()[0] + if not shutil.which(cmd): + return [None, None] + name = os.path.basename(cmd) + try: + command = _browsers[name.lower()] + except KeyError: + return [None, None] + # now attempt to clone to fit the new name: + controller = command[1] + if controller and name.lower() == controller.basename: + import copy + controller = copy.copy(controller) + controller.name = browser + controller.basename = os.path.basename(browser) + register(browser, None, controller, update_tryorder) + return [None, controller] + return [None, None] + + +# General parent classes + +class BaseBrowser(object): + """Parent class for all browsers. Do not use directly.""" + + args = ['%s'] + + def __init__(self, name=""): + self.name = name + self.basename = name + + def open(self, url, new=0, autoraise=True): + raise NotImplementedError + + def open_new(self, url): + return self.open(url, 1) + + def open_new_tab(self, url): + return self.open(url, 2) + + +class GenericBrowser(BaseBrowser): + """Class for all browsers started with a command + and without remote functionality.""" + + def __init__(self, name): + if isinstance(name, str): + self.name = name + self.args = ["%s"] + else: + # name should be a list with arguments + self.name = name[0] + self.args = name[1:] + self.basename = os.path.basename(self.name) + + def open(self, url, new=0, autoraise=True): + cmdline = [self.name] + [arg.replace("%s", url) + for arg in self.args] + try: + if sys.platform[:3] == 'win': + p = subprocess.Popen(cmdline) + else: + p = subprocess.Popen(cmdline, close_fds=True) + return not p.wait() + except OSError: + return False + + +class BackgroundBrowser(GenericBrowser): + """Class for all browsers which are to be started in the + background.""" + + def open(self, url, new=0, autoraise=True): + cmdline = [self.name] + [arg.replace("%s", url) + for arg in self.args] + try: + if sys.platform[:3] == 'win': + p = subprocess.Popen(cmdline) + else: + p = subprocess.Popen(cmdline, close_fds=True, + start_new_session=True) + return (p.poll() is None) + except OSError: + return False + + +class UnixBrowser(BaseBrowser): + """Parent class for all Unix browsers with remote functionality.""" + + raise_opts = None + background = False + redirect_stdout = True + # In remote_args, %s will be replaced with the requested URL. %action will + # be replaced depending on the value of 'new' passed to open. + # remote_action is used for new=0 (open). If newwin is not None, it is + # used for new=1 (open_new). If newtab is not None, it is used for + # new=3 (open_new_tab). After both substitutions are made, any empty + # strings in the transformed remote_args list will be removed. + remote_args = ['%action', '%s'] + remote_action = None + remote_action_newwin = None + remote_action_newtab = None + + def _invoke(self, args, remote, autoraise): + raise_opt = [] + if remote and self.raise_opts: + # use autoraise argument only for remote invocation + autoraise = int(autoraise) + opt = self.raise_opts[autoraise] + if opt: raise_opt = [opt] + + cmdline = [self.name] + raise_opt + args + + if remote or self.background: + inout = subprocess.DEVNULL + else: + # for TTY browsers, we need stdin/out + inout = None + p = subprocess.Popen(cmdline, close_fds=True, stdin=inout, + stdout=(self.redirect_stdout and inout or None), + stderr=inout, start_new_session=True) + if remote: + # wait at most five seconds. If the subprocess is not finished, the + # remote invocation has (hopefully) started a new instance. + try: + rc = p.wait(5) + # if remote call failed, open() will try direct invocation + return not rc + except subprocess.TimeoutExpired: + return True + elif self.background: + if p.poll() is None: + return True + else: + return False + else: + return not p.wait() + + def open(self, url, new=0, autoraise=True): + if new == 0: + action = self.remote_action + elif new == 1: + action = self.remote_action_newwin + elif new == 2: + if self.remote_action_newtab is None: + action = self.remote_action_newwin + else: + action = self.remote_action_newtab + else: + raise Error("Bad 'new' parameter to open(); " + + "expected 0, 1, or 2, got %s" % new) + + args = [arg.replace("%s", url).replace("%action", action) + for arg in self.remote_args] + args = [arg for arg in args if arg] + success = self._invoke(args, True, autoraise) + if not success: + # remote invocation failed, try straight way + args = [arg.replace("%s", url) for arg in self.args] + return self._invoke(args, False, False) + else: + return True + + +class Mozilla(UnixBrowser): + """Launcher class for Mozilla/Netscape browsers.""" + + raise_opts = ["-noraise", "-raise"] + remote_args = ['-remote', 'openURL(%s%action)'] + remote_action = "" + remote_action_newwin = ",new-window" + remote_action_newtab = ",new-tab" + background = True + +Netscape = Mozilla + + +class Galeon(UnixBrowser): + """Launcher class for Galeon/Epiphany browsers.""" + + raise_opts = ["-noraise", ""] + remote_args = ['%action', '%s'] + remote_action = "-n" + remote_action_newwin = "-w" + background = True + + +class Chrome(UnixBrowser): + "Launcher class for Google Chrome browser." + + remote_args = ['%action', '%s'] + remote_action = "" + remote_action_newwin = "--new-window" + remote_action_newtab = "" + background = True + +Chromium = Chrome + + +class Opera(UnixBrowser): + "Launcher class for Opera browser." + + raise_opts = ["-noraise", ""] + remote_args = ['-remote', 'openURL(%s%action)'] + remote_action = "" + remote_action_newwin = ",new-window" + remote_action_newtab = ",new-page" + background = True + + +class Elinks(UnixBrowser): + "Launcher class for Elinks browsers." + + remote_args = ['-remote', 'openURL(%s%action)'] + remote_action = "" + remote_action_newwin = ",new-window" + remote_action_newtab = ",new-tab" + background = False + + # elinks doesn't like its stdout to be redirected - + # it uses redirected stdout as a signal to do -dump + redirect_stdout = False + + +class Konqueror(BaseBrowser): + """Controller for the KDE File Manager (kfm, or Konqueror). + + See the output of ``kfmclient --commands`` + for more information on the Konqueror remote-control interface. + """ + + def open(self, url, new=0, autoraise=True): + # XXX Currently I know no way to prevent KFM from opening a new win. + if new == 2: + action = "newTab" + else: + action = "openURL" + + devnull = subprocess.DEVNULL + + try: + p = subprocess.Popen(["kfmclient", action, url], + close_fds=True, stdin=devnull, + stdout=devnull, stderr=devnull) + except OSError: + # fall through to next variant + pass + else: + p.wait() + # kfmclient's return code unfortunately has no meaning as it seems + return True + + try: + p = subprocess.Popen(["konqueror", "--silent", url], + close_fds=True, stdin=devnull, + stdout=devnull, stderr=devnull, + start_new_session=True) + except OSError: + # fall through to next variant + pass + else: + if p.poll() is None: + # Should be running now. + return True + + try: + p = subprocess.Popen(["kfm", "-d", url], + close_fds=True, stdin=devnull, + stdout=devnull, stderr=devnull, + start_new_session=True) + except OSError: + return False + else: + return (p.poll() is None) + + +class Grail(BaseBrowser): + # There should be a way to maintain a connection to Grail, but the + # Grail remote control protocol doesn't really allow that at this + # point. It probably never will! + def _find_grail_rc(self): + import glob + import pwd + import socket + import tempfile + tempdir = os.path.join(tempfile.gettempdir(), + ".grail-unix") + user = pwd.getpwuid(os.getuid())[0] + filename = os.path.join(tempdir, user + "-*") + maybes = glob.glob(filename) + if not maybes: + return None + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + for fn in maybes: + # need to PING each one until we find one that's live + try: + s.connect(fn) + except OSError: + # no good; attempt to clean it out, but don't fail: + try: + os.unlink(fn) + except OSError: + pass + else: + return s + + def _remote(self, action): + s = self._find_grail_rc() + if not s: + return 0 + s.send(action) + s.close() + return 1 + + def open(self, url, new=0, autoraise=True): + if new: + ok = self._remote("LOADNEW " + url) + else: + ok = self._remote("LOAD " + url) + return ok + + +# +# Platform support for Unix +# + +# These are the right tests because all these Unix browsers require either +# a console terminal or an X display to run. + +def register_X_browsers(): + + # use xdg-open if around + if shutil.which("xdg-open"): + register("xdg-open", None, BackgroundBrowser("xdg-open")) + + # The default GNOME3 browser + if "GNOME_DESKTOP_SESSION_ID" in os.environ and shutil.which("gvfs-open"): + register("gvfs-open", None, BackgroundBrowser("gvfs-open")) + + # The default GNOME browser + if "GNOME_DESKTOP_SESSION_ID" in os.environ and shutil.which("gnome-open"): + register("gnome-open", None, BackgroundBrowser("gnome-open")) + + # The default KDE browser + if "KDE_FULL_SESSION" in os.environ and shutil.which("kfmclient"): + register("kfmclient", Konqueror, Konqueror("kfmclient")) + + if shutil.which("x-www-browser"): + register("x-www-browser", None, BackgroundBrowser("x-www-browser")) + + # The Mozilla/Netscape browsers + for browser in ("mozilla-firefox", "firefox", + "mozilla-firebird", "firebird", + "iceweasel", "iceape", + "seamonkey", "mozilla", "netscape"): + if shutil.which(browser): + register(browser, None, Mozilla(browser)) + + # Konqueror/kfm, the KDE browser. + if shutil.which("kfm"): + register("kfm", Konqueror, Konqueror("kfm")) + elif shutil.which("konqueror"): + register("konqueror", Konqueror, Konqueror("konqueror")) + + # Gnome's Galeon and Epiphany + for browser in ("galeon", "epiphany"): + if shutil.which(browser): + register(browser, None, Galeon(browser)) + + # Skipstone, another Gtk/Mozilla based browser + if shutil.which("skipstone"): + register("skipstone", None, BackgroundBrowser("skipstone")) + + # Google Chrome/Chromium browsers + for browser in ("google-chrome", "chrome", "chromium", "chromium-browser"): + if shutil.which(browser): + register(browser, None, Chrome(browser)) + + # Opera, quite popular + if shutil.which("opera"): + register("opera", None, Opera("opera")) + + # Next, Mosaic -- old but still in use. + if shutil.which("mosaic"): + register("mosaic", None, BackgroundBrowser("mosaic")) + + # Grail, the Python browser. Does anybody still use it? + if shutil.which("grail"): + register("grail", Grail, None) + +# Prefer X browsers if present +if os.environ.get("DISPLAY"): + register_X_browsers() + +# Also try console browsers +if os.environ.get("TERM"): + if shutil.which("www-browser"): + register("www-browser", None, GenericBrowser("www-browser")) + # The Links/elinks browsers + if shutil.which("links"): + register("links", None, GenericBrowser("links")) + if shutil.which("elinks"): + register("elinks", None, Elinks("elinks")) + # The Lynx browser , + if shutil.which("lynx"): + register("lynx", None, GenericBrowser("lynx")) + # The w3m browser + if shutil.which("w3m"): + register("w3m", None, GenericBrowser("w3m")) + +# +# Platform support for Windows +# + +if sys.platform[:3] == "win": + class WindowsDefault(BaseBrowser): + def open(self, url, new=0, autoraise=True): + try: + os.startfile(url) + except OSError: + # [Error 22] No application is associated with the specified + # file for this operation: '' + return False + else: + return True + + _tryorder = [] + _browsers = {} + + # First try to use the default Windows browser + register("windows-default", WindowsDefault) + + # Detect some common Windows browsers, fallback to IE + iexplore = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"), + "Internet Explorer\\IEXPLORE.EXE") + for browser in ("firefox", "firebird", "seamonkey", "mozilla", + "netscape", "opera", iexplore): + if shutil.which(browser): + register(browser, None, BackgroundBrowser(browser)) + +# +# Platform support for MacOS +# + +if sys.platform == 'darwin': + # Adapted from patch submitted to SourceForge by Steven J. Burr + class MacOSX(BaseBrowser): + """Launcher class for Aqua browsers on Mac OS X + + Optionally specify a browser name on instantiation. Note that this + will not work for Aqua browsers if the user has moved the application + package after installation. + + If no browser is specified, the default browser, as specified in the + Internet System Preferences panel, will be used. + """ + def __init__(self, name): + self.name = name + + def open(self, url, new=0, autoraise=True): + assert "'" not in url + # hack for local urls + if not ':' in url: + url = 'file:'+url + + # new must be 0 or 1 + new = int(bool(new)) + if self.name == "default": + # User called open, open_new or get without a browser parameter + script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser + else: + # User called get and chose a browser + if self.name == "OmniWeb": + toWindow = "" + else: + # Include toWindow parameter of OpenURL command for browsers + # that support it. 0 == new window; -1 == existing + toWindow = "toWindow %d" % (new - 1) + cmd = 'OpenURL "%s"' % url.replace('"', '%22') + script = '''tell application "%s" + activate + %s %s + end tell''' % (self.name, cmd, toWindow) + # Open pipe to AppleScript through osascript command + osapipe = os.popen("osascript", "w") + if osapipe is None: + return False + # Write script to osascript's stdin + osapipe.write(script) + rc = osapipe.close() + return not rc + + class MacOSXOSAScript(BaseBrowser): + def __init__(self, name): + self._name = name + + def open(self, url, new=0, autoraise=True): + if self._name == 'default': + script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser + else: + script = ''' + tell application "%s" + activate + open location "%s" + end + '''%(self._name, url.replace('"', '%22')) + + osapipe = os.popen("osascript", "w") + if osapipe is None: + return False + + osapipe.write(script) + rc = osapipe.close() + return not rc + + + # Don't clear _tryorder or _browsers since OS X can use above Unix support + # (but we prefer using the OS X specific stuff) + register("safari", None, MacOSXOSAScript('safari'), -1) + register("firefox", None, MacOSXOSAScript('firefox'), -1) + register("MacOSX", None, MacOSXOSAScript('default'), -1) + + +# OK, now that we know what the default preference orders for each +# platform are, allow user to override them with the BROWSER variable. +if "BROWSER" in os.environ: + _userchoices = os.environ["BROWSER"].split(os.pathsep) + _userchoices.reverse() + + # Treat choices in same way as if passed into get() but do register + # and prepend to _tryorder + for cmdline in _userchoices: + if cmdline != '': + cmd = _synthesize(cmdline, -1) + if cmd[1] is None: + register(cmdline, None, GenericBrowser(cmdline), -1) + cmdline = None # to make del work if _userchoices was empty + del cmdline + del _userchoices + +# what to do if _tryorder is now empty? + + +def main(): + import getopt + usage = """Usage: %s [-n | -t] url + -n: open new window + -t: open new tab""" % sys.argv[0] + try: + opts, args = getopt.getopt(sys.argv[1:], 'ntd') + except getopt.error as msg: + print(msg, file=sys.stderr) + print(usage, file=sys.stderr) + sys.exit(1) + new_win = 0 + for o, a in opts: + if o == '-n': new_win = 1 + elif o == '-t': new_win = 2 + if len(args) != 1: + print(usage, file=sys.stderr) + sys.exit(1) + + url = args[0] + open(url, new_win) + + print("\a") + +if __name__ == "__main__": + main() +# Module doctest. +# Released to the public domain 16-Jan-2001, by Tim Peters (tim@python.org). +# Major enhancements and refactoring by: +# Jim Fulton +# Edward Loper + +# Provided as-is; use at your own risk; no warranty; no promises; enjoy! + +r"""Module doctest -- a framework for running examples in docstrings. + +In simplest use, end each module M to be tested with: + +def _test(): + import doctest + doctest.testmod() + +if __name__ == "__main__": + _test() + +Then running the module as a script will cause the examples in the +docstrings to get executed and verified: + +python M.py + +This won't display anything unless an example fails, in which case the +failing example(s) and the cause(s) of the failure(s) are printed to stdout +(why not stderr? because stderr is a lame hack <0.2 wink>), and the final +line of output is "Test failed.". + +Run it with the -v switch instead: + +python M.py -v + +and a detailed report of all examples tried is printed to stdout, along +with assorted summaries at the end. + +You can force verbose mode by passing "verbose=True" to testmod, or prohibit +it by passing "verbose=False". In either of those cases, sys.argv is not +examined by testmod. + +There are a variety of other ways to run doctests, including integration +with the unittest framework, and support for running non-Python text +files containing doctests. There are also many ways to override parts +of doctest's default behaviors. See the Library Reference Manual for +details. +""" + +__docformat__ = 'reStructuredText en' + +__all__ = [ + # 0, Option Flags + 'register_optionflag', + 'DONT_ACCEPT_TRUE_FOR_1', + 'DONT_ACCEPT_BLANKLINE', + 'NORMALIZE_WHITESPACE', + 'ELLIPSIS', + 'SKIP', + 'IGNORE_EXCEPTION_DETAIL', + 'COMPARISON_FLAGS', + 'REPORT_UDIFF', + 'REPORT_CDIFF', + 'REPORT_NDIFF', + 'REPORT_ONLY_FIRST_FAILURE', + 'REPORTING_FLAGS', + 'FAIL_FAST', + # 1. Utility Functions + # 2. Example & DocTest + 'Example', + 'DocTest', + # 3. Doctest Parser + 'DocTestParser', + # 4. Doctest Finder + 'DocTestFinder', + # 5. Doctest Runner + 'DocTestRunner', + 'OutputChecker', + 'DocTestFailure', + 'UnexpectedException', + 'DebugRunner', + # 6. Test Functions + 'testmod', + 'testfile', + 'run_docstring_examples', + # 7. Unittest Support + 'DocTestSuite', + 'DocFileSuite', + 'set_unittest_reportflags', + # 8. Debugging Support + 'script_from_examples', + 'testsource', + 'debug_src', + 'debug', +] + +import __future__ +import argparse +import difflib +import inspect +import linecache +import os +import pdb +import re +import sys +import traceback +import unittest +from io import StringIO +from collections import namedtuple + +TestResults = namedtuple('TestResults', 'failed attempted') + +# There are 4 basic classes: +# - Example: a pair, plus an intra-docstring line number. +# - DocTest: a collection of examples, parsed from a docstring, plus +# info about where the docstring came from (name, filename, lineno). +# - DocTestFinder: extracts DocTests from a given object's docstring and +# its contained objects' docstrings. +# - DocTestRunner: runs DocTest cases, and accumulates statistics. +# +# So the basic picture is: +# +# list of: +# +------+ +---------+ +-------+ +# |object| --DocTestFinder-> | DocTest | --DocTestRunner-> |results| +# +------+ +---------+ +-------+ +# | Example | +# | ... | +# | Example | +# +---------+ + +# Option constants. + +OPTIONFLAGS_BY_NAME = {} +def register_optionflag(name): + # Create a new flag unless `name` is already known. + return OPTIONFLAGS_BY_NAME.setdefault(name, 1 << len(OPTIONFLAGS_BY_NAME)) + +DONT_ACCEPT_TRUE_FOR_1 = register_optionflag('DONT_ACCEPT_TRUE_FOR_1') +DONT_ACCEPT_BLANKLINE = register_optionflag('DONT_ACCEPT_BLANKLINE') +NORMALIZE_WHITESPACE = register_optionflag('NORMALIZE_WHITESPACE') +ELLIPSIS = register_optionflag('ELLIPSIS') +SKIP = register_optionflag('SKIP') +IGNORE_EXCEPTION_DETAIL = register_optionflag('IGNORE_EXCEPTION_DETAIL') + +COMPARISON_FLAGS = (DONT_ACCEPT_TRUE_FOR_1 | + DONT_ACCEPT_BLANKLINE | + NORMALIZE_WHITESPACE | + ELLIPSIS | + SKIP | + IGNORE_EXCEPTION_DETAIL) + +REPORT_UDIFF = register_optionflag('REPORT_UDIFF') +REPORT_CDIFF = register_optionflag('REPORT_CDIFF') +REPORT_NDIFF = register_optionflag('REPORT_NDIFF') +REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE') +FAIL_FAST = register_optionflag('FAIL_FAST') + +REPORTING_FLAGS = (REPORT_UDIFF | + REPORT_CDIFF | + REPORT_NDIFF | + REPORT_ONLY_FIRST_FAILURE | + FAIL_FAST) + +# Special string markers for use in `want` strings: +BLANKLINE_MARKER = '' +ELLIPSIS_MARKER = '...' + +###################################################################### +## Table of Contents +###################################################################### +# 1. Utility Functions +# 2. Example & DocTest -- store test cases +# 3. DocTest Parser -- extracts examples from strings +# 4. DocTest Finder -- extracts test cases from objects +# 5. DocTest Runner -- runs test cases +# 6. Test Functions -- convenient wrappers for testing +# 7. Unittest Support +# 8. Debugging Support +# 9. Example Usage + +###################################################################### +## 1. Utility Functions +###################################################################### + +def _extract_future_flags(globs): + """ + Return the compiler-flags associated with the future features that + have been imported into the given namespace (globs). + """ + flags = 0 + for fname in __future__.all_feature_names: + feature = globs.get(fname, None) + if feature is getattr(__future__, fname): + flags |= feature.compiler_flag + return flags + +def _normalize_module(module, depth=2): + """ + Return the module specified by `module`. In particular: + - If `module` is a module, then return module. + - If `module` is a string, then import and return the + module with that name. + - If `module` is None, then return the calling module. + The calling module is assumed to be the module of + the stack frame at the given depth in the call stack. + """ + if inspect.ismodule(module): + return module + elif isinstance(module, str): + return __import__(module, globals(), locals(), ["*"]) + elif module is None: + return sys.modules[sys._getframe(depth).f_globals['__name__']] + else: + raise TypeError("Expected a module, string, or None") + +def _load_testfile(filename, package, module_relative, encoding): + if module_relative: + package = _normalize_module(package, 3) + filename = _module_relative_path(package, filename) + if getattr(package, '__loader__', None) is not None: + if hasattr(package.__loader__, 'get_data'): + file_contents = package.__loader__.get_data(filename) + file_contents = file_contents.decode(encoding) + # get_data() opens files as 'rb', so one must do the equivalent + # conversion as universal newlines would do. + return file_contents.replace(os.linesep, '\n'), filename + with open(filename, encoding=encoding) as f: + return f.read(), filename + +def _indent(s, indent=4): + """ + Add the given number of space characters to the beginning of + every non-blank line in `s`, and return the result. + """ + # This regexp matches the start of non-blank lines: + return re.sub('(?m)^(?!$)', indent*' ', s) + +def _exception_traceback(exc_info): + """ + Return a string containing a traceback message for the given + exc_info tuple (as returned by sys.exc_info()). + """ + # Get a traceback message. + excout = StringIO() + exc_type, exc_val, exc_tb = exc_info + traceback.print_exception(exc_type, exc_val, exc_tb, file=excout) + return excout.getvalue() + +# Override some StringIO methods. +class _SpoofOut(StringIO): + def getvalue(self): + result = StringIO.getvalue(self) + # If anything at all was written, make sure there's a trailing + # newline. There's no way for the expected output to indicate + # that a trailing newline is missing. + if result and not result.endswith("\n"): + result += "\n" + return result + + def truncate(self, size=None): + self.seek(size) + StringIO.truncate(self) + +# Worst-case linear-time ellipsis matching. +def _ellipsis_match(want, got): + """ + Essentially the only subtle case: + >>> _ellipsis_match('aa...aa', 'aaa') + False + """ + if ELLIPSIS_MARKER not in want: + return want == got + + # Find "the real" strings. + ws = want.split(ELLIPSIS_MARKER) + assert len(ws) >= 2 + + # Deal with exact matches possibly needed at one or both ends. + startpos, endpos = 0, len(got) + w = ws[0] + if w: # starts with exact match + if got.startswith(w): + startpos = len(w) + del ws[0] + else: + return False + w = ws[-1] + if w: # ends with exact match + if got.endswith(w): + endpos -= len(w) + del ws[-1] + else: + return False + + if startpos > endpos: + # Exact end matches required more characters than we have, as in + # _ellipsis_match('aa...aa', 'aaa') + return False + + # For the rest, we only need to find the leftmost non-overlapping + # match for each piece. If there's no overall match that way alone, + # there's no overall match period. + for w in ws: + # w may be '' at times, if there are consecutive ellipses, or + # due to an ellipsis at the start or end of `want`. That's OK. + # Search for an empty string succeeds, and doesn't change startpos. + startpos = got.find(w, startpos, endpos) + if startpos < 0: + return False + startpos += len(w) + + return True + +def _comment_line(line): + "Return a commented form of the given line" + line = line.rstrip() + if line: + return '# '+line + else: + return '#' + +def _strip_exception_details(msg): + # Support for IGNORE_EXCEPTION_DETAIL. + # Get rid of everything except the exception name; in particular, drop + # the possibly dotted module path (if any) and the exception message (if + # any). We assume that a colon is never part of a dotted name, or of an + # exception name. + # E.g., given + # "foo.bar.MyError: la di da" + # return "MyError" + # Or for "abc.def" or "abc.def:\n" return "def". + + start, end = 0, len(msg) + # The exception name must appear on the first line. + i = msg.find("\n") + if i >= 0: + end = i + # retain up to the first colon (if any) + i = msg.find(':', 0, end) + if i >= 0: + end = i + # retain just the exception name + i = msg.rfind('.', 0, end) + if i >= 0: + start = i+1 + return msg[start: end] + +class _OutputRedirectingPdb(pdb.Pdb): + """ + A specialized version of the python debugger that redirects stdout + to a given stream when interacting with the user. Stdout is *not* + redirected when traced code is executed. + """ + def __init__(self, out): + self.__out = out + self.__debugger_used = False + # do not play signal games in the pdb + pdb.Pdb.__init__(self, stdout=out, nosigint=True) + # still use input() to get user input + self.use_rawinput = 1 + + def set_trace(self, frame=None): + self.__debugger_used = True + if frame is None: + frame = sys._getframe().f_back + pdb.Pdb.set_trace(self, frame) + + def set_continue(self): + # Calling set_continue unconditionally would break unit test + # coverage reporting, as Bdb.set_continue calls sys.settrace(None). + if self.__debugger_used: + pdb.Pdb.set_continue(self) + + def trace_dispatch(self, *args): + # Redirect stdout to the given stream. + save_stdout = sys.stdout + sys.stdout = self.__out + # Call Pdb's trace dispatch method. + try: + return pdb.Pdb.trace_dispatch(self, *args) + finally: + sys.stdout = save_stdout + +# [XX] Normalize with respect to os.path.pardir? +def _module_relative_path(module, path): + if not inspect.ismodule(module): + raise TypeError('Expected a module: %r' % module) + if path.startswith('/'): + raise ValueError('Module-relative files may not have absolute paths') + + # Find the base directory for the path. + if hasattr(module, '__file__'): + # A normal module/package + basedir = os.path.split(module.__file__)[0] + elif module.__name__ == '__main__': + # An interactive session. + if len(sys.argv)>0 and sys.argv[0] != '': + basedir = os.path.split(sys.argv[0])[0] + else: + basedir = os.curdir + else: + # A module w/o __file__ (this includes builtins) + raise ValueError("Can't resolve paths relative to the module " + "%r (it has no __file__)" + % module.__name__) + + # Combine the base directory and the path. + return os.path.join(basedir, *(path.split('/'))) + +###################################################################### +## 2. Example & DocTest +###################################################################### +## - An "example" is a pair, where "source" is a +## fragment of source code, and "want" is the expected output for +## "source." The Example class also includes information about +## where the example was extracted from. +## +## - A "doctest" is a collection of examples, typically extracted from +## a string (such as an object's docstring). The DocTest class also +## includes information about where the string was extracted from. + +class Example: + """ + A single doctest example, consisting of source code and expected + output. `Example` defines the following attributes: + + - source: A single Python statement, always ending with a newline. + The constructor adds a newline if needed. + + - want: The expected output from running the source code (either + from stdout, or a traceback in case of exception). `want` ends + with a newline unless it's empty, in which case it's an empty + string. The constructor adds a newline if needed. + + - exc_msg: The exception message generated by the example, if + the example is expected to generate an exception; or `None` if + it is not expected to generate an exception. This exception + message is compared against the return value of + `traceback.format_exception_only()`. `exc_msg` ends with a + newline unless it's `None`. The constructor adds a newline + if needed. + + - lineno: The line number within the DocTest string containing + this Example where the Example begins. This line number is + zero-based, with respect to the beginning of the DocTest. + + - indent: The example's indentation in the DocTest string. + I.e., the number of space characters that precede the + example's first prompt. + + - options: A dictionary mapping from option flags to True or + False, which is used to override default options for this + example. Any option flags not contained in this dictionary + are left at their default value (as specified by the + DocTestRunner's optionflags). By default, no options are set. + """ + def __init__(self, source, want, exc_msg=None, lineno=0, indent=0, + options=None): + # Normalize inputs. + if not source.endswith('\n'): + source += '\n' + if want and not want.endswith('\n'): + want += '\n' + if exc_msg is not None and not exc_msg.endswith('\n'): + exc_msg += '\n' + # Store properties. + self.source = source + self.want = want + self.lineno = lineno + self.indent = indent + if options is None: options = {} + self.options = options + self.exc_msg = exc_msg + + def __eq__(self, other): + if type(self) is not type(other): + return NotImplemented + + return self.source == other.source and \ + self.want == other.want and \ + self.lineno == other.lineno and \ + self.indent == other.indent and \ + self.options == other.options and \ + self.exc_msg == other.exc_msg + + def __hash__(self): + return hash((self.source, self.want, self.lineno, self.indent, + self.exc_msg)) + +class DocTest: + """ + A collection of doctest examples that should be run in a single + namespace. Each `DocTest` defines the following attributes: + + - examples: the list of examples. + + - globs: The namespace (aka globals) that the examples should + be run in. + + - name: A name identifying the DocTest (typically, the name of + the object whose docstring this DocTest was extracted from). + + - filename: The name of the file that this DocTest was extracted + from, or `None` if the filename is unknown. + + - lineno: The line number within filename where this DocTest + begins, or `None` if the line number is unavailable. This + line number is zero-based, with respect to the beginning of + the file. + + - docstring: The string that the examples were extracted from, + or `None` if the string is unavailable. + """ + def __init__(self, examples, globs, name, filename, lineno, docstring): + """ + Create a new DocTest containing the given examples. The + DocTest's globals are initialized with a copy of `globs`. + """ + assert not isinstance(examples, str), \ + "DocTest no longer accepts str; use DocTestParser instead" + self.examples = examples + self.docstring = docstring + self.globs = globs.copy() + self.name = name + self.filename = filename + self.lineno = lineno + + def __repr__(self): + if len(self.examples) == 0: + examples = 'no examples' + elif len(self.examples) == 1: + examples = '1 example' + else: + examples = '%d examples' % len(self.examples) + return ('<%s %s from %s:%s (%s)>' % + (self.__class__.__name__, + self.name, self.filename, self.lineno, examples)) + + def __eq__(self, other): + if type(self) is not type(other): + return NotImplemented + + return self.examples == other.examples and \ + self.docstring == other.docstring and \ + self.globs == other.globs and \ + self.name == other.name and \ + self.filename == other.filename and \ + self.lineno == other.lineno + + def __hash__(self): + return hash((self.docstring, self.name, self.filename, self.lineno)) + + # This lets us sort tests by name: + def __lt__(self, other): + if not isinstance(other, DocTest): + return NotImplemented + return ((self.name, self.filename, self.lineno, id(self)) + < + (other.name, other.filename, other.lineno, id(other))) + +###################################################################### +## 3. DocTestParser +###################################################################### + +class DocTestParser: + """ + A class used to parse strings containing doctest examples. + """ + # This regular expression is used to find doctest examples in a + # string. It defines three groups: `source` is the source code + # (including leading indentation and prompts); `indent` is the + # indentation of the first (PS1) line of the source code; and + # `want` is the expected output (including leading indentation). + _EXAMPLE_RE = re.compile(r''' + # Source consists of a PS1 line followed by zero or more PS2 lines. + (?P + (?:^(?P [ ]*) >>> .*) # PS1 line + (?:\n [ ]* \.\.\. .*)*) # PS2 lines + \n? + # Want consists of any non-blank lines that do not start with PS1. + (?P (?:(?![ ]*$) # Not a blank line + (?![ ]*>>>) # Not a line starting with PS1 + .+$\n? # But any other line + )*) + ''', re.MULTILINE | re.VERBOSE) + + # A regular expression for handling `want` strings that contain + # expected exceptions. It divides `want` into three pieces: + # - the traceback header line (`hdr`) + # - the traceback stack (`stack`) + # - the exception message (`msg`), as generated by + # traceback.format_exception_only() + # `msg` may have multiple lines. We assume/require that the + # exception message is the first non-indented line starting with a word + # character following the traceback header line. + _EXCEPTION_RE = re.compile(r""" + # Grab the traceback header. Different versions of Python have + # said different things on the first traceback line. + ^(?P Traceback\ \( + (?: most\ recent\ call\ last + | innermost\ last + ) \) : + ) + \s* $ # toss trailing whitespace on the header. + (?P .*?) # don't blink: absorb stuff until... + ^ (?P \w+ .*) # a line *starts* with alphanum. + """, re.VERBOSE | re.MULTILINE | re.DOTALL) + + # A callable returning a true value iff its argument is a blank line + # or contains a single comment. + _IS_BLANK_OR_COMMENT = re.compile(r'^[ ]*(#.*)?$').match + + def parse(self, string, name=''): + """ + Divide the given string into examples and intervening text, + and return them as a list of alternating Examples and strings. + Line numbers for the Examples are 0-based. The optional + argument `name` is a name identifying this string, and is only + used for error messages. + """ + string = string.expandtabs() + # If all lines begin with the same indentation, then strip it. + min_indent = self._min_indent(string) + if min_indent > 0: + string = '\n'.join([l[min_indent:] for l in string.split('\n')]) + + output = [] + charno, lineno = 0, 0 + # Find all doctest examples in the string: + for m in self._EXAMPLE_RE.finditer(string): + # Add the pre-example text to `output`. + output.append(string[charno:m.start()]) + # Update lineno (lines before this example) + lineno += string.count('\n', charno, m.start()) + # Extract info from the regexp match. + (source, options, want, exc_msg) = \ + self._parse_example(m, name, lineno) + # Create an Example, and add it to the list. + if not self._IS_BLANK_OR_COMMENT(source): + output.append( Example(source, want, exc_msg, + lineno=lineno, + indent=min_indent+len(m.group('indent')), + options=options) ) + # Update lineno (lines inside this example) + lineno += string.count('\n', m.start(), m.end()) + # Update charno. + charno = m.end() + # Add any remaining post-example text to `output`. + output.append(string[charno:]) + return output + + def get_doctest(self, string, globs, name, filename, lineno): + """ + Extract all doctest examples from the given string, and + collect them into a `DocTest` object. + + `globs`, `name`, `filename`, and `lineno` are attributes for + the new `DocTest` object. See the documentation for `DocTest` + for more information. + """ + return DocTest(self.get_examples(string, name), globs, + name, filename, lineno, string) + + def get_examples(self, string, name=''): + """ + Extract all doctest examples from the given string, and return + them as a list of `Example` objects. Line numbers are + 0-based, because it's most common in doctests that nothing + interesting appears on the same line as opening triple-quote, + and so the first interesting line is called \"line 1\" then. + + The optional argument `name` is a name identifying this + string, and is only used for error messages. + """ + return [x for x in self.parse(string, name) + if isinstance(x, Example)] + + def _parse_example(self, m, name, lineno): + """ + Given a regular expression match from `_EXAMPLE_RE` (`m`), + return a pair `(source, want)`, where `source` is the matched + example's source code (with prompts and indentation stripped); + and `want` is the example's expected output (with indentation + stripped). + + `name` is the string's name, and `lineno` is the line number + where the example starts; both are used for error messages. + """ + # Get the example's indentation level. + indent = len(m.group('indent')) + + # Divide source into lines; check that they're properly + # indented; and then strip their indentation & prompts. + source_lines = m.group('source').split('\n') + self._check_prompt_blank(source_lines, indent, name, lineno) + self._check_prefix(source_lines[1:], ' '*indent + '.', name, lineno) + source = '\n'.join([sl[indent+4:] for sl in source_lines]) + + # Divide want into lines; check that it's properly indented; and + # then strip the indentation. Spaces before the last newline should + # be preserved, so plain rstrip() isn't good enough. + want = m.group('want') + want_lines = want.split('\n') + if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]): + del want_lines[-1] # forget final newline & spaces after it + self._check_prefix(want_lines, ' '*indent, name, + lineno + len(source_lines)) + want = '\n'.join([wl[indent:] for wl in want_lines]) + + # If `want` contains a traceback message, then extract it. + m = self._EXCEPTION_RE.match(want) + if m: + exc_msg = m.group('msg') + else: + exc_msg = None + + # Extract options from the source. + options = self._find_options(source, name, lineno) + + return source, options, want, exc_msg + + # This regular expression looks for option directives in the + # source code of an example. Option directives are comments + # starting with "doctest:". Warning: this may give false + # positives for string-literals that contain the string + # "#doctest:". Eliminating these false positives would require + # actually parsing the string; but we limit them by ignoring any + # line containing "#doctest:" that is *followed* by a quote mark. + _OPTION_DIRECTIVE_RE = re.compile(r'#\s*doctest:\s*([^\n\'"]*)$', + re.MULTILINE) + + def _find_options(self, source, name, lineno): + """ + Return a dictionary containing option overrides extracted from + option directives in the given source string. + + `name` is the string's name, and `lineno` is the line number + where the example starts; both are used for error messages. + """ + options = {} + # (note: with the current regexp, this will match at most once:) + for m in self._OPTION_DIRECTIVE_RE.finditer(source): + option_strings = m.group(1).replace(',', ' ').split() + for option in option_strings: + if (option[0] not in '+-' or + option[1:] not in OPTIONFLAGS_BY_NAME): + raise ValueError('line %r of the doctest for %s ' + 'has an invalid option: %r' % + (lineno+1, name, option)) + flag = OPTIONFLAGS_BY_NAME[option[1:]] + options[flag] = (option[0] == '+') + if options and self._IS_BLANK_OR_COMMENT(source): + raise ValueError('line %r of the doctest for %s has an option ' + 'directive on a line with no example: %r' % + (lineno, name, source)) + return options + + # This regular expression finds the indentation of every non-blank + # line in a string. + _INDENT_RE = re.compile('^([ ]*)(?=\S)', re.MULTILINE) + + def _min_indent(self, s): + "Return the minimum indentation of any non-blank line in `s`" + indents = [len(indent) for indent in self._INDENT_RE.findall(s)] + if len(indents) > 0: + return min(indents) + else: + return 0 + + def _check_prompt_blank(self, lines, indent, name, lineno): + """ + Given the lines of a source string (including prompts and + leading indentation), check to make sure that every prompt is + followed by a space character. If any line is not followed by + a space character, then raise ValueError. + """ + for i, line in enumerate(lines): + if len(line) >= indent+4 and line[indent+3] != ' ': + raise ValueError('line %r of the docstring for %s ' + 'lacks blank after %s: %r' % + (lineno+i+1, name, + line[indent:indent+3], line)) + + def _check_prefix(self, lines, prefix, name, lineno): + """ + Check that every line in the given list starts with the given + prefix; if any line does not, then raise a ValueError. + """ + for i, line in enumerate(lines): + if line and not line.startswith(prefix): + raise ValueError('line %r of the docstring for %s has ' + 'inconsistent leading whitespace: %r' % + (lineno+i+1, name, line)) + + +###################################################################### +## 4. DocTest Finder +###################################################################### + +class DocTestFinder: + """ + A class used to extract the DocTests that are relevant to a given + object, from its docstring and the docstrings of its contained + objects. Doctests can currently be extracted from the following + object types: modules, functions, classes, methods, staticmethods, + classmethods, and properties. + """ + + def __init__(self, verbose=False, parser=DocTestParser(), + recurse=True, exclude_empty=True): + """ + Create a new doctest finder. + + The optional argument `parser` specifies a class or + function that should be used to create new DocTest objects (or + objects that implement the same interface as DocTest). The + signature for this factory function should match the signature + of the DocTest constructor. + + If the optional argument `recurse` is false, then `find` will + only examine the given object, and not any contained objects. + + If the optional argument `exclude_empty` is false, then `find` + will include tests for objects with empty docstrings. + """ + self._parser = parser + self._verbose = verbose + self._recurse = recurse + self._exclude_empty = exclude_empty + + def find(self, obj, name=None, module=None, globs=None, extraglobs=None): + """ + Return a list of the DocTests that are defined by the given + object's docstring, or by any of its contained objects' + docstrings. + + The optional parameter `module` is the module that contains + the given object. If the module is not specified or is None, then + the test finder will attempt to automatically determine the + correct module. The object's module is used: + + - As a default namespace, if `globs` is not specified. + - To prevent the DocTestFinder from extracting DocTests + from objects that are imported from other modules. + - To find the name of the file containing the object. + - To help find the line number of the object within its + file. + + Contained objects whose module does not match `module` are ignored. + + If `module` is False, no attempt to find the module will be made. + This is obscure, of use mostly in tests: if `module` is False, or + is None but cannot be found automatically, then all objects are + considered to belong to the (non-existent) module, so all contained + objects will (recursively) be searched for doctests. + + The globals for each DocTest is formed by combining `globs` + and `extraglobs` (bindings in `extraglobs` override bindings + in `globs`). A new copy of the globals dictionary is created + for each DocTest. If `globs` is not specified, then it + defaults to the module's `__dict__`, if specified, or {} + otherwise. If `extraglobs` is not specified, then it defaults + to {}. + + """ + # If name was not specified, then extract it from the object. + if name is None: + name = getattr(obj, '__name__', None) + if name is None: + raise ValueError("DocTestFinder.find: name must be given " + "when obj.__name__ doesn't exist: %r" % + (type(obj),)) + + # Find the module that contains the given object (if obj is + # a module, then module=obj.). Note: this may fail, in which + # case module will be None. + if module is False: + module = None + elif module is None: + module = inspect.getmodule(obj) + + # Read the module's source code. This is used by + # DocTestFinder._find_lineno to find the line number for a + # given object's docstring. + try: + file = inspect.getsourcefile(obj) + except TypeError: + source_lines = None + else: + if not file: + # Check to see if it's one of our special internal "files" + # (see __patched_linecache_getlines). + file = inspect.getfile(obj) + if not file[0]+file[-2:] == '<]>': file = None + if file is None: + source_lines = None + else: + if module is not None: + # Supply the module globals in case the module was + # originally loaded via a PEP 302 loader and + # file is not a valid filesystem path + source_lines = linecache.getlines(file, module.__dict__) + else: + # No access to a loader, so assume it's a normal + # filesystem path + source_lines = linecache.getlines(file) + if not source_lines: + source_lines = None + + # Initialize globals, and merge in extraglobs. + if globs is None: + if module is None: + globs = {} + else: + globs = module.__dict__.copy() + else: + globs = globs.copy() + if extraglobs is not None: + globs.update(extraglobs) + if '__name__' not in globs: + globs['__name__'] = '__main__' # provide a default module name + + # Recursively explore `obj`, extracting DocTests. + tests = [] + self._find(tests, obj, name, module, source_lines, globs, {}) + # Sort the tests by alpha order of names, for consistency in + # verbose-mode output. This was a feature of doctest in Pythons + # <= 2.3 that got lost by accident in 2.4. It was repaired in + # 2.4.4 and 2.5. + tests.sort() + return tests + + def _from_module(self, module, object): + """ + Return true if the given object is defined in the given + module. + """ + if module is None: + return True + elif inspect.getmodule(object) is not None: + return module is inspect.getmodule(object) + elif inspect.isfunction(object): + return module.__dict__ is object.__globals__ + elif inspect.ismethoddescriptor(object): + if hasattr(object, '__objclass__'): + obj_mod = object.__objclass__.__module__ + elif hasattr(object, '__module__'): + obj_mod = object.__module__ + else: + return True # [XX] no easy way to tell otherwise + return module.__name__ == obj_mod + elif inspect.isclass(object): + return module.__name__ == object.__module__ + elif hasattr(object, '__module__'): + return module.__name__ == object.__module__ + elif isinstance(object, property): + return True # [XX] no way not be sure. + else: + raise ValueError("object must be a class or function") + + def _find(self, tests, obj, name, module, source_lines, globs, seen): + """ + Find tests for the given object and any contained objects, and + add them to `tests`. + """ + if self._verbose: + print('Finding tests in %s' % name) + + # If we've already processed this object, then ignore it. + if id(obj) in seen: + return + seen[id(obj)] = 1 + + # Find a test for this object, and add it to the list of tests. + test = self._get_test(obj, name, module, globs, source_lines) + if test is not None: + tests.append(test) + + # Look for tests in a module's contained objects. + if inspect.ismodule(obj) and self._recurse: + for valname, val in obj.__dict__.items(): + valname = '%s.%s' % (name, valname) + # Recurse to functions & classes. + if ((inspect.isroutine(inspect.unwrap(val)) + or inspect.isclass(val)) and + self._from_module(module, val)): + self._find(tests, val, valname, module, source_lines, + globs, seen) + + # Look for tests in a module's __test__ dictionary. + if inspect.ismodule(obj) and self._recurse: + for valname, val in getattr(obj, '__test__', {}).items(): + if not isinstance(valname, str): + raise ValueError("DocTestFinder.find: __test__ keys " + "must be strings: %r" % + (type(valname),)) + if not (inspect.isroutine(val) or inspect.isclass(val) or + inspect.ismodule(val) or isinstance(val, str)): + raise ValueError("DocTestFinder.find: __test__ values " + "must be strings, functions, methods, " + "classes, or modules: %r" % + (type(val),)) + valname = '%s.__test__.%s' % (name, valname) + self._find(tests, val, valname, module, source_lines, + globs, seen) + + # Look for tests in a class's contained objects. + if inspect.isclass(obj) and self._recurse: + for valname, val in obj.__dict__.items(): + # Special handling for staticmethod/classmethod. + if isinstance(val, staticmethod): + val = getattr(obj, valname) + if isinstance(val, classmethod): + val = getattr(obj, valname).__func__ + + # Recurse to methods, properties, and nested classes. + if ((inspect.isroutine(val) or inspect.isclass(val) or + isinstance(val, property)) and + self._from_module(module, val)): + valname = '%s.%s' % (name, valname) + self._find(tests, val, valname, module, source_lines, + globs, seen) + + def _get_test(self, obj, name, module, globs, source_lines): + """ + Return a DocTest for the given object, if it defines a docstring; + otherwise, return None. + """ + # Extract the object's docstring. If it doesn't have one, + # then return None (no test for this object). + if isinstance(obj, str): + docstring = obj + else: + try: + if obj.__doc__ is None: + docstring = '' + else: + docstring = obj.__doc__ + if not isinstance(docstring, str): + docstring = str(docstring) + except (TypeError, AttributeError): + docstring = '' + + # Find the docstring's location in the file. + lineno = self._find_lineno(obj, source_lines) + + # Don't bother if the docstring is empty. + if self._exclude_empty and not docstring: + return None + + # Return a DocTest for this object. + if module is None: + filename = None + else: + filename = getattr(module, '__file__', module.__name__) + if filename[-4:] == ".pyc": + filename = filename[:-1] + return self._parser.get_doctest(docstring, globs, name, + filename, lineno) + + def _find_lineno(self, obj, source_lines): + """ + Return a line number of the given object's docstring. Note: + this method assumes that the object has a docstring. + """ + lineno = None + + # Find the line number for modules. + if inspect.ismodule(obj): + lineno = 0 + + # Find the line number for classes. + # Note: this could be fooled if a class is defined multiple + # times in a single file. + if inspect.isclass(obj): + if source_lines is None: + return None + pat = re.compile(r'^\s*class\s*%s\b' % + getattr(obj, '__name__', '-')) + for i, line in enumerate(source_lines): + if pat.match(line): + lineno = i + break + + # Find the line number for functions & methods. + if inspect.ismethod(obj): obj = obj.__func__ + if inspect.isfunction(obj): obj = obj.__code__ + if inspect.istraceback(obj): obj = obj.tb_frame + if inspect.isframe(obj): obj = obj.f_code + if inspect.iscode(obj): + lineno = getattr(obj, 'co_firstlineno', None)-1 + + # Find the line number where the docstring starts. Assume + # that it's the first line that begins with a quote mark. + # Note: this could be fooled by a multiline function + # signature, where a continuation line begins with a quote + # mark. + if lineno is not None: + if source_lines is None: + return lineno+1 + pat = re.compile('(^|.*:)\s*\w*("|\')') + for lineno in range(lineno, len(source_lines)): + if pat.match(source_lines[lineno]): + return lineno + + # We couldn't find the line number. + return None + +###################################################################### +## 5. DocTest Runner +###################################################################### + +class DocTestRunner: + """ + A class used to run DocTest test cases, and accumulate statistics. + The `run` method is used to process a single DocTest case. It + returns a tuple `(f, t)`, where `t` is the number of test cases + tried, and `f` is the number of test cases that failed. + + >>> tests = DocTestFinder().find(_TestClass) + >>> runner = DocTestRunner(verbose=False) + >>> tests.sort(key = lambda test: test.name) + >>> for test in tests: + ... print(test.name, '->', runner.run(test)) + _TestClass -> TestResults(failed=0, attempted=2) + _TestClass.__init__ -> TestResults(failed=0, attempted=2) + _TestClass.get -> TestResults(failed=0, attempted=2) + _TestClass.square -> TestResults(failed=0, attempted=1) + + The `summarize` method prints a summary of all the test cases that + have been run by the runner, and returns an aggregated `(f, t)` + tuple: + + >>> runner.summarize(verbose=1) + 4 items passed all tests: + 2 tests in _TestClass + 2 tests in _TestClass.__init__ + 2 tests in _TestClass.get + 1 tests in _TestClass.square + 7 tests in 4 items. + 7 passed and 0 failed. + Test passed. + TestResults(failed=0, attempted=7) + + The aggregated number of tried examples and failed examples is + also available via the `tries` and `failures` attributes: + + >>> runner.tries + 7 + >>> runner.failures + 0 + + The comparison between expected outputs and actual outputs is done + by an `OutputChecker`. This comparison may be customized with a + number of option flags; see the documentation for `testmod` for + more information. If the option flags are insufficient, then the + comparison may also be customized by passing a subclass of + `OutputChecker` to the constructor. + + The test runner's display output can be controlled in two ways. + First, an output function (`out) can be passed to + `TestRunner.run`; this function will be called with strings that + should be displayed. It defaults to `sys.stdout.write`. If + capturing the output is not sufficient, then the display output + can be also customized by subclassing DocTestRunner, and + overriding the methods `report_start`, `report_success`, + `report_unexpected_exception`, and `report_failure`. + """ + # This divider string is used to separate failure messages, and to + # separate sections of the summary. + DIVIDER = "*" * 70 + + def __init__(self, checker=None, verbose=None, optionflags=0): + """ + Create a new test runner. + + Optional keyword arg `checker` is the `OutputChecker` that + should be used to compare the expected outputs and actual + outputs of doctest examples. + + Optional keyword arg 'verbose' prints lots of stuff if true, + only failures if false; by default, it's true iff '-v' is in + sys.argv. + + Optional argument `optionflags` can be used to control how the + test runner compares expected output to actual output, and how + it displays failures. See the documentation for `testmod` for + more information. + """ + self._checker = checker or OutputChecker() + if verbose is None: + verbose = '-v' in sys.argv + self._verbose = verbose + self.optionflags = optionflags + self.original_optionflags = optionflags + + # Keep track of the examples we've run. + self.tries = 0 + self.failures = 0 + self._name2ft = {} + + # Create a fake output target for capturing doctest output. + self._fakeout = _SpoofOut() + + #///////////////////////////////////////////////////////////////// + # Reporting methods + #///////////////////////////////////////////////////////////////// + + def report_start(self, out, test, example): + """ + Report that the test runner is about to process the given + example. (Only displays a message if verbose=True) + """ + if self._verbose: + if example.want: + out('Trying:\n' + _indent(example.source) + + 'Expecting:\n' + _indent(example.want)) + else: + out('Trying:\n' + _indent(example.source) + + 'Expecting nothing\n') + + def report_success(self, out, test, example, got): + """ + Report that the given example ran successfully. (Only + displays a message if verbose=True) + """ + if self._verbose: + out("ok\n") + + def report_failure(self, out, test, example, got): + """ + Report that the given example failed. + """ + out(self._failure_header(test, example) + + self._checker.output_difference(example, got, self.optionflags)) + + def report_unexpected_exception(self, out, test, example, exc_info): + """ + Report that the given example raised an unexpected exception. + """ + out(self._failure_header(test, example) + + 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) + + def _failure_header(self, test, example): + out = [self.DIVIDER] + if test.filename: + if test.lineno is not None and example.lineno is not None: + lineno = test.lineno + example.lineno + 1 + else: + lineno = '?' + out.append('File "%s", line %s, in %s' % + (test.filename, lineno, test.name)) + else: + out.append('Line %s, in %s' % (example.lineno+1, test.name)) + out.append('Failed example:') + source = example.source + out.append(_indent(source)) + return '\n'.join(out) + + #///////////////////////////////////////////////////////////////// + # DocTest Running + #///////////////////////////////////////////////////////////////// + + def __run(self, test, compileflags, out): + """ + Run the examples in `test`. Write the outcome of each example + with one of the `DocTestRunner.report_*` methods, using the + writer function `out`. `compileflags` is the set of compiler + flags that should be used to execute examples. Return a tuple + `(f, t)`, where `t` is the number of examples tried, and `f` + is the number of examples that failed. The examples are run + in the namespace `test.globs`. + """ + # Keep track of the number of failures and tries. + failures = tries = 0 + + # Save the option flags (since option directives can be used + # to modify them). + original_optionflags = self.optionflags + + SUCCESS, FAILURE, BOOM = range(3) # `outcome` state + + check = self._checker.check_output + + # Process each example. + for examplenum, example in enumerate(test.examples): + + # If REPORT_ONLY_FIRST_FAILURE is set, then suppress + # reporting after the first failure. + quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and + failures > 0) + + # Merge in the example's options. + self.optionflags = original_optionflags + if example.options: + for (optionflag, val) in example.options.items(): + if val: + self.optionflags |= optionflag + else: + self.optionflags &= ~optionflag + + # If 'SKIP' is set, then skip this example. + if self.optionflags & SKIP: + continue + + # Record that we started this example. + tries += 1 + if not quiet: + self.report_start(out, test, example) + + # Use a special filename for compile(), so we can retrieve + # the source code during interactive debugging (see + # __patched_linecache_getlines). + filename = '' % (test.name, examplenum) + + # Run the example in the given context (globs), and record + # any exception that gets raised. (But don't intercept + # keyboard interrupts.) + try: + # Don't blink! This is where the user's code gets run. + exec(compile(example.source, filename, "single", + compileflags, 1), test.globs) + self.debugger.set_continue() # ==== Example Finished ==== + exception = None + except KeyboardInterrupt: + raise + except: + exception = sys.exc_info() + self.debugger.set_continue() # ==== Example Finished ==== + + got = self._fakeout.getvalue() # the actual output + self._fakeout.truncate(0) + outcome = FAILURE # guilty until proved innocent or insane + + # If the example executed without raising any exceptions, + # verify its output. + if exception is None: + if check(example.want, got, self.optionflags): + outcome = SUCCESS + + # The example raised an exception: check if it was expected. + else: + exc_msg = traceback.format_exception_only(*exception[:2])[-1] + if not quiet: + got += _exception_traceback(exception) + + # If `example.exc_msg` is None, then we weren't expecting + # an exception. + if example.exc_msg is None: + outcome = BOOM + + # We expected an exception: see whether it matches. + elif check(example.exc_msg, exc_msg, self.optionflags): + outcome = SUCCESS + + # Another chance if they didn't care about the detail. + elif self.optionflags & IGNORE_EXCEPTION_DETAIL: + if check(_strip_exception_details(example.exc_msg), + _strip_exception_details(exc_msg), + self.optionflags): + outcome = SUCCESS + + # Report the outcome. + if outcome is SUCCESS: + if not quiet: + self.report_success(out, test, example, got) + elif outcome is FAILURE: + if not quiet: + self.report_failure(out, test, example, got) + failures += 1 + elif outcome is BOOM: + if not quiet: + self.report_unexpected_exception(out, test, example, + exception) + failures += 1 + else: + assert False, ("unknown outcome", outcome) + + if failures and self.optionflags & FAIL_FAST: + break + + # Restore the option flags (in case they were modified) + self.optionflags = original_optionflags + + # Record and return the number of failures and tries. + self.__record_outcome(test, failures, tries) + return TestResults(failures, tries) + + def __record_outcome(self, test, f, t): + """ + Record the fact that the given DocTest (`test`) generated `f` + failures out of `t` tried examples. + """ + f2, t2 = self._name2ft.get(test.name, (0,0)) + self._name2ft[test.name] = (f+f2, t+t2) + self.failures += f + self.tries += t + + __LINECACHE_FILENAME_RE = re.compile(r'.+)' + r'\[(?P\d+)\]>$') + def __patched_linecache_getlines(self, filename, module_globals=None): + m = self.__LINECACHE_FILENAME_RE.match(filename) + if m and m.group('name') == self.test.name: + example = self.test.examples[int(m.group('examplenum'))] + return example.source.splitlines(keepends=True) + else: + return self.save_linecache_getlines(filename, module_globals) + + def run(self, test, compileflags=None, out=None, clear_globs=True): + """ + Run the examples in `test`, and display the results using the + writer function `out`. + + The examples are run in the namespace `test.globs`. If + `clear_globs` is true (the default), then this namespace will + be cleared after the test runs, to help with garbage + collection. If you would like to examine the namespace after + the test completes, then use `clear_globs=False`. + + `compileflags` gives the set of flags that should be used by + the Python compiler when running the examples. If not + specified, then it will default to the set of future-import + flags that apply to `globs`. + + The output of each example is checked using + `DocTestRunner.check_output`, and the results are formatted by + the `DocTestRunner.report_*` methods. + """ + self.test = test + + if compileflags is None: + compileflags = _extract_future_flags(test.globs) + + save_stdout = sys.stdout + if out is None: + encoding = save_stdout.encoding + if encoding is None or encoding.lower() == 'utf-8': + out = save_stdout.write + else: + # Use backslashreplace error handling on write + def out(s): + s = str(s.encode(encoding, 'backslashreplace'), encoding) + save_stdout.write(s) + sys.stdout = self._fakeout + + # Patch pdb.set_trace to restore sys.stdout during interactive + # debugging (so it's not still redirected to self._fakeout). + # Note that the interactive output will go to *our* + # save_stdout, even if that's not the real sys.stdout; this + # allows us to write test cases for the set_trace behavior. + save_trace = sys.gettrace() + save_set_trace = pdb.set_trace + self.debugger = _OutputRedirectingPdb(save_stdout) + self.debugger.reset() + pdb.set_trace = self.debugger.set_trace + + # Patch linecache.getlines, so we can see the example's source + # when we're inside the debugger. + self.save_linecache_getlines = linecache.getlines + linecache.getlines = self.__patched_linecache_getlines + + # Make sure sys.displayhook just prints the value to stdout + save_displayhook = sys.displayhook + sys.displayhook = sys.__displayhook__ + + try: + return self.__run(test, compileflags, out) + finally: + sys.stdout = save_stdout + pdb.set_trace = save_set_trace + sys.settrace(save_trace) + linecache.getlines = self.save_linecache_getlines + sys.displayhook = save_displayhook + if clear_globs: + test.globs.clear() + import builtins + builtins._ = None + + #///////////////////////////////////////////////////////////////// + # Summarization + #///////////////////////////////////////////////////////////////// + def summarize(self, verbose=None): + """ + Print a summary of all the test cases that have been run by + this DocTestRunner, and return a tuple `(f, t)`, where `f` is + the total number of failed examples, and `t` is the total + number of tried examples. + + The optional `verbose` argument controls how detailed the + summary is. If the verbosity is not specified, then the + DocTestRunner's verbosity is used. + """ + if verbose is None: + verbose = self._verbose + notests = [] + passed = [] + failed = [] + totalt = totalf = 0 + for x in self._name2ft.items(): + name, (f, t) = x + assert f <= t + totalt += t + totalf += f + if t == 0: + notests.append(name) + elif f == 0: + passed.append( (name, t) ) + else: + failed.append(x) + if verbose: + if notests: + print(len(notests), "items had no tests:") + notests.sort() + for thing in notests: + print(" ", thing) + if passed: + print(len(passed), "items passed all tests:") + passed.sort() + for thing, count in passed: + print(" %3d tests in %s" % (count, thing)) + if failed: + print(self.DIVIDER) + print(len(failed), "items had failures:") + failed.sort() + for thing, (f, t) in failed: + print(" %3d of %3d in %s" % (f, t, thing)) + if verbose: + print(totalt, "tests in", len(self._name2ft), "items.") + print(totalt - totalf, "passed and", totalf, "failed.") + if totalf: + print("***Test Failed***", totalf, "failures.") + elif verbose: + print("Test passed.") + return TestResults(totalf, totalt) + + #///////////////////////////////////////////////////////////////// + # Backward compatibility cruft to maintain doctest.master. + #///////////////////////////////////////////////////////////////// + def merge(self, other): + d = self._name2ft + for name, (f, t) in other._name2ft.items(): + if name in d: + # Don't print here by default, since doing + # so breaks some of the buildbots + #print("*** DocTestRunner.merge: '" + name + "' in both" \ + # " testers; summing outcomes.") + f2, t2 = d[name] + f = f + f2 + t = t + t2 + d[name] = f, t + +class OutputChecker: + """ + A class used to check the whether the actual output from a doctest + example matches the expected output. `OutputChecker` defines two + methods: `check_output`, which compares a given pair of outputs, + and returns true if they match; and `output_difference`, which + returns a string describing the differences between two outputs. + """ + def _toAscii(self, s): + """ + Convert string to hex-escaped ASCII string. + """ + return str(s.encode('ASCII', 'backslashreplace'), "ASCII") + + def check_output(self, want, got, optionflags): + """ + Return True iff the actual output from an example (`got`) + matches the expected output (`want`). These strings are + always considered to match if they are identical; but + depending on what option flags the test runner is using, + several non-exact match types are also possible. See the + documentation for `TestRunner` for more information about + option flags. + """ + + # If `want` contains hex-escaped character such as "\u1234", + # then `want` is a string of six characters(e.g. [\,u,1,2,3,4]). + # On the other hand, `got` could be another sequence of + # characters such as [\u1234], so `want` and `got` should + # be folded to hex-escaped ASCII string to compare. + got = self._toAscii(got) + want = self._toAscii(want) + + # Handle the common case first, for efficiency: + # if they're string-identical, always return true. + if got == want: + return True + + # The values True and False replaced 1 and 0 as the return + # value for boolean comparisons in Python 2.3. + if not (optionflags & DONT_ACCEPT_TRUE_FOR_1): + if (got,want) == ("True\n", "1\n"): + return True + if (got,want) == ("False\n", "0\n"): + return True + + # can be used as a special sequence to signify a + # blank line, unless the DONT_ACCEPT_BLANKLINE flag is used. + if not (optionflags & DONT_ACCEPT_BLANKLINE): + # Replace in want with a blank line. + want = re.sub('(?m)^%s\s*?$' % re.escape(BLANKLINE_MARKER), + '', want) + # If a line in got contains only spaces, then remove the + # spaces. + got = re.sub('(?m)^\s*?$', '', got) + if got == want: + return True + + # This flag causes doctest to ignore any differences in the + # contents of whitespace strings. Note that this can be used + # in conjunction with the ELLIPSIS flag. + if optionflags & NORMALIZE_WHITESPACE: + got = ' '.join(got.split()) + want = ' '.join(want.split()) + if got == want: + return True + + # The ELLIPSIS flag says to let the sequence "..." in `want` + # match any substring in `got`. + if optionflags & ELLIPSIS: + if _ellipsis_match(want, got): + return True + + # We didn't find any match; return false. + return False + + # Should we do a fancy diff? + def _do_a_fancy_diff(self, want, got, optionflags): + # Not unless they asked for a fancy diff. + if not optionflags & (REPORT_UDIFF | + REPORT_CDIFF | + REPORT_NDIFF): + return False + + # If expected output uses ellipsis, a meaningful fancy diff is + # too hard ... or maybe not. In two real-life failures Tim saw, + # a diff was a major help anyway, so this is commented out. + # [todo] _ellipsis_match() knows which pieces do and don't match, + # and could be the basis for a kick-ass diff in this case. + ##if optionflags & ELLIPSIS and ELLIPSIS_MARKER in want: + ## return False + + # ndiff does intraline difference marking, so can be useful even + # for 1-line differences. + if optionflags & REPORT_NDIFF: + return True + + # The other diff types need at least a few lines to be helpful. + return want.count('\n') > 2 and got.count('\n') > 2 + + def output_difference(self, example, got, optionflags): + """ + Return a string describing the differences between the + expected output for a given example (`example`) and the actual + output (`got`). `optionflags` is the set of option flags used + to compare `want` and `got`. + """ + want = example.want + # If s are being used, then replace blank lines + # with in the actual output string. + if not (optionflags & DONT_ACCEPT_BLANKLINE): + got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got) + + # Check if we should use diff. + if self._do_a_fancy_diff(want, got, optionflags): + # Split want & got into lines. + want_lines = want.splitlines(keepends=True) + got_lines = got.splitlines(keepends=True) + # Use difflib to find their differences. + if optionflags & REPORT_UDIFF: + diff = difflib.unified_diff(want_lines, got_lines, n=2) + diff = list(diff)[2:] # strip the diff header + kind = 'unified diff with -expected +actual' + elif optionflags & REPORT_CDIFF: + diff = difflib.context_diff(want_lines, got_lines, n=2) + diff = list(diff)[2:] # strip the diff header + kind = 'context diff with expected followed by actual' + elif optionflags & REPORT_NDIFF: + engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK) + diff = list(engine.compare(want_lines, got_lines)) + kind = 'ndiff with -expected +actual' + else: + assert 0, 'Bad diff option' + # Remove trailing whitespace on diff output. + diff = [line.rstrip() + '\n' for line in diff] + return 'Differences (%s):\n' % kind + _indent(''.join(diff)) + + # If we're not using diff, then simply list the expected + # output followed by the actual output. + if want and got: + return 'Expected:\n%sGot:\n%s' % (_indent(want), _indent(got)) + elif want: + return 'Expected:\n%sGot nothing\n' % _indent(want) + elif got: + return 'Expected nothing\nGot:\n%s' % _indent(got) + else: + return 'Expected nothing\nGot nothing\n' + +class DocTestFailure(Exception): + """A DocTest example has failed in debugging mode. + + The exception instance has variables: + + - test: the DocTest object being run + + - example: the Example object that failed + + - got: the actual output + """ + def __init__(self, test, example, got): + self.test = test + self.example = example + self.got = got + + def __str__(self): + return str(self.test) + +class UnexpectedException(Exception): + """A DocTest example has encountered an unexpected exception + + The exception instance has variables: + + - test: the DocTest object being run + + - example: the Example object that failed + + - exc_info: the exception info + """ + def __init__(self, test, example, exc_info): + self.test = test + self.example = example + self.exc_info = exc_info + + def __str__(self): + return str(self.test) + +class DebugRunner(DocTestRunner): + r"""Run doc tests but raise an exception as soon as there is a failure. + + If an unexpected exception occurs, an UnexpectedException is raised. + It contains the test, the example, and the original exception: + + >>> runner = DebugRunner(verbose=False) + >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', + ... {}, 'foo', 'foo.py', 0) + >>> try: + ... runner.run(test) + ... except UnexpectedException as f: + ... failure = f + + >>> failure.test is test + True + + >>> failure.example.want + '42\n' + + >>> exc_info = failure.exc_info + >>> raise exc_info[1] # Already has the traceback + Traceback (most recent call last): + ... + KeyError + + We wrap the original exception to give the calling application + access to the test and example information. + + If the output doesn't match, then a DocTestFailure is raised: + + >>> test = DocTestParser().get_doctest(''' + ... >>> x = 1 + ... >>> x + ... 2 + ... ''', {}, 'foo', 'foo.py', 0) + + >>> try: + ... runner.run(test) + ... except DocTestFailure as f: + ... failure = f + + DocTestFailure objects provide access to the test: + + >>> failure.test is test + True + + As well as to the example: + + >>> failure.example.want + '2\n' + + and the actual output: + + >>> failure.got + '1\n' + + If a failure or error occurs, the globals are left intact: + + >>> del test.globs['__builtins__'] + >>> test.globs + {'x': 1} + + >>> test = DocTestParser().get_doctest(''' + ... >>> x = 2 + ... >>> raise KeyError + ... ''', {}, 'foo', 'foo.py', 0) + + >>> runner.run(test) + Traceback (most recent call last): + ... + doctest.UnexpectedException: + + >>> del test.globs['__builtins__'] + >>> test.globs + {'x': 2} + + But the globals are cleared if there is no error: + + >>> test = DocTestParser().get_doctest(''' + ... >>> x = 2 + ... ''', {}, 'foo', 'foo.py', 0) + + >>> runner.run(test) + TestResults(failed=0, attempted=1) + + >>> test.globs + {} + + """ + + def run(self, test, compileflags=None, out=None, clear_globs=True): + r = DocTestRunner.run(self, test, compileflags, out, False) + if clear_globs: + test.globs.clear() + return r + + def report_unexpected_exception(self, out, test, example, exc_info): + raise UnexpectedException(test, example, exc_info) + + def report_failure(self, out, test, example, got): + raise DocTestFailure(test, example, got) + +###################################################################### +## 6. Test Functions +###################################################################### +# These should be backwards compatible. + +# For backward compatibility, a global instance of a DocTestRunner +# class, updated by testmod. +master = None + +def testmod(m=None, name=None, globs=None, verbose=None, + report=True, optionflags=0, extraglobs=None, + raise_on_error=False, exclude_empty=False): + """m=None, name=None, globs=None, verbose=None, report=True, + optionflags=0, extraglobs=None, raise_on_error=False, + exclude_empty=False + + Test examples in docstrings in functions and classes reachable + from module m (or the current module if m is not supplied), starting + with m.__doc__. + + Also test examples reachable from dict m.__test__ if it exists and is + not None. m.__test__ maps names to functions, classes and strings; + function and class docstrings are tested even if the name is private; + strings are tested directly, as if they were docstrings. + + Return (#failures, #tests). + + See help(doctest) for an overview. + + Optional keyword arg "name" gives the name of the module; by default + use m.__name__. + + Optional keyword arg "globs" gives a dict to be used as the globals + when executing examples; by default, use m.__dict__. A copy of this + dict is actually used for each docstring, so that each docstring's + examples start with a clean slate. + + Optional keyword arg "extraglobs" gives a dictionary that should be + merged into the globals that are used to execute examples. By + default, no extra globals are used. This is new in 2.4. + + Optional keyword arg "verbose" prints lots of stuff if true, prints + only failures if false; by default, it's true iff "-v" is in sys.argv. + + Optional keyword arg "report" prints a summary at the end when true, + else prints nothing at the end. In verbose mode, the summary is + detailed, else very brief (in fact, empty if all tests passed). + + Optional keyword arg "optionflags" or's together module constants, + and defaults to 0. This is new in 2.3. Possible values (see the + docs for details): + + DONT_ACCEPT_TRUE_FOR_1 + DONT_ACCEPT_BLANKLINE + NORMALIZE_WHITESPACE + ELLIPSIS + SKIP + IGNORE_EXCEPTION_DETAIL + REPORT_UDIFF + REPORT_CDIFF + REPORT_NDIFF + REPORT_ONLY_FIRST_FAILURE + + Optional keyword arg "raise_on_error" raises an exception on the + first unexpected exception or failure. This allows failures to be + post-mortem debugged. + + Advanced tomfoolery: testmod runs methods of a local instance of + class doctest.Tester, then merges the results into (or creates) + global Tester instance doctest.master. Methods of doctest.master + can be called directly too, if you want to do something unusual. + Passing report=0 to testmod is especially useful then, to delay + displaying a summary. Invoke doctest.master.summarize(verbose) + when you're done fiddling. + """ + global master + + # If no module was given, then use __main__. + if m is None: + # DWA - m will still be None if this wasn't invoked from the command + # line, in which case the following TypeError is about as good an error + # as we should expect + m = sys.modules.get('__main__') + + # Check that we were actually given a module. + if not inspect.ismodule(m): + raise TypeError("testmod: module required; %r" % (m,)) + + # If no name was given, then use the module's name. + if name is None: + name = m.__name__ + + # Find, parse, and run all tests in the given module. + finder = DocTestFinder(exclude_empty=exclude_empty) + + if raise_on_error: + runner = DebugRunner(verbose=verbose, optionflags=optionflags) + else: + runner = DocTestRunner(verbose=verbose, optionflags=optionflags) + + for test in finder.find(m, name, globs=globs, extraglobs=extraglobs): + runner.run(test) + + if report: + runner.summarize() + + if master is None: + master = runner + else: + master.merge(runner) + + return TestResults(runner.failures, runner.tries) + +def testfile(filename, module_relative=True, name=None, package=None, + globs=None, verbose=None, report=True, optionflags=0, + extraglobs=None, raise_on_error=False, parser=DocTestParser(), + encoding=None): + """ + Test examples in the given file. Return (#failures, #tests). + + Optional keyword arg "module_relative" specifies how filenames + should be interpreted: + + - If "module_relative" is True (the default), then "filename" + specifies a module-relative path. By default, this path is + relative to the calling module's directory; but if the + "package" argument is specified, then it is relative to that + package. To ensure os-independence, "filename" should use + "/" characters to separate path segments, and should not + be an absolute path (i.e., it may not begin with "/"). + + - If "module_relative" is False, then "filename" specifies an + os-specific path. The path may be absolute or relative (to + the current working directory). + + Optional keyword arg "name" gives the name of the test; by default + use the file's basename. + + Optional keyword argument "package" is a Python package or the + name of a Python package whose directory should be used as the + base directory for a module relative filename. If no package is + specified, then the calling module's directory is used as the base + directory for module relative filenames. It is an error to + specify "package" if "module_relative" is False. + + Optional keyword arg "globs" gives a dict to be used as the globals + when executing examples; by default, use {}. A copy of this dict + is actually used for each docstring, so that each docstring's + examples start with a clean slate. + + Optional keyword arg "extraglobs" gives a dictionary that should be + merged into the globals that are used to execute examples. By + default, no extra globals are used. + + Optional keyword arg "verbose" prints lots of stuff if true, prints + only failures if false; by default, it's true iff "-v" is in sys.argv. + + Optional keyword arg "report" prints a summary at the end when true, + else prints nothing at the end. In verbose mode, the summary is + detailed, else very brief (in fact, empty if all tests passed). + + Optional keyword arg "optionflags" or's together module constants, + and defaults to 0. Possible values (see the docs for details): + + DONT_ACCEPT_TRUE_FOR_1 + DONT_ACCEPT_BLANKLINE + NORMALIZE_WHITESPACE + ELLIPSIS + SKIP + IGNORE_EXCEPTION_DETAIL + REPORT_UDIFF + REPORT_CDIFF + REPORT_NDIFF + REPORT_ONLY_FIRST_FAILURE + + Optional keyword arg "raise_on_error" raises an exception on the + first unexpected exception or failure. This allows failures to be + post-mortem debugged. + + Optional keyword arg "parser" specifies a DocTestParser (or + subclass) that should be used to extract tests from the files. + + Optional keyword arg "encoding" specifies an encoding that should + be used to convert the file to unicode. + + Advanced tomfoolery: testmod runs methods of a local instance of + class doctest.Tester, then merges the results into (or creates) + global Tester instance doctest.master. Methods of doctest.master + can be called directly too, if you want to do something unusual. + Passing report=0 to testmod is especially useful then, to delay + displaying a summary. Invoke doctest.master.summarize(verbose) + when you're done fiddling. + """ + global master + + if package and not module_relative: + raise ValueError("Package may only be specified for module-" + "relative paths.") + + # Relativize the path + text, filename = _load_testfile(filename, package, module_relative, + encoding or "utf-8") + + # If no name was given, then use the file's name. + if name is None: + name = os.path.basename(filename) + + # Assemble the globals. + if globs is None: + globs = {} + else: + globs = globs.copy() + if extraglobs is not None: + globs.update(extraglobs) + if '__name__' not in globs: + globs['__name__'] = '__main__' + + if raise_on_error: + runner = DebugRunner(verbose=verbose, optionflags=optionflags) + else: + runner = DocTestRunner(verbose=verbose, optionflags=optionflags) + + # Read the file, convert it to a test, and run it. + test = parser.get_doctest(text, globs, name, filename, 0) + runner.run(test) + + if report: + runner.summarize() + + if master is None: + master = runner + else: + master.merge(runner) + + return TestResults(runner.failures, runner.tries) + +def run_docstring_examples(f, globs, verbose=False, name="NoName", + compileflags=None, optionflags=0): + """ + Test examples in the given object's docstring (`f`), using `globs` + as globals. Optional argument `name` is used in failure messages. + If the optional argument `verbose` is true, then generate output + even if there are no failures. + + `compileflags` gives the set of flags that should be used by the + Python compiler when running the examples. If not specified, then + it will default to the set of future-import flags that apply to + `globs`. + + Optional keyword arg `optionflags` specifies options for the + testing and output. See the documentation for `testmod` for more + information. + """ + # Find, parse, and run all tests in the given module. + finder = DocTestFinder(verbose=verbose, recurse=False) + runner = DocTestRunner(verbose=verbose, optionflags=optionflags) + for test in finder.find(f, name, globs=globs): + runner.run(test, compileflags=compileflags) + +###################################################################### +## 7. Unittest Support +###################################################################### + +_unittest_reportflags = 0 + +def set_unittest_reportflags(flags): + """Sets the unittest option flags. + + The old flag is returned so that a runner could restore the old + value if it wished to: + + >>> import doctest + >>> old = doctest._unittest_reportflags + >>> doctest.set_unittest_reportflags(REPORT_NDIFF | + ... REPORT_ONLY_FIRST_FAILURE) == old + True + + >>> doctest._unittest_reportflags == (REPORT_NDIFF | + ... REPORT_ONLY_FIRST_FAILURE) + True + + Only reporting flags can be set: + + >>> doctest.set_unittest_reportflags(ELLIPSIS) + Traceback (most recent call last): + ... + ValueError: ('Only reporting flags allowed', 8) + + >>> doctest.set_unittest_reportflags(old) == (REPORT_NDIFF | + ... REPORT_ONLY_FIRST_FAILURE) + True + """ + global _unittest_reportflags + + if (flags & REPORTING_FLAGS) != flags: + raise ValueError("Only reporting flags allowed", flags) + old = _unittest_reportflags + _unittest_reportflags = flags + return old + + +class DocTestCase(unittest.TestCase): + + def __init__(self, test, optionflags=0, setUp=None, tearDown=None, + checker=None): + + unittest.TestCase.__init__(self) + self._dt_optionflags = optionflags + self._dt_checker = checker + self._dt_test = test + self._dt_setUp = setUp + self._dt_tearDown = tearDown + + def setUp(self): + test = self._dt_test + + if self._dt_setUp is not None: + self._dt_setUp(test) + + def tearDown(self): + test = self._dt_test + + if self._dt_tearDown is not None: + self._dt_tearDown(test) + + test.globs.clear() + + def runTest(self): + test = self._dt_test + old = sys.stdout + new = StringIO() + optionflags = self._dt_optionflags + + if not (optionflags & REPORTING_FLAGS): + # The option flags don't include any reporting flags, + # so add the default reporting flags + optionflags |= _unittest_reportflags + + runner = DocTestRunner(optionflags=optionflags, + checker=self._dt_checker, verbose=False) + + try: + runner.DIVIDER = "-"*70 + failures, tries = runner.run( + test, out=new.write, clear_globs=False) + finally: + sys.stdout = old + + if failures: + raise self.failureException(self.format_failure(new.getvalue())) + + def format_failure(self, err): + test = self._dt_test + if test.lineno is None: + lineno = 'unknown line number' + else: + lineno = '%s' % test.lineno + lname = '.'.join(test.name.split('.')[-1:]) + return ('Failed doctest test for %s\n' + ' File "%s", line %s, in %s\n\n%s' + % (test.name, test.filename, lineno, lname, err) + ) + + def debug(self): + r"""Run the test case without results and without catching exceptions + + The unit test framework includes a debug method on test cases + and test suites to support post-mortem debugging. The test code + is run in such a way that errors are not caught. This way a + caller can catch the errors and initiate post-mortem debugging. + + The DocTestCase provides a debug method that raises + UnexpectedException errors if there is an unexpected + exception: + + >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', + ... {}, 'foo', 'foo.py', 0) + >>> case = DocTestCase(test) + >>> try: + ... case.debug() + ... except UnexpectedException as f: + ... failure = f + + The UnexpectedException contains the test, the example, and + the original exception: + + >>> failure.test is test + True + + >>> failure.example.want + '42\n' + + >>> exc_info = failure.exc_info + >>> raise exc_info[1] # Already has the traceback + Traceback (most recent call last): + ... + KeyError + + If the output doesn't match, then a DocTestFailure is raised: + + >>> test = DocTestParser().get_doctest(''' + ... >>> x = 1 + ... >>> x + ... 2 + ... ''', {}, 'foo', 'foo.py', 0) + >>> case = DocTestCase(test) + + >>> try: + ... case.debug() + ... except DocTestFailure as f: + ... failure = f + + DocTestFailure objects provide access to the test: + + >>> failure.test is test + True + + As well as to the example: + + >>> failure.example.want + '2\n' + + and the actual output: + + >>> failure.got + '1\n' + + """ + + self.setUp() + runner = DebugRunner(optionflags=self._dt_optionflags, + checker=self._dt_checker, verbose=False) + runner.run(self._dt_test, clear_globs=False) + self.tearDown() + + def id(self): + return self._dt_test.name + + def __eq__(self, other): + if type(self) is not type(other): + return NotImplemented + + return self._dt_test == other._dt_test and \ + self._dt_optionflags == other._dt_optionflags and \ + self._dt_setUp == other._dt_setUp and \ + self._dt_tearDown == other._dt_tearDown and \ + self._dt_checker == other._dt_checker + + def __hash__(self): + return hash((self._dt_optionflags, self._dt_setUp, self._dt_tearDown, + self._dt_checker)) + + def __repr__(self): + name = self._dt_test.name.split('.') + return "%s (%s)" % (name[-1], '.'.join(name[:-1])) + + __str__ = __repr__ + + def shortDescription(self): + return "Doctest: " + self._dt_test.name + +class SkipDocTestCase(DocTestCase): + def __init__(self, module): + self.module = module + DocTestCase.__init__(self, None) + + def setUp(self): + self.skipTest("DocTestSuite will not work with -O2 and above") + + def test_skip(self): + pass + + def shortDescription(self): + return "Skipping tests from %s" % self.module.__name__ + + __str__ = shortDescription + + +class _DocTestSuite(unittest.TestSuite): + + def _removeTestAtIndex(self, index): + pass + + +def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, + **options): + """ + Convert doctest tests for a module to a unittest test suite. + + This converts each documentation string in a module that + contains doctest tests to a unittest test case. If any of the + tests in a doc string fail, then the test case fails. An exception + is raised showing the name of the file containing the test and a + (sometimes approximate) line number. + + The `module` argument provides the module to be tested. The argument + can be either a module or a module name. + + If no argument is given, the calling module is used. + + A number of options may be provided as keyword arguments: + + setUp + A set-up function. This is called before running the + tests in each file. The setUp function will be passed a DocTest + object. The setUp function can access the test globals as the + globs attribute of the test passed. + + tearDown + A tear-down function. This is called after running the + tests in each file. The tearDown function will be passed a DocTest + object. The tearDown function can access the test globals as the + globs attribute of the test passed. + + globs + A dictionary containing initial global variables for the tests. + + optionflags + A set of doctest option flags expressed as an integer. + """ + + if test_finder is None: + test_finder = DocTestFinder() + + module = _normalize_module(module) + tests = test_finder.find(module, globs=globs, extraglobs=extraglobs) + + if not tests and sys.flags.optimize >=2: + # Skip doctests when running with -O2 + suite = _DocTestSuite() + suite.addTest(SkipDocTestCase(module)) + return suite + + tests.sort() + suite = _DocTestSuite() + + for test in tests: + if len(test.examples) == 0: + continue + if not test.filename: + filename = module.__file__ + if filename[-4:] == ".pyc": + filename = filename[:-1] + test.filename = filename + suite.addTest(DocTestCase(test, **options)) + + return suite + +class DocFileCase(DocTestCase): + + def id(self): + return '_'.join(self._dt_test.name.split('.')) + + def __repr__(self): + return self._dt_test.filename + __str__ = __repr__ + + def format_failure(self, err): + return ('Failed doctest test for %s\n File "%s", line 0\n\n%s' + % (self._dt_test.name, self._dt_test.filename, err) + ) + +def DocFileTest(path, module_relative=True, package=None, + globs=None, parser=DocTestParser(), + encoding=None, **options): + if globs is None: + globs = {} + else: + globs = globs.copy() + + if package and not module_relative: + raise ValueError("Package may only be specified for module-" + "relative paths.") + + # Relativize the path. + doc, path = _load_testfile(path, package, module_relative, + encoding or "utf-8") + + if "__file__" not in globs: + globs["__file__"] = path + + # Find the file and read it. + name = os.path.basename(path) + + # Convert it to a test, and wrap it in a DocFileCase. + test = parser.get_doctest(doc, globs, name, path, 0) + return DocFileCase(test, **options) + +def DocFileSuite(*paths, **kw): + """A unittest suite for one or more doctest files. + + The path to each doctest file is given as a string; the + interpretation of that string depends on the keyword argument + "module_relative". + + A number of options may be provided as keyword arguments: + + module_relative + If "module_relative" is True, then the given file paths are + interpreted as os-independent module-relative paths. By + default, these paths are relative to the calling module's + directory; but if the "package" argument is specified, then + they are relative to that package. To ensure os-independence, + "filename" should use "/" characters to separate path + segments, and may not be an absolute path (i.e., it may not + begin with "/"). + + If "module_relative" is False, then the given file paths are + interpreted as os-specific paths. These paths may be absolute + or relative (to the current working directory). + + package + A Python package or the name of a Python package whose directory + should be used as the base directory for module relative paths. + If "package" is not specified, then the calling module's + directory is used as the base directory for module relative + filenames. It is an error to specify "package" if + "module_relative" is False. + + setUp + A set-up function. This is called before running the + tests in each file. The setUp function will be passed a DocTest + object. The setUp function can access the test globals as the + globs attribute of the test passed. + + tearDown + A tear-down function. This is called after running the + tests in each file. The tearDown function will be passed a DocTest + object. The tearDown function can access the test globals as the + globs attribute of the test passed. + + globs + A dictionary containing initial global variables for the tests. + + optionflags + A set of doctest option flags expressed as an integer. + + parser + A DocTestParser (or subclass) that should be used to extract + tests from the files. + + encoding + An encoding that will be used to convert the files to unicode. + """ + suite = _DocTestSuite() + + # We do this here so that _normalize_module is called at the right + # level. If it were called in DocFileTest, then this function + # would be the caller and we might guess the package incorrectly. + if kw.get('module_relative', True): + kw['package'] = _normalize_module(kw.get('package')) + + for path in paths: + suite.addTest(DocFileTest(path, **kw)) + + return suite + +###################################################################### +## 8. Debugging Support +###################################################################### + +def script_from_examples(s): + r"""Extract script from text with examples. + + Converts text with examples to a Python script. Example input is + converted to regular code. Example output and all other words + are converted to comments: + + >>> text = ''' + ... Here are examples of simple math. + ... + ... Python has super accurate integer addition + ... + ... >>> 2 + 2 + ... 5 + ... + ... And very friendly error messages: + ... + ... >>> 1/0 + ... To Infinity + ... And + ... Beyond + ... + ... You can use logic if you want: + ... + ... >>> if 0: + ... ... blah + ... ... blah + ... ... + ... + ... Ho hum + ... ''' + + >>> print(script_from_examples(text)) + # Here are examples of simple math. + # + # Python has super accurate integer addition + # + 2 + 2 + # Expected: + ## 5 + # + # And very friendly error messages: + # + 1/0 + # Expected: + ## To Infinity + ## And + ## Beyond + # + # You can use logic if you want: + # + if 0: + blah + blah + # + # Ho hum + + """ + output = [] + for piece in DocTestParser().parse(s): + if isinstance(piece, Example): + # Add the example's source code (strip trailing NL) + output.append(piece.source[:-1]) + # Add the expected output: + want = piece.want + if want: + output.append('# Expected:') + output += ['## '+l for l in want.split('\n')[:-1]] + else: + # Add non-example text. + output += [_comment_line(l) + for l in piece.split('\n')[:-1]] + + # Trim junk on both ends. + while output and output[-1] == '#': + output.pop() + while output and output[0] == '#': + output.pop(0) + # Combine the output, and return it. + # Add a courtesy newline to prevent exec from choking (see bug #1172785) + return '\n'.join(output) + '\n' + +def testsource(module, name): + """Extract the test sources from a doctest docstring as a script. + + Provide the module (or dotted name of the module) containing the + test to be debugged and the name (within the module) of the object + with the doc string with tests to be debugged. + """ + module = _normalize_module(module) + tests = DocTestFinder().find(module) + test = [t for t in tests if t.name == name] + if not test: + raise ValueError(name, "not found in tests") + test = test[0] + testsrc = script_from_examples(test.docstring) + return testsrc + +def debug_src(src, pm=False, globs=None): + """Debug a single doctest docstring, in argument `src`'""" + testsrc = script_from_examples(src) + debug_script(testsrc, pm, globs) + +def debug_script(src, pm=False, globs=None): + "Debug a test script. `src` is the script, as a string." + import pdb + + if globs: + globs = globs.copy() + else: + globs = {} + + if pm: + try: + exec(src, globs, globs) + except: + print(sys.exc_info()[1]) + p = pdb.Pdb(nosigint=True) + p.reset() + p.interaction(None, sys.exc_info()[2]) + else: + pdb.Pdb(nosigint=True).run("exec(%r)" % src, globs, globs) + +def debug(module, name, pm=False): + """Debug a single doctest docstring. + + Provide the module (or dotted name of the module) containing the + test to be debugged and the name (within the module) of the object + with the docstring with tests to be debugged. + """ + module = _normalize_module(module) + testsrc = testsource(module, name) + debug_script(testsrc, pm, module.__dict__) + +###################################################################### +## 9. Example Usage +###################################################################### +class _TestClass: + """ + A pointless class, for sanity-checking of docstring testing. + + Methods: + square() + get() + + >>> _TestClass(13).get() + _TestClass(-12).get() + 1 + >>> hex(_TestClass(13).square().get()) + '0xa9' + """ + + def __init__(self, val): + """val -> _TestClass object with associated value val. + + >>> t = _TestClass(123) + >>> print(t.get()) + 123 + """ + + self.val = val + + def square(self): + """square() -> square TestClass's associated value + + >>> _TestClass(13).square().get() + 169 + """ + + self.val = self.val ** 2 + return self + + def get(self): + """get() -> return TestClass's associated value. + + >>> x = _TestClass(-42) + >>> print(x.get()) + -42 + """ + + return self.val + +__test__ = {"_TestClass": _TestClass, + "string": r""" + Example of a string object, searched as-is. + >>> x = 1; y = 2 + >>> x + y, x * y + (3, 2) + """, + + "bool-int equivalence": r""" + In 2.2, boolean expressions displayed + 0 or 1. By default, we still accept + them. This can be disabled by passing + DONT_ACCEPT_TRUE_FOR_1 to the new + optionflags argument. + >>> 4 == 4 + 1 + >>> 4 == 4 + True + >>> 4 > 4 + 0 + >>> 4 > 4 + False + """, + + "blank lines": r""" + Blank lines can be marked with : + >>> print('foo\n\nbar\n') + foo + + bar + + """, + + "ellipsis": r""" + If the ellipsis flag is used, then '...' can be used to + elide substrings in the desired output: + >>> print(list(range(1000))) #doctest: +ELLIPSIS + [0, 1, 2, ..., 999] + """, + + "whitespace normalization": r""" + If the whitespace normalization flag is used, then + differences in whitespace are ignored. + >>> print(list(range(30))) #doctest: +NORMALIZE_WHITESPACE + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29] + """, + } + + +def _test(): + parser = argparse.ArgumentParser(description="doctest runner") + parser.add_argument('-v', '--verbose', action='store_true', default=False, + help='print very verbose output for all tests') + parser.add_argument('-o', '--option', action='append', + choices=OPTIONFLAGS_BY_NAME.keys(), default=[], + help=('specify a doctest option flag to apply' + ' to the test run; may be specified more' + ' than once to apply multiple options')) + parser.add_argument('-f', '--fail-fast', action='store_true', + help=('stop running tests after first failure (this' + ' is a shorthand for -o FAIL_FAST, and is' + ' in addition to any other -o options)')) + parser.add_argument('file', nargs='+', + help='file containing the tests to run') + args = parser.parse_args() + testfiles = args.file + # Verbose used to be handled by the "inspect argv" magic in DocTestRunner, + # but since we are using argparse we are passing it manually now. + verbose = args.verbose + options = 0 + for option in args.option: + options |= OPTIONFLAGS_BY_NAME[option] + if args.fail_fast: + options |= FAIL_FAST + for filename in testfiles: + if filename.endswith(".py"): + # It is a module -- insert its dir into sys.path and try to + # import it. If it is part of a package, that possibly + # won't work because of package imports. + dirname, filename = os.path.split(filename) + sys.path.insert(0, dirname) + m = __import__(filename[:-3]) + del sys.path[0] + failures, _ = testmod(m, verbose=verbose, optionflags=options) + else: + failures, _ = testfile(filename, module_relative=False, + verbose=verbose, optionflags=options) + if failures: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(_test()) +"""Faux ``threading`` version using ``dummy_thread`` instead of ``thread``. + +The module ``_dummy_threading`` is added to ``sys.modules`` in order +to not have ``threading`` considered imported. Had ``threading`` been +directly imported it would have made all subsequent imports succeed +regardless of whether ``_thread`` was available which is not desired. + +""" +from sys import modules as sys_modules + +import _dummy_thread + +# Declaring now so as to not have to nest ``try``s to get proper clean-up. +holding_thread = False +holding_threading = False +holding__threading_local = False + +try: + # Could have checked if ``_thread`` was not in sys.modules and gone + # a different route, but decided to mirror technique used with + # ``threading`` below. + if '_thread' in sys_modules: + held_thread = sys_modules['_thread'] + holding_thread = True + # Must have some module named ``_thread`` that implements its API + # in order to initially import ``threading``. + sys_modules['_thread'] = sys_modules['_dummy_thread'] + + if 'threading' in sys_modules: + # If ``threading`` is already imported, might as well prevent + # trying to import it more than needed by saving it if it is + # already imported before deleting it. + held_threading = sys_modules['threading'] + holding_threading = True + del sys_modules['threading'] + + if '_threading_local' in sys_modules: + # If ``_threading_local`` is already imported, might as well prevent + # trying to import it more than needed by saving it if it is + # already imported before deleting it. + held__threading_local = sys_modules['_threading_local'] + holding__threading_local = True + del sys_modules['_threading_local'] + + import threading + # Need a copy of the code kept somewhere... + sys_modules['_dummy_threading'] = sys_modules['threading'] + del sys_modules['threading'] + sys_modules['_dummy__threading_local'] = sys_modules['_threading_local'] + del sys_modules['_threading_local'] + from _dummy_threading import * + from _dummy_threading import __all__ + +finally: + # Put back ``threading`` if we overwrote earlier + + if holding_threading: + sys_modules['threading'] = held_threading + del held_threading + del holding_threading + + # Put back ``_threading_local`` if we overwrote earlier + + if holding__threading_local: + sys_modules['_threading_local'] = held__threading_local + del held__threading_local + del holding__threading_local + + # Put back ``thread`` if we overwrote, else del the entry we made + if holding_thread: + sys_modules['_thread'] = held_thread + del held_thread + else: + del sys_modules['_thread'] + del holding_thread + + del _dummy_thread + del sys_modules +r"""UUID objects (universally unique identifiers) according to RFC 4122. + +This module provides immutable UUID objects (class UUID) and the functions +uuid1(), uuid3(), uuid4(), uuid5() for generating version 1, 3, 4, and 5 +UUIDs as specified in RFC 4122. + +If all you want is a unique ID, you should probably call uuid1() or uuid4(). +Note that uuid1() may compromise privacy since it creates a UUID containing +the computer's network address. uuid4() creates a random UUID. + +Typical usage: + + >>> import uuid + + # make a UUID based on the host ID and current time + >>> uuid.uuid1() # doctest: +SKIP + UUID('a8098c1a-f86e-11da-bd1a-00112444be1e') + + # make a UUID using an MD5 hash of a namespace UUID and a name + >>> uuid.uuid3(uuid.NAMESPACE_DNS, 'python.org') + UUID('6fa459ea-ee8a-3ca4-894e-db77e160355e') + + # make a random UUID + >>> uuid.uuid4() # doctest: +SKIP + UUID('16fd2706-8baf-433b-82eb-8c7fada847da') + + # make a UUID using a SHA-1 hash of a namespace UUID and a name + >>> uuid.uuid5(uuid.NAMESPACE_DNS, 'python.org') + UUID('886313e1-3b8a-5372-9b90-0c9aee199e5d') + + # make a UUID from a string of hex digits (braces and hyphens ignored) + >>> x = uuid.UUID('{00010203-0405-0607-0809-0a0b0c0d0e0f}') + + # convert a UUID to a string of hex digits in standard form + >>> str(x) + '00010203-0405-0607-0809-0a0b0c0d0e0f' + + # get the raw 16 bytes of the UUID + >>> x.bytes + b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f' + + # make a UUID from a 16-byte string + >>> uuid.UUID(bytes=x.bytes) + UUID('00010203-0405-0607-0809-0a0b0c0d0e0f') +""" + +import os + +__author__ = 'Ka-Ping Yee ' + +RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, RESERVED_FUTURE = [ + 'reserved for NCS compatibility', 'specified in RFC 4122', + 'reserved for Microsoft compatibility', 'reserved for future definition'] + +int_ = int # The built-in int type +bytes_ = bytes # The built-in bytes type + +class UUID(object): + """Instances of the UUID class represent UUIDs as specified in RFC 4122. + UUID objects are immutable, hashable, and usable as dictionary keys. + Converting a UUID to a string with str() yields something in the form + '12345678-1234-1234-1234-123456789abc'. The UUID constructor accepts + five possible forms: a similar string of hexadecimal digits, or a tuple + of six integer fields (with 32-bit, 16-bit, 16-bit, 8-bit, 8-bit, and + 48-bit values respectively) as an argument named 'fields', or a string + of 16 bytes (with all the integer fields in big-endian order) as an + argument named 'bytes', or a string of 16 bytes (with the first three + fields in little-endian order) as an argument named 'bytes_le', or a + single 128-bit integer as an argument named 'int'. + + UUIDs have these read-only attributes: + + bytes the UUID as a 16-byte string (containing the six + integer fields in big-endian byte order) + + bytes_le the UUID as a 16-byte string (with time_low, time_mid, + and time_hi_version in little-endian byte order) + + fields a tuple of the six integer fields of the UUID, + which are also available as six individual attributes + and two derived attributes: + + time_low the first 32 bits of the UUID + time_mid the next 16 bits of the UUID + time_hi_version the next 16 bits of the UUID + clock_seq_hi_variant the next 8 bits of the UUID + clock_seq_low the next 8 bits of the UUID + node the last 48 bits of the UUID + + time the 60-bit timestamp + clock_seq the 14-bit sequence number + + hex the UUID as a 32-character hexadecimal string + + int the UUID as a 128-bit integer + + urn the UUID as a URN as specified in RFC 4122 + + variant the UUID variant (one of the constants RESERVED_NCS, + RFC_4122, RESERVED_MICROSOFT, or RESERVED_FUTURE) + + version the UUID version number (1 through 5, meaningful only + when the variant is RFC_4122) + """ + + def __init__(self, hex=None, bytes=None, bytes_le=None, fields=None, + int=None, version=None): + r"""Create a UUID from either a string of 32 hexadecimal digits, + a string of 16 bytes as the 'bytes' argument, a string of 16 bytes + in little-endian order as the 'bytes_le' argument, a tuple of six + integers (32-bit time_low, 16-bit time_mid, 16-bit time_hi_version, + 8-bit clock_seq_hi_variant, 8-bit clock_seq_low, 48-bit node) as + the 'fields' argument, or a single 128-bit integer as the 'int' + argument. When a string of hex digits is given, curly braces, + hyphens, and a URN prefix are all optional. For example, these + expressions all yield the same UUID: + + UUID('{12345678-1234-5678-1234-567812345678}') + UUID('12345678123456781234567812345678') + UUID('urn:uuid:12345678-1234-5678-1234-567812345678') + UUID(bytes='\x12\x34\x56\x78'*4) + UUID(bytes_le='\x78\x56\x34\x12\x34\x12\x78\x56' + + '\x12\x34\x56\x78\x12\x34\x56\x78') + UUID(fields=(0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678)) + UUID(int=0x12345678123456781234567812345678) + + Exactly one of 'hex', 'bytes', 'bytes_le', 'fields', or 'int' must + be given. The 'version' argument is optional; if given, the resulting + UUID will have its variant and version set according to RFC 4122, + overriding the given 'hex', 'bytes', 'bytes_le', 'fields', or 'int'. + """ + + if [hex, bytes, bytes_le, fields, int].count(None) != 4: + raise TypeError('one of the hex, bytes, bytes_le, fields, ' + 'or int arguments must be given') + if hex is not None: + hex = hex.replace('urn:', '').replace('uuid:', '') + hex = hex.strip('{}').replace('-', '') + if len(hex) != 32: + raise ValueError('badly formed hexadecimal UUID string') + int = int_(hex, 16) + if bytes_le is not None: + if len(bytes_le) != 16: + raise ValueError('bytes_le is not a 16-char string') + bytes = (bytes_le[4-1::-1] + bytes_le[6-1:4-1:-1] + + bytes_le[8-1:6-1:-1] + bytes_le[8:]) + if bytes is not None: + if len(bytes) != 16: + raise ValueError('bytes is not a 16-char string') + assert isinstance(bytes, bytes_), repr(bytes) + int = int_.from_bytes(bytes, byteorder='big') + if fields is not None: + if len(fields) != 6: + raise ValueError('fields is not a 6-tuple') + (time_low, time_mid, time_hi_version, + clock_seq_hi_variant, clock_seq_low, node) = fields + if not 0 <= time_low < 1<<32: + raise ValueError('field 1 out of range (need a 32-bit value)') + if not 0 <= time_mid < 1<<16: + raise ValueError('field 2 out of range (need a 16-bit value)') + if not 0 <= time_hi_version < 1<<16: + raise ValueError('field 3 out of range (need a 16-bit value)') + if not 0 <= clock_seq_hi_variant < 1<<8: + raise ValueError('field 4 out of range (need an 8-bit value)') + if not 0 <= clock_seq_low < 1<<8: + raise ValueError('field 5 out of range (need an 8-bit value)') + if not 0 <= node < 1<<48: + raise ValueError('field 6 out of range (need a 48-bit value)') + clock_seq = (clock_seq_hi_variant << 8) | clock_seq_low + int = ((time_low << 96) | (time_mid << 80) | + (time_hi_version << 64) | (clock_seq << 48) | node) + if int is not None: + if not 0 <= int < 1<<128: + raise ValueError('int is out of range (need a 128-bit value)') + if version is not None: + if not 1 <= version <= 5: + raise ValueError('illegal version number') + # Set the variant to RFC 4122. + int &= ~(0xc000 << 48) + int |= 0x8000 << 48 + # Set the version number. + int &= ~(0xf000 << 64) + int |= version << 76 + self.__dict__['int'] = int + + def __eq__(self, other): + if isinstance(other, UUID): + return self.int == other.int + return NotImplemented + + # Q. What's the value of being able to sort UUIDs? + # A. Use them as keys in a B-Tree or similar mapping. + + def __lt__(self, other): + if isinstance(other, UUID): + return self.int < other.int + return NotImplemented + + def __gt__(self, other): + if isinstance(other, UUID): + return self.int > other.int + return NotImplemented + + def __le__(self, other): + if isinstance(other, UUID): + return self.int <= other.int + return NotImplemented + + def __ge__(self, other): + if isinstance(other, UUID): + return self.int >= other.int + return NotImplemented + + def __hash__(self): + return hash(self.int) + + def __int__(self): + return self.int + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, str(self)) + + def __setattr__(self, name, value): + raise TypeError('UUID objects are immutable') + + def __str__(self): + hex = '%032x' % self.int + return '%s-%s-%s-%s-%s' % ( + hex[:8], hex[8:12], hex[12:16], hex[16:20], hex[20:]) + + @property + def bytes(self): + return self.int.to_bytes(16, 'big') + + @property + def bytes_le(self): + bytes = self.bytes + return (bytes[4-1::-1] + bytes[6-1:4-1:-1] + bytes[8-1:6-1:-1] + + bytes[8:]) + + @property + def fields(self): + return (self.time_low, self.time_mid, self.time_hi_version, + self.clock_seq_hi_variant, self.clock_seq_low, self.node) + + @property + def time_low(self): + return self.int >> 96 + + @property + def time_mid(self): + return (self.int >> 80) & 0xffff + + @property + def time_hi_version(self): + return (self.int >> 64) & 0xffff + + @property + def clock_seq_hi_variant(self): + return (self.int >> 56) & 0xff + + @property + def clock_seq_low(self): + return (self.int >> 48) & 0xff + + @property + def time(self): + return (((self.time_hi_version & 0x0fff) << 48) | + (self.time_mid << 32) | self.time_low) + + @property + def clock_seq(self): + return (((self.clock_seq_hi_variant & 0x3f) << 8) | + self.clock_seq_low) + + @property + def node(self): + return self.int & 0xffffffffffff + + @property + def hex(self): + return '%032x' % self.int + + @property + def urn(self): + return 'urn:uuid:' + str(self) + + @property + def variant(self): + if not self.int & (0x8000 << 48): + return RESERVED_NCS + elif not self.int & (0x4000 << 48): + return RFC_4122 + elif not self.int & (0x2000 << 48): + return RESERVED_MICROSOFT + else: + return RESERVED_FUTURE + + @property + def version(self): + # The version bits are only meaningful for RFC 4122 UUIDs. + if self.variant == RFC_4122: + return int((self.int >> 76) & 0xf) + +def _popen(command, *args): + import os, shutil, subprocess + executable = shutil.which(command) + if executable is None: + path = os.pathsep.join(('/sbin', '/usr/sbin')) + executable = shutil.which(command, path=path) + if executable is None: + return None + # LC_ALL=C to ensure English output, stderr=DEVNULL to prevent output + # on stderr (Note: we don't have an example where the words we search + # for are actually localized, but in theory some system could do so.) + env = dict(os.environ) + env['LC_ALL'] = 'C' + proc = subprocess.Popen((executable,) + args, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + env=env) + return proc + +def _find_mac(command, args, hw_identifiers, get_index): + try: + proc = _popen(command, *args.split()) + if not proc: + return + with proc: + for line in proc.stdout: + words = line.lower().rstrip().split() + for i in range(len(words)): + if words[i] in hw_identifiers: + try: + word = words[get_index(i)] + mac = int(word.replace(b':', b''), 16) + if mac: + return mac + except (ValueError, IndexError): + # Virtual interfaces, such as those provided by + # VPNs, do not have a colon-delimited MAC address + # as expected, but a 16-byte HWAddr separated by + # dashes. These should be ignored in favor of a + # real MAC address + pass + except OSError: + pass + +def _ifconfig_getnode(): + """Get the hardware address on Unix by running ifconfig.""" + # This works on Linux ('' or '-a'), Tru64 ('-av'), but not all Unixes. + for args in ('', '-a', '-av'): + mac = _find_mac('ifconfig', args, [b'hwaddr', b'ether'], lambda i: i+1) + if mac: + return mac + +def _ip_getnode(): + """Get the hardware address on Unix by running ip.""" + # This works on Linux with iproute2. + mac = _find_mac('ip', 'link list', [b'link/ether'], lambda i: i+1) + if mac: + return mac + +def _arp_getnode(): + """Get the hardware address on Unix by running arp.""" + import os, socket + try: + ip_addr = socket.gethostbyname(socket.gethostname()) + except OSError: + return None + + # Try getting the MAC addr from arp based on our IP address (Solaris). + return _find_mac('arp', '-an', [os.fsencode(ip_addr)], lambda i: -1) + +def _lanscan_getnode(): + """Get the hardware address on Unix by running lanscan.""" + # This might work on HP-UX. + return _find_mac('lanscan', '-ai', [b'lan0'], lambda i: 0) + +def _netstat_getnode(): + """Get the hardware address on Unix by running netstat.""" + # This might work on AIX, Tru64 UNIX and presumably on IRIX. + try: + proc = _popen('netstat', '-ia') + if not proc: + return + with proc: + words = proc.stdout.readline().rstrip().split() + try: + i = words.index(b'Address') + except ValueError: + return + for line in proc.stdout: + try: + words = line.rstrip().split() + word = words[i] + if len(word) == 17 and word.count(b':') == 5: + mac = int(word.replace(b':', b''), 16) + if mac: + return mac + except (ValueError, IndexError): + pass + except OSError: + pass + +def _ipconfig_getnode(): + """Get the hardware address on Windows by running ipconfig.exe.""" + import os, re + dirs = ['', r'c:\windows\system32', r'c:\winnt\system32'] + try: + import ctypes + buffer = ctypes.create_string_buffer(300) + ctypes.windll.kernel32.GetSystemDirectoryA(buffer, 300) + dirs.insert(0, buffer.value.decode('mbcs')) + except: + pass + for dir in dirs: + try: + pipe = os.popen(os.path.join(dir, 'ipconfig') + ' /all') + except OSError: + continue + with pipe: + for line in pipe: + value = line.split(':')[-1].strip().lower() + if re.match('([0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): + return int(value.replace('-', ''), 16) + +def _netbios_getnode(): + """Get the hardware address on Windows using NetBIOS calls. + See http://support.microsoft.com/kb/118623 for details.""" + import win32wnet, netbios + ncb = netbios.NCB() + ncb.Command = netbios.NCBENUM + ncb.Buffer = adapters = netbios.LANA_ENUM() + adapters._pack() + if win32wnet.Netbios(ncb) != 0: + return + adapters._unpack() + for i in range(adapters.length): + ncb.Reset() + ncb.Command = netbios.NCBRESET + ncb.Lana_num = ord(adapters.lana[i]) + if win32wnet.Netbios(ncb) != 0: + continue + ncb.Reset() + ncb.Command = netbios.NCBASTAT + ncb.Lana_num = ord(adapters.lana[i]) + ncb.Callname = '*'.ljust(16) + ncb.Buffer = status = netbios.ADAPTER_STATUS() + if win32wnet.Netbios(ncb) != 0: + continue + status._unpack() + bytes = status.adapter_address[:6] + if len(bytes) != 6: + continue + return int.from_bytes(bytes, 'big') + +# Thanks to Thomas Heller for ctypes and for his help with its use here. + +# If ctypes is available, use it to find system routines for UUID generation. +# XXX This makes the module non-thread-safe! +_uuid_generate_time = _UuidCreate = None +try: + import ctypes, ctypes.util + import sys + + # The uuid_generate_* routines are provided by libuuid on at least + # Linux and FreeBSD, and provided by libc on Mac OS X. + _libnames = ['uuid'] + if not sys.platform.startswith('win'): + _libnames.append('c') + for libname in _libnames: + try: + lib = ctypes.CDLL(ctypes.util.find_library(libname)) + except Exception: + continue + if hasattr(lib, 'uuid_generate_time'): + _uuid_generate_time = lib.uuid_generate_time + break + del _libnames + + # The uuid_generate_* functions are broken on MacOS X 10.5, as noted + # in issue #8621 the function generates the same sequence of values + # in the parent process and all children created using fork (unless + # those children use exec as well). + # + # Assume that the uuid_generate functions are broken from 10.5 onward, + # the test can be adjusted when a later version is fixed. + if sys.platform == 'darwin': + import os + if int(os.uname().release.split('.')[0]) >= 9: + _uuid_generate_time = None + + # On Windows prior to 2000, UuidCreate gives a UUID containing the + # hardware address. On Windows 2000 and later, UuidCreate makes a + # random UUID and UuidCreateSequential gives a UUID containing the + # hardware address. These routines are provided by the RPC runtime. + # NOTE: at least on Tim's WinXP Pro SP2 desktop box, while the last + # 6 bytes returned by UuidCreateSequential are fixed, they don't appear + # to bear any relationship to the MAC address of any network device + # on the box. + try: + lib = ctypes.windll.rpcrt4 + except: + lib = None + _UuidCreate = getattr(lib, 'UuidCreateSequential', + getattr(lib, 'UuidCreate', None)) +except: + pass + +def _unixdll_getnode(): + """Get the hardware address on Unix using ctypes.""" + _buffer = ctypes.create_string_buffer(16) + _uuid_generate_time(_buffer) + return UUID(bytes=bytes_(_buffer.raw)).node + +def _windll_getnode(): + """Get the hardware address on Windows using ctypes.""" + _buffer = ctypes.create_string_buffer(16) + if _UuidCreate(_buffer) == 0: + return UUID(bytes=bytes_(_buffer.raw)).node + +def _random_getnode(): + """Get a random node ID, with eighth bit set as suggested by RFC 4122.""" + import random + return random.getrandbits(48) | 0x010000000000 + +_node = None + +def getnode(): + """Get the hardware address as a 48-bit positive integer. + + The first time this runs, it may launch a separate program, which could + be quite slow. If all attempts to obtain the hardware address fail, we + choose a random 48-bit number with its eighth bit set to 1 as recommended + in RFC 4122. + """ + + global _node + if _node is not None: + return _node + + import sys + if sys.platform == 'win32': + getters = [_windll_getnode, _netbios_getnode, _ipconfig_getnode] + else: + getters = [_unixdll_getnode, _ifconfig_getnode, _ip_getnode, + _arp_getnode, _lanscan_getnode, _netstat_getnode] + + for getter in getters + [_random_getnode]: + try: + _node = getter() + except: + continue + if _node is not None: + return _node + +_last_timestamp = None + +def uuid1(node=None, clock_seq=None): + """Generate a UUID from a host ID, sequence number, and the current time. + If 'node' is not given, getnode() is used to obtain the hardware + address. If 'clock_seq' is given, it is used as the sequence number; + otherwise a random 14-bit sequence number is chosen.""" + + # When the system provides a version-1 UUID generator, use it (but don't + # use UuidCreate here because its UUIDs don't conform to RFC 4122). + if _uuid_generate_time and node is clock_seq is None: + _buffer = ctypes.create_string_buffer(16) + _uuid_generate_time(_buffer) + return UUID(bytes=bytes_(_buffer.raw)) + + global _last_timestamp + import time + nanoseconds = int(time.time() * 1e9) + # 0x01b21dd213814000 is the number of 100-ns intervals between the + # UUID epoch 1582-10-15 00:00:00 and the Unix epoch 1970-01-01 00:00:00. + timestamp = int(nanoseconds/100) + 0x01b21dd213814000 + if _last_timestamp is not None and timestamp <= _last_timestamp: + timestamp = _last_timestamp + 1 + _last_timestamp = timestamp + if clock_seq is None: + import random + clock_seq = random.getrandbits(14) # instead of stable storage + time_low = timestamp & 0xffffffff + time_mid = (timestamp >> 32) & 0xffff + time_hi_version = (timestamp >> 48) & 0x0fff + clock_seq_low = clock_seq & 0xff + clock_seq_hi_variant = (clock_seq >> 8) & 0x3f + if node is None: + node = getnode() + return UUID(fields=(time_low, time_mid, time_hi_version, + clock_seq_hi_variant, clock_seq_low, node), version=1) + +def uuid3(namespace, name): + """Generate a UUID from the MD5 hash of a namespace UUID and a name.""" + from hashlib import md5 + hash = md5(namespace.bytes + bytes(name, "utf-8")).digest() + return UUID(bytes=hash[:16], version=3) + +def uuid4(): + """Generate a random UUID.""" + return UUID(bytes=os.urandom(16), version=4) + +def uuid5(namespace, name): + """Generate a UUID from the SHA-1 hash of a namespace UUID and a name.""" + from hashlib import sha1 + hash = sha1(namespace.bytes + bytes(name, "utf-8")).digest() + return UUID(bytes=hash[:16], version=5) + +# The following standard UUIDs are for use with uuid3() or uuid5(). + +NAMESPACE_DNS = UUID('6ba7b810-9dad-11d1-80b4-00c04fd430c8') +NAMESPACE_URL = UUID('6ba7b811-9dad-11d1-80b4-00c04fd430c8') +NAMESPACE_OID = UUID('6ba7b812-9dad-11d1-80b4-00c04fd430c8') +NAMESPACE_X500 = UUID('6ba7b814-9dad-11d1-80b4-00c04fd430c8') +"""Weak reference support for Python. + +This module is an implementation of PEP 205: + +http://www.python.org/dev/peps/pep-0205/ +""" + +# Naming convention: Variables named "wr" are weak reference objects; +# they are called this instead of "ref" to avoid name collisions with +# the module-global ref() function imported from _weakref. + +from _weakref import ( + getweakrefcount, + getweakrefs, + ref, + proxy, + CallableProxyType, + ProxyType, + ReferenceType) + +from _weakrefset import WeakSet, _IterationGuard + +import collections # Import after _weakref to avoid circular import. +import sys +import itertools + +ProxyTypes = (ProxyType, CallableProxyType) + +__all__ = ["ref", "proxy", "getweakrefcount", "getweakrefs", + "WeakKeyDictionary", "ReferenceType", "ProxyType", + "CallableProxyType", "ProxyTypes", "WeakValueDictionary", + "WeakSet", "WeakMethod", "finalize"] + + +class WeakMethod(ref): + """ + A custom `weakref.ref` subclass which simulates a weak reference to + a bound method, working around the lifetime problem of bound methods. + """ + + __slots__ = "_func_ref", "_meth_type", "_alive", "__weakref__" + + def __new__(cls, meth, callback=None): + try: + obj = meth.__self__ + func = meth.__func__ + except AttributeError: + raise TypeError("argument should be a bound method, not {}" + .format(type(meth))) from None + def _cb(arg): + # The self-weakref trick is needed to avoid creating a reference + # cycle. + self = self_wr() + if self._alive: + self._alive = False + if callback is not None: + callback(self) + self = ref.__new__(cls, obj, _cb) + self._func_ref = ref(func, _cb) + self._meth_type = type(meth) + self._alive = True + self_wr = ref(self) + return self + + def __call__(self): + obj = super().__call__() + func = self._func_ref() + if obj is None or func is None: + return None + return self._meth_type(func, obj) + + def __eq__(self, other): + if isinstance(other, WeakMethod): + if not self._alive or not other._alive: + return self is other + return ref.__eq__(self, other) and self._func_ref == other._func_ref + return False + + def __ne__(self, other): + if isinstance(other, WeakMethod): + if not self._alive or not other._alive: + return self is not other + return ref.__ne__(self, other) or self._func_ref != other._func_ref + return True + + __hash__ = ref.__hash__ + + +class WeakValueDictionary(collections.MutableMapping): + """Mapping class that references values weakly. + + Entries in the dictionary will be discarded when no strong + reference to the value exists anymore + """ + # We inherit the constructor without worrying about the input + # dictionary; since it uses our .update() method, we get the right + # checks (if the other dictionary is a WeakValueDictionary, + # objects are unwrapped on the way out, and we always wrap on the + # way in). + + def __init__(*args, **kw): + if not args: + raise TypeError("descriptor '__init__' of 'WeakValueDictionary' " + "object needs an argument") + self, *args = args + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + def remove(wr, selfref=ref(self)): + self = selfref() + if self is not None: + if self._iterating: + self._pending_removals.append(wr.key) + else: + del self.data[wr.key] + self._remove = remove + # A list of keys to be removed + self._pending_removals = [] + self._iterating = set() + self.data = d = {} + self.update(*args, **kw) + + def _commit_removals(self): + l = self._pending_removals + d = self.data + # We shouldn't encounter any KeyError, because this method should + # always be called *before* mutating the dict. + while l: + del d[l.pop()] + + def __getitem__(self, key): + o = self.data[key]() + if o is None: + raise KeyError(key) + else: + return o + + def __delitem__(self, key): + if self._pending_removals: + self._commit_removals() + del self.data[key] + + def __len__(self): + return len(self.data) - len(self._pending_removals) + + def __contains__(self, key): + try: + o = self.data[key]() + except KeyError: + return False + return o is not None + + def __repr__(self): + return "<%s at %#x>" % (self.__class__.__name__, id(self)) + + def __setitem__(self, key, value): + if self._pending_removals: + self._commit_removals() + self.data[key] = KeyedRef(value, self._remove, key) + + def copy(self): + new = WeakValueDictionary() + for key, wr in self.data.items(): + o = wr() + if o is not None: + new[key] = o + return new + + __copy__ = copy + + def __deepcopy__(self, memo): + from copy import deepcopy + new = self.__class__() + for key, wr in self.data.items(): + o = wr() + if o is not None: + new[deepcopy(key, memo)] = o + return new + + def get(self, key, default=None): + try: + wr = self.data[key] + except KeyError: + return default + else: + o = wr() + if o is None: + # This should only happen + return default + else: + return o + + def items(self): + with _IterationGuard(self): + for k, wr in self.data.items(): + v = wr() + if v is not None: + yield k, v + + def keys(self): + with _IterationGuard(self): + for k, wr in self.data.items(): + if wr() is not None: + yield k + + __iter__ = keys + + def itervaluerefs(self): + """Return an iterator that yields the weak references to the values. + + The references are not guaranteed to be 'live' at the time + they are used, so the result of calling the references needs + to be checked before being used. This can be used to avoid + creating references that will cause the garbage collector to + keep the values around longer than needed. + + """ + with _IterationGuard(self): + yield from self.data.values() + + def values(self): + with _IterationGuard(self): + for wr in self.data.values(): + obj = wr() + if obj is not None: + yield obj + + def popitem(self): + if self._pending_removals: + self._commit_removals() + while True: + key, wr = self.data.popitem() + o = wr() + if o is not None: + return key, o + + def pop(self, key, *args): + if self._pending_removals: + self._commit_removals() + try: + o = self.data.pop(key)() + except KeyError: + if args: + return args[0] + raise + if o is None: + raise KeyError(key) + else: + return o + + def setdefault(self, key, default=None): + try: + wr = self.data[key] + except KeyError: + if self._pending_removals: + self._commit_removals() + self.data[key] = KeyedRef(default, self._remove, key) + return default + else: + return wr() + + def update(*args, **kwargs): + if not args: + raise TypeError("descriptor 'update' of 'WeakValueDictionary' " + "object needs an argument") + self, *args = args + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + dict = args[0] if args else None + if self._pending_removals: + self._commit_removals() + d = self.data + if dict is not None: + if not hasattr(dict, "items"): + dict = type({})(dict) + for key, o in dict.items(): + d[key] = KeyedRef(o, self._remove, key) + if len(kwargs): + self.update(kwargs) + + def valuerefs(self): + """Return a list of weak references to the values. + + The references are not guaranteed to be 'live' at the time + they are used, so the result of calling the references needs + to be checked before being used. This can be used to avoid + creating references that will cause the garbage collector to + keep the values around longer than needed. + + """ + return list(self.data.values()) + + +class KeyedRef(ref): + """Specialized reference that includes a key corresponding to the value. + + This is used in the WeakValueDictionary to avoid having to create + a function object for each key stored in the mapping. A shared + callback object can use the 'key' attribute of a KeyedRef instead + of getting a reference to the key from an enclosing scope. + + """ + + __slots__ = "key", + + def __new__(type, ob, callback, key): + self = ref.__new__(type, ob, callback) + self.key = key + return self + + def __init__(self, ob, callback, key): + super().__init__(ob, callback) + + +class WeakKeyDictionary(collections.MutableMapping): + """ Mapping class that references keys weakly. + + Entries in the dictionary will be discarded when there is no + longer a strong reference to the key. This can be used to + associate additional data with an object owned by other parts of + an application without adding attributes to those objects. This + can be especially useful with objects that override attribute + accesses. + """ + + def __init__(self, dict=None): + self.data = {} + def remove(k, selfref=ref(self)): + self = selfref() + if self is not None: + if self._iterating: + self._pending_removals.append(k) + else: + del self.data[k] + self._remove = remove + # A list of dead weakrefs (keys to be removed) + self._pending_removals = [] + self._iterating = set() + self._dirty_len = False + if dict is not None: + self.update(dict) + + def _commit_removals(self): + # NOTE: We don't need to call this method before mutating the dict, + # because a dead weakref never compares equal to a live weakref, + # even if they happened to refer to equal objects. + # However, it means keys may already have been removed. + l = self._pending_removals + d = self.data + while l: + try: + del d[l.pop()] + except KeyError: + pass + + def _scrub_removals(self): + d = self.data + self._pending_removals = [k for k in self._pending_removals if k in d] + self._dirty_len = False + + def __delitem__(self, key): + self._dirty_len = True + del self.data[ref(key)] + + def __getitem__(self, key): + return self.data[ref(key)] + + def __len__(self): + if self._dirty_len and self._pending_removals: + # self._pending_removals may still contain keys which were + # explicitly removed, we have to scrub them (see issue #21173). + self._scrub_removals() + return len(self.data) - len(self._pending_removals) + + def __repr__(self): + return "<%s at %#x>" % (self.__class__.__name__, id(self)) + + def __setitem__(self, key, value): + self.data[ref(key, self._remove)] = value + + def copy(self): + new = WeakKeyDictionary() + for key, value in self.data.items(): + o = key() + if o is not None: + new[o] = value + return new + + __copy__ = copy + + def __deepcopy__(self, memo): + from copy import deepcopy + new = self.__class__() + for key, value in self.data.items(): + o = key() + if o is not None: + new[o] = deepcopy(value, memo) + return new + + def get(self, key, default=None): + return self.data.get(ref(key),default) + + def __contains__(self, key): + try: + wr = ref(key) + except TypeError: + return False + return wr in self.data + + def items(self): + with _IterationGuard(self): + for wr, value in self.data.items(): + key = wr() + if key is not None: + yield key, value + + def keys(self): + with _IterationGuard(self): + for wr in self.data: + obj = wr() + if obj is not None: + yield obj + + __iter__ = keys + + def values(self): + with _IterationGuard(self): + for wr, value in self.data.items(): + if wr() is not None: + yield value + + def keyrefs(self): + """Return a list of weak references to the keys. + + The references are not guaranteed to be 'live' at the time + they are used, so the result of calling the references needs + to be checked before being used. This can be used to avoid + creating references that will cause the garbage collector to + keep the keys around longer than needed. + + """ + return list(self.data) + + def popitem(self): + self._dirty_len = True + while True: + key, value = self.data.popitem() + o = key() + if o is not None: + return o, value + + def pop(self, key, *args): + self._dirty_len = True + return self.data.pop(ref(key), *args) + + def setdefault(self, key, default=None): + return self.data.setdefault(ref(key, self._remove),default) + + def update(self, dict=None, **kwargs): + d = self.data + if dict is not None: + if not hasattr(dict, "items"): + dict = type({})(dict) + for key, value in dict.items(): + d[ref(key, self._remove)] = value + if len(kwargs): + self.update(kwargs) + + +class finalize: + """Class for finalization of weakrefable objects + + finalize(obj, func, *args, **kwargs) returns a callable finalizer + object which will be called when obj is garbage collected. The + first time the finalizer is called it evaluates func(*arg, **kwargs) + and returns the result. After this the finalizer is dead, and + calling it just returns None. + + When the program exits any remaining finalizers for which the + atexit attribute is true will be run in reverse order of creation. + By default atexit is true. + """ + + # Finalizer objects don't have any state of their own. They are + # just used as keys to lookup _Info objects in the registry. This + # ensures that they cannot be part of a ref-cycle. + + __slots__ = () + _registry = {} + _shutdown = False + _index_iter = itertools.count() + _dirty = False + _registered_with_atexit = False + + class _Info: + __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index") + + def __init__(self, obj, func, *args, **kwargs): + if not self._registered_with_atexit: + # We may register the exit function more than once because + # of a thread race, but that is harmless + import atexit + atexit.register(self._exitfunc) + finalize._registered_with_atexit = True + info = self._Info() + info.weakref = ref(obj, self) + info.func = func + info.args = args + info.kwargs = kwargs or None + info.atexit = True + info.index = next(self._index_iter) + self._registry[self] = info + finalize._dirty = True + + def __call__(self, _=None): + """If alive then mark as dead and return func(*args, **kwargs); + otherwise return None""" + info = self._registry.pop(self, None) + if info and not self._shutdown: + return info.func(*info.args, **(info.kwargs or {})) + + def detach(self): + """If alive then mark as dead and return (obj, func, args, kwargs); + otherwise return None""" + info = self._registry.get(self) + obj = info and info.weakref() + if obj is not None and self._registry.pop(self, None): + return (obj, info.func, info.args, info.kwargs or {}) + + def peek(self): + """If alive then return (obj, func, args, kwargs); + otherwise return None""" + info = self._registry.get(self) + obj = info and info.weakref() + if obj is not None: + return (obj, info.func, info.args, info.kwargs or {}) + + @property + def alive(self): + """Whether finalizer is alive""" + return self in self._registry + + @property + def atexit(self): + """Whether finalizer should be called at exit""" + info = self._registry.get(self) + return bool(info) and info.atexit + + @atexit.setter + def atexit(self, value): + info = self._registry.get(self) + if info: + info.atexit = bool(value) + + def __repr__(self): + info = self._registry.get(self) + obj = info and info.weakref() + if obj is None: + return '<%s object at %#x; dead>' % (type(self).__name__, id(self)) + else: + return '<%s object at %#x; for %r at %#x>' % \ + (type(self).__name__, id(self), type(obj).__name__, id(obj)) + + @classmethod + def _select_for_exit(cls): + # Return live finalizers marked for exit, oldest first + L = [(f,i) for (f,i) in cls._registry.items() if i.atexit] + L.sort(key=lambda item:item[1].index) + return [f for (f,i) in L] + + @classmethod + def _exitfunc(cls): + # At shutdown invoke finalizers for which atexit is true. + # This is called once all other non-daemonic threads have been + # joined. + reenable_gc = False + try: + if cls._registry: + import gc + if gc.isenabled(): + reenable_gc = True + gc.disable() + pending = None + while True: + if pending is None or finalize._dirty: + pending = cls._select_for_exit() + finalize._dirty = False + if not pending: + break + f = pending.pop() + try: + # gc is disabled, so (assuming no daemonic + # threads) the following is the only line in + # this function which might trigger creation + # of a new finalizer + f() + except Exception: + sys.excepthook(*sys.exc_info()) + assert f not in cls._registry + finally: + # prevent any more finalizers from executing during shutdown + finalize._shutdown = True + if reenable_gc: + gc.enable() +# +# Secret Labs' Regular Expression Engine +# +# various symbols used by the regular expression engine. +# run this script to update the _sre include files! +# +# Copyright (c) 1998-2001 by Secret Labs AB. All rights reserved. +# +# See the sre.py file for information on usage and redistribution. +# + +"""Internal support module for sre""" + +# update when constants are added or removed + +MAGIC = 20140917 + +from _sre import MAXREPEAT, MAXGROUPS + +# SRE standard exception (access as sre.error) +# should this really be here? + +class error(Exception): + def __init__(self, msg, pattern=None, pos=None): + self.msg = msg + self.pattern = pattern + self.pos = pos + if pattern is not None and pos is not None: + msg = '%s at position %d' % (msg, pos) + if isinstance(pattern, str): + newline = '\n' + else: + newline = b'\n' + self.lineno = pattern.count(newline, 0, pos) + 1 + self.colno = pos - pattern.rfind(newline, 0, pos) + if newline in pattern: + msg = '%s (line %d, column %d)' % (msg, self.lineno, self.colno) + else: + self.lineno = self.colno = None + super().__init__(msg) + + +class _NamedIntConstant(int): + def __new__(cls, value, name): + self = super(_NamedIntConstant, cls).__new__(cls, value) + self.name = name + return self + + def __str__(self): + return self.name + + __repr__ = __str__ + +MAXREPEAT = _NamedIntConstant(MAXREPEAT, 'MAXREPEAT') + +def _makecodes(names): + names = names.strip().split() + items = [_NamedIntConstant(i, name) for i, name in enumerate(names)] + globals().update({item.name: item for item in items}) + return items + +# operators +# failure=0 success=1 (just because it looks better that way :-) +OPCODES = _makecodes(""" + FAILURE SUCCESS + + ANY ANY_ALL + ASSERT ASSERT_NOT + AT + BRANCH + CALL + CATEGORY + CHARSET BIGCHARSET + GROUPREF GROUPREF_EXISTS GROUPREF_IGNORE + IN IN_IGNORE + INFO + JUMP + LITERAL LITERAL_IGNORE + MARK + MAX_UNTIL + MIN_UNTIL + NOT_LITERAL NOT_LITERAL_IGNORE + NEGATE + RANGE + REPEAT + REPEAT_ONE + SUBPATTERN + MIN_REPEAT_ONE + RANGE_IGNORE + + MIN_REPEAT MAX_REPEAT +""") +del OPCODES[-2:] # remove MIN_REPEAT and MAX_REPEAT + +# positions +ATCODES = _makecodes(""" + AT_BEGINNING AT_BEGINNING_LINE AT_BEGINNING_STRING + AT_BOUNDARY AT_NON_BOUNDARY + AT_END AT_END_LINE AT_END_STRING + AT_LOC_BOUNDARY AT_LOC_NON_BOUNDARY + AT_UNI_BOUNDARY AT_UNI_NON_BOUNDARY +""") + +# categories +CHCODES = _makecodes(""" + CATEGORY_DIGIT CATEGORY_NOT_DIGIT + CATEGORY_SPACE CATEGORY_NOT_SPACE + CATEGORY_WORD CATEGORY_NOT_WORD + CATEGORY_LINEBREAK CATEGORY_NOT_LINEBREAK + CATEGORY_LOC_WORD CATEGORY_LOC_NOT_WORD + CATEGORY_UNI_DIGIT CATEGORY_UNI_NOT_DIGIT + CATEGORY_UNI_SPACE CATEGORY_UNI_NOT_SPACE + CATEGORY_UNI_WORD CATEGORY_UNI_NOT_WORD + CATEGORY_UNI_LINEBREAK CATEGORY_UNI_NOT_LINEBREAK +""") + + +# replacement operations for "ignore case" mode +OP_IGNORE = { + GROUPREF: GROUPREF_IGNORE, + IN: IN_IGNORE, + LITERAL: LITERAL_IGNORE, + NOT_LITERAL: NOT_LITERAL_IGNORE, + RANGE: RANGE_IGNORE, +} + +AT_MULTILINE = { + AT_BEGINNING: AT_BEGINNING_LINE, + AT_END: AT_END_LINE +} + +AT_LOCALE = { + AT_BOUNDARY: AT_LOC_BOUNDARY, + AT_NON_BOUNDARY: AT_LOC_NON_BOUNDARY +} + +AT_UNICODE = { + AT_BOUNDARY: AT_UNI_BOUNDARY, + AT_NON_BOUNDARY: AT_UNI_NON_BOUNDARY +} + +CH_LOCALE = { + CATEGORY_DIGIT: CATEGORY_DIGIT, + CATEGORY_NOT_DIGIT: CATEGORY_NOT_DIGIT, + CATEGORY_SPACE: CATEGORY_SPACE, + CATEGORY_NOT_SPACE: CATEGORY_NOT_SPACE, + CATEGORY_WORD: CATEGORY_LOC_WORD, + CATEGORY_NOT_WORD: CATEGORY_LOC_NOT_WORD, + CATEGORY_LINEBREAK: CATEGORY_LINEBREAK, + CATEGORY_NOT_LINEBREAK: CATEGORY_NOT_LINEBREAK +} + +CH_UNICODE = { + CATEGORY_DIGIT: CATEGORY_UNI_DIGIT, + CATEGORY_NOT_DIGIT: CATEGORY_UNI_NOT_DIGIT, + CATEGORY_SPACE: CATEGORY_UNI_SPACE, + CATEGORY_NOT_SPACE: CATEGORY_UNI_NOT_SPACE, + CATEGORY_WORD: CATEGORY_UNI_WORD, + CATEGORY_NOT_WORD: CATEGORY_UNI_NOT_WORD, + CATEGORY_LINEBREAK: CATEGORY_UNI_LINEBREAK, + CATEGORY_NOT_LINEBREAK: CATEGORY_UNI_NOT_LINEBREAK +} + +# flags +SRE_FLAG_TEMPLATE = 1 # template mode (disable backtracking) +SRE_FLAG_IGNORECASE = 2 # case insensitive +SRE_FLAG_LOCALE = 4 # honour system locale +SRE_FLAG_MULTILINE = 8 # treat target as multiline string +SRE_FLAG_DOTALL = 16 # treat target as a single string +SRE_FLAG_UNICODE = 32 # use unicode "locale" +SRE_FLAG_VERBOSE = 64 # ignore whitespace and comments +SRE_FLAG_DEBUG = 128 # debugging +SRE_FLAG_ASCII = 256 # use ascii "locale" + +# flags for INFO primitive +SRE_INFO_PREFIX = 1 # has prefix +SRE_INFO_LITERAL = 2 # entire pattern is literal (given by prefix) +SRE_INFO_CHARSET = 4 # pattern starts with character from given set + +if __name__ == "__main__": + def dump(f, d, prefix): + items = sorted(d) + for item in items: + f.write("#define %s_%s %d\n" % (prefix, item, item)) + with open("sre_constants.h", "w") as f: + f.write("""\ +/* + * Secret Labs' Regular Expression Engine + * + * regular expression matching engine + * + * NOTE: This file is generated by sre_constants.py. If you need + * to change anything in here, edit sre_constants.py and run it. + * + * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. + * + * See the _sre.c file for information on usage and redistribution. + */ + +""") + + f.write("#define SRE_MAGIC %d\n" % MAGIC) + + dump(f, OPCODES, "SRE_OP") + dump(f, ATCODES, "SRE") + dump(f, CHCODES, "SRE") + + f.write("#define SRE_FLAG_TEMPLATE %d\n" % SRE_FLAG_TEMPLATE) + f.write("#define SRE_FLAG_IGNORECASE %d\n" % SRE_FLAG_IGNORECASE) + f.write("#define SRE_FLAG_LOCALE %d\n" % SRE_FLAG_LOCALE) + f.write("#define SRE_FLAG_MULTILINE %d\n" % SRE_FLAG_MULTILINE) + f.write("#define SRE_FLAG_DOTALL %d\n" % SRE_FLAG_DOTALL) + f.write("#define SRE_FLAG_UNICODE %d\n" % SRE_FLAG_UNICODE) + f.write("#define SRE_FLAG_VERBOSE %d\n" % SRE_FLAG_VERBOSE) + f.write("#define SRE_FLAG_DEBUG %d\n" % SRE_FLAG_DEBUG) + f.write("#define SRE_FLAG_ASCII %d\n" % SRE_FLAG_ASCII) + + f.write("#define SRE_INFO_PREFIX %d\n" % SRE_INFO_PREFIX) + f.write("#define SRE_INFO_LITERAL %d\n" % SRE_INFO_LITERAL) + f.write("#define SRE_INFO_CHARSET %d\n" % SRE_INFO_CHARSET) + + print("done") +"""Parse a Python module and describe its classes and methods. + +Parse enough of a Python file to recognize imports and class and +method definitions, and to find out the superclasses of a class. + +The interface consists of a single function: + readmodule_ex(module [, path]) +where module is the name of a Python module, and path is an optional +list of directories where the module is to be searched. If present, +path is prepended to the system search path sys.path. The return +value is a dictionary. The keys of the dictionary are the names of +the classes defined in the module (including classes that are defined +via the from XXX import YYY construct). The values are class +instances of the class Class defined here. One special key/value pair +is present for packages: the key '__path__' has a list as its value +which contains the package search path. + +A class is described by the class Class in this module. Instances +of this class have the following instance variables: + module -- the module name + name -- the name of the class + super -- a list of super classes (Class instances) + methods -- a dictionary of methods + file -- the file in which the class was defined + lineno -- the line in the file on which the class statement occurred +The dictionary of methods uses the method names as keys and the line +numbers on which the method was defined as values. +If the name of a super class is not recognized, the corresponding +entry in the list of super classes is not a class instance but a +string giving the name of the super class. Since import statements +are recognized and imported modules are scanned as well, this +shouldn't happen often. + +A function is described by the class Function in this module. +Instances of this class have the following instance variables: + module -- the module name + name -- the name of the class + file -- the file in which the class was defined + lineno -- the line in the file on which the class statement occurred +""" + +import io +import os +import sys +import importlib.util +import tokenize +from token import NAME, DEDENT, OP +from operator import itemgetter + +__all__ = ["readmodule", "readmodule_ex", "Class", "Function"] + +_modules = {} # cache of modules we've seen + +# each Python class is represented by an instance of this class +class Class: + '''Class to represent a Python class.''' + def __init__(self, module, name, super, file, lineno): + self.module = module + self.name = name + if super is None: + super = [] + self.super = super + self.methods = {} + self.file = file + self.lineno = lineno + + def _addmethod(self, name, lineno): + self.methods[name] = lineno + +class Function: + '''Class to represent a top-level Python function''' + def __init__(self, module, name, file, lineno): + self.module = module + self.name = name + self.file = file + self.lineno = lineno + +def readmodule(module, path=None): + '''Backwards compatible interface. + + Call readmodule_ex() and then only keep Class objects from the + resulting dictionary.''' + + res = {} + for key, value in _readmodule(module, path or []).items(): + if isinstance(value, Class): + res[key] = value + return res + +def readmodule_ex(module, path=None): + '''Read a module file and return a dictionary of classes. + + Search for MODULE in PATH and sys.path, read and parse the + module and return a dictionary with one entry for each class + found in the module. + ''' + return _readmodule(module, path or []) + +def _readmodule(module, path, inpackage=None): + '''Do the hard work for readmodule[_ex]. + + If INPACKAGE is given, it must be the dotted name of the package in + which we are searching for a submodule, and then PATH must be the + package search path; otherwise, we are searching for a top-level + module, and PATH is combined with sys.path. + ''' + # Compute the full module name (prepending inpackage if set) + if inpackage is not None: + fullmodule = "%s.%s" % (inpackage, module) + else: + fullmodule = module + + # Check in the cache + if fullmodule in _modules: + return _modules[fullmodule] + + # Initialize the dict for this module's contents + dict = {} + + # Check if it is a built-in module; we don't do much for these + if module in sys.builtin_module_names and inpackage is None: + _modules[module] = dict + return dict + + # Check for a dotted module name + i = module.rfind('.') + if i >= 0: + package = module[:i] + submodule = module[i+1:] + parent = _readmodule(package, path, inpackage) + if inpackage is not None: + package = "%s.%s" % (inpackage, package) + if not '__path__' in parent: + raise ImportError('No package named {}'.format(package)) + return _readmodule(submodule, parent['__path__'], package) + + # Search the path for the module + f = None + if inpackage is not None: + search_path = path + else: + search_path = path + sys.path + # XXX This will change once issue19944 lands. + spec = importlib.util._find_spec_from_path(fullmodule, search_path) + _modules[fullmodule] = dict + # is module a package? + if spec.submodule_search_locations is not None: + dict['__path__'] = spec.submodule_search_locations + try: + source = spec.loader.get_source(fullmodule) + if source is None: + return dict + except (AttributeError, ImportError): + # not Python source, can't do anything with this module + return dict + + fname = spec.loader.get_filename(fullmodule) + + f = io.StringIO(source) + + stack = [] # stack of (class, indent) pairs + + g = tokenize.generate_tokens(f.readline) + try: + for tokentype, token, start, _end, _line in g: + if tokentype == DEDENT: + lineno, thisindent = start + # close nested classes and defs + while stack and stack[-1][1] >= thisindent: + del stack[-1] + elif token == 'def': + lineno, thisindent = start + # close previous nested classes and defs + while stack and stack[-1][1] >= thisindent: + del stack[-1] + tokentype, meth_name, start = next(g)[0:3] + if tokentype != NAME: + continue # Syntax error + if stack: + cur_class = stack[-1][0] + if isinstance(cur_class, Class): + # it's a method + cur_class._addmethod(meth_name, lineno) + # else it's a nested def + else: + # it's a function + dict[meth_name] = Function(fullmodule, meth_name, + fname, lineno) + stack.append((None, thisindent)) # Marker for nested fns + elif token == 'class': + lineno, thisindent = start + # close previous nested classes and defs + while stack and stack[-1][1] >= thisindent: + del stack[-1] + tokentype, class_name, start = next(g)[0:3] + if tokentype != NAME: + continue # Syntax error + # parse what follows the class name + tokentype, token, start = next(g)[0:3] + inherit = None + if token == '(': + names = [] # List of superclasses + # there's a list of superclasses + level = 1 + super = [] # Tokens making up current superclass + while True: + tokentype, token, start = next(g)[0:3] + if token in (')', ',') and level == 1: + n = "".join(super) + if n in dict: + # we know this super class + n = dict[n] + else: + c = n.split('.') + if len(c) > 1: + # super class is of the form + # module.class: look in module for + # class + m = c[-2] + c = c[-1] + if m in _modules: + d = _modules[m] + if c in d: + n = d[c] + names.append(n) + super = [] + if token == '(': + level += 1 + elif token == ')': + level -= 1 + if level == 0: + break + elif token == ',' and level == 1: + pass + # only use NAME and OP (== dot) tokens for type name + elif tokentype in (NAME, OP) and level == 1: + super.append(token) + # expressions in the base list are not supported + inherit = names + cur_class = Class(fullmodule, class_name, inherit, + fname, lineno) + if not stack: + dict[class_name] = cur_class + stack.append((cur_class, thisindent)) + elif token == 'import' and start[1] == 0: + modules = _getnamelist(g) + for mod, _mod2 in modules: + try: + # Recursively read the imported module + if inpackage is None: + _readmodule(mod, path) + else: + try: + _readmodule(mod, path, inpackage) + except ImportError: + _readmodule(mod, []) + except: + # If we can't find or parse the imported module, + # too bad -- don't die here. + pass + elif token == 'from' and start[1] == 0: + mod, token = _getname(g) + if not mod or token != "import": + continue + names = _getnamelist(g) + try: + # Recursively read the imported module + d = _readmodule(mod, path, inpackage) + except: + # If we can't find or parse the imported module, + # too bad -- don't die here. + continue + # add any classes that were defined in the imported module + # to our name space if they were mentioned in the list + for n, n2 in names: + if n in d: + dict[n2 or n] = d[n] + elif n == '*': + # don't add names that start with _ + for n in d: + if n[0] != '_': + dict[n] = d[n] + except StopIteration: + pass + + f.close() + return dict + +def _getnamelist(g): + # Helper to get a comma-separated list of dotted names plus 'as' + # clauses. Return a list of pairs (name, name2) where name2 is + # the 'as' name, or None if there is no 'as' clause. + names = [] + while True: + name, token = _getname(g) + if not name: + break + if token == 'as': + name2, token = _getname(g) + else: + name2 = None + names.append((name, name2)) + while token != "," and "\n" not in token: + token = next(g)[1] + if token != ",": + break + return names + +def _getname(g): + # Helper to get a dotted name, return a pair (name, token) where + # name is the dotted name, or None if there was no dotted name, + # and token is the next input token. + parts = [] + tokentype, token = next(g)[0:2] + if tokentype != NAME and token != '*': + return (None, token) + parts.append(token) + while True: + tokentype, token = next(g)[0:2] + if token != '.': + break + tokentype, token = next(g)[0:2] + if tokentype != NAME: + break + parts.append(token) + return (".".join(parts), token) + +def _main(): + # Main program for testing. + import os + mod = sys.argv[1] + if os.path.exists(mod): + path = [os.path.dirname(mod)] + mod = os.path.basename(mod) + if mod.lower().endswith(".py"): + mod = mod[:-3] + else: + path = [] + dict = readmodule_ex(mod, path) + objs = list(dict.values()) + objs.sort(key=lambda a: getattr(a, 'lineno', 0)) + for obj in objs: + if isinstance(obj, Class): + print("class", obj.name, obj.super, obj.lineno) + methods = sorted(obj.methods.items(), key=itemgetter(1)) + for name, lineno in methods: + if name != "__path__": + print(" def", name, lineno) + elif isinstance(obj, Function): + print("def", obj.name, obj.lineno) + +if __name__ == "__main__": + _main() +"""Conversion pipeline templates. + +The problem: +------------ + +Suppose you have some data that you want to convert to another format, +such as from GIF image format to PPM image format. Maybe the +conversion involves several steps (e.g. piping it through compress or +uuencode). Some of the conversion steps may require that their input +is a disk file, others may be able to read standard input; similar for +their output. The input to the entire conversion may also be read +from a disk file or from an open file, and similar for its output. + +The module lets you construct a pipeline template by sticking one or +more conversion steps together. It will take care of creating and +removing temporary files if they are necessary to hold intermediate +data. You can then use the template to do conversions from many +different sources to many different destinations. The temporary +file names used are different each time the template is used. + +The templates are objects so you can create templates for many +different conversion steps and store them in a dictionary, for +instance. + + +Directions: +----------- + +To create a template: + t = Template() + +To add a conversion step to a template: + t.append(command, kind) +where kind is a string of two characters: the first is '-' if the +command reads its standard input or 'f' if it requires a file; the +second likewise for the output. The command must be valid /bin/sh +syntax. If input or output files are required, they are passed as +$IN and $OUT; otherwise, it must be possible to use the command in +a pipeline. + +To add a conversion step at the beginning: + t.prepend(command, kind) + +To convert a file to another file using a template: + sts = t.copy(infile, outfile) +If infile or outfile are the empty string, standard input is read or +standard output is written, respectively. The return value is the +exit status of the conversion pipeline. + +To open a file for reading or writing through a conversion pipeline: + fp = t.open(file, mode) +where mode is 'r' to read the file, or 'w' to write it -- just like +for the built-in function open() or for os.popen(). + +To create a new template object initialized to a given one: + t2 = t.clone() +""" # ' + + +import re +import os +import tempfile +# we import the quote function rather than the module for backward compat +# (quote used to be an undocumented but used function in pipes) +from shlex import quote + +__all__ = ["Template"] + +# Conversion step kinds + +FILEIN_FILEOUT = 'ff' # Must read & write real files +STDIN_FILEOUT = '-f' # Must write a real file +FILEIN_STDOUT = 'f-' # Must read a real file +STDIN_STDOUT = '--' # Normal pipeline element +SOURCE = '.-' # Must be first, writes stdout +SINK = '-.' # Must be last, reads stdin + +stepkinds = [FILEIN_FILEOUT, STDIN_FILEOUT, FILEIN_STDOUT, STDIN_STDOUT, \ + SOURCE, SINK] + + +class Template: + """Class representing a pipeline template.""" + + def __init__(self): + """Template() returns a fresh pipeline template.""" + self.debugging = 0 + self.reset() + + def __repr__(self): + """t.__repr__() implements repr(t).""" + return '