Added mypy testing to check type hints; improved testing for new installs

Author: mborsetti

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.8.1
+current_version = 3.8.2a0
 message = Release {new_version}
 parse = ^
 	(?P<major>\d+)

.deploy.bat

@@ -12,6 +12,11 @@ REM 4) run this file!
 REM To bump version from x.x.x.rc0 onwards:
 REM > bump2version --verbose --allow-dirty --no-commit --no-tag pre --dry-run
 
+REM For manual release in PyCharm:
+REM 1) Commit the release; comment is 'Release x.x'
+REM 2) In PyCharm Git (ALT+F9) -> Log, right click on the release and select New Tag. Enter tag in format vx.x
+REM 3) > git push origin unreleased:main --tags
+
 set "project=%~dp0"
 set "project=%project:~0,-1%"
 set "project=%project:\=" & set "project=%"
@@ -51,21 +56,14 @@ if %r% EQU N r=n
 if %r% EQU n exit /b
 
 echo.
-echo Bumping version to next %v%
+echo Bumping version to next %v%; this will also commit the changes.
 echo.
 bump2version --commit --tag --no-sign-tags %v%
-if NOT ["%errorlevel%"]==["0"] (
-    echo.
-    echo.
-    echo errorlevel: "%errorlevel%"
-    echo make sure to commit all files (other than those that will be modified by bump2version)
-    echo before rerunning this script
-    pause
-    exit /b %errorlevel%
-)
+
+pause
 
 echo.
-echo Pushing branch to main and setting release tag
+echo Pushing branch to main including release tag
 git push origin unreleased:main --tags
 if NOT ["%errorlevel%"]==["0"] (
     pause

.pre-commit-config.yaml

@@ -13,6 +13,11 @@
 
 # NOTE: in this package tox runs 'pre-commit run -a'
 
+# Force all unspecified python hooks to run python3
+default_language_version:
+    python: python3
+
+
 # A list of repository mappings.
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -39,8 +44,8 @@ repos:
 #        description: Checks that scripts with shebangs are executable
 #      - id: check-symlinks
 #        description: Checks for symlinks which do not point to anything
-#      - id: check-toml
-#        description: Attempts to load all TOML files to verify syntax.
+      - id: check-toml
+        description: Attempts to load all TOML files to verify syntax.
       - id: check-vcs-permalinks
         description: Ensures that links to vcs websites are permalinks
 #      - id: check-xml
@@ -123,8 +128,7 @@ repos:
         name: Find common security issues in Python code (bandit)
         entry: bandit
         language: python
-        language_version: python3
-        types: [ python ]
+        types: [python]
         args:  [--ini, .bandit, --recursive]
 #  isort not needed if running black
 #      - id: isort  # https://github.com/PyCQA/isort
@@ -139,37 +143,34 @@ repos:
         name: Convert % strings to f-strings (flynt)
         entry: flynt
         language: python
-        language_version: python3
         require_serial: true
         types: [python]
         args: [--fail-on-change]
       - id: black  # https://github.com/python/black
         name: Uncompromising code formatting (black)
         entry: black
         language: python
-        language_version: python3
         types: [python]
       - id: flake8
         name: Check for errors, style, [and over-complexity] (flake8)
         description: Code linter
         entry: flake8  # https://gitlab.com/pycqa/flake8
         language: python
-        language_version: python3
         types: [python]
         require_serial: true
         args: [--max-line-length, '120']
-#      - id: mypy  # https://github.com/python/mypy
-#        name: Static Typing for Python (mypy)
-#        entry: mypy
-#        language: python
-#        language_version: python3
-#        types: [python]
+      - id: mypy  # https://github.com/python/mypy
+        name: Static Typing for Python (mypy)
+        entry: mypy
+        language: python
+        types: [python]
+        pass_filenames: false
+        args: [-p, webchanges]
       - id: doc8  # https://github.com/pycqa/doc8
         name: Check for style in rst documentation (doc8)
         description: Documentation linter
         entry: doc8
         language: python
-        language_version: python3
         types: [rst]
         require_serial: true
         args: [--max-line-length, '120']

CHANGELOG.rst

@@ -31,6 +31,15 @@ can check out the `wish list <https://github.com/mborsetti/webchanges/blob/main/
    Internals, for changes that don't affect users. [triggers a minor patch]
 
 
+Version 3.8.2a0
+====================
+Unreleased
+
+Internals
+---------
+* Added mypy testing for type hint checks
+
+
 Version 3.8.1
 ====================
 2021-08-03

docs/cli.rst

@@ -94,7 +94,10 @@ Test run a job
 You can test a job and its filter by using the argument ``--test`` followed by the job index number (from ``--list``) or
 its URL/command; :program:`webchanges` will display the filtered output. This allows to easily test changes in
 filters. Use a negative index number to select a job from the bottom of your job list (i.e. -1 is the last job, -2 is
-the second to last job, etc.)
+the second to last job, etc.).  Combine ``--test`` with ``--verbose`` to get more information, for example the text
+returned from a website with a 4xx (client error) status code::
+
+   webchanges --verbose --test 1
 
 .. versionchanged:: 3.8
    Accepts negative indices.

docs/filters.rst

@@ -351,12 +351,12 @@ but may not yield the prettiest of results.
 
 ``bs4``
 ^^^^^^^
-This filter method extracts unformatted text from HTML using the `Beautiful Soup
-<https://pypi.org/project/beautifulsoup4/>`__, specifically its `get_text(strip=True)
+This filter method extracts human-readable text from HTML using the `Beautiful Soup
+<https://pypi.org/project/beautifulsoup4/>`__ Python package, specifically its `get_text(strip=True)
 <https://www.crummy.com/software/BeautifulSoup/bs4/doc/#get-text>`__ method.
 
 .. note:: As of Beautiful Soup version 4.9.0, when using the ``lxml`` or ``html.parser`` parser (see optional
-   sub-directive below), the contents of <script>, <style>, and <template> tags are not considered to be ‘text’, since
+   sub-directive below), the contents of <script>, <style>, and <template> tags are not considered to be ‘text’ since
    those tags are not part of the human-visible content of the page.
 
 .. code-block:: yaml

mypy.ini

@@ -0,0 +1,47 @@
+[mypy]
+# Static Typing for Python
+# Runs as part of pre-commit
+# File info at https://mypy.readthedocs.io/en/stable/config_file.html
+
+# Specifies the Python version used to parse and check the target program.
+python_version = 3.7
+
+# Shows error codes in error messages.
+show_error_codes = True
+
+# Suppresses error messages about imports that cannot be resolved.
+ignore_missing_imports = True
+
+# Disallows calling functions without type annotations from functions with type annotations.
+disallow_untyped_calls = True
+
+# Disallows defining functions without type annotations or with incomplete type annotations.
+disallow_untyped_defs = True
+
+# Reports an error whenever a function with type annotations is decorated with a decorator without annotations.
+disallow_untyped_decorators = True
+
+# Changes the treatment of arguments with a default value of None by not implicitly making their type Optional.
+no_implicit_optional = True
+
+# Warns about casting an expression to its inferred type.
+warn_redundant_casts = True
+
+# Warns about unneeded # type: ignore comments.
+warn_unused_ignores = True
+
+# Shows a warning when returning a value with type Any from a function declared with a non-Any return type.
+# warn_return_any = True
+
+# Shows a warning when encountering any code inferred to be unreachable or redundant after performing type analysis.
+warn_unreachable = False
+
+# Use visually nicer output in error messages: use soft word wrap, show source code snippets, and show error location
+# markers.
+pretty = True
+
+# Use an SQLite database to store the cache.
+sqlite_cache = True
+
+# Warns about per-module sections in the config file that do not match any files processed when invoking mypy.
+warn_unused_configs = True

pyproject.toml

@@ -1,3 +1,7 @@
+[build-system]
+# Minimum requirements for the build system to execute.
+requires = ['setuptools', 'wheel']  # PEP 508 specifications.
+
 [tool.black]
 # Uncompromising code formatting
 # Runs as part of pre-commit

setup.py

@@ -58,6 +58,7 @@
     'license': project.__license__,
     # Below to include in sdist the files read above (see https://stackoverflow.com/questions/37753833)
     # data_files is deprecated. It does not work with wheels, so it should be avoided.
+    'zip_safe': True,
     'install_requires': list(requirements),
     'entry_points': {'console_scripts': [f'{project.__project_name__}={project.__package__}.cli:main']},
     'extras_require': {
@@ -81,6 +82,7 @@
         'CI': f'{project.__code_url__}actions',
         'Documentation': project.__docs_url__,
         'Source Code': project.__code_url__,
+        'Changelog': f'{project.__docs_url__}changelog.html',
     },
 }
 SETUP['extras_require']['all'] = sorted(list(set(pkg for extra in SETUP['extras_require'].values() for pkg in extra)))

tests/command_test.py

@@ -114,7 +114,7 @@ def test_edit_hooks(capsys):
     with pytest.raises(SystemExit) as pytest_wrapped_e:
         urlwatch_command.handle_actions()
     setattr(command_config, 'edit_hooks', False)
-    assert pytest_wrapped_e.value.code is None
+    assert pytest_wrapped_e.value.code == 0
     message = capsys.readouterr().out
     assert message == f'Saved edits in {urlwatch_command.urlwatch_config.hooks}\n'
 
@@ -147,7 +147,7 @@ def test_show_features_and_verbose(capsys):
         urlwatch_command.handle_actions()
     setattr(command_config, 'features', False)
     setattr(command_config, 'verbose', False)
-    assert pytest_wrapped_e.value.code is None
+    assert pytest_wrapped_e.value.code == 0
     message = capsys.readouterr().out
     assert '* browser - Retrieve a URL, emulating a real web browser (use_browser: true).' in message
 
@@ -274,7 +274,7 @@ def test_test_diff_and_joblist(capsys):
         with pytest.raises(SystemExit) as pytest_wrapped_e:
             urlwatch_command.handle_actions()
         setattr(command_config, 'test_diff', None)
-        assert pytest_wrapped_e.value.code is None
+        assert pytest_wrapped_e.value.code == 0
         message = capsys.readouterr().out
         assert '=== Filtered diff between state 0 and state -1 ===\n' in message
         # rerun to reuse cached _generated_diff
@@ -296,7 +296,7 @@ def test_test_diff_and_joblist(capsys):
         with pytest.raises(SystemExit) as pytest_wrapped_e:
             urlwatch_command.handle_actions()
         setattr(command_config, 'test_diff', None)
-        assert pytest_wrapped_e.value.code is None
+        assert pytest_wrapped_e.value.code == 0
         message = capsys.readouterr().out
         assert '=== Filtered diff between state 0 and state -1 ===\n' in message
     finally:
@@ -470,7 +470,7 @@ def test_check_edit_config():
     with pytest.raises(SystemExit) as pytest_wrapped_e:
         urlwatch_command.check_edit_config()
     setattr(command_config, 'edit_config', False)
-    assert pytest_wrapped_e.value.code is None
+    assert pytest_wrapped_e.value.code == 0
 
 
 def test_check_edit_config_fail(capsys):

tests/jobs_test.py

@@ -388,7 +388,6 @@ def test_browser_block_elements_not_str_or_list():
     job = JobBase.unserialize(job_data)
     job_state = JobState(cache_storage, job)
     job_state.process()
-    print('job_state.exception', job_state.exception)
     assert isinstance(job_state.exception, TypeError)
 
 

tests/storage_test.py

@@ -218,7 +218,7 @@ def test_clean_cache_no_clean_all():
         assert timestamps[1] < timestamps[0]
 
         # clean cache without using clean_all
-        delattr(CacheSQLite3Storage, 'clean_all')
+        # delattr(CacheSQLite3Storage, 'clean_all')
         cache_storage.clean_cache([guid])
         history = cache_storage.get_history_data(guid)
         assert len(history) == 1

tox.ini

@@ -9,10 +9,10 @@
 minversion = 3.23.1
 envlist =
     pre-commit,
-    new-install,
     py{310,39,38,37},
     post,
-    docs
+    docs,
+    new-install,
 skip_missing_interpreters = true
 
 [testenv]
@@ -24,7 +24,7 @@ setenv =
     PYTHONUTF8 = 1
 # parallel_show_output = true
 depends =
-    py{310,39,38,37}: pre-commit, new-install
+#     py{310,39,38,37}: pre-commit, new-install
     post: py{310,39,38,37}
 
 [testenv:pre-commit]
@@ -40,19 +40,24 @@ commands =
 
 [testenv:new-install]
 # Settings defined in the top-level testenv section are automatically inherited if not overwritten
-# new-install tests a clean new installation, ensuring e.g. that all packages are installed as well
+# new-install tests a clean new installation using wheel, ensuring e.g. that all packages are installed as well
 basepython = python3.7
 commands =
+    python setup.py bdist_wheel
+    pip install --upgrade  --find-links={toxinidir}/dist webchanges
     webchanges -v --clean-cache
     python -c "from pathlib import Path; dir = Path.home().joinpath('Documents').joinpath('webchanges'); [f.unlink() for f in dir.iterdir()]; dir.rmdir()"
+deps = wheel
 setenv = USERPROFILE = {env:TEMP}
+download = true
+skip_install = true
 isolated_build = true
 
 [testenv:post]
 # Post-test cleanup
 commands =
     coverage combine
-    coverage html --fail-under=74
+    coverage html --fail-under=78
     # TODO The below works in Windows only
     cmd /c start "" htmlcov/index.html
 deps =

webchanges/__init__.py

@@ -16,14 +16,14 @@
 # * MINOR version when you add functionality in a backwards compatible manner, and
 # * MICRO or PATCH version when you make backwards compatible bug fixes. We no longer use '0'
 # If unsure on increments, use pkg_resources.parse_version to parse
-__version__ = '3.8.1'
+__version__ = '3.8.2a0'
 __description__ = 'Check web (or commands) for changes since last run and notify'
 __author__ = 'Mike Borsetti <[email protected]>'
 __copyright__ = 'Copyright 2020- Mike Borsetti'
 __license__ = 'MIT, BSD 3-Clause License'
 __url__ = f'https://pypi.org/project/{__project_name__}/'
 __code_url__ = f'https://www.github.com/mborsetti/{__project_name__}/'
-__docs_url__ = f'https://{__project_name__}.readthedocs.io'
+__docs_url__ = f'https://{__project_name__}.readthedocs.io/'
 __user_agent__ = f'{__project_name__}/{__version__} (+{__url__})'  # TODO find out why url is prepended by '+'
 
 from typing import Dict, Union

webchanges/cli.py

@@ -28,23 +28,19 @@
     YamlJobsStorage,
 )
 
-# directory where the config, jobs and hooks files are located
+# Directory where the config, jobs and hooks files are located
 if os.name != 'nt':
-    # typically ~/.config/{__project_name__}
-    config_path = user_config_path(__project_name__)
+    config_path = user_config_path(__project_name__)  # typically ~/.config/{__project_name__}
 else:
     config_path = Path.home().joinpath('Documents').joinpath(__project_name__)
 
 # directory where the database is located
 # typically ~/.cache/{__project_name__} or %LOCALAPPDATA%\{__project_name__}\{__project_name__}\Cache
 cache_path = user_cache_path(__project_name__)
 
-# Ignore SIGPIPE for stdout (see https://github.com/thp/urlwatch/issues/77)
-try:
+# Ignore signal SIGPIPE ("broken pipe") for stdout (see https://github.com/thp/urlwatch/issues/77)
+if os.name != 'nt':  # Windows does not have signal.SIGPIPE
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)  # type: ignore[attr-defined]  # not defined in Windows
-except AttributeError:
-    # Windows does not have signal.SIGPIPE
-    pass
 
 logger = logging.getLogger(__name__)