diff --git a/.circleci/config.yml b/.circleci/config.yml index 823ebb6309f..8ee8f9c0df9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,7 +100,7 @@ commands: name: Preparing environment - system command: | choco install -y --no-progress miniconda3 - choco install -y --no-progress openssl javaruntime + choco install -y --no-progress openssl openjdk11jre C:\tools\miniconda3\Scripts\conda.exe init powershell - run: name: Preparing environment - Hydra diff --git a/.gitignore b/.gitignore index 61e7b1526b8..c8773cfb671 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ pip-wheel-metadata .ipynb_checkpoints /.dmypy.json TODO.txt +/venv diff --git a/build_helpers/bin/antlr-4.11.1-complete.jar b/build_helpers/bin/antlr-4.11.1-complete.jar new file mode 100644 index 00000000000..bb96df95109 Binary files /dev/null and b/build_helpers/bin/antlr-4.11.1-complete.jar differ diff --git a/build_helpers/bin/antlr-4.9.3-complete.jar b/build_helpers/bin/antlr-4.9.3-complete.jar deleted file mode 100644 index 749296fe7b9..00000000000 Binary files a/build_helpers/bin/antlr-4.9.3-complete.jar and /dev/null differ diff --git a/build_helpers/bin/antlr4 b/build_helpers/bin/antlr4 index edafbd519c2..aa6a6fc67ee 100755 --- a/build_helpers/bin/antlr4 +++ b/build_helpers/bin/antlr4 @@ -4,7 +4,7 @@ import subprocess import sys my_dir = os.path.realpath(os.path.dirname(__file__)) -antlr = "antlr-4.9.3-complete.jar" +antlr = "antlr-4.11.1-complete.jar" args = ["java", "-jar", my_dir + "/" + antlr] args.extend(sys.argv[1:]) subprocess.check_call(args) diff --git a/build_helpers/bin/grun b/build_helpers/bin/grun index 6ae14225d04..4bcabd050cf 100755 --- a/build_helpers/bin/grun +++ b/build_helpers/bin/grun @@ -4,7 +4,7 @@ import subprocess import sys my_dir = os.path.realpath(os.path.dirname(__file__)) -antlr = "antlr-4.9.3-complete.jar" +antlr = "antlr-4.11.1-complete.jar" args = ["java", "-cp", my_dir + "/" + antlr, "org.antlr.v4.gui.TestRig"] args.extend(sys.argv[1:]) subprocess.check_call(args) diff --git a/build_helpers/build_helpers.py b/build_helpers/build_helpers.py index 7159d22615f..bd43111bdb1 100644 --- a/build_helpers/build_helpers.py +++ b/build_helpers/build_helpers.py @@ -6,7 +6,9 @@ import re import shutil import subprocess +from functools import partial from os.path import abspath, basename, dirname, exists, isdir, join +from pathlib import Path from typing import List, Optional from setuptools import Command @@ -185,7 +187,7 @@ def run(self) -> None: command = [ "java", "-jar", - join(root_dir, "bin/antlr-4.9.3-complete.jar"), + join(root_dir, "bin/antlr-4.11.1-complete.jar"), "-Dlanguage=Python3", "-o", join(project_root, "hydra/grammar/gen/"), @@ -198,8 +200,37 @@ def run(self) -> None: subprocess.check_call(command) + log.info("Replacing imports of antlr4 in generated parsers") + self._fix_imports() + def initialize_options(self) -> None: pass def finalize_options(self) -> None: pass + + def _fix_imports(self) -> None: + """Fix imports from the generated parsers to use the vendored antlr4 instead""" + build_dir = Path(__file__).parent.absolute() + project_root = build_dir.parent + lib = "antlr4" + pkgname = 'omegaconf.vendor' + + replacements = [ + partial( # import antlr4 -> import omegaconf.vendor.antlr4 + re.compile(r'(^\s*)import {}\n'.format(lib), flags=re.M).sub, + r'\1from {} import {}\n'.format(pkgname, lib) + ), + partial( # from antlr4 -> from fomegaconf.vendor.antlr4 + re.compile(r'(^\s*)from {}(\.|\s+)'.format(lib), flags=re.M).sub, + r'\1from {}.{}\2'.format(pkgname, lib) + ), + ] + + path = project_root / "hydra" / "grammar" / "gen" + for item in path.iterdir(): + if item.is_file() and item.name.endswith(".py"): + text = item.read_text('utf8') + for replacement in replacements: + text = replacement(text) + item.write_text(text, 'utf8') diff --git a/hydra/core/override_parser/overrides_parser.py b/hydra/core/override_parser/overrides_parser.py index b5678b3eb07..b095ccaad06 100644 --- a/hydra/core/override_parser/overrides_parser.py +++ b/hydra/core/override_parser/overrides_parser.py @@ -2,7 +2,10 @@ import sys from typing import Any, List, Optional -from antlr4.error.Errors import LexerNoViableAltException, RecognitionException +from omegaconf.vendor.antlr4.error.Errors import ( + LexerNoViableAltException, + RecognitionException, +) from hydra._internal.grammar import grammar_functions from hydra._internal.grammar.functions import Functions diff --git a/hydra/core/override_parser/overrides_visitor.py b/hydra/core/override_parser/overrides_visitor.py index db7a16f869b..6b094f68964 100644 --- a/hydra/core/override_parser/overrides_visitor.py +++ b/hydra/core/override_parser/overrides_visitor.py @@ -3,9 +3,13 @@ import warnings from typing import Any, Dict, List, Optional, Tuple, Union -from antlr4 import ParserRuleContext, TerminalNode, Token -from antlr4.error.ErrorListener import ErrorListener -from antlr4.tree.Tree import TerminalNodeImpl +from omegaconf.vendor.antlr4 import ( # type: ignore[attr-defined] + ParserRuleContext, + TerminalNode, + Token, +) +from omegaconf.vendor.antlr4.error.ErrorListener import ErrorListener +from omegaconf.vendor.antlr4.tree.Tree import TerminalNodeImpl from hydra._internal.grammar.functions import FunctionCall, Functions from hydra._internal.grammar.utils import _ESC_QUOTED_STR @@ -162,7 +166,7 @@ def visitOverride(self, ctx: OverrideParser.OverrideContext) -> Override: if ( override_type == OverrideType.DEL and isinstance(eq_node, TerminalNode) - and eq_node.symbol.type == Token.EOF + and eq_node.symbol.type == Token.EOF # type: ignore[attr-defined] ): value = None value_type = None @@ -244,26 +248,26 @@ def visitFunction(self, ctx: OverrideParser.FunctionContext) -> Any: ) from e def _createPrimitive( - self, ctx: ParserRuleContext + self, ctx: ParserRuleContext # type: ignore[valid-type] ) -> Optional[Union[QuotedString, int, bool, float, str]]: ret: Optional[Union[int, bool, float, str]] first_idx = 0 - last_idx = ctx.getChildCount() + last_idx = ctx.getChildCount() # type: ignore[attr-defined] # skip first if whitespace - if self.is_ws(ctx.getChild(0)): + if self.is_ws(ctx.getChild(0)): # type: ignore[attr-defined] if last_idx == 1: # Only whitespaces => this is not allowed. raise HydraException( "Trying to parse a primitive that is all whitespaces" ) first_idx = 1 - if self.is_ws(ctx.getChild(-1)): + if self.is_ws(ctx.getChild(-1)): # type: ignore[attr-defined] last_idx = last_idx - 1 num = last_idx - first_idx if num > 1: # Concatenate, while un-escaping as needed. tokens = [] - for i, n in enumerate(ctx.getChildren()): + for i, n in enumerate(ctx.getChildren()): # type: ignore[attr-defined] if n.symbol.type == OverrideLexer.WS and ( i < first_idx or i >= last_idx ): @@ -276,7 +280,7 @@ def _createPrimitive( ) ret = "".join(tokens) else: - node = ctx.getChild(first_idx) + node = ctx.getChild(first_idx) # type: ignore[attr-defined] if node.symbol.type == OverrideLexer.QUOTED_VALUE: text = node.getText() qc = text[0] @@ -360,7 +364,7 @@ def _unescape_quoted_string(self, text: str) -> str: return "".join(tokens) -class HydraErrorListener(ErrorListener): # type: ignore +class HydraErrorListener(ErrorListener): def syntaxError( self, recognizer: Any, diff --git a/hydra/core/plugins.py b/hydra/core/plugins.py index e63a35eaebd..3d4dbbd1fbb 100644 --- a/hydra/core/plugins.py +++ b/hydra/core/plugins.py @@ -187,13 +187,14 @@ def _scan_all_plugins( assert m is not None loaded_mod = m.load_module(modname) else: - spec = importer.find_spec(modname) + spec = importer.find_spec(modname) # type: ignore[call-arg] assert spec is not None if modname in sys.modules: loaded_mod = sys.modules[modname] else: loaded_mod = importlib.util.module_from_spec(spec) if loaded_mod is not None: + assert spec.loader is not None spec.loader.exec_module(loaded_mod) sys.modules[modname] = loaded_mod diff --git a/news/2699.maintenance b/news/2699.maintenance new file mode 100644 index 00000000000..1a820de0de1 --- /dev/null +++ b/news/2699.maintenance @@ -0,0 +1 @@ +Use OmegaConf's vendored version of `antlr4` runtime to prevent conflicts with other dependencies. See https://github.com/omry/omegaconf/pull/1114 for more details. diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 20f72cd4de6..0b365cc4014 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,4 +1,3 @@ -omegaconf>=2.2,<2.4 -antlr4-python3-runtime==4.9.* +omegaconf>=2.4.0.dev2 importlib-resources;python_version<'3.9' packaging