From 5467c723146d6abebd88441637baaaab5373c88e Mon Sep 17 00:00:00 2001 From: Filippo Vicentini Date: Tue, 18 Jan 2022 18:23:46 +0100 Subject: [PATCH 1/5] update gitignore gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 0242f00d..74fce62d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ Manifest.toml +LocalPreferences.toml aot/Manifest.toml aot/Project.toml aot/_julia_path @@ -6,3 +7,7 @@ aot/sys.* deps/deps.jl deps/PYTHON deps/build.log + +PyPreferences.jl/Manifest.toml + +.envrc \ No newline at end of file From 9dfa3c21de4d0758043d343d3af6c0b8dbc80dc7 Mon Sep 17 00:00:00 2001 From: Filippo Vicentini Date: Sun, 5 Dec 2021 13:45:34 +0100 Subject: [PATCH 2/5] use pyPreferences --- Project.toml | 3 ++- src/PyCall.jl | 1 + src/pyinit.jl | 2 +- src/startup.jl | 11 ++++++++--- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Project.toml b/Project.toml index 3dea4f2e..685fbeae 100644 --- a/Project.toml +++ b/Project.toml @@ -9,6 +9,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +PyPreferences = "cc9521c6-0242-4dda-8d66-c47a9d9eec02" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" VersionParsing = "81def892-9a0e-5fdd-b105-ffc91e053289" @@ -16,7 +17,7 @@ VersionParsing = "81def892-9a0e-5fdd-b105-ffc91e053289" Conda = "1.0" MacroTools = "0.4, 0.5" VersionParsing = "1.0" -julia = "0.7, 1.0" +julia = "1.6" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/src/PyCall.jl b/src/PyCall.jl index c09cc204..611c1147 100644 --- a/src/PyCall.jl +++ b/src/PyCall.jl @@ -4,6 +4,7 @@ if isdefined(Base, :Experimental) && isdefined(Base.Experimental, Symbol("@optle @eval Base.Experimental.@optlevel 1 end +using PyPreferences using VersionParsing export pycall, pycall!, pyimport, pyimport_e, pybuiltin, PyObject, PyReverseDims, diff --git a/src/pyinit.jl b/src/pyinit.jl index 42a928df..c6cff336 100644 --- a/src/pyinit.jl +++ b/src/pyinit.jl @@ -201,7 +201,7 @@ function __init__() if new_pyversion.major != pyversion.major error("PyCall precompiled with Python $pyversion, but now using Python $new_pyversion; ", - "you need to relaunch Julia and re-run Pkg.build(\"PyCall\")") + "you need to relaunch Julia and run `using PyPrefernces; PyPreferences.recompile()") end copy!(inspect, pyimport("inspect")) diff --git a/src/startup.jl b/src/startup.jl index d208040c..35527d17 100644 --- a/src/startup.jl +++ b/src/startup.jl @@ -38,17 +38,22 @@ else global symbols_present = hassym(proc_handle, :Py_GetVersion) end +if PyPreferences.inprocess + @assert symbols_present # TODO: better error +end + if !symbols_present + PyPreferences.assert_configured() + using PyPreferences: PYTHONHOME, conda, libpython, pyprogramname, python, pyversion_build # Python not present. Use deps.jl const depfile = joinpath(dirname(@__FILE__), "..", "deps", "deps.jl") - isfile(depfile) || error("PyCall not properly installed. Please run Pkg.build(\"PyCall\")") - include(depfile) # generated by Pkg.build("PyCall") + # Only to be used at top-level - pointer will be invalid after reload libpy_handle = try Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL) catch err if err isa ErrorException - error(err.msg, ". Please run `Pkg.build(\"PyCall\")` if your Python build has changed") + error(err.msg, "\n", PyPreferences.instruction_message()) else rethrow(err) end From 484346684a0a8ed465d865845f2abce99bb15237 Mon Sep 17 00:00:00 2001 From: Filippo Vicentini Date: Tue, 18 Jan 2022 23:55:41 +0100 Subject: [PATCH 3/5] add PyPreferences module remove artifact remove build phase start adapting CI fix ci fix ci fix ci fix ci fix ci conda fixes fix conda ci stop testing on 1.0 remove testers fix fix aot ci fixes dont test build fixes fix venv test refactor PyPreferences use env variable pyprefs project fix fixes stuff test fix conda revert changes fix cng fix python version custom which to work in julia 1.6 make test work with python 2 fixup docs --- .github/workflows/aot.yml | 6 +- .github/workflows/conda.yml | 12 +- .github/workflows/system.yml | 22 +-- Project.toml | 9 +- PyPreferences.jl/LICENSE | 21 ++ PyPreferences.jl/Project.toml | 24 +++ PyPreferences.jl/README.md | 8 + PyPreferences.jl/src/PyPreferences.jl | 41 ++++ PyPreferences.jl/src/api.jl | 62 ++++++ PyPreferences.jl/src/core.jl | 186 ++++++++++++++++++ .../src}/find_libpython.py | 23 ++- PyPreferences.jl/src/python_utils.jl | 157 +++++++++++++++ PyPreferences.jl/src/which.jl | 65 ++++++ PyPreferences.jl/test/runtests.jl | 10 + PyPreferences.jl/test/test_venv.jl | 123 ++++++++++++ aot/compile.jl | 2 +- deps/build.jl | 125 ------------ deps/buildutils.jl | 80 -------- src/PyCall.jl | 4 +- src/pyinit.jl | 33 +--- deps/depsutils.jl => src/startup_helpers.jl | 60 ------ test/runtests.jl | 18 +- test/test_venv.jl | 7 +- 23 files changed, 768 insertions(+), 330 deletions(-) create mode 100644 PyPreferences.jl/LICENSE create mode 100644 PyPreferences.jl/Project.toml create mode 100644 PyPreferences.jl/README.md create mode 100644 PyPreferences.jl/src/PyPreferences.jl create mode 100644 PyPreferences.jl/src/api.jl create mode 100644 PyPreferences.jl/src/core.jl rename {deps => PyPreferences.jl/src}/find_libpython.py (96%) mode change 100755 => 100644 create mode 100644 PyPreferences.jl/src/python_utils.jl create mode 100644 PyPreferences.jl/src/which.jl create mode 100644 PyPreferences.jl/test/runtests.jl create mode 100644 PyPreferences.jl/test/test_venv.jl delete mode 100644 deps/build.jl delete mode 100644 deps/buildutils.jl rename deps/depsutils.jl => src/startup_helpers.jl (58%) diff --git a/.github/workflows/aot.yml b/.github/workflows/aot.yml index 8fc32fa2..6f1ecb68 100644 --- a/.github/workflows/aot.yml +++ b/.github/workflows/aot.yml @@ -16,10 +16,11 @@ jobs: os: - ubuntu-latest architecture: [x64] - python-version: ['3.x'] + python-version: ['3.9'] julia-version: + - '1.6' - '1' - - '~1.7.0-rc1' + - '~1.7' # - 'nightly' # TODO: reenable fail-fast: false env: @@ -46,7 +47,6 @@ jobs: version: ${{ matrix.julia-version }} arch: ${{ matrix.architecture }} show-versioninfo: true - - run: julia -e 'using Pkg; pkg"add PackageCompiler@v1"' - run: aot/compile.jl diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index bfebbc80..b57e1b7a 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -18,14 +18,14 @@ jobs: - macos-latest - windows-latest architecture: [x64] - julia-version: ['1'] + julia-version: ['1.6', '1'] include: - os: windows-latest architecture: x86 julia-version: '1' fail-fast: false env: - PYTHON: "" + PYPREFERENCES_PYTHON: "conda" name: Test Julia ${{ matrix.julia-version }} Conda @@ -41,7 +41,13 @@ jobs: version: ${{ matrix.julia-version }} arch: ${{ matrix.architecture }} show-versioninfo: true - - uses: julia-actions/julia-buildpkg@v1 + - name: Install PyPreferences + shell: julia --project=@. {0} + run: | + using Pkg; + pkg"dev ./PyPreferences.jl"; + using PyPreferences; + PyPreferences.use_conda(); - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v1 diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index 2d6fb420..6a36c2b5 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -20,7 +20,7 @@ jobs: architecture: [x64] python-version: ['3.x'] julia-version: - - '1.0' + - '1.6' - '1' - 'nightly' include: @@ -34,14 +34,6 @@ jobs: architecture: x64 python-version: '3.9' julia-version: '1' - - os: ubuntu-latest - architecture: x64 - python-version: '3.x' - julia-version: '1.5' - - os: ubuntu-latest - architecture: x64 - python-version: '3.x' - julia-version: '1.4' # Test Python 2.7 only with a few combinations (TODO: drop 2.7). # Note that it does not work in macOS: - os: ubuntu-latest @@ -80,10 +72,14 @@ jobs: version: ${{ matrix.julia-version }} arch: ${{ matrix.architecture }} show-versioninfo: true - - uses: julia-actions/julia-buildpkg@v1 - env: - PYTHON: python - - run: julia test/check_deps_version.jl ${{ matrix.python-version }} + - name: Set PYTHON Env variable + run: echo "PYTHON=python" >> $GITHUB_ENV + if: ${{ matrix.python-version == '2.7' }} + - name: Install PyPreferences + shell: julia --project=@. {0} + run: | + using Pkg; + pkg"dev ./PyPreferences.jl" - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v1 diff --git a/Project.toml b/Project.toml index 685fbeae..9c243702 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PyCall" uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" authors = ["Steven G. Johnson ", "Yichao Yu ", "Takafumi Arakaki ", "Simon Kornblith ", "Páll Haraldsson ", "Jon Malmaud ", "Jake Bolewski ", "Keno Fischer ", "Joel Mason ", "Jameson Nash ", "The JuliaPy development team"] -version = "1.93.0" +version = "2.0.0" [deps] Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d" @@ -14,13 +14,14 @@ Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" VersionParsing = "81def892-9a0e-5fdd-b105-ffc91e053289" [compat] -Conda = "1.0" +Conda = "1" MacroTools = "0.4, 0.5" -VersionParsing = "1.0" +VersionParsing = "1" julia = "1.6" [extras] +Preferences = "21216c6a-2e73-6563-6e65-726566657250" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test"] +test = ["Test", "Preferences"] diff --git a/PyPreferences.jl/LICENSE b/PyPreferences.jl/LICENSE new file mode 100644 index 00000000..f59c07c6 --- /dev/null +++ b/PyPreferences.jl/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Takafumi Arakaki and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/PyPreferences.jl/Project.toml b/PyPreferences.jl/Project.toml new file mode 100644 index 00000000..aa60cdca --- /dev/null +++ b/PyPreferences.jl/Project.toml @@ -0,0 +1,24 @@ +name = "PyPreferences" +uuid = "cc9521c6-0242-4dda-8d66-c47a9d9eec02" +authors = ["Takafumi Arakaki and contributors"] +version = "0.1.0" + +[deps] +Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Preferences = "21216c6a-2e73-6563-6e65-726566657250" +VersionParsing = "81def892-9a0e-5fdd-b105-ffc91e053289" + +[compat] +Conda = "1" +julia = "1.6" +VersionParsing = "1" +Preferences = "1" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/PyPreferences.jl/README.md b/PyPreferences.jl/README.md new file mode 100644 index 00000000..eedb5262 --- /dev/null +++ b/PyPreferences.jl/README.md @@ -0,0 +1,8 @@ +# PyPreferences + +```julia +using PyPreferences + +PyPreferences.use_system("python_exe") +PyPreferences.use_conda() +``` \ No newline at end of file diff --git a/PyPreferences.jl/src/PyPreferences.jl b/PyPreferences.jl/src/PyPreferences.jl new file mode 100644 index 00000000..109a0a72 --- /dev/null +++ b/PyPreferences.jl/src/PyPreferences.jl @@ -0,0 +1,41 @@ +baremodule PyPreferences + +using Logging + +function use_system end +function use_conda end +# function use_jll end +function use_inprocess end +function recompile end + +# API to be used from PyCall +function assert_configured end +function instruction_message end + +# function diagnose end +function status end + +module Implementations + + module PythonUtils + include("python_utils.jl") + end + include("which.jl") + include("core.jl") + include("api.jl") +end + +let prefs = Implementations.setup_non_failing() + global const python = prefs.python + global const inprocess = prefs.inprocess + global const conda = prefs.conda + global const python_fullpath = prefs.python_fullpath + global const libpython = prefs.libpython + global const python_version = prefs.python_version + global const PYTHONHOME = prefs.PYTHONHOME +end + +const pyprogramname = python_fullpath +const pyversion_build = python_version + +end # baremodule PyPreferences diff --git a/PyPreferences.jl/src/api.jl b/PyPreferences.jl/src/api.jl new file mode 100644 index 00000000..269ce61c --- /dev/null +++ b/PyPreferences.jl/src/api.jl @@ -0,0 +1,62 @@ +using Conda + +function PyPreferences.status() + # TODO: compare with in-process values + code = """ + $(load_pypreferences_code()) + PyPreferences.Implementations.status_inprocess() + """ + cmd = include_stdin_cmd() + open(pipeline(cmd; stdout = stdout, stderr = stderr); write = true) do io + write(io, code) + end + return +end + +function PyPreferences.use_system(python::AbstractString = "python3") + """ + Use python from a provided executable or path (defaults to `python3`). + """ + return Implementations.set(python = python) +end + +function PyPreferences.use_conda() + """ + Use Python provided by Conda.jl + """ + Conda.add("numpy") + return Implementations.set(conda = true) +end + +#= +function use_jll() +end +=# + +function PyPreferences.use_inprocess() + return Implementations.set(inprocess = true) +end + +function PyPreferences.instruction_message() + return """ + PyPreferences.jl is not configured properly. Run: + using Pkg + Pkg.add("PyPreferences") + using PyPreferences + @doc PyPreferences + for usage. + """ +end + + +function PyPreferences.assert_configured() + if ( + PyPreferences.python === nothing || + PyPreferences.python_fullpath === nothing || + PyPreferences.libpython === nothing || + PyPreferences.python_version === nothing || + PyPreferences.PYTHONHOME === nothing + ) + error(PyPreferences.instruction_message()) + end +end \ No newline at end of file diff --git a/PyPreferences.jl/src/core.jl b/PyPreferences.jl/src/core.jl new file mode 100644 index 00000000..805f42a0 --- /dev/null +++ b/PyPreferences.jl/src/core.jl @@ -0,0 +1,186 @@ +using ..PyPreferences: PyPreferences +using .PythonUtils: find_libpython, python_version_of, pythonhome_of, conda_python_fullpath + +using Preferences: @set_preferences!, @load_preference, @delete_preferences! + +struct PythonPreferences + python::Union{Nothing,String} + inprocess::Bool + conda::Bool + # jll::Bool +end + +function Base.show(io::IO,x::PythonPreferences) + print(io, "PythonPreferences(python=$(x.python), inprocess=$(x.inprocess), conda=$(x.conda))") +end + + +set(; python = nothing, inprocess = false, conda = false) = + set(PythonPreferences(python, inprocess, conda)) + +function set(prefs::PythonPreferences) + @debug "setting new Python Preferences" prefs + if prefs.python === nothing + @delete_preferences!("python") + else + @set_preferences!("python" => prefs.python) + end + if prefs.inprocess + @set_preferences!("inprocess" => prefs.inprocess) + else + @delete_preferences!("inprocess") + end + if prefs.conda + @set_preferences!("conda" => prefs.conda) + else + @delete_preferences!("conda") + end + PyPreferences.recompile() + return prefs +end + +PythonPreferences(rawprefs::AbstractDict) = PythonPreferences( + get(rawprefs, "python", nothing), + get(rawprefs, "inprocess", false), + get(rawprefs, "conda", false), +) + +function _load_python_preferences() + # TODO: lookup v#.#? + _python = @load_preference("python", nothing) + _inprocess = @load_preference("inprocess", false) + _conda = @load_preference("conda", false) + #isempty(rawprefs) && return nothing + + # default value + # if !_inprocess && !_conda && _python === nothing + # _python = get_python_fullpath(get_default_python()) + # @info "Setting default Python interpreter to $(_python)" + # return set(python=_python) + # end + return PythonPreferences(_python, _inprocess, _conda) +end + +function load_pypreferences_code() + return """ + $(Base.load_path_setup_code()) + PyPreferences = Base.require(Base.PkgId( + Base.UUID("cc9521c6-0242-4dda-8d66-c47a9d9eec02"), + "PyPreferences", + )) + """ +end + +function include_stdin_cmd() + return ``` + $(Base.julia_cmd()) + --startup-file=no + -e "include_string(Main, read(stdin, String))" + ``` +end + +function PyPreferences.recompile() + code = """ + $(load_pypreferences_code()) + PyPreferences.assert_configured() + """ + cmd = include_stdin_cmd() + open(cmd; write = true) do io + write(io, code) + end + return +end + +""" +Returns the default python executable used by PyCall. This defaults to +`python3`, and can be overridden by `ENV["PYTHON"]` if it is desired. +""" +get_default_python() = get(ENV,"PYTHON", "python3") + +function get_python_fullpath(python) + python_fullpath = nothing + if python !== nothing + python_fullpath = _which(python) + if python_fullpath === nothing + @error "Failed to find a binary named `$(python)` in PATH." + else + @debug "Found path for command $(python)" python_fullpath + end + end + return python_fullpath +end + +function setup_non_failing() + python = nothing + inprocess = false + conda = false + python_fullpath = nothing + libpython = nothing + python_version = nothing + PYTHONHOME = nothing + + prefs = _load_python_preferences() + @debug "Loaded python preferences" prefs + python = prefs.python + inprocess = prefs.inprocess + conda = prefs.conda + + if !inprocess + if conda + python = python_fullpath = conda_python_fullpath() + elseif python === nothing + python = get_default_python() + end + + @debug "Python binary selected. Attempting to find the path" python + + try + if python !== nothing + python_fullpath = _which(python) + if python_fullpath === nothing + @error "Failed to find a binary named `$(python)` in PATH." + else + @debug "Found path for command $(python)" python_fullpath + end + end + + if python_fullpath !== nothing + libpython, = find_libpython(python_fullpath) + python_version = python_version_of(python_fullpath) + PYTHONHOME = pythonhome_of(python_fullpath) + end + catch err + @error( + "Failed to configure for `$python`", + exception = (err, catch_backtrace()) + ) + end + + @debug "Determined python binary path" python_fullpath libpython python_version PYTHONHOME + end + if python === nothing + python = python_fullpath + end + + return ( + python = python, + inprocess = inprocess, + conda = conda, + python_fullpath = python_fullpath, + libpython = libpython, + python_version = python_version, + PYTHONHOME = PYTHONHOME, + ) +end + +function status_inprocess() + print(""" + python : $(PyPreferences.python) + inprocess : $(PyPreferences.inprocess) + conda : $(PyPreferences.conda) + python_fullpath: $(PyPreferences.python_fullpath) + libpython : $(PyPreferences.libpython) + python_version : $(PyPreferences.python_version) + PYTHONHOME : $(PyPreferences.PYTHONHOME) + """) +end diff --git a/deps/find_libpython.py b/PyPreferences.jl/src/find_libpython.py old mode 100755 new mode 100644 similarity index 96% rename from deps/find_libpython.py rename to PyPreferences.jl/src/find_libpython.py index 422a0200..72e2270c --- a/deps/find_libpython.py +++ b/PyPreferences.jl/src/find_libpython.py @@ -27,14 +27,14 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function -from logging import getLogger import ctypes.util import functools import os import sys import sysconfig +from logging import getLogger # see `julia.core.logger` logger = getLogger("find_libpython") @@ -76,6 +76,9 @@ class Dl_info(ctypes.Structure): ] +# fmt: off + + def _linked_libpython_unix(): libdl = ctypes.CDLL(ctypes.util.find_library("dl")) libdl.dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(Dl_info)] @@ -88,6 +91,8 @@ def _linked_libpython_unix(): if retcode == 0: # means error return None path = os.path.realpath(dlinfo.dli_fname.decode()) + if not os.path.exists(path): + return None if path == os.path.realpath(sys.executable): return None return path @@ -182,9 +187,8 @@ def candidate_names(suffix=SHLIB_SUFFIX): sysdata = dict( v=sys.version_info, # VERSION is X.Y in Linux/macOS and XY in Windows: - VERSION=(sysconfig.get_python_version() or - "{v.major}.{v.minor}".format(v=sys.version_info) or - sysconfig.get_config_var("VERSION")), + VERSION=(sysconfig.get_config_var("VERSION") or + "{v.major}.{v.minor}".format(v=sys.version_info)), ABIFLAGS=(sysconfig.get_config_var("ABIFLAGS") or sysconfig.get_config_var("abiflags") or ""), ) @@ -264,8 +268,13 @@ def normalize_path(path, suffix=SHLIB_SUFFIX, is_apple=is_apple): Parameters ---------- - path : str ot None + path : str or None A candidate path to a shared library. + + Returns + ------- + path : str or None + Normalized existing path or `None`. """ if not path: return None @@ -392,4 +401,4 @@ def main(args=None): if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/PyPreferences.jl/src/python_utils.jl b/PyPreferences.jl/src/python_utils.jl new file mode 100644 index 00000000..a1c1b4f9 --- /dev/null +++ b/PyPreferences.jl/src/python_utils.jl @@ -0,0 +1,157 @@ +using Libdl: Libdl +using VersionParsing: vparse +using Conda: Conda + +conda_python_fullpath() = + abspath(Conda.PYTHONDIR, "python" * (Sys.iswindows() ? ".exe" : "")) + +# Fix the environment for running `python`, and setts IO encoding to UTF-8. +# If cmd is the Conda python, then additionally removes all PYTHON* and +# CONDA* environment variables. +function pythonenv(cmd::Cmd) + @assert cmd.env === nothing # TODO: handle non-nothing case + env = copy(ENV) + if dirname(cmd.exec[1]) == abspath(Conda.PYTHONDIR) + pythonvars = String[] + for var in keys(env) + if startswith(var, "CONDA") || startswith(var, "PYTHON") + push!(pythonvars, var) + end + end + for var in pythonvars + pop!(env, var) + end + end + + # set PYTHONIOENCODING when running python executable, so that + # we get UTF-8 encoded text as output (this is not the default on Windows). + env["PYTHONIOENCODING"] = "UTF-8" + setenv(cmd, env) +end + +pyvar(python::AbstractString, mod::AbstractString, var::AbstractString) = + chomp(read(pythonenv(`$python -c "import $mod; print($mod.$(var))"`), String)) + +pyconfigvar(python::AbstractString, var::AbstractString) = + pyvar(python, "distutils.sysconfig", "get_config_var('$(var)')") +pyconfigvar(python, var, default) = + let v = pyconfigvar(python, var) + v == "None" ? default : v + end + +function pythonhome_of(pyprogramname::AbstractString) + if Sys.iswindows() + # PYTHONHOME tells python where to look for both pure python + # and binary modules. When it is set, it replaces both + # `prefix` and `exec_prefix` and we thus need to set it to + # both in case they differ. This is also what the + # documentation recommends. However, they are documented + # to always be the same on Windows, where it causes + # problems if we try to include both. + script = """ + import sys + if hasattr(sys, "base_exec_prefix"): + sys.stdout.write(sys.base_exec_prefix) + else: + sys.stdout.write(sys.exec_prefix) + """ + else + script = """ + import sys + if hasattr(sys, "base_exec_prefix"): + sys.stdout.write(sys.base_prefix) + sys.stdout.write(":") + sys.stdout.write(sys.base_exec_prefix) + else: + sys.stdout.write(sys.prefix) + sys.stdout.write(":") + sys.stdout.write(sys.exec_prefix) + """ + # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHOME + end + return read(pythonenv(`$pyprogramname -c $script`), String) +end +# To support `venv` standard library (as well as `virtualenv`), we +# need to use `sys.base_prefix` and `sys.base_exec_prefix` here. +# Otherwise, initializing Python in `__init__` below fails with +# unrecoverable error: +# +# Fatal Python error: initfsencoding: unable to load the file system codec +# ModuleNotFoundError: No module named 'encodings' +# +# This is because `venv` does not symlink standard libraries like +# `virtualenv`. For example, `lib/python3.X/encodings` does not +# exist. Rather, `venv` relies on the behavior of Python runtime: +# +# If a file named "pyvenv.cfg" exists one directory above +# sys.executable, sys.prefix and sys.exec_prefix are set to that +# directory and it is also checked for site-packages +# --- https://docs.python.org/3/library/venv.html +# +# Thus, we need point `PYTHONHOME` to `sys.base_prefix` and +# `sys.base_exec_prefix`. If the virtual environment is created by +# `virtualenv`, those `sys.base_*` paths point to the virtual +# environment. Thus, above code supports both use cases. +# +# See also: +# * https://docs.python.org/3/library/venv.html +# * https://docs.python.org/3/library/site.html +# * https://docs.python.org/3/library/sys.html#sys.base_exec_prefix +# * https://github.com/JuliaPy/PyCall.jl/issues/410 + +python_version_of(python) = vparse(pyvar(python, "platform", "python_version()")) + +function find_libpython_py_path() + + return joinpath(@__DIR__, "find_libpython.py") +end + +function exec_find_libpython(python::AbstractString, options, verbose::Bool) + # Do not inline `@__DIR__` into the backticks to expand correctly. + # See: https://github.com/JuliaLang/julia/issues/26323 + script = find_libpython_py_path() + cmd = `$python $script $options` + if verbose + cmd = `$cmd --verbose` + end + return readlines(pythonenv(cmd)) +end + +# return libpython path, libpython pointer +function find_libpython( + python::AbstractString; + _dlopen = Libdl.dlopen, + verbose::Bool = false, +) + dlopen_flags = Libdl.RTLD_LAZY | Libdl.RTLD_DEEPBIND | Libdl.RTLD_GLOBAL + + libpaths = exec_find_libpython(python, `--list-all`, verbose) + for lib in libpaths + try + return (lib, _dlopen(lib, dlopen_flags)) + catch e + @warn "Failed to `dlopen` $lib" exception = (e, catch_backtrace()) + end + end + @warn """ + Python (`find_libpython.py`) failed to find `libpython`. + Falling back to `Libdl`-based discovery. + """ + + # Try all candidate libpython names and let Libdl find the path. + # We do this *last* because the libpython in the system + # library path might be the wrong one if multiple python + # versions are installed (we prefer the one in LIBDIR): + libs = exec_find_libpython(python, `--candidate-names`, verbose) + for lib in libs + lib = splitext(lib)[1] + try + libpython = _dlopen(lib, dlopen_flags) + return (Libdl.dlpath(libpython), libpython) + catch e + @debug "Failed to `dlopen` $lib" exception = (e, catch_backtrace()) + end + end + + return nothing, nothing +end diff --git a/PyPreferences.jl/src/which.jl b/PyPreferences.jl/src/which.jl new file mode 100644 index 00000000..ff2a2b7a --- /dev/null +++ b/PyPreferences.jl/src/which.jl @@ -0,0 +1,65 @@ +# This file vendors the utility `Sys.which` for Julia versions prior to 1.7, +# as in 1.6 it called `realpath` and therefore breaks virtual environment +# detection. (Without this, all `test/test_venv.jl` would fail). + +@static if VERSION >= v"1.7.0" + const _which = Sys.which +else + function _which(program_name::String) + if isempty(program_name) + return nothing + end + # Build a list of program names that we're going to try + program_names = String[] + base_pname = basename(program_name) + if Sys.iswindows() + # If the file already has an extension, try that name first + if !isempty(splitext(base_pname)[2]) + push!(program_names, base_pname) + end + + # But also try appending .exe and .com` + for pe in (".exe", ".com") + push!(program_names, string(base_pname, pe)) + end + else + # On non-windows, we just always search for what we've been given + push!(program_names, base_pname) + end + + path_dirs = String[] + program_dirname = dirname(program_name) + # If we've been given a path that has a directory name in it, then we + # check to see if that path exists. Otherwise, we search the PATH. + if isempty(program_dirname) + # If we have been given just a program name (not a relative or absolute + # path) then we should search `PATH` for it here: + pathsep = Sys.iswindows() ? ';' : ':' + path_dirs = abspath.(split(get(ENV, "PATH", ""), pathsep)) + + # On windows we always check the current directory as well + if Sys.iswindows() + pushfirst!(path_dirs, pwd()) + end + else + push!(path_dirs, abspath(program_dirname)) + end + + # Here we combine our directories with our program names, searching for the + # first match among all combinations. + for path_dir in path_dirs + for pname in program_names + program_path = joinpath(path_dir, pname) + # If we find something that matches our name and we can execute + if isfile(program_path) && Sys.isexecutable(program_path) + return program_path + end + end + end + + # If we couldn't find anything, don't return anything + nothing + end + + _which(program_name::AbstractString) = _which(String(program_name)) +end \ No newline at end of file diff --git a/PyPreferences.jl/test/runtests.jl b/PyPreferences.jl/test/runtests.jl new file mode 100644 index 00000000..2f0a8376 --- /dev/null +++ b/PyPreferences.jl/test/runtests.jl @@ -0,0 +1,10 @@ +using PyPreferences +using Test + +@testset "PyPreferences.jl" begin + # Write your tests here. +end + +if lowercase(get(ENV, "JULIA_PKGEVAL", "false")) != "true" + include("test_venv.jl") +end diff --git a/PyPreferences.jl/test/test_venv.jl b/PyPreferences.jl/test/test_venv.jl new file mode 100644 index 00000000..01255462 --- /dev/null +++ b/PyPreferences.jl/test/test_venv.jl @@ -0,0 +1,123 @@ +using PyCall, Test + + +function test_venv_has_python(path) + newpython = PyCall.python_cmd(venv=path).exec[1] + if !isfile(newpython) + @info """ + Python executable $newpython does not exists. + This directory contains only the following files: + $(join(readdir(dirname(newpython)), '\n')) + """ + end + @test isfile(newpython) +end + + +function test_venv_activation(path) + newpython = PyCall.python_cmd(venv=path).exec[1] + + # Run a fresh Julia process with new Python environment + code = """ + $(Base.load_path_setup_code()) + using PyCall + println(PyCall.pyimport("sys").executable) + println(PyCall.pyimport("sys").exec_prefix) + println(PyCall.pyimport("pip").__file__) + """ + # Note that `pip` is just some arbitrary non-standard + # library. Using standard library like `os` does not work + # because those files are not created. + env = copy(ENV) + env["PYCALL_JL_RUNTIME_PYTHON"] = newpython + jlcmd = setenv(`$(Base.julia_cmd()) --startup-file=no -e $code`, env) + if Sys.iswindows() + # Marking the test broken in Windows. It seems that + # venv copies .dll on Windows and libpython check in + # PyCall.__init__ detects that. + @test_broken begin + output = read(jlcmd, String) + sys_executable, exec_prefix, mod_file = split(output, "\n") + newpython == sys_executable + end + else + output = read(jlcmd, String) + sys_executable, exec_prefix, mod_file = split(output, "\n") + @test newpython == sys_executable + @test startswith(exec_prefix, path) + @test startswith(mod_file, path) + end +end + + +@testset "virtualenv activation" begin + pyname = "python$(pyversion.major).$(pyversion.minor)" + if Sys.which("virtualenv") === nothing + @info "No virtualenv command. Skipping the test..." + elseif Sys.which(pyname) === nothing + @info "No $pyname command. Skipping the test..." + else + mktempdir() do tmppath + if PyCall.pyversion.major == 2 + path = joinpath(tmppath, "kind") + else + path = joinpath(tmppath, "ϵνιℓ") + end + run(`virtualenv --python=$pyname $path`) + test_venv_has_python(path) + + newpython = PyCall.python_cmd(venv=path).exec[1] + venv_libpython = PyCall.find_libpython(newpython) + if venv_libpython != PyCall.libpython + @info """ + virtualenv created an environment with incompatible libpython: + $venv_libpython + """ + return + end + + test_venv_activation(path) + end + end +end + + +@testset "venv activation" begin + # In case PyCall is built with a Python executable created by + # `virtualenv`, let's try to find the original Python executable. + # Otherwise, `venv` does not work with this Python executable: + # https://bugs.python.org/issue30811 + sys = PyCall.pyimport("sys") + if hasproperty(sys, :real_prefix) + # sys.real_prefix is set by virtualenv and does not exist in + # standard Python: + # https://github.com/pypa/virtualenv/blob/16.0.0/virtualenv_embedded/site.py#L554 + candidates = [ + PyCall.venv_python(sys.real_prefix, "$(pyversion.major).$(pyversion.minor)"), + PyCall.venv_python(sys.real_prefix, "$(pyversion.major)"), + PyCall.venv_python(sys.real_prefix), + PyCall.pyprogramname, # must exists + ] + python = candidates[findfirst(isfile, candidates)] + else + python = PyCall.pyprogramname + end + + if PyCall.conda + @info "Skip venv test with conda." + elseif !success(PyCall.python_cmd(`-c "import venv"`, python=python)) + @info "Skip venv test since venv package is missing." + else + mktempdir() do tmppath + if PyCall.pyversion.major == 2 + path = joinpath(tmppath, "kind") + else + path = joinpath(tmppath, "ϵνιℓ") + end + # Create a new virtual environment + run(PyCall.python_cmd(`-m venv $path`, python=python)) + test_venv_has_python(path) + test_venv_activation(path) + end + end +end diff --git a/aot/compile.jl b/aot/compile.jl index 97af0a00..cb2b67a3 100755 --- a/aot/compile.jl +++ b/aot/compile.jl @@ -9,8 +9,8 @@ using PackageCompiler using Pkg Pkg.activate(@__DIR__) +Pkg.develop(PackageSpec(name="PyPreferences", path=joinpath(dirname(@__DIR__), "PyPreferences.jl"))) Pkg.develop(PackageSpec(name="PyCall", path=dirname(@__DIR__))) -Pkg.build("PyCall") Pkg.activate() sysimage_path = joinpath(@__DIR__, "sys.$(Libdl.dlext)") diff --git a/deps/build.jl b/deps/build.jl deleted file mode 100644 index 4dd63826..00000000 --- a/deps/build.jl +++ /dev/null @@ -1,125 +0,0 @@ -# In this file, we figure out how to link to Python (surprisingly complicated) -# and generate a deps/deps.jl file with the libpython name and other information -# needed for static compilation of PyCall. - -# As a result, if you switch to a different version or path of Python, you -# will probably need to re-run Pkg.build("PyCall"). - -using VersionParsing -import Conda, Libdl - -struct UseCondaPython <: Exception end - -include("buildutils.jl") -include("depsutils.jl") - -######################################################################### - -# we write configuration files only if they change, both -# to prevent unnecessary recompilation and to minimize -# problems in the unlikely event of read-only directories. -function writeifchanged(filename, str) - if !isfile(filename) || read(filename, String) != str - @info string(abspath(filename), " has been updated") - write(filename, str) - else - @info string(abspath(filename), " has not changed") - end -end - -# return the first arg that exists in the PATH -function whichfirst(args...) - for x in args - if Sys.which(x) !== nothing - return x - end - end - return "" -end - -const prefsfile = joinpath(first(DEPOT_PATH), "prefs", "PyCall") -mkpath(dirname(prefsfile)) - -try # make sure deps.jl file is removed on error - python = try - let py = get(ENV, "PYTHON", isfile(prefsfile) ? readchomp(prefsfile) : - (Sys.isunix() && !Sys.isapple()) ? - whichfirst("python3", "python") : "Conda"), - vers = isempty(py) || py == "Conda" ? v"0.0" : vparse(pyconfigvar(py,"VERSION","0.0")) - if vers < v"2.7" - if isempty(py) || py == "Conda" - throw(UseCondaPython()) - else - error("Python version $vers < 2.7 is not supported") - end - end - - # check word size of Python via sys.maxsize, since a common error - # on Windows is to link a 64-bit Julia to a 32-bit Python. - pywordsize = parse(UInt64, pysys(py, "maxsize")) > (UInt64(1)<<32) ? 64 : 32 - if pywordsize != Sys.WORD_SIZE - error("$py is $(pywordsize)-bit, but Julia is $(Sys.WORD_SIZE)-bit") - end - - py - end - catch e1 - if isa(e1, UseCondaPython) - @info string("Using the Python distribution in the Conda package by default.\n", - "To use a different Python version, set ENV[\"PYTHON\"]=\"pythoncommand\" and re-run Pkg.build(\"PyCall\").") - else - @info string("No system-wide Python was found; got the following error:\n", - "$e1\nusing the Python distribution in the Conda package") - end - abspath(Conda.PYTHONDIR, "python" * (Sys.iswindows() ? ".exe" : "")) - end - - use_conda = dirname(python) == abspath(Conda.PYTHONDIR) - if use_conda - Conda.add("numpy") - end - - (libpython, libpy_name) = find_libpython(python) - programname = pysys(python, "executable") - - # Get PYTHONHOME, either from the environment or from Python - # itself (if it is not in the environment or if we are using Conda) - PYTHONHOME = if !haskey(ENV, "PYTHONHOME") || use_conda - pythonhome_of(python) - else - ENV["PYTHONHOME"] - end - - # cache the Python version as a Julia VersionNumber - pyversion = vparse(pyvar(python, "platform", "python_version()")) - - @info "PyCall is using $python (Python $pyversion) at $programname, libpython = $libpy_name" - - if pyversion < v"2.7" - error("Python 2.7 or later is required for PyCall") - end - - writeifchanged("deps.jl", """ - const python = "$(escape_string(python))" - const libpython = "$(escape_string(libpy_name))" - const pyprogramname = "$(escape_string(programname))" - const pyversion_build = $(repr(pyversion)) - const PYTHONHOME = "$(escape_string(PYTHONHOME))" - - "True if we are using the Python distribution in the Conda package." - const conda = $use_conda - """) - - # Make subsequent builds (e.g. Pkg.update) use the same Python by default: - writeifchanged(prefsfile, use_conda ? "Conda" : isfile(programname) ? programname : python) - - ######################################################################### - -catch - - # remove deps.jl (if it exists) on an error, so that PyCall will - # not load until it is properly configured. - isfile("deps.jl") && rm("deps.jl") - rethrow() - -end diff --git a/deps/buildutils.jl b/deps/buildutils.jl deleted file mode 100644 index 5eb27b94..00000000 --- a/deps/buildutils.jl +++ /dev/null @@ -1,80 +0,0 @@ -# Included from build.jl and ../test/test_build.jl - -using VersionParsing -import Conda, Libdl - -pyvar(python::AbstractString, mod::AbstractString, var::AbstractString) = chomp(read(pythonenv(`$python -c "import $mod; print($mod.$(var))"`), String)) - -pyconfigvar(python::AbstractString, var::AbstractString) = pyvar(python, "distutils.sysconfig", "get_config_var('$(var)')") -pyconfigvar(python, var, default) = let v = pyconfigvar(python, var) - v == "None" ? default : v -end - -pysys(python::AbstractString, var::AbstractString) = pyvar(python, "sys", var) - -######################################################################### - -# print out extra info to help with remote debugging -const PYCALL_DEBUG_BUILD = "yes" == get(ENV, "PYCALL_DEBUG_BUILD", "no") - -function exec_find_libpython(python::AbstractString, options) - # Do not inline `@__DIR__` into the backticks to expand correctly. - # See: https://github.com/JuliaLang/julia/issues/26323 - script = joinpath(@__DIR__, "find_libpython.py") - cmd = `$python $script $options` - if PYCALL_DEBUG_BUILD - cmd = `$cmd --verbose` - end - return readlines(pythonenv(cmd)) -end - -function show_dlopen_error(lib, e) - if PYCALL_DEBUG_BUILD - println(stderr, "dlopen($lib) ==> ", e) - # Using STDERR since find_libpython.py prints debugging - # messages to STDERR too. - end -end - -# return libpython name, libpython pointer -function find_libpython(python::AbstractString; _dlopen = Libdl.dlopen) - dlopen_flags = Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL - - libpaths = exec_find_libpython(python, `--list-all`) - for lib in libpaths - try - return (_dlopen(lib, dlopen_flags), lib) - catch e - show_dlopen_error(lib, e) - end - end - - # Try all candidate libpython names and let Libdl find the path. - # We do this *last* because the libpython in the system - # library path might be the wrong one if multiple python - # versions are installed (we prefer the one in LIBDIR): - libs = exec_find_libpython(python, `--candidate-names`) - for lib in libs - lib = splitext(lib)[1] - try - libpython = _dlopen(lib, dlopen_flags) - # Store the fullpath to libpython in deps.jl. This makes - # it easier for users to investigate Python setup - # PyCall.jl trying to use. It also helps PyJulia to - # compare libpython. - return (libpython, Libdl.dlpath(libpython)) - catch e - show_dlopen_error(lib, e) - end - end - - v = pyconfigvar(python, "VERSION", "unknown") - error(""" - Couldn't find libpython; check your PYTHON environment variable. - - The python executable we tried was $python (= version $v). - Re-building with - ENV["PYCALL_DEBUG_BUILD"] = "yes" - may provide extra information for why it failed. - """) -end diff --git a/src/PyCall.jl b/src/PyCall.jl index 611c1147..1f2b008e 100644 --- a/src/PyCall.jl +++ b/src/PyCall.jl @@ -5,7 +5,7 @@ if isdefined(Base, :Experimental) && isdefined(Base.Experimental, Symbol("@optle end using PyPreferences -using VersionParsing +using VersionParsing: vparse export pycall, pycall!, pyimport, pyimport_e, pybuiltin, PyObject, PyReverseDims, PyPtr, pyincref, pydecref, pyversion, @@ -35,7 +35,7 @@ import Base.Iterators: filter ######################################################################### -include(joinpath(dirname(@__FILE__), "..", "deps","depsutils.jl")) +include("startup_helpers.jl") include("startup.jl") """ diff --git a/src/pyinit.jl b/src/pyinit.jl index c6cff336..35e735b6 100644 --- a/src/pyinit.jl +++ b/src/pyinit.jl @@ -98,7 +98,7 @@ function python_cmd(args::Cmd = ``; end function find_libpython(python::AbstractString) - script = joinpath(@__DIR__, "..", "deps", "find_libpython.py") + script = joinpath(@__DIR__, "..", "PyPreferences.jl", "src", "find_libpython.py") cmd = python_cmd(`$script`; python = python) try return read(cmd, String) @@ -154,34 +154,7 @@ function __init__() if !already_inited pyhome = PYTHONHOME - if isfile(get(ENV, "PYCALL_JL_RUNTIME_PYTHON", "")) - _current_python[] = ENV["PYCALL_JL_RUNTIME_PYTHON"] - - # Check libpython compatibility. - venv_libpython = find_libpython(current_python()) - if venv_libpython === nothing - error(""" - `libpython` for $(current_python()) cannot be found. - PyCall.jl cannot initialize Python safely. - """) - elseif venv_libpython != libpython - error(""" - Incompatible `libpython` detected. - `libpython` for $(current_python()) is: - $venv_libpython - `libpython` for $pyprogramname is: - $libpython - PyCall.jl only supports loading Python environment using - the same `libpython`. - """) - end - - if haskey(ENV, "PYCALL_JL_RUNTIME_PYTHONHOME") - pyhome = ENV["PYCALL_JL_RUNTIME_PYTHONHOME"] - else - pyhome = pythonhome_of(current_python()) - end - elseif conda && Sys.iswindows() + if conda && Sys.iswindows() # some Python modules on Windows need the PATH to include # Anaconda's Library\bin directory in order to find their DLL files ENV["PATH"] = Conda.bin_dir(Conda.ROOTENV) * ";" * get(ENV, "PATH", "") @@ -201,7 +174,7 @@ function __init__() if new_pyversion.major != pyversion.major error("PyCall precompiled with Python $pyversion, but now using Python $new_pyversion; ", - "you need to relaunch Julia and run `using PyPrefernces; PyPreferences.recompile()") + "you need to relaunch Julia and run `using PyPreferences; PyPreferences.recompile()") end copy!(inspect, pyimport("inspect")) diff --git a/deps/depsutils.jl b/src/startup_helpers.jl similarity index 58% rename from deps/depsutils.jl rename to src/startup_helpers.jl index 35ed1e12..2e3d9333 100644 --- a/deps/depsutils.jl +++ b/src/startup_helpers.jl @@ -95,63 +95,3 @@ function pythonenv(cmd::Cmd) setenv(cmd, env) end - -function pythonhome_of(pyprogramname::AbstractString) - if Sys.iswindows() - # PYTHONHOME tells python where to look for both pure python - # and binary modules. When it is set, it replaces both - # `prefix` and `exec_prefix` and we thus need to set it to - # both in case they differ. This is also what the - # documentation recommends. However, they are documented - # to always be the same on Windows, where it causes - # problems if we try to include both. - script = """ - import sys - if hasattr(sys, "base_exec_prefix"): - sys.stdout.write(sys.base_exec_prefix) - else: - sys.stdout.write(sys.exec_prefix) - """ - else - script = """ - import sys - if hasattr(sys, "base_exec_prefix"): - sys.stdout.write(sys.base_prefix) - sys.stdout.write(":") - sys.stdout.write(sys.base_exec_prefix) - else: - sys.stdout.write(sys.prefix) - sys.stdout.write(":") - sys.stdout.write(sys.exec_prefix) - """ - # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHOME - end - return read(pythonenv(`$pyprogramname -c $script`), String) -end -# To support `venv` standard library (as well as `virtualenv`), we -# need to use `sys.base_prefix` and `sys.base_exec_prefix` here. -# Otherwise, initializing Python in `__init__` below fails with -# unrecoverable error: -# -# Fatal Python error: initfsencoding: unable to load the file system codec -# ModuleNotFoundError: No module named 'encodings' -# -# This is because `venv` does not symlink standard libraries like -# `virtualenv`. For example, `lib/python3.X/encodings` does not -# exist. Rather, `venv` relies on the behavior of Python runtime: -# -# If a file named "pyvenv.cfg" exists one directory above -# sys.executable, sys.prefix and sys.exec_prefix are set to that -# directory and it is also checked for site-packages -# --- https://docs.python.org/3/library/venv.html -# -# Thus, we need point `PYTHONHOME` to `sys.base_prefix` and -# `sys.base_exec_prefix`. If the virtual environment is created by -# `virtualenv`, those `sys.base_*` paths point to the virtual -# environment. Thus, above code supports both use cases. -# -# See also: -# * https://docs.python.org/3/library/venv.html -# * https://docs.python.org/3/library/site.html -# * https://docs.python.org/3/library/sys.html#sys.base_exec_prefix -# * https://github.com/JuliaPy/PyCall.jl/issues/410 diff --git a/test/runtests.jl b/test/runtests.jl index 4594cc40..1529ecc4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,17 @@ +using Preferences +pypref_py = lowercase(get(ENV, "PYPREFERENCES_PYTHON", "")) +if pypref_py == "conda" + PyPrefs_UUID = Base.UUID("cc9521c6-0242-4dda-8d66-c47a9d9eec02") + delete_preferences!(PyPrefs_UUID, "python3", force=true) + set_preferences!(PyPrefs_UUID, "conda"=>true) + + using PyPreferences + PyPreferences.use_conda() +end + using PyCall using PyCall: hasproperty +using PyPreferences using Test, Dates, Serialization filter(f, itr) = collect(Iterators.filter(f, itr)) @@ -8,6 +20,9 @@ filter(f, d::AbstractDict) = Base.filter(f, d) PYTHONPATH=get(ENV,"PYTHONPATH","") PYTHONHOME=get(ENV,"PYTHONHOME","") PYTHONEXECUTABLE=get(ENV,"PYTHONEXECUTABLE","") + +PyPreferences.status() + @info "Python version $pyversion from $(PyCall.libpython), PYTHONHOME=$(PyCall.PYTHONHOME)\nENV[PYTHONPATH]=$PYTHONPATH\nENV[PYTHONHOME]=$PYTHONHOME\nENV[PYTHONEXECUTABLE]=$PYTHONEXECUTABLE" @testset "CI setup" begin @@ -16,6 +31,8 @@ PYTHONEXECUTABLE=get(ENV,"PYTHONEXECUTABLE","") end end +include("test_venv.jl") + roundtrip(T, x) = convert(T, PyObject(x)) roundtrip(x) = roundtrip(PyAny, x) roundtripeq(T, x) = roundtrip(T, x) == x @@ -817,7 +834,6 @@ include("test_pyfncall.jl") include("testpybuffer.jl") if lowercase(get(ENV, "JULIA_PKGEVAL", "false")) != "true" include("test_venv.jl") - include("test_build.jl") end @testset "@pyinclude" begin diff --git a/test/test_venv.jl b/test/test_venv.jl index 01255462..0716185e 100644 --- a/test/test_venv.jl +++ b/test/test_venv.jl @@ -18,8 +18,14 @@ function test_venv_activation(path) newpython = PyCall.python_cmd(venv=path).exec[1] # Run a fresh Julia process with new Python environment + # must manually set the preferences before loading PyPreferences + # in order not to have to run again the julia process code = """ $(Base.load_path_setup_code()) + using Preferences + PyPrefs_UUID = Base.UUID("cc9521c6-0242-4dda-8d66-c47a9d9eec02") + delete_preferences!(PyPrefs_UUID, "conda", force=true) + set_preferences!(PyPrefs_UUID, "python"=>"$newpython", force=true) using PyCall println(PyCall.pyimport("sys").executable) println(PyCall.pyimport("sys").exec_prefix) @@ -29,7 +35,6 @@ function test_venv_activation(path) # library. Using standard library like `os` does not work # because those files are not created. env = copy(ENV) - env["PYCALL_JL_RUNTIME_PYTHON"] = newpython jlcmd = setenv(`$(Base.julia_cmd()) --startup-file=no -e $code`, env) if Sys.iswindows() # Marking the test broken in Windows. It seems that From d9da0699df938d59f7f1f1bbdc9f71976347aef6 Mon Sep 17 00:00:00 2001 From: Filippo Vicentini Date: Tue, 18 Jan 2022 23:55:35 +0100 Subject: [PATCH 4/5] add tmate --- .github/workflows/system.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index 6a36c2b5..669053f3 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -87,3 +87,6 @@ jobs: file: ./lcov.info flags: unittests name: codecov-umbrella + - name: tmate session if tests fail + if: failure() && ${{ matrix.os == windows-latest }} + uses: mxschmitt/action-tmate@v3 From 7c8609646cacde0d4ed1523be891041f2c829055 Mon Sep 17 00:00:00 2001 From: Filippo Vicentini Date: Wed, 19 Jan 2022 00:05:35 +0100 Subject: [PATCH 5/5] fix --- .github/workflows/system.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/system.yml b/.github/workflows/system.yml index 669053f3..cb860c61 100644 --- a/.github/workflows/system.yml +++ b/.github/workflows/system.yml @@ -88,5 +88,5 @@ jobs: flags: unittests name: codecov-umbrella - name: tmate session if tests fail - if: failure() && ${{ matrix.os == windows-latest }} + if: failure() uses: mxschmitt/action-tmate@v3