loading: Delete normpath where possible

Unix path semantics have a design flaw, where if `a` is a symbolic
directory link, `x/a/..` refer's to the link target's parent directory
rather than `x/`. As a result, `normpath` is not in general safe
to run. We were doing this all over the place (either explicitly,
or implicitly in `abspath`) causing us to fail to load files if
their paths contained this pattern. Fix this by rm'ing normpath
in most places in loading and adding an `abspath` kwarg that skips
normalization. Where paths show up in printing, we can possibly
put some back after verifying that the path resolution result
does not change using `samefile`, but let's do that on a
case-by-case basis.

Author: Keno

base/initdefs.jl

@@ -302,15 +302,15 @@ function load_path_expand(env::AbstractString)::Union{String, Nothing}
             path = joinpath(depot, "environments", name)
             isdir(path) || continue
             for proj in project_names
-                file = abspath(path, proj)
+                file = joinpath(path, proj)
                 isfile_casesensitive(file) && return file
             end
         end
         isempty(DEPOT_PATH) && return nothing
-        return abspath(DEPOT_PATH[1], "environments", name, project_names[end])
+        return abspath(DEPOT_PATH[1], "environments", name, project_names[end]; normalize=false)
     end
     # otherwise, it's a path
-    path = abspath(env)
+    path = abspath(env; normalize=false)
     if isdir(path)
         # directory with a project file?
         for proj in project_names
@@ -337,7 +337,7 @@ function active_project(search_load_path::Bool=true)
         # there are backedges here from abspath(::AbstractString, ::String)
         project = project::String
         if !isfile_casesensitive(project) && basename(project) ∉ project_names
-            project = abspath(project, "Project.toml")
+            project = abspath(project, "Project.toml"; normalize=false)
         end
         return project
     end

base/loading.jl

@@ -964,7 +964,7 @@ function project_file_manifest_path(project_file::String)::Union{Nothing,String}
         manifest_path = get(cache.project_file_manifest_path, project_file, missing)
         manifest_path === missing || return manifest_path
     end
-    dir = abspath(dirname(project_file))
+    dir = abspath(dirname(project_file); normalize=false)
     isfile_casesensitive(project_file) || return nothing
     d = parsed_toml(project_file)
     base_manifest = workspace_manifest(project_file)
@@ -974,7 +974,7 @@ function project_file_manifest_path(project_file::String)::Union{Nothing,String}
     explicit_manifest = get(d, "manifest", nothing)::Union{String, Nothing}
     manifest_path = nothing
     if explicit_manifest !== nothing
-        manifest_file = normpath(joinpath(dir, explicit_manifest))
+        manifest_file = joinpath(dir, explicit_manifest)
         if isfile_casesensitive(manifest_file)
             manifest_path = manifest_file
         end
@@ -998,10 +998,10 @@ end
 # given a directory (implicit env from LOAD_PATH) and a name,
 # check if it is an implicit package
 function entry_point_and_project_file_inside(dir::String, name::String)::Union{Tuple{Nothing,Nothing},Tuple{String,Nothing},Tuple{String,String}}
-    path = normpath(joinpath(dir, "src", "$name.jl"))
+    path = joinpath(dir, "src", "$name.jl")
     isfile_casesensitive(path) || return nothing, nothing
     for proj in project_names
-        project_file = normpath(joinpath(dir, proj))
+        project_file = joinpath(dir, proj)
         isfile_casesensitive(project_file) || continue
         return path, project_file
     end
@@ -1018,7 +1018,7 @@ function entry_point_and_project_file(dir::String, name::String)::Union{Tuple{No
     path, project_file = entry_point_and_project_file_inside(dir_jl, name)
     path === nothing || return path, project_file
     # check for less likely case with a bare file and no src directory last to minimize stat calls
-    path = normpath(joinpath(dir, "$name.jl"))
+    path = joinpath(dir, "$name.jl")
     isfile_casesensitive(path) && return path, nothing
     return nothing, nothing
 end
@@ -1038,9 +1038,9 @@ end
 
 # given a path, name, and possibly an entryfile, return the entry point
 function entry_path(path::String, name::String, entryfile::Union{Nothing,String})::String
-    isfile_casesensitive(path) && return normpath(path)
+    isfile_casesensitive(path) && return path
     entrypoint = entryfile === nothing ? joinpath("src", "$name.jl") : entryfile
-    return normpath(joinpath(path, entrypoint))
+    return joinpath(path, entrypoint)
 end
 
 ## explicit project & manifest API ##
@@ -1204,7 +1204,7 @@ function explicit_manifest_uuid_load_spec(project_file::String, pkg::PkgId)::Uni
                     error("failed to find source of parent package: \"$name\"")
                 end
                 parent_path = parent_load_spec.path
-                p = normpath(dirname(parent_path), "..")
+                p = joinpath(dirname(parent_path), "..")
                 return PkgLoadSpec(find_ext_path(p, pkg.name), parent_load_spec.julia_syntax_version)
             end
         end
@@ -1228,7 +1228,7 @@ function explicit_manifest_entry_load_spec(manifest_file::String, pkg::PkgId, en
     path = get(entry, "path", nothing)::Union{Nothing, String}
     entryfile = get(entry, "entryfile", nothing)::Union{Nothing, String}
     if path !== nothing
-        path = entry_path(normpath(abspath(dirname(manifest_file), path)), pkg.name, entryfile)
+        path = entry_path(joinpath(dirname(manifest_file), path), pkg.name, entryfile)
         return PkgLoadSpec(path, syntax_version)
     end
     hash = get(entry, "git-tree-sha1", nothing)::Union{Nothing, String}
@@ -1248,7 +1248,7 @@ function explicit_manifest_entry_load_spec(manifest_file::String, pkg::PkgId, en
     for slug in (version_slug(uuid, hash), version_slug(uuid, hash, 4))
         for depot in DEPOT_PATH
             path = joinpath(depot, "packages", pkg.name, slug)
-            ispath(path) && return PkgLoadSpec(entry_path(abspath(path), pkg.name, entryfile), syntax_version)
+            ispath(path) && return PkgLoadSpec(entry_path(abspath(path; normalize=false), pkg.name, entryfile), syntax_version)
         end
     end
     # no depot contains the package, return missing to stop looking
@@ -3003,9 +3003,9 @@ function require_stdlib(package_uuidkey::PkgId, ext::Union{Nothing, String}, fro
         if from_stdlib
             # first since this is a stdlib, try to look there directly first
             if ext === nothing
-                sourcepath = normpath(env, this_uuidkey.name, "src", this_uuidkey.name * ".jl")
+                sourcepath = joinpath(env, this_uuidkey.name, "src", this_uuidkey.name * ".jl")
             else
-                sourcepath = find_ext_path(normpath(joinpath(env, package_uuidkey.name)), ext)
+                sourcepath = find_ext_path(joinpath(env, package_uuidkey.name), ext)
             end
             set_pkgorigin_version_path(this_uuidkey, sourcepath)
             newm = _require_search_from_serialized(this_uuidkey, PkgLoadSpec(sourcepath, VERSION), UInt128(0), false; DEPOT_PATH=depot_path)
@@ -3191,13 +3191,14 @@ function evalfile(path::AbstractString, args::Vector{String}=String[])
 end
 evalfile(path::AbstractString, args::Vector) = evalfile(path, String[args...])
 
+# Used in Pkg.jl
 function load_path_setup_code(load_path::Bool=true)
     code = """
-    append!(empty!(Base.DEPOT_PATH), $(repr(map(abspath, DEPOT_PATH))))
-    append!(empty!(Base.DL_LOAD_PATH), $(repr(map(abspath, DL_LOAD_PATH))))
+    append!(empty!(Base.DEPOT_PATH), $(repr(map(x -> abspath(x; normalize=false), DEPOT_PATH))))
+    append!(empty!(Base.DL_LOAD_PATH), $(repr(map(x -> abspath(x; normalize=false), DL_LOAD_PATH))))
     """
     if load_path
-        load_path = map(abspath, Base.load_path())
+        load_path = map(x -> abspath(x; normalize=false), Base.load_path())
         path_sep = Sys.iswindows() ? ';' : ':'
         any(path -> path_sep in path, load_path) &&
             error("LOAD_PATH entries cannot contain $(repr(path_sep))")
@@ -3283,9 +3284,9 @@ function create_expr_cache(pkg::PkgId, input::PkgLoadSpec, output::String, outpu
     @nospecialize internal_stderr internal_stdout
     rm(output, force=true)   # Remove file if it exists
     output_o === nothing || rm(output_o, force=true)
-    depot_path = String[abspath(x) for x in DEPOT_PATH]
-    dl_load_path = String[abspath(x) for x in DL_LOAD_PATH]
-    load_path = String[abspath(x) for x in Base.load_path()]
+    depot_path = String[abspath(x; normalize=false) for x in DEPOT_PATH]
+    dl_load_path = String[abspath(x; normalize=false) for x in DL_LOAD_PATH]
+    load_path = String[abspath(x; normalize=false) for x in Base.load_path()]
     # if pkg is a stdlib, append its parent Project.toml to the load path
     triggers = get(EXT_PRIMED, pkg, nothing)
     if triggers !== nothing
@@ -3343,7 +3344,7 @@ function create_expr_cache(pkg::PkgId, input::PkgLoadSpec, output::String, outpu
         Base.track_nested_precomp($(_pkg_str(vcat(Base.precompilation_stack, pkg))))
         Base.loadable_extensions = $(_pkg_str(loadable_exts))
         Base.precompiling_extension = $(loading_extension)
-        Base.include_package_for_output($(_pkg_str(pkg)), $(repr(abspath(input.path))), $(repr(input.julia_syntax_version)), $(repr(depot_path)), $(repr(dl_load_path)),
+        Base.include_package_for_output($(_pkg_str(pkg)), $(repr(abspath(input.path; normalize=false))), $(repr(input.julia_syntax_version)), $(repr(depot_path)), $(repr(dl_load_path)),
             $(repr(load_path)), $(_pkg_str(concrete_deps)), $(repr(source_path(nothing))))
         """)
     close(io.in)
@@ -3372,7 +3373,7 @@ function compilecache_path(pkg::PkgId, prefs_hash::UInt64; flags::CacheFlags=Cac
     cachepath = joinpath(DEPOT_PATH[1], entrypath)
     isdir(cachepath) || mkpath(cachepath)
     if pkg.uuid === nothing
-        abspath(cachepath, entryfile) * ".ji"
+        joinpath(cachepath, entryfile) * ".ji"
     else
         crc = _crc32c(project)
         crc = _crc32c(unsafe_string(JLOptions().image_file), crc)
@@ -3387,7 +3388,7 @@ function compilecache_path(pkg::PkgId, prefs_hash::UInt64; flags::CacheFlags=Cac
 
         crc = _crc32c(prefs_hash, crc)
         project_precompile_slug = slug(crc, 5)
-        abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji"))
+        joinpath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji"))
     end
 end
 
@@ -3612,7 +3613,7 @@ function normalize_depots_for_relocation()
         if isdirpath(d)
             d = dirname(d)
         end
-        push!(depots, abspath(d))
+        push!(depots, realpath(d))
     end
     return depots
 end

base/path.jl

@@ -386,6 +386,13 @@ julia> normpath("/home/myuser/../example.jl")
 julia> normpath("Documents/Julia") == joinpath("Documents", "Julia")
 true
 ```
+
+!!! warning
+    Normalization may change the meaning of a path if the directory immediately
+    preceding a ".." entry is a symbolic link. For example, if `a` is a symbolic
+    link in `a/../b`, then this path will resolve to `b` in the link target's
+    parent directory. Hoewver, `normpath("a/../b")` will return `b`, which will
+    resolve `b` in the current working directory.
 """
 function normpath(path::String)
     isabs = isabspath(path)
@@ -430,10 +437,10 @@ Convert a set of paths to a normalized path by joining them together and removin
 normpath(a::AbstractString, b::AbstractString...) = normpath(joinpath(a,b...))
 
 """
-    abspath(path::AbstractString)::String
+    abspath(path::AbstractString; normalize=true)::String
 
 Convert a path to an absolute path by adding the current directory if necessary.
-Also normalizes the path as in [`normpath`](@ref).
+If `normalize` is true, also normalizes the path as in [`normpath`](@ref).
 
 # Examples
 
@@ -444,8 +451,11 @@ If you are in a directory called `JuliaExample` and the data you are using is tw
 Which gives a path like `"/home/JuliaUser/data/"`.
 
 See also [`joinpath`](@ref), [`pwd`](@ref), [`expanduser`](@ref).
+
+!!! compat "Julia 1.14"
+    The `normalize` keyword argument was added in Julia 1.14.
 """
-function abspath(a::String)::String
+function abspath(a::String; normalize=true)::String
     if !isabspath(a)
         cwd = pwd()
         a_drive, a_nodrive = splitdrive(a)
@@ -456,7 +466,7 @@ function abspath(a::String)::String
             a = joinpath(cwd, a)
         end
     end
-    return normpath(a)
+    return normalize ? normpath(a) : a
 end
 
 """
@@ -465,7 +475,7 @@ end
 Convert a set of paths to an absolute path by joining them together and adding the
 current directory if necessary. Equivalent to `abspath(joinpath(path, paths...))`.
 """
-abspath(a::AbstractString, b::AbstractString...) = abspath(joinpath(a,b...))
+abspath(a::AbstractString, b::AbstractString...; kwargs...) = abspath(joinpath(a,b...); kwargs...)
 
 if Sys.iswindows()
 

test/cmdlineargs.jl

@@ -294,25 +294,25 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no`
 
     # ~ expansion in --project and JULIA_PROJECT
     if !Sys.iswindows()
-        let expanded = abspath(expanduser("~/foo/Project.toml"))
+        let expanded = abspath(expanduser("~/foo/Project.toml"); normalize=false)
             @test expanded == readchomp(`$exename --project='~/foo' -e 'println(Base.active_project())'`)
             @test expanded == readchomp(setenv(`$exename -e 'println(Base.active_project())'`, "JULIA_PROJECT" => "~/foo", "HOME" => homedir()))
         end
     end
 
     # handling of @projectname in --project and JULIA_PROJECT
-    let expanded = abspath(Base.load_path_expand("@foo"))
+    let expanded = abspath(Base.load_path_expand("@foo"); normalize=false)
         @test expanded == readchomp(`$exename --project='@foo' -e 'println(Base.active_project())'`)
         @test expanded == readchomp(addenv(`$exename -e 'println(Base.active_project())'`, "JULIA_PROJECT" => "@foo", "HOME" => homedir()))
     end
 
     # --project=@script handling
-    let expanded = abspath(joinpath(@__DIR__, "project", "ScriptProject"))
+    let expanded = abspath(joinpath(@__DIR__, "project", "ScriptProject"); normalize=false)
         script = joinpath(expanded, "bin", "script.jl")
         # Check running julia with --project=@script both within and outside the script directory
         @testset "--@script from $name" for (name, dir) in [("project", expanded), ("outside", pwd())]
-            @test joinpath(expanded, "Project.toml") == readchomp(Cmd(`$exename --project=@script $script`; dir))
-            @test joinpath(expanded, "SubProject", "Project.toml") == readchomp(Cmd(`$exename --project=@script/../SubProject $script`; dir))
+            @test samefile(joinpath(expanded, "Project.toml"), readchomp(Cmd(`$exename --project=@script $script`; dir)))
+            @test samefile(joinpath(expanded, "SubProject", "Project.toml"), readchomp(Cmd(`$exename --project=@script/../SubProject $script`; dir)))
         end
     end
 

test/loading.jl

@@ -274,7 +274,7 @@ end
         n = map(String, split(names, '.'))
         pkg = recurse_package(n...)
         @test pkg == PkgId(UUID(uuid), n[end])
-        @test joinpath(@__DIR__, normpath(path)) == locate_package(pkg)
+        @test samefile(joinpath(@__DIR__, path), locate_package(pkg))
         @test Base.compilecache_path(pkg, UInt64(0)) == Base.compilecache_path(pkg, UInt64(0))
     end
     @test identify_package("Baz") === nothing
@@ -350,19 +350,19 @@ module NotPkgModule; end
     @test Foo.which == "path"
 
     @testset "pathof" begin
-        @test pathof(Foo) == normpath(abspath(@__DIR__, "project/deps/Foo1/src/Foo.jl"))
+        @test samefile(pathof(Foo), joinpath(@__DIR__, "project/deps/Foo1/src/Foo.jl"))
         @test pathof(NotPkgModule) === nothing
     end
 
     @testset "pkgdir" begin
-        @test pkgdir(Foo) == normpath(abspath(@__DIR__, "project/deps/Foo1"))
-        @test pkgdir(Foo.SubFoo1) == normpath(abspath(@__DIR__, "project/deps/Foo1"))
-        @test pkgdir(Foo.SubFoo2) == normpath(abspath(@__DIR__, "project/deps/Foo1"))
+        @test samefile(pkgdir(Foo), joinpath(@__DIR__, "project/deps/Foo1"))
+        @test samefile(pkgdir(Foo.SubFoo1), joinpath(@__DIR__, "project/deps/Foo1"))
+        @test samefile(pkgdir(Foo.SubFoo2), joinpath(@__DIR__, "project/deps/Foo1"))
         @test pkgdir(NotPkgModule) === nothing
 
-        @test pkgdir(Foo, "src") == normpath(abspath(@__DIR__, "project/deps/Foo1/src"))
-        @test pkgdir(Foo.SubFoo1, "src") == normpath(abspath(@__DIR__, "project/deps/Foo1/src"))
-        @test pkgdir(Foo.SubFoo2, "src") == normpath(abspath(@__DIR__, "project/deps/Foo1/src"))
+        @test samefile(pkgdir(Foo, "src"), joinpath(@__DIR__, "project/deps/Foo1/src"))
+        @test samefile(pkgdir(Foo.SubFoo1, "src"), joinpath(@__DIR__, "project/deps/Foo1/src"))
+        @test samefile(pkgdir(Foo.SubFoo2, "src"), joinpath(@__DIR__, "project/deps/Foo1/src"))
         @test pkgdir(NotPkgModule, "src") === nothing
     end
 
@@ -1146,7 +1146,7 @@ end
         _ext = Base.get_extension(parent, ext)
         _ext isa Module || error("expected extension \$ext to be loaded")
         _pkgdir = pkgdir(_ext)
-        _pkgdir == pkgdir(parent) != nothing || error("unexpected extension \$ext pkgdir path: \$_pkgdir")
+        samefile(_pkgdir, pkgdir(parent)) || error("unexpected extension \$ext pkgdir path: \$_pkgdir")
         _pkgversion = pkgversion(_ext)
         _pkgversion == pkgversion(parent) || error("unexpected extension \$ext version: \$_pkgversion")
     end