Skip to content

Commit 4ef6b00

Browse files
authored
Fixes for python in /bin, /usr/bin, /usr/local/bin (#54)
For #53
1 parent 786b985 commit 4ef6b00

File tree

3 files changed

+126
-96
lines changed

3 files changed

+126
-96
lines changed

crates/pet-linux-global-python/src/lib.rs

Lines changed: 62 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
// Copyright (c) Microsoft Corporation.
22
// Licensed under the MIT License.
33

4-
use std::{fs, path::PathBuf};
4+
use std::fs;
55

6-
use log::error;
6+
use log::warn;
77
use pet_core::{
88
python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentCategory},
99
reporter::Reporter,
@@ -44,34 +44,35 @@ impl Locator for LinuxGlobalPython {
4444
return None;
4545
}
4646

47-
if let (Some(prefix), Some(_)) = (env.prefix.clone(), env.version.clone()) {
48-
let executable = env.executable.clone();
49-
50-
// If prefix or version is not available, then we cannot use this method.
51-
// 1. For files in /bin or /usr/bin, the prefix is always /usr
52-
// 2. For files in /usr/local/bin, the prefix is always /usr/local
53-
if !executable.starts_with("/bin")
54-
&& !executable.starts_with("/usr/bin")
55-
&& !executable.starts_with("/usr/local/bin")
56-
&& !prefix.starts_with("/usr")
57-
&& !prefix.starts_with("/usr/local")
58-
{
59-
return None;
60-
}
47+
// If we do not have a version, then we cannot use this method.
48+
// Without version means we have not spawned the Python exe, thus do not have the real info.
49+
env.version.clone()?;
50+
let prefix = env.prefix.clone()?;
51+
let executable = env.executable.clone();
52+
53+
// If prefix or version is not available, then we cannot use this method.
54+
// 1. For files in /bin or /usr/bin, the prefix is always /usr
55+
// 2. For files in /usr/local/bin, the prefix is always /usr/local
56+
if !executable.starts_with("/bin")
57+
&& !executable.starts_with("/usr/bin")
58+
&& !executable.starts_with("/usr/local/bin")
59+
&& !prefix.starts_with("/usr")
60+
&& !prefix.starts_with("/usr/local")
61+
{
62+
return None;
63+
}
6164

62-
// All known global linux are always installed in `/bin` or `/usr/bin`
63-
if executable.starts_with("/bin") || executable.starts_with("/usr/bin") {
64-
get_python_in_usr_bin(env)
65-
} else if executable.starts_with("/usr/local/bin") {
66-
get_python_in_usr_local_bin(env)
67-
} else {
68-
error!(
69-
"Invalid state, ex ({:?}) is not in any of /bin, /usr/bin, /usr/local/bin",
65+
// All known global linux are always installed in `/bin` or `/usr/bin` or `/usr/local/bin`
66+
if executable.starts_with("/bin")
67+
|| executable.starts_with("/usr/bin")
68+
|| executable.starts_with("/usr/local/bin")
69+
{
70+
get_python_in_bin(env)
71+
} else {
72+
warn!(
73+
"Unknown Python exe ({:?}), not in any of the known locations /bin, /usr/bin, /usr/local/bin",
7074
executable
7175
);
72-
None
73-
}
74-
} else {
7576
None
7677
}
7778
}
@@ -84,7 +85,7 @@ impl Locator for LinuxGlobalPython {
8485
}
8586
}
8687

87-
fn get_python_in_usr_bin(env: &PythonEnv) -> Option<PythonEnvironment> {
88+
fn get_python_in_bin(env: &PythonEnv) -> Option<PythonEnvironment> {
8889
// If we do not have the prefix, then do not try
8990
// This method will be called with resolved Python where prefix & version is available.
9091
if env.version.clone().is_none() || env.prefix.clone().is_none() {
@@ -94,86 +95,54 @@ fn get_python_in_usr_bin(env: &PythonEnv) -> Option<PythonEnvironment> {
9495
let mut symlinks = env.symlinks.clone().unwrap_or_default();
9596
symlinks.push(executable.clone());
9697

97-
let bin = PathBuf::from("/bin");
98-
let usr_bin = PathBuf::from("/usr/bin");
98+
let bin = executable.parent()?;
99+
100+
// Keep track of what the exe resolves to.
101+
// Will have a value only if the exe is in another dir
102+
// E.g. /bin/python3 might be a symlink to /usr/bin/python3.12
103+
// However due to legacy reasons we'll be treating these two as separate exes.
104+
// Hence they will be separate Python environments.
105+
let mut resolved_exe_is_from_another_dir = None;
99106

100107
// Possible this exe is a symlink to another file in the same directory.
101-
// E.g. /usr/bin/python3 is a symlink to /usr/bin/python3.12
108+
// E.g. Generally /usr/bin/python3 is a symlink to /usr/bin/python3.12
109+
// E.g. Generally /usr/local/bin/python3 is a symlink to /usr/local/bin/python3.12
110+
// E.g. Generally /bin/python3 is a symlink to /bin/python3.12
102111
// let bin = executable.parent()?;
103112
// We use canonicalize to get the real path of the symlink.
104113
// Only used in this case, see notes for resolve_symlink.
105114
if let Some(symlink) = resolve_symlink(&executable).or(fs::canonicalize(&executable).ok()) {
106115
// Ensure this is a symlink in the bin or usr/bin directory.
107-
if symlink.starts_with(&bin) || symlink.starts_with(&usr_bin) {
116+
if symlink.starts_with(bin) {
108117
symlinks.push(symlink);
118+
} else {
119+
resolved_exe_is_from_another_dir = Some(symlink);
109120
}
110121
}
111122

112-
// Look for other symlinks in /usr/bin and /bin folder
113-
// https://stackoverflow.com/questions/68728225/what-is-the-difference-between-usr-bin-python3-and-bin-python3
114-
// We know that on linux there are symlinks in both places.
123+
// Look for other symlinks in the same folder
124+
// We know that on linux there are sym links in the same folder as the exe.
115125
// & they all point to one exe and have the same version and same prefix.
116-
for possible_symlink in [find_executables(&bin), find_executables(&usr_bin)].concat() {
117-
if let Some(symlink) =
118-
resolve_symlink(&possible_symlink).or(fs::canonicalize(&possible_symlink).ok())
126+
for possible_symlink in find_executables(bin).iter() {
127+
if let Some(ref symlink) =
128+
resolve_symlink(&possible_symlink).or(fs::canonicalize(possible_symlink).ok())
119129
{
120-
// the file /bin/python3 is a symlink to /usr/bin/python3.12
121-
// the file /bin/python3.12 is a symlink to /usr/bin/python3.12
122-
// the file /usr/bin/python3 is a symlink to /usr/bin/python3.12
123-
// Thus we have 3 symlinks pointing to the same exe /usr/bin/python3.12
124-
if symlinks.contains(&symlink) {
125-
symlinks.push(possible_symlink);
130+
// Generally the file /bin/python3 is a symlink to /usr/bin/python3.12
131+
// Generally the file /bin/python3.12 is a symlink to /usr/bin/python3.12
132+
// Generally the file /usr/bin/python3 is a symlink to /usr/bin/python3.12
133+
// HOWEVER, we will be treating the files in /bin and /usr/bin as different.
134+
// Hence check whether the resolve symlink is in the same directory.
135+
if symlink.starts_with(bin) & symlinks.contains(symlink) {
136+
symlinks.push(possible_symlink.to_owned());
126137
}
127-
}
128-
}
129-
symlinks.sort();
130-
symlinks.dedup();
131-
132-
Some(
133-
PythonEnvironmentBuilder::new(PythonEnvironmentCategory::LinuxGlobal)
134-
.executable(Some(executable))
135-
.version(env.version.clone())
136-
.prefix(env.prefix.clone())
137-
.symlinks(Some(symlinks))
138-
.build(),
139-
)
140-
}
141-
142-
fn get_python_in_usr_local_bin(env: &PythonEnv) -> Option<PythonEnvironment> {
143-
// If we do not have the prefix, then do not try
144-
// This method will be called with resolved Python where prefix & version is available.
145-
if env.version.clone().is_none() || env.prefix.clone().is_none() {
146-
return None;
147-
}
148-
let executable = env.executable.clone();
149-
let mut symlinks = env.symlinks.clone().unwrap_or_default();
150-
symlinks.push(executable.clone());
151138

152-
let usr_local_bin = PathBuf::from("/usr/local/bin");
153-
154-
// Possible this exe is a symlink to another file in the same directory.
155-
// E.g. /usr/local/bin/python3 could be a symlink to /usr/local/bin/python3.12
156-
// let bin = executable.parent()?;
157-
// We use canonicalize to get the real path of the symlink.
158-
// Only used in this case, see notes for resolve_symlink.
159-
if let Some(symlink) = resolve_symlink(&executable).or(fs::canonicalize(&executable).ok()) {
160-
// Ensure this is a symlink in the bin or usr/local/bin directory.
161-
if symlink.starts_with(&usr_local_bin) {
162-
symlinks.push(symlink);
163-
}
164-
}
165-
166-
// Look for other symlinks in this same folder
167-
for possible_symlink in find_executables(&usr_local_bin) {
168-
if let Some(symlink) =
169-
resolve_symlink(&possible_symlink).or(fs::canonicalize(&possible_symlink).ok())
170-
{
171-
// the file /bin/python3 is a symlink to /usr/bin/python3.12
172-
// the file /bin/python3.12 is a symlink to /usr/bin/python3.12
173-
// the file /usr/bin/python3 is a symlink to /usr/bin/python3.12
174-
// Thus we have 3 symlinks pointing to the same exe /usr/bin/python3.12
175-
if symlinks.contains(&symlink) {
176-
symlinks.push(possible_symlink);
139+
// Possible the env.executable = /bin/python3
140+
// And the possible_symlink = /bin/python3.12
141+
// & possible that both of the above are symlinks and point to /usr/bin/python3.12
142+
// In this case /bin/python3 === /bin/python.3.12
143+
// However as mentioned earlier we will not be treating these the same as /usr/bin/python3.12
144+
if resolved_exe_is_from_another_dir == Some(symlink.to_owned()) {
145+
symlinks.push(possible_symlink.to_owned());
177146
}
178147
}
179148
}

crates/pet/src/find.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ use pet_python_utils::env::PythonEnv;
1414
use pet_python_utils::executable::{
1515
find_executable, find_executables, should_search_for_environments_in_path,
1616
};
17-
use std::collections::HashMap;
17+
use std::collections::BTreeMap;
1818
use std::fs;
1919
use std::path::PathBuf;
2020
use std::sync::Mutex;
@@ -25,7 +25,7 @@ use crate::locators::identify_python_environment_using_locators;
2525

2626
pub struct Summary {
2727
pub time: Duration,
28-
pub find_locators_times: HashMap<&'static str, Duration>,
28+
pub find_locators_times: BTreeMap<&'static str, Duration>,
2929
pub find_locators_time: Duration,
3030
pub find_path_time: Duration,
3131
pub find_global_virtual_envs_time: Duration,
@@ -40,7 +40,7 @@ pub fn find_and_report_envs(
4040
) -> Arc<Mutex<Summary>> {
4141
let summary = Arc::new(Mutex::new(Summary {
4242
time: Duration::from_secs(0),
43-
find_locators_times: HashMap::new(),
43+
find_locators_times: BTreeMap::new(),
4444
find_locators_time: Duration::from_secs(0),
4545
find_path_time: Duration::from_secs(0),
4646
find_global_virtual_envs_time: Duration::from_secs(0),

crates/pet/tests/ci_test.rs

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,67 @@ fn verify_validity_of_interpreter_info(environment: PythonEnvironment) {
208208
}
209209
}
210210

211+
#[cfg(unix)]
212+
#[cfg(target_os = "linux")]
213+
#[cfg_attr(feature = "ci", test)]
214+
#[allow(dead_code)]
215+
// On linux we /bin/python, /usr/bin/python and /usr/local/python are all separate environments.
216+
fn verify_bin_usr_bin_user_local_are_separate_python_envs() {
217+
use pet::{find::find_and_report_envs, locators::create_locators};
218+
use pet_conda::Conda;
219+
use pet_core::os_environment::EnvironmentApi;
220+
use pet_reporter::test;
221+
use std::sync::Arc;
222+
223+
let reporter = test::create_reporter();
224+
let environment = EnvironmentApi::new();
225+
let conda_locator = Arc::new(Conda::from(&environment));
226+
227+
find_and_report_envs(
228+
&reporter,
229+
Default::default(),
230+
&create_locators(conda_locator.clone()),
231+
conda_locator,
232+
);
233+
234+
let result = reporter.get_result();
235+
let environments = result.environments;
236+
237+
// Python env /bin/python cannot have symlinks in /usr/bin or /usr/local
238+
// Python env /usr/bin/python cannot have symlinks /bin or /usr/local
239+
// Python env /usr/local/bin/python cannot have symlinks in /bin or /usr/bin
240+
let bins = ["/bin", "/usr/bin", "/usr/local/bin"];
241+
for bin in bins.iter() {
242+
if let Some(bin_python) = environments.iter().find(|e| {
243+
e.executable.clone().is_some()
244+
&& e.executable
245+
.clone()
246+
.unwrap()
247+
.parent()
248+
.unwrap()
249+
.starts_with(bin)
250+
}) {
251+
// If the exe is in /bin, then we can never have any symlinks to other folders such as /usr/bin or /usr/local
252+
let other_bins = bins
253+
.iter()
254+
.filter(|b| *b != bin)
255+
.map(|b| PathBuf::from(*b))
256+
.collect::<Vec<PathBuf>>();
257+
if let Some(symlinks) = &bin_python.symlinks {
258+
for symlink in symlinks.iter() {
259+
let parent_of_symlink = symlink.parent().unwrap().to_path_buf();
260+
if other_bins.contains(&parent_of_symlink) {
261+
panic!(
262+
"Python environment {:?} cannot have a symlinks in {:?}",
263+
bin_python, other_bins
264+
);
265+
}
266+
}
267+
}
268+
}
269+
}
270+
}
271+
211272
#[allow(dead_code)]
212273
fn get_conda_exe() -> &'static str {
213274
// On CI we expect conda to be in the current path.

0 commit comments

Comments
 (0)