From 81daea252076379ab869761040f0c86bbc9e7488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Romain=20Tarti=C3=A8re?= Date: Thu, 24 Apr 2025 11:11:07 -1000 Subject: [PATCH 1/2] Add a unit test to illustrate the pyvenv bootstrap issue This module that manage python needs python to be installed in order to work properly under certain conditions. When python is not installed on a node, the python_version fact does not have a value. When using `python::pyvenv` on such a system without a specific version set, a call of the split() function on this unde value cause an error te be raised: ``` Error: Evaluation Error: Error while evaluating a Resource Statement, Evaluation Error: Error while evaluating a Function Call, The function 'split' was called with arguments it does not accept. It expects one of: (String str, String pattern) rejected: parameter 'str' expects a String value, got Undef (String str, Regexp pattern) rejected: parameter 'str' expects a String value, got Undef (String str, Type[Regexp] pattern) rejected: parameter 'str' expects a String value, got Undef (Sensitive[String] sensitive, String pattern) rejected: parameter 'sensitive' expects a Sensitive[String] value, got Undef (Sensitive[String] sensitive, Regexp pattern) rejected: parameter 'sensitive' expects a Sensitive[String] value, got Undef (Sensitive[String] sensitive, Type[Regexp] pattern) rejected: parameter 'sensitive' expects a Sensitive[String] value, got Undef (file: /etc/puppetlabs/code/environments/production/modules/python/manifests/pyvenv.pp, line: 48, column: 34) (file: /etc/puppetlabs/code/environments/production/modules/taiga/manifests/back/install.pp, line: 15) on node debian12-64-puppet7.example.com ``` --- spec/defines/pyvenv_spec.rb | 125 ++++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 57 deletions(-) diff --git a/spec/defines/pyvenv_spec.rb b/spec/defines/pyvenv_spec.rb index ba49fb37..b0578bec 100644 --- a/spec/defines/pyvenv_spec.rb +++ b/spec/defines/pyvenv_spec.rb @@ -6,83 +6,94 @@ on_supported_os.each do |os, facts| next if os == 'gentoo-3-x86_64' + let :title do + '/opt/env' + end + context "on #{os}" do - let :facts do - # python3 is required to use pyvenv - facts.merge( - python3_version: '3.5.1' - ) - end - let :title do - '/opt/env' + context 'with default parameters' do + let :facts do + facts + end + + it { is_expected.to compile } end - context 'with default parameters' do - it { is_expected.to contain_file('/opt/env').that_requires('Class[python::install]') } - it { is_expected.to contain_exec('python_virtualenv_/opt/env').with_command('pyvenv-3.5 --clear /opt/env && /opt/env/bin/pip --log /opt/env/pip.log install --upgrade pip && /opt/env/bin/pip --log /opt/env/pip.log install --upgrade setuptools') } + context 'with a specific python3 version' do + let :facts do + # python3 is required to use pyvenv + facts.merge( + python3_version: '3.5.1' + ) + end + + context 'with default parameters' do + it { is_expected.to contain_file('/opt/env').that_requires('Class[python::install]') } + it { is_expected.to contain_exec('python_virtualenv_/opt/env').with_command('pyvenv-3.5 --clear /opt/env && /opt/env/bin/pip --log /opt/env/pip.log install --upgrade pip && /opt/env/bin/pip --log /opt/env/pip.log install --upgrade setuptools') } + end + + describe 'when ensure' do + context 'is absent' do + let :params do + { + ensure: 'absent' + } + end + + it { + expect(subject).to contain_file('/opt/env').with_ensure('absent').with_purge(true) + } + end + end end - describe 'when ensure' do - context 'is absent' do + context "prompt on #{os} with python 3.6" do + let :facts do + # python 3.6 is required for venv and prompt + facts.merge( + python3_version: '3.6.1' + ) + end + let :title do + '/opt/env' + end + + context 'with prompt' do let :params do { - ensure: 'absent' + prompt: 'custom prompt', } end it { - expect(subject).to contain_file('/opt/env').with_ensure('absent').with_purge(true) + is_expected.to contain_file('/opt/env').that_requires('Class[python::install]') + is_expected.to contain_exec('python_virtualenv_/opt/env').with_command('python3.6 -m venv --clear --prompt custom\\ prompt /opt/env && /opt/env/bin/pip --log /opt/env/pip.log install --upgrade pip && /opt/env/bin/pip --log /opt/env/pip.log install --upgrade setuptools') } end end - end - context "prompt on #{os} with python 3.6" do - let :facts do - # python 3.6 is required for venv and prompt - facts.merge( - python3_version: '3.6.1' - ) - end - let :title do - '/opt/env' - end - - context 'with prompt' do - let :params do - { - prompt: 'custom prompt', - } + context "prompt on #{os} with python 3.5" do + let :facts do + facts.merge( + python3_version: '3.5.1' + ) + end + let :title do + '/opt/env' end - it { - is_expected.to contain_file('/opt/env').that_requires('Class[python::install]') - is_expected.to contain_exec('python_virtualenv_/opt/env').with_command('python3.6 -m venv --clear --prompt custom\\ prompt /opt/env && /opt/env/bin/pip --log /opt/env/pip.log install --upgrade pip && /opt/env/bin/pip --log /opt/env/pip.log install --upgrade setuptools') - } - end - end - - context "prompt on #{os} with python 3.5" do - let :facts do - facts.merge( - python3_version: '3.5.1' - ) - end - let :title do - '/opt/env' - end + context 'with prompt' do + let :params do + { + prompt: 'custom prompt', + } + end - context 'with prompt' do - let :params do - { - prompt: 'custom prompt', + it { + is_expected.to contain_file('/opt/env').that_requires('Class[python::install]') + is_expected.to contain_exec('python_virtualenv_/opt/env').with_command('pyvenv-3.5 --clear /opt/env && /opt/env/bin/pip --log /opt/env/pip.log install --upgrade pip && /opt/env/bin/pip --log /opt/env/pip.log install --upgrade setuptools') } end - - it { - is_expected.to contain_file('/opt/env').that_requires('Class[python::install]') - is_expected.to contain_exec('python_virtualenv_/opt/env').with_command('pyvenv-3.5 --clear /opt/env && /opt/env/bin/pip --log /opt/env/pip.log install --upgrade pip && /opt/env/bin/pip --log /opt/env/pip.log install --upgrade setuptools') - } end end end From 9ad7d6758197ce6df762e93ead1a65670942f5e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Romain=20Tarti=C3=A8re?= Date: Thu, 24 Apr 2025 12:00:02 -1000 Subject: [PATCH 2/2] Add fallback on the default system version of Python When Python is not installed, the python_version fact is undef, but we need to know which version of Python is going to be installed by the module in order to use the proper command invocation for building virtual envs. This issue only occurs when the user use the system version of Python, so add the default version of Python for supported operating systems into Hiera and use that as a fallback when the version of Python is unknown. --- REFERENCE.md | 7 +++++++ data/common.yaml | 1 + data/os/Archlinux.yaml | 1 + data/os/Debian/11.yaml | 1 + data/os/Debian/12.yaml | 1 + data/os/FreeBSD.yaml | 1 + data/os/Gentoo.yaml | 1 + data/os/RedHat/8.yaml | 1 + data/os/RedHat/9.yaml | 1 + data/os/Ubuntu/20.04.yaml | 1 + data/os/Ubuntu/22.04.yaml | 1 + data/os/Ubuntu/24.04.yaml | 1 + hiera.yaml | 23 +++++++++++++++++++++++ manifests/init.pp | 2 ++ manifests/pyvenv.pp | 2 +- 15 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 data/common.yaml create mode 100644 data/os/Archlinux.yaml create mode 100644 data/os/Debian/11.yaml create mode 100644 data/os/Debian/12.yaml create mode 100644 data/os/FreeBSD.yaml create mode 100644 data/os/Gentoo.yaml create mode 100644 data/os/RedHat/8.yaml create mode 100644 data/os/RedHat/9.yaml create mode 100644 data/os/Ubuntu/20.04.yaml create mode 100644 data/os/Ubuntu/22.04.yaml create mode 100644 data/os/Ubuntu/24.04.yaml create mode 100644 hiera.yaml diff --git a/REFERENCE.md b/REFERENCE.md index d690f047..4352dd43 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -67,6 +67,7 @@ class { 'python' : The following parameters are available in the `python` class: +* [`default_system_version`](#-python--default_system_version) * [`ensure`](#-python--ensure) * [`version`](#-python--version) * [`pip`](#-python--pip) @@ -92,6 +93,12 @@ The following parameters are available in the `python` class: * [`anaconda_installer_url`](#-python--anaconda_installer_url) * [`anaconda_install_path`](#-python--anaconda_install_path) +##### `default_system_version` + +Data type: `Python::Version` + +The default version of Python provided by the operating system. Only used as a fallback if Python is not installed yet to determine how to handle some actions that vary depending on the Python version used. + ##### `ensure` Data type: `Python::Package::Ensure` diff --git a/data/common.yaml b/data/common.yaml new file mode 100644 index 00000000..8d945465 --- /dev/null +++ b/data/common.yaml @@ -0,0 +1 @@ +python::default_system_version: "3.11" diff --git a/data/os/Archlinux.yaml b/data/os/Archlinux.yaml new file mode 100644 index 00000000..3bd58ed5 --- /dev/null +++ b/data/os/Archlinux.yaml @@ -0,0 +1 @@ +python::default_system_version: "3.13" diff --git a/data/os/Debian/11.yaml b/data/os/Debian/11.yaml new file mode 100644 index 00000000..c3b18ece --- /dev/null +++ b/data/os/Debian/11.yaml @@ -0,0 +1 @@ +python::default_system_version: "3.9" diff --git a/data/os/Debian/12.yaml b/data/os/Debian/12.yaml new file mode 100644 index 00000000..8d945465 --- /dev/null +++ b/data/os/Debian/12.yaml @@ -0,0 +1 @@ +python::default_system_version: "3.11" diff --git a/data/os/FreeBSD.yaml b/data/os/FreeBSD.yaml new file mode 100644 index 00000000..8d945465 --- /dev/null +++ b/data/os/FreeBSD.yaml @@ -0,0 +1 @@ +python::default_system_version: "3.11" diff --git a/data/os/Gentoo.yaml b/data/os/Gentoo.yaml new file mode 100644 index 00000000..68b52939 --- /dev/null +++ b/data/os/Gentoo.yaml @@ -0,0 +1 @@ +python::default_system_version: "3.12" diff --git a/data/os/RedHat/8.yaml b/data/os/RedHat/8.yaml new file mode 100644 index 00000000..5de86ac2 --- /dev/null +++ b/data/os/RedHat/8.yaml @@ -0,0 +1 @@ +python::default_system_version: "3.6" diff --git a/data/os/RedHat/9.yaml b/data/os/RedHat/9.yaml new file mode 100644 index 00000000..c3b18ece --- /dev/null +++ b/data/os/RedHat/9.yaml @@ -0,0 +1 @@ +python::default_system_version: "3.9" diff --git a/data/os/Ubuntu/20.04.yaml b/data/os/Ubuntu/20.04.yaml new file mode 100644 index 00000000..20aa9c3e --- /dev/null +++ b/data/os/Ubuntu/20.04.yaml @@ -0,0 +1 @@ +python::default_system_version: "3.8" diff --git a/data/os/Ubuntu/22.04.yaml b/data/os/Ubuntu/22.04.yaml new file mode 100644 index 00000000..383744c0 --- /dev/null +++ b/data/os/Ubuntu/22.04.yaml @@ -0,0 +1 @@ +python::default_system_version: "3.10" diff --git a/data/os/Ubuntu/24.04.yaml b/data/os/Ubuntu/24.04.yaml new file mode 100644 index 00000000..68b52939 --- /dev/null +++ b/data/os/Ubuntu/24.04.yaml @@ -0,0 +1 @@ +python::default_system_version: "3.12" diff --git a/hiera.yaml b/hiera.yaml new file mode 100644 index 00000000..9cea3e4d --- /dev/null +++ b/hiera.yaml @@ -0,0 +1,23 @@ +--- +version: 5 + +defaults: # Used for any hierarchy level that omits these keys. + datadir: data # This path is relative to hiera.yaml's directory. + data_hash: yaml_data # Use the built-in YAML backend. + +hierarchy: + - name: "archicture" + paths: + - "architecture/%{facts.os.architecture}.yaml" + - "architecture/common.yaml" + - name: "osfamily/major release" + paths: + - "os/%{facts.os.name}/%{facts.os.release.major}.yaml" # Used to distinguish between Debian and Ubuntu + - "os/%{facts.os.family}/%{facts.os.release.major}.yaml" # + - "os/%{facts.os.family}/%{facts.kernelrelease}.yaml" # Used for Solaris + - name: "osfamily" + paths: + - "os/%{facts.os.name}.yaml" + - "os/%{facts.os.family}.yaml" + - name: 'common' + path: 'common.yaml' diff --git a/manifests/init.pp b/manifests/init.pp index b457edbd..3f5ac23f 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -1,5 +1,6 @@ # @summary Installs and manages python, python-dev and gunicorn. # +# @param default_system_version The default version of Python provided by the operating system. Only used as a fallback if Python is not installed yet to determine how to handle some actions that vary depending on the Python version used. # @param ensure Desired installation state for the Python package. # @param version Python version to install. Beware that valid values for this differ a) by the provider you choose and b) by the osfamily/operatingsystem you are using. # Allowed values: @@ -38,6 +39,7 @@ # } # class python ( + Python::Version $default_system_version, Python::Package::Ensure $ensure = 'present', Python::Version $version = $facts['os']['family'] ? { 'Archlinux' => 'system', default => '3' }, Python::Package::Ensure $pip = 'present', diff --git a/manifests/pyvenv.pp b/manifests/pyvenv.pp index 33b37100..6627a406 100644 --- a/manifests/pyvenv.pp +++ b/manifests/pyvenv.pp @@ -41,7 +41,7 @@ if $ensure == 'present' { $python_version = $version ? { - 'system' => $facts['python3_version'], + 'system' => $facts['python3_version'].lest || { $python::default_system_version }, default => $version, }