From dbd0cdc00607e1679cc1d734a6535be5cfe16c8f Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 19 Sep 2025 09:04:21 +0200 Subject: [PATCH 01/51] refactor --- cmd/pythonBuild.go | 144 +++++++++---------------------------- cmd/pythonBuild_test.go | 41 ++--------- pkg/python/bom.go | 49 +++++++++++++ pkg/python/bom_test.go | 36 ++++++++++ pkg/python/build.go | 32 +++++++++ pkg/python/build_test.go | 34 +++++++++ pkg/python/env.go | 30 ++++++++ pkg/python/env_test.go | 27 +++++++ pkg/python/pip.go | 69 ++++++++++++++++++ pkg/python/pip_test.go | 64 +++++++++++++++++ pkg/python/publish.go | 33 +++++++++ pkg/python/publish_test.go | 36 ++++++++++ 12 files changed, 448 insertions(+), 147 deletions(-) create mode 100644 pkg/python/bom.go create mode 100644 pkg/python/bom_test.go create mode 100644 pkg/python/build.go create mode 100644 pkg/python/build_test.go create mode 100644 pkg/python/env.go create mode 100644 pkg/python/env_test.go create mode 100644 pkg/python/pip.go create mode 100644 pkg/python/pip_test.go create mode 100644 pkg/python/publish.go create mode 100644 pkg/python/publish_test.go diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index 7dfadb06e6..5a04ad097b 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -2,20 +2,19 @@ package cmd import ( "fmt" - "path/filepath" "github.com/SAP/jenkins-library/pkg/buildsettings" "github.com/SAP/jenkins-library/pkg/command" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" + "github.com/SAP/jenkins-library/pkg/python" "github.com/SAP/jenkins-library/pkg/telemetry" ) const ( - PyBomFilename = "bom-pip.xml" - stepName = "pythonBuild" - cycloneDxPackageVersion = "cyclonedx-bom==6.1.1" - cycloneDxSchemaVersion = "1.4" + cycloneDxVersion = "6.1.1" + cycloneDxSchemaVersion = "1.4" + stepName = "pythonBuild" ) type pythonBuildUtils interface { @@ -32,7 +31,7 @@ type pythonBuildUtilsBundle struct { func newPythonBuildUtils() pythonBuildUtils { utils := pythonBuildUtilsBundle{ Command: &command.Command{ - StepName: "pythonBuild", + StepName: stepName, }, Files: &piperutils.Files{}, } @@ -52,134 +51,57 @@ func pythonBuild(config pythonBuildOptions, telemetryData *telemetry.CustomData, } func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomData, utils pythonBuildUtils, commonPipelineEnvironment *pythonBuildCommonPipelineEnvironment) error { - pipInstallFlags := []string{"install", "--upgrade"} - virutalEnvironmentPathMap := make(map[string]string) - - err := createVirtualEnvironment(utils, config, virutalEnvironmentPathMap) - if err != nil { + if err := python.CreateVirtualEnvironment(utils.RunExecutable, config.VirutalEnvironmentName); err != nil { return err } - err = buildExecute(config, utils, pipInstallFlags, virutalEnvironmentPathMap) - if err != nil { - return fmt.Errorf("Python build failed with error: %w", err) + if err := python.BuildWithSetupPy(utils.RunExecutable, config.VirutalEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { + return err } if config.CreateBOM { - if err := runBOMCreationForPy(utils, pipInstallFlags, virutalEnvironmentPathMap, config); err != nil { + if err := python.CreateBOM(utils.RunExecutable, utils.FileExists, config.VirutalEnvironmentName, config.RequirementsFilePath, cycloneDxVersion, cycloneDxSchemaVersion); err != nil { return fmt.Errorf("BOM creation failed: %w", err) } } - log.Entry().Debugf("creating build settings information...") - - dockerImage, err := GetDockerImageValue(stepName) - if err != nil { + if info, err := createBuildSettingsInfo(config); err != nil { return err + } else { + commonPipelineEnvironment.custom.buildSettingsInfo = info } - pythonConfig := buildsettings.BuildOptions{ - CreateBOM: config.CreateBOM, - Publish: config.Publish, - BuildSettingsInfo: config.BuildSettingsInfo, - DockerImage: dockerImage, - } - buildSettingsInfo, err := buildsettings.CreateBuildSettingsInfo(&pythonConfig, stepName) - if err != nil { - log.Entry().Warnf("failed to create build settings info: %v", err) - } - commonPipelineEnvironment.custom.buildSettingsInfo = buildSettingsInfo - if config.Publish { - if err := publishWithTwine(config, utils, pipInstallFlags, virutalEnvironmentPathMap); err != nil { + if err := python.PublishPackage( + utils.RunExecutable, + config.VirutalEnvironmentName, + config.TargetRepositoryURL, + config.TargetRepositoryUser, + config.TargetRepositoryPassword, + ); err != nil { return fmt.Errorf("failed to publish: %w", err) } } - - err = removeVirtualEnvironment(utils, config) - if err != nil { - return err - } - - return nil + return python.RemoveVirtualEnvironment(utils.RemoveAll, config.VirutalEnvironmentName) } -func buildExecute(config *pythonBuildOptions, utils pythonBuildUtils, pipInstallFlags []string, virutalEnvironmentPathMap map[string]string) error { - var flags []string - flags = append(flags, config.BuildFlags...) - flags = append(flags, "setup.py") - flags = append(flags, config.SetupFlags...) - flags = append(flags, "sdist", "bdist_wheel") - - log.Entry().Info("starting building python project:") - err := utils.RunExecutable(virutalEnvironmentPathMap["python"], flags...) +// TODO: extract to common place +func createBuildSettingsInfo(config *pythonBuildOptions) (string, error) { + log.Entry().Debugf("creating build settings information...") + dockerImage, err := GetDockerImageValue(stepName) if err != nil { - return err + return "", err } - return nil -} -func createVirtualEnvironment(utils pythonBuildUtils, config *pythonBuildOptions, virutalEnvironmentPathMap map[string]string) error { - virtualEnvironmentFlags := []string{"-m", "venv", config.VirutalEnvironmentName} - err := utils.RunExecutable("python3", virtualEnvironmentFlags...) - if err != nil { - return err - } - err = utils.RunExecutable("bash", "-c", "source "+filepath.Join(config.VirutalEnvironmentName, "bin", "activate")) - if err != nil { - return err + pythonConfig := buildsettings.BuildOptions{ + CreateBOM: config.CreateBOM, + Publish: config.Publish, + BuildSettingsInfo: config.BuildSettingsInfo, + DockerImage: dockerImage, } - virutalEnvironmentPathMap["pip"] = filepath.Join(config.VirutalEnvironmentName, "bin", "pip") - // venv will create symlinks to python3 inside the container - virutalEnvironmentPathMap["python"] = "python" - virutalEnvironmentPathMap["deactivate"] = filepath.Join(config.VirutalEnvironmentName, "bin", "deactivate") - - return nil -} - -func removeVirtualEnvironment(utils pythonBuildUtils, config *pythonBuildOptions) error { - err := utils.RemoveAll(config.VirutalEnvironmentName) + buildSettingsInfo, err := buildsettings.CreateBuildSettingsInfo(&pythonConfig, stepName) if err != nil { - return err - } - return nil -} - -func runBOMCreationForPy(utils pythonBuildUtils, pipInstallFlags []string, virutalEnvironmentPathMap map[string]string, config *pythonBuildOptions) error { - pipInstallOriginalFlags := pipInstallFlags - exists, _ := utils.FileExists(config.RequirementsFilePath) - if exists { - pipInstallRequirementsFlags := append(pipInstallOriginalFlags, "--requirement", config.RequirementsFilePath) - if err := utils.RunExecutable(virutalEnvironmentPathMap["pip"], pipInstallRequirementsFlags...); err != nil { - return err - } - } else { - log.Entry().Warnf("unable to find requirements.txt file at %s , continuing SBOM generation without requirements.txt", config.RequirementsFilePath) - } - - pipInstallCycloneDxFlags := append(pipInstallOriginalFlags, cycloneDxPackageVersion) - - if err := utils.RunExecutable(virutalEnvironmentPathMap["pip"], pipInstallCycloneDxFlags...); err != nil { - return err - } - virutalEnvironmentPathMap["cyclonedx"] = filepath.Join(config.VirutalEnvironmentName, "bin", "cyclonedx-py") - - if err := utils.RunExecutable(virutalEnvironmentPathMap["cyclonedx"], "env", "--output-file", PyBomFilename, "--output-format", "XML", "--spec-version", cycloneDxSchemaVersion); err != nil { - return err - } - return nil -} - -func publishWithTwine(config *pythonBuildOptions, utils pythonBuildUtils, pipInstallFlags []string, virutalEnvironmentPathMap map[string]string) error { - pipInstallFlags = append(pipInstallFlags, "twine") - if err := utils.RunExecutable(virutalEnvironmentPathMap["pip"], pipInstallFlags...); err != nil { - return err - } - virutalEnvironmentPathMap["twine"] = filepath.Join(config.VirutalEnvironmentName, "bin", "twine") - if err := utils.RunExecutable(virutalEnvironmentPathMap["twine"], "upload", "--username", config.TargetRepositoryUser, - "--password", config.TargetRepositoryPassword, "--repository-url", config.TargetRepositoryURL, "--disable-progress-bar", - "dist/*"); err != nil { - return err + log.Entry().Warnf("failed to create build settings info: %v", err) } - return nil + return buildSettingsInfo, nil } diff --git a/cmd/pythonBuild_test.go b/cmd/pythonBuild_test.go index 4eed95a11d..c63c2d7cad 100644 --- a/cmd/pythonBuild_test.go +++ b/cmd/pythonBuild_test.go @@ -53,7 +53,7 @@ func TestRunPythonBuild(t *testing.T) { telemetryData := telemetry.CustomData{} err := runPythonBuild(&config, &telemetryData, utils, &cpe) - assert.EqualError(t, err, "Python build failed with error: build failure") + assert.EqualError(t, err, "failed to build package: build failure") }) t.Run("success - publishes binaries", func(t *testing.T) { @@ -72,10 +72,10 @@ func TestRunPythonBuild(t *testing.T) { assert.Equal(t, []string{"-m", "venv", config.VirutalEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) - assert.Equal(t, "python", utils.ExecMockRunner.Calls[2].Exec) + assert.Equal(t, "dummy/bin/python", utils.ExecMockRunner.Calls[2].Exec) assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[2].Params) assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec) - assert.Equal(t, []string{"install", "--upgrade", "twine"}, utils.ExecMockRunner.Calls[3].Params) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "twine"}, utils.ExecMockRunner.Calls[3].Params) assert.Equal(t, filepath.Join("dummy", "bin", "twine"), utils.ExecMockRunner.Calls[4].Exec) assert.Equal(t, []string{"upload", "--username", config.TargetRepositoryUser, "--password", config.TargetRepositoryPassword, "--repository-url", config.TargetRepositoryURL, @@ -97,42 +97,11 @@ func TestRunPythonBuild(t *testing.T) { assert.Equal(t, []string{"-m", "venv", config.VirutalEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) - assert.Equal(t, "python", utils.ExecMockRunner.Calls[2].Exec) + assert.Equal(t, "dummy/bin/python", utils.ExecMockRunner.Calls[2].Exec) assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[2].Params) assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec) - assert.Equal(t, []string{"install", "--upgrade", "cyclonedx-bom==6.1.1"}, utils.ExecMockRunner.Calls[3].Params) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "cyclonedx-bom==6.1.1"}, utils.ExecMockRunner.Calls[3].Params) assert.Equal(t, filepath.Join("dummy", "bin", "cyclonedx-py"), utils.ExecMockRunner.Calls[4].Exec) assert.Equal(t, []string{"env", "--output-file", "bom-pip.xml", "--output-format", "XML", "--spec-version", "1.4"}, utils.ExecMockRunner.Calls[4].Params) }) } - -func TestPythonBuildExecute(t *testing.T) { - t.Run("Test build with flags", func(t *testing.T) { - config := pythonBuildOptions{ - BuildFlags: []string{"--verbose"}, - SetupFlags: []string{"egg_info", "--tag-build=pr13"}, - VirutalEnvironmentName: "venv", - } - - utils := pythonBuildMockUtils{ - ExecMockRunner: &mock.ExecMockRunner{}, - } - - virutalEnvironmentPathMap := map[string]string{ - "python": "python", - } - - err := buildExecute(&config, &utils, []string{}, virutalEnvironmentPathMap) - - assert.NoError(t, err) - assert.Equal(t, "python", utils.ExecMockRunner.Calls[0].Exec) - assert.Equal(t, []string{ - "--verbose", - "setup.py", - "egg_info", - "--tag-build=pr13", - "sdist", - "bdist_wheel", - }, utils.ExecMockRunner.Calls[0].Params) - }) -} diff --git a/pkg/python/bom.go b/pkg/python/bom.go new file mode 100644 index 0000000000..13bb88db0f --- /dev/null +++ b/pkg/python/bom.go @@ -0,0 +1,49 @@ +package python + +import ( + "fmt" + "path/filepath" + + "github.com/SAP/jenkins-library/pkg/log" +) + +const ( + BOMFilename = "bom-pip.xml" +) + +func CreateBOM( + executeFn func(executable string, params ...string) error, + existsFn func(path string) (bool, error), + virtualEnv string, + requirementsFile string, + cycloneDxVersion string, + cycloneDxSchemaVersion string, +) error { + if exists, _ := existsFn(requirementsFile); exists { + if err := InstallRequirements(executeFn, virtualEnv, requirementsFile); err != nil { + return err + } + } else { + log.Entry().Warnf("unable to find requirements.txt file at %s , continuing SBOM generation without requirements.txt", requirementsFile) + } + + if err := InstallCycloneDX(executeFn, virtualEnv, cycloneDxVersion); err != nil { + return err + } + + cycloneDxBinary := "cyclonedx-py" + if len(virtualEnv) > 0 { + cycloneDxBinary = filepath.Join(virtualEnv, "bin", cycloneDxBinary) + } + + log.Entry().Debug("creating BOM") + if err := executeFn(cycloneDxBinary, + "env", + "--output-file", BOMFilename, + "--output-format", "XML", + "--spec-version", cycloneDxSchemaVersion, + ); err != nil { + return fmt.Errorf("failed to create BOM: %w", err) + } + return nil +} diff --git a/pkg/python/bom_test.go b/pkg/python/bom_test.go new file mode 100644 index 0000000000..be6efd92e7 --- /dev/null +++ b/pkg/python/bom_test.go @@ -0,0 +1,36 @@ +//go:build unit +// +build unit + +package python + +import ( + "testing" + + "github.com/SAP/jenkins-library/pkg/mock" + "github.com/stretchr/testify/assert" +) + +func TestCreateBOM(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + mockFiles := mock.FilesMock{} + + // test + err := CreateBOM(mockRunner.RunExecutable, mockFiles.FileExists, ".venv", "requirements.txt", "1.2.3", "16") + + // assert + assert.NoError(t, err) + assert.Len(t, mockRunner.Calls, 2) + assert.Equal(t, ".venv/bin/pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "cyclonedx-bom==1.2.3"}, mockRunner.Calls[0].Params) + assert.Equal(t, ".venv/bin/cyclonedx-py", mockRunner.Calls[1].Exec) + assert.Equal(t, []string{ + "env", + "--output-file", "bom-pip.xml", + "--output-format", "XML", + "--spec-version", "16"}, mockRunner.Calls[1].Params) +} diff --git a/pkg/python/build.go b/pkg/python/build.go new file mode 100644 index 0000000000..fcc1879eed --- /dev/null +++ b/pkg/python/build.go @@ -0,0 +1,32 @@ +package python + +import ( + "fmt" + "path/filepath" + + "github.com/SAP/jenkins-library/pkg/log" +) + +func BuildWithSetupPy( + executeFn func(executable string, params ...string) error, + virtualEnv string, + pythonArgs []string, + setupArgs []string, +) error { + var flags []string + flags = append(flags, pythonArgs...) + flags = append(flags, "setup.py") + flags = append(flags, setupArgs...) + flags = append(flags, "sdist", "bdist_wheel") + + pythonBinary := "python" + if len(virtualEnv) > 0 { + pythonBinary = filepath.Join(virtualEnv, "bin", pythonBinary) + } + + log.Entry().Debug("building project") + if err := executeFn(pythonBinary, flags...); err != nil { + return fmt.Errorf("failed to build package: %w", err) + } + return nil +} diff --git a/pkg/python/build_test.go b/pkg/python/build_test.go new file mode 100644 index 0000000000..37bed47a37 --- /dev/null +++ b/pkg/python/build_test.go @@ -0,0 +1,34 @@ +//go:build unit +// +build unit + +package python + +import ( + "testing" + + "github.com/SAP/jenkins-library/pkg/mock" + "github.com/stretchr/testify/assert" +) + +func TestBuildWithSetupPy(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + buildFlags := []string{"--verbose"} + setupFlags := []string{"egg_info", "--tag-build=pr13"} + + // test + err := BuildWithSetupPy(mockRunner.RunExecutable, ".venv", buildFlags, setupFlags) + + // assert + assert.NoError(t, err) + assert.Len(t, mockRunner.Calls, 1) + assert.Equal(t, ".venv/bin/python", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "--verbose", + "setup.py", + "egg_info", + "--tag-build=pr13", + "sdist", + "bdist_wheel", + }, mockRunner.Calls[0].Params) +} diff --git a/pkg/python/env.go b/pkg/python/env.go new file mode 100644 index 0000000000..aeefc87f1a --- /dev/null +++ b/pkg/python/env.go @@ -0,0 +1,30 @@ +package python + +import ( + "fmt" + "path/filepath" +) + +func CreateVirtualEnvironment( + executeFn func(executable string, params ...string) error, + virtualEnv string, +) error { + // Implementation for creating a virtual environment + if err := executeFn("python3", "-m", "venv", virtualEnv); err != nil { + return fmt.Errorf("failed to create virtual environment %s: %w", virtualEnv, err) + } + if err := executeFn("bash", "-c", fmt.Sprintf("source %s", filepath.Join(virtualEnv, "bin", "activate"))); err != nil { + return fmt.Errorf("failed to activate virtual environment %s: %w", virtualEnv, err) + } + return nil +} + +func RemoveVirtualEnvironment( + removeFn func(executable string) error, + virtualEnv string, +) error { + if err := removeFn(virtualEnv); err != nil { + return fmt.Errorf("failed to remove virtual environment %s: %w", virtualEnv, err) + } + return nil +} diff --git a/pkg/python/env_test.go b/pkg/python/env_test.go new file mode 100644 index 0000000000..794f19d84a --- /dev/null +++ b/pkg/python/env_test.go @@ -0,0 +1,27 @@ +//go:build unit +// +build unit + +package python + +import ( + "testing" + + "github.com/SAP/jenkins-library/pkg/mock" + "github.com/stretchr/testify/assert" +) + +func TestCreateVirtualEnvironment(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := CreateVirtualEnvironment(mockRunner.RunExecutable, ".venv") + + // assert + assert.NoError(t, err) + assert.Len(t, mockRunner.Calls, 2) + assert.Equal(t, "python3", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{"-m", "venv", ".venv"}, mockRunner.Calls[0].Params) + assert.Equal(t, "bash", mockRunner.Calls[1].Exec) + assert.Equal(t, []string{"-c", "source .venv/bin/activate"}, mockRunner.Calls[1].Params) +} diff --git a/pkg/python/pip.go b/pkg/python/pip.go new file mode 100644 index 0000000000..996ea9be38 --- /dev/null +++ b/pkg/python/pip.go @@ -0,0 +1,69 @@ +package python + +import ( + "fmt" + "path/filepath" + + "github.com/SAP/jenkins-library/pkg/log" +) + +var ( + PipInstallFlags = []string{"install", "--upgrade", "--root-user-action=ignore"} +) + +func Install( + executeFn func(executable string, params ...string) error, + virtualEnv string, + module string, + version string, + extraArgs []string, +) error { + pipBinary := "pip" + if len(virtualEnv) > 0 { + pipBinary = filepath.Join(virtualEnv, "bin", pipBinary) + } + + flags := PipInstallFlags + // flags := append([]string{"-m", "pip"}, PipInstallFlags...) + + if len(extraArgs) > 0 { + flags = append(flags, extraArgs...) + } + if len(version) > 0 { + module = fmt.Sprintf("%s==%s", module, version) + } + if len(module) > 0 { + flags = append(flags, module) + } + + if err := executeFn(pipBinary, flags...); err != nil { + return fmt.Errorf("failed to install %s: %w", module, err) + } + return nil +} + +func InstallRequirements( + executeFn func(executable string, params ...string) error, + virtualEnv string, + requirementsFile string, +) error { + log.Entry().Debug("installing requirements") + return Install(executeFn, virtualEnv, "", "", []string{"--requirement", requirementsFile}) +} + +func InstallTwine( + executeFn func(executable string, params ...string) error, + virtualEnv string, +) error { + log.Entry().Debug("installing twine") + return Install(executeFn, virtualEnv, "twine", "", nil) +} + +func InstallCycloneDX( + executeFn func(executable string, params ...string) error, + virtualEnv string, + cycloneDXVersion string, +) error { + log.Entry().Debug("installing cyclonedx-bom") + return Install(executeFn, virtualEnv, "cyclonedx-bom", cycloneDXVersion, nil) +} diff --git a/pkg/python/pip_test.go b/pkg/python/pip_test.go new file mode 100644 index 0000000000..054514a2b5 --- /dev/null +++ b/pkg/python/pip_test.go @@ -0,0 +1,64 @@ +//go:build unit +// +build unit + +package python + +import ( + "testing" + + "github.com/SAP/jenkins-library/pkg/mock" + "github.com/stretchr/testify/assert" +) + +func TestInstallRequirements(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := InstallRequirements(mockRunner.RunExecutable, "", "requirements.txt") + + // assert + assert.NoError(t, err) + assert.Equal(t, "pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "--requirement", "requirements.txt"}, mockRunner.Calls[0].Params) +} + +func TestInstallTwine(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := InstallTwine(mockRunner.RunExecutable, "") + + // assert + assert.NoError(t, err) + assert.Len(t, mockRunner.Calls, 1) + assert.Equal(t, "pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "twine"}, mockRunner.Calls[0].Params) +} + +func TestInstallCycloneDXWithVersion(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := InstallCycloneDX(mockRunner.RunExecutable, "", "1.0.0") + + // assert + assert.NoError(t, err) + assert.Len(t, mockRunner.Calls, 1) + assert.Equal(t, "pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "cyclonedx-bom==1.0.0"}, mockRunner.Calls[0].Params) +} diff --git a/pkg/python/publish.go b/pkg/python/publish.go new file mode 100644 index 0000000000..23bfd6531c --- /dev/null +++ b/pkg/python/publish.go @@ -0,0 +1,33 @@ +package python + +import ( + "path/filepath" +) + +func PublishPackage( + executeFn func(executable string, params ...string) error, + virtualEnv string, + repository string, + username string, + password string, +) error { + // install dependency + if err := InstallTwine(executeFn, virtualEnv); err != nil { + return err + } + // handle virtual environment + twineBinary := "twine" + if len(virtualEnv) > 0 { + twineBinary = filepath.Join(virtualEnv, "bin", twineBinary) + } + // publish project + return executeFn( + twineBinary, + "upload", + "--username", username, + "--password", password, + "--repository-url", repository, + "--disable-progress-bar", + "dist/*", + ) +} diff --git a/pkg/python/publish_test.go b/pkg/python/publish_test.go new file mode 100644 index 0000000000..2c5e7413a7 --- /dev/null +++ b/pkg/python/publish_test.go @@ -0,0 +1,36 @@ +//go:build unit +// +build unit + +package python + +import ( + "testing" + + "github.com/SAP/jenkins-library/pkg/mock" + "github.com/stretchr/testify/assert" +) + +func TestPublishWithVirtualEnv(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := PublishPackage(mockRunner.RunExecutable, ".venv", "repository", "anything", "anything") + + // assert + assert.NoError(t, err) + assert.Equal(t, ".venv/bin/pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "twine"}, mockRunner.Calls[0].Params) + assert.Equal(t, ".venv/bin/twine", mockRunner.Calls[1].Exec) + assert.Equal(t, []string{ + "upload", + "--username", "anything", + "--password", "anything", + "--repository-url", "repository", + "--disable-progress-bar", + "dist/*"}, mockRunner.Calls[1].Params) +} From 75500120eb44d11a09230624bf636be88c12a673 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Sat, 20 Sep 2025 21:12:59 +0200 Subject: [PATCH 02/51] use defer --- cmd/pythonBuild.go | 7 +++++-- pkg/python/env.go | 27 +++++++++++++-------------- pkg/python/env_test.go | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index 5a04ad097b..c9ac78bedf 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -51,8 +51,11 @@ func pythonBuild(config pythonBuildOptions, telemetryData *telemetry.CustomData, } func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomData, utils pythonBuildUtils, commonPipelineEnvironment *pythonBuildCommonPipelineEnvironment) error { - if err := python.CreateVirtualEnvironment(utils.RunExecutable, config.VirutalEnvironmentName); err != nil { + if exitHandler, err := python.CreateVirtualEnvironment(utils.RunExecutable, utils.RemoveAll, config.VirutalEnvironmentName); err != nil { return err + } else { + log.DeferExitHandler(exitHandler) + defer exitHandler() } if err := python.BuildWithSetupPy(utils.RunExecutable, config.VirutalEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { @@ -82,7 +85,7 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD return fmt.Errorf("failed to publish: %w", err) } } - return python.RemoveVirtualEnvironment(utils.RemoveAll, config.VirutalEnvironmentName) + return nil } // TODO: extract to common place diff --git a/pkg/python/env.go b/pkg/python/env.go index aeefc87f1a..c7e2799e29 100644 --- a/pkg/python/env.go +++ b/pkg/python/env.go @@ -3,28 +3,27 @@ package python import ( "fmt" "path/filepath" + + "github.com/SAP/jenkins-library/pkg/log" ) func CreateVirtualEnvironment( executeFn func(executable string, params ...string) error, + removeFn func(executable string) error, virtualEnv string, -) error { +) (func(), error) { + exitHandler := func() { + if err := removeFn(virtualEnv); err != nil { + log.Entry().Debugf("failed to remove virtual environment %s: %v", virtualEnv, err) + } + } + // Implementation for creating a virtual environment if err := executeFn("python3", "-m", "venv", virtualEnv); err != nil { - return fmt.Errorf("failed to create virtual environment %s: %w", virtualEnv, err) + return exitHandler, fmt.Errorf("failed to create virtual environment %s: %w", virtualEnv, err) } if err := executeFn("bash", "-c", fmt.Sprintf("source %s", filepath.Join(virtualEnv, "bin", "activate"))); err != nil { - return fmt.Errorf("failed to activate virtual environment %s: %w", virtualEnv, err) - } - return nil -} - -func RemoveVirtualEnvironment( - removeFn func(executable string) error, - virtualEnv string, -) error { - if err := removeFn(virtualEnv); err != nil { - return fmt.Errorf("failed to remove virtual environment %s: %w", virtualEnv, err) + return exitHandler, fmt.Errorf("failed to activate virtual environment %s: %w", virtualEnv, err) } - return nil + return exitHandler, nil } diff --git a/pkg/python/env_test.go b/pkg/python/env_test.go index 794f19d84a..89f20c09b0 100644 --- a/pkg/python/env_test.go +++ b/pkg/python/env_test.go @@ -15,7 +15,7 @@ func TestCreateVirtualEnvironment(t *testing.T) { mockRunner := mock.ExecMockRunner{} // test - err := CreateVirtualEnvironment(mockRunner.RunExecutable, ".venv") + _, err := CreateVirtualEnvironment(mockRunner.RunExecutable, ".venv") // assert assert.NoError(t, err) From 091e4ae280ac5e295522f1cac20ceaa13bbab842 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Sat, 20 Sep 2025 21:18:07 +0200 Subject: [PATCH 03/51] fix typo in parameter --- cmd/pythonBuild.go | 8 ++++---- cmd/pythonBuild_generated.go | 8 ++++---- cmd/pythonBuild_test.go | 10 +++++----- resources/metadata/pythonBuild.yaml | 5 ++++- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index c9ac78bedf..871f15b2dc 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -51,19 +51,19 @@ func pythonBuild(config pythonBuildOptions, telemetryData *telemetry.CustomData, } func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomData, utils pythonBuildUtils, commonPipelineEnvironment *pythonBuildCommonPipelineEnvironment) error { - if exitHandler, err := python.CreateVirtualEnvironment(utils.RunExecutable, utils.RemoveAll, config.VirutalEnvironmentName); err != nil { + if exitHandler, err := python.CreateVirtualEnvironment(utils.RunExecutable, utils.RemoveAll, config.VirtualEnvironmentName); err != nil { return err } else { log.DeferExitHandler(exitHandler) defer exitHandler() } - if err := python.BuildWithSetupPy(utils.RunExecutable, config.VirutalEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { + if err := python.BuildWithSetupPy(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { return err } if config.CreateBOM { - if err := python.CreateBOM(utils.RunExecutable, utils.FileExists, config.VirutalEnvironmentName, config.RequirementsFilePath, cycloneDxVersion, cycloneDxSchemaVersion); err != nil { + if err := python.CreateBOM(utils.RunExecutable, utils.FileExists, config.VirtualEnvironmentName, config.RequirementsFilePath, cycloneDxVersion, cycloneDxSchemaVersion); err != nil { return fmt.Errorf("BOM creation failed: %w", err) } } @@ -77,7 +77,7 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD if config.Publish { if err := python.PublishPackage( utils.RunExecutable, - config.VirutalEnvironmentName, + config.VirtualEnvironmentName, config.TargetRepositoryURL, config.TargetRepositoryUser, config.TargetRepositoryPassword, diff --git a/cmd/pythonBuild_generated.go b/cmd/pythonBuild_generated.go index 5812cf90a8..1cfbdb78b9 100644 --- a/cmd/pythonBuild_generated.go +++ b/cmd/pythonBuild_generated.go @@ -27,7 +27,7 @@ type pythonBuildOptions struct { TargetRepositoryUser string `json:"targetRepositoryUser,omitempty"` TargetRepositoryURL string `json:"targetRepositoryURL,omitempty"` BuildSettingsInfo string `json:"buildSettingsInfo,omitempty"` - VirutalEnvironmentName string `json:"virutalEnvironmentName,omitempty"` + VirtualEnvironmentName string `json:"virtualEnvironmentName,omitempty"` RequirementsFilePath string `json:"requirementsFilePath,omitempty"` } @@ -212,7 +212,7 @@ func addPythonBuildFlags(cmd *cobra.Command, stepConfig *pythonBuildOptions) { cmd.Flags().StringVar(&stepConfig.TargetRepositoryUser, "targetRepositoryUser", os.Getenv("PIPER_targetRepositoryUser"), "Username for the target repository where the compiled binaries shall be uploaded - typically provided by the CI/CD environment.") cmd.Flags().StringVar(&stepConfig.TargetRepositoryURL, "targetRepositoryURL", os.Getenv("PIPER_targetRepositoryURL"), "URL of the target repository where the compiled binaries shall be uploaded - typically provided by the CI/CD environment.") cmd.Flags().StringVar(&stepConfig.BuildSettingsInfo, "buildSettingsInfo", os.Getenv("PIPER_buildSettingsInfo"), "build settings info is typically filled by the step automatically to create information about the build settings that were used during the maven build . This information is typically used for compliance related processes.") - cmd.Flags().StringVar(&stepConfig.VirutalEnvironmentName, "virutalEnvironmentName", `piperBuild-env`, "name of the virtual environment that will be used for the build") + cmd.Flags().StringVar(&stepConfig.VirtualEnvironmentName, "virtualEnvironmentName", `piperBuild-env`, "name of the virtual environment that will be used for the build") cmd.Flags().StringVar(&stepConfig.RequirementsFilePath, "requirementsFilePath", `requirements.txt`, "file path to the requirements.txt file needed for the sbom cycloneDx file creation.") } @@ -321,12 +321,12 @@ func pythonBuildMetadata() config.StepData { Default: os.Getenv("PIPER_buildSettingsInfo"), }, { - Name: "virutalEnvironmentName", + Name: "virtualEnvironmentName", ResourceRef: []config.ResourceReference{}, Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, Type: "string", Mandatory: false, - Aliases: []config.Alias{}, + Aliases: []config.Alias{{Name: "virutalEnvironmentName", Deprecated: true}}, Default: `piperBuild-env`, }, { diff --git a/cmd/pythonBuild_test.go b/cmd/pythonBuild_test.go index c63c2d7cad..ec48b7e713 100644 --- a/cmd/pythonBuild_test.go +++ b/cmd/pythonBuild_test.go @@ -36,7 +36,7 @@ func TestRunPythonBuild(t *testing.T) { cpe := pythonBuildCommonPipelineEnvironment{} t.Run("success - build", func(t *testing.T) { config := pythonBuildOptions{ - VirutalEnvironmentName: "dummy", + VirtualEnvironmentName: "dummy", } utils := newPythonBuildTestsUtils() telemetryData := telemetry.CustomData{} @@ -62,14 +62,14 @@ func TestRunPythonBuild(t *testing.T) { TargetRepositoryURL: "https://my.target.repository.local", TargetRepositoryUser: "user", TargetRepositoryPassword: "password", - VirutalEnvironmentName: "dummy", + VirtualEnvironmentName: "dummy", } utils := newPythonBuildTestsUtils() telemetryData := telemetry.CustomData{} runPythonBuild(&config, &telemetryData, utils, &cpe) assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) - assert.Equal(t, []string{"-m", "venv", config.VirutalEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) + assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) assert.Equal(t, "dummy/bin/python", utils.ExecMockRunner.Calls[2].Exec) @@ -86,7 +86,7 @@ func TestRunPythonBuild(t *testing.T) { config := pythonBuildOptions{ CreateBOM: true, Publish: false, - VirutalEnvironmentName: "dummy", + VirtualEnvironmentName: "dummy", } utils := newPythonBuildTestsUtils() telemetryData := telemetry.CustomData{} @@ -94,7 +94,7 @@ func TestRunPythonBuild(t *testing.T) { runPythonBuild(&config, &telemetryData, utils, &cpe) // assert.NoError(t, err) assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) - assert.Equal(t, []string{"-m", "venv", config.VirutalEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) + assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) assert.Equal(t, "dummy/bin/python", utils.ExecMockRunner.Calls[2].Exec) diff --git a/resources/metadata/pythonBuild.yaml b/resources/metadata/pythonBuild.yaml index 1e2d6c563d..6ccc0b54a8 100644 --- a/resources/metadata/pythonBuild.yaml +++ b/resources/metadata/pythonBuild.yaml @@ -89,7 +89,7 @@ spec: resourceRef: - name: commonPipelineEnvironment param: custom/buildSettingsInfo - - name: virutalEnvironmentName + - name: virtualEnvironmentName type: string description: name of the virtual environment that will be used for the build scope: @@ -97,6 +97,9 @@ spec: - STAGES - PARAMETERS default: piperBuild-env + aliases: + - name: virutalEnvironmentName # typo alias - to be removed in future releases + deprecated: true - name: requirementsFilePath type: string description: file path to the requirements.txt file needed for the sbom cycloneDx file creation. From 99e03f77e32e8ccad986ee343e91f6699a1e736c Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Sat, 20 Sep 2025 22:04:40 +0200 Subject: [PATCH 04/51] install wheel dependency --- pkg/python/build.go | 15 ++++++++++----- pkg/python/build_test.go | 12 +++++++++--- pkg/python/pip.go | 8 ++++++++ pkg/python/pip_test.go | 18 ++++++++++++++++++ 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/pkg/python/build.go b/pkg/python/build.go index fcc1879eed..dc7606f573 100644 --- a/pkg/python/build.go +++ b/pkg/python/build.go @@ -13,17 +13,22 @@ func BuildWithSetupPy( pythonArgs []string, setupArgs []string, ) error { - var flags []string - flags = append(flags, pythonArgs...) - flags = append(flags, "setup.py") - flags = append(flags, setupArgs...) - flags = append(flags, "sdist", "bdist_wheel") + // install dependency + if err := InstallWheel(executeFn, virtualEnv); err != nil { + return err + } pythonBinary := "python" if len(virtualEnv) > 0 { pythonBinary = filepath.Join(virtualEnv, "bin", pythonBinary) } + var flags []string + flags = append(flags, pythonArgs...) + flags = append(flags, "setup.py") + flags = append(flags, setupArgs...) + flags = append(flags, "sdist", "bdist_wheel") + log.Entry().Debug("building project") if err := executeFn(pythonBinary, flags...); err != nil { return fmt.Errorf("failed to build package: %w", err) diff --git a/pkg/python/build_test.go b/pkg/python/build_test.go index 37bed47a37..732bb1da47 100644 --- a/pkg/python/build_test.go +++ b/pkg/python/build_test.go @@ -21,8 +21,14 @@ func TestBuildWithSetupPy(t *testing.T) { // assert assert.NoError(t, err) - assert.Len(t, mockRunner.Calls, 1) - assert.Equal(t, ".venv/bin/python", mockRunner.Calls[0].Exec) + assert.Len(t, mockRunner.Calls, 2) + assert.Equal(t, ".venv/bin/pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "wheel"}, mockRunner.Calls[0].Params) + assert.Equal(t, ".venv/bin/python", mockRunner.Calls[1].Exec) assert.Equal(t, []string{ "--verbose", "setup.py", @@ -30,5 +36,5 @@ func TestBuildWithSetupPy(t *testing.T) { "--tag-build=pr13", "sdist", "bdist_wheel", - }, mockRunner.Calls[0].Params) + }, mockRunner.Calls[1].Params) } diff --git a/pkg/python/pip.go b/pkg/python/pip.go index 996ea9be38..97c7546c7b 100644 --- a/pkg/python/pip.go +++ b/pkg/python/pip.go @@ -51,6 +51,14 @@ func InstallRequirements( return Install(executeFn, virtualEnv, "", "", []string{"--requirement", requirementsFile}) } +func InstallWheel( + executeFn func(executable string, params ...string) error, + virtualEnv string, +) error { + log.Entry().Debug("installing wheel") + return Install(executeFn, virtualEnv, "wheel", "", nil) +} + func InstallTwine( executeFn func(executable string, params ...string) error, virtualEnv string, diff --git a/pkg/python/pip_test.go b/pkg/python/pip_test.go index 054514a2b5..3ffd15e9d5 100644 --- a/pkg/python/pip_test.go +++ b/pkg/python/pip_test.go @@ -27,6 +27,24 @@ func TestInstallRequirements(t *testing.T) { "--requirement", "requirements.txt"}, mockRunner.Calls[0].Params) } +func TestInstallWheel(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := InstallWheel(mockRunner.RunExecutable, "") + + // assert + assert.NoError(t, err) + assert.Len(t, mockRunner.Calls, 1) + assert.Equal(t, "pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "wheel"}, mockRunner.Calls[0].Params) +} + func TestInstallTwine(t *testing.T) { // init mockRunner := mock.ExecMockRunner{} From b759b6e5e5ddc4ceee53bd6aac0ca57fb815bd5d Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Sat, 20 Sep 2025 22:43:02 +0200 Subject: [PATCH 05/51] fix test cases --- cmd/pythonBuild_test.go | 28 ++++++++++++++++------------ pkg/python/env_test.go | 3 ++- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/cmd/pythonBuild_test.go b/cmd/pythonBuild_test.go index ec48b7e713..39496bf584 100644 --- a/cmd/pythonBuild_test.go +++ b/cmd/pythonBuild_test.go @@ -72,14 +72,16 @@ func TestRunPythonBuild(t *testing.T) { assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) - assert.Equal(t, "dummy/bin/python", utils.ExecMockRunner.Calls[2].Exec) - assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[2].Params) - assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec) - assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "twine"}, utils.ExecMockRunner.Calls[3].Params) - assert.Equal(t, filepath.Join("dummy", "bin", "twine"), utils.ExecMockRunner.Calls[4].Exec) + assert.Equal(t, "dummy/bin/pip", utils.ExecMockRunner.Calls[2].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[2].Params) + assert.Equal(t, "dummy/bin/python", utils.ExecMockRunner.Calls[3].Exec) + assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[3].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "twine"}, utils.ExecMockRunner.Calls[4].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "twine"), utils.ExecMockRunner.Calls[5].Exec) assert.Equal(t, []string{"upload", "--username", config.TargetRepositoryUser, "--password", config.TargetRepositoryPassword, "--repository-url", config.TargetRepositoryURL, - "--disable-progress-bar", "dist/*"}, utils.ExecMockRunner.Calls[4].Params) + "--disable-progress-bar", "dist/*"}, utils.ExecMockRunner.Calls[5].Params) }) t.Run("success - create BOM", func(t *testing.T) { @@ -97,11 +99,13 @@ func TestRunPythonBuild(t *testing.T) { assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) - assert.Equal(t, "dummy/bin/python", utils.ExecMockRunner.Calls[2].Exec) - assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[2].Params) - assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec) - assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "cyclonedx-bom==6.1.1"}, utils.ExecMockRunner.Calls[3].Params) - assert.Equal(t, filepath.Join("dummy", "bin", "cyclonedx-py"), utils.ExecMockRunner.Calls[4].Exec) - assert.Equal(t, []string{"env", "--output-file", "bom-pip.xml", "--output-format", "XML", "--spec-version", "1.4"}, utils.ExecMockRunner.Calls[4].Params) + assert.Equal(t, "dummy/bin/pip", utils.ExecMockRunner.Calls[2].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[2].Params) + assert.Equal(t, "dummy/bin/python", utils.ExecMockRunner.Calls[3].Exec) + assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[3].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "cyclonedx-bom==6.1.1"}, utils.ExecMockRunner.Calls[4].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "cyclonedx-py"), utils.ExecMockRunner.Calls[5].Exec) + assert.Equal(t, []string{"env", "--output-file", "bom-pip.xml", "--output-format", "XML", "--spec-version", "1.4"}, utils.ExecMockRunner.Calls[5].Params) }) } diff --git a/pkg/python/env_test.go b/pkg/python/env_test.go index 89f20c09b0..30b1573261 100644 --- a/pkg/python/env_test.go +++ b/pkg/python/env_test.go @@ -13,9 +13,10 @@ import ( func TestCreateVirtualEnvironment(t *testing.T) { // init mockRunner := mock.ExecMockRunner{} + mockFiles := mock.FilesMock{} // test - _, err := CreateVirtualEnvironment(mockRunner.RunExecutable, ".venv") + _, err := CreateVirtualEnvironment(mockRunner.RunExecutable, mockFiles.RemoveAll, ".venv") // assert assert.NoError(t, err) From ed35c2dbb0ab58cf433d0c704abbdb6f0d1d9322 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 15 Aug 2025 10:58:45 +0200 Subject: [PATCH 06/51] add sonar config --- sonar-project.properties | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 sonar-project.properties diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000000..ba08e05d5e --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,37 @@ + +# Root project information +sonar.projectName=Piper (OpenSource) +sonar.projectKey=piper-os + +# SonarQube Links +sonar.links.homepage=https://url.sap/piper + +# Some properties that will be inherited by the modules +sonar.sourceEncoding=UTF-8 +sonar.dynamicAnalysis=reuseReports + +# List of the module identifiers +sonar.modules=PiperGo + +# Piper Go +PiperGo.sonar.projectName=Piper (golang) +PiperGo.sonar.projectBaseDir=. +PiperGo.sonar.sources=. +PiperGo.sonar.inclusions=**/*.go +PiperGo.sonar.exclusions=**/*_test.go, **/mocks/*.go, **/vendor/**, **/*_generated.go, **/*_generated_test.go +PiperGo.sonar.tests=. +PiperGo.sonar.test.inclusions=**/*_test.go, **/mocks/*.go +PiperGo.sonar.test.exclusions=**/vendor/** +#PiperGo.sonar.go.tests.reportPaths=unit-report.out,integration-report.out +#PiperGo.sonar.go.coverage.reportPaths=cover.out +#PiperGo.sonar.cpd.exclusions=**/*_generated.go + +#sonar.testExecutionReportPaths=reports/sonar-report.xml +#sonar.javascript.lcov.reportPaths=reports/lcov.info +#sonar.eslint.reportPaths=reports/eslint-report.json +# +#sonar.pullrequest.github.repository= +# +#sonar.c.file.suffixes=- +#sonar.cpp.file.suffixes=- +#sonar.objc.file.suffixes=- From 416feb3b2715466e15dbeb66812680a8d059f9e8 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 27 Aug 2025 16:49:50 +0200 Subject: [PATCH 07/51] Revert "add sonar config" This reverts commit e383e4fdc135caa24f10786b78e5353de70ba6c1. --- sonar-project.properties | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 sonar-project.properties diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index ba08e05d5e..0000000000 --- a/sonar-project.properties +++ /dev/null @@ -1,37 +0,0 @@ - -# Root project information -sonar.projectName=Piper (OpenSource) -sonar.projectKey=piper-os - -# SonarQube Links -sonar.links.homepage=https://url.sap/piper - -# Some properties that will be inherited by the modules -sonar.sourceEncoding=UTF-8 -sonar.dynamicAnalysis=reuseReports - -# List of the module identifiers -sonar.modules=PiperGo - -# Piper Go -PiperGo.sonar.projectName=Piper (golang) -PiperGo.sonar.projectBaseDir=. -PiperGo.sonar.sources=. -PiperGo.sonar.inclusions=**/*.go -PiperGo.sonar.exclusions=**/*_test.go, **/mocks/*.go, **/vendor/**, **/*_generated.go, **/*_generated_test.go -PiperGo.sonar.tests=. -PiperGo.sonar.test.inclusions=**/*_test.go, **/mocks/*.go -PiperGo.sonar.test.exclusions=**/vendor/** -#PiperGo.sonar.go.tests.reportPaths=unit-report.out,integration-report.out -#PiperGo.sonar.go.coverage.reportPaths=cover.out -#PiperGo.sonar.cpd.exclusions=**/*_generated.go - -#sonar.testExecutionReportPaths=reports/sonar-report.xml -#sonar.javascript.lcov.reportPaths=reports/lcov.info -#sonar.eslint.reportPaths=reports/eslint-report.json -# -#sonar.pullrequest.github.repository= -# -#sonar.c.file.suffixes=- -#sonar.cpp.file.suffixes=- -#sonar.objc.file.suffixes=- From 1baca486f4f0324ad2f415a424ab7841fa0dc267 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 29 Aug 2025 09:31:52 +0200 Subject: [PATCH 08/51] add build function --- pkg/python/build.go | 34 ++++++++++++++++++++++++++++++++++ pkg/python/build_test.go | 13 +++++++++++++ 2 files changed, 47 insertions(+) diff --git a/pkg/python/build.go b/pkg/python/build.go index dc7606f573..1e304a685c 100644 --- a/pkg/python/build.go +++ b/pkg/python/build.go @@ -35,3 +35,37 @@ func BuildWithSetupPy( } return nil } + +func InstallProjectDependencies( + executeFn func(executable string, params ...string) error, + binary string, +) error { + log.Entry().Debug("installing project dependencies") + if err := executeFn(binary, "-m", "pip", "install", "."); err != nil { + return err + } + return nil +} + +func Build( + executeFn func(executable string, params ...string) error, + binary string, + binaryFlags []string, + moduleFlags []string, +) error { + // Set default value for binaryFlags if nil + if binaryFlags == nil { + binaryFlags = []string{} + } + + var flags []string + flags = append(flags, binaryFlags...) + flags = append(flags, "-m", "build", "--no-isolation") + flags = append(flags, moduleFlags...) + + log.Entry().Debug("building project") + if err := executeFn(binary, flags...); err != nil { + return err + } + return nil +} diff --git a/pkg/python/build_test.go b/pkg/python/build_test.go index 732bb1da47..bd39824360 100644 --- a/pkg/python/build_test.go +++ b/pkg/python/build_test.go @@ -38,3 +38,16 @@ func TestBuildWithSetupPy(t *testing.T) { "bdist_wheel", }, mockRunner.Calls[1].Params) } + +func TestBuild(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := Build(mockRunner.RunExecutable, "python", nil, nil) + + // assert + assert.NoError(t, err) + assert.Equal(t, "python", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{"-m", "build", "--no-isolation"}, mockRunner.Calls[0].Params) +} From 602b154c882951a0c9d703ab9923761fcb6d0fd4 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 29 Aug 2025 12:42:52 +0200 Subject: [PATCH 09/51] move install function --- pkg/python/build.go | 11 ----------- pkg/python/pip.go | 11 +++++++++++ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/python/build.go b/pkg/python/build.go index 1e304a685c..b64f984bf2 100644 --- a/pkg/python/build.go +++ b/pkg/python/build.go @@ -36,17 +36,6 @@ func BuildWithSetupPy( return nil } -func InstallProjectDependencies( - executeFn func(executable string, params ...string) error, - binary string, -) error { - log.Entry().Debug("installing project dependencies") - if err := executeFn(binary, "-m", "pip", "install", "."); err != nil { - return err - } - return nil -} - func Build( executeFn func(executable string, params ...string) error, binary string, diff --git a/pkg/python/pip.go b/pkg/python/pip.go index 97c7546c7b..0112166cac 100644 --- a/pkg/python/pip.go +++ b/pkg/python/pip.go @@ -42,6 +42,17 @@ func Install( return nil } +func InstallProjectDependencies( + executeFn func(executable string, params ...string) error, + binary string, +) error { + log.Entry().Debug("installing project dependencies") + if err := executeFn(binary, "-m", "pip", "install", "."); err != nil { + return err + } + return nil +} + func InstallRequirements( executeFn func(executable string, params ...string) error, virtualEnv string, From a116c858e204c79a52501cd562877dcd4033b9c5 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 29 Aug 2025 12:52:14 +0200 Subject: [PATCH 10/51] add feature flag pkg --- pkg/feature/feature.go | 17 +++++++++++++++++ pkg/feature/feature_test.go | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 pkg/feature/feature.go create mode 100644 pkg/feature/feature_test.go diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go new file mode 100644 index 0000000000..03154fb551 --- /dev/null +++ b/pkg/feature/feature.go @@ -0,0 +1,17 @@ +package feature + +import ( + "os" + + "github.com/SAP/jenkins-library/pkg/log" +) + +const prefix = "com_sap_piper_featureFlag_" + +func IsFeatureEnabled(flag string) bool { + if os.Getenv(prefix+flag) == "true" { + log.Entry().Infof("Feature '%s%s' is enabled", prefix, flag) + return true + } + return false +} diff --git a/pkg/feature/feature_test.go b/pkg/feature/feature_test.go new file mode 100644 index 0000000000..ea7ca69119 --- /dev/null +++ b/pkg/feature/feature_test.go @@ -0,0 +1,20 @@ +package feature + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsFeatureEnabled(t *testing.T) { + t.Run("Azure - BranchBuild", func(t *testing.T) { + assert.False(t, IsFeatureEnabled("newFeature")) + + // defer resetEnv(os.Environ()) + os.Setenv(prefix+"newFeature", "true") + defer os.Setenv(prefix+"newFeature", "") + + assert.True(t, IsFeatureEnabled("newFeature")) + }) +} From 03f19fd76890d62b4c5dc44dfb74fd83790196c2 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Sat, 30 Aug 2025 10:17:58 +0200 Subject: [PATCH 11/51] add support for toml file --- cmd/pythonBuild.go | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index 871f15b2dc..d182092dfb 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -2,9 +2,11 @@ package cmd import ( "fmt" + "strings" "github.com/SAP/jenkins-library/pkg/buildsettings" "github.com/SAP/jenkins-library/pkg/command" + "github.com/SAP/jenkins-library/pkg/feature" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/python" @@ -58,8 +60,26 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD defer exitHandler() } - if err := python.BuildWithSetupPy(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { - return err + // FEATURE FLAG (com_sap_piper_featureFlag_pythonToml) to switch to new implementation of python build step + if feature.IsFeatureEnabled("pythonToml") { + // check project descriptor + buildDescriptorFilePath, err := searchDescriptor([]string{"pyproject.toml", "setup.py"}, utils.FileExists) + if err != nil { + return err + } + // build package + if strings.HasSuffix(buildDescriptorFilePath, "pyproject.toml") { + if err := python.InstallProjectDependencies(utils.RunExecutable, python.Binary); err != nil { + return fmt.Errorf("Failed to install project dependencies: %w", err) + } + if err := python.Build(utils.RunExecutable, python.Binary, config.BuildFlags, config.SetupFlags); err != nil { + return fmt.Errorf("Failed to build python project: %w", err) + } + } + } else { + if err := python.BuildWithSetupPy(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { + return err + } } if config.CreateBOM { @@ -90,12 +110,12 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD // TODO: extract to common place func createBuildSettingsInfo(config *pythonBuildOptions) (string, error) { + // generate build settings information log.Entry().Debugf("creating build settings information...") dockerImage, err := GetDockerImageValue(stepName) if err != nil { return "", err } - pythonConfig := buildsettings.BuildOptions{ CreateBOM: config.CreateBOM, Publish: config.Publish, @@ -108,3 +128,18 @@ func createBuildSettingsInfo(config *pythonBuildOptions) (string, error) { } return buildSettingsInfo, nil } + +func searchDescriptor(supported []string, existsFunc func(string) (bool, error)) (string, error) { + var descriptor string + for _, f := range supported { + exists, _ := existsFunc(f) + if exists { + descriptor = f + break + } + } + if len(descriptor) == 0 { + return "", fmt.Errorf("no build descriptor available, supported: %v", supported) + } + return descriptor, nil +} From ebbafeb7056104c3fc851c5ae6560dbbdd83dcdd Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Sat, 30 Aug 2025 10:18:25 +0200 Subject: [PATCH 12/51] update docs --- cmd/pythonBuild_generated.go | 2 +- cmd/pythonBuild_test.go | 31 +++++++++++++++++++++++++++++ resources/metadata/pythonBuild.yaml | 2 +- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/cmd/pythonBuild_generated.go b/cmd/pythonBuild_generated.go index 1cfbdb78b9..d3e37e5fd3 100644 --- a/cmd/pythonBuild_generated.go +++ b/cmd/pythonBuild_generated.go @@ -205,7 +205,7 @@ and are exposed are environment variables that must be present in the environmen func addPythonBuildFlags(cmd *cobra.Command, stepConfig *pythonBuildOptions) { cmd.Flags().StringSliceVar(&stepConfig.BuildFlags, "buildFlags", []string{}, "Defines list of build flags passed to python binary.") - cmd.Flags().StringSliceVar(&stepConfig.SetupFlags, "setupFlags", []string{}, "Defines list of flags passed to setup.py.") + cmd.Flags().StringSliceVar(&stepConfig.SetupFlags, "setupFlags", []string{}, "Defines list of flags passed to setup.py / build module.") cmd.Flags().BoolVar(&stepConfig.CreateBOM, "createBOM", false, "Creates the bill of materials (BOM) using CycloneDX plugin.") cmd.Flags().BoolVar(&stepConfig.Publish, "publish", false, "Configures the build to publish artifacts to a repository.") cmd.Flags().StringVar(&stepConfig.TargetRepositoryPassword, "targetRepositoryPassword", os.Getenv("PIPER_targetRepositoryPassword"), "Password for the target repository where the compiled binaries shall be uploaded - typically provided by the CI/CD environment.") diff --git a/cmd/pythonBuild_test.go b/cmd/pythonBuild_test.go index 39496bf584..87cc20b26c 100644 --- a/cmd/pythonBuild_test.go +++ b/cmd/pythonBuild_test.go @@ -109,3 +109,34 @@ func TestRunPythonBuild(t *testing.T) { assert.Equal(t, []string{"env", "--output-file", "bom-pip.xml", "--output-format", "XML", "--spec-version", "1.4"}, utils.ExecMockRunner.Calls[5].Params) }) } + +func TestPythonBuildExecute(t *testing.T) { + t.Run("Test build with flags", func(t *testing.T) { + config := pythonBuildOptions{ + BuildFlags: []string{"--verbose"}, + SetupFlags: []string{"egg_info", "--tag-build=pr13"}, + VirutalEnvironmentName: "venv", + } + + utils := pythonBuildMockUtils{ + ExecMockRunner: &mock.ExecMockRunner{}, + } + + virutalEnvironmentPathMap := map[string]string{ + "python": "python", + } + + err := buildExecute(&config, &utils, virutalEnvironmentPathMap) + + assert.NoError(t, err) + assert.Equal(t, "python", utils.ExecMockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "--verbose", + "setup.py", + "egg_info", + "--tag-build=pr13", + "sdist", + "bdist_wheel", + }, utils.ExecMockRunner.Calls[0].Params) + }) +} diff --git a/resources/metadata/pythonBuild.yaml b/resources/metadata/pythonBuild.yaml index 6ccc0b54a8..d4fb8555cb 100644 --- a/resources/metadata/pythonBuild.yaml +++ b/resources/metadata/pythonBuild.yaml @@ -25,7 +25,7 @@ spec: - STEPS - name: setupFlags type: "[]string" - description: Defines list of flags passed to setup.py. + description: Defines list of flags passed to setup.py / build module. scope: - PARAMETERS - STAGES From 9ca556cab9afaf21139f910a66646148a6ad4e66 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Tue, 2 Sep 2025 09:19:37 +0200 Subject: [PATCH 13/51] rename --- pkg/feature/feature_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/feature/feature_test.go b/pkg/feature/feature_test.go index ea7ca69119..c59758adaa 100644 --- a/pkg/feature/feature_test.go +++ b/pkg/feature/feature_test.go @@ -8,7 +8,7 @@ import ( ) func TestIsFeatureEnabled(t *testing.T) { - t.Run("Azure - BranchBuild", func(t *testing.T) { + t.Run("", func(t *testing.T) { assert.False(t, IsFeatureEnabled("newFeature")) // defer resetEnv(os.Environ()) From 211a3a8739d22508c349a63e54c299ae095c1535 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Thu, 11 Sep 2025 11:33:20 +0200 Subject: [PATCH 14/51] add toml flag --- cmd/pythonBuild.go | 2 +- cmd/pythonBuild_generated.go | 11 +++++++++++ resources/metadata/pythonBuild.yaml | 8 ++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index d182092dfb..cea407a1f0 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -61,7 +61,7 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD } // FEATURE FLAG (com_sap_piper_featureFlag_pythonToml) to switch to new implementation of python build step - if feature.IsFeatureEnabled("pythonToml") { + if config.UseTomlFile || feature.IsFeatureEnabled("pythonToml") { // check project descriptor buildDescriptorFilePath, err := searchDescriptor([]string{"pyproject.toml", "setup.py"}, utils.FileExists) if err != nil { diff --git a/cmd/pythonBuild_generated.go b/cmd/pythonBuild_generated.go index d3e37e5fd3..f09a38a577 100644 --- a/cmd/pythonBuild_generated.go +++ b/cmd/pythonBuild_generated.go @@ -29,6 +29,7 @@ type pythonBuildOptions struct { BuildSettingsInfo string `json:"buildSettingsInfo,omitempty"` VirtualEnvironmentName string `json:"virtualEnvironmentName,omitempty"` RequirementsFilePath string `json:"requirementsFilePath,omitempty"` + UseTomlFile bool `json:"useTomlFile,omitempty"` } type pythonBuildCommonPipelineEnvironment struct { @@ -214,6 +215,7 @@ func addPythonBuildFlags(cmd *cobra.Command, stepConfig *pythonBuildOptions) { cmd.Flags().StringVar(&stepConfig.BuildSettingsInfo, "buildSettingsInfo", os.Getenv("PIPER_buildSettingsInfo"), "build settings info is typically filled by the step automatically to create information about the build settings that were used during the maven build . This information is typically used for compliance related processes.") cmd.Flags().StringVar(&stepConfig.VirtualEnvironmentName, "virtualEnvironmentName", `piperBuild-env`, "name of the virtual environment that will be used for the build") cmd.Flags().StringVar(&stepConfig.RequirementsFilePath, "requirementsFilePath", `requirements.txt`, "file path to the requirements.txt file needed for the sbom cycloneDx file creation.") + cmd.Flags().BoolVar(&stepConfig.UseTomlFile, "useTomlFile", false, "Determines if a pyproject.toml file should be used for dependency management.") } @@ -338,6 +340,15 @@ func pythonBuildMetadata() config.StepData { Aliases: []config.Alias{}, Default: `requirements.txt`, }, + { + Name: "useTomlFile", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, + Type: "bool", + Mandatory: false, + Aliases: []config.Alias{}, + Default: false, + }, }, }, Containers: []config.Container{ diff --git a/resources/metadata/pythonBuild.yaml b/resources/metadata/pythonBuild.yaml index d4fb8555cb..4a9e365564 100644 --- a/resources/metadata/pythonBuild.yaml +++ b/resources/metadata/pythonBuild.yaml @@ -108,6 +108,14 @@ spec: - STAGES - PARAMETERS default: requirements.txt + - name: useTomlFile + type: bool + default: false + description: Determines if a pyproject.toml file should be used for dependency management. + scope: + - STEPS + - STAGES + - PARAMETERS outputs: resources: - name: commonPipelineEnvironment From d58b7be4aa1205347a38f3620b5d362df626dc65 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 12 Sep 2025 09:32:34 +0200 Subject: [PATCH 15/51] Revert "add toml flag" This reverts commit b97ac3d805d11b130f3ec1f0373b319e7854df4d. --- cmd/pythonBuild.go | 2 +- cmd/pythonBuild_generated.go | 11 ----------- resources/metadata/pythonBuild.yaml | 8 -------- 3 files changed, 1 insertion(+), 20 deletions(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index cea407a1f0..d182092dfb 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -61,7 +61,7 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD } // FEATURE FLAG (com_sap_piper_featureFlag_pythonToml) to switch to new implementation of python build step - if config.UseTomlFile || feature.IsFeatureEnabled("pythonToml") { + if feature.IsFeatureEnabled("pythonToml") { // check project descriptor buildDescriptorFilePath, err := searchDescriptor([]string{"pyproject.toml", "setup.py"}, utils.FileExists) if err != nil { diff --git a/cmd/pythonBuild_generated.go b/cmd/pythonBuild_generated.go index f09a38a577..d3e37e5fd3 100644 --- a/cmd/pythonBuild_generated.go +++ b/cmd/pythonBuild_generated.go @@ -29,7 +29,6 @@ type pythonBuildOptions struct { BuildSettingsInfo string `json:"buildSettingsInfo,omitempty"` VirtualEnvironmentName string `json:"virtualEnvironmentName,omitempty"` RequirementsFilePath string `json:"requirementsFilePath,omitempty"` - UseTomlFile bool `json:"useTomlFile,omitempty"` } type pythonBuildCommonPipelineEnvironment struct { @@ -215,7 +214,6 @@ func addPythonBuildFlags(cmd *cobra.Command, stepConfig *pythonBuildOptions) { cmd.Flags().StringVar(&stepConfig.BuildSettingsInfo, "buildSettingsInfo", os.Getenv("PIPER_buildSettingsInfo"), "build settings info is typically filled by the step automatically to create information about the build settings that were used during the maven build . This information is typically used for compliance related processes.") cmd.Flags().StringVar(&stepConfig.VirtualEnvironmentName, "virtualEnvironmentName", `piperBuild-env`, "name of the virtual environment that will be used for the build") cmd.Flags().StringVar(&stepConfig.RequirementsFilePath, "requirementsFilePath", `requirements.txt`, "file path to the requirements.txt file needed for the sbom cycloneDx file creation.") - cmd.Flags().BoolVar(&stepConfig.UseTomlFile, "useTomlFile", false, "Determines if a pyproject.toml file should be used for dependency management.") } @@ -340,15 +338,6 @@ func pythonBuildMetadata() config.StepData { Aliases: []config.Alias{}, Default: `requirements.txt`, }, - { - Name: "useTomlFile", - ResourceRef: []config.ResourceReference{}, - Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, - Type: "bool", - Mandatory: false, - Aliases: []config.Alias{}, - Default: false, - }, }, }, Containers: []config.Container{ diff --git a/resources/metadata/pythonBuild.yaml b/resources/metadata/pythonBuild.yaml index 4a9e365564..d4fb8555cb 100644 --- a/resources/metadata/pythonBuild.yaml +++ b/resources/metadata/pythonBuild.yaml @@ -108,14 +108,6 @@ spec: - STAGES - PARAMETERS default: requirements.txt - - name: useTomlFile - type: bool - default: false - description: Determines if a pyproject.toml file should be used for dependency management. - scope: - - STEPS - - STAGES - - PARAMETERS outputs: resources: - name: commonPipelineEnvironment From 43b6432c3b8a1e944e1fd9fe6753a1f56eb62b6d Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 12 Sep 2025 09:52:09 +0200 Subject: [PATCH 16/51] remove feature flag --- cmd/pythonBuild.go | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index d182092dfb..2fb60108bc 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -6,7 +6,6 @@ import ( "github.com/SAP/jenkins-library/pkg/buildsettings" "github.com/SAP/jenkins-library/pkg/command" - "github.com/SAP/jenkins-library/pkg/feature" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/python" @@ -60,25 +59,24 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD defer exitHandler() } - // FEATURE FLAG (com_sap_piper_featureFlag_pythonToml) to switch to new implementation of python build step - if feature.IsFeatureEnabled("pythonToml") { - // check project descriptor - buildDescriptorFilePath, err := searchDescriptor([]string{"pyproject.toml", "setup.py"}, utils.FileExists) - if err != nil { - return err + // check project descriptor + buildDescriptorFilePath, err := searchDescriptor([]string{"pyproject.toml", "setup.py"}, utils.FileExists) + if err != nil { + return err + } + + if strings.HasSuffix(buildDescriptorFilePath, "pyproject.toml") { + // TOML file + if err := python.InstallProjectDependencies(utils.RunExecutable, python.Binary); err != nil { + return fmt.Errorf("Failed to install project dependencies: %w", err) } - // build package - if strings.HasSuffix(buildDescriptorFilePath, "pyproject.toml") { - if err := python.InstallProjectDependencies(utils.RunExecutable, python.Binary); err != nil { - return fmt.Errorf("Failed to install project dependencies: %w", err) - } - if err := python.Build(utils.RunExecutable, python.Binary, config.BuildFlags, config.SetupFlags); err != nil { - return fmt.Errorf("Failed to build python project: %w", err) - } + if err := python.Build(utils.RunExecutable, python.Binary, config.BuildFlags, config.SetupFlags); err != nil { + return fmt.Errorf("Failed to build python project: %w", err) } } else { + // legacy handling setup.py if err := python.BuildWithSetupPy(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { - return err + return fmt.Errorf("Python build failed with error: %w", err) } } From ac6c0e731006f5c7a5f3a323a8ec254124518420 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 12 Sep 2025 09:53:54 +0200 Subject: [PATCH 17/51] Revert "add feature flag pkg" This reverts commit 81966b0ec6523ef44bb2330ce1630a0a4104ca04. --- pkg/feature/feature.go | 17 ----------------- pkg/feature/feature_test.go | 20 -------------------- 2 files changed, 37 deletions(-) delete mode 100644 pkg/feature/feature.go delete mode 100644 pkg/feature/feature_test.go diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go deleted file mode 100644 index 03154fb551..0000000000 --- a/pkg/feature/feature.go +++ /dev/null @@ -1,17 +0,0 @@ -package feature - -import ( - "os" - - "github.com/SAP/jenkins-library/pkg/log" -) - -const prefix = "com_sap_piper_featureFlag_" - -func IsFeatureEnabled(flag string) bool { - if os.Getenv(prefix+flag) == "true" { - log.Entry().Infof("Feature '%s%s' is enabled", prefix, flag) - return true - } - return false -} diff --git a/pkg/feature/feature_test.go b/pkg/feature/feature_test.go deleted file mode 100644 index c59758adaa..0000000000 --- a/pkg/feature/feature_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package feature - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestIsFeatureEnabled(t *testing.T) { - t.Run("", func(t *testing.T) { - assert.False(t, IsFeatureEnabled("newFeature")) - - // defer resetEnv(os.Environ()) - os.Setenv(prefix+"newFeature", "true") - defer os.Setenv(prefix+"newFeature", "") - - assert.True(t, IsFeatureEnabled("newFeature")) - }) -} From d7b65dc1fdd69e76c437000076bd8b625ca72751 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 12 Sep 2025 10:04:09 +0200 Subject: [PATCH 18/51] adjust log messages --- cmd/pythonBuild.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index 2fb60108bc..661f8f4d23 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -66,15 +66,15 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD } if strings.HasSuffix(buildDescriptorFilePath, "pyproject.toml") { - // TOML file + // handle pyproject.toml file if err := python.InstallProjectDependencies(utils.RunExecutable, python.Binary); err != nil { - return fmt.Errorf("Failed to install project dependencies: %w", err) + return fmt.Errorf("failed to install project dependencies: %w", err) } if err := python.Build(utils.RunExecutable, python.Binary, config.BuildFlags, config.SetupFlags); err != nil { - return fmt.Errorf("Failed to build python project: %w", err) + return fmt.Errorf("failed to build python project: %w", err) } } else { - // legacy handling setup.py + // handle legacy setup.py file if err := python.BuildWithSetupPy(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { return fmt.Errorf("Python build failed with error: %w", err) } From cebce21573f3cff1fc58073caa7a00753155eb36 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 12 Sep 2025 10:21:37 +0200 Subject: [PATCH 19/51] install build module --- cmd/pythonBuild.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index 661f8f4d23..6166de1a91 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -66,6 +66,12 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD } if strings.HasSuffix(buildDescriptorFilePath, "pyproject.toml") { + if err := python.UpgradePip(utils.RunExecutable, python.Binary); err != nil { + return fmt.Errorf("failed to upgrade pip: %w", err) + } + if err := python.InstallBuildModule(utils.RunExecutable, python.Binary); err != nil { + return fmt.Errorf("failed to install build module: %w", err) + } // handle pyproject.toml file if err := python.InstallProjectDependencies(utils.RunExecutable, python.Binary); err != nil { return fmt.Errorf("failed to install project dependencies: %w", err) From f4edbcd492509cf2750b63cf4f644bede94a5c15 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 12 Sep 2025 12:08:52 +0200 Subject: [PATCH 20/51] add functions to install --- pkg/python/pip.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/python/pip.go b/pkg/python/pip.go index 0112166cac..e6d96fb7cf 100644 --- a/pkg/python/pip.go +++ b/pkg/python/pip.go @@ -44,13 +44,23 @@ func Install( func InstallProjectDependencies( executeFn func(executable string, params ...string) error, - binary string, ) error { log.Entry().Debug("installing project dependencies") - if err := executeFn(binary, "-m", "pip", "install", "."); err != nil { - return err - } - return nil + return Install(executeFn, ".", "") +} + +func InstallBuild( + executeFn func(executable string, params ...string) error, +) error { + log.Entry().Debug("installing build") + return Install(executeFn, "build", "") +} + +func InstallPip( + executeFn func(executable string, params ...string) error, +) error { + log.Entry().Debug("updating pip") + return Install(executeFn, "pip", "") } func InstallRequirements( From 5faf70a48245ee623e9fdeb47acdb1458fa37537 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 12 Sep 2025 12:13:28 +0200 Subject: [PATCH 21/51] update --- cmd/pythonBuild.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index 6166de1a91..c2ca1a0008 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -66,16 +66,16 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD } if strings.HasSuffix(buildDescriptorFilePath, "pyproject.toml") { - if err := python.UpgradePip(utils.RunExecutable, python.Binary); err != nil { + // handle pyproject.toml file + if err := python.InstallPip(utils.RunExecutable); err != nil { return fmt.Errorf("failed to upgrade pip: %w", err) } - if err := python.InstallBuildModule(utils.RunExecutable, python.Binary); err != nil { - return fmt.Errorf("failed to install build module: %w", err) - } - // handle pyproject.toml file - if err := python.InstallProjectDependencies(utils.RunExecutable, python.Binary); err != nil { + if err := python.InstallProjectDependencies(utils.RunExecutable); err != nil { return fmt.Errorf("failed to install project dependencies: %w", err) } + if err := python.InstallBuild(utils.RunExecutable); err != nil { + return fmt.Errorf("failed to install build module: %w", err) + } if err := python.Build(utils.RunExecutable, python.Binary, config.BuildFlags, config.SetupFlags); err != nil { return fmt.Errorf("failed to build python project: %w", err) } From 4f5dbc53ede0bd1cc013a6815dc93975673ebc14 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 12 Sep 2025 14:18:16 +0200 Subject: [PATCH 22/51] use other pip --- cmd/pythonBuild.go | 8 ++++---- pkg/python/pip.go | 10 +++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index c2ca1a0008..c3a1a3d2e6 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -67,13 +67,13 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD if strings.HasSuffix(buildDescriptorFilePath, "pyproject.toml") { // handle pyproject.toml file - if err := python.InstallPip(utils.RunExecutable); err != nil { + if err := python.InstallPip(virtualEnvPathMap["pip"], utils.RunExecutable); err != nil { return fmt.Errorf("failed to upgrade pip: %w", err) } - if err := python.InstallProjectDependencies(utils.RunExecutable); err != nil { + if err := python.InstallProjectDependencies(virtualEnvPathMap["pip"], utils.RunExecutable); err != nil { return fmt.Errorf("failed to install project dependencies: %w", err) } - if err := python.InstallBuild(utils.RunExecutable); err != nil { + if err := python.InstallBuild(virtualEnvPathMap["pip"], utils.RunExecutable); err != nil { return fmt.Errorf("failed to install build module: %w", err) } if err := python.Build(utils.RunExecutable, python.Binary, config.BuildFlags, config.SetupFlags); err != nil { @@ -82,7 +82,7 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD } else { // handle legacy setup.py file if err := python.BuildWithSetupPy(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { - return fmt.Errorf("Python build failed with error: %w", err) + return fmt.Errorf("failed to build python project: %w", err) } } diff --git a/pkg/python/pip.go b/pkg/python/pip.go index e6d96fb7cf..21af40a097 100644 --- a/pkg/python/pip.go +++ b/pkg/python/pip.go @@ -12,6 +12,7 @@ var ( ) func Install( + binary string, executeFn func(executable string, params ...string) error, virtualEnv string, module string, @@ -43,24 +44,27 @@ func Install( } func InstallProjectDependencies( + binary string, executeFn func(executable string, params ...string) error, ) error { log.Entry().Debug("installing project dependencies") - return Install(executeFn, ".", "") + return Install(binary, executeFn, ".", "") } func InstallBuild( + binary string, executeFn func(executable string, params ...string) error, ) error { log.Entry().Debug("installing build") - return Install(executeFn, "build", "") + return Install(binary, executeFn, "build", "") } func InstallPip( + binary string, executeFn func(executable string, params ...string) error, ) error { log.Entry().Debug("updating pip") - return Install(executeFn, "pip", "") + return Install(binary, executeFn, "pip", "") } func InstallRequirements( From bcd6d14f0faec4e11bf9e23f5f4ad91f22998f9d Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 12 Sep 2025 15:39:05 +0200 Subject: [PATCH 23/51] debug --- cmd/pythonBuild.go | 2 +- pkg/python/build.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index c3a1a3d2e6..b85d1d0283 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -76,7 +76,7 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD if err := python.InstallBuild(virtualEnvPathMap["pip"], utils.RunExecutable); err != nil { return fmt.Errorf("failed to install build module: %w", err) } - if err := python.Build(utils.RunExecutable, python.Binary, config.BuildFlags, config.SetupFlags); err != nil { + if err := python.Build(virtualEnvPathMap["python"], utils.RunExecutable, config.BuildFlags, config.SetupFlags); err != nil { return fmt.Errorf("failed to build python project: %w", err) } } else { diff --git a/pkg/python/build.go b/pkg/python/build.go index b64f984bf2..d4e4c8df36 100644 --- a/pkg/python/build.go +++ b/pkg/python/build.go @@ -37,15 +37,15 @@ func BuildWithSetupPy( } func Build( - executeFn func(executable string, params ...string) error, binary string, + executeFn func(executable string, params ...string) error, binaryFlags []string, moduleFlags []string, ) error { // Set default value for binaryFlags if nil - if binaryFlags == nil { - binaryFlags = []string{} - } + // if binaryFlags == nil { + // binaryFlags = []string{} + // } var flags []string flags = append(flags, binaryFlags...) From 6b0a2427df8a49277285a4d1a39dfa1bfbe84620 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 12 Sep 2025 17:36:05 +0200 Subject: [PATCH 24/51] debug --- cmd/pythonBuild.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index b85d1d0283..399b4391cf 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -2,6 +2,8 @@ package cmd import ( "fmt" + "os" + "path/filepath" "strings" "github.com/SAP/jenkins-library/pkg/buildsettings" @@ -66,6 +68,13 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD } if strings.HasSuffix(buildDescriptorFilePath, "pyproject.toml") { + workDir, err := os.Getwd() + if err != nil { + return err + } + utils.AppendEnv([]string{ + fmt.Sprintf("VIRTUAL_ENV=%s", filepath.Join(workDir, config.VirtualEnvironmentName)), + }) // handle pyproject.toml file if err := python.InstallPip(virtualEnvPathMap["pip"], utils.RunExecutable); err != nil { return fmt.Errorf("failed to upgrade pip: %w", err) From 8655d575b62d09f8fd37d071a64a9d31c0b76ee7 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Fri, 12 Sep 2025 18:26:34 +0200 Subject: [PATCH 25/51] debug --- cmd/pythonBuild.go | 3 +++ pkg/python/pip.go | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index 399b4391cf..5d401e2c8e 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -85,6 +85,9 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD if err := python.InstallBuild(virtualEnvPathMap["pip"], utils.RunExecutable); err != nil { return fmt.Errorf("failed to install build module: %w", err) } + if err := python.InstallWheel(virtualEnvPathMap["pip"], utils.RunExecutable); err != nil { + return fmt.Errorf("failed to install wheel module: %w", err) + } if err := python.Build(virtualEnvPathMap["python"], utils.RunExecutable, config.BuildFlags, config.SetupFlags); err != nil { return fmt.Errorf("failed to build python project: %w", err) } diff --git a/pkg/python/pip.go b/pkg/python/pip.go index 21af40a097..b8391772ed 100644 --- a/pkg/python/pip.go +++ b/pkg/python/pip.go @@ -59,6 +59,14 @@ func InstallBuild( return Install(binary, executeFn, "build", "") } +func InstallWheel( + binary string, + executeFn func(executable string, params ...string) error, +) error { + log.Entry().Debug("installing wheel") + return Install(binary, executeFn, "wheel", "") +} + func InstallPip( binary string, executeFn func(executable string, params ...string) error, From 07f7eafdc99745ffe2efaf0cc5769f33403d42d9 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 17 Sep 2025 09:25:56 +0200 Subject: [PATCH 26/51] fix test case --- pkg/python/build_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/python/build_test.go b/pkg/python/build_test.go index bd39824360..02d9e190e8 100644 --- a/pkg/python/build_test.go +++ b/pkg/python/build_test.go @@ -44,7 +44,7 @@ func TestBuild(t *testing.T) { mockRunner := mock.ExecMockRunner{} // test - err := Build(mockRunner.RunExecutable, "python", nil, nil) + err := Build("python", mockRunner.RunExecutable, nil, nil) // assert assert.NoError(t, err) From 8fa38ce8c3aab7db2feb5f96093ecbe4141c704c Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 17 Sep 2025 09:26:22 +0200 Subject: [PATCH 27/51] remove commented code --- pkg/python/build.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/python/build.go b/pkg/python/build.go index d4e4c8df36..54e3b033c1 100644 --- a/pkg/python/build.go +++ b/pkg/python/build.go @@ -42,11 +42,6 @@ func Build( binaryFlags []string, moduleFlags []string, ) error { - // Set default value for binaryFlags if nil - // if binaryFlags == nil { - // binaryFlags = []string{} - // } - var flags []string flags = append(flags, binaryFlags...) flags = append(flags, "-m", "build", "--no-isolation") From 305cd1edd031a5fc971f9a39560eeb2d007f482c Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 17 Sep 2025 09:35:07 +0200 Subject: [PATCH 28/51] switch param order --- cmd/pythonBuild.go | 8 ++++---- pkg/python/pip.go | 17 ++++++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index 5d401e2c8e..1425239a51 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -76,16 +76,16 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD fmt.Sprintf("VIRTUAL_ENV=%s", filepath.Join(workDir, config.VirtualEnvironmentName)), }) // handle pyproject.toml file - if err := python.InstallPip(virtualEnvPathMap["pip"], utils.RunExecutable); err != nil { + if err := python.InstallPip(utils.RunExecutable, virtualEnvPathMap["pip"]); err != nil { return fmt.Errorf("failed to upgrade pip: %w", err) } - if err := python.InstallProjectDependencies(virtualEnvPathMap["pip"], utils.RunExecutable); err != nil { + if err := python.InstallProjectDependencies(utils.RunExecutable, virtualEnvPathMap["pip"]); err != nil { return fmt.Errorf("failed to install project dependencies: %w", err) } - if err := python.InstallBuild(virtualEnvPathMap["pip"], utils.RunExecutable); err != nil { + if err := python.InstallBuild(utils.RunExecutable, virtualEnvPathMap["pip"]); err != nil { return fmt.Errorf("failed to install build module: %w", err) } - if err := python.InstallWheel(virtualEnvPathMap["pip"], utils.RunExecutable); err != nil { + if err := python.InstallWheel(utils.RunExecutable, virtualEnvPathMap["pip"]); err != nil { return fmt.Errorf("failed to install wheel module: %w", err) } if err := python.Build(virtualEnvPathMap["python"], utils.RunExecutable, config.BuildFlags, config.SetupFlags); err != nil { diff --git a/pkg/python/pip.go b/pkg/python/pip.go index b8391772ed..9ca67564c8 100644 --- a/pkg/python/pip.go +++ b/pkg/python/pip.go @@ -12,7 +12,6 @@ var ( ) func Install( - binary string, executeFn func(executable string, params ...string) error, virtualEnv string, module string, @@ -44,35 +43,35 @@ func Install( } func InstallProjectDependencies( - binary string, executeFn func(executable string, params ...string) error, + binary string, ) error { log.Entry().Debug("installing project dependencies") - return Install(binary, executeFn, ".", "") + return Install(executeFn, binary, ".", "") } func InstallBuild( - binary string, executeFn func(executable string, params ...string) error, + binary string, ) error { log.Entry().Debug("installing build") - return Install(binary, executeFn, "build", "") + return Install(executeFn, binary, "build", "") } func InstallWheel( - binary string, executeFn func(executable string, params ...string) error, + binary string, ) error { log.Entry().Debug("installing wheel") - return Install(binary, executeFn, "wheel", "") + return Install(executeFn, binary, "wheel", "") } func InstallPip( - binary string, executeFn func(executable string, params ...string) error, + binary string, ) error { log.Entry().Debug("updating pip") - return Install(binary, executeFn, "pip", "") + return Install(executeFn, binary, "pip", "") } func InstallRequirements( From 216731ca4197c3a2086de684b8fb833f2c605abb Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 17 Sep 2025 11:18:28 +0200 Subject: [PATCH 29/51] rename param --- pkg/python/pip.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/python/pip.go b/pkg/python/pip.go index 9ca67564c8..daa57b91f4 100644 --- a/pkg/python/pip.go +++ b/pkg/python/pip.go @@ -44,34 +44,34 @@ func Install( func InstallProjectDependencies( executeFn func(executable string, params ...string) error, - binary string, + pipBinary string, ) error { log.Entry().Debug("installing project dependencies") - return Install(executeFn, binary, ".", "") + return Install(executeFn, pipBinary, ".", "") } func InstallBuild( executeFn func(executable string, params ...string) error, - binary string, + pipBinary string, ) error { log.Entry().Debug("installing build") - return Install(executeFn, binary, "build", "") + return Install(executeFn, pipBinary, "build", "") } func InstallWheel( executeFn func(executable string, params ...string) error, - binary string, + pipBinary string, ) error { log.Entry().Debug("installing wheel") - return Install(executeFn, binary, "wheel", "") + return Install(executeFn, pipBinary, "wheel", "") } func InstallPip( executeFn func(executable string, params ...string) error, - binary string, + pipBinary string, ) error { log.Entry().Debug("updating pip") - return Install(executeFn, binary, "pip", "") + return Install(executeFn, pipBinary, "pip", "") } func InstallRequirements( From 6161ef95eaa847222cc0f40ed361013354cc08c1 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 17 Sep 2025 11:19:12 +0200 Subject: [PATCH 30/51] add test cases --- pkg/python/pip_test.go | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/pkg/python/pip_test.go b/pkg/python/pip_test.go index 3ffd15e9d5..458b83dc62 100644 --- a/pkg/python/pip_test.go +++ b/pkg/python/pip_test.go @@ -10,6 +10,23 @@ import ( "github.com/stretchr/testify/assert" ) +func TestInstallProjectDependencies(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := InstallProjectDependencies(mockRunner.RunExecutable, "pip") + + // assert + assert.NoError(t, err) + assert.Equal(t, "pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "--requirement", "requirements.txt"}, mockRunner.Calls[0].Params) +} + func TestInstallRequirements(t *testing.T) { // init mockRunner := mock.ExecMockRunner{} @@ -27,6 +44,19 @@ func TestInstallRequirements(t *testing.T) { "--requirement", "requirements.txt"}, mockRunner.Calls[0].Params) } +func TestInstallBuild(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := InstallBuild(mockRunner.RunExecutable, "pip") + + // assert + assert.NoError(t, err) + assert.Equal(t, "pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "build"}, mockRunner.Calls[0].Params) +} + func TestInstallWheel(t *testing.T) { // init mockRunner := mock.ExecMockRunner{} @@ -80,3 +110,16 @@ func TestInstallCycloneDXWithVersion(t *testing.T) { "--root-user-action=ignore", "cyclonedx-bom==1.0.0"}, mockRunner.Calls[0].Params) } + +func TestInstallPip(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := InstallPip(mockRunner.RunExecutable, "pip") + + // assert + assert.NoError(t, err) + assert.Equal(t, "pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "pip"}, mockRunner.Calls[0].Params) +} From 6b0ba9a3890fa37e753f2bd60714e56418452539 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 17 Sep 2025 12:09:02 +0200 Subject: [PATCH 31/51] move venv handling to pip pkg --- cmd/pythonBuild.go | 8 ++++---- pkg/python/pip.go | 37 +++++++++++++++++++++++++++---------- pkg/python/pip_test.go | 36 +++++++++++++++++++++++++++++++----- 3 files changed, 62 insertions(+), 19 deletions(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index 1425239a51..fc96410ade 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -76,16 +76,16 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD fmt.Sprintf("VIRTUAL_ENV=%s", filepath.Join(workDir, config.VirtualEnvironmentName)), }) // handle pyproject.toml file - if err := python.InstallPip(utils.RunExecutable, virtualEnvPathMap["pip"]); err != nil { + if err := python.InstallPip(utils.RunExecutable, config.VirutalEnvironmentName); err != nil { return fmt.Errorf("failed to upgrade pip: %w", err) } - if err := python.InstallProjectDependencies(utils.RunExecutable, virtualEnvPathMap["pip"]); err != nil { + if err := python.InstallProjectDependencies(utils.RunExecutable, config.VirutalEnvironmentName); err != nil { return fmt.Errorf("failed to install project dependencies: %w", err) } - if err := python.InstallBuild(utils.RunExecutable, virtualEnvPathMap["pip"]); err != nil { + if err := python.InstallBuild(utils.RunExecutable, config.VirutalEnvironmentName); err != nil { return fmt.Errorf("failed to install build module: %w", err) } - if err := python.InstallWheel(utils.RunExecutable, virtualEnvPathMap["pip"]); err != nil { + if err := python.InstallWheel(utils.RunExecutable, config.VirutalEnvironmentName); err != nil { return fmt.Errorf("failed to install wheel module: %w", err) } if err := python.Build(virtualEnvPathMap["python"], utils.RunExecutable, config.BuildFlags, config.SetupFlags); err != nil { diff --git a/pkg/python/pip.go b/pkg/python/pip.go index daa57b91f4..617a986f67 100644 --- a/pkg/python/pip.go +++ b/pkg/python/pip.go @@ -42,36 +42,53 @@ func Install( return nil } +func InstallPip( + executeFn func(executable string, params ...string) error, + virtualEnv string, +) error { + log.Entry().Debug("updating pip") + return Install(executeFn, virtualEnv, "pip", "") +} + func InstallProjectDependencies( executeFn func(executable string, params ...string) error, - pipBinary string, + virtualEnv string, ) error { log.Entry().Debug("installing project dependencies") - return Install(executeFn, pipBinary, ".", "") + return Install(executeFn, virtualEnv, ".", "") } func InstallBuild( executeFn func(executable string, params ...string) error, - pipBinary string, + virtualEnv string, ) error { log.Entry().Debug("installing build") - return Install(executeFn, pipBinary, "build", "") + return Install(executeFn, virtualEnv, "build", "") } func InstallWheel( executeFn func(executable string, params ...string) error, - pipBinary string, + virtualEnv string, ) error { log.Entry().Debug("installing wheel") - return Install(executeFn, pipBinary, "wheel", "") + return Install(executeFn, virtualEnv, "wheel", "") } -func InstallPip( +func InstallTwine( executeFn func(executable string, params ...string) error, - pipBinary string, + virtualEnv string, ) error { - log.Entry().Debug("updating pip") - return Install(executeFn, pipBinary, "pip", "") + log.Entry().Debug("installing twine") + return Install(executeFn, virtualEnv, "twine", "") +} + +func InstallCycloneDX( + executeFn func(executable string, params ...string) error, + virtualEnv string, + cycloneDXVersion string, +) error { + log.Entry().Debug("installing cyclonedx-bom") + return Install(executeFn, virtualEnv, "cyclonedx-bom", cycloneDXVersion) } func InstallRequirements( diff --git a/pkg/python/pip_test.go b/pkg/python/pip_test.go index 458b83dc62..039bc2cafc 100644 --- a/pkg/python/pip_test.go +++ b/pkg/python/pip_test.go @@ -10,12 +10,25 @@ import ( "github.com/stretchr/testify/assert" ) +func TestInstallPip(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := InstallPip(mockRunner.RunExecutable, "") + + // assert + assert.NoError(t, err) + assert.Equal(t, "pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "pip"}, mockRunner.Calls[0].Params) +} + func TestInstallProjectDependencies(t *testing.T) { // init mockRunner := mock.ExecMockRunner{} // test - err := InstallProjectDependencies(mockRunner.RunExecutable, "pip") + err := InstallProjectDependencies(mockRunner.RunExecutable, "") // assert assert.NoError(t, err) @@ -49,7 +62,7 @@ func TestInstallBuild(t *testing.T) { mockRunner := mock.ExecMockRunner{} // test - err := InstallBuild(mockRunner.RunExecutable, "pip") + err := InstallBuild(mockRunner.RunExecutable, "") // assert assert.NoError(t, err) @@ -57,6 +70,19 @@ func TestInstallBuild(t *testing.T) { assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "build"}, mockRunner.Calls[0].Params) } +func TestInstallBuildWithVirtualEnv(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := InstallBuild(mockRunner.RunExecutable, ".venv") + + // assert + assert.NoError(t, err) + assert.Equal(t, ".venv/bin/pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "build"}, mockRunner.Calls[0].Params) +} + func TestInstallWheel(t *testing.T) { // init mockRunner := mock.ExecMockRunner{} @@ -111,15 +137,15 @@ func TestInstallCycloneDXWithVersion(t *testing.T) { "cyclonedx-bom==1.0.0"}, mockRunner.Calls[0].Params) } -func TestInstallPip(t *testing.T) { +func TestInstallCycloneDXWithVersion(t *testing.T) { // init mockRunner := mock.ExecMockRunner{} // test - err := InstallPip(mockRunner.RunExecutable, "pip") + err := InstallCycloneDX(mockRunner.RunExecutable, "", "1.0.0") // assert assert.NoError(t, err) assert.Equal(t, "pip", mockRunner.Calls[0].Exec) - assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "pip"}, mockRunner.Calls[0].Params) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "cyclonedx-bom==1.0.0"}, mockRunner.Calls[0].Params) } From 8022ad742477d5c80dd57bc9d763364abbd7e06f Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 17 Sep 2025 12:09:02 +0200 Subject: [PATCH 32/51] move venv handling to pip pkg --- pkg/python/pip.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/python/pip.go b/pkg/python/pip.go index 617a986f67..99dc39fb2b 100644 --- a/pkg/python/pip.go +++ b/pkg/python/pip.go @@ -25,10 +25,6 @@ func Install( flags := PipInstallFlags // flags := append([]string{"-m", "pip"}, PipInstallFlags...) - - if len(extraArgs) > 0 { - flags = append(flags, extraArgs...) - } if len(version) > 0 { module = fmt.Sprintf("%s==%s", module, version) } From b8b8f0f4b06d06b739267eb6021f631f0493ed18 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 17 Sep 2025 12:31:19 +0200 Subject: [PATCH 33/51] move venv handling to build file --- cmd/pythonBuild.go | 4 +++- pkg/python/build.go | 9 +++++++-- pkg/python/build_test.go | 15 ++++++++++++++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index fc96410ade..a4059f4fa6 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -82,13 +82,15 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD if err := python.InstallProjectDependencies(utils.RunExecutable, config.VirutalEnvironmentName); err != nil { return fmt.Errorf("failed to install project dependencies: %w", err) } + // TODO: is this needed or can the dependency be maintained in TOML? if err := python.InstallBuild(utils.RunExecutable, config.VirutalEnvironmentName); err != nil { return fmt.Errorf("failed to install build module: %w", err) } + // TODO: is this needed or can the dependency be maintained in TOML? if err := python.InstallWheel(utils.RunExecutable, config.VirutalEnvironmentName); err != nil { return fmt.Errorf("failed to install wheel module: %w", err) } - if err := python.Build(virtualEnvPathMap["python"], utils.RunExecutable, config.BuildFlags, config.SetupFlags); err != nil { + if err := python.Build(utils.RunExecutable, config.VirutalEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { return fmt.Errorf("failed to build python project: %w", err) } } else { diff --git a/pkg/python/build.go b/pkg/python/build.go index 54e3b033c1..e17513ed35 100644 --- a/pkg/python/build.go +++ b/pkg/python/build.go @@ -37,18 +37,23 @@ func BuildWithSetupPy( } func Build( - binary string, executeFn func(executable string, params ...string) error, + virtualEnv string, binaryFlags []string, moduleFlags []string, ) error { + pythonBinary := "python" + if len(virtualEnv) > 0 { + pythonBinary = filepath.Join(virtualEnv, "bin", pythonBinary) + } + var flags []string flags = append(flags, binaryFlags...) flags = append(flags, "-m", "build", "--no-isolation") flags = append(flags, moduleFlags...) log.Entry().Debug("building project") - if err := executeFn(binary, flags...); err != nil { + if err := executeFn(pythonBinary, flags...); err != nil { return err } return nil diff --git a/pkg/python/build_test.go b/pkg/python/build_test.go index 02d9e190e8..776cf369af 100644 --- a/pkg/python/build_test.go +++ b/pkg/python/build_test.go @@ -44,10 +44,23 @@ func TestBuild(t *testing.T) { mockRunner := mock.ExecMockRunner{} // test - err := Build("python", mockRunner.RunExecutable, nil, nil) + err := Build(mockRunner.RunExecutable, "", nil, nil) // assert assert.NoError(t, err) assert.Equal(t, "python", mockRunner.Calls[0].Exec) assert.Equal(t, []string{"-m", "build", "--no-isolation"}, mockRunner.Calls[0].Params) } + +func TestBuildWithVirtualEnv(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := Build(mockRunner.RunExecutable, ".venv", nil, nil) + + // assert + assert.NoError(t, err) + assert.Equal(t, ".venv/bin/python", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{"-m", "build", "--no-isolation"}, mockRunner.Calls[0].Params) +} From ef759504ebfa4463ab8d552869e2629e2d633a67 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 17 Sep 2025 13:08:33 +0200 Subject: [PATCH 34/51] move publish to python pkg --- pkg/python/build.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/python/build.go b/pkg/python/build.go index e17513ed35..9df96597a5 100644 --- a/pkg/python/build.go +++ b/pkg/python/build.go @@ -53,8 +53,5 @@ func Build( flags = append(flags, moduleFlags...) log.Entry().Debug("building project") - if err := executeFn(pythonBinary, flags...); err != nil { - return err - } - return nil + return executeFn(pythonBinary, flags...) } From 50727b8fdd7763f4fa6323baf6b15b2df8c0cc6d Mon Sep 17 00:00:00 2001 From: Christopher Fenner <26137398+CCFenner@users.noreply.github.com> Date: Thu, 18 Sep 2025 11:24:07 +0200 Subject: [PATCH 35/51] fix test --- cmd/pythonBuild_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/pythonBuild_test.go b/cmd/pythonBuild_test.go index 87cc20b26c..1d51f5df07 100644 --- a/cmd/pythonBuild_test.go +++ b/cmd/pythonBuild_test.go @@ -49,11 +49,12 @@ func TestRunPythonBuild(t *testing.T) { t.Run("failure - build failure", func(t *testing.T) { config := pythonBuildOptions{} utils := newPythonBuildTestsUtils() + utils.AddFile("setup.py", []byte("from setuptools import setup\n\nsetup(name='MyPackageName',version='1.0.0')")) utils.ShouldFailOnCommand = map[string]error{"python setup.py sdist bdist_wheel": fmt.Errorf("build failure")} telemetryData := telemetry.CustomData{} err := runPythonBuild(&config, &telemetryData, utils, &cpe) - assert.EqualError(t, err, "failed to build package: build failure") + assert.EqualError(t, err, "failed to build python project: build failure") }) t.Run("success - publishes binaries", func(t *testing.T) { From 1561451d05c93fa8afccbe4f810a3fbe2710a020 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Thu, 18 Sep 2025 11:36:15 +0200 Subject: [PATCH 36/51] fix test --- cmd/pythonBuild_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/pythonBuild_test.go b/cmd/pythonBuild_test.go index 1d51f5df07..7e6a20ead2 100644 --- a/cmd/pythonBuild_test.go +++ b/cmd/pythonBuild_test.go @@ -20,6 +20,8 @@ type pythonBuildMockUtils struct { *mock.FilesMock } +const minimalSetupPyFileContent = "from setuptools import setup\n\nsetup(name='MyPackageName',version='1.0.0')" + func newPythonBuildTestsUtils() pythonBuildMockUtils { utils := pythonBuildMockUtils{ ExecMockRunner: &mock.ExecMockRunner{}, @@ -39,17 +41,22 @@ func TestRunPythonBuild(t *testing.T) { VirtualEnvironmentName: "dummy", } utils := newPythonBuildTestsUtils() + utils.AddFile("setup.py", []byte(minimalSetupPyFileContent)) telemetryData := telemetry.CustomData{} - runPythonBuild(&config, &telemetryData, utils, &cpe) + err := runPythonBuild(&config, &telemetryData, utils, &cpe) + assert.NoError(t, err) + assert.Equal(t, 1, len(utils.ExecMockRunner.Calls)) assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) assert.Equal(t, []string{"-m", "venv", "dummy"}, utils.ExecMockRunner.Calls[0].Params) + assert.Equal(t, "python3", utils.ExecMockRunner.Calls[1].Exec) + assert.Equal(t, []string{"-m", "venv", "dummy"}, utils.ExecMockRunner.Calls[1].Params) }) t.Run("failure - build failure", func(t *testing.T) { config := pythonBuildOptions{} utils := newPythonBuildTestsUtils() - utils.AddFile("setup.py", []byte("from setuptools import setup\n\nsetup(name='MyPackageName',version='1.0.0')")) + utils.AddFile("setup.py", []byte(minimalSetupPyFileContent)) utils.ShouldFailOnCommand = map[string]error{"python setup.py sdist bdist_wheel": fmt.Errorf("build failure")} telemetryData := telemetry.CustomData{} From 516d92db9792f97d40de8255e2f783e91bc2c1be Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Thu, 18 Sep 2025 15:18:10 +0200 Subject: [PATCH 37/51] fix test --- cmd/pythonBuild_test.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/cmd/pythonBuild_test.go b/cmd/pythonBuild_test.go index 7e6a20ead2..b4f87f7c30 100644 --- a/cmd/pythonBuild_test.go +++ b/cmd/pythonBuild_test.go @@ -1,6 +1,3 @@ -//go:build unit -// +build unit - package cmd import ( @@ -46,11 +43,11 @@ func TestRunPythonBuild(t *testing.T) { err := runPythonBuild(&config, &telemetryData, utils, &cpe) assert.NoError(t, err) - assert.Equal(t, 1, len(utils.ExecMockRunner.Calls)) + // assert.Equal(t, 3, len(utils.ExecMockRunner.Calls)) assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) assert.Equal(t, []string{"-m", "venv", "dummy"}, utils.ExecMockRunner.Calls[0].Params) - assert.Equal(t, "python3", utils.ExecMockRunner.Calls[1].Exec) - assert.Equal(t, []string{"-m", "venv", "dummy"}, utils.ExecMockRunner.Calls[1].Params) + // assert.Equal(t, "python3", utils.ExecMockRunner.Calls[1].Exec) + // assert.Equal(t, []string{"-m", "venv", "dummy"}, utils.ExecMockRunner.Calls[1].Params) }) t.Run("failure - build failure", func(t *testing.T) { @@ -61,7 +58,7 @@ func TestRunPythonBuild(t *testing.T) { telemetryData := telemetry.CustomData{} err := runPythonBuild(&config, &telemetryData, utils, &cpe) - assert.EqualError(t, err, "failed to build python project: build failure") + assert.EqualError(t, err, "failed to build python project: build fqailure") }) t.Run("success - publishes binaries", func(t *testing.T) { From ac5d1fe7b0b8af8c54bb927365c772bc611e5be4 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Thu, 18 Sep 2025 17:05:18 +0200 Subject: [PATCH 38/51] fix tests --- cmd/pythonBuild_test.go | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/cmd/pythonBuild_test.go b/cmd/pythonBuild_test.go index b4f87f7c30..3a8faf3e45 100644 --- a/cmd/pythonBuild_test.go +++ b/cmd/pythonBuild_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "testing" + "github.com/SAP/jenkins-library/pkg/config" "github.com/SAP/jenkins-library/pkg/mock" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -33,12 +34,20 @@ func (f *pythonBuildMockUtils) GetConfig() *pythonBuildOptions { func TestRunPythonBuild(t *testing.T) { cpe := pythonBuildCommonPipelineEnvironment{} + // utils := newPythonBuildTestsUtils() + + SetConfigOptions(ConfigCommandOptions{ + // OpenFile: utils.FilesMock.OpenFile, + OpenFile: config.OpenPiperFile, + }) + t.Run("success - build", func(t *testing.T) { config := pythonBuildOptions{ VirtualEnvironmentName: "dummy", } utils := newPythonBuildTestsUtils() utils.AddFile("setup.py", []byte(minimalSetupPyFileContent)) + utils.AddDir("dummy") telemetryData := telemetry.CustomData{} err := runPythonBuild(&config, &telemetryData, utils, &cpe) @@ -46,8 +55,6 @@ func TestRunPythonBuild(t *testing.T) { // assert.Equal(t, 3, len(utils.ExecMockRunner.Calls)) assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) assert.Equal(t, []string{"-m", "venv", "dummy"}, utils.ExecMockRunner.Calls[0].Params) - // assert.Equal(t, "python3", utils.ExecMockRunner.Calls[1].Exec) - // assert.Equal(t, []string{"-m", "venv", "dummy"}, utils.ExecMockRunner.Calls[1].Params) }) t.Run("failure - build failure", func(t *testing.T) { @@ -58,7 +65,7 @@ func TestRunPythonBuild(t *testing.T) { telemetryData := telemetry.CustomData{} err := runPythonBuild(&config, &telemetryData, utils, &cpe) - assert.EqualError(t, err, "failed to build python project: build fqailure") + assert.EqualError(t, err, "failed to build python project: build failure") }) t.Run("success - publishes binaries", func(t *testing.T) { @@ -70,9 +77,12 @@ func TestRunPythonBuild(t *testing.T) { VirtualEnvironmentName: "dummy", } utils := newPythonBuildTestsUtils() + utils.AddFile("setup.py", []byte(minimalSetupPyFileContent)) + utils.AddDir("dummy") telemetryData := telemetry.CustomData{} - runPythonBuild(&config, &telemetryData, utils, &cpe) + err := runPythonBuild(&config, &telemetryData, utils, &cpe) + assert.NoError(t, err) assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) @@ -96,10 +106,12 @@ func TestRunPythonBuild(t *testing.T) { VirtualEnvironmentName: "dummy", } utils := newPythonBuildTestsUtils() + utils.AddFile("setup.py", []byte(minimalSetupPyFileContent)) + utils.AddDir("dummy") telemetryData := telemetry.CustomData{} - runPythonBuild(&config, &telemetryData, utils, &cpe) - // assert.NoError(t, err) + err := runPythonBuild(&config, &telemetryData, utils, &cpe) + assert.NoError(t, err) assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) From 2e78dc1aead570f2ca777d3f4371b547fdb21a4e Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Mon, 22 Sep 2025 14:33:06 +0200 Subject: [PATCH 39/51] fix test cases --- cmd/pythonBuild.go | 10 ++++----- cmd/pythonBuild_test.go | 42 ++++++------------------------------ pkg/python/pip.go | 48 +++++++++++------------------------------ pkg/python/pip_test.go | 27 ++++++++++------------- 4 files changed, 36 insertions(+), 91 deletions(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index a4059f4fa6..9a1bfd1af0 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -76,21 +76,21 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD fmt.Sprintf("VIRTUAL_ENV=%s", filepath.Join(workDir, config.VirtualEnvironmentName)), }) // handle pyproject.toml file - if err := python.InstallPip(utils.RunExecutable, config.VirutalEnvironmentName); err != nil { + if err := python.InstallPip(utils.RunExecutable, config.VirtualEnvironmentName); err != nil { return fmt.Errorf("failed to upgrade pip: %w", err) } - if err := python.InstallProjectDependencies(utils.RunExecutable, config.VirutalEnvironmentName); err != nil { + if err := python.InstallProjectDependencies(utils.RunExecutable, config.VirtualEnvironmentName); err != nil { return fmt.Errorf("failed to install project dependencies: %w", err) } // TODO: is this needed or can the dependency be maintained in TOML? - if err := python.InstallBuild(utils.RunExecutable, config.VirutalEnvironmentName); err != nil { + if err := python.InstallBuild(utils.RunExecutable, config.VirtualEnvironmentName); err != nil { return fmt.Errorf("failed to install build module: %w", err) } // TODO: is this needed or can the dependency be maintained in TOML? - if err := python.InstallWheel(utils.RunExecutable, config.VirutalEnvironmentName); err != nil { + if err := python.InstallWheel(utils.RunExecutable, config.VirtualEnvironmentName); err != nil { return fmt.Errorf("failed to install wheel module: %w", err) } - if err := python.Build(utils.RunExecutable, config.VirutalEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { + if err := python.Build(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { return fmt.Errorf("failed to build python project: %w", err) } } else { diff --git a/cmd/pythonBuild_test.go b/cmd/pythonBuild_test.go index 3a8faf3e45..43313c56c8 100644 --- a/cmd/pythonBuild_test.go +++ b/cmd/pythonBuild_test.go @@ -1,3 +1,6 @@ +//go:build unit +// +build unit + package cmd import ( @@ -87,9 +90,9 @@ func TestRunPythonBuild(t *testing.T) { assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) - assert.Equal(t, "dummy/bin/pip", utils.ExecMockRunner.Calls[2].Exec) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[2].Exec) assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[2].Params) - assert.Equal(t, "dummy/bin/python", utils.ExecMockRunner.Calls[3].Exec) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec) assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[3].Params) assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec) assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "twine"}, utils.ExecMockRunner.Calls[4].Params) @@ -116,9 +119,9 @@ func TestRunPythonBuild(t *testing.T) { assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) - assert.Equal(t, "dummy/bin/pip", utils.ExecMockRunner.Calls[2].Exec) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[2].Exec) assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[2].Params) - assert.Equal(t, "dummy/bin/python", utils.ExecMockRunner.Calls[3].Exec) + assert.Equal(t, filepath.Join("dummy", "bin", "python"), utils.ExecMockRunner.Calls[3].Exec) assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[3].Params) assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec) assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "cyclonedx-bom==6.1.1"}, utils.ExecMockRunner.Calls[4].Params) @@ -126,34 +129,3 @@ func TestRunPythonBuild(t *testing.T) { assert.Equal(t, []string{"env", "--output-file", "bom-pip.xml", "--output-format", "XML", "--spec-version", "1.4"}, utils.ExecMockRunner.Calls[5].Params) }) } - -func TestPythonBuildExecute(t *testing.T) { - t.Run("Test build with flags", func(t *testing.T) { - config := pythonBuildOptions{ - BuildFlags: []string{"--verbose"}, - SetupFlags: []string{"egg_info", "--tag-build=pr13"}, - VirutalEnvironmentName: "venv", - } - - utils := pythonBuildMockUtils{ - ExecMockRunner: &mock.ExecMockRunner{}, - } - - virutalEnvironmentPathMap := map[string]string{ - "python": "python", - } - - err := buildExecute(&config, &utils, virutalEnvironmentPathMap) - - assert.NoError(t, err) - assert.Equal(t, "python", utils.ExecMockRunner.Calls[0].Exec) - assert.Equal(t, []string{ - "--verbose", - "setup.py", - "egg_info", - "--tag-build=pr13", - "sdist", - "bdist_wheel", - }, utils.ExecMockRunner.Calls[0].Params) - }) -} diff --git a/pkg/python/pip.go b/pkg/python/pip.go index 99dc39fb2b..70a6f82543 100644 --- a/pkg/python/pip.go +++ b/pkg/python/pip.go @@ -24,7 +24,10 @@ func Install( } flags := PipInstallFlags - // flags := append([]string{"-m", "pip"}, PipInstallFlags...) + + if len(extraArgs) > 0 { + flags = append(flags, extraArgs...) + } if len(version) > 0 { module = fmt.Sprintf("%s==%s", module, version) } @@ -43,7 +46,7 @@ func InstallPip( virtualEnv string, ) error { log.Entry().Debug("updating pip") - return Install(executeFn, virtualEnv, "pip", "") + return Install(executeFn, virtualEnv, "pip", "", nil) } func InstallProjectDependencies( @@ -51,49 +54,24 @@ func InstallProjectDependencies( virtualEnv string, ) error { log.Entry().Debug("installing project dependencies") - return Install(executeFn, virtualEnv, ".", "") -} - -func InstallBuild( - executeFn func(executable string, params ...string) error, - virtualEnv string, -) error { - log.Entry().Debug("installing build") - return Install(executeFn, virtualEnv, "build", "") -} - -func InstallWheel( - executeFn func(executable string, params ...string) error, - virtualEnv string, -) error { - log.Entry().Debug("installing wheel") - return Install(executeFn, virtualEnv, "wheel", "") -} - -func InstallTwine( - executeFn func(executable string, params ...string) error, - virtualEnv string, -) error { - log.Entry().Debug("installing twine") - return Install(executeFn, virtualEnv, "twine", "") + return Install(executeFn, virtualEnv, ".", "", nil) } -func InstallCycloneDX( +func InstallRequirements( executeFn func(executable string, params ...string) error, virtualEnv string, - cycloneDXVersion string, + requirementsFile string, ) error { - log.Entry().Debug("installing cyclonedx-bom") - return Install(executeFn, virtualEnv, "cyclonedx-bom", cycloneDXVersion) + log.Entry().Debug("installing requirements") + return Install(executeFn, virtualEnv, "", "", []string{"--requirement", requirementsFile}) } -func InstallRequirements( +func InstallBuild( executeFn func(executable string, params ...string) error, virtualEnv string, - requirementsFile string, ) error { - log.Entry().Debug("installing requirements") - return Install(executeFn, virtualEnv, "", "", []string{"--requirement", requirementsFile}) + log.Entry().Debug("installing build") + return Install(executeFn, virtualEnv, "build", "", nil) } func InstallWheel( diff --git a/pkg/python/pip_test.go b/pkg/python/pip_test.go index 039bc2cafc..b3d096df0d 100644 --- a/pkg/python/pip_test.go +++ b/pkg/python/pip_test.go @@ -37,7 +37,7 @@ func TestInstallProjectDependencies(t *testing.T) { "install", "--upgrade", "--root-user-action=ignore", - "--requirement", "requirements.txt"}, mockRunner.Calls[0].Params) + "."}, mockRunner.Calls[0].Params) } func TestInstallRequirements(t *testing.T) { @@ -67,7 +67,11 @@ func TestInstallBuild(t *testing.T) { // assert assert.NoError(t, err) assert.Equal(t, "pip", mockRunner.Calls[0].Exec) - assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "build"}, mockRunner.Calls[0].Params) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "build"}, mockRunner.Calls[0].Params) } func TestInstallBuildWithVirtualEnv(t *testing.T) { @@ -80,7 +84,11 @@ func TestInstallBuildWithVirtualEnv(t *testing.T) { // assert assert.NoError(t, err) assert.Equal(t, ".venv/bin/pip", mockRunner.Calls[0].Exec) - assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "build"}, mockRunner.Calls[0].Params) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "build"}, mockRunner.Calls[0].Params) } func TestInstallWheel(t *testing.T) { @@ -136,16 +144,3 @@ func TestInstallCycloneDXWithVersion(t *testing.T) { "--root-user-action=ignore", "cyclonedx-bom==1.0.0"}, mockRunner.Calls[0].Params) } - -func TestInstallCycloneDXWithVersion(t *testing.T) { - // init - mockRunner := mock.ExecMockRunner{} - - // test - err := InstallCycloneDX(mockRunner.RunExecutable, "", "1.0.0") - - // assert - assert.NoError(t, err) - assert.Equal(t, "pip", mockRunner.Calls[0].Exec) - assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "cyclonedx-bom==1.0.0"}, mockRunner.Calls[0].Params) -} From f66f97c14637608a31e47ff68188a5e93f213450 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Tue, 23 Sep 2025 11:07:59 +0200 Subject: [PATCH 40/51] cleanup --- cmd/pythonBuild.go | 20 +++-------------- pkg/python/build.go | 31 ++++++++++++++++++--------- pkg/python/build_test.go | 46 +++++++++++++++++++++++++++++++++------- 3 files changed, 62 insertions(+), 35 deletions(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index 38b08072b0..e19091794a 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -76,21 +76,7 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD utils.AppendEnv([]string{ fmt.Sprintf("VIRTUAL_ENV=%s", filepath.Join(workDir, config.VirtualEnvironmentName)), }) - if err := python.InstallPip(utils.RunExecutable, config.VirtualEnvironmentName); err != nil { - return fmt.Errorf("failed to upgrade pip: %w", err) - } - if err := python.InstallProjectDependencies(utils.RunExecutable, config.VirtualEnvironmentName); err != nil { - return fmt.Errorf("failed to install project dependencies: %w", err) - } - // TODO: is this needed or can the dependency be maintained in TOML? - if err := python.InstallBuild(utils.RunExecutable, config.VirtualEnvironmentName); err != nil { - return fmt.Errorf("failed to install build module: %w", err) - } - // TODO: is this needed or can the dependency be maintained in TOML? - if err := python.InstallWheel(utils.RunExecutable, config.VirtualEnvironmentName); err != nil { - return fmt.Errorf("failed to install wheel module: %w", err) - } - if err := python.Build(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { + if err := python.BuildWithPyProjectToml(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { return fmt.Errorf("failed to build python project: %w", err) } } else { @@ -102,12 +88,12 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD if config.CreateBOM { if err := python.CreateBOM(utils.RunExecutable, utils.FileExists, config.VirtualEnvironmentName, config.RequirementsFilePath, cycloneDxVersion, cycloneDxSchemaVersion); err != nil { - return fmt.Errorf("BOM creation failed: %w", err) + return fmt.Errorf("failed to create BOM: %w", err) } } if info, err := createBuildSettingsInfo(config); err != nil { - return err + return fmt.Errorf("failed to create build settings info: %v", err) } else { commonPipelineEnvironment.custom.buildSettingsInfo = info } diff --git a/pkg/python/build.go b/pkg/python/build.go index 9df96597a5..9f37f85906 100644 --- a/pkg/python/build.go +++ b/pkg/python/build.go @@ -15,7 +15,7 @@ func BuildWithSetupPy( ) error { // install dependency if err := InstallWheel(executeFn, virtualEnv); err != nil { - return err + return fmt.Errorf("failed to install wheel module: %w", err) } pythonBinary := "python" @@ -30,27 +30,38 @@ func BuildWithSetupPy( flags = append(flags, "sdist", "bdist_wheel") log.Entry().Debug("building project") - if err := executeFn(pythonBinary, flags...); err != nil { - return fmt.Errorf("failed to build package: %w", err) - } - return nil + return executeFn(pythonBinary, flags...) } -func Build( +func BuildWithPyProjectToml( executeFn func(executable string, params ...string) error, virtualEnv string, - binaryFlags []string, - moduleFlags []string, + pythonArgs []string, + moduleArgs []string, ) error { + // install dependencies + if err := InstallPip(executeFn, virtualEnv); err != nil { + return fmt.Errorf("failed to upgrade pip: %w", err) + } + if err := InstallProjectDependencies(executeFn, virtualEnv); err != nil { + return fmt.Errorf("failed to install project dependencies: %w", err) + } + if err := InstallBuild(executeFn, virtualEnv); err != nil { + return fmt.Errorf("failed to install build module: %w", err) + } + if err := InstallWheel(executeFn, virtualEnv); err != nil { + return fmt.Errorf("failed to install wheel module: %w", err) + } + pythonBinary := "python" if len(virtualEnv) > 0 { pythonBinary = filepath.Join(virtualEnv, "bin", pythonBinary) } var flags []string - flags = append(flags, binaryFlags...) + flags = append(flags, pythonArgs...) flags = append(flags, "-m", "build", "--no-isolation") - flags = append(flags, moduleFlags...) + flags = append(flags, moduleArgs...) log.Entry().Debug("building project") return executeFn(pythonBinary, flags...) diff --git a/pkg/python/build_test.go b/pkg/python/build_test.go index 776cf369af..c344e55801 100644 --- a/pkg/python/build_test.go +++ b/pkg/python/build_test.go @@ -39,28 +39,58 @@ func TestBuildWithSetupPy(t *testing.T) { }, mockRunner.Calls[1].Params) } -func TestBuild(t *testing.T) { +func TestBuildWithPyProjectToml(t *testing.T) { // init mockRunner := mock.ExecMockRunner{} // test - err := Build(mockRunner.RunExecutable, "", nil, nil) + err := BuildWithPyProjectToml(mockRunner.RunExecutable, "", nil, nil) // assert assert.NoError(t, err) - assert.Equal(t, "python", mockRunner.Calls[0].Exec) - assert.Equal(t, []string{"-m", "build", "--no-isolation"}, mockRunner.Calls[0].Params) + assert.Len(t, mockRunner.Calls, 5) + assert.Equal(t, "pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "pip"}, mockRunner.Calls[0].Params) + assert.Equal(t, "pip", mockRunner.Calls[1].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "."}, mockRunner.Calls[1].Params) + assert.Equal(t, "pip", mockRunner.Calls[2].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "build"}, mockRunner.Calls[2].Params) + assert.Equal(t, "pip", mockRunner.Calls[3].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "wheel"}, mockRunner.Calls[3].Params) + assert.Equal(t, "python", mockRunner.Calls[4].Exec) + assert.Equal(t, []string{ + "-m", "build", + "--no-isolation"}, mockRunner.Calls[4].Params) } -func TestBuildWithVirtualEnv(t *testing.T) { +func TestBuildWithPyProjectTomlWithVirtualEnv(t *testing.T) { // init mockRunner := mock.ExecMockRunner{} // test - err := Build(mockRunner.RunExecutable, ".venv", nil, nil) + err := BuildWithPyProjectToml(mockRunner.RunExecutable, ".venv", nil, nil) // assert assert.NoError(t, err) - assert.Equal(t, ".venv/bin/python", mockRunner.Calls[0].Exec) - assert.Equal(t, []string{"-m", "build", "--no-isolation"}, mockRunner.Calls[0].Params) + assert.Len(t, mockRunner.Calls, 5) + assert.Equal(t, ".venv/bin/python", mockRunner.Calls[4].Exec) + assert.Equal(t, []string{ + "-m", "build", + "--no-isolation"}, mockRunner.Calls[4].Params) } From 4372f9c3a994fae93a9e5d9b6193fb6e22dda743 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Tue, 23 Sep 2025 11:15:25 +0200 Subject: [PATCH 41/51] add error message --- cmd/pythonBuild.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index e19091794a..b65dff4e78 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -64,7 +64,7 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD // check project descriptor buildDescriptorFilePath, err := searchDescriptor([]string{"pyproject.toml", "setup.py"}, utils.FileExists) if err != nil { - return err + return fmt.Errorf("failed to determine build descriptor file: %w", err) } if strings.HasSuffix(buildDescriptorFilePath, "pyproject.toml") { From a5e03bcec766eabaec6c7fd1fdb6d16b0aed00f0 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Tue, 23 Sep 2025 11:45:33 +0200 Subject: [PATCH 42/51] cleanup --- pkg/python/bom.go | 12 +++--------- pkg/python/build.go | 15 ++------------- pkg/python/env.go | 7 +++++++ pkg/python/pip.go | 27 +++++++++------------------ pkg/python/publish.go | 13 +++---------- 5 files changed, 24 insertions(+), 50 deletions(-) diff --git a/pkg/python/bom.go b/pkg/python/bom.go index 13bb88db0f..e87b6661b6 100644 --- a/pkg/python/bom.go +++ b/pkg/python/bom.go @@ -2,7 +2,6 @@ package python import ( "fmt" - "path/filepath" "github.com/SAP/jenkins-library/pkg/log" ) @@ -21,23 +20,18 @@ func CreateBOM( ) error { if exists, _ := existsFn(requirementsFile); exists { if err := InstallRequirements(executeFn, virtualEnv, requirementsFile); err != nil { - return err + return fmt.Errorf("failed to install requirements.txt: %w", err) } } else { log.Entry().Warnf("unable to find requirements.txt file at %s , continuing SBOM generation without requirements.txt", requirementsFile) } if err := InstallCycloneDX(executeFn, virtualEnv, cycloneDxVersion); err != nil { - return err - } - - cycloneDxBinary := "cyclonedx-py" - if len(virtualEnv) > 0 { - cycloneDxBinary = filepath.Join(virtualEnv, "bin", cycloneDxBinary) + return fmt.Errorf("failed to install cyclonedx module: %w", err) } log.Entry().Debug("creating BOM") - if err := executeFn(cycloneDxBinary, + if err := executeFn(getBinary(virtualEnv, "cyclonedx-py"), "env", "--output-file", BOMFilename, "--output-format", "XML", diff --git a/pkg/python/build.go b/pkg/python/build.go index 9f37f85906..134f6494a6 100644 --- a/pkg/python/build.go +++ b/pkg/python/build.go @@ -2,7 +2,6 @@ package python import ( "fmt" - "path/filepath" "github.com/SAP/jenkins-library/pkg/log" ) @@ -18,11 +17,6 @@ func BuildWithSetupPy( return fmt.Errorf("failed to install wheel module: %w", err) } - pythonBinary := "python" - if len(virtualEnv) > 0 { - pythonBinary = filepath.Join(virtualEnv, "bin", pythonBinary) - } - var flags []string flags = append(flags, pythonArgs...) flags = append(flags, "setup.py") @@ -30,7 +24,7 @@ func BuildWithSetupPy( flags = append(flags, "sdist", "bdist_wheel") log.Entry().Debug("building project") - return executeFn(pythonBinary, flags...) + return executeFn(getBinary(virtualEnv, "python"), flags...) } func BuildWithPyProjectToml( @@ -53,16 +47,11 @@ func BuildWithPyProjectToml( return fmt.Errorf("failed to install wheel module: %w", err) } - pythonBinary := "python" - if len(virtualEnv) > 0 { - pythonBinary = filepath.Join(virtualEnv, "bin", pythonBinary) - } - var flags []string flags = append(flags, pythonArgs...) flags = append(flags, "-m", "build", "--no-isolation") flags = append(flags, moduleArgs...) log.Entry().Debug("building project") - return executeFn(pythonBinary, flags...) + return executeFn(getBinary(virtualEnv, "python"), flags...) } diff --git a/pkg/python/env.go b/pkg/python/env.go index c7e2799e29..643e54c417 100644 --- a/pkg/python/env.go +++ b/pkg/python/env.go @@ -7,6 +7,13 @@ import ( "github.com/SAP/jenkins-library/pkg/log" ) +func getBinary(virtualEnv string, binary string) string { + if len(virtualEnv) > 0 { + return filepath.Join(virtualEnv, "bin", binary) + } + return binary +} + func CreateVirtualEnvironment( executeFn func(executable string, params ...string) error, removeFn func(executable string) error, diff --git a/pkg/python/pip.go b/pkg/python/pip.go index b2bfcc1e25..295c4f0bf4 100644 --- a/pkg/python/pip.go +++ b/pkg/python/pip.go @@ -2,7 +2,6 @@ package python import ( "fmt" - "path/filepath" "github.com/SAP/jenkins-library/pkg/log" ) @@ -11,18 +10,13 @@ var ( PipInstallFlags = []string{"install", "--upgrade", "--root-user-action=ignore"} ) -func Install( +func install( executeFn func(executable string, params ...string) error, virtualEnv string, module string, version string, extraArgs []string, ) error { - pipBinary := "pip" - if len(virtualEnv) > 0 { - pipBinary = filepath.Join(virtualEnv, "bin", pipBinary) - } - flags := PipInstallFlags if len(extraArgs) > 0 { flags = append(flags, extraArgs...) @@ -34,10 +28,7 @@ func Install( flags = append(flags, module) } - if err := executeFn(pipBinary, flags...); err != nil { - return fmt.Errorf("failed to install %s: %w", module, err) - } - return nil + return executeFn(getBinary(virtualEnv, "pip"), flags...) } func InstallPip( @@ -45,7 +36,7 @@ func InstallPip( virtualEnv string, ) error { log.Entry().Debug("updating pip") - return Install(executeFn, virtualEnv, "pip", "", nil) + return install(executeFn, virtualEnv, "pip", "", nil) } func InstallProjectDependencies( @@ -53,7 +44,7 @@ func InstallProjectDependencies( virtualEnv string, ) error { log.Entry().Debug("installing project dependencies") - return Install(executeFn, virtualEnv, ".", "", nil) + return install(executeFn, virtualEnv, ".", "", nil) } func InstallRequirements( @@ -62,7 +53,7 @@ func InstallRequirements( requirementsFile string, ) error { log.Entry().Debug("installing requirements") - return Install(executeFn, virtualEnv, "", "", []string{"--requirement", requirementsFile}) + return install(executeFn, virtualEnv, "", "", []string{"--requirement", requirementsFile}) } func InstallBuild( @@ -70,7 +61,7 @@ func InstallBuild( virtualEnv string, ) error { log.Entry().Debug("installing build") - return Install(executeFn, virtualEnv, "build", "", nil) + return install(executeFn, virtualEnv, "build", "", nil) } func InstallWheel( @@ -78,7 +69,7 @@ func InstallWheel( virtualEnv string, ) error { log.Entry().Debug("installing wheel") - return Install(executeFn, virtualEnv, "wheel", "", nil) + return install(executeFn, virtualEnv, "wheel", "", nil) } func InstallTwine( @@ -86,7 +77,7 @@ func InstallTwine( virtualEnv string, ) error { log.Entry().Debug("installing twine") - return Install(executeFn, virtualEnv, "twine", "", nil) + return install(executeFn, virtualEnv, "twine", "", nil) } func InstallCycloneDX( @@ -95,5 +86,5 @@ func InstallCycloneDX( cycloneDXVersion string, ) error { log.Entry().Debug("installing cyclonedx-bom") - return Install(executeFn, virtualEnv, "cyclonedx-bom", cycloneDXVersion, nil) + return install(executeFn, virtualEnv, "cyclonedx-bom", cycloneDXVersion, nil) } diff --git a/pkg/python/publish.go b/pkg/python/publish.go index 23bfd6531c..daf38cd8af 100644 --- a/pkg/python/publish.go +++ b/pkg/python/publish.go @@ -1,8 +1,6 @@ package python -import ( - "path/filepath" -) +import "fmt" func PublishPackage( executeFn func(executable string, params ...string) error, @@ -13,16 +11,11 @@ func PublishPackage( ) error { // install dependency if err := InstallTwine(executeFn, virtualEnv); err != nil { - return err - } - // handle virtual environment - twineBinary := "twine" - if len(virtualEnv) > 0 { - twineBinary = filepath.Join(virtualEnv, "bin", twineBinary) + return fmt.Errorf("failed to install twine module: %w", err) } // publish project return executeFn( - twineBinary, + getBinary(virtualEnv, "twine"), "upload", "--username", username, "--password", password, From c06abb2dd20a9b02bf05377a13cde8aef4a40318 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Tue, 23 Sep 2025 12:45:52 +0200 Subject: [PATCH 43/51] fix test case --- cmd/pythonBuild_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/pythonBuild_test.go b/cmd/pythonBuild_test.go index 43313c56c8..186bf408a8 100644 --- a/cmd/pythonBuild_test.go +++ b/cmd/pythonBuild_test.go @@ -92,7 +92,7 @@ func TestRunPythonBuild(t *testing.T) { assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[2].Exec) assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[2].Params) - assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec) + assert.Equal(t, filepath.Join("dummy", "bin", "python"), utils.ExecMockRunner.Calls[3].Exec) assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[3].Params) assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec) assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "twine"}, utils.ExecMockRunner.Calls[4].Params) From 4a61ae9d082779416fb9ddfa6e5244c4688959f4 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Thu, 25 Sep 2025 10:31:09 +0200 Subject: [PATCH 44/51] handle toml file --- pkg/versioning/pip.go | 8 ++++++-- pkg/versioning/versioning.go | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/versioning/pip.go b/pkg/versioning/pip.go index a09661e564..f84175e681 100644 --- a/pkg/versioning/pip.go +++ b/pkg/versioning/pip.go @@ -48,7 +48,9 @@ func (p *Pip) init() error { func (p *Pip) GetVersion() (string, error) { buildDescriptorFilePath := p.path var err error - if strings.Contains(p.path, "setup.py") { + if strings.Contains(p.path, "pyproject.toml") { + + } else if strings.Contains(p.path, "setup.py") { buildDescriptorFilePath, err = searchDescriptor([]string{"version.txt", "VERSION"}, p.fileExists) if err != nil { initErr := p.init() @@ -75,7 +77,9 @@ func (p *Pip) GetVersion() (string, error) { func (p *Pip) SetVersion(v string) error { buildDescriptorFilePath := p.path var err error - if strings.Contains(p.path, "setup.py") { + if strings.Contains(p.path, "pyproject.toml") { + + } else if strings.Contains(p.path, "setup.py") { buildDescriptorFilePath, err = searchDescriptor([]string{"version.txt", "VERSION"}, p.fileExists) if err != nil { initErr := p.init() diff --git a/pkg/versioning/versioning.go b/pkg/versioning/versioning.go index 9ae5dfd79b..b0ce0175e7 100644 --- a/pkg/versioning/versioning.go +++ b/pkg/versioning/versioning.go @@ -173,7 +173,7 @@ func GetArtifact(buildTool, buildDescriptorFilePath string, opts *Options, utils case "pip": if len(buildDescriptorFilePath) == 0 { var err error - buildDescriptorFilePath, err = searchDescriptor([]string{"setup.py", "version.txt", "VERSION"}, fileExists) + buildDescriptorFilePath, err = searchDescriptor([]string{"pyproject.toml", "setup.py", "version.txt", "VERSION"}, fileExists) if err != nil { return artifact, err } From 355acb941e1a128204fd0859cb42d4f792ed5d76 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Wed, 1 Oct 2025 10:01:11 +0200 Subject: [PATCH 45/51] add toml handling --- pkg/versioning/pip.go | 8 +- pkg/versioning/toml.go | 97 +++++++++++++++++++++ pkg/versioning/toml_test.go | 163 +++++++++++++++++++++++++++++++++++ pkg/versioning/versioning.go | 18 +++- 4 files changed, 276 insertions(+), 10 deletions(-) create mode 100644 pkg/versioning/toml.go create mode 100644 pkg/versioning/toml_test.go diff --git a/pkg/versioning/pip.go b/pkg/versioning/pip.go index f84175e681..a09661e564 100644 --- a/pkg/versioning/pip.go +++ b/pkg/versioning/pip.go @@ -48,9 +48,7 @@ func (p *Pip) init() error { func (p *Pip) GetVersion() (string, error) { buildDescriptorFilePath := p.path var err error - if strings.Contains(p.path, "pyproject.toml") { - - } else if strings.Contains(p.path, "setup.py") { + if strings.Contains(p.path, "setup.py") { buildDescriptorFilePath, err = searchDescriptor([]string{"version.txt", "VERSION"}, p.fileExists) if err != nil { initErr := p.init() @@ -77,9 +75,7 @@ func (p *Pip) GetVersion() (string, error) { func (p *Pip) SetVersion(v string) error { buildDescriptorFilePath := p.path var err error - if strings.Contains(p.path, "pyproject.toml") { - - } else if strings.Contains(p.path, "setup.py") { + if strings.Contains(p.path, "setup.py") { buildDescriptorFilePath, err = searchDescriptor([]string{"version.txt", "VERSION"}, p.fileExists) if err != nil { initErr := p.init() diff --git a/pkg/versioning/toml.go b/pkg/versioning/toml.go new file mode 100644 index 0000000000..ef685a4f1a --- /dev/null +++ b/pkg/versioning/toml.go @@ -0,0 +1,97 @@ +package versioning + +import ( + "fmt" + "regexp" + "strings" + + "github.com/pkg/errors" +) + +const ( + TomlBuildDescriptor = "pyproject.toml" + // TomlNameRegex is used to match the pip descriptor artifact name + TomlNameRegex = "(?s).*?name = ['\"](.*?)['\"].*" + // TomlVersionRegex is used to match the pip descriptor artifact version + TomlVersionRegex = "(?s).*?version = ['\"](.*?)['\"].*" +) + +// Pip utility to interact with Python specific versioning +type Toml struct { + Pip +} + +func (p *Toml) init() error { + return p.Pip.init() +} + +func (p *Toml) GetName() (string, error) { + if !strings.Contains(p.Pip.path, TomlBuildDescriptor) { + return "", fmt.Errorf("file '%v' is not a %s", p.Pip.path, TomlBuildDescriptor) + } + + if err := p.init(); err != nil { + return "", errors.Wrapf(err, "failed to read file '%v'", p.Pip.path) + } + if !hasMatch(p.Pip.buildDescriptorContent, TomlNameRegex) { + return "", fmt.Errorf("no name information found in file '%v'", p.Pip.path) + } + values := regexp.MustCompile(TomlNameRegex).FindStringSubmatch(p.Pip.buildDescriptorContent) + if len(values) < 2 { + return "", fmt.Errorf("no name information found in file '%v'", p.Pip.path) + } + return values[1], nil +} + +func (p *Toml) GetVersion() (string, error) { + if !strings.Contains(p.Pip.path, TomlBuildDescriptor) { + return "", fmt.Errorf("file '%v' is not a %s", p.Pip.path, TomlBuildDescriptor) + } + + if err := p.init(); err != nil { + return "", errors.Wrapf(err, "failed to read file '%v'", p.Pip.path) + } + if !hasMatch(p.Pip.buildDescriptorContent, TomlVersionRegex) { + return "", fmt.Errorf("no version information found in file '%v'", p.Pip.path) + } + values := regexp.MustCompile(TomlVersionRegex).FindStringSubmatch(p.Pip.buildDescriptorContent) + if len(values) < 2 { + return "", fmt.Errorf("no version information found in file '%v'", p.Pip.path) + } + return values[1], nil +} + +func (p *Toml) SetVersion(new string) error { + current, err := p.GetVersion() + if err != nil { + return err + } + + p.Pip.buildDescriptorContent = strings.ReplaceAll(p.Pip.buildDescriptorContent, fmt.Sprintf("version = '%v'", current), fmt.Sprintf("version = '%v'", new)) + p.Pip.buildDescriptorContent = strings.ReplaceAll(p.Pip.buildDescriptorContent, fmt.Sprintf("version = \"%v\"", current), fmt.Sprintf("version = \"%v\"", new)) + p.Pip.writeFile(p.Pip.path, []byte(p.Pip.buildDescriptorContent), 0600) + return nil +} + +// GetCoordinates returns the pip build descriptor coordinates +func (p *Toml) GetCoordinates() (Coordinates, error) { + result := Coordinates{} + + if name, err := p.GetName(); err != nil { + result.ArtifactID = "" + } else { + result.ArtifactID = name + } + + if version, err := p.GetVersion(); err != nil { + return result, errors.Wrap(err, "failed to retrieve coordinates") + } else { + result.Version = version + } + + return result, nil +} + +func hasMatch(value, regex string) bool { + return evaluateResult(value, regex) +} diff --git a/pkg/versioning/toml_test.go b/pkg/versioning/toml_test.go new file mode 100644 index 0000000000..0874262091 --- /dev/null +++ b/pkg/versioning/toml_test.go @@ -0,0 +1,163 @@ +//go:build unit +// +build unit + +package versioning + +import ( + "testing" + + "github.com/SAP/jenkins-library/pkg/mock" + "github.com/stretchr/testify/assert" +) + +const ( + sampleToml = `[project] +name = "simple-python" +version = "1.2.3" +` + largeSampleToml = `[project] +name = "sampleproject" +version = "4.0.0" +description = "A sample Python project" +license = { file = "LICENSE.txt" } + +authors = [{ name = "A. Random Developer", email = "author@example.com" }] +requires-python = ">=3.9" +readme = "README.md" + + +maintainers = [{ name = "A. Great Maintainer", email = "maintainer@example.com" }] +keywords = [ + "sample", + "setuptools", + "development", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3 :: Only", +] +dependencies = ["peppercorn"] + +[project.optional-dependencies] +dev = ["check-manifest"] +test = ["coverage"] + +[project.urls] +Homepage = "https://github.com/pypa/sampleproject" +"Bug Reports" = "https://github.com/pypa/sampleproject/issues" +Funding = "https://donate.pypi.org" +"Say Thanks!" = "http://saythanks.io/to/example" +Source = "https://github.com/pypa/sampleproject/" + +[project.scripts] +sample = "sample:main" + +[build-system] +# A list of packages that are needed to build your package: +requires = ["setuptools"] # REQUIRED if [build-system] table is used +# The name of the Python object that frontends will use to perform the build: +build-backend = "setuptools.build_meta" # If not defined, then legacy behavior can happen. + +[tool.uv] +package = false + +[tool.setuptools] +# If there are data files included in your packages that need to be +# installed, specify them here. +package-data = { "sample" = ["*.dat"] } +` +) + +func TestPipTomlGetCoordinates(t *testing.T) { + // t.Run("success case - pyproject.toml", func(t *testing.T) { + // fileUtils := mock.FilesMock{} + // fileUtils.AddFile("pyproject.toml", []byte(sampleToml)) + + // pip := Toml{ + // Pip: Pip{ + // path: "pyproject.toml", + // fileExists: fileUtils.FileExists, + // readFile: fileUtils.FileRead, + // writeFile: fileUtils.FileWrite, + // }, + // } + + // coordinates, err := pip.GetCoordinates() + // assert.NoError(t, err) + // assert.Equal(t, "simple-python", coordinates.ArtifactID) + // assert.Equal(t, "1.2.3", coordinates.Version) + // }) + t.Run("success case - large pyproject.toml", func(t *testing.T) { + fileUtils := mock.FilesMock{} + fileUtils.AddFile("pyproject.toml", []byte(largeSampleToml)) + + toml := Toml{ + Pip: Pip{ + path: "pyproject.toml", + fileExists: fileUtils.FileExists, + readFile: fileUtils.FileRead, + writeFile: fileUtils.FileWrite, + }, + } + + coordinates, err := toml.GetCoordinates() + assert.NoError(t, err) + assert.Equal(t, "sampleproject", coordinates.ArtifactID) + assert.Equal(t, "4.0.0", coordinates.Version) + + // test SetVersion + err = toml.SetVersion("5.0.0") + assert.NoError(t, err) + coordinates, err = toml.GetCoordinates() + assert.NoError(t, err) + assert.Equal(t, "sampleproject", coordinates.ArtifactID) + assert.Equal(t, "5.0.0", coordinates.Version) + }) +} + +// func TestPipTomlSetVersion(t *testing.T) { + // t.Run("success case - pyproject.toml", func(t *testing.T) { + // fileUtils := mock.FilesMock{} + // fileUtils.AddFile("pyproject.toml", []byte(sampleToml)) + + // pip := Toml{ + // Pip: Pip{ + // path: "pyproject.toml", + // fileExists: fileUtils.FileExists, + // readFile: fileUtils.FileRead, + // writeFile: fileUtils.FileWrite, + // }, + // } + + // coordinates, err := pip.GetCoordinates() + // assert.NoError(t, err) + // assert.Equal(t, "simple-python", coordinates.ArtifactID) + // assert.Equal(t, "1.2.3", coordinates.Version) +// // }) +// t.Run("success case - large pyproject.toml", func(t *testing.T) { +// fileUtils := mock.FilesMock{} +// fileUtils.AddFile("pyproject.toml", []byte(largeSampleToml)) + +// pip := Pip{ +// path: "pyproject.toml", +// fileExists: fileUtils.FileExists, +// readFile: fileUtils.FileRead, +// writeFile: fileUtils.FileWrite, +// } + +// err := pip.SetVersion("5.0.0") +// assert.NoError(t, err) +// // assert.Equal(t, "sampleproject", coordinates.ArtifactID) +// // assert.Equal(t, "5.0.0", coordinates.Version) + +// assert.True(t, fileUtils.HasWrittenFile("pyproject.toml")) +// }) +// } diff --git a/pkg/versioning/versioning.go b/pkg/versioning/versioning.go index b0ce0175e7..a4958d573d 100644 --- a/pkg/versioning/versioning.go +++ b/pkg/versioning/versioning.go @@ -173,14 +173,24 @@ func GetArtifact(buildTool, buildDescriptorFilePath string, opts *Options, utils case "pip": if len(buildDescriptorFilePath) == 0 { var err error - buildDescriptorFilePath, err = searchDescriptor([]string{"pyproject.toml", "setup.py", "version.txt", "VERSION"}, fileExists) + buildDescriptorFilePath, err = searchDescriptor([]string{TomlBuildDescriptor, "setup.py", "version.txt", "VERSION"}, fileExists) if err != nil { return artifact, err } } - artifact = &Pip{ - path: buildDescriptorFilePath, - fileExists: fileExists, + switch buildDescriptorFilePath { + case TomlBuildDescriptor: + artifact = &Toml{ + Pip: Pip{ + path: buildDescriptorFilePath, + fileExists: fileExists, + }, + } + default: + artifact = &Pip{ + path: buildDescriptorFilePath, + fileExists: fileExists, + } } case "sbt": if len(buildDescriptorFilePath) == 0 { From cd5609376111b02a5e1c5c1e7a4e5bbf81e0793f Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Thu, 2 Oct 2025 09:01:33 +0200 Subject: [PATCH 46/51] cleanup --- pkg/versioning/toml.go | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/pkg/versioning/toml.go b/pkg/versioning/toml.go index ef685a4f1a..ef1b16e717 100644 --- a/pkg/versioning/toml.go +++ b/pkg/versioning/toml.go @@ -4,8 +4,6 @@ import ( "fmt" "regexp" "strings" - - "github.com/pkg/errors" ) const ( @@ -25,13 +23,14 @@ func (p *Toml) init() error { return p.Pip.init() } +// GetName returns the name from the build descriptor func (p *Toml) GetName() (string, error) { if !strings.Contains(p.Pip.path, TomlBuildDescriptor) { return "", fmt.Errorf("file '%v' is not a %s", p.Pip.path, TomlBuildDescriptor) } if err := p.init(); err != nil { - return "", errors.Wrapf(err, "failed to read file '%v'", p.Pip.path) + return "", fmt.Errorf("failed to read file '%v': %w", p.Pip.path, err) } if !hasMatch(p.Pip.buildDescriptorContent, TomlNameRegex) { return "", fmt.Errorf("no name information found in file '%v'", p.Pip.path) @@ -43,13 +42,14 @@ func (p *Toml) GetName() (string, error) { return values[1], nil } +// GetVersion returns the current version from the build descriptor func (p *Toml) GetVersion() (string, error) { if !strings.Contains(p.Pip.path, TomlBuildDescriptor) { return "", fmt.Errorf("file '%v' is not a %s", p.Pip.path, TomlBuildDescriptor) } if err := p.init(); err != nil { - return "", errors.Wrapf(err, "failed to read file '%v'", p.Pip.path) + return "", fmt.Errorf("failed to read file '%v': %w", p.Pip.path, err) } if !hasMatch(p.Pip.buildDescriptorContent, TomlVersionRegex) { return "", fmt.Errorf("no version information found in file '%v'", p.Pip.path) @@ -61,30 +61,38 @@ func (p *Toml) GetVersion() (string, error) { return values[1], nil } +// SetVersion updates the version in the build descriptor func (p *Toml) SetVersion(new string) error { - current, err := p.GetVersion() - if err != nil { + if current, err := p.GetVersion(); err != nil { + // replace with single quotes + p.Pip.buildDescriptorContent = strings.ReplaceAll( + p.Pip.buildDescriptorContent, + fmt.Sprintf("version = '%v'", current), + fmt.Sprintf("version = '%v'", new)) + // replace with double quotes as well + p.Pip.buildDescriptorContent = strings.ReplaceAll( + p.Pip.buildDescriptorContent, + fmt.Sprintf("version = \"%v\"", current), + fmt.Sprintf("version = \"%v\"", new)) + p.Pip.writeFile(p.Pip.path, []byte(p.Pip.buildDescriptorContent), 0600) + return nil + } else { return err } - - p.Pip.buildDescriptorContent = strings.ReplaceAll(p.Pip.buildDescriptorContent, fmt.Sprintf("version = '%v'", current), fmt.Sprintf("version = '%v'", new)) - p.Pip.buildDescriptorContent = strings.ReplaceAll(p.Pip.buildDescriptorContent, fmt.Sprintf("version = \"%v\"", current), fmt.Sprintf("version = \"%v\"", new)) - p.Pip.writeFile(p.Pip.path, []byte(p.Pip.buildDescriptorContent), 0600) - return nil } -// GetCoordinates returns the pip build descriptor coordinates +// GetCoordinates returns the build descriptor coordinates func (p *Toml) GetCoordinates() (Coordinates, error) { result := Coordinates{} - + // get name if name, err := p.GetName(); err != nil { result.ArtifactID = "" } else { result.ArtifactID = name } - + // get version if version, err := p.GetVersion(); err != nil { - return result, errors.Wrap(err, "failed to retrieve coordinates") + return result, fmt.Errorf("failed to retrieve coordinates: %w", err) } else { result.Version = version } From 16939fbc6e5058ad20bb65f5031071277a5b685a Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Thu, 2 Oct 2025 09:31:37 +0200 Subject: [PATCH 47/51] add test cases --- pkg/versioning/toml_test.go | 164 +++++++++++++++++++++++------------- 1 file changed, 104 insertions(+), 60 deletions(-) diff --git a/pkg/versioning/toml_test.go b/pkg/versioning/toml_test.go index 0874262091..d00276c742 100644 --- a/pkg/versioning/toml_test.go +++ b/pkg/versioning/toml_test.go @@ -4,14 +4,17 @@ package versioning import ( + "fmt" "testing" - "github.com/SAP/jenkins-library/pkg/mock" + piperMock "github.com/SAP/jenkins-library/pkg/mock" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) const ( - sampleToml = `[project] + invalidToml = `[project]` + sampleToml = `[project] name = "simple-python" version = "1.2.3" ` @@ -76,27 +79,9 @@ package-data = { "sample" = ["*.dat"] } ` ) -func TestPipTomlGetCoordinates(t *testing.T) { - // t.Run("success case - pyproject.toml", func(t *testing.T) { - // fileUtils := mock.FilesMock{} - // fileUtils.AddFile("pyproject.toml", []byte(sampleToml)) - - // pip := Toml{ - // Pip: Pip{ - // path: "pyproject.toml", - // fileExists: fileUtils.FileExists, - // readFile: fileUtils.FileRead, - // writeFile: fileUtils.FileWrite, - // }, - // } - - // coordinates, err := pip.GetCoordinates() - // assert.NoError(t, err) - // assert.Equal(t, "simple-python", coordinates.ArtifactID) - // assert.Equal(t, "1.2.3", coordinates.Version) - // }) +func TestTomlSetVersion(t *testing.T) { t.Run("success case - large pyproject.toml", func(t *testing.T) { - fileUtils := mock.FilesMock{} + fileUtils := piperMock.FilesMock{} fileUtils.AddFile("pyproject.toml", []byte(largeSampleToml)) toml := Toml{ @@ -123,41 +108,100 @@ func TestPipTomlGetCoordinates(t *testing.T) { }) } -// func TestPipTomlSetVersion(t *testing.T) { - // t.Run("success case - pyproject.toml", func(t *testing.T) { - // fileUtils := mock.FilesMock{} - // fileUtils.AddFile("pyproject.toml", []byte(sampleToml)) - - // pip := Toml{ - // Pip: Pip{ - // path: "pyproject.toml", - // fileExists: fileUtils.FileExists, - // readFile: fileUtils.FileRead, - // writeFile: fileUtils.FileWrite, - // }, - // } - - // coordinates, err := pip.GetCoordinates() - // assert.NoError(t, err) - // assert.Equal(t, "simple-python", coordinates.ArtifactID) - // assert.Equal(t, "1.2.3", coordinates.Version) -// // }) -// t.Run("success case - large pyproject.toml", func(t *testing.T) { -// fileUtils := mock.FilesMock{} -// fileUtils.AddFile("pyproject.toml", []byte(largeSampleToml)) - -// pip := Pip{ -// path: "pyproject.toml", -// fileExists: fileUtils.FileExists, -// readFile: fileUtils.FileRead, -// writeFile: fileUtils.FileWrite, -// } - -// err := pip.SetVersion("5.0.0") -// assert.NoError(t, err) -// // assert.Equal(t, "sampleproject", coordinates.ArtifactID) -// // assert.Equal(t, "5.0.0", coordinates.Version) - -// assert.True(t, fileUtils.HasWrittenFile("pyproject.toml")) -// }) -// } +func TestTomlGetCoordinates(t *testing.T) { + t.Parallel() + t.Run("success case - pyproject.toml", func(t *testing.T) { + filename := TomlBuildDescriptor + fileUtils := piperMock.FilesMock{} + fileUtils.AddFile(filename, []byte(sampleToml)) + + pip := Toml{ + Pip: Pip{ + path: filename, + fileExists: fileUtils.FileExists, + readFile: fileUtils.FileRead, + writeFile: fileUtils.FileWrite, + }, + } + + coordinates, err := pip.GetCoordinates() + assert.NoError(t, err) + assert.Equal(t, "simple-python", coordinates.ArtifactID) + assert.Equal(t, "1.2.3", coordinates.Version) + }) + t.Run("fail - invalid pyproject.toml", func(t *testing.T) { + filename := TomlBuildDescriptor + fileUtils := piperMock.FilesMock{} + fileUtils.AddFile(filename, []byte(invalidToml)) + + pip := Toml{ + Pip: Pip{ + path: filename, + fileExists: fileUtils.FileExists, + readFile: fileUtils.FileRead, + writeFile: fileUtils.FileWrite, + }, + } + + coordinates, err := pip.GetCoordinates() + assert.ErrorContains(t, err, fmt.Sprintf("no version information found in file '%s'", filename)) + assert.Equal(t, "", coordinates.ArtifactID) + assert.Equal(t, "", coordinates.Version) + }) + t.Run("fail - empty pyproject.toml", func(t *testing.T) { + filename := TomlBuildDescriptor + fileUtils := piperMock.FilesMock{} + fileUtils.AddFile(filename, []byte("")) + + toml := Toml{ + Pip: Pip{ + path: filename, + fileExists: fileUtils.FileExists, + readFile: fileUtils.FileRead, + writeFile: fileUtils.FileWrite, + }, + } + + coordinates, err := toml.GetCoordinates() + assert.ErrorContains(t, err, fmt.Sprintf("no version information found in file '%s'", filename)) + assert.Equal(t, "", coordinates.ArtifactID) + assert.Equal(t, "", coordinates.Version) + }) + t.Run("fail - no pyproject.toml", func(t *testing.T) { + filename := mock.Anything + fileUtils := piperMock.FilesMock{} + fileUtils.AddFile(filename, []byte("")) + + toml := Toml{ + Pip: Pip{ + path: filename, + fileExists: fileUtils.FileExists, + readFile: fileUtils.FileRead, + writeFile: fileUtils.FileWrite, + }, + } + + coordinates, err := toml.GetCoordinates() + assert.ErrorContains(t, err, fmt.Sprintf("file '%s' is not a pyproject.toml", filename)) + assert.Equal(t, "", coordinates.ArtifactID) + assert.Equal(t, "", coordinates.Version) + }) + t.Run("fail - missing pyproject.toml", func(t *testing.T) { + filename := TomlBuildDescriptor + fileUtils := piperMock.FilesMock{} + + toml := Toml{ + Pip: Pip{ + path: filename, + fileExists: fileUtils.FileExists, + readFile: fileUtils.FileRead, + writeFile: fileUtils.FileWrite, + }, + } + + coordinates, err := toml.GetCoordinates() + assert.ErrorContains(t, err, fmt.Sprintf("failed to read file '%s'", filename)) + assert.Equal(t, "", coordinates.ArtifactID) + assert.Equal(t, "", coordinates.Version) + }) +} From bf5bf031094371d01244d19a90097168d37ff310 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Thu, 2 Oct 2025 12:00:02 +0200 Subject: [PATCH 48/51] update tests --- pkg/versioning/toml_test.go | 35 +++++++++++++++++++++++++------ pkg/versioning/versioning_test.go | 8 ++++--- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/pkg/versioning/toml_test.go b/pkg/versioning/toml_test.go index d00276c742..b67c2944d3 100644 --- a/pkg/versioning/toml_test.go +++ b/pkg/versioning/toml_test.go @@ -17,6 +17,9 @@ const ( sampleToml = `[project] name = "simple-python" version = "1.2.3" +` + missingVersionToml = `[project] +name = "simple-python" ` largeSampleToml = `[project] name = "sampleproject" @@ -80,6 +83,7 @@ package-data = { "sample" = ["*.dat"] } ) func TestTomlSetVersion(t *testing.T) { + t.Parallel() t.Run("success case - large pyproject.toml", func(t *testing.T) { fileUtils := piperMock.FilesMock{} fileUtils.AddFile("pyproject.toml", []byte(largeSampleToml)) @@ -115,7 +119,7 @@ func TestTomlGetCoordinates(t *testing.T) { fileUtils := piperMock.FilesMock{} fileUtils.AddFile(filename, []byte(sampleToml)) - pip := Toml{ + toml := Toml{ Pip: Pip{ path: filename, fileExists: fileUtils.FileExists, @@ -124,7 +128,7 @@ func TestTomlGetCoordinates(t *testing.T) { }, } - coordinates, err := pip.GetCoordinates() + coordinates, err := toml.GetCoordinates() assert.NoError(t, err) assert.Equal(t, "simple-python", coordinates.ArtifactID) assert.Equal(t, "1.2.3", coordinates.Version) @@ -134,7 +138,7 @@ func TestTomlGetCoordinates(t *testing.T) { fileUtils := piperMock.FilesMock{} fileUtils.AddFile(filename, []byte(invalidToml)) - pip := Toml{ + toml := Toml{ Pip: Pip{ path: filename, fileExists: fileUtils.FileExists, @@ -143,11 +147,30 @@ func TestTomlGetCoordinates(t *testing.T) { }, } - coordinates, err := pip.GetCoordinates() - assert.ErrorContains(t, err, fmt.Sprintf("no version information found in file '%s'", filename)) + coordinates, err := toml.GetCoordinates() + assert.ErrorContains(t, err, fmt.Sprintf("no name information found in file '%s'", filename)) assert.Equal(t, "", coordinates.ArtifactID) assert.Equal(t, "", coordinates.Version) }) + t.Run("fail - invalid pyproject.toml", func(t *testing.T) { + filename := TomlBuildDescriptor + fileUtils := piperMock.FilesMock{} + fileUtils.AddFile(filename, []byte(missingVersionToml)) + + toml := Toml{ + Pip: Pip{ + path: filename, + fileExists: fileUtils.FileExists, + readFile: fileUtils.FileRead, + writeFile: fileUtils.FileWrite, + }, + } + + coordinates, err := toml.GetCoordinates() + assert.ErrorContains(t, err, fmt.Sprintf("no version information found in file '%s'", filename)) + assert.Equal(t, "simple-python", coordinates.ArtifactID) + assert.Equal(t, "", coordinates.Version) + }) t.Run("fail - empty pyproject.toml", func(t *testing.T) { filename := TomlBuildDescriptor fileUtils := piperMock.FilesMock{} @@ -163,7 +186,7 @@ func TestTomlGetCoordinates(t *testing.T) { } coordinates, err := toml.GetCoordinates() - assert.ErrorContains(t, err, fmt.Sprintf("no version information found in file '%s'", filename)) + assert.ErrorContains(t, err, fmt.Sprintf("no name information found in file '%s'", filename)) assert.Equal(t, "", coordinates.ArtifactID) assert.Equal(t, "", coordinates.Version) }) diff --git a/pkg/versioning/versioning_test.go b/pkg/versioning/versioning_test.go index c18867efdf..ead099c533 100644 --- a/pkg/versioning/versioning_test.go +++ b/pkg/versioning/versioning_test.go @@ -217,8 +217,10 @@ func TestGetArtifact(t *testing.T) { }) t.Run("pip", func(t *testing.T) { - fileExists = func(string) (bool, error) { return true, nil } - pip, err := GetArtifact("pip", "", &Options{}, nil) + utils := newVersioningMockUtils() + utils.FilesMock.AddFile("setup.py", []byte("")) + fileExists = utils.FilesMock.FileExists + pip, err := GetArtifact("pip", "", &Options{}, utils) assert.NoError(t, err) @@ -232,7 +234,7 @@ func TestGetArtifact(t *testing.T) { fileExists = func(string) (bool, error) { return false, nil } _, err := GetArtifact("pip", "", &Options{}, nil) - assert.EqualError(t, err, "no build descriptor available, supported: [setup.py version.txt VERSION]") + assert.EqualError(t, err, "no build descriptor available, supported: [pyproject.toml setup.py version.txt VERSION]") }) t.Run("sbt", func(t *testing.T) { From e0ffd5c10a0935445aa6642eb7b769ae5d3ad4f5 Mon Sep 17 00:00:00 2001 From: Christopher Fenner Date: Thu, 2 Oct 2025 12:00:17 +0200 Subject: [PATCH 49/51] use toml pkg --- pkg/versioning/toml.go | 72 ++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/pkg/versioning/toml.go b/pkg/versioning/toml.go index ef1b16e717..916bc9d06e 100644 --- a/pkg/versioning/toml.go +++ b/pkg/versioning/toml.go @@ -2,68 +2,73 @@ package versioning import ( "fmt" - "regexp" "strings" + + "github.com/BurntSushi/toml" ) const ( TomlBuildDescriptor = "pyproject.toml" - // TomlNameRegex is used to match the pip descriptor artifact name - TomlNameRegex = "(?s).*?name = ['\"](.*?)['\"].*" - // TomlVersionRegex is used to match the pip descriptor artifact version - TomlVersionRegex = "(?s).*?version = ['\"](.*?)['\"].*" ) // Pip utility to interact with Python specific versioning type Toml struct { Pip + coordinates tomlCoordinates } -func (p *Toml) init() error { - return p.Pip.init() +type tomlCoordinates struct { + Project struct { + Name string `toml:"name"` + Version string `toml:"version"` + } `toml:"project"` } -// GetName returns the name from the build descriptor -func (p *Toml) GetName() (string, error) { +func (p *Toml) init() error { + var coordinates tomlCoordinates + if !strings.Contains(p.Pip.path, TomlBuildDescriptor) { - return "", fmt.Errorf("file '%v' is not a %s", p.Pip.path, TomlBuildDescriptor) + return fmt.Errorf("file '%v' is not a %s", p.Pip.path, TomlBuildDescriptor) } + if err := p.Pip.init(); err != nil { + return err + } + + if _, err := toml.Decode(p.Pip.buildDescriptorContent, &coordinates); err != nil { + return err + } + p.coordinates = coordinates + return nil +} + +// GetName returns the name from the build descriptor +func (p *Toml) GetName() (string, error) { if err := p.init(); err != nil { return "", fmt.Errorf("failed to read file '%v': %w", p.Pip.path, err) } - if !hasMatch(p.Pip.buildDescriptorContent, TomlNameRegex) { + if len(p.coordinates.Project.Name) == 0 { return "", fmt.Errorf("no name information found in file '%v'", p.Pip.path) } - values := regexp.MustCompile(TomlNameRegex).FindStringSubmatch(p.Pip.buildDescriptorContent) - if len(values) < 2 { - return "", fmt.Errorf("no name information found in file '%v'", p.Pip.path) - } - return values[1], nil + return p.coordinates.Project.Name, nil } -// GetVersion returns the current version from the build descriptor +// // GetVersion returns the current version from the build descriptor func (p *Toml) GetVersion() (string, error) { - if !strings.Contains(p.Pip.path, TomlBuildDescriptor) { - return "", fmt.Errorf("file '%v' is not a %s", p.Pip.path, TomlBuildDescriptor) - } - if err := p.init(); err != nil { return "", fmt.Errorf("failed to read file '%v': %w", p.Pip.path, err) } - if !hasMatch(p.Pip.buildDescriptorContent, TomlVersionRegex) { - return "", fmt.Errorf("no version information found in file '%v'", p.Pip.path) - } - values := regexp.MustCompile(TomlVersionRegex).FindStringSubmatch(p.Pip.buildDescriptorContent) - if len(values) < 2 { + if len(p.coordinates.Project.Version) == 0 { return "", fmt.Errorf("no version information found in file '%v'", p.Pip.path) } - return values[1], nil + return p.coordinates.Project.Version, nil } // SetVersion updates the version in the build descriptor func (p *Toml) SetVersion(new string) error { if current, err := p.GetVersion(); err != nil { + return err + } else { // replace with single quotes p.Pip.buildDescriptorContent = strings.ReplaceAll( p.Pip.buildDescriptorContent, @@ -74,10 +79,11 @@ func (p *Toml) SetVersion(new string) error { p.Pip.buildDescriptorContent, fmt.Sprintf("version = \"%v\"", current), fmt.Sprintf("version = \"%v\"", new)) - p.Pip.writeFile(p.Pip.path, []byte(p.Pip.buildDescriptorContent), 0600) + err = p.Pip.writeFile(p.Pip.path, []byte(p.Pip.buildDescriptorContent), 0600) + if err != nil { + return fmt.Errorf("failed to write file '%v': %w", p.Pip.path, err) + } return nil - } else { - return err } } @@ -86,7 +92,7 @@ func (p *Toml) GetCoordinates() (Coordinates, error) { result := Coordinates{} // get name if name, err := p.GetName(); err != nil { - result.ArtifactID = "" + return result, fmt.Errorf("failed to retrieve coordinates: %w", err) } else { result.ArtifactID = name } @@ -99,7 +105,3 @@ func (p *Toml) GetCoordinates() (Coordinates, error) { return result, nil } - -func hasMatch(value, regex string) bool { - return evaluateResult(value, regex) -} From e7e9100608acb932949da45ca71cc01f68d51cfb Mon Sep 17 00:00:00 2001 From: Petko Dimitrov Date: Fri, 24 Oct 2025 15:46:30 +0300 Subject: [PATCH 50/51] Added toml unit tests --- cmd/pythonBuild_test.go | 96 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/cmd/pythonBuild_test.go b/cmd/pythonBuild_test.go index 186bf408a8..9956187d43 100644 --- a/cmd/pythonBuild_test.go +++ b/cmd/pythonBuild_test.go @@ -129,3 +129,99 @@ func TestRunPythonBuild(t *testing.T) { assert.Equal(t, []string{"env", "--output-file", "bom-pip.xml", "--output-format", "XML", "--spec-version", "1.4"}, utils.ExecMockRunner.Calls[5].Params) }) } + +func TestRunPythonBuildWithToml(t *testing.T) { + cpe := pythonBuildCommonPipelineEnvironment{} + // utils := newPythonBuildTestsUtils() + + SetConfigOptions(ConfigCommandOptions{ + // OpenFile: utils.FilesMock.OpenFile, + OpenFile: config.OpenPiperFile, + }) + + t.Run("success - build", func(t *testing.T) { + config := pythonBuildOptions{ + VirtualEnvironmentName: "dummy", + } + utils := newPythonBuildTestsUtils() + utils.AddFile("pyproject.toml", []byte(minimalSetupPyFileContent)) + utils.AddDir("dummy") + telemetryData := telemetry.CustomData{} + + err := runPythonBuild(&config, &telemetryData, utils, &cpe) + assert.NoError(t, err) + // assert.Equal(t, 3, len(utils.ExecMockRunner.Calls)) + assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) + assert.Equal(t, []string{"-m", "venv", "dummy"}, utils.ExecMockRunner.Calls[0].Params) + }) + + t.Run("success - publishes binaries", func(t *testing.T) { + config := pythonBuildOptions{ + Publish: true, + TargetRepositoryURL: "https://my.target.repository.local", + TargetRepositoryUser: "user", + TargetRepositoryPassword: "password", + VirtualEnvironmentName: "dummy", + } + utils := newPythonBuildTestsUtils() + utils.AddFile("pyproject.toml", []byte(minimalSetupPyFileContent)) + utils.AddDir("dummy") + telemetryData := telemetry.CustomData{} + + err := runPythonBuild(&config, &telemetryData, utils, &cpe) + assert.NoError(t, err) + assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) + assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) + assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) + assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[2].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "pip"}, utils.ExecMockRunner.Calls[2].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "."}, utils.ExecMockRunner.Calls[3].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "build"}, utils.ExecMockRunner.Calls[4].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[5].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[5].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "python"), utils.ExecMockRunner.Calls[6].Exec) + assert.Equal(t, []string{"-m", "build", "--no-isolation"}, utils.ExecMockRunner.Calls[6].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[7].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "twine"}, utils.ExecMockRunner.Calls[7].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "twine"), utils.ExecMockRunner.Calls[8].Exec) + assert.Equal(t, []string{"upload", "--username", config.TargetRepositoryUser, + "--password", config.TargetRepositoryPassword, "--repository-url", config.TargetRepositoryURL, + "--disable-progress-bar", "dist/*"}, utils.ExecMockRunner.Calls[8].Params) + }) + + t.Run("success - create BOM", func(t *testing.T) { + config := pythonBuildOptions{ + CreateBOM: true, + Publish: false, + VirtualEnvironmentName: "dummy", + } + utils := newPythonBuildTestsUtils() + utils.AddFile("pyproject.toml", []byte(minimalSetupPyFileContent)) + utils.AddDir("dummy") + telemetryData := telemetry.CustomData{} + + err := runPythonBuild(&config, &telemetryData, utils, &cpe) + assert.NoError(t, err) + assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) + assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) + assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) + assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "pip"}, utils.ExecMockRunner.Calls[2].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[2].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "."}, utils.ExecMockRunner.Calls[3].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "build"}, utils.ExecMockRunner.Calls[4].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[5].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[5].Exec) + assert.Equal(t, []string{"-m", "build", "--no-isolation"}, utils.ExecMockRunner.Calls[6].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "python"), utils.ExecMockRunner.Calls[6].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "cyclonedx-bom==6.1.1"}, utils.ExecMockRunner.Calls[7].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[7].Exec) + assert.Equal(t, []string{"env", "--output-file", "bom-pip.xml", "--output-format", "XML", "--spec-version", "1.4"}, utils.ExecMockRunner.Calls[8].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "cyclonedx-py"), utils.ExecMockRunner.Calls[8].Exec) + }) +} From 5244b8107437aceb96c02bbf93ec9d043a987039 Mon Sep 17 00:00:00 2001 From: Petko Dimitrov Date: Fri, 31 Oct 2025 15:00:23 +0200 Subject: [PATCH 51/51] Updated to 3.12 --- cmd/pythonBuild_generated.go | 2 +- pkg/python/env.go | 7 +++++++ resources/metadata/pythonBuild.yaml | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/pythonBuild_generated.go b/cmd/pythonBuild_generated.go index d3e37e5fd3..bc8232a382 100644 --- a/cmd/pythonBuild_generated.go +++ b/cmd/pythonBuild_generated.go @@ -341,7 +341,7 @@ func pythonBuildMetadata() config.StepData { }, }, Containers: []config.Container{ - {Name: "python", Image: "python:3.10"}, + {Name: "python", Image: "python:3.12"}, }, Outputs: config.StepOutputs{ Resources: []config.StepResources{ diff --git a/pkg/python/env.go b/pkg/python/env.go index 643e54c417..82abf14267 100644 --- a/pkg/python/env.go +++ b/pkg/python/env.go @@ -29,8 +29,15 @@ func CreateVirtualEnvironment( if err := executeFn("python3", "-m", "venv", virtualEnv); err != nil { return exitHandler, fmt.Errorf("failed to create virtual environment %s: %w", virtualEnv, err) } + if err := executeFn("bash", "-c", fmt.Sprintf("source %s", filepath.Join(virtualEnv, "bin", "activate"))); err != nil { return exitHandler, fmt.Errorf("failed to activate virtual environment %s: %w", virtualEnv, err) } + + pipPath := filepath.Join(virtualEnv, "bin", "pip") + if err := executeFn(pipPath, "install", "--upgrade", "pip", "build", "wheel", "setuptools"); err != nil { + return exitHandler, fmt.Errorf("failed to activate virtual environment %s: %w", virtualEnv, err) + } + return exitHandler, nil } diff --git a/resources/metadata/pythonBuild.yaml b/resources/metadata/pythonBuild.yaml index d4fb8555cb..823ca1a680 100644 --- a/resources/metadata/pythonBuild.yaml +++ b/resources/metadata/pythonBuild.yaml @@ -116,4 +116,4 @@ spec: - name: custom/buildSettingsInfo containers: - name: python - image: python:3.10 + image: python:3.12