fix(deps): update python to v3.12 for pythonBuild

cmd/pythonBuild.go

@@ -2,6 +2,9 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
 
 	"github.com/SAP/jenkins-library/pkg/buildsettings"
 	"github.com/SAP/jenkins-library/pkg/command"
@@ -58,18 +61,39 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD
 		defer exitHandler()
 	}
 
-	if err := python.BuildWithSetupPy(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil {
-		return err
+	// check project descriptor
+	buildDescriptorFilePath, err := searchDescriptor([]string{"pyproject.toml", "setup.py"}, utils.FileExists)
+	if err != nil {
+		return fmt.Errorf("failed to determine build descriptor file: %w", err)
+	}
+
+	if strings.HasSuffix(buildDescriptorFilePath, "pyproject.toml") {
+		// handle pyproject.toml file
+		workDir, err := os.Getwd()
+		if err != nil {
+			return err
+		}
+		utils.AppendEnv([]string{
+			fmt.Sprintf("VIRTUAL_ENV=%s", filepath.Join(workDir, config.VirtualEnvironmentName)),
+		})
+		if err := python.BuildWithPyProjectToml(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil {
+			return fmt.Errorf("failed to build python project: %w", err)
+		}
+	} else {
+		// handle legacy setup.py file
+		if err := python.BuildWithSetupPy(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil {
+			return fmt.Errorf("failed to build python project: %w", err)
+		}
 	}
 
 	if config.CreateBOM {
 		if err := python.CreateBOM(utils.RunExecutable, utils.FileExists, config.VirtualEnvironmentName, config.RequirementsFilePath, cycloneDxVersion, cycloneDxSchemaVersion); err != nil {
-			return fmt.Errorf("BOM creation failed: %w", err)
+			return fmt.Errorf("failed to create BOM: %w", err)
 		}
 	}
 
 	if info, err := createBuildSettingsInfo(config); err != nil {
-		return err
+		return fmt.Errorf("failed to create build settings info: %v", err)
 	} else {
 		commonPipelineEnvironment.custom.buildSettingsInfo = info
 	}
@@ -95,7 +119,6 @@ func createBuildSettingsInfo(config *pythonBuildOptions) (string, error) {
 	if err != nil {
 		return "", err
 	}
-
 	pythonConfig := buildsettings.BuildOptions{
 		CreateBOM:         config.CreateBOM,
 		Publish:           config.Publish,
@@ -108,3 +131,18 @@ func createBuildSettingsInfo(config *pythonBuildOptions) (string, error) {
 	}
 	return buildSettingsInfo, nil
 }
+
+func searchDescriptor(supported []string, existsFunc func(string) (bool, error)) (string, error) {
+	var descriptor string
+	for _, f := range supported {
+		exists, _ := existsFunc(f)
+		if exists {
+			descriptor = f
+			break
+		}
+	}
+	if len(descriptor) == 0 {
+		return "", fmt.Errorf("no build descriptor available, supported: %v", supported)
+	}
+	return descriptor, nil
+}

cmd/pythonBuild_test.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/SAP/jenkins-library/pkg/config"
 	"github.com/SAP/jenkins-library/pkg/mock"
 	"github.com/SAP/jenkins-library/pkg/telemetry"
 
@@ -20,6 +21,8 @@ type pythonBuildMockUtils struct {
 	*mock.FilesMock
 }
 
+const minimalSetupPyFileContent = "from setuptools import setup\n\nsetup(name='MyPackageName',version='1.0.0')"
+
 func newPythonBuildTestsUtils() pythonBuildMockUtils {
 	utils := pythonBuildMockUtils{
 		ExecMockRunner: &mock.ExecMockRunner{},
@@ -34,26 +37,38 @@ func (f *pythonBuildMockUtils) GetConfig() *pythonBuildOptions {
 
 func TestRunPythonBuild(t *testing.T) {
 	cpe := pythonBuildCommonPipelineEnvironment{}
+	// utils := newPythonBuildTestsUtils()
+
+	SetConfigOptions(ConfigCommandOptions{
+		// OpenFile: utils.FilesMock.OpenFile,
+		OpenFile: config.OpenPiperFile,
+	})
+
 	t.Run("success - build", func(t *testing.T) {
 		config := pythonBuildOptions{
 			VirtualEnvironmentName: "dummy",
 		}
 		utils := newPythonBuildTestsUtils()
+		utils.AddFile("setup.py", []byte(minimalSetupPyFileContent))
+		utils.AddDir("dummy")
 		telemetryData := telemetry.CustomData{}
 
-		runPythonBuild(&config, &telemetryData, utils, &cpe)
+		err := runPythonBuild(&config, &telemetryData, utils, &cpe)
+		assert.NoError(t, err)
+		// assert.Equal(t, 3, len(utils.ExecMockRunner.Calls))
 		assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec)
 		assert.Equal(t, []string{"-m", "venv", "dummy"}, utils.ExecMockRunner.Calls[0].Params)
 	})
 
 	t.Run("failure - build failure", func(t *testing.T) {
 		config := pythonBuildOptions{}
 		utils := newPythonBuildTestsUtils()
+		utils.AddFile("setup.py", []byte(minimalSetupPyFileContent))
 		utils.ShouldFailOnCommand = map[string]error{"python setup.py sdist bdist_wheel": fmt.Errorf("build failure")}
 		telemetryData := telemetry.CustomData{}
 
 		err := runPythonBuild(&config, &telemetryData, utils, &cpe)
-		assert.EqualError(t, err, "failed to build package: build failure")
+		assert.EqualError(t, err, "failed to build python project: build failure")
 	})
 
 	t.Run("success - publishes binaries", func(t *testing.T) {
@@ -65,16 +80,19 @@ func TestRunPythonBuild(t *testing.T) {
 			VirtualEnvironmentName:   "dummy",
 		}
 		utils := newPythonBuildTestsUtils()
+		utils.AddFile("setup.py", []byte(minimalSetupPyFileContent))
+		utils.AddDir("dummy")
 		telemetryData := telemetry.CustomData{}
 
-		runPythonBuild(&config, &telemetryData, utils, &cpe)
+		err := runPythonBuild(&config, &telemetryData, utils, &cpe)
+		assert.NoError(t, err)
 		assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec)
 		assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params)
 		assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec)
 		assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params)
-		assert.Equal(t, "dummy/bin/pip", utils.ExecMockRunner.Calls[2].Exec)
+		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[2].Exec)
 		assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[2].Params)
-		assert.Equal(t, "dummy/bin/python", utils.ExecMockRunner.Calls[3].Exec)
+		assert.Equal(t, filepath.Join("dummy", "bin", "python"), utils.ExecMockRunner.Calls[3].Exec)
 		assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[3].Params)
 		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec)
 		assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "twine"}, utils.ExecMockRunner.Calls[4].Params)
@@ -91,21 +109,119 @@ func TestRunPythonBuild(t *testing.T) {
 			VirtualEnvironmentName: "dummy",
 		}
 		utils := newPythonBuildTestsUtils()
+		utils.AddFile("setup.py", []byte(minimalSetupPyFileContent))
+		utils.AddDir("dummy")
 		telemetryData := telemetry.CustomData{}
 
-		runPythonBuild(&config, &telemetryData, utils, &cpe)
-		// assert.NoError(t, err)
+		err := runPythonBuild(&config, &telemetryData, utils, &cpe)
+		assert.NoError(t, err)
 		assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec)
 		assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params)
 		assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec)
 		assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params)
-		assert.Equal(t, "dummy/bin/pip", utils.ExecMockRunner.Calls[2].Exec)
+		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[2].Exec)
 		assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[2].Params)
-		assert.Equal(t, "dummy/bin/python", utils.ExecMockRunner.Calls[3].Exec)
+		assert.Equal(t, filepath.Join("dummy", "bin", "python"), utils.ExecMockRunner.Calls[3].Exec)
 		assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[3].Params)
 		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec)
 		assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "cyclonedx-bom==6.1.1"}, utils.ExecMockRunner.Calls[4].Params)
 		assert.Equal(t, filepath.Join("dummy", "bin", "cyclonedx-py"), utils.ExecMockRunner.Calls[5].Exec)
 		assert.Equal(t, []string{"env", "--output-file", "bom-pip.xml", "--output-format", "XML", "--spec-version", "1.4"}, utils.ExecMockRunner.Calls[5].Params)
 	})
 }
+
+func TestRunPythonBuildWithToml(t *testing.T) {
+	cpe := pythonBuildCommonPipelineEnvironment{}
+	// utils := newPythonBuildTestsUtils()
+
+	SetConfigOptions(ConfigCommandOptions{
+		// OpenFile: utils.FilesMock.OpenFile,
+		OpenFile: config.OpenPiperFile,
+	})
+
+	t.Run("success - build", func(t *testing.T) {
+		config := pythonBuildOptions{
+			VirtualEnvironmentName: "dummy",
+		}
+		utils := newPythonBuildTestsUtils()
+		utils.AddFile("pyproject.toml", []byte(minimalSetupPyFileContent))
+		utils.AddDir("dummy")
+		telemetryData := telemetry.CustomData{}
+
+		err := runPythonBuild(&config, &telemetryData, utils, &cpe)
+		assert.NoError(t, err)
+		// assert.Equal(t, 3, len(utils.ExecMockRunner.Calls))
+		assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec)
+		assert.Equal(t, []string{"-m", "venv", "dummy"}, utils.ExecMockRunner.Calls[0].Params)
+	})
+
+	t.Run("success - publishes binaries", func(t *testing.T) {
+		config := pythonBuildOptions{
+			Publish:                  true,
+			TargetRepositoryURL:      "https://my.target.repository.local",
+			TargetRepositoryUser:     "user",
+			TargetRepositoryPassword: "password",
+			VirtualEnvironmentName:   "dummy",
+		}
+		utils := newPythonBuildTestsUtils()
+		utils.AddFile("pyproject.toml", []byte(minimalSetupPyFileContent))
+		utils.AddDir("dummy")
+		telemetryData := telemetry.CustomData{}
+
+		err := runPythonBuild(&config, &telemetryData, utils, &cpe)
+		assert.NoError(t, err)
+		assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec)
+		assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params)
+		assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec)
+		assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[2].Exec)
+		assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "pip"}, utils.ExecMockRunner.Calls[2].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec)
+		assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "."}, utils.ExecMockRunner.Calls[3].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec)
+		assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "build"}, utils.ExecMockRunner.Calls[4].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[5].Exec)
+		assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[5].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "python"), utils.ExecMockRunner.Calls[6].Exec)
+		assert.Equal(t, []string{"-m", "build", "--no-isolation"}, utils.ExecMockRunner.Calls[6].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[7].Exec)
+		assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "twine"}, utils.ExecMockRunner.Calls[7].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "twine"), utils.ExecMockRunner.Calls[8].Exec)
+		assert.Equal(t, []string{"upload", "--username", config.TargetRepositoryUser,
+			"--password", config.TargetRepositoryPassword, "--repository-url", config.TargetRepositoryURL,
+			"--disable-progress-bar", "dist/*"}, utils.ExecMockRunner.Calls[8].Params)
+	})
+
+	t.Run("success - create BOM", func(t *testing.T) {
+		config := pythonBuildOptions{
+			CreateBOM:              true,
+			Publish:                false,
+			VirtualEnvironmentName: "dummy",
+		}
+		utils := newPythonBuildTestsUtils()
+		utils.AddFile("pyproject.toml", []byte(minimalSetupPyFileContent))
+		utils.AddDir("dummy")
+		telemetryData := telemetry.CustomData{}
+
+		err := runPythonBuild(&config, &telemetryData, utils, &cpe)
+		assert.NoError(t, err)
+		assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec)
+		assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params)
+		assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec)
+		assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params)
+		assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "pip"}, utils.ExecMockRunner.Calls[2].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[2].Exec)
+		assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "."}, utils.ExecMockRunner.Calls[3].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec)
+		assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "build"}, utils.ExecMockRunner.Calls[4].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec)
+		assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[5].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[5].Exec)
+		assert.Equal(t, []string{"-m", "build", "--no-isolation"}, utils.ExecMockRunner.Calls[6].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "python"), utils.ExecMockRunner.Calls[6].Exec)
+		assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "cyclonedx-bom==6.1.1"}, utils.ExecMockRunner.Calls[7].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[7].Exec)
+		assert.Equal(t, []string{"env", "--output-file", "bom-pip.xml", "--output-format", "XML", "--spec-version", "1.4"}, utils.ExecMockRunner.Calls[8].Params)
+		assert.Equal(t, filepath.Join("dummy", "bin", "cyclonedx-py"), utils.ExecMockRunner.Calls[8].Exec)
+	})
+}

pkg/python/bom.go

@@ -2,7 +2,6 @@ package python
 
 import (
 	"fmt"
-	"path/filepath"
 
 	"github.com/SAP/jenkins-library/pkg/log"
 )
@@ -21,23 +20,18 @@ func CreateBOM(
 ) error {
 	if exists, _ := existsFn(requirementsFile); exists {
 		if err := InstallRequirements(executeFn, virtualEnv, requirementsFile); err != nil {
-			return err
+			return fmt.Errorf("failed to install requirements.txt: %w", err)
 		}
 	} else {
 		log.Entry().Warnf("unable to find requirements.txt file at %s , continuing SBOM generation without requirements.txt", requirementsFile)
 	}
 
 	if err := InstallCycloneDX(executeFn, virtualEnv, cycloneDxVersion); err != nil {
-		return err
-	}
-
-	cycloneDxBinary := "cyclonedx-py"
-	if len(virtualEnv) > 0 {
-		cycloneDxBinary = filepath.Join(virtualEnv, "bin", cycloneDxBinary)
+		return fmt.Errorf("failed to install cyclonedx module: %w", err)
 	}
 
 	log.Entry().Debug("creating BOM")
-	if err := executeFn(cycloneDxBinary,
+	if err := executeFn(getBinary(virtualEnv, "cyclonedx-py"),
 		"env",
 		"--output-file", BOMFilename,
 		"--output-format", "XML",

pkg/python/build.go

@@ -2,7 +2,6 @@ package python
 
 import (
 	"fmt"
-	"path/filepath"
 
 	"github.com/SAP/jenkins-library/pkg/log"
 )
@@ -15,12 +14,7 @@ func BuildWithSetupPy(
 ) error {
 	// install dependency
 	if err := InstallWheel(executeFn, virtualEnv); err != nil {
-		return err
-	}
-
-	pythonBinary := "python"
-	if len(virtualEnv) > 0 {
-		pythonBinary = filepath.Join(virtualEnv, "bin", pythonBinary)
+		return fmt.Errorf("failed to install wheel module: %w", err)
 	}
 
 	var flags []string
@@ -30,8 +24,34 @@ func BuildWithSetupPy(
 	flags = append(flags, "sdist", "bdist_wheel")
 
 	log.Entry().Debug("building project")
-	if err := executeFn(pythonBinary, flags...); err != nil {
-		return fmt.Errorf("failed to build package: %w", err)
+	return executeFn(getBinary(virtualEnv, "python"), flags...)
+}
+
+func BuildWithPyProjectToml(
+	executeFn func(executable string, params ...string) error,
+	virtualEnv string,
+	pythonArgs []string,
+	moduleArgs []string,
+) error {
+	// install dependencies
+	if err := InstallPip(executeFn, virtualEnv); err != nil {
+		return fmt.Errorf("failed to upgrade pip: %w", err)
+	}
+	if err := InstallProjectDependencies(executeFn, virtualEnv); err != nil {
+		return fmt.Errorf("failed to install project dependencies: %w", err)
 	}
-	return nil
+	if err := InstallBuild(executeFn, virtualEnv); err != nil {
+		return fmt.Errorf("failed to install build module: %w", err)
+	}
+	if err := InstallWheel(executeFn, virtualEnv); err != nil {
+		return fmt.Errorf("failed to install wheel module: %w", err)
+	}
+
+	var flags []string
+	flags = append(flags, pythonArgs...)
+	flags = append(flags, "-m", "build", "--no-isolation")
+	flags = append(flags, moduleArgs...)
+
+	log.Entry().Debug("building project")
+	return executeFn(getBinary(virtualEnv, "python"), flags...)
 }