diff --git a/cmd/pythonBuild.go b/cmd/pythonBuild.go index 871f15b2dc..b65dff4e78 100644 --- a/cmd/pythonBuild.go +++ b/cmd/pythonBuild.go @@ -2,6 +2,9 @@ package cmd import ( "fmt" + "os" + "path/filepath" + "strings" "github.com/SAP/jenkins-library/pkg/buildsettings" "github.com/SAP/jenkins-library/pkg/command" @@ -58,18 +61,39 @@ func runPythonBuild(config *pythonBuildOptions, telemetryData *telemetry.CustomD defer exitHandler() } - if err := python.BuildWithSetupPy(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { - return err + // check project descriptor + buildDescriptorFilePath, err := searchDescriptor([]string{"pyproject.toml", "setup.py"}, utils.FileExists) + if err != nil { + return fmt.Errorf("failed to determine build descriptor file: %w", err) + } + + if strings.HasSuffix(buildDescriptorFilePath, "pyproject.toml") { + // handle pyproject.toml file + workDir, err := os.Getwd() + if err != nil { + return err + } + utils.AppendEnv([]string{ + fmt.Sprintf("VIRTUAL_ENV=%s", filepath.Join(workDir, config.VirtualEnvironmentName)), + }) + if err := python.BuildWithPyProjectToml(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { + return fmt.Errorf("failed to build python project: %w", err) + } + } else { + // handle legacy setup.py file + if err := python.BuildWithSetupPy(utils.RunExecutable, config.VirtualEnvironmentName, config.BuildFlags, config.SetupFlags); err != nil { + return fmt.Errorf("failed to build python project: %w", err) + } } if config.CreateBOM { if err := python.CreateBOM(utils.RunExecutable, utils.FileExists, config.VirtualEnvironmentName, config.RequirementsFilePath, cycloneDxVersion, cycloneDxSchemaVersion); err != nil { - return fmt.Errorf("BOM creation failed: %w", err) + return fmt.Errorf("failed to create BOM: %w", err) } } if info, err := createBuildSettingsInfo(config); err != nil { - return err + return fmt.Errorf("failed to create build settings info: %v", err) } else { commonPipelineEnvironment.custom.buildSettingsInfo = info } @@ -95,7 +119,6 @@ func createBuildSettingsInfo(config *pythonBuildOptions) (string, error) { if err != nil { return "", err } - pythonConfig := buildsettings.BuildOptions{ CreateBOM: config.CreateBOM, Publish: config.Publish, @@ -108,3 +131,18 @@ func createBuildSettingsInfo(config *pythonBuildOptions) (string, error) { } return buildSettingsInfo, nil } + +func searchDescriptor(supported []string, existsFunc func(string) (bool, error)) (string, error) { + var descriptor string + for _, f := range supported { + exists, _ := existsFunc(f) + if exists { + descriptor = f + break + } + } + if len(descriptor) == 0 { + return "", fmt.Errorf("no build descriptor available, supported: %v", supported) + } + return descriptor, nil +} diff --git a/cmd/pythonBuild_generated.go b/cmd/pythonBuild_generated.go index 1cfbdb78b9..bc8232a382 100644 --- a/cmd/pythonBuild_generated.go +++ b/cmd/pythonBuild_generated.go @@ -205,7 +205,7 @@ and are exposed are environment variables that must be present in the environmen func addPythonBuildFlags(cmd *cobra.Command, stepConfig *pythonBuildOptions) { cmd.Flags().StringSliceVar(&stepConfig.BuildFlags, "buildFlags", []string{}, "Defines list of build flags passed to python binary.") - cmd.Flags().StringSliceVar(&stepConfig.SetupFlags, "setupFlags", []string{}, "Defines list of flags passed to setup.py.") + cmd.Flags().StringSliceVar(&stepConfig.SetupFlags, "setupFlags", []string{}, "Defines list of flags passed to setup.py / build module.") cmd.Flags().BoolVar(&stepConfig.CreateBOM, "createBOM", false, "Creates the bill of materials (BOM) using CycloneDX plugin.") cmd.Flags().BoolVar(&stepConfig.Publish, "publish", false, "Configures the build to publish artifacts to a repository.") cmd.Flags().StringVar(&stepConfig.TargetRepositoryPassword, "targetRepositoryPassword", os.Getenv("PIPER_targetRepositoryPassword"), "Password for the target repository where the compiled binaries shall be uploaded - typically provided by the CI/CD environment.") @@ -341,7 +341,7 @@ func pythonBuildMetadata() config.StepData { }, }, Containers: []config.Container{ - {Name: "python", Image: "python:3.10"}, + {Name: "python", Image: "python:3.12"}, }, Outputs: config.StepOutputs{ Resources: []config.StepResources{ diff --git a/cmd/pythonBuild_test.go b/cmd/pythonBuild_test.go index 39496bf584..9956187d43 100644 --- a/cmd/pythonBuild_test.go +++ b/cmd/pythonBuild_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "testing" + "github.com/SAP/jenkins-library/pkg/config" "github.com/SAP/jenkins-library/pkg/mock" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -20,6 +21,8 @@ type pythonBuildMockUtils struct { *mock.FilesMock } +const minimalSetupPyFileContent = "from setuptools import setup\n\nsetup(name='MyPackageName',version='1.0.0')" + func newPythonBuildTestsUtils() pythonBuildMockUtils { utils := pythonBuildMockUtils{ ExecMockRunner: &mock.ExecMockRunner{}, @@ -34,14 +37,25 @@ func (f *pythonBuildMockUtils) GetConfig() *pythonBuildOptions { func TestRunPythonBuild(t *testing.T) { cpe := pythonBuildCommonPipelineEnvironment{} + // utils := newPythonBuildTestsUtils() + + SetConfigOptions(ConfigCommandOptions{ + // OpenFile: utils.FilesMock.OpenFile, + OpenFile: config.OpenPiperFile, + }) + t.Run("success - build", func(t *testing.T) { config := pythonBuildOptions{ VirtualEnvironmentName: "dummy", } utils := newPythonBuildTestsUtils() + utils.AddFile("setup.py", []byte(minimalSetupPyFileContent)) + utils.AddDir("dummy") telemetryData := telemetry.CustomData{} - runPythonBuild(&config, &telemetryData, utils, &cpe) + err := runPythonBuild(&config, &telemetryData, utils, &cpe) + assert.NoError(t, err) + // assert.Equal(t, 3, len(utils.ExecMockRunner.Calls)) assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) assert.Equal(t, []string{"-m", "venv", "dummy"}, utils.ExecMockRunner.Calls[0].Params) }) @@ -49,11 +63,12 @@ func TestRunPythonBuild(t *testing.T) { t.Run("failure - build failure", func(t *testing.T) { config := pythonBuildOptions{} utils := newPythonBuildTestsUtils() + utils.AddFile("setup.py", []byte(minimalSetupPyFileContent)) utils.ShouldFailOnCommand = map[string]error{"python setup.py sdist bdist_wheel": fmt.Errorf("build failure")} telemetryData := telemetry.CustomData{} err := runPythonBuild(&config, &telemetryData, utils, &cpe) - assert.EqualError(t, err, "failed to build package: build failure") + assert.EqualError(t, err, "failed to build python project: build failure") }) t.Run("success - publishes binaries", func(t *testing.T) { @@ -65,16 +80,19 @@ func TestRunPythonBuild(t *testing.T) { VirtualEnvironmentName: "dummy", } utils := newPythonBuildTestsUtils() + utils.AddFile("setup.py", []byte(minimalSetupPyFileContent)) + utils.AddDir("dummy") telemetryData := telemetry.CustomData{} - runPythonBuild(&config, &telemetryData, utils, &cpe) + err := runPythonBuild(&config, &telemetryData, utils, &cpe) + assert.NoError(t, err) assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) - assert.Equal(t, "dummy/bin/pip", utils.ExecMockRunner.Calls[2].Exec) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[2].Exec) assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[2].Params) - assert.Equal(t, "dummy/bin/python", utils.ExecMockRunner.Calls[3].Exec) + assert.Equal(t, filepath.Join("dummy", "bin", "python"), utils.ExecMockRunner.Calls[3].Exec) assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[3].Params) assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec) assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "twine"}, utils.ExecMockRunner.Calls[4].Params) @@ -91,17 +109,19 @@ func TestRunPythonBuild(t *testing.T) { VirtualEnvironmentName: "dummy", } utils := newPythonBuildTestsUtils() + utils.AddFile("setup.py", []byte(minimalSetupPyFileContent)) + utils.AddDir("dummy") telemetryData := telemetry.CustomData{} - runPythonBuild(&config, &telemetryData, utils, &cpe) - // assert.NoError(t, err) + err := runPythonBuild(&config, &telemetryData, utils, &cpe) + assert.NoError(t, err) assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) - assert.Equal(t, "dummy/bin/pip", utils.ExecMockRunner.Calls[2].Exec) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[2].Exec) assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[2].Params) - assert.Equal(t, "dummy/bin/python", utils.ExecMockRunner.Calls[3].Exec) + assert.Equal(t, filepath.Join("dummy", "bin", "python"), utils.ExecMockRunner.Calls[3].Exec) assert.Equal(t, []string{"setup.py", "sdist", "bdist_wheel"}, utils.ExecMockRunner.Calls[3].Params) assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec) assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "cyclonedx-bom==6.1.1"}, utils.ExecMockRunner.Calls[4].Params) @@ -109,3 +129,99 @@ func TestRunPythonBuild(t *testing.T) { assert.Equal(t, []string{"env", "--output-file", "bom-pip.xml", "--output-format", "XML", "--spec-version", "1.4"}, utils.ExecMockRunner.Calls[5].Params) }) } + +func TestRunPythonBuildWithToml(t *testing.T) { + cpe := pythonBuildCommonPipelineEnvironment{} + // utils := newPythonBuildTestsUtils() + + SetConfigOptions(ConfigCommandOptions{ + // OpenFile: utils.FilesMock.OpenFile, + OpenFile: config.OpenPiperFile, + }) + + t.Run("success - build", func(t *testing.T) { + config := pythonBuildOptions{ + VirtualEnvironmentName: "dummy", + } + utils := newPythonBuildTestsUtils() + utils.AddFile("pyproject.toml", []byte(minimalSetupPyFileContent)) + utils.AddDir("dummy") + telemetryData := telemetry.CustomData{} + + err := runPythonBuild(&config, &telemetryData, utils, &cpe) + assert.NoError(t, err) + // assert.Equal(t, 3, len(utils.ExecMockRunner.Calls)) + assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) + assert.Equal(t, []string{"-m", "venv", "dummy"}, utils.ExecMockRunner.Calls[0].Params) + }) + + t.Run("success - publishes binaries", func(t *testing.T) { + config := pythonBuildOptions{ + Publish: true, + TargetRepositoryURL: "https://my.target.repository.local", + TargetRepositoryUser: "user", + TargetRepositoryPassword: "password", + VirtualEnvironmentName: "dummy", + } + utils := newPythonBuildTestsUtils() + utils.AddFile("pyproject.toml", []byte(minimalSetupPyFileContent)) + utils.AddDir("dummy") + telemetryData := telemetry.CustomData{} + + err := runPythonBuild(&config, &telemetryData, utils, &cpe) + assert.NoError(t, err) + assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) + assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) + assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) + assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[2].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "pip"}, utils.ExecMockRunner.Calls[2].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "."}, utils.ExecMockRunner.Calls[3].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "build"}, utils.ExecMockRunner.Calls[4].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[5].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[5].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "python"), utils.ExecMockRunner.Calls[6].Exec) + assert.Equal(t, []string{"-m", "build", "--no-isolation"}, utils.ExecMockRunner.Calls[6].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[7].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "twine"}, utils.ExecMockRunner.Calls[7].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "twine"), utils.ExecMockRunner.Calls[8].Exec) + assert.Equal(t, []string{"upload", "--username", config.TargetRepositoryUser, + "--password", config.TargetRepositoryPassword, "--repository-url", config.TargetRepositoryURL, + "--disable-progress-bar", "dist/*"}, utils.ExecMockRunner.Calls[8].Params) + }) + + t.Run("success - create BOM", func(t *testing.T) { + config := pythonBuildOptions{ + CreateBOM: true, + Publish: false, + VirtualEnvironmentName: "dummy", + } + utils := newPythonBuildTestsUtils() + utils.AddFile("pyproject.toml", []byte(minimalSetupPyFileContent)) + utils.AddDir("dummy") + telemetryData := telemetry.CustomData{} + + err := runPythonBuild(&config, &telemetryData, utils, &cpe) + assert.NoError(t, err) + assert.Equal(t, "python3", utils.ExecMockRunner.Calls[0].Exec) + assert.Equal(t, []string{"-m", "venv", config.VirtualEnvironmentName}, utils.ExecMockRunner.Calls[0].Params) + assert.Equal(t, "bash", utils.ExecMockRunner.Calls[1].Exec) + assert.Equal(t, []string{"-c", "source " + filepath.Join("dummy", "bin", "activate")}, utils.ExecMockRunner.Calls[1].Params) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "pip"}, utils.ExecMockRunner.Calls[2].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[2].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "."}, utils.ExecMockRunner.Calls[3].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[3].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "build"}, utils.ExecMockRunner.Calls[4].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[4].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "wheel"}, utils.ExecMockRunner.Calls[5].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[5].Exec) + assert.Equal(t, []string{"-m", "build", "--no-isolation"}, utils.ExecMockRunner.Calls[6].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "python"), utils.ExecMockRunner.Calls[6].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "cyclonedx-bom==6.1.1"}, utils.ExecMockRunner.Calls[7].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "pip"), utils.ExecMockRunner.Calls[7].Exec) + assert.Equal(t, []string{"env", "--output-file", "bom-pip.xml", "--output-format", "XML", "--spec-version", "1.4"}, utils.ExecMockRunner.Calls[8].Params) + assert.Equal(t, filepath.Join("dummy", "bin", "cyclonedx-py"), utils.ExecMockRunner.Calls[8].Exec) + }) +} diff --git a/pkg/python/bom.go b/pkg/python/bom.go index 13bb88db0f..e87b6661b6 100644 --- a/pkg/python/bom.go +++ b/pkg/python/bom.go @@ -2,7 +2,6 @@ package python import ( "fmt" - "path/filepath" "github.com/SAP/jenkins-library/pkg/log" ) @@ -21,23 +20,18 @@ func CreateBOM( ) error { if exists, _ := existsFn(requirementsFile); exists { if err := InstallRequirements(executeFn, virtualEnv, requirementsFile); err != nil { - return err + return fmt.Errorf("failed to install requirements.txt: %w", err) } } else { log.Entry().Warnf("unable to find requirements.txt file at %s , continuing SBOM generation without requirements.txt", requirementsFile) } if err := InstallCycloneDX(executeFn, virtualEnv, cycloneDxVersion); err != nil { - return err - } - - cycloneDxBinary := "cyclonedx-py" - if len(virtualEnv) > 0 { - cycloneDxBinary = filepath.Join(virtualEnv, "bin", cycloneDxBinary) + return fmt.Errorf("failed to install cyclonedx module: %w", err) } log.Entry().Debug("creating BOM") - if err := executeFn(cycloneDxBinary, + if err := executeFn(getBinary(virtualEnv, "cyclonedx-py"), "env", "--output-file", BOMFilename, "--output-format", "XML", diff --git a/pkg/python/build.go b/pkg/python/build.go index dc7606f573..134f6494a6 100644 --- a/pkg/python/build.go +++ b/pkg/python/build.go @@ -2,7 +2,6 @@ package python import ( "fmt" - "path/filepath" "github.com/SAP/jenkins-library/pkg/log" ) @@ -15,12 +14,7 @@ func BuildWithSetupPy( ) error { // install dependency if err := InstallWheel(executeFn, virtualEnv); err != nil { - return err - } - - pythonBinary := "python" - if len(virtualEnv) > 0 { - pythonBinary = filepath.Join(virtualEnv, "bin", pythonBinary) + return fmt.Errorf("failed to install wheel module: %w", err) } var flags []string @@ -30,8 +24,34 @@ func BuildWithSetupPy( flags = append(flags, "sdist", "bdist_wheel") log.Entry().Debug("building project") - if err := executeFn(pythonBinary, flags...); err != nil { - return fmt.Errorf("failed to build package: %w", err) + return executeFn(getBinary(virtualEnv, "python"), flags...) +} + +func BuildWithPyProjectToml( + executeFn func(executable string, params ...string) error, + virtualEnv string, + pythonArgs []string, + moduleArgs []string, +) error { + // install dependencies + if err := InstallPip(executeFn, virtualEnv); err != nil { + return fmt.Errorf("failed to upgrade pip: %w", err) + } + if err := InstallProjectDependencies(executeFn, virtualEnv); err != nil { + return fmt.Errorf("failed to install project dependencies: %w", err) } - return nil + if err := InstallBuild(executeFn, virtualEnv); err != nil { + return fmt.Errorf("failed to install build module: %w", err) + } + if err := InstallWheel(executeFn, virtualEnv); err != nil { + return fmt.Errorf("failed to install wheel module: %w", err) + } + + var flags []string + flags = append(flags, pythonArgs...) + flags = append(flags, "-m", "build", "--no-isolation") + flags = append(flags, moduleArgs...) + + log.Entry().Debug("building project") + return executeFn(getBinary(virtualEnv, "python"), flags...) } diff --git a/pkg/python/build_test.go b/pkg/python/build_test.go index 732bb1da47..c344e55801 100644 --- a/pkg/python/build_test.go +++ b/pkg/python/build_test.go @@ -38,3 +38,59 @@ func TestBuildWithSetupPy(t *testing.T) { "bdist_wheel", }, mockRunner.Calls[1].Params) } + +func TestBuildWithPyProjectToml(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := BuildWithPyProjectToml(mockRunner.RunExecutable, "", nil, nil) + + // assert + assert.NoError(t, err) + assert.Len(t, mockRunner.Calls, 5) + assert.Equal(t, "pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "pip"}, mockRunner.Calls[0].Params) + assert.Equal(t, "pip", mockRunner.Calls[1].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "."}, mockRunner.Calls[1].Params) + assert.Equal(t, "pip", mockRunner.Calls[2].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "build"}, mockRunner.Calls[2].Params) + assert.Equal(t, "pip", mockRunner.Calls[3].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "wheel"}, mockRunner.Calls[3].Params) + assert.Equal(t, "python", mockRunner.Calls[4].Exec) + assert.Equal(t, []string{ + "-m", "build", + "--no-isolation"}, mockRunner.Calls[4].Params) +} + +func TestBuildWithPyProjectTomlWithVirtualEnv(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := BuildWithPyProjectToml(mockRunner.RunExecutable, ".venv", nil, nil) + + // assert + assert.NoError(t, err) + assert.Len(t, mockRunner.Calls, 5) + assert.Equal(t, ".venv/bin/python", mockRunner.Calls[4].Exec) + assert.Equal(t, []string{ + "-m", "build", + "--no-isolation"}, mockRunner.Calls[4].Params) +} diff --git a/pkg/python/env.go b/pkg/python/env.go index c7e2799e29..82abf14267 100644 --- a/pkg/python/env.go +++ b/pkg/python/env.go @@ -7,6 +7,13 @@ import ( "github.com/SAP/jenkins-library/pkg/log" ) +func getBinary(virtualEnv string, binary string) string { + if len(virtualEnv) > 0 { + return filepath.Join(virtualEnv, "bin", binary) + } + return binary +} + func CreateVirtualEnvironment( executeFn func(executable string, params ...string) error, removeFn func(executable string) error, @@ -22,8 +29,15 @@ func CreateVirtualEnvironment( if err := executeFn("python3", "-m", "venv", virtualEnv); err != nil { return exitHandler, fmt.Errorf("failed to create virtual environment %s: %w", virtualEnv, err) } + if err := executeFn("bash", "-c", fmt.Sprintf("source %s", filepath.Join(virtualEnv, "bin", "activate"))); err != nil { return exitHandler, fmt.Errorf("failed to activate virtual environment %s: %w", virtualEnv, err) } + + pipPath := filepath.Join(virtualEnv, "bin", "pip") + if err := executeFn(pipPath, "install", "--upgrade", "pip", "build", "wheel", "setuptools"); err != nil { + return exitHandler, fmt.Errorf("failed to activate virtual environment %s: %w", virtualEnv, err) + } + return exitHandler, nil } diff --git a/pkg/python/pip.go b/pkg/python/pip.go index 97c7546c7b..295c4f0bf4 100644 --- a/pkg/python/pip.go +++ b/pkg/python/pip.go @@ -2,7 +2,6 @@ package python import ( "fmt" - "path/filepath" "github.com/SAP/jenkins-library/pkg/log" ) @@ -11,21 +10,14 @@ var ( PipInstallFlags = []string{"install", "--upgrade", "--root-user-action=ignore"} ) -func Install( +func install( executeFn func(executable string, params ...string) error, virtualEnv string, module string, version string, extraArgs []string, ) error { - pipBinary := "pip" - if len(virtualEnv) > 0 { - pipBinary = filepath.Join(virtualEnv, "bin", pipBinary) - } - flags := PipInstallFlags - // flags := append([]string{"-m", "pip"}, PipInstallFlags...) - if len(extraArgs) > 0 { flags = append(flags, extraArgs...) } @@ -36,10 +28,23 @@ func Install( flags = append(flags, module) } - if err := executeFn(pipBinary, flags...); err != nil { - return fmt.Errorf("failed to install %s: %w", module, err) - } - return nil + return executeFn(getBinary(virtualEnv, "pip"), flags...) +} + +func InstallPip( + executeFn func(executable string, params ...string) error, + virtualEnv string, +) error { + log.Entry().Debug("updating pip") + return install(executeFn, virtualEnv, "pip", "", nil) +} + +func InstallProjectDependencies( + executeFn func(executable string, params ...string) error, + virtualEnv string, +) error { + log.Entry().Debug("installing project dependencies") + return install(executeFn, virtualEnv, ".", "", nil) } func InstallRequirements( @@ -48,7 +53,15 @@ func InstallRequirements( requirementsFile string, ) error { log.Entry().Debug("installing requirements") - return Install(executeFn, virtualEnv, "", "", []string{"--requirement", requirementsFile}) + return install(executeFn, virtualEnv, "", "", []string{"--requirement", requirementsFile}) +} + +func InstallBuild( + executeFn func(executable string, params ...string) error, + virtualEnv string, +) error { + log.Entry().Debug("installing build") + return install(executeFn, virtualEnv, "build", "", nil) } func InstallWheel( @@ -56,7 +69,7 @@ func InstallWheel( virtualEnv string, ) error { log.Entry().Debug("installing wheel") - return Install(executeFn, virtualEnv, "wheel", "", nil) + return install(executeFn, virtualEnv, "wheel", "", nil) } func InstallTwine( @@ -64,7 +77,7 @@ func InstallTwine( virtualEnv string, ) error { log.Entry().Debug("installing twine") - return Install(executeFn, virtualEnv, "twine", "", nil) + return install(executeFn, virtualEnv, "twine", "", nil) } func InstallCycloneDX( @@ -73,5 +86,5 @@ func InstallCycloneDX( cycloneDXVersion string, ) error { log.Entry().Debug("installing cyclonedx-bom") - return Install(executeFn, virtualEnv, "cyclonedx-bom", cycloneDXVersion, nil) + return install(executeFn, virtualEnv, "cyclonedx-bom", cycloneDXVersion, nil) } diff --git a/pkg/python/pip_test.go b/pkg/python/pip_test.go index 3ffd15e9d5..b3d096df0d 100644 --- a/pkg/python/pip_test.go +++ b/pkg/python/pip_test.go @@ -10,6 +10,36 @@ import ( "github.com/stretchr/testify/assert" ) +func TestInstallPip(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := InstallPip(mockRunner.RunExecutable, "") + + // assert + assert.NoError(t, err) + assert.Equal(t, "pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{"install", "--upgrade", "--root-user-action=ignore", "pip"}, mockRunner.Calls[0].Params) +} + +func TestInstallProjectDependencies(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := InstallProjectDependencies(mockRunner.RunExecutable, "") + + // assert + assert.NoError(t, err) + assert.Equal(t, "pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "."}, mockRunner.Calls[0].Params) +} + func TestInstallRequirements(t *testing.T) { // init mockRunner := mock.ExecMockRunner{} @@ -27,6 +57,40 @@ func TestInstallRequirements(t *testing.T) { "--requirement", "requirements.txt"}, mockRunner.Calls[0].Params) } +func TestInstallBuild(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := InstallBuild(mockRunner.RunExecutable, "") + + // assert + assert.NoError(t, err) + assert.Equal(t, "pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "build"}, mockRunner.Calls[0].Params) +} + +func TestInstallBuildWithVirtualEnv(t *testing.T) { + // init + mockRunner := mock.ExecMockRunner{} + + // test + err := InstallBuild(mockRunner.RunExecutable, ".venv") + + // assert + assert.NoError(t, err) + assert.Equal(t, ".venv/bin/pip", mockRunner.Calls[0].Exec) + assert.Equal(t, []string{ + "install", + "--upgrade", + "--root-user-action=ignore", + "build"}, mockRunner.Calls[0].Params) +} + func TestInstallWheel(t *testing.T) { // init mockRunner := mock.ExecMockRunner{} diff --git a/pkg/python/publish.go b/pkg/python/publish.go index 23bfd6531c..daf38cd8af 100644 --- a/pkg/python/publish.go +++ b/pkg/python/publish.go @@ -1,8 +1,6 @@ package python -import ( - "path/filepath" -) +import "fmt" func PublishPackage( executeFn func(executable string, params ...string) error, @@ -13,16 +11,11 @@ func PublishPackage( ) error { // install dependency if err := InstallTwine(executeFn, virtualEnv); err != nil { - return err - } - // handle virtual environment - twineBinary := "twine" - if len(virtualEnv) > 0 { - twineBinary = filepath.Join(virtualEnv, "bin", twineBinary) + return fmt.Errorf("failed to install twine module: %w", err) } // publish project return executeFn( - twineBinary, + getBinary(virtualEnv, "twine"), "upload", "--username", username, "--password", password, diff --git a/pkg/versioning/toml.go b/pkg/versioning/toml.go new file mode 100644 index 0000000000..916bc9d06e --- /dev/null +++ b/pkg/versioning/toml.go @@ -0,0 +1,107 @@ +package versioning + +import ( + "fmt" + "strings" + + "github.com/BurntSushi/toml" +) + +const ( + TomlBuildDescriptor = "pyproject.toml" +) + +// Pip utility to interact with Python specific versioning +type Toml struct { + Pip + coordinates tomlCoordinates +} + +type tomlCoordinates struct { + Project struct { + Name string `toml:"name"` + Version string `toml:"version"` + } `toml:"project"` +} + +func (p *Toml) init() error { + var coordinates tomlCoordinates + + if !strings.Contains(p.Pip.path, TomlBuildDescriptor) { + return fmt.Errorf("file '%v' is not a %s", p.Pip.path, TomlBuildDescriptor) + } + + if err := p.Pip.init(); err != nil { + return err + } + + if _, err := toml.Decode(p.Pip.buildDescriptorContent, &coordinates); err != nil { + return err + } + p.coordinates = coordinates + return nil +} + +// GetName returns the name from the build descriptor +func (p *Toml) GetName() (string, error) { + if err := p.init(); err != nil { + return "", fmt.Errorf("failed to read file '%v': %w", p.Pip.path, err) + } + if len(p.coordinates.Project.Name) == 0 { + return "", fmt.Errorf("no name information found in file '%v'", p.Pip.path) + } + return p.coordinates.Project.Name, nil +} + +// // GetVersion returns the current version from the build descriptor +func (p *Toml) GetVersion() (string, error) { + if err := p.init(); err != nil { + return "", fmt.Errorf("failed to read file '%v': %w", p.Pip.path, err) + } + if len(p.coordinates.Project.Version) == 0 { + return "", fmt.Errorf("no version information found in file '%v'", p.Pip.path) + } + return p.coordinates.Project.Version, nil +} + +// SetVersion updates the version in the build descriptor +func (p *Toml) SetVersion(new string) error { + if current, err := p.GetVersion(); err != nil { + return err + } else { + // replace with single quotes + p.Pip.buildDescriptorContent = strings.ReplaceAll( + p.Pip.buildDescriptorContent, + fmt.Sprintf("version = '%v'", current), + fmt.Sprintf("version = '%v'", new)) + // replace with double quotes as well + p.Pip.buildDescriptorContent = strings.ReplaceAll( + p.Pip.buildDescriptorContent, + fmt.Sprintf("version = \"%v\"", current), + fmt.Sprintf("version = \"%v\"", new)) + err = p.Pip.writeFile(p.Pip.path, []byte(p.Pip.buildDescriptorContent), 0600) + if err != nil { + return fmt.Errorf("failed to write file '%v': %w", p.Pip.path, err) + } + return nil + } +} + +// GetCoordinates returns the build descriptor coordinates +func (p *Toml) GetCoordinates() (Coordinates, error) { + result := Coordinates{} + // get name + if name, err := p.GetName(); err != nil { + return result, fmt.Errorf("failed to retrieve coordinates: %w", err) + } else { + result.ArtifactID = name + } + // get version + if version, err := p.GetVersion(); err != nil { + return result, fmt.Errorf("failed to retrieve coordinates: %w", err) + } else { + result.Version = version + } + + return result, nil +} diff --git a/pkg/versioning/toml_test.go b/pkg/versioning/toml_test.go new file mode 100644 index 0000000000..b67c2944d3 --- /dev/null +++ b/pkg/versioning/toml_test.go @@ -0,0 +1,230 @@ +//go:build unit +// +build unit + +package versioning + +import ( + "fmt" + "testing" + + piperMock "github.com/SAP/jenkins-library/pkg/mock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +const ( + invalidToml = `[project]` + sampleToml = `[project] +name = "simple-python" +version = "1.2.3" +` + missingVersionToml = `[project] +name = "simple-python" +` + largeSampleToml = `[project] +name = "sampleproject" +version = "4.0.0" +description = "A sample Python project" +license = { file = "LICENSE.txt" } + +authors = [{ name = "A. Random Developer", email = "author@example.com" }] +requires-python = ">=3.9" +readme = "README.md" + + +maintainers = [{ name = "A. Great Maintainer", email = "maintainer@example.com" }] +keywords = [ + "sample", + "setuptools", + "development", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3 :: Only", +] +dependencies = ["peppercorn"] + +[project.optional-dependencies] +dev = ["check-manifest"] +test = ["coverage"] + +[project.urls] +Homepage = "https://github.com/pypa/sampleproject" +"Bug Reports" = "https://github.com/pypa/sampleproject/issues" +Funding = "https://donate.pypi.org" +"Say Thanks!" = "http://saythanks.io/to/example" +Source = "https://github.com/pypa/sampleproject/" + +[project.scripts] +sample = "sample:main" + +[build-system] +# A list of packages that are needed to build your package: +requires = ["setuptools"] # REQUIRED if [build-system] table is used +# The name of the Python object that frontends will use to perform the build: +build-backend = "setuptools.build_meta" # If not defined, then legacy behavior can happen. + +[tool.uv] +package = false + +[tool.setuptools] +# If there are data files included in your packages that need to be +# installed, specify them here. +package-data = { "sample" = ["*.dat"] } +` +) + +func TestTomlSetVersion(t *testing.T) { + t.Parallel() + t.Run("success case - large pyproject.toml", func(t *testing.T) { + fileUtils := piperMock.FilesMock{} + fileUtils.AddFile("pyproject.toml", []byte(largeSampleToml)) + + toml := Toml{ + Pip: Pip{ + path: "pyproject.toml", + fileExists: fileUtils.FileExists, + readFile: fileUtils.FileRead, + writeFile: fileUtils.FileWrite, + }, + } + + coordinates, err := toml.GetCoordinates() + assert.NoError(t, err) + assert.Equal(t, "sampleproject", coordinates.ArtifactID) + assert.Equal(t, "4.0.0", coordinates.Version) + + // test SetVersion + err = toml.SetVersion("5.0.0") + assert.NoError(t, err) + coordinates, err = toml.GetCoordinates() + assert.NoError(t, err) + assert.Equal(t, "sampleproject", coordinates.ArtifactID) + assert.Equal(t, "5.0.0", coordinates.Version) + }) +} + +func TestTomlGetCoordinates(t *testing.T) { + t.Parallel() + t.Run("success case - pyproject.toml", func(t *testing.T) { + filename := TomlBuildDescriptor + fileUtils := piperMock.FilesMock{} + fileUtils.AddFile(filename, []byte(sampleToml)) + + toml := Toml{ + Pip: Pip{ + path: filename, + fileExists: fileUtils.FileExists, + readFile: fileUtils.FileRead, + writeFile: fileUtils.FileWrite, + }, + } + + coordinates, err := toml.GetCoordinates() + assert.NoError(t, err) + assert.Equal(t, "simple-python", coordinates.ArtifactID) + assert.Equal(t, "1.2.3", coordinates.Version) + }) + t.Run("fail - invalid pyproject.toml", func(t *testing.T) { + filename := TomlBuildDescriptor + fileUtils := piperMock.FilesMock{} + fileUtils.AddFile(filename, []byte(invalidToml)) + + toml := Toml{ + Pip: Pip{ + path: filename, + fileExists: fileUtils.FileExists, + readFile: fileUtils.FileRead, + writeFile: fileUtils.FileWrite, + }, + } + + coordinates, err := toml.GetCoordinates() + assert.ErrorContains(t, err, fmt.Sprintf("no name information found in file '%s'", filename)) + assert.Equal(t, "", coordinates.ArtifactID) + assert.Equal(t, "", coordinates.Version) + }) + t.Run("fail - invalid pyproject.toml", func(t *testing.T) { + filename := TomlBuildDescriptor + fileUtils := piperMock.FilesMock{} + fileUtils.AddFile(filename, []byte(missingVersionToml)) + + toml := Toml{ + Pip: Pip{ + path: filename, + fileExists: fileUtils.FileExists, + readFile: fileUtils.FileRead, + writeFile: fileUtils.FileWrite, + }, + } + + coordinates, err := toml.GetCoordinates() + assert.ErrorContains(t, err, fmt.Sprintf("no version information found in file '%s'", filename)) + assert.Equal(t, "simple-python", coordinates.ArtifactID) + assert.Equal(t, "", coordinates.Version) + }) + t.Run("fail - empty pyproject.toml", func(t *testing.T) { + filename := TomlBuildDescriptor + fileUtils := piperMock.FilesMock{} + fileUtils.AddFile(filename, []byte("")) + + toml := Toml{ + Pip: Pip{ + path: filename, + fileExists: fileUtils.FileExists, + readFile: fileUtils.FileRead, + writeFile: fileUtils.FileWrite, + }, + } + + coordinates, err := toml.GetCoordinates() + assert.ErrorContains(t, err, fmt.Sprintf("no name information found in file '%s'", filename)) + assert.Equal(t, "", coordinates.ArtifactID) + assert.Equal(t, "", coordinates.Version) + }) + t.Run("fail - no pyproject.toml", func(t *testing.T) { + filename := mock.Anything + fileUtils := piperMock.FilesMock{} + fileUtils.AddFile(filename, []byte("")) + + toml := Toml{ + Pip: Pip{ + path: filename, + fileExists: fileUtils.FileExists, + readFile: fileUtils.FileRead, + writeFile: fileUtils.FileWrite, + }, + } + + coordinates, err := toml.GetCoordinates() + assert.ErrorContains(t, err, fmt.Sprintf("file '%s' is not a pyproject.toml", filename)) + assert.Equal(t, "", coordinates.ArtifactID) + assert.Equal(t, "", coordinates.Version) + }) + t.Run("fail - missing pyproject.toml", func(t *testing.T) { + filename := TomlBuildDescriptor + fileUtils := piperMock.FilesMock{} + + toml := Toml{ + Pip: Pip{ + path: filename, + fileExists: fileUtils.FileExists, + readFile: fileUtils.FileRead, + writeFile: fileUtils.FileWrite, + }, + } + + coordinates, err := toml.GetCoordinates() + assert.ErrorContains(t, err, fmt.Sprintf("failed to read file '%s'", filename)) + assert.Equal(t, "", coordinates.ArtifactID) + assert.Equal(t, "", coordinates.Version) + }) +} diff --git a/pkg/versioning/versioning.go b/pkg/versioning/versioning.go index 9ae5dfd79b..a4958d573d 100644 --- a/pkg/versioning/versioning.go +++ b/pkg/versioning/versioning.go @@ -173,14 +173,24 @@ func GetArtifact(buildTool, buildDescriptorFilePath string, opts *Options, utils case "pip": if len(buildDescriptorFilePath) == 0 { var err error - buildDescriptorFilePath, err = searchDescriptor([]string{"setup.py", "version.txt", "VERSION"}, fileExists) + buildDescriptorFilePath, err = searchDescriptor([]string{TomlBuildDescriptor, "setup.py", "version.txt", "VERSION"}, fileExists) if err != nil { return artifact, err } } - artifact = &Pip{ - path: buildDescriptorFilePath, - fileExists: fileExists, + switch buildDescriptorFilePath { + case TomlBuildDescriptor: + artifact = &Toml{ + Pip: Pip{ + path: buildDescriptorFilePath, + fileExists: fileExists, + }, + } + default: + artifact = &Pip{ + path: buildDescriptorFilePath, + fileExists: fileExists, + } } case "sbt": if len(buildDescriptorFilePath) == 0 { diff --git a/pkg/versioning/versioning_test.go b/pkg/versioning/versioning_test.go index c18867efdf..ead099c533 100644 --- a/pkg/versioning/versioning_test.go +++ b/pkg/versioning/versioning_test.go @@ -217,8 +217,10 @@ func TestGetArtifact(t *testing.T) { }) t.Run("pip", func(t *testing.T) { - fileExists = func(string) (bool, error) { return true, nil } - pip, err := GetArtifact("pip", "", &Options{}, nil) + utils := newVersioningMockUtils() + utils.FilesMock.AddFile("setup.py", []byte("")) + fileExists = utils.FilesMock.FileExists + pip, err := GetArtifact("pip", "", &Options{}, utils) assert.NoError(t, err) @@ -232,7 +234,7 @@ func TestGetArtifact(t *testing.T) { fileExists = func(string) (bool, error) { return false, nil } _, err := GetArtifact("pip", "", &Options{}, nil) - assert.EqualError(t, err, "no build descriptor available, supported: [setup.py version.txt VERSION]") + assert.EqualError(t, err, "no build descriptor available, supported: [pyproject.toml setup.py version.txt VERSION]") }) t.Run("sbt", func(t *testing.T) { diff --git a/resources/metadata/pythonBuild.yaml b/resources/metadata/pythonBuild.yaml index 6ccc0b54a8..823ca1a680 100644 --- a/resources/metadata/pythonBuild.yaml +++ b/resources/metadata/pythonBuild.yaml @@ -25,7 +25,7 @@ spec: - STEPS - name: setupFlags type: "[]string" - description: Defines list of flags passed to setup.py. + description: Defines list of flags passed to setup.py / build module. scope: - PARAMETERS - STAGES @@ -116,4 +116,4 @@ spec: - name: custom/buildSettingsInfo containers: - name: python - image: python:3.10 + image: python:3.12