Merge pull request #76 from asgrim/parallel-jobs-with-make

asgrim · web-flow · commit 27a5b97e3eea · 2024-10-24T16:14:14.000+01:00
Parallel jobs with make
diff --git a/src/Building/UnixBuild.php b/src/Building/UnixBuild.php
@@ -20,7 +20,7 @@ final class UnixBuild implements Build
 {
     private const PHPIZE_TIMEOUT_SECS    = 60; // 1 minute
     private const CONFIGURE_TIMEOUT_SECS = 120; // 2 minutes
-    private const MAKE_TIMEOUT_SECS      = 600; // 10 minutes
+    private const MAKE_TIMEOUT_SECS      = null; // unlimited
 
     /** {@inheritDoc} */
     public function __invoke(
@@ -52,7 +52,7 @@ public function __invoke(
         $optionsOutput = count($configureOptions) ? ' with options: ' . implode(' ', $configureOptions) : '.';
         $output->writeln('<info>Configure complete</info>' . $optionsOutput);
 
-        $makeOutput = $this->make($downloadedPackage);
+        $makeOutput = $this->make($targetPlatform, $downloadedPackage, $output);
         if ($output->isVeryVerbose()) {
             $output->writeln($makeOutput);
         }
@@ -93,10 +93,19 @@ private function configure(DownloadedPackage $downloadedPackage, array $configur
         );
     }
 
-    private function make(DownloadedPackage $downloadedPackage): string
+    private function make(TargetPlatform $targetPlatform, DownloadedPackage $downloadedPackage, OutputInterface $output): string
     {
+        $makeCommand = ['make'];
+
+        if ($targetPlatform->makeParallelJobs === 1) {
+            $output->writeln('Running make without parallelization - try providing -jN to PIE where N is the number of cores you have.');
+        } else {
+            $makeCommand[] = '--jobs';
+            $makeCommand[] = (string) $targetPlatform->makeParallelJobs;
+        }
+
         return Process::run(
-            ['make'],
+            $makeCommand,
             $downloadedPackage->extractedSourcePath,
             self::MAKE_TIMEOUT_SECS,
         );
diff --git a/src/Command/CommandHelper.php b/src/Command/CommandHelper.php
@@ -41,6 +41,7 @@ final class CommandHelper
     private const ARG_REQUESTED_PACKAGE_AND_VERSION = 'requested-package-and-version';
     private const OPTION_WITH_PHP_CONFIG            = 'with-php-config';
     private const OPTION_WITH_PHP_PATH              = 'with-php-path';
+    private const OPTION_MAKE_PARALLEL_JOBS         = 'make-parallel-jobs';
 
     /** @psalm-suppress UnusedConstructor */
     private function __construct()
@@ -52,13 +53,13 @@ public static function configurePhpConfigOptions(Command $command): void
         $command->addOption(
             self::OPTION_WITH_PHP_CONFIG,
             null,
-            InputOption::VALUE_OPTIONAL,
+            InputOption::VALUE_REQUIRED,
             'The path to the `php-config` binary to find the target PHP platform on ' . OperatingSystem::NonWindows->asFriendlyName() . ', e.g. --' . self::OPTION_WITH_PHP_CONFIG . '=/usr/bin/php-config7.4',
         );
         $command->addOption(
             self::OPTION_WITH_PHP_PATH,
             null,
-            InputOption::VALUE_OPTIONAL,
+            InputOption::VALUE_REQUIRED,
             'The path to the `php` binary to use as the target PHP platform on ' . OperatingSystem::Windows->asFriendlyName() . ', e.g. --' . self::OPTION_WITH_PHP_PATH . '=C:\usr\php7.4.33\php.exe',
         );
     }
@@ -70,6 +71,12 @@ public static function configureDownloadBuildInstallOptions(Command $command): v
             InputArgument::REQUIRED,
             'The extension name and version constraint to use, in the format {ext-name}{?:{?version-constraint}{?@stability}}, for example `xdebug/xdebug:^3.4@alpha`, `xdebug/xdebug:@alpha`, `xdebug/xdebug:^3.4`, etc.',
         );
+        $command->addOption(
+            self::OPTION_MAKE_PARALLEL_JOBS,
+            'j',
+            InputOption::VALUE_REQUIRED,
+            'Override many jobs to run in parallel when running compiling (this is passed to "make -jN" during build). PIE will try to detect this by default.',
+        );
 
         self::configurePhpConfigOptions($command);
 
@@ -112,7 +119,15 @@ public static function determineTargetPlatformFromInputs(InputInterface $input,
             $phpBinaryPath = PhpBinaryPath::fromPhpBinaryPath($withPhpPath);
         }
 
-        $targetPlatform = TargetPlatform::fromPhpBinaryPath($phpBinaryPath);
+        $makeParallelJobs = null; /** `null` means {@see TargetPlatform} will try to auto-detect */
+        if ($input->hasOption(self::OPTION_MAKE_PARALLEL_JOBS)) {
+            $makeParallelJobsOptions = (int) $input->getOption(self::OPTION_MAKE_PARALLEL_JOBS);
+            if ($makeParallelJobsOptions > 0) {
+                $makeParallelJobs = $makeParallelJobsOptions;
+            }
+        }
+
+        $targetPlatform = TargetPlatform::fromPhpBinaryPath($phpBinaryPath, $makeParallelJobs);
 
         $output->writeln(sprintf('<info>You are running PHP %s</info>', PHP_VERSION));
         $output->writeln(sprintf(
diff --git a/src/Platform/TargetPlatform.php b/src/Platform/TargetPlatform.php
@@ -5,6 +5,7 @@
 namespace Php\Pie\Platform;
 
 use Php\Pie\Platform\TargetPhp\PhpBinaryPath;
+use Php\Pie\Util\NumberOfCores;
 
 use function array_key_exists;
 use function curl_version;
@@ -27,6 +28,7 @@ public function __construct(
         public readonly PhpBinaryPath $phpBinaryPath,
         public readonly Architecture $architecture,
         public readonly ThreadSafetyMode $threadSafety,
+        public readonly int $makeParallelJobs,
         public readonly WindowsCompiler|null $windowsCompiler,
     ) {
     }
@@ -50,7 +52,7 @@ public static function isRunningAsRoot(): bool
         return function_exists('posix_getuid') && posix_getuid() === 0;
     }
 
-    public static function fromPhpBinaryPath(PhpBinaryPath $phpBinaryPath): self
+    public static function fromPhpBinaryPath(PhpBinaryPath $phpBinaryPath, int|null $makeParallelJobs): self
     {
         $os = $phpBinaryPath->operatingSystem();
 
@@ -122,11 +124,16 @@ public static function fromPhpBinaryPath(PhpBinaryPath $phpBinaryPath): self
             }
         }
 
+        if ($makeParallelJobs === null) {
+            $makeParallelJobs = NumberOfCores::determine();
+        }
+
         return new self(
             $os,
             $phpBinaryPath,
             $architecture,
             $threadSafety,
+            $makeParallelJobs,
             $windowsCompiler,
         );
     }
diff --git a/src/Util/NumberOfCores.php b/src/Util/NumberOfCores.php
@@ -0,0 +1,72 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Php\Pie\Util;
+
+use Symfony\Component\Process\Process;
+
+use function is_numeric;
+use function trim;
+
+final class NumberOfCores
+{
+    /** @psalm-suppress UnusedConstructor */
+    private function __construct()
+    {
+    }
+
+    /**
+     * @param list<string> $params
+     *
+     * @return positive-int
+     */
+    private static function tryCommand(string $command, array $params): int|null
+    {
+        $whichProcess = new Process(['which', $command]);
+        if ($whichProcess->run() === 0) {
+            $commandPath = trim($whichProcess->getOutput());
+
+            $commandProcess = new Process([$commandPath, ...$params]);
+            if ($commandProcess->run() === 0) {
+                $commandResult = trim($commandProcess->getOutput());
+                if (is_numeric($commandResult)) {
+                    $commandNumericResult = (int) $commandResult;
+
+                    if ($commandNumericResult > 0) {
+                        return $commandNumericResult;
+                    }
+                }
+            }
+        }
+
+        return null;
+    }
+
+    /**
+     * Try a few different ways of determining number of CPUs. If we can't guess
+     * how many CPUs someone has, then default to 1 for safety.
+     *
+     * @return positive-int
+     */
+    public static function determine(): int
+    {
+        $nproc = self::tryCommand('nproc', ['--all']);
+        if ($nproc !== null) {
+            return $nproc;
+        }
+
+        $getconf = self::tryCommand('getconf', ['_NPROCESSORS_ONLN']);
+        if ($getconf !== null) {
+            return $getconf;
+        }
+
+        $sysctl = self::tryCommand('sysctl', ['-n', 'hw.ncpu']);
+        if ($sysctl !== null) {
+            return $sysctl;
+        }
+
+        // Default to 1 for safety
+        return 1;
+    }
+}
diff --git a/test/integration/Building/UnixBuildTest.php b/test/integration/Building/UnixBuildTest.php
@@ -50,7 +50,7 @@ public function testUnixBuildCanBuildExtension(): void
         $unixBuilder = new UnixBuild();
         $unixBuilder->__invoke(
             $downloadedPackage,
-            TargetPlatform::fromPhpBinaryPath(PhpBinaryPath::fromCurrentProcess()),
+            TargetPlatform::fromPhpBinaryPath(PhpBinaryPath::fromCurrentProcess(), null),
             ['--enable-pie_test_ext'],
             $output,
         );
diff --git a/test/integration/DependencyResolver/ResolveDependencyWithComposerTest.php b/test/integration/DependencyResolver/ResolveDependencyWithComposerTest.php
@@ -69,7 +69,7 @@ public function testDependenciesAreResolvedToExpectedVersions(
         $resolve   = $container->get(DependencyResolver::class);
 
         $package = $resolve->__invoke(
-            TargetPlatform::fromPhpBinaryPath(PhpBinaryPath::fromCurrentProcess()),
+            TargetPlatform::fromPhpBinaryPath(PhpBinaryPath::fromCurrentProcess(), null),
             'asgrim/example-pie-extension',
             $requestedVersion,
         );
diff --git a/test/integration/Installing/UnixInstallTest.php b/test/integration/Installing/UnixInstallTest.php
@@ -75,7 +75,7 @@ public function testUnixInstallCanInstallExtension(string $phpConfig): void
         }
 
         $output         = new BufferedOutput();
-        $targetPlatform = TargetPlatform::fromPhpBinaryPath(PhpBinaryPath::fromPhpConfigExecutable($phpConfig));
+        $targetPlatform = TargetPlatform::fromPhpBinaryPath(PhpBinaryPath::fromPhpConfigExecutable($phpConfig), null);
         $extensionPath  = $targetPlatform->phpBinaryPath->extensionPath();
 
         $downloadedPackage = DownloadedPackage::fromPackageAndExtractedPath(
diff --git a/test/integration/Installing/WindowsInstallTest.php b/test/integration/Installing/WindowsInstallTest.php
@@ -65,6 +65,7 @@ public function testWindowsInstallCanInstallExtension(): void
             PhpBinaryPath::fromCurrentProcess(),
             Architecture::x86_64,
             ThreadSafetyMode::ThreadSafe,
+            1,
             WindowsCompiler::VS16,
         );
         $phpPath           = dirname($targetPlatform->phpBinaryPath->phpBinaryPath);
diff --git a/test/unit/Command/CommandHelperTest.php b/test/unit/Command/CommandHelperTest.php
@@ -92,7 +92,7 @@ public function testInvalidRequestedNameAndVersionPairThrowsExceptionWhenNoPacka
     public function testDownloadPackage(): void
     {
         $dependencyResolver          = $this->createMock(DependencyResolver::class);
-        $targetPlatform              = TargetPlatform::fromPhpBinaryPath(PhpBinaryPath::fromCurrentProcess());
+        $targetPlatform              = TargetPlatform::fromPhpBinaryPath(PhpBinaryPath::fromCurrentProcess(), null);
         $requestedNameAndVersionPair = ['name' => 'php/test-pie-ext', 'version' => '^1.2'];
         $downloadAndExtract          = $this->createMock(DownloadAndExtract::class);
         $output                      = $this->createMock(OutputInterface::class);
diff --git a/test/unit/DependencyResolver/ResolveDependencyWithComposerTest.php b/test/unit/DependencyResolver/ResolveDependencyWithComposerTest.php
@@ -57,6 +57,7 @@ public function testPackageThatCanBeResolved(): void
             $phpBinaryPath,
             Architecture::x86_64,
             ThreadSafetyMode::ThreadSafe,
+            1,
             null,
         );
 
@@ -97,6 +98,7 @@ public function testPackageThatCannotBeResolvedThrowsException(array $platformOv
             $phpBinaryPath,
             Architecture::x86_64,
             ThreadSafetyMode::ThreadSafe,
+            1,
             null,
         );
 
@@ -139,6 +141,7 @@ public function testZtsOnlyPackageCannotBeInstalledOnNtsSystem(): void
             $phpBinaryPath,
             Architecture::x86_64,
             ThreadSafetyMode::NonThreadSafe,
+            1,
             null,
         );
 
@@ -181,6 +184,7 @@ public function testNtsOnlyPackageCannotBeInstalledOnZtsSystem(): void
             $phpBinaryPath,
             Architecture::x86_64,
             ThreadSafetyMode::ThreadSafe,
+            1,
             null,
         );
 
diff --git a/test/unit/Downloading/Exception/CouldNotFindReleaseAssetTest.php b/test/unit/Downloading/Exception/CouldNotFindReleaseAssetTest.php
@@ -67,6 +67,7 @@ public function testForMissingWindowsCompiler(): void
             $phpBinary,
             Architecture::x86,
             ThreadSafetyMode::NonThreadSafe,
+            1,
             null,
         ));
 
diff --git a/test/unit/Downloading/GithubPackageReleaseAssetsTest.php b/test/unit/Downloading/GithubPackageReleaseAssetsTest.php
@@ -41,6 +41,7 @@ public function testUrlIsReturnedWhenFindingWindowsDownloadUrl(): void
             $phpBinaryPath,
             Architecture::x86,
             ThreadSafetyMode::ThreadSafe,
+            1,
             WindowsCompiler::VC14,
         );
 
@@ -97,6 +98,7 @@ public function testUrlIsReturnedWhenFindingWindowsDownloadUrlWithCompilerAndThr
             $phpBinaryPath,
             Architecture::x86,
             ThreadSafetyMode::ThreadSafe,
+            1,
             WindowsCompiler::VC14,
         );
 
@@ -148,6 +150,7 @@ public function testFindWindowsDownloadUrlForPackageThrowsExceptionWhenAssetNotF
             PhpBinaryPath::fromCurrentProcess(),
             Architecture::x86,
             ThreadSafetyMode::ThreadSafe,
+            1,
             WindowsCompiler::VC14,
         );
 
diff --git a/test/unit/Downloading/UnixDownloadAndExtractTest.php b/test/unit/Downloading/UnixDownloadAndExtractTest.php
@@ -32,6 +32,7 @@ public function testInvoke(): void
             PhpBinaryPath::fromCurrentProcess(),
             Architecture::x86,
             ThreadSafetyMode::NonThreadSafe,
+            1,
             null,
         );
 
diff --git a/test/unit/Downloading/WindowsDownloadAndExtractTest.php b/test/unit/Downloading/WindowsDownloadAndExtractTest.php
@@ -37,6 +37,7 @@ public function testInvoke(): void
             PhpBinaryPath::fromCurrentProcess(),
             Architecture::x86,
             ThreadSafetyMode::ThreadSafe,
+            1,
             WindowsCompiler::VC14,
         );
 
diff --git a/test/unit/Installing/InstallNotification/SendInstallNotificationUsingGuzzleTest.php b/test/unit/Installing/InstallNotification/SendInstallNotificationUsingGuzzleTest.php
@@ -53,6 +53,7 @@ public function setUp(): void
             $phpBinaryPath,
             Architecture::x86,
             ThreadSafetyMode::ThreadSafe,
+            1,
             WindowsCompiler::VC14,
         );
     }
diff --git a/test/unit/Platform/TargetPlatformTest.php b/test/unit/Platform/TargetPlatformTest.php
@@ -50,7 +50,7 @@ public function testWindowsPlatformFromPhpInfo(): void
 PHP Extension Build => API20230831,TS,VS16
 TEXT);
 
-        $platform = TargetPlatform::fromPhpBinaryPath($phpBinaryPath);
+        $platform = TargetPlatform::fromPhpBinaryPath($phpBinaryPath, null);
 
         self::assertSame(OperatingSystem::Windows, $platform->operatingSystem);
         self::assertSame(WindowsCompiler::VS16, $platform->windowsCompiler);
@@ -91,7 +91,7 @@ public function testLinuxPlatform(): void
 Thread Safety => disabled
 TEXT);
 
-        $platform = TargetPlatform::fromPhpBinaryPath($phpBinaryPath);
+        $platform = TargetPlatform::fromPhpBinaryPath($phpBinaryPath, null);
 
         self::assertSame(OperatingSystem::NonWindows, $platform->operatingSystem);
         self::assertSame(null, $platform->windowsCompiler);
diff --git a/test/unit/Platform/WindowsExtensionAssetNameTest.php b/test/unit/Platform/WindowsExtensionAssetNameTest.php
@@ -33,6 +33,7 @@ public function setUp(): void
             PhpBinaryPath::fromCurrentProcess(),
             Architecture::x86_64,
             ThreadSafetyMode::ThreadSafe,
+            1,
             WindowsCompiler::VC14,
         );
 

Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,7 @@`
`5`	`5`	`namespace Php\Pie\Platform;`
`6`	`6`
`7`	`7`	`use Php\Pie\Platform\TargetPhp\PhpBinaryPath;`
	`8`	`+use Php\Pie\Util\NumberOfCores;`
`8`	`9`
`9`	`10`	`use function array_key_exists;`
`10`	`11`	`use function curl_version;`
`@@ -27,6 +28,7 @@ public function __construct(`
`27`	`28`	`public readonly PhpBinaryPath $phpBinaryPath,`
`28`	`29`	`public readonly Architecture $architecture,`
`29`	`30`	`public readonly ThreadSafetyMode $threadSafety,`
	`31`	`+ public readonly int $makeParallelJobs,`
`30`	`32`	`public readonly WindowsCompiler\|null $windowsCompiler,`
`31`	`33`	`) {`
`32`	`34`	`}`
`@@ -50,7 +52,7 @@ public static function isRunningAsRoot(): bool`
`50`	`52`	`return function_exists('posix_getuid') && posix_getuid() === 0;`
`51`	`53`	`}`
`52`	`54`
`53`		`- public static function fromPhpBinaryPath(PhpBinaryPath $phpBinaryPath): self`
	`55`	`+ public static function fromPhpBinaryPath(PhpBinaryPath $phpBinaryPath, int\|null $makeParallelJobs): self`
`54`	`56`	`{`
`55`	`57`	`$os = $phpBinaryPath->operatingSystem();`
`56`	`58`
`@@ -122,11 +124,16 @@ public static function fromPhpBinaryPath(PhpBinaryPath $phpBinaryPath): self`
`122`	`124`	`}`
`123`	`125`	`}`
`124`	`126`
	`127`	`+ if ($makeParallelJobs === null) {`
	`128`	`+ $makeParallelJobs = NumberOfCores::determine();`
	`129`	`+ }`
	`130`	`+`
`125`	`131`	`return new self(`
`126`	`132`	`$os,`
`127`	`133`	`$phpBinaryPath,`
`128`	`134`	`$architecture,`
`129`	`135`	`$threadSafety,`
	`136`	`+ $makeParallelJobs,`
`130`	`137`	`$windowsCompiler,`
`131`	`138`	`);`
`132`	`139`	`}`
Original file line number	Diff line number	Diff line change
`@@ -75,7 +75,7 @@ public function testUnixInstallCanInstallExtension(string $phpConfig): void`
`75`	`75`	`}`
`76`	`76`
`77`	`77`	`$output = new BufferedOutput();`
`78`		`- $targetPlatform = TargetPlatform::fromPhpBinaryPath(PhpBinaryPath::fromPhpConfigExecutable($phpConfig));`
	`78`	`+ $targetPlatform = TargetPlatform::fromPhpBinaryPath(PhpBinaryPath::fromPhpConfigExecutable($phpConfig), null);`
`79`	`79`	`$extensionPath = $targetPlatform->phpBinaryPath->extensionPath();`
`80`	`80`
`81`	`81`	`$downloadedPackage = DownloadedPackage::fromPackageAndExtractedPath(`
Original file line number	Diff line number	Diff line change
`@@ -92,7 +92,7 @@ public function testInvalidRequestedNameAndVersionPairThrowsExceptionWhenNoPacka`
`92`	`92`	`public function testDownloadPackage(): void`
`93`	`93`	`{`
`94`	`94`	`$dependencyResolver = $this->createMock(DependencyResolver::class);`
`95`		`- $targetPlatform = TargetPlatform::fromPhpBinaryPath(PhpBinaryPath::fromCurrentProcess());`
	`95`	`+ $targetPlatform = TargetPlatform::fromPhpBinaryPath(PhpBinaryPath::fromCurrentProcess(), null);`
`96`	`96`	`$requestedNameAndVersionPair = ['name' => 'php/test-pie-ext', 'version' => '^1.2'];`
`97`	`97`	`$downloadAndExtract = $this->createMock(DownloadAndExtract::class);`
`98`	`98`	`$output = $this->createMock(OutputInterface::class);`
Original file line number	Diff line number	Diff line change
`@@ -53,6 +53,7 @@ public function setUp(): void`
`53`	`53`	`$phpBinaryPath,`
`54`	`54`	`Architecture::x86,`
`55`	`55`	`ThreadSafetyMode::ThreadSafe,`
	`56`	`+ 1,`
`56`	`57`	`WindowsCompiler::VC14,`
`57`	`58`	`);`
`58`	`59`	`}`