From ccbd38a03743195505e8fc7f800fdccd6c18d227 Mon Sep 17 00:00:00 2001 From: Kyle Butt Date: Wed, 14 Feb 2024 12:12:58 -0700 Subject: [PATCH 1/3] Add argument for build specific environment It is likely that a user may want to specify environment variables during the build phase as well as during pnpm install. Add an argument for a build specific environment. This allows the user to avoid duplicating the environment between a configure or prebuild step and installEnv, by using something like: let sharedEnv = { ... }; buildEnvOverrides = { ... }; installEnvOverrides = { ... }; in mkPnpmPackage { buildEnv = sharedEnv // buildEnvOverrides; installEnv = sharedEnv // installEnvOverrides; ... } --- derivation.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/derivation.nix b/derivation.nix index affaea8..a032422 100644 --- a/derivation.nix +++ b/derivation.nix @@ -26,6 +26,7 @@ in , distDir ? "dist" , installInPlace ? false , installEnv ? { } + , buildEnv ? { } , noDevDependencies ? false , extraNodeModuleSources ? [ ] , copyPnpmStore ? true @@ -68,6 +69,12 @@ in ''; buildPhase = '' + ${concatStringsSep "\n" ( + mapAttrsToList + (n: v: ''export ${n}="${v}"'') + buildEnv + )} + runHook preBuild pnpm run ${script} @@ -159,6 +166,6 @@ in }; }) - (attrs // { extraNodeModuleSources = null; installEnv = null; }) + (attrs // { extraNodeModuleSources = null; installEnv = null; buildEnv = null;}) ); } From a17fd10c3e3cfc3a827b78a9e5fa1c122770fa58 Mon Sep 17 00:00:00 2001 From: Kyle Butt Date: Mon, 12 Feb 2024 17:14:34 -0700 Subject: [PATCH 2/3] Add support for pnpm workspaces as single derivation Allow a caller to specify a workspace and a list of components rather than a single source directory. This allows for the building of pnpm monorepo projects with dependency links between the various projects and a single pnpm-lock.yaml for the whole workspace. Requires that the script has the same name in all components. The location of all of the per-component package.json files is overridable. A workspace project can still specify the location of the root package.json via packageJSON, as well as the components' package.json files via componentPackageJSONs. The list defaults to c/package.json for c in components. Similarly, the list of distDirs for the components is overridable. For a workspace, the distDirs are all subdirs of $out rather than dist becoming $out. The default list is c/dist for c in components. Allowing for override handles the case where one component is built by a different tool like next and produces a different directory like ".next" NB: Any "link:" dependencies will need to be recreated as a preBuild step. pnpm will create the symlinks during install, and nix removes them because when the node_modules derivation is built, they are dangling. Attempts to leave the non-workspace pnpm package support intact. --- derivation.nix | 107 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 89 insertions(+), 18 deletions(-) diff --git a/derivation.nix b/derivation.nix index a032422..556cca0 100644 --- a/derivation.nix +++ b/derivation.nix @@ -1,6 +1,7 @@ { lib , stdenv , nodejs +, rsync , pkg-config , callPackage , writeText @@ -15,15 +16,23 @@ let in { mkPnpmPackage = - { src + { workspace ? null + , components ? [] + , src ? if (workspace != null && components != []) then workspace else null , packageJSON ? src + "/package.json" + , componentPackageJSONs ? map (c: { + name = "${c}/package.json"; + value = src + "/${c}/package.json"; + }) components , pnpmLockYaml ? src + "/pnpm-lock.yaml" + , pnpmWorkspaceYaml ? (if workspace == null then null else workspace + "/pnpm-workspace.yaml") , pname ? (fromJSON (readFile packageJSON)).name , version ? (fromJSON (readFile packageJSON)).version or null , name ? if version != null then "${pname}-${version}" else pname , registry ? "https://registry.npmjs.org" , script ? "build" , distDir ? "dist" + , distDirs ? (if workspace == null then [distDir] else (map (c: "${c}/dist") components)) , installInPlace ? false , installEnv ? { } , buildEnv ? { } @@ -38,17 +47,65 @@ in , ... }@attrs: let + # Flag that can be computed from arguments, indicating a workspace was + # supplied. Only used in these let bindings. + isWorkspace = workspace != null && components != []; + # Utility functions + forEachConcat = f: xs: concatStringsSep "\n" (map f xs); + forEachComponent = f: forEachConcat f components; + # Computed values used below that don't loop nativeBuildInputs = [ nodejs pnpm pkg-config - ] ++ extraBuildInputs; + ] ++ extraBuildInputs ++ (optional copyNodeModules rsync); + copyLink = + if copyNodeModules + then "rsync -a --chmod=u+w" + else "ln -s"; + rsyncSlash = optionalString copyNodeModules "/"; + packageFilesWithoutLockfile = + [ + { name = "package.json"; value = packageJSON; } + ] ++ componentPackageJSONs ++ computedNodeModuleSources; + computedNodeModuleSources = + (if pnpmWorkspaceYaml == null + then [] + else [ + {name = "pnpm-workspace.yaml"; value = pnpmWorkspaceYaml;} + ] + ) ++ extraNodeModuleSources; + # Computed values that loop over something + nodeModulesDirs = + if isWorkspace then + ["node_modules"] ++ (map (c: "${c}/node_modules") components) + else ["node_modules"]; + filterString = concatStringsSep " " ( + ["--recursive" "--stream"] ++ + map (c: "--filter ./${c}") components + ) + " "; + buildScripts = '' + pnpm run ${optionalString isWorkspace filterString}${script} + ''; + # Flag derived from value computed above, indicating the single dist + # should be copied as $out directly, rather than $out/${distDir} + computedDistDirIsOut = + length distDirs == 1 && !isWorkspace; in stdenv.mkDerivation ( recursiveUpdate (rec { inherit src name nativeBuildInputs; + postUnpack = '' + ${optionalString (pnpmWorkspaceYaml != null) '' + cp ${pnpmWorkspaceYaml} pnpm-workspace.yaml + ''} + ${forEachComponent (component: + ''mkdir -p "${component}"'') + } + ''; + configurePhase = '' export HOME=$NIX_BUILD_TOP # Some packages need a writable HOME export npm_config_nodedir=${nodejs} @@ -57,12 +114,11 @@ in ${if installInPlace then passthru.nodeModules.buildPhase - else '' - ${if !copyNodeModules - then "ln -s" - else "cp -r" - } ${passthru.nodeModules}/node_modules node_modules - '' + else + forEachConcat ( + nodeModulesDir: '' + ${copyLink} ${passthru.nodeModules}/${nodeModulesDir}${rsyncSlash} ${nodeModulesDir} + '') nodeModulesDirs } runHook postConfigure @@ -77,7 +133,7 @@ in runHook preBuild - pnpm run ${script} + ${buildScripts} runHook postBuild ''; @@ -85,7 +141,17 @@ in installPhase = '' runHook preInstall - ${if distDir == "." then "cp -r" else "mv"} ${distDir} $out + ${if computedDistDirIsOut then '' + ${if distDir == "." then "cp -r" else "mv"} ${distDir} $out + '' + else '' + mkdir -p $out + ${forEachConcat (dDir: '' + cp -r --parents ${dDir} $out + '') distDirs + } + '' + } runHook postInstall ''; @@ -119,18 +185,19 @@ in inherit nativeBuildInputs; unpackPhase = concatStringsSep "\n" - ( + ( [ # components is an empty list for non workspace builds + (forEachComponent (component: '' + mkdir -p "${component}" + '')) ] ++ map (v: let nv = if isAttrs v then v else { name = "."; value = v; }; in - "cp -vr ${nv.value} ${nv.name}" + "cp -vr \"${nv.value}\" \"${nv.name}\"" ) - ([ - { name = "package.json"; value = packageJSON; } - { name = "pnpm-lock.yaml"; value = passthru.patchedLockfileYaml; } - ] ++ extraNodeModuleSources) + ([{ name = "pnpm-lock.yaml"; value = passthru.patchedLockfileYaml; }] + ++ packageFilesWithoutLockfile) ); buildPhase = '' @@ -147,7 +214,7 @@ in else "cp -RL" } ${passthru.pnpmStore} $(pnpm store path) - ${lib.optionalString copyPnpmStore "chmod -R +w $(pnpm store path)"} + ${optionalString copyPnpmStore "chmod -R +w $(pnpm store path)"} ${concatStringsSep "\n" ( mapAttrsToList @@ -155,12 +222,16 @@ in installEnv )} - pnpm install ${optionalString noDevDependencies "--prod "}--frozen-lockfile --offline + pnpm install --stream ${optionalString noDevDependencies "--prod "}--frozen-lockfile --offline ''; installPhase = '' mkdir -p $out cp -r node_modules/. $out/node_modules + ${forEachComponent (component: '' + mkdir -p $out/"${component}" + cp -r "${component}/node_modules" $out/"${component}/node_modules" + '')} ''; }; }; From 71a99689dac92d2460e57e89c0dd97ae59f0d222 Mon Sep 17 00:00:00 2001 From: Kyle Butt Date: Wed, 14 Feb 2024 12:16:19 -0700 Subject: [PATCH 3/3] Add support for placing additional artifacts into the result. A user may want the node_modules and package artifacts as a part of the output in order to have a runnable build artifact without needing the node-modules derivation or the sources available. Add options to control additional copies as a part of the install. Simplify the values used in the derivation so that the conditional logic is almost all in the let rather than in the derivation body. Some of this change should probably rebased into the first commit. --- derivation.nix | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/derivation.nix b/derivation.nix index 556cca0..9d92e89 100644 --- a/derivation.nix +++ b/derivation.nix @@ -33,6 +33,9 @@ in , script ? "build" , distDir ? "dist" , distDirs ? (if workspace == null then [distDir] else (map (c: "${c}/dist") components)) + , distDirIsOut ? true + , installNodeModules ? false + , installPackageFiles ? false , installInPlace ? false , installEnv ? { } , buildEnv ? { } @@ -76,6 +79,14 @@ in ] ) ++ extraNodeModuleSources; # Computed values that loop over something + computedDistFiles = + let + packageFileNames = ["pnpm-lock.yaml"] ++ + map ({ name, ... }: name) packageFilesWithoutLockfile; + in + distDirs ++ + optionals installNodeModules nodeModulesDirs ++ + optionals installPackageFiles packageFileNames; nodeModulesDirs = if isWorkspace then ["node_modules"] ++ (map (c: "${c}/node_modules") components) @@ -90,7 +101,7 @@ in # Flag derived from value computed above, indicating the single dist # should be copied as $out directly, rather than $out/${distDir} computedDistDirIsOut = - length distDirs == 1 && !isWorkspace; + length computedDistFiles == 1 && distDirIsOut && !isWorkspace; in stdenv.mkDerivation ( recursiveUpdate @@ -148,7 +159,7 @@ in mkdir -p $out ${forEachConcat (dDir: '' cp -r --parents ${dDir} $out - '') distDirs + '') computedDistFiles } '' }