From 64062c91d869563b9c1c19ab273ce3f4f6109477 Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Sun, 16 Aug 2020 10:42:09 +0200 Subject: [PATCH 1/2] makeBinaryWrapper: create binary wrappers When packaging programs we often need to change the environment the program is run in. Typically this is done using wrappers. Thus far, the wrappers produced were typically shell scripts, thus interpreted, and with a shebang. This is an issue on e.g. Darwin, where shebangs cannot point at interpreted scripts. This commit introduces a new program for creating binary wrappers. The program, written currently in Python, produces an instruction in JSON which is loaded into a Nim program and then compiled into a binary. With interpreted wrappers one can easily check the code to see what it is doing, but with a binary that is more difficult. To that end, an environment variable `NIX_DEBUG_WRAPPER` is introduced which, when set, causes an executed wrapper to print the embedded JSON instead of exec'ing into the wrapped executable. The Python program aims to be compatible with the existing `makeWrapper` but is lacking features, mostly because of difficulties with implementing them using `argparse`. --- .../make-binary-wrapper/default.nix | 55 ++++++++ .../make-binary-wrapper/src/bin/make-wrapper | 7 + .../make-binary-wrapper/src/bin/makeWrapper | 1 + .../src/lib/libwrapper/compile_wrapper.py | 28 ++++ .../src/lib/libwrapper/make_wrapper.py | 120 ++++++++++++++++++ .../src/lib/libwrapper/wrapper.nim | 102 +++++++++++++++ .../make-binary-wrapper/tests.nix | 48 +++++++ pkgs/top-level/all-packages.nix | 3 + 8 files changed, 364 insertions(+) create mode 100644 pkgs/build-support/make-binary-wrapper/default.nix create mode 100755 pkgs/build-support/make-binary-wrapper/src/bin/make-wrapper create mode 120000 pkgs/build-support/make-binary-wrapper/src/bin/makeWrapper create mode 100644 pkgs/build-support/make-binary-wrapper/src/lib/libwrapper/compile_wrapper.py create mode 100644 pkgs/build-support/make-binary-wrapper/src/lib/libwrapper/make_wrapper.py create mode 100644 pkgs/build-support/make-binary-wrapper/src/lib/libwrapper/wrapper.nim create mode 100644 pkgs/build-support/make-binary-wrapper/tests.nix diff --git a/pkgs/build-support/make-binary-wrapper/default.nix b/pkgs/build-support/make-binary-wrapper/default.nix new file mode 100644 index 0000000000000..657a850b75bcf --- /dev/null +++ b/pkgs/build-support/make-binary-wrapper/default.nix @@ -0,0 +1,55 @@ +{ stdenv +, targetPackages +, python3Minimal +, callPackage +}: + +# Python is used for creating the instructions for the Nim program which +# is compiled into a binary. +# +# The python3Minimal package is used because its available early on during bootstrapping. +# Python packaging tools are avoided because this needs to be available early on in bootstrapping. + +let + python = python3Minimal; + sitePackages = "${placeholder "out"}/${python.sitePackages}"; + nim = targetPackages.nim.override { + minimal = true; + }; +in stdenv.mkDerivation { + name = "make-binary-wrapper"; + + src = ./src; + + buildInputs = [ + python + ]; + + strictDeps = true; + + postPatch = '' + substituteInPlace lib/libwrapper/compile_wrapper.py \ + --replace 'NIM_EXECUTABLE = "nim"' 'NIM_EXECUTABLE = "${nim}/bin/nim"' \ + --replace 'STRIP_EXECUTABLE = "strip"' 'STRIP_EXECUTABLE = "${targetPackages.binutils-unwrapped}/bin/strip"' + substituteAllInPlace bin/make-wrapper + ''; + + inherit sitePackages; + + dontBuild = true; + + installPhase = '' + mkdir -p $out/${python.sitePackages} + mv bin $out/ + mv lib/libwrapper $out/${python.sitePackages} + ''; + + passthru.tests.test-wrapped-hello = callPackage ./tests.nix { + inherit python; + }; + + meta = { + description = "Tool to create binary wrappers"; + maintainers = with stdenv.lib.maintainers; [ fridh ]; + }; +} \ No newline at end of file diff --git a/pkgs/build-support/make-binary-wrapper/src/bin/make-wrapper b/pkgs/build-support/make-binary-wrapper/src/bin/make-wrapper new file mode 100755 index 0000000000000..91808ea449297 --- /dev/null +++ b/pkgs/build-support/make-binary-wrapper/src/bin/make-wrapper @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +import sys +sitepackages = "@sitePackages@" +sys.path.insert(0, sitepackages) +import libwrapper.make_wrapper +libwrapper.make_wrapper.main() diff --git a/pkgs/build-support/make-binary-wrapper/src/bin/makeWrapper b/pkgs/build-support/make-binary-wrapper/src/bin/makeWrapper new file mode 120000 index 0000000000000..0cb2c93e7a560 --- /dev/null +++ b/pkgs/build-support/make-binary-wrapper/src/bin/makeWrapper @@ -0,0 +1 @@ +make-wrapper \ No newline at end of file diff --git a/pkgs/build-support/make-binary-wrapper/src/lib/libwrapper/compile_wrapper.py b/pkgs/build-support/make-binary-wrapper/src/lib/libwrapper/compile_wrapper.py new file mode 100644 index 0000000000000..ce2040761184e --- /dev/null +++ b/pkgs/build-support/make-binary-wrapper/src/lib/libwrapper/compile_wrapper.py @@ -0,0 +1,28 @@ +import json +import pathlib +import shutil +import subprocess +import tempfile +from typing import Dict + + +WRAPPER_SOURCE = pathlib.Path(__file__).parent / "wrapper.nim" + +NIM_EXECUTABLE = "nim" +STRIP_EXECUTABLE = "strip" + + +def compile_wrapper(instruction: Dict): + """Compile a wrapper using the given instruction.""" + + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + shutil.copyfile(WRAPPER_SOURCE, tmpdir / "wrapper.nim" ) + with open(tmpdir / "wrapper.json", "w") as fout: + json.dump(instruction, fout) + subprocess.run( + f"cd {tmpdir} && {NIM_EXECUTABLE} --nimcache=. --gc:none -d:release --opt:size compile {tmpdir}/wrapper.nim && {STRIP_EXECUTABLE} -s {tmpdir}/wrapper", + shell=True, + ) + shutil.move(tmpdir / "wrapper", instruction["wrapper"]) + \ No newline at end of file diff --git a/pkgs/build-support/make-binary-wrapper/src/lib/libwrapper/make_wrapper.py b/pkgs/build-support/make-binary-wrapper/src/lib/libwrapper/make_wrapper.py new file mode 100644 index 0000000000000..ce70406a2dfb1 --- /dev/null +++ b/pkgs/build-support/make-binary-wrapper/src/lib/libwrapper/make_wrapper.py @@ -0,0 +1,120 @@ +import argparse +import textwrap +from typing import Dict + +import libwrapper.compile_wrapper + + +EPILOG = textwrap.dedent('''\ +This program creates a binary wrapper. The arguments given are +serialized to JSON. A binary wrapper is created and the JSON is +embedded into it. + +For debugging purposes it is possible to view the embedded JSON: + + NIX_DEBUG_PYTHON=1 my-wrapped-executable + +''') + + +def parse_args() -> Dict: + + parser = argparse.ArgumentParser( + description="Create a binary wrapper.", + epilog=EPILOG, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument("original", type=str, + help="Path of executable to wrap", + ) + parser.add_argument("wrapper", type=str, + help="Path of wrapper to create", + ) + # parser.add_argument( + # "--argv", nargs=1, type=str, metavar="NAME", default, + # help="Set name of executed process. Not used." + # ) + parser.add_argument( + "--set", nargs=2, type=str, metavar=("VAR", "VAL"), action="append", default=[], + help="Set environment variable to value", + ) + parser.add_argument( + "--set-default", nargs=2, type=str, metavar=("VAR", "VAL"), action="append", default=[], + help="Set environment variable to value, if not yet set in environment", + ) + parser.add_argument( + "--unset", nargs=1, type=str, metavar="VAR", action="append", default=[], + help="Unset variable from the environment" + ) + parser.add_argument( + "--run", nargs=1, type=str, metavar="COMMAND", action="append", + help="Run command before the executable" + ) + parser.add_argument( + "--add-flags", dest="flags", nargs=1, type=str, metavar="FLAGS", action="append", default=[], + help="Add flags to invocation of process" + ) + parser.add_argument( + "--prefix", nargs=3, type=str, metavar=("ENV", "SEP", "VAL"), action="append", default=[], + help="Prefix environment variable ENV with value VAL, separated by separator SEP" + ) + parser.add_argument( + "--suffix", nargs=3, type=str, metavar=("ENV", "SEP", "VAL"), action="append", default=[], + help="Suffix environment variable ENV with value VAL, separated by separator SEP" + ) + # TODO: Fix help message because we cannot use metavar with nargs="+". + # Note these hardly used in Nixpkgs and may as well be dropped. + # parser.add_argument( + # "--prefix-each", nargs="+", type=str, action="append", + # help="Prefix environment variable ENV with values VALS, separated by separator SEP." + # ) + # parser.add_argument( + # "--suffix-each", nargs="+", type=str, action="append", + # help="Suffix environment variable ENV with values VALS, separated by separator SEP." + # ) + # parser.add_argument( + # "--prefix-contents", nargs="+", type=str, action="append", + # help="Prefix environment variable ENV with values read from FILES, separated by separator SEP." + # ) + # parser.add_argument( + # "--suffix-contents", nargs="+", type=str, action="append", + # help="Suffix environment variable ENV with values read from FILES, separated by separator SEP." + # ) + return vars(parser.parse_args()) + + +def convert_args(args: Dict) -> Dict: + """Convert arguments to the JSON structure expected by the Nim wrapper.""" + output = {} + + # Would not need this if the Environment members were part of Wrapper. + output["original"] = args["original"] + output["wrapper"] = args["wrapper"] + output["run"] = args["run"] + output["flags"] = [item[0] for item in args["flags"]] + + output["environment"] = {} + for key, value in args.items(): + if key == "set": + output["environment"][key] = [dict(zip(["variable", "value"], item)) for item in value] + if key == "set_default": + output["environment"][key] = [dict(zip(["variable", "value"], item)) for item in value] + if key == "unset": + output["environment"][key] = [dict(zip(["variable"], item)) for item in value] + if key == "prefix": + output["environment"][key] = [dict(zip(["variable", "value", "separator"], item)) for item in value] + if key == "suffix": + output["environment"][key] = [dict(zip(["variable", "value", "separator"], item)) for item in value] + + return output + + +def main(): + args = parse_args() + args = convert_args(args) + libwrapper.compile_wrapper.compile_wrapper(args) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/pkgs/build-support/make-binary-wrapper/src/lib/libwrapper/wrapper.nim b/pkgs/build-support/make-binary-wrapper/src/lib/libwrapper/wrapper.nim new file mode 100644 index 0000000000000..9e00f709551c4 --- /dev/null +++ b/pkgs/build-support/make-binary-wrapper/src/lib/libwrapper/wrapper.nim @@ -0,0 +1,102 @@ +# This is the code of the wrapper that is generated. + +import json +import os +import posix +import sequtils +import strutils + +# Wrapper type as used by the wrapper-generation code as well in the actual wrapper. + +type + SetVar* = object + variable*: string + value*: string + + SetDefaultVar* = object + variable*: string + value*: string + + UnsetVar* = object + variable*: string + + PrefixVar* = object + variable*: string + values*: seq[string] + separator*: string + + SuffixVar* = object + variable*: string + values*: seq[string] + separator*: string + + # Maybe move the members into Wrapper directly? + Environment* = object + set*: seq[SetVar] + set_default*: seq[SetDefaultVar] + unset*: seq[UnsetVar] + prefix*: seq[PrefixVar] + suffix*: seq[SuffixVar] + + Wrapper* = object + original*: string + wrapper*: string + run*: string + flags*: seq[string] + environment*: Environment + +# File containing wrapper instructions +const jsonFilename = "./wrapper.json" + +# Embed the JSON string defining the wrapper in our binary +const jsonString = staticRead(jsonFilename) + +proc modifyEnv(item: SetVar) = + putEnv(item.variable, item.value) + +proc modifyEnv(item: SetDefaultVar) = + if not existsEnv(item.variable): + putEnv(item.variable, item.value) + +proc modifyEnv(item: UnsetVar) = + if existsEnv(item.variable): + delEnv(item.variable) + +proc modifyEnv(item: PrefixVar) = + let old_value = if existsEnv(item.variable): getEnv(item.variable) else: "" + let new_value = join(concat(item.values, @[old_value]), item.separator) + putEnv(item.variable, new_value) + +proc modifyEnv(item: SuffixVar) = + let old_value = if existsEnv(item.variable): getEnv(item.variable) else: "" + let new_value = join(concat(@[old_value], item.values), item.separator) + putEnv(item.variable, new_value) + +proc processEnvironment(environment: Environment) = + for item in environment.unset.items(): + item.modifyEnv() + for item in environment.set.items(): + item.modifyEnv() + for item in environment.set_default.items(): + item.modifyEnv() + for item in environment.prefix.items(): + item.modifyEnv() + for item in environment.suffix.items(): + item.modifyEnv() + + +if existsEnv("NIX_DEBUG_WRAPPER"): + echo(jsonString) +else: + # Unfortunately parsing JSON during compile-time is not supported. + let wrapperDescription = parseJson(jsonString) + let wrapper = to(wrapperDescription, Wrapper) + processEnvironment(wrapper.environment) + let argv = wrapper.original # convert target to cstring + let argc = allocCStringArray(wrapper.flags) + + # Run command in new environment but before executing our executable + discard execShellCmd(wrapper.run) + discard execvp(argv, argc) # Maybe use execvpe instead so we can pass an updated mapping? + + deallocCStringArray(argc) diff --git a/pkgs/build-support/make-binary-wrapper/tests.nix b/pkgs/build-support/make-binary-wrapper/tests.nix new file mode 100644 index 0000000000000..957d94ee711d3 --- /dev/null +++ b/pkgs/build-support/make-binary-wrapper/tests.nix @@ -0,0 +1,48 @@ +{ runCommand +, makeBinaryWrapper +, python +, stdenv +}: + +runCommand "test-wrapped-hello" { + nativeBuildInputs = [ + makeBinaryWrapper + ]; +} ('' + mkdir -p $out/bin + + # Test building of the wrapper. + + make-wrapper ${python.interpreter} $out/bin/python \ + --set FOO bar \ + --set-default BAR foo \ + --prefix MYPATH ":" zero \ + --suffix MYPATH ":" four \ + --unset UNSET_THIS + +'' + stdenv.lib.optionalString (stdenv.hostPlatform == stdenv.buildPlatform) '' + # When not cross-compiling we can execute the wrapper and test how it behaves. + + # See the following tests for why variables are set the way they are. + + # Test `set`: We set FOO to bar + $out/bin/python -c "import os; assert os.environ["FOO] == "bar" + + # Test `set-default`: We set BAR to bar, and then set-default BAR, thus expecting the original bar. + export BAR=bar + $out/bin/python -c "import os; assert os.environ["BAR] == "bar" + + # Test `unset`: # We set MYPATH and unset it in the wrapper. + export UNSET_THIS=1 + $out/bin/python -c "import os; assert "UNSET_THIS" not in os.environ.["BAR]" + + # Test `prefix`: + export MYPATH=one:two:three + $out/bin/python -c "import os; assert os.environ["MYPATH].split(":")[0] == "zero" + + # Test `suffix`: + $out/bin/python -c "import os; assert os.environ["MYPATH].split(":")[0] == "four" + + # Test `NIX_DEBUG_WRAPPER`: + NIX_DEBUG_WRAPPER=1 $out/bin/python +'') diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 56342cad1210e..271532e2fade3 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -439,6 +439,9 @@ in makeWrapper = makeSetupHook { deps = [ dieHook ]; substitutions = { shell = pkgs.runtimeShell; }; } ../build-support/setup-hooks/make-wrapper.sh; + + makeBinaryWrapper = callPackage ../build-support/make-binary-wrapper { }; + makeModulesClosure = { kernel, firmware, rootModules, allowMissing ? false }: callPackage ../build-support/kernel/modules-closure.nix { inherit kernel firmware rootModules allowMissing; From da5f53542645fdc240fa170af771fecd5367c7c0 Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Sun, 16 Aug 2020 17:41:01 +0200 Subject: [PATCH 2/2] nim: add `minimal` option that is stripped of its dependencies For `makeBinaryWrapper` we preferably keep the closure small so it can be used early on during bootstrapping. --- pkgs/development/compilers/nim/default.nix | 27 ++++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/pkgs/development/compilers/nim/default.nix b/pkgs/development/compilers/nim/default.nix index 543a6120577f3..dac4135e394ff 100644 --- a/pkgs/development/compilers/nim/default.nix +++ b/pkgs/development/compilers/nim/default.nix @@ -1,10 +1,12 @@ # based on https://github.com/nim-lang/Nim/blob/v0.18.0/.travis.yml -{ stdenv, lib, fetchurl, makeWrapper, openssl, pcre, readline, - boehmgc, sfml, sqlite }: +{ stdenv, lib, fetchurl, makeWrapper, openssl, pcre, readline +, boehmgc, sfml, sqlite +, minimal ? false +}: stdenv.mkDerivation rec { - pname = "nim"; + pname = "nim" + lib.optionalString (!minimal) "-minimal"; version = "1.2.6"; src = fetchurl { @@ -14,7 +16,13 @@ stdenv.mkDerivation rec { enableParallelBuilding = true; - NIX_LDFLAGS = "-lcrypto -lpcre -lreadline -lgc -lsqlite3"; + NIX_LDFLAGS = toString(lib.optionals (!minimal) [ + "-lcrypto" + "-lgc" + "-lpcre" + "-lreadline" + "-lsqlite3" + ]); # we could create a separate derivation for the "written in c" version of nim # used for bootstrapping, but koch insists on moving the nim compiler around @@ -24,8 +32,13 @@ stdenv.mkDerivation rec { makeWrapper ]; - buildInputs = [ - openssl pcre readline boehmgc sfml sqlite + buildInputs = lib.optionals (!minimal) [ + pcre + readline + boehmgc + openssl + sfml + sqlite ]; buildPhase = '' @@ -63,7 +76,7 @@ stdenv.mkDerivation rec { description = "Statically typed, imperative programming language"; homepage = "https://nim-lang.org/"; license = licenses.mit; - maintainers = with maintainers; [ ehmry ]; + maintainers = with maintainers; [ ehmry ] ++ lib.optionals (!minimal) [ fridh ]; platforms = with platforms; linux ++ darwin; # arbitrary }; }