From d01de3c6ba41db33dc98ab70d02e87d1d75a7f73 Mon Sep 17 00:00:00 2001 From: Stephen Sherratt Date: Wed, 20 Aug 2025 09:16:59 +1000 Subject: [PATCH 1/4] Move completion url to flake.nix Maintainers are expected to only modify flake.nix. While the url url of the completion script isn't expected to change (unless Tarides adopts the project), the goal of this change is to make it clear to maintainers where the completion script comes from. Signed-off-by: Stephen Sherratt --- README.md | 2 +- dune.nix | 5 +---- flake.nix | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4c3139a..ad455e0 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ better compatibility with non-glibc distros), x86_64-macos and aarch64-macos. ## Installation -To install the binaries released here, use the +To install the binaries released here, use the [dune-bin-install](https://github.com/ocaml-dune/dune-bin-install) script, or see that script for manual installation. diff --git a/dune.nix b/dune.nix index 3e25a6c..f5fc8e1 100644 --- a/dune.nix +++ b/dune.nix @@ -5,10 +5,7 @@ let ref = ref; rev = rev; }; - completion-src = fetchGit { - url = "https://github.com/gridbugs/dune-completion-scripts"; - inherit (completion) ref rev; - }; + completion-src = fetchGit completion; # This creates a git repo and creates an annotated tag named after the # current version of dune. This is necessary for the resulting dune # executable to print the correct version in the output of `dune --version`. diff --git a/flake.nix b/flake.nix index d6642d2..19d02d7 100644 --- a/flake.nix +++ b/flake.nix @@ -27,6 +27,7 @@ ref = "main"; rev = "fd307dfd897c0d764bbf5b151e01f99f6be05111"; completion = { + url = "https://github.com/gridbugs/dune-completion-scripts"; ref = "main"; rev = "42c6489d175cb81be58e052ca951186d6b2291d1"; }; From e73f67eff4ecb2dd2c76346c688965dbf1b4a21b Mon Sep 17 00:00:00 2001 From: Stephen Sherratt Date: Wed, 20 Aug 2025 09:27:51 +1000 Subject: [PATCH 2/4] Completion script is optional Passing `completion = null;` to dune.nix causes the dune package to be built without a completion script. Signed-off-by: Stephen Sherratt --- dune.nix | 15 +++++++++++---- extra/share/dune/env/env.bash | 2 +- extra/share/dune/env/env.zsh | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/dune.nix b/dune.nix index f5fc8e1..c6cd48e 100644 --- a/dune.nix +++ b/dune.nix @@ -5,7 +5,7 @@ let ref = ref; rev = rev; }; - completion-src = fetchGit completion; + completion-src = if isNull completion then null else fetchGit completion; # This creates a git repo and creates an annotated tag named after the # current version of dune. This is necessary for the resulting dune # executable to print the correct version in the output of `dune --version`. @@ -45,9 +45,16 @@ let dontAddPrefix = true; dontAddStaticConfigureFlags = !static; configurePlatforms = [ ]; - preInstall = '' - mkdir -p $out/share/bash-completion/completions - cp ${completion-src}/bash.sh $out/share/bash-completion/completions/dune + preInstall = let + for-completion = if isNull completion-src then + "" + else '' + mkdir -p $out/share/bash-completion/completions + cp ${completion-src}/bash.sh $out/share/bash-completion/completions/dune + ''; + in '' + ${for-completion} + mkdir -p $out/share cp -r ${./extra}/* $out ''; installFlags = [ "PREFIX=${placeholder "out"}" ]; diff --git a/extra/share/dune/env/env.bash b/extra/share/dune/env/env.bash index 043f17e..0057e50 100644 --- a/extra/share/dune/env/env.bash +++ b/extra/share/dune/env/env.bash @@ -47,7 +47,7 @@ __dune_env() { export __DUNE_SETUP_STATE=success # Only load completions if the shell is interactive. - if [ -t 0 ]; then + if [ -t 0 ] && [ -f "$ROOT"/share/bash-completion/completions/dune ]; then # Load bash completions for dune. # Suppress warning from shellcheck as it can't see the completion script. # shellcheck disable=SC1091 diff --git a/extra/share/dune/env/env.zsh b/extra/share/dune/env/env.zsh index e6a049f..768b7d9 100644 --- a/extra/share/dune/env/env.zsh +++ b/extra/share/dune/env/env.zsh @@ -44,7 +44,7 @@ __dune_env() { export __DUNE_SETUP_STATE=success # Only load completions if the shell is interactive. - if [ -t 0 ]; then + if [ -t 0 ] && [ -f "$ROOT"/share/bash-completion/completions/dune ]; then # completions via bash compat autoload -Uz compinit bashcompinit compinit From a2cc7e5516d051b68eb38d0b9f41b3199102bfa1 Mon Sep 17 00:00:00 2001 From: Stephen Sherratt Date: Wed, 20 Aug 2025 18:06:54 +1000 Subject: [PATCH 3/4] Fix typos in comments Signed-off-by: Stephen Sherratt --- extra/share/dune/env/env.bash | 4 ++-- extra/share/dune/env/env.fish | 4 ++-- extra/share/dune/env/env.sh | 4 ++-- extra/share/dune/env/env.zsh | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/extra/share/dune/env/env.bash b/extra/share/dune/env/env.bash index 0057e50..682ce23 100644 --- a/extra/share/dune/env/env.bash +++ b/extra/share/dune/env/env.bash @@ -68,8 +68,8 @@ __dune_env() { # current environment (possibly by the user sourcing their shell config # or nesting shell sessions), and the previous time it was called it # successfully modified the environment to give precedence to our dune - # executable. check that our dune still has precedence, and attempt to - # undo any opam-specific path shenanigans that have taken place since the + # executable. Check that our dune still has precedence, and attempt to + # undo any opam-specific PATH shenanigans that have taken place since the # last time this function was called. if [ "$(__dune_which)" != "$ROOT/bin/dune" ]; then case :"$PATH": in diff --git a/extra/share/dune/env/env.fish b/extra/share/dune/env/env.fish index 62ae5d8..aa953a3 100644 --- a/extra/share/dune/env/env.fish +++ b/extra/share/dune/env/env.fish @@ -33,8 +33,8 @@ function __dune_env # current environment (possibly by the user sourcing their shell config # or nesting shell sessions), and the previous time it was called it # successfully modified the environment to give precedence to our dune - # executable. check that our dune still has precedence, and attempt to - # undo any opam-specific path shenanigans that have taken place since the + # executable. Check that our dune still has precedence, and attempt to + # undo any opam-specific PATH shenanigans that have taken place since the # last time this function was called. if [ "$(type -P dune)" != "$dune_bin/dune" ] if contains "$dune_bin" $PATH diff --git a/extra/share/dune/env/env.sh b/extra/share/dune/env/env.sh index 9dc5d48..0a33a8b 100644 --- a/extra/share/dune/env/env.sh +++ b/extra/share/dune/env/env.sh @@ -57,8 +57,8 @@ __dune_env() { # current environment (possibly by the user sourcing their shell config # or nesting shell sessions), and the previous time it was called it # successfully modified the environment to give precedence to our dune - # executable. check that our dune still has precedence, and attempt to - # undo any opam-specific path shenanigans that have taken place since the + # executable. Check that our dune still has precedence, and attempt to + # undo any opam-specific PATH shenanigans that have taken place since the # last time this function was called. if [ "$(__dune_which)" != "$__DUNE_ROOT/bin/dune" ]; then case :"$PATH": in diff --git a/extra/share/dune/env/env.zsh b/extra/share/dune/env/env.zsh index 768b7d9..7f94d5a 100644 --- a/extra/share/dune/env/env.zsh +++ b/extra/share/dune/env/env.zsh @@ -67,8 +67,8 @@ __dune_env() { # current environment (possibly by the user sourcing their shell config # or nesting shell sessions), and the previous time it was called it # successfully modified the environment to give precedence to our dune - # executable. check that our dune still has precedence, and attempt to - # undo any opam-specific path shenanigans that have taken place since the + # executable. Check that our dune still has precedence, and attempt to + # undo any opam-specific PATH shenanigans that have taken place since the # last time this function was called. if [ "$(__dune_which)" != "$ROOT/bin/dune" ]; then case :"$PATH": in From 09f2526d2e998a9cf1f1867a6318c480a097bd8e Mon Sep 17 00:00:00 2001 From: Stephen Sherratt Date: Wed, 20 Aug 2025 18:13:08 +1000 Subject: [PATCH 4/4] Add documentation to readme file Signed-off-by: Stephen Sherratt --- README.md | 245 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 227 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ad455e0..fdaddd0 100644 --- a/README.md +++ b/README.md @@ -1,46 +1,252 @@ # Dune Binary Distro -Nix flake for building binary releases of the Dune build system. +Build scripts for binary releases of the [Dune build +system](https://github.com/ocaml/dune). See the +[Releases](https://github.com/ocaml-dune/dune-bin/releases) page for this +project for pre-built binaries. The goal of this project is to provide stable +binary releases of Dune which can be installed without opam. + + +## Installation + +To install the binary distro packages released here, use the +[dune-bin-install](https://github.com/ocaml-dune/dune-bin-install) script, or +see that script for manual installation. Alternatively, extract the package +archive into the root directory of a unix-looking directory structure +(typically a directory with at least a "bin" and "share" subdirectory such as +"~/.local" or "/usr"). + +Download and run the install script to install the latest binary release of Dune with: +``` +curl -fsSL https://github.com/ocaml-dune/dune-bin-install/releases/download/v2/install.sh | sh +``` + +The install script is interactive and will prompt before creating or modifying +any files on your computer. ## Contents Dune is distributed as a compressed tarball file (a `.tag.gz`) containing files -organized in a unix-style filesystem (`/bin`, `/share`, etc). No install script -is provided as yet however users can install these files by recursively copying -them into an appropriate directory such as `/usr/local`, `~/.local`, or `/opt`. +organized in a unix-style filesystem (`/bin`, `/share`, etc). The tarball +contains the Dune executable and documentation, env scripts for setting up Dune +in various shells, and may contain a bash completion script for `dune` commands. + + +## Platform Support + Supported platforms are currently x86_64-linux (statically linked with musl for better compatibility with non-glibc distros), x86_64-macos and aarch64-macos. -## Installation - -To install the binaries released here, use the -[dune-bin-install](https://github.com/ocaml-dune/dune-bin-install) script, or -see that script for manual installation. ## Release Process Modify the `flake.nix` file so that its `.#dune.dynamic` and `.#dune.static` outputs are built from the desired revision of -[ocaml/dune](https://github.com/ocaml/dune). This should just involve setting -the `ref` and `rev` fields of the attribute set passed to `make-duve-pkgs`, -such as: +[ocaml/dune](https://github.com/ocaml/dune). The `make-dune-pkgs` helper +function is provided to simplify this. Pass it a set with 3 elements: +- `ref` is the tag or branch name of the + [dune repo](https://github.com/ocaml/dune) that will be built. It will _also_ be + used to set the version number printed when running `dune --version`. +- `rev` is the revision of the [dune repo](https://github.com/ocaml/dune) that + will be built. It's needed for reproducibility as in the case where `ref` is + a branch. +- `completion` may be `null` or a set with keys `url`, `ref`, and `rev` + pointing to a revision of a git repo containing a bash completion script + named "bash.sh" (probably + [this repo](https://github.com/gridbugs/dune-completion-scripts)). See below + for more details about the completion script. + +Here's an example: ```nix packages = { dune = make-dune-pkgs { ref = "3.19.1"; rev = "76c0c3941798f81dcc13a305d7abb120c191f5fa"; + completion = { + url = "https://github.com/gridbugs/dune-completion-scripts"; + ref = "3.19.1"; + rev = "a56e105760f5cc49369ee012aa1b790f7443bd45"; + }; }; }; ``` You can test the change locally by running `nix build .#dune.dynamic`. Run the -resulting `dune` binary from `result/bin/dune`. +resulting `dune` binary from `result/bin/dune`. The entire package build +process can be tested by running `./make_tarball.sh output .#dune.dynamic` +which will create a file in the current directory called `output.tar.gz` +containing the binary distro package build for your current OS/architecture. Commit the changes and tag the commit with the corresponding version number (`3.19.1` continuing the example above). -Once the github action completes building the binaries they'll be available in -the github "Release" page corresponding to the tag name. +Push the tag to trigger a github action that builds the binary distro packages +for all supported platforms and makes them available on this project's releases +page. + + +## Completion Script + +The [gridbugs/dune-completion-scripts repo](https://github.com/gridbugs/dune-completion-scripts) contains bash +completion scripts for Dune. Its release process is to tag commits with the +version of dune supported by the completion scripts in that commit. For example +tag `3.19.1` of the completion scrips repo will suggest command completions +matching the commands accepted by `dune.3.19.1`. Thus when releasing the binary +distro, choose a version of the completion scripts repo whose tag matches the +tag of Dune included in the release. + +Dune does not contain a mechanism for generating its own completion script. The +completion script is generated using a semi-manual process. Dune uses the +[cmdliner](https://github.com/dbuenzli/cmdliner) library for its CLI which +currently lacks the ability to generate shell completion scripts. To generate +shell completion scripts for Dune, we replace its usage of cmdliner with an +alternative CLI library [climate](https://github.com/gridbugs/climate) which +can generate completion scripts. Climate provides a limited cmdliner +compatibility layer to simplify this process. There's also a +[branch](https://github.com/gridbugs/dune/tree/climate-3.20.0_alpha4) of Dune +(on gridbugs' fork) which can serve as a starting point where climate is +already vendored and a new command is added to Dune to print its completion +script. To generate completions for a particular version of Dune, rebase that +branch onto the desired revision of the Dune repo. Then follow the instructions +[here](https://github.com/gridbugs/dune-completion-scripts?tab=readme-ov-file#how-to-generate) +to generate the completion script. + +If this is too complicated then bundling an older version of the completion +script with a later version of Dune could be considered, though it may be +better UX to have no completions at all than completions that omit valid +suggestions. It's recommended to only use an older version of the completion +script if the CLI hasn't changed since the last release of Dune. + +To omit the completion script entirely, pass `completion = null;` to +`make-dune-pkgs`. E.g. +``` +packages = { + dune = make-dune-pkgs { + ref = "3.19.1"; + rev = "76c0c3941798f81dcc13a305d7abb120c191f5fa"; + completion = null; + }; +}; +``` + +Shell completion support for cmdliner is currently in development, so once this +work is complete Dune updates its vendored copy of cmdliner to include a +version with this feature, the process of generating completion scripts for +Dune will be greatly simplified. + + +## Env Scripts + +This project contains environment scripts in extra/share/dune/env which are +included in the released tarballs. There is an env script for various shells +(bash, zsh, fish, and posix sh). The logic in the env script is to set up the +shell so that running `dune` at the command line invokes the Dune executable +from the binary distro. This involves modifying the `PATH` variable to include +the `bin` directory from the binary distro. To use the env script in a shell, +source the script appropriate for your shell and run the function `__dune_env`, +passing it the location of the binary distro. For example if the binary distro +is installed in ~/.local, the following will set up a bash shell environment: + +```bash +. "$HOME/.local/share/dune/env/env.bash" +__dune_env "$HOME/.local" +``` + +For bash and zsh shells, this will also register bash completions for Dune in +the shell. + +## Version Numbers + +The version of Dune included in a package is determined by the `ref` passed to +`make-dune-pkgs` in flake.nix. This is both the snapshot of the Dune source +repository from which the binary distro is built, and also the value printed by +`dune --version`. Conversely, the version of the _binary distro_ package is +determined by the tag of this repo that was pushed to trigger the build of the +binary distro. This version appears in the filenames of packages (e.g. +`dune-3.19.1-aarch64-apple-darwin.tar.gz`) and also makes up the headings on +the project's releases page. + +By convention we manually ensure that the version of the binary distro is the +same as the version of Dune contained within it by naming the tag of this repo +after the Dune `ref` passed to `make-dune-pkgs`. This policy may change in the +future. + +One unfortunate consequence of this policy is that if the binary distro changes +as distinct from Dune (e.g. if the env or bash completion script changes), the +only way to update the binary distro to include these changes is to re-release +the same version by moving its tag to a commit containing the desired changes +and pushing it to github with `-f`. This causes the released tarballs to be +updated in place which is not ideal. Currently the install script doesn't +attempt to verify the checksum of the binary distro but if we ever add that +feature then we'll need to be more careful with how we update the binary distro +(one option would be to include an additional component in the binary distro's +version number so multiple different versions of the binary distro could be +released containing the same version of dune). + + +## Interactions with Opam + +We expect many users of the Dune binary distro to have pre-existing +installations of opam, and for most opam switches to contain an installation of +Dune. Therefore depending on the order of entries in the user's `PATH` variable, +it's possible that the `dune` command could run the Dune executable from the +binary distro or the current opam switch. + +The env scripts in the binary distro make changes to the user's `PATH` +variable in order to allow `dune` to be run from the command-line, but we try to +be as respectful of the user's `PATH` variable as we can. With this in mind, the +env scripts will not add an entry to the beginning of `PATH` if it's already +present somewhere later in `PATH`. This is because it's possible that the Dune +executable is not the only executable in its bin directory. Thus moving or +re-inserting its bin directory at the beginning of `PATH` may affect the +precedence of other unrelated executables, which we find unacceptable. + +However this poses some problems when co-existing with opam. + +Firstly, the default install location for Dune when using the install script is +~/.local, however it's possible that the user has already added ~/.local/bin to +their `PATH` when the install script runs. If ~/.local/bin is already in `PATH`, +and there's an opam switch earlier in `PATH` that ~/.local/bin, then if the Dune +binary distro is installed to ~/.local then any opam installation of Dune will +take precedence over the binary distro. The install script detects this case and +suggests installing the binary distro to ~/.dune instead, as it's unlikely that +~/.dune/bin is already in `PATH`. + +The second problem comes from the fact that when opam is installed with its +default configuration, every time the shell config is re-loaded, opam will +re-insert the bin directory of the default switch at the beginning of `PATH`. +As described above, Dune's env scripts do not re-insert directories already in +`PATH`, so the second time Dune's shell initialization runs, it doesn't add its +bin directory to the beginning of `PATH`. This means that when a shell config is +reloaded, if the binary distro's Dune executable had precedence prior to +reloading, opam's installation of the Dune executable will have precedence after +reloading. Reloading a shell config is rare, but it's very surprising if doing +so changes which instance of the Dune executable runs when you run the command +`dune`. To fix this, Dune's shell initialization logic detects if the shell +config has been reloaded since the first time it ran, and if so, strips all the +opam bin directories out of `PATH` that come before the Dune binary distro's bin +directory. Eventually we'd like for opam's initialization logic to be changed to +not unconditionally prepend its bin directory into `PATH`. However even if this +happens many users will still be sourcing opam's current shell init scripts in +their shell config for the foreseeable future. + +The third problem comes from the fact that opam's default configuration installs +a pre-command hook that runs immediately before every command runs. This is used +so that in projects with local opam switches, commands always run in the context +of that switch rather than requiring the user to run `eval $(opam env)` +explicitly (indeed the hook just runs `eval $(opam env)` automatically before +each command). This means that if the current opam switch contains an +installation of Dune then that Dune executable will run when the user runs the +`dune` command rather than the Dune binary from the binary distro, _no matter +what the `PATH` variable contains_. The env scripts don't attempt to fix this +problem, however the install script adds a line to the user's shell config that +removes the opam pre-command shell hook. This only takes effect if the Dune +binary distro shell initialization happens after opam's shell initialization, +which we take to mean that the user would prefer the `dune` command to resolve +to our executable rather than one from the opam switch. The user is still free +to run `eval $(opam env)` manually which will have the effect of giving +precedence to any `dune` executable in the current opam switch. + ## Relationship with the "Dune Developer Preview" @@ -51,6 +257,9 @@ experimental feature flags enabled. Its for downloading and building dune every day, uploading built artifacts to a webserver, a small website with information about the experimental features it contains, and an installation script that downloads the latest version of -Dune and updates your shell config to make it available in `PATH`. By contrast, -this repo is intended to be a minimal way of producing binaries of stable -versions of dune which correspond with its release on the opam package manager. +Dune and updates your shell config to make it available in `PATH`. It's similar +to the binary distro in that both projects distribute pre-compiled executables +of Dune installable with a shell script. Unlike the developer preview, the +binary distro releases stable versions of Dune with version numbers matching +those found in opam. It's not experimental, and intended for everyday use as an +alternative to installing Dune with opam.