CI: siblings on the merged PR's branch after a merge; references downloaded by the tests

.github/actions/checkout-siblings/action.yml

@@ -3,8 +3,8 @@ description: >
   Clone OM.jl and its sibling packages (Absyn.jl, DAE.jl, ..., OMParser.jl) at
   the branch matching the change under test: for a pull request from a feature
   branch that branch (in the author's fork or upstream), else the PR base
-  branch, else master; for a push the pushed branch, else master. See
-  checkout-siblings.sh.
+  branch, else master; for a push the branch of the pull request it was merged
+  from, else the pushed branch, else master. See checkout-siblings.sh.
 
 inputs:
   path:
@@ -35,17 +35,39 @@ inputs:
       pushed branch; for a package that has one branch for all lines (OMParser)
     required: false
     default: ""
+  token:
+    description: Token to look up the pull request a pushed commit was merged from
+    required: false
+    default: ${{ github.token }}
 
 runs:
   using: composite
   steps:
+    # After a merge, the siblings of a coordinated change that are not merged yet
+    # are still on its branch: the pushed commit's merged pull request names it.
+    # No pull request (a direct or bot push) or a failed lookup: none.
+    - name: Find the merged pull request
+      id: merged-pr
+      if: github.event_name == 'push'
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.token }}
+        REPOSITORY: ${{ github.repository }}
+        SHA: ${{ github.sha }}
+      run: |
+        head=$(gh api "repos/$REPOSITORY/commits/$SHA/pulls" \
+                 --jq '[.[] | select(.merged_at != null) | .head.ref][0] // empty' 2> /dev/null) || head=""
+        echo "head=$head" >> "$GITHUB_OUTPUT"
+        echo "Branch of the pull request $SHA was merged from: ${head:-none}"
+
     - name: Clone siblings
       shell: bash
       env:
         EVENT_NAME: ${{ github.event_name }}
         HEAD_REF: ${{ github.head_ref }}
         BASE_REF: ${{ github.base_ref }}
         REF_NAME: ${{ github.ref_name }}
+        PUSH_HEAD: ${{ steps.merged-pr.outputs.head }}
         HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
         BASE_REPO: ${{ github.repository }}
         LINE: ${{ inputs.line }}

.github/actions/checkout-siblings/checkout-siblings.sh

@@ -7,16 +7,20 @@
 # <dest dir>/<Package>.jl. The ref is the first branch that exists of
 #   pull request: <PR head branch> in the PR author's fork (fork PRs only),
 #                 <PR head branch>, <line>, master
-#   push:         <line>, master
+#   push:         <PUSH_HEAD>, <line>, master
 # where <line> is LINE if set, else the PR base branch or the pushed branch.
 # So coordinated changes across repositories (same branch name) are tested
-# together, and otherwise the siblings come from the line the change targets
-# (for example 1.13) instead of master. A head branch that is itself a line
-# (master, main, 1.13, ...) or equals the base branch is not a coordinated
-# change: forks carry stale copies of those, so it is skipped.
+# together, also after the first of them is merged (PUSH_HEAD: the siblings
+# not merged yet are still on that branch, the merged ones on master), and
+# otherwise the siblings come from the line the change targets (for example
+# 1.13) instead of master. A head branch that is itself a line (master, main,
+# 1.13, ...) or equals the base branch is not a coordinated change: forks
+# carry stale copies of those, so it is skipped.
 #
 # Environment (set by the action from the GitHub context; all optional):
 #   EVENT_NAME, HEAD_REF, BASE_REF, REF_NAME
+#   PUSH_HEAD             for a push, the head branch of the pull request the
+#                         pushed commit was merged from (the action looks it up)
 #   HEAD_REPO, BASE_REPO  owner/name of the PR head and base repositories
 #   LINE                  force the line (e.g. 1.13) instead of BASE_REF/REF_NAME
 #   OM_REPOSITORY         repository of OM.jl (default JKRT/OM.jl)
@@ -67,6 +71,8 @@ fi
 head=""
 if [ -n "$pr" ] && [ -n "${HEAD_REF:-}" ] && ! is_line "$HEAD_REF" && [ "$HEAD_REF" != "${BASE_REF:-}" ]; then
   head=$HEAD_REF
+elif [ -z "$pr" ] && [ -n "${PUSH_HEAD:-}" ] && ! is_line "$PUSH_HEAD" && [ "$PUSH_HEAD" != "${REF_NAME:-}" ]; then
+  head=$PUSH_HEAD
 fi
 
 summary=""

.github/workflows/omjl-ci.yml

@@ -7,8 +7,8 @@ name: OM.jl integration
 # 1. Check out OM.jl and every sibling package at the branch that matches the
 #    change (.github/actions/checkout-siblings: a PR's feature branch if it
 #    exists in that repository, else the PR base branch or `line`, e.g. 1.13,
-#    else master), in the layout of OM.jl's submodules, with the package under
-#    test in its place.
+#    else master; after a merge the merged PR's branch first), in the layout of
+#    OM.jl's submodules, with the package under test in its place.
 # 2. Build the native libraries (OMParser from source when no release exists
 #    for the Julia version), set up and precompile OM.jl (ci/setup.jl).
 # 3. Run the package's own tests against the checked-out siblings
@@ -126,10 +126,8 @@ jobs:
         run: julia --color=yes OM.jl/ci/test-package.jl "$PACKAGE"
 
       # Also after failing package tests: whether OM.jl still works is the point.
-      - name: MSL reference results
-        if: ${{ !cancelled() && inputs.om-tests && steps.setup.outcome == 'success' }}
-        run: OM.jl/ci/fetch-msl-refs.sh
-
+      # A test downloads the MSL reference results it compares against
+      # (validateMSLModel in test/testUtils.jl).
       - name: OM.jl tests (${{ matrix.group }})
         if: ${{ !cancelled() && inputs.om-tests && steps.setup.outcome == 'success' }}
         working-directory: OM.jl/test

test/mslTests.jl

@@ -978,7 +978,6 @@ end
      parameter array by a runtime index in an affect (UndefVarError), and ShowCompositeStep's
      `suspend = anyTrue({...}) or ...` had its initialization row on an alias outside the
      lifted names (refused). =#
-  #= The reference names are literals: ci/fetch-msl-refs.sh finds them next to validateMSLModel. =#
   @testset "IntegerNetwork1: a function of discrete variables only" begin
     sol = simulateMSL("Modelica.Blocks.Examples.IntegerNetwork1", 10.0)
     @test sol !== nothing

test/testUtils.jl

@@ -319,6 +319,31 @@ function resolveMTKVariable(sol, sym::Symbol)
   return sym
 end
 
+"""
+Held while a reference is downloaded: the tests run with threads.
+"""
+const REFERENCE_DOWNLOAD_LOCK = ReentrantLock()
+
+"""
+    fetchMSLReference(refFile) -> Union{Nothing, String}
+
+Download the MAP-LIB reference `refFile` (its CSV and signal list) into `MSL_REF_DIR` with
+OMLibraryTesting's `download_refs.sh`, unless its CSV is there already. `nothing` once the CSV is
+there, else why it is not.
+"""
+function fetchMSLReference(refFile::String)::Union{Nothing, String}
+  csv_path = joinpath(MSL_REF_DIR, "csv", refFile * ".csv")
+  return lock(REFERENCE_DOWNLOAD_LOCK) do
+    isfile(csv_path) && return nothing
+    isfile(joinpath(MSL_REF_DIR, "download_refs.sh")) || return "no download_refs.sh in $MSL_REF_DIR"
+    Sys.which("bash") === nothing && return "no bash to run download_refs.sh"
+    output = IOBuffer()
+    ok = success(pipeline(Cmd(`bash download_refs.sh $refFile`; dir = MSL_REF_DIR); stdout = output, stderr = output))
+    isfile(csv_path) && return nothing
+    return "download_refs.sh $refFile $(ok ? "downloaded no CSV" : "failed"): $(last(strip(String(take!(output))), 300))"
+  end
+end
+
 """
     validateMSLModel(sol, refFile; stopTime, reltol, atol, npoints) -> (passed, details)
 
@@ -329,7 +354,8 @@ Validate simulation solution against MAP-LIB reference data.
 - Returns `(passed::Bool, details::String)` where details describes any failures.
 
 Signals are loaded from `OMLibraryTesting.jl/reference/signals/<refFile>.txt`
-and reference values from `OMLibraryTesting.jl/reference/csv/<refFile>.csv`.
+and reference values from `OMLibraryTesting.jl/reference/csv/<refFile>.csv`,
+downloaded first when they are not there (`fetchMSLReference`).
 Signal names are converted from Modelica dot notation to underscore notation.
 """
 function validateMSLModel(sol, refFile::String;
@@ -340,8 +366,9 @@ function validateMSLModel(sol, refFile::String;
   csv_path = joinpath(MSL_REF_DIR, "csv", refFile * ".csv")
   signals_path = joinpath(MSL_REF_DIR, "signals", refFile * ".txt")
 
-  if !isfile(csv_path)
-    return (false, "Reference CSV not found: $csv_path")
+  missing_reason = fetchMSLReference(refFile)
+  if missing_reason !== nothing
+    return (false, "Reference CSV not found: $csv_path ($missing_reason)")
   end
 
   ref_time, ref_signals = loadReferenceCSV(csv_path)