accept yaml files (#1)

crschardt · web-flow · commit 4c90a9de2d9f · 2025-11-23T13:39:22.000-06:00
* accept yaml files * mount image with offset * make shrinking image optional * make tar for rubikpi and compress image * avoid sudo in compress image * Revert "avoid sudo in compress image" This reverts commit 1b54014.
diff --git a/.github/workflows/test-raspi.yml b/.github/workflows/test-raspi.yml
@@ -1,18 +1,18 @@
 name: 'Test modifying a Raspberry Pi image'
 on:
-  # push:
-  #   branches:
-  #     - 'main'
-  #     - 'releases/**'
-  # pull_request:
+  push:
+    branches:
+      - 'main'
+      - 'releases/**'
+  pull_request:
   workflow_dispatch:
 
 jobs:
   build:
     runs-on: ubuntu-24.04-arm
     steps:
       - uses: actions/checkout@v4
-      - uses: photonvision/photon-image-runner@HEAD
+      - uses: ./  # photonvision/photon-image-runner@HEAD
         with:
           image_url: https://downloads.raspberrypi.com/raspios_lite_arm64/images/raspios_lite_arm64-2025-05-13/2025-05-13-raspios-bookworm-arm64-lite.img.xz
           additional_mb: 200
@@ -28,7 +28,7 @@ jobs:
             sudo xz -T 0 -v ${{ steps.build_image.outputs.image }}
       - uses: actions/upload-artifact@v4
         with:
-          name: ${{ steps.build_image.outputs.image }}.xz
+          name: raspi.img.xz
           path: ${{ steps.build_image.outputs.image }}.xz
           compression-level: 0
           if-no-files-found: error
diff --git a/.github/workflows/test-rubikpi.yml b/.github/workflows/test-rubikpi.yml
@@ -1,4 +1,4 @@
-name: 'Test modifying a Raspberry Pi image'
+name: 'Test modifying a Rubik Pi image'
 on:
   push:
     branches:
@@ -12,23 +12,36 @@ jobs:
     runs-on: ubuntu-24.04-arm
     steps:
       - uses: actions/checkout@v4
-      - uses: photonvision/photon-image-runner@HEAD
+      - uses: ./
         with:
           image_url: https://people.canonical.com/~platform/images/qualcomm-iot/rubikpi3/ubuntu-server-24.04/x00/ubuntu-24.04-preinstalled-server-arm64+rubikpi3-20250912-127.yaml
           minimum_free_mb: 2000
+          root_location: "offset=569376768"
+          shrink_image: "no"
           commands: |
-              echo "Testing Raspberry Pi image"
+              echo "Testing Rubik Pi image"
               uname -a
               lsblk
               echo "${loopdev}"
         id: build_image
       - name: Compress image
         run: |
-            sudo xz -T 0 -v ${{ steps.build_image.outputs.image }}
+            imagedir=$(dirname ${{ steps.build_image.outputs.image }})
+            tardir=${RUNNER_TEMP}/photonvision_rubikpi3
+            mkdir --parents ${tardir}
+            if ls ${imagedir}*.tar.gz 1>/dev/null 2>&1; then
+              sudo tar -xzf ${imagedir}*.tar.gz -C ${tardir}
+            fi
+            sudo mv ${imagedir}/rawprogram*.xml ${tardir}/ 2>/dev/null || true
+            sudo mv ${imagedir}/dtb.bin ${tardir}/ 2>/dev/null || true
+            sudo mv ${imagedir}/*.img ${tardir}/ 2>/dev/null || true
+            sudo find ${tardir} -mindepth 2 -type f -exec mv {} ${tardir}/ \;
+            sudo find ${tardir} -mindepth 1 -type d -empty -delete
+            sudo tar -I 'xz -T0' -cf ./photonvision_rubikpi3.tar.xz ${tardir} --checkpoint=10000 --checkpoint-action=echo='%T'
       - uses: actions/upload-artifact@v4
         with:
-          name: ${{ steps.build_image.outputs.image }}.xz
-          path: ${{ steps.build_image.outputs.image }}.xz
+          name: photonvision_rubikpi3.tar.xz
+          path: ./photonvision_rubikpi3.tar.xz
           compression-level: 0
           if-no-files-found: error
           retention-days: 1
diff --git a/action.yml b/action.yml
@@ -22,6 +22,14 @@ inputs:
     description: 'Partition number for root'
     required: false
     default: '2'
+  root_location:
+    description: 'Location of the root as either "partition=X" or "offset=XXX"'
+    required: false
+    default: 'partition=2'
+  shrink_image:
+    description: 'Make the output image file as small as possible'
+    reqiured: false
+    default: 'yes'
 
 outputs:
   image:
@@ -33,41 +41,40 @@ runs:
   steps:
     - name: Get base image
       run: |
-          ${GITHUB_ACTION_PATH}/get_image.sh ${{ inputs.image_url }}
+          sudo --preserve-env bash ${GITHUB_ACTION_PATH}/get_image.sh ${{ inputs.image_url }}
       shell: bash
       id: get_image
 
-    # - name: Mount image
-    #   run: |
-    #       sudo --preserve-env bash "${GITHUB_ACTION_PATH}/mount_image.sh" "${{ steps.get_image.outputs.image }}" "${{ inputs.additional_mb }}" "${{ inputs.minimum_free_mb }}" "${{ inputs.root_partition }}"
-    #   shell: bash
-    #   id: mount_image
-
-    # - name: Run commands
-    #   run: |
-    #       echo "Running the commands"
-    #       chroot_scriptdir=/tmp/build
-    #       scriptdir="${{ env.rootdir }}${chroot_scriptdir}"
-    #       sudo mkdir --parents "${scriptdir}"
-    #       sudo mount --bind "${{ github.workspace }}" "${scriptdir}"
+    - name: Mount image
+      run: |
+          sudo --preserve-env bash "${GITHUB_ACTION_PATH}/mount_image.sh" "${{ steps.get_image.outputs.image }}" "${{ inputs.additional_mb }}" "${{ inputs.minimum_free_mb }}" "${{ inputs.root_location }}"
+      shell: bash
+      id: mount_image
 
-    #       cat >> "${scriptdir}/commands.sh" << EOF
-    #       #!/bin/bash
-    #       # set -exo pipefail
-    #       # export SHELLOPTS
-    #       # export BASHOPTS
-    #       cd "${chroot_scriptdir}"
-    #       ${{ inputs.commands }}
-    #       EOF
+    - name: Run commands
+      run: |
+          echo "Running the commands"
+          chroot_scriptdir=/tmp/build
+          scriptdir="${{ env.rootdir }}${chroot_scriptdir}"
+          sudo mkdir --parents "${scriptdir}"
+          sudo mount --bind "${{ github.workspace }}" "${scriptdir}"
 
-    #       sudo chmod +x "${scriptdir}/commands.sh"
+          cat >> "${scriptdir}/commands.sh" << EOF
+          #!/bin/bash
+          # set -exo pipefail
+          # export SHELLOPTS
+          # export BASHOPTS
+          cd "${chroot_scriptdir}"
+          ${{ inputs.commands }}
+          EOF
 
-    #       sudo --preserve-env chroot "${{ env.rootdir }}" bash -c "${chroot_scriptdir}/commands.sh"
-    #   shell: bash
-    #   id: run_commands
+          sudo chmod +x "${scriptdir}/commands.sh"
+          sudo --preserve-env chroot "${{ env.rootdir }}" bash -c "${chroot_scriptdir}/commands.sh"
+      shell: bash
+      id: run_commands
 
-    # - name: Clean up and pack image
-    #   run: |
-    #       sudo --preserve-env bash ${GITHUB_ACTION_PATH}/pack_image.sh "${{ steps.get_image.outputs.image }}"
-    #   shell: bash
-    #   id: pack_image
+    - name: Clean up and pack image
+      run: |
+          sudo --preserve-env bash ${GITHUB_ACTION_PATH}/pack_image.sh "${{ steps.get_image.outputs.image }}" "${{ inputs.shrink_image }}"
+      shell: bash
+      id: pack_image
diff --git a/get_image.sh b/get_image.sh
@@ -3,35 +3,42 @@ set -euo pipefail
 
 url=$1
 download_path="${RUNNER_TEMP}/image"
+echo "download_path=${download_path}" >> "$GITHUB_ENV"
 mkdir --parent "${download_path}"
 
-if [[ ${url} = @("*.yaml"|"*.yml") ]]; then
+image=""
+
+if [[ ${url} = @(*.yaml|*.yml) ]]; then
     apt-get --quiet update
     apt-get --yes --quiet install yq
     wget --no-verbose --output-document="manifest.yaml" "${url}"
     echo "=== Manifest contents ==="
     cat manifest.yaml
     echo "========================="
-    yq -r '.urls[] | "\(.url) \(.sha256sum)"' manifest.yaml | while read -r url sha; do
-        filename="$(basename ${url})"
-        echo "Downloading: ${filename} from ${url}"
-        wget --no-verbose --output-document=${download_path}/${filename} ${url}
-        echo "$sha $filename" | sha256sum -c -
+    yq -r '.urls[] | "\(.url) \(.sha256sum)"' ./manifest.yaml > urls
+    while read -r file_url sha; do
+        filename="$(basename ${file_url})"
+        echo "Downloading: ${filename} from ${file_url}"
+        wget --no-verbose --output-document=${download_path}/${filename} ${file_url}
+        echo "$sha ${download_path}/$filename" | sha256sum -c -
         [[ ${filename} = *.img.xz ]] && image="${download_path}/${filename}"
-    done
+    done < urls
 else 
     image="${download_path}/$(basename ${url})"
     echo "Downloading ${image} from ${url}"
     wget --no-verbose --output-document="${image}" ${url}
 fi
 
+echo "Image: ${image}"
+
+ls -l ${download_path}
+
 if [[ ${image} = *.xz ]]; then
     echo "Unzipping ${image}"
     unxz ${image}
     image=${image%.xz}
 fi
 
-echo "PWD: $(pwd)"
 ls -l ${download_path}
 
 if [[ ${image} != *.img ]]; then
diff --git a/mount_image.sh b/mount_image.sh
@@ -4,26 +4,41 @@ set -euo pipefail
 image="$1"
 additional_mb=$2
 minimum_free=$3
-rootpartition=$4
-echo "rootpartition=${rootpartition}" >> "$GITHUB_ENV"
+root_location=$4
 
 ####
 # Prepare and mount the image
 ####
 
-loopdev=$(losetup --find --show --partscan ${image})
+case ${root_location,,} in
+    partition* )
+        rootpartition=${root_location#*=}
+        loopdev=$(losetup --find --show --partscan ${image})
+        rootdev="${loopdev}p${rootpartition}"
+    ;;
+    offset* )
+        rootpartition=1
+        rootoffset=${root_location#*=}
+        loopdev=$(losetup --find --show --offset=${rootoffset} ${image})
+        rootdev="${loopdev}"
+    ;;
+    * ) 
+        echo "Don't understand value for root_location: ${root_location}"
+        exit 1
+    ;;
+esac
+
 echo "loopdev=${loopdev}" >> "$GITHUB_ENV"
+echo "rootpartition=${rootpartition}" >> "$GITHUB_ENV"
+echo "rootdev=${rootdev}" >> "$GITHUB_ENV"
+echo "Root device is: ${rootdev}"
 
 echo "Partitions in the mounted image:"
 lsblk "${loopdev}"
 
 part_type=$(blkid -o value -s PTTYPE "${loopdev}")
 echo "part_type=${part_type}" >> "$GITHUB_ENV"
-echo "Image is using ${part_type} partition table"
-
-rootdev="${loopdev}p${rootpartition}"
-echo "rootdev=${rootdev}" >> "$GITHUB_ENV"
-echo "Root device is: ${rootdev}"
+echo "Image is using ${part_type:=NO} partition table"
 
 rootdir="./rootfs"
 rootdir="$(realpath ${rootdir})"
@@ -48,9 +63,11 @@ if [[ ${additional_mb} -gt 0 ]]; then
     if [[ "${part_type}" == "gpt" ]]; then
         sgdisk --move-second-header "${loopdev}"
     fi
-    parted --script "${loopdev}" resizepart ${rootpartition} 100%
-    e2fsck -p -f "${loopdev}p${rootpartition}"
-    resize2fs "${loopdev}p${rootpartition}"
+    if [[ ${rootpartition} -gt 0 ]]; then
+        parted --script "${loopdev}" resizepart ${rootpartition} 100%
+    fi
+    e2fsck -p -f "${rootdev}"
+    resize2fs "${rootdev}"
     echo "Finished resizing disk image."
 fi
 
@@ -69,6 +86,7 @@ df --block-size=M "${rootdir}"
 # Set up the environment
 mount -t proc /proc "${rootdir}/proc"
 mount -t sysfs /sys "${rootdir}/sys"
+mount -t tmpfs /tmpfs "${rootdir}/run"
 mount --rbind /dev "${rootdir}/dev"
 
 # Temporarily replace resolv.conf for networking
diff --git a/pack_image.sh b/pack_image.sh
@@ -5,6 +5,7 @@ set -euo pipefail
 # Clean up and shrink image
 ####
 image="$1"
+shrink="$2"
 
 if [[ -e "${rootdir}/etc/resolv.conf.bak" ]]; then
     mv "${rootdir}/etc/resolv.conf.bak" "${rootdir}/etc/resolv.conf"
@@ -15,37 +16,39 @@ echo "Zero filling empty space"
 
 umount --recursive "${rootdir}"
 
-echo "Resizing root filesystem to minimal size."
-e2fsck -v -f -p -E discard "${rootdev}"
-resize2fs -M "${rootdev}"
-rootfs_blocksize=$(tune2fs -l ${rootdev} | grep "^Block size" | awk '{print $NF}')
-rootfs_blockcount=$(tune2fs -l ${rootdev} | grep "^Block count" | awk '{print $NF}')
-
-echo "Resizing rootfs partition."
-rootfs_partstart=$(parted -m --script "${loopdev}" unit B print | grep "^${rootpartition}:" | awk -F ":" '{print $2}' | tr -d 'B')
-rootfs_partsize=$((${rootfs_blockcount} * ${rootfs_blocksize}))
-rootfs_partend=$((${rootfs_partstart} + ${rootfs_partsize} - 1))
-rootfs_partoldend=$(parted -m --script "${loopdev}" unit B print | grep "^${rootpartition}:" | awk -F ":" '{print $3}' | tr -d 'B')
-if [ "$rootfs_partoldend" -gt "$rootfs_partend" ]; then
-    echo y | parted ---pretend-input-tty "${loopdev}" unit B resizepart "${rootpartition}" "${rootfs_partend}"
-else
-    echo "Rootfs partition not resized as it was not shrunk"
-fi
+if [[ ${shrink,,} = y* ]]; then
+    echo "Resizing root filesystem to minimal size."
+    e2fsck -v -f -p -E discard "${rootdev}"
+    resize2fs -M "${rootdev}"
+    rootfs_blocksize=$(tune2fs -l ${rootdev} | grep "^Block size" | awk '{print $NF}')
+    rootfs_blockcount=$(tune2fs -l ${rootdev} | grep "^Block count" | awk '{print $NF}')
+
+    echo "Resizing rootfs partition."
+    rootfs_partstart=$(parted -m --script "${loopdev}" unit B print | grep "^${rootpartition}:" | awk -F ":" '{print $2}' | tr -d 'B')
+    rootfs_partsize=$((${rootfs_blockcount} * ${rootfs_blocksize}))
+    rootfs_partend=$((${rootfs_partstart} + ${rootfs_partsize} - 1))
+    rootfs_partoldend=$(parted -m --script "${loopdev}" unit B print | grep "^${rootpartition}:" | awk -F ":" '{print $3}' | tr -d 'B')
+    if [ "$rootfs_partoldend" -gt "$rootfs_partend" ]; then
+        echo y | parted ---pretend-input-tty "${loopdev}" unit B resizepart "${rootpartition}" "${rootfs_partend}"
+    else
+        echo "Rootfs partition not resized as it was not shrunk"
+    fi
 
-free_space=$(parted -m --script "${loopdev}" unit B print free | tail -1)
-if [[ "${free_space}" =~ "free" ]]; then
-    initial_image_size=$(stat -L --printf="%s" "${image}")
-    image_size=$(echo "${free_space}" | awk -F ":" '{print $2}' | tr -d 'B')
-    if [[ "${part_type}" == "gpt" ]]; then
-        # for GPT partition table, leave space at the end for the secondary GPT 
-        # it requires 33 sectors, which is 16896 bytes
-        image_size=$((image_size + 16896))
-    fi            
-    echo "Shrinking image from ${initial_image_size} to ${image_size} bytes."
-    truncate -s "${image_size}" "${image}"
-    if [[ "${part_type}" == "gpt" ]]; then
-        # use sgdisk to fix the secondary GPT after truncation 
-        sgdisk -e "${image}"
+    free_space=$(parted -m --script "${loopdev}" unit B print free | tail -1)
+    if [[ "${free_space}" =~ "free" ]]; then
+        initial_image_size=$(stat -L --printf="%s" "${image}")
+        image_size=$(echo "${free_space}" | awk -F ":" '{print $2}' | tr -d 'B')
+        if [[ "${part_type}" == "gpt" ]]; then
+            # for GPT partition table, leave space at the end for the secondary GPT 
+            # it requires 33 sectors, which is 16896 bytes
+            image_size=$((image_size + 16896))
+        fi            
+        echo "Shrinking image from ${initial_image_size} to ${image_size} bytes."
+        truncate -s "${image_size}" "${image}"
+        if [[ "${part_type}" == "gpt" ]]; then
+            # use sgdisk to fix the secondary GPT after truncation 
+            sgdisk -e "${image}"
+        fi
     fi
 fi
 
