build-push.yml

name: build-push

on:
  schedule:
    - cron: "30 */6 * * *"
  push:
    branches:
    - master
  pull_request:
  workflow_dispatch:
    inputs:
      EDM4EIC_VERSION:
        required: false
        default: ''
        type: string
      EICRECON_VERSION:
        required: false
        default: ''
        type: string
      JUGGLER_VERSION:
        required: false
        default: ''
        type: string

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: false

permissions:
  contents: read
  packages: write

env:
  ## CUDA version and container operating system
  CUDA_VERSION: 12.5.1
  CUDA_OS: ubuntu24.04

  ## Default versions are specified in packages.yaml but can be overridden
  ## note: nightly builds will always use the master/main branch
  EDM4EIC_VERSION: ${{ inputs.EDM4EIC_VERSION }}
  EICRECON_VERSION: ${{ inputs.EICRECON_VERSION }}
  JUGGLER_VERSION: ${{ inputs.JUGGLER_VERSION }}

  ## Dockerhub registry
  DH_REGISTRY: docker.io
  DH_REGISTRY_USER: eicweb
  DH_PUSH: 0
  ## GitHub registry
  GH_REGISTRY: ghcr.io
  GH_REGISTRY_USER: eic
  GH_PUSH: 1 # required

  ## Number of jobs to start during container builds
  JOBS: 4

  ## Internal tag used for the CI
  INTERNAL_TAG: pipeline-${{ github.run_id }}

  ## Compressed image size caps (GiB).  Fail the build if a final image exceeds
  ## these limits to catch regressions early (e.g. accidental Spack-built LLVM).
  ## To update: measure the compressed sizes of the published master images,
  ## add ~15% headroom, round up, and update here and in .gitlab-ci.yml
  ## variables.
  SIZE_LIMIT_DEBIAN_STABLE_BASE_GIB: 0.8
  SIZE_LIMIT_EIC_CI_GIB: 3
  SIZE_LIMIT_EIC_XL_GIB: 4.5

jobs:
  env:
    ## The env context is not available in matrix, only in steps,
    ## so this job is merely to transfer env to job outputs which
    ## can be used in matrix.
    name: Turn env into outputs
    runs-on: ubuntu-latest
    outputs:
      CUDA_VERSION: ${{ steps.env.outputs.CUDA_VERSION }}
      CUDA_OS: ${{ steps.env.outputs.CUDA_OS }}
      BUILDWEEK: ${{ steps.env.outputs.BUILDWEEK }}
    steps:
      - name: Turn env into outputs
        id: env
        run: |
          echo "CUDA_VERSION=${{ env.CUDA_VERSION }}" >> $GITHUB_OUTPUT
          echo "CUDA_OS=${{ env.CUDA_OS }}" >> $GITHUB_OUTPUT
          echo "BUILDWEEK=$(date +%Yw%V)" >> $GITHUB_OUTPUT

  base:
    name: Build ${{ matrix.BUILD_IMAGE }} on ${{ matrix.arch }}
    runs-on: ${{ matrix.runner }}
    needs: env
    strategy:
      matrix:
        include:
        - BASE_IMAGE: debian:trixie-slim
          BUILD_IMAGE: debian_stable_base
          PLATFORM: linux/amd64
          runner: ubuntu-latest
          arch: amd64
        - BASE_IMAGE: debian:trixie-slim
          BUILD_IMAGE: debian_stable_base
          PLATFORM: linux/amd64/v3
          runner: ubuntu-latest
          arch: amd64_v3
        - BASE_IMAGE: debian:trixie-slim
          BUILD_IMAGE: debian_stable_base
          PLATFORM: linux/arm64
          runner: ubuntu-24.04-arm
          arch: arm64
        - BASE_IMAGE: nvidia/cuda:${{ needs.env.outputs.CUDA_VERSION }}-devel-${{ needs.env.outputs.CUDA_OS }}
          BUILD_IMAGE: cuda_devel
          PLATFORM: linux/amd64
          runner: ubuntu-latest
          arch: amd64
        - BASE_IMAGE: nvidia/cuda:${{ needs.env.outputs.CUDA_VERSION }}-runtime-${{ needs.env.outputs.CUDA_OS }}
          BUILD_IMAGE: cuda_runtime
          PLATFORM: linux/amd64
          runner: ubuntu-latest
          arch: amd64
      fail-fast: false
    steps:
      - name: Checkout
        uses: actions/checkout@v7
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v4
        with:
          platforms: linux/amd64,linux/amd64/v3,linux/arm64
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4
        id: buildx
      - name: Cache build mounts
        # Cache mount contents (ccache, apt cache, spack cache) using GitHub Actions cache
        # This is separate from the registry cache which stores image layers
        # Reference: https://docs.docker.com/build/ci/github-actions/cache/#cache-mounts
        # Implementation: https://github.com/reproducible-containers/buildkit-cache-dance
        uses: actions/cache@v6
        id: cache-base-mounts
        with:
          path: cache-mount-base-${{ matrix.BUILD_IMAGE }}-${{ matrix.arch }}
          key: ${{ matrix.BUILD_IMAGE }}-${{ matrix.arch }}-base-mounts-${{ github.ref_name }}-${{ needs.env.outputs.BUILDWEEK }}
          restore-keys: |
            ${{ matrix.BUILD_IMAGE }}-${{ matrix.arch }}-base-mounts-main-${{ needs.env.outputs.BUILDWEEK }}
            ${{ matrix.BUILD_IMAGE }}-${{ matrix.arch }}-base-mounts-main
            ${{ matrix.BUILD_IMAGE }}-${{ matrix.arch }}-base-mounts-
      - name: Inject cache mounts into builder
        uses: reproducible-containers/buildkit-cache-dance@v3
        with:
          builder: ${{ steps.buildx.outputs.name }}
          cache-dir: cache-mount-base-${{ matrix.BUILD_IMAGE }}-${{ matrix.arch }}
          cache-map: |
            {
              "var-cache-apt": "/var/cache/apt",
              "var-lib-apt": "/var/lib/apt",
              "var-cache-spack": "/var/cache/spack"
            }
          skip-extraction: ${{ steps.cache-base-mounts.outputs.cache-hit }}
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        if: ${{ env.GH_PUSH == '1' }}
        with:
          registry: ${{ env.GH_REGISTRY }}
          username: ${{ secrets.GHCR_REGISTRY_USER }}
          password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
      - name: Build and push
        env:
          BUILD_IMAGE: ${{ matrix.BUILD_IMAGE }}
          BASE_IMAGE: ${{ matrix.BASE_IMAGE }}
          PLATFORM: ${{ matrix.PLATFORM }}
          BUILDWEEK: ${{ needs.env.outputs.BUILDWEEK }}
          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
          METADATA_FILE: /tmp/build-metadata.json
        run: bash scripts/build-base.sh
      - name: Export digest to file
        run: |
          DIGEST=$(jq -r '."containerimage.digest"' /tmp/build-metadata.json)
          IMAGE="${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/${{ matrix.BUILD_IMAGE }}"
          mkdir -p /tmp/digests
          echo "${IMAGE}@${DIGEST}" > /tmp/digests/${{ matrix.BUILD_IMAGE }}-${{ matrix.arch }}.digest
      - name: Check image size
        run: |
          FULLNAME="${{ matrix.BUILD_IMAGE }}"
          LIMIT_VAR="SIZE_LIMIT_$(printf '%s' "${FULLNAME}" | tr '[:lower:]' '[:upper:]')_GIB"
          LIMIT="${!LIMIT_VAR}"
          if [ -n "${LIMIT}" ]; then
            scripts/check_image_size \
              "${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/${FULLNAME}@$(jq -r '."containerimage.digest"' /tmp/build-metadata.json)" \
              "${LIMIT}"
          else
            echo "No size limit configured for ${FULLNAME}, skipping check."
          fi
      - name: Upload digest as artifact
        uses: actions/upload-artifact@v7
        with:
          name: base-${{ matrix.BUILD_IMAGE }}-${{ matrix.arch }}-digest
          path: /tmp/digests/${{ matrix.BUILD_IMAGE }}-${{ matrix.arch }}.digest
          retention-days: 1

  base-manifest:
    name: Push ${{ matrix.BUILD_IMAGE }} manifest
    runs-on: ubuntu-latest
    needs: base
    strategy:
      matrix:
        include:
        - BUILD_IMAGE: debian_stable_base
        - BUILD_IMAGE: cuda_devel
        - BUILD_IMAGE: cuda_runtime
    steps:
      - name: Set up QEMU (for imagetools)
        uses: docker/setup-qemu-action@v4
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4
      - name: Login to Docker Hub
        uses: docker/login-action@v4
        if: ${{ env.DH_PUSH == '1' }}
        with:
          registry: ${{ env.DH_REGISTRY }}
          username: ${{ env.DH_REGISTRY_USER }}
          password: ${{ secrets.DH_EICWEB_TOKEN }}
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        if: ${{ env.GH_PUSH == '1' }}
        with:
          registry: ${{ env.GH_REGISTRY }}
          username: ${{ secrets.GHCR_REGISTRY_USER }}
          password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
      - name: Download all digest artifacts
        uses: actions/download-artifact@v8
        with:
          path: /tmp/digests
          pattern: base-${{ matrix.BUILD_IMAGE }}-*-digest
          merge-multiple: true
      - name: Analyze digest artifacts
        id: digests
        run: |
          # Loop over all available architecture artifacts
          ALL_DIGESTS=""
          FIRST_DIGEST=""
          shopt -s nullglob
          for digest_file in /tmp/digests/${{ matrix.BUILD_IMAGE }}-*.digest; do
            DIGEST=$(cat "$digest_file")
            echo "Found digest: $DIGEST"
            if [ -z "$ALL_DIGESTS" ]; then
              ALL_DIGESTS="$DIGEST"
            else
              ALL_DIGESTS="$ALL_DIGESTS $DIGEST"
            fi
            if [ -z "$FIRST_DIGEST" ]; then
              FIRST_DIGEST="$DIGEST"
            fi
          done
          shopt -u nullglob
          # Check if at least one digest was found
          if [ -z "$FIRST_DIGEST" ]; then
            echo "Error: No digest files found"
            exit 1
          fi
          # Get the base image name from the first digest
          REGISTRY_IMAGE=$(echo $FIRST_DIGEST | cut -d'@' -f1)
          REGISTRY=$(echo $REGISTRY_IMAGE | cut -d'/' -f1-2)
          IMAGE=$(echo $REGISTRY_IMAGE | cut -d'/' -f3)
          echo "Registry Name: $REGISTRY"
          echo "Image Name: $IMAGE"
          echo "All Digests: $ALL_DIGESTS"
          echo "registry=$REGISTRY" >> $GITHUB_OUTPUT
          echo "image=$IMAGE" >> $GITHUB_OUTPUT
          echo "all=$ALL_DIGESTS" >> $GITHUB_OUTPUT
      - name: Extract Docker metadata for final tags
        id: meta
        uses: docker/metadata-action@v6
        with:
          images: |
            name=${{ env.DH_REGISTRY }}/${{ env.DH_REGISTRY_USER }}/${{ steps.digests.outputs.image }},enable=${{ env.DH_PUSH != 0 }}
            name=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/${{ steps.digests.outputs.image }},enable=${{ env.GH_PUSH != 0 }}
          tags: |
            ${{ env.INTERNAL_TAG }}
            type=ref,prefix=unstable-pr-,event=pr
            type=match,pattern=^v(\d+\.\d+\.\d+-.*)$,group=1
      - name: Create and push manifest list
        run: |
          # Create tag arguments from the multi-line 'tags' output from the meta step
          TAG_ARGS=""
          while read -r line; do
            TAG_ARGS="$TAG_ARGS --tag $line"
          done <<< "${{ steps.meta.outputs.tags }}"
          # Create the manifest list and tag it with the final tags
          docker buildx imagetools create \
            $TAG_ARGS \
            ${{ steps.digests.outputs.all }}

  eic:
    name: Build ${{ matrix.BUILD_IMAGE }}${{ matrix.ENV }} on ${{ matrix.arch }}
    runs-on: ${{ matrix.runner }}
    needs: base-manifest
    strategy:
      matrix:
        include:
        - BUILD_IMAGE: eic_
          BUILD_TYPE: "default,nightly"
          BUILDER_IMAGE: debian_stable_base
          RUNTIME_IMAGE: debian_stable_base
          ENV: ci
          arch: amd64
          runner: ubuntu-latest
          PLATFORM: linux/amd64
          target: final
        - BUILD_IMAGE: eic_
          BUILD_TYPE: "default,nightly"
          BUILDER_IMAGE: debian_stable_base
          RUNTIME_IMAGE: debian_stable_base
          ENV: ci
          arch: amd64_v3
          runner: ubuntu-latest
          PLATFORM: linux/amd64/v3
          target: final
        - BUILD_IMAGE: eic_
          BUILD_TYPE: "default,nightly"
          BUILDER_IMAGE: debian_stable_base
          RUNTIME_IMAGE: debian_stable_base
          ENV: ci
          arch: arm64
          runner: ubuntu-24.04-arm
          PLATFORM: linux/arm64
          target: final
        - BUILD_IMAGE: eic_
          BUILD_TYPE: "default,nightly"
          BUILDER_IMAGE: debian_stable_base
          RUNTIME_IMAGE: debian_stable_base
          ENV: xl
          arch: amd64
          runner: ubuntu-latest
          PLATFORM: linux/amd64
          target: final
        - BUILD_IMAGE: eic_
          BUILD_TYPE: "default,nightly"
          BUILDER_IMAGE: debian_stable_base
          RUNTIME_IMAGE: debian_stable_base
          ENV: xl
          arch: amd64_v3
          runner: ubuntu-latest
          PLATFORM: linux/amd64/v3
          target: final
        - BUILD_IMAGE: eic_
          BUILD_TYPE: "default,nightly"
          BUILDER_IMAGE: debian_stable_base
          RUNTIME_IMAGE: debian_stable_base
          ENV: xl
          arch: arm64
          runner: ubuntu-24.04-arm
          PLATFORM: linux/arm64
          target: final
        - BUILD_IMAGE: eic_
          BUILD_TYPE: "default"
          BUILDER_IMAGE: cuda_devel
          RUNTIME_IMAGE: cuda_devel
          ENV: cuda
          arch: amd64
          runner: ubuntu-latest
          PLATFORM: linux/amd64
          target: builder_concretization_default
        - BUILD_IMAGE: eic_
          BUILD_TYPE: "default"
          BUILDER_IMAGE: cuda_devel
          RUNTIME_IMAGE: cuda_runtime
          ENV: tf
          arch: amd64
          runner: ubuntu-latest
          PLATFORM: linux/amd64
          target: builder_concretization_default
      fail-fast: false
    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@v1.3.1
        with:
          android: true
          dotnet: true
      - name: Checkout
        uses: actions/checkout@v7
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v4
        with:
          platforms: linux/amd64,linux/amd64/v3,linux/arm64
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4
        id: buildx
      - name: Cache build mounts
        # Cache mount contents (ccache, apt cache, spack cache) using GitHub Actions cache
        # This is separate from the registry cache which stores image layers
        # Reference: https://docs.docker.com/build/ci/github-actions/cache/#cache-mounts
        # Implementation: https://github.com/reproducible-containers/buildkit-cache-dance
        uses: actions/cache@v6
        id: cache-eic-mounts
        with:
          path: cache-mount-eic-${{ matrix.arch }}
          key: ${{ matrix.arch }}-eic-${{ matrix.ENV }}-mounts-${{ github.ref_name }}
          restore-keys: |
            ${{ matrix.arch }}-eic-${{ matrix.ENV }}-mounts-main
            ${{ matrix.arch }}-eic-${{ matrix.ENV }}-mounts-
            ${{ matrix.arch }}-eic-mounts-
      - name: Inject cache mounts into builder
        uses: reproducible-containers/buildkit-cache-dance@v3
        with:
          builder: ${{ steps.buildx.outputs.name }}
          cache-dir: cache-mount-eic-${{ matrix.arch }}
          cache-map: |
            {
              "ccache": "/ccache",
              "var-cache-spack": "/var/cache/spack"
            }
          skip-extraction: ${{ steps.cache-eic-mounts.outputs.cache-hit }}
      - name: Login to Docker Hub
        uses: docker/login-action@v4
        if: ${{ env.DH_PUSH == '1' }}
        with:
          registry: ${{ env.DH_REGISTRY }}
          username: ${{ env.DH_REGISTRY_USER }}
          password: ${{ secrets.DH_EICWEB_TOKEN }}
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        if: ${{ env.GH_PUSH == '1' }}
        with:
          registry: ${{ env.GH_REGISTRY }}
          username: ${{ secrets.GHCR_REGISTRY_USER }}
          password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
      - name: Build and push
        env:
          BUILD_IMAGE: ${{ matrix.BUILD_IMAGE }}
          ENV: ${{ matrix.ENV }}
          BUILD_TYPE: ${{ matrix.BUILD_TYPE }}
          BUILD_TARGET: ${{ matrix.target }}
          BUILDER_IMAGE: ${{ matrix.BUILDER_IMAGE }}
          RUNTIME_IMAGE: ${{ matrix.RUNTIME_IMAGE }}
          PLATFORM: ${{ matrix.PLATFORM }}
          CI_REGISTRY_USER: ${{ secrets.GHCR_REGISTRY_USER }}
          CI_REGISTRY_PASSWORD: ${{ secrets.GHCR_REGISTRY_TOKEN }}
          GITHUB_REGISTRY_USER: ${{ secrets.GHCR_REGISTRY_USER }}
          GITHUB_REGISTRY_TOKEN: ${{ secrets.GHCR_REGISTRY_TOKEN }}
          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
          METADATA_FILE: /tmp/build-metadata.json
        run: bash scripts/build-eic.sh
      - name: Export digests to files
        run: |
          IMAGE="${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/${{ matrix.BUILD_IMAGE }}${{ matrix.ENV }}"
          mkdir -p /tmp/digests
          for metadata_file in /tmp/build-metadata-*.json; do
            build_type="${metadata_file#/tmp/build-metadata-}"
            build_type="${build_type%.json}"
            DIGEST=$(jq -r '."containerimage.digest"' "${metadata_file}")
            echo "${IMAGE}@${DIGEST}" > "/tmp/digests/${{ matrix.arch }}-${build_type}.digest"
          done
      - name: Check image size
        # Only check final images; concretization-only targets (cuda, tf) don't produce a full image
        if: ${{ matrix.target == 'final' }}
        run: |
          FULLNAME="${{ matrix.BUILD_IMAGE }}${{ matrix.ENV }}"
          LIMIT_VAR="SIZE_LIMIT_$(printf '%s' "${FULLNAME}" | tr '[:lower:]' '[:upper:]')_GIB"
          LIMIT="${!LIMIT_VAR}"
          if [ -n "${LIMIT}" ]; then
            metadata_file=/tmp/build-metadata-default.json
            [ -f "${metadata_file}" ] || metadata_file=$(ls /tmp/build-metadata-*.json | head -1)
            scripts/check_image_size \
              "${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/${FULLNAME}@$(jq -r '."containerimage.digest"' "${metadata_file}")" \
              "${LIMIT}"
          else
            echo "No size limit configured for ${FULLNAME}, skipping check."
          fi
      - name: Check produced build types
        id: built-types
        run: |
          [ -f /tmp/digests/${{ matrix.arch }}-default.digest ] \
            && echo "has_default=true" >> $GITHUB_OUTPUT \
            || echo "has_default=false" >> $GITHUB_OUTPUT
          [ -f /tmp/digests/${{ matrix.arch }}-nightly.digest ] \
            && echo "has_nightly=true" >> $GITHUB_OUTPUT \
            || echo "has_nightly=false" >> $GITHUB_OUTPUT
      - name: Upload default digest as artifact
        if: steps.built-types.outputs.has_default == 'true'
        uses: actions/upload-artifact@v7
        with:
          name: eic-${{ matrix.ENV }}-${{ matrix.arch }}-default-digest
          path: /tmp/digests/${{ matrix.arch }}-default.digest
          retention-days: 1
      - name: Upload nightly digest as artifact
        if: steps.built-types.outputs.has_nightly == 'true'
        uses: actions/upload-artifact@v7
        with:
          name: eic-${{ matrix.ENV }}-${{ matrix.arch }}-nightly-digest
          path: /tmp/digests/${{ matrix.arch }}-nightly.digest
          retention-days: 1

  eic-manifest:
    name: Push ${{ matrix.BUILD_IMAGE }}${{ matrix.ENV }} ${{ matrix.BUILD_TYPE }} manifest
    runs-on: ubuntu-latest
    needs: eic
    strategy:
      matrix:
        BUILD_IMAGE: [eic_]
        ENV: [ci, xl]
        BUILD_TYPE: [default, nightly]
    steps:
      - name: Set up QEMU (for imagetools)
        uses: docker/setup-qemu-action@v4
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4
      - name: Login to Docker Hub
        uses: docker/login-action@v4
        if: ${{ env.DH_PUSH == '1' }}
        with:
          registry: ${{ env.DH_REGISTRY }}
          username: ${{ env.DH_REGISTRY_USER }}
          password: ${{ secrets.DH_EICWEB_TOKEN }}
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        if: ${{ env.GH_PUSH == '1' }}
        with:
          registry: ${{ env.GH_REGISTRY }}
          username: ${{ secrets.GHCR_REGISTRY_USER }}
          password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
      - name: Download all digest artifacts
        uses: actions/download-artifact@v8
        with:
          path: /tmp/digests
          pattern: eic-${{ matrix.ENV }}-*-${{ matrix.BUILD_TYPE }}-digest
          merge-multiple: true
      - name: Analyze digest artifacts
        id: digests
        run: |
          # Loop over all available architecture artifacts
          ALL_DIGESTS=""
          FIRST_DIGEST=""
          shopt -s nullglob
          for digest_file in /tmp/digests/*-${{ matrix.BUILD_TYPE }}.digest; do
            DIGEST=$(cat "$digest_file")
            echo "Found digest: $DIGEST"
            if [ -z "$ALL_DIGESTS" ]; then
              ALL_DIGESTS="$DIGEST"
            else
              ALL_DIGESTS="$ALL_DIGESTS $DIGEST"
            fi
            if [ -z "$FIRST_DIGEST" ]; then
              FIRST_DIGEST="$DIGEST"
            fi
          done
          shopt -u nullglob
          # Check if at least one digest was found
          if [ -z "$FIRST_DIGEST" ]; then
            echo "Error: No digest files found"
            exit 1
          fi
          # Get the base image name from the first digest
          REGISTRY_IMAGE=$(echo $FIRST_DIGEST | cut -d'@' -f1)
          REGISTRY=$(echo $REGISTRY_IMAGE | cut -d'/' -f1-2)
          IMAGE=$(echo $REGISTRY_IMAGE | cut -d'/' -f3)
          echo "Registry Name: $REGISTRY"
          echo "Image Name: $IMAGE"
          echo "All Digests: $ALL_DIGESTS"
          echo "registry=$REGISTRY" >> $GITHUB_OUTPUT
          echo "image=$IMAGE" >> $GITHUB_OUTPUT
          echo "all=$ALL_DIGESTS" >> $GITHUB_OUTPUT
      - name: Extract Docker metadata for final tags
        id: meta
        uses: docker/metadata-action@v6
        with:
          images: |
            name=${{ env.DH_REGISTRY }}/${{ env.DH_REGISTRY_USER }}/${{ steps.digests.outputs.image }},enable=${{ env.DH_PUSH != 0 }}
            name=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/${{ steps.digests.outputs.image }},enable=${{ env.GH_PUSH != 0 }}
          tags: |
            ${{ env.INTERNAL_TAG }}-${{ matrix.BUILD_TYPE }}
            type=ref,prefix=unstable-pr-,event=pr,suffix=-${{ matrix.BUILD_TYPE }}
            type=match,pattern=^v(\d+\.\d+\.\d+-.*)$,group=1,enable=${{ matrix.BUILD_TYPE == 'default' }}
            type=match,pattern=^v(\d+\.\d+\.\d+-.*)$,group=1,suffix=-nightly,enable=${{ matrix.BUILD_TYPE == 'nightly' }}
      - name: Create and push manifest list
        run: |
          # Create tag arguments from the multi-line 'tags' output from the meta step
          TAG_ARGS=""
          while read -r line; do
            TAG_ARGS="$TAG_ARGS --tag $line"
          done <<< "${{ steps.meta.outputs.tags }}"
          # Create the manifest list and tag it with the final tags
          docker buildx imagetools create \
            $TAG_ARGS \
            ${{ steps.digests.outputs.all }}

  npsim-gun:
    name: npsim (gun, ${{ matrix.particle }}, ${{ matrix.detector_config }}, ${{ matrix.arch }})
    runs-on: ${{ matrix.runner }}
    needs: eic-manifest
    container:
      image: ghcr.io/eic/eic_ci:pipeline-${{ github.run_id }}
      credentials:
        username: ${{ secrets.GHCR_REGISTRY_USER }}
        password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
    strategy:
      fail-fast: false
      matrix:
        particle: [pi, e]
        detector_config: [epic_craterlake]
        arch: [amd64, arm64]
        include:
        - arch: amd64
          runner: ubuntu-latest
        - arch: arm64
          runner: ubuntu-24.04-arm
    steps:
    - name: Produce simulation files
      shell: bash --login -eo pipefail {0}
      run: |
        source /opt/detector/epic-main/bin/thisepic.sh
        npsim \
          --compactFile ${DETECTOR_PATH}/${{ matrix.detector_config }}.xml \
          -G --random.seed 1 \
          --random.enableEventSeed \
          --gun.particle "${{ matrix.particle }}-" \
          --gun.momentumMin "1*GeV" --gun.momentumMax "20*GeV" \
          --gun.distribution "uniform" \
          -N 100 \
          --outputFile sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4hep.root \
          -v WARNING
    - id: upload_new_artifact
      uses: actions/upload-artifact@v7
      with:
        name: sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}.edm4hep.root
        path: sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4hep.root
        if-no-files-found: error
    - name: Download previous artifact
      id: download_previous_artifact
      uses: dawidd6/action-download-artifact@v21
      with:
        branch: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref_name }}
        path: ref/
        name: sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}.edm4hep.root
        workflow: ".github/workflows/build-push.yml"
        workflow_conclusion: "success"
        if_no_artifact_found: warn
    - name: Compare to previous artifacts
      shell: bash --login -eo pipefail {0}
      run: |
        mkdir capybara-reports
        shopt -s nullglob
        capybara bara ref/sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4hep.root* sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4hep.root
        mv capybara-reports sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}
        touch .sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}
    - name: Write artifact metadata
      env:
        NEW_URL: ${{ steps.upload_new_artifact.outputs.artifact-url }}
        FOUND_ARTIFACT: ${{ steps.download_previous_artifact.outputs.found_artifact }}
        REF_ARTIFACTS: ${{ steps.download_previous_artifact.outputs.artifacts }}
        REPOSITORY: ${{ github.repository }}
      run: |
        REF_URL=""
        if [ "$FOUND_ARTIFACT" = "true" ]; then
          REF_ART_ID=$(echo "$REF_ARTIFACTS" | jq -r '.[0].id // empty')
          REF_RUN_ID=$(echo "$REF_ARTIFACTS" | jq -r '.[0].workflow_run.id // empty')
          if [ -n "$REF_ART_ID" ] && [ -n "$REF_RUN_ID" ]; then
            REF_URL="https://github.com/$REPOSITORY/actions/runs/${REF_RUN_ID}/artifacts/${REF_ART_ID}"
          fi
        fi
        jq -nc --arg new "$NEW_URL" --arg ref "$REF_URL" '{new: $new, ref: $ref}' \
          > sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}/meta.json
    - uses: actions/upload-artifact@v7
      with:
        name: sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}.capy
        path: |
          .sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}
          sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}/
        if-no-files-found: error

  npsim-dis:
    name: npsim (DIS, ${{ matrix.beam }}, minQ2=${{ matrix.minq2 }}, ${{ matrix.detector_config }}, ${{ matrix.arch }})
    runs-on: ${{ matrix.runner }}
    needs: eic-manifest
    container:
      image: ghcr.io/eic/eic_ci:pipeline-${{ github.run_id }}
      credentials:
        username: ${{ secrets.GHCR_REGISTRY_USER }}
        password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
    strategy:
      fail-fast: false
      matrix:
        beam: [5x41, 10x100, 18x275]
        minq2: [1, 1000]
        detector_config: [epic_craterlake]
        arch: [amd64, arm64]
        include:
        - arch: amd64
          runner: ubuntu-latest
        - arch: arm64
          runner: ubuntu-24.04-arm
        exclude:
        - beam: 5x41
          minq2: 1000
        - beam: 10x100
          minq2: 1
          arch: arm64
        - beam: 10x100
          minq2: 1000
          arch: arm64
        - beam: 18x275
          minq2: 1
          arch: arm64
        - beam: 18x275
          minq2: 1000
          arch: arm64
    steps:
    - name: Produce simulation files
      shell: bash --login -eo pipefail {0}
      run: |
        source /opt/detector/epic-main/bin/thisepic.sh
        url=root://dtn-eic.jlab.org//volatile/eic/EPIC/EVGEN/DIS/NC/${{ matrix.beam }}/minQ2=${{ matrix.minq2 }}/pythia8NCDIS_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_beamEffects_xAngle=-0.025_hiDiv_1.hepmc3.tree.root
        npsim \
          --compactFile ${DETECTOR_PATH}/${{ matrix.detector_config }}.xml \
          -N 100 --inputFiles ${url} --random.seed 1 \
          --outputFile sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}.edm4hep.root \
          -v WARNING
    - id: upload_new_artifact
      uses: actions/upload-artifact@v7
      with:
        name: sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}.edm4hep.root
        path: sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}.edm4hep.root
        if-no-files-found: error
    - name: Download previous artifact
      id: download_previous_artifact
      uses: dawidd6/action-download-artifact@v21
      with:
        branch: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref_name }}
        path: ref/
        name: sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}.edm4hep.root
        workflow: ".github/workflows/build-push.yml"
        workflow_conclusion: "success"
        if_no_artifact_found: warn
    - name: Compare to previous artifacts
      shell: bash --login -eo pipefail {0}
      run: |
        mkdir capybara-reports
        shopt -s nullglob
        capybara bara ref/sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}.edm4hep.root* sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}.edm4hep.root
        mv capybara-reports sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}
        touch .sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}
    - name: Write artifact metadata
      env:
        NEW_URL: ${{ steps.upload_new_artifact.outputs.artifact-url }}
        FOUND_ARTIFACT: ${{ steps.download_previous_artifact.outputs.found_artifact }}
        REF_ARTIFACTS: ${{ steps.download_previous_artifact.outputs.artifacts }}
        GITHUB_REPOSITORY: ${{ github.repository }}
      run: |
        REF_URL=""
        if [ "$FOUND_ARTIFACT" = "true" ]; then
          REF_ART_ID=$(echo "$REF_ARTIFACTS" | jq -r '.[0].id // empty')
          REF_RUN_ID=$(echo "$REF_ARTIFACTS" | jq -r '.[0].workflow_run.id // empty')
          if [ -n "$REF_ART_ID" ] && [ -n "$REF_RUN_ID" ]; then
            REF_URL="https://github.com/$GITHUB_REPOSITORY/actions/runs/${REF_RUN_ID}/artifacts/${REF_ART_ID}"
          fi
        fi
        jq -nc --arg new "$NEW_URL" --arg ref "$REF_URL" '{new: $new, ref: $ref}' \
          > sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}/meta.json
    - uses: actions/upload-artifact@v7
      with:
        name: sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}.capy
        path: |
          .sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}
          sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}/
        if-no-files-found: error

  eicrecon-gun:
    name: eicrecon (gun, ${{ matrix.particle }}, ${{ matrix.detector_config }}, ${{ matrix.arch }})
    runs-on: ${{ matrix.runner }}
    needs: [eic-manifest, npsim-gun]
    container:
      image: ghcr.io/eic/eic_ci:pipeline-${{ github.run_id }}
      credentials:
        username: ${{ secrets.GHCR_REGISTRY_USER }}
        password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
    strategy:
      fail-fast: false
      matrix:
        particle: [pi, e]
        detector_config: [epic_craterlake]
        arch: [amd64, arm64]
        include:
        - arch: amd64
          runner: ubuntu-latest
        - arch: arm64
          runner: ubuntu-24.04-arm
    steps:
    - uses: actions/download-artifact@v8
      with:
        name: sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}.edm4hep.root
    - name: Run EICrecon
      shell: bash --login -eo pipefail {0}
      run: |
        source /opt/detector/epic-main/bin/thisepic.sh
        export DETECTOR_CONFIG=${{ matrix.detector_config }}
        eicrecon \
          -Ppodio:output_file=rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4eic.root \
          sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4hep.root
    - id: upload_new_artifact
      uses: actions/upload-artifact@v7
      with:
        name: rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}.edm4eic.root
        path: rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4eic.root
        if-no-files-found: error
    - name: Download previous artifact
      id: download_previous_artifact
      uses: dawidd6/action-download-artifact@v21
      with:
        branch: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref_name }}
        path: ref/
        name: rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}.edm4eic.root
        workflow: ".github/workflows/build-push.yml"
        workflow_conclusion: "success"
        if_no_artifact_found: warn
    - name: Compare to previous artifacts
      shell: bash --login -eo pipefail {0}
      run: |
        mkdir capybara-reports
        mkdir new
        ln -sf ../rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4eic.root new/
        shopt -s nullglob
        capybara bara ref/rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4eic.root* new/rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4eic.root
        mv capybara-reports rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}
        touch .rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}
    - name: Write artifact metadata
      env:
        NEW_URL: ${{ steps.upload_new_artifact.outputs.artifact-url }}
        FOUND_ARTIFACT: ${{ steps.download_previous_artifact.outputs.found_artifact }}
        REF_ARTIFACTS_JSON: ${{ steps.download_previous_artifact.outputs.artifacts }}
        GITHUB_REPOSITORY_NAME: ${{ github.repository }}
      run: |
        REF_URL=""
        if [ "$FOUND_ARTIFACT" = "true" ]; then
          REF_ART_ID=$(echo "$REF_ARTIFACTS_JSON" | jq -r '.[0].id // empty')
          REF_RUN_ID=$(echo "$REF_ARTIFACTS_JSON" | jq -r '.[0].workflow_run.id // empty')
          if [ -n "$REF_ART_ID" ] && [ -n "$REF_RUN_ID" ]; then
            REF_URL="https://github.com/${GITHUB_REPOSITORY_NAME}/actions/runs/${REF_RUN_ID}/artifacts/${REF_ART_ID}"
          fi
        fi
        jq -nc --arg new "$NEW_URL" --arg ref "$REF_URL" '{new: $new, ref: $ref}' \
          > rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}/meta.json
    - uses: actions/upload-artifact@v7
      with:
        name: rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}.capy
        path: |
          .rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}
          rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.arch }}/
        if-no-files-found: error

  eicrecon-dis:
    name: eicrecon (DIS, ${{ matrix.beam }}, minQ2=${{ matrix.minq2 }}, ${{ matrix.detector_config }}, ${{ matrix.arch }})
    runs-on: ${{ matrix.runner }}
    needs: [eic-manifest, npsim-dis]
    container:
      image: ghcr.io/eic/eic_ci:pipeline-${{ github.run_id }}
      credentials:
        username: ${{ secrets.GHCR_REGISTRY_USER }}
        password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
    strategy:
      fail-fast: false
      matrix:
        beam: [5x41, 10x100, 18x275]
        minq2: [1, 1000]
        detector_config: [epic_craterlake]
        arch: [amd64, arm64]
        include:
        - arch: amd64
          runner: ubuntu-latest
        - arch: arm64
          runner: ubuntu-24.04-arm
        exclude:
        - beam: 5x41
          minq2: 1000
        - beam: 10x100
          minq2: 1
          arch: arm64
        - beam: 10x100
          minq2: 1000
          arch: arm64
        - beam: 18x275
          minq2: 1
          arch: arm64
        - beam: 18x275
          minq2: 1000
          arch: arm64
    steps:
    - uses: actions/download-artifact@v8
      with:
        name: sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}.edm4hep.root
    - name: Run EICrecon
      shell: bash --login -eo pipefail {0}
      run: |
        source /opt/detector/epic-main/bin/thisepic.sh
        export DETECTOR_CONFIG=${{ matrix.detector_config }}
        eicrecon \
          -Ppodio:output_file=rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}.edm4eic.root \
          sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}.edm4hep.root
    - id: upload_new_artifact
      uses: actions/upload-artifact@v7
      with:
        name: rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}.edm4eic.root
        path: rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}.edm4eic.root
        if-no-files-found: error
    - name: Download previous artifact
      id: download_previous_artifact
      uses: dawidd6/action-download-artifact@v21
      with:
        branch: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref_name }}
        path: ref/
        name: rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}.edm4eic.root
        workflow: ".github/workflows/build-push.yml"
        workflow_conclusion: "success"
        if_no_artifact_found: warn
    - name: Compare to previous artifacts
      shell: bash --login -eo pipefail {0}
      run: |
        mkdir capybara-reports
        mkdir new
        ln -sf ../rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}.edm4eic.root new/
        shopt -s nullglob
        capybara bara ref/rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}.edm4eic.root* new/rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}.edm4eic.root
        mv capybara-reports rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}
        touch .rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}
    - name: Write artifact metadata
      env:
        NEW_URL: ${{ steps.upload_new_artifact.outputs.artifact-url }}
        FOUND_ARTIFACT: ${{ steps.download_previous_artifact.outputs.found_artifact }}
        REF_ARTIFACTS_JSON: ${{ steps.download_previous_artifact.outputs.artifacts }}
        GITHUB_REPOSITORY: ${{ github.repository }}
      run: |
        REF_URL=""
        if [ "$FOUND_ARTIFACT" = "true" ]; then
          REF_ART_ID=$(echo "$REF_ARTIFACTS_JSON" | jq -r '.[0].id // empty')
          REF_RUN_ID=$(echo "$REF_ARTIFACTS_JSON" | jq -r '.[0].workflow_run.id // empty')
          if [ -n "$REF_ART_ID" ] && [ -n "$REF_RUN_ID" ]; then
            REF_URL="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${REF_RUN_ID}/artifacts/${REF_ART_ID}"
          fi
        fi
        jq -nc --arg new "$NEW_URL" --arg ref "$REF_URL" '{new: $new, ref: $ref}' \
          > rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}/meta.json
    - uses: actions/upload-artifact@v7
      with:
        name: rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}.capy
        path: |
          .rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}
          rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_${{ matrix.arch }}/
        if-no-files-found: error

  npsim-dis-cross-arch:
    name: capybara (npsim DIS, ${{ matrix.beam }}, minQ2=${{ matrix.minq2 }}, ${{ matrix.detector_config }}, amd64 vs arm64)
    runs-on: ubuntu-latest
    needs: [eic-manifest, npsim-dis]
    container:
      image: ghcr.io/eic/eic_ci:pipeline-${{ github.run_id }}
      credentials:
        username: ${{ secrets.GHCR_REGISTRY_USER }}
        password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
    strategy:
      fail-fast: false
      matrix:
        beam: [5x41]
        minq2: [1]
        detector_config: [epic_craterlake]
    steps:
    - name: Download amd64 artifact
      uses: actions/download-artifact@v8
      with:
        name: sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_amd64.edm4hep.root
        path: amd64/
    - name: Download arm64 artifact
      uses: actions/download-artifact@v8
      with:
        name: sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_arm64.edm4hep.root
        path: arm64/
    - name: Compare amd64 vs arm64 artifacts
      shell: bash --login -eo pipefail {0}
      run: |
        mkdir capybara-reports
        capybara bara amd64/sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}.edm4hep.root arm64/sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}.edm4hep.root
        mv capybara-reports sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_amd64_vs_arm64
        touch .sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_amd64_vs_arm64
    - uses: actions/upload-artifact@v7
      with:
        name: sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_amd64_vs_arm64.capy
        path: |
          .sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_amd64_vs_arm64
          sim_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_amd64_vs_arm64/
        if-no-files-found: error

  eicrecon-dis-cross-arch:
    name: capybara (eicrecon DIS, ${{ matrix.beam }}, minQ2=${{ matrix.minq2 }}, ${{ matrix.detector_config }}, amd64 vs arm64)
    runs-on: ubuntu-latest
    needs: [eic-manifest, eicrecon-dis]
    container:
      image: ghcr.io/eic/eic_ci:pipeline-${{ github.run_id }}
      credentials:
        username: ${{ secrets.GHCR_REGISTRY_USER }}
        password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
    strategy:
      fail-fast: false
      matrix:
        beam: [5x41]
        minq2: [1]
        detector_config: [epic_craterlake]
    steps:
    - name: Download amd64 artifact
      uses: actions/download-artifact@v8
      with:
        name: rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_amd64.edm4eic.root
        path: amd64/
    - name: Download arm64 artifact
      uses: actions/download-artifact@v8
      with:
        name: rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_arm64.edm4eic.root
        path: arm64/
    - name: Compare amd64 vs arm64 artifacts
      shell: bash --login -eo pipefail {0}
      run: |
        mkdir capybara-reports
        capybara bara amd64/rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}.edm4eic.root arm64/rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}.edm4eic.root
        mv capybara-reports rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_amd64_vs_arm64
        touch .rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_amd64_vs_arm64
    - uses: actions/upload-artifact@v7
      with:
        name: rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_amd64_vs_arm64.capy
        path: |
          .rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_amd64_vs_arm64
          rec_dis_${{ matrix.beam }}_minQ2=${{ matrix.minq2 }}_${{ matrix.detector_config }}_amd64_vs_arm64/
        if-no-files-found: error

  npsim-gun-cross-arch:
    name: capybara (npsim gun, ${{ matrix.particle }}, ${{ matrix.detector_config }}, amd64 vs arm64)
    runs-on: ubuntu-latest
    needs: [eic-manifest, npsim-gun]
    container:
      image: ghcr.io/eic/eic_ci:pipeline-${{ github.run_id }}
      credentials:
        username: ${{ secrets.GHCR_REGISTRY_USER }}
        password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
    strategy:
      fail-fast: false
      matrix:
        particle: [pi, e]
        detector_config: [epic_craterlake]
    steps:
    - name: Download amd64 artifact
      uses: actions/download-artifact@v8
      with:
        name: sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_amd64.edm4hep.root
        path: amd64/
    - name: Download arm64 artifact
      uses: actions/download-artifact@v8
      with:
        name: sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_arm64.edm4hep.root
        path: arm64/
    - name: Compare amd64 vs arm64 artifacts
      shell: bash --login -eo pipefail {0}
      run: |
        mkdir capybara-reports
        capybara bara amd64/sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4hep.root arm64/sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4hep.root
        mv capybara-reports sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_amd64_vs_arm64
        touch .sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_amd64_vs_arm64
    - uses: actions/upload-artifact@v7
      with:
        name: sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_amd64_vs_arm64.capy
        path: |
          .sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_amd64_vs_arm64
          sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_amd64_vs_arm64/
        if-no-files-found: error

  eicrecon-gun-cross-arch:
    name: capybara (eicrecon gun, ${{ matrix.particle }}, ${{ matrix.detector_config }}, amd64 vs arm64)
    runs-on: ubuntu-latest
    needs: [eic-manifest, eicrecon-gun]
    container:
      image: ghcr.io/eic/eic_ci:pipeline-${{ github.run_id }}
      credentials:
        username: ${{ secrets.GHCR_REGISTRY_USER }}
        password: ${{ secrets.GHCR_REGISTRY_TOKEN }}
    strategy:
      fail-fast: false
      matrix:
        particle: [pi, e]
        detector_config: [epic_craterlake]
    steps:
    - name: Download amd64 artifact
      uses: actions/download-artifact@v8
      with:
        name: rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_amd64.edm4eic.root
        path: amd64/
    - name: Download arm64 artifact
      uses: actions/download-artifact@v8
      with:
        name: rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_arm64.edm4eic.root
        path: arm64/
    - name: Compare amd64 vs arm64 artifacts
      shell: bash --login -eo pipefail {0}
      run: |
        mkdir capybara-reports
        capybara bara amd64/rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4eic.root arm64/rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4eic.root
        mv capybara-reports rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_amd64_vs_arm64
        touch .rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_amd64_vs_arm64
    - uses: actions/upload-artifact@v7
      with:
        name: rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_amd64_vs_arm64.capy
        path: |
          .rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_amd64_vs_arm64
          rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_amd64_vs_arm64/
        if-no-files-found: error

  merge-capybara:
    name: Merge capybara reports
    runs-on: ubuntu-latest
    needs: [npsim-gun, npsim-gun-cross-arch, npsim-dis, npsim-dis-cross-arch, eicrecon-gun, eicrecon-gun-cross-arch, eicrecon-dis, eicrecon-dis-cross-arch]
    steps:
    - uses: actions/upload-artifact/merge@v7
      with:
        name: capybara-report
        pattern: "*.capy"
        delete-merged: true

  list-open-prs:
    uses: eic/actions/.github/workflows/list-open-prs.yml@main

  get-docs-from-open-prs:
    runs-on: ubuntu-latest
    needs:
    - list-open-prs
    - merge-capybara
    if: always() && !cancelled()
    strategy:
      matrix: ${{ fromJSON(needs.list-open-prs.outputs.json) }}
      fail-fast: false
      max-parallel: 4
    steps:
    - name: Download capybara artifact (other PRs)
      id: download_capybara
      uses: dawidd6/action-download-artifact@v21
      if: github.event.pull_request.number != matrix.pr
      with:
        commit: ${{ matrix.head_sha }}
        path: publishing_docs/pr/${{ matrix.pr }}/capybara/
        name: capybara-report
        workflow: ".github/workflows/build-push.yml"
        workflow_conclusion: "completed"
        if_no_artifact_found: ignore
    - name: Download capybara artifact (this PR)
      uses: actions/download-artifact@v8
      if: github.event.pull_request.number == matrix.pr
      with:
        name: capybara-report
        path: publishing_docs/pr/${{ matrix.pr }}/capybara/
    - uses: actions/upload-artifact@v7
      with:
        name: github-pages-staging-pr-${{ matrix.pr }}
        path: publishing_docs/
        if-no-files-found: ignore

  get-docs-from-main:
    runs-on: ubuntu-latest
    needs:
    - merge-capybara
    if: always() && !cancelled()
    steps:
    - name: Download capybara artifact (in PR)
      id: download_capybara
      uses: dawidd6/action-download-artifact@v21
      if: github.ref_name != 'master'
      with:
        branch: master
        path: publishing_docs/capybara/
        name: capybara-report
        workflow: ".github/workflows/build-push.yml"
        workflow_conclusion: "completed"
        if_no_artifact_found: ignore
    - name: Download capybara artifact (on master)
      uses: actions/download-artifact@v8
      if: github.ref_name == 'master'
      with:
        name: capybara-report
        path: publishing_docs/capybara/
    - name: Populate capybara summary (on master)
      if: github.ref_name == 'master' || steps.download_capybara.outputs.found_artifact == 'true'
      run: |
        echo "### Capybara summary for master" >> publishing_docs/capybara/index.md
        while IFS= read -r dir; do
          name=$(basename "$dir")
          line="- [${name}](https://eic.github.io/containers/capybara/${name}/index.html)"
          if [ -f "${dir}/meta.json" ]; then
            new_url=$(jq -r '.new // empty' "${dir}/meta.json" 2>/dev/null || true)
            ref_url=$(jq -r '.ref // empty' "${dir}/meta.json" 2>/dev/null || true)
            [ -n "$new_url" ] && line="${line} ([new artifact](${new_url}))"
            [ -n "$ref_url" ] && line="${line} ([ref artifact](${ref_url}))"
          fi
          echo "$line"
        done < <(find publishing_docs/capybara/ -mindepth 1 -maxdepth 1 -type d | sort) \
          >> publishing_docs/capybara/index.md
    - name: Create capybara step summary (on master)
      if: github.ref_name == 'master'
      run: |
        cat publishing_docs/capybara/index.md >> $GITHUB_STEP_SUMMARY
    - name: Ensure publishing_docs is non-empty
      run: mkdir -p publishing_docs && touch publishing_docs/.keep
    - uses: actions/upload-artifact@v7
      with:
        name: github-pages-staging-main
        path: publishing_docs/
        if-no-files-found: error
        include-hidden-files: true

  build-artifacts-page:
    runs-on: ubuntu-latest
    needs:
    - get-docs-from-open-prs
    - get-docs-from-main
    if: always() && !cancelled() && !failure()
    steps:
    - uses: actions/checkout@v7
    - name: Download artifacts for pages
      uses: actions/download-artifact@v8
      with:
        path: artifacts/
    - name: Assemble site
      run: |
        mkdir -p publishing_docs
        cp -r docs/. publishing_docs/
        for dir in artifacts/github-pages-staging-*/; do
          [ -d "$dir" ] && cp -r "$dir"/. publishing_docs/ || true
        done
    - name: Make PR summary page
      run: |
        mkdir -p publishing_docs/pr
        echo "### PRs" >> publishing_docs/pr/index.md
        find publishing_docs/pr -mindepth 1 -maxdepth 1 -type d \
          -printf "- [%f](%f/index.md)\n" | sort >> publishing_docs/pr/index.md
    - name: List content to publish
      run: find publishing_docs/
    - uses: actions/upload-pages-artifact@v5
      with:
        path: publishing_docs/
        retention-days: 7
    - name: Populate capybara summary
      if: ${{ github.event_name == 'pull_request' }}
      run: |
        echo "### Capybara summary for PR ${{ github.event.pull_request.number }}" \
          >> capybara_${{ github.event.pull_request.number }}.md
        CAPY_DIR="publishing_docs/pr/${{ github.event.pull_request.number }}/capybara"
        if [ -d "$CAPY_DIR" ]; then
          while IFS= read -r dir; do
            name=$(basename "$dir")
            line="- [${name}](https://eic.github.io/containers/pr/${{ github.event.pull_request.number }}/capybara/${name}/index.html)"
            if [ -f "${dir}/meta.json" ]; then
              new_url=$(jq -r '.new // empty' "${dir}/meta.json" 2>/dev/null || true)
              ref_url=$(jq -r '.ref // empty' "${dir}/meta.json" 2>/dev/null || true)
              [ -n "$new_url" ] && line="${line} ([new artifact](${new_url}))"
              [ -n "$ref_url" ] && line="${line} ([ref artifact](${ref_url}))"
            fi
            echo "$line"
          done < <(find "$CAPY_DIR" -mindepth 1 -maxdepth 1 -type d | sort) \
            >> capybara_${{ github.event.pull_request.number }}.md
        fi
        echo "<sup><sub>Last updated $(TZ=America/New_York date -Iminutes) ${{ github.event.pull_request.head.sha }}</sub></sup>" \
          >> capybara_${{ github.event.pull_request.number }}.md
    - name: Create capybara step summary
      if: ${{ github.event_name == 'pull_request' }}
      run: cat capybara_${{ github.event.pull_request.number }}.md >> $GITHUB_STEP_SUMMARY
    - name: Upload capybara summary
      if: ${{ github.event_name == 'pull_request' }}
      uses: actions/upload-artifact@v7
      with:
        name: capybara_${{ github.event.pull_request.number }}.md
        path: capybara_${{ github.event.pull_request.number }}.md

  deploy-artifacts-page:
    needs:
    - build-artifacts-page
    if: always() && !cancelled() && !failure()
    concurrency:
      group: pages
      cancel-in-progress: false
    permissions:
      pages: write
      id-token: write
      pull-requests: write
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    steps:
    - name: Deploy to GitHub Pages
      id: deployment
      continue-on-error: true
      uses: actions/deploy-pages@v5
    - name: Find capybara comment
      if: ${{ github.event_name == 'pull_request' }}
      id: find_comment
      uses: peter-evans/find-comment@v4
      with:
        issue-number: ${{ github.event.pull_request.number }}
        comment-author: 'github-actions[bot]'
        body-includes: Capybara summary
    - name: Fetch capybara summary
      if: ${{ github.event_name == 'pull_request' }}
      uses: actions/download-artifact@v8
      with:
        name: capybara_${{ github.event.pull_request.number }}.md
    - name: Create or update capybara comment
      if: ${{ github.event_name == 'pull_request' }}
      uses: peter-evans/create-or-update-comment@v5
      with:
        comment-id: ${{ steps.find_comment.outputs.comment-id }}
        issue-number: ${{ github.event.pull_request.number }}
        body-file: capybara_${{ github.event.pull_request.number }}.md
        edit-mode: replace