fix(backport-test): opt-out memory test #30
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| name: Required Checks | |
| on: | |
| push: | |
| branches: | |
| - 'ci-enable/**' | |
| - 'main' | |
| - 'release/**' | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| - labeled | |
| - unlabeled | |
| merge_group: | |
| workflow_dispatch: | |
| permissions: | |
| checks: write | |
| contents: read | |
| pull-requests: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/heads/release/') }} | |
| jobs: | |
| # Precheck decides which downstream jobs run for this event: | |
| # - On PR events, wait for the Pull Request Labeler workflow to finish so | |
| # the labels it applies (frontend, docs, infra, ...) are available, then | |
| # gate run_* outputs on those labels. | |
| # - run_frontend / run_amber / run_platform / run_pyamber / | |
| # run_agent_service: gate the main build stacks. Each labeler-applied | |
| # label maps to the stacks it requires (LABEL_STACKS below); the run | |
| # set is the union across all PR labels. Empty union (e.g. docs-only | |
| # PRs) skips every stack. Push and workflow_dispatch | |
| # events run every stack. | |
| # - backport_targets: JSON array of release/* labels currently on the PR. | |
| # Drives the backport matrix; empty array means no backport runs. | |
| precheck: | |
| name: Precheck | |
| runs-on: ubuntu-latest | |
| outputs: | |
| run_frontend: ${{ steps.decide.outputs.run_frontend }} | |
| run_amber: ${{ steps.decide.outputs.run_amber }} | |
| run_amber_integration: ${{ steps.decide.outputs.run_amber_integration }} | |
| run_platform: ${{ steps.decide.outputs.run_platform }} | |
| run_platform_integration: ${{ steps.decide.outputs.run_platform_integration }} | |
| run_pyamber: ${{ steps.decide.outputs.run_pyamber }} | |
| run_agent_service: ${{ steps.decide.outputs.run_agent_service }} | |
| run_infra: ${{ steps.decide.outputs.run_infra }} | |
| run_pyright_language_service: ${{ steps.decide.outputs.run_pyright_language_service }} | |
| backport_targets: ${{ steps.decide.outputs.backport_targets }} | |
| steps: | |
| - name: Wait for Pull Request Labeler | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v9 | |
| with: | |
| script: | | |
| const ref = context.payload.pull_request.head.sha; | |
| const maxAttempts = 30; | |
| for (let i = 0; i < maxAttempts; i++) { | |
| const { data } = await github.rest.checks.listForRef({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| ref, | |
| check_name: "labeler", | |
| }); | |
| const check = data.check_runs[0]; | |
| if (check && check.status === "completed") { | |
| core.info(`labeler ${check.conclusion}`); | |
| return; | |
| } | |
| core.info(`labeler not ready (attempt ${i + 1}/${maxAttempts})`); | |
| await new Promise((r) => setTimeout(r, 10000)); | |
| } | |
| core.warning("labeler did not complete within 5 minutes; proceeding with current labels."); | |
| - name: Decide which jobs to run | |
| id: decide | |
| uses: actions/github-script@v9 | |
| with: | |
| script: | | |
| const eventName = context.eventName; | |
| let labels = []; | |
| if (eventName === "pull_request") { | |
| // Re-fetch labels: the labeler may have just added some. | |
| const { data: pr } = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: context.payload.pull_request.number, | |
| }); | |
| labels = pr.labels.map((l) => l.name); | |
| core.info(`PR labels: ${labels.join(", ") || "(none)"}`); | |
| } | |
| // Map each labeler-applied label to the stacks it requires. The | |
| // run set is the union across all PR labels. Labels not listed | |
| // here (release/*, feature, fix, refactor) contribute no stacks. | |
| // dependencies is intentionally a no-op: every dep manifest the | |
| // labeler matches lives under a component dir and is already | |
| // covered by that component's label — including | |
| // pyright-language-service/package.json, kept honest by the | |
| // `pyright-language-service` component label below. | |
| // | |
| // `infra` and `pyright-language-service` are 1:1 self-mapping | |
| // labels (each fires only its own like-named stack); they are | |
| // omitted from the illustrative table below to keep it narrow. | |
| // | |
| // `platform-integration` mirrors `amber-integration`: a dedicated | |
| // `platform-integration` label fires only the integration stack | |
| // (a spec-only change) without the full `platform` job. It is | |
| // omitted from the table below to keep it narrow. No labeler rule | |
| // applies it yet — platform has no integration specs (#6047) — so | |
| // today it is a forward hook (applied manually until then). | |
| // | |
| // label | frontend | amber | amber-integ | platform | pyamber | agent-service | |
| // ------------------|----------|-------|-------------|----------|---------|-------------- | |
| // frontend | x | | | | | | |
| // pyamber | | | x | | x | | |
| // engine | | x | x | | | | |
| // amber-integration | | | x | | | | |
| // platform | | | | x | | | |
| // agent-service | | | | | | x | |
| // common | | x | x | x | | (root | |
| // scala | |
| // build/lint | |
| // config) | |
| // ddl-change | | x | x | x | | | |
| // ci | x | x | x | x | x | x | |
| // docs / deps / | | | | | | | |
| // release/* | | | | | | | |
| // * / branch | | | | | | | |
| // | |
| // amber-integration runs the Scala tests tagged | |
| // @org.apache.texera.amber.tags.IntegrationTest (e2e specs that | |
| // spawn Python UDF workers). The labeler attaches `pyamber` to | |
| // any *.py change (including amber/src/{main,test}/python/**), | |
| // so `engine` does not need to fire the pyamber stack itself — | |
| // pure-Python amber changes pick up `pyamber` directly. The | |
| // `amber-integration` label catches *IntegrationSpec.scala | |
| // edits so a test-only change does not trigger the full | |
| // Scala-only amber stack. | |
| const LABEL_STACKS = { | |
| frontend: ["frontend"], | |
| pyamber: ["amber-integration", "pyamber"], | |
| engine: ["amber", "amber-integration"], | |
| "amber-integration": ["amber-integration"], | |
| platform: ["platform", "platform-integration"], | |
| "platform-integration": ["platform-integration"], | |
| "agent-service": ["agent-service"], | |
| "pyright-language-service": ["pyright-language-service"], | |
| common: ["amber", "amber-integration", "platform", "platform-integration"], | |
| "ddl-change": ["amber", "amber-integration", "platform", "platform-integration"], | |
| infra: ["infra"], | |
| ci: ["frontend", "amber", "amber-integration", "platform", "platform-integration", "pyamber", "agent-service", "infra", "pyright-language-service"], | |
| }; | |
| let runFrontend = true; | |
| let runAmber = true; | |
| let runAmberIntegration = true; | |
| let runPlatform = true; | |
| let runPlatformIntegration = true; | |
| let runPyamber = true; | |
| let runAgentService = true; | |
| let runInfra = true; | |
| let runPyrightLanguageService = true; | |
| if (eventName === "pull_request") { | |
| const stacks = new Set(); | |
| for (const label of labels) { | |
| for (const stack of LABEL_STACKS[label] || []) { | |
| stacks.add(stack); | |
| } | |
| } | |
| runFrontend = stacks.has("frontend"); | |
| runAmber = stacks.has("amber"); | |
| runAmberIntegration = stacks.has("amber-integration"); | |
| runPlatform = stacks.has("platform"); | |
| runPlatformIntegration = stacks.has("platform-integration"); | |
| runPyamber = stacks.has("pyamber"); | |
| runAgentService = stacks.has("agent-service"); | |
| runInfra = stacks.has("infra"); | |
| runPyrightLanguageService = stacks.has("pyright-language-service"); | |
| core.info( | |
| `Stacks selected by label union: ${[...stacks].sort().join(", ") || "(none)"}` | |
| ); | |
| } | |
| core.setOutput("run_frontend", runFrontend ? "true" : "false"); | |
| core.setOutput("run_amber", runAmber ? "true" : "false"); | |
| core.setOutput("run_amber_integration", runAmberIntegration ? "true" : "false"); | |
| core.setOutput("run_platform", runPlatform ? "true" : "false"); | |
| core.setOutput("run_platform_integration", runPlatformIntegration ? "true" : "false"); | |
| core.setOutput("run_pyamber", runPyamber ? "true" : "false"); | |
| core.setOutput("run_agent_service", runAgentService ? "true" : "false"); | |
| core.setOutput("run_infra", runInfra ? "true" : "false"); | |
| core.setOutput("run_pyright_language_service", runPyrightLanguageService ? "true" : "false"); | |
| // Backport targets: all current release/* labels on the PR, unless | |
| // `no-backport-needed` vetoes backporting entirely (e.g. a fix for | |
| // a feature that does not exist on any release branch). The veto | |
| // wins even if release/* labels are also present, so adding it | |
| // mid-review cancels the backport apply-check/build on the re-run. | |
| let targets = [...new Set(labels.filter((n) => /^release\/.+$/.test(n)))].sort(); | |
| if (labels.includes("no-backport-needed")) { | |
| core.info("no-backport-needed present; skipping all backport targets."); | |
| targets = []; | |
| } else if (targets.length === 0) { | |
| core.info("No backport targets on PR."); | |
| } else { | |
| core.info(`Backport targets: ${targets.join(", ")}`); | |
| } | |
| core.setOutput("backport_targets", JSON.stringify(targets)); | |
| cleanup-stale-backport: | |
| if: ${{ github.event_name == 'pull_request' && github.event.action == 'unlabeled' && startsWith(github.event.label.name, 'release/') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Cancel obsolete backport check_runs for the removed target | |
| uses: actions/github-script@v9 | |
| with: | |
| script: | | |
| const { owner, repo } = context.repo; | |
| const target = context.payload.label.name; | |
| const headSha = context.payload.pull_request.head.sha; | |
| const prefix = `backport (${target}) `; | |
| const checks = await github.paginate( | |
| github.rest.checks.listForRef, | |
| { owner, repo, ref: headSha, per_page: 100 } | |
| ); | |
| for (const check of checks) { | |
| if (!check.name.startsWith(prefix)) continue; | |
| if (check.status === "completed" && check.conclusion === "cancelled") continue; | |
| try { | |
| await github.rest.checks.update({ | |
| owner, | |
| repo, | |
| check_run_id: check.id, | |
| status: "completed", | |
| conclusion: "cancelled", | |
| }); | |
| core.info(`Cancelled check ${check.name}`); | |
| } catch (e) { | |
| core.warning(`Failed to update check ${check.id} (${check.name}): ${e.message}`); | |
| } | |
| } | |
| build: | |
| needs: precheck | |
| uses: ./.github/workflows/build.yml | |
| with: | |
| run_frontend: ${{ needs.precheck.outputs.run_frontend == 'true' }} | |
| run_amber: ${{ needs.precheck.outputs.run_amber == 'true' }} | |
| run_amber_integration: ${{ needs.precheck.outputs.run_amber_integration == 'true' }} | |
| run_platform: ${{ needs.precheck.outputs.run_platform == 'true' }} | |
| run_platform_integration: ${{ needs.precheck.outputs.run_platform_integration == 'true' }} | |
| run_pyamber: ${{ needs.precheck.outputs.run_pyamber == 'true' }} | |
| run_agent_service: ${{ needs.precheck.outputs.run_agent_service == 'true' }} | |
| run_infra: ${{ needs.precheck.outputs.run_infra == 'true' }} | |
| run_pyright_language_service: ${{ needs.precheck.outputs.run_pyright_language_service == 'true' }} | |
| secrets: inherit | |
| # Fast pre-flight: does the change even cherry-pick onto each release branch? | |
| # This is a git-only job (no build), so a conflict is reported in seconds and | |
| # gates the expensive backport build below — a conflicting target never spins | |
| # up the full stack matrix. Post-merge, Direct Backport Push reads this job's | |
| # result as the authoritative "conflicts / applies" signal. | |
| backport-apply-check: | |
| needs: precheck | |
| if: ${{ needs.precheck.outputs.backport_targets != '[]' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: ${{ fromJson(needs.precheck.outputs.backport_targets) }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout PR head | |
| uses: actions/checkout@v7 | |
| with: | |
| ref: refs/pull/${{ github.event.pull_request.number }}/head | |
| fetch-depth: 0 | |
| - name: Dry-run cherry-pick onto ${{ matrix.target }} | |
| run: | | |
| bash ./.github/scripts/prepare-backport-checkout.sh \ | |
| "${{ matrix.target }}" \ | |
| "${{ format('{0}..{1}', github.event.pull_request.base.sha, github.event.pull_request.head.sha) }}" | |
| # Collect the targets whose pre-flight cherry-pick succeeded. always() so a | |
| # conflict on one target still lets the others build; an all-conflict run | |
| # yields an empty list and skips the backport build entirely. | |
| backport-buildable: | |
| needs: [precheck, backport-apply-check] | |
| if: ${{ always() && needs.precheck.outputs.backport_targets != '[]' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| targets: ${{ steps.collect.outputs.targets }} | |
| steps: | |
| - name: Collect targets that applied cleanly | |
| id: collect | |
| uses: actions/github-script@v9 | |
| env: | |
| REQUESTED: ${{ needs.precheck.outputs.backport_targets }} | |
| with: | |
| script: | | |
| const requested = JSON.parse(process.env.REQUESTED || "[]"); | |
| const { owner, repo } = context.repo; | |
| const jobs = await github.paginate( | |
| github.rest.actions.listJobsForWorkflowRun, | |
| { owner, repo, run_id: context.runId, per_page: 100 } | |
| ); | |
| const buildable = requested.filter((target) => { | |
| const job = jobs.find( | |
| (j) => j.name === `backport-apply-check (${target})` | |
| ); | |
| return job && job.conclusion === "success"; | |
| }); | |
| core.info(`Buildable backport targets: ${JSON.stringify(buildable)}`); | |
| core.setOutput("targets", JSON.stringify(buildable)); | |
| backport: | |
| needs: [precheck, backport-buildable] | |
| if: ${{ needs.backport-buildable.outputs.targets != '' && needs.backport-buildable.outputs.targets != '[]' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: ${{ fromJson(needs.backport-buildable.outputs.targets) }} | |
| uses: ./.github/workflows/build.yml | |
| with: | |
| checkout_ref: refs/pull/${{ github.event.pull_request.number }}/head | |
| backport_target_branch: ${{ matrix.target }} | |
| backport_commit_range: ${{ format('{0}..{1}', github.event.pull_request.base.sha, github.event.pull_request.head.sha) }} | |
| job_name_suffix: "" | |
| run_frontend: ${{ needs.precheck.outputs.run_frontend == 'true' }} | |
| run_amber: ${{ needs.precheck.outputs.run_amber == 'true' }} | |
| run_amber_integration: ${{ needs.precheck.outputs.run_amber_integration == 'true' }} | |
| run_platform: ${{ needs.precheck.outputs.run_platform == 'true' }} | |
| run_platform_integration: ${{ needs.precheck.outputs.run_platform_integration == 'true' }} | |
| run_pyamber: ${{ needs.precheck.outputs.run_pyamber == 'true' }} | |
| run_agent_service: ${{ needs.precheck.outputs.run_agent_service == 'true' }} | |
| run_infra: ${{ needs.precheck.outputs.run_infra == 'true' }} | |
| run_pyright_language_service: ${{ needs.precheck.outputs.run_pyright_language_service == 'true' }} | |
| secrets: inherit | |
| required-checks: | |
| # Do not rename this job — its display name is referenced in .asf.yaml. | |
| name: Required Checks | |
| # The backport job is deliberately NOT a dependency here: it is an advisory | |
| # signal, not a merge gate. A red pre-merge backport (a cherry-pick that | |
| # conflicts, or a backported tree that fails to build) must not block the | |
| # PR from landing on main — instead Direct Backport Push opens a draft | |
| # backport PR for the author after merge. Keeping backport out of `needs` | |
| # also lets this aggregator report as soon as build finishes. | |
| needs: [precheck, build] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Verify all required checks succeeded or were skipped | |
| run: | | |
| declare -A results=( | |
| [precheck]="${{ needs.precheck.result }}" | |
| [build]="${{ needs.build.result }}" | |
| ) | |
| failed=0 | |
| for job in "${!results[@]}"; do | |
| r="${results[$job]}" | |
| echo "${job}: ${r}" | |
| if [[ "$r" != "success" && "$r" != "skipped" ]]; then | |
| failed=1 | |
| fi | |
| done | |
| if (( failed )); then | |
| echo "::error::One or more required checks did not succeed." | |
| exit 1 | |
| fi | |
| echo "All required checks succeeded or were skipped." |