⚡ perf: Halve Per-Delta Redis Round Trips in Resumable Streams (#14313) #176
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: GitNexus Index | |
| on: | |
| # PR branches are NOT auto-indexed — an embeddings run is too slow to | |
| # spend on every PR push. Only main/dev are indexed automatically; | |
| # individual PRs are indexed on demand via the /gitnexus command or a | |
| # manual workflow_dispatch. | |
| push: | |
| branches: [main, dev] | |
| paths-ignore: ['**.md', 'docs/**', 'LICENSE', '.github/**'] | |
| workflow_dispatch: | |
| inputs: | |
| embeddings: | |
| description: 'Enable embedding generation (slow, increases index size)' | |
| type: boolean | |
| default: false | |
| force: | |
| description: 'Force full re-index' | |
| type: boolean | |
| default: false | |
| # When invoked from the /gitnexus index PR command, the command | |
| # workflow fills these so the index is built from the PR's head | |
| # ref and uploaded under the PR-numbered artifact name. | |
| pr_number: | |
| description: 'PR number to index (set by /gitnexus command)' | |
| type: string | |
| default: '' | |
| pr_ref: | |
| description: 'Optional PR head ref to check out; defaults to refs/pull/<pr_number>/head when pr_number is set' | |
| type: string | |
| default: '' | |
| deploy_after: | |
| description: 'Dispatch GitNexus Deploy after a successful index run' | |
| type: boolean | |
| default: false | |
| permissions: | |
| contents: read | |
| concurrency: | |
| # When triggered by the /gitnexus command, group by PR number so rapid | |
| # re-runs coalesce. Otherwise group by git ref as before. | |
| group: gitnexus-${{ inputs.pr_number != '' && format('pr-{0}', inputs.pr_number) || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| GITNEXUS_VERSION: '1.6.7' | |
| jobs: | |
| index: | |
| permissions: | |
| contents: read | |
| pull-requests: read # read changed files to decide whether embeddings are needed | |
| # Push + dispatch run unconditionally. The pull_request trigger is | |
| # disabled (see `on:` above), so this never runs automatically on a | |
| # PR. PRs are indexed on demand instead: | |
| # - /gitnexus index (PR comment command, contributor-gated) | |
| # - workflow_dispatch (manual dispatch from Actions UI) | |
| # Both arrive as workflow_dispatch. The pull_request guard is kept as | |
| # a safety net should the trigger ever be re-added. | |
| if: | | |
| github.event_name != 'pull_request' || | |
| github.event.pull_request.user.login == 'danny-avila' | |
| runs-on: ubuntu-latest | |
| # Embedding generation dominates the budget: ~45 min worst case on | |
| # standard runners since the 1.6.x graph (~23k nodes) doubled vs 1.5.x. | |
| timeout-minutes: 60 | |
| # Best-effort index: a tool-internal crash must not block PRs. Fail soft on | |
| # PR events; push/dispatch runs still fail loudly so regressions stay visible. | |
| continue-on-error: ${{ github.event_name == 'pull_request' }} | |
| steps: | |
| - name: Validate dispatch inputs | |
| if: github.event_name == 'workflow_dispatch' | |
| env: | |
| PR_NUMBER: ${{ inputs.pr_number }} | |
| PR_REF: ${{ inputs.pr_ref }} | |
| run: | | |
| set -euo pipefail | |
| if [ -n "$PR_NUMBER" ]; then | |
| if [[ ! "$PR_NUMBER" =~ ^[0-9]+$ ]]; then | |
| echo "::error::pr_number must be numeric" | |
| exit 1 | |
| fi | |
| EXPECTED_REF="refs/pull/${PR_NUMBER}/head" | |
| if [ -n "$PR_REF" ] && [ "$PR_REF" != "$EXPECTED_REF" ]; then | |
| echo "::error::pr_ref must match ${EXPECTED_REF}" | |
| exit 1 | |
| fi | |
| elif [ -n "$PR_REF" ]; then | |
| echo "::error::pr_ref requires pr_number" | |
| exit 1 | |
| fi | |
| - name: Resolve GitNexus flags | |
| id: flags | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| ENABLE_EMBEDDINGS_INPUT: ${{ inputs.embeddings }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUM: ${{ github.event.pull_request.number }} | |
| run: | | |
| set -euo pipefail | |
| # Decide whether to generate embeddings. Rules: | |
| # push (main/dev) -> always embed | |
| # pull_request -> embed ONLY when the PR changes files | |
| # under paths that also trigger backend | |
| # or frontend unit tests (api/, client/, | |
| # packages/). Docs/config-only PRs skip | |
| # embeddings to save ~3-5 min of CI. | |
| # workflow_dispatch -> respect the explicit `embeddings` input | |
| # (default false). This also covers the | |
| # /gitnexus index [embeddings] command. | |
| ENABLE_EMBEDDINGS=false | |
| case "$EVENT_NAME" in | |
| workflow_dispatch) | |
| [ "$ENABLE_EMBEDDINGS_INPUT" = "true" ] && ENABLE_EMBEDDINGS=true | |
| ;; | |
| push) | |
| ENABLE_EMBEDDINGS=true | |
| ;; | |
| pull_request) | |
| CHANGED=$(gh api "repos/${{ github.repository }}/pulls/$PR_NUM/files" \ | |
| --paginate --jq '.[].filename' 2>/dev/null || echo "") | |
| if printf '%s\n' "$CHANGED" | grep -qE '^(api/|client/|packages/)'; then | |
| echo "PR #$PR_NUM touches unit-test paths (api|client|packages) — enabling embeddings" | |
| ENABLE_EMBEDDINGS=true | |
| else | |
| echo "PR #$PR_NUM does not touch unit-test paths — graph-only index" | |
| fi | |
| ;; | |
| esac | |
| if [ "$ENABLE_EMBEDDINGS" = "true" ]; then | |
| echo "enable_embeddings=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "enable_embeddings=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '24.16.0' | |
| - name: Install GitNexus CLI | |
| working-directory: ${{ runner.temp }} | |
| env: | |
| NPM_CONFIG_AUDIT: false | |
| NPM_CONFIG_CACHE: ${{ runner.temp }}/gitnexus-npm-cache | |
| NPM_CONFIG_FUND: false | |
| NPM_CONFIG_GLOBALCONFIG: ${{ runner.temp }}/gitnexus-cli/global-npmrc | |
| NPM_CONFIG_REGISTRY: https://registry.npmjs.org/ | |
| NPM_CONFIG_USERCONFIG: ${{ runner.temp }}/gitnexus-cli/.npmrc | |
| run: | | |
| set -euo pipefail | |
| mkdir -p "$RUNNER_TEMP/gitnexus-cli" "$RUNNER_TEMP/gitnexus-npm-cache" | |
| : > "$RUNNER_TEMP/gitnexus-cli/global-npmrc" | |
| printf '%s\n' \ | |
| 'registry=https://registry.npmjs.org/' \ | |
| 'audit=false' \ | |
| 'fund=false' \ | |
| > "$RUNNER_TEMP/gitnexus-cli/.npmrc" | |
| # Keep GitNexus' native DB dependency deterministic in fresh CI installs. | |
| npm install \ | |
| --prefix "$RUNNER_TEMP/gitnexus-cli" \ | |
| --no-save \ | |
| --no-package-lock \ | |
| "gitnexus@${{ env.GITNEXUS_VERSION }}" \ | |
| "@ladybugdb/core@0.17.1" | |
| test -x "$RUNNER_TEMP/gitnexus-cli/node_modules/.bin/gitnexus" | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| # When the /gitnexus command dispatches us with a pr_ref, it's | |
| # a refs/pull/<N>/head ref that GitHub mirrors into the base | |
| # repo for every PR, so checkout works for fork PRs too. When | |
| # pr_ref is empty (native push/pull_request), fall back to the | |
| # default ref actions/checkout would use. | |
| ref: ${{ inputs.pr_ref || (inputs.pr_number != '' && format('refs/pull/{0}/head', inputs.pr_number) || '') }} | |
| fetch-depth: 1 | |
| persist-credentials: false | |
| # HuggingFace throttles anonymous model downloads from shared GHA | |
| # runner IPs (429s or stalled transfers). Cache the embedding model | |
| # across runs so warm runs never touch HF at all. | |
| - name: Cache HuggingFace embedding model | |
| if: steps.flags.outputs.enable_embeddings == 'true' | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ runner.temp }}/hf-cache | |
| key: hf-model-snowflake-arctic-embed-xs-v1 | |
| - name: Run GitNexus Analyze | |
| working-directory: ${{ runner.temp }} | |
| env: | |
| ENABLE_EMBEDDINGS: ${{ steps.flags.outputs.enable_embeddings }} | |
| FORCE: ${{ inputs.force }} | |
| GITNEXUS_BIN: ${{ runner.temp }}/gitnexus-cli/node_modules/.bin/gitnexus | |
| # Fail soft in ~2 min on stalled downloads instead of eating the | |
| # 25-min job budget; HF_TOKEN lifts the anonymous rate limit on | |
| # cold-cache runs (empty when the secret is unset — safe no-op). | |
| HF_DOWNLOAD_TIMEOUT_MS: '60000' | |
| HF_HOME: ${{ runner.temp }}/hf-cache | |
| HF_MAX_ATTEMPTS: '2' | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| NPM_CONFIG_AUDIT: false | |
| NPM_CONFIG_CACHE: ${{ runner.temp }}/gitnexus-npm-cache | |
| NPM_CONFIG_FUND: false | |
| NPM_CONFIG_GLOBALCONFIG: ${{ runner.temp }}/gitnexus-cli/global-npmrc | |
| NPM_CONFIG_REGISTRY: https://registry.npmjs.org/ | |
| NPM_CONFIG_USERCONFIG: ${{ runner.temp }}/gitnexus-cli/.npmrc | |
| run: | | |
| set -euo pipefail | |
| FLAGS=(--skip-agents-md --verbose) | |
| if [ "$ENABLE_EMBEDDINGS" = "true" ]; then | |
| FLAGS+=(--embeddings) | |
| fi | |
| if [ "$FORCE" = "true" ]; then | |
| FLAGS+=(--force) | |
| fi | |
| "$GITNEXUS_BIN" analyze "$GITHUB_WORKSPACE" "${FLAGS[@]}" | |
| - name: Verify index | |
| run: | | |
| if [ ! -d ".gitnexus" ] || [ ! -f ".gitnexus/meta.json" ]; then | |
| echo "::error::GitNexus index was not created" | |
| exit 1 | |
| fi | |
| echo "::group::Index metadata" | |
| cat .gitnexus/meta.json | |
| echo "" | |
| echo "::endgroup::" | |
| - name: Upload GitNexus index | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| # Artifact naming order of precedence: | |
| # 1. /gitnexus command dispatch: inputs.pr_number -> pr-<N> | |
| # 2. Native pull_request event: github.event.pull_request.number | |
| # 3. Push or manual dispatch without pr_number: github.ref_name | |
| name: >- | |
| gitnexus-index-${{ | |
| inputs.pr_number != '' | |
| && format('pr-{0}', inputs.pr_number) | |
| || (github.event_name == 'pull_request' | |
| && format('pr-{0}', github.event.pull_request.number) | |
| || github.ref_name) | |
| }} | |
| path: .gitnexus/ | |
| include-hidden-files: true | |
| retention-days: 30 | |
| post-index: | |
| needs: index | |
| if: | | |
| always() && | |
| (inputs.pr_number != '' || | |
| inputs.deploy_after) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| permissions: | |
| contents: read | |
| actions: write # dispatch gitnexus-deploy.yml when deploy_after is set | |
| pull-requests: write # post completion comments for /gitnexus command runs | |
| steps: | |
| # GitHub suppresses workflow_run events for workflow runs triggered | |
| # by GITHUB_TOKEN (to prevent recursive chaining). Dispatches without | |
| # a PR number can still opt into a deploy by setting deploy_after=true. | |
| - name: Trigger deploy workflow after non-PR dispatches | |
| if: inputs.deploy_after && inputs.pr_number == '' && needs.index.result == 'success' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| core.info('deploy_after=true; dispatching gitnexus-deploy.yml manually.'); | |
| await github.rest.actions.createWorkflowDispatch({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| workflow_id: 'gitnexus-deploy.yml', | |
| ref: 'main', | |
| inputs: { | |
| pr_number: '', | |
| }, | |
| }); | |
| # Reply on the PR when the /gitnexus command path runs so the | |
| # requester knows the index step finished. This fires when | |
| # inputs.pr_number is set and reports the index job result. | |
| - name: Comment on PR — index complete | |
| if: inputs.pr_number != '' | |
| uses: actions/github-script@v7 | |
| env: | |
| EMBEDDINGS_INPUT: ${{ inputs.embeddings }} | |
| INDEX_RESULT: ${{ needs.index.result }} | |
| PR_NUMBER: ${{ inputs.pr_number }} | |
| with: | |
| script: | | |
| const indexSucceeded = process.env.INDEX_RESULT === 'success'; | |
| const outcome = indexSucceeded ? '✅ indexed' : '❌ index failed'; | |
| const prNum = parseInt(process.env.PR_NUMBER || '', 10); | |
| if (!Number.isSafeInteger(prNum)) { | |
| core.setFailed(`Invalid PR number: ${process.env.PR_NUMBER}`); | |
| return; | |
| } | |
| const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | |
| const embeddingsFlag = process.env.EMBEDDINGS_INPUT === 'true' ? 'with embeddings' : 'graph-only'; | |
| const body = [ | |
| `### GitNexus: ${outcome}`, | |
| ``, | |
| `PR #${prNum} was indexed ${embeddingsFlag}.`, | |
| `[Index run](${runUrl})`, | |
| '', | |
| indexSucceeded | |
| ? 'PR-specific deploys are paused; only `LibreChat` and `LibreChat-dev` are currently served.' | |
| : '_Index run failed — the previous index (if any) continues to be served._', | |
| ].join('\n'); | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNum, | |
| body, | |
| }); |