docs-ingestion-ci.yml

name: Docs Ingestion CI

on:
  pull_request:
    branches: [main]
    paths:
      - 'docs-ingestion/**'
      - 'docs-embeddings/**'
      - 'api-references/**'
      - 'content/**'
  push:
    branches: [main]
    paths:
      - 'docs-ingestion/**'
      - 'docs-embeddings/**'
      - 'api-references/**'

jobs:
  build-and-test:
    name: Build & Test
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: docs-ingestion

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'npm'
          cache-dependency-path: docs-ingestion/package-lock.json

      - name: Install dependencies
        run: npm ci

      - name: Build
        run: npm run build

      # ── E. Test suite ──
      - name: Run tests
        run: npm test

  normalize-api-specs:
    name: API Spec Normalization
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: docs-ingestion

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'npm'
          cache-dependency-path: docs-ingestion/package-lock.json

      - name: Install dependencies
        run: npm ci

      # ── A. Normalization run ──
      - name: Run API spec normalization
        run: npm run normalize-api-specs

      - name: Verify output directory exists
        run: |
          if [ ! -d "../.api-reference-normalized" ]; then
            echo "FAIL: .api-reference-normalized/ directory does not exist"
            exit 1
          fi
          echo "PASS: Directory exists"

      - name: Verify file count
        run: |
          count=$(find ../.api-reference-normalized -name '*.md' | wc -l | tr -d ' ')
          echo "Found $count normalized files"
          if [ "$count" -lt 200 ]; then
            echo "FAIL: Expected at least 200 files, got $count"
            exit 1
          fi
          echo "PASS: File count ($count) >= 200"

      # ── B. Determinism check ──
      - name: Copy first run output
        run: cp -r ../.api-reference-normalized /tmp/api-ref-norm-run1

      - name: Run normalization again
        run: npm run normalize-api-specs

      - name: Verify determinism
        run: |
          diff_output=$(diff -r ../.api-reference-normalized /tmp/api-ref-norm-run1 2>&1) || true
          if [ -n "$diff_output" ]; then
            echo "FAIL: Normalization is not deterministic:"
            echo "$diff_output" | head -20
            exit 1
          fi
          echo "PASS: Output is deterministic"

      # ── C. Token limit compliance ──
      - name: Check token limits
        run: npm run check-token-limits

      # ── F. Git ignored state check ──
      - name: Verify .api-reference-normalized is gitignored
        run: |
          cd ..
          if git ls-files --error-unmatch .api-reference-normalized/ 2>/dev/null; then
            echo "FAIL: .api-reference-normalized/ is tracked by git"
            exit 1
          fi
          echo "PASS: .api-reference-normalized/ is not tracked"

      - name: Verify .docs-normalized is gitignored
        run: |
          cd ..
          if git ls-files --error-unmatch .docs-normalized/ 2>/dev/null; then
            echo "FAIL: .docs-normalized/ is tracked by git"
            exit 1
          fi
          echo "PASS: .docs-normalized/ is not tracked"

  ingestion-smoke-test:
    name: Ingestion Smoke Test
    runs-on: ubuntu-latest
    needs: [build-and-test, normalize-api-specs]
    defaults:
      run:
        working-directory: docs-ingestion

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'npm'
          cache-dependency-path: docs-ingestion/package-lock.json

      - name: Install dependencies
        run: npm ci

      - name: Build
        run: npm run build

      # Generate the normalized API specs (required for smoke test)
      - name: Normalize API specs
        run: npm run normalize-api-specs

      # Normalize MDX if content/ exists
      - name: Normalize MDX (if content exists)
        run: |
          if [ -d "../content" ]; then
            npm run normalize-mdx
          else
            echo "No content/ directory — skipping MDX normalization"
          fi

      # ── D. Ingestion smoke test ──
      - name: Run ingestion smoke test
        run: npm run smoke-test-ingestion

  embedding-dry-run:
    name: Embedding Dry Run
    runs-on: ubuntu-latest
    needs: [ingestion-smoke-test]

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'npm'
          cache-dependency-path: docs-ingestion/package-lock.json

      # Build ingestion pipeline and produce chunks.json
      - name: Install ingestion dependencies
        working-directory: docs-ingestion
        run: npm ci

      - name: Build ingestion
        working-directory: docs-ingestion
        run: npm run build

      - name: Normalize API specs
        working-directory: docs-ingestion
        run: npm run normalize-api-specs

      - name: Normalize MDX (if content exists)
        working-directory: docs-ingestion
        run: |
          if [ -d "../content" ]; then
            npm run normalize-mdx
          else
            echo "No content/ directory — skipping MDX normalization"
          fi

      - name: Run ingestion pipeline
        working-directory: docs-ingestion
        run: node dist/index.js

      - name: Verify chunks.json exists
        run: |
          if [ ! -f "docs-ingestion/output/chunks.json" ]; then
            echo "FAIL: chunks.json not produced"
            exit 1
          fi
          echo "PASS: chunks.json exists"

      # Build embeddings pipeline and run dry-run
      - name: Install embedding dependencies
        working-directory: docs-embeddings
        run: npm ci

      - name: Build embeddings
        working-directory: docs-embeddings
        run: npm run build

      # ── E. Embedding dry run ──
      - name: Run embedding dry run
        working-directory: docs-embeddings
        env:
          DRY_RUN: 'true'
          INGESTION_OUTPUT_PATH: ${{ github.workspace }}/docs-ingestion/output/chunks.json
        run: node dist/index.js --dry-run