feat: add docs-ingestion and docs-embeddings pipelines #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Docs Ingestion CI | |
| on: | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - 'docs-ingestion/**' | |
| - 'docs-embeddings/**' | |
| - 'api-references/**' | |
| - 'content/**' | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'docs-ingestion/**' | |
| - 'docs-embeddings/**' | |
| - 'api-references/**' | |
| jobs: | |
| build-and-test: | |
| name: Build & Test | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: docs-ingestion | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'npm' | |
| cache-dependency-path: docs-ingestion/package-lock.json | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Build | |
| run: npm run build | |
| # ── E. Test suite ── | |
| - name: Run tests | |
| run: npm test | |
| normalize-api-specs: | |
| name: API Spec Normalization | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: docs-ingestion | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'npm' | |
| cache-dependency-path: docs-ingestion/package-lock.json | |
| - name: Install dependencies | |
| run: npm ci | |
| # ── A. Normalization run ── | |
| - name: Run API spec normalization | |
| run: npm run normalize-api-specs | |
| - name: Verify output directory exists | |
| run: | | |
| if [ ! -d "../.api-reference-normalized" ]; then | |
| echo "FAIL: .api-reference-normalized/ directory does not exist" | |
| exit 1 | |
| fi | |
| echo "PASS: Directory exists" | |
| - name: Verify file count | |
| run: | | |
| count=$(find ../.api-reference-normalized -name '*.md' | wc -l | tr -d ' ') | |
| echo "Found $count normalized files" | |
| if [ "$count" -lt 200 ]; then | |
| echo "FAIL: Expected at least 200 files, got $count" | |
| exit 1 | |
| fi | |
| echo "PASS: File count ($count) >= 200" | |
| # ── B. Determinism check ── | |
| - name: Copy first run output | |
| run: cp -r ../.api-reference-normalized /tmp/api-ref-norm-run1 | |
| - name: Run normalization again | |
| run: npm run normalize-api-specs | |
| - name: Verify determinism | |
| run: | | |
| diff_output=$(diff -r ../.api-reference-normalized /tmp/api-ref-norm-run1 2>&1) || true | |
| if [ -n "$diff_output" ]; then | |
| echo "FAIL: Normalization is not deterministic:" | |
| echo "$diff_output" | head -20 | |
| exit 1 | |
| fi | |
| echo "PASS: Output is deterministic" | |
| # ── C. Token limit compliance ── | |
| - name: Check token limits | |
| run: npm run check-token-limits | |
| # ── F. Git ignored state check ── | |
| - name: Verify .api-reference-normalized is gitignored | |
| run: | | |
| cd .. | |
| if git ls-files --error-unmatch .api-reference-normalized/ 2>/dev/null; then | |
| echo "FAIL: .api-reference-normalized/ is tracked by git" | |
| exit 1 | |
| fi | |
| echo "PASS: .api-reference-normalized/ is not tracked" | |
| - name: Verify .docs-normalized is gitignored | |
| run: | | |
| cd .. | |
| if git ls-files --error-unmatch .docs-normalized/ 2>/dev/null; then | |
| echo "FAIL: .docs-normalized/ is tracked by git" | |
| exit 1 | |
| fi | |
| echo "PASS: .docs-normalized/ is not tracked" | |
| ingestion-smoke-test: | |
| name: Ingestion Smoke Test | |
| runs-on: ubuntu-latest | |
| needs: [build-and-test, normalize-api-specs] | |
| defaults: | |
| run: | |
| working-directory: docs-ingestion | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'npm' | |
| cache-dependency-path: docs-ingestion/package-lock.json | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Build | |
| run: npm run build | |
| # Generate the normalized API specs (required for smoke test) | |
| - name: Normalize API specs | |
| run: npm run normalize-api-specs | |
| # Normalize MDX if content/ exists | |
| - name: Normalize MDX (if content exists) | |
| run: | | |
| if [ -d "../content" ]; then | |
| npm run normalize-mdx | |
| else | |
| echo "No content/ directory — skipping MDX normalization" | |
| fi | |
| # ── D. Ingestion smoke test ── | |
| - name: Run ingestion smoke test | |
| run: npm run smoke-test-ingestion | |
| embedding-dry-run: | |
| name: Embedding Dry Run | |
| runs-on: ubuntu-latest | |
| needs: [ingestion-smoke-test] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'npm' | |
| cache-dependency-path: docs-ingestion/package-lock.json | |
| # Build ingestion pipeline and produce chunks.json | |
| - name: Install ingestion dependencies | |
| working-directory: docs-ingestion | |
| run: npm ci | |
| - name: Build ingestion | |
| working-directory: docs-ingestion | |
| run: npm run build | |
| - name: Normalize API specs | |
| working-directory: docs-ingestion | |
| run: npm run normalize-api-specs | |
| - name: Normalize MDX (if content exists) | |
| working-directory: docs-ingestion | |
| run: | | |
| if [ -d "../content" ]; then | |
| npm run normalize-mdx | |
| else | |
| echo "No content/ directory — skipping MDX normalization" | |
| fi | |
| - name: Run ingestion pipeline | |
| working-directory: docs-ingestion | |
| run: node dist/index.js | |
| - name: Verify chunks.json exists | |
| run: | | |
| if [ ! -f "docs-ingestion/output/chunks.json" ]; then | |
| echo "FAIL: chunks.json not produced" | |
| exit 1 | |
| fi | |
| echo "PASS: chunks.json exists" | |
| # Build embeddings pipeline and run dry-run | |
| - name: Install embedding dependencies | |
| working-directory: docs-embeddings | |
| run: npm ci | |
| - name: Build embeddings | |
| working-directory: docs-embeddings | |
| run: npm run build | |
| # ── E. Embedding dry run ── | |
| - name: Run embedding dry run | |
| working-directory: docs-embeddings | |
| env: | |
| DRY_RUN: 'true' | |
| INGESTION_OUTPUT_PATH: ${{ github.workspace }}/docs-ingestion/output/chunks.json | |
| run: node dist/index.js --dry-run |