Merge worktree-issue-119: Integrate DataOps with shared Cognito authe… #155
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy DataOps V1 Lambda | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - ".github/workflows/deploy-dataops-v1.yml" | |
| - "backend/**" | |
| - "content/**" | |
| - "frontend/**" | |
| - "infra/**" | |
| - "Makefile" | |
| - "package.json" | |
| - "package-lock.json" | |
| - "samconfig.toml" | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: dataops-v1-deploy | |
| cancel-in-progress: false | |
| env: | |
| AWS_REGION: eu-west-1 | |
| AWS_DEFAULT_REGION: eu-west-1 | |
| AWS_ROLE_ARN: arn:aws:iam::817685572750:role/dataops-github-actions-deploy | |
| DOCS_GITHUB_OWNER: DataTalksClub | |
| DOCS_GITHUB_REPO: dataops | |
| DOCS_GITHUB_BRANCH: main | |
| AUTH_BASE_URL: https://auth.dtcdev.click | |
| AUTH_USER_POOL_ID: us-east-1_H7nJu52Bs | |
| AUTH_ISSUER: https://cognito-idp.us-east-1.amazonaws.com/us-east-1_H7nJu52Bs | |
| AUTH_JWKS_URL: https://cognito-idp.us-east-1.amazonaws.com/us-east-1_H7nJu52Bs/.well-known/jwks.json | |
| AUTH_CLIENT_ID: 1kjv61esdjs3003s8u42sgr3hf | |
| AUTH_CALLBACK_URL: https://ops.dtcdev.click/auth/callback | |
| AUTH_LOGOUT_URL: https://ops.dtcdev.click/ | |
| AUTH_SESSION_LIFETIME_SECONDS: 28800 | |
| jobs: | |
| checks: | |
| name: Check DataOps v1 app | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: "20" | |
| cache: npm | |
| cache-dependency-path: package-lock.json | |
| - name: Install Node workspace dependencies | |
| run: npm ci | |
| - name: Install backend Playwright Chromium | |
| run: npm exec --workspace dataops-backend -- playwright install --with-deps chromium | |
| - name: Run backend tests | |
| run: npm --prefix backend test | |
| - name: Typecheck backend | |
| run: npm --prefix backend run typecheck | |
| - name: Build backend | |
| run: npm --prefix backend run build | |
| - name: Run backend Playwright E2E | |
| run: npm --prefix backend run test:e2e | |
| - name: Upload backend Playwright artifacts | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: backend-playwright-artifacts | |
| path: | | |
| backend/playwright-report/ | |
| backend/test-results/ | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Set up SAM | |
| uses: aws-actions/setup-sam@v3 | |
| with: | |
| use-installer: true | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Validate SAM template | |
| run: make sam-validate | |
| - name: Build SAM artifact | |
| run: make sam-build | |
| deploy: | |
| name: Deploy DataOps v1 app | |
| runs-on: ubuntu-latest | |
| needs: checks | |
| permissions: | |
| contents: read | |
| id-token: write | |
| env: | |
| EMAIL_DOCUMENT_INTAKE_SECRET_ARN: ${{ secrets.EMAIL_DOCUMENT_INTAKE_SECRET_ARN }} | |
| EMAIL_DOCUMENT_EXTERNAL_SOURCE_BUCKET: ${{ vars.EMAIL_DOCUMENT_EXTERNAL_SOURCE_BUCKET }} | |
| EMAIL_DOCUMENT_EXTERNAL_SOURCE_PREFIX: ${{ vars.EMAIL_DOCUMENT_EXTERNAL_SOURCE_PREFIX }} | |
| EMAIL_DOCUMENT_EXTERNAL_SOURCE_KMS_KEY_ARN: ${{ secrets.EMAIL_DOCUMENT_EXTERNAL_SOURCE_KMS_KEY_ARN }} | |
| MAILCHIMP_SECRET_ARN: ${{ secrets.MAILCHIMP_SECRET_ARN }} | |
| MAILING_EXPORTS_CONFIG: ${{ vars.MAILING_EXPORTS_CONFIG }} | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: "20" | |
| cache: npm | |
| cache-dependency-path: package-lock.json | |
| - name: Install Node workspace dependencies | |
| run: npm ci | |
| - name: Set up SAM | |
| uses: aws-actions/setup-sam@v3 | |
| with: | |
| use-installer: true | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v6.1.0 | |
| with: | |
| role-to-assume: ${{ env.AWS_ROLE_ARN }} | |
| role-session-name: dataops-v1-deploy | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Build SAM artifact | |
| run: make sam-build | |
| - name: Deploy DataOps v1 stack | |
| run: | | |
| if [ -z "$EMAIL_DOCUMENT_INTAKE_SECRET_ARN" ]; then | |
| echo "Required email-document intake secret ARN is not configured" >&2 | |
| exit 1 | |
| fi | |
| : "${EMAIL_DOCUMENT_EXTERNAL_SOURCE_PREFIX:=transfer/}" | |
| : "${MAILING_EXPORTS_CONFIG:=[]}" | |
| sam deploy \ | |
| --config-env full-sandbox \ | |
| --parameter-overrides \ | |
| "ParameterKey=GitHubOwner,ParameterValue=$DOCS_GITHUB_OWNER" \ | |
| "ParameterKey=GitHubRepo,ParameterValue=$DOCS_GITHUB_REPO" \ | |
| "ParameterKey=GitHubBranch,ParameterValue=$DOCS_GITHUB_BRANCH" \ | |
| "ParameterKey=AuthBaseUrl,ParameterValue=$AUTH_BASE_URL" \ | |
| "ParameterKey=AuthUserPoolId,ParameterValue=$AUTH_USER_POOL_ID" \ | |
| "ParameterKey=AuthIssuer,ParameterValue=$AUTH_ISSUER" \ | |
| "ParameterKey=AuthJwksUrl,ParameterValue=$AUTH_JWKS_URL" \ | |
| "ParameterKey=AuthClientId,ParameterValue=$AUTH_CLIENT_ID" \ | |
| "ParameterKey=AuthCallbackUrl,ParameterValue=$AUTH_CALLBACK_URL" \ | |
| "ParameterKey=AuthLogoutUrl,ParameterValue=$AUTH_LOGOUT_URL" \ | |
| "ParameterKey=AuthSessionLifetimeSeconds,ParameterValue=$AUTH_SESSION_LIFETIME_SECONDS" \ | |
| "ParameterKey=EmailDocumentIntakeSecretArn,ParameterValue=$EMAIL_DOCUMENT_INTAKE_SECRET_ARN" \ | |
| "ParameterKey=EmailDocumentExternalSourceBucketName,ParameterValue=$EMAIL_DOCUMENT_EXTERNAL_SOURCE_BUCKET" \ | |
| "ParameterKey=EmailDocumentExternalSourcePrefix,ParameterValue=$EMAIL_DOCUMENT_EXTERNAL_SOURCE_PREFIX" \ | |
| "ParameterKey=EmailDocumentExternalSourceKmsKeyArn,ParameterValue=$EMAIL_DOCUMENT_EXTERNAL_SOURCE_KMS_KEY_ARN" \ | |
| "ParameterKey=MailchimpSecretArn,ParameterValue=$MAILCHIMP_SECRET_ARN" \ | |
| "ParameterKey=MailingExportsConfig,ParameterValue=$MAILING_EXPORTS_CONFIG" | |
| - name: Seed runtime users, workflow templates, and recurring configs | |
| working-directory: ${{ github.workspace }}/backend | |
| run: | | |
| export DATAOPS_TASKS_TABLE="$(aws cloudformation describe-stacks \ | |
| --stack-name dataops-v1 \ | |
| --query "Stacks[0].Outputs[?OutputKey=='DataOpsTasksTableName'].OutputValue" \ | |
| --output text)" | |
| export DATAOPS_USERS_TABLE="$(aws cloudformation describe-stacks \ | |
| --stack-name dataops-v1 \ | |
| --query "Stacks[0].Outputs[?OutputKey=='DataOpsUsersTableName'].OutputValue" \ | |
| --output text)" | |
| export DATAOPS_TEMPLATES_TABLE="$(aws cloudformation describe-stacks \ | |
| --stack-name dataops-v1 \ | |
| --query "Stacks[0].Outputs[?OutputKey=='DataOpsTemplatesTableName'].OutputValue" \ | |
| --output text)" | |
| npm exec -- tsx scripts/seed-users.ts | |
| npm exec -- tsx scripts/seed-templates.ts | |
| npm exec -- tsx scripts/seed-recurring.ts | |
| - name: Smoke test deployed single-origin backend | |
| run: | | |
| backend_url="$(aws cloudformation describe-stacks \ | |
| --stack-name dataops-v1 \ | |
| --query "Stacks[0].Outputs[?OutputKey=='BackendUrl'].OutputValue" \ | |
| --output text)" | |
| # The Function URL output ends with a trailing slash; strip it so we hit | |
| # "/" and "/login" instead of malformed double-slash paths ("//", "//login"). | |
| backend_url="${backend_url%/}" | |
| # /login redirects to the shared Cognito authorize endpoint and root redirects locally. | |
| # The Function URL event normalization in handler.ts must work for this. | |
| # Retry a few times: immediately after a stack update the Function URL can | |
| # briefly return a cold-start/propagation blip before the new code settles. | |
| code="" | |
| login_code="" | |
| for attempt in 1 2 3 4 5; do | |
| code=$(curl -sS -o /dev/null -w "%{http_code}" "$backend_url/") | |
| login_code=$(curl -sS -o /dev/null -w "%{http_code}" "$backend_url/login") | |
| if [ "$code" = "302" ] && [ "$login_code" = "302" ]; then | |
| echo "Backend smoke OK on attempt $attempt: root=$code login=$login_code" | |
| exit 0 | |
| fi | |
| echo "Smoke attempt $attempt: root=$code login=$login_code (retrying)" | |
| sleep 10 | |
| done | |
| echo "Backend smoke test failed after retries: root expected 302 got $code; /login expected 302 got $login_code" | |
| exit 1 |