diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5dd0e2228d..e6052185be 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -549,9 +549,11 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: go test ./tests -run 'Test((AWS(StoreHooks|Secrets)|GCPSecrets|AzureSecrets)FlociE2E|LocalGitOpsPushE2E|Scaffold(AWSLandingZone|GCPLandingZone|AzureLandingZone|AWSApp)FlociE2E|InitFromTemplateRepoGiteaE2E|TerraformFlociTfmigrateS3History)' -count=1 -timeout 25m -v - # run k3s demo tests + # Run the approachable k3s demos plus the focused native Helm lifecycle + # scenario. Keep failure-path coverage under tests/fixtures so public examples + # remain small, happy-path configurations. k3s: - name: "[k3s-${{ matrix.flavor.target }}] ${{ matrix.demo-folder }}" + name: "[k3s-${{ matrix.flavor.target }}] ${{ matrix.fixture.name }}" needs: build runs-on: ${{ matrix.flavor.os }} env: @@ -564,11 +566,15 @@ jobs: flavor: - { os: ubuntu-latest, target: linux, artifact: linux } - { os: "macos-15-intel", target: macos, artifact: macos-intel } - demo-folder: - - demo-helmfile - - helm - - timeout-minutes: 60 + fixture: + - { name: demo-helmfile, path: examples/demo-helmfile } + - { name: helm, path: examples/helm } + - { name: helm-lifecycle, path: tests/fixtures/scenarios/helm-lifecycle } + + # The macOS matrix may spend up to 45 minutes starting Colima, followed by + # two bounded 45-minute test attempts. Keep the job ceiling above those + # nested budgets so the second retry can finish and emit diagnostics. + timeout-minutes: 180 steps: - name: Check out code into the Go module directory uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -616,9 +622,9 @@ jobs: mkdir -p ~/.aws echo '[default]' > ~/.aws/config - - name: Run tests for ${{ matrix.demo-folder }} + - name: Run tests for ${{ matrix.fixture.name }} run: | - cd examples/${{ matrix.demo-folder }} + cd "${{ matrix.fixture.path }}" run_with_timeout() { python3 - "$@" <<'PY' import os @@ -644,8 +650,20 @@ jobs: } cleanup_k3s() { - atmos emulator down kubernetes -s dev || true - docker rm -f atmos-dev-emulator-kubernetes >/dev/null 2>&1 || true + run_with_timeout 60 atmos emulator down kubernetes -s dev || true + run_with_timeout 30 docker rm -f atmos-dev-emulator-kubernetes >/dev/null 2>&1 || true + } + + diagnose_k3s() { + if ! run_with_timeout 15 docker inspect atmos-dev-emulator-kubernetes >/dev/null 2>&1; then + return + fi + + echo "::group::k3s workload diagnostics" + run_with_timeout 30 docker exec atmos-dev-emulator-kubernetes kubectl get pods,jobs,deployments -A -o wide || true + run_with_timeout 30 docker exec atmos-dev-emulator-kubernetes kubectl get events -A --sort-by=.lastTimestamp || true + run_with_timeout 30 docker exec atmos-dev-emulator-kubernetes ctr --namespace k8s.io images list || true + echo "::endgroup::" } # macOS k3s jobs can occasionally hang in the Docker/Colima stack. @@ -653,7 +671,7 @@ jobs: # whole job timeout and cancelling the required aggregate check. attempt_timeout=1500 if [ "${{ matrix.flavor.target }}" = "macos" ]; then - attempt_timeout=900 + attempt_timeout=2700 fi trap cleanup_k3s EXIT @@ -661,6 +679,7 @@ jobs: run_with_timeout "${attempt_timeout}" atmos test && exit 0 status=$? echo "atmos test failed (attempt ${attempt}/2, status ${status})" >&2 + diagnose_k3s cleanup_k3s [ "${attempt}" -lt 2 ] && sleep 15 done diff --git a/docs/fixes/2026-07-31-native-helm-release-lifecycle.md b/docs/fixes/2026-07-31-native-helm-release-lifecycle.md new file mode 100644 index 0000000000..f7af35f5cc --- /dev/null +++ b/docs/fixes/2026-07-31-native-helm-release-lifecycle.md @@ -0,0 +1,36 @@ +# Native Helm release lifecycle + +**Date:** 2026-07-31 + +Native Helm cluster operations now expose Helm 4 wait, timeout, recovery, +history, hook, and CRD controls through stack configuration and explicit command +flags. Apply and delete dry runs now reach the Helm SDK without persisting release +state. Caller cancellation propagates through direct and dependency-ordered +execution into install and upgrade actions and into delete wait and hook phases; +Helm 4 does not expose a context-aware uninstall request. + +Atmos reports the selected action and effective release policy before the Helm +action begins, including any `hookOnly` to `watcher` promotion required by +failure recovery. Bulk delete uses reverse dependency order so dependents are +removed before the releases they consume. + +## Migration notes + +- An omitted `release.timeout` remains `0s` (unbounded) for one minor release and emits a + warning. The omitted default becomes `5m` in the following minor. Configure + `release.timeout: 0s` explicitly to keep unbounded behavior without the warning. +- An omitted `release.history.max` retains ten upgrade revisions, matching the + Helm CLI. Configure `release.history.max: 0` to retain unlimited history. +- Failure recovery is operation-specific: use `release.install.on_failure: uninstall` + for failed first installs and `release.upgrade.on_failure: rollback` for failed + upgrades. Upgrade cleanup is controlled independently by + `release.upgrade.cleanup_on_failure`. +- Boolean `--wait=true` and `--wait=false` remain accepted temporarily; use + `--wait=watcher` and `--wait=hookOnly`. +- Explicit lifecycle flags cannot be combined with a non-Kubernetes provision + target. Stored lifecycle configuration is intentionally bypassed for external + delivery and is identified as such in the execution summary. +- Chart-loading commands do not fetch missing dependencies unless + `--dependency-update` is explicitly supplied. The opt-in follows Helm's + dependency-update behavior and may access repositories and mutate the chart's + `charts/` directory and lock file. diff --git a/docs/prd/native-helm-release-lifecycle.md b/docs/prd/native-helm-release-lifecycle.md index 2900637f75..8fcf68ef82 100644 --- a/docs/prd/native-helm-release-lifecycle.md +++ b/docs/prd/native-helm-release-lifecycle.md @@ -53,7 +53,6 @@ These gaps force users migrating from Helm or Helmfile to choose between depende - Fix apply dry-run propagation as a release-blocking safety prerequisite, then correctly propagate delete dry-run, cancellation, and deadlines through cluster operations. - Validate configuration before chart download or cluster mutation. - Keep template and diff complete by including Helm chart hook resources alongside the ordinary release manifest. -- Allow callers to opt into fetching missing chart dependencies with Helm-compatible `--dependency-update` semantics. - Keep the design compatible with future pre-rollback diagnostics without requiring another public configuration rename. - Follow Atmos schema, stack-processing, command parsing, provider, error, logging, and testing conventions. @@ -240,8 +239,6 @@ atmos helm delete demo-api -s example-prod \ `template`, `diff`, and `plan` do not register release-lifecycle flags because they do not perform a release operation. -All chart-loading operations (`template`, `diff`, `plan`, `apply`, and `deploy`) accept `--dependency-update`. Atmos invokes Helm's dependency manager only when a declared dependency is missing. The flag is intentionally invocation-scoped: without it, Atmos does not access dependency repositories or mutate the chart directory and instead reports both the equivalent `helm dependency build ` command and the opt-in flag. - ## Configuration Contract ### Release-Wide Defaults @@ -372,6 +369,7 @@ The following rules apply: - A release that fails and is successfully rolled back still returns failure to the scheduler. - A rollback or uninstall failure preserves the original release failure and adds the recovery failure. - Dependents never run after timeout, failed readiness, failed hooks, failed rollback, or cancellation. +- Bulk delete traverses the selected graph in reverse topological order so dependents are removed before their dependencies. - A future mixed-kind scheduler consumes the same provider result; it must not reinterpret Helm readiness. ## Timeout Semantics diff --git a/errors/errors.go b/errors/errors.go index c96af19ffc..aac0514868 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -1539,6 +1539,7 @@ var ( ErrHelmReleaseHistory = errors.New("failed to inspect helm release history") ErrHelmReleaseUpgrade = errors.New("failed to upgrade helm release") ErrHelmReleaseUninstall = errors.New("failed to uninstall helm release") + ErrHelmReleaseOperation = errors.New("helm release operation failed") ) // Stack dependency (`depends_on`) resolution errors. diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 83365237d3..2452d82a26 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -25,16 +25,6 @@ logs: file: "/dev/stderr" level: Info -toolchain: - install_path: .tools - aliases: - helm: helm/helm - registries: - - name: aqua-public - type: aqua - source: https://github.com/aquaproj/aqua-registry/tree/main/pkgs - priority: 10 - commands: - name: "test" description: "Render the local Helm chart, deploy it to the k3s emulator, verify it, then tear it down" diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index ae53528c78..3dcde80282 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -11,9 +11,6 @@ components: demo: metadata: component: demo - dependencies: - tools: - helm: "v3.21.2" # The chart reference. "." is the component directory itself # (components/helm/demo), which contains Chart.yaml. chart: "." @@ -29,9 +26,6 @@ components: demo-repo: metadata: component: demo - dependencies: - tools: - helm: "v3.21.2" repositories: - name: local url: !env HELM_DEMO_REPO_URL diff --git a/internal/exec/describe_affected_changed_files_index.go b/internal/exec/describe_affected_changed_files_index.go index 0a90dac7e8..5b854e9bce 100644 --- a/internal/exec/describe_affected_changed_files_index.go +++ b/internal/exec/describe_affected_changed_files_index.go @@ -20,6 +20,10 @@ type changedFilesIndex struct { // allFiles contains all changed files for fallback scenarios. allFiles []string + // allFilesSet provides constant-time lookup for dependencies that may live + // outside a component base path, such as native Helm values_files. + allFilesSet map[string]struct{} + mu sync.RWMutex } @@ -34,6 +38,7 @@ func newChangedFilesIndex(atmosConfig *schema.AtmosConfiguration, changedFiles [ index := &changedFilesIndex{ filesByBasePath: make(map[string][]string), allFiles: nil, // Set after normalization. + allFilesSet: make(map[string]struct{}, len(changedFiles)), } // Pre-compute absolute base paths for each component type. @@ -62,7 +67,9 @@ func newChangedFilesIndex(atmosConfig *schema.AtmosConfiguration, changedFiles [ absF = f } } + absF = filepath.Clean(absF) absAllFiles = append(absAllFiles, absF) + index.allFilesSet[absF] = struct{}{} } index.allFiles = absAllFiles @@ -83,7 +90,7 @@ func newChangedFilesIndex(atmosConfig *schema.AtmosConfiguration, changedFiles [ // Only includes non-empty component base paths to avoid indexing files under the root basePath. func buildNormalizedBasePaths(atmosConfig *schema.AtmosConfiguration) []string { // Collect base paths, skipping empty ones to prevent root basePath collisions. - basePaths := make([]string, 0, 4) + basePaths := make([]string, 0, 6) // Add terraform base path if configured. if atmosConfig.Components.Terraform.BasePath != "" { @@ -105,6 +112,11 @@ func buildNormalizedBasePaths(atmosConfig *schema.AtmosConfiguration) []string { basePaths = append(basePaths, filepath.Join(atmosConfig.BasePath, atmosConfig.Components.Kubernetes.BasePath)) } + // Add native Helm base path if configured. + if atmosConfig.Components.Helm.BasePath != "" { + basePaths = append(basePaths, filepath.Join(atmosConfig.BasePath, atmosConfig.Components.Helm.BasePath)) + } + // Add stacks base path if configured. if atmosConfig.Stacks.BasePath != "" { basePaths = append(basePaths, filepath.Join(atmosConfig.BasePath, atmosConfig.Stacks.BasePath)) @@ -211,3 +223,13 @@ func (idx *changedFilesIndex) getAllFiles() []string { defer idx.mu.RUnlock() return idx.allFiles } + +// isChangedFile reports whether the normalized absolute path is in the git +// change set. It is used for component dependencies that can be located +// outside the component's indexed base path. +func (idx *changedFilesIndex) isChangedFile(path string) bool { + idx.mu.RLock() + defer idx.mu.RUnlock() + _, ok := idx.allFilesSet[filepath.Clean(path)] + return ok +} diff --git a/internal/exec/describe_affected_components.go b/internal/exec/describe_affected_components.go index f9c616443f..b2ad46ff31 100644 --- a/internal/exec/describe_affected_components.go +++ b/internal/exec/describe_affected_components.go @@ -3,6 +3,7 @@ package exec import ( "fmt" + "path/filepath" "reflect" "github.com/go-viper/mapstructure/v2" @@ -670,6 +671,14 @@ func processHelmComponentsIndexed( } } + if helmValuesFileChanged(component, componentSection, atmosConfig, filesIndex) { + err := addAffectedComponent(&affected, atmosConfig, componentName, stackName, cfg.HelmComponentType, + &componentSection, affectedReasonStackValuesFile, includeSpaceliftAdminStacks, currentStacks, includeSettings) + if err != nil { + return nil, err + } + } + if err := addHelmSectionAffected(&affected, atmosConfig, componentName, stackName, &componentSection, remoteStacks, currentStacks, includeSpaceliftAdminStacks, includeSettings); err != nil { return nil, err } @@ -689,6 +698,56 @@ func processHelmComponentsIndexed( return affected, nil } +// helmValuesFileChanged reports whether a native Helm component consumes a +// values file changed by the compared git refs. Relative paths are resolved +// from the physical component directory, matching Helm values loading. The +// target may live outside the Helm component base path, so lookup uses the +// complete changed-file set rather than the component-path index. +func helmValuesFileChanged( + component string, + componentSection map[string]any, + atmosConfig *schema.AtmosConfiguration, + filesIndex *changedFilesIndex, +) bool { + if filesIndex == nil { + return false + } + + componentPath := filepath.Join(atmosConfig.BasePath, atmosConfig.Components.Helm.BasePath, component) + for _, ref := range stringSlice(componentSection[cfg.ValuesFilesSectionName]) { + path := ref + if !filepath.IsAbs(path) { + path = filepath.Join(componentPath, path) + } + absPath, err := filepath.Abs(path) + if err == nil && filesIndex.isChangedFile(absPath) { + return true + } + } + + return false +} + +func stringSlice(value any) []string { + switch typed := value.(type) { + case []string: + return typed + case []any: + result := make([]string, 0, len(typed)) + for _, item := range typed { + if str, ok := item.(string); ok && str != "" { + result = append(result, str) + } + } + return result + case string: + if typed != "" { + return []string{typed} + } + } + return nil +} + func addHelmSectionAffected( affected *[]schema.Affected, atmosConfig *schema.AtmosConfiguration, diff --git a/internal/exec/describe_affected_components_test.go b/internal/exec/describe_affected_components_test.go index d482551bb6..3c3045ef03 100644 --- a/internal/exec/describe_affected_components_test.go +++ b/internal/exec/describe_affected_components_test.go @@ -612,6 +612,59 @@ func TestProcessHelmComponentsIndexed_FolderChanged(t *testing.T) { assert.Contains(t, affected[0].AffectedAll, affectedReasonComponent) } +func TestProcessHelmComponentsIndexed_ValuesFilesChanged(t *testing.T) { + tempDir := t.TempDir() + atmosConfig := helmAtmosConfig() + atmosConfig.BasePath = tempDir + + componentFolder := "shared-chart" + componentPath := filepath.Join(tempDir, "components", "helm", componentFolder) + valuesFile := filepath.Join(tempDir, "config", "helm", "app-values.yaml") + valuesRef, err := filepath.Rel(componentPath, valuesFile) + require.NoError(t, err) + + for _, tt := range []struct { + name string + valuesFiles any + changedFile string + wantAffected bool + }{ + {name: "relative list", valuesFiles: []any{valuesRef}, changedFile: valuesFile, wantAffected: true}, + {name: "typed relative list", valuesFiles: []string{valuesRef}, changedFile: valuesFile, wantAffected: true}, + {name: "scalar", valuesFiles: valuesRef, changedFile: valuesFile, wantAffected: true}, + {name: "absolute", valuesFiles: []any{valuesFile}, changedFile: valuesFile, wantAffected: true}, + {name: "unrelated", valuesFiles: []string{valuesRef}, changedFile: filepath.Join(tempDir, "config", "helm", "other-values.yaml")}, + } { + t.Run(tt.name, func(t *testing.T) { + identical := map[string]any{ + cfg.ComponentSectionName: componentFolder, + sectionNameChart: ".", + sectionNameValuesF: tt.valuesFiles, + } + helmSection := map[string]any{helmTestComponent: identical} + remoteStacks := helmRemoteStacksWith(identical) + filesIndex := newChangedFilesIndex(atmosConfig, []string{tt.changedFile}, tempDir) + + affected, err := processHelmComponentsIndexed( + helmTestStack, helmSection, &remoteStacks, &remoteStacks, + atmosConfig, filesIndex, newComponentPathPatternCache(), + false, false, false, + ) + require.NoError(t, err) + + if !tt.wantAffected { + assert.Empty(t, affected) + return + } + + require.Len(t, affected, 1) + assert.Equal(t, helmTestComponent, affected[0].Component) + assert.Equal(t, cfg.HelmComponentType, affected[0].ComponentType) + assert.Contains(t, affected[0].AffectedAll, affectedReasonStackValuesFile) + }) + } +} + func TestProcessHelmComponentsIndexed_SkipsAbstractLockedAndInvalidSections(t *testing.T) { atmosConfig := helmAtmosConfig() remoteStacks := helmRemoteStacksWith(map[string]any{ diff --git a/internal/exec/describe_affected_optimizations_test.go b/internal/exec/describe_affected_optimizations_test.go index e8f8c729dc..254ba9dab9 100644 --- a/internal/exec/describe_affected_optimizations_test.go +++ b/internal/exec/describe_affected_optimizations_test.go @@ -2302,6 +2302,22 @@ func TestChangedFilesIndex_GetRelevantFiles_EdgeCases(t *testing.T) { }) } +func TestChangedFilesIndex_NativeHelmBasePath(t *testing.T) { + tempDir := t.TempDir() + atmosConfig := &schema.AtmosConfiguration{ + BasePath: tempDir, + Components: schema.Components{ + Helm: schema.Helm{BasePath: "components/helm"}, + }, + } + helmFile := filepath.Join(tempDir, "components", "helm", "app", "Chart.yaml") + unrelatedFile := filepath.Join(tempDir, "config", "helm", "app-values.yaml") + index := newChangedFilesIndex(atmosConfig, []string{helmFile, unrelatedFile}, tempDir) + + assert.Equal(t, []string{helmFile}, index.getRelevantFiles(cfg.HelmComponentType, atmosConfig)) + assert.True(t, index.isChangedFile(unrelatedFile)) +} + func TestComponentPathPatternCache_GetTerraformModulePatterns_EdgeCases(t *testing.T) { tempDir := t.TempDir() diff --git a/pkg/ci/plugins/helm/aggregate.go b/pkg/ci/plugins/helm/aggregate.go new file mode 100644 index 0000000000..8b4bca8faa --- /dev/null +++ b/pkg/ci/plugins/helm/aggregate.go @@ -0,0 +1,339 @@ +package helm + +import ( + "fmt" + "sort" + "strconv" + "strings" + "unicode/utf8" + + "github.com/cloudposse/atmos/pkg/ci/internal/plugin" + log "github.com/cloudposse/atmos/pkg/logger" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +const helmAggregateMarkdownMaxBytes = 960 * 1024 + +type helmAggregate struct { + Command string + Components []helmAggregateComponent + Counts helmAggregateCounts +} + +type helmAggregateComponent struct { + Result schema.HelmCIResult + Summary Summary + Status string +} + +type helmAggregateCounts struct { + Total int + Succeeded int + Failed int + Changed int + NoChanges int + Skipped int +} + +type helmAggregateCountRow struct { + label string + count int +} + +// onAfterAggregate writes one deterministic job summary for a graph-backed +// Helm plan or apply command. +func (p *Plugin) onAfterAggregate(ctx *plugin.HookContext) error { + defer perf.Track(ctx.Config, "helmci.Plugin.onAfterAggregate")() + + resultSet, ok := normalizeHelmAggregate(ctx.Aggregate) + if !ok { + log.Debug("Skipping aggregate Helm CI hook: no results") + return nil + } + if len(resultSet.Results) == 0 { + if ctx.CommandError == nil && ctx.ExitCode == 0 { + log.Debug("Skipping aggregate Helm CI hook: no results") + return nil + } + if !isSummaryEnabled(ctx.Config) { + return nil + } + writer := ctx.Provider.OutputWriter() + if writer == nil { + return nil + } + command := resultSet.Command + if command == "" { + command = ctx.Command + } + return writer.WriteSummary(renderHelmAggregateFailureMarkdown(command, ctx.CommandError, ctx.ExitCode)) + } + if !isSummaryEnabled(ctx.Config) { + return nil + } + writer := ctx.Provider.OutputWriter() + if writer == nil { + return nil + } + aggregate := buildHelmAggregate(resultSet) + return writer.WriteSummary(renderHelmAggregateMarkdown(&aggregate)) +} + +func renderHelmAggregateFailureMarkdown(command string, commandErr error, exitCode int) string { + message := fmt.Sprintf("command exited with code %d", exitCode) + if commandErr != nil { + message = commandErr.Error() + } + return enforceHelmAggregateMarkdownLimit(fmt.Sprintf( + "## Helm %s Summary\n\nCommand failed before any components were processed.\n\n**Error:** %s\n", + helmAggregateCommandLabel(normalizeHelmAggregateCommand(command)), + helmMarkdownCell(message), + )) +} + +func normalizeHelmAggregate(value any) (schema.HelmCIResultSet, bool) { + switch typed := value.(type) { + case schema.HelmCIResultSet: + return typed, true + case *schema.HelmCIResultSet: + if typed == nil { + return schema.HelmCIResultSet{}, false + } + return *typed, true + default: + return schema.HelmCIResultSet{}, false + } +} + +func buildHelmAggregate(resultSet schema.HelmCIResultSet) helmAggregate { + results := append([]schema.HelmCIResult(nil), resultSet.Results...) + sort.SliceStable(results, func(i, j int) bool { + if results[i].Stack != results[j].Stack { + return results[i].Stack < results[j].Stack + } + if results[i].Component != results[j].Component { + return results[i].Component < results[j].Component + } + return results[i].NodeID < results[j].NodeID + }) + + aggregate := helmAggregate{Command: normalizeHelmAggregateCommand(resultSet.Command)} + aggregate.Components = make([]helmAggregateComponent, 0, len(results)) + for i := range results { + component := helmAggregateComponent{ + Result: results[i], + Summary: normalizeSummary(results[i].Summary), + } + component.Status = helmAggregateStatus(aggregate.Command, &component) + aggregate.Components = append(aggregate.Components, component) + aggregate.Counts.add(component.Status) + } + return aggregate +} + +func normalizeHelmAggregateCommand(command string) string { + switch command { + case "apply", "deploy": + return "apply" + default: + return "plan" + } +} + +func helmAggregateStatus(command string, component *helmAggregateComponent) string { + if component.Result.Status == "failed" || component.Result.Error != "" || component.Result.ExitCode != 0 { + return "failed" + } + if component.Result.Status == "skipped" || !component.Result.Processed { + return "skipped" + } + if command == "plan" { + if strings.TrimSpace(component.Summary.Diff) != "" { + return "changed" + } + return "no changes" + } + return "succeeded" +} + +func (counts *helmAggregateCounts) add(status string) { + counts.Total++ + switch status { + case "failed": + counts.Failed++ + case "changed": + counts.Changed++ + case "no changes": + counts.NoChanges++ + case "skipped": + counts.Skipped++ + default: + counts.Succeeded++ + } +} + +func renderHelmAggregateMarkdown(aggregate *helmAggregate) string { + var builder strings.Builder + builder.WriteString("## Helm ") + builder.WriteString(helmAggregateCommandLabel(aggregate.Command)) + builder.WriteString(" Summary\n\n") + builder.WriteString(helmAggregateSummaryText(aggregate.Command, &aggregate.Counts)) + builder.WriteString("\n\n") + writeHelmAggregateCounts(&builder, aggregate.Command, &aggregate.Counts) + writeHelmAggregateComponents(&builder, aggregate.Components) + writeHelmAggregateDetails(&builder, aggregate.Command, aggregate.Components) + return enforceHelmAggregateMarkdownLimit(builder.String()) +} + +func helmAggregateCommandLabel(command string) string { + if command == "apply" { + return "Apply" + } + return "Plan" +} + +func helmAggregateSummaryText(command string, counts *helmAggregateCounts) string { + if command == "apply" { + return fmt.Sprintf( + "Processed %d component(s): %d succeeded, %d failed, %d skipped.", + counts.Total, + counts.Succeeded, + counts.Failed, + counts.Skipped, + ) + } + return fmt.Sprintf( + "Processed %d component(s): %d changed, %d unchanged, %d failed, %d skipped.", + counts.Total, + counts.Changed, + counts.NoChanges, + counts.Failed, + counts.Skipped, + ) +} + +func writeHelmAggregateCounts(builder *strings.Builder, command string, counts *helmAggregateCounts) { + builder.WriteString("| Result | Components |\n") + builder.WriteString("| --- | ---: |\n") + rows := make([]helmAggregateCountRow, 0, 4) + if command == "apply" { + rows = append(rows, helmAggregateCountRow{label: "Succeeded", count: counts.Succeeded}) + } else { + rows = append(rows, + helmAggregateCountRow{label: "Changed", count: counts.Changed}, + helmAggregateCountRow{label: "No changes", count: counts.NoChanges}, + ) + } + rows = append(rows, + helmAggregateCountRow{label: "Failed", count: counts.Failed}, + helmAggregateCountRow{label: "Skipped", count: counts.Skipped}, + ) + for _, row := range rows { + builder.WriteString("| ") + builder.WriteString(row.label) + builder.WriteString(" | ") + builder.WriteString(strconv.Itoa(row.count)) + builder.WriteString(" |\n") + } + builder.WriteString("\n") +} + +func writeHelmAggregateComponents(builder *strings.Builder, components []helmAggregateComponent) { + builder.WriteString("| Stack | Component | Status | Chart | Release | Namespace | Target | Duration |\n") + builder.WriteString("| --- | --- | --- | --- | --- | --- | --- | ---: |\n") + for i := range components { + component := &components[i] + values := []string{ + component.Result.Stack, + component.Result.Component, + component.Status, + component.Summary.Chart, + component.Summary.ReleaseName, + component.Summary.Namespace, + component.Summary.Target, + formatHelmAggregateDuration(component.Result.DurationMS), + } + builder.WriteString("| ") + for index, value := range values { + if index > 0 { + builder.WriteString(" | ") + } + builder.WriteString(helmMarkdownCell(value)) + } + builder.WriteString(" |\n") + } + builder.WriteString("\n") +} + +func writeHelmAggregateDetails(builder *strings.Builder, command string, components []helmAggregateComponent) { + for i := range components { + component := &components[i] + if component.Status != "failed" && !(command == "plan" && component.Status == "changed") { + continue + } + var detail strings.Builder + detail.WriteString("
") + detail.WriteString(helmMarkdownCell(component.Result.Stack + "/" + component.Result.Component + ": " + component.Status)) + detail.WriteString("\n\n") + if component.Status == "failed" { + detail.WriteString(helmMarkdownCodeBlock("text", plugin.TruncateDetail(component.Result.Error))) + } else { + detail.WriteString(helmMarkdownCodeBlock("diff", plugin.TruncateDetail(component.Summary.Diff))) + } + detail.WriteString("\n
\n\n") + if builder.Len()+detail.Len() > helmAggregateMarkdownMaxBytes { + builder.WriteString("> [!WARNING]\n> Additional component details were omitted to stay below GitHub Actions' job summary limit.\n") + return + } + builder.WriteString(detail.String()) + } +} + +func helmMarkdownCodeBlock(language, value string) string { + fence := "```" + for strings.Contains(value, fence) { + fence += "`" + } + return fence + language + "\n" + value + "\n" + fence + "\n" +} + +func formatHelmAggregateDuration(milliseconds int64) string { + if milliseconds <= 0 { + return "-" + } + return strconv.FormatInt(milliseconds, 10) + "ms" +} + +func helmMarkdownCell(value string) string { + value = strings.ReplaceAll(value, "|", "\\|") + value = strings.ReplaceAll(value, "\r", " ") + return strings.ReplaceAll(value, "\n", " ") +} + +func enforceHelmAggregateMarkdownLimit(markdown string) string { + if len(markdown) <= helmAggregateMarkdownMaxBytes { + return markdown + } + notice := "\n\n> [!WARNING]\n> Summary truncated to stay below GitHub Actions' job summary limit.\n" + limit := helmAggregateMarkdownMaxBytes - len(notice) + return trimHelmAggregateMarkdownToLimit(markdown, limit) + notice +} + +func trimHelmAggregateMarkdownToLimit(markdown string, maxBytes int) string { + if maxBytes <= 0 { + return "" + } + if len(markdown) <= maxBytes { + return markdown + } + + end := maxBytes + if lineEnd := strings.LastIndexByte(markdown[:maxBytes], '\n'); lineEnd > 0 { + end = lineEnd + } + for end > 0 && !utf8.ValidString(markdown[:end]) { + end-- + } + return strings.TrimRight(markdown[:end], "\r\n") +} diff --git a/pkg/ci/plugins/helm/aggregate_test.go b/pkg/ci/plugins/helm/aggregate_test.go new file mode 100644 index 0000000000..8c9e492df3 --- /dev/null +++ b/pkg/ci/plugins/helm/aggregate_test.go @@ -0,0 +1,167 @@ +package helm + +import ( + "errors" + "strings" + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/ci/internal/plugin" + "github.com/cloudposse/atmos/pkg/schema" +) + +func TestOnAfterAggregateRendersPlanSummary(t *testing.T) { + writer := &fakeWriter{} + ctx := &plugin.HookContext{ + Provider: fakeProvider{writer: writer}, + Aggregate: schema.HelmCIResultSet{ + Command: "plan", + Results: []schema.HelmCIResult{ + { + NodeID: "changed", Stack: "prod", Component: "api", Processed: true, DurationMS: 25, + Summary: map[string]any{ + "chart": "oci://registry/api", "release_name": "api", "namespace": "apps", + "target": "kubernetes", "diff": "+ kind: Deployment", + }, + }, + { + NodeID: "unchanged", Stack: "dev", Component: "web", Processed: true, + Summary: map[string]any{"chart": "web", "release_name": "web", "namespace": "apps", "target": "git"}, + }, + { + NodeID: "failed", Stack: "dev", Component: "db", ExitCode: 1, Error: "render failed", + Summary: map[string]any{"chart": "db"}, + }, + {NodeID: "skipped", Stack: "prod", Component: "worker"}, + }, + }, + } + + err := (&Plugin{}).onAfterAggregate(ctx) + require.NoError(t, err) + assert.Contains(t, writer.summary, "## Helm Plan Summary") + assert.Contains(t, writer.summary, "Processed 4 component(s): 1 changed, 1 unchanged, 1 failed, 1 skipped.") + assert.Contains(t, writer.summary, "| dev | db | failed | db |") + assert.Contains(t, writer.summary, "| dev | web | no changes | web | web | apps | git | - |") + assert.Contains(t, writer.summary, "| prod | api | changed | oci://registry/api | api | apps | kubernetes | 25ms |") + assert.Contains(t, writer.summary, "+ kind: Deployment") + assert.Contains(t, writer.summary, "render failed") + assert.Less(t, strings.Index(writer.summary, "| dev | db |"), strings.Index(writer.summary, "| prod | api |")) +} + +func TestOnAfterAggregateRendersApplySummary(t *testing.T) { + writer := &fakeWriter{} + err := (&Plugin{}).onAfterAggregate(&plugin.HookContext{ + Provider: fakeProvider{writer: writer}, + Aggregate: &schema.HelmCIResultSet{ + Command: "deploy", + Results: []schema.HelmCIResult{{ + Stack: "dev", Component: "api", Processed: true, + Summary: map[string]any{ + "chart": "api", "release_name": "api", "namespace": "apps", "target": "kubernetes", + }, + }}, + }, + }) + require.NoError(t, err) + assert.Contains(t, writer.summary, "## Helm Apply Summary") + assert.Contains(t, writer.summary, "Processed 1 component(s): 1 succeeded, 0 failed, 0 skipped.") + assert.Contains(t, writer.summary, "| dev | api | succeeded | api | api | apps | kubernetes | - |") +} + +func TestOnAfterAggregateUsesSafeDetailFences(t *testing.T) { + writer := &fakeWriter{} + err := (&Plugin{}).onAfterAggregate(&plugin.HookContext{ + Provider: fakeProvider{writer: writer}, + Aggregate: schema.HelmCIResultSet{ + Command: "plan", + Results: []schema.HelmCIResult{ + { + Stack: "dev", Component: "api", Processed: true, + Summary: map[string]any{"diff": "+ change\n```\nnot summary Markdown"}, + }, + { + Stack: "dev", Component: "worker", Status: "failed", + Error: "render failed\n```\nnot summary Markdown", + }, + }, + }, + }) + require.NoError(t, err) + assert.Contains(t, writer.summary, "````diff\n+ change\n```\nnot summary Markdown\n````") + assert.Contains(t, writer.summary, "````text\nrender failed\n```\nnot summary Markdown\n````") +} + +func TestOnAfterAggregateSkipsInvalidOrDisabledAndReturnsWriterError(t *testing.T) { + pluginUnderTest := &Plugin{} + require.NoError(t, pluginUnderTest.onAfterAggregate(&plugin.HookContext{Provider: fakeProvider{}, Aggregate: "invalid"})) + require.NoError(t, pluginUnderTest.onAfterAggregate(&plugin.HookContext{ + Provider: fakeProvider{}, + Aggregate: schema.HelmCIResultSet{}, + })) + + disabled := false + writer := &fakeWriter{} + require.NoError(t, pluginUnderTest.onAfterAggregate(&plugin.HookContext{ + Config: &schema.AtmosConfiguration{CI: schema.CIConfig{Summary: schema.CISummaryConfig{Enabled: &disabled}}}, + Provider: fakeProvider{writer: writer}, + Aggregate: schema.HelmCIResultSet{Results: []schema.HelmCIResult{{Processed: true}}}, + })) + assert.Empty(t, writer.summary) + + sentinel := errors.New("write failed") + err := pluginUnderTest.onAfterAggregate(&plugin.HookContext{ + Provider: fakeProvider{writer: &fakeWriter{err: sentinel}}, + Aggregate: schema.HelmCIResultSet{Results: []schema.HelmCIResult{{Processed: true}}}, + }) + require.ErrorIs(t, err, sentinel) +} + +func TestOnAfterAggregateRendersFailureWithoutResults(t *testing.T) { + writer := &fakeWriter{} + err := (&Plugin{}).onAfterAggregate(&plugin.HookContext{ + Provider: fakeProvider{writer: writer}, + Aggregate: schema.HelmCIResultSet{Command: "apply"}, + CommandError: errors.New("dependency graph contains a cycle"), + ExitCode: 1, + }) + require.NoError(t, err) + assert.Contains(t, writer.summary, "## Helm Apply Summary") + assert.Contains(t, writer.summary, "Command failed before any components were processed.") + assert.Contains(t, writer.summary, "dependency graph contains a cycle") +} + +func TestHelmAggregateHelpers(t *testing.T) { + resultSet := schema.HelmCIResultSet{Command: "diff"} + assert.Equal(t, resultSet, mustNormalizeHelmAggregate(t, resultSet)) + assert.Equal(t, resultSet, mustNormalizeHelmAggregate(t, &resultSet)) + _, ok := normalizeHelmAggregate((*schema.HelmCIResultSet)(nil)) + assert.False(t, ok) + _, ok = normalizeHelmAggregate("invalid") + assert.False(t, ok) + + assert.Equal(t, "plan", normalizeHelmAggregateCommand("diff")) + assert.Equal(t, "apply", normalizeHelmAggregateCommand("deploy")) + assert.Equal(t, "-", formatHelmAggregateDuration(0)) + assert.Equal(t, "15ms", formatHelmAggregateDuration(15)) + assert.Equal(t, `a\|b c`, helmMarkdownCell("a|b\nc")) + + oversized := strings.Repeat("x", helmAggregateMarkdownMaxBytes+100) + truncated := enforceHelmAggregateMarkdownLimit(oversized) + assert.LessOrEqual(t, len(truncated), helmAggregateMarkdownMaxBytes) + assert.Contains(t, truncated, "Summary truncated") + + unicodeOversized := strings.Repeat("界", helmAggregateMarkdownMaxBytes) + unicodeTruncated := enforceHelmAggregateMarkdownLimit(unicodeOversized) + assert.True(t, utf8.ValidString(unicodeTruncated)) +} + +func mustNormalizeHelmAggregate(t *testing.T, value any) schema.HelmCIResultSet { + t.Helper() + result, ok := normalizeHelmAggregate(value) + require.True(t, ok) + return result +} diff --git a/pkg/ci/plugins/helm/plugin.go b/pkg/ci/plugins/helm/plugin.go index abc597a5df..186678b2d1 100644 --- a/pkg/ci/plugins/helm/plugin.go +++ b/pkg/ci/plugins/helm/plugin.go @@ -40,8 +40,10 @@ func (p *Plugin) GetHookBindings() []plugin.HookBinding { return []plugin.HookBinding{ {Event: "after.helm.template", Handler: p.onAfterOperation}, {Event: "after.helm.diff", Handler: p.onAfterOperation}, + {Event: "after.helm.plan.aggregate", Handler: p.onAfterAggregate}, {Event: "after.helm.apply", Handler: p.onAfterOperation}, {Event: "after.helm.deploy", Handler: p.onAfterOperation}, + {Event: "after.helm.apply.aggregate", Handler: p.onAfterAggregate}, {Event: "after.helm.delete", Handler: p.onAfterOperation}, } } @@ -118,6 +120,7 @@ func (p *Plugin) buildTemplateContext(ctx *plugin.HookContext) *TemplateContext ObjectCount: data.ObjectCount, ObjectKinds: data.ObjectKinds, ManifestBytes: data.ManifestBytes, + Lifecycle: data.Lifecycle, Message: data.Message, Diff: plugin.TruncateDetail(data.Diff), } @@ -135,6 +138,7 @@ type TemplateContext struct { ObjectCount int ObjectKinds []string ManifestBytes int + Lifecycle map[string]any Message string // Diff is the unified diff produced by `helm diff`/`plan` (empty otherwise). Diff string @@ -152,6 +156,7 @@ type Summary struct { ObjectCount int ObjectKinds []string ManifestBytes int + Lifecycle map[string]any Message string // Diff is the unified diff produced by `helm diff`/`plan` (empty otherwise). Diff string @@ -185,6 +190,7 @@ func summaryFromMap(m map[string]any) Summary { ObjectCount: intValue(m["object_count"]), ObjectKinds: stringSliceValue(m["object_kinds"]), ManifestBytes: intValue(m["manifest_bytes"]), + Lifecycle: mapValue(m["release"]), Message: stringValue(m["message"]), Diff: stringValue(m["diff"]), } @@ -192,6 +198,18 @@ func summaryFromMap(m map[string]any) Summary { return s } +func mapValue(value any) map[string]any { + typed, ok := value.(map[string]any) + if !ok { + return nil + } + result := make(map[string]any, len(typed)) + for key, item := range typed { + result[key] = item + } + return result +} + func helmTemplateName(command string) string { switch command { case "render": diff --git a/pkg/ci/plugins/helm/plugin_test.go b/pkg/ci/plugins/helm/plugin_test.go index 83f212a4c9..73b09aea31 100644 --- a/pkg/ci/plugins/helm/plugin_test.go +++ b/pkg/ci/plugins/helm/plugin_test.go @@ -20,13 +20,15 @@ func TestPlugin_GetType(t *testing.T) { func TestPlugin_GetHookBindings(t *testing.T) { bindings := (&Plugin{}).GetHookBindings() - require.Len(t, bindings, 5) + require.Len(t, bindings, 7) for _, event := range []string{ "after.helm.template", "after.helm.diff", + "after.helm.plan.aggregate", "after.helm.apply", "after.helm.deploy", + "after.helm.apply.aggregate", "after.helm.delete", } { t.Run(event, func(t *testing.T) { @@ -50,7 +52,7 @@ func TestPlugin_BuildTemplateContext(t *testing.T) { Command: "deploy", Info: &schema.ConfigAndStacksInfo{ ComponentFromArg: "nginx", - Stack: "plat-ue2-dev", + Stack: "dev", }, Aggregate: map[string]any{ "chart": "bitnami/nginx", @@ -60,11 +62,15 @@ func TestPlugin_BuildTemplateContext(t *testing.T) { "object_count": 2, "object_kinds": []any{"Service", "Deployment"}, "manifest_bytes": 1234, + "release": map[string]any{ + "operation": "upgrade", + "wait": map[string]any{"strategy": "watcher"}, + }, }, }) assert.Equal(t, "nginx", ctx.Component) - assert.Equal(t, "plat-ue2-dev", ctx.Stack) + assert.Equal(t, "dev", ctx.Stack) assert.Equal(t, "deploy", ctx.Command) assert.Equal(t, "bitnami/nginx", ctx.Chart) assert.Equal(t, "nginx", ctx.ReleaseName) @@ -73,6 +79,8 @@ func TestPlugin_BuildTemplateContext(t *testing.T) { assert.Equal(t, 2, ctx.ObjectCount) assert.Equal(t, 1234, ctx.ManifestBytes) assert.Equal(t, []string{"Deployment", "Service"}, ctx.ObjectKinds) + assert.Equal(t, "upgrade", ctx.Lifecycle["operation"]) + assert.Equal(t, "watcher", ctx.Lifecycle["wait"].(map[string]any)["strategy"]) } func TestNormalizeSummary(t *testing.T) { @@ -81,6 +89,7 @@ func TestNormalizeSummary(t *testing.T) { assert.Equal(t, Summary{}, normalizeSummary((*Summary)(nil))) assert.Equal(t, Summary{}, normalizeSummary("not-summary")) + lifecycle := map[string]any{"operation": "install"} got := normalizeSummary(map[string]any{ "component": "app", "stack": "dev", @@ -94,6 +103,7 @@ func TestNormalizeSummary(t *testing.T) { "manifest_bytes": float64(123), "message": 42, "diff": "diff text", + "release": lifecycle, }) assert.Equal(t, "app", got.Component) assert.Equal(t, "dev", got.Stack) @@ -107,6 +117,12 @@ func TestNormalizeSummary(t *testing.T) { assert.Equal(t, 123, got.ManifestBytes) assert.Equal(t, "42", got.Message) assert.Equal(t, "diff text", got.Diff) + assert.Equal(t, map[string]any{"operation": "install"}, got.Lifecycle) + + lifecycle["operation"] = "upgrade" + assert.Equal(t, "install", got.Lifecycle["operation"]) + got.Lifecycle["timeout"] = "30m0s" + assert.NotContains(t, lifecycle, "timeout") } func TestPluginBuildTemplateContextFallbacksAndErrors(t *testing.T) { @@ -146,6 +162,7 @@ func TestSummaryEnabledAndPrimitiveConversions(t *testing.T) { assert.Zero(t, intValue("9")) assert.Equal(t, []string{"b", "a"}, stringSliceValue([]string{"b", "a"})) assert.Nil(t, stringSliceValue(1)) + assert.Nil(t, mapValue("not-a-map")) assert.Equal(t, "Helm", title("")) } @@ -227,26 +244,137 @@ func TestPluginOnAfterOperation(t *testing.T) { } func TestTemplateRendering(t *testing.T) { - ctx := (&Plugin{}).buildTemplateContext(&plugin.HookContext{ - Command: "apply", - Info: &schema.ConfigAndStacksInfo{ - ComponentFromArg: "nginx", - Stack: "plat-ue2-dev", + tests := []struct { + name string + command string + lifecycle map[string]any + contains []string + notContains []string + }{ + { + name: "cluster apply", + command: "apply", + lifecycle: map[string]any{ + "operation": "upgrade", + "wait": map[string]any{"strategy": "watcher", "jobs": true}, + "timeout": "30m0s", + "chart_hooks": true, + "on_failure": "rollback", + "cleanup_on_failure": true, + "history": map[string]any{"max": 10}, + }, + contains: []string{ + "Helm Apply Summary", "bitnami/nginx", "Deployment", "\n### Release lifecycle\n", + "\n| Operation | `upgrade` |\n", + "\n| Wait strategy | `watcher` |\n", + "| Timeout | `30m0s` |", + "| Chart hooks enabled | `true` |", + "| Wait for Jobs | `true` |", + "| On failure | `rollback` |\n" + + "| Cleanup on failure | `true` |\n" + + "| Maximum history | `10` |", + }, }, - Aggregate: Summary{ - Chart: "bitnami/nginx", - ReleaseName: "nginx", - Namespace: "apps", - Target: "kubernetes", - ObjectCount: 2, - ObjectKinds: []string{"Deployment", "Service"}, - ManifestBytes: 1234, + { + name: "cluster install", + command: "apply", + lifecycle: map[string]any{ + "operation": "install", + "wait": map[string]any{"strategy": "hookOnly", "jobs": false}, + "timeout": "5m0s", + "chart_hooks": true, + "on_failure": "keep", + "crds": "create", + }, + contains: []string{ + "Helm Apply Summary", "\n### Release lifecycle\n", + "\n| Operation | `install` |\n", + "| Wait strategy | `hookOnly` |", + "| Timeout | `5m0s` |", + "| Chart hooks enabled | `true` |", + "| Wait for Jobs | `false` |", + "| On failure | `keep` |\n" + + "| Install CRDs | `create` |", + }, + notContains: []string{"Maximum history"}, }, - }) + { + name: "external apply", + command: "apply", + lifecycle: map[string]any{ + "applied": false, "target_kind": "git", "reason": "external_target", + }, + contains: []string{ + "Helm Apply Summary", + "\n### Release lifecycle\n", + "\n| Applied | `false` |\n", + "| Target kind | `git` |", + "| Reason | `external_target` |", + }, + notContains: []string{"Wait strategy", "Timeout"}, + }, + { + name: "cluster delete", + command: "delete", + lifecycle: map[string]any{ + "operation": "delete", + "wait": map[string]any{"strategy": "legacy"}, + "timeout": "10m0s", + "chart_hooks": false, + }, + contains: []string{ + "Helm Delete Summary", "\n### Release lifecycle\n", + "\n| Operation | `delete` |\n", + "| Wait strategy | `legacy` |", + "| Timeout | `10m0s` |", + "| Chart hooks enabled | `false` |", + }, + }, + { + name: "external delete", + command: "delete", + lifecycle: map[string]any{ + "deleted": false, "target_kind": "git", "reason": "external_target", + }, + contains: []string{ + "Helm Delete Summary", + "\n### Release lifecycle\n", + "\n| Deleted | `false` |\n", + "| Target kind | `git` |", + "| Reason | `external_target` |", + }, + notContains: []string{"Wait strategy", "Timeout"}, + }, + } - rendered, err := templates.NewLoader(nil).LoadAndRender("helm", "apply", defaultTemplates, ctx) - require.NoError(t, err) - assert.Contains(t, rendered, "Helm Apply Summary") - assert.Contains(t, rendered, "bitnami/nginx") - assert.Contains(t, rendered, "Deployment") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := (&Plugin{}).buildTemplateContext(&plugin.HookContext{ + Command: tt.command, + Info: &schema.ConfigAndStacksInfo{ + ComponentFromArg: "nginx", + Stack: "dev", + }, + Aggregate: Summary{ + Chart: "bitnami/nginx", + ReleaseName: "nginx", + Namespace: "apps", + Target: "kubernetes", + ObjectCount: 2, + ObjectKinds: []string{"Deployment", "Service"}, + ManifestBytes: 1234, + Lifecycle: tt.lifecycle, + }, + }) + + rendered, err := templates.NewLoader(nil).LoadAndRender("helm", tt.command, defaultTemplates, ctx) + require.NoError(t, err) + for _, expected := range tt.contains { + assert.Contains(t, rendered, expected) + } + for _, unexpected := range tt.notContains { + assert.NotContains(t, rendered, unexpected) + } + }) + } } diff --git a/pkg/ci/plugins/helm/templates/apply.md b/pkg/ci/plugins/helm/templates/apply.md index ae3d8b6e3a..1e9e48bc39 100644 --- a/pkg/ci/plugins/helm/templates/apply.md +++ b/pkg/ci/plugins/helm/templates/apply.md @@ -14,6 +14,51 @@ | Objects | `{{ .ObjectCount }}` | | Manifest bytes | `{{ .ManifestBytes }}` | +{{ with .Lifecycle }} + +### Release lifecycle + +{{ if eq (index . "reason") "external_target" }} + +| Field | Value | +| --- | --- | +| Applied | `false` | +| Target kind | `{{ index . "target_kind" }}` | +| Reason | `external_target` | + +{{ else }} + +{{ if eq (index . "operation") "install" }} + +| Field | Value | +| --- | --- | +| Operation | `{{ index . "operation" }}` | +| Wait strategy | `{{ index (index . "wait") "strategy" }}` | +| Timeout | `{{ index . "timeout" }}` | +| Chart hooks enabled | `{{ index . "chart_hooks" }}` | +| Wait for Jobs | `{{ index (index . "wait") "jobs" }}` | +| On failure | `{{ index . "on_failure" }}` | +| Install CRDs | `{{ index . "crds" }}` | + +{{ else }} + +| Field | Value | +| --- | --- | +| Operation | `{{ index . "operation" }}` | +| Wait strategy | `{{ index (index . "wait") "strategy" }}` | +| Timeout | `{{ index . "timeout" }}` | +| Chart hooks enabled | `{{ index . "chart_hooks" }}` | +| Wait for Jobs | `{{ index (index . "wait") "jobs" }}` | +| On failure | `{{ index . "on_failure" }}` | +| Cleanup on failure | `{{ index . "cleanup_on_failure" }}` | +| Maximum history | `{{ index (index . "history") "max" }}` | + +{{ end }} + +{{ end }} + +{{ end }} + To reproduce locally: ```shell diff --git a/pkg/ci/plugins/helm/templates/delete.md b/pkg/ci/plugins/helm/templates/delete.md index 88577fab0f..c491cb3a22 100644 --- a/pkg/ci/plugins/helm/templates/delete.md +++ b/pkg/ci/plugins/helm/templates/delete.md @@ -10,6 +10,31 @@ | Release | `{{ .ReleaseName }}` | | Namespace | `{{ .Namespace }}` | +{{ with .Lifecycle }} + +### Release lifecycle + +{{ if eq (index . "reason") "external_target" }} + +| Field | Value | +| --- | --- | +| Deleted | `false` | +| Target kind | `{{ index . "target_kind" }}` | +| Reason | `external_target` | + +{{ else }} + +| Field | Value | +| --- | --- | +| Operation | `{{ index . "operation" }}` | +| Wait strategy | `{{ index (index . "wait") "strategy" }}` | +| Timeout | `{{ index . "timeout" }}` | +| Chart hooks enabled | `{{ index . "chart_hooks" }}` | + +{{ end }} + +{{ end }} + To reproduce locally: ```shell diff --git a/pkg/component/graph.go b/pkg/component/graph.go index 78187783ff..c3485a52f3 100644 --- a/pkg/component/graph.go +++ b/pkg/component/graph.go @@ -37,6 +37,13 @@ type GraphExecutionOptions struct { SubCommand string Flags map[string]any Selection *GraphSelection + ReverseOrder bool +} + +// GraphNodeSkipObserver is implemented by providers that need to record graph +// nodes skipped after execution stops before reaching them. +type GraphNodeSkipObserver interface { + OnGraphNodeSkipped(node *dependency.Node) } // ExecuteGraph runs selected components in dependency order and stops before @@ -59,21 +66,28 @@ func ExecuteGraph(ctx context.Context, opts *GraphExecutionOptions) error { return nil } - log.Info("Processing components in dependency order", "component_type", opts.ComponentType, "count", len(order)) + orderName := "dependency" + if opts.ReverseOrder { + orderName = "reverse_dependency" + } + log.Info("Processing components", "component_type", opts.ComponentType, "order", orderName, "count", len(order)) for i := range order { select { case <-ctx.Done(): + notifyGraphNodeSkips(opts.Provider, order[i:]) return fmt.Errorf(errUtils.ErrWrapFormat, errUtils.ErrGraphExecutionCanceled, ctx.Err()) default: } if err := executeGraphNode(ctx, opts, &order[i]); err != nil { + notifyGraphNodeSkips(opts.Provider, order[i+1:]) if ctxErr := ctx.Err(); ctxErr != nil { return fmt.Errorf(errUtils.ErrWrapFormat, errUtils.ErrGraphExecutionCanceled, errors.Join(ctxErr, err)) } return err } if ctxErr := ctx.Err(); ctxErr != nil { + notifyGraphNodeSkips(opts.Provider, order[i+1:]) return fmt.Errorf(errUtils.ErrWrapFormat, errUtils.ErrGraphExecutionCanceled, ctxErr) } } @@ -81,6 +95,16 @@ func ExecuteGraph(ctx context.Context, opts *GraphExecutionOptions) error { return nil } +func notifyGraphNodeSkips(provider ComponentProvider, nodes dependency.ExecutionOrder) { + observer, ok := provider.(GraphNodeSkipObserver) + if !ok { + return + } + for i := range nodes { + observer.OnGraphNodeSkipped(&nodes[i]) + } +} + // prepareExecutionOrder validates options, builds and filters the graph, and returns // the topologically sorted execution order. An empty order indicates no matching components. func prepareExecutionOrder(opts *GraphExecutionOptions) (dependency.ExecutionOrder, error) { @@ -108,9 +132,18 @@ func prepareExecutionOrder(opts *GraphExecutionOptions) (dependency.ExecutionOrd if err != nil { return nil, fmt.Errorf("%w: %w", errUtils.ErrTopologicalOrder, err) } + if opts.ReverseOrder { + reverseExecutionOrder(order) + } return order, nil } +func reverseExecutionOrder(order dependency.ExecutionOrder) { + for left, right := 0, len(order)-1; left < right; left, right = left+1, right-1 { + order[left], order[right] = order[right], order[left] + } +} + // executeGraphNode executes a single graph node through the component provider. func executeGraphNode(ctx context.Context, opts *GraphExecutionOptions, node *dependency.Node) error { nodeInfo := *opts.Info diff --git a/pkg/component/graph_test.go b/pkg/component/graph_test.go index 6eb080bde5..dd9ae1b7b9 100644 --- a/pkg/component/graph_test.go +++ b/pkg/component/graph_test.go @@ -164,6 +164,23 @@ func TestExecuteGraphRunsComponentsInDependencyOrder(t *testing.T) { } } +func TestExecuteGraphRunsDeleteInReverseDependencyOrder(t *testing.T) { + provider := &graphTestProvider{} + err := ExecuteGraph(context.Background(), &GraphExecutionOptions{ + Provider: provider, + Info: &schema.ConfigAndStacksInfo{}, + Stacks: graphTestStacks(), + ComponentType: cfg.KubernetesComponentType, + SubCommand: "delete", + ReverseOrder: true, + }) + + require.NoError(t, err) + require.Len(t, provider.calls, 4) + assertLessCallIndex(t, provider.calls, "api", "dev", "base", "dev") + assertLessCallIndex(t, provider.calls, "worker", "dev", "base", "prod") +} + func TestExecuteGraphNodeDoesNotRedispatchBulkSelection(t *testing.T) { provider := &graphTestProvider{} info := &schema.ConfigAndStacksInfo{ diff --git a/pkg/component/helm/aggregate_ci.go b/pkg/component/helm/aggregate_ci.go new file mode 100644 index 0000000000..c52db7bb33 --- /dev/null +++ b/pkg/component/helm/aggregate_ci.go @@ -0,0 +1,212 @@ +package helm + +import ( + "sort" + "sync" + "time" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/component" + "github.com/cloudposse/atmos/pkg/dependency" + "github.com/cloudposse/atmos/pkg/hooks" + log "github.com/cloudposse/atmos/pkg/logger" + "github.com/cloudposse/atmos/pkg/schema" +) + +const helmBulkCICollectorFlag = "_helm_bulk_ci_collector" + +// helmBulkCICollector accumulates one result per graph node so bulk Helm +// commands can write a single CI summary after execution completes. +type helmBulkCICollector struct { + mu sync.Mutex + command string + results map[string]*schema.HelmCIResult +} + +func newHelmBulkCICollector(command string) *helmBulkCICollector { + return &helmBulkCICollector{ + command: command, + results: make(map[string]*schema.HelmCIResult), + } +} + +// bulkCollectingProvider wraps the native Helm provider and records failures +// that happen before an operation produces its structured summary. +type bulkCollectingProvider struct { + component.ComponentProvider + collector *helmBulkCICollector +} + +func (p *bulkCollectingProvider) Execute(ctx *component.ExecutionContext) error { + startedAt := time.Now() + err := p.ComponentProvider.Execute(ctx) + p.collector.finish(ctx, startedAt, time.Now(), err) + return err +} + +func (p *bulkCollectingProvider) OnGraphNodeSkipped(node *dependency.Node) { + if p == nil || p.collector == nil || node == nil { + return + } + p.collector.markSkipped(node.Stack, node.Component) +} + +func helmBulkCollector(ctx *component.ExecutionContext) *helmBulkCICollector { + if ctx == nil { + return nil + } + collector, _ := ctx.Flags[helmBulkCICollectorFlag].(*helmBulkCICollector) + return collector +} + +func (c *helmBulkCICollector) setSummary(info *schema.ConfigAndStacksInfo, summary map[string]any, operationErr error) { + if c == nil || info == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + + result := c.ensure(info.Stack, info.ComponentFromArg) + result.Processed = true + result.Summary = cloneHelmSummary(summary) + applyHelmResultError(result, operationErr) +} + +func (c *helmBulkCICollector) finish(ctx *component.ExecutionContext, startedAt, finishedAt time.Time, execErr error) { + if c == nil || ctx == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + + componentName := ctx.ConfigAndStacksInfo.ComponentFromArg + if componentName == "" { + componentName = ctx.Component + } + result := c.ensure(ctx.Stack, componentName) + result.StartedAt = startedAt + result.FinishedAt = finishedAt + result.DurationMS = finishedAt.Sub(startedAt).Milliseconds() + applyHelmResultError(result, execErr) +} + +func (c *helmBulkCICollector) markSkipped(stack, componentName string) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + + result := c.ensure(stack, componentName) + result.Status = "skipped" +} + +func (c *helmBulkCICollector) ensure(stack, componentName string) *schema.HelmCIResult { + nodeID := component.GraphNodeID(componentName, stack) + if result, ok := c.results[nodeID]; ok { + return result + } + result := &schema.HelmCIResult{ + NodeID: nodeID, + Stack: stack, + Component: componentName, + Status: "succeeded", + } + c.results[nodeID] = result + return result +} + +func (c *helmBulkCICollector) resultSet() schema.HelmCIResultSet { + if c == nil { + return schema.HelmCIResultSet{} + } + c.mu.Lock() + defer c.mu.Unlock() + + results := make([]schema.HelmCIResult, 0, len(c.results)) + for _, result := range c.results { + copyResult := *result + copyResult.Summary = cloneHelmSummary(result.Summary) + results = append(results, copyResult) + } + sort.SliceStable(results, func(i, j int) bool { + if results[i].Stack != results[j].Stack { + return results[i].Stack < results[j].Stack + } + if results[i].Component != results[j].Component { + return results[i].Component < results[j].Component + } + return results[i].NodeID < results[j].NodeID + }) + return schema.HelmCIResultSet{Command: c.command, Results: results} +} + +func applyHelmResultError(result *schema.HelmCIResult, err error) { + if result == nil || err == nil { + return + } + result.Status = "failed" + result.ExitCode = errUtils.GetExitCode(err) + result.Error = err.Error() +} + +func cloneHelmSummary(summary map[string]any) map[string]any { + if summary == nil { + return nil + } + cloned := make(map[string]any, len(summary)) + for key, value := range summary { + cloned[key] = cloneHelmSummaryValue(value) + } + return cloned +} + +func cloneHelmSummaryValue(value any) any { + switch typed := value.(type) { + case map[string]any: + return cloneHelmSummary(typed) + case []any: + cloned := make([]any, len(typed)) + for i := range typed { + cloned[i] = cloneHelmSummaryValue(typed[i]) + } + return cloned + case []string: + return append([]string(nil), typed...) + default: + return value + } +} + +func supportsHelmAggregateCI(command string) bool { + switch command { + case "plan", "diff", "apply", "deploy": + return true + default: + return false + } +} + +func runHelmAggregateCIHook( + ctx *component.ExecutionContext, + atmosConfig *schema.AtmosConfiguration, + info *schema.ConfigAndStacksInfo, + resultSet schema.HelmCIResultSet, + commandErr error, +) { + event := hooks.AfterHelmPlanAggregate + if resultSet.Command == "apply" || resultSet.Command == "deploy" { + event = hooks.AfterHelmApplyAggregate + } + if err := runCIHooks(&hooks.RunCIHooksOptions{ + Event: event, + AtmosConfig: atmosConfig, + Info: info, + ForceCIMode: helmCIModeEnabled(ctx.Flags), + CommandError: commandErr, + ExitCode: errUtils.GetExitCode(commandErr), + Aggregate: resultSet, + }); err != nil { + log.Warn("Helm CI aggregate hook failed", "command", resultSet.Command, "error", err) + } +} diff --git a/pkg/component/helm/aggregate_ci_test.go b/pkg/component/helm/aggregate_ci_test.go new file mode 100644 index 0000000000..366b58bcfc --- /dev/null +++ b/pkg/component/helm/aggregate_ci_test.go @@ -0,0 +1,243 @@ +package helm + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/auth" + "github.com/cloudposse/atmos/pkg/component" + cfg "github.com/cloudposse/atmos/pkg/config" + "github.com/cloudposse/atmos/pkg/hooks" + "github.com/cloudposse/atmos/pkg/schema" +) + +func TestHelmBulkCICollectorRecordsAndSortsResults(t *testing.T) { + collector := newHelmBulkCICollector("plan") + collector.setSummary(&schema.ConfigAndStacksInfo{Stack: "prod", ComponentFromArg: "api"}, map[string]any{"diff": "+ change"}, nil) + collector.finish(&component.ExecutionContext{Stack: "prod", Component: "api"}, time.Unix(1, 0), time.Unix(1, int64(25*time.Millisecond)), nil) + + sentinel := errors.New("render failed") + collector.finish(&component.ExecutionContext{Stack: "dev", Component: "db"}, time.Unix(2, 0), time.Unix(2, int64(time.Millisecond)), sentinel) + + resultSet := collector.resultSet() + assert.Equal(t, "plan", resultSet.Command) + require.Len(t, resultSet.Results, 2) + assert.Equal(t, "db", resultSet.Results[0].Component) + assert.False(t, resultSet.Results[0].Processed) + assert.Equal(t, "failed", resultSet.Results[0].Status) + assert.Equal(t, "render failed", resultSet.Results[0].Error) + assert.Equal(t, "api", resultSet.Results[1].Component) + assert.True(t, resultSet.Results[1].Processed) + assert.Equal(t, int64(25), resultSet.Results[1].DurationMS) + assert.Equal(t, "+ change", resultSet.Results[1].Summary["diff"]) + + resultSet.Results[1].Summary["diff"] = "mutated" + assert.Equal(t, "+ change", collector.resultSet().Results[1].Summary["diff"]) +} + +func TestHelmBulkCICollectorDeepCopiesNestedSummary(t *testing.T) { + collector := newHelmBulkCICollector("apply") + wait := map[string]any{"strategy": "watcher"} + kinds := []string{"Deployment", "Service"} + summary := map[string]any{ + "release": map[string]any{"wait": wait}, + "object_kinds": kinds, + } + collector.setSummary(&schema.ConfigAndStacksInfo{Stack: "dev", ComponentFromArg: "api"}, summary, nil) + + wait["strategy"] = "legacy" + kinds[0] = "Job" + result := collector.resultSet().Results[0] + release := result.Summary["release"].(map[string]any) + assert.Equal(t, "watcher", release["wait"].(map[string]any)["strategy"]) + assert.Equal(t, []string{"Deployment", "Service"}, result.Summary["object_kinds"]) + + release["wait"].(map[string]any)["strategy"] = "mutated" + result.Summary["object_kinds"].([]string)[1] = "ConfigMap" + retained := collector.resultSet().Results[0].Summary + assert.Equal(t, "watcher", retained["release"].(map[string]any)["wait"].(map[string]any)["strategy"]) + assert.Equal(t, []string{"Deployment", "Service"}, retained["object_kinds"]) +} + +type helmBulkGraphTestProvider struct { + failComponent string +} + +func (p *helmBulkGraphTestProvider) GetType() string { return cfg.HelmComponentType } +func (p *helmBulkGraphTestProvider) GetGroup() string { return "test" } +func (p *helmBulkGraphTestProvider) GetBasePath(*schema.AtmosConfiguration) string { return "" } +func (p *helmBulkGraphTestProvider) ListComponents(context.Context, string, map[string]any) ([]string, error) { + return nil, nil +} +func (p *helmBulkGraphTestProvider) ValidateComponent(map[string]any) error { return nil } +func (p *helmBulkGraphTestProvider) Execute(ctx *component.ExecutionContext) error { + if ctx.Component == p.failComponent { + return errors.New("operation failed") + } + return nil +} +func (p *helmBulkGraphTestProvider) GenerateArtifacts(*component.ExecutionContext) error { return nil } +func (p *helmBulkGraphTestProvider) GetAvailableCommands() []string { return nil } + +func TestHelmBulkCICollectorRecordsDependencyBlockedComponents(t *testing.T) { + collector := newHelmBulkCICollector("apply") + provider := &bulkCollectingProvider{ + ComponentProvider: &helmBulkGraphTestProvider{failComponent: "base"}, + collector: collector, + } + stacks := map[string]any{ + "dev": map[string]any{ + cfg.ComponentsSectionName: map[string]any{ + cfg.HelmComponentType: map[string]any{ + "base": map[string]any{}, + "api": map[string]any{ + cfg.SettingsSectionName: map[string]any{"depends_on": []any{"base"}}, + }, + }, + }, + }, + } + + err := component.ExecuteGraph(context.Background(), &component.GraphExecutionOptions{ + Provider: provider, + Info: &schema.ConfigAndStacksInfo{}, + Stacks: stacks, + ComponentType: cfg.HelmComponentType, + SubCommand: "apply", + }) + require.Error(t, err) + + results := collector.resultSet().Results + require.Len(t, results, 2) + assert.Equal(t, "api", results[0].Component) + assert.Equal(t, "skipped", results[0].Status) + assert.False(t, results[0].Processed) + assert.Equal(t, "base", results[1].Component) + assert.Equal(t, "failed", results[1].Status) +} + +func TestHelmBulkCICollectorUsesProcessedComponentIdentity(t *testing.T) { + collector := newHelmBulkCICollector("plan") + info := schema.ConfigAndStacksInfo{Stack: "dev", ComponentFromArg: "apps/app"} + collector.setSummary(&info, map[string]any{"diff": "+ change"}, nil) + collector.finish(&component.ExecutionContext{ + Stack: "dev", + Component: "app", + ConfigAndStacksInfo: info, + }, time.Unix(1, 0), time.Unix(1, int64(time.Millisecond)), nil) + + results := collector.resultSet().Results + require.Len(t, results, 1) + assert.Equal(t, "apps/app", results[0].Component) + assert.True(t, results[0].Processed) + assert.Equal(t, int64(1), results[0].DurationMS) + assert.Equal(t, "+ change", results[0].Summary["diff"]) +} + +func TestRunWithHooksBulkCollectorSuppressesPerComponentCI(t *testing.T) { + originalHooks := getHooks + originalApply := applyHelmRelease + originalCI := runCIHooks + t.Cleanup(func() { + getHooks = originalHooks + applyHelmRelease = originalApply + runCIHooks = originalCI + }) + + getHooks = func(*schema.AtmosConfiguration, *schema.ConfigAndStacksInfo) (*hooks.Hooks, error) { + return &hooks.Hooks{}, nil + } + applyHelmRelease = func(context.Context, *chartSpec, bool) (releaseActionResult, error) { + return releaseActionResult{Manifest: helmExecutorManifest, Operation: releaseOperationInstall}, nil + } + ciCalls := 0 + runCIHooks = func(*hooks.RunCIHooksOptions) error { + ciCalls++ + return nil + } + + collector := newHelmBulkCICollector("apply") + ctx := &component.ExecutionContext{Flags: map[string]any{helmBulkCICollectorFlag: collector}} + info := &schema.ConfigAndStacksInfo{ + Stack: "dev", ComponentFromArg: "api", SubCommand: "apply", + ComponentSection: map[string]any{"chart": "api", "name": "api"}, + } + require.NoError(t, runWithHooks(ctx, &schema.AtmosConfiguration{}, info, OperationApply, "")) + assert.Zero(t, ciCalls) + results := collector.resultSet().Results + require.Len(t, results, 1) + assert.True(t, results[0].Processed) + assert.Equal(t, "api", results[0].Summary["chart"]) +} + +func TestExecuteBulkEmitsOneAggregateCIHook(t *testing.T) { + originalDescribe := executeDescribeStacks + originalGraph := executeGraph + originalCI := runCIHooks + t.Cleanup(func() { + executeDescribeStacks = originalDescribe + executeGraph = originalGraph + runCIHooks = originalCI + }) + + executeDescribeStacks = func( + *schema.AtmosConfiguration, + string, + []string, + []string, + []string, + bool, + bool, + bool, + bool, + []string, + auth.AuthManager, + ) (map[string]any, error) { + return map[string]any{"dev": map[string]any{}}, nil + } + sentinel := errors.New("graph failed") + executeGraph = func(_ context.Context, opts *component.GraphExecutionOptions) error { + assert.Equal(t, "plan", opts.SubCommand) + _, wrapped := opts.Provider.(*bulkCollectingProvider) + assert.True(t, wrapped) + collector, ok := opts.Flags[helmBulkCICollectorFlag].(*helmBulkCICollector) + require.True(t, ok) + collector.setSummary(&schema.ConfigAndStacksInfo{Stack: "dev", ComponentFromArg: "api"}, map[string]any{"diff": "+ change"}, nil) + collector.finish(&component.ExecutionContext{Stack: "dev", Component: "api"}, time.Now(), time.Now(), sentinel) + return sentinel + } + + var captured *hooks.RunCIHooksOptions + runCIHooks = func(opts *hooks.RunCIHooksOptions) error { + captured = opts + return nil + } + + ctx := &component.ExecutionContext{Flags: map[string]any{"ci": true}} + info := &schema.ConfigAndStacksInfo{All: true, SubCommand: "plan"} + require.ErrorIs(t, executeBulk(ctx, &schema.AtmosConfiguration{}, info, OperationDiff), sentinel) + require.NotNil(t, captured) + assert.Equal(t, hooks.AfterHelmPlanAggregate, captured.Event) + assert.True(t, captured.ForceCIMode) + assert.ErrorIs(t, captured.CommandError, sentinel) + resultSet, ok := captured.Aggregate.(schema.HelmCIResultSet) + require.True(t, ok) + assert.Equal(t, "plan", resultSet.Command) + require.Len(t, resultSet.Results, 1) + assert.Equal(t, "graph failed", resultSet.Results[0].Error) + _, retained := ctx.Flags[helmBulkCICollectorFlag] + assert.False(t, retained) +} + +func TestHelmAggregateCIHelpers(t *testing.T) { + assert.True(t, supportsHelmAggregateCI("plan")) + assert.True(t, supportsHelmAggregateCI("apply")) + assert.False(t, supportsHelmAggregateCI("template")) + assert.Nil(t, helmBulkCollector(nil)) + assert.Nil(t, helmBulkCollector(&component.ExecutionContext{})) +} diff --git a/pkg/component/helm/client.go b/pkg/component/helm/client.go index abe29f0b68..07f9a2f26a 100644 --- a/pkg/component/helm/client.go +++ b/pkg/component/helm/client.go @@ -85,6 +85,7 @@ func applyRelease(ctx context.Context, spec *chartSpec, dryRun bool) (releaseAct return releaseActionResult{Operation: releaseOperationInstall}, resolveErr } spec.Lifecycle = lifecycle + reportResolvedLifecycle(lifecycle) operationCtx, cancel := releaseOperationContext(ctx, lifecycle.Policy.Timeout) defer cancel() manifest, installErr := installRelease(operationCtx, actx, spec, dryRun) @@ -97,6 +98,7 @@ func applyRelease(ctx context.Context, spec *chartSpec, dryRun bool) (releaseAct return releaseActionResult{Operation: releaseOperationUpgrade}, resolveErr } spec.Lifecycle = lifecycle + reportResolvedLifecycle(lifecycle) operationCtx, cancel := releaseOperationContext(ctx, lifecycle.Policy.Timeout) defer cancel() manifest, upgradeErr := upgradeRelease(operationCtx, actx, spec, dryRun) @@ -132,7 +134,11 @@ func installRelease(ctx context.Context, actx *actionContext, spec *chartSpec, d if dryRun { client.DryRunStrategy = action.DryRunServer } - return runInstall(ctx, client, actx.settings, spec) + manifest, err := runInstall(ctx, client, actx.settings, spec) + if err != nil { + return "", releaseOperationError("install", spec, err) + } + return manifest, nil } func upgradeRelease(ctx context.Context, actx *actionContext, spec *chartSpec, dryRun bool) (string, error) { @@ -162,12 +168,9 @@ func upgradeRelease(ctx context.Context, actx *actionContext, spec *chartSpec, d rel, err := client.RunWithContext(ctx, spec.ReleaseName, loaded, spec.Values) if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { - return "", ctxErr - } - if errors.Is(err, errUtils.ErrHelmRenderFailed) { - return "", fmt.Errorf("failed to upgrade Helm release %q: %w", spec.ReleaseName, err) + return "", releaseOperationError("upgrade", spec, errors.Join(ctxErr, err)) } - return "", fmt.Errorf("%w %q: %w", errUtils.ErrHelmReleaseUpgrade, spec.ReleaseName, err) + return "", releaseOperationError("upgrade", spec, err) } rendered, ok := rel.(*release.Release) if !ok { @@ -242,7 +245,7 @@ func deleteRelease(ctx context.Context, spec *chartSpec, dryRun bool) error { if errors.Is(err, driver.ErrReleaseNotFound) { return nil } - uninstallErr := fmt.Errorf("%w %q: %w", errUtils.ErrHelmReleaseUninstall, spec.ReleaseName, err) + uninstallErr := releaseOperationError("delete", spec, err) if ctxErr := operationCtx.Err(); ctxErr != nil { return errors.Join(ctxErr, uninstallErr) } @@ -254,6 +257,19 @@ func deleteRelease(ctx context.Context, spec *chartSpec, dryRun bool) error { return nil } +func releaseOperationError(operation string, spec *chartSpec, cause error) error { + policy := spec.Lifecycle.Policy + return errUtils.Build(errUtils.ErrHelmReleaseOperation). + WithCause(cause). + WithContext("operation", operation). + WithContext("release", spec.ReleaseName). + WithContext("namespace", spec.Namespace). + WithContext("wait_strategy", policy.WaitStrategy). + WithContext("timeout", policy.Timeout). + WithContext("timeout_field", spec.Lifecycle.TimeoutField). + Err() +} + func configureInstallLifecycle(client *action.Install, policy effectiveReleasePolicy) { client.RollbackOnFailure = policy.OnFailure == failurePolicyUninstall client.WaitStrategy = policy.WaitStrategy diff --git a/pkg/component/helm/client_lifecycle_test.go b/pkg/component/helm/client_lifecycle_test.go index 9e3328008b..84c717220f 100644 --- a/pkg/component/helm/client_lifecycle_test.go +++ b/pkg/component/helm/client_lifecycle_test.go @@ -14,11 +14,13 @@ import ( "helm.sh/helm/v4/pkg/cli" kubefake "helm.sh/helm/v4/pkg/kube/fake" "helm.sh/helm/v4/pkg/registry" + helmrelease "helm.sh/helm/v4/pkg/release" release "helm.sh/helm/v4/pkg/release/v1" "helm.sh/helm/v4/pkg/storage" "helm.sh/helm/v4/pkg/storage/driver" errUtils "github.com/cloudposse/atmos/errors" + cfg "github.com/cloudposse/atmos/pkg/config" ) // memoryActionContext builds an actionContext backed by Helm's in-memory storage @@ -199,6 +201,7 @@ func TestApplyReleaseUsesLifecycleTimeoutAndWaitContext(t *testing.T) { elapsed := time.Since(started) require.ErrorIs(t, err, context.DeadlineExceeded) + require.ErrorIs(t, err, errUtils.ErrHelmReleaseOperation) assert.Equal(t, releaseOperationUpgrade, result.Operation) assert.Less(t, elapsed, time.Second, "upgrade must return at the lifecycle deadline") assert.NotEmpty(t, kubeClient.RecordedWaitOptions, "Helm waiters must receive the operation context") @@ -213,3 +216,66 @@ func TestReleaseOperationContextPreservesZeroTimeout(t *testing.T) { _, hasDeadline := ctx.Deadline() assert.False(t, hasDeadline) } + +func TestUpgradeReleaseHistoryRetention(t *testing.T) { + const revisions = cfg.HelmDefaultMaxHistory + 3 + tests := []struct { + name string + releaseName string + maxHistory int + override bool + expectedCount int + }{ + { + name: "default bounded history", + releaseName: "history", + expectedCount: cfg.HelmDefaultMaxHistory, + }, + { + name: "explicit unlimited history", + releaseName: "unlimited-history", + maxHistory: 0, + override: true, + expectedCount: revisions, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actx := memoryActionContext(t) + stubActionContext(t, actx) + spec := testdataChartSpec(t, tt.releaseName) + if tt.override { + maxHistory := tt.maxHistory + spec.Release.History.Max = &maxHistory + } + + for revision := 0; revision < revisions; revision++ { + spec.Values["replicaCount"] = revision + 1 + _, err := applyRelease(context.Background(), spec, false) + require.NoError(t, err) + } + + history, err := actx.cfg.Releases.History(spec.ReleaseName) + require.NoError(t, err) + assert.Len(t, history, tt.expectedCount) + expected := make([]int, tt.expectedCount) + firstRevision := revisions - tt.expectedCount + 1 + for i := range expected { + expected[i] = firstRevision + i + } + assert.Equal(t, expected, releaseVersions(t, history)) + }) + } +} + +func releaseVersions(t *testing.T, history []helmrelease.Releaser) []int { + t.Helper() + versions := make([]int, len(history)) + for i, item := range history { + typed, ok := item.(*release.Release) + require.True(t, ok) + versions[i] = typed.Version + } + return versions +} diff --git a/pkg/component/helm/client_test.go b/pkg/component/helm/client_test.go index f52f213cea..4494543aa5 100644 --- a/pkg/component/helm/client_test.go +++ b/pkg/component/helm/client_test.go @@ -99,6 +99,27 @@ func TestConfigureReleaseLifecycleActions(t *testing.T) { assert.True(t, uninstall.DryRun) } +func TestReleaseOperationErrorIncludesEffectivePolicy(t *testing.T) { + cause := context.DeadlineExceeded + err := releaseOperationError("upgrade", &chartSpec{ + ReleaseName: "demo", + Namespace: "apps", + Lifecycle: releaseLifecycleResolution{Policy: effectiveReleasePolicy{ + WaitStrategy: kube.StatusWatcherStrategy, + Timeout: 7 * time.Minute, + }, TimeoutField: "release.upgrade.timeout"}, + }, cause) + + require.ErrorIs(t, err, errUtils.ErrHelmReleaseOperation) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.True(t, errUtils.HasContext(err, "operation", "upgrade")) + assert.True(t, errUtils.HasContext(err, "release", "demo")) + assert.True(t, errUtils.HasContext(err, "namespace", "apps")) + assert.True(t, errUtils.HasContext(err, "wait_strategy", "watcher")) + assert.True(t, errUtils.HasContext(err, "timeout", "7m0s")) + assert.True(t, errUtils.HasContext(err, "timeout_field", "release.upgrade.timeout")) +} + func TestClusterOperationsReturnActionContextErrors(t *testing.T) { original := newActionContext t.Cleanup(func() { newActionContext = original }) diff --git a/pkg/component/helm/executor.go b/pkg/component/helm/executor.go index eef811c1a2..d250acc4d9 100644 --- a/pkg/component/helm/executor.go +++ b/pkg/component/helm/executor.go @@ -179,7 +179,7 @@ func runWithHooks( if err != nil { return err } - emitLifecycleWarnings(spec.Lifecycle.Warnings) + reportResolvedLifecycle(spec.Lifecycle) } if err := ctx.GoContext().Err(); err != nil { return err @@ -191,14 +191,18 @@ func runWithHooks( } summary, opErr := runOperation(ctx, atmosConfig, info, operation, spec) - runHelmCIHook(helmCIHookParams{ - ctx: ctx, - atmosConfig: atmosConfig, - info: info, - event: after, - summary: summary, - commandErr: opErr, - }) + if collector := helmBulkCollector(ctx); collector != nil { + collector.setSummary(info, summary, opErr) + } else { + runHelmCIHook(helmCIHookParams{ + ctx: ctx, + atmosConfig: atmosConfig, + info: info, + event: after, + summary: summary, + commandErr: opErr, + }) + } if opErr != nil { return opErr } @@ -247,6 +251,27 @@ func emitLifecycleWarnings(warnings []lifecycleWarning) { } } +func reportResolvedLifecycle(resolution releaseLifecycleResolution) { + emitLifecycleWarnings(resolution.Warnings) + reason := "configured" + for _, warning := range resolution.Warnings { + if warning.Code == warningWaitDerived { + reason = warning.Message + break + } + } + policy := resolution.Policy + log.Debug("Resolved Helm release lifecycle", + "operation", policy.Operation, + "wait_strategy", policy.WaitStrategy, + "wait_strategy_reason", reason, + "wait_jobs", policy.WaitForJobs, + "on_failure", policy.OnFailure, + "timeout", policy.Timeout, + "timeout_field", resolution.TimeoutField, + ) +} + // runTemplate renders the chart and writes the manifests per the render options. func runTemplate(ctx *component.ExecutionContext, atmosConfig *schema.AtmosConfiguration, info *schema.ConfigAndStacksInfo, spec *chartSpec) ([]*unstructured.Unstructured, error) { objects, err := renderObjects(ctx.GoContext(), spec) diff --git a/pkg/component/helm/executor_bulk.go b/pkg/component/helm/executor_bulk.go index f6da35e89b..1b6cb7be8c 100644 --- a/pkg/component/helm/executor_bulk.go +++ b/pkg/component/helm/executor_bulk.go @@ -42,16 +42,45 @@ func executeBulk( return err } - return executeGraph(ctx.GoContext(), &component.GraphExecutionOptions{ - Provider: &ComponentProvider{}, + command := info.SubCommand + if command == "" { + command = string(operation) + } + provider, collector, cleanup := setupHelmBulkAggregateCollector(ctx, command) + defer cleanup() + + graphErr := executeGraph(ctx.GoContext(), &component.GraphExecutionOptions{ + Provider: provider, AtmosConfig: atmosConfig, Info: info, Stacks: stacks, ComponentType: cfg.HelmComponentType, - SubCommand: string(operation), + SubCommand: command, Flags: ctx.Flags, Selection: selection, + ReverseOrder: operation == OperationDelete, }) + if collector != nil { + runHelmAggregateCIHook(ctx, atmosConfig, info, collector.resultSet(), graphErr) + } + return graphErr +} + +func setupHelmBulkAggregateCollector( + ctx *component.ExecutionContext, + command string, +) (component.ComponentProvider, *helmBulkCICollector, func()) { + if !supportsHelmAggregateCI(command) { + return &ComponentProvider{}, nil, func() {} + } + + collector := newHelmBulkCICollector(command) + if ctx.Flags == nil { + ctx.Flags = make(map[string]any) + } + ctx.Flags[helmBulkCICollectorFlag] = collector + provider := &bulkCollectingProvider{ComponentProvider: &ComponentProvider{}, collector: collector} + return provider, collector, func() { delete(ctx.Flags, helmBulkCICollectorFlag) } } func authManagerForBulk(atmosConfig *schema.AtmosConfiguration, info *schema.ConfigAndStacksInfo) (auth.AuthManager, error) { diff --git a/pkg/component/helm/executor_test.go b/pkg/component/helm/executor_test.go index a74145fcaf..94d433893a 100644 --- a/pkg/component/helm/executor_test.go +++ b/pkg/component/helm/executor_test.go @@ -190,6 +190,16 @@ func TestExecuteBulkInitializesConfigAndGraph(t *testing.T) { assert.Equal(t, cfg.HelmComponentType, graphOpts.ComponentType) assert.Equal(t, "template", graphOpts.SubCommand) assert.Equal(t, ctx.Flags, graphOpts.Flags) + assert.False(t, graphOpts.ReverseOrder) + + graphOpts = nil + require.NoError(t, executeBulk(ctx, &schema.AtmosConfiguration{}, &schema.ConfigAndStacksInfo{ + All: true, + Stack: "dev", + SubCommand: "delete", + }, OperationDelete)) + require.NotNil(t, graphOpts) + assert.True(t, graphOpts.ReverseOrder) } func TestExecuteSingleSkipsDisabledComponent(t *testing.T) { diff --git a/pkg/component/helm/lifecycle.go b/pkg/component/helm/lifecycle.go index 71c0d3c155..0c1019fea6 100644 --- a/pkg/component/helm/lifecycle.go +++ b/pkg/component/helm/lifecycle.go @@ -325,6 +325,21 @@ func decodeDeletePolicy(releaseMap map[string]any) (deletePolicyInput, error) { } func resolveReleaseLifecycle(input releasePolicyInput, operation string, emitMigrationWarning bool) (releaseLifecycleResolution, error) { + resolution, err := resolveReleaseLifecycleBase(input, operation, emitMigrationWarning) + if err != nil { + return releaseLifecycleResolution{}, err + } + if err := validateAndDeriveLifecycle(&resolution); err != nil { + return releaseLifecycleResolution{}, err + } + return resolution, nil +} + +// resolveReleaseLifecycleBase applies configuration precedence without deriving +// cross-field values. Callers that overlay CLI flags must do so before the +// single validateAndDeriveLifecycle pass, otherwise a derived watcher strategy +// loses both its source hookOnly value and the explanation for the promotion. +func resolveReleaseLifecycleBase(input releasePolicyInput, operation string, emitMigrationWarning bool) (releaseLifecycleResolution, error) { resolution := releaseLifecycleResolution{ Policy: defaultReleasePolicy(operation), TimeoutField: "built-in default", @@ -359,9 +374,6 @@ func resolveReleaseLifecycle(input releasePolicyInput, operation string, emitMig Message: "helm release timeout is omitted; this release preserves 0s, but the default will become 5m in the next minor release", }) } - if err := validateAndDeriveLifecycle(&resolution); err != nil { - return releaseLifecycleResolution{}, err - } return resolution, nil } @@ -448,7 +460,7 @@ func validateAndDeriveLifecycle(resolution *releaseLifecycleResolution) error { // resolveReleaseLifecycleWithFlags resolves configuration for the selected // action, then overlays only explicitly supplied CLI values at highest priority. func resolveReleaseLifecycleWithFlags(input releasePolicyInput, operation string, flags map[string]any) (releaseLifecycleResolution, error) { - resolution, err := resolveReleaseLifecycle(input, operation, true) + resolution, err := resolveReleaseLifecycleBase(input, operation, true) if err != nil { return releaseLifecycleResolution{}, err } @@ -502,7 +514,6 @@ func resolveReleaseLifecycleWithFlags(input releasePolicyInput, operation string resolution.Policy.CleanupOnFailure = value } - resolution.Warnings = removeLifecycleWarning(resolution.Warnings, warningWaitDerived) if err := validateAndDeriveLifecycle(&resolution); err != nil { return releaseLifecycleResolution{}, err } diff --git a/pkg/component/helm/lifecycle_test.go b/pkg/component/helm/lifecycle_test.go index b717e44612..ded1b3af66 100644 --- a/pkg/component/helm/lifecycle_test.go +++ b/pkg/component/helm/lifecycle_test.go @@ -111,6 +111,39 @@ func TestResolveReleaseLifecycleDerivedWaitStrategy(t *testing.T) { } } +func TestResolveReleaseLifecycleWithFlagsDerivesWaitStrategyAfterFlagOverlay(t *testing.T) { + for _, tt := range []struct { + name string + configuredFailure string + flags map[string]any + wantStrategy kube.WaitStrategy + wantDerived bool + }{ + {name: "configured rollback", configuredFailure: "rollback", wantStrategy: kube.StatusWatcherStrategy, wantDerived: true}, + {name: "flag rollback", flags: map[string]any{cfg.HelmOnFailureSectionName: "rollback"}, wantStrategy: kube.StatusWatcherStrategy, wantDerived: true}, + {name: "flag keep disables configured rollback", configuredFailure: "rollback", flags: map[string]any{cfg.HelmOnFailureSectionName: "keep"}, wantStrategy: kube.HookOnlyStrategy}, + } { + t.Run(tt.name, func(t *testing.T) { + upgrade := map[string]any{} + if tt.configuredFailure != "" { + upgrade[cfg.HelmOnFailureSectionName] = tt.configuredFailure + } + input, err := decodeReleasePolicy(map[string]any{ + cfg.HelmReleaseSectionName: map[string]any{ + cfg.HelmWaitSectionName: map[string]any{cfg.HelmWaitStrategySectionName: "hookOnly"}, + cfg.HelmUpgradeSectionName: upgrade, + }, + }) + require.NoError(t, err) + + resolution, err := resolveReleaseLifecycleWithFlags(input, releaseOperationUpgrade, tt.flags) + require.NoError(t, err) + assert.Equal(t, tt.wantStrategy, resolution.Policy.WaitStrategy) + assert.Equal(t, tt.wantDerived, hasLifecycleWarning(resolution.Warnings, warningWaitDerived)) + }) + } +} + func TestResolveReleaseLifecycleWithFlagsHighestPrecedence(t *testing.T) { input, err := decodeReleasePolicy(map[string]any{ cfg.HelmReleaseSectionName: map[string]any{ diff --git a/pkg/component/helm/provision.go b/pkg/component/helm/provision.go index 0c638e5d64..9fcc348d63 100644 --- a/pkg/component/helm/provision.go +++ b/pkg/component/helm/provision.go @@ -49,7 +49,6 @@ func deliverApply( if selected.Kind == target.KindKubernetes { result, err := applyHelmRelease(ctx, spec, info.DryRun) spec.Lifecycle = result.Lifecycle - emitLifecycleWarnings(result.Lifecycle.Warnings) summary["manifest_bytes"] = len(result.Manifest) summary["release"] = lifecycleSummary(result.Operation, result.Lifecycle.Policy) if objects, decodeErr := manifest.DecodeObjects([]byte(result.Manifest)); decodeErr == nil { diff --git a/pkg/hooks/event.go b/pkg/hooks/event.go index fee50582db..9d1645c4eb 100644 --- a/pkg/hooks/event.go +++ b/pkg/hooks/event.go @@ -40,8 +40,10 @@ const ( AfterHelmTemplate HookEvent = "after.helm.template" BeforeHelmDiff HookEvent = "before.helm.diff" AfterHelmDiff HookEvent = "after.helm.diff" + AfterHelmPlanAggregate HookEvent = "after.helm.plan.aggregate" BeforeHelmApply HookEvent = "before.helm.apply" AfterHelmApply HookEvent = "after.helm.apply" + AfterHelmApplyAggregate HookEvent = "after.helm.apply.aggregate" BeforeHelmDeploy HookEvent = "before.helm.deploy" AfterHelmDeploy HookEvent = "after.helm.deploy" BeforeHelmDelete HookEvent = "before.helm.delete" diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index 3bbbb1aae2..98cb0f3795 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -900,6 +900,29 @@ type TerraformPlanCIResult struct { Error string } +// HelmCIResultSet contains deterministic per-node Helm results for one +// graph-backed plan or apply run. +type HelmCIResultSet struct { + Command string + Results []HelmCIResult +} + +// HelmCIResult contains the execution outcome and structured Helm summary for +// one component in a graph-backed run. +type HelmCIResult struct { + NodeID string + Stack string + Component string + Status string + Processed bool + ExitCode int + Summary map[string]any + StartedAt time.Time + FinishedAt time.Time + DurationMS int64 + Error string +} + // KubernetesCIResult contains the compact result data rendered into native CI // job summaries for one Kubernetes component command. type KubernetesCIResult struct { diff --git a/tests/fixtures/scenarios/helm-lifecycle/README.md b/tests/fixtures/scenarios/helm-lifecycle/README.md new file mode 100644 index 0000000000..5eb49aa3a1 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/README.md @@ -0,0 +1,8 @@ +# Native Helm lifecycle scenario + +This fixture exercises native Helm lifecycle behavior against the Kubernetes +emulator. It intentionally contains failure cases, delayed readiness, hooks, +Jobs, CRDs, dependency ordering, rollback, cleanup, dry-run, and timeout +coverage. + +The user-facing happy-path demo remains in `examples/helm`. diff --git a/tests/fixtures/scenarios/helm-lifecycle/atmos.yaml b/tests/fixtures/scenarios/helm-lifecycle/atmos.yaml new file mode 100644 index 0000000000..ba3dda5a77 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/atmos.yaml @@ -0,0 +1,178 @@ +# Atmos configuration for the native Helm lifecycle integration scenario. + +base_path: "." + +components: + helm: + # Base path for native Helm components (local charts live here). + base_path: "components/helm" + +auth: + identities: + local-k3s: + kind: kubernetes/emulator + emulator: kubernetes + +stacks: + base_path: "stacks" + included_paths: + - "deploy/**/*" + excluded_paths: + - "**/_defaults.yaml" + name_template: "{{.vars.stage}}" +logs: + file: "/dev/stderr" + level: Info + +toolchain: + install_path: .tools + aliases: + helm: helm/helm + registries: + - name: aqua-public + type: aqua + source: https://github.com/aquaproj/aqua-registry/tree/main/pkgs + priority: 10 + +commands: + - name: "test" + description: "Exercise native Helm lifecycle behavior against the k3s emulator" + steps: + - atmos validate stacks + # Dependency acquisition is explicit: the opt-in flag fetches the missing + # file:// library and updates the local chart before the first render. + # Template output includes weighted hooks and leaves Helm `tpl` expressions + # in values for the chart to evaluate with its native .Values context. + - command: >- + artifact=$(mktemp ./atmos-helm-template.XXXXXX); + trap 'rm -f "$artifact"' EXIT; + atmos helm template demo -s dev --dependency-update --output="$artifact" && + test "$(grep -c '^ name: demo-hook-order$' "$artifact")" -eq 2 && + grep -q 'helm.sh/hook-weight: "-2"' "$artifact" && + grep -q 'helm.sh/hook-weight: "-1"' "$artifact" && + grep -q 'rendered: from-stack' "$artifact" + - command: atmos emulator up kubernetes -s dev + retry: + max_attempts: 2 + initial_delay: 15s + backoff_strategy: constant + # Pull workload images through the host runtime and import them into the + # nested k3s containerd. This avoids registry pulls through Colima's + # nested network while Helm is waiting on Jobs and Deployments. + - command: >- + docker pull busybox:1.36.1 && + docker pull nginx:1.27 && + docker save busybox:1.36.1 nginx:1.27 | + docker exec -i atmos-dev-emulator-kubernetes ctr --namespace k8s.io images import - + retry: + max_attempts: 2 + initial_delay: 15s + backoff_strategy: constant + - command: atmos helm diff demo -s dev --identity local-k3s + retry: + max_attempts: 2 + initial_delay: 15s + backoff_strategy: constant + # Apply dry-run must not persist a release or create Kubernetes objects. + - atmos helm apply demo -s dev --identity local-k3s --dry-run + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo 2>&1); then echo "dry-run unexpectedly created deployment/demo"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify deployment/demo absence: $output"; exit 1;; esac; fi + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo 2>&1); then echo "dry-run unexpectedly created service/demo"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify service/demo absence: $output"; exit 1;; esac; fi + - command: >- + if ! releases=$(atmos emulator exec kubernetes -s dev -- kubectl get secrets --all-namespaces -l owner=helm,name=demo -o name); then echo "failed to query Helm release records after dry-run"; exit 1; fi; + if [ -n "$releases" ]; then echo "dry-run unexpectedly persisted Helm release records: $releases"; exit 1; fi + - command: atmos helm apply demo -s dev --identity local-k3s + retry: + max_attempts: 2 + initial_delay: 15s + backoff_strategy: constant + # The -2 ConfigMap must exist before the -1 hook Job can mount it. + - atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-hook-order + # release.install.crds is a stack-level release default. + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl get crd widgets.lifecycle.atmos.test 2>&1); then echo "skip_crds unexpectedly installed widgets.lifecycle.atmos.test"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify widgets.lifecycle.atmos.test absence: $output"; exit 1;; esac; fi + - command: atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo + retry: + max_attempts: 3 + initial_delay: 10s + backoff_strategy: constant + - command: atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo + retry: + max_attempts: 3 + initial_delay: 10s + backoff_strategy: constant + # watcher + release.wait.jobs returns only after the ordinary Job completes. + - atmos helm apply demo-jobs -s dev --identity local-k3s + - command: >- + completed=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-jobs get job demo-jobs-job -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}'); + if [ "$completed" != "True" ]; then echo "wait_for_jobs returned before demo-jobs-job completed: status=$completed"; exit 1; fi + # Disabling chart hooks prevents both weighted hook resources. + - atmos helm apply demo-no-hooks -s dev --identity local-k3s + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-no-hooks get configmap demo-no-hooks-hook-order 2>&1); then echo "disable_chart_hooks unexpectedly ran Helm hooks"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify disabled Helm hooks: $output"; exit 1;; esac; fi + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-no-hooks get job demo-no-hooks-hook-order 2>&1); then echo "disable_chart_hooks unexpectedly ran the Helm hook Job"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify disabled Helm hook Job: $output"; exit 1;; esac; fi + # hookOnly returns after hooks without waiting for the Deployment's explicit readiness gate. + - atmos helm apply demo-hook-only -s dev --identity local-k3s + - command: >- + if ! available=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only get deployment demo-hook-only -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>&1); then echo "failed to query Deployment/demo-hook-only availability: $available"; exit 1; fi; + if [ "$available" = "True" ]; then echo "hookOnly unexpectedly satisfied the Deployment readiness gate"; exit 1; fi + - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only patch deployment demo-hook-only --type=json -p='[{"op":"remove","path":"/spec/template/spec/readinessGates"}]' + - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only rollout status deployment/demo-hook-only --timeout=2m + # watcher honors the release timeout and rollback removes the failed install. + - command: >- + if output=$(NO_COLOR=1 atmos helm apply demo-timeout -s dev --identity local-k3s 2>&1); then echo "timed release unexpectedly succeeded"; exit 1; fi; + case "$output" in *"helm release operation failed"*) ;; *) echo "timed release failed outside the Helm lifecycle operation: $output"; exit 1;; esac; + case "$output" in *"context"*"deadline exceeded"*|*"timed out waiting for condition"*) ;; *) echo "timed release did not report a lifecycle timeout: $output"; exit 1;; esac + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-timeout get deployment demo-timeout 2>&1); then echo "timeout rollback left release resources"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify timeout rollback cleanup: $output"; exit 1;; esac; fi + # A failed first install is reported as failure and rolled back. The kept + # hook ConfigMap survives recovery while ordinary release resources do not. + - command: >- + if output=$(NO_COLOR=1 atmos helm apply demo-install-fail -s dev --identity local-k3s 2>&1); then echo "failed install unexpectedly succeeded"; exit 1; fi; + case "$output" in *"helm release operation failed"*) ;; *) echo "failed install stopped outside the Helm lifecycle operation: $output"; exit 1;; esac; + case "$output" in *"job demo-install-fail-failing-hook failed: BackoffLimitExceeded"*) ;; *) echo "failed install did not report the expected hook failure: $output"; exit 1;; esac + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-install-fail get deployment demo-install-fail 2>&1); then echo "rollback left failed install resources"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify failed-install rollback cleanup: $output"; exit 1;; esac; fi + - atmos emulator exec kubernetes -s dev -- kubectl -n demo-install-fail get configmap demo-install-fail-hook-order + # A failed upgrade restores the successful demo release and cleanup removes + # the resource introduced only by the failed revision. + - command: >- + if ! before=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}' 2>&1); then echo "failed to capture Deployment/demo state before upgrade: $before"; exit 1; fi; + if output=$(NO_COLOR=1 atmos helm apply demo-upgrade-fail -s dev --identity local-k3s 2>&1); then echo "failed upgrade unexpectedly succeeded"; exit 1; fi; + case "$output" in *"helm release operation failed"*) ;; *) echo "failed upgrade stopped outside the Helm lifecycle operation: $output"; exit 1;; esac; + case "$output" in *"job demo-failing-hook failed"*"BackoffLimitExceeded"*) ;; *) echo "failed upgrade did not report the expected post-upgrade hook failure: $output"; exit 1;; esac; + if ! after=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}' 2>&1); then echo "failed to capture Deployment/demo state after rollback: $after"; exit 1; fi; + if [ "$before" != "$after" ]; then echo "rollback did not restore Deployment/demo: before=$before after=$after"; exit 1; fi + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-upgrade-only 2>&1); then echo "cleanup failure action left the upgrade-only ConfigMap"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify cleanup failure action: $output"; exit 1;; esac; fi + # Dependency execution gates the dependent render on the foundation's + # release-managed Deployment readiness state. + - atmos helm apply --all -s dev --identity local-k3s --tags lifecycle-dag + - atmos emulator exec kubernetes -s dev -- kubectl -n lifecycle-dag get deployment dag-dependent + - command: >- + observed=$(atmos emulator exec kubernetes -s dev -- kubectl -n lifecycle-dag get configmap dag-dependent-dependency-observed -o jsonpath='{.data.ready}'); + if [ "$observed" != "true" ]; then echo "dependent did not observe the ready foundation Deployment: ready=$observed"; exit 1; fi + # Delete dry-run must leave the deployed release and resources intact. + - atmos helm delete demo -s dev --identity local-k3s --dry-run --wait=watcher --timeout=2m + - atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo + - atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo + - command: >- + if ! releases=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get secrets -l owner=helm,name=demo -o name); then echo "failed to query Helm release records after delete dry-run"; exit 1; fi; + if [ -z "$releases" ]; then echo "delete dry-run removed the Helm release record"; exit 1; fi + - atmos helm delete demo -s dev --identity local-k3s + - atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-hook-order + - atmos helm delete demo-jobs -s dev --identity local-k3s + - atmos helm delete demo-no-hooks -s dev --identity local-k3s + - atmos helm delete demo-hook-only -s dev --identity local-k3s + - atmos helm delete --all -s dev --identity local-k3s --tags lifecycle-dag + - atmos emulator down kubernetes -s dev diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/.gitignore b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/.gitignore new file mode 100644 index 0000000000..ee3892e879 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/.gitignore @@ -0,0 +1 @@ +charts/ diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/Chart.lock b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/Chart.lock new file mode 100644 index 0000000000..3ae6829b29 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: helm-test-library + repository: file://../helm-test-library + version: 0.1.0 +digest: sha256:cf07c19b9e23d03dec7c67e6160c8d59a941059e48f9c47e86878cd29ce16098 +generated: "2026-08-01T15:03:42.45764+04:00" diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/Chart.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/Chart.yaml new file mode 100644 index 0000000000..d17a37337f --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: demo +description: A minimal local Helm chart used by the Atmos native Helm component demo. +type: application +version: 0.1.0 +appVersion: "1.0.0" +dependencies: + - name: helm-test-library + version: 0.1.0 + repository: file://../helm-test-library diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/crds/lifecycle.atmos.test_widgets.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/crds/lifecycle.atmos.test_widgets.yaml new file mode 100644 index 0000000000..390a2826d1 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/crds/lifecycle.atmos.test_widgets.yaml @@ -0,0 +1,18 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: widgets.lifecycle.atmos.test +spec: + group: lifecycle.atmos.test + scope: Namespaced + names: + plural: widgets + singular: widget + kind: Widget + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/dependency-observed.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/dependency-observed.yaml new file mode 100644 index 0000000000..4a6be62d2e --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/dependency-observed.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.hooks.requiredDeployment .Values.hooks.validateDependency }} +{{- $required := lookup "apps/v1" "Deployment" .Release.Namespace .Values.hooks.requiredDeployment }} +{{- if not $required }} +{{- fail (printf "required dependency Deployment %q is not ready" .Values.hooks.requiredDeployment) }} +{{- end }} +{{- $desired := int (dig "spec" "replicas" 1 $required) }} +{{- $available := int (dig "status" "availableReplicas" 0 $required) }} +{{- $generation := int64 (dig "metadata" "generation" 0 $required) }} +{{- $observedGeneration := int64 (dig "status" "observedGeneration" 0 $required) }} +{{- if or (lt $available $desired) (lt $observedGeneration $generation) }} +{{- fail (printf "required dependency Deployment %q is not ready: available=%d desired=%d observedGeneration=%d generation=%d" .Values.hooks.requiredDeployment $available $desired $observedGeneration $generation) }} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-dependency-observed + namespace: {{ .Release.Namespace }} +data: + requiredDeployment: {{ .Values.hooks.requiredDeployment | quote }} + ready: "true" + availableReplicas: {{ $available | quote }} +{{- end }} diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/deployment.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/deployment.yaml new file mode 100644 index 0000000000..ac8067e5cb --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ .Release.Name }} + spec: + {{- if .Values.deployment.readinessGate }} + readinessGates: + - conditionType: "lifecycle.atmos.test/ready" + {{- end }} + {{- if gt (int .Values.deployment.readinessDelaySeconds) 0 }} + initContainers: + - name: readiness-delay + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: ["/bin/sh", "-c", "sleep {{ .Values.deployment.readinessDelaySeconds }}"] + {{- end }} + containers: + - name: {{ .Release.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + ports: + - containerPort: {{ .Values.service.port }} diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/extra-configmap.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/extra-configmap.yaml new file mode 100644 index 0000000000..ea425677fc --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/extra-configmap.yaml @@ -0,0 +1,9 @@ +{{- if .Values.extraConfigMap.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-upgrade-only + namespace: {{ .Release.Namespace }} +data: + created: during-failed-upgrade +{{- end }} diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/failing-hook-job.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/failing-hook-job.yaml new file mode 100644 index 0000000000..d4f3c4b0f7 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/failing-hook-job.yaml @@ -0,0 +1,20 @@ +{{- if .Values.hooks.fail }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-failing-hook + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: fail + image: "{{ .Values.jobImage.repository }}:{{ .Values.jobImage.tag }}" + command: ["/bin/sh", "-c", "echo intentional lifecycle failure >&2; exit 1"] +{{- end }} diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/hook-order-configmap.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/hook-order-configmap.yaml new file mode 100644 index 0000000000..5403f9e8d0 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/hook-order-configmap.yaml @@ -0,0 +1,14 @@ +{{- if .Values.hooks.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-hook-order + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-2" + "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/resource-policy": keep +data: + order: first +{{- end }} diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/hook-order-job.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/hook-order-job.yaml new file mode 100644 index 0000000000..5b059044d2 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/hook-order-job.yaml @@ -0,0 +1,29 @@ +{{- if .Values.hooks.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-hook-order + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-1" + # Retain the completed Job so lifecycle integration tests can distinguish + # a disabled hook from a hook that ran successfully and self-deleted. + "helm.sh/hook-delete-policy": before-hook-creation +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: verify-weighted-configmap + image: "{{ .Values.jobImage.repository }}:{{ .Values.jobImage.tag }}" + command: ["/bin/sh", "-c", "test -f /hook-order/order"] + volumeMounts: + - name: hook-order + mountPath: /hook-order + volumes: + - name: hook-order + configMap: + name: {{ printf "%s-hook-order" .Release.Name }} +{{- end }} diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/job.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/job.yaml new file mode 100644 index 0000000000..3f0df8e448 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/job.yaml @@ -0,0 +1,16 @@ +{{- if .Values.job.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-job + namespace: {{ .Release.Namespace }} +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: ordinary-job + image: "{{ .Values.jobImage.repository }}:{{ .Values.jobImage.tag }}" + command: ["/bin/sh", "-c", "sleep {{ .Values.job.sleepSeconds }}"] +{{- end }} diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/service.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/service.yaml new file mode 100644 index 0000000000..3cfba0daed --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.service.type }} + selector: + app: {{ .Release.Name }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.port }} diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/tpl-configmap.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/tpl-configmap.yaml new file mode 100644 index 0000000000..165b7f6adb --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/tpl-configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-tpl + namespace: {{ .Release.Namespace }} +data: + rendered: {{ include "helmTestLibrary.renderValue" (dict "value" .Values.tpl.expression "context" .) | quote }} diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/values.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/values.yaml new file mode 100644 index 0000000000..3f7b52c0b5 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/values.yaml @@ -0,0 +1,27 @@ +# Default chart values. Atmos `values:` from the stack are merged on top of these. +replicaCount: 1 +image: + repository: nginx + tag: "latest" +jobImage: + repository: busybox + tag: "1.36.1" +service: + type: ClusterIP + port: 80 +deployment: + readinessDelaySeconds: 0 + readinessGate: false +job: + enabled: false + sleepSeconds: 0 +hooks: + enabled: true + fail: false + requiredDeployment: "" + validateDependency: false +extraConfigMap: + enabled: false +tpl: + source: default + expression: "{{ .Values.tpl.source }}" diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/helm-test-library/Chart.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/helm-test-library/Chart.yaml new file mode 100644 index 0000000000..04577030d2 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/helm-test-library/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: helm-test-library +description: Minimal library chart used to verify local Helm dependencies. +type: library +version: 0.1.0 diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/helm-test-library/templates/_render.tpl b/tests/fixtures/scenarios/helm-lifecycle/components/helm/helm-test-library/templates/_render.tpl new file mode 100644 index 0000000000..fc0dee51e0 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/helm-test-library/templates/_render.tpl @@ -0,0 +1,3 @@ +{{- define "helmTestLibrary.renderValue" -}} +{{- tpl .value .context -}} +{{- end -}} diff --git a/tests/fixtures/scenarios/helm-lifecycle/stacks/deploy/dev.yaml b/tests/fixtures/scenarios/helm-lifecycle/stacks/deploy/dev.yaml new file mode 100644 index 0000000000..b23e81fa70 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/stacks/deploy/dev.yaml @@ -0,0 +1,202 @@ +vars: + stage: dev + +# Native Helm lifecycle defaults for every Helm component in this stack. +helm: + release: + wait: + strategy: watcher + # Nested k3s runners can leave Helm's watcher blocked after release + # resources are gone. Live deletes use deterministic legacy waiting; the + # workflow still exercises watcher selection through its delete dry-run. + delete: + wait: + strategy: legacy + # Leave enough headroom for nested k3s workloads on resource-constrained + # macOS/Colima runners. Components that exercise timeout behavior override + # this value explicitly below. + timeout: 4m + history: + max: 10 + install: + crds: skip + +components: + emulator: + kubernetes: + driver: k3s + ephemeral: true + + helm: + demo: + metadata: + component: demo + # The chart reference. "." is the component directory itself + # (components/helm/demo), which contains Chart.yaml. + chart: "." + namespace: demo + release: + wait: + strategy: legacy + install: + on_failure: uninstall + upgrade: + on_failure: rollback + cleanup_on_failure: true + # Atmos `values:` are the Helm chart values, merged via Atmos inheritance. + values: + replicaCount: 2 + image: + tag: "1.27" + service: + port: 8080 + tpl: + source: from-stack + expression: !literal "{{ .Values.tpl.source }}" + + demo-repo: + metadata: + component: demo + repositories: + - name: local + url: !env HELM_DEMO_REPO_URL + chart: local/demo + version: 0.1.0 + namespace: demo-repo + values: + replicaCount: 2 + image: + tag: "1.27" + service: + port: 8080 + + demo-jobs: + metadata: + component: demo + chart: "." + name: demo-jobs + namespace: demo-jobs + release: + wait: + jobs: true + values: + hooks: + enabled: false + job: + enabled: true + sleepSeconds: 3 + + demo-no-hooks: + metadata: + component: demo + chart: "." + name: demo-no-hooks + namespace: demo-no-hooks + release: + chart_hooks: false + + demo-hook-only: + metadata: + component: demo + chart: "." + name: demo-hook-only + namespace: demo-hook-only + release: + wait: + strategy: hookOnly + values: + deployment: + readinessGate: true + + demo-timeout: + metadata: + component: demo + chart: "." + name: demo-timeout + namespace: demo-timeout + release: + # Keep the lifecycle deadline below the readiness delay while leaving + # enough time for recovery to finish on resource-constrained runners. + timeout: 15s + wait: + strategy: watcher + install: + on_failure: uninstall + upgrade: + on_failure: rollback + values: + deployment: + readinessDelaySeconds: 60 + hooks: + enabled: false + + demo-install-fail: + metadata: + component: demo + chart: "." + name: demo-install-fail + namespace: demo-install-fail + release: + wait: + strategy: legacy + install: + on_failure: uninstall + values: + hooks: + fail: true + + demo-upgrade-fail: + metadata: + component: demo + dependencies: + components: + - name: demo + chart: "." + # Intentionally targets the already-installed demo release. + name: demo + namespace: demo + release: + upgrade: + wait: + strategy: legacy + on_failure: rollback + cleanup_on_failure: true + values: + hooks: + fail: true + extraConfigMap: + enabled: true + + dag-foundation: + metadata: + component: demo + tags: [lifecycle-dag] + chart: "." + name: dag-foundation + namespace: lifecycle-dag + release: + wait: + strategy: legacy + values: + deployment: + readinessDelaySeconds: 3 + + dag-dependent: + metadata: + component: demo + tags: [lifecycle-dag] + dependencies: + components: + - name: dag-foundation + chart: "." + name: dag-dependent + namespace: lifecycle-dag + release: + wait: + strategy: legacy + values: + hooks: + requiredDeployment: dag-foundation + # The lifecycle DAG integration test explicitly opts into a live + # cluster lookup; ordinary offline chart rendering leaves this off. + validateDependency: true diff --git a/website/docs/ci/job-summaries.mdx b/website/docs/ci/job-summaries.mdx index c66cf626e7..200b54b549 100644 --- a/website/docs/ci/job-summaries.mdx +++ b/website/docs/ci/job-summaries.mdx @@ -124,6 +124,21 @@ comments, or artifacts. The summary includes component, stack, command status, a command, and Helm metadata such as release name, namespace, chart, target, object counts, object kinds, and rendered manifest size when available. +When `plan` or `apply` selects multiple components with `--all` or `--affected`, Atmos writes one +deterministic aggregate summary after the dependency-graph run completes instead of writing a +separate summary from every component. The aggregate includes result counts and a stable +per-component table with stack, component, chart, release, namespace, target, status, and duration. +Plan summaries distinguish changed and unchanged components and include collapsible diffs; failed +components include their error details. The same aggregate behavior applies to the `diff` and +`deploy` aliases and composes with tag and label filters. + +For a single cluster-backed apply/deploy/delete operation, the component summary also includes an +operation-specific `release` block with the effective wait strategy, timeout, chart-hook state, +and applicable recovery, Job-wait, CRD, cleanup, and history values. For external delivery, +apply/deploy reports `applied: false` while delete reports `deleted: false`; both include the +selected target kind and `reason: external_target` instead of presenting stored release policy as +active. + ## Helmfile Summaries Helmfile components write summaries for these operations when `ci.enabled: true` and CI mode is diff --git a/website/docs/cli/commands/helm/helm-apply.mdx b/website/docs/cli/commands/helm/helm-apply.mdx index 1312f84058..8695a38856 100644 --- a/website/docs/cli/commands/helm/helm-apply.mdx +++ b/website/docs/cli/commands/helm/helm-apply.mdx @@ -26,6 +26,18 @@ Install or upgrade the release in the cluster: atmos helm apply monitoring -s plat-ue2-dev ``` +Upgrade an existing release with an explicit production readiness and recovery policy: + +```shell +atmos helm apply monitoring -s plat-ue2-dev \ + --on-failure=rollback \ + --cleanup-on-failure \ + --wait=watcher \ + --wait-for-jobs \ + --timeout=30m \ + --history-max=10 +``` + Deliver rendered manifests to a provision target instead of the cluster: ```shell @@ -46,6 +58,10 @@ atmos helm apply --all --tags production,tier-1 atmos helm apply --affected --labels cost-center=platform ``` +In CI, bulk applies selected with `--all` or `--affected` write one aggregate GitHub job summary +after the dependency graph completes. It lists component status and Helm release metadata in stable +stack/component order and includes details for failures. + ## Flags
@@ -58,6 +74,33 @@ atmos helm apply --affected --labels cost-center=platform
`--dependency-update` (optional)
Fetch declared chart dependencies when they are missing. This may access dependency repositories and update the chart's charts/ directory and lock file.
+
`--dry-run` (optional)
+
Preview the install or upgrade without persisting release state or creating Kubernetes resources.
+ +
`--on-failure` (optional)
+
Failure action for the selected operation: `uninstall` or `keep` for install; `rollback` or `keep` for upgrade. The explicit flag overrides stack configuration for this invocation.
+ +
`--cleanup-on-failure` (optional)
+
Remove resources newly created during a failed upgrade, independently of rollback. Fails if the selected operation is install.
+ +
`--wait[=strategy]` (optional)
+
Use `watcher`, `hookOnly`, or `legacy`. Passing `--wait` without a value selects `watcher`. Boolean values remain accepted temporarily but are deprecated.
+ +
`--wait-for-jobs` (optional)
+
Wait for ordinary Jobs in the release manifest. Requires `watcher` or `legacy`.
+ +
`--timeout` (optional)
+
Helm release-operation timeout, such as `10m` or `1h`. `0s` is explicitly unbounded.
+ +
`--history-max` (optional)
+
Maximum retained release revisions for an upgrade. Defaults to `10`; `0` means unlimited. Fails if the selected operation is install.
+ +
`--no-hooks` (optional)
+
Disable Helm chart hooks. Atmos lifecycle hooks are unaffected.
+ +
`--skip-crds` (optional)
+
Skip CRD installation on a first install. Fails if the selected operation is upgrade.
+
`--all` (optional)
Apply all Helm components in dependency order.
@@ -73,3 +116,7 @@ atmos helm apply --affected --labels cost-center=platform
`--labels` (optional)
Filter by labels (comma-separated `key=value` or `key:value` pairs, matches all): `--labels=cost-center=platform,compliance=sox`. Composes with `--all`/`--affected`/`--tags`; cannot be combined with a single component argument.
+ +Lifecycle flags apply only to direct Kubernetes delivery. Combining an explicit +lifecycle flag with an external `--target` fails instead of implying that Atmos +waited for or rolled back a GitOps deployment. diff --git a/website/docs/cli/commands/helm/helm-delete.mdx b/website/docs/cli/commands/helm/helm-delete.mdx index 487369ca2a..b05468c74b 100644 --- a/website/docs/cli/commands/helm/helm-delete.mdx +++ b/website/docs/cli/commands/helm/helm-delete.mdx @@ -18,6 +18,9 @@ import Intro from "@site/src/components/Intro"; atmos helm delete --stack [options] atmos helm delete monitoring -s plat-ue2-dev +# Preview without removing release state or Kubernetes resources +atmos helm delete monitoring -s plat-ue2-dev --dry-run --wait=watcher --timeout=10m + # Delete components filtered by tags or labels (composes with --all/--affected) atmos helm delete --all --tags production,tier-1 ``` @@ -29,8 +32,20 @@ atmos helm delete --all --tags production,tier-1
Atmos stack.
`--all` / `--affected` / `--include-dependents` (optional)
-
Process multiple Helm components in dependency order.
+
Process multiple Helm components in reverse dependency order, deleting dependents before their dependencies.
`--tags` / `--labels` (optional)
Filter by tags (comma-separated, matches any) or labels (comma-separated `key=value` or `key:value` pairs, matches all): `--tags=production,tier-1`, `--labels=cost-center=platform`. Compose with `--all`/`--affected` to narrow the selected set further; cannot be combined with a single component argument.
+ +
`--wait[=strategy]` (optional)
+
Use `watcher`, `hookOnly`, or `legacy` while deleting. Passing `--wait` without a value selects `watcher`. Boolean values remain accepted temporarily but are deprecated.
+ +
`--timeout` (optional)
+
Helm uninstall timeout, such as `10m`. `0s` is explicitly unbounded.
+ +
`--no-hooks` (optional)
+
Disable Helm chart uninstall hooks. Atmos lifecycle hooks are unaffected.
+ +
`--dry-run` (optional)
+
Preview the uninstall without deleting release history or Kubernetes resources.
diff --git a/website/docs/cli/commands/helm/helm-deploy.mdx b/website/docs/cli/commands/helm/helm-deploy.mdx index 7b01cb1cfa..e958bc4622 100644 --- a/website/docs/cli/commands/helm/helm-deploy.mdx +++ b/website/docs/cli/commands/helm/helm-deploy.mdx @@ -34,6 +34,9 @@ atmos helm deploy --affected --labels cost-center=platform
`--target` (optional)
Provision target to deliver to. Defaults to `provision.default`, otherwise the cluster.
+
`--dependency-update` (optional)
+
Fetch declared chart dependencies when they are missing. See [`apply`](/cli/commands/helm/apply#flags).
+
`--all` / `--affected` / `--include-dependents` (optional)
Process multiple Helm components in dependency order. See [`apply`](/cli/commands/helm/apply).
diff --git a/website/docs/cli/commands/helm/helm-plan.mdx b/website/docs/cli/commands/helm/helm-plan.mdx index c71c337f1e..1badae913d 100644 --- a/website/docs/cli/commands/helm/helm-plan.mdx +++ b/website/docs/cli/commands/helm/helm-plan.mdx @@ -31,6 +31,11 @@ atmos helm plan monitoring -s plat-ue2-dev --against=target atmos helm plan --all --tags production ``` +In CI, bulk plans selected with `--all` or `--affected` write one aggregate GitHub job summary +after the dependency graph completes. It lists every attempted component in stable stack/component +order, distinguishes changed, unchanged, and failed results, and includes collapsible diffs for +changed components. + ## Flags
@@ -40,6 +45,9 @@ atmos helm plan --all --tags production
`--against=` / `--from-manifest=` / `--context=` (optional)
Select and tune the diff baseline. See [`diff`](/cli/commands/helm/diff#flags).
+
`--dependency-update` (optional)
+
Fetch declared chart dependencies when they are missing. See [`diff`](/cli/commands/helm/diff#flags).
+
`--all` / `--affected` / `--include-dependents` (optional)
Process multiple Helm components in dependency order.
diff --git a/website/docs/cli/commands/helm/usage.mdx b/website/docs/cli/commands/helm/usage.mdx index fb6aecc26c..4e28e62131 100644 --- a/website/docs/cli/commands/helm/usage.mdx +++ b/website/docs/cli/commands/helm/usage.mdx @@ -72,6 +72,21 @@ Supported summaries: Summaries include component, stack, command status, a local reproduction command, and Helm metadata such as release name, namespace, chart, target, object counts, object kinds, and rendered manifest size when available. +Cluster-backed release summaries also show the effective Helm lifecycle policy. +External-target summaries explicitly report that release lifecycle was bypassed. + +## Release lifecycle + +For cluster delivery, `apply`/`deploy` and `delete` expose Helm 4 wait and timeout +behavior. Apply/deploy additionally support rollback on failure, ordinary Job +waiting, failed-upgrade cleanup, history limits, chart-hook suppression, and CRD +skipping. Configure defaults and inherited policy in the stack component, then +use command flags only for an explicit invocation override. + +See [Helm stack release lifecycle](/stacks/components/helm#release-lifecycle), +[`helm apply`](/cli/commands/helm/apply), and +[`helm delete`](/cli/commands/helm/delete) for the complete field and flag +reference. ## Chart sources diff --git a/website/docs/cli/configuration/components/helm.mdx b/website/docs/cli/configuration/components/helm.mdx index dc82df2e08..388059f675 100644 --- a/website/docs/cli/configuration/components/helm.mdx +++ b/website/docs/cli/configuration/components/helm.mdx @@ -54,6 +54,16 @@ components:
+:::info Release policy belongs in stacks +Helm 4 lifecycle fields such as `release.wait.strategy`, `release.wait.jobs`, +`release.timeout`, `release.history.max`, and operation-specific +`release.install.on_failure` or `release.upgrade.on_failure` are stack +configuration, not project-wide `components.helm` settings in `atmos.yaml`. +This keeps release policy subject to stack imports, component inheritance, and +environment-specific overrides. See +[Helm stack configuration](/stacks/components/helm#release-lifecycle). +::: + ## Helm Repositories Global repositories are declared once in `atmos.yaml`: diff --git a/website/docs/stacks/components/helm.mdx b/website/docs/stacks/components/helm.mdx index 9bd712768a..497917bc9a 100644 --- a/website/docs/stacks/components/helm.mdx +++ b/website/docs/stacks/components/helm.mdx @@ -73,8 +73,126 @@ included in affected runs.
`provision`
Delivery targets for apply/deploy — the cluster (default) or an external target such as a Git deployment repository.
+ +
`secrets`
+
Component-scoped secret declarations and providers. Helm components support the same secret processing as other component types.
+## Type-Level Defaults and Overrides + +The top-level `helm` section in a stack manifest can provide defaults for every +native Helm component in that stack. Use `helm.values` for shared chart values; +component and inherited `values` are merged over those defaults. + +Use `helm.overrides` when a stack must enforce values after component-level +configuration is resolved. Type-level overrides are deep-merged over each +component's `overrides`, including components supplied by imported manifests. +The block accepts `values` plus the common component override sections such as +`vars`, `env`, `settings`, `auth`, `secrets`, and `retry`. + +```yaml +helm: + values: + cluster: shared + overrides: + values: + environment: production + +components: + helm: + monitoring: + chart: ./charts/monitoring + values: + replicaCount: 2 + overrides: + values: + image: + tag: stable +``` + +In this example, every Helm component receives `cluster: shared`, while the +stack-level override enforces `environment: production` after component +inheritance and component overrides are merged. + +## Release Lifecycle + +Cluster-backed `apply`, `deploy`, and `delete` operations can use Helm 4 release +lifecycle controls under `release`. Configure the tree at the top-level +`helm.release` section as defaults, on an abstract component for inheritance, or +on a concrete component. Atmos first deep-merges that complete tree, then +overlays the selected `install`, `upgrade`, or `delete` section. Explicit command +flags have the highest precedence. + +| Field | Default | Operations | Behavior | +| --- | --- | --- | --- | +| `release.wait.strategy` | `hookOnly` | install, upgrade, delete | `hookOnly`, `watcher`, or Helm 3-compatible `legacy`. | +| `release.wait.jobs` | `false` | install, upgrade | Wait for ordinary Jobs. Requires `watcher` or `legacy`; hook Jobs are already handled by Helm hooks. | +| `release.timeout` | `0s` during migration | install, upgrade, delete | Release-wide operation timeout. Each operation can override it. Explicit `0s` remains unbounded. | +| `release.history.max` | `10` | upgrade | Revisions retained. Set `0` for unlimited history. | +| `release.chart_hooks` | `true` | install, upgrade, delete | Enable Helm chart hooks. This does not control Atmos `hooks:`. | +| `release.install.crds` | `create` | install | Create or skip CRDs from the chart on first install. | +| `release.install.on_failure` | `keep` | install | `uninstall` removes a failed first install; `keep` preserves partial state. | +| `release.upgrade.on_failure` | `keep` | upgrade | `rollback` restores the prior release; `keep` preserves failed state. | +| `release.upgrade.cleanup_on_failure` | `false` | upgrade | Independently remove resources newly created by a failed upgrade. | + +`release.install.timeout`, `release.upgrade.timeout`, and +`release.delete.timeout` override the release-wide timeout only for that action. +The same operation sections can override `chart_hooks` and `wait`. + +:::warning Timeout and history migration +For one minor release, omitting `release.timeout` and the selected operation +timeout preserves the previous unbounded `0s` +behavior and emits a warning. The following minor changes the omitted default to +`5m`. Set `release.timeout: 0s` explicitly to remain unbounded, or set a duration +such as `30m`. An omitted `release.history.max` retains ten revisions; set it to +`0` if unlimited release history is required. +::: + + +```yaml +helm: + release: + wait: + strategy: watcher + timeout: 10m + history: + max: 10 + +components: + helm: + release-policy: + metadata: + type: abstract + release: + upgrade: + on_failure: rollback + cleanup_on_failure: true + + demo-release: + metadata: + inherits: + - release-policy + chart: ./charts/demo-release + namespace: demo + release: + install: + timeout: 60m + on_failure: uninstall + upgrade: + timeout: 30m +``` + + +With `dependencies.components`, a successful Helm DAG node means the selected +Helm action completed under this effective policy. `watcher` and `legacy` gate +dependents on resource readiness; `hookOnly` intentionally does not wait for +ordinary chart resources. A failed or rolled-back node remains failed, so its +dependents do not start. + +Lifecycle policy applies only to the Kubernetes target. Stored lifecycle values +are ignored and reported as bypassed for external Git delivery; explicitly +passing lifecycle flags with an external target is an error. + Reusable repository defaults can also be configured under `components.helm.repositories` in `atmos.yaml`. Component-level repositories override global entries with the same `name`. @@ -101,6 +219,17 @@ components: - name: prometheus-community url: https://prometheus-community.github.io/helm-charts namespace: monitoring + release: + timeout: 20m + wait: + strategy: watcher + jobs: true + history: + max: 10 + install: + on_failure: uninstall + upgrade: + on_failure: rollback values: grafana: enabled: true