From 678c287d92408ae69e76d0a6414256df2a01c5fd Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 02:48:05 +0400 Subject: [PATCH 01/62] docs: document Helm release lifecycle --- ...026-08-01-native-helm-release-lifecycle.md | 23 ++++++ errors/errors.go | 1 + examples/helm/README.md | 9 ++- examples/helm/atmos.yaml | 6 ++ examples/helm/stacks/deploy/dev.yaml | 8 +++ pkg/ci/plugins/helm/plugin.go | 16 +++++ pkg/ci/plugins/helm/plugin_test.go | 34 +++++++++ pkg/ci/plugins/helm/templates/apply.md | 31 ++++++++ pkg/ci/plugins/helm/templates/delete.md | 12 ++++ pkg/component/helm/client.go | 29 ++++++-- pkg/component/helm/client_lifecycle_test.go | 34 +++++++++ pkg/component/helm/client_test.go | 21 ++++++ website/docs/ci/job-summaries.mdx | 7 ++ website/docs/cli/commands/helm/helm-apply.mdx | 40 +++++++++++ .../docs/cli/commands/helm/helm-delete.mdx | 15 ++++ website/docs/cli/commands/helm/usage.mdx | 15 ++++ .../cli/configuration/components/helm.mdx | 8 +++ website/docs/stacks/components/helm.mdx | 71 +++++++++++++++++++ 18 files changed, 373 insertions(+), 7 deletions(-) create mode 100644 docs/fixes/2026-08-01-native-helm-release-lifecycle.md diff --git a/docs/fixes/2026-08-01-native-helm-release-lifecycle.md b/docs/fixes/2026-08-01-native-helm-release-lifecycle.md new file mode 100644 index 0000000000..ffb96701a9 --- /dev/null +++ b/docs/fixes/2026-08-01-native-helm-release-lifecycle.md @@ -0,0 +1,23 @@ +# Native Helm release lifecycle + +**Date:** 2026-08-01 + +Native Helm cluster operations now expose Helm 4 wait, timeout, recovery, +history, hook, and CRD controls through stack configuration and explicit command +flags. Apply and delete dry runs now reach the Helm SDK without persisting release +state, and caller cancellation propagates through direct and dependency-ordered +execution. + +## Migration notes + +- An omitted `timeout` remains `0s` (unbounded) for one minor release and emits a + warning. The omitted default becomes `5m` in the following minor. Configure + `timeout: 0s` explicitly to keep unbounded behavior without the warning. +- An omitted `max_history` now retains ten upgrade revisions, matching the Helm + CLI. Configure `max_history: 0` to retain unlimited history. +- `atomic` is deprecated in favor of `rollback_on_failure`. +- Boolean `--wait=true` and `--wait=false` remain accepted temporarily; use + `--wait=watcher` and `--wait=hookOnly`. +- Explicit lifecycle flags cannot be combined with a non-Kubernetes provision + target. Stored lifecycle configuration is intentionally bypassed for external + delivery and is identified as such in the execution summary. diff --git a/errors/errors.go b/errors/errors.go index c96af19ffc..aac0514868 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -1539,6 +1539,7 @@ var ( ErrHelmReleaseHistory = errors.New("failed to inspect helm release history") ErrHelmReleaseUpgrade = errors.New("failed to upgrade helm release") ErrHelmReleaseUninstall = errors.New("failed to uninstall helm release") + ErrHelmReleaseOperation = errors.New("helm release operation failed") ) // Stack dependency (`depends_on`) resolution errors. diff --git a/examples/helm/README.md b/examples/helm/README.md index 0f9d939617..8985028905 100644 --- a/examples/helm/README.md +++ b/examples/helm/README.md @@ -20,9 +20,11 @@ atmos validate stacks atmos helm template demo -s dev atmos emulator up kubernetes -s dev atmos helm diff demo -s dev --identity local-k3s -atmos helm apply demo -s dev --identity local-k3s +atmos helm apply demo -s dev --identity local-k3s --dry-run +atmos helm apply demo -s dev --identity local-k3s --rollback-on-failure --wait=watcher --timeout=2m atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo +atmos helm delete demo -s dev --identity local-k3s --dry-run --wait=watcher --timeout=2m atmos helm delete demo -s dev --identity local-k3s atmos emulator down kubernetes -s dev ``` @@ -69,6 +71,11 @@ atmos emulator up kubernetes -s dev atmos helm apply demo -s dev --identity local-k3s ``` +The stack sets `wait_strategy: watcher`, `timeout: 2m`, and +`max_history: 10` as native Helm type defaults. The component enables +`rollback_on_failure` and failed-upgrade cleanup. The `atmos test` workflow also +proves that apply and delete dry runs do not create or remove the Deployment. + ## Helm Repositories The `demo-repo` component shows the declarative Helm repository path: diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 83365237d3..2a96b872d7 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -51,6 +51,9 @@ commands: max_attempts: 2 initial_delay: 15s backoff_strategy: constant + # Apply dry-run must not persist a release or create Kubernetes objects. + - atmos helm apply demo -s dev --identity local-k3s --dry-run + - command: if atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo; then echo "dry-run unexpectedly created deployment/demo"; exit 1; fi - command: atmos helm apply demo -s dev --identity local-k3s retry: max_attempts: 2 @@ -66,5 +69,8 @@ commands: max_attempts: 3 initial_delay: 10s backoff_strategy: constant + # Delete dry-run must leave the deployed release and resources intact. + - atmos helm delete demo -s dev --identity local-k3s --dry-run --wait=watcher --timeout=2m + - atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo - atmos helm delete demo -s dev --identity local-k3s - atmos emulator down kubernetes -s dev diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index ae53528c78..e402aece42 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -1,6 +1,12 @@ vars: stage: dev +# Native Helm lifecycle defaults for every Helm component in this stack. +helm: + wait_strategy: watcher + timeout: 2m + max_history: 10 + components: emulator: kubernetes: @@ -18,6 +24,8 @@ components: # (components/helm/demo), which contains Chart.yaml. chart: "." namespace: demo + rollback_on_failure: true + cleanup_on_fail: true # Atmos `values:` are the Helm chart values, merged via Atmos inheritance. values: replicaCount: 2 diff --git a/pkg/ci/plugins/helm/plugin.go b/pkg/ci/plugins/helm/plugin.go index abc597a5df..123a065491 100644 --- a/pkg/ci/plugins/helm/plugin.go +++ b/pkg/ci/plugins/helm/plugin.go @@ -118,6 +118,7 @@ func (p *Plugin) buildTemplateContext(ctx *plugin.HookContext) *TemplateContext ObjectCount: data.ObjectCount, ObjectKinds: data.ObjectKinds, ManifestBytes: data.ManifestBytes, + Lifecycle: data.Lifecycle, Message: data.Message, Diff: plugin.TruncateDetail(data.Diff), } @@ -135,6 +136,7 @@ type TemplateContext struct { ObjectCount int ObjectKinds []string ManifestBytes int + Lifecycle map[string]any Message string // Diff is the unified diff produced by `helm diff`/`plan` (empty otherwise). Diff string @@ -152,6 +154,7 @@ type Summary struct { ObjectCount int ObjectKinds []string ManifestBytes int + Lifecycle map[string]any Message string // Diff is the unified diff produced by `helm diff`/`plan` (empty otherwise). Diff string @@ -185,6 +188,7 @@ func summaryFromMap(m map[string]any) Summary { ObjectCount: intValue(m["object_count"]), ObjectKinds: stringSliceValue(m["object_kinds"]), ManifestBytes: intValue(m["manifest_bytes"]), + Lifecycle: mapValue(m["lifecycle"]), Message: stringValue(m["message"]), Diff: stringValue(m["diff"]), } @@ -192,6 +196,18 @@ func summaryFromMap(m map[string]any) Summary { return s } +func mapValue(value any) map[string]any { + typed, ok := value.(map[string]any) + if !ok { + return nil + } + result := make(map[string]any, len(typed)) + for key, item := range typed { + result[key] = item + } + return result +} + func helmTemplateName(command string) string { switch command { case "render": diff --git a/pkg/ci/plugins/helm/plugin_test.go b/pkg/ci/plugins/helm/plugin_test.go index 83f212a4c9..c49ffd39b8 100644 --- a/pkg/ci/plugins/helm/plugin_test.go +++ b/pkg/ci/plugins/helm/plugin_test.go @@ -60,6 +60,10 @@ func TestPlugin_BuildTemplateContext(t *testing.T) { "object_count": 2, "object_kinds": []any{"Service", "Deployment"}, "manifest_bytes": 1234, + "lifecycle": map[string]any{ + "operation": "upgrade", + "wait_strategy": "watcher", + }, }, }) @@ -73,6 +77,8 @@ func TestPlugin_BuildTemplateContext(t *testing.T) { assert.Equal(t, 2, ctx.ObjectCount) assert.Equal(t, 1234, ctx.ManifestBytes) assert.Equal(t, []string{"Deployment", "Service"}, ctx.ObjectKinds) + assert.Equal(t, "upgrade", ctx.Lifecycle["operation"]) + assert.Equal(t, "watcher", ctx.Lifecycle["wait_strategy"]) } func TestNormalizeSummary(t *testing.T) { @@ -94,6 +100,7 @@ func TestNormalizeSummary(t *testing.T) { "manifest_bytes": float64(123), "message": 42, "diff": "diff text", + "lifecycle": map[string]any{"operation": "install"}, }) assert.Equal(t, "app", got.Component) assert.Equal(t, "dev", got.Stack) @@ -107,6 +114,7 @@ func TestNormalizeSummary(t *testing.T) { assert.Equal(t, 123, got.ManifestBytes) assert.Equal(t, "42", got.Message) assert.Equal(t, "diff text", got.Diff) + assert.Equal(t, map[string]any{"operation": "install"}, got.Lifecycle) } func TestPluginBuildTemplateContextFallbacksAndErrors(t *testing.T) { @@ -146,6 +154,7 @@ func TestSummaryEnabledAndPrimitiveConversions(t *testing.T) { assert.Zero(t, intValue("9")) assert.Equal(t, []string{"b", "a"}, stringSliceValue([]string{"b", "a"})) assert.Nil(t, stringSliceValue(1)) + assert.Nil(t, mapValue("not-a-map")) assert.Equal(t, "Helm", title("")) } @@ -241,6 +250,16 @@ func TestTemplateRendering(t *testing.T) { ObjectCount: 2, ObjectKinds: []string{"Deployment", "Service"}, ManifestBytes: 1234, + Lifecycle: map[string]any{ + "operation": "upgrade", + "wait_strategy": "watcher", + "timeout": "30m0s", + "chart_hooks_enabled": true, + "wait_for_jobs": true, + "rollback_on_failure": true, + "cleanup_on_fail": true, + "max_history": 10, + }, }, }) @@ -249,4 +268,19 @@ func TestTemplateRendering(t *testing.T) { assert.Contains(t, rendered, "Helm Apply Summary") assert.Contains(t, rendered, "bitnami/nginx") assert.Contains(t, rendered, "Deployment") + assert.Contains(t, rendered, "Release lifecycle") + assert.Contains(t, rendered, "watcher") + assert.Contains(t, rendered, "Maximum history") + assert.Contains(t, rendered, "`10`") + + external := (&Plugin{}).buildTemplateContext(&plugin.HookContext{ + Command: "apply", + Aggregate: Summary{Lifecycle: map[string]any{ + "applied": false, "target_kind": "git", "reason": "external_target", + }}, + }) + rendered, err = templates.NewLoader(nil).LoadAndRender("helm", "apply", defaultTemplates, external) + require.NoError(t, err) + assert.Contains(t, rendered, "external_target") + assert.NotContains(t, rendered, "Wait strategy") } diff --git a/pkg/ci/plugins/helm/templates/apply.md b/pkg/ci/plugins/helm/templates/apply.md index ae3d8b6e3a..b8dd0d3db1 100644 --- a/pkg/ci/plugins/helm/templates/apply.md +++ b/pkg/ci/plugins/helm/templates/apply.md @@ -14,6 +14,37 @@ | Objects | `{{ .ObjectCount }}` | | Manifest bytes | `{{ .ManifestBytes }}` | +{{- with .Lifecycle }} + +### Release lifecycle + +{{- if eq (index . "reason") "external_target" }} + +| Field | Value | +| --- | --- | +| Applied | `false` | +| Target kind | `{{ index . "target_kind" }}` | +| Reason | `external_target` | +{{- else }} + +| Field | Value | +| --- | --- | +| Operation | `{{ index . "operation" }}` | +| Wait strategy | `{{ index . "wait_strategy" }}` | +| Timeout | `{{ index . "timeout" }}` | +| Chart hooks enabled | `{{ index . "chart_hooks_enabled" }}` | +| Wait for Jobs | `{{ index . "wait_for_jobs" }}` | +| Rollback on failure | `{{ index . "rollback_on_failure" }}` | +{{- if eq (index . "operation") "install" }} +| Install CRDs | `{{ index . "install_crds" }}` | +{{- end }} +{{- if eq (index . "operation") "upgrade" }} +| Cleanup on failure | `{{ index . "cleanup_on_fail" }}` | +| Maximum history | `{{ index . "max_history" }}` | +{{- end }} +{{- end }} +{{- end }} + To reproduce locally: ```shell diff --git a/pkg/ci/plugins/helm/templates/delete.md b/pkg/ci/plugins/helm/templates/delete.md index 88577fab0f..8ae1020b51 100644 --- a/pkg/ci/plugins/helm/templates/delete.md +++ b/pkg/ci/plugins/helm/templates/delete.md @@ -10,6 +10,18 @@ | Release | `{{ .ReleaseName }}` | | Namespace | `{{ .Namespace }}` | +{{- with .Lifecycle }} + +### Release lifecycle + +| Field | Value | +| --- | --- | +| Operation | `{{ index . "operation" }}` | +| Wait strategy | `{{ index . "wait_strategy" }}` | +| Timeout | `{{ index . "timeout" }}` | +| Chart hooks enabled | `{{ index . "chart_hooks_enabled" }}` | +{{- end }} + To reproduce locally: ```shell diff --git a/pkg/component/helm/client.go b/pkg/component/helm/client.go index abe29f0b68..cab7ad88f0 100644 --- a/pkg/component/helm/client.go +++ b/pkg/component/helm/client.go @@ -16,6 +16,7 @@ import ( "helm.sh/helm/v4/pkg/storage/driver" errUtils "github.com/cloudposse/atmos/errors" + cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/perf" ) @@ -132,7 +133,11 @@ func installRelease(ctx context.Context, actx *actionContext, spec *chartSpec, d if dryRun { client.DryRunStrategy = action.DryRunServer } - return runInstall(ctx, client, actx.settings, spec) + manifest, err := runInstall(ctx, client, actx.settings, spec) + if err != nil { + return "", releaseOperationError("install", spec, err) + } + return manifest, nil } func upgradeRelease(ctx context.Context, actx *actionContext, spec *chartSpec, dryRun bool) (string, error) { @@ -164,10 +169,7 @@ func upgradeRelease(ctx context.Context, actx *actionContext, spec *chartSpec, d if ctxErr := ctx.Err(); ctxErr != nil { return "", ctxErr } - if errors.Is(err, errUtils.ErrHelmRenderFailed) { - return "", fmt.Errorf("failed to upgrade Helm release %q: %w", spec.ReleaseName, err) - } - return "", fmt.Errorf("%w %q: %w", errUtils.ErrHelmReleaseUpgrade, spec.ReleaseName, err) + return "", releaseOperationError("upgrade", spec, err) } rendered, ok := rel.(*release.Release) if !ok { @@ -242,7 +244,7 @@ func deleteRelease(ctx context.Context, spec *chartSpec, dryRun bool) error { if errors.Is(err, driver.ErrReleaseNotFound) { return nil } - uninstallErr := fmt.Errorf("%w %q: %w", errUtils.ErrHelmReleaseUninstall, spec.ReleaseName, err) + uninstallErr := releaseOperationError("delete", spec, err) if ctxErr := operationCtx.Err(); ctxErr != nil { return errors.Join(ctxErr, uninstallErr) } @@ -254,6 +256,21 @@ func deleteRelease(ctx context.Context, spec *chartSpec, dryRun bool) error { return nil } +func releaseOperationError(operation string, spec *chartSpec, cause error) error { + policy := spec.Lifecycle.Policy + return fmt.Errorf( + "%w: operation=%s release=%q namespace=%q wait_strategy=%s timeout=%s (component field %q): %w", + errUtils.ErrHelmReleaseOperation, + operation, + spec.ReleaseName, + spec.Namespace, + policy.WaitStrategy, + policy.Timeout, + cfg.HelmTimeoutSectionName, + cause, + ) +} + func configureInstallLifecycle(client *action.Install, policy effectiveReleasePolicy) { client.RollbackOnFailure = policy.OnFailure == failurePolicyUninstall client.WaitStrategy = policy.WaitStrategy diff --git a/pkg/component/helm/client_lifecycle_test.go b/pkg/component/helm/client_lifecycle_test.go index 9e3328008b..826b9d819f 100644 --- a/pkg/component/helm/client_lifecycle_test.go +++ b/pkg/component/helm/client_lifecycle_test.go @@ -213,3 +213,37 @@ func TestReleaseOperationContextPreservesZeroTimeout(t *testing.T) { _, hasDeadline := ctx.Deadline() assert.False(t, hasDeadline) } + +func TestUpgradeReleasePrunesDefaultHistory(t *testing.T) { + actx := memoryActionContext(t) + stubActionContext(t, actx) + spec := testdataChartSpec(t, "history") + + for revision := 0; revision < defaultHelmMaxHistory+3; revision++ { + spec.Values["replicaCount"] = revision + 1 + _, err := applyRelease(context.Background(), spec, false) + require.NoError(t, err) + } + + history, err := actx.cfg.Releases.History(spec.ReleaseName) + require.NoError(t, err) + assert.Len(t, history, defaultHelmMaxHistory) +} + +func TestUpgradeReleaseUnlimitedHistory(t *testing.T) { + actx := memoryActionContext(t) + stubActionContext(t, actx) + spec := testdataChartSpec(t, "unlimited-history") + spec.Lifecycle.Policy.MaxHistory = 0 + + const revisions = defaultHelmMaxHistory + 3 + for revision := 0; revision < revisions; revision++ { + spec.Values["replicaCount"] = revision + 1 + _, err := applyRelease(context.Background(), spec, false) + require.NoError(t, err) + } + + history, err := actx.cfg.Releases.History(spec.ReleaseName) + require.NoError(t, err) + assert.Len(t, history, revisions) +} diff --git a/pkg/component/helm/client_test.go b/pkg/component/helm/client_test.go index f52f213cea..33fbbe291e 100644 --- a/pkg/component/helm/client_test.go +++ b/pkg/component/helm/client_test.go @@ -99,6 +99,27 @@ func TestConfigureReleaseLifecycleActions(t *testing.T) { assert.True(t, uninstall.DryRun) } +func TestReleaseOperationErrorIncludesEffectivePolicy(t *testing.T) { + cause := context.DeadlineExceeded + err := releaseOperationError("upgrade", &chartSpec{ + ReleaseName: "demo", + Namespace: "apps", + Lifecycle: releaseLifecycleResolution{Policy: releaseLifecycle{ + WaitStrategy: kube.StatusWatcherStrategy, + Timeout: 7 * time.Minute, + }}, + }, cause) + + require.ErrorIs(t, err, errUtils.ErrHelmReleaseOperation) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Contains(t, err.Error(), "operation=upgrade") + assert.Contains(t, err.Error(), `release="demo"`) + assert.Contains(t, err.Error(), `namespace="apps"`) + assert.Contains(t, err.Error(), "wait_strategy=watcher") + assert.Contains(t, err.Error(), "timeout=7m0s") + assert.Contains(t, err.Error(), `component field "timeout"`) +} + func TestClusterOperationsReturnActionContextErrors(t *testing.T) { original := newActionContext t.Cleanup(func() { newActionContext = original }) diff --git a/website/docs/ci/job-summaries.mdx b/website/docs/ci/job-summaries.mdx index c66cf626e7..72655e0f34 100644 --- a/website/docs/ci/job-summaries.mdx +++ b/website/docs/ci/job-summaries.mdx @@ -124,6 +124,13 @@ comments, or artifacts. The summary includes component, stack, command status, a command, and Helm metadata such as release name, namespace, chart, target, object counts, object kinds, and rendered manifest size when available. +For cluster-backed apply/deploy/delete operations, the aggregate also includes +an operation-specific `lifecycle` block with the effective wait strategy, +timeout, chart-hook state, and applicable rollback, Job-wait, CRD, cleanup, and +history values. For external delivery, it reports `applied: false`, the selected +target kind, and `reason: external_target` instead of presenting stored release +policy as active. + ## Helmfile Summaries Helmfile components write summaries for these operations when `ci.enabled: true` and CI mode is diff --git a/website/docs/cli/commands/helm/helm-apply.mdx b/website/docs/cli/commands/helm/helm-apply.mdx index 1312f84058..d331f37e06 100644 --- a/website/docs/cli/commands/helm/helm-apply.mdx +++ b/website/docs/cli/commands/helm/helm-apply.mdx @@ -26,6 +26,18 @@ Install or upgrade the release in the cluster: atmos helm apply monitoring -s plat-ue2-dev ``` +Apply an explicit production readiness and recovery policy: + +```shell +atmos helm apply monitoring -s plat-ue2-dev \ + --rollback-on-failure \ + --wait=watcher \ + --wait-for-jobs \ + --timeout=30m \ + --cleanup-on-fail \ + --history-max=10 +``` + Deliver rendered manifests to a provision target instead of the cluster: ```shell @@ -58,6 +70,30 @@ atmos helm apply --affected --labels cost-center=platform
`--dependency-update` (optional)
Fetch declared chart dependencies when they are missing. This may access dependency repositories and update the chart's charts/ directory and lock file.
+
`--rollback-on-failure` (optional)
+
Uninstall a failed first install or roll a failed upgrade back. The deprecated alias is `--atomic`.
+ +
`--wait[=strategy]` (optional)
+
Use `watcher`, `hookOnly`, or `legacy`. Passing `--wait` without a value selects `watcher`. Boolean values remain accepted temporarily but are deprecated.
+ +
`--wait-for-jobs` (optional)
+
Wait for ordinary Jobs in the release manifest. Requires `watcher` or `legacy`.
+ +
`--timeout` (optional)
+
Helm release-operation timeout, such as `10m` or `1h`. `0s` is explicitly unbounded.
+ +
`--cleanup-on-fail` (optional)
+
Delete resources newly created by a failed upgrade before rollback.
+ +
`--history-max` (optional)
+
Maximum retained release revisions. Defaults to `10`; `0` means unlimited.
+ +
`--no-hooks` (optional)
+
Disable Helm chart hooks. Atmos lifecycle hooks are unaffected.
+ +
`--skip-crds` (optional)
+
Skip CRD installation on a first install.
+
`--all` (optional)
Apply all Helm components in dependency order.
@@ -73,3 +109,7 @@ atmos helm apply --affected --labels cost-center=platform
`--labels` (optional)
Filter by labels (comma-separated `key=value` or `key:value` pairs, matches all): `--labels=cost-center=platform,compliance=sox`. Composes with `--all`/`--affected`/`--tags`; cannot be combined with a single component argument.
+ +Lifecycle flags apply only to direct Kubernetes delivery. Combining an explicit +lifecycle flag with an external `--target` fails instead of implying that Atmos +waited for or rolled back a GitOps deployment. diff --git a/website/docs/cli/commands/helm/helm-delete.mdx b/website/docs/cli/commands/helm/helm-delete.mdx index 487369ca2a..f63db2d447 100644 --- a/website/docs/cli/commands/helm/helm-delete.mdx +++ b/website/docs/cli/commands/helm/helm-delete.mdx @@ -18,6 +18,9 @@ import Intro from "@site/src/components/Intro"; atmos helm delete --stack [options] atmos helm delete monitoring -s plat-ue2-dev +# Preview without removing release state or Kubernetes resources +atmos helm delete monitoring -s plat-ue2-dev --dry-run --wait=watcher --timeout=10m + # Delete components filtered by tags or labels (composes with --all/--affected) atmos helm delete --all --tags production,tier-1 ``` @@ -33,4 +36,16 @@ atmos helm delete --all --tags production,tier-1
`--tags` / `--labels` (optional)
Filter by tags (comma-separated, matches any) or labels (comma-separated `key=value` or `key:value` pairs, matches all): `--tags=production,tier-1`, `--labels=cost-center=platform`. Compose with `--all`/`--affected` to narrow the selected set further; cannot be combined with a single component argument.
+ +
`--wait[=strategy]` (optional)
+
Use `watcher`, `hookOnly`, or `legacy` while deleting. Passing `--wait` without a value selects `watcher`.
+ +
`--timeout` (optional)
+
Helm uninstall timeout, such as `10m`. `0s` is explicitly unbounded.
+ +
`--no-hooks` (optional)
+
Disable Helm chart uninstall hooks. Atmos lifecycle hooks are unaffected.
+ +
`--dry-run` (optional)
+
Preview the uninstall without deleting release history or Kubernetes resources.
diff --git a/website/docs/cli/commands/helm/usage.mdx b/website/docs/cli/commands/helm/usage.mdx index fb6aecc26c..4e28e62131 100644 --- a/website/docs/cli/commands/helm/usage.mdx +++ b/website/docs/cli/commands/helm/usage.mdx @@ -72,6 +72,21 @@ Supported summaries: Summaries include component, stack, command status, a local reproduction command, and Helm metadata such as release name, namespace, chart, target, object counts, object kinds, and rendered manifest size when available. +Cluster-backed release summaries also show the effective Helm lifecycle policy. +External-target summaries explicitly report that release lifecycle was bypassed. + +## Release lifecycle + +For cluster delivery, `apply`/`deploy` and `delete` expose Helm 4 wait and timeout +behavior. Apply/deploy additionally support rollback on failure, ordinary Job +waiting, failed-upgrade cleanup, history limits, chart-hook suppression, and CRD +skipping. Configure defaults and inherited policy in the stack component, then +use command flags only for an explicit invocation override. + +See [Helm stack release lifecycle](/stacks/components/helm#release-lifecycle), +[`helm apply`](/cli/commands/helm/apply), and +[`helm delete`](/cli/commands/helm/delete) for the complete field and flag +reference. ## Chart sources diff --git a/website/docs/cli/configuration/components/helm.mdx b/website/docs/cli/configuration/components/helm.mdx index dc82df2e08..3212b2e9cd 100644 --- a/website/docs/cli/configuration/components/helm.mdx +++ b/website/docs/cli/configuration/components/helm.mdx @@ -54,6 +54,14 @@ components: +:::info Release policy belongs in stacks +Helm 4 lifecycle fields such as `wait_strategy`, `timeout`, +`rollback_on_failure`, and `max_history` are stack configuration, not +project-wide `components.helm` settings in `atmos.yaml`. This keeps release +policy subject to stack imports, component inheritance, and environment-specific +overrides. See [Helm stack configuration](/stacks/components/helm#release-lifecycle). +::: + ## Helm Repositories Global repositories are declared once in `atmos.yaml`: diff --git a/website/docs/stacks/components/helm.mdx b/website/docs/stacks/components/helm.mdx index 9bd712768a..fe4d12ca74 100644 --- a/website/docs/stacks/components/helm.mdx +++ b/website/docs/stacks/components/helm.mdx @@ -75,6 +75,72 @@ included in affected runs.
Delivery targets for apply/deploy — the cluster (default) or an external target such as a Git deployment repository.
+## Release Lifecycle + +Cluster-backed `apply`, `deploy`, and `delete` operations can use Helm 4 release +lifecycle controls. Configure them at the top-level `helm` section as defaults, +on an abstract component for inheritance, or on a concrete component. Normal +Atmos precedence applies; concrete components override inherited and type-level +values. + +| Field | Default | Operations | Behavior | +| --- | --- | --- | --- | +| `rollback_on_failure` | `false` | install, upgrade | Uninstall a failed first install or roll a failed upgrade back. Enabling it promotes the default wait strategy to `watcher`. | +| `wait_strategy` | `hookOnly` | install, upgrade, delete | `hookOnly`, `watcher`, or Helm 3-compatible `legacy`. | +| `wait_for_jobs` | `false` | install, upgrade | Wait for ordinary Jobs. Requires `watcher` or `legacy`; hook Jobs are already handled by Helm hooks. | +| `timeout` | `0s` during migration | install, upgrade, delete | Helm operation timeout. Explicit `0s` remains unbounded. | +| `cleanup_on_fail` | `false` | upgrade | Remove resources newly created by a failed upgrade. | +| `max_history` | `10` | upgrade | Revisions retained. Set `0` for unlimited history. | +| `disable_chart_hooks` | `false` | install, upgrade, delete | Disable Helm chart hooks. This does not disable Atmos `hooks:`. | +| `skip_crds` | `false` | install | Do not install CRDs from the chart on first install. | + +`atomic` is a deprecated alias for `rollback_on_failure`; `wait: true` and +`wait: false` are convenience aliases for `watcher` and `hookOnly`. Canonical +fields win when both forms are inherited or configured. + +:::warning Timeout and history migration +For one minor release, omitting `timeout` preserves the previous unbounded `0s` +behavior and emits a warning. The following minor changes the omitted default to +`5m`. Set `timeout: 0s` explicitly to remain unbounded, or set a duration such as +`30m`. An omitted `max_history` now retains ten revisions; set +`max_history: 0` if unlimited release history is required. +::: + + +```yaml +helm: + wait_strategy: watcher + timeout: 10m + max_history: 10 + +components: + helm: + release-policy: + metadata: + type: abstract + rollback_on_failure: true + cleanup_on_fail: true + + demo-release: + metadata: + inherits: + - release-policy + chart: ./charts/demo-release + namespace: demo + timeout: 30m +``` + + +With `dependencies.components`, a successful Helm DAG node means the selected +Helm action completed under this effective policy. `watcher` and `legacy` gate +dependents on resource readiness; `hookOnly` intentionally does not wait for +ordinary chart resources. A failed or rolled-back node remains failed, so its +dependents do not start. + +Lifecycle policy applies only to the Kubernetes target. Stored lifecycle values +are ignored and reported as bypassed for external Git delivery; explicitly +passing lifecycle flags with an external target is an error. + Reusable repository defaults can also be configured under `components.helm.repositories` in `atmos.yaml`. Component-level repositories override global entries with the same `name`. @@ -101,6 +167,11 @@ components: - name: prometheus-community url: https://prometheus-community.github.io/helm-charts namespace: monitoring + wait_strategy: watcher + wait_for_jobs: true + timeout: 20m + rollback_on_failure: true + max_history: 10 values: grafana: enabled: true From bd6b3900758fc57c8b85d5de6e88296bd9ea9a38 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 02:54:51 +0400 Subject: [PATCH 02/62] test: cover Helm release lifecycle in k3s --- examples/helm/README.md | 8 +- examples/helm/atmos.yaml | 36 +++++++ .../crds/lifecycle.atmos.test_widgets.yaml | 18 ++++ .../helm/demo/templates/deployment.yaml | 6 ++ .../helm/demo/templates/extra-configmap.yaml | 9 ++ .../helm/demo/templates/failing-hook-job.yaml | 20 ++++ .../demo/templates/hook-order-configmap.yaml | 14 +++ .../helm/demo/templates/hook-order-job.yaml | 27 +++++ .../components/helm/demo/templates/job.yaml | 16 +++ .../helm/demo/templates/ready-marker.yaml | 13 +++ .../helm/components/helm/demo/values.yaml | 12 +++ examples/helm/stacks/deploy/dev.yaml | 99 +++++++++++++++++++ 12 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 examples/helm/components/helm/demo/crds/lifecycle.atmos.test_widgets.yaml create mode 100644 examples/helm/components/helm/demo/templates/extra-configmap.yaml create mode 100644 examples/helm/components/helm/demo/templates/failing-hook-job.yaml create mode 100644 examples/helm/components/helm/demo/templates/hook-order-configmap.yaml create mode 100644 examples/helm/components/helm/demo/templates/hook-order-job.yaml create mode 100644 examples/helm/components/helm/demo/templates/job.yaml create mode 100644 examples/helm/components/helm/demo/templates/ready-marker.yaml diff --git a/examples/helm/README.md b/examples/helm/README.md index 8985028905..24daa893bc 100644 --- a/examples/helm/README.md +++ b/examples/helm/README.md @@ -73,8 +73,12 @@ atmos helm apply demo -s dev --identity local-k3s The stack sets `wait_strategy: watcher`, `timeout: 2m`, and `max_history: 10` as native Helm type defaults. The component enables -`rollback_on_failure` and failed-upgrade cleanup. The `atmos test` workflow also -proves that apply and delete dry runs do not create or remove the Deployment. +`rollback_on_failure` and failed-upgrade cleanup. The `atmos test` workflow +covers non-mutating apply and delete dry runs, ordinary Job waiting, chart-hook +suppression, CRD skipping, weighted hook ordering, retained hook resources, +install and upgrade rollback, failed-upgrade cleanup, timeout handling, and +dependency-gated Helm releases. The intentionally slow resources are observed +with bounded Kubernetes readiness checks rather than fixed-delay assertions. ## Helm Repositories diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 2a96b872d7..8ba65315b0 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -59,6 +59,10 @@ commands: max_attempts: 2 initial_delay: 15s backoff_strategy: constant + # The -2 ConfigMap must exist before the -1 hook Job can mount it. + - atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-hook-order + # skip_crds is a stack-level lifecycle default. + - command: if atmos emulator exec kubernetes -s dev -- kubectl get crd widgets.lifecycle.atmos.test; then echo "skip_crds unexpectedly installed widgets.lifecycle.atmos.test"; exit 1; fi - command: atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo retry: max_attempts: 3 @@ -69,8 +73,40 @@ commands: max_attempts: 3 initial_delay: 10s backoff_strategy: constant + # watcher + wait_for_jobs returns only after the ordinary Job completes. + - atmos helm apply demo-jobs -s dev --identity local-k3s + - atmos emulator exec kubernetes -s dev -- kubectl -n demo-jobs wait --for=condition=complete job/demo-jobs-job --timeout=5s + # Disabling chart hooks prevents both weighted hook resources. + - atmos helm apply demo-no-hooks -s dev --identity local-k3s + - command: if atmos emulator exec kubernetes -s dev -- kubectl -n demo-no-hooks get configmap demo-no-hooks-hook-order; then echo "disable_chart_hooks unexpectedly ran Helm hooks"; exit 1; fi + # hookOnly returns after hooks without waiting for ordinary Deployment readiness. + - atmos helm apply demo-hook-only -s dev --identity local-k3s + - command: if atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only rollout status deployment/demo-hook-only --timeout=1s; then echo "hookOnly unexpectedly waited for Deployment readiness"; exit 1; fi + - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only rollout status deployment/demo-hook-only --timeout=30s + # watcher honors the release timeout and rollback removes the failed install. + - command: if atmos helm apply demo-timeout -s dev --identity local-k3s; then echo "timed release unexpectedly succeeded"; exit 1; fi + - command: if atmos emulator exec kubernetes -s dev -- kubectl -n demo-timeout get deployment demo-timeout; then echo "timeout rollback left release resources"; exit 1; fi + # A failed first install is reported as failure and rolled back. The kept + # hook ConfigMap survives recovery while ordinary release resources do not. + - command: if atmos helm apply demo-install-fail -s dev --identity local-k3s; then echo "failed install unexpectedly succeeded"; exit 1; fi + - command: if atmos emulator exec kubernetes -s dev -- kubectl -n demo-install-fail get deployment demo-install-fail; then echo "rollback left failed install resources"; exit 1; fi + - atmos emulator exec kubernetes -s dev -- kubectl -n demo-install-fail get configmap demo-install-fail-hook-order + # A failed upgrade restores the successful demo release and cleanup removes + # the resource introduced only by the failed revision. + - command: if atmos helm apply demo-upgrade-fail -s dev --identity local-k3s; then echo "failed upgrade unexpectedly succeeded"; exit 1; fi + - atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo + - command: if atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-upgrade-only; then echo "cleanup_on_fail left the upgrade-only ConfigMap"; exit 1; fi + # Dependency execution gates the dependent's pre-install hook on the + # foundation's post-readiness marker. + - atmos helm apply --all -s dev --identity local-k3s --tags lifecycle-dag + - atmos emulator exec kubernetes -s dev -- kubectl -n lifecycle-dag get deployment dag-dependent # Delete dry-run must leave the deployed release and resources intact. - atmos helm delete demo -s dev --identity local-k3s --dry-run --wait=watcher --timeout=2m - atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo - atmos helm delete demo -s dev --identity local-k3s + - atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-hook-order + - atmos helm delete demo-jobs -s dev --identity local-k3s + - atmos helm delete demo-no-hooks -s dev --identity local-k3s + - atmos helm delete demo-hook-only -s dev --identity local-k3s + - atmos helm delete --all -s dev --identity local-k3s --tags lifecycle-dag - atmos emulator down kubernetes -s dev diff --git a/examples/helm/components/helm/demo/crds/lifecycle.atmos.test_widgets.yaml b/examples/helm/components/helm/demo/crds/lifecycle.atmos.test_widgets.yaml new file mode 100644 index 0000000000..390a2826d1 --- /dev/null +++ b/examples/helm/components/helm/demo/crds/lifecycle.atmos.test_widgets.yaml @@ -0,0 +1,18 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: widgets.lifecycle.atmos.test +spec: + group: lifecycle.atmos.test + scope: Namespaced + names: + plural: widgets + singular: widget + kind: Widget + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object diff --git a/examples/helm/components/helm/demo/templates/deployment.yaml b/examples/helm/components/helm/demo/templates/deployment.yaml index 0efffbd106..d7028e4f3e 100644 --- a/examples/helm/components/helm/demo/templates/deployment.yaml +++ b/examples/helm/components/helm/demo/templates/deployment.yaml @@ -15,6 +15,12 @@ spec: labels: app: {{ .Release.Name }} spec: + {{- if gt (int .Values.deployment.readinessDelaySeconds) 0 }} + initContainers: + - name: readiness-delay + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: ["/bin/sh", "-c", "sleep {{ .Values.deployment.readinessDelaySeconds }}"] + {{- end }} containers: - name: {{ .Release.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/examples/helm/components/helm/demo/templates/extra-configmap.yaml b/examples/helm/components/helm/demo/templates/extra-configmap.yaml new file mode 100644 index 0000000000..ea425677fc --- /dev/null +++ b/examples/helm/components/helm/demo/templates/extra-configmap.yaml @@ -0,0 +1,9 @@ +{{- if .Values.extraConfigMap.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-upgrade-only + namespace: {{ .Release.Namespace }} +data: + created: during-failed-upgrade +{{- end }} diff --git a/examples/helm/components/helm/demo/templates/failing-hook-job.yaml b/examples/helm/components/helm/demo/templates/failing-hook-job.yaml new file mode 100644 index 0000000000..d3f2198459 --- /dev/null +++ b/examples/helm/components/helm/demo/templates/failing-hook-job.yaml @@ -0,0 +1,20 @@ +{{- if .Values.hooks.fail }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-failing-hook + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: fail + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: ["/bin/sh", "-c", "echo intentional lifecycle failure >&2; exit 1"] +{{- end }} diff --git a/examples/helm/components/helm/demo/templates/hook-order-configmap.yaml b/examples/helm/components/helm/demo/templates/hook-order-configmap.yaml new file mode 100644 index 0000000000..5403f9e8d0 --- /dev/null +++ b/examples/helm/components/helm/demo/templates/hook-order-configmap.yaml @@ -0,0 +1,14 @@ +{{- if .Values.hooks.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-hook-order + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-2" + "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/resource-policy": keep +data: + order: first +{{- end }} diff --git a/examples/helm/components/helm/demo/templates/hook-order-job.yaml b/examples/helm/components/helm/demo/templates/hook-order-job.yaml new file mode 100644 index 0000000000..d383fbbb5b --- /dev/null +++ b/examples/helm/components/helm/demo/templates/hook-order-job.yaml @@ -0,0 +1,27 @@ +{{- if .Values.hooks.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-hook-order + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-1" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: verify-weighted-configmap + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: ["/bin/sh", "-c", "test -f /hook-order/order"] + volumeMounts: + - name: hook-order + mountPath: /hook-order + volumes: + - name: hook-order + configMap: + name: {{ default (printf "%s-hook-order" .Release.Name) .Values.hooks.requiredConfigMap }} +{{- end }} diff --git a/examples/helm/components/helm/demo/templates/job.yaml b/examples/helm/components/helm/demo/templates/job.yaml new file mode 100644 index 0000000000..86edf72235 --- /dev/null +++ b/examples/helm/components/helm/demo/templates/job.yaml @@ -0,0 +1,16 @@ +{{- if .Values.job.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-job + namespace: {{ .Release.Namespace }} +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: ordinary-job + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: ["/bin/sh", "-c", "sleep {{ .Values.job.sleepSeconds }}"] +{{- end }} diff --git a/examples/helm/components/helm/demo/templates/ready-marker.yaml b/examples/helm/components/helm/demo/templates/ready-marker.yaml new file mode 100644 index 0000000000..2bb2ba86c6 --- /dev/null +++ b/examples/helm/components/helm/demo/templates/ready-marker.yaml @@ -0,0 +1,13 @@ +{{- if .Values.hooks.readyMarker }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-ready + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/resource-policy": keep +data: + ready: "true" +{{- end }} diff --git a/examples/helm/components/helm/demo/values.yaml b/examples/helm/components/helm/demo/values.yaml index f76f460afe..46f97bb5ec 100644 --- a/examples/helm/components/helm/demo/values.yaml +++ b/examples/helm/components/helm/demo/values.yaml @@ -6,3 +6,15 @@ image: service: type: ClusterIP port: 80 +deployment: + readinessDelaySeconds: 0 +job: + enabled: false + sleepSeconds: 0 +hooks: + enabled: true + fail: false + requiredConfigMap: "" + readyMarker: false +extraConfigMap: + enabled: false diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index e402aece42..65df1b4f23 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -6,6 +6,7 @@ helm: wait_strategy: watcher timeout: 2m max_history: 10 + skip_crds: true components: emulator: @@ -52,3 +53,101 @@ components: tag: "1.27" service: port: 8080 + + demo-jobs: + metadata: + component: demo + chart: "." + name: demo-jobs + namespace: demo-jobs + wait_for_jobs: true + values: + job: + enabled: true + sleepSeconds: 3 + + demo-no-hooks: + metadata: + component: demo + chart: "." + name: demo-no-hooks + namespace: demo-no-hooks + disable_chart_hooks: true + + demo-hook-only: + metadata: + component: demo + chart: "." + name: demo-hook-only + namespace: demo-hook-only + wait_strategy: hookOnly + values: + deployment: + readinessDelaySeconds: 8 + + demo-timeout: + metadata: + component: demo + chart: "." + name: demo-timeout + namespace: demo-timeout + wait_strategy: watcher + timeout: 2s + rollback_on_failure: true + values: + deployment: + readinessDelaySeconds: 20 + + demo-install-fail: + metadata: + component: demo + chart: "." + name: demo-install-fail + namespace: demo-install-fail + rollback_on_failure: true + cleanup_on_fail: true + values: + hooks: + fail: true + + demo-upgrade-fail: + metadata: + component: demo + chart: "." + # Intentionally targets the already-installed demo release. + name: demo + namespace: demo + rollback_on_failure: true + cleanup_on_fail: true + values: + hooks: + fail: true + extraConfigMap: + enabled: true + + dag-foundation: + metadata: + component: demo + tags: [lifecycle-dag] + chart: "." + name: dag-foundation + namespace: lifecycle-dag + values: + deployment: + readinessDelaySeconds: 3 + hooks: + readyMarker: true + + dag-dependent: + metadata: + component: demo + tags: [lifecycle-dag] + dependencies: + components: + - name: dag-foundation + chart: "." + name: dag-dependent + namespace: lifecycle-dag + values: + hooks: + requiredConfigMap: dag-foundation-ready From d51e32b53fc7f96eba1536d3a4ca2c85af3f5e87 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 04:06:16 +0400 Subject: [PATCH 03/62] fix: validate rendered Helm fixtures correctly --- .pre-commit-config.yaml | 2 +- ...ycle.md => 2026-07-31-native-helm-release-lifecycle.md} | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) rename docs/fixes/{2026-08-01-native-helm-release-lifecycle.md => 2026-07-31-native-helm-release-lifecycle.md} (81%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6bcc84f2f9..e9bc1f6ef2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -90,7 +90,7 @@ repos: stages: [pre-commit] exclude: ^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|.*\.svg|website/src/components/Screengrabs/) - id: check-yaml - exclude: ^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|tests/fixtures/) + exclude: ^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|tests/fixtures/|examples/helm/components/helm/demo/templates/) args: [--allow-multiple-documents, --unsafe] - id: check-added-large-files stages: [pre-commit] diff --git a/docs/fixes/2026-08-01-native-helm-release-lifecycle.md b/docs/fixes/2026-07-31-native-helm-release-lifecycle.md similarity index 81% rename from docs/fixes/2026-08-01-native-helm-release-lifecycle.md rename to docs/fixes/2026-07-31-native-helm-release-lifecycle.md index ffb96701a9..aa4c2ad04f 100644 --- a/docs/fixes/2026-08-01-native-helm-release-lifecycle.md +++ b/docs/fixes/2026-07-31-native-helm-release-lifecycle.md @@ -1,12 +1,13 @@ # Native Helm release lifecycle -**Date:** 2026-08-01 +**Date:** 2026-07-31 Native Helm cluster operations now expose Helm 4 wait, timeout, recovery, history, hook, and CRD controls through stack configuration and explicit command flags. Apply and delete dry runs now reach the Helm SDK without persisting release -state, and caller cancellation propagates through direct and dependency-ordered -execution. +state. Caller cancellation propagates through direct and dependency-ordered +execution into install and upgrade actions and into delete wait and hook phases; +Helm 4 does not expose a context-aware uninstall request. ## Migration notes From b961c246cc6bbd61385762573e757fcef3f9a86b Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 05:22:29 +0400 Subject: [PATCH 04/62] test: use shared Helm history default --- pkg/component/helm/client_lifecycle_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/component/helm/client_lifecycle_test.go b/pkg/component/helm/client_lifecycle_test.go index 826b9d819f..34701707b7 100644 --- a/pkg/component/helm/client_lifecycle_test.go +++ b/pkg/component/helm/client_lifecycle_test.go @@ -9,6 +9,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + cfg "github.com/cloudposse/atmos/pkg/config" "helm.sh/helm/v4/pkg/action" "helm.sh/helm/v4/pkg/chart/common" "helm.sh/helm/v4/pkg/cli" @@ -219,7 +221,7 @@ func TestUpgradeReleasePrunesDefaultHistory(t *testing.T) { stubActionContext(t, actx) spec := testdataChartSpec(t, "history") - for revision := 0; revision < defaultHelmMaxHistory+3; revision++ { + for revision := 0; revision < cfg.HelmDefaultMaxHistory+3; revision++ { spec.Values["replicaCount"] = revision + 1 _, err := applyRelease(context.Background(), spec, false) require.NoError(t, err) @@ -227,7 +229,7 @@ func TestUpgradeReleasePrunesDefaultHistory(t *testing.T) { history, err := actx.cfg.Releases.History(spec.ReleaseName) require.NoError(t, err) - assert.Len(t, history, defaultHelmMaxHistory) + assert.Len(t, history, cfg.HelmDefaultMaxHistory) } func TestUpgradeReleaseUnlimitedHistory(t *testing.T) { @@ -236,7 +238,7 @@ func TestUpgradeReleaseUnlimitedHistory(t *testing.T) { spec := testdataChartSpec(t, "unlimited-history") spec.Lifecycle.Policy.MaxHistory = 0 - const revisions = defaultHelmMaxHistory + 3 + const revisions = cfg.HelmDefaultMaxHistory + 3 for revision := 0; revision < revisions; revision++ { spec.Values["replicaCount"] = revision + 1 _, err := applyRelease(context.Background(), spec, false) From e0735f5b8bce200ece75261901214e76235b92a6 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 05:22:29 +0400 Subject: [PATCH 05/62] test: streamline Helm lifecycle k3s fixtures --- .../components/helm/demo/templates/failing-hook-job.yaml | 2 +- .../components/helm/demo/templates/hook-order-job.yaml | 2 +- examples/helm/components/helm/demo/templates/job.yaml | 2 +- .../components/helm/demo/templates/ready-marker.yaml | 1 + examples/helm/components/helm/demo/values.yaml | 3 +++ examples/helm/stacks/deploy/dev.yaml | 9 +++++++++ 6 files changed, 16 insertions(+), 3 deletions(-) diff --git a/examples/helm/components/helm/demo/templates/failing-hook-job.yaml b/examples/helm/components/helm/demo/templates/failing-hook-job.yaml index d3f2198459..d4f3c4b0f7 100644 --- a/examples/helm/components/helm/demo/templates/failing-hook-job.yaml +++ b/examples/helm/components/helm/demo/templates/failing-hook-job.yaml @@ -15,6 +15,6 @@ spec: restartPolicy: Never containers: - name: fail - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "{{ .Values.jobImage.repository }}:{{ .Values.jobImage.tag }}" command: ["/bin/sh", "-c", "echo intentional lifecycle failure >&2; exit 1"] {{- end }} diff --git a/examples/helm/components/helm/demo/templates/hook-order-job.yaml b/examples/helm/components/helm/demo/templates/hook-order-job.yaml index d383fbbb5b..c3a24fa154 100644 --- a/examples/helm/components/helm/demo/templates/hook-order-job.yaml +++ b/examples/helm/components/helm/demo/templates/hook-order-job.yaml @@ -15,7 +15,7 @@ spec: restartPolicy: Never containers: - name: verify-weighted-configmap - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "{{ .Values.jobImage.repository }}:{{ .Values.jobImage.tag }}" command: ["/bin/sh", "-c", "test -f /hook-order/order"] volumeMounts: - name: hook-order diff --git a/examples/helm/components/helm/demo/templates/job.yaml b/examples/helm/components/helm/demo/templates/job.yaml index 86edf72235..3f0df8e448 100644 --- a/examples/helm/components/helm/demo/templates/job.yaml +++ b/examples/helm/components/helm/demo/templates/job.yaml @@ -11,6 +11,6 @@ spec: restartPolicy: Never containers: - name: ordinary-job - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "{{ .Values.jobImage.repository }}:{{ .Values.jobImage.tag }}" command: ["/bin/sh", "-c", "sleep {{ .Values.job.sleepSeconds }}"] {{- end }} diff --git a/examples/helm/components/helm/demo/templates/ready-marker.yaml b/examples/helm/components/helm/demo/templates/ready-marker.yaml index 2bb2ba86c6..0c5a7bf4c9 100644 --- a/examples/helm/components/helm/demo/templates/ready-marker.yaml +++ b/examples/helm/components/helm/demo/templates/ready-marker.yaml @@ -10,4 +10,5 @@ metadata: "helm.sh/resource-policy": keep data: ready: "true" + order: "foundation-ready" {{- end }} diff --git a/examples/helm/components/helm/demo/values.yaml b/examples/helm/components/helm/demo/values.yaml index 46f97bb5ec..2a248a5e9f 100644 --- a/examples/helm/components/helm/demo/values.yaml +++ b/examples/helm/components/helm/demo/values.yaml @@ -3,6 +3,9 @@ replicaCount: 1 image: repository: nginx tag: "latest" +jobImage: + repository: busybox + tag: "1.36.1" service: type: ClusterIP port: 80 diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index 65df1b4f23..1c3d0aae67 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -25,6 +25,7 @@ components: # (components/helm/demo), which contains Chart.yaml. chart: "." namespace: demo + wait_strategy: legacy rollback_on_failure: true cleanup_on_fail: true # Atmos `values:` are the Helm chart values, merged via Atmos inheritance. @@ -62,6 +63,8 @@ components: namespace: demo-jobs wait_for_jobs: true values: + hooks: + enabled: false job: enabled: true sleepSeconds: 3 @@ -97,6 +100,8 @@ components: values: deployment: readinessDelaySeconds: 20 + hooks: + enabled: false demo-install-fail: metadata: @@ -104,6 +109,7 @@ components: chart: "." name: demo-install-fail namespace: demo-install-fail + wait_strategy: legacy rollback_on_failure: true cleanup_on_fail: true values: @@ -117,6 +123,7 @@ components: # Intentionally targets the already-installed demo release. name: demo namespace: demo + wait_strategy: legacy rollback_on_failure: true cleanup_on_fail: true values: @@ -132,6 +139,7 @@ components: chart: "." name: dag-foundation namespace: lifecycle-dag + wait_strategy: legacy values: deployment: readinessDelaySeconds: 3 @@ -148,6 +156,7 @@ components: chart: "." name: dag-dependent namespace: lifecycle-dag + wait_strategy: legacy values: hooks: requiredConfigMap: dag-foundation-ready From 490635bc318d156b40bf9000b48dc4707bb8f6d4 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 06:12:51 +0400 Subject: [PATCH 06/62] test: make Helm lifecycle assertions deterministic --- examples/helm/atmos.yaml | 18 +++++++++++++----- .../helm/demo/templates/deployment.yaml | 4 ++++ examples/helm/components/helm/demo/values.yaml | 1 + examples/helm/stacks/deploy/dev.yaml | 2 +- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 8ba65315b0..980651d76f 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -75,13 +75,18 @@ commands: backoff_strategy: constant # watcher + wait_for_jobs returns only after the ordinary Job completes. - atmos helm apply demo-jobs -s dev --identity local-k3s - - atmos emulator exec kubernetes -s dev -- kubectl -n demo-jobs wait --for=condition=complete job/demo-jobs-job --timeout=5s + - command: >- + completed=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-jobs get job demo-jobs-job -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}'); + if [ "$completed" != "True" ]; then echo "wait_for_jobs returned before demo-jobs-job completed: status=$completed"; exit 1; fi # Disabling chart hooks prevents both weighted hook resources. - atmos helm apply demo-no-hooks -s dev --identity local-k3s - command: if atmos emulator exec kubernetes -s dev -- kubectl -n demo-no-hooks get configmap demo-no-hooks-hook-order; then echo "disable_chart_hooks unexpectedly ran Helm hooks"; exit 1; fi - # hookOnly returns after hooks without waiting for ordinary Deployment readiness. + # hookOnly returns after hooks without waiting for the Deployment's explicit readiness gate. - atmos helm apply demo-hook-only -s dev --identity local-k3s - - command: if atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only rollout status deployment/demo-hook-only --timeout=1s; then echo "hookOnly unexpectedly waited for Deployment readiness"; exit 1; fi + - command: >- + available=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only get deployment demo-hook-only -o jsonpath='{.status.conditions[?(@.type=="Available")].status}'); + if [ "$available" = "True" ]; then echo "hookOnly unexpectedly satisfied the Deployment readiness gate"; exit 1; fi + - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only patch deployment demo-hook-only --type=json -p='[{"op":"remove","path":"/spec/template/spec/readinessGates"}]' - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only rollout status deployment/demo-hook-only --timeout=30s # watcher honors the release timeout and rollback removes the failed install. - command: if atmos helm apply demo-timeout -s dev --identity local-k3s; then echo "timed release unexpectedly succeeded"; exit 1; fi @@ -93,8 +98,11 @@ commands: - atmos emulator exec kubernetes -s dev -- kubectl -n demo-install-fail get configmap demo-install-fail-hook-order # A failed upgrade restores the successful demo release and cleanup removes # the resource introduced only by the failed revision. - - command: if atmos helm apply demo-upgrade-fail -s dev --identity local-k3s; then echo "failed upgrade unexpectedly succeeded"; exit 1; fi - - atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo + - command: >- + before=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}'); + if atmos helm apply demo-upgrade-fail -s dev --identity local-k3s; then echo "failed upgrade unexpectedly succeeded"; exit 1; fi; + after=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}'); + if [ "$before" != "$after" ]; then echo "rollback did not restore Deployment/demo: before=$before after=$after"; exit 1; fi - command: if atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-upgrade-only; then echo "cleanup_on_fail left the upgrade-only ConfigMap"; exit 1; fi # Dependency execution gates the dependent's pre-install hook on the # foundation's post-readiness marker. diff --git a/examples/helm/components/helm/demo/templates/deployment.yaml b/examples/helm/components/helm/demo/templates/deployment.yaml index d7028e4f3e..ac8067e5cb 100644 --- a/examples/helm/components/helm/demo/templates/deployment.yaml +++ b/examples/helm/components/helm/demo/templates/deployment.yaml @@ -15,6 +15,10 @@ spec: labels: app: {{ .Release.Name }} spec: + {{- if .Values.deployment.readinessGate }} + readinessGates: + - conditionType: "lifecycle.atmos.test/ready" + {{- end }} {{- if gt (int .Values.deployment.readinessDelaySeconds) 0 }} initContainers: - name: readiness-delay diff --git a/examples/helm/components/helm/demo/values.yaml b/examples/helm/components/helm/demo/values.yaml index 2a248a5e9f..9e85e35d49 100644 --- a/examples/helm/components/helm/demo/values.yaml +++ b/examples/helm/components/helm/demo/values.yaml @@ -11,6 +11,7 @@ service: port: 80 deployment: readinessDelaySeconds: 0 + readinessGate: false job: enabled: false sleepSeconds: 0 diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index 1c3d0aae67..bde3e19cbc 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -86,7 +86,7 @@ components: wait_strategy: hookOnly values: deployment: - readinessDelaySeconds: 8 + readinessGate: true demo-timeout: metadata: From edd277a46fd71381f37b948cfdac25f0da3b8508 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 06:39:07 +0400 Subject: [PATCH 07/62] fix: address Helm lifecycle integration review --- examples/helm/stacks/deploy/dev.yaml | 3 +++ pkg/ci/plugins/helm/templates/delete.md | 10 ++++++++++ pkg/component/helm/client.go | 20 +++++++++----------- pkg/component/helm/client_lifecycle_test.go | 14 ++++++++++++++ 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index bde3e19cbc..5032b1cbc2 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -119,6 +119,9 @@ components: demo-upgrade-fail: metadata: component: demo + dependencies: + components: + - name: demo chart: "." # Intentionally targets the already-installed demo release. name: demo diff --git a/pkg/ci/plugins/helm/templates/delete.md b/pkg/ci/plugins/helm/templates/delete.md index 8ae1020b51..1923bfd730 100644 --- a/pkg/ci/plugins/helm/templates/delete.md +++ b/pkg/ci/plugins/helm/templates/delete.md @@ -14,6 +14,15 @@ ### Release lifecycle +{{- if eq (index . "reason") "external_target" }} + +| Field | Value | +| --- | --- | +| Deleted | `false` | +| Target kind | `{{ index . "target_kind" }}` | +| Reason | `external_target` | +{{- else }} + | Field | Value | | --- | --- | | Operation | `{{ index . "operation" }}` | @@ -21,6 +30,7 @@ | Timeout | `{{ index . "timeout" }}` | | Chart hooks enabled | `{{ index . "chart_hooks_enabled" }}` | {{- end }} +{{- end }} To reproduce locally: diff --git a/pkg/component/helm/client.go b/pkg/component/helm/client.go index cab7ad88f0..16742c52d2 100644 --- a/pkg/component/helm/client.go +++ b/pkg/component/helm/client.go @@ -258,17 +258,15 @@ func deleteRelease(ctx context.Context, spec *chartSpec, dryRun bool) error { func releaseOperationError(operation string, spec *chartSpec, cause error) error { policy := spec.Lifecycle.Policy - return fmt.Errorf( - "%w: operation=%s release=%q namespace=%q wait_strategy=%s timeout=%s (component field %q): %w", - errUtils.ErrHelmReleaseOperation, - operation, - spec.ReleaseName, - spec.Namespace, - policy.WaitStrategy, - policy.Timeout, - cfg.HelmTimeoutSectionName, - cause, - ) + return errUtils.Build(errUtils.ErrHelmReleaseOperation). + WithCause(cause). + WithContext("operation", operation). + WithContext("release", spec.ReleaseName). + WithContext("namespace", spec.Namespace). + WithContext("wait_strategy", policy.WaitStrategy). + WithContext("timeout", policy.Timeout). + WithContext("timeout_field", cfg.HelmTimeoutSectionName). + Err() } func configureInstallLifecycle(client *action.Install, policy effectiveReleasePolicy) { diff --git a/pkg/component/helm/client_lifecycle_test.go b/pkg/component/helm/client_lifecycle_test.go index 34701707b7..3182bd117b 100644 --- a/pkg/component/helm/client_lifecycle_test.go +++ b/pkg/component/helm/client_lifecycle_test.go @@ -16,6 +16,7 @@ import ( "helm.sh/helm/v4/pkg/cli" kubefake "helm.sh/helm/v4/pkg/kube/fake" "helm.sh/helm/v4/pkg/registry" + helmrelease "helm.sh/helm/v4/pkg/release" release "helm.sh/helm/v4/pkg/release/v1" "helm.sh/helm/v4/pkg/storage" "helm.sh/helm/v4/pkg/storage/driver" @@ -230,6 +231,7 @@ func TestUpgradeReleasePrunesDefaultHistory(t *testing.T) { history, err := actx.cfg.Releases.History(spec.ReleaseName) require.NoError(t, err) assert.Len(t, history, cfg.HelmDefaultMaxHistory) + assert.Equal(t, []int{4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, releaseVersions(t, history)) } func TestUpgradeReleaseUnlimitedHistory(t *testing.T) { @@ -248,4 +250,16 @@ func TestUpgradeReleaseUnlimitedHistory(t *testing.T) { history, err := actx.cfg.Releases.History(spec.ReleaseName) require.NoError(t, err) assert.Len(t, history, revisions) + assert.Equal(t, []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, releaseVersions(t, history)) +} + +func releaseVersions(t *testing.T, history []helmrelease.Releaser) []int { + t.Helper() + versions := make([]int, len(history)) + for i, item := range history { + typed, ok := item.(*release.Release) + require.True(t, ok) + versions[i] = typed.Version + } + return versions } From 94a7a9dc4260856f467067ad3d9e46a7ccc09aee Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 08:38:06 +0400 Subject: [PATCH 08/62] test: assert Helm error context metadata --- pkg/component/helm/client_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/component/helm/client_test.go b/pkg/component/helm/client_test.go index 33fbbe291e..32653865e4 100644 --- a/pkg/component/helm/client_test.go +++ b/pkg/component/helm/client_test.go @@ -13,6 +13,7 @@ import ( "helm.sh/helm/v4/pkg/kube" errUtils "github.com/cloudposse/atmos/errors" + cfg "github.com/cloudposse/atmos/pkg/config" ) func TestResolveUpgradeChartRef(t *testing.T) { @@ -112,12 +113,12 @@ func TestReleaseOperationErrorIncludesEffectivePolicy(t *testing.T) { require.ErrorIs(t, err, errUtils.ErrHelmReleaseOperation) require.ErrorIs(t, err, context.DeadlineExceeded) - assert.Contains(t, err.Error(), "operation=upgrade") - assert.Contains(t, err.Error(), `release="demo"`) - assert.Contains(t, err.Error(), `namespace="apps"`) - assert.Contains(t, err.Error(), "wait_strategy=watcher") - assert.Contains(t, err.Error(), "timeout=7m0s") - assert.Contains(t, err.Error(), `component field "timeout"`) + assert.True(t, errUtils.HasContext(err, "operation", "upgrade")) + assert.True(t, errUtils.HasContext(err, "release", "demo")) + assert.True(t, errUtils.HasContext(err, "namespace", "apps")) + assert.True(t, errUtils.HasContext(err, "wait_strategy", "watcher")) + assert.True(t, errUtils.HasContext(err, "timeout", "7m0s")) + assert.True(t, errUtils.HasContext(err, "timeout_field", cfg.HelmTimeoutSectionName)) } func TestClusterOperationsReturnActionContextErrors(t *testing.T) { From bf895b849fb53afac6460d8905d2616bd5a75ff6 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 08:59:49 +0400 Subject: [PATCH 09/62] test: distinguish Helm resource absence errors --- examples/helm/atmos.yaml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 980651d76f..28b5121cb1 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -53,7 +53,9 @@ commands: backoff_strategy: constant # Apply dry-run must not persist a release or create Kubernetes objects. - atmos helm apply demo -s dev --identity local-k3s --dry-run - - command: if atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo; then echo "dry-run unexpectedly created deployment/demo"; exit 1; fi + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo 2>&1); then echo "dry-run unexpectedly created deployment/demo"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify deployment/demo absence: $output"; exit 1;; esac; fi - command: atmos helm apply demo -s dev --identity local-k3s retry: max_attempts: 2 @@ -62,7 +64,9 @@ commands: # The -2 ConfigMap must exist before the -1 hook Job can mount it. - atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-hook-order # skip_crds is a stack-level lifecycle default. - - command: if atmos emulator exec kubernetes -s dev -- kubectl get crd widgets.lifecycle.atmos.test; then echo "skip_crds unexpectedly installed widgets.lifecycle.atmos.test"; exit 1; fi + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl get crd widgets.lifecycle.atmos.test 2>&1); then echo "skip_crds unexpectedly installed widgets.lifecycle.atmos.test"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify widgets.lifecycle.atmos.test absence: $output"; exit 1;; esac; fi - command: atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo retry: max_attempts: 3 @@ -80,7 +84,9 @@ commands: if [ "$completed" != "True" ]; then echo "wait_for_jobs returned before demo-jobs-job completed: status=$completed"; exit 1; fi # Disabling chart hooks prevents both weighted hook resources. - atmos helm apply demo-no-hooks -s dev --identity local-k3s - - command: if atmos emulator exec kubernetes -s dev -- kubectl -n demo-no-hooks get configmap demo-no-hooks-hook-order; then echo "disable_chart_hooks unexpectedly ran Helm hooks"; exit 1; fi + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-no-hooks get configmap demo-no-hooks-hook-order 2>&1); then echo "disable_chart_hooks unexpectedly ran Helm hooks"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify disabled Helm hooks: $output"; exit 1;; esac; fi # hookOnly returns after hooks without waiting for the Deployment's explicit readiness gate. - atmos helm apply demo-hook-only -s dev --identity local-k3s - command: >- @@ -90,11 +96,15 @@ commands: - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only rollout status deployment/demo-hook-only --timeout=30s # watcher honors the release timeout and rollback removes the failed install. - command: if atmos helm apply demo-timeout -s dev --identity local-k3s; then echo "timed release unexpectedly succeeded"; exit 1; fi - - command: if atmos emulator exec kubernetes -s dev -- kubectl -n demo-timeout get deployment demo-timeout; then echo "timeout rollback left release resources"; exit 1; fi + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-timeout get deployment demo-timeout 2>&1); then echo "timeout rollback left release resources"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify timeout rollback cleanup: $output"; exit 1;; esac; fi # A failed first install is reported as failure and rolled back. The kept # hook ConfigMap survives recovery while ordinary release resources do not. - command: if atmos helm apply demo-install-fail -s dev --identity local-k3s; then echo "failed install unexpectedly succeeded"; exit 1; fi - - command: if atmos emulator exec kubernetes -s dev -- kubectl -n demo-install-fail get deployment demo-install-fail; then echo "rollback left failed install resources"; exit 1; fi + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-install-fail get deployment demo-install-fail 2>&1); then echo "rollback left failed install resources"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify failed-install rollback cleanup: $output"; exit 1;; esac; fi - atmos emulator exec kubernetes -s dev -- kubectl -n demo-install-fail get configmap demo-install-fail-hook-order # A failed upgrade restores the successful demo release and cleanup removes # the resource introduced only by the failed revision. @@ -103,7 +113,9 @@ commands: if atmos helm apply demo-upgrade-fail -s dev --identity local-k3s; then echo "failed upgrade unexpectedly succeeded"; exit 1; fi; after=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}'); if [ "$before" != "$after" ]; then echo "rollback did not restore Deployment/demo: before=$before after=$after"; exit 1; fi - - command: if atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-upgrade-only; then echo "cleanup_on_fail left the upgrade-only ConfigMap"; exit 1; fi + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-upgrade-only 2>&1); then echo "cleanup_on_fail left the upgrade-only ConfigMap"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify cleanup_on_fail removal: $output"; exit 1;; esac; fi # Dependency execution gates the dependent's pre-install hook on the # foundation's post-readiness marker. - atmos helm apply --all -s dev --identity local-k3s --tags lifecycle-dag From 879d38e4a02467c16939a5224c8e015370e7c39b Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 10:51:45 +0400 Subject: [PATCH 10/62] docs: explain Helm defaults and overrides --- website/docs/stacks/components/helm.mdx | 39 +++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/website/docs/stacks/components/helm.mdx b/website/docs/stacks/components/helm.mdx index fe4d12ca74..08e35f6c63 100644 --- a/website/docs/stacks/components/helm.mdx +++ b/website/docs/stacks/components/helm.mdx @@ -73,8 +73,47 @@ included in affected runs.
`provision`
Delivery targets for apply/deploy — the cluster (default) or an external target such as a Git deployment repository.
+ +
`secrets`
+
Component-scoped secret declarations and providers. Helm components support the same secret processing as other component types.
+## Type-Level Defaults and Overrides + +The top-level `helm` section in a stack manifest can provide defaults for every +native Helm component in that stack. Use `helm.values` for shared chart values; +component and inherited `values` are merged over those defaults. + +Use `helm.overrides` when a stack must enforce values after component-level +configuration is resolved. Type-level overrides are deep-merged over each +component's `overrides`, including components supplied by imported manifests. +The block accepts `values` plus the common component override sections such as +`vars`, `env`, `settings`, `auth`, `secrets`, and `retry`. + +```yaml +helm: + values: + cluster: shared + overrides: + values: + environment: production + +components: + helm: + monitoring: + chart: ./charts/monitoring + values: + replicaCount: 2 + overrides: + values: + image: + tag: stable +``` + +In this example, every Helm component receives `cluster: shared`, while the +stack-level override enforces `environment: production` after component +inheritance and component overrides are merged. + ## Release Lifecycle Cluster-backed `apply`, `deploy`, and `delete` operations can use Helm 4 release From 1db8f07285067fda30c106f2c9152c5ebdad17e2 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 13:09:10 +0400 Subject: [PATCH 11/62] test: preload Helm k3s workload images --- examples/helm/atmos.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 28b5121cb1..4b3db34b56 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -46,6 +46,18 @@ commands: max_attempts: 2 initial_delay: 15s backoff_strategy: constant + # Pull workload images through the host runtime and import them into the + # nested k3s containerd. This avoids registry pulls through Colima's + # nested network while Helm is waiting on Jobs and Deployments. + - command: >- + docker pull busybox:1.36.1 && + docker pull nginx:1.27 && + docker save busybox:1.36.1 nginx:1.27 | + docker exec -i atmos-dev-emulator-kubernetes ctr --namespace k8s.io images import - + retry: + max_attempts: 2 + initial_delay: 15s + backoff_strategy: constant - command: atmos helm diff demo -s dev --identity local-k3s retry: max_attempts: 2 From 8ef57a1799df8d39e1fee41355741e8f01ae51dc Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 13:33:57 +0400 Subject: [PATCH 12/62] test: verify Helm dry-run release state --- examples/helm/atmos.yaml | 10 ++++++++++ website/docs/cli/commands/helm/helm-delete.mdx | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 4b3db34b56..c6a2601fa8 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -68,6 +68,12 @@ commands: - command: >- if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo 2>&1); then echo "dry-run unexpectedly created deployment/demo"; exit 1; else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify deployment/demo absence: $output"; exit 1;; esac; fi + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo 2>&1); then echo "dry-run unexpectedly created service/demo"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify service/demo absence: $output"; exit 1;; esac; fi + - command: >- + releases=$(atmos emulator exec kubernetes -s dev -- kubectl get secrets --all-namespaces -l owner=helm,name=demo -o name); + if [ -n "$releases" ]; then echo "dry-run unexpectedly persisted Helm release records: $releases"; exit 1; fi - command: atmos helm apply demo -s dev --identity local-k3s retry: max_attempts: 2 @@ -135,6 +141,10 @@ commands: # Delete dry-run must leave the deployed release and resources intact. - atmos helm delete demo -s dev --identity local-k3s --dry-run --wait=watcher --timeout=2m - atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo + - atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo + - command: >- + releases=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get secrets -l owner=helm,name=demo -o name); + if [ -z "$releases" ]; then echo "delete dry-run removed the Helm release record"; exit 1; fi - atmos helm delete demo -s dev --identity local-k3s - atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-hook-order - atmos helm delete demo-jobs -s dev --identity local-k3s diff --git a/website/docs/cli/commands/helm/helm-delete.mdx b/website/docs/cli/commands/helm/helm-delete.mdx index f63db2d447..0e1e4c8766 100644 --- a/website/docs/cli/commands/helm/helm-delete.mdx +++ b/website/docs/cli/commands/helm/helm-delete.mdx @@ -38,7 +38,7 @@ atmos helm delete --all --tags production,tier-1
Filter by tags (comma-separated, matches any) or labels (comma-separated `key=value` or `key:value` pairs, matches all): `--tags=production,tier-1`, `--labels=cost-center=platform`. Compose with `--all`/`--affected` to narrow the selected set further; cannot be combined with a single component argument.
`--wait[=strategy]` (optional)
-
Use `watcher`, `hookOnly`, or `legacy` while deleting. Passing `--wait` without a value selects `watcher`.
+
Use `watcher`, `hookOnly`, or `legacy` while deleting. Passing `--wait` without a value selects `watcher`. Boolean values remain accepted temporarily but are deprecated.
`--timeout` (optional)
Helm uninstall timeout, such as `10m`. `0s` is explicitly unbounded.
From 9c646900ad2df8f9fbc9a511c0e01355c032cd33 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 15:27:53 +0400 Subject: [PATCH 13/62] test: cover Helm hooks dependencies and tpl values --- examples/helm/README.md | 5 ++++- examples/helm/atmos.yaml | 12 +++++++++++- examples/helm/components/helm/demo/.gitignore | 1 + examples/helm/components/helm/demo/Chart.lock | 6 ++++++ examples/helm/components/helm/demo/Chart.yaml | 4 ++++ .../helm/demo/templates/tpl-configmap.yaml | 7 +++++++ examples/helm/components/helm/demo/values.yaml | 3 +++ .../components/helm/helm-test-library/Chart.yaml | 5 +++++ .../helm/helm-test-library/templates/_render.tpl | 3 +++ examples/helm/stacks/deploy/dev.yaml | 3 +++ 10 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 examples/helm/components/helm/demo/.gitignore create mode 100644 examples/helm/components/helm/demo/Chart.lock create mode 100644 examples/helm/components/helm/demo/templates/tpl-configmap.yaml create mode 100644 examples/helm/components/helm/helm-test-library/Chart.yaml create mode 100644 examples/helm/components/helm/helm-test-library/templates/_render.tpl diff --git a/examples/helm/README.md b/examples/helm/README.md index 24daa893bc..06fbb2dd49 100644 --- a/examples/helm/README.md +++ b/examples/helm/README.md @@ -17,6 +17,7 @@ Run the local chart workflow end to end: ```shell atmos validate stacks +atmos toolchain exec helm -- dependency build --skip-refresh components/helm/demo atmos helm template demo -s dev atmos emulator up kubernetes -s dev atmos helm diff demo -s dev --identity local-k3s @@ -77,7 +78,9 @@ The stack sets `wait_strategy: watcher`, `timeout: 2m`, and covers non-mutating apply and delete dry runs, ordinary Job waiting, chart-hook suppression, CRD skipping, weighted hook ordering, retained hook resources, install and upgrade rollback, failed-upgrade cleanup, timeout handling, and -dependency-gated Helm releases. The intentionally slow resources are observed +dependency-gated Helm releases. Rendering also verifies a built `file://` library +dependency, hook manifests, and a Helm `tpl` expression preserved in stack values. +The intentionally slow resources are observed with bounded Kubernetes readiness checks rather than fixed-delay assertions. ## Helm Repositories diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index c6a2601fa8..6b4d3a55db 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -40,7 +40,17 @@ commands: description: "Render the local Helm chart, deploy it to the k3s emulator, verify it, then tear it down" steps: - atmos validate stacks - - atmos helm template demo -s dev + # Native Helm validates declared chart dependencies before rendering. Build + # the local library dependency exactly as a chart consumer would. + - atmos toolchain exec helm -- dependency build --skip-refresh components/helm/demo + - atmos helm template demo -s dev --output=/tmp/atmos-helm-template.yaml + # Template output includes weighted hooks and leaves Helm `tpl` expressions + # in values for the chart to evaluate with its native .Values context. + - command: >- + test "$(grep -c '^ name: demo-hook-order$' /tmp/atmos-helm-template.yaml)" -eq 2 && + grep -q 'helm.sh/hook-weight: "-2"' /tmp/atmos-helm-template.yaml && + grep -q 'helm.sh/hook-weight: "-1"' /tmp/atmos-helm-template.yaml && + grep -q 'rendered: from-stack' /tmp/atmos-helm-template.yaml - command: atmos emulator up kubernetes -s dev retry: max_attempts: 2 diff --git a/examples/helm/components/helm/demo/.gitignore b/examples/helm/components/helm/demo/.gitignore new file mode 100644 index 0000000000..ee3892e879 --- /dev/null +++ b/examples/helm/components/helm/demo/.gitignore @@ -0,0 +1 @@ +charts/ diff --git a/examples/helm/components/helm/demo/Chart.lock b/examples/helm/components/helm/demo/Chart.lock new file mode 100644 index 0000000000..3ae6829b29 --- /dev/null +++ b/examples/helm/components/helm/demo/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: helm-test-library + repository: file://../helm-test-library + version: 0.1.0 +digest: sha256:cf07c19b9e23d03dec7c67e6160c8d59a941059e48f9c47e86878cd29ce16098 +generated: "2026-08-01T15:03:42.45764+04:00" diff --git a/examples/helm/components/helm/demo/Chart.yaml b/examples/helm/components/helm/demo/Chart.yaml index fc807f2414..d17a37337f 100644 --- a/examples/helm/components/helm/demo/Chart.yaml +++ b/examples/helm/components/helm/demo/Chart.yaml @@ -4,3 +4,7 @@ description: A minimal local Helm chart used by the Atmos native Helm component type: application version: 0.1.0 appVersion: "1.0.0" +dependencies: + - name: helm-test-library + version: 0.1.0 + repository: file://../helm-test-library diff --git a/examples/helm/components/helm/demo/templates/tpl-configmap.yaml b/examples/helm/components/helm/demo/templates/tpl-configmap.yaml new file mode 100644 index 0000000000..165b7f6adb --- /dev/null +++ b/examples/helm/components/helm/demo/templates/tpl-configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-tpl + namespace: {{ .Release.Namespace }} +data: + rendered: {{ include "helmTestLibrary.renderValue" (dict "value" .Values.tpl.expression "context" .) | quote }} diff --git a/examples/helm/components/helm/demo/values.yaml b/examples/helm/components/helm/demo/values.yaml index 9e85e35d49..2cb84753a6 100644 --- a/examples/helm/components/helm/demo/values.yaml +++ b/examples/helm/components/helm/demo/values.yaml @@ -22,3 +22,6 @@ hooks: readyMarker: false extraConfigMap: enabled: false +tpl: + source: default + expression: "{{ .Values.tpl.source }}" diff --git a/examples/helm/components/helm/helm-test-library/Chart.yaml b/examples/helm/components/helm/helm-test-library/Chart.yaml new file mode 100644 index 0000000000..04577030d2 --- /dev/null +++ b/examples/helm/components/helm/helm-test-library/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: helm-test-library +description: Minimal library chart used to verify local Helm dependencies. +type: library +version: 0.1.0 diff --git a/examples/helm/components/helm/helm-test-library/templates/_render.tpl b/examples/helm/components/helm/helm-test-library/templates/_render.tpl new file mode 100644 index 0000000000..fc0dee51e0 --- /dev/null +++ b/examples/helm/components/helm/helm-test-library/templates/_render.tpl @@ -0,0 +1,3 @@ +{{- define "helmTestLibrary.renderValue" -}} +{{- tpl .value .context -}} +{{- end -}} diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index 5032b1cbc2..d2f6269a59 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -35,6 +35,9 @@ components: tag: "1.27" service: port: 8080 + tpl: + source: from-stack + expression: !literal "{{ .Values.tpl.source }}" demo-repo: metadata: From db83db5de0f65e4982a83fa4803d3679b6f52c07 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 15:37:50 +0400 Subject: [PATCH 14/62] test: group Helm lifecycle imports --- pkg/component/helm/client_lifecycle_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/component/helm/client_lifecycle_test.go b/pkg/component/helm/client_lifecycle_test.go index 3182bd117b..a1f02f2f68 100644 --- a/pkg/component/helm/client_lifecycle_test.go +++ b/pkg/component/helm/client_lifecycle_test.go @@ -7,10 +7,10 @@ import ( "testing" "time" + errUtils "github.com/cloudposse/atmos/errors" + cfg "github.com/cloudposse/atmos/pkg/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - cfg "github.com/cloudposse/atmos/pkg/config" "helm.sh/helm/v4/pkg/action" "helm.sh/helm/v4/pkg/chart/common" "helm.sh/helm/v4/pkg/cli" @@ -20,8 +20,6 @@ import ( release "helm.sh/helm/v4/pkg/release/v1" "helm.sh/helm/v4/pkg/storage" "helm.sh/helm/v4/pkg/storage/driver" - - errUtils "github.com/cloudposse/atmos/errors" ) // memoryActionContext builds an actionContext backed by Helm's in-memory storage From 20407235c58722df5ed4776e65033d567b722d96 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 16:53:15 +0400 Subject: [PATCH 15/62] test: assert Helm lifecycle timeout failure --- examples/helm/atmos.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 6b4d3a55db..8b2b0ee650 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -123,7 +123,10 @@ commands: - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only patch deployment demo-hook-only --type=json -p='[{"op":"remove","path":"/spec/template/spec/readinessGates"}]' - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only rollout status deployment/demo-hook-only --timeout=30s # watcher honors the release timeout and rollback removes the failed install. - - command: if atmos helm apply demo-timeout -s dev --identity local-k3s; then echo "timed release unexpectedly succeeded"; exit 1; fi + - command: >- + if output=$(atmos helm apply demo-timeout -s dev --identity local-k3s 2>&1); then echo "timed release unexpectedly succeeded"; exit 1; fi; + case "$output" in *"helm release operation failed"*) ;; *) echo "timed release failed outside the Helm lifecycle operation: $output"; exit 1;; esac; + case "$output" in *"context deadline exceeded"*|*"timed out waiting for condition"*) ;; *) echo "timed release did not report a lifecycle timeout: $output"; exit 1;; esac - command: >- if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-timeout get deployment demo-timeout 2>&1); then echo "timeout rollback left release resources"; exit 1; else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify timeout rollback cleanup: $output"; exit 1;; esac; fi From aa75a3e9ede26daade056aa16bc688bc2136dc53 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 18:01:27 +0400 Subject: [PATCH 16/62] test: pin Helm toolchain dependency build --- examples/helm/atmos.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 8b2b0ee650..4089aee760 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -40,9 +40,10 @@ commands: description: "Render the local Helm chart, deploy it to the k3s emulator, verify it, then tear it down" steps: - atmos validate stacks - # Native Helm validates declared chart dependencies before rendering. Build - # the local library dependency exactly as a chart consumer would. - - atmos toolchain exec helm -- dependency build --skip-refresh components/helm/demo + # Native Helm validates declared chart dependencies before rendering. Use + # the explicit tool spec so a clean checkout does not depend on a local + # .tool-versions default or a previously populated toolchain cache. + - atmos toolchain exec helm@v3.21.2 -- dependency build --skip-refresh components/helm/demo - atmos helm template demo -s dev --output=/tmp/atmos-helm-template.yaml # Template output includes weighted hooks and leaves Helm `tpl` expressions # in values for the chart to evaluate with its native .Values context. From 5f0fd28b1d40717e7db3fe8fe56d6ee81c15b70a Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 18:46:05 +0400 Subject: [PATCH 17/62] test: use provisioned Helm for dependency build --- examples/helm/atmos.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 4089aee760..b0f3767880 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -40,10 +40,9 @@ commands: description: "Render the local Helm chart, deploy it to the k3s emulator, verify it, then tear it down" steps: - atmos validate stacks - # Native Helm validates declared chart dependencies before rendering. Use - # the explicit tool spec so a clean checkout does not depend on a local - # .tool-versions default or a previously populated toolchain cache. - - atmos toolchain exec helm@v3.21.2 -- dependency build --skip-refresh components/helm/demo + # Native Helm validates declared chart dependencies before rendering. CI + # and local contributors provision Helm on PATH before running the demo. + - helm dependency build --skip-refresh components/helm/demo - atmos helm template demo -s dev --output=/tmp/atmos-helm-template.yaml # Template output includes weighted hooks and leaves Helm `tpl` expressions # in values for the chart to evaluate with its native .Values context. From da352786ab75f29b98577136b878d0c2ae3c8dab Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 18:49:04 +0400 Subject: [PATCH 18/62] test: remove redundant Helm tool dependencies --- examples/helm/stacks/deploy/dev.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index d2f6269a59..357f02ee95 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -18,9 +18,6 @@ components: demo: metadata: component: demo - dependencies: - tools: - helm: "v3.21.2" # The chart reference. "." is the component directory itself # (components/helm/demo), which contains Chart.yaml. chart: "." @@ -42,9 +39,6 @@ components: demo-repo: metadata: component: demo - dependencies: - tools: - helm: "v3.21.2" repositories: - name: local url: !env HELM_DEMO_REPO_URL From 405c91387c7f8fc192db6c3cc9d51246be60ee34 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 19:14:43 +0400 Subject: [PATCH 19/62] test: disable color in lifecycle error assertion --- examples/helm/atmos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index b0f3767880..66d65a64bb 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -124,7 +124,7 @@ commands: - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only rollout status deployment/demo-hook-only --timeout=30s # watcher honors the release timeout and rollback removes the failed install. - command: >- - if output=$(atmos helm apply demo-timeout -s dev --identity local-k3s 2>&1); then echo "timed release unexpectedly succeeded"; exit 1; fi; + if output=$(NO_COLOR=1 atmos helm apply demo-timeout -s dev --identity local-k3s 2>&1); then echo "timed release unexpectedly succeeded"; exit 1; fi; case "$output" in *"helm release operation failed"*) ;; *) echo "timed release failed outside the Helm lifecycle operation: $output"; exit 1;; esac; case "$output" in *"context deadline exceeded"*|*"timed out waiting for condition"*) ;; *) echo "timed release did not report a lifecycle timeout: $output"; exit 1;; esac - command: >- From 3efa0a0b8bab54d0334fb7dbff9996d0a594086b Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 19:25:09 +0400 Subject: [PATCH 20/62] docs: clarify Helm cleanup-on-fail behavior --- website/docs/cli/commands/helm/helm-apply.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/cli/commands/helm/helm-apply.mdx b/website/docs/cli/commands/helm/helm-apply.mdx index d331f37e06..6f4793b75c 100644 --- a/website/docs/cli/commands/helm/helm-apply.mdx +++ b/website/docs/cli/commands/helm/helm-apply.mdx @@ -83,7 +83,7 @@ atmos helm apply --affected --labels cost-center=platform
Helm release-operation timeout, such as `10m` or `1h`. `0s` is explicitly unbounded.
`--cleanup-on-fail` (optional)
-
Delete resources newly created by a failed upgrade before rollback.
+
Delete resources newly created by a failed upgrade.
`--history-max` (optional)
Maximum retained release revisions. Defaults to `10`; `0` means unlimited.
From e635b57d3e6fd49dca5edfd57365c649ca4f39fc Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 19:58:58 +0400 Subject: [PATCH 21/62] test: tighten Helm lifecycle failure coverage --- examples/helm/atmos.yaml | 5 ++++- examples/helm/stacks/deploy/dev.yaml | 1 - website/docs/cli/commands/helm/helm-apply.mdx | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 66d65a64bb..3663091a4c 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -132,7 +132,10 @@ commands: else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify timeout rollback cleanup: $output"; exit 1;; esac; fi # A failed first install is reported as failure and rolled back. The kept # hook ConfigMap survives recovery while ordinary release resources do not. - - command: if atmos helm apply demo-install-fail -s dev --identity local-k3s; then echo "failed install unexpectedly succeeded"; exit 1; fi + - command: >- + if output=$(NO_COLOR=1 atmos helm apply demo-install-fail -s dev --identity local-k3s 2>&1); then echo "failed install unexpectedly succeeded"; exit 1; fi; + case "$output" in *"helm release operation failed"*) ;; *) echo "failed install stopped outside the Helm lifecycle operation: $output"; exit 1;; esac; + case "$output" in *"job demo-install-fail-failing-hook failed: BackoffLimitExceeded"*) ;; *) echo "failed install did not report the expected hook failure: $output"; exit 1;; esac - command: >- if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-install-fail get deployment demo-install-fail 2>&1); then echo "rollback left failed install resources"; exit 1; else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify failed-install rollback cleanup: $output"; exit 1;; esac; fi diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index 357f02ee95..d79d03b00c 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -108,7 +108,6 @@ components: namespace: demo-install-fail wait_strategy: legacy rollback_on_failure: true - cleanup_on_fail: true values: hooks: fail: true diff --git a/website/docs/cli/commands/helm/helm-apply.mdx b/website/docs/cli/commands/helm/helm-apply.mdx index 6f4793b75c..0178e3be9a 100644 --- a/website/docs/cli/commands/helm/helm-apply.mdx +++ b/website/docs/cli/commands/helm/helm-apply.mdx @@ -71,7 +71,7 @@ atmos helm apply --affected --labels cost-center=platform
Fetch declared chart dependencies when they are missing. This may access dependency repositories and update the chart's charts/ directory and lock file.
`--rollback-on-failure` (optional)
-
Uninstall a failed first install or roll a failed upgrade back. The deprecated alias is `--atomic`.
+
Uninstall a failed first install or roll a failed upgrade back. Enabling it promotes the default wait strategy to `watcher`. The deprecated alias is `--atomic`.
`--wait[=strategy]` (optional)
Use `watcher`, `hookOnly`, or `legacy`. Passing `--wait` without a value selects `watcher`. Boolean values remain accepted temporarily but are deprecated.
From d465197caf41aa8312334fed85d2911e23a10abe Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 20:27:35 +0400 Subject: [PATCH 22/62] docs: exercise opt-in Helm dependency updates --- docs/fixes/2026-07-31-native-helm-release-lifecycle.md | 4 ++++ examples/helm/README.md | 8 ++++---- examples/helm/atmos.yaml | 7 +++---- website/docs/cli/commands/helm/helm-deploy.mdx | 3 +++ website/docs/cli/commands/helm/helm-plan.mdx | 3 +++ 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/fixes/2026-07-31-native-helm-release-lifecycle.md b/docs/fixes/2026-07-31-native-helm-release-lifecycle.md index aa4c2ad04f..7beaec6a9d 100644 --- a/docs/fixes/2026-07-31-native-helm-release-lifecycle.md +++ b/docs/fixes/2026-07-31-native-helm-release-lifecycle.md @@ -22,3 +22,7 @@ Helm 4 does not expose a context-aware uninstall request. - Explicit lifecycle flags cannot be combined with a non-Kubernetes provision target. Stored lifecycle configuration is intentionally bypassed for external delivery and is identified as such in the execution summary. +- Chart-loading commands do not fetch missing dependencies unless + `--dependency-update` is explicitly supplied. The opt-in follows Helm's + dependency-update behavior and may access repositories and mutate the chart's + `charts/` directory and lock file. diff --git a/examples/helm/README.md b/examples/helm/README.md index 06fbb2dd49..2405e64d5b 100644 --- a/examples/helm/README.md +++ b/examples/helm/README.md @@ -17,8 +17,7 @@ Run the local chart workflow end to end: ```shell atmos validate stacks -atmos toolchain exec helm -- dependency build --skip-refresh components/helm/demo -atmos helm template demo -s dev +atmos helm template demo -s dev --dependency-update atmos emulator up kubernetes -s dev atmos helm diff demo -s dev --identity local-k3s atmos helm apply demo -s dev --identity local-k3s --dry-run @@ -78,8 +77,9 @@ The stack sets `wait_strategy: watcher`, `timeout: 2m`, and covers non-mutating apply and delete dry runs, ordinary Job waiting, chart-hook suppression, CRD skipping, weighted hook ordering, retained hook resources, install and upgrade rollback, failed-upgrade cleanup, timeout handling, and -dependency-gated Helm releases. Rendering also verifies a built `file://` library -dependency, hook manifests, and a Helm `tpl` expression preserved in stack values. +dependency-gated Helm releases. Rendering also verifies opt-in acquisition of a +missing `file://` library dependency, hook manifests, and a Helm `tpl` expression +preserved in stack values. The intentionally slow resources are observed with bounded Kubernetes readiness checks rather than fixed-delay assertions. diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 3663091a4c..379dd096e7 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -40,10 +40,9 @@ commands: description: "Render the local Helm chart, deploy it to the k3s emulator, verify it, then tear it down" steps: - atmos validate stacks - # Native Helm validates declared chart dependencies before rendering. CI - # and local contributors provision Helm on PATH before running the demo. - - helm dependency build --skip-refresh components/helm/demo - - atmos helm template demo -s dev --output=/tmp/atmos-helm-template.yaml + # Dependency acquisition is explicit: the opt-in flag fetches the missing + # file:// library and updates the local chart before the first render. + - atmos helm template demo -s dev --dependency-update --output=/tmp/atmos-helm-template.yaml # Template output includes weighted hooks and leaves Helm `tpl` expressions # in values for the chart to evaluate with its native .Values context. - command: >- diff --git a/website/docs/cli/commands/helm/helm-deploy.mdx b/website/docs/cli/commands/helm/helm-deploy.mdx index 7b01cb1cfa..e958bc4622 100644 --- a/website/docs/cli/commands/helm/helm-deploy.mdx +++ b/website/docs/cli/commands/helm/helm-deploy.mdx @@ -34,6 +34,9 @@ atmos helm deploy --affected --labels cost-center=platform
`--target` (optional)
Provision target to deliver to. Defaults to `provision.default`, otherwise the cluster.
+
`--dependency-update` (optional)
+
Fetch declared chart dependencies when they are missing. See [`apply`](/cli/commands/helm/apply#flags).
+
`--all` / `--affected` / `--include-dependents` (optional)
Process multiple Helm components in dependency order. See [`apply`](/cli/commands/helm/apply).
diff --git a/website/docs/cli/commands/helm/helm-plan.mdx b/website/docs/cli/commands/helm/helm-plan.mdx index c71c337f1e..48975a5509 100644 --- a/website/docs/cli/commands/helm/helm-plan.mdx +++ b/website/docs/cli/commands/helm/helm-plan.mdx @@ -40,6 +40,9 @@ atmos helm plan --all --tags production
`--against=` / `--from-manifest=` / `--context=` (optional)
Select and tune the diff baseline. See [`diff`](/cli/commands/helm/diff#flags).
+
`--dependency-update` (optional)
+
Fetch declared chart dependencies when they are missing. See [`diff`](/cli/commands/helm/diff#flags).
+
`--all` / `--affected` / `--include-dependents` (optional)
Process multiple Helm components in dependency order.
From 20a18213b3494febcf823038d43e10a67471a6fe Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 20:51:49 +0400 Subject: [PATCH 23/62] docs: clarify Helm example wait strategy --- examples/helm/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/helm/README.md b/examples/helm/README.md index 2405e64d5b..d4e648451d 100644 --- a/examples/helm/README.md +++ b/examples/helm/README.md @@ -72,8 +72,9 @@ atmos helm apply demo -s dev --identity local-k3s ``` The stack sets `wait_strategy: watcher`, `timeout: 2m`, and -`max_history: 10` as native Helm type defaults. The component enables -`rollback_on_failure` and failed-upgrade cleanup. The `atmos test` workflow +`max_history: 10` as native Helm type defaults. The `demo` component overrides +the wait strategy to `legacy`, and enables `rollback_on_failure` and +failed-upgrade cleanup. The `atmos test` workflow covers non-mutating apply and delete dry runs, ordinary Job waiting, chart-hook suppression, CRD skipping, weighted hook ordering, retained hook resources, install and upgrade rollback, failed-upgrade cleanup, timeout handling, and From ed2331a5f6dfecf28370a312d09a95d5045f236a Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 21:40:48 +0400 Subject: [PATCH 24/62] test: tolerate wrapped Helm timeout details --- examples/helm/atmos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 379dd096e7..580513a6cb 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -125,7 +125,7 @@ commands: - command: >- if output=$(NO_COLOR=1 atmos helm apply demo-timeout -s dev --identity local-k3s 2>&1); then echo "timed release unexpectedly succeeded"; exit 1; fi; case "$output" in *"helm release operation failed"*) ;; *) echo "timed release failed outside the Helm lifecycle operation: $output"; exit 1;; esac; - case "$output" in *"context deadline exceeded"*|*"timed out waiting for condition"*) ;; *) echo "timed release did not report a lifecycle timeout: $output"; exit 1;; esac + case "$output" in *"context"*"deadline exceeded"*|*"timed out waiting for condition"*) ;; *) echo "timed release did not report a lifecycle timeout: $output"; exit 1;; esac - command: >- if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-timeout get deployment demo-timeout 2>&1); then echo "timeout rollback left release resources"; exit 1; else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify timeout rollback cleanup: $output"; exit 1;; esac; fi From 80ac62b1d40ed379103ab81c85eb37a1d31790ff Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 21:49:52 +0400 Subject: [PATCH 25/62] test: verify failed Helm upgrade cause --- examples/helm/atmos.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 580513a6cb..acf3896dd9 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -143,7 +143,9 @@ commands: # the resource introduced only by the failed revision. - command: >- before=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}'); - if atmos helm apply demo-upgrade-fail -s dev --identity local-k3s; then echo "failed upgrade unexpectedly succeeded"; exit 1; fi; + if output=$(NO_COLOR=1 atmos helm apply demo-upgrade-fail -s dev --identity local-k3s 2>&1); then echo "failed upgrade unexpectedly succeeded"; exit 1; fi; + case "$output" in *"helm release operation failed"*) ;; *) echo "failed upgrade stopped outside the Helm lifecycle operation: $output"; exit 1;; esac; + case "$output" in *"job demo-failing-hook failed"*"BackoffLimitExceeded"*) ;; *) echo "failed upgrade did not report the expected post-upgrade hook failure: $output"; exit 1;; esac; after=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}'); if [ "$before" != "$after" ]; then echo "rollback did not restore Deployment/demo: before=$before after=$after"; exit 1; fi - command: >- From fbf7b9c9e85f606d5fc46bf6b0358060f8281330 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 22:46:38 +0400 Subject: [PATCH 26/62] test: fail Helm dry-run check on query errors --- examples/helm/atmos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index acf3896dd9..2eb60883cb 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -81,7 +81,7 @@ commands: if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo 2>&1); then echo "dry-run unexpectedly created service/demo"; exit 1; else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify service/demo absence: $output"; exit 1;; esac; fi - command: >- - releases=$(atmos emulator exec kubernetes -s dev -- kubectl get secrets --all-namespaces -l owner=helm,name=demo -o name); + if ! releases=$(atmos emulator exec kubernetes -s dev -- kubectl get secrets --all-namespaces -l owner=helm,name=demo -o name 2>&1); then echo "failed to query Helm release records after dry-run: $releases"; exit 1; fi; if [ -n "$releases" ]; then echo "dry-run unexpectedly persisted Helm release records: $releases"; exit 1; fi - command: atmos helm apply demo -s dev --identity local-k3s retry: From ad6b34d277b6835d6aaf400ec5d596da9e8e76ff Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 22:56:50 +0400 Subject: [PATCH 27/62] docs: document Helm apply dry-run --- website/docs/cli/commands/helm/helm-apply.mdx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/docs/cli/commands/helm/helm-apply.mdx b/website/docs/cli/commands/helm/helm-apply.mdx index 0178e3be9a..ea6df0b0f5 100644 --- a/website/docs/cli/commands/helm/helm-apply.mdx +++ b/website/docs/cli/commands/helm/helm-apply.mdx @@ -70,6 +70,9 @@ atmos helm apply --affected --labels cost-center=platform
`--dependency-update` (optional)
Fetch declared chart dependencies when they are missing. This may access dependency repositories and update the chart's charts/ directory and lock file.
+
`--dry-run` (optional)
+
Preview the install or upgrade without persisting release state or creating Kubernetes resources.
+
`--rollback-on-failure` (optional)
Uninstall a failed first install or roll a failed upgrade back. Enabling it promotes the default wait strategy to `watcher`. The deprecated alias is `--atomic`.
From 0cc088d84fe52038a072497e93cb67f0599e90d5 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sat, 1 Aug 2026 23:22:29 +0400 Subject: [PATCH 28/62] test: keep Helm release query logs separate --- examples/helm/atmos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 2eb60883cb..da3576114f 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -81,7 +81,7 @@ commands: if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo 2>&1); then echo "dry-run unexpectedly created service/demo"; exit 1; else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify service/demo absence: $output"; exit 1;; esac; fi - command: >- - if ! releases=$(atmos emulator exec kubernetes -s dev -- kubectl get secrets --all-namespaces -l owner=helm,name=demo -o name 2>&1); then echo "failed to query Helm release records after dry-run: $releases"; exit 1; fi; + if ! releases=$(atmos emulator exec kubernetes -s dev -- kubectl get secrets --all-namespaces -l owner=helm,name=demo -o name); then echo "failed to query Helm release records after dry-run"; exit 1; fi; if [ -n "$releases" ]; then echo "dry-run unexpectedly persisted Helm release records: $releases"; exit 1; fi - command: atmos helm apply demo -s dev --identity local-k3s retry: From c7b6cba27faf62aa0a9f01cb619ae158702a4f2a Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sun, 2 Aug 2026 00:48:10 +0400 Subject: [PATCH 29/62] test: stabilize macOS Helm lifecycle smoke --- .github/workflows/test.yml | 16 +++++++++++++--- examples/helm/stacks/deploy/dev.yaml | 5 ++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5dd0e2228d..063adb15de 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -648,19 +648,29 @@ jobs: docker rm -f atmos-dev-emulator-kubernetes >/dev/null 2>&1 || true } + diagnose_k3s() { + if ! docker inspect atmos-dev-emulator-kubernetes >/dev/null 2>&1; then + return + fi + + echo "::group::k3s workload diagnostics" + docker exec atmos-dev-emulator-kubernetes k3s kubectl get pods,jobs,deployments -A -o wide || true + docker exec atmos-dev-emulator-kubernetes k3s kubectl get events -A --sort-by=.lastTimestamp || true + docker exec atmos-dev-emulator-kubernetes ctr --namespace k8s.io images list || true + echo "::endgroup::" + } + # macOS k3s jobs can occasionally hang in the Docker/Colima stack. # Bound each attempt so the matrix can retry instead of consuming the # whole job timeout and cancelling the required aggregate check. attempt_timeout=1500 - if [ "${{ matrix.flavor.target }}" = "macos" ]; then - attempt_timeout=900 - fi trap cleanup_k3s EXIT for attempt in 1 2; do run_with_timeout "${attempt_timeout}" atmos test && exit 0 status=$? echo "atmos test failed (attempt ${attempt}/2, status ${status})" >&2 + diagnose_k3s cleanup_k3s [ "${attempt}" -lt 2 ] && sleep 15 done diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index d79d03b00c..3f69995eb8 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -4,7 +4,10 @@ vars: # Native Helm lifecycle defaults for every Helm component in this stack. helm: wait_strategy: watcher - timeout: 2m + # Leave enough headroom for nested k3s workloads on resource-constrained + # macOS/Colima runners. Components that exercise timeout behavior override + # this value explicitly below. + timeout: 4m max_history: 10 skip_crds: true From 746fdad270eb2b71f5459a0fc95a10f15e91b4ee Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sun, 2 Aug 2026 00:58:21 +0400 Subject: [PATCH 30/62] docs: align Helm smoke test budgets --- .github/workflows/test.yml | 5 ++++- examples/helm/README.md | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 063adb15de..251f7690fc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -568,7 +568,10 @@ jobs: - demo-helmfile - helm - timeout-minutes: 60 + # The macOS matrix may spend up to 45 minutes starting Colima, followed by + # two bounded 25-minute test attempts. Keep the job ceiling above those + # nested budgets so the second retry can finish and emit diagnostics. + timeout-minutes: 105 steps: - name: Check out code into the Go module directory uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/examples/helm/README.md b/examples/helm/README.md index d4e648451d..3311c70b83 100644 --- a/examples/helm/README.md +++ b/examples/helm/README.md @@ -71,8 +71,8 @@ atmos emulator up kubernetes -s dev atmos helm apply demo -s dev --identity local-k3s ``` -The stack sets `wait_strategy: watcher`, `timeout: 2m`, and -`max_history: 10` as native Helm type defaults. The `demo` component overrides +The stack sets `wait_strategy: watcher`, `timeout: 4m`, `max_history: 10`, and +`skip_crds: true` as native Helm type defaults. The `demo` component overrides the wait strategy to `legacy`, and enables `rollback_on_failure` and failed-upgrade cleanup. The `atmos test` workflow covers non-mutating apply and delete dry runs, ordinary Job waiting, chart-hook From 7b5c1ba6b6c8cf81f0f0288bf53d44349145d4e3 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sun, 2 Aug 2026 01:08:31 +0400 Subject: [PATCH 31/62] test: verify Helm lifecycle map ownership --- pkg/ci/plugins/helm/plugin_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/ci/plugins/helm/plugin_test.go b/pkg/ci/plugins/helm/plugin_test.go index c49ffd39b8..35cd4d50ad 100644 --- a/pkg/ci/plugins/helm/plugin_test.go +++ b/pkg/ci/plugins/helm/plugin_test.go @@ -87,6 +87,7 @@ func TestNormalizeSummary(t *testing.T) { assert.Equal(t, Summary{}, normalizeSummary((*Summary)(nil))) assert.Equal(t, Summary{}, normalizeSummary("not-summary")) + lifecycle := map[string]any{"operation": "install"} got := normalizeSummary(map[string]any{ "component": "app", "stack": "dev", @@ -100,7 +101,7 @@ func TestNormalizeSummary(t *testing.T) { "manifest_bytes": float64(123), "message": 42, "diff": "diff text", - "lifecycle": map[string]any{"operation": "install"}, + "lifecycle": lifecycle, }) assert.Equal(t, "app", got.Component) assert.Equal(t, "dev", got.Stack) @@ -115,6 +116,11 @@ func TestNormalizeSummary(t *testing.T) { assert.Equal(t, "42", got.Message) assert.Equal(t, "diff text", got.Diff) assert.Equal(t, map[string]any{"operation": "install"}, got.Lifecycle) + + lifecycle["operation"] = "upgrade" + assert.Equal(t, "install", got.Lifecycle["operation"]) + got.Lifecycle["timeout"] = "30m0s" + assert.NotContains(t, lifecycle, "timeout") } func TestPluginBuildTemplateContextFallbacksAndErrors(t *testing.T) { From 08d82628bf15de19223b5f7072f032f63b89ce7b Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sun, 2 Aug 2026 02:19:14 +0400 Subject: [PATCH 32/62] test: extend macOS Helm smoke budget --- .github/workflows/test.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 251f7690fc..81ed3a03e8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -569,9 +569,9 @@ jobs: - helm # The macOS matrix may spend up to 45 minutes starting Colima, followed by - # two bounded 25-minute test attempts. Keep the job ceiling above those + # two bounded 45-minute test attempts. Keep the job ceiling above those # nested budgets so the second retry can finish and emit diagnostics. - timeout-minutes: 105 + timeout-minutes: 150 steps: - name: Check out code into the Go module directory uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -657,8 +657,8 @@ jobs: fi echo "::group::k3s workload diagnostics" - docker exec atmos-dev-emulator-kubernetes k3s kubectl get pods,jobs,deployments -A -o wide || true - docker exec atmos-dev-emulator-kubernetes k3s kubectl get events -A --sort-by=.lastTimestamp || true + docker exec atmos-dev-emulator-kubernetes kubectl get pods,jobs,deployments -A -o wide || true + docker exec atmos-dev-emulator-kubernetes kubectl get events -A --sort-by=.lastTimestamp || true docker exec atmos-dev-emulator-kubernetes ctr --namespace k8s.io images list || true echo "::endgroup::" } @@ -667,6 +667,9 @@ jobs: # Bound each attempt so the matrix can retry instead of consuming the # whole job timeout and cancelling the required aggregate check. attempt_timeout=1500 + if [ "${{ matrix.flavor.target }}" = "macos" ]; then + attempt_timeout=2700 + fi trap cleanup_k3s EXIT for attempt in 1 2; do From d8004af4a0e707192e2bf631f41cf02981400302 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sun, 2 Aug 2026 02:28:09 +0400 Subject: [PATCH 33/62] test: bound k3s diagnostics and cleanup --- .github/workflows/test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 81ed3a03e8..a2778885da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -647,19 +647,19 @@ jobs: } cleanup_k3s() { - atmos emulator down kubernetes -s dev || true - docker rm -f atmos-dev-emulator-kubernetes >/dev/null 2>&1 || true + run_with_timeout 60 atmos emulator down kubernetes -s dev || true + run_with_timeout 30 docker rm -f atmos-dev-emulator-kubernetes >/dev/null 2>&1 || true } diagnose_k3s() { - if ! docker inspect atmos-dev-emulator-kubernetes >/dev/null 2>&1; then + if ! run_with_timeout 15 docker inspect atmos-dev-emulator-kubernetes >/dev/null 2>&1; then return fi echo "::group::k3s workload diagnostics" - docker exec atmos-dev-emulator-kubernetes kubectl get pods,jobs,deployments -A -o wide || true - docker exec atmos-dev-emulator-kubernetes kubectl get events -A --sort-by=.lastTimestamp || true - docker exec atmos-dev-emulator-kubernetes ctr --namespace k8s.io images list || true + run_with_timeout 30 docker exec atmos-dev-emulator-kubernetes kubectl get pods,jobs,deployments -A -o wide || true + run_with_timeout 30 docker exec atmos-dev-emulator-kubernetes kubectl get events -A --sort-by=.lastTimestamp || true + run_with_timeout 30 docker exec atmos-dev-emulator-kubernetes ctr --namespace k8s.io images list || true echo "::endgroup::" } From 4c0ee8bf25e95c030ff7c8b6668a87590d0fe40f Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sun, 2 Aug 2026 02:40:44 +0400 Subject: [PATCH 34/62] test: cover helm lifecycle summaries --- pkg/ci/plugins/helm/plugin_test.go | 113 ++++++++++++++++++++--------- 1 file changed, 77 insertions(+), 36 deletions(-) diff --git a/pkg/ci/plugins/helm/plugin_test.go b/pkg/ci/plugins/helm/plugin_test.go index 35cd4d50ad..b26fe00e09 100644 --- a/pkg/ci/plugins/helm/plugin_test.go +++ b/pkg/ci/plugins/helm/plugin_test.go @@ -242,21 +242,17 @@ func TestPluginOnAfterOperation(t *testing.T) { } func TestTemplateRendering(t *testing.T) { - ctx := (&Plugin{}).buildTemplateContext(&plugin.HookContext{ - Command: "apply", - Info: &schema.ConfigAndStacksInfo{ - ComponentFromArg: "nginx", - Stack: "plat-ue2-dev", - }, - Aggregate: Summary{ - Chart: "bitnami/nginx", - ReleaseName: "nginx", - Namespace: "apps", - Target: "kubernetes", - ObjectCount: 2, - ObjectKinds: []string{"Deployment", "Service"}, - ManifestBytes: 1234, - Lifecycle: map[string]any{ + tests := []struct { + name string + command string + lifecycle map[string]any + contains []string + notContains []string + }{ + { + name: "cluster apply", + command: "apply", + lifecycle: map[string]any{ "operation": "upgrade", "wait_strategy": "watcher", "timeout": "30m0s", @@ -266,27 +262,72 @@ func TestTemplateRendering(t *testing.T) { "cleanup_on_fail": true, "max_history": 10, }, + contains: []string{ + "Helm Apply Summary", "bitnami/nginx", "Deployment", "Release lifecycle", + "Wait strategy", "watcher", "Maximum history", "`10`", + }, }, - }) + { + name: "external apply", + command: "apply", + lifecycle: map[string]any{ + "applied": false, "target_kind": "git", "reason": "external_target", + }, + contains: []string{"Helm Apply Summary", "external_target"}, + notContains: []string{"Wait strategy", "Timeout"}, + }, + { + name: "cluster delete", + command: "delete", + lifecycle: map[string]any{ + "operation": "uninstall", + "wait_strategy": "legacy", + "timeout": "10m0s", + "chart_hooks_enabled": false, + }, + contains: []string{ + "Helm Delete Summary", "Release lifecycle", "Wait strategy", "legacy", "10m0s", + }, + }, + { + name: "external delete", + command: "delete", + lifecycle: map[string]any{ + "deleted": false, "target_kind": "git", "reason": "external_target", + }, + contains: []string{"Helm Delete Summary", "external_target"}, + notContains: []string{"Wait strategy", "Timeout"}, + }, + } - rendered, err := templates.NewLoader(nil).LoadAndRender("helm", "apply", defaultTemplates, ctx) - require.NoError(t, err) - assert.Contains(t, rendered, "Helm Apply Summary") - assert.Contains(t, rendered, "bitnami/nginx") - assert.Contains(t, rendered, "Deployment") - assert.Contains(t, rendered, "Release lifecycle") - assert.Contains(t, rendered, "watcher") - assert.Contains(t, rendered, "Maximum history") - assert.Contains(t, rendered, "`10`") - - external := (&Plugin{}).buildTemplateContext(&plugin.HookContext{ - Command: "apply", - Aggregate: Summary{Lifecycle: map[string]any{ - "applied": false, "target_kind": "git", "reason": "external_target", - }}, - }) - rendered, err = templates.NewLoader(nil).LoadAndRender("helm", "apply", defaultTemplates, external) - require.NoError(t, err) - assert.Contains(t, rendered, "external_target") - assert.NotContains(t, rendered, "Wait strategy") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := (&Plugin{}).buildTemplateContext(&plugin.HookContext{ + Command: tt.command, + Info: &schema.ConfigAndStacksInfo{ + ComponentFromArg: "nginx", + Stack: "plat-ue2-dev", + }, + Aggregate: Summary{ + Chart: "bitnami/nginx", + ReleaseName: "nginx", + Namespace: "apps", + Target: "kubernetes", + ObjectCount: 2, + ObjectKinds: []string{"Deployment", "Service"}, + ManifestBytes: 1234, + Lifecycle: tt.lifecycle, + }, + }) + + rendered, err := templates.NewLoader(nil).LoadAndRender("helm", tt.command, defaultTemplates, ctx) + require.NoError(t, err) + for _, expected := range tt.contains { + assert.Contains(t, rendered, expected) + } + for _, unexpected := range tt.notContains { + assert.NotContains(t, rendered, unexpected) + } + }) + } } From dd26fbceefd745d48d868147adc564f4ccfaa7fc Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sun, 2 Aug 2026 02:52:47 +0400 Subject: [PATCH 35/62] test: strengthen helm lifecycle fixtures --- .github/workflows/test.yml | 2 +- examples/helm/atmos.yaml | 6 ++++++ .../helm/demo/templates/dependency-observed.yaml | 14 ++++++++++++++ .../helm/demo/templates/hook-order-job.yaml | 4 +++- website/docs/ci/job-summaries.mdx | 6 +++--- 5 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 examples/helm/components/helm/demo/templates/dependency-observed.yaml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a2778885da..3a892a95e0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -571,7 +571,7 @@ jobs: # The macOS matrix may spend up to 45 minutes starting Colima, followed by # two bounded 45-minute test attempts. Keep the job ceiling above those # nested budgets so the second retry can finish and emit diagnostics. - timeout-minutes: 150 + timeout-minutes: 180 steps: - name: Check out code into the Go module directory uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index da3576114f..f73aac3175 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -114,6 +114,9 @@ commands: - command: >- if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-no-hooks get configmap demo-no-hooks-hook-order 2>&1); then echo "disable_chart_hooks unexpectedly ran Helm hooks"; exit 1; else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify disabled Helm hooks: $output"; exit 1;; esac; fi + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-no-hooks get job demo-no-hooks-hook-order 2>&1); then echo "disable_chart_hooks unexpectedly ran the Helm hook Job"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify disabled Helm hook Job: $output"; exit 1;; esac; fi # hookOnly returns after hooks without waiting for the Deployment's explicit readiness gate. - atmos helm apply demo-hook-only -s dev --identity local-k3s - command: >- @@ -155,6 +158,9 @@ commands: # foundation's post-readiness marker. - atmos helm apply --all -s dev --identity local-k3s --tags lifecycle-dag - atmos emulator exec kubernetes -s dev -- kubectl -n lifecycle-dag get deployment dag-dependent + - command: >- + observed=$(atmos emulator exec kubernetes -s dev -- kubectl -n lifecycle-dag get configmap dag-dependent-dependency-observed -o jsonpath='{.data.ready}'); + if [ "$observed" != "true" ]; then echo "dependent did not observe the ready foundation marker: ready=$observed"; exit 1; fi # Delete dry-run must leave the deployed release and resources intact. - atmos helm delete demo -s dev --identity local-k3s --dry-run --wait=watcher --timeout=2m - atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo diff --git a/examples/helm/components/helm/demo/templates/dependency-observed.yaml b/examples/helm/components/helm/demo/templates/dependency-observed.yaml new file mode 100644 index 0000000000..6990b8be0d --- /dev/null +++ b/examples/helm/components/helm/demo/templates/dependency-observed.yaml @@ -0,0 +1,14 @@ +{{- if .Values.hooks.requiredConfigMap }} +{{- $required := lookup "v1" "ConfigMap" .Release.Namespace .Values.hooks.requiredConfigMap }} +{{- if not $required }} +{{- fail (printf "required dependency marker ConfigMap %q is not ready" .Values.hooks.requiredConfigMap) }} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-dependency-observed + namespace: {{ .Release.Namespace }} +data: + requiredConfigMap: {{ .Values.hooks.requiredConfigMap | quote }} + ready: {{ index $required.data "ready" | quote }} +{{- end }} diff --git a/examples/helm/components/helm/demo/templates/hook-order-job.yaml b/examples/helm/components/helm/demo/templates/hook-order-job.yaml index c3a24fa154..5cdde68ffc 100644 --- a/examples/helm/components/helm/demo/templates/hook-order-job.yaml +++ b/examples/helm/components/helm/demo/templates/hook-order-job.yaml @@ -7,7 +7,9 @@ metadata: annotations: "helm.sh/hook": pre-install,pre-upgrade "helm.sh/hook-weight": "-1" - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + # Retain the completed Job so lifecycle integration tests can distinguish + # a disabled hook from a hook that ran successfully and self-deleted. + "helm.sh/hook-delete-policy": before-hook-creation spec: backoffLimit: 0 template: diff --git a/website/docs/ci/job-summaries.mdx b/website/docs/ci/job-summaries.mdx index 72655e0f34..97c1cd4580 100644 --- a/website/docs/ci/job-summaries.mdx +++ b/website/docs/ci/job-summaries.mdx @@ -127,9 +127,9 @@ kinds, and rendered manifest size when available. For cluster-backed apply/deploy/delete operations, the aggregate also includes an operation-specific `lifecycle` block with the effective wait strategy, timeout, chart-hook state, and applicable rollback, Job-wait, CRD, cleanup, and -history values. For external delivery, it reports `applied: false`, the selected -target kind, and `reason: external_target` instead of presenting stored release -policy as active. +history values. For external delivery, apply/deploy reports `applied: false` +while delete reports `deleted: false`; both include the selected target kind and +`reason: external_target` instead of presenting stored release policy as active. ## Helmfile Summaries From 3fe8b4313980700acf29c08d8c7275b087ebd577 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sun, 2 Aug 2026 03:01:17 +0400 Subject: [PATCH 36/62] test: tighten helm lifecycle coverage --- .../helm/demo/templates/hook-order-job.yaml | 2 +- pkg/ci/plugins/helm/templates/apply.md | 7 ++ pkg/ci/plugins/helm/templates/delete.md | 3 + pkg/component/helm/client_lifecycle_test.go | 75 +++++++++++-------- 4 files changed, 55 insertions(+), 32 deletions(-) diff --git a/examples/helm/components/helm/demo/templates/hook-order-job.yaml b/examples/helm/components/helm/demo/templates/hook-order-job.yaml index 5cdde68ffc..5b059044d2 100644 --- a/examples/helm/components/helm/demo/templates/hook-order-job.yaml +++ b/examples/helm/components/helm/demo/templates/hook-order-job.yaml @@ -25,5 +25,5 @@ spec: volumes: - name: hook-order configMap: - name: {{ default (printf "%s-hook-order" .Release.Name) .Values.hooks.requiredConfigMap }} + name: {{ printf "%s-hook-order" .Release.Name }} {{- end }} diff --git a/pkg/ci/plugins/helm/templates/apply.md b/pkg/ci/plugins/helm/templates/apply.md index b8dd0d3db1..02d01bfffb 100644 --- a/pkg/ci/plugins/helm/templates/apply.md +++ b/pkg/ci/plugins/helm/templates/apply.md @@ -25,6 +25,7 @@ | Applied | `false` | | Target kind | `{{ index . "target_kind" }}` | | Reason | `external_target` | + {{- else }} | Field | Value | @@ -35,14 +36,20 @@ | Chart hooks enabled | `{{ index . "chart_hooks_enabled" }}` | | Wait for Jobs | `{{ index . "wait_for_jobs" }}` | | Rollback on failure | `{{ index . "rollback_on_failure" }}` | + {{- if eq (index . "operation") "install" }} | Install CRDs | `{{ index . "install_crds" }}` | + {{- end }} + {{- if eq (index . "operation") "upgrade" }} | Cleanup on failure | `{{ index . "cleanup_on_fail" }}` | | Maximum history | `{{ index . "max_history" }}` | + {{- end }} + {{- end }} + {{- end }} To reproduce locally: diff --git a/pkg/ci/plugins/helm/templates/delete.md b/pkg/ci/plugins/helm/templates/delete.md index 1923bfd730..6bd8b123ee 100644 --- a/pkg/ci/plugins/helm/templates/delete.md +++ b/pkg/ci/plugins/helm/templates/delete.md @@ -21,6 +21,7 @@ | Deleted | `false` | | Target kind | `{{ index . "target_kind" }}` | | Reason | `external_target` | + {{- else }} | Field | Value | @@ -29,7 +30,9 @@ | Wait strategy | `{{ index . "wait_strategy" }}` | | Timeout | `{{ index . "timeout" }}` | | Chart hooks enabled | `{{ index . "chart_hooks_enabled" }}` | + {{- end }} + {{- end }} To reproduce locally: diff --git a/pkg/component/helm/client_lifecycle_test.go b/pkg/component/helm/client_lifecycle_test.go index a1f02f2f68..e34de13221 100644 --- a/pkg/component/helm/client_lifecycle_test.go +++ b/pkg/component/helm/client_lifecycle_test.go @@ -215,40 +215,53 @@ func TestReleaseOperationContextPreservesZeroTimeout(t *testing.T) { assert.False(t, hasDeadline) } -func TestUpgradeReleasePrunesDefaultHistory(t *testing.T) { - actx := memoryActionContext(t) - stubActionContext(t, actx) - spec := testdataChartSpec(t, "history") - - for revision := 0; revision < cfg.HelmDefaultMaxHistory+3; revision++ { - spec.Values["replicaCount"] = revision + 1 - _, err := applyRelease(context.Background(), spec, false) - require.NoError(t, err) - } - - history, err := actx.cfg.Releases.History(spec.ReleaseName) - require.NoError(t, err) - assert.Len(t, history, cfg.HelmDefaultMaxHistory) - assert.Equal(t, []int{4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, releaseVersions(t, history)) -} - -func TestUpgradeReleaseUnlimitedHistory(t *testing.T) { - actx := memoryActionContext(t) - stubActionContext(t, actx) - spec := testdataChartSpec(t, "unlimited-history") - spec.Lifecycle.Policy.MaxHistory = 0 - +func TestUpgradeReleaseHistoryRetention(t *testing.T) { const revisions = cfg.HelmDefaultMaxHistory + 3 - for revision := 0; revision < revisions; revision++ { - spec.Values["replicaCount"] = revision + 1 - _, err := applyRelease(context.Background(), spec, false) - require.NoError(t, err) + tests := []struct { + name string + releaseName string + maxHistory int + override bool + expectedCount int + expected []int + }{ + { + name: "default bounded history", + releaseName: "history", + expectedCount: cfg.HelmDefaultMaxHistory, + expected: []int{4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, + }, + { + name: "explicit unlimited history", + releaseName: "unlimited-history", + maxHistory: 0, + override: true, + expectedCount: revisions, + expected: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, + }, } - history, err := actx.cfg.Releases.History(spec.ReleaseName) - require.NoError(t, err) - assert.Len(t, history, revisions) - assert.Equal(t, []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, releaseVersions(t, history)) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actx := memoryActionContext(t) + stubActionContext(t, actx) + spec := testdataChartSpec(t, tt.releaseName) + if tt.override { + spec.Lifecycle.Policy.MaxHistory = tt.maxHistory + } + + for revision := 0; revision < revisions; revision++ { + spec.Values["replicaCount"] = revision + 1 + _, err := applyRelease(context.Background(), spec, false) + require.NoError(t, err) + } + + history, err := actx.cfg.Releases.History(spec.ReleaseName) + require.NoError(t, err) + assert.Len(t, history, tt.expectedCount) + assert.Equal(t, tt.expected, releaseVersions(t, history)) + }) + } } func releaseVersions(t *testing.T, history []helmrelease.Releaser) []int { From 7bf6eea61cc33e746bfdce3bb5afc1b691b81f19 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sun, 2 Aug 2026 04:16:55 +0400 Subject: [PATCH 37/62] test: allow macos rollout convergence --- examples/helm/atmos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index f73aac3175..e000d0defe 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -123,7 +123,7 @@ commands: available=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only get deployment demo-hook-only -o jsonpath='{.status.conditions[?(@.type=="Available")].status}'); if [ "$available" = "True" ]; then echo "hookOnly unexpectedly satisfied the Deployment readiness gate"; exit 1; fi - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only patch deployment demo-hook-only --type=json -p='[{"op":"remove","path":"/spec/template/spec/readinessGates"}]' - - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only rollout status deployment/demo-hook-only --timeout=30s + - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only rollout status deployment/demo-hook-only --timeout=2m # watcher honors the release timeout and rollback removes the failed install. - command: >- if output=$(NO_COLOR=1 atmos helm apply demo-timeout -s dev --identity local-k3s 2>&1); then echo "timed release unexpectedly succeeded"; exit 1; fi; From 57b5764b8f8d70b1fb372bd3fb71d32f4d6e696b Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sun, 2 Aug 2026 04:27:37 +0400 Subject: [PATCH 38/62] test: preserve offline helm rendering --- .../helm/demo/templates/dependency-observed.yaml | 2 +- examples/helm/components/helm/demo/values.yaml | 1 + examples/helm/stacks/deploy/dev.yaml | 3 +++ pkg/ci/plugins/helm/plugin_test.go | 8 +++++++- pkg/component/helm/client_lifecycle_test.go | 10 ++++++---- 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/examples/helm/components/helm/demo/templates/dependency-observed.yaml b/examples/helm/components/helm/demo/templates/dependency-observed.yaml index 6990b8be0d..1eb9053c3b 100644 --- a/examples/helm/components/helm/demo/templates/dependency-observed.yaml +++ b/examples/helm/components/helm/demo/templates/dependency-observed.yaml @@ -1,4 +1,4 @@ -{{- if .Values.hooks.requiredConfigMap }} +{{- if and .Values.hooks.requiredConfigMap .Values.hooks.validateDependency }} {{- $required := lookup "v1" "ConfigMap" .Release.Namespace .Values.hooks.requiredConfigMap }} {{- if not $required }} {{- fail (printf "required dependency marker ConfigMap %q is not ready" .Values.hooks.requiredConfigMap) }} diff --git a/examples/helm/components/helm/demo/values.yaml b/examples/helm/components/helm/demo/values.yaml index 2cb84753a6..96e5a2aa83 100644 --- a/examples/helm/components/helm/demo/values.yaml +++ b/examples/helm/components/helm/demo/values.yaml @@ -19,6 +19,7 @@ hooks: enabled: true fail: false requiredConfigMap: "" + validateDependency: false readyMarker: false extraConfigMap: enabled: false diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index 3f69995eb8..2911a323e7 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -162,3 +162,6 @@ components: values: hooks: requiredConfigMap: dag-foundation-ready + # The lifecycle DAG integration test explicitly opts into a live + # cluster lookup; ordinary offline chart rendering leaves this off. + validateDependency: true diff --git a/pkg/ci/plugins/helm/plugin_test.go b/pkg/ci/plugins/helm/plugin_test.go index b26fe00e09..168869c1fb 100644 --- a/pkg/ci/plugins/helm/plugin_test.go +++ b/pkg/ci/plugins/helm/plugin_test.go @@ -264,7 +264,13 @@ func TestTemplateRendering(t *testing.T) { }, contains: []string{ "Helm Apply Summary", "bitnami/nginx", "Deployment", "Release lifecycle", - "Wait strategy", "watcher", "Maximum history", "`10`", + "| Wait strategy | `watcher` |", + "| Timeout | `30m0s` |", + "| Chart hooks enabled | `true` |", + "| Wait for Jobs | `true` |", + "| Rollback on failure | `true` |", + "| Cleanup on failure | `true` |", + "| Maximum history | `10` |", }, }, { diff --git a/pkg/component/helm/client_lifecycle_test.go b/pkg/component/helm/client_lifecycle_test.go index e34de13221..67fb2c643b 100644 --- a/pkg/component/helm/client_lifecycle_test.go +++ b/pkg/component/helm/client_lifecycle_test.go @@ -223,13 +223,11 @@ func TestUpgradeReleaseHistoryRetention(t *testing.T) { maxHistory int override bool expectedCount int - expected []int }{ { name: "default bounded history", releaseName: "history", expectedCount: cfg.HelmDefaultMaxHistory, - expected: []int{4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, }, { name: "explicit unlimited history", @@ -237,7 +235,6 @@ func TestUpgradeReleaseHistoryRetention(t *testing.T) { maxHistory: 0, override: true, expectedCount: revisions, - expected: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, }, } @@ -259,7 +256,12 @@ func TestUpgradeReleaseHistoryRetention(t *testing.T) { history, err := actx.cfg.Releases.History(spec.ReleaseName) require.NoError(t, err) assert.Len(t, history, tt.expectedCount) - assert.Equal(t, tt.expected, releaseVersions(t, history)) + expected := make([]int, tt.expectedCount) + firstRevision := revisions - tt.expectedCount + 1 + for i := range expected { + expected[i] = firstRevision + i + } + assert.Equal(t, expected, releaseVersions(t, history)) }) } } From a955957a83b7b07c855ef0df984b74480317ee91 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sun, 2 Aug 2026 04:36:27 +0400 Subject: [PATCH 39/62] test: distinguish helm query failures --- examples/helm/atmos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index e000d0defe..d337f3d933 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -166,7 +166,7 @@ commands: - atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo - atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo - command: >- - releases=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get secrets -l owner=helm,name=demo -o name); + if ! releases=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get secrets -l owner=helm,name=demo -o name); then echo "failed to query Helm release records after delete dry-run"; exit 1; fi; if [ -z "$releases" ]; then echo "delete dry-run removed the Helm release record"; exit 1; fi - atmos helm delete demo -s dev --identity local-k3s - atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-hook-order From ace29d4bbe44d8bb346ad6569a98e47a468c3a8c Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sun, 2 Aug 2026 04:47:02 +0400 Subject: [PATCH 40/62] test: assert helm summary lifecycle rows --- pkg/ci/plugins/helm/plugin_test.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/ci/plugins/helm/plugin_test.go b/pkg/ci/plugins/helm/plugin_test.go index 168869c1fb..69ace4b9f4 100644 --- a/pkg/ci/plugins/helm/plugin_test.go +++ b/pkg/ci/plugins/helm/plugin_test.go @@ -279,7 +279,12 @@ func TestTemplateRendering(t *testing.T) { lifecycle: map[string]any{ "applied": false, "target_kind": "git", "reason": "external_target", }, - contains: []string{"Helm Apply Summary", "external_target"}, + contains: []string{ + "Helm Apply Summary", + "| Applied | `false` |", + "| Target kind | `git` |", + "| Reason | `external_target` |", + }, notContains: []string{"Wait strategy", "Timeout"}, }, { @@ -292,7 +297,10 @@ func TestTemplateRendering(t *testing.T) { "chart_hooks_enabled": false, }, contains: []string{ - "Helm Delete Summary", "Release lifecycle", "Wait strategy", "legacy", "10m0s", + "Helm Delete Summary", "Release lifecycle", + "| Wait strategy | `legacy` |", + "| Timeout | `10m0s` |", + "| Chart hooks enabled | `false` |", }, }, { @@ -301,7 +309,12 @@ func TestTemplateRendering(t *testing.T) { lifecycle: map[string]any{ "deleted": false, "target_kind": "git", "reason": "external_target", }, - contains: []string{"Helm Delete Summary", "external_target"}, + contains: []string{ + "Helm Delete Summary", + "| Deleted | `false` |", + "| Target kind | `git` |", + "| Reason | `external_target` |", + }, notContains: []string{"Wait strategy", "Timeout"}, }, } From 9df475756cba456640329f68e66472d33e24cd2f Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sun, 2 Aug 2026 04:58:14 +0400 Subject: [PATCH 41/62] test: cover Helm install lifecycle summary --- pkg/ci/plugins/helm/plugin_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/ci/plugins/helm/plugin_test.go b/pkg/ci/plugins/helm/plugin_test.go index 69ace4b9f4..899b9217b7 100644 --- a/pkg/ci/plugins/helm/plugin_test.go +++ b/pkg/ci/plugins/helm/plugin_test.go @@ -273,6 +273,25 @@ func TestTemplateRendering(t *testing.T) { "| Maximum history | `10` |", }, }, + { + name: "cluster install", + command: "apply", + lifecycle: map[string]any{ + "operation": "install", + "wait_strategy": "hookOnly", + "timeout": "5m0s", + "chart_hooks_enabled": true, + "wait_for_jobs": false, + "rollback_on_failure": false, + "install_crds": true, + }, + contains: []string{ + "Helm Apply Summary", "Release lifecycle", + "| Operation | `install` |", + "| Install CRDs | `true` |", + }, + notContains: []string{"Cleanup on failure", "Maximum history"}, + }, { name: "external apply", command: "apply", From ead8862f0d68edaaa3bb2c62b28904711d1dbf88 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Sun, 2 Aug 2026 05:44:43 +0400 Subject: [PATCH 42/62] feat: add aggregate Helm CI summaries --- pkg/ci/plugins/helm/aggregate.go | 305 ++++++++++++++++++ pkg/ci/plugins/helm/aggregate_test.go | 130 ++++++++ pkg/ci/plugins/helm/plugin.go | 2 + pkg/ci/plugins/helm/plugin_test.go | 4 +- pkg/component/helm/aggregate_ci.go | 172 ++++++++++ pkg/component/helm/aggregate_ci_test.go | 143 ++++++++ pkg/component/helm/executor.go | 20 +- pkg/component/helm/executor_bulk.go | 26 +- pkg/hooks/event.go | 2 + pkg/schema/schema.go | 23 ++ website/docs/ci/job-summaries.mdx | 20 +- website/docs/cli/commands/helm/helm-apply.mdx | 4 + website/docs/cli/commands/helm/helm-plan.mdx | 5 + 13 files changed, 838 insertions(+), 18 deletions(-) create mode 100644 pkg/ci/plugins/helm/aggregate.go create mode 100644 pkg/ci/plugins/helm/aggregate_test.go create mode 100644 pkg/component/helm/aggregate_ci.go create mode 100644 pkg/component/helm/aggregate_ci_test.go diff --git a/pkg/ci/plugins/helm/aggregate.go b/pkg/ci/plugins/helm/aggregate.go new file mode 100644 index 0000000000..afdb714743 --- /dev/null +++ b/pkg/ci/plugins/helm/aggregate.go @@ -0,0 +1,305 @@ +package helm + +import ( + "fmt" + "sort" + "strconv" + "strings" + "unicode/utf8" + + "github.com/cloudposse/atmos/pkg/ci/internal/plugin" + log "github.com/cloudposse/atmos/pkg/logger" + "github.com/cloudposse/atmos/pkg/perf" + "github.com/cloudposse/atmos/pkg/schema" +) + +const helmAggregateMarkdownMaxBytes = 960 * 1024 + +type helmAggregate struct { + Command string + Components []helmAggregateComponent + Counts helmAggregateCounts +} + +type helmAggregateComponent struct { + Result schema.HelmCIResult + Summary Summary + Status string +} + +type helmAggregateCounts struct { + Total int + Succeeded int + Failed int + Changed int + NoChanges int + Skipped int +} + +type helmAggregateCountRow struct { + label string + count int +} + +// onAfterAggregate writes one deterministic job summary for a graph-backed +// Helm plan or apply command. +func (p *Plugin) onAfterAggregate(ctx *plugin.HookContext) error { + defer perf.Track(ctx.Config, "helmci.Plugin.onAfterAggregate")() + + resultSet, ok := normalizeHelmAggregate(ctx.Aggregate) + if !ok || len(resultSet.Results) == 0 { + log.Debug("Skipping aggregate Helm CI hook: no results") + return nil + } + if !isSummaryEnabled(ctx.Config) { + return nil + } + writer := ctx.Provider.OutputWriter() + if writer == nil { + return nil + } + aggregate := buildHelmAggregate(resultSet) + return writer.WriteSummary(renderHelmAggregateMarkdown(&aggregate)) +} + +func normalizeHelmAggregate(value any) (schema.HelmCIResultSet, bool) { + switch typed := value.(type) { + case schema.HelmCIResultSet: + return typed, true + case *schema.HelmCIResultSet: + if typed == nil { + return schema.HelmCIResultSet{}, false + } + return *typed, true + default: + return schema.HelmCIResultSet{}, false + } +} + +func buildHelmAggregate(resultSet schema.HelmCIResultSet) helmAggregate { + results := append([]schema.HelmCIResult(nil), resultSet.Results...) + sort.SliceStable(results, func(i, j int) bool { + if results[i].Stack != results[j].Stack { + return results[i].Stack < results[j].Stack + } + if results[i].Component != results[j].Component { + return results[i].Component < results[j].Component + } + return results[i].NodeID < results[j].NodeID + }) + + aggregate := helmAggregate{Command: normalizeHelmAggregateCommand(resultSet.Command)} + aggregate.Components = make([]helmAggregateComponent, 0, len(results)) + for i := range results { + component := helmAggregateComponent{ + Result: results[i], + Summary: normalizeSummary(results[i].Summary), + } + component.Status = helmAggregateStatus(aggregate.Command, &component) + aggregate.Components = append(aggregate.Components, component) + aggregate.Counts.add(component.Status) + } + return aggregate +} + +func normalizeHelmAggregateCommand(command string) string { + switch command { + case "apply", "deploy": + return "apply" + default: + return "plan" + } +} + +func helmAggregateStatus(command string, component *helmAggregateComponent) string { + if component.Result.Status == "failed" || component.Result.Error != "" || component.Result.ExitCode != 0 { + return "failed" + } + if component.Result.Status == "skipped" || !component.Result.Processed { + return "skipped" + } + if command == "plan" { + if strings.TrimSpace(component.Summary.Diff) != "" { + return "changed" + } + return "no changes" + } + return "succeeded" +} + +func (counts *helmAggregateCounts) add(status string) { + counts.Total++ + switch status { + case "failed": + counts.Failed++ + case "changed": + counts.Changed++ + case "no changes": + counts.NoChanges++ + case "skipped": + counts.Skipped++ + default: + counts.Succeeded++ + } +} + +func renderHelmAggregateMarkdown(aggregate *helmAggregate) string { + var builder strings.Builder + builder.WriteString("## Helm ") + builder.WriteString(helmAggregateCommandLabel(aggregate.Command)) + builder.WriteString(" Summary\n\n") + builder.WriteString(helmAggregateSummaryText(aggregate.Command, &aggregate.Counts)) + builder.WriteString("\n\n") + writeHelmAggregateCounts(&builder, aggregate.Command, &aggregate.Counts) + writeHelmAggregateComponents(&builder, aggregate.Components) + writeHelmAggregateDetails(&builder, aggregate.Command, aggregate.Components) + return enforceHelmAggregateMarkdownLimit(builder.String()) +} + +func helmAggregateCommandLabel(command string) string { + if command == "apply" { + return "Apply" + } + return "Plan" +} + +func helmAggregateSummaryText(command string, counts *helmAggregateCounts) string { + if command == "apply" { + return fmt.Sprintf( + "Processed %d component(s): %d succeeded, %d failed, %d skipped.", + counts.Total, + counts.Succeeded, + counts.Failed, + counts.Skipped, + ) + } + return fmt.Sprintf( + "Processed %d component(s): %d changed, %d unchanged, %d failed, %d skipped.", + counts.Total, + counts.Changed, + counts.NoChanges, + counts.Failed, + counts.Skipped, + ) +} + +func writeHelmAggregateCounts(builder *strings.Builder, command string, counts *helmAggregateCounts) { + builder.WriteString("| Result | Components |\n") + builder.WriteString("| --- | ---: |\n") + rows := make([]helmAggregateCountRow, 0, 4) + if command == "apply" { + rows = append(rows, helmAggregateCountRow{label: "Succeeded", count: counts.Succeeded}) + } else { + rows = append(rows, + helmAggregateCountRow{label: "Changed", count: counts.Changed}, + helmAggregateCountRow{label: "No changes", count: counts.NoChanges}, + ) + } + rows = append(rows, + helmAggregateCountRow{label: "Failed", count: counts.Failed}, + helmAggregateCountRow{label: "Skipped", count: counts.Skipped}, + ) + for _, row := range rows { + builder.WriteString("| ") + builder.WriteString(row.label) + builder.WriteString(" | ") + builder.WriteString(strconv.Itoa(row.count)) + builder.WriteString(" |\n") + } + builder.WriteString("\n") +} + +func writeHelmAggregateComponents(builder *strings.Builder, components []helmAggregateComponent) { + builder.WriteString("| Stack | Component | Status | Chart | Release | Namespace | Target | Duration |\n") + builder.WriteString("| --- | --- | --- | --- | --- | --- | --- | ---: |\n") + for i := range components { + component := &components[i] + values := []string{ + component.Result.Stack, + component.Result.Component, + component.Status, + component.Summary.Chart, + component.Summary.ReleaseName, + component.Summary.Namespace, + component.Summary.Target, + formatHelmAggregateDuration(component.Result.DurationMS), + } + builder.WriteString("| ") + for index, value := range values { + if index > 0 { + builder.WriteString(" | ") + } + builder.WriteString(helmMarkdownCell(value)) + } + builder.WriteString(" |\n") + } + builder.WriteString("\n") +} + +func writeHelmAggregateDetails(builder *strings.Builder, command string, components []helmAggregateComponent) { + for i := range components { + component := &components[i] + if component.Status != "failed" && !(command == "plan" && component.Status == "changed") { + continue + } + var detail strings.Builder + detail.WriteString("
") + detail.WriteString(helmMarkdownCell(component.Result.Stack + "/" + component.Result.Component + ": " + component.Status)) + detail.WriteString("\n\n") + if component.Status == "failed" { + detail.WriteString("```text\n") + detail.WriteString(plugin.TruncateDetail(component.Result.Error)) + detail.WriteString("\n```\n") + } else { + detail.WriteString("```diff\n") + detail.WriteString(plugin.TruncateDetail(component.Summary.Diff)) + detail.WriteString("\n```\n") + } + detail.WriteString("\n
\n\n") + if builder.Len()+detail.Len() > helmAggregateMarkdownMaxBytes { + builder.WriteString("> [!WARNING]\n> Additional component details were omitted to stay below GitHub Actions' job summary limit.\n") + return + } + builder.WriteString(detail.String()) + } +} + +func formatHelmAggregateDuration(milliseconds int64) string { + if milliseconds <= 0 { + return "-" + } + return strconv.FormatInt(milliseconds, 10) + "ms" +} + +func helmMarkdownCell(value string) string { + value = strings.ReplaceAll(value, "|", "\\|") + value = strings.ReplaceAll(value, "\r", " ") + return strings.ReplaceAll(value, "\n", " ") +} + +func enforceHelmAggregateMarkdownLimit(markdown string) string { + if len(markdown) <= helmAggregateMarkdownMaxBytes { + return markdown + } + notice := "\n\n> [!WARNING]\n> Summary truncated to stay below GitHub Actions' job summary limit.\n" + limit := helmAggregateMarkdownMaxBytes - len(notice) + return trimHelmAggregateMarkdownToLimit(markdown, limit) + notice +} + +func trimHelmAggregateMarkdownToLimit(markdown string, maxBytes int) string { + if maxBytes <= 0 { + return "" + } + if len(markdown) <= maxBytes { + return markdown + } + + end := maxBytes + if lineEnd := strings.LastIndexByte(markdown[:maxBytes], '\n'); lineEnd > 0 { + end = lineEnd + } + for end > 0 && !utf8.ValidString(markdown[:end]) { + end-- + } + return strings.TrimRight(markdown[:end], "\r\n") +} diff --git a/pkg/ci/plugins/helm/aggregate_test.go b/pkg/ci/plugins/helm/aggregate_test.go new file mode 100644 index 0000000000..555d4500a5 --- /dev/null +++ b/pkg/ci/plugins/helm/aggregate_test.go @@ -0,0 +1,130 @@ +package helm + +import ( + "errors" + "strings" + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/ci/internal/plugin" + "github.com/cloudposse/atmos/pkg/schema" +) + +func TestOnAfterAggregateRendersPlanSummary(t *testing.T) { + writer := &fakeWriter{} + ctx := &plugin.HookContext{ + Provider: fakeProvider{writer: writer}, + Aggregate: schema.HelmCIResultSet{ + Command: "plan", + Results: []schema.HelmCIResult{ + { + NodeID: "changed", Stack: "prod", Component: "api", Processed: true, DurationMS: 25, + Summary: map[string]any{ + "chart": "oci://registry/api", "release_name": "api", "namespace": "apps", + "target": "kubernetes", "diff": "+ kind: Deployment", + }, + }, + { + NodeID: "unchanged", Stack: "dev", Component: "web", Processed: true, + Summary: map[string]any{"chart": "web", "release_name": "web", "namespace": "apps", "target": "git"}, + }, + { + NodeID: "failed", Stack: "dev", Component: "db", ExitCode: 1, Error: "render failed", + Summary: map[string]any{"chart": "db"}, + }, + {NodeID: "skipped", Stack: "prod", Component: "worker"}, + }, + }, + } + + err := (&Plugin{}).onAfterAggregate(ctx) + require.NoError(t, err) + assert.Contains(t, writer.summary, "## Helm Plan Summary") + assert.Contains(t, writer.summary, "Processed 4 component(s): 1 changed, 1 unchanged, 1 failed, 1 skipped.") + assert.Contains(t, writer.summary, "| dev | db | failed | db |") + assert.Contains(t, writer.summary, "| dev | web | no changes | web | web | apps | git | - |") + assert.Contains(t, writer.summary, "| prod | api | changed | oci://registry/api | api | apps | kubernetes | 25ms |") + assert.Contains(t, writer.summary, "+ kind: Deployment") + assert.Contains(t, writer.summary, "render failed") + assert.Less(t, strings.Index(writer.summary, "| dev | db |"), strings.Index(writer.summary, "| prod | api |")) +} + +func TestOnAfterAggregateRendersApplySummary(t *testing.T) { + writer := &fakeWriter{} + err := (&Plugin{}).onAfterAggregate(&plugin.HookContext{ + Provider: fakeProvider{writer: writer}, + Aggregate: &schema.HelmCIResultSet{ + Command: "deploy", + Results: []schema.HelmCIResult{{ + Stack: "dev", Component: "api", Processed: true, + Summary: map[string]any{ + "chart": "api", "release_name": "api", "namespace": "apps", "target": "kubernetes", + }, + }}, + }, + }) + require.NoError(t, err) + assert.Contains(t, writer.summary, "## Helm Apply Summary") + assert.Contains(t, writer.summary, "Processed 1 component(s): 1 succeeded, 0 failed, 0 skipped.") + assert.Contains(t, writer.summary, "| dev | api | succeeded | api | api | apps | kubernetes | - |") +} + +func TestOnAfterAggregateSkipsInvalidOrDisabledAndReturnsWriterError(t *testing.T) { + pluginUnderTest := &Plugin{} + require.NoError(t, pluginUnderTest.onAfterAggregate(&plugin.HookContext{Provider: fakeProvider{}, Aggregate: "invalid"})) + require.NoError(t, pluginUnderTest.onAfterAggregate(&plugin.HookContext{ + Provider: fakeProvider{}, + Aggregate: schema.HelmCIResultSet{}, + })) + + disabled := false + writer := &fakeWriter{} + require.NoError(t, pluginUnderTest.onAfterAggregate(&plugin.HookContext{ + Config: &schema.AtmosConfiguration{CI: schema.CIConfig{Summary: schema.CISummaryConfig{Enabled: &disabled}}}, + Provider: fakeProvider{writer: writer}, + Aggregate: schema.HelmCIResultSet{Results: []schema.HelmCIResult{{Processed: true}}}, + })) + assert.Empty(t, writer.summary) + + sentinel := errors.New("write failed") + err := pluginUnderTest.onAfterAggregate(&plugin.HookContext{ + Provider: fakeProvider{writer: &fakeWriter{err: sentinel}}, + Aggregate: schema.HelmCIResultSet{Results: []schema.HelmCIResult{{Processed: true}}}, + }) + require.ErrorIs(t, err, sentinel) +} + +func TestHelmAggregateHelpers(t *testing.T) { + resultSet := schema.HelmCIResultSet{Command: "diff"} + assert.Equal(t, resultSet, mustNormalizeHelmAggregate(t, resultSet)) + assert.Equal(t, resultSet, mustNormalizeHelmAggregate(t, &resultSet)) + _, ok := normalizeHelmAggregate((*schema.HelmCIResultSet)(nil)) + assert.False(t, ok) + _, ok = normalizeHelmAggregate("invalid") + assert.False(t, ok) + + assert.Equal(t, "plan", normalizeHelmAggregateCommand("diff")) + assert.Equal(t, "apply", normalizeHelmAggregateCommand("deploy")) + assert.Equal(t, "-", formatHelmAggregateDuration(0)) + assert.Equal(t, "15ms", formatHelmAggregateDuration(15)) + assert.Equal(t, `a\|b c`, helmMarkdownCell("a|b\nc")) + + oversized := strings.Repeat("x", helmAggregateMarkdownMaxBytes+100) + truncated := enforceHelmAggregateMarkdownLimit(oversized) + assert.LessOrEqual(t, len(truncated), helmAggregateMarkdownMaxBytes) + assert.Contains(t, truncated, "Summary truncated") + + unicodeOversized := strings.Repeat("界", helmAggregateMarkdownMaxBytes) + unicodeTruncated := enforceHelmAggregateMarkdownLimit(unicodeOversized) + assert.True(t, utf8.ValidString(unicodeTruncated)) +} + +func mustNormalizeHelmAggregate(t *testing.T, value any) schema.HelmCIResultSet { + t.Helper() + result, ok := normalizeHelmAggregate(value) + require.True(t, ok) + return result +} diff --git a/pkg/ci/plugins/helm/plugin.go b/pkg/ci/plugins/helm/plugin.go index 123a065491..df1091468f 100644 --- a/pkg/ci/plugins/helm/plugin.go +++ b/pkg/ci/plugins/helm/plugin.go @@ -40,8 +40,10 @@ func (p *Plugin) GetHookBindings() []plugin.HookBinding { return []plugin.HookBinding{ {Event: "after.helm.template", Handler: p.onAfterOperation}, {Event: "after.helm.diff", Handler: p.onAfterOperation}, + {Event: "after.helm.plan.aggregate", Handler: p.onAfterAggregate}, {Event: "after.helm.apply", Handler: p.onAfterOperation}, {Event: "after.helm.deploy", Handler: p.onAfterOperation}, + {Event: "after.helm.apply.aggregate", Handler: p.onAfterAggregate}, {Event: "after.helm.delete", Handler: p.onAfterOperation}, } } diff --git a/pkg/ci/plugins/helm/plugin_test.go b/pkg/ci/plugins/helm/plugin_test.go index 899b9217b7..8410f69ef3 100644 --- a/pkg/ci/plugins/helm/plugin_test.go +++ b/pkg/ci/plugins/helm/plugin_test.go @@ -20,13 +20,15 @@ func TestPlugin_GetType(t *testing.T) { func TestPlugin_GetHookBindings(t *testing.T) { bindings := (&Plugin{}).GetHookBindings() - require.Len(t, bindings, 5) + require.Len(t, bindings, 7) for _, event := range []string{ "after.helm.template", "after.helm.diff", + "after.helm.plan.aggregate", "after.helm.apply", "after.helm.deploy", + "after.helm.apply.aggregate", "after.helm.delete", } { t.Run(event, func(t *testing.T) { diff --git a/pkg/component/helm/aggregate_ci.go b/pkg/component/helm/aggregate_ci.go new file mode 100644 index 0000000000..3b3a0b8234 --- /dev/null +++ b/pkg/component/helm/aggregate_ci.go @@ -0,0 +1,172 @@ +package helm + +import ( + "sort" + "sync" + "time" + + errUtils "github.com/cloudposse/atmos/errors" + "github.com/cloudposse/atmos/pkg/component" + "github.com/cloudposse/atmos/pkg/hooks" + log "github.com/cloudposse/atmos/pkg/logger" + "github.com/cloudposse/atmos/pkg/schema" +) + +const helmBulkCICollectorFlag = "_helm_bulk_ci_collector" + +// helmBulkCICollector accumulates one result per graph node so bulk Helm +// commands can write a single CI summary after execution completes. +type helmBulkCICollector struct { + mu sync.Mutex + command string + results map[string]*schema.HelmCIResult +} + +func newHelmBulkCICollector(command string) *helmBulkCICollector { + return &helmBulkCICollector{ + command: command, + results: make(map[string]*schema.HelmCIResult), + } +} + +// bulkCollectingProvider wraps the native Helm provider and records failures +// that happen before an operation produces its structured summary. +type bulkCollectingProvider struct { + *ComponentProvider + collector *helmBulkCICollector +} + +func (p *bulkCollectingProvider) Execute(ctx *component.ExecutionContext) error { + startedAt := time.Now() + err := p.ComponentProvider.Execute(ctx) + p.collector.finish(ctx, startedAt, time.Now(), err) + return err +} + +func helmBulkCollector(ctx *component.ExecutionContext) *helmBulkCICollector { + if ctx == nil { + return nil + } + collector, _ := ctx.Flags[helmBulkCICollectorFlag].(*helmBulkCICollector) + return collector +} + +func (c *helmBulkCICollector) setSummary(info *schema.ConfigAndStacksInfo, summary map[string]any, operationErr error) { + if c == nil || info == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + + result := c.ensure(info.Stack, info.ComponentFromArg) + result.Processed = true + result.Summary = cloneHelmSummary(summary) + applyHelmResultError(result, operationErr) +} + +func (c *helmBulkCICollector) finish(ctx *component.ExecutionContext, startedAt, finishedAt time.Time, execErr error) { + if c == nil || ctx == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + + result := c.ensure(ctx.Stack, ctx.Component) + result.StartedAt = startedAt + result.FinishedAt = finishedAt + result.DurationMS = finishedAt.Sub(startedAt).Milliseconds() + applyHelmResultError(result, execErr) +} + +func (c *helmBulkCICollector) ensure(stack, componentName string) *schema.HelmCIResult { + nodeID := component.GraphNodeID(componentName, stack) + if result, ok := c.results[nodeID]; ok { + return result + } + result := &schema.HelmCIResult{ + NodeID: nodeID, + Stack: stack, + Component: componentName, + Status: "succeeded", + } + c.results[nodeID] = result + return result +} + +func (c *helmBulkCICollector) resultSet() schema.HelmCIResultSet { + if c == nil { + return schema.HelmCIResultSet{} + } + c.mu.Lock() + defer c.mu.Unlock() + + results := make([]schema.HelmCIResult, 0, len(c.results)) + for _, result := range c.results { + copyResult := *result + copyResult.Summary = cloneHelmSummary(result.Summary) + results = append(results, copyResult) + } + sort.SliceStable(results, func(i, j int) bool { + if results[i].Stack != results[j].Stack { + return results[i].Stack < results[j].Stack + } + if results[i].Component != results[j].Component { + return results[i].Component < results[j].Component + } + return results[i].NodeID < results[j].NodeID + }) + return schema.HelmCIResultSet{Command: c.command, Results: results} +} + +func applyHelmResultError(result *schema.HelmCIResult, err error) { + if result == nil || err == nil { + return + } + result.Status = "failed" + result.ExitCode = errUtils.GetExitCode(err) + result.Error = err.Error() +} + +func cloneHelmSummary(summary map[string]any) map[string]any { + if summary == nil { + return nil + } + cloned := make(map[string]any, len(summary)) + for key, value := range summary { + cloned[key] = value + } + return cloned +} + +func supportsHelmAggregateCI(command string) bool { + switch command { + case "plan", "diff", "apply", "deploy": + return true + default: + return false + } +} + +func runHelmAggregateCIHook( + ctx *component.ExecutionContext, + atmosConfig *schema.AtmosConfiguration, + info *schema.ConfigAndStacksInfo, + resultSet schema.HelmCIResultSet, + commandErr error, +) { + event := hooks.AfterHelmPlanAggregate + if resultSet.Command == "apply" || resultSet.Command == "deploy" { + event = hooks.AfterHelmApplyAggregate + } + if err := runCIHooks(&hooks.RunCIHooksOptions{ + Event: event, + AtmosConfig: atmosConfig, + Info: info, + ForceCIMode: helmCIModeEnabled(ctx.Flags), + CommandError: commandErr, + ExitCode: errUtils.GetExitCode(commandErr), + Aggregate: resultSet, + }); err != nil { + log.Warn("Helm CI aggregate hook failed", "command", resultSet.Command, "error", err) + } +} diff --git a/pkg/component/helm/aggregate_ci_test.go b/pkg/component/helm/aggregate_ci_test.go new file mode 100644 index 0000000000..37d2af7214 --- /dev/null +++ b/pkg/component/helm/aggregate_ci_test.go @@ -0,0 +1,143 @@ +package helm + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/cloudposse/atmos/pkg/auth" + "github.com/cloudposse/atmos/pkg/component" + "github.com/cloudposse/atmos/pkg/hooks" + "github.com/cloudposse/atmos/pkg/schema" +) + +func TestHelmBulkCICollectorRecordsAndSortsResults(t *testing.T) { + collector := newHelmBulkCICollector("plan") + collector.setSummary(&schema.ConfigAndStacksInfo{Stack: "prod", ComponentFromArg: "api"}, map[string]any{"diff": "+ change"}, nil) + collector.finish(&component.ExecutionContext{Stack: "prod", Component: "api"}, time.Unix(1, 0), time.Unix(1, int64(25*time.Millisecond)), nil) + + sentinel := errors.New("render failed") + collector.finish(&component.ExecutionContext{Stack: "dev", Component: "db"}, time.Unix(2, 0), time.Unix(2, int64(time.Millisecond)), sentinel) + + resultSet := collector.resultSet() + assert.Equal(t, "plan", resultSet.Command) + require.Len(t, resultSet.Results, 2) + assert.Equal(t, "db", resultSet.Results[0].Component) + assert.False(t, resultSet.Results[0].Processed) + assert.Equal(t, "failed", resultSet.Results[0].Status) + assert.Equal(t, "render failed", resultSet.Results[0].Error) + assert.Equal(t, "api", resultSet.Results[1].Component) + assert.True(t, resultSet.Results[1].Processed) + assert.Equal(t, int64(25), resultSet.Results[1].DurationMS) + assert.Equal(t, "+ change", resultSet.Results[1].Summary["diff"]) + + resultSet.Results[1].Summary["diff"] = "mutated" + assert.Equal(t, "+ change", collector.resultSet().Results[1].Summary["diff"]) +} + +func TestRunWithHooksBulkCollectorSuppressesPerComponentCI(t *testing.T) { + originalHooks := getHooks + originalApply := applyHelmRelease + originalCI := runCIHooks + t.Cleanup(func() { + getHooks = originalHooks + applyHelmRelease = originalApply + runCIHooks = originalCI + }) + + getHooks = func(*schema.AtmosConfiguration, *schema.ConfigAndStacksInfo) (*hooks.Hooks, error) { + return &hooks.Hooks{}, nil + } + applyHelmRelease = func(context.Context, *chartSpec, bool) (releaseActionResult, error) { + return releaseActionResult{Manifest: helmExecutorManifest, Operation: releaseOperationInstall}, nil + } + ciCalls := 0 + runCIHooks = func(*hooks.RunCIHooksOptions) error { + ciCalls++ + return nil + } + + collector := newHelmBulkCICollector("apply") + ctx := &component.ExecutionContext{Flags: map[string]any{helmBulkCICollectorFlag: collector}} + info := &schema.ConfigAndStacksInfo{ + Stack: "dev", ComponentFromArg: "api", SubCommand: "apply", + ComponentSection: map[string]any{"chart": "api", "name": "api"}, + } + require.NoError(t, runWithHooks(ctx, &schema.AtmosConfiguration{}, info, OperationApply, "")) + assert.Zero(t, ciCalls) + results := collector.resultSet().Results + require.Len(t, results, 1) + assert.True(t, results[0].Processed) + assert.Equal(t, "api", results[0].Summary["chart"]) +} + +func TestExecuteBulkEmitsOneAggregateCIHook(t *testing.T) { + originalDescribe := executeDescribeStacks + originalGraph := executeGraph + originalCI := runCIHooks + t.Cleanup(func() { + executeDescribeStacks = originalDescribe + executeGraph = originalGraph + runCIHooks = originalCI + }) + + executeDescribeStacks = func( + *schema.AtmosConfiguration, + string, + []string, + []string, + []string, + bool, + bool, + bool, + bool, + []string, + auth.AuthManager, + ) (map[string]any, error) { + return map[string]any{"dev": map[string]any{}}, nil + } + sentinel := errors.New("graph failed") + executeGraph = func(_ context.Context, opts *component.GraphExecutionOptions) error { + assert.Equal(t, "plan", opts.SubCommand) + _, wrapped := opts.Provider.(*bulkCollectingProvider) + assert.True(t, wrapped) + collector, ok := opts.Flags[helmBulkCICollectorFlag].(*helmBulkCICollector) + require.True(t, ok) + collector.setSummary(&schema.ConfigAndStacksInfo{Stack: "dev", ComponentFromArg: "api"}, map[string]any{"diff": "+ change"}, nil) + collector.finish(&component.ExecutionContext{Stack: "dev", Component: "api"}, time.Now(), time.Now(), sentinel) + return sentinel + } + + var captured *hooks.RunCIHooksOptions + runCIHooks = func(opts *hooks.RunCIHooksOptions) error { + captured = opts + return nil + } + + ctx := &component.ExecutionContext{Flags: map[string]any{"ci": true}} + info := &schema.ConfigAndStacksInfo{All: true, SubCommand: "plan"} + require.ErrorIs(t, executeBulk(ctx, &schema.AtmosConfiguration{}, info, OperationDiff), sentinel) + require.NotNil(t, captured) + assert.Equal(t, hooks.AfterHelmPlanAggregate, captured.Event) + assert.True(t, captured.ForceCIMode) + assert.ErrorIs(t, captured.CommandError, sentinel) + resultSet, ok := captured.Aggregate.(schema.HelmCIResultSet) + require.True(t, ok) + assert.Equal(t, "plan", resultSet.Command) + require.Len(t, resultSet.Results, 1) + assert.Equal(t, "graph failed", resultSet.Results[0].Error) + _, retained := ctx.Flags[helmBulkCICollectorFlag] + assert.False(t, retained) +} + +func TestHelmAggregateCIHelpers(t *testing.T) { + assert.True(t, supportsHelmAggregateCI("plan")) + assert.True(t, supportsHelmAggregateCI("apply")) + assert.False(t, supportsHelmAggregateCI("template")) + assert.Nil(t, helmBulkCollector(nil)) + assert.Nil(t, helmBulkCollector(&component.ExecutionContext{})) +} diff --git a/pkg/component/helm/executor.go b/pkg/component/helm/executor.go index eef811c1a2..e11656a6b7 100644 --- a/pkg/component/helm/executor.go +++ b/pkg/component/helm/executor.go @@ -191,14 +191,18 @@ func runWithHooks( } summary, opErr := runOperation(ctx, atmosConfig, info, operation, spec) - runHelmCIHook(helmCIHookParams{ - ctx: ctx, - atmosConfig: atmosConfig, - info: info, - event: after, - summary: summary, - commandErr: opErr, - }) + if collector := helmBulkCollector(ctx); collector != nil { + collector.setSummary(info, summary, opErr) + } else { + runHelmCIHook(helmCIHookParams{ + ctx: ctx, + atmosConfig: atmosConfig, + info: info, + event: after, + summary: summary, + commandErr: opErr, + }) + } if opErr != nil { return opErr } diff --git a/pkg/component/helm/executor_bulk.go b/pkg/component/helm/executor_bulk.go index f6da35e89b..ab2d164fea 100644 --- a/pkg/component/helm/executor_bulk.go +++ b/pkg/component/helm/executor_bulk.go @@ -42,16 +42,36 @@ func executeBulk( return err } - return executeGraph(ctx.GoContext(), &component.GraphExecutionOptions{ - Provider: &ComponentProvider{}, + provider := component.ComponentProvider(&ComponentProvider{}) + command := info.SubCommand + if command == "" { + command = string(operation) + } + var collector *helmBulkCICollector + if supportsHelmAggregateCI(command) { + collector = newHelmBulkCICollector(command) + if ctx.Flags == nil { + ctx.Flags = make(map[string]any) + } + ctx.Flags[helmBulkCICollectorFlag] = collector + defer delete(ctx.Flags, helmBulkCICollectorFlag) + provider = &bulkCollectingProvider{ComponentProvider: &ComponentProvider{}, collector: collector} + } + + graphErr := executeGraph(ctx.GoContext(), &component.GraphExecutionOptions{ + Provider: provider, AtmosConfig: atmosConfig, Info: info, Stacks: stacks, ComponentType: cfg.HelmComponentType, - SubCommand: string(operation), + SubCommand: command, Flags: ctx.Flags, Selection: selection, }) + if collector != nil { + runHelmAggregateCIHook(ctx, atmosConfig, info, collector.resultSet(), graphErr) + } + return graphErr } func authManagerForBulk(atmosConfig *schema.AtmosConfiguration, info *schema.ConfigAndStacksInfo) (auth.AuthManager, error) { diff --git a/pkg/hooks/event.go b/pkg/hooks/event.go index fee50582db..9d1645c4eb 100644 --- a/pkg/hooks/event.go +++ b/pkg/hooks/event.go @@ -40,8 +40,10 @@ const ( AfterHelmTemplate HookEvent = "after.helm.template" BeforeHelmDiff HookEvent = "before.helm.diff" AfterHelmDiff HookEvent = "after.helm.diff" + AfterHelmPlanAggregate HookEvent = "after.helm.plan.aggregate" BeforeHelmApply HookEvent = "before.helm.apply" AfterHelmApply HookEvent = "after.helm.apply" + AfterHelmApplyAggregate HookEvent = "after.helm.apply.aggregate" BeforeHelmDeploy HookEvent = "before.helm.deploy" AfterHelmDeploy HookEvent = "after.helm.deploy" BeforeHelmDelete HookEvent = "before.helm.delete" diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index 3bbbb1aae2..98cb0f3795 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -900,6 +900,29 @@ type TerraformPlanCIResult struct { Error string } +// HelmCIResultSet contains deterministic per-node Helm results for one +// graph-backed plan or apply run. +type HelmCIResultSet struct { + Command string + Results []HelmCIResult +} + +// HelmCIResult contains the execution outcome and structured Helm summary for +// one component in a graph-backed run. +type HelmCIResult struct { + NodeID string + Stack string + Component string + Status string + Processed bool + ExitCode int + Summary map[string]any + StartedAt time.Time + FinishedAt time.Time + DurationMS int64 + Error string +} + // KubernetesCIResult contains the compact result data rendered into native CI // job summaries for one Kubernetes component command. type KubernetesCIResult struct { diff --git a/website/docs/ci/job-summaries.mdx b/website/docs/ci/job-summaries.mdx index 97c1cd4580..f2bfd16fef 100644 --- a/website/docs/ci/job-summaries.mdx +++ b/website/docs/ci/job-summaries.mdx @@ -124,12 +124,20 @@ comments, or artifacts. The summary includes component, stack, command status, a command, and Helm metadata such as release name, namespace, chart, target, object counts, object kinds, and rendered manifest size when available. -For cluster-backed apply/deploy/delete operations, the aggregate also includes -an operation-specific `lifecycle` block with the effective wait strategy, -timeout, chart-hook state, and applicable rollback, Job-wait, CRD, cleanup, and -history values. For external delivery, apply/deploy reports `applied: false` -while delete reports `deleted: false`; both include the selected target kind and -`reason: external_target` instead of presenting stored release policy as active. +When `plan` or `apply` selects multiple components with `--all` or `--affected`, Atmos writes one +deterministic aggregate summary after the dependency-graph run completes instead of writing a +separate summary from every component. The aggregate includes result counts and a stable +per-component table with stack, component, chart, release, namespace, target, status, and duration. +Plan summaries distinguish changed and unchanged components and include collapsible diffs; failed +components include their error details. The same aggregate behavior applies to the `diff` and +`deploy` aliases and composes with tag and label filters. + +For a single cluster-backed apply/deploy/delete operation, the component summary also includes an +operation-specific `lifecycle` block with the effective wait strategy, timeout, chart-hook state, +and applicable rollback, Job-wait, CRD, cleanup, and history values. For external delivery, +apply/deploy reports `applied: false` while delete reports `deleted: false`; both include the +selected target kind and `reason: external_target` instead of presenting stored release policy as +active. ## Helmfile Summaries diff --git a/website/docs/cli/commands/helm/helm-apply.mdx b/website/docs/cli/commands/helm/helm-apply.mdx index ea6df0b0f5..356e2101cc 100644 --- a/website/docs/cli/commands/helm/helm-apply.mdx +++ b/website/docs/cli/commands/helm/helm-apply.mdx @@ -58,6 +58,10 @@ atmos helm apply --all --tags production,tier-1 atmos helm apply --affected --labels cost-center=platform ``` +In CI, bulk applies selected with `--all` or `--affected` write one aggregate GitHub job summary +after the dependency graph completes. It lists component status and Helm release metadata in stable +stack/component order and includes details for failures. + ## Flags
diff --git a/website/docs/cli/commands/helm/helm-plan.mdx b/website/docs/cli/commands/helm/helm-plan.mdx index 48975a5509..1badae913d 100644 --- a/website/docs/cli/commands/helm/helm-plan.mdx +++ b/website/docs/cli/commands/helm/helm-plan.mdx @@ -31,6 +31,11 @@ atmos helm plan monitoring -s plat-ue2-dev --against=target atmos helm plan --all --tags production ``` +In CI, bulk plans selected with `--all` or `--affected` write one aggregate GitHub job summary +after the dependency graph completes. It lists every attempted component in stable stack/component +order, distinguishes changed, unchanged, and failed results, and includes collapsible diffs for +changed components. + ## Flags
From 2014df0e101541f935030c5f3592ae864e73e027 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Mon, 3 Aug 2026 14:51:33 +0400 Subject: [PATCH 43/62] test: validate Helm DAG deployment readiness --- examples/helm/atmos.yaml | 6 +++--- .../demo/templates/dependency-observed.yaml | 18 +++++++++++++----- .../helm/demo/templates/ready-marker.yaml | 14 -------------- examples/helm/components/helm/demo/values.yaml | 3 +-- examples/helm/stacks/deploy/dev.yaml | 4 +--- 5 files changed, 18 insertions(+), 27 deletions(-) delete mode 100644 examples/helm/components/helm/demo/templates/ready-marker.yaml diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index d337f3d933..98999aa1f8 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -154,13 +154,13 @@ commands: - command: >- if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-upgrade-only 2>&1); then echo "cleanup_on_fail left the upgrade-only ConfigMap"; exit 1; else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify cleanup_on_fail removal: $output"; exit 1;; esac; fi - # Dependency execution gates the dependent's pre-install hook on the - # foundation's post-readiness marker. + # Dependency execution gates the dependent render on the foundation's + # release-managed Deployment readiness state. - atmos helm apply --all -s dev --identity local-k3s --tags lifecycle-dag - atmos emulator exec kubernetes -s dev -- kubectl -n lifecycle-dag get deployment dag-dependent - command: >- observed=$(atmos emulator exec kubernetes -s dev -- kubectl -n lifecycle-dag get configmap dag-dependent-dependency-observed -o jsonpath='{.data.ready}'); - if [ "$observed" != "true" ]; then echo "dependent did not observe the ready foundation marker: ready=$observed"; exit 1; fi + if [ "$observed" != "true" ]; then echo "dependent did not observe the ready foundation Deployment: ready=$observed"; exit 1; fi # Delete dry-run must leave the deployed release and resources intact. - atmos helm delete demo -s dev --identity local-k3s --dry-run --wait=watcher --timeout=2m - atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo diff --git a/examples/helm/components/helm/demo/templates/dependency-observed.yaml b/examples/helm/components/helm/demo/templates/dependency-observed.yaml index 1eb9053c3b..4a6be62d2e 100644 --- a/examples/helm/components/helm/demo/templates/dependency-observed.yaml +++ b/examples/helm/components/helm/demo/templates/dependency-observed.yaml @@ -1,7 +1,14 @@ -{{- if and .Values.hooks.requiredConfigMap .Values.hooks.validateDependency }} -{{- $required := lookup "v1" "ConfigMap" .Release.Namespace .Values.hooks.requiredConfigMap }} +{{- if and .Values.hooks.requiredDeployment .Values.hooks.validateDependency }} +{{- $required := lookup "apps/v1" "Deployment" .Release.Namespace .Values.hooks.requiredDeployment }} {{- if not $required }} -{{- fail (printf "required dependency marker ConfigMap %q is not ready" .Values.hooks.requiredConfigMap) }} +{{- fail (printf "required dependency Deployment %q is not ready" .Values.hooks.requiredDeployment) }} +{{- end }} +{{- $desired := int (dig "spec" "replicas" 1 $required) }} +{{- $available := int (dig "status" "availableReplicas" 0 $required) }} +{{- $generation := int64 (dig "metadata" "generation" 0 $required) }} +{{- $observedGeneration := int64 (dig "status" "observedGeneration" 0 $required) }} +{{- if or (lt $available $desired) (lt $observedGeneration $generation) }} +{{- fail (printf "required dependency Deployment %q is not ready: available=%d desired=%d observedGeneration=%d generation=%d" .Values.hooks.requiredDeployment $available $desired $observedGeneration $generation) }} {{- end }} apiVersion: v1 kind: ConfigMap @@ -9,6 +16,7 @@ metadata: name: {{ .Release.Name }}-dependency-observed namespace: {{ .Release.Namespace }} data: - requiredConfigMap: {{ .Values.hooks.requiredConfigMap | quote }} - ready: {{ index $required.data "ready" | quote }} + requiredDeployment: {{ .Values.hooks.requiredDeployment | quote }} + ready: "true" + availableReplicas: {{ $available | quote }} {{- end }} diff --git a/examples/helm/components/helm/demo/templates/ready-marker.yaml b/examples/helm/components/helm/demo/templates/ready-marker.yaml deleted file mode 100644 index 0c5a7bf4c9..0000000000 --- a/examples/helm/components/helm/demo/templates/ready-marker.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if .Values.hooks.readyMarker }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-ready - namespace: {{ .Release.Namespace }} - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-weight": "0" - "helm.sh/resource-policy": keep -data: - ready: "true" - order: "foundation-ready" -{{- end }} diff --git a/examples/helm/components/helm/demo/values.yaml b/examples/helm/components/helm/demo/values.yaml index 96e5a2aa83..3f7b52c0b5 100644 --- a/examples/helm/components/helm/demo/values.yaml +++ b/examples/helm/components/helm/demo/values.yaml @@ -18,9 +18,8 @@ job: hooks: enabled: true fail: false - requiredConfigMap: "" + requiredDeployment: "" validateDependency: false - readyMarker: false extraConfigMap: enabled: false tpl: diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index 2911a323e7..79de711e18 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -145,8 +145,6 @@ components: values: deployment: readinessDelaySeconds: 3 - hooks: - readyMarker: true dag-dependent: metadata: @@ -161,7 +159,7 @@ components: wait_strategy: legacy values: hooks: - requiredConfigMap: dag-foundation-ready + requiredDeployment: dag-foundation # The lifecycle DAG integration test explicitly opts into a live # cluster lookup; ordinary offline chart rendering leaves this off. validateDependency: true From 571e8538b89c65a7b8aff2a08488c593388834f0 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Mon, 3 Aug 2026 15:34:05 +0400 Subject: [PATCH 44/62] refactor: extract Helm CI collector setup --- pkg/component/helm/executor_bulk.go | 30 ++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/pkg/component/helm/executor_bulk.go b/pkg/component/helm/executor_bulk.go index ab2d164fea..fcac97b456 100644 --- a/pkg/component/helm/executor_bulk.go +++ b/pkg/component/helm/executor_bulk.go @@ -42,21 +42,12 @@ func executeBulk( return err } - provider := component.ComponentProvider(&ComponentProvider{}) command := info.SubCommand if command == "" { command = string(operation) } - var collector *helmBulkCICollector - if supportsHelmAggregateCI(command) { - collector = newHelmBulkCICollector(command) - if ctx.Flags == nil { - ctx.Flags = make(map[string]any) - } - ctx.Flags[helmBulkCICollectorFlag] = collector - defer delete(ctx.Flags, helmBulkCICollectorFlag) - provider = &bulkCollectingProvider{ComponentProvider: &ComponentProvider{}, collector: collector} - } + provider, collector, cleanup := setupHelmBulkAggregateCollector(ctx, command) + defer cleanup() graphErr := executeGraph(ctx.GoContext(), &component.GraphExecutionOptions{ Provider: provider, @@ -74,6 +65,23 @@ func executeBulk( return graphErr } +func setupHelmBulkAggregateCollector( + ctx *component.ExecutionContext, + command string, +) (component.ComponentProvider, *helmBulkCICollector, func()) { + if !supportsHelmAggregateCI(command) { + return &ComponentProvider{}, nil, func() {} + } + + collector := newHelmBulkCICollector(command) + if ctx.Flags == nil { + ctx.Flags = make(map[string]any) + } + ctx.Flags[helmBulkCICollectorFlag] = collector + provider := &bulkCollectingProvider{ComponentProvider: &ComponentProvider{}, collector: collector} + return provider, collector, func() { delete(ctx.Flags, helmBulkCICollectorFlag) } +} + func authManagerForBulk(atmosConfig *schema.AtmosConfiguration, info *schema.ConfigAndStacksInfo) (auth.AuthManager, error) { if info.Identity == "" { return nil, nil From b6b7c9023a68928e67afd03c709d0cc0092988ee Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Mon, 3 Aug 2026 16:06:19 +0400 Subject: [PATCH 45/62] fix: harden Helm aggregate failure reporting --- examples/helm/atmos.yaml | 6 ++-- pkg/ci/plugins/helm/aggregate.go | 32 ++++++++++++++++++++- pkg/ci/plugins/helm/aggregate_test.go | 14 +++++++++ pkg/ci/plugins/helm/plugin_test.go | 6 ++++ pkg/component/helm/client_lifecycle_test.go | 5 ++-- 5 files changed, 57 insertions(+), 6 deletions(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 98999aa1f8..be225190d7 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -120,7 +120,7 @@ commands: # hookOnly returns after hooks without waiting for the Deployment's explicit readiness gate. - atmos helm apply demo-hook-only -s dev --identity local-k3s - command: >- - available=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only get deployment demo-hook-only -o jsonpath='{.status.conditions[?(@.type=="Available")].status}'); + if ! available=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only get deployment demo-hook-only -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>&1); then echo "failed to query Deployment/demo-hook-only availability: $available"; exit 1; fi; if [ "$available" = "True" ]; then echo "hookOnly unexpectedly satisfied the Deployment readiness gate"; exit 1; fi - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only patch deployment demo-hook-only --type=json -p='[{"op":"remove","path":"/spec/template/spec/readinessGates"}]' - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only rollout status deployment/demo-hook-only --timeout=2m @@ -145,11 +145,11 @@ commands: # A failed upgrade restores the successful demo release and cleanup removes # the resource introduced only by the failed revision. - command: >- - before=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}'); + if ! before=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}' 2>&1); then echo "failed to capture Deployment/demo state before upgrade: $before"; exit 1; fi; if output=$(NO_COLOR=1 atmos helm apply demo-upgrade-fail -s dev --identity local-k3s 2>&1); then echo "failed upgrade unexpectedly succeeded"; exit 1; fi; case "$output" in *"helm release operation failed"*) ;; *) echo "failed upgrade stopped outside the Helm lifecycle operation: $output"; exit 1;; esac; case "$output" in *"job demo-failing-hook failed"*"BackoffLimitExceeded"*) ;; *) echo "failed upgrade did not report the expected post-upgrade hook failure: $output"; exit 1;; esac; - after=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}'); + if ! after=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}' 2>&1); then echo "failed to capture Deployment/demo state after rollback: $after"; exit 1; fi; if [ "$before" != "$after" ]; then echo "rollback did not restore Deployment/demo: before=$before after=$after"; exit 1; fi - command: >- if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-upgrade-only 2>&1); then echo "cleanup_on_fail left the upgrade-only ConfigMap"; exit 1; diff --git a/pkg/ci/plugins/helm/aggregate.go b/pkg/ci/plugins/helm/aggregate.go index afdb714743..193eedaf4b 100644 --- a/pkg/ci/plugins/helm/aggregate.go +++ b/pkg/ci/plugins/helm/aggregate.go @@ -47,10 +47,28 @@ func (p *Plugin) onAfterAggregate(ctx *plugin.HookContext) error { defer perf.Track(ctx.Config, "helmci.Plugin.onAfterAggregate")() resultSet, ok := normalizeHelmAggregate(ctx.Aggregate) - if !ok || len(resultSet.Results) == 0 { + if !ok { log.Debug("Skipping aggregate Helm CI hook: no results") return nil } + if len(resultSet.Results) == 0 { + if ctx.CommandError == nil && ctx.ExitCode == 0 { + log.Debug("Skipping aggregate Helm CI hook: no results") + return nil + } + if !isSummaryEnabled(ctx.Config) { + return nil + } + writer := ctx.Provider.OutputWriter() + if writer == nil { + return nil + } + command := resultSet.Command + if command == "" { + command = ctx.Command + } + return writer.WriteSummary(renderHelmAggregateFailureMarkdown(command, ctx.CommandError, ctx.ExitCode)) + } if !isSummaryEnabled(ctx.Config) { return nil } @@ -62,6 +80,18 @@ func (p *Plugin) onAfterAggregate(ctx *plugin.HookContext) error { return writer.WriteSummary(renderHelmAggregateMarkdown(&aggregate)) } +func renderHelmAggregateFailureMarkdown(command string, commandErr error, exitCode int) string { + message := fmt.Sprintf("command exited with code %d", exitCode) + if commandErr != nil { + message = commandErr.Error() + } + return enforceHelmAggregateMarkdownLimit(fmt.Sprintf( + "## Helm %s Summary\n\nCommand failed before any components were processed.\n\n**Error:** %s\n", + helmAggregateCommandLabel(normalizeHelmAggregateCommand(command)), + helmMarkdownCell(message), + )) +} + func normalizeHelmAggregate(value any) (schema.HelmCIResultSet, bool) { switch typed := value.(type) { case schema.HelmCIResultSet: diff --git a/pkg/ci/plugins/helm/aggregate_test.go b/pkg/ci/plugins/helm/aggregate_test.go index 555d4500a5..f5f568116e 100644 --- a/pkg/ci/plugins/helm/aggregate_test.go +++ b/pkg/ci/plugins/helm/aggregate_test.go @@ -97,6 +97,20 @@ func TestOnAfterAggregateSkipsInvalidOrDisabledAndReturnsWriterError(t *testing. require.ErrorIs(t, err, sentinel) } +func TestOnAfterAggregateRendersFailureWithoutResults(t *testing.T) { + writer := &fakeWriter{} + err := (&Plugin{}).onAfterAggregate(&plugin.HookContext{ + Provider: fakeProvider{writer: writer}, + Aggregate: schema.HelmCIResultSet{Command: "apply"}, + CommandError: errors.New("dependency graph contains a cycle"), + ExitCode: 1, + }) + require.NoError(t, err) + assert.Contains(t, writer.summary, "## Helm Apply Summary") + assert.Contains(t, writer.summary, "Command failed before any components were processed.") + assert.Contains(t, writer.summary, "dependency graph contains a cycle") +} + func TestHelmAggregateHelpers(t *testing.T) { resultSet := schema.HelmCIResultSet{Command: "diff"} assert.Equal(t, resultSet, mustNormalizeHelmAggregate(t, resultSet)) diff --git a/pkg/ci/plugins/helm/plugin_test.go b/pkg/ci/plugins/helm/plugin_test.go index 8410f69ef3..2166d73ee1 100644 --- a/pkg/ci/plugins/helm/plugin_test.go +++ b/pkg/ci/plugins/helm/plugin_test.go @@ -290,6 +290,11 @@ func TestTemplateRendering(t *testing.T) { contains: []string{ "Helm Apply Summary", "Release lifecycle", "| Operation | `install` |", + "| Wait strategy | `hookOnly` |", + "| Timeout | `5m0s` |", + "| Chart hooks enabled | `true` |", + "| Wait for Jobs | `false` |", + "| Rollback on failure | `false` |", "| Install CRDs | `true` |", }, notContains: []string{"Cleanup on failure", "Maximum history"}, @@ -319,6 +324,7 @@ func TestTemplateRendering(t *testing.T) { }, contains: []string{ "Helm Delete Summary", "Release lifecycle", + "| Operation | `uninstall` |", "| Wait strategy | `legacy` |", "| Timeout | `10m0s` |", "| Chart hooks enabled | `false` |", diff --git a/pkg/component/helm/client_lifecycle_test.go b/pkg/component/helm/client_lifecycle_test.go index 67fb2c643b..0db7c6e4a5 100644 --- a/pkg/component/helm/client_lifecycle_test.go +++ b/pkg/component/helm/client_lifecycle_test.go @@ -7,8 +7,6 @@ import ( "testing" "time" - errUtils "github.com/cloudposse/atmos/errors" - cfg "github.com/cloudposse/atmos/pkg/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "helm.sh/helm/v4/pkg/action" @@ -20,6 +18,9 @@ import ( release "helm.sh/helm/v4/pkg/release/v1" "helm.sh/helm/v4/pkg/storage" "helm.sh/helm/v4/pkg/storage/driver" + + errUtils "github.com/cloudposse/atmos/errors" + cfg "github.com/cloudposse/atmos/pkg/config" ) // memoryActionContext builds an actionContext backed by Helm's in-memory storage From a5894a78417f1f6ce5ffa9d4ece7be729fcdf3ff Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Mon, 3 Aug 2026 18:31:37 +0400 Subject: [PATCH 46/62] fix: address final Helm lifecycle review findings --- examples/helm/README.md | 2 +- pkg/component/helm/aggregate_ci.go | 6 +++++- pkg/component/helm/aggregate_ci_test.go | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/examples/helm/README.md b/examples/helm/README.md index 3311c70b83..abc6766bdf 100644 --- a/examples/helm/README.md +++ b/examples/helm/README.md @@ -35,7 +35,7 @@ this same lifecycle. ## Render (no cluster, no credentials) ```shell -atmos helm template demo -s dev +atmos helm template demo -s dev --dependency-update # Render the same chart through a declarative Helm repository. HELM_DEMO_REPO_URL=http://127.0.0.1:8080 atmos helm template demo-repo -s dev diff --git a/pkg/component/helm/aggregate_ci.go b/pkg/component/helm/aggregate_ci.go index 3b3a0b8234..271aceb2cb 100644 --- a/pkg/component/helm/aggregate_ci.go +++ b/pkg/component/helm/aggregate_ci.go @@ -71,7 +71,11 @@ func (c *helmBulkCICollector) finish(ctx *component.ExecutionContext, startedAt, c.mu.Lock() defer c.mu.Unlock() - result := c.ensure(ctx.Stack, ctx.Component) + componentName := ctx.ConfigAndStacksInfo.ComponentFromArg + if componentName == "" { + componentName = ctx.Component + } + result := c.ensure(ctx.Stack, componentName) result.StartedAt = startedAt result.FinishedAt = finishedAt result.DurationMS = finishedAt.Sub(startedAt).Milliseconds() diff --git a/pkg/component/helm/aggregate_ci_test.go b/pkg/component/helm/aggregate_ci_test.go index 37d2af7214..8d0b8bb3f0 100644 --- a/pkg/component/helm/aggregate_ci_test.go +++ b/pkg/component/helm/aggregate_ci_test.go @@ -39,6 +39,24 @@ func TestHelmBulkCICollectorRecordsAndSortsResults(t *testing.T) { assert.Equal(t, "+ change", collector.resultSet().Results[1].Summary["diff"]) } +func TestHelmBulkCICollectorUsesProcessedComponentIdentity(t *testing.T) { + collector := newHelmBulkCICollector("plan") + info := schema.ConfigAndStacksInfo{Stack: "dev", ComponentFromArg: "apps/app"} + collector.setSummary(&info, map[string]any{"diff": "+ change"}, nil) + collector.finish(&component.ExecutionContext{ + Stack: "dev", + Component: "app", + ConfigAndStacksInfo: info, + }, time.Unix(1, 0), time.Unix(1, int64(time.Millisecond)), nil) + + results := collector.resultSet().Results + require.Len(t, results, 1) + assert.Equal(t, "apps/app", results[0].Component) + assert.True(t, results[0].Processed) + assert.Equal(t, int64(1), results[0].DurationMS) + assert.Equal(t, "+ change", results[0].Summary["diff"]) +} + func TestRunWithHooksBulkCollectorSuppressesPerComponentCI(t *testing.T) { originalHooks := getHooks originalApply := applyHelmRelease From 323180c1702a97a29d11907ad7293d64a99964ee Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Mon, 3 Aug 2026 21:41:58 +0400 Subject: [PATCH 47/62] docs: adopt Helm on_failure actions --- .../2026-07-31-native-helm-release-lifecycle.md | 2 +- examples/helm/README.md | 6 +++--- examples/helm/atmos.yaml | 4 ++-- examples/helm/stacks/deploy/dev.yaml | 10 ++++------ pkg/ci/plugins/helm/plugin_test.go | 12 +++++------- pkg/ci/plugins/helm/templates/apply.md | 3 +-- website/docs/cli/commands/helm/helm-apply.mdx | 10 +++------- .../docs/cli/configuration/components/helm.mdx | 2 +- website/docs/stacks/components/helm.mdx | 16 ++++++++-------- 9 files changed, 28 insertions(+), 37 deletions(-) diff --git a/docs/fixes/2026-07-31-native-helm-release-lifecycle.md b/docs/fixes/2026-07-31-native-helm-release-lifecycle.md index 7beaec6a9d..fcf0f96931 100644 --- a/docs/fixes/2026-07-31-native-helm-release-lifecycle.md +++ b/docs/fixes/2026-07-31-native-helm-release-lifecycle.md @@ -16,7 +16,7 @@ Helm 4 does not expose a context-aware uninstall request. `timeout: 0s` explicitly to keep unbounded behavior without the warning. - An omitted `max_history` now retains ten upgrade revisions, matching the Helm CLI. Configure `max_history: 0` to retain unlimited history. -- `atomic` is deprecated in favor of `rollback_on_failure`. +- Failure recovery uses `on_failure: [rollback, cleanup]` and the matching `--on-failure` list flag. - Boolean `--wait=true` and `--wait=false` remain accepted temporarily; use `--wait=watcher` and `--wait=hookOnly`. - Explicit lifecycle flags cannot be combined with a non-Kubernetes provision diff --git a/examples/helm/README.md b/examples/helm/README.md index abc6766bdf..cc12b6806b 100644 --- a/examples/helm/README.md +++ b/examples/helm/README.md @@ -21,7 +21,7 @@ atmos helm template demo -s dev --dependency-update atmos emulator up kubernetes -s dev atmos helm diff demo -s dev --identity local-k3s atmos helm apply demo -s dev --identity local-k3s --dry-run -atmos helm apply demo -s dev --identity local-k3s --rollback-on-failure --wait=watcher --timeout=2m +atmos helm apply demo -s dev --identity local-k3s --on-failure=rollback --wait=watcher --timeout=2m atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo atmos helm delete demo -s dev --identity local-k3s --dry-run --wait=watcher --timeout=2m @@ -73,8 +73,8 @@ atmos helm apply demo -s dev --identity local-k3s The stack sets `wait_strategy: watcher`, `timeout: 4m`, `max_history: 10`, and `skip_crds: true` as native Helm type defaults. The `demo` component overrides -the wait strategy to `legacy`, and enables `rollback_on_failure` and -failed-upgrade cleanup. The `atmos test` workflow +the wait strategy to `legacy`, and enables the `rollback` and `cleanup` +failure actions. The `atmos test` workflow covers non-mutating apply and delete dry runs, ordinary Job waiting, chart-hook suppression, CRD skipping, weighted hook ordering, retained hook resources, install and upgrade rollback, failed-upgrade cleanup, timeout handling, and diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index be225190d7..3c182f9400 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -152,8 +152,8 @@ commands: if ! after=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}' 2>&1); then echo "failed to capture Deployment/demo state after rollback: $after"; exit 1; fi; if [ "$before" != "$after" ]; then echo "rollback did not restore Deployment/demo: before=$before after=$after"; exit 1; fi - command: >- - if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-upgrade-only 2>&1); then echo "cleanup_on_fail left the upgrade-only ConfigMap"; exit 1; - else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify cleanup_on_fail removal: $output"; exit 1;; esac; fi + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-upgrade-only 2>&1); then echo "cleanup failure action left the upgrade-only ConfigMap"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify cleanup failure action: $output"; exit 1;; esac; fi # Dependency execution gates the dependent render on the foundation's # release-managed Deployment readiness state. - atmos helm apply --all -s dev --identity local-k3s --tags lifecycle-dag diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index 79de711e18..1e4aaf75b6 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -26,8 +26,7 @@ components: chart: "." namespace: demo wait_strategy: legacy - rollback_on_failure: true - cleanup_on_fail: true + on_failure: [rollback, cleanup] # Atmos `values:` are the Helm chart values, merged via Atmos inheritance. values: replicaCount: 2 @@ -96,7 +95,7 @@ components: namespace: demo-timeout wait_strategy: watcher timeout: 2s - rollback_on_failure: true + on_failure: [rollback] values: deployment: readinessDelaySeconds: 20 @@ -110,7 +109,7 @@ components: name: demo-install-fail namespace: demo-install-fail wait_strategy: legacy - rollback_on_failure: true + on_failure: [rollback] values: hooks: fail: true @@ -126,8 +125,7 @@ components: name: demo namespace: demo wait_strategy: legacy - rollback_on_failure: true - cleanup_on_fail: true + on_failure: [rollback, cleanup] values: hooks: fail: true diff --git a/pkg/ci/plugins/helm/plugin_test.go b/pkg/ci/plugins/helm/plugin_test.go index 2166d73ee1..d8e8cd01aa 100644 --- a/pkg/ci/plugins/helm/plugin_test.go +++ b/pkg/ci/plugins/helm/plugin_test.go @@ -260,8 +260,7 @@ func TestTemplateRendering(t *testing.T) { "timeout": "30m0s", "chart_hooks_enabled": true, "wait_for_jobs": true, - "rollback_on_failure": true, - "cleanup_on_fail": true, + "on_failure": []string{"rollback", "cleanup"}, "max_history": 10, }, contains: []string{ @@ -270,8 +269,7 @@ func TestTemplateRendering(t *testing.T) { "| Timeout | `30m0s` |", "| Chart hooks enabled | `true` |", "| Wait for Jobs | `true` |", - "| Rollback on failure | `true` |", - "| Cleanup on failure | `true` |", + "| On failure | `rollback, cleanup` |", "| Maximum history | `10` |", }, }, @@ -284,7 +282,7 @@ func TestTemplateRendering(t *testing.T) { "timeout": "5m0s", "chart_hooks_enabled": true, "wait_for_jobs": false, - "rollback_on_failure": false, + "on_failure": []string{}, "install_crds": true, }, contains: []string{ @@ -294,10 +292,10 @@ func TestTemplateRendering(t *testing.T) { "| Timeout | `5m0s` |", "| Chart hooks enabled | `true` |", "| Wait for Jobs | `false` |", - "| Rollback on failure | `false` |", + "| On failure | `` |", "| Install CRDs | `true` |", }, - notContains: []string{"Cleanup on failure", "Maximum history"}, + notContains: []string{"Maximum history"}, }, { name: "external apply", diff --git a/pkg/ci/plugins/helm/templates/apply.md b/pkg/ci/plugins/helm/templates/apply.md index 02d01bfffb..65c727e3f3 100644 --- a/pkg/ci/plugins/helm/templates/apply.md +++ b/pkg/ci/plugins/helm/templates/apply.md @@ -35,7 +35,7 @@ | Timeout | `{{ index . "timeout" }}` | | Chart hooks enabled | `{{ index . "chart_hooks_enabled" }}` | | Wait for Jobs | `{{ index . "wait_for_jobs" }}` | -| Rollback on failure | `{{ index . "rollback_on_failure" }}` | +| On failure | `{{ range $i, $action := index . "on_failure" }}{{ if $i }}, {{ end }}{{ $action }}{{ end }}` | {{- if eq (index . "operation") "install" }} | Install CRDs | `{{ index . "install_crds" }}` | @@ -43,7 +43,6 @@ {{- end }} {{- if eq (index . "operation") "upgrade" }} -| Cleanup on failure | `{{ index . "cleanup_on_fail" }}` | | Maximum history | `{{ index . "max_history" }}` | {{- end }} diff --git a/website/docs/cli/commands/helm/helm-apply.mdx b/website/docs/cli/commands/helm/helm-apply.mdx index 356e2101cc..5bdefe8757 100644 --- a/website/docs/cli/commands/helm/helm-apply.mdx +++ b/website/docs/cli/commands/helm/helm-apply.mdx @@ -30,11 +30,10 @@ Apply an explicit production readiness and recovery policy: ```shell atmos helm apply monitoring -s plat-ue2-dev \ - --rollback-on-failure \ + --on-failure=rollback,cleanup \ --wait=watcher \ --wait-for-jobs \ --timeout=30m \ - --cleanup-on-fail \ --history-max=10 ``` @@ -77,8 +76,8 @@ stack/component order and includes details for failures.
`--dry-run` (optional)
Preview the install or upgrade without persisting release state or creating Kubernetes resources.
-
`--rollback-on-failure` (optional)
-
Uninstall a failed first install or roll a failed upgrade back. Enabling it promotes the default wait strategy to `watcher`. The deprecated alias is `--atomic`.
+
`--on-failure` (optional)
+
Comma-separated failure actions: `rollback`, `cleanup`, or both. The flag replaces the component policy for this invocation; `--on-failure=` clears inherited actions.
`--wait[=strategy]` (optional)
Use `watcher`, `hookOnly`, or `legacy`. Passing `--wait` without a value selects `watcher`. Boolean values remain accepted temporarily but are deprecated.
@@ -89,9 +88,6 @@ stack/component order and includes details for failures.
`--timeout` (optional)
Helm release-operation timeout, such as `10m` or `1h`. `0s` is explicitly unbounded.
-
`--cleanup-on-fail` (optional)
-
Delete resources newly created by a failed upgrade.
-
`--history-max` (optional)
Maximum retained release revisions. Defaults to `10`; `0` means unlimited.
diff --git a/website/docs/cli/configuration/components/helm.mdx b/website/docs/cli/configuration/components/helm.mdx index 3212b2e9cd..429a55f9fb 100644 --- a/website/docs/cli/configuration/components/helm.mdx +++ b/website/docs/cli/configuration/components/helm.mdx @@ -56,7 +56,7 @@ components: :::info Release policy belongs in stacks Helm 4 lifecycle fields such as `wait_strategy`, `timeout`, -`rollback_on_failure`, and `max_history` are stack configuration, not +`on_failure`, and `max_history` are stack configuration, not project-wide `components.helm` settings in `atmos.yaml`. This keeps release policy subject to stack imports, component inheritance, and environment-specific overrides. See [Helm stack configuration](/stacks/components/helm#release-lifecycle). diff --git a/website/docs/stacks/components/helm.mdx b/website/docs/stacks/components/helm.mdx index 08e35f6c63..9014d876ef 100644 --- a/website/docs/stacks/components/helm.mdx +++ b/website/docs/stacks/components/helm.mdx @@ -124,18 +124,19 @@ values. | Field | Default | Operations | Behavior | | --- | --- | --- | --- | -| `rollback_on_failure` | `false` | install, upgrade | Uninstall a failed first install or roll a failed upgrade back. Enabling it promotes the default wait strategy to `watcher`. | +| `on_failure` | `[]` | install, upgrade | Failure actions: `rollback` uninstalls a failed first install or rolls a failed upgrade back; `cleanup` removes resources newly created by a failed upgrade. | | `wait_strategy` | `hookOnly` | install, upgrade, delete | `hookOnly`, `watcher`, or Helm 3-compatible `legacy`. | | `wait_for_jobs` | `false` | install, upgrade | Wait for ordinary Jobs. Requires `watcher` or `legacy`; hook Jobs are already handled by Helm hooks. | | `timeout` | `0s` during migration | install, upgrade, delete | Helm operation timeout. Explicit `0s` remains unbounded. | -| `cleanup_on_fail` | `false` | upgrade | Remove resources newly created by a failed upgrade. | | `max_history` | `10` | upgrade | Revisions retained. Set `0` for unlimited history. | | `disable_chart_hooks` | `false` | install, upgrade, delete | Disable Helm chart hooks. This does not disable Atmos `hooks:`. | | `skip_crds` | `false` | install | Do not install CRDs from the chart on first install. | -`atomic` is a deprecated alias for `rollback_on_failure`; `wait: true` and -`wait: false` are convenience aliases for `watcher` and `hookOnly`. Canonical -fields win when both forms are inherited or configured. +`on_failure` follows Atmos list merge behavior. With the default `replace` +strategy, a component can clear inherited actions with `on_failure: []`; +`append` can add actions across inheritance layers. Duplicate actions are +normalized. `wait: true` and `wait: false` remain convenience aliases for +`watcher` and `hookOnly`. :::warning Timeout and history migration For one minor release, omitting `timeout` preserves the previous unbounded `0s` @@ -157,8 +158,7 @@ components: release-policy: metadata: type: abstract - rollback_on_failure: true - cleanup_on_fail: true + on_failure: [rollback, cleanup] demo-release: metadata: @@ -209,7 +209,7 @@ components: wait_strategy: watcher wait_for_jobs: true timeout: 20m - rollback_on_failure: true + on_failure: [rollback] max_history: 10 values: grafana: From 008d6e7d9de0cf4e1760169045d6733d553fbfb0 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Tue, 4 Aug 2026 17:52:15 +0400 Subject: [PATCH 48/62] docs: align Helm examples with release hierarchy --- ...026-07-31-native-helm-release-lifecycle.md | 13 +-- docs/prd/native-helm-release-lifecycle.md | 3 - examples/helm/README.md | 11 +-- examples/helm/atmos.yaml | 4 +- examples/helm/stacks/deploy/dev.yaml | 73 ++++++++++++----- pkg/ci/plugins/helm/plugin.go | 2 +- pkg/ci/plugins/helm/plugin_test.go | 60 +++++++------- pkg/ci/plugins/helm/templates/apply.md | 13 +-- pkg/ci/plugins/helm/templates/delete.md | 4 +- pkg/component/helm/client_lifecycle_test.go | 3 +- website/docs/ci/job-summaries.mdx | 4 +- website/docs/cli/commands/helm/helm-apply.mdx | 12 ++- website/docs/stacks/components/helm.mdx | 81 ++++++++++++------- 13 files changed, 170 insertions(+), 113 deletions(-) diff --git a/docs/fixes/2026-07-31-native-helm-release-lifecycle.md b/docs/fixes/2026-07-31-native-helm-release-lifecycle.md index fcf0f96931..7ac505960b 100644 --- a/docs/fixes/2026-07-31-native-helm-release-lifecycle.md +++ b/docs/fixes/2026-07-31-native-helm-release-lifecycle.md @@ -11,12 +11,15 @@ Helm 4 does not expose a context-aware uninstall request. ## Migration notes -- An omitted `timeout` remains `0s` (unbounded) for one minor release and emits a +- An omitted `release.timeout` remains `0s` (unbounded) for one minor release and emits a warning. The omitted default becomes `5m` in the following minor. Configure - `timeout: 0s` explicitly to keep unbounded behavior without the warning. -- An omitted `max_history` now retains ten upgrade revisions, matching the Helm - CLI. Configure `max_history: 0` to retain unlimited history. -- Failure recovery uses `on_failure: [rollback, cleanup]` and the matching `--on-failure` list flag. + `release.timeout: 0s` explicitly to keep unbounded behavior without the warning. +- An omitted `release.history.max` retains ten upgrade revisions, matching the + Helm CLI. Configure `release.history.max: 0` to retain unlimited history. +- Failure recovery is operation-specific: use `release.install.on_failure: uninstall` + for failed first installs and `release.upgrade.on_failure: rollback` for failed + upgrades. Upgrade cleanup is controlled independently by + `release.upgrade.cleanup_on_failure`. - Boolean `--wait=true` and `--wait=false` remain accepted temporarily; use `--wait=watcher` and `--wait=hookOnly`. - Explicit lifecycle flags cannot be combined with a non-Kubernetes provision diff --git a/docs/prd/native-helm-release-lifecycle.md b/docs/prd/native-helm-release-lifecycle.md index 2900637f75..77aae75a2a 100644 --- a/docs/prd/native-helm-release-lifecycle.md +++ b/docs/prd/native-helm-release-lifecycle.md @@ -53,7 +53,6 @@ These gaps force users migrating from Helm or Helmfile to choose between depende - Fix apply dry-run propagation as a release-blocking safety prerequisite, then correctly propagate delete dry-run, cancellation, and deadlines through cluster operations. - Validate configuration before chart download or cluster mutation. - Keep template and diff complete by including Helm chart hook resources alongside the ordinary release manifest. -- Allow callers to opt into fetching missing chart dependencies with Helm-compatible `--dependency-update` semantics. - Keep the design compatible with future pre-rollback diagnostics without requiring another public configuration rename. - Follow Atmos schema, stack-processing, command parsing, provider, error, logging, and testing conventions. @@ -240,8 +239,6 @@ atmos helm delete demo-api -s example-prod \ `template`, `diff`, and `plan` do not register release-lifecycle flags because they do not perform a release operation. -All chart-loading operations (`template`, `diff`, `plan`, `apply`, and `deploy`) accept `--dependency-update`. Atmos invokes Helm's dependency manager only when a declared dependency is missing. The flag is intentionally invocation-scoped: without it, Atmos does not access dependency repositories or mutate the chart directory and instead reports both the equivalent `helm dependency build ` command and the opt-in flag. - ## Configuration Contract ### Release-Wide Defaults diff --git a/examples/helm/README.md b/examples/helm/README.md index cc12b6806b..c6d3dca1b3 100644 --- a/examples/helm/README.md +++ b/examples/helm/README.md @@ -21,7 +21,7 @@ atmos helm template demo -s dev --dependency-update atmos emulator up kubernetes -s dev atmos helm diff demo -s dev --identity local-k3s atmos helm apply demo -s dev --identity local-k3s --dry-run -atmos helm apply demo -s dev --identity local-k3s --on-failure=rollback --wait=watcher --timeout=2m +atmos helm apply demo -s dev --identity local-k3s --on-failure=uninstall --wait=watcher --timeout=2m atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo atmos helm delete demo -s dev --identity local-k3s --dry-run --wait=watcher --timeout=2m @@ -71,10 +71,11 @@ atmos emulator up kubernetes -s dev atmos helm apply demo -s dev --identity local-k3s ``` -The stack sets `wait_strategy: watcher`, `timeout: 4m`, `max_history: 10`, and -`skip_crds: true` as native Helm type defaults. The `demo` component overrides -the wait strategy to `legacy`, and enables the `rollback` and `cleanup` -failure actions. The `atmos test` workflow +The stack sets `release.wait.strategy: watcher`, `release.timeout: 4m`, +`release.history.max: 10`, and `release.install.crds: skip` as native Helm type +defaults. The `demo` component overrides the wait strategy to `legacy`, uses +install uninstall-on-failure, and enables upgrade rollback with independent +failed-upgrade cleanup. The `atmos test` workflow covers non-mutating apply and delete dry runs, ordinary Job waiting, chart-hook suppression, CRD skipping, weighted hook ordering, retained hook resources, install and upgrade rollback, failed-upgrade cleanup, timeout handling, and diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 3c182f9400..4acd562e5b 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -90,7 +90,7 @@ commands: backoff_strategy: constant # The -2 ConfigMap must exist before the -1 hook Job can mount it. - atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-hook-order - # skip_crds is a stack-level lifecycle default. + # release.install.crds is a stack-level release default. - command: >- if output=$(atmos emulator exec kubernetes -s dev -- kubectl get crd widgets.lifecycle.atmos.test 2>&1); then echo "skip_crds unexpectedly installed widgets.lifecycle.atmos.test"; exit 1; else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify widgets.lifecycle.atmos.test absence: $output"; exit 1;; esac; fi @@ -104,7 +104,7 @@ commands: max_attempts: 3 initial_delay: 10s backoff_strategy: constant - # watcher + wait_for_jobs returns only after the ordinary Job completes. + # watcher + release.wait.jobs returns only after the ordinary Job completes. - atmos helm apply demo-jobs -s dev --identity local-k3s - command: >- completed=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-jobs get job demo-jobs-job -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}'); diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index 1e4aaf75b6..098f97f13c 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -3,13 +3,17 @@ vars: # Native Helm lifecycle defaults for every Helm component in this stack. helm: - wait_strategy: watcher - # Leave enough headroom for nested k3s workloads on resource-constrained - # macOS/Colima runners. Components that exercise timeout behavior override - # this value explicitly below. - timeout: 4m - max_history: 10 - skip_crds: true + release: + wait: + strategy: watcher + # Leave enough headroom for nested k3s workloads on resource-constrained + # macOS/Colima runners. Components that exercise timeout behavior override + # this value explicitly below. + timeout: 4m + history: + max: 10 + install: + crds: skip components: emulator: @@ -25,8 +29,14 @@ components: # (components/helm/demo), which contains Chart.yaml. chart: "." namespace: demo - wait_strategy: legacy - on_failure: [rollback, cleanup] + release: + wait: + strategy: legacy + install: + on_failure: uninstall + upgrade: + on_failure: rollback + cleanup_on_failure: true # Atmos `values:` are the Helm chart values, merged via Atmos inheritance. values: replicaCount: 2 @@ -60,7 +70,9 @@ components: chart: "." name: demo-jobs namespace: demo-jobs - wait_for_jobs: true + release: + wait: + jobs: true values: hooks: enabled: false @@ -74,7 +86,8 @@ components: chart: "." name: demo-no-hooks namespace: demo-no-hooks - disable_chart_hooks: true + release: + chart_hooks: false demo-hook-only: metadata: @@ -82,7 +95,9 @@ components: chart: "." name: demo-hook-only namespace: demo-hook-only - wait_strategy: hookOnly + release: + wait: + strategy: hookOnly values: deployment: readinessGate: true @@ -93,9 +108,14 @@ components: chart: "." name: demo-timeout namespace: demo-timeout - wait_strategy: watcher - timeout: 2s - on_failure: [rollback] + release: + timeout: 2s + wait: + strategy: watcher + install: + on_failure: uninstall + upgrade: + on_failure: rollback values: deployment: readinessDelaySeconds: 20 @@ -108,8 +128,11 @@ components: chart: "." name: demo-install-fail namespace: demo-install-fail - wait_strategy: legacy - on_failure: [rollback] + release: + wait: + strategy: legacy + install: + on_failure: uninstall values: hooks: fail: true @@ -124,8 +147,12 @@ components: # Intentionally targets the already-installed demo release. name: demo namespace: demo - wait_strategy: legacy - on_failure: [rollback, cleanup] + release: + upgrade: + wait: + strategy: legacy + on_failure: rollback + cleanup_on_failure: true values: hooks: fail: true @@ -139,7 +166,9 @@ components: chart: "." name: dag-foundation namespace: lifecycle-dag - wait_strategy: legacy + release: + wait: + strategy: legacy values: deployment: readinessDelaySeconds: 3 @@ -154,7 +183,9 @@ components: chart: "." name: dag-dependent namespace: lifecycle-dag - wait_strategy: legacy + release: + wait: + strategy: legacy values: hooks: requiredDeployment: dag-foundation diff --git a/pkg/ci/plugins/helm/plugin.go b/pkg/ci/plugins/helm/plugin.go index df1091468f..186678b2d1 100644 --- a/pkg/ci/plugins/helm/plugin.go +++ b/pkg/ci/plugins/helm/plugin.go @@ -190,7 +190,7 @@ func summaryFromMap(m map[string]any) Summary { ObjectCount: intValue(m["object_count"]), ObjectKinds: stringSliceValue(m["object_kinds"]), ManifestBytes: intValue(m["manifest_bytes"]), - Lifecycle: mapValue(m["lifecycle"]), + Lifecycle: mapValue(m["release"]), Message: stringValue(m["message"]), Diff: stringValue(m["diff"]), } diff --git a/pkg/ci/plugins/helm/plugin_test.go b/pkg/ci/plugins/helm/plugin_test.go index d8e8cd01aa..6fb1a99dba 100644 --- a/pkg/ci/plugins/helm/plugin_test.go +++ b/pkg/ci/plugins/helm/plugin_test.go @@ -52,7 +52,7 @@ func TestPlugin_BuildTemplateContext(t *testing.T) { Command: "deploy", Info: &schema.ConfigAndStacksInfo{ ComponentFromArg: "nginx", - Stack: "plat-ue2-dev", + Stack: "dev", }, Aggregate: map[string]any{ "chart": "bitnami/nginx", @@ -62,15 +62,15 @@ func TestPlugin_BuildTemplateContext(t *testing.T) { "object_count": 2, "object_kinds": []any{"Service", "Deployment"}, "manifest_bytes": 1234, - "lifecycle": map[string]any{ - "operation": "upgrade", - "wait_strategy": "watcher", + "release": map[string]any{ + "operation": "upgrade", + "wait": map[string]any{"strategy": "watcher"}, }, }, }) assert.Equal(t, "nginx", ctx.Component) - assert.Equal(t, "plat-ue2-dev", ctx.Stack) + assert.Equal(t, "dev", ctx.Stack) assert.Equal(t, "deploy", ctx.Command) assert.Equal(t, "bitnami/nginx", ctx.Chart) assert.Equal(t, "nginx", ctx.ReleaseName) @@ -80,7 +80,7 @@ func TestPlugin_BuildTemplateContext(t *testing.T) { assert.Equal(t, 1234, ctx.ManifestBytes) assert.Equal(t, []string{"Deployment", "Service"}, ctx.ObjectKinds) assert.Equal(t, "upgrade", ctx.Lifecycle["operation"]) - assert.Equal(t, "watcher", ctx.Lifecycle["wait_strategy"]) + assert.Equal(t, "watcher", ctx.Lifecycle["wait"].(map[string]any)["strategy"]) } func TestNormalizeSummary(t *testing.T) { @@ -103,7 +103,7 @@ func TestNormalizeSummary(t *testing.T) { "manifest_bytes": float64(123), "message": 42, "diff": "diff text", - "lifecycle": lifecycle, + "release": lifecycle, }) assert.Equal(t, "app", got.Component) assert.Equal(t, "dev", got.Stack) @@ -255,13 +255,13 @@ func TestTemplateRendering(t *testing.T) { name: "cluster apply", command: "apply", lifecycle: map[string]any{ - "operation": "upgrade", - "wait_strategy": "watcher", - "timeout": "30m0s", - "chart_hooks_enabled": true, - "wait_for_jobs": true, - "on_failure": []string{"rollback", "cleanup"}, - "max_history": 10, + "operation": "upgrade", + "wait": map[string]any{"strategy": "watcher", "jobs": true}, + "timeout": "30m0s", + "chart_hooks": true, + "on_failure": "rollback", + "cleanup_on_failure": true, + "history": map[string]any{"max": 10}, }, contains: []string{ "Helm Apply Summary", "bitnami/nginx", "Deployment", "Release lifecycle", @@ -269,7 +269,8 @@ func TestTemplateRendering(t *testing.T) { "| Timeout | `30m0s` |", "| Chart hooks enabled | `true` |", "| Wait for Jobs | `true` |", - "| On failure | `rollback, cleanup` |", + "| On failure | `rollback` |", + "| Cleanup on failure | `true` |", "| Maximum history | `10` |", }, }, @@ -277,13 +278,12 @@ func TestTemplateRendering(t *testing.T) { name: "cluster install", command: "apply", lifecycle: map[string]any{ - "operation": "install", - "wait_strategy": "hookOnly", - "timeout": "5m0s", - "chart_hooks_enabled": true, - "wait_for_jobs": false, - "on_failure": []string{}, - "install_crds": true, + "operation": "install", + "wait": map[string]any{"strategy": "hookOnly", "jobs": false}, + "timeout": "5m0s", + "chart_hooks": true, + "on_failure": "keep", + "crds": "create", }, contains: []string{ "Helm Apply Summary", "Release lifecycle", @@ -292,8 +292,8 @@ func TestTemplateRendering(t *testing.T) { "| Timeout | `5m0s` |", "| Chart hooks enabled | `true` |", "| Wait for Jobs | `false` |", - "| On failure | `` |", - "| Install CRDs | `true` |", + "| On failure | `keep` |", + "| Install CRDs | `create` |", }, notContains: []string{"Maximum history"}, }, @@ -315,14 +315,14 @@ func TestTemplateRendering(t *testing.T) { name: "cluster delete", command: "delete", lifecycle: map[string]any{ - "operation": "uninstall", - "wait_strategy": "legacy", - "timeout": "10m0s", - "chart_hooks_enabled": false, + "operation": "delete", + "wait": map[string]any{"strategy": "legacy"}, + "timeout": "10m0s", + "chart_hooks": false, }, contains: []string{ "Helm Delete Summary", "Release lifecycle", - "| Operation | `uninstall` |", + "| Operation | `delete` |", "| Wait strategy | `legacy` |", "| Timeout | `10m0s` |", "| Chart hooks enabled | `false` |", @@ -350,7 +350,7 @@ func TestTemplateRendering(t *testing.T) { Command: tt.command, Info: &schema.ConfigAndStacksInfo{ ComponentFromArg: "nginx", - Stack: "plat-ue2-dev", + Stack: "dev", }, Aggregate: Summary{ Chart: "bitnami/nginx", diff --git a/pkg/ci/plugins/helm/templates/apply.md b/pkg/ci/plugins/helm/templates/apply.md index 65c727e3f3..f7dc6e6ced 100644 --- a/pkg/ci/plugins/helm/templates/apply.md +++ b/pkg/ci/plugins/helm/templates/apply.md @@ -31,19 +31,20 @@ | Field | Value | | --- | --- | | Operation | `{{ index . "operation" }}` | -| Wait strategy | `{{ index . "wait_strategy" }}` | +| Wait strategy | `{{ index (index . "wait") "strategy" }}` | | Timeout | `{{ index . "timeout" }}` | -| Chart hooks enabled | `{{ index . "chart_hooks_enabled" }}` | -| Wait for Jobs | `{{ index . "wait_for_jobs" }}` | -| On failure | `{{ range $i, $action := index . "on_failure" }}{{ if $i }}, {{ end }}{{ $action }}{{ end }}` | +| Chart hooks enabled | `{{ index . "chart_hooks" }}` | +| Wait for Jobs | `{{ index (index . "wait") "jobs" }}` | +| On failure | `{{ index . "on_failure" }}` | {{- if eq (index . "operation") "install" }} -| Install CRDs | `{{ index . "install_crds" }}` | +| Install CRDs | `{{ index . "crds" }}` | {{- end }} {{- if eq (index . "operation") "upgrade" }} -| Maximum history | `{{ index . "max_history" }}` | +| Cleanup on failure | `{{ index . "cleanup_on_failure" }}` | +| Maximum history | `{{ index (index . "history") "max" }}` | {{- end }} diff --git a/pkg/ci/plugins/helm/templates/delete.md b/pkg/ci/plugins/helm/templates/delete.md index 6bd8b123ee..0a25629ded 100644 --- a/pkg/ci/plugins/helm/templates/delete.md +++ b/pkg/ci/plugins/helm/templates/delete.md @@ -27,9 +27,9 @@ | Field | Value | | --- | --- | | Operation | `{{ index . "operation" }}` | -| Wait strategy | `{{ index . "wait_strategy" }}` | +| Wait strategy | `{{ index (index . "wait") "strategy" }}` | | Timeout | `{{ index . "timeout" }}` | -| Chart hooks enabled | `{{ index . "chart_hooks_enabled" }}` | +| Chart hooks enabled | `{{ index . "chart_hooks" }}` | {{- end }} diff --git a/pkg/component/helm/client_lifecycle_test.go b/pkg/component/helm/client_lifecycle_test.go index 0db7c6e4a5..407c28a9a2 100644 --- a/pkg/component/helm/client_lifecycle_test.go +++ b/pkg/component/helm/client_lifecycle_test.go @@ -245,7 +245,8 @@ func TestUpgradeReleaseHistoryRetention(t *testing.T) { stubActionContext(t, actx) spec := testdataChartSpec(t, tt.releaseName) if tt.override { - spec.Lifecycle.Policy.MaxHistory = tt.maxHistory + maxHistory := tt.maxHistory + spec.Release.History.Max = &maxHistory } for revision := 0; revision < revisions; revision++ { diff --git a/website/docs/ci/job-summaries.mdx b/website/docs/ci/job-summaries.mdx index f2bfd16fef..200b54b549 100644 --- a/website/docs/ci/job-summaries.mdx +++ b/website/docs/ci/job-summaries.mdx @@ -133,8 +133,8 @@ components include their error details. The same aggregate behavior applies to t `deploy` aliases and composes with tag and label filters. For a single cluster-backed apply/deploy/delete operation, the component summary also includes an -operation-specific `lifecycle` block with the effective wait strategy, timeout, chart-hook state, -and applicable rollback, Job-wait, CRD, cleanup, and history values. For external delivery, +operation-specific `release` block with the effective wait strategy, timeout, chart-hook state, +and applicable recovery, Job-wait, CRD, cleanup, and history values. For external delivery, apply/deploy reports `applied: false` while delete reports `deleted: false`; both include the selected target kind and `reason: external_target` instead of presenting stored release policy as active. diff --git a/website/docs/cli/commands/helm/helm-apply.mdx b/website/docs/cli/commands/helm/helm-apply.mdx index 5bdefe8757..4c35b5de90 100644 --- a/website/docs/cli/commands/helm/helm-apply.mdx +++ b/website/docs/cli/commands/helm/helm-apply.mdx @@ -30,7 +30,8 @@ Apply an explicit production readiness and recovery policy: ```shell atmos helm apply monitoring -s plat-ue2-dev \ - --on-failure=rollback,cleanup \ + --on-failure=rollback \ + --cleanup-on-failure \ --wait=watcher \ --wait-for-jobs \ --timeout=30m \ @@ -77,7 +78,10 @@ stack/component order and includes details for failures.
Preview the install or upgrade without persisting release state or creating Kubernetes resources.
`--on-failure` (optional)
-
Comma-separated failure actions: `rollback`, `cleanup`, or both. The flag replaces the component policy for this invocation; `--on-failure=` clears inherited actions.
+
Failure action for the selected operation: `uninstall` or `keep` for install; `rollback` or `keep` for upgrade. The explicit flag overrides stack configuration for this invocation.
+ +
`--cleanup-on-failure` (optional)
+
Remove resources newly created during a failed upgrade, independently of rollback. Fails if the selected operation is install.
`--wait[=strategy]` (optional)
Use `watcher`, `hookOnly`, or `legacy`. Passing `--wait` without a value selects `watcher`. Boolean values remain accepted temporarily but are deprecated.
@@ -89,13 +93,13 @@ stack/component order and includes details for failures.
Helm release-operation timeout, such as `10m` or `1h`. `0s` is explicitly unbounded.
`--history-max` (optional)
-
Maximum retained release revisions. Defaults to `10`; `0` means unlimited.
+
Maximum retained release revisions for an upgrade. Defaults to `10`; `0` means unlimited. Fails if the selected operation is install.
`--no-hooks` (optional)
Disable Helm chart hooks. Atmos lifecycle hooks are unaffected.
`--skip-crds` (optional)
-
Skip CRD installation on a first install.
+
Skip CRD installation on a first install. Fails if the selected operation is upgrade.
`--all` (optional)
Apply all Helm components in dependency order.
diff --git a/website/docs/stacks/components/helm.mdx b/website/docs/stacks/components/helm.mdx index 9014d876ef..497917bc9a 100644 --- a/website/docs/stacks/components/helm.mdx +++ b/website/docs/stacks/components/helm.mdx @@ -117,48 +117,56 @@ inheritance and component overrides are merged. ## Release Lifecycle Cluster-backed `apply`, `deploy`, and `delete` operations can use Helm 4 release -lifecycle controls. Configure them at the top-level `helm` section as defaults, -on an abstract component for inheritance, or on a concrete component. Normal -Atmos precedence applies; concrete components override inherited and type-level -values. +lifecycle controls under `release`. Configure the tree at the top-level +`helm.release` section as defaults, on an abstract component for inheritance, or +on a concrete component. Atmos first deep-merges that complete tree, then +overlays the selected `install`, `upgrade`, or `delete` section. Explicit command +flags have the highest precedence. | Field | Default | Operations | Behavior | | --- | --- | --- | --- | -| `on_failure` | `[]` | install, upgrade | Failure actions: `rollback` uninstalls a failed first install or rolls a failed upgrade back; `cleanup` removes resources newly created by a failed upgrade. | -| `wait_strategy` | `hookOnly` | install, upgrade, delete | `hookOnly`, `watcher`, or Helm 3-compatible `legacy`. | -| `wait_for_jobs` | `false` | install, upgrade | Wait for ordinary Jobs. Requires `watcher` or `legacy`; hook Jobs are already handled by Helm hooks. | -| `timeout` | `0s` during migration | install, upgrade, delete | Helm operation timeout. Explicit `0s` remains unbounded. | -| `max_history` | `10` | upgrade | Revisions retained. Set `0` for unlimited history. | -| `disable_chart_hooks` | `false` | install, upgrade, delete | Disable Helm chart hooks. This does not disable Atmos `hooks:`. | -| `skip_crds` | `false` | install | Do not install CRDs from the chart on first install. | - -`on_failure` follows Atmos list merge behavior. With the default `replace` -strategy, a component can clear inherited actions with `on_failure: []`; -`append` can add actions across inheritance layers. Duplicate actions are -normalized. `wait: true` and `wait: false` remain convenience aliases for -`watcher` and `hookOnly`. +| `release.wait.strategy` | `hookOnly` | install, upgrade, delete | `hookOnly`, `watcher`, or Helm 3-compatible `legacy`. | +| `release.wait.jobs` | `false` | install, upgrade | Wait for ordinary Jobs. Requires `watcher` or `legacy`; hook Jobs are already handled by Helm hooks. | +| `release.timeout` | `0s` during migration | install, upgrade, delete | Release-wide operation timeout. Each operation can override it. Explicit `0s` remains unbounded. | +| `release.history.max` | `10` | upgrade | Revisions retained. Set `0` for unlimited history. | +| `release.chart_hooks` | `true` | install, upgrade, delete | Enable Helm chart hooks. This does not control Atmos `hooks:`. | +| `release.install.crds` | `create` | install | Create or skip CRDs from the chart on first install. | +| `release.install.on_failure` | `keep` | install | `uninstall` removes a failed first install; `keep` preserves partial state. | +| `release.upgrade.on_failure` | `keep` | upgrade | `rollback` restores the prior release; `keep` preserves failed state. | +| `release.upgrade.cleanup_on_failure` | `false` | upgrade | Independently remove resources newly created by a failed upgrade. | + +`release.install.timeout`, `release.upgrade.timeout`, and +`release.delete.timeout` override the release-wide timeout only for that action. +The same operation sections can override `chart_hooks` and `wait`. :::warning Timeout and history migration -For one minor release, omitting `timeout` preserves the previous unbounded `0s` +For one minor release, omitting `release.timeout` and the selected operation +timeout preserves the previous unbounded `0s` behavior and emits a warning. The following minor changes the omitted default to -`5m`. Set `timeout: 0s` explicitly to remain unbounded, or set a duration such as -`30m`. An omitted `max_history` now retains ten revisions; set -`max_history: 0` if unlimited release history is required. +`5m`. Set `release.timeout: 0s` explicitly to remain unbounded, or set a duration +such as `30m`. An omitted `release.history.max` retains ten revisions; set it to +`0` if unlimited release history is required. ::: ```yaml helm: - wait_strategy: watcher - timeout: 10m - max_history: 10 + release: + wait: + strategy: watcher + timeout: 10m + history: + max: 10 components: helm: release-policy: metadata: type: abstract - on_failure: [rollback, cleanup] + release: + upgrade: + on_failure: rollback + cleanup_on_failure: true demo-release: metadata: @@ -166,7 +174,12 @@ components: - release-policy chart: ./charts/demo-release namespace: demo - timeout: 30m + release: + install: + timeout: 60m + on_failure: uninstall + upgrade: + timeout: 30m ``` @@ -206,11 +219,17 @@ components: - name: prometheus-community url: https://prometheus-community.github.io/helm-charts namespace: monitoring - wait_strategy: watcher - wait_for_jobs: true - timeout: 20m - on_failure: [rollback] - max_history: 10 + release: + timeout: 20m + wait: + strategy: watcher + jobs: true + history: + max: 10 + install: + on_failure: uninstall + upgrade: + on_failure: rollback values: grafana: enabled: true From 187ceec70e936ce51f3cc49bab0d20e1156f5c0e Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Tue, 4 Aug 2026 17:52:24 +0400 Subject: [PATCH 49/62] fix: report effective Helm timeout source --- pkg/component/helm/client.go | 3 +-- pkg/component/helm/client_test.go | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/component/helm/client.go b/pkg/component/helm/client.go index 16742c52d2..04e47accd7 100644 --- a/pkg/component/helm/client.go +++ b/pkg/component/helm/client.go @@ -16,7 +16,6 @@ import ( "helm.sh/helm/v4/pkg/storage/driver" errUtils "github.com/cloudposse/atmos/errors" - cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/perf" ) @@ -265,7 +264,7 @@ func releaseOperationError(operation string, spec *chartSpec, cause error) error WithContext("namespace", spec.Namespace). WithContext("wait_strategy", policy.WaitStrategy). WithContext("timeout", policy.Timeout). - WithContext("timeout_field", cfg.HelmTimeoutSectionName). + WithContext("timeout_field", spec.Lifecycle.TimeoutField). Err() } diff --git a/pkg/component/helm/client_test.go b/pkg/component/helm/client_test.go index 32653865e4..4494543aa5 100644 --- a/pkg/component/helm/client_test.go +++ b/pkg/component/helm/client_test.go @@ -13,7 +13,6 @@ import ( "helm.sh/helm/v4/pkg/kube" errUtils "github.com/cloudposse/atmos/errors" - cfg "github.com/cloudposse/atmos/pkg/config" ) func TestResolveUpgradeChartRef(t *testing.T) { @@ -105,10 +104,10 @@ func TestReleaseOperationErrorIncludesEffectivePolicy(t *testing.T) { err := releaseOperationError("upgrade", &chartSpec{ ReleaseName: "demo", Namespace: "apps", - Lifecycle: releaseLifecycleResolution{Policy: releaseLifecycle{ + Lifecycle: releaseLifecycleResolution{Policy: effectiveReleasePolicy{ WaitStrategy: kube.StatusWatcherStrategy, Timeout: 7 * time.Minute, - }}, + }, TimeoutField: "release.upgrade.timeout"}, }, cause) require.ErrorIs(t, err, errUtils.ErrHelmReleaseOperation) @@ -118,7 +117,7 @@ func TestReleaseOperationErrorIncludesEffectivePolicy(t *testing.T) { assert.True(t, errUtils.HasContext(err, "namespace", "apps")) assert.True(t, errUtils.HasContext(err, "wait_strategy", "watcher")) assert.True(t, errUtils.HasContext(err, "timeout", "7m0s")) - assert.True(t, errUtils.HasContext(err, "timeout_field", cfg.HelmTimeoutSectionName)) + assert.True(t, errUtils.HasContext(err, "timeout_field", "release.upgrade.timeout")) } func TestClusterOperationsReturnActionContextErrors(t *testing.T) { From 03d7f99435ebe97b3dd07d4962298d2cdc917fd4 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Tue, 4 Aug 2026 19:23:43 +0400 Subject: [PATCH 50/62] fix: harden aggregate Helm result reporting --- pkg/ci/plugins/helm/aggregate.go | 16 +++-- pkg/ci/plugins/helm/aggregate_test.go | 23 +++++++ pkg/component/graph.go | 19 ++++++ pkg/component/helm/aggregate_ci.go | 40 +++++++++++- pkg/component/helm/aggregate_ci_test.go | 82 +++++++++++++++++++++++++ 5 files changed, 172 insertions(+), 8 deletions(-) diff --git a/pkg/ci/plugins/helm/aggregate.go b/pkg/ci/plugins/helm/aggregate.go index 193eedaf4b..8b4bca8faa 100644 --- a/pkg/ci/plugins/helm/aggregate.go +++ b/pkg/ci/plugins/helm/aggregate.go @@ -277,13 +277,9 @@ func writeHelmAggregateDetails(builder *strings.Builder, command string, compone detail.WriteString(helmMarkdownCell(component.Result.Stack + "/" + component.Result.Component + ": " + component.Status)) detail.WriteString("\n\n") if component.Status == "failed" { - detail.WriteString("```text\n") - detail.WriteString(plugin.TruncateDetail(component.Result.Error)) - detail.WriteString("\n```\n") + detail.WriteString(helmMarkdownCodeBlock("text", plugin.TruncateDetail(component.Result.Error))) } else { - detail.WriteString("```diff\n") - detail.WriteString(plugin.TruncateDetail(component.Summary.Diff)) - detail.WriteString("\n```\n") + detail.WriteString(helmMarkdownCodeBlock("diff", plugin.TruncateDetail(component.Summary.Diff))) } detail.WriteString("\n\n\n") if builder.Len()+detail.Len() > helmAggregateMarkdownMaxBytes { @@ -294,6 +290,14 @@ func writeHelmAggregateDetails(builder *strings.Builder, command string, compone } } +func helmMarkdownCodeBlock(language, value string) string { + fence := "```" + for strings.Contains(value, fence) { + fence += "`" + } + return fence + language + "\n" + value + "\n" + fence + "\n" +} + func formatHelmAggregateDuration(milliseconds int64) string { if milliseconds <= 0 { return "-" diff --git a/pkg/ci/plugins/helm/aggregate_test.go b/pkg/ci/plugins/helm/aggregate_test.go index f5f568116e..8c9e492df3 100644 --- a/pkg/ci/plugins/helm/aggregate_test.go +++ b/pkg/ci/plugins/helm/aggregate_test.go @@ -72,6 +72,29 @@ func TestOnAfterAggregateRendersApplySummary(t *testing.T) { assert.Contains(t, writer.summary, "| dev | api | succeeded | api | api | apps | kubernetes | - |") } +func TestOnAfterAggregateUsesSafeDetailFences(t *testing.T) { + writer := &fakeWriter{} + err := (&Plugin{}).onAfterAggregate(&plugin.HookContext{ + Provider: fakeProvider{writer: writer}, + Aggregate: schema.HelmCIResultSet{ + Command: "plan", + Results: []schema.HelmCIResult{ + { + Stack: "dev", Component: "api", Processed: true, + Summary: map[string]any{"diff": "+ change\n```\nnot summary Markdown"}, + }, + { + Stack: "dev", Component: "worker", Status: "failed", + Error: "render failed\n```\nnot summary Markdown", + }, + }, + }, + }) + require.NoError(t, err) + assert.Contains(t, writer.summary, "````diff\n+ change\n```\nnot summary Markdown\n````") + assert.Contains(t, writer.summary, "````text\nrender failed\n```\nnot summary Markdown\n````") +} + func TestOnAfterAggregateSkipsInvalidOrDisabledAndReturnsWriterError(t *testing.T) { pluginUnderTest := &Plugin{} require.NoError(t, pluginUnderTest.onAfterAggregate(&plugin.HookContext{Provider: fakeProvider{}, Aggregate: "invalid"})) diff --git a/pkg/component/graph.go b/pkg/component/graph.go index 78187783ff..ece6af48e6 100644 --- a/pkg/component/graph.go +++ b/pkg/component/graph.go @@ -39,6 +39,12 @@ type GraphExecutionOptions struct { Selection *GraphSelection } +// GraphNodeSkipObserver is implemented by providers that need to record graph +// nodes skipped after execution stops before reaching them. +type GraphNodeSkipObserver interface { + OnGraphNodeSkipped(node *dependency.Node) +} + // ExecuteGraph runs selected components in dependency order and stops before // starting another component when the caller context is canceled. func ExecuteGraph(ctx context.Context, opts *GraphExecutionOptions) error { @@ -63,17 +69,20 @@ func ExecuteGraph(ctx context.Context, opts *GraphExecutionOptions) error { for i := range order { select { case <-ctx.Done(): + notifyGraphNodeSkips(opts.Provider, order[i:]) return fmt.Errorf(errUtils.ErrWrapFormat, errUtils.ErrGraphExecutionCanceled, ctx.Err()) default: } if err := executeGraphNode(ctx, opts, &order[i]); err != nil { + notifyGraphNodeSkips(opts.Provider, order[i+1:]) if ctxErr := ctx.Err(); ctxErr != nil { return fmt.Errorf(errUtils.ErrWrapFormat, errUtils.ErrGraphExecutionCanceled, errors.Join(ctxErr, err)) } return err } if ctxErr := ctx.Err(); ctxErr != nil { + notifyGraphNodeSkips(opts.Provider, order[i+1:]) return fmt.Errorf(errUtils.ErrWrapFormat, errUtils.ErrGraphExecutionCanceled, ctxErr) } } @@ -81,6 +90,16 @@ func ExecuteGraph(ctx context.Context, opts *GraphExecutionOptions) error { return nil } +func notifyGraphNodeSkips(provider ComponentProvider, nodes dependency.ExecutionOrder) { + observer, ok := provider.(GraphNodeSkipObserver) + if !ok { + return + } + for i := range nodes { + observer.OnGraphNodeSkipped(&nodes[i]) + } +} + // prepareExecutionOrder validates options, builds and filters the graph, and returns // the topologically sorted execution order. An empty order indicates no matching components. func prepareExecutionOrder(opts *GraphExecutionOptions) (dependency.ExecutionOrder, error) { diff --git a/pkg/component/helm/aggregate_ci.go b/pkg/component/helm/aggregate_ci.go index 271aceb2cb..c52db7bb33 100644 --- a/pkg/component/helm/aggregate_ci.go +++ b/pkg/component/helm/aggregate_ci.go @@ -7,6 +7,7 @@ import ( errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/component" + "github.com/cloudposse/atmos/pkg/dependency" "github.com/cloudposse/atmos/pkg/hooks" log "github.com/cloudposse/atmos/pkg/logger" "github.com/cloudposse/atmos/pkg/schema" @@ -32,7 +33,7 @@ func newHelmBulkCICollector(command string) *helmBulkCICollector { // bulkCollectingProvider wraps the native Helm provider and records failures // that happen before an operation produces its structured summary. type bulkCollectingProvider struct { - *ComponentProvider + component.ComponentProvider collector *helmBulkCICollector } @@ -43,6 +44,13 @@ func (p *bulkCollectingProvider) Execute(ctx *component.ExecutionContext) error return err } +func (p *bulkCollectingProvider) OnGraphNodeSkipped(node *dependency.Node) { + if p == nil || p.collector == nil || node == nil { + return + } + p.collector.markSkipped(node.Stack, node.Component) +} + func helmBulkCollector(ctx *component.ExecutionContext) *helmBulkCICollector { if ctx == nil { return nil @@ -82,6 +90,17 @@ func (c *helmBulkCICollector) finish(ctx *component.ExecutionContext, startedAt, applyHelmResultError(result, execErr) } +func (c *helmBulkCICollector) markSkipped(stack, componentName string) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + + result := c.ensure(stack, componentName) + result.Status = "skipped" +} + func (c *helmBulkCICollector) ensure(stack, componentName string) *schema.HelmCIResult { nodeID := component.GraphNodeID(componentName, stack) if result, ok := c.results[nodeID]; ok { @@ -137,11 +156,28 @@ func cloneHelmSummary(summary map[string]any) map[string]any { } cloned := make(map[string]any, len(summary)) for key, value := range summary { - cloned[key] = value + cloned[key] = cloneHelmSummaryValue(value) } return cloned } +func cloneHelmSummaryValue(value any) any { + switch typed := value.(type) { + case map[string]any: + return cloneHelmSummary(typed) + case []any: + cloned := make([]any, len(typed)) + for i := range typed { + cloned[i] = cloneHelmSummaryValue(typed[i]) + } + return cloned + case []string: + return append([]string(nil), typed...) + default: + return value + } +} + func supportsHelmAggregateCI(command string) bool { switch command { case "plan", "diff", "apply", "deploy": diff --git a/pkg/component/helm/aggregate_ci_test.go b/pkg/component/helm/aggregate_ci_test.go index 8d0b8bb3f0..366b58bcfc 100644 --- a/pkg/component/helm/aggregate_ci_test.go +++ b/pkg/component/helm/aggregate_ci_test.go @@ -11,6 +11,7 @@ import ( "github.com/cloudposse/atmos/pkg/auth" "github.com/cloudposse/atmos/pkg/component" + cfg "github.com/cloudposse/atmos/pkg/config" "github.com/cloudposse/atmos/pkg/hooks" "github.com/cloudposse/atmos/pkg/schema" ) @@ -39,6 +40,87 @@ func TestHelmBulkCICollectorRecordsAndSortsResults(t *testing.T) { assert.Equal(t, "+ change", collector.resultSet().Results[1].Summary["diff"]) } +func TestHelmBulkCICollectorDeepCopiesNestedSummary(t *testing.T) { + collector := newHelmBulkCICollector("apply") + wait := map[string]any{"strategy": "watcher"} + kinds := []string{"Deployment", "Service"} + summary := map[string]any{ + "release": map[string]any{"wait": wait}, + "object_kinds": kinds, + } + collector.setSummary(&schema.ConfigAndStacksInfo{Stack: "dev", ComponentFromArg: "api"}, summary, nil) + + wait["strategy"] = "legacy" + kinds[0] = "Job" + result := collector.resultSet().Results[0] + release := result.Summary["release"].(map[string]any) + assert.Equal(t, "watcher", release["wait"].(map[string]any)["strategy"]) + assert.Equal(t, []string{"Deployment", "Service"}, result.Summary["object_kinds"]) + + release["wait"].(map[string]any)["strategy"] = "mutated" + result.Summary["object_kinds"].([]string)[1] = "ConfigMap" + retained := collector.resultSet().Results[0].Summary + assert.Equal(t, "watcher", retained["release"].(map[string]any)["wait"].(map[string]any)["strategy"]) + assert.Equal(t, []string{"Deployment", "Service"}, retained["object_kinds"]) +} + +type helmBulkGraphTestProvider struct { + failComponent string +} + +func (p *helmBulkGraphTestProvider) GetType() string { return cfg.HelmComponentType } +func (p *helmBulkGraphTestProvider) GetGroup() string { return "test" } +func (p *helmBulkGraphTestProvider) GetBasePath(*schema.AtmosConfiguration) string { return "" } +func (p *helmBulkGraphTestProvider) ListComponents(context.Context, string, map[string]any) ([]string, error) { + return nil, nil +} +func (p *helmBulkGraphTestProvider) ValidateComponent(map[string]any) error { return nil } +func (p *helmBulkGraphTestProvider) Execute(ctx *component.ExecutionContext) error { + if ctx.Component == p.failComponent { + return errors.New("operation failed") + } + return nil +} +func (p *helmBulkGraphTestProvider) GenerateArtifacts(*component.ExecutionContext) error { return nil } +func (p *helmBulkGraphTestProvider) GetAvailableCommands() []string { return nil } + +func TestHelmBulkCICollectorRecordsDependencyBlockedComponents(t *testing.T) { + collector := newHelmBulkCICollector("apply") + provider := &bulkCollectingProvider{ + ComponentProvider: &helmBulkGraphTestProvider{failComponent: "base"}, + collector: collector, + } + stacks := map[string]any{ + "dev": map[string]any{ + cfg.ComponentsSectionName: map[string]any{ + cfg.HelmComponentType: map[string]any{ + "base": map[string]any{}, + "api": map[string]any{ + cfg.SettingsSectionName: map[string]any{"depends_on": []any{"base"}}, + }, + }, + }, + }, + } + + err := component.ExecuteGraph(context.Background(), &component.GraphExecutionOptions{ + Provider: provider, + Info: &schema.ConfigAndStacksInfo{}, + Stacks: stacks, + ComponentType: cfg.HelmComponentType, + SubCommand: "apply", + }) + require.Error(t, err) + + results := collector.resultSet().Results + require.Len(t, results, 2) + assert.Equal(t, "api", results[0].Component) + assert.Equal(t, "skipped", results[0].Status) + assert.False(t, results[0].Processed) + assert.Equal(t, "base", results[1].Component) + assert.Equal(t, "failed", results[1].Status) +} + func TestHelmBulkCICollectorUsesProcessedComponentIdentity(t *testing.T) { collector := newHelmBulkCICollector("plan") info := schema.ConfigAndStacksInfo{Stack: "dev", ComponentFromArg: "apps/app"} From 2a4a77aaac4501ca2a833fcefcbc9de138846f94 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Thu, 6 Aug 2026 01:13:52 +0400 Subject: [PATCH 51/62] fix: preserve Helm timeout error context --- pkg/component/helm/client.go | 2 +- pkg/component/helm/client_lifecycle_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/component/helm/client.go b/pkg/component/helm/client.go index 04e47accd7..756fda9140 100644 --- a/pkg/component/helm/client.go +++ b/pkg/component/helm/client.go @@ -166,7 +166,7 @@ func upgradeRelease(ctx context.Context, actx *actionContext, spec *chartSpec, d rel, err := client.RunWithContext(ctx, spec.ReleaseName, loaded, spec.Values) if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { - return "", ctxErr + return "", releaseOperationError("upgrade", spec, errors.Join(ctxErr, err)) } return "", releaseOperationError("upgrade", spec, err) } diff --git a/pkg/component/helm/client_lifecycle_test.go b/pkg/component/helm/client_lifecycle_test.go index 407c28a9a2..84c717220f 100644 --- a/pkg/component/helm/client_lifecycle_test.go +++ b/pkg/component/helm/client_lifecycle_test.go @@ -201,6 +201,7 @@ func TestApplyReleaseUsesLifecycleTimeoutAndWaitContext(t *testing.T) { elapsed := time.Since(started) require.ErrorIs(t, err, context.DeadlineExceeded) + require.ErrorIs(t, err, errUtils.ErrHelmReleaseOperation) assert.Equal(t, releaseOperationUpgrade, result.Operation) assert.Less(t, elapsed, time.Second, "upgrade must return at the lifecycle deadline") assert.NotEmpty(t, kubeClient.RecordedWaitOptions, "Helm waiters must receive the operation context") From a4e3bded91a4127163bf59b33113ea9400fc280c Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Thu, 6 Aug 2026 02:05:21 +0400 Subject: [PATCH 52/62] fix: track native Helm values files as affected --- .../describe_affected_changed_files_index.go | 24 +++++++- internal/exec/describe_affected_components.go | 59 +++++++++++++++++++ .../exec/describe_affected_components_test.go | 55 +++++++++++++++++ .../describe_affected_optimizations_test.go | 16 +++++ 4 files changed, 153 insertions(+), 1 deletion(-) diff --git a/internal/exec/describe_affected_changed_files_index.go b/internal/exec/describe_affected_changed_files_index.go index 0a90dac7e8..5b854e9bce 100644 --- a/internal/exec/describe_affected_changed_files_index.go +++ b/internal/exec/describe_affected_changed_files_index.go @@ -20,6 +20,10 @@ type changedFilesIndex struct { // allFiles contains all changed files for fallback scenarios. allFiles []string + // allFilesSet provides constant-time lookup for dependencies that may live + // outside a component base path, such as native Helm values_files. + allFilesSet map[string]struct{} + mu sync.RWMutex } @@ -34,6 +38,7 @@ func newChangedFilesIndex(atmosConfig *schema.AtmosConfiguration, changedFiles [ index := &changedFilesIndex{ filesByBasePath: make(map[string][]string), allFiles: nil, // Set after normalization. + allFilesSet: make(map[string]struct{}, len(changedFiles)), } // Pre-compute absolute base paths for each component type. @@ -62,7 +67,9 @@ func newChangedFilesIndex(atmosConfig *schema.AtmosConfiguration, changedFiles [ absF = f } } + absF = filepath.Clean(absF) absAllFiles = append(absAllFiles, absF) + index.allFilesSet[absF] = struct{}{} } index.allFiles = absAllFiles @@ -83,7 +90,7 @@ func newChangedFilesIndex(atmosConfig *schema.AtmosConfiguration, changedFiles [ // Only includes non-empty component base paths to avoid indexing files under the root basePath. func buildNormalizedBasePaths(atmosConfig *schema.AtmosConfiguration) []string { // Collect base paths, skipping empty ones to prevent root basePath collisions. - basePaths := make([]string, 0, 4) + basePaths := make([]string, 0, 6) // Add terraform base path if configured. if atmosConfig.Components.Terraform.BasePath != "" { @@ -105,6 +112,11 @@ func buildNormalizedBasePaths(atmosConfig *schema.AtmosConfiguration) []string { basePaths = append(basePaths, filepath.Join(atmosConfig.BasePath, atmosConfig.Components.Kubernetes.BasePath)) } + // Add native Helm base path if configured. + if atmosConfig.Components.Helm.BasePath != "" { + basePaths = append(basePaths, filepath.Join(atmosConfig.BasePath, atmosConfig.Components.Helm.BasePath)) + } + // Add stacks base path if configured. if atmosConfig.Stacks.BasePath != "" { basePaths = append(basePaths, filepath.Join(atmosConfig.BasePath, atmosConfig.Stacks.BasePath)) @@ -211,3 +223,13 @@ func (idx *changedFilesIndex) getAllFiles() []string { defer idx.mu.RUnlock() return idx.allFiles } + +// isChangedFile reports whether the normalized absolute path is in the git +// change set. It is used for component dependencies that can be located +// outside the component's indexed base path. +func (idx *changedFilesIndex) isChangedFile(path string) bool { + idx.mu.RLock() + defer idx.mu.RUnlock() + _, ok := idx.allFilesSet[filepath.Clean(path)] + return ok +} diff --git a/internal/exec/describe_affected_components.go b/internal/exec/describe_affected_components.go index f9c616443f..b2ad46ff31 100644 --- a/internal/exec/describe_affected_components.go +++ b/internal/exec/describe_affected_components.go @@ -3,6 +3,7 @@ package exec import ( "fmt" + "path/filepath" "reflect" "github.com/go-viper/mapstructure/v2" @@ -670,6 +671,14 @@ func processHelmComponentsIndexed( } } + if helmValuesFileChanged(component, componentSection, atmosConfig, filesIndex) { + err := addAffectedComponent(&affected, atmosConfig, componentName, stackName, cfg.HelmComponentType, + &componentSection, affectedReasonStackValuesFile, includeSpaceliftAdminStacks, currentStacks, includeSettings) + if err != nil { + return nil, err + } + } + if err := addHelmSectionAffected(&affected, atmosConfig, componentName, stackName, &componentSection, remoteStacks, currentStacks, includeSpaceliftAdminStacks, includeSettings); err != nil { return nil, err } @@ -689,6 +698,56 @@ func processHelmComponentsIndexed( return affected, nil } +// helmValuesFileChanged reports whether a native Helm component consumes a +// values file changed by the compared git refs. Relative paths are resolved +// from the physical component directory, matching Helm values loading. The +// target may live outside the Helm component base path, so lookup uses the +// complete changed-file set rather than the component-path index. +func helmValuesFileChanged( + component string, + componentSection map[string]any, + atmosConfig *schema.AtmosConfiguration, + filesIndex *changedFilesIndex, +) bool { + if filesIndex == nil { + return false + } + + componentPath := filepath.Join(atmosConfig.BasePath, atmosConfig.Components.Helm.BasePath, component) + for _, ref := range stringSlice(componentSection[cfg.ValuesFilesSectionName]) { + path := ref + if !filepath.IsAbs(path) { + path = filepath.Join(componentPath, path) + } + absPath, err := filepath.Abs(path) + if err == nil && filesIndex.isChangedFile(absPath) { + return true + } + } + + return false +} + +func stringSlice(value any) []string { + switch typed := value.(type) { + case []string: + return typed + case []any: + result := make([]string, 0, len(typed)) + for _, item := range typed { + if str, ok := item.(string); ok && str != "" { + result = append(result, str) + } + } + return result + case string: + if typed != "" { + return []string{typed} + } + } + return nil +} + func addHelmSectionAffected( affected *[]schema.Affected, atmosConfig *schema.AtmosConfiguration, diff --git a/internal/exec/describe_affected_components_test.go b/internal/exec/describe_affected_components_test.go index d482551bb6..c0ec654049 100644 --- a/internal/exec/describe_affected_components_test.go +++ b/internal/exec/describe_affected_components_test.go @@ -612,6 +612,61 @@ func TestProcessHelmComponentsIndexed_FolderChanged(t *testing.T) { assert.Contains(t, affected[0].AffectedAll, affectedReasonComponent) } +func TestProcessHelmComponentsIndexed_ValuesFileChangedOutsideComponentBasePath(t *testing.T) { + tempDir := t.TempDir() + atmosConfig := helmAtmosConfig() + atmosConfig.BasePath = tempDir + + componentFolder := "shared-chart" + componentPath := filepath.Join(tempDir, "components", "helm", componentFolder) + valuesFile := filepath.Join(tempDir, "config", "helm", "app-values.yaml") + valuesRef, err := filepath.Rel(componentPath, valuesFile) + require.NoError(t, err) + + identical := map[string]any{ + cfg.ComponentSectionName: componentFolder, + sectionNameChart: ".", + sectionNameValuesF: []any{valuesRef}, + } + helmSection := map[string]any{helmTestComponent: identical} + remoteStacks := helmRemoteStacksWith(identical) + filesIndex := newChangedFilesIndex(atmosConfig, []string{valuesFile}, tempDir) + + affected, err := processHelmComponentsIndexed( + helmTestStack, helmSection, &remoteStacks, &remoteStacks, + atmosConfig, filesIndex, newComponentPathPatternCache(), + false, false, false, + ) + require.NoError(t, err) + + require.Len(t, affected, 1) + assert.Equal(t, helmTestComponent, affected[0].Component) + assert.Equal(t, cfg.HelmComponentType, affected[0].ComponentType) + assert.Contains(t, affected[0].AffectedAll, affectedReasonStackValuesFile) +} + +func TestProcessHelmComponentsIndexed_UnrelatedFileDoesNotAffectValuesFile(t *testing.T) { + tempDir := t.TempDir() + atmosConfig := helmAtmosConfig() + atmosConfig.BasePath = tempDir + + identical := map[string]any{ + sectionNameChart: ".", + sectionNameValuesF: []string{"../../../config/helm/app-values.yaml"}, + } + helmSection := map[string]any{helmTestComponent: identical} + remoteStacks := helmRemoteStacksWith(identical) + filesIndex := newChangedFilesIndex(atmosConfig, []string{filepath.Join(tempDir, "config", "helm", "other-values.yaml")}, tempDir) + + affected, err := processHelmComponentsIndexed( + helmTestStack, helmSection, &remoteStacks, &remoteStacks, + atmosConfig, filesIndex, newComponentPathPatternCache(), + false, false, false, + ) + require.NoError(t, err) + assert.Empty(t, affected) +} + func TestProcessHelmComponentsIndexed_SkipsAbstractLockedAndInvalidSections(t *testing.T) { atmosConfig := helmAtmosConfig() remoteStacks := helmRemoteStacksWith(map[string]any{ diff --git a/internal/exec/describe_affected_optimizations_test.go b/internal/exec/describe_affected_optimizations_test.go index e8f8c729dc..254ba9dab9 100644 --- a/internal/exec/describe_affected_optimizations_test.go +++ b/internal/exec/describe_affected_optimizations_test.go @@ -2302,6 +2302,22 @@ func TestChangedFilesIndex_GetRelevantFiles_EdgeCases(t *testing.T) { }) } +func TestChangedFilesIndex_NativeHelmBasePath(t *testing.T) { + tempDir := t.TempDir() + atmosConfig := &schema.AtmosConfiguration{ + BasePath: tempDir, + Components: schema.Components{ + Helm: schema.Helm{BasePath: "components/helm"}, + }, + } + helmFile := filepath.Join(tempDir, "components", "helm", "app", "Chart.yaml") + unrelatedFile := filepath.Join(tempDir, "config", "helm", "app-values.yaml") + index := newChangedFilesIndex(atmosConfig, []string{helmFile, unrelatedFile}, tempDir) + + assert.Equal(t, []string{helmFile}, index.getRelevantFiles(cfg.HelmComponentType, atmosConfig)) + assert.True(t, index.isChangedFile(unrelatedFile)) +} + func TestComponentPathPatternCache_GetTerraformModulePatterns_EdgeCases(t *testing.T) { tempDir := t.TempDir() From c9ca2c25a3c54f786072342a178d185e8ef01cce Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Thu, 6 Aug 2026 03:17:49 +0400 Subject: [PATCH 53/62] fix: report Helm lifecycle and reverse delete order --- ...026-07-31-native-helm-release-lifecycle.md | 5 +++ docs/prd/native-helm-release-lifecycle.md | 1 + pkg/component/graph.go | 16 ++++++- pkg/component/graph_test.go | 17 ++++++++ pkg/component/helm/client.go | 2 + pkg/component/helm/executor.go | 23 +++++++++- pkg/component/helm/executor_bulk.go | 1 + pkg/component/helm/executor_test.go | 10 +++++ pkg/component/helm/lifecycle.go | 21 +++++++--- pkg/component/helm/lifecycle_test.go | 42 +++++++++++++++++++ pkg/component/helm/provision.go | 1 - .../docs/cli/commands/helm/helm-delete.mdx | 2 +- 12 files changed, 132 insertions(+), 9 deletions(-) diff --git a/docs/fixes/2026-07-31-native-helm-release-lifecycle.md b/docs/fixes/2026-07-31-native-helm-release-lifecycle.md index 7ac505960b..f7af35f5cc 100644 --- a/docs/fixes/2026-07-31-native-helm-release-lifecycle.md +++ b/docs/fixes/2026-07-31-native-helm-release-lifecycle.md @@ -9,6 +9,11 @@ state. Caller cancellation propagates through direct and dependency-ordered execution into install and upgrade actions and into delete wait and hook phases; Helm 4 does not expose a context-aware uninstall request. +Atmos reports the selected action and effective release policy before the Helm +action begins, including any `hookOnly` to `watcher` promotion required by +failure recovery. Bulk delete uses reverse dependency order so dependents are +removed before the releases they consume. + ## Migration notes - An omitted `release.timeout` remains `0s` (unbounded) for one minor release and emits a diff --git a/docs/prd/native-helm-release-lifecycle.md b/docs/prd/native-helm-release-lifecycle.md index 77aae75a2a..8fcf68ef82 100644 --- a/docs/prd/native-helm-release-lifecycle.md +++ b/docs/prd/native-helm-release-lifecycle.md @@ -369,6 +369,7 @@ The following rules apply: - A release that fails and is successfully rolled back still returns failure to the scheduler. - A rollback or uninstall failure preserves the original release failure and adds the recovery failure. - Dependents never run after timeout, failed readiness, failed hooks, failed rollback, or cancellation. +- Bulk delete traverses the selected graph in reverse topological order so dependents are removed before their dependencies. - A future mixed-kind scheduler consumes the same provider result; it must not reinterpret Helm readiness. ## Timeout Semantics diff --git a/pkg/component/graph.go b/pkg/component/graph.go index ece6af48e6..c3485a52f3 100644 --- a/pkg/component/graph.go +++ b/pkg/component/graph.go @@ -37,6 +37,7 @@ type GraphExecutionOptions struct { SubCommand string Flags map[string]any Selection *GraphSelection + ReverseOrder bool } // GraphNodeSkipObserver is implemented by providers that need to record graph @@ -65,7 +66,11 @@ func ExecuteGraph(ctx context.Context, opts *GraphExecutionOptions) error { return nil } - log.Info("Processing components in dependency order", "component_type", opts.ComponentType, "count", len(order)) + orderName := "dependency" + if opts.ReverseOrder { + orderName = "reverse_dependency" + } + log.Info("Processing components", "component_type", opts.ComponentType, "order", orderName, "count", len(order)) for i := range order { select { case <-ctx.Done(): @@ -127,9 +132,18 @@ func prepareExecutionOrder(opts *GraphExecutionOptions) (dependency.ExecutionOrd if err != nil { return nil, fmt.Errorf("%w: %w", errUtils.ErrTopologicalOrder, err) } + if opts.ReverseOrder { + reverseExecutionOrder(order) + } return order, nil } +func reverseExecutionOrder(order dependency.ExecutionOrder) { + for left, right := 0, len(order)-1; left < right; left, right = left+1, right-1 { + order[left], order[right] = order[right], order[left] + } +} + // executeGraphNode executes a single graph node through the component provider. func executeGraphNode(ctx context.Context, opts *GraphExecutionOptions, node *dependency.Node) error { nodeInfo := *opts.Info diff --git a/pkg/component/graph_test.go b/pkg/component/graph_test.go index 6eb080bde5..dd9ae1b7b9 100644 --- a/pkg/component/graph_test.go +++ b/pkg/component/graph_test.go @@ -164,6 +164,23 @@ func TestExecuteGraphRunsComponentsInDependencyOrder(t *testing.T) { } } +func TestExecuteGraphRunsDeleteInReverseDependencyOrder(t *testing.T) { + provider := &graphTestProvider{} + err := ExecuteGraph(context.Background(), &GraphExecutionOptions{ + Provider: provider, + Info: &schema.ConfigAndStacksInfo{}, + Stacks: graphTestStacks(), + ComponentType: cfg.KubernetesComponentType, + SubCommand: "delete", + ReverseOrder: true, + }) + + require.NoError(t, err) + require.Len(t, provider.calls, 4) + assertLessCallIndex(t, provider.calls, "api", "dev", "base", "dev") + assertLessCallIndex(t, provider.calls, "worker", "dev", "base", "prod") +} + func TestExecuteGraphNodeDoesNotRedispatchBulkSelection(t *testing.T) { provider := &graphTestProvider{} info := &schema.ConfigAndStacksInfo{ diff --git a/pkg/component/helm/client.go b/pkg/component/helm/client.go index 756fda9140..07f9a2f26a 100644 --- a/pkg/component/helm/client.go +++ b/pkg/component/helm/client.go @@ -85,6 +85,7 @@ func applyRelease(ctx context.Context, spec *chartSpec, dryRun bool) (releaseAct return releaseActionResult{Operation: releaseOperationInstall}, resolveErr } spec.Lifecycle = lifecycle + reportResolvedLifecycle(lifecycle) operationCtx, cancel := releaseOperationContext(ctx, lifecycle.Policy.Timeout) defer cancel() manifest, installErr := installRelease(operationCtx, actx, spec, dryRun) @@ -97,6 +98,7 @@ func applyRelease(ctx context.Context, spec *chartSpec, dryRun bool) (releaseAct return releaseActionResult{Operation: releaseOperationUpgrade}, resolveErr } spec.Lifecycle = lifecycle + reportResolvedLifecycle(lifecycle) operationCtx, cancel := releaseOperationContext(ctx, lifecycle.Policy.Timeout) defer cancel() manifest, upgradeErr := upgradeRelease(operationCtx, actx, spec, dryRun) diff --git a/pkg/component/helm/executor.go b/pkg/component/helm/executor.go index e11656a6b7..d250acc4d9 100644 --- a/pkg/component/helm/executor.go +++ b/pkg/component/helm/executor.go @@ -179,7 +179,7 @@ func runWithHooks( if err != nil { return err } - emitLifecycleWarnings(spec.Lifecycle.Warnings) + reportResolvedLifecycle(spec.Lifecycle) } if err := ctx.GoContext().Err(); err != nil { return err @@ -251,6 +251,27 @@ func emitLifecycleWarnings(warnings []lifecycleWarning) { } } +func reportResolvedLifecycle(resolution releaseLifecycleResolution) { + emitLifecycleWarnings(resolution.Warnings) + reason := "configured" + for _, warning := range resolution.Warnings { + if warning.Code == warningWaitDerived { + reason = warning.Message + break + } + } + policy := resolution.Policy + log.Debug("Resolved Helm release lifecycle", + "operation", policy.Operation, + "wait_strategy", policy.WaitStrategy, + "wait_strategy_reason", reason, + "wait_jobs", policy.WaitForJobs, + "on_failure", policy.OnFailure, + "timeout", policy.Timeout, + "timeout_field", resolution.TimeoutField, + ) +} + // runTemplate renders the chart and writes the manifests per the render options. func runTemplate(ctx *component.ExecutionContext, atmosConfig *schema.AtmosConfiguration, info *schema.ConfigAndStacksInfo, spec *chartSpec) ([]*unstructured.Unstructured, error) { objects, err := renderObjects(ctx.GoContext(), spec) diff --git a/pkg/component/helm/executor_bulk.go b/pkg/component/helm/executor_bulk.go index fcac97b456..1b6cb7be8c 100644 --- a/pkg/component/helm/executor_bulk.go +++ b/pkg/component/helm/executor_bulk.go @@ -58,6 +58,7 @@ func executeBulk( SubCommand: command, Flags: ctx.Flags, Selection: selection, + ReverseOrder: operation == OperationDelete, }) if collector != nil { runHelmAggregateCIHook(ctx, atmosConfig, info, collector.resultSet(), graphErr) diff --git a/pkg/component/helm/executor_test.go b/pkg/component/helm/executor_test.go index a74145fcaf..94d433893a 100644 --- a/pkg/component/helm/executor_test.go +++ b/pkg/component/helm/executor_test.go @@ -190,6 +190,16 @@ func TestExecuteBulkInitializesConfigAndGraph(t *testing.T) { assert.Equal(t, cfg.HelmComponentType, graphOpts.ComponentType) assert.Equal(t, "template", graphOpts.SubCommand) assert.Equal(t, ctx.Flags, graphOpts.Flags) + assert.False(t, graphOpts.ReverseOrder) + + graphOpts = nil + require.NoError(t, executeBulk(ctx, &schema.AtmosConfiguration{}, &schema.ConfigAndStacksInfo{ + All: true, + Stack: "dev", + SubCommand: "delete", + }, OperationDelete)) + require.NotNil(t, graphOpts) + assert.True(t, graphOpts.ReverseOrder) } func TestExecuteSingleSkipsDisabledComponent(t *testing.T) { diff --git a/pkg/component/helm/lifecycle.go b/pkg/component/helm/lifecycle.go index 71c0d3c155..0c1019fea6 100644 --- a/pkg/component/helm/lifecycle.go +++ b/pkg/component/helm/lifecycle.go @@ -325,6 +325,21 @@ func decodeDeletePolicy(releaseMap map[string]any) (deletePolicyInput, error) { } func resolveReleaseLifecycle(input releasePolicyInput, operation string, emitMigrationWarning bool) (releaseLifecycleResolution, error) { + resolution, err := resolveReleaseLifecycleBase(input, operation, emitMigrationWarning) + if err != nil { + return releaseLifecycleResolution{}, err + } + if err := validateAndDeriveLifecycle(&resolution); err != nil { + return releaseLifecycleResolution{}, err + } + return resolution, nil +} + +// resolveReleaseLifecycleBase applies configuration precedence without deriving +// cross-field values. Callers that overlay CLI flags must do so before the +// single validateAndDeriveLifecycle pass, otherwise a derived watcher strategy +// loses both its source hookOnly value and the explanation for the promotion. +func resolveReleaseLifecycleBase(input releasePolicyInput, operation string, emitMigrationWarning bool) (releaseLifecycleResolution, error) { resolution := releaseLifecycleResolution{ Policy: defaultReleasePolicy(operation), TimeoutField: "built-in default", @@ -359,9 +374,6 @@ func resolveReleaseLifecycle(input releasePolicyInput, operation string, emitMig Message: "helm release timeout is omitted; this release preserves 0s, but the default will become 5m in the next minor release", }) } - if err := validateAndDeriveLifecycle(&resolution); err != nil { - return releaseLifecycleResolution{}, err - } return resolution, nil } @@ -448,7 +460,7 @@ func validateAndDeriveLifecycle(resolution *releaseLifecycleResolution) error { // resolveReleaseLifecycleWithFlags resolves configuration for the selected // action, then overlays only explicitly supplied CLI values at highest priority. func resolveReleaseLifecycleWithFlags(input releasePolicyInput, operation string, flags map[string]any) (releaseLifecycleResolution, error) { - resolution, err := resolveReleaseLifecycle(input, operation, true) + resolution, err := resolveReleaseLifecycleBase(input, operation, true) if err != nil { return releaseLifecycleResolution{}, err } @@ -502,7 +514,6 @@ func resolveReleaseLifecycleWithFlags(input releasePolicyInput, operation string resolution.Policy.CleanupOnFailure = value } - resolution.Warnings = removeLifecycleWarning(resolution.Warnings, warningWaitDerived) if err := validateAndDeriveLifecycle(&resolution); err != nil { return releaseLifecycleResolution{}, err } diff --git a/pkg/component/helm/lifecycle_test.go b/pkg/component/helm/lifecycle_test.go index b717e44612..0b685a50b4 100644 --- a/pkg/component/helm/lifecycle_test.go +++ b/pkg/component/helm/lifecycle_test.go @@ -111,6 +111,48 @@ func TestResolveReleaseLifecycleDerivedWaitStrategy(t *testing.T) { } } +func TestResolveReleaseLifecycleWithFlagsReportsDerivedWaitStrategy(t *testing.T) { + input, err := decodeReleasePolicy(map[string]any{ + cfg.HelmReleaseSectionName: map[string]any{ + cfg.HelmWaitSectionName: map[string]any{ + cfg.HelmWaitStrategySectionName: "hookOnly", + }, + cfg.HelmUpgradeSectionName: map[string]any{ + cfg.HelmOnFailureSectionName: "rollback", + cfg.HelmWaitSectionName: map[string]any{ + cfg.HelmWaitJobsSectionName: true, + }, + }, + }, + }) + require.NoError(t, err) + + resolution, err := resolveReleaseLifecycleWithFlags(input, releaseOperationUpgrade, nil) + require.NoError(t, err) + assert.Equal(t, kube.StatusWatcherStrategy, resolution.Policy.WaitStrategy) + assert.True(t, resolution.Policy.WaitForJobs) + assert.True(t, hasLifecycleWarning(resolution.Warnings, warningWaitDerived)) +} + +func TestResolveReleaseLifecycleWithFlagsCanDisableDerivedWaitStrategy(t *testing.T) { + input, err := decodeReleasePolicy(map[string]any{ + cfg.HelmReleaseSectionName: map[string]any{ + cfg.HelmWaitSectionName: map[string]any{cfg.HelmWaitStrategySectionName: "hookOnly"}, + cfg.HelmUpgradeSectionName: map[string]any{ + cfg.HelmOnFailureSectionName: "rollback", + }, + }, + }) + require.NoError(t, err) + + resolution, err := resolveReleaseLifecycleWithFlags(input, releaseOperationUpgrade, map[string]any{ + cfg.HelmOnFailureSectionName: "keep", + }) + require.NoError(t, err) + assert.Equal(t, kube.HookOnlyStrategy, resolution.Policy.WaitStrategy) + assert.False(t, hasLifecycleWarning(resolution.Warnings, warningWaitDerived)) +} + func TestResolveReleaseLifecycleWithFlagsHighestPrecedence(t *testing.T) { input, err := decodeReleasePolicy(map[string]any{ cfg.HelmReleaseSectionName: map[string]any{ diff --git a/pkg/component/helm/provision.go b/pkg/component/helm/provision.go index 0c638e5d64..9fcc348d63 100644 --- a/pkg/component/helm/provision.go +++ b/pkg/component/helm/provision.go @@ -49,7 +49,6 @@ func deliverApply( if selected.Kind == target.KindKubernetes { result, err := applyHelmRelease(ctx, spec, info.DryRun) spec.Lifecycle = result.Lifecycle - emitLifecycleWarnings(result.Lifecycle.Warnings) summary["manifest_bytes"] = len(result.Manifest) summary["release"] = lifecycleSummary(result.Operation, result.Lifecycle.Policy) if objects, decodeErr := manifest.DecodeObjects([]byte(result.Manifest)); decodeErr == nil { diff --git a/website/docs/cli/commands/helm/helm-delete.mdx b/website/docs/cli/commands/helm/helm-delete.mdx index 0e1e4c8766..b05468c74b 100644 --- a/website/docs/cli/commands/helm/helm-delete.mdx +++ b/website/docs/cli/commands/helm/helm-delete.mdx @@ -32,7 +32,7 @@ atmos helm delete --all --tags production,tier-1
Atmos stack.
`--all` / `--affected` / `--include-dependents` (optional)
-
Process multiple Helm components in dependency order.
+
Process multiple Helm components in reverse dependency order, deleting dependents before their dependencies.
`--tags` / `--labels` (optional)
Filter by tags (comma-separated, matches any) or labels (comma-separated `key=value` or `key:value` pairs, matches all): `--tags=production,tier-1`, `--labels=cost-center=platform`. Compose with `--all`/`--affected` to narrow the selected set further; cannot be combined with a single component argument.
From 2be60fd7e081b9b4d1de79475452316b812485fc Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Thu, 6 Aug 2026 04:32:11 +0400 Subject: [PATCH 54/62] test: address Helm lifecycle review feedback --- examples/helm/atmos.yaml | 12 ++-- .../exec/describe_affected_components_test.go | 28 ++++++++ pkg/ci/plugins/helm/plugin_test.go | 19 +++--- pkg/ci/plugins/helm/templates/apply.md | 18 ++--- pkg/ci/plugins/helm/templates/delete.md | 10 +-- pkg/component/helm/lifecycle_test.go | 67 ++++++++----------- website/docs/cli/commands/helm/helm-apply.mdx | 2 +- 7 files changed, 90 insertions(+), 66 deletions(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 4acd562e5b..a3fdeb421f 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -42,14 +42,16 @@ commands: - atmos validate stacks # Dependency acquisition is explicit: the opt-in flag fetches the missing # file:// library and updates the local chart before the first render. - - atmos helm template demo -s dev --dependency-update --output=/tmp/atmos-helm-template.yaml # Template output includes weighted hooks and leaves Helm `tpl` expressions # in values for the chart to evaluate with its native .Values context. - command: >- - test "$(grep -c '^ name: demo-hook-order$' /tmp/atmos-helm-template.yaml)" -eq 2 && - grep -q 'helm.sh/hook-weight: "-2"' /tmp/atmos-helm-template.yaml && - grep -q 'helm.sh/hook-weight: "-1"' /tmp/atmos-helm-template.yaml && - grep -q 'rendered: from-stack' /tmp/atmos-helm-template.yaml + artifact=$(mktemp ./atmos-helm-template.XXXXXX.yaml); + trap 'rm -f "$artifact"' EXIT; + atmos helm template demo -s dev --dependency-update --output="$artifact" && + test "$(grep -c '^ name: demo-hook-order$' "$artifact")" -eq 2 && + grep -q 'helm.sh/hook-weight: "-2"' "$artifact" && + grep -q 'helm.sh/hook-weight: "-1"' "$artifact" && + grep -q 'rendered: from-stack' "$artifact" - command: atmos emulator up kubernetes -s dev retry: max_attempts: 2 diff --git a/internal/exec/describe_affected_components_test.go b/internal/exec/describe_affected_components_test.go index c0ec654049..ffac1b4d94 100644 --- a/internal/exec/describe_affected_components_test.go +++ b/internal/exec/describe_affected_components_test.go @@ -645,6 +645,34 @@ func TestProcessHelmComponentsIndexed_ValuesFileChangedOutsideComponentBasePath( assert.Contains(t, affected[0].AffectedAll, affectedReasonStackValuesFile) } +func TestProcessHelmComponentsIndexed_AbsoluteValuesFileChanged(t *testing.T) { + tempDir := t.TempDir() + atmosConfig := helmAtmosConfig() + atmosConfig.BasePath = tempDir + + valuesFile := filepath.Join(tempDir, "config", "helm", "app-values.yaml") + identical := map[string]any{ + cfg.ComponentSectionName: "shared-chart", + sectionNameChart: ".", + sectionNameValuesF: []any{valuesFile}, + } + helmSection := map[string]any{helmTestComponent: identical} + remoteStacks := helmRemoteStacksWith(identical) + filesIndex := newChangedFilesIndex(atmosConfig, []string{valuesFile}, tempDir) + + affected, err := processHelmComponentsIndexed( + helmTestStack, helmSection, &remoteStacks, &remoteStacks, + atmosConfig, filesIndex, newComponentPathPatternCache(), + false, false, false, + ) + require.NoError(t, err) + + require.Len(t, affected, 1) + assert.Equal(t, helmTestComponent, affected[0].Component) + assert.Equal(t, cfg.HelmComponentType, affected[0].ComponentType) + assert.Contains(t, affected[0].AffectedAll, affectedReasonStackValuesFile) +} + func TestProcessHelmComponentsIndexed_UnrelatedFileDoesNotAffectValuesFile(t *testing.T) { tempDir := t.TempDir() atmosConfig := helmAtmosConfig() diff --git a/pkg/ci/plugins/helm/plugin_test.go b/pkg/ci/plugins/helm/plugin_test.go index 6fb1a99dba..a1fed9087a 100644 --- a/pkg/ci/plugins/helm/plugin_test.go +++ b/pkg/ci/plugins/helm/plugin_test.go @@ -264,8 +264,9 @@ func TestTemplateRendering(t *testing.T) { "history": map[string]any{"max": 10}, }, contains: []string{ - "Helm Apply Summary", "bitnami/nginx", "Deployment", "Release lifecycle", - "| Wait strategy | `watcher` |", + "Helm Apply Summary", "bitnami/nginx", "Deployment", "\n### Release lifecycle\n", + "\n| Operation | `upgrade` |\n", + "\n| Wait strategy | `watcher` |\n", "| Timeout | `30m0s` |", "| Chart hooks enabled | `true` |", "| Wait for Jobs | `true` |", @@ -286,8 +287,8 @@ func TestTemplateRendering(t *testing.T) { "crds": "create", }, contains: []string{ - "Helm Apply Summary", "Release lifecycle", - "| Operation | `install` |", + "Helm Apply Summary", "\n### Release lifecycle\n", + "\n| Operation | `install` |\n", "| Wait strategy | `hookOnly` |", "| Timeout | `5m0s` |", "| Chart hooks enabled | `true` |", @@ -305,7 +306,8 @@ func TestTemplateRendering(t *testing.T) { }, contains: []string{ "Helm Apply Summary", - "| Applied | `false` |", + "\n### Release lifecycle\n", + "\n| Applied | `false` |\n", "| Target kind | `git` |", "| Reason | `external_target` |", }, @@ -321,8 +323,8 @@ func TestTemplateRendering(t *testing.T) { "chart_hooks": false, }, contains: []string{ - "Helm Delete Summary", "Release lifecycle", - "| Operation | `delete` |", + "Helm Delete Summary", "\n### Release lifecycle\n", + "\n| Operation | `delete` |\n", "| Wait strategy | `legacy` |", "| Timeout | `10m0s` |", "| Chart hooks enabled | `false` |", @@ -336,7 +338,8 @@ func TestTemplateRendering(t *testing.T) { }, contains: []string{ "Helm Delete Summary", - "| Deleted | `false` |", + "\n### Release lifecycle\n", + "\n| Deleted | `false` |\n", "| Target kind | `git` |", "| Reason | `external_target` |", }, diff --git a/pkg/ci/plugins/helm/templates/apply.md b/pkg/ci/plugins/helm/templates/apply.md index f7dc6e6ced..f1957747c2 100644 --- a/pkg/ci/plugins/helm/templates/apply.md +++ b/pkg/ci/plugins/helm/templates/apply.md @@ -14,11 +14,11 @@ | Objects | `{{ .ObjectCount }}` | | Manifest bytes | `{{ .ManifestBytes }}` | -{{- with .Lifecycle }} +{{ with .Lifecycle }} ### Release lifecycle -{{- if eq (index . "reason") "external_target" }} +{{ if eq (index . "reason") "external_target" }} | Field | Value | | --- | --- | @@ -26,7 +26,7 @@ | Target kind | `{{ index . "target_kind" }}` | | Reason | `external_target` | -{{- else }} +{{ else }} | Field | Value | | --- | --- | @@ -37,20 +37,20 @@ | Wait for Jobs | `{{ index (index . "wait") "jobs" }}` | | On failure | `{{ index . "on_failure" }}` | -{{- if eq (index . "operation") "install" }} +{{ if eq (index . "operation") "install" }} | Install CRDs | `{{ index . "crds" }}` | -{{- end }} +{{ end }} -{{- if eq (index . "operation") "upgrade" }} +{{ if eq (index . "operation") "upgrade" }} | Cleanup on failure | `{{ index . "cleanup_on_failure" }}` | | Maximum history | `{{ index (index . "history") "max" }}` | -{{- end }} +{{ end }} -{{- end }} +{{ end }} -{{- end }} +{{ end }} To reproduce locally: diff --git a/pkg/ci/plugins/helm/templates/delete.md b/pkg/ci/plugins/helm/templates/delete.md index 0a25629ded..c491cb3a22 100644 --- a/pkg/ci/plugins/helm/templates/delete.md +++ b/pkg/ci/plugins/helm/templates/delete.md @@ -10,11 +10,11 @@ | Release | `{{ .ReleaseName }}` | | Namespace | `{{ .Namespace }}` | -{{- with .Lifecycle }} +{{ with .Lifecycle }} ### Release lifecycle -{{- if eq (index . "reason") "external_target" }} +{{ if eq (index . "reason") "external_target" }} | Field | Value | | --- | --- | @@ -22,7 +22,7 @@ | Target kind | `{{ index . "target_kind" }}` | | Reason | `external_target` | -{{- else }} +{{ else }} | Field | Value | | --- | --- | @@ -31,9 +31,9 @@ | Timeout | `{{ index . "timeout" }}` | | Chart hooks enabled | `{{ index . "chart_hooks" }}` | -{{- end }} +{{ end }} -{{- end }} +{{ end }} To reproduce locally: diff --git a/pkg/component/helm/lifecycle_test.go b/pkg/component/helm/lifecycle_test.go index 0b685a50b4..ded1b3af66 100644 --- a/pkg/component/helm/lifecycle_test.go +++ b/pkg/component/helm/lifecycle_test.go @@ -111,46 +111,37 @@ func TestResolveReleaseLifecycleDerivedWaitStrategy(t *testing.T) { } } -func TestResolveReleaseLifecycleWithFlagsReportsDerivedWaitStrategy(t *testing.T) { - input, err := decodeReleasePolicy(map[string]any{ - cfg.HelmReleaseSectionName: map[string]any{ - cfg.HelmWaitSectionName: map[string]any{ - cfg.HelmWaitStrategySectionName: "hookOnly", - }, - cfg.HelmUpgradeSectionName: map[string]any{ - cfg.HelmOnFailureSectionName: "rollback", - cfg.HelmWaitSectionName: map[string]any{ - cfg.HelmWaitJobsSectionName: true, +func TestResolveReleaseLifecycleWithFlagsDerivesWaitStrategyAfterFlagOverlay(t *testing.T) { + for _, tt := range []struct { + name string + configuredFailure string + flags map[string]any + wantStrategy kube.WaitStrategy + wantDerived bool + }{ + {name: "configured rollback", configuredFailure: "rollback", wantStrategy: kube.StatusWatcherStrategy, wantDerived: true}, + {name: "flag rollback", flags: map[string]any{cfg.HelmOnFailureSectionName: "rollback"}, wantStrategy: kube.StatusWatcherStrategy, wantDerived: true}, + {name: "flag keep disables configured rollback", configuredFailure: "rollback", flags: map[string]any{cfg.HelmOnFailureSectionName: "keep"}, wantStrategy: kube.HookOnlyStrategy}, + } { + t.Run(tt.name, func(t *testing.T) { + upgrade := map[string]any{} + if tt.configuredFailure != "" { + upgrade[cfg.HelmOnFailureSectionName] = tt.configuredFailure + } + input, err := decodeReleasePolicy(map[string]any{ + cfg.HelmReleaseSectionName: map[string]any{ + cfg.HelmWaitSectionName: map[string]any{cfg.HelmWaitStrategySectionName: "hookOnly"}, + cfg.HelmUpgradeSectionName: upgrade, }, - }, - }, - }) - require.NoError(t, err) - - resolution, err := resolveReleaseLifecycleWithFlags(input, releaseOperationUpgrade, nil) - require.NoError(t, err) - assert.Equal(t, kube.StatusWatcherStrategy, resolution.Policy.WaitStrategy) - assert.True(t, resolution.Policy.WaitForJobs) - assert.True(t, hasLifecycleWarning(resolution.Warnings, warningWaitDerived)) -} - -func TestResolveReleaseLifecycleWithFlagsCanDisableDerivedWaitStrategy(t *testing.T) { - input, err := decodeReleasePolicy(map[string]any{ - cfg.HelmReleaseSectionName: map[string]any{ - cfg.HelmWaitSectionName: map[string]any{cfg.HelmWaitStrategySectionName: "hookOnly"}, - cfg.HelmUpgradeSectionName: map[string]any{ - cfg.HelmOnFailureSectionName: "rollback", - }, - }, - }) - require.NoError(t, err) + }) + require.NoError(t, err) - resolution, err := resolveReleaseLifecycleWithFlags(input, releaseOperationUpgrade, map[string]any{ - cfg.HelmOnFailureSectionName: "keep", - }) - require.NoError(t, err) - assert.Equal(t, kube.HookOnlyStrategy, resolution.Policy.WaitStrategy) - assert.False(t, hasLifecycleWarning(resolution.Warnings, warningWaitDerived)) + resolution, err := resolveReleaseLifecycleWithFlags(input, releaseOperationUpgrade, tt.flags) + require.NoError(t, err) + assert.Equal(t, tt.wantStrategy, resolution.Policy.WaitStrategy) + assert.Equal(t, tt.wantDerived, hasLifecycleWarning(resolution.Warnings, warningWaitDerived)) + }) + } } func TestResolveReleaseLifecycleWithFlagsHighestPrecedence(t *testing.T) { diff --git a/website/docs/cli/commands/helm/helm-apply.mdx b/website/docs/cli/commands/helm/helm-apply.mdx index 4c35b5de90..8695a38856 100644 --- a/website/docs/cli/commands/helm/helm-apply.mdx +++ b/website/docs/cli/commands/helm/helm-apply.mdx @@ -26,7 +26,7 @@ Install or upgrade the release in the cluster: atmos helm apply monitoring -s plat-ue2-dev ``` -Apply an explicit production readiness and recovery policy: +Upgrade an existing release with an explicit production readiness and recovery policy: ```shell atmos helm apply monitoring -s plat-ue2-dev \ From fec8503891d37cb49aba7d821ff12f4891138c68 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Thu, 6 Aug 2026 04:48:03 +0400 Subject: [PATCH 55/62] test: cover portable Helm smoke paths --- examples/helm/atmos.yaml | 2 +- .../exec/describe_affected_components_test.go | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index a3fdeb421f..5efbdefb53 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -45,7 +45,7 @@ commands: # Template output includes weighted hooks and leaves Helm `tpl` expressions # in values for the chart to evaluate with its native .Values context. - command: >- - artifact=$(mktemp ./atmos-helm-template.XXXXXX.yaml); + artifact=$(mktemp ./atmos-helm-template.XXXXXX); trap 'rm -f "$artifact"' EXIT; atmos helm template demo -s dev --dependency-update --output="$artifact" && test "$(grep -c '^ name: demo-hook-order$' "$artifact")" -eq 2 && diff --git a/internal/exec/describe_affected_components_test.go b/internal/exec/describe_affected_components_test.go index ffac1b4d94..8794554db8 100644 --- a/internal/exec/describe_affected_components_test.go +++ b/internal/exec/describe_affected_components_test.go @@ -645,6 +645,39 @@ func TestProcessHelmComponentsIndexed_ValuesFileChangedOutsideComponentBasePath( assert.Contains(t, affected[0].AffectedAll, affectedReasonStackValuesFile) } +func TestProcessHelmComponentsIndexed_ScalarValuesFileChanged(t *testing.T) { + tempDir := t.TempDir() + atmosConfig := helmAtmosConfig() + atmosConfig.BasePath = tempDir + + componentFolder := "shared-chart" + componentPath := filepath.Join(tempDir, "components", "helm", componentFolder) + valuesFile := filepath.Join(tempDir, "config", "helm", "app-values.yaml") + valuesRef, err := filepath.Rel(componentPath, valuesFile) + require.NoError(t, err) + + identical := map[string]any{ + cfg.ComponentSectionName: componentFolder, + sectionNameChart: ".", + sectionNameValuesF: valuesRef, + } + helmSection := map[string]any{helmTestComponent: identical} + remoteStacks := helmRemoteStacksWith(identical) + filesIndex := newChangedFilesIndex(atmosConfig, []string{valuesFile}, tempDir) + + affected, err := processHelmComponentsIndexed( + helmTestStack, helmSection, &remoteStacks, &remoteStacks, + atmosConfig, filesIndex, newComponentPathPatternCache(), + false, false, false, + ) + require.NoError(t, err) + + require.Len(t, affected, 1) + assert.Equal(t, helmTestComponent, affected[0].Component) + assert.Equal(t, cfg.HelmComponentType, affected[0].ComponentType) + assert.Contains(t, affected[0].AffectedAll, affectedReasonStackValuesFile) +} + func TestProcessHelmComponentsIndexed_AbsoluteValuesFileChanged(t *testing.T) { tempDir := t.TempDir() atmosConfig := helmAtmosConfig() From 63d870fae7c433a8ded650b529f72592487e4733 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Thu, 6 Aug 2026 05:08:45 +0400 Subject: [PATCH 56/62] docs: align Helm lifecycle review details --- .../exec/describe_affected_components_test.go | 138 +++++------------- pkg/ci/plugins/helm/plugin_test.go | 10 +- pkg/ci/plugins/helm/templates/apply.md | 10 +- .../cli/configuration/components/helm.mdx | 12 +- 4 files changed, 52 insertions(+), 118 deletions(-) diff --git a/internal/exec/describe_affected_components_test.go b/internal/exec/describe_affected_components_test.go index 8794554db8..fb89b75d1c 100644 --- a/internal/exec/describe_affected_components_test.go +++ b/internal/exec/describe_affected_components_test.go @@ -612,7 +612,7 @@ func TestProcessHelmComponentsIndexed_FolderChanged(t *testing.T) { assert.Contains(t, affected[0].AffectedAll, affectedReasonComponent) } -func TestProcessHelmComponentsIndexed_ValuesFileChangedOutsideComponentBasePath(t *testing.T) { +func TestProcessHelmComponentsIndexed_ValuesFilesChanged(t *testing.T) { tempDir := t.TempDir() atmosConfig := helmAtmosConfig() atmosConfig.BasePath = tempDir @@ -623,109 +623,45 @@ func TestProcessHelmComponentsIndexed_ValuesFileChangedOutsideComponentBasePath( valuesRef, err := filepath.Rel(componentPath, valuesFile) require.NoError(t, err) - identical := map[string]any{ - cfg.ComponentSectionName: componentFolder, - sectionNameChart: ".", - sectionNameValuesF: []any{valuesRef}, - } - helmSection := map[string]any{helmTestComponent: identical} - remoteStacks := helmRemoteStacksWith(identical) - filesIndex := newChangedFilesIndex(atmosConfig, []string{valuesFile}, tempDir) - - affected, err := processHelmComponentsIndexed( - helmTestStack, helmSection, &remoteStacks, &remoteStacks, - atmosConfig, filesIndex, newComponentPathPatternCache(), - false, false, false, - ) - require.NoError(t, err) - - require.Len(t, affected, 1) - assert.Equal(t, helmTestComponent, affected[0].Component) - assert.Equal(t, cfg.HelmComponentType, affected[0].ComponentType) - assert.Contains(t, affected[0].AffectedAll, affectedReasonStackValuesFile) -} - -func TestProcessHelmComponentsIndexed_ScalarValuesFileChanged(t *testing.T) { - tempDir := t.TempDir() - atmosConfig := helmAtmosConfig() - atmosConfig.BasePath = tempDir - - componentFolder := "shared-chart" - componentPath := filepath.Join(tempDir, "components", "helm", componentFolder) - valuesFile := filepath.Join(tempDir, "config", "helm", "app-values.yaml") - valuesRef, err := filepath.Rel(componentPath, valuesFile) - require.NoError(t, err) - - identical := map[string]any{ - cfg.ComponentSectionName: componentFolder, - sectionNameChart: ".", - sectionNameValuesF: valuesRef, - } - helmSection := map[string]any{helmTestComponent: identical} - remoteStacks := helmRemoteStacksWith(identical) - filesIndex := newChangedFilesIndex(atmosConfig, []string{valuesFile}, tempDir) - - affected, err := processHelmComponentsIndexed( - helmTestStack, helmSection, &remoteStacks, &remoteStacks, - atmosConfig, filesIndex, newComponentPathPatternCache(), - false, false, false, - ) - require.NoError(t, err) - - require.Len(t, affected, 1) - assert.Equal(t, helmTestComponent, affected[0].Component) - assert.Equal(t, cfg.HelmComponentType, affected[0].ComponentType) - assert.Contains(t, affected[0].AffectedAll, affectedReasonStackValuesFile) -} - -func TestProcessHelmComponentsIndexed_AbsoluteValuesFileChanged(t *testing.T) { - tempDir := t.TempDir() - atmosConfig := helmAtmosConfig() - atmosConfig.BasePath = tempDir - - valuesFile := filepath.Join(tempDir, "config", "helm", "app-values.yaml") - identical := map[string]any{ - cfg.ComponentSectionName: "shared-chart", - sectionNameChart: ".", - sectionNameValuesF: []any{valuesFile}, - } - helmSection := map[string]any{helmTestComponent: identical} - remoteStacks := helmRemoteStacksWith(identical) - filesIndex := newChangedFilesIndex(atmosConfig, []string{valuesFile}, tempDir) - - affected, err := processHelmComponentsIndexed( - helmTestStack, helmSection, &remoteStacks, &remoteStacks, - atmosConfig, filesIndex, newComponentPathPatternCache(), - false, false, false, - ) - require.NoError(t, err) - - require.Len(t, affected, 1) - assert.Equal(t, helmTestComponent, affected[0].Component) - assert.Equal(t, cfg.HelmComponentType, affected[0].ComponentType) - assert.Contains(t, affected[0].AffectedAll, affectedReasonStackValuesFile) -} + for _, tt := range []struct { + name string + valuesFiles any + changedFile string + wantAffected bool + }{ + {name: "relative list", valuesFiles: []any{valuesRef}, changedFile: valuesFile, wantAffected: true}, + {name: "scalar", valuesFiles: valuesRef, changedFile: valuesFile, wantAffected: true}, + {name: "absolute", valuesFiles: []any{valuesFile}, changedFile: valuesFile, wantAffected: true}, + {name: "unrelated", valuesFiles: []string{valuesRef}, changedFile: filepath.Join(tempDir, "config", "helm", "other-values.yaml")}, + } { + t.Run(tt.name, func(t *testing.T) { + identical := map[string]any{ + cfg.ComponentSectionName: componentFolder, + sectionNameChart: ".", + sectionNameValuesF: tt.valuesFiles, + } + helmSection := map[string]any{helmTestComponent: identical} + remoteStacks := helmRemoteStacksWith(identical) + filesIndex := newChangedFilesIndex(atmosConfig, []string{tt.changedFile}, tempDir) + + affected, err := processHelmComponentsIndexed( + helmTestStack, helmSection, &remoteStacks, &remoteStacks, + atmosConfig, filesIndex, newComponentPathPatternCache(), + false, false, false, + ) + require.NoError(t, err) -func TestProcessHelmComponentsIndexed_UnrelatedFileDoesNotAffectValuesFile(t *testing.T) { - tempDir := t.TempDir() - atmosConfig := helmAtmosConfig() - atmosConfig.BasePath = tempDir + if !tt.wantAffected { + assert.Empty(t, affected) + return + } - identical := map[string]any{ - sectionNameChart: ".", - sectionNameValuesF: []string{"../../../config/helm/app-values.yaml"}, + require.Len(t, affected, 1) + assert.Equal(t, helmTestComponent, affected[0].Component) + assert.Equal(t, cfg.HelmComponentType, affected[0].ComponentType) + assert.Contains(t, affected[0].AffectedAll, affectedReasonStackValuesFile) + }) } - helmSection := map[string]any{helmTestComponent: identical} - remoteStacks := helmRemoteStacksWith(identical) - filesIndex := newChangedFilesIndex(atmosConfig, []string{filepath.Join(tempDir, "config", "helm", "other-values.yaml")}, tempDir) - - affected, err := processHelmComponentsIndexed( - helmTestStack, helmSection, &remoteStacks, &remoteStacks, - atmosConfig, filesIndex, newComponentPathPatternCache(), - false, false, false, - ) - require.NoError(t, err) - assert.Empty(t, affected) } func TestProcessHelmComponentsIndexed_SkipsAbstractLockedAndInvalidSections(t *testing.T) { diff --git a/pkg/ci/plugins/helm/plugin_test.go b/pkg/ci/plugins/helm/plugin_test.go index a1fed9087a..73b09aea31 100644 --- a/pkg/ci/plugins/helm/plugin_test.go +++ b/pkg/ci/plugins/helm/plugin_test.go @@ -270,9 +270,9 @@ func TestTemplateRendering(t *testing.T) { "| Timeout | `30m0s` |", "| Chart hooks enabled | `true` |", "| Wait for Jobs | `true` |", - "| On failure | `rollback` |", - "| Cleanup on failure | `true` |", - "| Maximum history | `10` |", + "| On failure | `rollback` |\n" + + "| Cleanup on failure | `true` |\n" + + "| Maximum history | `10` |", }, }, { @@ -293,8 +293,8 @@ func TestTemplateRendering(t *testing.T) { "| Timeout | `5m0s` |", "| Chart hooks enabled | `true` |", "| Wait for Jobs | `false` |", - "| On failure | `keep` |", - "| Install CRDs | `create` |", + "| On failure | `keep` |\n" + + "| Install CRDs | `create` |", }, notContains: []string{"Maximum history"}, }, diff --git a/pkg/ci/plugins/helm/templates/apply.md b/pkg/ci/plugins/helm/templates/apply.md index f1957747c2..e9c7127de9 100644 --- a/pkg/ci/plugins/helm/templates/apply.md +++ b/pkg/ci/plugins/helm/templates/apply.md @@ -36,16 +36,12 @@ | Chart hooks enabled | `{{ index . "chart_hooks" }}` | | Wait for Jobs | `{{ index (index . "wait") "jobs" }}` | | On failure | `{{ index . "on_failure" }}` | - -{{ if eq (index . "operation") "install" }} +{{ if eq (index . "operation") "install" -}} | Install CRDs | `{{ index . "crds" }}` | - -{{ end }} - -{{ if eq (index . "operation") "upgrade" }} +{{ end -}} +{{ if eq (index . "operation") "upgrade" -}} | Cleanup on failure | `{{ index . "cleanup_on_failure" }}` | | Maximum history | `{{ index (index . "history") "max" }}` | - {{ end }} {{ end }} diff --git a/website/docs/cli/configuration/components/helm.mdx b/website/docs/cli/configuration/components/helm.mdx index 429a55f9fb..388059f675 100644 --- a/website/docs/cli/configuration/components/helm.mdx +++ b/website/docs/cli/configuration/components/helm.mdx @@ -55,11 +55,13 @@ components:
:::info Release policy belongs in stacks -Helm 4 lifecycle fields such as `wait_strategy`, `timeout`, -`on_failure`, and `max_history` are stack configuration, not -project-wide `components.helm` settings in `atmos.yaml`. This keeps release -policy subject to stack imports, component inheritance, and environment-specific -overrides. See [Helm stack configuration](/stacks/components/helm#release-lifecycle). +Helm 4 lifecycle fields such as `release.wait.strategy`, `release.wait.jobs`, +`release.timeout`, `release.history.max`, and operation-specific +`release.install.on_failure` or `release.upgrade.on_failure` are stack +configuration, not project-wide `components.helm` settings in `atmos.yaml`. +This keeps release policy subject to stack imports, component inheritance, and +environment-specific overrides. See +[Helm stack configuration](/stacks/components/helm#release-lifecycle). ::: ## Helm Repositories From 8dee762a93ba607be0ecefc977eba404c8e654c9 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Thu, 6 Aug 2026 05:19:28 +0400 Subject: [PATCH 57/62] fix: keep Helm lifecycle tables lintable --- pkg/ci/plugins/helm/templates/apply.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pkg/ci/plugins/helm/templates/apply.md b/pkg/ci/plugins/helm/templates/apply.md index e9c7127de9..1e9e48bc39 100644 --- a/pkg/ci/plugins/helm/templates/apply.md +++ b/pkg/ci/plugins/helm/templates/apply.md @@ -28,6 +28,8 @@ {{ else }} +{{ if eq (index . "operation") "install" }} + | Field | Value | | --- | --- | | Operation | `{{ index . "operation" }}` | @@ -36,12 +38,21 @@ | Chart hooks enabled | `{{ index . "chart_hooks" }}` | | Wait for Jobs | `{{ index (index . "wait") "jobs" }}` | | On failure | `{{ index . "on_failure" }}` | -{{ if eq (index . "operation") "install" -}} | Install CRDs | `{{ index . "crds" }}` | -{{ end -}} -{{ if eq (index . "operation") "upgrade" -}} + +{{ else }} + +| Field | Value | +| --- | --- | +| Operation | `{{ index . "operation" }}` | +| Wait strategy | `{{ index (index . "wait") "strategy" }}` | +| Timeout | `{{ index . "timeout" }}` | +| Chart hooks enabled | `{{ index . "chart_hooks" }}` | +| Wait for Jobs | `{{ index (index . "wait") "jobs" }}` | +| On failure | `{{ index . "on_failure" }}` | | Cleanup on failure | `{{ index . "cleanup_on_failure" }}` | | Maximum history | `{{ index (index . "history") "max" }}` | + {{ end }} {{ end }} From 1c61fdeb80efec410c5c328dd347cc1e8fa38439 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Thu, 6 Aug 2026 06:03:19 +0400 Subject: [PATCH 58/62] test: cover typed Helm values file lists --- internal/exec/describe_affected_components_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/exec/describe_affected_components_test.go b/internal/exec/describe_affected_components_test.go index fb89b75d1c..3c3045ef03 100644 --- a/internal/exec/describe_affected_components_test.go +++ b/internal/exec/describe_affected_components_test.go @@ -630,6 +630,7 @@ func TestProcessHelmComponentsIndexed_ValuesFilesChanged(t *testing.T) { wantAffected bool }{ {name: "relative list", valuesFiles: []any{valuesRef}, changedFile: valuesFile, wantAffected: true}, + {name: "typed relative list", valuesFiles: []string{valuesRef}, changedFile: valuesFile, wantAffected: true}, {name: "scalar", valuesFiles: valuesRef, changedFile: valuesFile, wantAffected: true}, {name: "absolute", valuesFiles: []any{valuesFile}, changedFile: valuesFile, wantAffected: true}, {name: "unrelated", valuesFiles: []string{valuesRef}, changedFile: filepath.Join(tempDir, "config", "helm", "other-values.yaml")}, From 63ff5bd63461fadb70a3aa8a127200a2d975c3ba Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Thu, 6 Aug 2026 06:38:22 +0400 Subject: [PATCH 59/62] test: stabilize Helm Job teardown --- examples/helm/stacks/deploy/dev.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index 098f97f13c..60c30b0106 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -73,6 +73,12 @@ components: release: wait: jobs: true + # Helm's watcher can remain blocked after deleting a completed ordinary + # Job even though the release resources are already gone. Keep watcher + # coverage for install and use deterministic legacy cleanup here. + delete: + wait: + strategy: legacy values: hooks: enabled: false From ad465c188d670d6ee5041bd7953e26bc908e0389 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Thu, 6 Aug 2026 07:19:51 +0400 Subject: [PATCH 60/62] test: stabilize k3s Helm teardown strategy --- examples/helm/stacks/deploy/dev.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index 60c30b0106..daab8f0b31 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -6,6 +6,12 @@ helm: release: wait: strategy: watcher + # Nested k3s runners can leave Helm's watcher blocked after release + # resources are gone. Live deletes use deterministic legacy waiting; the + # workflow still exercises watcher selection through its delete dry-run. + delete: + wait: + strategy: legacy # Leave enough headroom for nested k3s workloads on resource-constrained # macOS/Colima runners. Components that exercise timeout behavior override # this value explicitly below. @@ -73,12 +79,6 @@ components: release: wait: jobs: true - # Helm's watcher can remain blocked after deleting a completed ordinary - # Job even though the release resources are already gone. Keep watcher - # coverage for install and use deterministic legacy cleanup here. - delete: - wait: - strategy: legacy values: hooks: enabled: false From 784d37a96acfd25202c003c76deb270f429c2b5d Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Thu, 6 Aug 2026 08:21:47 +0400 Subject: [PATCH 61/62] test: allow Helm timeout recovery to finish --- examples/helm/stacks/deploy/dev.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index daab8f0b31..b23e81fa70 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -115,7 +115,9 @@ components: name: demo-timeout namespace: demo-timeout release: - timeout: 2s + # Keep the lifecycle deadline below the readiness delay while leaving + # enough time for recovery to finish on resource-constrained runners. + timeout: 15s wait: strategy: watcher install: @@ -124,7 +126,7 @@ components: on_failure: rollback values: deployment: - readinessDelaySeconds: 20 + readinessDelaySeconds: 60 hooks: enabled: false From ed44f4a4ea3f029504266360afe349b5df790929 Mon Sep 17 00:00:00 2001 From: Mikhail Shirkov Date: Thu, 6 Aug 2026 20:00:41 +0400 Subject: [PATCH 62/62] refactor: isolate Helm lifecycle test scenario --- .github/workflows/test.yml | 17 +- .pre-commit-config.yaml | 2 +- examples/helm/README.md | 22 +- examples/helm/atmos.yaml | 121 +---------- examples/helm/components/helm/demo/Chart.yaml | 4 - .../helm/demo/templates/deployment.yaml | 10 - .../helm/components/helm/demo/values.yaml | 19 -- examples/helm/stacks/deploy/dev.yaml | 162 -------------- .../scenarios/helm-lifecycle/README.md | 8 + .../scenarios/helm-lifecycle/atmos.yaml | 178 +++++++++++++++ .../components/helm/demo/.gitignore | 0 .../components/helm/demo/Chart.lock | 0 .../components/helm/demo/Chart.yaml | 10 + .../crds/lifecycle.atmos.test_widgets.yaml | 0 .../demo/templates/dependency-observed.yaml | 0 .../helm/demo/templates/deployment.yaml | 32 +++ .../helm/demo/templates/extra-configmap.yaml | 0 .../helm/demo/templates/failing-hook-job.yaml | 0 .../demo/templates/hook-order-configmap.yaml | 0 .../helm/demo/templates/hook-order-job.yaml | 0 .../components/helm/demo/templates/job.yaml | 0 .../helm/demo/templates/service.yaml | 12 ++ .../helm/demo/templates/tpl-configmap.yaml | 0 .../components/helm/demo/values.yaml | 27 +++ .../helm/helm-test-library/Chart.yaml | 0 .../helm-test-library/templates/_render.tpl | 0 .../helm-lifecycle/stacks/deploy/dev.yaml | 202 ++++++++++++++++++ 27 files changed, 484 insertions(+), 342 deletions(-) create mode 100644 tests/fixtures/scenarios/helm-lifecycle/README.md create mode 100644 tests/fixtures/scenarios/helm-lifecycle/atmos.yaml rename {examples/helm => tests/fixtures/scenarios/helm-lifecycle}/components/helm/demo/.gitignore (100%) rename {examples/helm => tests/fixtures/scenarios/helm-lifecycle}/components/helm/demo/Chart.lock (100%) create mode 100644 tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/Chart.yaml rename {examples/helm => tests/fixtures/scenarios/helm-lifecycle}/components/helm/demo/crds/lifecycle.atmos.test_widgets.yaml (100%) rename {examples/helm => tests/fixtures/scenarios/helm-lifecycle}/components/helm/demo/templates/dependency-observed.yaml (100%) create mode 100644 tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/deployment.yaml rename {examples/helm => tests/fixtures/scenarios/helm-lifecycle}/components/helm/demo/templates/extra-configmap.yaml (100%) rename {examples/helm => tests/fixtures/scenarios/helm-lifecycle}/components/helm/demo/templates/failing-hook-job.yaml (100%) rename {examples/helm => tests/fixtures/scenarios/helm-lifecycle}/components/helm/demo/templates/hook-order-configmap.yaml (100%) rename {examples/helm => tests/fixtures/scenarios/helm-lifecycle}/components/helm/demo/templates/hook-order-job.yaml (100%) rename {examples/helm => tests/fixtures/scenarios/helm-lifecycle}/components/helm/demo/templates/job.yaml (100%) create mode 100644 tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/service.yaml rename {examples/helm => tests/fixtures/scenarios/helm-lifecycle}/components/helm/demo/templates/tpl-configmap.yaml (100%) create mode 100644 tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/values.yaml rename {examples/helm => tests/fixtures/scenarios/helm-lifecycle}/components/helm/helm-test-library/Chart.yaml (100%) rename {examples/helm => tests/fixtures/scenarios/helm-lifecycle}/components/helm/helm-test-library/templates/_render.tpl (100%) create mode 100644 tests/fixtures/scenarios/helm-lifecycle/stacks/deploy/dev.yaml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3a892a95e0..e6052185be 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -549,9 +549,11 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: go test ./tests -run 'Test((AWS(StoreHooks|Secrets)|GCPSecrets|AzureSecrets)FlociE2E|LocalGitOpsPushE2E|Scaffold(AWSLandingZone|GCPLandingZone|AzureLandingZone|AWSApp)FlociE2E|InitFromTemplateRepoGiteaE2E|TerraformFlociTfmigrateS3History)' -count=1 -timeout 25m -v - # run k3s demo tests + # Run the approachable k3s demos plus the focused native Helm lifecycle + # scenario. Keep failure-path coverage under tests/fixtures so public examples + # remain small, happy-path configurations. k3s: - name: "[k3s-${{ matrix.flavor.target }}] ${{ matrix.demo-folder }}" + name: "[k3s-${{ matrix.flavor.target }}] ${{ matrix.fixture.name }}" needs: build runs-on: ${{ matrix.flavor.os }} env: @@ -564,9 +566,10 @@ jobs: flavor: - { os: ubuntu-latest, target: linux, artifact: linux } - { os: "macos-15-intel", target: macos, artifact: macos-intel } - demo-folder: - - demo-helmfile - - helm + fixture: + - { name: demo-helmfile, path: examples/demo-helmfile } + - { name: helm, path: examples/helm } + - { name: helm-lifecycle, path: tests/fixtures/scenarios/helm-lifecycle } # The macOS matrix may spend up to 45 minutes starting Colima, followed by # two bounded 45-minute test attempts. Keep the job ceiling above those @@ -619,9 +622,9 @@ jobs: mkdir -p ~/.aws echo '[default]' > ~/.aws/config - - name: Run tests for ${{ matrix.demo-folder }} + - name: Run tests for ${{ matrix.fixture.name }} run: | - cd examples/${{ matrix.demo-folder }} + cd "${{ matrix.fixture.path }}" run_with_timeout() { python3 - "$@" <<'PY' import os diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e9bc1f6ef2..6bcc84f2f9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -90,7 +90,7 @@ repos: stages: [pre-commit] exclude: ^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|.*\.svg|website/src/components/Screengrabs/) - id: check-yaml - exclude: ^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|tests/fixtures/|examples/helm/components/helm/demo/templates/) + exclude: ^(vendor/|tests/test-cases/|tests/testdata/|tests/snapshots/|tests/fixtures/) args: [--allow-multiple-documents, --unsafe] - id: check-added-large-files stages: [pre-commit] diff --git a/examples/helm/README.md b/examples/helm/README.md index c6d3dca1b3..0f9d939617 100644 --- a/examples/helm/README.md +++ b/examples/helm/README.md @@ -17,14 +17,12 @@ Run the local chart workflow end to end: ```shell atmos validate stacks -atmos helm template demo -s dev --dependency-update +atmos helm template demo -s dev atmos emulator up kubernetes -s dev atmos helm diff demo -s dev --identity local-k3s -atmos helm apply demo -s dev --identity local-k3s --dry-run -atmos helm apply demo -s dev --identity local-k3s --on-failure=uninstall --wait=watcher --timeout=2m +atmos helm apply demo -s dev --identity local-k3s atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo -atmos helm delete demo -s dev --identity local-k3s --dry-run --wait=watcher --timeout=2m atmos helm delete demo -s dev --identity local-k3s atmos emulator down kubernetes -s dev ``` @@ -35,7 +33,7 @@ this same lifecycle. ## Render (no cluster, no credentials) ```shell -atmos helm template demo -s dev --dependency-update +atmos helm template demo -s dev # Render the same chart through a declarative Helm repository. HELM_DEMO_REPO_URL=http://127.0.0.1:8080 atmos helm template demo-repo -s dev @@ -71,20 +69,6 @@ atmos emulator up kubernetes -s dev atmos helm apply demo -s dev --identity local-k3s ``` -The stack sets `release.wait.strategy: watcher`, `release.timeout: 4m`, -`release.history.max: 10`, and `release.install.crds: skip` as native Helm type -defaults. The `demo` component overrides the wait strategy to `legacy`, uses -install uninstall-on-failure, and enables upgrade rollback with independent -failed-upgrade cleanup. The `atmos test` workflow -covers non-mutating apply and delete dry runs, ordinary Job waiting, chart-hook -suppression, CRD skipping, weighted hook ordering, retained hook resources, -install and upgrade rollback, failed-upgrade cleanup, timeout handling, and -dependency-gated Helm releases. Rendering also verifies opt-in acquisition of a -missing `file://` library dependency, hook manifests, and a Helm `tpl` expression -preserved in stack values. -The intentionally slow resources are observed -with bounded Kubernetes readiness checks rather than fixed-delay assertions. - ## Helm Repositories The `demo-repo` component shows the declarative Helm repository path: diff --git a/examples/helm/atmos.yaml b/examples/helm/atmos.yaml index 5efbdefb53..2452d82a26 100644 --- a/examples/helm/atmos.yaml +++ b/examples/helm/atmos.yaml @@ -25,77 +25,27 @@ logs: file: "/dev/stderr" level: Info -toolchain: - install_path: .tools - aliases: - helm: helm/helm - registries: - - name: aqua-public - type: aqua - source: https://github.com/aquaproj/aqua-registry/tree/main/pkgs - priority: 10 - commands: - name: "test" description: "Render the local Helm chart, deploy it to the k3s emulator, verify it, then tear it down" steps: - atmos validate stacks - # Dependency acquisition is explicit: the opt-in flag fetches the missing - # file:// library and updates the local chart before the first render. - # Template output includes weighted hooks and leaves Helm `tpl` expressions - # in values for the chart to evaluate with its native .Values context. - - command: >- - artifact=$(mktemp ./atmos-helm-template.XXXXXX); - trap 'rm -f "$artifact"' EXIT; - atmos helm template demo -s dev --dependency-update --output="$artifact" && - test "$(grep -c '^ name: demo-hook-order$' "$artifact")" -eq 2 && - grep -q 'helm.sh/hook-weight: "-2"' "$artifact" && - grep -q 'helm.sh/hook-weight: "-1"' "$artifact" && - grep -q 'rendered: from-stack' "$artifact" + - atmos helm template demo -s dev - command: atmos emulator up kubernetes -s dev retry: max_attempts: 2 initial_delay: 15s backoff_strategy: constant - # Pull workload images through the host runtime and import them into the - # nested k3s containerd. This avoids registry pulls through Colima's - # nested network while Helm is waiting on Jobs and Deployments. - - command: >- - docker pull busybox:1.36.1 && - docker pull nginx:1.27 && - docker save busybox:1.36.1 nginx:1.27 | - docker exec -i atmos-dev-emulator-kubernetes ctr --namespace k8s.io images import - - retry: - max_attempts: 2 - initial_delay: 15s - backoff_strategy: constant - command: atmos helm diff demo -s dev --identity local-k3s retry: max_attempts: 2 initial_delay: 15s backoff_strategy: constant - # Apply dry-run must not persist a release or create Kubernetes objects. - - atmos helm apply demo -s dev --identity local-k3s --dry-run - - command: >- - if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo 2>&1); then echo "dry-run unexpectedly created deployment/demo"; exit 1; - else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify deployment/demo absence: $output"; exit 1;; esac; fi - - command: >- - if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo 2>&1); then echo "dry-run unexpectedly created service/demo"; exit 1; - else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify service/demo absence: $output"; exit 1;; esac; fi - - command: >- - if ! releases=$(atmos emulator exec kubernetes -s dev -- kubectl get secrets --all-namespaces -l owner=helm,name=demo -o name); then echo "failed to query Helm release records after dry-run"; exit 1; fi; - if [ -n "$releases" ]; then echo "dry-run unexpectedly persisted Helm release records: $releases"; exit 1; fi - command: atmos helm apply demo -s dev --identity local-k3s retry: max_attempts: 2 initial_delay: 15s backoff_strategy: constant - # The -2 ConfigMap must exist before the -1 hook Job can mount it. - - atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-hook-order - # release.install.crds is a stack-level release default. - - command: >- - if output=$(atmos emulator exec kubernetes -s dev -- kubectl get crd widgets.lifecycle.atmos.test 2>&1); then echo "skip_crds unexpectedly installed widgets.lifecycle.atmos.test"; exit 1; - else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify widgets.lifecycle.atmos.test absence: $output"; exit 1;; esac; fi - command: atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo retry: max_attempts: 3 @@ -106,74 +56,5 @@ commands: max_attempts: 3 initial_delay: 10s backoff_strategy: constant - # watcher + release.wait.jobs returns only after the ordinary Job completes. - - atmos helm apply demo-jobs -s dev --identity local-k3s - - command: >- - completed=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-jobs get job demo-jobs-job -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}'); - if [ "$completed" != "True" ]; then echo "wait_for_jobs returned before demo-jobs-job completed: status=$completed"; exit 1; fi - # Disabling chart hooks prevents both weighted hook resources. - - atmos helm apply demo-no-hooks -s dev --identity local-k3s - - command: >- - if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-no-hooks get configmap demo-no-hooks-hook-order 2>&1); then echo "disable_chart_hooks unexpectedly ran Helm hooks"; exit 1; - else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify disabled Helm hooks: $output"; exit 1;; esac; fi - - command: >- - if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-no-hooks get job demo-no-hooks-hook-order 2>&1); then echo "disable_chart_hooks unexpectedly ran the Helm hook Job"; exit 1; - else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify disabled Helm hook Job: $output"; exit 1;; esac; fi - # hookOnly returns after hooks without waiting for the Deployment's explicit readiness gate. - - atmos helm apply demo-hook-only -s dev --identity local-k3s - - command: >- - if ! available=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only get deployment demo-hook-only -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>&1); then echo "failed to query Deployment/demo-hook-only availability: $available"; exit 1; fi; - if [ "$available" = "True" ]; then echo "hookOnly unexpectedly satisfied the Deployment readiness gate"; exit 1; fi - - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only patch deployment demo-hook-only --type=json -p='[{"op":"remove","path":"/spec/template/spec/readinessGates"}]' - - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only rollout status deployment/demo-hook-only --timeout=2m - # watcher honors the release timeout and rollback removes the failed install. - - command: >- - if output=$(NO_COLOR=1 atmos helm apply demo-timeout -s dev --identity local-k3s 2>&1); then echo "timed release unexpectedly succeeded"; exit 1; fi; - case "$output" in *"helm release operation failed"*) ;; *) echo "timed release failed outside the Helm lifecycle operation: $output"; exit 1;; esac; - case "$output" in *"context"*"deadline exceeded"*|*"timed out waiting for condition"*) ;; *) echo "timed release did not report a lifecycle timeout: $output"; exit 1;; esac - - command: >- - if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-timeout get deployment demo-timeout 2>&1); then echo "timeout rollback left release resources"; exit 1; - else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify timeout rollback cleanup: $output"; exit 1;; esac; fi - # A failed first install is reported as failure and rolled back. The kept - # hook ConfigMap survives recovery while ordinary release resources do not. - - command: >- - if output=$(NO_COLOR=1 atmos helm apply demo-install-fail -s dev --identity local-k3s 2>&1); then echo "failed install unexpectedly succeeded"; exit 1; fi; - case "$output" in *"helm release operation failed"*) ;; *) echo "failed install stopped outside the Helm lifecycle operation: $output"; exit 1;; esac; - case "$output" in *"job demo-install-fail-failing-hook failed: BackoffLimitExceeded"*) ;; *) echo "failed install did not report the expected hook failure: $output"; exit 1;; esac - - command: >- - if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-install-fail get deployment demo-install-fail 2>&1); then echo "rollback left failed install resources"; exit 1; - else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify failed-install rollback cleanup: $output"; exit 1;; esac; fi - - atmos emulator exec kubernetes -s dev -- kubectl -n demo-install-fail get configmap demo-install-fail-hook-order - # A failed upgrade restores the successful demo release and cleanup removes - # the resource introduced only by the failed revision. - - command: >- - if ! before=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}' 2>&1); then echo "failed to capture Deployment/demo state before upgrade: $before"; exit 1; fi; - if output=$(NO_COLOR=1 atmos helm apply demo-upgrade-fail -s dev --identity local-k3s 2>&1); then echo "failed upgrade unexpectedly succeeded"; exit 1; fi; - case "$output" in *"helm release operation failed"*) ;; *) echo "failed upgrade stopped outside the Helm lifecycle operation: $output"; exit 1;; esac; - case "$output" in *"job demo-failing-hook failed"*"BackoffLimitExceeded"*) ;; *) echo "failed upgrade did not report the expected post-upgrade hook failure: $output"; exit 1;; esac; - if ! after=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}' 2>&1); then echo "failed to capture Deployment/demo state after rollback: $after"; exit 1; fi; - if [ "$before" != "$after" ]; then echo "rollback did not restore Deployment/demo: before=$before after=$after"; exit 1; fi - - command: >- - if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-upgrade-only 2>&1); then echo "cleanup failure action left the upgrade-only ConfigMap"; exit 1; - else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify cleanup failure action: $output"; exit 1;; esac; fi - # Dependency execution gates the dependent render on the foundation's - # release-managed Deployment readiness state. - - atmos helm apply --all -s dev --identity local-k3s --tags lifecycle-dag - - atmos emulator exec kubernetes -s dev -- kubectl -n lifecycle-dag get deployment dag-dependent - - command: >- - observed=$(atmos emulator exec kubernetes -s dev -- kubectl -n lifecycle-dag get configmap dag-dependent-dependency-observed -o jsonpath='{.data.ready}'); - if [ "$observed" != "true" ]; then echo "dependent did not observe the ready foundation Deployment: ready=$observed"; exit 1; fi - # Delete dry-run must leave the deployed release and resources intact. - - atmos helm delete demo -s dev --identity local-k3s --dry-run --wait=watcher --timeout=2m - - atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo - - atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo - - command: >- - if ! releases=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get secrets -l owner=helm,name=demo -o name); then echo "failed to query Helm release records after delete dry-run"; exit 1; fi; - if [ -z "$releases" ]; then echo "delete dry-run removed the Helm release record"; exit 1; fi - atmos helm delete demo -s dev --identity local-k3s - - atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-hook-order - - atmos helm delete demo-jobs -s dev --identity local-k3s - - atmos helm delete demo-no-hooks -s dev --identity local-k3s - - atmos helm delete demo-hook-only -s dev --identity local-k3s - - atmos helm delete --all -s dev --identity local-k3s --tags lifecycle-dag - atmos emulator down kubernetes -s dev diff --git a/examples/helm/components/helm/demo/Chart.yaml b/examples/helm/components/helm/demo/Chart.yaml index d17a37337f..fc807f2414 100644 --- a/examples/helm/components/helm/demo/Chart.yaml +++ b/examples/helm/components/helm/demo/Chart.yaml @@ -4,7 +4,3 @@ description: A minimal local Helm chart used by the Atmos native Helm component type: application version: 0.1.0 appVersion: "1.0.0" -dependencies: - - name: helm-test-library - version: 0.1.0 - repository: file://../helm-test-library diff --git a/examples/helm/components/helm/demo/templates/deployment.yaml b/examples/helm/components/helm/demo/templates/deployment.yaml index ac8067e5cb..0efffbd106 100644 --- a/examples/helm/components/helm/demo/templates/deployment.yaml +++ b/examples/helm/components/helm/demo/templates/deployment.yaml @@ -15,16 +15,6 @@ spec: labels: app: {{ .Release.Name }} spec: - {{- if .Values.deployment.readinessGate }} - readinessGates: - - conditionType: "lifecycle.atmos.test/ready" - {{- end }} - {{- if gt (int .Values.deployment.readinessDelaySeconds) 0 }} - initContainers: - - name: readiness-delay - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - command: ["/bin/sh", "-c", "sleep {{ .Values.deployment.readinessDelaySeconds }}"] - {{- end }} containers: - name: {{ .Release.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/examples/helm/components/helm/demo/values.yaml b/examples/helm/components/helm/demo/values.yaml index 3f7b52c0b5..f76f460afe 100644 --- a/examples/helm/components/helm/demo/values.yaml +++ b/examples/helm/components/helm/demo/values.yaml @@ -3,25 +3,6 @@ replicaCount: 1 image: repository: nginx tag: "latest" -jobImage: - repository: busybox - tag: "1.36.1" service: type: ClusterIP port: 80 -deployment: - readinessDelaySeconds: 0 - readinessGate: false -job: - enabled: false - sleepSeconds: 0 -hooks: - enabled: true - fail: false - requiredDeployment: "" - validateDependency: false -extraConfigMap: - enabled: false -tpl: - source: default - expression: "{{ .Values.tpl.source }}" diff --git a/examples/helm/stacks/deploy/dev.yaml b/examples/helm/stacks/deploy/dev.yaml index b23e81fa70..3dcde80282 100644 --- a/examples/helm/stacks/deploy/dev.yaml +++ b/examples/helm/stacks/deploy/dev.yaml @@ -1,26 +1,6 @@ vars: stage: dev -# Native Helm lifecycle defaults for every Helm component in this stack. -helm: - release: - wait: - strategy: watcher - # Nested k3s runners can leave Helm's watcher blocked after release - # resources are gone. Live deletes use deterministic legacy waiting; the - # workflow still exercises watcher selection through its delete dry-run. - delete: - wait: - strategy: legacy - # Leave enough headroom for nested k3s workloads on resource-constrained - # macOS/Colima runners. Components that exercise timeout behavior override - # this value explicitly below. - timeout: 4m - history: - max: 10 - install: - crds: skip - components: emulator: kubernetes: @@ -35,14 +15,6 @@ components: # (components/helm/demo), which contains Chart.yaml. chart: "." namespace: demo - release: - wait: - strategy: legacy - install: - on_failure: uninstall - upgrade: - on_failure: rollback - cleanup_on_failure: true # Atmos `values:` are the Helm chart values, merged via Atmos inheritance. values: replicaCount: 2 @@ -50,9 +22,6 @@ components: tag: "1.27" service: port: 8080 - tpl: - source: from-stack - expression: !literal "{{ .Values.tpl.source }}" demo-repo: metadata: @@ -69,134 +38,3 @@ components: tag: "1.27" service: port: 8080 - - demo-jobs: - metadata: - component: demo - chart: "." - name: demo-jobs - namespace: demo-jobs - release: - wait: - jobs: true - values: - hooks: - enabled: false - job: - enabled: true - sleepSeconds: 3 - - demo-no-hooks: - metadata: - component: demo - chart: "." - name: demo-no-hooks - namespace: demo-no-hooks - release: - chart_hooks: false - - demo-hook-only: - metadata: - component: demo - chart: "." - name: demo-hook-only - namespace: demo-hook-only - release: - wait: - strategy: hookOnly - values: - deployment: - readinessGate: true - - demo-timeout: - metadata: - component: demo - chart: "." - name: demo-timeout - namespace: demo-timeout - release: - # Keep the lifecycle deadline below the readiness delay while leaving - # enough time for recovery to finish on resource-constrained runners. - timeout: 15s - wait: - strategy: watcher - install: - on_failure: uninstall - upgrade: - on_failure: rollback - values: - deployment: - readinessDelaySeconds: 60 - hooks: - enabled: false - - demo-install-fail: - metadata: - component: demo - chart: "." - name: demo-install-fail - namespace: demo-install-fail - release: - wait: - strategy: legacy - install: - on_failure: uninstall - values: - hooks: - fail: true - - demo-upgrade-fail: - metadata: - component: demo - dependencies: - components: - - name: demo - chart: "." - # Intentionally targets the already-installed demo release. - name: demo - namespace: demo - release: - upgrade: - wait: - strategy: legacy - on_failure: rollback - cleanup_on_failure: true - values: - hooks: - fail: true - extraConfigMap: - enabled: true - - dag-foundation: - metadata: - component: demo - tags: [lifecycle-dag] - chart: "." - name: dag-foundation - namespace: lifecycle-dag - release: - wait: - strategy: legacy - values: - deployment: - readinessDelaySeconds: 3 - - dag-dependent: - metadata: - component: demo - tags: [lifecycle-dag] - dependencies: - components: - - name: dag-foundation - chart: "." - name: dag-dependent - namespace: lifecycle-dag - release: - wait: - strategy: legacy - values: - hooks: - requiredDeployment: dag-foundation - # The lifecycle DAG integration test explicitly opts into a live - # cluster lookup; ordinary offline chart rendering leaves this off. - validateDependency: true diff --git a/tests/fixtures/scenarios/helm-lifecycle/README.md b/tests/fixtures/scenarios/helm-lifecycle/README.md new file mode 100644 index 0000000000..5eb49aa3a1 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/README.md @@ -0,0 +1,8 @@ +# Native Helm lifecycle scenario + +This fixture exercises native Helm lifecycle behavior against the Kubernetes +emulator. It intentionally contains failure cases, delayed readiness, hooks, +Jobs, CRDs, dependency ordering, rollback, cleanup, dry-run, and timeout +coverage. + +The user-facing happy-path demo remains in `examples/helm`. diff --git a/tests/fixtures/scenarios/helm-lifecycle/atmos.yaml b/tests/fixtures/scenarios/helm-lifecycle/atmos.yaml new file mode 100644 index 0000000000..ba3dda5a77 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/atmos.yaml @@ -0,0 +1,178 @@ +# Atmos configuration for the native Helm lifecycle integration scenario. + +base_path: "." + +components: + helm: + # Base path for native Helm components (local charts live here). + base_path: "components/helm" + +auth: + identities: + local-k3s: + kind: kubernetes/emulator + emulator: kubernetes + +stacks: + base_path: "stacks" + included_paths: + - "deploy/**/*" + excluded_paths: + - "**/_defaults.yaml" + name_template: "{{.vars.stage}}" +logs: + file: "/dev/stderr" + level: Info + +toolchain: + install_path: .tools + aliases: + helm: helm/helm + registries: + - name: aqua-public + type: aqua + source: https://github.com/aquaproj/aqua-registry/tree/main/pkgs + priority: 10 + +commands: + - name: "test" + description: "Exercise native Helm lifecycle behavior against the k3s emulator" + steps: + - atmos validate stacks + # Dependency acquisition is explicit: the opt-in flag fetches the missing + # file:// library and updates the local chart before the first render. + # Template output includes weighted hooks and leaves Helm `tpl` expressions + # in values for the chart to evaluate with its native .Values context. + - command: >- + artifact=$(mktemp ./atmos-helm-template.XXXXXX); + trap 'rm -f "$artifact"' EXIT; + atmos helm template demo -s dev --dependency-update --output="$artifact" && + test "$(grep -c '^ name: demo-hook-order$' "$artifact")" -eq 2 && + grep -q 'helm.sh/hook-weight: "-2"' "$artifact" && + grep -q 'helm.sh/hook-weight: "-1"' "$artifact" && + grep -q 'rendered: from-stack' "$artifact" + - command: atmos emulator up kubernetes -s dev + retry: + max_attempts: 2 + initial_delay: 15s + backoff_strategy: constant + # Pull workload images through the host runtime and import them into the + # nested k3s containerd. This avoids registry pulls through Colima's + # nested network while Helm is waiting on Jobs and Deployments. + - command: >- + docker pull busybox:1.36.1 && + docker pull nginx:1.27 && + docker save busybox:1.36.1 nginx:1.27 | + docker exec -i atmos-dev-emulator-kubernetes ctr --namespace k8s.io images import - + retry: + max_attempts: 2 + initial_delay: 15s + backoff_strategy: constant + - command: atmos helm diff demo -s dev --identity local-k3s + retry: + max_attempts: 2 + initial_delay: 15s + backoff_strategy: constant + # Apply dry-run must not persist a release or create Kubernetes objects. + - atmos helm apply demo -s dev --identity local-k3s --dry-run + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo 2>&1); then echo "dry-run unexpectedly created deployment/demo"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify deployment/demo absence: $output"; exit 1;; esac; fi + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo 2>&1); then echo "dry-run unexpectedly created service/demo"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify service/demo absence: $output"; exit 1;; esac; fi + - command: >- + if ! releases=$(atmos emulator exec kubernetes -s dev -- kubectl get secrets --all-namespaces -l owner=helm,name=demo -o name); then echo "failed to query Helm release records after dry-run"; exit 1; fi; + if [ -n "$releases" ]; then echo "dry-run unexpectedly persisted Helm release records: $releases"; exit 1; fi + - command: atmos helm apply demo -s dev --identity local-k3s + retry: + max_attempts: 2 + initial_delay: 15s + backoff_strategy: constant + # The -2 ConfigMap must exist before the -1 hook Job can mount it. + - atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-hook-order + # release.install.crds is a stack-level release default. + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl get crd widgets.lifecycle.atmos.test 2>&1); then echo "skip_crds unexpectedly installed widgets.lifecycle.atmos.test"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify widgets.lifecycle.atmos.test absence: $output"; exit 1;; esac; fi + - command: atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo + retry: + max_attempts: 3 + initial_delay: 10s + backoff_strategy: constant + - command: atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo + retry: + max_attempts: 3 + initial_delay: 10s + backoff_strategy: constant + # watcher + release.wait.jobs returns only after the ordinary Job completes. + - atmos helm apply demo-jobs -s dev --identity local-k3s + - command: >- + completed=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-jobs get job demo-jobs-job -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}'); + if [ "$completed" != "True" ]; then echo "wait_for_jobs returned before demo-jobs-job completed: status=$completed"; exit 1; fi + # Disabling chart hooks prevents both weighted hook resources. + - atmos helm apply demo-no-hooks -s dev --identity local-k3s + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-no-hooks get configmap demo-no-hooks-hook-order 2>&1); then echo "disable_chart_hooks unexpectedly ran Helm hooks"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify disabled Helm hooks: $output"; exit 1;; esac; fi + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-no-hooks get job demo-no-hooks-hook-order 2>&1); then echo "disable_chart_hooks unexpectedly ran the Helm hook Job"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify disabled Helm hook Job: $output"; exit 1;; esac; fi + # hookOnly returns after hooks without waiting for the Deployment's explicit readiness gate. + - atmos helm apply demo-hook-only -s dev --identity local-k3s + - command: >- + if ! available=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only get deployment demo-hook-only -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>&1); then echo "failed to query Deployment/demo-hook-only availability: $available"; exit 1; fi; + if [ "$available" = "True" ]; then echo "hookOnly unexpectedly satisfied the Deployment readiness gate"; exit 1; fi + - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only patch deployment demo-hook-only --type=json -p='[{"op":"remove","path":"/spec/template/spec/readinessGates"}]' + - atmos emulator exec kubernetes -s dev -- kubectl -n demo-hook-only rollout status deployment/demo-hook-only --timeout=2m + # watcher honors the release timeout and rollback removes the failed install. + - command: >- + if output=$(NO_COLOR=1 atmos helm apply demo-timeout -s dev --identity local-k3s 2>&1); then echo "timed release unexpectedly succeeded"; exit 1; fi; + case "$output" in *"helm release operation failed"*) ;; *) echo "timed release failed outside the Helm lifecycle operation: $output"; exit 1;; esac; + case "$output" in *"context"*"deadline exceeded"*|*"timed out waiting for condition"*) ;; *) echo "timed release did not report a lifecycle timeout: $output"; exit 1;; esac + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-timeout get deployment demo-timeout 2>&1); then echo "timeout rollback left release resources"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify timeout rollback cleanup: $output"; exit 1;; esac; fi + # A failed first install is reported as failure and rolled back. The kept + # hook ConfigMap survives recovery while ordinary release resources do not. + - command: >- + if output=$(NO_COLOR=1 atmos helm apply demo-install-fail -s dev --identity local-k3s 2>&1); then echo "failed install unexpectedly succeeded"; exit 1; fi; + case "$output" in *"helm release operation failed"*) ;; *) echo "failed install stopped outside the Helm lifecycle operation: $output"; exit 1;; esac; + case "$output" in *"job demo-install-fail-failing-hook failed: BackoffLimitExceeded"*) ;; *) echo "failed install did not report the expected hook failure: $output"; exit 1;; esac + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo-install-fail get deployment demo-install-fail 2>&1); then echo "rollback left failed install resources"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify failed-install rollback cleanup: $output"; exit 1;; esac; fi + - atmos emulator exec kubernetes -s dev -- kubectl -n demo-install-fail get configmap demo-install-fail-hook-order + # A failed upgrade restores the successful demo release and cleanup removes + # the resource introduced only by the failed revision. + - command: >- + if ! before=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}' 2>&1); then echo "failed to capture Deployment/demo state before upgrade: $before"; exit 1; fi; + if output=$(NO_COLOR=1 atmos helm apply demo-upgrade-fail -s dev --identity local-k3s 2>&1); then echo "failed upgrade unexpectedly succeeded"; exit 1; fi; + case "$output" in *"helm release operation failed"*) ;; *) echo "failed upgrade stopped outside the Helm lifecycle operation: $output"; exit 1;; esac; + case "$output" in *"job demo-failing-hook failed"*"BackoffLimitExceeded"*) ;; *) echo "failed upgrade did not report the expected post-upgrade hook failure: $output"; exit 1;; esac; + if ! after=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo -o jsonpath='{.spec.replicas} {.spec.template.spec.containers[0].image} {.spec.template.spec.containers[0].ports[0].containerPort}' 2>&1); then echo "failed to capture Deployment/demo state after rollback: $after"; exit 1; fi; + if [ "$before" != "$after" ]; then echo "rollback did not restore Deployment/demo: before=$before after=$after"; exit 1; fi + - command: >- + if output=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-upgrade-only 2>&1); then echo "cleanup failure action left the upgrade-only ConfigMap"; exit 1; + else case "$output" in *"(NotFound)"*) ;; *) echo "failed to verify cleanup failure action: $output"; exit 1;; esac; fi + # Dependency execution gates the dependent render on the foundation's + # release-managed Deployment readiness state. + - atmos helm apply --all -s dev --identity local-k3s --tags lifecycle-dag + - atmos emulator exec kubernetes -s dev -- kubectl -n lifecycle-dag get deployment dag-dependent + - command: >- + observed=$(atmos emulator exec kubernetes -s dev -- kubectl -n lifecycle-dag get configmap dag-dependent-dependency-observed -o jsonpath='{.data.ready}'); + if [ "$observed" != "true" ]; then echo "dependent did not observe the ready foundation Deployment: ready=$observed"; exit 1; fi + # Delete dry-run must leave the deployed release and resources intact. + - atmos helm delete demo -s dev --identity local-k3s --dry-run --wait=watcher --timeout=2m + - atmos emulator exec kubernetes -s dev -- kubectl -n demo get deployment demo + - atmos emulator exec kubernetes -s dev -- kubectl -n demo get service demo + - command: >- + if ! releases=$(atmos emulator exec kubernetes -s dev -- kubectl -n demo get secrets -l owner=helm,name=demo -o name); then echo "failed to query Helm release records after delete dry-run"; exit 1; fi; + if [ -z "$releases" ]; then echo "delete dry-run removed the Helm release record"; exit 1; fi + - atmos helm delete demo -s dev --identity local-k3s + - atmos emulator exec kubernetes -s dev -- kubectl -n demo get configmap demo-hook-order + - atmos helm delete demo-jobs -s dev --identity local-k3s + - atmos helm delete demo-no-hooks -s dev --identity local-k3s + - atmos helm delete demo-hook-only -s dev --identity local-k3s + - atmos helm delete --all -s dev --identity local-k3s --tags lifecycle-dag + - atmos emulator down kubernetes -s dev diff --git a/examples/helm/components/helm/demo/.gitignore b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/.gitignore similarity index 100% rename from examples/helm/components/helm/demo/.gitignore rename to tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/.gitignore diff --git a/examples/helm/components/helm/demo/Chart.lock b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/Chart.lock similarity index 100% rename from examples/helm/components/helm/demo/Chart.lock rename to tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/Chart.lock diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/Chart.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/Chart.yaml new file mode 100644 index 0000000000..d17a37337f --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: demo +description: A minimal local Helm chart used by the Atmos native Helm component demo. +type: application +version: 0.1.0 +appVersion: "1.0.0" +dependencies: + - name: helm-test-library + version: 0.1.0 + repository: file://../helm-test-library diff --git a/examples/helm/components/helm/demo/crds/lifecycle.atmos.test_widgets.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/crds/lifecycle.atmos.test_widgets.yaml similarity index 100% rename from examples/helm/components/helm/demo/crds/lifecycle.atmos.test_widgets.yaml rename to tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/crds/lifecycle.atmos.test_widgets.yaml diff --git a/examples/helm/components/helm/demo/templates/dependency-observed.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/dependency-observed.yaml similarity index 100% rename from examples/helm/components/helm/demo/templates/dependency-observed.yaml rename to tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/dependency-observed.yaml diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/deployment.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/deployment.yaml new file mode 100644 index 0000000000..ac8067e5cb --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ .Release.Name }} + spec: + {{- if .Values.deployment.readinessGate }} + readinessGates: + - conditionType: "lifecycle.atmos.test/ready" + {{- end }} + {{- if gt (int .Values.deployment.readinessDelaySeconds) 0 }} + initContainers: + - name: readiness-delay + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + command: ["/bin/sh", "-c", "sleep {{ .Values.deployment.readinessDelaySeconds }}"] + {{- end }} + containers: + - name: {{ .Release.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + ports: + - containerPort: {{ .Values.service.port }} diff --git a/examples/helm/components/helm/demo/templates/extra-configmap.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/extra-configmap.yaml similarity index 100% rename from examples/helm/components/helm/demo/templates/extra-configmap.yaml rename to tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/extra-configmap.yaml diff --git a/examples/helm/components/helm/demo/templates/failing-hook-job.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/failing-hook-job.yaml similarity index 100% rename from examples/helm/components/helm/demo/templates/failing-hook-job.yaml rename to tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/failing-hook-job.yaml diff --git a/examples/helm/components/helm/demo/templates/hook-order-configmap.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/hook-order-configmap.yaml similarity index 100% rename from examples/helm/components/helm/demo/templates/hook-order-configmap.yaml rename to tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/hook-order-configmap.yaml diff --git a/examples/helm/components/helm/demo/templates/hook-order-job.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/hook-order-job.yaml similarity index 100% rename from examples/helm/components/helm/demo/templates/hook-order-job.yaml rename to tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/hook-order-job.yaml diff --git a/examples/helm/components/helm/demo/templates/job.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/job.yaml similarity index 100% rename from examples/helm/components/helm/demo/templates/job.yaml rename to tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/job.yaml diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/service.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/service.yaml new file mode 100644 index 0000000000..3cfba0daed --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.service.type }} + selector: + app: {{ .Release.Name }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.port }} diff --git a/examples/helm/components/helm/demo/templates/tpl-configmap.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/tpl-configmap.yaml similarity index 100% rename from examples/helm/components/helm/demo/templates/tpl-configmap.yaml rename to tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/templates/tpl-configmap.yaml diff --git a/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/values.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/values.yaml new file mode 100644 index 0000000000..3f7b52c0b5 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/components/helm/demo/values.yaml @@ -0,0 +1,27 @@ +# Default chart values. Atmos `values:` from the stack are merged on top of these. +replicaCount: 1 +image: + repository: nginx + tag: "latest" +jobImage: + repository: busybox + tag: "1.36.1" +service: + type: ClusterIP + port: 80 +deployment: + readinessDelaySeconds: 0 + readinessGate: false +job: + enabled: false + sleepSeconds: 0 +hooks: + enabled: true + fail: false + requiredDeployment: "" + validateDependency: false +extraConfigMap: + enabled: false +tpl: + source: default + expression: "{{ .Values.tpl.source }}" diff --git a/examples/helm/components/helm/helm-test-library/Chart.yaml b/tests/fixtures/scenarios/helm-lifecycle/components/helm/helm-test-library/Chart.yaml similarity index 100% rename from examples/helm/components/helm/helm-test-library/Chart.yaml rename to tests/fixtures/scenarios/helm-lifecycle/components/helm/helm-test-library/Chart.yaml diff --git a/examples/helm/components/helm/helm-test-library/templates/_render.tpl b/tests/fixtures/scenarios/helm-lifecycle/components/helm/helm-test-library/templates/_render.tpl similarity index 100% rename from examples/helm/components/helm/helm-test-library/templates/_render.tpl rename to tests/fixtures/scenarios/helm-lifecycle/components/helm/helm-test-library/templates/_render.tpl diff --git a/tests/fixtures/scenarios/helm-lifecycle/stacks/deploy/dev.yaml b/tests/fixtures/scenarios/helm-lifecycle/stacks/deploy/dev.yaml new file mode 100644 index 0000000000..b23e81fa70 --- /dev/null +++ b/tests/fixtures/scenarios/helm-lifecycle/stacks/deploy/dev.yaml @@ -0,0 +1,202 @@ +vars: + stage: dev + +# Native Helm lifecycle defaults for every Helm component in this stack. +helm: + release: + wait: + strategy: watcher + # Nested k3s runners can leave Helm's watcher blocked after release + # resources are gone. Live deletes use deterministic legacy waiting; the + # workflow still exercises watcher selection through its delete dry-run. + delete: + wait: + strategy: legacy + # Leave enough headroom for nested k3s workloads on resource-constrained + # macOS/Colima runners. Components that exercise timeout behavior override + # this value explicitly below. + timeout: 4m + history: + max: 10 + install: + crds: skip + +components: + emulator: + kubernetes: + driver: k3s + ephemeral: true + + helm: + demo: + metadata: + component: demo + # The chart reference. "." is the component directory itself + # (components/helm/demo), which contains Chart.yaml. + chart: "." + namespace: demo + release: + wait: + strategy: legacy + install: + on_failure: uninstall + upgrade: + on_failure: rollback + cleanup_on_failure: true + # Atmos `values:` are the Helm chart values, merged via Atmos inheritance. + values: + replicaCount: 2 + image: + tag: "1.27" + service: + port: 8080 + tpl: + source: from-stack + expression: !literal "{{ .Values.tpl.source }}" + + demo-repo: + metadata: + component: demo + repositories: + - name: local + url: !env HELM_DEMO_REPO_URL + chart: local/demo + version: 0.1.0 + namespace: demo-repo + values: + replicaCount: 2 + image: + tag: "1.27" + service: + port: 8080 + + demo-jobs: + metadata: + component: demo + chart: "." + name: demo-jobs + namespace: demo-jobs + release: + wait: + jobs: true + values: + hooks: + enabled: false + job: + enabled: true + sleepSeconds: 3 + + demo-no-hooks: + metadata: + component: demo + chart: "." + name: demo-no-hooks + namespace: demo-no-hooks + release: + chart_hooks: false + + demo-hook-only: + metadata: + component: demo + chart: "." + name: demo-hook-only + namespace: demo-hook-only + release: + wait: + strategy: hookOnly + values: + deployment: + readinessGate: true + + demo-timeout: + metadata: + component: demo + chart: "." + name: demo-timeout + namespace: demo-timeout + release: + # Keep the lifecycle deadline below the readiness delay while leaving + # enough time for recovery to finish on resource-constrained runners. + timeout: 15s + wait: + strategy: watcher + install: + on_failure: uninstall + upgrade: + on_failure: rollback + values: + deployment: + readinessDelaySeconds: 60 + hooks: + enabled: false + + demo-install-fail: + metadata: + component: demo + chart: "." + name: demo-install-fail + namespace: demo-install-fail + release: + wait: + strategy: legacy + install: + on_failure: uninstall + values: + hooks: + fail: true + + demo-upgrade-fail: + metadata: + component: demo + dependencies: + components: + - name: demo + chart: "." + # Intentionally targets the already-installed demo release. + name: demo + namespace: demo + release: + upgrade: + wait: + strategy: legacy + on_failure: rollback + cleanup_on_failure: true + values: + hooks: + fail: true + extraConfigMap: + enabled: true + + dag-foundation: + metadata: + component: demo + tags: [lifecycle-dag] + chart: "." + name: dag-foundation + namespace: lifecycle-dag + release: + wait: + strategy: legacy + values: + deployment: + readinessDelaySeconds: 3 + + dag-dependent: + metadata: + component: demo + tags: [lifecycle-dag] + dependencies: + components: + - name: dag-foundation + chart: "." + name: dag-dependent + namespace: lifecycle-dag + release: + wait: + strategy: legacy + values: + hooks: + requiredDeployment: dag-foundation + # The lifecycle DAG integration test explicitly opts into a live + # cluster lookup; ordinary offline chart rendering leaves this off. + validateDependency: true