feat(schema): config schema generation pipeline (foundation)

.github/workflows/schemas.yml

@@ -0,0 +1,71 @@
+name: Schemas
+
+# Drift check for the generated schemas/ artifacts (committed in-tree,
+# consumed by nebari-docs). Runs schemagen and fails if the working tree
+# diverges from what was committed. Authors keep schemas/ in sync by
+# running `make schemas` locally before pushing.
+#
+# Paths-filtered to avoid running on PRs that can't affect schema output
+# (e.g. docs-only PRs, CI tweaks).
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'pkg/config/**'
+      - 'pkg/provider/**'
+      - 'pkg/dnsprovider/**'
+      - 'pkg/nic/registry.go'
+      - 'pkg/nic/config_types.go'
+      - 'pkg/storage/longhorn/**'
+      - 'pkg/git/**'
+      - 'pkg/configschema/**'
+      - 'cmd/schemagen/**'
+      - 'schemas/**'
+      - 'go.mod'
+      - 'go.sum'
+      - 'Makefile'
+      - '.github/workflows/schemas.yml'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'pkg/config/**'
+      - 'pkg/provider/**'
+      - 'pkg/dnsprovider/**'
+      - 'pkg/nic/registry.go'
+      - 'pkg/nic/config_types.go'
+      - 'pkg/storage/longhorn/**'
+      - 'pkg/git/**'
+      - 'pkg/configschema/**'
+      - 'cmd/schemagen/**'
+      - 'schemas/**'
+      - 'go.mod'
+      - 'go.sum'
+      - 'Makefile'
+      - '.github/workflows/schemas.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  drift-check:
+    name: schemas/ drift check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Regenerate schemas
+        run: make schemas
+
+      - name: Fail on drift
+        run: |
+          if ! git diff --exit-code schemas/; then
+            echo "::error::schemas/ is out of sync with the code. Run 'make schemas' locally and commit the result."
+            exit 1
+          fi

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help build test test-unit test-integration test-coverage test-race clean fmt vet lint install pre-commit release-snapshot localkind-up localkind-down
+.PHONY: help build test test-unit test-integration test-coverage test-race clean fmt vet lint install pre-commit release-snapshot localkind-up localkind-down schemas
 
 # Variables
 BINARY_NAME=nic
@@ -166,6 +166,11 @@ deps: ## Download Go dependencies
 	go mod verify
 	@echo "Dependencies downloaded successfully"
 
+schemas: ## Regenerate JSON Schema + commented YAML reference under schemas/
+	@echo "Regenerating schemas/..."
+	go run ./cmd/schemagen -out ./schemas
+	@echo "Schemas regenerated. Run 'git diff schemas/' to review."
+
 deps-update: ## Update Go dependencies
 	@echo "Updating dependencies..."
 	go get -u ./...

cmd/schemagen/main.go

@@ -0,0 +1,232 @@
+// schemagen emits JSON Schema documents for nebari-config.yaml and each
+// registered provider's Config struct. It is an internal build/CI tool,
+// not a user-facing subcommand of nic.
+//
+// Output layout (default `-out ./schemas`):
+//
+//	schemas/
+//	  manifest.json
+//	  nebari-config.json
+//	  providers/
+//	    <name>.json    (one per registered cluster + DNS provider)
+//
+// The provider list is sourced from the nic registry (pkg/nic/registry.go)
+// via (*nic.Client).RegisteredConfigTypes; there is no parallel hard-coded
+// list. Adding a new provider to the registry automatically extends the
+// schemagen output on the next CI run.
+//
+// Invocation: `make schemas` or `go run ./cmd/schemagen -out ./schemas`.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"strings"
+
+	"github.com/nebari-dev/nebari-infrastructure-core/pkg/config"
+	"github.com/nebari-dev/nebari-infrastructure-core/pkg/configschema"
+	"github.com/nebari-dev/nebari-infrastructure-core/pkg/nic"
+)
+
+func main() {
+	var (
+		outDir    string
+		providers string
+		pkgRoot   string
+		version   string
+	)
+	flag.StringVar(&outDir, "out", "./schemas", "output directory for generated schema files")
+	flag.StringVar(&providers, "providers", "", "comma-separated subset to regenerate (default: all registered)")
+	flag.StringVar(&pkgRoot, "pkg-root", "./pkg", "root directory whose Go packages are scanned for field godoc")
+	flag.StringVar(&version, "version", "", "version string stamped into manifest.json (default: empty)")
+	flag.Parse()
+
+	ctx := context.Background()
+	if err := run(ctx, outDir, providers, pkgRoot, version); err != nil {
+		slog.Error("schemagen failed", "error", err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context, outDir, providersFlag, pkgRoot, version string) error {
+	if err := os.MkdirAll(filepath.Join(outDir, "providers"), 0o755); err != nil {
+		return fmt.Errorf("mkdir %s: %w", outDir, err)
+	}
+
+	pkgPaths, err := collectPackagePaths(pkgRoot)
+	if err != nil {
+		return fmt.Errorf("collect package paths under %s: %w", pkgRoot, err)
+	}
+
+	client, err := nic.NewClient(ctx)
+	if err != nil {
+		return fmt.Errorf("build nic client: %w", err)
+	}
+	types := client.RegisteredConfigTypes(ctx)
+
+	filter := parseFilter(providersFlag)
+	emitTopLevel := len(filter) == 0
+
+	clusterNames := sortedKeys(types.Cluster)
+	dnsNames := sortedKeys(types.DNS)
+
+	if emitTopLevel {
+		if err := writeSchema(ctx, outDir, "nebari-config.json",
+			reflect.TypeFor[config.NebariConfig](),
+			"Nebari config", pkgPaths); err != nil {
+			return err
+		}
+	}
+
+	for _, name := range clusterNames {
+		if !accepts(filter, name) {
+			continue
+		}
+		if err := writeSchema(ctx, outDir, filepath.Join("providers", name+".json"),
+			types.Cluster[name],
+			fmt.Sprintf("%s cluster provider configuration", name), pkgPaths); err != nil {
+			return err
+		}
+	}
+
+	for _, name := range dnsNames {
+		if !accepts(filter, name) {
+			continue
+		}
+		if err := writeSchema(ctx, outDir, filepath.Join("providers", name+".json"),
+			types.DNS[name],
+			fmt.Sprintf("%s DNS provider configuration", name), pkgPaths); err != nil {
+			return err
+		}
+	}
+
+	if emitTopLevel {
+		if err := writeManifest(outDir, version, clusterNames, dnsNames); err != nil {
+			return err
+		}
+	}
+
+	fmt.Printf("schemagen wrote schemas under %s\n", outDir)
+	fmt.Printf("  cluster providers: %v\n", clusterNames)
+	fmt.Printf("  dns providers:     %v\n", dnsNames)
+	return nil
+}
+
+func writeSchema(ctx context.Context, outDir, relPath string, t reflect.Type, title string, pkgPaths []string) error {
+	data, err := configschema.Generate(ctx, t, configschema.FormatJSON, configschema.Options{
+		Title:        title,
+		PackagePaths: pkgPaths,
+	})
+	if err != nil {
+		return fmt.Errorf("generate %s: %w", relPath, err)
+	}
+	full := filepath.Join(outDir, relPath)
+	if err := os.WriteFile(full, data, 0o644); err != nil {
+		return fmt.Errorf("write %s: %w", full, err)
+	}
+	return nil
+}
+
+// manifest is the shape of schemas/manifest.json. The docs site fetches
+// this first to discover what schemas exist, then fetches each referenced
+// file. Adding a new provider extends Providers/DNS automatically.
+type manifest struct {
+	Version   string   `json:"version,omitempty"`
+	Providers []string `json:"providers"`
+	DNS       []string `json:"dns"`
+	TopLevel  string   `json:"top_level"`
+}
+
+func writeManifest(outDir, version string, cluster, dns []string) error {
+	m := manifest{
+		Version:   version,
+		Providers: cluster,
+		DNS:       dns,
+		TopLevel:  "nebari-config.json",
+	}
+	data, err := json.MarshalIndent(m, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal manifest: %w", err)
+	}
+	data = append(data, '\n')
+	return os.WriteFile(filepath.Join(outDir, "manifest.json"), data, 0o644)
+}
+
+// collectPackagePaths walks root and returns every subdirectory that
+// contains at least one non-test .go file. These paths are passed to
+// configschema.Generate as Options.PackagePaths so invopop/jsonschema
+// can pick up godoc comments wherever the type tree leads.
+func collectPackagePaths(root string) ([]string, error) {
+	var paths []string
+	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		if !d.IsDir() {
+			return nil
+		}
+		name := d.Name()
+		if strings.HasPrefix(name, ".") || name == "vendor" || name == "testdata" {
+			return fs.SkipDir
+		}
+		entries, err := os.ReadDir(path)
+		if err != nil {
+			return err
+		}
+		for _, e := range entries {
+			if e.IsDir() {
+				continue
+			}
+			n := e.Name()
+			if strings.HasSuffix(n, ".go") && !strings.HasSuffix(n, "_test.go") {
+				paths = append(paths, path)
+				return nil
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(paths)
+	return paths, nil
+}
+
+func sortedKeys[V any](m map[string]V) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+func parseFilter(raw string) map[string]struct{} {
+	if raw == "" {
+		return nil
+	}
+	out := make(map[string]struct{})
+	for name := range strings.SplitSeq(raw, ",") {
+		name = strings.TrimSpace(name)
+		if name != "" {
+			out[name] = struct{}{}
+		}
+	}
+	return out
+}
+
+func accepts(filter map[string]struct{}, name string) bool {
+	if filter == nil {
+		return true
+	}
+	_, ok := filter[name]
+	return ok
+}

go.mod

@@ -69,7 +69,9 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.30.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7 // indirect
+	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/buger/jsonparser v1.1.2 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/cloudflare/circl v1.6.1 // indirect
@@ -112,6 +114,7 @@ require (
 	github.com/hashicorp/terraform-json v0.27.1 // indirect
 	github.com/huandu/xstrings v1.5.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/invopop/jsonschema v0.14.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmoiron/sqlx v1.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -139,6 +142,7 @@ require (
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/pb33f/ordered-map/v2 v2.3.1 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pjbgf/sha1cd v0.3.2 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
@@ -167,6 +171,7 @@ require (
 	go.opentelemetry.io/proto/otlp v1.7.1 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect
 	golang.org/x/net v0.53.0 // indirect
 	golang.org/x/oauth2 v0.30.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect