Skip to content

Commit 65cfde1

Browse files
joecoralljoecorall
authored
[major] Stabilize template checkout lifecycle for v1.0.0 (#119)
Stabilize the schema 1 plugin contract and harden local and remote template checkout provenance, cancellation, ownership, and cleanup.
1 parent 9bae545 commit 65cfde1

7 files changed

Lines changed: 1451 additions & 251 deletions

‎pkg/plugin/compose_template.go‎

Lines changed: 83 additions & 159 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,12 @@ package plugin
33
import (
44
"bytes"
55
"context"
6-
"encoding/base64"
6+
"errors"
77
"fmt"
88
"io"
99
"os"
1010
"os/exec"
11+
"path"
1112
"path/filepath"
1213
"strings"
1314
"sync"
@@ -310,31 +311,62 @@ func (s *SDK) EnsureComposeTemplateCheckoutContext(runCtx context.Context, out i
310311
}
311312

312313
func (s *SDK) ensureLocalComposeTemplateCheckout(runCtx context.Context, out io.Writer, req ComposeCreateRequest, projectDir string) (bool, error) {
313-
if err := runCtx.Err(); err != nil {
314-
return false, err
315-
}
316-
entries, err := os.ReadDir(projectDir)
317-
if err == nil && len(entries) > 0 {
318-
return false, nil
314+
projectDirExisted, notEmpty, err := localProjectDirectoryState(runCtx, projectDir)
315+
if err != nil {
316+
return false, fmt.Errorf("inspect project directory: %w", err)
319317
}
320-
if err != nil && !os.IsNotExist(err) {
321-
return false, fmt.Errorf("read project directory %q: %w", projectDir, err)
318+
if notEmpty {
319+
return false, fmt.Errorf("project directory %q is not empty; choose checkout source %q to use an existing checkout", projectDir, CheckoutSourceExisting)
322320
}
323321
if err := os.MkdirAll(filepath.Dir(projectDir), 0o750); err != nil {
324322
return false, fmt.Errorf("create parent directory for %q: %w", projectDir, err)
325323
}
324+
ownedProjectDir := false
325+
if !projectDirExisted {
326+
if err := os.Mkdir(projectDir, 0o750); err != nil {
327+
return false, fmt.Errorf("claim project directory %q for template checkout: %w", projectDir, err)
328+
}
329+
ownedProjectDir = true
330+
}
326331
fmt.Fprintf(out, "Cloning %s into %s\n", req.TemplateRepo, projectDir)
327-
if err := s.CloneTemplateRepo(GitTemplateOptions{
332+
if err := s.CloneTemplateRepoContext(runCtx, GitTemplateOptions{
328333
TemplateRepo: req.TemplateRepo,
329334
TemplateBranch: req.TemplateBranch,
330335
ProjectDir: projectDir,
331336
Quiet: true,
332337
}); err != nil {
338+
if !ownedProjectDir {
339+
return false, err
340+
}
341+
if cleanupErr := os.RemoveAll(projectDir); cleanupErr != nil {
342+
return false, errors.Join(err, fmt.Errorf("clean up failed template checkout %q: %w", projectDir, cleanupErr))
343+
}
333344
return false, err
334345
}
335346
return true, nil
336347
}
337348

349+
func localProjectDirectoryState(runCtx context.Context, projectDir string) (bool, bool, error) {
350+
if err := runCtx.Err(); err != nil {
351+
return false, false, err
352+
}
353+
info, err := os.Lstat(projectDir)
354+
if err != nil {
355+
if os.IsNotExist(err) {
356+
return false, false, nil
357+
}
358+
return false, false, fmt.Errorf("inspect project directory %q: %w", projectDir, err)
359+
}
360+
if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() {
361+
return true, false, fmt.Errorf("project directory %q must be a real directory, not a symlink or other file", projectDir)
362+
}
363+
entries, err := os.ReadDir(projectDir)
364+
if err != nil {
365+
return false, false, fmt.Errorf("read project directory %q: %w", projectDir, err)
366+
}
367+
return true, len(entries) > 0, nil
368+
}
369+
338370
func refreshCreateContextComposeIdentity(ctx *config.Context, req ComposeCreateRequest) error {
339371
if ctx == nil {
340372
return nil
@@ -359,180 +391,72 @@ func refreshCreateContextComposeIdentity(ctx *config.Context, req ComposeCreateR
359391
}
360392

361393
func (s *SDK) ensureRemoteComposeTemplateCheckout(runCtx context.Context, out io.Writer, req ComposeCreateRequest, ctx *config.Context) (bool, error) {
362-
runRemote := runComposeProjectRemoteShellCommandContext
363-
output, err := runRemote(runCtx, ctx, nil, nil, fmt.Sprintf("if [ -d %s ] && [ -n \"$(ls -A %s 2>/dev/null)\" ]; then echo present; fi", shellQuote(ctx.ProjectDir), shellQuote(ctx.ProjectDir)))
394+
connection, err := openRemoteTemplateConnection(runCtx, ctx)
395+
if err != nil {
396+
return false, fmt.Errorf("open remote template connection: %w", err)
397+
}
398+
defer connection.Close()
399+
projectDirExisted, notEmpty, err := remoteProjectDirectoryState(runCtx, connection, ctx.ProjectDir)
364400
if err != nil {
365401
return false, fmt.Errorf("inspect remote project directory: %w", err)
366402
}
367-
if strings.TrimSpace(output) == "present" {
368-
return false, nil
403+
if notEmpty {
404+
return false, fmt.Errorf("remote project directory %q is not empty; choose checkout source %q to use an existing checkout", ctx.ProjectDir, CheckoutSourceExisting)
369405
}
370406
templateRepo, err := validateTemplateRepository(req.TemplateRepo)
371407
if err != nil {
372408
return false, err
373409
}
374-
if _, err := runRemote(runCtx, ctx, nil, nil, fmt.Sprintf("mkdir -p %s", shellQuote(filepath.Dir(ctx.ProjectDir)))); err != nil {
410+
if err := connection.MkdirAll(path.Dir(ctx.ProjectDir)); err != nil {
375411
return false, fmt.Errorf("prepare remote parent directory: %w", err)
376412
}
413+
ownedProjectDir := false
414+
if !projectDirExisted {
415+
if err := connection.Mkdir(ctx.ProjectDir); err != nil {
416+
return false, fmt.Errorf("claim remote project directory %q for template checkout: %w", ctx.ProjectDir, err)
417+
}
418+
ownedProjectDir = true
419+
if err := connection.Chmod(ctx.ProjectDir, 0o750); err != nil {
420+
claimErr := fmt.Errorf("set remote project directory permissions: %w", err)
421+
return false, cleanupOwnedRemoteTemplateCheckout(connection, ctx.ProjectDir, claimErr)
422+
}
423+
}
377424
cloneArgs := []string{"git", "clone"}
378425
if strings.TrimSpace(req.TemplateBranch) != "" {
379426
cloneArgs = append(cloneArgs, "--branch", req.TemplateBranch)
380427
}
381428
cloneArgs = append(cloneArgs, "--", templateRepo, ctx.ProjectDir)
382429
fmt.Fprintf(out, "Cloning %s into %s on %s\n", templateRepo, ctx.ProjectDir, ctx.SSHHostname)
383-
if _, err := runRemote(runCtx, ctx, io.Discard, io.Discard, shellJoin(cloneArgs)); err != nil {
384-
return false, err
385-
}
386-
inspectionOutput, err := runRemote(runCtx, ctx, nil, nil, remoteTemplateInspectionCommand(ctx.ProjectDir))
387-
if err != nil {
388-
return false, fmt.Errorf("inspect remote template checkout: %w", err)
430+
if _, err := connection.Run(runCtx, io.Discard, nil, cloneArgs...); err != nil {
431+
cloneErr := fmt.Errorf("clone remote template repo %q: %w", templateRepo, err)
432+
if !ownedProjectDir {
433+
return false, cloneErr
434+
}
435+
return false, cleanupOwnedRemoteTemplateCheckout(connection, ctx.ProjectDir, cloneErr)
389436
}
390-
metadata, err := parseRemoteTemplateInspection(inspectionOutput)
437+
metadata, err := inspectRemoteTemplateCheckout(runCtx, connection, ctx.ProjectDir)
391438
if err != nil {
392-
return false, err
439+
if !ownedProjectDir {
440+
return false, err
441+
}
442+
return false, cleanupOwnedRemoteTemplateCheckout(connection, ctx.ProjectDir, err)
393443
}
394444
sitectl, plugins := s.templateLockPackages()
395445
lock, err := buildTemplateLock(templateRepo, metadata, sitectl, plugins)
396446
if err != nil {
397-
return false, err
398-
}
399-
finalizeCommand := remoteTemplateFinalizeCommand(ctx.ProjectDir, req.TemplateBranch, lock)
400-
if _, err := runRemote(runCtx, ctx, io.Discard, io.Discard, finalizeCommand); err != nil {
401-
return false, fmt.Errorf("finalize remote template checkout: %w", err)
402-
}
403-
return true, nil
404-
}
405-
406-
func remoteTemplateInspectionCommand(projectDir string) string {
407-
return fmt.Sprintf(`set -euo pipefail
408-
project=%s
409-
if [ ! -d "${project}/.git" ]; then
410-
echo "remote template checkout has no Git history" >&2
411-
exit 1
412-
fi
413-
commit="$(git -C "${project}" rev-parse --verify 'HEAD^{commit}')"
414-
libops="${project}/.libops"
415-
if [ -L "${libops}" ] || { [ -e "${libops}" ] && [ ! -d "${libops}" ]; }; then
416-
echo "remote template .libops must be a real directory" >&2
417-
exit 1
418-
fi
419-
lock_path="${libops}/template.lock.yaml"
420-
if [ -L "${lock_path}" ] || [ -e "${lock_path}" ]; then
421-
echo "source template must not contain .libops/template.lock.yaml; sitectl creates that downstream provenance file" >&2
422-
exit 1
423-
fi
424-
encode_metadata_file() {
425-
local relative="$1"
426-
local maximum="$2"
427-
local path="${project}/${relative}"
428-
if [ -L "${path}" ] || { [ -e "${path}" ] && [ ! -f "${path}" ]; }; then
429-
echo "remote template metadata ${relative} must be a regular file" >&2
430-
exit 1
431-
fi
432-
if [ ! -e "${path}" ]; then
433-
return 0
434-
fi
435-
local bytes
436-
bytes="$(LC_ALL=C wc -c < "${path}")"
437-
if [[ ! "${bytes}" =~ ^[0-9]+$ ]] || [ "${bytes}" -gt "${maximum}" ]; then
438-
echo "remote template metadata ${relative} exceeds ${maximum} bytes" >&2
439-
exit 1
440-
fi
441-
base64 < "${path}" | tr -d '\n'
442-
}
443-
printf 'commit=%%s\n' "${commit}"
444-
printf 'contract=%%s\n' "$(encode_metadata_file %s %d)"
445-
printf 'component_defaults=%%s\n' "$(encode_metadata_file %s %d)"`,
446-
shellQuote(projectDir),
447-
shellQuote(templateContractPath), maxTemplateContractBytes,
448-
shellQuote(componentDefaultsRevisionPath), maxComponentRevisionBytes,
449-
)
450-
}
451-
452-
func parseRemoteTemplateInspection(output string) (templateCheckoutMetadata, error) {
453-
lines := strings.Split(strings.TrimSpace(output), "\n")
454-
if len(lines) != 3 {
455-
return templateCheckoutMetadata{}, fmt.Errorf("parse remote template metadata: expected three fields")
456-
}
457-
values := map[string]string{}
458-
for _, line := range lines {
459-
key, value, found := strings.Cut(line, "=")
460-
if !found || (key != "commit" && key != "contract" && key != "component_defaults") {
461-
return templateCheckoutMetadata{}, fmt.Errorf("parse remote template metadata: invalid field")
462-
}
463-
if _, duplicate := values[key]; duplicate {
464-
return templateCheckoutMetadata{}, fmt.Errorf("parse remote template metadata: duplicate %s field", key)
465-
}
466-
values[key] = value
467-
}
468-
for _, key := range []string{"commit", "contract", "component_defaults"} {
469-
if _, present := values[key]; !present {
470-
return templateCheckoutMetadata{}, fmt.Errorf("parse remote template metadata: missing %s field", key)
471-
}
472-
}
473-
if !templateCommitPattern.MatchString(values["commit"]) {
474-
return templateCheckoutMetadata{}, fmt.Errorf("parse remote template metadata: invalid Git object id")
475-
}
476-
metadata := templateCheckoutMetadata{Commit: strings.ToLower(values["commit"])}
477-
if values["contract"] != "" {
478-
contract, err := base64.StdEncoding.Strict().DecodeString(values["contract"])
479-
if err != nil || len(contract) > maxTemplateContractBytes {
480-
return templateCheckoutMetadata{}, fmt.Errorf("parse remote template metadata: invalid contract payload")
481-
}
482-
metadata.Contract = contract
483-
revision, err := validateTemplateContract(contract)
484-
if err != nil {
485-
return templateCheckoutMetadata{}, err
447+
if !ownedProjectDir {
448+
return false, err
486449
}
487-
metadata.ComponentDefaultsRevision = revision
450+
return false, cleanupOwnedRemoteTemplateCheckout(connection, ctx.ProjectDir, err)
488451
}
489-
if values["component_defaults"] != "" {
490-
revisionData, err := base64.StdEncoding.Strict().DecodeString(values["component_defaults"])
491-
if err != nil || len(revisionData) > maxComponentRevisionBytes {
492-
return templateCheckoutMetadata{}, fmt.Errorf("parse remote template metadata: invalid component defaults payload")
452+
if err := finalizeRemoteTemplateCheckout(runCtx, connection, ctx.ProjectDir, req.TemplateBranch, lock); err != nil {
453+
finalizeErr := fmt.Errorf("finalize remote template checkout: %w", err)
454+
if !ownedProjectDir {
455+
return false, finalizeErr
493456
}
494-
revision, err := validateComponentDefaultsRevision(string(revisionData))
495-
if err != nil {
496-
return templateCheckoutMetadata{}, fmt.Errorf("parse remote component defaults revision: %w", err)
497-
}
498-
if metadata.ComponentDefaultsRevision != "" && metadata.ComponentDefaultsRevision != revision {
499-
return templateCheckoutMetadata{}, fmt.Errorf("component defaults revision differs between %s and %s", templateContractPath, componentDefaultsRevisionPath)
500-
}
501-
metadata.ComponentDefaultsRevision = revision
502-
}
503-
return metadata, nil
504-
}
505-
506-
func remoteTemplateFinalizeCommand(projectDir, branch string, lock []byte) string {
507-
initArgs := []string{"git", "-C", projectDir, "init"}
508-
if strings.TrimSpace(branch) != "" {
509-
initArgs = append(initArgs, "-b", branch)
510-
}
511-
encodedLock := base64.StdEncoding.EncodeToString(lock)
512-
return fmt.Sprintf(`set -euo pipefail
513-
project=%s
514-
libops="${project}/.libops"
515-
lock_path="${libops}/template.lock.yaml"
516-
if [ -L "${libops}" ] || { [ -e "${libops}" ] && [ ! -d "${libops}" ]; }; then
517-
echo "remote template .libops must be a real directory" >&2
518-
exit 1
519-
fi
520-
if [ -L "${lock_path}" ] || { [ -e "${lock_path}" ] && [ ! -f "${lock_path}" ]; }; then
521-
echo "remote template lock path must be a regular file" >&2
522-
exit 1
523-
fi
524-
if [ ! -e "${libops}" ]; then
525-
mkdir -m 0750 -- "${libops}"
526-
fi
527-
temporary="$(mktemp "${libops}/.template.lock.yaml.tmp-XXXXXX")"
528-
cleanup() { rm -f -- "${temporary}"; }
529-
trap cleanup EXIT
530-
printf '%%s' %s | base64 --decode > "${temporary}"
531-
chmod 0644 "${temporary}"
532-
rm -rf -- "${project}/.git"
533-
%s
534-
mv -f -- "${temporary}" "${lock_path}"
535-
trap - EXIT`, shellQuote(projectDir), shellQuote(encodedLock), shellJoin(initArgs))
457+
return false, cleanupOwnedRemoteTemplateCheckout(connection, ctx.ProjectDir, finalizeErr)
458+
}
459+
return true, nil
536460
}
537461

538462
// RunComposeProjectCommandList runs a list of shell commands in a compose

0 commit comments

Comments
 (0)