Skip to content

test(e2e): show CI timings and keep services on fork timeout #173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
aprimakina wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

test(e2e): show CI timings and keep services on fork timeout #173

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,8 @@ jobs:
TIGER_SECRET_KEY_INTEGRATION: ${{ secrets.TIGER_SECRET_KEY_INTEGRATION }}
# Runs the live-CDN upgrade test (TestUpgradeLiveCDNIntegration)
TIGER_UPGRADE_INTEGRATION: "1"
run: go test -race -timeout 15m -coverprofile=coverage.out -coverpkg=./... ./...
# -v prints per-test durations so E2E timings are visible in CI logs.
run: go test -v -race -timeout 15m -coverprofile=coverage.out -coverpkg=./... ./...

- name: Generate coverage report
run: go tool cover -html=coverage.out -o=coverage.html
Expand Down
106 changes: 66 additions & 40 deletions internal/tiger/cmd/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"regexp"
Expand Down Expand Up @@ -1747,6 +1748,57 @@ func TestServiceForkIntegration(t *testing.T) {
var sourceServiceID string
var forkedServiceID string

// On a fork wait-timeout, keep the source and forked services instead of
// deleting them so the timeout can be investigated. The stale-service sweeper
// reclaims them after 1h.
var keepForDebug bool

// handleForkErr fails the test on a fork error, keeping services on a timeout.
handleForkErr := func(t *testing.T, err error, output string) {
if err == nil {
return
}
var ece common.ExitCodeError
if errors.As(err, &ece) && ece.ExitCode() == common.ExitTimeout {
forkID := extractServiceIDFromCreateOutput(t, output)
if forkID != "" {
forkedServiceID = forkID
}
keepForDebug = true
t.Fatalf("Fork timed out — KEEPING source %q and fork %q for debugging "+
"(stale-service sweeper reclaims after 1h): %v\nOutput: %s",
sourceServiceID, forkID, err, output)
}
t.Fatalf("Service fork failed: %v\nOutput: %s", err, output)
}

// skipIfKeeping skips remaining subtests once a fork has timed out.
skipIfKeeping := func(t *testing.T) {
if keepForDebug {
t.Skip("a prior fork timed out; preserving services for debugging")
}
}

// cleanupService deletes *id at test end, unless a fork timeout flagged the run
// to keep services for debugging. The ID is read at cleanup time so it picks up
// the value assigned during the subtests.
cleanupService := func(label string, id *string) func() {
return func() {
if keepForDebug {
t.Logf("⚠️ KEEPING %s service %s for debugging (a fork timed out); stale sweeper reclaims after 1h", label, *id)
return
}
if *id == "" {
return
}
t.Logf("Cleaning up %s service: %s", label, *id)
if _, err := executeIntegrationCommand(t.Context(),
"service", "delete", *id, "--confirm", "--wait-timeout", "5m"); err != nil {
t.Logf("Warning: Failed to cleanup %s service %s: %v", label, *id, err)
}
}
}

// Always logout at the end to clean up credentials
defer func() {
t.Logf("Cleaning up authentication")
Expand All @@ -1756,37 +1808,9 @@ func TestServiceForkIntegration(t *testing.T) {
}
}()

// Cleanup function to ensure source service is deleted
defer func() {
if sourceServiceID != "" {
t.Logf("Cleaning up source service: %s", sourceServiceID)
_, err := executeIntegrationCommand(
t.Context(),
"service", "delete", sourceServiceID,
"--confirm",
"--wait-timeout", "5m",
)
if err != nil {
t.Logf("Warning: Failed to cleanup source service %s: %v", sourceServiceID, err)
}
}
}()

// Cleanup function to ensure forked service is deleted
defer func() {
if forkedServiceID != "" {
t.Logf("Cleaning up forked service: %s", forkedServiceID)
_, err := executeIntegrationCommand(
t.Context(),
"service", "delete", forkedServiceID,
"--confirm",
"--wait-timeout", "5m",
)
if err != nil {
t.Logf("Warning: Failed to cleanup forked service %s: %v", forkedServiceID, err)
}
}
}()
// Delete the source and forked services at test end (kept on a fork timeout).
defer cleanupService("source", &sourceServiceID)()
defer cleanupService("forked", &forkedServiceID)()

t.Run("Login", func(t *testing.T) {
t.Logf("Logging in with public key: %s", publicKey[:8]+"...") // Only show first 8 chars
Expand Down Expand Up @@ -1917,9 +1941,7 @@ func TestServiceForkIntegration(t *testing.T) {
"--output", "json",
)

if err != nil {
t.Fatalf("Service fork with --last-snapshot failed: %v\nOutput: %s", err, output)
}
handleForkErr(t, err, output)

extractedServiceID := extractServiceIDFromCreateOutput(t, output)
if extractedServiceID == "" {
Expand All @@ -1946,6 +1968,7 @@ func TestServiceForkIntegration(t *testing.T) {
})

t.Run("ForkService_Now", func(t *testing.T) {
skipIfKeeping(t)
if sourceServiceID == "" {
t.Skip("No source service ID available")
}
Expand All @@ -1961,9 +1984,7 @@ func TestServiceForkIntegration(t *testing.T) {
"--output", "json",
)

if err != nil {
t.Fatalf("Service fork failed: %v\nOutput: %s", err, output)
}
handleForkErr(t, err, output)

extractedServiceID := extractServiceIDFromCreateOutput(t, output)
if extractedServiceID == "" {
Expand All @@ -1975,6 +1996,7 @@ func TestServiceForkIntegration(t *testing.T) {
})

t.Run("VerifyForkedData", func(t *testing.T) {
skipIfKeeping(t)
if forkedServiceID == "" {
t.Skip("No forked service ID available")
}
Expand Down Expand Up @@ -2006,6 +2028,7 @@ func TestServiceForkIntegration(t *testing.T) {
})

t.Run("VerifyDataIndependence", func(t *testing.T) {
skipIfKeeping(t)
if sourceServiceID == "" || forkedServiceID == "" {
t.Skip("Source or forked service ID not available")
}
Expand Down Expand Up @@ -2076,6 +2099,7 @@ func TestServiceForkIntegration(t *testing.T) {
})

t.Run("DeleteForkedService_Now", func(t *testing.T) {
skipIfKeeping(t)
if forkedServiceID == "" {
t.Skip("No forked service ID available")
}
Expand All @@ -2099,6 +2123,7 @@ func TestServiceForkIntegration(t *testing.T) {
})

t.Run("ForkService_LastSnapshot_Success", func(t *testing.T) {
skipIfKeeping(t)
if sourceServiceID == "" {
t.Skip("No source service ID available")
}
Expand All @@ -2117,9 +2142,7 @@ func TestServiceForkIntegration(t *testing.T) {
"--output", "json",
)

if err != nil {
t.Fatalf("Service fork with --last-snapshot failed: %v\nOutput: %s", err, output)
}
handleForkErr(t, err, output)

extractedServiceID := extractServiceIDFromCreateOutput(t, output)
if extractedServiceID == "" {
Expand All @@ -2131,6 +2154,7 @@ func TestServiceForkIntegration(t *testing.T) {
})

t.Run("VerifyLastSnapshotForkWorks", func(t *testing.T) {
skipIfKeeping(t)
if forkedServiceID == "" {
t.Skip("No forked service ID available")
}
Expand All @@ -2155,6 +2179,7 @@ func TestServiceForkIntegration(t *testing.T) {
})

t.Run("DeleteForkedService_LastSnapshot", func(t *testing.T) {
skipIfKeeping(t)
if forkedServiceID == "" {
t.Skip("No forked service ID available")
}
Expand All @@ -2178,6 +2203,7 @@ func TestServiceForkIntegration(t *testing.T) {
})

t.Run("DeleteSourceService", func(t *testing.T) {
skipIfKeeping(t)
if sourceServiceID == "" {
t.Skip("No source service ID available")
}
Expand Down
Loading