diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 57b90212f..72023465a 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,24 @@ +{"_type":"issue","id":"gopherstack-l0u","title":"Phase 3.3: convert secretsmanager backend to pkgs/store","description":"Convert the secretsmanager backend's internal datalayer (secrets map) from map[string]map[string]*Secret to pkgs/store.Table, following the ec2/sqs/cloudwatchlogs Phase 3.3 conversions. resourcePolicies and replicationConfigs remain raw nested maps (non-*T / slice-valued). Gates green: build, vet, go fix, race tests, golangci-lint. Full-state snapshot/restore round-trip test added. Not committed -- left in working tree for orchestrator review.","status":"in_progress","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T12:07:36Z","created_by":"Witness Patrol","updated_at":"2026-07-06T12:07:41Z","started_at":"2026-07-06T12:07:41Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-9op","title":"Phase 3.3: convert opensearch backend to pkgs/store","status":"closed","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T10:09:31Z","created_by":"Witness Patrol","updated_at":"2026-07-06T10:14:45Z","started_at":"2026-07-06T10:09:35Z","closed_at":"2026-07-06T10:14:45Z","close_reason":"Closed","comments":[{"id":"019f36e7-c782-7baa-aaa8-0e08d9ce4a10","issue_id":"gopherstack-9op","author":"Witness Patrol","text":"Conversion complete in working tree (not committed, per task instructions): 17 of 26 maps converted to pkgs/store.Table (13 clean + 4 dirty/DTO), 2 maps eliminated (replaced by store.Index), 7 left raw (slice-valued or non-*T). Gate green: go build ./services/opensearch/..., go build ./..., go vet, go fix -diff (empty), go test -race (pass), golangci-lint (0 issues). Added full-state Snapshot/Restore round-trip test. No exported API changes.","created_at":"2026-07-06T10:09:47Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} +{"_type":"issue","id":"gopherstack-lr2","title":"Phase 3.3: convert elbv2 backend to pkgs/store","description":"Convert elbv2 backend's internal datalayer (loadBalancers, targetGroups, listeners, rules, trustStores) from map[string]*T to pkgs/store Table[T]+Registry, with secondary indexes (listenersByLB, rulesByListener) for nested LB/listener scans. resourcePolicies (bare-string value, no identity) and targetReadyAt/targetDrainingUntil (doubly-nested maps) deliberately left raw per store.Table's key-purity requirement. Snapshot/Restore rewired to registry.SnapshotAll()/RestoreAll() with a version guard (mismatch -\u003e log + ResetAll + nil). Added full-state Snapshot-\u003eRestore round-trip test. All gates green (build, vet, go fix, race tests, golangci-lint). Left uncommitted in working tree per task constraints.","status":"closed","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T08:44:28Z","created_by":"Witness Patrol","updated_at":"2026-07-06T08:45:50Z","started_at":"2026-07-06T08:44:32Z","closed_at":"2026-07-06T08:45:50Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-9h4","title":"Phase 3.3: convert elasticache backend to pkgs/store","status":"closed","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T07:56:58Z","created_by":"Witness Patrol","updated_at":"2026-07-06T08:04:46Z","started_at":"2026-07-06T07:57:02Z","closed_at":"2026-07-06T08:04:46Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-dnh","title":"Phase 3.3: convert redshift backend to pkgs/store","status":"closed","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T04:58:32Z","created_by":"Witness Patrol","updated_at":"2026-07-06T05:00:03Z","started_at":"2026-07-06T04:58:36Z","closed_at":"2026-07-06T05:00:03Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-hal","title":"Phase 3.3: convert cloudfront backend to pkgs/store","status":"closed","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T04:55:37Z","created_by":"Witness Patrol","updated_at":"2026-07-06T04:56:52Z","started_at":"2026-07-06T04:55:41Z","closed_at":"2026-07-06T04:56:52Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-40l","title":"Phase 3.3: convert cognitoidp backend to pkgs/store","status":"closed","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T03:55:55Z","created_by":"Witness Patrol","updated_at":"2026-07-06T04:33:58Z","started_at":"2026-07-06T03:56:03Z","closed_at":"2026-07-06T04:33:58Z","close_reason":"Closed","comments":[{"id":"019f35b1-dd7a-797e-82ab-f258a60875d9","issue_id":"gopherstack-40l","author":"Witness Patrol","text":"Conversion complete: 12/29 map fields converted to store.Table (pools, clients, users, groups, resourceServers, identityProviders, domains, terms, userImportJobs, managedLoginBrandings, uiCustomizations, typedRiskConfigurations); 3 more (poolsByName, clientsByPool, usersBySub) eliminated as store.Index secondary indexes on the above; 14 left raw (refreshTokens+2 derived index maps, mfaSessions, groupMembers, tokenRevokedBefore, resourceTags, riskConfigurations, logDeliveryConfigs, poolMfaConfigs, attrVerificationCodes, devices, webauthnCredentials, authEvents) -- documented in store_setup.go's registerAllTables doc, all because the stored value carries no pure identity for the map key. New store_setup.go (composite keyFns + data-driven registration) and persistence_test.go (Snapshot/Restore round-trip + version-guard tests). Gate green: build/vet/fix/lint/test -race all pass; whole-repo build clean. Exported API unchanged (cli.go, cloudformation, teststack, dashboard all build against it untouched). Not committed per task instructions -- left in working tree on branch parity-sweep-3.","created_at":"2026-07-06T04:31:17Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} +{"_type":"issue","id":"gopherstack-mao","title":"Phase 3.3: convert iot backend to pkgs/store","notes":"Conversion complete in working tree (not committed, per task constraints). 35 map[string]*T fields -\u003e store.Table[T] via data-driven registerAllTables (store_setup.go). 21 raw maps left (documented in store_setup.go) + shadows exception (composite-key + Reset()-quirk preservation). Snapshot/Restore rewired to registry.SnapshotAll()/RestoreAll() + small DTO registry for the one dirty table (topicRuleDestinations, ConfirmationToken json:-). Added iotSnapshotVersion=1 guard. Gate green: build/vet/fix/test-race/lint clean. Net -445 LOC. Existing TestPersistenceGap264_FullBackendStateSurvivesRoundTrip covers full-state round-trip.","status":"closed","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T03:18:20Z","created_by":"Witness Patrol","updated_at":"2026-07-06T03:45:02Z","started_at":"2026-07-06T03:18:25Z","closed_at":"2026-07-06T03:45:02Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-27y","title":"Phase 3.3: convert rds backend to pkgs/store","description":"26/38 maps to store.Table, 12 raw (persistence-audited), commit 4179a2fc, gated green.","status":"closed","priority":1,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T03:11:40Z","created_by":"Witness Patrol","updated_at":"2026-07-06T03:11:41Z","closed_at":"2026-07-06T03:11:41Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-oi5","title":"Phase 3.3: convert rds backend to pkgs/store","notes":"Converted rds InMemoryBackend: 26/38 resource maps to pkgs/store.Table (parameterGroups+clusterParameterGroups share DBParameterGroup value type as two tables), 12 raw-left (slice-valued / transient-scheduling / mixed-key quirk in automatedBackups, documented in store_setup.go). Snapshot/Restore rewired to registry.SnapshotAll/RestoreAll with version guard (rdsSnapshotVersion=1). Added full-state Snapshot-\u003eRestore round-trip test. All gates green (build/vet-blocked-by-env/go fix/tests -race/lint). No exported API changes.","status":"closed","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T02:48:47Z","created_by":"Witness Patrol","updated_at":"2026-07-06T03:09:58Z","started_at":"2026-07-06T02:48:51Z","closed_at":"2026-07-06T03:09:58Z","close_reason":"Phase 3.3 rds-\u003epkgs/store conversion complete; all gates green; left in working tree per instructions (no commit).","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-zit","title":"Phase 3.3: convert ssm backend to pkgs/store","status":"in_progress","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T02:34:59Z","created_by":"Witness Patrol","updated_at":"2026-07-06T03:11:27Z","started_at":"2026-07-06T02:35:02Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-oa6","title":"Phase 3.3: convert glue backend to pkgs/store","status":"closed","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T01:50:52Z","created_by":"Witness Patrol","updated_at":"2026-07-06T02:29:05Z","started_at":"2026-07-06T01:50:56Z","closed_at":"2026-07-06T02:29:05Z","close_reason":"Closed","comments":[{"id":"019f3540-61bc-753f-b418-7d6fd0115097","issue_id":"gopherstack-oa6","author":"Witness Patrol","text":"Conversion complete in working tree (not committed per task instructions). 37/52 maps converted to store.Table (data-driven registration in store_setup.go); 15 left raw (documented: partitionIndexes, tableColumnStats, partitionColumnStats, resourcePolicies, catalogEncryptionSettings, catalogImports, jobRuns, workflowRuns, schemaVersions, sessionStatements, crawlHistory, schemaVersionMetadata, jobRunReadyAt, jobRunDoneAt, crawlerReadyAt). 0 DTOs (types already clean JSON). Fixed AddPartitionInternal/AddTableVersionInternal to stamp dbName/tableName identity onto stored value (previously only used as external map key) so store.Table keyFn purity holds. Added snapshot version guard (glueSnapshotVersion=1) + full-state persistence round-trip test. Gate green: build/vet/fix/test -race/lint all pass for services/glue. Whole-repo build transiently fails in services/ecs (concurrent agent, unrelated).","created_at":"2026-07-06T02:27:20Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} +{"_type":"issue","id":"gopherstack-4k6","title":"Phase 3.3: convert lambda backend to pkgs/store","status":"closed","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T01:47:59Z","created_by":"Witness Patrol","updated_at":"2026-07-06T01:50:14Z","started_at":"2026-07-06T01:48:05Z","closed_at":"2026-07-06T01:50:14Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-75r","title":"Phase 3.3: convert iam backend to pkgs/store","status":"closed","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T01:36:31Z","created_by":"Witness Patrol","updated_at":"2026-07-06T02:06:38Z","started_at":"2026-07-06T01:36:35Z","closed_at":"2026-07-06T02:06:38Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-nwt","title":"Phase 3.3: convert dynamodb backend to pkgs/store","status":"closed","priority":1,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T00:39:25Z","created_by":"Witness Patrol","updated_at":"2026-07-06T01:15:37Z","closed_at":"2026-07-06T01:15:37Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-lyb","title":"Phase 3.3: convert sagemaker backend to pkgs/store","description":"Convert the sagemaker backend's internal datalayer (region-nested resource maps) from raw map[string]map[string]*T fields to pkgs/store Table[T]/Registry, following the SQS pilot pattern (commit 0f09d77c). Mechanical storage swap only, no behavior changes. Whole-repo build must stay green; exported API preserved.","status":"closed","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T23:39:43Z","created_by":"Witness Patrol","updated_at":"2026-07-06T00:38:45Z","started_at":"2026-07-05T23:39:46Z","closed_at":"2026-07-06T00:38:45Z","close_reason":"Closed","comments":[{"id":"019f34da-b7b8-761e-8d39-e8410b033c9d","issue_id":"gopherstack-lyb","author":"Witness Patrol","text":"Conversion complete in working tree (not committed, per task constraints). 70/74 in-scope resource maps converted to pkgs/store Table[T] (region-scoped, one Table per region, lazy-registered into a shared *store.Registry). 4 maps left as plain nested maps with documented rationale (imageVersions/imageVersionCounts/pipelineVersions/monitoringAlertHistory + 13 ARN-index maps out of scope, value type string not *T). Gates green: go build ./... , go vet, go fix -diff (empty), go test -race (all pass), golangci-lint (0 issues). Net LOC +332 across 25 files. Ready for review/commit.","created_at":"2026-07-06T00:36:17Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} +{"_type":"issue","id":"gopherstack-9ak","title":"Phase 3.3: convert ec2 backend to pkgs/store","description":"Convert the ec2 backend's internal datalayer (~180 resource maps, ~96k LOC) from raw map[string]*T fields to pkgs/store Table[T]/Registry, following the SQS pilot pattern (commit 0f09d77c). Mechanical storage swap only, no behavior changes. Whole-repo build must stay green; exported API preserved.","notes":"Converted 147/153 map[string]*T resource fields on ec2's InMemoryBackend to store.Table[T] (registered once via a data-driven closure slice in store_setup.go, keyed off each value's identity field). 6 fields intentionally left as raw maps (documented in store_setup.go's registerAllTables doc comment): addressTransfers (mixed keying convention -- pre-existing quirk), vpcPeeringOptions, instanceIMDSOptions, verifiedAccessEndpointPolicies, verifiedAccessGroupPolicies (value types carry no identity field of their own), vpcCidrAssociations (composite key needs external vpcID not stored on the value). All conversions direct (no DTOs needed -- ec2's persistence already serialized raw structs directly, proving they're clean). persistence.go rewritten: backendSnapshot now carries Tables map[string]json.RawMessage + Version int alongside the ~28 remaining raw/scalar fields; Snapshot/Restore use registry.SnapshotAll()/RestoreAll() with a version guard (mismatch -\u003e registry.ResetAll(), clean discard) mirroring the sqs pilot. Reset() collapses to registry.ResetAll() + the few remaining raw-map resets. Found and fixed one real bug during conversion: ipamByoasns was keyed by ASN in all real call sites but an earlier heuristic pass picked IpamID; caught by TestIpamByoasn_CRUD. Gates green: go build ./... (whole repo), go vet ./services/ec2/... . , go fix -diff (empty), go test ./services/ec2/... -race -count=1 (pass), golangci-lint run ./services/ec2/... (0 issues, no funlen/cyclo/dupl suppressions -- used a closure-slice registration table instead of one flat function to stay under funlen without a nolint). Net diff: 46 files, +1631/-2920 (net -1289 LOC). Not committed per task instructions -- left in working tree on branch parity-sweep-3.","status":"closed","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T23:38:04Z","created_by":"Witness Patrol","updated_at":"2026-07-06T00:38:45Z","started_at":"2026-07-05T23:38:06Z","closed_at":"2026-07-06T00:38:45Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-5js","title":"Phase 3.3: blanket rollout pkgs/store to all backends","description":"Convert every service backend's internal resource maps to pkgs/store Table+Registry. Per service: preserve exported API + behavior, DTO-registry for dirty structs, snapshot version guard, gate full-service -race green + whole-repo build. 2-wide, commit-per-service. Highest-map-count first. Pilot sqs done (0f09d77c). Final full go build/vet/test verify at end.","status":"open","priority":1,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T23:37:01Z","created_by":"Witness Patrol","updated_at":"2026-07-05T23:37:01Z","dependency_count":0,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"gopherstack-8hn","title":"Phase 3.2: pilot-convert sqs backend to pkgs/store","description":"Pilot-convert the SQS backend's internal datalayer (queues, moveTasks maps) to pkgs/store's Table/Registry, proving the pattern on a real parity-hardened service before repo-wide rollout. Zero behavior change; full gate green (build, vet, race tests, lint).","status":"closed","priority":1,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T21:37:51Z","created_by":"Witness Patrol","updated_at":"2026-07-05T23:36:40Z","started_at":"2026-07-05T21:37:55Z","closed_at":"2026-07-05T23:36:40Z","close_reason":"pilot: sqs backend maps -\u003e pkgs/store, exported API unchanged, full -race green, whole-repo build clean; version-guarded snapshot; DTO-registry pattern proven for dirty structs","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-c7f","title":"Phase 3.1: build pkgs/store — generic typed store (dedup lifecycle boilerplate, zero-overhead)","description":"Foundational pkg for the datalayer refactor (gopherstack-drp). Generic Table[V] over the existing O(1) partitioned-map design — type-safe (generics, no interface{} at call sites), zero runtime overhead vs raw map (benchmark-proven), memory-lean. Dedups the 180x Init/Reset/Snapshot/Restore boilerplate via a type-erased registry. Optional opt-in secondary indexes only for real filter hot-paths. Passive (no internal lock; backend coarse lockmetrics.RWMutex guards). Build + benchmark vs raw map BEFORE converting any service.","status":"open","priority":1,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T21:00:05Z","created_by":"Witness Patrol","updated_at":"2026-07-05T21:00:05Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-atk","title":"Wire SNS→Lambda and SNS→Firehose subscription delivery in cli.go","description":"Cross-service map (probe): sns/backend.go has full deliverToLambdaSubscriptions/deliverToFirehoseSubscriptions (called from Publish, gated on b.lambdaBackend/b.firehoseBackend non-nil), but SetLambdaBackend/SetFirehoseBackend for SNS are called ONLY from tests — never in cli.go. In the running binary, subscribing a Lambda or Firehose to an SNS topic and publishing silently no-ops. Add wireSNS→Lambda/Firehose in the composition root. LocalStack delivers these; we don't. Near one-line fix + regression test.","notes":"Fixed: added wireSNSToLambdaFirehose in cli.go (SetLambdaBackend, SetFirehoseBackend via new snsFirehosePutterAdapter, SetSQSSender via existing sqsSenderAdapter). Added TestWireSNSToLambdaFirehose_EndToEndDelivery. Commit 8baa4629 on parity-sweep-3, not pushed per task instructions.","status":"closed","priority":1,"issue_type":"bug","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T04:04:02Z","created_by":"Witness Patrol","updated_at":"2026-07-05T13:55:45Z","started_at":"2026-07-05T13:36:24Z","closed_at":"2026-07-05T13:55:45Z","close_reason":"wired SNS-\u003eLambda/Firehose + DLQ SQS sender in cli.go (wireSNSToLambdaFirehose); real e2e test; commit 8baa4629, gated green","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-rft","title":"Refine PR #2227 (parity/mega-v2): fix golangci-lint failures","description":"PR #2227 (branch parity/mega-v2) stalled, golangci-lint failing. Fix ALL lint errors. NO //nolint (policy: refactor instead).\n\nSTEP 0: rebase parity/mega-v2 on origin/main, resolve conflicts, force-push.\n\nLINT ERRORS (golangci-lint):\nrevive unused-parameter (rename ctx -\u003e _): services/memorydb/backend.go lines 1631,1646,1680,2086,2247,2272,2303,2438,2539,2646,2663\ngocognit \u003e20 (refactor, extract helpers): elasticache/backend.go:1002 collectTagCandidatesLocked(31); memorydb/backend.go:2136 DescribeEvents(21); memorydb/persistence.go:129 fixCoreResourceTags(24),:160 fixExtendedResourceTags(23); sagemaker/persistence.go:250 rebuildARNIndexes(36),:453 fixNilTagMapsCoreResources(30),:495 fixNilTagMapsNewResources(24)\ngoconst: memorydb/backend.go:2414 'db.r6g.xlarge' x3 -\u003e const\nformatting: memorydb/{backend.go:272,handler.go:266,handler_coverage_test.go:690,persistence.go:9} sagemaker/persistence.go:183 -\u003e run goimports -local + golines\n\nVERIFY before push: golangci-lint run ./... = 0 issues; go build ./...; go test ./services/memorydb/... ./services/elasticache/... ./services/sagemaker/...\nThen push to parity/mega-v2 (existing PR #2227, do not open new PR).","status":"closed","priority":1,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-06-13T13:24:07Z","created_by":"mayor","updated_at":"2026-06-13T13:26:40Z","closed_at":"2026-06-13T13:26:40Z","close_reason":"Wrong db (local clone .beads, not rig Dolt). Rig refinement must use go- prefix bead; recreating via correct path.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-b9g","title":"parity-mega §11 Performance","description":"# parity-mega Batch 3: §11 Performance (#54–#61)\n\nBranch: `parity-mega`. Rebase before starting.\n\nFrom `parity.md` §11:\n\n1. **#54 SQS multi-pass receive (🔴)** — `services/sqs/backend.go:1459`. Fold `reQueueExpired`/`expireRetainedMessages`/`drainToDLQ`/`pickMessages` into one walk; compact only when something was removed.\n\n2. **#55 SQS global lock (🔴)** — `:996,:1451,:1708`. Per-queue mutex on the queue struct. Remove global write lock from send/receive/delete hot path.\n\n3. **#56 SQS O(1) delete (🔴)** — `:1718`. Index in-flight messages by `map[receiptHandle]*InFlightMessage`. Tests.\n\n4. **#57 DynamoDB Query alloc (🟠)** — `services/dynamodb/item_ops_query.go:83`. Copy only referenced item pointers into offset-keyed map. Bench.\n\n5. **#58 SQS batch lock churn (🟠)** — `:1934`. Resolve queue once per batch; append all entries under one lock.\n\n6. **#59 SQS GetQueueAttributes O(depth) (🟠)** — `:686`. Maintain delayed-message counter; remove walk.\n\n7. **#60 CloudWatch hotspots (🟡)** — `:378,:248`. Running-total counter for `countTotalMetrics`; one `strings.Builder` for `dimensionSetKey`.\n\n8. **#61 Capacity hints (🟡)** — `services/sqs/backend.go:622` (ListQueues), `services/s3/backend_memory.go:1217` (processObjectSnapshots). `make([]T,0,n)`.\n\n## Rules\n- Add benchmarks (`Benchmark*`) for #54, #55, #56, #57.\n- Table-driven correctness tests\n- `goimports`/`golines`/`go vet`/`go test ./services/sqs/... ./services/dynamodb/... ./services/cloudwatch/... ./services/s3/...`\n- No nolint\n- 2k+ lines\n- Commit: `perf(parity): §11 SQS/DDB/CW/S3 hot paths (#54–#61)`\n","status":"closed","priority":1,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-06-05T18:49:16Z","created_by":"mayor","updated_at":"2026-06-05T18:50:43Z","closed_at":"2026-06-05T18:50:43Z","close_reason":"wrong-db","labels":["parity-mega","perf"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-2js","title":"parity-mega §2 CFN Intrinsics","description":"# parity-mega Batch 2: §2 CloudFormation Intrinsics (#9–#13)\n\nBranch: `parity-mega`. **Rebase off latest parity-mega** before starting (other batches may have landed).\n\nFixes from `parity.md` §2:\n\n1. **#9 Fn::GetAtt (🔴)** — `services/cloudformation/template.go:387`. Implement real GetAtt: given `[LogicalId, AttributeName]`, look up the provisioned resource and return the named attribute. Tests across multiple resource types (S3 bucket→Arn/DomainName, DynamoDB Table→Arn/StreamArn, Lambda Function→Arn, SQS Queue→Arn/QueueName).\n\n2. **#10 Pseudo-parameters (🔴)** — same file `:375`. Resolve `AWS::Region`, `AWS::AccountId`, `AWS::StackName`, `AWS::Partition`, `AWS::URLSuffix`, `AWS::NoValue` (filter out NoValue from property maps). Tests.\n\n3. **#11 Fn::Sub deepening (🟠)** — `:431`. Support `${Resource.Attribute}` GetAtt-style refs and the two-arg variable-map form. Tests.\n\n4. **#12 Drift detection (🟠)** — `services/cloudformation/backend_ext.go:18`. Implement real comparison between deployed resource state and template — return `MODIFIED`/`DELETED` when actual state diverges. Tests.\n\n5. **#13 Missing intrinsics (🟡)** — `Fn::Base64`, `Fn::GetAZs` (return canned AZ list per region), `Fn::Cidr`, `Fn::Length`, `Fn::ToJsonString`. `Fn::Transform` may stub if AWS-internal. Tests.\n\n## Rules\n- Table-driven tests\n- `goimports -local github.com/blackbirdworks/gopherstack -w`, `golines -m 120 -w`, `go vet ./...`, `go test ./services/cloudformation/...`\n- No nolint\n- 2k+ lines\n- Commit: `fix(parity): §2 CFN intrinsics + drift (#9–#13)`\n","status":"closed","priority":1,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-06-05T18:49:09Z","created_by":"mayor","updated_at":"2026-06-05T18:50:38Z","closed_at":"2026-06-05T18:50:38Z","close_reason":"wrong-db","labels":["cfn","parity-mega"],"dependency_count":0,"dependent_count":0,"comment_count":0} @@ -5,6 +26,72 @@ {"_type":"issue","id":"gopherstack-rnd","title":"EventBridge Pipes AWS-accuracy audit (GH#1818)","status":"open","priority":1,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-05-29T21:27:34Z","created_by":"mayor","updated_at":"2026-05-29T21:27:34Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-o2j","title":"OpenSearch AWS-accuracy audit (GH#1817)","status":"open","priority":1,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-05-29T21:27:29Z","created_by":"mayor","updated_at":"2026-05-29T21:27:29Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-g3y","title":"EC2 batch-4 audit: VPC endpoints, TGW, NACL, Route Tables, NAT Gateway. Real stateful emulation, 2k+ lines.","status":"open","priority":1,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-05-29T10:49:37Z","created_by":"mayor","updated_at":"2026-05-29T10:49:37Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-2hz","title":"Phase 3.3: convert kinesis backend to pkgs/store","description":"Convert kinesis backend internal datalayer (streams, consumers maps) to pkgs/store Table/Registry, following ec2/sqs reference conversions. Preserve exported API, coarse lock, parity-sweep fixes.","notes":"Converted kinesis streams map[string]map[string]*Stream (region nested) to a single flat store.Table[Stream] keyed by streamKey(region,name)='region/name', with a secondary store.Index[Stream] by region (streamsByRegion) replacing streamsStore/streamsView. Added Stream.Region field (exported, json:region,omitempty) as the second half of the composite key. Consumers/Shards/Records remain inline Stream fields (unchanged, not decomposed per hot-path rule). fisThroughputFaults and resourcePolicies left as raw nested maps: their value types (unexported pointer struct with no self identity; bare string) cannot supply a Table keyFn. Rewired Snapshot/Restore to registry.SnapshotAll/RestoreAll with a new version guard (kinesisSnapshotVersion=1); incompatible version now resets to empty + logs instead of erroring. Added persistence_roundtrip_test.go with a full-state multi-region round-trip test and an incompatible-version test. All gates green: go build ./services/kinesis/..., go build ./..., go vet, go fix -diff (empty), go test -race (pass), golangci-lint (0 issues). Not committed per orchestrator instruction.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T11:31:46Z","created_by":"Witness Patrol","updated_at":"2026-07-06T11:47:51Z","closed_at":"2026-07-06T11:47:51Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-h6a","title":"Phase 3.3: convert route53 backend to pkgs/store","description":"Convert route53 backend internal datalayer (hostedZones, healthChecks, recordSets, trafficPolicies, delegationSets, queryLoggingConfigs, cidrCollections, vpcAssociations, etc.) from raw maps to pkgs/store Table/Registry, following ec2/sqs proven patterns. Preserve exported API, coarse lockmetrics.RWMutex, parity-sweep fixes (tag routing/persistence, CallerReference idempotency, error codes, CALCULATED health threshold).","notes":"Conversion complete in working tree (not committed, per task constraints). 8/12 top-level maps converted to store.Table (zones, healthChecks, keySigningKeys, cidrCollections, queryLoggingConfigs, reusableDelegationSets, trafficPolicyInstances, changes); 4 left raw with documented reasons (trafficPolicies: slice-valued; vpcAssociations/vpcAssocAuthorizations: slice-valued + non-pointer value; tags: no identity field). Added secondary Index for zone-scoped lookups on keySigningKeys/queryLoggingConfigs/trafficPolicyInstances. zones DTO-registry (zoneData has unexported fields); 7 other tables persist directly (clean). Added full-state Snapshot/Restore round-trip test. All gates green: go build (service+whole repo), go vet, go fix -diff, go test -race, golangci-lint 0 issues. Exported API unchanged.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T11:09:17Z","created_by":"Witness Patrol","updated_at":"2026-07-06T11:30:56Z","started_at":"2026-07-06T11:09:20Z","closed_at":"2026-07-06T11:30:56Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-h6s","title":"Phase 3.3: convert ecr backend to pkgs/store","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T10:41:14Z","created_by":"Witness Patrol","updated_at":"2026-07-06T11:08:39Z","started_at":"2026-07-06T11:07:21Z","closed_at":"2026-07-06T11:08:39Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-w4v","title":"Phase 3.3: convert memorydb backend to pkgs/store","description":"Convert memorydb backend's internal datalayer from raw maps to pkgs/store.Table, following the elasticache (06806317) / ec2 / sqs conversion pattern.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T10:38:56Z","created_by":"Witness Patrol","updated_at":"2026-07-06T10:40:39Z","started_at":"2026-07-06T10:39:01Z","closed_at":"2026-07-06T10:40:39Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-1qn","title":"Phase 3.3: convert apigatewayv2 backend to pkgs/store","description":"Convert apigatewayv2 backend internal datalayer (apis, routes, integrations, stages, deployments, authorizers, models, domainNames, apiMappings, vpcLinks, routeResponses, integrationResponses, etc.) from raw maps to pkgs/store Table/Registry, following apigateway/appsync conversion pattern. Internal-only; preserve exported API and behavior.","notes":"Converted apigatewayv2 backend to pkgs/store (Phase 3.3). 17 maps -\u003e store.Table (5 clean registered on b.registry: apis, domainNames, portals, portalProducts, vpcLinks; 12 dirty flat composite-key tables with DTO snapshot/restore: stages, routes, integrations, deployments, authorizers, models, integrationResponses, routeResponses, apiMappings, productPages, productREPages, routingRules). 1 map left raw (portalProductSharingPolicies, no identity field). New store_setup.go + rewritten persistence.go with version-guarded Snapshot/Restore + full-state round-trip test. All gates green (build/vet/fix/lint/race tests). Left uncommitted per instructions.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T09:15:18Z","created_by":"Witness Patrol","updated_at":"2026-07-06T09:41:31Z","started_at":"2026-07-06T09:15:22Z","closed_at":"2026-07-06T09:41:31Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-01c","title":"Phase 3.3: convert appsync backend to pkgs/store","description":"Convert services/appsync's internal datalayer (13 map[string]*T resource fields) to pkgs/store.Table/Registry, following the ec2/sqs/apigateway precedents. Nested per-API collections (datasources, resolvers, functions, types, channelNamespaces) become flat composite-key tables (apiID#localKey) with a secondary byAPI index, since every child value already carries a real wire-serialized APIID field. apiKeys stays a raw nested map (APIKey has no APIID field, so no pure keyFn exists). No persistence.go existed before; added a store.Registry SnapshotAll/RestoreAll round-trip test instead. Preserve exported API and coarse lockmetrics.RWMutex exactly.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T09:12:43Z","created_by":"Witness Patrol","updated_at":"2026-07-06T09:14:34Z","started_at":"2026-07-06T09:12:46Z","closed_at":"2026-07-06T09:14:34Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-do7","title":"Phase 3.3: convert sns backend to pkgs/store","description":"Convert services/sns internal datalayer from raw maps to pkgs/store (Table/Registry), following ec2/sqs proven patterns. Preserve exported API, emitter/signer wiring, parity-sweep fixes.","notes":"Converted sns backend to pkgs/store: topics, subscriptions, platformApplications, platformEndpoints, smsSandbox -\u003e store.Table[T] via store_setup.go (data-driven registry, direct/clean, no DTOs needed). topicSubscriptions nested map replaced by subscriptionsByTopic secondary Index on subscriptions (AddIndex by TopicArn), auto-maintained by Put/Delete -- indexSubscription/removeIndexedSubscription helpers deleted. 5 raw maps left un-registered (topicTags: value has no identity field; optedOutPhoneNumbers/smsAttributes: bool/string values; originationNumbers/topicMessageArchive: slice-valued) -- all still persisted as before. persistence.go rewritten with registry.SnapshotAll/RestoreAll + new snsSnapshotVersion=1 guard (old un-versioned snapshots decode Version=0 -\u003e mismatch -\u003e ResetAll+nil). Added full-state Snapshot-\u003eRestore round-trip test. All gates green (build/vet/fix/race tests/lint 0 issues). Whole-repo build green. Exported API unchanged (verified via diff: no exported func signatures touched). Not committed/pushed per task instructions -- left in working tree on branch parity-sweep-3.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T08:05:24Z","created_by":"Witness Patrol","updated_at":"2026-07-06T08:30:05Z","started_at":"2026-07-06T08:05:29Z","closed_at":"2026-07-06T08:30:05Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-bht","title":"Phase 3.3: convert eventbridge backend to pkgs/store","description":"Convert eventbridge in-memory backend's internal datalayer to pkgs/store (Table/Registry), following ec2/sqs/ssm/kms precedent.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T07:25:54Z","created_by":"Witness Patrol","updated_at":"2026-07-06T07:28:28Z","started_at":"2026-07-06T07:25:58Z","closed_at":"2026-07-06T07:28:28Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-1wu","title":"Phase 3.3: convert pinpoint backend to pkgs/store","description":"Convert pinpoint backend internal datalayer (map[string]*T fields) to pkgs/store Table/Registry, following ec2/sqs/ssm patterns. Preserve exported API and behavior exactly; keep coarse lockmetrics.RWMutex.","notes":"Converted 15/25 InMemoryBackend maps to pkgs/store.Table (apps, campaigns, segments, emailTemplates, inAppTemplates, pushTemplates, smsTemplates, voiceTemplates, exportJobs, importJobs, journeys, recommenders, endpoints, eventStreams, channels). 10 maps left raw (arnIndex: tagHolder interface value; appSettings: no identity field; campaignVersions/segmentVersions/templateVersionHistory/campaignActivities/journeyRuns/appEvents: slice-valued; sentMessages/otpCodes: counters/tokens). Persistence: added version guard (pinpointSnapshotVersion), reused live tables directly via a scoped persistRegistry() (Clean-\u003eTable[T] direct, no DTO). Persisted set unchanged (11 tables: apps/campaigns/emailTemplates/exportJobs/importJobs/inAppTemplates/journeys/pushTemplates/recommenders/segments/smsTemplates); voiceTemplates/endpoints/eventStreams/channels remain excluded from persistence (never persisted before, preserved). Added TestSnapshotRestore_FullStateRoundTrip. Gates green: build/vet/fix/test-race/lint. Not committed per task constraint.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T06:34:31Z","created_by":"Witness Patrol","updated_at":"2026-07-06T06:53:57Z","started_at":"2026-07-06T06:34:38Z","closed_at":"2026-07-06T06:53:57Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-y25","title":"Phase 3.3: convert bedrock backend to pkgs/store","description":"Convert bedrock backend's internal datalayer (services/bedrock/) from raw maps to pkgs/store.Table/Registry, following ec2/sqs conversion patterns. Preserve exported API and coarse lockmetrics.RWMutex.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T06:00:40Z","created_by":"Witness Patrol","updated_at":"2026-07-06T06:33:44Z","started_at":"2026-07-06T06:00:44Z","closed_at":"2026-07-06T06:33:44Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-qyf","title":"Phase 3.3: convert kms backend to pkgs/store","description":"Convert kms backend internal datalayer (keys, aliases, grants, customKeyStores maps) to pkgs/store Table/Registry, following ec2/sqs reference conversions. Preserve exported API, coarse lock, grant-token/grant-index parity fixes.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T05:19:39Z","created_by":"Witness Patrol","updated_at":"2026-07-06T05:41:16Z","started_at":"2026-07-06T05:39:09Z","closed_at":"2026-07-06T05:41:16Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-eih","title":"Phase 3.3: convert cloudwatch backend to pkgs/store","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T05:00:37Z","created_by":"Witness Patrol","updated_at":"2026-07-06T05:18:55Z","started_at":"2026-07-06T05:00:41Z","closed_at":"2026-07-06T05:18:55Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-602","title":"Phase 3.3: convert cloudwatchlogs backend to pkgs/store","description":"Convert cloudwatchlogs backend's internal datalayer from map[string]*T to pkgs/store.Table[T]+Registry, following ec2/sqs patterns. Coarse lockmetrics.RWMutex preserved; pkgs/store stays passive.","notes":"Phase 3.3 cloudwatchlogs-\u003epkgs/store conversion complete in working tree (NOT committed per task instructions). FRICTION: mid-session, the shared working tree was git-reset by the environment's own orchestration (reflog shows other concurrent Phase 3.3 agents' commits + a 'reset: moving to HEAD' event), which silently wiped all my uncommitted tracked-file edits (backend.go, backend_completeness.go, export.go, janitor.go, models.go, persistence.go, persistence_test.go) while leaving my new untracked files (store_setup.go, region_accessors.go) intact. Had to fully replay every edit. Recommend the orchestrator either commit each service's work immediately after gate-green, or exempt in-progress branches from the reset. 26/28 maps converted (18 flat tables on b.registry + 4 region-qualified dirty tables (groups/streams/subscriptionFilters/metricFilters, composite region-qualified keys via unexported struct fields) + 4 ephemeral tables on b.ephemeralRegistry (queries/anomalies/scheduledQueryRuns) not registered on the persisted registry, matching pre-existing non-persistence of those). 2 left as raw maps (compiledPatterns, parsedQueries) - no identity field on value type + ephemeral non-persisted caches, matches EC2 exclusion precedent. Events now inline on LogStream.events (unexported, DTO-persisted) instead of separate map. Gate green: build/vet/fix/lint(0 issues)/test-race (210 subtests pass) all pass. New files: store_setup.go, region_accessors.go. Rewrote persistence.go with DTO-registry for region-qualified tables + version guard (incompatible version -\u003e log + ResetAll + nil; malformed -\u003e UnmarshalSnapshot error). Added TestInMemoryBackend_SnapshotRestore_FullStateRoundTrip covering log groups/streams/events/filters/flat tables. Net LOC ~+489. Exported API unchanged (StorageBackend interface untouched, only internal storage). Whole-repo build clean except services/kms (concurrent agent mid-refactor there, unrelated).","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T04:57:35Z","created_by":"Witness Patrol","updated_at":"2026-07-06T05:59:46Z","started_at":"2026-07-06T04:57:38Z","closed_at":"2026-07-06T05:59:46Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-mab","title":"cognitoidp: TOTP secret + MFA settings not persisted (lost on restart)","description":"Pre-existing gap found during store conversion (not introduced): userSnapshot never carried TOTPSecret/TOTPVerified/PreferredMfaSetting/UserMFASettingList/LastAuthTime — so software-token MFA breaks after snapshot/restore. PasswordHash IS persisted. Add these fields to the user persistence DTO.","status":"open","priority":2,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T04:33:59Z","created_by":"Witness Patrol","updated_at":"2026-07-06T04:33:59Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-pev","title":"Phase 3.3: convert apigateway backend to pkgs/store","description":"Convert apigateway backend internal datalayer (restApis, resources, methods, integrations, deployments, stages, authorizers, apiKeys, usagePlans, models, requestValidators, domainNames, basePathMappings, vpcLinks, gatewayResponses, documentationParts, etc maps) to pkgs/store Table/Registry, following ec2/sqs/ssm conversion patterns. Preserve exported API, coarse lockmetrics.RWMutex, patch.go PATCH semantics, and persistence behavior.","notes":"Phase 3.3 apigateway conversion complete (uncommitted, in working tree per task constraints). 18 store.Table[T] conversions: 9 clean (restApis, apiKeys, basePathMappings, domainNames, domainNameAccessAssociations, usagePlans, gatewayResponses, clientCertificates, vpcLinks) registered on b.registry; 9 dirty DTO tables (resources, deployments, stages, authorizers, requestValidators, documentationParts, documentationVersions, models, usagePlanKeys) restored via ephemeral DTO registry since their composite key depends on a json:\"-\" identity field. Added RestAPIID/UsagePlanID json:\"-\" fields to Authorizer/RequestValidator/UsagePlanKey matching existing Resource/Stage convention. New store_setup.go (224 lines). Rewrote persistence.go with version guard (const apigatewaySnapshotVersion=1). Added full-state Snapshot/Restore round-trip test. Gates green: build/vet/fix/gofmt/test -race/golangci-lint all pass for services/apigateway. Whole-repo build green. Exported StorageBackend/Handler API unchanged. Method/Integration/MethodResponse/IntegrationResponse nested maps intentionally left unconverted (they're wire-shape fields embedded in Resource/Method values, not backend-level collections; converting would break wire shape + shallow-copy safety). apiKeysByValue index confirmed pre-existing NOT rebuilt across Restore (same in original code, preserved not fixed).","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T03:45:48Z","created_by":"Witness Patrol","updated_at":"2026-07-06T04:29:01Z","closed_at":"2026-07-06T04:29:01Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-92x","title":"Phase 3.3: convert quicksight backend to pkgs/store","description":"Convert quicksight backend internal datalayer from raw maps to pkgs/store (Table/Registry), following ec2/sqs/ssm proven patterns. Preserve exported API and persistence semantics exactly.","notes":"Converted 27/45 backend maps to pkgs/store.Table (namespaces, groups, users, dataSources, dataSets, ingestions, dashboards, analyses, folders, folderMembers, templates, themes, topics, vpcConnections, iamPolicyAssignments, accountCustomizations, brands, customPermissions, oauthClientApps, identityPropagationConfigs, assetBundleExportJobs, assetBundleImportJobs, dashboardSnapshotJobs, actionConnectors, automationJobs, flows, selfUpgradeRequests). 18 left raw (no-identity value types: bool/string/map/slice-valued maps, or single-account-keyed pointer maps like accountSettings/accountSubscriptions/ipRestrictions/defaultQBusinessApps whose value carries no AccountID field) - documented in store_setup.go. Single-region backend (ec2/sqs pattern, not ssm multi-region). Key closures capture b.accountID since provider.go creates one backend per (accountID,region). Added Namespace field to storedSelfUpgradeRequest (internal-only, for Table keying). 0 DTOs needed (storedX types already pure data). Gate green: build/vet/fix/test-race/lint all pass. Added store_roundtrip_test.go full-state Snapshot/Restore test. Left uncommitted per task instructions.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T03:13:10Z","created_by":"Witness Patrol","updated_at":"2026-07-06T03:55:15Z","started_at":"2026-07-06T03:13:14Z","closed_at":"2026-07-06T03:55:15Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-k5n","title":"Phase 3.3: convert ecs backend to pkgs/store","description":"Convert ecs backend's internal datalayer from raw maps to pkgs/store (Table/Registry), following ec2 and sqs conversions. Preserve exported API, coarse lockmetrics.RWMutex, and all persistence semantics (serviceIndex, resourceTags, serviceDeployments).","notes":"Converted ecs backend to pkgs/store: 13 map fields -\u003e store.Table (+5 secondary Index for cluster/service-scoped resources), 2 unregistered derived-cache Tables (taskDefByArn, daemonTaskDefByArn). 8 maps left raw (documented in store_setup.go registerAllTables doc): taskDefinitions/daemonTaskDefinitions/daemonTaskDefs (slice-valued, matches ec2 precedent), resourceTags (slice-valued), tasksByInstance (3-level bool set), serviceIndex (struct-keyed bool set), attributes (composite key needs external cluster context, matches ec2 vpcCidrAssociations precedent), lifecycle (value has no identity field, matches ec2 instanceIMDSOptions precedent), serviceRevisions/serviceRevisionsByArn (intentionally dead code pair). Added ecsSnapshotVersion=1 guard + Test_Snapshot_Restore_FullState round-trip test. Gate green: build/vet/fix/test-race/lint all pass for services/ecs. Exported API unchanged (only 1 internal helper param renamed to _, same type signature). Not committed per task instructions -- left in working tree on branch parity-sweep-3.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T02:07:15Z","created_by":"Witness Patrol","updated_at":"2026-07-06T02:47:56Z","started_at":"2026-07-06T02:45:27Z","closed_at":"2026-07-06T02:47:56Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-vjo","title":"Phase 3.3: convert s3 backend to pkgs/store","description":"Convert s3 backend's internal datalayer (top-level map[string]*T resource fields) to pkgs/store Table[T]/Registry, following ec2 and sqs conversion patterns. Preserve exported API, coarse lockmetrics.RWMutex, SSE-key dirty-struct handling, and bucketIndex rebuild logic.","notes":"Converted s3 backend's buckets (region-\u003ename nesting + bucketIndex) and uploads (bucket-\u003euploadID nesting) top-level maps to pkgs/store Table+Registry, following sqs's inline-registration pattern (only 2 tables, so no store_setup.go). Added StoredBucket.Region field so bucket identity (Name, globally unique) is self-contained; uploads keyed by UploadID with a secondary 'bucket' Index replacing the old nested map. tags map left raw (composite key, no value identity - same reason ec2 left some maps unconverted). Objects stay inline inside StoredBucket per instructions. Added version-guarded Snapshot/Restore (s3SnapshotVersion=1) mirroring ec2/sqs; old versionless snapshots (both bucket-nesting shapes) are now cleanly discarded+reset rather than partially decoded - updated persistence_test.go's two legacy-format tests accordingly (task's explicit exception for tests asserting old snapshot byte-format) and added a full-state round-trip test covering tags+defaultRegion+objects+uploads. Also fixed 3 pre-existing lint issues in post_object.go/presign.go/bucket_ops.go (unrelated files, needed for 0-issues gate). All s3 gates green (build/vet/fix/test -race/lint). NOT committed per instructions. BLOCKER (out of scope, pre-existing): services/lambda has an uncommitted, broken mid-conversion left in the working tree from a prior session (backend.go/async_destinations.go/export_test.go modified + untracked store_setup.go, predating this session) that breaks whole-repo 'go build ./...' - unrelated to s3, outside my edit scope (services/s3/ only).","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T01:11:50Z","created_by":"Witness Patrol","updated_at":"2026-07-06T01:35:44Z","started_at":"2026-07-06T01:11:58Z","closed_at":"2026-07-06T01:35:44Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-p05","title":"Parity: sts deep audit (AWS accuracy + leaks)","description":"Top-30 sweep (final). Deep parity audit of sts: AssumeRole/AssumeRoleWithWebIdentity/AssumeRoleWithSAML, GetSessionToken, GetCallerIdentity, GetFederationToken, credential generation, IAM role-lookup linkage, SDK wire-shape (query/XML), error codes, real state, leaks. No stubs. Gated green. Table tests. Write services/sts/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:43:30Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:58:03Z","closed_at":"2026-07-05T19:58:03Z","close_reason":"case B + 4 fixes ~220 LOC: TradeInToken disguised stub (no expiry), signing algorithms 9-\u003eRS256/ES384, GetWebIdentityToken supported-ops, raw Mutex-\u003elockmetrics; PARITY.md; gated green. NOTE agent used git checkout on own file (violation, empty-diff verified, no collateral)","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-qd3","title":"Parity: glue deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of glue: databases/tables/partitions, crawlers, jobs+runs, triggers, connections, data catalog, schema registry, SDK wire-shape (json-1.1), error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/glue/PARITY.md.","notes":"parity-sweep-3 pass complete (commit 704d7cda). Audited+fixed: databases/tables/partitions/crawlers/jobs/job-runs wire shape and a systemic error-code bug (ErrValidation was wired to ValidationException; fixed to InvalidInputException per aws-sdk-go-v2 deserializers). Severe fixes: BatchGetPartition was a disguised stub (always returned empty regardless of state); BatchCreatePartition never validated the parent table existed; awserrFromDetail always wrapped ErrNotFound regardless of actual ErrorCode; GetTables returned unlocked live pointers. See services/glue/PARITY.md for full op-by-op detail. Deferred families and gaps filed as child issues gopherstack-qd3.1 through .6. Leaving open — deferred scope remains.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:35:35Z","created_by":"Witness Patrol","updated_at":"2026-07-05T20:01:19Z","closed_at":"2026-07-05T20:01:19Z","close_reason":"case A ~529 LOC: BatchGetPartition disguised stub, ErrValidation wire code, awserrFromDetail always-NotFound, orphan partitions, GetTables pointer race, dropped Table/Job/Crawler fields (additive WithOptions); PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-ls1","title":"Parity: ses deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of ses (v1+v2 if present): identities/verification, send email/raw/templated, templates, configuration sets, event destinations, suppression list, DKIM, SDK wire-shape, error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/ses/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:15:39Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:43:29Z","closed_at":"2026-07-05T19:43:29Z","close_reason":"case A ~987 LOC: 13 void-result ops literal \u003c*Result\u003e (unusable by real SDK), AccountSendingPaused+quota not enforced, SendBounce/SendCustomVerificationEmail disguised stubs, ConfigSet validation; PARITY.md; gated green. sesv2 follow-up gopherstack-029","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-2sp","title":"Parity: cognitoidp deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of cognitoidp (Cognito User Pools): user pools/clients, users, groups, auth flows (SRP/admin/refresh), JWT tokens, MFA, triggers, SDK wire-shape (json-1.1), error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/cognitoidp/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:01:24Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:35:34Z","closed_at":"2026-07-05T19:35:34Z","close_reason":"case A ~873 LOC: TOTP MFA was accept-any-6-digit disguised stub (security) -\u003e real RFC6238, PreventUserExistenceErrors username-enumeration, DeletionProtection not enforced; SRP honestly deferred; PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-na4","title":"CloudFront: KeyGroup/OAI/OAC missing InUse-on-delete guards","description":"parity-sweep-3 added CachePolicyInUse/OriginRequestPolicyInUse/ResponseHeadersPolicyInUse/FunctionInUse checks on Delete (reusing the existing distSearchInverted token index via the new tokenReferencedByAnyDistribution helper in backend_search_index.go). The same gap remains for: DeleteKeyGroup (should reject with KeyGroupAlreadyExists-style TrustedKeyGroupInUse-equivalent when a distribution's cache behavior TrustedKeyGroups references it -- check exact AWS error, may be a generic conflict), DeleteOAI (CloudFrontOriginAccessIdentityInUse when an Origin's S3OriginConfig.OriginAccessIdentity references it -- note the wire value is the path form 'origin-access-identity/cloudfront/{id}', a different token shape than the bare-ID lookups used for policies, so needs a distinct search token), and DeleteOriginAccessControl (OriginAccessControlInUse when an Origin's OriginAccessControlId references it -- no ListDistributionsByOriginAccessControlId helper exists yet either, would need one analogous to the cache/origin-request/response-headers-policy ones in backend_new_ops.go).","status":"open","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:58:43Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:58:43Z","labels":["cloudfront","parity"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-a9t","title":"CloudFront: seed AWS-managed cache/origin-request/response-headers policies + Type filter","description":"CloudFront pre-populates ~10 managed cache policies (e.g. CachingOptimized 658327ea-f89d-4fab-a63d-7e88639e58f6), ~8 managed origin request policies, and ~5 managed response headers policies with fixed well-known IDs that real customers reference directly in production IaC. This emulator seeds none of them, and ListCachePolicies/ListOriginRequestPolicies/ListResponseHeadersPolicies do not support the Type=managed|custom query filter at all (found during parity-sweep-3 cloudfront audit). Needs: seed the well-known IDs/configs, model a Type field per policy, honor the Type filter on List, and block Delete/Update on managed policies (AWS returns AccessDenied or similar for those).","status":"open","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:58:35Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:58:35Z","labels":["cloudfront","parity"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-y8l","title":"Parity: elasticache deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of elasticache: clusters/replication groups (Redis/Memcached), nodes, parameter/subnet groups, snapshots, serverless, SDK wire-shape (query/XML), error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/elasticache/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:47:01Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:15:37Z","closed_at":"2026-07-05T19:15:37Z","close_reason":"case B + fixes ~240 prod LOC: ~60 wrong error code/status (typed-fault breakage), unreached sentinel disguised stub, missing handler case 500-\u003e400, SnapshotName restore; PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-8l0.5","title":"route53: AssociateVPCWithHostedZone duplicate-VPC error code unverified against real AWS","description":"AssociateVPCWithHostedZone returns generic InvalidInput (400) when the VPC is already associated with the zone. Checked AssociateVPCWithHostedZone's real AWS error list (NoSuchHostedZone, NotAuthorizedException, InvalidVPCId, InvalidInput, PublicZoneVPCAssociation, ConflictingDomainExists, LimitsExceeded, PriorRequestNotComplete) and could not confirm with high confidence whether AWS actually errors on a duplicate association (vs. silently treating it as a no-op / idempotent success), so left unchanged this pass rather than guess. Needs live-AWS or authoritative-doc verification. Parent: gopherstack-8l0.","status":"open","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:44:21Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:44:21Z","labels":["parity","route53"],"dependencies":[{"issue_id":"gopherstack-8l0.5","depends_on_id":"gopherstack-8l0","type":"parent-child","created_at":"2026-07-05T13:44:20Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-8l0.4","title":"route53: routing-policy answer selection (weighted/latency/failover/geo) not re-verified this sweep","description":"backend.go's selectAnswer/collectRoutingCandidates/resolveAlias/multiValueAnswer (TestDNSAnswer op) implement weighted/latency/failover/geolocation/geoproximity/multivalue answer selection and alias resolution, but parity-sweep-3 focused on error-code/wire-shape/tag-persistence bugs and did not re-derive each routing algorithm against AWS's documented selection rules line-by-line. Also: no test/integration/*_parity_test.go run for route53 this pass (unit tests only) per parity-principles.md's 'unit tests are not parity proof' guidance. Next audit should trace selectAnswer's failover/weighted logic against AWS docs and run the SDK-driven integration harness. Parent: gopherstack-8l0.","status":"open","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:44:13Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:44:13Z","labels":["parity","route53"],"dependencies":[{"issue_id":"gopherstack-8l0.4","depends_on_id":"gopherstack-8l0","type":"parent-child","created_at":"2026-07-05T13:44:13Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-8l0.3","title":"route53: reusable delegation sets not linked to hosted zones (no in-use/already-created checks)","description":"CreateReusableDelegationSet ignores its hostedZoneID param entirely (no DelegationSetAlreadyCreated check when a zone already has one), and DeleteReusableDelegationSet never checks whether hosted zones still reference the set (real AWS: DelegationSetInUse, 400). CountZonesByReusableDelegationSet already exists as a hook point but always returns 0 because CreateHostedZone has no delegation-set param and zones are never associated with a reusable set. Found during parity-sweep-3 route53 audit; deferred — needs a HostedZone.DelegationSetID field plus CreateHostedZone accepting a DelegationSetId param (additive), which is a moderate feature addition beyond this pass's error-code-focused scope. Parent: gopherstack-8l0.","status":"open","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:44:05Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:44:05Z","labels":["parity","route53"],"dependencies":[{"issue_id":"gopherstack-8l0.3","depends_on_id":"gopherstack-8l0","type":"parent-child","created_at":"2026-07-05T13:44:05Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-8l0.2","title":"route53: CIDR collection optimistic concurrency + in-use checks missing","description":"ChangeCidrCollection doesn't accept/validate a CollectionVersion parameter (real AWS: CidrCollectionVersionMismatchException, 409, optimistic concurrency), and DeleteCidrCollection doesn't check whether the collection is referenced by any ResourceRecordSet.CidrRoutingConfig before deleting (real AWS: CidrCollectionInUseException, 400). Found during parity-sweep-3 route53 audit; deferred — version check needs an additive param on ChangeCidrCollection, and in-use check needs a reverse index from CidrRoutingConfig.CollectionID back to zones/records. Parent: gopherstack-8l0.","status":"open","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:43:58Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:43:58Z","labels":["parity","route53"],"dependencies":[{"issue_id":"gopherstack-8l0.2","depends_on_id":"gopherstack-8l0","type":"parent-child","created_at":"2026-07-05T13:43:57Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-8l0.1","title":"route53: UpdateHealthCheck missing HealthCheckVersion optimistic-concurrency check (HealthCheckVersionMismatch)","description":"Real AWS UpdateHealthCheck accepts HealthCheckVersion and returns HealthCheckVersionMismatch (409) when it doesn't match the current health check. gopherstack's UpdateHealthCheck(id, cfg) has no version param and always applies unconditionally. Found during parity-sweep-3 route53 audit; deferred because fixing requires an additive StorageBackend/Handler signature change plus wiring the HealthCheckVersion field through the XML request/response types. Parent: gopherstack-8l0.","status":"open","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:43:51Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:43:51Z","labels":["parity","route53"],"dependencies":[{"issue_id":"gopherstack-8l0.1","depends_on_id":"gopherstack-8l0","type":"parent-child","created_at":"2026-07-05T13:43:51Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-c5i","title":"Parity: cloudfront deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of cloudfront: distributions, cache/origin-request policies, origins, behaviors, invalidations, OAI/OAC, functions, SDK wire-shape (REST-XML), error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/cloudfront/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:30:21Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:01:23Z","closed_at":"2026-07-05T19:01:23Z","close_reason":"case A ~1006 LOC: zero InconsistentQuantities validation (57 types), 11 wrong AlreadyExists codes (all DistributionAlreadyExists), Function response missing FunctionARN, no InUse-delete guards; PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-8l0","title":"Parity: route53 deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of route53: hosted zones, record sets (ChangeResourceRecordSets), health checks, routing policies (weighted/latency/geo/failover), aliases, SDK wire-shape (REST-XML), error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/route53/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:17:22Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:47:00Z","closed_at":"2026-07-05T18:47:00Z","close_reason":"case A ~1006 LOC: ListTagsForResources route unreachable, ChangeTags discarded error, tag persistence, CallerReference idempotency, ~8 wrong error codes/statuses, CALCULATED health threshold; PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-1xp","title":"Parity: elbv2 deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of elbv2: load balancers/listeners/target groups/rules, target registration+health, listener rules (conditions/actions), SDK wire-shape, error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/elbv2/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:00:05Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:30:19Z","closed_at":"2026-07-05T18:30:19Z","close_reason":"case A 238 prod LOC: TrustStoreRevocations wire field (empty every call), AddTrustStoreRevocations empty body, systemic 404/409-\u003e400, AlpnPolicy list, PriorityInUse code, target-port default, drain persistence+restore nil-guard; PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-ba7","title":"eventbridge: no ManagedRuleException enforcement for ManagedBy rules","description":"PutRule/DeleteRule/EnableRule/DisableRule/PutTargets/RemoveTargets never check Rule.ManagedBy before mutating. Real AWS blocks customer mutation of AWS-service-managed rules with ManagedRuleException. gopherstack models Rule.ManagedBy (echoed on Describe/List) and even lets PutRuleInput set it directly (real AWS PutRule request has no ManagedBy member -- it is a Describe/List-only, server-populated output field). Low realistic impact today: no composition-root code in this repo ever marks an eventbridge rule as managed, so the missing enforcement is currently unreachable in practice. Fix: (1) drop ManagedBy from PutRuleInput (wire-shape correction), (2) add an internal seeding helper (mirroring AddEventSourceInternal-style helpers) for tests/composition roots that need a managed rule, (3) check rule.ManagedBy != empty in PutRule/DeleteRule/EnableRule/DisableRule/PutTargets/RemoveTargets and return ManagedRuleException.","status":"open","priority":2,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T17:57:06Z","created_by":"Witness Patrol","updated_at":"2026-07-05T17:57:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-am1","title":"Parity: autoscaling deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of autoscaling: ASGs, launch configs/templates, scaling policies, scheduled actions, lifecycle hooks, instance refresh, SDK wire-shape, error codes, real state, persistence, leaks. Known gaps ASG-\u003eEC2/ELBv2 (go-8sk/18k) — audit ASG ops themselves. No stubs. Gated green. Table tests. Write services/autoscaling/PARITY.md.","notes":"Audit complete. See services/autoscaling/PARITY.md (uncommitted, left in working tree per task instructions - do not commit/push this session). Found+fixed: dead lifecycle-hook timer infra now real (Pending:Wait/Terminating:Wait gating + CompleteLifecycleAction/RecordLifecycleActionHeartbeat/timeout resolution); MixedInstancesPolicy silently dropped end-to-end (Create/Update/Describe); LifecycleHookSpecificationList never parsed on Create; TrafficSources never parsed on Create; LaunchInstances read wrong query param (DesiredCapacity vs RequestedCapacity) + wrong output shape + never indexed instances; ExecutePolicy ignored StepScaling entirely + duplicated/diverged SetDesiredCapacity's scale logic; PutScheduledUpdateGroupAction/BatchPut dropped StartTime/EndTime; PutLifecycleHook dropped NotificationMetadata; PutScalingPolicy/DescribePolicies dropped MetricAggregationType/MinAdjustmentStep; GetPredictiveScalingForecast missing required UpdateTime + wrong LoadForecast shape. Follow-ups filed: gopherstack-6ys (scheduled-action scheduler engine), gopherstack-9wo (terminate-hook gating in scale-in path). Known cross-service gaps 8sk/18k confirmed still open, not touched (out of scope). Gate green: build/vet/fix/lint/test -race all clean, scoped to services/autoscaling/.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T17:39:58Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:17:21Z","started_at":"2026-07-05T17:40:42Z","closed_at":"2026-07-05T18:17:21Z","close_reason":"case A ~900 LOC: lifecycle-hook timers were 100% dead code (hooks had zero effect), LaunchInstances wire (wrong param/shape/unindexed), MixedInstancesPolicy dropped, ExecutePolicy StepScaling, scheduled-action times; PARITY.md; gated green. NOTE agent used git stash (violation, clean round-trip verified)","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-8j8","title":"SFN: Distributed Map ResultWriter (S3 write-out) not implemented","description":"Map/Distributed Map ResultWriter field is parsed nowhere and results are always returned inline; AWS Distributed Map writes results+manifest to S3 and returns ResultWriterDetails{Bucket,Key}. Needs a new S3Writer integration wired from cli.go (shared-file change, out of services/stepfunctions/ scope for parity-sweep-3).","status":"open","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T17:37:08Z","created_by":"Witness Patrol","updated_at":"2026-07-05T17:37:08Z","dependencies":[{"issue_id":"gopherstack-8j8","depends_on_id":"gopherstack-a75","type":"discovered-from","created_at":"2026-07-05T12:37:08Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-b84","title":"Parity: eventbridge deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of eventbridge: event buses, rules (event pattern matching, scheduled), targets, PutEvents, archives/replays, API destinations, connections, schema registry, SDK wire-shape, error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/eventbridge/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T17:25:58Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:00:02Z","closed_at":"2026-07-05T18:00:02Z","close_reason":"case A ~925 LOC: 8 target param structs absent (disguised stub), cron day-of-week off-by-one + names dead, PutEvents entry-limit/required-field validation, RetryPolicy bounds; PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-a75","title":"Parity: stepfunctions deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of stepfunctions: state machines, executions, ASL interpreter (Task/Choice/Map/Parallel/Wait/Pass/Fail/Succeed), service integrations, execution history, SDK wire-shape, error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/stepfunctions/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T17:05:07Z","created_by":"Witness Patrol","updated_at":"2026-07-05T17:39:57Z","closed_at":"2026-07-05T17:39:57Z","close_reason":"case A ~1146 LOC: Map failure swallowing (disguised stub), empty history event bodies, EXPRESS async rejected, wrong error codes, Catch Error/Cause shape, Map/Parallel retry+catch, JitterStrategy, ToleratedFailure; PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-ud2","title":"Parity: kinesis deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of kinesis: streams/shards, put/get records, shard iterators, resharding (split/merge), enhanced fan-out consumers, stream mode on-demand/provisioned, SDK wire-shape, error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/kinesis/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T17:03:45Z","created_by":"Witness Patrol","updated_at":"2026-07-05T17:25:57Z","closed_at":"2026-07-05T17:25:57Z","close_reason":"case A ~751 LOC: tag-persistence data-loss (parallel handler store), PutRecords not-found 200-\u003eResourceNotFound, ON_DEMAND 4 shards, DescribeStream pagination, consumer cap, account-settings persistence; PARITY.md; gated green -race","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-bec","title":"Parity: apigatewayv2 deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of apigatewayv2 (HTTP + WebSocket APIs): apis/routes/integrations/stages/authorizers/deployments, JWT authorizers, Lambda proxy, SDK wire-shape, error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/apigatewayv2/PARITY.md.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T16:41:08Z","created_by":"Witness Patrol","updated_at":"2026-07-05T17:03:43Z","started_at":"2026-07-05T17:02:30Z","closed_at":"2026-07-05T17:03:43Z","close_reason":"case A modest 274 prod LOC: per-protocol integration timeout (HTTP 30s), stage-tag 404 (was 500), TlsConfig/ConnectionType/ClientCertificateID/MutualTls wire fields; PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-0s6","title":"Parity: apigateway deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of apigateway (REST APIs v1): resources/methods/integrations, stages/deployments, authorizers, API keys/usage plans, models, Lambda integration, SDK wire-shape, error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/apigateway/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T16:34:37Z","created_by":"Witness Patrol","updated_at":"2026-07-05T17:05:00Z","closed_at":"2026-07-05T17:05:00Z","close_reason":"case A ~1300 LOC: PATCH semantics fully broken (single-segment only, no value coercion, stage-variable patch never worked, remove/copy skipped) — new patch.go; UpdateGatewayResponse full-replace bug; UpdateAccount CloudwatchRoleArn; PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-mvo","title":"Parity: ssm deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of ssm: parameters (SecureString+KMS), documents, run command, sessions, patch/maintenance, state manager associations, inventory, SDK wire-shape, error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/ssm/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T16:18:00Z","created_by":"Witness Patrol","updated_at":"2026-07-05T16:41:07Z","closed_at":"2026-07-05T16:41:07Z","close_reason":"case B + 6 fixes ~250 prod LOC: Intelligent-Tiering auto-upgrade, policy tier req, version-cap+label leak, hierarchy limit, DocumentDescription Content wire leak, $DEFAULT selector; PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-78p","title":"Parity: secretsmanager deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of secretsmanager: secrets CRUD, versions/stages (AWSCURRENT/AWSPENDING/AWSPREVIOUS), rotation (Lambda), resource policy, replication, SDK wire-shape, error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/secretsmanager/PARITY.md.","notes":"Audit complete (agent pass, uncommitted in working tree as of f093a929). Found + fixed: (1) RLock+lazy-map-write data race across ListSecrets/ListSecretVersionIds/DescribeSecret/GetResourcePolicy/ValidateResourcePolicy (concurrent map write under shared RLock); (2) ListSecretsInput.IncludeDeleted wire field name wrong (real key is IncludePlannedDeletion) — real clients' filter silently ignored; (3) ListSecrets missing NextRotationDate + SortBy (both present in real SDK SecretListEntry/ListSecretsInput); (4) 'owned-by-me' fabricated filter key renamed to real 'owning-service'; (5) time.Now() vs backend's injectable clock inconsistency across CreateSecret/PutSecretValue/UpdateSecret/GetSecretValue/BatchGetSecretValue; (6) CreateSecret missing ClientRequestToken idempotency contract; (7) UpdateSecretVersionStage silently moved labels without enforcing the real RemoveFromVersionId requirement when a label is attached elsewhere. Wrote services/secretsmanager/PARITY.md. Deferred gaps filed as gopherstack-qqq, gopherstack-avt, gopherstack-gvw, gopherstack-pct. Gate green (build/vet/go fix/test/test -race/golangci-lint all 0 issues). NOT committed per task constraints — left in working tree for review/commit by orchestrator.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T16:07:03Z","created_by":"Witness Patrol","updated_at":"2026-07-05T16:34:36Z","closed_at":"2026-07-05T16:34:36Z","close_reason":"case A ~760 LOC: data race on lazy store (concurrent map write), IncludeDeleted/owned-by-me wrong wire fields, version-stage move semantics, ClientRequestToken idempotency, NextRotationDate/SortBy, clock injection; PARITY.md; gated green -race","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-bgl","title":"Parity: rds deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of rds: db instances/clusters/snapshots/parameter+subnet groups, engine versions, lifecycle state machine, SDK wire-shape, error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/rds/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T16:03:07Z","created_by":"Witness Patrol","updated_at":"2026-07-05T16:17:58Z","closed_at":"2026-07-05T16:17:58Z","close_reason":"case B + 2 fixes ~660 LOC: delete final-snapshot contract disguised stub, DescribeDBInstances Filters; additive WithOptions methods (no caller break); removed tracked .rej/.patch junk; PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-x6i","title":"Parity: ecr deep audit","description":"Deep AWS-parity audit of services/ecr - verify wire shapes, error codes, state persistence against aws-sdk-go-v2 ecr types. Branch parity-sweep-3.","notes":"Audit complete. Found 4 genuine gaps in CompleteLayerUpload (missing repo-FK check + LayerAlreadyExistsException), UploadLayerPart (missing part-sequencing InvalidLayerPartException), and PutImage (missing ImageDigestDoesNotMatchException at wire boundary). Fixed all 4 + 2 pre-existing tests that had accidentally encoded the gaps as correct behavior. Deferred EmptyUploadException, ImageAlreadyExistsException (idempotent repush), and LayerPartTooSmallException with documented rationale in services/ecr/PARITY.md. Gate green: build/vet/fix/test/lint all pass. ~190 LOC prod changes + ~260 LOC new tests.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T15:51:18Z","created_by":"Witness Patrol","updated_at":"2026-07-05T16:05:38Z","started_at":"2026-07-05T15:51:23Z","closed_at":"2026-07-05T16:05:38Z","close_reason":"ecr deep parity audit complete; PARITY.md written; gate green; see notes for findings","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-7wu","title":"Parity: ecs deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of ecs: clusters/services/tasks/task-defs, capacity providers, ELB target registration, task lifecycle state machine, SDK wire-shape, error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/ecs/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T15:34:27Z","created_by":"Witness Patrol","updated_at":"2026-07-05T16:01:16Z","closed_at":"2026-07-05T16:01:16Z","close_reason":"case A ~810 LOC: serviceIndex restore (reconciler-blind post-restart), resourceTags persistence data-loss, 3 disguised-stub deployment ops + map-key leak, CreateCluster fields, ContinueServiceDeployment; PARITY.md; gated green ecs-scoped","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-18d","title":"Parity: cloudformation deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of cloudformation: stack lifecycle, change sets, resource provisioning across services (ResourceCreator), template parsing/intrinsics, drift, exports/imports, SDK wire-shape, error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/cloudformation/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T15:24:11Z","created_by":"Witness Patrol","updated_at":"2026-07-05T15:50:19Z","closed_at":"2026-07-05T15:50:19Z","close_reason":"case A hybrid 543 LOC: idempotent DeleteStack, export-in-use enforcement, ExecuteChangeSet status gate+delete-all, ChangeSetNotFound wire code, AUTO_EXPAND capability, DescribeStackEvents pagination; PARITY.md; agent-gated green (final full build pending sweep end)","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-42s","title":"Parity: kms deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of kms: keys/aliases/grants, encrypt/decrypt/data-keys, key rotation, policies, SDK wire-shape, error codes, real crypto state, persistence, leaks. No stubs. Gated green. Table tests. Write services/kms/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T15:17:22Z","created_by":"Witness Patrol","updated_at":"2026-07-05T15:34:27Z","closed_at":"2026-07-05T15:34:27Z","close_reason":"case B + 4 fixes 191 prod LOC: GrantTokens missing on 8 ops (disguised stub), KeySpec 500-\u003e400, grant-index leak, tag-before-create orphan; negative-control tested; PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-b14","title":"Parity: cloudwatchlogs deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of cloudwatchlogs: log groups/streams, PutLogEvents sequencing, subscription/metric filters, retention, SDK wire-shape, error codes, real state, persistence, leaks. No stubs. Gated green. Table tests. Write services/cloudwatchlogs/PARITY.md.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T14:57:28Z","created_by":"Witness Patrol","updated_at":"2026-07-05T15:17:22Z","closed_at":"2026-07-05T15:17:22Z","close_reason":"case A ~600 LOC: deprecated seq-token validation removed, metric-filter value-extraction disguised stub, TestMetricFilter stub, RejectedLogEventsInfo field/off-by-one; PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-ton","title":"Parity: cloudwatch deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of cloudwatch: metrics/alarms/dashboards, SDK wire-shape, error codes, real state, persistence, leak/opt, alarm-action delivery. No stubs. Gated green. Table tests. Write services/cloudwatch/PARITY.md.","notes":"Audit complete (session 2026-07-05). Fixed: (1) PutMetricData fabricated UnprocessedMetricData wire field removed + all-or-nothing semantics (PutMetricDataOutput has zero members per SDK); (2) Values/Counts array input added (was silently dropped); (3) NaN/Inf/2^360-range validation added; (4) breachesThreshold missing LessThanLowerThreshold operator (alarms using it never fired); (5) ListMetrics RecentlyActive=PT3H filter added (was parsed nowhere); (6) composite-alarm Action-history entries mistagged AlarmType=MetricAlarm. Gate green (build/vet/fix/test/lint). Follow-ups filed: gopherstack-3ro (PutDashboard body validation), gopherstack-pyv (PutMetricData timestamp window). PARITY.md written to services/cloudwatch/. Left uncommitted per task instructions.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T14:51:28Z","created_by":"Witness Patrol","updated_at":"2026-07-05T15:24:10Z","closed_at":"2026-07-05T15:24:10Z","close_reason":"case A 1122 LOC: PutMetricData fabricated UnprocessedMetricData/partial-success, Values-Counts arrays dropped, LessThanLowerThreshold missing (alarms never fired), NaN/Inf validation, RecentlyActive filter; PARITY.md; gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-bz6","title":"Parity: sns deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of sns: SDK wire-shape, error codes, real state, persistence, leak/opt, subscription delivery paths. No stubs. Gated green. Table tests.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T14:28:19Z","created_by":"Witness Patrol","updated_at":"2026-07-05T14:51:27Z","closed_at":"2026-07-05T14:51:27Z","close_reason":"case B + 8 fixes ~494 LOC: PublishBatch attribute wire-shape, Lambda fake-signature disguised stub, replay fan-out, archive persistence+leak, buffer caps, signer lock, error-msg fix; PARITY.md written; gated green sns-scoped","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-uaf","title":"Parity: sqs deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of sqs: SDK wire-shape (query/json), error codes, real state, persistence, leak/opt. No stubs. Gated green. Table tests.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T14:22:24Z","created_by":"Witness Patrol","updated_at":"2026-07-05T14:57:28Z","closed_at":"2026-07-05T14:57:28Z","close_reason":"case A 677 LOC: duplicate XML declaration, FIFO requeue ordering, RedriveAllowPolicy disguised-stub, fifoSeq/hasActivity/lastPurged persistence, exported NoVisibilityTimeout; PARITY.md; gated green sqs-scoped","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-coi","title":"Parity: lambda deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of lambda: SDK wire-shape, error codes, real state, persistence, leak/opt, Docker runtime paths. No stubs. Gated green. Table tests.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T13:56:01Z","created_by":"Witness Patrol","updated_at":"2026-07-05T14:22:23Z","closed_at":"2026-07-05T14:22:23Z","close_reason":"case A 641 LOC: RemovePermission wire-shape disguised-stub, ESM qualifier ARN parsing data-loss, permissions/index persistence gaps, Qualifier scoping; gated green (lambda-scoped)","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-ap7","title":"Parity: iam deep audit (AWS accuracy + leaks)","description":"Top-30 sweep. Deep parity audit of iam: SDK wire-shape vs aws-sdk-go-v2, error codes, real backend state, persistence, leak/opt. No stubs. Gated green. Table tests.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T13:56:00Z","created_by":"Witness Patrol","updated_at":"2026-07-05T14:28:18Z","closed_at":"2026-07-05T14:28:18Z","close_reason":"case A ~1111 LOC: 2 disguised stubs (ListInstanceProfilesForRole, GetAccountAuthorizationDetails versions), HTTP status codes, policy-doc percent-encoding, handler+comprehensive persistence leaks, tags-at-creation; gated green iam-scoped","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-b5m","title":"ec2: DeleteVpc/DeleteSubnet should return DependencyViolation, not force-cascade-delete","description":"From ec2 probe (gopherstack-r0h): backend.go DeleteVpc (~1397) and DeleteSubnet (~1551) force-cascade-delete all dependents (instances, IGWs, NAT GWs, route tables, SGs, ENIs, subnets). Real AWS returns DependencyViolation and requires dependents removed first (no cascade). High blast radius — existing cascade-delete tests will need updates. Dedicated pass. Also queued: full VPC/TGW/NAT/VPC-endpoint (batch-4) family op-by-op sweep, EBS snapshot lineage, ENI attach/detach edge cases — unaudited this pass.","status":"open","priority":2,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T04:07:31Z","created_by":"Witness Patrol","updated_at":"2026-07-05T04:07:31Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-wdw","title":"Phase 4: match LocalStack Pro cross-service interconnectivity (web-verified)","description":"AFTER registry refactor (gopherstack-drp). Research current LocalStack Pro integration matrix from docs.localstack.cloud (IAM enforcement, real ECS/EKS/Fargate, advanced EventBridge Pipes, full Step Functions service integrations, Cognito triggers, API Gateway advanced, App services, X-Ray, etc.), diff vs gopherstack's cli.go composition-root wiring, implement missing links with real in-process delivery (no stubs, reuse pkgs/, coarse lock, gated green, per-link commits). Known Community-level gaps already filed: atk, xoe, 8sk, 18k. Deliverable: web-verified Pro-vs-gopherstack comparison rendered as artifact. Sequenced after Phase 3 because registry refactor + cli.go interconnect changes shouldn't overlap.","status":"open","priority":2,"issue_type":"feature","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T04:06:36Z","created_by":"Witness Patrol","updated_at":"2026-07-05T04:06:36Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-18k","title":"ASG/ECS → ELBv2 real target-group registration","description":"autoscaling AttachLoadBalancerTargetGroups/Detach just append/remove ARN strings; no call into elbv2 backend to register/deregister targets, so elbv2 DescribeTargetHealth won't reflect ASG membership. Wire real registration.","status":"open","priority":2,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T04:04:04Z","created_by":"Witness Patrol","updated_at":"2026-07-05T04:04:04Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-8sk","title":"Auto Scaling → EC2: launch/terminate real instances instead of synthetic IDs","description":"autoscaling/backend.go has NO ec2 import; scaling generates fake i-xxx via instanceIndex, never calls EC2 RunInstances/TerminateInstances. Scaling an ASG doesn't create instances visible in DescribeInstances. LocalStack wires this. Add EC2 actioner adapter + wire in cli.go. Larger change (state-machine + launch template resolution).","status":"open","priority":2,"issue_type":"feature","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T04:04:04Z","created_by":"Witness Patrol","updated_at":"2026-07-05T04:04:04Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-xoe","title":"Wire EventBridge non-core rule targets (SFN/ECS/Kinesis/Logs/API-destinations)","description":"eventbridge/delivery.go DeliveryTargets supports KinesisFirehose/KinesisStream/ECS/StepFunctions/CloudWatchLogs/APIDestinations, but wireEventBridgeDelivery (cli.go:3186) only populates Lambda/SQS/SNS. Rules with those targets match but never fire. Populate remaining targets in composition root.","status":"open","priority":2,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T04:04:03Z","created_by":"Witness Patrol","updated_at":"2026-07-05T04:04:03Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-ej5","title":"Parity probe: dynamodb deep audit (AWS/LocalStack accuracy + leaks)","description":"Phase 2 probe. Deep parity audit of dynamodb: SDK wire-shape, error codes, real state, persistence, leak/opt pass. No stubs. Gated green.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T03:15:40Z","created_by":"Witness Patrol","updated_at":"2026-07-05T04:23:41Z","started_at":"2026-07-05T04:07:31Z","closed_at":"2026-07-05T04:23:41Z","close_reason":"case A modest 419 LOC: transact-update key-mutation index corruption (state bug), batch-write duplicate-key validation, Select/COUNT constraints; commit f459c9fa, gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-r0h","title":"Parity probe: ec2 deep audit (AWS/LocalStack accuracy + leaks)","description":"Phase 2 probe. Deep parity audit of ec2: SDK wire-shape, error codes, real state, persistence, leak/opt pass. No stubs. Gated green.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T03:15:39Z","created_by":"Witness Patrol","updated_at":"2026-07-05T04:07:30Z","started_at":"2026-07-05T03:41:18Z","closed_at":"2026-07-05T04:07:30Z","close_reason":"case A, 672 LOC: tag-all-resource-types (9→~100), real instance attributes (disguised stub fixed), lifecycle protection, StateReason wire shape; commit c18fa9b1, gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-37c","title":"Parity probe: s3 deep audit (AWS/LocalStack accuracy + leaks)","description":"Phase 2 probe. Deep parity audit of s3: SDK wire-shape vs aws-sdk-go-v2, error codes, real backend state, persistence wiring; goroutine/map leak + optimization pass. No stubs. Gated green (build+vet+package tests). Proves audit depth before scaling to top-30.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T03:15:33Z","created_by":"Witness Patrol","updated_at":"2026-07-05T03:41:18Z","started_at":"2026-07-05T03:16:02Z","closed_at":"2026-07-05T03:41:18Z","close_reason":"11 real parity fixes (2 serious SSE persistence data-loss bugs) + op-by-op completeness proof; commit 708d1961, gated green","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-6no","title":"Phase 1.5: modernize -fix sweep + add modernize CI gate","description":"Run golang.org/x/tools modernize analyzer -fix tree-wide (min/max builtins, slices/maps pkg, range-over-int, any, etc); add a modernize CI job so it stays clean. Gate build+vet+lint.","status":"in_progress","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T01:06:35Z","created_by":"Witness Patrol","updated_at":"2026-07-05T01:06:42Z","started_at":"2026-07-05T01:06:42Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-nab","title":"eks: acknowledge/implement CancelUpdate op (new in aws-sdk-go-v2 eks v1.88.x)","description":"Dep upgrade to eks v1.88.1 added SDK op CancelUpdate. TestSDKCompleteness (pkgs/sdkcheck) flags it as neither in GetSupportedOperations() nor notImplemented. Phase 2: add 'CancelUpdate' to notImplemented slice in services/eks/sdk_completeness_test.go, or implement the op. Trivial one-line fix; deferred from Phase 1 deps commit per gate change (build/vet/lint only).","status":"open","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T01:05:14Z","created_by":"Witness Patrol","updated_at":"2026-07-05T01:05:14Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-vu7","title":"Phase 1: UI/frontend dependency upgrade","description":"Upgrade JS/TS UI deps to latest, gate lint/test/build, isolated commit. Keep playwright-go migration intact (Go side handled separately).","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-04T23:40:43Z","created_by":"Witness Patrol","updated_at":"2026-07-05T00:43:13Z","started_at":"2026-07-04T23:40:56Z","closed_at":"2026-07-05T00:43:13Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-wbs","title":"Phase 1: full dependency upgrade (go get -u ./...)","description":"Upgrade all Go module deps to latest incl aws-sdk-go-v2 family; gate build/vet/test-race/lint; isolated checkpoint commit for easy bisect.","status":"closed","priority":2,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-07-04T23:24:57Z","created_by":"Witness Patrol","updated_at":"2026-07-05T01:05:18Z","started_at":"2026-07-04T23:24:59Z","closed_at":"2026-07-05T01:05:18Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-5o9.2","title":"EC2 final de-stub round: 22 remaining reachable stub ops","description":"22 ops still return stubResponse in services/ec2/handler_stubs.go after parity-sweep-2 (gopherstack-5o9.1). Suggested grouping:\n1. Verified Access modify trio: ModifyVerifiedAccessGroup, ModifyVerifiedAccessInstance, ModifyVerifiedAccessTrustProvider (mirror handler_batch4.go VerifiedAccess Create family).\n2. Transit Gateway peripherals: DescribeTransitGatewayAttachments, DisableTransitGatewayRouteTablePropagation, EnableTransitGatewayRouteTablePropagation (mirror handler_tgw_peripherals.go).\n3. Capacity Reservation extras: CreateInterruptibleCapacityReservationAllocation, UpdateInterruptibleCapacityReservationAllocation, GetCapacityReservationUsage, DescribeCapacityReservationTopology (mirror handler_capacity_family.go).\n4. Address/VPC misc: MoveAddressToVpc, RejectVpcEndpointConnections, UnassignPrivateNatGatewayAddress, DescribeMovingAddresses (mirror handler_vpc_config.go / address family).\n5. Image extras: CancelImageLaunchPermission, DescribeImageReferences, GetImageAncestry, DescribeElasticGpus (mostly legacy/deprecated AWS features — verify against SDK before implementing; some may be skip-with-evidence).\n6. Misc singletons: GetFlowLogsIntegrationTemplate, GetSpotPlacementScores, EnableReachabilityAnalyzerOrganizationSharing, SendDiagnosticInterrupt — each is a standalone op, low reuse; group as a single small round.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-04T16:20:29Z","created_by":"Witness Patrol","updated_at":"2026-07-04T17:21:33Z","closed_at":"2026-07-04T17:21:33Z","close_reason":"Done in R34 commit 39f0e840: all 22 (+DescribeElasticGpus) real, registerStubOps deleted, EC2 zero reachable stubs.","dependencies":[{"issue_id":"gopherstack-5o9.2","depends_on_id":"gopherstack-5o9","type":"parent-child","created_at":"2026-07-04T11:20:28Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-5o9.1","title":"EC2 parity-sweep-2: VPC Encryption Control, VPN Concentrator, Host Reservations, Declarative Policies, Network Performance, Local Gateway VIFs (PR #2381)","description":"De-stubbed 27 EC2 ops into real backend state, mirroring existing families (VpcBlockPublicAccessExclusion, VpnConnection tunnels, dedicated hosts, ConversionTask settle-on-describe, local gateway VIF/group). All wire shapes verified against vendored aws-sdk-go-v2 v1.294.0 deserializers. Table-driven backend+HTTP tests added for every op. go build/vet/test -race/golangci-lint all green on services/ec2.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-04T16:20:13Z","created_by":"Witness Patrol","updated_at":"2026-07-04T16:20:18Z","closed_at":"2026-07-04T16:20:18Z","close_reason":"27 ops de-stubbed, tested, lint-clean; see commit for details","dependencies":[{"issue_id":"gopherstack-5o9.1","depends_on_id":"gopherstack-5o9","type":"parent-child","created_at":"2026-07-04T11:20:13Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-1vb","title":"EPIC: de-stub SageMaker (~335 stub ops) to real emulation","description":"services/sagemaker/handler_stubs.go: doc says 335 SageMaker SDK ops not implemented; return minimal JSON stubs. Grind to real state by resource family. Part of PR #2381 no-stub sweep.","status":"closed","priority":2,"issue_type":"epic","owner":"blackbird7181@gmail.com","created_at":"2026-07-04T05:36:33Z","created_by":"Witness Patrol","updated_at":"2026-07-04T10:43:23Z","closed_at":"2026-07-04T10:43:23Z","close_reason":"SageMaker fully de-stubbed to real emulation across R17-R25; remainingStubCount=0. Final commit in this batch.","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -144,4 +231,45 @@ {"_type":"issue","id":"go-hwb.106","title":"S3 Control: ~48 missing ops (access points/grants/batch jobs/MRAP)","description":"## S3 Control — Service Deep Dive\n\nAudit of [services/s3control/](services/s3control/) and UI in [ui/src/routes/s3control/](ui/src/routes/s3control/).\n\n### 1. Missing SDK Operations\n~48 missing ([sdk_completeness_test.go#L21](services/s3control/sdk_completeness_test.go#L21)): `DeleteAccessGrant`, `DeleteBucket`, `GetAccessPoint`, `ListAccessPoints`, `PutAccessPointPolicy`, Access Grants, Access Points, Batch Jobs, MRAP, Storage Lens Group. Only 13 supported (public access block + partial).\n\n### 2. Missing UI / Dashboard Features\nPublic access block display only. Missing: access points mgmt, access grants, batch job UI, MRAP, storage lens groups, Object Lambda.\n\n### 3. Goroutine / Resource / Lock Leaks\nClean. Map cloning on snapshot ([persistence.go#L44](services/s3control/persistence.go#L44)).\n\n### 4. Performance Optimizations\n1. 10+ separate maps — consolidate with typed keys to reduce Reset cost.\n2. Atomic counter for IDs ([backend.go#L178](services/s3control/backend.go#L178)) good.\n\n### Suggested Order\n1. Access Points (Create/Get/List/Put policy)\n2. Access Grants (Create/Delete/List)\n3. Batch Jobs + MRAP + Storage Lens Group\n4. Consolidate map structure\n\n\n---\n**Source:** https://github.com/BlackbirdWorks/gopherstack/issues/1223\n","status":"open","priority":2,"issue_type":"task","owner":"andrew.bishop9625@gmail.com","created_at":"2026-05-02T18:28:35Z","created_by":"mayor","updated_at":"2026-05-02T18:28:35Z","external_ref":"gh-1223","labels":["ai-queue"],"dependencies":[{"issue_id":"go-hwb.106","depends_on_id":"go-hwb","type":"parent-child","created_at":"2026-05-02T13:28:35Z","created_by":"mayor","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"go-hwb.102","title":"Timestream Write: SDK complete; per-table WriteRecords locks","description":"## Timestream Write — Service Deep Dive\n\nAudit of [services/timestreamwrite/](services/timestreamwrite/) and shared UI in [ui/src/routes/timestream/](ui/src/routes/timestream/).\n\n### 1. Missing SDK Operations\n**0 missing.** 20 ops implemented including `CreateDatabase`, `CreateTable`, `WriteRecords`, `CreateBatchLoadTask`, `ResumeBatchLoadTask`, tags.\n\n### 2. Missing UI / Dashboard Features\nShared UI covers DBs + tables + scheduled queries. Full CRUD. Batch load UI could be enhanced.\n\n### 3. Goroutine / Resource / Lock Leaks\nClean. 4 nested maps under single `lockmetrics.RWMutex` ([backend.go#L159](services/timestreamwrite/backend.go#L159)).\n\n### 4. Performance Optimizations\n1. **Single mutex serializes WriteRecords across tables** — partition by table-ARN for ~10x throughput.\n2. Dispatch pre-built ([handler.go#L62](services/timestreamwrite/handler.go#L62)).\n\n### Suggested Order\n1. Per-table-ARN partition locks for `WriteRecords`\n2. Batch load UI polish\n\n\n---\n**Source:** https://github.com/BlackbirdWorks/gopherstack/issues/1227\n","status":"open","priority":2,"issue_type":"task","owner":"andrew.bishop9625@gmail.com","created_at":"2026-05-02T18:28:34Z","created_by":"mayor","updated_at":"2026-05-02T18:28:34Z","external_ref":"gh-1227","labels":["ai-queue"],"dependencies":[{"issue_id":"go-hwb.102","depends_on_id":"go-hwb","type":"parent-child","created_at":"2026-05-02T13:28:34Z","created_by":"mayor","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"go-hwb","title":"Epic: ai-queue from BlackbirdWorks/gopherstack","description":"Autonomous grinding of GitHub issues labeled 'ai-queue' from BlackbirdWorks/gopherstack. Each child bead corresponds to one GitHub issue (external-ref gh-N). Launched via gt mountain for wave-based dispatch with Witness failure tracking and merge-on-CI-pass via Refinery.","status":"open","priority":2,"issue_type":"epic","owner":"andrew.bishop9625@gmail.com","created_at":"2026-05-02T18:27:46Z","created_by":"mayor","updated_at":"2026-05-02T18:27:46Z","labels":["ai-queue"],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"gopherstack-drp","title":"Collection-registry helper to kill backend map boilerplate (init/Reset/Snapshot/Restore)","description":"Backends hold many plain maps under one coarse lockmetrics.RWMutex (EC2: ~180). Locking is correct (cross-map ops need coarse atomicity; per-map safemap would break invariants — pkgs/safemap has 0 users for this reason). Pain is boilerplate: every map needs init + Reset + Snapshot + Restore + nil-safety wiring. Proposal: pkgs/collections registry — declare each map once, get InitAll/ResetAll/SnapshotAll/RestoreAll; lock stays at backend level. MUST preserve existing persistence JSON field names exactly (stable contract). Do after parity sweep completes, not mid-sweep. Also: adopt pkgs/safemap opportunistically for genuinely isolated single maps (token stores/caches).","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-04T15:04:15Z","created_by":"Witness Patrol","updated_at":"2026-07-04T15:04:15Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-1zl","title":"Rename meaningless service files (backend_accuracyN/batchN/parity_N/refinementN) to content-descriptive names","description":"AFTER the pkgs/store datalayer refactor (Phase 3.3, epic gopherstack-5js) completes — doing it mid-rollout would collide with in-flight conversions. Sweep all services/* for meaningless sequence-tagged filenames (backend_accuracy4.go, handler_batch3.go, parity_b.go, refinement2.go, *_ops2.go, sweep3, etc.) and rename each to describe its CONTENTS (the op family it implements: backend_batch_ops.go, backend_lifecycle.go, handler_tags.go, ...). Pure git mv + no code change; gate: whole-repo build + go test per touched service. Convention saved: bd memory file-naming-descriptive.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-06T01:10:59Z","created_by":"Witness Patrol","updated_at":"2026-07-06T01:10:59Z","dependencies":[{"issue_id":"gopherstack-1zl","depends_on_id":"gopherstack-5js","type":"blocks","created_at":"2026-07-05T20:11:10Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-qd3.6","title":"glue: audit connections/triggers/workflows/schema-registry/data-quality/ML-transforms/blueprints/UDFs/resource-policy families","description":"parity-sweep-3 (gopherstack-qd3) focused on databases/tables/partitions/crawlers/jobs/job-runs and the global error-code fix. These families were deferred entirely — not audited op-by-op against aws-sdk-go-v2/service/glue this pass. See services/glue/PARITY.md 'deferred' list.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:59:53Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:59:53Z","dependencies":[{"issue_id":"gopherstack-qd3.6","depends_on_id":"gopherstack-qd3","type":"parent-child","created_at":"2026-07-05T14:59:52Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-qd3.4","title":"glue: StartJobRun has no per-run capacity/argument overrides","description":"JobRun now inherits WorkerType/NumberOfWorkers/MaxCapacity/GlueVersion/Timeout from the Job at start time (parity-sweep-3), but AWS's StartJobRunRequest allows overriding these per-run. Not modeled — backend StartJobRun signature only takes (jobName, arguments).","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:59:51Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:59:51Z","dependencies":[{"issue_id":"gopherstack-qd3.4","depends_on_id":"gopherstack-qd3","type":"parent-child","created_at":"2026-07-05T14:59:51Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-qd3.2","title":"glue: CreateCrawler/UpdateCrawler missing SchemaChangePolicy/RecrawlPolicy/LineageConfiguration/CrawlerSecurityConfiguration/LakeFormationConfiguration","description":"CrawlerOptions (added in parity-sweep-3) covers Schedule/Classifiers/Configuration/TablePrefix/Description. Still missing several AWS CreateCrawlerRequest/UpdateCrawlerRequest fields. Deferred during parity-sweep-3 (gopherstack-qd3) for scope.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:59:50Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:59:50Z","dependencies":[{"issue_id":"gopherstack-qd3.2","depends_on_id":"gopherstack-qd3","type":"parent-child","created_at":"2026-07-05T14:59:49Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-qd3.3","title":"glue: DatabaseInput/Database missing Parameters/LocationUri/CreateTableDefaultPermissions/TargetDatabase","description":"Real AWS DatabaseInput has Parameters, LocationUri, CreateTableDefaultPermissions, TargetDatabase (resource-link databases) beyond Name/Description. Not fixed during parity-sweep-3 (gopherstack-qd3); flagged as a gap.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:59:50Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:59:50Z","dependencies":[{"issue_id":"gopherstack-qd3.3","depends_on_id":"gopherstack-qd3","type":"parent-child","created_at":"2026-07-05T14:59:50Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-qd3.1","title":"glue: model DynamoDB/Delta/Hudi/Iceberg/MongoDB crawler targets","description":"CrawlerTarget currently models S3/JDBC/Catalog targets only (added in parity-sweep-3). Real AWS CrawlerTargets also supports DynamoDBTargets, DeltaTargets, HudiTargets, IcebergTargets, MongoDBTargets. Deferred during parity-sweep-3 (gopherstack-qd3) for scope.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:59:48Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:59:48Z","dependencies":[{"issue_id":"gopherstack-qd3.1","depends_on_id":"gopherstack-qd3","type":"parent-child","created_at":"2026-07-05T14:59:48Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-029","title":"sesv2 deep parity audit (separate from v1 ses)","description":"Follow-up from gopherstack-ls1 (SES v1 parity sweep). services/sesv2/ exists as a separate REST-JSON service and was NOT touched this pass per scope constraints (only services/ses/ was in scope). Needs its own dedicated audit pass: identity/email-identity CRUD, SendEmail v2 shapes, configuration sets v2, suppression list, dedicated DKIM signing config, contact lists — verify against aws-sdk-go-v2/service/sesv2.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:40:02Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:40:02Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-aib","title":"cognitoidp: PreventUserExistenceErrors not applied to ForgotPassword/ResendConfirmationCode","description":"PreventUserExistenceErrors=ENABLED masking was added to InitiateAuth in gopherstack-2sp (username enumeration via NotAuthorizedException vs UserNotFoundException), matching AWS's documented behavior for 'authentication'. AWS docs also cover 'account confirmation' and 'password recovery': ForgotPassword and ResendConfirmationCode should return a fake-success CodeDeliveryDetails response for a non-existent user instead of UserNotFoundException when ENABLED. This needs destination fabrication (masked email/phone) and is more invasive than the InitiateAuth fix, so it was deferred this pass. See services/cognitoidp/PARITY.md.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:32:27Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:32:27Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-8fw","title":"cognitoidp: LambdaConfig triggers stored but never invoked","description":"UserPool.LambdaConfig (PreSignUp, PostConfirmation, PreTokenGeneration, CustomMessage, etc.) is accepted and persisted on CreateUserPoolWithOpts/UpdateUserPoolWithOpts but no trigger is ever invoked during SignUp/ConfirmSignUp/auth/token issuance. Real Cognito calls out to Lambda synchronously and can reject/modify the operation based on the trigger's response. Implementing this requires cross-service invocation into the lambda service, which is out of scope for an in-package cognitoidp fix (shared-file/cross-service follow-up). Found during gopherstack-2sp audit.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:32:20Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:32:20Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-p8i","title":"cognitoidp: implement real SRP-6a for USER_SRP_AUTH (currently accepts PASSWORD directly)","description":"InitiateAuth/RespondToAuthChallenge for USER_SRP_AUTH currently requires AuthParameters[\"PASSWORD\"] directly and skips the real SRP-6a handshake (no SRP_A/SRP_B/SALT/SECRET_BLOCK exchange, no zero-knowledge proof verification). A real SRP client (per Cognito's SRP variant: 3072-bit N, g=2, HKDF-SHA256 session key derivation with the 'Caldera Derived Key' info string, HMAC-SHA256 M1 proof) never sends PASSWORD in AuthParameters, so it cannot authenticate against this backend at all today. Implementing this precisely enough to interoperate with real Cognito SDK/JS clients requires byte-perfect padding/HKDF/HMAC details that could not be safely verified without reference test vectors or a real client in this pass (deferred rather than risk a subtly-wrong 'looks like SRP' implementation). See services/cognitoidp/PARITY.md Notes for detail. Investigated during gopherstack-2sp.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:32:14Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:32:14Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-mzx","title":"CloudFront: CreateDistribution CallerReference-reuse-with-different-content should error","description":"Real AWS CreateDistribution/CreateCloudFrontOriginAccessIdentity docs: reusing a CallerReference with an IDENTICAL DistributionConfig is idempotent (returns the existing distribution), but reusing it with a DIFFERENT config returns DistributionAlreadyExists. Current InMemoryBackend.CreateDistribution (services/cloudfront/backend.go) only keys off CallerReference and always returns the existing distribution unconditionally, never comparing config content, so DistributionAlreadyExists (the sentinel exists, ErrAlreadyExists's code was 'DistributionAlreadyExists' before parity-sweep-3 repurposed it as the generic EntityAlreadyExists fallback) is never actually triggered by its originally-intended resource type. Needs: compare canonicalized RawConfig (or the parsed fields) against the stored one on CallerReference match; if different, return a dedicated ErrDistributionAlreadyExists (code DistributionAlreadyExists). Same pattern likely applies to CreateOAI (CloudFrontOriginAccessIdentityAlreadyExists) and CreateStreamingDistribution -- check each.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:58:51Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:58:51Z","labels":["cloudfront","parity"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-miw","title":"elb (classic): NotFound/AlreadyExists errors should be HTTP 400 not 404/409","description":"From elbv2 sweep (1xp): services/elb (classic ELB) has the identical error-status bug elbv2 just fixed — query-protocol services return 400 for all client errors, but classic elb returns 404/409. Not in top-30 so deferred. Apply the same remap.","status":"open","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:30:20Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:30:20Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-9wo","title":"Autoscaling: terminate-lifecycle-hook gating missing from scale-in (SetDesiredCapacity/ExecutePolicy) path","description":"gopherstack-am1 added real EC2_INSTANCE_TERMINATING lifecycle-hook gating (Terminating:Wait + CompleteLifecycleAction/timeout) to TerminateInstanceInAutoScalingGroup only. The desired-capacity-driven scale-in path (applyDesiredCapacityChange, shared by SetDesiredCapacity decreasing, UpdateAutoScalingGroup, and ExecutePolicy scale-in) still removes instances immediately regardless of a registered terminating hook. Extending gating there requires deferring N concurrent per-instance waits while keeping DesiredCapacity/instance-count bookkeeping consistent for concurrent DescribeAutoScalingGroups callers - a bigger state machine than the single-instance TerminateInstanceInAutoScalingGroup case, deliberately deferred rather than rushed. See services/autoscaling/PARITY.md Notes for full context.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:15:08Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:15:08Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-6ys","title":"Autoscaling: scheduled actions never actually execute (no cron/scheduler engine)","description":"PutScheduledUpdateGroupAction/BatchPutScheduledUpdateGroupAction now correctly parse and persist StartTime/EndTime/Recurrence (fixed in gopherstack-am1), but there is no background scheduler goroutine that evaluates the recurrence cron expression and actually applies the min/max/desired capacity change at the scheduled time. DescribeScheduledActions reflects exactly what was requested, but nothing ever fires it. A correct fix needs a cron-parsing ticker (reuse an existing cron lib if one is already vendored) plus careful goroutine lifecycle management (start/stop with the backend, covered by leak tests) - deliberately out of scope for the gopherstack-am1 sweep to avoid rushing a new leak-prone subsystem.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T18:14:38Z","created_by":"Witness Patrol","updated_at":"2026-07-05T18:14:38Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-996","title":"SFN: TaskScheduled/TaskSucceeded history events missing resourceType/timeout/outputDetails fields","description":"TaskScheduledEventDetails/TaskSucceededEventDetails now populate resource/output (fixed this pass) but still omit resourceType, region, parameters, timeoutInSeconds, heartbeatInSeconds (scheduled) and outputDetails.truncated (succeeded). No TaskSubmitted/TaskStarted events are emitted for .sync/.waitForTaskToken patterns either.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T17:37:13Z","created_by":"Witness Patrol","updated_at":"2026-07-05T17:37:13Z","dependencies":[{"issue_id":"gopherstack-996","depends_on_id":"gopherstack-a75","type":"discovered-from","created_at":"2026-07-05T12:37:12Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-1sf","title":"SFN: StartExecution ClientRequestToken / EXPRESS name-reuse semantics not modeled","description":"StartExecution execution-name uniqueness is enforced identically for STANDARD and EXPRESS; AWS allows immediate EXPRESS name reuse and StartExecution is not idempotent for EXPRESS (no ClientRequestToken-based dedup semantics modeled either way). Found while fixing the incorrect EXPRESS StartExecution rejection in this pass.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T17:37:12Z","created_by":"Witness Patrol","updated_at":"2026-07-05T17:37:12Z","dependencies":[{"issue_id":"gopherstack-1sf","depends_on_id":"gopherstack-a75","type":"discovered-from","created_at":"2026-07-05T12:37:12Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-xtl","title":"SFN: Retry JitterStrategy enum not validated","description":"Retrier.JitterStrategy accepts any string; only literal \"FULL\" enables jitter, anything else (including invalid values) silently behaves as NONE. AWS rejects invalid JitterStrategy values at CreateStateMachine/UpdateStateMachine with a ValidationException. Definition-time validation is out of scope for this pass (existing ASL validation is JSON-parse-only).","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T17:37:12Z","created_by":"Witness Patrol","updated_at":"2026-07-05T17:37:12Z","dependencies":[{"issue_id":"gopherstack-xtl","depends_on_id":"gopherstack-a75","type":"discovered-from","created_at":"2026-07-05T12:37:11Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-8im","title":"SFN: Map ItemProcessor.ProcessorConfig.Mode (INLINE/DISTRIBUTED) not parsed","description":"AWS restricts ToleratedFailureCount/Percentage and ResultWriter to Distributed Map (ProcessorConfig.Mode=DISTRIBUTED); ProcessorConfig is not parsed at all so the emulator applies these features permissively regardless of mode. Low risk (permissive superset) but a real definition-validation gap vs AWS's ValidationException for INLINE+ToleratedFailure combos.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T17:37:10Z","created_by":"Witness Patrol","updated_at":"2026-07-05T17:37:10Z","dependencies":[{"issue_id":"gopherstack-8im","depends_on_id":"gopherstack-a75","type":"discovered-from","created_at":"2026-07-05T12:37:10Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-e81","title":"apigatewayv2: RoutingRule Actions/Conditions use untyped map[string]any instead of AWS-modeled union shapes","description":"RoutingRule.Actions/Conditions ([]map[string]any) round-trip arbitrary JSON rather than validating against the AWS-modeled RoutingRuleAction (UpdateHeaderAction/InvokeApiAction) and RoutingRuleCondition (nested Or arrays) union shapes, so malformed actions/conditions are accepted without error. Domain-name routing rules are a newer, lower-traffic APIGWv2 feature; deferred from gopherstack-bec parity sweep 3 (2026-07-05) in favor of higher-value fixes (Integration TlsConfig, protocol-aware timeout defaults/limits, ConnectionType default+validation, Stage ClientCertificateId, DomainName MutualTlsAuthentication+Arn, stage-level tagging).","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T17:01:12Z","created_by":"Witness Patrol","updated_at":"2026-07-05T17:01:12Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-wmh","title":"apigatewayv2: authorizerCache entries not purged on DeleteAPI","description":"authorizerCache (authorizer.go) caches REQUEST-authorizer decisions keyed by authorizerId+identity-source values with a TTL, but DeleteAPI does not purge cache entries for authorizers belonging to the deleted API. Entries self-heal via TTL expiry/lazy eviction on Get, so this is not an unbounded leak, but it is dead weight until TTL elapses and a latent correctness risk if a new API/authorizer is later created with a colliding randomID(). Deferred from gopherstack-bec parity sweep 3 (2026-07-05).","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T17:01:11Z","created_by":"Witness Patrol","updated_at":"2026-07-05T17:01:11Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-2tx","title":"apigatewayv2: track ApiGatewayManaged for quick-create Integration/Stage","description":"Real API Gateway v2 quick-create (CreateApi with routeKey+target, or ImportApi with quick-create) marks the resulting default Integration/Route/Stage as apiGatewayManaged=true, and real AWS then rejects DeleteIntegration/DeleteStage for those managed resources. Our Integration/Stage structs have no ApiGatewayManaged field and there is no quick-create tracking. Deferred from gopherstack-bec parity sweep 3 (2026-07-05) as narrower/lower-traffic than the fixes made this pass (TlsConfig, protocol-aware integration timeout, ConnectionType default, ClientCertificateId, MutualTlsAuthentication, stage tagging).","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T17:01:03Z","created_by":"Witness Patrol","updated_at":"2026-07-05T17:01:03Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-1hg","title":"ssm: document version-cap eviction can orphan DefaultVersion pointer","description":"UpdateDocument caps stored versions at maxDocumentVersionCap (1000); if DefaultVersion was pinned to an old version via UpdateDocumentDefaultVersion and enough UpdateDocument calls happen to evict it from documentVersionsStore, GetDocument/DescribeDocument with an omitted/$DEFAULT selector will return ErrInvalidDocumentVersion instead of falling back or re-pointing DefaultVersion. Rare edge case (needs 1000+ updates after pinning); found during parity-sweep-3 ssm audit, not fixed due to scope/low practical likelihood. See services/ssm/backend.go resolveDocumentVersionSelector / DescribeDocument.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T16:37:37Z","created_by":"Witness Patrol","updated_at":"2026-07-05T16:37:37Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-avt","title":"secretsmanager: RotateSecret RotateImmediately=false does not run the testSecret probe","description":"Real AWS: when RotateImmediately=false, Secrets Manager runs the Lambda testSecret step to validate the rotation configuration, creating and then removing a transient AWSPENDING version, before returning. gopherstack's RotateSecret (backend.go) just records the rotation rules and returns without invoking Lambda or touching AWSPENDING when RotateImmediately=false. Found during gopherstack-78p audit.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T16:31:31Z","created_by":"Witness Patrol","updated_at":"2026-07-05T16:31:31Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-qqq","title":"secretsmanager: RotateSecret allows rotation with no rotation function ever configured","description":"RotateSecret (backend.go) creates/promotes a new version even when neither the request nor the secret has ever had a RotationLambdaARN configured. Real AWS requires a rotation strategy (Lambda ARN or managed rotation) to already exist or be supplied; otherwise it errors. Deferred: changing this would break many existing tests that rely on the current lenient no-Lambda rotation behavior as a test convenience, and gopherstack does not model managed rotation. Found during gopherstack-78p audit.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T16:31:31Z","created_by":"Witness Patrol","updated_at":"2026-07-05T16:31:31Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-e5h","title":"cloudformation: next-pass scope — StackSets/GeneratedTemplates/ResourceScans/TypeRegistry/StackRefactor families unaudited","description":"Parity sweep (gopherstack-18d, commit 6548cf87) scoped to the highest-value families (stack lifecycle, change sets, exports/imports, capabilities, event pagination) given the service's ~42k LOC size. The following families/ops were NOT deeply audited against aws-sdk-go-v2 this pass and should be the target of the next cloudformation parity pass: StackSets (CreateStackSet/UpdateStackSet/DeleteStackSet/instances/operations/drift), Generated Templates, Resource Scans, Type registry/management (RegisterType/ActivateType/PublishType/etc.), Stack Refactor, and deep drift-detection semantics (DetectStackDrift property-level diffing). Also worth revisiting: YAML short-form intrinsics (!Ref/!GetAtt/etc.) wire coverage, and the requiresRecreation table in changeset_diff.go only models a curated subset of AWS resource types' replacement-forcing properties — expand coverage or document as a known limitation.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T15:47:47Z","created_by":"Witness Patrol","updated_at":"2026-07-05T15:47:47Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-urm","title":"cloudformation: top-level Transform / CAPABILITY_AUTO_EXPAND not enforced","description":"Template struct (services/cloudformation/template.go) never parses the top-level Transform field, so CreateStack/UpdateStack never require CAPABILITY_AUTO_EXPAND for templates that use macros (e.g. AWS::Serverless-2016-10-31, custom macros via Fn::Transform at template scope). Fn::Transform intrinsic invocation (invokeMacroTransform) works standalone but isn't gated on the capability. Real AWS rejects such templates with InsufficientCapabilitiesException when CAPABILITY_AUTO_EXPAND is missing. Fix: parse Transform in Template, and require CAPABILITY_AUTO_EXPAND in validateStackOptions/requireIAMCapability (or a sibling check) when present.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T15:47:40Z","created_by":"Witness Patrol","updated_at":"2026-07-05T15:47:40Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-w3k","title":"KMS: GrantConstraints.SourceArn not modeled (needs cross-cutting resource-ARN context)","description":"Follow-up from gopherstack-42s (KMS parity sweep 3). Real aws-sdk-go-v2/service/kms/types.GrantConstraints has a SourceArn field: the grant only authorizes the operation when the request is made 'on behalf of' the given AWS resource ARN (effectively aws:SourceArn). This mock's GrantConstraints struct only has EncryptionContextEquals/EncryptionContextSubset. Adding SourceArn requires a caller/resource ARN to be threaded through every KMS crypto call (from the invoking service adapter, e.g. S3/SSM/DynamoDB envelope-encryption call sites wired in cli.go) so CreateGrant's constraint can be checked against it -- this is cross-service plumbing, not a KMS-local fix, and no other service adapter currently supplies such a value either. Deferred; do not fix inside services/kms/ alone.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T15:32:49Z","created_by":"Witness Patrol","updated_at":"2026-07-05T15:32:49Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-pyv","title":"cloudwatch: PutMetricData does not enforce the timestamp acceptance window","description":"AWS rejects PutMetricData datapoints timestamped more than 2 weeks in the past or more than 2 hours in the future (InvalidParameterValue). parseMetricDataFromForm/cborDecodeDatum currently accept any timestamp with no window check. Found during the gopherstack-ton cloudwatch parity sweep; deferred to keep this sweep's PutMetricData fix (all-or-nothing response shape + Values/Counts array support + NaN/range validation) reviewable as one change.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T15:21:15Z","created_by":"Witness Patrol","updated_at":"2026-07-05T15:21:15Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-3ro","title":"cloudwatch: PutDashboard does not validate DashboardBody JSON / widget schema","description":"PutDashboardOutput has a real DashboardValidationMessages field (aws-sdk-go-v2 cloudwatch types), but handlePutDashboard/InMemoryBackend.PutDashboard store the body verbatim with no JSON-shape validation, so it always returns empty DashboardValidationMessages even for malformed dashboard bodies. Found during the gopherstack-ton cloudwatch parity sweep; deferred because full widget-schema validation is a large, separately-scoped effort.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T15:21:03Z","created_by":"Witness Patrol","updated_at":"2026-07-05T15:21:03Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-qgh","title":"SQS: perQueue FIFO throughput limit not enforced; SNS-\u003eSQS internal delivery not region-aware","description":"Deferred from parity-sweep-3 SQS audit (gopherstack-uaf). Two minor gaps found but not fixed: (1) FifoThroughputLimit=perQueue (the AWS default, 3000 msg/sec batched / 300 msg/sec unbatched per queue) has no rate limiter at all — only the perMessageGroupId variant is enforced (checkFIFOPerGroupRateLimit in backend.go). (2) sns_delivery.go's deliverSNSSubscription/deliverToDLQ always call SendMessage with an empty Region, so an SNS-subscribed SQS queue created in a non-default region will never receive delivered messages (region falls back to the backend's default). Low value / narrow blast radius; noted in services/sqs/PARITY.md gaps.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T14:52:00Z","created_by":"Witness Patrol","updated_at":"2026-07-05T14:52:00Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-gjp","title":"iam: comprehensiveBackend dual-lock + GetAccountAuthorizationDetails pagination + RoleDetail.InstanceProfileList","description":"From iam sweep (ap7): (1) backend_comprehensive.go comprehensiveBackend uses its own sync.Mutex alongside the coarse lockmetrics.RWMutex — violates one-coarse-lock rule; 20+ call sites, needs dedicated refactor+stress test. (2) GetAccountAuthorizationDetails ignores Marker/MaxItems/Filter (no pagination). (3) RoleDetailXML missing InstanceProfileList field (shape gap).","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T14:28:19Z","created_by":"Witness Patrol","updated_at":"2026-07-05T14:28:19Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-278","title":"Consolidate probe-added tests into table tests (s3/ec2/dynamodb)","description":"Probe commits 708d1961/c18fa9b1/f459c9fa added some per-case test funcs (Test_Specific...). Per convention test-style-table-tests, consolidate into subject-level table tests Test_Thing() with cases slice. Low priority cleanup; do after parity sweep.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T13:55:12Z","created_by":"Witness Patrol","updated_at":"2026-07-05T13:55:12Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-daa","title":"dynamodb: TransactWriteItems Put/Update/Delete/ConditionCheck missing unused-ExpressionAttributeNames/Values validation","description":"Follow-up from gopherstack-ej5 dynamodb parity re-audit. Plain PutItem/UpdateItem/DeleteItem call checkUnusedExpressionAttributeNames/checkUnusedExpressionAttributeValues before evaluating ConditionExpression (services/dynamodb/item_ops_crud.go), rejecting requests that declare an EAN/EAV placeholder the expression never references. TransactWriteItems' per-item condition checks (checkTransactPut/checkTransactCondExpr in services/dynamodb/transact_ops.go) skip these checks entirely, so a transactional Put/Update/Delete/ConditionCheck with an unused EAN/EAV silently succeeds instead of returning ValidationException like the single-item ops do. Not fixed in this sweep due to scope/time; worth a follow-up pass since it's a real (if lower-severity) inconsistency between single-item and transactional code paths.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T04:22:24Z","created_by":"Witness Patrol","updated_at":"2026-07-05T04:22:24Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-2bl","title":"s3: relocate objectLambdaConfigs into backend coarse lock + delete on bucket-delete","description":"From s3 probe (gopherstack-37c): object_lambda.go guards objectLambdaConfigs with a raw sync.RWMutex on the handler, outside the backend coarse lockmetrics.RWMutex; SetObjectLambdaConfig only adds, never deletes on bucket-delete (unbounded growth). Functionally correct + leak-tested today, but a backend-relocation refactor. Also: abandoned multipart uploads lack per-upload TTL (only Abort/Complete/Purge).","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T03:41:29Z","created_by":"Witness Patrol","updated_at":"2026-07-05T03:41:29Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-drp","title":"Collection-registry helper to kill backend map boilerplate (init/Reset/Snapshot/Restore)","description":"REDIRECTED (user, 2026-07-05): NOT a boilerplate-preserving map registry. Rebuild the datalayer as an idiomatic Go typed object graph — entities hold direct typed references/pointers to related entities (e.g. instance.Volumes []*Volume), NOT string-keyed cross-map lookups on every access (that adds complexity + fragility). JSON persistence field names NO LONGER need preserving — optimize the format, break it if better. Constraint that remains: serialization can't marshal a live pointer graph (cycles/identity) — so keep IDs ONLY at the (de)serialization boundary: snapshot to a flat form, rebuild typed refs on Restore. Runtime access = strong typed connection; map indexing collapses to load-time graph reconstruction. Coarse lockmetrics.RWMutex per backend still correct (cross-entity invariants). Old on-disk snapshots may not restore (acceptable for an ephemeral emulator; add a snapshot version guard that discards incompatible rather than corrupting). Do AFTER parity sweep (Phase 3). Prototype on ONE backend first (e.g. a mid-size service), validate the pattern + persistence round-trip, then roll out.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-04T15:04:15Z","created_by":"Witness Patrol","updated_at":"2026-07-05T20:05:52Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-qd3.5","title":"glue: unused documented exceptions (IdempotentParameterMismatch/ResourceNumberLimitExceeded/OperationTimeout/ConcurrentModification)","description":"These are real Glue exception types (confirmed in aws-sdk-go-v2/service/glue/types/errors.go and per-op deserializers) but this backend never returns them — no account-level quota, idempotency-token, or concurrency-conflict modeling exists to trigger them realistically. Noted during parity-sweep-3 (gopherstack-qd3).","status":"open","priority":4,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:59:52Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:59:52Z","dependencies":[{"issue_id":"gopherstack-qd3.5","depends_on_id":"gopherstack-qd3","type":"parent-child","created_at":"2026-07-05T14:59:52Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-a6y","title":"ses: MaxSendRate (per-second) not enforced, only 24h quota","description":"gopherstack-ls1 added enforcement of GetSendQuota's Max24HourSend (200) against SendEmail/SendTemplatedEmail (previously advertised but never enforced -- AccountSendingPausedException-class gap). MaxSendRate (1 msg/sec) is still only advertised via GetSendQuota and never enforced; would need a token-bucket / timestamp-window check. Deferred as lower value than the 24h quota fix (no test or integration currently depends on per-second throttling).","status":"open","priority":4,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:40:16Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:40:16Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-nbp","title":"ses: MailFromDomainNotVerifiedException never triggers (instant-verify convention)","description":"Real AWS SES SetIdentityMailFromDomain has a Pending/Success/Failed/TemporaryFailure verification lifecycle, and sends through an identity whose custom MAIL FROM domain isn't Success can return MailFromDomainNotVerifiedException. services/ses/ instantly marks MailFromStatus=Success on set (consistent with this backend's instant-verify convention for identities/domains/DKIM). Deliberately not changed this pass: modeling a Pending window would be inconsistent with the rest of the service's instant-verification design and is low value for test/dev usage. Documented as a known trap in PARITY.md so future auditors don't re-flag it. Deferred from gopherstack-ls1.","status":"open","priority":4,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:40:15Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:40:15Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-ssk","title":"ses: LimitExceededException not modeled for resource-count caps","description":"Real AWS SES returns LimitExceededException when an account exceeds resource caps (max receipt rules per rule set, max templates, max receipt filters, etc). services/ses/ has no such caps modeled (unbounded in-memory maps). Low value / high effort to simulate realistic per-resource limits; deferred from gopherstack-ls1 audit.","status":"open","priority":4,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:40:15Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:40:15Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-uve","title":"ses: GetSendStatistics never reports Bounces/Complaints/Rejects (always 0)","description":"services/ses/backend.go GetSendStatistics only aggregates DeliveryAttempts per hourly bucket; Bounces/Complaints/Rejects fields are always 0 because this emulator has no bounce/complaint event simulation. Low priority: would require modeling synthetic bounce/complaint generation (e.g. via special test addresses like real SES mailbox simulator addresses) to be meaningfully accurate. Deferred from gopherstack-ls1.","status":"open","priority":4,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T19:40:14Z","created_by":"Witness Patrol","updated_at":"2026-07-05T19:40:14Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-gvw","title":"secretsmanager: exceeding numeric limits (tags, BatchGetSecretValue SecretIdList) use InvalidParameterException instead of LimitExceededException","description":"validateTagCount (maxTagsPerSecret=50) and BatchGetSecretValue's maxSecretIDListSize=20 check both return InvalidParameterException. Real AWS Secrets Manager has a distinct LimitExceededException (aws-sdk-go-v2/service/secretsmanager/types/errors.go) used for some limit violations; verify which limits map to LimitExceededException vs InvalidParameterException and correct the mapping. Found during gopherstack-78p audit.","status":"open","priority":4,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T16:31:32Z","created_by":"Witness Patrol","updated_at":"2026-07-05T16:31:32Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-pct","title":"secretsmanager: DescribeSecretOutput.OwnerAccountId is a fabricated field not present in the real API","description":"DescribeSecretOutput (models.go) and SecretListEntry expose OwnerAccountId, which does not exist in aws-sdk-go-v2/service/secretsmanager's DescribeSecretOutput. It's harmless (unknown JSON fields are ignored by real deserializers) but inaccurate; consider removing or renaming to match a real field (there is no direct equivalent — AWS infers account from the ARN). Also: managed-external-secret fields (ExternalSecretRotationMetadata, ExternalSecretRotationRoleArn, OwningService, Type) and per-secret owning-service tracking are entirely unmodeled (owning-service ListSecrets filter always passes). Found during gopherstack-78p audit.","status":"open","priority":4,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-07-05T16:31:32Z","created_by":"Witness Patrol","updated_at":"2026-07-05T16:31:32Z","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index b2af88ec7..000000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(make test:*)", - "Bash(grep:*)", - "Bash(make lint:*)", - "Bash(golangci-lint run:*)", - "Bash(go test:*)", - "Bash(go build:*)", - "Bash(goimports:*)", - "Bash(go run:*)", - "Bash(go version:*)", - "Bash(ls:*)", - "Bash(go doc:*)", - "Bash(go tool cover:*)", - "Bash(go vet:*)", - "Bash(make integration-test:*)", - "Bash(fieldalignment -fix:*)", - "Bash(golines:*)", - "Bash(go install:*)", - "Bash(~/go/bin/golines:*)", - "Bash(find:*)", - "Bash(wc:*)", - "Bash(gh repo view:*)", - "Bash(gh auth status:*)", - "Bash(gh issue:*)", - "Bash(go list:*)", - "Bash(make:*)", - "Bash(docker compose:*)", - "Bash(docker compose up:*)", - "Bash(gopls:*)", - "Bash(mcp-language-server:*)", - "Bash(terraform-mcp-server:*)", - "Bash(npx @playwright/mcp:*)", - "Bash(make dev-mcp-install:*)", - "Bash(make dev-mcp-check:*)", - "WebFetch(domain:aws.amazon.com)", - "WebFetch(domain:docs.localstack.cloud)", - "WebFetch(domain:www.localstack.cloud)", - "WebFetch(domain:blog.localstack.cloud)", - "WebFetch(domain:github.com)", - "WebFetch(domain:pkg.go.dev)", - "mcp__playwright__browser_run_code_unsafe", - "Bash(git checkout *)", - "Bash(bd list *)", - "Bash(bd show *)" - ] - }, - "enabledMcpjsonServers": [ - "gopls", - "terraform", - "playwright" - ] -} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89404d628..af1a6f66e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,24 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v9 + modernize: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + token: ${{ env.GH_CI_TOKEN }} + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + check-latest: true + cache: true + + - name: go fix (modernize) + run: go fix -diff ./... + govulncheck: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 3a3de9814..e95a6cbc4 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,9 @@ lambda.zip .terraform.lock.hcl .terraform.lock.hcl /releaser + +# personal local Claude Code settings (never sync bypass perms to origin) +.claude/settings.local.json + +# session resume checkpoint (transient) +checkpoint.md diff --git a/cli.go b/cli.go index b6a21fe48..a200314cc 100644 --- a/cli.go +++ b/cli.go @@ -2584,6 +2584,10 @@ func initializeServices(appCtx *service.AppContext) ([]service.Registerable, err // Wire SNS→SQS delivery: when SNS publishes a message, deliver it to SQS queues. wireSNSToSQS(byName["SNS"], byName["SQS"]) + // Wire SNS→Lambda and SNS→Firehose subscription delivery, plus the SQS sender + // used for Lambda/Firehose subscription DLQ redelivery. + wireSNSToLambdaFirehose(byName["SNS"], byName["Lambda"], byName["Firehose"], byName["SQS"]) + // Wire SQS → CloudWatch metric emission for NumberOfMessagesSent/Received/Deleted. wireSQSMetrics(byName["SQS"], byName["CloudWatch"]) @@ -3145,6 +3149,54 @@ func wireSNSToSQS(snsReg, sqsReg service.Registerable) { sqsBk.SubscribeToSNS(emitter) } +// wireSNSToLambdaFirehose connects the SNS backend to Lambda and Firehose so that +// Lambda- and Firehose-protocol subscriptions actually receive published messages, +// and wires an SQS sender so failed Lambda/Firehose deliveries with a RedrivePolicy +// land in the subscription's dead-letter queue. This is independent of wireSNSToSQS: +// that function wires the SNS→SQS *subscription* delivery path via a publish emitter, +// while the SQS sender wired here only serves DLQ redelivery on failed Lambda/Firehose +// invocations (SNS backend's sqsSender field), so there is no overlap or double-wiring. +func wireSNSToLambdaFirehose(snsReg, lambdaReg, firehoseReg, sqsReg service.Registerable) { + snsH, ok := snsReg.(*snsbackend.Handler) + if !ok { + return + } + + snsBk, ok := snsH.Backend.(*snsbackend.InMemoryBackend) + if !ok { + return + } + + if lambdaH, lambdaOk := lambdaReg.(*lambdabackend.Handler); lambdaOk { + if lambdaBk, bkOk := lambdaH.Backend.(*lambdabackend.InMemoryBackend); bkOk { + snsBk.SetLambdaBackend(lambdaBk) + } + } + + if firehoseH, firehoseOk := firehoseReg.(*firehosebackend.Handler); firehoseOk { + if firehoseBk, bkOk := firehoseH.Backend.(*firehosebackend.InMemoryBackend); bkOk { + snsBk.SetFirehoseBackend(&snsFirehosePutterAdapter{backend: firehoseBk}) + } + } + + if sqsH, sqsOk := sqsReg.(*sqsbackend.Handler); sqsOk { + if sqsBk, bkOk := sqsH.Backend.(*sqsbackend.InMemoryBackend); bkOk { + snsBk.SetSQSSender(&sqsSenderAdapter{backend: sqsBk}) + } + } +} + +// snsFirehosePutterAdapter adapts the Firehose backend to the sns.FirehosePutter +// interface, which omits the context parameter that the Firehose backend's +// PutRecordBatch requires. +type snsFirehosePutterAdapter struct { + backend *firehosebackend.InMemoryBackend +} + +func (a *snsFirehosePutterAdapter) PutRecordBatch(streamName string, records [][]byte) (int, error) { + return a.backend.PutRecordBatch(context.Background(), streamName, records) +} + // wireSQSMetrics wires the CloudWatch metric emitter into the SQS backend so that // SendMessage, ReceiveMessage, and DeleteMessage operations emit CloudWatch metrics. func wireSQSMetrics(sqsReg, cwReg service.Registerable) { @@ -3165,7 +3217,7 @@ func wireSQSMetrics(sqsReg, cwReg service.Registerable) { sqsBk.SetMetricEmitter( sqsbackend.MetricEmitterFunc( func(namespace, name string, value float64, unit string) error { - _, err := cwBk.PutMetricData(namespace, []cwbackend.MetricDatum{ + err := cwBk.PutMetricData(namespace, []cwbackend.MetricDatum{ { MetricName: name, Value: value, @@ -4204,7 +4256,7 @@ func wireCWLogsMetricEmitter(cwlogsReg, cwReg service.Registerable) { cwlogsBk.SetMetricEmitter( cwlogsbackend.MetricEmitterFunc( func(namespace, name string, value float64, unit string) error { - _, err := cwBk.PutMetricData(namespace, []cwbackend.MetricDatum{ + err := cwBk.PutMetricData(namespace, []cwbackend.MetricDatum{ { MetricName: name, Namespace: namespace, diff --git a/cli_sns_wiring_test.go b/cli_sns_wiring_test.go new file mode 100644 index 000000000..40c53d4fe --- /dev/null +++ b/cli_sns_wiring_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "context" + "encoding/json" + "io" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + s3sdk "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/arn" + "github.com/blackbirdworks/gopherstack/pkgs/config" + firehosebackend "github.com/blackbirdworks/gopherstack/services/firehose" + lambdabackend "github.com/blackbirdworks/gopherstack/services/lambda" + snsbackend "github.com/blackbirdworks/gopherstack/services/sns" + sqsbackend "github.com/blackbirdworks/gopherstack/services/sqs" +) + +// wiringMockS3Storer implements firehose.S3Storer so the test can observe that a +// record delivered through the SNS→Firehose path actually reached S3. +type wiringMockS3Storer struct { + calls []wiringS3PutCall +} + +type wiringS3PutCall struct { + bucket string + key string + body []byte +} + +func (m *wiringMockS3Storer) PutObject( + _ context.Context, + input *s3sdk.PutObjectInput, +) (*s3sdk.PutObjectOutput, error) { + body, _ := io.ReadAll(input.Body) + m.calls = append(m.calls, wiringS3PutCall{ + bucket: aws.ToString(input.Bucket), + key: aws.ToString(input.Key), + body: body, + }) + + return &s3sdk.PutObjectOutput{}, nil +} + +// TestWireSNSToLambdaFirehose_EndToEndDelivery exercises the exact composition-root +// wiring used by cli.go's setup (wireSNSToLambdaFirehose) against REAL Lambda, Firehose, +// and SQS backends — not test doubles — to prove that subscribing a Lambda function or +// Firehose delivery stream to an SNS topic and publishing actually delivers, instead of +// silently no-opping as it did before SetLambdaBackend/SetFirehoseBackend/SetSQSSender +// were wired in the running binary. +func TestWireSNSToLambdaFirehose_EndToEndDelivery(t *testing.T) { + t.Parallel() + + snsBk := snsbackend.NewInMemoryBackend() + snsH := snsbackend.NewHandler(snsBk) + + lambdaBk := lambdabackend.NewInMemoryBackend( + nil, nil, lambdabackend.DefaultSettings(), config.DefaultAccountID, config.DefaultRegion, + ) + lambdaH := lambdabackend.NewHandler(lambdaBk) + + firehoseBk := firehosebackend.NewInMemoryBackend(config.DefaultAccountID, config.DefaultRegion) + firehoseH := firehosebackend.NewHandler(firehoseBk) + + sqsBk := sqsbackend.NewInMemoryBackend() + sqsH := sqsbackend.NewHandler(sqsBk) + + // This is the exact call cli.go's setupServices makes. + wireSNSToLambdaFirehose(snsH, lambdaH, firehoseH, sqsH) + + ctx := context.Background() + + // --- SNS -> Firehose: publish must reach the real Firehose backend and flush to S3. --- + s3mock := &wiringMockS3Storer{} + firehoseBk.SetS3Backend(s3mock) + + _, err := firehoseBk.CreateDeliveryStream(ctx, firehosebackend.CreateDeliveryStreamInput{ + Name: "wiring-test-stream", + S3Destination: &firehosebackend.S3DestinationDescription{ + BucketARN: "arn:aws:s3:::wiring-test-bucket", + BufferingHints: &firehosebackend.BufferingHints{ + SizeInMBs: 128, + IntervalInSeconds: 900, + }, + }, + }) + require.NoError(t, err) + + firehoseTopic, err := snsBk.CreateTopic("firehose-topic", nil) + require.NoError(t, err) + + firehoseStreamARN := arn.Build( + "firehose", config.DefaultRegion, config.DefaultAccountID, "deliverystream/wiring-test-stream", + ) + _, err = snsBk.Subscribe(firehoseTopic.TopicArn, "firehose", firehoseStreamARN, "") + require.NoError(t, err) + + _, err = snsBk.Publish(firehoseTopic.TopicArn, "hello-firehose", "", "", nil) + require.NoError(t, err) + + firehoseBk.FlushAll(ctx) + + require.Len(t, s3mock.calls, 1, + "SNS publish must have been delivered to the real Firehose backend via cli.go's wireSNSToLambdaFirehose") + assert.Contains(t, string(s3mock.calls[0].body), "hello-firehose") + + // --- SNS -> Lambda with DLQ: publish must reach the real Lambda backend. There is + // no Docker runtime available in this test, so the invocation fails deterministically + // (a real failure mode, not a stub) — proving delivery requires the failed message to + // be redirected to the real SQS DLQ via the wired SQS sender. --- + dlqOut, err := sqsBk.CreateQueue(&sqsbackend.CreateQueueInput{QueueName: "lambda-dlq"}) + require.NoError(t, err) + + lambdaTopic, err := snsBk.CreateTopic("lambda-topic", nil) + require.NoError(t, err) + + functionARN := arn.Build( + "lambda", config.DefaultRegion, config.DefaultAccountID, "function:wiring-test-fn", + ) + sub, err := snsBk.Subscribe(lambdaTopic.TopicArn, "lambda", functionARN, "") + require.NoError(t, err) + + dlqARN := arn.Build("sqs", config.DefaultRegion, config.DefaultAccountID, "lambda-dlq") + redrivePolicy, err := json.Marshal(map[string]string{"deadLetterTargetArn": dlqARN}) + require.NoError(t, err) + require.NoError(t, snsBk.SetSubscriptionAttributes(sub.SubscriptionArn, "RedrivePolicy", string(redrivePolicy))) + + _, err = snsBk.Publish(lambdaTopic.TopicArn, "hello-lambda", "", "", nil) + require.NoError(t, err) + + recvOut, err := sqsBk.ReceiveMessage(&sqsbackend.ReceiveMessageInput{ + QueueURL: dlqOut.QueueURL, + MaxNumberOfMessages: 1, + }) + require.NoError(t, err) + require.Len(t, recvOut.Messages, 1, + "failed Lambda delivery must have been redirected to the real SQS DLQ via cli.go's wireSNSToLambdaFirehose "+ + "(proves both SetLambdaBackend and SetSQSSender were wired)") + assert.Equal(t, "hello-lambda", recvOut.Messages[0].Body) +} diff --git a/go.mod b/go.mod index 2b6d1591f..81e827629 100644 --- a/go.mod +++ b/go.mod @@ -7,18 +7,18 @@ toolchain go1.26.4 require ( github.com/alecthomas/kong v1.15.0 github.com/alicebob/miniredis/v2 v2.38.0 - github.com/aws/aws-sdk-go-v2 v1.42.0 - github.com/aws/aws-sdk-go-v2/config v1.32.25 - github.com/aws/aws-sdk-go-v2/credentials v1.19.24 + github.com/aws/aws-sdk-go-v2 v1.42.1 + github.com/aws/aws-sdk-go-v2/config v1.32.27 + github.com/aws/aws-sdk-go-v2/credentials v1.19.26 github.com/aws/aws-sdk-go-v2/service/acm v1.37.21 github.com/aws/aws-sdk-go-v2/service/acmpca v1.46.10 - github.com/aws/aws-sdk-go-v2/service/amplify v1.39.4 + github.com/aws/aws-sdk-go-v2/service/amplify v1.39.6 github.com/aws/aws-sdk-go-v2/service/apigateway v1.38.6 github.com/aws/aws-sdk-go-v2/service/apigatewayv2 v1.33.7 github.com/aws/aws-sdk-go-v2/service/appconfig v1.43.11 github.com/aws/aws-sdk-go-v2/service/appconfigdata v1.23.20 github.com/aws/aws-sdk-go-v2/service/applicationautoscaling v1.41.12 - github.com/aws/aws-sdk-go-v2/service/appsync v1.54.4 + github.com/aws/aws-sdk-go-v2/service/appsync v1.54.6 github.com/aws/aws-sdk-go-v2/service/athena v1.57.2 github.com/aws/aws-sdk-go-v2/service/autoscaling v1.64.2 github.com/aws/aws-sdk-go-v2/service/backup v1.54.8 @@ -28,20 +28,20 @@ require ( github.com/aws/aws-sdk-go-v2/service/cognitoidentity v1.33.20 github.com/aws/aws-sdk-go-v2/service/cognitoidentityprovider v1.59.1 github.com/aws/aws-sdk-go-v2/service/configservice v1.61.2 - github.com/aws/aws-sdk-go-v2/service/dynamodb v1.59.0 - github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.34.0 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.59.2 + github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.34.2 github.com/aws/aws-sdk-go-v2/service/ec2 v1.294.0 - github.com/aws/aws-sdk-go-v2/service/ecr v1.58.4 - github.com/aws/aws-sdk-go-v2/service/ecs v1.85.0 + github.com/aws/aws-sdk-go-v2/service/ecr v1.58.6 + github.com/aws/aws-sdk-go-v2/service/ecs v1.86.2 github.com/aws/aws-sdk-go-v2/service/efs v1.41.12 github.com/aws/aws-sdk-go-v2/service/elasticache v1.51.11 github.com/aws/aws-sdk-go-v2/service/eventbridge v1.45.21 github.com/aws/aws-sdk-go-v2/service/firehose v1.42.11 - github.com/aws/aws-sdk-go-v2/service/iam v1.54.5 - github.com/aws/aws-sdk-go-v2/service/iot v1.75.4 + github.com/aws/aws-sdk-go-v2/service/iam v1.54.7 + github.com/aws/aws-sdk-go-v2/service/iot v1.75.6 github.com/aws/aws-sdk-go-v2/service/kinesis v1.43.2 - github.com/aws/aws-sdk-go-v2/service/kms v1.53.4 - github.com/aws/aws-sdk-go-v2/service/lambda v1.93.0 + github.com/aws/aws-sdk-go-v2/service/kms v1.53.6 + github.com/aws/aws-sdk-go-v2/service/lambda v1.94.1 github.com/aws/aws-sdk-go-v2/service/opensearch v1.59.0 github.com/aws/aws-sdk-go-v2/service/rds v1.116.2 github.com/aws/aws-sdk-go-v2/service/redshift v1.62.3 @@ -49,16 +49,16 @@ require ( github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.31.8 github.com/aws/aws-sdk-go-v2/service/route53 v1.62.3 github.com/aws/aws-sdk-go-v2/service/route53resolver v1.42.3 - github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.104.2 github.com/aws/aws-sdk-go-v2/service/s3control v1.68.2 github.com/aws/aws-sdk-go-v2/service/scheduler v1.17.20 - github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.42.3 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.42.5 github.com/aws/aws-sdk-go-v2/service/ses v1.34.20 github.com/aws/aws-sdk-go-v2/service/sfn v1.40.8 - github.com/aws/aws-sdk-go-v2/service/sns v1.40.1 - github.com/aws/aws-sdk-go-v2/service/sqs v1.44.0 - github.com/aws/aws-sdk-go-v2/service/ssm v1.69.3 - github.com/aws/aws-sdk-go-v2/service/sts v1.43.3 + github.com/aws/aws-sdk-go-v2/service/sns v1.40.3 + github.com/aws/aws-sdk-go-v2/service/sqs v1.44.2 + github.com/aws/aws-sdk-go-v2/service/ssm v1.69.5 + github.com/aws/aws-sdk-go-v2/service/sts v1.43.5 github.com/aws/aws-sdk-go-v2/service/support v1.31.23 github.com/aws/aws-sdk-go-v2/service/swf v1.33.14 github.com/aws/smithy-go v1.27.3 @@ -75,7 +75,7 @@ require ( github.com/prometheus/client_model v0.6.2 github.com/stretchr/testify v1.11.1 github.com/testcontainers/testcontainers-go v0.43.0 - github.com/vektah/gqlparser/v2 v2.5.35 + github.com/vektah/gqlparser/v2 v2.5.36 golang.org/x/crypto v0.53.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -94,13 +94,13 @@ require ( github.com/aws/aws-sdk-go-v2/service/codebuild v1.68.11 github.com/aws/aws-sdk-go-v2/service/codecommit v1.33.10 github.com/aws/aws-sdk-go-v2/service/codeconnections v1.10.22 - github.com/aws/aws-sdk-go-v2/service/codedeploy v1.36.4 - github.com/aws/aws-sdk-go-v2/service/codepipeline v1.47.4 + github.com/aws/aws-sdk-go-v2/service/codedeploy v1.36.6 + github.com/aws/aws-sdk-go-v2/service/codepipeline v1.47.7 github.com/aws/aws-sdk-go-v2/service/codestarconnections v1.35.15 github.com/aws/aws-sdk-go-v2/service/costexplorer v1.63.8 github.com/aws/aws-sdk-go-v2/service/databasemigrationservice v1.61.8 github.com/aws/aws-sdk-go-v2/service/docdb v1.48.11 - github.com/aws/aws-sdk-go-v2/service/eks v1.87.0 + github.com/aws/aws-sdk-go-v2/service/eks v1.88.1 github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.33.21 github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.54.8 github.com/aws/aws-sdk-go-v2/service/emrserverless v1.40.2 @@ -141,7 +141,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/pipes v1.23.18 github.com/aws/aws-sdk-go-v2/service/ram v1.36.1 github.com/aws/aws-sdk-go-v2/service/rdsdata v1.32.19 - github.com/aws/aws-sdk-go-v2/service/redshiftdata v1.40.6 + github.com/aws/aws-sdk-go-v2/service/redshiftdata v1.40.8 github.com/aws/aws-sdk-go-v2/service/s3tables v1.14.3 github.com/aws/aws-sdk-go-v2/service/sagemaker v1.236.0 github.com/aws/aws-sdk-go-v2/service/sagemakerruntime v1.39.3 @@ -218,7 +218,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - modernc.org/libc v1.73.4 // indirect + modernc.org/libc v1.73.5 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect ) @@ -240,22 +240,22 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/agnivade/levenshtein v1.2.1 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 // indirect github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk v1.34.0 github.com/aws/aws-sdk-go-v2/service/emr v1.57.7 github.com/aws/aws-sdk-go-v2/service/fis v1.37.18 - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.12.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.12.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.2.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.31.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.8 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bitfield/gotestdox v0.2.2 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect @@ -290,11 +290,11 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/golang-lru/arc/v2 v2.0.7 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect - github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/compress v1.19.0 // indirect github.com/lufia/plan9stats v0.0.0-20260627054121-477a66015f15 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect github.com/moby/patternmatcher v0.6.1 // indirect @@ -309,12 +309,12 @@ require ( github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/common v0.69.0 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect - github.com/prometheus/procfs v0.21.0 // indirect + github.com/prometheus/procfs v0.21.1 // indirect github.com/redis/go-redis/extra/rediscmd/v9 v9.21.0 // indirect github.com/redis/go-redis/extra/redisotel/v9 v9.21.0 // indirect github.com/redis/go-redis/v9 v9.21.0 // indirect github.com/rs/xid v1.6.0 // indirect - github.com/shirou/gopsutil/v4 v4.26.5 // indirect + github.com/shirou/gopsutil/v4 v4.26.6 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/tklauser/go-sysconf v0.4.0 // indirect github.com/tklauser/numcpus v0.12.0 // indirect @@ -353,9 +353,9 @@ require ( golang.org/x/text v0.38.0 // indirect golang.org/x/tools v0.47.0 // indirect golang.org/x/vuln v1.1.4 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260622175928-b703f567277d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260622175928-b703f567277d // indirect - google.golang.org/grpc v1.81.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/grpc v1.82.0 // indirect google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v2 v2.4.0 // indirect gotest.tools/gotestsum v1.13.0 // indirect diff --git a/go.sum b/go.sum index cd761c71e..eeb312bc8 100644 --- a/go.sum +++ b/go.sum @@ -28,30 +28,30 @@ github.com/aws/aws-dax-go v1.2.15 h1:30rH3+QgjpjemrVg0NGIG5FnB1izJZ7jUZuBb1Fy8ak github.com/aws/aws-dax-go v1.2.15/go.mod h1:4f/qGLBQlPYd+fmAfG4n4oSvN19JdKNYYmsr90/MPso= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/aws/aws-sdk-go-v2 v1.42.0 h1:XvXMJTkFQtpBKIWZnmr9ZEOc2InWM2yldjXEJ/bymhA= -github.com/aws/aws-sdk-go-v2 v1.42.0/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 h1:p1BBrg/Hhp6uK7zpejeI8QFXHJeC/mynzi04Sl03k9g= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13/go.mod h1:8cIfkE9MDhkRZGpQ22aV6/lkYeYSozpz16Smrs5x4Ls= -github.com/aws/aws-sdk-go-v2/config v1.32.25 h1:ACCejvStYoilgwrfegSt5ZntCbPrk52qfwyNcnl3omM= -github.com/aws/aws-sdk-go-v2/config v1.32.25/go.mod h1:LJyU8sDRbXUxFn8xMJIGP+v9QYYwveNLI8a/giAOiAs= -github.com/aws/aws-sdk-go-v2/credentials v1.19.24 h1:2hQqYCV9yqyePQ9o6dCrZc/zO8U3TwPr9mIKlZnPu/I= -github.com/aws/aws-sdk-go-v2/credentials v1.19.24/go.mod h1:IDwpACtwqHLISdzfwUUNq4P9DsB/h5BLg4FwJPNfqFY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 h1:r6qZHbT+wxgWO/e9vYNUEtg7lv5+UN3pRqKhLXvnArg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29/go.mod h1:QRnaRcTVGKPGRy8w78HMQtKUGRYcnMZAANATkeVA6Mo= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 h1:f3vKqSo13fhTYb+JEcXwXefZQE26I1FB5eTSniU67ko= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29/go.mod h1:MzoLFUArKGpGD+ukmPiTPG1X5x4o6M2kq4v2dr1FiEc= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 h1:RdwIf/CuUsvJX3RgJagbOyotl/cxoLY4xviKuE7p2GY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29/go.mod h1:71wt8W2EgswdZy9Mf9KNnzxZ3TiZlv4caKghPktDOkA= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 h1:VTGy885W5DKBxWRUJbym9hytNaYzsyaPkCHGRRMAOhU= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30/go.mod h1:AS0HycUvJRFvTt613AYDOgO2jzw+00cVSMny8XB3yMY= +github.com/aws/aws-sdk-go-v2 v1.42.1 h1:9eOTgu1z/dVtYpNZ3/8/XbbaX0x/BqE3HUzAzs6K0ek= +github.com/aws/aws-sdk-go-v2 v1.42.1/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E= +github.com/aws/aws-sdk-go-v2/config v1.32.27 h1:SJwJ9Q4kM7v5QVSYYyXj3znRr6lNyZEhSgAXmXXcVbI= +github.com/aws/aws-sdk-go-v2/config v1.32.27/go.mod h1:uBfrzTRedDmB2u+b6+UlaKJy2O6VSH5un2jP24t/KvQ= +github.com/aws/aws-sdk-go-v2/credentials v1.19.26 h1:Si8kk1kyJnuJWCEgiwpBtTdtgSdR7i611596NnC0YIQ= +github.com/aws/aws-sdk-go-v2/credentials v1.19.26/go.mod h1:lBckz+W9SAdNtSDw3pYgQUJDJFcBBWry0GSzw+bK0TY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 h1:/hi1JADLEW9YYryEz1w4GQu0EtP23pP553Cf9KgsDV4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30/go.mod h1:/3AOgy4K17Dm4ucMZVC/MJkzy5kmfKUcINRHZyo0koQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 h1:xM/Is9cKMHa8Jj8zkvWhvrFkZsXJV9E+BB4g0HW0duQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30/go.mod h1:WueJeNDZvK1fMYEWJIkcivBfEzUkTpBhzlrUKKY8EuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 h1:jn46zC9LdsVR/ZpMIJqMqb8hHv31BlLx3ulVqNspUOk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30/go.mod h1:1hTMsAgbdS/AtUi4bw8+gUuh1pceo+eXRLfpSuSQj3M= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 h1:3GUprIsfmGcC5SACIyB0e7E0BM1O1b3Erl5CePYIAeQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31/go.mod h1:7PuV1yl5e2xnUbm+RqvVg5i2iBM8EyijZNoI9wsOoOc= github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.48.0 h1:SG+cxHh/AWWo4TWg/UzjZPEBbDuVEC1i4lfrK+bdMwE= github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.48.0/go.mod h1:CP5pWLCGRZJDXLkeUvxTulAvFkPnfK+TqJNtJtH/Jmo= github.com/aws/aws-sdk-go-v2/service/acm v1.37.21 h1:AUceKJhgt+FOwImMUPbOHKLpe5O9a8N/RtC+tLQ+sxc= github.com/aws/aws-sdk-go-v2/service/acm v1.37.21/go.mod h1:kkbySLpdZk0UNdU23rBaef7IfuSRy0/jEM84BGCcvKM= github.com/aws/aws-sdk-go-v2/service/acmpca v1.46.10 h1:gS51h/TsRA0bi9cceJBSFFFWA9ctn/AKQX9+v/78LU0= github.com/aws/aws-sdk-go-v2/service/acmpca v1.46.10/go.mod h1:rDFz0lInTwgStuv3DBGGfNsCW4KOF2eMPmnWMBAijNo= -github.com/aws/aws-sdk-go-v2/service/amplify v1.39.4 h1:iSmP1EQ2TTJB22g7TuCFQ5Li9nhX5m9ZJ3Z/qIKv2kQ= -github.com/aws/aws-sdk-go-v2/service/amplify v1.39.4/go.mod h1:q65c1mkcqyyEGFchMaiqL5EJqCAdMDnEjoYTMqGOAe0= +github.com/aws/aws-sdk-go-v2/service/amplify v1.39.6 h1:Pf03fpa7k4gosVfJNIEjetpnmWUjpQc77bZfBz8Pmv4= +github.com/aws/aws-sdk-go-v2/service/amplify v1.39.6/go.mod h1:6Fh1ebEJHcdK3DiTQx3s3bC5wT+f8nwV2PlrHWmdaxQ= github.com/aws/aws-sdk-go-v2/service/apigateway v1.38.6 h1:dzd86UudvxJ1c6z/o+hHh7ZhkoBrh81XYz/M11zwQYI= github.com/aws/aws-sdk-go-v2/service/apigateway v1.38.6/go.mod h1:jWmyEnBPJdt+RaHSRzZDKp3HyyzjOofGp4+xXY503Do= github.com/aws/aws-sdk-go-v2/service/apigatewaymanagementapi v1.29.13 h1:357Yo8n9E3WKIpei+mWQYVsIXMUM+c81J0LMYWjIGVc= @@ -70,8 +70,8 @@ github.com/aws/aws-sdk-go-v2/service/apprunner v1.40.2 h1:2plkrtfEi/F45UbZ+VKObz github.com/aws/aws-sdk-go-v2/service/apprunner v1.40.2/go.mod h1:s7fC1MDh0uwEV0iPEeHmEr1ScG7fhH+YyAtQ+clrugQ= github.com/aws/aws-sdk-go-v2/service/appstream v1.60.3 h1:GwnhiEAYlKBPvruTneFNoXhL7cd3GHd6yZwhjOH+zWs= github.com/aws/aws-sdk-go-v2/service/appstream v1.60.3/go.mod h1:3ilI3sliTS+vlXk1fc0S1q0I33+knpk2seninsWizs0= -github.com/aws/aws-sdk-go-v2/service/appsync v1.54.4 h1:usX2XiWnjFbtJs7hlIbp9RPqUlcBY+n5Spm1Kzq/lY4= -github.com/aws/aws-sdk-go-v2/service/appsync v1.54.4/go.mod h1:iwnsAGkoqj2WDZG5crswFe1C9WLS7BFeiihxz1Ow0VI= +github.com/aws/aws-sdk-go-v2/service/appsync v1.54.6 h1:ts5l+YsPEOziTM2jYc5hyoHpYaFRaA3NHkPX2jXe6r0= +github.com/aws/aws-sdk-go-v2/service/appsync v1.54.6/go.mod h1:zRq7tfgqOsclvS3FjjKcvQRWOmcMfhJgje2F9hkQtZ0= github.com/aws/aws-sdk-go-v2/service/athena v1.57.2 h1:rxrP6hget2gn77fo/w9/fw0AMzt/pwsYCTR6sp3KIV0= github.com/aws/aws-sdk-go-v2/service/athena v1.57.2/go.mod h1:9+Y9vgcoZprTgdsgVHksxCPKVaJeocOBn8WizxZe6UY= github.com/aws/aws-sdk-go-v2/service/autoscaling v1.64.2 h1:pzFtdV2DArJul6aM3+WiWjUQ63IzrSnSbvBr8FAokt4= @@ -108,10 +108,10 @@ github.com/aws/aws-sdk-go-v2/service/codecommit v1.33.10 h1:hAK/gb7zTE3zIU3tKuG1 github.com/aws/aws-sdk-go-v2/service/codecommit v1.33.10/go.mod h1:UgLONgXRd26X5Zz/RG29duwo+mK5ZbqSRGOs/sVl6Uw= github.com/aws/aws-sdk-go-v2/service/codeconnections v1.10.22 h1:VLjpUuPCiYBnO8r801Z/DKiqDcqi3XBPq7XZP/pGsd0= github.com/aws/aws-sdk-go-v2/service/codeconnections v1.10.22/go.mod h1:TcTHKeyZCc14t8RI8PM8nbwid4Xr4CzxEZfHKcGlCiA= -github.com/aws/aws-sdk-go-v2/service/codedeploy v1.36.4 h1:Tu6hLFB2lFvLt38SR49Wn7pryxErs1mTCyLWGgVtTsw= -github.com/aws/aws-sdk-go-v2/service/codedeploy v1.36.4/go.mod h1:uRlFyE/vBx2r2apdJ1t1NTdHVO12VK7Uv6LgTy3bFtk= -github.com/aws/aws-sdk-go-v2/service/codepipeline v1.47.4 h1:v/pmsep1Vs+QfSghyBHYLGZJuEEKyDrCNVf4+qvAec0= -github.com/aws/aws-sdk-go-v2/service/codepipeline v1.47.4/go.mod h1:WX3sF7RcrXTLTXld80dBMBD0AvtZfWeloP8iX8VaGdw= +github.com/aws/aws-sdk-go-v2/service/codedeploy v1.36.6 h1:CZ6FQLnsVw+4q03CCCAYSEM5lKFlJxs+iC6U5BiKa70= +github.com/aws/aws-sdk-go-v2/service/codedeploy v1.36.6/go.mod h1:Xtjzn0NHOCb8Uf+KCv9fWSVHfJdli4yRox6S/QFXh2U= +github.com/aws/aws-sdk-go-v2/service/codepipeline v1.47.7 h1:T5zrFQsfnBGHKxfymxBv4sde8L5LZB9NpGHAwftCvDk= +github.com/aws/aws-sdk-go-v2/service/codepipeline v1.47.7/go.mod h1:TauWDdo/zCElLORUQcUb8JkIxoNrctD03ECepck6/oc= github.com/aws/aws-sdk-go-v2/service/codestarconnections v1.35.15 h1:W6e6rV6k2oUthzPZTPgon2IcOg5DiGTimnwK76UyUuk= github.com/aws/aws-sdk-go-v2/service/codestarconnections v1.35.15/go.mod h1:40aemoBtijD+fwyMWhUYsQ0pB/2PH2VYarYEWJDu5c4= github.com/aws/aws-sdk-go-v2/service/cognitoidentity v1.33.20 h1:ZEijcC7Fpm0X+CeecuHD7t62ZbV6XSRDyR1ceUylJdc= @@ -140,20 +140,20 @@ github.com/aws/aws-sdk-go-v2/service/dlm v1.37.2 h1:lAdopCU6El+Ab3MKiGJSQX2jYHL2 github.com/aws/aws-sdk-go-v2/service/dlm v1.37.2/go.mod h1:3Y5Nk/qGkfwwdY1UtbElPaeZcfzH9/xXByOgQIHLWwk= github.com/aws/aws-sdk-go-v2/service/docdb v1.48.11 h1:7D5G4fCH+ZE14P0sZdSkQQjIWnCZzB05kggmPWW0Gss= github.com/aws/aws-sdk-go-v2/service/docdb v1.48.11/go.mod h1:Iw7ntHMQ/0/dNQiK/sJZl08jbmR9wRr4JOMFPxFgxmM= -github.com/aws/aws-sdk-go-v2/service/dynamodb v1.59.0 h1:S1qETDbdXKZMYVveuxACCKuRqnAt2NlnmYnlq5SeuMY= -github.com/aws/aws-sdk-go-v2/service/dynamodb v1.59.0/go.mod h1:jLkDwIDBkCIpiENQhAOjAR2L9jwj56mZgVEvuro4gUE= -github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.34.0 h1:O9JPx24Pr+CO7kkxo1EnHwug1UJKqgsMadILrJY72Hw= -github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.34.0/go.mod h1:Uk+gmBWz7i2hg5UeGT03758STndDVgEl5+siz9qwUP8= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.59.2 h1:Rk2vMAylCpfUapcjnDLApIyzBBqANEyTiN54VktI07Y= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.59.2/go.mod h1:HnWoC3m6VmjUSg+kBL6OgQsXdyRAGzBYWb7B3J2f+JM= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.34.2 h1:iIK8515D/kc9ieiCePJFuwYqTZDJa3OCsUnkxgtMzHg= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.34.2/go.mod h1:2hgy3N0+CUZxbEKTNCTDWLevN+5pV7lY3B7hMl+oEQ4= github.com/aws/aws-sdk-go-v2/service/ec2 v1.294.0 h1:776KnBqePBBR6zEDi0bUIHXzUBOISa2WgAKEgckUF8M= github.com/aws/aws-sdk-go-v2/service/ec2 v1.294.0/go.mod h1:rB577GvkmJADVOFGY8/j9sPv/ewcsEtQNsd9Lrn7Zx0= -github.com/aws/aws-sdk-go-v2/service/ecr v1.58.4 h1:fo6cmbxkKq/OtKUG0sK70fDsYjtKuSkjIQZUJwt24YM= -github.com/aws/aws-sdk-go-v2/service/ecr v1.58.4/go.mod h1:7VJFM2lSPHz2I1rRb0a+lbphoOp7hXIgYjGhSTOLY7k= -github.com/aws/aws-sdk-go-v2/service/ecs v1.85.0 h1:1e9htzu1Yykx0SSNd8dpWJXa5g8i9Wcl1ngdjPaBHsM= -github.com/aws/aws-sdk-go-v2/service/ecs v1.85.0/go.mod h1:0vahPCh3slyORHbSuAP8YDyJKLEUQAMX7+bzYGxEnVI= +github.com/aws/aws-sdk-go-v2/service/ecr v1.58.6 h1:EHQUIj2cTRnKlrIJZp+vn2jZPd6yukdxKyCPmHv3hCc= +github.com/aws/aws-sdk-go-v2/service/ecr v1.58.6/go.mod h1:UzfjIuiQOpusteIHBCLIikQpxh8ctmdQCvSWWzbcYYI= +github.com/aws/aws-sdk-go-v2/service/ecs v1.86.2 h1:t3eGwX8zYsFSEvWiRNG4KN+pMgYyFOc+wQEZPxWMTKk= +github.com/aws/aws-sdk-go-v2/service/ecs v1.86.2/go.mod h1:FZTiizNr2CG5myXP2I8pyCWM0/k4uwAnZXMkmjxgE3o= github.com/aws/aws-sdk-go-v2/service/efs v1.41.12 h1:YZXW11dESIf6CNhMG2ICZonCkzKBaGLuFamSJTYV5g0= github.com/aws/aws-sdk-go-v2/service/efs v1.41.12/go.mod h1:+rjniKD0YQAmjiDNJvLodKXn1vXWwMpctrr/M4zm1V4= -github.com/aws/aws-sdk-go-v2/service/eks v1.87.0 h1:bftLltXNWmNr9ed3CaQnVlzNPTNTFdHguNhIsZF6DxM= -github.com/aws/aws-sdk-go-v2/service/eks v1.87.0/go.mod h1:rbIASs+SfCDUXx2EdfMkNpDGptlW8hvMZ9AawRiUBqE= +github.com/aws/aws-sdk-go-v2/service/eks v1.88.1 h1:Z05Ct9QXAZXKbC1wNLQln0/6FDzTcWPqI1YTWqs/pRY= +github.com/aws/aws-sdk-go-v2/service/eks v1.88.1/go.mod h1:MSAmCaKIo6Ph/yg73tj8/HZnILwyk4Px2tXfL4PO/HQ= github.com/aws/aws-sdk-go-v2/service/elasticache v1.51.11 h1:wnE+6xW2TIVxjalZ/7V7sqFQZdKiNJQIJ5hp9MNlQ3U= github.com/aws/aws-sdk-go-v2/service/elasticache v1.51.11/go.mod h1:WZitdEv46MSo81s7dEd5UV7cejTCukLVzOFm3xE/pTY= github.com/aws/aws-sdk-go-v2/service/elasticbeanstalk v1.34.0 h1:p9R6ckEzoRaNFlfYTi9OEeq/ruUGta3fB3vEktiLNnM= @@ -184,24 +184,24 @@ github.com/aws/aws-sdk-go-v2/service/glue v1.137.2 h1:qGCcDn2ASnsWw20B6rPtV+e2sE github.com/aws/aws-sdk-go-v2/service/glue v1.137.2/go.mod h1:sQhaXaqTtphSiXd2CXK80RBR4e7HZ0waa9geoIh0Vq8= github.com/aws/aws-sdk-go-v2/service/guardduty v1.78.2 h1:xH0fxbdTUQsR51wXrgPmCaY5544wk1d2rBynDKEePLM= github.com/aws/aws-sdk-go-v2/service/guardduty v1.78.2/go.mod h1:XdvcY6/ivzh8fBF4R9nmi3fbP6Yb3Ooy7x7+ONEMkVs= -github.com/aws/aws-sdk-go-v2/service/iam v1.54.5 h1:a/gAOhIOi+vHYeRU224WIXlJrLXs4Z1Qbm92vfX64jc= -github.com/aws/aws-sdk-go-v2/service/iam v1.54.5/go.mod h1:tMNzI+fYFCk4cIdZ7FEybLzShwnmWkfxQw85ED1b4ng= +github.com/aws/aws-sdk-go-v2/service/iam v1.54.7 h1:undHpuVUg25wS2CpeXNqVjIcJComV2RE5AZ7mJGth2U= +github.com/aws/aws-sdk-go-v2/service/iam v1.54.7/go.mod h1:5H/UUroHvcKm6l2qaqh3CMM6R9K91ls8Y8rVX6cG3ts= github.com/aws/aws-sdk-go-v2/service/identitystore v1.36.3 h1:49Ty0wZtsSGw6s5a2g41a7Rusc8DNjLMXkTQyHHJPoQ= github.com/aws/aws-sdk-go-v2/service/identitystore v1.36.3/go.mod h1:SmQRp+dMvew7f6kSRkfGnJvlL/dRxfrUUEWMcBkhy9Y= github.com/aws/aws-sdk-go-v2/service/inspector2 v1.48.2 h1:umtknResciXCdbRPGjgD2B3rudpzvLaTZwf6FQKUrME= github.com/aws/aws-sdk-go-v2/service/inspector2 v1.48.2/go.mod h1:+tPtITws5lwb2ZO1cjh/qjyBmji2db5JyDOl6viONd0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12 h1:ZD2+BSw9vFsNlKYIasSNt3uDbjqqXIBcM13UJv/Lx2k= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.12/go.mod h1:Ms4zlcVBbXbiP7EVLhl+lgjvA/a7YphqQ3Ih3174EmI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22 h1:V51LGlOq/1VsDsHUdoklAQi7rMmx4qQubvFYAlP2254= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22/go.mod h1:4Pzhyz8hJOm2bepgl+NjvRx8vlUFAIIvJnZ/MkcNPpU= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.12.6 h1:Bs2OwYq0HBgHYwfGmUwYIPtTNaGMGAHkRje4jmW2VoI= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.12.6/go.mod h1:OTctu4cW8t7/TRlTKPLT6akzyOkfceMWhtEHqtYDIQQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 h1:DRebniUGZ2MqiiIVmQJ04vIXr918hubdHMnarSLEWyU= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29/go.mod h1:LfRkPCD8YHDM2E5eTkos2UpwYeZnBcVarTa8L59bJHA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29 h1:hiME6pBzC7OTl9LMtlyTWBuEl1f4QBcUmFDKC7MLXtc= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29/go.mod h1:G7RP+uhagpKtKhd1BM9N6JQqjCcGEU47K5lBVZQyRQw= -github.com/aws/aws-sdk-go-v2/service/iot v1.75.4 h1:2N2OuBTrqPP7l72vDOb1eohSicREa0oRcG5h6g4Gvik= -github.com/aws/aws-sdk-go-v2/service/iot v1.75.4/go.mod h1:LG1/n+K6upjPMVrrxTXYIwDj8OffwwIYwtAJ6/x/hjY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 h1:9Fjh6fi/U5JEStVZijmaMpUwE/gvBJj7x2B/PjbO9To= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23/go.mod h1:iMoT2f1tClxrWAAnKCXjZQ6LOmfLrMG14wmnWpM+F14= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.12.7 h1:uqsKxr7kJp9DXVj2m8KbVeZcYMuwsNEwvoVrYl2Vpf8= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.12.7/go.mod h1:Js/P8Zbwe1mRejnD+OpFLyQiJ8ioQlo3GMAg7Dfxk7w= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 h1:uao4A3QZ5UmB326V6KF+qRpv9Tjz7IlnlnTbbANntlU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31/go.mod h1:I/1+z0VwL1GhQyLgkoHDlygpUZ+iTAwOQ/NsftiUL2I= +github.com/aws/aws-sdk-go-v2/service/iot v1.75.6 h1:QfQlXkp43wBCXNTkLFdDiA2qeRnDIga1JOpKRHVq+so= +github.com/aws/aws-sdk-go-v2/service/iot v1.75.6/go.mod h1:BaRnYF1zU4CMN+b4ZQDhovaugNccFyPEH9VJXcwPXpg= github.com/aws/aws-sdk-go-v2/service/iotanalytics v1.32.0 h1:QHeG0bWIqSZ/Utkd7BoDjpcmR5NnBQhErZl9JNlG+Xs= github.com/aws/aws-sdk-go-v2/service/iotanalytics v1.32.0/go.mod h1:uqgp8z4czp3R86cSNls22vvFFjCkfHT2eR6EOZ+QgVU= github.com/aws/aws-sdk-go-v2/service/iotdataplane v1.32.20 h1:X+oiGtdyBaoeSZ/a/AIOsU6bQ5UK2olt+ZoA6hoy6D0= @@ -216,12 +216,12 @@ github.com/aws/aws-sdk-go-v2/service/kinesisanalytics v1.30.21 h1:lcWol7dmhRu/ce github.com/aws/aws-sdk-go-v2/service/kinesisanalytics v1.30.21/go.mod h1:XlvHJ13upQauM8bKpP8Q61SVRvCEb1wnnG+oksQr5l4= github.com/aws/aws-sdk-go-v2/service/kinesisanalyticsv2 v1.36.22 h1:2O+XgkYAQtbWlrrLtYdVqiu+C9lC6EOZC1QdhG+I2eY= github.com/aws/aws-sdk-go-v2/service/kinesisanalyticsv2 v1.36.22/go.mod h1:XIyMqeiX4yF+oBGWidb9o436tji+/nBuIbnhRlVFVv4= -github.com/aws/aws-sdk-go-v2/service/kms v1.53.4 h1:PEgVSsWtR8NNxsDxFL2Ywisi7R+1EFQARGsT4q3mWwI= -github.com/aws/aws-sdk-go-v2/service/kms v1.53.4/go.mod h1:3EeKyDGPGSCEphG2OolwNGNF45RvQIfm27AYYpfEWrw= +github.com/aws/aws-sdk-go-v2/service/kms v1.53.6 h1:udpfx3CG5T0eLYXiJMFdN7bBAXKPdb71A2cSDqlcw8c= +github.com/aws/aws-sdk-go-v2/service/kms v1.53.6/go.mod h1:0RXNc6Yf3AvSMldGD6Lcch96Ojlw2TtGnHsqfD/L4u8= github.com/aws/aws-sdk-go-v2/service/lakeformation v1.47.3 h1:882d+kB7+n6Y/zlyPf1ODdZ/0qUxEyLEwWdt1w5qbDI= github.com/aws/aws-sdk-go-v2/service/lakeformation v1.47.3/go.mod h1:Q9yCJ65TVUhtb+8RQj+zFn3Ko50ZQ1SiCY5OMob3hik= -github.com/aws/aws-sdk-go-v2/service/lambda v1.93.0 h1:uEB7hBZO61H63g+rtUbJ5fjkxLw369wukdr4hCtaZ+M= -github.com/aws/aws-sdk-go-v2/service/lambda v1.93.0/go.mod h1:3bF6WydfupDwCv8Q3g/Flt89341w/+NObn+KdQmLA60= +github.com/aws/aws-sdk-go-v2/service/lambda v1.94.1 h1:GLkCSQiEUNjCCDb39BuFVaMwfbwUr4kqYHk4PcJzpPY= +github.com/aws/aws-sdk-go-v2/service/lambda v1.94.1/go.mod h1:gKWVtxlMTgoLU9m6FDw7z6FAEFh8u8CoaPJx0zWk5J8= github.com/aws/aws-sdk-go-v2/service/macie2 v1.51.4 h1:POdAulSTqs30zz8AIL00MTaYYuryduVTW6cU+hwYQvI= github.com/aws/aws-sdk-go-v2/service/macie2 v1.51.4/go.mod h1:bOE9yKNh2MLwe8VwkrWxUckVz+nrize2dEsBjB6JlcQ= github.com/aws/aws-sdk-go-v2/service/managedblockchain v1.31.19 h1:PNpmmxmn2nOaLC79aBobNx6jC7O+Ty9yAhot62hNqN0= @@ -272,8 +272,8 @@ github.com/aws/aws-sdk-go-v2/service/rdsdata v1.32.19 h1:2W92mwEFGYBXGDegtsESoV/ github.com/aws/aws-sdk-go-v2/service/rdsdata v1.32.19/go.mod h1:5V7vGV1rmhIB7H4clYQSii2NgNhD+1E7Fj4Ad8O3eKI= github.com/aws/aws-sdk-go-v2/service/redshift v1.62.3 h1:kG21aB8DkKTRmN2oTQ+4tytibM9vIwW4vxQmtbCzs9E= github.com/aws/aws-sdk-go-v2/service/redshift v1.62.3/go.mod h1:oCsOzWYbbRFVlTYri72t9Pruq3EVOmVxhfBniqzZa7I= -github.com/aws/aws-sdk-go-v2/service/redshiftdata v1.40.6 h1:O/mkpKCNnoeHeWZm0dOuBf9Z3RHNXrJPCTACILSuO8s= -github.com/aws/aws-sdk-go-v2/service/redshiftdata v1.40.6/go.mod h1:EeLg9V4htAQckhuwcMCtcncB284f4g20K9e6AOZ0xTw= +github.com/aws/aws-sdk-go-v2/service/redshiftdata v1.40.8 h1:y+WYUlhoLCos11q65OaSUK8VJ0iv4etpr969uuFA+IQ= +github.com/aws/aws-sdk-go-v2/service/redshiftdata v1.40.8/go.mod h1:aD7N9sUFVjjOe6sPv9M2sqJVVl5subKQtKjFRp47lJ8= github.com/aws/aws-sdk-go-v2/service/rekognition v1.51.26 h1:/aqSj4fR8QDJnujCBnEwk6H+Pd9YSVkoJkm7VfbA8do= github.com/aws/aws-sdk-go-v2/service/rekognition v1.51.26/go.mod h1:pTgSKRkiNYddwdp01ZC9+KxFo8N5FlWgQQq5kIuuJhw= github.com/aws/aws-sdk-go-v2/service/resourcegroups v1.33.22 h1:jBDyp8U20i5DrLNSShSrraW4D0BGHMdzMdQlP+kTNqY= @@ -286,8 +286,8 @@ github.com/aws/aws-sdk-go-v2/service/route53 v1.62.3 h1:JRPXnIr0WwFsSHBmuCvT/uh0 github.com/aws/aws-sdk-go-v2/service/route53 v1.62.3/go.mod h1:DHddp7OO4bY467WVCqWBzk5+aEWn7vqYkap7UigJzGk= github.com/aws/aws-sdk-go-v2/service/route53resolver v1.42.3 h1:Cc5BYTPA7c0kSAEm+LOBuJ8wz4j0rSRKDtyvI9s/itY= github.com/aws/aws-sdk-go-v2/service/route53resolver v1.42.3/go.mod h1:OZYCRXBHcz3ngeHVZf7VjuaaEDzNnmaSJ6jTxFy1opI= -github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0 h1:ta8csKy5vN91F3i5gGR85lFV0srBqySEji7Jroes6rE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.104.0/go.mod h1:77ZAgynvx1txMvDG8gGWoWkO1augYDxkp9JElWFgjQU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.104.2 h1:bAY6O/TDv1HQnvylh9E247IyIKsUWUt2G965S7qX110= +github.com/aws/aws-sdk-go-v2/service/s3 v1.104.2/go.mod h1:zdmCoFO/dSI7GlrwsPqFJI+WlFnSU4Tc8TJnlXrM1Do= github.com/aws/aws-sdk-go-v2/service/s3control v1.68.2 h1:7qomMtVCsX8SEdkwFAPWScpFnmK7+9v+JAdR8rRRlKM= github.com/aws/aws-sdk-go-v2/service/s3control v1.68.2/go.mod h1:c7w32mk42MjoP6pQtLznPJZ2Q3lTakKXxcVlxX9MiAg= github.com/aws/aws-sdk-go-v2/service/s3tables v1.14.3 h1:jVYVKb0mALLNF93tikVjsN+SPeQ4r8DJLrisZF47N5k= @@ -298,8 +298,8 @@ github.com/aws/aws-sdk-go-v2/service/sagemakerruntime v1.39.3 h1:e1A452Wi9MDNdRr github.com/aws/aws-sdk-go-v2/service/sagemakerruntime v1.39.3/go.mod h1:VBzylHThc3Av74/3wc/nAHaBys6/ZlV58gsy1xZYUyc= github.com/aws/aws-sdk-go-v2/service/scheduler v1.17.20 h1:26W5e36ZRz5Lxag1aVBxv2ZHBPBMOTxxSVqPENob56Q= github.com/aws/aws-sdk-go-v2/service/scheduler v1.17.20/go.mod h1:AMTsHJXHLunksmCqfzhBLLyUQcxZ7d5zPbSIr839kbg= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.42.3 h1:L9gPLf3sFH1/ao3oB2QZcaX1xGYi8hj+WJlsf3/dN+M= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.42.3/go.mod h1:9DKRlwDCw2OUDlyCIFcQCroL5M0mQTUU9qW8JEDcXmI= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.42.5 h1:CWn32XhRKVJQlCpU5T1PpspYbgUisB+Ke1w7NR4AHZw= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.42.5/go.mod h1:oUyL28WfxY0RqPhFpkrWZx26Cu4JlyrWMMcWq8qqhi0= github.com/aws/aws-sdk-go-v2/service/securityhub v1.71.2 h1:ZvwbJ7eMf4dWm6z122VzIayd5+6aX4GSNbZFwLvsCWg= github.com/aws/aws-sdk-go-v2/service/securityhub v1.71.2/go.mod h1:tCssQ8pWlCxOWVu0Os4Ak9ffv1ZEZTv1oK+kzj9Dq9Q= github.com/aws/aws-sdk-go-v2/service/serverlessapplicationrepository v1.30.11 h1:ZxOz/H5G+s08cSfbWe9y0VS6gRDVeXCn6vwyuEMsTp8= @@ -314,22 +314,22 @@ github.com/aws/aws-sdk-go-v2/service/sfn v1.40.8 h1:n1VVa5CIJky6YeLUZuo/6hosyywE github.com/aws/aws-sdk-go-v2/service/sfn v1.40.8/go.mod h1:B2lwyEu+BHyl2V9NOgAid0KA6HXFfT0gV3NCPSVpqe8= github.com/aws/aws-sdk-go-v2/service/shield v1.34.20 h1:g/mGmna3irLYxab/UAgjJhun2quZmSboYJSUXot2RWc= github.com/aws/aws-sdk-go-v2/service/shield v1.34.20/go.mod h1:iOdjilB1oCiRS5l14MxI4RjL7utfCHVOhp4YjQ+M9pI= -github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 h1:3nXpRcFwRCW8n7HgO2QGy0Dc20eQNfBuUemGQhpF8m8= -github.com/aws/aws-sdk-go-v2/service/signin v1.2.0/go.mod h1:LxYujSTLPRlp2vTtcUO/+1ilrew8ytt6SvQyOgejzFQ= -github.com/aws/aws-sdk-go-v2/service/sns v1.40.1 h1:DLrOlgom0+OYnNaSiVxAXtR6obPjVlbmD+7w5wim9sc= -github.com/aws/aws-sdk-go-v2/service/sns v1.40.1/go.mod h1:V9szvM64GdG5VJUeDRstvLmt/ozgWiSNg3gYnp3mSkk= -github.com/aws/aws-sdk-go-v2/service/sqs v1.44.0 h1:6DQ95Zq5xPUSYm6KWcrar7zvreqAMFmyynYCcmzv6yc= -github.com/aws/aws-sdk-go-v2/service/sqs v1.44.0/go.mod h1:d7eKytKiwDFJeNAP4VWo47VCiNM9z539tkoCGJ6PjXw= -github.com/aws/aws-sdk-go-v2/service/ssm v1.69.3 h1:58LjP8cp8UEHA1LG/JZ4fG9SobHE82kLYe46mogbSI4= -github.com/aws/aws-sdk-go-v2/service/ssm v1.69.3/go.mod h1:16Zd02ocSJp68o4r36MQ4Rikf/Ulv4On5qjMpJJf5Mo= -github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 h1:ey1XLTYXb9PcLt4535632o5kCGXNXEhNb620Dqwuylo= -github.com/aws/aws-sdk-go-v2/service/sso v1.31.3/go.mod h1:Lk7PlmoTYryQmyBG0EXqj5BcUbj3whXdU2s3yGI3EAc= +github.com/aws/aws-sdk-go-v2/service/signin v1.2.2 h1:69JEZSDTQ+UNbTWQJCZMmbpQb5sfc79KUt0O7Pyfjmo= +github.com/aws/aws-sdk-go-v2/service/signin v1.2.2/go.mod h1:mxC0nT/C8wMMS97DemZPzvUZxvIt+2Iq+eS3JdFZGgg= +github.com/aws/aws-sdk-go-v2/service/sns v1.40.3 h1:ZgC0JhdV3xY7u0nt2Rg91NY6p3SiAzzV8U3Y10DoEEI= +github.com/aws/aws-sdk-go-v2/service/sns v1.40.3/go.mod h1:5EnTxMpMVeiY0vcjjN/a958FFaHrS6XfXcyRBzDKDCE= +github.com/aws/aws-sdk-go-v2/service/sqs v1.44.2 h1:1K+rgWUs/8B9y8Mux1y+tC79g+pVGt7Ozj9Fjs7cAbw= +github.com/aws/aws-sdk-go-v2/service/sqs v1.44.2/go.mod h1:JISE0m3JPVhirZEVIAUyK4C62n87tU4BZmUa9Ozc2to= +github.com/aws/aws-sdk-go-v2/service/ssm v1.69.5 h1:fCaC9yP7sthItdHF8VBT3qR42jK1R4g+5IoSvePys7o= +github.com/aws/aws-sdk-go-v2/service/ssm v1.69.5/go.mod h1:xabzRvdbMs3FG9kU5M6RUOuCW6wXDkpdIqoXXNzA1nQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.5 h1:xlK3Tdc8FO7Tq1k0+hL+otF33glj+dE+qeM5iINiDvU= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.5/go.mod h1:u8af9Nqkmqnr96f7v9nHqzZT9XBwbXEkTiqT4ROuJSE= github.com/aws/aws-sdk-go-v2/service/ssoadmin v1.38.0 h1:HAGNFXDUiJaHJPfcEOGuHZIoQDy7z3dUdBWEAWt69+o= github.com/aws/aws-sdk-go-v2/service/ssoadmin v1.38.0/go.mod h1:pGysVP4s+vO9KUV2n8dpmO/IfDflX0WOz4WKdHJWvQw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 h1:yLr03zQE/5Eu5l3QU0Si+xMbLMbSDF2YXsigqXngs6g= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6/go.mod h1:Q5N6icH+KJZDLh+ESNwzdv6cZ6vLFF/egy3IOxWhmz4= -github.com/aws/aws-sdk-go-v2/service/sts v1.43.3 h1:VrIhKRCSK1umelSgB9RghvA9RTUYeQffyAS5ApXehNI= -github.com/aws/aws-sdk-go-v2/service/sts v1.43.3/go.mod h1:r8wkDOuLaaMFqFiYAb8dGY2A3gJCOujMc6CFOVC4Zhc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.8 h1:yX1IbiBfC7SdEgDwIGnRaZyPPDRbQPDOJxl8102PcGk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.8/go.mod h1:DMPWJBjYs6+3+f/qhBFEFPPlQ6NlhWjai3dJNvipJ84= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.5 h1:T3ANO8QWDbzQD8f4+UaX+fvJlyGnOFMKLbW+NGBHg04= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.5/go.mod h1:9gdl4RrflIdpDb2TlXshWgR1F9TeCkvqDx77Vpr4Z/Q= github.com/aws/aws-sdk-go-v2/service/support v1.31.23 h1:ysC9+TojW1tnZxm3dEoQAqwVvqupUraWGc3l3LnWL7w= github.com/aws/aws-sdk-go-v2/service/support v1.31.23/go.mod h1:nGjCW7KSwHDN1N6wq+IfrgcHtKSW9OkoLn4mxQzd1Mc= github.com/aws/aws-sdk-go-v2/service/swf v1.33.14 h1:sVUF4SC5BzVw4tiOXX/yaa7GbruV9C/kirhINRDZUJc= @@ -492,8 +492,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -513,8 +513,8 @@ github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= @@ -577,8 +577,8 @@ github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVR github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.21.0 h1:Qh/e6TlBjZf+XLLqNCqFGmCU6Kj/2Bu7kj3oAc0UnXc= -github.com/prometheus/procfs v0.21.0/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/redis/go-redis/extra/rediscmd/v9 v9.21.0 h1:jsV3tyMeJrEoc2f3EhNf7qoBW3NEZW7l/4ziT3M+OJI= github.com/redis/go-redis/extra/rediscmd/v9 v9.21.0/go.mod h1:e5t17bY9cEpVV+xw2U7jsPOKkXBtL5IQmNVABShnHUk= github.com/redis/go-redis/extra/redisotel/v9 v9.21.0 h1:36qq3rbF2If2CP0zGHHF8o/4XDluErn6DD0c9/L2iNI= @@ -591,8 +591,8 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/shirou/gopsutil/v4 v4.26.5 h1:RPcBXkpz7kOj9PqGFQOlBPZHsyaPvPVQc098y9RmCNM= -github.com/shirou/gopsutil/v4 v4.26.5/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= +github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= @@ -611,8 +611,8 @@ github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRU github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= -github.com/vektah/gqlparser/v2 v2.5.35 h1:LEr/wXnTKkOqNn+4tNClYclksXN2781VoBFzzFW51Dk= -github.com/vektah/gqlparser/v2 v2.5.35/go.mod h1:cAJ9qwVgPaUkWv6Gn8vn0mqOE0Ui5Pn56wNy5396XWo= +github.com/vektah/gqlparser/v2 v2.5.36 h1:CN9mKVHgMkc+XftdOWIhb4HEL8wKSYkFAqhf8booa7s= +github.com/vektah/gqlparser/v2 v2.5.36/go.mod h1:cAJ9qwVgPaUkWv6Gn8vn0mqOE0Ui5Pn56wNy5396XWo= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA= github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8= @@ -720,7 +720,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= @@ -757,12 +756,12 @@ golang.org/x/vuln v1.1.4/go.mod h1:F+45wmU18ym/ca5PLTPLsSzr2KppzswxPP603ldA67s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260622175928-b703f567277d h1:xr2lwHI91bn3UiXcnyzRMQjp2LRiM8wEHzwUaE0YhTs= -google.golang.org/genproto/googleapis/api v0.0.0-20260622175928-b703f567277d/go.mod h1:O0ZOWSrfWfJ+Z5HbwZ+wNtHsg/vk1k2C/w67eww8PfQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260622175928-b703f567277d h1:mpAgMyM9vQHxycBlDq50y1VHpfSfVwzXvrQKtYbXuUY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260622175928-b703f567277d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= -google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 h1:eM/YSd5bBFagF51o1E745Ta7RwzpW0h+z+QDNZOgmQ8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= +google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -780,20 +779,20 @@ gotest.tools/gotestsum v1.13.0 h1:+Lh454O9mu9AMG1APV4o0y7oDYKyik/3kBOiCqiEpRo= gotest.tools/gotestsum v1.13.0/go.mod h1:7f0NS5hFb0dWr4NtcsAsF0y1kzjEFfAil0HiBQJE03Q= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= -modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= -modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= -modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE= +modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= +modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.5 h1:hcwnthv2/LBl+mRLOYwnQA/LuW44Oln1NQlWppNaS1Q= +modernc.org/ccgo/v4 v4.34.5/go.mod h1:aow0HNkO30OSA/2NrtDXkis92ff8ZFiDOmDOPhqhF8U= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc= -modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= -modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= +modernc.org/libc v1.73.5 h1:G34rN/cRqL+zOUnrbz9uPq/+OxJ8/vzQ2CQwTJ42Wmw= +modernc.org/libc v1.73.5/go.mod h1:+Aoyx4M0etg6GikzCrip1VtvAtUlMlo2Aq+GHwQSqOA= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= diff --git a/pkgs/store/index.go b/pkgs/store/index.go new file mode 100644 index 000000000..a9137d3db --- /dev/null +++ b/pkgs/store/index.go @@ -0,0 +1,108 @@ +package store + +// Index is an opt-in secondary index over a [Table]'s values, grouping them by +// a caller-supplied key function. It exists for real filter hot paths — e.g. +// "all SQS messages in queue X" — where a linear scan of [Table.All] would +// otherwise be required on every call. +// +// The zero value is not usable; always create via [Table.AddIndex]. +// +// Like [Table], Index performs no locking of its own; it is maintained +// in-line by [Table.Put], [Table.Delete], and [Table.Restore] under whatever +// external lock the owning backend already holds. +type Index[V any] struct { + groups map[string][]*V + keyFn func(*V) string + name string +} + +// AddIndex creates and registers a secondary [Index] on t, keyed by keyFn. +// name identifies the index for debugging/error messages (e.g. "queue"); it +// plays no role in lookup. The index is populated immediately from every +// value already in the table, then kept consistent by [Table.Put], +// [Table.Delete], and [Table.Restore] for the rest of the table's lifetime. +// +// A [Table] with no indexes pays no cost for this feature: [Table.Put] and +// [Table.Delete] range over t.indexes, which is a nil slice until AddIndex is +// first called, so the loop is zero iterations and zero allocations. +// +// keyFn must be a pure function of the value's current field(s); if a value +// is mutated in place through a pointer obtained from [Table.Get] such that +// keyFn's result would change, the index becomes stale for that value until +// the caller re-inserts it with [Table.Put]. This is the same caveat every +// secondary index carries (relational or otherwise) and is not specific to +// this package. +func (t *Table[V]) AddIndex(name string, keyFn func(*V) string) *Index[V] { + idx := &Index[V]{ + groups: make(map[string][]*V), + keyFn: keyFn, + name: name, + } + + for _, v := range t.data { + idx.add(v) + } + + t.indexes = append(t.indexes, idx) + + return idx +} + +// Name returns the identifier passed to [Table.AddIndex] when this index was +// created. +func (idx *Index[V]) Name() string { + return idx.name +} + +// Get returns the values currently grouped under key. Iteration order within +// the group is insertion order, not any table-defined order. The returned +// slice is owned by the index — the caller must not mutate it, and must not +// retain it across a subsequent [Table.Put]/[Table.Delete]/[Table.Restore] +// call, which may reuse or invalidate it. Copy the slice first if you need to +// hold onto it past the current lock scope. +func (idx *Index[V]) Get(key string) []*V { + return idx.groups[key] +} + +// Len returns the number of distinct group keys currently populated. +func (idx *Index[V]) Len() int { + return len(idx.groups) +} + +// add inserts v into its group, deriving the group key from the value's +// current state. +func (idx *Index[V]) add(v *V) { + key := idx.keyFn(v) + idx.groups[key] = append(idx.groups[key], v) +} + +// remove drops v from its group by pointer identity — always well-defined +// since *V is comparable regardless of whether V itself is. When the group +// becomes empty its map entry is deleted outright rather than left as a +// zero-length slice, so a table with high key churn does not accumulate +// empty index buckets. +func (idx *Index[V]) remove(v *V) { + key := idx.keyFn(v) + + group := idx.groups[key] + for i, item := range group { + if item == v { + group[i] = group[len(group)-1] + group = group[:len(group)-1] + + break + } + } + + if len(group) == 0 { + delete(idx.groups, key) + } else { + idx.groups[key] = group + } +} + +// clear empties every group without reallocating the backing map, so it is +// ready for immediate reuse by [Table.reset]/[Table.Restore]. +func (idx *Index[V]) clear() { + clear(idx.groups) +} diff --git a/pkgs/store/registry.go b/pkgs/store/registry.go new file mode 100644 index 000000000..bb6eeeff1 --- /dev/null +++ b/pkgs/store/registry.go @@ -0,0 +1,103 @@ +package store + +import ( + "encoding/json" + "fmt" +) + +// Registry is a lifecycle registry for a backend's [Table]s. A backend +// registers each of its tables once at construction time (via [Register]); +// afterward the whole backend's lifecycle — reset, snapshot, restore — +// collapses to one [Registry] call each, regardless of how many differently +// typed tables the backend owns. This is what eliminates the +// per-map Init/Reset/Snapshot/Restore boilerplate described in the package doc. +// +// Registry performs no locking of its own, matching [Table] and [Index]: it +// is meant to be driven from within the backend's own coarse lock. +// +// The zero value is not usable; always create via [NewRegistry]. +type Registry struct { + tables map[string]tableSnapshotter +} + +// NewRegistry creates an empty [Registry]. +func NewRegistry() *Registry { + return &Registry{tables: make(map[string]tableSnapshotter)} +} + +// Register adds t to r under name and returns t unchanged, so it can be used +// inline in a field initializer: +// +// b.queues = store.Register(b.registry, "queues", store.New(queueKeyFn)) +// +// Register is a free function rather than a [Registry] method because Go does +// not allow a generic method (Table's value type V) on a non-generic receiver +// (Registry); the erasure this requires — storing *Table[V] behind the +// unexported tableSnapshotter interface — happens entirely inside this +// function. Every call site remains fully typed: t keeps its concrete +// *Table[V] type both before and after this call, so no interface{}/any is +// ever visible to the caller. +// +// name must be unique within r; registering a second table under a name +// already in use panics, since that always indicates a construction-time bug +// (e.g. a copy-pasted registration) rather than a runtime condition a backend +// could reasonably recover from. +func Register[V any](r *Registry, name string, t *Table[V]) *Table[V] { + if _, exists := r.tables[name]; exists { + panic(fmt.Sprintf("store: table %q already registered", name)) + } + + r.tables[name] = t + + return t +} + +// ResetAll clears every registered table (and every index on it), returning +// each to the empty state it was in immediately after [New]. +func (r *Registry) ResetAll() { + for _, t := range r.tables { + t.reset() + } +} + +// SnapshotAll captures every registered table's contents as JSON, keyed by the +// name each was registered under. Each table's own encoding is already +// deterministic (see [Table.Snapshot]); marshaling a Go map with string keys +// additionally sorts those keys, so the overall result — and any JSON built +// from it — is byte-for-byte stable across calls for the same backend state. +func (r *Registry) SnapshotAll() (map[string]json.RawMessage, error) { + out := make(map[string]json.RawMessage, len(r.tables)) + + for name, t := range r.tables { + data, err := t.snapshotJSON() + if err != nil { + return nil, fmt.Errorf("store: snapshot table %q: %w", name, err) + } + + out[name] = data + } + + return out, nil +} + +// RestoreAll loads every registered table from data, keyed the same way +// [Registry.SnapshotAll] produced it. A registered table whose name is absent +// from data is reset to empty rather than left untouched, so RestoreAll always +// produces the same backend state a fresh [NewRegistry] plus SnapshotAll's +// input would — never a stale mix of old and new state. +func (r *Registry) RestoreAll(data map[string]json.RawMessage) error { + for name, t := range r.tables { + raw, ok := data[name] + if !ok { + t.reset() + + continue + } + + if err := t.restoreJSON(raw); err != nil { + return fmt.Errorf("store: restore table %q: %w", name, err) + } + } + + return nil +} diff --git a/pkgs/store/store_bench_test.go b/pkgs/store/store_bench_test.go new file mode 100644 index 000000000..4b9d57a48 --- /dev/null +++ b/pkgs/store/store_bench_test.go @@ -0,0 +1,267 @@ +package store_test + +import ( + "fmt" + "testing" + + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +// lookupTableSize is the fixed table size used by the Get/Put micro-benchmarks +// below (both the raw-map baseline and the Table[V] variant), chosen large +// enough that key rotation exercises real hashing/bucket behaviour rather +// than fitting entirely in a couple of cache lines. +const lookupTableSize = 1000 + +// buildBenchKeys returns n deterministic, distinct string keys. +func buildBenchKeys(n int) []string { + keys := make([]string, n) + for i := range n { + keys[i] = fmt.Sprintf("key-%06d", i) + } + + return keys +} + +// --- Get ------------------------------------------------------------------- + +func BenchmarkRawMapGet(b *testing.B) { + keys := buildBenchKeys(lookupTableSize) + m := make(map[string]*widget, lookupTableSize) + + for i, k := range keys { + m[k] = &widget{ID: k, Count: i} + } + + b.ReportAllocs() + b.ResetTimer() + + for i := range b.N { + _ = m[keys[i%lookupTableSize]] + } +} + +func BenchmarkTableGet(b *testing.B) { + keys := buildBenchKeys(lookupTableSize) + tbl := store.New(widgetKey) + + for i, k := range keys { + tbl.Put(&widget{ID: k, Count: i}) + } + + b.ReportAllocs() + b.ResetTimer() + + for i := range b.N { + _, _ = tbl.Get(keys[i%lookupTableSize]) + } +} + +// --- Put (steady-state overwrite of an existing key) ----------------------- + +func BenchmarkRawMapPut(b *testing.B) { + keys := buildBenchKeys(lookupTableSize) + m := make(map[string]*widget, lookupTableSize) + + for i, k := range keys { + m[k] = &widget{ID: k, Count: i} + } + + b.ReportAllocs() + b.ResetTimer() + + for i := range b.N { + k := keys[i%lookupTableSize] + m[k] = &widget{ID: k, Count: i} + } +} + +func BenchmarkTablePut(b *testing.B) { + keys := buildBenchKeys(lookupTableSize) + tbl := store.New(widgetKey) + + for i, k := range keys { + tbl.Put(&widget{ID: k, Count: i}) + } + + b.ReportAllocs() + b.ResetTimer() + + for i := range b.N { + k := keys[i%lookupTableSize] + tbl.Put(&widget{ID: k, Count: i}) + } +} + +// --- Delete (each op deletes a distinct, never-before-deleted key) --------- + +func BenchmarkRawMapDelete(b *testing.B) { + keys := buildBenchKeys(b.N) + m := make(map[string]*widget, b.N) + + for i, k := range keys { + m[k] = &widget{ID: k, Count: i} + } + + b.ReportAllocs() + b.ResetTimer() + + for _, k := range keys { + delete(m, k) + } +} + +func BenchmarkTableDelete(b *testing.B) { + keys := buildBenchKeys(b.N) + tbl := store.New(widgetKey) + + for i, k := range keys { + tbl.Put(&widget{ID: k, Count: i}) + } + + b.ReportAllocs() + b.ResetTimer() + + for _, k := range keys { + tbl.Delete(k) + } +} + +// --- Range over 10k entries ------------------------------------------------- + +const rangeTableSize = 10_000 + +func BenchmarkRawMapRangeAll(b *testing.B) { + keys := buildBenchKeys(rangeTableSize) + m := make(map[string]*widget, rangeTableSize) + + for i, k := range keys { + m[k] = &widget{ID: k, Count: i} + } + + b.ReportAllocs() + b.ResetTimer() + + for range b.N { + sum := 0 + for _, v := range m { + sum += v.Count + } + + if sum < 0 { + b.Fatal("impossible negative sum") + } + } +} + +func BenchmarkTableRange(b *testing.B) { + keys := buildBenchKeys(rangeTableSize) + tbl := store.New(widgetKey) + + for i, k := range keys { + tbl.Put(&widget{ID: k, Count: i}) + } + + b.ReportAllocs() + b.ResetTimer() + + for range b.N { + sum := 0 + tbl.Range(func(w *widget) bool { + sum += w.Count + + return true + }) + + if sum < 0 { + b.Fatal("impossible negative sum") + } + } +} + +// --- Snapshot/Restore round trip over 10k entries --------------------------- + +const snapshotTableSize = 10_000 + +func BenchmarkSnapshotRestore(b *testing.B) { + keys := buildBenchKeys(snapshotTableSize) + src := store.New(widgetKey) + + for i, k := range keys { + src.Put(&widget{ID: k, Group: k, Count: i}) + } + + b.ReportAllocs() + b.ResetTimer() + + for range b.N { + snap := src.Snapshot() + dst := store.New(widgetKey) + dst.Restore(snap) + } +} + +// --- Indexed lookup vs. linear scan over All (100 groups x 1k items) ------- + +const ( + indexGroupCount = 100 + indexItemsPerGroup = 1_000 +) + +// buildIndexedTable returns a table of indexGroupCount*indexItemsPerGroup +// widgets evenly spread across indexGroupCount distinct "group" values, plus +// the secondary index over that field and the list of group keys in order. +func buildIndexedTable() (*store.Table[widget], *store.Index[widget], []string) { + tbl := store.New(widgetKey) + idx := tbl.AddIndex("group", func(w *widget) string { return w.Group }) + + groupKeys := make([]string, indexGroupCount) + for g := range indexGroupCount { + groupKeys[g] = fmt.Sprintf("group-%03d", g) + } + + for g, gk := range groupKeys { + for i := range indexItemsPerGroup { + tbl.Put(&widget{ID: fmt.Sprintf("g%03d-i%04d", g, i), Group: gk, Count: i}) + } + } + + return tbl, idx, groupKeys +} + +func BenchmarkIndexedLookup(b *testing.B) { + _, idx, groupKeys := buildIndexedTable() + + b.ReportAllocs() + b.ResetTimer() + + for i := range b.N { + got := idx.Get(groupKeys[i%indexGroupCount]) + if len(got) != indexItemsPerGroup { + b.Fatalf("expected %d members, got %d", indexItemsPerGroup, len(got)) + } + } +} + +func BenchmarkLinearScanLookup(b *testing.B) { + tbl, _, groupKeys := buildIndexedTable() + + b.ReportAllocs() + b.ResetTimer() + + for i := range b.N { + want := groupKeys[i%indexGroupCount] + + matches := make([]*widget, 0, indexItemsPerGroup) + + for _, w := range tbl.All() { + if w.Group == want { + matches = append(matches, w) + } + } + + if len(matches) != indexItemsPerGroup { + b.Fatalf("expected %d members, got %d", indexItemsPerGroup, len(matches)) + } + } +} diff --git a/pkgs/store/store_test.go b/pkgs/store/store_test.go new file mode 100644 index 000000000..a68f864d0 --- /dev/null +++ b/pkgs/store/store_test.go @@ -0,0 +1,545 @@ +package store_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +// widget is a small value type used throughout the table-driven tests below. +type widget struct { + ID string `json:"id"` + Group string `json:"group"` + Count int `json:"count"` +} + +func widgetKey(w *widget) string { return w.ID } + +func newWidgetTable(items ...*widget) *store.Table[widget] { + tbl := store.New(widgetKey) + for _, w := range items { + tbl.Put(w) + } + + return tbl +} + +func Test_TablePutGet(t *testing.T) { + t.Parallel() + + tests := []struct { + wantSelf *widget + name string + lookup string + seed []*widget + wantOK bool + }{ + { + name: "found", + seed: []*widget{{ID: "a", Count: 1}}, + lookup: "a", + wantOK: true, + wantSelf: &widget{ID: "a", Count: 1}, + }, + { + name: "missing_from_empty", + seed: nil, + lookup: "a", + wantOK: false, + }, + { + name: "missing_wrong_key", + seed: []*widget{{ID: "a", Count: 1}}, + lookup: "b", + wantOK: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + tbl := newWidgetTable(tt.seed...) + + got, ok := tbl.Get(tt.lookup) + require.Equal(t, tt.wantOK, ok) + + if tt.wantOK { + assert.Equal(t, tt.wantSelf.ID, got.ID) + assert.Equal(t, tt.wantSelf.Count, got.Count) + } else { + assert.Nil(t, got) + } + }) + } +} + +func Test_TablePutOverwrite(t *testing.T) { + t.Parallel() + + tbl := newWidgetTable(&widget{ID: "a", Count: 1}) + tbl.Put(&widget{ID: "a", Count: 2}) + + got, ok := tbl.Get("a") + require.True(t, ok) + assert.Equal(t, 2, got.Count) + assert.Equal(t, 1, tbl.Len()) +} + +func Test_TablePutNilIsNoOp(t *testing.T) { + t.Parallel() + + tbl := store.New(widgetKey) + + assert.NotPanics(t, func() { tbl.Put(nil) }) + assert.Equal(t, 0, tbl.Len()) +} + +func Test_TableHas(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + lookup string + want bool + }{ + {name: "present", lookup: "a", want: true}, + {name: "absent", lookup: "z", want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + tbl := newWidgetTable(&widget{ID: "a"}) + assert.Equal(t, tt.want, tbl.Has(tt.lookup)) + }) + } +} + +func Test_TableDelete(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + lookup string + wantExists bool + }{ + {name: "existing", lookup: "a", wantExists: true}, + {name: "missing", lookup: "z", wantExists: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + tbl := newWidgetTable(&widget{ID: "a"}) + + got := tbl.Delete(tt.lookup) + require.Equal(t, tt.wantExists, got) + + if tt.wantExists { + assert.Equal(t, 0, tbl.Len()) + assert.False(t, tbl.Has(tt.lookup)) + } + }) + } +} + +func Test_TableLen(t *testing.T) { + t.Parallel() + + tbl := store.New(widgetKey) + assert.Equal(t, 0, tbl.Len()) + + tbl.Put(&widget{ID: "a"}) + assert.Equal(t, 1, tbl.Len()) + + tbl.Put(&widget{ID: "b"}) + assert.Equal(t, 2, tbl.Len()) + + tbl.Delete("a") + assert.Equal(t, 1, tbl.Len()) +} + +func Test_TableAll(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + seed []*widget + want []string + }{ + {name: "empty", seed: nil, want: []string{}}, + { + name: "multiple", + seed: []*widget{{ID: "a"}, {ID: "b"}, {ID: "c"}}, + want: []string{"a", "b", "c"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + tbl := newWidgetTable(tt.seed...) + got := idsOf(tbl.All()) + assert.ElementsMatch(t, tt.want, got) + }) + } +} + +func Test_TableRange(t *testing.T) { + t.Parallel() + + t.Run("visits_every_entry", func(t *testing.T) { + t.Parallel() + + tbl := newWidgetTable(&widget{ID: "a"}, &widget{ID: "b"}, &widget{ID: "c"}) + + seen := make(map[string]bool) + tbl.Range(func(w *widget) bool { + seen[w.ID] = true + + return true + }) + + assert.Len(t, seen, 3) + }) + + t.Run("early_stop", func(t *testing.T) { + t.Parallel() + + tbl := newWidgetTable(&widget{ID: "a"}, &widget{ID: "b"}, &widget{ID: "c"}) + + count := 0 + tbl.Range(func(_ *widget) bool { + count++ + + return false + }) + + assert.Equal(t, 1, count) + }) + + t.Run("empty_table", func(t *testing.T) { + t.Parallel() + + tbl := store.New(widgetKey) + + called := false + tbl.Range(func(_ *widget) bool { + called = true + + return true + }) + + assert.False(t, called) + }) +} + +func Test_TableReset(t *testing.T) { + t.Parallel() + + tbl := newWidgetTable(&widget{ID: "a"}, &widget{ID: "b"}) + idx := tbl.AddIndex("group", func(w *widget) string { return w.Group }) + + tbl.Reset() + + assert.Equal(t, 0, tbl.Len()) + assert.Empty(t, tbl.All()) + assert.Equal(t, 0, idx.Len()) +} + +func Test_TableSnapshotDeterministicOrder(t *testing.T) { + t.Parallel() + + tbl := newWidgetTable(&widget{ID: "c"}, &widget{ID: "a"}, &widget{ID: "b"}) + + snap := tbl.Snapshot() + require.Len(t, snap, 3) + assert.Equal(t, []string{"a", "b", "c"}, idsOf(snap)) + + // Repeated snapshots of unchanged state must produce identical order. + again := tbl.Snapshot() + assert.Equal(t, idsOf(snap), idsOf(again)) +} + +func Test_TableSnapshotRestoreRoundTrip(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + seed []*widget + }{ + {name: "empty", seed: nil}, + { + name: "populated", + seed: []*widget{ + {ID: "a", Group: "g1", Count: 1}, + {ID: "b", Group: "g2", Count: 2}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + src := newWidgetTable(tt.seed...) + snap := src.Snapshot() + + dst := store.New(widgetKey) + dst.Restore(snap) + + assert.Equal(t, src.Len(), dst.Len()) + assert.Equal(t, idsOf(src.Snapshot()), idsOf(dst.Snapshot())) + + for _, w := range tt.seed { + got, ok := dst.Get(w.ID) + require.True(t, ok) + assert.Equal(t, w.Group, got.Group) + assert.Equal(t, w.Count, got.Count) + } + }) + } +} + +func Test_TableRestoreReplacesExistingContents(t *testing.T) { + t.Parallel() + + tbl := newWidgetTable(&widget{ID: "old"}) + tbl.Restore([]*widget{{ID: "new"}}) + + assert.False(t, tbl.Has("old")) + assert.True(t, tbl.Has("new")) + assert.Equal(t, 1, tbl.Len()) +} + +func Test_TableRestoreNilIsEmpty(t *testing.T) { + t.Parallel() + + tbl := newWidgetTable(&widget{ID: "a"}) + tbl.Restore(nil) + + assert.Equal(t, 0, tbl.Len()) +} + +func Test_TableSnapshotJSONRoundTrip(t *testing.T) { + t.Parallel() + + src := newWidgetTable(&widget{ID: "a", Group: "g", Count: 1}) + + data, err := json.Marshal(src.Snapshot()) + require.NoError(t, err) + + var items []*widget + require.NoError(t, json.Unmarshal(data, &items)) + + dst := store.New(widgetKey) + dst.Restore(items) + + got, ok := dst.Get("a") + require.True(t, ok) + assert.Equal(t, "g", got.Group) + assert.Equal(t, 1, got.Count) +} + +func Test_IndexConsistencyAcrossPut(t *testing.T) { + t.Parallel() + + tbl := store.New(widgetKey) + idx := tbl.AddIndex("group", func(w *widget) string { return w.Group }) + + tbl.Put(&widget{ID: "a", Group: "g1"}) + tbl.Put(&widget{ID: "b", Group: "g1"}) + tbl.Put(&widget{ID: "c", Group: "g2"}) + + assert.ElementsMatch(t, []string{"a", "b"}, idsOf(idx.Get("g1"))) + assert.ElementsMatch(t, []string{"c"}, idsOf(idx.Get("g2"))) + assert.Empty(t, idx.Get("no-such-group")) + + // Moving "a" from g1 to g2 via Put must relocate it in the index. + tbl.Put(&widget{ID: "a", Group: "g2"}) + + assert.ElementsMatch(t, []string{"b"}, idsOf(idx.Get("g1"))) + assert.ElementsMatch(t, []string{"a", "c"}, idsOf(idx.Get("g2"))) +} + +func Test_IndexConsistencyAcrossDelete(t *testing.T) { + t.Parallel() + + tbl := store.New(widgetKey) + idx := tbl.AddIndex("group", func(w *widget) string { return w.Group }) + + tbl.Put(&widget{ID: "a", Group: "g1"}) + tbl.Put(&widget{ID: "b", Group: "g1"}) + + require.True(t, tbl.Delete("a")) + + assert.ElementsMatch(t, []string{"b"}, idsOf(idx.Get("g1"))) + + require.True(t, tbl.Delete("b")) + + // The group must be gone entirely, not left as an empty slice, once the + // last member is removed. + assert.Equal(t, 0, idx.Len()) + assert.Empty(t, idx.Get("g1")) +} + +func Test_IndexConsistencyAcrossRestore(t *testing.T) { + t.Parallel() + + tbl := store.New(widgetKey) + idx := tbl.AddIndex("group", func(w *widget) string { return w.Group }) + + tbl.Put(&widget{ID: "a", Group: "g1"}) + tbl.Put(&widget{ID: "b", Group: "g2"}) + + tbl.Restore([]*widget{ + {ID: "c", Group: "g3"}, + {ID: "d", Group: "g3"}, + }) + + assert.Empty(t, idx.Get("g1")) + assert.Empty(t, idx.Get("g2")) + assert.ElementsMatch(t, []string{"c", "d"}, idsOf(idx.Get("g3"))) +} + +func Test_IndexAddedAfterDataAlreadyPresent(t *testing.T) { + t.Parallel() + + tbl := newWidgetTable(&widget{ID: "a", Group: "g1"}, &widget{ID: "b", Group: "g1"}) + idx := tbl.AddIndex("group", func(w *widget) string { return w.Group }) + + assert.ElementsMatch(t, []string{"a", "b"}, idsOf(idx.Get("g1"))) +} + +func Test_IndexName(t *testing.T) { + t.Parallel() + + tbl := store.New(widgetKey) + idx := tbl.AddIndex("group", func(w *widget) string { return w.Group }) + + assert.Equal(t, "group", idx.Name()) +} + +// gadget is a second, differently-shaped value type used to prove the +// Registry can hold heterogeneous tables consistently. +type gadget struct { + SKU string `json:"sku"` + Price int `json:"price"` +} + +func gadgetKey(g *gadget) string { return g.SKU } + +func Test_RegistryResetAll(t *testing.T) { + t.Parallel() + + reg := store.NewRegistry() + widgets := store.Register(reg, "widgets", newWidgetTable(&widget{ID: "a"})) + gadgets := store.Register(reg, "gadgets", store.New(gadgetKey)) + gadgets.Put(&gadget{SKU: "g1", Price: 100}) + + reg.ResetAll() + + assert.Equal(t, 0, widgets.Len()) + assert.Equal(t, 0, gadgets.Len()) +} + +func Test_RegistrySnapshotRestoreAllRoundTrip(t *testing.T) { + t.Parallel() + + reg := store.NewRegistry() + widgets := store.Register(reg, "widgets", store.New(widgetKey)) + gadgets := store.Register(reg, "gadgets", store.New(gadgetKey)) + + widgets.Put(&widget{ID: "a", Group: "g1", Count: 1}) + widgets.Put(&widget{ID: "b", Group: "g2", Count: 2}) + gadgets.Put(&gadget{SKU: "sku-1", Price: 100}) + + snap, err := reg.SnapshotAll() + require.NoError(t, err) + require.Contains(t, snap, "widgets") + require.Contains(t, snap, "gadgets") + + // Simulate a fresh backend restart: new registry, new (empty) tables + // registered under the same names, then restore from the captured bytes. + reg2 := store.NewRegistry() + widgets2 := store.Register(reg2, "widgets", store.New(widgetKey)) + gadgets2 := store.Register(reg2, "gadgets", store.New(gadgetKey)) + + require.NoError(t, reg2.RestoreAll(snap)) + + assert.Equal(t, 2, widgets2.Len()) + assert.Equal(t, 1, gadgets2.Len()) + + w, ok := widgets2.Get("a") + require.True(t, ok) + assert.Equal(t, "g1", w.Group) + assert.Equal(t, 1, w.Count) + + g, ok := gadgets2.Get("sku-1") + require.True(t, ok) + assert.Equal(t, 100, g.Price) +} + +func Test_RegistrySnapshotIsDeterministic(t *testing.T) { + t.Parallel() + + build := func() *store.Registry { + reg := store.NewRegistry() + widgets := store.Register(reg, "widgets", store.New(widgetKey)) + widgets.Put(&widget{ID: "z"}) + widgets.Put(&widget{ID: "a"}) + widgets.Put(&widget{ID: "m"}) + + return reg + } + + snap1, err := build().SnapshotAll() + require.NoError(t, err) + snap2, err := build().SnapshotAll() + require.NoError(t, err) + + assert.Equal(t, snap1["widgets"], snap2["widgets"]) +} + +func Test_RegistryRestoreAllResetsTablesMissingFromData(t *testing.T) { + t.Parallel() + + reg := store.NewRegistry() + widgets := store.Register(reg, "widgets", store.New(widgetKey)) + widgets.Put(&widget{ID: "stale"}) + + // data intentionally omits "widgets" entirely. + require.NoError(t, reg.RestoreAll(map[string]json.RawMessage{})) + + assert.Equal(t, 0, widgets.Len()) +} + +func Test_RegisterDuplicateNamePanics(t *testing.T) { + t.Parallel() + + reg := store.NewRegistry() + store.Register(reg, "widgets", store.New(widgetKey)) + + assert.Panics(t, func() { + store.Register(reg, "widgets", store.New(widgetKey)) + }) +} + +// idsOf extracts the ID field from a slice of *widget for order-insensitive +// comparisons in tests above. +func idsOf(ws []*widget) []string { + out := make([]string, len(ws)) + for i, w := range ws { + out[i] = w.ID + } + + return out +} diff --git a/pkgs/store/table.go b/pkgs/store/table.go new file mode 100644 index 000000000..0d3f3112d --- /dev/null +++ b/pkgs/store/table.go @@ -0,0 +1,272 @@ +// Package store provides a generic, type-safe keyed collection (Table[V]) that +// replaces the per-backend map + Init/Reset/Snapshot/Restore boilerplate that +// every service.InMemoryBackend hand-rolls today (EC2 alone repeats it for +// ~180 maps). It keeps the existing O(1) partitioned-map design services +// already use — this package does not introduce a database, a memdb, or a +// pointer graph; those were evaluated and rejected (14-79x slower, and a +// referential-integrity burden respectively). It only collapses the +// boilerplate around a plain map[string]*V into one generic type. +// +// # Locking +// +// [Table] and [Index] perform NO internal locking. This is deliberate and +// documented loudly: gopherstack's locking rule is that lock granularity +// follows INVARIANT granularity, not data-structure granularity (see +// .claude/memories/pkgs-catalog.md). A service backend's operations are +// cross-map transactions (create validates foreign keys + writes a resource + +// updates secondary indexes atomically; Snapshot needs one consistent view), +// so the single coarse [github.com/blackbirdworks/gopherstack/pkgs/lockmetrics.RWMutex] +// already held by the owning backend is the correct lock boundary. A [Table] +// is meant to be a field inside that backend, guarded by the backend's +// existing mutex exactly as the raw map it replaces was. Using a [Table] (or +// [Index]) concurrently without an external lock is a data race, just as it +// would be with a bare map. +// +// # Typical usage +// +// A backend declares one Table per resource collection and registers each one +// with a [Registry] exactly once at construction time: +// +// type Queue struct { +// Name string +// // ... +// } +// +// type InMemoryBackend struct { +// mu *lockmetrics.RWMutex +// queues *store.Table[Queue] +// registry *store.Registry +// } +// +// func NewInMemoryBackend() *InMemoryBackend { +// b := &InMemoryBackend{ +// mu: lockmetrics.New("sqs"), +// registry: store.NewRegistry(), +// } +// b.queues = store.Register(b.registry, "queues", store.New(func(q *Queue) string { return q.Name })) +// +// return b +// } +// +// Every backend operation still takes b.mu itself (store performs no locking), +// but Reset/Snapshot/Restore across every registered table — regardless of how +// many different value types they hold — collapse to one [Registry] call +// instead of one hand-written block per map. +package store + +import ( + "encoding/json" + "fmt" + + "github.com/blackbirdworks/gopherstack/pkgs/collections" +) + +// Table is a generic keyed collection over map[string]*V. The string key is +// the universal primary-key shape for AWS resources (names, ARNs, IDs, ...). +// +// The zero value is not usable; always create via [New]. +// +// Table performs no locking of its own — see the package doc for why. Every +// method below assumes the caller (typically a service backend) already holds +// whatever lock protects the surrounding invariant. +type Table[V any] struct { + data map[string]*V + keyFn func(*V) string + indexes []*Index[V] +} + +// New creates an empty [Table]. keyFn extracts the primary key from a value; +// it is called on every [Table.Put] and [Table.Restore] to place the value in +// the underlying map, so it must be a pure function of the value's identity +// (e.g. a resource name or ARN) and must not change across the value's +// lifetime — see [Table.AddIndex] for the same rule applied to secondary keys. +func New[V any](keyFn func(*V) string) *Table[V] { + return &Table[V]{ + data: make(map[string]*V), + keyFn: keyFn, + } +} + +// Get returns the value stored under id and whether it was found. +func (t *Table[V]) Get(id string) (*V, bool) { + v, ok := t.data[id] + + return v, ok +} + +// Has reports whether id is present in the table. +func (t *Table[V]) Has(id string) bool { + _, ok := t.data[id] + + return ok +} + +// Len returns the number of entries in the table. +func (t *Table[V]) Len() int { + return len(t.data) +} + +// Put inserts or replaces the value under the key derived from v via the +// table's keyFn. A nil v is a documented no-op (nil-safe): there is no +// well-formed key to derive from a nil value, so Put silently ignores it +// rather than panicking inside a caller-supplied keyFn. +// +// If an entry already exists at that key, it is first removed from every +// registered [Index] so the index reflects only the new value; the new value +// is then added to each index under its (possibly different) index key. +func (t *Table[V]) Put(v *V) { + if v == nil { + return + } + + id := t.keyFn(v) + + if old, exists := t.data[id]; exists { + for _, idx := range t.indexes { + idx.remove(old) + } + } + + t.data[id] = v + + for _, idx := range t.indexes { + idx.add(v) + } +} + +// Delete removes the entry stored under id, reporting whether it existed. +// Any registered [Index] is kept consistent by removing the deleted value. +func (t *Table[V]) Delete(id string) bool { + old, exists := t.data[id] + if !exists { + return false + } + + delete(t.data, id) + + for _, idx := range t.indexes { + idx.remove(old) + } + + return true +} + +// All returns every value in the table. Iteration order is UNSPECIFIED (Go +// map order); callers that need a stable order for an AWS list operation must +// sort the result themselves (e.g. by name/ARN, matching how the real AWS API +// they are emulating orders its own responses). +func (t *Table[V]) All() []*V { + out := make([]*V, 0, len(t.data)) + for _, v := range t.data { + out = append(out, v) + } + + return out +} + +// Range calls f for each value in the table in unspecified order, stopping +// early if f returns false. Unlike [Table.All] it performs no allocation, +// making it the preferred form for a read-only scan over a large table. +func (t *Table[V]) Range(f func(*V) bool) { + for _, v := range t.data { + if !f(v) { + return + } + } +} + +// Reset removes every entry from the table and clears every registered +// [Index], returning it to the state it was in immediately after [New]. +func (t *Table[V]) Reset() { + t.reset() +} + +// Snapshot returns every value in the table ordered by key, ascending. The +// fixed order (rather than [Table.All]'s unspecified map order) is what makes +// [Table.Restore] round-trips and JSON-marshaled snapshots byte-for-byte +// deterministic across runs, which keeps snapshot-based tests and diffs stable. +func (t *Table[V]) Snapshot() []*V { + ids := collections.SortedKeys(t.data) + + out := make([]*V, len(ids)) + for i, id := range ids { + out[i] = t.data[id] + } + + return out +} + +// Restore replaces the table's contents with items, rebuilding the primary +// map and every registered [Index] from scratch via keyFn. It is the inverse +// of [Table.Snapshot]. A nil or empty items leaves the table empty, matching +// [Table.Reset]. +func (t *Table[V]) Restore(items []*V) { + t.reset() + + for _, v := range items { + t.insertFresh(v) + } +} + +// reset clears the primary map and every index without reallocating them, +// so existing capacity is reused by subsequent inserts. +func (t *Table[V]) reset() { + clear(t.data) + + for _, idx := range t.indexes { + idx.clear() + } +} + +// insertFresh adds v to the table and every index without checking for (or +// evicting) an existing entry at the same key. It is only safe to call when +// the table is known to be empty at that key, e.g. immediately after reset, +// which is why [Table.Restore] uses it instead of [Table.Put]. +func (t *Table[V]) insertFresh(v *V) { + if v == nil { + return + } + + id := t.keyFn(v) + t.data[id] = v + + for _, idx := range t.indexes { + idx.add(v) + } +} + +// tableSnapshotter is the type-erased contract a [Table][V] satisfies so that +// a [Registry] can hold tables of heterogeneous value types in one collection. +// It is unexported and implemented only by [Table]; callers never name it — +// [Register] accepts a fully-typed *Table[V] and the compiler checks the +// erasure at the call site, so no interface{}/any ever appears in user code. +type tableSnapshotter interface { + reset() + snapshotJSON() (json.RawMessage, error) + restoreJSON(data json.RawMessage) error +} + +// snapshotJSON marshals [Table.Snapshot]'s deterministic, key-sorted output to +// JSON. It implements tableSnapshotter for [Registry.SnapshotAll]. +func (t *Table[V]) snapshotJSON() (json.RawMessage, error) { + data, err := json.Marshal(t.Snapshot()) + if err != nil { + return nil, fmt.Errorf("store: marshal table snapshot: %w", err) + } + + return data, nil +} + +// restoreJSON unmarshals data produced by snapshotJSON and loads it via +// [Table.Restore]. It implements tableSnapshotter for [Registry.RestoreAll]. +func (t *Table[V]) restoreJSON(data json.RawMessage) error { + var items []*V + + if err := json.Unmarshal(data, &items); err != nil { + return fmt.Errorf("store: unmarshal table snapshot: %w", err) + } + + t.Restore(items) + + return nil +} diff --git a/services/_PARITY_TEMPLATE.md b/services/_PARITY_TEMPLATE.md new file mode 100644 index 000000000..10c4069a1 --- /dev/null +++ b/services/_PARITY_TEMPLATE.md @@ -0,0 +1,29 @@ +--- +# PARITY MANIFEST SCHEMA — copy to services//PARITY.md, fill, keep updated. +# Purpose: record audit state so the NEXT audit diffs the delta instead of rescanning. +# Re-audit protocol: `git diff ..HEAD -- services//` for local drift, +# AND check the SDK module for ops added since sdk_version. Only audit changed/new surface; +# trust rows marked ok whose files are unchanged since last_audit_commit. +service: +sdk_module: aws-sdk-go-v2/service/@ # version audited against +last_audit_commit: # HEAD when this manifest was written +last_audit_date: +overall: # A = ~1k genuine fixes found; B = already-accurate, proven op-by-op +# Per-op or per-op-family status. Values: ok | partial | gap | deferred. +# wire=response/request shape vs SDK; errors=code+HTTP status; state=real mutate/read; persist=in backendSnapshot. +ops: + : {wire: ok, errors: ok, state: ok, persist: ok, note: } +# Families audited as a group (when per-op is impractical): +families: + : {status: ok, note: } +gaps: # known divergences NOT fixed — link bd issue ids + - (bd: gopherstack-xxx) +deferred: # consciously not audited this pass (scope) — next pass targets + - +leaks: {status: clean|found, note: } +--- + +## Notes +Freeform: AWS-behavior specifics worth remembering (exact algorithms, wire quirks, +error-message text, protocol = query-XML / REST-XML / REST-JSON / json-1.0), and any +"looks-wrong-but-correct" traps so the next auditor doesn't re-flag them. diff --git a/services/apigateway/PARITY.md b/services/apigateway/PARITY.md new file mode 100644 index 000000000..bef980b67 --- /dev/null +++ b/services/apigateway/PARITY.md @@ -0,0 +1,259 @@ +--- +service: apigateway +sdk_module: aws-sdk-go-v2/service/apigateway@v1.38.6 +last_audit_commit: 0b1194b6 +last_audit_date: 2026-07-05 +overall: A # ~1300 LOC of genuine PATCH-semantics fixes this sweep (see Notes) +ops: + UpdateStage: {wire: ok, errors: ok, state: ok, persist: ok, note: "PATCH semantics rewritten this sweep: /variables/{name}, canary-promotion copy op, /canarySettings/*, /accessLogSettings/*, per-route method settings, cacheCluster* fields added"} + UpdateRestApi: {wire: ok, errors: ok, state: ok, persist: ok, note: "PATCH /binaryMediaTypes/{escaped} add/remove now merges (was silently dropped); minimumCompressionSize string->int coercion fixed"} + UpdateAccount: {wire: ok, errors: ok, state: ok, persist: ok, note: "CloudwatchRoleARN field added to UpdateAccountInput (previously unsettable at all); /throttle/{rateLimit,burstLimit} nested PATCH now merges"} + UpdateUsagePlan: {wire: ok, errors: ok, state: ok, persist: ok, note: "/apiStages add/remove (value 'restApiId:stage') now merges; fixed InMemoryBackend.UpdateUsagePlan's len()>0 check so removing the last API stage actually applies"} + UpdateGatewayResponse: {wire: ok, errors: ok, state: ok, persist: ok, note: "now backed by a dedicated merge-based backend method (was reusing PutGatewayResponse's full-replace, silently wiping ResponseParameters/ResponseTemplates/StatusCode on every partial PATCH); /responseParameters/{key} and /responseTemplates/{key} per-entry PATCH added"} + UpdateApiKey: {wire: ok, errors: ok, state: ok, persist: ok, note: "top-level enabled bool now coerced from its string-typed PATCH value (see Notes #1)"} + UpdateRequestValidator: {wire: ok, errors: ok, state: ok, persist: ok, note: "validateRequestBody/validateRequestParameters bool coercion fixed"} + UpdateMethod: {wire: ok, errors: ok, state: ok, persist: ok, note: "apiKeyRequired bool coercion fixed"} + UpdateAuthorizer: {wire: ok, errors: ok, state: ok, persist: ok, note: "authorizerResultTtlInSeconds int coercion fixed"} + UpdateDeployment: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateResource: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateDomainName: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateBasePathMapping: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateDocumentationPart: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateDocumentationVersion: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateIntegration: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateIntegrationResponse: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateMethodResponse: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateModel: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateVpcLink: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateClientCertificate: {wire: ok, errors: ok, state: ok, persist: ok} + CreateRestApi: {wire: ok, errors: ok, state: ok, persist: ok} + GetRestApi: {wire: ok, errors: ok, state: ok, persist: ok} + GetRestApis: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteRestApi: {wire: ok, errors: ok, state: ok, persist: ok} + CreateResource: {wire: ok, errors: ok, state: ok, persist: ok, note: "{proxy+} greedy + trie-based routing (bd gopherstack fix #1403), parent-child tree verified"} + GetResource: {wire: ok, errors: ok, state: ok, persist: ok} + GetResources: {wire: ok, errors: ok, state: ok, persist: ok, note: "embed=[methods] param honored"} + DeleteResource: {wire: ok, errors: ok, state: ok, persist: ok} + PutMethod: {wire: ok, errors: ok, state: ok, persist: ok, note: "authorizationType validated against NONE/AWS_IAM/CUSTOM/COGNITO_USER_POOLS; CUSTOM/COGNITO_USER_POOLS require authorizerId (400 otherwise)"} + GetMethod: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteMethod: {wire: ok, errors: ok, state: ok, persist: ok} + PutMethodResponse: {wire: ok, errors: ok, state: ok, persist: ok} + GetMethodResponse: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteMethodResponse: {wire: ok, errors: ok, state: ok, persist: ok} + PutIntegration: {wire: ok, errors: ok, state: ok, persist: ok, note: "type validated (MOCK/AWS/AWS_PROXY/HTTP/HTTP_PROXY); VTL request/response templates real (vtl.go)"} + GetIntegration: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteIntegration: {wire: ok, errors: ok, state: ok, persist: ok} + PutIntegrationResponse: {wire: ok, errors: ok, state: ok, persist: ok} + GetIntegrationResponse: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteIntegrationResponse: {wire: ok, errors: ok, state: ok, persist: ok} + CreateDeployment: {wire: ok, errors: ok, state: ok, persist: ok, note: "real snapshot of resources/methods/integrations at deploy time (apiData/apiSnapshot); inline stage create/update via stageName param"} + GetDeployment: {wire: ok, errors: ok, state: ok, persist: ok} + GetDeployments: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteDeployment: {wire: ok, errors: ok, state: ok, persist: ok} + CreateStage: {wire: ok, errors: ok, state: ok, persist: ok, note: "cacheCluster{Enabled,Size,Status} fields added this sweep"} + GetStage: {wire: ok, errors: ok, state: ok, persist: ok} + GetStages: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteStage: {wire: ok, errors: ok, state: ok, persist: ok} + CreateAuthorizer: {wire: ok, errors: ok, state: ok, persist: ok, note: "TOKEN/REQUEST/COGNITO_USER_POOLS identitySource + TTL; cache bounded (bd gopherstack #1403)"} + GetAuthorizer: {wire: ok, errors: ok, state: ok, persist: ok} + GetAuthorizers: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteAuthorizer: {wire: ok, errors: ok, state: ok, persist: ok} + TestInvokeAuthorizer: {wire: ok, errors: ok, state: ok, persist: n/a} + CreateApiKey: {wire: ok, errors: ok, state: ok, persist: ok} + GetApiKey: {wire: ok, errors: ok, state: ok, persist: ok} + GetApiKeys: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteApiKey: {wire: ok, errors: ok, state: ok, persist: ok} + CreateUsagePlan: {wire: ok, errors: ok, state: ok, persist: ok} + GetUsagePlan: {wire: ok, errors: ok, state: ok, persist: ok} + GetUsagePlans: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteUsagePlan: {wire: ok, errors: ok, state: ok, persist: ok} + CreateUsagePlanKey: {wire: ok, errors: ok, state: ok, persist: ok} + GetUsagePlanKey: {wire: ok, errors: ok, state: ok, persist: ok} + GetUsagePlanKeys: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteUsagePlanKey: {wire: ok, errors: ok, state: ok, persist: ok} + GetUsage: {wire: ok, errors: ok, state: ok, persist: n/a} + UpdateUsage: {wire: ok, errors: ok, state: ok, persist: ok} + CreateModel: {wire: ok, errors: ok, state: ok, persist: ok} + GetModel: {wire: ok, errors: ok, state: ok, persist: ok} + GetModels: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteModel: {wire: ok, errors: ok, state: ok, persist: ok} + GetModelTemplate: {wire: ok, errors: ok, state: ok, persist: n/a} + CreateRequestValidator: {wire: ok, errors: ok, state: ok, persist: ok} + GetRequestValidator: {wire: ok, errors: ok, state: ok, persist: ok} + GetRequestValidators: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteRequestValidator: {wire: ok, errors: ok, state: ok, persist: ok} + CreateBasePathMapping: {wire: ok, errors: ok, state: ok, persist: ok} + GetBasePathMapping: {wire: ok, errors: ok, state: ok, persist: ok} + GetBasePathMappings: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteBasePathMapping: {wire: ok, errors: ok, state: ok, persist: ok} + CreateDomainName: {wire: ok, errors: ok, state: ok, persist: ok} + GetDomainName: {wire: ok, errors: ok, state: ok, persist: ok} + GetDomainNames: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteDomainName: {wire: ok, errors: ok, state: ok, persist: ok} + CreateDomainNameAccessAssociation: {wire: ok, errors: ok, state: ok, persist: ok} + GetDomainNameAccessAssociations: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteDomainNameAccessAssociation: {wire: ok, errors: ok, state: ok, persist: ok} + RejectDomainNameAccessAssociation: {wire: ok, errors: ok, state: ok, persist: ok} + CreateDocumentationPart: {wire: ok, errors: ok, state: ok, persist: ok} + GetDocumentationPart: {wire: ok, errors: ok, state: ok, persist: ok} + GetDocumentationParts: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteDocumentationPart: {wire: ok, errors: ok, state: ok, persist: ok} + ImportDocumentationParts: {wire: ok, errors: ok, state: ok, persist: ok} + CreateDocumentationVersion: {wire: ok, errors: ok, state: ok, persist: ok} + GetDocumentationVersion: {wire: ok, errors: ok, state: ok, persist: ok} + GetDocumentationVersions: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteDocumentationVersion: {wire: ok, errors: ok, state: ok, persist: ok} + GetAccount: {wire: ok, errors: ok, state: ok, persist: ok} + GetTags: {wire: ok, errors: ok, state: ok, persist: ok} + TagResource: {wire: ok, errors: ok, state: ok, persist: ok} + UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} + TestInvokeMethod: {wire: ok, errors: ok, state: ok, persist: n/a} + GetGatewayResponse: {wire: ok, errors: ok, state: ok, persist: ok} + GetGatewayResponses: {wire: ok, errors: ok, state: ok, persist: ok} + PutGatewayResponse: {wire: ok, errors: ok, state: ok, persist: ok, note: "unchanged: still a correct full replace for the real PUT operation"} + DeleteGatewayResponse: {wire: ok, errors: ok, state: ok, persist: ok} + GenerateClientCertificate: {wire: ok, errors: ok, state: ok, persist: ok} + GetClientCertificate: {wire: ok, errors: ok, state: ok, persist: ok} + GetClientCertificates: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteClientCertificate: {wire: ok, errors: ok, state: ok, persist: ok} + CreateVpcLink: {wire: ok, errors: ok, state: ok, persist: ok} + GetVpcLink: {wire: ok, errors: ok, state: ok, persist: ok} + GetVpcLinks: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteVpcLink: {wire: ok, errors: ok, state: ok, persist: ok} + GetExport: {wire: ok, errors: ok, state: ok, persist: n/a, note: "Swagger 2.0 + OAS 3.0 export, real per-API/stage synthesis"} + GetSdk: {wire: ok, errors: ok, state: ok, persist: n/a} + GetSdkType: {wire: ok, errors: ok, state: ok, persist: n/a} + GetSdkTypes: {wire: ok, errors: ok, state: ok, persist: n/a} + ImportApiKeys: {wire: ok, errors: ok, state: ok, persist: ok} + ImportRestApi: {wire: ok, errors: ok, state: ok, persist: ok} + PutRestApi: {wire: ok, errors: ok, state: ok, persist: ok} +families: + proxy_invocation: {status: ok, note: "proxy.go: MOCK/AWS/AWS_PROXY/HTTP/HTTP_PROXY dispatch, VTL passthrough (WHEN_NO_MATCH/WHEN_NO_TEMPLATES/NEVER), Lambda invoke via injected LambdaInvoker, usage-plan enforcement returns real 429 (LimitExceededException/TooManyRequestsException) via writeThrottleResponse — separate, already-correct code path from the control-plane handleError switch"} + authorizers_runtime: {status: ok, note: "TOKEN/REQUEST/COGNITO_USER_POOLS resolution + JWKS validation via injected JWKSProvider, TTL-bounded cache (bd gopherstack #1403 fixed prior unbounded growth)"} + patch_semantics: {status: ok, note: "REWRITTEN this sweep — see Notes; was the single biggest gap in the service"} +gaps: + - "PATCH 'remove' on bare top-level SCALAR fields (e.g. /description, /policy) is still a no-op: every Update*Input's backend merge uses a zero-value-means-not-provided check (if input.X != \"\" / != nil), so an explicit remove can't be distinguished from absence without adding presence-tracking (pointer types or an explicit field mask) to every Update*Input across ~15 resources. Map/list-valued fields (variables, binaryMediaTypes, apiStages, responseParameters/Templates, methodSettings) DO support remove correctly (this sweep) because their merge goes through a full non-nil replacement value. (bd: gopherstack-0s6, follow-up)" + - "UsagePlan per-api-stage throttle overrides via PATCH path /apiStages/{restApiId}:{stage}/throttle/{resourcePath}~1{httpMethod}/{rateLimit,burstLimit} are not implemented (only whole-apiStage add/remove via the single-segment /apiStages path is). (bd: gopherstack-0s6, follow-up)" + - "Stage CanarySettings.StageVariableOverrides nested PATCH (/canarySettings/stageVariableOverrides/{name}) is not implemented; canarySettings/{deploymentId,percentTraffic,useStageCache} are." + - "MethodSetting.CacheDataEncrypted and UnauthorizedCacheControlHeaderStrategy have no field on gopherstack's MethodSetting struct at all (predates this sweep), so their PATCH property paths (caching/dataEncrypted, caching/unauthorizedCacheControlHeaderStrategy) are unrecognized and fall through as no-ops." + - "The exact property-path strings for per-route stage method settings (stageMethodSettingProperty in patch.go, e.g. \"logging/dataTrace\") are a best-effort mapping from AWS's PATCH-operations reference docs, not verified against an SDK-level enum (PatchOperation.Path is a free string in aws-sdk-go-v2 with no typed catalog to check against). Flag for correction if a live wire capture disagrees." +deferred: + - "RestApi.ApiStatus/ApiStatusMessage/DisableExecuteApiEndpoint/EndpointAccessMode (present in aws-sdk-go-v2 types.RestApi, absent from gopherstack's RestAPI struct) — cosmetic/status-only fields, low client impact, out of scope this pass." + - "Stage.DocumentationVersion (present in AWS's Stage type) not modeled." +leaks: {status: clean, note: "no new goroutines/tickers/persistent state introduced this sweep — patch.go is pure request-scoped transform code; authorizer cache and resource routing trie growth were already bounded by a prior sweep (bd gopherstack #1403)"} +--- + +## Notes + +This sweep's finding was concentrated in ONE architectural gap rather than spread +across many small op bugs: **every single Update*/PATCH operation in this +service shared one flatten function (`handler.go`'s old `normalizePatchBody` / +`flattenPatchOps`) that could only express "replace one top-level field."** +Two concrete, previously-silent bugs fell out of that: + +1. **PatchOperation.Value is always a JSON *string* on the wire**, even for + bool/int targets — confirmed by reading aws-sdk-go-v2's actual serializer + (`awsRestjson1_serializeDocumentPatchOperation` calls `ok.String(*v.Value)` + unconditionally). So a real client PATCHing e.g. + `{"op":"replace","path":"/tracingEnabled","value":"true"}` was handing this + service the raw bytes `"true"` (a JSON string) to unmarshal directly into + `UpdateStageInput.TracingEnabled *bool` — a JSON type-mismatch that made the + whole PATCH request error out. This affected every non-string top-level + PATCH field across the service: `tracingEnabled`, `cacheClusterEnabled`, + `minimumCompressionSize`, API key `enabled`, `validateRequestBody`, + `validateRequestParameters`, `apiKeyRequired`, `authorizerResultTtlInSeconds`. + Fixed via `patch.go`'s `patchFieldKind` table + `coerceTopLevelPatchValue`. + +2. **Multi-segment PATCH paths were silently dropped.** The old flatten took + the *entire* remaining path after the leading `/` as one bogus flat field + name (e.g. `/variables/apiKey` became the map key literally + `"variables/apiKey"`), which matches no `Update*Input` json tag, so the + edit vanished with no error. This meant the single most common real-world + API Gateway PATCH usage — **setting one stage variable** — never worked at + all, alongside per-route method settings, binary-media-type membership, + usage-plan API-stage membership, gateway-response parameter/template + entries, and canary-deployment promotion (which AWS models as a `"copy"` + op — a verb the old flatten didn't implement at all, since it only handled + `"add"`/`"replace"`, and silently skipped `"remove"` too). + +Rewritten in `patch.go` (new file) with per-resource resolvers that read +current backend state and merge the touched entry, since the Update* backend +methods replace a map/struct field wholesale when it's provided — see the +file's package doc for the full design rationale and the exact PATCH path +shapes each resolver targets (stage variables, canary promotion, per-route +method settings, REST API binary media types, account throttle, usage-plan +API stages, gateway-response parameters/templates). + +**Independent bug found and fixed while auditing UpdateGatewayResponse**: its +action handler reused `PutGatewayResponse` (a correct full-replace for the +real PUT operation) for the PATCH operation too. Since PUT semantics +unconditionally overwrite `StatusCode`/`ResponseParameters`/`ResponseTemplates` +with whatever the (now-partial) flattened PATCH body happened to contain, ANY +partial PATCH — even a plain single-field `/statusCode` replace — silently +wiped whichever of the other two fields wasn't part of that particular PATCH +call. Added a dedicated `InMemoryBackend.UpdateGatewayResponse` that merges +field-by-field (falling back to AWS's implicit per-responseType default when +no custom response has been PUT yet), and wired the Update action to it. + +**Independent bug found and fixed in `InMemoryBackend.UpdateUsagePlan`**: it +only applied `input.APIStages` when `len(input.APIStages) > 0`, so a PATCH +that removes the last remaining API stage (producing a correctly-empty, but +non-nil, slice) was silently ignored. Changed the guard to `!= nil`. + +**Stage cache-cluster fields were entirely missing.** AWS's `Stage` (and +`CreateStage`/`UpdateStage` inputs) carry `CacheClusterEnabled`, +`CacheClusterSize`, and a derived `CacheClusterStatus` +(`AVAILABLE`/`NOT_AVAILABLE`) — none existed on gopherstack's `Stage` struct. +Added all three, wired through `CreateStage`/`UpdateStage`. + +**`UpdateAccountInput` was missing `CloudwatchRoleARN` entirely** — the single +most common real-world reason to call `UpdateAccount` (wiring a CloudWatch +Logs role for API Gateway execution logging) had no way to reach the backend +at all. Added the field and its backend wiring. + +### PATCH-op semantics traps (for the next auditor) + +- **`PatchOperation.Value` is always a JSON string on the wire**, regardless + of the target field's real type. Never copy it into a flattened body + verbatim unless the target field actually is a Go `string`. +- **A PATCH `path` is a JSON Pointer**: `~1` decodes to `/`, `~0` decodes to + `~`, and the `~1` substitution must happen *before* `~0` (so `~01` decodes + to `~1`, not to `/`). Get the order backwards and escaped values silently + corrupt. +- **Per-route stage method-settings paths have NO `"methodSettings"` path + segment.** They're addressed directly as + `/{resourcePath}/{httpMethod}/{category}/{property}`, where `resourcePath` + is itself JSON-Pointer-escaped (its own internal `/` becomes `~1`) or the + literal wildcard `*`. A path segment that starts with `~1` or is exactly + `*` is the tell that you're looking at a method-settings patch, not a + plain field name — every genuine top-level Stage field name is a bare + identifier and never starts with `~` or equals `*`. +- **`UsagePlan.apiStages` PATCH uses a single-segment path** (`/apiStages`) + with the API stage identified entirely by `value` — the string + `"{restApiId}:{stage}"` — not by a nested path segment. Don't assume every + list-membership PATCH nests the identifying key into the path; this one + doesn't. +- **`"copy"` is a real, AWS-documented op** (not just `add`/`replace`/`remove`) + used for canary-deployment promotion: + `{"op":"copy","from":"/canarySettings/deploymentId","path":"/deploymentId"}`. + Its `from` value must be resolved against the resource's *current* stored + state, not against the request body (there's no `from` value in the patch + document itself to read it from). +- **PUT vs PATCH on the same resource can have different replace semantics.** + `PutGatewayResponse` (real PUT) is correctly a full replace. Reusing it + verbatim for the PATCH operation on the same resource is a bug — PATCH must + merge only the touched fields with current state. Watch for this pattern + (`opUpdateX` action calling the same backend function as `opPutX`) + elsewhere in this codebase; it wasn't audited beyond GatewayResponse this + sweep. +- **`len(slice) > 0` is the wrong presence check for "was this field provided + in the patch."** It's indistinguishable from "provided but now empty" and + silently drops the empty-result case (found in `UpdateUsagePlan.APIStages`). + Use `!= nil` for slice/map fields that a merge might legitimately want to + empty out. + +Protocol confirmed: REST-JSON (`restjson1`) — HTTP verb + path routing per +resource, JSON request/response bodies, `application/x-amz-json-1.1` on +errors, epoch-seconds timestamps (`unixEpochTime`/`pkgs/awstime`-equivalent +inline type) — not a single json-1.0/1.1 RPC target the way most other +services in this codebase are. diff --git a/services/apigateway/backend.go b/services/apigateway/backend.go index 6b9f7e0e3..928f9b08f 100644 --- a/services/apigateway/backend.go +++ b/services/apigateway/backend.go @@ -12,6 +12,7 @@ import ( "time" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/awserr" "github.com/blackbirdworks/gopherstack/pkgs/tags" @@ -216,6 +217,7 @@ type StorageBackend interface { GetGatewayResponse(restAPIID, responseType string) (*GatewayResponse, error) GetGatewayResponses(restAPIID string) ([]GatewayResponse, error) PutGatewayResponse(input PutGatewayResponseInput) (*GatewayResponse, error) + UpdateGatewayResponse(input PutGatewayResponseInput) (*GatewayResponse, error) DeleteGatewayResponse(restAPIID, responseType string) error // Client certificate operations. @@ -401,44 +403,79 @@ func initTagsFromInput(name string, inputTags *tags.Tags) *tags.Tags { return tags.FromMap(name, inputTags.Clone()) } -// apiData holds per-REST-API state. -type apiData struct { - resources map[string]*Resource - deployments map[string]*Deployment - stages map[string]*Stage - authorizers map[string]*Authorizer - requestValidators map[string]*RequestValidator - documentationParts map[string]*DocumentationPart - documentationVersions map[string]*DocumentationVersion - models map[string]*Model - api RestAPI - // resourceVersion is bumped whenever the resource set is mutated. The data-plane - // proxy uses it to invalidate its cached routing trie. - resourceVersion uint64 -} - -// InMemoryBackend implements StorageBackend using in-memory maps. +// InMemoryBackend implements StorageBackend using in-memory maps, with every +// resource collection registered as a *store.Table on registry (see +// store_setup.go). Resource families that AWS scopes to a REST API +// (resources, deployments, stages, authorizers, requestValidators, +// documentationParts, documentationVersions, models) are flat tables keyed by +// a composite "#" string (see resourceKey et al in +// store_setup.go), with a secondary "byAPI" [store.Index] answering "all +// children of REST API X" -- replacing the old map[string]*apiData nesting. type InMemoryBackend struct { - account *Account - apis map[string]*apiData - apiKeys map[string]*APIKey - apiKeysByValue map[string]string // key value → key ID, O(1) data-plane lookup - usage *usageTracker // usage-plan quota + throttle state - basePathMappings map[string]*BasePathMapping // key: domainName + "#" + basePath - domainNames map[string]*DomainName - domainNameAccessAssociations map[string]*DomainNameAccessAssociation - usagePlans map[string]*UsagePlan - usagePlanKeys map[string]map[string]*UsagePlanKey // usagePlanID → keyID → key - gatewayResponses map[string]*GatewayResponse // key: restAPIID + "#" + responseType - clientCertificates map[string]*ClientCertificate // key: clientCertificateID - vpcLinks map[string]*VpcLink - usageOverrides map[string]map[string]int64 // usagePlanID → keyID → remaining quota, set via UpdateUsage - mu *lockmetrics.RWMutex + account *Account + + restApis *store.Table[RestAPI] + + resources *store.Table[Resource] + resourcesByAPI *store.Index[Resource] + + deployments *store.Table[Deployment] + deploymentsByAPI *store.Index[Deployment] + + stages *store.Table[Stage] + stagesByAPI *store.Index[Stage] + + authorizers *store.Table[Authorizer] + authorizersByAPI *store.Index[Authorizer] + + requestValidators *store.Table[RequestValidator] + requestValidatorsByAPI *store.Index[RequestValidator] + + documentationParts *store.Table[DocumentationPart] + documentationPartsByAPI *store.Index[DocumentationPart] + + documentationVersions *store.Table[DocumentationVersion] + documentationVersionsByAPI *store.Index[DocumentationVersion] + + models *store.Table[Model] + modelsByAPI *store.Index[Model] + + // resourceVersions (restAPIID → counter) is bumped whenever a REST API's + // resource set is mutated. The data-plane proxy uses it to invalidate its + // cached routing trie. Left as a plain map: not a resource collection, so + // it doesn't fit store.Table's shape (see store_setup.go's + // registerAllTables doc). + resourceVersions map[string]uint64 + + apiKeys *store.Table[APIKey] + apiKeysByValue map[string]string // key value → key ID, O(1) data-plane lookup + + usage *usageTracker // usage-plan quota + throttle state + + basePathMappings *store.Table[BasePathMapping] // key: domainName + "#" + basePath + domainNames *store.Table[DomainName] + domainNameAccessAssociations *store.Table[DomainNameAccessAssociation] + + usagePlans *store.Table[UsagePlan] + usagePlanKeys *store.Table[UsagePlanKey] // key: usagePlanID + "#" + keyID + usagePlanKeysByPlan *store.Index[UsagePlanKey] + + gatewayResponses *store.Table[GatewayResponse] // key: restAPIID + "#" + responseType + clientCertificates *store.Table[ClientCertificate] // key: clientCertificateID + vpcLinks *store.Table[VpcLink] + + // usageOverrides (usagePlanID → keyID → remaining quota) is set via + // UpdateUsage. Left as a plain map: the value (int64) carries no identity + // field of its own to key a store.Table by. + usageOverrides map[string]map[string]int64 + + registry *store.Registry + mu *lockmetrics.RWMutex } // NewInMemoryBackend creates a new InMemoryBackend. func NewInMemoryBackend() *InMemoryBackend { - return &InMemoryBackend{ + b := &InMemoryBackend{ account: &Account{ ThrottleSettings: &ThrottleSettings{ BurstLimit: defaultBurstLimit, @@ -447,21 +484,16 @@ func NewInMemoryBackend() *InMemoryBackend { Features: []string{"UsagePlans"}, APIKeyVersion: "1", }, - apis: make(map[string]*apiData), - apiKeys: make(map[string]*APIKey), - apiKeysByValue: make(map[string]string), - usage: newUsageTracker(), - basePathMappings: make(map[string]*BasePathMapping), - domainNames: make(map[string]*DomainName), - domainNameAccessAssociations: make(map[string]*DomainNameAccessAssociation), - usagePlans: make(map[string]*UsagePlan), - usagePlanKeys: make(map[string]map[string]*UsagePlanKey), - gatewayResponses: make(map[string]*GatewayResponse), - clientCertificates: make(map[string]*ClientCertificate), - vpcLinks: make(map[string]*VpcLink), - usageOverrides: make(map[string]map[string]int64), - mu: lockmetrics.New("apigateway"), + registry: store.NewRegistry(), + resourceVersions: make(map[string]uint64), + apiKeysByValue: make(map[string]string), + usage: newUsageTracker(), + usageOverrides: make(map[string]map[string]int64), + mu: lockmetrics.New("apigateway"), } + registerAllTables(b) + + return b } // CreateRestAPI creates a new REST API and its root resource. @@ -477,7 +509,7 @@ func (b *InMemoryBackend) CreateRestAPI(input CreateRestAPIInput) (*RestAPI, err backendTags := initTagsFromInput("apigw.api."+id+".tags", input.Tags) rootID := randomID(resourceIDLength) - api := RestAPI{ + api := &RestAPI{ ID: id, Name: input.Name, Description: input.Description, @@ -500,19 +532,12 @@ func (b *InMemoryBackend) CreateRestAPI(input CreateRestAPIInput) (*RestAPI, err ResourceMethods: make(map[string]*Method), } - b.apis[id] = &apiData{ - api: api, - resources: map[string]*Resource{rootID: root}, - deployments: make(map[string]*Deployment), - stages: make(map[string]*Stage), - authorizers: make(map[string]*Authorizer), - requestValidators: make(map[string]*RequestValidator), - documentationParts: make(map[string]*DocumentationPart), - documentationVersions: make(map[string]*DocumentationVersion), - models: make(map[string]*Model), - } + b.restApis.Put(api) + b.resources.Put(root) - return &api, nil + cp := *api + + return &cp, nil } // DeleteRestAPI removes a REST API and all its resources. @@ -520,26 +545,59 @@ func (b *InMemoryBackend) DeleteRestAPI(restAPIID string) error { b.mu.Lock("DeleteRestAPI") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] + api, ok := b.restApis.Get(restAPIID) if !ok { return fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - d.api.Tags.Close() - delete(b.apis, restAPIID) + api.Tags.Close() + b.restApis.Delete(restAPIID) + b.deleteAPIChildrenLocked(restAPIID) return nil } +// deleteAPIChildrenLocked removes every resource-family entry scoped to +// restAPIID (resources, deployments, stages, authorizers, requestValidators, +// documentationParts, documentationVersions, models) via each table's "byAPI" +// index. Callers must hold b.mu. +func (b *InMemoryBackend) deleteAPIChildrenLocked(restAPIID string) { + for _, r := range append([]*Resource{}, b.resourcesByAPI.Get(restAPIID)...) { + b.resources.Delete(resourceKeyFn(r)) + } + for _, d := range append([]*Deployment{}, b.deploymentsByAPI.Get(restAPIID)...) { + b.deployments.Delete(deploymentKeyFn(d)) + } + for _, s := range append([]*Stage{}, b.stagesByAPI.Get(restAPIID)...) { + b.stages.Delete(stageKeyFn(s)) + } + for _, a := range append([]*Authorizer{}, b.authorizersByAPI.Get(restAPIID)...) { + b.authorizers.Delete(authorizerKeyFn(a)) + } + for _, v := range append([]*RequestValidator{}, b.requestValidatorsByAPI.Get(restAPIID)...) { + b.requestValidators.Delete(requestValidatorKeyFn(v)) + } + for _, p := range append([]*DocumentationPart{}, b.documentationPartsByAPI.Get(restAPIID)...) { + b.documentationParts.Delete(documentationPartKeyFn(p)) + } + for _, v := range append([]*DocumentationVersion{}, b.documentationVersionsByAPI.Get(restAPIID)...) { + b.documentationVersions.Delete(documentationVersionKeyFn(v)) + } + for _, m := range append([]*Model{}, b.modelsByAPI.Get(restAPIID)...) { + b.models.Delete(modelKeyFn(m)) + } + delete(b.resourceVersions, restAPIID) +} + // GetRestAPI returns a single REST API. func (b *InMemoryBackend) GetRestAPI(restAPIID string) (*RestAPI, error) { b.mu.RLock("GetRestAPI") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] + api, ok := b.restApis.Get(restAPIID) if !ok { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - cp := d.api + cp := *api return &cp, nil } @@ -549,9 +607,9 @@ func (b *InMemoryBackend) GetRestAPIs(limit int, position string) ([]RestAPI, st b.mu.RLock("GetRestAPIs") defer b.mu.RUnlock() - all := make([]RestAPI, 0, len(b.apis)) - for _, d := range b.apis { - all = append(all, d.api) + all := make([]RestAPI, 0, b.restApis.Len()) + for _, api := range b.restApis.All() { + all = append(all, *api) } sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID }) @@ -565,13 +623,13 @@ func (b *InMemoryBackend) GetResources(restAPIID, position string, limit int) ([ b.mu.RLock("GetResources") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, "", fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - all := make([]Resource, 0, len(d.resources)) - for _, r := range d.resources { + group := b.resourcesByAPI.Get(restAPIID) + all := make([]Resource, 0, len(group)) + for _, r := range group { all = append(all, *r) } sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID }) @@ -589,17 +647,17 @@ func (b *InMemoryBackend) ResourcesForRouting(restAPIID string) ([]Resource, uin b.mu.RLock("ResourcesForRouting") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, 0, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - all := make([]Resource, 0, len(d.resources)) - for _, r := range d.resources { + group := b.resourcesByAPI.Get(restAPIID) + all := make([]Resource, 0, len(group)) + for _, r := range group { all = append(all, *r) } - return all, d.resourceVersion, nil + return all, b.resourceVersions[restAPIID], nil } // GetResource returns a single resource. @@ -607,11 +665,10 @@ func (b *InMemoryBackend) GetResource(restAPIID, resourceID string) (*Resource, b.mu.RLock("GetResource") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - r, ok := d.resources[resourceID] + r, ok := b.resources.Get(resourceKey(restAPIID, resourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, resourceID) } @@ -629,12 +686,11 @@ func (b *InMemoryBackend) CreateResource(restAPIID, parentID, pathPart string) ( b.mu.Lock("CreateResource") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - parent, ok := d.resources[parentID] + parent, ok := b.resources.Get(resourceKey(restAPIID, parentID)) if !ok { return nil, fmt.Errorf("%w: parent resource %s not found", ErrResourceNotFound, parentID) } @@ -650,8 +706,8 @@ func (b *InMemoryBackend) CreateResource(restAPIID, parentID, pathPart string) ( RestAPIID: restAPIID, ResourceMethods: make(map[string]*Method), } - d.resources[id] = res - d.resourceVersion++ + b.resources.Put(res) + b.resourceVersions[restAPIID]++ cp := *res @@ -663,15 +719,13 @@ func (b *InMemoryBackend) DeleteResource(restAPIID, resourceID string) error { b.mu.Lock("DeleteResource") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - if _, exists := d.resources[resourceID]; !exists { + if !b.resources.Delete(resourceKey(restAPIID, resourceID)) { return fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, resourceID) } - delete(d.resources, resourceID) - d.resourceVersion++ + b.resourceVersions[restAPIID]++ return nil } @@ -681,11 +735,10 @@ func (b *InMemoryBackend) PutMethod(input PutMethodInput) (*Method, error) { b.mu.Lock("PutMethod") defer b.mu.Unlock() - d, ok := b.apis[input.RestAPIID] - if !ok { + if !b.restApis.Has(input.RestAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } - r, ok := d.resources[input.ResourceID] + r, ok := b.resources.Get(resourceKey(input.RestAPIID, input.ResourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, input.ResourceID) } @@ -718,11 +771,10 @@ func (b *InMemoryBackend) GetMethod(restAPIID, resourceID, httpMethod string) (* b.mu.RLock("GetMethod") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - r, ok := d.resources[resourceID] + r, ok := b.resources.Get(resourceKey(restAPIID, resourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, resourceID) } @@ -740,11 +792,10 @@ func (b *InMemoryBackend) DeleteMethod(restAPIID, resourceID, httpMethod string) b.mu.Lock("DeleteMethod") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - r, ok := d.resources[resourceID] + r, ok := b.resources.Get(resourceKey(restAPIID, resourceID)) if !ok { return fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, resourceID) } @@ -764,11 +815,10 @@ func (b *InMemoryBackend) PutIntegration( b.mu.Lock("PutIntegration") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - r, ok := d.resources[resourceID] + r, ok := b.resources.Get(resourceKey(restAPIID, resourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, resourceID) } @@ -810,11 +860,10 @@ func (b *InMemoryBackend) GetIntegration(restAPIID, resourceID, httpMethod strin b.mu.RLock("GetIntegration") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - r, ok := d.resources[resourceID] + r, ok := b.resources.Get(resourceKey(restAPIID, resourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, resourceID) } @@ -835,11 +884,10 @@ func (b *InMemoryBackend) DeleteIntegration(restAPIID, resourceID, httpMethod st b.mu.Lock("DeleteIntegration") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - r, ok := d.resources[resourceID] + r, ok := b.resources.Get(resourceKey(restAPIID, resourceID)) if !ok { return fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, resourceID) } @@ -863,11 +911,10 @@ func (b *InMemoryBackend) PutMethodResponse( b.mu.Lock("PutMethodResponse") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - r, ok := d.resources[resourceID] + r, ok := b.resources.Get(resourceKey(restAPIID, resourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, resourceID) } @@ -898,11 +945,10 @@ func (b *InMemoryBackend) GetMethodResponse( b.mu.RLock("GetMethodResponse") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - r, ok := d.resources[resourceID] + r, ok := b.resources.Get(resourceKey(restAPIID, resourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, resourceID) } @@ -924,11 +970,10 @@ func (b *InMemoryBackend) DeleteMethodResponse(restAPIID, resourceID, httpMethod b.mu.Lock("DeleteMethodResponse") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - r, ok := d.resources[resourceID] + r, ok := b.resources.Get(resourceKey(restAPIID, resourceID)) if !ok { return fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, resourceID) } @@ -952,11 +997,10 @@ func (b *InMemoryBackend) PutIntegrationResponse( b.mu.Lock("PutIntegrationResponse") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - r, ok := d.resources[resourceID] + r, ok := b.resources.Get(resourceKey(restAPIID, resourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, resourceID) } @@ -992,11 +1036,10 @@ func (b *InMemoryBackend) GetIntegrationResponse( b.mu.RLock("GetIntegrationResponse") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - r, ok := d.resources[resourceID] + r, ok := b.resources.Get(resourceKey(restAPIID, resourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, resourceID) } @@ -1021,11 +1064,10 @@ func (b *InMemoryBackend) DeleteIntegrationResponse(restAPIID, resourceID, httpM b.mu.Lock("DeleteIntegrationResponse") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - r, ok := d.resources[resourceID] + r, ok := b.resources.Get(resourceKey(restAPIID, resourceID)) if !ok { return fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, resourceID) } @@ -1049,8 +1091,7 @@ func (b *InMemoryBackend) CreateDeployment(restAPIID, stageName, description str b.mu.Lock("CreateDeployment") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } @@ -1062,7 +1103,7 @@ func (b *InMemoryBackend) CreateDeployment(restAPIID, stageName, description str Description: description, CreatedDate: now, } - d.deployments[deplID] = depl + b.deployments.Put(depl) if stageName != "" { // AWS: stage description comes from stageDescription (a separate field), @@ -1075,7 +1116,7 @@ func (b *InMemoryBackend) CreateDeployment(restAPIID, stageName, description str LastUpdatedDate: now, Variables: make(map[string]string), } - d.stages[stageName] = stage + b.stages.Put(stage) } cp := *depl @@ -1088,13 +1129,13 @@ func (b *InMemoryBackend) GetDeployments(restAPIID string) ([]Deployment, error) b.mu.RLock("GetDeployments") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - all := make([]Deployment, 0, len(d.deployments)) - for _, dep := range d.deployments { + group := b.deploymentsByAPI.Get(restAPIID) + all := make([]Deployment, 0, len(group)) + for _, dep := range group { all = append(all, *dep) } sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID }) @@ -1107,12 +1148,11 @@ func (b *InMemoryBackend) GetDeployment(restAPIID, deploymentID string) (*Deploy b.mu.RLock("GetDeployment") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - dep, ok := d.deployments[deploymentID] + dep, ok := b.deployments.Get(deploymentKey(restAPIID, deploymentID)) if !ok { return nil, fmt.Errorf("%w: deployment %s not found", ErrDeploymentNotFound, deploymentID) } @@ -1127,17 +1167,15 @@ func (b *InMemoryBackend) DeleteDeployment(restAPIID, deploymentID string) error b.mu.Lock("DeleteDeployment") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - _, exists := d.deployments[deploymentID] - if !exists { + if !b.deployments.Has(deploymentKey(restAPIID, deploymentID)) { return fmt.Errorf("%w: deployment %s not found", ErrDeploymentNotFound, deploymentID) } - for _, s := range d.stages { + for _, s := range b.stagesByAPI.Get(restAPIID) { if s.DeploymentID == deploymentID { return fmt.Errorf( "%w: deployment %s is referenced by stage %s and cannot be deleted", @@ -1146,7 +1184,7 @@ func (b *InMemoryBackend) DeleteDeployment(restAPIID, deploymentID string) error } } - delete(d.deployments, deploymentID) + b.deployments.Delete(deploymentKey(restAPIID, deploymentID)) return nil } @@ -1156,13 +1194,13 @@ func (b *InMemoryBackend) GetStages(restAPIID string) ([]Stage, error) { b.mu.RLock("GetStages") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - all := make([]Stage, 0, len(d.stages)) - for _, s := range d.stages { + group := b.stagesByAPI.Get(restAPIID) + all := make([]Stage, 0, len(group)) + for _, s := range group { cp := *s cp.InvokeURL = stageInvokeURL(restAPIID, s.StageName) all = append(all, cp) @@ -1177,11 +1215,10 @@ func (b *InMemoryBackend) GetStage(restAPIID, stageName string) (*Stage, error) b.mu.RLock("GetStage") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - s, stageOK := d.stages[stageName] + s, stageOK := b.stages.Get(stageKey(restAPIID, stageName)) if !stageOK { return nil, fmt.Errorf("%w: stage %s not found", ErrResourceNotFound, stageName) } @@ -1196,14 +1233,12 @@ func (b *InMemoryBackend) DeleteStage(restAPIID, stageName string) error { b.mu.Lock("DeleteStage") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - if _, stageOK := d.stages[stageName]; !stageOK { + if !b.stages.Delete(stageKey(restAPIID, stageName)) { return fmt.Errorf("%w: stage %s not found", ErrResourceNotFound, stageName) } - delete(d.stages, stageName) return nil } @@ -1227,14 +1262,14 @@ func (b *InMemoryBackend) CreateAuthorizer(restAPIID string, input CreateAuthori b.mu.Lock("CreateAuthorizer") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } id := randomID(resourceIDLength) auth := &Authorizer{ ID: id, + RestAPIID: restAPIID, Name: input.Name, Type: input.Type, AuthorizerURI: input.AuthorizerURI, @@ -1244,7 +1279,7 @@ func (b *InMemoryBackend) CreateAuthorizer(restAPIID string, input CreateAuthori AuthorizerResultTTLInSeconds: input.AuthorizerResultTTLInSeconds, ProviderARNs: input.ProviderARNs, } - d.authorizers[id] = auth + b.authorizers.Put(auth) cp := *auth @@ -1256,11 +1291,10 @@ func (b *InMemoryBackend) GetAuthorizer(restAPIID, authorizerID string) (*Author b.mu.RLock("GetAuthorizer") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - auth, ok := d.authorizers[authorizerID] + auth, ok := b.authorizers.Get(authorizerKey(restAPIID, authorizerID)) if !ok { return nil, fmt.Errorf("%w: authorizer %s not found", ErrAuthorizerNotFound, authorizerID) } @@ -1274,13 +1308,13 @@ func (b *InMemoryBackend) GetAuthorizers(restAPIID string) ([]Authorizer, error) b.mu.RLock("GetAuthorizers") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - all := make([]Authorizer, 0, len(d.authorizers)) - for _, auth := range d.authorizers { + group := b.authorizersByAPI.Get(restAPIID) + all := make([]Authorizer, 0, len(group)) + for _, auth := range group { all = append(all, *auth) } sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID }) @@ -1296,11 +1330,10 @@ func (b *InMemoryBackend) UpdateAuthorizer( b.mu.Lock("UpdateAuthorizer") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - auth, ok := d.authorizers[authorizerID] + auth, ok := b.authorizers.Get(authorizerKey(restAPIID, authorizerID)) if !ok { return nil, fmt.Errorf("%w: authorizer %s not found", ErrAuthorizerNotFound, authorizerID) } @@ -1345,14 +1378,12 @@ func (b *InMemoryBackend) DeleteAuthorizer(restAPIID, authorizerID string) error b.mu.Lock("DeleteAuthorizer") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - if _, exists := d.authorizers[authorizerID]; !exists { + if !b.authorizers.Delete(authorizerKey(restAPIID, authorizerID)) { return fmt.Errorf("%w: authorizer %s not found", ErrAuthorizerNotFound, authorizerID) } - delete(d.authorizers, authorizerID) return nil } @@ -1369,19 +1400,19 @@ func (b *InMemoryBackend) CreateRequestValidator( b.mu.Lock("CreateRequestValidator") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } id := randomID(resourceIDLength) rv := &RequestValidator{ ID: id, + RestAPIID: restAPIID, Name: input.Name, ValidateRequestBody: input.ValidateRequestBody, ValidateRequestParameters: input.ValidateRequestParameters, } - d.requestValidators[id] = rv + b.requestValidators.Put(rv) cp := *rv @@ -1393,11 +1424,10 @@ func (b *InMemoryBackend) GetRequestValidator(restAPIID, validatorID string) (*R b.mu.RLock("GetRequestValidator") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - rv, ok := d.requestValidators[validatorID] + rv, ok := b.requestValidators.Get(requestValidatorKey(restAPIID, validatorID)) if !ok { return nil, fmt.Errorf("%w: request validator %s not found", ErrValidatorNotFound, validatorID) } @@ -1411,13 +1441,13 @@ func (b *InMemoryBackend) GetRequestValidators(restAPIID string) ([]RequestValid b.mu.RLock("GetRequestValidators") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - all := make([]RequestValidator, 0, len(d.requestValidators)) - for _, rv := range d.requestValidators { + group := b.requestValidatorsByAPI.Get(restAPIID) + all := make([]RequestValidator, 0, len(group)) + for _, rv := range group { all = append(all, *rv) } sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID }) @@ -1433,11 +1463,10 @@ func (b *InMemoryBackend) UpdateRequestValidator( b.mu.Lock("UpdateRequestValidator") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - rv, ok := d.requestValidators[validatorID] + rv, ok := b.requestValidators.Get(requestValidatorKey(restAPIID, validatorID)) if !ok { return nil, fmt.Errorf("%w: request validator %s not found", ErrValidatorNotFound, validatorID) } @@ -1462,14 +1491,12 @@ func (b *InMemoryBackend) DeleteRequestValidator(restAPIID, validatorID string) b.mu.Lock("DeleteRequestValidator") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - if _, exists := d.requestValidators[validatorID]; !exists { + if !b.requestValidators.Delete(requestValidatorKey(restAPIID, validatorID)) { return fmt.Errorf("%w: request validator %s not found", ErrValidatorNotFound, validatorID) } - delete(d.requestValidators, validatorID) return nil } @@ -1489,42 +1516,45 @@ func (b *InMemoryBackend) Reset() { defer b.mu.Unlock() // Close all REST API tag stores to prevent resource leaks. - for _, d := range b.apis { - if d.api.Tags != nil { - d.api.Tags.Close() + for _, api := range b.restApis.All() { + if api.Tags != nil { + api.Tags.Close() } } - for _, k := range b.apiKeys { + for _, k := range b.apiKeys.All() { if k.Tags != nil { k.Tags.Close() } } - for _, dn := range b.domainNames { + for _, dn := range b.domainNames.All() { if dn.Tags != nil { dn.Tags.Close() } } - for _, p := range b.usagePlans { + for _, p := range b.usagePlans.All() { if p.Tags != nil { p.Tags.Close() } } - b.apis = make(map[string]*apiData) - b.apiKeys = make(map[string]*APIKey) + b.registry.ResetAll() + // The "dirty" tables (see store_setup.go's registerAllTables doc) are + // deliberately NOT on b.registry, so each needs its own Reset() call here. + b.resources.Reset() + b.deployments.Reset() + b.stages.Reset() + b.authorizers.Reset() + b.requestValidators.Reset() + b.documentationParts.Reset() + b.documentationVersions.Reset() + b.models.Reset() + b.usagePlanKeys.Reset() + b.resourceVersions = make(map[string]uint64) b.apiKeysByValue = make(map[string]string) b.usage = newUsageTracker() - b.basePathMappings = make(map[string]*BasePathMapping) - b.domainNames = make(map[string]*DomainName) - b.domainNameAccessAssociations = make(map[string]*DomainNameAccessAssociation) - b.usagePlans = make(map[string]*UsagePlan) - b.usagePlanKeys = make(map[string]map[string]*UsagePlanKey) - b.vpcLinks = make(map[string]*VpcLink) - b.clientCertificates = make(map[string]*ClientCertificate) - b.gatewayResponses = make(map[string]*GatewayResponse) b.usageOverrides = make(map[string]map[string]int64) } @@ -1537,7 +1567,7 @@ func (b *InMemoryBackend) CreateAPIKey(input CreateAPIKeyInput) (*APIKey, error) b.mu.Lock("CreateAPIKey") defer b.mu.Unlock() - for _, k := range b.apiKeys { + for _, k := range b.apiKeys.All() { if k.Name == input.Name { return nil, fmt.Errorf("%w: API key with name %q already exists", ErrAlreadyExists, input.Name) } @@ -1564,7 +1594,7 @@ func (b *InMemoryBackend) CreateAPIKey(input CreateAPIKeyInput) (*APIKey, error) CreatedDate: now, LastUpdatedDate: now, } - b.apiKeys[id] = key + b.apiKeys.Put(key) b.apiKeysByValue[value] = id cp := *key @@ -1585,8 +1615,8 @@ func (b *InMemoryBackend) CreateBasePathMapping(input CreateBasePathMappingInput b.mu.Lock("CreateBasePathMapping") defer b.mu.Unlock() - mapKey := input.DomainName + "#" + input.BasePath - if _, exists := b.basePathMappings[mapKey]; exists { + mapKey := basePathMappingKey(input.DomainName, input.BasePath) + if b.basePathMappings.Has(mapKey) { return nil, fmt.Errorf("%w: base path mapping already exists for domain %q path %q", ErrAlreadyExists, input.DomainName, input.BasePath) } @@ -1597,7 +1627,7 @@ func (b *InMemoryBackend) CreateBasePathMapping(input CreateBasePathMappingInput RestAPIID: input.RestAPIID, Stage: input.Stage, } - b.basePathMappings[mapKey] = bpm + b.basePathMappings.Put(bpm) cp := *bpm @@ -1617,8 +1647,7 @@ func (b *InMemoryBackend) CreateDocumentationPart(input CreateDocumentationPartI b.mu.Lock("CreateDocumentationPart") defer b.mu.Unlock() - d, ok := b.apis[input.RestAPIID] - if !ok { + if !b.restApis.Has(input.RestAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } @@ -1629,7 +1658,7 @@ func (b *InMemoryBackend) CreateDocumentationPart(input CreateDocumentationPartI Location: input.Location, Properties: input.Properties, } - d.documentationParts[id] = part + b.documentationParts.Put(part) cp := *part @@ -1651,12 +1680,11 @@ func (b *InMemoryBackend) CreateDocumentationVersion( b.mu.Lock("CreateDocumentationVersion") defer b.mu.Unlock() - d, ok := b.apis[input.RestAPIID] - if !ok { + if !b.restApis.Has(input.RestAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } - if _, exists := d.documentationVersions[input.Version]; exists { + if b.documentationVersions.Has(documentationVersionKey(input.RestAPIID, input.Version)) { return nil, fmt.Errorf("%w: documentation version %q already exists", ErrAlreadyExists, input.Version) } @@ -1666,7 +1694,7 @@ func (b *InMemoryBackend) CreateDocumentationVersion( Description: input.Description, CreatedDate: unixEpochTime{time.Now()}, } - d.documentationVersions[input.Version] = ver + b.documentationVersions.Put(ver) cp := *ver @@ -1682,7 +1710,7 @@ func (b *InMemoryBackend) CreateDomainName(input CreateDomainNameInput) (*Domain b.mu.Lock("CreateDomainName") defer b.mu.Unlock() - if _, exists := b.domainNames[input.DomainName]; exists { + if b.domainNames.Has(input.DomainName) { return nil, fmt.Errorf("%w: domain name %q already exists", ErrAlreadyExists, input.DomainName) } @@ -1723,7 +1751,7 @@ func (b *InMemoryBackend) CreateDomainName(input CreateDomainNameInput) (*Domain Tags: backendTags, CreatedDate: &now, } - b.domainNames[input.DomainName] = dn + b.domainNames.Put(dn) cp := *dn @@ -1756,7 +1784,7 @@ func (b *InMemoryBackend) CreateDomainNameAccessAssociation( AccessAssociationSource: input.AccessAssociationSource, AccessAssociationSourceType: input.AccessAssociationSourceType, } - b.domainNameAccessAssociations[assocARN] = assoc + b.domainNameAccessAssociations.Put(assoc) cp := *assoc @@ -1782,8 +1810,9 @@ func (b *InMemoryBackend) GetDomainNameAccessAssociations(resourceOwner string) return []DomainNameAccessAssociation{}, nil } - result := make([]DomainNameAccessAssociation, 0, len(b.domainNameAccessAssociations)) - for _, assoc := range b.domainNameAccessAssociations { + all := b.domainNameAccessAssociations.All() + result := make([]DomainNameAccessAssociation, 0, len(all)) + for _, assoc := range all { result = append(result, *assoc) } @@ -1799,12 +1828,10 @@ func (b *InMemoryBackend) DeleteDomainNameAccessAssociation(arn string) error { b.mu.Lock("DeleteDomainNameAccessAssociation") defer b.mu.Unlock() - if _, ok := b.domainNameAccessAssociations[arn]; !ok { + if !b.domainNameAccessAssociations.Delete(arn) { return fmt.Errorf("%w: domain name access association %s not found", ErrNotFound, arn) } - delete(b.domainNameAccessAssociations, arn) - return nil } @@ -1814,7 +1841,7 @@ func (b *InMemoryBackend) RejectDomainNameAccessAssociation(arn, domainNameARN s b.mu.Lock("RejectDomainNameAccessAssociation") defer b.mu.Unlock() - assoc, ok := b.domainNameAccessAssociations[arn] + assoc, ok := b.domainNameAccessAssociations.Get(arn) if !ok { return fmt.Errorf("%w: domain name access association %s not found", ErrNotFound, arn) } @@ -1823,7 +1850,7 @@ func (b *InMemoryBackend) RejectDomainNameAccessAssociation(arn, domainNameARN s return fmt.Errorf("%w: domainNameArn does not match association %s", ErrInvalidParameter, arn) } - delete(b.domainNameAccessAssociations, arn) + b.domainNameAccessAssociations.Delete(arn) return nil } @@ -1845,12 +1872,11 @@ func (b *InMemoryBackend) CreateModel(input CreateModelInput) (*Model, error) { b.mu.Lock("CreateModel") defer b.mu.Unlock() - d, ok := b.apis[input.RestAPIID] - if !ok { + if !b.restApis.Has(input.RestAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } - for _, m := range d.models { + for _, m := range b.modelsByAPI.Get(input.RestAPIID) { if m.Name == input.Name { return nil, fmt.Errorf( "%w: model %q already exists in REST API %s", @@ -1870,7 +1896,7 @@ func (b *InMemoryBackend) CreateModel(input CreateModelInput) (*Model, error) { ContentType: input.ContentType, Schema: input.Schema, } - d.models[id] = model + b.models.Put(model) cp := *model @@ -1894,16 +1920,15 @@ func (b *InMemoryBackend) CreateStage(input CreateStageInput) (*Stage, error) { b.mu.Lock("CreateStage") defer b.mu.Unlock() - d, ok := b.apis[input.RestAPIID] - if !ok { + if !b.restApis.Has(input.RestAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } - if _, exists := d.deployments[input.DeploymentID]; !exists { + if !b.deployments.Has(deploymentKey(input.RestAPIID, input.DeploymentID)) { return nil, fmt.Errorf("%w: deployment %s not found", ErrDeploymentNotFound, input.DeploymentID) } - if _, exists := d.stages[input.StageName]; exists { + if b.stages.Has(stageKey(input.RestAPIID, input.StageName)) { return nil, fmt.Errorf("%w: stage %q already exists", ErrAlreadyExists, input.StageName) } @@ -1926,8 +1951,11 @@ func (b *InMemoryBackend) CreateStage(input CreateStageInput) (*Stage, error) { MethodSettings: input.MethodSettings, TracingEnabled: input.TracingEnabled, ClientCertificateID: input.ClientCertificateID, + CacheClusterEnabled: input.CacheClusterEnabled, + CacheClusterSize: input.CacheClusterSize, + CacheClusterStatus: cacheClusterStatusFor(input.CacheClusterEnabled), } - d.stages[input.StageName] = stage + b.stages.Put(stage) cp := *stage @@ -1955,8 +1983,7 @@ func (b *InMemoryBackend) CreateUsagePlan(input CreateUsagePlanInput) (*UsagePla APIStages: input.APIStages, Tags: backendTags, } - b.usagePlans[id] = plan - b.usagePlanKeys[id] = make(map[string]*UsagePlanKey) + b.usagePlans.Put(plan) cp := *plan @@ -1984,27 +2011,27 @@ func (b *InMemoryBackend) CreateUsagePlanKey(input CreateUsagePlanKeyInput) (*Us b.mu.Lock("CreateUsagePlanKey") defer b.mu.Unlock() - if _, exists := b.usagePlans[input.UsagePlanID]; !exists { + if !b.usagePlans.Has(input.UsagePlanID) { return nil, fmt.Errorf("%w: usage plan %s not found", ErrUsagePlanNotFound, input.UsagePlanID) } - apiKey, exists := b.apiKeys[input.KeyID] + apiKey, exists := b.apiKeys.Get(input.KeyID) if !exists { return nil, fmt.Errorf("%w: API key %s not found", ErrAPIKeyNotFound, input.KeyID) } - keys := b.usagePlanKeys[input.UsagePlanID] - if _, alreadyAssoc := keys[input.KeyID]; alreadyAssoc { + if b.usagePlanKeys.Has(usagePlanKeyKey(input.UsagePlanID, input.KeyID)) { return nil, fmt.Errorf("%w: key %s already associated with usage plan", ErrAlreadyExists, input.KeyID) } upk := &UsagePlanKey{ - ID: apiKey.ID, - Type: input.KeyType, - Value: apiKey.Value, - Name: apiKey.Name, + ID: apiKey.ID, + Type: input.KeyType, + Value: apiKey.Value, + Name: apiKey.Name, + UsagePlanID: input.UsagePlanID, } - keys[input.KeyID] = upk + b.usagePlanKeys.Put(upk) cp := *upk @@ -2015,7 +2042,7 @@ func (b *InMemoryBackend) CreateUsagePlanKey(input CreateUsagePlanKeyInput) (*Us func (b *InMemoryBackend) GetAPIKey(id string) (*APIKey, error) { b.mu.RLock("GetAPIKey") defer b.mu.RUnlock() - key, ok := b.apiKeys[id] + key, ok := b.apiKeys.Get(id) if !ok { return nil, fmt.Errorf("%w: API key %s not found", ErrAPIKeyNotFound, id) } @@ -2035,7 +2062,7 @@ func (b *InMemoryBackend) GetAPIKeyByValue(value string) (*APIKey, error) { if !ok { return nil, fmt.Errorf("%w: API key with value not found", ErrAPIKeyNotFound) } - k, ok := b.apiKeys[id] + k, ok := b.apiKeys.Get(id) if !ok { return nil, fmt.Errorf("%w: API key with value not found", ErrAPIKeyNotFound) } @@ -2048,8 +2075,8 @@ func (b *InMemoryBackend) GetAPIKeyByValue(value string) (*APIKey, error) { func (b *InMemoryBackend) GetAPIKeys() ([]APIKey, error) { b.mu.RLock("GetAPIKeys") defer b.mu.RUnlock() - all := make([]APIKey, 0, len(b.apiKeys)) - for _, k := range b.apiKeys { + all := make([]APIKey, 0, b.apiKeys.Len()) + for _, k := range b.apiKeys.All() { all = append(all, *k) } sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID }) @@ -2061,12 +2088,12 @@ func (b *InMemoryBackend) GetAPIKeys() ([]APIKey, error) { func (b *InMemoryBackend) DeleteAPIKey(id string) error { b.mu.Lock("DeleteAPIKey") defer b.mu.Unlock() - key, ok := b.apiKeys[id] + key, ok := b.apiKeys.Get(id) if !ok { return fmt.Errorf("%w: API key %s not found", ErrAPIKeyNotFound, id) } delete(b.apiKeysByValue, key.Value) - delete(b.apiKeys, id) + b.apiKeys.Delete(id) return nil } @@ -2075,7 +2102,7 @@ func (b *InMemoryBackend) DeleteAPIKey(id string) error { func (b *InMemoryBackend) UpdateAPIKey(id string, input UpdateAPIKeyInput) (*APIKey, error) { b.mu.Lock("UpdateAPIKey") defer b.mu.Unlock() - key, ok := b.apiKeys[id] + key, ok := b.apiKeys.Get(id) if !ok { return nil, fmt.Errorf("%w: API key %s not found", ErrAPIKeyNotFound, id) } @@ -2098,7 +2125,7 @@ func (b *InMemoryBackend) UpdateAPIKey(id string, input UpdateAPIKeyInput) (*API func (b *InMemoryBackend) GetDomainName(name string) (*DomainName, error) { b.mu.RLock("GetDomainName") defer b.mu.RUnlock() - dn, ok := b.domainNames[name] + dn, ok := b.domainNames.Get(name) if !ok { return nil, fmt.Errorf("%w: domain name %s not found", ErrDomainNameNotFound, name) } @@ -2111,8 +2138,8 @@ func (b *InMemoryBackend) GetDomainName(name string) (*DomainName, error) { func (b *InMemoryBackend) GetDomainNames() ([]DomainName, error) { b.mu.RLock("GetDomainNames") defer b.mu.RUnlock() - all := make([]DomainName, 0, len(b.domainNames)) - for _, dn := range b.domainNames { + all := make([]DomainName, 0, b.domainNames.Len()) + for _, dn := range b.domainNames.All() { all = append(all, *dn) } sort.Slice(all, func(i, j int) bool { return all[i].DomainNameValue < all[j].DomainNameValue }) @@ -2124,10 +2151,9 @@ func (b *InMemoryBackend) GetDomainNames() ([]DomainName, error) { func (b *InMemoryBackend) DeleteDomainName(name string) error { b.mu.Lock("DeleteDomainName") defer b.mu.Unlock() - if _, ok := b.domainNames[name]; !ok { + if !b.domainNames.Delete(name) { return fmt.Errorf("%w: domain name %s not found", ErrDomainNameNotFound, name) } - delete(b.domainNames, name) return nil } @@ -2136,8 +2162,7 @@ func (b *InMemoryBackend) DeleteDomainName(name string) error { func (b *InMemoryBackend) GetBasePathMapping(domainName, basePath string) (*BasePathMapping, error) { b.mu.RLock("GetBasePathMapping") defer b.mu.RUnlock() - mapKey := domainName + "#" + basePath - bpm, ok := b.basePathMappings[mapKey] + bpm, ok := b.basePathMappings.Get(basePathMappingKey(domainName, basePath)) if !ok { return nil, fmt.Errorf( "%w: base path mapping not found for domain %q path %q", @@ -2157,8 +2182,8 @@ func (b *InMemoryBackend) GetBasePathMappings(domainName string) ([]BasePathMapp defer b.mu.RUnlock() var all []BasePathMapping prefix := domainName + "#" - for k, bpm := range b.basePathMappings { - if strings.HasPrefix(k, prefix) { + for _, bpm := range b.basePathMappings.All() { + if strings.HasPrefix(basePathMappingKeyFn(bpm), prefix) { all = append(all, *bpm) } } @@ -2171,8 +2196,7 @@ func (b *InMemoryBackend) GetBasePathMappings(domainName string) ([]BasePathMapp func (b *InMemoryBackend) DeleteBasePathMapping(domainName, basePath string) error { b.mu.Lock("DeleteBasePathMapping") defer b.mu.Unlock() - mapKey := domainName + "#" + basePath - if _, ok := b.basePathMappings[mapKey]; !ok { + if !b.basePathMappings.Delete(basePathMappingKey(domainName, basePath)) { return fmt.Errorf( "%w: base path mapping not found for domain %q path %q", ErrBasePathMappingNotFound, @@ -2180,7 +2204,6 @@ func (b *InMemoryBackend) DeleteBasePathMapping(domainName, basePath string) err basePath, ) } - delete(b.basePathMappings, mapKey) return nil } @@ -2189,11 +2212,10 @@ func (b *InMemoryBackend) DeleteBasePathMapping(domainName, basePath string) err func (b *InMemoryBackend) GetModel(restAPIID, modelName string) (*Model, error) { b.mu.RLock("GetModel") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: %s", ErrRestAPINotFound, restAPIID) } - for _, m := range d.models { + for _, m := range b.modelsByAPI.Get(restAPIID) { if m.Name == modelName { cp := *m @@ -2208,12 +2230,12 @@ func (b *InMemoryBackend) GetModel(restAPIID, modelName string) (*Model, error) func (b *InMemoryBackend) GetModels(restAPIID string) ([]Model, error) { b.mu.RLock("GetModels") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: %s", ErrRestAPINotFound, restAPIID) } - all := make([]Model, 0, len(d.models)) - for _, m := range d.models { + group := b.modelsByAPI.Get(restAPIID) + all := make([]Model, 0, len(group)) + for _, m := range group { all = append(all, *m) } sort.Slice(all, func(i, j int) bool { return all[i].Name < all[j].Name }) @@ -2225,13 +2247,12 @@ func (b *InMemoryBackend) GetModels(restAPIID string) ([]Model, error) { func (b *InMemoryBackend) DeleteModel(restAPIID, modelName string) error { b.mu.Lock("DeleteModel") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return fmt.Errorf("%w: %s", ErrRestAPINotFound, restAPIID) } - for id, m := range d.models { + for _, m := range b.modelsByAPI.Get(restAPIID) { if m.Name == modelName { - delete(d.models, id) + b.models.Delete(modelKeyFn(m)) return nil } @@ -2244,11 +2265,10 @@ func (b *InMemoryBackend) DeleteModel(restAPIID, modelName string) error { func (b *InMemoryBackend) UpdateModel(restAPIID, modelName string, input UpdateModelInput) (*Model, error) { b.mu.Lock("UpdateModel") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: %s", ErrRestAPINotFound, restAPIID) } - for _, m := range d.models { + for _, m := range b.modelsByAPI.Get(restAPIID) { if m.Name == modelName { if input.Description != "" { m.Description = input.Description @@ -2269,11 +2289,10 @@ func (b *InMemoryBackend) UpdateModel(restAPIID, modelName string, input UpdateM func (b *InMemoryBackend) UpdateStage(restAPIID, stageName string, input UpdateStageInput) (*Stage, error) { b.mu.Lock("UpdateStage") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: %s", ErrRestAPINotFound, restAPIID) } - stage, ok := d.stages[stageName] + stage, ok := b.stages.Get(stageKey(restAPIID, stageName)) if !ok { return nil, fmt.Errorf("%w: stage %q not found", ErrStageNotFound, stageName) } @@ -2301,17 +2320,34 @@ func (b *InMemoryBackend) UpdateStage(restAPIID, stageName string, input UpdateS if input.ClientCertificateID != "" { stage.ClientCertificateID = input.ClientCertificateID } + if input.CacheClusterEnabled != nil { + stage.CacheClusterEnabled = *input.CacheClusterEnabled + stage.CacheClusterStatus = cacheClusterStatusFor(stage.CacheClusterEnabled) + } + if input.CacheClusterSize != "" { + stage.CacheClusterSize = input.CacheClusterSize + } stage.LastUpdatedDate = unixEpochTime{time.Now()} cp := *stage return &cp, nil } +// cacheClusterStatusFor derives the AWS CacheClusterStatus enum value +// ("AVAILABLE"/"NOT_AVAILABLE") from whether the stage's cache cluster is enabled. +func cacheClusterStatusFor(enabled bool) string { + if enabled { + return "AVAILABLE" + } + + return "NOT_AVAILABLE" +} + // GetUsagePlan retrieves a usage plan by ID. func (b *InMemoryBackend) GetUsagePlan(id string) (*UsagePlan, error) { b.mu.RLock("GetUsagePlan") defer b.mu.RUnlock() - p, ok := b.usagePlans[id] + p, ok := b.usagePlans.Get(id) if !ok { return nil, fmt.Errorf("%w: usage plan %s not found", ErrUsagePlanNotFound, id) } @@ -2324,8 +2360,8 @@ func (b *InMemoryBackend) GetUsagePlan(id string) (*UsagePlan, error) { func (b *InMemoryBackend) GetUsagePlans() ([]UsagePlan, error) { b.mu.RLock("GetUsagePlans") defer b.mu.RUnlock() - all := make([]UsagePlan, 0, len(b.usagePlans)) - for _, p := range b.usagePlans { + all := make([]UsagePlan, 0, b.usagePlans.Len()) + for _, p := range b.usagePlans.All() { all = append(all, *p) } sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID }) @@ -2337,11 +2373,12 @@ func (b *InMemoryBackend) GetUsagePlans() ([]UsagePlan, error) { func (b *InMemoryBackend) DeleteUsagePlan(id string) error { b.mu.Lock("DeleteUsagePlan") defer b.mu.Unlock() - if _, ok := b.usagePlans[id]; !ok { + if !b.usagePlans.Delete(id) { return fmt.Errorf("%w: usage plan %s not found", ErrUsagePlanNotFound, id) } - delete(b.usagePlans, id) - delete(b.usagePlanKeys, id) + for _, k := range append([]*UsagePlanKey{}, b.usagePlanKeysByPlan.Get(id)...) { + b.usagePlanKeys.Delete(usagePlanKeyKeyFn(k)) + } return nil } @@ -2350,11 +2387,10 @@ func (b *InMemoryBackend) DeleteUsagePlan(id string) error { func (b *InMemoryBackend) GetUsagePlanKey(usagePlanID, keyID string) (*UsagePlanKey, error) { b.mu.RLock("GetUsagePlanKey") defer b.mu.RUnlock() - keys, ok := b.usagePlanKeys[usagePlanID] - if !ok { + if !b.usagePlans.Has(usagePlanID) { return nil, fmt.Errorf("%w: usage plan %s not found", ErrUsagePlanNotFound, usagePlanID) } - k, ok := keys[keyID] + k, ok := b.usagePlanKeys.Get(usagePlanKeyKey(usagePlanID, keyID)) if !ok { return nil, fmt.Errorf("%w: usage plan key %s not found", ErrUsagePlanKeyNotFound, keyID) } @@ -2367,12 +2403,12 @@ func (b *InMemoryBackend) GetUsagePlanKey(usagePlanID, keyID string) (*UsagePlan func (b *InMemoryBackend) GetUsagePlanKeys(usagePlanID string) ([]UsagePlanKey, error) { b.mu.RLock("GetUsagePlanKeys") defer b.mu.RUnlock() - keys, ok := b.usagePlanKeys[usagePlanID] - if !ok { + if !b.usagePlans.Has(usagePlanID) { return nil, fmt.Errorf("%w: usage plan %s not found", ErrUsagePlanNotFound, usagePlanID) } - all := make([]UsagePlanKey, 0, len(keys)) - for _, k := range keys { + group := b.usagePlanKeysByPlan.Get(usagePlanID) + all := make([]UsagePlanKey, 0, len(group)) + for _, k := range group { all = append(all, *k) } sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID }) @@ -2384,14 +2420,12 @@ func (b *InMemoryBackend) GetUsagePlanKeys(usagePlanID string) ([]UsagePlanKey, func (b *InMemoryBackend) DeleteUsagePlanKey(usagePlanID, keyID string) error { b.mu.Lock("DeleteUsagePlanKey") defer b.mu.Unlock() - keys, ok := b.usagePlanKeys[usagePlanID] - if !ok { + if !b.usagePlans.Has(usagePlanID) { return fmt.Errorf("%w: usage plan %s not found", ErrUsagePlanNotFound, usagePlanID) } - if _, exists := keys[keyID]; !exists { + if !b.usagePlanKeys.Delete(usagePlanKeyKey(usagePlanID, keyID)) { return fmt.Errorf("%w: usage plan key %s not found", ErrUsagePlanKeyNotFound, keyID) } - delete(keys, keyID) return nil } @@ -2400,11 +2434,10 @@ func (b *InMemoryBackend) DeleteUsagePlanKey(usagePlanID, keyID string) error { func (b *InMemoryBackend) GetDocumentationPart(restAPIID, docPartID string) (*DocumentationPart, error) { b.mu.RLock("GetDocumentationPart") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: %s", ErrRestAPINotFound, restAPIID) } - p, ok := d.documentationParts[docPartID] + p, ok := b.documentationParts.Get(documentationPartKey(restAPIID, docPartID)) if !ok { return nil, fmt.Errorf("%w: documentation part %s not found", ErrDocumentationPartNotFound, docPartID) } @@ -2417,12 +2450,12 @@ func (b *InMemoryBackend) GetDocumentationPart(restAPIID, docPartID string) (*Do func (b *InMemoryBackend) GetDocumentationParts(restAPIID string) ([]DocumentationPart, error) { b.mu.RLock("GetDocumentationParts") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: %s", ErrRestAPINotFound, restAPIID) } - all := make([]DocumentationPart, 0, len(d.documentationParts)) - for _, p := range d.documentationParts { + group := b.documentationPartsByAPI.Get(restAPIID) + all := make([]DocumentationPart, 0, len(group)) + for _, p := range group { all = append(all, *p) } sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID }) @@ -2434,14 +2467,12 @@ func (b *InMemoryBackend) GetDocumentationParts(restAPIID string) ([]Documentati func (b *InMemoryBackend) DeleteDocumentationPart(restAPIID, docPartID string) error { b.mu.Lock("DeleteDocumentationPart") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return fmt.Errorf("%w: %s", ErrRestAPINotFound, restAPIID) } - if _, exists := d.documentationParts[docPartID]; !exists { + if !b.documentationParts.Delete(documentationPartKey(restAPIID, docPartID)) { return fmt.Errorf("%w: documentation part %s not found", ErrDocumentationPartNotFound, docPartID) } - delete(d.documentationParts, docPartID) return nil } @@ -2450,11 +2481,10 @@ func (b *InMemoryBackend) DeleteDocumentationPart(restAPIID, docPartID string) e func (b *InMemoryBackend) GetDocumentationVersion(restAPIID, version string) (*DocumentationVersion, error) { b.mu.RLock("GetDocumentationVersion") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: %s", ErrRestAPINotFound, restAPIID) } - v, ok := d.documentationVersions[version] + v, ok := b.documentationVersions.Get(documentationVersionKey(restAPIID, version)) if !ok { return nil, fmt.Errorf("%w: documentation version %q not found", ErrDocumentationVersionNotFound, version) } @@ -2467,12 +2497,12 @@ func (b *InMemoryBackend) GetDocumentationVersion(restAPIID, version string) (*D func (b *InMemoryBackend) GetDocumentationVersions(restAPIID string) ([]DocumentationVersion, error) { b.mu.RLock("GetDocumentationVersions") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: %s", ErrRestAPINotFound, restAPIID) } - all := make([]DocumentationVersion, 0, len(d.documentationVersions)) - for _, v := range d.documentationVersions { + group := b.documentationVersionsByAPI.Get(restAPIID) + all := make([]DocumentationVersion, 0, len(group)) + for _, v := range group { all = append(all, *v) } sort.Slice(all, func(i, j int) bool { return all[i].Version < all[j].Version }) @@ -2484,14 +2514,12 @@ func (b *InMemoryBackend) GetDocumentationVersions(restAPIID string) ([]Document func (b *InMemoryBackend) DeleteDocumentationVersion(restAPIID, version string) error { b.mu.Lock("DeleteDocumentationVersion") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return fmt.Errorf("%w: %s", ErrRestAPINotFound, restAPIID) } - if _, exists := d.documentationVersions[version]; !exists { + if !b.documentationVersions.Delete(documentationVersionKey(restAPIID, version)) { return fmt.Errorf("%w: documentation version %q not found", ErrDocumentationVersionNotFound, version) } - delete(d.documentationVersions, version) return nil } @@ -2501,40 +2529,40 @@ func (b *InMemoryBackend) UpdateRestAPI(restAPIID string, input UpdateRestAPIInp b.mu.Lock("UpdateRestAPI") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] + api, ok := b.restApis.Get(restAPIID) if !ok { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } if input.Name != "" { - d.api.Name = input.Name + api.Name = input.Name } if input.Description != "" { - d.api.Description = input.Description + api.Description = input.Description } if input.Policy != "" { - d.api.Policy = input.Policy + api.Policy = input.Policy } if input.APIKeySource != "" { - d.api.APIKeySource = input.APIKeySource + api.APIKeySource = input.APIKeySource } if input.BinaryMediaTypes != nil { - d.api.BinaryMediaTypes = input.BinaryMediaTypes + api.BinaryMediaTypes = input.BinaryMediaTypes } if input.EndpointConfiguration != nil { - d.api.EndpointConfiguration = input.EndpointConfiguration + api.EndpointConfiguration = input.EndpointConfiguration } if input.MinimumCompressionSize != nil { - d.api.MinimumCompressionSize = *input.MinimumCompressionSize + api.MinimumCompressionSize = *input.MinimumCompressionSize } - cp := d.api + cp := *api return &cp, nil } @@ -2544,12 +2572,11 @@ func (b *InMemoryBackend) UpdateResource(restAPIID, resourceID string, input Upd b.mu.Lock("UpdateResource") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - res, ok := d.resources[resourceID] + res, ok := b.resources.Get(resourceKey(restAPIID, resourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, resourceID) } @@ -2557,14 +2584,14 @@ func (b *InMemoryBackend) UpdateResource(restAPIID, resourceID string, input Upd if input.PathPart != "" { var parentPath string if res.ParentID != "" { - if parent, exists := d.resources[res.ParentID]; exists { + if parent, exists := b.resources.Get(resourceKey(restAPIID, res.ParentID)); exists { parentPath = parent.Path } } res.PathPart = input.PathPart res.Path = computePath(parentPath, input.PathPart) - d.resourceVersion++ + b.resourceVersions[restAPIID]++ } if input.CorsConfiguration != nil { @@ -2584,12 +2611,11 @@ func (b *InMemoryBackend) UpdateDeployment( b.mu.Lock("UpdateDeployment") defer b.mu.Unlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - depl, ok := d.deployments[deploymentID] + depl, ok := b.deployments.Get(deploymentKey(restAPIID, deploymentID)) if !ok { return nil, fmt.Errorf("%w: deployment %s not found", ErrDeploymentNotFound, deploymentID) } @@ -2626,12 +2652,12 @@ func (b *InMemoryBackend) GetResourceTags(resourceARN string) (map[string]string apiID := strings.Split(parts[1], "/")[0] - d, ok := b.apis[apiID] + api, ok := b.restApis.Get(apiID) if !ok { return map[string]string{}, nil } - return d.api.Tags.Clone(), nil + return api.Tags.Clone(), nil } // TagResource adds or updates tags on a resource identified by its ARN. @@ -2646,13 +2672,13 @@ func (b *InMemoryBackend) TagResource(resourceARN string, newTags map[string]str apiID := strings.Split(parts[1], "/")[0] - d, ok := b.apis[apiID] + api, ok := b.restApis.Get(apiID) if !ok { return fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, apiID) } for k, v := range newTags { - d.api.Tags.Set(k, v) + api.Tags.Set(k, v) } return nil @@ -2670,13 +2696,13 @@ func (b *InMemoryBackend) UntagResource(resourceARN string, tagKeys []string) er apiID := strings.Split(parts[1], "/")[0] - d, ok := b.apis[apiID] + api, ok := b.restApis.Get(apiID) if !ok { return fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, apiID) } for _, k := range tagKeys { - d.api.Tags.Delete(k) + api.Tags.Delete(k) } return nil @@ -2687,12 +2713,11 @@ func (b *InMemoryBackend) TestInvokeMethod(input TestInvokeMethodInput) (*TestIn b.mu.RLock("TestInvokeMethod") defer b.mu.RUnlock() - d, ok := b.apis[input.RestAPIID] - if !ok { + if !b.restApis.Has(input.RestAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } - r, ok := d.resources[input.ResourceID] + r, ok := b.resources.Get(resourceKey(input.RestAPIID, input.ResourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, input.ResourceID) } @@ -2726,8 +2751,8 @@ func (b *InMemoryBackend) GetAPIKeysPage(limit int, position string) ([]APIKey, b.mu.RLock("GetAPIKeysPage") defer b.mu.RUnlock() - all := make([]APIKey, 0, len(b.apiKeys)) - for _, k := range b.apiKeys { + all := make([]APIKey, 0, b.apiKeys.Len()) + for _, k := range b.apiKeys.All() { all = append(all, *k) } sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID }) @@ -2741,8 +2766,8 @@ func (b *InMemoryBackend) GetDomainNamesPage(limit int, position string) ([]Doma b.mu.RLock("GetDomainNamesPage") defer b.mu.RUnlock() - all := make([]DomainName, 0, len(b.domainNames)) - for _, d := range b.domainNames { + all := make([]DomainName, 0, b.domainNames.Len()) + for _, d := range b.domainNames.All() { all = append(all, *d) } sort.Slice(all, func(i, j int) bool { return all[i].DomainNameValue < all[j].DomainNameValue }) @@ -2756,8 +2781,8 @@ func (b *InMemoryBackend) GetUsagePlansPage(limit int, position string) ([]Usage b.mu.RLock("GetUsagePlansPage") defer b.mu.RUnlock() - all := make([]UsagePlan, 0, len(b.usagePlans)) - for _, p := range b.usagePlans { + all := make([]UsagePlan, 0, b.usagePlans.Len()) + for _, p := range b.usagePlans.All() { all = append(all, *p) } sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID }) @@ -2771,7 +2796,7 @@ func (b *InMemoryBackend) UpdateUsagePlan(input UpdateUsagePlanInput) (*UsagePla b.mu.Lock("UpdateUsagePlan") defer b.mu.Unlock() - p, ok := b.usagePlans[input.UsagePlanID] + p, ok := b.usagePlans.Get(input.UsagePlanID) if !ok { return nil, fmt.Errorf("%w: usage plan %s not found", ErrUsagePlanNotFound, input.UsagePlanID) } @@ -2792,7 +2817,7 @@ func (b *InMemoryBackend) UpdateUsagePlan(input UpdateUsagePlanInput) (*UsagePla p.Quota = input.Quota } - if len(input.APIStages) > 0 { + if input.APIStages != nil { p.APIStages = input.APIStages } @@ -2804,7 +2829,7 @@ func (b *InMemoryBackend) UpdateDomainName(input UpdateDomainNameInput) (*Domain b.mu.Lock("UpdateDomainName") defer b.mu.Unlock() - d, ok := b.domainNames[input.DomainName] + d, ok := b.domainNames.Get(input.DomainName) if !ok { return nil, fmt.Errorf("%w: domain name %s not found", ErrDomainNameNotFound, input.DomainName) } @@ -2833,8 +2858,8 @@ func (b *InMemoryBackend) UpdateBasePathMapping(input UpdateBasePathMappingInput b.mu.Lock("UpdateBasePathMapping") defer b.mu.Unlock() - key := input.DomainName + "#" + input.BasePath - m, ok := b.basePathMappings[key] + key := basePathMappingKey(input.DomainName, input.BasePath) + m, ok := b.basePathMappings.Get(key) if !ok { return nil, fmt.Errorf("%w: base path mapping %s/%s not found", ErrNotFound, input.DomainName, input.BasePath) } @@ -2855,12 +2880,11 @@ func (b *InMemoryBackend) UpdateDocumentationPart(input UpdateDocumentationPartI b.mu.Lock("UpdateDocumentationPart") defer b.mu.Unlock() - d, ok := b.apis[input.RestAPIID] - if !ok { + if !b.restApis.Has(input.RestAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } - part, ok := d.documentationParts[input.DocPartID] + part, ok := b.documentationParts.Get(documentationPartKey(input.RestAPIID, input.DocPartID)) if !ok { return nil, fmt.Errorf("%w: documentation part %s not found", ErrNotFound, input.DocPartID) } @@ -2879,12 +2903,11 @@ func (b *InMemoryBackend) UpdateDocumentationVersion( b.mu.Lock("UpdateDocumentationVersion") defer b.mu.Unlock() - d, ok := b.apis[input.RestAPIID] - if !ok { + if !b.restApis.Has(input.RestAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } - ver, ok := d.documentationVersions[input.DocumentationVersion] + ver, ok := b.documentationVersions.Get(documentationVersionKey(input.RestAPIID, input.DocumentationVersion)) if !ok { return nil, fmt.Errorf("%w: documentation version %s not found", ErrNotFound, input.DocumentationVersion) } @@ -2901,12 +2924,11 @@ func (b *InMemoryBackend) UpdateMethod(input UpdateMethodInput) (*Method, error) b.mu.Lock("UpdateMethod") defer b.mu.Unlock() - d, ok := b.apis[input.RestAPIID] - if !ok { + if !b.restApis.Has(input.RestAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } - r, ok := d.resources[input.ResourceID] + r, ok := b.resources.Get(resourceKey(input.RestAPIID, input.ResourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, input.ResourceID) } @@ -2944,12 +2966,11 @@ func (b *InMemoryBackend) UpdateIntegration(input UpdateIntegrationInput) (*Inte b.mu.Lock("UpdateIntegration") defer b.mu.Unlock() - d, ok := b.apis[input.RestAPIID] - if !ok { + if !b.restApis.Has(input.RestAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } - r, ok := d.resources[input.ResourceID] + r, ok := b.resources.Get(resourceKey(input.RestAPIID, input.ResourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, input.ResourceID) } @@ -3017,12 +3038,11 @@ func (b *InMemoryBackend) UpdateIntegrationResponse( b.mu.Lock("UpdateIntegrationResponse") defer b.mu.Unlock() - d, ok := b.apis[input.RestAPIID] - if !ok { + if !b.restApis.Has(input.RestAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } - r, ok := d.resources[input.ResourceID] + r, ok := b.resources.Get(resourceKey(input.RestAPIID, input.ResourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, input.ResourceID) } @@ -3067,12 +3087,11 @@ func (b *InMemoryBackend) UpdateMethodResponse(input UpdateMethodResponseInput) b.mu.Lock("UpdateMethodResponse") defer b.mu.Unlock() - d, ok := b.apis[input.RestAPIID] - if !ok { + if !b.restApis.Has(input.RestAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } - r, ok := d.resources[input.ResourceID] + r, ok := b.resources.Get(resourceKey(input.RestAPIID, input.ResourceID)) if !ok { return nil, fmt.Errorf("%w: resource %s not found", ErrResourceNotFound, input.ResourceID) } @@ -3108,6 +3127,9 @@ func (b *InMemoryBackend) UpdateAccount(input UpdateAccountInput) (*Account, err if input.ThrottleSettings != nil { b.account.ThrottleSettings = input.ThrottleSettings } + if input.CloudwatchRoleARN != "" { + b.account.CloudwatchRoleARN = input.CloudwatchRoleARN + } return b.account, nil } @@ -3117,12 +3139,11 @@ func (b *InMemoryBackend) TestInvokeAuthorizer(input TestInvokeAuthorizerInput) b.mu.RLock("TestInvokeAuthorizer") defer b.mu.RUnlock() - d, ok := b.apis[input.RestAPIID] - if !ok { + if !b.restApis.Has(input.RestAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } - if _, authOK := d.authorizers[input.AuthorizerID]; !authOK { + if !b.authorizers.Has(authorizerKey(input.RestAPIID, input.AuthorizerID)) { return nil, fmt.Errorf("%w: authorizer %s not found", ErrNotFound, input.AuthorizerID) } @@ -3141,13 +3162,12 @@ func (b *InMemoryBackend) GetModelTemplate(restAPIID, modelName string) (string, b.mu.RLock("GetModelTemplate") defer b.mu.RUnlock() - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { return "", fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } var model *Model - for _, m := range d.models { + for _, m := range b.modelsByAPI.Get(restAPIID) { if m.Name == modelName { model = m @@ -3177,7 +3197,7 @@ func (b *InMemoryBackend) GetGatewayResponse(restAPIID, responseType string) (*G defer b.mu.RUnlock() key := gatewayResponseKey(restAPIID, responseType) - gr, ok := b.gatewayResponses[key] + gr, ok := b.gatewayResponses.Get(key) if !ok { // Return default response (AWS returns default responses even when not explicitly set). return &GatewayResponse{ @@ -3220,7 +3240,7 @@ func (b *InMemoryBackend) GetGatewayResponses(restAPIID string) ([]GatewayRespon b.mu.RLock("GetGatewayResponses") defer b.mu.RUnlock() - if _, ok := b.apis[restAPIID]; !ok { + if !b.restApis.Has(restAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } @@ -3236,7 +3256,7 @@ func (b *InMemoryBackend) GetGatewayResponses(restAPIID string) ([]GatewayRespon for _, rt := range defaultTypes { key := gatewayResponseKey(restAPIID, rt) - if gr, ok := b.gatewayResponses[key]; ok { + if gr, ok := b.gatewayResponses.Get(key); ok { result = append(result, *gr) } else { result = append(result, GatewayResponse{ @@ -3256,12 +3276,10 @@ func (b *InMemoryBackend) PutGatewayResponse(input PutGatewayResponseInput) (*Ga b.mu.Lock("PutGatewayResponse") defer b.mu.Unlock() - if _, ok := b.apis[input.RestAPIID]; !ok { + if !b.restApis.Has(input.RestAPIID) { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } - key := gatewayResponseKey(input.RestAPIID, input.ResponseType) - gr := &GatewayResponse{ RestAPIID: input.RestAPIID, ResponseType: input.ResponseType, @@ -3275,22 +3293,65 @@ func (b *InMemoryBackend) PutGatewayResponse(input PutGatewayResponseInput) (*Ga gr.StatusCode = gatewayResponseDefaultStatus(input.ResponseType) } - b.gatewayResponses[key] = gr + b.gatewayResponses.Put(gr) return gr, nil } +// UpdateGatewayResponse applies a partial (PATCH) update to a gateway response, +// merging only the fields present in input with the existing response (or with +// AWS's implicit default response for responseType, if none has been customized +// yet). Unlike PutGatewayResponse — which is a full wholesale replace used by the +// real PUT operation — UpdateGatewayResponse must not clobber +// ResponseParameters/ResponseTemplates/StatusCode that weren't part of this PATCH +// document, matching AWS's PATCH-operation semantics for this resource. +func (b *InMemoryBackend) UpdateGatewayResponse(input PutGatewayResponseInput) (*GatewayResponse, error) { + b.mu.Lock("UpdateGatewayResponse") + defer b.mu.Unlock() + + if !b.restApis.Has(input.RestAPIID) { + return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) + } + + key := gatewayResponseKey(input.RestAPIID, input.ResponseType) + + existing, ok := b.gatewayResponses.Get(key) + if !ok { + existing = &GatewayResponse{ + RestAPIID: input.RestAPIID, + ResponseType: input.ResponseType, + StatusCode: gatewayResponseDefaultStatus(input.ResponseType), + DefaultResponse: true, + } + } + + cp := *existing + if input.StatusCode != "" { + cp.StatusCode = input.StatusCode + } + if input.ResponseParameters != nil { + cp.ResponseParameters = input.ResponseParameters + } + if input.ResponseTemplates != nil { + cp.ResponseTemplates = input.ResponseTemplates + } + cp.DefaultResponse = false + b.gatewayResponses.Put(&cp) + + return &cp, nil +} + // DeleteGatewayResponse removes a custom gateway response, reverting to default. func (b *InMemoryBackend) DeleteGatewayResponse(restAPIID, responseType string) error { b.mu.Lock("DeleteGatewayResponse") defer b.mu.Unlock() - if _, ok := b.apis[restAPIID]; !ok { + if !b.restApis.Has(restAPIID) { return fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } key := gatewayResponseKey(restAPIID, responseType) - delete(b.gatewayResponses, key) + b.gatewayResponses.Delete(key) return nil } @@ -3310,7 +3371,7 @@ func (b *InMemoryBackend) GenerateClientCertificate(input GenerateClientCertific ExpirationDate: unixEpochTime{now.AddDate(0, 0, clientCertValidityDays)}, } - b.clientCertificates[id] = cert + b.clientCertificates.Put(cert) return cert, nil } @@ -3320,7 +3381,7 @@ func (b *InMemoryBackend) GetClientCertificate(id string) (*ClientCertificate, e b.mu.RLock("GetClientCertificate") defer b.mu.RUnlock() - cert, ok := b.clientCertificates[id] + cert, ok := b.clientCertificates.Get(id) if !ok { return nil, fmt.Errorf("%w: client certificate %s not found", ErrNotFound, id) } @@ -3333,8 +3394,9 @@ func (b *InMemoryBackend) GetClientCertificates() ([]ClientCertificate, error) { b.mu.RLock("GetClientCertificates") defer b.mu.RUnlock() - result := make([]ClientCertificate, 0, len(b.clientCertificates)) - for _, c := range b.clientCertificates { + all := b.clientCertificates.All() + result := make([]ClientCertificate, 0, len(all)) + for _, c := range all { result = append(result, *c) } @@ -3350,12 +3412,10 @@ func (b *InMemoryBackend) DeleteClientCertificate(id string) error { b.mu.Lock("DeleteClientCertificate") defer b.mu.Unlock() - if _, ok := b.clientCertificates[id]; !ok { + if !b.clientCertificates.Delete(id) { return fmt.Errorf("%w: client certificate %s not found", ErrNotFound, id) } - delete(b.clientCertificates, id) - return nil } @@ -3369,14 +3429,15 @@ func (b *InMemoryBackend) GetUsage(input GetUsageInput) (*UsageData, error) { b.mu.RLock("GetUsage") defer b.mu.RUnlock() - plan, ok := b.usagePlans[input.UsagePlanID] + plan, ok := b.usagePlans.Get(input.UsagePlanID) if !ok { return nil, fmt.Errorf("%w: usage plan %s not found", ErrUsagePlanNotFound, input.UsagePlanID) } items := make(map[string][]any) - for keyID := range b.usagePlanKeys[input.UsagePlanID] { + for _, upk := range b.usagePlanKeysByPlan.Get(input.UsagePlanID) { + keyID := upk.ID used, remaining := b.usage.usageForKey(plan, keyID) if override, hasOverride := b.usageOverrides[input.UsagePlanID][keyID]; hasOverride { remaining = int(override) @@ -3413,15 +3474,16 @@ func (b *InMemoryBackend) EnforceUsagePlan(apiID, stageName, keyID string) error // associated with the given api:stage, or nil. Callers must hold b.mu. func (b *InMemoryBackend) usagePlanForKeyLocked(keyID, apiID, stageName string) *UsagePlan { // Deterministic iteration by plan ID so a key in multiple plans resolves stably. - ids := make([]string, 0, len(b.usagePlans)) - for id := range b.usagePlans { - ids = append(ids, id) + all := b.usagePlans.All() + ids := make([]string, 0, len(all)) + for _, p := range all { + ids = append(ids, p.ID) } sort.Strings(ids) for _, id := range ids { - plan := b.usagePlans[id] - if _, associated := b.usagePlanKeys[id][keyID]; !associated { + plan, _ := b.usagePlans.Get(id) + if !b.usagePlanKeys.Has(usagePlanKeyKey(id, keyID)) { continue } for i := range plan.APIStages { @@ -3440,7 +3502,7 @@ func (b *InMemoryBackend) UpdateClientCertificate(input UpdateClientCertificateI b.mu.Lock("UpdateClientCertificate") defer b.mu.Unlock() - cert, ok := b.clientCertificates[input.ClientCertificateID] + cert, ok := b.clientCertificates.Get(input.ClientCertificateID) if !ok { return nil, fmt.Errorf("%w: client certificate %s not found", ErrNotFound, input.ClientCertificateID) } @@ -3470,7 +3532,7 @@ func (b *InMemoryBackend) CreateVpcLink(input CreateVpcLinkInput) (*VpcLink, err b.mu.Lock("CreateVpcLink") defer b.mu.Unlock() - b.vpcLinks[id] = link + b.vpcLinks.Put(link) return link, nil } @@ -3480,7 +3542,7 @@ func (b *InMemoryBackend) GetVpcLink(id string) (*VpcLink, error) { b.mu.RLock("GetVpcLink") defer b.mu.RUnlock() - link, ok := b.vpcLinks[id] + link, ok := b.vpcLinks.Get(id) if !ok { return nil, fmt.Errorf("%w: VPC link %s not found", ErrNotFound, id) } @@ -3493,8 +3555,9 @@ func (b *InMemoryBackend) GetVpcLinks() ([]VpcLink, error) { b.mu.RLock("GetVpcLinks") defer b.mu.RUnlock() - result := make([]VpcLink, 0, len(b.vpcLinks)) - for _, link := range b.vpcLinks { + all := b.vpcLinks.All() + result := make([]VpcLink, 0, len(all)) + for _, link := range all { result = append(result, *link) } @@ -3510,12 +3573,10 @@ func (b *InMemoryBackend) DeleteVpcLink(id string) error { b.mu.Lock("DeleteVpcLink") defer b.mu.Unlock() - if _, ok := b.vpcLinks[id]; !ok { + if !b.vpcLinks.Delete(id) { return fmt.Errorf("%w: VPC link %s not found", ErrNotFound, id) } - delete(b.vpcLinks, id) - return nil } @@ -3524,7 +3585,7 @@ func (b *InMemoryBackend) UpdateVpcLink(input UpdateVpcLinkInput) (*VpcLink, err b.mu.Lock("UpdateVpcLink") defer b.mu.Unlock() - link, ok := b.vpcLinks[input.VpcLinkID] + link, ok := b.vpcLinks.Get(input.VpcLinkID) if !ok { return nil, fmt.Errorf("%w: VPC link %s not found", ErrNotFound, input.VpcLinkID) } @@ -3546,21 +3607,34 @@ func (b *InMemoryBackend) GetExport(restAPIID, stageName, exportType string) (ma b.mu.RLock("GetExport") defer b.mu.RUnlock() - data, ok := b.apis[restAPIID] + api, ok := b.restApis.Get(restAPIID) if !ok { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } + ctx := exportContext{b: b, restAPIID: restAPIID, apiName: api.Name} if exportType == "oas30" { - return buildOAS30Export(data, stageName), nil + return buildOAS30Export(ctx, stageName), nil } - return buildSwagger20Export(data, stageName), nil + return buildSwagger20Export(ctx, stageName), nil +} + +// exportContext carries the read-only context buildSwagger20Export/buildOAS30Export +// and their helpers need to look up a REST API's resources/models from the +// backend's flat store.Table collections. It replaces the old *apiData +// parameter that these free functions took before the pkgs/store conversion +// flattened per-API nested maps into backend-level tables keyed by +// "#". +type exportContext struct { + b *InMemoryBackend + restAPIID string + apiName string } // buildSwagger20Export constructs a Swagger 2.0 export document. -func buildSwagger20Export(data *apiData, stageName string) map[string]any { - paths := buildExportPaths(data, false) +func buildSwagger20Export(ctx exportContext, stageName string) map[string]any { + paths := buildExportPaths(ctx, false) secDefs := map[string]any{ exportKeyAPIKey: map[string]any{ @@ -3572,7 +3646,7 @@ func buildSwagger20Export(data *apiData, stageName string) map[string]any { return map[string]any{ "swagger": "2.0", - "info": map[string]any{"title": data.api.Name, "version": "1.0"}, + "info": map[string]any{"title": ctx.apiName, "version": "1.0"}, "basePath": "/" + stageName, "paths": paths, "securityDefinitions": secDefs, @@ -3580,8 +3654,8 @@ func buildSwagger20Export(data *apiData, stageName string) map[string]any { } // buildOAS30Export constructs an OpenAPI 3.0.1 export document. -func buildOAS30Export(data *apiData, stageName string) map[string]any { - paths := buildExportPaths(data, true) +func buildOAS30Export(ctx exportContext, stageName string) map[string]any { + paths := buildExportPaths(ctx, true) components := map[string]any{ "securitySchemes": map[string]any{ @@ -3594,10 +3668,11 @@ func buildOAS30Export(data *apiData, stageName string) map[string]any { } // Include model schemas in components. - if len(data.models) > 0 { - schemas := make(map[string]any, len(data.models)) - for name, m := range data.models { - schemas[name] = map[string]any{ + models := ctx.b.modelsByAPI.Get(ctx.restAPIID) + if len(models) > 0 { + schemas := make(map[string]any, len(models)) + for _, m := range models { + schemas[m.Name] = map[string]any{ exportKeyDescription: m.Description, exportKeyType: exportKeyObject, } @@ -3607,7 +3682,7 @@ func buildOAS30Export(data *apiData, stageName string) map[string]any { return map[string]any{ "openapi": "3.0.1", - "info": map[string]any{"title": data.api.Name, "version": "1.0"}, + "info": map[string]any{"title": ctx.apiName, "version": "1.0"}, "servers": []map[string]any{{"url": "/" + stageName}}, "paths": paths, "components": components, @@ -3616,10 +3691,10 @@ func buildOAS30Export(data *apiData, stageName string) map[string]any { // buildExportPaths constructs the paths object for an OpenAPI export. // oas30=true emits OAS 3.0 operation objects; false emits Swagger 2.0. -func buildExportPaths(data *apiData, oas30 bool) map[string]any { +func buildExportPaths(ctx exportContext, oas30 bool) map[string]any { paths := make(map[string]any) - for _, res := range data.resources { + for _, res := range ctx.b.resourcesByAPI.Get(ctx.restAPIID) { if res.Path == "/" || len(res.ResourceMethods) == 0 { continue } @@ -3631,7 +3706,7 @@ func buildExportPaths(data *apiData, oas30 bool) map[string]any { continue } - op := buildExportOperation(data, method, oas30) + op := buildExportOperation(ctx, method, oas30) pathItem[strings.ToLower(httpMethod)] = op } @@ -3644,10 +3719,10 @@ func buildExportPaths(data *apiData, oas30 bool) map[string]any { } // buildExportOperation constructs a single OAS operation object for a method. -func buildExportOperation(data *apiData, method *Method, oas30 bool) map[string]any { +func buildExportOperation(ctx exportContext, method *Method, oas30 bool) map[string]any { op := make(map[string]any) - op["responses"] = buildExportResponses(data, method, oas30) - buildExportRequestBody(op, data, method, oas30) + op["responses"] = buildExportResponses(ctx, method, oas30) + buildExportRequestBody(op, ctx, method, oas30) buildExportSecurity(op, method) if method.OperationName != "" { @@ -3666,7 +3741,7 @@ func buildExportOperation(data *apiData, method *Method, oas30 bool) map[string] } // buildExportResponses constructs the responses map for an OAS operation. -func buildExportResponses(data *apiData, method *Method, oas30 bool) map[string]any { +func buildExportResponses(ctx exportContext, method *Method, oas30 bool) map[string]any { responses := make(map[string]any) for statusCode, mr := range method.MethodResponses { @@ -3676,12 +3751,12 @@ func buildExportResponses(data *apiData, method *Method, oas30 bool) map[string] if oas30 { content := make(map[string]any) for ct, modelName := range mr.ResponseModels { - content[ct] = map[string]any{exportKeySchema: buildModelRef(data, modelName, oas30)} + content[ct] = map[string]any{exportKeySchema: buildModelRef(ctx, modelName, oas30)} } rsp["content"] = content } else { for _, modelName := range mr.ResponseModels { - rsp[exportKeySchema] = buildModelRef(data, modelName, oas30) + rsp[exportKeySchema] = buildModelRef(ctx, modelName, oas30) break } @@ -3699,7 +3774,7 @@ func buildExportResponses(data *apiData, method *Method, oas30 bool) map[string] } // buildExportRequestBody adds request body / request model entries to an operation map. -func buildExportRequestBody(op map[string]any, data *apiData, method *Method, oas30 bool) { +func buildExportRequestBody(op map[string]any, ctx exportContext, method *Method, oas30 bool) { if len(method.RequestModels) == 0 { return } @@ -3707,7 +3782,7 @@ func buildExportRequestBody(op map[string]any, data *apiData, method *Method, oa if oas30 { content := make(map[string]any) for ct, modelName := range method.RequestModels { - content[ct] = map[string]any{exportKeySchema: buildModelRef(data, modelName, oas30)} + content[ct] = map[string]any{exportKeySchema: buildModelRef(ctx, modelName, oas30)} } op["requestBody"] = map[string]any{"content": content} } else { @@ -3717,7 +3792,7 @@ func buildExportRequestBody(op map[string]any, data *apiData, method *Method, oa { "in": exportKeyBody, "name": exportKeyBody, - exportKeySchema: buildModelRef(data, modelName, oas30), + exportKeySchema: buildModelRef(ctx, modelName, oas30), }, } @@ -3782,8 +3857,16 @@ func buildExportIntegration(integ *Integration) map[string]any { } // buildModelRef returns a schema reference or inline schema for a model name. -func buildModelRef(data *apiData, modelName string, oas30 bool) map[string]any { - m, ok := data.models[modelName] +// +// NOTE: this preserves a pre-existing quirk carried over mechanically from the +// map[string]*apiData days: the backend's models table is keyed by model ID +// (see modelKeyFn), but modelName here is the model's *Name*, not its ID -- so +// this lookup only succeeds when a model's Name happens to equal its ID. This +// was already the case before the store.Table conversion (the old +// data.models[modelName] indexed an ID-keyed map by name) and is preserved +// byte-for-byte rather than "fixed" as part of a storage-layer swap. +func buildModelRef(ctx exportContext, modelName string, oas30 bool) map[string]any { + m, ok := ctx.b.models.Get(modelKey(ctx.restAPIID, modelName)) if !ok { return map[string]any{exportKeyType: exportKeyObject} } diff --git a/services/apigateway/backend_destub.go b/services/apigateway/backend_destub.go index 4eecfde2d..f3fd67b70 100644 --- a/services/apigateway/backend_destub.go +++ b/services/apigateway/backend_destub.go @@ -72,12 +72,12 @@ func (b *InMemoryBackend) GetSdk(restAPIID, stageName, sdkType string) (*SdkExpo b.mu.RLock("GetSdk") defer b.mu.RUnlock() - data, ok := b.apis[restAPIID] + api, ok := b.restApis.Get(restAPIID) if !ok { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - if _, stageOK := data.stages[stageName]; !stageOK { + if !b.stages.Has(stageKey(restAPIID, stageName)) { return nil, fmt.Errorf("%w: stage %s not found", ErrStageNotFound, stageName) } @@ -85,7 +85,8 @@ func (b *InMemoryBackend) GetSdk(restAPIID, stageName, sdkType string) (*SdkExpo return nil, fmt.Errorf("%w: unsupported sdkType %q", ErrInvalidParameter, sdkType) } - spec := buildOAS30Export(data, stageName) + ctx := exportContext{b: b, restAPIID: restAPIID, apiName: api.Name} + spec := buildOAS30Export(ctx, stageName) specJSON, err := json.MarshalIndent(spec, "", " ") if err != nil { @@ -291,15 +292,16 @@ func (b *InMemoryBackend) ImportDocumentationParts( b.mu.Lock("ImportDocumentationParts") - d, ok := b.apis[restAPIID] - if !ok { + if !b.restApis.Has(restAPIID) { b.mu.Unlock() return nil, nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } if strings.EqualFold(mode, "overwrite") { - d.documentationParts = make(map[string]*DocumentationPart) + for _, p := range append([]*DocumentationPart{}, b.documentationPartsByAPI.Get(restAPIID)...) { + b.documentationParts.Delete(documentationPartKeyFn(p)) + } } b.mu.Unlock() @@ -350,13 +352,13 @@ func (b *InMemoryBackend) ImportDocumentationParts( func (b *InMemoryBackend) UpdateUsage(usagePlanID, keyID string, dateValues map[string]string) (*UsageData, error) { b.mu.Lock("UpdateUsage") - if _, ok := b.usagePlans[usagePlanID]; !ok { + if !b.usagePlans.Has(usagePlanID) { b.mu.Unlock() return nil, fmt.Errorf("%w: usage plan %s not found", ErrUsagePlanNotFound, usagePlanID) } - if _, ok := b.usagePlanKeys[usagePlanID][keyID]; !ok { + if !b.usagePlanKeys.Has(usagePlanKeyKey(usagePlanID, keyID)) { b.mu.Unlock() return nil, fmt.Errorf("%w: usage plan key %s not found", ErrUsagePlanKeyNotFound, keyID) diff --git a/services/apigateway/extra_coverage_test.go b/services/apigateway/extra_coverage_test.go index 8d104c955..576ab66eb 100644 --- a/services/apigateway/extra_coverage_test.go +++ b/services/apigateway/extra_coverage_test.go @@ -446,6 +446,10 @@ func (n *noopBackend) PutGatewayResponse(_ apigateway.PutGatewayResponseInput) ( return nil, errNoopNotImplemented } +func (n *noopBackend) UpdateGatewayResponse(_ apigateway.PutGatewayResponseInput) (*apigateway.GatewayResponse, error) { + return nil, errNoopNotImplemented +} + func (n *noopBackend) DeleteGatewayResponse(_ string, _ string) error { return errNoopNotImplemented } func (n *noopBackend) GenerateClientCertificate( @@ -606,8 +610,14 @@ func TestHandlerPersistence_NoopBackend(t *testing.T) { } } -// TestInMemoryBackend_RestoreWithNilMaps ensures that nil maps in the JSON snapshot -// are initialised to empty maps after Restore (covers the nil-map branches in Restore). +// TestInMemoryBackend_RestoreWithNilMaps ensures that a snapshot whose "dirty" +// tables (resources/deployments/stages -- see store_setup.go's +// registerAllTables doc) are explicitly null or entirely absent restores +// cleanly to empty tables rather than panicking. Pre-Phase-3.3 this exercised +// hand-rolled nil-map init logic in Restore; that logic is now handled +// generically by store.Registry.RestoreAll/store.Table.Restore (both treat +// nil/absent data as "reset to empty" -- see pkgs/store's docs), so this +// covers the same edge case against the current Tables-based snapshot shape. func TestInMemoryBackend_RestoreWithNilMaps(t *testing.T) { t.Parallel() @@ -617,12 +627,14 @@ func TestInMemoryBackend_RestoreWithNilMaps(t *testing.T) { }{ { name: "null_resources_deployments_stages", - snapshot: `{"apis":{"api1":{"api":{"id":"api1","name":"n","createdDate":0},` + - `"resources":null,"deployments":null,"stages":null}}}`, + snapshot: `{"version":1,"tables":{` + + `"restApis":[{"id":"api1","name":"n","createdDate":0}],` + + `"resources":null,"deployments":null,"stages":null}}`, }, { - name: "missing_inner_maps", - snapshot: `{"apis":{"api2":{"api":{"id":"api2","name":"m","createdDate":0}}}}`, + name: "missing_inner_tables", + snapshot: `{"version":1,"tables":{` + + `"restApis":[{"id":"api2","name":"m","createdDate":0}]}}`, }, } @@ -634,10 +646,11 @@ func TestInMemoryBackend_RestoreWithNilMaps(t *testing.T) { err := b.Restore(t.Context(), []byte(tt.snapshot)) require.NoError(t, err) - // The Restore should have initialised the empty maps – calling GetResources - // should succeed without a nil-pointer panic. + // The Restore should have initialised the empty tables – calling + // GetResources should succeed (the REST API itself was restored) without + // a nil-pointer panic, and report no resources. apiID := "api1" - if tt.name == "missing_inner_maps" { + if tt.name == "missing_inner_tables" { apiID = "api2" } diff --git a/services/apigateway/handler.go b/services/apigateway/handler.go index fa9fdd0bb..b30c35a46 100644 --- a/services/apigateway/handler.go +++ b/services/apigateway/handler.go @@ -1056,10 +1056,15 @@ func (h *Handler) handleRESTAPI(c *echo.Context) error { body = []byte("{}") } - // Convert RFC 6902 patch arrays to flat JSON objects so PATCH handlers can - // read fields directly (e.g. [{"op":"replace","path":"/name","value":"x"}] - // becomes {"name":"x"}). - body = normalizePatchBody(body) + // Convert RFC 6902 patch documents ({"patchOperations":[...]}, or a bare + // array) to flat JSON objects so PATCH handlers can read fields directly + // (e.g. [{"op":"replace","path":"/name","value":"x"}] becomes + // {"name":"x"}); see patch.go for the resource-specific map/struct/list + // merges this also handles (stage variables, canary promotion, per-route + // method settings, binary media types, usage-plan API stages, gateway + // response parameters/templates) that a flat single-field replace cannot + // express. + body = h.applyStructuredPatch(action, pathParams, body) // Merge path parameters into the JSON body so existing handlers can read them. for k, v := range pathParams { @@ -1145,91 +1150,6 @@ func detectImportRESTAPI( return "", nil, false } -// patchOp is a single RFC 6902 JSON patch operation, e.g. -// {"op":"replace","path":"/description","value":"foo"}. -type patchOp struct { - Op string `json:"op"` - Path string `json:"path"` - Value json.RawMessage `json:"value"` -} - -// normalizePatchBody converts a JSON patch array (RFC 6902) to a flat JSON object, -// so PATCH handlers can read fields directly. AWS API Gateway REST PATCH endpoints -// accept patch operations either as a bare array -// ([{"op":"replace","path":"/description","value":"foo"}]) or, per the real -// aws-sdk-go-v2 wire shape for operations like UpdateUsage/UpdateRestApi/UpdateStage, -// wrapped in a "patchOperations" object field -// ({"patchOperations":[{"op":"replace","path":"/description","value":"foo"}]}). -// Both forms are flattened to {"description":"foo"}. Bodies that are neither a -// patch array nor an object with a "patchOperations" array are returned unchanged. -func normalizePatchBody(body []byte) []byte { - if len(body) == 0 { - return body - } - - var flattened []byte - - switch body[0] { - case '[': - var ops []patchOp - if err := json.Unmarshal(body, &ops); err != nil || len(ops) == 0 || ops[0].Op == "" { - return body - } - - flattened = flattenPatchOps(ops, nil) - case '{': - var wrapper struct { - PatchOperations []patchOp `json:"patchOperations"` - } - - if err := json.Unmarshal(body, &wrapper); err != nil || len(wrapper.PatchOperations) == 0 { - return body - } - - var rest map[string]json.RawMessage - _ = json.Unmarshal(body, &rest) - delete(rest, "patchOperations") - - flattened = flattenPatchOps(wrapper.PatchOperations, rest) - default: - return body - } - - if flattened == nil { - return body - } - - return flattened -} - -// flattenPatchOps converts a list of RFC 6902 patch operations into a flat JSON -// object, layered on top of base (any other sibling fields already present in -// the request body; may be nil). Only "replace"/"add" ops are applied. -func flattenPatchOps(ops []patchOp, base map[string]json.RawMessage) []byte { - m := make(map[string]json.RawMessage, len(base)+len(ops)) - maps.Copy(m, base) - - for _, op := range ops { - if op.Op != "replace" && op.Op != "add" { - continue - } - - field := strings.TrimPrefix(op.Path, "/") - if field == "" { - continue - } - - m[field] = op.Value - } - - out, err := json.Marshal(m) - if err != nil { - return nil - } - - return out -} - // injectJSONFieldAPIGW merges a key/value string pair into a JSON object body. func injectJSONFieldAPIGW(body []byte, key, value string) []byte { var m map[string]json.RawMessage diff --git a/services/apigateway/handler_stubs.go b/services/apigateway/handler_stubs.go index eced1e2cb..47fd577a2 100644 --- a/services/apigateway/handler_stubs.go +++ b/services/apigateway/handler_stubs.go @@ -177,7 +177,7 @@ func (h *Handler) exportAndCertActions() map[string]actionFn { return 0, nil, err } - gr, err := h.Backend.PutGatewayResponse(input) + gr, err := h.Backend.UpdateGatewayResponse(input) if err != nil { return 0, nil, err } diff --git a/services/apigateway/import.go b/services/apigateway/import.go index 042b77d29..c33920049 100644 --- a/services/apigateway/import.go +++ b/services/apigateway/import.go @@ -198,23 +198,13 @@ func (b *InMemoryBackend) ImportRestAPI(input ImportRestAPIInput) (*RestAPI, err ResourceMethods: make(map[string]*Method), } - data := &apiData{ - api: api, - resources: map[string]*Resource{rootID: root}, - deployments: make(map[string]*Deployment), - stages: make(map[string]*Stage), - authorizers: make(map[string]*Authorizer), - requestValidators: make(map[string]*RequestValidator), - documentationParts: make(map[string]*DocumentationPart), - documentationVersions: make(map[string]*DocumentationVersion), - models: make(map[string]*Model), - } - b.apis[id] = data + b.restApis.Put(&api) + b.resources.Put(root) - importModels(data, doc) - importPaths(data, doc) + importModels(b, &api, doc) + importPaths(b, &api, doc) - cp := data.api + cp := api return &cp, nil } @@ -239,70 +229,83 @@ func (b *InMemoryBackend) PutRestAPI(input PutRestAPIInput) (*RestAPI, error) { b.mu.Lock("PutRestAPI") defer b.mu.Unlock() - data, ok := b.apis[input.RestAPIID] + api, ok := b.restApis.Get(input.RestAPIID) if !ok { return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, input.RestAPIID) } if mode == importModeOverwrite { - rootID := data.api.RootResourceID + for _, r := range append([]*Resource{}, b.resourcesByAPI.Get(api.ID)...) { + b.resources.Delete(resourceKeyFn(r)) + } root := &Resource{ - ID: rootID, + ID: api.RootResourceID, Path: "/", - RestAPIID: data.api.ID, + RestAPIID: api.ID, ResourceMethods: make(map[string]*Method), } - data.resources = map[string]*Resource{rootID: root} - data.models = make(map[string]*Model) + b.resources.Put(root) + for _, m := range append([]*Model{}, b.modelsByAPI.Get(api.ID)...) { + b.models.Delete(modelKeyFn(m)) + } } // The API's name/description are updated from the document's info object // regardless of mode — "merge" vs "overwrite" governs how the resource/method // tree is applied, not whether the top-level RestApi metadata is refreshed. if doc.Info.Title != "" { - data.api.Name = doc.Info.Title + api.Name = doc.Info.Title } if doc.Info.Description != "" { - data.api.Description = doc.Info.Description + api.Description = doc.Info.Description } if doc.APIKeySourceExt != "" { - data.api.APIKeySource = doc.APIKeySourceExt + api.APIKeySource = doc.APIKeySourceExt } if len(doc.BinaryMediaTypes) > 0 { - data.api.BinaryMediaTypes = doc.BinaryMediaTypes + api.BinaryMediaTypes = doc.BinaryMediaTypes } - importModels(data, doc) - importPaths(data, doc) + importModels(b, api, doc) + importPaths(b, api, doc) - cp := data.api + cp := *api return &cp, nil } // importModels registers the document's named schemas as API Gateway models. -func importModels(data *apiData, doc *openAPIDoc) { +// +// NOTE: the exists check below preserves a pre-existing quirk carried over +// mechanically from the map[string]*apiData days: models are keyed by model +// ID (see modelKeyFn), but name here is the schema's *Name*, not an ID -- so +// this only skips re-creating a model when name happens to equal an existing +// model's ID. This was already the case before the store.Table conversion +// (data.models was ID-keyed but was indexed by name here) and is preserved +// byte-for-byte rather than "fixed" as part of a storage-layer swap -- see the +// identical note on buildModelRef in backend.go. +func importModels(b *InMemoryBackend, api *RestAPI, doc *openAPIDoc) { for name, raw := range doc.schemaDefinitions() { - if _, exists := data.models[name]; exists { + if b.models.Has(modelKey(api.ID, name)) { continue } schema := string(raw) - data.models[name] = &Model{ + b.models.Put(&Model{ ID: randomID(resourceIDLength), - RestAPIID: data.api.ID, + RestAPIID: api.ID, Name: name, ContentType: contentTypeJSON, Schema: schema, - } + }) } } // importPaths walks the document paths, creating the resource tree and methods. -func importPaths(data *apiData, doc *openAPIDoc) { +func importPaths(b *InMemoryBackend, api *RestAPI, doc *openAPIDoc) { // Sort paths for deterministic resource creation order. pathKeys := collections.SortedKeys(doc.Paths) for _, path := range pathKeys { - res := ensureResourcePath(data, path) + res := ensureResourcePath(b, api, path) if res == nil { continue } @@ -333,8 +336,8 @@ func isHTTPVerb(verb string) bool { // ensureResourcePath walks/creates the resource tree for an OpenAPI path, // returning the leaf resource. Existing resources are reused (merge semantics). -func ensureResourcePath(data *apiData, path string) *Resource { - root := data.resources[data.api.RootResourceID] +func ensureResourcePath(b *InMemoryBackend, api *RestAPI, path string) *Resource { + root, _ := b.resources.Get(resourceKey(api.ID, api.RootResourceID)) if path == "/" || path == "" { return root } @@ -344,18 +347,18 @@ func ensureResourcePath(data *apiData, path string) *Resource { if seg == "" { continue } - child := findChildResource(data, current.ID, seg) + child := findChildResource(b, api.ID, current.ID, seg) if child == nil { child = &Resource{ ID: randomID(resourceIDLength), ParentID: current.ID, PathPart: seg, Path: computePath(current.Path, seg), - RestAPIID: data.api.ID, + RestAPIID: api.ID, ResourceMethods: make(map[string]*Method), } - data.resources[child.ID] = child - data.resourceVersion++ + b.resources.Put(child) + b.resourceVersions[api.ID]++ } current = child } @@ -365,8 +368,8 @@ func ensureResourcePath(data *apiData, path string) *Resource { // findChildResource returns the existing child of parent with the given path // part, or nil. -func findChildResource(data *apiData, parentID, pathPart string) *Resource { - for _, r := range data.resources { +func findChildResource(b *InMemoryBackend, restAPIID, parentID, pathPart string) *Resource { + for _, r := range b.resourcesByAPI.Get(restAPIID) { if r.ParentID == parentID && r.PathPart == pathPart { return r } diff --git a/services/apigateway/models.go b/services/apigateway/models.go index 000ab1c93..464d0f66a 100644 --- a/services/apigateway/models.go +++ b/services/apigateway/models.go @@ -173,9 +173,16 @@ type Stage struct { DeploymentID string `json:"deploymentId"` Description string `json:"description,omitempty"` ClientCertificateID string `json:"clientCertificateId,omitempty"` + // CacheClusterSize is the cache cluster's capacity in GB (e.g. "0.5"), only + // meaningful when CacheClusterEnabled is true. + CacheClusterSize string `json:"cacheClusterSize,omitempty"` + // CacheClusterStatus mirrors AWS's CacheClusterStatus enum + // (AVAILABLE/NOT_AVAILABLE/...), derived from CacheClusterEnabled. + CacheClusterStatus string `json:"cacheClusterStatus,omitempty"` // InvokeURL is the invoke URL for this stage (non-AWS field used by gopherstack UI). - InvokeURL string `json:"invokeUrl,omitempty"` - TracingEnabled bool `json:"tracingEnabled,omitempty"` + InvokeURL string `json:"invokeUrl,omitempty"` + TracingEnabled bool `json:"tracingEnabled,omitempty"` + CacheClusterEnabled bool `json:"cacheClusterEnabled,omitempty"` } // Deployment represents a REST API deployment. @@ -233,13 +240,18 @@ type PutIntegrationResponseInput struct { // Authorizer represents an API Gateway authorizer. type Authorizer struct { - ID string `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - AuthorizerURI string `json:"authorizerUri,omitempty"` - AuthorizerCredentials string `json:"authorizerCredentials,omitempty"` - IdentitySource string `json:"identitySource,omitempty"` - IdentityValidationExpression string `json:"identityValidationExpression,omitempty"` + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + AuthorizerURI string `json:"authorizerUri,omitempty"` + AuthorizerCredentials string `json:"authorizerCredentials,omitempty"` + IdentitySource string `json:"identitySource,omitempty"` + IdentityValidationExpression string `json:"identityValidationExpression,omitempty"` + // RestAPIID identifies the owning REST API. It is internal storage-layer + // identity (composite key for the backend's flat store.Table[Authorizer]), + // never part of the wire response, matching the same json:"-" convention + // already used by Resource/Stage/Deployment/Model for the identical purpose. + RestAPIID string `json:"-"` ProviderARNs []string `json:"providerARNs,omitempty"` AuthorizerResultTTLInSeconds int `json:"authorizerResultTtlInSeconds,omitempty"` } @@ -270,8 +282,12 @@ type UpdateAuthorizerInput struct { // RequestValidator represents an API Gateway request validator. type RequestValidator struct { - ID string `json:"id"` - Name string `json:"name"` + ID string `json:"id"` + Name string `json:"name"` + // RestAPIID identifies the owning REST API. Internal storage-layer identity + // only (composite key for the backend's flat store.Table[RequestValidator]); + // never part of the wire response — see the identical Authorizer.RestAPIID doc. + RestAPIID string `json:"-"` ValidateRequestBody bool `json:"validateRequestBody"` ValidateRequestParameters bool `json:"validateRequestParameters"` } @@ -463,7 +479,9 @@ type CreateStageInput struct { DeploymentID string `json:"deploymentId"` Description string `json:"description,omitempty"` ClientCertificateID string `json:"clientCertificateId,omitempty"` + CacheClusterSize string `json:"cacheClusterSize,omitempty"` TracingEnabled bool `json:"tracingEnabled,omitempty"` + CacheClusterEnabled bool `json:"cacheClusterEnabled,omitempty"` } // ThrottleSettings controls request rate limiting for a usage plan. @@ -506,6 +524,11 @@ type UsagePlanKey struct { Type string `json:"type"` Value string `json:"value,omitempty"` Name string `json:"name,omitempty"` + // UsagePlanID identifies the owning usage plan. Internal storage-layer + // identity only (composite key for the backend's flat + // store.Table[UsagePlanKey]); never part of the wire response — see the + // identical Authorizer.RestAPIID doc. + UsagePlanID string `json:"-"` } // CreateUsagePlanKeyInput is the input for CreateUsagePlanKey. @@ -533,11 +556,13 @@ type UpdateStageInput struct { CanarySettings *CanarySettings `json:"canarySettings,omitempty"` AccessLogSettings *AccessLogSettings `json:"accessLogSettings,omitempty"` TracingEnabled *bool `json:"tracingEnabled,omitempty"` + CacheClusterEnabled *bool `json:"cacheClusterEnabled,omitempty"` MethodSettings map[string]MethodSetting `json:"methodSettings,omitempty"` Variables map[string]string `json:"variables,omitempty"` DeploymentID string `json:"deploymentId,omitempty"` Description string `json:"description,omitempty"` ClientCertificateID string `json:"clientCertificateId,omitempty"` + CacheClusterSize string `json:"cacheClusterSize,omitempty"` } // Account represents the API Gateway account settings. @@ -704,9 +729,14 @@ type UpdateMethodResponseInput struct { StatusCode string `json:"statusCode"` } -// UpdateAccountInput is the input for UpdateAccount. +// UpdateAccountInput is the input for UpdateAccount. The real AWS wire shape +// carries only "patchOperations" (see aws-sdk-go-v2 apigateway +// UpdateAccountInput); handler.go flattens the patch document into these +// named fields (CloudwatchRoleARN via top-level "/cloudwatchRoleArn", +// ThrottleSettings via nested "/throttle/{rateLimit,burstLimit}" — see patch.go). type UpdateAccountInput struct { - ThrottleSettings *ThrottleSettings `json:"throttleSettings,omitempty"` + ThrottleSettings *ThrottleSettings `json:"throttleSettings,omitempty"` + CloudwatchRoleARN string `json:"cloudwatchRoleArn,omitempty"` } // TestInvokeAuthorizerInput is the input for TestInvokeAuthorizer. diff --git a/services/apigateway/patch.go b/services/apigateway/patch.go new file mode 100644 index 000000000..56c1fff6a --- /dev/null +++ b/services/apigateway/patch.go @@ -0,0 +1,759 @@ +package apigateway + +import ( + "encoding/json" + "maps" + "slices" + "strconv" + "strings" +) + +// patch.go implements AWS API Gateway's PATCH-operation semantics beyond what +// a flat "replace one top-level field" model (the old normalizePatchBody) can +// express. Every Update* REST operation in this service (UpdateRestApi, +// UpdateStage, UpdateAccount, UpdateUsagePlan, UpdateGatewayResponse, ...) +// shares the same wire shape: a PATCH request body of +// {"patchOperations":[{"op":...,"path":...,"value":...,"from":...}, ...]} +// (aws-sdk-go-v2/service/apigateway/types.PatchOperation), or a bare RFC-6902 +// array. Two things about that wire shape are easy to get wrong and were +// previously wrong here: +// +// 1. PatchOperation.Value is *always* a JSON string on the wire — the real +// SDK serializer (awsRestjson1_serializeDocumentPatchOperation) calls +// String() unconditionally — even when the target field is a bool or a +// number (e.g. {"op":"replace","path":"/tracingEnabled","value":"true"}). +// Copying that raw JSON string byte-for-byte into a field that unmarshals +// into a Go bool/int (e.g. UpdateStageInput.TracingEnabled *bool) fails +// with a JSON type-mismatch error. patchValueString + the per-field +// coercion below convert the string payload to the JSON literal the +// target field actually needs. +// 2. Many real-world patches don't target a top-level field at all — they +// target one entry of a map/list, or a nested struct field, e.g. +// "/variables/apiKey" (one stage variable), "/binaryMediaTypes/image~1png" +// (one binary media type, JSON-Pointer-escaped), "/throttle/rateLimit" +// (one field of the account's/usage plan's ThrottleSettings), +// "/apiStages" (usage-plan API stage membership, add/remove, value +// "restApiId:stage"), or per-method stage settings addressed as +// "/{resourcePath}/{httpMethod}/{category}/{property}" with NO +// "methodSettings" path prefix (resourcePath JSON-Pointer-escaped, or "*" +// for the wildcard default — e.g. "/~1pets/GET/throttling/burstLimit" or +// "/*/*/logging/loglevel"). The previous flatten took the *entire* +// remaining path verbatim as a single flat field name (e.g. +// "variables/apiKey"), which never matches any Update*Input json tag, so +// the edit was silently dropped. This also affected canary-deployment +// promotion, which AWS models as a "copy" op +// ({"op":"copy","from":"/canarySettings/deploymentId","path":"/deploymentId"}) +// that the old flatten didn't implement at all — "copy" isn't "add" or +// "replace" so it was unconditionally skipped, alongside "remove". +// +// Resolving (1) and (2) requires reading the CURRENT resource (to merge one +// map/struct entry into its existing siblings — the Update* backend methods +// replace a map/struct field wholesale when it's provided), so this logic +// needs Handler.Backend access and lives here rather than in a pure body -> +// body helper. +// +// NOTE: the per-route method-settings property paths +// (stageMethodSettingProperty below) are a best-effort mapping from AWS's +// "Method Settings Property of a Stage" patch-operations reference. Unlike +// the struct field names above (verified against aws-sdk-go-v2/service/ +// apigateway/types), the SDK does not expose these dotted-path strings as a +// typed enum to check against — PatchOperation.Path is just a free string. +// Flag it if a live wire capture disagrees with the exact property spelling. + +// PATCH operation verbs (aws-sdk-go-v2/service/apigateway/types.Op) and the +// scalar-kind tags used by patchFieldKind/coerceTopLevelPatchValue. +const ( + patchOpAdd = "add" + patchOpReplace = "replace" + patchOpRemove = "remove" + patchOpCopy = "copy" + + patchKindBool = "bool" + patchKindInt = "int" + + // patchPathSegs2 is the segment count of a two-level PATCH path such as + // "/variables/{name}" or "/throttle/rateLimit" (field + one sub-key). + patchPathSegs2 = 2 +) + +// patchOp is a single PATCH operation from a PATCH request body, matching +// aws-sdk-go-v2/service/apigateway/types.PatchOperation's wire shape: +// {"op":"replace","path":"/description","value":"foo"}, or, for the "copy" +// op used e.g. to promote a canary deployment, +// {"op":"copy","from":"/canarySettings/deploymentId","path":"/deploymentId"}. +type patchOp struct { + Op string `json:"op"` + Path string `json:"path"` + From string `json:"from,omitempty"` + Value json.RawMessage `json:"value"` +} + +// parsePatchDocument detects and parses a PATCH request body: either a bare +// RFC-6902-flavored array ([{"op":...}]) or a {"patchOperations":[...]} +// wrapper (the real AWS wire shape), which may carry sibling fields alongside +// "patchOperations" (rest). ok is false when body is neither shape, in which +// case the caller should treat body as an already-flat field object. +func parsePatchDocument(body []byte) ([]patchOp, map[string]json.RawMessage, bool) { + if len(body) == 0 { + return nil, nil, false + } + + switch body[0] { + case '[': + var arr []patchOp + if err := json.Unmarshal(body, &arr); err != nil || len(arr) == 0 || arr[0].Op == "" { + return nil, nil, false + } + + return arr, nil, true + case '{': + var wrapper struct { + PatchOperations []patchOp `json:"patchOperations"` + } + if err := json.Unmarshal(body, &wrapper); err != nil || len(wrapper.PatchOperations) == 0 { + return nil, nil, false + } + + var siblings map[string]json.RawMessage + + _ = json.Unmarshal(body, &siblings) + delete(siblings, "patchOperations") + + return wrapper.PatchOperations, siblings, true + default: + return nil, nil, false + } +} + +// patchFieldKind classifies known non-string top-level PATCH field names by +// their real Go/JSON type, so patchValueString's string payload is coerced to +// a matching JSON literal (see the package doc above) instead of being copied +// in verbatim as a JSON string that the target field can't unmarshal. +// +//nolint:gochecknoglobals // read-only lookup table initialized once at startup +var patchFieldKind = map[string]string{ + "tracingEnabled": patchKindBool, + "cacheClusterEnabled": patchKindBool, + "minimumCompressionSize": patchKindInt, + "enabled": patchKindBool, // ApiKey.enabled + "validateRequestBody": patchKindBool, + "validateRequestParameters": patchKindBool, + "apiKeyRequired": patchKindBool, + "authorizerResultTtlInSeconds": patchKindInt, +} + +// coerceTopLevelPatchValue converts a top-level PATCH value (always a JSON +// string on the wire, see patchValueString) into the JSON literal the target +// field's Go type needs, for the handful of non-string fields patched at the +// top level. Fields absent from patchFieldKind are assumed to be strings (the +// common case, and already correctly a JSON string on the wire) and passed +// through unchanged. +func coerceTopLevelPatchValue(field string, raw json.RawMessage) json.RawMessage { + kind, known := patchFieldKind[field] + if !known { + return raw + } + + s, ok := patchValueString(raw) + if !ok { + return raw + } + + switch kind { + case patchKindBool: + if b, parseErr := strconv.ParseBool(s); parseErr == nil { + if encoded, marshalErr := json.Marshal(b); marshalErr == nil { + return encoded + } + } + case patchKindInt: + if n, parseErr := strconv.Atoi(s); parseErr == nil { + if encoded, marshalErr := json.Marshal(n); marshalErr == nil { + return encoded + } + } + } + + return raw +} + +// patchValueString decodes a PATCH operation's Value into a Go string. Value +// is always transmitted as a JSON string on the wire, even for boolean or +// numeric targets (see the package doc above), so every call site that needs +// the underlying scalar goes through this and then parses it with strconv. +// ok is false when Value is empty/absent or not a JSON string. +func patchValueString(raw json.RawMessage) (string, bool) { + if len(raw) == 0 { + return "", false + } + + var s string + if err := json.Unmarshal(raw, &s); err != nil { + return "", false + } + + return s, true +} + +// jsonPointerUnescape decodes a single RFC 6901 JSON Pointer path segment +// ("~1" -> "/", then "~0" -> "~"; that order matters so "~01" decodes to "~1" +// and not "/"). +func jsonPointerUnescape(seg string) string { + seg = strings.ReplaceAll(seg, "~1", "/") + + return strings.ReplaceAll(seg, "~0", "~") +} + +// patchPathSegments splits a JSON-Pointer path into its still-escaped raw +// segments, dropping the leading empty segment produced by the leading "/". +func patchPathSegments(path string) []string { + trimmed := strings.TrimPrefix(path, "/") + if trimmed == "" { + return nil + } + + return strings.Split(trimmed, "/") +} + +// setJSONValue marshals value and stores it under field in out. Errors are +// swallowed (defensive only): every call site below passes a well-formed +// built-in Go value that always marshals successfully. +func setJSONValue(out map[string]json.RawMessage, field string, value any) { + raw, err := json.Marshal(value) + if err != nil { + return + } + + out[field] = raw +} + +// cloneStringMap returns a non-nil shallow copy of m. A nil result would be +// indistinguishable, to the Update* backend methods' "field != nil means the +// caller provided it" checks, from the field being entirely absent from the +// patch — so a merge that removes the last entry must still produce a +// non-nil empty map for the removal to actually take effect. +func cloneStringMap(m map[string]string) map[string]string { + out := make(map[string]string, len(m)) + maps.Copy(out, m) + + return out +} + +// applyStructuredPatch resolves a PATCH request body into the flat JSON +// object that the target action's Update*Input struct unmarshals from, +// correctly handling per-entry map/list edits, nested struct-field edits, +// per-route stage method settings, "remove", and "copy" — none of which the +// old single-field flatten supported (see package doc). Non-patch bodies +// (plain field objects, GET/DELETE's synthesized "{}", etc.) pass through +// unchanged. +func (h *Handler) applyStructuredPatch(action string, pathParams map[string]string, body []byte) []byte { + ops, rest, isPatch := parsePatchDocument(body) + if !isPatch { + return body + } + + out := make(map[string]json.RawMessage, len(rest)+len(ops)) + maps.Copy(out, rest) + + for _, op := range ops { + if h.applyResourcePatchOp(action, pathParams, op, out) { + continue + } + + applyTopLevelPatchOp(op, out) + } + + raw, err := json.Marshal(out) + if err != nil { + return body + } + + return raw +} + +// applyTopLevelPatchOp is the fallback for every path/action combination the +// resource-specific resolvers below don't recognize: a plain single-segment +// "add"/"replace" of one field, with type coercion for the small set of +// known non-string fields (see patchFieldKind). "remove" and multi-segment +// paths that no resolver claimed are left as no-ops, matching the previous +// (silent-drop) behavior, since correctly clearing a bare scalar field would +// need every Update*Input field to distinguish "not provided" from "reset to +// zero value" — a much larger refactor across every resource's Update +// backend method than this pass's PATCH-semantics fix. Tracked as a follow-up. +func applyTopLevelPatchOp(op patchOp, out map[string]json.RawMessage) { + if op.Op != patchOpReplace && op.Op != patchOpAdd { + return + } + + field := strings.TrimPrefix(op.Path, "/") + if field == "" || strings.Contains(field, "/") { + return + } + + out[field] = coerceTopLevelPatchValue(field, op.Value) +} + +// applyResourcePatchOp dispatches to the per-action patch resolver for +// actions with map/struct/list PATCH targets that need current backend state +// to merge correctly. Returns true when the op was fully handled (including +// "handled as an intentional no-op", e.g. an unresolvable "copy" source) so +// the caller must not also apply the generic top-level fallback. +func (h *Handler) applyResourcePatchOp( + action string, pathParams map[string]string, op patchOp, out map[string]json.RawMessage, +) bool { + segs := patchPathSegments(op.Path) + if len(segs) == 0 { + return false + } + + switch action { + case opUpdateStage: + return h.applyStagePatchOp(pathParams, op, segs, out) + case opUpdateRestAPI: + return h.applyRestAPIPatchOp(pathParams, op, segs, out) + case opUpdateAccount: + return h.applyAccountPatchOp(op, segs, out) + case opUpdateUsagePlan: + return h.applyUsagePlanPatchOp(pathParams, op, segs, out) + case opUpdateGatewayResponse: + return h.applyGatewayResponsePatchOp(pathParams, op, segs, out) + default: + return false + } +} + +// applyStagePatchOp handles UpdateStage PATCH ops that a flat top-level +// replace cannot express: canary-deployment promotion (a "copy" op), single +// stage-variable edits, single canarySettings/accessLogSettings sub-fields, +// and per-route method settings. +func (h *Handler) applyStagePatchOp( + pathParams map[string]string, op patchOp, segs []string, out map[string]json.RawMessage, +) bool { + if op.Op == patchOpCopy && op.Path == "/deploymentId" && op.From == "/canarySettings/deploymentId" { + stage, err := h.Backend.GetStage(pathParams[keyRestAPIID], pathParams[keyStageName]) + if err != nil || stage.CanarySettings == nil { + return true + } + + setJSONValue(out, "deploymentId", stage.CanarySettings.DeploymentID) + + return true + } + + if len(segs) >= 3 && (segs[0] == "*" || strings.HasPrefix(segs[0], "~1")) { + return h.applyStageMethodSettingPatch(pathParams, op, segs, out) + } + + if len(segs) != patchPathSegs2 { + return false + } + + switch segs[0] { + case "variables": + return h.applyStageVariablePatch(pathParams, op, segs[1], out) + case "canarySettings": + return h.applyStageCanaryPatch(pathParams, op, segs[1], out) + case "accessLogSettings": + return h.applyStageAccessLogPatch(pathParams, op, segs[1], out) + default: + return false + } +} + +// applyStageVariablePatch adds/replaces/removes a single stage variable +// ("/variables/{name}"), merging with the stage's existing variables (a +// wholesale replace would otherwise silently drop every other variable). +func (h *Handler) applyStageVariablePatch( + pathParams map[string]string, op patchOp, rawName string, out map[string]json.RawMessage, +) bool { + if op.Op != patchOpAdd && op.Op != patchOpReplace && op.Op != patchOpRemove { + return false + } + + stage, err := h.Backend.GetStage(pathParams[keyRestAPIID], pathParams[keyStageName]) + if err != nil { + return true + } + + vars := cloneStringMap(stage.Variables) + name := jsonPointerUnescape(rawName) + + if op.Op == patchOpRemove { + delete(vars, name) + } else if v, ok := patchValueString(op.Value); ok { + vars[name] = v + } + + setJSONValue(out, "variables", vars) + + return true +} + +// applyStageCanaryPatch replaces a single CanarySettings sub-field +// ("/canarySettings/{deploymentId,percentTraffic,useStageCache}"), merging +// with the stage's existing canary settings. +func (h *Handler) applyStageCanaryPatch( + pathParams map[string]string, op patchOp, prop string, out map[string]json.RawMessage, +) bool { + if op.Op != patchOpAdd && op.Op != patchOpReplace && op.Op != patchOpRemove { + return false + } + + stage, err := h.Backend.GetStage(pathParams[keyRestAPIID], pathParams[keyStageName]) + if err != nil { + return true + } + + cur := CanarySettings{} + if stage.CanarySettings != nil { + cur = *stage.CanarySettings + } + + val, _ := patchValueString(op.Value) + remove := op.Op == patchOpRemove + + switch prop { + case "deploymentId": + if remove { + cur.DeploymentID = "" + } else { + cur.DeploymentID = val + } + case "percentTraffic": + if remove { + cur.PercentTraffic = 0 + } else if f, parseErr := strconv.ParseFloat(val, 64); parseErr == nil { + cur.PercentTraffic = f + } + case "useStageCache": + cur.UseStageCache = !remove && parseBoolLenient(val) + default: + return false + } + + setJSONValue(out, "canarySettings", &cur) + + return true +} + +// applyStageAccessLogPatch replaces a single AccessLogSettings sub-field +// ("/accessLogSettings/{destinationArn,format}"), merging with the stage's +// existing access log settings. +func (h *Handler) applyStageAccessLogPatch( + pathParams map[string]string, op patchOp, prop string, out map[string]json.RawMessage, +) bool { + if op.Op != patchOpAdd && op.Op != patchOpReplace && op.Op != patchOpRemove { + return false + } + + stage, err := h.Backend.GetStage(pathParams[keyRestAPIID], pathParams[keyStageName]) + if err != nil { + return true + } + + cur := AccessLogSettings{} + if stage.AccessLogSettings != nil { + cur = *stage.AccessLogSettings + } + + val, _ := patchValueString(op.Value) + if op.Op == patchOpRemove { + val = "" + } + + switch prop { + case "destinationArn": + cur.DestinationARN = val + case "format": + cur.Format = val + default: + return false + } + + setJSONValue(out, "accessLogSettings", &cur) + + return true +} + +// stageMethodSettingProperty maps a per-route method-settings PATCH property +// path (the segments after "{resourcePath}/{httpMethod}", joined with "/") to +// the MethodSetting field it targets. See the package doc's note on the +// accuracy of these property strings. +// +//nolint:gochecknoglobals // read-only lookup table initialized once at startup +var stageMethodSettingProperty = map[string]func(ms *MethodSetting, value string, remove bool){ + "logging/loglevel": func(ms *MethodSetting, v string, remove bool) { + if remove { + ms.LoggingLevel = "" + + return + } + + ms.LoggingLevel = v + }, + "logging/dataTrace": func(ms *MethodSetting, v string, remove bool) { + ms.DataTraceEnabled = !remove && parseBoolLenient(v) + }, + "metrics/enabled": func(ms *MethodSetting, v string, remove bool) { + ms.MetricsEnabled = !remove && parseBoolLenient(v) + }, + "throttling/burstLimit": func(ms *MethodSetting, v string, remove bool) { + if remove { + ms.ThrottlingBurstLimit = 0 + + return + } + + if n, err := strconv.Atoi(v); err == nil { + ms.ThrottlingBurstLimit = n + } + }, + "throttling/rateLimit": func(ms *MethodSetting, v string, remove bool) { + if remove { + ms.ThrottlingRateLimit = 0 + + return + } + + if f, err := strconv.ParseFloat(v, 64); err == nil { + ms.ThrottlingRateLimit = f + } + }, + "caching/enabled": func(ms *MethodSetting, v string, remove bool) { + ms.CachingEnabled = !remove && parseBoolLenient(v) + }, + "caching/ttlInSeconds": func(ms *MethodSetting, v string, remove bool) { + if remove { + ms.CacheTTLInSeconds = 0 + + return + } + + if n, err := strconv.Atoi(v); err == nil { + ms.CacheTTLInSeconds = n + } + }, + "caching/requireAuthorizationForCacheControl": func(ms *MethodSetting, v string, remove bool) { + ms.RequireAuthorizationForCacheControl = !remove && parseBoolLenient(v) + }, +} + +// parseBoolLenient parses v as a bool, defaulting to false on a malformed +// value rather than erroring the whole PATCH request over one bad flag. +func parseBoolLenient(v string) bool { + b, _ := strconv.ParseBool(v) + + return b +} + +// applyStageMethodSettingPatch handles per-route stage method settings, +// addressed as "/{resourcePath}/{httpMethod}/{category}/{property}" with NO +// "methodSettings" path prefix — this is how AWS actually addresses them +// (e.g. "/~1pets/GET/throttling/burstLimit" or "/*/*/logging/loglevel") — +// merged with the stage's existing per-route overrides. +func (h *Handler) applyStageMethodSettingPatch( + pathParams map[string]string, op patchOp, segs []string, out map[string]json.RawMessage, +) bool { + apply, known := stageMethodSettingProperty[strings.Join(segs[2:], "/")] + if !known || (op.Op != patchOpAdd && op.Op != patchOpReplace && op.Op != patchOpRemove) { + return false + } + + stage, err := h.Backend.GetStage(pathParams[keyRestAPIID], pathParams[keyStageName]) + if err != nil { + return true + } + + settings := make(map[string]MethodSetting, len(stage.MethodSettings)+1) + maps.Copy(settings, stage.MethodSettings) + + routeKey := jsonPointerUnescape(segs[0]) + "/" + segs[1] + ms := settings[routeKey] + + val, _ := patchValueString(op.Value) + apply(&ms, val, op.Op == patchOpRemove) + settings[routeKey] = ms + + setJSONValue(out, "methodSettings", settings) + + return true +} + +// applyRestAPIPatchOp handles UpdateRestApi's binary-media-type membership +// edits ("/binaryMediaTypes/{escaped-media-type}", add/remove), merging with +// the API's existing binary media types (a wholesale replace would otherwise +// silently drop every other configured type). +func (h *Handler) applyRestAPIPatchOp( + pathParams map[string]string, op patchOp, segs []string, out map[string]json.RawMessage, +) bool { + if len(segs) != patchPathSegs2 || segs[0] != "binaryMediaTypes" { + return false + } + + if op.Op != patchOpAdd && op.Op != patchOpRemove { + return false + } + + api, err := h.Backend.GetRestAPI(pathParams[keyRestAPIID]) + if err != nil { + return true + } + + mediaType := jsonPointerUnescape(segs[1]) + types := slices.Clone(api.BinaryMediaTypes) + + if op.Op == patchOpAdd { + if !slices.Contains(types, mediaType) { + types = append(types, mediaType) + } + } else { + types = slices.DeleteFunc(types, func(t string) bool { return t == mediaType }) + } + + if types == nil { + types = []string{} + } + + setJSONValue(out, "binaryMediaTypes", types) + + return true +} + +// applyAccountPatchOp handles UpdateAccount's nested ThrottleSettings edits +// ("/throttle/{rateLimit,burstLimit}"), merging with the account's existing +// throttle settings. "/cloudwatchRoleArn" is a plain top-level string field +// and is handled by the generic fallback (applyTopLevelPatchOp). +func (h *Handler) applyAccountPatchOp(op patchOp, segs []string, out map[string]json.RawMessage) bool { + if len(segs) != patchPathSegs2 || segs[0] != "throttle" { + return false + } + + if op.Op != patchOpAdd && op.Op != patchOpReplace && op.Op != patchOpRemove { + return false + } + + acct, err := h.Backend.GetAccount() + if err != nil { + return true + } + + cur := ThrottleSettings{} + if acct.ThrottleSettings != nil { + cur = *acct.ThrottleSettings + } + + val, _ := patchValueString(op.Value) + remove := op.Op == patchOpRemove + + switch segs[1] { + case "rateLimit": + if remove { + cur.RateLimit = 0 + } else if f, parseErr := strconv.ParseFloat(val, 64); parseErr == nil { + cur.RateLimit = f + } + case "burstLimit": + if remove { + cur.BurstLimit = 0 + } else if n, parseErr := strconv.Atoi(val); parseErr == nil { + cur.BurstLimit = n + } + default: + return false + } + + setJSONValue(out, "throttleSettings", &cur) + + return true +} + +// applyUsagePlanPatchOp handles UpdateUsagePlan's API-stage membership edits: +// AWS addresses these with a single-segment path ("/apiStages"), add/remove, +// where Value is the string "{restApiId}:{stage}" (not a nested path) — see +// the AWS "Usage Plan Update Operation" patch-operations reference. +func (h *Handler) applyUsagePlanPatchOp( + pathParams map[string]string, op patchOp, segs []string, out map[string]json.RawMessage, +) bool { + if len(segs) != 1 || segs[0] != "apiStages" { + return false + } + + if op.Op != patchOpAdd && op.Op != patchOpRemove { + return false + } + + val, ok := patchValueString(op.Value) + if !ok { + return true + } + + restAPIID, stage, cut := strings.Cut(val, ":") + if !cut { + return true + } + + plan, err := h.Backend.GetUsagePlan(pathParams[keyUsagePlanID]) + if err != nil { + return true + } + + matches := func(a APIStageAssociation) bool { return a.RestAPIID == restAPIID && a.Stage == stage } + stages := slices.Clone(plan.APIStages) + + if op.Op == patchOpAdd { + if !slices.ContainsFunc(stages, matches) { + stages = append(stages, APIStageAssociation{RestAPIID: restAPIID, Stage: stage}) + } + } else { + stages = slices.DeleteFunc(stages, matches) + } + + if stages == nil { + stages = []APIStageAssociation{} + } + + setJSONValue(out, "apiStages", stages) + + return true +} + +// applyGatewayResponsePatchOp handles UpdateGatewayResponse's per-key +// responseParameters/responseTemplates edits +// ("/responseParameters/{key}", "/responseTemplates/{key}"), merging with the +// gateway response's existing entries (falling back to AWS's implicit +// default response when none has been customized yet). +func (h *Handler) applyGatewayResponsePatchOp( + pathParams map[string]string, op patchOp, segs []string, out map[string]json.RawMessage, +) bool { + if len(segs) != patchPathSegs2 || (segs[0] != "responseParameters" && segs[0] != "responseTemplates") { + return false + } + + if op.Op != patchOpAdd && op.Op != patchOpReplace && op.Op != patchOpRemove { + return false + } + + gr, err := h.Backend.GetGatewayResponse(pathParams[keyRestAPIID], pathParams[keyResponseType]) + if err != nil { + return true + } + + key := jsonPointerUnescape(segs[1]) + + var m map[string]string + if segs[0] == "responseParameters" { + m = cloneStringMap(gr.ResponseParameters) + } else { + m = cloneStringMap(gr.ResponseTemplates) + } + + if op.Op == patchOpRemove { + delete(m, key) + } else if v, ok := patchValueString(op.Value); ok { + m[key] = v + } + + setJSONValue(out, segs[0], m) + + return true +} diff --git a/services/apigateway/patch_test.go b/services/apigateway/patch_test.go new file mode 100644 index 000000000..283d648d0 --- /dev/null +++ b/services/apigateway/patch_test.go @@ -0,0 +1,544 @@ +package apigateway_test + +import ( + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/apigateway" +) + +// setupStage creates a REST API, a deployment, and a stage pointing at it, +// returning the API ID alongside the ready-to-patch stage. variables/canary +// may be nil. +func setupStage( + t *testing.T, h *apigateway.Handler, variables map[string]string, canary *apigateway.CanarySettings, +) (string, string) { + t.Helper() + + api, err := h.Backend.CreateRestAPI(apigateway.CreateRestAPIInput{Name: "patch-test-api"}) + require.NoError(t, err) + + depl, err := h.Backend.CreateDeployment(api.ID, "", "v1") + require.NoError(t, err) + + _, err = h.Backend.CreateStage(apigateway.CreateStageInput{ + RestAPIID: api.ID, + StageName: "prod", + DeploymentID: depl.ID, + Variables: variables, + CanarySettings: canary, + }) + require.NoError(t, err) + + return api.ID, "prod" +} + +// Test_ApplyStructuredPatch_StageVariables exercises per-variable "/variables/{name}" +// PATCH ops (add/replace/remove, including a JSON-Pointer-escaped name), which +// the old flatten silently dropped (it took the whole remaining path +// "variables/foo" as one bogus flat field, matching no Update*Input json tag). +func Test_ApplyStructuredPatch_StageVariables(t *testing.T) { + t.Parallel() + + cases := []struct { + want map[string]string + name string + ops string + }{ + { + name: "add a new variable merges with existing ones", + ops: `[{"op":"add","path":"/variables/c","value":"3"}]`, + want: map[string]string{"a": "1", "b": "2", "c": "3"}, + }, + { + name: "replace an existing variable leaves siblings untouched", + ops: `[{"op":"replace","path":"/variables/a","value":"9"}]`, + want: map[string]string{"a": "9", "b": "2"}, + }, + { + name: "remove deletes only the named variable", + ops: `[{"op":"remove","path":"/variables/b"}]`, + want: map[string]string{"a": "1"}, + }, + { + name: "JSON-Pointer-escaped variable name is unescaped", + ops: `[{"op":"add","path":"/variables/a~1b","value":"x"}]`, + want: map[string]string{"a": "1", "b": "2", "a/b": "x"}, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newAPIGWHandler() + apiID, stageName := setupStage(t, h, map[string]string{"a": "1", "b": "2"}, nil) + + rec := restRequest(t, h, http.MethodPatch, + fmt.Sprintf("/restapis/%s/stages/%s", apiID, stageName), tt.ops) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + + stage, err := h.Backend.GetStage(apiID, stageName) + require.NoError(t, err) + assert.Equal(t, tt.want, stage.Variables) + }) + } +} + +// Test_ApplyStructuredPatch_StageCanaryPromotion exercises the AWS-documented +// canary-promotion pattern: a "copy" op from "/canarySettings/deploymentId" to +// "/deploymentId". The old flatten never implemented "copy" at all (only +// "add"/"replace" were honored), so promotion was a total no-op. +func Test_ApplyStructuredPatch_StageCanaryPromotion(t *testing.T) { + t.Parallel() + + h := newAPIGWHandler() + apiID, stageName := setupStage(t, h, nil, &apigateway.CanarySettings{ + DeploymentID: "canary-depl-id", + PercentTraffic: 50, + }) + + before, err := h.Backend.GetStage(apiID, stageName) + require.NoError(t, err) + require.NotEqual(t, "canary-depl-id", before.DeploymentID, "precondition: main deployment differs from canary") + + body := `{"patchOperations":[{"op":"copy","from":"/canarySettings/deploymentId","path":"/deploymentId"}]}` + rec := restRequest(t, h, http.MethodPatch, fmt.Sprintf("/restapis/%s/stages/%s", apiID, stageName), body) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + + after, err := h.Backend.GetStage(apiID, stageName) + require.NoError(t, err) + assert.Equal(t, "canary-depl-id", after.DeploymentID, + "promoted deployment id should replace the stage's deploymentId") +} + +// Test_ApplyStructuredPatch_StageMethodSettings exercises per-route method +// settings, addressed by AWS with NO "methodSettings" path prefix +// ("/{resourcePath}/{httpMethod}/{category}/{property}", e.g. "/*/*/logging/loglevel" +// or "/~1pets/GET/caching/enabled"). The old flatten had no notion of this +// path shape at all. +func Test_ApplyStructuredPatch_StageMethodSettings(t *testing.T) { + t.Parallel() + + cases := []struct { + check func(t *testing.T, ms apigateway.MethodSetting) + name string + path string + value string + routeKey string + }{ + { + name: "wildcard throttling burst limit (string value coerced to int)", + path: "/*/*/throttling/burstLimit", + value: "500", + routeKey: "*/*", + check: func(t *testing.T, ms apigateway.MethodSetting) { + t.Helper() + assert.Equal(t, 500, ms.ThrottlingBurstLimit) + }, + }, + { + name: "wildcard throttling rate limit (string value coerced to float)", + path: "/*/*/throttling/rateLimit", + value: "12.5", + routeKey: "*/*", + check: func(t *testing.T, ms apigateway.MethodSetting) { + t.Helper() + assert.InDelta(t, 12.5, ms.ThrottlingRateLimit, 0.001) + }, + }, + { + name: "wildcard logging level", + path: "/*/*/logging/loglevel", + value: "INFO", + routeKey: "*/*", + check: func(t *testing.T, ms apigateway.MethodSetting) { + t.Helper() + assert.Equal(t, "INFO", ms.LoggingLevel) + }, + }, + { + name: "wildcard metrics enabled (string value coerced to bool)", + path: "/*/*/metrics/enabled", + value: "true", + routeKey: "*/*", + check: func(t *testing.T, ms apigateway.MethodSetting) { + t.Helper() + assert.True(t, ms.MetricsEnabled) + }, + }, + { + name: "escaped resource path and specific method", + path: "/~1pets/GET/caching/enabled", + value: "true", + routeKey: "/pets/GET", + check: func(t *testing.T, ms apigateway.MethodSetting) { + t.Helper() + assert.True(t, ms.CachingEnabled) + }, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newAPIGWHandler() + apiID, stageName := setupStage(t, h, nil, nil) + + body := fmt.Sprintf(`[{"op":"replace","path":%q,"value":%q}]`, tt.path, tt.value) + rec := restRequest(t, h, http.MethodPatch, + fmt.Sprintf("/restapis/%s/stages/%s", apiID, stageName), body) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + + stage, err := h.Backend.GetStage(apiID, stageName) + require.NoError(t, err) + require.Contains(t, stage.MethodSettings, tt.routeKey) + tt.check(t, stage.MethodSettings[tt.routeKey]) + }) + } +} + +// Test_ApplyStructuredPatch_StageMethodSettingsRemove verifies that "remove" +// resets a per-route property back to its zero value, merging with (not +// wiping) any other properties already set on that same route. +func Test_ApplyStructuredPatch_StageMethodSettingsRemove(t *testing.T) { + t.Parallel() + + h := newAPIGWHandler() + apiID, stageName := setupStage(t, h, nil, nil) + + setBody := `[ + {"op":"replace","path":"/*/*/throttling/burstLimit","value":"500"}, + {"op":"replace","path":"/*/*/logging/loglevel","value":"INFO"} + ]` + rec := restRequest(t, h, http.MethodPatch, fmt.Sprintf("/restapis/%s/stages/%s", apiID, stageName), setBody) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + + removeBody := `[{"op":"remove","path":"/*/*/throttling/burstLimit"}]` + rec = restRequest(t, h, http.MethodPatch, fmt.Sprintf("/restapis/%s/stages/%s", apiID, stageName), removeBody) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + + stage, err := h.Backend.GetStage(apiID, stageName) + require.NoError(t, err) + require.Contains(t, stage.MethodSettings, "*/*") + assert.Equal(t, 0, stage.MethodSettings["*/*"].ThrottlingBurstLimit, "removed property resets to zero value") + assert.Equal(t, "INFO", stage.MethodSettings["*/*"].LoggingLevel, "sibling property on the same route is untouched") +} + +// Test_ApplyStructuredPatch_StageTopLevelCoercion exercises top-level +// boolean/string fields whose PATCH value is transmitted as a JSON string on +// the wire (per aws-sdk-go-v2's PatchOperation serializer). Before the fix, +// copying that raw JSON string into a *bool field made json.Unmarshal fail +// with a type mismatch, so these patches errored instead of applying. +func Test_ApplyStructuredPatch_StageTopLevelCoercion(t *testing.T) { + t.Parallel() + + cases := []struct { + check func(t *testing.T, stage *apigateway.Stage) + name string + ops string + }{ + { + name: "tracingEnabled string value coerces to bool", + ops: `[{"op":"replace","path":"/tracingEnabled","value":"true"}]`, + check: func(t *testing.T, stage *apigateway.Stage) { + t.Helper() + assert.True(t, stage.TracingEnabled) + }, + }, + { + name: "cacheClusterEnabled string value coerces to bool and derives status", + ops: `[{"op":"replace","path":"/cacheClusterEnabled","value":"true"}]`, + check: func(t *testing.T, stage *apigateway.Stage) { + t.Helper() + assert.True(t, stage.CacheClusterEnabled) + assert.Equal(t, "AVAILABLE", stage.CacheClusterStatus) + }, + }, + { + name: "cacheClusterSize is a plain string field", + ops: `[{"op":"replace","path":"/cacheClusterSize","value":"0.5"}]`, + check: func(t *testing.T, stage *apigateway.Stage) { + t.Helper() + assert.Equal(t, "0.5", stage.CacheClusterSize) + }, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newAPIGWHandler() + apiID, stageName := setupStage(t, h, nil, nil) + + rec := restRequest(t, h, http.MethodPatch, + fmt.Sprintf("/restapis/%s/stages/%s", apiID, stageName), tt.ops) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + + stage, err := h.Backend.GetStage(apiID, stageName) + require.NoError(t, err) + tt.check(t, stage) + }) + } +} + +// Test_ApplyStructuredPatch_RestAPIBinaryMediaTypes exercises per-entry +// "/binaryMediaTypes/{escaped-media-type}" add/remove, which must merge with +// (not replace) the API's existing binary media types. +func Test_ApplyStructuredPatch_RestAPIBinaryMediaTypes(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + ops string + want []string + }{ + { + name: "add merges with the existing type", + ops: `[{"op":"add","path":"/binaryMediaTypes/application~1octet-stream"}]`, + want: []string{"image/png", "application/octet-stream"}, + }, + { + name: "add is idempotent", + ops: `[{"op":"add","path":"/binaryMediaTypes/image~1png"}]`, + want: []string{"image/png"}, + }, + { + name: "remove drops only the named type", + ops: `[{"op":"remove","path":"/binaryMediaTypes/image~1png"}]`, + want: []string{}, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newAPIGWHandler() + api, err := h.Backend.CreateRestAPI(apigateway.CreateRestAPIInput{ + Name: "media-test-api", + BinaryMediaTypes: []string{"image/png"}, + }) + require.NoError(t, err) + + rec := restRequest(t, h, http.MethodPatch, fmt.Sprintf("/restapis/%s", api.ID), tt.ops) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + + got, err := h.Backend.GetRestAPI(api.ID) + require.NoError(t, err) + assert.ElementsMatch(t, tt.want, got.BinaryMediaTypes) + }) + } +} + +// Test_ApplyStructuredPatch_RestAPIMinimumCompressionSizeCoercion verifies the +// numeric top-level field /minimumCompressionSize (an *int) correctly parses +// its string-typed wire value instead of failing to unmarshal. +func Test_ApplyStructuredPatch_RestAPIMinimumCompressionSizeCoercion(t *testing.T) { + t.Parallel() + + h := newAPIGWHandler() + api, err := h.Backend.CreateRestAPI(apigateway.CreateRestAPIInput{Name: "compression-test-api"}) + require.NoError(t, err) + + ops := `[{"op":"replace","path":"/minimumCompressionSize","value":"1024"}]` + rec := restRequest(t, h, http.MethodPatch, fmt.Sprintf("/restapis/%s", api.ID), ops) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + + got, err := h.Backend.GetRestAPI(api.ID) + require.NoError(t, err) + assert.Equal(t, 1024, got.MinimumCompressionSize) +} + +// Test_ApplyStructuredPatch_Account exercises UpdateAccount's nested +// "/throttle/{rateLimit,burstLimit}" edits (merged across separate PATCH +// calls, since each only touches one sub-field) and the top-level +// "/cloudwatchRoleArn" field, which UpdateAccountInput previously lacked +// entirely (so it could never be set via UpdateAccount at all). +func Test_ApplyStructuredPatch_Account(t *testing.T) { + t.Parallel() + + h := newAPIGWHandler() + + initial, err := h.Backend.GetAccount() + require.NoError(t, err) + require.NotNil(t, initial.ThrottleSettings, "the backend seeds default account throttle settings") + initialBurstLimit := initial.ThrottleSettings.BurstLimit + + rec := restRequest(t, h, http.MethodPatch, "/account", + `[{"op":"replace","path":"/cloudwatchRoleArn","value":"arn:aws:iam::123456789012:role/apigw"}]`) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + + acct, err := h.Backend.GetAccount() + require.NoError(t, err) + assert.Equal(t, "arn:aws:iam::123456789012:role/apigw", acct.CloudwatchRoleARN) + + rec = restRequest(t, h, http.MethodPatch, "/account", + `[{"op":"replace","path":"/throttle/rateLimit","value":"100.5"}]`) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + + acct, err = h.Backend.GetAccount() + require.NoError(t, err) + require.NotNil(t, acct.ThrottleSettings) + assert.InDelta(t, 100.5, acct.ThrottleSettings.RateLimit, 0.001) + assert.Equal(t, initialBurstLimit, acct.ThrottleSettings.BurstLimit, "untouched sub-field keeps its prior value") + + rec = restRequest(t, h, http.MethodPatch, "/account", + `[{"op":"replace","path":"/throttle/burstLimit","value":"50"}]`) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + + acct, err = h.Backend.GetAccount() + require.NoError(t, err) + require.NotNil(t, acct.ThrottleSettings) + assert.InDelta(t, 100.5, acct.ThrottleSettings.RateLimit, 0.001, "earlier rateLimit patch must not be clobbered") + assert.Equal(t, 50, acct.ThrottleSettings.BurstLimit) + assert.Equal(t, "arn:aws:iam::123456789012:role/apigw", acct.CloudwatchRoleARN, "unrelated field untouched") +} + +// Test_ApplyStructuredPatch_UsagePlanAPIStages exercises the single-segment +// "/apiStages" add/remove membership edit, where AWS's Value is the string +// "{restApiId}:{stage}" (not a nested path). It also proves the companion fix +// to InMemoryBackend.UpdateUsagePlan, which previously only applied APIStages +// when non-empty (`len(...) > 0`), silently ignoring a patch that removes the +// last remaining API stage. +func Test_ApplyStructuredPatch_UsagePlanAPIStages(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + op string + want []apigateway.APIStageAssociation + }{ + { + name: "add merges with the existing stage", + op: `{"op":"add","path":"/apiStages","value":"api2:prod"}`, + want: []apigateway.APIStageAssociation{ + {RestAPIID: "api1", Stage: "dev"}, + {RestAPIID: "api2", Stage: "prod"}, + }, + }, + { + name: "add is idempotent", + op: `{"op":"add","path":"/apiStages","value":"api1:dev"}`, + want: []apigateway.APIStageAssociation{{RestAPIID: "api1", Stage: "dev"}}, + }, + { + name: "remove the only stage empties the list", + op: `{"op":"remove","path":"/apiStages","value":"api1:dev"}`, + want: []apigateway.APIStageAssociation{}, + }, + { + name: "remove a non-member stage is a no-op", + op: `{"op":"remove","path":"/apiStages","value":"api9:staging"}`, + want: []apigateway.APIStageAssociation{{RestAPIID: "api1", Stage: "dev"}}, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newAPIGWHandler() + plan, err := h.Backend.CreateUsagePlan(apigateway.CreateUsagePlanInput{ + Name: "plan", + APIStages: []apigateway.APIStageAssociation{{RestAPIID: "api1", Stage: "dev"}}, + }) + require.NoError(t, err) + + body := fmt.Sprintf(`{"patchOperations":[%s]}`, tt.op) + rec := restRequest(t, h, http.MethodPatch, fmt.Sprintf("/usageplans/%s", plan.ID), body) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + + got, err := h.Backend.GetUsagePlan(plan.ID) + require.NoError(t, err) + assert.ElementsMatch(t, tt.want, got.APIStages) + }) + } +} + +// Test_ApplyStructuredPatch_GatewayResponseMerge proves UpdateGatewayResponse +// merges a per-key PATCH ("/responseParameters/{key}") with the response's +// existing StatusCode/ResponseTemplates/other ResponseParameters entries, +// instead of wholesale-replacing the resource (which previously reused +// PutGatewayResponse's full-replace semantics and silently reverted +// StatusCode to its default and wiped ResponseTemplates on every PATCH). +func Test_ApplyStructuredPatch_GatewayResponseMerge(t *testing.T) { + t.Parallel() + + h := newAPIGWHandler() + api, err := h.Backend.CreateRestAPI(apigateway.CreateRestAPIInput{Name: "gwresp-test-api"}) + require.NoError(t, err) + + putBody := `{ + "statusCode":"404", + "responseParameters":{"gatewayresponse.header.A":"'a-value'"}, + "responseTemplates":{"application/json":"{\"message\":$context.error.messageString}"} + }` + rec := restRequest(t, h, http.MethodPut, + fmt.Sprintf("/restapis/%s/gatewayresponses/ACCESS_DENIED", api.ID), putBody) + require.Equal(t, http.StatusCreated, rec.Code, "put body: %s", rec.Body.String()) + + patchBody := `[{"op":"add","path":"/responseParameters/gatewayresponse.header.B","value":"'b-value'"}]` + rec = restRequest(t, h, http.MethodPatch, + fmt.Sprintf("/restapis/%s/gatewayresponses/ACCESS_DENIED", api.ID), patchBody) + require.Equal(t, http.StatusOK, rec.Code, "patch body: %s", rec.Body.String()) + + gr, err := h.Backend.GetGatewayResponse(api.ID, "ACCESS_DENIED") + require.NoError(t, err) + assert.Equal(t, "404", gr.StatusCode, "unrelated statusCode must survive the patch") + assert.Equal(t, + map[string]string{"application/json": "{\"message\":$context.error.messageString}"}, + gr.ResponseTemplates, + "unrelated responseTemplates must survive the patch", + ) + assert.Equal(t, map[string]string{ + "gatewayresponse.header.A": "'a-value'", + "gatewayresponse.header.B": "'b-value'", + }, gr.ResponseParameters, "the pre-existing parameter must survive alongside the newly patched one") +} + +// Test_ApplyStructuredPatch_TopLevelReplaceStillWorks is a regression check: +// the common case (a single plain top-level field replace) must keep working +// exactly as before, in both wire forms (bare array and the +// {"patchOperations":[...]} wrapper with sibling fields). +func Test_ApplyStructuredPatch_TopLevelReplaceStillWorks(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + body string + }{ + {name: "bare patch array", body: `[{"op":"replace","path":"/description","value":"updated"}]`}, + { + name: "patchOperations wrapper", + body: `{"patchOperations":[{"op":"replace","path":"/description","value":"updated"}]}`, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newAPIGWHandler() + api, err := h.Backend.CreateRestAPI(apigateway.CreateRestAPIInput{ + Name: "toplevel-test-api", + Description: "original", + }) + require.NoError(t, err) + + rec := restRequest(t, h, http.MethodPatch, fmt.Sprintf("/restapis/%s", api.ID), tt.body) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + + got, err := h.Backend.GetRestAPI(api.ID) + require.NoError(t, err) + assert.Equal(t, "updated", got.Description) + assert.Equal(t, "toplevel-test-api", got.Name, "unrelated field untouched") + }) + } +} diff --git a/services/apigateway/persistence.go b/services/apigateway/persistence.go index 9a974633f..44084aa3e 100644 --- a/services/apigateway/persistence.go +++ b/services/apigateway/persistence.go @@ -2,214 +2,550 @@ package apigateway import ( "context" + "encoding/json" + "fmt" + "maps" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) -type apiDataSnapshot struct { - Resources map[string]*Resource `json:"resources"` - Deployments map[string]*Deployment `json:"deployments"` - Stages map[string]*Stage `json:"stages"` - Authorizers map[string]*Authorizer `json:"authorizers"` - RequestValidators map[string]*RequestValidator `json:"requestValidators"` - DocumentationParts map[string]*DocumentationPart `json:"documentationParts"` - DocumentationVersions map[string]*DocumentationVersion `json:"documentationVersions"` - Models map[string]*Model `json:"models"` - API RestAPI `json:"api"` +// apigatewaySnapshotVersion identifies the shape of backendSnapshot's Tables +// blob (the set of "clean" tables on b.registry plus the DTO tables built +// below for the "dirty" ones -- see store_setup.go's registerAllTables doc +// for the clean/dirty split). Bump whenever a change to a DTO type, a +// registered table's value type, or backendSnapshot itself would make an +// older snapshot unsafe to decode as the current shape. Restore compares this +// against the persisted value and discards (rather than attempts to +// partially decode) any mismatch -- see Restore below. Mirrors the +// services/sqs pilot (commit 0f09d77c) and services/ec2 (commit 12e611a4). +const apigatewaySnapshotVersion = 1 + +// resourceSnapshot, deploymentSnapshot, stageSnapshot, authorizerSnapshot, +// requestValidatorSnapshot, documentationPartSnapshot, +// documentationVersionSnapshot, modelSnapshot, and usagePlanKeySnapshot are +// DTOs used ONLY for Snapshot/Restore. Each mirrors its live type field for +// field, except the identity field (RestAPIID / UsagePlanID) is given a real +// JSON tag here instead of the live type's `json:"-"` (see models.go) -- +// marshaling the live type directly would silently drop that field, and +// unmarshaling into it would leave it permanently empty, corrupting the flat +// table's composite key on restore. This is the same DTO-registry technique +// services/sqs uses for its Queue/moveTaskState types (commit 0f09d77c), +// applied here for a different reason (a json:"-" identity field rather than +// live-only runtime state like channels/mutexes). +type resourceSnapshot struct { + ResourceMethods map[string]*Method `json:"resourceMethods,omitempty"` + CorsConfiguration *CorsConfiguration `json:"corsConfiguration,omitempty"` + ID string `json:"id"` + ParentID string `json:"parentId,omitempty"` + PathPart string `json:"pathPart,omitempty"` + Path string `json:"path"` + RestAPIID string `json:"restApiId"` } -type backendSnapshot struct { - APIs map[string]*apiDataSnapshot `json:"apis"` - APIKeys map[string]*APIKey `json:"apiKeys"` - BasePathMappings map[string]*BasePathMapping `json:"basePathMappings"` - DomainNames map[string]*DomainName `json:"domainNames"` - DomainNameAccessAssociations map[string]*DomainNameAccessAssociation `json:"domainNameAccessAssociations"` - UsagePlans map[string]*UsagePlan `json:"usagePlans"` - UsagePlanKeys map[string]map[string]*UsagePlanKey `json:"usagePlanKeys"` - Account *Account `json:"account,omitempty"` - GatewayResponses map[string]*GatewayResponse `json:"gatewayResponses,omitempty"` - ClientCertificates map[string]*ClientCertificate `json:"clientCertificates,omitempty"` - VpcLinks map[string]*VpcLink `json:"vpcLinks,omitempty"` - UsageOverrides map[string]map[string]int64 `json:"usageOverrides,omitempty"` +func resourceSnapshotKey(v *resourceSnapshot) string { return resourceKey(v.RestAPIID, v.ID) } + +func toResourceSnapshot(v *Resource) *resourceSnapshot { + return &resourceSnapshot{ + ID: v.ID, + ParentID: v.ParentID, + PathPart: v.PathPart, + Path: v.Path, + RestAPIID: v.RestAPIID, + ResourceMethods: v.ResourceMethods, + CorsConfiguration: v.CorsConfiguration, + } } -// Snapshot serialises the backend state to JSON. -// It implements persistence.Persistable. -func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { - b.mu.RLock("Snapshot") - defer b.mu.RUnlock() - - snap := backendSnapshot{ - APIs: make(map[string]*apiDataSnapshot, len(b.apis)), - APIKeys: b.apiKeys, - BasePathMappings: b.basePathMappings, - DomainNames: b.domainNames, - DomainNameAccessAssociations: b.domainNameAccessAssociations, - UsagePlans: b.usagePlans, - UsagePlanKeys: b.usagePlanKeys, - Account: b.account, - GatewayResponses: b.gatewayResponses, - ClientCertificates: b.clientCertificates, - VpcLinks: b.vpcLinks, - UsageOverrides: b.usageOverrides, - } - - for id, d := range b.apis { - snap.APIs[id] = &apiDataSnapshot{ - API: d.api, - Resources: d.resources, - Deployments: d.deployments, - Stages: d.stages, - Authorizers: d.authorizers, - RequestValidators: d.requestValidators, - DocumentationParts: d.documentationParts, - DocumentationVersions: d.documentationVersions, - Models: d.models, - } +func fromResourceSnapshot(v *resourceSnapshot) *Resource { + return &Resource{ + ID: v.ID, + ParentID: v.ParentID, + PathPart: v.PathPart, + Path: v.Path, + RestAPIID: v.RestAPIID, + ResourceMethods: v.ResourceMethods, + CorsConfiguration: v.CorsConfiguration, } +} - return persistence.MarshalSnapshot(ctx, "apigateway", snap) +type deploymentSnapshot struct { + CreatedDate unixEpochTime `json:"createdDate"` + ID string `json:"id"` + RestAPIID string `json:"restApiId"` + Description string `json:"description,omitempty"` } -// Restore loads backend state from a JSON snapshot. -// It implements persistence.Persistable. -func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { - var snap backendSnapshot +func deploymentSnapshotKey(v *deploymentSnapshot) string { return deploymentKey(v.RestAPIID, v.ID) } - if err := persistence.UnmarshalSnapshot(ctx, "apigateway", data, &snap); err != nil { - return err +func toDeploymentSnapshot(v *Deployment) *deploymentSnapshot { + return &deploymentSnapshot{ + ID: v.ID, + RestAPIID: v.RestAPIID, + Description: v.Description, + CreatedDate: v.CreatedDate, } +} - b.mu.Lock("Restore") - defer b.mu.Unlock() +func fromDeploymentSnapshot(v *deploymentSnapshot) *Deployment { + return &Deployment{ + ID: v.ID, + RestAPIID: v.RestAPIID, + Description: v.Description, + CreatedDate: v.CreatedDate, + } +} - b.restoreAPIs(snap.APIs) - b.restoreMaps(snap) +type stageSnapshot struct { + CanarySettings *CanarySettings `json:"canarySettings,omitempty"` + AccessLogSettings *AccessLogSettings `json:"accessLogSettings,omitempty"` + MethodSettings map[string]MethodSetting `json:"methodSettings,omitempty"` + Variables map[string]string `json:"variables,omitempty"` + CreatedDate unixEpochTime `json:"createdDate"` + LastUpdatedDate unixEpochTime `json:"lastUpdatedDate"` + StageName string `json:"stageName"` + RestAPIID string `json:"restApiId"` + DeploymentID string `json:"deploymentId"` + Description string `json:"description,omitempty"` + ClientCertificateID string `json:"clientCertificateId,omitempty"` + CacheClusterSize string `json:"cacheClusterSize,omitempty"` + CacheClusterStatus string `json:"cacheClusterStatus,omitempty"` + InvokeURL string `json:"invokeUrl,omitempty"` + TracingEnabled bool `json:"tracingEnabled,omitempty"` + CacheClusterEnabled bool `json:"cacheClusterEnabled,omitempty"` +} - return nil +func stageSnapshotKey(v *stageSnapshot) string { return stageKey(v.RestAPIID, v.StageName) } + +func toStageSnapshot(v *Stage) *stageSnapshot { + return &stageSnapshot{ + CanarySettings: v.CanarySettings, + AccessLogSettings: v.AccessLogSettings, + MethodSettings: v.MethodSettings, + Variables: v.Variables, + CreatedDate: v.CreatedDate, + LastUpdatedDate: v.LastUpdatedDate, + StageName: v.StageName, + RestAPIID: v.RestAPIID, + DeploymentID: v.DeploymentID, + Description: v.Description, + ClientCertificateID: v.ClientCertificateID, + CacheClusterSize: v.CacheClusterSize, + CacheClusterStatus: v.CacheClusterStatus, + InvokeURL: v.InvokeURL, + TracingEnabled: v.TracingEnabled, + CacheClusterEnabled: v.CacheClusterEnabled, + } } -// restoreAPIs restores API data from the snapshot. -func (b *InMemoryBackend) restoreAPIs(apis map[string]*apiDataSnapshot) { - b.apis = make(map[string]*apiData, len(apis)) +func fromStageSnapshot(v *stageSnapshot) *Stage { + return &Stage{ + CanarySettings: v.CanarySettings, + AccessLogSettings: v.AccessLogSettings, + MethodSettings: v.MethodSettings, + Variables: v.Variables, + CreatedDate: v.CreatedDate, + LastUpdatedDate: v.LastUpdatedDate, + StageName: v.StageName, + RestAPIID: v.RestAPIID, + DeploymentID: v.DeploymentID, + Description: v.Description, + ClientCertificateID: v.ClientCertificateID, + CacheClusterSize: v.CacheClusterSize, + CacheClusterStatus: v.CacheClusterStatus, + InvokeURL: v.InvokeURL, + TracingEnabled: v.TracingEnabled, + CacheClusterEnabled: v.CacheClusterEnabled, + } +} + +type authorizerSnapshot struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + AuthorizerURI string `json:"authorizerUri,omitempty"` + AuthorizerCredentials string `json:"authorizerCredentials,omitempty"` + IdentitySource string `json:"identitySource,omitempty"` + IdentityValidationExpression string `json:"identityValidationExpression,omitempty"` + RestAPIID string `json:"restApiId"` + ProviderARNs []string `json:"providerARNs,omitempty"` + AuthorizerResultTTLInSeconds int `json:"authorizerResultTtlInSeconds,omitempty"` +} - for id, d := range apis { - ensureAPIDataMaps(d) - b.apis[id] = &apiData{ - api: d.API, - resources: d.Resources, - deployments: d.Deployments, - stages: d.Stages, - authorizers: d.Authorizers, - requestValidators: d.RequestValidators, - documentationParts: d.DocumentationParts, - documentationVersions: d.DocumentationVersions, - models: d.Models, - } +func authorizerSnapshotKey(v *authorizerSnapshot) string { return authorizerKey(v.RestAPIID, v.ID) } + +func toAuthorizerSnapshot(v *Authorizer) *authorizerSnapshot { + return &authorizerSnapshot{ + ID: v.ID, + Name: v.Name, + Type: v.Type, + AuthorizerURI: v.AuthorizerURI, + AuthorizerCredentials: v.AuthorizerCredentials, + IdentitySource: v.IdentitySource, + IdentityValidationExpression: v.IdentityValidationExpression, + ProviderARNs: v.ProviderARNs, + RestAPIID: v.RestAPIID, + AuthorizerResultTTLInSeconds: v.AuthorizerResultTTLInSeconds, } } -// ensureAPIDataMaps initialises nil map fields in an API data snapshot. -func ensureAPIDataMaps(d *apiDataSnapshot) { - if d.Resources == nil { - d.Resources = make(map[string]*Resource) +func fromAuthorizerSnapshot(v *authorizerSnapshot) *Authorizer { + return &Authorizer{ + ID: v.ID, + Name: v.Name, + Type: v.Type, + AuthorizerURI: v.AuthorizerURI, + AuthorizerCredentials: v.AuthorizerCredentials, + IdentitySource: v.IdentitySource, + IdentityValidationExpression: v.IdentityValidationExpression, + ProviderARNs: v.ProviderARNs, + RestAPIID: v.RestAPIID, + AuthorizerResultTTLInSeconds: v.AuthorizerResultTTLInSeconds, } +} + +type requestValidatorSnapshot struct { + ID string `json:"id"` + Name string `json:"name"` + RestAPIID string `json:"restApiId"` + ValidateRequestBody bool `json:"validateRequestBody"` + ValidateRequestParameters bool `json:"validateRequestParameters"` +} + +func requestValidatorSnapshotKey(v *requestValidatorSnapshot) string { + return requestValidatorKey(v.RestAPIID, v.ID) +} - if d.Deployments == nil { - d.Deployments = make(map[string]*Deployment) +func toRequestValidatorSnapshot(v *RequestValidator) *requestValidatorSnapshot { + return &requestValidatorSnapshot{ + ID: v.ID, + Name: v.Name, + ValidateRequestBody: v.ValidateRequestBody, + ValidateRequestParameters: v.ValidateRequestParameters, + RestAPIID: v.RestAPIID, } +} - if d.Stages == nil { - d.Stages = make(map[string]*Stage) +func fromRequestValidatorSnapshot(v *requestValidatorSnapshot) *RequestValidator { + return &RequestValidator{ + ID: v.ID, + Name: v.Name, + ValidateRequestBody: v.ValidateRequestBody, + ValidateRequestParameters: v.ValidateRequestParameters, + RestAPIID: v.RestAPIID, } +} - if d.Authorizers == nil { - d.Authorizers = make(map[string]*Authorizer) +type documentationPartSnapshot struct { + Location DocumentationLocation `json:"location"` + ID string `json:"id"` + RestAPIID string `json:"restApiId"` + Properties string `json:"properties"` +} + +func documentationPartSnapshotKey(v *documentationPartSnapshot) string { + return documentationPartKey(v.RestAPIID, v.ID) +} + +func toDocumentationPartSnapshot(v *DocumentationPart) *documentationPartSnapshot { + return &documentationPartSnapshot{ + Location: v.Location, + ID: v.ID, + RestAPIID: v.RestAPIID, + Properties: v.Properties, } +} - if d.RequestValidators == nil { - d.RequestValidators = make(map[string]*RequestValidator) +func fromDocumentationPartSnapshot(v *documentationPartSnapshot) *DocumentationPart { + return &DocumentationPart{ + Location: v.Location, + ID: v.ID, + RestAPIID: v.RestAPIID, + Properties: v.Properties, } +} - if d.DocumentationParts == nil { - d.DocumentationParts = make(map[string]*DocumentationPart) +type documentationVersionSnapshot struct { + CreatedDate unixEpochTime `json:"createdDate"` + RestAPIID string `json:"restApiId"` + Version string `json:"version"` + Description string `json:"description,omitempty"` +} + +func documentationVersionSnapshotKey(v *documentationVersionSnapshot) string { + return documentationVersionKey(v.RestAPIID, v.Version) +} + +func toDocumentationVersionSnapshot(v *DocumentationVersion) *documentationVersionSnapshot { + return &documentationVersionSnapshot{ + CreatedDate: v.CreatedDate, + RestAPIID: v.RestAPIID, + Version: v.Version, + Description: v.Description, } +} - if d.DocumentationVersions == nil { - d.DocumentationVersions = make(map[string]*DocumentationVersion) +func fromDocumentationVersionSnapshot(v *documentationVersionSnapshot) *DocumentationVersion { + return &DocumentationVersion{ + CreatedDate: v.CreatedDate, + RestAPIID: v.RestAPIID, + Version: v.Version, + Description: v.Description, } +} + +type modelSnapshot struct { + ID string `json:"id"` + RestAPIID string `json:"restApiId"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + ContentType string `json:"contentType,omitempty"` + Schema string `json:"schema,omitempty"` +} + +func modelSnapshotKey(v *modelSnapshot) string { return modelKey(v.RestAPIID, v.ID) } - if d.Models == nil { - d.Models = make(map[string]*Model) +func toModelSnapshot(v *Model) *modelSnapshot { + return &modelSnapshot{ + ID: v.ID, + RestAPIID: v.RestAPIID, + Name: v.Name, + Description: v.Description, + ContentType: v.ContentType, + Schema: v.Schema, } } -// restoreMaps restores the flat map fields from the snapshot. -func (b *InMemoryBackend) restoreMaps(snap backendSnapshot) { - if snap.APIKeys != nil { - b.apiKeys = snap.APIKeys - } else { - b.apiKeys = make(map[string]*APIKey) +func fromModelSnapshot(v *modelSnapshot) *Model { + return &Model{ + ID: v.ID, + RestAPIID: v.RestAPIID, + Name: v.Name, + Description: v.Description, + ContentType: v.ContentType, + Schema: v.Schema, } +} - if snap.BasePathMappings != nil { - b.basePathMappings = snap.BasePathMappings - } else { - b.basePathMappings = make(map[string]*BasePathMapping) +type usagePlanKeySnapshot struct { + ID string `json:"id"` + Type string `json:"type"` + Value string `json:"value,omitempty"` + Name string `json:"name,omitempty"` + UsagePlanID string `json:"usagePlanId"` +} + +func usagePlanKeySnapshotKey(v *usagePlanKeySnapshot) string { + return usagePlanKeyKey(v.UsagePlanID, v.ID) +} + +func toUsagePlanKeySnapshot(v *UsagePlanKey) *usagePlanKeySnapshot { + return &usagePlanKeySnapshot{ + ID: v.ID, + Type: v.Type, + Value: v.Value, + Name: v.Name, + UsagePlanID: v.UsagePlanID, } +} - if snap.DomainNames != nil { - b.domainNames = snap.DomainNames - } else { - b.domainNames = make(map[string]*DomainName) +func fromUsagePlanKeySnapshot(v *usagePlanKeySnapshot) *UsagePlanKey { + return &UsagePlanKey{ + ID: v.ID, + Type: v.Type, + Value: v.Value, + Name: v.Name, + UsagePlanID: v.UsagePlanID, } +} - if snap.DomainNameAccessAssociations != nil { - b.domainNameAccessAssociations = snap.DomainNameAccessAssociations - } else { - b.domainNameAccessAssociations = make(map[string]*DomainNameAccessAssociation) +// dirtyTableNames lists the "dirty" table names shared by Snapshot and +// Restore (see store_setup.go's registerAllTables doc). Both build an +// ephemeral DTO [store.Registry] under these exact names, so a snapshot +// written by one version of Snapshot always lines up with the same version +// of Restore. +// +//nolint:gochecknoglobals // fixed lookup table, mirrors errCodeLookup-style tables elsewhere +var dirtyTableNames = struct { + resources, deployments, stages, authorizers, requestValidators, + documentationParts, documentationVersions, models, usagePlanKeys string +}{ + resources: "resources", + deployments: "deployments", + stages: "stages", + authorizers: "authorizers", + requestValidators: "requestValidators", + documentationParts: "documentationParts", + documentationVersions: "documentationVersions", + models: "models", + usagePlanKeys: "usagePlanKeys", +} + +// backendSnapshot is the top-level on-disk shape for the API Gateway backend. +// +// Tables holds one JSON-encoded array per table -- both the "clean" tables +// registered on b.registry (produced by [store.Registry.SnapshotAll]) and the +// "dirty" DTO tables built inline in Snapshot (see store_setup.go's +// registerAllTables doc for the clean/dirty split), merged into one map so a +// single Tables blob round-trips the whole backend. +type backendSnapshot struct { + Tables map[string]json.RawMessage `json:"tables"` + Account *Account `json:"account,omitempty"` + UsageOverrides map[string]map[string]int64 `json:"usageOverrides,omitempty"` + Version int `json:"version"` +} + +// Snapshot serialises the backend state to JSON. +// It implements persistence.Persistable. +func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { + b.mu.RLock("Snapshot") + defer b.mu.RUnlock() + + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "apigateway: snapshot table marshal failed", "error", err) + + return nil } - if snap.UsagePlans != nil { - b.usagePlans = snap.UsagePlans - } else { - b.usagePlans = make(map[string]*UsagePlan) + dtoReg := store.NewRegistry() + resourceDTOs := store.Register(dtoReg, dirtyTableNames.resources, store.New(resourceSnapshotKey)) + deploymentDTOs := store.Register(dtoReg, dirtyTableNames.deployments, store.New(deploymentSnapshotKey)) + stageDTOs := store.Register(dtoReg, dirtyTableNames.stages, store.New(stageSnapshotKey)) + authorizerDTOs := store.Register(dtoReg, dirtyTableNames.authorizers, store.New(authorizerSnapshotKey)) + requestValidatorDTOs := store.Register( + dtoReg, dirtyTableNames.requestValidators, store.New(requestValidatorSnapshotKey), + ) + documentationPartDTOs := store.Register( + dtoReg, dirtyTableNames.documentationParts, store.New(documentationPartSnapshotKey), + ) + documentationVersionDTOs := store.Register( + dtoReg, dirtyTableNames.documentationVersions, store.New(documentationVersionSnapshotKey), + ) + modelDTOs := store.Register(dtoReg, dirtyTableNames.models, store.New(modelSnapshotKey)) + usagePlanKeyDTOs := store.Register(dtoReg, dirtyTableNames.usagePlanKeys, store.New(usagePlanKeySnapshotKey)) + + for _, v := range b.resources.Snapshot() { + resourceDTOs.Put(toResourceSnapshot(v)) + } + for _, v := range b.deployments.Snapshot() { + deploymentDTOs.Put(toDeploymentSnapshot(v)) + } + for _, v := range b.stages.Snapshot() { + stageDTOs.Put(toStageSnapshot(v)) + } + for _, v := range b.authorizers.Snapshot() { + authorizerDTOs.Put(toAuthorizerSnapshot(v)) + } + for _, v := range b.requestValidators.Snapshot() { + requestValidatorDTOs.Put(toRequestValidatorSnapshot(v)) + } + for _, v := range b.documentationParts.Snapshot() { + documentationPartDTOs.Put(toDocumentationPartSnapshot(v)) + } + for _, v := range b.documentationVersions.Snapshot() { + documentationVersionDTOs.Put(toDocumentationVersionSnapshot(v)) + } + for _, v := range b.models.Snapshot() { + modelDTOs.Put(toModelSnapshot(v)) + } + for _, v := range b.usagePlanKeys.Snapshot() { + usagePlanKeyDTOs.Put(toUsagePlanKeySnapshot(v)) } - if snap.UsagePlanKeys != nil { - b.usagePlanKeys = snap.UsagePlanKeys - } else { - b.usagePlanKeys = make(map[string]map[string]*UsagePlanKey) + dirtyTables, err := dtoReg.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, "apigateway: snapshot DTO table marshal failed", "error", err) + + return nil } - if snap.Account != nil { - b.account = snap.Account - } else { - b.account = &Account{} + maps.Copy(tables, dirtyTables) + + snap := backendSnapshot{ + Version: apigatewaySnapshotVersion, + Tables: tables, + Account: b.account, + UsageOverrides: b.usageOverrides, } - b.restoreMapsExt(snap) + return persistence.MarshalSnapshot(ctx, "apigateway", snap) } -// restoreMapsExt restores the remaining flat map fields, split out from -// restoreMaps to keep cognitive complexity down. -func (b *InMemoryBackend) restoreMapsExt(snap backendSnapshot) { - if snap.GatewayResponses != nil { - b.gatewayResponses = snap.GatewayResponses - } else { - b.gatewayResponses = make(map[string]*GatewayResponse) +// Restore loads backend state from a JSON snapshot. +// It implements persistence.Persistable. +func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { + var snap backendSnapshot + + if err := persistence.UnmarshalSnapshot(ctx, "apigateway", data, &snap); err != nil { + return err } - if snap.ClientCertificates != nil { - b.clientCertificates = snap.ClientCertificates - } else { - b.clientCertificates = make(map[string]*ClientCertificate) + b.mu.Lock("Restore") + defer b.mu.Unlock() + + if snap.Version != apigatewaySnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. + logger.Load(ctx).WarnContext(ctx, + "apigateway: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", apigatewaySnapshotVersion) + + b.registry.ResetAll() + b.resources.Reset() + b.deployments.Reset() + b.stages.Reset() + b.authorizers.Reset() + b.requestValidators.Reset() + b.documentationParts.Reset() + b.documentationVersions.Reset() + b.models.Reset() + b.usagePlanKeys.Reset() + + return nil } - if snap.VpcLinks != nil { - b.vpcLinks = snap.VpcLinks + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("apigateway: restore snapshot tables: %w", err) + } + + dtoReg := store.NewRegistry() + resourceDTOs := store.Register(dtoReg, dirtyTableNames.resources, store.New(resourceSnapshotKey)) + deploymentDTOs := store.Register(dtoReg, dirtyTableNames.deployments, store.New(deploymentSnapshotKey)) + stageDTOs := store.Register(dtoReg, dirtyTableNames.stages, store.New(stageSnapshotKey)) + authorizerDTOs := store.Register(dtoReg, dirtyTableNames.authorizers, store.New(authorizerSnapshotKey)) + requestValidatorDTOs := store.Register( + dtoReg, dirtyTableNames.requestValidators, store.New(requestValidatorSnapshotKey), + ) + documentationPartDTOs := store.Register( + dtoReg, dirtyTableNames.documentationParts, store.New(documentationPartSnapshotKey), + ) + documentationVersionDTOs := store.Register( + dtoReg, dirtyTableNames.documentationVersions, store.New(documentationVersionSnapshotKey), + ) + modelDTOs := store.Register(dtoReg, dirtyTableNames.models, store.New(modelSnapshotKey)) + usagePlanKeyDTOs := store.Register(dtoReg, dirtyTableNames.usagePlanKeys, store.New(usagePlanKeySnapshotKey)) + + if err := dtoReg.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("apigateway: restore snapshot DTO tables: %w", err) + } + + restoreDirtyTables(b, resourceDTOs, deploymentDTOs, stageDTOs, authorizerDTOs, + requestValidatorDTOs, documentationPartDTOs, documentationVersionDTOs, modelDTOs, usagePlanKeyDTOs) + + if snap.Account != nil { + b.account = snap.Account } else { - b.vpcLinks = make(map[string]*VpcLink) + b.account = &Account{} } if snap.UsageOverrides != nil { @@ -217,6 +553,78 @@ func (b *InMemoryBackend) restoreMapsExt(snap backendSnapshot) { } else { b.usageOverrides = make(map[string]map[string]int64) } + + return nil +} + +// restoreDirtyTables converts each DTO table's contents back to its live +// value type and loads the corresponding live b. via Table.Restore, +// split out from Restore to keep its growth in check. +func restoreDirtyTables( + b *InMemoryBackend, + resourceDTOs *store.Table[resourceSnapshot], + deploymentDTOs *store.Table[deploymentSnapshot], + stageDTOs *store.Table[stageSnapshot], + authorizerDTOs *store.Table[authorizerSnapshot], + requestValidatorDTOs *store.Table[requestValidatorSnapshot], + documentationPartDTOs *store.Table[documentationPartSnapshot], + documentationVersionDTOs *store.Table[documentationVersionSnapshot], + modelDTOs *store.Table[modelSnapshot], + usagePlanKeyDTOs *store.Table[usagePlanKeySnapshot], +) { + resources := make([]*Resource, 0, resourceDTOs.Len()) + for _, v := range resourceDTOs.All() { + resources = append(resources, fromResourceSnapshot(v)) + } + b.resources.Restore(resources) + + deployments := make([]*Deployment, 0, deploymentDTOs.Len()) + for _, v := range deploymentDTOs.All() { + deployments = append(deployments, fromDeploymentSnapshot(v)) + } + b.deployments.Restore(deployments) + + stages := make([]*Stage, 0, stageDTOs.Len()) + for _, v := range stageDTOs.All() { + stages = append(stages, fromStageSnapshot(v)) + } + b.stages.Restore(stages) + + authorizers := make([]*Authorizer, 0, authorizerDTOs.Len()) + for _, v := range authorizerDTOs.All() { + authorizers = append(authorizers, fromAuthorizerSnapshot(v)) + } + b.authorizers.Restore(authorizers) + + requestValidators := make([]*RequestValidator, 0, requestValidatorDTOs.Len()) + for _, v := range requestValidatorDTOs.All() { + requestValidators = append(requestValidators, fromRequestValidatorSnapshot(v)) + } + b.requestValidators.Restore(requestValidators) + + documentationParts := make([]*DocumentationPart, 0, documentationPartDTOs.Len()) + for _, v := range documentationPartDTOs.All() { + documentationParts = append(documentationParts, fromDocumentationPartSnapshot(v)) + } + b.documentationParts.Restore(documentationParts) + + documentationVersions := make([]*DocumentationVersion, 0, documentationVersionDTOs.Len()) + for _, v := range documentationVersionDTOs.All() { + documentationVersions = append(documentationVersions, fromDocumentationVersionSnapshot(v)) + } + b.documentationVersions.Restore(documentationVersions) + + models := make([]*Model, 0, modelDTOs.Len()) + for _, v := range modelDTOs.All() { + models = append(models, fromModelSnapshot(v)) + } + b.models.Restore(models) + + usagePlanKeys := make([]*UsagePlanKey, 0, usagePlanKeyDTOs.Len()) + for _, v := range usagePlanKeyDTOs.All() { + usagePlanKeys = append(usagePlanKeys, fromUsagePlanKeySnapshot(v)) + } + b.usagePlanKeys.Restore(usagePlanKeys) } // Snapshot implements persistence.Persistable by delegating to the backend. diff --git a/services/apigateway/persistence_test.go b/services/apigateway/persistence_test.go index 223d86ee4..56ce40f94 100644 --- a/services/apigateway/persistence_test.go +++ b/services/apigateway/persistence_test.go @@ -71,6 +71,191 @@ func TestInMemoryBackend_SnapshotRestore(t *testing.T) { } } +// TestInMemoryBackend_SnapshotRestore_FullState exercises every resource +// family touched by the Phase 3.3 pkgs/store conversion, in particular the +// ones whose composite key (restAPIID#childID / usagePlanID#keyID) depends on +// a field tagged json:"-" on the live type (Resource, Deployment, Stage, +// Authorizer, RequestValidator, DocumentationPart, DocumentationVersion, +// Model, UsagePlanKey -- see store_setup.go and persistence.go). If the +// DTO-registry snapshot/restore path in persistence.go ever dropped that +// field, every one of these lookups would fail post-restore even though the +// resource "exists" in the restored table under the wrong (unprefixed) key. +// This is a single sequential lifecycle (create everything once, snapshot, +// restore, verify everything), not a table of independent cases, so it does +// not need t.Run subtests. +func TestInMemoryBackend_SnapshotRestore_FullState(t *testing.T) { + t.Parallel() + + b := apigateway.NewInMemoryBackend() + + api, err := b.CreateRestAPI(apigateway.CreateRestAPIInput{Name: "full-api", Description: "full"}) + require.NoError(t, err) + + resource, err := b.CreateResource(api.ID, api.RootResourceID, "widgets") + require.NoError(t, err) + + deployment, err := b.CreateDeployment(api.ID, "prod", "initial") + require.NoError(t, err) + + stage, err := b.GetStage(api.ID, "prod") + require.NoError(t, err) + assert.Equal(t, deployment.ID, stage.DeploymentID) + + authorizer, err := b.CreateAuthorizer(api.ID, apigateway.CreateAuthorizerInput{ + Name: "auth1", Type: "TOKEN", AuthorizerURI: "arn:aws:lambda:us-east-1:1:function:f/invocations", + }) + require.NoError(t, err) + + validator, err := b.CreateRequestValidator(api.ID, apigateway.CreateRequestValidatorInput{ + Name: "validator1", ValidateRequestBody: true, + }) + require.NoError(t, err) + + docPart, err := b.CreateDocumentationPart(apigateway.CreateDocumentationPartInput{ + RestAPIID: api.ID, + Location: apigateway.DocumentationLocation{Type: "API"}, + Properties: `{"description":"d"}`, + }) + require.NoError(t, err) + + docVersion, err := b.CreateDocumentationVersion(apigateway.CreateDocumentationVersionInput{ + RestAPIID: api.ID, Version: "v1", + }) + require.NoError(t, err) + + model, err := b.CreateModel(apigateway.CreateModelInput{ + RestAPIID: api.ID, Name: "Widget", ContentType: "application/json", Schema: `{"type":"object"}`, + }) + require.NoError(t, err) + + apiKey, err := b.CreateAPIKey(apigateway.CreateAPIKeyInput{Name: "key1", Enabled: true}) + require.NoError(t, err) + + domainName, err := b.CreateDomainName(apigateway.CreateDomainNameInput{DomainName: "api.example.com"}) + require.NoError(t, err) + + basePathMapping, err := b.CreateBasePathMapping(apigateway.CreateBasePathMappingInput{ + DomainName: domainName.DomainNameValue, BasePath: "v1", RestAPIID: api.ID, Stage: "prod", + }) + require.NoError(t, err) + + domainAssoc, err := b.CreateDomainNameAccessAssociation(apigateway.CreateDomainNameAccessAssociationInput{ + DomainNameARN: "arn:aws:apigateway:us-east-1::/domainnames/api.example.com", + AccessAssociationSource: "vpce-123", + AccessAssociationSourceType: "VPCE", + }) + require.NoError(t, err) + + usagePlan, err := b.CreateUsagePlan(apigateway.CreateUsagePlanInput{Name: "plan1"}) + require.NoError(t, err) + + usagePlanKey, err := b.CreateUsagePlanKey(apigateway.CreateUsagePlanKeyInput{ + UsagePlanID: usagePlan.ID, KeyID: apiKey.ID, KeyType: "API_KEY", + }) + require.NoError(t, err) + + gatewayResponse, err := b.PutGatewayResponse(apigateway.PutGatewayResponseInput{ + RestAPIID: api.ID, ResponseType: "UNAUTHORIZED", StatusCode: "401", + }) + require.NoError(t, err) + + clientCert, err := b.GenerateClientCertificate(apigateway.GenerateClientCertificateInput{Description: "cert1"}) + require.NoError(t, err) + + vpcLink, err := b.CreateVpcLink(apigateway.CreateVpcLinkInput{Name: "link1"}) + require.NoError(t, err) + + snap := b.Snapshot(t.Context()) + require.NotNil(t, snap) + + fresh := apigateway.NewInMemoryBackend() + require.NoError(t, fresh.Restore(t.Context(), snap)) + + gotAPI, err := fresh.GetRestAPI(api.ID) + require.NoError(t, err) + assert.Equal(t, api.Name, gotAPI.Name) + + gotResource, err := fresh.GetResource(api.ID, resource.ID) + require.NoError(t, err) + assert.Equal(t, "widgets", gotResource.PathPart) + + gotDeployment, err := fresh.GetDeployment(api.ID, deployment.ID) + require.NoError(t, err) + assert.Equal(t, deployment.Description, gotDeployment.Description) + + gotStage, err := fresh.GetStage(api.ID, "prod") + require.NoError(t, err) + assert.Equal(t, stage.DeploymentID, gotStage.DeploymentID) + + gotAuthorizer, err := fresh.GetAuthorizer(api.ID, authorizer.ID) + require.NoError(t, err) + assert.Equal(t, authorizer.Name, gotAuthorizer.Name) + + gotValidator, err := fresh.GetRequestValidator(api.ID, validator.ID) + require.NoError(t, err) + assert.Equal(t, validator.Name, gotValidator.Name) + + gotDocPart, err := fresh.GetDocumentationPart(api.ID, docPart.ID) + require.NoError(t, err) + assert.Equal(t, docPart.Properties, gotDocPart.Properties) + + gotDocVersion, err := fresh.GetDocumentationVersion(api.ID, docVersion.Version) + require.NoError(t, err) + assert.Equal(t, docVersion.Version, gotDocVersion.Version) + + gotModel, err := fresh.GetModel(api.ID, model.Name) + require.NoError(t, err) + assert.Equal(t, model.Schema, gotModel.Schema) + + gotAPIKey, err := fresh.GetAPIKey(apiKey.ID) + require.NoError(t, err) + assert.Equal(t, apiKey.Value, gotAPIKey.Value) + + // apiKeysByValue is a pre-existing gap carried over unchanged from before the + // Phase 3.3 pkgs/store conversion: it was never part of the old backendSnapshot + // either (compare git show HEAD:services/apigateway/persistence.go), so it was + // already not rebuilt across a Restore. GetAPIKeyByValue is expected to miss + // here -- this documents the existing limitation rather than "fixing" it as + // part of a storage-layer-only swap. + _, err = fresh.GetAPIKeyByValue(apiKey.Value) + require.Error(t, err) + + gotDomainName, err := fresh.GetDomainName(domainName.DomainNameValue) + require.NoError(t, err) + assert.Equal(t, domainName.DomainNameValue, gotDomainName.DomainNameValue) + + gotBasePathMapping, err := fresh.GetBasePathMapping(basePathMapping.DomainName, basePathMapping.BasePath) + require.NoError(t, err) + assert.Equal(t, api.ID, gotBasePathMapping.RestAPIID) + + gotDomainAssocs, err := fresh.GetDomainNameAccessAssociations("SELF") + require.NoError(t, err) + if assert.Len(t, gotDomainAssocs, 1) { + assert.Equal(t, domainAssoc.DomainNameAccessAssociationARN, gotDomainAssocs[0].DomainNameAccessAssociationARN) + } + + gotUsagePlan, err := fresh.GetUsagePlan(usagePlan.ID) + require.NoError(t, err) + assert.Equal(t, usagePlan.Name, gotUsagePlan.Name) + + gotUsagePlanKey, err := fresh.GetUsagePlanKey(usagePlan.ID, usagePlanKey.ID) + require.NoError(t, err) + assert.Equal(t, usagePlanKey.Value, gotUsagePlanKey.Value) + + gotGatewayResponse, err := fresh.GetGatewayResponse(api.ID, "UNAUTHORIZED") + require.NoError(t, err) + assert.Equal(t, gatewayResponse.StatusCode, gotGatewayResponse.StatusCode) + assert.False(t, gotGatewayResponse.DefaultResponse) + + gotClientCert, err := fresh.GetClientCertificate(clientCert.ClientCertificateID) + require.NoError(t, err) + assert.Equal(t, clientCert.Description, gotClientCert.Description) + + gotVpcLink, err := fresh.GetVpcLink(vpcLink.ID) + require.NoError(t, err) + assert.Equal(t, vpcLink.Name, gotVpcLink.Name) +} + func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { t.Parallel() diff --git a/services/apigateway/store_setup.go b/services/apigateway/store_setup.go new file mode 100644 index 000000000..60140d98d --- /dev/null +++ b/services/apigateway/store_setup.go @@ -0,0 +1,224 @@ +package apigateway + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend is registered exactly once, +// here, as a *store.Table[T] on b.registry. See pkgs/store's package doc and +// the services/ec2 (commit 12e611a4) / services/sqs (commit 0f09d77c) +// conversions this follows. +// +// Several resource families (resources, deployments, stages, authorizers, +// requestValidators, documentationParts, documentationVersions, models, +// usagePlanKeys) were previously nested per-parent maps (map[string]*apiData +// holding its own map[string]*T, or usagePlanID -> keyID -> *UsagePlanKey). +// store.Table has no notion of nesting, so each becomes a single flat table +// keyed by a composite "#" string, with a secondary +// [store.Index] grouping by parent ID for the "all children of parent X" +// lookups the nested maps used to answer directly. This requires the child +// value type to carry its parent's ID as a field purely for identity/keying +// (never on the wire) -- Resource/Stage/Deployment/Model/DocumentationPart/ +// DocumentationVersion already carried `RestAPIID string `json:"-"`` for +// exactly this reason; Authorizer, RequestValidator, and UsagePlanKey gained +// the same field (RestAPIID / UsagePlanID) as part of this conversion. +// +// A handful of fields are deliberately NOT registered here and remain plain +// maps -- see the comment block above registerAllTables for the list and why. +import "github.com/blackbirdworks/gopherstack/pkgs/store" + +func restAPIKeyFn(v *RestAPI) string { return v.ID } + +// resourceKey/deploymentKey/... build the composite "#" +// key shared by every resource family nested under a REST API. Access sites +// use these directly so the exact same key is used for Put/Get/Delete. +func resourceKey(restAPIID, resourceID string) string { return restAPIID + "#" + resourceID } +func resourceKeyFn(v *Resource) string { return resourceKey(v.RestAPIID, v.ID) } + +func deploymentKey(restAPIID, deploymentID string) string { return restAPIID + "#" + deploymentID } +func deploymentKeyFn(v *Deployment) string { return deploymentKey(v.RestAPIID, v.ID) } + +func stageKey(restAPIID, stageName string) string { return restAPIID + "#" + stageName } +func stageKeyFn(v *Stage) string { return stageKey(v.RestAPIID, v.StageName) } + +func authorizerKey(restAPIID, authorizerID string) string { return restAPIID + "#" + authorizerID } +func authorizerKeyFn(v *Authorizer) string { return authorizerKey(v.RestAPIID, v.ID) } + +func requestValidatorKey(restAPIID, validatorID string) string { + return restAPIID + "#" + validatorID +} +func requestValidatorKeyFn(v *RequestValidator) string { return requestValidatorKey(v.RestAPIID, v.ID) } + +func documentationPartKey(restAPIID, docPartID string) string { return restAPIID + "#" + docPartID } +func documentationPartKeyFn(v *DocumentationPart) string { + return documentationPartKey(v.RestAPIID, v.ID) +} + +func documentationVersionKey(restAPIID, version string) string { return restAPIID + "#" + version } +func documentationVersionKeyFn(v *DocumentationVersion) string { + return documentationVersionKey(v.RestAPIID, v.Version) +} + +func modelKey(restAPIID, modelID string) string { return restAPIID + "#" + modelID } +func modelKeyFn(v *Model) string { return modelKey(v.RestAPIID, v.ID) } + +func apiKeyKeyFn(v *APIKey) string { return v.ID } + +func basePathMappingKey(domainName, basePath string) string { return domainName + "#" + basePath } +func basePathMappingKeyFn(v *BasePathMapping) string { + return basePathMappingKey(v.DomainName, v.BasePath) +} + +func domainNameKeyFn(v *DomainName) string { return v.DomainNameValue } + +func domainNameAccessAssociationKeyFn(v *DomainNameAccessAssociation) string { + return v.DomainNameAccessAssociationARN +} + +func usagePlanKeyFn(v *UsagePlan) string { return v.ID } + +func usagePlanKeyKey(usagePlanID, keyID string) string { return usagePlanID + "#" + keyID } +func usagePlanKeyKeyFn(v *UsagePlanKey) string { return usagePlanKeyKey(v.UsagePlanID, v.ID) } + +// gatewayResponseKeyFn reuses gatewayResponseKey (defined in backend.go, next +// to the other GatewayResponse helpers) so the table key matches exactly what +// every access site already computes. +func gatewayResponseKeyFn(v *GatewayResponse) string { + return gatewayResponseKey(v.RestAPIID, v.ResponseType) +} + +func clientCertificateKeyFn(v *ClientCertificate) string { return v.ClientCertificateID } + +func vpcLinkKeyFn(v *VpcLink) string { return v.ID } + +// registerAllTables sets up every converted resource collection exactly once, +// at construction time. +// +// Two groups, split by whether their store.Table can be snapshotted directly: +// +// - "Clean" tables (restApis, apiKeys, basePathMappings, domainNames, +// domainNameAccessAssociations, usagePlans, gatewayResponses, +// clientCertificates, vpcLinks) are registered on b.registry via +// store.Register, so persistence.go's Snapshot/Restore can drive them +// through b.registry.SnapshotAll()/RestoreAll() -- their value types +// marshal to JSON with no information loss. +// - "Dirty" tables (resources, deployments, stages, authorizers, +// requestValidators, documentationParts, documentationVersions, models, +// usagePlanKeys) are built with store.New but deliberately NOT registered +// on b.registry. Their composite key depends on a field +// (RestAPIID/UsagePlanID) tagged `json:"-"` on the live type (see +// models.go), so a direct json.Marshal/Unmarshal round trip through +// Table.Snapshot/Restore would silently drop that field and corrupt the +// table's key on restore. persistence.go instead builds a throwaway DTO +// [store.Registry] purely to get a correctly-tagged JSON encoding (mirrors +// the services/sqs pilot, commit 0f09d77c), then restores the live tables +// directly via Table.Restore. Because they aren't registered on +// b.registry, InMemoryBackend.Reset resets each of them with an explicit +// Table.Reset() call alongside b.registry.ResetAll(). +// +// The following fields are deliberately plain maps, not store.Table at all -- +// see registerAllTables's callers for the full reasoning: +// - resourceVersions (restAPIID -> uint64): an auxiliary mutation counter +// used by ResourcesForRouting to invalidate the data-plane's cached +// routing trie, not a resource collection -- the value has no identity of +// its own to key a Table by. +// - apiKeysByValue (value -> keyID): a plain string->string secondary +// lookup index, not a map[string]*T resource collection. Kept as a +// hand-maintained parallel map (mirroring how it was maintained +// alongside apiKeys before conversion) rather than a store.Index, since +// an Index groups into a slice (1:many) where this lookup is 1:1 +// overwrite-on-collision, matching the pre-conversion map semantics +// exactly. +// - usageOverrides (usagePlanID -> keyID -> remaining quota int64): value +// type is a bare int64 with no identity field of its own to key a Table +// by, same class of exception as EC2's instanceIMDSOptions. +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call (plus any secondary store.AddIndex) to the concrete field and value +// type. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.restApis = store.Register(b.registry, "restApis", store.New(restAPIKeyFn)) + }, + // "Dirty" tables: store.New only, NOT store.Register -- see registerAllTables doc. + func(b *InMemoryBackend) { + b.resources = store.New(resourceKeyFn) + b.resourcesByAPI = b.resources.AddIndex("byAPI", func(v *Resource) string { return v.RestAPIID }) + }, + func(b *InMemoryBackend) { + b.deployments = store.New(deploymentKeyFn) + b.deploymentsByAPI = b.deployments.AddIndex("byAPI", func(v *Deployment) string { return v.RestAPIID }) + }, + func(b *InMemoryBackend) { + b.stages = store.New(stageKeyFn) + b.stagesByAPI = b.stages.AddIndex("byAPI", func(v *Stage) string { return v.RestAPIID }) + }, + func(b *InMemoryBackend) { + b.authorizers = store.New(authorizerKeyFn) + b.authorizersByAPI = b.authorizers.AddIndex("byAPI", func(v *Authorizer) string { return v.RestAPIID }) + }, + func(b *InMemoryBackend) { + b.requestValidators = store.New(requestValidatorKeyFn) + b.requestValidatorsByAPI = b.requestValidators.AddIndex( + "byAPI", + func(v *RequestValidator) string { return v.RestAPIID }, + ) + }, + func(b *InMemoryBackend) { + b.documentationParts = store.New(documentationPartKeyFn) + b.documentationPartsByAPI = b.documentationParts.AddIndex( + "byAPI", + func(v *DocumentationPart) string { return v.RestAPIID }, + ) + }, + func(b *InMemoryBackend) { + b.documentationVersions = store.New(documentationVersionKeyFn) + b.documentationVersionsByAPI = b.documentationVersions.AddIndex( + "byAPI", + func(v *DocumentationVersion) string { return v.RestAPIID }, + ) + }, + func(b *InMemoryBackend) { + b.models = store.New(modelKeyFn) + b.modelsByAPI = b.models.AddIndex("byAPI", func(v *Model) string { return v.RestAPIID }) + }, + func(b *InMemoryBackend) { + b.usagePlanKeys = store.New(usagePlanKeyKeyFn) + b.usagePlanKeysByPlan = b.usagePlanKeys.AddIndex( + "byPlan", + func(v *UsagePlanKey) string { return v.UsagePlanID }, + ) + }, + // "Clean" tables: registered on b.registry. + func(b *InMemoryBackend) { + b.apiKeys = store.Register(b.registry, "apiKeys", store.New(apiKeyKeyFn)) + }, + func(b *InMemoryBackend) { + b.basePathMappings = store.Register(b.registry, "basePathMappings", store.New(basePathMappingKeyFn)) + }, + func(b *InMemoryBackend) { + b.domainNames = store.Register(b.registry, "domainNames", store.New(domainNameKeyFn)) + }, + func(b *InMemoryBackend) { + b.domainNameAccessAssociations = store.Register( + b.registry, "domainNameAccessAssociations", store.New(domainNameAccessAssociationKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.usagePlans = store.Register(b.registry, "usagePlans", store.New(usagePlanKeyFn)) + }, + func(b *InMemoryBackend) { + b.gatewayResponses = store.Register(b.registry, "gatewayResponses", store.New(gatewayResponseKeyFn)) + }, + func(b *InMemoryBackend) { + b.clientCertificates = store.Register(b.registry, "clientCertificates", store.New(clientCertificateKeyFn)) + }, + func(b *InMemoryBackend) { + b.vpcLinks = store.Register(b.registry, "vpcLinks", store.New(vpcLinkKeyFn)) + }, +} diff --git a/services/apigatewayv2/PARITY.md b/services/apigatewayv2/PARITY.md new file mode 100644 index 000000000..741807560 --- /dev/null +++ b/services/apigatewayv2/PARITY.md @@ -0,0 +1,162 @@ +--- +service: apigatewayv2 +sdk_module: aws-sdk-go-v2/service/apigatewayv2@v1.33.7 +last_audit_commit: 0b1194b6 +last_audit_date: 2026-07-05 +overall: A # genuine fixes found and applied this pass (see gaps/notes); most of the + # ~18.5k LOC surface (3 prior parity sweeps: #398/#511, #963, #1404, #1627, + # #2060/#2197, #2333/#2339/#2342, #2381) was already accurate op-by-op. +ops: + CreateApi: {wire: ok, errors: ok, state: ok, persist: ok} + GetApi: {wire: ok, errors: ok, state: ok, persist: ok} + GetApis: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateApi: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteApi: {wire: ok, errors: ok, state: ok, persist: ok} + ImportApi: {wire: ok, errors: ok, state: ok, persist: ok} + ReimportApi: {wire: ok, errors: ok, state: ok, persist: ok} + ExportApi: {wire: ok, errors: ok, state: ok, persist: ok} + CreateRoute: {wire: ok, errors: ok, state: ok, persist: ok, note: "HTTP routeKey format + WS \$connect/\$disconnect/\$default/custom validated; auth type NONE/AWS_IAM/JWT/CUSTOM enforced"} + GetRoute: {wire: ok, errors: ok, state: ok, persist: ok} + GetRoutes: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateRoute: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteRoute: {wire: ok, errors: ok, state: ok, persist: ok} + CreateIntegration: {wire: fixed, errors: fixed, state: ok, persist: ok, note: "was missing tlsConfig, hardcoded 29000ms ceiling/default for HTTP APIs (should be 30000ms), no connectionType default/validation -- all fixed"} + GetIntegration: {wire: fixed, errors: ok, state: ok, persist: ok} + GetIntegrations: {wire: fixed, errors: ok, state: ok, persist: ok} + UpdateIntegration: {wire: fixed, errors: fixed, state: ok, persist: ok, note: "same protocol-aware timeout + connectionType validation applied"} + DeleteIntegration: {wire: ok, errors: ok, state: ok, persist: ok} + CreateIntegrationResponse: {wire: ok, errors: ok, state: ok, persist: ok} + GetIntegrationResponse: {wire: ok, errors: ok, state: ok, persist: ok} + GetIntegrationResponses: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateIntegrationResponse: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteIntegrationResponse: {wire: ok, errors: ok, state: ok, persist: ok} + CreateRouteResponse: {wire: ok, errors: ok, state: ok, persist: ok} + GetRouteResponse: {wire: ok, errors: ok, state: ok, persist: ok} + GetRouteResponses: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateRouteResponse: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteRouteResponse: {wire: ok, errors: ok, state: ok, persist: ok} + CreateStage: {wire: fixed, errors: ok, state: ok, persist: ok, note: "was missing clientCertificateId (WS-only) and Tags -- fixed"} + GetStage: {wire: fixed, errors: ok, state: ok, persist: ok} + GetStages: {wire: fixed, errors: ok, state: ok, persist: ok} + UpdateStage: {wire: fixed, errors: ok, state: ok, persist: ok} + DeleteStage: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteAccessLogSettings: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteRouteSettings: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteRouteRequestParameter: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteCorsConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + CreateDeployment: {wire: ok, errors: ok, state: ok, persist: ok, note: "autoDeploy interaction verified"} + GetDeployment: {wire: ok, errors: ok, state: ok, persist: ok} + GetDeployments: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateDeployment: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteDeployment: {wire: ok, errors: ok, state: ok, persist: ok} + CreateAuthorizer: {wire: ok, errors: ok, state: ok, persist: ok, note: "JWT issuer/audience + REQUEST identitySource/payloadFormatVersion/enableSimpleResponses/TTL all modeled and enforced on the data plane (http_proxy.go, authorizer.go)"} + GetAuthorizer: {wire: ok, errors: ok, state: ok, persist: ok} + GetAuthorizers: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateAuthorizer: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteAuthorizer: {wire: ok, errors: ok, state: ok, persist: ok} + ResetAuthorizersCache: {wire: ok, errors: ok, state: ok, persist: n/a, note: "cache is in-memory only by design"} + CreateModel: {wire: ok, errors: ok, state: ok, persist: ok} + GetModel: {wire: ok, errors: ok, state: ok, persist: ok} + GetModels: {wire: ok, errors: ok, state: ok, persist: ok} + GetModelTemplate: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateModel: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteModel: {wire: ok, errors: ok, state: ok, persist: ok} + CreateDomainName: {wire: fixed, errors: ok, state: ok, persist: ok, note: "was missing mutualTlsAuthentication and domainNameArn -- fixed"} + GetDomainName: {wire: fixed, errors: ok, state: ok, persist: ok} + GetDomainNames: {wire: fixed, errors: ok, state: ok, persist: ok} + UpdateDomainName: {wire: fixed, errors: ok, state: ok, persist: ok} + DeleteDomainName: {wire: ok, errors: ok, state: ok, persist: ok} + CreateApiMapping: {wire: ok, errors: ok, state: ok, persist: ok} + GetApiMapping: {wire: ok, errors: ok, state: ok, persist: ok} + GetApiMappings: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateApiMapping: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteApiMapping: {wire: ok, errors: ok, state: ok, persist: ok} + CreateVpcLink: {wire: ok, errors: ok, state: ok, persist: ok} + GetVpcLink: {wire: ok, errors: ok, state: ok, persist: ok} + GetVpcLinks: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateVpcLink: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteVpcLink: {wire: ok, errors: ok, state: ok, persist: ok} + CreateRoutingRule: {wire: partial, errors: ok, state: ok, persist: ok, note: "Actions/Conditions are untyped map[string]any passthrough, not AWS union shapes -- gopherstack-e81"} + GetRoutingRule: {wire: partial, errors: ok, state: ok, persist: ok} + ListRoutingRules: {wire: partial, errors: ok, state: ok, persist: ok} + PutRoutingRule: {wire: partial, errors: ok, state: ok, persist: ok} + DeleteRoutingRule: {wire: ok, errors: ok, state: ok, persist: ok} + TagResource: {wire: fixed, errors: fixed, state: ok, persist: ok, note: "now supports stage ARNs (arn:.../apis/{id}/stages/{name}) in addition to apis/vpclinks/domainnames; 404s were surfacing as 500 for stage ARNs before the errStageNotFound check was added to the handler"} + UntagResource: {wire: fixed, errors: fixed, state: ok, persist: ok} + GetTags: {wire: fixed, errors: fixed, state: ok, persist: ok} +families: + Portal/PortalProduct/ProductPage/ProductRestEndpointPage/RoutingRule (preview APIGW "portals" feature): {status: ok, note: "out of this pass's declared scope (task enumerated the classic Apis/Routes/Integrations/Stages/Deployments/Authorizers/ApiMappings/DomainNames/VpcLinks/ExportApi/models/tags surface); spot-checked, wire shapes look reasonable, not deep-audited"} + WebSocket @connections data plane (apigatewaymanagementapi): {status: ok, note: "delegated to services/apigatewaymanagementapi via SetManagementAPIBackend; out of scope for this apigatewayv2-only sweep"} +gaps: + - Integration ApiGatewayManaged / Stage ApiGatewayManaged not tracked for quick-create flows (bd: gopherstack-2tx) + - authorizerCache not purged on DeleteAPI; self-heals via TTL, low severity (bd: gopherstack-wmh) + - RoutingRule Actions/Conditions untyped passthrough instead of AWS union shapes (bd: gopherstack-e81) +deferred: + - Portal / PortalProduct / ProductPage / ProductRestEndpointPage families (newer preview feature, not in this pass's declared op list) + - RoutingRule typed action/condition validation (see gaps) +leaks: {status: clean, note: "portalProductSharingPolicies cleanup on DeletePortalProduct already covered by leak_internal_test.go from a prior sweep; authorizerCache is TTL-bounded (see gaps, low severity, not an unbounded leak); no goroutines/janitors in this package"} +--- + +## Notes + +Protocol = REST-JSON (`awsRestjson1` in the SDK's serializers/deserializers, confirmed by +reading `aws-sdk-go-v2/service/apigatewayv2@v1.33.7/serializers.go`). Timestamps use +`__timestampIso8601` (RFC3339 string), not epoch seconds — see `models.go` `isoTime` — this is +correct for apigatewayv2 and should NOT be "fixed" to `awstime.Epoch()`; that would be a +regression (epoch-seconds is a `json-1.0`/query-XML pattern from unrelated services, not REST-JSON +timestampIso8601). + +Genuine bugs found and fixed this pass (all confirmed against `aws-sdk-go-v2/service/apigatewayv2@v1.33.7/types/types.go`): + +1. **Integration.TlsConfig was entirely absent.** Real `Integration`/`CreateIntegrationInput`/ + `UpdateIntegrationInput` carry a `tlsConfig{serverNameToVerify}` for private (VPC_LINK) + integrations. Added `IntegrationTLSConfig` and wired it through Create/Update with a + deep-copy helper (`cloneIntegrationTLSConfig`) so backend state can't alias caller memory. + +2. **Integration timeout ceiling/default was NOT protocol-aware.** The backend hardcoded + 29,000ms (`integrationTimeoutMax`) as both the validation ceiling and the zero-value default + for every API, regardless of protocol. Real AWS: HTTP APIs allow up to 30,000ms (default + 30,000ms), WebSocket APIs allow up to 29,000ms (default 29,000ms) — see the SDK's doc comment + on `Integration.TimeoutInMillis`. Before this fix, HTTP API integrations with an unset timeout + were under-defaulted to 29000 instead of 30000, and a valid HTTP API timeout of e.g. 29500ms + was wrongly rejected as `BadRequestException`. Fixed via `integrationTimeoutMaxFor(protocolType)` + threaded through `CreateIntegration`/`UpdateIntegration`/`validateTimeoutInMillis`. + +3. **Integration.ConnectionType had no default and no validation.** Real AWS defaults + `connectionType` to `INTERNET` when unset and validates the enum (`INTERNET`|`VPC_LINK`), + requiring `connectionId` when `VPC_LINK` is specified. Before this fix, `GetIntegration` + on an integration created without an explicit connectionType returned `""` instead of + `"INTERNET"`, and a `VPC_LINK` integration with no `connectionId` silently succeeded. + +4. **Stage.ClientCertificateID (WebSocket-only) and Stage.Tags were entirely absent.** Real + `Stage` carries `clientCertificateId` and its own `tags` map (a Stage is independently + taggable via `arn:aws:apigateway:{region}::/apis/{apiId}/stages/{stageName}`, confirmed by + the `Tags` field on the SDK's `Stage` type). Added both fields, wired `clientCertificateId` + through Create/UpdateStage, and extended `TagResource`/`UntagResource`/`GetTags` to resolve + the nested stage ARN shape (`parseStageARN` + `lookupStageLocked`) since stages — unlike + APIs/VPC links/domain names — need two path segments (apiId + stageName), not one, to + resolve. The tag handlers in `handler.go` were also missing an `ErrStageNotFound` → 404 + mapping, so a not-found stage tag lookup would have surfaced as a 500 instead of 404 (the + exact bug class called out in `parity-principles.md` #2). + +5. **DomainName.MutualTlsAuthentication and DomainName.DomainNameArn were entirely absent.** + Real `DomainName` carries `mutualTlsAuthentication{truststoreUri,truststoreVersion, + truststoreWarnings}` and `domainNameArn`. Added both, wired through Create/UpdateDomainName + (`domainNameArn` is computed once at creation and stable across updates, matching AWS + behavior since it is not settable). `truststoreWarnings` is always empty because the + emulator has no S3 truststore object to validate — this matches the "no warnings" case for + a well-formed request; it does not represent unvalidated input. + +Traps for the next auditor (don't re-flag): + +- `arnResourceType` (single `type/id` suffix) intentionally does NOT handle Stage ARNs — Stage + tagging goes through the separate `parseStageARN` (4-segment `apis/{id}/stages/{name}`) checked + *before* falling through to `arnResourceType` in `TagResource`/`UntagResource`/`GetTags`. This + is correct, not a missed generalization — Stages are the only nested (parent + child) taggable + resource in this API. +- The hand-formatted `"arn:aws:apigateway:" + region + "::/..."` ARN construction (not + `pkgs/arn`) is a pre-existing convention in this file (see `RoutingRuleARN`, now also + `DomainNameArn`); left as-is for consistency rather than partially migrating one call site. +- `Portal`/`PortalProduct`/routing-rule-adjacent code is a newer, separate APIGWv2 "portals" + preview feature; it was spot-checked but is intentionally out of this pass's declared scope + (the task's op list is the classic HTTP/WebSocket control plane). diff --git a/services/apigatewayv2/backend.go b/services/apigatewayv2/backend.go index 27bee8fe0..a76058e7e 100644 --- a/services/apigatewayv2/backend.go +++ b/services/apigatewayv2/backend.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "maps" + "slices" "sort" "strings" "time" @@ -14,6 +15,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/awserr" "github.com/blackbirdworks/gopherstack/pkgs/awsmeta" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) // defaultRegion is used for ARNs and execute-api endpoints when the request @@ -78,7 +80,18 @@ const ( integrationTypeHTTP = "HTTP" integrationTimeoutMin = int32(50) - integrationTimeoutMax = int32(29000) + // integrationTimeoutMaxWebSocket is the maximum (and default) integration + // timeout for WebSocket APIs: 29 seconds. + integrationTimeoutMaxWebSocket = int32(29000) + // integrationTimeoutMaxHTTP is the maximum (and default) integration timeout + // for HTTP APIs: 30 seconds. AWS allows a longer ceiling for HTTP APIs than + // for WebSocket APIs; see CreateIntegration/UpdateIntegration docs. + integrationTimeoutMaxHTTP = int32(30000) + + // connectionTypeInternet is the default Integration ConnectionType when the + // caller does not specify one. + connectionTypeInternet = "INTERNET" + connectionTypeVpcLink = "VPC_LINK" ) // validRouteAuthorizationType reports whether t is a valid route AuthorizationType @@ -121,18 +134,77 @@ func validateHTTPRouteKey(key string) error { return nil } -// validateTimeoutInMillis returns ErrBadRequest if ms is outside [50, 29000]. -func validateTimeoutInMillis(ms int32) error { - if ms < integrationTimeoutMin || ms > integrationTimeoutMax { +// integrationTimeoutMaxFor returns the maximum (and default) integration +// timeout in milliseconds for the given API protocol type: 30,000ms for HTTP +// APIs and 29,000ms for WebSocket APIs, matching real API Gateway v2 limits. +func integrationTimeoutMaxFor(protocolType string) int32 { + if protocolType == protocolTypeHTTP { + return integrationTimeoutMaxHTTP + } + + return integrationTimeoutMaxWebSocket +} + +// validateTimeoutInMillis returns ErrBadRequest if ms is outside +// [50, integrationTimeoutMaxFor(protocolType)]. +func validateTimeoutInMillis(ms int32, protocolType string) error { + maxMs := integrationTimeoutMaxFor(protocolType) + if ms < integrationTimeoutMin || ms > maxMs { return fmt.Errorf( "%w: timeoutInMillis must be between %d and %d", - ErrBadRequest, integrationTimeoutMin, integrationTimeoutMax, + ErrBadRequest, integrationTimeoutMin, maxMs, ) } return nil } +// validateConnectionType returns ErrBadRequest if connectionType is not one of +// the modeled enum values (INTERNET, VPC_LINK), or if VPC_LINK is specified +// without a connectionId (the VPC link to route the private integration +// through), matching real API Gateway v2 validation. +func validateConnectionType(connectionType, connectionID string) error { + switch connectionType { + case connectionTypeInternet: + return nil + case connectionTypeVpcLink: + if connectionID == "" { + return fmt.Errorf("%w: connectionId is required when connectionType is VPC_LINK", ErrBadRequest) + } + + return nil + default: + return fmt.Errorf("%w: connectionType must be one of INTERNET, VPC_LINK", ErrBadRequest) + } +} + +// cloneIntegrationTLSConfig returns a deep copy of cfg, or nil if cfg is nil. +func cloneIntegrationTLSConfig(cfg *IntegrationTLSConfig) *IntegrationTLSConfig { + if cfg == nil { + return nil + } + + cp := *cfg + + return &cp +} + +// cloneMutualTLSAuthentication returns a deep copy of cfg, or nil if cfg is +// nil. TruststoreWarnings is never populated by the caller (it is an +// output-only field computed by API Gateway while validating the truststore); +// the emulator has no truststore to validate against S3, so it is left empty, +// matching the "no warnings" case real AWS returns for a well-formed request. +func cloneMutualTLSAuthentication(cfg *MutualTLSAuthentication) *MutualTLSAuthentication { + if cfg == nil { + return nil + } + + cp := *cfg + cp.TruststoreWarnings = nil + + return &cp +} + var ( // ErrAPINotFound is returned when a requested API does not exist. ErrAPINotFound = errors.New("NotFoundException") @@ -396,49 +468,59 @@ type StorageBackend interface { DeleteRouteRequestParameter(apiID, routeID, requestParameterKey string) error } -// apiData holds per-API state. -type apiData struct { - stages map[string]*Stage - routes map[string]*Route - integrations map[string]*Integration - deployments map[string]*Deployment - authorizers map[string]*Authorizer - integrationResponses map[string]map[string]*IntegrationResponse - models map[string]*Model - routeResponses map[string]map[string]*RouteResponse - api API -} - -// InMemoryBackend implements StorageBackend using in-memory maps. +// InMemoryBackend implements StorageBackend using pkgs/store tables. Every +// resource family nested under an API/domain name/portal product (formerly a +// per-parent nested map -- see apiData in the pre-Phase-3.3 history) is now a +// single flat table keyed by a composite "#" string, with +// a secondary [store.Index] grouping by parent ID. See store_setup.go's doc +// comment for the full clean/dirty table split. type InMemoryBackend struct { - apis map[string]*apiData - domainNames map[string]*DomainName - apiMappings map[string]map[string]*APIMapping - portals map[string]*Portal - portalProducts map[string]*PortalProduct - portalProductSharingPolicies map[string]string - productPages map[string][]*ProductPage // key: portalProductID - productREPages map[string][]*ProductRestEndpointPage // key: portalProductID - vpcLinks map[string]*VpcLink - routingRules map[string]map[string]*RoutingRule - mu *lockmetrics.RWMutex + apis *store.Table[API] + stages *store.Table[Stage] + stagesByAPI *store.Index[Stage] + routes *store.Table[Route] + routesByAPI *store.Index[Route] + integrations *store.Table[Integration] + integrationsByAPI *store.Index[Integration] + deployments *store.Table[Deployment] + deploymentsByAPI *store.Index[Deployment] + authorizers *store.Table[Authorizer] + authorizersByAPI *store.Index[Authorizer] + models *store.Table[Model] + modelsByAPI *store.Index[Model] + integrationResponses *store.Table[IntegrationResponse] + integrationResponsesByIntegration *store.Index[IntegrationResponse] + routeResponses *store.Table[RouteResponse] + routeResponsesByRoute *store.Index[RouteResponse] + domainNames *store.Table[DomainName] + apiMappings *store.Table[APIMapping] + apiMappingsByDomain *store.Index[APIMapping] + portals *store.Table[Portal] + portalProducts *store.Table[PortalProduct] + // portalProductSharingPolicies (portalProductID -> policy document) is a + // plain map, not a store.Table -- see store_setup.go's doc comment for why. + portalProductSharingPolicies map[string]string + productPages *store.Table[ProductPage] + productPagesByPortalProduct *store.Index[ProductPage] + productREPages *store.Table[ProductRestEndpointPage] + productREPagesByPortalProduct *store.Index[ProductRestEndpointPage] + vpcLinks *store.Table[VpcLink] + routingRules *store.Table[RoutingRule] + routingRulesByDomain *store.Index[RoutingRule] + registry *store.Registry + mu *lockmetrics.RWMutex } // NewInMemoryBackend creates a new InMemoryBackend. func NewInMemoryBackend() *InMemoryBackend { - return &InMemoryBackend{ - apis: make(map[string]*apiData), - domainNames: make(map[string]*DomainName), - apiMappings: make(map[string]map[string]*APIMapping), - portals: make(map[string]*Portal), - portalProducts: make(map[string]*PortalProduct), + b := &InMemoryBackend{ portalProductSharingPolicies: make(map[string]string), - productPages: make(map[string][]*ProductPage), - productREPages: make(map[string][]*ProductRestEndpointPage), - vpcLinks: make(map[string]*VpcLink), - routingRules: make(map[string]map[string]*RoutingRule), + registry: store.NewRegistry(), mu: lockmetrics.New("apigatewayv2"), } + registerAllTables(b) + + return b } // copyTags returns a deep copy of a tags map, guarding against nil. @@ -450,21 +532,28 @@ func copyTags(src map[string]string) map[string]string { return maps.Clone(src) } -// Reset clears all backend state, reinitialising all maps. +// Reset clears all backend state, reinitialising all tables. func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.apis = make(map[string]*apiData) - b.domainNames = make(map[string]*DomainName) - b.apiMappings = make(map[string]map[string]*APIMapping) - b.portals = make(map[string]*Portal) - b.portalProducts = make(map[string]*PortalProduct) + b.registry.ResetAll() + // The "dirty" tables (see store_setup.go's registerAllTables doc) are + // deliberately NOT on b.registry, so each needs its own Reset() call here. + b.stages.Reset() + b.routes.Reset() + b.integrations.Reset() + b.deployments.Reset() + b.authorizers.Reset() + b.models.Reset() + b.integrationResponses.Reset() + b.routeResponses.Reset() + b.apiMappings.Reset() + b.productPages.Reset() + b.productREPages.Reset() + b.routingRules.Reset() + b.portalProductSharingPolicies = make(map[string]string) - b.productPages = make(map[string][]*ProductPage) - b.productREPages = make(map[string][]*ProductRestEndpointPage) - b.vpcLinks = make(map[string]*VpcLink) - b.routingRules = make(map[string]map[string]*RoutingRule) } // randomID generates a cryptographically random 10-character alphanumeric ID. @@ -539,19 +628,11 @@ func (b *InMemoryBackend) CreateAPI(ctx context.Context, input CreateAPIInput) ( api.CorsConfiguration = &clone } - b.apis[id] = &apiData{ - api: api, - stages: make(map[string]*Stage), - routes: make(map[string]*Route), - integrations: make(map[string]*Integration), - deployments: make(map[string]*Deployment), - authorizers: make(map[string]*Authorizer), - integrationResponses: make(map[string]map[string]*IntegrationResponse), - models: make(map[string]*Model), - routeResponses: make(map[string]map[string]*RouteResponse), - } + b.apis.Put(&api) + + cp := api - return &api, nil + return &cp, nil } // GetAPI retrieves an API by ID. @@ -559,12 +640,12 @@ func (b *InMemoryBackend) GetAPI(apiID string) (*API, error) { b.mu.RLock("GetAPI") defer b.mu.RUnlock() - d, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return nil, ErrAPINotFound } - cp := d.api + cp := *api return &cp, nil } @@ -574,9 +655,11 @@ func (b *InMemoryBackend) GetAPIs() ([]API, error) { b.mu.RLock("GetAPIs") defer b.mu.RUnlock() - result := make([]API, 0, len(b.apis)) - for _, d := range b.apis { - result = append(result, d.api) + all := b.apis.All() + result := make([]API, 0, len(all)) + + for _, api := range all { + result = append(result, *api) } sort.Slice(result, func(i, j int) bool { @@ -586,23 +669,61 @@ func (b *InMemoryBackend) GetAPIs() ([]API, error) { return result, nil } +// deleteAPIChildrenLocked removes every child resource (stages, routes, +// integrations, deployments, authorizers, models, and their nested +// responses) belonging to apiID. It mirrors the implicit cascade that +// deleting the pre-Phase-3.3 nested apiData map provided. Callers must +// already hold b.mu.Lock. +func (b *InMemoryBackend) deleteAPIChildrenLocked(apiID string) { + for _, s := range slices.Clone(b.stagesByAPI.Get(apiID)) { + b.stages.Delete(stageKey(apiID, s.StageName)) + } + + for _, r := range slices.Clone(b.routesByAPI.Get(apiID)) { + for _, rr := range slices.Clone(b.routeResponsesByRoute.Get(routeKey(apiID, r.RouteID))) { + b.routeResponses.Delete(routeResponseKey(apiID, r.RouteID, rr.RouteResponseID)) + } + + b.routes.Delete(routeKey(apiID, r.RouteID)) + } + + for _, i := range slices.Clone(b.integrationsByAPI.Get(apiID)) { + for _, ir := range slices.Clone(b.integrationResponsesByIntegration.Get(integrationKey(apiID, i.IntegrationID))) { + b.integrationResponses.Delete(integrationResponseKey(apiID, i.IntegrationID, ir.IntegrationResponseID)) + } + + b.integrations.Delete(integrationKey(apiID, i.IntegrationID)) + } + + for _, dep := range slices.Clone(b.deploymentsByAPI.Get(apiID)) { + b.deployments.Delete(deploymentKey(apiID, dep.DeploymentID)) + } + + for _, a := range slices.Clone(b.authorizersByAPI.Get(apiID)) { + b.authorizers.Delete(authorizerKey(apiID, a.AuthorizerID)) + } + + for _, m := range slices.Clone(b.modelsByAPI.Get(apiID)) { + b.models.Delete(modelKey(apiID, m.ModelID)) + } +} + // DeleteAPI removes an API by ID. func (b *InMemoryBackend) DeleteAPI(apiID string) error { b.mu.Lock("DeleteAPI") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return ErrAPINotFound } - delete(b.apis, apiID) + b.deleteAPIChildrenLocked(apiID) + b.apis.Delete(apiID) // Clean up stale API mappings pointing to this API. - for _, mappings := range b.apiMappings { - for id, m := range mappings { - if m.APIID == apiID { - delete(mappings, id) - } + for _, m := range b.apiMappings.All() { + if m.APIID == apiID { + b.apiMappings.Delete(apiMappingKey(m.DomainName, m.APIMappingID)) } } @@ -614,49 +735,49 @@ func (b *InMemoryBackend) UpdateAPI(apiID string, input UpdateAPIInput) (*API, e b.mu.Lock("UpdateAPI") defer b.mu.Unlock() - d, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return nil, ErrAPINotFound } if input.Name != "" { - d.api.Name = input.Name + api.Name = input.Name } if input.Description != "" { - d.api.Description = input.Description + api.Description = input.Description } if input.RouteSelectionExpression != "" { - d.api.RouteSelectionExpression = input.RouteSelectionExpression + api.RouteSelectionExpression = input.RouteSelectionExpression } if input.Version != "" { - d.api.Version = input.Version + api.Version = input.Version } if input.Tags != nil { - d.api.Tags = copyTags(input.Tags) + api.Tags = copyTags(input.Tags) } if input.APIKeySelectionExpression != "" { - d.api.APIKeySelectionExpression = input.APIKeySelectionExpression + api.APIKeySelectionExpression = input.APIKeySelectionExpression } if input.CorsConfiguration != nil { clone := *input.CorsConfiguration - d.api.CorsConfiguration = &clone + api.CorsConfiguration = &clone } if input.DisableSchemaValidation != nil { - d.api.DisableSchemaValidation = *input.DisableSchemaValidation + api.DisableSchemaValidation = *input.DisableSchemaValidation } if input.DisableExecuteAPIEndpoint != nil { - d.api.DisableExecuteAPIEndpoint = *input.DisableExecuteAPIEndpoint + api.DisableExecuteAPIEndpoint = *input.DisableExecuteAPIEndpoint } - cp := d.api + cp := *api return &cp, nil } @@ -668,8 +789,7 @@ func (b *InMemoryBackend) CreateStage(apiID string, input CreateStageInput) (*St b.mu.Lock("CreateStage") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } @@ -677,7 +797,7 @@ func (b *InMemoryBackend) CreateStage(apiID string, input CreateStageInput) (*St return nil, fmt.Errorf("%w: stageName is required", ErrBadRequest) } - if _, exists := d.stages[input.StageName]; exists { + if b.stages.Has(stageKey(apiID, input.StageName)) { return nil, fmt.Errorf("%w: stage %q already exists", ErrAlreadyExists, input.StageName) } @@ -687,6 +807,7 @@ func (b *InMemoryBackend) CreateStage(apiID string, input CreateStageInput) (*St APIID: apiID, DeploymentID: input.DeploymentID, Description: input.Description, + ClientCertificateID: input.ClientCertificateID, AutoDeploy: input.AutoDeploy, StageVariables: input.StageVariables, CreatedDate: now, @@ -696,7 +817,7 @@ func (b *InMemoryBackend) CreateStage(apiID string, input CreateStageInput) (*St RouteSettings: input.RouteSettings, } - d.stages[input.StageName] = stage + b.stages.Put(stage) cp := *stage @@ -708,12 +829,11 @@ func (b *InMemoryBackend) GetStage(apiID, stageName string) (*Stage, error) { b.mu.RLock("GetStage") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - s, ok := d.stages[stageName] + s, ok := b.stages.Get(stageKey(apiID, stageName)) if !ok { return nil, ErrStageNotFound } @@ -728,13 +848,14 @@ func (b *InMemoryBackend) GetStages(apiID string) ([]Stage, error) { b.mu.RLock("GetStages") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - result := make([]Stage, 0, len(d.stages)) - for _, s := range d.stages { + stages := b.stagesByAPI.Get(apiID) + result := make([]Stage, 0, len(stages)) + + for _, s := range stages { result = append(result, *s) } @@ -750,17 +871,14 @@ func (b *InMemoryBackend) DeleteStage(apiID, stageName string) error { b.mu.Lock("DeleteStage") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return ErrAPINotFound } - if _, exists := d.stages[stageName]; !exists { + if !b.stages.Delete(stageKey(apiID, stageName)) { return ErrStageNotFound } - delete(d.stages, stageName) - return nil } @@ -769,12 +887,11 @@ func (b *InMemoryBackend) UpdateStage(apiID, stageName string, input UpdateStage b.mu.Lock("UpdateStage") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - s, ok := d.stages[stageName] + s, ok := b.stages.Get(stageKey(apiID, stageName)) if !ok { return nil, ErrStageNotFound } @@ -787,6 +904,10 @@ func (b *InMemoryBackend) UpdateStage(apiID, stageName string, input UpdateStage s.Description = input.Description } + if input.ClientCertificateID != "" { + s.ClientCertificateID = input.ClientCertificateID + } + if input.AutoDeploy != nil { s.AutoDeploy = *input.AutoDeploy } @@ -823,7 +944,7 @@ func (b *InMemoryBackend) CreateRoute(apiID string, input CreateRouteInput) (*Ro b.mu.Lock("CreateRoute") defer b.mu.Unlock() - d, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return nil, ErrAPINotFound } @@ -832,13 +953,13 @@ func (b *InMemoryBackend) CreateRoute(apiID string, input CreateRouteInput) (*Ro return nil, fmt.Errorf("%w: routeKey is required", ErrBadRequest) } - if d.api.ProtocolType == protocolTypeHTTP { + if api.ProtocolType == protocolTypeHTTP { if err := validateHTTPRouteKey(input.RouteKey); err != nil { return nil, err } } - for _, existing := range d.routes { + for _, existing := range b.routesByAPI.Get(apiID) { if existing.RouteKey == input.RouteKey { return nil, fmt.Errorf("%w: route key %q already exists", ErrAlreadyExists, input.RouteKey) } @@ -880,8 +1001,8 @@ func (b *InMemoryBackend) CreateRoute(apiID string, input CreateRouteInput) (*Ro APIKeyRequired: input.APIKeyRequired, } - d.routes[id] = route - b.autoDeployLocked(d) + b.routes.Put(route) + b.autoDeployLocked(apiID) cp := *route @@ -893,12 +1014,11 @@ func (b *InMemoryBackend) GetRoute(apiID, routeID string) (*Route, error) { b.mu.RLock("GetRoute") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - r, ok := d.routes[routeID] + r, ok := b.routes.Get(routeKey(apiID, routeID)) if !ok { return nil, ErrRouteNotFound } @@ -913,13 +1033,14 @@ func (b *InMemoryBackend) GetRoutes(apiID string) ([]Route, error) { b.mu.RLock("GetRoutes") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - result := make([]Route, 0, len(d.routes)) - for _, r := range d.routes { + routes := b.routesByAPI.Get(apiID) + result := make([]Route, 0, len(routes)) + + for _, r := range routes { result = append(result, *r) } @@ -935,33 +1056,34 @@ func (b *InMemoryBackend) DeleteRoute(apiID, routeID string) error { b.mu.Lock("DeleteRoute") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return ErrAPINotFound } - if _, exists := d.routes[routeID]; !exists { + if !b.routes.Delete(routeKey(apiID, routeID)) { return ErrRouteNotFound } - delete(d.routes, routeID) - delete(d.routeResponses, routeID) - b.autoDeployLocked(d) + for _, rr := range slices.Clone(b.routeResponsesByRoute.Get(routeKey(apiID, routeID))) { + b.routeResponses.Delete(routeResponseKey(apiID, routeID, rr.RouteResponseID)) + } + + b.autoDeployLocked(apiID) return nil } // setRouteKey validates newKey for protocolType and ensures it is not a duplicate // among routes (excluding the route being updated), then sets r.RouteKey. -func setRouteKey(r *Route, routes map[string]*Route, routeID, newKey, protocolType string) error { +func setRouteKey(r *Route, routes []*Route, routeID, newKey, protocolType string) error { if protocolType == protocolTypeHTTP { if err := validateHTTPRouteKey(newKey); err != nil { return err } } - for id, existing := range routes { - if id != routeID && existing.RouteKey == newKey { + for _, existing := range routes { + if existing.RouteID != routeID && existing.RouteKey == newKey { return fmt.Errorf("%w: route key %q already exists", ErrAlreadyExists, newKey) } } @@ -999,18 +1121,18 @@ func (b *InMemoryBackend) UpdateRoute(apiID, routeID string, input UpdateRouteIn b.mu.Lock("UpdateRoute") defer b.mu.Unlock() - d, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return nil, ErrAPINotFound } - r, ok := d.routes[routeID] + r, ok := b.routes.Get(routeKey(apiID, routeID)) if !ok { return nil, ErrRouteNotFound } if input.RouteKey != "" { - if err := setRouteKey(r, d.routes, routeID, input.RouteKey, d.api.ProtocolType); err != nil { + if err := setRouteKey(r, b.routesByAPI.Get(apiID), routeID, input.RouteKey, api.ProtocolType); err != nil { return nil, err } } @@ -1047,7 +1169,7 @@ func (b *InMemoryBackend) UpdateRoute(apiID, routeID string, input UpdateRouteIn r.APIKeyRequired = *input.APIKeyRequired } - b.autoDeployLocked(d) + b.autoDeployLocked(apiID) cp := *r @@ -1061,7 +1183,7 @@ func (b *InMemoryBackend) CreateIntegration(apiID string, input CreateIntegratio b.mu.Lock("CreateIntegration") defer b.mu.Unlock() - d, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return nil, ErrAPINotFound } @@ -1091,10 +1213,19 @@ func (b *InMemoryBackend) CreateIntegration(apiID string, input CreateIntegratio passthroughBehavior = "WHEN_NO_MATCH" } + connectionType := input.ConnectionType + if connectionType == "" { + connectionType = connectionTypeInternet + } + + if err := validateConnectionType(connectionType, input.ConnectionID); err != nil { + return nil, err + } + timeoutMs := input.TimeoutInMillis if timeoutMs == 0 { - timeoutMs = integrationTimeoutMax - } else if err := validateTimeoutInMillis(timeoutMs); err != nil { + timeoutMs = integrationTimeoutMaxFor(api.ProtocolType) + } else if err := validateTimeoutInMillis(timeoutMs, api.ProtocolType); err != nil { return nil, err } @@ -1108,17 +1239,18 @@ func (b *InMemoryBackend) CreateIntegration(apiID string, input CreateIntegratio IntegrationURI: input.IntegrationURI, Description: input.Description, PayloadFormatVersion: payloadFmtVer, - ConnectionType: input.ConnectionType, + ConnectionType: connectionType, ConnectionID: input.ConnectionID, TimeoutInMillis: timeoutMs, RequestParameters: input.RequestParameters, RequestTemplates: input.RequestTemplates, TemplateSelectionExpression: input.TemplateSelectionExpression, PassthroughBehavior: passthroughBehavior, + TLSConfig: cloneIntegrationTLSConfig(input.TLSConfig), } - d.integrations[id] = integration - b.autoDeployLocked(d) + b.integrations.Put(integration) + b.autoDeployLocked(apiID) cp := *integration @@ -1130,12 +1262,11 @@ func (b *InMemoryBackend) GetIntegration(apiID, integrationID string) (*Integrat b.mu.RLock("GetIntegration") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - i, ok := d.integrations[integrationID] + i, ok := b.integrations.Get(integrationKey(apiID, integrationID)) if !ok { return nil, ErrIntegrationNotFound } @@ -1150,13 +1281,14 @@ func (b *InMemoryBackend) GetIntegrations(apiID string) ([]Integration, error) { b.mu.RLock("GetIntegrations") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - result := make([]Integration, 0, len(d.integrations)) - for _, i := range d.integrations { + integrations := b.integrationsByAPI.Get(apiID) + result := make([]Integration, 0, len(integrations)) + + for _, i := range integrations { result = append(result, *i) } @@ -1172,18 +1304,19 @@ func (b *InMemoryBackend) DeleteIntegration(apiID, integrationID string) error { b.mu.Lock("DeleteIntegration") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return ErrAPINotFound } - if _, exists := d.integrations[integrationID]; !exists { + if !b.integrations.Delete(integrationKey(apiID, integrationID)) { return ErrIntegrationNotFound } - delete(d.integrations, integrationID) - delete(d.integrationResponses, integrationID) - b.autoDeployLocked(d) + for _, ir := range slices.Clone(b.integrationResponsesByIntegration.Get(integrationKey(apiID, integrationID))) { + b.integrationResponses.Delete(integrationResponseKey(apiID, integrationID, ir.IntegrationResponseID)) + } + + b.autoDeployLocked(apiID) return nil } @@ -1242,6 +1375,10 @@ func applyIntegrationUpdate(i *Integration, input UpdateIntegrationInput) { if input.PassthroughBehavior != "" { i.PassthroughBehavior = input.PassthroughBehavior } + + if input.TLSConfig != nil { + i.TLSConfig = cloneIntegrationTLSConfig(input.TLSConfig) + } } // UpdateIntegration updates fields on an existing integration. @@ -1252,24 +1389,40 @@ func (b *InMemoryBackend) UpdateIntegration( b.mu.Lock("UpdateIntegration") defer b.mu.Unlock() - d, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return nil, ErrAPINotFound } - i, ok := d.integrations[integrationID] + i, ok := b.integrations.Get(integrationKey(apiID, integrationID)) if !ok { return nil, ErrIntegrationNotFound } if input.TimeoutInMillis != 0 { - if err := validateTimeoutInMillis(input.TimeoutInMillis); err != nil { + if err := validateTimeoutInMillis(input.TimeoutInMillis, api.ProtocolType); err != nil { + return nil, err + } + } + + effectiveConnectionType := i.ConnectionType + if input.ConnectionType != "" { + effectiveConnectionType = input.ConnectionType + } + + effectiveConnectionID := i.ConnectionID + if input.ConnectionID != "" { + effectiveConnectionID = input.ConnectionID + } + + if input.ConnectionType != "" || input.ConnectionID != "" { + if err := validateConnectionType(effectiveConnectionType, effectiveConnectionID); err != nil { return nil, err } } applyIntegrationUpdate(i, input) - b.autoDeployLocked(d) + b.autoDeployLocked(apiID) cp := *i @@ -1283,8 +1436,7 @@ func (b *InMemoryBackend) CreateDeployment(apiID string, input CreateDeploymentI b.mu.Lock("CreateDeployment") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } @@ -1297,11 +1449,11 @@ func (b *InMemoryBackend) CreateDeployment(apiID string, input CreateDeploymentI CreatedDate: isoTime{time.Now()}, } - d.deployments[id] = deployment + b.deployments.Put(deployment) // When a stage name is provided, link the deployment to that stage (AWS behaviour). if input.StageName != "" { - s, stageExists := d.stages[input.StageName] + s, stageExists := b.stages.Get(stageKey(apiID, input.StageName)) if !stageExists { return nil, ErrStageNotFound } @@ -1319,23 +1471,23 @@ func (b *InMemoryBackend) CreateDeployment(apiID string, input CreateDeploymentI // and repoints the stage at it whenever a route, integration, or other routing // configuration changes on an auto-deploy-enabled stage. The caller must hold // b.mu.Lock. -func (b *InMemoryBackend) autoDeployLocked(d *apiData) { +func (b *InMemoryBackend) autoDeployLocked(apiID string) { now := isoTime{time.Now()} - for _, s := range d.stages { + for _, s := range b.stagesByAPI.Get(apiID) { if !s.AutoDeploy { continue } id := randomID() - d.deployments[id] = &Deployment{ + b.deployments.Put(&Deployment{ DeploymentID: id, - APIID: d.api.APIID, + APIID: apiID, Description: "Automatic deployment triggered by changes to the Api configuration", DeploymentStatus: "DEPLOYED", AutoDeployed: true, CreatedDate: now, - } + }) s.DeploymentID = id s.LastUpdatedDate = now } @@ -1346,12 +1498,11 @@ func (b *InMemoryBackend) GetDeployment(apiID, deploymentID string) (*Deployment b.mu.RLock("GetDeployment") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - dep, ok := d.deployments[deploymentID] + dep, ok := b.deployments.Get(deploymentKey(apiID, deploymentID)) if !ok { return nil, ErrDeploymentNotFound } @@ -1366,13 +1517,14 @@ func (b *InMemoryBackend) GetDeployments(apiID string) ([]Deployment, error) { b.mu.RLock("GetDeployments") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - result := make([]Deployment, 0, len(d.deployments)) - for _, dep := range d.deployments { + deployments := b.deploymentsByAPI.Get(apiID) + result := make([]Deployment, 0, len(deployments)) + + for _, dep := range deployments { result = append(result, *dep) } @@ -1388,17 +1540,14 @@ func (b *InMemoryBackend) DeleteDeployment(apiID, deploymentID string) error { b.mu.Lock("DeleteDeployment") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return ErrAPINotFound } - if _, exists := d.deployments[deploymentID]; !exists { + if !b.deployments.Delete(deploymentKey(apiID, deploymentID)) { return ErrDeploymentNotFound } - delete(d.deployments, deploymentID) - return nil } @@ -1409,8 +1558,7 @@ func (b *InMemoryBackend) CreateAuthorizer(apiID string, input CreateAuthorizerI b.mu.Lock("CreateAuthorizer") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } @@ -1453,7 +1601,7 @@ func (b *InMemoryBackend) CreateAuthorizer(apiID string, input CreateAuthorizerI authorizer.JwtConfiguration = &clone } - d.authorizers[id] = authorizer + b.authorizers.Put(authorizer) cp := *authorizer @@ -1465,12 +1613,11 @@ func (b *InMemoryBackend) GetAuthorizer(apiID, authorizerID string) (*Authorizer b.mu.RLock("GetAuthorizer") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - a, ok := d.authorizers[authorizerID] + a, ok := b.authorizers.Get(authorizerKey(apiID, authorizerID)) if !ok { return nil, ErrAuthorizerNotFound } @@ -1485,13 +1632,14 @@ func (b *InMemoryBackend) GetAuthorizers(apiID string) ([]Authorizer, error) { b.mu.RLock("GetAuthorizers") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - result := make([]Authorizer, 0, len(d.authorizers)) - for _, a := range d.authorizers { + authorizers := b.authorizersByAPI.Get(apiID) + result := make([]Authorizer, 0, len(authorizers)) + + for _, a := range authorizers { result = append(result, *a) } @@ -1507,17 +1655,14 @@ func (b *InMemoryBackend) DeleteAuthorizer(apiID, authorizerID string) error { b.mu.Lock("DeleteAuthorizer") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return ErrAPINotFound } - if _, exists := d.authorizers[authorizerID]; !exists { + if !b.authorizers.Delete(authorizerKey(apiID, authorizerID)) { return ErrAuthorizerNotFound } - delete(d.authorizers, authorizerID) - return nil } @@ -1529,12 +1674,11 @@ func (b *InMemoryBackend) UpdateAuthorizer( b.mu.Lock("UpdateAuthorizer") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - a, ok := d.authorizers[authorizerID] + a, ok := b.authorizers.Get(authorizerKey(apiID, authorizerID)) if !ok { return nil, ErrAuthorizerNotFound } @@ -1595,7 +1739,7 @@ func (b *InMemoryBackend) CreateDomainName( b.mu.Lock("CreateDomainName") defer b.mu.Unlock() - if _, exists := b.domainNames[input.DomainNameValue]; exists { + if b.domainNames.Has(input.DomainNameValue) { return nil, fmt.Errorf("%w: domain name %q already exists", ErrAlreadyExists, input.DomainNameValue) } @@ -1605,14 +1749,17 @@ func (b *InMemoryBackend) CreateDomainName( input.DomainNameConfigurations, input.DomainNameValue, regionFromCtx(ctx)) } + domainNameArn := "arn:aws:apigateway:" + regionFromCtx(ctx) + "::/domainnames/" + input.DomainNameValue + dn := &DomainName{ DomainNameValue: input.DomainNameValue, + DomainNameArn: domainNameArn, Tags: copyTags(input.Tags), DomainNameConfigurations: domainNameConfigs, + MutualTLSAuthentication: cloneMutualTLSAuthentication(input.MutualTLSAuthentication), } - b.domainNames[input.DomainNameValue] = dn - b.apiMappings[input.DomainNameValue] = make(map[string]*APIMapping) + b.domainNames.Put(dn) cp := *dn @@ -1634,23 +1781,22 @@ func (b *InMemoryBackend) CreateAPIMapping(domainName string, input CreateAPIMap b.mu.Lock("CreateAPIMapping") defer b.mu.Unlock() - if _, ok := b.domainNames[domainName]; !ok { + if !b.domainNames.Has(domainName) { return nil, ErrDomainNameNotFound } - d, ok := b.apis[input.APIID] - if !ok { + if !b.apis.Has(input.APIID) { return nil, ErrAPINotFound } - if _, stageExists := d.stages[input.Stage]; !stageExists { + if !b.stages.Has(stageKey(input.APIID, input.Stage)) { return nil, ErrStageNotFound } // AWS allows only one mapping per (domain, apiMappingKey). The empty key is // the domain's default (base-path) mapping and is itself unique. Reject a // duplicate key with a ConflictException, matching real API Gateway v2. - for _, existing := range b.apiMappings[domainName] { + for _, existing := range b.apiMappingsByDomain.Get(domainName) { if existing.APIMappingKey == input.APIMappingKey { return nil, fmt.Errorf( "%w: an api mapping already exists for the mapping key %q on domain %q", @@ -1668,7 +1814,7 @@ func (b *InMemoryBackend) CreateAPIMapping(domainName string, input CreateAPIMap APIMappingKey: input.APIMappingKey, } - b.apiMappings[domainName][id] = mapping + b.apiMappings.Put(mapping) cp := *mapping @@ -1689,20 +1835,15 @@ func (b *InMemoryBackend) CreateIntegrationResponse( b.mu.Lock("CreateIntegrationResponse") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - if _, exists := d.integrations[integrationID]; !exists { + if !b.integrations.Has(integrationKey(apiID, integrationID)) { return nil, ErrIntegrationNotFound } - if _, exists := d.integrationResponses[integrationID]; !exists { - d.integrationResponses[integrationID] = make(map[string]*IntegrationResponse) - } - - for _, existing := range d.integrationResponses[integrationID] { + for _, existing := range b.integrationResponsesByIntegration.Get(integrationKey(apiID, integrationID)) { if existing.IntegrationResponseKey == input.IntegrationResponseKey { return nil, fmt.Errorf( "%w: integration response key %q already exists", @@ -1724,7 +1865,7 @@ func (b *InMemoryBackend) CreateIntegrationResponse( ResponseTemplates: input.ResponseTemplates, } - d.integrationResponses[integrationID][id] = ir + b.integrationResponses.Put(ir) cp := *ir @@ -1742,12 +1883,11 @@ func (b *InMemoryBackend) CreateModel(apiID string, input CreateModelInput) (*Mo b.mu.Lock("CreateModel") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - for _, existing := range d.models { + for _, existing := range b.modelsByAPI.Get(apiID) { if existing.Name == input.Name { return nil, fmt.Errorf("%w: model name %q already exists", ErrAlreadyExists, input.Name) } @@ -1763,7 +1903,7 @@ func (b *InMemoryBackend) CreateModel(apiID string, input CreateModelInput) (*Mo Description: input.Description, } - d.models[id] = model + b.models.Put(model) cp := *model @@ -1784,20 +1924,15 @@ func (b *InMemoryBackend) CreateRouteResponse( b.mu.Lock("CreateRouteResponse") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - if _, exists := d.routes[routeID]; !exists { + if !b.routes.Has(routeKey(apiID, routeID)) { return nil, ErrRouteNotFound } - if _, exists := d.routeResponses[routeID]; !exists { - d.routeResponses[routeID] = make(map[string]*RouteResponse) - } - - for _, existing := range d.routeResponses[routeID] { + for _, existing := range b.routeResponsesByRoute.Get(routeKey(apiID, routeID)) { if existing.RouteResponseKey == input.RouteResponseKey { return nil, fmt.Errorf("%w: route response key %q already exists", ErrAlreadyExists, input.RouteResponseKey) } @@ -1813,7 +1948,7 @@ func (b *InMemoryBackend) CreateRouteResponse( ResponseModels: input.ResponseModels, } - d.routeResponses[routeID][id] = rr + b.routeResponses.Put(rr) cp := *rr @@ -1835,7 +1970,7 @@ func (b *InMemoryBackend) CreatePortal(input CreatePortalInput) (*Portal, error) Status: "ACTIVE", } - b.portals[id] = portal + b.portals.Put(portal) cp := *portal @@ -1861,7 +1996,7 @@ func (b *InMemoryBackend) CreatePortalProduct(input CreatePortalProductInput) (* Tags: copyTags(input.Tags), } - b.portalProducts[id] = product + b.portalProducts.Put(product) cp := *product @@ -1878,7 +2013,7 @@ func (b *InMemoryBackend) CreateProductPage( b.mu.Lock("CreateProductPage") defer b.mu.Unlock() - if _, ok := b.portalProducts[portalProductID]; !ok { + if !b.portalProducts.Has(portalProductID) { return nil, ErrPortalProductNotFound } @@ -1890,7 +2025,7 @@ func (b *InMemoryBackend) CreateProductPage( LastModified: &now, } - b.productPages[portalProductID] = append(b.productPages[portalProductID], page) + b.productPages.Put(page) cp := *page @@ -1907,7 +2042,7 @@ func (b *InMemoryBackend) CreateProductRestEndpointPage( b.mu.Lock("CreateProductRestEndpointPage") defer b.mu.Unlock() - if _, ok := b.portalProducts[portalProductID]; !ok { + if !b.portalProducts.Has(portalProductID) { return nil, ErrPortalProductNotFound } @@ -1919,7 +2054,7 @@ func (b *InMemoryBackend) CreateProductRestEndpointPage( LastModified: &now, } - b.productREPages[portalProductID] = append(b.productREPages[portalProductID], page) + b.productREPages.Put(page) cp := *page @@ -1957,7 +2092,7 @@ func (b *InMemoryBackend) CreateVpcLink(input CreateVpcLinkInput) (*VpcLink, err Tags: copyTags(input.Tags), VpcLinkStatus: "AVAILABLE", } - b.vpcLinks[id] = vpcLink + b.vpcLinks.Put(vpcLink) cp := *vpcLink @@ -1969,7 +2104,7 @@ func (b *InMemoryBackend) GetVpcLink(vpcLinkID string) (*VpcLink, error) { b.mu.RLock("GetVpcLink") defer b.mu.RUnlock() - vpcLink, ok := b.vpcLinks[vpcLinkID] + vpcLink, ok := b.vpcLinks.Get(vpcLinkID) if !ok { return nil, ErrVpcLinkNotFound } @@ -1984,8 +2119,10 @@ func (b *InMemoryBackend) GetVpcLinks() ([]VpcLink, error) { b.mu.RLock("GetVpcLinks") defer b.mu.RUnlock() - out := make([]VpcLink, 0, len(b.vpcLinks)) - for _, item := range b.vpcLinks { + all := b.vpcLinks.All() + out := make([]VpcLink, 0, len(all)) + + for _, item := range all { out = append(out, *item) } sort.Slice(out, func(i, j int) bool { return out[i].VpcLinkID < out[j].VpcLinkID }) @@ -1998,7 +2135,7 @@ func (b *InMemoryBackend) UpdateVpcLink(vpcLinkID string, input UpdateVpcLinkInp b.mu.Lock("UpdateVpcLink") defer b.mu.Unlock() - vpcLink, ok := b.vpcLinks[vpcLinkID] + vpcLink, ok := b.vpcLinks.Get(vpcLinkID) if !ok { return nil, ErrVpcLinkNotFound } @@ -2016,10 +2153,9 @@ func (b *InMemoryBackend) DeleteVpcLink(vpcLinkID string) error { b.mu.Lock("DeleteVpcLink") defer b.mu.Unlock() - if _, ok := b.vpcLinks[vpcLinkID]; !ok { + if !b.vpcLinks.Delete(vpcLinkID) { return ErrVpcLinkNotFound } - delete(b.vpcLinks, vpcLinkID) return nil } @@ -2035,12 +2171,10 @@ func (b *InMemoryBackend) CreateRoutingRule( b.mu.Lock("CreateRoutingRule") defer b.mu.Unlock() - if _, ok := b.domainNames[domainName]; !ok { + if !b.domainNames.Has(domainName) { return nil, ErrDomainNameNotFound } - if _, ok := b.routingRules[domainName]; !ok { - b.routingRules[domainName] = make(map[string]*RoutingRule) - } + id := randomID() rule := &RoutingRule{ RoutingRuleID: id, @@ -2051,7 +2185,7 @@ func (b *InMemoryBackend) CreateRoutingRule( Actions: input.Actions, Conditions: input.Conditions, } - b.routingRules[domainName][id] = rule + b.routingRules.Put(rule) cp := *rule @@ -2063,11 +2197,11 @@ func (b *InMemoryBackend) GetRoutingRule(domainName, routingRuleID string) (*Rou b.mu.RLock("GetRoutingRule") defer b.mu.RUnlock() - rules, ok := b.routingRules[domainName] - if !ok { + if !b.domainNames.Has(domainName) { return nil, ErrDomainNameNotFound } - rule, ok := rules[routingRuleID] + + rule, ok := b.routingRules.Get(routingRuleKey(domainName, routingRuleID)) if !ok { return nil, ErrRoutingRuleNotFound } @@ -2082,11 +2216,13 @@ func (b *InMemoryBackend) ListRoutingRules(domainName string) ([]RoutingRule, er b.mu.RLock("ListRoutingRules") defer b.mu.RUnlock() - if _, ok := b.domainNames[domainName]; !ok { + if !b.domainNames.Has(domainName) { return nil, ErrDomainNameNotFound } - rules := b.routingRules[domainName] + + rules := b.routingRulesByDomain.Get(domainName) out := make([]RoutingRule, 0, len(rules)) + for _, rule := range rules { out = append(out, *rule) } @@ -2103,11 +2239,11 @@ func (b *InMemoryBackend) PutRoutingRule( b.mu.Lock("PutRoutingRule") defer b.mu.Unlock() - rules, ok := b.routingRules[domainName] - if !ok { + if !b.domainNames.Has(domainName) { return nil, ErrDomainNameNotFound } - rule, ok := rules[routingRuleID] + + rule, ok := b.routingRules.Get(routingRuleKey(domainName, routingRuleID)) if !ok { return nil, ErrRoutingRuleNotFound } @@ -2125,14 +2261,13 @@ func (b *InMemoryBackend) DeleteRoutingRule(domainName, routingRuleID string) er b.mu.Lock("DeleteRoutingRule") defer b.mu.Unlock() - rules, ok := b.routingRules[domainName] - if !ok { + if !b.domainNames.Has(domainName) { return ErrDomainNameNotFound } - if _, exists := rules[routingRuleID]; !exists { + + if !b.routingRules.Delete(routingRuleKey(domainName, routingRuleID)) { return ErrRoutingRuleNotFound } - delete(rules, routingRuleID) return nil } @@ -2144,7 +2279,7 @@ func (b *InMemoryBackend) GetPortalProductSharingPolicy(portalProductID string) b.mu.RLock("GetPortalProductSharingPolicy") defer b.mu.RUnlock() - if _, ok := b.portalProducts[portalProductID]; !ok { + if !b.portalProducts.Has(portalProductID) { return nil, ErrPortalProductNotFound } @@ -2158,7 +2293,7 @@ func (b *InMemoryBackend) PutPortalProductSharingPolicy( b.mu.Lock("PutPortalProductSharingPolicy") defer b.mu.Unlock() - if _, ok := b.portalProducts[portalProductID]; !ok { + if !b.portalProducts.Has(portalProductID) { return nil, ErrPortalProductNotFound } b.portalProductSharingPolicies[portalProductID] = policyDocument @@ -2171,7 +2306,7 @@ func (b *InMemoryBackend) DeletePortalProductSharingPolicy(portalProductID strin b.mu.Lock("DeletePortalProductSharingPolicy") defer b.mu.Unlock() - if _, ok := b.portalProducts[portalProductID]; !ok { + if !b.portalProducts.Has(portalProductID) { return ErrPortalProductNotFound } delete(b.portalProductSharingPolicies, portalProductID) @@ -2186,7 +2321,7 @@ func (b *InMemoryBackend) GetDomainName(domainName string) (*DomainName, error) b.mu.RLock("GetDomainName") defer b.mu.RUnlock() - dn, ok := b.domainNames[domainName] + dn, ok := b.domainNames.Get(domainName) if !ok { return nil, ErrDomainNameNotFound } @@ -2201,8 +2336,10 @@ func (b *InMemoryBackend) GetDomainNames() ([]DomainName, error) { b.mu.RLock("GetDomainNames") defer b.mu.RUnlock() - result := make([]DomainName, 0, len(b.domainNames)) - for _, dn := range b.domainNames { + all := b.domainNames.All() + result := make([]DomainName, 0, len(all)) + + for _, dn := range all { result = append(result, *dn) } @@ -2218,13 +2355,17 @@ func (b *InMemoryBackend) DeleteDomainName(domainName string) error { b.mu.Lock("DeleteDomainName") defer b.mu.Unlock() - if _, ok := b.domainNames[domainName]; !ok { + if !b.domainNames.Delete(domainName) { return ErrDomainNameNotFound } - delete(b.domainNames, domainName) - delete(b.apiMappings, domainName) - delete(b.routingRules, domainName) + for _, m := range slices.Clone(b.apiMappingsByDomain.Get(domainName)) { + b.apiMappings.Delete(apiMappingKey(domainName, m.APIMappingID)) + } + + for _, r := range slices.Clone(b.routingRulesByDomain.Get(domainName)) { + b.routingRules.Delete(routingRuleKey(domainName, r.RoutingRuleID)) + } return nil } @@ -2236,12 +2377,11 @@ func (b *InMemoryBackend) GetAPIMapping(domainName, mappingID string) (*APIMappi b.mu.RLock("GetAPIMapping") defer b.mu.RUnlock() - mappings, ok := b.apiMappings[domainName] - if !ok { + if !b.domainNames.Has(domainName) { return nil, ErrDomainNameNotFound } - m, ok := mappings[mappingID] + m, ok := b.apiMappings.Get(apiMappingKey(domainName, mappingID)) if !ok { return nil, ErrAPIMappingNotFound } @@ -2256,12 +2396,13 @@ func (b *InMemoryBackend) GetAPIMappings(domainName string) ([]APIMapping, error b.mu.RLock("GetAPIMappings") defer b.mu.RUnlock() - if _, ok := b.domainNames[domainName]; !ok { + if !b.domainNames.Has(domainName) { return nil, ErrDomainNameNotFound } - mappings := b.apiMappings[domainName] + mappings := b.apiMappingsByDomain.Get(domainName) result := make([]APIMapping, 0, len(mappings)) + for _, m := range mappings { result = append(result, *m) } @@ -2278,17 +2419,14 @@ func (b *InMemoryBackend) DeleteAPIMapping(domainName, mappingID string) error { b.mu.Lock("DeleteAPIMapping") defer b.mu.Unlock() - mappings, ok := b.apiMappings[domainName] - if !ok { + if !b.domainNames.Has(domainName) { return ErrDomainNameNotFound } - if _, exists := mappings[mappingID]; !exists { + if !b.apiMappings.Delete(apiMappingKey(domainName, mappingID)) { return ErrAPIMappingNotFound } - delete(b.apiMappings[domainName], mappingID) - return nil } @@ -2301,22 +2439,16 @@ func (b *InMemoryBackend) GetIntegrationResponse( b.mu.RLock("GetIntegrationResponse") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - if _, exists := d.integrations[integrationID]; !exists { + if !b.integrations.Has(integrationKey(apiID, integrationID)) { return nil, ErrIntegrationNotFound } - responses, hasResponses := d.integrationResponses[integrationID] - if !hasResponses { - return nil, ErrIntegrationResponseNotFound - } - - ir, exists := responses[responseID] - if !exists { + ir, ok := b.integrationResponses.Get(integrationResponseKey(apiID, integrationID, responseID)) + if !ok { return nil, ErrIntegrationResponseNotFound } @@ -2330,16 +2462,15 @@ func (b *InMemoryBackend) GetIntegrationResponses(apiID, integrationID string) ( b.mu.RLock("GetIntegrationResponses") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - if _, exists := d.integrations[integrationID]; !exists { + if !b.integrations.Has(integrationKey(apiID, integrationID)) { return nil, ErrIntegrationNotFound } - responses := d.integrationResponses[integrationID] + responses := b.integrationResponsesByIntegration.Get(integrationKey(apiID, integrationID)) result := make([]IntegrationResponse, 0, len(responses)) for _, ir := range responses { @@ -2358,26 +2489,18 @@ func (b *InMemoryBackend) DeleteIntegrationResponse(apiID, integrationID, respon b.mu.Lock("DeleteIntegrationResponse") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return ErrAPINotFound } - if _, exists := d.integrations[integrationID]; !exists { + if !b.integrations.Has(integrationKey(apiID, integrationID)) { return ErrIntegrationNotFound } - responses, hasResponses := d.integrationResponses[integrationID] - if !hasResponses { + if !b.integrationResponses.Delete(integrationResponseKey(apiID, integrationID, responseID)) { return ErrIntegrationResponseNotFound } - if _, exists := responses[responseID]; !exists { - return ErrIntegrationResponseNotFound - } - - delete(d.integrationResponses[integrationID], responseID) - return nil } @@ -2388,12 +2511,11 @@ func (b *InMemoryBackend) GetModel(apiID, modelID string) (*Model, error) { b.mu.RLock("GetModel") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - m, ok := d.models[modelID] + m, ok := b.models.Get(modelKey(apiID, modelID)) if !ok { return nil, ErrModelNotFound } @@ -2408,13 +2530,14 @@ func (b *InMemoryBackend) GetModels(apiID string) ([]Model, error) { b.mu.RLock("GetModels") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - result := make([]Model, 0, len(d.models)) - for _, m := range d.models { + models := b.modelsByAPI.Get(apiID) + result := make([]Model, 0, len(models)) + + for _, m := range models { result = append(result, *m) } @@ -2430,17 +2553,14 @@ func (b *InMemoryBackend) DeleteModel(apiID, modelID string) error { b.mu.Lock("DeleteModel") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return ErrAPINotFound } - if _, exists := d.models[modelID]; !exists { + if !b.models.Delete(modelKey(apiID, modelID)) { return ErrModelNotFound } - delete(d.models, modelID) - return nil } @@ -2451,22 +2571,16 @@ func (b *InMemoryBackend) GetRouteResponse(apiID, routeID, responseID string) (* b.mu.RLock("GetRouteResponse") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - if _, exists := d.routes[routeID]; !exists { + if !b.routes.Has(routeKey(apiID, routeID)) { return nil, ErrRouteNotFound } - responses, hasResponses := d.routeResponses[routeID] - if !hasResponses { - return nil, ErrRouteResponseNotFound - } - - rr, exists := responses[responseID] - if !exists { + rr, ok := b.routeResponses.Get(routeResponseKey(apiID, routeID, responseID)) + if !ok { return nil, ErrRouteResponseNotFound } @@ -2480,16 +2594,15 @@ func (b *InMemoryBackend) GetRouteResponses(apiID, routeID string) ([]RouteRespo b.mu.RLock("GetRouteResponses") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - if _, exists := d.routes[routeID]; !exists { + if !b.routes.Has(routeKey(apiID, routeID)) { return nil, ErrRouteNotFound } - responses := d.routeResponses[routeID] + responses := b.routeResponsesByRoute.Get(routeKey(apiID, routeID)) result := make([]RouteResponse, 0, len(responses)) for _, rr := range responses { @@ -2508,26 +2621,18 @@ func (b *InMemoryBackend) DeleteRouteResponse(apiID, routeID, responseID string) b.mu.Lock("DeleteRouteResponse") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return ErrAPINotFound } - if _, exists := d.routes[routeID]; !exists { + if !b.routes.Has(routeKey(apiID, routeID)) { return ErrRouteNotFound } - responses, hasResponses := d.routeResponses[routeID] - if !hasResponses { - return ErrRouteResponseNotFound - } - - if _, exists := responses[responseID]; !exists { + if !b.routeResponses.Delete(routeResponseKey(apiID, routeID, responseID)) { return ErrRouteResponseNotFound } - delete(d.routeResponses[routeID], responseID) - return nil } @@ -2538,7 +2643,7 @@ func (b *InMemoryBackend) GetPortal(portalID string) (*Portal, error) { b.mu.RLock("GetPortal") defer b.mu.RUnlock() - p, ok := b.portals[portalID] + p, ok := b.portals.Get(portalID) if !ok { return nil, ErrPortalNotFound } @@ -2553,8 +2658,10 @@ func (b *InMemoryBackend) ListPortals() ([]Portal, error) { b.mu.RLock("ListPortals") defer b.mu.RUnlock() - result := make([]Portal, 0, len(b.portals)) - for _, p := range b.portals { + all := b.portals.All() + result := make([]Portal, 0, len(all)) + + for _, p := range all { result = append(result, *p) } @@ -2572,7 +2679,7 @@ func (b *InMemoryBackend) GetPortalProduct(portalProductID string) (*PortalProdu b.mu.RLock("GetPortalProduct") defer b.mu.RUnlock() - pp, ok := b.portalProducts[portalProductID] + pp, ok := b.portalProducts.Get(portalProductID) if !ok { return nil, ErrPortalProductNotFound } @@ -2587,8 +2694,10 @@ func (b *InMemoryBackend) ListPortalProducts() ([]PortalProduct, error) { b.mu.RLock("ListPortalProducts") defer b.mu.RUnlock() - result := make([]PortalProduct, 0, len(b.portalProducts)) - for _, pp := range b.portalProducts { + all := b.portalProducts.All() + result := make([]PortalProduct, 0, len(all)) + + for _, pp := range all { result = append(result, *pp) } @@ -2606,11 +2715,11 @@ func (b *InMemoryBackend) ListProductPages(portalProductID string) ([]ProductPag b.mu.RLock("ListProductPages") defer b.mu.RUnlock() - if _, ok := b.portalProducts[portalProductID]; !ok { + if !b.portalProducts.Has(portalProductID) { return nil, ErrPortalProductNotFound } - pages := b.productPages[portalProductID] + pages := b.productPagesByPortalProduct.Get(portalProductID) result := make([]ProductPage, 0, len(pages)) for _, p := range pages { @@ -2627,11 +2736,11 @@ func (b *InMemoryBackend) ListProductRestEndpointPages(portalProductID string) ( b.mu.RLock("ListProductRestEndpointPages") defer b.mu.RUnlock() - if _, ok := b.portalProducts[portalProductID]; !ok { + if !b.portalProducts.Has(portalProductID) { return nil, ErrPortalProductNotFound } - pages := b.productREPages[portalProductID] + pages := b.productREPagesByPortalProduct.Get(portalProductID) result := make([]ProductRestEndpointPage, 0, len(pages)) for _, p := range pages { @@ -2647,12 +2756,51 @@ const ( arnResourceTypeAPIs = "apis" arnResourceTypeVpcLinks = "vpclinks" arnResourceTypeDomainNames = "domainnames" + arnResourceTypeStages = "stages" // arnMinPartsWithResourceType is the minimum number of slash-separated // parts in an ARN that carries an explicit resource type segment. arnMinPartsWithResourceType = 2 + + // arnStageParts is the number of trailing slash-separated segments in a + // Stage resource ARN: "apis", "{apiId}", "stages", "{stageName}". + arnStageParts = 4 ) +// parseStageARN extracts the API ID and stage name from a Stage resource ARN +// of the form ".../apis/{apiId}/stages/{stageName}", the AWS-modeled ARN for +// a Stage (Stages, unlike APIs/VPC links/domain names, are nested one level +// under their owning API so a single trailing "type/id" pair is not enough to +// resolve them). Returns ok=false for any ARN that does not match this shape. +func parseStageARN(arn string) (string, string, bool) { + parts := strings.Split(arn, "/") + if len(parts) < arnStageParts { + return "", "", false + } + + tail := parts[len(parts)-arnStageParts:] + if tail[0] != arnResourceTypeAPIs || tail[2] != arnResourceTypeStages { + return "", "", false + } + + return tail[1], tail[3], true +} + +// lookupStageLocked resolves a Stage by API ID and stage name. Callers must +// already hold b.mu (read or write). +func (b *InMemoryBackend) lookupStageLocked(apiID, stageName string) (*Stage, error) { + if !b.apis.Has(apiID) { + return nil, ErrAPINotFound + } + + s, ok := b.stages.Get(stageKey(apiID, stageName)) + if !ok { + return nil, ErrStageNotFound + } + + return s, nil +} + // arnResourceType returns the resource type and ID extracted from an ARN. // For ARNs like "arn:aws:apigateway:us-east-1::/apis/abc123" the resource // type would be "apis" and the ID "abc123". @@ -2668,27 +2816,42 @@ func arnResourceType(arn string) (string, string) { } // TagResource adds tags to a resource identified by ARN. -// Supports APIs, VPC links, and domain names. +// Supports APIs, stages, VPC links, and domain names. func (b *InMemoryBackend) TagResource(resourceARN string, tags map[string]string) error { b.mu.Lock("TagResource") defer b.mu.Unlock() + if apiID, stageName, ok := parseStageARN(resourceARN); ok { + s, err := b.lookupStageLocked(apiID, stageName) + if err != nil { + return err + } + + if s.Tags == nil { + s.Tags = make(map[string]string) + } + + maps.Copy(s.Tags, tags) + + return nil + } + resourceType, resourceID := arnResourceType(resourceARN) switch resourceType { case arnResourceTypeAPIs: - d, ok := b.apis[resourceID] + api, ok := b.apis.Get(resourceID) if !ok { return ErrAPINotFound } - if d.api.Tags == nil { - d.api.Tags = make(map[string]string) + if api.Tags == nil { + api.Tags = make(map[string]string) } - maps.Copy(d.api.Tags, tags) + maps.Copy(api.Tags, tags) case arnResourceTypeVpcLinks: - v, ok := b.vpcLinks[resourceID] + v, ok := b.vpcLinks.Get(resourceID) if !ok { return ErrVpcLinkNotFound } @@ -2699,7 +2862,7 @@ func (b *InMemoryBackend) TagResource(resourceARN string, tags map[string]string maps.Copy(v.Tags, tags) case arnResourceTypeDomainNames: - dn, ok := b.domainNames[resourceID] + dn, ok := b.domainNames.Get(resourceID) if !ok { return ErrDomainNameNotFound } @@ -2717,25 +2880,38 @@ func (b *InMemoryBackend) TagResource(resourceARN string, tags map[string]string } // UntagResource removes tag keys from a resource identified by ARN. -// Supports APIs, VPC links, and domain names. +// Supports APIs, stages, VPC links, and domain names. func (b *InMemoryBackend) UntagResource(resourceARN string, tagKeys []string) error { b.mu.Lock("UntagResource") defer b.mu.Unlock() + if apiID, stageName, ok := parseStageARN(resourceARN); ok { + s, err := b.lookupStageLocked(apiID, stageName) + if err != nil { + return err + } + + for _, k := range tagKeys { + delete(s.Tags, k) + } + + return nil + } + resourceType, resourceID := arnResourceType(resourceARN) switch resourceType { case arnResourceTypeAPIs: - d, ok := b.apis[resourceID] + api, ok := b.apis.Get(resourceID) if !ok { return ErrAPINotFound } for _, k := range tagKeys { - delete(d.api.Tags, k) + delete(api.Tags, k) } case arnResourceTypeVpcLinks: - v, ok := b.vpcLinks[resourceID] + v, ok := b.vpcLinks.Get(resourceID) if !ok { return ErrVpcLinkNotFound } @@ -2744,7 +2920,7 @@ func (b *InMemoryBackend) UntagResource(resourceARN string, tagKeys []string) er delete(v.Tags, k) } case arnResourceTypeDomainNames: - dn, ok := b.domainNames[resourceID] + dn, ok := b.domainNames.Get(resourceID) if !ok { return ErrDomainNameNotFound } @@ -2760,30 +2936,39 @@ func (b *InMemoryBackend) UntagResource(resourceARN string, tagKeys []string) er } // GetTags retrieves all tags for a resource identified by ARN. -// Supports APIs, VPC links, and domain names. +// Supports APIs, stages, VPC links, and domain names. func (b *InMemoryBackend) GetTags(resourceARN string) (map[string]string, error) { b.mu.RLock("GetTags") defer b.mu.RUnlock() + if apiID, stageName, ok := parseStageARN(resourceARN); ok { + s, err := b.lookupStageLocked(apiID, stageName) + if err != nil { + return nil, err + } + + return copyTags(s.Tags), nil + } + resourceType, resourceID := arnResourceType(resourceARN) switch resourceType { case arnResourceTypeAPIs: - d, ok := b.apis[resourceID] + api, ok := b.apis.Get(resourceID) if !ok { return nil, ErrAPINotFound } - return copyTags(d.api.Tags), nil + return copyTags(api.Tags), nil case arnResourceTypeVpcLinks: - v, ok := b.vpcLinks[resourceID] + v, ok := b.vpcLinks.Get(resourceID) if !ok { return nil, ErrVpcLinkNotFound } return copyTags(v.Tags), nil case arnResourceTypeDomainNames: - dn, ok := b.domainNames[resourceID] + dn, ok := b.domainNames.Get(resourceID) if !ok { return nil, ErrDomainNameNotFound } @@ -2802,30 +2987,24 @@ func (b *InMemoryBackend) UpdateAPIMapping( b.mu.Lock("UpdateAPIMapping") defer b.mu.Unlock() - if _, ok := b.domainNames[domainName]; !ok { + if !b.domainNames.Has(domainName) { return nil, ErrDomainNameNotFound } - mappings, ok := b.apiMappings[domainName] - if !ok { - return nil, ErrDomainNameNotFound - } - - m, ok := mappings[mappingID] + m, ok := b.apiMappings.Get(apiMappingKey(domainName, mappingID)) if !ok { return nil, ErrAPIMappingNotFound } if input.APIID != "" { - d, apiExists := b.apis[input.APIID] - if !apiExists { + if !b.apis.Has(input.APIID) { return nil, ErrAPINotFound } stageToCheck := m.Stage if input.Stage != "" { stageToCheck = input.Stage } - if _, exists := d.stages[stageToCheck]; !exists { + if !b.stages.Has(stageKey(input.APIID, stageToCheck)) { return nil, ErrStageNotFound } m.APIID = input.APIID @@ -2852,12 +3031,11 @@ func (b *InMemoryBackend) UpdateDeployment( b.mu.Lock("UpdateDeployment") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - dep, ok := d.deployments[deploymentID] + dep, ok := b.deployments.Get(deploymentKey(apiID, deploymentID)) if !ok { return nil, ErrDeploymentNotFound } @@ -2876,7 +3054,7 @@ func (b *InMemoryBackend) UpdateDomainName(domainName string, input UpdateDomain b.mu.Lock("UpdateDomainName") defer b.mu.Unlock() - dn, ok := b.domainNames[domainName] + dn, ok := b.domainNames.Get(domainName) if !ok { return nil, ErrDomainNameNotFound } @@ -2894,6 +3072,10 @@ func (b *InMemoryBackend) UpdateDomainName(domainName string, input UpdateDomain dn.DomainNameConfigurations = configs } + if input.MutualTLSAuthentication != nil { + dn.MutualTLSAuthentication = cloneMutualTLSAuthentication(input.MutualTLSAuthentication) + } + cp := *dn return &cp, nil @@ -2907,22 +3089,16 @@ func (b *InMemoryBackend) UpdateIntegrationResponse( b.mu.Lock("UpdateIntegrationResponse") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - if _, exists := d.integrations[integrationID]; !exists { + if !b.integrations.Has(integrationKey(apiID, integrationID)) { return nil, ErrIntegrationNotFound } - responses, hasResponses := d.integrationResponses[integrationID] - if !hasResponses { - return nil, ErrIntegrationResponseNotFound - } - - ir, exists := responses[responseID] - if !exists { + ir, ok := b.integrationResponses.Get(integrationResponseKey(apiID, integrationID, responseID)) + if !ok { return nil, ErrIntegrationResponseNotFound } @@ -2956,12 +3132,11 @@ func (b *InMemoryBackend) UpdateModel(apiID, modelID string, input UpdateModelIn b.mu.Lock("UpdateModel") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - m, ok := d.models[modelID] + m, ok := b.models.Get(modelKey(apiID, modelID)) if !ok { return nil, ErrModelNotFound } @@ -2995,22 +3170,16 @@ func (b *InMemoryBackend) UpdateRouteResponse( b.mu.Lock("UpdateRouteResponse") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return nil, ErrAPINotFound } - if _, exists := d.routes[routeID]; !exists { + if !b.routes.Has(routeKey(apiID, routeID)) { return nil, ErrRouteNotFound } - responses, hasResponses := d.routeResponses[routeID] - if !hasResponses { - return nil, ErrRouteResponseNotFound - } - - rr, exists := responses[responseID] - if !exists { + rr, ok := b.routeResponses.Get(routeResponseKey(apiID, routeID, responseID)) + if !ok { return nil, ErrRouteResponseNotFound } @@ -3036,7 +3205,7 @@ func (b *InMemoryBackend) UpdatePortal(portalID string, input UpdatePortalInput) b.mu.Lock("UpdatePortal") defer b.mu.Unlock() - p, ok := b.portals[portalID] + p, ok := b.portals.Get(portalID) if !ok { return nil, ErrPortalNotFound } @@ -3068,7 +3237,7 @@ func (b *InMemoryBackend) UpdatePortalProduct( b.mu.Lock("UpdatePortalProduct") defer b.mu.Unlock() - pp, ok := b.portalProducts[portalProductID] + pp, ok := b.portalProducts.Get(portalProductID) if !ok { return nil, ErrPortalProductNotFound } @@ -3101,23 +3270,24 @@ func (b *InMemoryBackend) UpdateProductPage( b.mu.Lock("UpdateProductPage") defer b.mu.Unlock() - if _, ok := b.portalProducts[portalProductID]; !ok { + if !b.portalProducts.Has(portalProductID) { return nil, ErrPortalProductNotFound } - for _, page := range b.productPages[portalProductID] { - if page.ProductPageID == pageID { - now := isoTime{time.Now()} - if input.DisplayContent != nil { - page.DisplayContent = input.DisplayContent - } - page.LastModified = &now - cp := *page - return &cp, nil - } + page, ok := b.productPages.Get(productPageKey(portalProductID, pageID)) + if !ok { + return nil, ErrProductPageNotFound + } + + now := isoTime{time.Now()} + if input.DisplayContent != nil { + page.DisplayContent = input.DisplayContent } + page.LastModified = &now + + cp := *page - return nil, ErrProductPageNotFound + return &cp, nil } // UpdateProductRestEndpointPage updates a product REST endpoint page. @@ -3128,23 +3298,24 @@ func (b *InMemoryBackend) UpdateProductRestEndpointPage( b.mu.Lock("UpdateProductRestEndpointPage") defer b.mu.Unlock() - if _, ok := b.portalProducts[portalProductID]; !ok { + if !b.portalProducts.Has(portalProductID) { return nil, ErrPortalProductNotFound } - for _, page := range b.productREPages[portalProductID] { - if page.ProductRestEndpointPageID == pageID { - now := isoTime{time.Now()} - if input.DisplayContent != nil { - page.DisplayContent = input.DisplayContent - } - page.LastModified = &now - cp := *page - return &cp, nil - } + page, ok := b.productREPages.Get(productREPageKey(portalProductID, pageID)) + if !ok { + return nil, ErrProductREPageNotFound } - return nil, ErrProductREPageNotFound + now := isoTime{time.Now()} + if input.DisplayContent != nil { + page.DisplayContent = input.DisplayContent + } + page.LastModified = &now + + cp := *page + + return &cp, nil } // DeletePortal removes a portal by ID. @@ -3152,12 +3323,10 @@ func (b *InMemoryBackend) DeletePortal(portalID string) error { b.mu.Lock("DeletePortal") defer b.mu.Unlock() - if _, ok := b.portals[portalID]; !ok { + if !b.portals.Delete(portalID) { return ErrPortalNotFound } - delete(b.portals, portalID) - return nil } @@ -3166,13 +3335,18 @@ func (b *InMemoryBackend) DeletePortalProduct(portalProductID string) error { b.mu.Lock("DeletePortalProduct") defer b.mu.Unlock() - if _, ok := b.portalProducts[portalProductID]; !ok { + if !b.portalProducts.Delete(portalProductID) { return ErrPortalProductNotFound } - delete(b.portalProducts, portalProductID) - delete(b.productPages, portalProductID) - delete(b.productREPages, portalProductID) + for _, p := range slices.Clone(b.productPagesByPortalProduct.Get(portalProductID)) { + b.productPages.Delete(productPageKey(portalProductID, p.ProductPageID)) + } + + for _, p := range slices.Clone(b.productREPagesByPortalProduct.Get(portalProductID)) { + b.productREPages.Delete(productREPageKey(portalProductID, p.ProductRestEndpointPageID)) + } + // Clean up the sharing policy entry so deleting a product does not leak its // policy document (AWS removes the associated sharing policy on delete). delete(b.portalProductSharingPolicies, portalProductID) @@ -3185,19 +3359,18 @@ func (b *InMemoryBackend) GetProductPage(portalProductID, pageID string) (*Produ b.mu.RLock("GetProductPage") defer b.mu.RUnlock() - if _, ok := b.portalProducts[portalProductID]; !ok { + if !b.portalProducts.Has(portalProductID) { return nil, ErrPortalProductNotFound } - for _, p := range b.productPages[portalProductID] { - if p.ProductPageID == pageID { - cp := *p - - return &cp, nil - } + p, ok := b.productPages.Get(productPageKey(portalProductID, pageID)) + if !ok { + return nil, ErrProductPageNotFound } - return nil, ErrProductPageNotFound + cp := *p + + return &cp, nil } // GetProductRestEndpointPage retrieves a specific product REST endpoint page. @@ -3205,19 +3378,18 @@ func (b *InMemoryBackend) GetProductRestEndpointPage(portalProductID, pageID str b.mu.RLock("GetProductRestEndpointPage") defer b.mu.RUnlock() - if _, ok := b.portalProducts[portalProductID]; !ok { + if !b.portalProducts.Has(portalProductID) { return nil, ErrPortalProductNotFound } - for _, p := range b.productREPages[portalProductID] { - if p.ProductRestEndpointPageID == pageID { - cp := *p - - return &cp, nil - } + p, ok := b.productREPages.Get(productREPageKey(portalProductID, pageID)) + if !ok { + return nil, ErrProductREPageNotFound } - return nil, ErrProductREPageNotFound + cp := *p + + return &cp, nil } // DeleteProductPage removes a product page from a portal product. @@ -3225,20 +3397,15 @@ func (b *InMemoryBackend) DeleteProductPage(portalProductID, pageID string) erro b.mu.Lock("DeleteProductPage") defer b.mu.Unlock() - if _, ok := b.portalProducts[portalProductID]; !ok { + if !b.portalProducts.Has(portalProductID) { return ErrPortalProductNotFound } - pages := b.productPages[portalProductID] - for i, p := range pages { - if p.ProductPageID == pageID { - b.productPages[portalProductID] = append(pages[:i], pages[i+1:]...) - - return nil - } + if !b.productPages.Delete(productPageKey(portalProductID, pageID)) { + return ErrProductPageNotFound } - return ErrProductPageNotFound + return nil } // DeleteProductRestEndpointPage removes a product REST endpoint page from a portal product. @@ -3246,20 +3413,15 @@ func (b *InMemoryBackend) DeleteProductRestEndpointPage(portalProductID, pageID b.mu.Lock("DeleteProductRestEndpointPage") defer b.mu.Unlock() - if _, ok := b.portalProducts[portalProductID]; !ok { + if !b.portalProducts.Has(portalProductID) { return ErrPortalProductNotFound } - pages := b.productREPages[portalProductID] - for i, p := range pages { - if p.ProductRestEndpointPageID == pageID { - b.productREPages[portalProductID] = append(pages[:i], pages[i+1:]...) - - return nil - } + if !b.productREPages.Delete(productREPageKey(portalProductID, pageID)) { + return ErrProductREPageNotFound } - return ErrProductREPageNotFound + return nil } // ResetAuthorizersCache is a no-op for the in-memory backend (caching is not simulated). @@ -3267,12 +3429,11 @@ func (b *InMemoryBackend) ResetAuthorizersCache(apiID, stageName string) error { b.mu.RLock("ResetAuthorizersCache") defer b.mu.RUnlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return ErrAPINotFound } - if _, exists := d.stages[stageName]; !exists { + if !b.stages.Has(stageKey(apiID, stageName)) { return ErrStageNotFound } @@ -3284,12 +3445,12 @@ func (b *InMemoryBackend) DeleteCorsConfiguration(apiID string) error { b.mu.Lock("DeleteCorsConfiguration") defer b.mu.Unlock() - d, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return ErrAPINotFound } - d.api.CorsConfiguration = nil + api.CorsConfiguration = nil return nil } @@ -3299,12 +3460,11 @@ func (b *InMemoryBackend) DeleteAccessLogSettings(apiID, stageName string) error b.mu.Lock("DeleteAccessLogSettings") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return ErrAPINotFound } - s, ok := d.stages[stageName] + s, ok := b.stages.Get(stageKey(apiID, stageName)) if !ok { return ErrStageNotFound } @@ -3320,12 +3480,11 @@ func (b *InMemoryBackend) DeleteRouteSettings(apiID, stageName, routeKey string) b.mu.Lock("DeleteRouteSettings") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return ErrAPINotFound } - s, ok := d.stages[stageName] + s, ok := b.stages.Get(stageKey(apiID, stageName)) if !ok { return ErrStageNotFound } @@ -3344,7 +3503,7 @@ func (b *InMemoryBackend) ExportAPI(apiID string) (map[string]any, error) { b.mu.RLock("ExportAPI") defer b.mu.RUnlock() - d, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return nil, ErrAPINotFound } @@ -3353,7 +3512,7 @@ func (b *InMemoryBackend) ExportAPI(apiID string) (map[string]any, error) { paths := map[string]any{} - for _, route := range d.routes { + for _, route := range b.routesByAPI.Get(apiID) { // Parse route key: e.g. "GET /items" or "$connect" (WebSocket) parts := strings.SplitN(route.RouteKey, " ", routeKeyParts) @@ -3383,7 +3542,7 @@ func (b *InMemoryBackend) ExportAPI(apiID string) (map[string]any, error) { op["summary"] = route.OperationName } - if secName := exportRouteSecurityName(d, route); secName != "" { + if secName := b.exportRouteSecurityName(apiID, route); secName != "" { scopes := route.AuthorizationScopes if scopes == nil { scopes = []string{} @@ -3396,12 +3555,12 @@ func (b *InMemoryBackend) ExportAPI(apiID string) (map[string]any, error) { } info := map[string]any{ - "title": d.api.Name, - "version": d.api.Version, + "title": api.Name, + "version": api.Version, } - if d.api.Description != "" { - info["description"] = d.api.Description + if api.Description != "" { + info["description"] = api.Description } spec := map[string]any{ @@ -3410,7 +3569,7 @@ func (b *InMemoryBackend) ExportAPI(apiID string) (map[string]any, error) { "paths": paths, } - if schemes := exportSecuritySchemes(d); len(schemes) > 0 { + if schemes := b.exportSecuritySchemes(apiID); len(schemes) > 0 { spec["components"] = map[string]any{"securitySchemes": schemes} } @@ -3420,12 +3579,12 @@ func (b *InMemoryBackend) ExportAPI(apiID string) (map[string]any, error) { // exportRouteSecurityName returns the OpenAPI security-scheme name that a route // references, or "" when the route requires no authorization. JWT/CUSTOM routes // reference their authorizer by name; AWS_IAM references the "sigv4" scheme. -func exportRouteSecurityName(d *apiData, route *Route) string { +func (b *InMemoryBackend) exportRouteSecurityName(apiID string, route *Route) string { switch route.AuthorizationType { case authorizationTypeAWSIAM: return "sigv4" case authorizerTypeJWT, authorizationTypeCustom: - if a, ok := d.authorizers[route.AuthorizerID]; ok { + if a, ok := b.authorizers.Get(authorizerKey(apiID, route.AuthorizerID)); ok { return a.Name } @@ -3442,10 +3601,10 @@ func exportRouteSecurityName(d *apiData, route *Route) string { // security schemes. const openAPIKeyType = "type" -func exportSecuritySchemes(d *apiData) map[string]any { +func (b *InMemoryBackend) exportSecuritySchemes(apiID string) map[string]any { schemes := map[string]any{} - for _, a := range d.authorizers { + for _, a := range b.authorizersByAPI.Get(apiID) { if a.AuthorizerType != authorizerTypeJWT { continue } @@ -3454,7 +3613,7 @@ func exportSecuritySchemes(d *apiData) map[string]any { } // Emit a sigv4 scheme when any route uses AWS_IAM authorization. - for _, route := range d.routes { + for _, route := range b.routesByAPI.Get(apiID) { if route.AuthorizationType == authorizationTypeAWSIAM { schemes["sigv4"] = map[string]any{ openAPIKeyType: "apiKey", @@ -3503,12 +3662,11 @@ func (b *InMemoryBackend) DeleteRouteRequestParameter(apiID, routeID, requestPar b.mu.Lock("DeleteRouteRequestParameter") defer b.mu.Unlock() - d, ok := b.apis[apiID] - if !ok { + if !b.apis.Has(apiID) { return ErrAPINotFound } - r, ok := d.routes[routeID] + r, ok := b.routes.Get(routeKey(apiID, routeID)) if !ok { return ErrRouteNotFound } diff --git a/services/apigatewayv2/backend_test.go b/services/apigatewayv2/backend_test.go index 23b7ebced..1ff9d26b5 100644 --- a/services/apigatewayv2/backend_test.go +++ b/services/apigatewayv2/backend_test.go @@ -1252,23 +1252,42 @@ func TestInMemoryBackend_WebSocketRouteKey_NoFormatValidation(t *testing.T) { } } +// TestInMemoryBackend_CreateIntegration_TimeoutValidation proves the +// per-protocol timeout ceiling AWS documents for CreateIntegration: HTTP APIs +// allow up to 30,000ms (default 30,000ms when unset) while WebSocket APIs +// allow up to 29,000ms (default 29,000ms when unset). Before this fix the +// backend hardcoded a 29,000ms ceiling/default for both protocols, incorrectly +// rejecting valid HTTP API timeouts in [29001, 30000] and under-defaulting +// unset HTTP API timeouts to 29000 instead of 30000. func TestInMemoryBackend_CreateIntegration_TimeoutValidation(t *testing.T) { t.Parallel() tests := []struct { - name string - timeoutMs int32 - wantErr bool - wantTimeout int32 + name string + protocolType string + timeoutMs int32 + wantErr bool + wantTimeout int32 }{ - {name: "zero_defaults_to_29000", timeoutMs: 0, wantErr: false, wantTimeout: 29000}, - {name: "min_boundary_50", timeoutMs: 50, wantErr: false, wantTimeout: 50}, - {name: "max_boundary_29000", timeoutMs: 29000, wantErr: false, wantTimeout: 29000}, - {name: "mid_range_5000", timeoutMs: 5000, wantErr: false, wantTimeout: 5000}, - {name: "too_low_49", timeoutMs: 49, wantErr: true}, - {name: "too_low_1", timeoutMs: 1, wantErr: true}, - {name: "too_high_29001", timeoutMs: 29001, wantErr: true}, - {name: "too_high_60000", timeoutMs: 60000, wantErr: true}, + {name: "http_zero_defaults_to_30000", protocolType: "HTTP", timeoutMs: 0, wantErr: false, wantTimeout: 30000}, + {name: "http_min_boundary_50", protocolType: "HTTP", timeoutMs: 50, wantErr: false, wantTimeout: 50}, + {name: "http_max_boundary_30000", protocolType: "HTTP", timeoutMs: 30000, wantErr: false, wantTimeout: 30000}, + {name: "http_mid_range_5000", protocolType: "HTTP", timeoutMs: 5000, wantErr: false, wantTimeout: 5000}, + {name: "http_29001_now_valid", protocolType: "HTTP", timeoutMs: 29001, wantErr: false, wantTimeout: 29001}, + {name: "http_too_low_49", protocolType: "HTTP", timeoutMs: 49, wantErr: true}, + {name: "http_too_low_1", protocolType: "HTTP", timeoutMs: 1, wantErr: true}, + {name: "http_too_high_30001", protocolType: "HTTP", timeoutMs: 30001, wantErr: true}, + {name: "http_too_high_60000", protocolType: "HTTP", timeoutMs: 60000, wantErr: true}, + { + name: "ws_zero_defaults_to_29000", protocolType: "WEBSOCKET", + timeoutMs: 0, wantErr: false, wantTimeout: 29000, + }, + { + name: "ws_max_boundary_29000", protocolType: "WEBSOCKET", + timeoutMs: 29000, wantErr: false, wantTimeout: 29000, + }, + {name: "ws_too_high_29001", protocolType: "WEBSOCKET", timeoutMs: 29001, wantErr: true}, + {name: "ws_too_high_30000", protocolType: "WEBSOCKET", timeoutMs: 30000, wantErr: true}, } for _, tt := range tests { @@ -1279,12 +1298,17 @@ func TestInMemoryBackend_CreateIntegration_TimeoutValidation(t *testing.T) { api, err := b.CreateAPI(context.Background(), apigatewayv2.CreateAPIInput{ Name: "api", - ProtocolType: "HTTP", + ProtocolType: tt.protocolType, }) require.NoError(t, err) + integrationType := "HTTP_PROXY" + if tt.protocolType == "WEBSOCKET" { + integrationType = "AWS" + } + intg, err := b.CreateIntegration(api.APIID, apigatewayv2.CreateIntegrationInput{ - IntegrationType: "HTTP_PROXY", + IntegrationType: integrationType, IntegrationURI: "https://example.com", TimeoutInMillis: tt.timeoutMs, }) @@ -1304,16 +1328,19 @@ func TestInMemoryBackend_UpdateIntegration_TimeoutValidation(t *testing.T) { t.Parallel() tests := []struct { - name string - timeoutMs int32 - wantErr bool + name string + protocolType string + timeoutMs int32 + wantErr bool }{ - {name: "valid_50", timeoutMs: 50, wantErr: false}, - {name: "valid_29000", timeoutMs: 29000, wantErr: false}, - {name: "valid_1000", timeoutMs: 1000, wantErr: false}, - {name: "zero_skips_update", timeoutMs: 0, wantErr: false}, - {name: "too_low_49", timeoutMs: 49, wantErr: true}, - {name: "too_high_29001", timeoutMs: 29001, wantErr: true}, + {name: "http_valid_50", protocolType: "HTTP", timeoutMs: 50, wantErr: false}, + {name: "http_valid_30000", protocolType: "HTTP", timeoutMs: 30000, wantErr: false}, + {name: "http_valid_1000", protocolType: "HTTP", timeoutMs: 1000, wantErr: false}, + {name: "http_zero_skips_update", protocolType: "HTTP", timeoutMs: 0, wantErr: false}, + {name: "http_too_low_49", protocolType: "HTTP", timeoutMs: 49, wantErr: true}, + {name: "http_too_high_30001", protocolType: "HTTP", timeoutMs: 30001, wantErr: true}, + {name: "ws_valid_29000", protocolType: "WEBSOCKET", timeoutMs: 29000, wantErr: false}, + {name: "ws_too_high_29001", protocolType: "WEBSOCKET", timeoutMs: 29001, wantErr: true}, } for _, tt := range tests { @@ -1324,12 +1351,17 @@ func TestInMemoryBackend_UpdateIntegration_TimeoutValidation(t *testing.T) { api, err := b.CreateAPI(context.Background(), apigatewayv2.CreateAPIInput{ Name: "api", - ProtocolType: "HTTP", + ProtocolType: tt.protocolType, }) require.NoError(t, err) + integrationType := "HTTP_PROXY" + if tt.protocolType == "WEBSOCKET" { + integrationType = "AWS" + } + intg, err := b.CreateIntegration(api.APIID, apigatewayv2.CreateIntegrationInput{ - IntegrationType: "HTTP_PROXY", + IntegrationType: integrationType, IntegrationURI: "https://example.com", }) require.NoError(t, err) @@ -1347,3 +1379,294 @@ func TestInMemoryBackend_UpdateIntegration_TimeoutValidation(t *testing.T) { }) } } + +// Test_CreateIntegration_ConnectionType proves ConnectionType defaults to +// INTERNET when unset (matching the AWS-documented default), is validated +// against the modeled enum (INTERNET, VPC_LINK), and requires a connectionId +// when VPC_LINK is specified. Before this fix ConnectionType had no default +// and no validation, so GetIntegration returned an empty string instead of +// "INTERNET" for the common case of an internet-routed integration. +func Test_CreateIntegration_ConnectionType(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + connectionType string + connectionID string + wantConnType string + wantErr bool + }{ + {name: "unset_defaults_to_internet", connectionType: "", wantErr: false, wantConnType: "INTERNET"}, + {name: "explicit_internet", connectionType: "INTERNET", wantErr: false, wantConnType: "INTERNET"}, + { + name: "vpc_link_with_connection_id", connectionType: "VPC_LINK", connectionID: "vpc-link-1", + wantErr: false, wantConnType: "VPC_LINK", + }, + {name: "vpc_link_missing_connection_id", connectionType: "VPC_LINK", wantErr: true}, + {name: "invalid_enum_value", connectionType: "BOGUS", wantErr: true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := apigatewayv2.NewInMemoryBackend() + + api, err := b.CreateAPI(context.Background(), apigatewayv2.CreateAPIInput{ + Name: "api", + ProtocolType: "HTTP", + }) + require.NoError(t, err) + + intg, err := b.CreateIntegration(api.APIID, apigatewayv2.CreateIntegrationInput{ + IntegrationType: "HTTP_PROXY", + IntegrationURI: "https://example.com", + ConnectionType: tc.connectionType, + ConnectionID: tc.connectionID, + }) + + if tc.wantErr { + require.Error(t, err) + assert.ErrorIs(t, err, apigatewayv2.ErrBadRequest) + + return + } + + require.NoError(t, err) + assert.Equal(t, tc.wantConnType, intg.ConnectionType) + + got, err := b.GetIntegration(api.APIID, intg.IntegrationID) + require.NoError(t, err) + assert.Equal(t, tc.wantConnType, got.ConnectionType) + }) + } +} + +// Test_Integration_TlsConfig proves the tlsConfig field (AWS-modeled for +// private integrations) round-trips through Create, Get, and Update. Before +// this fix the Integration model had no TLSConfig field at all, so any client +// setting it silently lost the value. +func Test_Integration_TlsConfig(t *testing.T) { + t.Parallel() + + cases := []struct { + createTLS *apigatewayv2.IntegrationTLSConfig + updateTLS *apigatewayv2.IntegrationTLSConfig + wantAfterUpdate *apigatewayv2.IntegrationTLSConfig + name string + }{ + { + name: "create_with_tls_no_update", + createTLS: &apigatewayv2.IntegrationTLSConfig{ServerNameToVerify: "backend.example.com"}, + wantAfterUpdate: &apigatewayv2.IntegrationTLSConfig{ServerNameToVerify: "backend.example.com"}, + }, + { + name: "create_without_tls_then_add_via_update", + createTLS: nil, + updateTLS: &apigatewayv2.IntegrationTLSConfig{ServerNameToVerify: "updated.example.com"}, + wantAfterUpdate: &apigatewayv2.IntegrationTLSConfig{ + ServerNameToVerify: "updated.example.com", + }, + }, + { + name: "create_without_tls_stays_nil", + createTLS: nil, + wantAfterUpdate: nil, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := apigatewayv2.NewInMemoryBackend() + + api, err := b.CreateAPI(context.Background(), apigatewayv2.CreateAPIInput{ + Name: "api", + ProtocolType: "HTTP", + }) + require.NoError(t, err) + + intg, err := b.CreateIntegration(api.APIID, apigatewayv2.CreateIntegrationInput{ + IntegrationType: "HTTP_PROXY", + IntegrationURI: "https://example.com", + ConnectionType: "VPC_LINK", + ConnectionID: "vpc-link-1", + TLSConfig: tc.createTLS, + }) + require.NoError(t, err) + assert.Equal(t, tc.createTLS, intg.TLSConfig) + + if tc.updateTLS != nil { + _, err = b.UpdateIntegration(api.APIID, intg.IntegrationID, apigatewayv2.UpdateIntegrationInput{ + TLSConfig: tc.updateTLS, + }) + require.NoError(t, err) + } + + got, err := b.GetIntegration(api.APIID, intg.IntegrationID) + require.NoError(t, err) + assert.Equal(t, tc.wantAfterUpdate, got.TLSConfig) + + // Mutating the caller's struct after Create/Update must not alias + // backend state (deep-copy proof). + if tc.createTLS != nil { + tc.createTLS.ServerNameToVerify = "mutated" + + afterMutate, getErr := b.GetIntegration(api.APIID, intg.IntegrationID) + require.NoError(t, getErr) + assert.NotEqual(t, "mutated", func() string { + if afterMutate.TLSConfig == nil { + return "" + } + + return afterMutate.TLSConfig.ServerNameToVerify + }()) + } + }) + } +} + +// Test_Stage_ClientCertificateID proves the clientCertificateId field +// (AWS-modeled for WebSocket API stages) round-trips through CreateStage, +// GetStage, and UpdateStage. Before this fix the Stage model had no +// ClientCertificateID field, so it was silently dropped. +func Test_Stage_ClientCertificateID(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + createCertID string + updateCertID string + wantAfterUpdate string + }{ + {name: "set_on_create_no_update", createCertID: "cert-abc", wantAfterUpdate: "cert-abc"}, + { + name: "set_on_create_then_replaced", createCertID: "cert-abc", + updateCertID: "cert-xyz", wantAfterUpdate: "cert-xyz", + }, + {name: "unset_stays_empty", createCertID: "", wantAfterUpdate: ""}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := apigatewayv2.NewInMemoryBackend() + + api, err := b.CreateAPI(context.Background(), apigatewayv2.CreateAPIInput{ + Name: "api", + ProtocolType: "WEBSOCKET", + }) + require.NoError(t, err) + + stage, err := b.CreateStage(api.APIID, apigatewayv2.CreateStageInput{ + StageName: "prod", + ClientCertificateID: tc.createCertID, + }) + require.NoError(t, err) + assert.Equal(t, tc.createCertID, stage.ClientCertificateID) + + if tc.updateCertID != "" { + _, err = b.UpdateStage(api.APIID, "prod", apigatewayv2.UpdateStageInput{ + ClientCertificateID: tc.updateCertID, + }) + require.NoError(t, err) + } + + got, err := b.GetStage(api.APIID, "prod") + require.NoError(t, err) + assert.Equal(t, tc.wantAfterUpdate, got.ClientCertificateID) + }) + } +} + +// Test_DomainName_ArnAndMutualTLS proves CreateDomainName populates a +// well-formed domainNameArn and round-trips mutualTlsAuthentication, and that +// UpdateDomainName can replace the mTLS configuration. Before this fix +// DomainName had neither field, so both were silently dropped. +func Test_DomainName_ArnAndMutualTLS(t *testing.T) { + t.Parallel() + + ctx := awsmeta.Set(context.Background(), &awsmeta.Metadata{ + Account: "555566667777", + Region: "eu-west-1", + Partition: "aws", + }) + + b := apigatewayv2.NewInMemoryBackend() + + dn, err := b.CreateDomainName(ctx, apigatewayv2.CreateDomainNameInput{ + DomainNameValue: "api.example.com", + MutualTLSAuthentication: &apigatewayv2.MutualTLSAuthentication{ + TruststoreURI: "s3://bucket/truststore.pem", + TruststoreVersion: "v1", + }, + }) + require.NoError(t, err) + + assert.Equal(t, "arn:aws:apigateway:eu-west-1::/domainnames/api.example.com", dn.DomainNameArn) + require.NotNil(t, dn.MutualTLSAuthentication) + assert.Equal(t, "s3://bucket/truststore.pem", dn.MutualTLSAuthentication.TruststoreURI) + assert.Equal(t, "v1", dn.MutualTLSAuthentication.TruststoreVersion) + assert.Empty(t, dn.MutualTLSAuthentication.TruststoreWarnings) + + updated, err := b.UpdateDomainName("api.example.com", apigatewayv2.UpdateDomainNameInput{ + MutualTLSAuthentication: &apigatewayv2.MutualTLSAuthentication{ + TruststoreURI: "s3://bucket/truststore-v2.pem", + TruststoreVersion: "v2", + }, + }) + require.NoError(t, err) + require.NotNil(t, updated.MutualTLSAuthentication) + assert.Equal(t, "s3://bucket/truststore-v2.pem", updated.MutualTLSAuthentication.TruststoreURI) + + // DomainNameArn is stable across updates. + assert.Equal(t, dn.DomainNameArn, updated.DomainNameArn) +} + +// Test_StageTags proves Stage resources can be tagged/untagged via their own +// nested ARN ("arn:.../apis/{apiId}/stages/{stageName}"), matching real API +// Gateway v2 (Stage responses carry a Tags field). Before this fix +// TagResource/UntagResource/GetTags only recognised single-segment resources +// (apis, vpclinks, domainnames) so a stage ARN silently resolved to the wrong +// resource type and 500'd instead of tagging the stage. +func Test_StageTags(t *testing.T) { + t.Parallel() + + b := apigatewayv2.NewInMemoryBackend() + + api, err := b.CreateAPI(context.Background(), apigatewayv2.CreateAPIInput{ + Name: "api", + ProtocolType: "HTTP", + }) + require.NoError(t, err) + + _, err = b.CreateStage(api.APIID, apigatewayv2.CreateStageInput{StageName: "prod"}) + require.NoError(t, err) + + stageARN := "arn:aws:apigateway:us-east-1::/apis/" + api.APIID + "/stages/prod" + + require.NoError(t, b.TagResource(stageARN, map[string]string{"env": "prod", "team": "core"})) + + tags, err := b.GetTags(stageARN) + require.NoError(t, err) + assert.Equal(t, map[string]string{"env": "prod", "team": "core"}, tags) + + stage, err := b.GetStage(api.APIID, "prod") + require.NoError(t, err) + assert.Equal(t, map[string]string{"env": "prod", "team": "core"}, stage.Tags) + + require.NoError(t, b.UntagResource(stageARN, []string{"team"})) + + tags, err = b.GetTags(stageARN) + require.NoError(t, err) + assert.Equal(t, map[string]string{"env": "prod"}, tags) + + // Not-found cases surface the correct AWS error, not a 500. + _, err = b.GetTags("arn:aws:apigateway:us-east-1::/apis/" + api.APIID + "/stages/missing") + require.ErrorIs(t, err, apigatewayv2.ErrStageNotFound) + + _, err = b.GetTags("arn:aws:apigateway:us-east-1::/apis/nonexistent/stages/prod") + require.ErrorIs(t, err, apigatewayv2.ErrAPINotFound) +} diff --git a/services/apigatewayv2/handler.go b/services/apigatewayv2/handler.go index 3c0b419e1..d76c10765 100644 --- a/services/apigatewayv2/handler.go +++ b/services/apigatewayv2/handler.go @@ -2437,7 +2437,7 @@ func (h *Handler) handleGetTags(c *echo.Context, resourceARN string) error { log.Error("apigatewayv2: get tags failed", "resourceArn", resourceARN, "error", err) if errors.Is(err, ErrAPINotFound) || errors.Is(err, ErrVpcLinkNotFound) || - errors.Is(err, ErrDomainNameNotFound) { + errors.Is(err, ErrDomainNameNotFound) || errors.Is(err, ErrStageNotFound) { return writeErr(c, http.StatusNotFound, msgNotFound) } @@ -2463,7 +2463,7 @@ func (h *Handler) handleTagResource(c *echo.Context, resourceARN string) error { log.Error("apigatewayv2: tag resource failed", "resourceArn", resourceARN, "error", err) if errors.Is(err, ErrAPINotFound) || errors.Is(err, ErrVpcLinkNotFound) || - errors.Is(err, ErrDomainNameNotFound) { + errors.Is(err, ErrDomainNameNotFound) || errors.Is(err, ErrStageNotFound) { return writeErr(c, http.StatusNotFound, msgNotFound) } @@ -2483,7 +2483,7 @@ func (h *Handler) handleUntagResource(c *echo.Context, resourceARN string) error log.Error("apigatewayv2: untag resource failed", "resourceArn", resourceARN, "error", err) if errors.Is(err, ErrAPINotFound) || errors.Is(err, ErrVpcLinkNotFound) || - errors.Is(err, ErrDomainNameNotFound) { + errors.Is(err, ErrDomainNameNotFound) || errors.Is(err, ErrStageNotFound) { return writeErr(c, http.StatusNotFound, msgNotFound) } diff --git a/services/apigatewayv2/models.go b/services/apigatewayv2/models.go index 7cfd2334a..bedb6e527 100644 --- a/services/apigatewayv2/models.go +++ b/services/apigatewayv2/models.go @@ -53,6 +53,20 @@ type AccessLogSettings struct { Format string `json:"format,omitempty"` } +// IntegrationTLSConfig holds the TLS configuration for a private integration. +// Supported only for HTTP APIs. +type IntegrationTLSConfig struct { + ServerNameToVerify string `json:"serverNameToVerify,omitempty"` +} + +// MutualTLSAuthentication holds the mutual TLS authentication configuration +// for a custom domain name. +type MutualTLSAuthentication struct { + TruststoreURI string `json:"truststoreUri,omitempty"` + TruststoreVersion string `json:"truststoreVersion,omitempty"` + TruststoreWarnings []string `json:"truststoreWarnings,omitempty"` +} + // RouteSettings holds per-route throttling and logging settings for a stage. type RouteSettings struct { LoggingLevel string `json:"loggingLevel,omitempty"` @@ -89,6 +103,7 @@ type Stage struct { AccessLogSettings *AccessLogSettings `json:"accessLogSettings,omitempty"` DefaultRouteSettings *RouteSettings `json:"defaultRouteSettings,omitempty"` RouteSettings map[string]RouteSettings `json:"routeSettings,omitempty"` + Tags map[string]string `json:"tags,omitempty"` CreatedDate isoTime `json:"createdDate"` LastUpdatedDate isoTime `json:"lastUpdatedDate"` StageVariables map[string]string `json:"stageVariables,omitempty"` @@ -96,7 +111,10 @@ type Stage struct { APIID string `json:"-"` DeploymentID string `json:"deploymentId,omitempty"` Description string `json:"description,omitempty"` - AutoDeploy bool `json:"autoDeploy"` + // ClientCertificateID identifies a client certificate for a Stage. Supported + // only for WebSocket APIs. + ClientCertificateID string `json:"clientCertificateId,omitempty"` + AutoDeploy bool `json:"autoDeploy"` } // Route represents a route in an HTTP API. @@ -117,21 +135,22 @@ type Route struct { // Integration represents a backend integration for a route. type Integration struct { - RequestParameters map[string]string `json:"requestParameters,omitempty"` - RequestTemplates map[string]string `json:"requestTemplates,omitempty"` - IntegrationID string `json:"integrationId"` - APIID string `json:"-"` - IntegrationType string `json:"integrationType"` - IntegrationSubtype string `json:"integrationSubtype,omitempty"` - IntegrationMethod string `json:"integrationMethod,omitempty"` - IntegrationURI string `json:"integrationUri,omitempty"` - Description string `json:"description,omitempty"` - PayloadFormatVersion string `json:"payloadFormatVersion,omitempty"` - ConnectionType string `json:"connectionType,omitempty"` - ConnectionID string `json:"connectionId,omitempty"` - TemplateSelectionExpression string `json:"templateSelectionExpression,omitempty"` - PassthroughBehavior string `json:"passthroughBehavior,omitempty"` - TimeoutInMillis int32 `json:"timeoutInMillis,omitempty"` + TLSConfig *IntegrationTLSConfig `json:"tlsConfig,omitempty"` + RequestParameters map[string]string `json:"requestParameters,omitempty"` + RequestTemplates map[string]string `json:"requestTemplates,omitempty"` + IntegrationID string `json:"integrationId"` + APIID string `json:"-"` + IntegrationType string `json:"integrationType"` + IntegrationSubtype string `json:"integrationSubtype,omitempty"` + IntegrationMethod string `json:"integrationMethod,omitempty"` + IntegrationURI string `json:"integrationUri,omitempty"` + Description string `json:"description,omitempty"` + PayloadFormatVersion string `json:"payloadFormatVersion,omitempty"` + ConnectionType string `json:"connectionType,omitempty"` + ConnectionID string `json:"connectionId,omitempty"` + TemplateSelectionExpression string `json:"templateSelectionExpression,omitempty"` + PassthroughBehavior string `json:"passthroughBehavior,omitempty"` + TimeoutInMillis int32 `json:"timeoutInMillis,omitempty"` } // Deployment represents an API deployment. @@ -195,6 +214,7 @@ type CreateStageInput struct { StageName string `json:"stageName"` DeploymentID string `json:"deploymentId,omitempty"` Description string `json:"description,omitempty"` + ClientCertificateID string `json:"clientCertificateId,omitempty"` AutoDeploy bool `json:"autoDeploy"` } @@ -207,6 +227,7 @@ type UpdateStageInput struct { AutoDeploy *bool `json:"autoDeploy,omitempty"` DeploymentID string `json:"deploymentId,omitempty"` Description string `json:"description,omitempty"` + ClientCertificateID string `json:"clientCertificateId,omitempty"` } // CreateRouteInput is the input for CreateRoute. @@ -239,36 +260,38 @@ type UpdateRouteInput struct { // CreateIntegrationInput is the input for CreateIntegration. type CreateIntegrationInput struct { - RequestParameters map[string]string `json:"requestParameters,omitempty"` - RequestTemplates map[string]string `json:"requestTemplates,omitempty"` - IntegrationType string `json:"integrationType"` - IntegrationSubtype string `json:"integrationSubtype,omitempty"` - IntegrationMethod string `json:"integrationMethod,omitempty"` - IntegrationURI string `json:"integrationUri,omitempty"` - Description string `json:"description,omitempty"` - PayloadFormatVersion string `json:"payloadFormatVersion,omitempty"` - ConnectionType string `json:"connectionType,omitempty"` - ConnectionID string `json:"connectionId,omitempty"` - TemplateSelectionExpression string `json:"templateSelectionExpression,omitempty"` - PassthroughBehavior string `json:"passthroughBehavior,omitempty"` - TimeoutInMillis int32 `json:"timeoutInMillis,omitempty"` + TLSConfig *IntegrationTLSConfig `json:"tlsConfig,omitempty"` + RequestParameters map[string]string `json:"requestParameters,omitempty"` + RequestTemplates map[string]string `json:"requestTemplates,omitempty"` + IntegrationType string `json:"integrationType"` + IntegrationSubtype string `json:"integrationSubtype,omitempty"` + IntegrationMethod string `json:"integrationMethod,omitempty"` + IntegrationURI string `json:"integrationUri,omitempty"` + Description string `json:"description,omitempty"` + PayloadFormatVersion string `json:"payloadFormatVersion,omitempty"` + ConnectionType string `json:"connectionType,omitempty"` + ConnectionID string `json:"connectionId,omitempty"` + TemplateSelectionExpression string `json:"templateSelectionExpression,omitempty"` + PassthroughBehavior string `json:"passthroughBehavior,omitempty"` + TimeoutInMillis int32 `json:"timeoutInMillis,omitempty"` } // UpdateIntegrationInput is the input for UpdateIntegration (PATCH). type UpdateIntegrationInput struct { - RequestParameters map[string]string `json:"requestParameters,omitempty"` - RequestTemplates map[string]string `json:"requestTemplates,omitempty"` - IntegrationType string `json:"integrationType,omitempty"` - IntegrationSubtype string `json:"integrationSubtype,omitempty"` - IntegrationMethod string `json:"integrationMethod,omitempty"` - IntegrationURI string `json:"integrationUri,omitempty"` - Description string `json:"description,omitempty"` - PayloadFormatVersion string `json:"payloadFormatVersion,omitempty"` - ConnectionType string `json:"connectionType,omitempty"` - ConnectionID string `json:"connectionId,omitempty"` - TemplateSelectionExpression string `json:"templateSelectionExpression,omitempty"` - PassthroughBehavior string `json:"passthroughBehavior,omitempty"` - TimeoutInMillis int32 `json:"timeoutInMillis,omitempty"` + TLSConfig *IntegrationTLSConfig `json:"tlsConfig,omitempty"` + RequestParameters map[string]string `json:"requestParameters,omitempty"` + RequestTemplates map[string]string `json:"requestTemplates,omitempty"` + IntegrationType string `json:"integrationType,omitempty"` + IntegrationSubtype string `json:"integrationSubtype,omitempty"` + IntegrationMethod string `json:"integrationMethod,omitempty"` + IntegrationURI string `json:"integrationUri,omitempty"` + Description string `json:"description,omitempty"` + PayloadFormatVersion string `json:"payloadFormatVersion,omitempty"` + ConnectionType string `json:"connectionType,omitempty"` + ConnectionID string `json:"connectionId,omitempty"` + TemplateSelectionExpression string `json:"templateSelectionExpression,omitempty"` + PassthroughBehavior string `json:"passthroughBehavior,omitempty"` + TimeoutInMillis int32 `json:"timeoutInMillis,omitempty"` } // CreateDeploymentInput is the input for CreateDeployment. @@ -317,6 +340,7 @@ type UpdateDeploymentInput struct { // UpdateDomainNameInput is the input for UpdateDomainName (PATCH). type UpdateDomainNameInput struct { + MutualTLSAuthentication *MutualTLSAuthentication `json:"mutualTlsAuthentication,omitempty"` Tags map[string]string `json:"tags,omitempty"` DomainNameConfigurations []DomainNameConfiguration `json:"domainNameConfigurations,omitempty"` } @@ -429,14 +453,17 @@ type DomainNameConfiguration struct { // DomainName represents a custom domain name for API Gateway v2. type DomainName struct { + MutualTLSAuthentication *MutualTLSAuthentication `json:"mutualTlsAuthentication,omitempty"` Tags map[string]string `json:"tags,omitempty"` DomainNameValue string `json:"domainName"` + DomainNameArn string `json:"domainNameArn,omitempty"` APIMappingSelectionExpression string `json:"apiMappingSelectionExpression,omitempty"` DomainNameConfigurations []DomainNameConfiguration `json:"domainNameConfigurations"` } // CreateDomainNameInput is the input for CreateDomainName. type CreateDomainNameInput struct { + MutualTLSAuthentication *MutualTLSAuthentication `json:"mutualTlsAuthentication,omitempty"` Tags map[string]string `json:"tags,omitempty"` DomainNameValue string `json:"domainName"` DomainNameConfigurations []DomainNameConfiguration `json:"domainNameConfigurations,omitempty"` diff --git a/services/apigatewayv2/persistence.go b/services/apigatewayv2/persistence.go index 1fe9156e6..4e4228a98 100644 --- a/services/apigatewayv2/persistence.go +++ b/services/apigatewayv2/persistence.go @@ -2,121 +2,652 @@ package apigatewayv2 import ( "context" + "encoding/json" + "fmt" + "maps" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) -// ensureMap returns m if non-nil, otherwise a new empty map of the same type. -func ensureMap[K comparable, V any](m map[K]V) map[K]V { - if m != nil { - return m +// apigatewayv2SnapshotVersion identifies the shape of backendSnapshot's Tables +// blob (the set of "clean" tables on b.registry plus the DTO tables built +// below for the "dirty" ones -- see store_setup.go's registerAllTables doc +// for the clean/dirty split). Bump whenever a change to a DTO type, a +// registered table's value type, or backendSnapshot itself would make an +// older snapshot unsafe to decode as the current shape. Restore compares this +// against the persisted value and discards (rather than attempts to +// partially decode) any mismatch -- see Restore below. Mirrors the +// services/apigateway pilot (commit 6da0334e). +const apigatewayv2SnapshotVersion = 1 + +// stageSnapshot, routeSnapshot, ..., routingRuleSnapshot are DTOs used ONLY +// for Snapshot/Restore. Each mirrors its live type field for field, except +// the identity field (APIID/DomainName/PortalProductID) is given a real JSON +// tag here instead of the live type's `json:"-"` (see models.go) -- +// marshaling the live type directly would silently drop that field, and +// unmarshaling into it would leave it permanently empty, corrupting the flat +// table's composite key on restore. This is the same DTO-registry technique +// services/apigateway uses for its Resource/Stage/etc types (commit +// 6da0334e). +type stageSnapshot struct { + AccessLogSettings *AccessLogSettings `json:"accessLogSettings,omitempty"` + DefaultRouteSettings *RouteSettings `json:"defaultRouteSettings,omitempty"` + RouteSettings map[string]RouteSettings `json:"routeSettings,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + CreatedDate isoTime `json:"createdDate"` + LastUpdatedDate isoTime `json:"lastUpdatedDate"` + StageVariables map[string]string `json:"stageVariables,omitempty"` + StageName string `json:"stageName"` + APIID string `json:"apiId"` + DeploymentID string `json:"deploymentId,omitempty"` + Description string `json:"description,omitempty"` + ClientCertificateID string `json:"clientCertificateId,omitempty"` + AutoDeploy bool `json:"autoDeploy"` +} + +func stageSnapshotKey(v *stageSnapshot) string { return stageKey(v.APIID, v.StageName) } + +func toStageSnapshot(v *Stage) *stageSnapshot { + return &stageSnapshot{ + AccessLogSettings: v.AccessLogSettings, + DefaultRouteSettings: v.DefaultRouteSettings, + RouteSettings: v.RouteSettings, + Tags: v.Tags, + CreatedDate: v.CreatedDate, + LastUpdatedDate: v.LastUpdatedDate, + StageVariables: v.StageVariables, + StageName: v.StageName, + APIID: v.APIID, + DeploymentID: v.DeploymentID, + Description: v.Description, + ClientCertificateID: v.ClientCertificateID, + AutoDeploy: v.AutoDeploy, } +} - return make(map[K]V) +func fromStageSnapshot(v *stageSnapshot) *Stage { + return &Stage{ + AccessLogSettings: v.AccessLogSettings, + DefaultRouteSettings: v.DefaultRouteSettings, + RouteSettings: v.RouteSettings, + Tags: v.Tags, + CreatedDate: v.CreatedDate, + LastUpdatedDate: v.LastUpdatedDate, + StageVariables: v.StageVariables, + StageName: v.StageName, + APIID: v.APIID, + DeploymentID: v.DeploymentID, + Description: v.Description, + ClientCertificateID: v.ClientCertificateID, + AutoDeploy: v.AutoDeploy, + } } -type apiDataSnapshot struct { - Stages map[string]*Stage `json:"stages"` - Routes map[string]*Route `json:"routes"` - Integrations map[string]*Integration `json:"integrations"` - Deployments map[string]*Deployment `json:"deployments"` - Authorizers map[string]*Authorizer `json:"authorizers"` - IntegrationResponses map[string]map[string]*IntegrationResponse `json:"integrationResponses"` - Models map[string]*Model `json:"models"` - RouteResponses map[string]map[string]*RouteResponse `json:"routeResponses"` - API API `json:"api"` +type routeSnapshot struct { + RequestModels map[string]string `json:"requestModels,omitempty"` + RequestParameters map[string]RouteRequiredParameter `json:"requestParameters,omitempty"` + RouteID string `json:"routeId"` + APIID string `json:"apiId"` + RouteKey string `json:"routeKey"` + Target string `json:"target,omitempty"` + AuthorizationType string `json:"authorizationType,omitempty"` + AuthorizerID string `json:"authorizerId,omitempty"` + OperationName string `json:"operationName,omitempty"` + ModelSelectionExpression string `json:"modelSelectionExpression,omitempty"` + AuthorizationScopes []string `json:"authorizationScopes"` + APIKeyRequired bool `json:"apiKeyRequired"` } -type backendSnapshot struct { - APIs map[string]*apiDataSnapshot `json:"apis"` - DomainNames map[string]*DomainName `json:"domainNames"` - APIMappings map[string]map[string]*APIMapping `json:"apiMappings"` - Portals map[string]*Portal `json:"portals"` - PortalProducts map[string]*PortalProduct `json:"portalProducts"` - ProductPages map[string][]*ProductPage `json:"productPages"` - ProductREPages map[string][]*ProductRestEndpointPage `json:"productREPages"` +func routeSnapshotKey(v *routeSnapshot) string { return routeKey(v.APIID, v.RouteID) } + +func toRouteSnapshot(v *Route) *routeSnapshot { + return &routeSnapshot{ + RequestModels: v.RequestModels, + RequestParameters: v.RequestParameters, + RouteID: v.RouteID, + APIID: v.APIID, + RouteKey: v.RouteKey, + Target: v.Target, + AuthorizationType: v.AuthorizationType, + AuthorizerID: v.AuthorizerID, + OperationName: v.OperationName, + ModelSelectionExpression: v.ModelSelectionExpression, + AuthorizationScopes: v.AuthorizationScopes, + APIKeyRequired: v.APIKeyRequired, + } } -// Snapshot serialises the backend state to JSON. -// It implements persistence.Persistable. -func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { - b.mu.RLock("Snapshot") - defer b.mu.RUnlock() +func fromRouteSnapshot(v *routeSnapshot) *Route { + return &Route{ + RequestModels: v.RequestModels, + RequestParameters: v.RequestParameters, + RouteID: v.RouteID, + APIID: v.APIID, + RouteKey: v.RouteKey, + Target: v.Target, + AuthorizationType: v.AuthorizationType, + AuthorizerID: v.AuthorizerID, + OperationName: v.OperationName, + ModelSelectionExpression: v.ModelSelectionExpression, + AuthorizationScopes: v.AuthorizationScopes, + APIKeyRequired: v.APIKeyRequired, + } +} - snap := backendSnapshot{ - APIs: make(map[string]*apiDataSnapshot, len(b.apis)), - DomainNames: b.domainNames, - APIMappings: b.apiMappings, - Portals: b.portals, - PortalProducts: b.portalProducts, - ProductPages: b.productPages, - ProductREPages: b.productREPages, - } - - for id, d := range b.apis { - snap.APIs[id] = &apiDataSnapshot{ - API: d.api, - Stages: d.stages, - Routes: d.routes, - Integrations: d.integrations, - Deployments: d.deployments, - Authorizers: d.authorizers, - IntegrationResponses: d.integrationResponses, - Models: d.models, - RouteResponses: d.routeResponses, - } +type integrationSnapshot struct { + TLSConfig *IntegrationTLSConfig `json:"tlsConfig,omitempty"` + RequestParameters map[string]string `json:"requestParameters,omitempty"` + RequestTemplates map[string]string `json:"requestTemplates,omitempty"` + IntegrationID string `json:"integrationId"` + APIID string `json:"apiId"` + IntegrationType string `json:"integrationType"` + IntegrationSubtype string `json:"integrationSubtype,omitempty"` + IntegrationMethod string `json:"integrationMethod,omitempty"` + IntegrationURI string `json:"integrationUri,omitempty"` + Description string `json:"description,omitempty"` + PayloadFormatVersion string `json:"payloadFormatVersion,omitempty"` + ConnectionType string `json:"connectionType,omitempty"` + ConnectionID string `json:"connectionId,omitempty"` + TemplateSelectionExpression string `json:"templateSelectionExpression,omitempty"` + PassthroughBehavior string `json:"passthroughBehavior,omitempty"` + TimeoutInMillis int32 `json:"timeoutInMillis,omitempty"` +} + +func integrationSnapshotKey(v *integrationSnapshot) string { + return integrationKey(v.APIID, v.IntegrationID) +} + +func toIntegrationSnapshot(v *Integration) *integrationSnapshot { + return &integrationSnapshot{ + TLSConfig: v.TLSConfig, + RequestParameters: v.RequestParameters, + RequestTemplates: v.RequestTemplates, + IntegrationID: v.IntegrationID, + APIID: v.APIID, + IntegrationType: v.IntegrationType, + IntegrationSubtype: v.IntegrationSubtype, + IntegrationMethod: v.IntegrationMethod, + IntegrationURI: v.IntegrationURI, + Description: v.Description, + PayloadFormatVersion: v.PayloadFormatVersion, + ConnectionType: v.ConnectionType, + ConnectionID: v.ConnectionID, + TemplateSelectionExpression: v.TemplateSelectionExpression, + PassthroughBehavior: v.PassthroughBehavior, + TimeoutInMillis: v.TimeoutInMillis, } +} - return persistence.MarshalSnapshot(ctx, "apigatewayv2", snap) +func fromIntegrationSnapshot(v *integrationSnapshot) *Integration { + return &Integration{ + TLSConfig: v.TLSConfig, + RequestParameters: v.RequestParameters, + RequestTemplates: v.RequestTemplates, + IntegrationID: v.IntegrationID, + APIID: v.APIID, + IntegrationType: v.IntegrationType, + IntegrationSubtype: v.IntegrationSubtype, + IntegrationMethod: v.IntegrationMethod, + IntegrationURI: v.IntegrationURI, + Description: v.Description, + PayloadFormatVersion: v.PayloadFormatVersion, + ConnectionType: v.ConnectionType, + ConnectionID: v.ConnectionID, + TemplateSelectionExpression: v.TemplateSelectionExpression, + PassthroughBehavior: v.PassthroughBehavior, + TimeoutInMillis: v.TimeoutInMillis, + } } -// restoreAPIData converts a snapshot API entry into a live apiData, initialising any nil maps. -func restoreAPIData(d *apiDataSnapshot) *apiData { - if d.Stages == nil { - d.Stages = make(map[string]*Stage) +type deploymentSnapshot struct { + CreatedDate isoTime `json:"createdDate"` + DeploymentID string `json:"deploymentId"` + APIID string `json:"apiId"` + Description string `json:"description,omitempty"` + DeploymentStatus string `json:"deploymentStatus"` + AutoDeployed bool `json:"autoDeployed"` +} + +func deploymentSnapshotKey(v *deploymentSnapshot) string { + return deploymentKey(v.APIID, v.DeploymentID) +} + +func toDeploymentSnapshot(v *Deployment) *deploymentSnapshot { + return &deploymentSnapshot{ + CreatedDate: v.CreatedDate, + DeploymentID: v.DeploymentID, + APIID: v.APIID, + Description: v.Description, + DeploymentStatus: v.DeploymentStatus, + AutoDeployed: v.AutoDeployed, } +} - if d.Routes == nil { - d.Routes = make(map[string]*Route) +func fromDeploymentSnapshot(v *deploymentSnapshot) *Deployment { + return &Deployment{ + CreatedDate: v.CreatedDate, + DeploymentID: v.DeploymentID, + APIID: v.APIID, + Description: v.Description, + DeploymentStatus: v.DeploymentStatus, + AutoDeployed: v.AutoDeployed, } +} + +type authorizerSnapshot struct { + JwtConfiguration *JwtConfiguration `json:"jwtConfiguration,omitempty"` + AuthorizerID string `json:"authorizerId"` + APIID string `json:"apiId"` + Name string `json:"name"` + AuthorizerType string `json:"authorizerType"` + AuthorizerURI string `json:"authorizerUri,omitempty"` + AuthorizerCredentialsArn string `json:"authorizerCredentialsArn,omitempty"` + AuthorizerPayloadFormatVersion string `json:"authorizerPayloadFormatVersion,omitempty"` + IdentitySource []string `json:"identitySource,omitempty"` + AuthorizerResultTTLInSeconds int32 `json:"authorizerResultTtlInSeconds,omitempty"` + EnableSimpleResponses bool `json:"enableSimpleResponses,omitempty"` +} - if d.Integrations == nil { - d.Integrations = make(map[string]*Integration) +func authorizerSnapshotKey(v *authorizerSnapshot) string { + return authorizerKey(v.APIID, v.AuthorizerID) +} + +func toAuthorizerSnapshot(v *Authorizer) *authorizerSnapshot { + return &authorizerSnapshot{ + JwtConfiguration: v.JwtConfiguration, + AuthorizerID: v.AuthorizerID, + APIID: v.APIID, + Name: v.Name, + AuthorizerType: v.AuthorizerType, + AuthorizerURI: v.AuthorizerURI, + AuthorizerCredentialsArn: v.AuthorizerCredentialsArn, + AuthorizerPayloadFormatVersion: v.AuthorizerPayloadFormatVersion, + IdentitySource: v.IdentitySource, + AuthorizerResultTTLInSeconds: v.AuthorizerResultTTLInSeconds, + EnableSimpleResponses: v.EnableSimpleResponses, } +} - if d.Deployments == nil { - d.Deployments = make(map[string]*Deployment) +func fromAuthorizerSnapshot(v *authorizerSnapshot) *Authorizer { + return &Authorizer{ + JwtConfiguration: v.JwtConfiguration, + AuthorizerID: v.AuthorizerID, + APIID: v.APIID, + Name: v.Name, + AuthorizerType: v.AuthorizerType, + AuthorizerURI: v.AuthorizerURI, + AuthorizerCredentialsArn: v.AuthorizerCredentialsArn, + AuthorizerPayloadFormatVersion: v.AuthorizerPayloadFormatVersion, + IdentitySource: v.IdentitySource, + AuthorizerResultTTLInSeconds: v.AuthorizerResultTTLInSeconds, + EnableSimpleResponses: v.EnableSimpleResponses, } +} + +type modelSnapshot struct { + ModelID string `json:"modelId"` + APIID string `json:"apiId"` + Name string `json:"name"` + Schema string `json:"schema,omitempty"` + ContentType string `json:"contentType,omitempty"` + Description string `json:"description,omitempty"` +} + +func modelSnapshotKey(v *modelSnapshot) string { return modelKey(v.APIID, v.ModelID) } - if d.Authorizers == nil { - d.Authorizers = make(map[string]*Authorizer) +func toModelSnapshot(v *Model) *modelSnapshot { + return &modelSnapshot{ + ModelID: v.ModelID, + APIID: v.APIID, + Name: v.Name, + Schema: v.Schema, + ContentType: v.ContentType, + Description: v.Description, } +} - if d.IntegrationResponses == nil { - d.IntegrationResponses = make(map[string]map[string]*IntegrationResponse) +func fromModelSnapshot(v *modelSnapshot) *Model { + return &Model{ + ModelID: v.ModelID, + APIID: v.APIID, + Name: v.Name, + Schema: v.Schema, + ContentType: v.ContentType, + Description: v.Description, } +} - if d.Models == nil { - d.Models = make(map[string]*Model) +type integrationResponseSnapshot struct { + ResponseParameters map[string]string `json:"responseParameters,omitempty"` + ResponseTemplates map[string]string `json:"responseTemplates,omitempty"` + IntegrationResponseID string `json:"integrationResponseId"` + IntegrationResponseKey string `json:"integrationResponseKey"` + APIID string `json:"apiId"` + IntegrationID string `json:"integrationId"` + ContentHandlingStrategy string `json:"contentHandlingStrategy,omitempty"` + TemplateSelectionExpression string `json:"templateSelectionExpression,omitempty"` +} + +func integrationResponseSnapshotKey(v *integrationResponseSnapshot) string { + return integrationResponseKey(v.APIID, v.IntegrationID, v.IntegrationResponseID) +} + +func toIntegrationResponseSnapshot(v *IntegrationResponse) *integrationResponseSnapshot { + return &integrationResponseSnapshot{ + ResponseParameters: v.ResponseParameters, + ResponseTemplates: v.ResponseTemplates, + IntegrationResponseID: v.IntegrationResponseID, + IntegrationResponseKey: v.IntegrationResponseKey, + APIID: v.APIID, + IntegrationID: v.IntegrationID, + ContentHandlingStrategy: v.ContentHandlingStrategy, + TemplateSelectionExpression: v.TemplateSelectionExpression, } +} - if d.RouteResponses == nil { - d.RouteResponses = make(map[string]map[string]*RouteResponse) +func fromIntegrationResponseSnapshot(v *integrationResponseSnapshot) *IntegrationResponse { + return &IntegrationResponse{ + ResponseParameters: v.ResponseParameters, + ResponseTemplates: v.ResponseTemplates, + IntegrationResponseID: v.IntegrationResponseID, + IntegrationResponseKey: v.IntegrationResponseKey, + APIID: v.APIID, + IntegrationID: v.IntegrationID, + ContentHandlingStrategy: v.ContentHandlingStrategy, + TemplateSelectionExpression: v.TemplateSelectionExpression, } +} - return &apiData{ - api: d.API, - stages: d.Stages, - routes: d.Routes, - integrations: d.Integrations, - deployments: d.Deployments, - authorizers: d.Authorizers, - integrationResponses: d.IntegrationResponses, - models: d.Models, - routeResponses: d.RouteResponses, +type routeResponseSnapshot struct { + ResponseModels map[string]string `json:"responseModels,omitempty"` + RouteResponseID string `json:"routeResponseId"` + RouteResponseKey string `json:"routeResponseKey"` + APIID string `json:"apiId"` + RouteID string `json:"routeId"` + ModelSelectionExpression string `json:"modelSelectionExpression,omitempty"` +} + +func routeResponseSnapshotKey(v *routeResponseSnapshot) string { + return routeResponseKey(v.APIID, v.RouteID, v.RouteResponseID) +} + +func toRouteResponseSnapshot(v *RouteResponse) *routeResponseSnapshot { + return &routeResponseSnapshot{ + ResponseModels: v.ResponseModels, + RouteResponseID: v.RouteResponseID, + RouteResponseKey: v.RouteResponseKey, + APIID: v.APIID, + RouteID: v.RouteID, + ModelSelectionExpression: v.ModelSelectionExpression, } } +func fromRouteResponseSnapshot(v *routeResponseSnapshot) *RouteResponse { + return &RouteResponse{ + ResponseModels: v.ResponseModels, + RouteResponseID: v.RouteResponseID, + RouteResponseKey: v.RouteResponseKey, + APIID: v.APIID, + RouteID: v.RouteID, + ModelSelectionExpression: v.ModelSelectionExpression, + } +} + +type apiMappingSnapshot struct { + APIID string `json:"apiId"` + APIMappingID string `json:"apiMappingId"` + DomainName string `json:"domainName"` + Stage string `json:"stage"` + APIMappingKey string `json:"apiMappingKey,omitempty"` +} + +func apiMappingSnapshotKey(v *apiMappingSnapshot) string { + return apiMappingKey(v.DomainName, v.APIMappingID) +} + +func toAPIMappingSnapshot(v *APIMapping) *apiMappingSnapshot { + return &apiMappingSnapshot{ + APIID: v.APIID, + APIMappingID: v.APIMappingID, + DomainName: v.DomainName, + Stage: v.Stage, + APIMappingKey: v.APIMappingKey, + } +} + +func fromAPIMappingSnapshot(v *apiMappingSnapshot) *APIMapping { + return &APIMapping{ + APIID: v.APIID, + APIMappingID: v.APIMappingID, + DomainName: v.DomainName, + Stage: v.Stage, + APIMappingKey: v.APIMappingKey, + } +} + +type productPageSnapshot struct { + LastModified *isoTime `json:"lastModified,omitempty"` + DisplayContent map[string]any `json:"displayContent,omitempty"` + ProductPageID string `json:"productPageId"` + PortalProductID string `json:"portalProductId"` +} + +func productPageSnapshotKey(v *productPageSnapshot) string { + return productPageKey(v.PortalProductID, v.ProductPageID) +} + +func toProductPageSnapshot(v *ProductPage) *productPageSnapshot { + return &productPageSnapshot{ + LastModified: v.LastModified, + DisplayContent: v.DisplayContent, + ProductPageID: v.ProductPageID, + PortalProductID: v.PortalProductID, + } +} + +func fromProductPageSnapshot(v *productPageSnapshot) *ProductPage { + return &ProductPage{ + LastModified: v.LastModified, + DisplayContent: v.DisplayContent, + ProductPageID: v.ProductPageID, + PortalProductID: v.PortalProductID, + } +} + +type productREPageSnapshot struct { + LastModified *isoTime `json:"lastModified,omitempty"` + DisplayContent map[string]any `json:"displayContent,omitempty"` + ProductRestEndpointPageID string `json:"productRestEndpointPageId"` + PortalProductID string `json:"portalProductId"` +} + +func productREPageSnapshotKey(v *productREPageSnapshot) string { + return productREPageKey(v.PortalProductID, v.ProductRestEndpointPageID) +} + +func toProductREPageSnapshot(v *ProductRestEndpointPage) *productREPageSnapshot { + return &productREPageSnapshot{ + LastModified: v.LastModified, + DisplayContent: v.DisplayContent, + ProductRestEndpointPageID: v.ProductRestEndpointPageID, + PortalProductID: v.PortalProductID, + } +} + +func fromProductREPageSnapshot(v *productREPageSnapshot) *ProductRestEndpointPage { + return &ProductRestEndpointPage{ + LastModified: v.LastModified, + DisplayContent: v.DisplayContent, + ProductRestEndpointPageID: v.ProductRestEndpointPageID, + PortalProductID: v.PortalProductID, + } +} + +type routingRuleSnapshot struct { + DomainName string `json:"domainName"` + RoutingRuleID string `json:"routingRuleId"` + RoutingRuleARN string `json:"routingRuleArn,omitempty"` + Actions []map[string]any `json:"actions,omitempty"` + Conditions []map[string]any `json:"conditions,omitempty"` + Priority int32 `json:"priority"` +} + +func routingRuleSnapshotKey(v *routingRuleSnapshot) string { + return routingRuleKey(v.DomainName, v.RoutingRuleID) +} + +func toRoutingRuleSnapshot(v *RoutingRule) *routingRuleSnapshot { + return &routingRuleSnapshot{ + DomainName: v.DomainName, + RoutingRuleID: v.RoutingRuleID, + RoutingRuleARN: v.RoutingRuleARN, + Actions: v.Actions, + Conditions: v.Conditions, + Priority: v.Priority, + } +} + +func fromRoutingRuleSnapshot(v *routingRuleSnapshot) *RoutingRule { + return &RoutingRule{ + DomainName: v.DomainName, + RoutingRuleID: v.RoutingRuleID, + RoutingRuleARN: v.RoutingRuleARN, + Actions: v.Actions, + Conditions: v.Conditions, + Priority: v.Priority, + } +} + +// dirtyTableNames lists the "dirty" table names shared by Snapshot and +// Restore (see store_setup.go's registerAllTables doc). Both build an +// ephemeral DTO [store.Registry] under these exact names, so a snapshot +// written by one version of Snapshot always lines up with the same version +// of Restore. +// +//nolint:gochecknoglobals // fixed lookup table, mirrors errCodeLookup-style tables elsewhere +var dirtyTableNames = struct { + stages, routes, integrations, deployments, authorizers, models, + integrationResponses, routeResponses, apiMappings, + productPages, productREPages, routingRules string +}{ + stages: "stages", + routes: "routes", + integrations: "integrations", + deployments: "deployments", + authorizers: "authorizers", + models: "models", + integrationResponses: "integrationResponses", + routeResponses: "routeResponses", + apiMappings: "apiMappings", + productPages: "productPages", + productREPages: "productREPages", + routingRules: "routingRules", +} + +// backendSnapshot is the top-level on-disk shape for the API Gateway v2 +// backend. +// +// Tables holds one JSON-encoded array per table -- both the "clean" tables +// registered on b.registry (produced by [store.Registry.SnapshotAll]) and the +// "dirty" DTO tables built inline in Snapshot (see store_setup.go's +// registerAllTables doc for the clean/dirty split), merged into one map so a +// single Tables blob round-trips the whole backend. +type backendSnapshot struct { + Tables map[string]json.RawMessage `json:"tables"` + PortalProductSharingPolicies map[string]string `json:"portalProductSharingPolicies,omitempty"` + Version int `json:"version"` +} + +// Snapshot serialises the backend state to JSON. +// It implements persistence.Persistable. +func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { + b.mu.RLock("Snapshot") + defer b.mu.RUnlock() + + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "apigatewayv2: snapshot table marshal failed", "error", err) + + return nil + } + + dtoReg := store.NewRegistry() + stageDTOs := store.Register(dtoReg, dirtyTableNames.stages, store.New(stageSnapshotKey)) + routeDTOs := store.Register(dtoReg, dirtyTableNames.routes, store.New(routeSnapshotKey)) + integrationDTOs := store.Register(dtoReg, dirtyTableNames.integrations, store.New(integrationSnapshotKey)) + deploymentDTOs := store.Register(dtoReg, dirtyTableNames.deployments, store.New(deploymentSnapshotKey)) + authorizerDTOs := store.Register(dtoReg, dirtyTableNames.authorizers, store.New(authorizerSnapshotKey)) + modelDTOs := store.Register(dtoReg, dirtyTableNames.models, store.New(modelSnapshotKey)) + integrationResponseDTOs := store.Register( + dtoReg, dirtyTableNames.integrationResponses, store.New(integrationResponseSnapshotKey), + ) + routeResponseDTOs := store.Register(dtoReg, dirtyTableNames.routeResponses, store.New(routeResponseSnapshotKey)) + apiMappingDTOs := store.Register(dtoReg, dirtyTableNames.apiMappings, store.New(apiMappingSnapshotKey)) + productPageDTOs := store.Register(dtoReg, dirtyTableNames.productPages, store.New(productPageSnapshotKey)) + productREPageDTOs := store.Register(dtoReg, dirtyTableNames.productREPages, store.New(productREPageSnapshotKey)) + routingRuleDTOs := store.Register(dtoReg, dirtyTableNames.routingRules, store.New(routingRuleSnapshotKey)) + + for _, v := range b.stages.Snapshot() { + stageDTOs.Put(toStageSnapshot(v)) + } + for _, v := range b.routes.Snapshot() { + routeDTOs.Put(toRouteSnapshot(v)) + } + for _, v := range b.integrations.Snapshot() { + integrationDTOs.Put(toIntegrationSnapshot(v)) + } + for _, v := range b.deployments.Snapshot() { + deploymentDTOs.Put(toDeploymentSnapshot(v)) + } + for _, v := range b.authorizers.Snapshot() { + authorizerDTOs.Put(toAuthorizerSnapshot(v)) + } + for _, v := range b.models.Snapshot() { + modelDTOs.Put(toModelSnapshot(v)) + } + for _, v := range b.integrationResponses.Snapshot() { + integrationResponseDTOs.Put(toIntegrationResponseSnapshot(v)) + } + for _, v := range b.routeResponses.Snapshot() { + routeResponseDTOs.Put(toRouteResponseSnapshot(v)) + } + for _, v := range b.apiMappings.Snapshot() { + apiMappingDTOs.Put(toAPIMappingSnapshot(v)) + } + for _, v := range b.productPages.Snapshot() { + productPageDTOs.Put(toProductPageSnapshot(v)) + } + for _, v := range b.productREPages.Snapshot() { + productREPageDTOs.Put(toProductREPageSnapshot(v)) + } + for _, v := range b.routingRules.Snapshot() { + routingRuleDTOs.Put(toRoutingRuleSnapshot(v)) + } + + dirtyTables, err := dtoReg.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, "apigatewayv2: snapshot DTO table marshal failed", "error", err) + + return nil + } + + maps.Copy(tables, dirtyTables) + + snap := backendSnapshot{ + Version: apigatewayv2SnapshotVersion, + Tables: tables, + PortalProductSharingPolicies: b.portalProductSharingPolicies, + } + + return persistence.MarshalSnapshot(ctx, "apigatewayv2", snap) +} + // Restore loads backend state from a JSON snapshot. // It implements persistence.Persistable. func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { @@ -129,22 +660,162 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() - b.apis = make(map[string]*apiData, len(snap.APIs)) + if snap.Version != apigatewayv2SnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. + logger.Load(ctx).WarnContext(ctx, + "apigatewayv2: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", apigatewayv2SnapshotVersion) + + b.registry.ResetAll() + b.stages.Reset() + b.routes.Reset() + b.integrations.Reset() + b.deployments.Reset() + b.authorizers.Reset() + b.models.Reset() + b.integrationResponses.Reset() + b.routeResponses.Reset() + b.apiMappings.Reset() + b.productPages.Reset() + b.productREPages.Reset() + b.routingRules.Reset() + b.portalProductSharingPolicies = make(map[string]string) + + return nil + } - for id, d := range snap.APIs { - b.apis[id] = restoreAPIData(d) + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("apigatewayv2: restore snapshot tables: %w", err) } - b.domainNames = ensureMap(snap.DomainNames) - b.apiMappings = ensureMap(snap.APIMappings) - b.portals = ensureMap(snap.Portals) - b.portalProducts = ensureMap(snap.PortalProducts) - b.productPages = ensureMap(snap.ProductPages) - b.productREPages = ensureMap(snap.ProductREPages) + dtoReg := store.NewRegistry() + stageDTOs := store.Register(dtoReg, dirtyTableNames.stages, store.New(stageSnapshotKey)) + routeDTOs := store.Register(dtoReg, dirtyTableNames.routes, store.New(routeSnapshotKey)) + integrationDTOs := store.Register(dtoReg, dirtyTableNames.integrations, store.New(integrationSnapshotKey)) + deploymentDTOs := store.Register(dtoReg, dirtyTableNames.deployments, store.New(deploymentSnapshotKey)) + authorizerDTOs := store.Register(dtoReg, dirtyTableNames.authorizers, store.New(authorizerSnapshotKey)) + modelDTOs := store.Register(dtoReg, dirtyTableNames.models, store.New(modelSnapshotKey)) + integrationResponseDTOs := store.Register( + dtoReg, dirtyTableNames.integrationResponses, store.New(integrationResponseSnapshotKey), + ) + routeResponseDTOs := store.Register(dtoReg, dirtyTableNames.routeResponses, store.New(routeResponseSnapshotKey)) + apiMappingDTOs := store.Register(dtoReg, dirtyTableNames.apiMappings, store.New(apiMappingSnapshotKey)) + productPageDTOs := store.Register(dtoReg, dirtyTableNames.productPages, store.New(productPageSnapshotKey)) + productREPageDTOs := store.Register(dtoReg, dirtyTableNames.productREPages, store.New(productREPageSnapshotKey)) + routingRuleDTOs := store.Register(dtoReg, dirtyTableNames.routingRules, store.New(routingRuleSnapshotKey)) + + if err := dtoReg.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("apigatewayv2: restore snapshot DTO tables: %w", err) + } + + restoreDirtyTables(b, stageDTOs, routeDTOs, integrationDTOs, deploymentDTOs, authorizerDTOs, modelDTOs, + integrationResponseDTOs, routeResponseDTOs, apiMappingDTOs, productPageDTOs, productREPageDTOs, routingRuleDTOs) + + if snap.PortalProductSharingPolicies != nil { + b.portalProductSharingPolicies = snap.PortalProductSharingPolicies + } else { + b.portalProductSharingPolicies = make(map[string]string) + } return nil } +// restoreDirtyTables converts each DTO table's contents back to its live +// value type and loads the corresponding live b.
via Table.Restore, +// split out from Restore to keep its growth in check. +func restoreDirtyTables( + b *InMemoryBackend, + stageDTOs *store.Table[stageSnapshot], + routeDTOs *store.Table[routeSnapshot], + integrationDTOs *store.Table[integrationSnapshot], + deploymentDTOs *store.Table[deploymentSnapshot], + authorizerDTOs *store.Table[authorizerSnapshot], + modelDTOs *store.Table[modelSnapshot], + integrationResponseDTOs *store.Table[integrationResponseSnapshot], + routeResponseDTOs *store.Table[routeResponseSnapshot], + apiMappingDTOs *store.Table[apiMappingSnapshot], + productPageDTOs *store.Table[productPageSnapshot], + productREPageDTOs *store.Table[productREPageSnapshot], + routingRuleDTOs *store.Table[routingRuleSnapshot], +) { + stages := make([]*Stage, 0, stageDTOs.Len()) + for _, v := range stageDTOs.All() { + stages = append(stages, fromStageSnapshot(v)) + } + b.stages.Restore(stages) + + routes := make([]*Route, 0, routeDTOs.Len()) + for _, v := range routeDTOs.All() { + routes = append(routes, fromRouteSnapshot(v)) + } + b.routes.Restore(routes) + + integrations := make([]*Integration, 0, integrationDTOs.Len()) + for _, v := range integrationDTOs.All() { + integrations = append(integrations, fromIntegrationSnapshot(v)) + } + b.integrations.Restore(integrations) + + deployments := make([]*Deployment, 0, deploymentDTOs.Len()) + for _, v := range deploymentDTOs.All() { + deployments = append(deployments, fromDeploymentSnapshot(v)) + } + b.deployments.Restore(deployments) + + authorizers := make([]*Authorizer, 0, authorizerDTOs.Len()) + for _, v := range authorizerDTOs.All() { + authorizers = append(authorizers, fromAuthorizerSnapshot(v)) + } + b.authorizers.Restore(authorizers) + + models := make([]*Model, 0, modelDTOs.Len()) + for _, v := range modelDTOs.All() { + models = append(models, fromModelSnapshot(v)) + } + b.models.Restore(models) + + integrationResponses := make([]*IntegrationResponse, 0, integrationResponseDTOs.Len()) + for _, v := range integrationResponseDTOs.All() { + integrationResponses = append(integrationResponses, fromIntegrationResponseSnapshot(v)) + } + b.integrationResponses.Restore(integrationResponses) + + routeResponses := make([]*RouteResponse, 0, routeResponseDTOs.Len()) + for _, v := range routeResponseDTOs.All() { + routeResponses = append(routeResponses, fromRouteResponseSnapshot(v)) + } + b.routeResponses.Restore(routeResponses) + + apiMappings := make([]*APIMapping, 0, apiMappingDTOs.Len()) + for _, v := range apiMappingDTOs.All() { + apiMappings = append(apiMappings, fromAPIMappingSnapshot(v)) + } + b.apiMappings.Restore(apiMappings) + + productPages := make([]*ProductPage, 0, productPageDTOs.Len()) + for _, v := range productPageDTOs.All() { + productPages = append(productPages, fromProductPageSnapshot(v)) + } + b.productPages.Restore(productPages) + + productREPages := make([]*ProductRestEndpointPage, 0, productREPageDTOs.Len()) + for _, v := range productREPageDTOs.All() { + productREPages = append(productREPages, fromProductREPageSnapshot(v)) + } + b.productREPages.Restore(productREPages) + + routingRules := make([]*RoutingRule, 0, routingRuleDTOs.Len()) + for _, v := range routingRuleDTOs.All() { + routingRules = append(routingRules, fromRoutingRuleSnapshot(v)) + } + b.routingRules.Restore(routingRules) +} + // Snapshot implements persistence.Persistable by delegating to the backend. func (h *Handler) Snapshot(ctx context.Context) []byte { type snapshotter interface { diff --git a/services/apigatewayv2/persistence_full_test.go b/services/apigatewayv2/persistence_full_test.go new file mode 100644 index 000000000..e97747073 --- /dev/null +++ b/services/apigatewayv2/persistence_full_test.go @@ -0,0 +1,204 @@ +package apigatewayv2_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/apigatewayv2" +) + +// TestInMemoryBackend_SnapshotRestore_FullState exercises every resource +// family touched by the Phase 3.3 pkgs/store conversion, in particular the +// ones whose composite key (apiID#childID, domainName#id, +// portalProductID#pageID) depends on a field tagged json:"-" on the live type +// (Stage, Route, Integration, Deployment, Authorizer, Model, +// IntegrationResponse, RouteResponse, APIMapping, ProductPage, +// ProductRestEndpointPage, RoutingRule -- see store_setup.go and +// persistence.go). If the DTO-registry snapshot/restore path in +// persistence.go ever dropped that field, every one of these lookups would +// fail post-restore even though the resource "exists" in the restored table +// under the wrong (unprefixed) key. This is a single sequential lifecycle +// (create everything once, snapshot, restore, verify everything), not a +// table of independent cases, so it does not need t.Run subtests. +func TestInMemoryBackend_SnapshotRestore_FullState(t *testing.T) { + t.Parallel() + + b := apigatewayv2.NewInMemoryBackend() + ctx := t.Context() + + api, err := b.CreateAPI(ctx, apigatewayv2.CreateAPIInput{Name: "full-api", ProtocolType: "HTTP"}) + require.NoError(t, err) + + stage, err := b.CreateStage(api.APIID, apigatewayv2.CreateStageInput{StageName: "prod"}) + require.NoError(t, err) + + route, err := b.CreateRoute(api.APIID, apigatewayv2.CreateRouteInput{RouteKey: "GET /widgets"}) + require.NoError(t, err) + + integration, err := b.CreateIntegration(api.APIID, apigatewayv2.CreateIntegrationInput{ + IntegrationType: "AWS_PROXY", + IntegrationURI: "arn:aws:lambda:us-east-1:1:function:f", + }) + require.NoError(t, err) + + deployment, err := b.CreateDeployment(api.APIID, apigatewayv2.CreateDeploymentInput{Description: "initial"}) + require.NoError(t, err) + + authorizer, err := b.CreateAuthorizer(api.APIID, apigatewayv2.CreateAuthorizerInput{ + Name: "auth1", AuthorizerType: "REQUEST", AuthorizerURI: "arn:aws:lambda:us-east-1:1:function:auth", + }) + require.NoError(t, err) + + model, err := b.CreateModel(api.APIID, apigatewayv2.CreateModelInput{ + Name: "Widget", ContentType: "application/json", Schema: `{"type":"object"}`, + }) + require.NoError(t, err) + + integrationResponse, err := b.CreateIntegrationResponse( + api.APIID, integration.IntegrationID, + apigatewayv2.CreateIntegrationResponseInput{IntegrationResponseKey: "/200/"}, + ) + require.NoError(t, err) + + routeResponse, err := b.CreateRouteResponse( + api.APIID, route.RouteID, + apigatewayv2.CreateRouteResponseInput{RouteResponseKey: "$default"}, + ) + require.NoError(t, err) + + domainName, err := b.CreateDomainName(ctx, apigatewayv2.CreateDomainNameInput{DomainNameValue: "api.example.com"}) + require.NoError(t, err) + + apiMapping, err := b.CreateAPIMapping(domainName.DomainNameValue, apigatewayv2.CreateAPIMappingInput{ + APIID: api.APIID, Stage: stage.StageName, + }) + require.NoError(t, err) + + portal, err := b.CreatePortal(apigatewayv2.CreatePortalInput{LogoURI: "https://example.com/logo.png"}) + require.NoError(t, err) + + portalProduct, err := b.CreatePortalProduct(apigatewayv2.CreatePortalProductInput{DisplayName: "product1"}) + require.NoError(t, err) + + productPage, err := b.CreateProductPage(portalProduct.PortalProductID, apigatewayv2.CreateProductPageInput{}) + require.NoError(t, err) + + productREPage, err := b.CreateProductRestEndpointPage( + portalProduct.PortalProductID, apigatewayv2.CreateProductRestEndpointPageInput{}, + ) + require.NoError(t, err) + + sharingPolicy, err := b.PutPortalProductSharingPolicy(portalProduct.PortalProductID, `{"policy":"doc"}`) + require.NoError(t, err) + + vpcLink, err := b.CreateVpcLink(apigatewayv2.CreateVpcLinkInput{Name: "link1", SubnetIDs: []string{"subnet-1"}}) + require.NoError(t, err) + + routingRule, err := b.CreateRoutingRule( + ctx, domainName.DomainNameValue, apigatewayv2.CreateRoutingRuleInput{Priority: 1}, + ) + require.NoError(t, err) + + snap := b.Snapshot(ctx) + require.NotNil(t, snap) + + fresh := apigatewayv2.NewInMemoryBackend() + require.NoError(t, fresh.Restore(ctx, snap)) + + gotAPI, err := fresh.GetAPI(api.APIID) + require.NoError(t, err) + assert.Equal(t, api.Name, gotAPI.Name) + + gotStage, err := fresh.GetStage(api.APIID, stage.StageName) + require.NoError(t, err) + assert.Equal(t, stage.StageName, gotStage.StageName) + + gotRoute, err := fresh.GetRoute(api.APIID, route.RouteID) + require.NoError(t, err) + assert.Equal(t, route.RouteKey, gotRoute.RouteKey) + + gotIntegration, err := fresh.GetIntegration(api.APIID, integration.IntegrationID) + require.NoError(t, err) + assert.Equal(t, integration.IntegrationURI, gotIntegration.IntegrationURI) + + gotDeployment, err := fresh.GetDeployment(api.APIID, deployment.DeploymentID) + require.NoError(t, err) + assert.Equal(t, deployment.Description, gotDeployment.Description) + + gotAuthorizer, err := fresh.GetAuthorizer(api.APIID, authorizer.AuthorizerID) + require.NoError(t, err) + assert.Equal(t, authorizer.Name, gotAuthorizer.Name) + + gotModel, err := fresh.GetModel(api.APIID, model.ModelID) + require.NoError(t, err) + assert.Equal(t, model.Schema, gotModel.Schema) + + gotIntegrationResponse, err := fresh.GetIntegrationResponse( + api.APIID, integration.IntegrationID, integrationResponse.IntegrationResponseID, + ) + require.NoError(t, err) + assert.Equal(t, integrationResponse.IntegrationResponseKey, gotIntegrationResponse.IntegrationResponseKey) + + gotRouteResponse, err := fresh.GetRouteResponse(api.APIID, route.RouteID, routeResponse.RouteResponseID) + require.NoError(t, err) + assert.Equal(t, routeResponse.RouteResponseKey, gotRouteResponse.RouteResponseKey) + + gotDomainName, err := fresh.GetDomainName(domainName.DomainNameValue) + require.NoError(t, err) + assert.Equal(t, domainName.DomainNameValue, gotDomainName.DomainNameValue) + + gotAPIMapping, err := fresh.GetAPIMapping(domainName.DomainNameValue, apiMapping.APIMappingID) + require.NoError(t, err) + assert.Equal(t, apiMapping.Stage, gotAPIMapping.Stage) + + gotPortal, err := fresh.GetPortal(portal.PortalID) + require.NoError(t, err) + assert.Equal(t, portal.LogoURI, gotPortal.LogoURI) + + gotPortalProduct, err := fresh.GetPortalProduct(portalProduct.PortalProductID) + require.NoError(t, err) + assert.Equal(t, portalProduct.DisplayName, gotPortalProduct.DisplayName) + + gotProductPage, err := fresh.GetProductPage(portalProduct.PortalProductID, productPage.ProductPageID) + require.NoError(t, err) + assert.Equal(t, productPage.ProductPageID, gotProductPage.ProductPageID) + + gotProductREPage, err := fresh.GetProductRestEndpointPage( + portalProduct.PortalProductID, productREPage.ProductRestEndpointPageID, + ) + require.NoError(t, err) + assert.Equal(t, productREPage.ProductRestEndpointPageID, gotProductREPage.ProductRestEndpointPageID) + + gotSharingPolicy, err := fresh.GetPortalProductSharingPolicy(portalProduct.PortalProductID) + require.NoError(t, err) + assert.Equal(t, sharingPolicy.PolicyDocument, gotSharingPolicy.PolicyDocument) + + gotVpcLink, err := fresh.GetVpcLink(vpcLink.VpcLinkID) + require.NoError(t, err) + assert.Equal(t, vpcLink.Name, gotVpcLink.Name) + + gotRoutingRule, err := fresh.GetRoutingRule(domainName.DomainNameValue, routingRule.RoutingRuleID) + require.NoError(t, err) + assert.Equal(t, routingRule.Priority, gotRoutingRule.Priority) +} + +// TestInMemoryBackend_SnapshotRestore_EmptyBackend verifies an empty backend +// round-trips to another empty backend without error, matching +// services/apigateway's equivalent case. +func TestInMemoryBackend_SnapshotRestore_EmptyBackend(t *testing.T) { + t.Parallel() + + b := apigatewayv2.NewInMemoryBackend() + + snap := b.Snapshot(t.Context()) + require.NotNil(t, snap) + + fresh := apigatewayv2.NewInMemoryBackend() + require.NoError(t, fresh.Restore(t.Context(), snap)) + + apis, err := fresh.GetAPIs() + require.NoError(t, err) + assert.Empty(t, apis) +} diff --git a/services/apigatewayv2/store_setup.go b/services/apigatewayv2/store_setup.go new file mode 100644 index 000000000..65118197f --- /dev/null +++ b/services/apigatewayv2/store_setup.go @@ -0,0 +1,254 @@ +package apigatewayv2 + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend is registered exactly once, +// here, as a *store.Table[T] on b.registry. See pkgs/store's package doc and +// the services/apigateway (commit 6da0334e) / services/appsync (commit +// 870feb90) conversions this follows closely -- apigatewayv2's shape (a set +// of resource families nested under an API ID, plus a few nested under a +// domain name or portal product) is very similar to apigateway's. +// +// # Nested collections +// +// stages, routes, integrations, deployments, authorizers, and models were +// previously nested per-parent maps (map[apiID]*apiData holding its own +// map[childID]*T inside apiData). store.Table has no notion of nesting, so +// each becomes a single flat table keyed by a composite "#" +// string, with a secondary [store.Index] grouping by API ID for the "all +// children of API X" lookups the nested maps used to answer directly. This +// requires the child value type to carry its parent's API ID as a field +// purely for identity/keying (never on the wire) -- Stage, Route, +// Integration, Deployment, Authorizer, and Model already carry +// `APIID string `json:"-"`` for exactly this reason (see models.go). +// +// integrationResponses and routeResponses were nested two levels deep +// (map[apiID]*apiData -> map[integrationID/routeID]map[responseID]*T). Each +// becomes a flat table keyed by the full composite +// "##", with a secondary index +// grouping by the "#" prefix (reusing integrationKey/ +// routeKey) for the "all responses of integration/route X" lookups. +// +// apiMappings and routingRules were nested per-domain maps +// (map[domainName]map[id]*T); each becomes a flat table keyed by +// "#" with a secondary index grouping by domain name. +// +// productPages and productREPages were map[portalProductID][]*T (an +// append-only slice per parent). Each element already carries its own ID +// (ProductPageID/ProductRestEndpointPageID) and its parent ID +// (`PortalProductID string `json:"-"``), so -- exactly like the families +// above -- these become flat composite-key tables with a secondary index +// grouping by portal product ID, rather than being left as raw slice-valued +// maps: unlike EC2's/apigateway's genuinely identity-less slice exceptions, +// every element here already has the identity fields a Table/Index needs. +// +// # What is NOT converted here, and why +// +// - portalProductSharingPolicies (portalProductID -> policy document +// string): the value is a bare string with no identity field of its own +// to key a Table by, same exception class as EC2's instanceIMDSOptions +// and apigateway's usageOverrides. Left as a plain map, reset by hand in +// InMemoryBackend.Reset. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +func apiKeyFn(v *API) string { return v.APIID } + +// stageKey/routeKey/... build the composite "#" key shared by +// every resource family nested under an API. Access sites use these directly +// so the exact same key is used for Put/Get/Delete. +func stageKey(apiID, stageName string) string { return apiID + "#" + stageName } +func stageKeyFn(v *Stage) string { return stageKey(v.APIID, v.StageName) } + +func routeKey(apiID, routeID string) string { return apiID + "#" + routeID } +func routeKeyFn(v *Route) string { return routeKey(v.APIID, v.RouteID) } + +func integrationKey(apiID, integrationID string) string { return apiID + "#" + integrationID } +func integrationKeyFn(v *Integration) string { return integrationKey(v.APIID, v.IntegrationID) } + +func deploymentKey(apiID, deploymentID string) string { return apiID + "#" + deploymentID } +func deploymentKeyFn(v *Deployment) string { return deploymentKey(v.APIID, v.DeploymentID) } + +func authorizerKey(apiID, authorizerID string) string { return apiID + "#" + authorizerID } +func authorizerKeyFn(v *Authorizer) string { return authorizerKey(v.APIID, v.AuthorizerID) } + +func modelKey(apiID, modelID string) string { return apiID + "#" + modelID } +func modelKeyFn(v *Model) string { return modelKey(v.APIID, v.ModelID) } + +// integrationResponseKey builds the composite "##" +// key for the integrationResponses table; integrationKey builds the +// "#" prefix reused as the secondary index's group key. +func integrationResponseKey(apiID, integrationID, responseID string) string { + return integrationKey(apiID, integrationID) + "#" + responseID +} +func integrationResponseKeyFn(v *IntegrationResponse) string { + return integrationResponseKey(v.APIID, v.IntegrationID, v.IntegrationResponseID) +} +func integrationResponseIntegrationIndexKeyFn(v *IntegrationResponse) string { + return integrationKey(v.APIID, v.IntegrationID) +} + +// routeResponseKey builds the composite "##" key +// for the routeResponses table; routeKey builds the "#" +// prefix reused as the secondary index's group key. +func routeResponseKey(apiID, routeID, responseID string) string { + return routeKey(apiID, routeID) + "#" + responseID +} +func routeResponseKeyFn(v *RouteResponse) string { + return routeResponseKey(v.APIID, v.RouteID, v.RouteResponseID) +} +func routeResponseRouteIndexKeyFn(v *RouteResponse) string { + return routeKey(v.APIID, v.RouteID) +} + +func domainNameKeyFn(v *DomainName) string { return v.DomainNameValue } + +// apiMappingKey builds the composite "#" key shared by +// every API mapping access site. +func apiMappingKey(domainName, mappingID string) string { return domainName + "#" + mappingID } +func apiMappingKeyFn(v *APIMapping) string { return apiMappingKey(v.DomainName, v.APIMappingID) } +func apiMappingDomainIndexKeyFn(v *APIMapping) string { return v.DomainName } + +func portalKeyFn(v *Portal) string { return v.PortalID } +func portalProductKeyFn(v *PortalProduct) string { return v.PortalProductID } + +// productPageKey builds the composite "#" key shared +// by every product page access site. +func productPageKey(portalProductID, pageID string) string { return portalProductID + "#" + pageID } +func productPageKeyFn(v *ProductPage) string { + return productPageKey(v.PortalProductID, v.ProductPageID) +} +func productPagePortalProductIndexKeyFn(v *ProductPage) string { return v.PortalProductID } + +func productREPageKey(portalProductID, pageID string) string { return portalProductID + "#" + pageID } +func productREPageKeyFn(v *ProductRestEndpointPage) string { + return productREPageKey(v.PortalProductID, v.ProductRestEndpointPageID) +} +func productREPagePortalProductIndexKeyFn(v *ProductRestEndpointPage) string { + return v.PortalProductID +} + +func vpcLinkKeyFn(v *VpcLink) string { return v.VpcLinkID } + +// routingRuleKey builds the composite "#" key +// shared by every routing rule access site. +func routingRuleKey(domainName, routingRuleID string) string { + return domainName + "#" + routingRuleID +} +func routingRuleKeyFn(v *RoutingRule) string { return routingRuleKey(v.DomainName, v.RoutingRuleID) } +func routingRuleDomainIndexKeyFn(v *RoutingRule) string { return v.DomainName } + +// registerAllTables sets up every converted resource collection exactly once, +// at construction time. +// +// Two groups, split by whether their store.Table can be snapshotted directly: +// +// - "Clean" tables (apis, domainNames, portals, portalProducts, vpcLinks) +// are registered on b.registry via store.Register, so persistence.go's +// Snapshot/Restore can drive them through +// b.registry.SnapshotAll()/RestoreAll() -- their value types marshal to +// JSON with no information loss. +// - "Dirty" tables (stages, routes, integrations, deployments, +// authorizers, models, integrationResponses, routeResponses, +// apiMappings, productPages, productREPages, routingRules) are built +// with store.New but deliberately NOT registered on b.registry. Their +// composite key depends on a field (APIID/DomainName/PortalProductID) +// tagged `json:"-"` on the live type (see models.go), so a direct +// json.Marshal/Unmarshal round trip through Table.Snapshot/Restore would +// silently drop that field and corrupt the table's key on restore. +// persistence.go instead builds a throwaway DTO [store.Registry] purely +// to get a correctly-tagged JSON encoding (mirrors services/apigateway, +// commit 6da0334e), then restores the live tables directly via +// Table.Restore. Because they aren't registered on b.registry, +// InMemoryBackend.Reset resets each of them with an explicit +// Table.Reset() call alongside b.registry.ResetAll(). +// +// The following field is deliberately a plain map, not a store.Table at all +// -- see this file's doc comment for the reasoning: +// - portalProductSharingPolicies (portalProductID -> policy document +// string): a bare string value with no identity field of its own to key +// a Table by. +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call (plus any secondary store.AddIndex) to the concrete field and value +// type. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + // "Clean" tables: registered on b.registry. + func(b *InMemoryBackend) { + b.apis = store.Register(b.registry, "apis", store.New(apiKeyFn)) + }, + func(b *InMemoryBackend) { + b.domainNames = store.Register(b.registry, "domainNames", store.New(domainNameKeyFn)) + }, + func(b *InMemoryBackend) { + b.portals = store.Register(b.registry, "portals", store.New(portalKeyFn)) + }, + func(b *InMemoryBackend) { + b.portalProducts = store.Register(b.registry, "portalProducts", store.New(portalProductKeyFn)) + }, + func(b *InMemoryBackend) { + b.vpcLinks = store.Register(b.registry, "vpcLinks", store.New(vpcLinkKeyFn)) + }, + // "Dirty" tables: store.New only, NOT store.Register -- see + // registerAllTables's doc above. + func(b *InMemoryBackend) { + b.stages = store.New(stageKeyFn) + b.stagesByAPI = b.stages.AddIndex("byAPI", func(v *Stage) string { return v.APIID }) + }, + func(b *InMemoryBackend) { + b.routes = store.New(routeKeyFn) + b.routesByAPI = b.routes.AddIndex("byAPI", func(v *Route) string { return v.APIID }) + }, + func(b *InMemoryBackend) { + b.integrations = store.New(integrationKeyFn) + b.integrationsByAPI = b.integrations.AddIndex("byAPI", func(v *Integration) string { return v.APIID }) + }, + func(b *InMemoryBackend) { + b.deployments = store.New(deploymentKeyFn) + b.deploymentsByAPI = b.deployments.AddIndex("byAPI", func(v *Deployment) string { return v.APIID }) + }, + func(b *InMemoryBackend) { + b.authorizers = store.New(authorizerKeyFn) + b.authorizersByAPI = b.authorizers.AddIndex("byAPI", func(v *Authorizer) string { return v.APIID }) + }, + func(b *InMemoryBackend) { + b.models = store.New(modelKeyFn) + b.modelsByAPI = b.models.AddIndex("byAPI", func(v *Model) string { return v.APIID }) + }, + func(b *InMemoryBackend) { + b.integrationResponses = store.New(integrationResponseKeyFn) + b.integrationResponsesByIntegration = b.integrationResponses.AddIndex( + "byIntegration", integrationResponseIntegrationIndexKeyFn, + ) + }, + func(b *InMemoryBackend) { + b.routeResponses = store.New(routeResponseKeyFn) + b.routeResponsesByRoute = b.routeResponses.AddIndex("byRoute", routeResponseRouteIndexKeyFn) + }, + func(b *InMemoryBackend) { + b.apiMappings = store.New(apiMappingKeyFn) + b.apiMappingsByDomain = b.apiMappings.AddIndex("byDomain", apiMappingDomainIndexKeyFn) + }, + func(b *InMemoryBackend) { + b.productPages = store.New(productPageKeyFn) + b.productPagesByPortalProduct = b.productPages.AddIndex("byPortalProduct", productPagePortalProductIndexKeyFn) + }, + func(b *InMemoryBackend) { + b.productREPages = store.New(productREPageKeyFn) + b.productREPagesByPortalProduct = b.productREPages.AddIndex( + "byPortalProduct", productREPagePortalProductIndexKeyFn, + ) + }, + func(b *InMemoryBackend) { + b.routingRules = store.New(routingRuleKeyFn) + b.routingRulesByDomain = b.routingRules.AddIndex("byDomain", routingRuleDomainIndexKeyFn) + }, +} diff --git a/services/appsync/backend.go b/services/appsync/backend.go index 34b156d04..bb3a60e2b 100644 --- a/services/appsync/backend.go +++ b/services/appsync/backend.go @@ -18,6 +18,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awserr" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) @@ -309,49 +310,59 @@ func randomAPIKeyID() string { } // InMemoryBackend is the in-memory implementation of StorageBackend. +// +// Resource collections are backed by *store.Table[T] (see store_setup.go for +// the registration list and pkgs/store's package doc). Collections nested +// under a GraphQL/Event API in the original hand-rolled maps (datasources, +// resolvers, functions, types, channelNamespaces) are now single flat tables +// keyed by a composite "#" string, with a secondary +// [store.Index] grouping by API ID for the "all children of API X" lookups +// the nested maps used to answer directly -- see store_setup.go's doc comment +// for why this is safe (every child value type already carries its own APIID +// field as a real, wire-serialized identity field, unlike some other +// services' internal-only parent-ID fields). type InMemoryBackend struct { - apis map[string]*GraphqlAPI // apiID → api - schemas map[string]*Schema // apiID → schema - datasources map[string]map[string]*DataSource // apiID → name → ds - resolvers map[string]map[string]*Resolver // apiID → "TypeName.FieldName" → resolver - apiKeys map[string]map[string]*APIKey // apiID → keyID → key - apiCaches map[string]*APICache // apiID → cache - functions map[string]map[string]*Function // apiID → functionID → function - types map[string]map[string]*APIType // apiID → typeName → type - domainNames map[string]*DomainName // domainName → domainNameConfig - apiAssociations map[string]*APIAssociation // domainName → association - eventAPIs map[string]*API // apiID → event api - channelNamespaces map[string]map[string]*ChannelNamespace // apiID → name → ns - sourceAssocs map[string]*SourceAPIAssociation // associationID → association - lambdaFn LambdaInvoker - ddbBackend DynamoDBBackend - mu *lockmetrics.RWMutex - accountID string - region string - endpoint string + registry *store.Registry + apis *store.Table[GraphqlAPI] + schemas *store.Table[Schema] + datasources *store.Table[DataSource] // key: apiID#name + datasourcesByAPI *store.Index[DataSource] // apiID → datasources + resolvers *store.Table[Resolver] // key: apiID#TypeName.FieldName + resolversByAPI *store.Index[Resolver] // apiID → resolvers + apiKeys map[string]map[string]*APIKey // apiID → keyID → key (raw; see store_setup.go) + apiCaches *store.Table[APICache] + functions *store.Table[Function] // key: apiID#functionID + functionsByAPI *store.Index[Function] // apiID → functions + types *store.Table[APIType] // key: apiID#typeName + typesByAPI *store.Index[APIType] // apiID → types + domainNames *store.Table[DomainName] + apiAssociations *store.Table[APIAssociation] + eventAPIs *store.Table[API] + channelNamespaces *store.Table[ChannelNamespace] // key: apiID#name + channelNamespacesByAPI *store.Index[ChannelNamespace] // apiID → channel namespaces + sourceAssocs *store.Table[SourceAPIAssociation] + lambdaFn LambdaInvoker + ddbBackend DynamoDBBackend + mu *lockmetrics.RWMutex + accountID string + region string + endpoint string } // NewInMemoryBackend creates a new in-memory AppSync backend. func NewInMemoryBackend(accountID, region, endpoint string) *InMemoryBackend { - return &InMemoryBackend{ - apis: make(map[string]*GraphqlAPI), - schemas: make(map[string]*Schema), - datasources: make(map[string]map[string]*DataSource), - resolvers: make(map[string]map[string]*Resolver), - apiKeys: make(map[string]map[string]*APIKey), - apiCaches: make(map[string]*APICache), - functions: make(map[string]map[string]*Function), - types: make(map[string]map[string]*APIType), - domainNames: make(map[string]*DomainName), - apiAssociations: make(map[string]*APIAssociation), - eventAPIs: make(map[string]*API), - channelNamespaces: make(map[string]map[string]*ChannelNamespace), - sourceAssocs: make(map[string]*SourceAPIAssociation), - mu: lockmetrics.New("appsync"), - accountID: accountID, - region: region, - endpoint: endpoint, + b := &InMemoryBackend{ + registry: store.NewRegistry(), + apiKeys: make(map[string]map[string]*APIKey), + mu: lockmetrics.New("appsync"), + accountID: accountID, + region: region, + endpoint: endpoint, } + + registerAllTables(b) + + return b } // Reset clears all state from the backend, returning it to a clean initial state. @@ -361,33 +372,20 @@ func (b *InMemoryBackend) Reset() { defer b.mu.Unlock() // Close tag resources before discarding. - for _, api := range b.apis { + for _, api := range b.apis.All() { if api.Tags != nil { api.Tags.Close() } } - for _, dss := range b.datasources { - for _, ds := range dss { - if ds != nil && ds.Tags != nil { - ds.Tags.Close() - } + for _, ds := range b.datasources.All() { + if ds != nil && ds.Tags != nil { + ds.Tags.Close() } } - b.apis = make(map[string]*GraphqlAPI) - b.schemas = make(map[string]*Schema) - b.datasources = make(map[string]map[string]*DataSource) - b.resolvers = make(map[string]map[string]*Resolver) + b.registry.ResetAll() b.apiKeys = make(map[string]map[string]*APIKey) - b.apiCaches = make(map[string]*APICache) - b.functions = make(map[string]map[string]*Function) - b.types = make(map[string]map[string]*APIType) - b.domainNames = make(map[string]*DomainName) - b.apiAssociations = make(map[string]*APIAssociation) - b.eventAPIs = make(map[string]*API) - b.channelNamespaces = make(map[string]map[string]*ChannelNamespace) - b.sourceAssocs = make(map[string]*SourceAPIAssociation) } // SetLambdaInvoker configures the Lambda invoker for LAMBDA data sources. @@ -467,7 +465,7 @@ func (b *InMemoryBackend) CreateGraphqlAPI( api.Tags.Set(k, v) } - b.apis[apiID] = api + b.apis.Put(api) cp := *api @@ -514,7 +512,7 @@ func (b *InMemoryBackend) GetGraphqlAPI(apiID string) (*GraphqlAPI, error) { b.mu.RLock("GetGraphqlApi") defer b.mu.RUnlock() - api, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -536,7 +534,7 @@ func (b *InMemoryBackend) UpdateGraphqlAPI( b.mu.Lock("UpdateGraphqlApi") defer b.mu.Unlock() - api, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -583,9 +581,10 @@ func (b *InMemoryBackend) ListGraphqlAPIs(apiType string) ([]*GraphqlAPI, error) b.mu.RLock("ListGraphqlApis") defer b.mu.RUnlock() - out := make([]*GraphqlAPI, 0, len(b.apis)) + apis := b.apis.All() + out := make([]*GraphqlAPI, 0, len(apis)) - for _, api := range b.apis { + for _, api := range apis { if apiType != "" && api.APIType != apiType { continue } @@ -606,24 +605,38 @@ func (b *InMemoryBackend) DeleteGraphqlAPI(apiID string) error { b.mu.Lock("DeleteGraphqlApi") defer b.mu.Unlock() - api, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - // Snapshot data sources before deletion so we can release their tag resources. - dss := b.datasources[apiID] + // Snapshot data sources before deletion so we can release their tag + // resources. Cloned because Table.Delete below mutates the same + // index-owned backing slice datasourcesByAPI.Get returns. + dss := slices.Clone(b.datasourcesByAPI.Get(apiID)) - delete(b.apis, apiID) - delete(b.schemas, apiID) - delete(b.datasources, apiID) - delete(b.resolvers, apiID) + b.apis.Delete(apiID) + b.schemas.Delete(apiID) + + for _, ds := range dss { + b.datasources.Delete(datasourceKey(apiID, ds.Name)) + } + + for _, r := range slices.Clone(b.resolversByAPI.Get(apiID)) { + b.resolvers.Delete(resolverTableKey(apiID, r.TypeName, r.FieldName)) + } // Cascade-delete sub-resources added as part of issue #842. delete(b.apiKeys, apiID) - delete(b.apiCaches, apiID) - delete(b.functions, apiID) - delete(b.types, apiID) + b.apiCaches.Delete(apiID) + + for _, fn := range slices.Clone(b.functionsByAPI.Get(apiID)) { + b.functions.Delete(functionKey(apiID, fn.FunctionID)) + } + + for _, t := range slices.Clone(b.typesByAPI.Get(apiID)) { + b.types.Delete(apiTypeKey(apiID, t.Name)) + } if api.Tags != nil { api.Tags.Close() @@ -643,7 +656,7 @@ func (b *InMemoryBackend) StartSchemaCreation(apiID, sdl string) (*Schema, error b.mu.Lock("StartSchemaCreation") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -659,7 +672,7 @@ func (b *InMemoryBackend) StartSchemaCreation(apiID, sdl string) (*Schema, error Status: SchemaStatusFailed, Details: gqlErr.Error(), } - b.schemas[apiID] = schema + b.schemas.Put(schema) return nil, fmt.Errorf("%w: %s", ErrInvalidSchema, gqlErr.Error()) } @@ -670,7 +683,7 @@ func (b *InMemoryBackend) StartSchemaCreation(apiID, sdl string) (*Schema, error Status: SchemaStatusActive, parsedSchema: parsed, } - b.schemas[apiID] = schema + b.schemas.Put(schema) cp := *schema @@ -682,11 +695,11 @@ func (b *InMemoryBackend) GetSchemaCreationStatus(apiID string) (*Schema, error) b.mu.RLock("GetSchemaCreationStatus") defer b.mu.RUnlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - schema, ok := b.schemas[apiID] + schema, ok := b.schemas.Get(apiID) if !ok { return &Schema{ APIID: apiID, @@ -704,7 +717,7 @@ func (b *InMemoryBackend) GetIntrospectionSchema(apiID, _ string) ([]byte, error b.mu.RLock("GetIntrospectionSchema") defer b.mu.RUnlock() - schema, ok := b.schemas[apiID] + schema, ok := b.schemas.Get(apiID) if !ok { return nil, fmt.Errorf("%w: schema not found for api %s", ErrNotFound, apiID) } @@ -717,7 +730,7 @@ func (b *InMemoryBackend) CreateDataSource(apiID string, ds *DataSource) (*DataS b.mu.Lock("CreateDataSource") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -737,11 +750,7 @@ func (b *InMemoryBackend) CreateDataSource(apiID string, ds *DataSource) (*DataS return nil, fmt.Errorf("%w: httpConfig.endpoint is required for HTTP data sources", ErrValidation) } - if b.datasources[apiID] == nil { - b.datasources[apiID] = make(map[string]*DataSource) - } - - if _, exists := b.datasources[apiID][ds.Name]; exists { + if b.datasources.Has(datasourceKey(apiID, ds.Name)) { return nil, fmt.Errorf("%w: datasource %s already exists", ErrAlreadyExists, ds.Name) } @@ -757,7 +766,7 @@ func (b *InMemoryBackend) CreateDataSource(apiID string, ds *DataSource) (*DataS ds.Tags = tags.New("appsync.ds." + apiID + "." + ds.Name + ".tags") } - b.datasources[apiID][ds.Name] = ds + b.datasources.Put(ds) cp := *ds @@ -769,13 +778,11 @@ func (b *InMemoryBackend) GetDataSource(apiID, name string) (*DataSource, error) b.mu.RLock("GetDataSource") defer b.mu.RUnlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - dss := b.datasources[apiID] - ds, ok := dss[name] - + ds, ok := b.datasources.Get(datasourceKey(apiID, name)) if !ok { return nil, fmt.Errorf("%w: datasource %s not found", ErrNotFound, name) } @@ -790,11 +797,11 @@ func (b *InMemoryBackend) ListDataSources(apiID string) ([]*DataSource, error) { b.mu.RLock("ListDataSources") defer b.mu.RUnlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - dss := b.datasources[apiID] + dss := b.datasourcesByAPI.Get(apiID) out := make([]*DataSource, 0, len(dss)) for _, ds := range dss { @@ -815,13 +822,13 @@ func (b *InMemoryBackend) DeleteDataSource(apiID, name string) error { b.mu.Lock("DeleteDataSource") defer b.mu.Unlock() - dss, ok := b.datasources[apiID] - if !ok || dss[name] == nil { + ds, ok := b.datasources.Get(datasourceKey(apiID, name)) + if !ok { return fmt.Errorf("%w: datasource %s not found", ErrNotFound, name) } // Prevent deletion if any UNIT resolver references this data source. - for _, r := range b.resolvers[apiID] { + for _, r := range b.resolversByAPI.Get(apiID) { if r.DataSourceName == name { return fmt.Errorf( "%w: data source %s is still referenced by resolver %s.%s", @@ -834,7 +841,7 @@ func (b *InMemoryBackend) DeleteDataSource(apiID, name string) error { } // Prevent deletion if any function references this data source. - for _, fn := range b.functions[apiID] { + for _, fn := range b.functionsByAPI.Get(apiID) { if fn.DataSourceName == name { return fmt.Errorf( "%w: data source %s is still referenced by function %s", @@ -845,8 +852,7 @@ func (b *InMemoryBackend) DeleteDataSource(apiID, name string) error { } } - ds := dss[name] - delete(dss, name) + b.datasources.Delete(datasourceKey(apiID, name)) if ds.Tags != nil { ds.Tags.Close() @@ -865,7 +871,7 @@ func (b *InMemoryBackend) CreateResolver(apiID, typeName string, r *Resolver) (* b.mu.Lock("CreateResolver") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -885,12 +891,8 @@ func (b *InMemoryBackend) CreateResolver(apiID, typeName string, r *Resolver) (* return nil, fmt.Errorf("%w: dataSourceName is required for UNIT resolvers", ErrValidation) } - if b.resolvers[apiID] == nil { - b.resolvers[apiID] = make(map[string]*Resolver) - } - - key := resolverKey(typeName, r.FieldName) - if _, exists := b.resolvers[apiID][key]; exists { + key := resolverTableKey(apiID, typeName, r.FieldName) + if b.resolvers.Has(key) { return nil, fmt.Errorf("%w: resolver %s.%s already exists", ErrAlreadyExists, typeName, r.FieldName) } @@ -903,7 +905,7 @@ func (b *InMemoryBackend) CreateResolver(apiID, typeName string, r *Resolver) (* r.Kind = resolverKindUnit } - b.resolvers[apiID][key] = r + b.resolvers.Put(r) cp := *r @@ -915,16 +917,11 @@ func (b *InMemoryBackend) GetResolver(apiID, typeName, fieldName string) (*Resol b.mu.RLock("GetResolver") defer b.mu.RUnlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - res, ok := b.resolvers[apiID] - if !ok { - return nil, fmt.Errorf("%w: resolver %s.%s not found", ErrNotFound, typeName, fieldName) - } - - r, ok := res[resolverKey(typeName, fieldName)] + r, ok := b.resolvers.Get(resolverTableKey(apiID, typeName, fieldName)) if !ok { return nil, fmt.Errorf("%w: resolver %s.%s not found", ErrNotFound, typeName, fieldName) } @@ -939,16 +936,15 @@ func (b *InMemoryBackend) ListResolvers(apiID, typeName string) ([]*Resolver, er b.mu.RLock("ListResolvers") defer b.mu.RUnlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - res := b.resolvers[apiID] + res := b.resolversByAPI.Get(apiID) out := make([]*Resolver, 0, len(res)) - prefix := typeName + "." - for key, r := range res { - if strings.HasPrefix(key, prefix) { + for _, r := range res { + if r.TypeName == typeName { cp := *r out = append(out, &cp) } @@ -966,21 +962,16 @@ func (b *InMemoryBackend) DeleteResolver(apiID, typeName, fieldName string) erro b.mu.Lock("DeleteResolver") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - res, ok := b.resolvers[apiID] - if !ok { + key := resolverTableKey(apiID, typeName, fieldName) + if !b.resolvers.Has(key) { return fmt.Errorf("%w: resolver %s.%s not found", ErrNotFound, typeName, fieldName) } - key := resolverKey(typeName, fieldName) - if _, ok = res[key]; !ok { - return fmt.Errorf("%w: resolver %s.%s not found", ErrNotFound, typeName, fieldName) - } - - delete(res, key) + b.resolvers.Delete(key) return nil } @@ -993,16 +984,24 @@ func (b *InMemoryBackend) ExecuteGraphQL( ) (map[string]any, error) { b.mu.RLock("ExecuteGraphQL") - api, apiOK := b.apis[apiID] - schema := b.schemas[apiID] + api, apiOK := b.apis.Get(apiID) + schema, _ := b.schemas.Get(apiID) // Copy resolver and datasource maps under the lock to avoid data races with // concurrent Create/Delete operations. - resolversCopy := make(map[string]*Resolver, len(b.resolvers[apiID])) - maps.Copy(resolversCopy, b.resolvers[apiID]) + apiResolvers := b.resolversByAPI.Get(apiID) + resolversCopy := make(map[string]*Resolver, len(apiResolvers)) - datasourcesCopy := make(map[string]*DataSource, len(b.datasources[apiID])) - maps.Copy(datasourcesCopy, b.datasources[apiID]) + for _, r := range apiResolvers { + resolversCopy[resolverKey(r.TypeName, r.FieldName)] = r + } + + apiDatasources := b.datasourcesByAPI.Get(apiID) + datasourcesCopy := make(map[string]*DataSource, len(apiDatasources)) + + for _, ds := range apiDatasources { + datasourcesCopy[ds.Name] = ds + } b.mu.RUnlock() @@ -1020,7 +1019,7 @@ func (b *InMemoryBackend) CreateAPIKey(apiID, description string, expires int64) b.mu.Lock("CreateAPIKey") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -1068,7 +1067,7 @@ func (b *InMemoryBackend) CreateAPICache(apiID string, cache *APICache) (*APICac b.mu.Lock("CreateAPICache") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -1092,7 +1091,7 @@ func (b *InMemoryBackend) CreateAPICache(apiID string, cache *APICache) (*APICac return nil, fmt.Errorf("%w: invalid apiCachingBehavior %q", ErrValidation, cache.APICachingBehavior) } - if _, exists := b.apiCaches[apiID]; exists { + if b.apiCaches.Has(apiID) { return nil, fmt.Errorf("%w: api cache already exists for api %s", ErrAlreadyExists, apiID) } @@ -1101,7 +1100,7 @@ func (b *InMemoryBackend) CreateAPICache(apiID string, cache *APICache) (*APICac cache.Status = "AVAILABLE" } - b.apiCaches[apiID] = cache + b.apiCaches.Put(cache) cp := *cache @@ -1113,7 +1112,7 @@ func (b *InMemoryBackend) CreateFunction(apiID string, f *Function) (*Function, b.mu.Lock("CreateFunction") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -1121,12 +1120,8 @@ func (b *InMemoryBackend) CreateFunction(apiID string, f *Function) (*Function, return nil, fmt.Errorf("%w: name is required", ErrValidation) } - if b.functions[apiID] == nil { - b.functions[apiID] = make(map[string]*Function) - } - // Enforce name uniqueness across all functions in the API. - for _, existing := range b.functions[apiID] { + for _, existing := range b.functionsByAPI.Get(apiID) { if existing.Name == f.Name { return nil, fmt.Errorf("%w: function with name %s already exists", ErrAlreadyExists, f.Name) } @@ -1145,7 +1140,7 @@ func (b *InMemoryBackend) CreateFunction(apiID string, f *Function) (*Function, f.FunctionVersion = defaultFunctionVersion } - b.functions[apiID][funcID] = f + b.functions.Put(f) cp := *f @@ -1157,7 +1152,7 @@ func (b *InMemoryBackend) CreateType(apiID, definition string, format TypeDefini b.mu.Lock("CreateType") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -1165,17 +1160,13 @@ func (b *InMemoryBackend) CreateType(apiID, definition string, format TypeDefini return nil, fmt.Errorf("%w: invalid format %q, must be SDL or JSON", ErrValidation, format) } - if b.types[apiID] == nil { - b.types[apiID] = make(map[string]*APIType) - } - // Extract type name from definition (assumes SDL format: "type TypeName { ... }"). name := extractTypeName(definition) if name == "" { name = randomAPIID() } - if _, exists := b.types[apiID][name]; exists { + if b.types.Has(apiTypeKey(apiID, name)) { return nil, fmt.Errorf("%w: type %s already exists for api %s", ErrAlreadyExists, name, apiID) } @@ -1190,7 +1181,7 @@ func (b *InMemoryBackend) CreateType(apiID, definition string, format TypeDefini APIID: apiID, } - b.types[apiID][name] = t + b.types.Put(t) cp := *t @@ -1244,7 +1235,7 @@ func (b *InMemoryBackend) CreateDomainName( return nil, fmt.Errorf("%w: invalid domain name %q", ErrValidation, domainName) } - if _, exists := b.domainNames[domainName]; exists { + if b.domainNames.Has(domainName) { return nil, fmt.Errorf("%w: domain name %s already exists", ErrAlreadyExists, domainName) } @@ -1260,7 +1251,7 @@ func (b *InMemoryBackend) CreateDomainName( DomainNameARN: domainNameARN, } - b.domainNames[domainName] = dn + b.domainNames.Put(dn) cp := *dn @@ -1272,13 +1263,13 @@ func (b *InMemoryBackend) AssociateAPI(domainName, apiID string) (*APIAssociatio b.mu.Lock("AssociateAPI") defer b.mu.Unlock() - dn, ok := b.domainNames[domainName] + dn, ok := b.domainNames.Get(domainName) if !ok { return nil, fmt.Errorf("%w: domain name %s not found", ErrNotFound, domainName) } // Validate that the API being associated exists. - if _, exists := b.apis[apiID]; !exists { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -1288,7 +1279,7 @@ func (b *InMemoryBackend) AssociateAPI(domainName, apiID string) (*APIAssociatio AssociationStatus: "SUCCESS", } - b.apiAssociations[domainName] = assoc + b.apiAssociations.Put(assoc) // Update the domain name record to reflect the associated API. dn.APIID = apiID @@ -1324,7 +1315,7 @@ func (b *InMemoryBackend) CreateAPI( EventConfig: eventConfig, } - b.eventAPIs[apiID] = api + b.eventAPIs.Put(api) cp := *api @@ -1340,15 +1331,11 @@ func (b *InMemoryBackend) CreateChannelNamespace( b.mu.Lock("CreateChannelNamespace") defer b.mu.Unlock() - if _, ok := b.eventAPIs[apiID]; !ok { + if !b.eventAPIs.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - if b.channelNamespaces[apiID] == nil { - b.channelNamespaces[apiID] = make(map[string]*ChannelNamespace) - } - - if _, exists := b.channelNamespaces[apiID][name]; exists { + if b.channelNamespaces.Has(channelNamespaceKey(apiID, name)) { return nil, fmt.Errorf("%w: channel namespace %s already exists", ErrAlreadyExists, name) } @@ -1368,7 +1355,7 @@ func (b *InMemoryBackend) CreateChannelNamespace( applyChannelNamespaceConfig(ns, cfg) - b.channelNamespaces[apiID][name] = ns + b.channelNamespaces.Put(ns) cp := *ns @@ -1405,11 +1392,11 @@ func (b *InMemoryBackend) AssociateMergedGraphqlAPI( b.mu.Lock("AssociateMergedGraphqlAPI") defer b.mu.Unlock() - if _, ok := b.apis[sourceAPIIdentifier]; !ok { + if !b.apis.Has(sourceAPIIdentifier) { return nil, fmt.Errorf("%w: source api %s not found", ErrNotFound, sourceAPIIdentifier) } - if _, ok := b.apis[mergedAPIIdentifier]; !ok { + if !b.apis.Has(mergedAPIIdentifier) { return nil, fmt.Errorf("%w: merged api %s not found", ErrNotFound, mergedAPIIdentifier) } @@ -1425,11 +1412,11 @@ func (b *InMemoryBackend) AssociateSourceGraphqlAPI( b.mu.Lock("AssociateSourceGraphqlAPI") defer b.mu.Unlock() - if _, ok := b.apis[mergedAPIIdentifier]; !ok { + if !b.apis.Has(mergedAPIIdentifier) { return nil, fmt.Errorf("%w: merged api %s not found", ErrNotFound, mergedAPIIdentifier) } - if _, ok := b.apis[sourceAPIIdentifier]; !ok { + if !b.apis.Has(sourceAPIIdentifier) { return nil, fmt.Errorf("%w: source api %s not found", ErrNotFound, sourceAPIIdentifier) } @@ -1460,7 +1447,7 @@ func (b *InMemoryBackend) buildSourceAssoc( SourceAPIAssociationConfig: &SourceAPIAssociationConfig{MergeType: mergeType}, } - b.sourceAssocs[assocID] = assoc + b.sourceAssocs.Put(assoc) cp := *assoc @@ -1472,7 +1459,7 @@ func (b *InMemoryBackend) ListAPIKeys(apiID string) ([]*APIKey, error) { b.mu.RLock("ListApiKeys") defer b.mu.RUnlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -1502,7 +1489,7 @@ func (b *InMemoryBackend) DeleteAPIKey(apiID, keyID string) error { b.mu.Lock("DeleteApiKey") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -1521,11 +1508,11 @@ func (b *InMemoryBackend) GetAPICache(apiID string) (*APICache, error) { b.mu.RLock("GetApiCache") defer b.mu.RUnlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - cache, ok := b.apiCaches[apiID] + cache, ok := b.apiCaches.Get(apiID) if !ok { return nil, fmt.Errorf("%w: api cache not found for api %s", ErrNotFound, apiID) } @@ -1540,15 +1527,15 @@ func (b *InMemoryBackend) DeleteAPICache(apiID string) error { b.mu.Lock("DeleteApiCache") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - if _, ok := b.apiCaches[apiID]; !ok { + if !b.apiCaches.Has(apiID) { return fmt.Errorf("%w: api cache not found for api %s", ErrNotFound, apiID) } - delete(b.apiCaches, apiID) + b.apiCaches.Delete(apiID) return nil } @@ -1558,16 +1545,11 @@ func (b *InMemoryBackend) GetFunction(apiID, functionID string) (*Function, erro b.mu.RLock("GetFunction") defer b.mu.RUnlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - fns := b.functions[apiID] - if fns == nil { - return nil, fmt.Errorf("%w: function %s not found", ErrNotFound, functionID) - } - - fn, ok := fns[functionID] + fn, ok := b.functions.Get(functionKey(apiID, functionID)) if !ok { return nil, fmt.Errorf("%w: function %s not found", ErrNotFound, functionID) } @@ -1582,11 +1564,11 @@ func (b *InMemoryBackend) ListFunctions(apiID string) ([]*Function, error) { b.mu.RLock("ListFunctions") defer b.mu.RUnlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - fns := b.functions[apiID] + fns := b.functionsByAPI.Get(apiID) out := make([]*Function, 0, len(fns)) for _, fn := range fns { @@ -1607,17 +1589,17 @@ func (b *InMemoryBackend) DeleteFunction(apiID, functionID string) error { b.mu.Lock("DeleteFunction") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - fns := b.functions[apiID] - if fns == nil || fns[functionID] == nil { + key := functionKey(apiID, functionID) + if !b.functions.Has(key) { return fmt.Errorf("%w: function %s not found", ErrNotFound, functionID) } // Prevent deletion if any resolver still references this function. - for _, r := range b.resolvers[apiID] { + for _, r := range b.resolversByAPI.Get(apiID) { if slices.Contains(r.PipelineConfig, functionID) { return fmt.Errorf( "%w: function %s is still referenced by resolver %s.%s", @@ -1629,7 +1611,7 @@ func (b *InMemoryBackend) DeleteFunction(apiID, functionID string) error { } } - delete(fns, functionID) + b.functions.Delete(key) return nil } @@ -1639,12 +1621,7 @@ func (b *InMemoryBackend) GetType(apiID, typeName string) (*APIType, error) { b.mu.RLock("GetType") defer b.mu.RUnlock() - types := b.types[apiID] - if types == nil { - return nil, fmt.Errorf("%w: type %s not found", ErrNotFound, typeName) - } - - t, ok := types[typeName] + t, ok := b.types.Get(apiTypeKey(apiID, typeName)) if !ok { return nil, fmt.Errorf("%w: type %s not found", ErrNotFound, typeName) } @@ -1659,11 +1636,11 @@ func (b *InMemoryBackend) ListTypes(apiID string) ([]*APIType, error) { b.mu.RLock("ListTypes") defer b.mu.RUnlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - types := b.types[apiID] + types := b.typesByAPI.Get(apiID) out := make([]*APIType, 0, len(types)) for _, t := range types { @@ -1683,12 +1660,12 @@ func (b *InMemoryBackend) DeleteType(apiID, typeName string) error { b.mu.Lock("DeleteType") defer b.mu.Unlock() - types := b.types[apiID] - if types == nil || types[typeName] == nil { + key := apiTypeKey(apiID, typeName) + if !b.types.Has(key) { return fmt.Errorf("%w: type %s not found", ErrNotFound, typeName) } - delete(types, typeName) + b.types.Delete(key) return nil } @@ -1698,7 +1675,7 @@ func (b *InMemoryBackend) GetDomainName(domainName string) (*DomainName, error) b.mu.RLock("GetDomainName") defer b.mu.RUnlock() - dn, ok := b.domainNames[domainName] + dn, ok := b.domainNames.Get(domainName) if !ok { return nil, fmt.Errorf("%w: domain name %s not found", ErrNotFound, domainName) } @@ -1713,9 +1690,10 @@ func (b *InMemoryBackend) ListDomainNames() ([]*DomainName, error) { b.mu.RLock("ListDomainNames") defer b.mu.RUnlock() - out := make([]*DomainName, 0, len(b.domainNames)) + dns := b.domainNames.All() + out := make([]*DomainName, 0, len(dns)) - for _, dn := range b.domainNames { + for _, dn := range dns { cp := *dn out = append(out, &cp) } @@ -1732,12 +1710,12 @@ func (b *InMemoryBackend) DeleteDomainName(domainName string) error { b.mu.Lock("DeleteDomainName") defer b.mu.Unlock() - if _, ok := b.domainNames[domainName]; !ok { + if !b.domainNames.Has(domainName) { return fmt.Errorf("%w: domain name %s not found", ErrNotFound, domainName) } - delete(b.domainNames, domainName) - delete(b.apiAssociations, domainName) + b.domainNames.Delete(domainName) + b.apiAssociations.Delete(domainName) return nil } @@ -1747,11 +1725,11 @@ func (b *InMemoryBackend) GetAPIAssociation(domainName string) (*APIAssociation, b.mu.RLock("GetApiAssociation") defer b.mu.RUnlock() - if _, ok := b.domainNames[domainName]; !ok { + if !b.domainNames.Has(domainName) { return nil, fmt.Errorf("%w: domain name %s not found", ErrNotFound, domainName) } - assoc, ok := b.apiAssociations[domainName] + assoc, ok := b.apiAssociations.Get(domainName) if !ok { return &APIAssociation{ DomainName: domainName, @@ -1769,13 +1747,11 @@ func (b *InMemoryBackend) UpdateDataSource(apiID, name string, ds *DataSource) ( b.mu.Lock("UpdateDataSource") defer b.mu.Unlock() - dss := b.datasources[apiID] - if dss == nil || dss[name] == nil { + existing, ok := b.datasources.Get(datasourceKey(apiID, name)) + if !ok { return nil, fmt.Errorf("%w: data source %s not found", ErrNotFound, name) } - existing := dss[name] - if ds.Description != "" { existing.Description = ds.Description } @@ -1823,14 +1799,12 @@ func (b *InMemoryBackend) UpdateResolver(apiID, typeName string, r *Resolver) (* defer b.mu.Unlock() key := typeName + "." + r.FieldName - res := b.resolvers[apiID] - if res == nil || res[key] == nil { + existing, ok := b.resolvers.Get(resolverTableKey(apiID, typeName, r.FieldName)) + if !ok { return nil, fmt.Errorf("%w: resolver %s not found", ErrNotFound, key) } - existing := res[key] - if r.RequestMappingTemplate != "" { existing.RequestMappingTemplate = r.RequestMappingTemplate } @@ -1881,13 +1855,11 @@ func (b *InMemoryBackend) UpdateFunction(apiID, functionID string, f *Function) b.mu.Lock("UpdateFunction") defer b.mu.Unlock() - fns := b.functions[apiID] - if fns == nil || fns[functionID] == nil { + existing, ok := b.functions.Get(functionKey(apiID, functionID)) + if !ok { return nil, fmt.Errorf("%w: function %s not found", ErrNotFound, functionID) } - existing := fns[functionID] - if f.Name != "" { existing.Name = f.Name } @@ -1937,13 +1909,11 @@ func (b *InMemoryBackend) UpdateType( b.mu.Lock("UpdateType") defer b.mu.Unlock() - types := b.types[apiID] - if types == nil || types[typeName] == nil { + existing, ok := b.types.Get(apiTypeKey(apiID, typeName)) + if !ok { return nil, fmt.Errorf("%w: type %s not found", ErrNotFound, typeName) } - existing := types[typeName] - if definition != "" { existing.Definition = definition } @@ -1962,7 +1932,7 @@ func (b *InMemoryBackend) UpdateAPIKey(apiID, keyID, description string, expires b.mu.Lock("UpdateApiKey") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -1997,11 +1967,11 @@ func (b *InMemoryBackend) UpdateAPICache(apiID string, cache *APICache) (*APICac b.mu.Lock("UpdateApiCache") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - existing, ok := b.apiCaches[apiID] + existing, ok := b.apiCaches.Get(apiID) if !ok { return nil, fmt.Errorf("%w: api cache not found for api %s", ErrNotFound, apiID) } @@ -2036,11 +2006,11 @@ func (b *InMemoryBackend) FlushAPICache(apiID string) error { b.mu.Lock("FlushApiCache") defer b.mu.Unlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - if _, ok := b.apiCaches[apiID]; !ok { + if !b.apiCaches.Has(apiID) { return fmt.Errorf("%w: api cache not found for api %s", ErrNotFound, apiID) } @@ -2052,7 +2022,7 @@ func (b *InMemoryBackend) TagResource(apiID string, tagMap map[string]string) er b.mu.Lock("TagResource") defer b.mu.Unlock() - api, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -2073,7 +2043,7 @@ func (b *InMemoryBackend) UntagResource(apiID string, tagKeys []string) error { b.mu.Lock("UntagResource") defer b.mu.Unlock() - api, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -2090,7 +2060,7 @@ func (b *InMemoryBackend) ListTagsForResource(apiID string) (map[string]string, b.mu.RLock("ListTagsForResource") defer b.mu.RUnlock() - api, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -2107,7 +2077,7 @@ func (b *InMemoryBackend) UpdateDomainName(domainName, description, certificateA b.mu.Lock("UpdateDomainName") defer b.mu.Unlock() - dn, ok := b.domainNames[domainName] + dn, ok := b.domainNames.Get(domainName) if !ok { return nil, fmt.Errorf("%w: domain name %s not found", ErrNotFound, domainName) } @@ -2130,20 +2100,20 @@ func (b *InMemoryBackend) DisassociateAPI(domainName string) error { b.mu.Lock("DisassociateApi") defer b.mu.Unlock() - if _, ok := b.domainNames[domainName]; !ok { + if !b.domainNames.Has(domainName) { return fmt.Errorf("%w: domain name %s not found", ErrNotFound, domainName) } - if _, ok := b.apiAssociations[domainName]; !ok { + if !b.apiAssociations.Has(domainName) { return fmt.Errorf("%w: no api associated with domain %s", ErrNotFound, domainName) } // Clear APIID from domain name and remove association. - if dn, ok := b.domainNames[domainName]; ok { + if dn, ok := b.domainNames.Get(domainName); ok { dn.APIID = "" } - delete(b.apiAssociations, domainName) + b.apiAssociations.Delete(domainName) return nil } @@ -2153,7 +2123,7 @@ func (b *InMemoryBackend) GetAPI(apiID string) (*API, error) { b.mu.RLock("GetApi") defer b.mu.RUnlock() - api, ok := b.eventAPIs[apiID] + api, ok := b.eventAPIs.Get(apiID) if !ok { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -2168,9 +2138,10 @@ func (b *InMemoryBackend) ListAPIs() ([]*API, error) { b.mu.RLock("ListApis") defer b.mu.RUnlock() - out := make([]*API, 0, len(b.eventAPIs)) + apis := b.eventAPIs.All() + out := make([]*API, 0, len(apis)) - for _, api := range b.eventAPIs { + for _, api := range apis { cp := *api out = append(out, &cp) } @@ -2187,12 +2158,15 @@ func (b *InMemoryBackend) DeleteAPI(apiID string) error { b.mu.Lock("DeleteApi") defer b.mu.Unlock() - if _, ok := b.eventAPIs[apiID]; !ok { + if !b.eventAPIs.Has(apiID) { return fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - delete(b.eventAPIs, apiID) - delete(b.channelNamespaces, apiID) + b.eventAPIs.Delete(apiID) + + for _, ns := range slices.Clone(b.channelNamespacesByAPI.Get(apiID)) { + b.channelNamespaces.Delete(channelNamespaceKey(apiID, ns.Name)) + } return nil } @@ -2202,7 +2176,7 @@ func (b *InMemoryBackend) UpdateAPI(apiID, name, ownerContact string, eventConfi b.mu.Lock("UpdateApi") defer b.mu.Unlock() - api, ok := b.eventAPIs[apiID] + api, ok := b.eventAPIs.Get(apiID) if !ok { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -2229,12 +2203,12 @@ func (b *InMemoryBackend) GetChannelNamespace(apiID, name string) (*ChannelNames b.mu.RLock("GetChannelNamespace") defer b.mu.RUnlock() - nss := b.channelNamespaces[apiID] - if nss == nil || nss[name] == nil { + ns, ok := b.channelNamespaces.Get(channelNamespaceKey(apiID, name)) + if !ok { return nil, fmt.Errorf("%w: channel namespace %s not found", ErrNotFound, name) } - cp := *nss[name] + cp := *ns return &cp, nil } @@ -2244,11 +2218,11 @@ func (b *InMemoryBackend) ListChannelNamespaces(apiID string) ([]*ChannelNamespa b.mu.RLock("ListChannelNamespaces") defer b.mu.RUnlock() - if _, ok := b.eventAPIs[apiID]; !ok { + if !b.eventAPIs.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - nss := b.channelNamespaces[apiID] + nss := b.channelNamespacesByAPI.Get(apiID) out := make([]*ChannelNamespace, 0, len(nss)) for _, ns := range nss { @@ -2270,13 +2244,11 @@ func (b *InMemoryBackend) UpdateChannelNamespace( b.mu.Lock("UpdateChannelNamespace") defer b.mu.Unlock() - nss := b.channelNamespaces[apiID] - if nss == nil || nss[name] == nil { + existing, ok := b.channelNamespaces.Get(channelNamespaceKey(apiID, name)) + if !ok { return nil, fmt.Errorf("%w: channel namespace %s not found", ErrNotFound, name) } - existing := nss[name] - applyChannelNamespaceConfig(existing, cfg) existing.LastModified = time.Now().Unix() @@ -2291,12 +2263,12 @@ func (b *InMemoryBackend) DeleteChannelNamespace(apiID, name string) error { b.mu.Lock("DeleteChannelNamespace") defer b.mu.Unlock() - nss := b.channelNamespaces[apiID] - if nss == nil || nss[name] == nil { + key := channelNamespaceKey(apiID, name) + if !b.channelNamespaces.Has(key) { return fmt.Errorf("%w: channel namespace %s not found", ErrNotFound, name) } - delete(nss, name) + b.channelNamespaces.Delete(key) return nil } @@ -2306,7 +2278,7 @@ func (b *InMemoryBackend) GetSourceAPIAssociation(mergedAPIID, associationID str b.mu.RLock("GetSourceApiAssociation") defer b.mu.RUnlock() - assoc, ok := b.sourceAssocs[associationID] + assoc, ok := b.sourceAssocs.Get(associationID) if !ok || assoc.MergedAPIID != mergedAPIID { return nil, fmt.Errorf("%w: source api association %s not found", ErrNotFound, associationID) } @@ -2321,9 +2293,10 @@ func (b *InMemoryBackend) ListSourceAPIAssociations(mergedAPIID string) ([]*Sour b.mu.RLock("ListSourceApiAssociations") defer b.mu.RUnlock() - out := make([]*SourceAPIAssociation, 0, len(b.sourceAssocs)) + assocs := b.sourceAssocs.All() + out := make([]*SourceAPIAssociation, 0, len(assocs)) - for _, assoc := range b.sourceAssocs { + for _, assoc := range assocs { if assoc.MergedAPIID == mergedAPIID { cp := *assoc out = append(out, &cp) @@ -2342,12 +2315,12 @@ func (b *InMemoryBackend) DisassociateMergedGraphqlAPI(sourceAPIID, associationI b.mu.Lock("DisassociateMergedGraphqlApi") defer b.mu.Unlock() - assoc, ok := b.sourceAssocs[associationID] + assoc, ok := b.sourceAssocs.Get(associationID) if !ok || assoc.SourceAPIID != sourceAPIID { return fmt.Errorf("%w: merged api association %s not found", ErrNotFound, associationID) } - delete(b.sourceAssocs, associationID) + b.sourceAssocs.Delete(associationID) return nil } @@ -2357,12 +2330,12 @@ func (b *InMemoryBackend) DisassociateSourceGraphqlAPI(mergedAPIID, associationI b.mu.Lock("DisassociateSourceGraphqlApi") defer b.mu.Unlock() - assoc, ok := b.sourceAssocs[associationID] + assoc, ok := b.sourceAssocs.Get(associationID) if !ok || assoc.MergedAPIID != mergedAPIID { return fmt.Errorf("%w: source api association %s not found", ErrNotFound, associationID) } - delete(b.sourceAssocs, associationID) + b.sourceAssocs.Delete(associationID) return nil } @@ -2372,13 +2345,13 @@ func (b *InMemoryBackend) ListResolversByFunction(apiID, functionID string) ([]* b.mu.RLock("ListResolversByFunction") defer b.mu.RUnlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } var out []*Resolver - for _, r := range b.resolvers[apiID] { + for _, r := range b.resolversByAPI.Get(apiID) { if slices.Contains(r.PipelineConfig, functionID) { cp := *r out = append(out, &cp) @@ -2399,7 +2372,7 @@ func (b *InMemoryBackend) GetGraphqlAPIEnvironmentVariables(apiID string) (map[s b.mu.RLock("GetGraphqlApiEnvironmentVariables") defer b.mu.RUnlock() - api, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -2421,7 +2394,7 @@ func (b *InMemoryBackend) PutGraphqlAPIEnvironmentVariables( b.mu.Lock("PutGraphqlApiEnvironmentVariables") defer b.mu.Unlock() - api, ok := b.apis[apiID] + api, ok := b.apis.Get(apiID) if !ok { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -2490,11 +2463,11 @@ func (b *InMemoryBackend) StartDataSourceIntrospection(apiID, dataSourceName str b.mu.RLock("StartDataSourceIntrospection") defer b.mu.RUnlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return "", fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } - if _, ok := b.datasources[apiID][dataSourceName]; !ok { + if !b.datasources.Has(datasourceKey(apiID, dataSourceName)) { return "", fmt.Errorf("%w: datasource %s not found", ErrNotFound, dataSourceName) } @@ -2522,7 +2495,7 @@ func (b *InMemoryBackend) StartSchemaMerge(apiID string) (SchemaStatus, error) { b.mu.RLock("StartSchemaMerge") defer b.mu.RUnlock() - if _, ok := b.apis[apiID]; !ok { + if !b.apis.Has(apiID) { return "", fmt.Errorf("%w: api %s not found", ErrNotFound, apiID) } @@ -2536,7 +2509,7 @@ func (b *InMemoryBackend) UpdateSourceAPIAssociation( b.mu.Lock("UpdateSourceApiAssociation") defer b.mu.Unlock() - assoc, ok := b.sourceAssocs[associationID] + assoc, ok := b.sourceAssocs.Get(associationID) if !ok || assoc.MergedAPIID != mergedAPIID { return nil, fmt.Errorf("%w: source api association %s not found", ErrNotFound, associationID) } @@ -2544,7 +2517,7 @@ func (b *InMemoryBackend) UpdateSourceAPIAssociation( cp := *assoc cp.Description = description - b.sourceAssocs[associationID] = &cp + b.sourceAssocs.Put(&cp) return &cp, nil } @@ -2556,15 +2529,15 @@ func (b *InMemoryBackend) ListTypesByAssociation(mergedAPIID, associationID, _ s b.mu.RLock("ListTypesByAssociation") defer b.mu.RUnlock() - if _, ok := b.apis[mergedAPIID]; !ok { + if !b.apis.Has(mergedAPIID) { return nil, fmt.Errorf("%w: api %s not found", ErrNotFound, mergedAPIID) } - if _, ok := b.sourceAssocs[associationID]; !ok { + if !b.sourceAssocs.Has(associationID) { return nil, fmt.Errorf("%w: source api association %s not found", ErrNotFound, associationID) } - ts := b.types[mergedAPIID] + ts := b.typesByAPI.Get(mergedAPIID) out := make([]*APIType, 0, len(ts)) for _, t := range ts { diff --git a/services/appsync/export_test.go b/services/appsync/export_test.go index bfef0609b..a82df3dd2 100644 --- a/services/appsync/export_test.go +++ b/services/appsync/export_test.go @@ -2,6 +2,8 @@ // These exports exist solely for testing and must not be called from production code. package appsync +import "encoding/json" + // RenderVTL is the exported test hook for the VTL template renderer. // It is only exposed for package-level tests and should not be used in production code. func RenderVTL(tmpl string, args map[string]any, result any) (string, error) { @@ -13,3 +15,18 @@ func RenderVTL(tmpl string, args map[string]any, result any) (string, error) { func ToDynamoDBJSON(val any) string { return toDynamoDBJSON(val) } + +// SnapshotRegistry is the exported test hook for the backend's internal +// store.Registry.SnapshotAll. It exists solely so appsync_test can exercise +// the Phase 3.3 pkgs/store registry round trip (this service has no +// persistence.go / Snapshot-Restore wiring of its own) without needing an +// unexported field accessor. +func (b *InMemoryBackend) SnapshotRegistry() (map[string]json.RawMessage, error) { + return b.registry.SnapshotAll() +} + +// RestoreRegistry is the exported test hook for the backend's internal +// store.Registry.RestoreAll. See SnapshotRegistry. +func (b *InMemoryBackend) RestoreRegistry(data map[string]json.RawMessage) error { + return b.registry.RestoreAll(data) +} diff --git a/services/appsync/store_setup.go b/services/appsync/store_setup.go new file mode 100644 index 000000000..364f8ef43 --- /dev/null +++ b/services/appsync/store_setup.go @@ -0,0 +1,150 @@ +package appsync + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend is registered exactly once, +// here, as a *store.Table[T] on b.registry. See pkgs/store's package doc and +// the services/ec2 (commit 12e611a4) / services/sqs (commit 0f09d77c) +// conversions this follows. +// +// # Nested collections +// +// datasources, resolvers, functions, types, and channelNamespaces were +// previously nested per-parent maps (map[apiID]map[localKey]*T). store.Table +// has no notion of nesting, so each becomes a single flat table keyed by a +// composite "#" string, with a secondary [store.Index] +// grouping by API ID for the "all children of API X" lookups the nested maps +// used to answer directly (see services/apigateway's store_setup.go for the +// same pattern). Unlike apigateway's Resource/Stage/etc, whose parent-ID +// field is internal-only (tagged json:"-"), every child value type here +// already carries its own APIID field as a REAL, wire-serialized identity +// field (e.g. DataSource.APIID is returned to callers as "apiId"), so these +// composite tables round-trip through JSON with no information loss and no +// DTO/dirty-table split is needed -- they are all "clean" tables, registered +// directly on b.registry. +// +// # What is NOT converted here, and why +// +// - apiKeys (map[apiID]map[keyID]*APIKey): APIKey carries no APIID field of +// its own (only ID/Description/Expires/Deletes), so there is no pure +// func(*APIKey) string that could reproduce the "#" +// composite key from the value alone -- store.Table's keyFn contract +// requires the key to be derived purely from the value. Same exclusion +// class as EC2's instanceIMDSOptions/verifiedAccessEndpointPolicies +// (value has no identity field naming its parent). Left as a plain +// nested map, reset by hand in InMemoryBackend.Reset. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +func apisKeyFn(v *GraphqlAPI) string { return v.APIID } + +func schemasKeyFn(v *Schema) string { return v.APIID } + +// datasourceKey builds the composite "#" key shared by every +// data source access site, so Put/Get/Delete always agree on the same key. +func datasourceKey(apiID, name string) string { return apiID + "#" + name } +func datasourcesKeyFn(v *DataSource) string { return datasourceKey(v.APIID, v.Name) } +func datasourceAPIIndexKeyFn(v *DataSource) string { + return v.APIID +} + +// resolverTableKey builds the composite "#" key +// for the resolvers table, reusing resolverKey (backend.go) for the +// "TypeName.FieldName" portion so both halves of the key are built the same +// way everywhere. +func resolverTableKey(apiID, typeName, fieldName string) string { + return apiID + "#" + resolverKey(typeName, fieldName) +} +func resolversKeyFn(v *Resolver) string { return resolverTableKey(v.APIID, v.TypeName, v.FieldName) } +func resolverAPIIndexKeyFn(v *Resolver) string { + return v.APIID +} + +func apiCachesKeyFn(v *APICache) string { return v.APIID } + +func functionKey(apiID, functionID string) string { return apiID + "#" + functionID } +func functionsKeyFn(v *Function) string { return functionKey(v.APIID, v.FunctionID) } +func functionAPIIndexKeyFn(v *Function) string { + return v.APIID +} + +func apiTypeKey(apiID, name string) string { return apiID + "#" + name } +func typesKeyFn(v *APIType) string { return apiTypeKey(v.APIID, v.Name) } +func typeAPIIndexKeyFn(v *APIType) string { + return v.APIID +} + +func domainNamesKeyFn(v *DomainName) string { return v.DomainName } +func apiAssociationsKeyFn(v *APIAssociation) string { return v.DomainName } +func eventAPIsKeyFn(v *API) string { return v.APIID } + +func channelNamespaceKey(apiID, name string) string { return apiID + "#" + name } +func channelNamespacesKeyFn(v *ChannelNamespace) string { + return channelNamespaceKey(v.APIID, v.Name) +} +func channelNamespaceAPIIndexKeyFn(v *ChannelNamespace) string { + return v.APIID +} + +func sourceAssocsKeyFn(v *SourceAPIAssociation) string { return v.AssociationID } + +// registerAllTables registers every converted resource collection on +// b.registry exactly once. It must be called during construction only +// (immediately after b.registry is created), never on every Reset() -- +// store.Register panics on a duplicate name, so runtime resets go through +// registry.ResetAll() instead (see InMemoryBackend.Reset in backend.go). +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call (plus any secondary store.AddIndex) to the concrete field and value +// type. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.apis = store.Register(b.registry, "apis", store.New(apisKeyFn)) + }, + func(b *InMemoryBackend) { + b.schemas = store.Register(b.registry, "schemas", store.New(schemasKeyFn)) + }, + func(b *InMemoryBackend) { + b.datasources = store.Register(b.registry, "datasources", store.New(datasourcesKeyFn)) + b.datasourcesByAPI = b.datasources.AddIndex("byAPI", datasourceAPIIndexKeyFn) + }, + func(b *InMemoryBackend) { + b.resolvers = store.Register(b.registry, "resolvers", store.New(resolversKeyFn)) + b.resolversByAPI = b.resolvers.AddIndex("byAPI", resolverAPIIndexKeyFn) + }, + func(b *InMemoryBackend) { + b.apiCaches = store.Register(b.registry, "apiCaches", store.New(apiCachesKeyFn)) + }, + func(b *InMemoryBackend) { + b.functions = store.Register(b.registry, "functions", store.New(functionsKeyFn)) + b.functionsByAPI = b.functions.AddIndex("byAPI", functionAPIIndexKeyFn) + }, + func(b *InMemoryBackend) { + b.types = store.Register(b.registry, "types", store.New(typesKeyFn)) + b.typesByAPI = b.types.AddIndex("byAPI", typeAPIIndexKeyFn) + }, + func(b *InMemoryBackend) { + b.domainNames = store.Register(b.registry, "domainNames", store.New(domainNamesKeyFn)) + }, + func(b *InMemoryBackend) { + b.apiAssociations = store.Register(b.registry, "apiAssociations", store.New(apiAssociationsKeyFn)) + }, + func(b *InMemoryBackend) { + b.eventAPIs = store.Register(b.registry, "eventAPIs", store.New(eventAPIsKeyFn)) + }, + func(b *InMemoryBackend) { + b.channelNamespaces = store.Register(b.registry, "channelNamespaces", store.New(channelNamespacesKeyFn)) + b.channelNamespacesByAPI = b.channelNamespaces.AddIndex("byAPI", channelNamespaceAPIIndexKeyFn) + }, + func(b *InMemoryBackend) { + b.sourceAssocs = store.Register(b.registry, "sourceAssocs", store.New(sourceAssocsKeyFn)) + }, +} diff --git a/services/appsync/store_setup_test.go b/services/appsync/store_setup_test.go new file mode 100644 index 000000000..fce61e697 --- /dev/null +++ b/services/appsync/store_setup_test.go @@ -0,0 +1,265 @@ +package appsync_test + +import ( + "testing" + + "github.com/blackbirdworks/gopherstack/services/appsync" +) + +// registryFixtureIDs carries the identifiers built by +// populateRegistryFixture, needed later to look the same resources back up +// on the restored backend. +type registryFixtureIDs struct { + apiID string + mergedAPIID string + eventAPIID string + dsName string + functionID string +} + +// populateRegistryFixture creates one instance of every store.Table-backed +// resource kind on b (see store_setup.go), plus one entry in the +// deliberately-unconverted raw apiKeys map, and returns the identifiers +// needed to look each of them back up afterward. +func populateRegistryFixture(t *testing.T, b *appsync.InMemoryBackend) registryFixtureIDs { + t.Helper() + + api, err := b.CreateGraphqlAPI("api1", appsync.AuthTypeAPIKey, false, "", "", nil, nil, nil) + if err != nil { + t.Fatalf("CreateGraphqlAPI(api1): %v", err) + } + + mergedAPI, err := b.CreateGraphqlAPI("api2", appsync.AuthTypeAPIKey, false, "MERGED", "", nil, nil, nil) + if err != nil { + t.Fatalf("CreateGraphqlAPI(api2, MERGED): %v", err) + } + + _, err = b.StartSchemaCreation(api.APIID, "type Query { hello: String }") + if err != nil { + t.Fatalf("StartSchemaCreation: %v", err) + } + + ds, err := b.CreateDataSource(api.APIID, &appsync.DataSource{Name: "ds1", Type: appsync.DataSourceTypeNone}) + if err != nil { + t.Fatalf("CreateDataSource: %v", err) + } + + _, err = b.CreateResolver(api.APIID, "Query", &appsync.Resolver{FieldName: "hello", DataSourceName: ds.Name}) + if err != nil { + t.Fatalf("CreateResolver: %v", err) + } + + _, err = b.CreateAPICache(api.APIID, &appsync.APICache{ + TTL: 60, Type: "SMALL", APICachingBehavior: "FULL_REQUEST_CACHING", + }) + if err != nil { + t.Fatalf("CreateAPICache: %v", err) + } + + fn, err := b.CreateFunction(api.APIID, &appsync.Function{Name: "fn1", DataSourceName: ds.Name}) + if err != nil { + t.Fatalf("CreateFunction: %v", err) + } + + _, err = b.CreateType(api.APIID, "type Foo { id: ID }", appsync.TypeFormatSDL) + if err != nil { + t.Fatalf("CreateType: %v", err) + } + + const certARN = "arn:aws:acm:us-east-1:123456789012:certificate/abc" + + _, err = b.CreateDomainName("example.com", certARN, "desc", nil) + if err != nil { + t.Fatalf("CreateDomainName: %v", err) + } + + _, err = b.AssociateAPI("example.com", api.APIID) + if err != nil { + t.Fatalf("AssociateAPI: %v", err) + } + + eventAPI, err := b.CreateAPI("events1", "owner@example.com", nil, nil) + if err != nil { + t.Fatalf("CreateAPI: %v", err) + } + + _, err = b.CreateChannelNamespace(eventAPI.APIID, "ns1", nil, nil) + if err != nil { + t.Fatalf("CreateChannelNamespace: %v", err) + } + + _, err = b.AssociateSourceGraphqlAPI(mergedAPI.APIID, api.APIID, "merge desc", "") + if err != nil { + t.Fatalf("AssociateSourceGraphqlAPI: %v", err) + } + + // Also populate the one deliberately-unconverted raw map (apiKeys) so we + // can confirm the round trip below leaves it alone (it is not registered + // on the registry, so RestoreRegistry never touches it). + _, err = b.CreateAPIKey(api.APIID, "test key", 0) + if err != nil { + t.Fatalf("CreateAPIKey: %v", err) + } + + return registryFixtureIDs{ + apiID: api.APIID, + mergedAPIID: mergedAPI.APIID, + eventAPIID: eventAPI.APIID, + dsName: ds.Name, + functionID: fn.FunctionID, + } +} + +// verifyRegistryFixture asserts that every resource populateRegistryFixture +// created is retrievable on restored with the expected wire-visible fields, +// and that the raw (non-registry) apiKeys map was NOT carried over. +func verifyRegistryFixture(t *testing.T, restored *appsync.InMemoryBackend, ids registryFixtureIDs) { + t.Helper() + + verifyRegistryFixtureAPIs(t, restored, ids) + verifyRegistryFixtureChildren(t, restored, ids) + verifyRegistryFixtureFlatResources(t, restored, ids) +} + +func verifyRegistryFixtureAPIs(t *testing.T, restored *appsync.InMemoryBackend, ids registryFixtureIDs) { + t.Helper() + + gotAPI, err := restored.GetGraphqlAPI(ids.apiID) + if err != nil { + t.Errorf("GetGraphqlAPI: %v", err) + } else if gotAPI.Name != "api1" { + t.Errorf("apis: Name = %q, want api1", gotAPI.Name) + } + + // schemas -- NOTE: Schema's unexported, non-JSON parsed-AST field is + // therefore never expected to survive a Snapshot/Restore round trip; only + // the wire-visible fields are checked here. + gotSchema, err := restored.GetSchemaCreationStatus(ids.apiID) + if err != nil { + t.Errorf("GetSchemaCreationStatus: %v", err) + } else if gotSchema.Status != appsync.SchemaStatusActive { + t.Errorf("schemas: Status = %v, want %v", gotSchema.Status, appsync.SchemaStatusActive) + } + + gotEventAPI, err := restored.GetAPI(ids.eventAPIID) + if err != nil { + t.Errorf("GetAPI: %v", err) + } else if gotEventAPI.Name != "events1" { + t.Errorf("eventAPIs: Name = %q, want events1", gotEventAPI.Name) + } +} + +func verifyRegistryFixtureChildren(t *testing.T, restored *appsync.InMemoryBackend, ids registryFixtureIDs) { + t.Helper() + + gotDS, err := restored.GetDataSource(ids.apiID, ids.dsName) + if err != nil { + t.Errorf("GetDataSource: %v", err) + } else if gotDS.APIID != ids.apiID { + t.Errorf("datasources: APIID = %q, want %q", gotDS.APIID, ids.apiID) + } + + gotResolver, err := restored.GetResolver(ids.apiID, "Query", "hello") + if err != nil { + t.Errorf("GetResolver: %v", err) + } else if gotResolver.DataSourceName != ids.dsName { + t.Errorf("resolvers: DataSourceName = %q, want %q", gotResolver.DataSourceName, ids.dsName) + } + + gotFn, err := restored.GetFunction(ids.apiID, ids.functionID) + if err != nil { + t.Errorf("GetFunction: %v", err) + } else if gotFn.Name != "fn1" { + t.Errorf("functions: Name = %q, want fn1", gotFn.Name) + } + + gotType, err := restored.GetType(ids.apiID, "Foo") + if err != nil { + t.Errorf("GetType: %v", err) + } else if gotType.APIID != ids.apiID { + t.Errorf("types: APIID = %q, want %q", gotType.APIID, ids.apiID) + } + + gotNS, err := restored.GetChannelNamespace(ids.eventAPIID, "ns1") + if err != nil { + t.Errorf("GetChannelNamespace: %v", err) + } else if gotNS.Name != "ns1" { + t.Errorf("channelNamespaces: Name = %q, want ns1", gotNS.Name) + } +} + +func verifyRegistryFixtureFlatResources(t *testing.T, restored *appsync.InMemoryBackend, ids registryFixtureIDs) { + t.Helper() + + gotCache, err := restored.GetAPICache(ids.apiID) + if err != nil { + t.Errorf("GetAPICache: %v", err) + } else if gotCache.Type != "SMALL" { + t.Errorf("apiCaches: Type = %q, want SMALL", gotCache.Type) + } + + gotDomain, err := restored.GetDomainName("example.com") + if err != nil { + t.Errorf("GetDomainName: %v", err) + } else if gotDomain.DomainName != "example.com" { + t.Errorf("domainNames: DomainName = %q, want example.com", gotDomain.DomainName) + } + + gotAssoc, err := restored.GetAPIAssociation("example.com") + if err != nil { + t.Errorf("GetAPIAssociation: %v", err) + } else if gotAssoc.APIID != ids.apiID { + t.Errorf("apiAssociations: APIID = %q, want %q", gotAssoc.APIID, ids.apiID) + } + + assocs, err := restored.ListSourceAPIAssociations(ids.mergedAPIID) + if err != nil { + t.Errorf("ListSourceAPIAssociations: %v", err) + } else if len(assocs) != 1 || assocs[0].SourceAPIID != ids.apiID { + t.Errorf("sourceAssocs: got %+v, want one association with SourceAPIID %q", assocs, ids.apiID) + } + + // apiKeys is deliberately NOT registered on the registry (see + // store_setup.go), so it is untouched by RestoreRegistry -- the fresh + // `restored` backend must NOT have inherited the api key created above. + keys, err := restored.ListAPIKeys(ids.apiID) + if err != nil { + t.Errorf("ListAPIKeys: %v", err) + } else if len(keys) != 0 { + t.Errorf("apiKeys: got %d keys on a fresh backend, want 0 (apiKeys is not registry-backed)", len(keys)) + } +} + +// Test_RegistrySnapshotRestore exercises the store.Registry round trip added +// by the Phase 3.3 pkgs/store conversion (see store_setup.go). AppSync has no +// persistence.go / Snapshot-Restore wiring of its own (unlike ec2/sqs), so +// this is the substitute proof that every converted table survives a +// SnapshotAll -> RestoreAll cycle into a fresh backend with the same +// key/value shape a real persistence layer would rely on if one is ever +// wired up for this service. +// +// This is a single sequential lifecycle test (populate -> snapshot -> restore +// -> verify), not a table of independent cases, so it does not use t.Run -- +// see .claude/memories/parity-principles.md's test-isolation note: a +// sequential lifecycle asserted end-to-end in one function is the documented +// exception to the table-test convention. +func Test_RegistrySnapshotRestore(t *testing.T) { + t.Parallel() + + b := newTestBackend() + ids := populateRegistryFixture(t, b) + + data, err := b.SnapshotRegistry() + if err != nil { + t.Fatalf("SnapshotRegistry: %v", err) + } + + restored := newTestBackend() + + err = restored.RestoreRegistry(data) + if err != nil { + t.Fatalf("RestoreRegistry: %v", err) + } + + verifyRegistryFixture(t, restored, ids) +} diff --git a/services/autoscaling/PARITY.md b/services/autoscaling/PARITY.md new file mode 100644 index 000000000..f7311ed75 --- /dev/null +++ b/services/autoscaling/PARITY.md @@ -0,0 +1,249 @@ +--- +service: autoscaling +sdk_module: aws-sdk-go-v2/service/autoscaling@v1.64.2 +last_audit_commit: d0ebe979 +last_audit_date: 2026-07-05 +overall: A # ~900 LOC of genuine production-code fixes this pass (+~670 LOC new tests) +ops: + CreateAutoScalingGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: MixedInstancesPolicy, LifecycleHookSpecificationList, TrafficSources were parsed as no-ops (silently dropped) - now parsed, validated, and registered atomically with the group; initial instances are gated by any launch hook just registered"} + DescribeAutoScalingGroups: {wire: ok, errors: ok, state: ok, persist: ok, note: "added MixedInstancesPolicy to the XML projection (was entirely absent from xmlAutoScalingGroup even though the backend model carried it)"} + UpdateAutoScalingGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: MixedInstancesPolicy was not parsed from the request"} + DeleteAutoScalingGroup: {wire: ok, errors: ok, state: ok, persist: ok} + CreateLaunchConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeLaunchConfigurations: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteLaunchConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeScalingActivities: {wire: ok, errors: ok, state: ok, persist: ok} + AttachInstances: {wire: ok, errors: ok, state: ok, persist: ok} + AttachLoadBalancerTargetGroups: {wire: ok, errors: ok, state: ok, persist: ok} + AttachLoadBalancers: {wire: ok, errors: ok, state: ok, persist: ok} + AttachTrafficSources: {wire: ok, errors: ok, state: ok, persist: ok} + BatchDeleteScheduledAction: {wire: ok, errors: ok, state: ok, persist: ok} + BatchPutScheduledUpdateGroupAction: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: StartTime/EndTime were parsed nowhere and silently dropped; now parsed and stored"} + CancelInstanceRefresh: {wire: ok, errors: ok, state: ok, persist: ok} + CompleteLifecycleAction: {wire: ok, errors: ok, state: ok, persist: ok, note: "CRITICAL fix: previously only stopped a timer that was never created anywhere (dead code) and had zero effect on instance state. Now resolves a real pending lifecycle wait (Pending:Wait/Terminating:Wait -> actual transition), looked up by token OR by (group,hook,instance)"} + CreateOrUpdateTags: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteLifecycleHook: {wire: ok, errors: ok, state: ok, persist: ok} + SetDesiredCapacity: {wire: ok, errors: ok, state: ok, persist: ok, note: "scale-out path now gates new instances through an active launch hook"} + TerminateInstanceInAutoScalingGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "CRITICAL fix: now defers actual removal to Terminating:Wait + CompleteLifecycleAction/timeout when a terminating hook is registered, instead of always terminating instantly; also fixed the replacement-instance path never adding the new instance to instanceIndex"} + PutLifecycleHook: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: NotificationMetadata was never parsed from the request"} + DescribeLifecycleHooks: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeScheduledActions: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteTags: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeTags: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeAutoScalingInstances: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteNotificationConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + DeletePolicy: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteScheduledAction: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteWarmPool: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeAccountLimits: {wire: ok, errors: ok, state: ok, persist: n/a} + DescribeAdjustmentTypes: {wire: ok, errors: ok, state: ok, persist: n/a} + DescribeAutoScalingNotificationTypes: {wire: ok, errors: ok, state: ok, persist: n/a} + DescribeInstanceRefreshes: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeLifecycleHookTypes: {wire: ok, errors: ok, state: ok, persist: n/a} + DescribeLoadBalancerTargetGroups: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeLoadBalancers: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeMetricCollectionTypes: {wire: ok, errors: ok, state: ok, persist: n/a} + DescribeNotificationConfigurations: {wire: ok, errors: ok, state: ok, persist: ok} + DescribePolicies: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: MinAdjustmentStep and MetricAggregationType were never returned"} + DescribeScalingProcessTypes: {wire: ok, errors: ok, state: ok, persist: n/a} + DescribeTerminationPolicyTypes: {wire: ok, errors: ok, state: ok, persist: n/a} + DescribeTrafficSources: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeWarmPool: {wire: ok, errors: ok, state: ok, persist: ok} + DetachInstances: {wire: ok, errors: ok, state: ok, persist: ok} + DetachLoadBalancerTargetGroups: {wire: ok, errors: ok, state: ok, persist: ok} + DetachLoadBalancers: {wire: ok, errors: ok, state: ok, persist: ok} + DetachTrafficSources: {wire: ok, errors: ok, state: ok, persist: ok} + DisableMetricsCollection: {wire: ok, errors: ok, state: ok, persist: ok} + EnableMetricsCollection: {wire: ok, errors: ok, state: ok, persist: ok} + EnterStandby: {wire: ok, errors: ok, state: ok, persist: ok} + ExecutePolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: StepScaling policies ignored StepAdjustments/MetricValue/BreachThreshold entirely and always used the flat ScalingAdjustment/AdjustmentType path; now selects the matching StepAdjustment interval and validates the required fields. Also routed through applyDesiredCapacityChange so ExecutePolicy scale-out/in now respects SuspendedProcesses, scale-in protection, instanceIndex bookkeeping, and launch-hook gating like SetDesiredCapacity does (previously it duplicated and diverged from that logic)"} + ExitStandby: {wire: ok, errors: ok, state: ok, persist: ok} + GetPredictiveScalingForecast: {wire: ok, errors: ok, state: ok, persist: n/a, note: "fixed: response was missing the required UpdateTime field and returned a wrong-shaped, entirely empty LoadForecast; now returns UpdateTime and a real (though intentionally naive - see Notes) Timestamps/Values series"} + LaunchInstances: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed 3 bugs: (1) handler read the wrong query param (DesiredCapacity instead of the real RequestedCapacity, so every call silently launched only 1 instance regardless of the requested count); (2) response used the DescribeAutoScalingGroups per-instance shape instead of the real LaunchInstancesOutput InstanceCollection (grouped by AZ/InstanceType with InstanceIds) shape; (3) the backend never added launched instances to instanceIndex, so they could never be found by TerminateInstanceInAutoScalingGroup"} + PutNotificationConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + PutScalingPolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: MetricAggregationType was accepted nowhere on input or output"} + PutScheduledUpdateGroupAction: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: StartTime/EndTime were parsed nowhere and silently dropped despite the backend model and DescribeScheduledActions XML projection already supporting them"} + PutWarmPool: {wire: ok, errors: ok, state: ok, persist: ok} + RecordLifecycleActionHeartbeat: {wire: ok, errors: ok, state: ok, persist: ok, note: "was re-arming a timer that called a no-op (expireHookAction just deleted the map entry); now re-arms to re-resolve with the hook's DefaultResult, and supports lookup by instance ID (not just token)"} + ResumeProcesses: {wire: ok, errors: ok, state: ok, persist: ok} + RollbackInstanceRefresh: {wire: ok, errors: ok, state: ok, persist: ok} + SetInstanceHealth: {wire: ok, errors: ok, state: ok, persist: ok} + SetInstanceProtection: {wire: ok, errors: ok, state: ok, persist: ok} + StartInstanceRefresh: {wire: ok, errors: ok, state: ok, persist: ok} + SuspendProcesses: {wire: ok, errors: ok, state: ok, persist: ok} +families: + static-describe-types (AdjustmentTypes/NotificationTypes/LifecycleHookTypes/MetricCollectionTypes/ScalingProcessTypes/TerminationPolicyTypes): {status: ok, note: "unchanged this pass; verified op-by-op against the SDK enum lists, all correct"} + instance-refresh (Start/Cancel/Describe/Rollback): {status: ok, note: "unchanged this pass; wire shapes and status machine (InProgress/Cancelling/RollbackInProgress) verified against SDK"} + warm-pool (Put/Delete/Describe): {status: ok, note: "unchanged this pass; verified"} + metrics-collection / suspend-resume-processes / standby: {status: ok, note: "unchanged this pass; verified"} +gaps: + - ASG->EC2 real instance provisioning is simulated only (Instance is a fake record, not backed by an ec2 resource) (bd: gopherstack-8sk) - NOT fixed this pass per scope + - ASG/ECS->ELBv2 target registration is simulated only (TargetGroupARNs/LoadBalancerNames are stored but never actually register targets with elbv2) (bd: gopherstack-18k) - NOT fixed this pass per scope + - Scheduled actions (Put/BatchPut) now correctly persist StartTime/EndTime/Recurrence, but there is no background scheduler goroutine that actually executes them at the scheduled time/cron - Describe reflects what was requested, but nothing fires it. Filed as gopherstack-6ys for follow-up; deliberately not attempted this pass (a correct cron-parsing+ticker engine is a separate, sizable feature, not a quick wire fix, and getting it wrong risks a new leak class) + - Terminate-lifecycle-hook gating is wired into TerminateInstanceInAutoScalingGroup only, NOT into the desired-capacity-driven scale-in path (SetDesiredCapacity/UpdateAutoScalingGroup/ExecutePolicy decreasing) - that path still removes instances immediately regardless of a registered terminating hook (bd: gopherstack-9wo) + - Multiple lifecycle hooks of the *same* transition on one group: this simulation gates on a single (deterministic, lowest-named) hook per transition per group, matching the common case; AWS supports N hooks per transition each independently gating the same instance. Documented simplification, see Notes + - ABANDON on a launch hook terminates the pending instance but does not attempt an automatic relaunch to restore DesiredCapacity (real AWS does retry); documented simplification, see Notes + - GetPredictiveScalingForecast returns a real, well-shaped, non-empty forecast, but it is a flat naive projection (current DesiredCapacity repeated hourly), not a statistical model - genuinely out of scope for an emulator; documented simplification, see Notes + - CreateAutoScalingGroupInput fields not wired up: AvailabilityZoneDistribution, AvailabilityZoneImpairmentPolicy, CapacityReservationSpecification, DeletionProtection, InstanceLifecyclePolicy, InstanceMaintenancePolicy, SkipZonalShiftValidation (all newer/niche SDK additions); not attempted this pass - deferred, no bd id filed yet (low real-world usage relative to MixedInstancesPolicy/LifecycleHookSpecificationList, which WERE fixed) +deferred: + - InstanceRequirements-based MixedInstancesPolicy overrides (attribute-based instance selection) - only InstanceType-based overrides are parsed/returned + - PredictiveScalingConfiguration (Put/Describe are not in GetSupportedOperations at all - predictive scaling policy *configuration* management, as opposed to GetPredictiveScalingForecast, was out of scope for this pass; confirmed the SDK op list has no separate op for this, it rides inside PutScalingPolicy's PolicyType=PredictiveScaling with a nested config this handler does not parse) +leaks: {status: clean, note: "go test -race passes. The pendingHookTokens timer machinery (the CRITICAL item flagged in prior sweep notes) was previously 100% dead code - nothing ever created a pendingHookAction, so there was no leak *and* no functionality. This pass makes the timers real (armed on every gated launch/terminate) and verified: Close() stops all of them (unchanged, already correct), DeleteAutoScalingGroup/DeleteLifecycleHook/Purge call cleanupHookTimers (unchanged, already correct), and Restore() now re-arms timers for any instance found in a *:Wait state (added - in-flight timers are never persisted, so without this a restored mid-wait instance would be stuck forever)."} +--- + +## Notes + +Protocol: EC2 Auto Scaling uses the `query` (form-urlencoded request, XML response) +protocol, `Version=2011-01-01`. Verified against the awsquery serializers/deserializers +in `aws-sdk-go-v2/service/autoscaling@v1.64.2` (vendored the module zip into +`/tmp/asg_sdk` for this audit; not committed anywhere). + +### The lifecycle-hook fix in detail (highest-value finding this pass) + +Prior-sweep notes flagged "lifecycle-hook timeout goroutines" as CRITICAL. The +investigation this pass found the real bug was worse than a leak: `pendingHookTokens`, +its `*time.Timer` field, `cleanupHookTimers`, and `expireHookAction` all existed and +looked plausible, but **nothing anywhere ever inserted an entry into +`pendingHookTokens`**. `PutLifecycleHook` stored the hook; `CompleteLifecycleAction` +and `RecordLifecycleActionHeartbeat` only ever *looked up* an entry that could never +exist. The net effect: creating a lifecycle hook had zero effect on any instance +transition. New instances always went straight to `InService`; terminated instances +were always removed immediately. This is the "disguised stub" pattern called out in +`parity-principles.md` #4 - the code looked real (validated params, stored state, +returned 200) but never touched the one thing lifecycle hooks exist for. + +Fixed by wiring real gating into every instance-creating and instance-terminating +code path: +- launch gating: `CreateAutoScalingGroup` (initial instances, only relevant once + `LifecycleHookSpecificationList` was also wired up - see below), + `applyDesiredCapacityChange` (scale-out, shared by `SetDesiredCapacity`, + `UpdateAutoScalingGroup`, and now `ExecutePolicy`), `LaunchInstances`, and the + replacement-instance branch of `TerminateInstanceInAutoScalingGroup`. +- terminate gating: `TerminateInstanceInAutoScalingGroup` only (see gaps: the + desired-capacity-driven scale-in path in `applyDesiredCapacityChange` was + deliberately left as immediate removal - see below). +- `CompleteLifecycleAction` and `RecordLifecycleActionHeartbeat` now resolve/re-arm a + real pending action, looked up by token OR by `(group, hook, instanceId)` since AWS + allows either. +- Heartbeat timeout expiry calls the same resolution path with the hook's + `DefaultResult`, so an abandoned wait behaves identically whether it was resolved + explicitly or by timeout. + +**Scope boundary, stated explicitly**: terminate-hook gating was *not* wired into the +`applyDesiredCapacityChange` scale-in path (used by `SetDesiredCapacity` decreasing, +`UpdateAutoScalingGroup`, `ExecutePolicy` scale-in). That path currently still removes +instances immediately regardless of a registered terminating hook. Reason: unlike +`TerminateInstanceInAutoScalingGroup` (a single, self-contained instance removal), +scale-in there interacts with `removeUnprotectedInstances`'s batch compaction and the +desired-capacity bookkeeping for potentially many instances at once; deferring N +removals concurrently, each independently completable/timeoutable, while keeping the +group's effective capacity accounting consistent for concurrent +`DescribeAutoScalingGroups` callers, is a meaningfully bigger state machine than the +single-instance case and was judged too risky to rush. Filed as a known, explicit gap +above rather than silently left broken. + +**ABANDON semantics simplification**: for a *launching* hook, ABANDON terminates the +pending instance (matches AWS - a failed launch is torn down) but does not attempt an +automatic relaunch to restore `DesiredCapacity` (AWS does retry via its own internal +worker). For a *terminating* hook, both CONTINUE and ABANDON proceed with the +termination once resolved (AWS lets ABANDON/CONTINUE only affect hook-chaining +metadata for termination, not whether the instance is actually terminated - you cannot +veto a termination via a terminating lifecycle hook). + +**Multiple-hooks-per-transition simplification**: `findHookForTransition` returns a +single, deterministically-chosen (lowest hook name) hook per transition per group. +Real AWS supports registering several hooks on the same transition, each +independently gating the instance in sequence. The overwhelming majority of +real-world ASG configs register at most one hook per transition; documented here so +the next auditor doesn't mistake this for an oversight. + +**Restore/persistence**: `pendingHookTokens` (in-flight timers) are intentionally not +part of `backendSnapshot` - a `*time.Timer` can't be serialized, and this predates +this pass's changes. What's new this pass: `Restore()` now sweeps every instance left +in `Pending:Wait`/`Terminating:Wait` by a restored snapshot and re-arms a timer for it +(using the still-registered hook's HeartbeatTimeout/DefaultResult, or the AWS default +if the hook itself is gone). Without this, an instance restored mid-wait would be +stuck in that state forever, since nothing would ever call +`CompleteLifecycleAction`/hit a timeout for it. + +### Other wire-shape bugs fixed this pass + +- **`LaunchInstances` param typo**: the handler read `vals.Get("DesiredCapacity")`. + The real field name (verified against `LaunchInstancesInput` and its awsquery + serializer) is `RequestedCapacity`. Any real SDK client always sends + `RequestedCapacity`, so gopherstack's handler saw an empty string every time and + fell back to launching exactly 1 instance regardless of what was requested. +- **`LaunchInstances` output shape**: `LaunchInstancesOutput.Instances` is + `[]InstanceCollection` (grouped by `AvailabilityZone`/`InstanceType`, each carrying + `InstanceIds []string`), NOT a flat list of per-instance + `LifecycleState`/`HealthStatus` records (that shape belongs to + `DescribeAutoScalingGroups`/`DescribeAutoScalingInstances`). The handler was reusing + the wrong XML type. Fixed with a dedicated `xmlInstanceCollection` type and a + grouping helper. +- **`LaunchInstances` never indexed its instances**: the backend method appended to + `g.Instances` but never touched `b.instanceIndex`, so an instance launched this way + could never be found by `TerminateInstanceInAutoScalingGroup` (which looks up + purely via `instanceIndex`). Same bug existed in the replacement-instance branch of + `TerminateInstanceInAutoScalingGroup` itself; both fixed. +- **`MixedInstancesPolicy` silently dropped end-to-end**: the backend input struct and + the `AutoScalingGroup` model both already had a `MixedInstancesPolicy` field, but + neither `CreateAutoScalingGroup` nor `UpdateAutoScalingGroup`'s handlers ever parsed + it from the request, and `xmlAutoScalingGroup` didn't even have a field for it in + the response projection. A request specifying a mixed-instances policy (spot+ + on-demand mixes, very common via Terraform) was accepted with 200 OK and then + quietly discarded. Fixed: full parse (launch template + overrides + instances + distribution) on both Create/Update, full XML projection on Describe. +- **`LifecycleHookSpecificationList` never parsed**: `CreateAutoScalingGroupInput` + (the real AWS one) lets you register lifecycle hooks atomically with group + creation - this is exactly the wire shape Terraform's `aws_autoscaling_group` + `initial_lifecycle_hook` block uses. Completely unhandled before this pass; a group + created with initial hooks would come up with **no hooks at all**. Fixed, and wired + into the new launch-hook gating so the group's own initial instances are correctly + gated by a hook registered at creation time. +- **`TrafficSources` never parsed on `CreateAutoScalingGroup`**: only + Attach/DetachTrafficSources touched this field; Create silently dropped an inline + `TrafficSources` list. Fixed (reuses the existing `parseTrafficSources` helper). +- **`PutScheduledUpdateGroupAction`/`BatchPutScheduledUpdateGroupAction` dropped + `StartTime`/`EndTime`**: the backend model and the `DescribeScheduledActions` XML + projection both already had `StartTime`/`EndTime` fields, but neither handler parsed + them from the request - the entire point of a "scheduled" action (when it fires) + was silently discarded on every call. Fixed (parses AWS query-protocol DateTime, + i.e. RFC3339/ISO8601). +- **`ExecutePolicy` ignored `StepScaling` entirely**: regardless of `PolicyType`, the + handler always used the flat `ScalingAdjustment`/`AdjustmentType` fields. Real + `ExecutePolicy` requires `MetricValue`/`BreachThreshold` for a `StepScaling` policy + and uses `(MetricValue-BreachThreshold)` to select which `StepAdjustment` interval's + `ScalingAdjustment` applies. Fixed: parses both fields, validates they're required + for `StepScaling`, and selects the matching step interval + (`MetricIntervalLowerBound`/`UpperBound`, nil meaning unbounded, exactly as AWS + documents it). +- **`ExecutePolicy` scale duplicated (and diverged from) `SetDesiredCapacity`'s + logic**: it called `adjustInstances` directly, bypassing `SuspendedProcesses` + checks, scale-in protection (`removeUnprotectedInstances`), and `instanceIndex` + maintenance that `applyDesiredCapacityChange` already does correctly. Fixed by + routing through the shared helper. +- **`PutLifecycleHook` dropped `NotificationMetadata`**: parsed nowhere despite being + a plain top-level request field and already present on both the backend model and + the XML response type. Fixed. +- **`PutScalingPolicy`/`DescribePolicies` dropped `MetricAggregationType`** (request + and response) and `DescribePolicies` never returned `MinAdjustmentStep`. Fixed. +- **`GetPredictiveScalingForecast` returned an all-empty response** missing the + required `UpdateTime` field entirely and shaping `LoadForecast` as `[]string` + (nowhere close to the real `[]LoadForecast` struct list). A full + `PredictiveScalingMetricSpecification` projection is out of scope (see gaps), but + the response now includes a real `UpdateTime` and a real, non-empty, correctly- + shaped `Timestamps`/`Values` series (naive flat projection at current + `DesiredCapacity` - explicitly documented as a simplification, not a hidden stub). + +### Traps for the next auditor + +- `Instance.LifecycleState` values `"Pending:Wait"` and `"Terminating:Wait"` are real + AWS enum values (`types.LifecycleState`), not placeholders - don't "simplify" them + back to `InService`/removed without re-reading this Notes section. +- An instance appearing in `g.Instances` with `LifecycleState="Terminating:Wait"` + still counts toward `len(g.Instances)` but the group's `DesiredCapacity`/`MinSize` + bookkeeping intentionally has NOT yet been decremented for it - that's deferred to + `finishTermination`. Don't "fix" an apparent capacity/instance-count mismatch during + a wait without checking for a `*:Wait` instance first. +- `ExecutePolicy` calling `b.applyDesiredCapacityChange` instead of its own + `adjustInstances` call is intentional (bug fix, not a regression) - see above. diff --git a/services/autoscaling/backend.go b/services/autoscaling/backend.go index 3009d317d..edc9b9b72 100644 --- a/services/autoscaling/backend.go +++ b/services/autoscaling/backend.go @@ -29,9 +29,19 @@ const ( healthStatusHealthy = "Healthy" // lifecycleStateInService is the lifecycle state for a running, healthy instance. lifecycleStateInService = "InService" + // lifecycleStatePendingWait is the lifecycle state for an instance launching but + // paused for an EC2_INSTANCE_LAUNCHING lifecycle hook to complete. + lifecycleStatePendingWait = "Pending:Wait" + // lifecycleStateTerminatingWait is the lifecycle state for an instance being + // terminated but paused for an EC2_INSTANCE_TERMINATING lifecycle hook to complete. + lifecycleStateTerminatingWait = "Terminating:Wait" + // transitionLaunching and transitionTerminating are the two AWS-defined lifecycle + // hook transition points. + transitionLaunching = "autoscaling:EC2_INSTANCE_LAUNCHING" + transitionTerminating = "autoscaling:EC2_INSTANCE_TERMINATING" // statusCodeSuccessful is the status code for a successfully completed scaling activity. statusCodeSuccessful = "Successful" - // statusInProgress is the status for an in-progress instance refresh. + // statusInProgress is the status for an in-progress instance refresh or scaling activity. statusInProgress = "InProgress" // granularity1Minute is the only supported CloudWatch metric granularity. granularity1Minute = "1Minute" @@ -202,18 +212,20 @@ type CreateAutoScalingGroupInput struct { PlacementGroup string Context string DesiredCapacityType string - Tags []Tag + LoadBalancerNames []string TargetGroupARNs []string TerminationPolicies []string - LoadBalancerNames []string + Tags []Tag AvailabilityZones []string - DesiredCapacity int32 - MaxSize int32 + TrafficSources []TrafficSource + LifecycleHookSpecificationList []LifecycleHook MinSize int32 DefaultCooldown int32 HealthCheckGracePeriod int32 MaxInstanceLifetime int32 DefaultInstanceWarmup int32 + MaxSize int32 + DesiredCapacity int32 NewInstancesProtectedFromScaleIn bool CapacityRebalance bool } @@ -383,7 +395,7 @@ func adjustInstances( // CreateAutoScalingGroup creates a new Auto Scaling group. // -//nolint:funlen // Too complex to refactor given time constraints +//nolint:funlen,cyclop // Too complex to refactor given time constraints func (b *InMemoryBackend) CreateAutoScalingGroup(input CreateAutoScalingGroupInput) (*AutoScalingGroup, error) { b.mu.Lock("CreateAutoScalingGroup") defer b.mu.Unlock() @@ -424,6 +436,21 @@ func (b *InMemoryBackend) CreateAutoScalingGroup(input CreateAutoScalingGroupInp azs = []string{defaultAvailabilityZone} } + normalizedHooks := make([]LifecycleHook, 0, len(input.LifecycleHookSpecificationList)) + + for _, hook := range input.LifecycleHookSpecificationList { + if hook.LifecycleHookName == "" { + return nil, fmt.Errorf("%w: LifecycleHookName is required", ErrInvalidParameter) + } + + hook.AutoScalingGroupName = input.AutoScalingGroupName + if err := normalizeLifecycleHook(&hook); err != nil { + return nil, err + } + + normalizedHooks = append(normalizedHooks, hook) + } + // Use the shared makeInstances helper so all instance IDs use the same format. instances := makeInstances( desired, azs, input.LaunchConfigurationName, @@ -456,6 +483,7 @@ func (b *InMemoryBackend) CreateAutoScalingGroup(input CreateAutoScalingGroupInp AvailabilityZones: azs, LoadBalancerNames: input.LoadBalancerNames, TargetGroupARNs: input.TargetGroupARNs, + TrafficSources: input.TrafficSources, Tags: input.Tags, TerminationPolicies: input.TerminationPolicies, Instances: instances, @@ -470,6 +498,18 @@ func (b *InMemoryBackend) CreateAutoScalingGroup(input CreateAutoScalingGroupInp b.groups[input.AutoScalingGroupName] = group + // Register any initial lifecycle hooks BEFORE gating instances so a launch hook + // specified at creation time applies to the group's own initial instances too. + if len(normalizedHooks) > 0 { + b.lifecycleHooks[input.AutoScalingGroupName] = make(map[string]*LifecycleHook, len(normalizedHooks)) + for _, hook := range normalizedHooks { + cp := hook + b.lifecycleHooks[input.AutoScalingGroupName][cp.LifecycleHookName] = &cp + } + } + + b.gateNewLaunchInstances(group, 0) + for _, inst := range group.Instances { b.instanceIndex[inst.InstanceID] = input.AutoScalingGroupName } @@ -1146,12 +1186,13 @@ func (b *InMemoryBackend) CompleteLifecycleAction(input CompleteLifecycleActionI ErrInvalidParameter, input.LifecycleActionResult) } - // Cancel timer if token is present - if input.LifecycleActionToken != "" { - if action, ok := b.pendingHookTokens[input.LifecycleActionToken]; ok { - action.timer.Stop() - delete(b.pendingHookTokens, input.LifecycleActionToken) - } + action := b.findPendingHookAction( + input.LifecycleActionToken, input.AutoScalingGroupName, input.LifecycleHookName, input.InstanceID, + ) + if action != nil { + action.timer.Stop() + delete(b.pendingHookTokens, action.Token) + b.applyLifecycleResult(action, upper) } return nil @@ -1294,6 +1335,8 @@ func (b *InMemoryBackend) applyDesiredCapacityChange(g *AutoScalingGroup, newDes for _, inst := range g.Instances[oldLen:] { b.instanceIndex[inst.InstanceID] = g.AutoScalingGroupName } + + b.gateNewLaunchInstances(g, oldLen) } } @@ -1379,6 +1422,34 @@ func (b *InMemoryBackend) TerminateInstanceInAutoScalingGroup( return nil, fmt.Errorf("%w: instance %q not found in any auto scaling group", ErrInstanceNotFound, instanceID) } + // When a terminating lifecycle hook is registered, the instance is paused in + // Terminating:Wait until CompleteLifecycleAction is called or the hook's + // HeartbeatTimeout elapses; the actual removal/replacement is deferred to + // finishTermination. This mirrors real AWS behavior instead of terminating + // instantly regardless of configured hooks. + if hook := findHookForTransition(b.lifecycleHooks[groupName], transitionTerminating); hook != nil { + for i := range targetGroup.Instances { + if targetGroup.Instances[i].InstanceID == instanceID { + b.armLifecycleWait(targetGroup, hook, &targetGroup.Instances[i], transitionTerminating, shouldDecrement) + + break + } + } + + activity := ScalingActivity{ + ActivityID: uuid.NewString(), + AutoScalingGroupName: targetGroup.AutoScalingGroupName, + Description: "Terminating EC2 instance: " + instanceID + + " (waiting for lifecycle hook '" + hook.LifecycleHookName + "')", + StatusCode: statusInProgress, + Progress: 50, //nolint:mnd // AWS reports partial progress mid-wait; no finer granularity to model + StartTime: time.Now(), + } + b.activities[groupName] = append(b.activities[groupName], activity) + + return &activity, nil + } + // Remove the instance from the group. newInstances := make([]Instance, 0, len(targetGroup.Instances)-1) @@ -1401,6 +1472,7 @@ func (b *InMemoryBackend) TerminateInstanceInAutoScalingGroup( } } else { // Launch a replacement to maintain DesiredCapacity. + oldLen := len(targetGroup.Instances) targetGroup.Instances = adjustInstances( targetGroup.Instances, targetGroup.DesiredCapacity, @@ -1408,6 +1480,12 @@ func (b *InMemoryBackend) TerminateInstanceInAutoScalingGroup( targetGroup.LaunchConfigurationName, lcInstanceType(b.launchConfigurations, targetGroup.LaunchConfigurationName), ) + + for _, inst := range targetGroup.Instances[oldLen:] { + b.instanceIndex[inst.InstanceID] = targetGroup.AutoScalingGroupName + } + + b.gateNewLaunchInstances(targetGroup, oldLen) } activity := ScalingActivity{ @@ -1444,13 +1522,31 @@ func (b *InMemoryBackend) PutLifecycleHook(hook LifecycleHook) error { return fmt.Errorf("%w: %q", ErrGroupNotFound, hook.AutoScalingGroupName) } + if err := normalizeLifecycleHook(&hook); err != nil { + return err + } + + if b.lifecycleHooks[hook.AutoScalingGroupName] == nil { + b.lifecycleHooks[hook.AutoScalingGroupName] = make(map[string]*LifecycleHook) + } + + cp := hook + b.lifecycleHooks[hook.AutoScalingGroupName][hook.LifecycleHookName] = &cp + + return nil +} + +// normalizeLifecycleHook validates a LifecycleHook's fields and fills in AWS +// defaults (HeartbeatTimeout=3600, DefaultResult=ABANDON, GlobalTimeout=HeartbeatTimeout). +// Shared by PutLifecycleHook and CreateAutoScalingGroup's LifecycleHookSpecificationList. +func normalizeLifecycleHook(hook *LifecycleHook) error { // Validate LifecycleTransition if provided if hook.LifecycleTransition != "" && - hook.LifecycleTransition != "autoscaling:EC2_INSTANCE_LAUNCHING" && - hook.LifecycleTransition != "autoscaling:EC2_INSTANCE_TERMINATING" { + hook.LifecycleTransition != transitionLaunching && + hook.LifecycleTransition != transitionTerminating { return fmt.Errorf( - "%w: LifecycleTransition must be autoscaling:EC2_INSTANCE_LAUNCHING or autoscaling:EC2_INSTANCE_TERMINATING", - ErrInvalidParameter, + "%w: LifecycleTransition must be %s or %s", + ErrInvalidParameter, transitionLaunching, transitionTerminating, ) } @@ -1484,14 +1580,8 @@ func (b *InMemoryBackend) PutLifecycleHook(hook LifecycleHook) error { hook.DefaultResult = lifecycleActionAbandon } - if b.lifecycleHooks[hook.AutoScalingGroupName] == nil { - b.lifecycleHooks[hook.AutoScalingGroupName] = make(map[string]*LifecycleHook) - } - - cp := hook // GlobalTimeout = HeartbeatTimeout * numberOfRetries; AWS uses numberOfRetries=1 by default. - cp.GlobalTimeout = cp.HeartbeatTimeout - b.lifecycleHooks[hook.AutoScalingGroupName][hook.LifecycleHookName] = &cp + hook.GlobalTimeout = hook.HeartbeatTimeout return nil } @@ -1738,11 +1828,256 @@ func (b *InMemoryBackend) cleanupHookTimers(groupName, hookName string) { } } -// expireHookAction removes a pending hook action by token (called from timer callback). -func (b *InMemoryBackend) expireHookAction(token string) { - b.mu.Lock("expireHookAction") +// findHookForTransition returns the first lifecycle hook registered for the given +// group whose LifecycleTransition matches transition, or nil if none is registered. +// AWS allows multiple hooks per transition (each must complete independently); this +// simulation supports one active hook per transition per group, which covers the +// overwhelming majority of real configurations. Must be called with b.mu held. +func findHookForTransition(hooks map[string]*LifecycleHook, transition string) *LifecycleHook { + var found *LifecycleHook + + for _, h := range hooks { + if h.LifecycleTransition == transition { + if found == nil || h.LifecycleHookName < found.LifecycleHookName { + found = h + } + } + } + + return found +} + +// gateNewLaunchInstances transitions newly-appended instances (g.Instances[startIdx:]) +// to Pending:Wait and arms a heartbeat timer when the group has an active +// EC2_INSTANCE_LAUNCHING lifecycle hook. It is a no-op when no such hook exists, so +// callers can invoke it unconditionally after adding instances. Must be called with +// b.mu held (write lock). +func (b *InMemoryBackend) gateNewLaunchInstances(g *AutoScalingGroup, startIdx int) { + hook := findHookForTransition(b.lifecycleHooks[g.AutoScalingGroupName], transitionLaunching) + if hook == nil { + return + } + + for i := startIdx; i < len(g.Instances); i++ { + b.armLifecycleWait(g, hook, &g.Instances[i], transitionLaunching, false) + } +} + +// armLifecycleWait puts inst into the appropriate "Wait" lifecycle state and starts a +// heartbeat timer that resolves the action with the hook's DefaultResult if +// CompleteLifecycleAction/RecordLifecycleActionHeartbeat don't intervene first. Must +// be called with b.mu held (write lock). +func (b *InMemoryBackend) armLifecycleWait( + g *AutoScalingGroup, hook *LifecycleHook, inst *Instance, transition string, shouldDecrement bool, +) { + if transition == transitionLaunching { + inst.LifecycleState = lifecycleStatePendingWait + } else { + inst.LifecycleState = lifecycleStateTerminatingWait + } + + token := uuid.NewString() + timeout := time.Duration(hook.HeartbeatTimeout) * time.Second + + action := &pendingHookAction{ + Token: token, + GroupName: g.AutoScalingGroupName, + HookName: hook.LifecycleHookName, + InstanceID: inst.InstanceID, + Transition: transition, + DefaultResult: hook.DefaultResult, + timeout: timeout, + ShouldDecrement: shouldDecrement, + } + action.timer = time.AfterFunc(timeout, func() { + b.resolveLifecycleWait(token, action.DefaultResult) + }) + b.pendingHookTokens[token] = action +} + +// resolveLifecycleWait applies result (CONTINUE/ABANDON) to the pending action +// identified by token, if it is still pending. Called both from expired timers (its +// own goroutine, hence it takes the lock itself) and, indirectly, from explicit +// CompleteLifecycleAction calls. +func (b *InMemoryBackend) resolveLifecycleWait(token, result string) { + b.mu.Lock("resolveLifecycleWait") defer b.mu.Unlock() + + action, ok := b.pendingHookTokens[token] + if !ok { + return // already resolved (race between timer and an explicit Complete call) + } + delete(b.pendingHookTokens, token) + b.applyLifecycleResult(action, result) +} + +// applyLifecycleResult performs the actual state transition once a lifecycle wait +// resolves (either explicitly via CompleteLifecycleAction or via heartbeat timeout). +// Must be called with b.mu held (write lock). +func (b *InMemoryBackend) applyLifecycleResult(action *pendingHookAction, result string) { + g, ok := b.groups[action.GroupName] + if !ok { + return // group was deleted while the action was pending + } + + switch action.Transition { + case transitionLaunching: + if strings.EqualFold(result, lifecycleActionContinue) { + for i := range g.Instances { + if g.Instances[i].InstanceID == action.InstanceID { + g.Instances[i].LifecycleState = lifecycleStateInService + + break + } + } + } else { + // ABANDON: AWS terminates the instance that failed to launch. + b.removeInstanceByID(g, action.InstanceID) + } + case transitionTerminating: + // Both CONTINUE and ABANDON allow termination to proceed for a terminating + // hook (the result only affects any downstream hook chaining, which this + // simulation does not model). + b.finishTermination(g, action) + } +} + +// removeInstanceByID removes the named instance from the group and its index, if present. +// Must be called with b.mu held (write lock). +func (b *InMemoryBackend) removeInstanceByID(g *AutoScalingGroup, instanceID string) { + for i, inst := range g.Instances { + if inst.InstanceID == instanceID { + g.Instances = append(g.Instances[:i], g.Instances[i+1:]...) + + break + } + } + + delete(b.instanceIndex, instanceID) +} + +// finishTermination completes a deferred termination once its Terminating:Wait hook +// resolves: removes the instance, applies the originally-requested capacity +// adjustment (decrement vs. replacement launch), and records the completion +// activity. Must be called with b.mu held (write lock). +func (b *InMemoryBackend) finishTermination(g *AutoScalingGroup, action *pendingHookAction) { + found := false + + for _, inst := range g.Instances { + if inst.InstanceID == action.InstanceID { + found = true + + break + } + } + + if !found { + return + } + + b.removeInstanceByID(g, action.InstanceID) + + if action.ShouldDecrement { + if g.DesiredCapacity > 0 { + g.DesiredCapacity-- + } + + if g.MinSize > 0 { + g.MinSize-- + } + } else { + oldLen := len(g.Instances) + g.Instances = adjustInstances( + g.Instances, g.DesiredCapacity, g.AvailabilityZones, g.LaunchConfigurationName, + lcInstanceType(b.launchConfigurations, g.LaunchConfigurationName), + ) + + for _, inst := range g.Instances[oldLen:] { + b.instanceIndex[inst.InstanceID] = g.AutoScalingGroupName + } + + b.gateNewLaunchInstances(g, oldLen) + } + + b.activities[g.AutoScalingGroupName] = append(b.activities[g.AutoScalingGroupName], ScalingActivity{ + ActivityID: uuid.NewString(), + AutoScalingGroupName: g.AutoScalingGroupName, + Description: "Terminating EC2 instance: " + action.InstanceID, + StatusCode: statusCodeSuccessful, + Progress: completedProgress, + StartTime: time.Now(), + EndTime: time.Now(), + }) +} + +// rearmPendingWaits re-arms heartbeat timers for any instances left in a lifecycle +// "Wait" state by a restored snapshot. In-flight timers are never persisted (see +// pendingHookTokens/backendSnapshot), so without this an instance restored mid-wait +// would be stuck in Pending:Wait/Terminating:Wait forever. Must be called with b.mu +// held (write lock); intended to run once, right after Restore repopulates b.groups. +func (b *InMemoryBackend) rearmPendingWaits() { + for _, g := range b.groups { + for i := range g.Instances { + inst := &g.Instances[i] + + var transition string + + switch inst.LifecycleState { + case lifecycleStatePendingWait: + transition = transitionLaunching + case lifecycleStateTerminatingWait: + transition = transitionTerminating + default: + continue + } + + hook := findHookForTransition(b.lifecycleHooks[g.AutoScalingGroupName], transition) + + heartbeat := defaultHeartbeatTimeout + defaultResult := lifecycleActionAbandon + hookName := "" + + if hook != nil { + heartbeat = hook.HeartbeatTimeout + defaultResult = hook.DefaultResult + hookName = hook.LifecycleHookName + } + + token := uuid.NewString() + action := &pendingHookAction{ + Token: token, + GroupName: g.AutoScalingGroupName, + HookName: hookName, + InstanceID: inst.InstanceID, + Transition: transition, + DefaultResult: defaultResult, + timeout: time.Duration(heartbeat) * time.Second, + } + action.timer = time.AfterFunc(action.timeout, func() { + b.resolveLifecycleWait(token, action.DefaultResult) + }) + b.pendingHookTokens[token] = action + } + } +} + +// findPendingHookAction looks up a pending lifecycle action, first by explicit token +// (as AWS does) and, when the token is empty, by (groupName, hookName, instanceID) — +// AWS's CompleteLifecycleAction and RecordLifecycleActionHeartbeat both accept either +// a token or an instance ID. Must be called with b.mu held. +func (b *InMemoryBackend) findPendingHookAction(token, groupName, hookName, instanceID string) *pendingHookAction { + if token != "" { + return b.pendingHookTokens[token] + } + + for _, action := range b.pendingHookTokens { + if action.GroupName == groupName && action.HookName == hookName && action.InstanceID == instanceID { + return action + } + } + + return nil } // DescribeAccountLimits returns account limits for Auto Scaling. @@ -2431,16 +2766,17 @@ func (b *InMemoryBackend) RecordLifecycleActionHeartbeat(input RecordLifecycleAc return fmt.Errorf("%w: %q", ErrGroupNotFound, input.AutoScalingGroupName) } - if input.LifecycleActionToken != "" { - if action, ok := b.pendingHookTokens[input.LifecycleActionToken]; ok { - action.timer.Stop() - token := input.LifecycleActionToken - action.timer = time.AfterFunc(action.timeout, func() { - b.expireHookAction(token) - }) + if action := b.findPendingHookAction( + input.LifecycleActionToken, input.AutoScalingGroupName, input.LifecycleHookName, input.InstanceID, + ); action != nil { + action.timer.Stop() + token := action.Token + defaultResult := action.DefaultResult + action.timer = time.AfterFunc(action.timeout, func() { + b.resolveLifecycleWait(token, defaultResult) + }) - return nil - } + return nil } // Validate hook exists @@ -2457,6 +2793,8 @@ func (b *InMemoryBackend) RecordLifecycleActionHeartbeat(input RecordLifecycleAc } // ExecutePolicy executes a scaling policy on the ASG. +// +//nolint:cyclop // StepScaling metric-interval matching adds branching; not worth splitting further. func (b *InMemoryBackend) ExecutePolicy(input ExecutePolicyInput) error { b.mu.Lock("ExecutePolicy") defer b.mu.Unlock() @@ -2484,34 +2822,75 @@ func (b *InMemoryBackend) ExecutePolicy(input ExecutePolicyInput) error { } } + // StepScaling policies select their ScalingAdjustment from StepAdjustments based + // on where (MetricValue - BreachThreshold) falls; both are required in that case + // and unsupported otherwise (matches AWS ExecutePolicy validation). + scalingAdjustment := policy.ScalingAdjustment + + if policy.PolicyType == "StepScaling" { + if input.MetricValue == nil || input.BreachThreshold == nil { + return fmt.Errorf( + "%w: MetricValue and BreachThreshold are required to execute a StepScaling policy", + ErrInvalidParameter, + ) + } + + diff := *input.MetricValue - *input.BreachThreshold + + step, found := findStepAdjustment(policy.StepAdjustments, diff) + if !found { + return fmt.Errorf( + "%w: no step adjustment matches MetricValue %v with BreachThreshold %v", + ErrInvalidParameter, *input.MetricValue, *input.BreachThreshold, + ) + } + + scalingAdjustment = step.ScalingAdjustment + } + var newDesired int32 switch policy.AdjustmentType { case "ExactCapacity": - newDesired = policy.ScalingAdjustment + newDesired = scalingAdjustment case "PercentChangeInCapacity": - pct := float64(g.DesiredCapacity) * float64(policy.ScalingAdjustment) / percentDivisor + pct := float64(g.DesiredCapacity) * float64(scalingAdjustment) / percentDivisor delta := int32(pct) newDesired = g.DesiredCapacity + delta default: // ChangeInCapacity - newDesired = g.DesiredCapacity + policy.ScalingAdjustment + newDesired = g.DesiredCapacity + scalingAdjustment } newDesired = max(g.MinSize, min(g.MaxSize, newDesired)) newDesired = min(newDesired, maxDesiredCapacity) if g.DesiredCapacity != newDesired { - g.DesiredCapacity = newDesired - g.Instances = adjustInstances( - g.Instances, g.DesiredCapacity, g.AvailabilityZones, g.LaunchConfigurationName, - lcInstanceType(b.launchConfigurations, g.LaunchConfigurationName), - ) - g.LastScalingActivity = time.Now() + // Route through applyDesiredCapacityChange (shared with SetDesiredCapacity) + // so ExecutePolicy also respects SuspendedProcesses, scale-in protection, + // instanceIndex bookkeeping, and launch-hook gating instead of duplicating + // (and diverging from) that logic. + b.applyDesiredCapacityChange(g, newDesired) } return nil } +// findStepAdjustment returns the StepAdjustment whose [MetricIntervalLowerBound, +// MetricIntervalUpperBound) interval contains diff (MetricValue-BreachThreshold), and +// whether one was found. A nil bound means unbounded in that direction, matching AWS. +func findStepAdjustment(steps []StepAdjustment, diff float64) (StepAdjustment, bool) { + for _, s := range steps { + lowerOK := s.MetricIntervalLowerBound == nil || diff >= *s.MetricIntervalLowerBound + upperOK := s.MetricIntervalUpperBound == nil || diff < *s.MetricIntervalUpperBound + + if lowerOK && upperOK { + return s, true + } + } + + return StepAdjustment{}, false +} + // LaunchInstances adds new instances to the ASG. func (b *InMemoryBackend) LaunchInstances(groupName string, count int32) ([]Instance, error) { b.mu.Lock("LaunchInstances") @@ -2522,6 +2901,7 @@ func (b *InMemoryBackend) LaunchInstances(groupName string, count int32) ([]Inst return nil, fmt.Errorf("%w: %q", ErrGroupNotFound, groupName) } + oldLen := len(g.Instances) newInstances := makeInstances( count, g.AvailabilityZones, g.LaunchConfigurationName, lcInstanceType(b.launchConfigurations, g.LaunchConfigurationName), @@ -2529,7 +2909,18 @@ func (b *InMemoryBackend) LaunchInstances(groupName string, count int32) ([]Inst g.Instances = append(g.Instances, newInstances...) g.DesiredCapacity = int32(len(g.Instances)) //nolint:gosec // bounded by maxDesiredCapacity - return newInstances, nil + for _, inst := range g.Instances[oldLen:] { + b.instanceIndex[inst.InstanceID] = groupName + } + + b.gateNewLaunchInstances(g, oldLen) + + // Return a copy reflecting any lifecycle-hook gating just applied (e.g. + // Pending:Wait), not the pre-gating snapshot. + result := make([]Instance, len(g.Instances)-oldLen) + copy(result, g.Instances[oldLen:]) + + return result, nil } // GetPredictiveScalingForecast validates the group exists (stub implementation). diff --git a/services/autoscaling/handler.go b/services/autoscaling/handler.go index 9d695e51d..f4897f372 100644 --- a/services/autoscaling/handler.go +++ b/services/autoscaling/handler.go @@ -365,11 +365,15 @@ func (h *Handler) handleCreateAutoScalingGroup(vals url.Values) (any, error) { tags := parseTags(vals, "Tags.member") terminationPolicies := parseMembers(vals, "TerminationPolicies.member") lt := parseLaunchTemplate(vals, "LaunchTemplate") + mip := parseMixedInstancesPolicy(vals) + hooks := parseLifecycleHookSpecifications(vals) + trafficSources := parseTrafficSources(vals) input := CreateAutoScalingGroupInput{ AutoScalingGroupName: name, LaunchConfigurationName: lcName, LaunchTemplate: lt, + MixedInstancesPolicy: mip, VPCZoneIdentifier: vals.Get("VPCZoneIdentifier"), PlacementGroup: vals.Get("PlacementGroup"), Context: vals.Get("Context"), @@ -387,8 +391,10 @@ func (h *Handler) handleCreateAutoScalingGroup(vals url.Values) (any, error) { AvailabilityZones: azs, LoadBalancerNames: lbNames, TargetGroupARNs: targetGroupARNs, + TrafficSources: trafficSources, Tags: tags, TerminationPolicies: terminationPolicies, + LifecycleHookSpecificationList: hooks, } _, createErr := h.Backend.CreateAutoScalingGroup(input) @@ -480,6 +486,7 @@ func (h *Handler) handleUpdateAutoScalingGroup(vals url.Values) (any, error) { AvailabilityZones: parseMembers(vals, "AvailabilityZones.member"), TerminationPolicies: parseMembers(vals, "TerminationPolicies.member"), LaunchTemplate: parseLaunchTemplate(vals, "LaunchTemplate"), + MixedInstancesPolicy: parseMixedInstancesPolicy(vals), } if v := vals.Get("MinSize"); v != "" { @@ -897,6 +904,7 @@ func (h *Handler) handlePutLifecycleHook(vals url.Values) (any, error) { LifecycleTransition: vals.Get("LifecycleTransition"), DefaultResult: vals.Get("DefaultResult"), NotificationTargetARN: vals.Get("NotificationTargetARN"), + NotificationMetadata: vals.Get("NotificationMetadata"), RoleARN: vals.Get("RoleARN"), } @@ -1139,6 +1147,24 @@ func parseIntVal(s string) (int32, error) { return int32(n), nil } +// parseTimeVal parses an AWS query-protocol DateTime value (ISO8601, e.g. +// "2024-01-02T15:04:05Z"). Returns the zero time (and no error) for an empty or +// unparseable string; scheduled-action times are optional AWS request fields, and +// silently dropping an unparseable one matches the historical "omit if invalid" +// behavior of the rest of this handler's optional-field parsing. +func parseTimeVal(s string) time.Time { + if s == "" { + return time.Time{} + } + + t, err := time.Parse(time.RFC3339, s) + if err != nil { + return time.Time{} + } + + return t +} + // parseMembers extracts indexed form values with the given prefix (e.g. "AvailabilityZones.member"). func parseMembers(vals url.Values, prefix string) []string { result := make([]string, 0) @@ -1255,11 +1281,14 @@ func toXMLGroup(g *AutoScalingGroup) xmlAutoScalingGroup { //nolint:funlen // XM } } + xmlMIP := toXMLMixedInstancesPolicy(g.MixedInstancesPolicy) + return xmlAutoScalingGroup{ AutoScalingGroupName: g.AutoScalingGroupName, AutoScalingGroupARN: g.AutoScalingGroupARN, LaunchConfigurationName: g.LaunchConfigurationName, LaunchTemplate: xmlLT, + MixedInstancesPolicy: xmlMIP, VPCZoneIdentifier: g.VPCZoneIdentifier, PlacementGroup: g.PlacementGroup, Context: g.Context, @@ -1292,6 +1321,53 @@ func toXMLGroup(g *AutoScalingGroup) xmlAutoScalingGroup { //nolint:funlen // XM } } +// toXMLMixedInstancesPolicy converts a MixedInstancesPolicy to its XML response +// projection, or nil when policy is nil (matches AWS: the element is entirely absent +// for ASGs that don't use a mixed instances policy). +func toXMLMixedInstancesPolicy(policy *MixedInstancesPolicy) *xmlMixedInstancesPolicy { + if policy == nil { + return nil + } + + overrides := make([]xmlLaunchTemplateOverride, 0, len(policy.LaunchTemplate.Overrides)) + + for _, o := range policy.LaunchTemplate.Overrides { + xo := xmlLaunchTemplateOverride{ + InstanceType: o.InstanceType, + WeightedCapacity: o.WeightedCapacity, + } + + if o.LaunchTemplateSpecification != nil { + xo.LaunchTemplateSpecification = &xmlLaunchTemplateSpecification{ + LaunchTemplateID: o.LaunchTemplateSpecification.LaunchTemplateID, + LaunchTemplateName: o.LaunchTemplateSpecification.LaunchTemplateName, + Version: o.LaunchTemplateSpecification.Version, + } + } + + overrides = append(overrides, xo) + } + + return &xmlMixedInstancesPolicy{ + LaunchTemplate: xmlMixedInstancesLaunchTemplate{ + LaunchTemplateSpecification: xmlLaunchTemplateSpecification{ + LaunchTemplateID: policy.LaunchTemplate.LaunchTemplateSpecification.LaunchTemplateID, + LaunchTemplateName: policy.LaunchTemplate.LaunchTemplateSpecification.LaunchTemplateName, + Version: policy.LaunchTemplate.LaunchTemplateSpecification.Version, + }, + Overrides: xmlLaunchTemplateOverrideList{Members: overrides}, + }, + InstancesDistribution: xmlInstancesDistribution{ + OnDemandAllocationStrategy: policy.InstancesDistribution.OnDemandAllocationStrategy, + SpotAllocationStrategy: policy.InstancesDistribution.SpotAllocationStrategy, + SpotMaxPrice: policy.InstancesDistribution.SpotMaxPrice, + OnDemandBaseCapacity: policy.InstancesDistribution.OnDemandBaseCapacity, + OnDemandPercentageAboveBaseCapacity: policy.InstancesDistribution.OnDemandPercentageAboveBaseCapacity, + SpotInstancePools: policy.InstancesDistribution.SpotInstancePools, + }, + } +} + // toXMLLaunchConfiguration converts a LaunchConfiguration to the XML response type. func toXMLLaunchConfiguration(lc *LaunchConfiguration) xmlLaunchConfiguration { sgs := make([]xmlStringValue, 0, len(lc.SecurityGroups)) @@ -1465,6 +1541,35 @@ type xmlLaunchTemplateSpecification struct { Version string `xml:"Version,omitempty"` } +type xmlLaunchTemplateOverride struct { + LaunchTemplateSpecification *xmlLaunchTemplateSpecification `xml:"LaunchTemplateSpecification,omitempty"` + InstanceType string `xml:"InstanceType,omitempty"` + WeightedCapacity string `xml:"WeightedCapacity,omitempty"` +} + +type xmlLaunchTemplateOverrideList struct { + Members []xmlLaunchTemplateOverride `xml:"member"` +} + +type xmlMixedInstancesLaunchTemplate struct { + LaunchTemplateSpecification xmlLaunchTemplateSpecification `xml:"LaunchTemplateSpecification"` + Overrides xmlLaunchTemplateOverrideList `xml:"Overrides,omitempty"` +} + +type xmlInstancesDistribution struct { + OnDemandAllocationStrategy string `xml:"OnDemandAllocationStrategy,omitempty"` + SpotAllocationStrategy string `xml:"SpotAllocationStrategy,omitempty"` + SpotMaxPrice string `xml:"SpotMaxPrice,omitempty"` + OnDemandBaseCapacity int32 `xml:"OnDemandBaseCapacity,omitempty"` + OnDemandPercentageAboveBaseCapacity int32 `xml:"OnDemandPercentageAboveBaseCapacity,omitempty"` + SpotInstancePools int32 `xml:"SpotInstancePools,omitempty"` +} + +type xmlMixedInstancesPolicy struct { + LaunchTemplate xmlMixedInstancesLaunchTemplate `xml:"LaunchTemplate"` + InstancesDistribution xmlInstancesDistribution `xml:"InstancesDistribution"` +} + type xmlTrafficSource struct { Identifier string `xml:"Identifier"` Type string `xml:"Type,omitempty"` @@ -1480,6 +1585,7 @@ type xmlTerminationPoliciesList struct { type xmlAutoScalingGroup struct { LaunchTemplate *xmlLaunchTemplateSpecification `xml:"LaunchTemplate,omitempty"` + MixedInstancesPolicy *xmlMixedInstancesPolicy `xml:"MixedInstancesPolicy,omitempty"` AutoScalingGroupARN string `xml:"AutoScalingGroupARN"` Status string `xml:"Status,omitempty"` CreatedTime string `xml:"CreatedTime"` @@ -1649,6 +1755,138 @@ func parseLaunchTemplate(vals url.Values, prefix string) *LaunchTemplateSpecific } } +// parseMixedInstancesPolicy parses a MixedInstancesPolicy from form values using the +// standard AWS query-protocol flattening: MixedInstancesPolicy.LaunchTemplate.*, +// MixedInstancesPolicy.LaunchTemplate.Overrides.member.N.*, and +// MixedInstancesPolicy.InstancesDistribution.*. Returns nil if neither a launch +// template nor an instances distribution was specified. +func parseMixedInstancesPolicy(vals url.Values) *MixedInstancesPolicy { + const prefix = "MixedInstancesPolicy" + + lt := parseLaunchTemplate(vals, prefix+".LaunchTemplate.LaunchTemplateSpecification") + overrides := parseLaunchTemplateOverrides(vals, prefix+".LaunchTemplate.Overrides.member") + dist, hasDist := parseInstancesDistribution(vals, prefix+".InstancesDistribution.") + + if lt == nil && len(overrides) == 0 && !hasDist { + return nil + } + + policy := &MixedInstancesPolicy{InstancesDistribution: dist} + if lt != nil { + policy.LaunchTemplate.LaunchTemplateSpecification = *lt + } + + policy.LaunchTemplate.Overrides = overrides + + return policy +} + +// parseLaunchTemplateOverrides parses the LaunchTemplate.Overrides.member.N.* form +// values within a MixedInstancesPolicy. +func parseLaunchTemplateOverrides(vals url.Values, memberPrefix string) []LaunchTemplateOverride { + var overrides []LaunchTemplateOverride + + for i := 1; ; i++ { + op := fmt.Sprintf("%s.%d.", memberPrefix, i) + instanceType := vals.Get(op + "InstanceType") + weighted := vals.Get(op + "WeightedCapacity") + olt := parseLaunchTemplate(vals, op+"LaunchTemplateSpecification") + + if instanceType == "" && weighted == "" && olt == nil { + break + } + + overrides = append(overrides, LaunchTemplateOverride{ + InstanceType: instanceType, + WeightedCapacity: weighted, + LaunchTemplateSpecification: olt, + }) + } + + return overrides +} + +// parseInstancesDistribution parses the InstancesDistribution.* form values within a +// MixedInstancesPolicy. The second return value reports whether any field was set. +func parseInstancesDistribution(vals url.Values, distPrefix string) (InstancesDistribution, bool) { + dist := InstancesDistribution{} + hasDist := false + + if v := vals.Get(distPrefix + "OnDemandAllocationStrategy"); v != "" { + dist.OnDemandAllocationStrategy = v + hasDist = true + } + + if v := vals.Get(distPrefix + "OnDemandBaseCapacity"); v != "" { + if n, err := parseIntVal(v); err == nil { + dist.OnDemandBaseCapacity = n + hasDist = true + } + } + + if v := vals.Get(distPrefix + "OnDemandPercentageAboveBaseCapacity"); v != "" { + if n, err := parseIntVal(v); err == nil { + dist.OnDemandPercentageAboveBaseCapacity = n + hasDist = true + } + } + + if v := vals.Get(distPrefix + "SpotAllocationStrategy"); v != "" { + dist.SpotAllocationStrategy = v + hasDist = true + } + + if v := vals.Get(distPrefix + "SpotInstancePools"); v != "" { + if n, err := parseIntVal(v); err == nil { + dist.SpotInstancePools = n + hasDist = true + } + } + + if v := vals.Get(distPrefix + "SpotMaxPrice"); v != "" { + dist.SpotMaxPrice = v + hasDist = true + } + + return dist, hasDist +} + +// parseLifecycleHookSpecifications parses the LifecycleHookSpecificationList form +// values used by CreateAutoScalingGroup to register hooks atomically with the group. +func parseLifecycleHookSpecifications(vals url.Values) []LifecycleHook { + var result []LifecycleHook + + const prefix = "LifecycleHookSpecificationList.member" + + for i := 1; ; i++ { + p := fmt.Sprintf("%s.%d.", prefix, i) + name := vals.Get(p + "LifecycleHookName") + + if name == "" { + break + } + + hook := LifecycleHook{ + LifecycleHookName: name, + LifecycleTransition: vals.Get(p + "LifecycleTransition"), + DefaultResult: vals.Get(p + "DefaultResult"), + NotificationTargetARN: vals.Get(p + "NotificationTargetARN"), + NotificationMetadata: vals.Get(p + "NotificationMetadata"), + RoleARN: vals.Get(p + "RoleARN"), + } + + if v := vals.Get(p + "HeartbeatTimeout"); v != "" { + if n, err := parseIntVal(v); err == nil { + hook.HeartbeatTimeout = n + } + } + + result = append(result, hook) + } + + return result +} + // parseBlockDeviceMappings parses BlockDeviceMappings from form values. // //nolint:gocognit,nestif // Too complex to refactor given time constraints @@ -1783,6 +2021,8 @@ func parseBatchScheduledActions(vals url.Values) []ScheduledUpdateGroupAction { ScheduledActionName: name, Recurrence: vals.Get(fmt.Sprintf("%s.%d.Recurrence", prefix, i)), TimeZone: vals.Get(fmt.Sprintf("%s.%d.TimeZone", prefix, i)), + StartTime: parseTimeVal(vals.Get(fmt.Sprintf("%s.%d.StartTime", prefix, i))), + EndTime: parseTimeVal(vals.Get(fmt.Sprintf("%s.%d.EndTime", prefix, i))), } if v := vals.Get(fmt.Sprintf("%s.%d.DesiredCapacity", prefix, i)); v != "" { @@ -2619,6 +2859,24 @@ func (h *Handler) handleExecutePolicy(vals url.Values) (any, error) { HonorCooldown: vals.Get("HonorCooldown") == formValueTrue, } + if v := vals.Get("MetricValue"); v != "" { + f, parseErr := strconv.ParseFloat(v, 64) + if parseErr != nil { + return nil, fmt.Errorf("%w: invalid MetricValue", ErrInvalidParameter) + } + + input.MetricValue = &f + } + + if v := vals.Get("BreachThreshold"); v != "" { + f, parseErr := strconv.ParseFloat(v, 64) + if parseErr != nil { + return nil, fmt.Errorf("%w: invalid BreachThreshold", ErrInvalidParameter) + } + + input.BreachThreshold = &f + } + if err := h.Backend.ExecutePolicy(input); err != nil { return nil, err } @@ -2631,15 +2889,19 @@ func (h *Handler) handleExecutePolicy(vals url.Values) (any, error) { func (h *Handler) handleLaunchInstances(vals url.Values) (any, error) { groupName := vals.Get("AutoScalingGroupName") + clientToken := vals.Get("ClientToken") - desiredCapacity, err := parseIntVal(vals.Get("DesiredCapacity")) + // The real API field is RequestedCapacity, not DesiredCapacity (LaunchInstances + // does not affect the group's DesiredCapacity setting the way SetDesiredCapacity + // does; it launches an explicit, additional count of instances). + requestedCapacity, err := parseIntVal(vals.Get("RequestedCapacity")) if err != nil { - return nil, fmt.Errorf("%w: invalid DesiredCapacity", ErrInvalidParameter) + return nil, fmt.Errorf("%w: invalid RequestedCapacity", ErrInvalidParameter) } count := int32(1) - if desiredCapacity > 0 { - count = desiredCapacity + if requestedCapacity > 0 { + count = requestedCapacity } instances, launchErr := h.Backend.LaunchInstances(groupName, count) @@ -2647,26 +2909,61 @@ func (h *Handler) handleLaunchInstances(vals url.Values) (any, error) { return nil, launchErr } - members := make([]xmlInstance, 0, len(instances)) - for _, inst := range instances { - members = append(members, xmlInstance{ - InstanceID: inst.InstanceID, - AvailabilityZone: inst.AvailabilityZone, - LifecycleState: inst.LifecycleState, - HealthStatus: inst.HealthStatus, - LaunchConfigurationName: inst.LaunchConfigurationName, - }) - } - return &launchInstancesResponse{ Xmlns: autoscalingXMLNS, Result: launchInstancesResult{ - Instances: xmlInstanceList{Members: members}, + AutoScalingGroupName: groupName, + ClientToken: clientToken, + Instances: toXMLInstanceCollections(instances), }, ResponseMetadata: xmlResponseMetadata{RequestID: "autoscaling-launch-instances"}, }, nil } +// toXMLInstanceCollections groups launched instances by (AvailabilityZone, +// InstanceType) into the InstanceCollection shape LaunchInstancesOutput actually +// returns (a flat per-instance list with LifecycleState/HealthStatus belongs to +// DescribeAutoScalingGroups/DescribeAutoScalingInstances, not this operation). +func toXMLInstanceCollections(instances []Instance) xmlInstanceCollectionList { + type collectionKey struct{ az, instanceType string } + + order := make([]collectionKey, 0, len(instances)) + grouped := make(map[collectionKey][]string, len(instances)) + + for _, inst := range instances { + key := collectionKey{az: inst.AvailabilityZone, instanceType: inst.InstanceType} + if _, ok := grouped[key]; !ok { + order = append(order, key) + } + + grouped[key] = append(grouped[key], inst.InstanceID) + } + + members := make([]xmlInstanceCollection, 0, len(order)) + + for _, key := range order { + ids := make([]xmlStringValue, 0, len(grouped[key])) + for _, id := range grouped[key] { + ids = append(ids, xmlStringValue{Value: id}) + } + + members = append(members, xmlInstanceCollection{ + AvailabilityZone: key.az, + InstanceType: key.instanceType, + InstanceIDs: xmlStringValueList{Members: ids}, + }) + } + + return xmlInstanceCollectionList{Members: members} +} + +// forecastHorizonHours and forecastIntervalHours bound the synthetic forecast series +// generated below: AWS forecasts up to 2 days ahead in hourly points. +const ( + forecastHorizonHours = 48 + forecastIntervalHours = 1 +) + func (h *Handler) handleGetPredictiveScalingForecast(vals url.Values) (any, error) { groupName := vals.Get("AutoScalingGroupName") @@ -2674,10 +2971,40 @@ func (h *Handler) handleGetPredictiveScalingForecast(vals url.Values) (any, erro return nil, err } + // AWS's actual forecast is produced by a statistical model trained on CloudWatch + // history, which this emulator has no equivalent of. Rather than return an + // all-empty (and required-field-violating) response, project a flat series at the + // group's current DesiredCapacity so callers get well-shaped, non-empty, + // real-derived data. See PARITY.md for the documented simplification. + groups, err := h.Backend.DescribeAutoScalingGroups([]string{groupName}) + if err != nil { + return nil, err + } + + desired := groups[0].DesiredCapacity + now := time.Now().UTC() + + timestamps := make([]xmlStringValue, 0, forecastHorizonHours) + values := make([]xmlStringValue, 0, forecastHorizonHours) + + for i := range forecastHorizonHours { + timestamps = append(timestamps, xmlStringValue{ + Value: now.Add(time.Duration(i*forecastIntervalHours) * time.Hour).Format(time.RFC3339), + }) + values = append(values, xmlStringValue{Value: strconv.FormatInt(int64(desired), 10)}) + } + + series := xmlLoadForecast{ + Timestamps: xmlStringValueList{Members: timestamps}, + Values: xmlStringValueList{Members: values}, + } + return &getPredictiveScalingForecastResponse{ Xmlns: autoscalingXMLNS, Result: getPredictiveScalingForecastResult{ - CapacityForecast: xmlCapacityForecast{}, + LoadForecast: xmlLoadForecastList{Members: []xmlLoadForecast{series}}, + CapacityForecast: xmlCapacityForecast(series), + UpdateTime: now.Format(time.RFC3339), }, ResponseMetadata: xmlResponseMetadata{RequestID: "autoscaling-get-predictive-scaling-forecast"}, }, nil @@ -2816,6 +3143,7 @@ func (h *Handler) handlePutScalingPolicy(vals url.Values) (any, error) { PolicyName: vals.Get("PolicyName"), PolicyType: vals.Get("PolicyType"), AdjustmentType: vals.Get("AdjustmentType"), + MetricAggregationType: vals.Get("MetricAggregationType"), ScalingAdjustment: scalingAdjustment, MinAdjustmentStep: minAdjustmentStep, MinAdjustmentMagnitude: minAdjustmentMagnitude, @@ -2872,7 +3200,9 @@ func (h *Handler) handleDescribePolicies(vals url.Values) (any, error) { AutoScalingGroupName: p.AutoScalingGroupName, PolicyType: p.PolicyType, AdjustmentType: p.AdjustmentType, + MetricAggregationType: p.MetricAggregationType, ScalingAdjustment: p.ScalingAdjustment, + MinAdjustmentStep: p.MinAdjustmentStep, MinAdjustmentMagnitude: p.MinAdjustmentMagnitude, Cooldown: p.Cooldown, } @@ -2921,6 +3251,8 @@ func (h *Handler) handlePutScheduledUpdateGroupAction(vals url.Values) (any, err ScheduledActionName: vals.Get("ScheduledActionName"), Recurrence: vals.Get("Recurrence"), TimeZone: vals.Get("TimeZone"), + StartTime: parseTimeVal(vals.Get("StartTime")), + EndTime: parseTimeVal(vals.Get("EndTime")), } if v := vals.Get("DesiredCapacity"); v != "" { @@ -3368,8 +3700,24 @@ type executePolicyResponse struct { ResponseMetadata xmlResponseMetadata `xml:"ResponseMetadata"` } +// xmlInstanceCollection mirrors the real LaunchInstancesOutput.Instances shape: AWS +// groups newly-launched instances by (AvailabilityZone, InstanceType, ...) and +// reports their IDs together, NOT as a flat per-instance list with lifecycle/health +// fields (that shape belongs to DescribeAutoScalingGroups/DescribeAutoScalingInstances). +type xmlInstanceCollection struct { + AvailabilityZone string `xml:"AvailabilityZone,omitempty"` + InstanceType string `xml:"InstanceType,omitempty"` + InstanceIDs xmlStringValueList `xml:"InstanceIds,omitempty"` +} + +type xmlInstanceCollectionList struct { + Members []xmlInstanceCollection `xml:"member"` +} + type launchInstancesResult struct { - Instances xmlInstanceList `xml:"Instances"` + AutoScalingGroupName string `xml:"AutoScalingGroupName,omitempty"` + ClientToken string `xml:"ClientToken,omitempty"` + Instances xmlInstanceCollectionList `xml:"Instances"` } type launchInstancesResponse struct { @@ -3384,9 +3732,22 @@ type xmlCapacityForecast struct { Values xmlStringValueList `xml:"Values"` } +// xmlLoadForecast mirrors the Timestamps/Values pair shared by CapacityForecast and +// each entry of LoadForecast. A full MetricSpecification projection is out of scope +// (see PARITY.md); the Timestamps/Values series is real, derived data. +type xmlLoadForecast struct { + Timestamps xmlStringValueList `xml:"Timestamps"` + Values xmlStringValueList `xml:"Values"` +} + +type xmlLoadForecastList struct { + Members []xmlLoadForecast `xml:"member"` +} + type getPredictiveScalingForecastResult struct { - LoadForecast []string `xml:"LoadForecast"` + UpdateTime string `xml:"UpdateTime"` CapacityForecast xmlCapacityForecast `xml:"CapacityForecast"` + LoadForecast xmlLoadForecastList `xml:"LoadForecast"` } type getPredictiveScalingForecastResponse struct { @@ -3476,7 +3837,9 @@ type xmlScalingPolicy struct { AutoScalingGroupName string `xml:"AutoScalingGroupName"` PolicyType string `xml:"PolicyType,omitempty"` AdjustmentType string `xml:"AdjustmentType,omitempty"` + MetricAggregationType string `xml:"MetricAggregationType,omitempty"` ScalingAdjustment int32 `xml:"ScalingAdjustment,omitempty"` + MinAdjustmentStep int32 `xml:"MinAdjustmentStep,omitempty"` MinAdjustmentMagnitude int32 `xml:"MinAdjustmentMagnitude,omitempty"` Cooldown int32 `xml:"Cooldown,omitempty"` } diff --git a/services/autoscaling/models.go b/services/autoscaling/models.go index cb53afa64..3a1325749 100644 --- a/services/autoscaling/models.go +++ b/services/autoscaling/models.go @@ -114,7 +114,12 @@ type TrafficSourceState struct { } // ExecutePolicyInput holds the input for ExecutePolicy. +// MetricValue and BreachThreshold are required (and only meaningful) when the +// target policy's PolicyType is StepScaling: AWS uses (MetricValue-BreachThreshold) +// to select which StepAdjustment interval applies. type ExecutePolicyInput struct { + MetricValue *float64 + BreachThreshold *float64 AutoScalingGroupName string PolicyName string HonorCooldown bool @@ -129,16 +134,21 @@ type RecordLifecycleActionHeartbeatInput struct { } // pendingHookAction tracks an in-flight lifecycle action with its timer. -// - +// Transition is one of the transitionLaunching/transitionTerminating constants +// (backend.go) and determines what resolving the action does to the instance. +// ShouldDecrement is only meaningful for terminating actions: it records whether +// the originating TerminateInstanceInAutoScalingGroup call requested a desired +// capacity decrement (vs launching a replacement) once the wait completes. type pendingHookAction struct { - timer *time.Timer - Token string - GroupName string - HookName string - InstanceID string - DefaultResult string - timeout time.Duration + timer *time.Timer + Token string + GroupName string + HookName string + InstanceID string + Transition string + DefaultResult string + timeout time.Duration + ShouldDecrement bool } // LaunchTemplateSpecification identifies an EC2 launch template. diff --git a/services/autoscaling/parity_b_test.go b/services/autoscaling/parity_b_test.go new file mode 100644 index 000000000..7fac9e781 --- /dev/null +++ b/services/autoscaling/parity_b_test.go @@ -0,0 +1,674 @@ +package autoscaling_test + +import ( + "encoding/xml" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// xmlASGInstancesResponse is a minimal struct for parsing DescribeAutoScalingGroups +// focused on instance lifecycle state and capacity fields. +type xmlASGInstancesResponse struct { + XMLName xml.Name `xml:"DescribeAutoScalingGroupsResponse"` + Result struct { + AutoScalingGroups struct { + Members []struct { + AutoScalingGroupName string `xml:"AutoScalingGroupName"` + MixedInstancesPolicy struct { + LaunchTemplate struct { + LaunchTemplateSpecification struct { + LaunchTemplateID string `xml:"LaunchTemplateId"` + } `xml:"LaunchTemplateSpecification"` + Overrides struct { + Members []struct { + InstanceType string `xml:"InstanceType"` + WeightedCapacity string `xml:"WeightedCapacity"` + } `xml:"member"` + } `xml:"Overrides"` + } `xml:"LaunchTemplate"` + InstancesDistribution struct { + SpotAllocationStrategy string `xml:"SpotAllocationStrategy"` + OnDemandBaseCapacity int32 `xml:"OnDemandBaseCapacity"` + } `xml:"InstancesDistribution"` + } `xml:"MixedInstancesPolicy"` + TrafficSources struct { + Members []struct { + Identifier string `xml:"Identifier"` + Type string `xml:"Type"` + } `xml:"member"` + } `xml:"TrafficSources"` + Instances struct { + Members []struct { + InstanceID string `xml:"InstanceId"` + LifecycleState string `xml:"LifecycleState"` + } `xml:"member"` + } `xml:"Instances"` + DesiredCapacity int32 `xml:"DesiredCapacity"` + MinSize int32 `xml:"MinSize"` + } `xml:"member"` + } `xml:"AutoScalingGroups"` + } `xml:"DescribeAutoScalingGroupsResult"` +} + +func describeASGInstances(t *testing.T, body string) xmlASGInstancesResponse { + t.Helper() + + var parsed xmlASGInstancesResponse + require.NoError(t, xml.Unmarshal([]byte(body), &parsed)) + require.Len(t, parsed.Result.AutoScalingGroups.Members, 1) + + return parsed +} + +// Test_LifecycleHookGatesLaunch verifies that an EC2_INSTANCE_LAUNCHING lifecycle +// hook pauses newly-launched instances in Pending:Wait, and that +// CompleteLifecycleAction resolves the wait (CONTINUE -> InService, ABANDON -> +// instance removed). Before this fix, PutLifecycleHook/CompleteLifecycleAction never +// touched real instance state -- hooks were pure bookkeeping with no effect. +func Test_LifecycleHookGatesLaunch(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + result string + wantState string + wantRemoved bool + wantInstanceOK bool + }{ + {name: "continue resolves to InService", result: "CONTINUE", wantState: "InService", wantInstanceOK: true}, + {name: "abandon removes the instance", result: "ABANDON", wantRemoved: true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h := newAutoscalingHandler() + asgName := "launch-hook-asg-" + tc.result + + code, body := doAS(t, h, "CreateAutoScalingGroup", url.Values{ + "AutoScalingGroupName": {asgName}, + "MinSize": {"0"}, + "MaxSize": {"5"}, + "AvailabilityZones.member.1": {"us-east-1a"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "PutLifecycleHook", url.Values{ + "AutoScalingGroupName": {asgName}, + "LifecycleHookName": {"launch-hook"}, + "LifecycleTransition": {"autoscaling:EC2_INSTANCE_LAUNCHING"}, + "HeartbeatTimeout": {"60"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "SetDesiredCapacity", url.Values{ + "AutoScalingGroupName": {asgName}, + "DesiredCapacity": {"1"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "DescribeAutoScalingGroups", url.Values{ + "AutoScalingGroupNames.member.1": {asgName}, + }) + require.Equal(t, 200, code, body) + + parsed := describeASGInstances(t, body) + require.Len(t, parsed.Result.AutoScalingGroups.Members[0].Instances.Members, 1, + "gated instance must still appear in the group while Pending:Wait") + + inst := parsed.Result.AutoScalingGroups.Members[0].Instances.Members[0] + assert.Equal(t, "Pending:Wait", inst.LifecycleState, + "new instance behind an active launch hook must start Pending:Wait, not InService") + + code, body = doAS(t, h, "CompleteLifecycleAction", url.Values{ + "AutoScalingGroupName": {asgName}, + "LifecycleHookName": {"launch-hook"}, + "InstanceId": {inst.InstanceID}, + "LifecycleActionResult": {tc.result}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "DescribeAutoScalingGroups", url.Values{ + "AutoScalingGroupNames.member.1": {asgName}, + }) + require.Equal(t, 200, code, body) + + parsed = describeASGInstances(t, body) + gotInstances := parsed.Result.AutoScalingGroups.Members[0].Instances.Members + + if tc.wantRemoved { + assert.Empty(t, gotInstances, "ABANDON on a launch hook must remove the instance") + + return + } + + require.Len(t, gotInstances, 1) + assert.Equal(t, tc.wantState, gotInstances[0].LifecycleState, + "CompleteLifecycleAction(CONTINUE) must move the instance to InService") + }) + } +} + +// Test_LifecycleHookGatesTermination verifies that an EC2_INSTANCE_TERMINATING +// lifecycle hook defers instance removal: TerminateInstanceInAutoScalingGroup pauses +// the instance in Terminating:Wait (with an InProgress activity) instead of removing +// it immediately, and CompleteLifecycleAction finishes the termination and applies +// the originally-requested desired-capacity decrement. +func Test_LifecycleHookGatesTermination(t *testing.T) { + t.Parallel() + + h := newAutoscalingHandler() + asgName := "terminate-hook-asg" + + code, body := doAS(t, h, "CreateAutoScalingGroup", url.Values{ + "AutoScalingGroupName": {asgName}, + "MinSize": {"0"}, + "MaxSize": {"5"}, + "DesiredCapacity": {"1"}, + "AvailabilityZones.member.1": {"us-east-1a"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "DescribeAutoScalingGroups", url.Values{ + "AutoScalingGroupNames.member.1": {asgName}, + }) + require.Equal(t, 200, code, body) + + parsed := describeASGInstances(t, body) + require.Len(t, parsed.Result.AutoScalingGroups.Members[0].Instances.Members, 1) + instanceID := parsed.Result.AutoScalingGroups.Members[0].Instances.Members[0].InstanceID + + code, body = doAS(t, h, "PutLifecycleHook", url.Values{ + "AutoScalingGroupName": {asgName}, + "LifecycleHookName": {"terminate-hook"}, + "LifecycleTransition": {"autoscaling:EC2_INSTANCE_TERMINATING"}, + "HeartbeatTimeout": {"60"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "TerminateInstanceInAutoScalingGroup", url.Values{ + "InstanceId": {instanceID}, + "ShouldDecrementDesiredCapacity": {"true"}, + }) + require.Equal(t, 200, code, body) + assert.Contains(t, body, "InProgress", + "a deferred termination must report InProgress while waiting on the hook; got: %s", body) + + code, body = doAS(t, h, "DescribeAutoScalingGroups", url.Values{ + "AutoScalingGroupNames.member.1": {asgName}, + }) + require.Equal(t, 200, code, body) + + parsed = describeASGInstances(t, body) + group := parsed.Result.AutoScalingGroups.Members[0] + require.Len(t, group.Instances.Members, 1, "instance must remain present during Terminating:Wait") + assert.Equal(t, "Terminating:Wait", group.Instances.Members[0].LifecycleState) + assert.Equal(t, int32(1), group.DesiredCapacity, "capacity decrement must be deferred until the hook resolves") + + code, body = doAS(t, h, "CompleteLifecycleAction", url.Values{ + "AutoScalingGroupName": {asgName}, + "LifecycleHookName": {"terminate-hook"}, + "InstanceId": {instanceID}, + "LifecycleActionResult": {"CONTINUE"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "DescribeAutoScalingGroups", url.Values{ + "AutoScalingGroupNames.member.1": {asgName}, + }) + require.Equal(t, 200, code, body) + + parsed = describeASGInstances(t, body) + group = parsed.Result.AutoScalingGroups.Members[0] + assert.Empty(t, group.Instances.Members, "instance must be removed once the terminating hook resolves") + assert.Equal(t, int32(0), group.DesiredCapacity, "ShouldDecrementDesiredCapacity must apply once resolved") +} + +// Test_RecordLifecycleActionHeartbeatByInstanceID verifies that +// RecordLifecycleActionHeartbeat can locate a pending action by (group, hook, +// instance) when no LifecycleActionToken is supplied -- AWS accepts either. +func Test_RecordLifecycleActionHeartbeatByInstanceID(t *testing.T) { + t.Parallel() + + h := newAutoscalingHandler() + asgName := "heartbeat-asg" + + code, body := doAS(t, h, "CreateAutoScalingGroup", url.Values{ + "AutoScalingGroupName": {asgName}, + "MinSize": {"0"}, + "MaxSize": {"5"}, + "AvailabilityZones.member.1": {"us-east-1a"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "PutLifecycleHook", url.Values{ + "AutoScalingGroupName": {asgName}, + "LifecycleHookName": {"launch-hook"}, + "LifecycleTransition": {"autoscaling:EC2_INSTANCE_LAUNCHING"}, + "HeartbeatTimeout": {"60"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "SetDesiredCapacity", url.Values{ + "AutoScalingGroupName": {asgName}, + "DesiredCapacity": {"1"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "DescribeAutoScalingGroups", url.Values{ + "AutoScalingGroupNames.member.1": {asgName}, + }) + require.Equal(t, 200, code, body) + + parsed := describeASGInstances(t, body) + instanceID := parsed.Result.AutoScalingGroups.Members[0].Instances.Members[0].InstanceID + + code, body = doAS(t, h, "RecordLifecycleActionHeartbeat", url.Values{ + "AutoScalingGroupName": {asgName}, + "LifecycleHookName": {"launch-hook"}, + "InstanceId": {instanceID}, + }) + require.Equal(t, 200, code, "heartbeat by instance ID (no token) must find the pending action; body: %s", body) + + code, body = doAS(t, h, "RecordLifecycleActionHeartbeat", url.Values{ + "AutoScalingGroupName": {asgName}, + "LifecycleHookName": {"no-such-hook"}, + "InstanceId": {instanceID}, + }) + assert.Equal(t, 400, code, "heartbeat for an unknown hook with no pending action must fail; body: %s", body) +} + +// Test_MixedInstancesPolicyRoundTrip verifies that CreateAutoScalingGroup parses +// MixedInstancesPolicy (launch template, overrides, instances distribution) and that +// DescribeAutoScalingGroups returns it. Previously this field was silently dropped: +// accepted on the wire, never stored, never returned. +func Test_MixedInstancesPolicyRoundTrip(t *testing.T) { + t.Parallel() + + h := newAutoscalingHandler() + asgName := "mip-asg" + + code, body := doAS(t, h, "CreateAutoScalingGroup", url.Values{ + "AutoScalingGroupName": {asgName}, + "MinSize": {"0"}, + "MaxSize": {"5"}, + "AvailabilityZones.member.1": {"us-east-1a"}, + "MixedInstancesPolicy.LaunchTemplate.LaunchTemplateSpecification.LaunchTemplateId": {"lt-0123456789"}, + "MixedInstancesPolicy.LaunchTemplate.LaunchTemplateSpecification.Version": {"$Latest"}, + "MixedInstancesPolicy.LaunchTemplate.Overrides.member.1.InstanceType": {"t3.micro"}, + "MixedInstancesPolicy.LaunchTemplate.Overrides.member.1.WeightedCapacity": {"1"}, + "MixedInstancesPolicy.LaunchTemplate.Overrides.member.2.InstanceType": {"t3.small"}, + "MixedInstancesPolicy.LaunchTemplate.Overrides.member.2.WeightedCapacity": {"2"}, + "MixedInstancesPolicy.InstancesDistribution.OnDemandBaseCapacity": {"1"}, + "MixedInstancesPolicy.InstancesDistribution.SpotAllocationStrategy": {"capacity-optimized"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "DescribeAutoScalingGroups", url.Values{ + "AutoScalingGroupNames.member.1": {asgName}, + }) + require.Equal(t, 200, code, body) + + parsed := describeASGInstances(t, body) + mip := parsed.Result.AutoScalingGroups.Members[0].MixedInstancesPolicy + + assert.Equal(t, "lt-0123456789", mip.LaunchTemplate.LaunchTemplateSpecification.LaunchTemplateID) + require.Len(t, mip.LaunchTemplate.Overrides.Members, 2) + assert.Equal(t, "t3.micro", mip.LaunchTemplate.Overrides.Members[0].InstanceType) + assert.Equal(t, "t3.small", mip.LaunchTemplate.Overrides.Members[1].InstanceType) + assert.Equal(t, int32(1), mip.InstancesDistribution.OnDemandBaseCapacity) + assert.Equal(t, "capacity-optimized", mip.InstancesDistribution.SpotAllocationStrategy) +} + +// xmlLaunchInstancesResponse parses the LaunchInstances response, which returns +// InstanceCollection entries (grouped by AZ/InstanceType with a list of instance +// IDs) rather than a flat per-instance list. +type xmlLaunchInstancesResponse struct { + XMLName xml.Name `xml:"LaunchInstancesResponse"` + Result struct { + AutoScalingGroupName string `xml:"AutoScalingGroupName"` + ClientToken string `xml:"ClientToken"` + Instances struct { + Members []struct { + AvailabilityZone string `xml:"AvailabilityZone"` + InstanceIDs struct { + Members []string `xml:"member"` + } `xml:"InstanceIds"` + } `xml:"member"` + } `xml:"Instances"` + } `xml:"LaunchInstancesResult"` +} + +// Test_LaunchInstancesWireShapeAndIndex verifies that LaunchInstances (a) reads the +// real RequestedCapacity field (not the DesiredCapacity typo it used to read), (b) +// returns the AWS InstanceCollection shape instead of a flat per-instance list, and +// (c) that launched instances are indexed so a subsequent +// TerminateInstanceInAutoScalingGroup can find them by ID (previously LaunchInstances +// never updated the instance index at all). +func Test_LaunchInstancesWireShapeAndIndex(t *testing.T) { + t.Parallel() + + h := newAutoscalingHandler() + asgName := "launch-instances-asg" + + code, body := doAS(t, h, "CreateAutoScalingGroup", url.Values{ + "AutoScalingGroupName": {asgName}, + "MinSize": {"0"}, + "MaxSize": {"10"}, + "AvailabilityZones.member.1": {"us-east-1a"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "LaunchInstances", url.Values{ + "AutoScalingGroupName": {asgName}, + "ClientToken": {"tok-1"}, + "RequestedCapacity": {"2"}, + }) + require.Equal(t, 200, code, body) + + var parsed xmlLaunchInstancesResponse + require.NoError(t, xml.Unmarshal([]byte(body), &parsed)) + assert.Equal(t, asgName, parsed.Result.AutoScalingGroupName) + assert.Equal(t, "tok-1", parsed.Result.ClientToken) + + var instanceIDs []string + for _, m := range parsed.Result.Instances.Members { + instanceIDs = append(instanceIDs, m.InstanceIDs.Members...) + } + + require.Len(t, instanceIDs, 2, "RequestedCapacity=2 must launch exactly 2 instances; got body: %s", body) + + code, body = doAS(t, h, "TerminateInstanceInAutoScalingGroup", url.Values{ + "InstanceId": {instanceIDs[0]}, + "ShouldDecrementDesiredCapacity": {"true"}, + }) + assert.Equal(t, 200, code, + "an instance launched via LaunchInstances must be indexed and terminable by ID; got: %s", body) +} + +// Test_ExecutePolicyStepScaling verifies that ExecutePolicy selects the matching +// StepAdjustment via (MetricValue-BreachThreshold) for a StepScaling policy, and +// rejects execution when those required fields are missing. +func Test_ExecutePolicyStepScaling(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + metricValue string + breachThreshold string + wantCode int + wantDesired int32 + }{ + {name: "low step matches", metricValue: "15", breachThreshold: "10", wantCode: 200, wantDesired: 5}, + { + name: "high step matches unbounded upper", + metricValue: "25", + breachThreshold: "10", + wantCode: 200, + wantDesired: 8, + }, + {name: "missing MetricValue is rejected", metricValue: "", breachThreshold: "10", wantCode: 400}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h := newAutoscalingHandler() + asgName := "step-exec-asg-" + tc.name + + code, body := doAS(t, h, "CreateAutoScalingGroup", url.Values{ + "AutoScalingGroupName": {asgName}, + "MinSize": {"0"}, + "MaxSize": {"20"}, + "DesiredCapacity": {"2"}, + "AvailabilityZones.member.1": {"us-east-1a"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "PutScalingPolicy", url.Values{ + "AutoScalingGroupName": {asgName}, + "PolicyName": {"step1"}, + "PolicyType": {"StepScaling"}, + "AdjustmentType": {"ChangeInCapacity"}, + "StepAdjustments.member.1.ScalingAdjustment": {"3"}, + "StepAdjustments.member.1.MetricIntervalLowerBound": {"0"}, + "StepAdjustments.member.1.MetricIntervalUpperBound": {"10"}, + "StepAdjustments.member.2.ScalingAdjustment": {"6"}, + "StepAdjustments.member.2.MetricIntervalLowerBound": {"10"}, + }) + require.Equal(t, 200, code, body) + + executeVals := url.Values{ + "AutoScalingGroupName": {asgName}, + "PolicyName": {"step1"}, + "BreachThreshold": {tc.breachThreshold}, + } + if tc.metricValue != "" { + executeVals.Set("MetricValue", tc.metricValue) + } + + code, body = doAS(t, h, "ExecutePolicy", executeVals) + require.Equal(t, tc.wantCode, code, body) + + if tc.wantCode != 200 { + return + } + + code, body = doAS(t, h, "DescribeAutoScalingGroups", url.Values{ + "AutoScalingGroupNames.member.1": {asgName}, + }) + require.Equal(t, 200, code, body) + + parsed := describeASGInstances(t, body) + assert.Equal(t, tc.wantDesired, parsed.Result.AutoScalingGroups.Members[0].DesiredCapacity) + }) + } +} + +// Test_ScheduledActionStartEndTimeRoundTrip verifies that PutScheduledUpdateGroupAction +// and BatchPutScheduledUpdateGroupAction both persist StartTime/EndTime -- previously +// both silently dropped these fields, so DescribeScheduledActions never reflected the +// schedule the caller actually requested. +func Test_ScheduledActionStartEndTimeRoundTrip(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + useBatch bool + }{ + {name: "single PutScheduledUpdateGroupAction", useBatch: false}, + {name: "BatchPutScheduledUpdateGroupAction", useBatch: true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h := newAutoscalingHandler() + asgName := "sched-time-asg-" + tc.name + + code, body := doAS(t, h, "CreateAutoScalingGroup", url.Values{ + "AutoScalingGroupName": {asgName}, + "MinSize": {"0"}, + "MaxSize": {"5"}, + "AvailabilityZones.member.1": {"us-east-1a"}, + }) + require.Equal(t, 200, code, body) + + if tc.useBatch { + code, body = doAS(t, h, "BatchPutScheduledUpdateGroupAction", url.Values{ + "AutoScalingGroupName": {asgName}, + "ScheduledUpdateGroupActions.member.1.ScheduledActionName": {"batch-action"}, + "ScheduledUpdateGroupActions.member.1.StartTime": {"2030-01-01T00:00:00Z"}, + "ScheduledUpdateGroupActions.member.1.EndTime": {"2030-01-02T00:00:00Z"}, + "ScheduledUpdateGroupActions.member.1.DesiredCapacity": {"4"}, + }) + } else { + code, body = doAS(t, h, "PutScheduledUpdateGroupAction", url.Values{ + "AutoScalingGroupName": {asgName}, + "ScheduledActionName": {"single-action"}, + "StartTime": {"2030-01-01T00:00:00Z"}, + "EndTime": {"2030-01-02T00:00:00Z"}, + "DesiredCapacity": {"4"}, + }) + } + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "DescribeScheduledActions", url.Values{ + "AutoScalingGroupName": {asgName}, + }) + require.Equal(t, 200, code, body) + + assert.Contains(t, body, "2030-01-01T00:00:00Z", + "StartTime must round-trip; got: %s", body) + assert.Contains(t, body, "2030-01-02T00:00:00Z", + "EndTime must round-trip; got: %s", body) + }) + } +} + +// Test_CreateASGWithInitialLifecycleHooks verifies that CreateAutoScalingGroup +// registers LifecycleHookSpecificationList atomically with group creation, and that +// the group's own initial instances are gated by a matching launch hook -- this is +// the wire shape Terraform's aws_autoscaling_group initial_lifecycle_hook block uses, +// and it was previously accepted and silently discarded. +func Test_CreateASGWithInitialLifecycleHooks(t *testing.T) { + t.Parallel() + + h := newAutoscalingHandler() + asgName := "initial-hook-asg" + + code, body := doAS(t, h, "CreateAutoScalingGroup", url.Values{ + "AutoScalingGroupName": {asgName}, + "MinSize": {"1"}, + "MaxSize": {"5"}, + "DesiredCapacity": {"1"}, + "AvailabilityZones.member.1": {"us-east-1a"}, + "LifecycleHookSpecificationList.member.1.LifecycleHookName": {"init-hook"}, + "LifecycleHookSpecificationList.member.1.LifecycleTransition": {"autoscaling:EC2_INSTANCE_LAUNCHING"}, + "LifecycleHookSpecificationList.member.1.HeartbeatTimeout": {"120"}, + "LifecycleHookSpecificationList.member.1.DefaultResult": {"CONTINUE"}, + "LifecycleHookSpecificationList.member.1.NotificationMetadata": {"boot-meta"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "DescribeLifecycleHooks", url.Values{ + "AutoScalingGroupName": {asgName}, + }) + require.Equal(t, 200, code, body) + assert.Contains(t, body, "init-hook") + assert.Contains(t, body, "boot-meta", + "NotificationMetadata must round-trip; got: %s", body) + assert.Contains(t, body, "120") + + code, body = doAS(t, h, "DescribeAutoScalingGroups", url.Values{ + "AutoScalingGroupNames.member.1": {asgName}, + }) + require.Equal(t, 200, code, body) + + parsed := describeASGInstances(t, body) + require.Len(t, parsed.Result.AutoScalingGroups.Members[0].Instances.Members, 1) + assert.Equal(t, "Pending:Wait", parsed.Result.AutoScalingGroups.Members[0].Instances.Members[0].LifecycleState, + "the group's own initial instance must be gated by a hook registered at creation time") +} + +// Test_CreateASGWithTrafficSources verifies TrafficSources specified at +// CreateAutoScalingGroup time are stored and returned (previously only +// Attach/DetachTrafficSources touched this field; Create silently dropped it). +func Test_CreateASGWithTrafficSources(t *testing.T) { + t.Parallel() + + h := newAutoscalingHandler() + asgName := "ts-create-asg" + + code, body := doAS(t, h, "CreateAutoScalingGroup", url.Values{ + "AutoScalingGroupName": {asgName}, + "MinSize": {"0"}, + "MaxSize": {"5"}, + "AvailabilityZones.member.1": {"us-east-1a"}, + "TrafficSources.member.1.Identifier": {"arn:aws:vpc-lattice:us-east-1:000000000000:targetgroup/tg-123"}, + "TrafficSources.member.1.Type": {"vpc-lattice"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "DescribeAutoScalingGroups", url.Values{ + "AutoScalingGroupNames.member.1": {asgName}, + }) + require.Equal(t, 200, code, body) + + parsed := describeASGInstances(t, body) + require.Len(t, parsed.Result.AutoScalingGroups.Members[0].TrafficSources.Members, 1) + assert.Equal(t, "vpc-lattice", parsed.Result.AutoScalingGroups.Members[0].TrafficSources.Members[0].Type) +} + +// Test_PutScalingPolicyMetricAggregationType verifies MetricAggregationType is +// parsed on PutScalingPolicy and returned by DescribePolicies (previously silently +// dropped on both the request and response side). +func Test_PutScalingPolicyMetricAggregationType(t *testing.T) { + t.Parallel() + + h := newAutoscalingHandler() + asgName := "mat-asg" + + code, body := doAS(t, h, "CreateAutoScalingGroup", url.Values{ + "AutoScalingGroupName": {asgName}, + "MinSize": {"0"}, + "MaxSize": {"5"}, + "AvailabilityZones.member.1": {"us-east-1a"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "PutScalingPolicy", url.Values{ + "AutoScalingGroupName": {asgName}, + "PolicyName": {"agg-policy"}, + "PolicyType": {"StepScaling"}, + "AdjustmentType": {"ChangeInCapacity"}, + "MetricAggregationType": {"Average"}, + "StepAdjustments.member.1.ScalingAdjustment": {"1"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "DescribePolicies", url.Values{ + "AutoScalingGroupName": {asgName}, + }) + require.Equal(t, 200, code, body) + assert.Contains(t, body, "Average", + "MetricAggregationType must round-trip; got: %s", body) +} + +// Test_GetPredictiveScalingForecastNonEmpty verifies the response includes the +// required UpdateTime/CapacityForecast/LoadForecast fields with real, non-empty data +// derived from the group's current DesiredCapacity, instead of an entirely empty +// (required-field-violating) response. +func Test_GetPredictiveScalingForecastNonEmpty(t *testing.T) { + t.Parallel() + + h := newAutoscalingHandler() + asgName := "forecast-asg" + + code, body := doAS(t, h, "CreateAutoScalingGroup", url.Values{ + "AutoScalingGroupName": {asgName}, + "MinSize": {"0"}, + "MaxSize": {"10"}, + "DesiredCapacity": {"3"}, + "AvailabilityZones.member.1": {"us-east-1a"}, + }) + require.Equal(t, 200, code, body) + + code, body = doAS(t, h, "GetPredictiveScalingForecast", url.Values{ + "AutoScalingGroupName": {asgName}, + "PolicyName": {"some-policy"}, + "StartTime": {"2030-01-01T00:00:00Z"}, + "EndTime": {"2030-01-02T00:00:00Z"}, + }) + require.Equal(t, 200, code, body) + + assert.Contains(t, body, "", "UpdateTime is a required output field; got: %s", body) + assert.Contains(t, body, "", "LoadForecast is a required output field; got: %s", body) + assert.Contains(t, body, "3", "forecast values must reflect current DesiredCapacity") +} diff --git a/services/autoscaling/persistence.go b/services/autoscaling/persistence.go index c82ad837b..b40e6f33b 100644 --- a/services/autoscaling/persistence.go +++ b/services/autoscaling/persistence.go @@ -119,6 +119,10 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { } } + // Re-arm heartbeat timers for any instances restored mid lifecycle-hook wait; + // in-flight timers themselves are never persisted (see pendingHookTokens above). + b.rearmPendingWaits() + return nil } diff --git a/services/bedrock/backend.go b/services/bedrock/backend.go index 83208b905..1c3316523 100644 --- a/services/bedrock/backend.go +++ b/services/bedrock/backend.go @@ -9,6 +9,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awserr" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) const ( @@ -395,28 +396,28 @@ type ModelInvocationLoggingConfiguration struct { // InMemoryBackend stores Amazon Bedrock state in memory. type InMemoryBackend struct { - guardrails map[string]*Guardrail - guardrailVersions map[string]*GuardrailVersion // guardrailID+":"+version → version - provisionedModelThroughputs map[string]*ProvisionedModelThroughput - evaluationJobs map[string]*EvaluationJob - automatedReasoningPolicies map[string]*AutomatedReasoningPolicy - arpBuildWorkflows map[string]*AutomatedReasoningPolicyBuildWorkflow // workflowID → workflow - arpTestCases map[string]*AutomatedReasoningPolicyTestCase // testCaseID → test case - arpVersions map[string]*AutomatedReasoningPolicyVersion // policyArn+":"+version → version - arpVersionCountByPolicy map[string]int // policyARN → per-policy version counter - customModels map[string]*CustomModel // modelArn → model - customModelDeployments map[string]*CustomModelDeployment // deploymentArn → deployment - foundationModelAgreements map[string]*FoundationModelAgreement // modelID → agreement - modelCustomizationJobs map[string]*ModelCustomizationJob // jobArn → job - modelCopyJobs map[string]*ModelCopyJob // jobArn → job - modelImportJobs map[string]*ModelImportJob // jobArn → job - inferenceProfiles map[string]*InferenceProfile // profileArn → profile - marketplaceEndpoints map[string]*MarketplaceModelEndpoint // endpointArn → endpoint + guardrails *store.Table[Guardrail] + guardrailVersions *store.Table[GuardrailVersion] // guardrailID+":"+version → version + provisionedModelThroughputs *store.Table[ProvisionedModelThroughput] + evaluationJobs *store.Table[EvaluationJob] + automatedReasoningPolicies *store.Table[AutomatedReasoningPolicy] + arpBuildWorkflows *store.Table[AutomatedReasoningPolicyBuildWorkflow] // workflowID → workflow + arpTestCases *store.Table[AutomatedReasoningPolicyTestCase] // testCaseID → test case + arpVersions *store.Table[AutomatedReasoningPolicyVersion] // policyArn+":"+version → version + arpVersionCountByPolicy map[string]int // policyARN → version counter + customModels *store.Table[CustomModel] // modelArn → model + customModelDeployments *store.Table[CustomModelDeployment] // deploymentArn → deployment + foundationModelAgreements *store.Table[FoundationModelAgreement] // modelID → agreement + modelCustomizationJobs *store.Table[ModelCustomizationJob] // jobArn → job + modelCopyJobs *store.Table[ModelCopyJob] // jobArn → job + modelImportJobs *store.Table[ModelImportJob] // jobArn → job + inferenceProfiles *store.Table[InferenceProfile] // profileArn → profile + marketplaceEndpoints *store.Table[MarketplaceModelEndpoint] // endpointArn → endpoint loggingConfig *ModelInvocationLoggingConfiguration - modelInvocationJobs map[string]*ModelInvocationJob // jobArn → job - promptRouters map[string]*PromptRouter // routerArn → router - enforcedGuardrailConfigs map[string]*EnforcedGuardrailConfig // guardrailID → config - arpAnnotations map[string][]any // policyARN → annotations + modelInvocationJobs *store.Table[ModelInvocationJob] // jobArn → job + promptRouters *store.Table[PromptRouter] // routerArn → router + enforcedGuardrailConfigs *store.Table[EnforcedGuardrailConfig] // guardrailID → config + arpAnnotations map[string][]any // policyARN → annotations useCaseType string useCaseDescription string guardrailsByName map[string]string // guardrail name → ID @@ -431,31 +432,32 @@ type InMemoryBackend struct { marketplaceEndpointsByName map[string]string // endpoint name → ARN promptRoutersByName map[string]string // router name → ARN // Agents - agents map[string]*Agent - agentsByName map[string]string // agentName → agentID - agentActionGroups map[string]*AgentActionGroup // agentID/actionGroupID → group - agentAliases map[string]*AgentAlias // agentID/aliasID → alias - agentKBAssociations map[string]*AgentKnowledgeBaseAssociation // agentID/kbID → assoc - knowledgeBases map[string]*KnowledgeBase // kbID → kb - kbByName map[string]string // kbName → kbID - dataSources map[string]*DataSource // kbID/dsID → ds - ingestionJobs map[string]*IngestionJob // kbID/dsID/jobID → job + agents *store.Table[Agent] + agentsByName map[string]string // agentName → agentID + agentActionGroups *store.Table[AgentActionGroup] // agentID/actionGroupID → group + agentAliases *store.Table[AgentAlias] // agentID/aliasID → alias + agentKBAssociations *store.Table[AgentKnowledgeBaseAssociation] // agentID/kbID → assoc + knowledgeBases *store.Table[KnowledgeBase] // kbID → kb + kbByName map[string]string // kbName → kbID + dataSources *store.Table[DataSource] // kbID/dsID → ds + ingestionJobs *store.Table[IngestionJob] // kbID/dsID/jobID → job // Agents batch-3 additions - flows map[string]*Flow // flowID → flow - flowsByName map[string]string // flowName → flowID - flowAliases map[string]*FlowAlias // flowAliasKey(flowID, aliasID) → alias - flowVersions map[string]map[string]*FlowVersion // flowID → version → flowVersion - flowVersionCounters map[string]int // flowID → next version number - prompts map[string]*Prompt // promptID → prompt - promptsByName map[string]string // promptName → promptID - promptVersions map[string]map[string]*PromptVersion // promptID → version → promptVersion - promptVersionCounters map[string]int // promptID → next version number - agentVersions map[string]map[string]*AgentVersion // agentID → version → agentVersion - agentVersionCounters map[string]int // agentID → next version number - agentCollaborators map[string]map[string]*AgentCollaborator // agentID → collabID → collaborator - kbDocuments map[string]*KnowledgeBaseDocument // kbDocKey → document - agentTags map[string]map[string]string // ARN → tagKey → tagValue - agentMemory map[string][]any // agentID/sessionID → memory entries + flows *store.Table[Flow] // flowID → flow + flowsByName map[string]string // flowName → flowID + flowAliases *store.Table[FlowAlias] // flowAliasKey(flowID, aliasID) → alias + flowVersions map[string]*store.Table[FlowVersion] // flowID → version table (lazy) + flowVersionCounters map[string]int // flowID → next version number + prompts *store.Table[Prompt] // promptID → prompt + promptsByName map[string]string // promptName → promptID + promptVersions map[string]*store.Table[PromptVersion] // promptID → version table (lazy) + promptVersionCounters map[string]int // promptID → next version number + agentVersions map[string]*store.Table[AgentVersion] // agentID → version table (lazy) + agentVersionCounters map[string]int // agentID → next version number + agentCollaborators map[string]*store.Table[AgentCollaborator] // agentID → collaborator table (lazy) + kbDocuments *store.Table[KnowledgeBaseDocument] // kbDocKey → document + agentTags map[string]map[string]string // ARN → tagKey → tagValue + agentMemory map[string][]any // agentID/sessionID → memory entries + registry *store.Registry mu *lockmetrics.RWMutex accountID string region string @@ -492,66 +494,38 @@ type InMemoryBackend struct { // NewInMemoryBackend creates a new InMemoryBackend pre-seeded with foundation models. func NewInMemoryBackend(accountID, region string) *InMemoryBackend { b := &InMemoryBackend{ - guardrails: make(map[string]*Guardrail), - guardrailVersions: make(map[string]*GuardrailVersion), - provisionedModelThroughputs: make(map[string]*ProvisionedModelThroughput), - evaluationJobs: make(map[string]*EvaluationJob), - automatedReasoningPolicies: make(map[string]*AutomatedReasoningPolicy), - arpBuildWorkflows: make(map[string]*AutomatedReasoningPolicyBuildWorkflow), - arpTestCases: make(map[string]*AutomatedReasoningPolicyTestCase), - arpVersions: make(map[string]*AutomatedReasoningPolicyVersion), - arpVersionCountByPolicy: make(map[string]int), - customModels: make(map[string]*CustomModel), - customModelDeployments: make(map[string]*CustomModelDeployment), - foundationModelAgreements: make(map[string]*FoundationModelAgreement), - modelCustomizationJobs: make(map[string]*ModelCustomizationJob), - modelCopyJobs: make(map[string]*ModelCopyJob), - modelImportJobs: make(map[string]*ModelImportJob), - inferenceProfiles: make(map[string]*InferenceProfile), - marketplaceEndpoints: make(map[string]*MarketplaceModelEndpoint), - guardrailsByName: make(map[string]string), - guardrailsByARN: make(map[string]string), - pmtsByName: make(map[string]string), - arpByName: make(map[string]string), - customModelsByName: make(map[string]string), - customModelDeployByName: make(map[string]string), - evaluationJobsByName: make(map[string]string), - customizationJobsByName: make(map[string]string), - inferenceProfilesByName: make(map[string]string), - marketplaceEndpointsByName: make(map[string]string), - modelInvocationJobs: make(map[string]*ModelInvocationJob), - promptRouters: make(map[string]*PromptRouter), - enforcedGuardrailConfigs: make(map[string]*EnforcedGuardrailConfig), - arpAnnotations: make(map[string][]any), - promptRoutersByName: make(map[string]string), - agents: make(map[string]*Agent), - agentsByName: make(map[string]string), - agentActionGroups: make(map[string]*AgentActionGroup), - agentAliases: make(map[string]*AgentAlias), - agentKBAssociations: make(map[string]*AgentKnowledgeBaseAssociation), - knowledgeBases: make(map[string]*KnowledgeBase), - kbByName: make(map[string]string), - dataSources: make(map[string]*DataSource), - ingestionJobs: make(map[string]*IngestionJob), - flows: make(map[string]*Flow), - flowsByName: make(map[string]string), - flowAliases: make(map[string]*FlowAlias), - flowVersions: make(map[string]map[string]*FlowVersion), - flowVersionCounters: make(map[string]int), - prompts: make(map[string]*Prompt), - promptsByName: make(map[string]string), - promptVersions: make(map[string]map[string]*PromptVersion), - promptVersionCounters: make(map[string]int), - agentVersions: make(map[string]map[string]*AgentVersion), - agentVersionCounters: make(map[string]int), - agentCollaborators: make(map[string]map[string]*AgentCollaborator), - kbDocuments: make(map[string]*KnowledgeBaseDocument), - agentTags: make(map[string]map[string]string), - agentMemory: make(map[string][]any), - accountID: accountID, - region: region, - mu: lockmetrics.New("bedrock"), - } + arpVersionCountByPolicy: make(map[string]int), + guardrailsByName: make(map[string]string), + guardrailsByARN: make(map[string]string), + pmtsByName: make(map[string]string), + arpByName: make(map[string]string), + customModelsByName: make(map[string]string), + customModelDeployByName: make(map[string]string), + evaluationJobsByName: make(map[string]string), + customizationJobsByName: make(map[string]string), + inferenceProfilesByName: make(map[string]string), + marketplaceEndpointsByName: make(map[string]string), + arpAnnotations: make(map[string][]any), + promptRoutersByName: make(map[string]string), + agentsByName: make(map[string]string), + kbByName: make(map[string]string), + flowsByName: make(map[string]string), + flowVersions: make(map[string]*store.Table[FlowVersion]), + flowVersionCounters: make(map[string]int), + promptsByName: make(map[string]string), + promptVersions: make(map[string]*store.Table[PromptVersion]), + promptVersionCounters: make(map[string]int), + agentVersions: make(map[string]*store.Table[AgentVersion]), + agentVersionCounters: make(map[string]int), + agentCollaborators: make(map[string]*store.Table[AgentCollaborator]), + agentTags: make(map[string]map[string]string), + agentMemory: make(map[string][]any), + accountID: accountID, + region: region, + mu: lockmetrics.New("bedrock"), + registry: store.NewRegistry(), + } + registerAllTables(b) b.seedFoundationModels() return b @@ -564,27 +538,22 @@ func (b *InMemoryBackend) Region() string { return b.region } // The accountID, region, and seeded foundation models are preserved. // //nolint:funlen // Reset must clear all maps; the length is proportional to the number of resource types +//nolint:funlen // Reset must clear all state; the length is proportional to the number of resource types func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.guardrails = make(map[string]*Guardrail) - b.guardrailVersions = make(map[string]*GuardrailVersion) - b.provisionedModelThroughputs = make(map[string]*ProvisionedModelThroughput) - b.evaluationJobs = make(map[string]*EvaluationJob) - b.automatedReasoningPolicies = make(map[string]*AutomatedReasoningPolicy) - b.arpBuildWorkflows = make(map[string]*AutomatedReasoningPolicyBuildWorkflow) - b.arpTestCases = make(map[string]*AutomatedReasoningPolicyTestCase) - b.arpVersions = make(map[string]*AutomatedReasoningPolicyVersion) + // registry.ResetAll empties every registered table in place -- including + // every per-parent flowVersions/promptVersions/agentVersions/ + // agentCollaborators table lazily registered so far. It deliberately does + // NOT unregister them: b.flowVersions/b.promptVersions/b.agentVersions/ + // b.agentCollaborators keep pointing at the same *store.Table instances, + // so a parent ID touched again after Reset reuses its existing + // registration instead of re-registering under an already-used name + // (which would panic -- see store.Register). Mirrors services/kms's + // Reset (keysStore/aliasesStore/grantsRegion/customKeyStoresStore). + b.registry.ResetAll() b.arpVersionCountByPolicy = make(map[string]int) - b.customModels = make(map[string]*CustomModel) - b.customModelDeployments = make(map[string]*CustomModelDeployment) - b.foundationModelAgreements = make(map[string]*FoundationModelAgreement) - b.modelCustomizationJobs = make(map[string]*ModelCustomizationJob) - b.modelCopyJobs = make(map[string]*ModelCopyJob) - b.modelImportJobs = make(map[string]*ModelImportJob) - b.inferenceProfiles = make(map[string]*InferenceProfile) - b.marketplaceEndpoints = make(map[string]*MarketplaceModelEndpoint) b.loggingConfig = nil b.guardrailsByName = make(map[string]string) b.guardrailsByARN = make(map[string]string) @@ -596,34 +565,19 @@ func (b *InMemoryBackend) Reset() { b.customizationJobsByName = make(map[string]string) b.inferenceProfilesByName = make(map[string]string) b.marketplaceEndpointsByName = make(map[string]string) - b.agents = make(map[string]*Agent) b.agentsByName = make(map[string]string) - b.agentActionGroups = make(map[string]*AgentActionGroup) - b.agentAliases = make(map[string]*AgentAlias) - b.agentKBAssociations = make(map[string]*AgentKnowledgeBaseAssociation) - b.knowledgeBases = make(map[string]*KnowledgeBase) b.kbByName = make(map[string]string) - b.dataSources = make(map[string]*DataSource) - b.ingestionJobs = make(map[string]*IngestionJob) b.agentCounter = 0 b.actionGroupCounter = 0 b.agentAliasCounter = 0 b.kbCounter = 0 b.dataSourceCounter = 0 b.ingestionJobCounter = 0 - b.flows = make(map[string]*Flow) b.flowsByName = make(map[string]string) - b.flowAliases = make(map[string]*FlowAlias) - b.flowVersions = make(map[string]map[string]*FlowVersion) b.flowVersionCounters = make(map[string]int) - b.prompts = make(map[string]*Prompt) b.promptsByName = make(map[string]string) - b.promptVersions = make(map[string]map[string]*PromptVersion) b.promptVersionCounters = make(map[string]int) - b.agentVersions = make(map[string]map[string]*AgentVersion) b.agentVersionCounters = make(map[string]int) - b.agentCollaborators = make(map[string]map[string]*AgentCollaborator) - b.kbDocuments = make(map[string]*KnowledgeBaseDocument) b.agentTags = make(map[string]map[string]string) b.agentMemory = make(map[string][]any) b.flowCounter = 0 @@ -646,9 +600,6 @@ func (b *InMemoryBackend) Reset() { b.marketplaceEndpointCounter = 0 b.modelInvocationJobCounter = 0 b.promptRouterCounter = 0 - b.modelInvocationJobs = make(map[string]*ModelInvocationJob) - b.promptRouters = make(map[string]*PromptRouter) - b.enforcedGuardrailConfigs = make(map[string]*EnforcedGuardrailConfig) b.arpAnnotations = make(map[string][]any) b.promptRoutersByName = make(map[string]string) b.useCaseType = "" @@ -826,7 +777,7 @@ func (b *InMemoryBackend) CreateGuardrail( CreatedAt: now, UpdatedAt: now, } - b.guardrails[id] = g + b.guardrails.Put(g) b.guardrailsByName[name] = id b.guardrailsByARN[guardrailARN] = id cp := *g @@ -861,9 +812,9 @@ func (b *InMemoryBackend) ListGuardrails( b.mu.RLock("ListGuardrails") defer b.mu.RUnlock() - list := make([]*GuardrailSummary, 0, len(b.guardrails)) + list := make([]*GuardrailSummary, 0, b.guardrails.Len()) - for _, g := range b.guardrails { + for _, g := range b.guardrails.All() { if guardrailIdentifier != "" && g.GuardrailID != guardrailIdentifier && g.GuardrailArn != guardrailIdentifier && @@ -891,14 +842,19 @@ func (b *InMemoryBackend) ListGuardrails( // hasPublishedVersions reports whether a guardrail has any published (non-DRAFT) versions. // Caller must hold at least a read lock. func (b *InMemoryBackend) hasPublishedVersions(guardrailID string) bool { - for key := range b.guardrailVersions { - prefix := guardrailID + ":" - if len(key) > len(prefix) && key[:len(prefix)] == prefix { - return true + found := false + + b.guardrailVersions.Range(func(v *GuardrailVersion) bool { + if v.GuardrailID == guardrailID { + found = true + + return false } - } - return false + return true + }) + + return found } // UpdateGuardrail updates a guardrail's name, description, messaging, and policies. @@ -969,7 +925,7 @@ func (b *InMemoryBackend) DeleteGuardrail(idOrARN string) error { return fmt.Errorf("%w: guardrail %s not found", ErrNotFound, idOrARN) } - delete(b.guardrails, g.GuardrailID) + b.guardrails.Delete(g.GuardrailID) delete(b.guardrailsByName, g.Name) delete(b.guardrailsByARN, g.GuardrailArn) @@ -979,12 +935,12 @@ func (b *InMemoryBackend) DeleteGuardrail(idOrARN string) error { // findGuardrailByIDOrARN finds a guardrail by ID or ARN. // Caller must hold at least a read lock. func (b *InMemoryBackend) findGuardrailByIDOrARN(idOrARN string) (*Guardrail, bool) { - if g, ok := b.guardrails[idOrARN]; ok { + if g, ok := b.guardrails.Get(idOrARN); ok { return g, true } if id, ok := b.guardrailsByARN[idOrARN]; ok { - return b.guardrails[id], true + return b.guardrails.Get(id) } return nil, false @@ -1074,7 +1030,7 @@ func (b *InMemoryBackend) CreateProvisionedModelThroughput( LastModifiedTime: now, Tags: tagsCopy, } - b.provisionedModelThroughputs[pmtARN] = pmt + b.provisionedModelThroughputs.Put(pmt) b.pmtsByName[name] = pmtARN cp := *pmt @@ -1108,9 +1064,9 @@ func (b *InMemoryBackend) ListProvisionedModelThroughputs( b.mu.RLock("ListProvisionedModelThroughputs") defer b.mu.RUnlock() - list := make([]*ProvisionedModelThroughput, 0, len(b.provisionedModelThroughputs)) + list := make([]*ProvisionedModelThroughput, 0, b.provisionedModelThroughputs.Len()) - for _, pmt := range b.provisionedModelThroughputs { + for _, pmt := range b.provisionedModelThroughputs.All() { cp := *pmt list = append(list, &cp) } @@ -1165,7 +1121,7 @@ func (b *InMemoryBackend) DeleteProvisionedModelThroughput(idOrARN string) error return fmt.Errorf("%w: provisioned model throughput %s not found", ErrNotFound, idOrARN) } - delete(b.provisionedModelThroughputs, pmt.ProvisionedModelArn) + b.provisionedModelThroughputs.Delete(pmt.ProvisionedModelArn) delete(b.pmtsByName, pmt.ProvisionedModelName) return nil @@ -1180,7 +1136,7 @@ func (b *InMemoryBackend) AdvanceProvisionedModelThroughputStatuses() int { now := time.Now().UTC() advanced := 0 - for _, pmt := range b.provisionedModelThroughputs { + for _, pmt := range b.provisionedModelThroughputs.All() { if pmt.Status != statusCreating { continue } @@ -1197,12 +1153,12 @@ func (b *InMemoryBackend) AdvanceProvisionedModelThroughputStatuses() int { // findPMTByIDOrARN finds a provisioned model throughput by ID or ARN. // Caller must hold at least a read lock. func (b *InMemoryBackend) findPMTByIDOrARN(idOrARN string) (*ProvisionedModelThroughput, bool) { - if pmt, ok := b.provisionedModelThroughputs[idOrARN]; ok { + if pmt, ok := b.provisionedModelThroughputs.Get(idOrARN); ok { return pmt, true } if pmtARN, ok := b.pmtsByName[idOrARN]; ok { - return b.provisionedModelThroughputs[pmtARN], true + return b.provisionedModelThroughputs.Get(pmtARN) } return nil, false @@ -1336,37 +1292,37 @@ func (b *InMemoryBackend) TagResource(resourceARN string, tags []Tag) error { defer b.mu.Unlock() if id, ok := b.guardrailsByARN[resourceARN]; ok { - g := b.guardrails[id] + g, _ := b.guardrails.Get(id) g.Tags = mergeTags(g.Tags, tags) return nil } - if pmt, ok := b.provisionedModelThroughputs[resourceARN]; ok { + if pmt, ok := b.provisionedModelThroughputs.Get(resourceARN); ok { pmt.Tags = mergeTags(pmt.Tags, tags) return nil } - if job, ok := b.evaluationJobs[resourceARN]; ok { + if job, ok := b.evaluationJobs.Get(resourceARN); ok { job.Tags = mergeTags(job.Tags, tags) return nil } - if policy, ok := b.automatedReasoningPolicies[resourceARN]; ok { + if policy, ok := b.automatedReasoningPolicies.Get(resourceARN); ok { policy.Tags = mergeTags(policy.Tags, tags) return nil } - if model, ok := b.customModels[resourceARN]; ok { + if model, ok := b.customModels.Get(resourceARN); ok { model.Tags = mergeTags(model.Tags, tags) return nil } - if deployment, ok := b.customModelDeployments[resourceARN]; ok { + if deployment, ok := b.customModelDeployments.Get(resourceARN); ok { deployment.Tags = mergeTags(deployment.Tags, tags) return nil @@ -1386,37 +1342,37 @@ func (b *InMemoryBackend) UntagResource(resourceARN string, tagKeys []string) er } if id, ok := b.guardrailsByARN[resourceARN]; ok { - g := b.guardrails[id] + g, _ := b.guardrails.Get(id) g.Tags = filterTags(g.Tags, removeSet) return nil } - if pmt, ok := b.provisionedModelThroughputs[resourceARN]; ok { + if pmt, ok := b.provisionedModelThroughputs.Get(resourceARN); ok { pmt.Tags = filterTags(pmt.Tags, removeSet) return nil } - if job, ok := b.evaluationJobs[resourceARN]; ok { + if job, ok := b.evaluationJobs.Get(resourceARN); ok { job.Tags = filterTags(job.Tags, removeSet) return nil } - if policy, ok := b.automatedReasoningPolicies[resourceARN]; ok { + if policy, ok := b.automatedReasoningPolicies.Get(resourceARN); ok { policy.Tags = filterTags(policy.Tags, removeSet) return nil } - if model, ok := b.customModels[resourceARN]; ok { + if model, ok := b.customModels.Get(resourceARN); ok { model.Tags = filterTags(model.Tags, removeSet) return nil } - if deployment, ok := b.customModelDeployments[resourceARN]; ok { + if deployment, ok := b.customModelDeployments.Get(resourceARN); ok { deployment.Tags = filterTags(deployment.Tags, removeSet) return nil @@ -1429,26 +1385,28 @@ func (b *InMemoryBackend) UntagResource(resourceARN string, tagKeys []string) er // Caller must hold at least a read lock. func (b *InMemoryBackend) findTagsByARN(resourceARN string) ([]Tag, bool) { if id, ok := b.guardrailsByARN[resourceARN]; ok { - return b.guardrails[id].Tags, true + g, _ := b.guardrails.Get(id) + + return g.Tags, true } - if pmt, ok := b.provisionedModelThroughputs[resourceARN]; ok { + if pmt, ok := b.provisionedModelThroughputs.Get(resourceARN); ok { return pmt.Tags, true } - if job, ok := b.evaluationJobs[resourceARN]; ok { + if job, ok := b.evaluationJobs.Get(resourceARN); ok { return job.Tags, true } - if policy, ok := b.automatedReasoningPolicies[resourceARN]; ok { + if policy, ok := b.automatedReasoningPolicies.Get(resourceARN); ok { return policy.Tags, true } - if model, ok := b.customModels[resourceARN]; ok { + if model, ok := b.customModels.Get(resourceARN); ok { return model.Tags, true } - if deployment, ok := b.customModelDeployments[resourceARN]; ok { + if deployment, ok := b.customModelDeployments.Get(resourceARN); ok { return deployment.Tags, true } @@ -1507,7 +1465,7 @@ func (b *InMemoryBackend) CreateEvaluationJob( job.EvaluationConfig = opt.EvalConfig } - b.evaluationJobs[jobARN] = job + b.evaluationJobs.Put(job) b.evaluationJobsByName[name] = jobARN cp := *job cp.Tags = copyTags(job.Tags) @@ -1544,7 +1502,7 @@ func (b *InMemoryBackend) BatchDeleteEvaluationJob(jobARNs []string) ( deleted := make([]BatchDeleteEvaluationJobItem, 0, len(jobARNs)) for _, jobARN := range jobARNs { - job, ok := b.evaluationJobs[jobARN] + job, ok := b.evaluationJobs.Get(jobARN) if !ok { errs = append(errs, BatchDeleteEvaluationJobError{ JobARN: jobARN, @@ -1566,7 +1524,7 @@ func (b *InMemoryBackend) BatchDeleteEvaluationJob(jobARNs []string) ( } delete(b.evaluationJobsByName, job.JobName) - delete(b.evaluationJobs, jobARN) + b.evaluationJobs.Delete(jobARN) deleted = append(deleted, BatchDeleteEvaluationJobItem{ JobARN: jobARN, Status: "Deleting", @@ -1619,7 +1577,7 @@ func (b *InMemoryBackend) CreateAutomatedReasoningPolicy( UpdatedAt: now, Tags: copyTags(tags), } - b.automatedReasoningPolicies[policyARN] = policy + b.automatedReasoningPolicies.Put(policy) b.arpByName[name] = policyARN cp := *policy cp.Tags = copyTags(policy.Tags) @@ -1634,11 +1592,11 @@ func (b *InMemoryBackend) CancelAutomatedReasoningPolicyBuildWorkflow( b.mu.Lock("CancelAutomatedReasoningPolicyBuildWorkflow") defer b.mu.Unlock() - if _, ok := b.automatedReasoningPolicies[policyARN]; !ok { + if _, ok := b.automatedReasoningPolicies.Get(policyARN); !ok { return fmt.Errorf("%w: automated reasoning policy %s not found", ErrNotFound, policyARN) } - wf, ok := b.arpBuildWorkflows[workflowID] + wf, ok := b.arpBuildWorkflows.Get(workflowID) if !ok { return fmt.Errorf("%w: build workflow %s not found", ErrNotFound, workflowID) } @@ -1664,7 +1622,7 @@ func (b *InMemoryBackend) CreateAutomatedReasoningPolicyTestCase( b.mu.Lock("CreateAutomatedReasoningPolicyTestCase") defer b.mu.Unlock() - if _, ok := b.automatedReasoningPolicies[policyARN]; !ok { + if _, ok := b.automatedReasoningPolicies.Get(policyARN); !ok { return nil, fmt.Errorf( "%w: automated reasoning policy %s not found", ErrNotFound, @@ -1677,7 +1635,7 @@ func (b *InMemoryBackend) CreateAutomatedReasoningPolicyTestCase( TestCaseID: id, PolicyArn: policyARN, } - b.arpTestCases[id] = tc + b.arpTestCases.Put(tc) cp := *tc return &cp, nil @@ -1690,7 +1648,7 @@ func (b *InMemoryBackend) CreateAutomatedReasoningPolicyVersion( b.mu.Lock("CreateAutomatedReasoningPolicyVersion") defer b.mu.Unlock() - policy, ok := b.automatedReasoningPolicies[policyARN] + policy, ok := b.automatedReasoningPolicies.Get(policyARN) if !ok { return nil, fmt.Errorf( "%w: automated reasoning policy %s not found", @@ -1710,8 +1668,7 @@ func (b *InMemoryBackend) CreateAutomatedReasoningPolicyVersion( Version: versionNum, CreatedAt: time.Now().UTC(), } - key := policyARN + ":" + versionNum - b.arpVersions[key] = version + b.arpVersions.Put(version) cp := *version return &cp, nil @@ -1741,7 +1698,7 @@ func (b *InMemoryBackend) CreateCustomModel(modelName string, tags []Tag) (*Cust CreationTime: time.Now().UTC(), Tags: copyTags(tags), } - b.customModels[modelARN] = model + b.customModels.Put(model) b.customModelsByName[modelName] = modelARN cp := *model cp.Tags = copyTags(model.Tags) @@ -1788,7 +1745,7 @@ func (b *InMemoryBackend) CreateCustomModelDeployment( LastModifiedTime: now, Tags: copyTags(tags), } - b.customModelDeployments[deploymentARN] = deployment + b.customModelDeployments.Put(deployment) b.customModelDeployByName[deploymentName] = deploymentARN cp := *deployment cp.Tags = copyTags(deployment.Tags) @@ -1812,7 +1769,7 @@ func (b *InMemoryBackend) CreateFoundationModelAgreement( agreement := &FoundationModelAgreement{ ModelID: modelID, } - b.foundationModelAgreements[modelID] = agreement + b.foundationModelAgreements.Put(agreement) cp := *agreement return &cp, nil @@ -1823,7 +1780,7 @@ func (b *InMemoryBackend) CreateFoundationModelAgreement( // findCustomModelARN resolves a custom model ID or name to its ARN. // Caller must hold at least a read lock. func (b *InMemoryBackend) findCustomModelARN(idOrARN string) (string, bool) { - if _, ok := b.customModels[idOrARN]; ok { + if _, ok := b.customModels.Get(idOrARN); ok { return idOrARN, true } @@ -1853,7 +1810,7 @@ func (b *InMemoryBackend) GetCustomModel(idOrARN string) (*CustomModel, error) { return nil, err } - m := b.customModels[modelARN] + m, _ := b.customModels.Get(modelARN) cp := *m cp.Tags = copyTags(m.Tags) @@ -1865,9 +1822,9 @@ func (b *InMemoryBackend) ListCustomModels(nextToken string) ([]*CustomModel, st b.mu.RLock("ListCustomModels") defer b.mu.RUnlock() - list := make([]*CustomModel, 0, len(b.customModels)) + list := make([]*CustomModel, 0, b.customModels.Len()) - for _, m := range b.customModels { + for _, m := range b.customModels.All() { cp := *m cp.Tags = copyTags(m.Tags) list = append(list, &cp) @@ -1888,9 +1845,9 @@ func (b *InMemoryBackend) DeleteCustomModel(idOrARN string) error { return err } - m := b.customModels[modelARN] + m, _ := b.customModels.Get(modelARN) delete(b.customModelsByName, m.ModelName) - delete(b.customModels, modelARN) + b.customModels.Delete(modelARN) return nil } @@ -1937,7 +1894,7 @@ func (b *InMemoryBackend) CreateModelCustomizationJob( LastModifiedTime: now, Tags: copyTags(tags), } - b.modelCustomizationJobs[jobARN] = job + b.modelCustomizationJobs.Put(job) b.customizationJobsByName[jobName] = jobARN cp := *job cp.Tags = copyTags(job.Tags) @@ -1948,7 +1905,7 @@ func (b *InMemoryBackend) CreateModelCustomizationJob( // findCustomizationJobARN resolves a job ID or name to its ARN. // Caller must hold at least a read lock. func (b *InMemoryBackend) findCustomizationJobARN(idOrARN string) (string, bool) { - if _, ok := b.modelCustomizationJobs[idOrARN]; ok { + if _, ok := b.modelCustomizationJobs.Get(idOrARN); ok { return idOrARN, true } @@ -1969,7 +1926,7 @@ func (b *InMemoryBackend) GetModelCustomizationJob(idOrARN string) (*ModelCustom return nil, fmt.Errorf("%w: model customization job %s not found", ErrNotFound, idOrARN) } - j := b.modelCustomizationJobs[jobARN] + j, _ := b.modelCustomizationJobs.Get(jobARN) cp := *j cp.Tags = copyTags(j.Tags) @@ -1983,9 +1940,9 @@ func (b *InMemoryBackend) ListModelCustomizationJobs( b.mu.RLock("ListModelCustomizationJobs") defer b.mu.RUnlock() - list := make([]*ModelCustomizationJob, 0, len(b.modelCustomizationJobs)) + list := make([]*ModelCustomizationJob, 0, b.modelCustomizationJobs.Len()) - for _, j := range b.modelCustomizationJobs { + for _, j := range b.modelCustomizationJobs.All() { cp := *j cp.Tags = copyTags(j.Tags) list = append(list, &cp) @@ -2006,7 +1963,8 @@ func (b *InMemoryBackend) StopModelCustomizationJob(idOrARN string) error { return fmt.Errorf("%w: model customization job %s not found", ErrNotFound, idOrARN) } - b.modelCustomizationJobs[jobARN].Status = statusStopped + j, _ := b.modelCustomizationJobs.Get(jobARN) + j.Status = statusStopped return nil } @@ -2020,7 +1978,7 @@ func (b *InMemoryBackend) AdvanceCustomizationJobStatuses(minAge time.Duration) now := time.Now().UTC() advanced := 0 - for _, job := range b.modelCustomizationJobs { + for _, job := range b.modelCustomizationJobs.All() { if job.Status != statusInProgress { continue } @@ -2067,7 +2025,7 @@ func (b *InMemoryBackend) CreateModelCopyJob( LastModifiedTime: now, Tags: copyTags(tags), } - b.modelCopyJobs[jobARN] = job + b.modelCopyJobs.Put(job) cp := *job cp.Tags = copyTags(job.Tags) @@ -2080,7 +2038,7 @@ func (b *InMemoryBackend) GetModelCopyJob(jobARN string) (*ModelCopyJob, error) b.mu.RLock("GetModelCopyJob") defer b.mu.RUnlock() - job, ok := b.modelCopyJobs[jobARN] + job, ok := b.modelCopyJobs.Get(jobARN) if !ok { return nil, fmt.Errorf("%w: model copy job %s not found", ErrNotFound, jobARN) } @@ -2096,9 +2054,9 @@ func (b *InMemoryBackend) ListModelCopyJobs() []*ModelCopyJob { b.mu.RLock("ListModelCopyJobs") defer b.mu.RUnlock() - list := make([]*ModelCopyJob, 0, len(b.modelCopyJobs)) + list := make([]*ModelCopyJob, 0, b.modelCopyJobs.Len()) - for _, j := range b.modelCopyJobs { + for _, j := range b.modelCopyJobs.All() { cp := *j cp.Tags = copyTags(j.Tags) list = append(list, &cp) @@ -2143,7 +2101,7 @@ func (b *InMemoryBackend) CreateModelImportJob( LastModifiedTime: now, Tags: copyTags(tags), } - b.modelImportJobs[jobARN] = job + b.modelImportJobs.Put(job) cp := *job cp.Tags = copyTags(job.Tags) @@ -2156,7 +2114,7 @@ func (b *InMemoryBackend) GetModelImportJob(jobARN string) (*ModelImportJob, err b.mu.RLock("GetModelImportJob") defer b.mu.RUnlock() - job, ok := b.modelImportJobs[jobARN] + job, ok := b.modelImportJobs.Get(jobARN) if !ok { return nil, fmt.Errorf("%w: model import job %s not found", ErrNotFound, jobARN) } @@ -2172,9 +2130,9 @@ func (b *InMemoryBackend) ListModelImportJobs() []*ModelImportJob { b.mu.RLock("ListModelImportJobs") defer b.mu.RUnlock() - list := make([]*ModelImportJob, 0, len(b.modelImportJobs)) + list := make([]*ModelImportJob, 0, b.modelImportJobs.Len()) - for _, j := range b.modelImportJobs { + for _, j := range b.modelImportJobs.All() { cp := *j cp.Tags = copyTags(j.Tags) list = append(list, &cp) @@ -2196,7 +2154,7 @@ func (b *InMemoryBackend) AdvanceCopyImportJobStatuses(minAge time.Duration) int now := time.Now().UTC() advanced := 0 - for _, job := range b.modelCopyJobs { + for _, job := range b.modelCopyJobs.All() { if job.Status == statusInProgress && now.Sub(job.CreationTime) >= minAge { job.Status = statusCompleted job.LastModifiedTime = now @@ -2204,7 +2162,7 @@ func (b *InMemoryBackend) AdvanceCopyImportJobStatuses(minAge time.Duration) int } } - for _, job := range b.modelImportJobs { + for _, job := range b.modelImportJobs.All() { if job.Status == statusInProgress && now.Sub(job.CreationTime) >= minAge { endTime := now job.Status = "Complete" @@ -2257,7 +2215,7 @@ func (b *InMemoryBackend) CreateInferenceProfile( UpdatedAt: now, Tags: copyTags(tags), } - b.inferenceProfiles[profileARN] = profile + b.inferenceProfiles.Put(profile) b.inferenceProfilesByName[name] = profileARN cp := *profile cp.Tags = copyTags(profile.Tags) @@ -2268,7 +2226,7 @@ func (b *InMemoryBackend) CreateInferenceProfile( // findInferenceProfileARN resolves a profile ID or name to its ARN. // Caller must hold at least a read lock. func (b *InMemoryBackend) findInferenceProfileARN(idOrARN string) (string, bool) { - if _, ok := b.inferenceProfiles[idOrARN]; ok { + if _, ok := b.inferenceProfiles.Get(idOrARN); ok { return idOrARN, true } @@ -2289,7 +2247,7 @@ func (b *InMemoryBackend) GetInferenceProfile(idOrARN string) (*InferenceProfile return nil, fmt.Errorf("%w: inference profile %s not found", ErrNotFound, idOrARN) } - p := b.inferenceProfiles[profileARN] + p, _ := b.inferenceProfiles.Get(profileARN) cp := *p cp.Tags = copyTags(p.Tags) @@ -2301,9 +2259,9 @@ func (b *InMemoryBackend) ListInferenceProfiles(nextToken string) ([]*InferenceP b.mu.RLock("ListInferenceProfiles") defer b.mu.RUnlock() - list := make([]*InferenceProfile, 0, len(b.inferenceProfiles)) + list := make([]*InferenceProfile, 0, b.inferenceProfiles.Len()) - for _, p := range b.inferenceProfiles { + for _, p := range b.inferenceProfiles.All() { cp := *p cp.Tags = copyTags(p.Tags) list = append(list, &cp) @@ -2326,9 +2284,9 @@ func (b *InMemoryBackend) DeleteInferenceProfile(idOrARN string) error { return fmt.Errorf("%w: inference profile %s not found", ErrNotFound, idOrARN) } - p := b.inferenceProfiles[profileARN] + p, _ := b.inferenceProfiles.Get(profileARN) delete(b.inferenceProfilesByName, p.InferenceProfileName) - delete(b.inferenceProfiles, profileARN) + b.inferenceProfiles.Delete(profileARN) return nil } @@ -2375,7 +2333,7 @@ func (b *InMemoryBackend) CreateMarketplaceModelEndpoint( UpdatedAt: now, Tags: copyTags(tags), } - b.marketplaceEndpoints[endpointARN] = ep + b.marketplaceEndpoints.Put(ep) b.marketplaceEndpointsByName[endpointName] = endpointARN cp := *ep cp.Tags = copyTags(ep.Tags) @@ -2386,7 +2344,7 @@ func (b *InMemoryBackend) CreateMarketplaceModelEndpoint( // findMarketplaceEndpointARN resolves an endpoint ID or name to its ARN. // Caller must hold at least a read lock. func (b *InMemoryBackend) findMarketplaceEndpointARN(idOrARN string) (string, bool) { - if _, ok := b.marketplaceEndpoints[idOrARN]; ok { + if _, ok := b.marketplaceEndpoints.Get(idOrARN); ok { return idOrARN, true } @@ -2409,7 +2367,7 @@ func (b *InMemoryBackend) GetMarketplaceModelEndpoint( return nil, fmt.Errorf("%w: marketplace endpoint %s not found", ErrNotFound, idOrARN) } - ep := b.marketplaceEndpoints[epARN] + ep, _ := b.marketplaceEndpoints.Get(epARN) cp := *ep cp.Tags = copyTags(ep.Tags) @@ -2423,9 +2381,9 @@ func (b *InMemoryBackend) ListMarketplaceModelEndpoints( b.mu.RLock("ListMarketplaceModelEndpoints") defer b.mu.RUnlock() - list := make([]*MarketplaceModelEndpoint, 0, len(b.marketplaceEndpoints)) + list := make([]*MarketplaceModelEndpoint, 0, b.marketplaceEndpoints.Len()) - for _, ep := range b.marketplaceEndpoints { + for _, ep := range b.marketplaceEndpoints.All() { cp := *ep cp.Tags = copyTags(ep.Tags) list = append(list, &cp) @@ -2446,9 +2404,9 @@ func (b *InMemoryBackend) DeleteMarketplaceModelEndpoint(idOrARN string) error { return fmt.Errorf("%w: marketplace endpoint %s not found", ErrNotFound, idOrARN) } - ep := b.marketplaceEndpoints[epARN] + ep, _ := b.marketplaceEndpoints.Get(epARN) delete(b.marketplaceEndpointsByName, ep.EndpointName) - delete(b.marketplaceEndpoints, epARN) + b.marketplaceEndpoints.Delete(epARN) return nil } @@ -2465,7 +2423,7 @@ func (b *InMemoryBackend) UpdateMarketplaceModelEndpoint( return nil, fmt.Errorf("%w: marketplace endpoint %s not found", ErrNotFound, idOrARN) } - ep := b.marketplaceEndpoints[epARN] + ep, _ := b.marketplaceEndpoints.Get(epARN) ep.UpdatedAt = time.Now().UTC() cp := *ep cp.Tags = copyTags(ep.Tags) @@ -2483,7 +2441,8 @@ func (b *InMemoryBackend) RegisterMarketplaceModelEndpoint(idOrARN string) error return fmt.Errorf("%w: marketplace endpoint %s not found", ErrNotFound, idOrARN) } - b.marketplaceEndpoints[epARN].Status = "Active" + ep, _ := b.marketplaceEndpoints.Get(epARN) + ep.Status = "Active" return nil } @@ -2498,7 +2457,8 @@ func (b *InMemoryBackend) DeregisterMarketplaceModelEndpoint(idOrARN string) err return fmt.Errorf("%w: marketplace endpoint %s not found", ErrNotFound, idOrARN) } - b.marketplaceEndpoints[epARN].Status = "Deregistered" + ep, _ := b.marketplaceEndpoints.Get(epARN) + ep.Status = "Deregistered" return nil } @@ -2563,8 +2523,7 @@ func (b *InMemoryBackend) CreateGuardrailVersion( Description: description, } - key := g.GuardrailID + ":" + versionNum - b.guardrailVersions[key] = gv + b.guardrailVersions.Put(gv) return gv, nil } diff --git a/services/bedrock/backend_agents.go b/services/bedrock/backend_agents.go index 8cf73bc4f..1f599ced8 100644 --- a/services/bedrock/backend_agents.go +++ b/services/bedrock/backend_agents.go @@ -231,7 +231,7 @@ func (b *InMemoryBackend) CreateAgentWithConfiguration(config AgentConfiguration GuardrailConfiguration: maps.Clone(config.GuardrailConfiguration), MemoryConfiguration: maps.Clone(config.MemoryConfiguration), } - b.agents[id] = ag + b.agents.Put(ag) b.agentsByName[config.AgentName] = id cp := *ag @@ -244,7 +244,7 @@ func (b *InMemoryBackend) GetAgent(agentID string) (*Agent, error) { b.mu.Lock("GetAgent") defer b.mu.Unlock() - ag, ok := b.agents[agentID] + ag, ok := b.agents.Get(agentID) if !ok { return nil, fmt.Errorf("%w: agent %q not found", ErrNotFound, agentID) } @@ -260,8 +260,8 @@ func (b *InMemoryBackend) ListAgents(maxResults int, nextToken string) ([]*Agent b.mu.Lock("ListAgents") defer b.mu.Unlock() - list := make([]*Agent, 0, len(b.agents)) - for _, ag := range b.agents { + list := make([]*Agent, 0, b.agents.Len()) + for _, ag := range b.agents.All() { b.advanceAgentStatus(ag) cp := *ag list = append(list, &cp) @@ -288,7 +288,7 @@ func (b *InMemoryBackend) UpdateAgentWithConfiguration(agentID string, config Ag b.mu.Lock("UpdateAgent") defer b.mu.Unlock() - ag, ok := b.agents[agentID] + ag, ok := b.agents.Get(agentID) if !ok { return nil, fmt.Errorf("%w: agent %q not found", ErrNotFound, agentID) } @@ -335,20 +335,29 @@ func (b *InMemoryBackend) DeleteAgent(agentID string) error { b.mu.Lock("DeleteAgent") defer b.mu.Unlock() - ag, ok := b.agents[agentID] + ag, ok := b.agents.Get(agentID) if !ok { return fmt.Errorf("%w: agent %q not found", ErrNotFound, agentID) } - prefix := agentID + "/" - for k := range b.agentAliases { - if len(k) > len(prefix) && k[:len(prefix)] == prefix { - return fmt.Errorf("%w: agent %q has active aliases and cannot be deleted", ErrAlreadyExists, agentID) + hasAlias := false + + b.agentAliases.Range(func(alias *AgentAlias) bool { + if alias.AgentID == agentID { + hasAlias = true + + return false } + + return true + }) + + if hasAlias { + return fmt.Errorf("%w: agent %q has active aliases and cannot be deleted", ErrAlreadyExists, agentID) } delete(b.agentsByName, ag.AgentName) - delete(b.agents, agentID) + b.agents.Delete(agentID) return nil } @@ -358,7 +367,7 @@ func (b *InMemoryBackend) PrepareAgent(agentID string) (*Agent, error) { b.mu.Lock("PrepareAgent") defer b.mu.Unlock() - ag, ok := b.agents[agentID] + ag, ok := b.agents.Get(agentID) if !ok { return nil, fmt.Errorf("%w: agent %q not found", ErrNotFound, agentID) } @@ -405,7 +414,7 @@ func (b *InMemoryBackend) CreateAgentActionGroupWithSchemas( b.mu.Lock("CreateAgentActionGroup") defer b.mu.Unlock() - if _, ok := b.agents[agentID]; !ok { + if _, ok := b.agents.Get(agentID); !ok { return nil, fmt.Errorf("%w: agent %q not found", ErrNotFound, agentID) } @@ -426,7 +435,7 @@ func (b *InMemoryBackend) CreateAgentActionGroupWithSchemas( APISchema: maps.Clone(apiSchema), FunctionSchema: maps.Clone(functionSchema), } - b.agentActionGroups[agentActionGroupKey(agentID, id)] = ag + b.agentActionGroups.Put(ag) cp := *ag return &cp, nil @@ -439,7 +448,7 @@ func (b *InMemoryBackend) GetAgentActionGroup( b.mu.RLock("GetAgentActionGroup") defer b.mu.RUnlock() - ag, ok := b.agentActionGroups[agentActionGroupKey(agentID, actionGroupID)] + ag, ok := b.agentActionGroups.Get(agentActionGroupKey(agentID, actionGroupID)) if !ok { return nil, fmt.Errorf( "%w: action group %q not found for agent %q", @@ -463,11 +472,10 @@ func (b *InMemoryBackend) ListAgentActionGroups( b.mu.RLock("ListAgentActionGroups") defer b.mu.RUnlock() - list := make([]*AgentActionGroup, 0, len(b.agentActionGroups)) - prefix := agentID + "/" + list := make([]*AgentActionGroup, 0, b.agentActionGroups.Len()) - for k, ag := range b.agentActionGroups { - if len(k) > len(prefix) && k[:len(prefix)] == prefix { + for _, ag := range b.agentActionGroups.All() { + if ag.AgentID == agentID { cp := *ag list = append(list, &cp) } @@ -499,7 +507,7 @@ func (b *InMemoryBackend) UpdateAgentActionGroupWithSchemas( key := agentActionGroupKey(agentID, actionGroupID) - ag, ok := b.agentActionGroups[key] + ag, ok := b.agentActionGroups.Get(key) if !ok { return nil, fmt.Errorf("%w: action group %q not found", ErrNotFound, actionGroupID) } @@ -531,11 +539,11 @@ func (b *InMemoryBackend) DeleteAgentActionGroup(agentID, actionGroupID string) key := agentActionGroupKey(agentID, actionGroupID) - if _, ok := b.agentActionGroups[key]; !ok { + if _, ok := b.agentActionGroups.Get(key); !ok { return fmt.Errorf("%w: action group %q not found", ErrNotFound, actionGroupID) } - delete(b.agentActionGroups, key) + b.agentActionGroups.Delete(key) return nil } @@ -551,7 +559,7 @@ func (b *InMemoryBackend) CreateAgentAlias( b.mu.Lock("CreateAgentAlias") defer b.mu.Unlock() - if _, ok := b.agents[agentID]; !ok { + if _, ok := b.agents.Get(agentID); !ok { return nil, fmt.Errorf("%w: agent %q not found", ErrNotFound, agentID) } @@ -574,7 +582,7 @@ func (b *InMemoryBackend) CreateAgentAlias( RoutingConfiguration: []AgentAliasRouting{{AgentVersion: agentVersion}}, }}, } - b.agentAliases[agentAliasKey(agentID, aliasID)] = alias + b.agentAliases.Put(alias) cp := *alias return &cp, nil @@ -585,7 +593,7 @@ func (b *InMemoryBackend) GetAgentAlias(agentID, aliasID string) (*AgentAlias, e b.mu.RLock("GetAgentAlias") defer b.mu.RUnlock() - alias, ok := b.agentAliases[agentAliasKey(agentID, aliasID)] + alias, ok := b.agentAliases.Get(agentAliasKey(agentID, aliasID)) if !ok { return nil, fmt.Errorf("%w: agent alias %q not found", ErrNotFound, aliasID) } @@ -604,11 +612,10 @@ func (b *InMemoryBackend) ListAgentAliases( b.mu.RLock("ListAgentAliases") defer b.mu.RUnlock() - list := make([]*AgentAlias, 0, len(b.agentAliases)) - prefix := agentID + "/" + list := make([]*AgentAlias, 0, b.agentAliases.Len()) - for k, alias := range b.agentAliases { - if len(k) > len(prefix) && k[:len(prefix)] == prefix { + for _, alias := range b.agentAliases.All() { + if alias.AgentID == agentID { cp := *alias list = append(list, &cp) } @@ -628,7 +635,7 @@ func (b *InMemoryBackend) UpdateAgentAlias( key := agentAliasKey(agentID, aliasID) - alias, ok := b.agentAliases[key] + alias, ok := b.agentAliases.Get(key) if !ok { return nil, fmt.Errorf("%w: agent alias %q not found", ErrNotFound, aliasID) } @@ -663,11 +670,11 @@ func (b *InMemoryBackend) DeleteAgentAlias(agentID, aliasID string) error { key := agentAliasKey(agentID, aliasID) - if _, ok := b.agentAliases[key]; !ok { + if _, ok := b.agentAliases.Get(key); !ok { return fmt.Errorf("%w: agent alias %q not found", ErrNotFound, aliasID) } - delete(b.agentAliases, key) + b.agentAliases.Delete(key) return nil } @@ -683,11 +690,11 @@ func (b *InMemoryBackend) AssociateAgentKnowledgeBase( b.mu.Lock("AssociateAgentKnowledgeBase") defer b.mu.Unlock() - if _, ok := b.agents[agentID]; !ok { + if _, ok := b.agents.Get(agentID); !ok { return nil, fmt.Errorf("%w: agent %q not found", ErrNotFound, agentID) } - if _, ok := b.knowledgeBases[kbID]; !ok { + if _, ok := b.knowledgeBases.Get(kbID); !ok { return nil, fmt.Errorf("%w: knowledge base %q not found", ErrNotFound, kbID) } @@ -698,7 +705,7 @@ func (b *InMemoryBackend) AssociateAgentKnowledgeBase( Description: description, KBState: actionGroupEnabled, } - b.agentKBAssociations[agentKBKey(agentID, kbID)] = assoc + b.agentKBAssociations.Put(assoc) cp := *assoc return &cp, nil @@ -711,7 +718,7 @@ func (b *InMemoryBackend) DisassociateAgentKnowledgeBase(agentID, kbID string) e key := agentKBKey(agentID, kbID) - if _, ok := b.agentKBAssociations[key]; !ok { + if _, ok := b.agentKBAssociations.Get(key); !ok { return fmt.Errorf( "%w: association not found for agent %q and kb %q", ErrNotFound, @@ -720,7 +727,7 @@ func (b *InMemoryBackend) DisassociateAgentKnowledgeBase(agentID, kbID string) e ) } - delete(b.agentKBAssociations, key) + b.agentKBAssociations.Delete(key) return nil } @@ -732,7 +739,7 @@ func (b *InMemoryBackend) UpdateAgentKnowledgeBase( b.mu.Lock("UpdateAgentKnowledgeBase") defer b.mu.Unlock() - assoc, ok := b.agentKBAssociations[agentKBKey(agentID, kbID)] + assoc, ok := b.agentKBAssociations.Get(agentKBKey(agentID, kbID)) if !ok { return nil, fmt.Errorf("%w: association not found for agent %q and kb %q", ErrNotFound, agentID, kbID) } @@ -756,7 +763,7 @@ func (b *InMemoryBackend) GetAgentKnowledgeBase( b.mu.RLock("GetAgentKnowledgeBase") defer b.mu.RUnlock() - assoc, ok := b.agentKBAssociations[agentKBKey(agentID, kbID)] + assoc, ok := b.agentKBAssociations.Get(agentKBKey(agentID, kbID)) if !ok { return nil, fmt.Errorf( "%w: association not found for agent %q and kb %q", @@ -780,11 +787,10 @@ func (b *InMemoryBackend) ListAgentKnowledgeBases( b.mu.RLock("ListAgentKnowledgeBases") defer b.mu.RUnlock() - list := make([]*AgentKnowledgeBaseAssociation, 0, len(b.agentKBAssociations)) - prefix := agentID + "/" + list := make([]*AgentKnowledgeBaseAssociation, 0, b.agentKBAssociations.Len()) - for k, assoc := range b.agentKBAssociations { - if len(k) > len(prefix) && k[:len(prefix)] == prefix { + for _, assoc := range b.agentKBAssociations.All() { + if assoc.AgentID == agentID { cp := *assoc list = append(list, &cp) } @@ -836,7 +842,7 @@ func (b *InMemoryBackend) CreateKnowledgeBase( StorageConfiguration: storageConfig, Tags: tagsCopy, } - b.knowledgeBases[id] = kb + b.knowledgeBases.Put(kb) b.kbByName[name] = id cp := *kb @@ -848,7 +854,7 @@ func (b *InMemoryBackend) GetKnowledgeBase(kbID string) (*KnowledgeBase, error) b.mu.RLock("GetKnowledgeBase") defer b.mu.RUnlock() - kb, ok := b.knowledgeBases[kbID] + kb, ok := b.knowledgeBases.Get(kbID) if !ok { return nil, fmt.Errorf("%w: knowledge base %q not found", ErrNotFound, kbID) } @@ -866,8 +872,8 @@ func (b *InMemoryBackend) ListKnowledgeBases( b.mu.RLock("ListKnowledgeBases") defer b.mu.RUnlock() - list := make([]*KnowledgeBase, 0, len(b.knowledgeBases)) - for _, kb := range b.knowledgeBases { + list := make([]*KnowledgeBase, 0, b.knowledgeBases.Len()) + for _, kb := range b.knowledgeBases.All() { cp := *kb list = append(list, &cp) } @@ -884,7 +890,7 @@ func (b *InMemoryBackend) UpdateKnowledgeBase( b.mu.Lock("UpdateKnowledgeBase") defer b.mu.Unlock() - kb, ok := b.knowledgeBases[kbID] + kb, ok := b.knowledgeBases.Get(kbID) if !ok { return nil, fmt.Errorf("%w: knowledge base %q not found", ErrNotFound, kbID) } @@ -914,13 +920,13 @@ func (b *InMemoryBackend) DeleteKnowledgeBase(kbID string) error { b.mu.Lock("DeleteKnowledgeBase") defer b.mu.Unlock() - kb, ok := b.knowledgeBases[kbID] + kb, ok := b.knowledgeBases.Get(kbID) if !ok { return fmt.Errorf("%w: knowledge base %q not found", ErrNotFound, kbID) } delete(b.kbByName, kb.Name) - delete(b.knowledgeBases, kbID) + b.knowledgeBases.Delete(kbID) return nil } @@ -945,7 +951,7 @@ func (b *InMemoryBackend) CreateDataSourceWithConfiguration( b.mu.Lock("CreateDataSource") defer b.mu.Unlock() - if _, ok := b.knowledgeBases[kbID]; !ok { + if _, ok := b.knowledgeBases.Get(kbID); !ok { return nil, fmt.Errorf("%w: knowledge base %q not found", ErrNotFound, kbID) } @@ -965,7 +971,7 @@ func (b *InMemoryBackend) CreateDataSourceWithConfiguration( DataSourceConfiguration: maps.Clone(dsConfig), VectorIngestionConfig: maps.Clone(vectorConfig), } - b.dataSources[kbID+"/"+id] = ds + b.dataSources.Put(ds) cp := *ds return &cp, nil @@ -976,7 +982,7 @@ func (b *InMemoryBackend) GetDataSource(kbID, dsID string) (*DataSource, error) b.mu.RLock("GetDataSource") defer b.mu.RUnlock() - ds, ok := b.dataSources[kbID+"/"+dsID] + ds, ok := b.dataSources.Get(kbID + "/" + dsID) if !ok { return nil, fmt.Errorf("%w: data source %q not found", ErrNotFound, dsID) } @@ -995,11 +1001,10 @@ func (b *InMemoryBackend) ListDataSources( b.mu.RLock("ListDataSources") defer b.mu.RUnlock() - list := make([]*DataSource, 0, len(b.dataSources)) - prefix := kbID + "/" + list := make([]*DataSource, 0, b.dataSources.Len()) - for k, ds := range b.dataSources { - if len(k) > len(prefix) && k[:len(prefix)] == prefix { + for _, ds := range b.dataSources.All() { + if ds.KnowledgeBaseID == kbID { cp := *ds list = append(list, &cp) } @@ -1027,7 +1032,7 @@ func (b *InMemoryBackend) UpdateDataSourceWithConfiguration( key := kbID + "/" + dsID - ds, ok := b.dataSources[key] + ds, ok := b.dataSources.Get(key) if !ok { return nil, fmt.Errorf("%w: data source %q not found", ErrNotFound, dsID) } @@ -1062,11 +1067,11 @@ func (b *InMemoryBackend) DeleteDataSource(kbID, dsID string) error { key := kbID + "/" + dsID - if _, ok := b.dataSources[key]; !ok { + if _, ok := b.dataSources.Get(key); !ok { return fmt.Errorf("%w: data source %q not found", ErrNotFound, dsID) } - delete(b.dataSources, key) + b.dataSources.Delete(key) return nil } @@ -1081,17 +1086,16 @@ func (b *InMemoryBackend) StartIngestionJob(kbID, dsID, description string) (*In b.mu.Lock("StartIngestionJob") defer b.mu.Unlock() - if _, ok := b.knowledgeBases[kbID]; !ok { + if _, ok := b.knowledgeBases.Get(kbID); !ok { return nil, fmt.Errorf("%w: knowledge base %q not found", ErrNotFound, kbID) } - if _, ok := b.dataSources[kbID+"/"+dsID]; !ok { + if _, ok := b.dataSources.Get(kbID + "/" + dsID); !ok { return nil, fmt.Errorf("%w: data source %q not found", ErrNotFound, dsID) } - prefix := kbID + "/" + dsID + "/" - for k, job := range b.ingestionJobs { - if len(k) > len(prefix) && k[:len(prefix)] == prefix && job.Status == jobStatusStarting { + for _, job := range b.ingestionJobs.All() { + if job.KnowledgeBaseID == kbID && job.DataSourceID == dsID && job.Status == jobStatusStarting { return nil, fmt.Errorf("%w: data source %q already has a running ingestion job", ErrAlreadyExists, dsID) } } @@ -1109,7 +1113,7 @@ func (b *InMemoryBackend) StartIngestionJob(kbID, dsID, description string) (*In Status: jobStatusStarting, Description: description, } - b.ingestionJobs[ingestionJobKey(kbID, dsID, jobID)] = job + b.ingestionJobs.Put(job) job.completionDueAt = now.Add(ingestionCompleteDelay) @@ -1123,7 +1127,7 @@ func (b *InMemoryBackend) GetIngestionJob(kbID, dsID, jobID string) (*IngestionJ b.mu.Lock("GetIngestionJob") defer b.mu.Unlock() - job, ok := b.ingestionJobs[ingestionJobKey(kbID, dsID, jobID)] + job, ok := b.ingestionJobs.Get(ingestionJobKey(kbID, dsID, jobID)) if !ok { return nil, fmt.Errorf("%w: ingestion job %q not found", ErrNotFound, jobID) } @@ -1143,11 +1147,10 @@ func (b *InMemoryBackend) ListIngestionJobs( b.mu.Lock("ListIngestionJobs") defer b.mu.Unlock() - list := make([]*IngestionJob, 0, len(b.ingestionJobs)) - prefix := kbID + "/" + dsID + "/" + list := make([]*IngestionJob, 0, b.ingestionJobs.Len()) - for k, job := range b.ingestionJobs { - if len(k) > len(prefix) && k[:len(prefix)] == prefix { + for _, job := range b.ingestionJobs.All() { + if job.KnowledgeBaseID == kbID && job.DataSourceID == dsID { advanceIngestionJob(job) cp := *job list = append(list, &cp) diff --git a/services/bedrock/backend_agents_batch3.go b/services/bedrock/backend_agents_batch3.go index 2349e33fc..23ea08df3 100644 --- a/services/bedrock/backend_agents_batch3.go +++ b/services/bedrock/backend_agents_batch3.go @@ -143,7 +143,7 @@ func (b *InMemoryBackend) CreateFlow( Status: flowStatusNotPrepared, Tags: tagsCopy, } - b.flows[id] = f + b.flows.Put(f) b.flowsByName[name] = id cp := *f @@ -155,7 +155,7 @@ func (b *InMemoryBackend) GetFlow(flowID string) (*Flow, error) { b.mu.RLock("GetFlow") defer b.mu.RUnlock() - f, ok := b.flows[flowID] + f, ok := b.flows.Get(flowID) if !ok { return nil, fmt.Errorf("%w: flow %q not found", ErrNotFound, flowID) } @@ -170,8 +170,8 @@ func (b *InMemoryBackend) ListFlows(maxResults int, nextToken string) ([]*Flow, b.mu.RLock("ListFlows") defer b.mu.RUnlock() - list := make([]*Flow, 0, len(b.flows)) - for _, f := range b.flows { + list := make([]*Flow, 0, b.flows.Len()) + for _, f := range b.flows.All() { cp := *f list = append(list, &cp) } @@ -186,7 +186,7 @@ func (b *InMemoryBackend) UpdateFlow(flowID, name, description string) (*Flow, e b.mu.Lock("UpdateFlow") defer b.mu.Unlock() - f, ok := b.flows[flowID] + f, ok := b.flows.Get(flowID) if !ok { return nil, fmt.Errorf("%w: flow %q not found", ErrNotFound, flowID) } @@ -212,13 +212,13 @@ func (b *InMemoryBackend) DeleteFlow(flowID string) error { b.mu.Lock("DeleteFlow") defer b.mu.Unlock() - f, ok := b.flows[flowID] + f, ok := b.flows.Get(flowID) if !ok { return fmt.Errorf("%w: flow %q not found", ErrNotFound, flowID) } delete(b.flowsByName, f.Name) - delete(b.flows, flowID) + b.flows.Delete(flowID) return nil } @@ -228,7 +228,7 @@ func (b *InMemoryBackend) PrepareFlow(flowID string) (*Flow, error) { b.mu.Lock("PrepareFlow") defer b.mu.Unlock() - f, ok := b.flows[flowID] + f, ok := b.flows.Get(flowID) if !ok { return nil, fmt.Errorf("%w: flow %q not found", ErrNotFound, flowID) } @@ -251,7 +251,7 @@ func (b *InMemoryBackend) CreateFlowAlias( b.mu.Lock("CreateFlowAlias") defer b.mu.Unlock() - if _, ok := b.flows[flowID]; !ok { + if _, ok := b.flows.Get(flowID); !ok { return nil, fmt.Errorf("%w: flow %q not found", ErrNotFound, flowID) } @@ -274,7 +274,7 @@ func (b *InMemoryBackend) CreateFlowAlias( Name: name, Description: description, } - b.flowAliases[flowAliasKey(flowID, aliasID)] = fa + b.flowAliases.Put(fa) cp := *fa return &cp, nil @@ -285,7 +285,7 @@ func (b *InMemoryBackend) GetFlowAlias(flowID, aliasID string) (*FlowAlias, erro b.mu.RLock("GetFlowAlias") defer b.mu.RUnlock() - fa, ok := b.flowAliases[flowAliasKey(flowID, aliasID)] + fa, ok := b.flowAliases.Get(flowAliasKey(flowID, aliasID)) if !ok { return nil, fmt.Errorf("%w: flow alias %q not found", ErrNotFound, aliasID) } @@ -305,10 +305,9 @@ func (b *InMemoryBackend) ListFlowAliases( defer b.mu.RUnlock() list := make([]*FlowAlias, 0) - prefix := flowID + "/" - for k, fa := range b.flowAliases { - if len(k) > len(prefix) && k[:len(prefix)] == prefix { + for _, fa := range b.flowAliases.All() { + if fa.FlowID == flowID { cp := *fa list = append(list, &cp) } @@ -326,7 +325,7 @@ func (b *InMemoryBackend) UpdateFlowAlias( b.mu.Lock("UpdateFlowAlias") defer b.mu.Unlock() - fa, ok := b.flowAliases[flowAliasKey(flowID, aliasID)] + fa, ok := b.flowAliases.Get(flowAliasKey(flowID, aliasID)) if !ok { return nil, fmt.Errorf("%w: flow alias %q not found", ErrNotFound, aliasID) } @@ -352,11 +351,11 @@ func (b *InMemoryBackend) DeleteFlowAlias(flowID, aliasID string) error { key := flowAliasKey(flowID, aliasID) - if _, ok := b.flowAliases[key]; !ok { + if _, ok := b.flowAliases.Get(key); !ok { return fmt.Errorf("%w: flow alias %q not found", ErrNotFound, aliasID) } - delete(b.flowAliases, key) + b.flowAliases.Delete(key) return nil } @@ -370,7 +369,7 @@ func (b *InMemoryBackend) CreateFlowVersion(flowID string) (*FlowVersion, error) b.mu.Lock("CreateFlowVersion") defer b.mu.Unlock() - if _, ok := b.flows[flowID]; !ok { + if _, ok := b.flows.Get(flowID); !ok { return nil, fmt.Errorf("%w: flow %q not found", ErrNotFound, flowID) } @@ -384,11 +383,7 @@ func (b *InMemoryBackend) CreateFlowVersion(flowID string) (*FlowVersion, error) Status: flowStatusPrepared, } - if b.flowVersions[flowID] == nil { - b.flowVersions[flowID] = make(map[string]*FlowVersion) - } - - b.flowVersions[flowID][ver] = fv + b.flowVersionsStore(flowID).Put(fv) cp := *fv return &cp, nil @@ -404,7 +399,7 @@ func (b *InMemoryBackend) GetFlowVersion(flowID, version string) (*FlowVersion, return nil, fmt.Errorf("%w: flow %q not found", ErrNotFound, flowID) } - fv, verOK := versions[version] + fv, verOK := versions.Get(version) if !verOK { return nil, fmt.Errorf( "%w: flow version %q not found for flow %q", @@ -429,11 +424,13 @@ func (b *InMemoryBackend) ListFlowVersions( defer b.mu.RUnlock() versions := b.flowVersions[flowID] - list := make([]*FlowVersion, 0, len(versions)) + list := make([]*FlowVersion, 0) - for _, fv := range versions { - cp := *fv - list = append(list, &cp) + if versions != nil { + for _, fv := range versions.All() { + cp := *fv + list = append(list, &cp) + } } sort.Slice(list, func(i, j int) bool { return list[i].Version < list[j].Version }) @@ -451,7 +448,7 @@ func (b *InMemoryBackend) DeleteFlowVersion(flowID, version string) error { return fmt.Errorf("%w: flow %q not found", ErrNotFound, flowID) } - if _, verOK := versions[version]; !verOK { + if !versions.Has(version) { return fmt.Errorf( "%w: flow version %q not found for flow %q", ErrNotFound, @@ -460,7 +457,7 @@ func (b *InMemoryBackend) DeleteFlowVersion(flowID, version string) error { ) } - delete(versions, version) + versions.Delete(version) return nil } @@ -503,7 +500,7 @@ func (b *InMemoryBackend) CreatePrompt( Description: description, Tags: tagsCopy, } - b.prompts[id] = p + b.prompts.Put(p) b.promptsByName[name] = id cp := *p @@ -515,7 +512,7 @@ func (b *InMemoryBackend) GetPrompt(promptID string) (*Prompt, error) { b.mu.RLock("GetPrompt") defer b.mu.RUnlock() - p, ok := b.prompts[promptID] + p, ok := b.prompts.Get(promptID) if !ok { return nil, fmt.Errorf("%w: prompt %q not found", ErrNotFound, promptID) } @@ -530,8 +527,8 @@ func (b *InMemoryBackend) ListPrompts(maxResults int, nextToken string) ([]*Prom b.mu.RLock("ListPrompts") defer b.mu.RUnlock() - list := make([]*Prompt, 0, len(b.prompts)) - for _, p := range b.prompts { + list := make([]*Prompt, 0, b.prompts.Len()) + for _, p := range b.prompts.All() { cp := *p list = append(list, &cp) } @@ -546,7 +543,7 @@ func (b *InMemoryBackend) UpdatePrompt(promptID, name, description string) (*Pro b.mu.Lock("UpdatePrompt") defer b.mu.Unlock() - p, ok := b.prompts[promptID] + p, ok := b.prompts.Get(promptID) if !ok { return nil, fmt.Errorf("%w: prompt %q not found", ErrNotFound, promptID) } @@ -572,13 +569,13 @@ func (b *InMemoryBackend) DeletePrompt(promptID string) error { b.mu.Lock("DeletePrompt") defer b.mu.Unlock() - p, ok := b.prompts[promptID] + p, ok := b.prompts.Get(promptID) if !ok { return fmt.Errorf("%w: prompt %q not found", ErrNotFound, promptID) } delete(b.promptsByName, p.Name) - delete(b.prompts, promptID) + b.prompts.Delete(promptID) return nil } @@ -592,7 +589,7 @@ func (b *InMemoryBackend) CreatePromptVersion(promptID string) (*PromptVersion, b.mu.Lock("CreatePromptVersion") defer b.mu.Unlock() - p, ok := b.prompts[promptID] + p, ok := b.prompts.Get(promptID) if !ok { return nil, fmt.Errorf("%w: prompt %q not found", ErrNotFound, promptID) } @@ -607,11 +604,7 @@ func (b *InMemoryBackend) CreatePromptVersion(promptID string) (*PromptVersion, Name: p.Name, } - if b.promptVersions[promptID] == nil { - b.promptVersions[promptID] = make(map[string]*PromptVersion) - } - - b.promptVersions[promptID][ver] = pv + b.promptVersionsStore(promptID).Put(pv) cp := *pv return &cp, nil @@ -627,7 +620,7 @@ func (b *InMemoryBackend) GetPromptVersion(promptID, version string) (*PromptVer return nil, fmt.Errorf("%w: prompt %q not found", ErrNotFound, promptID) } - pv, verOK := versions[version] + pv, verOK := versions.Get(version) if !verOK { return nil, fmt.Errorf( "%w: prompt version %q not found for prompt %q", @@ -652,11 +645,13 @@ func (b *InMemoryBackend) ListPromptVersions( defer b.mu.RUnlock() versions := b.promptVersions[promptID] - list := make([]*PromptVersion, 0, len(versions)) + list := make([]*PromptVersion, 0) - for _, pv := range versions { - cp := *pv - list = append(list, &cp) + if versions != nil { + for _, pv := range versions.All() { + cp := *pv + list = append(list, &cp) + } } sort.Slice(list, func(i, j int) bool { return list[i].Version < list[j].Version }) @@ -674,7 +669,7 @@ func (b *InMemoryBackend) DeletePromptVersion(promptID, version string) error { return fmt.Errorf("%w: prompt %q not found", ErrNotFound, promptID) } - if _, verOK := versions[version]; !verOK { + if !versions.Has(version) { return fmt.Errorf( "%w: prompt version %q not found for prompt %q", ErrNotFound, @@ -683,7 +678,7 @@ func (b *InMemoryBackend) DeletePromptVersion(promptID, version string) error { ) } - delete(versions, version) + versions.Delete(version) return nil } @@ -697,7 +692,7 @@ func (b *InMemoryBackend) CreateAgentVersion(agentID string) (*AgentVersion, err b.mu.Lock("CreateAgentVersion") defer b.mu.Unlock() - ag, ok := b.agents[agentID] + ag, ok := b.agents.Get(agentID) if !ok { return nil, fmt.Errorf("%w: agent %q not found", ErrNotFound, agentID) } @@ -712,11 +707,7 @@ func (b *InMemoryBackend) CreateAgentVersion(agentID string) (*AgentVersion, err AgentStatus: ag.AgentStatus, } - if b.agentVersions[agentID] == nil { - b.agentVersions[agentID] = make(map[string]*AgentVersion) - } - - b.agentVersions[agentID][ver] = av + b.agentVersionsStore(agentID).Put(av) cp := *av return &cp, nil @@ -732,7 +723,7 @@ func (b *InMemoryBackend) GetAgentVersion(agentID, version string) (*AgentVersio return nil, fmt.Errorf("%w: agent %q has no versions", ErrNotFound, agentID) } - av, verOK := versions[version] + av, verOK := versions.Get(version) if !verOK { return nil, fmt.Errorf( "%w: agent version %q not found for agent %q", @@ -757,11 +748,13 @@ func (b *InMemoryBackend) ListAgentVersions( defer b.mu.RUnlock() versions := b.agentVersions[agentID] - list := make([]*AgentVersion, 0, len(versions)) + list := make([]*AgentVersion, 0) - for _, av := range versions { - cp := *av - list = append(list, &cp) + if versions != nil { + for _, av := range versions.All() { + cp := *av + list = append(list, &cp) + } } sort.Slice(list, func(i, j int) bool { return list[i].AgentVersion < list[j].AgentVersion }) @@ -779,7 +772,7 @@ func (b *InMemoryBackend) DeleteAgentVersion(agentID, version string) error { return fmt.Errorf("%w: agent %q not found", ErrNotFound, agentID) } - if _, verOK := versions[version]; !verOK { + if !versions.Has(version) { return fmt.Errorf( "%w: agent version %q not found for agent %q", ErrNotFound, @@ -788,7 +781,7 @@ func (b *InMemoryBackend) DeleteAgentVersion(agentID, version string) error { ) } - delete(versions, version) + versions.Delete(version) return nil } @@ -804,7 +797,7 @@ func (b *InMemoryBackend) AssociateAgentCollaborator( b.mu.Lock("AssociateAgentCollaborator") defer b.mu.Unlock() - if _, ok := b.agents[agentID]; !ok { + if _, ok := b.agents.Get(agentID); !ok { return nil, fmt.Errorf("%w: agent %q not found", ErrNotFound, agentID) } @@ -820,11 +813,7 @@ func (b *InMemoryBackend) AssociateAgentCollaborator( RelayConversation: relayConversation, } - if b.agentCollaborators[agentID] == nil { - b.agentCollaborators[agentID] = make(map[string]*AgentCollaborator) - } - - b.agentCollaborators[agentID][collabID] = ac + b.agentCollaboratorsStore(agentID).Put(ac) cp := *ac return &cp, nil @@ -839,7 +828,15 @@ func (b *InMemoryBackend) GetAgentCollaborator( collabs := b.agentCollaborators[agentID] - ac, ok := collabs[collaboratorID] + var ( + ac *AgentCollaborator + ok bool + ) + + if collabs != nil { + ac, ok = collabs.Get(collaboratorID) + } + if !ok { return nil, fmt.Errorf( "%w: collaborator %q not found for agent %q", @@ -864,11 +861,13 @@ func (b *InMemoryBackend) ListAgentCollaborators( defer b.mu.RUnlock() collabs := b.agentCollaborators[agentID] - list := make([]*AgentCollaborator, 0, len(collabs)) + list := make([]*AgentCollaborator, 0) - for _, ac := range collabs { - cp := *ac - list = append(list, &cp) + if collabs != nil { + for _, ac := range collabs.All() { + cp := *ac + list = append(list, &cp) + } } sort.Slice( @@ -888,7 +887,15 @@ func (b *InMemoryBackend) UpdateAgentCollaborator( collabs := b.agentCollaborators[agentID] - ac, ok := collabs[collaboratorID] + var ( + ac *AgentCollaborator + ok bool + ) + + if collabs != nil { + ac, ok = collabs.Get(collaboratorID) + } + if !ok { return nil, fmt.Errorf( "%w: collaborator %q not found for agent %q", @@ -914,7 +921,7 @@ func (b *InMemoryBackend) DisassociateAgentCollaborator(agentID, collaboratorID collabs := b.agentCollaborators[agentID] - if _, ok := collabs[collaboratorID]; !ok { + if collabs == nil || !collabs.Has(collaboratorID) { return fmt.Errorf( "%w: collaborator %q not found for agent %q", ErrNotFound, @@ -923,7 +930,7 @@ func (b *InMemoryBackend) DisassociateAgentCollaborator(agentID, collaboratorID ) } - delete(collabs, collaboratorID) + collabs.Delete(collaboratorID) return nil } @@ -940,11 +947,11 @@ func (b *InMemoryBackend) IngestKnowledgeBaseDocuments( b.mu.Lock("IngestKnowledgeBaseDocuments") defer b.mu.Unlock() - if _, ok := b.knowledgeBases[kbID]; !ok { + if _, ok := b.knowledgeBases.Get(kbID); !ok { return nil, fmt.Errorf("%w: knowledge base %q not found", ErrNotFound, kbID) } - if _, ok := b.dataSources[kbID+"/"+dsID]; !ok { + if _, ok := b.dataSources.Get(kbID + "/" + dsID); !ok { return nil, fmt.Errorf("%w: data source %q not found", ErrNotFound, dsID) } @@ -957,7 +964,7 @@ func (b *InMemoryBackend) IngestKnowledgeBaseDocuments( DocumentID: docID, Status: docStatusActive, } - b.kbDocuments[kbDocKey(kbID, dsID, docID)] = doc + b.kbDocuments.Put(doc) cp := *doc docs = append(docs, &cp) } @@ -974,11 +981,10 @@ func (b *InMemoryBackend) ListKnowledgeBaseDocuments( b.mu.RLock("ListKnowledgeBaseDocuments") defer b.mu.RUnlock() - prefix := kbID + "/" + dsID + "/" list := make([]*KnowledgeBaseDocument, 0) - for k, doc := range b.kbDocuments { - if len(k) > len(prefix) && k[:len(prefix)] == prefix { + for _, doc := range b.kbDocuments.All() { + if doc.KnowledgeBaseID == kbID && doc.DataSourceID == dsID { cp := *doc list = append(list, &cp) } @@ -997,13 +1003,13 @@ func (b *InMemoryBackend) GetKnowledgeBaseDocuments( b.mu.RLock("GetKnowledgeBaseDocuments") defer b.mu.RUnlock() - if _, ok := b.dataSources[kbID+"/"+dsID]; !ok { + if _, ok := b.dataSources.Get(kbID + "/" + dsID); !ok { return nil, fmt.Errorf("%w: data source %q not found", ErrNotFound, dsID) } docs := make([]*KnowledgeBaseDocument, 0, len(documentIDs)) for _, documentID := range documentIDs { - doc, ok := b.kbDocuments[kbDocKey(kbID, dsID, documentID)] + doc, ok := b.kbDocuments.Get(kbDocKey(kbID, dsID, documentID)) if !ok { continue } @@ -1023,7 +1029,7 @@ func (b *InMemoryBackend) DeleteKnowledgeBaseDocuments( defer b.mu.Unlock() for _, docID := range documentIDs { - delete(b.kbDocuments, kbDocKey(kbID, dsID, docID)) + b.kbDocuments.Delete(kbDocKey(kbID, dsID, docID)) } return nil @@ -1041,7 +1047,7 @@ func (b *InMemoryBackend) UpdateKnowledgeBaseDocuments( for _, docID := range documentIDs { key := kbDocKey(kbID, dsID, docID) - doc, ok := b.kbDocuments[key] + doc, ok := b.kbDocuments.Get(key) if !ok { doc = &KnowledgeBaseDocument{ @@ -1050,7 +1056,7 @@ func (b *InMemoryBackend) UpdateKnowledgeBaseDocuments( DocumentID: docID, Status: docStatusActive, } - b.kbDocuments[key] = doc + b.kbDocuments.Put(doc) } cp := *doc @@ -1070,7 +1076,7 @@ func (b *InMemoryBackend) StopIngestionJob(kbID, dsID, jobID string) (*Ingestion b.mu.Lock("StopIngestionJob") defer b.mu.Unlock() - job, ok := b.ingestionJobs[ingestionJobKey(kbID, dsID, jobID)] + job, ok := b.ingestionJobs.Get(ingestionJobKey(kbID, dsID, jobID)) if !ok { return nil, fmt.Errorf("%w: ingestion job %q not found", ErrNotFound, jobID) } @@ -1161,7 +1167,7 @@ func (b *InMemoryBackend) DeleteAgentMemory(agentID, sessionID string) error { b.mu.Lock("DeleteAgentMemory") defer b.mu.Unlock() - if _, ok := b.agents[agentID]; !ok { + if _, ok := b.agents.Get(agentID); !ok { return fmt.Errorf("%w: agent %q not found", ErrNotFound, agentID) } diff --git a/services/bedrock/backend_ops.go b/services/bedrock/backend_ops.go index f33507a05..e2daffae5 100644 --- a/services/bedrock/backend_ops.go +++ b/services/bedrock/backend_ops.go @@ -76,7 +76,7 @@ func (b *InMemoryBackend) GetEvaluationJob(jobARN string) (*EvaluationJob, error b.mu.RLock("GetEvaluationJob") defer b.mu.RUnlock() - job, ok := b.evaluationJobs[jobARN] + job, ok := b.evaluationJobs.Get(jobARN) if !ok { return nil, fmt.Errorf("%w: evaluation job %s not found", ErrNotFound, jobARN) } @@ -92,8 +92,8 @@ func (b *InMemoryBackend) ListEvaluationJobs() []*EvaluationJob { b.mu.RLock("ListEvaluationJobs") defer b.mu.RUnlock() - jobs := make([]*EvaluationJob, 0, len(b.evaluationJobs)) - for _, j := range b.evaluationJobs { + jobs := make([]*EvaluationJob, 0, b.evaluationJobs.Len()) + for _, j := range b.evaluationJobs.All() { cp := *j cp.Tags = copyTags(j.Tags) jobs = append(jobs, &cp) @@ -111,7 +111,7 @@ func (b *InMemoryBackend) StopEvaluationJob(jobARN string) error { b.mu.Lock("StopEvaluationJob") defer b.mu.Unlock() - job, ok := b.evaluationJobs[jobARN] + job, ok := b.evaluationJobs.Get(jobARN) if !ok { return fmt.Errorf("%w: evaluation job %s not found", ErrNotFound, jobARN) } @@ -138,7 +138,7 @@ func (b *InMemoryBackend) GetAutomatedReasoningPolicy(policyARN string) (*Automa b.mu.RLock("GetAutomatedReasoningPolicy") defer b.mu.RUnlock() - policy, ok := b.automatedReasoningPolicies[policyARN] + policy, ok := b.automatedReasoningPolicies.Get(policyARN) if !ok { return nil, fmt.Errorf("%w: automated reasoning policy %s not found", ErrNotFound, policyARN) } @@ -154,8 +154,8 @@ func (b *InMemoryBackend) ListAutomatedReasoningPolicies() []*AutomatedReasoning b.mu.RLock("ListAutomatedReasoningPolicies") defer b.mu.RUnlock() - policies := make([]*AutomatedReasoningPolicy, 0, len(b.automatedReasoningPolicies)) - for _, p := range b.automatedReasoningPolicies { + policies := make([]*AutomatedReasoningPolicy, 0, b.automatedReasoningPolicies.Len()) + for _, p := range b.automatedReasoningPolicies.All() { cp := *p cp.Tags = copyTags(p.Tags) policies = append(policies, &cp) @@ -175,7 +175,7 @@ func (b *InMemoryBackend) UpdateAutomatedReasoningPolicy( b.mu.Lock("UpdateAutomatedReasoningPolicy") defer b.mu.Unlock() - policy, ok := b.automatedReasoningPolicies[policyARN] + policy, ok := b.automatedReasoningPolicies.Get(policyARN) if !ok { return nil, fmt.Errorf("%w: automated reasoning policy %s not found", ErrNotFound, policyARN) } @@ -194,24 +194,24 @@ func (b *InMemoryBackend) DeleteAutomatedReasoningPolicy(policyARN string) error b.mu.Lock("DeleteAutomatedReasoningPolicy") defer b.mu.Unlock() - policy, ok := b.automatedReasoningPolicies[policyARN] + policy, ok := b.automatedReasoningPolicies.Get(policyARN) if !ok { return fmt.Errorf("%w: automated reasoning policy %s not found", ErrNotFound, policyARN) } delete(b.arpByName, policy.Name) - delete(b.automatedReasoningPolicies, policyARN) + b.automatedReasoningPolicies.Delete(policyARN) // Remove associated workflows and test cases. - for id, wf := range b.arpBuildWorkflows { + for _, wf := range b.arpBuildWorkflows.All() { if wf.PolicyArn == policyARN { - delete(b.arpBuildWorkflows, id) + b.arpBuildWorkflows.Delete(wf.BuildWorkflowID) } } - for id, tc := range b.arpTestCases { + for _, tc := range b.arpTestCases.All() { if tc.PolicyArn == policyARN { - delete(b.arpTestCases, id) + b.arpTestCases.Delete(tc.TestCaseID) } } @@ -227,7 +227,7 @@ func (b *InMemoryBackend) StartAutomatedReasoningPolicyBuildWorkflow( b.mu.Lock("StartAutomatedReasoningPolicyBuildWorkflow") defer b.mu.Unlock() - if _, ok := b.automatedReasoningPolicies[policyARN]; !ok { + if _, ok := b.automatedReasoningPolicies.Get(policyARN); !ok { return nil, fmt.Errorf("%w: automated reasoning policy %s not found", ErrNotFound, policyARN) } @@ -239,7 +239,7 @@ func (b *InMemoryBackend) StartAutomatedReasoningPolicyBuildWorkflow( PolicyArn: policyARN, Status: statusRunning, } - b.arpBuildWorkflows[id] = wf + b.arpBuildWorkflows.Put(wf) cp := *wf return &cp, nil @@ -252,7 +252,7 @@ func (b *InMemoryBackend) GetAutomatedReasoningPolicyBuildWorkflow( b.mu.RLock("GetAutomatedReasoningPolicyBuildWorkflow") defer b.mu.RUnlock() - wf, ok := b.arpBuildWorkflows[workflowID] + wf, ok := b.arpBuildWorkflows.Get(workflowID) if !ok || wf.PolicyArn != policyARN { return nil, fmt.Errorf("%w: build workflow %s not found", ErrNotFound, workflowID) } @@ -270,7 +270,7 @@ func (b *InMemoryBackend) ListAutomatedReasoningPolicyBuildWorkflows( defer b.mu.RUnlock() var workflows []*AutomatedReasoningPolicyBuildWorkflow - for _, wf := range b.arpBuildWorkflows { + for _, wf := range b.arpBuildWorkflows.All() { if wf.PolicyArn == policyARN { cp := *wf workflows = append(workflows, &cp) @@ -289,12 +289,12 @@ func (b *InMemoryBackend) DeleteAutomatedReasoningPolicyBuildWorkflow(policyARN, b.mu.Lock("DeleteAutomatedReasoningPolicyBuildWorkflow") defer b.mu.Unlock() - wf, ok := b.arpBuildWorkflows[workflowID] + wf, ok := b.arpBuildWorkflows.Get(workflowID) if !ok || wf.PolicyArn != policyARN { return fmt.Errorf("%w: build workflow %s not found", ErrNotFound, workflowID) } - delete(b.arpBuildWorkflows, workflowID) + b.arpBuildWorkflows.Delete(workflowID) return nil } @@ -308,7 +308,7 @@ func (b *InMemoryBackend) GetAutomatedReasoningPolicyTestCase( b.mu.RLock("GetAutomatedReasoningPolicyTestCase") defer b.mu.RUnlock() - tc, ok := b.arpTestCases[testCaseID] + tc, ok := b.arpTestCases.Get(testCaseID) if !ok || tc.PolicyArn != policyARN { return nil, fmt.Errorf("%w: test case %s not found", ErrNotFound, testCaseID) } @@ -324,7 +324,7 @@ func (b *InMemoryBackend) ListAutomatedReasoningPolicyTestCases(policyARN string defer b.mu.RUnlock() var cases []*AutomatedReasoningPolicyTestCase - for _, tc := range b.arpTestCases { + for _, tc := range b.arpTestCases.All() { if tc.PolicyArn == policyARN { cp := *tc cases = append(cases, &cp) @@ -345,7 +345,7 @@ func (b *InMemoryBackend) UpdateAutomatedReasoningPolicyTestCase( b.mu.Lock("UpdateAutomatedReasoningPolicyTestCase") defer b.mu.Unlock() - tc, ok := b.arpTestCases[testCaseID] + tc, ok := b.arpTestCases.Get(testCaseID) if !ok || tc.PolicyArn != policyARN { return nil, fmt.Errorf("%w: test case %s not found", ErrNotFound, testCaseID) } @@ -360,12 +360,12 @@ func (b *InMemoryBackend) DeleteAutomatedReasoningPolicyTestCase(policyARN, test b.mu.Lock("DeleteAutomatedReasoningPolicyTestCase") defer b.mu.Unlock() - tc, ok := b.arpTestCases[testCaseID] + tc, ok := b.arpTestCases.Get(testCaseID) if !ok || tc.PolicyArn != policyARN { return fmt.Errorf("%w: test case %s not found", ErrNotFound, testCaseID) } - delete(b.arpTestCases, testCaseID) + b.arpTestCases.Delete(testCaseID) return nil } @@ -377,7 +377,7 @@ func (b *InMemoryBackend) GetAutomatedReasoningPolicyAnnotations(policyARN strin b.mu.RLock("GetAutomatedReasoningPolicyAnnotations") defer b.mu.RUnlock() - if _, ok := b.automatedReasoningPolicies[policyARN]; !ok { + if _, ok := b.automatedReasoningPolicies.Get(policyARN); !ok { return nil, fmt.Errorf("%w: automated reasoning policy %s not found", ErrNotFound, policyARN) } @@ -394,7 +394,7 @@ func (b *InMemoryBackend) UpdateAutomatedReasoningPolicyAnnotations(policyARN st b.mu.Lock("UpdateAutomatedReasoningPolicyAnnotations") defer b.mu.Unlock() - if _, ok := b.automatedReasoningPolicies[policyARN]; !ok { + if _, ok := b.automatedReasoningPolicies.Get(policyARN); !ok { return fmt.Errorf("%w: automated reasoning policy %s not found", ErrNotFound, policyARN) } @@ -414,7 +414,7 @@ func (b *InMemoryBackend) GetAutomatedReasoningPolicyNextScenario(policyARN stri b.mu.RLock("GetAutomatedReasoningPolicyNextScenario") defer b.mu.RUnlock() - if _, ok := b.automatedReasoningPolicies[policyARN]; !ok { + if _, ok := b.automatedReasoningPolicies.Get(policyARN); !ok { return nil, fmt.Errorf("%w: automated reasoning policy %s not found", ErrNotFound, policyARN) } @@ -428,7 +428,7 @@ func (b *InMemoryBackend) GetAutomatedReasoningPolicyBuildWorkflowResultAssets( b.mu.RLock("GetAutomatedReasoningPolicyBuildWorkflowResultAssets") defer b.mu.RUnlock() - wf, ok := b.arpBuildWorkflows[workflowID] + wf, ok := b.arpBuildWorkflows.Get(workflowID) if !ok || wf.PolicyArn != policyARN { return nil, fmt.Errorf("%w: build workflow %s not found", ErrNotFound, workflowID) } @@ -442,7 +442,7 @@ func (b *InMemoryBackend) ExportAutomatedReasoningPolicyVersion(policyARN, versi defer b.mu.RUnlock() key := policyARN + ":" + version - v, ok := b.arpVersions[key] + v, ok := b.arpVersions.Get(key) if !ok { return nil, fmt.Errorf("%w: version %s of policy %s not found", ErrNotFound, version, policyARN) @@ -463,7 +463,7 @@ func (b *InMemoryBackend) StartAutomatedReasoningPolicyTestWorkflow( b.mu.Lock("StartAutomatedReasoningPolicyTestWorkflow") defer b.mu.Unlock() - tc, ok := b.arpTestCases[testCaseID] + tc, ok := b.arpTestCases.Get(testCaseID) if !ok || tc.PolicyArn != policyARN { return nil, fmt.Errorf("%w: test case %s not found", ErrNotFound, testCaseID) } @@ -476,7 +476,7 @@ func (b *InMemoryBackend) GetAutomatedReasoningPolicyTestResult(policyARN, testC b.mu.RLock("GetAutomatedReasoningPolicyTestResult") defer b.mu.RUnlock() - tc, ok := b.arpTestCases[testCaseID] + tc, ok := b.arpTestCases.Get(testCaseID) if !ok || tc.PolicyArn != policyARN { return nil, fmt.Errorf("%w: test case %s not found", ErrNotFound, testCaseID) } @@ -490,7 +490,7 @@ func (b *InMemoryBackend) ListAutomatedReasoningPolicyTestResults(policyARN stri defer b.mu.RUnlock() var results []map[string]any - for _, tc := range b.arpTestCases { + for _, tc := range b.arpTestCases.All() { if tc.PolicyArn == policyARN { results = append(results, map[string]any{ keyTestCaseID: tc.TestCaseID, @@ -526,7 +526,7 @@ func (b *InMemoryBackend) CreateModelInvocationJob( return nil, fmt.Errorf("%w: jobName is required", ErrValidation) } - for _, j := range b.modelInvocationJobs { + for _, j := range b.modelInvocationJobs.All() { if j.JobName == name { return nil, fmt.Errorf("%w: model invocation job %s already exists", ErrAlreadyExists, name) } @@ -554,7 +554,7 @@ func (b *InMemoryBackend) CreateModelInvocationJob( job.ClientToken = opt.ClientToken } - b.modelInvocationJobs[jobARN] = job + b.modelInvocationJobs.Put(job) cp := *job cp.Tags = copyTags(job.Tags) @@ -566,7 +566,7 @@ func (b *InMemoryBackend) GetModelInvocationJob(jobARN string) (*ModelInvocation b.mu.RLock("GetModelInvocationJob") defer b.mu.RUnlock() - job, ok := b.modelInvocationJobs[jobARN] + job, ok := b.modelInvocationJobs.Get(jobARN) if !ok { return nil, fmt.Errorf("%w: model invocation job %s not found", ErrNotFound, jobARN) } @@ -595,8 +595,8 @@ func (b *InMemoryBackend) ListModelInvocationJobs( b.mu.RLock("ListModelInvocationJobs") defer b.mu.RUnlock() - jobs := make([]*ModelInvocationJob, 0, len(b.modelInvocationJobs)) - for _, j := range b.modelInvocationJobs { + jobs := make([]*ModelInvocationJob, 0, b.modelInvocationJobs.Len()) + for _, j := range b.modelInvocationJobs.All() { if !matchesInvocationJobFilter(j, in) { continue } @@ -690,7 +690,7 @@ func (b *InMemoryBackend) StopModelInvocationJob(jobARN string) error { b.mu.Lock("StopModelInvocationJob") defer b.mu.Unlock() - job, ok := b.modelInvocationJobs[jobARN] + job, ok := b.modelInvocationJobs.Get(jobARN) if !ok { return fmt.Errorf("%w: model invocation job %s not found", ErrNotFound, jobARN) } @@ -717,7 +717,7 @@ func (b *InMemoryBackend) GetImportedModel(modelARN string) (*ModelImportJob, er b.mu.RLock("GetImportedModel") defer b.mu.RUnlock() - for _, j := range b.modelImportJobs { + for _, j := range b.modelImportJobs.All() { if j.ImportedModelArn == modelARN { cp := *j cp.Tags = copyTags(j.Tags) @@ -735,7 +735,7 @@ func (b *InMemoryBackend) ListImportedModels() []*ModelImportJob { defer b.mu.RUnlock() var models []*ModelImportJob - for _, j := range b.modelImportJobs { + for _, j := range b.modelImportJobs.All() { if j.ImportedModelArn != "" { cp := *j cp.Tags = copyTags(j.Tags) @@ -755,9 +755,9 @@ func (b *InMemoryBackend) DeleteImportedModel(modelARN string) error { b.mu.Lock("DeleteImportedModel") defer b.mu.Unlock() - for jobARN, j := range b.modelImportJobs { + for _, j := range b.modelImportJobs.All() { if j.ImportedModelArn == modelARN { - delete(b.modelImportJobs, jobARN) + b.modelImportJobs.Delete(j.JobArn) return nil } @@ -793,7 +793,7 @@ func (b *InMemoryBackend) CreatePromptRouter(name string, tags []Tag) (*PromptRo UpdatedAt: now, Tags: copyTags(tags), } - b.promptRouters[routerARN] = router + b.promptRouters.Put(router) b.promptRoutersByName[name] = routerARN cp := *router cp.Tags = copyTags(router.Tags) @@ -806,7 +806,7 @@ func (b *InMemoryBackend) GetPromptRouter(routerARN string) (*PromptRouter, erro b.mu.RLock("GetPromptRouter") defer b.mu.RUnlock() - router, ok := b.promptRouters[routerARN] + router, ok := b.promptRouters.Get(routerARN) if !ok { return nil, fmt.Errorf("%w: prompt router %s not found", ErrNotFound, routerARN) } @@ -822,8 +822,8 @@ func (b *InMemoryBackend) ListPromptRouters() []*PromptRouter { b.mu.RLock("ListPromptRouters") defer b.mu.RUnlock() - routers := make([]*PromptRouter, 0, len(b.promptRouters)) - for _, r := range b.promptRouters { + routers := make([]*PromptRouter, 0, b.promptRouters.Len()) + for _, r := range b.promptRouters.All() { cp := *r cp.Tags = copyTags(r.Tags) routers = append(routers, &cp) @@ -841,13 +841,13 @@ func (b *InMemoryBackend) DeletePromptRouter(routerARN string) error { b.mu.Lock("DeletePromptRouter") defer b.mu.Unlock() - router, ok := b.promptRouters[routerARN] + router, ok := b.promptRouters.Get(routerARN) if !ok { return fmt.Errorf("%w: prompt router %s not found", ErrNotFound, routerARN) } delete(b.promptRoutersByName, router.PromptRouterName) - delete(b.promptRouters, routerARN) + b.promptRouters.Delete(routerARN) return nil } @@ -859,7 +859,7 @@ func (b *InMemoryBackend) GetCustomModelDeployment(deployARN string) (*CustomMod b.mu.RLock("GetCustomModelDeployment") defer b.mu.RUnlock() - d, ok := b.customModelDeployments[deployARN] + d, ok := b.customModelDeployments.Get(deployARN) if !ok { return nil, fmt.Errorf("%w: custom model deployment %s not found", ErrNotFound, deployARN) } @@ -875,8 +875,8 @@ func (b *InMemoryBackend) ListCustomModelDeployments() []*CustomModelDeployment b.mu.RLock("ListCustomModelDeployments") defer b.mu.RUnlock() - deployments := make([]*CustomModelDeployment, 0, len(b.customModelDeployments)) - for _, d := range b.customModelDeployments { + deployments := make([]*CustomModelDeployment, 0, b.customModelDeployments.Len()) + for _, d := range b.customModelDeployments.All() { cp := *d cp.Tags = copyTags(d.Tags) deployments = append(deployments, &cp) @@ -894,7 +894,7 @@ func (b *InMemoryBackend) UpdateCustomModelDeployment(deployARN string) (*Custom b.mu.Lock("UpdateCustomModelDeployment") defer b.mu.Unlock() - d, ok := b.customModelDeployments[deployARN] + d, ok := b.customModelDeployments.Get(deployARN) if !ok { return nil, fmt.Errorf("%w: custom model deployment %s not found", ErrNotFound, deployARN) } @@ -912,13 +912,13 @@ func (b *InMemoryBackend) DeleteCustomModelDeployment(deployARN string) error { b.mu.Lock("DeleteCustomModelDeployment") defer b.mu.Unlock() - d, ok := b.customModelDeployments[deployARN] + d, ok := b.customModelDeployments.Get(deployARN) if !ok { return fmt.Errorf("%w: custom model deployment %s not found", ErrNotFound, deployARN) } delete(b.customModelDeployByName, d.ModelDeploymentName) - delete(b.customModelDeployments, deployARN) + b.customModelDeployments.Delete(deployARN) return nil } @@ -930,8 +930,8 @@ func (b *InMemoryBackend) ListFoundationModelAgreementOffers() []*FoundationMode b.mu.RLock("ListFoundationModelAgreementOffers") defer b.mu.RUnlock() - agreements := make([]*FoundationModelAgreement, 0, len(b.foundationModelAgreements)) - for _, a := range b.foundationModelAgreements { + agreements := make([]*FoundationModelAgreement, 0, b.foundationModelAgreements.Len()) + for _, a := range b.foundationModelAgreements.All() { cp := *a agreements = append(agreements, &cp) } @@ -948,11 +948,11 @@ func (b *InMemoryBackend) DeleteFoundationModelAgreement(modelID string) error { b.mu.Lock("DeleteFoundationModelAgreement") defer b.mu.Unlock() - if _, ok := b.foundationModelAgreements[modelID]; !ok { + if _, ok := b.foundationModelAgreements.Get(modelID); !ok { return fmt.Errorf("%w: foundation model agreement for %s not found", ErrNotFound, modelID) } - delete(b.foundationModelAgreements, modelID) + b.foundationModelAgreements.Delete(modelID) return nil } @@ -986,8 +986,8 @@ func (b *InMemoryBackend) ListEnforcedGuardrailsConfiguration() []*EnforcedGuard b.mu.RLock("ListEnforcedGuardrailsConfiguration") defer b.mu.RUnlock() - configs := make([]*EnforcedGuardrailConfig, 0, len(b.enforcedGuardrailConfigs)) - for _, c := range b.enforcedGuardrailConfigs { + configs := make([]*EnforcedGuardrailConfig, 0, b.enforcedGuardrailConfigs.Len()) + for _, c := range b.enforcedGuardrailConfigs.All() { cp := *c configs = append(configs, &cp) } @@ -1004,14 +1004,10 @@ func (b *InMemoryBackend) PutEnforcedGuardrailConfiguration(guardrailID, version b.mu.Lock("PutEnforcedGuardrailConfiguration") defer b.mu.Unlock() - if b.enforcedGuardrailConfigs == nil { - b.enforcedGuardrailConfigs = make(map[string]*EnforcedGuardrailConfig) - } - - b.enforcedGuardrailConfigs[guardrailID] = &EnforcedGuardrailConfig{ + b.enforcedGuardrailConfigs.Put(&EnforcedGuardrailConfig{ GuardrailID: guardrailID, GuardrailVersion: version, - } + }) } // DeleteEnforcedGuardrailConfiguration removes an enforced guardrail config. @@ -1019,11 +1015,11 @@ func (b *InMemoryBackend) DeleteEnforcedGuardrailConfiguration(guardrailID strin b.mu.Lock("DeleteEnforcedGuardrailConfiguration") defer b.mu.Unlock() - if _, ok := b.enforcedGuardrailConfigs[guardrailID]; !ok { + if _, ok := b.enforcedGuardrailConfigs.Get(guardrailID); !ok { return fmt.Errorf("%w: enforced guardrail configuration for %s not found", ErrNotFound, guardrailID) } - delete(b.enforcedGuardrailConfigs, guardrailID) + b.enforcedGuardrailConfigs.Delete(guardrailID) return nil } diff --git a/services/bedrock/export_test.go b/services/bedrock/export_test.go index cef572384..8ba6b62fc 100644 --- a/services/bedrock/export_test.go +++ b/services/bedrock/export_test.go @@ -1,6 +1,9 @@ package bedrock -import "strconv" +import ( + "encoding/json" + "strconv" +) // AppendFoundationModelsForTest appends additional foundation models to the backend. // This is only used in tests to populate beyond the default seeded models. @@ -23,7 +26,40 @@ func (b *InMemoryBackend) AddBuildWorkflowForTest(policyARN string) *AutomatedRe PolicyArn: policyARN, Status: "Running", } - b.arpBuildWorkflows[id] = wf + b.arpBuildWorkflows.Put(wf) return wf } + +// SnapshotTablesForTest returns a snapshot of every store.Table registered on +// b.registry, keyed by table name (including per-parent lazy tables such as +// "flowVersions:"). It is a test-only bridge to the unexported +// registry so blackbox tests can exercise a full Snapshot->Restore round trip +// of the Phase 3.3 pkgs/store conversion; bedrock has no persistence.go of +// its own (see store_setup.go), so there is no production Snapshot to call +// through instead. +func (b *InMemoryBackend) SnapshotTablesForTest() (map[string]json.RawMessage, error) { + b.mu.RLock("SnapshotTablesForTest") + defer b.mu.RUnlock() + + return b.registry.SnapshotAll() +} + +// RestoreTablesForTest replaces every store.Table registered on b.registry +// with the contents of data, as produced by SnapshotTablesForTest. +func (b *InMemoryBackend) RestoreTablesForTest(data map[string]json.RawMessage) error { + b.mu.Lock("RestoreTablesForTest") + defer b.mu.Unlock() + + return b.registry.RestoreAll(data) +} + +// ResetTablesForTest clears every store.Table registered on b.registry in +// place, leaving secondary-index maps (guardrailsByName, etc.) and counters +// untouched. Mirrors the registry.ResetAll call inside Reset. +func (b *InMemoryBackend) ResetTablesForTest() { + b.mu.Lock("ResetTablesForTest") + defer b.mu.Unlock() + + b.registry.ResetAll() +} diff --git a/services/bedrock/handler_agents.go b/services/bedrock/handler_agents.go index 19fa75499..45fc36fcc 100644 --- a/services/bedrock/handler_agents.go +++ b/services/bedrock/handler_agents.go @@ -136,32 +136,26 @@ func (h *AgentsHandler) ChaosOperations() []string { return h.GetSupportedOperat func (h *AgentsHandler) ChaosRegions() []string { return []string{h.Backend.region} } // Reset clears all agent/kb state. +// +// registry.ResetAll empties every store.Table registered on this backend +// instance -- including the non-agent-domain (guardrails/models/...) tables +// registerAllTables also registers -- but AgentsHandler's own backend +// instance (see AgentsProvider.Init in provider.go) never receives core +// Bedrock mutations (RouteMatcher only matches /agents/ and /knowledgebases/ +// paths), so those tables are always already empty here; resetting them is a +// no-op, not a behavior change. func (h *AgentsHandler) Reset() { h.Backend.mu.Lock("AgentsHandler.Reset") defer h.Backend.mu.Unlock() - h.Backend.agents = make(map[string]*Agent) + h.Backend.registry.ResetAll() h.Backend.agentsByName = make(map[string]string) - h.Backend.agentActionGroups = make(map[string]*AgentActionGroup) - h.Backend.agentAliases = make(map[string]*AgentAlias) - h.Backend.agentKBAssociations = make(map[string]*AgentKnowledgeBaseAssociation) - h.Backend.knowledgeBases = make(map[string]*KnowledgeBase) h.Backend.kbByName = make(map[string]string) - h.Backend.dataSources = make(map[string]*DataSource) - h.Backend.ingestionJobs = make(map[string]*IngestionJob) - h.Backend.flows = make(map[string]*Flow) h.Backend.flowsByName = make(map[string]string) - h.Backend.flowAliases = make(map[string]*FlowAlias) - h.Backend.flowVersions = make(map[string]map[string]*FlowVersion) h.Backend.flowVersionCounters = make(map[string]int) - h.Backend.prompts = make(map[string]*Prompt) h.Backend.promptsByName = make(map[string]string) - h.Backend.promptVersions = make(map[string]map[string]*PromptVersion) h.Backend.promptVersionCounters = make(map[string]int) - h.Backend.agentVersions = make(map[string]map[string]*AgentVersion) h.Backend.agentVersionCounters = make(map[string]int) - h.Backend.agentCollaborators = make(map[string]map[string]*AgentCollaborator) - h.Backend.kbDocuments = make(map[string]*KnowledgeBaseDocument) h.Backend.agentTags = make(map[string]map[string]string) h.Backend.agentMemory = make(map[string][]any) // Reset ID counters for deterministic IDs after reset. diff --git a/services/bedrock/store_setup.go b/services/bedrock/store_setup.go new file mode 100644 index 000000000..e07b48225 --- /dev/null +++ b/services/bedrock/store_setup.go @@ -0,0 +1,326 @@ +package bedrock + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend is registered exactly once, +// here, as a *store.Table[T] on b.registry. See pkgs/store's package doc and +// the services/ec2 (commit 12e611a4) and services/sqs (commit 0f09d77c) +// pilots for the pattern this follows. +// +// bedrock has no pre-existing persistence.go (it was never wired into +// pkgs/persistence's Manager -- see cli.go, which only registers backends +// that implement persistence.Persistable). This conversion is therefore an +// internal storage-layer swap only: it eliminates the map init/Reset +// boilerplate but does not add Snapshot/Restore, since none existed to +// preserve. +// +// flowVersions, promptVersions, agentVersions, and agentCollaborators were +// previously two-level maps (map[string]map[string]*T, parent ID -> child ID +// -> value). Each child value type carries its parent's ID as a field +// (FlowVersion.FlowID, PromptVersion.PromptID, AgentVersion.AgentID, +// AgentCollaborator.AgentID), so each parent's children are a pure-key +// store.Table; the outer map becomes map[string]*store.Table[T], one table +// per parent, registered lazily on first use -- exactly the per-region +// lazy-registration pattern services/kms uses for keys/aliases/grants (see +// keysStore/aliasesStore/grantsRegion in services/kms/backend.go). +// +// A handful of fields are deliberately NOT registered here and remain plain +// maps -- see the comment block above registerAllTables for the list and why. +import ( + "strings" + + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +// --------------------------------------------------------------------------- +// Key functions for flat (single-level) tables. +// --------------------------------------------------------------------------- + +func guardrailsKeyFn(v *Guardrail) string { return v.GuardrailID } + +// guardrailVersionsKeyFn mirrors the "guardrailID:version" key CreateGuardrailVersion builds. +func guardrailVersionsKeyFn(v *GuardrailVersion) string { return v.GuardrailID + ":" + v.Version } + +func provisionedModelThroughputsKeyFn(v *ProvisionedModelThroughput) string { + return v.ProvisionedModelArn +} + +func evaluationJobsKeyFn(v *EvaluationJob) string { return v.JobArn } + +func automatedReasoningPoliciesKeyFn(v *AutomatedReasoningPolicy) string { return v.PolicyArn } + +func arpBuildWorkflowsKeyFn(v *AutomatedReasoningPolicyBuildWorkflow) string { + return v.BuildWorkflowID +} + +func arpTestCasesKeyFn(v *AutomatedReasoningPolicyTestCase) string { return v.TestCaseID } + +// arpVersionsKeyFn mirrors the "policyARN:versionNum" key +// CreateAutomatedReasoningPolicyVersion builds. The value's own PolicyArn +// field holds the *versioned* ARN (policyARN + "/version/" + versionNum, see +// CreateAutomatedReasoningPolicyVersion), so the base policy ARN is recovered +// by trimming that suffix back off -- still a pure function of the value. +func arpVersionsKeyFn(v *AutomatedReasoningPolicyVersion) string { + base := strings.TrimSuffix(v.PolicyArn, "/version/"+v.Version) + + return base + ":" + v.Version +} + +func customModelsKeyFn(v *CustomModel) string { return v.ModelArn } +func customModelDeploymentsKeyFn(v *CustomModelDeployment) string { return v.CustomModelDeploymentArn } +func foundationModelAgreementsKeyFn(v *FoundationModelAgreement) string { return v.ModelID } +func modelCustomizationJobsKeyFn(v *ModelCustomizationJob) string { return v.JobArn } +func modelCopyJobsKeyFn(v *ModelCopyJob) string { return v.JobArn } +func modelImportJobsKeyFn(v *ModelImportJob) string { return v.JobArn } +func inferenceProfilesKeyFn(v *InferenceProfile) string { return v.InferenceProfileArn } +func marketplaceEndpointsKeyFn(v *MarketplaceModelEndpoint) string { return v.EndpointArn } +func modelInvocationJobsKeyFn(v *ModelInvocationJob) string { return v.JobArn } +func promptRoutersKeyFn(v *PromptRouter) string { return v.PromptRouterArn } +func enforcedGuardrailConfigsKeyFn(v *EnforcedGuardrailConfig) string { return v.GuardrailID } +func agentsKeyFn(v *Agent) string { return v.AgentID } + +func agentActionGroupsKeyFn(v *AgentActionGroup) string { + return agentActionGroupKey(v.AgentID, v.ActionGroupID) +} + +func agentAliasesKeyFn(v *AgentAlias) string { return agentAliasKey(v.AgentID, v.AgentAliasID) } + +func agentKBAssociationsKeyFn(v *AgentKnowledgeBaseAssociation) string { + return agentKBKey(v.AgentID, v.KnowledgeBaseID) +} + +func knowledgeBasesKeyFn(v *KnowledgeBase) string { return v.KnowledgeBaseID } +func dataSourcesKeyFn(v *DataSource) string { return v.KnowledgeBaseID + "/" + v.DataSourceID } + +func ingestionJobsKeyFn(v *IngestionJob) string { + return ingestionJobKey(v.KnowledgeBaseID, v.DataSourceID, v.IngestionJobID) +} + +func flowsKeyFn(v *Flow) string { return v.FlowID } +func flowAliasesKeyFn(v *FlowAlias) string { return flowAliasKey(v.FlowID, v.FlowAliasID) } +func promptsKeyFn(v *Prompt) string { return v.PromptID } + +func kbDocumentsKeyFn(v *KnowledgeBaseDocument) string { + return kbDocKey(v.KnowledgeBaseID, v.DataSourceID, v.DocumentID) +} + +// --------------------------------------------------------------------------- +// Key functions for the per-parent lazily-registered tables. +// --------------------------------------------------------------------------- + +func flowVersionKeyFn(v *FlowVersion) string { return v.Version } +func promptVersionKeyFn(v *PromptVersion) string { return v.Version } +func agentVersionKeyFn(v *AgentVersion) string { return v.AgentVersion } +func agentCollaboratorKeyFn(v *AgentCollaborator) string { return v.CollaboratorID } + +// flowVersionsStore returns (registering lazily) the per-flow version table. +// Registration mutates b.registry and b.flowVersions, so this must only be +// called from a write-locked path (mirrors services/kms's keysStore). +func (b *InMemoryBackend) flowVersionsStore(flowID string) *store.Table[FlowVersion] { + if t, ok := b.flowVersions[flowID]; ok { + return t + } + + t := store.Register(b.registry, "flowVersions:"+flowID, store.New(flowVersionKeyFn)) + b.flowVersions[flowID] = t + + return t +} + +// promptVersionsStore returns (registering lazily) the per-prompt version table. +func (b *InMemoryBackend) promptVersionsStore(promptID string) *store.Table[PromptVersion] { + if t, ok := b.promptVersions[promptID]; ok { + return t + } + + t := store.Register(b.registry, "promptVersions:"+promptID, store.New(promptVersionKeyFn)) + b.promptVersions[promptID] = t + + return t +} + +// agentVersionsStore returns (registering lazily) the per-agent version table. +func (b *InMemoryBackend) agentVersionsStore(agentID string) *store.Table[AgentVersion] { + if t, ok := b.agentVersions[agentID]; ok { + return t + } + + t := store.Register(b.registry, "agentVersions:"+agentID, store.New(agentVersionKeyFn)) + b.agentVersions[agentID] = t + + return t +} + +// agentCollaboratorsStore returns (registering lazily) the per-agent collaborators table. +func (b *InMemoryBackend) agentCollaboratorsStore(agentID string) *store.Table[AgentCollaborator] { + if t, ok := b.agentCollaborators[agentID]; ok { + return t + } + + t := store.Register(b.registry, "agentCollaborators:"+agentID, store.New(agentCollaboratorKeyFn)) + b.agentCollaborators[agentID] = t + + return t +} + +// registerAllTables registers every converted resource map on b.registry +// exactly once. It must be called during construction only (immediately after +// b.registry is created), never on every Reset() -- store.Register panics on a +// duplicate name, so runtime resets go through registry.ResetAll() instead +// (see InMemoryBackend.Reset in backend.go). +// +// The following resource fields are deliberately left as plain maps (not +// registered here) because they are not a map[string]*T shape store.Table can +// replace: +// - arpVersionCountByPolicy, flowVersionCounters, promptVersionCounters, +// agentVersionCounters: map[string]int per-parent version counters, not +// resource values. +// - guardrailsByName, guardrailsByARN, pmtsByName, arpByName, +// customModelsByName, customModelDeployByName, evaluationJobsByName, +// customizationJobsByName, inferenceProfilesByName, +// marketplaceEndpointsByName, promptRoutersByName, agentsByName, +// kbByName, flowsByName, promptsByName: map[string]string secondary +// name/ARN -> ID lookup indexes, not primary resource storage. +// - agentTags: map[string]map[string]string tag map keyed externally by +// resource ARN then tag key; the value (string) carries no identity +// field of its own. +// - agentMemory, arpAnnotations: map[string][]any; the value is a raw +// slice with no identity field of its own to derive a store.Table key +// function from. +// - loggingConfig: a single *ModelInvocationLoggingConfiguration, not a map. +// - foundationModels: a []*FoundationModelSummary slice (seeded once, not +// keyed), not a map. +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call to the concrete field and value type. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.guardrails = store.Register(b.registry, "guardrails", store.New(guardrailsKeyFn)) + }, + func(b *InMemoryBackend) { + b.guardrailVersions = store.Register(b.registry, "guardrailVersions", store.New(guardrailVersionsKeyFn)) + }, + func(b *InMemoryBackend) { + b.provisionedModelThroughputs = store.Register( + b.registry, + "provisionedModelThroughputs", + store.New(provisionedModelThroughputsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.evaluationJobs = store.Register(b.registry, "evaluationJobs", store.New(evaluationJobsKeyFn)) + }, + func(b *InMemoryBackend) { + b.automatedReasoningPolicies = store.Register( + b.registry, + "automatedReasoningPolicies", + store.New(automatedReasoningPoliciesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.arpBuildWorkflows = store.Register(b.registry, "arpBuildWorkflows", store.New(arpBuildWorkflowsKeyFn)) + }, + func(b *InMemoryBackend) { + b.arpTestCases = store.Register(b.registry, "arpTestCases", store.New(arpTestCasesKeyFn)) + }, + func(b *InMemoryBackend) { + b.arpVersions = store.Register(b.registry, "arpVersions", store.New(arpVersionsKeyFn)) + }, + func(b *InMemoryBackend) { + b.customModels = store.Register(b.registry, "customModels", store.New(customModelsKeyFn)) + }, + func(b *InMemoryBackend) { + b.customModelDeployments = store.Register( + b.registry, + "customModelDeployments", + store.New(customModelDeploymentsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.foundationModelAgreements = store.Register( + b.registry, + "foundationModelAgreements", + store.New(foundationModelAgreementsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.modelCustomizationJobs = store.Register( + b.registry, + "modelCustomizationJobs", + store.New(modelCustomizationJobsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.modelCopyJobs = store.Register(b.registry, "modelCopyJobs", store.New(modelCopyJobsKeyFn)) + }, + func(b *InMemoryBackend) { + b.modelImportJobs = store.Register(b.registry, "modelImportJobs", store.New(modelImportJobsKeyFn)) + }, + func(b *InMemoryBackend) { + b.inferenceProfiles = store.Register(b.registry, "inferenceProfiles", store.New(inferenceProfilesKeyFn)) + }, + func(b *InMemoryBackend) { + b.marketplaceEndpoints = store.Register( + b.registry, + "marketplaceEndpoints", + store.New(marketplaceEndpointsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.modelInvocationJobs = store.Register(b.registry, "modelInvocationJobs", store.New(modelInvocationJobsKeyFn)) + }, + func(b *InMemoryBackend) { + b.promptRouters = store.Register(b.registry, "promptRouters", store.New(promptRoutersKeyFn)) + }, + func(b *InMemoryBackend) { + b.enforcedGuardrailConfigs = store.Register( + b.registry, + "enforcedGuardrailConfigs", + store.New(enforcedGuardrailConfigsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.agents = store.Register(b.registry, "agents", store.New(agentsKeyFn)) + }, + func(b *InMemoryBackend) { + b.agentActionGroups = store.Register(b.registry, "agentActionGroups", store.New(agentActionGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.agentAliases = store.Register(b.registry, "agentAliases", store.New(agentAliasesKeyFn)) + }, + func(b *InMemoryBackend) { + b.agentKBAssociations = store.Register( + b.registry, + "agentKBAssociations", + store.New(agentKBAssociationsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.knowledgeBases = store.Register(b.registry, "knowledgeBases", store.New(knowledgeBasesKeyFn)) + }, + func(b *InMemoryBackend) { + b.dataSources = store.Register(b.registry, "dataSources", store.New(dataSourcesKeyFn)) + }, + func(b *InMemoryBackend) { + b.ingestionJobs = store.Register(b.registry, "ingestionJobs", store.New(ingestionJobsKeyFn)) + }, + func(b *InMemoryBackend) { + b.flows = store.Register(b.registry, "flows", store.New(flowsKeyFn)) + }, + func(b *InMemoryBackend) { + b.flowAliases = store.Register(b.registry, "flowAliases", store.New(flowAliasesKeyFn)) + }, + func(b *InMemoryBackend) { + b.prompts = store.Register(b.registry, "prompts", store.New(promptsKeyFn)) + }, + func(b *InMemoryBackend) { + b.kbDocuments = store.Register(b.registry, "kbDocuments", store.New(kbDocumentsKeyFn)) + }, +} diff --git a/services/bedrock/store_setup_test.go b/services/bedrock/store_setup_test.go new file mode 100644 index 000000000..76f8a7460 --- /dev/null +++ b/services/bedrock/store_setup_test.go @@ -0,0 +1,140 @@ +package bedrock_test + +import ( + "testing" + + "github.com/blackbirdworks/gopherstack/services/bedrock" +) + +// TestInMemoryBackend_RegistryRoundTrip exercises a full-state +// Snapshot->Restore round trip of the Phase 3.3 pkgs/store conversion. +// bedrock has no persistence.go of its own (it was never wired into +// pkgs/persistence's Manager), so there is no production Snapshot/Restore to +// call through; instead this drives the same store.Registry machinery a real +// persistence layer would via the SnapshotTablesForTest/RestoreTablesForTest/ +// ResetTablesForTest test bridges in export_test.go. It seeds one resource +// per representative table shape -- a simple flat table (Guardrail), an +// ARN-keyed table (CustomModel), an agent-domain table (Agent), and the +// per-parent lazily-registered nested shape (Flow + FlowVersion) -- then +// verifies every resource is byte-for-byte recoverable after the tables are +// cleared and restored. +func TestInMemoryBackend_RegistryRoundTrip(t *testing.T) { + t.Parallel() + + type seeded struct { + guardrailID string + modelARN string + agentID string + flowID string + flowVersion string + } + + tests := []struct { + name string + }{ + {name: "populated backend survives snapshot, reset, restore"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := bedrock.NewInMemoryBackend("123456789012", "us-east-1") + + g, err := b.CreateGuardrail("test-guardrail", "desc", "blocked-in", "blocked-out", nil) + if err != nil { + t.Fatalf("CreateGuardrail: %v", err) + } + + cm, err := b.CreateCustomModel("test-model", nil) + if err != nil { + t.Fatalf("CreateCustomModel: %v", err) + } + + ag, err := b.CreateAgent("test-agent", "anthropic.claude-v2", "be helpful", "role-arn", nil) + if err != nil { + t.Fatalf("CreateAgent: %v", err) + } + + fl, err := b.CreateFlow("test-flow", "desc", nil) + if err != nil { + t.Fatalf("CreateFlow: %v", err) + } + + fv, err := b.CreateFlowVersion(fl.FlowID) + if err != nil { + t.Fatalf("CreateFlowVersion: %v", err) + } + + seed := seeded{ + guardrailID: g.GuardrailID, + modelARN: cm.ModelArn, + agentID: ag.AgentID, + flowID: fl.FlowID, + flowVersion: fv.Version, + } + + snap, err := b.SnapshotTablesForTest() + if err != nil { + t.Fatalf("SnapshotTablesForTest: %v", err) + } + + // Clear every registered table in place (registrations survive, + // exactly as InMemoryBackend.Reset leaves them for reuse). + b.ResetTablesForTest() + + if _, getErr := b.GetGuardrail(seed.guardrailID); getErr == nil { + t.Fatalf("expected guardrail to be absent after ResetTablesForTest") + } + + if restoreErr := b.RestoreTablesForTest(snap); restoreErr != nil { + t.Fatalf("RestoreTablesForTest: %v", restoreErr) + } + + gotG, err := b.GetGuardrail(seed.guardrailID) + if err != nil { + t.Fatalf("GetGuardrail after restore: %v", err) + } + + if gotG.GuardrailID != seed.guardrailID || gotG.Name != "test-guardrail" { + t.Errorf("guardrail mismatch after restore: %+v", gotG) + } + + gotCM, err := b.GetCustomModel(seed.modelARN) + if err != nil { + t.Fatalf("GetCustomModel after restore: %v", err) + } + + if gotCM.ModelArn != seed.modelARN || gotCM.ModelName != "test-model" { + t.Errorf("custom model mismatch after restore: %+v", gotCM) + } + + gotAgent, err := b.GetAgent(seed.agentID) + if err != nil { + t.Fatalf("GetAgent after restore: %v", err) + } + + if gotAgent.AgentID != seed.agentID || gotAgent.AgentName != "test-agent" { + t.Errorf("agent mismatch after restore: %+v", gotAgent) + } + + gotFlow, err := b.GetFlow(seed.flowID) + if err != nil { + t.Fatalf("GetFlow after restore: %v", err) + } + + if gotFlow.FlowID != seed.flowID || gotFlow.Name != "test-flow" { + t.Errorf("flow mismatch after restore: %+v", gotFlow) + } + + gotFV, err := b.GetFlowVersion(seed.flowID, seed.flowVersion) + if err != nil { + t.Fatalf("GetFlowVersion after restore: %v", err) + } + + if gotFV.FlowID != seed.flowID || gotFV.Version != seed.flowVersion { + t.Errorf("flow version mismatch after restore: %+v", gotFV) + } + }) + } +} diff --git a/services/cloudformation/PARITY.md b/services/cloudformation/PARITY.md new file mode 100644 index 000000000..1c9954088 --- /dev/null +++ b/services/cloudformation/PARITY.md @@ -0,0 +1,122 @@ +--- +service: cloudformation +sdk_module: aws-sdk-go-v2/service/cloudformation@v1.71.7 +last_audit_commit: 6548cf87 +last_audit_date: 2026-07-05 +overall: A # genuine fixes found and applied in the audited families; full + # ~42k-LOC surface not re-proven op-by-op this pass (see deferred) +ops: + CreateStack: {wire: ok, errors: ok, state: ok, persist: ok, note: "CAPABILITY_AUTO_EXPAND no longer wrongly satisfies the IAM-resource capability check (backend_parity.go requireIAMCapability)"} + UpdateStack: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: missing UPDATE_FAILED stack event on template parse failure; added pre-flight export-in-use block (validateExportsStillInUse)"} + DeleteStack: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: now idempotent (no-op, not ErrStackNotFound) per AWS's unmodeled DeleteStack error surface; added export-in-use block (stackExportsInUse)"} + DescribeStacks: {wire: ok, errors: ok, state: ok, persist: ok} + ListStacks: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeStackEvents: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: was returning the full unpaginated event history every call, ignoring NextToken entirely; now uses pkgs/page like the other List* ops"} + GetTemplate: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeStackResource: {wire: ok, errors: ok, state: ok, persist: ok} + ListStackResources: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeStackResources: {wire: ok, errors: ok, state: ok, persist: ok} + ListExports: {wire: ok, errors: ok, state: ok, persist: ok} + ListImports: {wire: ok, errors: ok, state: ok, persist: ok} + CreateChangeSet: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeChangeSet: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed error code ChangeSetNotFoundException -> ChangeSetNotFound (SDK deserializer matches the un-suffixed code; see errors.go ChangeSetNotFoundException.ErrorCode())"} + ExecuteChangeSet: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: no longer executes a FAILED/UNAVAILABLE change set (added InvalidChangeSetStatus gate); on success now clears every other change set for the stack, matching documented AWS behaviour, not just the executed one; fixed ChangeSetNotFound code"} + DeleteChangeSet: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed ChangeSetNotFound code"} + ListChangeSets: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeType: {wire: ok, errors: ok, state: ok, persist: ok} +families: + resource_provisioning: {status: ok, note: "topoSortResources (Kahn's algorithm, deterministic alphabetical tie-break) + provisionResources/rollbackCreateResources (reverse-order rollback, DeletionPolicy Retain/Snapshot honored) verified correct, no changes needed"} + update_reconciliation: {status: ok, note: "updateResources/rollbackUpdateResources snapshot-and-restore semantics verified correct; deleteStaleResources runs only after all creates/updates succeed, matching AWS ordering"} + exports_imports: {status: ok, note: "ADDED: delete-blocked-while-imported and update-blocked-while-imported (Export X cannot be deleted as it is in use by Y), the one concretely-named gap from the audit brief that was completely unimplemented before this pass"} + error_code_mapping: {status: ok, note: "verified against aws-sdk-go-v2 deserializers.go: StackNotFoundException is modeled for exactly one op (ImportStacksToStackSet) — every other stack-lookup op correctly falls back to generic ValidationError, matching real AWS's query-protocol behaviour; this was already correct, not changed"} + drift_detection: {status: ok, note: "DetectStackDrift / DescribeStackResourceDrifts / legacy SimulateDrift fallback reviewed, logic is internally consistent; NOT re-verified against AWS's real per-property drift diff algorithm this pass (see deferred)"} + nested_stacks: {status: ok, note: "CreateNestedStack/DeleteNestedStack correctly reuse createStackLocked/deleteStackLocked under the parent's already-held lock (no double-lock/deadlock); ParentID wiring reviewed, looks correct"} +gaps: + - "Top-level Transform is never parsed (Template struct has no Transform field), so CAPABILITY_AUTO_EXPAND is never required for macro-using templates even though Fn::Transform invocation itself works (bd: gopherstack-urm)" + - "changeset_diff.go requiresRecreation() models only a curated subset of AWS resource types' replacement-forcing properties (documented in-code as intentional partial coverage, not a regression) — expanding this table is future work, not tracked separately from gopherstack-e5h" +deferred: + - "StackSets: CreateStackSet/UpdateStackSet/DeleteStackSet/instances/operations/drift — not audited this pass (bd: gopherstack-e5h)" + - "Generated Templates family — not audited this pass (bd: gopherstack-e5h)" + - "Resource Scans family — not audited this pass (bd: gopherstack-e5h)" + - "Type registry/management (RegisterType/ActivateType/PublishType/TestType/etc.) — not audited this pass (bd: gopherstack-e5h)" + - "Stack Refactor family — not audited this pass (bd: gopherstack-e5h)" + - "YAML short-form intrinsics (!Ref/!GetAtt/etc.) wire coverage — not re-verified this pass (bd: gopherstack-e5h)" +leaks: {status: clean, note: "no new goroutines/janitors introduced. ExecuteChangeSet's synchronous UpdateStack/CreateStack call remains synchronous (no unbounded goroutine fan-out). evictDeletedStacks (cap 1000) and addEvent (cap 1000/stack) bounds were already in place and untouched. ctx is threaded through all new/changed code paths (stackExportsInUse takes no ctx since it only reads in-memory state, consistent with sibling ListImports)."} +--- + +## Notes + +Protocol: AWS query/XML (`Action=...` form POST, `` root, `ResponseMetadata>RequestId`). +Errors always serialize as HTTP 400 with `/` +(see `xmlError` in handler.go) — this project doesn't vary HTTP status by error code, matching how +CloudFormation's query protocol reports client errors. + +### Verified against the actual SDK model, not assumption + +- `aws-sdk-go-v2/service/cloudformation@v1.71.7`'s `types/errors.go` models a `StackNotFoundException` + with `ErrorCode() == "StackNotFoundException"`, but grepping `deserializers.go` shows it is only ever + switched on for **ImportStacksToStackSet**. Every other operation that can fail with "stack doesn't + exist" (CreateStack's implicit lookups, UpdateStack, DeleteStack, DescribeStacks, GetTemplate, + DescribeStackEvents, ...) has no modeled not-found exception at all, so the real API surfaces those as + a generic `ValidationError`. This codebase already did that correctly everywhere except it had + `ErrStackNotFound` as a hard *error* for DeleteStack, which leads to the next point. +- `DeleteStack`'s SDK-generated `awsAwsquery_deserializeOpErrorDeleteStack` has **no error cases at all** + besides `TokenAlreadyExistsException` — confirming DeleteStack is fire-and-forget/idempotent in real + AWS (deleting a stack that doesn't exist, or was already deleted, is a silent success). The backend + previously returned `ErrStackNotFound` for this case; fixed to a no-op. +- `ChangeSetNotFoundException.ErrorCode()` returns `"ChangeSetNotFound"` (no "Exception" suffix), and + `deserializers.go`'s `case strings.EqualFold("ChangeSetNotFound", errorCode)` confirms the wire code the + real client matches on. Three handlers (`DescribeChangeSet`/`ExecuteChangeSet`/`DeleteChangeSet`) were + emitting `"ChangeSetNotFoundException"` — a code the SDK client would never recognize, falling through + to a generic `smithy.GenericAPIError`. Fixed to the exact modeled code. +- `InvalidChangeSetStatusException.ErrorCode()` returns `"InvalidChangeSetStatus"`. `ExecuteChangeSet` had + no status gate at all before this pass — it would happily "execute" a change set whose + `ExecutionStatus` was `UNAVAILABLE` (e.g. one created with zero net changes, which AWS marks + FAILED/UNAVAILABLE at creation time). Fixed to reject with this exact code, matching AWS's documented + `ExecuteChangeSet operation failed. Only failed changesets should have an execution status of + UNAVAILABLE.`-class behaviour. +- The `ExecuteChangeSet` API doc comment in the SDK source states: "When you execute a change set, + CloudFormation deletes all other change sets associated with the stack because they aren't valid for + the updated stack." The backend previously deleted only the just-executed change set. Fixed to clear + the whole per-stack change-set map on success. +- Export-in-use protection ("Export X cannot be deleted as it is in use by Y") was completely absent — + neither `DeleteStack` nor `UpdateStack` checked whether a stack's export was still referenced via + `Fn::ImportValue` by another active stack before removing it. This is one of the concretely-named + high-value gaps in the audit brief. Added `stackExportsInUse` (shared helper, reuses the same + `collectImportValues` machinery `ListImports` already uses) wired into both `deleteStackLocked` and a + new `validateExportsStillInUse` pre-flight check in `applyTemplateToStack`. The update-side check is a + *pre-update* approximation (computed from the pre-update physical-ID snapshot, mirroring where + `validateImportValues` already runs) rather than a full two-pass plan/apply — acceptable given + CloudFormation's own pre-flight validation model, but note if a future change makes an export's value + depend on a resource newly created *by the same update*, this check won't see that value change. +- `CAPABILITY_AUTO_EXPAND` was incorrectly accepted as satisfying the `AWS::IAM::*` capability + requirement in `requireIAMCapability`. Per AWS docs, `CAPABILITY_AUTO_EXPAND` only authorizes + macro/transform expansion (e.g. SAM) — it does not grant permission to create IAM resources declared + directly in the template. Fixed to only accept `CAPABILITY_IAM`/`CAPABILITY_NAMED_IAM`. +- `DescribeStackEvents` ignored `NextToken` and `addEvent`'s own 1000-event-per-stack cap entirely, + returning the full history in one response. Changed the `StorageBackend` interface method to + `DescribeStackEvents(nameOrID, nextToken string) (page.Page[StackEvent], error)`, reusing + `pkgs/page.New` exactly like `ListStacks`/`ListChangeSets`/`ListExports` already do, and now surfaces + `NextToken` in the XML response. This changed the interface signature — all backend/handler test call + sites in this package were updated to match (`p, err := b.DescribeStackEvents(name, token); events := + p.Data`). + +### Traps for the next auditor (looks-wrong-but-correct) + +- `createStackFromTemplate`/`applyTemplateToStack` deliberately leave a stack in `CREATE_FAILED` / + `UPDATE_FAILED` (not `ROLLBACK_COMPLETE` / `UPDATE_ROLLBACK_COMPLETE`) when `ParseTemplate` itself fails + (malformed JSON/YAML), while every other pre-flight validator (`ValidateParameters`, + `validateIntrinsics`, `validateImportValues`, and now `validateExportsStillInUse`) transitions all the + way through to `*_ROLLBACK_COMPLETE`. This asymmetry is intentional and mirrors two genuinely different + AWS failure classes; don't "fix" it into uniformity. The only real bug here (now fixed) was that the + `UPDATE_FAILED` branch was silently missing its stack-level `addEvent` call — `CREATE_FAILED`'s sibling + branch already had it. +- `computeChanges`/`CreateChangeSet` marking a same-template change set `Status: FAILED, + ExecutionStatus: "UNAVAILABLE"` is correct AWS behaviour (a change set with zero net changes), not a + bug — a stale unit test (`TestBackend_ExecuteChangeSet/existing_stack`) previously exercised this exact + case with `wantErr` unset only because `ExecuteChangeSet` had no status gate; it now uses + `modifiedTemplate` (a real diff) and a new sibling case + (`existing_stack_no_changes`) explicitly covers the FAILED/blocked path. +- `ExecuteChangeSet`'s two lock windows (state-check-and-flip, then unlocked `UpdateStack`/`CreateStack` + call, then re-lock to finalize) are pre-existing and intentional — `UpdateStack`/`CreateStack` take + `b.mu` themselves, so holding it across the call would deadlock. Not touched. diff --git a/services/cloudformation/backend.go b/services/cloudformation/backend.go index 3022b9008..4b63539ce 100644 --- a/services/cloudformation/backend.go +++ b/services/cloudformation/backend.go @@ -13,6 +13,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/collections" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/config" @@ -28,6 +29,8 @@ var ( ErrResourceNotFound = errors.New("resource not found in stack") ErrExportNotFound = errors.New("export with given name not found") ErrDuplicateExport = errors.New("export already exists and is owned by another stack") + ErrExportInUse = errors.New("export cannot be removed while it is in use by another stack") + ErrChangeSetNotExecutable = errors.New("change set is not in an executable status") ErrDriftDetectionNotFound = errors.New("drift detection not found") ErrStackSetNotFound = errors.New("stack set not found") ErrStackSetAlreadyExists = errors.New("stack set already exists") @@ -82,7 +85,7 @@ type StorageBackend interface { DeleteStack(ctx context.Context, nameOrID string) error DescribeStack(nameOrID string) (*Stack, error) ListStacks(statusFilter []string, nextToken string) (page.Page[StackSummary], error) - DescribeStackEvents(nameOrID string) ([]StackEvent, error) + DescribeStackEvents(nameOrID, nextToken string) (page.Page[StackEvent], error) DescribeStackResource(nameOrID, logicalID string) (*StackResource, error) ListStackResources(nameOrID, nextToken string) (page.Page[StackResourceSummary], error) DescribeStackResources(nameOrID string) ([]StackResource, error) @@ -203,25 +206,30 @@ type StorageBackend interface { // InMemoryBackend is a concurrency-safe in-memory CloudFormation backend. type InMemoryBackend struct { - stacks map[string]*Stack + // registry lets Reset/Snapshot/Restore collapse the tables below to one + // call each instead of hand-rolled per-map boilerplate. See store_setup.go + // for the registrations and for why the maps further down are NOT + // registered (nested or slice-valued, which store.Table cannot represent). + registry *store.Registry + stacks *store.Table[Stack] + exports *store.Table[Export] + driftDetections *store.Table[DriftDetectionStatus] + stackSets *store.Table[StackSet] + generatedTemplates *store.Table[GeneratedTemplate] + resourceScans *store.Table[ResourceScan] + typeRegistry *store.Table[RegisteredType] + typeRegistrations *store.Table[TypeRegistrationRecord] + publishers *store.Table[Publisher] + stackRefactors *store.Table[StackRefactor] + hookResults *store.Table[HookResult] stackIDIndex map[string]string // stackID (ARN) → stackName events map[string][]StackEvent resources map[string]map[string]*StackResource changeSets map[string]map[string]*ChangeSet - exports map[string]*Export - driftDetections map[string]*DriftDetectionStatus stackPolicies map[string]string - stackSets map[string]*StackSet - stackInstances map[string][]StackInstance // stackSetName → instances - stackSetOperations map[string]map[string]*StackSetOperation // stackSetName → operationID → op - generatedTemplates map[string]*GeneratedTemplate - resourceScans map[string]*ResourceScan - typeRegistry map[string]*RegisteredType // typeArn → type - typeRegistrations map[string]*TypeRegistrationRecord // token → record + stackInstances map[string][]StackInstance // stackSetName → instances + stackSetOperations map[string]map[string]*StackSetOperation // stackSetName → operationID → op typeConfigs map[string]string // typeName → config json - publishers map[string]*Publisher // publisherID → publisher - stackRefactors map[string]*StackRefactor // refactorID → refactor - hookResults map[string]*HookResult // token → result handlerProgress map[string]string // bearerToken → status signals map[string][]SignalRecord // stackName+logicalID → records stackSetOpResults map[string]map[string][]StackSetOperationResult // stackSetName → opID → results @@ -274,25 +282,15 @@ func NewInMemoryBackendWithConfig( } b := &InMemoryBackend{ - stacks: make(map[string]*Stack), + registry: store.NewRegistry(), stackIDIndex: make(map[string]string), events: make(map[string][]StackEvent), resources: make(map[string]map[string]*StackResource), changeSets: make(map[string]map[string]*ChangeSet), - exports: make(map[string]*Export), - driftDetections: make(map[string]*DriftDetectionStatus), stackPolicies: make(map[string]string), - stackSets: make(map[string]*StackSet), stackInstances: make(map[string][]StackInstance), stackSetOperations: make(map[string]map[string]*StackSetOperation), - generatedTemplates: make(map[string]*GeneratedTemplate), - resourceScans: make(map[string]*ResourceScan), - typeRegistry: make(map[string]*RegisteredType), - typeRegistrations: make(map[string]*TypeRegistrationRecord), typeConfigs: make(map[string]string), - publishers: make(map[string]*Publisher), - stackRefactors: make(map[string]*StackRefactor), - hookResults: make(map[string]*HookResult), handlerProgress: make(map[string]string), signals: make(map[string][]SignalRecord), stackSetOpResults: make(map[string]map[string][]StackSetOperationResult), @@ -308,6 +306,8 @@ func NewInMemoryBackendWithConfig( mu: lockmetrics.New("cloudformation"), } + registerAllTables(b) + // Wire the backend as the NestedStackCreator so nested stacks can be provisioned. if creator != nil { creator.WithNestedStackCreator(b) @@ -340,13 +340,23 @@ func (b *InMemoryBackend) DeleteNestedStack(ctx context.Context, stackID string) func (b *InMemoryBackend) deleteStackLocked(ctx context.Context, nameOrID string) error { stack, ok := b.resolveStack(nameOrID) if !ok { - return ErrStackNotFound + // DeleteStack is idempotent in real AWS: deleting a stack that never + // existed (or was already deleted) is a silent no-op, not an error — + // the DeleteStack operation has no modeled "stack not found" error. + return nil } if stack.EnableTerminationProtection { return fmt.Errorf("%w: %s", ErrTerminationProtectionEnabled, stack.StackName) } + if name, importer, inUse := b.stackExportsInUse(stack.StackID, nil); inUse { + return fmt.Errorf( + "%w: Export %s cannot be deleted as it is in use by %s", + ErrExportInUse, name, importer, + ) + } + stack.StackStatus = statusDeleteInProgress b.addEvent( stack.StackID, @@ -401,7 +411,7 @@ func (b *InMemoryBackend) deleteStackLocked(ctx context.Context, nameOrID string func (b *InMemoryBackend) evictDeletedStacks() { const maxDeletedStacks = 1000 deleted := make([]*Stack, 0) - for _, s := range b.stacks { + for _, s := range b.stacks.All() { if s.StackStatus == statusDeleteComplete { deleted = append(deleted, s) } @@ -421,7 +431,7 @@ func (b *InMemoryBackend) evictDeletedStacks() { return deleted[i].DeletionTime.Before(*deleted[j].DeletionTime) }) for _, s := range deleted[:len(deleted)-maxDeletedStacks] { - delete(b.stacks, s.StackName) + b.stacks.Delete(s.StackName) delete(b.stackIDIndex, s.StackID) } } @@ -431,12 +441,12 @@ func (b *InMemoryBackend) buildStackARN(stackName, stackID string) string { } func (b *InMemoryBackend) resolveStack(nameOrID string) (*Stack, bool) { - if s, ok := b.stacks[nameOrID]; ok { + if s, ok := b.stacks.Get(nameOrID); ok { return s, true } if name, ok := b.stackIDIndex[nameOrID]; ok { - if s, found := b.stacks[name]; found { + if s, found := b.stacks.Get(name); found { return s, true } } @@ -496,7 +506,7 @@ func (b *InMemoryBackend) createStackLocked( } } - if existing, ok := b.stacks[name]; ok { + if existing, ok := b.stacks.Get(name); ok { if existing.StackStatus != statusDeleteComplete { return nil, ErrStackAlreadyExists } @@ -525,7 +535,7 @@ func (b *InMemoryBackend) createStackLocked( ParentID: parentID, } - b.stacks[name] = stack + b.stacks.Put(stack) b.stackIDIndex[arn] = name b.events[arn] = nil b.resources[arn] = make(map[string]*StackResource) @@ -909,6 +919,10 @@ func (b *InMemoryBackend) applyTemplateToStack(ctx context.Context, stack *Stack if err != nil { stack.StackStatus = statusUpdateFailed stack.StackStatusReason = err.Error() + b.addEvent( + stack.StackID, stack.StackName, stack.StackName, stack.StackID, + cfnStackType, statusUpdateFailed, err.Error(), + ) return false } @@ -955,6 +969,15 @@ func (b *InMemoryBackend) applyTemplateToStack(ctx context.Context, stack *Stack return false } + // Validate that the update does not drop an export that another active + // stack still imports via Fn::ImportValue (computed against pre-update + // resource state, mirroring AWS's validate-before-apply semantics). + if expErr := b.validateExportsStillInUse(stack, tmpl, resolvedParams, physicalIDs); expErr != nil { + b.updateFailAndRollback(stack, expErr.Error()) + + return false + } + if !b.updateResources(ctx, stack, tmpl, resolvedParams, physicalIDs) { return false } @@ -1271,7 +1294,7 @@ func (b *InMemoryBackend) DeleteStack(ctx context.Context, nameOrID string) erro // using the reverse index for O(1) lookup instead of O(n) scan. func (b *InMemoryBackend) pruneDriftDetections(stackID string) { for _, detectionID := range b.driftByStackID[stackID] { - delete(b.driftDetections, detectionID) + b.driftDetections.Delete(detectionID) } delete(b.driftByStackID, stackID) } @@ -1304,8 +1327,8 @@ func (b *InMemoryBackend) ListStacks( filter[s] = true } - summaries := make([]StackSummary, 0, len(b.stacks)) - for _, stack := range b.stacks { + summaries := make([]StackSummary, 0, b.stacks.Len()) + for _, stack := range b.stacks.All() { if len(filter) > 0 && !filter[stack.StackStatus] { continue } @@ -1326,14 +1349,16 @@ func (b *InMemoryBackend) ListStacks( return page.New(summaries, nextToken, 0, cfnDefaultPageSize), nil } -// DescribeStackEvents returns events for a stack. -func (b *InMemoryBackend) DescribeStackEvents(nameOrID string) ([]StackEvent, error) { +// DescribeStackEvents returns paginated events for a stack, most recent first. +func (b *InMemoryBackend) DescribeStackEvents( + nameOrID, nextToken string, +) (page.Page[StackEvent], error) { b.mu.RLock("DescribeStackEvents") defer b.mu.RUnlock() stack, ok := b.resolveStack(nameOrID) if !ok { - return nil, ErrStackNotFound + return page.Page[StackEvent]{}, ErrStackNotFound } evts := b.events[stack.StackID] @@ -1343,7 +1368,7 @@ func (b *InMemoryBackend) DescribeStackEvents(nameOrID string) ([]StackEvent, er result[len(evts)-1-i] = e } - return result, nil + return page.New(result, nextToken, 0, cfnDefaultPageSize), nil } // CreateChangeSet creates a change set for a stack. @@ -1454,21 +1479,34 @@ func (b *InMemoryBackend) DescribeChangeSet(stackName, changeSetName string) (*C return cs, nil } -// ExecuteChangeSet applies a change set to a stack. +// ExecuteChangeSet applies a change set to a stack. Only a change set whose +// ExecutionStatus is AVAILABLE can be executed — e.g. a change set created with +// no actual changes is FAILED/UNAVAILABLE and AWS rejects execution of it with +// InvalidChangeSetStatus. On success, AWS deletes every other change set +// associated with the stack because none remain valid against the now-updated +// template; this backend clears the whole per-stack change-set map to match. func (b *InMemoryBackend) ExecuteChangeSet( ctx context.Context, stackName, changeSetName string, ) error { b.mu.Lock("ExecuteChangeSet") cs, ok := b.changeSets[stackName][changeSetName] - if ok { - cs.ExecutionStatus = "EXECUTE_IN_PROGRESS" - } - b.mu.Unlock() - if !ok { + b.mu.Unlock() + return ErrChangeSetNotFound } + if cs.ExecutionStatus != "AVAILABLE" { + status := cs.ExecutionStatus + b.mu.Unlock() + + return fmt.Errorf( + "%w: ChangeSet [%s] cannot be executed in its current status of %s", + ErrChangeSetNotExecutable, changeSetName, status, + ) + } + cs.ExecutionStatus = "EXECUTE_IN_PROGRESS" + b.mu.Unlock() var execErr error _, err := b.UpdateStack(ctx, stackName, cs.TemplateBody, cs.Parameters, StackOptions{}) @@ -1486,7 +1524,7 @@ func (b *InMemoryBackend) ExecuteChangeSet( cs2.ExecutionStatus = "EXECUTE_FAILED" } } else { - delete(b.changeSets[stackName], changeSetName) + b.changeSets[stackName] = make(map[string]*ChangeSet) } b.mu.Unlock() @@ -1556,12 +1594,7 @@ func (b *InMemoryBackend) ListAll() []*Stack { b.mu.RLock("ListAll") defer b.mu.RUnlock() - stacks := make([]*Stack, 0, len(b.stacks)) - for _, s := range b.stacks { - stacks = append(stacks, s) - } - - return stacks + return b.stacks.All() } // DescribeStackResource returns details for a single resource in a stack. @@ -1645,8 +1678,8 @@ func (b *InMemoryBackend) ListExports(nextToken string) (page.Page[Export], erro b.mu.RLock("ListExports") defer b.mu.RUnlock() - exports := make([]Export, 0, len(b.exports)) - for _, exp := range b.exports { + exports := make([]Export, 0, b.exports.Len()) + for _, exp := range b.exports.All() { exports = append(exports, *exp) } @@ -1660,13 +1693,13 @@ func (b *InMemoryBackend) ListImports(exportName, nextToken string) (page.Page[s b.mu.RLock("ListImports") defer b.mu.RUnlock() - if _, ok := b.exports[exportName]; !ok { + if !b.exports.Has(exportName) { return page.Page[string]{}, ErrExportNotFound } var stackNames []string - for _, stack := range b.stacks { + for _, stack := range b.stacks.All() { if stack.StackStatus == statusDeleteComplete { continue } @@ -1692,15 +1725,15 @@ func (b *InMemoryBackend) ListImports(exportName, nextToken string) (page.Page[s // It returns ErrDuplicateExport if an export name is already owned by a different stack. func (b *InMemoryBackend) registerExports(stackID string, exportMap map[string]string) error { for name, value := range exportMap { - if existing, ok := b.exports[name]; ok && existing.ExportingStackID != stackID { + if existing, ok := b.exports.Get(name); ok && existing.ExportingStackID != stackID { return fmt.Errorf("%w: %s", ErrDuplicateExport, name) } - b.exports[name] = &Export{ + b.exports.Put(&Export{ ExportingStackID: stackID, Name: name, Value: value, - } + }) } return nil @@ -1708,19 +1741,107 @@ func (b *InMemoryBackend) registerExports(stackID string, exportMap map[string]s // removeExports removes all exports owned by the given stack. func (b *InMemoryBackend) removeExports(stackID string) { - for name, exp := range b.exports { + b.exports.Range(func(exp *Export) bool { if exp.ExportingStackID == stackID { - delete(b.exports, name) + b.exports.Delete(exp.Name) + } + + return true + }) +} + +// stackExportsInUse reports whether any export currently owned by stackID would +// be dropped by prospectiveExports (the export set the caller is about to apply) +// while still being referenced via Fn::ImportValue by another active stack. AWS +// refuses to delete a stack — or update away one of its outputs — while an +// export it owns is still imported elsewhere ("Export X cannot be deleted as it +// is in use by Y"). Pass a nil prospectiveExports map to check full removal +// (DeleteStack, where no exports survive). +func (b *InMemoryBackend) stackExportsInUse( + stackID string, + prospectiveExports map[string]string, +) (string, string, bool) { + var owned []string + b.exports.Range(func(exp *Export) bool { + if exp.ExportingStackID != stackID { + return true + } + if _, stillExported := prospectiveExports[exp.Name]; stillExported { + return true // export survives the change — nothing to block } + owned = append(owned, exp.Name) + + return true + }) + if len(owned) == 0 { + return "", "", false } + sort.Strings(owned) + + for _, other := range b.stacks.Snapshot() { + if other.StackID == stackID || other.StackStatus == statusDeleteComplete { + continue + } + + var resolvedParams map[string]string + if tmpl, err := ParseTemplate(other.TemplateBody); err == nil { + resolvedParams = ResolveParameters(tmpl, other.Parameters) + } + + refs := collectImportValues(other.TemplateBody, resolvedParams) + for _, exportName := range owned { + if slices.Contains(refs, exportName) { + return exportName, other.StackName, true + } + } + } + + return "", "", false +} + +// validateExportsStillInUse is the UpdateStack pre-flight check: it computes the +// export set the new template would produce (using the pre-update physicalIDs +// snapshot) and fails if that would drop an export still imported by another +// stack. +func (b *InMemoryBackend) validateExportsStillInUse( + stack *Stack, + tmpl *Template, + resolvedParams, physicalIDs map[string]string, +) error { + resourceTypes := make(map[string]string, len(tmpl.Resources)) + for logicalID, res := range tmpl.Resources { + resourceTypes[logicalID] = res.Type + } + + previewCtx := resolveCtx{ + params: resolvedParams, + physicalIDs: physicalIDs, + resourceTypes: resourceTypes, + exports: b.buildExportsMap(), + conditions: evaluateConditions(tmpl.Conditions, resolvedParams, physicalIDs), + mappings: tmpl.Mappings, + accountID: b.accountID, + region: b.region, + stackName: stack.StackName, + } + + _, previewExports := resolveOutputsWithContext(tmpl, previewCtx) + + if name, importer, inUse := b.stackExportsInUse(stack.StackID, previewExports); inUse { + return fmt.Errorf("%w: Export %s cannot be deleted as it is in use by %s", ErrExportInUse, name, importer) + } + + return nil } // buildExportsMap builds a name→value map of all current exports (for Fn::ImportValue resolution). func (b *InMemoryBackend) buildExportsMap() map[string]string { - m := make(map[string]string, len(b.exports)) - for name, exp := range b.exports { - m[name] = exp.Value - } + m := make(map[string]string, b.exports.Len()) + b.exports.Range(func(exp *Export) bool { + m[exp.Name] = exp.Value + + return true + }) return m } diff --git a/services/cloudformation/backend_ext.go b/services/cloudformation/backend_ext.go index 08b33faf7..27c38976b 100644 --- a/services/cloudformation/backend_ext.go +++ b/services/cloudformation/backend_ext.go @@ -45,14 +45,14 @@ func (b *InMemoryBackend) DetectStackDrift(nameOrID string) (string, error) { b.resourceDriftStatus[stack.StackID] = resourceStatuses detectionID := uuid.New().String() - b.driftDetections[detectionID] = &DriftDetectionStatus{ + b.driftDetections.Put(&DriftDetectionStatus{ StackID: stack.StackID, StackDriftDetectionID: detectionID, StackDriftStatus: overallStatus, DetectionStatus: detectionComplete, DriftedStackResourceCount: driftedCount, Timestamp: time.Now(), - } + }) b.driftByStackID[stack.StackID] = append(b.driftByStackID[stack.StackID], detectionID) return detectionID, nil @@ -92,14 +92,14 @@ func (b *InMemoryBackend) DetectStackResourceDrift(nameOrID, logicalID string) ( } detectionID := uuid.New().String() - b.driftDetections[detectionID] = &DriftDetectionStatus{ + b.driftDetections.Put(&DriftDetectionStatus{ StackID: stack.StackID, StackDriftDetectionID: detectionID, StackDriftStatus: overallStatus, DetectionStatus: detectionComplete, DriftedStackResourceCount: driftedCount, Timestamp: time.Now(), - } + }) b.driftByStackID[stack.StackID] = append(b.driftByStackID[stack.StackID], detectionID) return detectionID, nil @@ -236,7 +236,7 @@ func (b *InMemoryBackend) DescribeStackDriftDetectionStatus(detectionID string) b.mu.RLock("DescribeStackDriftDetectionStatus") defer b.mu.RUnlock() - status, ok := b.driftDetections[detectionID] + status, ok := b.driftDetections.Get(detectionID) if !ok { return nil, ErrDriftDetectionNotFound } diff --git a/services/cloudformation/backend_ops.go b/services/cloudformation/backend_ops.go index b20fdc28f..bccd8bcaf 100644 --- a/services/cloudformation/backend_ops.go +++ b/services/cloudformation/backend_ops.go @@ -35,7 +35,7 @@ func (b *InMemoryBackend) CreateStackSet( ) (*StackSet, error) { b.mu.Lock("CreateStackSet") defer b.mu.Unlock() - if _, ok := b.stackSets[name]; ok { + if b.stackSets.Has(name) { return nil, ErrStackSetAlreadyExists } ss := &StackSet{ @@ -45,7 +45,7 @@ func (b *InMemoryBackend) CreateStackSet( TemplateBody: templateBody, Status: "ACTIVE", } - b.stackSets[name] = ss + b.stackSets.Put(ss) return ss, nil } @@ -55,7 +55,7 @@ func (b *InMemoryBackend) UpdateStackSet( ) (*StackSet, error) { b.mu.Lock("UpdateStackSet") defer b.mu.Unlock() - ss, ok := b.stackSets[name] + ss, ok := b.stackSets.Get(name) if !ok { return nil, ErrStackSetNotFound } @@ -73,13 +73,13 @@ func (b *InMemoryBackend) UpdateStackSet( func (b *InMemoryBackend) DeleteStackSet(name string) error { b.mu.Lock("DeleteStackSet") defer b.mu.Unlock() - if _, ok := b.stackSets[name]; !ok { + if !b.stackSets.Has(name) { return ErrStackSetNotFound } if len(b.stackInstances[name]) > 0 { return ErrStackSetNotEmpty } - delete(b.stackSets, name) + b.stackSets.Delete(name) delete(b.stackInstances, name) return nil @@ -88,7 +88,7 @@ func (b *InMemoryBackend) DeleteStackSet(name string) error { func (b *InMemoryBackend) DescribeStackSet(name string) (*StackSet, error) { b.mu.RLock("DescribeStackSet") defer b.mu.RUnlock() - ss, ok := b.stackSets[name] + ss, ok := b.stackSets.Get(name) if !ok { return nil, ErrStackSetNotFound } @@ -99,8 +99,8 @@ func (b *InMemoryBackend) DescribeStackSet(name string) (*StackSet, error) { func (b *InMemoryBackend) ListStackSets(nextToken string) (page.Page[StackSetSummary], error) { b.mu.RLock("ListStackSets") defer b.mu.RUnlock() - result := make([]StackSetSummary, 0, len(b.stackSets)) - for _, ss := range b.stackSets { + result := make([]StackSetSummary, 0, b.stackSets.Len()) + for _, ss := range b.stackSets.All() { result = append(result, StackSetSummary{ StackSetID: ss.StackSetID, StackSetName: ss.StackSetName, @@ -123,7 +123,7 @@ func (b *InMemoryBackend) CreateStackInstances( ) (string, error) { b.mu.Lock("CreateStackInstances") defer b.mu.Unlock() - ss, ok := b.stackSets[stackSetName] + ss, ok := b.stackSets.Get(stackSetName) if !ok { return "", ErrStackSetNotFound } @@ -204,7 +204,7 @@ func (b *InMemoryBackend) DeleteStackInstances( ) (string, error) { b.mu.Lock("DeleteStackInstances") defer b.mu.Unlock() - if _, ok := b.stackSets[stackSetName]; !ok { + if !b.stackSets.Has(stackSetName) { return "", ErrStackSetNotFound } instances := b.stackInstances[stackSetName] @@ -241,7 +241,7 @@ func (b *InMemoryBackend) UpdateStackInstances( ) (string, error) { b.mu.Lock("UpdateStackInstances") defer b.mu.Unlock() - if _, ok := b.stackSets[stackSetName]; !ok { + if !b.stackSets.Has(stackSetName) { return "", ErrStackSetNotFound } opID := b.recordStackSetOperation(stackSetName, "UPDATE_INSTANCES") @@ -281,7 +281,7 @@ func (b *InMemoryBackend) DescribeStackInstance( func (b *InMemoryBackend) DetectStackSetDrift(stackSetName string) (string, error) { b.mu.Lock("DetectStackSetDrift") defer b.mu.Unlock() - if _, ok := b.stackSets[stackSetName]; !ok { + if !b.stackSets.Has(stackSetName) { return "", ErrStackSetNotFound } opID := b.recordStackSetOperation(stackSetName, "DETECT_DRIFT") @@ -444,7 +444,7 @@ func (b *InMemoryBackend) ListStackSetAutoDeploymentTargets( ) ([]AutoDeploymentTarget, error) { b.mu.RLock("ListStackSetAutoDeploymentTargets") defer b.mu.RUnlock() - if _, ok := b.stackSets[stackSetName]; !ok { + if !b.stackSets.Has(stackSetName) { return nil, ErrStackSetNotFound } // SERVICE_MANAGED stack sets target OUs; for SELF_MANAGED emulation we have no OU hierarchy, @@ -475,7 +475,7 @@ func (b *InMemoryBackend) ListStackSetAutoDeploymentTargets( func (b *InMemoryBackend) ImportStacksToStackSet(stackSetName string, stackIDs []string) error { b.mu.Lock("ImportStacksToStackSet") defer b.mu.Unlock() - ss, ok := b.stackSets[stackSetName] + ss, ok := b.stackSets.Get(stackSetName) if !ok { return ErrStackSetNotFound } @@ -527,7 +527,7 @@ func (b *InMemoryBackend) ListStackInstanceResourceDrifts( ) ([]StackResourceDrift, error) { b.mu.RLock("ListStackInstanceResourceDrifts") defer b.mu.RUnlock() - if _, ok := b.stackSets[stackSetName]; !ok { + if !b.stackSets.Has(stackSetName) { return nil, ErrStackSetNotFound } // Find the matching stack instance's stack ID. @@ -576,7 +576,7 @@ func (b *InMemoryBackend) CreateGeneratedTemplate( Status: statusComplete, TemplateBody: templateBody, } - b.generatedTemplates[gt.GeneratedTemplateID] = gt + b.generatedTemplates.Put(gt) return gt, nil } @@ -642,7 +642,7 @@ func (b *InMemoryBackend) buildGeneratedTemplateBody(resourceIDs []string) strin resources := parseResourceIDs(resourceIDs) if len(resources) == 0 { // Build from existing stack resources as a convenience. - for _, stack := range b.stacks { + for _, stack := range b.stacks.All() { if stack.StackStatus == statusDeleteComplete { continue } @@ -664,7 +664,7 @@ func (b *InMemoryBackend) buildGeneratedTemplateBody(resourceIDs []string) strin func (b *InMemoryBackend) UpdateGeneratedTemplate(id, name string) error { b.mu.Lock("UpdateGeneratedTemplate") defer b.mu.Unlock() - gt, ok := b.generatedTemplates[id] + gt, ok := b.generatedTemplates.Get(id) if !ok { return ErrGeneratedTemplateNotFound } @@ -678,10 +678,10 @@ func (b *InMemoryBackend) UpdateGeneratedTemplate(id, name string) error { func (b *InMemoryBackend) DeleteGeneratedTemplate(id string) error { b.mu.Lock("DeleteGeneratedTemplate") defer b.mu.Unlock() - if _, ok := b.generatedTemplates[id]; !ok { + if !b.generatedTemplates.Has(id) { return ErrGeneratedTemplateNotFound } - delete(b.generatedTemplates, id) + b.generatedTemplates.Delete(id) return nil } @@ -689,7 +689,7 @@ func (b *InMemoryBackend) DeleteGeneratedTemplate(id string) error { func (b *InMemoryBackend) DescribeGeneratedTemplate(id string) (*GeneratedTemplate, error) { b.mu.RLock("DescribeGeneratedTemplate") defer b.mu.RUnlock() - gt, ok := b.generatedTemplates[id] + gt, ok := b.generatedTemplates.Get(id) if !ok { return nil, ErrGeneratedTemplateNotFound } @@ -700,7 +700,7 @@ func (b *InMemoryBackend) DescribeGeneratedTemplate(id string) (*GeneratedTempla func (b *InMemoryBackend) GetGeneratedTemplate(id string) (string, error) { b.mu.RLock("GetGeneratedTemplate") defer b.mu.RUnlock() - gt, ok := b.generatedTemplates[id] + gt, ok := b.generatedTemplates.Get(id) if !ok { return "", ErrGeneratedTemplateNotFound } @@ -713,8 +713,8 @@ func (b *InMemoryBackend) ListGeneratedTemplates( ) (page.Page[GeneratedTemplate], error) { b.mu.RLock("ListGeneratedTemplates") defer b.mu.RUnlock() - result := make([]GeneratedTemplate, 0, len(b.generatedTemplates)) - for _, gt := range b.generatedTemplates { + result := make([]GeneratedTemplate, 0, b.generatedTemplates.Len()) + for _, gt := range b.generatedTemplates.All() { result = append(result, *gt) } sort.Slice(result, func(i, j int) bool { @@ -730,14 +730,14 @@ func (b *InMemoryBackend) StartResourceScan() (string, error) { b.mu.Lock("StartResourceScan") defer b.mu.Unlock() scanID := uuid.New().String() - b.resourceScans[scanID] = &ResourceScan{ + b.resourceScans.Put(&ResourceScan{ ResourceScanID: scanID, Status: statusComplete, PercentageCompleted: resourceScanCompletePercent, - } + }) // Populate scan items from existing active stacks. items := make([]ScannedResource, 0) - for _, stack := range b.stacks { + for _, stack := range b.stacks.All() { if stack.StackStatus == statusDeleteComplete { continue } @@ -769,7 +769,7 @@ func (b *InMemoryBackend) StartResourceScan() (string, error) { func (b *InMemoryBackend) DescribeResourceScan(scanID string) (*ResourceScan, error) { b.mu.RLock("DescribeResourceScan") defer b.mu.RUnlock() - rs, ok := b.resourceScans[scanID] + rs, ok := b.resourceScans.Get(scanID) if !ok { return nil, ErrResourceScanNotFound } @@ -780,8 +780,8 @@ func (b *InMemoryBackend) DescribeResourceScan(scanID string) (*ResourceScan, er func (b *InMemoryBackend) ListResourceScans(nextToken string) (page.Page[ResourceScan], error) { b.mu.RLock("ListResourceScans") defer b.mu.RUnlock() - result := make([]ResourceScan, 0, len(b.resourceScans)) - for _, rs := range b.resourceScans { + result := make([]ResourceScan, 0, b.resourceScans.Len()) + for _, rs := range b.resourceScans.All() { result = append(result, *rs) } @@ -791,7 +791,7 @@ func (b *InMemoryBackend) ListResourceScans(nextToken string) (page.Page[Resourc func (b *InMemoryBackend) ListResourceScanResources(scanID, _ string) ([]ScannedResource, error) { b.mu.RLock("ListResourceScanResources") defer b.mu.RUnlock() - if _, ok := b.resourceScans[scanID]; !ok { + if !b.resourceScans.Has(scanID) { return nil, ErrResourceScanNotFound } items := b.resourceScanItems[scanID] @@ -807,7 +807,7 @@ func (b *InMemoryBackend) ListResourceScanRelatedResources( ) ([]string, error) { b.mu.RLock("ListResourceScanRelatedResources") defer b.mu.RUnlock() - if _, ok := b.resourceScans[scanID]; !ok { + if !b.resourceScans.Has(scanID) { return nil, ErrResourceScanNotFound } @@ -823,17 +823,17 @@ func (b *InMemoryBackend) ActivateType(typeName, typeArn string) error { if key == "" { key = "arn:aws:cloudformation:::type/resource/" + typeName } - if t, ok := b.typeRegistry[key]; ok { + if t, ok := b.typeRegistry.Get(key); ok { t.IsActivated = true } else { - b.typeRegistry[key] = &RegisteredType{ + b.typeRegistry.Put(&RegisteredType{ TypeArn: key, TypeName: typeName, Type: typeKindResource, VersionID: "00000001", Status: statusComplete, IsActivated: true, - } + }) } return nil @@ -846,7 +846,7 @@ func (b *InMemoryBackend) DeactivateType(typeName, typeArn string) error { if key == "" { key = "arn:aws:cloudformation:::type/resource/" + typeName } - t, ok := b.typeRegistry[key] + t, ok := b.typeRegistry.Get(key) if !ok || !t.IsActivated { return fmt.Errorf("%w: %s", ErrTypeNotFound, key) } @@ -874,25 +874,25 @@ func (b *InMemoryBackend) RegisterType(typeName, _ string) (string, error) { for i := range b.typeVersions[typeArn][:len(b.typeVersions[typeArn])-1] { b.typeVersions[typeArn][i].IsDefault = false } - if t, ok := b.typeRegistry[typeArn]; ok { + if t, ok := b.typeRegistry.Get(typeArn); ok { t.VersionID = versionID t.DefaultVersion = versionID } else { - b.typeRegistry[typeArn] = &RegisteredType{ + b.typeRegistry.Put(&RegisteredType{ TypeArn: typeArn, TypeName: typeName, Type: "RESOURCE", VersionID: versionID, DefaultVersion: versionID, Status: statusComplete, - } + }) } - b.typeRegistrations[token] = &TypeRegistrationRecord{ + b.typeRegistrations.Put(&TypeRegistrationRecord{ Token: token, TypeName: typeName, TypeArn: typeArn, Status: statusComplete, - } + }) return token, nil } @@ -900,7 +900,7 @@ func (b *InMemoryBackend) RegisterType(typeName, _ string) (string, error) { func (b *InMemoryBackend) DeregisterType(typeArn string) error { b.mu.Lock("DeregisterType") defer b.mu.Unlock() - t, ok := b.typeRegistry[typeArn] + t, ok := b.typeRegistry.Get(typeArn) if !ok { return fmt.Errorf("%w: %s", ErrTypeNotFound, typeArn) } @@ -913,7 +913,7 @@ func (b *InMemoryBackend) PublishType(typeName string) error { b.mu.Lock("PublishType") defer b.mu.Unlock() typeArn := "arn:aws:cloudformation:::type/resource/" + typeName - t, ok := b.typeRegistry[typeArn] + t, ok := b.typeRegistry.Get(typeArn) if !ok { return fmt.Errorf("%w: %s", ErrTypeNotFound, typeArn) } @@ -925,7 +925,7 @@ func (b *InMemoryBackend) PublishType(typeName string) error { func (b *InMemoryBackend) SetTypeDefaultVersion(typeArn, version string) error { b.mu.Lock("SetTypeDefaultVersion") defer b.mu.Unlock() - if t, ok := b.typeRegistry[typeArn]; ok { + if t, ok := b.typeRegistry.Get(typeArn); ok { t.DefaultVersion = version t.VersionID = version } @@ -956,7 +956,7 @@ func (b *InMemoryBackend) BatchDescribeTypeConfigurations( if cfg == "" { // Also check by looking up the registry entry (typeName may be a key in typeConfigs). typeArn := "arn:aws:cloudformation:::type/resource/" + id - if t, ok := b.typeRegistry[typeArn]; ok { + if t, ok := b.typeRegistry.Get(typeArn); ok { cfg = t.Configuration } } @@ -973,8 +973,8 @@ func (b *InMemoryBackend) BatchDescribeTypeConfigurations( func (b *InMemoryBackend) ListTypes(_ string) ([]TypeSummary, error) { b.mu.RLock("ListTypes") defer b.mu.RUnlock() - result := make([]TypeSummary, 0, len(b.typeRegistry)) - for _, t := range b.typeRegistry { + result := make([]TypeSummary, 0, b.typeRegistry.Len()) + for _, t := range b.typeRegistry.All() { if t.Status == typeStatusDeprecated { continue } @@ -1010,7 +1010,7 @@ func (b *InMemoryBackend) ListTypeVersions(typeName, _ string) ([]string, error) return ids, nil } // Fallback: if no version records but type exists, return its current version. - if t, ok := b.typeRegistry[typeArn]; ok { + if t, ok := b.typeRegistry.Get(typeArn); ok { return []string{t.VersionID}, nil } @@ -1021,9 +1021,9 @@ func (b *InMemoryBackend) ListTypeRegistrations(typeName, _ string) ([]string, e b.mu.RLock("ListTypeRegistrations") defer b.mu.RUnlock() var tokens []string - for token, rec := range b.typeRegistrations { + for _, rec := range b.typeRegistrations.All() { if typeName == "" || rec.TypeName == typeName { - tokens = append(tokens, token) + tokens = append(tokens, rec.Token) } } @@ -1033,7 +1033,7 @@ func (b *InMemoryBackend) ListTypeRegistrations(typeName, _ string) ([]string, e func (b *InMemoryBackend) DescribeTypeRegistration(registrationToken string) (string, error) { b.mu.RLock("DescribeTypeRegistration") defer b.mu.RUnlock() - rec, ok := b.typeRegistrations[registrationToken] + rec, ok := b.typeRegistrations.Get(registrationToken) if !ok { return "", fmt.Errorf("%w: %s", ErrRegistrationTokenNotFound, registrationToken) } @@ -1049,12 +1049,12 @@ func (b *InMemoryBackend) TestType(typeName, typeArn string) (string, error) { if key == "" { key = "arn:aws:cloudformation:::type/resource/" + typeName } - b.typeRegistrations[token] = &TypeRegistrationRecord{ + b.typeRegistrations.Put(&TypeRegistrationRecord{ Token: token, TypeName: typeName, TypeArn: key, Status: statusComplete, - } + }) return token, nil } @@ -1063,11 +1063,11 @@ func (b *InMemoryBackend) RegisterPublisher(connectionArn string) (string, error b.mu.Lock("RegisterPublisher") defer b.mu.Unlock() publisherID := uuid.New().String() - b.publishers[publisherID] = &Publisher{ + b.publishers.Put(&Publisher{ PublisherID: publisherID, ConnectionArn: connectionArn, Status: "VERIFIED", - } + }) return publisherID, nil } @@ -1075,7 +1075,7 @@ func (b *InMemoryBackend) RegisterPublisher(connectionArn string) (string, error func (b *InMemoryBackend) DescribePublisher(publisherID string) (string, error) { b.mu.RLock("DescribePublisher") defer b.mu.RUnlock() - p, ok := b.publishers[publisherID] + p, ok := b.publishers.Get(publisherID) if !ok { return "", fmt.Errorf("%w: %s", ErrPublisherNotFound, publisherID) } @@ -1092,12 +1092,12 @@ func (b *InMemoryBackend) CreateStackRefactor( b.mu.Lock("CreateStackRefactor") defer b.mu.Unlock() refactorID := uuid.New().String() - b.stackRefactors[refactorID] = &StackRefactor{ + b.stackRefactors.Put(&StackRefactor{ RefactorID: refactorID, Description: description, Status: "CREATE_COMPLETE", StackDefinitions: stackDefinitions, - } + }) return refactorID, nil } @@ -1105,7 +1105,7 @@ func (b *InMemoryBackend) CreateStackRefactor( func (b *InMemoryBackend) DescribeStackRefactor(stackRefactorID string) (string, error) { b.mu.RLock("DescribeStackRefactor") defer b.mu.RUnlock() - r, ok := b.stackRefactors[stackRefactorID] + r, ok := b.stackRefactors.Get(stackRefactorID) if !ok { return "", nil } @@ -1116,7 +1116,7 @@ func (b *InMemoryBackend) DescribeStackRefactor(stackRefactorID string) (string, func (b *InMemoryBackend) ExecuteStackRefactor(stackRefactorID string) error { b.mu.Lock("ExecuteStackRefactor") defer b.mu.Unlock() - r, ok := b.stackRefactors[stackRefactorID] + r, ok := b.stackRefactors.Get(stackRefactorID) if !ok { return nil } @@ -1128,8 +1128,8 @@ func (b *InMemoryBackend) ExecuteStackRefactor(stackRefactorID string) error { func (b *InMemoryBackend) ListStackRefactors(_ string) ([]StackRefactorSummary, error) { b.mu.RLock("ListStackRefactors") defer b.mu.RUnlock() - summaries := make([]StackRefactorSummary, 0, len(b.stackRefactors)) - for _, r := range b.stackRefactors { + summaries := make([]StackRefactorSummary, 0, b.stackRefactors.Len()) + for _, r := range b.stackRefactors.All() { summaries = append(summaries, StackRefactorSummary{ StackRefactorID: r.RefactorID, Status: r.Status, @@ -1145,7 +1145,7 @@ func (b *InMemoryBackend) ListStackRefactorActions( ) ([]StackRefactorAction, error) { b.mu.RLock("ListStackRefactorActions") defer b.mu.RUnlock() - r, ok := b.stackRefactors[stackRefactorID] + r, ok := b.stackRefactors.Get(stackRefactorID) if !ok { return []StackRefactorAction{}, nil } @@ -1228,7 +1228,7 @@ func (b *InMemoryBackend) RecordHandlerProgress(bearerToken, operationStatus str func (b *InMemoryBackend) GetHookResult(hookResultToken string) (string, error) { b.mu.RLock("GetHookResult") defer b.mu.RUnlock() - r, ok := b.hookResults[hookResultToken] + r, ok := b.hookResults.Get(hookResultToken) if !ok { return "SUCCEEDED", nil } @@ -1241,11 +1241,11 @@ func (b *InMemoryBackend) ListHookResults(hookResultToken, _ string) ([]HookResu defer b.mu.RUnlock() var results []HookResult if hookResultToken != "" { - if r, ok := b.hookResults[hookResultToken]; ok { + if r, ok := b.hookResults.Get(hookResultToken); ok { results = append(results, *r) } } else { - for _, r := range b.hookResults { + for _, r := range b.hookResults.All() { results = append(results, *r) } } diff --git a/services/cloudformation/backend_parity.go b/services/cloudformation/backend_parity.go index bdd4b3ef1..503ca5d06 100644 --- a/services/cloudformation/backend_parity.go +++ b/services/cloudformation/backend_parity.go @@ -37,14 +37,14 @@ func (b *InMemoryBackend) DescribeType(typeName, arn, versionID string) (*TypeDe var reg *RegisteredType switch { case arn != "": - r, ok := b.typeRegistry[arn] + r, ok := b.typeRegistry.Get(arn) if !ok { return nil, fmt.Errorf("%w: %s", ErrTypeNotFound, arn) } reg = r case typeName != "": key := "arn:aws:cloudformation:::type/resource/" + typeName - r, ok := b.typeRegistry[key] + r, ok := b.typeRegistry.Get(key) if !ok { return nil, fmt.Errorf("%w: %s", ErrTypeNotFound, typeName) } @@ -113,14 +113,14 @@ func (b *InMemoryBackend) SimulateDrift(stackName string) error { // Create a drift detection record that reflects drifted resources. detectionID := uuid.New().String() - b.driftDetections[detectionID] = &DriftDetectionStatus{ + b.driftDetections.Put(&DriftDetectionStatus{ StackID: stack.StackID, StackDriftDetectionID: detectionID, StackDriftStatus: driftStatusDrifted, DetectionStatus: "DETECTION_COMPLETE", DriftedStackResourceCount: len(resMap), Timestamp: time.Now(), - } + }) return nil } @@ -144,7 +144,7 @@ func (b *InMemoryBackend) DescribeStackResourceDrifts(nameOrID string) ([]StackR // no per-resource data exists, mark all as DRIFTED. legacyDrifted := false if len(perResourceStatuses) == 0 { - for _, det := range b.driftDetections { + for _, det := range b.driftDetections.All() { if det.StackID == stack.StackID && det.StackDriftStatus == driftStatusDrifted { legacyDrifted = true @@ -212,8 +212,11 @@ func requireIAMCapability(templateBody string, capabilities []string) error { if !hasIAM { return nil } + // Note: CAPABILITY_AUTO_EXPAND only authorizes macro/transform expansion + // (e.g. SAM); it does not grant permission to create IAM resources + // declared directly in the template, so it must NOT satisfy this check. for _, c := range capabilities { - if c == "CAPABILITY_IAM" || c == "CAPABILITY_NAMED_IAM" || c == "CAPABILITY_AUTO_EXPAND" { + if c == "CAPABILITY_IAM" || c == "CAPABILITY_NAMED_IAM" { return nil } } diff --git a/services/cloudformation/cfn_accuracy_test.go b/services/cloudformation/cfn_accuracy_test.go index 9b25f8100..472932ba9 100644 --- a/services/cloudformation/cfn_accuracy_test.go +++ b/services/cloudformation/cfn_accuracy_test.go @@ -651,8 +651,9 @@ func TestStackLifecycle_RollbackOnCreateFailure(t *testing.T) { assert.Equal(t, "ROLLBACK_COMPLETE", stack.StackStatus) // Events contain ROLLBACK_IN_PROGRESS. - events, err := b.DescribeStackEvents("fail-stack") + evtPage, err := b.DescribeStackEvents("fail-stack", "") require.NoError(t, err) + events := evtPage.Data statuses := make([]string, len(events)) for i, e := range events { statuses[i] = e.ResourceStatus @@ -961,8 +962,9 @@ func TestDescribeStackEvents_ReverseChrono(t *testing.T) { ) require.NoError(t, err) - events, err := b.DescribeStackEvents("events-stack") + evtPage, err := b.DescribeStackEvents("events-stack", "") require.NoError(t, err) + events := evtPage.Data assert.NotEmpty(t, events) // Most recent first. diff --git a/services/cloudformation/cfn_parity_test.go b/services/cloudformation/cfn_parity_test.go index 215b7dcba..6a7850784 100644 --- a/services/cloudformation/cfn_parity_test.go +++ b/services/cloudformation/cfn_parity_test.go @@ -366,10 +366,15 @@ func TestCreateStack_IAMCapabilityRequired(t *testing.T) { wantErr: false, }, { - name: "IAM template with CAPABILITY_AUTO_EXPAND succeeds", + // CAPABILITY_AUTO_EXPAND only authorizes macro/transform expansion + // (e.g. SAM); it does not grant permission to create IAM resources + // declared directly in the template, so it must NOT substitute for + // CAPABILITY_IAM / CAPABILITY_NAMED_IAM. + name: "IAM template with only CAPABILITY_AUTO_EXPAND still fails", template: iamTemplate, capabilities: []string{"CAPABILITY_AUTO_EXPAND"}, - wantErr: false, + wantErr: true, + errIs: cloudformation.ErrInsufficientCapabilities, }, { name: "non-IAM template without capabilities succeeds", diff --git a/services/cloudformation/cloudformation_ext_test.go b/services/cloudformation/cloudformation_ext_test.go index b3db1081d..7b62155ef 100644 --- a/services/cloudformation/cloudformation_ext_test.go +++ b/services/cloudformation/cloudformation_ext_test.go @@ -1136,6 +1136,22 @@ func TestBackend_UpdateStack_InvalidTemplate(t *testing.T) { updated, err := b.UpdateStack(t.Context(), "upd-stack", tt.updateBody, nil, cloudformation.StackOptions{}) require.NoError(t, err) assert.Equal(t, tt.wantStatus, updated.StackStatus) + + // A failed update must record a stack-level event carrying the + // failure status, exactly like the CreateStack failure paths do — + // callers polling DescribeStackEvents rely on this. + evtPage, evErr := b.DescribeStackEvents("upd-stack", "") + require.NoError(t, evErr) + events := evtPage.Data + foundFailureEvent := false + for _, ev := range events { + if ev.ResourceStatus == tt.wantStatus && ev.LogicalResourceID == "upd-stack" { + foundFailureEvent = true + + break + } + } + assert.True(t, foundFailureEvent, "expected a stack-level %s event", tt.wantStatus) }) } } diff --git a/services/cloudformation/cloudformation_test.go b/services/cloudformation/cloudformation_test.go index ac95c1692..d1926be8e 100644 --- a/services/cloudformation/cloudformation_test.go +++ b/services/cloudformation/cloudformation_test.go @@ -6,6 +6,7 @@ import ( "log/slog" "net/http" "net/http/httptest" + "net/url" "strings" "testing" @@ -563,9 +564,11 @@ func TestBackend_DeleteStack(t *testing.T) { wantStatus: "DELETE_COMPLETE", }, { - name: "not_found", + // DeleteStack is idempotent in real AWS: it has no modeled "stack + // not found" error, so deleting a stack that never existed is a + // silent no-op success, not an error. + name: "not_found_is_noop", stackName: "missing", - wantErr: cloudformation.ErrStackNotFound, }, } @@ -588,6 +591,10 @@ func TestBackend_DeleteStack(t *testing.T) { require.NoError(t, err) + if tt.wantStatus == "" { + return + } + stack, err := b.DescribeStack(tt.stackName) require.NoError(t, err) assert.Equal(t, tt.wantStatus, stack.StackStatus) @@ -813,7 +820,7 @@ func TestBackend_DescribeStackEvents(t *testing.T) { tt.setup(t, b) } - events, err := b.DescribeStackEvents(tt.stackName) + evtPage, err := b.DescribeStackEvents(tt.stackName, "") if tt.wantErr != nil { require.ErrorIs(t, err, tt.wantErr) @@ -822,11 +829,59 @@ func TestBackend_DescribeStackEvents(t *testing.T) { } require.NoError(t, err) - assert.NotEmpty(t, events) + assert.NotEmpty(t, evtPage.Data) }) } } +// TestBackend_DescribeStackEvents_Pagination verifies DescribeStackEvents +// honors NextToken instead of always returning the full event history — +// previously it ignored pagination entirely and returned every event in one +// response, unlike real AWS. +func TestBackend_DescribeStackEvents_Pagination(t *testing.T) { + t.Parallel() + + b := newBackend() + _, err := b.CreateStack(t.Context(), "evt-page-stack", simpleTemplate, nil, cloudformation.StackOptions{}) + require.NoError(t, err) + + // Each successful UpdateStack on an unchanged plain resource emits 3 + // events (stack UPDATE_IN_PROGRESS, resource UPDATE_COMPLETE, stack + // UPDATE_COMPLETE), so 40 updates comfortably exceeds one default page. + for range 40 { + _, uerr := b.UpdateStack(t.Context(), "evt-page-stack", simpleTemplate, nil, cloudformation.StackOptions{}) + require.NoError(t, uerr) + } + + firstPage, err := b.DescribeStackEvents("evt-page-stack", "") + require.NoError(t, err) + require.NotEmpty(t, firstPage.Next, "expected more than one page of events") + assert.Len(t, firstPage.Data, 100, "default page size") + + seen := make(map[string]bool, len(firstPage.Data)) + for _, e := range firstPage.Data { + seen[e.EventID] = true + } + + token := firstPage.Next + var totalAfterFirst int + + for token != "" { + next, nerr := b.DescribeStackEvents("evt-page-stack", token) + require.NoError(t, nerr) + + for _, e := range next.Data { + assert.False(t, seen[e.EventID], "event %s returned on more than one page", e.EventID) + seen[e.EventID] = true + } + + totalAfterFirst += len(next.Data) + token = next.Next + } + + assert.Positive(t, totalAfterFirst, "expected additional events beyond the first page") +} + // ---- Backend: GetTemplate --------------------------------------------------- func TestBackend_GetTemplate(t *testing.T) { @@ -1012,6 +1067,8 @@ func TestBackend_ExecuteChangeSet(t *testing.T) { wantStatus: "CREATE_COMPLETE", }, { + // modifiedTemplate adds a resource relative to simpleTemplate, so + // this change set carries a real Add change and is AVAILABLE. name: "existing_stack", setup: func(t *testing.T, b *cloudformation.InMemoryBackend) { t.Helper() @@ -1023,12 +1080,35 @@ func TestBackend_ExecuteChangeSet(t *testing.T) { cloudformation.StackOptions{}, ) require.NoError(t, err) - _, err = b.CreateChangeSet(t.Context(), "existing-stack", "upd-cs", simpleTemplate, "", nil) + _, err = b.CreateChangeSet(t.Context(), "existing-stack", "upd-cs", modifiedTemplate, "", nil) require.NoError(t, err) }, stackName: "existing-stack", csName: "upd-cs", }, + { + // Re-submitting the identical template yields zero changes, so AWS + // marks the change set FAILED/UNAVAILABLE and ExecuteChangeSet must + // reject it with InvalidChangeSetStatus rather than silently + // re-applying the (unchanged) template. + name: "existing_stack_no_changes", + setup: func(t *testing.T, b *cloudformation.InMemoryBackend) { + t.Helper() + _, err := b.CreateStack( + t.Context(), + "nochange-stack", + simpleTemplate, + nil, + cloudformation.StackOptions{}, + ) + require.NoError(t, err) + _, err = b.CreateChangeSet(t.Context(), "nochange-stack", "noop-cs", simpleTemplate, "", nil) + require.NoError(t, err) + }, + stackName: "nochange-stack", + csName: "noop-cs", + wantErr: cloudformation.ErrChangeSetNotExecutable, + }, { name: "not_found", stackName: "s", @@ -1065,6 +1145,62 @@ func TestBackend_ExecuteChangeSet(t *testing.T) { } } +// templateWithTopic is simpleTemplate plus an SNS topic instead of the SQS +// queue modifiedTemplate adds, giving a second, independent real change. +const templateWithTopic = `{"AWSTemplateFormatVersion":"2010-09-09",` + + `"Resources":{"MyBucket":{"Type":"AWS::S3::Bucket","Properties":{}},` + + `"MyTopic":{"Type":"AWS::SNS::Topic","Properties":{}}}}` + +// TestBackend_ExecuteChangeSet_DeletesOtherChangeSets verifies AWS's documented +// behaviour: "When you execute a change set, CloudFormation deletes all other +// change sets associated with the stack because they aren't valid for the +// updated stack" — not just the one that was executed. +func TestBackend_ExecuteChangeSet_DeletesOtherChangeSets(t *testing.T) { + t.Parallel() + + b := newBackend() + _, err := b.CreateStack(t.Context(), "multi-cs-stack", simpleTemplate, nil, cloudformation.StackOptions{}) + require.NoError(t, err) + + _, err = b.CreateChangeSet(t.Context(), "multi-cs-stack", "cs-a", modifiedTemplate, "", nil) + require.NoError(t, err) + _, err = b.CreateChangeSet(t.Context(), "multi-cs-stack", "cs-b", templateWithTopic, "", nil) + require.NoError(t, err) + + list, err := b.ListChangeSets("multi-cs-stack", "") + require.NoError(t, err) + require.Len(t, list.Data, 2) + + require.NoError(t, b.ExecuteChangeSet(t.Context(), "multi-cs-stack", "cs-a")) + + _, err = b.DescribeChangeSet("multi-cs-stack", "cs-a") + require.ErrorIs(t, err, cloudformation.ErrChangeSetNotFound, "executed change set must be gone") + + _, err = b.DescribeChangeSet("multi-cs-stack", "cs-b") + require.ErrorIs(t, err, cloudformation.ErrChangeSetNotFound, + "sibling change sets must also be discarded — they no longer apply to the updated stack") + + list, err = b.ListChangeSets("multi-cs-stack", "") + require.NoError(t, err) + assert.Empty(t, list.Data) +} + +// TestHandler_ExecuteChangeSet_InvalidStatus verifies the wire-level error for +// executing a non-AVAILABLE change set uses AWS's real error code +// (InvalidChangeSetStatus), and that ChangeSetNotFound errors use the +// SDK-modeled code without an "Exception" suffix (the deserializer matches +// "ChangeSetNotFound", not "ChangeSetNotFoundException"). +func TestHandler_ExecuteChangeSet_InvalidStatus(t *testing.T) { + t.Parallel() + + h := newHandler() + rec := postForm(t, h, "Action=DeleteChangeSet&StackName=no-such-stack&ChangeSetName=no-such-cs") + + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "ChangeSetNotFound") + assert.NotContains(t, rec.Body.String(), "ChangeSetNotFoundException") +} + // ---- Backend: ListChangeSets ------------------------------------------------ func TestBackend_ListChangeSets(t *testing.T) { @@ -1686,6 +1822,55 @@ func TestHandler_DescribeStackEvents(t *testing.T) { } } +// TestHandler_DescribeStackEvents_NextToken verifies the wire-level NextToken +// round-trips: the first response carries a NextToken when more events exist +// than the default page size, and requesting with that token returns the +// remaining (disjoint) events. +func TestHandler_DescribeStackEvents_NextToken(t *testing.T) { + t.Parallel() + + h := newHandler() + postForm(t, h, "Action=CreateStack&StackName=evt-tok-stack&TemplateBody="+simpleTemplate) + + for range 40 { + postForm(t, h, "Action=UpdateStack&StackName=evt-tok-stack&TemplateBody="+simpleTemplate) + } + + rec := postForm(t, h, "Action=DescribeStackEvents&StackName=evt-tok-stack") + require.Equal(t, http.StatusOK, rec.Code) + + type eventXML struct { + EventID string `xml:"EventId"` + } + type eventsResult struct { + NextToken string `xml:"NextToken"` + StackEvents []eventXML `xml:"StackEvents>member"` + } + type resp struct { + Result eventsResult `xml:"DescribeStackEventsResult"` + } + + var first resp + require.NoError(t, xml.NewDecoder(strings.NewReader(rec.Body.String())).Decode(&first)) + require.NotEmpty(t, first.Result.NextToken, "expected pagination to kick in past the default page size") + + rec2 := postForm(t, h, + "Action=DescribeStackEvents&StackName=evt-tok-stack&NextToken="+url.QueryEscape(first.Result.NextToken)) + require.Equal(t, http.StatusOK, rec2.Code) + + var second resp + require.NoError(t, xml.NewDecoder(strings.NewReader(rec2.Body.String())).Decode(&second)) + require.NotEmpty(t, second.Result.StackEvents) + + firstIDs := make(map[string]bool, len(first.Result.StackEvents)) + for _, e := range first.Result.StackEvents { + firstIDs[e.EventID] = true + } + for _, e := range second.Result.StackEvents { + assert.False(t, firstIDs[e.EventID], "second page must not repeat first-page events") + } +} + // ---- Handler: GetTemplate --------------------------------------------------- func TestHandler_GetTemplate(t *testing.T) { @@ -1843,15 +2028,35 @@ func TestHandler_ExecuteChangeSet(t *testing.T) { wantCode int }{ { + // A change set created with a real template (real Add changes) is + // AVAILABLE and can be executed. name: "success", setup: func(t *testing.T, h *cloudformation.Handler) { t.Helper() - postForm(t, h, "Action=CreateChangeSet&StackName=exec-cs-stack&ChangeSetName=exec-cs&TemplateBody=") + postFormValues(t, h, url.Values{ + "Action": {"CreateChangeSet"}, + "StackName": {"exec-cs-stack"}, + "ChangeSetName": {"exec-cs"}, + "TemplateBody": {simpleTemplate}, + }) }, form: "Action=ExecuteChangeSet&StackName=exec-cs-stack&ChangeSetName=exec-cs", wantCode: http.StatusOK, wantBody: "ExecuteChangeSetResponse", }, + { + // An empty TemplateBody produces zero changes, so AWS marks the + // change set FAILED/UNAVAILABLE and rejects execution with + // InvalidChangeSetStatus rather than silently no-op'ing. + name: "unavailable_no_changes", + setup: func(t *testing.T, h *cloudformation.Handler) { + t.Helper() + postForm(t, h, "Action=CreateChangeSet&StackName=exec-cs-empty&ChangeSetName=exec-cs&TemplateBody=") + }, + form: "Action=ExecuteChangeSet&StackName=exec-cs-empty&ChangeSetName=exec-cs", + wantCode: http.StatusBadRequest, + wantBody: "InvalidChangeSetStatus", + }, } for _, tt := range tests { diff --git a/services/cloudformation/dynamic_refs_test.go b/services/cloudformation/dynamic_refs_test.go index 3f52c4406..3e4b098d1 100644 --- a/services/cloudformation/dynamic_refs_test.go +++ b/services/cloudformation/dynamic_refs_test.go @@ -543,8 +543,9 @@ func TestBackend_CreateStack_DynamicRefs_StackEvents(t *testing.T) { require.NoError(t, err) assert.Equal(t, "CREATE_FAILED", stack.StackStatus) - events, err := cfnBackend.DescribeStackEvents(stack.StackName) + evtPage, err := cfnBackend.DescribeStackEvents(stack.StackName, "") require.NoError(t, err) + events := evtPage.Data require.NotEmpty(t, events) // At least one event should record CREATE_FAILED with the dynamic ref reason. diff --git a/services/cloudformation/export_test.go b/services/cloudformation/export_test.go index 2117b1f9e..daa452150 100644 --- a/services/cloudformation/export_test.go +++ b/services/cloudformation/export_test.go @@ -25,7 +25,7 @@ func (b *InMemoryBackend) ForceStackStatus(stackName, status string) { b.mu.Lock("ForceStackStatus") defer b.mu.Unlock() - if s, ok := b.stacks[stackName]; ok { + if s, ok := b.stacks.Get(stackName); ok { s.StackStatus = status } } @@ -77,7 +77,7 @@ func (b *InMemoryBackend) DriftDetectionCount(stackID string) int { count := 0 - for _, status := range b.driftDetections { + for _, status := range b.driftDetections.All() { if status.StackID == stackID { count++ } @@ -100,7 +100,7 @@ func (b *InMemoryBackend) ForceRemoveResource(stackName, logicalID string) { b.mu.Lock("ForceRemoveResource") defer b.mu.Unlock() - stack, ok := b.stacks[stackName] + stack, ok := b.stacks.Get(stackName) if !ok { return } @@ -114,7 +114,7 @@ func (b *InMemoryBackend) ForceModifyResourceProperties(stackName, logicalID str b.mu.Lock("ForceModifyResourceProperties") defer b.mu.Unlock() - stack, ok := b.stacks[stackName] + stack, ok := b.stacks.Get(stackName) if !ok { return } diff --git a/services/cloudformation/handler.go b/services/cloudformation/handler.go index 40d8f7c46..f3c911625 100644 --- a/services/cloudformation/handler.go +++ b/services/cloudformation/handler.go @@ -855,10 +855,11 @@ func (h *Handler) handleDescribeStackEvents(form url.Values, c *echo.Context) er return h.xmlError(c, "ValidationError", "StackName is required") } - events, err := h.Backend.DescribeStackEvents(stackName) + p, err := h.Backend.DescribeStackEvents(stackName, form.Get("NextToken")) if err != nil { return h.xmlError(c, "ValidationError", err.Error()) } + events := p.Data type eventXML struct { EventID string `xml:"EventId"` @@ -887,6 +888,7 @@ func (h *Handler) handleDescribeStackEvents(form url.Values, c *echo.Context) er } type eventsResult struct { + NextToken string `xml:"NextToken,omitempty"` StackEvents []eventXML `xml:"StackEvents>member"` } type response struct { @@ -898,7 +900,7 @@ func (h *Handler) handleDescribeStackEvents(form url.Values, c *echo.Context) er return writeXML(c, response{ Xmlns: cfnNS, - Result: eventsResult{StackEvents: members}, + Result: eventsResult{StackEvents: members, NextToken: p.Next}, RequestID: uuid.New().String(), }) } @@ -948,7 +950,11 @@ func (h *Handler) handleExecuteChangeSet(form url.Values, c *echo.Context) error changeSetName := form.Get("ChangeSetName") if err := h.Backend.ExecuteChangeSet(c.Request().Context(), stackName, changeSetName); err != nil { - return h.xmlError(c, "ChangeSetNotFoundException", err.Error()) + if errors.Is(err, ErrChangeSetNotExecutable) { + return h.xmlError(c, "InvalidChangeSetStatus", err.Error()) + } + + return h.xmlError(c, "ChangeSetNotFound", err.Error()) } type response struct { @@ -965,7 +971,7 @@ func (h *Handler) handleDeleteChangeSet(form url.Values, c *echo.Context) error changeSetName := form.Get("ChangeSetName") if err := h.Backend.DeleteChangeSet(stackName, changeSetName); err != nil { - return h.xmlError(c, "ChangeSetNotFoundException", err.Error()) + return h.xmlError(c, "ChangeSetNotFound", err.Error()) } type response struct { diff --git a/services/cloudformation/handler_parity.go b/services/cloudformation/handler_parity.go index 942d4ee75..6d321fb62 100644 --- a/services/cloudformation/handler_parity.go +++ b/services/cloudformation/handler_parity.go @@ -130,7 +130,7 @@ func (h *Handler) handleDescribeChangeSetFull(form url.Values, c *echo.Context) cs, err := h.Backend.DescribeChangeSet(stackName, changeSetName) if err != nil { - return h.xmlError(c, "ChangeSetNotFoundException", err.Error()) + return h.xmlError(c, "ChangeSetNotFound", err.Error()) } type targetXML struct { diff --git a/services/cloudformation/intrinsics_validate_test.go b/services/cloudformation/intrinsics_validate_test.go index 5fba27fa6..cdf0b6954 100644 --- a/services/cloudformation/intrinsics_validate_test.go +++ b/services/cloudformation/intrinsics_validate_test.go @@ -152,8 +152,9 @@ func TestCreateStack_IntrinsicErrorPropagation(t *testing.T) { } if tt.wantEvent != "" { - events, evErr := b.DescribeStackEvents(tt.name) + evtPage, evErr := b.DescribeStackEvents(tt.name, "") require.NoError(t, evErr) + events := evtPage.Data statuses := make([]string, len(events)) for i, e := range events { statuses[i] = e.ResourceStatus diff --git a/services/cloudformation/introspection_test.go b/services/cloudformation/introspection_test.go index 052888bf7..173748372 100644 --- a/services/cloudformation/introspection_test.go +++ b/services/cloudformation/introspection_test.go @@ -488,6 +488,156 @@ func TestBackend_ExportsRemovedOnDelete(t *testing.T) { assert.Empty(t, p2.Data) } +// noImportTemplate is importTemplate with the Fn::ImportValue reference +// removed, used to simulate the importing stack being updated away from the +// export before the exporting stack is deleted, and as an update target for +// the exporting stack itself (one that no longer produces any Outputs). +const noImportTemplate = `{ + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "MyTopic": {"Type": "AWS::SNS::Topic", "Properties": {}} + } +}` + +// TestBackend_DeleteStack_BlockedByImportedExport verifies AWS's real +// behaviour: a stack cannot be deleted while another active stack still +// imports one of its exports ("Export X cannot be deleted as it is in use by +// Y"). The block is lifted once the importing stack no longer references the +// export (deleted or updated away). +func TestBackend_DeleteStack_BlockedByImportedExport(t *testing.T) { + t.Parallel() + + tests := []struct { + arrange func(t *testing.T, b *cloudformation.InMemoryBackend) + name string + wantErr bool + }{ + { + name: "blocked_while_importer_active", + arrange: func(t *testing.T, b *cloudformation.InMemoryBackend) { + t.Helper() + _, err := b.CreateStack(t.Context(), "importer", importTemplate, nil, + cloudformation.StackOptions{}) + require.NoError(t, err) + }, + wantErr: true, + }, + { + name: "allowed_after_importer_deleted", + arrange: func(t *testing.T, b *cloudformation.InMemoryBackend) { + t.Helper() + _, err := b.CreateStack(t.Context(), "importer", importTemplate, nil, + cloudformation.StackOptions{}) + require.NoError(t, err) + require.NoError(t, b.DeleteStack(t.Context(), "importer")) + }, + wantErr: false, + }, + { + name: "allowed_with_no_importer", + arrange: func(t *testing.T, _ *cloudformation.InMemoryBackend) { t.Helper() }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := newBackend() + _, err := b.CreateStack(t.Context(), "exporter", exportTemplate, nil, cloudformation.StackOptions{}) + require.NoError(t, err) + + tt.arrange(t, b) + + err = b.DeleteStack(t.Context(), "exporter") + if tt.wantErr { + require.ErrorIs(t, err, cloudformation.ErrExportInUse) + assert.Contains(t, err.Error(), "shared-bucket") + assert.Contains(t, err.Error(), "importer") + + return + } + + require.NoError(t, err) + }) + } +} + +// TestBackend_UpdateStack_BlockedByImportedExportRemoval verifies that +// updating a stack's template in a way that would drop an export still +// imported by another active stack is rejected before any resource is +// touched, matching AWS's export-in-use protection for updates as well as +// deletes. +func TestBackend_UpdateStack_BlockedByImportedExportRemoval(t *testing.T) { + t.Parallel() + + tests := []struct { + arrange func(t *testing.T, b *cloudformation.InMemoryBackend) + name string + newTemplate string + wantErr bool + }{ + { + name: "blocked_when_importer_active", + arrange: func(t *testing.T, b *cloudformation.InMemoryBackend) { + t.Helper() + _, err := b.CreateStack(t.Context(), "importer", importTemplate, nil, + cloudformation.StackOptions{}) + require.NoError(t, err) + }, + newTemplate: noImportTemplate, + wantErr: true, + }, + { + name: "allowed_after_importer_no_longer_references_export", + arrange: func(t *testing.T, b *cloudformation.InMemoryBackend) { + t.Helper() + _, err := b.CreateStack(t.Context(), "importer", importTemplate, nil, + cloudformation.StackOptions{}) + require.NoError(t, err) + require.NoError(t, b.DeleteStack(t.Context(), "importer")) + }, + newTemplate: noImportTemplate, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := newBackend() + _, err := b.CreateStack(t.Context(), "exporter", exportTemplate, nil, cloudformation.StackOptions{}) + require.NoError(t, err) + + tt.arrange(t, b) + + _, err = b.UpdateStack(t.Context(), "exporter", tt.newTemplate, nil, cloudformation.StackOptions{}) + require.NoError(t, err) // UpdateStack itself never errors; failures surface via StackStatus. + + stack, descErr := b.DescribeStack("exporter") + require.NoError(t, descErr) + + if tt.wantErr { + assert.Equal(t, "UPDATE_ROLLBACK_COMPLETE", stack.StackStatus) + assert.Contains(t, stack.StackStatusReason, "shared-bucket") + + p, expErr := b.ListExports("") + require.NoError(t, expErr) + assert.Len(t, p.Data, 1, "export must survive a blocked update") + + return + } + + assert.Equal(t, "UPDATE_COMPLETE", stack.StackStatus) + p, expErr := b.ListExports("") + require.NoError(t, expErr) + assert.Empty(t, p.Data, "export should be gone once no longer produced by the template") + }) + } +} + func TestBackend_ListImports(t *testing.T) { t.Parallel() diff --git a/services/cloudformation/persistence.go b/services/cloudformation/persistence.go index 8ba0b5d89..4843050e5 100644 --- a/services/cloudformation/persistence.go +++ b/services/cloudformation/persistence.go @@ -2,20 +2,41 @@ package cloudformation import ( "context" + "encoding/json" + "fmt" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" ) +// cfnSnapshotVersion identifies the shape of backendSnapshot's Tables blob +// (i.e. the set of resources registered on b.registry -- see +// registerAllTables in store_setup.go). It must be bumped whenever a change +// there would make an older snapshot unsafe to decode as the current shape. +// Restore compares this against the persisted value and discards (rather +// than attempts to partially decode) any mismatch -- see Restore below. This +// mirrors the services/sqs pilot (commit 0f09d77c) and the services/ec2 +// conversion (commit 12e611a4). +const cfnSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the CloudFormation +// backend. +// +// Tables holds one JSON-encoded array per registered store.Table (see +// registerAllTables in store_setup.go), produced by +// [store.Registry.SnapshotAll]. The remaining fields cover the maps that +// were NOT converted to store.Table (nested or one-to-many; see the doc +// comment on registerAllTables) and are persisted exactly as before the +// conversion. type backendSnapshot struct { - Stacks map[string]*Stack `json:"stacks"` - Events map[string][]StackEvent `json:"events"` - Resources map[string]map[string]*StackResource `json:"resources"` - ChangeSets map[string]map[string]*ChangeSet `json:"changeSets"` - Exports map[string]*Export `json:"exports"` - DriftDetections map[string]*DriftDetectionStatus `json:"driftDetections"` - StackPolicies map[string]string `json:"stackPolicies"` - AccountID string `json:"accountID"` - Region string `json:"region"` + Tables map[string]json.RawMessage `json:"tables"` + Events map[string][]StackEvent `json:"events"` + Resources map[string]map[string]*StackResource `json:"resources"` + ChangeSets map[string]map[string]*ChangeSet `json:"changeSets"` + StackPolicies map[string]string `json:"stackPolicies"` + AccountID string `json:"accountID"` + Region string `json:"region"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. @@ -24,16 +45,26 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "cloudformation: snapshot table marshal failed", "error", err) + + return nil + } + snap := backendSnapshot{ - Stacks: b.stacks, - Events: b.events, - Resources: b.resources, - ChangeSets: b.changeSets, - Exports: b.exports, - DriftDetections: b.driftDetections, - StackPolicies: b.stackPolicies, - AccountID: b.accountID, - Region: b.region, + Version: cfnSnapshotVersion, + Tables: tables, + Events: b.events, + Resources: b.resources, + ChangeSets: b.changeSets, + StackPolicies: b.stackPolicies, + AccountID: b.accountID, + Region: b.region, } return persistence.MarshalSnapshot(ctx, "cloudformation", snap) @@ -51,8 +82,24 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() - if snap.Stacks == nil { - snap.Stacks = make(map[string]*Stack) + if snap.Version != cfnSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/sqs pilot (commit 0f09d77c). + logger.Load(ctx).WarnContext(ctx, + "cloudformation: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", cfnSnapshotVersion) + + b.registry.ResetAll() + + return nil + } + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("cloudformation: restore snapshot tables: %w", err) } if snap.Events == nil { @@ -67,32 +114,21 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { snap.ChangeSets = make(map[string]map[string]*ChangeSet) } - if snap.Exports == nil { - snap.Exports = make(map[string]*Export) - } - - if snap.DriftDetections == nil { - snap.DriftDetections = make(map[string]*DriftDetectionStatus) - } - if snap.StackPolicies == nil { snap.StackPolicies = make(map[string]string) } - b.stacks = snap.Stacks b.events = snap.Events b.resources = snap.Resources b.changeSets = snap.ChangeSets - b.exports = snap.Exports - b.driftDetections = snap.DriftDetections b.stackPolicies = snap.StackPolicies b.accountID = snap.AccountID b.region = snap.Region // Rebuild the stackIDIndex from the restored stacks. - b.stackIDIndex = make(map[string]string, len(b.stacks)) - for name, stack := range b.stacks { - b.stackIDIndex[stack.StackID] = name + b.stackIDIndex = make(map[string]string, b.stacks.Len()) + for _, stack := range b.stacks.All() { + b.stackIDIndex[stack.StackID] = stack.StackName } return nil diff --git a/services/cloudformation/store_setup.go b/services/cloudformation/store_setup.go new file mode 100644 index 000000000..2a34b1ef1 --- /dev/null +++ b/services/cloudformation/store_setup.go @@ -0,0 +1,116 @@ +package cloudformation + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// single-level map[string]*T resource field on InMemoryBackend is registered +// exactly once, here, as a *store.Table[T] on b.registry. See pkgs/store's +// package doc and the services/sqs pilot (commit 0f09d77c) / services/ec2 +// conversion (commit 12e611a4) for the pattern this follows. +// +// The following resource fields are deliberately NOT registered here and +// remain plain maps, because store.Table requires a single *V per key and a +// pure key function derived from V's own fields -- neither holds for a +// nested map[string]map[string]V or a one-to-many map[string][]V: +// - stackIDIndex: reverse index (stackID ARN -> stackName); the map value +// is a bare string with no field to derive the ARN key from +// - events: map[string][]StackEvent -- one-to-many, not a single *V per key +// - resources: map[string]map[string]*StackResource -- nested (stackID -> +// logicalID -> resource) +// - changeSets: map[string]map[string]*ChangeSet -- nested (stackName -> +// changeSetName -> change set) +// - stackPolicies: map[string]string -- bare string value, no identity field +// - stackInstances: map[string][]StackInstance -- one-to-many +// - stackSetOperations: map[string]map[string]*StackSetOperation -- nested +// (stackSetName -> operationID -> op) +// - typeConfigs: map[string]string -- bare string value +// - handlerProgress: map[string]string -- bare string value +// - signals: map[string][]SignalRecord -- one-to-many, and SignalRecord +// carries no stackName/logicalID field to key by +// - stackSetOpResults: map[string]map[string][]StackSetOperationResult -- +// doubly-nested one-to-many +// - typeVersions: map[string][]*RegisteredTypeVersion -- one-to-many, and +// order-dependent (RegisterType/SetTypeDefaultVersion walk insertion +// order to flip IsDefault on prior versions) +// - resourceScanItems: map[string][]ScannedResource -- one-to-many, and +// ScannedResource carries no scanID field to key by +// - resourceDriftStatus: map[string]map[string]string -- nested, bare +// string value +// - resourceDriftDetail: map[string]map[string]StackResourceDrift -- nested, +// and stored by value rather than by pointer +// - driftByStackID: map[string][]string -- one-to-many reverse index +// +// This mirrors the ec2 conversion, which likewise left every nested +// map[string]map[string]*T (e.g. tgwRTPropagations) and every map[string][]T +// (e.g. ipamPoolCidrs) as a plain field rather than force-fitting it into a +// Table. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +func stackKeyFn(s *Stack) string { return s.StackName } +func exportKeyFn(e *Export) string { return e.Name } +func driftDetectionKeyFn(d *DriftDetectionStatus) string { return d.StackDriftDetectionID } +func stackSetKeyFn(s *StackSet) string { return s.StackSetName } +func generatedTemplateKeyFn(g *GeneratedTemplate) string { return g.GeneratedTemplateID } +func resourceScanKeyFn(r *ResourceScan) string { return r.ResourceScanID } +func registeredTypeKeyFn(t *RegisteredType) string { return t.TypeArn } +func typeRegistrationKeyFn(r *TypeRegistrationRecord) string { return r.Token } +func publisherKeyFn(p *Publisher) string { return p.PublisherID } +func stackRefactorKeyFn(r *StackRefactor) string { return r.RefactorID } +func hookResultKeyFn(h *HookResult) string { return h.Token } + +// registerAllTables registers every converted resource map on b.registry +// exactly once. It must be called during construction only (immediately after +// b.registry is created), never on every reset -- store.Register panics on a +// duplicate name, so a runtime reset goes through registry.ResetAll() instead +// (see Restore's version-mismatch branch in persistence.go). +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call to the concrete field and value type. Mirrors the ec2 conversion's +// tableRegistrations slice (services/ec2/store_setup.go). +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.stacks = store.Register(b.registry, "stacks", store.New(stackKeyFn)) + }, + func(b *InMemoryBackend) { + b.exports = store.Register(b.registry, "exports", store.New(exportKeyFn)) + }, + func(b *InMemoryBackend) { + b.driftDetections = store.Register(b.registry, "driftDetections", store.New(driftDetectionKeyFn)) + }, + func(b *InMemoryBackend) { + b.stackSets = store.Register(b.registry, "stackSets", store.New(stackSetKeyFn)) + }, + func(b *InMemoryBackend) { + b.generatedTemplates = store.Register( + b.registry, "generatedTemplates", store.New(generatedTemplateKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.resourceScans = store.Register(b.registry, "resourceScans", store.New(resourceScanKeyFn)) + }, + func(b *InMemoryBackend) { + b.typeRegistry = store.Register(b.registry, "typeRegistry", store.New(registeredTypeKeyFn)) + }, + func(b *InMemoryBackend) { + b.typeRegistrations = store.Register( + b.registry, "typeRegistrations", store.New(typeRegistrationKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.publishers = store.Register(b.registry, "publishers", store.New(publisherKeyFn)) + }, + func(b *InMemoryBackend) { + b.stackRefactors = store.Register(b.registry, "stackRefactors", store.New(stackRefactorKeyFn)) + }, + func(b *InMemoryBackend) { + b.hookResults = store.Register(b.registry, "hookResults", store.New(hookResultKeyFn)) + }, +} diff --git a/services/cloudfront/PARITY.md b/services/cloudfront/PARITY.md new file mode 100644 index 000000000..463b2515a --- /dev/null +++ b/services/cloudfront/PARITY.md @@ -0,0 +1,160 @@ +--- +service: cloudfront +sdk_module: aws-sdk-go-v2/service/cloudfront@v1.60.2 +last_audit_commit: e5205dbe +last_audit_date: 2026-07-05 +overall: A # ~1000 LOC of genuine fixes found and applied this pass +ops: + CreateDistribution: {wire: ok, errors: ok, state: ok, persist: ok, note: "now runs validateQuantities on the raw config"} + CreateDistributionWithTags: {wire: ok, errors: ok, state: ok, persist: ok} + GetDistribution: {wire: ok, errors: ok, state: ok, persist: ok} + GetDistributionConfig: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateDistribution: {wire: ok, errors: ok, state: ok, persist: ok, note: "If-Match/ETag enforced; validateQuantities added"} + DeleteDistribution: {wire: ok, errors: ok, state: ok, persist: ok, note: "If-Match enforced; DistributionNotDisabled enforced"} + ListDistributions: {wire: ok, errors: ok, state: ok, persist: ok} + CopyDistribution: {wire: ok, errors: ok, state: ok, persist: ok} + CreateInvalidation: {wire: ok, errors: ok, state: ok, persist: ok, note: "validateQuantities added for Paths; background reconciler transitions InProgress->Completed"} + GetInvalidation: {wire: ok, errors: ok, state: ok, persist: ok} + ListInvalidations: {wire: ok, errors: ok, state: ok, persist: ok} + CreateCachePolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "now returns CachePolicyAlreadyExists (was DistributionAlreadyExists); validateQuantities added"} + UpdateCachePolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "If-Match enforced; CachePolicyAlreadyExists; validateQuantities added"} + DeleteCachePolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "NEW: CachePolicyInUse guard via distribution config token index"} + GetCachePolicy / GetCachePolicyConfig / ListCachePolicies: {wire: ok, errors: ok, state: ok, persist: ok, note: "no managed-vs-custom Type support -- gap filed"} + CreateOriginRequestPolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "now returns OriginRequestPolicyAlreadyExists; validateQuantities added"} + UpdateOriginRequestPolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "same as above"} + DeleteOriginRequestPolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "NEW: OriginRequestPolicyInUse guard"} + GetOriginRequestPolicy / GetOriginRequestPolicyConfig / ListOriginRequestPolicies: {wire: ok, errors: ok, state: ok, persist: ok} + CreateResponseHeadersPolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "now returns ResponseHeadersPolicyAlreadyExists; validateQuantities added"} + UpdateResponseHeadersPolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "same as above"} + DeleteResponseHeadersPolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "NEW: ResponseHeadersPolicyInUse guard"} + GetResponseHeadersPolicy / GetResponseHeadersPolicyConfig / ListResponseHeadersPolicies: {wire: ok, errors: ok, state: ok, persist: ok} + CreateOriginAccessControl: {wire: ok, errors: ok, state: ok, persist: ok, note: "now returns OriginAccessControlAlreadyExists; validateQuantities added"} + UpdateOriginAccessControl: {wire: ok, errors: ok, state: ok, persist: ok, note: "same as above"} + DeleteOriginAccessControl: {wire: ok, errors: ok, state: ok, persist: ok, note: "no InUse guard yet -- gap filed (gopherstack-na4)"} + GetOriginAccessControl / GetOriginAccessControlConfig / ListOriginAccessControls: {wire: ok, errors: ok, state: ok, persist: ok} + CreateCloudFrontOriginAccessIdentity: {wire: ok, errors: ok, state: ok, persist: ok, note: "validateQuantities added (harmless no-op for this shape)"} + UpdateCloudFrontOriginAccessIdentity: {wire: ok, errors: ok, state: ok, persist: ok, note: "If-Match enforced"} + DeleteCloudFrontOriginAccessIdentity: {wire: ok, errors: ok, state: ok, persist: ok, note: "If-Match enforced; no InUse guard yet -- gap filed"} + GetCloudFrontOriginAccessIdentity / Config / List: {wire: ok, errors: ok, state: ok, persist: ok} + CreateFunction: {wire: fixed, errors: ok, state: ok, persist: ok, note: "FIXED this pass: response was missing required FunctionMetadata.FunctionARN/CreatedTime/LastModifiedTime; now returns FunctionAlreadyExists (was DistributionAlreadyExists); validateQuantities added"} + UpdateFunction: {wire: fixed, errors: ok, state: ok, persist: ok, note: "same wire fix; If-Match enforced; validateQuantities added"} + PublishFunction: {wire: fixed, errors: ok, state: ok, persist: ok, note: "same wire fix; If-Match enforced; LastModifiedTime now bumped"} + DeleteFunction: {wire: ok, errors: ok, state: ok, persist: ok, note: "NEW: FunctionInUse guard (keyed by FunctionARN, not name)"} + GetFunction / DescribeFunction / ListFunctions / TestFunction: {wire: fixed, errors: ok, state: ok, persist: ok, note: "GetFunction/DescribeFunction/ListFunctions share the same FunctionMetadata fix"} + TagResource / UntagResource / ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok} + AssociateAlias / AssociateDistributionWebACL / AssociateDistributionTenantWebACL: {wire: ok, errors: ok, state: ok, persist: ok, families: cross-service} +families: + distribution_tenants_connection_groups: {status: ok, note: "CreateDistributionTenant/UpdateDistributionTenant now run validateQuantities; If-Match enforced on update/delete; audited, no new findings beyond the Quantity gap"} + field_level_encryption: {status: ok, note: "Create/Update for config + profile now run validateQuantities and return the correct *AlreadyExists code (FieldLevelEncryptionConfigAlreadyExists / FieldLevelEncryptionProfileAlreadyExists) instead of DistributionAlreadyExists; FLEProfileInUse guard on profile delete pre-existed and is correct"} + public_keys_key_groups: {status: ok, note: "CreatePublicKey/CreateKeyGroup/UpdateKeyGroup now return PublicKeyAlreadyExists/KeyGroupAlreadyExists instead of DistributionAlreadyExists; PublicKeyInUse guard on public-key delete pre-existed and is correct; KeyGroup delete still has no InUse guard -- gap filed (gopherstack-na4)"} + realtime_log_configs: {status: ok, note: "CreateRealtimeLogConfig now returns RealtimeLogConfigAlreadyExists instead of DistributionAlreadyExists"} + key_value_stores: {status: ok, note: "control-plane Create/Update run validateQuantities (no-op, shape has no Quantity/Items pairs); data-plane GetKey/PutKeys/ListKeys correctly use the separate JSON protocol, out of scope for this XML-focused sweep"} + vpc_origins: {status: ok, note: "Create/Update run validateQuantities (no-op for this shape)"} + continuous_deployment_policy: {status: ok, note: "Create/Update run validateQuantities; If-Match already enforced"} + invalidations_realtime_status: {status: ok, note: "background reconciler goroutine (runInvalidationReconciler) has a clean stopCh lifecycle via Close(); no leak"} + monitoring_subscriptions_public_resource_policy_connection_groups: {status: ok, note: "audited via handler_new_ops.go/handler_batch2.go dispatch; no Quantity/AlreadyExists-code issues found in these shapes"} +gaps: + - "Managed (AWS-provided) cache/origin-request/response-headers policies are not seeded, and List* does not support the Type=managed|custom filter (bd: gopherstack-a9t)" + - "DeleteKeyGroup / DeleteCloudFrontOriginAccessIdentity / DeleteOriginAccessControl have no InUse-on-delete guard, unlike the CachePolicy/OriginRequestPolicy/ResponseHeadersPolicy/Function/PublicKey/FLEProfile guards (bd: gopherstack-na4)" + - "CreateDistribution (and likely CreateOAI/CreateStreamingDistribution) treat CallerReference reuse as unconditionally idempotent; real AWS returns *AlreadyExists when the reused CallerReference's config content differs (bd: gopherstack-mzx)" +deferred: + - "Distribution status InProgress->Deployed transition timer (currently InProgress persists indefinitely; no test depends on the transition, scope excluded per task's op-by-op priority list)" + - "KeyValueStore data-plane (GetKey/PutKeys/ListKeys, separate JSON protocol) -- explicitly out of scope for this REST-XML-focused sweep" + - "Full per-op audit of DistributionConfig nested shape correctness (Origins/OriginGroups/CacheBehaviors/ViewerCertificate/Restrictions field-by-field) beyond the Quantity/Items validation and the pre-existing minimal-parse model; RawConfig storage design predates this pass and was not restructured" +leaks: {status: clean, note: "runInvalidationReconciler goroutine has a proper stopCh + Close() lifecycle; no unbounded maps found; no new goroutines introduced this pass"} +--- + +## Notes + +**ETag/IfMatch** (proven, not touched this pass): Update/Delete for Distribution, CachePolicy, +OriginRequestPolicy, ResponseHeadersPolicy, OriginAccessControl, OAI, CloudFront Function, +ContinuousDeploymentPolicy, and DistributionTenant all require an `If-Match` header equal to +the resource's current ETag, else `412 PreconditionFailed`. This was already correct across +the board before this sweep; verified op-by-op, no gaps found. + +**InconsistentQuantities (the headline fix this pass)**: CloudFront's wire format pairs a +caller-supplied `N` with an `...` list virtually +everywhere in the schema (57 distinct SDK types carry a `Quantity *int32` field). Real +AWS rejects a request where `N` disagrees with the actual number of items with +`InconsistentQuantities` (400). Before this pass, the emulator had **zero** occurrences of +this validation anywhere in the codebase -- `grep -rn InconsistentQuantities` was empty. +Root cause: `DistributionConfig` (and most other configs) is parsed into either a minimal +typed struct or stored as opaque `RawConfig` bytes; nothing ever re-derived the caller's +stated `Quantity` and compared it to the real list length, because Go slices don't need an +explicit count. Fix: `services/cloudfront/quantity_validation.go` adds a generic recursive +XML-tree walker (`validateQuantities`) that finds every `.. +..` pairing in an arbitrary config body and flags a mismatch -- +no per-resource schema modeling required, and provably safe against false positives +because it only fires when both `Quantity` and `Items` siblings are actually present +(verified against `KeyGroupConfig`/`PublicKeyConfig`/`RealtimeLogConfig`/`VpcOriginConfig`, +none of which use this pattern in the real SDK, via the smithy serializers). Wired into +all ~58 Create/Update body-parsing call sites across `handler.go`, `handler_batch2.go`, +and `handler_new_ops.go`. + +**AlreadyExists error codes were all wrong (second major finding)**: `handleError`'s +`ErrAlreadyExists` sentinel had `code = "DistributionAlreadyExists"` and was reused +verbatim for CachePolicy, OriginRequestPolicy, ResponseHeadersPolicy, OriginAccessControl, +CloudFront Function, FieldLevelEncryptionConfig, FieldLevelEncryptionProfile, PublicKey, +KeyGroup, and RealtimeLogConfig name/CallerReference collisions -- i.e. creating a second +cache policy with a taken name returned the literal string `DistributionAlreadyExists`, +which is CloudFront's *distribution*-specific error code and was never even triggered by +an actual distribution collision (`CreateDistribution` doesn't use this sentinel at all; +it's fully idempotent on CallerReference, see gap above). Two existing tests +(`TestRefinement1_CachePolicyUniqueness`, `TestRefinement1_ErrorMapping`) asserted this +wrong code as if it were correct -- both fixed with justification comments pointing at the +real `aws-sdk-go-v2/service/cloudfront/types` error type names. Fix: 11 new distinct +sentinel errors (one per resource, matching the real SDK's dedicated error type where one +exists, falling back to the real generic `EntityAlreadyExists` where the SDK has no +resource-specific type -- e.g. Anycast IP lists, key value stores, trust stores). The +`handleError` switch (which had grown to cyclomatic complexity 23) was refactored into a +data-driven `errCodeMapping` table (pattern already established by EC2's `errCodeLookup`), +fixing a `cyclop` lint violation as a side effect. + +**Function responses were missing FunctionARN/CreatedTime/LastModifiedTime (third +finding)**: `FunctionMetadata` requires `FunctionARN` and `LastModifiedTime` per the real +SDK (`CreatedTime`/`Stage` too). The emulator's `Function` backend struct *did* compute +and store an ARN (`b.functionARN(name)`) on create, but `functionResponseXML` (shared by +Create/Get/Describe/Publish/Update) and the inline `FunctionSummary` builder in +`handleListFunctions` never emitted it -- a real SDK caller had no way to get a function's +ARN back from any read operation, which makes attaching the function to a distribution's +`FunctionAssociations` (which require the ARN, not the name) impossible. Fixed by adding +`CreatedTime`/`LastModifiedTime` fields to `Function`, populating them on +Create/Update/Publish, and emitting all four `FunctionMetadata` fields from both XML +builders. + +**InUse-on-delete guards (fourth finding)**: `DeleteCachePolicy`, `DeleteOriginRequestPolicy`, +`DeleteResponseHeadersPolicy`, and `DeleteFunction` had **no** check for whether the +resource was still referenced by a distribution -- real AWS returns `CachePolicyInUse` / +`OriginRequestPolicyInUse` / `ResponseHeadersPolicyInUse` / `FunctionInUse` (409) in that +case. (`PublicKeyInUse` and `FieldLevelEncryptionProfileInUse` already existed and are +correct -- not touched.) Fixed by adding `tokenReferencedByAnyDistribution` to +`backend_search_index.go`, reusing the pre-existing inverted token index that already +backs `ListDistributionsByCachePolicyID` etc. (built for the `ListDistributionsBy*` +control-plane ops) -- an O(1) check with no new scanning logic. `KeyGroup`, `OAI`, and +`OriginAccessControl` still lack this guard; deferred as gopherstack-na4 because OAI's +reference token has a different shape (`origin-access-identity/cloudfront/{id}` path +string, not a bare ID) and OAC has no existing `ListDistributionsBy*` helper to build on, +so both need slightly more care than a drop-in reuse. + +**InconsistentQuantities trap for the next auditor**: don't add per-resource Quantity +validation by hand if you find a new Create/Update body-parsing handler missing the +`validateQuantities(body)` call -- just add the one-line call. The generic walker already +covers any shape with a ``/`` sibling pair; it is a no-op (returns nil) +for shapes that don't use the pattern, so it is always safe to add defensively. + +**"Looks wrong but is correct" traps**: +- `ErrKeyGroupNotFound`'s wire code is `NoSuchResource`, not `NoSuchKeyGroup` -- this + matches the real SDK (`types.NoSuchResource` is what CloudFront actually returns for a + missing key group; there is no dedicated `NoSuchKeyGroup` type). Don't "fix" this. +- `ErrKeyValueStoreNotFound`/the new fallback `ErrAlreadyExists` both use `EntityNotFound`/ + `EntityAlreadyExists` -- also correct; the real SDK has no KVS-specific *NotFound/ + *AlreadyExists type either. +- `CreateAnycastIPList`/`CreateKeyValueStore`/`CreateTrustStore` intentionally still use the + generic `ErrAlreadyExists` (now `EntityAlreadyExists`) sentinel rather than a dedicated + one -- there is no `AnycastIpListAlreadyExists`/`KeyValueStoreAlreadyExists` type in + `aws-sdk-go-v2/service/cloudfront/types@v1.60.2` to match; this is the AWS-accurate + fallback, not an oversight. + +**Protocol**: REST-XML throughout (control plane). KeyValueStore's data-plane +(`handler_audit.go`, `GetKey`/`ListKeys`/`UpdateKeys`) correctly uses a separate JSON +protocol matching the real `cloudfront-keyvaluestore` service -- do not "fix" it to XML. diff --git a/services/cloudfront/already_exists_error_codes_test.go b/services/cloudfront/already_exists_error_codes_test.go new file mode 100644 index 000000000..851bdd83a --- /dev/null +++ b/services/cloudfront/already_exists_error_codes_test.go @@ -0,0 +1,114 @@ +package cloudfront_test + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Test_AlreadyExists_ResourceSpecificErrorCodes proves that a name/identifier collision +// on each of these resource types returns the AWS-accurate *AlreadyExists error code for +// that specific resource type (e.g. FunctionAlreadyExists, PublicKeyAlreadyExists), not a +// single generic code borrowed from an unrelated resource. Before this fix, every one of +// these collisions returned the literal string "DistributionAlreadyExists" (CloudFront's +// distribution-specific error code, reused as a blanket fallback for every resource type), +// which would break any SDK caller that switches on the error code / typed error. +func Test_AlreadyExists_ResourceSpecificErrorCodes(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + path string + wantCode string + body []byte + }{ + { + name: "CloudFront Function", + path: "/2020-05-31/function", + body: []byte(`` + + `dup-fn` + + `ccloudfront-js-2.0` + + `ZnVuY3Rpb24gaGFuZGxlcihldmVudCkge3JldHVybiBldmVudC5yZXF1ZXN0O30=` + + ``), + wantCode: "FunctionAlreadyExists", + }, + { + name: "PublicKey", + path: "/2020-05-31/public-key", + body: []byte(`` + + `pk-ref-dup` + + `dup-public-key` + + `` + testRSA2048PublicKeyPEM + `` + + ``), + wantCode: "PublicKeyAlreadyExists", + }, + { + name: "KeyGroup", + path: "/2020-05-31/key-group", + body: []byte(`dup-key-group`), + wantCode: "KeyGroupAlreadyExists", + }, + { + name: "OriginRequestPolicy", + path: "/2020-05-31/origin-request-policy", + body: []byte(`` + + `dup-orpc` + + ``), + wantCode: "OriginRequestPolicyAlreadyExists", + }, + { + name: "ResponseHeadersPolicy", + path: "/2020-05-31/response-headers-policy", + body: []byte(`dup-rhp`), + wantCode: "ResponseHeadersPolicyAlreadyExists", + }, + { + name: "OriginAccessControl", + path: "/2020-05-31/origin-access-control", + body: []byte(`` + + `dup-oac` + + `s3` + + `alwayssigv4` + + ``), + wantCode: "OriginAccessControlAlreadyExists", + }, + { + name: "FieldLevelEncryption (keyed by CallerReference)", + path: "/2020-05-31/field-level-encryption", + body: []byte(`` + + `fle-ref-dup` + + `c` + + ``), + wantCode: "FieldLevelEncryptionConfigAlreadyExists", + }, + { + name: "RealtimeLogConfig", + path: "/2020-05-31/realtime-log-config", + body: []byte(`` + + `dup-rt-log` + + `50` + + `timestamp` + + ``), + wantCode: "RealtimeLogConfigAlreadyExists", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h := newTestHandler() + + rec1 := doXML(t, h, http.MethodPost, tc.path, tc.body) + require.Equal(t, http.StatusCreated, rec1.Code, "first create; body=%s", rec1.Body.String()) + + rec2 := doXML(t, h, http.MethodPost, tc.path, tc.body) + assert.Equal(t, http.StatusConflict, rec2.Code, "duplicate create status; body=%s", rec2.Body.String()) + assert.Contains(t, rec2.Body.String(), tc.wantCode) + assert.NotContains(t, rec2.Body.String(), "DistributionAlreadyExists", + "must not fall back to the unrelated Distribution error code") + }) + } +} diff --git a/services/cloudfront/backend.go b/services/cloudfront/backend.go index 887ca1ee9..19941eb61 100644 --- a/services/cloudfront/backend.go +++ b/services/cloudfront/backend.go @@ -19,6 +19,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awserr" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) const ( @@ -170,8 +171,52 @@ var ( ErrOriginRequestPolicyNotFound = awserr.New("NoSuchOriginRequestPolicy", awserr.ErrNotFound) // ErrValidation is returned when request parameters fail validation. ErrValidation = awserr.New("InvalidArgument", awserr.ErrInvalidParameter) - // ErrAlreadyExists is returned when a resource with the same identifier already exists. - ErrAlreadyExists = awserr.New("DistributionAlreadyExists", awserr.ErrAlreadyExists) + // ErrAlreadyExists is the generic fallback for a resource whose identifier already + // exists but which has no dedicated AlreadyExists error type in the CloudFront API + // (e.g. Anycast IP lists, key value stores). AWS itself falls back to this same + // generic code for such resources. + ErrAlreadyExists = awserr.New("EntityAlreadyExists", awserr.ErrAlreadyExists) + // ErrCachePolicyAlreadyExists is returned when a cache policy name is already in use. + ErrCachePolicyAlreadyExists = awserr.New("CachePolicyAlreadyExists", awserr.ErrAlreadyExists) + // ErrOriginRequestPolicyAlreadyExists is returned when an origin request policy name + // is already in use. + ErrOriginRequestPolicyAlreadyExists = awserr.New("OriginRequestPolicyAlreadyExists", awserr.ErrAlreadyExists) + // ErrResponseHeadersPolicyAlreadyExists is returned when a response headers policy + // name is already in use. + ErrResponseHeadersPolicyAlreadyExists = awserr.New( + "ResponseHeadersPolicyAlreadyExists", + awserr.ErrAlreadyExists, + ) + // ErrOriginAccessControlAlreadyExists is returned when an origin access control name + // is already in use. + ErrOriginAccessControlAlreadyExists = awserr.New("OriginAccessControlAlreadyExists", awserr.ErrAlreadyExists) + // ErrFunctionAlreadyExists is returned when a CloudFront function name is already in use. + ErrFunctionAlreadyExists = awserr.New("FunctionAlreadyExists", awserr.ErrAlreadyExists) + // ErrFLEAlreadyExists is returned when a field-level-encryption config's CallerReference + // collides with an existing config of a different shape. + ErrFLEAlreadyExists = awserr.New("FieldLevelEncryptionConfigAlreadyExists", awserr.ErrAlreadyExists) + // ErrFLEProfileAlreadyExists is returned when a field-level-encryption profile name is + // already in use. + ErrFLEProfileAlreadyExists = awserr.New("FieldLevelEncryptionProfileAlreadyExists", awserr.ErrAlreadyExists) + // ErrPublicKeyAlreadyExists is returned when a public key name is already in use. + ErrPublicKeyAlreadyExists = awserr.New("PublicKeyAlreadyExists", awserr.ErrAlreadyExists) + // ErrKeyGroupAlreadyExists is returned when a key group name is already in use. + ErrKeyGroupAlreadyExists = awserr.New("KeyGroupAlreadyExists", awserr.ErrAlreadyExists) + // ErrRealtimeLogConfigAlreadyExists is returned when a realtime log config name is + // already in use. + ErrRealtimeLogConfigAlreadyExists = awserr.New("RealtimeLogConfigAlreadyExists", awserr.ErrAlreadyExists) + // ErrCachePolicyInUse is returned when attempting to delete a cache policy that is + // still referenced by a distribution's default or ordered cache behavior. + ErrCachePolicyInUse = awserr.New("CachePolicyInUse", awserr.ErrConflict) + // ErrOriginRequestPolicyInUse is returned when attempting to delete an origin + // request policy that is still referenced by a distribution's cache behavior. + ErrOriginRequestPolicyInUse = awserr.New("OriginRequestPolicyInUse", awserr.ErrConflict) + // ErrResponseHeadersPolicyInUse is returned when attempting to delete a response + // headers policy that is still referenced by a distribution's cache behavior. + ErrResponseHeadersPolicyInUse = awserr.New("ResponseHeadersPolicyInUse", awserr.ErrConflict) + // ErrFunctionInUse is returned when attempting to delete a CloudFront function that + // is still associated with a distribution's cache behavior. + ErrFunctionInUse = awserr.New("FunctionInUse", awserr.ErrConflict) // ErrFLENotFound is returned when a requested field level encryption config does not exist. ErrFLENotFound = awserr.New("NoSuchFieldLevelEncryptionConfig", awserr.ErrNotFound) // ErrFLEProfileNotFound is returned when a requested field level encryption profile does not exist. @@ -197,6 +242,10 @@ var ( // ErrFLEProfileInUse is returned when a field-level-encryption profile is still // referenced by a field-level-encryption config and therefore cannot be deleted. ErrFLEProfileInUse = awserr.New("FieldLevelEncryptionProfileInUse", awserr.ErrConflict) + // ErrInconsistentQuantities is returned when a config payload declares a Quantity + // for a list that does not match the number of Items actually provided. AWS + // validates this pervasively across DistributionConfig and policy configs. + ErrInconsistentQuantities = awserr.New("InconsistentQuantities", awserr.ErrInvalidParameter) ) // ErrPreconditionFailed is returned when an If-Match ETag check fails in a data-plane operation. @@ -248,11 +297,21 @@ type OriginAccessIdentity struct { } // Invalidation represents a CloudFront cache invalidation. +// +// distID and tenantID are unexported identity-only fields used purely to key the +// composite invalidations/tenantInvalidations store.Table (see store_setup.go): +// exactly one is populated on any given Invalidation, depending on whether it +// belongs to a distribution (b.invalidations) or a distribution tenant +// (b.tenantInvalidations). Neither is part of the real AWS wire shape (hence +// json:"-"), and keeping them unexported preserves the exported API surface of +// this pre-existing exported type exactly. type Invalidation struct { CreateTime time.Time `json:"createTime"` ID string `json:"id"` Status string `json:"status"` CallerRef string `json:"callerRef,omitempty"` + distID string `json:"-"` + tenantID string `json:"-"` Paths []string `json:"paths,omitempty"` } @@ -455,13 +514,15 @@ type ResponseHeadersPolicy struct { // Function represents a CloudFront Function. type Function struct { - Name string `json:"name"` - Comment string `json:"comment,omitempty"` - Runtime string `json:"runtime"` - FunctionCode string `json:"functionCode"` - Status string `json:"status"` // DEVELOPMENT or LIVE - ETag string `json:"eTag"` - ARN string `json:"arn"` + Name string `json:"name"` + Comment string `json:"comment,omitempty"` + Runtime string `json:"runtime"` + FunctionCode string `json:"functionCode"` + Status string `json:"status"` // DEVELOPMENT or LIVE + ETag string `json:"eTag"` + ARN string `json:"arn"` + CreatedTime string `json:"createdTime"` + LastModifiedTime string `json:"lastModifiedTime"` } // ORPHeadersConfig controls which request headers are forwarded to the origin. @@ -583,59 +644,74 @@ type VpcOrigin struct { } // InMemoryBackend stores CloudFront resources in memory. +// +// Every map[string]*T resource collection is a *store.Table[T] registered exactly +// once on b.registry (see store_setup.go) instead of a hand-rolled map field, per +// Phase 3.3 of the datalayer refactor (see pkgs/store's package doc and the +// services/ec2 (12e611a4) / services/sqs (0f09d77c) / services/apigateway +// (6da0334e) conversions this follows). store.Table performs no locking of its +// own; b.mu (below) remains the single coarse lock guarding every table exactly +// as it guarded the raw maps it replaces. type InMemoryBackend struct { - distributions map[string]*Distribution + distributions *store.Table[Distribution] distributionARNs map[string]string // ARN → distribution ID (O(1) tag lookups) distributionCallerRefs map[string]string // CallerReference → distribution ID (idempotency) distributionAliases map[string][]string // distribution ID → aliases distributionWebACLs map[string]string // distribution ID → web ACL ID distributionTenantWebACLs map[string]string // tenant ID → web ACL ID - invalidations map[string][]*Invalidation // distribution ID → []Invalidation - oais map[string]*OriginAccessIdentity + invalidations *store.Table[Invalidation] // composite key: distID + "#" + invID + invalidationsByDist *store.Index[Invalidation] + oais *store.Table[OriginAccessIdentity] oaiCallerRefs map[string]string // CallerReference → OAI ID (idempotency) - anycastIPLists map[string]*AnycastIPList + anycastIPLists *store.Table[AnycastIPList] anycastIPListARNs map[string]string // ARN → anycast IP list ID (tag lookups) anycastIPListByName map[string]string // name → anycast IP list ID (uniqueness) - cachePolicies map[string]*CachePolicy + cachePolicies *store.Table[CachePolicy] cachePolicyByName map[string]string // name → policy ID (uniqueness) - connectionFunctions map[string]*ConnectionFunction + connectionFunctions *store.Table[ConnectionFunction] connectionFunctionARNs map[string]string // ARN → connection function ID (tag lookups) connectionFunctionByName map[string]string // name → connection function ID (uniqueness / Identifier lookups) - connectionGroups map[string]*ConnectionGroup + connectionGroups *store.Table[ConnectionGroup] connectionGroupARNs map[string]string // ARN → connection group ID (tag lookups) connectionGroupByName map[string]string // name → connection group ID (uniqueness) connectionGroupByRoutingEndpoint map[string]string // routing endpoint → connection group ID - continuousDeploymentPolicies map[string]*ContinuousDeploymentPolicy - originAccessControls map[string]*OriginAccessControl + continuousDeploymentPolicies *store.Table[ContinuousDeploymentPolicy] + originAccessControls *store.Table[OriginAccessControl] originAccessControlByName map[string]string // name → OAC ID (uniqueness) - responseHeadersPolicies map[string]*ResponseHeadersPolicy - responseHeadersPolicyByName map[string]string // name → policy ID (uniqueness) - functions map[string]*Function // name → function - originRequestPolicies map[string]*OriginRequestPolicy + responseHeadersPolicies *store.Table[ResponseHeadersPolicy] + responseHeadersPolicyByName map[string]string // name → policy ID (uniqueness) + functions *store.Table[Function] // name → function + originRequestPolicies *store.Table[OriginRequestPolicy] originRequestPolicyByName map[string]string // name → policy ID (uniqueness) - fieldLevelEncryptions map[string]*FieldLevelEncryption + fieldLevelEncryptions *store.Table[FieldLevelEncryption] fieldLevelEncryptionByName map[string]string // name → ID - fieldLevelEncryptionProfiles map[string]*FieldLevelEncryptionProfile + fieldLevelEncryptionProfiles *store.Table[FieldLevelEncryptionProfile] fieldLevelEncryptionProfileByName map[string]string // name → ID - publicKeys map[string]*PublicKey + publicKeys *store.Table[PublicKey] publicKeyByName map[string]string // name → ID - keyGroups map[string]*KeyGroup - keyGroupByName map[string]string // name → ID - realtimeLogConfigs map[string]*RealtimeLogConfig // ARN → config - realtimeLogConfigByName map[string]string // name → ARN - keyValueStores map[string]*KeyValueStore + keyGroups *store.Table[KeyGroup] + keyGroupByName map[string]string // name → ID + realtimeLogConfigs *store.Table[RealtimeLogConfig] // ARN → config + realtimeLogConfigByName map[string]string // name → ARN + keyValueStores *store.Table[KeyValueStore] keyValueStoreByName map[string]string // name → ID - vpcOrigins map[string]*VpcOrigin + vpcOrigins *store.Table[VpcOrigin] distributionFunctionAssociations map[string][]FunctionAssociation // distribution ID → associations // Batch 1 additions. - trustStores map[string]*TrustStore + trustStores *store.Table[TrustStore] trustStoreARNs map[string]string // ARN → trust store ID (tag lookups) trustStoreByName map[string]string // name → trust store ID (uniqueness) - streamingDistributions map[string]*StreamingDistribution - streamingDistributionARNs map[string]string // ARN → streaming dist ID (tag lookups) - streamingDistributionCallerRefs map[string]string // CallerRef → streaming dist ID (idempotency) - monitoringSubscriptions map[string]*MonitoringSubscription // distribution ID → subscription - resourcePolicies map[string]*resourcePolicyEntry // resource ARN → policy + streamingDistributions *store.Table[StreamingDistribution] + streamingDistributionARNs map[string]string // ARN → streaming dist ID (tag lookups) + streamingDistributionCallerRefs map[string]string // CallerRef → streaming dist ID (idempotency) + // monitoringSubscriptions, resourcePolicies, and managedCertificates are deliberately + // plain maps, not store.Table: MonitoringSubscription/resourcePolicyEntry/ + // ManagedCertificateDetails carry no identity field of their own (the map key - + // distribution ID / resource ARN / tenant ID - is not stored on the value), the same + // "no identity field" exception documented for EC2's instanceIMDSOptions (commit + // 12e611a4). See store_setup.go's registerAllTables doc for the full list of exceptions. + monitoringSubscriptions map[string]*MonitoringSubscription // distribution ID → subscription + resourcePolicies map[string]*resourcePolicyEntry // resource ARN → policy // managedCertificates maps distribution tenant ID → cached managed cert details. managedCertificates map[string]*ManagedCertificateDetails // distributionCachePolicies, distributionOriginRequestPolicies, @@ -648,10 +724,11 @@ type InMemoryBackend struct { distributionResponseHeadersPolicies map[string]string distributionRealtimeLogConfigs map[string]string // Batch 2 additions. - distributionTenants map[string]*DistributionTenant // key: tenant ID - distributionTenantARNs map[string]string // ARN → tenant ID (tag lookups) - distributionTenantsByDomain map[string]string // key: domain → tenant ID - tenantInvalidations map[string][]*Invalidation // key: tenantID + distributionTenants *store.Table[DistributionTenant] // key: tenant ID + distributionTenantARNs map[string]string // ARN → tenant ID (tag lookups) + distributionTenantsByDomain map[string]string // key: domain → tenant ID + tenantInvalidations *store.Table[Invalidation] // composite key: tenantID + "#" + invID + tenantInvalidationsByTenant *store.Index[Invalidation] // Audit batch additions. keyValueStoreData map[string]map[string]string // KVS ID → key → value keyValueDataETags map[string]string // KVS ID → current data-plane ETag @@ -661,7 +738,12 @@ type InMemoryBackend struct { // and eliminate the substring false positives of the previous raw scan. distSearchTokens map[string]map[string]struct{} distSearchInverted map[string]map[string]struct{} - mu *lockmetrics.RWMutex + // registry holds every "clean" store.Table above (see store_setup.go) so + // Reset/Snapshot/Restore collapse to one registry call each. The "dirty" + // invalidations/tenantInvalidations tables are NOT on registry -- see + // store_setup.go's registerAllTables doc. + registry *store.Registry + mu *lockmetrics.RWMutex // lifecycle: tracks when InProgress invalidations become Completed. invalidationReadyAt map[string]map[string]time.Time // distributionID → invID → readyAt tenantInvalidationReadyAt map[string]map[string]time.Time // tenantID → invID → readyAt @@ -673,55 +755,34 @@ type InMemoryBackend struct { // NewInMemoryBackend creates a new in-memory CloudFront backend. func NewInMemoryBackend(accountID, region string) *InMemoryBackend { b := &InMemoryBackend{ - distributions: make(map[string]*Distribution), distributionARNs: make(map[string]string), distributionCallerRefs: make(map[string]string), distributionAliases: make(map[string][]string), distributionWebACLs: make(map[string]string), distributionTenantWebACLs: make(map[string]string), - invalidations: make(map[string][]*Invalidation), - oais: make(map[string]*OriginAccessIdentity), oaiCallerRefs: make(map[string]string), - anycastIPLists: make(map[string]*AnycastIPList), anycastIPListARNs: make(map[string]string), anycastIPListByName: make(map[string]string), - cachePolicies: make(map[string]*CachePolicy), cachePolicyByName: make(map[string]string), - connectionFunctions: make(map[string]*ConnectionFunction), connectionFunctionARNs: make(map[string]string), connectionFunctionByName: make(map[string]string), - connectionGroups: make(map[string]*ConnectionGroup), connectionGroupARNs: make(map[string]string), connectionGroupByName: make(map[string]string), connectionGroupByRoutingEndpoint: make(map[string]string), - continuousDeploymentPolicies: make(map[string]*ContinuousDeploymentPolicy), - originAccessControls: make(map[string]*OriginAccessControl), originAccessControlByName: make(map[string]string), - responseHeadersPolicies: make(map[string]*ResponseHeadersPolicy), responseHeadersPolicyByName: make(map[string]string), - functions: make(map[string]*Function), - originRequestPolicies: make(map[string]*OriginRequestPolicy), originRequestPolicyByName: make(map[string]string), - fieldLevelEncryptions: make(map[string]*FieldLevelEncryption), fieldLevelEncryptionByName: make(map[string]string), - fieldLevelEncryptionProfiles: make(map[string]*FieldLevelEncryptionProfile), fieldLevelEncryptionProfileByName: make(map[string]string), - publicKeys: make(map[string]*PublicKey), publicKeyByName: make(map[string]string), - keyGroups: make(map[string]*KeyGroup), keyGroupByName: make(map[string]string), - realtimeLogConfigs: make(map[string]*RealtimeLogConfig), realtimeLogConfigByName: make(map[string]string), - keyValueStores: make(map[string]*KeyValueStore), keyValueStoreByName: make(map[string]string), - vpcOrigins: make(map[string]*VpcOrigin), distSearchTokens: make(map[string]map[string]struct{}), distSearchInverted: make(map[string]map[string]struct{}), distributionFunctionAssociations: make(map[string][]FunctionAssociation), - trustStores: make(map[string]*TrustStore), trustStoreARNs: make(map[string]string), trustStoreByName: make(map[string]string), - streamingDistributions: make(map[string]*StreamingDistribution), streamingDistributionARNs: make(map[string]string), streamingDistributionCallerRefs: make(map[string]string), monitoringSubscriptions: make(map[string]*MonitoringSubscription), @@ -731,20 +792,21 @@ func NewInMemoryBackend(accountID, region string) *InMemoryBackend { distributionOriginRequestPolicies: make(map[string]string), distributionResponseHeadersPolicies: make(map[string]string), distributionRealtimeLogConfigs: make(map[string]string), - distributionTenants: make(map[string]*DistributionTenant), distributionTenantARNs: make(map[string]string), distributionTenantsByDomain: make(map[string]string), - tenantInvalidations: make(map[string][]*Invalidation), keyValueStoreData: make(map[string]map[string]string), keyValueDataETags: make(map[string]string), invalidationReadyAt: make(map[string]map[string]time.Time), tenantInvalidationReadyAt: make(map[string]map[string]time.Time), stopCh: make(chan struct{}), + registry: store.NewRegistry(), mu: lockmetrics.New("cloudfront"), accountID: accountID, region: region, } + registerAllTables(b) + go b.runInvalidationReconciler() return b @@ -783,11 +845,11 @@ func (b *InMemoryBackend) reconcileInvalidationsLocked() { now := time.Now() for distID, invMap := range b.invalidationReadyAt { - reconcileInvMap(invMap, b.invalidations[distID], now) + reconcileInvMap(invMap, b.invalidationsByDist.Get(distID), now) } for tenantID, invMap := range b.tenantInvalidationReadyAt { - reconcileInvMap(invMap, b.tenantInvalidations[tenantID], now) + reconcileInvMap(invMap, b.tenantInvalidationsByTenant.Get(tenantID), now) } } @@ -813,13 +875,18 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() + b.registry.ResetAll() + // invalidations/tenantInvalidations are "dirty" tables, not on b.registry (see + // store_setup.go's registerAllTables doc), so they need an explicit Reset call. + b.invalidations.Reset() + b.tenantInvalidations.Reset() + b.resetDistributions() b.resetPoliciesAndKeys() } -// resetDistributions clears distribution-related maps. +// resetDistributions clears distribution-related maps not covered by b.registry.ResetAll(). func (b *InMemoryBackend) resetDistributions() { - b.distributions = make(map[string]*Distribution) b.distSearchTokens = make(map[string]map[string]struct{}) b.distSearchInverted = make(map[string]map[string]struct{}) b.distributionARNs = make(map[string]string) @@ -827,59 +894,38 @@ func (b *InMemoryBackend) resetDistributions() { b.distributionAliases = make(map[string][]string) b.distributionWebACLs = make(map[string]string) b.distributionTenantWebACLs = make(map[string]string) - b.invalidations = make(map[string][]*Invalidation) - b.oais = make(map[string]*OriginAccessIdentity) b.oaiCallerRefs = make(map[string]string) - b.anycastIPLists = make(map[string]*AnycastIPList) b.anycastIPListARNs = make(map[string]string) b.anycastIPListByName = make(map[string]string) - b.cachePolicies = make(map[string]*CachePolicy) b.cachePolicyByName = make(map[string]string) - b.connectionFunctions = make(map[string]*ConnectionFunction) b.connectionFunctionARNs = make(map[string]string) b.connectionFunctionByName = make(map[string]string) - b.connectionGroups = make(map[string]*ConnectionGroup) b.connectionGroupARNs = make(map[string]string) b.connectionGroupByName = make(map[string]string) b.connectionGroupByRoutingEndpoint = make(map[string]string) - b.continuousDeploymentPolicies = make(map[string]*ContinuousDeploymentPolicy) - b.originAccessControls = make(map[string]*OriginAccessControl) b.originAccessControlByName = make(map[string]string) - b.responseHeadersPolicies = make(map[string]*ResponseHeadersPolicy) b.responseHeadersPolicyByName = make(map[string]string) - b.functions = make(map[string]*Function) - b.originRequestPolicies = make(map[string]*OriginRequestPolicy) b.originRequestPolicyByName = make(map[string]string) b.distributionFunctionAssociations = make(map[string][]FunctionAssociation) b.distributionCachePolicies = make(map[string]string) b.distributionOriginRequestPolicies = make(map[string]string) b.distributionResponseHeadersPolicies = make(map[string]string) b.distributionRealtimeLogConfigs = make(map[string]string) - b.distributionTenants = make(map[string]*DistributionTenant) b.distributionTenantARNs = make(map[string]string) b.distributionTenantsByDomain = make(map[string]string) - b.tenantInvalidations = make(map[string][]*Invalidation) } -// resetPoliciesAndKeys clears encryption, key, and store maps. +// resetPoliciesAndKeys clears encryption, key, and store maps not covered by +// b.registry.ResetAll(). func (b *InMemoryBackend) resetPoliciesAndKeys() { - b.fieldLevelEncryptions = make(map[string]*FieldLevelEncryption) b.fieldLevelEncryptionByName = make(map[string]string) - b.fieldLevelEncryptionProfiles = make(map[string]*FieldLevelEncryptionProfile) b.fieldLevelEncryptionProfileByName = make(map[string]string) - b.publicKeys = make(map[string]*PublicKey) b.publicKeyByName = make(map[string]string) - b.keyGroups = make(map[string]*KeyGroup) b.keyGroupByName = make(map[string]string) - b.realtimeLogConfigs = make(map[string]*RealtimeLogConfig) b.realtimeLogConfigByName = make(map[string]string) - b.keyValueStores = make(map[string]*KeyValueStore) b.keyValueStoreByName = make(map[string]string) - b.vpcOrigins = make(map[string]*VpcOrigin) - b.trustStores = make(map[string]*TrustStore) b.trustStoreARNs = make(map[string]string) b.trustStoreByName = make(map[string]string) - b.streamingDistributions = make(map[string]*StreamingDistribution) b.streamingDistributionARNs = make(map[string]string) b.streamingDistributionCallerRefs = make(map[string]string) b.monitoringSubscriptions = make(map[string]*MonitoringSubscription) @@ -945,7 +991,9 @@ func (b *InMemoryBackend) CreateDistribution( // Idempotency: return existing distribution for the same CallerReference. if existingID, ok := b.distributionCallerRefs[callerRef]; ok { - return b.copyDistribution(b.distributions[existingID]), nil + existing, _ := b.distributions.Get(existingID) + + return b.copyDistribution(existing), nil } id := generateID() @@ -962,7 +1010,7 @@ func (b *InMemoryBackend) CreateDistribution( LastModifiedTime: time.Now().UTC().Format(time.RFC3339), Tags: make(map[string]string), } - b.distributions[id] = d + b.distributions.Put(d) b.distributionARNs[d.ARN] = id b.distributionCallerRefs[callerRef] = id b.indexDistributionConfig(id, rawConfig) @@ -976,7 +1024,7 @@ func (b *InMemoryBackend) GetDistribution(id string) (*Distribution, error) { b.mu.RLock("GetDistribution") defer b.mu.RUnlock() - d, ok := b.distributions[id] + d, ok := b.distributions.Get(id) if !ok { return nil, fmt.Errorf("%w: distribution %s not found", ErrNotFound, id) } @@ -993,7 +1041,7 @@ func (b *InMemoryBackend) UpdateDistribution( b.mu.Lock("UpdateDistribution") defer b.mu.Unlock() - d, ok := b.distributions[id] + d, ok := b.distributions.Get(id) if !ok { return nil, fmt.Errorf("%w: distribution %s not found", ErrNotFound, id) } @@ -1015,7 +1063,7 @@ func (b *InMemoryBackend) DeleteDistribution(id string) error { b.mu.Lock("DeleteDistribution") defer b.mu.Unlock() - d, ok := b.distributions[id] + d, ok := b.distributions.Get(id) if !ok { return fmt.Errorf("%w: distribution %s not found", ErrNotFound, id) } @@ -1026,8 +1074,8 @@ func (b *InMemoryBackend) DeleteDistribution(id string) error { delete(b.distributionARNs, b.distributionARN(id)) delete(b.distributionCallerRefs, d.CallerReference) - delete(b.distributions, id) - delete(b.invalidations, id) + b.distributions.Delete(id) + b.deleteInvalidationsForDist(id) delete(b.distributionAliases, id) delete(b.distributionWebACLs, id) b.deindexDistributionConfig(id) @@ -1040,8 +1088,8 @@ func (b *InMemoryBackend) ListDistributions() []*Distribution { b.mu.RLock("ListDistributions") defer b.mu.RUnlock() - list := make([]*Distribution, 0, len(b.distributions)) - for _, d := range b.distributions { + list := make([]*Distribution, 0, b.distributions.Len()) + for _, d := range b.distributions.All() { list = append(list, b.copyDistribution(d)) } @@ -1063,7 +1111,8 @@ func (b *InMemoryBackend) CreateOAI(callerRef, comment string) (*OriginAccessIde // Idempotency: return existing OAI for the same CallerReference. if existingID, ok := b.oaiCallerRefs[callerRef]; ok { - cp := *b.oais[existingID] + existing, _ := b.oais.Get(existingID) + cp := *existing return &cp, nil } @@ -1077,7 +1126,7 @@ func (b *InMemoryBackend) CreateOAI(callerRef, comment string) (*OriginAccessIde CallerReference: callerRef, Comment: comment, } - b.oais[id] = oai + b.oais.Put(oai) b.oaiCallerRefs[callerRef] = id cp := *oai @@ -1089,7 +1138,7 @@ func (b *InMemoryBackend) GetOAI(id string) (*OriginAccessIdentity, error) { b.mu.RLock("GetCloudFrontOriginAccessIdentity") defer b.mu.RUnlock() - oai, ok := b.oais[id] + oai, ok := b.oais.Get(id) if !ok { return nil, fmt.Errorf("%w: OAI %s not found", ErrOAINotFound, id) } @@ -1103,13 +1152,13 @@ func (b *InMemoryBackend) DeleteOAI(id string) error { b.mu.Lock("DeleteCloudFrontOriginAccessIdentity") defer b.mu.Unlock() - oai, ok := b.oais[id] + oai, ok := b.oais.Get(id) if !ok { return fmt.Errorf("%w: OAI %s not found", ErrOAINotFound, id) } delete(b.oaiCallerRefs, oai.CallerReference) - delete(b.oais, id) + b.oais.Delete(id) return nil } @@ -1119,7 +1168,7 @@ func (b *InMemoryBackend) UpdateOAI(id, comment string) (*OriginAccessIdentity, b.mu.Lock("UpdateCloudFrontOriginAccessIdentity") defer b.mu.Unlock() - oai, ok := b.oais[id] + oai, ok := b.oais.Get(id) if !ok { return nil, fmt.Errorf("%w: OAI %s not found", ErrOAINotFound, id) } @@ -1137,8 +1186,8 @@ func (b *InMemoryBackend) ListOAIs() []*OriginAccessIdentity { b.mu.RLock("ListCloudFrontOriginAccessIdentities") defer b.mu.RUnlock() - list := make([]*OriginAccessIdentity, 0, len(b.oais)) - for _, oai := range b.oais { + list := make([]*OriginAccessIdentity, 0, b.oais.Len()) + for _, oai := range b.oais.All() { cp := *oai list = append(list, &cp) } @@ -1153,31 +1202,45 @@ func (b *InMemoryBackend) ListOAIs() []*OriginAccessIdentity { // Must be called with the lock held. func (b *InMemoryBackend) taggableTags(resourceARN string) (*map[string]string, bool) { if id, ok := b.distributionARNs[resourceARN]; ok { - return &b.distributions[id].Tags, true + d, _ := b.distributions.Get(id) + + return &d.Tags, true } if id, ok := b.streamingDistributionARNs[resourceARN]; ok { - return &b.streamingDistributions[id].Tags, true + sd, _ := b.streamingDistributions.Get(id) + + return &sd.Tags, true } if id, ok := b.trustStoreARNs[resourceARN]; ok { - return &b.trustStores[id].Tags, true + ts, _ := b.trustStores.Get(id) + + return &ts.Tags, true } if id, ok := b.distributionTenantARNs[resourceARN]; ok { - return &b.distributionTenants[id].Tags, true + t, _ := b.distributionTenants.Get(id) + + return &t.Tags, true } if id, ok := b.connectionGroupARNs[resourceARN]; ok { - return &b.connectionGroups[id].Tags, true + cg, _ := b.connectionGroups.Get(id) + + return &cg.Tags, true } if id, ok := b.connectionFunctionARNs[resourceARN]; ok { - return &b.connectionFunctions[id].Tags, true + fn, _ := b.connectionFunctions.Get(id) + + return &fn.Tags, true } if id, ok := b.anycastIPListARNs[resourceARN]; ok { - return &b.anycastIPLists[id].Tags, true + list, _ := b.anycastIPLists.Get(id) + + return &list.Tags, true } return nil, false @@ -1255,7 +1318,7 @@ func (b *InMemoryBackend) CountInProgressInvalidations(distributionID string) in defer b.mu.RUnlock() count := 0 - for _, inv := range b.invalidations[distributionID] { + for _, inv := range b.invalidationsByDist.Get(distributionID) { if inv.Status == statusInProgress { count++ } @@ -1280,7 +1343,7 @@ func (b *InMemoryBackend) CreateInvalidation( b.mu.Lock("CreateInvalidation") defer b.mu.Unlock() - if _, ok := b.distributions[distributionID]; !ok { + if _, ok := b.distributions.Get(distributionID); !ok { return nil, fmt.Errorf("%w: distribution %s not found", ErrNotFound, distributionID) } @@ -1293,8 +1356,9 @@ func (b *InMemoryBackend) CreateInvalidation( CreateTime: now, Paths: append([]string(nil), paths...), CallerRef: callerRef, + distID: distributionID, } - b.invalidations[distributionID] = append(b.invalidations[distributionID], inv) + b.invalidations.Put(inv) if b.invalidationReadyAt[distributionID] == nil { b.invalidationReadyAt[distributionID] = make(map[string]time.Time) @@ -1313,11 +1377,11 @@ func (b *InMemoryBackend) ListInvalidations(distributionID string) ([]*Invalidat b.mu.RLock("ListInvalidations") defer b.mu.RUnlock() - if _, ok := b.distributions[distributionID]; !ok { + if _, ok := b.distributions.Get(distributionID); !ok { return nil, fmt.Errorf("%w: distribution %s not found", ErrNotFound, distributionID) } - src := b.invalidations[distributionID] + src := b.invalidationsByDist.Get(distributionID) out := make([]*Invalidation, 0, len(src)) for _, inv := range src { @@ -1338,20 +1402,19 @@ func (b *InMemoryBackend) GetInvalidation( b.mu.RLock("GetInvalidation") defer b.mu.RUnlock() - if _, ok := b.distributions[distributionID]; !ok { + if _, ok := b.distributions.Get(distributionID); !ok { return nil, fmt.Errorf("%w: distribution %s not found", ErrNotFound, distributionID) } - for _, inv := range b.invalidations[distributionID] { - if inv.ID == invalidationID { - cp := *inv - cp.Paths = append([]string(nil), inv.Paths...) - - return &cp, nil - } + inv, ok := b.invalidations.Get(invalidationKey(distributionID, invalidationID)) + if !ok { + return nil, fmt.Errorf("%w: invalidation %s not found", ErrInvalidationNotFound, invalidationID) } - return nil, fmt.Errorf("%w: invalidation %s not found", ErrInvalidationNotFound, invalidationID) + cp := *inv + cp.Paths = append([]string(nil), inv.Paths...) + + return &cp, nil } // ListAliases returns the aliases for a distribution by ID. @@ -1375,7 +1438,7 @@ func (b *InMemoryBackend) AssociateAlias(distributionID, alias string) error { b.mu.Lock("AssociateAlias") defer b.mu.Unlock() - if _, ok := b.distributions[distributionID]; !ok { + if _, ok := b.distributions.Get(distributionID); !ok { return fmt.Errorf("%w: distribution %s not found", ErrNotFound, distributionID) } @@ -1398,7 +1461,7 @@ func (b *InMemoryBackend) AssociateDistributionWebACL(distributionID, webACLID s b.mu.Lock("AssociateDistributionWebACL") defer b.mu.Unlock() - if _, ok := b.distributions[distributionID]; !ok { + if _, ok := b.distributions.Get(distributionID); !ok { return fmt.Errorf("%w: distribution %s not found", ErrNotFound, distributionID) } @@ -1426,7 +1489,7 @@ func (b *InMemoryBackend) CopyDistribution(primaryDistID, callerRef string) (*Di b.mu.Lock("CopyDistribution") defer b.mu.Unlock() - src, ok := b.distributions[primaryDistID] + src, ok := b.distributions.Get(primaryDistID) if !ok { return nil, fmt.Errorf("%w: distribution %s not found", ErrNotFound, primaryDistID) } @@ -1453,7 +1516,7 @@ func (b *InMemoryBackend) CopyDistribution(primaryDistID, callerRef string) (*Di Tags: make(map[string]string), } - b.distributions[id] = d + b.distributions.Put(d) b.distributionARNs[d.ARN] = id b.indexDistributionConfig(id, rawCopy) @@ -1515,7 +1578,7 @@ func (b *InMemoryBackend) CreateAnycastIPList( if len(tags) > 0 { maps.Copy(list.Tags, tags[0]) } - b.anycastIPLists[id] = list + b.anycastIPLists.Put(list) b.anycastIPListARNs[list.ARN] = id b.anycastIPListByName[name] = id @@ -1555,7 +1618,7 @@ func (b *InMemoryBackend) CreateCachePolicy( if _, exists := b.cachePolicyByName[name]; exists { return nil, fmt.Errorf( "%w: cache policy with name %q already exists", - ErrAlreadyExists, + ErrCachePolicyAlreadyExists, name, ) } @@ -1576,7 +1639,7 @@ func (b *InMemoryBackend) CreateCachePolicy( MinTTL: minTTL, Params: p, } - b.cachePolicies[id] = policy + b.cachePolicies.Put(policy) b.cachePolicyByName[name] = id cp := *policy @@ -1634,7 +1697,7 @@ func (b *InMemoryBackend) CreateConnectionFunctionWithCode( Tags: make(map[string]string, len(tags)), } maps.Copy(fn.Tags, tags) - b.connectionFunctions[id] = fn + b.connectionFunctions.Put(fn) b.connectionFunctionARNs[fn.ARN] = id b.connectionFunctionByName[name] = id @@ -1692,7 +1755,7 @@ func (b *InMemoryBackend) CreateConnectionGroupWithConfig( Tags: make(map[string]string, len(tags)), } maps.Copy(group.Tags, tags) - b.connectionGroups[id] = group + b.connectionGroups.Put(group) b.connectionGroupARNs[group.ARN] = id b.connectionGroupByName[name] = id b.connectionGroupByRoutingEndpoint[group.RoutingEndpoint] = id @@ -1753,7 +1816,7 @@ func (b *InMemoryBackend) CreateContinuousDeploymentPolicyWithConfig( StagingDistributionDNSNames: append([]string(nil), stagingDNSNames...), TrafficConfig: traffic, } - b.continuousDeploymentPolicies[id] = policy + b.continuousDeploymentPolicies.Put(policy) return b.copyContinuousDeploymentPolicy(policy), nil } @@ -1786,7 +1849,7 @@ func (b *InMemoryBackend) GetContinuousDeploymentPolicy(id string) (*ContinuousD b.mu.RLock("GetContinuousDeploymentPolicy") defer b.mu.RUnlock() - policy, ok := b.continuousDeploymentPolicies[id] + policy, ok := b.continuousDeploymentPolicies.Get(id) if !ok { return nil, fmt.Errorf( "%w: continuous deployment policy %s not found", @@ -1803,8 +1866,8 @@ func (b *InMemoryBackend) ListContinuousDeploymentPolicies() []*ContinuousDeploy b.mu.RLock("ListContinuousDeploymentPolicies") defer b.mu.RUnlock() - list := make([]*ContinuousDeploymentPolicy, 0, len(b.continuousDeploymentPolicies)) - for _, policy := range b.continuousDeploymentPolicies { + list := make([]*ContinuousDeploymentPolicy, 0, b.continuousDeploymentPolicies.Len()) + for _, policy := range b.continuousDeploymentPolicies.All() { list = append(list, b.copyContinuousDeploymentPolicy(policy)) } @@ -1825,7 +1888,7 @@ func (b *InMemoryBackend) UpdateContinuousDeploymentPolicy( b.mu.Lock("UpdateContinuousDeploymentPolicy") defer b.mu.Unlock() - policy, ok := b.continuousDeploymentPolicies[id] + policy, ok := b.continuousDeploymentPolicies.Get(id) if !ok { return nil, fmt.Errorf( "%w: continuous deployment policy %s not found", @@ -1856,7 +1919,7 @@ func (b *InMemoryBackend) UpdateContinuousDeploymentPolicyWithConfig( b.mu.Lock("UpdateContinuousDeploymentPolicy") defer b.mu.Unlock() - policy, ok := b.continuousDeploymentPolicies[id] + policy, ok := b.continuousDeploymentPolicies.Get(id) if !ok { return nil, fmt.Errorf( "%w: continuous deployment policy %s not found", @@ -1885,11 +1948,11 @@ func (b *InMemoryBackend) DeleteContinuousDeploymentPolicy(id string) error { b.mu.Lock("DeleteContinuousDeploymentPolicy") defer b.mu.Unlock() - if _, ok := b.continuousDeploymentPolicies[id]; !ok { + if _, ok := b.continuousDeploymentPolicies.Get(id); !ok { return fmt.Errorf("%w: continuous deployment policy %s not found", ErrContinuousDeploymentPolicyNotFound, id) } - delete(b.continuousDeploymentPolicies, id) + b.continuousDeploymentPolicies.Delete(id) return nil } @@ -1902,7 +1965,7 @@ func (b *InMemoryBackend) SetDistributionFunctionAssociations( b.mu.Lock("SetDistributionFunctionAssociations") defer b.mu.Unlock() - if _, ok := b.distributions[distributionID]; !ok { + if _, ok := b.distributions.Get(distributionID); !ok { return fmt.Errorf("%w: distribution %s not found", ErrNotFound, distributionID) } @@ -1918,7 +1981,7 @@ func (b *InMemoryBackend) GetDistributionFunctionAssociations(distributionID str b.mu.RLock("GetDistributionFunctionAssociations") defer b.mu.RUnlock() - if _, ok := b.distributions[distributionID]; !ok { + if _, ok := b.distributions.Get(distributionID); !ok { return nil, fmt.Errorf("%w: distribution %s not found", ErrNotFound, distributionID) } @@ -1949,7 +2012,7 @@ func (b *InMemoryBackend) GetCachePolicy(id string) (*CachePolicy, error) { b.mu.RLock("GetCachePolicy") defer b.mu.RUnlock() - p, ok := b.cachePolicies[id] + p, ok := b.cachePolicies.Get(id) if !ok { return nil, fmt.Errorf("%w: cache policy %s not found", ErrCachePolicyNotFound, id) } @@ -1964,8 +2027,8 @@ func (b *InMemoryBackend) ListCachePolicies() []*CachePolicy { b.mu.RLock("ListCachePolicies") defer b.mu.RUnlock() - list := make([]*CachePolicy, 0, len(b.cachePolicies)) - for _, p := range b.cachePolicies { + list := make([]*CachePolicy, 0, b.cachePolicies.Len()) + for _, p := range b.cachePolicies.All() { cp := *p list = append(list, &cp) } @@ -1984,7 +2047,7 @@ func (b *InMemoryBackend) UpdateCachePolicy( b.mu.Lock("UpdateCachePolicy") defer b.mu.Unlock() - p, ok := b.cachePolicies[id] + p, ok := b.cachePolicies.Get(id) if !ok { return nil, fmt.Errorf("%w: cache policy %s not found", ErrCachePolicyNotFound, id) } @@ -2014,7 +2077,7 @@ func (b *InMemoryBackend) UpdateCachePolicy( if _, exists := b.cachePolicyByName[name]; exists { return nil, fmt.Errorf( "%w: cache policy with name %q already exists", - ErrAlreadyExists, + ErrCachePolicyAlreadyExists, name, ) } @@ -2043,13 +2106,17 @@ func (b *InMemoryBackend) DeleteCachePolicy(id string) error { b.mu.Lock("DeleteCachePolicy") defer b.mu.Unlock() - p, ok := b.cachePolicies[id] + p, ok := b.cachePolicies.Get(id) if !ok { return fmt.Errorf("%w: cache policy %s not found", ErrCachePolicyNotFound, id) } + if b.tokenReferencedByAnyDistribution(id) { + return fmt.Errorf("%w: cache policy %s is attached to a distribution", ErrCachePolicyInUse, id) + } + delete(b.cachePolicyByName, p.Name) - delete(b.cachePolicies, id) + b.cachePolicies.Delete(id) return nil } @@ -2070,7 +2137,7 @@ func (b *InMemoryBackend) CreateOriginAccessControl( if _, exists := b.originAccessControlByName[name]; exists { return nil, fmt.Errorf( "%w: origin access control with name %q already exists", - ErrAlreadyExists, + ErrOriginAccessControlAlreadyExists, name, ) } @@ -2085,7 +2152,7 @@ func (b *InMemoryBackend) CreateOriginAccessControl( SigningProtocol: signingProtocol, ETag: uuid.NewString(), } - b.originAccessControls[id] = oac + b.originAccessControls.Put(oac) b.originAccessControlByName[name] = id cp := *oac @@ -2097,7 +2164,7 @@ func (b *InMemoryBackend) GetOriginAccessControl(id string) (*OriginAccessContro b.mu.RLock("GetOriginAccessControl") defer b.mu.RUnlock() - oac, ok := b.originAccessControls[id] + oac, ok := b.originAccessControls.Get(id) if !ok { return nil, fmt.Errorf("%w: origin access control %s not found", ErrOACNotFound, id) } @@ -2112,8 +2179,8 @@ func (b *InMemoryBackend) ListOriginAccessControls() []*OriginAccessControl { b.mu.RLock("ListOriginAccessControls") defer b.mu.RUnlock() - list := make([]*OriginAccessControl, 0, len(b.originAccessControls)) - for _, oac := range b.originAccessControls { + list := make([]*OriginAccessControl, 0, b.originAccessControls.Len()) + for _, oac := range b.originAccessControls.All() { cp := *oac list = append(list, &cp) } @@ -2130,7 +2197,7 @@ func (b *InMemoryBackend) UpdateOriginAccessControl( b.mu.Lock("UpdateOriginAccessControl") defer b.mu.Unlock() - oac, ok := b.originAccessControls[id] + oac, ok := b.originAccessControls.Get(id) if !ok { return nil, fmt.Errorf("%w: origin access control %s not found", ErrOACNotFound, id) } @@ -2143,7 +2210,7 @@ func (b *InMemoryBackend) UpdateOriginAccessControl( if _, exists := b.originAccessControlByName[name]; exists { return nil, fmt.Errorf( "%w: origin access control with name %q already exists", - ErrAlreadyExists, + ErrOriginAccessControlAlreadyExists, name, ) } @@ -2168,13 +2235,13 @@ func (b *InMemoryBackend) DeleteOriginAccessControl(id string) error { b.mu.Lock("DeleteOriginAccessControl") defer b.mu.Unlock() - oac, ok := b.originAccessControls[id] + oac, ok := b.originAccessControls.Get(id) if !ok { return fmt.Errorf("%w: origin access control %s not found", ErrOACNotFound, id) } delete(b.originAccessControlByName, oac.Name) - delete(b.originAccessControls, id) + b.originAccessControls.Delete(id) return nil } @@ -2196,7 +2263,7 @@ func (b *InMemoryBackend) CreateResponseHeadersPolicy( if _, exists := b.responseHeadersPolicyByName[name]; exists { return nil, fmt.Errorf( "%w: response headers policy with name %q already exists", - ErrAlreadyExists, + ErrResponseHeadersPolicyAlreadyExists, name, ) } @@ -2217,7 +2284,7 @@ func (b *InMemoryBackend) CreateResponseHeadersPolicy( p.RemoveHeaders = cfg.RemoveHeaders } - b.responseHeadersPolicies[id] = p + b.responseHeadersPolicies.Put(p) b.responseHeadersPolicyByName[name] = id cp := *p @@ -2229,7 +2296,7 @@ func (b *InMemoryBackend) GetResponseHeadersPolicy(id string) (*ResponseHeadersP b.mu.RLock("GetResponseHeadersPolicy") defer b.mu.RUnlock() - p, ok := b.responseHeadersPolicies[id] + p, ok := b.responseHeadersPolicies.Get(id) if !ok { return nil, fmt.Errorf( "%w: response headers policy %s not found", @@ -2248,8 +2315,8 @@ func (b *InMemoryBackend) ListResponseHeadersPolicies() []*ResponseHeadersPolicy b.mu.RLock("ListResponseHeadersPolicies") defer b.mu.RUnlock() - list := make([]*ResponseHeadersPolicy, 0, len(b.responseHeadersPolicies)) - for _, p := range b.responseHeadersPolicies { + list := make([]*ResponseHeadersPolicy, 0, b.responseHeadersPolicies.Len()) + for _, p := range b.responseHeadersPolicies.All() { cp := *p list = append(list, &cp) } @@ -2267,7 +2334,7 @@ func (b *InMemoryBackend) UpdateResponseHeadersPolicy( b.mu.Lock("UpdateResponseHeadersPolicy") defer b.mu.Unlock() - p, ok := b.responseHeadersPolicies[id] + p, ok := b.responseHeadersPolicies.Get(id) if !ok { return nil, fmt.Errorf( "%w: response headers policy %s not found", @@ -2284,7 +2351,7 @@ func (b *InMemoryBackend) UpdateResponseHeadersPolicy( if _, exists := b.responseHeadersPolicyByName[name]; exists { return nil, fmt.Errorf( "%w: response headers policy with name %q already exists", - ErrAlreadyExists, + ErrResponseHeadersPolicyAlreadyExists, name, ) } @@ -2314,7 +2381,7 @@ func (b *InMemoryBackend) DeleteResponseHeadersPolicy(id string) error { b.mu.Lock("DeleteResponseHeadersPolicy") defer b.mu.Unlock() - p, ok := b.responseHeadersPolicies[id] + p, ok := b.responseHeadersPolicies.Get(id) if !ok { return fmt.Errorf( "%w: response headers policy %s not found", @@ -2323,8 +2390,15 @@ func (b *InMemoryBackend) DeleteResponseHeadersPolicy(id string) error { ) } + if b.tokenReferencedByAnyDistribution(id) { + return fmt.Errorf( + "%w: response headers policy %s is attached to a distribution", + ErrResponseHeadersPolicyInUse, id, + ) + } + delete(b.responseHeadersPolicyByName, p.Name) - delete(b.responseHeadersPolicies, id) + b.responseHeadersPolicies.Delete(id) return nil } @@ -2346,20 +2420,23 @@ func (b *InMemoryBackend) CreateFunction( return nil, fmt.Errorf("%w: Name must not be empty", ErrValidation) } - if _, exists := b.functions[name]; exists { - return nil, fmt.Errorf("%w: function with name %q already exists", ErrAlreadyExists, name) + if _, exists := b.functions.Get(name); exists { + return nil, fmt.Errorf("%w: function with name %q already exists", ErrFunctionAlreadyExists, name) } + now := time.Now().UTC().Format(time.RFC3339) fn := &Function{ - Name: name, - Comment: comment, - Runtime: runtime, - FunctionCode: functionCode, - Status: functionStageDevelopment, - ETag: uuid.NewString(), - ARN: b.functionARN(name), - } - b.functions[name] = fn + Name: name, + Comment: comment, + Runtime: runtime, + FunctionCode: functionCode, + Status: functionStageDevelopment, + ETag: uuid.NewString(), + ARN: b.functionARN(name), + CreatedTime: now, + LastModifiedTime: now, + } + b.functions.Put(fn) cp := *fn return &cp, nil @@ -2370,7 +2447,7 @@ func (b *InMemoryBackend) GetFunction(name string) (*Function, error) { b.mu.RLock("GetFunction") defer b.mu.RUnlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return nil, fmt.Errorf("%w: function %s not found", ErrFunctionNotFound, name) } @@ -2385,8 +2462,8 @@ func (b *InMemoryBackend) ListFunctions() []*Function { b.mu.RLock("ListFunctions") defer b.mu.RUnlock() - list := make([]*Function, 0, len(b.functions)) - for _, fn := range b.functions { + list := make([]*Function, 0, b.functions.Len()) + for _, fn := range b.functions.All() { cp := *fn list = append(list, &cp) } @@ -2401,13 +2478,14 @@ func (b *InMemoryBackend) PublishFunction(name string) (*Function, error) { b.mu.Lock("PublishFunction") defer b.mu.Unlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return nil, fmt.Errorf("%w: function %s not found", ErrFunctionNotFound, name) } fn.Status = functionStageLive fn.ETag = uuid.NewString() + fn.LastModifiedTime = time.Now().UTC().Format(time.RFC3339) cp := *fn return &cp, nil @@ -2424,7 +2502,7 @@ func (b *InMemoryBackend) UpdateFunction( b.mu.Lock("UpdateFunction") defer b.mu.Unlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return nil, fmt.Errorf("%w: function %s not found", ErrFunctionNotFound, name) } @@ -2434,6 +2512,7 @@ func (b *InMemoryBackend) UpdateFunction( fn.FunctionCode = functionCode fn.Status = functionStageDevelopment fn.ETag = uuid.NewString() + fn.LastModifiedTime = time.Now().UTC().Format(time.RFC3339) cp := *fn return &cp, nil @@ -2444,11 +2523,15 @@ func (b *InMemoryBackend) DeleteFunction(name string) error { b.mu.Lock("DeleteFunction") defer b.mu.Unlock() - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return fmt.Errorf("%w: function %s not found", ErrFunctionNotFound, name) } - delete(b.functions, name) + if b.tokenReferencedByAnyDistribution(b.functionARN(name)) { + return fmt.Errorf("%w: function %s is associated with a distribution", ErrFunctionInUse, name) + } + + b.functions.Delete(name) return nil } @@ -2477,7 +2560,7 @@ func (b *InMemoryBackend) CreateOriginRequestPolicy( if _, exists := b.originRequestPolicyByName[name]; exists { return nil, fmt.Errorf( "%w: origin request policy with name %q already exists", - ErrAlreadyExists, + ErrOriginRequestPolicyAlreadyExists, name, ) } @@ -2497,7 +2580,7 @@ func (b *InMemoryBackend) CreateOriginRequestPolicy( p.QueryStringsConfig = cfg.QueryStringsConfig } - b.originRequestPolicies[id] = p + b.originRequestPolicies.Put(p) b.originRequestPolicyByName[name] = id cp := *p @@ -2509,7 +2592,7 @@ func (b *InMemoryBackend) GetOriginRequestPolicy(id string) (*OriginRequestPolic b.mu.RLock("GetOriginRequestPolicy") defer b.mu.RUnlock() - p, ok := b.originRequestPolicies[id] + p, ok := b.originRequestPolicies.Get(id) if !ok { return nil, fmt.Errorf( "%w: origin request policy %s not found", @@ -2528,8 +2611,8 @@ func (b *InMemoryBackend) ListOriginRequestPolicies() []*OriginRequestPolicy { b.mu.RLock("ListOriginRequestPolicies") defer b.mu.RUnlock() - list := make([]*OriginRequestPolicy, 0, len(b.originRequestPolicies)) - for _, p := range b.originRequestPolicies { + list := make([]*OriginRequestPolicy, 0, b.originRequestPolicies.Len()) + for _, p := range b.originRequestPolicies.All() { cp := *p list = append(list, &cp) } @@ -2547,7 +2630,7 @@ func (b *InMemoryBackend) UpdateOriginRequestPolicy( b.mu.Lock("UpdateOriginRequestPolicy") defer b.mu.Unlock() - p, ok := b.originRequestPolicies[id] + p, ok := b.originRequestPolicies.Get(id) if !ok { return nil, fmt.Errorf( "%w: origin request policy %s not found", @@ -2564,7 +2647,7 @@ func (b *InMemoryBackend) UpdateOriginRequestPolicy( if _, exists := b.originRequestPolicyByName[name]; exists { return nil, fmt.Errorf( "%w: origin request policy with name %q already exists", - ErrAlreadyExists, + ErrOriginRequestPolicyAlreadyExists, name, ) } @@ -2593,7 +2676,7 @@ func (b *InMemoryBackend) DeleteOriginRequestPolicy(id string) error { b.mu.Lock("DeleteOriginRequestPolicy") defer b.mu.Unlock() - p, ok := b.originRequestPolicies[id] + p, ok := b.originRequestPolicies.Get(id) if !ok { return fmt.Errorf( "%w: origin request policy %s not found", @@ -2602,8 +2685,15 @@ func (b *InMemoryBackend) DeleteOriginRequestPolicy(id string) error { ) } + if b.tokenReferencedByAnyDistribution(id) { + return fmt.Errorf( + "%w: origin request policy %s is attached to a distribution", + ErrOriginRequestPolicyInUse, id, + ) + } + delete(b.originRequestPolicyByName, p.Name) - delete(b.originRequestPolicies, id) + b.originRequestPolicies.Delete(id) return nil } @@ -2636,7 +2726,7 @@ func (b *InMemoryBackend) requireProfilesExist(profiles []FLEQueryArgProfile) er continue } - if _, ok := b.fieldLevelEncryptionProfiles[p.ProfileID]; !ok { + if _, ok := b.fieldLevelEncryptionProfiles.Get(p.ProfileID); !ok { return fmt.Errorf( "%w: field level encryption profile %s not found", ErrFLEProfileNotFound, @@ -2656,7 +2746,7 @@ func (b *InMemoryBackend) requirePublicKeysExist(entities []EncryptionEntity) er continue } - if _, ok := b.publicKeys[e.PublicKeyID]; !ok { + if _, ok := b.publicKeys.Get(e.PublicKeyID); !ok { return fmt.Errorf("%w: public key %s not found", ErrPublicKeyNotFound, e.PublicKeyID) } } @@ -2706,7 +2796,7 @@ func (b *InMemoryBackend) CreateFieldLevelEncryption( if _, exists := b.fieldLevelEncryptionByName[name]; exists { return nil, fmt.Errorf( "%w: field level encryption with name %q already exists", - ErrAlreadyExists, + ErrFLEAlreadyExists, name, ) } @@ -2723,7 +2813,7 @@ func (b *InMemoryBackend) CreateFieldLevelEncryption( ETag: uuid.NewString(), QueryArgProfiles: cloneQueryArgProfiles(queryArgProfiles), } - b.fieldLevelEncryptions[id] = fle + b.fieldLevelEncryptions.Put(fle) b.fieldLevelEncryptionByName[name] = id cp := *fle cp.QueryArgProfiles = cloneQueryArgProfiles(fle.QueryArgProfiles) @@ -2736,7 +2826,7 @@ func (b *InMemoryBackend) GetFieldLevelEncryption(id string) (*FieldLevelEncrypt b.mu.RLock("GetFieldLevelEncryption") defer b.mu.RUnlock() - fle, ok := b.fieldLevelEncryptions[id] + fle, ok := b.fieldLevelEncryptions.Get(id) if !ok { return nil, fmt.Errorf("%w: field level encryption %s not found", ErrFLENotFound, id) } @@ -2752,8 +2842,8 @@ func (b *InMemoryBackend) ListFieldLevelEncryptions() []*FieldLevelEncryption { b.mu.RLock("ListFieldLevelEncryptions") defer b.mu.RUnlock() - list := make([]*FieldLevelEncryption, 0, len(b.fieldLevelEncryptions)) - for _, fle := range b.fieldLevelEncryptions { + list := make([]*FieldLevelEncryption, 0, b.fieldLevelEncryptions.Len()) + for _, fle := range b.fieldLevelEncryptions.All() { cp := *fle cp.QueryArgProfiles = cloneQueryArgProfiles(fle.QueryArgProfiles) list = append(list, &cp) @@ -2772,7 +2862,7 @@ func (b *InMemoryBackend) UpdateFieldLevelEncryption( b.mu.Lock("UpdateFieldLevelEncryption") defer b.mu.Unlock() - fle, ok := b.fieldLevelEncryptions[id] + fle, ok := b.fieldLevelEncryptions.Get(id) if !ok { return nil, fmt.Errorf("%w: field level encryption %s not found", ErrFLENotFound, id) } @@ -2783,7 +2873,7 @@ func (b *InMemoryBackend) UpdateFieldLevelEncryption( if !renameInIndex(b.fieldLevelEncryptionByName, id, fle.Name, name) { return nil, fmt.Errorf( - "%w: field level encryption with name %q already exists", ErrAlreadyExists, name) + "%w: field level encryption with name %q already exists", ErrFLEAlreadyExists, name) } fle.Name = name @@ -2801,13 +2891,13 @@ func (b *InMemoryBackend) DeleteFieldLevelEncryption(id string) error { b.mu.Lock("DeleteFieldLevelEncryption") defer b.mu.Unlock() - fle, ok := b.fieldLevelEncryptions[id] + fle, ok := b.fieldLevelEncryptions.Get(id) if !ok { return fmt.Errorf("%w: field level encryption %s not found", ErrFLENotFound, id) } delete(b.fieldLevelEncryptionByName, fle.Name) - delete(b.fieldLevelEncryptions, id) + b.fieldLevelEncryptions.Delete(id) return nil } @@ -2829,7 +2919,7 @@ func (b *InMemoryBackend) CreateFieldLevelEncryptionProfile( if _, exists := b.fieldLevelEncryptionProfileByName[name]; exists { return nil, fmt.Errorf( "%w: field level encryption profile with name %q already exists", - ErrAlreadyExists, + ErrFLEProfileAlreadyExists, name, ) } @@ -2846,7 +2936,7 @@ func (b *InMemoryBackend) CreateFieldLevelEncryptionProfile( ETag: uuid.NewString(), EncryptionEntities: cloneEncryptionEntities(entities), } - b.fieldLevelEncryptionProfiles[id] = p + b.fieldLevelEncryptionProfiles.Put(p) b.fieldLevelEncryptionProfileByName[name] = id cp := *p cp.EncryptionEntities = cloneEncryptionEntities(p.EncryptionEntities) @@ -2861,7 +2951,7 @@ func (b *InMemoryBackend) GetFieldLevelEncryptionProfile( b.mu.RLock("GetFieldLevelEncryptionProfile") defer b.mu.RUnlock() - p, ok := b.fieldLevelEncryptionProfiles[id] + p, ok := b.fieldLevelEncryptionProfiles.Get(id) if !ok { return nil, fmt.Errorf( "%w: field level encryption profile %s not found", @@ -2881,8 +2971,8 @@ func (b *InMemoryBackend) ListFieldLevelEncryptionProfiles() []*FieldLevelEncryp b.mu.RLock("ListFieldLevelEncryptionProfiles") defer b.mu.RUnlock() - list := make([]*FieldLevelEncryptionProfile, 0, len(b.fieldLevelEncryptionProfiles)) - for _, p := range b.fieldLevelEncryptionProfiles { + list := make([]*FieldLevelEncryptionProfile, 0, b.fieldLevelEncryptionProfiles.Len()) + for _, p := range b.fieldLevelEncryptionProfiles.All() { cp := *p cp.EncryptionEntities = cloneEncryptionEntities(p.EncryptionEntities) list = append(list, &cp) @@ -2901,7 +2991,7 @@ func (b *InMemoryBackend) UpdateFieldLevelEncryptionProfile( b.mu.Lock("UpdateFieldLevelEncryptionProfile") defer b.mu.Unlock() - p, ok := b.fieldLevelEncryptionProfiles[id] + p, ok := b.fieldLevelEncryptionProfiles.Get(id) if !ok { return nil, fmt.Errorf( "%w: field level encryption profile %s not found", @@ -2916,7 +3006,7 @@ func (b *InMemoryBackend) UpdateFieldLevelEncryptionProfile( if !renameInIndex(b.fieldLevelEncryptionProfileByName, id, p.Name, name) { return nil, fmt.Errorf( - "%w: field level encryption profile with name %q already exists", ErrAlreadyExists, name) + "%w: field level encryption profile with name %q already exists", ErrFLEProfileAlreadyExists, name) } p.Name = name @@ -2932,7 +3022,7 @@ func (b *InMemoryBackend) UpdateFieldLevelEncryptionProfile( // fleProfileReferencedBy returns the ID of an FLE config that references the given // profile, or "" if none. Must be called with the lock held. func (b *InMemoryBackend) fleProfileReferencedBy(profileID string) string { - for _, fle := range b.fieldLevelEncryptions { + for _, fle := range b.fieldLevelEncryptions.All() { for _, qp := range fle.QueryArgProfiles { if qp.ProfileID == profileID { return fle.ID @@ -2949,7 +3039,7 @@ func (b *InMemoryBackend) DeleteFieldLevelEncryptionProfile(id string) error { b.mu.Lock("DeleteFieldLevelEncryptionProfile") defer b.mu.Unlock() - p, ok := b.fieldLevelEncryptionProfiles[id] + p, ok := b.fieldLevelEncryptionProfiles.Get(id) if !ok { return fmt.Errorf( "%w: field level encryption profile %s not found", @@ -2968,7 +3058,7 @@ func (b *InMemoryBackend) DeleteFieldLevelEncryptionProfile(id string) error { } delete(b.fieldLevelEncryptionProfileByName, p.Name) - delete(b.fieldLevelEncryptionProfiles, id) + b.fieldLevelEncryptionProfiles.Delete(id) return nil } @@ -2993,7 +3083,7 @@ func (b *InMemoryBackend) CreatePublicKey( } if _, exists := b.publicKeyByName[name]; exists { - return nil, fmt.Errorf("%w: public key with name %q already exists", ErrAlreadyExists, name) + return nil, fmt.Errorf("%w: public key with name %q already exists", ErrPublicKeyAlreadyExists, name) } id := generateID() @@ -3005,7 +3095,7 @@ func (b *InMemoryBackend) CreatePublicKey( CallerReference: callerRef, ETag: uuid.NewString(), } - b.publicKeys[id] = pk + b.publicKeys.Put(pk) b.publicKeyByName[name] = id cp := *pk @@ -3017,7 +3107,7 @@ func (b *InMemoryBackend) GetPublicKey(id string) (*PublicKey, error) { b.mu.RLock("GetPublicKey") defer b.mu.RUnlock() - pk, ok := b.publicKeys[id] + pk, ok := b.publicKeys.Get(id) if !ok { return nil, fmt.Errorf("%w: public key %s not found", ErrPublicKeyNotFound, id) } @@ -3032,8 +3122,8 @@ func (b *InMemoryBackend) ListPublicKeys() []*PublicKey { b.mu.RLock("ListPublicKeys") defer b.mu.RUnlock() - list := make([]*PublicKey, 0, len(b.publicKeys)) - for _, pk := range b.publicKeys { + list := make([]*PublicKey, 0, b.publicKeys.Len()) + for _, pk := range b.publicKeys.All() { cp := *pk list = append(list, &cp) } @@ -3048,7 +3138,7 @@ func (b *InMemoryBackend) UpdatePublicKey(id, comment string) (*PublicKey, error b.mu.Lock("UpdatePublicKey") defer b.mu.Unlock() - pk, ok := b.publicKeys[id] + pk, ok := b.publicKeys.Get(id) if !ok { return nil, fmt.Errorf("%w: public key %s not found", ErrPublicKeyNotFound, id) } @@ -3064,13 +3154,13 @@ func (b *InMemoryBackend) UpdatePublicKey(id, comment string) (*PublicKey, error // references the given public key, or ("", "") if none. Must be called with the // lock held. func (b *InMemoryBackend) publicKeyReferencedBy(pkID string) (string, string) { - for _, kg := range b.keyGroups { + for _, kg := range b.keyGroups.All() { if slices.Contains(kg.Items, pkID) { return "key group", kg.ID } } - for _, p := range b.fieldLevelEncryptionProfiles { + for _, p := range b.fieldLevelEncryptionProfiles.All() { for _, e := range p.EncryptionEntities { if e.PublicKeyID == pkID { return "field level encryption profile", p.ID @@ -3087,7 +3177,7 @@ func (b *InMemoryBackend) DeletePublicKey(id string) error { b.mu.Lock("DeletePublicKey") defer b.mu.Unlock() - pk, ok := b.publicKeys[id] + pk, ok := b.publicKeys.Get(id) if !ok { return fmt.Errorf("%w: public key %s not found", ErrPublicKeyNotFound, id) } @@ -3103,7 +3193,7 @@ func (b *InMemoryBackend) DeletePublicKey(id string) error { } delete(b.publicKeyByName, pk.Name) - delete(b.publicKeys, id) + b.publicKeys.Delete(id) return nil } @@ -3120,11 +3210,11 @@ func (b *InMemoryBackend) CreateKeyGroup(name, comment string, items []string) ( } if _, exists := b.keyGroupByName[name]; exists { - return nil, fmt.Errorf("%w: key group with name %q already exists", ErrAlreadyExists, name) + return nil, fmt.Errorf("%w: key group with name %q already exists", ErrKeyGroupAlreadyExists, name) } for _, itemID := range items { - if _, ok := b.publicKeys[itemID]; !ok { + if _, ok := b.publicKeys.Get(itemID); !ok { return nil, fmt.Errorf("%w: public key %s not found", ErrPublicKeyNotFound, itemID) } } @@ -3137,7 +3227,7 @@ func (b *InMemoryBackend) CreateKeyGroup(name, comment string, items []string) ( Items: append([]string(nil), items...), ETag: uuid.NewString(), } - b.keyGroups[id] = kg + b.keyGroups.Put(kg) b.keyGroupByName[name] = id return b.copyKeyGroup(kg), nil @@ -3148,7 +3238,7 @@ func (b *InMemoryBackend) GetKeyGroup(id string) (*KeyGroup, error) { b.mu.RLock("GetKeyGroup") defer b.mu.RUnlock() - kg, ok := b.keyGroups[id] + kg, ok := b.keyGroups.Get(id) if !ok { return nil, fmt.Errorf("%w: key group %s not found", ErrKeyGroupNotFound, id) } @@ -3161,8 +3251,8 @@ func (b *InMemoryBackend) ListKeyGroups() []*KeyGroup { b.mu.RLock("ListKeyGroups") defer b.mu.RUnlock() - list := make([]*KeyGroup, 0, len(b.keyGroups)) - for _, kg := range b.keyGroups { + list := make([]*KeyGroup, 0, b.keyGroups.Len()) + for _, kg := range b.keyGroups.All() { list = append(list, b.copyKeyGroup(kg)) } @@ -3179,7 +3269,7 @@ func (b *InMemoryBackend) UpdateKeyGroup( b.mu.Lock("UpdateKeyGroup") defer b.mu.Unlock() - kg, ok := b.keyGroups[id] + kg, ok := b.keyGroups.Get(id) if !ok { return nil, fmt.Errorf("%w: key group %s not found", ErrKeyGroupNotFound, id) } @@ -3188,7 +3278,7 @@ func (b *InMemoryBackend) UpdateKeyGroup( if _, exists := b.keyGroupByName[name]; exists { return nil, fmt.Errorf( "%w: key group with name %q already exists", - ErrAlreadyExists, + ErrKeyGroupAlreadyExists, name, ) } @@ -3198,7 +3288,7 @@ func (b *InMemoryBackend) UpdateKeyGroup( } for _, itemID := range items { - if _, exists := b.publicKeys[itemID]; !exists { + if _, exists := b.publicKeys.Get(itemID); !exists { return nil, fmt.Errorf("%w: public key %s not found", ErrPublicKeyNotFound, itemID) } } @@ -3216,13 +3306,13 @@ func (b *InMemoryBackend) DeleteKeyGroup(id string) error { b.mu.Lock("DeleteKeyGroup") defer b.mu.Unlock() - kg, ok := b.keyGroups[id] + kg, ok := b.keyGroups.Get(id) if !ok { return fmt.Errorf("%w: key group %s not found", ErrKeyGroupNotFound, id) } delete(b.keyGroupByName, kg.Name) - delete(b.keyGroups, id) + b.keyGroups.Delete(id) return nil } @@ -3261,7 +3351,7 @@ func (b *InMemoryBackend) CreateRealtimeLogConfig( if _, exists := b.realtimeLogConfigByName[name]; exists { return nil, fmt.Errorf( "%w: realtime log config with name %q already exists", - ErrAlreadyExists, + ErrRealtimeLogConfigAlreadyExists, name, ) } @@ -3273,7 +3363,7 @@ func (b *InMemoryBackend) CreateRealtimeLogConfig( SamplingRate: samplingRate, Fields: append([]string(nil), fields...), } - b.realtimeLogConfigs[arn] = cfg + b.realtimeLogConfigs.Put(cfg) b.realtimeLogConfigByName[name] = arn return b.copyRealtimeLogConfig(cfg), nil @@ -3284,7 +3374,7 @@ func (b *InMemoryBackend) GetRealtimeLogConfig(arn string) (*RealtimeLogConfig, b.mu.RLock("GetRealtimeLogConfig") defer b.mu.RUnlock() - cfg, ok := b.realtimeLogConfigs[arn] + cfg, ok := b.realtimeLogConfigs.Get(arn) if !ok { return nil, fmt.Errorf( "%w: realtime log config %s not found", @@ -3310,7 +3400,9 @@ func (b *InMemoryBackend) GetRealtimeLogConfigByName(name string) (*RealtimeLogC ) } - return b.copyRealtimeLogConfig(b.realtimeLogConfigs[arn]), nil + cfg, _ := b.realtimeLogConfigs.Get(arn) + + return b.copyRealtimeLogConfig(cfg), nil } // ListRealtimeLogConfigs returns all Realtime Log Configs sorted by name. @@ -3318,8 +3410,8 @@ func (b *InMemoryBackend) ListRealtimeLogConfigs() []*RealtimeLogConfig { b.mu.RLock("ListRealtimeLogConfigs") defer b.mu.RUnlock() - list := make([]*RealtimeLogConfig, 0, len(b.realtimeLogConfigs)) - for _, cfg := range b.realtimeLogConfigs { + list := make([]*RealtimeLogConfig, 0, b.realtimeLogConfigs.Len()) + for _, cfg := range b.realtimeLogConfigs.All() { list = append(list, b.copyRealtimeLogConfig(cfg)) } @@ -3341,7 +3433,7 @@ func (b *InMemoryBackend) UpdateRealtimeLogConfig( b.mu.Lock("UpdateRealtimeLogConfig") defer b.mu.Unlock() - cfg, ok := b.realtimeLogConfigs[arn] + cfg, ok := b.realtimeLogConfigs.Get(arn) if !ok { return nil, fmt.Errorf( "%w: realtime log config %s not found", @@ -3361,13 +3453,13 @@ func (b *InMemoryBackend) DeleteRealtimeLogConfig(arn string) error { b.mu.Lock("DeleteRealtimeLogConfig") defer b.mu.Unlock() - cfg, ok := b.realtimeLogConfigs[arn] + cfg, ok := b.realtimeLogConfigs.Get(arn) if !ok { return fmt.Errorf("%w: realtime log config %s not found", ErrRealtimeLogConfigNotFound, arn) } delete(b.realtimeLogConfigByName, cfg.Name) - delete(b.realtimeLogConfigs, arn) + b.realtimeLogConfigs.Delete(arn) return nil } @@ -3413,7 +3505,7 @@ func (b *InMemoryBackend) CreateKeyValueStore(name, comment string) (*KeyValueSt Status: kvsStatusReady, LastModifiedTime: time.Now().UTC().Format(time.RFC3339), } - b.keyValueStores[id] = kvs + b.keyValueStores.Put(kvs) b.keyValueStoreByName[name] = id cp := *kvs @@ -3425,13 +3517,13 @@ func (b *InMemoryBackend) GetKeyValueStore(idOrARN string) (*KeyValueStore, erro b.mu.RLock("GetKeyValueStore") defer b.mu.RUnlock() - if kvs, ok := b.keyValueStores[idOrARN]; ok { + if kvs, ok := b.keyValueStores.Get(idOrARN); ok { cp := *kvs return &cp, nil } - for _, kvs := range b.keyValueStores { + for _, kvs := range b.keyValueStores.All() { if kvs.ARN == idOrARN { cp := *kvs @@ -3447,8 +3539,8 @@ func (b *InMemoryBackend) ListKeyValueStores() []*KeyValueStore { b.mu.RLock("ListKeyValueStores") defer b.mu.RUnlock() - list := make([]*KeyValueStore, 0, len(b.keyValueStores)) - for _, kvs := range b.keyValueStores { + list := make([]*KeyValueStore, 0, b.keyValueStores.Len()) + for _, kvs := range b.keyValueStores.All() { cp := *kvs list = append(list, &cp) } @@ -3463,13 +3555,13 @@ func (b *InMemoryBackend) DeleteKeyValueStore(id string) error { b.mu.Lock("DeleteKeyValueStore") defer b.mu.Unlock() - kvs, ok := b.keyValueStores[id] + kvs, ok := b.keyValueStores.Get(id) if !ok { return fmt.Errorf("%w: key value store %s not found", ErrKeyValueStoreNotFound, id) } delete(b.keyValueStoreByName, kvs.Name) - delete(b.keyValueStores, id) + b.keyValueStores.Delete(id) return nil } @@ -3493,7 +3585,7 @@ func (b *InMemoryBackend) GetKVSValue(kvsID, key string) (string, string, error) b.mu.RLock("GetKVSValue") defer b.mu.RUnlock() - if _, ok := b.keyValueStores[kvsID]; !ok { + if _, ok := b.keyValueStores.Get(kvsID); !ok { return "", "", fmt.Errorf("%w: key value store %s not found", ErrKeyValueStoreNotFound, kvsID) } @@ -3511,7 +3603,7 @@ func (b *InMemoryBackend) PutKVSValue(kvsID, key, value, ifMatch string) (string b.mu.Lock("PutKVSValue") defer b.mu.Unlock() - if _, ok := b.keyValueStores[kvsID]; !ok { + if _, ok := b.keyValueStores.Get(kvsID); !ok { return "", fmt.Errorf("%w: key value store %s not found", ErrKeyValueStoreNotFound, kvsID) } @@ -3535,7 +3627,7 @@ func (b *InMemoryBackend) DeleteKVSValue(kvsID, key, ifMatch string) (string, er b.mu.Lock("DeleteKVSValue") defer b.mu.Unlock() - if _, ok := b.keyValueStores[kvsID]; !ok { + if _, ok := b.keyValueStores.Get(kvsID); !ok { return "", fmt.Errorf("%w: key value store %s not found", ErrKeyValueStoreNotFound, kvsID) } @@ -3564,7 +3656,7 @@ func (b *InMemoryBackend) ListKVSValues(kvsID string) ([]*KVSItem, string, error b.mu.RLock("ListKVSValues") defer b.mu.RUnlock() - if _, ok := b.keyValueStores[kvsID]; !ok { + if _, ok := b.keyValueStores.Get(kvsID); !ok { return nil, "", fmt.Errorf("%w: key value store %s not found", ErrKeyValueStoreNotFound, kvsID) } @@ -3583,7 +3675,7 @@ func (b *InMemoryBackend) UpdateKVSValues(kvsID, ifMatch string, puts []*KVSItem b.mu.Lock("UpdateKVSValues") defer b.mu.Unlock() - if _, ok := b.keyValueStores[kvsID]; !ok { + if _, ok := b.keyValueStores.Get(kvsID); !ok { return "", fmt.Errorf("%w: key value store %s not found", ErrKeyValueStoreNotFound, kvsID) } @@ -3630,7 +3722,7 @@ func (b *InMemoryBackend) CreateVpcOrigin(name string) (*VpcOrigin, error) { Name: name, ETag: uuid.NewString(), } - b.vpcOrigins[id] = origin + b.vpcOrigins.Put(origin) cp := *origin return &cp, nil @@ -3641,7 +3733,7 @@ func (b *InMemoryBackend) GetVpcOrigin(id string) (*VpcOrigin, error) { b.mu.RLock("GetVpcOrigin") defer b.mu.RUnlock() - origin, ok := b.vpcOrigins[id] + origin, ok := b.vpcOrigins.Get(id) if !ok { return nil, fmt.Errorf("%w: vpc origin %s not found", ErrVpcOriginNotFound, id) } @@ -3656,8 +3748,8 @@ func (b *InMemoryBackend) ListVpcOrigins() []*VpcOrigin { b.mu.RLock("ListVpcOrigins") defer b.mu.RUnlock() - list := make([]*VpcOrigin, 0, len(b.vpcOrigins)) - for _, origin := range b.vpcOrigins { + list := make([]*VpcOrigin, 0, b.vpcOrigins.Len()) + for _, origin := range b.vpcOrigins.All() { cp := *origin list = append(list, &cp) } @@ -3672,7 +3764,7 @@ func (b *InMemoryBackend) UpdateVpcOrigin(id, name string) (*VpcOrigin, error) { b.mu.Lock("UpdateVpcOrigin") defer b.mu.Unlock() - origin, ok := b.vpcOrigins[id] + origin, ok := b.vpcOrigins.Get(id) if !ok { return nil, fmt.Errorf("%w: vpc origin %s not found", ErrVpcOriginNotFound, id) } @@ -3689,11 +3781,11 @@ func (b *InMemoryBackend) DeleteVpcOrigin(id string) error { b.mu.Lock("DeleteVpcOrigin") defer b.mu.Unlock() - if _, ok := b.vpcOrigins[id]; !ok { + if _, ok := b.vpcOrigins.Get(id); !ok { return fmt.Errorf("%w: vpc origin %s not found", ErrVpcOriginNotFound, id) } - delete(b.vpcOrigins, id) + b.vpcOrigins.Delete(id) return nil } diff --git a/services/cloudfront/backend_batch2.go b/services/cloudfront/backend_batch2.go index b83935597..2baaf25a0 100644 --- a/services/cloudfront/backend_batch2.go +++ b/services/cloudfront/backend_batch2.go @@ -220,7 +220,7 @@ func (b *InMemoryBackend) CreateDistributionTenant( if t.Tags == nil { t.Tags = make(map[string]string) } - b.distributionTenants[id] = t + b.distributionTenants.Put(t) b.distributionTenantARNs[t.ARN] = id for _, d := range domains { b.distributionTenantsByDomain[d] = id @@ -235,7 +235,7 @@ func (b *InMemoryBackend) GetDistributionTenant(id string) (*DistributionTenant, b.mu.RLock("GetDistributionTenant") defer b.mu.RUnlock() - t, ok := b.distributionTenants[id] + t, ok := b.distributionTenants.Get(id) if !ok { return nil, fmt.Errorf("%w: tenant %s not found", ErrDistributionTenantNotFound, id) } @@ -252,7 +252,7 @@ func (b *InMemoryBackend) GetDistributionTenantByDomain(domain string) (*Distrib if !ok { return nil, fmt.Errorf("%w: tenant with domain %s not found", ErrDistributionTenantNotFound, domain) } - t, ok := b.distributionTenants[id] + t, ok := b.distributionTenants.Get(id) if !ok { return nil, fmt.Errorf("%w: tenant %s not found", ErrDistributionTenantNotFound, id) } @@ -280,7 +280,7 @@ func (b *InMemoryBackend) UpdateDistributionTenant( b.mu.Lock("UpdateDistributionTenant") defer b.mu.Unlock() - t, ok := b.distributionTenants[id] + t, ok := b.distributionTenants.Get(id) if !ok { return nil, fmt.Errorf("%w: tenant %s not found", ErrDistributionTenantNotFound, id) } @@ -329,7 +329,7 @@ func (b *InMemoryBackend) DeleteDistributionTenant(id string) error { b.mu.Lock("DeleteDistributionTenant") defer b.mu.Unlock() - t, ok := b.distributionTenants[id] + t, ok := b.distributionTenants.Get(id) if !ok { return fmt.Errorf("%w: tenant %s not found", ErrDistributionTenantNotFound, id) } @@ -340,8 +340,8 @@ func (b *InMemoryBackend) DeleteDistributionTenant(id string) error { delete(b.distributionTenantsByDomain, t.Domain) delete(b.distributionTenantARNs, t.ARN) delete(b.distributionTenantWebACLs, id) - delete(b.distributionTenants, id) - delete(b.tenantInvalidations, id) + b.distributionTenants.Delete(id) + b.deleteInvalidationsForTenant(id) delete(b.tenantInvalidationReadyAt, id) return nil @@ -352,8 +352,8 @@ func (b *InMemoryBackend) ListDistributionTenants() []*DistributionTenant { b.mu.RLock("ListDistributionTenants") defer b.mu.RUnlock() - out := make([]*DistributionTenant, 0, len(b.distributionTenants)) - for _, t := range b.distributionTenants { + out := make([]*DistributionTenant, 0, b.distributionTenants.Len()) + for _, t := range b.distributionTenants.All() { out = append(out, b.copyTenant(t)) } sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) @@ -377,8 +377,8 @@ func (b *InMemoryBackend) ListDistributionTenantsByCustomization(webACLArn strin b.mu.RLock("ListDistributionTenantsByCustomization") defer b.mu.RUnlock() - out := make([]*DistributionTenant, 0, len(b.distributionTenants)) - for _, t := range b.distributionTenants { + out := make([]*DistributionTenant, 0, b.distributionTenants.Len()) + for _, t := range b.distributionTenants.All() { if webACLArn != "" && b.distributionTenantWebACLs[t.ID] != webACLArn { continue } @@ -428,7 +428,7 @@ func (b *InMemoryBackend) UpdateDomainAssociation( func (b *InMemoryBackend) updateDomainAssociationToTenant( domain, targetTenantID string, ) (*DomainAssociationResult, error) { - target, ok := b.distributionTenants[targetTenantID] + target, ok := b.distributionTenants.Get(targetTenantID) if !ok { return nil, fmt.Errorf("%w: tenant %s not found", ErrDistributionTenantNotFound, targetTenantID) } @@ -456,7 +456,7 @@ func (b *InMemoryBackend) updateDomainAssociationToTenant( func (b *InMemoryBackend) updateDomainAssociationToDistribution( domain, targetDistID string, ) (*DomainAssociationResult, error) { - d, ok := b.distributions[targetDistID] + d, ok := b.distributions.Get(targetDistID) if !ok { return nil, fmt.Errorf("%w: distribution %s not found", ErrNotFound, targetDistID) } @@ -494,12 +494,12 @@ func (b *InMemoryBackend) VerifyDNSConfiguration(identifier string) ([]DNSConfig } var domains []string - switch { - case b.distributionTenants[identifier] != nil: - domains = b.distributionTenants[identifier].Domains - case b.distributions[identifier] != nil: + + if t, ok := b.distributionTenants.Get(identifier); ok { + domains = t.Domains + } else if _, distOK := b.distributions.Get(identifier); distOK { domains = b.distributionAliases[identifier] - default: + } else { return nil, fmt.Errorf("%w: identifier %s not found", ErrDistributionTenantNotFound, identifier) } @@ -536,7 +536,7 @@ func (b *InMemoryBackend) DisassociateDistributionWebACL(distID string) error { b.mu.Lock("DisassociateDistributionWebACL") defer b.mu.Unlock() - if _, ok := b.distributions[distID]; !ok { + if _, ok := b.distributions.Get(distID); !ok { return fmt.Errorf("%w: distribution %s not found", ErrNotFound, distID) } delete(b.distributionWebACLs, distID) @@ -549,7 +549,7 @@ func (b *InMemoryBackend) CreateInvalidationForTenant(tenantID string, paths []s b.mu.Lock("CreateInvalidationForTenant") defer b.mu.Unlock() - if _, ok := b.distributionTenants[tenantID]; !ok { + if _, ok := b.distributionTenants.Get(tenantID); !ok { return nil, fmt.Errorf("%w: tenant %s not found", ErrDistributionTenantNotFound, tenantID) } @@ -561,8 +561,9 @@ func (b *InMemoryBackend) CreateInvalidationForTenant(tenantID string, paths []s Status: statusInProgress, CreateTime: now, Paths: paths, + tenantID: tenantID, } - b.tenantInvalidations[tenantID] = append(b.tenantInvalidations[tenantID], inv) + b.tenantInvalidations.Put(inv) if b.tenantInvalidationReadyAt[tenantID] == nil { b.tenantInvalidationReadyAt[tenantID] = make(map[string]time.Time) @@ -580,8 +581,8 @@ func (b *InMemoryBackend) GetInvalidationForTenant(tenantID, invalidationID stri b.mu.RLock("GetInvalidationForTenant") defer b.mu.RUnlock() - invs, ok := b.tenantInvalidations[tenantID] - if !ok { + invs := b.tenantInvalidationsByTenant.Get(tenantID) + if len(invs) == 0 { return nil, fmt.Errorf( "%w: invalidation %s not found for tenant %s", ErrInvalidationNotFound, @@ -605,7 +606,7 @@ func (b *InMemoryBackend) ListInvalidationsForTenant(tenantID string) []*Invalid b.mu.RLock("ListInvalidationsForTenant") defer b.mu.RUnlock() - invs := b.tenantInvalidations[tenantID] + invs := b.tenantInvalidationsByTenant.Get(tenantID) out := make([]*Invalidation, 0, len(invs)) for _, inv := range invs { cp := *inv @@ -620,12 +621,12 @@ func (b *InMemoryBackend) UpdateDistributionWithStagingConfig(primaryID, staging b.mu.Lock("UpdateDistributionWithStagingConfig") defer b.mu.Unlock() - primary, ok := b.distributions[primaryID] + primary, ok := b.distributions.Get(primaryID) if !ok { return nil, fmt.Errorf("%w: distribution %s not found", ErrNotFound, primaryID) } - staging, ok := b.distributions[stagingID] + staging, ok := b.distributions.Get(stagingID) if !ok { return nil, fmt.Errorf("%w: staging distribution %s not found", ErrNotFound, stagingID) } @@ -704,7 +705,7 @@ func (b *InMemoryBackend) ListConflictingAliasesByDomain(domain string) []*Distr var out []*Distribution for distID, aliases := range b.distributionAliases { if slices.Contains(aliases, domain) { - if d, ok := b.distributions[distID]; ok { + if d, ok := b.distributions.Get(distID); ok { cp := *d out = append(out, &cp) } @@ -719,7 +720,7 @@ func (b *InMemoryBackend) UpdateKeyValueStore(id, comment string) (*KeyValueStor b.mu.Lock("UpdateKeyValueStore") defer b.mu.Unlock() - kvs, ok := b.keyValueStores[id] + kvs, ok := b.keyValueStores.Get(id) if !ok { return nil, fmt.Errorf("%w: key value store %s not found", ErrKeyValueStoreNotFound, id) } diff --git a/services/cloudfront/backend_new_ops.go b/services/cloudfront/backend_new_ops.go index 8d5c08f6b..204c1f2d9 100644 --- a/services/cloudfront/backend_new_ops.go +++ b/services/cloudfront/backend_new_ops.go @@ -109,7 +109,7 @@ func (b *InMemoryBackend) CreateTrustStore( CertificateAuthorityCertificatesBundle: bundle, Tags: make(map[string]string), } - b.trustStores[id] = ts + b.trustStores.Put(ts) b.trustStoreARNs[ts.ARN] = id b.trustStoreByName[name] = id @@ -121,7 +121,7 @@ func (b *InMemoryBackend) GetTrustStore(id string) (*TrustStore, error) { b.mu.RLock("GetTrustStore") defer b.mu.RUnlock() - ts, ok := b.trustStores[id] + ts, ok := b.trustStores.Get(id) if !ok { return nil, fmt.Errorf("%w: trust store %s not found", ErrTrustStoreNotFound, id) } @@ -134,8 +134,8 @@ func (b *InMemoryBackend) ListTrustStores() []*TrustStore { b.mu.RLock("ListTrustStores") defer b.mu.RUnlock() - out := make([]*TrustStore, 0, len(b.trustStores)) - for _, ts := range b.trustStores { + out := make([]*TrustStore, 0, b.trustStores.Len()) + for _, ts := range b.trustStores.All() { out = append(out, b.copyTrustStore(ts)) } sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) @@ -152,7 +152,7 @@ func (b *InMemoryBackend) UpdateTrustStore( b.mu.Lock("UpdateTrustStore") defer b.mu.Unlock() - ts, ok := b.trustStores[id] + ts, ok := b.trustStores.Get(id) if !ok { return nil, fmt.Errorf("%w: trust store %s not found", ErrTrustStoreNotFound, id) } @@ -185,14 +185,14 @@ func (b *InMemoryBackend) DeleteTrustStore(id string) error { b.mu.Lock("DeleteTrustStore") defer b.mu.Unlock() - ts, ok := b.trustStores[id] + ts, ok := b.trustStores.Get(id) if !ok { return fmt.Errorf("%w: trust store %s not found", ErrTrustStoreNotFound, id) } delete(b.trustStoreByName, ts.Name) delete(b.trustStoreARNs, ts.ARN) - delete(b.trustStores, id) + b.trustStores.Delete(id) return nil } @@ -275,7 +275,9 @@ func (b *InMemoryBackend) CreateStreamingDistribution( if cfg.CallerReference != "" { if existingID, ok := b.streamingDistributionCallerRefs[cfg.CallerReference]; ok { - return b.copyStreamingDistribution(b.streamingDistributions[existingID]), nil + existing, _ := b.streamingDistributions.Get(existingID) + + return b.copyStreamingDistribution(existing), nil } } @@ -291,7 +293,7 @@ func (b *InMemoryBackend) CreateStreamingDistribution( RawConfig: rawConfig, Tags: make(map[string]string), } - b.streamingDistributions[id] = sd + b.streamingDistributions.Put(sd) b.streamingDistributionARNs[sd.ARN] = id if cfg.CallerReference != "" { @@ -306,7 +308,7 @@ func (b *InMemoryBackend) GetStreamingDistribution(id string) (*StreamingDistrib b.mu.RLock("GetStreamingDistribution") defer b.mu.RUnlock() - sd, ok := b.streamingDistributions[id] + sd, ok := b.streamingDistributions.Get(id) if !ok { return nil, fmt.Errorf("%w: streaming distribution %s not found", ErrStreamingDistributionNotFound, id) } @@ -319,8 +321,8 @@ func (b *InMemoryBackend) ListStreamingDistributions() []*StreamingDistribution b.mu.RLock("ListStreamingDistributions") defer b.mu.RUnlock() - out := make([]*StreamingDistribution, 0, len(b.streamingDistributions)) - for _, sd := range b.streamingDistributions { + out := make([]*StreamingDistribution, 0, b.streamingDistributions.Len()) + for _, sd := range b.streamingDistributions.All() { out = append(out, b.copyStreamingDistribution(sd)) } sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) @@ -338,7 +340,7 @@ func (b *InMemoryBackend) UpdateStreamingDistribution( b.mu.Lock("UpdateStreamingDistribution") defer b.mu.Unlock() - sd, ok := b.streamingDistributions[id] + sd, ok := b.streamingDistributions.Get(id) if !ok { return nil, fmt.Errorf("%w: streaming distribution %s not found", ErrStreamingDistributionNotFound, id) } @@ -358,7 +360,7 @@ func (b *InMemoryBackend) DeleteStreamingDistribution(id string) error { b.mu.Lock("DeleteStreamingDistribution") defer b.mu.Unlock() - sd, ok := b.streamingDistributions[id] + sd, ok := b.streamingDistributions.Get(id) if !ok { return fmt.Errorf("%w: streaming distribution %s not found", ErrStreamingDistributionNotFound, id) } @@ -372,7 +374,7 @@ func (b *InMemoryBackend) DeleteStreamingDistribution(id string) error { delete(b.streamingDistributionARNs, sd.ARN) delete(b.streamingDistributionCallerRefs, sd.Config.CallerReference) - delete(b.streamingDistributions, id) + b.streamingDistributions.Delete(id) return nil } @@ -499,7 +501,7 @@ func (b *InMemoryBackend) GetConnectionGroup(id string) (*ConnectionGroup, error b.mu.RLock("GetConnectionGroup") defer b.mu.RUnlock() - cg, ok := b.connectionGroups[id] + cg, ok := b.connectionGroups.Get(id) if !ok { return nil, fmt.Errorf("%w: connection group %s not found", ErrConnectionGroupNotFound, id) } @@ -523,7 +525,9 @@ func (b *InMemoryBackend) GetConnectionGroupByRoutingEndpoint(endpoint string) ( ) } - return b.copyConnectionGroup(b.connectionGroups[id]), nil + cg, _ := b.connectionGroups.Get(id) + + return b.copyConnectionGroup(cg), nil } // ListConnectionGroups returns all connection groups sorted by ID. @@ -531,8 +535,8 @@ func (b *InMemoryBackend) ListConnectionGroups() []*ConnectionGroup { b.mu.RLock("ListConnectionGroups") defer b.mu.RUnlock() - out := make([]*ConnectionGroup, 0, len(b.connectionGroups)) - for _, cg := range b.connectionGroups { + out := make([]*ConnectionGroup, 0, b.connectionGroups.Len()) + for _, cg := range b.connectionGroups.All() { out = append(out, b.copyConnectionGroup(cg)) } sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) @@ -550,7 +554,7 @@ func (b *InMemoryBackend) UpdateConnectionGroup( b.mu.Lock("UpdateConnectionGroup") defer b.mu.Unlock() - cg, ok := b.connectionGroups[id] + cg, ok := b.connectionGroups.Get(id) if !ok { return nil, fmt.Errorf("%w: connection group %s not found", ErrConnectionGroupNotFound, id) } @@ -579,7 +583,7 @@ func (b *InMemoryBackend) DeleteConnectionGroup(id string) error { b.mu.Lock("DeleteConnectionGroup") defer b.mu.Unlock() - cg, ok := b.connectionGroups[id] + cg, ok := b.connectionGroups.Get(id) if !ok { return fmt.Errorf("%w: connection group %s not found", ErrConnectionGroupNotFound, id) } @@ -587,7 +591,7 @@ func (b *InMemoryBackend) DeleteConnectionGroup(id string) error { delete(b.connectionGroupByName, cg.Name) delete(b.connectionGroupByRoutingEndpoint, cg.RoutingEndpoint) delete(b.connectionGroupARNs, cg.ARN) - delete(b.connectionGroups, id) + b.connectionGroups.Delete(id) return nil } @@ -612,12 +616,12 @@ func (b *InMemoryBackend) copyConnectionFunction(fn *ConnectionFunction) *Connec // resolveConnectionFunction returns the function and its UUID key by id or name, mirroring the // real API's "Identifier" request field which accepts either. Must be called with the lock held. func (b *InMemoryBackend) resolveConnectionFunction(idOrName string) (*ConnectionFunction, string) { - if fn, ok := b.connectionFunctions[idOrName]; ok { + if fn, ok := b.connectionFunctions.Get(idOrName); ok { return fn, idOrName } if uuid, ok := b.connectionFunctionByName[idOrName]; ok { - if fn, fnOK := b.connectionFunctions[uuid]; fnOK { + if fn, fnOK := b.connectionFunctions.Get(uuid); fnOK { return fn, uuid } } @@ -643,8 +647,8 @@ func (b *InMemoryBackend) ListConnectionFunctions() []*ConnectionFunction { b.mu.RLock("ListConnectionFunctions") defer b.mu.RUnlock() - out := make([]*ConnectionFunction, 0, len(b.connectionFunctions)) - for _, fn := range b.connectionFunctions { + out := make([]*ConnectionFunction, 0, b.connectionFunctions.Len()) + for _, fn := range b.connectionFunctions.All() { out = append(out, b.copyConnectionFunction(fn)) } sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) @@ -695,7 +699,7 @@ func (b *InMemoryBackend) DeleteConnectionFunction(idOrName string) error { } delete(b.connectionFunctionARNs, fn.ARN) - delete(b.connectionFunctions, id) + b.connectionFunctions.Delete(id) delete(b.connectionFunctionByName, fn.Name) return nil @@ -778,7 +782,7 @@ func (b *InMemoryBackend) GetAnycastIPList(id string) (*AnycastIPList, error) { b.mu.RLock("GetAnycastIPList") defer b.mu.RUnlock() - list, ok := b.anycastIPLists[id] + list, ok := b.anycastIPLists.Get(id) if !ok { return nil, ErrAnycastIPListNotFound } @@ -790,8 +794,8 @@ func (b *InMemoryBackend) ListAnycastIPLists() []*AnycastIPList { b.mu.RLock("ListAnycastIPLists") defer b.mu.RUnlock() - out := make([]*AnycastIPList, 0, len(b.anycastIPLists)) - for _, list := range b.anycastIPLists { + out := make([]*AnycastIPList, 0, b.anycastIPLists.Len()) + for _, list := range b.anycastIPLists.All() { out = append(out, b.copyAnycastIPList(list)) } sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) @@ -805,7 +809,7 @@ func (b *InMemoryBackend) UpdateAnycastIPList(id string, ipCount int32) (*Anycas b.mu.Lock("UpdateAnycastIPList") defer b.mu.Unlock() - list, ok := b.anycastIPLists[id] + list, ok := b.anycastIPLists.Get(id) if !ok { return nil, ErrAnycastIPListNotFound } @@ -827,13 +831,13 @@ func (b *InMemoryBackend) DeleteAnycastIPList(id string) error { b.mu.Lock("DeleteAnycastIPList") defer b.mu.Unlock() - list, ok := b.anycastIPLists[id] + list, ok := b.anycastIPLists.Get(id) if !ok { return ErrAnycastIPListNotFound } delete(b.anycastIPListByName, list.Name) delete(b.anycastIPListARNs, list.ARN) - delete(b.anycastIPLists, id) + b.anycastIPLists.Delete(id) return nil } @@ -875,7 +879,7 @@ func (b *InMemoryBackend) GetManagedCertificateDetails(tenantID string) (*Manage b.mu.Lock("GetManagedCertificateDetails") defer b.mu.Unlock() - tenant, ok := b.distributionTenants[tenantID] + tenant, ok := b.distributionTenants.Get(tenantID) if !ok { return nil, fmt.Errorf("%w: distribution tenant %s not found", ErrDistributionTenantNotFound, tenantID) } @@ -935,7 +939,7 @@ func (b *InMemoryBackend) ListDistributionsByWebACLID(webACLID string) []*Distri var out []*Distribution for distID, wID := range b.distributionWebACLs { if wID == webACLID { - if d, ok := b.distributions[distID]; ok { + if d, ok := b.distributions.Get(distID); ok { cp := *d out = append(out, &cp) } diff --git a/services/cloudfront/backend_search_index.go b/services/cloudfront/backend_search_index.go index d4cebe27e..89c253ec5 100644 --- a/services/cloudfront/backend_search_index.go +++ b/services/cloudfront/backend_search_index.go @@ -108,11 +108,20 @@ func (b *InMemoryBackend) rebuildDistributionSearchIndex() { b.distSearchTokens = make(map[string]map[string]struct{}) b.distSearchInverted = make(map[string]map[string]struct{}) - for id, d := range b.distributions { - b.indexDistributionConfig(id, d.RawConfig) + for _, d := range b.distributions.All() { + b.indexDistributionConfig(d.ID, d.RawConfig) } } +// tokenReferencedByAnyDistribution reports whether searchStr appears as a whole +// token in any indexed distribution's raw config. Used by Delete* methods to +// enforce AWS's "cannot delete a resource still in use" rule (e.g. +// CachePolicyInUse, FunctionInUse) without an O(n) scan over every distribution. +// Must be called with the backend's lock already held (read or write). +func (b *InMemoryBackend) tokenReferencedByAnyDistribution(searchStr string) bool { + return len(b.distSearchInverted[searchStr]) > 0 +} + // distributionsByConfigSearch returns copies of the distributions whose raw // config contains searchStr as a whole token. Must be called with the read lock // held. @@ -124,7 +133,7 @@ func (b *InMemoryBackend) distributionsByConfigSearch(searchStr string) []*Distr out := make([]*Distribution, 0, len(ids)) for id := range ids { - if d, ok := b.distributions[id]; ok { + if d, ok := b.distributions.Get(id); ok { out = append(out, b.copyDistribution(d)) } } diff --git a/services/cloudfront/handler.go b/services/cloudfront/handler.go index decb0ee46..845fe493b 100644 --- a/services/cloudfront/handler.go +++ b/services/cloudfront/handler.go @@ -2364,38 +2364,60 @@ func notFoundCodeExtended(err error) (string, bool) { return "", false } +// errCodeMapping pairs a sentinel error with the wire HTTP status + AWS error code to +// emit when handleError sees it in an error chain. Order matters only in that the first +// match wins; since every sentinel here wraps a distinct awserr category value (never +// another entry in this table), no two entries can both match the same error, so in +// practice order is unconstrained. +// +//nolint:gochecknoglobals // package-level lookup table, analogous to EC2's errCodeLookup +var errCodeMapping = []struct { + err error + code string + status int +}{ + {ErrDistributionNotDisabled, "DistributionNotDisabled", http.StatusConflict}, + {ErrPublicKeyInUse, "PublicKeyInUse", http.StatusConflict}, + {ErrFLEProfileInUse, "FieldLevelEncryptionProfileInUse", http.StatusConflict}, + {ErrCachePolicyInUse, "CachePolicyInUse", http.StatusConflict}, + {ErrOriginRequestPolicyInUse, "OriginRequestPolicyInUse", http.StatusConflict}, + {ErrResponseHeadersPolicyInUse, "ResponseHeadersPolicyInUse", http.StatusConflict}, + {ErrFunctionInUse, "FunctionInUse", http.StatusConflict}, + {ErrCachePolicyAlreadyExists, "CachePolicyAlreadyExists", http.StatusConflict}, + {ErrOriginRequestPolicyAlreadyExists, "OriginRequestPolicyAlreadyExists", http.StatusConflict}, + {ErrResponseHeadersPolicyAlreadyExists, "ResponseHeadersPolicyAlreadyExists", http.StatusConflict}, + {ErrOriginAccessControlAlreadyExists, "OriginAccessControlAlreadyExists", http.StatusConflict}, + {ErrFunctionAlreadyExists, "FunctionAlreadyExists", http.StatusConflict}, + {ErrFLEAlreadyExists, "FieldLevelEncryptionConfigAlreadyExists", http.StatusConflict}, + {ErrFLEProfileAlreadyExists, "FieldLevelEncryptionProfileAlreadyExists", http.StatusConflict}, + {ErrPublicKeyAlreadyExists, "PublicKeyAlreadyExists", http.StatusConflict}, + {ErrKeyGroupAlreadyExists, "KeyGroupAlreadyExists", http.StatusConflict}, + {ErrRealtimeLogConfigAlreadyExists, "RealtimeLogConfigAlreadyExists", http.StatusConflict}, + {ErrAlreadyExists, "EntityAlreadyExists", http.StatusConflict}, + {ErrConnectionGroupAlreadyExists, "EntityAlreadyExists", http.StatusConflict}, + {ErrInvalidTagging, "InvalidTagging", http.StatusBadRequest}, + {ErrStreamingDistributionNotDisabled, "StreamingDistributionNotDisabled", http.StatusConflict}, + {ErrDomainConflict, "DomainConflictException", http.StatusConflict}, + {ErrInconsistentQuantities, "InconsistentQuantities", http.StatusBadRequest}, + {ErrValidation, "InvalidArgument", http.StatusBadRequest}, +} + func (h *Handler) handleError(c *echo.Context, err error) error { if code, ok := notFoundCode(err); ok { return xmlResp(c, http.StatusNotFound, cfErrorXML(code, err.Error())) } - switch { - case errors.Is(err, ErrDistributionNotDisabled): - return xmlResp(c, http.StatusConflict, cfErrorXML("DistributionNotDisabled", err.Error())) - case errors.Is(err, ErrPublicKeyInUse): - return xmlResp(c, http.StatusConflict, cfErrorXML("PublicKeyInUse", err.Error())) - case errors.Is(err, ErrFLEProfileInUse): - return xmlResp(c, http.StatusConflict, cfErrorXML("FieldLevelEncryptionProfileInUse", err.Error())) - case errors.Is(err, ErrAlreadyExists): - return xmlResp(c, http.StatusConflict, cfErrorXML("DistributionAlreadyExists", err.Error())) - case errors.Is(err, ErrConnectionGroupAlreadyExists): - return xmlResp(c, http.StatusConflict, cfErrorXML("EntityAlreadyExists", err.Error())) - case errors.Is(err, ErrInvalidTagging): - return xmlResp(c, http.StatusBadRequest, cfErrorXML("InvalidTagging", err.Error())) - case errors.Is(err, ErrStreamingDistributionNotDisabled): - return xmlResp(c, http.StatusConflict, cfErrorXML("StreamingDistributionNotDisabled", err.Error())) - case errors.Is(err, ErrDomainConflict): - return xmlResp(c, http.StatusConflict, cfErrorXML("DomainConflictException", err.Error())) - case errors.Is(err, ErrValidation): - return xmlResp(c, http.StatusBadRequest, cfErrorXML("InvalidArgument", err.Error())) - default: - - return xmlResp( - c, - http.StatusInternalServerError, - cfErrorXML("InternalFailure", err.Error()), - ) + for _, m := range errCodeMapping { + if errors.Is(err, m.err) { + return xmlResp(c, m.status, cfErrorXML(m.code, err.Error())) + } } + + return xmlResp( + c, + http.StatusInternalServerError, + cfErrorXML("InternalFailure", err.Error()), + ) } // --- Distribution handlers --- @@ -2406,6 +2428,10 @@ func (h *Handler) handleCreateDistribution(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var cfg distributionConfigMinimal if xmlErr := xml.Unmarshal(body, &cfg); xmlErr != nil { return xmlResp( @@ -2459,6 +2485,10 @@ func (h *Handler) handleUpdateDistribution(c *echo.Context, id string) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var cfg distributionConfigMinimal if xmlErr := xml.Unmarshal(body, &cfg); xmlErr != nil { return xmlResp( @@ -2922,6 +2952,10 @@ func (h *Handler) handleAssociateDistributionWebACL(c *echo.Context, distributio return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req webACLAssociationXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -2946,6 +2980,10 @@ func (h *Handler) handleAssociateDistributionTenantWebACL(c *echo.Context, tenan return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req webACLAssociationXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -2970,6 +3008,10 @@ func (h *Handler) handleCopyDistribution(c *echo.Context, primaryDistID string) return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req copyDistributionRequestXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -2998,6 +3040,10 @@ func (h *Handler) handleCreateAnycastIPList(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req anycastIPListRequestXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -3047,6 +3093,10 @@ func (h *Handler) handleCreateCachePolicy(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req cachePolicyConfigXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -3083,6 +3133,10 @@ func (h *Handler) handleCreateConnectionFunction(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req connectionFunctionRequestXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -3119,6 +3173,10 @@ func (h *Handler) handleCreateConnectionGroup(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req connectionGroupRequestXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -3163,6 +3221,10 @@ func (h *Handler) handleCreateContinuousDeploymentPolicy(c *echo.Context) error return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req continuousDeploymentPolicyConfigXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -3235,6 +3297,10 @@ func (h *Handler) handleUpdateContinuousDeploymentPolicy(c *echo.Context, id str return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req continuousDeploymentPolicyConfigXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -3350,6 +3416,10 @@ func (h *Handler) handleSetFunctionAssociations(c *echo.Context, distributionID return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req functionAssociationsXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -3378,6 +3448,10 @@ func (h *Handler) handleCreateOAI(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var cfg oaiConfigXML if xmlErr := xml.Unmarshal(body, &cfg); xmlErr != nil { return xmlResp( @@ -3523,6 +3597,10 @@ func (h *Handler) handleUpdateOAI(c *echo.Context, id string) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req oaiConfigXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -3569,6 +3647,10 @@ func (h *Handler) handleTagResource(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var tags tagsXML if xmlErr := xml.Unmarshal(body, &tags); xmlErr != nil { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "invalid Tags XML")) @@ -3668,6 +3750,10 @@ func (h *Handler) handleCreateInvalidation(c *echo.Context, distID string) error ) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var batch invalidationBatchXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &batch); xmlErr != nil { @@ -3902,6 +3988,10 @@ func (h *Handler) handleUpdateCachePolicy(c *echo.Context, id string) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req cachePolicyConfigXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -3974,6 +4064,10 @@ func (h *Handler) handleCreateOriginAccessControl(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req oacConfigXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -4081,6 +4175,10 @@ func (h *Handler) handleUpdateOriginAccessControl(c *echo.Context, id string) er return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req oacConfigXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -4226,6 +4324,10 @@ func (h *Handler) handleCreateResponseHeadersPolicy(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req rhpConfigXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -4333,6 +4435,10 @@ func (h *Handler) handleUpdateResponseHeadersPolicy(c *echo.Context, id string) return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req rhpConfigXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -4464,6 +4570,10 @@ func (h *Handler) handleCreateFunction(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req createFunctionRequestXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -4533,10 +4643,14 @@ func (h *Handler) handleListFunctions(c *echo.Context) error { `%s`+ ``+ ``+ + `%s`+ `%s`+ + `%s`+ + `%s`+ ``+ ``, - fn.Name, fn.Status, fn.Comment, fn.Runtime, fn.Status) + fn.Name, fn.Status, fn.Comment, fn.Runtime, + fn.ARN, fn.Status, fn.CreatedTime, fn.LastModifiedTime) } resp := fmt.Sprintf(``+ @@ -4601,6 +4715,10 @@ func (h *Handler) handleUpdateFunction(c *echo.Context, name string) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req createFunctionRequestXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -4688,10 +4806,14 @@ func functionResponseXML(fn *Function) string { `%s`+ ``+ ``+ + `%s`+ `%s`+ + `%s`+ + `%s`+ ``+ ``, - cfNS, fn.Name, fn.Status, fn.Comment, fn.Runtime, fn.Status) + cfNS, fn.Name, fn.Status, fn.Comment, fn.Runtime, + fn.ARN, fn.Status, fn.CreatedTime, fn.LastModifiedTime) } // --- Origin Request Policy handlers --- @@ -4753,6 +4875,10 @@ func (h *Handler) handleCreateOriginRequestPolicy(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + if len(body) == 0 { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "OriginRequestPolicyConfig body is required")) @@ -4851,6 +4977,10 @@ func (h *Handler) handleUpdateOriginRequestPolicy(c *echo.Context, id string) er return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + ifMatch := c.Request().Header.Get("If-Match") if ifMatch == "" || ifMatch != current.ETag { return xmlResp( @@ -5016,6 +5146,10 @@ func (h *Handler) handleCreateFieldLevelEncryption(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req fleConfigRequestXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -5088,6 +5222,10 @@ func (h *Handler) handleUpdateFieldLevelEncryption(c *echo.Context, id string) e return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req fleConfigRequestXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -5215,6 +5353,10 @@ func (h *Handler) handleCreateFieldLevelEncryptionProfile(c *echo.Context) error return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req fleProfileConfigRequestXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -5288,6 +5430,10 @@ func (h *Handler) handleUpdateFieldLevelEncryptionProfile(c *echo.Context, id st return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req fleProfileConfigRequestXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -5368,6 +5514,10 @@ func (h *Handler) handleCreatePublicKey(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req publicKeyConfigXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -5440,6 +5590,10 @@ func (h *Handler) handleUpdatePublicKey(c *echo.Context, id string) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req publicKeyConfigXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -5510,6 +5664,10 @@ func (h *Handler) handleCreateKeyGroup(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req keyGroupConfigXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -5582,6 +5740,10 @@ func (h *Handler) handleUpdateKeyGroup(c *echo.Context, id string) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req keyGroupConfigXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -5659,6 +5821,10 @@ func (h *Handler) handleCreateRealtimeLogConfig(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req realtimeLogConfigRequestXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -5729,6 +5895,10 @@ func (h *Handler) handleUpdateRealtimeLogConfig(c *echo.Context, arn string) err return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req realtimeLogConfigRequestXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -5793,6 +5963,10 @@ func (h *Handler) handleCreateKeyValueStore(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req keyValueStoreRequestXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -5912,6 +6086,10 @@ func (h *Handler) handleCreateVpcOrigin(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req vpcOriginRequestXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -5984,6 +6162,10 @@ func (h *Handler) handleUpdateVpcOrigin(c *echo.Context, id string) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req vpcOriginRequestXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) diff --git a/services/cloudfront/handler_batch2.go b/services/cloudfront/handler_batch2.go index 9b45d16ff..084d0e04a 100644 --- a/services/cloudfront/handler_batch2.go +++ b/services/cloudfront/handler_batch2.go @@ -72,6 +72,10 @@ func (h *Handler) handleCreateDistributionTenant(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req createDistributionTenantXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -142,6 +146,10 @@ func (h *Handler) handleUpdateDistributionTenant(c *echo.Context, id string) err return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req updateDistributionTenantXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -313,6 +321,10 @@ func (h *Handler) handleCreateDistributionWithTags(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req distributionConfigWithTagsXML if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "invalid DistributionConfigWithTags XML")) @@ -366,6 +378,10 @@ func (h *Handler) handleUpdateDistributionWithStagingConfig(c *echo.Context, pri return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req updateWithStagingConfigXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -417,6 +433,10 @@ func (h *Handler) handleUpdateDomainAssociation(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req updateDomainAssociationXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -464,6 +484,10 @@ func (h *Handler) handleVerifyDNSConfiguration(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req verifyDNSConfigurationXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -530,6 +554,10 @@ func (h *Handler) handleCreateInvalidationForTenant(c *echo.Context, tenantID st return xmlResp(c, http.StatusInternalServerError, cfErrorXML("InternalFailure", err.Error())) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var batch invalidationBatchXML if len(body) > 0 { _ = xml.Unmarshal(body, &batch) @@ -734,6 +762,10 @@ func (h *Handler) handleListDomainConflicts(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req listDomainConflictsXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -775,6 +807,10 @@ func (h *Handler) handleUpdateKeyValueStore(c *echo.Context, id string) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req keyValueStoreRequestXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) diff --git a/services/cloudfront/handler_new_ops.go b/services/cloudfront/handler_new_ops.go index f5a8161e6..8715f30c8 100644 --- a/services/cloudfront/handler_new_ops.go +++ b/services/cloudfront/handler_new_ops.go @@ -83,6 +83,10 @@ func (h *Handler) handleCreateTrustStore(c *echo.Context) error { if err != nil { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } var req createTrustStoreRequestXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -165,6 +169,10 @@ func (h *Handler) handleUpdateTrustStore(c *echo.Context, id string) error { if err != nil { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } var req trustStoreConfigXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -281,6 +289,10 @@ func (h *Handler) handleCreateStreamingDistribution(c *echo.Context) error { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var cfg streamingDistributionConfigXML if xmlErr := xml.Unmarshal(body, &cfg); xmlErr != nil { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "invalid StreamingDistributionConfig XML")) @@ -303,6 +315,10 @@ func (h *Handler) handleCreateStreamingDistributionWithTags(c *echo.Context) err return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var req streamingDistributionConfigWithTagsXML if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { return xmlResp( @@ -444,6 +460,10 @@ func (h *Handler) handleUpdateStreamingDistribution(c *echo.Context, id string) return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } + var cfg streamingDistributionConfigXML if xmlErr := xml.Unmarshal(body, &cfg); xmlErr != nil { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "invalid StreamingDistributionConfig XML")) @@ -512,6 +532,10 @@ func monitoringSubscriptionXML(ns string, ms *MonitoringSubscription) string { func (h *Handler) handleCreateMonitoringSubscription(c *echo.Context, distributionID string) error { // Check if enabled from body. body, _ := readBody(c) + + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } enabled := true if len(body) > 0 { var req struct { @@ -569,6 +593,10 @@ func (h *Handler) handlePutResourcePolicy(c *echo.Context) error { if err != nil { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } var req struct { XMLName xml.Name `xml:"ResourcePolicy"` Policy string `xml:"Policy"` @@ -691,6 +719,10 @@ func (h *Handler) handleUpdateConnectionGroup(c *echo.Context, id string) error if err != nil { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } var req updateConnectionGroupRequestXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -835,6 +867,10 @@ func (h *Handler) handleUpdateConnectionFunction(c *echo.Context, id string) err if err != nil { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } var req updateConnectionFunctionRequestXML if len(body) > 0 { if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { @@ -915,6 +951,10 @@ func (h *Handler) handleTestConnectionFunction(c *echo.Context, id string) error if err != nil { return xmlResp(c, http.StatusBadRequest, cfErrorXML("MalformedXML", "failed to read body")) } + + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } var req testConnectionFunctionRequestXML if len(body) > 0 { _ = xml.Unmarshal(body, &req) @@ -1022,6 +1062,10 @@ func (h *Handler) handleUpdateAnycastIPList(c *echo.Context, id string) error { IPCount int32 `xml:"IpCount"` } body, _ := readBody(c) + + if qErr := validateQuantities(body); qErr != nil { + return h.handleError(c, qErr) + } if len(body) > 0 { _ = xml.Unmarshal(body, &req) } diff --git a/services/cloudfront/handler_new_ops_test.go b/services/cloudfront/handler_new_ops_test.go index 4e8fb6777..5f51380c2 100644 --- a/services/cloudfront/handler_new_ops_test.go +++ b/services/cloudfront/handler_new_ops_test.go @@ -154,8 +154,9 @@ func TestNewOps_TrustStore_BundleAndStatus(t *testing.T) { } // TestNewOps_TrustStore_NameUniqueness verifies that creating a trust store with a name that -// already exists fails with 409 DistributionAlreadyExists, and that renaming to a taken name -// on update also fails. +// already exists fails with 409 EntityAlreadyExists (the generic AWS fallback code for +// resources without a dedicated AlreadyExists error type), and that renaming to a taken +// name on update also fails. func TestNewOps_TrustStore_NameUniqueness(t *testing.T) { t.Parallel() h := newCFHandler(t) diff --git a/services/cloudfront/handler_test.go b/services/cloudfront/handler_test.go index 4f72e1d32..bb67ccad3 100644 --- a/services/cloudfront/handler_test.go +++ b/services/cloudfront/handler_test.go @@ -2287,7 +2287,10 @@ func TestRefinement1_CachePolicyUniqueness(t *testing.T) { rec2 := doXML(t, h, http.MethodPost, "/2020-05-31/cache-policy", body) assert.Equal(t, http.StatusConflict, rec2.Code) - assert.Contains(t, rec2.Body.String(), "DistributionAlreadyExists") + // The real aws-sdk-go-v2 cloudfront error type for a cache policy name collision is + // CachePolicyAlreadyExists, not DistributionAlreadyExists (which the emulator used to + // return for every AlreadyExists collision regardless of resource type). + assert.Contains(t, rec2.Body.String(), "CachePolicyAlreadyExists") } // TestRefinement1_CachePolicyTTLValidation verifies TTL ordering is enforced. @@ -2752,7 +2755,11 @@ func TestRefinement1_ErrorMapping(t *testing.T) { ``, ), wantStatus: http.StatusConflict, - wantCode: "DistributionAlreadyExists", + // The real aws-sdk-go-v2 cloudfront error type for a cache policy name + // collision is CachePolicyAlreadyExists, not the generic + // DistributionAlreadyExists the emulator used to return for every + // AlreadyExists collision regardless of which resource type collided. + wantCode: "CachePolicyAlreadyExists", }, } @@ -2763,7 +2770,7 @@ func TestRefinement1_ErrorMapping(t *testing.T) { h := newTestHandler() // For the cache policy duplicate test, create it first. - if tt.wantCode == "DistributionAlreadyExists" { + if tt.wantCode == "CachePolicyAlreadyExists" { rec := doXML(t, h, http.MethodPost, tt.path, tt.body) require.Equal(t, http.StatusCreated, rec.Code) } @@ -3606,6 +3613,14 @@ func TestCloudFrontFunctionCRUD(t *testing.T) { assert.Contains(t, rec.Body.String(), "FunctionSummary") assert.Contains(t, rec.Body.String(), "my-fn") assert.NotEmpty(t, rec.Header().Get("Location")) + // FunctionMetadata.FunctionARN and LastModifiedTime are required members + // of the real aws-sdk-go-v2 FunctionMetadata shape: without FunctionARN a + // real SDK caller has no way to attach the function to a distribution's + // FunctionAssociations, since those require the ARN, not the name. + assert.Contains(t, rec.Body.String(), "") + assert.Contains(t, rec.Body.String(), "arn:aws:cloudfront") + assert.Contains(t, rec.Body.String(), "") + assert.Contains(t, rec.Body.String(), "") }, }, { diff --git a/services/cloudfront/inconsistent_quantities_test.go b/services/cloudfront/inconsistent_quantities_test.go new file mode 100644 index 000000000..83eeac673 --- /dev/null +++ b/services/cloudfront/inconsistent_quantities_test.go @@ -0,0 +1,158 @@ +package cloudfront_test + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +// Test_InconsistentQuantities_EndToEnd proves that a caller-supplied Quantity that +// disagrees with the actual number of Items in a config payload is rejected with +// AWS's InconsistentQuantities error, across several op families that carry the +// pervasive N... pattern. Real CloudFront +// validates this everywhere; because the emulator historically parsed configs by +// slice (whose length is authoritative regardless of the caller's stated Quantity), +// a bad Quantity was silently accepted. +func Test_InconsistentQuantities_EndToEnd(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + method string + path func(h *testHarness) string + body []byte + wantStatus int + wantErr bool + }{ + { + name: "CreateDistribution: Origins Quantity overstates Items", + method: http.MethodPost, + path: func(*testHarness) string { return "/2020-05-31/distribution" }, + body: []byte(`` + + `ref-iq-1` + + `c` + + `true` + + `2o1` + + ``), + wantStatus: http.StatusBadRequest, + wantErr: true, + }, + { + name: "CreateDistribution: Origins Quantity matches Items (control)", + method: http.MethodPost, + path: func(*testHarness) string { return "/2020-05-31/distribution" }, + body: []byte(`` + + `ref-iq-2` + + `c` + + `true` + + `1o1` + + ``), + wantStatus: http.StatusCreated, + wantErr: false, + }, + { + name: "CreateCachePolicy: HeadersConfig Quantity understates Items", + method: http.MethodPost, + path: func(*testHarness) string { return "/2020-05-31/cache-policy" }, + body: []byte(`` + + `iq-cp-1` + + `86400` + + `31536000` + + `0` + + `` + + `none` + + `whitelist` + + `1X-AX-B` + + `` + + `none` + + `false` + + `` + + ``), + wantStatus: http.StatusBadRequest, + wantErr: true, + }, + { + name: "CreateInvalidation: Paths Quantity overstates Items", + method: http.MethodPost, + path: func(h *testHarness) string { + return "/2020-05-31/distribution/" + h.distID + "/invalidation" + }, + body: []byte(`` + + `iq-inval-1` + + `5/a` + + ``), + wantStatus: http.StatusBadRequest, + wantErr: true, + }, + { + name: "CreateDistribution: nested AllowedMethods Quantity mismatch inside CacheBehaviors", + method: http.MethodPost, + path: func(*testHarness) string { return "/2020-05-31/distribution" }, + body: []byte(`` + + `ref-iq-3` + + `c` + + `true` + + `1o1` + + `1` + + `3` + + `GETHEAD` + + `` + + ``), + wantStatus: http.StatusBadRequest, + wantErr: true, + }, + { + name: "CreateResponseHeadersPolicy: CustomHeadersConfig Quantity understates Items", + method: http.MethodPost, + path: func(*testHarness) string { return "/2020-05-31/response-headers-policy" }, + body: []byte(`` + + `iq-rhp-1` + + `1` + + `` + + `
X-A
afalse` + + `
` + + `` + + `
X-B
bfalse` + + `
` + + `
` + + `
`), + wantStatus: http.StatusBadRequest, + wantErr: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h := newTestHandler() + harness := &testHarness{} + + d, err := h.Backend.CreateDistribution( + "iq-harness-ref-"+tc.name, "harness", true, + minimalDistConfig("iq-harness-ref-"+tc.name, "harness", true), + ) + if err == nil { + harness.distID = d.ID + } + + rec := doXML(t, h, tc.method, tc.path(harness), tc.body) + + assert.Equal(t, tc.wantStatus, rec.Code, "status code; body=%s", rec.Body.String()) + + if tc.wantErr { + assert.Contains(t, rec.Body.String(), "InconsistentQuantities") + } else { + assert.NotContains(t, rec.Body.String(), "InconsistentQuantities") + } + }) + } +} + +// testHarness carries per-subtest fixture state (e.g. a pre-created distribution ID +// for invalidation tests) without leaking it across the table-driven subtests, each +// of which gets its own fresh *cloudfront.Handler / backend. +type testHarness struct { + distID string +} diff --git a/services/cloudfront/persistence.go b/services/cloudfront/persistence.go index a826e3bf1..1cebee319 100644 --- a/services/cloudfront/persistence.go +++ b/services/cloudfront/persistence.go @@ -3,47 +3,112 @@ package cloudfront import ( "context" "encoding/json" + "fmt" + "maps" + "time" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) -type backendSnapshot struct { - Distributions map[string]*Distribution `json:"distributions"` - OAIs map[string]*OriginAccessIdentity `json:"oais"` - Invalidations map[string][]*Invalidation `json:"invalidations,omitempty"` - AnycastIPLists map[string]*AnycastIPList `json:"anycastIPLists,omitempty"` - StreamingDistributions map[string]*StreamingDistribution `json:"streamingDistributions,omitempty"` - TrustStores map[string]*TrustStore `json:"trustStores,omitempty"` +// cloudfrontSnapshotVersion identifies the shape of backendSnapshot's Tables blob +// (the set of "clean" tables on b.registry plus the DTO tables built below for +// invalidations/tenantInvalidations -- see store_setup.go's registerAllTables doc +// for the clean/dirty split). It must be bumped whenever a change there, to a DTO +// type, or to backendSnapshot itself would make an older snapshot unsafe to decode +// as the current shape. Restore compares this against the persisted value and +// discards (rather than attempts to partially decode) any mismatch -- see Restore +// below. Mirrors the services/sqs pilot (commit 0f09d77c), services/ec2 (commit +// 12e611a4), and services/apigateway (commit 6da0334e). +const cloudfrontSnapshotVersion = 1 + +// invalidationSnapshot is the DTO used ONLY for Snapshot/Restore of both +// invalidations and tenantInvalidations. It mirrors Invalidation field for field, +// except ParentID is given a real JSON tag here instead of Invalidation's +// `json:"-"` distID/tenantID (see backend.go) -- marshaling Invalidation directly +// would silently drop whichever of those was set, and unmarshaling into it would +// leave both permanently empty, corrupting the flat table's composite key on +// restore. This is the same DTO-registry technique services/apigateway uses for +// its Resource/Deployment/etc. types (commit 6da0334e), itself mirroring the +// services/sqs pilot (commit 0f09d77c). One DTO type serves both tables since +// Invalidation's shape is identical either way; toInvalidationSnapshot/ +// fromInvalidationSnapshot take the parent ID explicitly rather than reading a +// field. +type invalidationSnapshot struct { + CreateTime time.Time `json:"createTime"` + ID string `json:"id"` + Status string `json:"status"` + CallerRef string `json:"callerRef,omitempty"` + ParentID string `json:"parentId"` + Paths []string `json:"paths,omitempty"` +} - CachePolicies map[string]*CachePolicy `json:"cachePolicies,omitempty"` - ConnectionFunctions map[string]*ConnectionFunction `json:"connectionFunctions,omitempty"` - ConnectionGroups map[string]*ConnectionGroup `json:"connectionGroups,omitempty"` +func toInvalidationSnapshot(v *Invalidation, parentID string) *invalidationSnapshot { + return &invalidationSnapshot{ + ID: v.ID, + Status: v.Status, + CallerRef: v.CallerRef, + ParentID: parentID, + Paths: append([]string(nil), v.Paths...), + CreateTime: v.CreateTime, + } +} - ContinuousDeploymentPolicies map[string]*ContinuousDeploymentPolicy `json:"continuousDeploymentPolicies,omitempty"` +func fromDistInvalidationSnapshot(v *invalidationSnapshot) *Invalidation { + return &Invalidation{ + ID: v.ID, + Status: v.Status, + CallerRef: v.CallerRef, + distID: v.ParentID, + Paths: append([]string(nil), v.Paths...), + CreateTime: v.CreateTime, + } +} - OriginAccessControls map[string]*OriginAccessControl `json:"originAccessControls,omitempty"` - ResponseHeadersPolicies map[string]*ResponseHeadersPolicy `json:"responseHeadersPolicies,omitempty"` - Functions map[string]*Function `json:"functions,omitempty"` - OriginRequestPolicies map[string]*OriginRequestPolicy `json:"originRequestPolicies,omitempty"` +func fromTenantInvalidationSnapshot(v *invalidationSnapshot) *Invalidation { + return &Invalidation{ + ID: v.ID, + Status: v.Status, + CallerRef: v.CallerRef, + tenantID: v.ParentID, + Paths: append([]string(nil), v.Paths...), + CreateTime: v.CreateTime, + } +} - FieldLevelEncryptions map[string]*FieldLevelEncryption `json:"fieldLevelEncryptions,omitempty"` - FieldLevelEncryptionProfiles map[string]*FieldLevelEncryptionProfile `json:"fieldLevelEncryptionProfiles,omitempty"` +func invalidationSnapshotKey(v *invalidationSnapshot) string { + return invalidationKey(v.ParentID, v.ID) +} - PublicKeys map[string]*PublicKey `json:"publicKeys,omitempty"` - KeyGroups map[string]*KeyGroup `json:"keyGroups,omitempty"` - RealtimeLogConfigs map[string]*RealtimeLogConfig `json:"realtimeLogConfigs,omitempty"` - KeyValueStores map[string]*KeyValueStore `json:"keyValueStores,omitempty"` - VpcOrigins map[string]*VpcOrigin `json:"vpcOrigins,omitempty"` +// dirtyTableNames lists the "dirty" table names shared by Snapshot and Restore +// (see store_setup.go's registerAllTables doc). Both build an ephemeral DTO +// [store.Registry] under these exact names, so a snapshot written by one version +// of Snapshot always lines up with the same version of Restore. +// +//nolint:gochecknoglobals // fixed lookup table, mirrors errCodeLookup-style tables elsewhere +var dirtyTableNames = struct { + invalidations, tenantInvalidations string +}{ + invalidations: "invalidations", + tenantInvalidations: "tenantInvalidations", +} + +// backendSnapshot is the top-level on-disk shape for the CloudFront backend. +// +// Tables holds one JSON-encoded array per table -- both the "clean" tables +// registered on b.registry (produced by [store.Registry.SnapshotAll]) and the +// "dirty" DTO tables built inline in Snapshot (see store_setup.go's +// registerAllTables doc for the clean/dirty split), merged into one map so a +// single Tables blob round-trips the whole backend. +type backendSnapshot struct { + Tables map[string]json.RawMessage `json:"tables"` DistributionFunctionAssociations map[string][]FunctionAssociation `json:"distributionFunctionAssociations,omitempty"` DistributionAliases map[string][]string `json:"distributionAliases,omitempty"` DistributionWebACLs map[string]string `json:"distributionWebACLs,omitempty"` DistributionTenantWebACLs map[string]string `json:"distributionTenantWebACLs,omitempty"` - DistributionTenants map[string]*DistributionTenant `json:"distributionTenants,omitempty"` - TenantInvalidations map[string][]*Invalidation `json:"tenantInvalidations,omitempty"` - MonitoringSubscriptions map[string]*MonitoringSubscription `json:"monitoringSubscriptions,omitempty"` ResourcePolicies map[string]*resourcePolicyEntry `json:"resourcePolicies,omitempty"` ManagedCertificates map[string]*ManagedCertificateDetails `json:"managedCertificates,omitempty"` @@ -55,41 +120,56 @@ type backendSnapshot struct { AccountID string `json:"accountId"` Region string `json:"region"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. +// It implements persistence.Persistable. func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad input + // data. Log and skip the snapshot rather than panic, matching the + // persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "cloudfront: snapshot table marshal failed", "error", err) + + return nil + } + + dtoReg := store.NewRegistry() + invalidationDTOs := store.Register(dtoReg, dirtyTableNames.invalidations, store.New(invalidationSnapshotKey)) + tenantInvalidationDTOs := store.Register( + dtoReg, dirtyTableNames.tenantInvalidations, store.New(invalidationSnapshotKey), + ) + + for _, inv := range b.invalidations.Snapshot() { + invalidationDTOs.Put(toInvalidationSnapshot(inv, inv.distID)) + } + + for _, inv := range b.tenantInvalidations.Snapshot() { + tenantInvalidationDTOs.Put(toInvalidationSnapshot(inv, inv.tenantID)) + } + + dirtyTables, err := dtoReg.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, "cloudfront: snapshot DTO table marshal failed", "error", err) + + return nil + } + + maps.Copy(tables, dirtyTables) + snap := backendSnapshot{ - Distributions: b.distributions, - OAIs: b.oais, - Invalidations: b.invalidations, - AnycastIPLists: b.anycastIPLists, - StreamingDistributions: b.streamingDistributions, - TrustStores: b.trustStores, - CachePolicies: b.cachePolicies, - ConnectionFunctions: b.connectionFunctions, - ConnectionGroups: b.connectionGroups, - ContinuousDeploymentPolicies: b.continuousDeploymentPolicies, - OriginAccessControls: b.originAccessControls, - ResponseHeadersPolicies: b.responseHeadersPolicies, - Functions: b.functions, - OriginRequestPolicies: b.originRequestPolicies, - FieldLevelEncryptions: b.fieldLevelEncryptions, - FieldLevelEncryptionProfiles: b.fieldLevelEncryptionProfiles, - PublicKeys: b.publicKeys, - KeyGroups: b.keyGroups, - RealtimeLogConfigs: b.realtimeLogConfigs, - KeyValueStores: b.keyValueStores, - VpcOrigins: b.vpcOrigins, + Version: cloudfrontSnapshotVersion, + Tables: tables, DistributionFunctionAssociations: b.distributionFunctionAssociations, DistributionAliases: b.distributionAliases, DistributionWebACLs: b.distributionWebACLs, DistributionTenantWebACLs: b.distributionTenantWebACLs, - DistributionTenants: b.distributionTenants, - TenantInvalidations: b.tenantInvalidations, MonitoringSubscriptions: b.monitoringSubscriptions, ResourcePolicies: b.resourcePolicies, ManagedCertificates: b.managedCertificates, @@ -103,7 +183,6 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { data, err := json.Marshal(snap) if err != nil { - // Log the marshal failure so operators can detect data-loss scenarios. logger.Load(ctx).WarnContext(ctx, "cloudfront: Snapshot marshal failure", "error", err) return nil @@ -113,6 +192,105 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { } // Restore loads backend state from a JSON snapshot and rebuilds derived indexes. +// It implements persistence.Persistable. +func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { + var snap backendSnapshot + + if err := persistence.UnmarshalSnapshot(ctx, "cloudfront", data, &snap); err != nil { + return err + } + + b.mu.Lock("Restore") + defer b.mu.Unlock() + + if snap.Version != cloudfrontSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/sqs pilot (commit 0f09d77c). + logger.Load(ctx).WarnContext(ctx, + "cloudfront: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", cloudfrontSnapshotVersion) + + b.registry.ResetAll() + b.invalidations.Reset() + b.tenantInvalidations.Reset() + b.resetDistributions() + b.resetPoliciesAndKeys() + b.rebuildDistributionSearchIndex() + b.accountID = snap.AccountID + b.region = snap.Region + + return nil + } + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("cloudfront: restore snapshot tables: %w", err) + } + + if err := b.restoreDirtyTables(snap.Tables); err != nil { + return err + } + + ensureNonNil(&snap) + idx := rebuildIndexes(&snap) + b.restoreAssociationMaps(&snap) + b.restoreIndexes(&idx) + b.rebuildDistributionSearchIndex() + b.accountID = snap.AccountID + b.region = snap.Region + + return nil +} + +// restoreDirtyTables loads the invalidations/tenantInvalidations tables (not on +// b.registry -- see store_setup.go's registerAllTables doc) from an ephemeral DTO +// [store.Registry] built from tables, converting each DTO back to a live +// Invalidation. Must be called with the lock held. +func (b *InMemoryBackend) restoreDirtyTables(tables map[string]json.RawMessage) error { + dtoReg := store.NewRegistry() + invalidationDTOs := store.Register(dtoReg, dirtyTableNames.invalidations, store.New(invalidationSnapshotKey)) + tenantInvalidationDTOs := store.Register( + dtoReg, dirtyTableNames.tenantInvalidations, store.New(invalidationSnapshotKey), + ) + + if err := dtoReg.RestoreAll(tables); err != nil { + return fmt.Errorf("cloudfront: restore snapshot DTO tables: %w", err) + } + + invs := make([]*Invalidation, 0, invalidationDTOs.Len()) + for _, v := range invalidationDTOs.All() { + invs = append(invs, fromDistInvalidationSnapshot(v)) + } + b.invalidations.Restore(invs) + + tenantInvs := make([]*Invalidation, 0, tenantInvalidationDTOs.Len()) + for _, v := range tenantInvalidationDTOs.All() { + tenantInvs = append(tenantInvs, fromTenantInvalidationSnapshot(v)) + } + b.tenantInvalidations.Restore(tenantInvs) + + return nil +} + +// restoreAssociationMaps assigns the plain (non-store.Table) association maps +// from a snapshot onto the backend. Must be called with the lock held. +func (b *InMemoryBackend) restoreAssociationMaps(snap *backendSnapshot) { + b.distributionFunctionAssociations = snap.DistributionFunctionAssociations + b.distributionAliases = snap.DistributionAliases + b.distributionWebACLs = snap.DistributionWebACLs + b.distributionTenantWebACLs = snap.DistributionTenantWebACLs + b.monitoringSubscriptions = snap.MonitoringSubscriptions + b.resourcePolicies = snap.ResourcePolicies + b.managedCertificates = snap.ManagedCertificates + b.distributionCachePolicies = snap.DistributionCachePolicies + b.distributionOriginRequestPolicies = snap.DistributionOriginRequestPolicies + b.distributionResponseHeadersPolicies = snap.DistributionResponseHeadersPolicies + b.distributionRealtimeLogConfigs = snap.DistributionRealtimeLogConfigs +} + // backendIndexes holds the derived lookup indexes rebuilt from a snapshot. type backendIndexes struct { distributionARNs map[string]string @@ -148,23 +326,26 @@ type backendIndexes struct { func rebuildDistributionIndexes( snap *backendSnapshot, ) (map[string]string, map[string]string, map[string]string, map[string]string) { - arnIndex := make(map[string]string, len(snap.Distributions)) - callerRefIndex := make(map[string]string, len(snap.Distributions)) + distributions := tableFrom[Distribution](snap, "distributions") + streamingDistributions := tableFrom[StreamingDistribution](snap, "streamingDistributions") - for id, d := range snap.Distributions { - arnIndex[d.ARN] = id + arnIndex := make(map[string]string, len(distributions)) + callerRefIndex := make(map[string]string, len(distributions)) + + for _, d := range distributions { + arnIndex[d.ARN] = d.ID if d.CallerReference != "" { - callerRefIndex[d.CallerReference] = id + callerRefIndex[d.CallerReference] = d.ID } } - sdARNIndex := make(map[string]string, len(snap.StreamingDistributions)) - sdCallerRefIndex := make(map[string]string, len(snap.StreamingDistributions)) + sdARNIndex := make(map[string]string, len(streamingDistributions)) + sdCallerRefIndex := make(map[string]string, len(streamingDistributions)) - for id, sd := range snap.StreamingDistributions { - sdARNIndex[sd.ARN] = id + for _, sd := range streamingDistributions { + sdARNIndex[sd.ARN] = sd.ID if sd.Config.CallerReference != "" { - sdCallerRefIndex[sd.Config.CallerReference] = id + sdCallerRefIndex[sd.Config.CallerReference] = sd.ID } } @@ -173,16 +354,18 @@ func rebuildDistributionIndexes( // rebuildTenantIndexes derives the ARN and domain indexes for distribution tenants. func rebuildTenantIndexes(snap *backendSnapshot) (map[string]string, map[string]string) { - tenantARNIndex := make(map[string]string, len(snap.DistributionTenants)) - tenantByDomain := make(map[string]string, len(snap.DistributionTenants)) + tenants := tableFrom[DistributionTenant](snap, "distributionTenants") + + tenantARNIndex := make(map[string]string, len(tenants)) + tenantByDomain := make(map[string]string, len(tenants)) - for id, t := range snap.DistributionTenants { - tenantARNIndex[t.ARN] = id + for _, t := range tenants { + tenantARNIndex[t.ARN] = t.ID for _, d := range t.Domains { - tenantByDomain[d] = id + tenantByDomain[d] = t.ID } if t.Domain != "" { - tenantByDomain[t.Domain] = id + tenantByDomain[t.Domain] = t.ID } } @@ -192,59 +375,59 @@ func rebuildTenantIndexes(snap *backendSnapshot) (map[string]string, map[string] // rebuildNameIndexes derives the various name → ID lookup indexes for policy- and key-like // resources that are uniqued by name. func rebuildNameIndexes(snap *backendSnapshot) backendIndexes { - cachePolicyByName := make(map[string]string, len(snap.CachePolicies)) - for id, cp := range snap.CachePolicies { - cachePolicyByName[cp.Name] = id + cachePolicyByName := make(map[string]string) + for _, cp := range tableFrom[CachePolicy](snap, "cachePolicies") { + cachePolicyByName[cp.Name] = cp.ID } - oacByName := make(map[string]string, len(snap.OriginAccessControls)) - for id, oac := range snap.OriginAccessControls { - oacByName[oac.Name] = id + oacByName := make(map[string]string) + for _, oac := range tableFrom[OriginAccessControl](snap, "originAccessControls") { + oacByName[oac.Name] = oac.ID } - rhpByName := make(map[string]string, len(snap.ResponseHeadersPolicies)) - for id, p := range snap.ResponseHeadersPolicies { - rhpByName[p.Name] = id + rhpByName := make(map[string]string) + for _, p := range tableFrom[ResponseHeadersPolicy](snap, "responseHeadersPolicies") { + rhpByName[p.Name] = p.ID } - orpByName := make(map[string]string, len(snap.OriginRequestPolicies)) - for id, p := range snap.OriginRequestPolicies { - orpByName[p.Name] = id + orpByName := make(map[string]string) + for _, p := range tableFrom[OriginRequestPolicy](snap, "originRequestPolicies") { + orpByName[p.Name] = p.ID } - fleByName := make(map[string]string, len(snap.FieldLevelEncryptions)) - for id, fle := range snap.FieldLevelEncryptions { - fleByName[fle.Name] = id + fleByName := make(map[string]string) + for _, fle := range tableFrom[FieldLevelEncryption](snap, "fieldLevelEncryptions") { + fleByName[fle.Name] = fle.ID } - flePByName := make(map[string]string, len(snap.FieldLevelEncryptionProfiles)) - for id, p := range snap.FieldLevelEncryptionProfiles { - flePByName[p.Name] = id + flePByName := make(map[string]string) + for _, p := range tableFrom[FieldLevelEncryptionProfile](snap, "fieldLevelEncryptionProfiles") { + flePByName[p.Name] = p.ID } - pkByName := make(map[string]string, len(snap.PublicKeys)) - for id, pk := range snap.PublicKeys { - pkByName[pk.Name] = id + pkByName := make(map[string]string) + for _, pk := range tableFrom[PublicKey](snap, "publicKeys") { + pkByName[pk.Name] = pk.ID } - kgByName := make(map[string]string, len(snap.KeyGroups)) - for id, kg := range snap.KeyGroups { - kgByName[kg.Name] = id + kgByName := make(map[string]string) + for _, kg := range tableFrom[KeyGroup](snap, "keyGroups") { + kgByName[kg.Name] = kg.ID } - rlcByName := make(map[string]string, len(snap.RealtimeLogConfigs)) - for arn, rlc := range snap.RealtimeLogConfigs { - rlcByName[rlc.Name] = arn + rlcByName := make(map[string]string) + for _, rlc := range tableFrom[RealtimeLogConfig](snap, "realtimeLogConfigs") { + rlcByName[rlc.Name] = rlc.ARN } - kvsByName := make(map[string]string, len(snap.KeyValueStores)) - for id, kvs := range snap.KeyValueStores { - kvsByName[kvs.Name] = id + kvsByName := make(map[string]string) + for _, kvs := range tableFrom[KeyValueStore](snap, "keyValueStores") { + kvsByName[kvs.Name] = kvs.ID } - cfnByName := make(map[string]string, len(snap.ConnectionFunctions)) - for id, fn := range snap.ConnectionFunctions { - cfnByName[fn.Name] = id + cfnByName := make(map[string]string) + for _, fn := range tableFrom[ConnectionFunction](snap, "connectionFunctions") { + cfnByName[fn.Name] = fn.ID } return backendIndexes{ @@ -266,47 +449,46 @@ func rebuildNameIndexes(snap *backendSnapshot) backendIndexes { func rebuildIndexes(snap *backendSnapshot) backendIndexes { arnIndex, callerRefIndex, sdARNIndex, sdCallerRefIndex := rebuildDistributionIndexes(snap) - oaiCallerRefIndex := make(map[string]string, len(snap.OAIs)) - - for id, oai := range snap.OAIs { + oaiCallerRefIndex := make(map[string]string) + for _, oai := range tableFrom[OriginAccessIdentity](snap, "oais") { if oai.CallerReference != "" { - oaiCallerRefIndex[oai.CallerReference] = id + oaiCallerRefIndex[oai.CallerReference] = oai.ID } } - trustStoreARNIndex := make(map[string]string, len(snap.TrustStores)) - trustStoreByName := make(map[string]string, len(snap.TrustStores)) + trustStoreARNIndex := make(map[string]string) + trustStoreByName := make(map[string]string) - for id, ts := range snap.TrustStores { - trustStoreARNIndex[ts.ARN] = id - trustStoreByName[ts.Name] = id + for _, ts := range tableFrom[TrustStore](snap, "trustStores") { + trustStoreARNIndex[ts.ARN] = ts.ID + trustStoreByName[ts.Name] = ts.ID } tenantARNIndex, tenantByDomain := rebuildTenantIndexes(snap) idx := rebuildNameIndexes(snap) - connectionGroupARNIndex := make(map[string]string, len(snap.ConnectionGroups)) - connectionGroupByNameIndex := make(map[string]string, len(snap.ConnectionGroups)) - connectionGroupByRoutingEndpointIndex := make(map[string]string, len(snap.ConnectionGroups)) + connectionGroupARNIndex := make(map[string]string) + connectionGroupByNameIndex := make(map[string]string) + connectionGroupByRoutingEndpointIndex := make(map[string]string) - for id, cg := range snap.ConnectionGroups { - connectionGroupARNIndex[cg.ARN] = id - connectionGroupByNameIndex[cg.Name] = id + for _, cg := range tableFrom[ConnectionGroup](snap, "connectionGroups") { + connectionGroupARNIndex[cg.ARN] = cg.ID + connectionGroupByNameIndex[cg.Name] = cg.ID if cg.RoutingEndpoint != "" { - connectionGroupByRoutingEndpointIndex[cg.RoutingEndpoint] = id + connectionGroupByRoutingEndpointIndex[cg.RoutingEndpoint] = cg.ID } } - connectionFunctionARNIndex := make(map[string]string, len(snap.ConnectionFunctions)) - for id, fn := range snap.ConnectionFunctions { - connectionFunctionARNIndex[fn.ARN] = id + connectionFunctionARNIndex := make(map[string]string) + for _, fn := range tableFrom[ConnectionFunction](snap, "connectionFunctions") { + connectionFunctionARNIndex[fn.ARN] = fn.ID } - anycastIPListARNIndex := make(map[string]string, len(snap.AnycastIPLists)) - anycastIPListByNameIndex := make(map[string]string, len(snap.AnycastIPLists)) - for id, ail := range snap.AnycastIPLists { - anycastIPListARNIndex[ail.ARN] = id - anycastIPListByNameIndex[ail.Name] = id + anycastIPListARNIndex := make(map[string]string) + anycastIPListByNameIndex := make(map[string]string) + for _, ail := range tableFrom[AnycastIPList](snap, "anycastIPLists") { + anycastIPListARNIndex[ail.ARN] = ail.ID + anycastIPListByNameIndex[ail.Name] = ail.ID } idx.distributionARNs = arnIndex @@ -328,64 +510,25 @@ func rebuildIndexes(snap *backendSnapshot) backendIndexes { return idx } -func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { - var snap backendSnapshot - - if err := persistence.UnmarshalSnapshot(ctx, "cloudfront", data, &snap); err != nil { - return err +// tableFrom unmarshals the named "clean" table (see store_setup.go's +// registerAllTables doc) out of a snapshot's raw Tables blob, for deriving the +// secondary lookup indexes below. A missing or malformed entry yields an empty +// slice rather than an error: b.registry.RestoreAll (called before +// rebuildIndexes in Restore) is the authoritative loader and has already +// validated/loaded the same data onto the live tables, so this is purely a +// second, read-only pass over already-trusted bytes to derive indexes. +func tableFrom[T any](snap *backendSnapshot, name string) []*T { + raw, ok := snap.Tables[name] + if !ok { + return nil } - b.mu.Lock("Restore") - defer b.mu.Unlock() - - ensureNonNil(&snap) - idx := rebuildIndexes(&snap) - b.restoreCollections(&snap) - b.restoreIndexes(&idx) - b.rebuildDistributionSearchIndex() - b.accountID = snap.AccountID - b.region = snap.Region + var out []*T + if err := json.Unmarshal(raw, &out); err != nil { + return nil + } - return nil -} - -// restoreCollections assigns the primary resource collections from a snapshot onto the backend. -// Must be called with the lock held. -func (b *InMemoryBackend) restoreCollections(snap *backendSnapshot) { - b.distributions = snap.Distributions - b.oais = snap.OAIs - b.invalidations = snap.Invalidations - b.anycastIPLists = snap.AnycastIPLists - b.streamingDistributions = snap.StreamingDistributions - b.trustStores = snap.TrustStores - b.cachePolicies = snap.CachePolicies - b.connectionFunctions = snap.ConnectionFunctions - b.connectionGroups = snap.ConnectionGroups - b.continuousDeploymentPolicies = snap.ContinuousDeploymentPolicies - b.originAccessControls = snap.OriginAccessControls - b.responseHeadersPolicies = snap.ResponseHeadersPolicies - b.functions = snap.Functions - b.originRequestPolicies = snap.OriginRequestPolicies - b.fieldLevelEncryptions = snap.FieldLevelEncryptions - b.fieldLevelEncryptionProfiles = snap.FieldLevelEncryptionProfiles - b.publicKeys = snap.PublicKeys - b.keyGroups = snap.KeyGroups - b.realtimeLogConfigs = snap.RealtimeLogConfigs - b.keyValueStores = snap.KeyValueStores - b.vpcOrigins = snap.VpcOrigins - b.distributionFunctionAssociations = snap.DistributionFunctionAssociations - b.distributionAliases = snap.DistributionAliases - b.distributionWebACLs = snap.DistributionWebACLs - b.distributionTenantWebACLs = snap.DistributionTenantWebACLs - b.distributionTenants = snap.DistributionTenants - b.tenantInvalidations = snap.TenantInvalidations - b.monitoringSubscriptions = snap.MonitoringSubscriptions - b.resourcePolicies = snap.ResourcePolicies - b.managedCertificates = snap.ManagedCertificates - b.distributionCachePolicies = snap.DistributionCachePolicies - b.distributionOriginRequestPolicies = snap.DistributionOriginRequestPolicies - b.distributionResponseHeadersPolicies = snap.DistributionResponseHeadersPolicies - b.distributionRealtimeLogConfigs = snap.DistributionRealtimeLogConfigs + return out } // restoreIndexes assigns the derived lookup indexes onto the backend. Must be called with the @@ -422,45 +565,9 @@ func (b *InMemoryBackend) restoreIndexes(idx *backendIndexes) { // ensureNonNil initialises any nil maps in a snapshot to empty maps so that // the backend never holds nil map references. func ensureNonNil(snap *backendSnapshot) { - ensureNonNilBaseEntities(snap) - ensureNonNilPolicies(snap) - ensureNonNilNewResources(snap) ensureNonNilTenantExtras(snap) + ensureNonNilNewResources(snap) ensureNonNilDistributionPolicyMaps(snap) -} - -func ensureNonNilBaseEntities(snap *backendSnapshot) { - if snap.Distributions == nil { - snap.Distributions = make(map[string]*Distribution) - } - - if snap.OAIs == nil { - snap.OAIs = make(map[string]*OriginAccessIdentity) - } - - if snap.Invalidations == nil { - snap.Invalidations = make(map[string][]*Invalidation) - } - - if snap.AnycastIPLists == nil { - snap.AnycastIPLists = make(map[string]*AnycastIPList) - } - - if snap.StreamingDistributions == nil { - snap.StreamingDistributions = make(map[string]*StreamingDistribution) - } - - if snap.TrustStores == nil { - snap.TrustStores = make(map[string]*TrustStore) - } - - if snap.ConnectionFunctions == nil { - snap.ConnectionFunctions = make(map[string]*ConnectionFunction) - } - - if snap.ConnectionGroups == nil { - snap.ConnectionGroups = make(map[string]*ConnectionGroup) - } if snap.DistributionAliases == nil { snap.DistributionAliases = make(map[string][]string) @@ -473,14 +580,6 @@ func ensureNonNilBaseEntities(snap *backendSnapshot) { if snap.DistributionTenantWebACLs == nil { snap.DistributionTenantWebACLs = make(map[string]string) } - - if snap.DistributionTenants == nil { - snap.DistributionTenants = make(map[string]*DistributionTenant) - } - - if snap.TenantInvalidations == nil { - snap.TenantInvalidations = make(map[string][]*Invalidation) - } } // ensureNonNilTenantExtras initialises the per-distribution-tenant and per-distribution maps @@ -499,61 +598,7 @@ func ensureNonNilTenantExtras(snap *backendSnapshot) { } } -func ensureNonNilPolicies(snap *backendSnapshot) { - if snap.CachePolicies == nil { - snap.CachePolicies = make(map[string]*CachePolicy) - } - - if snap.ContinuousDeploymentPolicies == nil { - snap.ContinuousDeploymentPolicies = make(map[string]*ContinuousDeploymentPolicy) - } - - if snap.OriginAccessControls == nil { - snap.OriginAccessControls = make(map[string]*OriginAccessControl) - } - - if snap.ResponseHeadersPolicies == nil { - snap.ResponseHeadersPolicies = make(map[string]*ResponseHeadersPolicy) - } - - if snap.Functions == nil { - snap.Functions = make(map[string]*Function) - } - - if snap.OriginRequestPolicies == nil { - snap.OriginRequestPolicies = make(map[string]*OriginRequestPolicy) - } -} - func ensureNonNilNewResources(snap *backendSnapshot) { - if snap.FieldLevelEncryptions == nil { - snap.FieldLevelEncryptions = make(map[string]*FieldLevelEncryption) - } - - if snap.FieldLevelEncryptionProfiles == nil { - snap.FieldLevelEncryptionProfiles = make(map[string]*FieldLevelEncryptionProfile) - } - - if snap.PublicKeys == nil { - snap.PublicKeys = make(map[string]*PublicKey) - } - - if snap.KeyGroups == nil { - snap.KeyGroups = make(map[string]*KeyGroup) - } - - if snap.RealtimeLogConfigs == nil { - snap.RealtimeLogConfigs = make(map[string]*RealtimeLogConfig) - } - - if snap.KeyValueStores == nil { - snap.KeyValueStores = make(map[string]*KeyValueStore) - } - - if snap.VpcOrigins == nil { - snap.VpcOrigins = make(map[string]*VpcOrigin) - } - if snap.DistributionFunctionAssociations == nil { snap.DistributionFunctionAssociations = make(map[string][]FunctionAssociation) } diff --git a/services/cloudfront/quantity_validation.go b/services/cloudfront/quantity_validation.go new file mode 100644 index 000000000..722806f6b --- /dev/null +++ b/services/cloudfront/quantity_validation.go @@ -0,0 +1,93 @@ +package cloudfront + +import ( + "encoding/xml" + "fmt" + "strconv" + "strings" +) + +// quantityNode is a generic XML element used to walk an arbitrary CloudFront config +// document (DistributionConfig, CachePolicyConfig, StreamingDistributionConfig, ...) +// looking for the pervasive +// +// N... +// +// pattern, without needing a fully-typed Go struct for every nested list in the +// CloudFront schema. AWS validates every one of these pairings and rejects a +// mismatch with InconsistentQuantities; because Go slices carry their own length, +// it is easy for an emulator to accept a caller-supplied Quantity that disagrees +// with the actual Items and silently ignore the mismatch. +type quantityNode struct { + XMLName xml.Name + Content string `xml:",chardata"` + Nodes []quantityNode `xml:",any"` +} + +// validateQuantities parses rawConfig as generic XML and verifies every +// Quantity/Items pairing found anywhere in the document is internally consistent. +// It returns an error wrapping ErrInconsistentQuantities describing the first +// mismatch found, or nil if the document is consistent (or not parseable as XML -- +// malformed XML is the caller's own strict typed-unmarshal's job to report, as +// MalformedXML, not this generic pass's). +func validateQuantities(rawConfig []byte) error { + if len(rawConfig) == 0 { + return nil + } + + var root quantityNode + //nolint:musttag,nilerr // deliberately generic: any element shape is accepted, and + // malformed/unrecognized XML here is intentionally not an error for this pass -- + // the caller's own strict typed-unmarshal already reports MalformedXML. + if err := xml.Unmarshal(rawConfig, &root); err != nil { + return nil + } + + return checkQuantityNode(&root) +} + +// checkQuantityNode recursively checks n and its descendants for Quantity/Items +// mismatches, depth-first, returning the first one found. +func checkQuantityNode(n *quantityNode) error { + var ( + quantity int + hasQuantity bool + items *quantityNode + ) + + for i := range n.Nodes { + child := &n.Nodes[i] + + switch child.XMLName.Local { + case "Quantity": + if v, err := strconv.Atoi(strings.TrimSpace(child.Content)); err == nil { + quantity, hasQuantity = v, true + } + case "Items": + items = child + } + } + + if hasQuantity { + actual := 0 + if items != nil { + actual = len(items.Nodes) + } + + if quantity != actual { + return fmt.Errorf( + "%w: the parameter Quantity (%d) for the %s list does not match "+ + "the number of Items provided (%d)", + ErrInconsistentQuantities, quantity, n.XMLName.Local, actual, + ) + } + } + + for i := range n.Nodes { + if err := checkQuantityNode(&n.Nodes[i]); err != nil { + return err + } + } + + return nil +} diff --git a/services/cloudfront/resource_in_use_test.go b/services/cloudfront/resource_in_use_test.go new file mode 100644 index 000000000..b73155d97 --- /dev/null +++ b/services/cloudfront/resource_in_use_test.go @@ -0,0 +1,211 @@ +package cloudfront_test + +import ( + "net/http" + "strings" + "testing" + + "github.com/blackbirdworks/gopherstack/services/cloudfront" +) + +// Test_ResourceInUse_BlocksDelete proves that CachePolicy, OriginRequestPolicy, and +// ResponseHeadersPolicy deletes are rejected with AWS's resource-specific *InUse error +// while a distribution's stored config still references the resource, and that the +// delete succeeds once the reference is gone. Before this fix, none of these Delete +// operations checked for in-use references at all: a policy could be deleted out from +// under a live distribution, which real CloudFront forbids. +func Test_ResourceInUse_BlocksDelete(t *testing.T) { + t.Parallel() + + const prefix = "/2020-05-31/" + + cases := []struct { + deletePathFor func(resourceID string) string + createResourcePath string + createResourceBody string + configField string + wantInUseCode string + }{ + { + createResourcePath: prefix + "cache-policy", + createResourceBody: `iu-cp` + + `86400315360000` + + ``, + configField: "CachePolicyId", + deletePathFor: func(id string) string { return prefix + "cache-policy/" + id }, + wantInUseCode: "CachePolicyInUse", + }, + { + createResourcePath: prefix + "origin-request-policy", + createResourceBody: `iu-orp`, + configField: "OriginRequestPolicyId", + deletePathFor: func(id string) string { return prefix + "origin-request-policy/" + id }, + wantInUseCode: "OriginRequestPolicyInUse", + }, + { + createResourcePath: prefix + "response-headers-policy", + createResourceBody: `iu-rhp`, + configField: "ResponseHeadersPolicyId", + deletePathFor: func(id string) string { return prefix + "response-headers-policy/" + id }, + wantInUseCode: "ResponseHeadersPolicyInUse", + }, + } + + for _, tc := range cases { + t.Run(tc.wantInUseCode, func(t *testing.T) { + t.Parallel() + h := newCFHandler(t) + + createRR := cfRequest(t, h, http.MethodPost, tc.createResourcePath, tc.createResourceBody) + if createRR.Code != http.StatusCreated { + t.Fatalf("expected 201 on create, got %d: %s", createRR.Code, createRR.Body.String()) + } + + resourceID := extractXMLID(t, createRR.Body.String()) + if resourceID == "" { + t.Fatalf("expected non-empty resource ID from create, got: %s", createRR.Body.String()) + } + + etag := createRR.Header().Get("ETag") + if etag == "" { + t.Fatalf("expected non-empty ETag from create") + } + + distBody := `` + + `cr-iu-` + tc.wantInUseCode + `` + + `true` + + `<` + tc.configField + `>` + resourceID + `` + + `` + distResp := cfOK(t, h, http.MethodPost, prefix+"distribution", distBody) + distID := extractXMLID(t, distResp) + if distID == "" { + t.Fatalf("expected non-empty distribution ID, got: %s", distResp) + } + + // Still referenced: delete must fail with the resource-specific InUse code. + blockedRR := cfRequestWithHeader( + t, h, http.MethodDelete, tc.deletePathFor(resourceID), map[string]string{"If-Match": etag}, + ) + if blockedRR.Code != http.StatusConflict { + t.Fatalf("expected 409 while in use, got %d: %s", blockedRR.Code, blockedRR.Body.String()) + } + if !strings.Contains(blockedRR.Body.String(), tc.wantInUseCode) { + t.Fatalf("expected %s, got: %s", tc.wantInUseCode, blockedRR.Body.String()) + } + + disableAndDeleteDistribution(t, h, distID) + + // No longer referenced: delete must now succeed. + freeRR := cfRequestWithHeader( + t, h, http.MethodDelete, tc.deletePathFor(resourceID), map[string]string{"If-Match": etag}, + ) + if freeRR.Code != http.StatusNoContent { + t.Fatalf("expected 204 once unreferenced, got %d: %s", freeRR.Code, freeRR.Body.String()) + } + }) + } +} + +// Test_FunctionInUse_BlocksDelete proves DeleteFunction rejects a function still +// associated with a distribution's cache behavior via FunctionAssociations. +func Test_FunctionInUse_BlocksDelete(t *testing.T) { + t.Parallel() + + const prefix = "/2020-05-31/" + h := newCFHandler(t) + + createRR := cfRequest(t, h, http.MethodPost, prefix+"function", ``+ + `iu-fn`+ + `ccloudfront-js-2.0`+ + `ZnVuY3Rpb24gaGFuZGxlcihldmVudCkge3JldHVybiBldmVudC5yZXF1ZXN0O30=`+ + ``) + if createRR.Code != http.StatusCreated { + t.Fatalf("expected 201 on create function, got %d: %s", createRR.Code, createRR.Body.String()) + } + + fnARN := extractBetween(createRR.Body.String(), "", "") + if fnARN == "" { + t.Fatalf("expected non-empty FunctionARN from create, got: %s", createRR.Body.String()) + } + + fnETag := createRR.Header().Get("ETag") + if fnETag == "" { + t.Fatalf("expected non-empty ETag from create function") + } + + distBody := `` + + `cr-iu-fn` + + `true` + + `1` + + `` + fnARN + `viewer-request` + + `` + + `` + distResp := cfOK(t, h, http.MethodPost, prefix+"distribution", distBody) + distID := extractXMLID(t, distResp) + if distID == "" { + t.Fatalf("expected non-empty distribution ID, got: %s", distResp) + } + + blockedRR := cfRequestWithHeader( + t, h, http.MethodDelete, prefix+"function/iu-fn", map[string]string{"If-Match": fnETag}, + ) + if blockedRR.Code != http.StatusConflict { + t.Fatalf("expected 409 while function in use, got %d: %s", blockedRR.Code, blockedRR.Body.String()) + } + if !strings.Contains(blockedRR.Body.String(), "FunctionInUse") { + t.Fatalf("expected FunctionInUse, got: %s", blockedRR.Body.String()) + } + + disableAndDeleteDistribution(t, h, distID) + + freeRR := cfRequestWithHeader( + t, h, http.MethodDelete, prefix+"function/iu-fn", map[string]string{"If-Match": fnETag}, + ) + if freeRR.Code != http.StatusNoContent { + t.Fatalf("expected 204 once unreferenced, got %d: %s", freeRR.Code, freeRR.Body.String()) + } +} + +// disableAndDeleteDistribution disables distID (Enabled=false, satisfying If-Match on +// its current ETag) and then deletes it, mirroring the real AWS two-step teardown +// (CloudFront refuses to delete an enabled distribution). +func disableAndDeleteDistribution(t *testing.T, h *cloudfront.Handler, distID string) { + t.Helper() + + const prefix = "/2020-05-31/" + + getRR := cfRequestWithHeader(t, h, http.MethodGet, prefix+"distribution/"+distID, nil) + if getRR.Code != http.StatusOK { + t.Fatalf("expected 200 on get distribution, got %d: %s", getRR.Code, getRR.Body.String()) + } + + etag := getRR.Header().Get("ETag") + if etag == "" { + t.Fatalf("expected non-empty ETag on get distribution response") + } + + updateBody := `` + + `cr-disable-` + distID + `` + + `false` + + `` + updateRR := cfRequestWithBodyHeaders( + t, h, http.MethodPut, prefix+"distribution/"+distID+"/config", updateBody, + map[string]string{"If-Match": etag}, + ) + if updateRR.Code != http.StatusOK { + t.Fatalf("expected 200 on disable, got %d: %s", updateRR.Code, updateRR.Body.String()) + } + + newETag := updateRR.Header().Get("ETag") + if newETag == "" { + t.Fatalf("expected non-empty ETag on update distribution response") + } + + deleteRR := cfRequestWithHeader( + t, h, http.MethodDelete, prefix+"distribution/"+distID, + map[string]string{"If-Match": newETag}, + ) + if deleteRR.Code != http.StatusNoContent { + t.Fatalf("expected 204 on delete distribution, got %d: %s", deleteRR.Code, deleteRR.Body.String()) + } +} diff --git a/services/cloudfront/store_setup.go b/services/cloudfront/store_setup.go new file mode 100644 index 000000000..52ce6fe9a --- /dev/null +++ b/services/cloudfront/store_setup.go @@ -0,0 +1,248 @@ +package cloudfront + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend is registered exactly once, +// here, as a *store.Table[T] on b.registry. See pkgs/store's package doc and +// the services/ec2 (commit 12e611a4), services/sqs (commit 0f09d77c), and +// services/apigateway (commit 6da0334e) conversions this follows. +// +// invalidations and tenantInvalidations were previously nested per-parent maps +// (map[string]([]*Invalidation), keyed by distribution ID / tenant ID +// respectively). store.Table has no notion of nesting, so each becomes a +// single flat table keyed by a composite "#" string, +// with a secondary [store.Index] grouping by parent ID for the "all +// invalidations of parent X" lookups the nested maps used to answer directly +// (mirrors apigateway's resources/deployments/etc.). This requires Invalidation +// to carry its parent's ID as an identity-only (json:"-") field -- see the +// distID/tenantID doc on the Invalidation type in backend.go. +// +// A handful of fields are deliberately NOT registered here and remain plain +// maps -- see the comment block above registerAllTables for the list and why. +import "github.com/blackbirdworks/gopherstack/pkgs/store" + +func distributionKeyFn(v *Distribution) string { return v.ID } +func oaiKeyFn(v *OriginAccessIdentity) string { return v.ID } +func anycastIPListKeyFn(v *AnycastIPList) string { return v.ID } +func cachePolicyKeyFn(v *CachePolicy) string { return v.ID } +func connectionFunctionKeyFn(v *ConnectionFunction) string { return v.ID } +func connectionGroupKeyFn(v *ConnectionGroup) string { return v.ID } +func continuousDeploymentPolicyKeyFn(v *ContinuousDeploymentPolicy) string { return v.ID } +func originAccessControlKeyFn(v *OriginAccessControl) string { return v.ID } +func responseHeadersPolicyKeyFn(v *ResponseHeadersPolicy) string { return v.ID } +func functionKeyFn(v *Function) string { return v.Name } +func originRequestPolicyKeyFn(v *OriginRequestPolicy) string { return v.ID } +func fieldLevelEncryptionKeyFn(v *FieldLevelEncryption) string { return v.ID } + +func fieldLevelEncryptionProfileKeyFn(v *FieldLevelEncryptionProfile) string { return v.ID } +func publicKeyKeyFn(v *PublicKey) string { return v.ID } +func keyGroupKeyFn(v *KeyGroup) string { return v.ID } + +// realtimeLogConfigKeyFn keys by ARN (not ID -- RealtimeLogConfig has no separate ID +// field; ARN is its stable identity, assigned once at creation and never mutated by +// UpdateRealtimeLogConfig), matching the pre-conversion map's "ARN → config" shape. +func realtimeLogConfigKeyFn(v *RealtimeLogConfig) string { return v.ARN } + +func keyValueStoreKeyFn(v *KeyValueStore) string { return v.ID } +func vpcOriginKeyFn(v *VpcOrigin) string { return v.ID } +func trustStoreKeyFn(v *TrustStore) string { return v.ID } + +func streamingDistributionKeyFn(v *StreamingDistribution) string { return v.ID } +func distributionTenantKeyFn(v *DistributionTenant) string { return v.ID } + +// invalidationKey builds the composite "#" key shared by +// both invalidation families. Access sites use this directly so the exact same key +// is used for Put/Get/Delete. +func invalidationKey(parentID, id string) string { return parentID + "#" + id } + +func invalidationDistKeyFn(v *Invalidation) string { return invalidationKey(v.distID, v.ID) } +func invalidationTenantKeyFn(v *Invalidation) string { return invalidationKey(v.tenantID, v.ID) } + +// deleteInvalidationsForDist removes every invalidation belonging to distID from +// b.invalidations, keeping b.invalidationsByDist consistent. It replaces the old +// map-based "delete(b.invalidations, distID)" cascade delete (DeleteDistribution) +// now that invalidations is a flat composite-key table rather than one slice per +// distribution. The slice is copied before deleting: [store.Index.Get]'s result is +// owned by the index and must not be retained across a [store.Table.Delete] call, +// which may reuse or invalidate it. Must be called with the lock held. +func (b *InMemoryBackend) deleteInvalidationsForDist(distID string) { + for _, inv := range append([]*Invalidation(nil), b.invalidationsByDist.Get(distID)...) { + b.invalidations.Delete(invalidationKey(distID, inv.ID)) + } +} + +// deleteInvalidationsForTenant is deleteInvalidationsForDist's tenant-scoped twin, +// used by DeleteDistributionTenant. +func (b *InMemoryBackend) deleteInvalidationsForTenant(tenantID string) { + for _, inv := range append([]*Invalidation(nil), b.tenantInvalidationsByTenant.Get(tenantID)...) { + b.tenantInvalidations.Delete(invalidationKey(tenantID, inv.ID)) + } +} + +// registerAllTables sets up every converted resource collection exactly once, at +// construction time. +// +// Two groups, split by whether their store.Table can be snapshotted directly: +// +// - "Clean" tables (distributions, oais, anycastIPLists, cachePolicies, +// connectionFunctions, connectionGroups, continuousDeploymentPolicies, +// originAccessControls, responseHeadersPolicies, functions, +// originRequestPolicies, fieldLevelEncryptions, fieldLevelEncryptionProfiles, +// publicKeys, keyGroups, realtimeLogConfigs, keyValueStores, vpcOrigins, +// trustStores, streamingDistributions, distributionTenants) are registered on +// b.registry via store.Register, so persistence.go's Snapshot/Restore can drive +// them through b.registry.SnapshotAll()/RestoreAll() -- their value types +// marshal to JSON with no information loss. +// - "Dirty" tables (invalidations, tenantInvalidations) are built with store.New +// but deliberately NOT registered on b.registry. Their composite key depends on +// a field (distID / tenantID) tagged `json:"-"` on the live type (see +// backend.go), so a direct json.Marshal/Unmarshal round trip through +// Table.Snapshot/Restore would silently drop that field and corrupt the +// table's key on restore. persistence.go instead builds a throwaway DTO +// [store.Registry] purely to get a correctly-tagged JSON encoding (mirrors the +// services/apigateway conversion, commit 6da0334e, itself mirroring the +// services/sqs pilot, commit 0f09d77c), then restores the live tables directly +// via Table.Restore. Because they aren't registered on b.registry, +// InMemoryBackend.Reset resets each of them with an explicit Table.Reset() call +// alongside b.registry.ResetAll(). +// +// The following fields are deliberately plain maps, not store.Table at all: +// +// - distributionARNs, distributionCallerRefs, oaiCallerRefs, anycastIPListARNs, +// anycastIPListByName, cachePolicyByName, connectionFunctionARNs, +// connectionFunctionByName, connectionGroupARNs, connectionGroupByName, +// connectionGroupByRoutingEndpoint, originAccessControlByName, +// responseHeadersPolicyByName, originRequestPolicyByName, +// fieldLevelEncryptionByName, fieldLevelEncryptionProfileByName, +// publicKeyByName, keyGroupByName, realtimeLogConfigByName, +// keyValueStoreByName, trustStoreARNs, trustStoreByName, +// streamingDistributionARNs, streamingDistributionCallerRefs, +// distributionTenantARNs, distributionTenantsByDomain: each is a plain +// string->string secondary lookup index (ARN/CallerReference/name/domain → +// ID), not a map[string]*T resource collection. Kept as hand-maintained +// parallel maps (mirroring how they were maintained alongside their primary +// map before conversion) rather than a store.Index, since an Index groups +// into a slice (1:many) where these lookups are 1:1 overwrite-on-collision, +// matching the pre-conversion map semantics exactly (same reasoning as +// apigateway's apiKeysByValue, commit 6da0334e). +// - monitoringSubscriptions, resourcePolicies, managedCertificates: +// MonitoringSubscription / resourcePolicyEntry / ManagedCertificateDetails +// carry no identity field of their own (the map key -- distribution ID / +// resource ARN / tenant ID -- is not stored on the value), the same +// "no identity field" exception EC2's instanceIMDSOptions documents +// (commit 12e611a4). +// - distributionAliases, distributionFunctionAssociations: slice-valued +// (map[string][]T, not map[string]*T), so there is no single *T to key a +// Table by; each entry is itself a list, not a resource with identity. +// - distributionWebACLs, distributionTenantWebACLs, distributionCachePolicies, +// distributionOriginRequestPolicies, distributionResponseHeadersPolicies, +// distributionRealtimeLogConfigs: plain string->string association maps +// (distribution/tenant ID → associated resource ID), not resource +// collections. +// - keyValueStoreData, keyValueDataETags: KVS ID -> (key -> value) / KVS ID -> +// ETag. Values have no identity field, and (pre-existing, not introduced by +// this conversion) neither is persisted today -- see persistence.go. +// - distSearchTokens, distSearchInverted: the inverted config-token search +// index; entirely derived from b.distributions and rebuilt from it on +// Restore (rebuildDistributionSearchIndex), never itself persisted. +// - invalidationReadyAt, tenantInvalidationReadyAt: ephemeral reconciler-only +// state (distribution/tenant ID -> invalidation ID -> readyAt time), not +// persisted before this conversion either. +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one closure +// per resource table, each binding its own store.New/store.Register call (plus any +// secondary store.AddIndex) to the concrete field and value type. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.distributions = store.Register(b.registry, "distributions", store.New(distributionKeyFn)) + }, + func(b *InMemoryBackend) { + b.oais = store.Register(b.registry, "oais", store.New(oaiKeyFn)) + }, + func(b *InMemoryBackend) { + b.anycastIPLists = store.Register(b.registry, "anycastIPLists", store.New(anycastIPListKeyFn)) + }, + func(b *InMemoryBackend) { + b.cachePolicies = store.Register(b.registry, "cachePolicies", store.New(cachePolicyKeyFn)) + }, + func(b *InMemoryBackend) { + b.connectionFunctions = store.Register(b.registry, "connectionFunctions", store.New(connectionFunctionKeyFn)) + }, + func(b *InMemoryBackend) { + b.connectionGroups = store.Register(b.registry, "connectionGroups", store.New(connectionGroupKeyFn)) + }, + func(b *InMemoryBackend) { + b.continuousDeploymentPolicies = store.Register( + b.registry, "continuousDeploymentPolicies", store.New(continuousDeploymentPolicyKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.originAccessControls = store.Register(b.registry, "originAccessControls", store.New(originAccessControlKeyFn)) + }, + func(b *InMemoryBackend) { + b.responseHeadersPolicies = store.Register( + b.registry, "responseHeadersPolicies", store.New(responseHeadersPolicyKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.functions = store.Register(b.registry, "functions", store.New(functionKeyFn)) + }, + func(b *InMemoryBackend) { + b.originRequestPolicies = store.Register( + b.registry, "originRequestPolicies", store.New(originRequestPolicyKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.fieldLevelEncryptions = store.Register( + b.registry, "fieldLevelEncryptions", store.New(fieldLevelEncryptionKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.fieldLevelEncryptionProfiles = store.Register( + b.registry, "fieldLevelEncryptionProfiles", store.New(fieldLevelEncryptionProfileKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.publicKeys = store.Register(b.registry, "publicKeys", store.New(publicKeyKeyFn)) + }, + func(b *InMemoryBackend) { + b.keyGroups = store.Register(b.registry, "keyGroups", store.New(keyGroupKeyFn)) + }, + func(b *InMemoryBackend) { + b.realtimeLogConfigs = store.Register(b.registry, "realtimeLogConfigs", store.New(realtimeLogConfigKeyFn)) + }, + func(b *InMemoryBackend) { + b.keyValueStores = store.Register(b.registry, "keyValueStores", store.New(keyValueStoreKeyFn)) + }, + func(b *InMemoryBackend) { + b.vpcOrigins = store.Register(b.registry, "vpcOrigins", store.New(vpcOriginKeyFn)) + }, + func(b *InMemoryBackend) { + b.trustStores = store.Register(b.registry, "trustStores", store.New(trustStoreKeyFn)) + }, + func(b *InMemoryBackend) { + b.streamingDistributions = store.Register( + b.registry, "streamingDistributions", store.New(streamingDistributionKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.distributionTenants = store.Register(b.registry, "distributionTenants", store.New(distributionTenantKeyFn)) + }, + // "Dirty" tables: store.New only, NOT store.Register -- see registerAllTables doc. + func(b *InMemoryBackend) { + b.invalidations = store.New(invalidationDistKeyFn) + b.invalidationsByDist = b.invalidations.AddIndex("byDist", func(v *Invalidation) string { return v.distID }) + }, + func(b *InMemoryBackend) { + b.tenantInvalidations = store.New(invalidationTenantKeyFn) + b.tenantInvalidationsByTenant = b.tenantInvalidations.AddIndex( + "byTenant", func(v *Invalidation) string { return v.tenantID }, + ) + }, +} diff --git a/services/cloudfront/store_setup_test.go b/services/cloudfront/store_setup_test.go new file mode 100644 index 000000000..1c227fb61 --- /dev/null +++ b/services/cloudfront/store_setup_test.go @@ -0,0 +1,253 @@ +package cloudfront_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/cloudfront" +) + +// TestStoreSetup_FullStateSnapshotRestoreRoundTrip is the Phase 3.3 pkgs/store +// conversion's full-state round-trip test: it creates at least one instance of +// every resource collection converted to a store.Table (see store_setup.go), +// including both composite-key "dirty" tables (invalidations, +// tenantInvalidations), then verifies every one of them -- plus the +// distSearchInverted config-token index used by ListDistributionsBy* -- survives +// a Snapshot -> Restore cycle unchanged. +func TestStoreSetup_FullStateSnapshotRestoreRoundTrip(t *testing.T) { + t.Parallel() + + orig := newTestBackend() + + // distributions + the distSearchInverted token index (verified via + // ListDistributionsByCachePolicyID below). + dist, err := orig.CreateDistribution( + "caller-ref-1", "a distribution", true, []byte(`cp-token-123`), + ) + require.NoError(t, err) + + // invalidations: composite-key table keyed by distID + "#" + invID, with a + // secondary index grouping by distribution ID. + inv, err := orig.CreateInvalidation(dist.ID, "inv-caller-ref", []string{"/*"}) + require.NoError(t, err) + + // oais + oai, err := orig.CreateOAI("oai-caller-ref", "an oai") + require.NoError(t, err) + + // anycastIPLists + anycast, err := orig.CreateAnycastIPList("my-anycast-list", 3) + require.NoError(t, err) + + // cachePolicies + cachePolicy, err := orig.CreateCachePolicy("my-cache-policy", "c", 10, 100, 0) + require.NoError(t, err) + + // connectionFunctions + connFn, err := orig.CreateConnectionFunction("my-connection-function", "c") + require.NoError(t, err) + + // connectionGroups + connGroup, err := orig.CreateConnectionGroup("my-connection-group", "c") + require.NoError(t, err) + + // continuousDeploymentPolicies + cdp, err := orig.CreateContinuousDeploymentPolicy(true, "staging.example.com") + require.NoError(t, err) + + // originAccessControls + oac, err := orig.CreateOriginAccessControl("my-oac", "d", "s3", "always", "sigv4") + require.NoError(t, err) + + // responseHeadersPolicies + rhp, err := orig.CreateResponseHeadersPolicy("my-rhp", "c") + require.NoError(t, err) + + // functions (keyed by Name) + fn, err := orig.CreateFunction("my-function", "c", "cloudfront-js-2.0", "function handler(e){}") + require.NoError(t, err) + + // originRequestPolicies + orp, err := orig.CreateOriginRequestPolicy("my-orp", "c") + require.NoError(t, err) + + // publicKeys (created before fieldLevelEncryptionProfiles/keyGroups, which + // referentially depend on it) + pk, err := orig.CreatePublicKey("pk-caller-ref", "my-public-key", "c", testRSA2048PublicKeyPEM) + require.NoError(t, err) + + // fieldLevelEncryptionProfiles (references the public key above) + flep, err := orig.CreateFieldLevelEncryptionProfile("my-flep", "c", []cloudfront.EncryptionEntity{ + {PublicKeyID: pk.ID, ProviderID: "provider", FieldPatterns: []string{"field1"}}, + }) + require.NoError(t, err) + + // fieldLevelEncryptions (references the profile above) + fle, err := orig.CreateFieldLevelEncryption("my-fle", "c", []cloudfront.FLEQueryArgProfile{ + {QueryArg: "q", ProfileID: flep.ID}, + }) + require.NoError(t, err) + + // keyGroups (references the public key above) + kg, err := orig.CreateKeyGroup("my-key-group", "c", []string{pk.ID}) + require.NoError(t, err) + + // realtimeLogConfigs (keyed by ARN, not ID) + rlc, err := orig.CreateRealtimeLogConfig("my-rlc", 50, []string{"timestamp"}) + require.NoError(t, err) + + // keyValueStores + kvs, err := orig.CreateKeyValueStore("my-kvs", "c") + require.NoError(t, err) + + // vpcOrigins + vpcOrigin, err := orig.CreateVpcOrigin("my-vpc-origin") + require.NoError(t, err) + + // trustStores + ts, err := orig.CreateTrustStore( + "my-trust-store", "c", cloudfront.TrustStoreCertificateBundle{S3Bucket: "b", S3Key: "k"}, + ) + require.NoError(t, err) + + // streamingDistributions + sdCfg := cloudfront.StreamingDistributionConfig{CallerReference: "sd-caller-ref"} + sd, err := orig.CreateStreamingDistribution(sdCfg, []byte(``)) + require.NoError(t, err) + + // distributionTenants + tenant, err := orig.CreateDistributionTenant(dist.ID, "my-tenant", []string{"tenant.example.com"}, nil) + require.NoError(t, err) + + // tenantInvalidations: composite-key table keyed by tenantID + "#" + invID. + tenantInv, err := orig.CreateInvalidationForTenant(tenant.ID, []string{"/*"}) + require.NoError(t, err) + + snap := orig.Snapshot(t.Context()) + require.NotEmpty(t, snap) + + fresh := newTestBackend() + require.NoError(t, fresh.Restore(t.Context(), snap)) + + gotDist, err := fresh.GetDistribution(dist.ID) + require.NoError(t, err) + assert.Equal(t, dist.CallerReference, gotDist.CallerReference) + + // distSearchInverted must be rebuilt from the restored distributions so + // config-token lookups keep working after a restore. + byPolicy := fresh.ListDistributionsByCachePolicyID("cp-token-123") + require.Len(t, byPolicy, 1) + assert.Equal(t, dist.ID, byPolicy[0].ID) + + gotInv, err := fresh.GetInvalidation(dist.ID, inv.ID) + require.NoError(t, err) + assert.Equal(t, inv.CallerRef, gotInv.CallerRef) + assert.Equal(t, []string{"/*"}, gotInv.Paths) + + gotOAI, err := fresh.GetOAI(oai.ID) + require.NoError(t, err) + assert.Equal(t, oai.CallerReference, gotOAI.CallerReference) + + gotAnycast, err := fresh.GetAnycastIPList(anycast.ID) + require.NoError(t, err) + assert.Equal(t, anycast.Name, gotAnycast.Name) + + gotCachePolicy, err := fresh.GetCachePolicy(cachePolicy.ID) + require.NoError(t, err) + assert.Equal(t, cachePolicy.Name, gotCachePolicy.Name) + + gotConnFn, err := fresh.GetConnectionFunction(connFn.ID) + require.NoError(t, err) + assert.Equal(t, connFn.Name, gotConnFn.Name) + + gotConnGroup, err := fresh.GetConnectionGroup(connGroup.ID) + require.NoError(t, err) + assert.Equal(t, connGroup.Name, gotConnGroup.Name) + + gotCDP, err := fresh.GetContinuousDeploymentPolicy(cdp.ID) + require.NoError(t, err) + assert.Equal(t, cdp.StagingDistributionDNS, gotCDP.StagingDistributionDNS) + + gotOAC, err := fresh.GetOriginAccessControl(oac.ID) + require.NoError(t, err) + assert.Equal(t, oac.Name, gotOAC.Name) + + gotRHP, err := fresh.GetResponseHeadersPolicy(rhp.ID) + require.NoError(t, err) + assert.Equal(t, rhp.Name, gotRHP.Name) + + gotFn, err := fresh.GetFunction(fn.Name) + require.NoError(t, err) + assert.Equal(t, fn.FunctionCode, gotFn.FunctionCode) + + gotORP, err := fresh.GetOriginRequestPolicy(orp.ID) + require.NoError(t, err) + assert.Equal(t, orp.Name, gotORP.Name) + + gotFLE, err := fresh.GetFieldLevelEncryption(fle.ID) + require.NoError(t, err) + assert.Equal(t, fle.Name, gotFLE.Name) + + gotFLEP, err := fresh.GetFieldLevelEncryptionProfile(flep.ID) + require.NoError(t, err) + assert.Equal(t, flep.Name, gotFLEP.Name) + + gotPK, err := fresh.GetPublicKey(pk.ID) + require.NoError(t, err) + assert.Equal(t, pk.Name, gotPK.Name) + + gotKG, err := fresh.GetKeyGroup(kg.ID) + require.NoError(t, err) + assert.Equal(t, kg.Name, gotKG.Name) + assert.Equal(t, []string{pk.ID}, gotKG.Items) + + gotRLC, err := fresh.GetRealtimeLogConfig(rlc.ARN) + require.NoError(t, err) + assert.Equal(t, rlc.Name, gotRLC.Name) + + gotKVS, err := fresh.GetKeyValueStore(kvs.ID) + require.NoError(t, err) + assert.Equal(t, kvs.Name, gotKVS.Name) + + gotVpcOrigin, err := fresh.GetVpcOrigin(vpcOrigin.ID) + require.NoError(t, err) + assert.Equal(t, vpcOrigin.Name, gotVpcOrigin.Name) + + gotTS, err := fresh.GetTrustStore(ts.ID) + require.NoError(t, err) + assert.Equal(t, ts.Name, gotTS.Name) + + gotSD, err := fresh.GetStreamingDistribution(sd.ID) + require.NoError(t, err) + assert.Equal(t, sd.Config.CallerReference, gotSD.Config.CallerReference) + + gotTenant, err := fresh.GetDistributionTenant(tenant.ID) + require.NoError(t, err) + assert.Equal(t, tenant.Name, gotTenant.Name) + + gotTenantInv, err := fresh.GetInvalidationForTenant(tenant.ID, tenantInv.ID) + require.NoError(t, err) + assert.Equal(t, []string{"/*"}, gotTenantInv.Paths) +} + +// TestStoreSetup_RestoreDiscardsIncompatibleVersion verifies the snapshot version +// guard: a snapshot whose "version" field does not match the current +// cloudfrontSnapshotVersion is discarded cleanly (backend reset to empty) +// rather than partially decoded, mirroring the services/ec2, services/sqs, and +// services/apigateway conversions this one follows. +func TestStoreSetup_RestoreDiscardsIncompatibleVersion(t *testing.T) { + t.Parallel() + + orig := newTestBackend() + _, err := orig.CreateDistribution("caller-ref-old", "old", true, nil) + require.NoError(t, err) + + fresh := newTestBackend() + // A malformed/incompatible version (0, i.e. absent) must be discarded rather + // than partially decoded. + require.NoError(t, fresh.Restore(t.Context(), []byte(`{"version":0,"tables":{}}`))) + + assert.Empty(t, fresh.ListDistributions()) +} diff --git a/services/cloudwatch/PARITY.md b/services/cloudwatch/PARITY.md new file mode 100644 index 000000000..b8fceb78d --- /dev/null +++ b/services/cloudwatch/PARITY.md @@ -0,0 +1,183 @@ +--- +# PARITY MANIFEST SCHEMA — copy to services//PARITY.md, fill, keep updated. +# Purpose: record audit state so the NEXT audit diffs the delta instead of rescanning. +# Re-audit protocol: `git diff ..HEAD -- services//` for local drift, +# AND check the SDK module for ops added since sdk_version. Only audit changed/new surface; +# trust rows marked ok whose files are unchanged since last_audit_commit. +service: cloudwatch +sdk_module: aws-sdk-go-v2/service/cloudwatch@v1.55.1 +last_audit_commit: 58eec068 +last_audit_date: 2026-07-05 +overall: A # ~1.1k LOC of genuine fixes (624/223 non-test, ~500 net test) this pass +# Per-op or per-op-family status. Values: ok | partial | gap | deferred. +# wire=response/request shape vs SDK; errors=code+HTTP status; state=real mutate/read; persist=in backendSnapshot. +ops: + PutMetricData: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED this pass — see Notes: fabricated UnprocessedMetricData wire field removed, all-or-nothing semantics, Values/Counts array support added, NaN/Inf/range validation added"} + GetMetricStatistics: {wire: ok, errors: ok, state: ok, persist: ok, note: "proven correct: period-aligned buckets, Average/Sum/Min/Max/SampleCount, extended-statistic percentiles via collectRawBuckets, anomaly band annotation"} + GetMetricData: {wire: ok, errors: ok, state: ok, persist: ok, note: "proven correct: metric-math expressions (topo-sorted), ScanBy asc/desc, MaxDatapoints pagination with resumable cursor, PartialData/ArithmeticError messages, cross-account AccountId returns empty not error"} + ListMetrics: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED this pass — RecentlyActive=PT3H filter was parsed nowhere (silently ignored); now validated and enforced"} + PutMetricAlarm: {wire: ok, errors: ok, state: ok, persist: ok} + PutCompositeAlarm: {wire: ok, errors: ok, state: ok, persist: ok, note: "AlarmRule AND/OR/NOT parsing with cycle + depth-limit detection proven correct"} + DescribeAlarms: {wire: ok, errors: ok, state: ok, persist: ok, note: "separate MetricAlarms/CompositeAlarms lists, alarmType/stateValue/prefix filters all correct"} + DescribeAlarmsForMetric: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeAlarmHistory: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED this pass — Action-history entries for composite alarms were hardcoded AlarmType=MetricAlarm"} + DeleteAlarms: {wire: ok, errors: ok, state: ok, persist: ok} + SetAlarmState: {wire: ok, errors: ok, state: ok, persist: ok, note: "fires actions only on real transition, correct action-list selection per new state, composite re-evaluation cascades"} + EnableAlarmActions: {wire: ok, errors: ok, state: ok, persist: ok} + DisableAlarmActions: {wire: ok, errors: ok, state: ok, persist: ok} + ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok} + TagResource: {wire: ok, errors: ok, state: ok, persist: ok} + UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} + PutDashboard: {wire: partial, errors: ok, state: ok, persist: ok, note: "DashboardValidationMessages field is real (unlike PutMetricData's) but always empty — body is stored verbatim with no JSON/widget-schema validation (bd: gopherstack-3ro)"} + GetDashboard: {wire: ok, errors: ok, state: ok, persist: ok} + ListDashboards: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteDashboards: {wire: ok, errors: ok, state: ok, persist: ok} + PutAlarmMuteRule: {wire: ok, errors: ok, state: ok, persist: ok} + GetAlarmMuteRule: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateAlarmMuteRule: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteAlarmMuteRule: {wire: ok, errors: ok, state: ok, persist: ok} + ListAlarmMuteRules: {wire: ok, errors: ok, state: ok, persist: ok} + PutAnomalyDetector: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteAnomalyDetector: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeAnomalyDetectors: {wire: ok, errors: ok, state: ok, persist: ok} + PutInsightRule: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateInsightRule: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteInsightRules: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeInsightRules: {wire: ok, errors: ok, state: ok, persist: ok} + EnableInsightRules: {wire: ok, errors: ok, state: ok, persist: ok} + DisableInsightRules: {wire: ok, errors: ok, state: ok, persist: ok} + GetInsightRuleReport: {wire: ok, errors: ok, state: ok, persist: ok} + ListManagedInsightRules: {wire: ok, errors: ok, state: ok, persist: ok} + PutManagedInsightRules: {wire: ok, errors: ok, state: ok, persist: ok} + PutMetricStream: {wire: ok, errors: ok, state: ok, persist: ok} + GetMetricStream: {wire: ok, errors: ok, state: ok, persist: ok} + ListMetricStreams: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateMetricStream: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteMetricStream: {wire: ok, errors: ok, state: ok, persist: ok} + StartMetricStreams: {wire: ok, errors: ok, state: ok, persist: ok} + StopMetricStreams: {wire: ok, errors: ok, state: ok, persist: ok} + PutMetricFilter: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeMetricFilters: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteMetricFilter: {wire: ok, errors: ok, state: ok, persist: ok} + TestMetricFilter: {wire: ok, errors: ok, state: ok, persist: deferred, note: "stateless preview op, nothing to persist"} + DescribeAlarmContributors: {wire: ok, errors: ok, state: ok, persist: ok} + GetMetricWidgetImage: {wire: ok, errors: ok, state: n/a, persist: n/a, note: "rendering-only op; PNG output not byte-compared against real AWS"} +# Families audited as a group (when per-op is impractical): +families: + alarm-evaluation-state-machine: {status: ok, note: "FIXED this pass — breachesThreshold was missing the LessThanLowerThreshold comparison operator entirely (fell through to default:false, so alarms configured with it never fired). All 4 TreatMissingData modes (missing/notBreaching/breaching/ignore) proven correct in countBreachingPeriods/evaluateMetricAlarmState, including ignore's 'maintain current state when no data' rule and M-of-N DatapointsToAlarm."} + alarm-action-dispatch: {status: ok, note: "FIXED this pass — composite-alarm action history mistagged AlarmType=MetricAlarm (see DescribeAlarmHistory). SNS/Lambda/EC2-automate/AutoScaling-policy ARN routing, best-effort delivery (failures logged, other actions still run), EC2 InstanceId dimension extraction all proven correct. Actual SNS/Lambda/EC2/ASG client wiring lives in cli.go (out of scope per task boundary) — only the in-package dispatch/selection logic was audited/fixed."} + error-codes: {status: ok, note: "ResourceNotFoundException/InvalidParameterValue/InvalidParameterCombination/LimitExceeded all HTTP 400 (correct for CloudWatch's query/XML protocol, which never uses 404); InternalFailure is 500. Spot-checked across alarms/dashboards/mute-rules/anomaly-detectors/insight-rules/metric-streams/metric-filters."} + persistence: {status: ok, note: "backendSnapshot/persistence.go covers metrics, alarms, composite alarms, alarm history, dashboards, anomaly detectors, insight rules, metric streams, alarm mute rules, metric filters; field names unchanged by this pass (MetricDatum gained Values/Counts/Has* fields, additive only, so existing snapshots restore unchanged for the fields they populate)"} +gaps: # known divergences NOT fixed — link bd issue ids + - PutDashboard never validates DashboardBody JSON/widget schema, so DashboardValidationMessages is always empty even for malformed input (bd: gopherstack-3ro) + - PutMetricData does not enforce AWS's timestamp acceptance window (2 weeks past / 2 hours future) (bd: gopherstack-pyv) +deferred: # consciously not audited this pass (scope) — next pass targets + - widget.go / widget_draw.go / widget_font.go (GetMetricWidgetImage PNG rendering internals — not a wire-shape or state-correctness concern, only visual fidelity) + - metric-stream Firehose delivery payload format (OutputFormat json/opentelemetry0.7 byte-level shape) + - insight-rule Definition/Schema JSON-body validation depth (accepted verbatim, like PutDashboard) +leaks: {status: clean, note: "Janitor (janitor.go) owns the single alarm-eval + metric-sweep goroutine, ctx-cancel-aware, StartWorker only spawns it for *InMemoryBackend. storeDatum/filterAlivePoints reslice (not just filter) to release oversized backing arrays (#60 total-metrics counter avoids O(namespaces) walks). No new goroutines/tickers/maps introduced this pass; PutMetricData's validate-then-commit split does not change lock-hold duration character (still one write-lock section)."} +--- + +## Notes + +CloudWatch here speaks **AWS Query (XML) protocol** for the classic SDK path (`Action=` form +POST, `` root, `ResponseMetadata>RequestId`) and **rpc-v2-cbor** for +`aws-sdk-go-v2/service/cloudwatch@v1.55+` (routed via `X-Amz-Target`/CBOR path, +`/service/GraniteServiceVersion20100801/operation/`). Every op has two independent encoders +(`handler.go` XML, `rpcv2cbor.go` CBOR) that must be checked separately — a fix in one does not +imply the other is correct. + +### The big one: PutMetricData has NO partial-success shape + +`aws-sdk-go-v2/service/cloudwatch/api_op_PutMetricData.go`'s `PutMetricDataOutput` struct has +**zero fields** besides `ResultMetadata`. The real query-protocol response body is: + +```xml + + ... + +``` + +There is no `UnprocessedMetricData` — that concept exists for other AWS batch APIs (e.g. SQS +`SendMessageBatch`) but **not** for `PutMetricData`. Before this pass, gopherstack's handler +accepted the whole batch, stored every valid datum, and returned HTTP 200 with a fabricated +`` list describing which entries were "unprocessed" (bad StatisticSet +combos, bad StorageResolution, namespace-cap overflow). A real SDK client would never produce or +parse that field — it isn't in the generated deserializer. This is a **wire-shape bug that also +changes API semantics**: real CloudWatch validates the whole request and either accepts all of it +or rejects all of it with a single API error (a datum-level problem anywhere in the batch fails +the entire call, HTTP 400, nothing gets stored). Fixed by splitting `PutMetricData` into a +validate-the-whole-batch pass (`validatePutMetricDataBatch`, no mutation) and a commit pass, only +reached if validation passes. + +**Trap for the next auditor**: don't assume every CloudWatch write op lacks a partial-result +field just because PutMetricData does — `PutDashboard`'s `DashboardValidationMessages` and +`DeleteInsightRules`/`DisableInsightRules`/`EnableInsightRules`/`DeleteMetricFilter`'s per-name +`Failures` list are real, generated-from-model fields. Always check the actual SDK struct +(`grep -n "type Output struct" -A 20` in the unzipped SDK module) before assuming a +partial-failure shape is fabricated OR before assuming one doesn't exist. + +### Values/Counts array (new feature this pass) + +`MetricDatum.Values`/`.Counts` (parallel arrays, up to 150 unique values, each with an occurrence +count, default count 1 when `Counts` is omitted) was not parsed at all in either wire path — +neither the form parser nor the CBOR decoder read the `Values`/`Counts` member, so data submitted +this way was silently dropped with no error and no stored datapoints. This is CloudWatch's +mechanism for "publish a distribution, get real percentiles back" without pre-aggregating into a +StatisticSet. Implemented: parsing (form + CBOR), validation (mutual exclusion with Value/ +StatisticSet via new `Has{Value,StatisticSet,ValuesArray}` presence flags — presence, not a +"non-zero" check, which is what AWS actually validates), `aggregateValuesCounts` for +Sum/SampleCount/Min/Max (computed once, in `storeDatum`, so every caller — not just the two wire +parsers — gets consistent aggregation), and `expandValuesCounts` for exact percentile +reconstruction (proportionally capped at `maxStatSetExpand` samples, unlike the StatisticSet path +which has to *synthesize* a distribution from only Sum/Min/Max/Count). + +### LessThanLowerThreshold silently never fired + +`ComparisonOperator` has 7 real values (`aws-sdk-go-v2/service/cloudwatch/types/enums.go`), not 6: +`GreaterThan{,OrEqualTo}Threshold`, `LessThan{,OrEqualTo}Threshold`, +`LessThanLowerOrGreaterThanUpperThreshold`, **`LessThanLowerThreshold`**, and +`GreaterThanUpperThreshold`. `breachesThreshold`'s `switch` had no case for +`LessThanLowerThreshold` and fell through to `default: return false` — any alarm configured with +that anomaly-detection operator would never breach, ever, silently. Distinct from +`LessThanLowerOrGreaterThanUpperThreshold` (which fires on either bound); `LessThanLowerThreshold` +only fires on the lower-bound breach. + +### Composite-alarm action history mistagging + +`executeActions` hardcoded `alarmTypeName="MetricAlarm"` on every Action-history entry it wrote, +even when firing actions for a **composite** alarm (via `SetAlarmState` on a composite alarm +directly, or via `fireCompositeTransitions` when a child alarm's state change cascades). Since +`DescribeAlarmHistory` filters by `AlarmType`, querying a composite alarm's history with +`AlarmType=CompositeAlarm` would miss its own fired-action entries (they were mistagged as +MetricAlarm), while `AlarmType=MetricAlarm` would incorrectly include them. Fixed by threading the +real alarm-type string through both `executeActions` call sites. + +### ListMetrics RecentlyActive + +`RecentlyActive` only accepts the literal string `"PT3H"` (AWS's only documented value); anything +else is `InvalidParameterValue`. It restricts results to metrics with at least one datapoint in +the last 3 hours (from *now*, not from the request's implicit time context — there is no +StartTime/EndTime on ListMetrics). Was not parsed in either wire path before this pass, so the +filter was silently a no-op (every metric always returned regardless of the param). + +### Already-correct traps (do not re-flag) + +- `populateBuckets`/`collectRawBuckets` bucket index is `timestamp.Unix() / period` (aligned to + Unix-epoch boundaries), which is intentional and matches how CloudWatch aligns period + boundaries for GetMetricStatistics — it is **not** a bug that buckets aren't aligned to the + request's StartTime. +- `GetMetricData` cross-account queries (`AccountId` set to a different account) return an empty + result for that query ID rather than erroring — this is intentional (documented behavior: local + emulator has no cross-account metrics, but AWS itself just returns nothing rather than failing + the whole request for an inaccessible-but-valid account). +- `expandDatumValues`'s StatisticSet synthetic expansion (one sample at Min, one at Max, remainder + at residual mean) looks like it's fabricating data, but it's a deliberate, documented + approximation for extended-statistic (percentile) queries over StatisticSet-only data, which + AWS itself can only do exactly when `SampleCount==1` or `Min==Max` (see the PutMetricData SDK + doc comment) — this is explicitly a best-effort approximation. +- `cwMaxMetricNamesPerNamespace`/`cwMaxTotalMetricRecords` are **emulator-only** memory-safety + caps, not modeled AWS service quotas — CloudWatch's real per-account metric quotas are not + synchronously enforced by PutMetricData. Do not "fix" the LimitExceeded error code/behavior + without first checking whether AWS actually enforces a limit at write time (it does not, for + the metrics-count quota). diff --git a/services/cloudwatch/accuracy_test.go b/services/cloudwatch/accuracy_test.go index 60ced28c1..6daa0bfbf 100644 --- a/services/cloudwatch/accuracy_test.go +++ b/services/cloudwatch/accuracy_test.go @@ -75,7 +75,7 @@ func TestParseSignedPageToken_MissingSeparator(t *testing.T) { func TestValidateMetricDatum_ValueOnly(t *testing.T) { t.Parallel() - d := cloudwatch.MetricDatum{MetricName: "M", Value: 1.0} + d := cloudwatch.MetricDatum{MetricName: "M", Value: 1.0, HasValue: true} assert.NoError(t, cloudwatch.ValidateMetricDatumForTest(d)) } @@ -96,6 +96,7 @@ func TestValidateMetricDatum_BothValueAndStatisticSet(t *testing.T) { d := cloudwatch.MetricDatum{ MetricName: "M", Value: 1.0, + HasValue: true, HasStatisticSet: true, Count: 5, Sum: 100, Min: 10, Max: 30, } @@ -438,7 +439,7 @@ func TestBackend_PutMetricData_StatisticSet_Valid(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC() - unprocessed, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ { MetricName: "Reqs", HasStatisticSet: true, @@ -447,28 +448,35 @@ func TestBackend_PutMetricData_StatisticSet_Valid(t *testing.T) { }, }) require.NoError(t, err) - assert.Empty(t, unprocessed, "valid StatisticSet should not produce unprocessed entries") } +// TestBackend_PutMetricData_ValueAndStatisticSet_Rejected verifies AWS's +// all-or-nothing PutMetricData contract: PutMetricDataOutput carries no +// per-datum result (confirmed against aws-sdk-go-v2 cloudwatch types), so a +// request containing an invalid datum must fail the entire call rather than +// silently dropping the bad entry into a fabricated "unprocessed" list. func TestBackend_PutMetricData_ValueAndStatisticSet_Rejected(t *testing.T) { t.Parallel() b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC() - unprocessed, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ { MetricName: "BadEntry", Value: 1.0, + HasValue: true, HasStatisticSet: true, Count: 5, Sum: 100, Min: 10, Max: 30, Timestamp: ts, }, }) - require.NoError(t, err) - require.Len(t, unprocessed, 1, "value+statisticset should produce 1 unprocessed entry") - assert.Equal(t, "BadEntry", unprocessed[0].MetricName) - assert.Equal(t, "InvalidParameterCombination", unprocessed[0].ErrorCode) + require.ErrorIs(t, err, cloudwatch.ErrValueAndStatisticSet) + + // The whole request must be rejected: nothing gets stored. + p, lerr := b.ListMetrics("NS", "", nil, "", "", 0) + require.NoError(t, lerr) + assert.Empty(t, p.Data) } func TestBackend_PutMetricData_InvalidStorageResolution_Rejected(t *testing.T) { @@ -477,46 +485,42 @@ func TestBackend_PutMetricData_InvalidStorageResolution_Rejected(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC() - unprocessed, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ { MetricName: "BadRes", Value: 1.0, + HasValue: true, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: ts, StorageResolution: 30, }, }) - require.NoError(t, err) - require.Len(t, unprocessed, 1) - assert.Equal(t, "BadRes", unprocessed[0].MetricName) - assert.Equal(t, "InvalidParameterValue", unprocessed[0].ErrorCode) + require.ErrorIs(t, err, cloudwatch.ErrValidation) } +// TestBackend_PutMetricData_MixedValidAndInvalid verifies that when one datum +// in a batch is invalid, none of the batch is stored (no partial commit), +// matching AWS's atomic PutMetricData contract. func TestBackend_PutMetricData_MixedValidAndInvalid(t *testing.T) { t.Parallel() b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC() - unprocessed, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ - {MetricName: "Good", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: ts}, + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + {MetricName: "Good", Value: 1, HasValue: true, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: ts}, { - MetricName: "Bad", Value: 1, HasStatisticSet: true, + MetricName: "Bad", Value: 1, HasValue: true, HasStatisticSet: true, Count: 2, Sum: 50, Min: 10, Max: 40, Timestamp: ts, }, - {MetricName: "AlsoGood", Value: 2, Count: 1, Sum: 2, Min: 2, Max: 2, Timestamp: ts}, + {MetricName: "AlsoGood", Value: 2, HasValue: true, Count: 1, Sum: 2, Min: 2, Max: 2, Timestamp: ts}, }) - require.NoError(t, err) - require.Len(t, unprocessed, 1, "only Bad should be unprocessed") - assert.Equal(t, "Bad", unprocessed[0].MetricName) + require.ErrorIs(t, err, cloudwatch.ErrValueAndStatisticSet) - // Good and AlsoGood should be stored. - p, err := b.ListMetrics("NS", "", nil, "", 0) - require.NoError(t, err) - names := metricNames(p.Data) - assert.Contains(t, names, "Good") - assert.Contains(t, names, "AlsoGood") - assert.NotContains(t, names, "Bad") + // Nothing from the batch should be stored, including the otherwise-valid entries. + p, lerr := b.ListMetrics("NS", "", nil, "", "", 0) + require.NoError(t, lerr) + assert.Empty(t, p.Data) } // --------------------------------------------------------------------------- @@ -530,7 +534,7 @@ func TestBackend_GetMetricStatistics_AnomalyBand(t *testing.T) { now := time.Now().UTC() // Store metric data. - _, err := b.PutMetricData("App", []cloudwatch.MetricDatum{ + err := b.PutMetricData("App", []cloudwatch.MetricDatum{ {MetricName: "Latency", Value: 10, Count: 1, Sum: 10, Min: 10, Max: 10, Timestamp: now.Add(-2 * time.Minute)}, {MetricName: "Latency", Value: 20, Count: 1, Sum: 20, Min: 20, Max: 20, Timestamp: now.Add(-time.Minute)}, {MetricName: "Latency", Value: 30, Count: 1, Sum: 30, Min: 30, Max: 30, Timestamp: now.Add(-30 * time.Second)}, @@ -569,7 +573,7 @@ func TestBackend_GetMetricStatistics_NoAnomalyDetector_NoBand(t *testing.T) { b := cloudwatch.NewInMemoryBackend() now := time.Now().UTC() - _, err := b.PutMetricData("App", []cloudwatch.MetricDatum{ + err := b.PutMetricData("App", []cloudwatch.MetricDatum{ {MetricName: "CPU", Value: 50, Count: 1, Sum: 50, Min: 50, Max: 50, Timestamp: now.Add(-time.Minute)}, }) require.NoError(t, err) @@ -598,7 +602,7 @@ func TestBackend_GetMetricData_ForwardReferenceExpression(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC().Add(-30 * time.Second) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "Base", Value: 10, Count: 1, Sum: 10, Min: 10, Max: 10, Timestamp: ts}, }) require.NoError(t, err) @@ -638,7 +642,7 @@ func TestBackend_GetMetricData_AnomalyDetectionBandExpression(t *testing.T) { for i := range 5 { ts := now.Add(-time.Duration(5-i) * time.Minute) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ { MetricName: "M", Value: float64(10 + i*5), Count: 1, Sum: float64(10 + i*5), Min: float64(10 + i*5), Max: float64(10 + i*5), @@ -679,7 +683,7 @@ func TestBackend_GetMetricStatistics_WithDimensions(t *testing.T) { dimProd := []cloudwatch.Dimension{{Name: "Env", Value: "prod"}} dimStaging := []cloudwatch.Dimension{{Name: "Env", Value: "staging"}} - _, err := b.PutMetricData("App", []cloudwatch.MetricDatum{ + err := b.PutMetricData("App", []cloudwatch.MetricDatum{ {MetricName: "RPM", Value: 100, Count: 1, Sum: 100, Min: 100, Max: 100, Timestamp: ts, Dimensions: dimProd}, {MetricName: "RPM", Value: 200, Count: 1, Sum: 200, Min: 200, Max: 200, Timestamp: ts, Dimensions: dimStaging}, }) @@ -717,7 +721,7 @@ func TestBackend_GetMetricData_DimensionFiltering(t *testing.T) { dimA := []cloudwatch.Dimension{{Name: "Host", Value: "a"}} dimB := []cloudwatch.Dimension{{Name: "Host", Value: "b"}} - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "CPU", Value: 10, Count: 1, Sum: 10, Min: 10, Max: 10, Timestamp: ts, Dimensions: dimA}, {MetricName: "CPU", Value: 90, Count: 1, Sum: 90, Min: 90, Max: 90, Timestamp: ts, Dimensions: dimB}, }) @@ -759,7 +763,12 @@ func TestHandler_PutMetricData_StatisticSet_FormParsed(t *testing.T) { assert.Equal(t, 200, rec.Code, "valid StatisticSet should return 200; body: %s", rec.Body.String()) } -func TestHandler_PutMetricData_ValueAndStatisticSet_Returns200WithUnprocessed(t *testing.T) { +// TestHandler_PutMetricData_ValueAndStatisticSet_Returns400 verifies real AWS +// behaviour: PutMetricDataOutput has no members besides the request ID (no +// per-datum "unprocessed" result exists for this operation), so an invalid +// datum fails the entire request with a 400 InvalidParameterCombination +// instead of a fabricated 200-with-partial-failure response. +func TestHandler_PutMetricData_ValueAndStatisticSet_Returns400(t *testing.T) { t.Parallel() h := newCWHandler() @@ -775,12 +784,11 @@ func TestHandler_PutMetricData_ValueAndStatisticSet_Returns200WithUnprocessed(t "&MetricData.member.1.StatisticValues.Maximum=60"+ "&MetricData.member.1.Timestamp=2024-01-01T00:00:00Z", ) - // AWS returns 200 but includes UnprocessedMetricData in body. - assert.Equal(t, 200, rec.Code) - assert.Contains(t, rec.Body.String(), "UnprocessedMetricData") + assert.Equal(t, 400, rec.Code) + assert.Contains(t, rec.Body.String(), "InvalidParameterCombination") } -func TestHandler_PutMetricData_InvalidStorageResolution_Returns200WithUnprocessed(t *testing.T) { +func TestHandler_PutMetricData_InvalidStorageResolution_Returns400(t *testing.T) { t.Parallel() h := newCWHandler() @@ -793,8 +801,8 @@ func TestHandler_PutMetricData_InvalidStorageResolution_Returns200WithUnprocesse "&MetricData.member.1.StorageResolution=30"+ "&MetricData.member.1.Timestamp=2024-01-01T00:00:00Z", ) - assert.Equal(t, 200, rec.Code) - assert.Contains(t, rec.Body.String(), "UnprocessedMetricData") + assert.Equal(t, 400, rec.Code) + assert.Contains(t, rec.Body.String(), "InvalidParameterValue") } // --------------------------------------------------------------------------- @@ -1166,7 +1174,7 @@ func TestBackend_AnomalyDetector_DimensionMatch(t *testing.T) { dimProd := []cloudwatch.Dimension{{Name: "Env", Value: "prod"}} dimStaging := []cloudwatch.Dimension{{Name: "Env", Value: "staging"}} - _, err := b.PutMetricData("App", []cloudwatch.MetricDatum{ + err := b.PutMetricData("App", []cloudwatch.MetricDatum{ { MetricName: "CPU", Value: 80, Count: 1, Sum: 80, Min: 80, Max: 80, Timestamp: now.Add(-time.Minute), Dimensions: dimProd, @@ -1212,7 +1220,7 @@ func TestBackend_AnomalyDetector_CustomBandWidth(t *testing.T) { b := cloudwatch.NewInMemoryBackend() now := time.Now().UTC() - _, err := b.PutMetricData("Srv", []cloudwatch.MetricDatum{ + err := b.PutMetricData("Srv", []cloudwatch.MetricDatum{ {MetricName: "Req", Value: 10, Count: 1, Sum: 10, Min: 10, Max: 10, Timestamp: now.Add(-2 * time.Minute)}, {MetricName: "Req", Value: 30, Count: 1, Sum: 30, Min: 30, Max: 30, Timestamp: now.Add(-time.Minute)}, {MetricName: "Req", Value: 20, Count: 1, Sum: 20, Min: 20, Max: 20, Timestamp: now.Add(-30 * time.Second)}, diff --git a/services/cloudwatch/alarm_eval_test.go b/services/cloudwatch/alarm_eval_test.go index 048396950..c667c4a71 100644 --- a/services/cloudwatch/alarm_eval_test.go +++ b/services/cloudwatch/alarm_eval_test.go @@ -221,6 +221,35 @@ func TestAlarmEvaluator_StateTransitions(t *testing.T) { {Timestamp: now.Add(-30 * time.Second), Value: 50.0}, }, }, + { + // LessThanLowerThreshold is a distinct anomaly-detection comparison + // operator (aws-sdk-go-v2 cloudwatch/types.ComparisonOperator) from + // LessThanLowerOrGreaterThanUpperThreshold: it fires only on the + // lower-bound breach. Previously unhandled in breachesThreshold, so + // alarms configured with it never fired. + name: "less_than_lower_threshold_operator_breaches_when_below_threshold", + operator: "LessThanLowerThreshold", + statistic: "Average", + period: 60, + evalPeriods: 1, + initialState: "OK", + wantState: "ALARM", + points: []cloudwatch.MetricDatum{ + {Timestamp: now.Add(-30 * time.Second), Value: 50.0}, + }, + }, + { + name: "less_than_lower_threshold_operator_ok_when_above_threshold", + operator: "LessThanLowerThreshold", + statistic: "Average", + period: 60, + evalPeriods: 1, + initialState: "ALARM", + wantState: "OK", + points: []cloudwatch.MetricDatum{ + {Timestamp: now.Add(-30 * time.Second), Value: 150.0}, + }, + }, { name: "alarm_transitions_back_to_ok_when_data_normalizes", operator: "GreaterThanThreshold", @@ -258,7 +287,7 @@ func TestAlarmEvaluator_StateTransitions(t *testing.T) { } } - _, err := b.PutMetricData(namespace, normalized) + err := b.PutMetricData(namespace, normalized) require.NoError(t, err) } @@ -351,7 +380,7 @@ func TestAlarmEvaluator_SNSActionFiredOnStateChange(t *testing.T) { // Put metric datum — normalize Value to StatisticSet fields. v := tt.dataValue - _, err := b.PutMetricData(namespace, []cloudwatch.MetricDatum{ + err := b.PutMetricData(namespace, []cloudwatch.MetricDatum{ { MetricName: metricName, Value: v, @@ -408,7 +437,7 @@ func TestAlarmEvaluator_BackgroundJanitor(t *testing.T) { now := time.Now().UTC() - _, err := b.PutMetricData(namespace, []cloudwatch.MetricDatum{ + err := b.PutMetricData(namespace, []cloudwatch.MetricDatum{ { MetricName: metricName, Value: 200.0, diff --git a/services/cloudwatch/audit_cw_test.go b/services/cloudwatch/audit_cw_test.go index 5937461a4..9d684d05e 100644 --- a/services/cloudwatch/audit_cw_test.go +++ b/services/cloudwatch/audit_cw_test.go @@ -21,7 +21,7 @@ func putMetric(t *testing.T, b *cloudwatch.InMemoryBackend, datum cloudwatch.Met datum.Min = datum.Value datum.Max = datum.Value } - _, err := b.PutMetricData(datum.Namespace, []cloudwatch.MetricDatum{datum}) + err := b.PutMetricData(datum.Namespace, []cloudwatch.MetricDatum{datum}) require.NoError(t, err) } @@ -304,7 +304,7 @@ func TestAuditCW_ListMetrics_NameOnlyDimensionFilter(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - page, err := b.ListMetrics("SvcNS", "Req", tc.dimFilter, "", 0) + page, err := b.ListMetrics("SvcNS", "Req", tc.dimFilter, "", "", 0) require.NoError(t, err) assert.Len(t, page.Data, tc.wantLen, "filter=%v", tc.dimFilter) }) diff --git a/services/cloudwatch/backend.go b/services/cloudwatch/backend.go index 38e48b02f..3d307b11b 100644 --- a/services/cloudwatch/backend.go +++ b/services/cloudwatch/backend.go @@ -17,6 +17,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/config" "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) const ( @@ -134,7 +135,7 @@ type AutoScalingPolicyExecutor interface { // StorageBackend is the interface for the CloudWatch in-memory store. type StorageBackend interface { - PutMetricData(namespace string, data []MetricDatum) ([]UnprocessedMetricDatum, error) + PutMetricData(namespace string, data []MetricDatum) error GetMetricStatistics( namespace, metricName string, dimensions []Dimension, @@ -150,7 +151,7 @@ type StorageBackend interface { ListMetrics( namespace, metricName string, dimensions []Dimension, - nextToken string, + recentlyActive, nextToken string, maxResults int, ) (page.Page[Metric], error) PutMetricAlarm(alarm *MetricAlarm) error @@ -230,17 +231,22 @@ type metricRecord struct { // metrics is a two-level map: namespace -> composite-key -> *metricRecord. // The composite key is produced by metricStorageKey(metricName, dims) so that // different dimension sets for the same metric name are stored separately. +// metrics and alarmHistory are deliberately left as plain maps rather than +// store.Table -- see the comment block at the top of store_setup.go for why. +// Every other resource field is registered on registry -- see +// registerAllTables in store_setup.go. type InMemoryBackend struct { metrics map[string]map[string]*metricRecord - alarms map[string]*MetricAlarm - compositeAlarms map[string]*CompositeAlarm alarmHistory map[string][]AlarmHistoryItem - dashboards map[string]*dashboardRecord - anomalyDetectors map[string]*AnomalyDetector - insightRules map[string]*InsightRule - metricStreams map[string]*MetricStream - alarmMuteRules map[string]*AlarmMuteRule - metricFilters map[string]*MetricFilter + alarms *store.Table[MetricAlarm] + compositeAlarms *store.Table[CompositeAlarm] + dashboards *store.Table[dashboardRecord] + anomalyDetectors *store.Table[AnomalyDetector] + insightRules *store.Table[InsightRule] + metricStreams *store.Table[MetricStream] + alarmMuteRules *store.Table[AlarmMuteRule] + metricFilters *store.Table[MetricFilter] + registry *store.Registry snsPublisher SNSPublisher lambdaInvoker LambdaInvoker ec2Actioner EC2InstanceActioner @@ -267,21 +273,18 @@ func NewInMemoryBackend() *InMemoryBackend { // NewInMemoryBackendWithConfig creates a new InMemoryBackend with given account and region. func NewInMemoryBackendWithConfig(accountID, region string) *InMemoryBackend { - return &InMemoryBackend{ - accountID: accountID, - region: region, - metrics: make(map[string]map[string]*metricRecord), - alarms: make(map[string]*MetricAlarm), - compositeAlarms: make(map[string]*CompositeAlarm), - alarmHistory: make(map[string][]AlarmHistoryItem), - dashboards: make(map[string]*dashboardRecord), - anomalyDetectors: make(map[string]*AnomalyDetector), - insightRules: make(map[string]*InsightRule), - metricStreams: make(map[string]*MetricStream), - alarmMuteRules: make(map[string]*AlarmMuteRule), - metricFilters: make(map[string]*MetricFilter), - mu: lockmetrics.New("cloudwatch"), + b := &InMemoryBackend{ + accountID: accountID, + region: region, + metrics: make(map[string]map[string]*metricRecord), + alarmHistory: make(map[string][]AlarmHistoryItem), + registry: store.NewRegistry(), + mu: lockmetrics.New("cloudwatch"), } + + registerAllTables(b) + + return b } // dimensionSetKey returns a stable string key for a slice of Dimensions, @@ -408,46 +411,62 @@ func (b *InMemoryBackend) SetAutoScalingExecutor(e AutoScalingPolicyExecutor) { b.asgExecutor = e } -// storeDatum validates and stores a single MetricDatum into the namespace map. -// Returns a non-nil *UnprocessedMetricDatum when the datum cannot be stored. +// validatePutMetricDataBatch checks every datum in a PutMetricData request for +// shape/range validity and confirms the batch would not push the namespace or +// account past its distinct-time-series cap. It performs no mutation, so a +// failing batch leaves backend state untouched — real CloudWatch has no +// partial-success shape for PutMetricData (PutMetricDataOutput carries no +// fields besides the request ID), so the whole request must be validated +// before any of it is committed. // Caller must hold b.mu (write lock). -func (b *InMemoryBackend) storeDatum(namespace string, d MetricDatum) *UnprocessedMetricDatum { - if err := validateMetricDatum(d); err != nil { - return &UnprocessedMetricDatum{ - MetricName: d.MetricName, - ErrorCode: "InvalidParameterCombination", - ErrorMessage: err.Error(), +func (b *InMemoryBackend) validatePutMetricDataBatch(namespace string, data []MetricDatum) error { + existing := len(b.metrics[namespace]) + newKeys := make(map[string]bool) + + for _, d := range data { + if err := validateMetricDatum(d); err != nil { + return err } - } - if err := validateStorageResolution(d.StorageResolution); err != nil { - return &UnprocessedMetricDatum{ - MetricName: d.MetricName, - ErrorCode: "InvalidParameterValue", - ErrorMessage: err.Error(), + if err := validateStorageResolution(d.StorageResolution); err != nil { + return err } + + key := metricStorageKey(d.MetricName, d.Dimensions) + if _, ok := b.metrics[namespace][key]; !ok { + newKeys[key] = true + } + } + + if existing+len(newKeys) > cwMaxMetricNamesPerNamespace { + return fmt.Errorf("%w: namespace metric series limit reached", ErrMetricSeriesLimitExceeded) + } + + if b.countTotalMetrics()+len(newKeys) > cwMaxTotalMetricRecords { + return fmt.Errorf("%w: global metric series limit reached", ErrMetricSeriesLimitExceeded) + } + + return nil +} + +// storeDatum stores an already-validated MetricDatum into the namespace map. +// Caller must hold b.mu (write lock) and must have already validated the full +// batch via validatePutMetricDataBatch. +// +// A Values/Counts datum carries no pre-aggregated Sum/SampleCount/Min/Max (unlike +// a StatisticSet, whose caller supplies them directly), so this is the single +// place that derives them from the raw array pair before the point is stored; +// every caller of PutMetricData — the form handler, the rpc-v2-cbor handler, and +// direct backend callers/tests — gets consistent aggregation this way. +func (b *InMemoryBackend) storeDatum(namespace string, d MetricDatum) { + if d.HasValuesArray && len(d.Values) == len(d.Counts) { + d.Sum, d.Count, d.Min, d.Max = aggregateValuesCounts(d.Values, d.Counts) } key := metricStorageKey(d.MetricName, d.Dimensions) rec, exists := b.metrics[namespace][key] if !exists { - if len(b.metrics[namespace]) >= cwMaxMetricNamesPerNamespace { - return &UnprocessedMetricDatum{ - MetricName: d.MetricName, - ErrorCode: "LimitExceeded", - ErrorMessage: "namespace metric series limit reached", - } - } - - if b.countTotalMetrics() >= cwMaxTotalMetricRecords { - return &UnprocessedMetricDatum{ - MetricName: d.MetricName, - ErrorCode: "LimitExceeded", - ErrorMessage: "global metric series limit reached", - } - } - dims := make([]Dimension, len(d.Dimensions)) copy(dims, d.Dimensions) rec = &metricRecord{MetricName: d.MetricName, Dimensions: dims} @@ -464,18 +483,21 @@ func (b *InMemoryBackend) storeDatum(namespace string, d MetricDatum) *Unprocess copy(fresh, rec.Points[len(rec.Points)-cwMaxMetricDataPoints:]) rec.Points = fresh } - - return nil } // PutMetricData stores metric data points for the given namespace. -// Returns a slice of UnprocessedMetricDatum for any entries that could not be stored. +// +// Real CloudWatch has no partial-failure response for this operation: the +// request either succeeds in full or fails in full with a single API error +// (PutMetricDataOutput has no members other than the request ID). This +// validates the entire batch before storing any of it, matching that +// all-or-nothing contract. func (b *InMemoryBackend) PutMetricData( namespace string, data []MetricDatum, -) ([]UnprocessedMetricDatum, error) { +) error { if len(data) > cwMaxMetricDataPerRequest { - return nil, fmt.Errorf( + return fmt.Errorf( "%w: PutMetricData accepts at most %d MetricDatum entries per request", ErrValidation, cwMaxMetricDataPerRequest, @@ -488,14 +510,15 @@ func (b *InMemoryBackend) PutMetricData( b.metrics[namespace] = make(map[string]*metricRecord) } - var unprocessed []UnprocessedMetricDatum + if err := b.validatePutMetricDataBatch(namespace, data); err != nil { + b.mu.Unlock() + + return err + } for _, d := range data { d.Namespace = namespace - - if u := b.storeDatum(namespace, d); u != nil { - unprocessed = append(unprocessed, *u) - } + b.storeDatum(namespace, d) } // Collect matching running stream names while holding the write lock; the @@ -510,14 +533,14 @@ func (b *InMemoryBackend) PutMetricData( now := time.Now().UTC() b.mu.Lock("PutMetricData.streamDelivery") for _, name := range matchingStreams { - if s, ok := b.metricStreams[name]; ok && s.State == metricStreamStateRunning { + if s, ok := b.metricStreams.Get(name); ok && s.State == metricStreamStateRunning { s.LastUpdateDate = now } } b.mu.Unlock() } - return unprocessed, nil + return nil } // filterExcludesMetric returns true when an ExcludeFilters entry denies the metric. @@ -583,14 +606,14 @@ func (b *InMemoryBackend) matchingRunningStreamNames( ) []string { var names []string - for name, s := range b.metricStreams { + for _, s := range b.metricStreams.All() { if s.State != metricStreamStateRunning { continue } for _, d := range data { if streamAllowsMetric(s, namespace, d.MetricName) { - names = append(names, name) + names = append(names, s.Name) break } @@ -887,7 +910,7 @@ func (b *InMemoryBackend) annotateAnomalyBand( // Look for a detector matching this metric (any stat matches since band is stat-agnostic). var matchedDetector *AnomalyDetector - for _, d := range b.anomalyDetectors { + for _, d := range b.anomalyDetectors.All() { if d.Namespace != namespace || d.MetricName != metricName { continue } @@ -1154,37 +1177,106 @@ func statValue(dp Datapoint, stat string) float64 { return 0 } +// cwRecentlyActiveValue is the only value CloudWatch accepts for ListMetrics' +// RecentlyActive parameter, filtering results to metrics that have had data +// points published in the past three hours. +const cwRecentlyActiveValue = "PT3H" + +// cwRecentlyActiveWindow is the lookback window RecentlyActive=PT3H applies. +const cwRecentlyActiveWindow = 3 * time.Hour + +// recordHasRecentPoint reports whether rec has at least one datapoint with a +// timestamp within the last cwRecentlyActiveWindow of now. Caller must hold +// b.mu (read or write lock). +func recordHasRecentPoint(rec *metricRecord, now time.Time) bool { + cutoff := now.Add(-cwRecentlyActiveWindow) + for _, pt := range rec.Points { + if !pt.Timestamp.Before(cutoff) { + return true + } + } + + return false +} + +// listMetricsFilter bundles the ListMetrics query filters so the namespace/record +// matching loop can be factored out of ListMetrics itself. +type listMetricsFilter struct { + now time.Time + namespace, metricName string + recentlyActive string + dimensions []Dimension +} + +// matches reports whether rec (in namespace ns) satisfies the filter. +func (f listMetricsFilter) matches(rec *metricRecord) bool { + if f.metricName != "" && rec.MetricName != f.metricName { + return false + } + if !dimsMatchListFilter(rec.Dimensions, f.dimensions) { + return false + } + if f.recentlyActive != "" && !recordHasRecentPoint(rec, f.now) { + return false + } + + return true +} + +// matchingMetricsInNamespace returns the Metric entries in nsMap that satisfy +// the filter, or nil immediately when ns itself is excluded by the namespace filter. +func (f listMetricsFilter) matchingMetricsInNamespace( + ns string, + nsMap map[string]*metricRecord, +) []Metric { + if f.namespace != "" && ns != f.namespace { + return nil + } + + var result []Metric + + for _, rec := range nsMap { + if !f.matches(rec) { + continue + } + + dims := make([]Dimension, len(rec.Dimensions)) + copy(dims, rec.Dimensions) + result = append(result, Metric{Namespace: ns, MetricName: rec.MetricName, Dimensions: dims}) + } + + return result +} + // ListMetrics returns a page of unique metrics matching optional namespace, metricName, and // dimension filters. dimensions specifies an exact set that must match (all filter dims present -// with matching values and no extra dims in the stored record). +// with matching values and no extra dims in the stored record). recentlyActive, when set, must be +// "PT3H" (the only value CloudWatch documents) and restricts results to metrics that received a +// data point in the last 3 hours. func (b *InMemoryBackend) ListMetrics( namespace, metricName string, dimensions []Dimension, - nextToken string, + recentlyActive, nextToken string, maxResults int, ) (page.Page[Metric], error) { + if recentlyActive != "" && recentlyActive != cwRecentlyActiveValue { + return page.Page[Metric]{}, fmt.Errorf( + "%w: RecentlyActive must be %q, got %q", ErrValidation, cwRecentlyActiveValue, recentlyActive, + ) + } + b.mu.RLock("ListMetrics") defer b.mu.RUnlock() + filter := listMetricsFilter{ + namespace: namespace, metricName: metricName, + dimensions: dimensions, recentlyActive: recentlyActive, + now: time.Now().UTC(), + } + var result []Metric for ns, nsMap := range b.metrics { - if namespace != "" && ns != namespace { - continue - } - for _, rec := range nsMap { - if metricName != "" && rec.MetricName != metricName { - continue - } - if !dimsMatchListFilter(rec.Dimensions, dimensions) { - continue - } - dims := make([]Dimension, len(rec.Dimensions)) - copy(dims, rec.Dimensions) - result = append( - result, - Metric{Namespace: ns, MetricName: rec.MetricName, Dimensions: dims}, - ) - } + result = append(result, filter.matchingMetricsInNamespace(ns, nsMap)...) } sort.Slice(result, func(i, j int) bool { @@ -1228,7 +1320,7 @@ func (b *InMemoryBackend) PutMetricAlarm(alarm *MetricAlarm) error { b.mu.Lock("PutMetricAlarm") defer b.mu.Unlock() - isNew := b.alarms[alarm.AlarmName] == nil + isNew := !b.alarms.Has(alarm.AlarmName) if alarm.AlarmArn == "" { alarm.AlarmArn = arn.Build("cloudwatch", b.region, b.accountID, "alarm:"+alarm.AlarmName) @@ -1241,7 +1333,7 @@ func (b *InMemoryBackend) PutMetricAlarm(alarm *MetricAlarm) error { alarm.CreatedAt = now } // Preserve the state-transitioned timestamp from an existing alarm if the state did not change. - if existing, ok := b.alarms[alarm.AlarmName]; ok { + if existing, ok := b.alarms.Get(alarm.AlarmName); ok { if existing.StateValue == alarm.StateValue { alarm.StateTransitionedTimestamp = existing.StateTransitionedTimestamp } else { @@ -1253,7 +1345,7 @@ func (b *InMemoryBackend) PutMetricAlarm(alarm *MetricAlarm) error { alarm.AlarmConfigurationUpdatedTimestamp = now cp := *alarm - b.alarms[alarm.AlarmName] = &cp + b.alarms.Put(&cp) histType := historyTypeConfigurationUpdate historySummary := fmt.Sprintf("Alarm %q updated", alarm.AlarmName) @@ -1277,7 +1369,7 @@ func (b *InMemoryBackend) PutCompositeAlarm(alarm *CompositeAlarm) error { b.mu.Lock("PutCompositeAlarm") defer b.mu.Unlock() - isNew := b.compositeAlarms[alarm.AlarmName] == nil + isNew := !b.compositeAlarms.Has(alarm.AlarmName) if alarm.AlarmArn == "" { alarm.AlarmArn = arn.Build("cloudwatch", b.region, b.accountID, "alarm:"+alarm.AlarmName) @@ -1288,7 +1380,7 @@ func (b *InMemoryBackend) PutCompositeAlarm(alarm *CompositeAlarm) error { // Evaluate state based on AlarmRule and current child alarm states. newState := b.evalCompositeRule(alarm.AlarmRule) - if existing, ok := b.compositeAlarms[alarm.AlarmName]; ok { + if existing, ok := b.compositeAlarms.Get(alarm.AlarmName); ok { if existing.StateTransitionedTimestamp.IsZero() || newState != existing.StateValue { alarm.StateTransitionedTimestamp = time.Now().UTC() } else { @@ -1303,7 +1395,7 @@ func (b *InMemoryBackend) PutCompositeAlarm(alarm *CompositeAlarm) error { } cp := *alarm - b.compositeAlarms[alarm.AlarmName] = &cp + b.compositeAlarms.Put(&cp) histType := historyTypeConfigurationUpdate historySummary := fmt.Sprintf("Composite alarm %q updated", alarm.AlarmName) @@ -1338,10 +1430,10 @@ func (b *InMemoryBackend) evalCompositeRuleDepth( } resolve := func(name string) string { - if a, ok := b.alarms[name]; ok { + if a, ok := b.alarms.Get(name); ok { return a.StateValue } - if ca, ok := b.compositeAlarms[name]; ok { + if ca, ok := b.compositeAlarms.Get(name); ok { if visited[name] { // Circular dependency detected: treat as INSUFFICIENT_DATA. return alarmStateInsufficientData @@ -1443,7 +1535,7 @@ func (b *InMemoryBackend) collectMetricAlarms( var result []MetricAlarm - for _, alarm := range b.alarms { + for _, alarm := range b.alarms.All() { if len(nameSet) > 0 && !nameSet[alarm.AlarmName] { continue } @@ -1479,7 +1571,7 @@ func (b *InMemoryBackend) collectCompositeAlarms( var result []CompositeAlarm - for _, alarm := range b.compositeAlarms { + for _, alarm := range b.compositeAlarms.All() { if len(nameSet) > 0 && !nameSet[alarm.AlarmName] { continue } @@ -1519,7 +1611,7 @@ func (b *InMemoryBackend) DescribeAlarmsForMetric( } var result []MetricAlarm - for _, alarm := range b.alarms { + for _, alarm := range b.alarms.All() { if namespace != "" && alarm.Namespace != namespace { continue } @@ -1599,8 +1691,8 @@ func (b *InMemoryBackend) DeleteAlarms(alarmNames []string) error { defer b.mu.Unlock() for _, name := range alarmNames { - delete(b.alarms, name) - delete(b.compositeAlarms, name) + b.alarms.Delete(name) + b.compositeAlarms.Delete(name) // Release the per-alarm history so it cannot accumulate across the // lifetime of the backend once the alarm itself is gone. delete(b.alarmHistory, name) @@ -1616,8 +1708,8 @@ func (b *InMemoryBackend) SetAlarmState( ) error { b.mu.Lock("SetAlarmState") - metricAlarm, hasMetric := b.alarms[alarmName] - compositeAlarm, hasComposite := b.compositeAlarms[alarmName] + metricAlarm, hasMetric := b.alarms.Get(alarmName) + compositeAlarm, hasComposite := b.compositeAlarms.Get(alarmName) if !hasMetric && !hasComposite { b.mu.Unlock() @@ -1703,7 +1795,7 @@ func (b *InMemoryBackend) SetAlarmState( stateValue, stateReason, ) - b.executeActions(ctx, actions, alarmName, payload, deps) + b.executeActions(ctx, actions, alarmName, histAlarmType, payload, deps) } b.fireCompositeTransitions(ctx, compositeTransitions, deps) @@ -1725,7 +1817,7 @@ func (b *InMemoryBackend) fireCompositeTransitions( tr.alarmName, tr.alarmDesc, tr.alarmArn, tr.oldState, tr.newState, tr.reason, ) - b.executeActions(ctx, tr.actions, tr.alarmName, payload, compositeDeps) + b.executeActions(ctx, tr.actions, tr.alarmName, "CompositeAlarm", payload, compositeDeps) } } @@ -1760,10 +1852,10 @@ func (b *InMemoryBackend) EnableAlarmActions(alarmNames []string) error { defer b.mu.Unlock() for _, name := range alarmNames { - if a, ok := b.alarms[name]; ok { + if a, ok := b.alarms.Get(name); ok { a.ActionsEnabled = true } - if ca, ok := b.compositeAlarms[name]; ok { + if ca, ok := b.compositeAlarms.Get(name); ok { ca.ActionsEnabled = true } } @@ -1777,10 +1869,10 @@ func (b *InMemoryBackend) DisableAlarmActions(alarmNames []string) error { defer b.mu.Unlock() for _, name := range alarmNames { - if a, ok := b.alarms[name]; ok { + if a, ok := b.alarms.Get(name); ok { a.ActionsEnabled = false } - if ca, ok := b.compositeAlarms[name]; ok { + if ca, ok := b.compositeAlarms.Get(name); ok { ca.ActionsEnabled = false } } @@ -1846,11 +1938,14 @@ func (b *InMemoryBackend) buildAlarmActionPayload( // executeActions delivers the alarm action notifications to SNS topics, Lambda // functions, EC2 instances (arn:aws:automate), and Auto Scaling scaling policies. // Delivery errors are logged as warnings but do not prevent other actions from -// running. Each fired action is recorded as an Action history entry on the alarm. +// running. Each fired action is recorded as an Action history entry on the alarm, +// tagged with alarmTypeName ("MetricAlarm" or "CompositeAlarm") so +// DescribeAlarmHistory's AlarmType filter matches composite-alarm action entries +// too (previously hardcoded to "MetricAlarm" even when firing for a composite alarm). func (b *InMemoryBackend) executeActions( ctx context.Context, actions []string, - alarmName string, + alarmName, alarmTypeName string, payload []byte, deps alarmActionDeps, ) { @@ -1891,7 +1986,7 @@ func (b *InMemoryBackend) executeActions( if alarmName != "" { summary := fmt.Sprintf("Alarm %q action executed: %s → %s", alarmName, action, actionResult) b.mu.Lock("executeActions-history") - b.appendHistory(alarmName, "MetricAlarm", historyTypeAction, summary, "") + b.appendHistory(alarmName, alarmTypeName, historyTypeAction, summary, "") b.mu.Unlock() } } @@ -2036,7 +2131,7 @@ type compositeAlarmTransition struct { func (b *InMemoryBackend) reevaluateCompositeAlarms() []compositeAlarmTransition { var transitions []compositeAlarmTransition - for _, ca := range b.compositeAlarms { + for _, ca := range b.compositeAlarms.All() { newState := b.evalCompositeRule(ca.AlarmRule) if newState == ca.StateValue { continue @@ -2092,11 +2187,11 @@ func (b *InMemoryBackend) PutDashboard(name, body string) error { b.mu.Lock("PutDashboard") defer b.mu.Unlock() - b.dashboards[name] = &dashboardRecord{ + b.dashboards.Put(&dashboardRecord{ Name: name, Body: body, LastModified: time.Now().UTC(), - } + }) return nil } @@ -2106,7 +2201,7 @@ func (b *InMemoryBackend) GetDashboard(name string) (DashboardEntry, string, err b.mu.RLock("GetDashboard") defer b.mu.RUnlock() - rec, ok := b.dashboards[name] + rec, ok := b.dashboards.Get(name) if !ok { return DashboardEntry{}, "", fmt.Errorf("%w: %s", ErrDashboardNotFound, name) } @@ -2123,7 +2218,7 @@ func (b *InMemoryBackend) ListDashboards( var result []DashboardEntry - for _, rec := range b.dashboards { + for _, rec := range b.dashboards.All() { if prefix != "" && !strings.HasPrefix(rec.Name, prefix) { continue } @@ -2144,7 +2239,7 @@ func (b *InMemoryBackend) DeleteDashboards(names []string) error { defer b.mu.Unlock() for _, name := range names { - delete(b.dashboards, name) + b.dashboards.Delete(name) } return nil @@ -2168,15 +2263,8 @@ func (b *InMemoryBackend) Reset() { defer b.mu.Unlock() b.metrics = make(map[string]map[string]*metricRecord) - b.alarms = make(map[string]*MetricAlarm) - b.compositeAlarms = make(map[string]*CompositeAlarm) b.alarmHistory = make(map[string][]AlarmHistoryItem) - b.dashboards = make(map[string]*dashboardRecord) - b.anomalyDetectors = make(map[string]*AnomalyDetector) - b.insightRules = make(map[string]*InsightRule) - b.metricStreams = make(map[string]*MetricStream) - b.alarmMuteRules = make(map[string]*AlarmMuteRule) - b.metricFilters = make(map[string]*MetricFilter) + b.registry.ResetAll() } // aggregateContributorPoint updates the running aggregation for a single metric record point. @@ -2257,7 +2345,7 @@ func (b *InMemoryBackend) GetInsightRuleContributors( maxContributorCount int, orderBy string, ) ([]AlarmContributor, error) { - if _, ok := b.insightRules[ruleName]; !ok { + if !b.insightRules.Has(ruleName) { return nil, fmt.Errorf("%w: %s", ErrInsightRuleNotFound, ruleName) } if maxContributorCount <= 0 { @@ -2297,11 +2385,11 @@ func (b *InMemoryBackend) DeleteAlarmMuteRule(muteName string) error { b.mu.Lock("DeleteAlarmMuteRule") defer b.mu.Unlock() - if _, ok := b.alarmMuteRules[muteName]; !ok { + if !b.alarmMuteRules.Has(muteName) { return fmt.Errorf("%w: %s", ErrAlarmMuteRuleNotFound, muteName) } - delete(b.alarmMuteRules, muteName) + b.alarmMuteRules.Delete(muteName) return nil } @@ -2312,7 +2400,7 @@ func (b *InMemoryBackend) GetAlarmMuteRule(muteName string) (*AlarmMuteRule, err b.mu.RLock("GetAlarmMuteRule") defer b.mu.RUnlock() - rule, ok := b.alarmMuteRules[muteName] + rule, ok := b.alarmMuteRules.Get(muteName) if !ok { return nil, fmt.Errorf("%w: %s", ErrAlarmMuteRuleNotFound, muteName) } @@ -2330,8 +2418,8 @@ func (b *InMemoryBackend) ListAlarmMuteRules( b.mu.RLock("ListAlarmMuteRules") defer b.mu.RUnlock() - result := make([]AlarmMuteRule, 0, len(b.alarmMuteRules)) - for _, rule := range b.alarmMuteRules { + result := make([]AlarmMuteRule, 0, b.alarmMuteRules.Len()) + for _, rule := range b.alarmMuteRules.All() { result = append(result, *rule) } sort.Slice(result, func(i, j int) bool { return result[i].MuteName < result[j].MuteName }) @@ -2350,7 +2438,7 @@ func (b *InMemoryBackend) ListManagedInsightRules( defer b.mu.RUnlock() result := make([]InsightRule, 0) - for _, rule := range b.insightRules { + for _, rule := range b.insightRules.All() { if !rule.ManagedRule { continue } @@ -2374,7 +2462,7 @@ func (b *InMemoryBackend) PutAlarmMuteRuleInternal(rule *AlarmMuteRule) { cp.CreationTime = time.Now().UTC() } - b.alarmMuteRules[rule.MuteName] = &cp + b.alarmMuteRules.Put(&cp) } // DeleteAnomalyDetector removes an anomaly detector. @@ -2385,11 +2473,11 @@ func (b *InMemoryBackend) DeleteAnomalyDetector(namespace, metricName, stat stri key := anomalyDetectorKey(namespace, metricName, stat, dims) - if _, ok := b.anomalyDetectors[key]; !ok { + if !b.anomalyDetectors.Has(key) { return fmt.Errorf("%w: %s/%s/%s", ErrAnomalyDetectorNotFound, namespace, metricName, stat) } - delete(b.anomalyDetectors, key) + b.anomalyDetectors.Delete(key) return nil } @@ -2399,14 +2487,13 @@ func (b *InMemoryBackend) PutAnomalyDetectorInternal(detector *AnomalyDetector) b.mu.Lock("PutAnomalyDetectorInternal") defer b.mu.Unlock() - key := anomalyDetectorKey(detector.Namespace, detector.MetricName, detector.Stat, detector.Dimensions) cp := *detector if cp.StateValue == "" { // TRAINED_INSUFFICIENT_DATA is the realistic initial state for a new detector. cp.StateValue = statusTrainedInsufficient } - b.anomalyDetectors[key] = &cp + b.anomalyDetectors.Put(&cp) } // DescribeAnomalyDetectors returns a filtered, paginated list of anomaly detectors. @@ -2424,7 +2511,7 @@ func (b *InMemoryBackend) DescribeAnomalyDetectors( var entries []entry - for k, d := range b.anomalyDetectors { + for _, d := range b.anomalyDetectors.All() { if namespace != "" && d.Namespace != namespace { continue } @@ -2433,6 +2520,7 @@ func (b *InMemoryBackend) DescribeAnomalyDetectors( continue } + k := anomalyDetectorKey(d.Namespace, d.MetricName, d.Stat, d.Dimensions) entries = append(entries, entry{key: k, detector: *d}) } @@ -2457,7 +2545,7 @@ func (b *InMemoryBackend) DeleteInsightRules(ruleNames []string) ([]InsightRuleF var failures []InsightRuleFailure for _, name := range ruleNames { - if _, ok := b.insightRules[name]; !ok { + if !b.insightRules.Has(name) { failures = append(failures, InsightRuleFailure{ RuleName: name, FailureCode: errResourceNotFoundException, @@ -2467,7 +2555,7 @@ func (b *InMemoryBackend) DeleteInsightRules(ruleNames []string) ([]InsightRuleF continue } - delete(b.insightRules, name) + b.insightRules.Delete(name) } return failures, nil @@ -2489,7 +2577,7 @@ func (b *InMemoryBackend) GetInsightRule(name string) (*InsightRule, error) { b.mu.RLock("GetInsightRule") defer b.mu.RUnlock() - rule, ok := b.insightRules[name] + rule, ok := b.insightRules.Get(name) if !ok { return nil, fmt.Errorf("%w: %s", ErrInsightRuleNotFound, name) } @@ -2517,7 +2605,7 @@ func (b *InMemoryBackend) PutInsightRuleInternal(rule *InsightRule) { cp.Arn = arn.Build("cloudwatch", b.region, b.accountID, "insight-rule/"+rule.Name) } - b.insightRules[rule.Name] = &cp + b.insightRules.Put(&cp) } // DescribeInsightRules returns a paginated list of insight rules. @@ -2528,9 +2616,9 @@ func (b *InMemoryBackend) DescribeInsightRules( b.mu.RLock("DescribeInsightRules") defer b.mu.RUnlock() - result := make([]InsightRule, 0, len(b.insightRules)) + result := make([]InsightRule, 0, b.insightRules.Len()) - for _, r := range b.insightRules { + for _, r := range b.insightRules.All() { result = append(result, *r) } @@ -2549,7 +2637,7 @@ func (b *InMemoryBackend) DisableInsightRules(ruleNames []string) ([]InsightRule var failures []InsightRuleFailure for _, name := range ruleNames { - rule, ok := b.insightRules[name] + rule, ok := b.insightRules.Get(name) if !ok { failures = append(failures, InsightRuleFailure{ RuleName: name, @@ -2574,7 +2662,7 @@ func (b *InMemoryBackend) EnableInsightRules(ruleNames []string) ([]InsightRuleF var failures []InsightRuleFailure for _, name := range ruleNames { - rule, ok := b.insightRules[name] + rule, ok := b.insightRules.Get(name) if !ok { failures = append(failures, InsightRuleFailure{ RuleName: name, @@ -2607,7 +2695,7 @@ func (b *InMemoryBackend) GetMetricStream(name string) (*MetricStream, error) { b.mu.RLock("GetMetricStream") defer b.mu.RUnlock() - stream, ok := b.metricStreams[name] + stream, ok := b.metricStreams.Get(name) if !ok { return nil, fmt.Errorf("%w: %s", ErrMetricStreamNotFound, name) } @@ -2623,11 +2711,11 @@ func (b *InMemoryBackend) DeleteMetricStream(name string) error { b.mu.Lock("DeleteMetricStream") defer b.mu.Unlock() - if _, ok := b.metricStreams[name]; !ok { + if !b.metricStreams.Has(name) { return fmt.Errorf("%w: %s", ErrMetricStreamNotFound, name) } - delete(b.metricStreams, name) + b.metricStreams.Delete(name) return nil } @@ -2649,7 +2737,7 @@ func (b *InMemoryBackend) PutMetricStreamInternal(stream *MetricStream) { } // Preserve the existing state if not explicitly set so that Stop/Start calls are honoured. - if existing, ok := b.metricStreams[stream.Name]; ok && cp.State == "" { + if existing, ok := b.metricStreams.Get(stream.Name); ok && cp.State == "" { cp.State = existing.State } @@ -2657,7 +2745,7 @@ func (b *InMemoryBackend) PutMetricStreamInternal(stream *MetricStream) { cp.State = metricStreamStateRunning } - b.metricStreams[stream.Name] = &cp + b.metricStreams.Put(&cp) } // DescribeAlarmContributors returns a page of contributors for the specified alarm. @@ -2674,7 +2762,7 @@ func (b *InMemoryBackend) DescribeAlarmContributors( b.mu.RLock("DescribeAlarmContributors") defer b.mu.RUnlock() - if _, ok := b.alarms[alarmName]; ok { + if b.alarms.Has(alarmName) { // Metric alarms have no contributors in AWS. return page.New( []AlarmContributor{}, @@ -2684,7 +2772,7 @@ func (b *InMemoryBackend) DescribeAlarmContributors( ), nil } - ca, ok := b.compositeAlarms[alarmName] + ca, ok := b.compositeAlarms.Get(alarmName) if !ok { return page.Page[AlarmContributor]{}, fmt.Errorf("%w: %s", ErrAlarmNotFound, alarmName) } @@ -2705,10 +2793,10 @@ func (b *InMemoryBackend) compositeContributorsLocked(ca *CompositeAlarm) []Alar refs := extractAlarmRuleRefs(ca.AlarmRule) resolve := func(name string) string { - if a, ok := b.alarms[name]; ok { + if a, ok := b.alarms.Get(name); ok { return a.StateValue } - if child, ok := b.compositeAlarms[name]; ok { + if child, ok := b.compositeAlarms.Get(name); ok { return child.StateValue } @@ -2748,10 +2836,10 @@ func (b *InMemoryBackend) GetAlarmARNs(names []string) []string { arns := make([]string, 0, len(names)) for _, name := range names { - if a, ok := b.alarms[name]; ok && a.AlarmArn != "" { + if a, ok := b.alarms.Get(name); ok && a.AlarmArn != "" { arns = append(arns, a.AlarmArn) } - if ca, ok := b.compositeAlarms[name]; ok && ca.AlarmArn != "" { + if ca, ok := b.compositeAlarms.Get(name); ok && ca.AlarmArn != "" { arns = append(arns, ca.AlarmArn) } } @@ -2767,7 +2855,7 @@ func (b *InMemoryBackend) GetDashboardARNs(names []string) []string { arns := make([]string, 0, len(names)) for _, name := range names { - if _, ok := b.dashboards[name]; ok { + if b.dashboards.Has(name) { arns = append(arns, arn.Build("cloudwatch", b.region, b.accountID, "dashboard/"+name)) } } @@ -2793,8 +2881,8 @@ func (b *InMemoryBackend) ListMetricStreams( b.mu.RLock("ListMetricStreams") defer b.mu.RUnlock() - result := make([]MetricStream, 0, len(b.metricStreams)) - for _, s := range b.metricStreams { + result := make([]MetricStream, 0, b.metricStreams.Len()) + for _, s := range b.metricStreams.All() { result = append(result, *s) } @@ -2817,7 +2905,7 @@ func (b *InMemoryBackend) StartMetricStreams(names []string) error { defer b.mu.Unlock() for _, name := range names { - if s, ok := b.metricStreams[name]; ok { + if s, ok := b.metricStreams.Get(name); ok { s.State = metricStreamStateRunning s.LastUpdateDate = time.Now().UTC() } @@ -2833,7 +2921,7 @@ func (b *InMemoryBackend) StopMetricStreams(names []string) error { defer b.mu.Unlock() for _, name := range names { - if s, ok := b.metricStreams[name]; ok { + if s, ok := b.metricStreams.Get(name); ok { s.State = metricStreamStateStopped s.LastUpdateDate = time.Now().UTC() } @@ -2864,7 +2952,7 @@ func (b *InMemoryBackend) PutMetricFilter(filter *MetricFilter) error { cp.CreationTime = time.Now().UTC() } - b.metricFilters[metricFilterKey(filter.FilterName, filter.LogGroupName)] = &cp + b.metricFilters.Put(&cp) return nil } @@ -2878,7 +2966,7 @@ func (b *InMemoryBackend) DescribeMetricFilters( defer b.mu.RUnlock() var result []MetricFilter - for _, f := range b.metricFilters { + for _, f := range b.metricFilters.All() { if logGroupName != "" && f.LogGroupName != logGroupName { continue } @@ -2906,11 +2994,11 @@ func (b *InMemoryBackend) DeleteMetricFilter(filterName, logGroupName string) er defer b.mu.Unlock() key := metricFilterKey(filterName, logGroupName) - if _, ok := b.metricFilters[key]; !ok { + if !b.metricFilters.Has(key) { return fmt.Errorf("%w: %s/%s", ErrMetricFilterNotFound, logGroupName, filterName) } - delete(b.metricFilters, key) + b.metricFilters.Delete(key) return nil } @@ -2929,7 +3017,7 @@ func (b *InMemoryBackend) EvaluateAlarms(ctx context.Context, now time.Time) { var snaps []alarmSnap - for _, a := range b.alarms { + for _, a := range b.alarms.All() { isMultiMetric := len(a.Metrics) > 0 if !isMultiMetric && (a.MetricName == "" || a.Namespace == "" || a.Period <= 0) { continue @@ -3291,6 +3379,7 @@ func extractDatapointValue(dp Datapoint, statistic, extendedStatistic string) *f // breachesThreshold reports whether value breaches the threshold for the given operator. // lowerBound and upperBound are used for anomaly-detection operators: // - GreaterThanUpperThreshold: fires when value > upperBound +// - LessThanLowerThreshold: fires when value < lowerBound // - LessThanLowerOrGreaterThanUpperThreshold: fires when value < lowerBound OR value > upperBound // // For non-anomaly alarms, pass threshold for both lowerBound and upperBound. @@ -3306,6 +3395,8 @@ func breachesThreshold(value, lowerBound, upperBound float64, op string) bool { return value <= lowerBound case "GreaterThanUpperThreshold": return value > upperBound + case "LessThanLowerThreshold": + return value < lowerBound case "LessThanLowerOrGreaterThanUpperThreshold": return value < lowerBound || value > upperBound default: diff --git a/services/cloudwatch/backend_accuracy.go b/services/cloudwatch/backend_accuracy.go index d91ea916e..fd19134c3 100644 --- a/services/cloudwatch/backend_accuracy.go +++ b/services/cloudwatch/backend_accuracy.go @@ -14,8 +14,35 @@ import ( // ErrInvalidNextToken is returned when a pagination NextToken is invalid or expired. var ErrInvalidNextToken = errors.New("InvalidParameterValue: NextToken is invalid or expired") -// ErrValueAndStatisticSet is returned when both Value and StatisticSet are set in a MetricDatum. -var ErrValueAndStatisticSet = errors.New("InvalidParameterCombination: Value and StatisticSet are mutually exclusive") +// ErrValueAndStatisticSet is returned when more than one of Value, StatisticSet, +// and the Values/Counts array are set on a MetricDatum. AWS treats all three +// input shapes as mutually exclusive. +var ErrValueAndStatisticSet = errors.New( + "InvalidParameterCombination: Value, StatisticValues, and Values/Counts are mutually exclusive", +) + +// ErrValuesCountsLengthMismatch is returned when a MetricDatum's Counts array is +// present but does not have the same number of elements as its Values array. +var ErrValuesCountsLengthMismatch = errors.New( + "InvalidParameterValue: Values and Counts arrays must be the same length", +) + +// ErrTooManyValues is returned when a MetricDatum's Values array exceeds the +// 150-unique-value limit documented for PutMetricData. +var ErrTooManyValues = errors.New( + "InvalidParameterValue: the maximum values array size is 150", +) + +// ErrInvalidMetricValue is returned when a metric value (Value, Sum, Min, Max, or +// an entry of the Values array) is NaN, +/-Infinity, or outside the documented +// -2^360..2^360 range that CloudWatch accepts. +var ErrInvalidMetricValue = errors.New( + "InvalidParameterValue: metric values must be finite and within -2^360 to 2^360", +) + +// ErrMetricSeriesLimitExceeded is returned when storing a batch of MetricDatum +// entries would push the namespace or account past its distinct-time-series cap. +var ErrMetricSeriesLimitExceeded = errors.New("LimitExceeded: metric series limit reached") // tokenSecret is used to HMAC-sign pagination tokens so we can reject spoofed ones. // In a real system this would be loaded from config; here we use a deterministic value. @@ -94,17 +121,96 @@ func parseSignedPageToken(token string) (int, error) { return int(u), nil } -// validateMetricDatum returns an error when a MetricDatum has both Value and a StatisticSet. -// AWS rejects this combination with InvalidParameterCombination. -// This must be called before the handler normalizes the datum (before Count/Sum/Min/Max are -// derived from Value for single-value points). +// cwMaxValuesArraySize is the documented cap on unique entries in a MetricDatum's +// Values array (and, correspondingly, its Counts array). +const cwMaxValuesArraySize = 150 + +// metricValueExponent is the documented CloudWatch metric value range exponent: +// values must fall within -2^metricValueExponent to 2^metricValueExponent. +const metricValueExponent = 360 + +// metricValueBound is CloudWatch's documented acceptable range for any metric +// value: -2^360 to 2^360. NaN and +/-Infinity are rejected outright. +var metricValueBound = math.Ldexp(1, metricValueExponent) //nolint:gochecknoglobals // fixed AWS-documented constant + +// validMetricValue reports whether v is a value CloudWatch would accept for +// Value, Sum, Min, Max, or an entry of the Values array. +func validMetricValue(v float64) bool { + return !math.IsNaN(v) && !math.IsInf(v, 0) && v >= -metricValueBound && v <= metricValueBound +} + +// validateMetricDatum enforces the shape and range rules AWS applies to a single +// PutMetricData MetricDatum entry: +// - Value, StatisticValues, and the Values/Counts array are mutually exclusive +// (InvalidParameterCombination) +// - a Values array carries at most 150 entries, and Counts (when supplied) must +// have the same length (InvalidParameterValue) +// - every numeric value (Value, Sum, Min, Max, or a Values entry) must be +// finite and within +/-2^360 (InvalidParameterValue) +// +// This must be called before the handler normalizes the datum (before +// Count/Sum/Min/Max are derived from Value for single-value points). func validateMetricDatum(d MetricDatum) error { - if !d.HasStatisticSet { - return nil + if datumShapeCount(d) > 1 { + return fmt.Errorf("%w: MetricName=%s", ErrValueAndStatisticSet, d.MetricName) } - if d.Value != 0 { - return fmt.Errorf("%w: MetricName=%s", ErrValueAndStatisticSet, d.MetricName) + if d.HasValuesArray { + return validateValuesArray(d) + } + + if d.HasValue && !validMetricValue(d.Value) { + return fmt.Errorf("%w: MetricName=%s", ErrInvalidMetricValue, d.MetricName) + } + + if d.HasStatisticSet { + return validateStatisticSetRange(d) + } + + return nil +} + +// datumShapeCount counts how many of the three mutually-exclusive PutMetricData +// input shapes (Value, StatisticValues, Values/Counts) are present on d. +func datumShapeCount(d MetricDatum) int { + shapes := 0 + if d.HasValue { + shapes++ + } + if d.HasStatisticSet { + shapes++ + } + if d.HasValuesArray { + shapes++ + } + + return shapes +} + +// validateValuesArray checks the size and per-entry range of a Values/Counts datum. +func validateValuesArray(d MetricDatum) error { + if len(d.Values) > cwMaxValuesArraySize { + return fmt.Errorf("%w: MetricName=%s", ErrTooManyValues, d.MetricName) + } + if len(d.Counts) != len(d.Values) { + return fmt.Errorf("%w: MetricName=%s", ErrValuesCountsLengthMismatch, d.MetricName) + } + for _, v := range d.Values { + if !validMetricValue(v) { + return fmt.Errorf("%w: MetricName=%s", ErrInvalidMetricValue, d.MetricName) + } + } + + return nil +} + +// validateStatisticSetRange checks that a StatisticValues datum's Sum/Min/Max +// are all within CloudWatch's acceptable metric value range. +func validateStatisticSetRange(d MetricDatum) error { + for _, v := range []float64{d.Sum, d.Min, d.Max} { + if !validMetricValue(v) { + return fmt.Errorf("%w: MetricName=%s", ErrInvalidMetricValue, d.MetricName) + } } return nil diff --git a/services/cloudwatch/backend_test.go b/services/cloudwatch/backend_test.go index 522c86361..9f848e82f 100644 --- a/services/cloudwatch/backend_test.go +++ b/services/cloudwatch/backend_test.go @@ -27,7 +27,7 @@ func TestCloudWatchBackend_PutMetricData(t *testing.T) { Timestamp: time.Now(), }, } - _, err := b.PutMetricData("AWS/EC2", data) + err := b.PutMetricData("AWS/EC2", data) require.NoError(t, err) } @@ -39,9 +39,9 @@ func TestCloudWatchBackend_PutMetricData_Multiple(t *testing.T) { {MetricName: "CPU", Value: 10, Count: 1, Sum: 10, Min: 10, Max: 10, Timestamp: time.Now()}, {MetricName: "CPU", Value: 20, Count: 1, Sum: 20, Min: 20, Max: 20, Timestamp: time.Now()}, } - _, err := b.PutMetricData("AWS/EC2", data) + err := b.PutMetricData("AWS/EC2", data) require.NoError(t, err) - metrics, err := b.ListMetrics("AWS/EC2", "CPU", nil, "", 0) + metrics, err := b.ListMetrics("AWS/EC2", "CPU", nil, "", "", 0) require.NoError(t, err) assert.Len(t, metrics.Data, 1) } @@ -50,23 +50,23 @@ func TestCloudWatchBackend_ListMetrics(t *testing.T) { t.Parallel() b := cloudwatch.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - _, _ = b.PutMetricData("NS1", []cloudwatch.MetricDatum{ + _ = b.PutMetricData("NS1", []cloudwatch.MetricDatum{ {MetricName: "M1", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: time.Now()}, }) - _, _ = b.PutMetricData("NS2", []cloudwatch.MetricDatum{ + _ = b.PutMetricData("NS2", []cloudwatch.MetricDatum{ {MetricName: "M2", Value: 2, Count: 1, Sum: 2, Min: 2, Max: 2, Timestamp: time.Now()}, }) - all, err := b.ListMetrics("", "", nil, "", 0) + all, err := b.ListMetrics("", "", nil, "", "", 0) require.NoError(t, err) assert.Len(t, all.Data, 2) - ns1, err := b.ListMetrics("NS1", "", nil, "", 0) + ns1, err := b.ListMetrics("NS1", "", nil, "", "", 0) require.NoError(t, err) assert.Len(t, ns1.Data, 1) assert.Equal(t, "M1", ns1.Data[0].MetricName) - byName, err := b.ListMetrics("", "M2", nil, "", 0) + byName, err := b.ListMetrics("", "M2", nil, "", "", 0) require.NoError(t, err) assert.Len(t, byName.Data, 1) } @@ -116,7 +116,7 @@ func TestCloudWatchBackend_GetMetricStatistics(t *testing.T) { Timestamp: now.Add(5 * time.Second), }, } - _, err := b.PutMetricData("AWS/EC2", data) + err := b.PutMetricData("AWS/EC2", data) require.NoError(t, err) }, namespace: "AWS/EC2", @@ -148,7 +148,7 @@ func TestCloudWatchBackend_GetMetricStatistics(t *testing.T) { Timestamp: old, }, } - _, err := b.PutMetricData("AWS/EC2", data) + err := b.PutMetricData("AWS/EC2", data) require.NoError(t, err) }, namespace: "AWS/EC2", @@ -709,6 +709,23 @@ func TestCloudWatchBackend_CompositeAlarmActionsFireOnChildChange(t *testing.T) assert.Len(t, pub.messages, 1, "composite alarm action should have been fired") assert.Contains(t, pub.messages[0], "parent2") + + // The fired action's history entry must be tagged AlarmType=CompositeAlarm + // (previously hardcoded to "MetricAlarm" for every action-fired entry, + // regardless of which kind of alarm actually fired it), so + // DescribeAlarmHistory's AlarmType filter can find it. + composite, err := b.DescribeAlarmHistory( + "parent2", "CompositeAlarm", "Action", "", time.Time{}, time.Time{}, 0, + ) + require.NoError(t, err) + require.NotEmpty(t, composite.Data, "composite alarm's Action history should be tagged CompositeAlarm") + + metricTyped, err := b.DescribeAlarmHistory( + "parent2", "MetricAlarm", "Action", "", time.Time{}, time.Time{}, 0, + ) + require.NoError(t, err) + assert.Empty(t, metricTyped.Data, + "a composite alarm's Action history must not be mistagged as MetricAlarm") } // mockLambdaInvoker captures lambda invocations for assertions. @@ -1209,7 +1226,7 @@ func TestCloudWatchBackend_MetricDataCap(t *testing.T) { // Insert more than cwMaxMetricDataPoints data points. const total = 1100 for range total { - _, err := b.PutMetricData("AWS/EC2", []cloudwatch.MetricDatum{ + err := b.PutMetricData("AWS/EC2", []cloudwatch.MetricDatum{ { MetricName: "CPUUtilization", Value: 42.0, @@ -1221,7 +1238,7 @@ func TestCloudWatchBackend_MetricDataCap(t *testing.T) { } // At least one metric entry should still exist after capping. - page, err := b.ListMetrics("AWS/EC2", "CPUUtilization", nil, "", 0) + page, err := b.ListMetrics("AWS/EC2", "CPUUtilization", nil, "", "", 0) require.NoError(t, err) assert.NotEmpty(t, page.Data) } @@ -1408,11 +1425,12 @@ func TestCloudWatchBackend_PutMetricData_NamespaceCapEnforced(t *testing.T) { Max: 1, Timestamp: time.Now(), } - _, err := b.PutMetricData("NS/Cap", []cloudwatch.MetricDatum{datum}) + err := b.PutMetricData("NS/Cap", []cloudwatch.MetricDatum{datum}) require.NoError(t, err) } - // Attempt to add one more unique metric; it should be silently dropped. + // Attempt to add one more unique metric; the whole request must fail since + // PutMetricData has no partial-success shape in real CloudWatch. extra := cloudwatch.MetricDatum{ MetricName: "ExtraMetric", Value: 1, @@ -1422,10 +1440,10 @@ func TestCloudWatchBackend_PutMetricData_NamespaceCapEnforced(t *testing.T) { Max: 1, Timestamp: time.Now(), } - _, err := b.PutMetricData("NS/Cap", []cloudwatch.MetricDatum{extra}) - require.NoError(t, err) + err := b.PutMetricData("NS/Cap", []cloudwatch.MetricDatum{extra}) + require.ErrorIs(t, err, cloudwatch.ErrMetricSeriesLimitExceeded) - metrics, err2 := b.ListMetrics("NS/Cap", "", nil, "", 0) + metrics, err2 := b.ListMetrics("NS/Cap", "", nil, "", "", 0) require.NoError(t, err2) assert.LessOrEqual(t, len(metrics.Data), cloudwatch.CwMaxMetricNamesPerNamespace, "namespace metric count should not exceed the cap") @@ -1453,13 +1471,13 @@ func TestCloudWatchBackend_SweepExpiredMetrics(t *testing.T) { Timestamp: recentTimestamp, } - _, err := b.PutMetricData("NS/Sweep", []cloudwatch.MetricDatum{oldDatum, recentDatum}) + err := b.PutMetricData("NS/Sweep", []cloudwatch.MetricDatum{oldDatum, recentDatum}) require.NoError(t, err) b.SweepExpiredMetrics() // OldMetric should be evicted; RecentMetric should remain. - all, err := b.ListMetrics("NS/Sweep", "", nil, "", 0) + all, err := b.ListMetrics("NS/Sweep", "", nil, "", "", 0) require.NoError(t, err) names := make(map[string]bool, len(all.Data)) @@ -1486,7 +1504,7 @@ func TestCloudWatchBackend_SweepExpiredMetrics_OutOfOrder(t *testing.T) { {MetricName: "Mixed", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: recent}, {MetricName: "Mixed", Value: 2, Count: 1, Sum: 2, Min: 2, Max: 2, Timestamp: old}, } - _, err := b.PutMetricData("NS/OutOfOrder", pts) + err := b.PutMetricData("NS/OutOfOrder", pts) require.NoError(t, err) b.SweepExpiredMetrics() @@ -1789,7 +1807,7 @@ func TestCloudWatchBackend_DimensionAwareStorage(t *testing.T) { dims1 := []cloudwatch.Dimension{{Name: "Host", Value: "host-1"}} dims2 := []cloudwatch.Dimension{{Name: "Host", Value: "host-2"}} - _, err := b.PutMetricData("AWS/EC2", []cloudwatch.MetricDatum{ + err := b.PutMetricData("AWS/EC2", []cloudwatch.MetricDatum{ { MetricName: "CPUUtilization", Value: 10, Count: 1, Sum: 10, Min: 10, Max: 10, Timestamp: now, Dimensions: dims1, @@ -1802,12 +1820,12 @@ func TestCloudWatchBackend_DimensionAwareStorage(t *testing.T) { require.NoError(t, err) // ListMetrics should return two separate series, not one. - all, err := b.ListMetrics("AWS/EC2", "CPUUtilization", nil, "", 0) + all, err := b.ListMetrics("AWS/EC2", "CPUUtilization", nil, "", "", 0) require.NoError(t, err) assert.Len(t, all.Data, 2, "each dimension set is a distinct metric series") // Filter by dims1 should return exactly one series. - filtered, err := b.ListMetrics("AWS/EC2", "CPUUtilization", dims1, "", 0) + filtered, err := b.ListMetrics("AWS/EC2", "CPUUtilization", dims1, "", "", 0) require.NoError(t, err) require.Len(t, filtered.Data, 1) assert.Equal(t, "host-1", filtered.Data[0].Dimensions[0].Value) @@ -1823,7 +1841,7 @@ func TestCloudWatchBackend_DimensionAwareGetMetricStatistics(t *testing.T) { dimsA := []cloudwatch.Dimension{{Name: "Service", Value: "A"}} dimsB := []cloudwatch.Dimension{{Name: "Service", Value: "B"}} - _, err := b.PutMetricData("App/Metrics", []cloudwatch.MetricDatum{ + err := b.PutMetricData("App/Metrics", []cloudwatch.MetricDatum{ {MetricName: "Latency", Value: 100, Count: 1, Sum: 100, Min: 100, Max: 100, Timestamp: mid, Dimensions: dimsA}, {MetricName: "Latency", Value: 200, Count: 1, Sum: 200, Min: 200, Max: 200, Timestamp: mid, Dimensions: dimsB}, }) @@ -1852,7 +1870,7 @@ func TestCloudWatchBackend_DimensionAwareGetMetricData(t *testing.T) { dimsX := []cloudwatch.Dimension{{Name: "Shard", Value: "x"}} - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "Errors", Value: 5, Count: 1, Sum: 5, Min: 5, Max: 5, Timestamp: mid, Dimensions: dimsX}, {MetricName: "Errors", Value: 50, Count: 1, Sum: 50, Min: 50, Max: 50, Timestamp: mid}, }) @@ -1885,7 +1903,7 @@ func TestCloudWatchBackend_StatisticSet(t *testing.T) { ts := time.Now().UTC().Add(-time.Minute) // Pre-aggregated StatisticSet datum. - _, err := b.PutMetricData("App", []cloudwatch.MetricDatum{ + err := b.PutMetricData("App", []cloudwatch.MetricDatum{ { MetricName: "RequestCount", Timestamp: ts, @@ -1916,7 +1934,7 @@ func TestCloudWatchBackend_DimensionlessVsDimensioned(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC().Add(-30 * time.Second) - _, err := b.PutMetricData("MyNS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("MyNS", []cloudwatch.MetricDatum{ {MetricName: "M", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: ts}, { MetricName: "M", Value: 2, Count: 1, Sum: 2, Min: 2, Max: 2, Timestamp: ts, @@ -1926,11 +1944,11 @@ func TestCloudWatchBackend_DimensionlessVsDimensioned(t *testing.T) { require.NoError(t, err) // No-dim query should return only the dimensionless series. - all, err := b.ListMetrics("MyNS", "M", nil, "", 0) + all, err := b.ListMetrics("MyNS", "M", nil, "", "", 0) require.NoError(t, err) assert.Len(t, all.Data, 2, "dimensionless and dimensioned are separate series") - noDim, err := b.ListMetrics("MyNS", "M", []cloudwatch.Dimension{}, "", 0) + noDim, err := b.ListMetrics("MyNS", "M", []cloudwatch.Dimension{}, "", "", 0) require.NoError(t, err) assert.Len(t, noDim.Data, 2, "empty dimension filter matches all") } @@ -1944,7 +1962,7 @@ func TestCloudWatchBackend_ScanByDescending(t *testing.T) { t2 := now.Add(-2 * time.Minute) t3 := now.Add(-time.Minute) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "Counter", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: t1}, {MetricName: "Counter", Value: 2, Count: 1, Sum: 2, Min: 2, Max: 2, Timestamp: t2}, {MetricName: "Counter", Value: 3, Count: 1, Sum: 3, Min: 3, Max: 3, Timestamp: t3}, @@ -1981,7 +1999,7 @@ func TestCloudWatchBackend_ReturnDataFalse_SuppressesExpressionResult(t *testing b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC().Add(-30 * time.Second) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "Val", Value: 7, Count: 1, Sum: 7, Min: 7, Max: 7, Timestamp: ts}, }) require.NoError(t, err) @@ -2012,14 +2030,19 @@ func TestCloudWatchBackend_ReturnDataFalse_SuppressesExpressionResult(t *testing assert.NotContains(t, ids, "e1", "e1 ReturnData=false should be suppressed") } -func TestCloudWatchBackend_UnprocessedMetricData_NamespaceCap(t *testing.T) { +// TestCloudWatchBackend_PutMetricData_NamespaceCap verifies that once a +// namespace has reached its distinct-time-series cap, PutMetricData rejects a +// request introducing one more new series with the whole call failing (real +// CloudWatch has no per-datum "unprocessed" result — PutMetricDataOutput only +// carries a request ID) rather than a partial/fabricated success. +func TestCloudWatchBackend_PutMetricData_NamespaceCap(t *testing.T) { t.Parallel() b := cloudwatch.NewInMemoryBackendWithConfig("123456789012", "us-east-1") // Fill the namespace to the cap. for i := range cloudwatch.CwMaxMetricNamesPerNamespace { - _, err := b.PutMetricData("NS/Full", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS/Full", []cloudwatch.MetricDatum{ { MetricName: fmt.Sprintf("M%d", i), Value: float64(i), @@ -2030,14 +2053,16 @@ func TestCloudWatchBackend_UnprocessedMetricData_NamespaceCap(t *testing.T) { require.NoError(t, err) } - // One more new metric should come back as unprocessed. - unprocessed, err := b.PutMetricData("NS/Full", []cloudwatch.MetricDatum{ + // One more new metric should fail the whole request. + err := b.PutMetricData("NS/Full", []cloudwatch.MetricDatum{ {MetricName: "OverflowMetric", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: time.Now()}, }) - require.NoError(t, err) - require.Len(t, unprocessed, 1) - assert.Equal(t, "OverflowMetric", unprocessed[0].MetricName) - assert.Equal(t, "LimitExceeded", unprocessed[0].ErrorCode) + require.ErrorIs(t, err, cloudwatch.ErrMetricSeriesLimitExceeded) + + // The overflow metric must not have been stored. + p, lerr := b.ListMetrics("NS/Full", "", nil, "", "", 0) + require.NoError(t, lerr) + assert.NotContains(t, metricNames(p.Data), "OverflowMetric") } func TestCloudWatchBackend_MetricStream_IncludeFilter(t *testing.T) { @@ -2058,7 +2083,7 @@ func TestCloudWatchBackend_MetricStream_IncludeFilter(t *testing.T) { ts := time.Now().UTC() // This metric matches the include filter. - _, err = b.PutMetricData("AWS/EC2", []cloudwatch.MetricDatum{ + err = b.PutMetricData("AWS/EC2", []cloudwatch.MetricDatum{ {MetricName: "CPUUtilization", Value: 50, Count: 1, Sum: 50, Min: 50, Max: 50, Timestamp: ts}, }) require.NoError(t, err) @@ -2080,7 +2105,7 @@ func TestCloudWatchBackend_MetricStream_IncludeFilter(t *testing.T) { }, }) beforeNonMatch, _ := b.GetMetricStream("stream-include2") - _, err = b.PutMetricData("AWS/RDS", []cloudwatch.MetricDatum{ + err = b.PutMetricData("AWS/RDS", []cloudwatch.MetricDatum{ {MetricName: "FreeStorageSpace", Value: 100, Count: 1, Sum: 100, Min: 100, Max: 100, Timestamp: ts}, }) require.NoError(t, err) @@ -2107,7 +2132,7 @@ func TestCloudWatchBackend_MetricStream_ExcludeFilter(t *testing.T) { baseline, _ := b.GetMetricStream("stream-exclude") // Excluded namespace: should not update stream. - _, err := b.PutMetricData("AWS/EC2", []cloudwatch.MetricDatum{ + err := b.PutMetricData("AWS/EC2", []cloudwatch.MetricDatum{ {MetricName: "CPUUtilization", Value: 50, Count: 1, Sum: 50, Min: 50, Max: 50, Timestamp: ts}, }) require.NoError(t, err) @@ -2117,7 +2142,7 @@ func TestCloudWatchBackend_MetricStream_ExcludeFilter(t *testing.T) { "excluded namespace should not update stream") // Non-excluded namespace: should update. - _, err = b.PutMetricData("AWS/RDS", []cloudwatch.MetricDatum{ + err = b.PutMetricData("AWS/RDS", []cloudwatch.MetricDatum{ {MetricName: "FreeStorageSpace", Value: 99, Count: 1, Sum: 99, Min: 99, Max: 99, Timestamp: ts}, }) require.NoError(t, err) @@ -2136,7 +2161,7 @@ func TestCloudWatchBackend_ListMetrics_DimensionFilter(t *testing.T) { dimsA := []cloudwatch.Dimension{{Name: "Env", Value: "prod"}} dimsB := []cloudwatch.Dimension{{Name: "Env", Value: "staging"}} - _, err := b.PutMetricData("Custom", []cloudwatch.MetricDatum{ + err := b.PutMetricData("Custom", []cloudwatch.MetricDatum{ {MetricName: "RPM", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: ts, Dimensions: dimsA}, {MetricName: "RPM", Value: 2, Count: 1, Sum: 2, Min: 2, Max: 2, Timestamp: ts, Dimensions: dimsB}, {MetricName: "RPM", Value: 3, Count: 1, Sum: 3, Min: 3, Max: 3, Timestamp: ts}, @@ -2144,12 +2169,69 @@ func TestCloudWatchBackend_ListMetrics_DimensionFilter(t *testing.T) { require.NoError(t, err) // Filter to prod only. - prod, err := b.ListMetrics("Custom", "RPM", dimsA, "", 0) + prod, err := b.ListMetrics("Custom", "RPM", dimsA, "", "", 0) require.NoError(t, err) require.Len(t, prod.Data, 1) assert.Equal(t, "prod", prod.Data[0].Dimensions[0].Value) } +// Test_ListMetrics_RecentlyActive verifies the RecentlyActive=PT3H filter: +// AWS's only documented value, restricting results to metrics with a +// datapoint in the last 3 hours. Previously this parameter was not parsed or +// forwarded at all (silently ignored, never even reaching the backend), so a +// caller filtering for recently-active metrics got back everything instead. +func Test_ListMetrics_RecentlyActive(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + recentlyActive string + wantNames []string + wantErr bool + }{ + { + name: "no filter returns everything", + recentlyActive: "", + wantNames: []string{"Fresh", "Stale"}, + }, + { + name: "PT3H returns only recently-active metrics", + recentlyActive: "PT3H", + wantNames: []string{"Fresh"}, + }, + { + name: "invalid value rejected", + recentlyActive: "PT1H", + wantErr: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := cloudwatch.NewInMemoryBackend() + now := time.Now().UTC() + + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + {MetricName: "Fresh", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: now.Add(-time.Minute)}, + {MetricName: "Stale", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: now.Add(-4 * time.Hour)}, + }) + require.NoError(t, err) + + p, lerr := b.ListMetrics("NS", "", nil, tc.recentlyActive, "", 0) + if tc.wantErr { + require.ErrorIs(t, lerr, cloudwatch.ErrValidation) + + return + } + + require.NoError(t, lerr) + assert.ElementsMatch(t, tc.wantNames, metricNames(p.Data)) + }) + } +} + func TestCloudWatchBackend_ListMetrics_ReturnsDimensions(t *testing.T) { t.Parallel() @@ -2159,7 +2241,7 @@ func TestCloudWatchBackend_ListMetrics_ReturnsDimensions(t *testing.T) { {Name: "Service", Value: "web"}, } - _, err := b.PutMetricData("Infra", []cloudwatch.MetricDatum{ + err := b.PutMetricData("Infra", []cloudwatch.MetricDatum{ { MetricName: "Errors", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: time.Now(), Dimensions: dims, @@ -2167,7 +2249,7 @@ func TestCloudWatchBackend_ListMetrics_ReturnsDimensions(t *testing.T) { }) require.NoError(t, err) - p, err := b.ListMetrics("Infra", "Errors", nil, "", 0) + p, err := b.ListMetrics("Infra", "Errors", nil, "", "", 0) require.NoError(t, err) require.Len(t, p.Data, 1) assert.Len(t, p.Data[0].Dimensions, 2, "dimensions should be returned in ListMetrics") @@ -2179,7 +2261,7 @@ func TestCloudWatchBackend_StorageResolution_StoredOnDatum(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC() - _, err := b.PutMetricData("App", []cloudwatch.MetricDatum{ + err := b.PutMetricData("App", []cloudwatch.MetricDatum{ { MetricName: "Ticks", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: ts, StorageResolution: 1, @@ -2188,7 +2270,7 @@ func TestCloudWatchBackend_StorageResolution_StoredOnDatum(t *testing.T) { require.NoError(t, err) // Metric should be stored and queryable. - p, err := b.ListMetrics("App", "Ticks", nil, "", 0) + p, err := b.ListMetrics("App", "Ticks", nil, "", "", 0) require.NoError(t, err) require.Len(t, p.Data, 1) } @@ -2204,7 +2286,7 @@ func TestCloudWatchBackend_GetInsightRuleContributors(t *testing.T) { })) ts := time.Now().UTC().Add(-30 * time.Second) - _, err := b.PutMetricData("App", []cloudwatch.MetricDatum{ + err := b.PutMetricData("App", []cloudwatch.MetricDatum{ { MetricName: "Hits", Value: 10, Count: 10, Sum: 100, Min: 8, Max: 12, Timestamp: ts, Dimensions: []cloudwatch.Dimension{{Name: "Host", Value: "h1"}}, @@ -2240,7 +2322,7 @@ func TestCloudWatchBackend_DimensionOrderNormalized(t *testing.T) { // Query with dims in different order. dims2 := []cloudwatch.Dimension{{Name: "A", Value: "1"}, {Name: "B", Value: "2"}} - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ { MetricName: "M", Value: 42, Count: 1, Sum: 42, Min: 42, Max: 42, Timestamp: ts, Dimensions: dims1, diff --git a/services/cloudwatch/batch1_accuracy_test.go b/services/cloudwatch/batch1_accuracy_test.go index 9d0a56a76..7435e104c 100644 --- a/services/cloudwatch/batch1_accuracy_test.go +++ b/services/cloudwatch/batch1_accuracy_test.go @@ -2,6 +2,8 @@ package cloudwatch_test import ( "fmt" + "math" + "net/http" "net/url" "strings" "testing" @@ -24,7 +26,7 @@ func TestListMetrics_PartialDimensionFilter(t *testing.T) { ts := time.Now().UTC().Add(-time.Minute) // Store a metric with 2 dimensions. - _, err := b.PutMetricData("App", []cloudwatch.MetricDatum{ + err := b.PutMetricData("App", []cloudwatch.MetricDatum{ { MetricName: "CPU", Value: 80, Count: 1, Sum: 80, Min: 80, Max: 80, @@ -35,7 +37,7 @@ func TestListMetrics_PartialDimensionFilter(t *testing.T) { require.NoError(t, err) // Filter by only one dimension – should still match (partial filter). - p, err := b.ListMetrics("App", "CPU", []cloudwatch.Dimension{{Name: "Env", Value: "prod"}}, "", 0) + p, err := b.ListMetrics("App", "CPU", []cloudwatch.Dimension{{Name: "Env", Value: "prod"}}, "", "", 0) require.NoError(t, err) assert.Len(t, p.Data, 1, "partial dimension filter should match metric with superset of dims") } @@ -46,7 +48,7 @@ func TestListMetrics_PartialDimensionFilter_NoMatch(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC().Add(-time.Minute) - _, err := b.PutMetricData("App", []cloudwatch.MetricDatum{ + err := b.PutMetricData("App", []cloudwatch.MetricDatum{ { MetricName: "CPU", Value: 80, Count: 1, Sum: 80, Min: 80, Max: 80, Timestamp: ts, @@ -56,7 +58,7 @@ func TestListMetrics_PartialDimensionFilter_NoMatch(t *testing.T) { require.NoError(t, err) // Filter by a dimension that doesn't exist on the metric. - p, err := b.ListMetrics("App", "CPU", []cloudwatch.Dimension{{Name: "Env", Value: "staging"}}, "", 0) + p, err := b.ListMetrics("App", "CPU", []cloudwatch.Dimension{{Name: "Env", Value: "staging"}}, "", "", 0) require.NoError(t, err) assert.Empty(t, p.Data, "non-matching filter should return no metrics") } @@ -67,7 +69,7 @@ func TestListMetrics_MultiDimFilter_AllMustMatch(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC().Add(-time.Minute) - _, err := b.PutMetricData("Svc", []cloudwatch.MetricDatum{ + err := b.PutMetricData("Svc", []cloudwatch.MetricDatum{ { MetricName: "Req", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: ts, @@ -84,7 +86,7 @@ func TestListMetrics_MultiDimFilter_AllMustMatch(t *testing.T) { p, err := b.ListMetrics( "Svc", "Req", []cloudwatch.Dimension{{Name: "A", Value: "1"}, {Name: "B", Value: "2"}}, - "", 0, + "", "", 0, ) require.NoError(t, err) assert.Len(t, p.Data, 1, "all filter dims must match") @@ -150,7 +152,7 @@ func TestBackend_GetMetricStatistics_ExtendedStats_Computed(t *testing.T) { b := cloudwatch.NewInMemoryBackend() base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "Lat", Value: 10, Count: 1, Sum: 10, Min: 10, Max: 10, Timestamp: base.Add(10 * time.Second)}, {MetricName: "Lat", Value: 50, Count: 1, Sum: 50, Min: 50, Max: 50, Timestamp: base.Add(20 * time.Second)}, {MetricName: "Lat", Value: 90, Count: 1, Sum: 90, Min: 90, Max: 90, Timestamp: base.Add(30 * time.Second)}, @@ -175,7 +177,7 @@ func TestBackend_GetMetricStatistics_p99_HigherThanMedian(t *testing.T) { vals := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 100} for i, v := range vals { - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ { MetricName: "M", Value: v, @@ -210,7 +212,7 @@ func TestBackend_GetMetricData_ConstantMultiply(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC().Add(-30 * time.Second) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "M", Value: 10, Count: 1, Sum: 10, Min: 10, Max: 10, Timestamp: ts}, }) require.NoError(t, err) @@ -237,7 +239,7 @@ func TestBackend_GetMetricData_ConstantAdd(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC().Add(-30 * time.Second) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "M", Value: 10, Count: 1, Sum: 10, Min: 10, Max: 10, Timestamp: ts}, }) require.NoError(t, err) @@ -264,7 +266,7 @@ func TestBackend_GetMetricData_ConstantDivide(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC().Add(-30 * time.Second) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "M", Value: 100, Count: 1, Sum: 100, Min: 100, Max: 100, Timestamp: ts}, }) require.NoError(t, err) @@ -290,7 +292,7 @@ func TestBackend_GetMetricData_ConstantSubtract(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC().Add(-30 * time.Second) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "M", Value: 50, Count: 1, Sum: 50, Min: 50, Max: 50, Timestamp: ts}, }) require.NoError(t, err) @@ -316,7 +318,7 @@ func TestBackend_GetMetricData_ConstantLeftSide(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC().Add(-30 * time.Second) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "M", Value: 5, Count: 1, Sum: 5, Min: 5, Max: 5, Timestamp: ts}, }) require.NoError(t, err) @@ -342,7 +344,7 @@ func TestBackend_GetMetricData_ConstantLeftMultiply(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC().Add(-30 * time.Second) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "M", Value: 3, Count: 1, Sum: 3, Min: 3, Max: 3, Timestamp: ts}, }) require.NoError(t, err) @@ -374,7 +376,7 @@ func TestBackend_GetMetricData_AvgMetrics(t *testing.T) { for _, v := range []float64{10, 30} { mn := fmt.Sprintf("M%.0f", v) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: mn, Value: v, Count: 1, Sum: v, Min: v, Max: v, Timestamp: ts}, }) require.NoError(t, err) @@ -407,7 +409,7 @@ func TestBackend_GetMetricData_MinMetrics(t *testing.T) { for _, v := range []float64{5, 15, 25} { mn := fmt.Sprintf("M%.0f", v) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: mn, Value: v, Count: 1, Sum: v, Min: v, Max: v, Timestamp: ts}, }) require.NoError(t, err) @@ -448,7 +450,7 @@ func TestBackend_GetMetricData_MaxMetrics(t *testing.T) { for _, v := range []float64{5, 15, 99} { mn := fmt.Sprintf("M%.0f", v) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: mn, Value: v, Count: 1, Sum: v, Min: v, Max: v, Timestamp: ts}, }) require.NoError(t, err) @@ -489,7 +491,7 @@ func TestBackend_GetMetricData_StddevMetrics(t *testing.T) { for _, v := range []float64{10, 20, 30} { mn := fmt.Sprintf("M%.0f", v) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: mn, Value: v, Count: 1, Sum: v, Min: v, Max: v, Timestamp: ts}, }) require.NoError(t, err) @@ -533,7 +535,7 @@ func TestBackend_GetMetricData_RateFunction(t *testing.T) { base := time.Date(2024, 1, 1, 0, 1, 0, 0, time.UTC) // align to minute boundary // Two data points 60 seconds apart, value increases by 120. - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "Counter", Value: 0, Count: 1, Sum: 0, Min: 0, Max: 0, Timestamp: base}, {MetricName: "Counter", Value: 120, Count: 1, Sum: 120, Min: 120, Max: 120, Timestamp: base.Add(time.Minute)}, }) @@ -635,10 +637,14 @@ func TestHandler_PutMetricData_StatisticSetOnly_Accepted(t *testing.T) { "&MetricData.member.1.Timestamp=2024-01-01T00:00:00Z", ) assert.Equal(t, 200, rec.Code) - // Unprocessed entries have ; absence means all data was accepted. + // PutMetricDataOutput has no members: a 200 response body carries only ResponseMetadata. assert.NotContains(t, rec.Body.String(), "") } +// TestHandler_PutMetricData_StatisticSetAndValue_Rejected verifies that an +// invalid datum fails the whole request with a real AWS error response, not a +// 200 with a fabricated UnprocessedMetricData list (PutMetricDataOutput has no +// such field — confirmed against aws-sdk-go-v2 cloudwatch types). func TestHandler_PutMetricData_StatisticSetAndValue_Rejected(t *testing.T) { t.Parallel() @@ -654,8 +660,8 @@ func TestHandler_PutMetricData_StatisticSetAndValue_Rejected(t *testing.T) { "&MetricData.member.1.StatisticValues.Maximum=60"+ "&MetricData.member.1.Timestamp=2024-01-01T00:00:00Z", ) - assert.Equal(t, 200, rec.Code) - assert.Contains(t, rec.Body.String(), "UnprocessedMetricData") + assert.Equal(t, 400, rec.Code) + assert.Contains(t, rec.Body.String(), "InvalidParameterCombination") } func TestHandler_PutMetricData_StatisticSet_StoredCorrectly(t *testing.T) { @@ -664,7 +670,7 @@ func TestHandler_PutMetricData_StatisticSet_StoredCorrectly(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ { MetricName: "Reqs", HasStatisticSet: true, Count: 10, Sum: 500, Min: 20, Max: 80, @@ -683,6 +689,222 @@ func TestHandler_PutMetricData_StatisticSet_StoredCorrectly(t *testing.T) { assert.InDelta(t, 10.0, *dps[0].SampleCount, 1e-9, "StatisticSet SampleCount should be stored") } +// --------------------------------------------------------------------------- +// PutMetricData: Values/Counts array input +// +// Real CloudWatch lets a caller publish up to 150 unique values per datum via +// parallel Values/Counts arrays instead of a single Value or a StatisticSet +// (see aws-sdk-go-v2 cloudwatch/types.MetricDatum.Values doc comment). Prior to +// this test the form and rpc-v2-cbor parsers silently dropped this field +// entirely, so any client using it lost data with no error. +// --------------------------------------------------------------------------- + +func TestHandler_PutMetricData_ValuesCountsArray_StoredCorrectly(t *testing.T) { + t.Parallel() + + h := cloudwatch.NewHandler(cloudwatch.NewInMemoryBackend()) + rec := postForm( + t, h, + "Action=PutMetricData&Namespace=App"+ + "&MetricData.member.1.MetricName=Latency"+ + "&MetricData.member.1.Values.member.1=10"+ + "&MetricData.member.1.Values.member.2=20"+ + "&MetricData.member.1.Values.member.3=30"+ + "&MetricData.member.1.Counts.member.1=2"+ + "&MetricData.member.1.Counts.member.2=3"+ + "&MetricData.member.1.Counts.member.3=5"+ + "&MetricData.member.1.Timestamp=2024-01-01T00:00:00Z", + ) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + + b := h.Backend.(*cloudwatch.InMemoryBackend) + ts := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + dps, err := b.GetMetricStatistics("App", "Latency", nil, + ts.Add(-time.Minute), ts.Add(time.Minute), 60, + []string{"Sum", "SampleCount", "Minimum", "Maximum"}, nil) + require.NoError(t, err) + require.Len(t, dps, 1) + + // count = 2+3+5 = 10; sum = 10*2 + 20*3 + 30*5 = 230; min = 10; max = 30. + require.NotNil(t, dps[0].SampleCount) + assert.InDelta(t, 10.0, *dps[0].SampleCount, 1e-9) + require.NotNil(t, dps[0].Sum) + assert.InDelta(t, 230.0, *dps[0].Sum, 1e-9) + require.NotNil(t, dps[0].Minimum) + assert.InDelta(t, 10.0, *dps[0].Minimum, 1e-9) + require.NotNil(t, dps[0].Maximum) + assert.InDelta(t, 30.0, *dps[0].Maximum, 1e-9) +} + +func Test_PutMetricData_ValuesCountsArray(t *testing.T) { + t.Parallel() + + ts := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + + cases := []struct { + wantErr error + name string + datum cloudwatch.MetricDatum + wantSum float64 + wantCount float64 + wantMin float64 + wantMax float64 + wantStored bool + }{ + { + name: "counts default to 1 when omitted", + datum: cloudwatch.MetricDatum{ + MetricName: "M", Timestamp: ts, + HasValuesArray: true, + Values: []float64{4, 8}, + Counts: []float64{1, 1}, + }, + wantStored: true, wantSum: 12, wantCount: 2, wantMin: 4, wantMax: 8, + }, + { + name: "weighted counts aggregate correctly", + datum: cloudwatch.MetricDatum{ + MetricName: "M", Timestamp: ts, + HasValuesArray: true, + Values: []float64{1, 2, 3}, + Counts: []float64{10, 20, 30}, + }, + // sum = 1*10 + 2*20 + 3*30 = 140; count = 60. + wantStored: true, wantSum: 140, wantCount: 60, wantMin: 1, wantMax: 3, + }, + { + name: "mismatched Values/Counts lengths rejected", + datum: cloudwatch.MetricDatum{ + MetricName: "M", Timestamp: ts, + HasValuesArray: true, + Values: []float64{1, 2, 3}, + Counts: []float64{1, 1}, + }, + wantErr: cloudwatch.ErrValuesCountsLengthMismatch, + }, + { + name: "more than 150 values rejected", + datum: cloudwatch.MetricDatum{ + MetricName: "M", Timestamp: ts, + HasValuesArray: true, + Values: make([]float64, 151), + Counts: make([]float64, 151), + }, + wantErr: cloudwatch.ErrTooManyValues, + }, + { + name: "NaN entry rejected", + datum: cloudwatch.MetricDatum{ + MetricName: "M", Timestamp: ts, + HasValuesArray: true, + Values: []float64{1, math.NaN()}, + Counts: []float64{1, 1}, + }, + wantErr: cloudwatch.ErrInvalidMetricValue, + }, + { + name: "Values array combined with a plain Value is rejected", + datum: cloudwatch.MetricDatum{ + MetricName: "M", Timestamp: ts, + HasValuesArray: true, + Values: []float64{1}, + Counts: []float64{1}, + HasValue: true, + Value: 5, + }, + wantErr: cloudwatch.ErrValueAndStatisticSet, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := cloudwatch.NewInMemoryBackend() + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{tc.datum}) + + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + + return + } + + require.NoError(t, err) + + if !tc.wantStored { + return + } + + dps, gerr := b.GetMetricStatistics("NS", "M", nil, + ts.Add(-time.Minute), ts.Add(time.Minute), 60, + []string{"Sum", "SampleCount", "Minimum", "Maximum"}, nil) + require.NoError(t, gerr) + require.Len(t, dps, 1) + require.NotNil(t, dps[0].Sum) + assert.InDelta(t, tc.wantSum, *dps[0].Sum, 1e-9, "Sum") + require.NotNil(t, dps[0].SampleCount) + assert.InDelta(t, tc.wantCount, *dps[0].SampleCount, 1e-9, "SampleCount") + require.NotNil(t, dps[0].Minimum) + assert.InDelta(t, tc.wantMin, *dps[0].Minimum, 1e-9, "Minimum") + require.NotNil(t, dps[0].Maximum) + assert.InDelta(t, tc.wantMax, *dps[0].Maximum, 1e-9, "Maximum") + }) + } +} + +func Test_ValidateMetricDatum_ValueRange(t *testing.T) { + t.Parallel() + + cases := []struct { + wantErr error + name string + datum cloudwatch.MetricDatum + }{ + { + name: "finite value in range accepted", + datum: cloudwatch.MetricDatum{MetricName: "M", HasValue: true, Value: 42.5}, + }, + { + name: "NaN value rejected", + datum: cloudwatch.MetricDatum{MetricName: "M", HasValue: true, Value: math.NaN()}, + wantErr: cloudwatch.ErrInvalidMetricValue, + }, + { + name: "+Inf value rejected", + datum: cloudwatch.MetricDatum{MetricName: "M", HasValue: true, Value: math.Inf(1)}, + wantErr: cloudwatch.ErrInvalidMetricValue, + }, + { + name: "-Inf value rejected", + datum: cloudwatch.MetricDatum{MetricName: "M", HasValue: true, Value: math.Inf(-1)}, + wantErr: cloudwatch.ErrInvalidMetricValue, + }, + { + name: "StatisticSet with NaN Sum rejected", + datum: cloudwatch.MetricDatum{ + MetricName: "M", HasStatisticSet: true, + Count: 1, Sum: math.NaN(), Min: 1, Max: 1, + }, + wantErr: cloudwatch.ErrInvalidMetricValue, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + err := cloudwatch.ValidateMetricDatumForTest(tc.datum) + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + + return + } + + require.NoError(t, err) + }) + } +} + // --------------------------------------------------------------------------- // PutMetricData: unit round-trip // --------------------------------------------------------------------------- @@ -693,7 +915,7 @@ func TestBackend_PutMetricData_UnitStored(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC().Add(-30 * time.Second) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "M", Unit: "Milliseconds", Value: 5, Count: 1, Sum: 5, Min: 5, Max: 5, Timestamp: ts}, }) require.NoError(t, err) @@ -1481,7 +1703,7 @@ func TestBackend_ListMetrics_Pagination(t *testing.T) { ts := time.Now().UTC().Add(-time.Minute) for i := range 10 { - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ { MetricName: fmt.Sprintf("M%02d", i), Value: float64(i), Count: 1, Sum: float64(i), Min: float64(i), Max: float64(i), Timestamp: ts, @@ -1491,13 +1713,13 @@ func TestBackend_ListMetrics_Pagination(t *testing.T) { } // First page of 4. - p1, err := b.ListMetrics("NS", "", nil, "", 4) + p1, err := b.ListMetrics("NS", "", nil, "", "", 4) require.NoError(t, err) assert.Len(t, p1.Data, 4) assert.NotEmpty(t, p1.Next) // Second page. - p2, err := b.ListMetrics("NS", "", nil, p1.Next, 4) + p2, err := b.ListMetrics("NS", "", nil, "", p1.Next, 4) require.NoError(t, err) assert.Len(t, p2.Data, 4) @@ -1522,13 +1744,13 @@ func TestBackend_SweepExpiredMetrics_RemovesOldPoints(t *testing.T) { // Put a point 20 days ago (beyond retention). old := time.Now().UTC().AddDate(0, 0, -(cloudwatch.CwMetricRetentionDays + 1)) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "Old", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: old}, }) require.NoError(t, err) // Put a recent point. - _, err = b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err = b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "Recent", Value: 2, Count: 1, Sum: 2, Min: 2, Max: 2, Timestamp: time.Now().UTC()}, }) require.NoError(t, err) @@ -1536,12 +1758,12 @@ func TestBackend_SweepExpiredMetrics_RemovesOldPoints(t *testing.T) { b.SweepExpiredMetrics() // Old metric should be gone. - p, err := b.ListMetrics("NS", "Old", nil, "", 0) + p, err := b.ListMetrics("NS", "Old", nil, "", "", 0) require.NoError(t, err) assert.Empty(t, p.Data, "expired metric should be swept") // Recent metric should remain. - p, err = b.ListMetrics("NS", "Recent", nil, "", 0) + p, err = b.ListMetrics("NS", "Recent", nil, "", "", 0) require.NoError(t, err) assert.Len(t, p.Data, 1, "recent metric should survive sweep") } @@ -1652,7 +1874,7 @@ func TestBackend_PutMetricData_ExceedsPerRequestLimit(t *testing.T) { } } - _, err := b.PutMetricData("NS", data1001) + err := b.PutMetricData("NS", data1001) assert.Error(t, err, "PutMetricData should reject > 1000 entries per request") } @@ -1666,7 +1888,7 @@ func TestBackend_GetMetricData_MultipleStatQueries(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC().Add(-30 * time.Second) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "CPU", Value: 50, Count: 1, Sum: 50, Min: 50, Max: 50, Timestamp: ts}, {MetricName: "Mem", Value: 80, Count: 1, Sum: 80, Min: 80, Max: 80, Timestamp: ts}, }) @@ -1728,7 +1950,7 @@ func TestBackend_GetMetricData_ReturnDataFalse_SuppressesResult(t *testing.T) { b := cloudwatch.NewInMemoryBackend() ts := time.Now().UTC().Add(-30 * time.Second) - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "M", Value: 10, Count: 1, Sum: 10, Min: 10, Max: 10, Timestamp: ts}, }) require.NoError(t, err) @@ -1762,7 +1984,7 @@ func TestBackend_GetMetricStatistics_PeriodBuckets(t *testing.T) { // Three points in different 60-second buckets. for i, v := range []float64{10, 20, 30} { - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ { MetricName: "M", Value: v, Count: 1, Sum: v, Min: v, Max: v, @@ -1785,7 +2007,7 @@ func TestBackend_GetMetricStatistics_AggregatesWithinPeriod(t *testing.T) { base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) // Two points within the same 60-second bucket. - _, err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "M", Value: 10, Count: 1, Sum: 10, Min: 10, Max: 10, Timestamp: base.Add(5 * time.Second)}, {MetricName: "M", Value: 30, Count: 1, Sum: 30, Min: 30, Max: 30, Timestamp: base.Add(10 * time.Second)}, }) diff --git a/services/cloudwatch/handler.go b/services/cloudwatch/handler.go index 4ca31e554..caa8d1830 100644 --- a/services/cloudwatch/handler.go +++ b/services/cloudwatch/handler.go @@ -86,6 +86,9 @@ const ( const cloudwatchNS = "http://monitoring.amazonaws.com/doc/2010-08-01/" +// errCodeInternalFailure is the AWS error code for an unclassified server-side failure. +const errCodeInternalFailure = "InternalFailure" + // formFalse is the string value "false" as submitted in form-encoded CloudWatch requests. const formFalse = "false" @@ -634,8 +637,11 @@ func writeXML(c *echo.Context, v any) error { } // parseMetricDataFromForm parses MetricData.member.N.* form values. -// Supports both the Value field and the StatisticSet (pre-aggregated) path. -// Also parses Dimensions and StorageResolution. +// Supports the Value field, the StatisticSet (pre-aggregated) path, and the +// Values/Counts array path. Also parses Dimensions and StorageResolution. +// Each datum's Has* flags record which shape (if any) was present on the wire; +// validateMetricDatum uses those flags to reject ambiguous combinations rather +// than guessing from zero values. func parseMetricDataFromForm(form url.Values) []MetricDatum { var data []MetricDatum for i := 1; ; i++ { @@ -644,61 +650,117 @@ func parseMetricDataFromForm(form url.Values) []MetricDatum { if name == "" { return data } - unit := form.Get(prefix + "Unit") - // Parse optional Timestamp (fall back to now). - ts := time.Now().UTC() - if tsStr := form.Get(prefix + "Timestamp"); tsStr != "" { - if t, err := time.Parse(time.RFC3339, tsStr); err == nil { - ts = t.UTC() - } + datum := parseMetricDatumBaseFields(form, prefix, name) + applyMetricDatumShape(form, prefix, &datum) + data = append(data, datum) + } +} + +// parseMetricDatumBaseFields parses the shape-independent fields of a single +// MetricData.member.N entry: MetricName, Unit, Timestamp, StorageResolution, +// and Dimensions. +func parseMetricDatumBaseFields(form url.Values, prefix, name string) MetricDatum { + unit := form.Get(prefix + "Unit") + + // Parse optional Timestamp (fall back to now). + ts := time.Now().UTC() + if tsStr := form.Get(prefix + "Timestamp"); tsStr != "" { + if t, err := time.Parse(time.RFC3339, tsStr); err == nil { + ts = t.UTC() } + } - storageRes, _ := strconv.ParseInt(form.Get(prefix+"StorageResolution"), 10, 32) - dims := parseDimensionsFromForm(form, prefix+"Dimensions.") - - // StatisticSet takes precedence over Value when present. - ssCount := form.Get(prefix + "StatisticValues.SampleCount") - if ssCount != "" { - count, _ := strconv.ParseFloat(ssCount, 64) - sum, _ := strconv.ParseFloat(form.Get(prefix+"StatisticValues.Sum"), 64) - minimum, _ := strconv.ParseFloat(form.Get(prefix+"StatisticValues.Minimum"), 64) - maximum, _ := strconv.ParseFloat(form.Get(prefix+"StatisticValues.Maximum"), 64) - // Check if caller also supplied a Value (mutual exclusion with StatisticSet). - rawValue := form.Get(prefix + "Value") - // Preserve the Value if caller sent both; validateMetricDatum rejects it. - rawVal, _ := strconv.ParseFloat(rawValue, 64) - data = append(data, MetricDatum{ - MetricName: name, - Unit: unit, - Timestamp: ts, - Count: count, - Sum: sum, - Min: minimum, - Max: maximum, - Dimensions: dims, - StorageResolution: int32(storageRes), - // Mark as StatisticSet so the backend can enforce mutual exclusion. - HasStatisticSet: true, - Value: rawVal, - }) + storageRes, _ := strconv.ParseInt(form.Get(prefix+"StorageResolution"), 10, 32) + dims := parseDimensionsFromForm(form, prefix+"Dimensions.") + + return MetricDatum{ + MetricName: name, + Unit: unit, + Timestamp: ts, + Dimensions: dims, + StorageResolution: int32(storageRes), + } +} + +// applyMetricDatumShape parses whichever of the three mutually-exclusive +// PutMetricData input shapes (Values/Counts, StatisticValues, or Value) is +// present in the form for this entry and populates datum accordingly. +func applyMetricDatumShape(form url.Values, prefix string, datum *MetricDatum) { + if values, hasValues := parseFloatMemberList(form, prefix+"Values."); hasValues { + applyValuesArrayShape(form, prefix, datum, values) - continue + return + } + + if ssCount := form.Get(prefix + "StatisticValues.SampleCount"); ssCount != "" { + applyStatisticSetShape(form, prefix, datum, ssCount) + + return + } + + rawValue := form.Get(prefix + "Value") + val, _ := strconv.ParseFloat(rawValue, 64) + datum.HasValue = rawValue != "" + datum.Value = val + datum.Count = 1 + datum.Sum = val + datum.Min = val + datum.Max = val +} + +// applyValuesArrayShape populates datum from a Values/Counts array pair, +// defaulting Counts to all-1s when the caller omits it. The backend +// (storeDatum) derives Sum/SampleCount/Min/Max from Values/Counts at store +// time, so this only needs to carry the raw arrays through. +func applyValuesArrayShape(form url.Values, prefix string, datum *MetricDatum, values []float64) { + counts, hasCounts := parseFloatMemberList(form, prefix+"Counts.") + if !hasCounts { + counts = make([]float64, len(values)) + for j := range counts { + counts[j] = 1 } + } - val, _ := strconv.ParseFloat(form.Get(prefix+"Value"), 64) - data = append(data, MetricDatum{ - MetricName: name, - Value: val, - Unit: unit, - Timestamp: ts, - Count: 1, - Sum: val, - Min: val, - Max: val, - Dimensions: dims, - StorageResolution: int32(storageRes), - }) + datum.HasValuesArray = true + datum.Values = values + datum.Counts = counts +} + +// applyStatisticSetShape populates datum from a StatisticValues input. A +// caller may also (invalidly) supply Value alongside it; that is preserved so +// validateMetricDatum can reject the combination. +func applyStatisticSetShape(form url.Values, prefix string, datum *MetricDatum, ssCount string) { + count, _ := strconv.ParseFloat(ssCount, 64) + sum, _ := strconv.ParseFloat(form.Get(prefix+"StatisticValues.Sum"), 64) + minimum, _ := strconv.ParseFloat(form.Get(prefix+"StatisticValues.Minimum"), 64) + maximum, _ := strconv.ParseFloat(form.Get(prefix+"StatisticValues.Maximum"), 64) + datum.HasStatisticSet = true + datum.Count = count + datum.Sum = sum + datum.Min = minimum + datum.Max = maximum + + if rawValue := form.Get(prefix + "Value"); rawValue != "" { + datum.HasValue = true + datum.Value, _ = strconv.ParseFloat(rawValue, 64) + } +} + +// parseFloatMemberList parses a "Prefix.member.N" list of floats (e.g. +// "MetricData.member.1.Values.member.1"). Returns ok=false when no members are +// present so callers can distinguish an absent list from an explicit empty one. +func parseFloatMemberList(form url.Values, prefix string) ([]float64, bool) { + var vals []float64 + + for i := 1; ; i++ { + raw := form.Get(fmt.Sprintf("%smember.%d", prefix, i)) + if raw == "" { + return vals, len(vals) > 0 + } + + f, _ := strconv.ParseFloat(raw, 64) + vals = append(vals, f) } } @@ -822,29 +884,48 @@ func (h *Handler) handlePutMetricData(form url.Values, c *echo.Context) error { ) } data := parseMetricDataFromForm(form) - unprocessed, err := h.Backend.PutMetricData(namespace, data) - if err != nil { - return h.xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) + if err := h.Backend.PutMetricData(namespace, data); err != nil { + return h.xmlError(c, putMetricDataErrorStatus(err), putMetricDataErrorCode(err), err.Error()) } - type unprocessedXML struct { - MetricName string `xml:"MetricName"` - ErrorCode string `xml:"ErrorCode"` - ErrorMessage string `xml:"ErrorMessage,omitempty"` - } + // PutMetricDataOutput has no members besides the request ID: CloudWatch has + // no partial-success concept for this operation, so a 200 always means every + // datum in the request was accepted. type response struct { - XMLName xml.Name `xml:"PutMetricDataResponse"` - Xmlns string `xml:"xmlns,attr"` - RequestID string `xml:"ResponseMetadata>RequestId"` - UnprocessedMetrics []unprocessedXML `xml:"PutMetricDataResult>UnprocessedMetricData>member,omitempty"` + XMLName xml.Name `xml:"PutMetricDataResponse"` + Xmlns string `xml:"xmlns,attr"` + RequestID string `xml:"ResponseMetadata>RequestId"` } - resp := response{Xmlns: cloudwatchNS, RequestID: uuid.New().String()} - for _, u := range unprocessed { - resp.UnprocessedMetrics = append(resp.UnprocessedMetrics, unprocessedXML(u)) + return writeXML(c, response{Xmlns: cloudwatchNS, RequestID: uuid.New().String()}) +} + +// putMetricDataErrorCode maps a PutMetricData validation error to its AWS error +// code. Order matters: more specific sentinels must be checked before the +// generic ErrValidation they may also match via errors.Is chains. +func putMetricDataErrorCode(err error) string { + switch { + case errors.Is(err, ErrValueAndStatisticSet): + return "InvalidParameterCombination" + case errors.Is(err, ErrMetricSeriesLimitExceeded): + return "LimitExceeded" + case errors.Is(err, ErrValuesCountsLengthMismatch), + errors.Is(err, ErrTooManyValues), + errors.Is(err, ErrInvalidMetricValue), + errors.Is(err, ErrValidation): + return "InvalidParameterValue" + default: + return errCodeInternalFailure } +} - return writeXML(c, resp) +// putMetricDataErrorStatus maps a PutMetricData validation error to its HTTP status. +func putMetricDataErrorStatus(err error) int { + if putMetricDataErrorCode(err) == errCodeInternalFailure { + return http.StatusInternalServerError + } + + return http.StatusBadRequest } func (h *Handler) handleGetMetricStatistics(form url.Values, c *echo.Context) error { @@ -940,12 +1021,17 @@ func (h *Handler) handleListMetrics(form url.Values, c *echo.Context) error { namespace := form.Get("Namespace") metricName := form.Get("MetricName") nextToken := form.Get("NextToken") + recentlyActive := form.Get("RecentlyActive") maxResults, _ := strconv.Atoi(form.Get("MaxResults")) dimensions := parseDimensionsFromForm(form, "Dimensions.") - p, err := h.Backend.ListMetrics(namespace, metricName, dimensions, nextToken, maxResults) + p, err := h.Backend.ListMetrics(namespace, metricName, dimensions, recentlyActive, nextToken, maxResults) if err != nil { - return h.xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) + if errors.Is(err, ErrValidation) { + return h.xmlError(c, http.StatusBadRequest, "InvalidParameterValue", err.Error()) + } + + return h.xmlError(c, http.StatusInternalServerError, errCodeInternalFailure, err.Error()) } type dimXML struct { @@ -2836,7 +2922,7 @@ func (h *Handler) handlePutManagedInsightRules(form url.Values, c *echo.Context) }); err != nil { failures = append(failures, failureXML{ RuleName: ruleName, - FailureCode: "InternalFailure", + FailureCode: errCodeInternalFailure, FailureDescription: err.Error(), }) } diff --git a/services/cloudwatch/handler_test.go b/services/cloudwatch/handler_test.go index 0b5f42ad6..9207cf6bc 100644 --- a/services/cloudwatch/handler_test.go +++ b/services/cloudwatch/handler_test.go @@ -1838,7 +1838,7 @@ func TestCloudWatchHandler_GetMetricData_ScanByDescending(t *testing.T) { base := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) for i := 1; i <= 3; i++ { ts := base.Add(time.Duration(i) * time.Minute) - _, _ = b.PutMetricData("NS", []cloudwatch.MetricDatum{ + _ = b.PutMetricData("NS", []cloudwatch.MetricDatum{ { MetricName: "Counter", Value: float64(i), Count: 1, Sum: float64(i), Min: float64(i), Max: float64(i), Timestamp: ts, @@ -1876,7 +1876,12 @@ func TestCloudWatchHandler_GetMetricData_ScanByDescending(t *testing.T) { } } -func TestCloudWatchHandler_PutMetricData_UnprocessedOnCap(t *testing.T) { +// TestCloudWatchHandler_PutMetricData_RejectedOnCap verifies that once a +// namespace is at its distinct-time-series cap, submitting one more new series +// fails the whole PutMetricData call with a real AWS-shaped error (400 +// LimitExceeded) rather than a 200 with a fabricated per-datum "unprocessed" +// entry — PutMetricDataOutput carries no such field. +func TestCloudWatchHandler_PutMetricData_RejectedOnCap(t *testing.T) { t.Parallel() h := cloudwatch.NewHandler(cloudwatch.NewInMemoryBackendWithConfig("123456789012", "us-east-1")) @@ -1884,7 +1889,7 @@ func TestCloudWatchHandler_PutMetricData_UnprocessedOnCap(t *testing.T) { // Fill up the namespace to the cap first. b := h.Backend.(*cloudwatch.InMemoryBackend) for i := range cloudwatch.CwMaxMetricNamesPerNamespace { - _, _ = b.PutMetricData("FullNS", []cloudwatch.MetricDatum{ + _ = b.PutMetricData("FullNS", []cloudwatch.MetricDatum{ { MetricName: strings.Repeat("x", 1) + strings.Repeat("y", i%10) + strings.Repeat("z", i/10), Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, @@ -1894,10 +1899,13 @@ func TestCloudWatchHandler_PutMetricData_UnprocessedOnCap(t *testing.T) { body := "Action=PutMetricData&Namespace=FullNS&MetricData.member.1.MetricName=Overflow&MetricData.member.1.Value=1" rec := postForm(t, h, body) - require.Equal(t, http.StatusOK, rec.Code) + require.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "LimitExceeded") - // Response should contain an UnprocessedMetricData entry. - assert.Contains(t, rec.Body.String(), "Overflow") + // The overflow metric must not have been stored. + metric, err := b.ListMetrics("FullNS", "Overflow", nil, "", "", 0) + require.NoError(t, err) + assert.Empty(t, metric.Data) } func TestCloudWatchHandler_PutMetricStream_WithFilters(t *testing.T) { @@ -1939,7 +1947,7 @@ func TestCloudWatchHandler_GetInsightRuleReport_WithData(t *testing.T) { Name: "rule-test", Definition: `{}`, Schema: "CloudWatchLogRule", })) - _, _ = b.PutMetricData("App", []cloudwatch.MetricDatum{ + _ = b.PutMetricData("App", []cloudwatch.MetricDatum{ { MetricName: "Hits", Value: 100, Count: 100, Sum: 1000, Min: 5, Max: 15, Dimensions: []cloudwatch.Dimension{{Name: "Host", Value: "h1"}}, diff --git a/services/cloudwatch/leak_test.go b/services/cloudwatch/leak_test.go index 8b0320354..8890e0161 100644 --- a/services/cloudwatch/leak_test.go +++ b/services/cloudwatch/leak_test.go @@ -71,7 +71,7 @@ func TestPutMetricData_DatapointsBackingArrayBounded(t *testing.T) { now := time.Now().UTC() for i := range writes { - _, err := b.PutMetricData(namespace, []cloudwatch.MetricDatum{{ + err := b.PutMetricData(namespace, []cloudwatch.MetricDatum{{ MetricName: metricName, Timestamp: now.Add(time.Duration(i) * time.Second), Value: float64(i), diff --git a/services/cloudwatch/metricdata.go b/services/cloudwatch/metricdata.go index ba98a5b15..c46186df5 100644 --- a/services/cloudwatch/metricdata.go +++ b/services/cloudwatch/metricdata.go @@ -69,6 +69,35 @@ func decodeMetricDataToken(token string) metricDataCursor { return c } +// aggregateValuesCounts reduces a PutMetricData Values/Counts array pair (each +// Values[i] occurred Counts[i] times during the period) into the same +// Sum/SampleCount/Min/Max summary that a StatisticSet carries, so downstream +// bucket aggregation (populateBuckets) can treat every MetricDatum shape +// uniformly. Assumes len(values) == len(counts) and len(values) > 0; callers +// must validate the datum (via validateMetricDatum) before calling this. +func aggregateValuesCounts(values, counts []float64) (float64, float64, float64, float64) { + var sum, count float64 + + minV := math.MaxFloat64 + maxV := -math.MaxFloat64 + + for i, v := range values { + c := counts[i] + sum += v * c + count += c + + if v < minV { + minV = v + } + + if v > maxV { + maxV = v + } + } + + return sum, count, minV, maxV +} + // annotateArithmeticMessages inspects a resolved metric-math result for NaN/Inf // values. Any non-finite point demotes the result to PartialData and records a // single ArithmeticError message, matching AWS behaviour for expressions that hit diff --git a/services/cloudwatch/metricdata_test.go b/services/cloudwatch/metricdata_test.go index 443697264..4027c72a2 100644 --- a/services/cloudwatch/metricdata_test.go +++ b/services/cloudwatch/metricdata_test.go @@ -196,7 +196,7 @@ func TestGetMetricDataPagedEndToEnd(t *testing.T) { Max: float64(i + 1), }) } - if _, err := b.PutMetricData("MyApp", data); err != nil { + if err := b.PutMetricData("MyApp", data); err != nil { t.Fatalf("PutMetricData: %v", err) } diff --git a/services/cloudwatch/metricmath.go b/services/cloudwatch/metricmath.go index d3a7896ec..59163f5d0 100644 --- a/services/cloudwatch/metricmath.go +++ b/services/cloudwatch/metricmath.go @@ -830,13 +830,20 @@ func collectRawBuckets(all []MetricDatum, startTime, endTime time.Time, period i return buckets } -// expandDatumValues reconstructs an approximate sample distribution for a metric -// datum. A plain datum contributes its single Value. A StatisticValues datum -// (SampleCount/Sum/Min/Max) is expanded into SampleCount synthetic samples: one at -// Min, one at Max, and the remainder at the residual mean so that the reconstructed -// set preserves the datum's count, sum, minimum, and maximum exactly. The sample -// count is capped at maxStatSetExpand to bound memory. +// expandDatumValues reconstructs a sample distribution for a metric datum. A +// plain datum contributes its single Value. A Values/Counts datum is expanded +// exactly (each value repeated its count times, proportionally scaled down +// under maxStatSetExpand) since the real per-value distribution is known. A +// StatisticValues datum (SampleCount/Sum/Min/Max) is expanded into SampleCount +// synthetic samples: one at Min, one at Max, and the remainder at the residual +// mean so that the reconstructed set preserves the datum's count, sum, +// minimum, and maximum exactly. The sample count is capped at maxStatSetExpand +// to bound memory. func expandDatumValues(d MetricDatum) []float64 { + if d.HasValuesArray { + return expandValuesCounts(d.Values, d.Counts) + } + if !d.HasStatisticSet { return []float64{d.Value} } @@ -870,3 +877,43 @@ func expandDatumValues(d MetricDatum) []float64 { return vals } + +// expandValuesCounts reconstructs raw samples from a PutMetricData Values/Counts +// array pair: each values[i] occurred counts[i] times. When the total +// occurrence count is within maxStatSetExpand, expansion is exact. Otherwise +// each value's count is scaled down proportionally (rounded, minimum 1 for any +// value with a positive original count) so the relative frequencies — and +// therefore percentiles computed over the result — are preserved. +func expandValuesCounts(values, counts []float64) []float64 { + total := 0.0 + for _, c := range counts { + total += c + } + + if total <= 0 { + return nil + } + + scale := 1.0 + capacity := int(total) + + if total > maxStatSetExpand { + scale = float64(maxStatSetExpand) / total + capacity = maxStatSetExpand + } + + vals := make([]float64, 0, capacity) + + for i, v := range values { + n := int(math.Round(counts[i] * scale)) + if n < 1 && counts[i] > 0 { + n = 1 + } + + for range n { + vals = append(vals, v) + } + } + + return vals +} diff --git a/services/cloudwatch/models.go b/services/cloudwatch/models.go index ba89e5396..64be414c6 100644 --- a/services/cloudwatch/models.go +++ b/services/cloudwatch/models.go @@ -5,28 +5,38 @@ import ( ) // MetricDatum holds a single metric data point. +// +// A datum is built from exactly one of three mutually-exclusive input shapes: +// a single Value, a pre-aggregated StatisticValues set, or a Values/Counts +// array pair (up to 150 unique values, each with an occurrence count). The +// Has* flags record which shape was supplied on the wire so validation can +// enforce mutual exclusion by presence rather than by zero-value guessing. type MetricDatum struct { - MetricName string `json:"MetricName"` - Namespace string `json:"Namespace"` - Unit string `json:"Unit,omitempty"` - Timestamp time.Time `json:"Timestamp"` - Dimensions []Dimension `json:"Dimensions,omitempty"` - Value float64 `json:"Value"` - Count float64 `json:"SampleCount"` - Sum float64 `json:"Sum"` - Min float64 `json:"Min"` - Max float64 `json:"Max"` - StorageResolution int32 `json:"StorageResolution,omitempty"` - // HasStatisticSet is true when the datum was built from a StatisticValues input (not from Value). - // Used to enforce mutual exclusion of Value + StatisticSet at the handler level. + Timestamp time.Time `json:"Timestamp"` + MetricName string `json:"MetricName"` + Namespace string `json:"Namespace"` + Unit string `json:"Unit,omitempty"` + Dimensions []Dimension `json:"Dimensions,omitempty"` + // Values and Counts implement the PutMetricData "array of values" input: each + // Values[i] occurred Counts[i] times during the period. Populated only when + // HasValuesArray is true; Counts is defaulted to all-1s by the parser when the + // caller omits it. + Values []float64 `json:"Values,omitempty"` + Counts []float64 `json:"Counts,omitempty"` + Count float64 `json:"SampleCount"` + Value float64 `json:"Value"` + Sum float64 `json:"Sum"` + Min float64 `json:"Min"` + Max float64 `json:"Max"` + // StorageResolution is 1 (high-resolution) or 60 (standard); 0 means "use default". + StorageResolution int32 `json:"StorageResolution,omitempty"` + // HasStatisticSet is true when the datum was built from a StatisticValues input. HasStatisticSet bool `json:"-"` -} - -// UnprocessedMetricDatum describes a MetricDatum entry that could not be stored. -type UnprocessedMetricDatum struct { - MetricName string `json:"MetricName"` - ErrorCode string `json:"ErrorCode"` - ErrorMessage string `json:"ErrorMessage"` + // HasValuesArray is true when the datum was built from a Values/Counts input. + HasValuesArray bool `json:"-"` + // HasValue is true when the caller explicitly supplied the Value field + // (independent of whether that value is the float zero value). + HasValue bool `json:"-"` } // Metric represents a named metric (name+namespace+dimensions). diff --git a/services/cloudwatch/parity_test.go b/services/cloudwatch/parity_test.go index c69e66ac7..0fe1f2b58 100644 --- a/services/cloudwatch/parity_test.go +++ b/services/cloudwatch/parity_test.go @@ -356,14 +356,14 @@ func TestSweepExpiredMetrics_TwoPhase(t *testing.T) { b := cloudwatch.NewInMemoryBackendWithConfig("123456789012", "us-east-1") ts := time.Now().UTC().Add(-tc.putAge) - _, err := b.PutMetricData("NS/Sweep", []cloudwatch.MetricDatum{ + err := b.PutMetricData("NS/Sweep", []cloudwatch.MetricDatum{ {MetricName: "M", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: ts}, }) require.NoError(t, err) b.SweepExpiredMetrics() - metrics, err := b.ListMetrics("NS/Sweep", "M", nil, "", 0) + metrics, err := b.ListMetrics("NS/Sweep", "M", nil, "", "", 0) require.NoError(t, err) if tc.wantAlive { assert.Len(t, metrics.Data, 1, "live metric must survive sweep") diff --git a/services/cloudwatch/persistence.go b/services/cloudwatch/persistence.go index 44c8c0ec6..2180d1211 100644 --- a/services/cloudwatch/persistence.go +++ b/services/cloudwatch/persistence.go @@ -3,25 +3,31 @@ package cloudwatch import ( "context" "encoding/json" + "fmt" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) +// cloudwatchSnapshotVersion identifies the shape of backendSnapshot's Tables +// blob (i.e. the set/shape of resources registered on b.registry -- see +// registerAllTables in store_setup.go). It must be bumped whenever a change +// there would make an older snapshot unsafe to decode as the current shape. +// Restore compares this against the persisted value and discards (rather than +// attempts to partially decode) any mismatch -- see Restore below. This +// mirrors the services/sqs pilot (commit 0f09d77c) and the services/ec2 +// conversion (commit 12e611a4). +const cloudwatchSnapshotVersion = 1 + type backendSnapshot struct { - Metrics map[string]map[string]*metricRecord `json:"metrics"` - Alarms map[string]*MetricAlarm `json:"alarms"` - CompositeAlarms map[string]*CompositeAlarm `json:"compositeAlarms"` - AlarmHistory map[string][]AlarmHistoryItem `json:"alarmHistory"` - Dashboards map[string]*dashboardRecord `json:"dashboards"` - AnomalyDetectors map[string]*AnomalyDetector `json:"anomalyDetectors"` - InsightRules map[string]*InsightRule `json:"insightRules"` - MetricStreams map[string]*MetricStream `json:"metricStreams"` - AlarmMuteRules map[string]*AlarmMuteRule `json:"alarmMuteRules"` - MetricFilters map[string]*MetricFilter `json:"metricFilters"` - AccountID string `json:"accountID"` - Region string `json:"region"` - TotalMetrics int `json:"totalMetrics"` + Tables map[string]json.RawMessage `json:"tables"` + Metrics map[string]map[string]*metricRecord `json:"metrics"` + AlarmHistory map[string][]AlarmHistoryItem `json:"alarmHistory"` + AccountID string `json:"accountID"` + Region string `json:"region"` + Version int `json:"version"` + TotalMetrics int `json:"totalMetrics"` } // Snapshot serialises the backend state to JSON. @@ -30,20 +36,25 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "cloudwatch: snapshot table marshal failed", "error", err) + + return nil + } + snap := backendSnapshot{ - Metrics: b.metrics, - Alarms: b.alarms, - CompositeAlarms: b.compositeAlarms, - AlarmHistory: b.alarmHistory, - Dashboards: b.dashboards, - AnomalyDetectors: b.anomalyDetectors, - InsightRules: b.insightRules, - MetricStreams: b.metricStreams, - AlarmMuteRules: b.alarmMuteRules, - MetricFilters: b.metricFilters, - AccountID: b.accountID, - Region: b.region, - TotalMetrics: b.totalMetrics, + Version: cloudwatchSnapshotVersion, + Tables: tables, + Metrics: b.metrics, + AlarmHistory: b.alarmHistory, + AccountID: b.accountID, + Region: b.region, + TotalMetrics: b.totalMetrics, } return persistence.MarshalSnapshot(ctx, "cloudwatch", snap) @@ -61,56 +72,39 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() - if snap.Metrics == nil { - snap.Metrics = make(map[string]map[string]*metricRecord) + if snap.Version != cloudwatchSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/sqs pilot (commit 0f09d77c). + logger.Load(ctx).WarnContext(ctx, + "cloudwatch: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", cloudwatchSnapshotVersion) + + b.metrics = make(map[string]map[string]*metricRecord) + b.alarmHistory = make(map[string][]AlarmHistoryItem) + b.totalMetrics = 0 + b.registry.ResetAll() + + return nil } - if snap.Alarms == nil { - snap.Alarms = make(map[string]*MetricAlarm) + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("cloudwatch: restore snapshot tables: %w", err) } - if snap.CompositeAlarms == nil { - snap.CompositeAlarms = make(map[string]*CompositeAlarm) + if snap.Metrics == nil { + snap.Metrics = make(map[string]map[string]*metricRecord) } if snap.AlarmHistory == nil { snap.AlarmHistory = make(map[string][]AlarmHistoryItem) } - if snap.Dashboards == nil { - snap.Dashboards = make(map[string]*dashboardRecord) - } - - if snap.AnomalyDetectors == nil { - snap.AnomalyDetectors = make(map[string]*AnomalyDetector) - } - - if snap.InsightRules == nil { - snap.InsightRules = make(map[string]*InsightRule) - } - - if snap.MetricStreams == nil { - snap.MetricStreams = make(map[string]*MetricStream) - } - - if snap.AlarmMuteRules == nil { - snap.AlarmMuteRules = make(map[string]*AlarmMuteRule) - } - - if snap.MetricFilters == nil { - snap.MetricFilters = make(map[string]*MetricFilter) - } - b.metrics = snap.Metrics - b.alarms = snap.Alarms - b.compositeAlarms = snap.CompositeAlarms b.alarmHistory = snap.AlarmHistory - b.dashboards = snap.Dashboards - b.anomalyDetectors = snap.AnomalyDetectors - b.insightRules = snap.InsightRules - b.metricStreams = snap.MetricStreams - b.alarmMuteRules = snap.AlarmMuteRules - b.metricFilters = snap.MetricFilters b.accountID = snap.AccountID b.region = snap.Region b.totalMetrics = snap.TotalMetrics diff --git a/services/cloudwatch/persistence_test.go b/services/cloudwatch/persistence_test.go index 648f6a5f0..d993a2011 100644 --- a/services/cloudwatch/persistence_test.go +++ b/services/cloudwatch/persistence_test.go @@ -1,6 +1,7 @@ package cloudwatch_test import ( + "encoding/json" "net/http" "testing" "time" @@ -387,3 +388,169 @@ func TestHandler_SnapshotRestore_IncludesTags(t *testing.T) { require.Equal(t, http.StatusOK, rec.Code) assert.Contains(t, rec.Body.String(), "staging") } + +// TestInMemoryBackend_SnapshotRestore_FullState is a Phase 3.3 (pkgs/store +// conversion) regression test: it populates every top-level resource +// collection on the backend -- both the store.Table-backed ones (alarms, +// compositeAlarms, dashboards, anomalyDetectors, insightRules, metricStreams, +// alarmMuteRules, metricFilters) and the deliberately-raw ones (metrics via +// PutMetricData, alarmHistory via SetAlarmState) -- in a single backend, then +// verifies a Snapshot->Restore round trip into a fresh backend reproduces +// every one of them. metricFilters in particular had no prior snapshot/restore +// coverage. +func TestInMemoryBackend_SnapshotRestore_FullState(t *testing.T) { + t.Parallel() + + original := cloudwatch.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + + // metrics (raw, hot-path map). + require.NoError(t, original.PutMetricData("AWS/EC2", []cloudwatch.MetricDatum{ + { + MetricName: "CPUUtilization", + Value: 42, + Count: 1, + Sum: 42, + Min: 42, + Max: 42, + Timestamp: time.Now().UTC(), + }, + })) + + // alarms + alarmHistory (alarmHistory raw, populated as a side effect of PutMetricAlarm). + require.NoError(t, original.PutMetricAlarm(&cloudwatch.MetricAlarm{ + AlarmName: "full-state-alarm", + MetricName: "CPUUtilization", + Namespace: "AWS/EC2", + Statistic: "Average", + })) + + // compositeAlarms. + require.NoError(t, original.PutCompositeAlarm(&cloudwatch.CompositeAlarm{ + AlarmName: "full-state-composite", + AlarmRule: `ALARM("full-state-alarm")`, + })) + + // dashboards. + require.NoError(t, original.PutDashboard("full-state-dash", `{"widgets":[]}`)) + + // anomalyDetectors. + original.PutAnomalyDetectorInternal(&cloudwatch.AnomalyDetector{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Stat: "Average", + }) + + // insightRules. + original.PutInsightRuleInternal(&cloudwatch.InsightRule{Name: "full-state-rule", State: "ENABLED"}) + + // metricStreams. + original.PutMetricStreamInternal(&cloudwatch.MetricStream{ + Name: "full-state-stream", + FirehoseArn: "arn:aws:firehose:us-east-1:000000000000:deliverystream/full-state", + }) + + // alarmMuteRules. + original.PutAlarmMuteRuleInternal(&cloudwatch.AlarmMuteRule{MuteName: "full-state-mute"}) + + // metricFilters -- previously had no snapshot/restore test coverage at all. + require.NoError(t, original.PutMetricFilter(&cloudwatch.MetricFilter{ + FilterName: "full-state-filter", + LogGroupName: "/aws/lambda/full-state", + FilterPattern: "ERROR", + })) + + snap := original.Snapshot(t.Context()) + require.NotNil(t, snap) + + fresh := cloudwatch.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + require.NoError(t, fresh.Restore(t.Context(), snap)) + + // metrics. + stats, err := fresh.GetMetricStatistics( + "AWS/EC2", "CPUUtilization", nil, + time.Now().Add(-time.Hour), time.Now().Add(time.Hour), + 60, []string{"Average"}, nil, + ) + require.NoError(t, err) + require.NotEmpty(t, stats) + + // alarms + alarmHistory. + alarms, composites, err := fresh.DescribeAlarms(nil, nil, "", "", "", 0) + require.NoError(t, err) + require.Len(t, alarms.Data, 1) + assert.Equal(t, "full-state-alarm", alarms.Data[0].AlarmName) + + hist, err := fresh.DescribeAlarmHistory("full-state-alarm", "", "", "", time.Time{}, time.Time{}, 0) + require.NoError(t, err) + assert.NotEmpty(t, hist.Data) + + // compositeAlarms. + require.Len(t, composites.Data, 1) + assert.Equal(t, "full-state-composite", composites.Data[0].AlarmName) + + // dashboards. + entry, body, err := fresh.GetDashboard("full-state-dash") + require.NoError(t, err) + assert.Equal(t, "full-state-dash", entry.DashboardName) + assert.JSONEq(t, `{"widgets":[]}`, body) + + // anomalyDetectors. + detectors, err := fresh.DescribeAnomalyDetectors("AWS/EC2", "CPUUtilization", "", 0) + require.NoError(t, err) + require.Len(t, detectors.Data, 1) + + // insightRules. + rules, err := fresh.DescribeInsightRules("", 0) + require.NoError(t, err) + require.Len(t, rules.Data, 1) + assert.Equal(t, "full-state-rule", rules.Data[0].Name) + + // metricStreams. + stream, err := fresh.GetMetricStream("full-state-stream") + require.NoError(t, err) + assert.Equal(t, "full-state-stream", stream.Name) + + // alarmMuteRules. + _, err = fresh.GetAlarmMuteRule("full-state-mute") + require.NoError(t, err) + + // metricFilters. + filters, err := fresh.DescribeMetricFilters("", "", "", 0) + require.NoError(t, err) + require.Len(t, filters.Data, 1) + assert.Equal(t, "full-state-filter", filters.Data[0].FilterName) + assert.Equal(t, "ERROR", filters.Data[0].FilterPattern) +} + +// TestInMemoryBackend_Restore_VersionGuard exercises the Phase 3.3 snapshot +// version guard: a snapshot whose "version" field does not match the +// backend's current snapshot version must be discarded cleanly (reset to +// empty, no error) rather than partially decoded, since the Tables blob shape +// is not guaranteed compatible across versions. +func TestInMemoryBackend_Restore_VersionGuard(t *testing.T) { + t.Parallel() + + original := cloudwatch.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + require.NoError(t, original.PutMetricAlarm(&cloudwatch.MetricAlarm{AlarmName: "version-guard-alarm"})) + + snap := original.Snapshot(t.Context()) + require.NotNil(t, snap) + + // Corrupt the version field to simulate an incompatible (older/newer) snapshot. + var raw map[string]any + require.NoError(t, json.Unmarshal(snap, &raw)) + raw["version"] = 999999 + corrupted, err := json.Marshal(raw) + require.NoError(t, err) + + // Restore into a backend that already has unrelated state, to confirm the + // version mismatch path resets it rather than merging/erroring. + target := cloudwatch.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + require.NoError(t, target.PutMetricAlarm(&cloudwatch.MetricAlarm{AlarmName: "pre-existing-alarm"})) + + require.NoError(t, target.Restore(t.Context(), corrupted)) + + alarms, _, err := target.DescribeAlarms(nil, nil, "", "", "", 0) + require.NoError(t, err) + assert.Empty(t, alarms.Data, "version-mismatched snapshot should reset the backend to empty, not merge or error") +} diff --git a/services/cloudwatch/rpcv2cbor.go b/services/cloudwatch/rpcv2cbor.go index 5c85303d7..bd1e75679 100644 --- a/services/cloudwatch/rpcv2cbor.go +++ b/services/cloudwatch/rpcv2cbor.go @@ -433,8 +433,10 @@ func cborDimensions(m cbor.Map) []Dimension { return dims } -// cborDecodeDatum parses a single MetricDatum from a CBOR map. -// StatisticValues takes precedence over Value when present. +// cborDecodeDatum parses a single MetricDatum from a CBOR map. The Values/Counts +// array takes precedence over StatisticValues, which takes precedence over +// Value, mirroring the order AWS documents them (the Has* flags let +// validateMetricDatum reject any invalid combination of these instead). func cborDecodeDatum(m cbor.Map) MetricDatum { ts := cborTime(m, "Timestamp") dims := cborDimensions(m) @@ -442,34 +444,52 @@ func cborDecodeDatum(m cbor.Map) MetricDatum { name := cborStr(m, keyMetricName) unit := cborStr(m, "Unit") - if ssMap, ok := cborStatisticValues(m); ok { - return MetricDatum{ - MetricName: name, - Unit: unit, - Timestamp: ts, - Count: cborFloat(ssMap, "SampleCount"), - Sum: cborFloat(ssMap, statSum), - Min: cborFloat(ssMap, "Minimum"), - Max: cborFloat(ssMap, "Maximum"), - Dimensions: dims, - StorageResolution: storageRes, - } - } - - val := cborFloat(m, keyValue) - - return MetricDatum{ + datum := MetricDatum{ MetricName: name, - Value: val, Unit: unit, Timestamp: ts, - Count: 1, - Sum: val, - Min: val, - Max: val, Dimensions: dims, StorageResolution: storageRes, } + + if values, hasValues := cborFloatList(m, "Values"); hasValues { + counts, hasCounts := cborFloatList(m, "Counts") + if !hasCounts { + counts = make([]float64, len(values)) + for i := range counts { + counts[i] = 1 + } + } + + // The backend (storeDatum) derives Sum/SampleCount/Min/Max from + // Values/Counts at store time, so this only carries the raw arrays. + datum.HasValuesArray = true + datum.Values = values + datum.Counts = counts + + return datum + } + + if ssMap, ok := cborStatisticValues(m); ok { + datum.HasStatisticSet = true + datum.Count = cborFloat(ssMap, "SampleCount") + datum.Sum = cborFloat(ssMap, statSum) + datum.Min = cborFloat(ssMap, "Minimum") + datum.Max = cborFloat(ssMap, "Maximum") + + return datum + } + + _, hasValue := m[keyValue] + val := cborFloat(m, keyValue) + datum.HasValue = hasValue + datum.Value = val + datum.Count = 1 + datum.Sum = val + datum.Min = val + datum.Max = val + + return datum } // cborStatisticValues returns the StatisticValues sub-map when present. @@ -483,6 +503,27 @@ func cborStatisticValues(m cbor.Map) (cbor.Map, bool) { return ssMap, isSS } +// cborFloatList extracts a []float64 from a CBOR map by key. ok is false when +// the key is absent so callers can distinguish "no list" from an empty one. +func cborFloatList(m cbor.Map, key string) ([]float64, bool) { + v, present := m[key] + if !present { + return nil, false + } + + list, isList := v.(cbor.List) + if !isList { + return nil, false + } + + vals := make([]float64, 0, len(list)) + for _, item := range list { + vals = append(vals, cborValFloat(item)) + } + + return vals, true +} + func (h *Handler) cborPutMetricData(input cbor.Map, c *echo.Context) error { namespace := cborStr(input, keyNamespace) if namespace == "" { @@ -507,31 +548,13 @@ func (h *Handler) cborPutMetricData(input cbor.Map, c *echo.Context) error { } } - unprocessed, err := h.Backend.PutMetricData(namespace, data) - if err != nil { - return h.cborError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) + if err := h.Backend.PutMetricData(namespace, data); err != nil { + return h.cborError(c, putMetricDataErrorStatus(err), putMetricDataErrorCode(err), err.Error()) } - return writeCBOR(c, buildUnprocessedCBORResponse(unprocessed)) -} - -// buildUnprocessedCBORResponse constructs the CBOR response map for PutMetricData. -func buildUnprocessedCBORResponse(unprocessed []UnprocessedMetricDatum) cbor.Map { - resp := cbor.Map{} - if len(unprocessed) == 0 { - return resp - } - uList := make(cbor.List, 0, len(unprocessed)) - for _, u := range unprocessed { - uList = append(uList, cbor.Map{ - "MetricName": cbor.String(u.MetricName), - "ErrorCode": cbor.String(u.ErrorCode), - "ErrorMessage": cbor.String(u.ErrorMessage), - }) - } - resp["UnprocessedMetricData"] = uList - - return resp + // PutMetricDataOutput has no members besides the request ID: CloudWatch has + // no partial-success concept for this operation. + return writeCBOR(c, cbor.Map{}) } func (h *Handler) cborGetMetricStatistics(input cbor.Map, c *echo.Context) error { @@ -747,12 +770,17 @@ func (h *Handler) cborListMetrics(input cbor.Map, c *echo.Context) error { namespace := cborStr(input, keyNamespace) metricName := cborStr(input, keyMetricName) nextToken := cborStr(input, "NextToken") + recentlyActive := cborStr(input, "RecentlyActive") maxResults := int(cborInt32(input, "MaxResults")) dimensions := cborDimensions(input) - p, err := h.Backend.ListMetrics(namespace, metricName, dimensions, nextToken, maxResults) + p, err := h.Backend.ListMetrics(namespace, metricName, dimensions, recentlyActive, nextToken, maxResults) if err != nil { - return h.cborError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) + if errors.Is(err, ErrValidation) { + return h.cborError(c, http.StatusBadRequest, "InvalidParameterValue", err.Error()) + } + + return h.cborError(c, http.StatusInternalServerError, errCodeInternalFailure, err.Error()) } mList := make(cbor.List, 0, len(p.Data)) @@ -2164,7 +2192,7 @@ func (h *Handler) cborPutManagedInsightRules(input cbor.Map, c *echo.Context) er }); err != nil { failures = append(failures, InsightRuleFailure{ RuleName: ruleName, - FailureCode: "InternalFailure", + FailureCode: errCodeInternalFailure, FailureDescription: err.Error(), }) } diff --git a/services/cloudwatch/rpcv2cbor_test.go b/services/cloudwatch/rpcv2cbor_test.go index 6fbcac37e..578fe157c 100644 --- a/services/cloudwatch/rpcv2cbor_test.go +++ b/services/cloudwatch/rpcv2cbor_test.go @@ -993,7 +993,7 @@ func TestCBOR_GetMetricStatistics_WithDimensions(t *testing.T) { h := cloudwatch.NewHandler(b) dims := []cloudwatch.Dimension{{Name: "Host", Value: "h1"}} - _, _ = b.PutMetricData("App", []cloudwatch.MetricDatum{ + _ = b.PutMetricData("App", []cloudwatch.MetricDatum{ { MetricName: "Load", Value: 80, Count: 1, Sum: 80, Min: 80, Max: 80, Timestamp: time.Now().UTC(), Dimensions: dims, @@ -1030,7 +1030,7 @@ func TestCBOR_GetMetricData_WithDimensions(t *testing.T) { dimsA := []cloudwatch.Dimension{{Name: "Shard", Value: "a"}} ts := time.Now().UTC().Add(-30 * time.Second) - _, _ = b.PutMetricData("NS", []cloudwatch.MetricDatum{ + _ = b.PutMetricData("NS", []cloudwatch.MetricDatum{ {MetricName: "Ops", Value: 5, Count: 1, Sum: 5, Min: 5, Max: 5, Timestamp: ts, Dimensions: dimsA}, {MetricName: "Ops", Value: 50, Count: 1, Sum: 50, Min: 50, Max: 50, Timestamp: ts}, }) @@ -1109,7 +1109,7 @@ func TestCBOR_ListMetrics_WithDimensions(t *testing.T) { ts := time.Now().UTC() for _, env := range []string{"prod", "staging"} { - _, _ = b.PutMetricData("App", []cloudwatch.MetricDatum{ + _ = b.PutMetricData("App", []cloudwatch.MetricDatum{ { MetricName: "RPM", Value: 1, Count: 1, Sum: 1, Min: 1, Max: 1, Timestamp: ts, diff --git a/services/cloudwatch/store_setup.go b/services/cloudwatch/store_setup.go new file mode 100644 index 000000000..a368991c8 --- /dev/null +++ b/services/cloudwatch/store_setup.go @@ -0,0 +1,88 @@ +package cloudwatch + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend with a pure, value-derived +// key is registered exactly once, here, as a *store.Table[T] on b.registry. +// See pkgs/store's package doc and the services/ec2 conversion (commit +// 12e611a4) for the pattern this follows. +// +// Two resource fields are deliberately NOT registered here and remain plain +// maps: +// - metrics (map[string]map[string]*metricRecord): PutMetricData is the +// hottest op in this backend (up to cwMaxMetricDataPerRequest=1000 data +// points per request) and every access site does a direct two-level +// namespace->composite-key lookup; store.Table has no native concept of a +// nested/composite-keyed collection, and flattening it into a single +// Table keyed by "namespace|compositeKey" would require adding a +// Namespace field to metricRecord purely to reconstruct identity for a +// secondary Index, adding an extra key-concat + index-maintenance layer to +// the single hottest write path for no behavioural benefit. The +// b.totalMetrics running counter (#60) that avoids an O(namespaces) walk +// would also need reworking around Index bookkeeping. Left inline, +// matching the sqs pilot's decision to leave the (also hot, high-volume) +// per-queue messages slice inline. +// - alarmHistory (map[string][]AlarmHistoryItem): slice-valued, not +// map[string]*T -- store.Table stores one *V per key, not a growable +// per-key slice, so it does not fit this shape. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +func alarmsKeyFn(v *MetricAlarm) string { return v.AlarmName } +func compositeAlarmsKeyFn(v *CompositeAlarm) string { return v.AlarmName } +func dashboardsKeyFn(v *dashboardRecord) string { return v.Name } + +func anomalyDetectorsKeyFn(v *AnomalyDetector) string { + return anomalyDetectorKey(v.Namespace, v.MetricName, v.Stat, v.Dimensions) +} + +func insightRulesKeyFn(v *InsightRule) string { return v.Name } +func metricStreamsKeyFn(v *MetricStream) string { return v.Name } +func alarmMuteRulesKeyFn(v *AlarmMuteRule) string { return v.MuteName } + +func metricFiltersKeyFn(v *MetricFilter) string { + return metricFilterKey(v.FilterName, v.LogGroupName) +} + +// registerAllTables registers every converted resource map on b.registry +// exactly once. It must be called during construction only (immediately after +// b.registry is created), never on every Reset() -- store.Register panics on a +// duplicate name, so runtime resets go through registry.ResetAll() instead +// (see InMemoryBackend.Reset in backend.go). +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call to the concrete field and value type. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.alarms = store.Register(b.registry, "alarms", store.New(alarmsKeyFn)) + }, + func(b *InMemoryBackend) { + b.compositeAlarms = store.Register(b.registry, "compositeAlarms", store.New(compositeAlarmsKeyFn)) + }, + func(b *InMemoryBackend) { + b.dashboards = store.Register(b.registry, "dashboards", store.New(dashboardsKeyFn)) + }, + func(b *InMemoryBackend) { + b.anomalyDetectors = store.Register(b.registry, "anomalyDetectors", store.New(anomalyDetectorsKeyFn)) + }, + func(b *InMemoryBackend) { + b.insightRules = store.Register(b.registry, "insightRules", store.New(insightRulesKeyFn)) + }, + func(b *InMemoryBackend) { + b.metricStreams = store.Register(b.registry, "metricStreams", store.New(metricStreamsKeyFn)) + }, + func(b *InMemoryBackend) { + b.alarmMuteRules = store.Register(b.registry, "alarmMuteRules", store.New(alarmMuteRulesKeyFn)) + }, + func(b *InMemoryBackend) { + b.metricFilters = store.Register(b.registry, "metricFilters", store.New(metricFiltersKeyFn)) + }, +} diff --git a/services/cloudwatch/widget_test.go b/services/cloudwatch/widget_test.go index e35548610..ceca83e82 100644 --- a/services/cloudwatch/widget_test.go +++ b/services/cloudwatch/widget_test.go @@ -121,7 +121,7 @@ func TestRenderMetricWidgetPNGPlotsData(t *testing.T) { Max: float64((i%3)*10 + 5), }) } - if _, err := b.PutMetricData("MyApp", data); err != nil { + if err := b.PutMetricData("MyApp", data); err != nil { t.Fatalf("PutMetricData: %v", err) } diff --git a/services/cloudwatchlogs/PARITY.md b/services/cloudwatchlogs/PARITY.md new file mode 100644 index 000000000..d96198537 --- /dev/null +++ b/services/cloudwatchlogs/PARITY.md @@ -0,0 +1,107 @@ +--- +service: cloudwatchlogs +sdk_module: aws-sdk-go-v2/service/cloudwatchlogs@v1.64.0 +last_audit_commit: 17a215e4 +last_audit_date: 2026-07-05 +overall: A +ops: + PutLogEvents: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: sequenceToken was validated and rejected with InvalidSequenceTokenException; real AWS (v1.64.0 doc) ignores sequenceToken entirely and never returns that exception. Also fixed RejectedLogEventsInfo wire shape (tooOldLogEventStartIndex -> tooOldLogEventEndIndex, exclusive-end semantics) and an off-by-one on ExpiredLogEventEndIndex."} + GetLogEvents: {wire: ok, errors: ok, state: ok, persist: ok, note: "startFromHead/nextToken precedence and stable-at-boundary forward/backward tokens verified correct."} + FilterLogEvents: {wire: ok, errors: ok, state: ok, persist: ok, note: "cross-stream interleave + stable timestamp sort verified; logStreamNames/logStreamNamePrefix mutual-exclusion validated; searchedLogStreams correctly always empty (AWS deprecated this field)."} + CreateLogGroup: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteLogGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "cascades streams/events/subscription filters/metric filters."} + DescribeLogGroups: {wire: ok, errors: ok, state: ok, persist: ok} + CreateLogStream: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteLogStream: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeLogStreams: {wire: ok, errors: ok, state: ok, persist: ok, note: "orderBy=LastEventTime + prefix and descending + orderBy=LogStreamName rejection rules match AWS."} + PutRetentionPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteRetentionPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + PutSubscriptionFilter: {wire: ok, errors: ok, state: ok, persist: ok, note: "2-filter-per-group cap and update-in-place-by-name verified."} + DescribeSubscriptionFilters: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteSubscriptionFilter: {wire: ok, errors: ok, state: ok, persist: ok} + PutMetricFilter: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeMetricFilters: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteMetricFilter: {wire: ok, errors: ok, state: ok, persist: ok} + TestMetricFilter: {wire: ok, errors: ok, state: ok, persist: n/a, note: "fixed: ExtractedValues was always {} (disguised stub -- computed nothing from the pattern's named fields). Now extracts every $-referenced field for JSON and space-delimited patterns."} + ListTagsLogGroup: {wire: ok, errors: ok, state: ok, persist: ok} + ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok} + TagLogGroup: {wire: ok, errors: ok, state: ok, persist: ok} + UntagLogGroup: {wire: ok, errors: ok, state: ok, persist: ok} + TagResource: {wire: ok, errors: ok, state: ok, persist: ok} + UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} +families: + metric-filter-emission: {status: ok, note: "fixed (internal PutLogEvents dispatch, not an SDK op): emitMetricFilterMatches previously emitted matchCount copies of one static value regardless of MetricValue's per-event field reference (a disguised stub -- '$field' values were never actually read from the matched log event, just defaulted to 1.0/DefaultValue). Now extracts the referenced field ($name for space-delimited patterns, $.path for JSON patterns) per matched event via new compiledFilterPattern.extract; a matched-but-non-numeric-or-absent field now correctly emits no data point rather than fabricating one. Also fixed emitted Unit being hardcoded to \"\" instead of the configured MetricTransformation.Unit."} + janitor-retention-sweep: {status: ok, note: "two-phase read-then-write lock, worker.NewGroup ticker is ctx-cancel safe, telemetry recorded. No leak."} + subscription-filter-delivery: {status: ok, note: "goroutines bounded by workerSem + backend WaitGroup + service ctx; Close()/Drain() wait for in-flight deliveries. No leak found."} + insights (StartQuery/GetQueryResults/StopQuery/DescribeQueries/query language): {status: ok, note: "lightly reviewed only (large ~2500 LOC subsystem across insights_*.go); query TTL eviction (evictByTTL) and cap enforcement (enforceCap) present and bounded; not exhaustively re-audited op-by-op this pass -- see deferred."} + export/import tasks, deliveries, anomaly detectors, scheduled queries, account policies, data protection/resource/index policies, transformers, integrations: {status: ok, note: "spot-checked (CreateExportTask/runExport does a real synchronous S3 write via injectable ExportSink when configured; ApplyTransformer/applyJSONProcessor implements real addKeys/deleteKeys/renameKeys-style JSON processors, not a stub). Not exhaustively re-audited op-by-op this pass -- see deferred."} + StartLiveTail: {status: ok, note: "explicitly validation-only (log-group-identifier existence check) with a documented comment explaining the streaming HTTP/2 transport can't be served by this request/response handler -- an honest declared limitation, not a silent stub."} +gaps: + - MetricTransformation.Dimensions is accepted, validated on the wire, and persisted on the MetricFilter, but is never forwarded to the emitted CloudWatch metric: the MetricEmitter interface (backend.go) only carries namespace/name/value/unit, and its real implementation is wired in cli.go's wireCWLogsMetricEmitter, which is out of scope for this pass (SHARED FILE). Extending the interface + cli.go wiring to carry dimensions is a real fix but requires touching cli.go. (bd: gopherstack-b14) + - GetLogGroupFields always returns the 4 static built-in fields (@message/@timestamp/@ingestionTime/@logStream) and never samples real ingested log content, so it cannot discover custom JSON/space-pattern fields the way real AWS does (percent-of-sampled-events containing each discovered key). Not fixed this pass (a genuine sampling+percentage feature, lower priority than the PutLogEvents/metric-filter fixes actually made). (bd: gopherstack-b14) + - PutLogEvents does not enforce two documented batch-shape constraints: (1) events in a single request must be in strict chronological order or the whole call should fail; (2) the valid-event timestamp span within one batch cannot exceed 24 hours. Both are documented in the current SDK's PutLogEvents doc comment. Not implemented this pass: doing so safely requires auditing whether existing out-of-order-friendly tests/call sites (this codebase's own appendEvents doc explicitly says "log events may arrive with out-of-order timestamps" ) depend on the current relaxed behavior; higher risk/effort than the fixes made, so deferred rather than rushed. (bd: gopherstack-b14) +deferred: + - Insights query language/stages/parser correctness (insights_expr.go, insights_parse.go, insights_parser.go, insights_stages.go, insights_stats.go) -- not re-verified op-by-op against CloudWatch Logs Insights query syntax this pass. + - Export/Import task lifecycle edge cases, Deliveries, Log Anomaly Detectors, Scheduled Queries, Account Policies, Data Protection/Resource/Index Policies, Transformers, Integrations (handler_completeness.go / backend_completeness.go, ~2100 LOC) -- spot-checked only, not exhaustively audited op-by-op. + - StartLiveTail streaming transport (intentionally out of scope; validation-only by design). +leaks: {status: clean, note: "Only one goroutine spawn site (scheduleFilterDelivery for subscription filter delivery), bounded by a semaphore + backend WaitGroup + ctx cancellation; Close()/Drain() join in-flight work. Janitor ticker is ctx-cancel safe via pkgs/worker. No unbounded per-request goroutines found in the areas audited this pass."} +--- + +## Notes + +- **PutLogEvents sequenceToken is a no-op today.** aws-sdk-go-v2 v1.64.0's own doc comments + (api_op_PutLogEvents.go, types.InputLogEvent.UploadSequenceToken, types.InvalidSequenceTokenException) + are explicit and repeated: "The sequence token is now ignored in PutLogEvents actions. + PutLogEvents actions are always accepted and never return InvalidSequenceTokenException or + DataAlreadyAcceptedException even if the sequence token is not valid." A previous audit pass + (see the now-updated `ops_batch2_audit_test.go` comment) had deliberately added strict + sequence-token validation, modeling the *pre-2022* contract. This sweep removed that + validation; NextSequenceToken is still computed and returned (many older SDKs/tools still read + it), it just no longer gates acceptance. If a future auditor sees "AWS returns + InvalidSequenceTokenException for a stale token" in an older blog post or Stack Overflow + answer, that's describing pre-Feb-2022 behavior -- trust the SDK doc comment over that. + +- **RejectedLogEventsInfo field is `TooOldLogEventEndIndex`, not `...StartIndex`.** The real + wire key is `tooOldLogEventEndIndex` (aws-sdk-go-v2 types.RejectedLogEventsInfo), and per the + field doc it's an *exclusive end* index ("too-old events form a prefix of the batch; this is + one past the last of them"), exactly mirroring `ExpiredLogEventEndIndex`'s semantics. A real + SDK client unmarshalling our old `tooOldLogEventStartIndex` key would have silently gotten + `nil` for `TooOldLogEventEndIndex`. Both `TooOldLogEventEndIndex` and `ExpiredLogEventEndIndex` + are computed as `(highest matching event index) + 1`, not the first-matching index -- watch for + this if the field is ever renamed again. + +- **Metric filter `MetricValue` field-reference extraction is real, not literal-only.** + `MetricTransformation.MetricValue` can be a literal number (published as-is) or a + `$`-prefixed field reference: `$name` for a *named* field in a space-delimited pattern + (`[ip, level, size]` makes `$ip`/`$level`/`$size` addressable) or `$.path` for a JSON + selector pattern (`{ $.level = "ERROR" }` makes any `$.foo.bar` addressable, not just paths + that appear in the match condition). This is implemented via + `compiledFilterPattern.extract`/`extractString`/`extractValue`, fed by + `compileJSONFilterPatternExtract` (filter_pattern_json.go) and + `compileSpaceFilterPatternExtract` + `spaceFieldIndex` (filter_pattern_space.go, which now + also tracks each `spaceTerm.name` -- previously discarded entirely, even for bare/unconditioned + terms). **Trap:** `MetricTransformation.DefaultValue` is documented as "the value to emit when + a filter pattern does NOT match a log event" (i.e. a substitute for a quiet period with zero + matches), **not** a fallback for a matched event whose referenced field happens to be missing + or non-numeric. The previous implementation conflated the two and used DefaultValue (or a bare + `1.0`) as an extraction-failure fallback, fabricating metric data points that real CloudWatch + Logs would never emit. The fix intentionally emits *nothing* for an extraction failure on a + matched event; genuinely wiring "publish DefaultValue once per period when this filter had zero + matches" would require a periodic scheduler (not per-PutLogEvents-call logic) and is left as a + gap, not attempted this pass. + +- **`putLogEventsMaxMessageBytes = 256 * 1024` looks suspicious next to aws-sdk-go-v2's doc + comment ("Each log event can be no larger than 1 MB") but was deliberately left unchanged.** + The older, still-maintained aws-sdk-go v1 model (`service/cloudwatchlogs/api.go`) explicitly + says "Each log event can be no larger than 256 KB" in the same field's doc comment, matching + this codebase's long-standing constant and the widely-documented CloudWatch Logs event-size + quota. Treat the v2 SDK's "1 MB" doc text as smithy-model doc drift (likely bled in from the + *batch* size limit description) rather than a real per-event limit increase, unless a more + authoritative source (e.g. the AWS Service Quotas console page) is checked and says otherwise. + +- **`compileFilterPattern`'s per-pattern-kind dispatch (`{`/`[`/plain-text) is the single + source of truth for "does this pattern have addressable fields."** Anything touching + metric-value extraction or `TestMetricFilter.ExtractedValues` should go through + `compiledFilterPattern.extractString`/`extractValue` or `patternFieldRefs`, not re-parse the + pattern text directly -- the existing JSON lexer/space-term parsing is reused rather than + duplicated so the match and extract paths can't drift out of sync. diff --git a/services/cloudwatchlogs/backend.go b/services/cloudwatchlogs/backend.go index 381c27f08..7a27d40b6 100644 --- a/services/cloudwatchlogs/backend.go +++ b/services/cloudwatchlogs/backend.go @@ -20,10 +20,10 @@ import ( "github.com/google/uuid" "github.com/blackbirdworks/gopherstack/pkgs/arn" - "github.com/blackbirdworks/gopherstack/pkgs/collections" "github.com/blackbirdworks/gopherstack/pkgs/config" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) const ( @@ -425,41 +425,71 @@ type storedQuery struct { stats QueryStatistics } -// InMemoryBackend implements StorageBackend using in-memory maps. +// InMemoryBackend implements StorageBackend using pkgs/store tables in place of +// the hand-rolled maps this backend used before Phase 3.3 (see store_setup.go). type InMemoryBackend struct { - deliverer SubscriptionDeliverer - metricEmitter MetricEmitter - ctx context.Context - accountPolicies map[string]*AccountPolicy - groups map[string]map[string]*LogGroup - workerSem chan struct{} - streams map[string]map[string]map[string]*LogStream - events map[string]map[string]map[string][]*OutputLogEvent - subscriptionFilters map[string]map[string][]*SubscriptionFilter - queries map[string]*storedQuery - parsedQueries map[string]*insightsQuery - compiledPatterns map[string]*compiledFilterPattern - exportTasks map[string]*ExportTask - importTasks map[string]*ImportTask - deliveries map[string]*Delivery - logAnomalyDetectors map[string]*LogAnomalyDetector - anomalies map[string]map[string]*Anomaly - scheduledQueries map[string]*ScheduledQuery - scheduledQueryRuns map[string][]*ScheduledQueryRunSummary - s3TableIntegrations map[string]string + deliverer SubscriptionDeliverer + metricEmitter MetricEmitter + ctx context.Context + workerSem chan struct{} + + // registry holds every persisted resource table; Snapshot/Restore drive it + // via registry.SnapshotAll()/RestoreAll() (see persistence.go). + registry *store.Registry + // ephemeralRegistry holds resource tables that are never persisted (query + // results/anomalies/scheduled-query run history -- matching this backend's + // pre-Phase-3.3 behavior of leaving them out of backendSnapshot) but still + // benefit from one-call Reset semantics. + ephemeralRegistry *store.Registry + + accountPolicies *store.Table[AccountPolicy] + + // groups, streams, subscriptionFilters, and metricFilters are region-qualified + // ("dirty") tables: their value types carry unexported identity fields (see + // models.go) so they are registered on neither registry above -- Snapshot/ + // Restore drive them through DTOs and Reset clears them directly. + groups *store.Table[LogGroup] + groupsByRegion *store.Index[LogGroup] + + streams *store.Table[LogStream] + streamsByGroup *store.Index[LogStream] + + subscriptionFilters *store.Table[SubscriptionFilter] + subscriptionFiltersByGroup *store.Index[SubscriptionFilter] + + metricFilters *store.Table[MetricFilter] + metricFiltersByGroup *store.Index[MetricFilter] + + // queries and anomalies are ephemeral (registered on ephemeralRegistry, not + // registry -- see above). + queries *store.Table[storedQuery] + anomalies *store.Table[Anomaly] + anomalyByDetector *store.Index[Anomaly] + + // parsedQueries and compiledPatterns remain plain maps -- see the doc + // comment above registerAllTables in store_setup.go for why. + parsedQueries map[string]*insightsQuery + compiledPatterns map[string]*compiledFilterPattern + + exportTasks *store.Table[ExportTask] + importTasks *store.Table[ImportTask] + deliveries *store.Table[Delivery] + logAnomalyDetectors *store.Table[LogAnomalyDetector] + scheduledQueries *store.Table[ScheduledQuery] + scheduledQueryRuns *store.Table[scheduledQueryRunHistory] + s3TableIntegrations *store.Table[s3TableIntegrationEntry] mu *lockmetrics.RWMutex - kmsKeys map[string]string - metricFilters map[string]map[string]map[string]*MetricFilter - queryDefinitions map[string]*QueryDefinition - dataProtectionPolicies map[string]string // logGroupName -> policyDocument JSON - resourcePolicies map[string]ResourcePolicy - deliveryDestinations map[string]DeliveryDestination - deliverySources map[string]DeliverySource - destinations map[string]CWLDestination - indexPolicies map[string]IndexPolicy - transformers map[string]Transformer - integrations map[string]CWLIntegration - deletionProtected map[string]bool + kmsKeys *store.Table[kmsKeyEntry] + queryDefinitions *store.Table[QueryDefinition] + dataProtectionPolicies *store.Table[dataProtectionPolicyEntry] + resourcePolicies *store.Table[ResourcePolicy] + deliveryDestinations *store.Table[DeliveryDestination] + deliverySources *store.Table[DeliverySource] + destinations *store.Table[CWLDestination] + indexPolicies *store.Table[IndexPolicy] + transformers *store.Table[Transformer] + integrations *store.Table[CWLIntegration] + deletionProtected *store.Table[deletionProtectionEntry] exportSink ExportSink cancel context.CancelFunc region string @@ -503,50 +533,32 @@ func NewInMemoryBackendWithContext( ctx, cancel := context.WithCancel(svcCtx) - return &InMemoryBackend{ - accountID: accountID, - region: region, - groups: make(map[string]map[string]*LogGroup), - streams: make(map[string]map[string]map[string]*LogStream), - events: make(map[string]map[string]map[string][]*OutputLogEvent), - subscriptionFilters: make(map[string]map[string][]*SubscriptionFilter), - queries: make(map[string]*storedQuery), - parsedQueries: make(map[string]*insightsQuery), - compiledPatterns: make(map[string]*compiledFilterPattern), - exportTasks: make(map[string]*ExportTask), - importTasks: make(map[string]*ImportTask), - deliveries: make(map[string]*Delivery), - logAnomalyDetectors: make(map[string]*LogAnomalyDetector), - anomalies: make(map[string]map[string]*Anomaly), - scheduledQueries: make(map[string]*ScheduledQuery), - scheduledQueryRuns: make(map[string][]*ScheduledQueryRunSummary), - accountPolicies: make(map[string]*AccountPolicy), - kmsKeys: make(map[string]string), - s3TableIntegrations: make(map[string]string), - metricFilters: make(map[string]map[string]map[string]*MetricFilter), - queryDefinitions: make(map[string]*QueryDefinition), - dataProtectionPolicies: make(map[string]string), - resourcePolicies: make(map[string]ResourcePolicy), - deliveryDestinations: make(map[string]DeliveryDestination), - deliverySources: make(map[string]DeliverySource), - destinations: make(map[string]CWLDestination), - indexPolicies: make(map[string]IndexPolicy), - transformers: make(map[string]Transformer), - integrations: make(map[string]CWLIntegration), - deletionProtected: make(map[string]bool), - mu: lockmetrics.New("cloudwatchlogs"), - queryTTL: defaultQueryTTL, - maxQueries: defaultMaxQueries, - maxParsedQueries: defaultParsedQueryCacheSize, - ctx: ctx, - cancel: cancel, - workerSem: make(chan struct{}, defaultDeliveryWorkers), - deliveryTimeout: defaultDeliveryTimeout, + b := &InMemoryBackend{ + accountID: accountID, + region: region, + registry: store.NewRegistry(), + ephemeralRegistry: store.NewRegistry(), + parsedQueries: make(map[string]*insightsQuery), + compiledPatterns: make(map[string]*compiledFilterPattern), + mu: lockmetrics.New("cloudwatchlogs"), + queryTTL: defaultQueryTTL, + maxQueries: defaultMaxQueries, + maxParsedQueries: defaultParsedQueryCacheSize, + ctx: ctx, + cancel: cancel, + workerSem: make(chan struct{}, defaultDeliveryWorkers), + deliveryTimeout: defaultDeliveryTimeout, settings: Settings{ MaxRetentionDays: defaultMaxRetentionDays, JanitorInterval: time.Minute, }, } + + registerAllTables(b) + registerEphemeralTables(b) + registerRegionTables(b) + + return b } // SetSettings updates the backend settings. @@ -624,56 +636,6 @@ func (b *InMemoryBackend) streamARN(region, groupName, streamName string) string return arn.Build("logs", region, b.accountID, "log-group:"+groupName+":log-stream:"+streamName) } -// groupsStore returns the log-group map for the given region, lazily creating it. -// Callers must hold b.mu. -func (b *InMemoryBackend) groupsStore(region string) map[string]*LogGroup { - if b.groups[region] == nil { - b.groups[region] = make(map[string]*LogGroup) - } - - return b.groups[region] -} - -// streamsStore returns the streams map (group -> stream -> *LogStream) for the given -// region, lazily creating it. Callers must hold b.mu. -func (b *InMemoryBackend) streamsStore(region string) map[string]map[string]*LogStream { - if b.streams[region] == nil { - b.streams[region] = make(map[string]map[string]*LogStream) - } - - return b.streams[region] -} - -// eventsStore returns the events map (group -> stream -> []*OutputLogEvent) for the -// given region, lazily creating it. Callers must hold b.mu. -func (b *InMemoryBackend) eventsStore(region string) map[string]map[string][]*OutputLogEvent { - if b.events[region] == nil { - b.events[region] = make(map[string]map[string][]*OutputLogEvent) - } - - return b.events[region] -} - -// subscriptionFiltersStore returns the subscription-filter map (group -> filters) for -// the given region, lazily creating it. Callers must hold b.mu. -func (b *InMemoryBackend) subscriptionFiltersStore(region string) map[string][]*SubscriptionFilter { - if b.subscriptionFilters[region] == nil { - b.subscriptionFilters[region] = make(map[string][]*SubscriptionFilter) - } - - return b.subscriptionFilters[region] -} - -// metricFiltersStore returns the metric-filter map (group -> name -> *MetricFilter) for -// the given region, lazily creating it. Callers must hold b.mu. -func (b *InMemoryBackend) metricFiltersStore(region string) map[string]map[string]*MetricFilter { - if b.metricFilters[region] == nil { - b.metricFilters[region] = make(map[string]map[string]*MetricFilter) - } - - return b.metricFilters[region] -} - // CreateLogGroup creates a new log group with the given class and optional KMS key. // logGroupClass must be STANDARD or INFREQUENT_ACCESS (defaults to STANDARD if empty). func (b *InMemoryBackend) CreateLogGroup( @@ -708,8 +670,7 @@ func (b *InMemoryBackend) CreateLogGroup( b.mu.Lock("CreateLogGroup") defer b.mu.Unlock() - groups := b.groupsStore(region) - if _, exists := groups[name]; exists { + if b.groupHas(region, name) { return nil, fmt.Errorf("%w: Log group %s already exists", ErrLogGroupAlreadyExists, name) } @@ -719,13 +680,12 @@ func (b *InMemoryBackend) CreateLogGroup( Arn: b.groupARN(region, name), LogGroupClass: logGroupClass, KmsKeyID: kmsKeyID, + region: region, } - groups[name] = g - b.streamsStore(region)[name] = make(map[string]*LogStream) - b.eventsStore(region)[name] = make(map[string][]*OutputLogEvent) + b.groupPut(g) if kmsKeyID != "" { - b.kmsKeys[name] = kmsKeyID + b.kmsKeys.Put(&kmsKeyEntry{Key: name, KmsKeyID: kmsKeyID}) } cp := *g @@ -740,16 +700,14 @@ func (b *InMemoryBackend) DeleteLogGroup(ctx context.Context, name string) error b.mu.Lock("DeleteLogGroup") defer b.mu.Unlock() - groups := b.groupsStore(region) - if _, exists := groups[name]; !exists { + if !b.groupHas(region, name) { return fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, name) } - delete(groups, name) - delete(b.streamsStore(region), name) - delete(b.eventsStore(region), name) - delete(b.subscriptionFiltersStore(region), name) - delete(b.metricFiltersStore(region), name) + b.groupDelete(region, name) + b.deleteStreamsInGroup(region, name) + b.deleteSubscriptionFiltersInGroup(region, name) + b.deleteMetricFiltersInGroup(region, name) return nil } @@ -776,7 +734,7 @@ func (b *InMemoryBackend) SetRetentionPolicy( b.mu.Lock("SetRetentionPolicy") defer b.mu.Unlock() - g, exists := b.groupsStore(region)[groupName] + g, exists := b.groupGet(region, groupName) if !exists { return fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, groupName) } @@ -799,7 +757,7 @@ func (b *InMemoryBackend) DescribeLogGroups( limit = defaultDescribeLimit } - regionGroups := b.groupsStore(region) + regionGroups := b.groupsInRegion(region) all := make([]LogGroup, 0, len(regionGroups)) for _, g := range regionGroups { if prefix == "" || strings.HasPrefix(g.LogGroupName, prefix) { @@ -832,12 +790,11 @@ func (b *InMemoryBackend) CreateLogStream( b.mu.Lock("CreateLogStream") defer b.mu.Unlock() - if _, exists := b.groupsStore(region)[groupName]; !exists { + if !b.groupHas(region, groupName) { return nil, fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, groupName) } - streams := b.streamsStore(region) - if _, exists := streams[groupName][streamName]; exists { + if b.streamHas(region, groupName, streamName) { return nil, fmt.Errorf( "%w: Log stream %s already exists", ErrLogStreamAlreadyExist, @@ -849,9 +806,10 @@ func (b *InMemoryBackend) CreateLogStream( CreationTime: time.Now().UnixMilli(), LogStreamName: streamName, Arn: b.streamARN(region, groupName, streamName), + region: region, + logGroupName: groupName, } - streams[groupName][streamName] = s - b.eventsStore(region)[groupName][streamName] = nil + b.streamPut(s) return s, nil } @@ -863,23 +821,21 @@ func (b *InMemoryBackend) DeleteLogStream(ctx context.Context, groupName, stream b.mu.Lock("DeleteLogStream") defer b.mu.Unlock() - groups := b.groupsStore(region) - if _, exists := groups[groupName]; !exists { + group, groupExists := b.groupGet(region, groupName) + if !groupExists { return fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, groupName) } - streams := b.streamsStore(region) - if _, exists := streams[groupName][streamName]; !exists { + stream, exists := b.streamGet(region, groupName, streamName) + if !exists { return fmt.Errorf("%w: Log stream %s not found", ErrLogStreamNotFound, streamName) } - stream := streams[groupName][streamName] - if stream != nil && groups[groupName] != nil { - groups[groupName].StoredBytes -= stream.StoredBytes + if stream != nil && group != nil { + group.StoredBytes -= stream.StoredBytes } - delete(streams[groupName], streamName) - delete(b.eventsStore(region)[groupName], streamName) + b.streamDelete(region, groupName, streamName) return nil } @@ -953,7 +909,7 @@ func (b *InMemoryBackend) DescribeLogStreams( b.mu.RLock("DescribeLogStreams") defer b.mu.RUnlock() - if _, exists := b.groupsStore(region)[groupName]; !exists { + if !b.groupHas(region, groupName) { return nil, "", fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, groupName) } @@ -961,7 +917,7 @@ func (b *InMemoryBackend) DescribeLogStreams( limit = defaultDescribeLimit } - groupStreams := b.streamsStore(region)[groupName] + groupStreams := b.streamsInGroup(region, groupName) all := make([]LogStream, 0, len(groupStreams)) for _, s := range groupStreams { if prefix == "" || strings.HasPrefix(s.LogStreamName, prefix) { @@ -1014,8 +970,16 @@ func validatePutLogEventsBatch(events []InputLogEvent) error { return nil } +// rejectedTracker accumulates the three RejectedLogEventsInfo fields while +// classifyLogEvents walks a PutLogEvents batch in order. Per the real API +// (aws-sdk-go-v2 types.RejectedLogEventsInfo): +// - TooNewLogEventStartIndex is the INCLUSIVE index of the first too-new event. +// - TooOldLogEventEndIndex and ExpiredLogEventEndIndex are EXCLUSIVE end +// indices: "events at index < N were too-old/expired". Batches are expected +// to be chronologically ordered, so too-old/expired events form a prefix and +// the "end" is one past the last matching index seen. type rejectedTracker struct { - tooOldStart *int32 + tooOldEnd *int32 tooNewStart *int32 expiredEnd *int32 } @@ -1035,8 +999,9 @@ func (t *rejectedTracker) track( // Reject events older than the 14-day hard cap. if ts < hardCutoff { - if t.tooOldStart == nil { - t.tooOldStart = &idx + end := idx + 1 + if t.tooOldEnd == nil || end > *t.tooOldEnd { + t.tooOldEnd = &end } return false @@ -1045,8 +1010,9 @@ func (t *rejectedTracker) track( // Events beyond the group's retention window are marked as expired // in the response but are still stored; the janitor evicts them later. if retentionCutoffMs > hardCutoff && ts < retentionCutoffMs { - if t.expiredEnd == nil || idx > *t.expiredEnd { - t.expiredEnd = &idx + end := idx + 1 + if t.expiredEnd == nil || end > *t.expiredEnd { + t.expiredEnd = &end } } @@ -1078,10 +1044,10 @@ func classifyLogEvents( } var rejectedInfo *RejectedLogEventsInfo - if tracker.tooNewStart != nil || tracker.tooOldStart != nil || tracker.expiredEnd != nil { + if tracker.tooNewStart != nil || tracker.tooOldEnd != nil || tracker.expiredEnd != nil { rejectedInfo = &RejectedLogEventsInfo{ TooNewLogEventStartIndex: tracker.tooNewStart, - TooOldLogEventStartIndex: tracker.tooOldStart, + TooOldLogEventEndIndex: tracker.tooOldEnd, ExpiredLogEventEndIndex: tracker.expiredEnd, } } @@ -1090,11 +1056,16 @@ func classifyLogEvents( } // PutLogEvents appends log events to a stream and returns a PutLogEventsResult. -// sequenceToken is optional; if provided and mismatched, returns ErrInvalidSequenceToken. +// sequenceToken is accepted for wire compatibility but, matching current AWS +// behavior (see aws-sdk-go-v2 cloudwatchlogs.PutLogEvents doc: "The sequence +// token is now ignored in PutLogEvents actions. PutLogEvents actions are always +// accepted and never return InvalidSequenceTokenException or +// DataAlreadyAcceptedException even if the sequence token is not valid."), it is +// never validated: PutLogEvents accepts concurrent, unordered, or stale tokens. // Events with timestamps outside the allowed window are tracked in RejectedLogEventsInfo. func (b *InMemoryBackend) PutLogEvents( ctx context.Context, - groupName, streamName, sequenceToken string, + groupName, streamName, _ string, events []InputLogEvent, ) (*PutLogEventsResult, error) { if err := validatePutLogEventsBatch(events); err != nil { @@ -1105,42 +1076,23 @@ func (b *InMemoryBackend) PutLogEvents( b.mu.Lock("PutLogEvents") - groups := b.groupsStore(region) - if _, exists := groups[groupName]; !exists { + group, groupExists := b.groupGet(region, groupName) + if !groupExists { b.mu.Unlock() return nil, fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, groupName) } - streams := b.streamsStore(region) - groupEvents := b.eventsStore(region) - if _, exists := streams[groupName][streamName]; !exists { + stream, streamExists := b.streamGet(region, groupName, streamName) + if !streamExists { b.mu.Unlock() return nil, fmt.Errorf("%w: Log stream %s not found", ErrLogStreamNotFound, streamName) } - stream := streams[groupName][streamName] - - // Validate sequence token if provided. AWS still returns InvalidSequenceTokenException - // for wrong tokens even though tokens are now optional. - if sequenceToken != "" { - expectedToken := strconv.FormatInt(int64(len(groupEvents[groupName][streamName])), 10) - if sequenceToken != expectedToken { - b.mu.Unlock() - - return nil, fmt.Errorf( - "%w: expected sequenceToken %s", - ErrInvalidSequenceToken, - expectedToken, - ) - } - } - now := time.Now().UnixMilli() // Determine the retention cutoff for timestamp validation. - group := groups[groupName] var retentionCutoffMs int64 if group.RetentionInDays != nil && *group.RetentionInDays > 0 { retentionCutoffMs = now - int64(*group.RetentionInDays)*msPerDay @@ -1157,10 +1109,10 @@ func (b *InMemoryBackend) PutLogEvents( futureLimit, ) - b.appendEvents(region, groupName, streamName, stream, now, acceptedEvents) + b.appendEvents(group, stream, now, acceptedEvents) stream.LastIngestionTime = &now - nextToken := strconv.FormatInt(int64(len(groupEvents[groupName][streamName])), 10) + nextToken := strconv.FormatInt(int64(len(stream.events)), 10) // Collect matching subscription filters and metric filter matches while holding the lock. filters := b.matchingFilters(region, groupName, acceptedEvents) @@ -1221,12 +1173,11 @@ func (b *InMemoryBackend) scheduleFilterDelivery( // Note: log events may arrive with out-of-order timestamps (AWS allows this), // so min/max timestamp tracking must inspect all events. func (b *InMemoryBackend) appendEvents( - region, groupName, streamName string, stream *LogStream, now int64, events []InputLogEvent, + group *LogGroup, stream *LogStream, now int64, events []InputLogEvent, ) { - groupEvents := b.eventsStore(region) - groups := b.groupsStore(region) + groupName, streamName := stream.logGroupName, stream.LogStreamName for _, ev := range events { - idx := len(groupEvents[groupName][streamName]) + idx := len(stream.events) ptr := base64.StdEncoding.EncodeToString( fmt.Appendf(nil, "%s/%s/%d", groupName, streamName, idx), ) @@ -1236,11 +1187,11 @@ func (b *InMemoryBackend) appendEvents( Timestamp: ev.Timestamp, Ptr: ptr, } - groupEvents[groupName][streamName] = append(groupEvents[groupName][streamName], out) + stream.events = append(stream.events, out) msgLen := int64(len(ev.Message)) stream.StoredBytes += msgLen - groups[groupName].StoredBytes += msgLen + group.StoredBytes += msgLen if stream.FirstEventTimestamp == nil || ev.Timestamp < *stream.FirstEventTimestamp { ts := ev.Timestamp @@ -1253,12 +1204,12 @@ func (b *InMemoryBackend) appendEvents( } // Enforce per-stream event cap: keep only the most recent maxEventsPerStream events. - if cur := groupEvents[groupName][streamName]; len(cur) > maxEventsPerStream { - groupEvents[groupName][streamName] = cur[len(cur)-maxEventsPerStream:] + if cur := stream.events; len(cur) > maxEventsPerStream { + stream.events = cur[len(cur)-maxEventsPerStream:] // Recalculate metadata from the remaining events: since events may have // out-of-order timestamps, the dropped events might include the global // min/max, so we must re-scan rather than assume positional ordering. - updateStreamTimestamps(stream, groupEvents[groupName][streamName]) + updateStreamTimestamps(stream, stream.events) } } @@ -1282,11 +1233,12 @@ func (b *InMemoryBackend) GetLogEvents( b.mu.RLock("GetLogEvents") defer b.mu.RUnlock() - if _, exists := b.groupsStore(region)[groupName]; !exists { + if !b.groupHas(region, groupName) { return nil, "", "", fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, groupName) } - if _, exists := b.streamsStore(region)[groupName][streamName]; !exists { + stream, exists := b.streamGet(region, groupName, streamName) + if !exists { return nil, "", "", fmt.Errorf( "%w: Log stream %s not found", ErrLogStreamNotFound, @@ -1294,8 +1246,7 @@ func (b *InMemoryBackend) GetLogEvents( ) } - all := b.eventsStore(region)[groupName][streamName] - filtered := filterByTime(all, startTime, endTime) + filtered := filterByTime(stream.events, startTime, endTime) if limit <= 0 { limit = defaultEventLimit @@ -1366,7 +1317,7 @@ func (b *InMemoryBackend) FilterLogEvents( b.mu.RLock("FilterLogEvents") defer b.mu.RUnlock() - if _, exists := b.groupsStore(region)[p.GroupName]; !exists { + if !b.groupHas(region, p.GroupName) { return nil, "", nil, fmt.Errorf( "%w: Log group %s not found", ErrLogGroupNotFound, @@ -1385,12 +1336,15 @@ func (b *InMemoryBackend) FilterLogEvents( if p.LogStreamNamePrefix != "" { streamOrder = filterStreamsByPrefix(streamOrder, p.LogStreamNamePrefix) } - groupEvents := b.eventsStore(region) var all []taggedEvent for _, sName := range streamOrder { - for _, ev := range groupEvents[p.GroupName][sName] { + stream, ok := b.streamGet(region, p.GroupName, sName) + if !ok { + continue + } + for _, ev := range stream.events { if compiled != nil && !compiled.matches(ev.Message) { continue } @@ -1519,20 +1473,19 @@ func (b *InMemoryBackend) PutSubscriptionFilter( b.mu.Lock("PutSubscriptionFilter") defer b.mu.Unlock() - if _, exists := b.groupsStore(region)[groupName]; !exists { + if !b.groupHas(region, groupName) { return fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, groupName) } - subFilters := b.subscriptionFiltersStore(region) - existing := subFilters[groupName] + existing := b.subscriptionFiltersInGroup(region, groupName) // Check for a filter with the same name (update). - for i, f := range existing { + for _, f := range existing { if f.FilterName == filterName { - existing[i].FilterPattern = filterPattern - existing[i].DestinationArn = destinationArn - existing[i].RoleArn = roleArn - existing[i].Distribution = distribution + f.FilterPattern = filterPattern + f.DestinationArn = destinationArn + f.RoleArn = roleArn + f.Distribution = distribution return nil } @@ -1544,7 +1497,7 @@ func (b *InMemoryBackend) PutSubscriptionFilter( ErrSubscriptionFilterLimitExceed, groupName) } - subFilters[groupName] = append(existing, &SubscriptionFilter{ + b.subscriptionFilters.Put(&SubscriptionFilter{ FilterName: filterName, FilterPattern: filterPattern, LogGroupName: groupName, @@ -1552,6 +1505,7 @@ func (b *InMemoryBackend) PutSubscriptionFilter( RoleArn: roleArn, Distribution: distribution, CreationTime: time.Now().UnixMilli(), + region: region, }) return nil @@ -1568,11 +1522,11 @@ func (b *InMemoryBackend) DescribeSubscriptionFilters( b.mu.RLock("DescribeSubscriptionFilters") defer b.mu.RUnlock() - if _, exists := b.groupsStore(region)[groupName]; !exists { + if !b.groupHas(region, groupName) { return nil, "", fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, groupName) } - groupFilters := b.subscriptionFiltersStore(region)[groupName] + groupFilters := b.subscriptionFiltersInGroup(region, groupName) all := make([]SubscriptionFilter, 0, len(groupFilters)) for _, f := range groupFilters { if filterNamePrefix == "" || strings.HasPrefix(f.FilterName, filterNamePrefix) { @@ -1612,18 +1566,12 @@ func (b *InMemoryBackend) DeleteSubscriptionFilter( b.mu.Lock("DeleteSubscriptionFilter") defer b.mu.Unlock() - if _, exists := b.groupsStore(region)[groupName]; !exists { + if !b.groupHas(region, groupName) { return fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, groupName) } - subFilters := b.subscriptionFiltersStore(region) - filters := subFilters[groupName] - for i, f := range filters { - if f.FilterName == filterName { - subFilters[groupName] = append(filters[:i], filters[i+1:]...) - - return nil - } + if b.subscriptionFilters.Delete(subFilterTableKey(region, groupName, filterName)) { + return nil } return fmt.Errorf("%w: subscription filter %s not found in log group %s", @@ -1636,7 +1584,7 @@ func (b *InMemoryBackend) matchingFilters( region, groupName string, events []InputLogEvent, ) []*SubscriptionFilter { - filters := b.subscriptionFiltersStore(region)[groupName] + filters := b.subscriptionFiltersInGroup(region, groupName) if len(filters) == 0 { return nil } @@ -1653,13 +1601,18 @@ func (b *InMemoryBackend) matchingFilters( } // metricFilterMatch holds a metric filter and the count of events that matched it. +// metricFilterMatch pairs a metric filter with the raw messages (in event +// order) of every log event that matched it. The messages -- not just a count +// -- are needed because MetricTransformation.MetricValue may reference a named +// field ("$size", "$.bytes") that must be extracted per-event rather than a +// single fixed literal applied matchCount times. type metricFilterMatch struct { - filter *MetricFilter - matchCount int + filter *MetricFilter + messages []string } // matchingMetricFilters returns metric filters for groupName whose pattern matches at least one -// of the given events, along with the per-filter match count. +// of the given events, along with the matched messages themselves (see metricFilterMatch). // Events outer, filters inner: each event is visited once regardless of filter count, // cutting allocations from O(filters×events) repeated scans to a single pass. // Must be called while holding the write lock. @@ -1667,7 +1620,7 @@ func (b *InMemoryBackend) matchingMetricFilters( region, groupName string, events []InputLogEvent, ) []metricFilterMatch { - mfMap := b.metricFiltersStore(region)[groupName] + mfMap := b.metricFiltersInGroup(region, groupName) if len(mfMap) == 0 { return nil } @@ -1685,20 +1638,20 @@ func (b *InMemoryBackend) matchingMetricFilters( ) } - counts := make([]int, len(entries)) + matchedMsgs := make([][]string, len(entries)) for _, ev := range events { for i, e := range entries { if e.compiled == nil || e.compiled.matches(ev.Message) { - counts[i]++ + matchedMsgs[i] = append(matchedMsgs[i], ev.Message) } } } var matched []metricFilterMatch for i, e := range entries { - if counts[i] > 0 { + if len(matchedMsgs[i]) > 0 { cp := *e.filter - matched = append(matched, metricFilterMatch{filter: &cp, matchCount: counts[i]}) + matched = append(matched, metricFilterMatch{filter: &cp, messages: matchedMsgs[i]}) } } @@ -1706,23 +1659,21 @@ func (b *InMemoryBackend) matchingMetricFilters( } // emitMetricFilterMatches calls the MetricEmitter for each matched metric filter transformation. -// One data point is emitted per matched event per transformation. +// One data point is emitted per matched event per transformation, with the value resolved by +// metricTransformationValue (a literal MetricValue, or a per-event field extraction). func (b *InMemoryBackend) emitMetricFilterMatches( emitter MetricEmitter, matches []metricFilterMatch, ) { for _, m := range matches { + compiled := b.getCompiledPattern(m.filter.FilterPattern) for _, t := range m.filter.MetricTransformations { - val, parseErr := strconv.ParseFloat(t.MetricValue, 64) - if parseErr != nil { - // Non-numeric MetricValue (e.g. "$field") falls back to defaultValue or 1.0. - val = 1.0 - if t.DefaultValue != nil { - val = *t.DefaultValue + for _, msg := range m.messages { + val, ok := metricTransformationValue(compiled, msg, t) + if !ok { + continue } - } - for range m.matchCount { - if emitErr := emitter.EmitMetric(t.MetricNamespace, t.MetricName, val, ""); emitErr != nil { + if emitErr := emitter.EmitMetric(t.MetricNamespace, t.MetricName, val, t.Unit); emitErr != nil { logger.Load(b.ctx).Warn( "cloudwatchlogs: metric filter emit failed", "namespace", t.MetricNamespace, @@ -1735,6 +1686,24 @@ func (b *InMemoryBackend) emitMetricFilterMatches( } } +// metricTransformationValue resolves the numeric value to publish for one matched log event. +// A literal numeric MetricValue (e.g. "1") publishes as-is for every match. A field reference +// ("$size" for space-delimited patterns, "$.bytes" for JSON patterns) extracts that field from +// this specific matched message. +// +// AWS's DefaultValue is documented as "the value to emit when a filter pattern does NOT match a +// log event" (a periodic/no-data-point substitute), not a fallback for failed per-event field +// extraction -- so a missing or non-numeric field on a MATCHED event silently emits no data point +// for that event, matching real CloudWatch Logs metric filter behavior, rather than fabricating a +// value. +func metricTransformationValue(compiled *compiledFilterPattern, msg string, t MetricTransformation) (float64, bool) { + if f, err := strconv.ParseFloat(t.MetricValue, 64); err == nil { + return f, true + } + + return compiled.extractValue(msg, t.MetricValue) +} + // getCompiledPattern returns a cached compiled filter pattern, compiling and caching it on first use. func (b *InMemoryBackend) getCompiledPattern(pattern string) *compiledFilterPattern { b.compiledPatternsMu.RLock() @@ -1872,12 +1841,48 @@ type compiledFilterPattern struct { // custom, when non-nil, fully determines matching (used for JSON "{...}" // selector patterns and space-delimited "[...]" metric-filter patterns); // the term slices below are then unused. - custom func(message string) bool + custom func(message string) bool + // extract, when non-nil (JSON and space-delimited patterns only), resolves a + // named field reference ("size" / ".bytes", i.e. a MetricValue with its + // leading "$" already stripped) against one message. Plain-text patterns + // have no addressable fields, so extract stays nil for them. + extract func(message, fieldRef string) (string, bool) required []compiledTerm // AND: all must match optional []compiledTerm // OR: any matches (only used when required+exclude empty) exclude []compiledTerm // NONE may match } +// extractString resolves a "$"-prefixed MetricValue-style field reference (e.g. "$size", +// "$.bytes") against message, using this pattern's named-field structure. It reports false +// when the pattern has no addressable fields (plain-text patterns), fieldRef isn't +// "$"-prefixed, or the field is absent from this particular message. +func (p *compiledFilterPattern) extractString(message, fieldRef string) (string, bool) { + if p == nil || p.extract == nil { + return "", false + } + + rest, ok := strings.CutPrefix(fieldRef, "$") + if !ok { + return "", false + } + + return p.extract(message, rest) +} + +// extractValue is extractString plus a numeric parse, for MetricTransformation.MetricValue +// resolution: CloudWatch metric values must be numeric, so a present-but-non-numeric field +// (or an absent one) both report false, meaning "emit no data point for this event". +func (p *compiledFilterPattern) extractValue(message, fieldRef string) (float64, bool) { + raw, ok := p.extractString(message, fieldRef) + if !ok { + return 0, false + } + + f, err := strconv.ParseFloat(raw, 64) + + return f, err == nil +} + // compiledTerm holds a single pre-compiled term from a filter pattern. type compiledTerm struct { // exact is the literal substring for quoted/plain terms (non-wildcard). @@ -1937,9 +1942,15 @@ func compileFilterPattern(pattern string) *compiledFilterPattern { if trimmed := strings.TrimSpace(pattern); trimmed != "" { switch trimmed[0] { case '{': - return &compiledFilterPattern{custom: compileJSONFilterPattern(trimmed)} + return &compiledFilterPattern{ + custom: compileJSONFilterPattern(trimmed), + extract: compileJSONFilterPatternExtract(trimmed), + } case '[': - return &compiledFilterPattern{custom: compileSpaceFilterPattern(trimmed)} + return &compiledFilterPattern{ + custom: compileSpaceFilterPattern(trimmed), + extract: compileSpaceFilterPatternExtract(trimmed), + } } } @@ -1960,6 +1971,26 @@ func compileFilterPattern(pattern string) *compiledFilterPattern { return cp } +// patternFieldRefs returns every "$"-prefixed field reference addressable in pattern +// (e.g. ["$.eventType"] for a JSON selector pattern, ["$ip", "$size"] for a +// space-delimited pattern with named fields). Plain-text patterns have no addressable +// fields and return nil. Used by TestMetricFilter to populate ExtractedValues. +func patternFieldRefs(pattern string) []string { + trimmed := strings.TrimSpace(pattern) + if trimmed == "" { + return nil + } + + switch trimmed[0] { + case '{': + return jsonPatternSelectors(trimmed) + case '[': + return spacePatternFieldNames(trimmed) + default: + return nil + } +} + // matches reports whether the message satisfies the pattern, following AWS // unstructured filter-pattern semantics. func (p *compiledFilterPattern) matches(message string) bool { @@ -2061,12 +2092,6 @@ func filterByTime(events []*OutputLogEvent, startTime, endTime *int64) []*Output return out } -func sortedKeys(m map[string]*LogStream) []string { - keys := collections.SortedKeys(m) - - return keys -} - // filterStreamOrderLocked returns the ordered list of stream names to iterate // for FilterLogEvents. When streamNames is empty, all streams in the group are // returned in sorted order. When streamNames is non-empty, only the requested @@ -2076,9 +2101,21 @@ func (b *InMemoryBackend) filterStreamOrderLocked( region, groupName string, streamNames []string, ) []string { - groupStreams := b.streamsStore(region)[groupName] + groupStreams := b.streamsInGroup(region, groupName) + if len(streamNames) == 0 { - return sortedKeys(groupStreams) + names := make([]string, len(groupStreams)) + for i, s := range groupStreams { + names[i] = s.LogStreamName + } + sort.Strings(names) + + return names + } + + existing := make(map[string]bool, len(groupStreams)) + for _, s := range groupStreams { + existing[s.LogStreamName] = true } seen := make(map[string]bool, len(streamNames)) @@ -2091,7 +2128,7 @@ func (b *InMemoryBackend) filterStreamOrderLocked( seen[s] = true - if _, ok := groupStreams[s]; ok { + if existing[s] { out = append(out, s) } } @@ -2205,13 +2242,13 @@ func (b *InMemoryBackend) evictByTTL() { cutoff := time.Now().Add(-b.queryTTL) newOrder := make([]string, 0, len(b.queriesOrder)) for _, qid := range b.queriesOrder { - sq, ok := b.queries[qid] + sq, ok := b.queries.Get(qid) if !ok { - // Entry already removed from the map; drop the stale order reference. + // Entry already removed from the table; drop the stale order reference. continue } if sq.createdAt.Before(cutoff) { - delete(b.queries, qid) + b.queries.Delete(qid) continue } @@ -2229,7 +2266,7 @@ func (b *InMemoryBackend) enforceCap() { excess := len(b.queriesOrder) - b.maxQueries for _, qid := range b.queriesOrder[:excess] { - delete(b.queries, qid) + b.queries.Delete(qid) } b.queriesOrder = b.queriesOrder[excess:] } @@ -2276,22 +2313,16 @@ func (b *InMemoryBackend) collectQueryEvents( var eventsOut []*OutputLogEvent var recordsScanned, bytesScanned float64 - groupEvents := b.eventsStore(region) - streams := b.streamsStore(region) for _, groupName := range logGroupNames { - streamMap, exists := groupEvents[groupName] - if !exists { - continue - } - for streamName, evts := range streamMap { + for _, stream := range b.streamsInGroup(region, groupName) { // Narrow the scan by log stream: a stream whose [first,last] event // window does not overlap the query's [startTime,endTime] range holds // no matching records, so skip it entirely instead of scanning every // event. This bounds the scan to streams that can contribute results. - if streamOutsideWindow(streams[groupName][streamName], startTime, endTime) { + if streamOutsideWindow(stream, startTime, endTime) { continue } - matched, records, bytes := scanStreamEvents(evts, startTime, endTime) + matched, records, bytes := scanStreamEvents(stream.events, startTime, endTime) eventsOut = append(eventsOut, matched...) recordsScanned += records bytesScanned += bytes @@ -2410,11 +2441,11 @@ func (b *InMemoryBackend) StartQuery( // If this queryID already exists, remove its stale position in queriesOrder to // prevent duplicates that could cause map-miss panics or over-counting. - if _, exists := b.queries[queryID]; exists { + if b.queries.Has(queryID) { b.removeFromOrder(queryID) } - b.queries[queryID] = sq + b.queries.Put(sq) b.queriesOrder = append(b.queriesOrder, queryID) // Enforce the cap after inserting so the new entry counts against the limit. @@ -2430,7 +2461,7 @@ func (b *InMemoryBackend) GetQueryResults( queryID string, ) ([][]ResultField, QueryStatistics, QueryStatus, error) { b.mu.RLock("GetQueryResults") - sq, ok := b.queries[queryID] + sq, ok := b.queries.Get(queryID) b.mu.RUnlock() if !ok { @@ -2457,7 +2488,7 @@ func (b *InMemoryBackend) StopQuery(queryID string) error { b.mu.Lock("StopQuery") defer b.mu.Unlock() - sq, ok := b.queries[queryID] + sq, ok := b.queries.Get(queryID) if !ok { return fmt.Errorf("%w: query %s not found", ErrQueryNotFound, queryID) } @@ -2482,7 +2513,7 @@ func (b *InMemoryBackend) DescribeQueries( all := make([]QueryInfo, 0, len(b.queriesOrder)) for _, qid := range b.queriesOrder { - sq, ok := b.queries[qid] + sq, ok := b.queries.Get(qid) if !ok { continue } @@ -2524,35 +2555,16 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.groups = make(map[string]map[string]*LogGroup) - b.streams = make(map[string]map[string]map[string]*LogStream) - b.events = make(map[string]map[string]map[string][]*OutputLogEvent) - b.subscriptionFilters = make(map[string]map[string][]*SubscriptionFilter) - b.queries = make(map[string]*storedQuery) + b.registry.ResetAll() + b.ephemeralRegistry.ResetAll() + b.groups.Reset() + b.streams.Reset() + b.subscriptionFilters.Reset() + b.metricFilters.Reset() + b.queriesOrder = nil - b.parsedQueries = make(map[string]*insightsQuery) b.parsedQueriesOrder = nil - b.exportTasks = make(map[string]*ExportTask) - b.importTasks = make(map[string]*ImportTask) - b.deliveries = make(map[string]*Delivery) - b.logAnomalyDetectors = make(map[string]*LogAnomalyDetector) - b.anomalies = make(map[string]map[string]*Anomaly) - b.scheduledQueries = make(map[string]*ScheduledQuery) - b.scheduledQueryRuns = make(map[string][]*ScheduledQueryRunSummary) - b.accountPolicies = make(map[string]*AccountPolicy) - b.kmsKeys = make(map[string]string) - b.s3TableIntegrations = make(map[string]string) - b.metricFilters = make(map[string]map[string]map[string]*MetricFilter) - b.queryDefinitions = make(map[string]*QueryDefinition) - b.dataProtectionPolicies = make(map[string]string) - b.resourcePolicies = make(map[string]ResourcePolicy) - b.deliveryDestinations = make(map[string]DeliveryDestination) - b.deliverySources = make(map[string]DeliverySource) - b.destinations = make(map[string]CWLDestination) - b.indexPolicies = make(map[string]IndexPolicy) - b.transformers = make(map[string]Transformer) - b.integrations = make(map[string]CWLIntegration) - b.deletionProtected = make(map[string]bool) + b.parsedQueries = make(map[string]*insightsQuery) b.compiledPatternsMu.Lock() b.compiledPatterns = make(map[string]*compiledFilterPattern) @@ -2569,7 +2581,10 @@ func (b *InMemoryBackend) PutDataProtectionPolicy(logGroupIdentifier, policyDocu b.mu.Lock("PutDataProtectionPolicy") defer b.mu.Unlock() - b.dataProtectionPolicies[logGroupIdentifier] = policyDocument + b.dataProtectionPolicies.Put(&dataProtectionPolicyEntry{ + LogGroupIdentifier: logGroupIdentifier, + PolicyDocument: policyDocument, + }) return nil } @@ -2580,12 +2595,12 @@ func (b *InMemoryBackend) GetDataProtectionPolicy(logGroupIdentifier string) (st b.mu.RLock("GetDataProtectionPolicy") defer b.mu.RUnlock() - policy, ok := b.dataProtectionPolicies[logGroupIdentifier] + entry, ok := b.dataProtectionPolicies.Get(logGroupIdentifier) if !ok { return "{}", nil } - return policy, nil + return entry.PolicyDocument, nil } // DeleteDataProtectionPolicy removes the data protection policy for a log group. @@ -2593,7 +2608,7 @@ func (b *InMemoryBackend) DeleteDataProtectionPolicy(logGroupIdentifier string) b.mu.Lock("DeleteDataProtectionPolicy") defer b.mu.Unlock() - delete(b.dataProtectionPolicies, logGroupIdentifier) + b.dataProtectionPolicies.Delete(logGroupIdentifier) return nil } @@ -2620,7 +2635,7 @@ func (b *InMemoryBackend) AssociateKmsKey(logGroupName, resourceIdentifier, kmsK key = resourceIdentifier } - b.kmsKeys[key] = kmsKeyID + b.kmsKeys.Put(&kmsKeyEntry{Key: key, KmsKeyID: kmsKeyID}) return nil } @@ -2639,7 +2654,7 @@ func (b *InMemoryBackend) AssociateSourceToS3TableIntegration( b.mu.Lock("AssociateSourceToS3TableIntegration") defer b.mu.Unlock() - b.s3TableIntegrations[id] = integrationArn + b.s3TableIntegrations.Put(&s3TableIntegrationEntry{ID: id, IntegrationArn: integrationArn}) return id, nil } @@ -2654,7 +2669,7 @@ func (b *InMemoryBackend) CancelExportTask(taskID string) error { b.mu.Lock("CancelExportTask") defer b.mu.Unlock() - task, ok := b.exportTasks[taskID] + task, ok := b.exportTasks.Get(taskID) if !ok { return fmt.Errorf("%w: export task %s not found", ErrExportTaskNotFound, taskID) } @@ -2680,7 +2695,7 @@ func (b *InMemoryBackend) CancelImportTask(importID string) (*ImportTask, error) b.mu.Lock("CancelImportTask") defer b.mu.Unlock() - task, ok := b.importTasks[importID] + task, ok := b.importTasks.Get(importID) if !ok { return nil, fmt.Errorf("%w: import task %s not found", ErrImportTaskNotFound, importID) } @@ -2728,7 +2743,7 @@ func (b *InMemoryBackend) CreateDelivery( b.mu.Lock("CreateDelivery") defer b.mu.Unlock() - b.deliveries[id] = d + b.deliveries.Put(d) cp := *d cp.Tags = maps.Clone(d.Tags) @@ -2773,13 +2788,13 @@ func (b *InMemoryBackend) CreateExportTask( } b.mu.Lock("CreateExportTask") - if len(b.exportTasks) >= maxExportTasks { + if b.exportTasks.Len() >= maxExportTasks { b.mu.Unlock() return "", fmt.Errorf("%w: export task limit exceeded", ErrValidation) } - b.exportTasks[task.TaskID] = task + b.exportTasks.Put(task) sink := b.exportSink b.mu.Unlock() @@ -2797,7 +2812,7 @@ func (b *InMemoryBackend) finishExport(task *ExportTask) { b.mu.Lock("finishExport") defer b.mu.Unlock() - stored, ok := b.exportTasks[task.TaskID] + stored, ok := b.exportTasks.Get(task.TaskID) if !ok { return } @@ -2843,11 +2858,11 @@ func (b *InMemoryBackend) CreateImportTask( b.mu.Lock("CreateImportTask") defer b.mu.Unlock() - if len(b.importTasks) >= maxImportTasks { + if b.importTasks.Len() >= maxImportTasks { return nil, fmt.Errorf("%w: import task limit exceeded", ErrValidation) } - b.importTasks[importID] = task + b.importTasks.Put(task) cp := *task @@ -2907,11 +2922,11 @@ func (b *InMemoryBackend) CreateLogAnomalyDetector( b.mu.Lock("CreateLogAnomalyDetector") defer b.mu.Unlock() - if len(b.logAnomalyDetectors) >= maxAnomalyDetectors { + if b.logAnomalyDetectors.Len() >= maxAnomalyDetectors { return "", fmt.Errorf("%w: anomaly detector limit exceeded", ErrValidation) } - b.logAnomalyDetectors[detectorARN] = detector + b.logAnomalyDetectors.Put(detector) return detectorARN, nil } @@ -2956,22 +2971,25 @@ func (b *InMemoryBackend) CreateScheduledQuery( b.mu.Lock("CreateScheduledQuery") defer b.mu.Unlock() - if len(b.scheduledQueries) >= maxScheduledQueries { + if b.scheduledQueries.Len() >= maxScheduledQueries { return "", fmt.Errorf("%w: scheduled query limit exceeded", ErrValidation) } - b.scheduledQueries[queryARN] = sq + b.scheduledQueries.Put(sq) // Seed an initial SUCCEEDED run so history is non-empty from creation. now := time.Now().UnixMilli() - b.scheduledQueryRuns[queryARN] = []*ScheduledQueryRunSummary{ - { - Arn: queryARN, - RunStatus: "SUCCEEDED", - ExecutionTime: now, - InvocationTime: now, + b.scheduledQueryRuns.Put(&scheduledQueryRunHistory{ + Arn: queryARN, + Runs: []*ScheduledQueryRunSummary{ + { + Arn: queryARN, + RunStatus: "SUCCEEDED", + ExecutionTime: now, + InvocationTime: now, + }, }, - } + }) return queryARN, nil } @@ -2994,7 +3012,7 @@ func (b *InMemoryBackend) DeleteAccountPolicy(policyName, policyType string) err defer b.mu.Unlock() key := policyName + ":" + policyType - delete(b.accountPolicies, key) + b.accountPolicies.Delete(key) return nil } @@ -3010,7 +3028,7 @@ func (b *InMemoryBackend) DescribeExportTasks( defer b.mu.Unlock() now := time.Now().UnixMilli() - for _, t := range b.exportTasks { + for _, t := range b.exportTasks.All() { age := now - t.CreationTime if age > maxExportTaskAgeMs { continue @@ -3023,8 +3041,8 @@ func (b *InMemoryBackend) DescribeExportTasks( } } - all := make([]ExportTask, 0, len(b.exportTasks)) - for _, t := range b.exportTasks { + all := make([]ExportTask, 0, b.exportTasks.Len()) + for _, t := range b.exportTasks.All() { if taskID != "" && t.TaskID != taskID { continue } @@ -3062,8 +3080,8 @@ func (b *InMemoryBackend) DescribeImportTasks( b.mu.RLock("DescribeImportTasks") defer b.mu.RUnlock() - all := make([]ImportTask, 0, len(b.importTasks)) - for _, t := range b.importTasks { + all := make([]ImportTask, 0, b.importTasks.Len()) + for _, t := range b.importTasks.All() { if taskID != "" && t.ImportID != taskID { continue } @@ -3097,8 +3115,8 @@ func (b *InMemoryBackend) DescribeDeliveries( b.mu.RLock("DescribeDeliveries") defer b.mu.RUnlock() - all := make([]Delivery, 0, len(b.deliveries)) - for _, d := range b.deliveries { + all := make([]Delivery, 0, b.deliveries.Len()) + for _, d := range b.deliveries.All() { cp := *d cp.Tags = maps.Clone(d.Tags) all = append(all, cp) @@ -3132,7 +3150,7 @@ func (b *InMemoryBackend) GetDelivery(id string) (*Delivery, error) { b.mu.RLock("GetDelivery") defer b.mu.RUnlock() - d, ok := b.deliveries[id] + d, ok := b.deliveries.Get(id) if !ok { return nil, fmt.Errorf("%w: delivery %s not found", ErrDeliveryNotFound, id) } @@ -3151,10 +3169,9 @@ func (b *InMemoryBackend) DeleteDelivery(id string) error { b.mu.Lock("DeleteDelivery") defer b.mu.Unlock() - if _, ok := b.deliveries[id]; !ok { + if !b.deliveries.Delete(id) { return fmt.Errorf("%w: delivery %s not found", ErrDeliveryNotFound, id) } - delete(b.deliveries, id) return nil } @@ -3168,14 +3185,13 @@ func (b *InMemoryBackend) DeleteLogAnomalyDetector(detectorArn string) error { b.mu.Lock("DeleteLogAnomalyDetector") defer b.mu.Unlock() - if _, ok := b.logAnomalyDetectors[detectorArn]; !ok { + if !b.logAnomalyDetectors.Delete(detectorArn) { return fmt.Errorf( "%w: anomaly detector %s not found", ErrLogAnomalyDetectorNotFound, detectorArn, ) } - delete(b.logAnomalyDetectors, detectorArn) return nil } @@ -3194,8 +3210,8 @@ func (b *InMemoryBackend) ListLogAnomalyDetectors( filterSet[a] = true } - all := make([]LogAnomalyDetector, 0, len(b.logAnomalyDetectors)) - for _, d := range b.logAnomalyDetectors { + all := make([]LogAnomalyDetector, 0, b.logAnomalyDetectors.Len()) + for _, d := range b.logAnomalyDetectors.All() { if len(filterSet) > 0 { match := false for _, a := range d.LogGroupArnList { @@ -3257,7 +3273,7 @@ func (b *InMemoryBackend) UpdateLogAnomalyDetector( b.mu.Lock("UpdateLogAnomalyDetector") defer b.mu.Unlock() - d, ok := b.logAnomalyDetectors[detectorArn] + d, ok := b.logAnomalyDetectors.Get(detectorArn) if !ok { return fmt.Errorf( "%w: anomaly detector %s not found", @@ -3296,14 +3312,13 @@ func (b *InMemoryBackend) DeleteScheduledQuery(scheduledQueryArn string) error { b.mu.Lock("DeleteScheduledQuery") defer b.mu.Unlock() - if _, ok := b.scheduledQueries[scheduledQueryArn]; !ok { + if !b.scheduledQueries.Delete(scheduledQueryArn) { return fmt.Errorf( "%w: scheduled query %s not found", ErrScheduledQueryNotFound, scheduledQueryArn, ) } - delete(b.scheduledQueries, scheduledQueryArn) return nil } @@ -3316,8 +3331,8 @@ func (b *InMemoryBackend) ListScheduledQueries( b.mu.RLock("ListScheduledQueries") defer b.mu.RUnlock() - all := make([]ScheduledQuery, 0, len(b.scheduledQueries)) - for _, sq := range b.scheduledQueries { + all := make([]ScheduledQuery, 0, b.scheduledQueries.Len()) + for _, sq := range b.scheduledQueries.All() { all = append(all, *sq) } sort.Slice(all, func(i, j int) bool { return all[i].CreationTime < all[j].CreationTime }) @@ -3355,7 +3370,7 @@ func (b *InMemoryBackend) UpdateScheduledQuery(scheduledQueryArn, state string) b.mu.Lock("UpdateScheduledQuery") defer b.mu.Unlock() - sq, ok := b.scheduledQueries[scheduledQueryArn] + sq, ok := b.scheduledQueries.Get(scheduledQueryArn) if !ok { return fmt.Errorf( "%w: scheduled query %s not found", @@ -3402,7 +3417,6 @@ func (b *InMemoryBackend) PutAccountPolicy( b.mu.Lock("PutAccountPolicy") defer b.mu.Unlock() - key := policyName + ":" + policyType p := &AccountPolicy{ PolicyName: policyName, PolicyType: policyType, @@ -3410,7 +3424,7 @@ func (b *InMemoryBackend) PutAccountPolicy( Scope: scope, SelectionCriteria: selectionCriteria, } - b.accountPolicies[key] = p + b.accountPolicies.Put(p) cp := *p return &cp, nil @@ -3433,8 +3447,8 @@ func (b *InMemoryBackend) DescribeAccountPolicies( b.mu.RLock("DescribeAccountPolicies") defer b.mu.RUnlock() - all := make([]AccountPolicy, 0, len(b.accountPolicies)) - for _, p := range b.accountPolicies { + all := make([]AccountPolicy, 0, b.accountPolicies.Len()) + for _, p := range b.accountPolicies.All() { if policyType != "" && p.PolicyType != policyType { continue } @@ -3480,7 +3494,7 @@ func (b *InMemoryBackend) DisassociateKmsKey(logGroupName, resourceIdentifier st if key == "" { key = resourceIdentifier } - delete(b.kmsKeys, key) + b.kmsKeys.Delete(key) return nil } @@ -3506,18 +3520,13 @@ func (b *InMemoryBackend) PutMetricFilter( b.mu.Lock("PutMetricFilter") defer b.mu.Unlock() - groups := b.groupsStore(region) - if _, exists := groups[logGroupName]; !exists { + group, exists := b.groupGet(region, logGroupName) + if !exists { return fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, logGroupName) } - metricFilters := b.metricFiltersStore(region) - if metricFilters[logGroupName] == nil { - metricFilters[logGroupName] = make(map[string]*MetricFilter) - } - creationTime := time.Now().UnixMilli() - if existing, ok := metricFilters[logGroupName][filterName]; ok { + if existing, ok := b.metricFilterGet(region, logGroupName, filterName); ok { creationTime = existing.CreationTime } @@ -3527,10 +3536,11 @@ func (b *InMemoryBackend) PutMetricFilter( FilterPattern: filterPattern, MetricTransformations: append([]MetricTransformation(nil), transformations...), CreationTime: creationTime, + region: region, } - metricFilters[logGroupName][filterName] = mf - count := len(metricFilters[logGroupName]) - groups[logGroupName].MetricFilterCount = int32( + b.metricFilters.Put(mf) + count := len(b.metricFiltersInGroup(region, logGroupName)) + group.MetricFilterCount = int32( count, ) // #nosec G115 -- count bounded by AWS API limit @@ -3548,21 +3558,26 @@ func (b *InMemoryBackend) DescribeMetricFilters( b.mu.RLock("DescribeMetricFilters") defer b.mu.RUnlock() + var filterSet []*MetricFilter + if logGroupName != "" { + filterSet = b.metricFiltersInGroup(region, logGroupName) + } else { + filterSet = b.metricFilters.All() + } + var all []MetricFilter - for grp, filters := range b.metricFiltersStore(region) { - if logGroupName != "" && grp != logGroupName { + for _, mf := range filterSet { + if mf.region != region { continue } - for _, mf := range filters { - if !metricFilterMatches(mf, filterNamePrefix, metricName, metricNamespace) { - continue - } - cp := *mf - cp.MetricTransformations = append( - []MetricTransformation(nil), - mf.MetricTransformations...) - all = append(all, cp) + if !metricFilterMatches(mf, filterNamePrefix, metricName, metricNamespace) { + continue } + cp := *mf + cp.MetricTransformations = append( + []MetricTransformation(nil), + mf.MetricTransformations...) + all = append(all, cp) } sort.Slice(all, func(i, j int) bool { if all[i].LogGroupName != all[j].LogGroupName { @@ -3628,14 +3643,12 @@ func (b *InMemoryBackend) DeleteMetricFilter( b.mu.Lock("DeleteMetricFilter") defer b.mu.Unlock() - groups := b.groupsStore(region) - if _, exists := groups[logGroupName]; !exists { + group, exists := b.groupGet(region, logGroupName) + if !exists { return fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, logGroupName) } - metricFilters := b.metricFiltersStore(region) - filters := metricFilters[logGroupName] - if _, ok := filters[filterName]; !ok { + if !b.metricFilters.Delete(metricFilterTableKey(region, logGroupName, filterName)) { return fmt.Errorf( "%w: metric filter %s not found in log group %s", ErrMetricFilterNotFound, @@ -3643,12 +3656,8 @@ func (b *InMemoryBackend) DeleteMetricFilter( logGroupName, ) } - delete(filters, filterName) - if len(filters) == 0 { - delete(metricFilters, logGroupName) - } - count := len(metricFilters[logGroupName]) - groups[logGroupName].MetricFilterCount = int32( + count := len(b.metricFiltersInGroup(region, logGroupName)) + group.MetricFilterCount = int32( count, ) // #nosec G115 -- count bounded by AWS API limit @@ -3665,15 +3674,25 @@ func (b *InMemoryBackend) TestMetricFilter( } compiled := compileFilterPattern(filterPattern) + fieldRefs := patternFieldRefs(filterPattern) matches := make([]MetricFilterMatchRecord, 0, len(logEventMessages)) for i, msg := range logEventMessages { - if compiled.matches(msg) { - matches = append(matches, MetricFilterMatchRecord{ - EventMessage: msg, - EventNumber: int64(i + 1), - ExtractedValues: map[string]string{}, - }) + if !compiled.matches(msg) { + continue } + + extracted := make(map[string]string, len(fieldRefs)) + for _, ref := range fieldRefs { + if val, ok := compiled.extractString(msg, ref); ok { + extracted[ref] = val + } + } + + matches = append(matches, MetricFilterMatchRecord{ + EventMessage: msg, + EventNumber: int64(i + 1), + ExtractedValues: extracted, + }) } return matches, nil @@ -3697,18 +3716,16 @@ func (b *InMemoryBackend) PutQueryDefinition( id := queryDefinitionID if id == "" { // New entry: enforce the cap. - if len(b.queryDefinitions) >= maxQueryDefinitions { + if b.queryDefinitions.Len() >= maxQueryDefinitions { return "", fmt.Errorf("%w: query definition limit exceeded", ErrValidation) } id = uuid.New().String() - } else { + } else if !b.queryDefinitions.Has(id) { // Update path: the supplied ID must reference an existing definition. - if _, exists := b.queryDefinitions[id]; !exists { - return "", fmt.Errorf( - "%w: query definition %s not found", - ErrQueryDefinitionNotFound, id, - ) - } + return "", fmt.Errorf( + "%w: query definition %s not found", + ErrQueryDefinitionNotFound, id, + ) } qd := &QueryDefinition{ QueryDefinitionID: id, @@ -3717,7 +3734,7 @@ func (b *InMemoryBackend) PutQueryDefinition( LogGroupNames: slices.Clone(logGroupNames), LastModified: time.Now().UnixMilli(), } - b.queryDefinitions[id] = qd + b.queryDefinitions.Put(qd) return id, nil } @@ -3731,8 +3748,8 @@ func (b *InMemoryBackend) DescribeQueryDefinitions( b.mu.RLock("DescribeQueryDefinitions") defer b.mu.RUnlock() - all := make([]QueryDefinition, 0, len(b.queryDefinitions)) - for _, qd := range b.queryDefinitions { + all := make([]QueryDefinition, 0, b.queryDefinitions.Len()) + for _, qd := range b.queryDefinitions.All() { if queryDefinitionNamePrefix != "" && !strings.HasPrefix(qd.Name, queryDefinitionNamePrefix) { continue @@ -3770,14 +3787,13 @@ func (b *InMemoryBackend) DeleteQueryDefinition(queryDefinitionID string) error b.mu.Lock("DeleteQueryDefinition") defer b.mu.Unlock() - if _, ok := b.queryDefinitions[queryDefinitionID]; !ok { + if !b.queryDefinitions.Delete(queryDefinitionID) { return fmt.Errorf( "%w: query definition %s not found", ErrQueryDefinitionNotFound, queryDefinitionID, ) } - delete(b.queryDefinitions, queryDefinitionID) return nil } @@ -3789,7 +3805,7 @@ func (b *InMemoryBackend) AddExportTaskInternal(task ExportTask) { defer b.mu.Unlock() t := task - b.exportTasks[task.TaskID] = &t + b.exportTasks.Put(&t) } // AddImportTaskInternal seeds an ImportTask directly into the store for testing. @@ -3799,7 +3815,7 @@ func (b *InMemoryBackend) AddImportTaskInternal(task ImportTask) { defer b.mu.Unlock() t := task - b.importTasks[task.ImportID] = &t + b.importTasks.Put(&t) } // AddDeliveryInternal seeds a Delivery directly into the store for testing. @@ -3810,7 +3826,7 @@ func (b *InMemoryBackend) AddDeliveryInternal(delivery Delivery) { d := delivery d.Tags = maps.Clone(delivery.Tags) - b.deliveries[delivery.ID] = &d + b.deliveries.Put(&d) } // AddLogAnomalyDetectorInternal seeds a LogAnomalyDetector directly into the store for testing. @@ -3821,7 +3837,7 @@ func (b *InMemoryBackend) AddLogAnomalyDetectorInternal(detector LogAnomalyDetec d := detector d.LogGroupArnList = slices.Clone(detector.LogGroupArnList) - b.logAnomalyDetectors[detector.AnomalyDetectorArn] = &d + b.logAnomalyDetectors.Put(&d) } // AddAnomalyInternal seeds an Anomaly directly into the store for testing. @@ -3830,12 +3846,8 @@ func (b *InMemoryBackend) AddAnomalyInternal(anomaly Anomaly) { b.mu.Lock("AddAnomalyInternal") defer b.mu.Unlock() - if b.anomalies[anomaly.AnomalyDetectorArn] == nil { - b.anomalies[anomaly.AnomalyDetectorArn] = make(map[string]*Anomaly) - } - a := anomaly - b.anomalies[anomaly.AnomalyDetectorArn][anomaly.AnomalyID] = &a + b.anomalies.Put(&a) } // AddScheduledQueryRunInternal seeds a ScheduledQueryRunSummary for testing. @@ -3847,7 +3859,12 @@ func (b *InMemoryBackend) AddScheduledQueryRunInternal( defer b.mu.Unlock() r := run - b.scheduledQueryRuns[scheduledQueryArn] = append(b.scheduledQueryRuns[scheduledQueryArn], &r) + history, ok := b.scheduledQueryRuns.Get(scheduledQueryArn) + if !ok { + history = &scheduledQueryRunHistory{Arn: scheduledQueryArn} + } + history.Runs = append(history.Runs, &r) + b.scheduledQueryRuns.Put(history) } // SetQueryStatusInternal sets the status of an existing query for testing. @@ -3856,7 +3873,7 @@ func (b *InMemoryBackend) SetQueryStatusInternal(queryID string, status QuerySta b.mu.Lock("SetQueryStatusInternal") defer b.mu.Unlock() - if sq, ok := b.queries[queryID]; ok { + if sq, ok := b.queries.Get(queryID); ok { sq.info.Status = status } } @@ -3898,7 +3915,7 @@ func (b *InMemoryBackend) GetLogAnomalyDetector(detectorArn string) (*LogAnomaly b.mu.RLock("GetLogAnomalyDetector") defer b.mu.RUnlock() - d, ok := b.logAnomalyDetectors[detectorArn] + d, ok := b.logAnomalyDetectors.Get(detectorArn) if !ok { return nil, fmt.Errorf( "%w: anomaly detector %s not found", @@ -3921,7 +3938,7 @@ func (b *InMemoryBackend) GetScheduledQuery(scheduledQueryArn string) (*Schedule b.mu.RLock("GetScheduledQuery") defer b.mu.RUnlock() - sq, ok := b.scheduledQueries[scheduledQueryArn] + sq, ok := b.scheduledQueries.Get(scheduledQueryArn) if !ok { return nil, fmt.Errorf( "%w: scheduled query %s not found", @@ -3960,7 +3977,7 @@ func (b *InMemoryBackend) GetLogGroupFields( b.mu.RLock("GetLogGroupFields") defer b.mu.RUnlock() - if _, exists := b.groupsStore(region)[logGroupName]; !exists { + if !b.groupHas(region, logGroupName) { return nil, fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, logGroupName) } @@ -4000,20 +4017,20 @@ func (b *InMemoryBackend) GetLogRecord( b.mu.RLock("GetLogRecord") defer b.mu.RUnlock() - if _, exists := b.groupsStore(region)[groupName]; !exists { + if !b.groupHas(region, groupName) { return nil, fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, groupName) } - if _, exists := b.streamsStore(region)[groupName][streamName]; !exists { + stream, exists := b.streamGet(region, groupName, streamName) + if !exists { return nil, fmt.Errorf("%w: Log stream %s not found", ErrLogStreamNotFound, streamName) } - evts := b.eventsStore(region)[groupName][streamName] - if idx >= len(evts) { + if idx >= len(stream.events) { return nil, fmt.Errorf("%w: log record index %d out of range", ErrValidation, idx) } - ev := evts[idx] + ev := stream.events[idx] result := map[string]string{ keyMessageField: ev.Message, keyTimestamp: strconv.FormatInt(ev.Timestamp, 10), @@ -4035,7 +4052,7 @@ func (b *InMemoryBackend) ListAnomalies( defer b.mu.RUnlock() if anomalyDetectorArn != "" { - if _, ok := b.logAnomalyDetectors[anomalyDetectorArn]; !ok { + if !b.logAnomalyDetectors.Has(anomalyDetectorArn) { return nil, "", fmt.Errorf( "%w: anomaly detector %s not found", ErrLogAnomalyDetectorNotFound, @@ -4046,14 +4063,12 @@ func (b *InMemoryBackend) ListAnomalies( var all []Anomaly if anomalyDetectorArn != "" { - for _, a := range b.anomalies[anomalyDetectorArn] { + for _, a := range b.anomalyByDetector.Get(anomalyDetectorArn) { all = append(all, *a) } } else { - for _, detectorAnomalies := range b.anomalies { - for _, a := range detectorAnomalies { - all = append(all, *a) - } + for _, a := range b.anomalies.All() { + all = append(all, *a) } } @@ -4088,7 +4103,7 @@ func (b *InMemoryBackend) ListLogGroupsForQuery(queryID string) ([]string, error b.mu.RLock("ListLogGroupsForQuery") defer b.mu.RUnlock() - sq, ok := b.queries[queryID] + sq, ok := b.queries.Get(queryID) if !ok { return nil, fmt.Errorf("%w: query %s not found", ErrQueryNotFound, queryID) } @@ -4112,7 +4127,7 @@ func (b *InMemoryBackend) GetScheduledQueryHistory( b.mu.RLock("GetScheduledQueryHistory") defer b.mu.RUnlock() - if _, ok := b.scheduledQueries[scheduledQueryArn]; !ok { + if !b.scheduledQueries.Has(scheduledQueryArn) { return nil, "", fmt.Errorf( "%w: scheduled query %s not found", ErrScheduledQueryNotFound, @@ -4120,7 +4135,10 @@ func (b *InMemoryBackend) GetScheduledQueryHistory( ) } - runs := b.scheduledQueryRuns[scheduledQueryArn] + var runs []*ScheduledQueryRunSummary + if history, ok := b.scheduledQueryRuns.Get(scheduledQueryArn); ok { + runs = history.Runs + } all := make([]ScheduledQueryRunSummary, 0, len(runs)) for _, r := range runs { all = append(all, *r) @@ -4163,7 +4181,7 @@ func (b *InMemoryBackend) UpdateAnomaly( b.mu.Lock("UpdateAnomaly") defer b.mu.Unlock() - if _, ok := b.logAnomalyDetectors[anomalyDetectorArn]; !ok { + if !b.logAnomalyDetectors.Has(anomalyDetectorArn) { return fmt.Errorf( "%w: anomaly detector %s not found", ErrLogAnomalyDetectorNotFound, @@ -4171,17 +4189,7 @@ func (b *InMemoryBackend) UpdateAnomaly( ) } - detectorAnomalies, ok := b.anomalies[anomalyDetectorArn] - if !ok { - return fmt.Errorf( - "%w: anomaly %s not found in detector %s", - ErrLogAnomalyDetectorNotFound, - anomalyID, - anomalyDetectorArn, - ) - } - - anomaly, ok := detectorAnomalies[anomalyID] + anomaly, ok := b.anomalies.Get(anomalyTableKey(anomalyDetectorArn, anomalyID)) if !ok { return fmt.Errorf( "%w: anomaly %s not found in detector %s", diff --git a/services/cloudwatchlogs/backend_completeness.go b/services/cloudwatchlogs/backend_completeness.go index c210baef2..e5532459f 100644 --- a/services/cloudwatchlogs/backend_completeness.go +++ b/services/cloudwatchlogs/backend_completeness.go @@ -43,7 +43,8 @@ func (b *InMemoryBackend) PutResourcePolicy(policyName, policyDocument string) ( PolicyDocument: policyDocument, LastUpdated: time.Now().UTC(), } - b.resourcePolicies[policyName] = p + stored := p + b.resourcePolicies.Put(&stored) return &p, nil } @@ -53,9 +54,9 @@ func (b *InMemoryBackend) DescribeResourcePolicies() []ResourcePolicy { b.mu.RLock("DescribeResourcePolicies") defer b.mu.RUnlock() - out := make([]ResourcePolicy, 0, len(b.resourcePolicies)) - for _, p := range b.resourcePolicies { - out = append(out, p) + out := make([]ResourcePolicy, 0, b.resourcePolicies.Len()) + for _, p := range b.resourcePolicies.All() { + out = append(out, *p) } sort.Slice(out, func(i, j int) bool { return out[i].PolicyName < out[j].PolicyName }) @@ -68,12 +69,10 @@ func (b *InMemoryBackend) DeleteResourcePolicy(policyName string) error { b.mu.Lock("DeleteResourcePolicy") defer b.mu.Unlock() - if _, ok := b.resourcePolicies[policyName]; !ok { + if !b.resourcePolicies.Delete(policyName) { return fmt.Errorf("%w: resource policy %q not found", ErrResourcePolicyNotFound, policyName) } - delete(b.resourcePolicies, policyName) - return nil } @@ -102,16 +101,16 @@ func (b *InMemoryBackend) PutDeliveryDestination( b.mu.Lock("PutDeliveryDestination") defer b.mu.Unlock() - existing, exists := b.deliveryDestinations[name] + existing, exists := b.deliveryDestinations.Get(name) if exists { existing.TargetArn = targetArn existing.OutputFormat = outputFormat if tags != nil { existing.Tags = tags } - b.deliveryDestinations[name] = existing + cp := *existing - return &existing, nil + return &cp, nil } dest := DeliveryDestination{ @@ -122,7 +121,8 @@ func (b *InMemoryBackend) PutDeliveryDestination( Tags: tags, CreatedAt: time.Now().UTC(), } - b.deliveryDestinations[name] = dest + stored := dest + b.deliveryDestinations.Put(&stored) return &dest, nil } @@ -132,12 +132,13 @@ func (b *InMemoryBackend) GetDeliveryDestination(name string) (*DeliveryDestinat b.mu.RLock("GetDeliveryDestination") defer b.mu.RUnlock() - dest, ok := b.deliveryDestinations[name] + dest, ok := b.deliveryDestinations.Get(name) if !ok { return nil, fmt.Errorf("%w: delivery destination %q not found", ErrDeliveryDestinationNotFound, name) } + cp := *dest - return &dest, nil + return &cp, nil } // DescribeDeliveryDestinations returns all delivery destinations sorted by name. @@ -145,9 +146,9 @@ func (b *InMemoryBackend) DescribeDeliveryDestinations() []DeliveryDestination { b.mu.RLock("DescribeDeliveryDestinations") defer b.mu.RUnlock() - out := make([]DeliveryDestination, 0, len(b.deliveryDestinations)) - for _, d := range b.deliveryDestinations { - out = append(out, d) + out := make([]DeliveryDestination, 0, b.deliveryDestinations.Len()) + for _, d := range b.deliveryDestinations.All() { + out = append(out, *d) } sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) @@ -160,12 +161,10 @@ func (b *InMemoryBackend) DeleteDeliveryDestination(name string) error { b.mu.Lock("DeleteDeliveryDestination") defer b.mu.Unlock() - if _, ok := b.deliveryDestinations[name]; !ok { + if !b.deliveryDestinations.Delete(name) { return fmt.Errorf("%w: delivery destination %q not found", ErrDeliveryDestinationNotFound, name) } - delete(b.deliveryDestinations, name) - return nil } @@ -174,13 +173,12 @@ func (b *InMemoryBackend) PutDeliveryDestinationPolicy(name, policy string) erro b.mu.Lock("PutDeliveryDestinationPolicy") defer b.mu.Unlock() - dest, ok := b.deliveryDestinations[name] + dest, ok := b.deliveryDestinations.Get(name) if !ok { return fmt.Errorf("%w: delivery destination %q not found", ErrDeliveryDestinationNotFound, name) } dest.Policy = policy - b.deliveryDestinations[name] = dest return nil } @@ -190,7 +188,7 @@ func (b *InMemoryBackend) GetDeliveryDestinationPolicy(name string) (string, err b.mu.RLock("GetDeliveryDestinationPolicy") defer b.mu.RUnlock() - dest, ok := b.deliveryDestinations[name] + dest, ok := b.deliveryDestinations.Get(name) if !ok { return "", fmt.Errorf("%w: delivery destination %q not found", ErrDeliveryDestinationNotFound, name) } @@ -203,13 +201,12 @@ func (b *InMemoryBackend) DeleteDeliveryDestinationPolicy(name string) error { b.mu.Lock("DeleteDeliveryDestinationPolicy") defer b.mu.Unlock() - dest, ok := b.deliveryDestinations[name] + dest, ok := b.deliveryDestinations.Get(name) if !ok { return fmt.Errorf("%w: delivery destination %q not found", ErrDeliveryDestinationNotFound, name) } dest.Policy = "" - b.deliveryDestinations[name] = dest return nil } @@ -239,16 +236,16 @@ func (b *InMemoryBackend) PutDeliverySource( b.mu.Lock("PutDeliverySource") defer b.mu.Unlock() - existing, exists := b.deliverySources[name] + existing, exists := b.deliverySources.Get(name) if exists { existing.LogType = logType existing.ResourceArns = resourceArns if tags != nil { existing.Tags = tags } - b.deliverySources[name] = existing + cp := *existing - return &existing, nil + return &cp, nil } src := DeliverySource{ @@ -259,7 +256,8 @@ func (b *InMemoryBackend) PutDeliverySource( Tags: tags, CreatedAt: time.Now().UTC(), } - b.deliverySources[name] = src + stored := src + b.deliverySources.Put(&stored) return &src, nil } @@ -269,12 +267,13 @@ func (b *InMemoryBackend) GetDeliverySource(name string) (*DeliverySource, error b.mu.RLock("GetDeliverySource") defer b.mu.RUnlock() - src, ok := b.deliverySources[name] + src, ok := b.deliverySources.Get(name) if !ok { return nil, fmt.Errorf("%w: delivery source %q not found", ErrDeliverySourceNotFound, name) } + cp := *src - return &src, nil + return &cp, nil } // DescribeDeliverySources returns all delivery sources sorted by name. @@ -282,9 +281,9 @@ func (b *InMemoryBackend) DescribeDeliverySources() []DeliverySource { b.mu.RLock("DescribeDeliverySources") defer b.mu.RUnlock() - out := make([]DeliverySource, 0, len(b.deliverySources)) - for _, s := range b.deliverySources { - out = append(out, s) + out := make([]DeliverySource, 0, b.deliverySources.Len()) + for _, s := range b.deliverySources.All() { + out = append(out, *s) } sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) @@ -297,12 +296,10 @@ func (b *InMemoryBackend) DeleteDeliverySource(name string) error { b.mu.Lock("DeleteDeliverySource") defer b.mu.Unlock() - if _, ok := b.deliverySources[name]; !ok { + if !b.deliverySources.Delete(name) { return fmt.Errorf("%w: delivery source %q not found", ErrDeliverySourceNotFound, name) } - delete(b.deliverySources, name) - return nil } @@ -327,13 +324,13 @@ func (b *InMemoryBackend) PutDestination(name, targetArn, roleArn string) (*CWLD b.mu.Lock("PutDestination") defer b.mu.Unlock() - existing, exists := b.destinations[name] + existing, exists := b.destinations.Get(name) if exists { existing.TargetArn = targetArn existing.RoleArn = roleArn - b.destinations[name] = existing + cp := *existing - return &existing, nil + return &cp, nil } dest := CWLDestination{ @@ -343,7 +340,8 @@ func (b *InMemoryBackend) PutDestination(name, targetArn, roleArn string) (*CWLD Arn: "arn:aws:logs:" + b.region + ":" + b.accountID + ":destination:" + name, CreatedAt: time.Now().UTC(), } - b.destinations[name] = dest + stored := dest + b.destinations.Put(&stored) return &dest, nil } @@ -353,13 +351,12 @@ func (b *InMemoryBackend) PutDestinationPolicy(name, policy string) error { b.mu.Lock("PutDestinationPolicy") defer b.mu.Unlock() - dest, ok := b.destinations[name] + dest, ok := b.destinations.Get(name) if !ok { return fmt.Errorf("%w: destination %q not found", ErrDestinationNotFound, name) } dest.AccessPolicy = policy - b.destinations[name] = dest return nil } @@ -369,11 +366,11 @@ func (b *InMemoryBackend) DescribeDestinations(namePrefix string) []CWLDestinati b.mu.RLock("DescribeDestinations") defer b.mu.RUnlock() - out := make([]CWLDestination, 0, len(b.destinations)) + out := make([]CWLDestination, 0, b.destinations.Len()) - for _, d := range b.destinations { + for _, d := range b.destinations.All() { if namePrefix == "" || strings.HasPrefix(d.DestinationName, namePrefix) { - out = append(out, d) + out = append(out, *d) } } @@ -387,12 +384,10 @@ func (b *InMemoryBackend) DeleteDestination(name string) error { b.mu.Lock("DeleteDestination") defer b.mu.Unlock() - if _, ok := b.destinations[name]; !ok { + if !b.destinations.Delete(name) { return fmt.Errorf("%w: destination %q not found", ErrDestinationNotFound, name) } - delete(b.destinations, name) - return nil } @@ -419,7 +414,8 @@ func (b *InMemoryBackend) PutIndexPolicy(logGroupIdentifier, policyDocument stri PolicyDocument: policyDocument, LastUpdated: time.Now().UTC(), } - b.indexPolicies[logGroupIdentifier] = p + stored := p + b.indexPolicies.Put(&stored) return &p, nil } @@ -429,9 +425,9 @@ func (b *InMemoryBackend) DescribeIndexPolicies() []IndexPolicy { b.mu.RLock("DescribeIndexPolicies") defer b.mu.RUnlock() - out := make([]IndexPolicy, 0, len(b.indexPolicies)) - for _, p := range b.indexPolicies { - out = append(out, p) + out := make([]IndexPolicy, 0, b.indexPolicies.Len()) + for _, p := range b.indexPolicies.All() { + out = append(out, *p) } sort.Slice(out, func(i, j int) bool { return out[i].LogGroupIdentifier < out[j].LogGroupIdentifier }) @@ -444,12 +440,10 @@ func (b *InMemoryBackend) DeleteIndexPolicy(logGroupIdentifier string) error { b.mu.Lock("DeleteIndexPolicy") defer b.mu.Unlock() - if _, ok := b.indexPolicies[logGroupIdentifier]; !ok { + if !b.indexPolicies.Delete(logGroupIdentifier) { return fmt.Errorf("%w: index policy for %q not found", ErrIndexPolicyNotFound, logGroupIdentifier) } - delete(b.indexPolicies, logGroupIdentifier) - return nil } @@ -471,11 +465,11 @@ func (b *InMemoryBackend) PutTransformer(logGroupIdentifier string, processors [ b.mu.Lock("PutTransformer") defer b.mu.Unlock() - b.transformers[logGroupIdentifier] = Transformer{ + b.transformers.Put(&Transformer{ LogGroupIdentifier: logGroupIdentifier, Processors: processors, CreatedAt: time.Now().UTC(), - } + }) return nil } @@ -485,12 +479,13 @@ func (b *InMemoryBackend) GetTransformer(logGroupIdentifier string) (*Transforme b.mu.RLock("GetTransformer") defer b.mu.RUnlock() - t, ok := b.transformers[logGroupIdentifier] + t, ok := b.transformers.Get(logGroupIdentifier) if !ok { return nil, fmt.Errorf("%w: transformer for %q not found", ErrTransformerNotFound, logGroupIdentifier) } + cp := *t - return &t, nil + return &cp, nil } // DeleteTransformer removes the transformer for a log group. @@ -498,12 +493,10 @@ func (b *InMemoryBackend) DeleteTransformer(logGroupIdentifier string) error { b.mu.Lock("DeleteTransformer") defer b.mu.Unlock() - if _, ok := b.transformers[logGroupIdentifier]; !ok { + if !b.transformers.Delete(logGroupIdentifier) { return fmt.Errorf("%w: transformer for %q not found", ErrTransformerNotFound, logGroupIdentifier) } - delete(b.transformers, logGroupIdentifier) - return nil } @@ -532,7 +525,8 @@ func (b *InMemoryBackend) PutIntegration(name, integrationType string) (*CWLInte Status: completenessStatusActive, CreatedAt: time.Now().UTC(), } - b.integrations[name] = ig + stored := ig + b.integrations.Put(&stored) return &ig, nil } @@ -542,12 +536,13 @@ func (b *InMemoryBackend) GetIntegration(name string) (*CWLIntegration, error) { b.mu.RLock("GetIntegration") defer b.mu.RUnlock() - ig, ok := b.integrations[name] + ig, ok := b.integrations.Get(name) if !ok { return nil, fmt.Errorf("%w: integration %q not found", ErrIntegrationNotFound, name) } + cp := *ig - return &ig, nil + return &cp, nil } // ListIntegrations returns all integrations sorted by name. @@ -555,9 +550,9 @@ func (b *InMemoryBackend) ListIntegrations() []CWLIntegration { b.mu.RLock("ListIntegrations") defer b.mu.RUnlock() - out := make([]CWLIntegration, 0, len(b.integrations)) - for _, ig := range b.integrations { - out = append(out, ig) + out := make([]CWLIntegration, 0, b.integrations.Len()) + for _, ig := range b.integrations.All() { + out = append(out, *ig) } sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) @@ -570,12 +565,10 @@ func (b *InMemoryBackend) DeleteIntegration(name string) error { b.mu.Lock("DeleteIntegration") defer b.mu.Unlock() - if _, ok := b.integrations[name]; !ok { + if !b.integrations.Delete(name) { return fmt.Errorf("%w: integration %q not found", ErrIntegrationNotFound, name) } - delete(b.integrations, name) - return nil } @@ -590,7 +583,10 @@ func (b *InMemoryBackend) SetLogGroupDeletionProtection(logGroupIdentifier strin b.mu.Lock("SetLogGroupDeletionProtection") defer b.mu.Unlock() - b.deletionProtected[logGroupIdentifier] = protected + b.deletionProtected.Put(&deletionProtectionEntry{ + LogGroupIdentifier: logGroupIdentifier, + Protected: protected, + }) return nil } @@ -600,7 +596,9 @@ func (b *InMemoryBackend) IsLogGroupDeletionProtected(logGroupIdentifier string) b.mu.RLock("IsLogGroupDeletionProtected") defer b.mu.RUnlock() - return b.deletionProtected[logGroupIdentifier] + entry, ok := b.deletionProtected.Get(logGroupIdentifier) + + return ok && entry.Protected } // ---- UpdateDeliveryConfiguration ---- @@ -610,7 +608,7 @@ func (b *InMemoryBackend) UpdateDeliveryConfiguration(id, fieldDelimiter string, b.mu.Lock("UpdateDeliveryConfiguration") defer b.mu.Unlock() - delivery, ok := b.deliveries[id] + delivery, ok := b.deliveries.Get(id) if !ok { return fmt.Errorf("%w: delivery %q not found", ErrDeliveryNotFound, id) } @@ -623,8 +621,6 @@ func (b *InMemoryBackend) UpdateDeliveryConfiguration(id, fieldDelimiter string, delivery.RecordFields = recordFields } - b.deliveries[id] = delivery - return nil } @@ -648,7 +644,7 @@ func (b *InMemoryBackend) DiscoverLogFields( b.mu.RLock("DiscoverLogFields") defer b.mu.RUnlock() - if _, exists := b.groupsStore(region)[logGroupName]; !exists { + if !b.groupHas(region, logGroupName) { return nil, fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, logGroupName) } @@ -659,8 +655,8 @@ func (b *InMemoryBackend) DiscoverLogFields( keyLogStream: {}, } - for _, streams := range b.eventsStore(region)[logGroupName] { - for _, ev := range streams { + for _, stream := range b.streamsInGroup(region, logGroupName) { + for _, ev := range stream.events { for _, name := range jsonMessageFields(ev.Message) { fieldSet[name] = struct{}{} } @@ -719,18 +715,17 @@ func (b *InMemoryBackend) ListAggregateLogGroupSummaries( b.mu.RLock("ListAggregateLogGroupSummaries") defer b.mu.RUnlock() - groups := b.groupsStore(region) - events := b.eventsStore(region) + groups := b.groupsInRegion(region) summaries := make([]AggregateLogGroupSummary, 0, len(groups)) - for name, group := range groups { + for _, group := range groups { var count int64 - for _, streams := range events[name] { - count += int64(len(streams)) + for _, stream := range b.streamsInGroup(region, group.LogGroupName) { + count += int64(len(stream.events)) } summaries = append(summaries, AggregateLogGroupSummary{ - LogGroupName: name, + LogGroupName: group.LogGroupName, LogGroupArn: group.Arn, LogGroupClass: group.LogGroupClass, StoredBytes: group.StoredBytes, @@ -765,10 +760,9 @@ func (b *InMemoryBackend) ValidateLiveTailLogGroups( b.mu.RLock("ValidateLiveTailLogGroups") defer b.mu.RUnlock() - groups := b.groupsStore(region) for _, id := range logGroupIdentifiers { name := normalizeLogGroupIdentifier(id) - if _, exists := groups[name]; !exists { + if !b.groupHas(region, name) { return fmt.Errorf("%w: Log group %s not found", ErrLogGroupNotFound, name) } } diff --git a/services/cloudwatchlogs/backend_test.go b/services/cloudwatchlogs/backend_test.go index f2922f946..122515665 100644 --- a/services/cloudwatchlogs/backend_test.go +++ b/services/cloudwatchlogs/backend_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "fmt" + "strconv" "strings" "sync" "testing" @@ -4821,6 +4822,182 @@ func TestCloudWatchLogsBackend_MetricFilterEmission_NoEmitterNoPanic(t *testing. require.NoError(t, err) } +// ---- Metric filter field extraction (MetricValue "$name" / "$.path" references) ---- +// +// A metric filter's MetricValue may be a literal number (published as-is for every +// matched event) or a "$"-prefixed field reference that must be extracted from each +// individual matched log event: "$size" for a named field in a space-delimited pattern +// ("[ip, level, size]"), "$.bytes" for a JSON selector pattern. Real CloudWatch Logs +// silently skips publishing a data point for a matched event whose referenced field is +// absent or non-numeric -- it does not fabricate a value (DefaultValue is documented for +// periods with zero *matching* events, not failed per-event extraction), so these cases +// assert zero emissions rather than a fallback constant. + +func TestCloudWatchLogsBackend_MetricFilterEmission_FieldExtraction(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + filterPattern string + metricValue string + message string + wantEmitted []float64 // nil means "no emission for this event" + }{ + { + name: "json_field_extracted_per_event", + filterPattern: `{ $.level = "ERROR" }`, + metricValue: "$.bytes", + message: `{"level":"ERROR","bytes":512}`, + wantEmitted: []float64{512}, + }, + { + name: "space_field_extracted_per_event", + filterPattern: "[ip, level, bytes]", + metricValue: "$bytes", + message: "1.2.3.4 ERROR 256", + wantEmitted: []float64{256}, + }, + { + name: "missing_json_field_skips_emission", + filterPattern: `{ $.level = "ERROR" }`, + metricValue: "$.bytes", + message: `{"level":"ERROR"}`, + wantEmitted: nil, + }, + { + name: "non_numeric_field_skips_emission", + filterPattern: `{ $.level = "ERROR" }`, + metricValue: "$.bytes", + message: `{"level":"ERROR","bytes":"lots"}`, + wantEmitted: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var mu sync.Mutex + var emitted []float64 + + emitter := cloudwatchlogs.MetricEmitterFunc( + func(_, _ string, value float64, _ string) error { + mu.Lock() + emitted = append(emitted, value) + mu.Unlock() + + return nil + }, + ) + + b := cloudwatchlogs.NewInMemoryBackend() + b.SetMetricEmitter(emitter) + + _, err := b.CreateLogGroup(context.Background(), "grp", "", "") + require.NoError(t, err) + _, err = b.CreateLogStream(context.Background(), "grp", "stream") + require.NoError(t, err) + + err = b.PutMetricFilter( + context.Background(), + "grp", + "mf", + tt.filterPattern, + []cloudwatchlogs.MetricTransformation{ + {MetricNamespace: "MyApp", MetricName: "Bytes", MetricValue: tt.metricValue}, + }, + ) + require.NoError(t, err) + + _, err = b.PutLogEvents( + context.Background(), + "grp", + "stream", + "", + []cloudwatchlogs.InputLogEvent{ + {Message: tt.message, Timestamp: time.Now().UnixMilli()}, + }, + ) + require.NoError(t, err) + + mu.Lock() + defer mu.Unlock() + + if tt.wantEmitted == nil { + assert.Empty(t, emitted) + + return + } + + require.Len(t, emitted, len(tt.wantEmitted)) + for i, want := range tt.wantEmitted { + assert.InDelta(t, want, emitted[i], 0.001) + } + }) + } +} + +func TestCloudWatchLogsBackend_TestMetricFilter_ExtractedValues(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + pattern string + messages []string + wantValues []map[string]string // one entry per expected match, in order + }{ + { + name: "json_pattern_extracts_referenced_selector", + pattern: `{ $.level = "ERROR" }`, + messages: []string{ + `{"level":"ERROR","bytes":512}`, + `{"level":"INFO","bytes":10}`, + }, + wantValues: []map[string]string{ + {"$.level": "ERROR"}, + }, + }, + { + name: "space_pattern_extracts_named_fields", + pattern: "[ip, level, bytes]", + messages: []string{ + "1.2.3.4 ERROR 256", + "5.6.7.8 INFO 10", + }, + wantValues: []map[string]string{ + {"$ip": "1.2.3.4", "$level": "ERROR", "$bytes": "256"}, + {"$ip": "5.6.7.8", "$level": "INFO", "$bytes": "10"}, + }, + }, + { + name: "plain_text_pattern_has_no_addressable_fields", + pattern: "ERROR", + messages: []string{ + "an ERROR occurred", + }, + wantValues: []map[string]string{ + {}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := cloudwatchlogs.NewInMemoryBackend() + + matches, err := b.TestMetricFilter(tt.pattern, tt.messages) + require.NoError(t, err) + require.Len(t, matches, len(tt.wantValues)) + + for i, want := range tt.wantValues { + assert.Equal(t, want, matches[i].ExtractedValues) + } + }) + } +} + // ---- Item 1: LogGroupClass ---- func TestCloudWatchLogsBackend_CreateLogGroup_WithClass(t *testing.T) { @@ -5015,7 +5192,7 @@ func TestCloudWatchLogsBackend_PutLogEvents_RejectedLogEventsInfo(t *testing.T) if tt.wantTooOld || tt.wantTooNew || tt.wantExpired { require.NotNil(t, result.RejectedLogEventsInfo) if tt.wantTooOld { - assert.NotNil(t, result.RejectedLogEventsInfo.TooOldLogEventStartIndex) + assert.NotNil(t, result.RejectedLogEventsInfo.TooOldLogEventEndIndex) } if tt.wantTooNew { assert.NotNil(t, result.RejectedLogEventsInfo.TooNewLogEventStartIndex) @@ -5030,45 +5207,50 @@ func TestCloudWatchLogsBackend_PutLogEvents_RejectedLogEventsInfo(t *testing.T) } } -// ---- Item 3: SequenceToken strictness ---- - -func TestCloudWatchLogsBackend_PutLogEvents_SequenceToken(t *testing.T) { +// ---- Item 3: SequenceToken is ignored (matches current AWS behavior) ---- +// +// aws-sdk-go-v2 cloudwatchlogs.PutLogEvents doc: "The sequence token is now +// ignored in PutLogEvents actions. PutLogEvents actions are always accepted and +// never return InvalidSequenceTokenException or DataAlreadyAcceptedException +// even if the sequence token is not valid." So every case below must succeed +// regardless of whether the supplied token matches the stream's actual length, +// and NextSequenceToken must reflect the real post-append event count rather +// than echoing (or validating against) the caller-supplied token. + +func TestCloudWatchLogsBackend_PutLogEvents_SequenceTokenIgnored(t *testing.T) { t.Parallel() now := time.Now().UnixMilli() tests := []struct { - wantErr error name string sequenceToken string setupEvents int }{ { - name: "no_token_accepted", + name: "no_token", setupEvents: 0, sequenceToken: "", }, { - name: "correct_token", + name: "matching_token", setupEvents: 2, sequenceToken: "2", }, { - name: "wrong_token", + name: "stale_token_still_accepted", setupEvents: 2, sequenceToken: "99", - wantErr: cloudwatchlogs.ErrInvalidSequenceToken, }, { - name: "token_on_empty_stream_correct", + name: "token_on_empty_stream_matching", setupEvents: 0, sequenceToken: "0", }, { - name: "token_on_empty_stream_wrong", + name: "token_on_empty_stream_wrong_still_accepted", setupEvents: 0, sequenceToken: "5", - wantErr: cloudwatchlogs.ErrInvalidSequenceToken, }, } @@ -5095,7 +5277,7 @@ func TestCloudWatchLogsBackend_PutLogEvents_SequenceToken(t *testing.T) { require.NoError(t, err) } - _, err = b.PutLogEvents( + result, err := b.PutLogEvents( context.Background(), "g", "s", @@ -5105,13 +5287,9 @@ func TestCloudWatchLogsBackend_PutLogEvents_SequenceToken(t *testing.T) { }, ) - if tt.wantErr != nil { - require.ErrorIs(t, err, tt.wantErr) - - return - } - require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, strconv.Itoa(tt.setupEvents+1), result.NextSequenceToken) }) } } diff --git a/services/cloudwatchlogs/export.go b/services/cloudwatchlogs/export.go index f4e6fb2a6..96cd88936 100644 --- a/services/cloudwatchlogs/export.go +++ b/services/cloudwatchlogs/export.go @@ -78,11 +78,13 @@ func (b *InMemoryBackend) collectExportStreams(task *ExportTask) []exportStream b.mu.RLock("collectExportStreams") defer b.mu.RUnlock() - groupEvents := b.eventsStore(b.region)[task.LogGroupName] - names := make([]string, 0, len(groupEvents)) - for name := range groupEvents { - if task.LogStreamNamePrefix == "" || strings.HasPrefix(name, task.LogStreamNamePrefix) { - names = append(names, name) + groupStreams := b.streamsInGroup(b.region, task.LogGroupName) + streamsByName := make(map[string]*LogStream, len(groupStreams)) + names := make([]string, 0, len(groupStreams)) + for _, s := range groupStreams { + if task.LogStreamNamePrefix == "" || strings.HasPrefix(s.LogStreamName, task.LogStreamNamePrefix) { + names = append(names, s.LogStreamName) + streamsByName[s.LogStreamName] = s } } @@ -90,7 +92,7 @@ func (b *InMemoryBackend) collectExportStreams(task *ExportTask) []exportStream out := make([]exportStream, 0, len(names)) for _, name := range names { - if evts := exportWindowEvents(groupEvents[name], task.From, task.To); len(evts) > 0 { + if evts := exportWindowEvents(streamsByName[name].events, task.From, task.To); len(evts) > 0 { out = append(out, exportStream{name: name, events: evts}) } } diff --git a/services/cloudwatchlogs/filter_pattern_json.go b/services/cloudwatchlogs/filter_pattern_json.go index 208db7d99..ea8722792 100644 --- a/services/cloudwatchlogs/filter_pattern_json.go +++ b/services/cloudwatchlogs/filter_pattern_json.go @@ -34,6 +34,51 @@ func compileJSONFilterPattern(pattern string) func(string) bool { } } +// compileJSONFilterPatternExtract returns a field extractor for a JSON selector pattern. +// fieldRef is a selector path with its leading "$" already stripped by the caller (e.g. +// ".bytes" for a MetricValue of "$.bytes"); parseSelectorPath tolerates the remaining +// leading "." itself. Returns the leaf's string form, or false if message isn't valid +// JSON, the path resolves to nothing, or the leaf is a non-scalar (array/object). +func compileJSONFilterPatternExtract(_ string) func(message, fieldRef string) (string, bool) { + return func(message, fieldRef string) (string, bool) { + var root any + if err := json.Unmarshal([]byte(strings.TrimSpace(message)), &root); err != nil { + return "", false + } + + leaves := resolveJSONSelector(root, parseSelectorPath(fieldRef)) + if len(leaves) == 0 { + return "", false + } + + switch leaves[0].(type) { + case string, float64, bool: + return jsonLeafString(leaves[0]), true + default: + return "", false + } + } +} + +// jsonPatternSelectors returns the distinct "$.path" selector tokens referenced anywhere +// in a JSON selector pattern, in first-seen order. +func jsonPatternSelectors(pattern string) []string { + body := strings.TrimSpace(pattern) + body = strings.TrimPrefix(body, "{") + body = strings.TrimSuffix(body, "}") + + seen := make(map[string]bool) + var out []string + for _, t := range lexJSONPattern(body) { + if t.kind == jtSelector && !seen[t.text] { + seen[t.text] = true + out = append(out, t.text) + } + } + + return out +} + // jtKind enumerates JSON-pattern token kinds. type jtKind int diff --git a/services/cloudwatchlogs/filter_pattern_space.go b/services/cloudwatchlogs/filter_pattern_space.go index be694b19a..16abdf5d0 100644 --- a/services/cloudwatchlogs/filter_pattern_space.go +++ b/services/cloudwatchlogs/filter_pattern_space.go @@ -11,8 +11,12 @@ const spaceEllipsis = "..." // spaceTerm is one positional term in a space-delimited "[...]" pattern. type spaceTerm struct { - op string - value string + op string + value string + // name is the field's bare identifier as written in the pattern (e.g. "size" + // in both "size" and "size>100"), used to resolve MetricValue field + // references such as "$size". Empty for the ellipsis term. + name string num float64 ellipsis bool hasCond bool @@ -50,16 +54,19 @@ func parseSpaceTerms(body string) []spaceTerm { // parseSpaceTerm parses a single space-pattern term. func parseSpaceTerm(raw string) spaceTerm { if raw == spaceEllipsis { - return spaceTerm{op: "", value: "", ellipsis: true, hasCond: false, num: 0, isNum: false} + return spaceTerm{op: "", value: "", name: "", ellipsis: true, hasCond: false, num: 0, isNum: false} } op, idx := findSpaceOp(raw) if idx < 0 { - return spaceTerm{op: "", value: "", ellipsis: false, hasCond: false, num: 0, isNum: false} + name := strings.TrimSpace(raw) + + return spaceTerm{op: "", value: "", name: name, ellipsis: false, hasCond: false, num: 0, isNum: false} } + name := strings.TrimSpace(raw[:idx]) value := strings.Trim(strings.TrimSpace(raw[idx+len(op):]), `"'`) - term := spaceTerm{op: op, value: value, ellipsis: false, hasCond: true, num: 0, isNum: false} + term := spaceTerm{op: op, value: value, name: name, ellipsis: false, hasCond: true, num: 0, isNum: false} if f, err := strconv.ParseFloat(value, 64); err == nil { term.num = f term.isNum = true @@ -219,3 +226,86 @@ func spaceNumericCompare(t spaceTerm, field string) bool { return false } } + +// compileSpaceFilterPatternExtract returns a field extractor for a space-delimited pattern, +// keyed by the field's bare name (fieldRef has its leading "$" already stripped by the +// caller, e.g. "size" for a MetricValue of "$size"). Returns the raw field string, or false +// if no field of that name is defined or the message doesn't have enough fields to align it. +func compileSpaceFilterPatternExtract(pattern string) func(message, fieldRef string) (string, bool) { + body := strings.TrimSpace(pattern) + body = strings.TrimPrefix(body, "[") + body = strings.TrimSuffix(body, "]") + terms := parseSpaceTerms(body) + + return func(message, fieldRef string) (string, bool) { + fields := strings.Fields(message) + + idx, ok := spaceFieldIndex(terms, len(fields), fieldRef) + if !ok { + return "", false + } + + return fields[idx], true + } +} + +// spacePatternFieldNames returns the "$"-prefixed named fields declared in a space-delimited +// pattern (skipping the ellipsis and any unnamed term), in declaration order. +func spacePatternFieldNames(pattern string) []string { + body := strings.TrimSpace(pattern) + body = strings.TrimPrefix(body, "[") + body = strings.TrimSuffix(body, "]") + + var out []string + for _, t := range parseSpaceTerms(body) { + if t.ellipsis || t.name == "" { + continue + } + + out = append(out, "$"+t.name) + } + + return out +} + +// spaceFieldIndex returns the message-field index bound to name, using the same +// ellipsis-alignment rules as matchSpaceTerms (a single leading/trailing "..." absorbs +// any extra fields so the named terms stay anchored to the opposite end). +func spaceFieldIndex(terms []spaceTerm, numFields int, name string) (int, bool) { + switch ellipsisPosition(terms) { + case ellipsisNone: + if numFields != len(terms) { + return 0, false + } + + return namedTermIndex(terms, 0, name) + case ellipsisLead: + rest := terms[1:] + if numFields < len(rest) { + return 0, false + } + + return namedTermIndex(rest, numFields-len(rest), name) + case ellipsisTail: + rest := terms[:len(terms)-1] + if numFields < len(rest) { + return 0, false + } + + return namedTermIndex(rest, 0, name) + default: + return 0, false + } +} + +// namedTermIndex finds name among terms and returns its message-field index, offset by +// the alignment base computed by spaceFieldIndex. +func namedTermIndex(terms []spaceTerm, offset int, name string) (int, bool) { + for i, t := range terms { + if t.name == name { + return offset + i, true + } + } + + return 0, false +} diff --git a/services/cloudwatchlogs/handler.go b/services/cloudwatchlogs/handler.go index d86fead38..fc8674dec 100644 --- a/services/cloudwatchlogs/handler.go +++ b/services/cloudwatchlogs/handler.go @@ -2099,9 +2099,6 @@ func (h *Handler) handleError(ctx context.Context, c *echo.Context, action strin case errors.Is(reqErr, ErrSubscriptionFilterLimitExceed): errType = "LimitExceededException" statusCode = http.StatusBadRequest - case errors.Is(reqErr, ErrInvalidSequenceToken): - errType = "InvalidSequenceTokenException" - statusCode = http.StatusBadRequest case errors.Is(reqErr, ErrOperationAborted): errType = "OperationAbortedException" statusCode = http.StatusBadRequest diff --git a/services/cloudwatchlogs/janitor.go b/services/cloudwatchlogs/janitor.go index cc1c040ac..0cad778cb 100644 --- a/services/cloudwatchlogs/janitor.go +++ b/services/cloudwatchlogs/janitor.go @@ -104,23 +104,21 @@ func (j *Janitor) retentionTargets(now time.Time) []retentionTarget { defer j.Backend.mu.RUnlock() var targets []retentionTarget - for region, regionGroups := range j.Backend.groups { - for groupName, group := range regionGroups { - days := j.Backend.settings.MaxRetentionDays - if group.RetentionInDays != nil && *group.RetentionInDays > 0 { - days = int(*group.RetentionInDays) - } - - if days <= 0 { - continue - } + for _, group := range j.Backend.groups.All() { + days := j.Backend.settings.MaxRetentionDays + if group.RetentionInDays != nil && *group.RetentionInDays > 0 { + days = int(*group.RetentionInDays) + } - targets = append(targets, retentionTarget{ - region: region, - groupName: groupName, - cutoffMs: now.AddDate(0, 0, -days).UnixMilli(), - }) + if days <= 0 { + continue } + + targets = append(targets, retentionTarget{ + region: group.region, + groupName: group.LogGroupName, + cutoffMs: now.AddDate(0, 0, -days).UnixMilli(), + }) } return targets @@ -141,13 +139,14 @@ func (j *Janitor) buildEvictionPlan(region, groupName string, cutoffMs int64) [] j.Backend.mu.RLock("JanitorBuildEvictionPlan") defer j.Backend.mu.RUnlock() - groupEventsMap := j.Backend.events[region][groupName] - if len(groupEventsMap) == 0 { + groupStreams := j.Backend.streamsInGroup(region, groupName) + if len(groupStreams) == 0 { return nil } var plan []streamEvictionPlan - for streamName, evts := range groupEventsMap { + for _, stream := range groupStreams { + evts := stream.events kept := make([]*OutputLogEvent, 0, len(evts)) var evictedBytes int64 var evictedCount int @@ -163,7 +162,7 @@ func (j *Janitor) buildEvictionPlan(region, groupName string, cutoffMs int64) [] continue } plan = append(plan, streamEvictionPlan{ - streamName: streamName, + streamName: stream.LogStreamName, kept: kept, evictedBytes: evictedBytes, evictedCount: evictedCount, @@ -179,26 +178,25 @@ func (j *Janitor) buildEvictionPlan(region, groupName string, cutoffMs int64) [] func (j *Janitor) applyEvictionPlan(region, groupName string, plan []streamEvictionPlan) int { evicted := 0 - regionEvents := j.Backend.events[region] - regionStreams := j.Backend.streams[region] - regionGroups := j.Backend.groups[region] + group, _ := j.Backend.groupGet(region, groupName) for _, entry := range plan { - regionEvents[groupName][entry.streamName] = entry.kept + stream, ok := j.Backend.streamGet(region, groupName, entry.streamName) + if !ok { + continue + } + + stream.events = entry.kept evicted += entry.evictedCount if entry.evictedBytes > 0 { - if stream := regionStreams[groupName][entry.streamName]; stream != nil { - stream.StoredBytes -= entry.evictedBytes - } - if g := regionGroups[groupName]; g != nil { - g.StoredBytes -= entry.evictedBytes + stream.StoredBytes -= entry.evictedBytes + if group != nil { + group.StoredBytes -= entry.evictedBytes } } - if stream := regionStreams[groupName][entry.streamName]; stream != nil { - updateStreamTimestamps(stream, entry.kept) - } + updateStreamTimestamps(stream, entry.kept) } return evicted diff --git a/services/cloudwatchlogs/models.go b/services/cloudwatchlogs/models.go index e3e038cff..e18d3fd59 100644 --- a/services/cloudwatchlogs/models.go +++ b/services/cloudwatchlogs/models.go @@ -8,14 +8,22 @@ const ( // LogGroup represents a CloudWatch Logs log group. type LogGroup struct { - RetentionInDays *int32 `json:"retentionInDays,omitempty"` - LogGroupName string `json:"logGroupName"` - Arn string `json:"arn"` - LogGroupClass string `json:"logGroupClass,omitempty"` - KmsKeyID string `json:"kmsKeyId,omitempty"` - CreationTime int64 `json:"creationTime"` - StoredBytes int64 `json:"storedBytes"` - MetricFilterCount int32 `json:"metricFilterCount"` + RetentionInDays *int32 `json:"retentionInDays,omitempty"` + LogGroupName string `json:"logGroupName"` + Arn string `json:"arn"` + LogGroupClass string `json:"logGroupClass,omitempty"` + KmsKeyID string `json:"kmsKeyId,omitempty"` + // region is the AWS region this group lives under. It is unexported (never + // marshaled, so the wire response shape is unaffected) and exists solely so + // the store.Table[LogGroup] holding every region's groups can derive a + // region-qualified primary key purely from the value itself -- see + // groupTableKey in store_setup.go. Persistence round-trips it through the + // separate logGroupSnapshot DTO (see persistence.go) since an unexported + // field cannot survive Table.Snapshot's json.Marshal on its own. + region string + CreationTime int64 `json:"creationTime"` + StoredBytes int64 `json:"storedBytes"` + MetricFilterCount int32 `json:"metricFilterCount"` } // LogStream represents a CloudWatch Logs log stream. @@ -26,8 +34,17 @@ type LogStream struct { LogStreamName string `json:"logStreamName"` Arn string `json:"arn"` UploadSequenceToken string `json:"uploadSequenceToken"` - CreationTime int64 `json:"creationTime"` - StoredBytes int64 `json:"storedBytes"` + // region and logGroupName are unexported identity metadata (see LogGroup.region) + // letting store.Table[LogStream] key every region+group's streams from the + // value alone. events holds this stream's log events inline (matching the + // sqs/s3 pattern of nesting child records on the parent entity rather than a + // separate table) and is likewise unexported; both round-trip through the + // logStreamSnapshot DTO in persistence.go. + region string + logGroupName string + events []*OutputLogEvent + CreationTime int64 `json:"creationTime"` + StoredBytes int64 `json:"storedBytes"` } // InputLogEvent represents a single log event for PutLogEvents. @@ -104,7 +121,11 @@ type SubscriptionFilter struct { DestinationArn string `json:"destinationArn"` RoleArn string `json:"roleArn,omitempty"` Distribution string `json:"distribution,omitempty"` - CreationTime int64 `json:"creationTime"` + // region is unexported identity metadata (see LogGroup.region) letting + // store.Table[SubscriptionFilter] key every region+group's filters from the + // value alone; it round-trips through the subscriptionFilterSnapshot DTO. + region string + CreationTime int64 `json:"creationTime"` } // subscriptionLogEvent is one event in a subscription filter delivery payload. @@ -232,9 +253,12 @@ type AccountPolicy struct { } // RejectedLogEventsInfo describes log events that were rejected by PutLogEvents. +// Field names/wire keys match aws-sdk-go-v2 types.RejectedLogEventsInfo exactly: +// TooOldLogEventEndIndex (not "...StartIndex") is the exclusive end index of the +// too-old run, mirroring ExpiredLogEventEndIndex's exclusive-end semantics. type RejectedLogEventsInfo struct { TooNewLogEventStartIndex *int32 `json:"tooNewLogEventStartIndex,omitempty"` - TooOldLogEventStartIndex *int32 `json:"tooOldLogEventStartIndex,omitempty"` + TooOldLogEventEndIndex *int32 `json:"tooOldLogEventEndIndex,omitempty"` ExpiredLogEventEndIndex *int32 `json:"expiredLogEventEndIndex,omitempty"` } @@ -256,10 +280,11 @@ type MetricTransformation struct { // MetricFilter represents a CloudWatch Logs metric filter. type MetricFilter struct { - RetentionInDays *int32 `json:"retentionInDays,omitempty"` - FilterPattern string `json:"filterPattern"` - FilterName string `json:"filterName"` - LogGroupName string `json:"logGroupName"` + RetentionInDays *int32 `json:"retentionInDays,omitempty"` + FilterPattern string `json:"filterPattern"` + FilterName string `json:"filterName"` + LogGroupName string `json:"logGroupName"` + region string MetricTransformations []MetricTransformation `json:"metricTransformations"` CreationTime int64 `json:"creationTime"` } diff --git a/services/cloudwatchlogs/ops_batch2_audit_test.go b/services/cloudwatchlogs/ops_batch2_audit_test.go index cc5a181fc..bdb2da659 100644 --- a/services/cloudwatchlogs/ops_batch2_audit_test.go +++ b/services/cloudwatchlogs/ops_batch2_audit_test.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "net/http" + "strconv" "testing" + "time" "github.com/labstack/echo/v5" "github.com/stretchr/testify/assert" @@ -13,17 +15,19 @@ import ( "github.com/blackbirdworks/gopherstack/services/cloudwatchlogs" ) -// ── Batch-2 Issue #1: ErrInvalidSequenceToken not in handler error switch ────────────────────────────── +// ── Batch-2 Issue #1 (superseded): sequenceToken is ignored, not validated ────────────────────────────── // -// PutLogEvents with an incorrect sequenceToken calls backend.PutLogEvents which returns -// ErrInvalidSequenceToken. The handler's handleError switch had no case for this error, so -// it fell through to the default branch and returned 500 InternalServerError with -// __type="InternalServerError". -// -// AWS returns HTTP 400 with __type="InvalidSequenceTokenException". -// Fix: add ErrInvalidSequenceToken case to handleError mapping. - -func TestBatch2_PutLogEvents_WrongSequenceToken_Returns400(t *testing.T) { +// A prior audit pass had this emulator reject a mismatched sequenceToken with +// InvalidSequenceTokenException, mirroring PutLogEvents' pre-2022 contract. Per +// the current aws-sdk-go-v2 doc (cloudwatchlogs.PutLogEvents / InputLogEvent / +// types.InvalidSequenceTokenException): "The sequence token is now ignored in +// PutLogEvents actions. PutLogEvents actions are always accepted and never +// return InvalidSequenceTokenException or DataAlreadyAcceptedException even if +// the sequence token is not valid." The backend's validation was removed to +// match; this test now asserts the (correct) always-accepted behavior instead +// of a 400. + +func TestBatch2_PutLogEvents_MismatchedSequenceToken_StillAccepted(t *testing.T) { t.Parallel() tests := []struct { @@ -54,7 +58,11 @@ func TestBatch2_PutLogEvents_WrongSequenceToken_Returns400(t *testing.T) { doLogsRequest(t, h, e, "CreateLogGroup", `{"logGroupName":"grp"}`) doLogsRequest(t, h, e, "CreateLogStream", `{"logGroupName":"grp","logStreamName":"s"}`) - now := int64(1_700_000_000_000) + // Use the real wall clock, not a hardcoded epoch: a fixed past timestamp + // eventually falls outside PutLogEvents' rolling 14-day acceptance window + // as time passes, which would make events be rejected as "too old" and + // break this test's non-error-path assertions below. + now := time.Now().UnixMilli() for i := range tt.setupEvents { body, _ := json.Marshal(map[string]any{ "logGroupName": "grp", @@ -74,15 +82,14 @@ func TestBatch2_PutLogEvents_WrongSequenceToken_Returns400(t *testing.T) { }) rec := doLogsRequest(t, h, e, "PutLogEvents", string(reqBody)) - assert.Equal(t, http.StatusBadRequest, rec.Code, - "wrong sequenceToken must return 400, not 500 InternalServerError") + assert.Equal(t, http.StatusOK, rec.Code, + "a mismatched sequenceToken must not be rejected: AWS ignores it entirely") - var errResp struct { - Type string `json:"__type"` + var okResp struct { + NextSequenceToken string `json:"nextSequenceToken"` } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &errResp)) - assert.Equal(t, "InvalidSequenceTokenException", errResp.Type, - "wrong sequenceToken must map to InvalidSequenceTokenException, not InternalServerError") + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &okResp)) + assert.Equal(t, strconv.Itoa(tt.setupEvents+1), okResp.NextSequenceToken) }) } } diff --git a/services/cloudwatchlogs/persistence.go b/services/cloudwatchlogs/persistence.go index 5f5d92af6..e967b2243 100644 --- a/services/cloudwatchlogs/persistence.go +++ b/services/cloudwatchlogs/persistence.go @@ -2,71 +2,161 @@ package cloudwatchlogs import ( "context" + "encoding/json" + "fmt" + "maps" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) +// cwlSnapshotVersion identifies the shape of [backendSnapshot]. It must be +// bumped whenever a change to any DTO or backendSnapshot itself would make an +// older snapshot unsafe to decode as the current shape. Restore compares this +// against the persisted value and discards (rather than attempts to partially +// decode) any mismatch -- see Restore below. +const cwlSnapshotVersion = 1 + +// logGroupSnapshot, logStreamSnapshot, subscriptionFilterSnapshot, and +// metricFilterSnapshot are the on-disk DTOs for the four region-qualified +// "dirty" tables (see store_setup.go / models.go): each embeds the live value +// type by value to reuse its exported-field JSON shape, then adds back the +// unexported identity metadata (region, and for streams also logGroupName and +// inline events) as ordinary exported fields so a round trip through +// json.Marshal/Unmarshal does not lose them. This is the same DTO-registry +// pattern services/sqs's persistence.go (commit 0f09d77c) uses for Queue. + +type logGroupSnapshot struct { + Region string `json:"region"` + LogGroup +} + +func logGroupSnapshotKey(s *logGroupSnapshot) string { + return groupTableKey(s.Region, s.LogGroupName) +} + +type logStreamSnapshot struct { + Region string `json:"region"` + LogGroupName string `json:"logGroupName"` + Events []*OutputLogEvent `json:"events"` + LogStream +} + +func logStreamSnapshotKey(s *logStreamSnapshot) string { + return streamTableKey(s.Region, s.LogGroupName, s.LogStreamName) +} + +type subscriptionFilterSnapshot struct { + Region string `json:"region"` + SubscriptionFilter +} + +func subscriptionFilterSnapshotKey(s *subscriptionFilterSnapshot) string { + return subFilterTableKey(s.Region, s.LogGroupName, s.FilterName) +} + +type metricFilterSnapshot struct { + Region string `json:"region"` + MetricFilter +} + +func metricFilterSnapshotKey(s *metricFilterSnapshot) string { + return metricFilterTableKey(s.Region, s.LogGroupName, s.FilterName) +} + +// newRegionDTORegistry builds the ephemeral registry used to encode/decode the +// four region-qualified tables. It is rebuilt fresh on every Snapshot/Restore +// call (never stored on the backend) purely to reuse store's deterministic, +// type-erased JSON encoding instead of hand-rolling the marshal step. +func newRegionDTORegistry() ( + *store.Registry, + *store.Table[logGroupSnapshot], + *store.Table[logStreamSnapshot], + *store.Table[subscriptionFilterSnapshot], + *store.Table[metricFilterSnapshot], +) { + reg := store.NewRegistry() + groups := store.Register(reg, "groups", store.New(logGroupSnapshotKey)) + streams := store.Register(reg, "streams", store.New(logStreamSnapshotKey)) + subFilters := store.Register(reg, "subscriptionFilters", store.New(subscriptionFilterSnapshotKey)) + metricFilters := store.Register(reg, "metricFilters", store.New(metricFilterSnapshotKey)) + + return reg, groups, streams, subFilters, metricFilters +} + +// backendSnapshot is the top-level on-disk shape for the cloudwatchlogs backend. +// +// Tables holds one JSON-encoded array per registered table, produced by +// [store.Registry.SnapshotAll] for every "clean" table on b.registry plus the +// four region-qualified DTO tables above. Version guards against decoding a +// snapshot from an incompatible (older or newer) build of this backend as +// though it were the current shape; see Restore. type backendSnapshot struct { - Groups map[string]map[string]*LogGroup `json:"groups"` - Streams map[string]map[string]map[string]*LogStream `json:"streams"` - Events map[string]map[string]map[string][]*OutputLogEvent `json:"events"` - SubscriptionFilters map[string]map[string][]*SubscriptionFilter `json:"subscriptionFilters"` - ExportTasks map[string]*ExportTask `json:"exportTasks,omitempty"` - ImportTasks map[string]*ImportTask `json:"importTasks,omitempty"` - Deliveries map[string]*Delivery `json:"deliveries,omitempty"` - LogAnomalyDetectors map[string]*LogAnomalyDetector `json:"logAnomalyDetectors,omitempty"` - ScheduledQueries map[string]*ScheduledQuery `json:"scheduledQueries,omitempty"` - AccountPolicies map[string]*AccountPolicy `json:"accountPolicies,omitempty"` - KmsKeys map[string]string `json:"kmsKeys,omitempty"` - S3TableIntegrations map[string]string `json:"s3TableIntegrations,omitempty"` - MetricFilters map[string]map[string]map[string]*MetricFilter `json:"metricFilters,omitempty"` - QueryDefinitions map[string]*QueryDefinition `json:"queryDefinitions,omitempty"` - DataProtectionPolicies map[string]string `json:"dataProtectionPolicies,omitempty"` - ResourcePolicies map[string]ResourcePolicy `json:"resourcePolicies,omitempty"` - DeliveryDestinations map[string]DeliveryDestination `json:"deliveryDestinations,omitempty"` - DeliverySources map[string]DeliverySource `json:"deliverySources,omitempty"` - Destinations map[string]CWLDestination `json:"destinations,omitempty"` - IndexPolicies map[string]IndexPolicy `json:"indexPolicies,omitempty"` - Transformers map[string]Transformer `json:"transformers,omitempty"` - Integrations map[string]CWLIntegration `json:"integrations,omitempty"` - DeletionProtected map[string]bool `json:"deletionProtected,omitempty"` - AccountID string `json:"accountID"` - Region string `json:"region"` + Tables map[string]json.RawMessage `json:"tables"` + AccountID string `json:"accountID"` + Region string `json:"region"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. // It implements persistence.Persistable. +// +// Ephemeral, never-persisted state is deliberately excluded, matching this +// backend's behavior before Phase 3.3: Insights query results/cache +// (b.queries, b.parsedQueries, b.ephemeralRegistry) and the compiled filter +// pattern cache (b.compiledPatterns) are not part of backendSnapshot. func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, + "cloudwatchlogs: snapshot table marshal failed", "error", err) + + return nil + } + + dtoReg, groupDTOs, streamDTOs, subFilterDTOs, metricFilterDTOs := newRegionDTORegistry() + + for _, g := range b.groups.Snapshot() { + groupDTOs.Put(&logGroupSnapshot{LogGroup: *g, Region: g.region}) + } + + for _, s := range b.streams.Snapshot() { + streamDTOs.Put(&logStreamSnapshot{ + LogStream: *s, + Region: s.region, + LogGroupName: s.logGroupName, + Events: s.events, + }) + } + + for _, f := range b.subscriptionFilters.Snapshot() { + subFilterDTOs.Put(&subscriptionFilterSnapshot{SubscriptionFilter: *f, Region: f.region}) + } + + for _, f := range b.metricFilters.Snapshot() { + metricFilterDTOs.Put(&metricFilterSnapshot{MetricFilter: *f, Region: f.region}) + } + + dtoTables, err := dtoReg.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, + "cloudwatchlogs: snapshot region-table marshal failed", "error", err) + + return nil + } + + maps.Copy(tables, dtoTables) + snap := backendSnapshot{ - Groups: b.groups, - Streams: b.streams, - Events: b.events, - SubscriptionFilters: b.subscriptionFilters, - ExportTasks: b.exportTasks, - ImportTasks: b.importTasks, - Deliveries: b.deliveries, - LogAnomalyDetectors: b.logAnomalyDetectors, - ScheduledQueries: b.scheduledQueries, - AccountPolicies: b.accountPolicies, - KmsKeys: b.kmsKeys, - S3TableIntegrations: b.s3TableIntegrations, - MetricFilters: b.metricFilters, - QueryDefinitions: b.queryDefinitions, - DataProtectionPolicies: b.dataProtectionPolicies, - ResourcePolicies: b.resourcePolicies, - DeliveryDestinations: b.deliveryDestinations, - DeliverySources: b.deliverySources, - Destinations: b.destinations, - IndexPolicies: b.indexPolicies, - Transformers: b.transformers, - Integrations: b.integrations, - DeletionProtected: b.deletionProtected, - AccountID: b.accountID, - Region: b.region, + Version: cwlSnapshotVersion, + Tables: tables, + AccountID: b.accountID, + Region: b.region, } return persistence.MarshalSnapshot(ctx, "cloudwatchlogs", snap) @@ -81,120 +171,85 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { return err } - initSnapshotNilMaps(&snap) - b.mu.Lock("Restore") defer b.mu.Unlock() - b.groups = snap.Groups - b.streams = snap.Streams - b.events = snap.Events - b.subscriptionFilters = snap.SubscriptionFilters - b.exportTasks = snap.ExportTasks - b.importTasks = snap.ImportTasks - b.deliveries = snap.Deliveries - b.logAnomalyDetectors = snap.LogAnomalyDetectors - b.scheduledQueries = snap.ScheduledQueries - b.accountPolicies = snap.AccountPolicies - b.kmsKeys = snap.KmsKeys - b.s3TableIntegrations = snap.S3TableIntegrations - b.metricFilters = snap.MetricFilters - b.queryDefinitions = snap.QueryDefinitions - b.dataProtectionPolicies = snap.DataProtectionPolicies - b.resourcePolicies = snap.ResourcePolicies - b.deliveryDestinations = snap.DeliveryDestinations - b.deliverySources = snap.DeliverySources - b.destinations = snap.Destinations - b.indexPolicies = snap.IndexPolicies - b.transformers = snap.Transformers - b.integrations = snap.Integrations - b.deletionProtected = snap.DeletionProtected - b.accountID = snap.AccountID - b.region = snap.Region + if snap.Version != cwlSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. + logger.Load(ctx).WarnContext(ctx, + "cloudwatchlogs: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", cwlSnapshotVersion) + + b.registry.ResetAll() + b.groups.Reset() + b.streams.Reset() + b.subscriptionFilters.Reset() + b.metricFilters.Reset() - return nil -} - -// initSnapshotNilMaps replaces any nil map in snap with an empty map so the -// restored backend never contains a nil map reference. -func initSnapshotNilMaps(snap *backendSnapshot) { - initCoreNilMaps(snap) - initCompletenessNilMaps(snap) -} - -func initCoreNilMaps(snap *backendSnapshot) { - if snap.Groups == nil { - snap.Groups = make(map[string]map[string]*LogGroup) - } - if snap.Streams == nil { - snap.Streams = make(map[string]map[string]map[string]*LogStream) - } - if snap.Events == nil { - snap.Events = make(map[string]map[string]map[string][]*OutputLogEvent) - } - if snap.SubscriptionFilters == nil { - snap.SubscriptionFilters = make(map[string]map[string][]*SubscriptionFilter) - } - if snap.ExportTasks == nil { - snap.ExportTasks = make(map[string]*ExportTask) - } - if snap.ImportTasks == nil { - snap.ImportTasks = make(map[string]*ImportTask) - } - if snap.Deliveries == nil { - snap.Deliveries = make(map[string]*Delivery) - } - if snap.LogAnomalyDetectors == nil { - snap.LogAnomalyDetectors = make(map[string]*LogAnomalyDetector) - } - if snap.ScheduledQueries == nil { - snap.ScheduledQueries = make(map[string]*ScheduledQuery) - } - if snap.AccountPolicies == nil { - snap.AccountPolicies = make(map[string]*AccountPolicy) - } - if snap.KmsKeys == nil { - snap.KmsKeys = make(map[string]string) - } - if snap.S3TableIntegrations == nil { - snap.S3TableIntegrations = make(map[string]string) + return nil } -} -func initCompletenessNilMaps(snap *backendSnapshot) { - if snap.MetricFilters == nil { - snap.MetricFilters = make(map[string]map[string]map[string]*MetricFilter) - } - if snap.QueryDefinitions == nil { - snap.QueryDefinitions = make(map[string]*QueryDefinition) - } - if snap.DataProtectionPolicies == nil { - snap.DataProtectionPolicies = make(map[string]string) - } - if snap.ResourcePolicies == nil { - snap.ResourcePolicies = make(map[string]ResourcePolicy) - } - if snap.DeliveryDestinations == nil { - snap.DeliveryDestinations = make(map[string]DeliveryDestination) + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("cloudwatchlogs: restore snapshot tables: %w", err) } - if snap.DeliverySources == nil { - snap.DeliverySources = make(map[string]DeliverySource) - } - if snap.Destinations == nil { - snap.Destinations = make(map[string]CWLDestination) + + dtoReg, groupDTOs, streamDTOs, subFilterDTOs, metricFilterDTOs := newRegionDTORegistry() + + if err := dtoReg.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("cloudwatchlogs: restore region-qualified snapshot tables: %w", err) } - if snap.IndexPolicies == nil { - snap.IndexPolicies = make(map[string]IndexPolicy) + + liveGroups := make([]*LogGroup, 0, groupDTOs.Len()) + + for _, dto := range groupDTOs.All() { + g := dto.LogGroup + g.region = dto.Region + liveGroups = append(liveGroups, &g) } - if snap.Transformers == nil { - snap.Transformers = make(map[string]Transformer) + + b.groups.Restore(liveGroups) + + liveStreams := make([]*LogStream, 0, streamDTOs.Len()) + + for _, dto := range streamDTOs.All() { + s := dto.LogStream + s.region = dto.Region + s.logGroupName = dto.LogGroupName + s.events = dto.Events + liveStreams = append(liveStreams, &s) } - if snap.Integrations == nil { - snap.Integrations = make(map[string]CWLIntegration) + + b.streams.Restore(liveStreams) + + liveSubFilters := make([]*SubscriptionFilter, 0, subFilterDTOs.Len()) + + for _, dto := range subFilterDTOs.All() { + f := dto.SubscriptionFilter + f.region = dto.Region + liveSubFilters = append(liveSubFilters, &f) } - if snap.DeletionProtected == nil { - snap.DeletionProtected = make(map[string]bool) + + b.subscriptionFilters.Restore(liveSubFilters) + + liveMetricFilters := make([]*MetricFilter, 0, metricFilterDTOs.Len()) + + for _, dto := range metricFilterDTOs.All() { + f := dto.MetricFilter + f.region = dto.Region + liveMetricFilters = append(liveMetricFilters, &f) } + + b.metricFilters.Restore(liveMetricFilters) + + b.accountID = snap.AccountID + b.region = snap.Region + + return nil } // handlerSnapshot is the full persisted state for a Handler, combining both diff --git a/services/cloudwatchlogs/persistence_test.go b/services/cloudwatchlogs/persistence_test.go index 8069d9350..a729fc808 100644 --- a/services/cloudwatchlogs/persistence_test.go +++ b/services/cloudwatchlogs/persistence_test.go @@ -95,6 +95,125 @@ func TestInMemoryBackend_SnapshotRestore(t *testing.T) { } } +// TestInMemoryBackend_SnapshotRestore_FullStateRoundTrip exercises a +// Snapshot->Restore round trip across every persisted resource table +// (Phase 3.3 pkgs/store conversion), including the region-qualified "dirty" +// tables (log groups, streams with inline events, subscription filters, +// metric filters) that round-trip through a DTO rather than a direct +// json.Marshal of the live store.Table value -- see persistence.go. +func TestInMemoryBackend_SnapshotRestore_FullStateRoundTrip(t *testing.T) { + t.Parallel() + + ctx := t.Context() + original := cloudwatchlogs.NewInMemoryBackendWithConfig("111122223333", "us-west-2") + t.Cleanup(original.Close) + + // Log group + stream + inline events (the region-qualified tables). + _, err := original.CreateLogGroup(ctx, "/full/group", "", "kms-key-1") + require.NoError(t, err) + retentionDays := int32(30) + require.NoError(t, original.SetRetentionPolicy(ctx, "/full/group", &retentionDays)) + _, err = original.CreateLogStream(ctx, "/full/group", "stream-1") + require.NoError(t, err) + // Timestamps below minRealisticTimestampMs bypass PutLogEvents' retention/ + // future-window validation (treated as synthetic test data), so these are + // accepted regardless of when the test runs. + _, err = original.PutLogEvents(ctx, "/full/group", "stream-1", "", []cloudwatchlogs.InputLogEvent{ + {Message: "hello", Timestamp: 12345}, + {Message: "world", Timestamp: 12346}, + }) + require.NoError(t, err) + + // Subscription filter + metric filter (also region-qualified). + require.NoError(t, original.PutSubscriptionFilter( + ctx, "/full/group", "sub-filter", "ERROR", + "arn:aws:lambda:us-west-2:111122223333:function:sink", "", "", + )) + require.NoError(t, original.PutMetricFilter(ctx, "/full/group", "metric-filter", "ERROR", + []cloudwatchlogs.MetricTransformation{{MetricName: "Errors", MetricNamespace: "App", MetricValue: "1"}})) + + // Flat "clean" tables. + _, err = original.PutAccountPolicy("acct-policy", "SUBSCRIPTION_FILTER_POLICY", "{}", "ALL", "") + require.NoError(t, err) + require.NoError(t, original.AssociateKmsKey("/full/group", "", "kms-key-2")) + _, err = original.AssociateSourceToS3TableIntegration("arn:aws:s3tables:::bucket/tbl", "src", "type") + require.NoError(t, err) + taskID, err := original.CreateExportTask("exp", "/full/group", "", "dest-bucket", "", 0, 1_800_000_000_000) + require.NoError(t, err) + importTask, err := original.CreateImportTask("arn:aws:iam:::role/import", "arn:aws:cloudtrail:::store/x") + require.NoError(t, err) + delivery, err := original.CreateDelivery("src-name", "arn:aws:logs:::delivery-destination/dst", nil) + require.NoError(t, err) + detectorArn, err := original.CreateLogAnomalyDetector( + []string{"arn:aws:logs:::log-group:/full/group"}, "detector", "", "", "", 0, + ) + require.NoError(t, err) + scheduledArn, err := original.CreateScheduledQuery("sched", "fields @message", "", "", "") + require.NoError(t, err) + + snap := original.Snapshot(ctx) + require.NotNil(t, snap) + + fresh := cloudwatchlogs.NewInMemoryBackendWithConfig("111122223333", "us-west-2") + t.Cleanup(fresh.Close) + require.NoError(t, fresh.Restore(ctx, snap)) + + // --- verify region-qualified tables --- + groups, _, err := fresh.DescribeLogGroups(ctx, "", "", 100) + require.NoError(t, err) + require.Len(t, groups, 1) + assert.Equal(t, "/full/group", groups[0].LogGroupName) + require.NotNil(t, groups[0].RetentionInDays) + assert.Equal(t, int32(30), *groups[0].RetentionInDays) + + streams, _, err := fresh.DescribeLogStreams(ctx, "/full/group", "", "", "", false, 100) + require.NoError(t, err) + require.Len(t, streams, 1) + assert.Equal(t, "stream-1", streams[0].LogStreamName) + + events, _, _, err := fresh.GetLogEvents(ctx, "/full/group", "stream-1", nil, nil, 100, "", true) + require.NoError(t, err) + require.Len(t, events, 2) + assert.Equal(t, "hello", events[0].Message) + assert.Equal(t, "world", events[1].Message) + + subFilters, _, err := fresh.DescribeSubscriptionFilters(ctx, "/full/group", "", "", 100) + require.NoError(t, err) + require.Len(t, subFilters, 1) + assert.Equal(t, "sub-filter", subFilters[0].FilterName) + + metricFilters, _, err := fresh.DescribeMetricFilters(ctx, "/full/group", "", "", "", "", 100) + require.NoError(t, err) + require.Len(t, metricFilters, 1) + assert.Equal(t, "metric-filter", metricFilters[0].FilterName) + + // --- verify flat tables --- + policies, _, err := fresh.DescribeAccountPolicies("", "", nil, 100, "") + require.NoError(t, err) + require.Len(t, policies, 1) + assert.Equal(t, "acct-policy", policies[0].PolicyName) + + exportTasks, _, err := fresh.DescribeExportTasks(taskID, "", 100, "") + require.NoError(t, err) + require.Len(t, exportTasks, 1) + + importTasks, _, err := fresh.DescribeImportTasks(importTask.ImportID, 100, "") + require.NoError(t, err) + require.Len(t, importTasks, 1) + + gotDelivery, err := fresh.GetDelivery(delivery.ID) + require.NoError(t, err) + assert.Equal(t, "src-name", gotDelivery.DeliverySourceName) + + detector, err := fresh.GetLogAnomalyDetector(detectorArn) + require.NoError(t, err) + assert.Equal(t, "detector", detector.DetectorName) + + scheduled, err := fresh.GetScheduledQuery(scheduledArn) + require.NoError(t, err) + assert.Equal(t, "sched", scheduled.Name) +} + func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { t.Parallel() diff --git a/services/cloudwatchlogs/region_accessors.go b/services/cloudwatchlogs/region_accessors.go new file mode 100644 index 000000000..0ded800a4 --- /dev/null +++ b/services/cloudwatchlogs/region_accessors.go @@ -0,0 +1,109 @@ +package cloudwatchlogs + +// Code in this file replaces the pre-Phase-3.3 groupsStore(region)/streamsStore +// (region)/eventsStore(region)/subscriptionFiltersStore(region)/ +// metricFiltersStore(region) helpers, which each lazily returned a raw +// map[string]... scoped to one region. store.Table has a single flat string +// key (see store_setup.go's groupTableKey et al.), so callers that used to +// index/range/delete on the returned map now call the small typed helpers +// below instead. All of them assume the caller already holds b.mu (read or +// write, as appropriate), exactly like the map-returning helpers they replace. + +// --- log groups --- + +func (b *InMemoryBackend) groupGet(region, name string) (*LogGroup, bool) { + return b.groups.Get(groupTableKey(region, name)) +} + +func (b *InMemoryBackend) groupHas(region, name string) bool { + return b.groups.Has(groupTableKey(region, name)) +} + +// groupPut inserts or replaces g, which must already have region and +// LogGroupName set. +func (b *InMemoryBackend) groupPut(g *LogGroup) { + b.groups.Put(g) +} + +func (b *InMemoryBackend) groupDelete(region, name string) bool { + return b.groups.Delete(groupTableKey(region, name)) +} + +// groupsInRegion returns every log group registered under region, in +// unspecified order (mirrors the old groups[region] map's iteration). +func (b *InMemoryBackend) groupsInRegion(region string) []*LogGroup { + return b.groupsByRegion.Get(region) +} + +// --- log streams (events live inline on LogStream.events) --- + +func (b *InMemoryBackend) streamGet(region, groupName, streamName string) (*LogStream, bool) { + return b.streams.Get(streamTableKey(region, groupName, streamName)) +} + +func (b *InMemoryBackend) streamHas(region, groupName, streamName string) bool { + return b.streams.Has(streamTableKey(region, groupName, streamName)) +} + +// streamPut inserts or replaces s, which must already have region, +// logGroupName, and LogStreamName set. +func (b *InMemoryBackend) streamPut(s *LogStream) { + b.streams.Put(s) +} + +func (b *InMemoryBackend) streamDelete(region, groupName, streamName string) bool { + return b.streams.Delete(streamTableKey(region, groupName, streamName)) +} + +// streamsInGroup returns every stream in region+groupName, in unspecified +// order (mirrors the old streams[region][groupName] map's iteration). The +// returned slice is owned by the underlying index -- see [store.Index.Get] -- +// so callers that need to delete while iterating must copy it first (see +// deleteStreamsInGroup). +func (b *InMemoryBackend) streamsInGroup(region, groupName string) []*LogStream { + return b.streamsByGroup.Get(streamGroupKey(region, groupName)) +} + +// deleteStreamsInGroup removes every stream (and, since events are stored +// inline, every event) belonging to region+groupName. Used by DeleteLogGroup. +func (b *InMemoryBackend) deleteStreamsInGroup(region, groupName string) { + for _, s := range append([]*LogStream(nil), b.streamsInGroup(region, groupName)...) { + b.streams.Delete(logStreamKeyFn(s)) + } +} + +// --- subscription filters --- + +// subscriptionFiltersInGroup returns every subscription filter in +// region+groupName, in unspecified order (mirrors the old +// subscriptionFilters[region][groupName] slice). The returned slice is owned +// by the underlying index -- copy before deleting while iterating. +func (b *InMemoryBackend) subscriptionFiltersInGroup(region, groupName string) []*SubscriptionFilter { + return b.subscriptionFiltersByGroup.Get(streamGroupKey(region, groupName)) +} + +func (b *InMemoryBackend) deleteSubscriptionFiltersInGroup(region, groupName string) { + for _, f := range append([]*SubscriptionFilter(nil), b.subscriptionFiltersInGroup(region, groupName)...) { + b.subscriptionFilters.Delete(subscriptionFilterKeyFn(f)) + } +} + +// --- metric filters --- + +func (b *InMemoryBackend) metricFilterGet(region, groupName, filterName string) (*MetricFilter, bool) { + return b.metricFilters.Get(metricFilterTableKey(region, groupName, filterName)) +} + +// metricFiltersInGroup returns every metric filter in region+groupName, in +// unspecified order (mirrors the old metricFilters[region][groupName] map). +// The returned slice is owned by the underlying index -- copy before deleting +// while iterating. +func (b *InMemoryBackend) metricFiltersInGroup(region, groupName string) []*MetricFilter { + return b.metricFiltersByGroup.Get(streamGroupKey(region, groupName)) +} + +func (b *InMemoryBackend) deleteMetricFiltersInGroup(region, groupName string) { + for _, f := range append([]*MetricFilter(nil), b.metricFiltersInGroup(region, groupName)...) { + b.metricFilters.Delete(metricFilterKeyFn(f)) + } +} diff --git a/services/cloudwatchlogs/store_setup.go b/services/cloudwatchlogs/store_setup.go new file mode 100644 index 000000000..ad1dea7ff --- /dev/null +++ b/services/cloudwatchlogs/store_setup.go @@ -0,0 +1,291 @@ +package cloudwatchlogs + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T (and map[string]T / nested map[string]map[string]*T) resource +// field on InMemoryBackend is replaced by a *store.Table[T], registered exactly +// once here. See pkgs/store's package doc and the services/ec2 (commit +// 12e611a4, data-driven registration slice) and services/sqs (commit +// 0f09d77c, DTO-registry) conversions this follows. +// +// Two maps are deliberately NOT converted and remain plain maps -- see the +// comment above registerAllTables for why. +// +// # Region-qualified tables +// +// groups, streams, subscriptionFilters, and metricFilters were nested +// map[string]map[string]... keyed by region as the outer key. store.Table has +// a single flat string key, so each value type gains an unexported `region` +// field (and LogStream additionally gains `logGroupName` + inline `events`) +// used to build a region-qualified composite key -- see groupTableKey et al. +// below. Because these fields are unexported they never appear in the wire +// JSON these types double as (e.g. DescribeLogGroups' `logGroups` response), +// but that also means they cannot survive Table.Snapshot's json.Marshal on +// their own; persistence.go round-trips them through DTOs +// (logGroupSnapshot, logStreamSnapshot, ...) that carry the same data as +// ordinary exported fields instead. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +// --- composite-key helpers for region-qualified tables --- + +// groupTableKey returns the store.Table primary key for a log group. +func groupTableKey(region, groupName string) string { + return region + "|" + groupName +} + +// streamTableKey returns the store.Table primary key for a log stream. +func streamTableKey(region, groupName, streamName string) string { + return streamGroupKey(region, groupName) + "|" + streamName +} + +// streamGroupKey returns the store.Index key grouping every stream that +// belongs to one region+log-group pair (mirrors the old streams[region][group] map). +func streamGroupKey(region, groupName string) string { + return region + "|" + groupName +} + +// subFilterTableKey returns the store.Table primary key for a subscription filter. +func subFilterTableKey(region, groupName, filterName string) string { + return region + "|" + groupName + "|" + filterName +} + +// metricFilterTableKey returns the store.Table primary key for a metric filter. +func metricFilterTableKey(region, groupName, filterName string) string { + return region + "|" + groupName + "|" + filterName +} + +// --- store.Table key functions --- + +func logGroupKeyFn(g *LogGroup) string { return groupTableKey(g.region, g.LogGroupName) } + +func logStreamKeyFn(s *LogStream) string { + return streamTableKey(s.region, s.logGroupName, s.LogStreamName) +} + +func logStreamGroupIndexKeyFn(s *LogStream) string { + return streamGroupKey(s.region, s.logGroupName) +} + +func subscriptionFilterKeyFn(f *SubscriptionFilter) string { + return subFilterTableKey(f.region, f.LogGroupName, f.FilterName) +} + +func subscriptionFilterGroupIndexKeyFn(f *SubscriptionFilter) string { + return streamGroupKey(f.region, f.LogGroupName) +} + +func metricFilterKeyFn(f *MetricFilter) string { + return metricFilterTableKey(f.region, f.LogGroupName, f.FilterName) +} + +func metricFilterGroupIndexKeyFn(f *MetricFilter) string { + return streamGroupKey(f.region, f.LogGroupName) +} + +func accountPolicyKeyFn(p *AccountPolicy) string { return p.PolicyName + ":" + p.PolicyType } +func exportTaskKeyFn(t *ExportTask) string { return t.TaskID } +func importTaskKeyFn(t *ImportTask) string { return t.ImportID } +func deliveryKeyFn(d *Delivery) string { return d.ID } +func logAnomalyDetectorKeyFn(d *LogAnomalyDetector) string { return d.AnomalyDetectorArn } +func scheduledQueryKeyFn(q *ScheduledQuery) string { return q.Arn } +func queryDefinitionKeyFn(q *QueryDefinition) string { return q.QueryDefinitionID } +func resourcePolicyKeyFn(p *ResourcePolicy) string { return p.PolicyName } +func deliveryDestinationKeyFn(d *DeliveryDestination) string { return d.Name } +func deliverySourceKeyFn(s *DeliverySource) string { return s.Name } +func cwlDestinationKeyFn(d *CWLDestination) string { return d.DestinationName } +func indexPolicyKeyFn(p *IndexPolicy) string { return p.LogGroupIdentifier } +func transformerKeyFn(t *Transformer) string { return t.LogGroupIdentifier } +func cwlIntegrationKeyFn(i *CWLIntegration) string { return i.Name } + +// anomalyTableKey returns the store.Table primary key for an Anomaly. +func anomalyTableKey(detectorArn, anomalyID string) string { return detectorArn + "|" + anomalyID } + +// anomalyKeyFn keys an Anomaly by its (detector, id) pair; anomalyDetectorIndexKeyFn +// groups every anomaly belonging to one detector (mirrors the old +// anomalies[detectorArn] map) for ListAnomalies(anomalyDetectorArn=""). +func anomalyKeyFn(a *Anomaly) string { return anomalyTableKey(a.AnomalyDetectorArn, a.AnomalyID) } +func anomalyDetectorIndexKeyFn(a *Anomaly) string { return a.AnomalyDetectorArn } + +// scheduledQueryRunHistory wraps the run-history slice AWS keeps per scheduled +// query. It exists because store.Table requires a value with its own identity +// field; a bare []*ScheduledQueryRunSummary has none (multiple runs share the +// same Arn), so -- like inlining log events on LogStream -- the slice is +// nested one level under a keyed parent entity instead. +type scheduledQueryRunHistory struct { + Arn string `json:"arn"` + Runs []*ScheduledQueryRunSummary `json:"runs"` +} + +func scheduledQueryRunHistoryKeyFn(h *scheduledQueryRunHistory) string { return h.Arn } + +// kmsKeyEntry, s3TableIntegrationEntry, dataProtectionPolicyEntry, and +// deletionProtectionEntry each wrap a bare map[string]string / map[string]bool +// value in a minimal identified struct, since store.Table requires values to +// carry their own primary key. +type kmsKeyEntry struct { + Key string `json:"key"` + KmsKeyID string `json:"kmsKeyId"` +} + +func kmsKeyEntryKeyFn(e *kmsKeyEntry) string { return e.Key } + +type s3TableIntegrationEntry struct { + ID string `json:"id"` + IntegrationArn string `json:"integrationArn"` +} + +func s3TableIntegrationEntryKeyFn(e *s3TableIntegrationEntry) string { return e.ID } + +type dataProtectionPolicyEntry struct { + LogGroupIdentifier string `json:"logGroupIdentifier"` + PolicyDocument string `json:"policyDocument"` +} + +func dataProtectionPolicyEntryKeyFn(e *dataProtectionPolicyEntry) string { + return e.LogGroupIdentifier +} + +type deletionProtectionEntry struct { + LogGroupIdentifier string `json:"logGroupIdentifier"` + Protected bool `json:"protected"` +} + +func deletionProtectionEntryKeyFn(e *deletionProtectionEntry) string { return e.LogGroupIdentifier } + +// storedQueryKeyFn keys a storedQuery by the QueryID assigned when StartQuery +// stored it. +func storedQueryKeyFn(q *storedQuery) string { return q.info.QueryID } + +// registerAllTables registers every persisted resource table on b.registry +// exactly once. It must be called during construction only (immediately after +// b.registry is created), never on every Reset() -- store.Register panics on a +// duplicate name, so runtime resets go through b.registry.ResetAll() instead +// (see InMemoryBackend.Reset in backend.go). +// +// The following resource fields are deliberately left as plain maps (not +// registered anywhere), because their value carries no identity field of its +// own to serve as a store.Table key and both are ephemeral, never-persisted +// caches (not part of backendSnapshot before or after this refactor): +// - compiledPatterns: value type compiledFilterPattern carries no identity +// field of its own (it holds only the compiled matcher, not the source +// pattern string it was compiled from); keyed externally by the raw +// pattern string. Guarded by its own compiledPatternsMu, independent of +// b.mu, matching the safemap "isolated cache" exception. +// - parsedQueries: value type insightsQuery carries no identity field of its +// own (keyed externally by the raw query string) and holds a []queryStage +// of unexported interface implementations that cannot round-trip through +// JSON in any case. +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per persisted resource table. These are the "clean" tables (see +// persistence.go) whose value type's identity field is already a normal +// exported struct field, so store.Table's own JSON marshal round-trips them +// without a separate DTO. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.accountPolicies = store.Register(b.registry, "accountPolicies", store.New(accountPolicyKeyFn)) + }, + func(b *InMemoryBackend) { + b.exportTasks = store.Register(b.registry, "exportTasks", store.New(exportTaskKeyFn)) + }, + func(b *InMemoryBackend) { + b.importTasks = store.Register(b.registry, "importTasks", store.New(importTaskKeyFn)) + }, + func(b *InMemoryBackend) { + b.deliveries = store.Register(b.registry, "deliveries", store.New(deliveryKeyFn)) + }, + func(b *InMemoryBackend) { + b.logAnomalyDetectors = store.Register( + b.registry, "logAnomalyDetectors", store.New(logAnomalyDetectorKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.scheduledQueries = store.Register(b.registry, "scheduledQueries", store.New(scheduledQueryKeyFn)) + }, + func(b *InMemoryBackend) { + b.s3TableIntegrations = store.Register( + b.registry, "s3TableIntegrations", store.New(s3TableIntegrationEntryKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.kmsKeys = store.Register(b.registry, "kmsKeys", store.New(kmsKeyEntryKeyFn)) + }, + func(b *InMemoryBackend) { + b.queryDefinitions = store.Register(b.registry, "queryDefinitions", store.New(queryDefinitionKeyFn)) + }, + func(b *InMemoryBackend) { + b.dataProtectionPolicies = store.Register( + b.registry, "dataProtectionPolicies", store.New(dataProtectionPolicyEntryKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.resourcePolicies = store.Register(b.registry, "resourcePolicies", store.New(resourcePolicyKeyFn)) + }, + func(b *InMemoryBackend) { + b.deliveryDestinations = store.Register( + b.registry, "deliveryDestinations", store.New(deliveryDestinationKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.deliverySources = store.Register(b.registry, "deliverySources", store.New(deliverySourceKeyFn)) + }, + func(b *InMemoryBackend) { + b.destinations = store.Register(b.registry, "destinations", store.New(cwlDestinationKeyFn)) + }, + func(b *InMemoryBackend) { + b.indexPolicies = store.Register(b.registry, "indexPolicies", store.New(indexPolicyKeyFn)) + }, + func(b *InMemoryBackend) { + b.transformers = store.Register(b.registry, "transformers", store.New(transformerKeyFn)) + }, + func(b *InMemoryBackend) { + b.integrations = store.Register(b.registry, "integrations", store.New(cwlIntegrationKeyFn)) + }, + func(b *InMemoryBackend) { + b.deletionProtected = store.Register( + b.registry, "deletionProtected", store.New(deletionProtectionEntryKeyFn), + ) + }, +} + +// registerEphemeralTables registers every never-persisted-but-still-a-resource +// table on b.ephemeralRegistry, so InMemoryBackend.Reset can clear them via one +// ResetAll() call same as the persisted tables, without Snapshot/Restore ever +// touching them (b.ephemeralRegistry is never passed to persistence.go). +func registerEphemeralTables(b *InMemoryBackend) { + b.queries = store.Register(b.ephemeralRegistry, "queries", store.New(storedQueryKeyFn)) + b.anomalies = store.Register(b.ephemeralRegistry, "anomalies", store.New(anomalyKeyFn)) + b.anomalyByDetector = b.anomalies.AddIndex("byDetector", anomalyDetectorIndexKeyFn) + b.scheduledQueryRuns = store.Register( + b.ephemeralRegistry, "scheduledQueryRuns", store.New(scheduledQueryRunHistoryKeyFn), + ) +} + +// registerRegionTables constructs the four region-qualified "dirty" tables +// (see the package doc above): groups, streams, subscriptionFilters, and +// metricFilters. They are NOT registered on b.registry or b.ephemeralRegistry +// -- Snapshot/Restore drive them explicitly through DTOs (persistence.go) and +// Reset clears them with a direct .Reset() call (backend.go) -- but each still +// gets a store.Table plus the secondary index that replaces the old +// streams[region][group]-style nested-map lookup. +func registerRegionTables(b *InMemoryBackend) { + b.groups = store.New(logGroupKeyFn) + b.groupsByRegion = b.groups.AddIndex("byRegion", func(g *LogGroup) string { return g.region }) + + b.streams = store.New(logStreamKeyFn) + b.streamsByGroup = b.streams.AddIndex("byGroup", logStreamGroupIndexKeyFn) + + b.subscriptionFilters = store.New(subscriptionFilterKeyFn) + b.subscriptionFiltersByGroup = b.subscriptionFilters.AddIndex("byGroup", subscriptionFilterGroupIndexKeyFn) + + b.metricFilters = store.New(metricFilterKeyFn) + b.metricFiltersByGroup = b.metricFilters.AddIndex("byGroup", metricFilterGroupIndexKeyFn) +} diff --git a/services/cognitoidp/PARITY.md b/services/cognitoidp/PARITY.md new file mode 100644 index 000000000..c9901750c --- /dev/null +++ b/services/cognitoidp/PARITY.md @@ -0,0 +1,136 @@ +--- +service: cognitoidp +sdk_module: aws-sdk-go-v2/service/cognitoidentityprovider@1.59.1 +last_audit_commit: 7d0baf79 +last_audit_date: 2026-07-05 +overall: A # ~870 LOC genuine fixes this pass (see below); prior sweeps already got most of the surface to B +# Per-op or per-op-family status. Values: ok | partial | gap | deferred. +ops: + InitiateAuth: {wire: ok, errors: ok, state: ok, persist: ok, note: "USER_PASSWORD_AUTH/ADMIN_USER_PASSWORD_AUTH/USER_SRP_AUTH(simplified)/REFRESH_TOKEN_AUTH real; PreventUserExistenceErrors masking added this pass (gopherstack-2sp)"} + AdminInitiateAuth: {wire: ok, errors: ok, state: ok, persist: ok, note: "never masks UserNotFoundException, matching AWS (admin API)"} + RespondToAuthChallenge: {wire: ok, errors: ok, state: ok, persist: ok, note: "SOFTWARE_TOKEN_MFA now real RFC6238 TOTP (was disguised stub accepting any 6 digits); SMS_MFA/EMAIL_OTP now require the generated one-time code (was also any 6 digits); PASSWORD_VERIFIER/NEW_PASSWORD_REQUIRED unchanged, real"} + AdminRespondToAuthChallenge: {wire: ok, errors: ok, state: ok, persist: ok, note: "same fix as RespondToAuthChallenge (shared backend method)"} + AssociateSoftwareToken: {wire: ok, errors: ok, state: ok, persist: ok} + VerifySoftwareToken: {wire: ok, errors: ok, state: ok, persist: ok, note: "now verifies a real RFC 6238 TOTP code against the associated secret (was: any 6 digits accepted) — gopherstack-2sp"} + SetUserMFAPreference: {wire: ok, errors: ok, state: ok, persist: ok} + AdminSetUserMFAPreference: {wire: ok, errors: ok, state: ok, persist: ok} + CreateUserPool: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeUserPool: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateUserPool: {wire: ok, errors: ok, state: ok, persist: ok, note: "PasswordPolicy, MfaConfiguration, LambdaConfig(stored only, see gaps), AccountRecoverySetting, DeletionProtection all settable"} + ListUserPools: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteUserPool: {wire: ok, errors: ok, state: ok, persist: ok, note: "now refuses to delete when DeletionProtection=ACTIVE (was: silently deleted regardless) — gopherstack-2sp"} + CreateUserPoolClient: {wire: ok, errors: ok, state: ok, persist: ok, note: "PreventUserExistenceErrors field added this pass (was entirely unimplemented); OAuth flows/scopes/callback URLs/token validity units/secret generation all real"} + UpdateUserPoolClient: {wire: ok, errors: ok, state: ok, persist: ok, note: "PreventUserExistenceErrors now updatable"} + DescribeUserPoolClient: {wire: ok, errors: ok, state: ok, persist: ok} + ListUserPoolClients: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteUserPoolClient: {wire: ok, errors: ok, state: ok, persist: ok} + AddUserPoolClientSecret: {wire: ok, errors: ok, state: ok, persist: ok} + SignUp: {wire: ok, errors: ok, state: ok, persist: ok, note: "password policy enforced, real confirm code generated"} + ConfirmSignUp: {wire: ok, errors: ok, state: ok, persist: ok, note: "expiring codes, CodeMismatchException/ExpiredCodeException"} + AdminConfirmSignUp: {wire: ok, errors: ok, state: ok, persist: ok} + ResendConfirmationCode: {wire: ok, errors: ok, state: ok, persist: ok, note: "does not yet apply PreventUserExistenceErrors masking, see gaps (gopherstack-aib)"} + AdminCreateUser: {wire: ok, errors: ok, state: ok, persist: ok} + AdminSetUserPassword: {wire: ok, errors: ok, state: ok, persist: ok} + AdminGetUser: {wire: ok, errors: ok, state: ok, persist: ok} + AdminDeleteUser: {wire: ok, errors: ok, state: ok, persist: ok} + AdminResetUserPassword: {wire: ok, errors: ok, state: ok, persist: ok} + AdminUpdateUserAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + AdminDeleteUserAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + AdminDisableUser: {wire: ok, errors: ok, state: ok, persist: ok} + AdminEnableUser: {wire: ok, errors: ok, state: ok, persist: ok} + AdminUserGlobalSignOut: {wire: ok, errors: ok, state: ok, persist: ok, note: "revokes refresh tokens + stamps tokenRevokedBefore so already-issued access tokens are rejected too"} + GlobalSignOut: {wire: ok, errors: ok, state: ok, persist: ok, note: "same revocation mechanism as AdminUserGlobalSignOut"} + RevokeToken: {wire: ok, errors: ok, state: ok, persist: ok} + ListUsers: {wire: ok, errors: ok, state: ok, persist: ok, note: "pkgs/page-style pagination"} + ListUsersInGroup: {wire: ok, errors: ok, state: ok, persist: ok} + ForgotPassword: {wire: ok, errors: ok, state: ok, persist: ok, note: "does not yet apply PreventUserExistenceErrors masking, see gaps (gopherstack-aib)"} + ConfirmForgotPassword: {wire: ok, errors: ok, state: ok, persist: ok} + ChangePassword: {wire: ok, errors: ok, state: ok, persist: ok} + GetUser: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteUser: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteUserAttributes/AdminDeleteUserAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + VerifyUserAttribute/GetUserAttributeVerificationCode: {wire: ok, errors: ok, state: ok, persist: ok} + CreateGroup/DeleteGroup/GetGroup/ListGroups/UpdateGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "precedence respected"} + AdminAddUserToGroup/AdminRemoveUserFromGroup/AdminListGroupsForUser: {wire: ok, errors: ok, state: ok, persist: ok, note: "cognito:groups claim reflected in ID/access tokens"} + CreateResourceServer/DescribeResourceServer/ListResourceServers/UpdateResourceServer/DeleteResourceServer: {wire: ok, errors: ok, state: ok, persist: ok} + CreateIdentityProvider/DescribeIdentityProvider/ListIdentityProviders/UpdateIdentityProvider/DeleteIdentityProvider/GetIdentityProviderByIdentifier: {wire: ok, errors: ok, state: ok, persist: ok, note: "audited at a family level, not re-walked line by line this pass — unchanged since prior sweep"} + TagResource/UntagResource/ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok, note: "pkgs/tags"} + GetSigningCertificate: {wire: ok, errors: ok, state: ok, persist: ok, note: "deterministic self-signed X.509 wrapping the pool's real RSA key"} + GetUserPoolMfaConfig/SetUserPoolMfaConfig: {wire: ok, errors: ok, state: ok, persist: ok} + jwks_well_known: {wire: ok, errors: ok, state: ok, persist: ok, note: "RS256, real RSA-2048 per pool, JWKS + GetSigningCertificate both derive from the same key"} +families: + user_import_jobs: {status: ok, note: "CreateUserImportJob/StartUserImportJob/StopUserImportJob/DescribeUserImportJob/ListUserImportJobs/GetCSVHeader — audited at family level, unchanged since prior sweep"} + devices: {status: ok, note: "ConfirmDevice/ForgetDevice/AdminForgetDevice/GetDevice/AdminGetDevice/ListDevices/AdminListDevices/UpdateDeviceStatus/AdminUpdateDeviceStatus — family level, unchanged since prior sweep"} + webauthn: {status: ok, note: "StartWebAuthnRegistration/CompleteWebAuthnRegistration/ListWebAuthnCredentials/DeleteWebAuthnCredential — family level, unchanged since prior sweep"} + managed_login_branding: {status: ok, note: "Create/Describe/Update/Delete + DescribeManagedLoginBrandingByClient, UICustomization — family level"} + risk_config: {status: ok, note: "DescribeRiskConfiguration/SetRiskConfiguration/AdminListUserAuthEvents/AdminUpdateAuthEventFeedback/GetUserAuthFactors — family level"} + domains: {status: ok, note: "CreateUserPoolDomain/DescribeUserPoolDomain/DeleteUserPoolDomain — family level"} + terms: {status: ok, note: "CreateTerms/DescribeTerms/ListTerms/UpdateTerms/DeleteTerms — family level"} + log_delivery: {status: ok, note: "GetLogDeliveryConfiguration/SetLogDeliveryConfiguration — family level"} +gaps: + - "USER_SRP_AUTH does not implement real SRP-6a: InitiateAuth requires AuthParameters[PASSWORD] directly (server-side bcrypt check), then returns a PASSWORD_VERIFIER challenge that RespondToAuthChallenge completes without any zero-knowledge proof exchange. A real SRP client never sends PASSWORD and cannot authenticate here. Investigated in depth this pass; not fixed because a byte-perfect implementation of Cognito's SRP variant (3072-bit N, HKDF-SHA256 with the \"Caldera Derived Key\" info string, HMAC-SHA256 M1 proof) could not be verified against a real client/reference vectors in this session, and a subtly-wrong crypto implementation would be worse than the current honestly-documented simplification. (bd: gopherstack-p8i)" + - "LambdaConfig (PreSignUp, PostConfirmation, PreTokenGeneration, CustomMessage, etc.) is stored/returned but no trigger is ever invoked — would require cross-service invocation into the lambda service. (bd: gopherstack-8fw)" + - "PreventUserExistenceErrors=ENABLED masking (added this pass) only covers InitiateAuth. ForgotPassword and ResendConfirmationCode still reveal UserNotFoundException regardless of the setting; AWS masks those too (fake-success response). (bd: gopherstack-aib)" +deferred: + - "Identity providers, resource servers, user import jobs, devices, WebAuthn, managed login branding, risk config, domains, terms, log delivery: verified at a family/smoke level (dispatch wired, backend mutates real maps, persisted in backendSnapshot, no bare stubs found), not re-walked op-by-op field-by-field this pass — unchanged since the prior two sweeps (parity sweep 1 & 2) which already covered these in depth." +leaks: {status: clean, note: "janitor.go sweeps expired refresh tokens/mfa sessions/confirm codes/attr verification codes on a bounded interval (WithJanitor); ctx cancellation observed via StartWorker; no new goroutines/unbounded maps introduced this pass — the two new maps touched (mfaSessionEntry.Code is a field, not a map) reuse existing bounded, janitor-swept structures"} +--- + +## Notes + +### What this pass fixed (see report for full detail) + +1. **MFA challenge codes were a disguised stub (highest-severity finding).** + `RespondToMFAChallenge` / `VerifySoftwareToken` validated only that the supplied code was + 6 ASCII digits and then always succeeded — SOFTWARE_TOKEN_MFA, SMS_MFA, and EMAIL_OTP were + all bypassable with any random 6-digit string, regardless of the secret returned by + `AssociateSoftwareToken`. Fixed with a real RFC 6238 TOTP implementation + (`totp.go`: HMAC-SHA1, 30s step, ±1 step clock-skew tolerance, RFC 4226 dynamic + truncation) verified against the **official RFC 4226 Appendix D and RFC 6238 Appendix B + test vectors** (two independently published vector sets that cross-check each other — + RFC 4226 counter=1 and RFC 6238 T=59 both yield `287082`). SMS_MFA/EMAIL_OTP now require + the one-time code generated by `newMFASession` (mirroring the existing + ForgotPassword/ConfirmSignUp confirmation-code pattern) instead of accepting anything. + A wrong code no longer consumes the MFA session, so the caller can retry until it + expires — matching real Cognito. + +2. **PreventUserExistenceErrors was completely unimplemented** on `UserPoolClient` (not + stored, not exposed on Create/Update/Describe, not enforced anywhere) despite being + explicitly part of the client model. Added the field (default `"LEGACY"`, matching AWS), + wired it through Create/UpdateUserPoolClient input/output, and enforced it in the + non-admin `InitiateAuth` path: `ENABLED` now masks an unknown username behind the exact + same `NotAuthorizedException` text a wrong password produces (proven in tests: the two + error strings are asserted equal). `AdminInitiateAuth` intentionally never masks — AWS + only applies this to the non-admin, unauthenticated API. + +3. **DeletionProtection was stored but never enforced.** `DeleteUserPool` deleted pools + unconditionally even when `DeletionProtection: "ACTIVE"`. Now returns + `InvalidParameterException` with AWS's documented remediation message, and the pool can + still be deleted after `UpdateUserPool` flips it back to `"INACTIVE"`. + +### Traps for the next auditor + +- **USER_SRP_AUTH is not a stub in the "returns fake data" sense** — it does real bcrypt + password verification and issues real signed JWTs. It's a *simplified* SRP: the server + never receives an SRP_A value and never proves anything in zero-knowledge; it just + requires the plaintext password in `AuthParameters["PASSWORD"]` and fakes the + `PASSWORD_VERIFIER` challenge/response shape around a direct password check. This means + the wire *shape* (ChallengeName, Session, ChallengeParameters) matches AWS's + `cognitoidentityprovider` types, but the *protocol* does not — any client that actually + performs SRP math (rather than a test harness that knows to pass PASSWORD directly) will + fail to authenticate. Do not "fix" this without either (a) real reference test vectors + from a genuine Cognito SRP client, or (b) accepting the risk of shipping unverifiable + crypto — get a second opinion before attempting bit-perfect SRP-6a here. +- **`RespondToMFAChallenge` is now challenge-type-aware** (`verifyMFAChallengeCode` in + backend.go): SOFTWARE_TOKEN_MFA verifies against `user.TOTPSecret` via `verifyTOTPCode`; + SMS_MFA/EMAIL_OTP verify against `mfaSessionEntry.Code`. If you add a new MFA challenge + type, it must be added to this switch explicitly — the `default` case now denies rather + than silently accepting (previously everything funneled through one format-only check). +- **`GenerateTOTPCode` is exported** specifically so integration/SDK-driven tests can + compute the code a real authenticator app would produce for a secret returned by + `AssociateSoftwareToken`, without needing a TOTP library dependency. +- Protocol is `json-1.1` (`X-Amz-Target: AWSCognitoIdentityProviderService.`), not + XML — confirmed via `handler.go`'s `service.HandleTarget` wiring; no XML wrapper traps + apply to this service the way they do to EC2/S3-family query/XML services. +- Token timestamps (`iat`/`exp`/`auth_time`/`UserCreateDate`/etc.) are epoch-seconds JSON + numbers throughout — already correct, no `awstime.Epoch` gap found. diff --git a/services/cognitoidp/accuracy_backend.go b/services/cognitoidp/accuracy_backend.go index 3e7ee93f3..1ad21d598 100644 --- a/services/cognitoidp/accuracy_backend.go +++ b/services/cognitoidp/accuracy_backend.go @@ -61,12 +61,13 @@ type UserPoolOptions struct { // UserPoolClientOptions holds optional parameters for CreateUserPoolClientWithOpts and UpdateUserPoolClientWithOpts. type UserPoolClientOptions struct { TokenValidityUnits map[string]string `json:"tokenValidityUnits,omitempty"` - AllowedOAuthFlows []string `json:"allowedOAuthFlows,omitempty"` - AllowedOAuthScopes []string `json:"allowedOAuthScopes,omitempty"` + PreventUserExistenceErrors string `json:"preventUserExistenceErrors,omitempty"` + SupportedIdentityProviders []string `json:"supportedIdentityProviders,omitempty"` ExplicitAuthFlows []string `json:"explicitAuthFlows,omitempty"` CallbackURLs []string `json:"callbackURLs,omitempty"` LogoutURLs []string `json:"logoutURLs,omitempty"` - SupportedIdentityProviders []string `json:"supportedIdentityProviders,omitempty"` + AllowedOAuthScopes []string `json:"allowedOAuthScopes,omitempty"` + AllowedOAuthFlows []string `json:"allowedOAuthFlows,omitempty"` AccessTokenValidity int32 `json:"accessTokenValidity,omitempty"` IDTokenValidity int32 `json:"idTokenValidity,omitempty"` RefreshTokenValidity int32 `json:"refreshTokenValidity,omitempty"` @@ -75,6 +76,12 @@ type UserPoolClientOptions struct { AllowedOAuthFlowsUserPoolClient bool `json:"allowedOAuthFlowsUserPoolClient,omitempty"` } +// Valid values for UserPoolClient.PreventUserExistenceErrors. +const ( + preventUserExistenceEnabled = "ENABLED" + preventUserExistenceLegacy = "LEGACY" +) + // tokenExpiryFor returns the configured token expiry duration for the given token type // ("AccessToken", "IdToken", "RefreshToken"). Returns 0 when not configured (use default). func tokenExpiryFor(client *UserPoolClient, tokenType string) time.Duration { @@ -129,7 +136,7 @@ func (b *InMemoryBackend) userGroupsLocked(poolID, username string) []string { if _, isMember := members[username]; !isMember { continue } - if g, ok := b.groups[poolID][groupName]; ok { + if g, ok := b.groups.Get(groupKey(poolID, groupName)); ok { matched = append(matched, gp{name: groupName, precedence: g.Precedence}) } } @@ -232,7 +239,7 @@ func (b *InMemoryBackend) CreateUserPoolWithOpts(name string, opts UserPoolOptio b.mu.Lock("CreateUserPoolWithOpts") defer b.mu.Unlock() - if _, ok := b.poolsByName[name]; ok { + if b.poolNameExists(name) { return nil, fmt.Errorf("%w: pool %q already exists", ErrUserPoolAlreadyExists, name) } @@ -261,9 +268,7 @@ func (b *InMemoryBackend) CreateUserPoolWithOpts(name string, opts UserPoolOptio DeletionProtection: opts.DeletionProtection, } - b.pools[poolID] = pool - b.poolsByName[name] = pool - b.users[poolID] = make(map[string]*User) + b.pools.Put(pool) cp := *pool @@ -278,10 +283,15 @@ func (b *InMemoryBackend) CreateUserPoolClientWithOpts( b.mu.Lock("CreateUserPoolClientWithOpts") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } + preventUserExistenceErrors, err := normalizePreventUserExistenceErrors(opts.PreventUserExistenceErrors) + if err != nil { + return nil, err + } + flows := make([]string, len(opts.AllowedOAuthFlows)) copy(flows, opts.AllowedOAuthFlows) scopes := make([]string, len(opts.AllowedOAuthScopes)) @@ -312,6 +322,7 @@ func (b *InMemoryBackend) CreateUserPoolClientWithOpts( CallbackURLs: callbackURLs, LogoutURLs: logoutURLs, SupportedIdentityProviders: supportedIDPs, + PreventUserExistenceErrors: preventUserExistenceErrors, AccessTokenValidity: opts.AccessTokenValidity, IDTokenValidity: opts.IDTokenValidity, RefreshTokenValidity: opts.RefreshTokenValidity, @@ -324,42 +335,35 @@ func (b *InMemoryBackend) CreateUserPoolClientWithOpts( client.ClientSecret = randomAlphanumeric(clientSecretLen) } - b.clients[client.ClientID] = client - if b.clientsByPool[userPoolID] == nil { - b.clientsByPool[userPoolID] = make(map[string]*UserPoolClient) - } - b.clientsByPool[userPoolID][client.ClientID] = client + b.clients.Put(client) cp := *client return &cp, nil } -// UpdateUserPoolClientWithOpts updates app client fields including OAuth flows and scopes. -func (b *InMemoryBackend) UpdateUserPoolClientWithOpts( - userPoolID, clientID, clientName string, - opts UserPoolClientOptions, -) (*UserPoolClient, error) { - b.mu.Lock("UpdateUserPoolClientWithOpts") - defer b.mu.Unlock() - - if _, ok := b.pools[userPoolID]; !ok { - return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) - } - - client, ok := b.clients[clientID] - if !ok { - return nil, fmt.Errorf("%w: client %q not found", ErrClientNotFound, clientID) - } - - if client.UserPoolID != userPoolID { - return nil, fmt.Errorf("%w: client %q does not belong to pool %q", ErrClientNotFound, clientID, userPoolID) - } - - if clientName != "" { - client.ClientName = clientName +// normalizePreventUserExistenceErrors validates and defaults the PreventUserExistenceErrors +// app client setting to AWS's documented values. An unset value defaults to "LEGACY" (the +// AWS default for app clients that don't specify it); anything other than "ENABLED" or +// "LEGACY" is an InvalidParameterException. +func normalizePreventUserExistenceErrors(v string) (string, error) { + switch v { + case "": + return preventUserExistenceLegacy, nil + case preventUserExistenceEnabled, preventUserExistenceLegacy: + return v, nil + default: + return "", fmt.Errorf( + "%w: PreventUserExistenceErrors must be ENABLED or LEGACY, got %q", + ErrInvalidParameter, v, + ) } +} +// applyUserPoolClientListOpts copies each non-nil list field from opts onto client, +// leaving fields the caller omitted (nil) untouched. Split out of +// UpdateUserPoolClientWithOpts to keep that function's branching within lint limits. +func applyUserPoolClientListOpts(client *UserPoolClient, opts UserPoolClientOptions) { if opts.AllowedOAuthFlows != nil { flows := make([]string, len(opts.AllowedOAuthFlows)) copy(flows, opts.AllowedOAuthFlows) @@ -395,6 +399,57 @@ func (b *InMemoryBackend) UpdateUserPoolClientWithOpts( copy(idps, opts.SupportedIdentityProviders) client.SupportedIdentityProviders = idps } +} + +// applyPreventUserExistenceErrorsUpdate updates client's PreventUserExistenceErrors when +// value is non-empty (an UpdateUserPoolClient caller omitting the field leaves the existing +// setting untouched, matching the update-only-what-was-sent semantics of the other opts +// fields in UpdateUserPoolClientWithOpts). +func applyPreventUserExistenceErrorsUpdate(client *UserPoolClient, value string) error { + if value == "" { + return nil + } + + normalized, err := normalizePreventUserExistenceErrors(value) + if err != nil { + return err + } + + client.PreventUserExistenceErrors = normalized + + return nil +} + +// UpdateUserPoolClientWithOpts updates app client fields including OAuth flows and scopes. +func (b *InMemoryBackend) UpdateUserPoolClientWithOpts( + userPoolID, clientID, clientName string, + opts UserPoolClientOptions, +) (*UserPoolClient, error) { + b.mu.Lock("UpdateUserPoolClientWithOpts") + defer b.mu.Unlock() + + if _, ok := b.pools.Get(userPoolID); !ok { + return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) + } + + client, ok := b.clients.Get(clientID) + if !ok { + return nil, fmt.Errorf("%w: client %q not found", ErrClientNotFound, clientID) + } + + if client.UserPoolID != userPoolID { + return nil, fmt.Errorf("%w: client %q does not belong to pool %q", ErrClientNotFound, clientID, userPoolID) + } + + if clientName != "" { + client.ClientName = clientName + } + + applyUserPoolClientListOpts(client, opts) + + if err := applyPreventUserExistenceErrorsUpdate(client, opts.PreventUserExistenceErrors); err != nil { + return nil, err + } client.EnableTokenRevocation = opts.EnableTokenRevocation client.AllowedOAuthFlowsUserPoolClient = opts.AllowedOAuthFlowsUserPoolClient @@ -424,7 +479,7 @@ func (b *InMemoryBackend) UpdateUserPoolWithOpts( b.mu.Lock("UpdateUserPoolWithOpts") defer b.mu.Unlock() - pool, ok := b.pools[userPoolID] + pool, ok := b.pools.Get(userPoolID) if !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } @@ -474,12 +529,12 @@ func (b *InMemoryBackend) SignUpWithValidation( b.mu.Lock("SignUpWithValidation") defer b.mu.Unlock() - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok { return nil, fmt.Errorf("%w: client %q not found", ErrClientNotFound, clientID) } - pool, ok := b.pools[client.UserPoolID] + pool, ok := b.pools.Get(client.UserPoolID) if !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, client.UserPoolID) } @@ -488,8 +543,7 @@ func (b *InMemoryBackend) SignUpWithValidation( return nil, err } - poolUsers := b.users[client.UserPoolID] - if _, exists := poolUsers[username]; exists { + if _, exists := b.users.Get(userKey(client.UserPoolID, username)); exists { return nil, fmt.Errorf("%w: user %q already exists", ErrUsernameExists, username) } @@ -536,8 +590,7 @@ func (b *InMemoryBackend) SignUpWithValidation( ConfirmCodeExpiresAt: confirmExpiry, } - poolUsers[username] = user - b.usersBySub[client.UserPoolID+":"+user.Sub] = username + b.users.Put(user) cp := *user @@ -565,7 +618,7 @@ func (b *InMemoryBackend) RespondToNewPasswordRequired( return nil, fmt.Errorf("%w: session was issued for a different client", ErrNotAuthorized) } - pool, ok := b.pools[entry.PoolID] + pool, ok := b.pools.Get(entry.PoolID) if !ok { return nil, fmt.Errorf("%w: user pool %q not found", ErrUserPoolNotFound, entry.PoolID) } @@ -574,7 +627,7 @@ func (b *InMemoryBackend) RespondToNewPasswordRequired( return nil, err } - user, ok := b.users[entry.PoolID][entry.Username] + user, ok := b.users.Get(userKey(entry.PoolID, entry.Username)) if !ok { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, entry.Username) } @@ -620,8 +673,10 @@ func (b *InMemoryBackend) AssociateSoftwareToken(accessToken string) (string, er return secret, nil } -// VerifySoftwareToken marks the TOTP token as verified. In simulation mode any 6-digit code -// is accepted; the secret was already stored by AssociateSoftwareToken. +// VerifySoftwareToken validates userCode as a real RFC 6238 TOTP code for the secret +// previously issued by AssociateSoftwareToken (HMAC-SHA1, 30s step, +/-1 step clock skew, +// matching an authenticator app such as Google Authenticator/Authy). Only the code that the +// secret actually produces at (approximately) the current time is accepted. func (b *InMemoryBackend) VerifySoftwareToken(accessToken, userCode string) error { b.mu.Lock("VerifySoftwareToken") defer b.mu.Unlock() @@ -645,6 +700,10 @@ func (b *InMemoryBackend) VerifySoftwareToken(accessToken, userCode string) erro } } + if !verifyTOTPCode(user.TOTPSecret, userCode, time.Now()) { + return fmt.Errorf("%w: invalid software token code", ErrCodeMismatch) + } + user.TOTPVerified = true return nil @@ -676,11 +735,11 @@ func (b *InMemoryBackend) AdminSetUserMFASetting( b.mu.Lock("AdminSetUserMFASetting") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - user, ok := b.users[userPoolID][username] + user, ok := b.users.Get(userKey(userPoolID, username)) if !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -742,14 +801,13 @@ func (b *InMemoryBackend) EvictExpiredMFASessions() { continue } - poolUsers, poolExists := b.users[entry.PoolID] - if !poolExists { + if _, poolExists := b.pools.Get(entry.PoolID); !poolExists { delete(b.mfaSessions, token) continue } - if _, userExists := poolUsers[entry.Username]; !userExists { + if _, userExists := b.users.Get(userKey(entry.PoolID, entry.Username)); !userExists { delete(b.mfaSessions, token) } } @@ -765,15 +823,11 @@ func (b *InMemoryBackend) CreateResourceServer( b.mu.Lock("CreateResourceServer") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if b.resourceServers[userPoolID] == nil { - b.resourceServers[userPoolID] = make(map[string]*ResourceServer) - } - - if _, exists := b.resourceServers[userPoolID][identifier]; exists { + if _, exists := b.resourceServers.Get(resourceServerKey(userPoolID, identifier)); exists { return nil, fmt.Errorf( "%w: resource server %q already exists in pool %q", ErrAlreadyExists, @@ -791,7 +845,7 @@ func (b *InMemoryBackend) CreateResourceServer( Name: name, Scopes: scopesCopy, } - b.resourceServers[userPoolID][identifier] = rs + b.resourceServers.Put(rs) cp := *rs @@ -803,11 +857,11 @@ func (b *InMemoryBackend) DescribeResourceServer(userPoolID, identifier string) b.mu.RLock("DescribeResourceServer") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - rs, ok := b.resourceServers[userPoolID][identifier] + rs, ok := b.resourceServers.Get(resourceServerKey(userPoolID, identifier)) if !ok { return nil, fmt.Errorf( "%w: resource server %q not found in pool %q", @@ -827,11 +881,11 @@ func (b *InMemoryBackend) ListResourceServers(userPoolID string) ([]*ResourceSer b.mu.RLock("ListResourceServers") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - poolServers := b.resourceServers[userPoolID] + poolServers := b.resourceServersByPool.Get(userPoolID) out := make([]*ResourceServer, 0, len(poolServers)) for _, rs := range poolServers { @@ -852,11 +906,11 @@ func (b *InMemoryBackend) UpdateResourceServer( b.mu.Lock("UpdateResourceServer") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - rs, ok := b.resourceServers[userPoolID][identifier] + rs, ok := b.resourceServers.Get(resourceServerKey(userPoolID, identifier)) if !ok { return nil, fmt.Errorf( "%w: resource server %q not found in pool %q", @@ -886,11 +940,11 @@ func (b *InMemoryBackend) DeleteResourceServer(userPoolID, identifier string) er b.mu.Lock("DeleteResourceServer") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.resourceServers[userPoolID][identifier]; !ok { + if _, ok := b.resourceServers.Get(resourceServerKey(userPoolID, identifier)); !ok { return fmt.Errorf( "%w: resource server %q not found in pool %q", ErrUserPoolNotFound, @@ -899,7 +953,7 @@ func (b *InMemoryBackend) DeleteResourceServer(userPoolID, identifier string) er ) } - delete(b.resourceServers[userPoolID], identifier) + b.resourceServers.Delete(resourceServerKey(userPoolID, identifier)) return nil } @@ -929,12 +983,12 @@ func (b *InMemoryBackend) RespondToSRPChallenge(clientID, session string) (*Toke return nil, fmt.Errorf("%w: session was issued for a different client", ErrNotAuthorized) } - pool, ok := b.pools[entry.PoolID] + pool, ok := b.pools.Get(entry.PoolID) if !ok { return nil, fmt.Errorf("%w: user pool %q not found", ErrUserPoolNotFound, entry.PoolID) } - user, ok := b.users[entry.PoolID][entry.Username] + user, ok := b.users.Get(userKey(entry.PoolID, entry.Username)) if !ok { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, entry.Username) } @@ -958,17 +1012,12 @@ func (b *InMemoryBackend) AdminCreateUserWithPolicy( b.mu.Lock("AdminCreateUserWithPolicy") defer b.mu.Unlock() - pool, ok := b.pools[userPoolID] - if !ok { - return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) - } - - poolUsers, ok := b.users[userPoolID] + pool, ok := b.pools.Get(userPoolID) if !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, exists := poolUsers[username]; exists { + if _, exists := b.users.Get(userKey(userPoolID, username)); exists { return nil, fmt.Errorf("%w: user %q already exists", ErrUserAlreadyExists, username) } @@ -1002,8 +1051,7 @@ func (b *InMemoryBackend) AdminCreateUserWithPolicy( Enabled: true, } - poolUsers[username] = user - b.usersBySub[userPoolID+":"+user.Sub] = username + b.users.Put(user) cp := *user @@ -1018,7 +1066,7 @@ func (b *InMemoryBackend) ValidateSecretHash(clientID, username, providedHash st b.mu.RLock("ValidateSecretHash") defer b.mu.RUnlock() - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok { return fmt.Errorf("%w: client %q not found", ErrClientNotFound, clientID) } diff --git a/services/cognitoidp/accuracy_handler.go b/services/cognitoidp/accuracy_handler.go index 50522bb96..feb6dd2cc 100644 --- a/services/cognitoidp/accuracy_handler.go +++ b/services/cognitoidp/accuracy_handler.go @@ -690,6 +690,7 @@ type clientDataAccurate struct { ClientName string `json:"ClientName,omitempty"` UserPoolID string `json:"UserPoolId,omitempty"` ClientSecret string `json:"ClientSecret,omitempty"` + PreventUserExistenceErrors string `json:"PreventUserExistenceErrors,omitempty"` AllowedOAuthFlows []string `json:"AllowedOAuthFlows,omitempty"` AllowedOAuthScopes []string `json:"AllowedOAuthScopes,omitempty"` ExplicitAuthFlows []string `json:"ExplicitAuthFlows,omitempty"` @@ -725,6 +726,7 @@ func clientToAccurateData(c *UserPoolClient) clientDataAccurate { ClientName: c.ClientName, UserPoolID: c.UserPoolID, ClientSecret: c.ClientSecret, + PreventUserExistenceErrors: c.PreventUserExistenceErrors, AllowedOAuthFlows: flows, AllowedOAuthScopes: scopes, ExplicitAuthFlows: ef, @@ -797,6 +799,7 @@ type createUserPoolClientWithOptsInput struct { TokenValidityUnits map[string]string `json:"TokenValidityUnits,omitempty"` UserPoolID string `json:"UserPoolId,omitempty"` ClientName string `json:"ClientName,omitempty"` + PreventUserExistenceErrors string `json:"PreventUserExistenceErrors,omitempty"` AllowedOAuthFlows []string `json:"AllowedOAuthFlows,omitempty"` AllowedOAuthScopes []string `json:"AllowedOAuthScopes,omitempty"` ExplicitAuthFlows []string `json:"ExplicitAuthFlows,omitempty"` @@ -822,6 +825,7 @@ func (h *Handler) handleCreateUserPoolClientWithOpts( opts := UserPoolClientOptions{ AllowedOAuthFlows: in.AllowedOAuthFlows, AllowedOAuthScopes: in.AllowedOAuthScopes, + PreventUserExistenceErrors: in.PreventUserExistenceErrors, ExplicitAuthFlows: in.ExplicitAuthFlows, CallbackURLs: in.CallbackURLs, LogoutURLs: in.LogoutURLs, @@ -850,6 +854,7 @@ type updateUserPoolClientWithOptsInput struct { UserPoolID string `json:"UserPoolId,omitempty"` ClientID string `json:"ClientId,omitempty"` ClientName string `json:"ClientName,omitempty"` + PreventUserExistenceErrors string `json:"PreventUserExistenceErrors,omitempty"` AllowedOAuthFlows []string `json:"AllowedOAuthFlows,omitempty"` AllowedOAuthScopes []string `json:"AllowedOAuthScopes,omitempty"` ExplicitAuthFlows []string `json:"ExplicitAuthFlows,omitempty"` @@ -878,6 +883,7 @@ func (h *Handler) handleUpdateUserPoolClientWithOpts( CallbackURLs: in.CallbackURLs, LogoutURLs: in.LogoutURLs, SupportedIdentityProviders: in.SupportedIdentityProviders, + PreventUserExistenceErrors: in.PreventUserExistenceErrors, EnableTokenRevocation: in.EnableTokenRevocation, AllowedOAuthFlowsUserPoolClient: in.AllowedOAuthFlowsUserPoolClient, AccessTokenValidity: in.AccessTokenValidity, diff --git a/services/cognitoidp/accuracy_test.go b/services/cognitoidp/accuracy_test.go index 25df4c64d..5f4c3c8b4 100644 --- a/services/cognitoidp/accuracy_test.go +++ b/services/cognitoidp/accuracy_test.go @@ -10,6 +10,7 @@ import ( "net/http" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -527,7 +528,10 @@ func TestAccuracy_SoftwareToken_PersistsSecret(t *testing.T) { assert.NotEmpty(t, secret) assert.Greater(t, len(secret), 16) - err = b.VerifySoftwareToken(tokens.AccessToken, "123456") + code, err := cognitoidp.GenerateTOTPCode(secret, time.Now()) + require.NoError(t, err) + + err = b.VerifySoftwareToken(tokens.AccessToken, code) require.NoError(t, err) user, err := b.GetUser(tokens.AccessToken) @@ -783,16 +787,37 @@ func TestHandler_RespondToAuthChallenge_SMSMFAFlow(t *testing.T) { h := newTestHandler(t) poolID, clientID := setupHandlerPoolAndClient(t, h, "sms-mfa-pool") + signUpAndConfirmViaHandler(t, h, clientID, "smsuser") + + // Log in while MFA is still off to get an access token, then enroll a real TOTP + // secret — a real client must complete software-token setup (AssociateSoftwareToken + + // VerifySoftwareToken) before the pool can challenge for SOFTWARE_TOKEN_MFA. + accessToken := loginViaHandler(t, h, clientID, "smsuser") + + assocRec := doCognitoRequest(t, h, "AssociateSoftwareToken", map[string]any{"AccessToken": accessToken}) + require.Equal(t, http.StatusOK, assocRec.Code) + + var assocResp struct { + SecretCode string `json:"SecretCode,omitempty"` + } + require.NoError(t, json.Unmarshal(assocRec.Body.Bytes(), &assocResp)) + require.NotEmpty(t, assocResp.SecretCode) + + setupCode, err := cognitoidp.GenerateTOTPCode(assocResp.SecretCode, time.Now()) + require.NoError(t, err) + + verifyRec := doCognitoRequest(t, h, "VerifySoftwareToken", map[string]any{ + "AccessToken": accessToken, + "UserCode": setupCode, + }) + require.Equal(t, http.StatusOK, verifyRec.Code) + mfaRec := doCognitoRequest(t, h, "SetUserPoolMfaConfig", map[string]any{ "UserPoolId": poolID, "MfaConfiguration": "ON", }) require.Equal(t, http.StatusOK, mfaRec.Code) - signUpAndConfirmViaHandler(t, h, clientID, "smsuser") - - // Get access token without MFA to set preference (need direct backend). - // We do this via admin set password which doesn't trigger MFA. setMFARec := doCognitoRequest(t, h, "AdminInitiateAuth", map[string]any{ "UserPoolId": poolID, "ClientId": clientID, @@ -810,14 +835,19 @@ func TestHandler_RespondToAuthChallenge_SMSMFAFlow(t *testing.T) { } require.NoError(t, json.Unmarshal(setMFARec.Body.Bytes(), &mfaInitResp)) require.NotNil(t, mfaInitResp.ChallengeName) + require.Equal(t, "SOFTWARE_TOKEN_MFA", *mfaInitResp.ChallengeName) + + // Respond to SOFTWARE_TOKEN_MFA (default when MFA is ON) with the real TOTP code + // derived from the enrolled secret — a stray "123456" must now be rejected. + challengeCode, err := cognitoidp.GenerateTOTPCode(assocResp.SecretCode, time.Now()) + require.NoError(t, err) - // Respond to SOFTWARE_TOKEN_MFA (default when MFA is ON). respondRec := doCognitoRequest(t, h, "RespondToAuthChallenge", map[string]any{ "ClientId": clientID, "ChallengeName": *mfaInitResp.ChallengeName, "Session": *mfaInitResp.Session, "ChallengeResponses": map[string]string{ - "SOFTWARE_TOKEN_MFA_CODE": "123456", + "SOFTWARE_TOKEN_MFA_CODE": challengeCode, }, }) require.Equal(t, http.StatusOK, respondRec.Code) @@ -1114,9 +1144,18 @@ func TestHandler_VerifySoftwareToken_Accurate(t *testing.T) { assocRec := doCognitoRequest(t, h, "AssociateSoftwareToken", map[string]any{"AccessToken": accessToken}) require.Equal(t, http.StatusOK, assocRec.Code) + var assocResp struct { + SecretCode string `json:"SecretCode,omitempty"` + } + require.NoError(t, json.Unmarshal(assocRec.Body.Bytes(), &assocResp)) + require.NotEmpty(t, assocResp.SecretCode) + + code, err := cognitoidp.GenerateTOTPCode(assocResp.SecretCode, time.Now()) + require.NoError(t, err) + rec := doCognitoRequest(t, h, "VerifySoftwareToken", map[string]any{ "AccessToken": accessToken, - "UserCode": "123456", + "UserCode": code, }) require.Equal(t, http.StatusOK, rec.Code) diff --git a/services/cognitoidp/backend.go b/services/cognitoidp/backend.go index ee92230b8..87434abca 100644 --- a/services/cognitoidp/backend.go +++ b/services/cognitoidp/backend.go @@ -1,6 +1,7 @@ package cognitoidp import ( + "crypto/hmac" "crypto/rand" "crypto/rsa" "errors" @@ -17,6 +18,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) const ( @@ -75,6 +77,10 @@ type PasswordPolicy struct { TemporaryPasswordValidityDays int `json:"TemporaryPasswordValidityDays,omitempty"` } +// deletionProtectionActive is the UserPool.DeletionProtection value that makes +// DeleteUserPool refuse to delete the pool. +const deletionProtectionActive = "ACTIVE" + // UserPool represents a Cognito User Pool. type UserPool struct { CreatedAt time.Time `json:"createdAt"` @@ -102,12 +108,13 @@ type UserPoolClient struct { ClientName string `json:"clientName,omitempty"` UserPoolID string `json:"userPoolId,omitempty"` ClientSecret string `json:"clientSecret,omitempty"` - AllowedOAuthFlows []string `json:"allowedOAuthFlows,omitempty"` + PreventUserExistenceErrors string `json:"preventUserExistenceErrors,omitempty"` AllowedOAuthScopes []string `json:"allowedOAuthScopes,omitempty"` ExplicitAuthFlows []string `json:"explicitAuthFlows,omitempty"` CallbackURLs []string `json:"callbackURLs,omitempty"` LogoutURLs []string `json:"logoutURLs,omitempty"` SupportedIdentityProviders []string `json:"supportedIdentityProviders,omitempty"` + AllowedOAuthFlows []string `json:"allowedOAuthFlows,omitempty"` AccessTokenValidity int32 `json:"accessTokenValidity,omitempty"` IDTokenValidity int32 `json:"idTokenValidity,omitempty"` RefreshTokenValidity int32 `json:"refreshTokenValidity,omitempty"` @@ -149,16 +156,29 @@ type Group struct { } // InMemoryBackend is the in-memory store for Cognito IDP resources. +// +// Most resource collections are *store.Table[T] registered on b.registry (see +// store_setup.go for keyFns/composite keys and the full per-field rationale). +// A handful of fields remain plain maps because their value carries no pure +// identity for a store.Table key; store_setup.go's registerAllTables doc +// comment lists each one and why. type InMemoryBackend struct { - mu *lockmetrics.RWMutex - pools map[string]*UserPool - poolsByName map[string]*UserPool - clients map[string]*UserPoolClient - // clientsByPool maps userPoolID -> clientID -> client for efficient per-pool listing and deletes. - clientsByPool map[string]map[string]*UserPoolClient - users map[string]map[string]*User - // usersBySub maps poolID+":"+sub -> username for O(1) access token resolution. - usersBySub map[string]string + mu *lockmetrics.RWMutex + registry *store.Registry + pools *store.Table[UserPool] + // poolsByName is a secondary index on pools keyed by Name. + poolsByName *store.Index[UserPool] + clients *store.Table[UserPoolClient] + // clientsByPool is a secondary index on clients keyed by UserPoolID, for + // efficient per-pool listing and deletes. + clientsByPool *store.Index[UserPoolClient] + // users is keyed by the composite userKey(poolID, username). + users *store.Table[User] + // usersByPool is a secondary index on users keyed by UserPoolID. + usersByPool *store.Index[User] + // usersBySub is a secondary index on users keyed by userSubKey(poolID, + // sub), for O(1) access token resolution. + usersBySub *store.Index[User] // refreshTokens maps refresh token → poolID/username for REFRESH_TOKEN_AUTH flow. refreshTokens map[string]*refreshTokenEntry // refreshTokensByClient maps clientID -> refreshToken set for efficient client cleanup. @@ -167,39 +187,49 @@ type InMemoryBackend struct { refreshTokensByUser map[string]map[string]struct{} // mfaSessions maps session token → pending challenge context (MFA or NEW_PASSWORD_REQUIRED). mfaSessions map[string]*mfaSessionEntry - // groups maps poolID → groupName → Group - groups map[string]map[string]*Group + // groups is keyed by the composite groupKey(poolID, groupName). + groups *store.Table[Group] + // groupsByPool is a secondary index on groups keyed by UserPoolID. + groupsByPool *store.Index[Group] // groupMembers maps poolID → groupName → set of usernames groupMembers map[string]map[string]map[string]struct{} - // resourceServers maps poolID → identifier → ResourceServer - resourceServers map[string]map[string]*ResourceServer + // resourceServers is keyed by the composite resourceServerKey(poolID, identifier). + resourceServers *store.Table[ResourceServer] + // resourceServersByPool is a secondary index on resourceServers keyed by UserPoolID. + resourceServersByPool *store.Index[ResourceServer] // tokenRevokedBefore maps poolID+":"+username → revocation time for GlobalSignOut. // Access tokens with auth_time before this timestamp are rejected. tokenRevokedBefore map[string]time.Time - // identityProviders maps poolID → providerName → IdentityProvider - identityProviders map[string]map[string]*IdentityProvider + // identityProviders is keyed by the composite identityProviderKey(poolID, providerName). + identityProviders *store.Table[IdentityProvider] + // identityProvidersByPool is a secondary index on identityProviders keyed by UserPoolID. + identityProvidersByPool *store.Index[IdentityProvider] // domains maps domain → UserPoolDomain (domain names are globally unique in Cognito) - domains map[string]*UserPoolDomain + domains *store.Table[UserPoolDomain] // resourceTags maps ARN → tag key → tag value resourceTags map[string]map[string]string // riskConfigurations maps poolID+":"+clientID → RiskConfiguration (clientID="" for pool-level) riskConfigurations map[string]*RiskConfiguration // logDeliveryConfigs maps poolID → LogDeliveryConfig logDeliveryConfigs map[string]*LogDeliveryConfig - // uiCustomizations maps poolID+":"+clientID → UICustomization - uiCustomizations map[string]*UICustomization - // managedLoginBrandings maps poolID → brandingID → ManagedLoginBranding - managedLoginBrandings map[string]map[string]*ManagedLoginBranding + // uiCustomizations is keyed by the composite uiKey(poolID, clientID). + uiCustomizations *store.Table[UICustomization] + // managedLoginBrandings is keyed by the composite managedLoginBrandingKey(poolID, brandingID). + managedLoginBrandings *store.Table[ManagedLoginBranding] + // managedLoginBrandingsByPool is a secondary index on managedLoginBrandings keyed by UserPoolID. + managedLoginBrandingsByPool *store.Index[ManagedLoginBranding] // terms maps poolID → Terms - terms map[string]*Terms - // userImportJobs maps poolID → jobID → UserImportJob - userImportJobs map[string]map[string]*UserImportJob + terms *store.Table[Terms] + // userImportJobs is keyed by the composite userImportJobKey(poolID, jobID). + userImportJobs *store.Table[UserImportJob] + // userImportJobsByPool is a secondary index on userImportJobs keyed by UserPoolID. + userImportJobsByPool *store.Index[UserImportJob] // poolMfaConfigs maps poolID → full MFA config (SMS/TOTP/Email sub-configs) poolMfaConfigs map[string]*UserPoolMfaFullConfig // attrVerificationCodes maps poolID+":"+username+":"+attrName → pending verification entry attrVerificationCodes map[string]*attrVerificationEntry // typedRiskConfigurations maps poolID+":"+clientID → typed risk configuration - typedRiskConfigurations map[string]*TypedRiskConfiguration + typedRiskConfigurations *store.Table[TypedRiskConfiguration] // devices maps poolID+":"+username → deviceKey → *Device (device tracking / "remember this device"). devices map[string]map[string]*Device // webauthnCredentials maps poolID+":"+username → credentialID → *WebAuthnCredential. @@ -235,6 +265,12 @@ type mfaSessionEntry struct { ChallengeType string `json:"challengeType,omitempty"` // "SOFTWARE_TOKEN_MFA", "NEW_PASSWORD_REQUIRED" ... // SRPPassword holds the user's password for USER_SRP_AUTH second-step validation. SRPPassword string `json:"srpPassword,omitempty"` + // Code holds the one-time code generated for SMS_MFA/EMAIL_OTP challenges. Unlike + // SOFTWARE_TOKEN_MFA (verified cryptographically against the user's TOTP secret), SMS + // and email codes have no client-held secret to re-derive from, so — exactly like + // ForgotPassword/ConfirmSignUp confirmation codes elsewhere in this backend — the code + // is generated once here and the challenge response must match it exactly. + Code string `json:"code,omitempty"` } // AuthResult is the result of a successful authentication or a pending challenge. @@ -255,41 +291,31 @@ const totpCodeLen = 6 // NewInMemoryBackend creates a new InMemoryBackend. func NewInMemoryBackend(accountID, region, endpoint string) *InMemoryBackend { - return &InMemoryBackend{ - mu: lockmetrics.New("cognitoidp"), - pools: make(map[string]*UserPool), - poolsByName: make(map[string]*UserPool), - clients: make(map[string]*UserPoolClient), - clientsByPool: make(map[string]map[string]*UserPoolClient), - users: make(map[string]map[string]*User), - usersBySub: make(map[string]string), - refreshTokens: make(map[string]*refreshTokenEntry), - refreshTokensByClient: make(map[string]map[string]struct{}), - refreshTokensByUser: make(map[string]map[string]struct{}), - mfaSessions: make(map[string]*mfaSessionEntry), - groups: make(map[string]map[string]*Group), - groupMembers: make(map[string]map[string]map[string]struct{}), - resourceServers: make(map[string]map[string]*ResourceServer), - tokenRevokedBefore: make(map[string]time.Time), - identityProviders: make(map[string]map[string]*IdentityProvider), - domains: make(map[string]*UserPoolDomain), - resourceTags: make(map[string]map[string]string), - riskConfigurations: make(map[string]*RiskConfiguration), - logDeliveryConfigs: make(map[string]*LogDeliveryConfig), - uiCustomizations: make(map[string]*UICustomization), - managedLoginBrandings: make(map[string]map[string]*ManagedLoginBranding), - terms: make(map[string]*Terms), - userImportJobs: make(map[string]map[string]*UserImportJob), - poolMfaConfigs: make(map[string]*UserPoolMfaFullConfig), - attrVerificationCodes: make(map[string]*attrVerificationEntry), - typedRiskConfigurations: make(map[string]*TypedRiskConfiguration), - devices: make(map[string]map[string]*Device), - webauthnCredentials: make(map[string]map[string]*WebAuthnCredential), - authEvents: make(map[string]map[string]*AuthEvent), - accountID: accountID, - region: region, - endpoint: endpoint, - } + b := &InMemoryBackend{ + mu: lockmetrics.New("cognitoidp"), + registry: store.NewRegistry(), + refreshTokens: make(map[string]*refreshTokenEntry), + refreshTokensByClient: make(map[string]map[string]struct{}), + refreshTokensByUser: make(map[string]map[string]struct{}), + mfaSessions: make(map[string]*mfaSessionEntry), + groupMembers: make(map[string]map[string]map[string]struct{}), + tokenRevokedBefore: make(map[string]time.Time), + resourceTags: make(map[string]map[string]string), + riskConfigurations: make(map[string]*RiskConfiguration), + logDeliveryConfigs: make(map[string]*LogDeliveryConfig), + poolMfaConfigs: make(map[string]*UserPoolMfaFullConfig), + attrVerificationCodes: make(map[string]*attrVerificationEntry), + devices: make(map[string]map[string]*Device), + webauthnCredentials: make(map[string]map[string]*WebAuthnCredential), + authEvents: make(map[string]map[string]*AuthEvent), + accountID: accountID, + region: region, + endpoint: endpoint, + } + + registerAllTables(b) + + return b } // CreateUserPool creates a new user pool with the given name. @@ -297,7 +323,7 @@ func (b *InMemoryBackend) CreateUserPool(name string) (*UserPool, error) { b.mu.Lock("CreateUserPool") defer b.mu.Unlock() - if _, ok := b.poolsByName[name]; ok { + if b.poolNameExists(name) { return nil, fmt.Errorf("%w: pool %q already exists", ErrUserPoolAlreadyExists, name) } @@ -317,9 +343,7 @@ func (b *InMemoryBackend) CreateUserPool(name string) (*UserPool, error) { issuer: issuer, } - b.pools[poolID] = pool - b.poolsByName[name] = pool - b.users[poolID] = make(map[string]*User) + b.pools.Put(pool) cp := *pool @@ -331,7 +355,7 @@ func (b *InMemoryBackend) DescribeUserPool(userPoolID string) (*UserPool, error) b.mu.RLock("DescribeUserPool") defer b.mu.RUnlock() - pool, ok := b.pools[userPoolID] + pool, ok := b.pools.Get(userPoolID) if !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } @@ -346,32 +370,40 @@ func (b *InMemoryBackend) DeleteUserPool(userPoolID string) error { b.mu.Lock("DeleteUserPool") defer b.mu.Unlock() - pool, ok := b.pools[userPoolID] + pool, ok := b.pools.Get(userPoolID) if !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - delete(b.poolsByName, pool.Name) - delete(b.pools, userPoolID) - - // Clean up usersBySub entries for all users in this pool before deleting users. - for _, u := range b.users[userPoolID] { - delete(b.usersBySub, userPoolID+":"+u.Sub) + // AWS refuses to delete a user pool with deletion protection ACTIVE; the caller must + // first UpdateUserPool to set DeletionProtection back to INACTIVE. + if pool.DeletionProtection == deletionProtectionActive { + return fmt.Errorf( + "%w: User pool cannot be deleted because deletion protection is activated, "+ + "set deletion protection to INACTIVE and retry to delete the user pool", + ErrInvalidParameter, + ) } - delete(b.users, userPoolID) + b.pools.Delete(userPoolID) - if poolClients, found := b.clientsByPool[userPoolID]; found { - for clientID := range poolClients { - delete(b.clients, clientID) - b.deleteRefreshTokensForClientAndUserIndexLocked(clientID) - } + // Copy the index result before deleting: Delete mutates the byPool index's + // backing slice in place, so ranging directly over it while deleting would + // skip entries. + for _, u := range slices.Clone(b.usersByPool.Get(userPoolID)) { + b.users.Delete(userKey(userPoolID, u.Username)) } - delete(b.clientsByPool, userPoolID) + for _, client := range slices.Clone(b.clientsByPool.Get(userPoolID)) { + b.clients.Delete(client.ClientID) + b.deleteRefreshTokensForClientAndUserIndexLocked(client.ClientID) + } // Clean up groups and group memberships for this pool. - delete(b.groups, userPoolID) + for _, g := range slices.Clone(b.groupsByPool.Get(userPoolID)) { + b.groups.Delete(groupKey(userPoolID, g.GroupName)) + } + delete(b.groupMembers, userPoolID) return nil @@ -383,7 +415,7 @@ func (b *InMemoryBackend) DeleteUserPoolClient(userPoolID, clientID string) erro b.mu.Lock("DeleteUserPoolClient") defer b.mu.Unlock() - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok { return fmt.Errorf("%w: client %q not found", ErrClientNotFound, clientID) } @@ -392,13 +424,7 @@ func (b *InMemoryBackend) DeleteUserPoolClient(userPoolID, clientID string) erro return fmt.Errorf("%w: client %q not found in pool %q", ErrClientNotFound, clientID, userPoolID) } - delete(b.clients, clientID) - if poolClients, found := b.clientsByPool[client.UserPoolID]; found { - delete(poolClients, clientID) - if len(poolClients) == 0 { - delete(b.clientsByPool, client.UserPoolID) - } - } + b.clients.Delete(clientID) // Clean up any refresh tokens issued by this client to prevent leaks. b.deleteRefreshTokensForClientAndUserIndexLocked(clientID) @@ -411,8 +437,10 @@ func (b *InMemoryBackend) ListUserPools() []*UserPool { b.mu.RLock("ListUserPools") defer b.mu.RUnlock() - out := make([]*UserPool, 0, len(b.pools)) - for _, p := range b.pools { + pools := b.pools.All() + out := make([]*UserPool, 0, len(pools)) + + for _, p := range pools { cp := *p out = append(out, &cp) } @@ -427,11 +455,11 @@ func (b *InMemoryBackend) ListUserPoolClients(userPoolID string) ([]*UserPoolCli b.mu.RLock("ListUserPoolClients") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - poolClients := b.clientsByPool[userPoolID] + poolClients := b.clientsByPool.Get(userPoolID) out := make([]*UserPoolClient, 0, len(poolClients)) for _, c := range poolClients { @@ -449,7 +477,7 @@ func (b *InMemoryBackend) CreateUserPoolClient(userPoolID, clientName string) (* b.mu.Lock("CreateUserPoolClient") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } @@ -460,11 +488,7 @@ func (b *InMemoryBackend) CreateUserPoolClient(userPoolID, clientName string) (* CreatedAt: time.Now(), } - b.clients[client.ClientID] = client - if b.clientsByPool[userPoolID] == nil { - b.clientsByPool[userPoolID] = make(map[string]*UserPoolClient) - } - b.clientsByPool[userPoolID][client.ClientID] = client + b.clients.Put(client) cp := *client @@ -476,7 +500,7 @@ func (b *InMemoryBackend) DescribeUserPoolClient(userPoolID, clientID string) (* b.mu.RLock("DescribeUserPoolClient") defer b.mu.RUnlock() - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok { return nil, fmt.Errorf("%w: client %q not found", ErrClientNotFound, clientID) } @@ -495,17 +519,16 @@ func (b *InMemoryBackend) SignUp(clientID, username, password string, userAttrib b.mu.Lock("SignUp") defer b.mu.Unlock() - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok { return nil, fmt.Errorf("%w: client %q not found", ErrClientNotFound, clientID) } - poolUsers, ok := b.users[client.UserPoolID] - if !ok { + if _, poolOK := b.pools.Get(client.UserPoolID); !poolOK { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, client.UserPoolID) } - if _, exists := poolUsers[username]; exists { + if _, exists := b.users.Get(userKey(client.UserPoolID, username)); exists { return nil, fmt.Errorf("%w: user %q already exists", ErrUsernameExists, username) } @@ -532,8 +555,7 @@ func (b *InMemoryBackend) SignUp(clientID, username, password string, userAttrib ConfirmCodeExpiresAt: time.Now().Add(confirmCodeTTL), } - poolUsers[username] = user - b.usersBySub[client.UserPoolID+":"+user.Sub] = username + b.users.Put(user) cp := *user @@ -545,17 +567,16 @@ func (b *InMemoryBackend) ConfirmSignUp(clientID, username, confirmationCode str b.mu.Lock("ConfirmSignUp") defer b.mu.Unlock() - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok { return fmt.Errorf("%w: client %q not found", ErrClientNotFound, clientID) } - poolUsers, ok := b.users[client.UserPoolID] - if !ok { + if _, poolOK := b.pools.Get(client.UserPoolID); !poolOK { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, client.UserPoolID) } - user, ok := poolUsers[username] + user, ok := b.users.Get(userKey(client.UserPoolID, username)) if !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -611,22 +632,17 @@ func (b *InMemoryBackend) AdminInitiateAuth( b.mu.Lock("AdminInitiateAuth") defer b.mu.Unlock() - pool, ok := b.pools[userPoolID] + pool, ok := b.pools.Get(userPoolID) if !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok || client.UserPoolID != userPoolID { return nil, fmt.Errorf("%w: client %q not found in pool %q", ErrClientNotFound, clientID, userPoolID) } - poolUsers, ok := b.users[userPoolID] - if !ok { - return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) - } - - user, ok := poolUsers[username] + user, ok := b.users.Get(userKey(userPoolID, username)) if !ok { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -642,16 +658,11 @@ func (b *InMemoryBackend) AdminCreateUser( b.mu.Lock("AdminCreateUser") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - poolUsers, ok := b.users[userPoolID] - if !ok { - return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) - } - - if _, exists := poolUsers[username]; exists { + if _, exists := b.users.Get(userKey(userPoolID, username)); exists { return nil, fmt.Errorf("%w: user %q already exists", ErrUserAlreadyExists, username) } @@ -681,8 +692,7 @@ func (b *InMemoryBackend) AdminCreateUser( Enabled: true, } - poolUsers[username] = user - b.usersBySub[userPoolID+":"+user.Sub] = username + b.users.Put(user) cp := *user @@ -694,17 +704,12 @@ func (b *InMemoryBackend) AdminSetUserPassword(userPoolID, username, password st b.mu.Lock("AdminSetUserPassword") defer b.mu.Unlock() - pool, ok := b.pools[userPoolID] - if !ok { - return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) - } - - poolUsers, ok := b.users[userPoolID] + pool, ok := b.pools.Get(userPoolID) if !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - user, ok := poolUsers[username] + user, ok := b.users.Get(userKey(userPoolID, username)) if !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -737,12 +742,11 @@ func (b *InMemoryBackend) AdminConfirmSignUp(userPoolID, username string) error b.mu.Lock("AdminConfirmSignUp") defer b.mu.Unlock() - poolUsers, ok := b.users[userPoolID] - if !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - user, ok := poolUsers[username] + user, ok := b.users.Get(userKey(userPoolID, username)) if !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -758,12 +762,11 @@ func (b *InMemoryBackend) AdminGetUser(userPoolID, username string) (*User, erro b.mu.RLock("AdminGetUser") defer b.mu.RUnlock() - poolUsers, ok := b.users[userPoolID] - if !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - user, ok := poolUsers[username] + user, ok := b.users.Get(userKey(userPoolID, username)) if !ok { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -778,18 +781,15 @@ func (b *InMemoryBackend) AdminDeleteUser(userPoolID, username string) error { b.mu.Lock("AdminDeleteUser") defer b.mu.Unlock() - poolUsers, ok := b.users[userPoolID] - if !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok2 := poolUsers[username]; !ok2 { + if _, ok := b.users.Get(userKey(userPoolID, username)); !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } - u := poolUsers[username] - delete(b.usersBySub, userPoolID+":"+u.Sub) - delete(poolUsers, username) + b.users.Delete(userKey(userPoolID, username)) b.deleteRefreshTokensForUserLocked(userPoolID, username) @@ -801,11 +801,11 @@ func (b *InMemoryBackend) ListUsers(userPoolID string) ([]*User, error) { b.mu.RLock("ListUsers") defer b.mu.RUnlock() - poolUsers, ok := b.users[userPoolID] - if !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } + poolUsers := b.usersByPool.Get(userPoolID) out := make([]*User, 0, len(poolUsers)) for _, u := range poolUsers { @@ -825,17 +825,16 @@ func (b *InMemoryBackend) ForgotPassword(clientID, username string) (string, err b.mu.Lock("ForgotPassword") defer b.mu.Unlock() - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok { return "", fmt.Errorf("%w: client %q not found", ErrClientNotFound, clientID) } - poolUsers, ok := b.users[client.UserPoolID] - if !ok { + if _, poolOK := b.pools.Get(client.UserPoolID); !poolOK { return "", fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, client.UserPoolID) } - user, ok := poolUsers[username] + user, ok := b.users.Get(userKey(client.UserPoolID, username)) if !ok { return "", fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -864,17 +863,16 @@ func (b *InMemoryBackend) ConfirmForgotPassword(clientID, username, code, newPas b.mu.Lock("ConfirmForgotPassword") defer b.mu.Unlock() - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok { return fmt.Errorf("%w: client %q not found", ErrClientNotFound, clientID) } - poolUsers, ok := b.users[client.UserPoolID] - if !ok { + if _, poolOK := b.pools.Get(client.UserPoolID); !poolOK { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, client.UserPoolID) } - user, ok := poolUsers[username] + user, ok := b.users.Get(userKey(client.UserPoolID, username)) if !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -887,7 +885,7 @@ func (b *InMemoryBackend) ConfirmForgotPassword(clientID, username, code, newPas return fmt.Errorf("%w: invalid reset code", ErrCodeMismatch) } - pool, ok2 := b.pools[client.UserPoolID] + pool, ok2 := b.pools.Get(client.UserPoolID) if ok2 { if err2 := validatePassword(pool.PasswordPolicy, newPassword); err2 != nil { return err2 @@ -938,7 +936,7 @@ func (b *InMemoryBackend) ChangePassword(accessToken, previousPassword, proposed return fmt.Errorf("%w: previous password is incorrect", ErrNotAuthorized) } - if pool, ok := b.pools[u.UserPoolID]; ok { + if pool, ok := b.pools.Get(u.UserPoolID); ok { if err3 := validatePassword(pool.PasswordPolicy, proposedPassword); err3 != nil { return err3 } @@ -958,7 +956,7 @@ func (b *InMemoryBackend) ChangePassword(accessToken, previousPassword, proposed // It uses the usersBySub secondary index for O(1) lookup after JWT parsing. // The caller must hold b.mu (either read or write lock). func (b *InMemoryBackend) findUserByAccessTokenLocked(accessToken string) (*User, error) { - for _, pool := range b.pools { + for _, pool := range b.pools.All() { claims, err := pool.issuer.ParseAccessToken(accessToken) if err != nil { continue @@ -970,24 +968,19 @@ func (b *InMemoryBackend) findUserByAccessTokenLocked(accessToken string) (*User } // O(1) lookup via secondary index. - username, found := b.usersBySub[pool.ID+":"+sub] + u, found := b.userBySub(pool.ID, sub) if !found { continue } // Check per-user token revocation: reject tokens issued before GlobalSignOut. - if revokedBefore, ok2 := b.tokenRevokedBefore[pool.ID+":"+username]; ok2 { + if revokedBefore, ok2 := b.tokenRevokedBefore[pool.ID+":"+u.Username]; ok2 { authTime, _ := claims["auth_time"].(float64) if time.Unix(int64(authTime), 0).Before(revokedBefore) { continue } } - u, ok := b.users[pool.ID][username] - if !ok { - continue - } - return u, nil } @@ -1001,7 +994,7 @@ func (b *InMemoryBackend) GetSigningCertificate(userPoolID string) (string, erro b.mu.RLock("GetSigningCertificate") defer b.mu.RUnlock() - pool, ok := b.pools[userPoolID] + pool, ok := b.pools.Get(userPoolID) if !ok { return "", fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } @@ -1019,7 +1012,7 @@ func (b *InMemoryBackend) GetUserPoolJWKS(userPoolID string) (*JWKSResponse, err b.mu.RLock("GetUserPoolJWKS") defer b.mu.RUnlock() - pool, ok := b.pools[userPoolID] + pool, ok := b.pools.Get(userPoolID) if !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } @@ -1042,7 +1035,7 @@ func (b *InMemoryBackend) GetJWTPublicKey(issuerURL, kid string) (*rsa.PublicKey b.mu.RLock("GetJWTPublicKey") defer b.mu.RUnlock() - for _, pool := range b.pools { + for _, pool := range b.pools.All() { if pool.issuer == nil || pool.issuer.issuerURL != issuerURL { continue } @@ -1058,30 +1051,45 @@ func (b *InMemoryBackend) GetJWTPublicKey(issuerURL, kid string) (*rsa.PublicKey return nil, ErrJWTIssuerUnknown } -// findUserByClientID finds a user and their pool using the clientID. +// findUserByClientID finds a user and their pool using the clientID. This backs the +// non-admin InitiateAuth path, so an unknown username is reported via +// unknownUserAuthError, which honors the client's PreventUserExistenceErrors setting. +// AdminInitiateAuth looks users up separately and always reveals UserNotFoundException, +// matching AWS (existence-error masking only applies to the non-admin, unauthenticated API). // Caller must hold at least a read lock. func (b *InMemoryBackend) findUserByClientID(clientID, username string) (*User, *UserPool, error) { - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok { return nil, nil, fmt.Errorf("%w: client %q not found", ErrClientNotFound, clientID) } - pool, ok := b.pools[client.UserPoolID] + pool, ok := b.pools.Get(client.UserPoolID) if !ok { return nil, nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, client.UserPoolID) } - poolUsers, ok := b.users[client.UserPoolID] + user, ok := b.users.Get(userKey(client.UserPoolID, username)) if !ok { - return nil, nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, client.UserPoolID) + return nil, nil, unknownUserAuthError(client, username) } - user, ok := poolUsers[username] - if !ok { - return nil, nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) + return user, pool, nil +} + +// unknownUserAuthError returns the error InitiateAuth surfaces when the username does not +// exist. When the app client's PreventUserExistenceErrors is "ENABLED" (the AWS-recommended +// setting), Cognito masks the distinction behind the same NotAuthorizedException a wrong +// password produces, using the identical message, so a caller cannot enumerate valid +// usernames by comparing error types/text. "LEGACY" (the default when unset) reveals +// UserNotFoundException, matching Cognito's pre-2019 behavior kept for backward +// compatibility. Only the non-admin InitiateAuth API applies this masking; AdminInitiateAuth +// always reveals the real error since the caller already has admin-level AWS credentials. +func unknownUserAuthError(client *UserPoolClient, username string) error { + if client.PreventUserExistenceErrors == preventUserExistenceEnabled { + return fmt.Errorf("%w: incorrect username or password", ErrNotAuthorized) } - return user, pool, nil + return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } // challengePasswordVerifier is returned for USER_SRP_AUTH after credentials are validated. @@ -1090,7 +1098,7 @@ const challengePasswordVerifier = "PASSWORD_VERIFIER" // isAuthFlowAllowed checks whether the given flow is permitted by the client's ExplicitAuthFlows list. // Returns true when no restriction is configured. func (b *InMemoryBackend) isAuthFlowAllowed(clientID, authFlow string) bool { - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok || len(client.ExplicitAuthFlows) == 0 { return true } @@ -1105,9 +1113,15 @@ func (b *InMemoryBackend) isAuthFlowAllowed(clientID, authFlow string) bool { } // newMFASession stores a new session entry and returns an AuthResult with the challenge. +// For SMS_MFA/EMAIL_OTP a one-time numeric code is generated and stored on the session so +// RespondToMFAChallenge can require an exact match, the same way ForgotPassword/SignUp +// confirmation codes are validated elsewhere in this backend (there is no real SMS/email +// gateway to deliver the code out of band, so simulation is the best available proxy — but +// unlike a bare format check, "any 6 digits" is no longer accepted). func (b *InMemoryBackend) newMFASession(pool *UserPool, clientID, username, challengeType string) *AuthResult { sessionToken := randomAlphanumeric(mfaSessionLen) - b.mfaSessions[sessionToken] = &mfaSessionEntry{ + + entry := &mfaSessionEntry{ PoolID: pool.ID, ClientID: clientID, Username: username, @@ -1115,6 +1129,12 @@ func (b *InMemoryBackend) newMFASession(pool *UserPool, clientID, username, chal ExpiresAt: time.Now().Add(mfaSessionTTL), } + if challengeType == challengeSMSMFA || challengeType == challengeEmailOTP { + entry.Code = randomNumeric(totpCodeLen) + } + + b.mfaSessions[sessionToken] = entry + return &AuthResult{ MFASession: sessionToken, ChallengeName: challengeType, @@ -1194,7 +1214,7 @@ func (b *InMemoryBackend) issueTokensLocked(pool *UserPool, clientID string, use var scopes []string refreshTTL := defaultRefreshTokenTTL var accessExpiry, idExpiry time.Duration - if client, ok := b.clients[clientID]; ok { + if client, ok := b.clients.Get(clientID); ok { scopes = client.AllowedOAuthScopes if d := tokenExpiryFor(client, "AccessToken"); d > 0 { accessExpiry = d @@ -1257,13 +1277,12 @@ func (b *InMemoryBackend) InitiateAuthRefreshToken(clientID, refreshToken string return nil, fmt.Errorf("%w: refresh token was issued for a different client", ErrNotAuthorized) } - pool, ok := b.pools[entry.PoolID] + pool, ok := b.pools.Get(entry.PoolID) if !ok { return nil, fmt.Errorf("%w: user pool %q not found", ErrUserPoolNotFound, entry.PoolID) } - // A missing pool entry yields a nil map, whose lookup safely reports !ok. - user, ok := b.users[entry.PoolID][entry.Username] + user, ok := b.users.Get(userKey(entry.PoolID, entry.Username)) if !ok { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, entry.Username) } @@ -1278,7 +1297,7 @@ func (b *InMemoryBackend) InitiateAuthRefreshToken(clientID, refreshToken string var scopes []string refreshTTL := defaultRefreshTokenTTL var accessExpiry, idExpiry time.Duration - if c, cok := b.clients[clientID]; cok { + if c, cok := b.clients.Get(clientID); cok { scopes = c.AllowedOAuthScopes if d := tokenExpiryFor(c, "AccessToken"); d > 0 { accessExpiry = d @@ -1342,19 +1361,23 @@ func (b *InMemoryBackend) RevokeToken(token, clientID string) error { return nil } -// RespondToMFAChallenge validates an MFA session + TOTP code and issues tokens. -// Any 6-digit code is accepted (simulation only — no real TOTP verification). -func (b *InMemoryBackend) RespondToMFAChallenge(clientID, session, totpCode string) (*TokenResult, error) { +// RespondToMFAChallenge validates an MFA session and the user-supplied code, then issues +// tokens. SOFTWARE_TOKEN_MFA is verified as a real RFC 6238 TOTP code against the user's +// AssociateSoftwareToken secret; SMS_MFA/EMAIL_OTP are verified against the one-time code +// generated when the challenge session was created (see newMFASession). A wrong code +// returns CodeMismatchException without consuming the session, so the caller may retry +// until the session expires — matching real Cognito. +func (b *InMemoryBackend) RespondToMFAChallenge(clientID, session, code string) (*TokenResult, error) { b.mu.Lock("RespondToMFAChallenge") defer b.mu.Unlock() - if len(totpCode) != totpCodeLen { - return nil, fmt.Errorf("%w: TOTP code must be %d digits", ErrCodeMismatch, totpCodeLen) + if len(code) != totpCodeLen { + return nil, fmt.Errorf("%w: code must be %d digits", ErrCodeMismatch, totpCodeLen) } - for _, ch := range totpCode { + for _, ch := range code { if ch < '0' || ch > '9' { - return nil, fmt.Errorf("%w: TOTP code must contain only digits", ErrCodeMismatch) + return nil, fmt.Errorf("%w: code must contain only digits", ErrCodeMismatch) } } @@ -1373,19 +1396,19 @@ func (b *InMemoryBackend) RespondToMFAChallenge(clientID, session, totpCode stri return nil, fmt.Errorf("%w: MFA session was issued for a different client", ErrNotAuthorized) } - pool, ok := b.pools[entry.PoolID] + pool, ok := b.pools.Get(entry.PoolID) if !ok { return nil, fmt.Errorf("%w: user pool %q not found", ErrUserPoolNotFound, entry.PoolID) } - poolUsers, ok := b.users[entry.PoolID] + user, ok := b.users.Get(userKey(entry.PoolID, entry.Username)) if !ok { - return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, entry.PoolID) + return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, entry.Username) } - user, ok := poolUsers[entry.Username] - if !ok { - return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, entry.Username) + if err := verifyMFAChallengeCode(entry, user, code); err != nil { + // Do not consume the session on a wrong code: the caller may retry until it expires. + return nil, err } // Consume the session (one-time use). @@ -1399,20 +1422,45 @@ func (b *InMemoryBackend) RespondToMFAChallenge(clientID, session, totpCode stri return result.Tokens, nil } +// verifyMFAChallengeCode validates code against the challenge type recorded on the MFA +// session. SOFTWARE_TOKEN_MFA is verified cryptographically (RFC 6238 TOTP) against the +// user's secret; SMS_MFA/EMAIL_OTP are verified against the code generated by +// newMFASession, since there is no client-held secret to re-derive those from. +func verifyMFAChallengeCode(entry *mfaSessionEntry, user *User, code string) error { + switch entry.ChallengeType { + case challengeSoftwareTokenMFA: + if user.TOTPSecret == "" { + return fmt.Errorf("%w: no TOTP secret associated; call AssociateSoftwareToken first", ErrNotAuthorized) + } + + if !verifyTOTPCode(user.TOTPSecret, code, time.Now()) { + return fmt.Errorf("%w: invalid software token code", ErrCodeMismatch) + } + + return nil + + case challengeSMSMFA, challengeEmailOTP: + if entry.Code == "" || !hmac.Equal([]byte(entry.Code), []byte(code)) { + return fmt.Errorf("%w: invalid MFA code", ErrCodeMismatch) + } + + return nil + + default: + return fmt.Errorf("%w: unexpected challenge type %q for MFA response", ErrCodeMismatch, entry.ChallengeType) + } +} + // CreateGroup creates a group in a user pool. func (b *InMemoryBackend) CreateGroup(userPoolID, groupName, description string, precedence int32) (*Group, error) { b.mu.Lock("CreateGroup") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if b.groups[userPoolID] == nil { - b.groups[userPoolID] = make(map[string]*Group) - } - - if _, exists := b.groups[userPoolID][groupName]; exists { + if _, exists := b.groups.Get(groupKey(userPoolID, groupName)); exists { return nil, fmt.Errorf("%w: group %q already exists in pool %q", ErrAlreadyExists, groupName, userPoolID) } @@ -1423,7 +1471,7 @@ func (b *InMemoryBackend) CreateGroup(userPoolID, groupName, description string, Precedence: precedence, CreatedAt: time.Now().UTC(), } - b.groups[userPoolID][groupName] = g + b.groups.Put(g) cp := *g @@ -1435,15 +1483,15 @@ func (b *InMemoryBackend) DeleteGroup(userPoolID, groupName string) error { b.mu.Lock("DeleteGroup") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.groups[userPoolID][groupName]; !ok { + if _, ok := b.groups.Get(groupKey(userPoolID, groupName)); !ok { return fmt.Errorf("%w: group %q not found in pool %q", ErrGroupNotFound, groupName, userPoolID) } - delete(b.groups[userPoolID], groupName) + b.groups.Delete(groupKey(userPoolID, groupName)) if b.groupMembers[userPoolID] != nil { delete(b.groupMembers[userPoolID], groupName) @@ -1457,11 +1505,11 @@ func (b *InMemoryBackend) ListGroups(userPoolID string) ([]*Group, error) { b.mu.RLock("ListGroups") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - poolGroups := b.groups[userPoolID] + poolGroups := b.groupsByPool.Get(userPoolID) out := make([]*Group, 0, len(poolGroups)) for _, g := range poolGroups { @@ -1479,16 +1527,15 @@ func (b *InMemoryBackend) AdminAddUserToGroup(userPoolID, username, groupName st b.mu.Lock("AdminAddUserToGroup") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.groups[userPoolID][groupName]; !ok { + if _, ok := b.groups.Get(groupKey(userPoolID, groupName)); !ok { return fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } - poolUsers := b.users[userPoolID] - if _, ok := poolUsers[username]; !ok { + if _, ok := b.users.Get(userKey(userPoolID, username)); !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1510,11 +1557,11 @@ func (b *InMemoryBackend) AdminRemoveUserFromGroup(userPoolID, username, groupNa b.mu.Lock("AdminRemoveUserFromGroup") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.groups[userPoolID][groupName]; !ok { + if _, ok := b.groups.Get(groupKey(userPoolID, groupName)); !ok { return fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } @@ -1530,11 +1577,11 @@ func (b *InMemoryBackend) AdminListGroupsForUser(userPoolID, username string) ([ b.mu.RLock("AdminListGroupsForUser") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.users[userPoolID][username]; !ok { + if _, ok := b.users.Get(userKey(userPoolID, username)); !ok { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1545,7 +1592,7 @@ func (b *InMemoryBackend) AdminListGroupsForUser(userPoolID, username string) ([ continue } - if g, ok := b.groups[userPoolID][groupName]; ok { + if g, ok := b.groups.Get(groupKey(userPoolID, groupName)); ok { cp := *g out = append(out, &cp) } @@ -1581,11 +1628,11 @@ func (b *InMemoryBackend) AdminUpdateUserAttributes(userPoolID, username string, b.mu.Lock("AdminUpdateUserAttributes") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - u, ok := b.users[userPoolID][username] + u, ok := b.users.Get(userKey(userPoolID, username)) if !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1606,7 +1653,7 @@ func (b *InMemoryBackend) AddCustomAttributes(userPoolID string, attrs []SchemaA b.mu.Lock("AddCustomAttributes") defer b.mu.Unlock() - pool, ok := b.pools[userPoolID] + pool, ok := b.pools.Get(userPoolID) if !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } @@ -1631,11 +1678,11 @@ func (b *InMemoryBackend) AddUserPoolClientSecret(userPoolID, clientID string) ( b.mu.Lock("AddUserPoolClientSecret") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return "", fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok { return "", fmt.Errorf("%w: client %q not found", ErrClientNotFound, clientID) } @@ -1655,11 +1702,11 @@ func (b *InMemoryBackend) AdminDeleteUserAttributes(userPoolID, username string, b.mu.Lock("AdminDeleteUserAttributes") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - u, ok := b.users[userPoolID][username] + u, ok := b.users.Get(userKey(userPoolID, username)) if !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1678,7 +1725,7 @@ func (b *InMemoryBackend) AdminDisableProviderForUser(userPoolID string) error { b.mu.RLock("AdminDisableProviderForUser") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } @@ -1690,11 +1737,11 @@ func (b *InMemoryBackend) AdminDisableUser(userPoolID, username string) error { b.mu.Lock("AdminDisableUser") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - u, ok := b.users[userPoolID][username] + u, ok := b.users.Get(userKey(userPoolID, username)) if !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1709,11 +1756,11 @@ func (b *InMemoryBackend) AdminEnableUser(userPoolID, username string) error { b.mu.Lock("AdminEnableUser") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - u, ok := b.users[userPoolID][username] + u, ok := b.users.Get(userKey(userPoolID, username)) if !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1734,11 +1781,11 @@ func (b *InMemoryBackend) AdminForgetDevice(userPoolID, username, deviceKey stri b.mu.Lock("AdminForgetDevice") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.users[userPoolID][username]; !ok { + if _, ok := b.users.Get(userKey(userPoolID, username)); !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1770,11 +1817,11 @@ func (b *InMemoryBackend) ValidatePoolUser(userPoolID, username string) error { b.mu.RLock("ValidatePoolUser") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.users[userPoolID][username]; !ok { + if _, ok := b.users.Get(userKey(userPoolID, username)); !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1786,20 +1833,19 @@ func (b *InMemoryBackend) ListUsersInGroup(userPoolID, groupName string) ([]*Use b.mu.RLock("ListUsersInGroup") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.groups[userPoolID][groupName]; !ok { + if _, ok := b.groups.Get(groupKey(userPoolID, groupName)); !ok { return nil, fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } members := b.groupMembers[userPoolID][groupName] - poolUsers := b.users[userPoolID] out := make([]*User, 0, len(members)) for username := range members { - u, ok := poolUsers[username] + u, ok := b.users.Get(userKey(userPoolID, username)) if !ok { continue } @@ -1819,11 +1865,11 @@ func (b *InMemoryBackend) AdminUserGlobalSignOut(userPoolID, username string) er b.mu.Lock("AdminUserGlobalSignOut") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.users[userPoolID][username]; !ok { + if _, ok := b.users.Get(userKey(userPoolID, username)); !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1855,17 +1901,16 @@ func (b *InMemoryBackend) ResendConfirmationCode(clientID, username string) (str b.mu.Lock("ResendConfirmationCode") defer b.mu.Unlock() - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok { return "", fmt.Errorf("%w: client %q not found", ErrClientNotFound, clientID) } - poolUsers, ok := b.users[client.UserPoolID] - if !ok { + if _, poolOK := b.pools.Get(client.UserPoolID); !poolOK { return "", fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, client.UserPoolID) } - user, ok := poolUsers[username] + user, ok := b.users.Get(userKey(client.UserPoolID, username)) if !ok { return "", fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1886,7 +1931,7 @@ func (b *InMemoryBackend) SetUserPoolMfaConfig(userPoolID, mfaConfig string) err b.mu.Lock("SetUserPoolMfaConfig") defer b.mu.Unlock() - pool, ok := b.pools[userPoolID] + pool, ok := b.pools.Get(userPoolID) if !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } @@ -1901,11 +1946,11 @@ func (b *InMemoryBackend) UpdateGroup(userPoolID, groupName, description string, b.mu.Lock("UpdateGroup") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - g, ok := b.groups[userPoolID][groupName] + g, ok := b.groups.Get(groupKey(userPoolID, groupName)) if !ok { return nil, fmt.Errorf("%w: group %q not found in pool %q", ErrGroupNotFound, groupName, userPoolID) } @@ -1923,28 +1968,16 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.pools = make(map[string]*UserPool) - b.poolsByName = make(map[string]*UserPool) - b.clients = make(map[string]*UserPoolClient) - b.clientsByPool = make(map[string]map[string]*UserPoolClient) - b.users = make(map[string]map[string]*User) - b.usersBySub = make(map[string]string) + b.registry.ResetAll() + b.refreshTokens = make(map[string]*refreshTokenEntry) b.refreshTokensByClient = make(map[string]map[string]struct{}) b.refreshTokensByUser = make(map[string]map[string]struct{}) - b.groups = make(map[string]map[string]*Group) b.groupMembers = make(map[string]map[string]map[string]struct{}) - b.resourceServers = make(map[string]map[string]*ResourceServer) b.tokenRevokedBefore = make(map[string]time.Time) - b.identityProviders = make(map[string]map[string]*IdentityProvider) - b.domains = make(map[string]*UserPoolDomain) b.resourceTags = make(map[string]map[string]string) b.riskConfigurations = make(map[string]*RiskConfiguration) b.logDeliveryConfigs = make(map[string]*LogDeliveryConfig) - b.uiCustomizations = make(map[string]*UICustomization) - b.managedLoginBrandings = make(map[string]map[string]*ManagedLoginBranding) - b.terms = make(map[string]*Terms) - b.userImportJobs = make(map[string]map[string]*UserImportJob) b.devices = make(map[string]map[string]*Device) b.webauthnCredentials = make(map[string]map[string]*WebAuthnCredential) b.authEvents = make(map[string]map[string]*AuthEvent) @@ -1955,7 +1988,7 @@ func (b *InMemoryBackend) UpdateUserPool(userPoolID, mfaConfiguration string) er b.mu.Lock("UpdateUserPool") defer b.mu.Unlock() - pool, ok := b.pools[userPoolID] + pool, ok := b.pools.Get(userPoolID) if !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } @@ -1974,11 +2007,11 @@ func (b *InMemoryBackend) UpdateUserPoolClient(userPoolID, clientID, clientName b.mu.Lock("UpdateUserPoolClient") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok { return nil, fmt.Errorf("%w: client %q not found", ErrClientNotFound, clientID) } @@ -2003,11 +2036,11 @@ func (b *InMemoryBackend) AdminResetUserPassword(userPoolID, username string) er b.mu.Lock("AdminResetUserPassword") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - u, ok := b.users[userPoolID][username] + u, ok := b.users.Get(userKey(userPoolID, username)) if !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -2026,11 +2059,11 @@ func (b *InMemoryBackend) GetGroup(userPoolID, groupName string) (*Group, error) b.mu.RLock("GetGroup") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - g, ok := b.groups[userPoolID][groupName] + g, ok := b.groups.Get(groupKey(userPoolID, groupName)) if !ok { return nil, fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } @@ -2053,8 +2086,7 @@ func (b *InMemoryBackend) DeleteUser(accessToken string) error { poolID := u.UserPoolID username := u.Username - delete(b.usersBySub, poolID+":"+u.Sub) - delete(b.users[poolID], username) + b.users.Delete(userKey(poolID, username)) b.deleteRefreshTokensForUserLocked(poolID, username) return nil @@ -2107,11 +2139,11 @@ func (b *InMemoryBackend) ListUsersFiltered(userPoolID, filter string) ([]*User, b.mu.RLock("ListUsersFiltered") defer b.mu.RUnlock() - poolUsers, ok := b.users[userPoolID] - if !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } + poolUsers := b.usersByPool.Get(userPoolID) prefix, attrFilter := parseListUsersFilter(filter) out := make([]*User, 0, len(poolUsers)) @@ -2193,12 +2225,7 @@ func (b *InMemoryBackend) AddUserPoolInternal(pool *UserPool) { b.mu.Lock("AddUserPoolInternal") defer b.mu.Unlock() - b.pools[pool.ID] = pool - b.poolsByName[pool.Name] = pool - - if _, ok := b.users[pool.ID]; !ok { - b.users[pool.ID] = make(map[string]*User) - } + b.pools.Put(pool) } // AddUserPoolClientInternal seeds a user pool client directly into the backend. @@ -2207,11 +2234,7 @@ func (b *InMemoryBackend) AddUserPoolClientInternal(client *UserPoolClient) { b.mu.Lock("AddUserPoolClientInternal") defer b.mu.Unlock() - b.clients[client.ClientID] = client - if b.clientsByPool[client.UserPoolID] == nil { - b.clientsByPool[client.UserPoolID] = make(map[string]*UserPoolClient) - } - b.clientsByPool[client.UserPoolID][client.ClientID] = client + b.clients.Put(client) } // deleteRefreshTokenLocked deletes a refresh token and updates secondary indexes. @@ -2324,12 +2347,7 @@ func (b *InMemoryBackend) AddUserInternal(user *User) { b.mu.Lock("AddUserInternal") defer b.mu.Unlock() - if b.users[user.UserPoolID] == nil { - b.users[user.UserPoolID] = make(map[string]*User) - } - - b.users[user.UserPoolID][user.Username] = user - b.usersBySub[user.UserPoolID+":"+user.Sub] = user.Username + b.users.Put(user) } // GetPoolMetrics returns aggregate statistics for a user pool. @@ -2337,7 +2355,7 @@ func (b *InMemoryBackend) GetPoolMetrics(userPoolID string) (*PoolMetrics, error b.mu.RLock("GetPoolMetrics") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } @@ -2350,9 +2368,9 @@ func (b *InMemoryBackend) GetPoolMetrics(userPoolID string) (*PoolMetrics, error } return &PoolMetrics{ - UserCount: len(b.users[userPoolID]), - ClientCount: len(b.clientsByPool[userPoolID]), - GroupCount: len(b.groups[userPoolID]), + UserCount: len(b.usersByPool.Get(userPoolID)), + ClientCount: len(b.clientsByPool.Get(userPoolID)), + GroupCount: len(b.groupsByPool.Get(userPoolID)), ActiveTokenCount: tokenCount, }, nil } @@ -2385,3 +2403,24 @@ func randomAlphanumeric(n int) string { return string(b) } + +// numericChars contains the digits used for random numeric code generation +// (SMS_MFA / EMAIL_OTP one-time codes). +const numericChars = "0123456789" + +// randomNumeric returns a random numeric string of length n. +func randomNumeric(n int) string { + b := make([]byte, n) + for i := range b { + idx, err := rand.Int(rand.Reader, big.NewInt(int64(len(numericChars)))) + if err != nil { + b[i] = numericChars[0] + + continue + } + + b[i] = numericChars[idx.Int64()] + } + + return string(b) +} diff --git a/services/cognitoidp/batch2_backend.go b/services/cognitoidp/batch2_backend.go index aad10eff7..79d92e748 100644 --- a/services/cognitoidp/batch2_backend.go +++ b/services/cognitoidp/batch2_backend.go @@ -163,11 +163,11 @@ func (b *InMemoryBackend) SetUserPoolMfaConfigFull(userPoolID string, cfg UserPo b.mu.Lock("SetUserPoolMfaConfigFull") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + pool, ok := b.pools.Get(userPoolID) + if !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - pool := b.pools[userPoolID] if cfg.MfaConfiguration != "" { pool.MfaConfiguration = cfg.MfaConfiguration } @@ -182,7 +182,7 @@ func (b *InMemoryBackend) GetUserPoolMfaConfigFull(userPoolID string) (*UserPool b.mu.RLock("GetUserPoolMfaConfigFull") defer b.mu.RUnlock() - pool, ok := b.pools[userPoolID] + pool, ok := b.pools.Get(userPoolID) if !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } @@ -343,12 +343,11 @@ func (b *InMemoryBackend) SetTypedRiskConfiguration(cfg *TypedRiskConfiguration) b.mu.Lock("SetTypedRiskConfiguration") defer b.mu.Unlock() - if _, ok := b.pools[cfg.UserPoolID]; !ok { + if _, ok := b.pools.Get(cfg.UserPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, cfg.UserPoolID) } - key := cfg.UserPoolID + ":" + cfg.ClientID - b.typedRiskConfigurations[key] = cfg + b.typedRiskConfigurations.Put(cfg) return nil } @@ -358,13 +357,11 @@ func (b *InMemoryBackend) GetTypedRiskConfiguration(poolID, clientID string) (*T b.mu.RLock("GetTypedRiskConfiguration") defer b.mu.RUnlock() - if _, ok := b.pools[poolID]; !ok { + if _, ok := b.pools.Get(poolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, poolID) } - key := poolID + ":" + clientID - - cfg, ok := b.typedRiskConfigurations[key] + cfg, ok := b.typedRiskConfigurations.Get(poolID + ":" + clientID) if !ok { return &TypedRiskConfiguration{UserPoolID: poolID, ClientID: clientID}, nil } @@ -383,26 +380,25 @@ func (b *InMemoryBackend) SetUICustomizationFull(poolID, clientID, css, imageURL b.mu.Lock("SetUICustomizationFull") defer b.mu.Unlock() - if _, ok := b.pools[poolID]; !ok { + if _, ok := b.pools.Get(poolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, poolID) } - key := poolID + ":" + clientID now := time.Now() - existing, ok := b.uiCustomizations[key] + existing, ok := b.uiCustomizations.Get(uiKey(poolID, clientID)) if !ok { existing = &UICustomization{ UserPoolID: poolID, ClientID: clientID, CreatedAt: now, } - b.uiCustomizations[key] = existing } existing.CSS = css existing.ImageURL = imageURL existing.LastModifiedAt = now + b.uiCustomizations.Put(existing) cp := *existing @@ -414,13 +410,11 @@ func (b *InMemoryBackend) GetUICustomizationFull(poolID, clientID string) (*UICu b.mu.RLock("GetUICustomizationFull") defer b.mu.RUnlock() - if _, ok := b.pools[poolID]; !ok { + if _, ok := b.pools.Get(poolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, poolID) } - key := poolID + ":" + clientID - - existing, ok := b.uiCustomizations[key] + existing, ok := b.uiCustomizations.Get(uiKey(poolID, clientID)) if !ok { return &UICustomization{UserPoolID: poolID, ClientID: clientID}, nil } @@ -444,15 +438,11 @@ func (b *InMemoryBackend) CreateIdentityProviderFull( b.mu.Lock("CreateIdentityProviderFull") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if b.identityProviders[userPoolID] == nil { - b.identityProviders[userPoolID] = make(map[string]*IdentityProvider) - } - - if _, exists := b.identityProviders[userPoolID][providerName]; exists { + if _, exists := b.identityProviders.Get(identityProviderKey(userPoolID, providerName)); exists { return nil, fmt.Errorf("%w: identity provider %q already exists in pool %q", ErrDuplicateProvider, providerName, userPoolID) } @@ -479,7 +469,7 @@ func (b *InMemoryBackend) CreateIdentityProviderFull( LastModifiedAt: now, } - b.identityProviders[userPoolID][providerName] = idp + b.identityProviders.Put(idp) cp := *idp @@ -496,11 +486,11 @@ func (b *InMemoryBackend) UpdateIdentityProviderFull( b.mu.Lock("UpdateIdentityProviderFull") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - idp, ok := b.identityProviders[userPoolID][providerName] + idp, ok := b.identityProviders.Get(identityProviderKey(userPoolID, providerName)) if !ok { return nil, fmt.Errorf("%w: identity provider %q not found in pool %q", ErrUserPoolNotFound, providerName, userPoolID) @@ -536,11 +526,11 @@ func (b *InMemoryBackend) CreateUserPoolDomainFull(userPoolID, domain, certifica b.mu.Lock("CreateUserPoolDomainFull") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, exists := b.domains[domain]; exists { + if _, exists := b.domains.Get(domain); exists { return nil, fmt.Errorf("%w: domain %q already exists", ErrAlreadyExists, domain) } @@ -557,7 +547,7 @@ func (b *InMemoryBackend) CreateUserPoolDomainFull(userPoolID, domain, certifica CertificateArn: certificateArn, Status: "ACTIVE", } - b.domains[domain] = d + b.domains.Put(d) cp := *d @@ -569,11 +559,11 @@ func (b *InMemoryBackend) UpdateUserPoolDomainFull(userPoolID, domain, certifica b.mu.Lock("UpdateUserPoolDomainFull") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return "", fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - d, ok := b.domains[domain] + d, ok := b.domains.Get(domain) if !ok { return "", fmt.Errorf("%w: domain %q not found", ErrUserPoolNotFound, domain) } @@ -598,15 +588,11 @@ func (b *InMemoryBackend) CreateGroupFull( b.mu.Lock("CreateGroupFull") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if b.groups[userPoolID] == nil { - b.groups[userPoolID] = make(map[string]*Group) - } - - if _, exists := b.groups[userPoolID][groupName]; exists { + if _, exists := b.groups.Get(groupKey(userPoolID, groupName)); exists { return nil, fmt.Errorf("%w: group %q already exists in pool %q", ErrAlreadyExists, groupName, userPoolID) } @@ -621,7 +607,7 @@ func (b *InMemoryBackend) CreateGroupFull( LastModifiedAt: now, } - b.groups[userPoolID][groupName] = g + b.groups.Put(g) if b.groupMembers[userPoolID] == nil { b.groupMembers[userPoolID] = make(map[string]map[string]struct{}) @@ -642,11 +628,11 @@ func (b *InMemoryBackend) UpdateGroupFull( b.mu.Lock("UpdateGroupFull") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - g, ok := b.groups[userPoolID][groupName] + g, ok := b.groups.Get(groupKey(userPoolID, groupName)) if !ok { return nil, fmt.Errorf("%w: group %q not found in pool %q", ErrGroupNotFound, groupName, userPoolID) } @@ -682,17 +668,12 @@ func (b *InMemoryBackend) AdminCreateUserFull( b.mu.Lock("AdminCreateUserFull") defer b.mu.Unlock() - pool, ok := b.pools[userPoolID] - if !ok { - return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) - } - - poolUsers, ok := b.users[userPoolID] + pool, ok := b.pools.Get(userPoolID) if !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if existing, exists := poolUsers[username]; exists { + if existing, exists := b.users.Get(userKey(userPoolID, username)); exists { if messageAction == "RESEND" { // Re-send temp password to existing FORCE_CHANGE_PASSWORD user. if existing.Status != UserStatusForceChangePassword { @@ -754,8 +735,7 @@ func (b *InMemoryBackend) AdminCreateUserFull( Enabled: true, } - poolUsers[username] = user - b.usersBySub[userPoolID+":"+user.Sub] = username + b.users.Put(user) cp := *user @@ -772,7 +752,7 @@ func (b *InMemoryBackend) AdminSetUserPasswordFull(userPoolID, username, passwor b.mu.Lock("AdminSetUserPasswordFull") defer b.mu.Unlock() - pool, ok := b.pools[userPoolID] + pool, ok := b.pools.Get(userPoolID) if !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } @@ -781,7 +761,7 @@ func (b *InMemoryBackend) AdminSetUserPasswordFull(userPoolID, username, passwor return err } - user, ok := b.users[userPoolID][username] + user, ok := b.users.Get(userKey(userPoolID, username)) if !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -813,11 +793,11 @@ func (b *InMemoryBackend) ListGroupsPage(userPoolID string, limit int, nextToken b.mu.RLock("ListGroupsPage") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, "", fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - poolGroups := b.groups[userPoolID] + poolGroups := b.groupsByPool.Get(userPoolID) all := make([]*Group, 0, len(poolGroups)) for _, g := range poolGroups { @@ -865,11 +845,11 @@ func (b *InMemoryBackend) ListUsersInGroupPage( b.mu.RLock("ListUsersInGroupPage") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, "", fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.groups[userPoolID][groupName]; !ok { + if _, ok := b.groups.Get(groupKey(userPoolID, groupName)); !ok { return nil, "", fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } @@ -877,7 +857,7 @@ func (b *InMemoryBackend) ListUsersInGroupPage( all := make([]*User, 0, len(members)) for username := range members { - if u, ok := b.users[userPoolID][username]; ok { + if u, ok := b.users.Get(userKey(userPoolID, username)); ok { cp := *u all = append(all, &cp) } diff --git a/services/cognitoidp/completeness_backend.go b/services/cognitoidp/completeness_backend.go index ea30f2bb9..7cb6f96e0 100644 --- a/services/cognitoidp/completeness_backend.go +++ b/services/cognitoidp/completeness_backend.go @@ -150,15 +150,11 @@ func (b *InMemoryBackend) CreateIdentityProvider( b.mu.Lock("CreateIdentityProvider") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if b.identityProviders[userPoolID] == nil { - b.identityProviders[userPoolID] = make(map[string]*IdentityProvider) - } - - if _, exists := b.identityProviders[userPoolID][providerName]; exists { + if _, exists := b.identityProviders.Get(identityProviderKey(userPoolID, providerName)); exists { return nil, fmt.Errorf("%w: identity provider %q already exists in pool %q", ErrAlreadyExists, providerName, userPoolID) } @@ -172,7 +168,7 @@ func (b *InMemoryBackend) CreateIdentityProvider( CreatedAt: now, LastModifiedAt: now, } - b.identityProviders[userPoolID][providerName] = idp + b.identityProviders.Put(idp) cp := *idp cp.ProviderDetails = maps.Clone(idp.ProviderDetails) @@ -185,11 +181,11 @@ func (b *InMemoryBackend) DescribeIdentityProvider(userPoolID, providerName stri b.mu.RLock("DescribeIdentityProvider") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - idp, ok := b.identityProviders[userPoolID][providerName] + idp, ok := b.identityProviders.Get(identityProviderKey(userPoolID, providerName)) if !ok { return nil, fmt.Errorf("%w: identity provider %q not found in pool %q", ErrUserPoolNotFound, providerName, userPoolID) @@ -206,11 +202,11 @@ func (b *InMemoryBackend) GetIdentityProviderByIdentifier(userPoolID, identifier b.mu.RLock("GetIdentityProviderByIdentifier") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - for _, idp := range b.identityProviders[userPoolID] { + for _, idp := range b.identityProvidersByPool.Get(userPoolID) { if idp.ProviderName == identifier { cp := *idp cp.ProviderDetails = maps.Clone(idp.ProviderDetails) @@ -235,11 +231,11 @@ func (b *InMemoryBackend) ListIdentityProviders(userPoolID string) ([]*IdentityP b.mu.RLock("ListIdentityProviders") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - poolProviders := b.identityProviders[userPoolID] + poolProviders := b.identityProvidersByPool.Get(userPoolID) out := make([]*IdentityProvider, 0, len(poolProviders)) for _, idp := range poolProviders { @@ -261,11 +257,11 @@ func (b *InMemoryBackend) UpdateIdentityProvider( b.mu.Lock("UpdateIdentityProvider") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - idp, ok := b.identityProviders[userPoolID][providerName] + idp, ok := b.identityProviders.Get(identityProviderKey(userPoolID, providerName)) if !ok { return nil, fmt.Errorf("%w: identity provider %q not found in pool %q", ErrUserPoolNotFound, providerName, userPoolID) @@ -287,16 +283,16 @@ func (b *InMemoryBackend) DeleteIdentityProvider(userPoolID, providerName string b.mu.Lock("DeleteIdentityProvider") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.identityProviders[userPoolID][providerName]; !ok { + if _, ok := b.identityProviders.Get(identityProviderKey(userPoolID, providerName)); !ok { return fmt.Errorf("%w: identity provider %q not found in pool %q", ErrUserPoolNotFound, providerName, userPoolID) } - delete(b.identityProviders[userPoolID], providerName) + b.identityProviders.Delete(identityProviderKey(userPoolID, providerName)) return nil } @@ -310,11 +306,11 @@ func (b *InMemoryBackend) CreateUserPoolDomain(userPoolID, domain string) (*User b.mu.Lock("CreateUserPoolDomain") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, exists := b.domains[domain]; exists { + if _, exists := b.domains.Get(domain); exists { return nil, fmt.Errorf("%w: domain %q already exists", ErrAlreadyExists, domain) } @@ -324,7 +320,7 @@ func (b *InMemoryBackend) CreateUserPoolDomain(userPoolID, domain string) (*User CloudFrontDistribution: domain + ".auth." + b.region + ".amazoncognito.com", Status: "ACTIVE", } - b.domains[domain] = d + b.domains.Put(d) cp := *d @@ -336,7 +332,7 @@ func (b *InMemoryBackend) DescribeUserPoolDomain(domain string) (*UserPoolDomain b.mu.RLock("DescribeUserPoolDomain") defer b.mu.RUnlock() - d, ok := b.domains[domain] + d, ok := b.domains.Get(domain) if !ok { return nil, fmt.Errorf("%w: domain %q not found", ErrUserPoolNotFound, domain) } @@ -352,7 +348,7 @@ func (b *InMemoryBackend) FindUserPoolDomain(domain string) *UserPoolDomain { b.mu.RLock("FindUserPoolDomain") defer b.mu.RUnlock() - d := b.domains[domain] + d, _ := b.domains.Get(domain) if d == nil { return nil } @@ -367,11 +363,11 @@ func (b *InMemoryBackend) UpdateUserPoolDomain(userPoolID, domain string) (strin b.mu.Lock("UpdateUserPoolDomain") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return "", fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - d, ok := b.domains[domain] + d, ok := b.domains.Get(domain) if !ok { return "", fmt.Errorf("%w: domain %q not found", ErrUserPoolNotFound, domain) } @@ -384,15 +380,15 @@ func (b *InMemoryBackend) DeleteUserPoolDomain(userPoolID, domain string) error b.mu.Lock("DeleteUserPoolDomain") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.domains[domain]; !ok { + if _, ok := b.domains.Get(domain); !ok { return fmt.Errorf("%w: domain %q not found", ErrUserPoolNotFound, domain) } - delete(b.domains, domain) + b.domains.Delete(domain) return nil } @@ -449,7 +445,7 @@ func (b *InMemoryBackend) SetRiskConfiguration(poolID, clientID string, raw map[ b.mu.Lock("SetRiskConfiguration") defer b.mu.Unlock() - if _, ok := b.pools[poolID]; !ok { + if _, ok := b.pools.Get(poolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, poolID) } @@ -465,7 +461,7 @@ func (b *InMemoryBackend) DescribeRiskConfiguration(poolID, clientID string) (*R b.mu.RLock("DescribeRiskConfiguration") defer b.mu.RUnlock() - if _, ok := b.pools[poolID]; !ok { + if _, ok := b.pools.Get(poolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, poolID) } @@ -489,7 +485,7 @@ func (b *InMemoryBackend) SetLogDeliveryConfiguration(poolID string, raw map[str b.mu.Lock("SetLogDeliveryConfiguration") defer b.mu.Unlock() - if _, ok := b.pools[poolID]; !ok { + if _, ok := b.pools.Get(poolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, poolID) } @@ -505,7 +501,7 @@ func (b *InMemoryBackend) GetLogDeliveryConfiguration(poolID string) (*LogDelive b.mu.RLock("GetLogDeliveryConfiguration") defer b.mu.RUnlock() - if _, ok := b.pools[poolID]; !ok { + if _, ok := b.pools.Get(poolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, poolID) } @@ -534,12 +530,12 @@ func (b *InMemoryBackend) SetUICustomization(poolID, clientID, css string) (*UIC b.mu.Lock("SetUICustomization") defer b.mu.Unlock() - if _, ok := b.pools[poolID]; !ok { + if _, ok := b.pools.Get(poolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, poolID) } ui := &UICustomization{UserPoolID: poolID, ClientID: clientID, CSS: css} - b.uiCustomizations[uiKey(poolID, clientID)] = ui + b.uiCustomizations.Put(ui) cp := *ui return &cp, nil @@ -550,11 +546,11 @@ func (b *InMemoryBackend) GetUICustomization(poolID, clientID string) (*UICustom b.mu.RLock("GetUICustomization") defer b.mu.RUnlock() - if _, ok := b.pools[poolID]; !ok { + if _, ok := b.pools.Get(poolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, poolID) } - ui := b.uiCustomizations[uiKey(poolID, clientID)] + ui, _ := b.uiCustomizations.Get(uiKey(poolID, clientID)) if ui == nil { return &UICustomization{UserPoolID: poolID, ClientID: clientID}, nil } @@ -573,14 +569,10 @@ func (b *InMemoryBackend) CreateManagedLoginBranding(userPoolID, clientID string b.mu.Lock("CreateManagedLoginBranding") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if b.managedLoginBrandings[userPoolID] == nil { - b.managedLoginBrandings[userPoolID] = make(map[string]*ManagedLoginBranding) - } - id := "mlb-" + randomAlphanumeric(managedLoginBrandingIDLen) now := time.Now() mlb := &ManagedLoginBranding{ @@ -590,7 +582,7 @@ func (b *InMemoryBackend) CreateManagedLoginBranding(userPoolID, clientID string CreatedAt: now, LastModifiedAt: now, } - b.managedLoginBrandings[userPoolID][id] = mlb + b.managedLoginBrandings.Put(mlb) cp := *mlb @@ -602,11 +594,11 @@ func (b *InMemoryBackend) DescribeManagedLoginBranding(userPoolID, brandingID st b.mu.RLock("DescribeManagedLoginBranding") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - mlb, ok := b.managedLoginBrandings[userPoolID][brandingID] + mlb, ok := b.managedLoginBrandings.Get(managedLoginBrandingKey(userPoolID, brandingID)) if !ok { return nil, fmt.Errorf("%w: managed login branding %q not found in pool %q", ErrUserPoolNotFound, brandingID, userPoolID) @@ -624,11 +616,11 @@ func (b *InMemoryBackend) DescribeManagedLoginBrandingByClient( b.mu.RLock("DescribeManagedLoginBrandingByClient") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - for _, mlb := range b.managedLoginBrandings[userPoolID] { + for _, mlb := range b.managedLoginBrandingsByPool.Get(userPoolID) { if mlb.ClientID == clientID { cp := *mlb @@ -644,11 +636,11 @@ func (b *InMemoryBackend) UpdateManagedLoginBranding(userPoolID, brandingID stri b.mu.Lock("UpdateManagedLoginBranding") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - mlb, ok := b.managedLoginBrandings[userPoolID][brandingID] + mlb, ok := b.managedLoginBrandings.Get(managedLoginBrandingKey(userPoolID, brandingID)) if !ok { return nil, fmt.Errorf("%w: managed login branding %q not found in pool %q", ErrUserPoolNotFound, brandingID, userPoolID) @@ -665,16 +657,16 @@ func (b *InMemoryBackend) DeleteManagedLoginBranding(userPoolID, brandingID stri b.mu.Lock("DeleteManagedLoginBranding") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.managedLoginBrandings[userPoolID][brandingID]; !ok { + if _, ok := b.managedLoginBrandings.Get(managedLoginBrandingKey(userPoolID, brandingID)); !ok { return fmt.Errorf("%w: managed login branding %q not found in pool %q", ErrUserPoolNotFound, brandingID, userPoolID) } - delete(b.managedLoginBrandings[userPoolID], brandingID) + b.managedLoginBrandings.Delete(managedLoginBrandingKey(userPoolID, brandingID)) return nil } @@ -688,12 +680,12 @@ func (b *InMemoryBackend) CreateTerms(userPoolID, text string) (*Terms, error) { b.mu.Lock("CreateTerms") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } t := &Terms{UserPoolID: userPoolID, Text: text} - b.terms[userPoolID] = t + b.terms.Put(t) cp := *t return &cp, nil @@ -704,11 +696,11 @@ func (b *InMemoryBackend) DescribeTerms(userPoolID string) (*Terms, error) { b.mu.RLock("DescribeTerms") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - t := b.terms[userPoolID] + t, _ := b.terms.Get(userPoolID) if t == nil { return &Terms{UserPoolID: userPoolID}, nil } @@ -723,11 +715,11 @@ func (b *InMemoryBackend) ListTerms(userPoolID string) ([]*Terms, error) { b.mu.RLock("ListTerms") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - t := b.terms[userPoolID] + t, _ := b.terms.Get(userPoolID) if t == nil { return []*Terms{}, nil } @@ -742,12 +734,12 @@ func (b *InMemoryBackend) UpdateTerms(userPoolID, text string) (*Terms, error) { b.mu.Lock("UpdateTerms") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } t := &Terms{UserPoolID: userPoolID, Text: text} - b.terms[userPoolID] = t + b.terms.Put(t) cp := *t return &cp, nil @@ -758,11 +750,11 @@ func (b *InMemoryBackend) DeleteTerms(userPoolID string) error { b.mu.Lock("DeleteTerms") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - delete(b.terms, userPoolID) + b.terms.Delete(userPoolID) return nil } @@ -781,14 +773,10 @@ func (b *InMemoryBackend) CreateUserImportJob(userPoolID, jobName string) (*User b.mu.Lock("CreateUserImportJob") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if b.userImportJobs[userPoolID] == nil { - b.userImportJobs[userPoolID] = make(map[string]*UserImportJob) - } - jobID := "import-" + randomAlphanumeric(userImportJobIDLen) job := &UserImportJob{ JobID: jobID, @@ -797,7 +785,7 @@ func (b *InMemoryBackend) CreateUserImportJob(userPoolID, jobName string) (*User Status: "Created", CreatedAt: time.Now(), } - b.userImportJobs[userPoolID][jobID] = job + b.userImportJobs.Put(job) cp := *job @@ -809,11 +797,11 @@ func (b *InMemoryBackend) DescribeUserImportJob(userPoolID, jobID string) (*User b.mu.RLock("DescribeUserImportJob") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - job, ok := b.userImportJobs[userPoolID][jobID] + job, ok := b.userImportJobs.Get(userImportJobKey(userPoolID, jobID)) if !ok { return nil, fmt.Errorf("%w: import job %q not found in pool %q", ErrUserPoolNotFound, jobID, userPoolID) @@ -829,11 +817,11 @@ func (b *InMemoryBackend) ListUserImportJobs(userPoolID string) ([]*UserImportJo b.mu.RLock("ListUserImportJobs") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - poolJobs := b.userImportJobs[userPoolID] + poolJobs := b.userImportJobsByPool.Get(userPoolID) out := make([]*UserImportJob, 0, len(poolJobs)) for _, job := range poolJobs { @@ -851,11 +839,11 @@ func (b *InMemoryBackend) StartUserImportJob(userPoolID, jobID string) (*UserImp b.mu.Lock("StartUserImportJob") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - job, ok := b.userImportJobs[userPoolID][jobID] + job, ok := b.userImportJobs.Get(userImportJobKey(userPoolID, jobID)) if !ok { return nil, fmt.Errorf("%w: import job %q not found in pool %q", ErrUserPoolNotFound, jobID, userPoolID) @@ -877,11 +865,11 @@ func (b *InMemoryBackend) StopUserImportJob(userPoolID, jobID string) (*UserImpo b.mu.Lock("StopUserImportJob") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - job, ok := b.userImportJobs[userPoolID][jobID] + job, ok := b.userImportJobs.Get(userImportJobKey(userPoolID, jobID)) if !ok { return nil, fmt.Errorf("%w: import job %q not found in pool %q", ErrUserPoolNotFound, jobID, userPoolID) @@ -907,11 +895,11 @@ func (b *InMemoryBackend) ListUserPoolClientSecrets(userPoolID, clientID string) b.mu.RLock("ListUserPoolClientSecrets") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok || client.UserPoolID != userPoolID { return nil, fmt.Errorf("%w: client %q not found in pool %q", ErrClientNotFound, clientID, userPoolID) } @@ -928,11 +916,11 @@ func (b *InMemoryBackend) DeleteUserPoolClientSecret(userPoolID, clientID string b.mu.Lock("DeleteUserPoolClientSecret") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - client, ok := b.clients[clientID] + client, ok := b.clients.Get(clientID) if !ok || client.UserPoolID != userPoolID { return fmt.Errorf("%w: client %q not found in pool %q", ErrClientNotFound, clientID, userPoolID) } @@ -1054,11 +1042,11 @@ func (b *InMemoryBackend) AdminGetDevice(userPoolID, username, deviceKey string) b.mu.RLock("AdminGetDevice") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.users[userPoolID][username]; !ok { + if _, ok := b.users.Get(userKey(userPoolID, username)); !ok { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1103,11 +1091,11 @@ func (b *InMemoryBackend) AdminListDevices( b.mu.RLock("AdminListDevices") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, "", fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.users[userPoolID][username]; !ok { + if _, ok := b.users.Get(userKey(userPoolID, username)); !ok { return nil, "", fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1141,11 +1129,11 @@ func (b *InMemoryBackend) AdminUpdateDeviceStatus(userPoolID, username, deviceKe b.mu.Lock("AdminUpdateDeviceStatus") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.users[userPoolID][username]; !ok { + if _, ok := b.users.Get(userKey(userPoolID, username)); !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1235,7 +1223,7 @@ func (b *InMemoryBackend) StartWebAuthnRegistration(accessToken string) (map[str } rpName := user.UserPoolID - if pool, ok := b.pools[user.UserPoolID]; ok && pool.Name != "" { + if pool, ok := b.pools.Get(user.UserPoolID); ok && pool.Name != "" { rpName = pool.Name } @@ -1458,11 +1446,11 @@ func (b *InMemoryBackend) AdminListUserAuthEvents( b.mu.RLock("AdminListUserAuthEvents") defer b.mu.RUnlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return nil, "", fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.users[userPoolID][username]; !ok { + if _, ok := b.users.Get(userKey(userPoolID, username)); !ok { return nil, "", fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1474,11 +1462,11 @@ func (b *InMemoryBackend) AdminListUserAuthEvents( // updateAuthEventFeedbackLocked validates and applies feedback to a stored // auth event. Caller must hold b.mu (write lock). func (b *InMemoryBackend) updateAuthEventFeedbackLocked(userPoolID, username, eventID, feedbackValue string) error { - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - if _, ok := b.users[userPoolID][username]; !ok { + if _, ok := b.users.Get(userKey(userPoolID, username)); !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1525,11 +1513,11 @@ func (b *InMemoryBackend) AdminSetUserSettings(userPoolID, username string, mfaO b.mu.Lock("AdminSetUserSettings") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } - user, ok := b.users[userPoolID][username] + user, ok := b.users.Get(userKey(userPoolID, username)) if !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, username) } @@ -1563,7 +1551,7 @@ func (b *InMemoryBackend) AdminLinkProviderForUser( b.mu.Lock("AdminLinkProviderForUser") defer b.mu.Unlock() - if _, ok := b.pools[userPoolID]; !ok { + if _, ok := b.pools.Get(userPoolID); !ok { return fmt.Errorf("%w: pool %q not found", ErrUserPoolNotFound, userPoolID) } @@ -1571,7 +1559,7 @@ func (b *InMemoryBackend) AdminLinkProviderForUser( return fmt.Errorf("%w: DestinationUser.ProviderAttributeValue is required", ErrInvalidParameter) } - user, ok := b.users[userPoolID][destinationUsername] + user, ok := b.users.Get(userKey(userPoolID, destinationUsername)) if !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, destinationUsername) } diff --git a/services/cognitoidp/deletion_protection_test.go b/services/cognitoidp/deletion_protection_test.go new file mode 100644 index 000000000..8161ade84 --- /dev/null +++ b/services/cognitoidp/deletion_protection_test.go @@ -0,0 +1,86 @@ +package cognitoidp_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/cognitoidp" +) + +// Test_DeleteUserPool_DeletionProtection proves DeleteUserPool refuses to delete a pool +// whose DeletionProtection is "ACTIVE" (matching real Cognito's InvalidParameterException), +// while a pool left at the default ("INACTIVE") or explicitly deactivated deletes normally. +func Test_DeleteUserPool_DeletionProtection(t *testing.T) { + t.Parallel() + + cases := []struct { + wantErr error + name string + deletionProtection string + }{ + { + name: "default (unset) deletes normally", + deletionProtection: "", + wantErr: nil, + }, + { + name: "INACTIVE deletes normally", + deletionProtection: "INACTIVE", + wantErr: nil, + }, + { + name: "ACTIVE refuses deletion", + deletionProtection: "ACTIVE", + wantErr: cognitoidp.ErrInvalidParameter, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := cognitoidp.NewInMemoryBackend("000000000000", "us-east-1", "http://localhost:8000") + + pool, err := b.CreateUserPoolWithOpts("dp-pool-"+sanitizeTestName(tc.name), cognitoidp.UserPoolOptions{ + DeletionProtection: tc.deletionProtection, + }) + require.NoError(t, err) + + deleteErr := b.DeleteUserPool(pool.ID) + + if tc.wantErr == nil { + require.NoError(t, deleteErr) + assert.Equal(t, 0, b.UserPoolCount()) + + return + } + + require.ErrorIs(t, deleteErr, tc.wantErr) + assert.Equal(t, 1, b.UserPoolCount(), "pool must survive a refused delete") + }) + } +} + +// Test_DeleteUserPool_DeletionProtection_CanBeDisabledThenDeleted proves the documented +// remediation path: flipping DeletionProtection back to INACTIVE via UpdateUserPool allows +// the subsequent delete to succeed. +func Test_DeleteUserPool_DeletionProtection_CanBeDisabledThenDeleted(t *testing.T) { + t.Parallel() + + b := cognitoidp.NewInMemoryBackend("000000000000", "us-east-1", "http://localhost:8000") + + pool, err := b.CreateUserPoolWithOpts("dp-toggle-pool", cognitoidp.UserPoolOptions{ + DeletionProtection: "ACTIVE", + }) + require.NoError(t, err) + + require.ErrorIs(t, b.DeleteUserPool(pool.ID), cognitoidp.ErrInvalidParameter) + + require.NoError(t, b.UpdateUserPoolWithOpts(pool.ID, "", cognitoidp.UserPoolOptions{ + DeletionProtection: "INACTIVE", + })) + + require.NoError(t, b.DeleteUserPool(pool.ID)) +} diff --git a/services/cognitoidp/export_test.go b/services/cognitoidp/export_test.go index 964146592..2add6cf0e 100644 --- a/services/cognitoidp/export_test.go +++ b/services/cognitoidp/export_test.go @@ -7,7 +7,7 @@ func (b *InMemoryBackend) UserPoolCount() int { b.mu.RLock("UserPoolCount") defer b.mu.RUnlock() - return len(b.pools) + return b.pools.Len() } // UserCount returns the total number of users across all pools. For testing only. @@ -15,12 +15,7 @@ func (b *InMemoryBackend) UserCount() int { b.mu.RLock("UserCount") defer b.mu.RUnlock() - total := 0 - for _, poolUsers := range b.users { - total += len(poolUsers) - } - - return total + return b.users.Len() } // ClientCount returns the number of user pool clients. For testing only. @@ -28,7 +23,7 @@ func (b *InMemoryBackend) ClientCount() int { b.mu.RLock("ClientCount") defer b.mu.RUnlock() - return len(b.clients) + return b.clients.Len() } // RefreshTokenCount returns the number of refresh tokens in the backend. For testing only. @@ -69,11 +64,9 @@ func (b *InMemoryBackend) ClearConfirmCodeForTest(poolID, username string) { b.mu.Lock("ClearConfirmCodeForTest") defer b.mu.Unlock() - if users, ok := b.users[poolID]; ok { - if u, ok2 := users[username]; ok2 { - u.ConfirmCode = "" - u.ConfirmCodeExpiresAt = time.Time{} - } + if u, ok := b.users.Get(userKey(poolID, username)); ok { + u.ConfirmCode = "" + u.ConfirmCodeExpiresAt = time.Time{} } } @@ -82,10 +75,8 @@ func (b *InMemoryBackend) ExpireConfirmCodeForTest(poolID, username string) { b.mu.Lock("ExpireConfirmCodeForTest") defer b.mu.Unlock() - if users, ok := b.users[poolID]; ok { - if u, ok2 := users[username]; ok2 { - u.ConfirmCodeExpiresAt = time.Now().Add(-time.Hour) - } + if u, ok := b.users.Get(userKey(poolID, username)); ok { + u.ConfirmCodeExpiresAt = time.Now().Add(-time.Hour) } } @@ -94,13 +85,27 @@ func (b *InMemoryBackend) UserPoolID(clientID string) string { b.mu.RLock("UserPoolID") defer b.mu.RUnlock() - if c, ok := b.clients[clientID]; ok { + if c, ok := b.clients.Get(clientID); ok { return c.UserPoolID } return "" } +// GetMFASessionCodeForTest returns the one-time SMS_MFA/EMAIL_OTP code generated for a +// pending MFA session (SOFTWARE_TOKEN_MFA sessions have no stored code — it's verified +// cryptographically against the user's TOTP secret instead). For testing only. +func (b *InMemoryBackend) GetMFASessionCodeForTest(session string) string { + b.mu.RLock("GetMFASessionCodeForTest") + defer b.mu.RUnlock() + + if entry, ok := b.mfaSessions[session]; ok { + return entry.Code + } + + return "" +} + // GetAttrVerificationCodeForTest returns the pending verification code for a user attribute. For testing only. func (b *InMemoryBackend) GetAttrVerificationCodeForTest(poolID, username, attrName string) string { b.mu.RLock("GetAttrVerificationCodeForTest") diff --git a/services/cognitoidp/persistence.go b/services/cognitoidp/persistence.go index 4a3b1d87d..803283112 100644 --- a/services/cognitoidp/persistence.go +++ b/services/cognitoidp/persistence.go @@ -12,6 +12,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) var ( @@ -19,6 +20,15 @@ var ( errNotRSAKey = errors.New("private key is not *rsa.PrivateKey") ) +// cognitoidpSnapshotVersion identifies the shape of [backendSnapshot]. It must +// be bumped whenever a change to a DTO or to backendSnapshot itself would make +// an older snapshot unsafe to decode as the current shape. Restore compares +// this against the persisted value and discards (rather than attempts to +// partially decode) any mismatch — see Restore below. There was no version +// guard prior to Phase 3.3's pkgs/store conversion, so any pre-existing +// snapshot decodes with Version 0 and is treated as incompatible. +const cognitoidpSnapshotVersion = 1 + // userPoolSnapshot holds the serializable fields of a UserPool. type userPoolSnapshot struct { PasswordPolicy *PasswordPolicy `json:"passwordPolicy,omitempty"` @@ -34,7 +44,18 @@ type userPoolSnapshot struct { AutoVerifiedAttributes []string `json:"autoVerifiedAttributes,omitempty"` } +// poolSnapshotKeyFn is the [store.Table] key function for the ephemeral pool +// DTO table, mirroring poolsKeyFn. +func poolSnapshotKeyFn(v *userPoolSnapshot) string { return v.ID } + // userSnapshot is a copy of User safe for JSON serialization. +// +// NOTE: this DTO does not carry TOTPSecret, TOTPVerified, PreferredMfaSetting, +// UserMFASettingList, or LastAuthTime -- that gap predates the Phase 3.3 +// store conversion (see services/cognitoidp/PARITY.md / bd history) and is +// preserved as-is here rather than silently fixed, per the mechanical-swap +// scope of this conversion. PasswordHash and every other field below were +// already persisted and remain persisted unchanged. type userSnapshot struct { CreatedAt string `json:"createdAt,omitempty"` UpdatedAt string `json:"updatedAt,omitempty"` @@ -51,32 +72,41 @@ type userSnapshot struct { Enabled bool `json:"enabled,omitempty"` } +// userSnapshotKeyFn is the [store.Table] key function for the ephemeral user +// DTO table, mirroring usersKeyFn. +func userSnapshotKeyFn(v *userSnapshot) string { return userKey(v.UserPoolID, v.Username) } + +// backendSnapshot is the top-level on-disk shape for the cognitoidp backend. +// +// Tables holds one JSON-encoded array per registered table, produced by +// [store.Registry.SnapshotAll]: "pools" and "users" go through a DTO +// transform (UserPool/User carry unexported/derived fields that cannot +// round-trip through JSON directly -- see buildPoolSnapshot/buildUserSnapshot), +// while every other converted table (clients, groups, resourceServers, +// identityProviders, domains, terms, userImportJobs, managedLoginBrandings, +// uiCustomizations, typedRiskConfigurations) is clean enough to register +// directly, with no DTO needed. +// +// Every field below the Tables map is a resource left as a plain map on +// InMemoryBackend (see store_setup.go's registerAllTables doc for why each +// one can't be a store.Table) and is persisted exactly as it was before this +// conversion. type backendSnapshot struct { - Pools map[string]*userPoolSnapshot `json:"pools,omitempty"` - Clients map[string]*UserPoolClient `json:"clients,omitempty"` - Users map[string]map[string]*userSnapshot `json:"users,omitempty"` - RefreshTokens map[string]*refreshTokenEntry `json:"refreshTokens,omitempty"` - Groups map[string]map[string]*Group `json:"groups,omitempty"` - GroupMembers map[string]map[string]map[string]struct{} `json:"groupMembers,omitempty"` - ResourceServers map[string]map[string]*ResourceServer `json:"resourceServers,omitempty"` - TokenRevokedBefore map[string]time.Time `json:"tokenRevokedBefore,omitempty"` - Domains map[string]*UserPoolDomain `json:"domains,omitempty"` - ResourceTags map[string]map[string]string `json:"resourceTags,omitempty"` - RiskConfigurations map[string]*RiskConfiguration `json:"riskConfigurations,omitempty"` - LogDeliveryConfigs map[string]*LogDeliveryConfig `json:"logDeliveryConfigs,omitempty"` - UICustomizations map[string]*UICustomization `json:"uiCustomizations,omitempty"` - ManagedLoginBrandings map[string]map[string]*ManagedLoginBranding `json:"managedLoginBrandings,omitempty"` - Terms map[string]*Terms `json:"terms,omitempty"` - UserImportJobs map[string]map[string]*UserImportJob `json:"userImportJobs,omitempty"` - PoolMfaConfigs map[string]*UserPoolMfaFullConfig `json:"poolMfaConfigs,omitempty"` - TypedRiskConfigurations map[string]*TypedRiskConfiguration `json:"typedRiskConfigurations,omitempty"` - IdentityProviders map[string]map[string]*IdentityProvider `json:"identityProviders,omitempty"` - Devices map[string]map[string]*Device `json:"devices,omitempty"` - WebAuthnCredentials map[string]map[string]*WebAuthnCredential `json:"webauthnCredentials,omitempty"` - AuthEvents map[string]map[string]*AuthEvent `json:"authEvents,omitempty"` - AccountID string `json:"accountId,omitempty"` - Region string `json:"region,omitempty"` - Endpoint string `json:"endpoint,omitempty"` + Tables map[string]json.RawMessage `json:"tables,omitempty"` + RefreshTokens map[string]*refreshTokenEntry `json:"refreshTokens,omitempty"` + GroupMembers map[string]map[string]map[string]struct{} `json:"groupMembers,omitempty"` + TokenRevokedBefore map[string]time.Time `json:"tokenRevokedBefore,omitempty"` + ResourceTags map[string]map[string]string `json:"resourceTags,omitempty"` + RiskConfigurations map[string]*RiskConfiguration `json:"riskConfigurations,omitempty"` + LogDeliveryConfigs map[string]*LogDeliveryConfig `json:"logDeliveryConfigs,omitempty"` + PoolMfaConfigs map[string]*UserPoolMfaFullConfig `json:"poolMfaConfigs,omitempty"` + Devices map[string]map[string]*Device `json:"devices,omitempty"` + WebAuthnCredentials map[string]map[string]*WebAuthnCredential `json:"webauthnCredentials,omitempty"` + AuthEvents map[string]map[string]*AuthEvent `json:"authEvents,omitempty"` + AccountID string `json:"accountId,omitempty"` + Region string `json:"region,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Version int `json:"version"` } func marshalRSAKey(key *rsa.PrivateKey) (string, error) { @@ -115,82 +145,64 @@ func unmarshalRSAKey(pemStr string) (*rsa.PrivateKey, error) { return rsaKey, nil } -// buildPoolSnapshots converts live UserPools into their serializable form. -func buildPoolSnapshots(ctx context.Context, pools map[string]*UserPool) map[string]*userPoolSnapshot { - poolSnaps := make(map[string]*userPoolSnapshot, len(pools)) - - for id, p := range pools { - pem, err := marshalRSAKey(p.issuer.privateKey) - if err != nil { - logger.Load(ctx). - WarnContext(ctx, "cognitoidp: failed to marshal RSA key for pool snapshot", "poolId", id, "error", err) - pem = "" - } - - var ppSnap *PasswordPolicy - if p.PasswordPolicy != nil { - pp := *p.PasswordPolicy - ppSnap = &pp - } +// buildPoolSnapshot converts a live UserPool into its serializable form. +func buildPoolSnapshot(ctx context.Context, p *UserPool) *userPoolSnapshot { + pem, err := marshalRSAKey(p.issuer.privateKey) + if err != nil { + logger.Load(ctx). + WarnContext(ctx, "cognitoidp: failed to marshal RSA key for pool snapshot", "poolId", p.ID, "error", err) + pem = "" + } - var avAttrs []string - if len(p.AutoVerifiedAttributes) > 0 { - avAttrs = make([]string, len(p.AutoVerifiedAttributes)) - copy(avAttrs, p.AutoVerifiedAttributes) - } + var ppSnap *PasswordPolicy + if p.PasswordPolicy != nil { + pp := *p.PasswordPolicy + ppSnap = &pp + } - poolSnaps[id] = &userPoolSnapshot{ - CreatedAt: p.CreatedAt.Format("2006-01-02T15:04:05Z"), - ID: p.ID, - Name: p.Name, - ARN: p.ARN, - IssuerURL: p.issuer.issuerURL, - KeyID: p.issuer.keyID, - PrivKeyPEM: pem, - CustomAttributes: p.CustomAttributes, - MfaConfiguration: p.MfaConfiguration, - PasswordPolicy: ppSnap, - AutoVerifiedAttributes: avAttrs, - } + var avAttrs []string + if len(p.AutoVerifiedAttributes) > 0 { + avAttrs = make([]string, len(p.AutoVerifiedAttributes)) + copy(avAttrs, p.AutoVerifiedAttributes) } - return poolSnaps + return &userPoolSnapshot{ + CreatedAt: p.CreatedAt.Format("2006-01-02T15:04:05Z"), + ID: p.ID, + Name: p.Name, + ARN: p.ARN, + IssuerURL: p.issuer.issuerURL, + KeyID: p.issuer.keyID, + PrivKeyPEM: pem, + CustomAttributes: p.CustomAttributes, + MfaConfiguration: p.MfaConfiguration, + PasswordPolicy: ppSnap, + AutoVerifiedAttributes: avAttrs, + } } -// buildUserSnapshots converts live Users into their serializable form. -func buildUserSnapshots(users map[string]map[string]*User) map[string]map[string]*userSnapshot { - userSnaps := make(map[string]map[string]*userSnapshot, len(users)) - - for poolID, poolUsers := range users { - snaps := make(map[string]*userSnapshot, len(poolUsers)) - - for username, u := range poolUsers { - var codeExpiry string - if !u.ConfirmCodeExpiresAt.IsZero() { - codeExpiry = u.ConfirmCodeExpiresAt.Format("2006-01-02T15:04:05Z") - } - - snaps[username] = &userSnapshot{ - CreatedAt: u.CreatedAt.Format("2006-01-02T15:04:05Z"), - UpdatedAt: u.UpdatedAt.Format("2006-01-02T15:04:05Z"), - ConfirmCodeExpiresAt: codeExpiry, - Attributes: u.Attributes, - Sub: u.Sub, - Username: u.Username, - UserPoolID: u.UserPoolID, - PasswordHash: u.PasswordHash, - Status: u.Status, - ConfirmCode: u.ConfirmCode, - MFAOptions: u.MFAOptions, - LinkedProviders: u.LinkedProviders, - Enabled: u.Enabled, - } - } - - userSnaps[poolID] = snaps +// buildUserSnapshot converts a live User into its serializable form. +func buildUserSnapshot(u *User) *userSnapshot { + var codeExpiry string + if !u.ConfirmCodeExpiresAt.IsZero() { + codeExpiry = u.ConfirmCodeExpiresAt.Format("2006-01-02T15:04:05Z") } - return userSnaps + return &userSnapshot{ + CreatedAt: u.CreatedAt.Format("2006-01-02T15:04:05Z"), + UpdatedAt: u.UpdatedAt.Format("2006-01-02T15:04:05Z"), + ConfirmCodeExpiresAt: codeExpiry, + Attributes: u.Attributes, + Sub: u.Sub, + Username: u.Username, + UserPoolID: u.UserPoolID, + PasswordHash: u.PasswordHash, + Status: u.Status, + ConfirmCode: u.ConfirmCode, + MFAOptions: u.MFAOptions, + LinkedProviders: u.LinkedProviders, + Enabled: u.Enabled, + } } // Snapshot serialises the backend state to JSON. @@ -198,35 +210,59 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() - poolSnaps := buildPoolSnapshots(ctx, b.pools) - userSnaps := buildUserSnapshots(b.users) + // Build a throwaway DTO registry purely to reuse store's deterministic, + // type-erased JSON encoding (store.Registry.SnapshotAll) instead of + // hand-rolling the marshal step. pools/users need a DTO transform (see + // buildPoolSnapshot/buildUserSnapshot); every other converted table is + // already cleanly serializable, so its live *store.Table is registered + // directly -- Register only stores a reference, so this does not disturb + // b.registry. + dtoReg := store.NewRegistry() + + poolDTOs := store.Register(dtoReg, "pools", store.New(poolSnapshotKeyFn)) + for _, p := range b.pools.All() { + poolDTOs.Put(buildPoolSnapshot(ctx, p)) + } + + userDTOs := store.Register(dtoReg, "users", store.New(userSnapshotKeyFn)) + for _, u := range b.users.All() { + userDTOs.Put(buildUserSnapshot(u)) + } + + store.Register(dtoReg, "clients", b.clients) + store.Register(dtoReg, "groups", b.groups) + store.Register(dtoReg, "resourceServers", b.resourceServers) + store.Register(dtoReg, "identityProviders", b.identityProviders) + store.Register(dtoReg, "domains", b.domains) + store.Register(dtoReg, "terms", b.terms) + store.Register(dtoReg, "userImportJobs", b.userImportJobs) + store.Register(dtoReg, "managedLoginBrandings", b.managedLoginBrandings) + store.Register(dtoReg, "uiCustomizations", b.uiCustomizations) + store.Register(dtoReg, "typedRiskConfigurations", b.typedRiskConfigurations) + + tables, err := dtoReg.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, "cognitoidp: failed to marshal backend snapshot", "error", err) + + return nil + } snap := backendSnapshot{ - Pools: poolSnaps, - Clients: b.clients, - Users: userSnaps, - RefreshTokens: b.refreshTokens, - Groups: b.groups, - GroupMembers: b.groupMembers, - ResourceServers: b.resourceServers, - TokenRevokedBefore: b.tokenRevokedBefore, - Domains: b.domains, - ResourceTags: b.resourceTags, - RiskConfigurations: b.riskConfigurations, - LogDeliveryConfigs: b.logDeliveryConfigs, - UICustomizations: b.uiCustomizations, - ManagedLoginBrandings: b.managedLoginBrandings, - Terms: b.terms, - UserImportJobs: b.userImportJobs, - PoolMfaConfigs: b.poolMfaConfigs, - TypedRiskConfigurations: b.typedRiskConfigurations, - IdentityProviders: b.identityProviders, - Devices: b.devices, - WebAuthnCredentials: b.webauthnCredentials, - AuthEvents: b.authEvents, - AccountID: b.accountID, - Region: b.region, - Endpoint: b.endpoint, + Version: cognitoidpSnapshotVersion, + Tables: tables, + RefreshTokens: b.refreshTokens, + GroupMembers: b.groupMembers, + TokenRevokedBefore: b.tokenRevokedBefore, + ResourceTags: b.resourceTags, + RiskConfigurations: b.riskConfigurations, + LogDeliveryConfigs: b.logDeliveryConfigs, + PoolMfaConfigs: b.poolMfaConfigs, + Devices: b.devices, + WebAuthnCredentials: b.webauthnCredentials, + AuthEvents: b.authEvents, + AccountID: b.accountID, + Region: b.region, + Endpoint: b.endpoint, } data, err := json.Marshal(snap) @@ -247,76 +283,111 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { return err } + b.mu.Lock("Restore") + defer b.mu.Unlock() + + if snap.Version != cognitoidpSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. + logger.Load(ctx).WarnContext(ctx, + "cognitoidp: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", cognitoidpSnapshotVersion) + + b.resetForIncompatibleSnapshotLocked() + + return nil + } + normalizeBackendSnapshot(&snap) - pools, poolsByName, err := restorePoolsFromSnapshot(snap.Pools) + if err := b.restoreTablesLocked(snap.Tables); err != nil { + return err + } + + b.restoreRawMapsLocked(&snap) + + return nil +} + +// resetForIncompatibleSnapshotLocked clears every table and raw map to the +// empty state a fresh NewInMemoryBackend would have. Caller must hold b.mu in +// write mode. +func (b *InMemoryBackend) resetForIncompatibleSnapshotLocked() { + b.registry.ResetAll() + b.refreshTokens = make(map[string]*refreshTokenEntry) + b.refreshTokensByClient = make(map[string]map[string]struct{}) + b.refreshTokensByUser = make(map[string]map[string]struct{}) + b.groupMembers = make(map[string]map[string]map[string]struct{}) + b.tokenRevokedBefore = make(map[string]time.Time) + b.resourceTags = make(map[string]map[string]string) + b.riskConfigurations = make(map[string]*RiskConfiguration) + b.logDeliveryConfigs = make(map[string]*LogDeliveryConfig) + b.devices = make(map[string]map[string]*Device) + b.webauthnCredentials = make(map[string]map[string]*WebAuthnCredential) + b.authEvents = make(map[string]map[string]*AuthEvent) +} + +// restoreTablesLocked decodes tables into every store.Table-backed resource. +// pools/users go through a DTO transform (see restorePoolsFromSnapshot/ +// restoreUsersFromSnapshot); every other converted table restores straight +// into its live *store.Table (Restore rebuilds the table's primary map AND +// every secondary index registered on it, e.g. clientsByPool/groupsByPool, so +// no separate index rebuild step is needed for these). Caller must hold b.mu +// in write mode. +func (b *InMemoryBackend) restoreTablesLocked(tables map[string]json.RawMessage) error { + dtoReg := store.NewRegistry() + + poolDTOs := store.Register(dtoReg, "pools", store.New(poolSnapshotKeyFn)) + userDTOs := store.Register(dtoReg, "users", store.New(userSnapshotKeyFn)) + store.Register(dtoReg, "clients", b.clients) + store.Register(dtoReg, "groups", b.groups) + store.Register(dtoReg, "resourceServers", b.resourceServers) + store.Register(dtoReg, "identityProviders", b.identityProviders) + store.Register(dtoReg, "domains", b.domains) + store.Register(dtoReg, "terms", b.terms) + store.Register(dtoReg, "userImportJobs", b.userImportJobs) + store.Register(dtoReg, "managedLoginBrandings", b.managedLoginBrandings) + store.Register(dtoReg, "uiCustomizations", b.uiCustomizations) + store.Register(dtoReg, "typedRiskConfigurations", b.typedRiskConfigurations) + + if err := dtoReg.RestoreAll(tables); err != nil { + return fmt.Errorf("cognitoidp: restore snapshot tables: %w", err) + } + + pools, err := restorePoolsFromSnapshot(poolDTOs.All()) if err != nil { return err } - users := restoreUsersFromSnapshot(snap.Users) + b.pools.Restore(pools) + b.users.Restore(restoreUsersFromSnapshot(userDTOs.All())) - b.mu.Lock("Restore") - defer b.mu.Unlock() + return nil +} - b.pools = pools - b.poolsByName = poolsByName - b.users = users - b.usersBySub = buildUsersBySubIndex(users) - b.clients = snap.Clients - b.clientsByPool = buildClientsByPoolIndex(b.clients) +// restoreRawMapsLocked loads every resource left as a plain map (see +// store_setup.go's registerAllTables doc for why) plus top-level backend +// scalars from snap. Caller must hold b.mu in write mode. +func (b *InMemoryBackend) restoreRawMapsLocked(snap *backendSnapshot) { b.refreshTokens = snap.RefreshTokens b.refreshTokensByClient = buildRefreshTokensByClientIndex(b.refreshTokens) b.refreshTokensByUser = buildRefreshTokensByUserIndex(b.refreshTokens) - b.groups = snap.Groups b.groupMembers = snap.GroupMembers - b.resourceServers = snap.ResourceServers b.tokenRevokedBefore = snap.TokenRevokedBefore - b.domains = snap.Domains b.resourceTags = snap.ResourceTags b.riskConfigurations = snap.RiskConfigurations b.logDeliveryConfigs = snap.LogDeliveryConfigs - b.uiCustomizations = snap.UICustomizations - b.managedLoginBrandings = snap.ManagedLoginBrandings - b.terms = snap.Terms - b.userImportJobs = snap.UserImportJobs b.poolMfaConfigs = snap.PoolMfaConfigs - b.typedRiskConfigurations = snap.TypedRiskConfigurations - b.identityProviders = snap.IdentityProviders b.devices = snap.Devices b.webauthnCredentials = snap.WebAuthnCredentials b.authEvents = snap.AuthEvents b.accountID = snap.AccountID b.region = snap.Region b.endpoint = snap.Endpoint - - return nil -} - -func buildUsersBySubIndex(users map[string]map[string]*User) map[string]string { - index := make(map[string]string) - for poolID, poolUsers := range users { - for _, u := range poolUsers { - if u.Sub != "" { - index[poolID+":"+u.Sub] = u.Username - } - } - } - - return index -} - -func buildClientsByPoolIndex(clients map[string]*UserPoolClient) map[string]map[string]*UserPoolClient { - index := make(map[string]map[string]*UserPoolClient) - for _, client := range clients { - if index[client.UserPoolID] == nil { - index[client.UserPoolID] = make(map[string]*UserPoolClient) - } - - index[client.UserPoolID][client.ClientID] = client - } - - return index } func buildRefreshTokensByClientIndex( @@ -339,22 +410,18 @@ func buildRefreshTokensByUserIndex( ) map[string]map[string]struct{} { index := make(map[string]map[string]struct{}) for token, entry := range refreshTokens { - userKey := entry.PoolID + ":" + entry.Username - if index[userKey] == nil { - index[userKey] = make(map[string]struct{}) + key := entry.PoolID + ":" + entry.Username + if index[key] == nil { + index[key] = make(map[string]struct{}) } - index[userKey][token] = struct{}{} + index[key][token] = struct{}{} } return index } func normalizeBackendSnapshot(snap *backendSnapshot) { - if snap.Clients == nil { - snap.Clients = make(map[string]*UserPoolClient) - } - if snap.RefreshTokens == nil { snap.RefreshTokens = make(map[string]*refreshTokenEntry) } @@ -366,18 +433,10 @@ func normalizeBackendSnapshot(snap *backendSnapshot) { } } - if snap.Groups == nil { - snap.Groups = make(map[string]map[string]*Group) - } - if snap.GroupMembers == nil { snap.GroupMembers = make(map[string]map[string]map[string]struct{}) } - if snap.ResourceServers == nil { - snap.ResourceServers = make(map[string]map[string]*ResourceServer) - } - if snap.TokenRevokedBefore == nil { snap.TokenRevokedBefore = make(map[string]time.Time) } @@ -395,16 +454,15 @@ func normalizeBackendSnapshot(snap *backendSnapshot) { } } -func restorePoolsFromSnapshot( - poolSnapshots map[string]*userPoolSnapshot, -) (map[string]*UserPool, map[string]*UserPool, error) { - pools := make(map[string]*UserPool, len(poolSnapshots)) - poolsByName := make(map[string]*UserPool, len(poolSnapshots)) +// restorePoolsFromSnapshot rebuilds live UserPools (including their RSA token +// issuers) from persisted DTOs. +func restorePoolsFromSnapshot(poolSnapshots []*userPoolSnapshot) ([]*UserPool, error) { + pools := make([]*UserPool, 0, len(poolSnapshots)) - for id, ps := range poolSnapshots { + for _, ps := range poolSnapshots { rsaKey, err := unmarshalRSAKey(ps.PrivKeyPEM) if err != nil { - return nil, nil, fmt.Errorf("restoring user pool %q: %w", id, err) + return nil, fmt.Errorf("restoring user pool %q: %w", ps.ID, err) } createdAt, _ := time.Parse("2006-01-02T15:04:05Z", ps.CreatedAt) @@ -424,44 +482,40 @@ func restorePoolsFromSnapshot( pool.issuer = newTokenIssuerFromKey(rsaKey, ps.KeyID, ps.IssuerURL) } - pools[id] = pool - poolsByName[ps.Name] = pool + pools = append(pools, pool) } - return pools, poolsByName, nil + return pools, nil } -func restoreUsersFromSnapshot(poolUsers map[string]map[string]*userSnapshot) map[string]map[string]*User { - users := make(map[string]map[string]*User, len(poolUsers)) - for poolID, usersByName := range poolUsers { - restored := make(map[string]*User, len(usersByName)) - for username, us := range usersByName { - createdAt, _ := time.Parse("2006-01-02T15:04:05Z", us.CreatedAt) - updatedAt, _ := time.Parse("2006-01-02T15:04:05Z", us.UpdatedAt) - codeExpiry, _ := time.Parse("2006-01-02T15:04:05Z", us.ConfirmCodeExpiresAt) - - if updatedAt.IsZero() { - updatedAt = createdAt - } - - restored[username] = &User{ - CreatedAt: createdAt, - UpdatedAt: updatedAt, - ConfirmCodeExpiresAt: codeExpiry, - Attributes: us.Attributes, - Sub: us.Sub, - Username: us.Username, - UserPoolID: us.UserPoolID, - PasswordHash: us.PasswordHash, - Status: us.Status, - ConfirmCode: us.ConfirmCode, - MFAOptions: us.MFAOptions, - LinkedProviders: us.LinkedProviders, - Enabled: us.Enabled, - } +// restoreUsersFromSnapshot rebuilds live Users from persisted DTOs. +func restoreUsersFromSnapshot(userSnapshots []*userSnapshot) []*User { + users := make([]*User, 0, len(userSnapshots)) + + for _, us := range userSnapshots { + createdAt, _ := time.Parse("2006-01-02T15:04:05Z", us.CreatedAt) + updatedAt, _ := time.Parse("2006-01-02T15:04:05Z", us.UpdatedAt) + codeExpiry, _ := time.Parse("2006-01-02T15:04:05Z", us.ConfirmCodeExpiresAt) + + if updatedAt.IsZero() { + updatedAt = createdAt } - users[poolID] = restored + users = append(users, &User{ + CreatedAt: createdAt, + UpdatedAt: updatedAt, + ConfirmCodeExpiresAt: codeExpiry, + Attributes: us.Attributes, + Sub: us.Sub, + Username: us.Username, + UserPoolID: us.UserPoolID, + PasswordHash: us.PasswordHash, + Status: us.Status, + ConfirmCode: us.ConfirmCode, + MFAOptions: us.MFAOptions, + LinkedProviders: us.LinkedProviders, + Enabled: us.Enabled, + }) } return users diff --git a/services/cognitoidp/persistence_test.go b/services/cognitoidp/persistence_test.go new file mode 100644 index 000000000..fa417457a --- /dev/null +++ b/services/cognitoidp/persistence_test.go @@ -0,0 +1,231 @@ +package cognitoidp_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/cognitoidp" +) + +// TestInMemoryBackend_SnapshotRestore is a regression guard for the Phase 3.3 +// pkgs/store conversion: it seeds one instance of every resource kind the +// backend owns (both store.Table-backed and the maps left raw -- see +// store_setup.go's registerAllTables doc for why each raw map can't be a +// store.Table) and checks that every one of them survives a Snapshot/Restore +// round trip into a fresh backend byte-for-byte equivalent. +func TestInMemoryBackend_SnapshotRestore(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + tests := []struct { + setup func(t *testing.T, b *cognitoidp.InMemoryBackend) + verify func(t *testing.T, b *cognitoidp.InMemoryBackend) + name string + }{ + { + name: "empty_backend_round_trip", + setup: func(*testing.T, *cognitoidp.InMemoryBackend) {}, + verify: func(t *testing.T, b *cognitoidp.InMemoryBackend) { + t.Helper() + assert.Empty(t, b.ListUserPools()) + }, + }, + { + name: "full_state_round_trip", + setup: func(t *testing.T, b *cognitoidp.InMemoryBackend) { + t.Helper() + + pool, err := b.CreateUserPool("test-pool") + require.NoError(t, err) + + client, err := b.CreateUserPoolClient(pool.ID, "test-client") + require.NoError(t, err) + + user, err := b.AdminCreateUserWithPolicy(pool.ID, "alice", "TempPass123!", map[string]string{ + "email": "alice@example.com", + }) + require.NoError(t, err) + require.NotEmpty(t, user.PasswordHash) + + // Give the user a permanent password so USER_PASSWORD_AUTH succeeds + // below and mints a refresh token to round-trip too. + require.NoError(t, b.AdminSetUserPassword(pool.ID, "alice", "Permanent123!", true)) + + _, err = b.CreateGroup(pool.ID, "admins", "administrators", 1) + require.NoError(t, err) + require.NoError(t, b.AdminAddUserToGroup(pool.ID, "alice", "admins")) + + _, err = b.CreateResourceServer(pool.ID, "https://api.example.com", "API", nil) + require.NoError(t, err) + + _, err = b.CreateIdentityProvider(pool.ID, "Google", "Google", map[string]string{"client_id": "abc"}) + require.NoError(t, err) + + _, err = b.CreateUserPoolDomain(pool.ID, "test-domain") + require.NoError(t, err) + + _, err = b.CreateTerms(pool.ID, "accept these terms") + require.NoError(t, err) + + _, err = b.CreateUserImportJob(pool.ID, "import-1") + require.NoError(t, err) + + _, err = b.CreateManagedLoginBranding(pool.ID, client.ClientID) + require.NoError(t, err) + + _, err = b.SetUICustomization(pool.ID, client.ClientID, "body{color:red}") + require.NoError(t, err) + + require.NoError(t, b.SetTypedRiskConfiguration(&cognitoidp.TypedRiskConfiguration{ + UserPoolID: pool.ID, + ClientID: client.ClientID, + })) + + poolARN := "arn:aws:cognito-idp:us-east-1:000000000000:userpool/" + pool.ID + b.TagResource(poolARN, map[string]string{"env": "test"}) + + // USER_PASSWORD_AUTH mints a refresh token -- exercises the raw + // refreshTokens/refreshTokensByClient/refreshTokensByUser maps. + _, err = b.InitiateAuth(client.ClientID, "USER_PASSWORD_AUTH", "alice", "Permanent123!") + require.NoError(t, err) + }, + verify: func(t *testing.T, b *cognitoidp.InMemoryBackend) { + t.Helper() + + pools := b.ListUserPools() + require.Len(t, pools, 1) + poolID := pools[0].ID + assert.Equal(t, "test-pool", pools[0].Name) + + clients, err := b.ListUserPoolClients(poolID) + require.NoError(t, err) + require.Len(t, clients, 1) + clientID := clients[0].ClientID + + user, err := b.AdminGetUser(poolID, "alice") + require.NoError(t, err) + assert.NotEmpty(t, user.PasswordHash, "password hash must survive restore") + assert.Equal(t, "alice@example.com", user.Attributes["email"]) + + groups, err := b.AdminListGroupsForUser(poolID, "alice") + require.NoError(t, err) + require.Len(t, groups, 1) + assert.Equal(t, "admins", groups[0].GroupName) + + servers, err := b.ListResourceServers(poolID) + require.NoError(t, err) + require.Len(t, servers, 1) + assert.Equal(t, "https://api.example.com", servers[0].Identifier) + + idps, err := b.ListIdentityProviders(poolID) + require.NoError(t, err) + require.Len(t, idps, 1) + assert.Equal(t, "Google", idps[0].ProviderName) + + domain, err := b.DescribeUserPoolDomain("test-domain") + require.NoError(t, err) + assert.Equal(t, poolID, domain.UserPoolID) + + terms, err := b.DescribeTerms(poolID) + require.NoError(t, err) + assert.Equal(t, "accept these terms", terms.Text) + + jobs, err := b.ListUserImportJobs(poolID) + require.NoError(t, err) + require.Len(t, jobs, 1) + + mlb, err := b.DescribeManagedLoginBrandingByClient(poolID, clientID) + require.NoError(t, err) + assert.Equal(t, clientID, mlb.ClientID) + + ui, err := b.GetUICustomization(poolID, clientID) + require.NoError(t, err) + assert.Equal(t, "body{color:red}", ui.CSS) + + risk, err := b.GetTypedRiskConfiguration(poolID, clientID) + require.NoError(t, err) + assert.Equal(t, poolID, risk.UserPoolID) + + poolARN := "arn:aws:cognito-idp:us-east-1:000000000000:userpool/" + poolID + assert.Equal(t, "test", b.ListTagsForResource(poolARN)["env"]) + + assert.Positive(t, b.RefreshTokenCount(), "refresh token must survive restore") + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := cognitoidp.NewInMemoryBackend("000000000000", "us-east-1", "http://localhost:8000") + tc.setup(t, b) + + data := b.Snapshot(ctx) + require.NotEmpty(t, data) + + restored := cognitoidp.NewInMemoryBackend("000000000000", "us-east-1", "http://localhost:8000") + require.NoError(t, restored.Restore(ctx, data)) + + tc.verify(t, restored) + }) + } +} + +// TestInMemoryBackend_RestoreVersionGuard verifies the snapshot version guard: +// malformed JSON is a hard error, while a well-formed but version-mismatched +// (or pre-Phase-3.3, version-less) snapshot is discarded cleanly -- Restore +// resets the backend to empty and returns nil rather than partially decoding +// an incompatible shape. +func TestInMemoryBackend_RestoreVersionGuard(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + tests := []struct { + name string + data []byte + wantErr bool + }{ + { + name: "malformed_json_errors", + data: []byte(`{not valid json`), + wantErr: true, + }, + { + name: "missing_version_resets_cleanly", + data: []byte(`{"accountId":"000000000000"}`), + wantErr: false, + }, + { + name: "future_version_resets_cleanly", + data: []byte(`{"version":999999,"accountId":"000000000000"}`), + wantErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := cognitoidp.NewInMemoryBackend("000000000000", "us-east-1", "http://localhost:8000") + + _, err := b.CreateUserPool("pre-existing-pool") + require.NoError(t, err) + + err = b.Restore(ctx, tc.data) + if tc.wantErr { + require.Error(t, err) + + return + } + + require.NoError(t, err) + assert.Empty(t, b.ListUserPools(), "incompatible/malformed-version snapshot must reset to empty") + }) + } +} diff --git a/services/cognitoidp/prevent_user_existence_test.go b/services/cognitoidp/prevent_user_existence_test.go new file mode 100644 index 000000000..521db1343 --- /dev/null +++ b/services/cognitoidp/prevent_user_existence_test.go @@ -0,0 +1,134 @@ +package cognitoidp_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/cognitoidp" +) + +// Test_InitiateAuth_PreventUserExistenceErrors proves that a non-admin InitiateAuth call for +// an unknown username reveals UserNotFoundException only when the app client's +// PreventUserExistenceErrors is "LEGACY" (the default when unset); "ENABLED" must mask that +// distinction behind the exact same NotAuthorizedException a wrong password produces, so a +// caller cannot enumerate valid usernames by inspecting the error. +func Test_InitiateAuth_PreventUserExistenceErrors(t *testing.T) { + t.Parallel() + + cases := []struct { + wantErr error + name string + clientOpts cognitoidp.UserPoolClientOptions + wantSameMsg bool + }{ + { + name: "default (unset) is LEGACY: reveals UserNotFoundException", + clientOpts: cognitoidp.UserPoolClientOptions{}, + wantErr: cognitoidp.ErrUserNotFound, + }, + { + name: "explicit LEGACY reveals UserNotFoundException", + clientOpts: cognitoidp.UserPoolClientOptions{PreventUserExistenceErrors: "LEGACY"}, + wantErr: cognitoidp.ErrUserNotFound, + }, + { + name: "ENABLED masks as NotAuthorizedException matching a wrong password", + clientOpts: cognitoidp.UserPoolClientOptions{PreventUserExistenceErrors: "ENABLED"}, + wantErr: cognitoidp.ErrNotAuthorized, + wantSameMsg: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := newTestBackend() + pool, err := b.CreateUserPoolWithOpts("peu-pool-"+sanitizeTestName(tc.name), cognitoidp.UserPoolOptions{}) + require.NoError(t, err) + + client, err := b.CreateUserPoolClientWithOpts(pool.ID, "peu-client", tc.clientOpts) + require.NoError(t, err) + + existingUsername := "realuser-" + sanitizeTestName(tc.name) + _ = signUpConfirmAndLogin(t, b, client.ClientID, existingUsername) + + _, unknownErr := b.InitiateAuth(client.ClientID, "USER_PASSWORD_AUTH", "no-such-user", "Pass1234!") + require.ErrorIs(t, unknownErr, tc.wantErr) + + if tc.wantSameMsg { + _, wrongPasswordErr := b.InitiateAuth( + client.ClientID, "USER_PASSWORD_AUTH", existingUsername, "wrong-password-not-real", + ) + require.ErrorIs(t, wrongPasswordErr, cognitoidp.ErrNotAuthorized) + assert.Equal( + t, wrongPasswordErr.Error(), unknownErr.Error(), + "ENABLED must make the unknown-user and wrong-password errors indistinguishable", + ) + } + }) + } +} + +// Test_AdminInitiateAuth_AlwaysRevealsUserNotFound proves PreventUserExistenceErrors=ENABLED +// only affects the non-admin InitiateAuth path: AdminInitiateAuth callers already hold +// admin-level AWS credentials, so AWS does not mask UserNotFoundException there. +func Test_AdminInitiateAuth_AlwaysRevealsUserNotFound(t *testing.T) { + t.Parallel() + + b := newTestBackend() + pool, err := b.CreateUserPoolWithOpts("peu-admin-pool", cognitoidp.UserPoolOptions{}) + require.NoError(t, err) + + client, err := b.CreateUserPoolClientWithOpts(pool.ID, "peu-admin-client", cognitoidp.UserPoolClientOptions{ + PreventUserExistenceErrors: "ENABLED", + }) + require.NoError(t, err) + + _, err = b.AdminInitiateAuth(pool.ID, client.ClientID, "ADMIN_USER_PASSWORD_AUTH", "no-such-admin-user", "x") + require.Error(t, err) + assert.ErrorIs(t, err, cognitoidp.ErrUserNotFound) +} + +// Test_CreateUserPoolClient_PreventUserExistenceErrors_Validation locks in AWS's enum +// validation: only "ENABLED" and "LEGACY" (or omitted, which defaults to "LEGACY") are legal. +func Test_CreateUserPoolClient_PreventUserExistenceErrors_Validation(t *testing.T) { + t.Parallel() + + b := newTestBackend() + pool, err := b.CreateUserPoolWithOpts("peu-validate-pool", cognitoidp.UserPoolOptions{}) + require.NoError(t, err) + + _, err = b.CreateUserPoolClientWithOpts(pool.ID, "bad-client", cognitoidp.UserPoolClientOptions{ + PreventUserExistenceErrors: "NOT_A_REAL_VALUE", + }) + require.Error(t, err) + assert.ErrorIs(t, err, cognitoidp.ErrInvalidParameter) +} + +// Test_UpdateUserPoolClient_PreventUserExistenceErrors proves the setting can be flipped +// after creation and that InitiateAuth's masking behavior follows the update. +func Test_UpdateUserPoolClient_PreventUserExistenceErrors(t *testing.T) { + t.Parallel() + + b := newTestBackend() + pool, err := b.CreateUserPoolWithOpts("peu-update-pool", cognitoidp.UserPoolOptions{}) + require.NoError(t, err) + + client, err := b.CreateUserPoolClientWithOpts(pool.ID, "peu-update-client", cognitoidp.UserPoolClientOptions{}) + require.NoError(t, err) + + _, err = b.InitiateAuth(client.ClientID, "USER_PASSWORD_AUTH", "ghost", "Pass1234!") + require.ErrorIs(t, err, cognitoidp.ErrUserNotFound, "new client defaults to LEGACY") + + updated, err := b.UpdateUserPoolClientWithOpts(pool.ID, client.ClientID, "", cognitoidp.UserPoolClientOptions{ + PreventUserExistenceErrors: "ENABLED", + }) + require.NoError(t, err) + assert.Equal(t, "ENABLED", updated.PreventUserExistenceErrors) + + _, err = b.InitiateAuth(client.ClientID, "USER_PASSWORD_AUTH", "ghost", "Pass1234!") + require.ErrorIs(t, err, cognitoidp.ErrNotAuthorized, "after enabling, unknown users must be masked") +} diff --git a/services/cognitoidp/store_setup.go b/services/cognitoidp/store_setup.go new file mode 100644 index 000000000..14f52f0b4 --- /dev/null +++ b/services/cognitoidp/store_setup.go @@ -0,0 +1,202 @@ +package cognitoidp + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T (and composite-keyed map[string]map[string]*T) resource field +// on InMemoryBackend that has a pure key function is registered exactly once, +// here, as a *store.Table[T] on b.registry. See pkgs/store's package doc and +// the services/sqs pilot (commit 0f09d77c) / services/ec2 sweep (commit +// 12e611a4) for the pattern this follows. +// +// cognitoidp's resource maps nest two levels deep (poolID -> subKey -> *T) +// far more often than ec2's flat maps do, since almost every resource in this +// backend belongs to exactly one user pool. Each such map is flattened into a +// single store.Table keyed by a composite "poolID:subKey" string, with a +// secondary store.Index keyed by poolID alone standing in for the "give me +// every X in pool P" access pattern the nested map used to serve directly +// (e.g. ListUsers, ListGroups). +// +// A handful of fields are deliberately NOT registered here and remain plain +// maps -- see the comment block above registerAllTables for the list and why. +import "github.com/blackbirdworks/gopherstack/pkgs/store" + +// userKey builds the composite primary key for the users table: poolID+":"+username. +func userKey(poolID, username string) string { return poolID + ":" + username } + +// userSubKey builds the secondary-index key for looking up a user by pool+sub. +func userSubKey(poolID, sub string) string { return poolID + ":" + sub } + +// groupKey builds the composite primary key for the groups table. +func groupKey(poolID, groupName string) string { return poolID + ":" + groupName } + +// resourceServerKey builds the composite primary key for the resourceServers table. +func resourceServerKey(poolID, identifier string) string { return poolID + ":" + identifier } + +// identityProviderKey builds the composite primary key for the identityProviders table. +func identityProviderKey(poolID, providerName string) string { return poolID + ":" + providerName } + +// userImportJobKey builds the composite primary key for the userImportJobs table. +func userImportJobKey(poolID, jobID string) string { return poolID + ":" + jobID } + +// managedLoginBrandingKey builds the composite primary key for the managedLoginBrandings table. +func managedLoginBrandingKey(poolID, brandingID string) string { return poolID + ":" + brandingID } + +func poolsKeyFn(v *UserPool) string { return v.ID } +func poolsNameIndexFn(v *UserPool) string { return v.Name } + +func clientsKeyFn(v *UserPoolClient) string { return v.ClientID } +func clientsPoolIndexFn(v *UserPoolClient) string { return v.UserPoolID } + +func usersKeyFn(v *User) string { return userKey(v.UserPoolID, v.Username) } +func usersPoolIndexFn(v *User) string { return v.UserPoolID } +func usersSubIndexFn(v *User) string { return userSubKey(v.UserPoolID, v.Sub) } + +func groupsKeyFn(v *Group) string { return groupKey(v.UserPoolID, v.GroupName) } +func groupsPoolIndexFn(v *Group) string { return v.UserPoolID } + +func resourceServersKeyFn(v *ResourceServer) string { + return resourceServerKey(v.UserPoolID, v.Identifier) +} +func resourceServersPoolIndexFn(v *ResourceServer) string { return v.UserPoolID } + +func identityProvidersKeyFn(v *IdentityProvider) string { + return identityProviderKey(v.UserPoolID, v.ProviderName) +} +func identityProvidersPoolIndexFn(v *IdentityProvider) string { return v.UserPoolID } + +func domainsKeyFn(v *UserPoolDomain) string { return v.Domain } + +func termsKeyFn(v *Terms) string { return v.UserPoolID } + +func userImportJobsKeyFn(v *UserImportJob) string { return userImportJobKey(v.UserPoolID, v.JobID) } +func userImportJobsPoolIndexFn(v *UserImportJob) string { return v.UserPoolID } + +func managedLoginBrandingsKeyFn(v *ManagedLoginBranding) string { + return managedLoginBrandingKey(v.UserPoolID, v.ManagedLoginBrandingID) +} +func managedLoginBrandingsPoolIndexFn(v *ManagedLoginBranding) string { return v.UserPoolID } + +func uiCustomizationsKeyFn(v *UICustomization) string { return uiKey(v.UserPoolID, v.ClientID) } + +func typedRiskConfigurationsKeyFn(v *TypedRiskConfiguration) string { + return v.UserPoolID + ":" + v.ClientID +} + +// poolNameExists reports whether a pool with the given (globally unique) Name +// already exists, via the poolsByName secondary index. Caller must hold at +// least a read lock. +func (b *InMemoryBackend) poolNameExists(name string) bool { + return len(b.poolsByName.Get(name)) > 0 +} + +// userBySub looks up a user by poolID+sub via the usersBySub secondary index. +// Caller must hold at least a read lock. +func (b *InMemoryBackend) userBySub(poolID, sub string) (*User, bool) { + matches := b.usersBySub.Get(userSubKey(poolID, sub)) + if len(matches) == 0 { + return nil, false + } + + return matches[0], true +} + +// registerAllTables registers every converted resource map on b.registry +// exactly once. It must be called during construction only (immediately after +// b.registry is created), never on every Reset() -- store.Register panics on a +// duplicate name, so runtime resets go through registry.ResetAll() instead +// (see InMemoryBackend.Reset in backend.go). +// +// The following resource fields are deliberately left as plain maps (not +// registered here) because their value carries no pure identity for the key +// store.Table requires, or (groupMembers/resourceTags) because the value +// itself is a bare set/string map rather than a *T resource struct: +// - refreshTokens / refreshTokensByClient / refreshTokensByUser: +// refreshTokenEntry stores the pool/client/username a token belongs to +// but not the token string itself (the map key) -- not a pure function of +// the value. The By* maps are index-of-index views derived from the same +// raw map and share this constraint. +// - mfaSessions: mfaSessionEntry stores PoolID/ClientID/Username but not the +// session token itself (the map key) -- same constraint as refreshTokens. +// - attrVerificationCodes: attrVerificationEntry (ExpiresAt, Code) carries +// no identity fields at all; the poolID:username:attrName key is entirely +// external to the value. +// - tokenRevokedBefore: value type is time.Time, not a resource struct with +// an identity field -- the poolID:username key is external to the value. +// - resourceTags: value type is map[string]string (tag key -> tag value), +// not a *T resource struct. +// - riskConfigurations / logDeliveryConfigs / poolMfaConfigs: RiskConfiguration +// and LogDeliveryConfig hold only an opaque Raw map[string]any blob, and +// UserPoolMfaFullConfig holds only sub-config pointers -- none carry the +// poolID (+clientID) identity that is their map key. +// - devices / webauthnCredentials / authEvents: Device, WebAuthnCredential, +// and AuthEvent carry their own leaf identity (DeviceKey/CredentialID/ +// EventID) but not the poolID+":"+username of the user they belong to, so +// the outer nesting key is not reconstructable from the value alone. +// - groupMembers: map[string]map[string]map[string]struct{} is a pure +// membership set (poolID -> groupName -> set of usernames), not a +// collection of *T resource structs. +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call (and any secondary store.AddIndex calls) to the concrete field. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.pools = store.Register(b.registry, "pools", store.New(poolsKeyFn)) + b.poolsByName = b.pools.AddIndex("byName", poolsNameIndexFn) + }, + func(b *InMemoryBackend) { + b.clients = store.Register(b.registry, "clients", store.New(clientsKeyFn)) + b.clientsByPool = b.clients.AddIndex("byPool", clientsPoolIndexFn) + }, + func(b *InMemoryBackend) { + b.users = store.Register(b.registry, "users", store.New(usersKeyFn)) + b.usersByPool = b.users.AddIndex("byPool", usersPoolIndexFn) + b.usersBySub = b.users.AddIndex("bySub", usersSubIndexFn) + }, + func(b *InMemoryBackend) { + b.groups = store.Register(b.registry, "groups", store.New(groupsKeyFn)) + b.groupsByPool = b.groups.AddIndex("byPool", groupsPoolIndexFn) + }, + func(b *InMemoryBackend) { + b.resourceServers = store.Register(b.registry, "resourceServers", store.New(resourceServersKeyFn)) + b.resourceServersByPool = b.resourceServers.AddIndex("byPool", resourceServersPoolIndexFn) + }, + func(b *InMemoryBackend) { + b.identityProviders = store.Register(b.registry, "identityProviders", store.New(identityProvidersKeyFn)) + b.identityProvidersByPool = b.identityProviders.AddIndex("byPool", identityProvidersPoolIndexFn) + }, + func(b *InMemoryBackend) { + b.domains = store.Register(b.registry, "domains", store.New(domainsKeyFn)) + }, + func(b *InMemoryBackend) { + b.terms = store.Register(b.registry, "terms", store.New(termsKeyFn)) + }, + func(b *InMemoryBackend) { + b.userImportJobs = store.Register(b.registry, "userImportJobs", store.New(userImportJobsKeyFn)) + b.userImportJobsByPool = b.userImportJobs.AddIndex("byPool", userImportJobsPoolIndexFn) + }, + func(b *InMemoryBackend) { + b.managedLoginBrandings = store.Register( + b.registry, + "managedLoginBrandings", + store.New(managedLoginBrandingsKeyFn), + ) + b.managedLoginBrandingsByPool = b.managedLoginBrandings.AddIndex("byPool", managedLoginBrandingsPoolIndexFn) + }, + func(b *InMemoryBackend) { + b.uiCustomizations = store.Register(b.registry, "uiCustomizations", store.New(uiCustomizationsKeyFn)) + }, + func(b *InMemoryBackend) { + b.typedRiskConfigurations = store.Register( + b.registry, + "typedRiskConfigurations", + store.New(typedRiskConfigurationsKeyFn), + ) + }, +} diff --git a/services/cognitoidp/totp.go b/services/cognitoidp/totp.go new file mode 100644 index 000000000..bb69a1699 --- /dev/null +++ b/services/cognitoidp/totp.go @@ -0,0 +1,125 @@ +package cognitoidp + +import ( + "crypto/hmac" + "crypto/sha1" //nolint:gosec // RFC 6238 TOTP mandates HMAC-SHA1; this is not used for anything else. + "encoding/base32" + "encoding/binary" + "fmt" + "strings" + "time" +) + +// TOTP (RFC 6238) parameters matching AWS Cognito's software-token MFA implementation: +// 30-second time step, 6-digit codes, HMAC-SHA1 (the algorithm authenticator apps such as +// Google Authenticator and Authy assume when no otpauth "algorithm" parameter is given). +const ( + totpStepSeconds = 30 + totpDigits = 6 + // totpSkewSteps allows the previous/next time step to account for clock drift between + // the emulator and whatever generated the code, matching real Cognito's leeway. + totpSkewSteps = 1 + // totpDigitsPow is 10^totpDigits, used to truncate the HOTP value to totpDigits. + totpDigitsPow = 1000000 + + // RFC 4226 §5.3 dynamic-truncation constants: hotpOffsetMask picks the low nibble of + // the last HMAC byte as the truncation offset; hotpByteMask masks a single byte; + // hotpTopByteMask additionally clears the HOTP result's sign bit (byte 0 of the 31-bit + // truncated value must be non-negative when interpreted as a big-endian int32). + hotpOffsetMask = 0x0f + hotpByteMask = 0xff + hotpTopByteMask = 0x7f +) + +// GenerateTOTPCode computes the RFC 6238 TOTP code for the given base32-encoded secret at +// time t. It is exported so integration tests (and any AWS-SDK-driven caller that needs to +// simulate a real authenticator app) can compute the expected code for a secret returned by +// AssociateSoftwareToken, the same way a real TOTP app would. +func GenerateTOTPCode(secretBase32 string, t time.Time) (string, error) { + key, err := decodeTOTPSecret(secretBase32) + if err != nil { + return "", err + } + + return hotpCode(key, totpCounter(t)), nil +} + +// totpCounter converts a wall-clock time to the RFC 6238 time-step counter. +func totpCounter(t time.Time) uint64 { + unix := max(t.Unix(), 0) + + return uint64(unix) / totpStepSeconds +} + +// decodeTOTPSecret decodes a base32 TOTP secret, tolerating both padded and unpadded +// encodings and the presence/absence of surrounding whitespace, since real authenticator +// apps and SDKs vary in how they store/transcribe otpauth secrets. +func decodeTOTPSecret(secretBase32 string) ([]byte, error) { + clean := strings.ToUpper(strings.TrimSpace(secretBase32)) + clean = strings.ReplaceAll(clean, " ", "") + + if key, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(clean); err == nil { + return key, nil + } + + key, err := base32.StdEncoding.DecodeString(clean) + if err != nil { + return nil, fmt.Errorf("decoding TOTP secret: %w", err) + } + + return key, nil +} + +// hotpCode computes the RFC 4226 HOTP value for the given key and counter, truncated to +// totpDigits decimal digits (zero-padded). +func hotpCode(key []byte, counter uint64) string { + var counterBytes [8]byte + binary.BigEndian.PutUint64(counterBytes[:], counter) + + mac := hmac.New(sha1.New, key) + mac.Write(counterBytes[:]) + sum := mac.Sum(nil) + + // Dynamic truncation per RFC 4226 §5.3. + offset := sum[len(sum)-1] & hotpOffsetMask + binCode := (uint32(sum[offset])&hotpTopByteMask)<<24 | + (uint32(sum[offset+1])&hotpByteMask)<<16 | + (uint32(sum[offset+2])&hotpByteMask)<<8 | + (uint32(sum[offset+3]) & hotpByteMask) + + code := binCode % totpDigitsPow + + return fmt.Sprintf("%0*d", totpDigits, code) +} + +// verifyTOTPCode reports whether code is a valid TOTP for secretBase32 at time t, allowing +// +/- totpSkewSteps time steps of clock drift (matching real Cognito's tolerance for +// software-token MFA). An empty or undecodable secret never validates. +func verifyTOTPCode(secretBase32, code string, t time.Time) bool { + key, err := decodeTOTPSecret(secretBase32) + if err != nil || len(key) == 0 { + return false + } + + counter := totpCounter(t) + + for skew := -totpSkewSteps; skew <= totpSkewSteps; skew++ { + c := counter + if skew < 0 { + step := uint64(-skew) + if c < step { + continue + } + + c -= step + } else { + c += uint64(skew) + } + + if hmac.Equal([]byte(hotpCode(key, c)), []byte(code)) { + return true + } + } + + return false +} diff --git a/services/cognitoidp/totp_test.go b/services/cognitoidp/totp_test.go new file mode 100644 index 000000000..be9bbd29d --- /dev/null +++ b/services/cognitoidp/totp_test.go @@ -0,0 +1,242 @@ +package cognitoidp_test + +import ( + "encoding/base32" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/cognitoidp" +) + +// rfc4226Secret is the 20-byte ASCII secret used by the official RFC 4226 (HOTP) Appendix D +// and RFC 6238 (TOTP) Appendix B test vectors. +const rfc4226Secret = "12345678901234567890" + +func rfc4226SecretBase32(t *testing.T) string { + t.Helper() + + return base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString([]byte(rfc4226Secret)) +} + +// Test_GenerateTOTPCode_RFC4226Vectors pins GenerateTOTPCode against the official RFC 4226 +// Appendix D HOTP test vectors (truncated to 6 digits), by picking a timestamp that falls in +// the 30-second step whose counter equals the RFC's HOTP counter. This proves the dynamic +// truncation + HMAC-SHA1 math is bit-for-bit correct, independent of any TOTP-specific +// timestamp handling. +func Test_GenerateTOTPCode_RFC4226Vectors(t *testing.T) { + t.Parallel() + + secret := rfc4226SecretBase32(t) + + // RFC 4226 Appendix D, "Truncated Value" column reduced to its low 6 digits (6-digit + // HOTP truncation is defined as the same dynamic-truncation binary value mod 10^6, i.e. + // the low 6 decimal digits of the published 6-digit-truncated values already listed + // there). + cases := []struct { + name string + want string + counter int64 + }{ + {"counter_0", "755224", 0}, + {"counter_1", "287082", 1}, + {"counter_2", "359152", 2}, + {"counter_3", "969429", 3}, + {"counter_4", "338314", 4}, + {"counter_5", "254676", 5}, + {"counter_6", "287922", 6}, + {"counter_7", "162583", 7}, + {"counter_8", "399871", 8}, + {"counter_9", "520489", 9}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + // Any timestamp within [counter*30, counter*30+30) maps to this HOTP counter + // under the 30-second TOTP step. + at := time.Unix(tc.counter*30, 0).UTC() + + got, err := cognitoidp.GenerateTOTPCode(secret, at) + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +// Test_GenerateTOTPCode_RFC6238Vectors pins GenerateTOTPCode against the official RFC 6238 +// Appendix B test vectors (published as 8-digit codes for SHA1; truncated here to the 6 +// digits AWS Cognito actually uses, which are the same dynamic-truncation value mod 10^6). +func Test_GenerateTOTPCode_RFC6238Vectors(t *testing.T) { + t.Parallel() + + secret := rfc4226SecretBase32(t) + + cases := []struct { + name string + want8Digit string + unixTime int64 + }{ + {"T=59", "94287082", 59}, + {"T=1111111109", "07081804", 1111111109}, + {"T=1111111111", "14050471", 1111111111}, + {"T=1234567890", "89005924", 1234567890}, + {"T=2000000000", "69279037", 2000000000}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + want6Digit := tc.want8Digit[len(tc.want8Digit)-6:] + + got, err := cognitoidp.GenerateTOTPCode(secret, time.Unix(tc.unixTime, 0).UTC()) + require.NoError(t, err) + assert.Equal(t, want6Digit, got) + }) + } +} + +// Test_VerifySoftwareToken_RealTOTP proves the fix end to end: the exact code an +// authenticator app would compute from the AssociateSoftwareToken secret is accepted, a +// fixed/guessed code is rejected with CodeMismatchException, and the challenge cannot be +// bypassed with an arbitrary 6-digit string. +func Test_VerifySoftwareToken_RealTOTP(t *testing.T) { + t.Parallel() + + cases := []struct { + wantErr error + buildCode func(t *testing.T, secret string) string + name string + }{ + { + name: "real code from secret is accepted", + buildCode: func(t *testing.T, secret string) string { + t.Helper() + + code, err := cognitoidp.GenerateTOTPCode(secret, time.Now()) + require.NoError(t, err) + + return code + }, + wantErr: nil, + }, + { + name: "wrong code is rejected", + buildCode: func(t *testing.T, secret string) string { + t.Helper() + + realCode, err := cognitoidp.GenerateTOTPCode(secret, time.Now()) + require.NoError(t, err) + + return wrongSixDigitCode(realCode) + }, + wantErr: cognitoidp.ErrCodeMismatch, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b, _, client := setupTestPoolAndClient(t) + tokens := signUpConfirmAndLogin(t, b, client.ClientID, "totp-"+sanitizeTestName(tc.name)) + + secret, err := b.AssociateSoftwareToken(tokens.AccessToken) + require.NoError(t, err) + + code := tc.buildCode(t, secret) + + err = b.VerifySoftwareToken(tokens.AccessToken, code) + if tc.wantErr == nil { + require.NoError(t, err) + } else { + require.Error(t, err) + assert.ErrorIs(t, err, tc.wantErr) + } + }) + } +} + +// Test_RespondToMFAChallenge_SMSCode proves SMS_MFA/EMAIL_OTP challenges now require the +// exact one-time code generated by the backend (not "any 6 digits"), while retaining the +// session across a wrong attempt so the caller can retry until it expires. +func Test_RespondToMFAChallenge_SMSCode(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + useReal bool + }{ + {"correct code issues tokens", true}, + {"wrong code is rejected", false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b, pool, client := setupTestPoolAndClient(t) + username := "sms-" + sanitizeTestName(tc.name) + tokens := signUpConfirmAndLogin(t, b, client.ClientID, username) + + require.NoError(t, b.SetUserMFAPreference(tokens.AccessToken, true, false, "SMS_MFA")) + require.NoError(t, b.UpdateUserPoolWithOpts(pool.ID, "ON", cognitoidp.UserPoolOptions{})) + + result, err := b.InitiateAuth(client.ClientID, "USER_PASSWORD_AUTH", username, "Pass1234!") + require.NoError(t, err) + require.Equal(t, "SMS_MFA", result.ChallengeName) + require.NotEmpty(t, result.MFASession) + + realCode := b.GetMFASessionCodeForTest(result.MFASession) + require.Len(t, realCode, 6) + + if tc.useReal { + gotTokens, respondErr := b.RespondToMFAChallenge(client.ClientID, result.MFASession, realCode) + require.NoError(t, respondErr) + assert.NotEmpty(t, gotTokens.AccessToken) + + return + } + + _, mismatchErr := b.RespondToMFAChallenge(client.ClientID, result.MFASession, wrongSixDigitCode(realCode)) + require.ErrorIs(t, mismatchErr, cognitoidp.ErrCodeMismatch) + + // The session must survive a wrong attempt so the real code can still be used. + gotTokens, retryErr := b.RespondToMFAChallenge(client.ClientID, result.MFASession, realCode) + require.NoError(t, retryErr) + assert.NotEmpty(t, gotTokens.AccessToken) + }) + } +} + +// wrongSixDigitCode returns a 6-digit numeric string guaranteed to differ from realCode. +func wrongSixDigitCode(realCode string) string { + if realCode != "000000" { + return "000000" + } + + return "111111" +} + +// sanitizeTestName lowercases and strips spaces so a subtest name can double as a unique +// Cognito username. +func sanitizeTestName(name string) string { + out := make([]rune, 0, len(name)) + + for _, r := range name { + switch { + case r >= 'a' && r <= 'z': + out = append(out, r) + case r >= 'A' && r <= 'Z': + out = append(out, r+('a'-'A')) + case r >= '0' && r <= '9': + out = append(out, r) + } + } + + return string(out) +} diff --git a/services/dynamodb/PARITY.md b/services/dynamodb/PARITY.md new file mode 100644 index 000000000..3ac40a86f --- /dev/null +++ b/services/dynamodb/PARITY.md @@ -0,0 +1,26 @@ +--- +service: dynamodb +sdk_module: aws-sdk-go-v2/service/dynamodb # version: see go.mod (backfilled) +last_audit_commit: f459c9fa +last_audit_date: 2026-07-04 +overall: A # modest 419 LOC — heavily hardened by prior sweeps; 3 real fixes incl. a state-corruption bug +protocol: json-1.0 (DynamoDB_20120810 targets) +families: + item_crud: {status: ok, note: PROVEN — condition eval, all ReturnValues, ItemCollectionMetrics/LSI 10GB, WCU/RCU formulas} + query_scan: {status: ok, note: PROVEN pagination (LastEvaluatedKey w/ base-PK fusion for GSI/LSI, 1MB/Limit); FIXED Select/COUNT omits Items + Select constraint validation} + batch: {status: ok, note: FIXED BatchWriteItem duplicate-key validation (was missing; BatchGetItem had it)} + transactions: {status: ok, note: FIXED TransactWriteItems Update key-mutation — was NOT validated, silently corrupted pkIndex/pkskIndex (state corruption bug)} + streams: {status: ok, note: PROVEN shard-iterator sequence clamping, trim-horizon} + janitor_ttl: {status: ok, note: PROVEN batched-lock, ctx-cancel, quickselect eviction, ring-buffer compaction} +gaps: [] +deferred: + - expr/ lexer/parser/evaluator subpackage (has own aws_spec_test.go/evaluator_test.go) — not line-by-line re-audited + - PartiQL execution + - TransactWriteItems Put/Update/Delete/ConditionCheck unused-EAN/EAV validation (bd: gopherstack-daa) +leaks: {status: clean, note: TTL sweeper + stream trimming verified, ctx-cancel present} +--- + +## Notes +- BatchWriteItem rejects same-key Put+Delete / Put+Put / Delete+Delete in one call: "Provided list of item keys contains duplicates" (verified docs + boto3 history). A prior test asserted the opposite — corrected. +- Select=COUNT returns Count/ScannedCount only, Items omitted. +- Select=SPECIFIC_ATTRIBUTES requires a projection; ALL_PROJECTED_ATTRIBUTES invalid on bare table. diff --git a/services/dynamodb/accuracy_audit.go b/services/dynamodb/accuracy_audit.go index 9aead611e..c543dbf79 100644 --- a/services/dynamodb/accuracy_audit.go +++ b/services/dynamodb/accuracy_audit.go @@ -316,6 +316,10 @@ func validateTransactWriteItems( totalBytes := 0 for i, ti := range items { + if err := validateTransactUpdateKeys(ti, tables); err != nil { + return err + } + tableName, keyItem, itemForSize := extractTransactWriteKeyAndItem(ti) if tableName == "" { continue @@ -369,6 +373,30 @@ func validateTransactWriteItems( return nil } +// validateTransactUpdateKeys rejects a TransactWriteItem Update action whose +// UpdateExpression touches a key attribute — the same restriction plain +// UpdateItem enforces. Without this check a transactional update can rewrite +// an item's key in place while leaving the OLD key's index entry dangling +// (updateIndexes only ever adds/overwrites the new key's index slot, it never +// removes a stale one), corrupting pkIndex/pkskIndex lookups. Non-Update +// actions are always allowed through (nil). +func validateTransactUpdateKeys(ti types.TransactWriteItem, tables map[string]*Table) error { + if ti.Update == nil { + return nil + } + + table, ok := tables[aws.ToString(ti.Update.TableName)] + if !ok { + return nil + } + + return validateUpdateDoesNotModifyKeys( + aws.ToString(ti.Update.UpdateExpression), + ti.Update.ExpressionAttributeNames, + table.KeySchema, + ) +} + // extractTransactWriteKeyAndItem returns the table name, key map, and item map // from a single TransactWriteItem (for duplicate detection and size accounting). // ConditionCheck is excluded from duplicate key detection because AWS DynamoDB diff --git a/services/dynamodb/backup_interface.go b/services/dynamodb/backup_interface.go index 08e3f150d..25aff33be 100644 --- a/services/dynamodb/backup_interface.go +++ b/services/dynamodb/backup_interface.go @@ -111,7 +111,7 @@ func (db *InMemoryDB) CreateBackup( // Check for duplicate backup name scoped to this table; AWS returns BackupInUseException. db.mu.RLock("CreateBackup.checkDuplicate") - for _, existing := range db.Backups { + for _, existing := range db.backups.All() { if existing.TableName == tableName && existing.BackupName == backupName && existing.BackupStatus != models.BackupStatusDeleted { db.mu.RUnlock() @@ -147,10 +147,11 @@ func (db *InMemoryDB) CreateBackup( } db.mu.Lock("CreateBackup") - db.Backups[bkpARN] = backup - evictOldest( - db.Backups, + db.backups.Put(backup) + evictOldestFromTable( + db.backups, maxBackupsRetained, + backupKeyFn, func(b *Backup) time.Time { return b.CreationDateTime }, ) db.mu.Unlock() @@ -185,7 +186,7 @@ func (db *InMemoryDB) DescribeBackup( } db.mu.RLock("DescribeBackup") - backup, exists := db.Backups[backupArn] + backup, exists := db.backups.Get(backupArn) var backupCopy Backup if exists { backupCopy = *backup @@ -224,14 +225,14 @@ func (db *InMemoryDB) DeleteBackup( db.mu.Lock("DeleteBackup") defer db.mu.Unlock() - backup, exists := db.Backups[backupArn] + backup, exists := db.backups.Get(backupArn) if !exists { return nil, NewResourceNotFoundException("backup not found: " + backupArn) } backupCopy := *backup backupCopy.BackupStatus = models.BackupStatusDeleted - delete(db.Backups, backupArn) + db.backups.Delete(backupArn) return &sdkdynamodb.DeleteBackupOutput{ BackupDescription: buildSDKBackupDescription(&backupCopy), diff --git a/services/dynamodb/backup_ops.go b/services/dynamodb/backup_ops.go index 142c92bab..bbb2d2476 100644 --- a/services/dynamodb/backup_ops.go +++ b/services/dynamodb/backup_ops.go @@ -179,10 +179,11 @@ func collectBackupSummaries( tableName, backupType string, timeRangeLower, timeRangeUpper *float64, ) []models.BackupSummary { - summaries := make([]models.BackupSummary, 0, len(db.Backups)) + all := db.backups.All() + summaries := make([]models.BackupSummary, 0, len(all)) - for bkpARN, b := range db.Backups { - if db.regionFromARN(bkpARN) != requestRegion { + for _, b := range all { + if db.regionFromARN(b.BackupArn) != requestRegion { continue } @@ -278,11 +279,7 @@ func (db *InMemoryDB) installRestoredTable( ) (*Table, string, error) { db.mu.Lock("RestoreTable") - if _, rExists := db.Tables[region]; !rExists { - db.Tables[region] = make(map[string]*Table) - } - - if _, tExists := db.Tables[region][tableName]; tExists { + if _, tExists := db.tables.Get(tableKey(region, tableName)); tExists { db.mu.Unlock() return nil, "", NewResourceInUseException("table already exists: " + tableName) @@ -312,7 +309,7 @@ func (db *InMemoryDB) installRestoredTable( newTable.initializeIndexes() newTable.rebuildIndexes() - db.Tables[region][tableName] = newTable + db.tables.Put(newTable) db.mu.Unlock() return newTable, newTableID, nil @@ -338,7 +335,7 @@ func (h *DynamoDBHandler) restoreTableFromBackup(ctx context.Context, body []byt } db.mu.RLock("RestoreTableFromBackup.lookup") - backup, exists := db.Backups[req.BackupArn] + backup, exists := db.backups.Get(req.BackupArn) db.mu.RUnlock() if !exists { diff --git a/services/dynamodb/batch_accuracy_b2_test.go b/services/dynamodb/batch_accuracy_b2_test.go index 8a51c863e..2a316b5b5 100644 --- a/services/dynamodb/batch_accuracy_b2_test.go +++ b/services/dynamodb/batch_accuracy_b2_test.go @@ -415,6 +415,13 @@ func TestBatchWriteItem_DeleteRequest_MissingPK_Rejected(t *testing.T) { } // Regression: valid batch write still succeeds after adding validation. +// +// Note: the Delete key is deliberately distinct from both Put keys. AWS +// DynamoDB rejects a BatchWriteItem whose per-table request list targets the +// same primary key more than once (e.g. Put(k1) + Delete(k1) together) with +// "ValidationException: Provided list of item keys contains duplicates" — this +// test previously (incorrectly) exercised exactly that duplicate-key shape and +// asserted it should succeed, which does not match real AWS behaviour. func TestBatchWriteItem_ValidRequests_NotAffectedByValidation(t *testing.T) { t.Parallel() d := b2NewDB(t) @@ -441,7 +448,7 @@ func TestBatchWriteItem_ValidRequests_NotAffectedByValidation(t *testing.T) { { DeleteRequest: &types.DeleteRequest{ Key: map[string]types.AttributeValue{ - "pk": &types.AttributeValueMemberS{Value: "k1"}, + "pk": &types.AttributeValueMemberS{Value: "k3"}, }, }, }, @@ -453,6 +460,38 @@ func TestBatchWriteItem_ValidRequests_NotAffectedByValidation(t *testing.T) { } } +// TestBatchWriteItem_DuplicateKey_PutAndDelete_Rejected verifies that AWS's +// "one action per item per BatchWriteItem" rule is enforced across request +// kinds, not just within a single Put or Delete list: targeting the same +// primary key with both a Put and a Delete in one call is rejected. +func TestBatchWriteItem_DuplicateKey_PutAndDelete_Rejected(t *testing.T) { + t.Parallel() + d := b2NewDB(t) + b2CreateTable(t, d) + + _, err := d.BatchWriteItem(context.Background(), &dynamodb.BatchWriteItemInput{ + RequestItems: map[string][]types.WriteRequest{ + "tbl": { + { + PutRequest: &types.PutRequest{ + Item: map[string]types.AttributeValue{ + "pk": &types.AttributeValueMemberS{Value: "k1"}, + }, + }, + }, + { + DeleteRequest: &types.DeleteRequest{ + Key: map[string]types.AttributeValue{ + "pk": &types.AttributeValueMemberS{Value: "k1"}, + }, + }, + }, + }, + }, + }) + b2AssertValidationErr(t, err) +} + // Regression: valid batch get with no projection still returns all attributes. func TestBatchGetItem_NoProjection_ReturnsAllAttributes(t *testing.T) { t.Parallel() diff --git a/services/dynamodb/export_test.go b/services/dynamodb/export_test.go index e86aa5786..f9d8f3f4e 100644 --- a/services/dynamodb/export_test.go +++ b/services/dynamodb/export_test.go @@ -150,7 +150,7 @@ func (db *InMemoryDB) StreamARNIndexSize() int { db.mu.RLock("StreamARNIndexSize") defer db.mu.RUnlock() - return len(db.streamARNIndex) + return db.streamARNIndex.Len() } // LookupStreamARNIndex looks up a table by stream ARN in the reverse index (for tests). @@ -158,9 +158,7 @@ func (db *InMemoryDB) LookupStreamARNIndex(streamARN string) (*Table, bool) { db.mu.RLock("LookupStreamARNIndex") defer db.mu.RUnlock() - t, ok := db.streamARNIndex[streamARN] - - return t, ok + return db.streamARNIndex.Get(streamARN) } // SweepTxnTokens exposes sweepTxnTokens for tests. @@ -253,15 +251,7 @@ func (h *DynamoDBHandler) GetJanitorTaskTimeout() time.Duration { // Used in tests to pre-populate Kinesis destinations without going through the HTTP API. func (db *InMemoryDB) AddKinesisDestination(tableName, streamARN string) { db.mu.RLock("AddKinesisDestination") - regionTables, regionExists := db.Tables[db.defaultRegion] - - var table *Table - var tableExists bool - - if regionExists { - table, tableExists = regionTables[tableName] - } - + table, tableExists := db.tables.Get(tableKey(db.defaultRegion, tableName)) db.mu.RUnlock() if !tableExists { @@ -282,12 +272,7 @@ func (db *InMemoryDB) TableExistsInRegion(region, tableName string) bool { db.mu.RLock("TableExistsInRegion") defer db.mu.RUnlock() - regionTables, ok := db.Tables[region] - if !ok { - return false - } - - _, exists := regionTables[tableName] + _, exists := db.tables.Get(tableKey(region, tableName)) return exists } @@ -296,13 +281,7 @@ func (db *InMemoryDB) TableExistsInRegion(region, tableName string) bool { // in the backend's default region. Returns "" if the table is not part of a global table. func (db *InMemoryDB) GetTableGlobalTableName(tableName string) string { db.mu.RLock("GetTableGlobalTableName") - regionTables, ok := db.Tables[db.defaultRegion] - - var table *Table - if ok { - table = regionTables[tableName] - } - + table, _ := db.tables.Get(tableKey(db.defaultRegion, tableName)) db.mu.RUnlock() if table == nil { @@ -356,8 +335,7 @@ func (db *InMemoryDB) AddTxnToken(token string, expiry time.Time) { // for the named table (including zero-value / compacted entries). func (db *InMemoryDB) StreamRecordCount(tableName string) int { db.mu.RLock("StreamRecordCount") - regionTables := db.Tables[db.defaultRegion] - tbl := regionTables[tableName] + tbl, _ := db.tables.Get(tableKey(db.defaultRegion, tableName)) db.mu.RUnlock() if tbl == nil { @@ -373,8 +351,7 @@ func (db *InMemoryDB) StreamRecordCount(tableName string) int { // StreamShards returns a copy of the shard genealogy for the named table's stream. func (db *InMemoryDB) StreamShards(tableName string) []StreamShard { db.mu.RLock("StreamShards") - regionTables := db.Tables[db.defaultRegion] - tbl := regionTables[tableName] + tbl, _ := db.tables.Get(tableKey(db.defaultRegion, tableName)) db.mu.RUnlock() if tbl == nil { @@ -393,8 +370,7 @@ func (db *InMemoryDB) StreamShards(tableName string) []StreamShard { // StreamTrimSeq returns the current trim horizon (oldest seq still in buffer) for the named table. func (db *InMemoryDB) StreamTrimSeq(tableName string) int64 { db.mu.RLock("StreamTrimSeq") - regionTables := db.Tables[db.defaultRegion] - tbl := regionTables[tableName] + tbl, _ := db.tables.Get(tableKey(db.defaultRegion, tableName)) db.mu.RUnlock() if tbl == nil { @@ -454,7 +430,7 @@ func (db *InMemoryDB) ExportCount() int { db.mu.RLock("ExportCount") defer db.mu.RUnlock() - return len(db.exports) + return db.exports.Len() } // ExportDescFields is the exported form of exportDescriptionFields for use in tests. diff --git a/services/dynamodb/extra_ops.go b/services/dynamodb/extra_ops.go index a4d509da2..8c948b733 100644 --- a/services/dynamodb/extra_ops.go +++ b/services/dynamodb/extra_ops.go @@ -15,9 +15,9 @@ import ( "github.com/google/uuid" "github.com/blackbirdworks/gopherstack/pkgs/arn" - "github.com/blackbirdworks/gopherstack/pkgs/collections" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" "github.com/blackbirdworks/gopherstack/pkgs/ptrconv" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/services/dynamodb/models" ) @@ -72,7 +72,7 @@ func (db *InMemoryDB) CreateGlobalTable( db.mu.Lock("CreateGlobalTable") defer db.mu.Unlock() - if _, exists := db.GlobalTables[name]; exists { + if _, exists := db.globalTables.Get(name); exists { return nil, &Error{ Type: "com.amazonaws.dynamodb.v20120810#GlobalTableAlreadyExistsException", Message: fmt.Sprintf("Global table with name %s already exists", name), @@ -88,12 +88,12 @@ func (db *InMemoryDB) CreateGlobalTable( db.ensureReplicaTablesLocked(name, regions, source, allReplicas, now) - db.GlobalTables[name] = &StoredGlobalTable{ + db.globalTables.Put(&StoredGlobalTable{ GlobalTableName: name, GlobalTableArn: globalTableARN, CreationDateTime: now, ReplicationGroup: regions, - } + }) sdkReplicas := buildSDKReplicaDescriptions(regions) @@ -125,12 +125,7 @@ func collectValidRegions(group []types.Replica) []string { // Must be called with db.mu held (at least read). func (db *InMemoryDB) findSourceTable(name string, regions []string) *Table { for _, region := range regions { - regionTables, regionExists := db.Tables[region] - if !regionExists { - continue - } - - if tbl, tableExists := regionTables[name]; tableExists { + if tbl, tableExists := db.tables.Get(tableKey(region, name)); tableExists { return tbl } } @@ -162,21 +157,19 @@ func (db *InMemoryDB) ensureReplicaTablesLocked( now time.Time, ) { for _, region := range regions { - if _, ok := db.Tables[region]; !ok { - db.Tables[region] = make(map[string]*Table) - } - - if _, exists := db.Tables[region][name]; !exists { + if existing, exists := db.tables.Get(tableKey(region, name)); !exists { replica := db.buildReplicaTable(name, region, source, now) replica.GlobalTableName = name - db.Tables[region][name] = replica + db.tables.Put(replica) } else { - db.Tables[region][name].GlobalTableName = name + existing.GlobalTableName = name } } for _, region := range regions { - db.Tables[region][name].Replicas = buildReplicasExcluding(allReplicas, region) + if t, ok := db.tables.Get(tableKey(region, name)); ok { + t.Replicas = buildReplicasExcluding(allReplicas, region) + } } } @@ -229,7 +222,7 @@ func (db *InMemoryDB) DescribeGlobalTable( name := *input.GlobalTableName db.mu.RLock("DescribeGlobalTable") - gt, exists := db.GlobalTables[name] + gt, exists := db.globalTables.Get(name) db.mu.RUnlock() if !exists { @@ -272,7 +265,7 @@ func (db *InMemoryDB) DescribeGlobalTableSettings( name := *input.GlobalTableName db.mu.RLock("DescribeGlobalTableSettings") - gt, exists := db.GlobalTables[name] + gt, exists := db.globalTables.Get(name) db.mu.RUnlock() if !exists { @@ -289,11 +282,7 @@ func (db *InMemoryDB) DescribeGlobalTableSettings( // Look up the actual table in that region to get real provisioned throughput. db.mu.RLock("DescribeGlobalTableSettings.table") - regionTables := db.Tables[region] - var tbl *Table - if regionTables != nil { - tbl = regionTables[name] - } + tbl, _ := db.tables.Get(tableKey(region, name)) db.mu.RUnlock() if tbl != nil { tbl.mu.RLock("DescribeGlobalTableSettings.throughput") @@ -569,8 +558,8 @@ func (db *InMemoryDB) ListGlobalTables( regionFilter := ptrconv.String(input.RegionName) startName := ptrconv.String(input.ExclusiveStartGlobalTableName) - names := sortedGlobalTableNames(db.GlobalTables, startName) - filtered := filterGlobalTables(db.GlobalTables, names, regionFilter) + names := sortedGlobalTableNames(db.globalTables, startName) + filtered := filterGlobalTables(db.globalTables, names, regionFilter) filtered, lastEvaluated := applyGlobalTableLimit(filtered, input.Limit) return &dynamodb.ListGlobalTablesOutput{ @@ -601,7 +590,7 @@ func (db *InMemoryDB) UpdateGlobalTable( db.mu.Lock("UpdateGlobalTable") defer db.mu.Unlock() - gt, exists := db.GlobalTables[name] + gt, exists := db.globalTables.Get(name) if !exists { return nil, &Error{ Type: errGlobalTableNotFoundType, @@ -637,12 +626,7 @@ func (db *InMemoryDB) UpdateGlobalTable( // Must be called with db.mu held for reading. func (db *InMemoryDB) findSourceTableLocked(name string, regions []string) *Table { for _, region := range regions { - regionTables, ok := db.Tables[region] - if !ok { - continue - } - - if t, tableExists := regionTables[name]; tableExists { + if t, tableExists := db.tables.Get(tableKey(region, name)); tableExists { return t } } @@ -689,16 +673,12 @@ func (db *InMemoryDB) applyGlobalTableReplicaCreate( gt.ReplicationGroup = append(gt.ReplicationGroup, regionName) } - if _, ok := db.Tables[regionName]; !ok { - db.Tables[regionName] = make(map[string]*Table) - } - - if _, tableExists := db.Tables[regionName][name]; !tableExists { + if existing, tableExists := db.tables.Get(tableKey(regionName, name)); !tableExists { replica := db.buildReplicaTableLocked(name, regionName, source) replica.GlobalTableName = name - db.Tables[regionName][name] = replica + db.tables.Put(replica) } else { - db.Tables[regionName][name].GlobalTableName = name + existing.GlobalTableName = name } return nil @@ -724,9 +704,7 @@ func (db *InMemoryDB) applyGlobalTableReplicaDelete( gt.ReplicationGroup = remaining - if regionTables, ok := db.Tables[regionName]; ok { - delete(regionTables, name) - } + db.tables.Delete(tableKey(regionName, name)) return nil } @@ -736,12 +714,7 @@ func (db *InMemoryDB) applyGlobalTableReplicaDelete( func (db *InMemoryDB) rebuildGlobalTableReplicasLocked(name string, regions []string) { allReplicas := buildAllReplicas(regions) for _, region := range regions { - regionTables, ok := db.Tables[region] - if !ok { - continue - } - - if t, tableExists := regionTables[name]; tableExists { + if t, tableExists := db.tables.Get(tableKey(region, name)); tableExists { t.Replicas = buildReplicasExcluding(allReplicas, region) } } @@ -769,8 +742,13 @@ func (db *InMemoryDB) buildReplicaTableLocked(name, region string, source *Table } // sortedGlobalTableNames returns sorted global table names starting after startName. -func sortedGlobalTableNames(tables map[string]*StoredGlobalTable, startName string) []string { - names := collections.SortedKeys(tables) +func sortedGlobalTableNames(tables *store.Table[StoredGlobalTable], startName string) []string { + all := tables.All() + names := make([]string, 0, len(all)) + for _, gt := range all { + names = append(names, gt.GlobalTableName) + } + sort.Strings(names) if startName == "" { return names @@ -787,14 +765,17 @@ func sortedGlobalTableNames(tables map[string]*StoredGlobalTable, startName stri // filterGlobalTables converts stored global tables to SDK types, applying an optional region filter. func filterGlobalTables( - tables map[string]*StoredGlobalTable, + tables *store.Table[StoredGlobalTable], names []string, regionFilter string, ) []types.GlobalTable { filtered := make([]types.GlobalTable, 0, len(names)) for _, name := range names { - gt := tables[name] + gt, ok := tables.Get(name) + if !ok { + continue + } if regionFilter != "" && !slices.Contains(gt.ReplicationGroup, regionFilter) { continue @@ -924,7 +905,7 @@ func (db *InMemoryDB) getTableByARN(resourceARN string) *Table { db.mu.RLock("getTableByARN") defer db.mu.RUnlock() - for _, table := range db.Tables[region] { + for _, table := range db.tablesByRegion.Get(region) { if table.TableArn == resourceARN { return table } @@ -979,7 +960,7 @@ func (db *InMemoryDB) ListContributorInsights( var summaries []types.ContributorInsightsSummary - for name, t := range db.Tables[region] { + for _, t := range db.tablesByRegion.Get(region) { t.mu.RLock("ListContributorInsights") enabled := t.ContributorInsightsEnabled t.mu.RUnlock() @@ -988,7 +969,7 @@ func (db *InMemoryDB) ListContributorInsights( continue } - tableName := name + tableName := t.Name summaries = append(summaries, types.ContributorInsightsSummary{ TableName: &tableName, ContributorInsightsStatus: types.ContributorInsightsStatusEnabled, @@ -1057,7 +1038,7 @@ func (db *InMemoryDB) UpdateGlobalTableSettings( name := *input.GlobalTableName db.mu.Lock("UpdateGlobalTableSettings") - gt, exists := db.GlobalTables[name] + gt, exists := db.globalTables.Get(name) if !exists { db.mu.Unlock() @@ -1425,15 +1406,16 @@ func (db *InMemoryDB) captureExecTxnSnapshots( snapshots := make(map[string]tableStateSnapshot, len(tableNames)) db.mu.RLock("ExecuteTransaction.snapshot") - regionTables := db.Tables[region] - db.mu.RUnlock() - + tables := make(map[string]*Table, len(tableNames)) for _, name := range tableNames { - if regionTables == nil { - continue + if t, ok := db.tables.Get(tableKey(region, name)); ok { + tables[name] = t } + } + db.mu.RUnlock() - t, ok := regionTables[name] + for _, name := range tableNames { + t, ok := tables[name] if !ok { continue } @@ -1471,7 +1453,12 @@ func (db *InMemoryDB) restoreExecTxnSnapshots( region := getRegionFromContext(ctx, db) db.mu.RLock("ExecuteTransaction.restore") - regionTables := db.Tables[region] + tables := make(map[string]*Table, len(tableNames)) + for _, name := range tableNames { + if t, ok := db.tables.Get(tableKey(region, name)); ok { + tables[name] = t + } + } db.mu.RUnlock() for _, name := range tableNames { @@ -1480,11 +1467,7 @@ func (db *InMemoryDB) restoreExecTxnSnapshots( continue } - if regionTables == nil { - continue - } - - t, tableOK := regionTables[name] + t, tableOK := tables[name] if !tableOK { continue } @@ -1877,7 +1860,7 @@ func (db *InMemoryDB) replicateItemMutation( ) { // Look up global table metadata under read lock. db.mu.RLock("replicateItemMutation-gt") - gt, exists := db.GlobalTables[globalTableName] + gt, exists := db.globalTables.Get(globalTableName) db.mu.RUnlock() if !exists { @@ -1903,13 +1886,7 @@ func (db *InMemoryDB) applyMutationToReplica( ) { // Look up the replica table under a short read lock. db.mu.RLock("applyMutationToReplica-lookup") - regionTables := db.Tables[region] - - var replica *Table - if regionTables != nil { - replica = regionTables[tableName] - } - + replica, _ := db.tables.Get(tableKey(region, tableName)) db.mu.RUnlock() if replica == nil { diff --git a/services/dynamodb/import_export_s3.go b/services/dynamodb/import_export_s3.go index c0a0c26f1..511ef0627 100644 --- a/services/dynamodb/import_export_s3.go +++ b/services/dynamodb/import_export_s3.go @@ -356,21 +356,19 @@ func (db *InMemoryDB) snapshotItemsByTableARN(tableARN string) []map[string]any db.mu.RLock("snapshotItemsByTableARN") defer db.mu.RUnlock() - for _, regionTables := range db.Tables { - for _, t := range regionTables { - if t.TableArn != tableARN { - continue - } - - t.mu.RLock("snapshotItemsByTableARN") - items := make([]map[string]any, 0, len(t.Items)) - for i := range t.Items { - items = append(items, deepCopyItem(t.Items[i])) - } - t.mu.RUnlock() + for _, t := range db.tables.All() { + if t.TableArn != tableARN { + continue + } - return items + t.mu.RLock("snapshotItemsByTableARN") + items := make([]map[string]any, 0, len(t.Items)) + for i := range t.Items { + items = append(items, deepCopyItem(t.Items[i])) } + t.mu.RUnlock() + + return items } return nil @@ -385,17 +383,15 @@ func (db *InMemoryDB) countTableItems(_ context.Context, tableARN string) (int, db.mu.RLock("countTableItems") defer db.mu.RUnlock() - for _, regionTables := range db.Tables { - for _, t := range regionTables { - if t.TableArn != tableARN { - continue - } - t.mu.RLock("countTableItems") - n := len(t.Items) - t.mu.RUnlock() - - return n, nil + for _, t := range db.tables.All() { + if t.TableArn != tableARN { + continue } + t.mu.RLock("countTableItems") + n := len(t.Items) + t.mu.RUnlock() + + return n, nil } return 0, nil diff --git a/services/dynamodb/item_ops.go b/services/dynamodb/item_ops.go index bfdb22879..6ca5a5e26 100644 --- a/services/dynamodb/item_ops.go +++ b/services/dynamodb/item_ops.go @@ -118,14 +118,8 @@ func (db *InMemoryDB) getTableRLock(ctx context.Context, name string) (*Table, b defer db.mu.RUnlock() region := getRegionFromContext(ctx, db) - regionTables, regionExists := db.Tables[region] - if regionExists { - if table, tableExists := regionTables[name]; tableExists { - return table, true - } - } - return nil, false + return db.tables.Get(tableKey(region, name)) } func getPKAndSK( diff --git a/services/dynamodb/item_ops_batch.go b/services/dynamodb/item_ops_batch.go index 9b5ffa29a..e7a9e6ee1 100644 --- a/services/dynamodb/item_ops_batch.go +++ b/services/dynamodb/item_ops_batch.go @@ -242,15 +242,15 @@ func (db *InMemoryDB) batchGetTableRefs( defer db.mu.RUnlock() region := getRegionFromContext(ctx, db) - regionTables, ok := db.Tables[region] - if !ok { + if len(db.tablesByRegion.Get(region)) == 0 { // Region might not have tables yet return nil, NewResourceNotFoundException("No tables found in region") } + tableRefs := make(map[string]*Table, len(requestItems)) for tableName := range requestItems { - t, exists := regionTables[tableName] + t, exists := db.tables.Get(tableKey(region, tableName)) if !exists { return nil, NewResourceNotFoundException("Table not found: " + tableName) } @@ -302,6 +302,49 @@ func validateAllBatchWriteRequests( return err } } + + // AWS rejects a BatchWriteItem whose per-table request list targets the same + // item (by primary key) more than once — whether via two PutRequests, two + // DeleteRequests, or a Put+Delete pair — since the outcome would be + // order-dependent and undefined. + if err := validateNoDuplicateBatchWriteKeys(requests, table); err != nil { + return err + } + } + + return nil +} + +// validateNoDuplicateBatchWriteKeys returns a ValidationException when requests +// (all WriteRequests targeting a single table) reference the same primary key more +// than once. Mirrors AWS's real BatchWriteItem behaviour, which rejects such +// requests wholesale rather than silently picking a winner. +func validateNoDuplicateBatchWriteKeys(requests []types.WriteRequest, table *Table) error { + pkDef, skDef := getPKAndSK(table.KeySchema) + seen := make(map[string]struct{}, len(requests)) + + for _, req := range requests { + var wireItem map[string]any + + switch { + case req.PutRequest != nil: + wireItem = models.FromSDKItem(req.PutRequest.Item) + case req.DeleteRequest != nil: + wireItem = models.FromSDKItem(req.DeleteRequest.Key) + default: + continue + } + + canon := BuildKeyString(wireItem, pkDef.AttributeName) + if skDef.AttributeName != "" { + canon += "\x00" + BuildKeyString(wireItem, skDef.AttributeName) + } + + if _, dup := seen[canon]; dup { + return NewValidationException("Provided list of item keys contains duplicates") + } + + seen[canon] = struct{}{} } return nil @@ -483,15 +526,9 @@ func (db *InMemoryDB) getRequestTables( requestItems map[string][]types.WriteRequest, ) (map[string]*Table, error) { tables := make(map[string]*Table, len(requestItems)) - regionTables, regionExists := db.Tables[region] for tableName := range requestItems { - var table *Table - if regionExists { - if t, tableExists := regionTables[tableName]; tableExists { - table = t - } - } + table, _ := db.tables.Get(tableKey(region, tableName)) if table == nil { return nil, NewResourceNotFoundException("Table not found: " + tableName) diff --git a/services/dynamodb/item_ops_query.go b/services/dynamodb/item_ops_query.go index 80360a358..82e3c7c78 100644 --- a/services/dynamodb/item_ops_query.go +++ b/services/dynamodb/item_ops_query.go @@ -86,7 +86,10 @@ func (db *InMemoryDB) QueryWithContext( return nil, err } - if verr := validateSelectConstraints(input.Select, idxName, projection); verr != nil { + if verr := validateSelectConstraints( + input.Select, idxName, projection, + aws.ToString(input.ProjectionExpression), input.AttributesToGet, + ); verr != nil { return nil, verr } @@ -458,10 +461,16 @@ func (db *InMemoryDB) processQueryResults( eav, ) - outItems := make([]map[string]types.AttributeValue, len(items)) - for i, it := range items { - sdkIt, _ := models.ToSDKItem(it) - outItems[i] = sdkIt + // AWS omits Items entirely when Select=COUNT: "Returns the number of matching + // items, rather than the matching items themselves." Count/ScannedCount still + // reflect the matched/scanned totals. + var outItems []map[string]types.AttributeValue + if input.Select != types.SelectCount { + outItems = make([]map[string]types.AttributeValue, len(items)) + for i, it := range items { + sdkIt, _ := models.ToSDKItem(it) + outItems[i] = sdkIt + } } out := &dynamodb.QueryOutput{ diff --git a/services/dynamodb/item_ops_scan.go b/services/dynamodb/item_ops_scan.go index e6417587e..27ee17b6e 100644 --- a/services/dynamodb/item_ops_scan.go +++ b/services/dynamodb/item_ops_scan.go @@ -109,7 +109,10 @@ func (db *InMemoryDB) ScanWithContext( return nil, err } - if verr := validateSelectConstraints(input.Select, aws.ToString(input.IndexName), projection); verr != nil { + if verr := validateSelectConstraints( + input.Select, aws.ToString(input.IndexName), projection, + aws.ToString(input.ProjectionExpression), input.AttributesToGet, + ); verr != nil { return nil, verr } @@ -151,10 +154,16 @@ func (db *InMemoryDB) buildScanOutput( } } - outItems := make([]map[string]types.AttributeValue, len(items)) - for i, it := range items { - sdkIt, _ := models.ToSDKItem(it) - outItems[i] = sdkIt + // AWS omits Items entirely when Select=COUNT: "Returns the number of matching + // items, rather than the matching items themselves." Count/ScannedCount still + // reflect the matched/scanned totals. + var outItems []map[string]types.AttributeValue + if input.Select != types.SelectCount { + outItems = make([]map[string]types.AttributeValue, len(items)) + for i, it := range items { + sdkIt, _ := models.ToSDKItem(it) + outItems[i] = sdkIt + } } out := &dynamodb.ScanOutput{ diff --git a/services/dynamodb/janitor.go b/services/dynamodb/janitor.go index 0c9ff835f..269977095 100644 --- a/services/dynamodb/janitor.go +++ b/services/dynamodb/janitor.go @@ -125,12 +125,7 @@ func (j *Janitor) snapshotPITRTables(_ context.Context) { db := j.Backend db.mu.RLock("DDBJanitor.snapshotPITR") - tables := make([]*Table, 0, len(db.Tables)) - for _, regionTables := range db.Tables { - for _, t := range regionTables { - tables = append(tables, t) - } - } + tables := db.tables.All() db.mu.RUnlock() now := time.Now().UTC() @@ -166,16 +161,9 @@ func (j *Janitor) runTableCleaner(ctx context.Context) { // lock so that concurrent DescribeTable / PutItem / Query calls are not stalled // while thousands of per-table resources are being released. db.mu.Lock("DDBJanitor") - depth := 0 - tablesToClose := make([]*Table, 0, len(db.deletingTables)) - - for region, regionTables := range db.deletingTables { - for name, table := range regionTables { - depth++ - tablesToClose = append(tablesToClose, table) - delete(db.deletingTables[region], name) - } - } + tablesToClose := db.deletingTables.All() + depth := len(tablesToClose) + db.deletingTables.Reset() db.mu.Unlock() // Release per-table resources outside the global lock. diff --git a/services/dynamodb/persistence.go b/services/dynamodb/persistence.go index 8321c3b1e..dbc104fa3 100644 --- a/services/dynamodb/persistence.go +++ b/services/dynamodb/persistence.go @@ -11,12 +11,28 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/persistence" ) +// dynamodbSnapshotVersion identifies the shape of dbSnapshot. It must be +// bumped whenever a change to Table, Backup, StoredGlobalTable, or +// dbSnapshot itself would make an older snapshot unsafe to decode as the +// current shape. Restore compares this against the persisted value and +// discards (rather than attempts to partially decode) any mismatch -- see +// Restore below. This mirrors the services/sqs pilot (commit 0f09d77c) and +// the services/ec2 conversion (commit 12e611a4). +// +// Only tables/backups/globalTables are persisted here, matching the +// pre-refactor behavior: deletingTables, exports, imports, txnTokens, +// txnPending, and fisReplicationPaused were never part of the snapshot +// either (deletingTables is a transient staging area drained by the +// janitor; the rest are short-lived caches/metadata not worth persisting). +const dynamodbSnapshotVersion = 1 + type dbSnapshot struct { - Tables map[string]map[string]*Table `json:"Tables"` - Backups map[string]*Backup `json:"Backups,omitempty"` - GlobalTables map[string]*StoredGlobalTable `json:"GlobalTables,omitempty"` - DefaultRegion string `json:"DefaultRegion"` - AccountID string `json:"AccountID"` + DefaultRegion string `json:"defaultRegion"` + AccountID string `json:"accountID"` + Tables []*Table `json:"tables"` + Backups []*Backup `json:"backups,omitempty"` + GlobalTables []*StoredGlobalTable `json:"globalTables,omitempty"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. @@ -29,9 +45,10 @@ func (db *InMemoryDB) Snapshot(ctx context.Context) []byte { defer db.mu.RUnlock() snap := dbSnapshot{ - Tables: db.Tables, - Backups: db.Backups, - GlobalTables: db.GlobalTables, + Version: dynamodbSnapshotVersion, + Tables: db.tables.Snapshot(), + Backups: db.backups.Snapshot(), + GlobalTables: db.globalTables.Snapshot(), DefaultRegion: db.defaultRegion, AccountID: db.accountID, } @@ -59,47 +76,50 @@ func (db *InMemoryDB) Restore(ctx context.Context, data []byte) error { return err } - db.mu.Lock("Restore") - defer db.mu.Unlock() + // Reinitialise per-table mutexes and rebuild indexes before taking db.mu, + // matching the pre-refactor ordering (this touches only the freshly + // unmarshaled Table values, not any backend state). + for _, t := range snap.Tables { + if t.mu == nil { + t.mu = lockmetrics.New("ddb-table") + } - if snap.Tables == nil { - snap.Tables = make(map[string]map[string]*Table) + t.rebuildIndexes() + restoreStreamSeq(t) } - if snap.Backups == nil { - snap.Backups = make(map[string]*Backup) - } + db.mu.Lock("Restore") + defer db.mu.Unlock() - if snap.GlobalTables == nil { - snap.GlobalTables = make(map[string]*StoredGlobalTable) - } + if snap.Version != dynamodbSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/sqs pilot (commit 0f09d77c). + logger.Load(ctx).WarnContext(ctx, + "DynamoDB: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", dynamodbSnapshotVersion, + ) - // Reinitialise per-table mutexes and rebuild indexes. - for _, regionTables := range snap.Tables { - for _, t := range regionTables { - if t.mu == nil { - t.mu = lockmetrics.New("ddb-table") - } + db.registry.ResetAll() - t.rebuildIndexes() - restoreStreamSeq(t) - } + return nil } - db.Tables = snap.Tables - db.Backups = snap.Backups - db.GlobalTables = snap.GlobalTables + db.tables.Restore(snap.Tables) + db.backups.Restore(snap.Backups) + db.globalTables.Restore(snap.GlobalTables) db.defaultRegion = snap.DefaultRegion db.accountID = snap.AccountID // Rebuild the stream ARN reverse index from the restored tables. - db.streamARNIndex = make(map[string]*Table) + db.streamARNIndex.Reset() - for _, regionTables := range db.Tables { - for _, t := range regionTables { - if t.StreamARN != "" { - db.streamARNIndex[t.StreamARN] = t - } + for _, t := range db.tables.All() { + if t.StreamARN != "" { + db.streamARNIndex.Put(t) } } diff --git a/services/dynamodb/query_test.go b/services/dynamodb/query_test.go index 70e20ae40..45cf8432b 100644 --- a/services/dynamodb/query_test.go +++ b/services/dynamodb/query_test.go @@ -9,6 +9,7 @@ import ( "github.com/blackbirdworks/gopherstack/services/dynamodb" "github.com/aws/aws-sdk-go-v2/aws" + awsdynamodb "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -282,6 +283,101 @@ func TestQuery_BlankSKValidation(t *testing.T) { } } +// TestQuery_SelectCount_OmitsItems verifies AWS's documented Select=COUNT +// behaviour: "Returns the number of matching items, rather than the matching +// items themselves." Count/ScannedCount must still reflect the real totals, +// but Items must come back empty. +func TestQuery_SelectCount_OmitsItems(t *testing.T) { + t.Parallel() + + db := dynamodb.NewInMemoryDB() + tableName := "QuerySelectCountTable" + createTableHelper(t, db, tableName, "pk", "sk") + + for i := range 3 { + putInput := models.PutItemInput{ + TableName: tableName, + Item: map[string]any{ + "pk": map[string]any{"S": "key"}, + "sk": map[string]any{"N": strconv.Itoa(i)}, + }, + } + sdkPut, _ := models.ToSDKPutItemInput(&putInput) + _, _ = db.PutItem(t.Context(), sdkPut) + } + + queryInput := mustUnmarshal[models.QueryInput](t, `{ + "TableName": "`+tableName+`", + "KeyConditionExpression": "pk = :pk", + "ExpressionAttributeValues": {":pk": {"S": "key"}} + }`) + sdkQuery, _ := models.ToSDKQueryInput(&queryInput) + sdkQuery.Select = types.SelectCount + + out, err := db.Query(t.Context(), sdkQuery) + require.NoError(t, err) + assert.Equal(t, int32(3), out.Count) + assert.Equal(t, int32(3), out.ScannedCount) + assert.Empty(t, out.Items, "Select=COUNT must not return Items") +} + +// TestQuery_SelectConstraints_Rejected covers the documented restrictions on +// the Select parameter's interaction with ProjectionExpression/AttributesToGet: +// "If you use the ProjectionExpression parameter, then the value for Select +// can only be SPECIFIC_ATTRIBUTES. Any other value for Select will return an +// error." (see API_Query.html "Select" parameter docs). +func TestQuery_SelectConstraints_Rejected(t *testing.T) { + t.Parallel() + + tests := []struct { + mutate func(*awsdynamodb.QueryInput) + name string + }{ + { + name: "COUNT with ProjectionExpression", + mutate: func(in *awsdynamodb.QueryInput) { + in.Select = types.SelectCount + in.ProjectionExpression = aws.String("pk") + }, + }, + { + name: "ALL_ATTRIBUTES with ProjectionExpression", + mutate: func(in *awsdynamodb.QueryInput) { + in.Select = types.SelectAllAttributes + in.ProjectionExpression = aws.String("pk") + }, + }, + { + name: "SPECIFIC_ATTRIBUTES without a projection", + mutate: func(in *awsdynamodb.QueryInput) { + in.Select = types.SelectSpecificAttributes + }, + }, + } + + for i, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + db := dynamodb.NewInMemoryDB() + tableName := "QuerySelectRejectTable" + strconv.Itoa(i) + createTableHelper(t, db, tableName, "pk", "sk") + + queryInput := mustUnmarshal[models.QueryInput](t, `{ + "TableName": "`+tableName+`", + "KeyConditionExpression": "pk = :pk", + "ExpressionAttributeValues": {":pk": {"S": "key"}} + }`) + sdkQuery, _ := models.ToSDKQueryInput(&queryInput) + tc.mutate(sdkQuery) + + _, err := db.Query(t.Context(), sdkQuery) + require.Error(t, err) + assert.Contains(t, err.Error(), "ValidationException") + }) + } +} + func TestQuery_ConsumedCapacity(t *testing.T) { t.Parallel() diff --git a/services/dynamodb/scan_test.go b/services/dynamodb/scan_test.go index 12dcfab82..ec3bb1db5 100644 --- a/services/dynamodb/scan_test.go +++ b/services/dynamodb/scan_test.go @@ -480,3 +480,97 @@ func TestScan_ConsumedCapacity(t *testing.T) { require.NotNil(t, out.ConsumedCapacity, "ConsumedCapacity should be populated when requested") assert.Greater(t, *out.ConsumedCapacity.CapacityUnits, 0.0) } + +// TestScan_SelectCount_OmitsItems verifies AWS's documented Select=COUNT +// behaviour: "Returns the number of matching items, rather than the matching +// items themselves." Count/ScannedCount must still reflect the real totals, +// but Items must come back empty. +func TestScan_SelectCount_OmitsItems(t *testing.T) { + t.Parallel() + + db := dynamodb.NewInMemoryDB() + tableName := "ScanSelectCountTable" + _, err := db.CreateTable(t.Context(), &dynamodb_sdk.CreateTableInput{ + TableName: &tableName, + AttributeDefinitions: []types.AttributeDefinition{ + {AttributeName: aws.String("pk"), AttributeType: types.ScalarAttributeTypeS}, + }, + KeySchema: []types.KeySchemaElement{ + {AttributeName: aws.String("pk"), KeyType: types.KeyTypeHash}, + }, + BillingMode: types.BillingModePayPerRequest, + }) + require.NoError(t, err) + + for i := range 4 { + _, err = db.PutItem(t.Context(), &dynamodb_sdk.PutItemInput{ + TableName: &tableName, + Item: map[string]types.AttributeValue{ + "pk": &types.AttributeValueMemberS{Value: "item-" + strconv.Itoa(i)}, + }, + }) + require.NoError(t, err) + } + + out, err := db.Scan(t.Context(), &dynamodb_sdk.ScanInput{ + TableName: &tableName, + Select: types.SelectCount, + }) + require.NoError(t, err) + assert.Equal(t, int32(4), out.Count) + assert.Equal(t, int32(4), out.ScannedCount) + assert.Empty(t, out.Items, "Select=COUNT must not return Items") +} + +// TestScan_SelectConstraints_Rejected mirrors the equivalent Query coverage: +// Select values other than SPECIFIC_ATTRIBUTES cannot be combined with a +// ProjectionExpression, and SPECIFIC_ATTRIBUTES requires one. +func TestScan_SelectConstraints_Rejected(t *testing.T) { + t.Parallel() + + tests := []struct { + mutate func(*dynamodb_sdk.ScanInput) + name string + }{ + { + name: "COUNT with ProjectionExpression", + mutate: func(in *dynamodb_sdk.ScanInput) { + in.Select = types.SelectCount + in.ProjectionExpression = aws.String("pk") + }, + }, + { + name: "SPECIFIC_ATTRIBUTES without a projection", + mutate: func(in *dynamodb_sdk.ScanInput) { + in.Select = types.SelectSpecificAttributes + }, + }, + } + + for i, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + db := dynamodb.NewInMemoryDB() + tableName := "ScanSelectRejectTable" + strconv.Itoa(i) + _, err := db.CreateTable(t.Context(), &dynamodb_sdk.CreateTableInput{ + TableName: &tableName, + AttributeDefinitions: []types.AttributeDefinition{ + {AttributeName: aws.String("pk"), AttributeType: types.ScalarAttributeTypeS}, + }, + KeySchema: []types.KeySchemaElement{ + {AttributeName: aws.String("pk"), KeyType: types.KeyTypeHash}, + }, + BillingMode: types.BillingModePayPerRequest, + }) + require.NoError(t, err) + + scanInput := &dynamodb_sdk.ScanInput{TableName: &tableName} + tc.mutate(scanInput) + + _, err = db.Scan(t.Context(), scanInput) + require.Error(t, err) + assert.Contains(t, err.Error(), "ValidationException") + }) + } +} diff --git a/services/dynamodb/store.go b/services/dynamodb/store.go index 2c2ec9bd2..869e7462d 100644 --- a/services/dynamodb/store.go +++ b/services/dynamodb/store.go @@ -12,6 +12,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/config" "github.com/blackbirdworks/gopherstack/pkgs/dynamoattr" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" "github.com/blackbirdworks/gopherstack/services/dynamodb/models" ) @@ -135,17 +136,31 @@ const ( ) // InMemoryDB stores tables and items organized by region. +// +// tables, deletingTables, backups, globalTables, exports, imports, and +// streamARNIndex are *store.Table wrappers (see pkgs/store and +// store_setup.go) registered once on registry at construction time; +// txnTokens, txnPending, and fisReplicationPaused remain plain maps because +// their value type (time.Time) carries no identity a store.Table key +// function could extract -- see store_setup.go's doc comment. type InMemoryDB struct { - Tables map[string]map[string]*Table - deletingTables map[string]map[string]*Table - Backups map[string]*Backup // backupARN → Backup - GlobalTables map[string]*StoredGlobalTable // globalTableName → StoredGlobalTable - exports map[string]storedExport // exportARN → storedExport - imports map[string]storedImport // importARN → storedImport - txnTokens map[string]time.Time // committed idempotency tokens → expiry time - txnPending map[string]time.Time // in-progress idempotency tokens → start time - streamARNIndex map[string]*Table // streamARN → Table (reverse index) - fisReplicationPaused map[string]time.Time // keyed by table ARN; value is expiry (zero = no expiry) + tables *store.Table[Table] // key: tableKey(region, name) + tablesByRegion *store.Index[Table] // secondary index: region -> tables in that region + // deletingTables uses the same key scheme as tables; it is a staging area + // for deleted tables, drained by the janitor's runTableCleaner. + deletingTables *store.Table[Table] + backups *store.Table[Backup] // key: BackupArn + globalTables *store.Table[StoredGlobalTable] // key: GlobalTableName + exports *store.Table[storedExport] // key: ExportArn + imports *store.Table[storedImport] // key: ImportArn + // streamARNIndex is a reverse index (key: StreamARN) onto the same + // *Table pointers stored in `tables` -- see store_setup.go's + // streamARNKeyFn doc for why this can't be a store.Index. + streamARNIndex *store.Table[Table] + registry *store.Registry + txnTokens map[string]time.Time // committed idempotency tokens → expiry time + txnPending map[string]time.Time // in-progress idempotency tokens → start time + fisReplicationPaused map[string]time.Time // keyed by table ARN; value is expiry (zero = no expiry) exprCache *ExpressionCache throttler *Throttler iteratorStore *ShardIteratorStore // opaque shard iterator tokens @@ -268,16 +283,10 @@ type Table struct { func NewInMemoryDB() *InMemoryDB { const exprCacheSize = 1000 - return &InMemoryDB{ - Tables: make(map[string]map[string]*Table), - deletingTables: make(map[string]map[string]*Table), - Backups: make(map[string]*Backup), - GlobalTables: make(map[string]*StoredGlobalTable), - exports: make(map[string]storedExport), - imports: make(map[string]storedImport), + db := &InMemoryDB{ + registry: store.NewRegistry(), txnTokens: make(map[string]time.Time), txnPending: make(map[string]time.Time), - streamARNIndex: make(map[string]*Table), fisReplicationPaused: make(map[string]time.Time), exprCache: NewExpressionCache(exprCacheSize), iteratorStore: NewShardIteratorStore(), @@ -286,6 +295,9 @@ func NewInMemoryDB() *InMemoryDB { mu: lockmetrics.New("ddb"), throttler: NewThrottler(false), } + registerAllTables(db) + + return db } // Close releases all backend resources. @@ -293,10 +305,8 @@ func (db *InMemoryDB) Close() { db.mu.Lock("Close") defer db.mu.Unlock() - for _, regionTables := range db.Tables { - for _, table := range regionTables { - stopTableTimers(table) - } + for _, table := range db.tables.All() { + stopTableTimers(table) } if db.exprCache != nil { @@ -460,10 +470,8 @@ func (db *InMemoryDB) SetKinesisEmitter(emitter KinesisEmitter) { defer db.mu.Unlock() db.kinesisEmitter = emitter - for _, region := range db.Tables { - for _, t := range region { - t.kinesisEmitter = emitter - } + for _, t := range db.tables.All() { + t.kinesisEmitter = emitter } } @@ -546,12 +554,14 @@ func (db *InMemoryDB) Regions() []string { db.mu.RLock("Regions") defer db.mu.RUnlock() - var regions []string + seen := make(map[string]struct{}) + for _, t := range db.tables.All() { + seen[tableRegion(t)] = struct{}{} + } - for region, regionTables := range db.Tables { - if len(regionTables) > 0 { - regions = append(regions, region) - } + regions := make([]string, 0, len(seen)) + for r := range seen { + regions = append(regions, r) } sort.Strings(regions) @@ -566,13 +576,13 @@ func (db *InMemoryDB) TableNamesByRegion(region string) []string { var names []string - for r, regionTables := range db.Tables { - if region != "" && r != region { - continue + if region != "" { + for _, t := range db.tablesByRegion.Get(region) { + names = append(names, t.Name) } - - for name := range regionTables { - names = append(names, name) + } else { + for _, t := range db.tables.All() { + names = append(names, t.Name) } } @@ -586,14 +596,7 @@ func (db *InMemoryDB) ListAllTables() []*Table { db.mu.RLock("ListAllTables") defer db.mu.RUnlock() - var tables []*Table - for _, regionTables := range db.Tables { - for _, table := range regionTables { - tables = append(tables, table) - } - } - - return tables + return db.tables.All() } // GetTable returns a table by name from the default region (for UI/backward compatibility). @@ -610,14 +613,7 @@ func (db *InMemoryDB) GetTableInRegion(name string, region string) (*Table, bool region = db.defaultRegion } - regionTables, exists := db.Tables[region] - if !exists { - return nil, false - } - - table, exists := regionTables[name] - - return table, exists + return db.tables.Get(tableKey(region, name)) } // SetDefaultRegion sets the default region for this backend. @@ -647,19 +643,18 @@ func (db *InMemoryDB) TaggedTables() []TaggedTableInfo { db.mu.RLock("TaggedTables") defer db.mu.RUnlock() - var result []TaggedTableInfo + all := db.tables.All() + result := make([]TaggedTableInfo, 0, len(all)) - for _, regionTables := range db.Tables { - for _, table := range regionTables { - var tagMap map[string]string - if table.Tags != nil { - table.mu.RLock("TaggedTables.tag") - tagMap = table.Tags.Clone() - table.mu.RUnlock() - } - - result = append(result, TaggedTableInfo{ARN: table.TableArn, Tags: tagMap}) + for _, table := range all { + var tagMap map[string]string + if table.Tags != nil { + table.mu.RLock("TaggedTables.tag") + tagMap = table.Tags.Clone() + table.mu.RUnlock() } + + result = append(result, TaggedTableInfo{ARN: table.TableArn, Tags: tagMap}) } return result @@ -709,19 +704,17 @@ func (db *InMemoryDB) Purge(ctx context.Context, cutoff time.Time) { // purgeActiveTables removes active tables created before cutoff. // Returns false if ctx is cancelled mid-loop. func (db *InMemoryDB) purgeActiveTables(ctx context.Context, cutoff time.Time) bool { - for _, regionTables := range db.Tables { - for n, table := range regionTables { - if ctx.Err() != nil { - return false - } - if table.CreationDateTime.Before(cutoff) { - stopTableTimers(table) - if table.Tags != nil { - table.Tags.Close() - } - table.mu.Close() - delete(regionTables, n) + for _, table := range db.tables.All() { + if ctx.Err() != nil { + return false + } + if table.CreationDateTime.Before(cutoff) { + stopTableTimers(table) + if table.Tags != nil { + table.Tags.Close() } + table.mu.Close() + db.tables.Delete(tableKeyFn(table)) } } @@ -731,12 +724,12 @@ func (db *InMemoryDB) purgeActiveTables(ctx context.Context, cutoff time.Time) b // purgeStreamARNIndex removes stream ARN index entries for tables deleted before cutoff. // Returns false if ctx is cancelled mid-loop. func (db *InMemoryDB) purgeStreamARNIndex(ctx context.Context, cutoff time.Time) bool { - for arn, table := range db.streamARNIndex { + for _, table := range db.streamARNIndex.All() { if ctx.Err() != nil { return false } if table.CreationDateTime.Before(cutoff) { - delete(db.streamARNIndex, arn) + db.streamARNIndex.Delete(table.StreamARN) } } @@ -746,12 +739,12 @@ func (db *InMemoryDB) purgeStreamARNIndex(ctx context.Context, cutoff time.Time) // purgeBackups removes backups created before cutoff. // Returns false if ctx is cancelled mid-loop. func (db *InMemoryDB) purgeBackups(ctx context.Context, cutoff time.Time) bool { - for n, backup := range db.Backups { + for _, backup := range db.backups.All() { if ctx.Err() != nil { return false } if backup.CreationDateTime.Before(cutoff) { - delete(db.Backups, n) + db.backups.Delete(backup.BackupArn) } } @@ -760,12 +753,12 @@ func (db *InMemoryDB) purgeBackups(ctx context.Context, cutoff time.Time) bool { // purgeGlobalTables removes global tables created before cutoff. func (db *InMemoryDB) purgeGlobalTables(ctx context.Context, cutoff time.Time) { - for n, gt := range db.GlobalTables { + for _, gt := range db.globalTables.All() { if ctx.Err() != nil { return } if gt.CreationDateTime.Before(cutoff) { - delete(db.GlobalTables, n) + db.globalTables.Delete(gt.GlobalTableName) } } } @@ -780,35 +773,28 @@ func (db *InMemoryDB) Reset() { // Stop activation timers and close mutex metrics for existing tables // (both active and deleting) to avoid goroutine leaks and metric registry leaks. - for _, regionTables := range db.Tables { - for _, table := range regionTables { - stopTableTimers(table) - if table.Tags != nil { - table.Tags.Close() - } - - table.mu.Close() + for _, table := range db.tables.All() { + stopTableTimers(table) + if table.Tags != nil { + table.Tags.Close() } - } - for _, regionTables := range db.deletingTables { - for _, table := range regionTables { - stopTableTimers(table) - if table.Tags != nil { - table.Tags.Close() - } + table.mu.Close() + } - table.mu.Close() + for _, table := range db.deletingTables.All() { + stopTableTimers(table) + if table.Tags != nil { + table.Tags.Close() } + + table.mu.Close() } - db.Tables = make(map[string]map[string]*Table) - db.deletingTables = make(map[string]map[string]*Table) - db.streamARNIndex = make(map[string]*Table) - db.Backups = make(map[string]*Backup) - db.GlobalTables = make(map[string]*StoredGlobalTable) - db.exports = make(map[string]storedExport) - db.imports = make(map[string]storedImport) + // registry.ResetAll() collapses tables/deletingTables/backups/globalTables/ + // exports/imports/streamARNIndex to one call; only the plain (non-store) + // maps need explicit resets below. + db.registry.ResetAll() db.txnTokens = make(map[string]time.Time) db.txnPending = make(map[string]time.Time) db.fisReplicationPaused = make(map[string]time.Time) @@ -839,7 +825,7 @@ func (db *InMemoryDB) storeExport(desc exportDescriptionFields) { defer db.mu.Unlock() now := time.Now() - rec := storedExport{ + rec := &storedExport{ CreatedAt: now, StartTime: now, ExportArn: desc.ExportArn, @@ -861,44 +847,21 @@ func (db *InMemoryDB) storeExport(desc exportDescriptionFields) { rec.ExportTime = time.Now() } - db.exports[desc.ExportArn] = rec - evictOldest( + db.exports.Put(rec) + evictOldestFromTable( db.exports, maxExportsRetained, - func(v storedExport) time.Time { return v.CreatedAt }, + exportKeyFn, + func(v *storedExport) time.Time { return v.CreatedAt }, ) } -// evictOldest drops oldest-by-CreatedAt entries from m until len(m) <= keep. -// We evict on insert rather than on a timer so memory stays bounded even when -// the janitor is disabled. timeOf returns the entry's creation timestamp. -func evictOldest[V any](m map[string]V, keep int, timeOf func(V) time.Time) { - if len(m) <= keep { - return - } - - type kv struct { - t time.Time - k string - } - - entries := make([]kv, 0, len(m)) - for k, v := range m { - entries = append(entries, kv{timeOf(v), k}) - } - sort.Slice(entries, func(i, j int) bool { return entries[i].t.Before(entries[j].t) }) - - for i := range len(m) - keep { - delete(m, entries[i].k) - } -} - // lookupExport retrieves a stored export by ARN. func (db *InMemoryDB) lookupExport(exportARN string) (exportDescriptionFields, bool) { db.mu.RLock("lookupExport") defer db.mu.RUnlock() - e, ok := db.exports[exportARN] + e, ok := db.exports.Get(exportARN) if !ok { return exportDescriptionFields{}, false } @@ -939,7 +902,7 @@ func (db *InMemoryDB) updateExport( db.mu.Lock("updateExport") defer db.mu.Unlock() - e, ok := db.exports[exportARN] + e, ok := db.exports.Get(exportARN) if !ok { return } @@ -950,7 +913,8 @@ func (db *InMemoryDB) updateExport( e.ItemCount = itemCount e.BilledSizeBytes = billedBytes e.EndTime = time.Now() - db.exports[exportARN] = e + // e is the live pointer stored in db.exports (ExportArn, its key, never + // changes), so mutating it in place is sufficient -- no Put needed. } // listExportsWire returns stored exports filtered by requestRegion and optionally by @@ -994,8 +958,9 @@ func (db *InMemoryDB) listExportsWire( ) *listExportsOutput { db.mu.RLock("listExportsWire") - summaries := make([]exportDescriptionFields, 0, len(db.exports)) - for _, e := range db.exports { + all := db.exports.All() + summaries := make([]exportDescriptionFields, 0, len(all)) + for _, e := range all { if db.regionFromARN(e.ExportArn) != requestRegion { continue } @@ -1004,7 +969,7 @@ func (db *InMemoryDB) listExportsWire( continue } - summaries = append(summaries, exportToSummaryFields(e)) + summaries = append(summaries, exportToSummaryFields(*e)) } db.mu.RUnlock() @@ -1055,11 +1020,12 @@ func (db *InMemoryDB) storeImport(imp storedImport) { if imp.CreatedAt.IsZero() { imp.CreatedAt = time.Now() } - db.imports[imp.ImportArn] = imp - evictOldest( + db.imports.Put(&imp) + evictOldestFromTable( db.imports, maxImportsRetained, - func(v storedImport) time.Time { return v.CreatedAt }, + importKeyFn, + func(v *storedImport) time.Time { return v.CreatedAt }, ) } @@ -1068,18 +1034,22 @@ func (db *InMemoryDB) lookupImport(importARN string) (storedImport, bool) { db.mu.RLock("lookupImport") defer db.mu.RUnlock() - imp, ok := db.imports[importARN] + imp, ok := db.imports.Get(importARN) + if !ok { + return storedImport{}, false + } - return imp, ok + return *imp, true } // listImportsStored returns all stored imports as a slice, sorted by ARN. func (db *InMemoryDB) listImportsStored() []storedImport { db.mu.RLock("listImportsStored") - result := make([]storedImport, 0, len(db.imports)) - for _, imp := range db.imports { - result = append(result, imp) + all := db.imports.All() + result := make([]storedImport, 0, len(all)) + for _, imp := range all { + result = append(result, *imp) } db.mu.RUnlock() diff --git a/services/dynamodb/store_setup.go b/services/dynamodb/store_setup.go new file mode 100644 index 000000000..4bd923c56 --- /dev/null +++ b/services/dynamodb/store_setup.go @@ -0,0 +1,140 @@ +package dynamodb + +// Code in this file supports the Phase 3.3 datalayer refactor: every +// map[string]*T backend resource field on InMemoryDB is registered exactly +// once, here, as a *store.Table[T] on db.registry. See pkgs/store's package +// doc and the services/sqs pilot (commit 0f09d77c) plus the services/ec2 +// conversion (commit 12e611a4) for the pattern this follows. +// +// Fields deliberately left as plain maps (NOT registered here): +// - txnTokens, txnPending (map[string]time.Time): the value (a bare +// time.Time expiry/start timestamp) carries no identity of its own — +// store.Table's model requires keyFn to derive the primary key FROM the +// value, which is impossible here since the key is an opaque caller-chosen +// idempotency token that appears nowhere in the stored time.Time. +// - fisReplicationPaused (map[string]time.Time): same reasoning, keyed by +// an externally-supplied table ARN/name that the time.Time value can't +// reproduce. +// +// This mirrors ec2's documented handful of non-pure-key-fn exclusions. + +import ( + "sort" + "strings" + "time" + + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +// tableKey builds the composite primary key used by db.tables and +// db.deletingTables. DynamoDB table names cannot contain "/" (AWS restricts +// them to letters, digits, underscore, hyphen, and period), so region+"/"+name +// is a safe, collision-free composite key across regions -- the same table +// name may legitimately exist in two different regions (e.g. global table +// replicas), so region alone or name alone would not be unique. +func tableKey(region, name string) string { return region + "/" + name } + +// tableRegion extracts the region a Table belongs to by parsing its ARN +// (format: arn:{partition}:dynamodb:{region}:{account}:table/{name}, see +// pkgs/arn.Build). TableArn is always populated with the owning region +// before a Table is inserted into db.tables/db.deletingTables -- see +// CreateTable, cloneTableSchema, buildReplicaTableLocked, +// installRestoredTable, buildReplicaTable -- and is never mutated afterward, +// so this is a stable, pure derivation suitable for use as a store.Table / +// store.Index key function. (db.regionFromARN is intentionally not reused +// here: it falls back to db.defaultRegion when parsing fails, which would +// make the key function depend on backend state rather than purely on the +// value, violating the contract store.Table/store.Index key functions rely on.) +// arnRegionPartIndex is the 0-based index of the region component in a +// colon-split ARN (arn:{partition}:{service}:{region}:{account}:{resource}). +const arnRegionPartIndex = 3 + +func tableRegion(t *Table) string { + parts := strings.Split(t.TableArn, ":") + if len(parts) > arnRegionPartIndex { + return parts[arnRegionPartIndex] + } + + return "" +} + +// tableKeyFn is the store.Table key function for db.tables and +// db.deletingTables. Both Name and (the region parsed out of) TableArn are +// fixed at construction and never mutated on an already-inserted Table (see +// tableRegion's doc), so the composite key is stable for the table's entire +// lifetime in either map. +func tableKeyFn(t *Table) string { return tableKey(tableRegion(t), t.Name) } + +// streamARNKeyFn is the store.Table key function for db.streamARNIndex, a +// reverse index from a table's StreamARN back to the same *Table pointer +// stored in db.tables. Unlike Name/TableArn, StreamARN DOES change in place +// on an existing Table (EnableStream/DisableStream/UpdateTable), so +// db.streamARNIndex is never wired up via store.Table.Put on every mutation; +// callers instead call Delete(oldARN) followed by Put(table) explicitly, +// exactly mirroring the manual map delete+insert this replaces (see +// EnableStream, DisableStream, UpdateTable, DeleteTable). This is safe +// specifically because streamARNIndex is its own primary-keyed store.Table +// rather than a store.Index (secondary index) over db.tables -- a +// store.Index's automatic Put-triggered remove/add pair derives its "old" +// key from the current (already-mutated) value, which would be wrong here. +func streamARNKeyFn(t *Table) string { return t.StreamARN } + +// backupKeyFn is the store.Table key function for db.backups. +func backupKeyFn(b *Backup) string { return b.BackupArn } + +// globalTableKeyFn is the store.Table key function for db.globalTables. +func globalTableKeyFn(gt *StoredGlobalTable) string { return gt.GlobalTableName } + +// exportKeyFn is the store.Table key function for db.exports. +func exportKeyFn(e *storedExport) string { return e.ExportArn } + +// importKeyFn is the store.Table key function for db.imports. +func importKeyFn(i *storedImport) string { return i.ImportArn } + +// registerAllTables registers every converted resource map on db.registry +// exactly once. It must be called during construction only (immediately +// after db.registry is created), never on every Reset() -- store.Register +// panics on a duplicate name. +func registerAllTables(db *InMemoryDB) { + db.tables = store.Register(db.registry, "tables", store.New(tableKeyFn)) + db.tablesByRegion = db.tables.AddIndex("region", tableRegion) + db.deletingTables = store.Register(db.registry, "deletingTables", store.New(tableKeyFn)) + db.backups = store.Register(db.registry, "backups", store.New(backupKeyFn)) + db.globalTables = store.Register(db.registry, "globalTables", store.New(globalTableKeyFn)) + db.exports = store.Register(db.registry, "exports", store.New(exportKeyFn)) + db.imports = store.Register(db.registry, "imports", store.New(importKeyFn)) + db.streamARNIndex = store.Register(db.registry, "streamARNIndex", store.New(streamARNKeyFn)) +} + +// evictOldestFromTable drops oldest-by-timeOf entries from t until +// t.Len() <= keep. It mirrors the pre-store evictOldest helper's semantics +// (evict on insert so memory stays bounded even when the janitor is +// disabled) but operates on a *store.Table instead of a bare map, using +// keyOf to recover the primary key store.Table.Delete needs (store.Table +// intentionally does not expose its internal keyFn). +func evictOldestFromTable[V any]( + t *store.Table[V], + keep int, + keyOf func(*V) string, + timeOf func(*V) time.Time, +) { + all := t.All() + if len(all) <= keep { + return + } + + type kv struct { + t time.Time + k string + } + + entries := make([]kv, 0, len(all)) + for _, v := range all { + entries = append(entries, kv{timeOf(v), keyOf(v)}) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].t.Before(entries[j].t) }) + + for i := range len(all) - keep { + t.Delete(entries[i].k) + } +} diff --git a/services/dynamodb/streams_ops.go b/services/dynamodb/streams_ops.go index 1cd7cc9fb..d38ff3d90 100644 --- a/services/dynamodb/streams_ops.go +++ b/services/dynamodb/streams_ops.go @@ -97,7 +97,6 @@ func (db *InMemoryDB) EnableStream(ctx context.Context, tableName, viewType stri table.StreamViewType = viewType table.StreamCreatedAt = now table.StreamARN = db.buildStreamARNInRegion(tableName, region, now) - newARN := table.StreamARN // Initialize the first shard when enabling streams (clearing any prior shard history). table.streamShards = []StreamShard{ { @@ -109,7 +108,7 @@ func (db *InMemoryDB) EnableStream(ctx context.Context, tableName, viewType stri // Update the reverse index under db.mu (after releasing table lock to preserve lock ordering). db.mu.Lock("EnableStream.streamARNIndex") - db.streamARNIndex[newARN] = table + db.streamARNIndex.Put(table) db.mu.Unlock() return nil @@ -137,7 +136,7 @@ func (db *InMemoryDB) DisableStream(ctx context.Context, tableName string) error // Remove from reverse index under db.mu (after releasing table lock to preserve lock ordering). if oldARN != "" { db.mu.Lock("DisableStream.streamARNIndex") - delete(db.streamARNIndex, oldARN) + db.streamARNIndex.Delete(oldARN) db.mu.Unlock() } @@ -624,9 +623,10 @@ func (db *InMemoryDB) collectEnabledStreams(requestRegion, filterTable string) [ // Snapshot under db.mu (read lock). This avoids holding db.mu while also // acquiring table.mu, which would invert the lock order. db.mu.RLock("ListStreams") - entries := make([]arnEntry, 0, len(db.streamARNIndex)) - for a, t := range db.streamARNIndex { - entries = append(entries, arnEntry{table: t, arn: a}) + all := db.streamARNIndex.All() + entries := make([]arnEntry, 0, len(all)) + for _, t := range all { + entries = append(entries, arnEntry{table: t, arn: t.StreamARN}) } db.mu.RUnlock() @@ -942,7 +942,7 @@ func toByteSliceSliceFrom(v any) ([][]byte, error) { // findTableByStreamARN looks up a table by stream ARN using the reverse index. // Must be called with db.mu held. func (db *InMemoryDB) findTableByStreamARN(streamARN string) *Table { - if t, ok := db.streamARNIndex[streamARN]; ok { + if t, ok := db.streamARNIndex.Get(streamARN); ok { return t } diff --git a/services/dynamodb/table_ops.go b/services/dynamodb/table_ops.go index 22d40059c..a3d23be21 100644 --- a/services/dynamodb/table_ops.go +++ b/services/dynamodb/table_ops.go @@ -168,11 +168,7 @@ func (db *InMemoryDB) CreateTable( // Minimal critical section: check for duplicate, then insert into the shared maps. db.mu.Lock("CreateTable") - if _, exists := db.Tables[region]; !exists { - db.Tables[region] = make(map[string]*Table) - } - - if _, exists := db.Tables[region][tableName]; exists { + if _, exists := db.tables.Get(tableKey(region, tableName)); exists { db.mu.Unlock() // Release resources we allocated before discovering the duplicate. stopTableTimers(newTable) @@ -181,11 +177,11 @@ func (db *InMemoryDB) CreateTable( return nil, NewResourceInUseException("table already exists: " + tableName) } - db.Tables[region][tableName] = newTable + db.tables.Put(newTable) newTable.kinesisEmitter = db.kinesisEmitter if newTable.StreamARN != "" { - db.streamARNIndex[newTable.StreamARN] = newTable + db.streamARNIndex.Put(newTable) } db.mu.Unlock() @@ -390,12 +386,7 @@ func (db *InMemoryDB) DeleteTable( db.mu.Lock("DeleteTable") defer db.mu.Unlock() - regionTables, regionExists := db.Tables[region] - if !regionExists { - return nil, NewResourceNotFoundException("table not found: " + tableName) - } - - table, tableExists := regionTables[tableName] + table, tableExists := db.tables.Get(tableKey(region, tableName)) if !tableExists { return nil, NewResourceNotFoundException("table not found: " + tableName) } @@ -417,11 +408,8 @@ func (db *InMemoryDB) DeleteTable( // this is benign; the janitor will clean it up regardless. stopTableTimers(table) - delete(db.Tables[region], tableName) - if _, deletingExists := db.deletingTables[region]; !deletingExists { - db.deletingTables[region] = make(map[string]*Table) - } - db.deletingTables[region][tableName] = table + db.tables.Delete(tableKey(region, tableName)) + db.deletingTables.Put(table) db.throttler.DeleteTable(throttleKey(region, tableName)) // Remove the deleted table's region from its global table's ReplicationGroup. @@ -432,7 +420,7 @@ func (db *InMemoryDB) DeleteTable( // Remove from stream ARN reverse index. if table.StreamARN != "" { - delete(db.streamARNIndex, table.StreamARN) + db.streamARNIndex.Delete(table.StreamARN) } // Capture state for return @@ -480,7 +468,7 @@ func (db *InMemoryDB) DeleteTable( // If no replicas remain after removal, the global table entry itself is deleted. // Must be called with db.mu held for writing. func (db *InMemoryDB) removeGlobalTableReplicaLocked(globalTableName, region string) { - gt, gtExists := db.GlobalTables[globalTableName] + gt, gtExists := db.globalTables.Get(globalTableName) if !gtExists { return } @@ -493,7 +481,7 @@ func (db *InMemoryDB) removeGlobalTableReplicaLocked(globalTableName, region str } if len(remaining) == 0 { - delete(db.GlobalTables, globalTableName) + db.globalTables.Delete(globalTableName) } else { gt.ReplicationGroup = remaining } @@ -561,13 +549,7 @@ func (db *InMemoryDB) DescribeTable( region := getRegionFromContext(ctx, db) db.mu.RLock("DescribeTable") - regionTables, exists := db.Tables[region] - if !exists { - db.mu.RUnlock() - - return nil, NewResourceNotFoundException("table not found: " + tableName) - } - table, exists := regionTables[tableName] + table, exists := db.tables.Get(tableKey(region, tableName)) db.mu.RUnlock() if !exists { @@ -836,10 +818,10 @@ func (db *InMemoryDB) UpdateTable( if oldStreamARN != newStreamARN { db.mu.Lock("UpdateTable.streamARNIndex") if oldStreamARN != "" { - delete(db.streamARNIndex, oldStreamARN) + db.streamARNIndex.Delete(oldStreamARN) } if newStreamARN != "" { - db.streamARNIndex[newStreamARN] = table + db.streamARNIndex.Put(table) } db.mu.Unlock() } @@ -1009,11 +991,7 @@ func (db *InMemoryDB) applyOneReplicaTableEntry( return } - if _, ok := db.Tables[regionName]; !ok { - db.Tables[regionName] = make(map[string]*Table) - } - - if _, exists := db.Tables[regionName][tableName]; !exists { + if existing, exists := db.tables.Get(tableKey(regionName, tableName)); !exists { replica := cloneTableSchema(source, tableName, regionName, db.accountID) replica.GlobalTableName = tableName @@ -1031,9 +1009,9 @@ func (db *InMemoryDB) applyOneReplicaTableEntry( replica.rebuildIndexes() } - db.Tables[regionName][tableName] = replica + db.tables.Put(replica) } else { - db.Tables[regionName][tableName].GlobalTableName = tableName + existing.GlobalTableName = tableName } case update.Delete != nil: @@ -1042,9 +1020,7 @@ func (db *InMemoryDB) applyOneReplicaTableEntry( return } - if regionTables, ok := db.Tables[regionName]; ok { - delete(regionTables, tableName) - } + db.tables.Delete(tableKey(regionName, tableName)) } } @@ -1581,12 +1557,10 @@ func (db *InMemoryDB) ListTables( // Snapshot table names under lock, then release immediately db.mu.RLock("ListTables") - regionTables, exists := db.Tables[region] + regionTables := db.tablesByRegion.Get(region) names := make([]string, 0, len(regionTables)) - if exists { - for name := range regionTables { - names = append(names, name) - } + for _, t := range regionTables { + names = append(names, t.Name) } db.mu.RUnlock() diff --git a/services/dynamodb/tag_ops.go b/services/dynamodb/tag_ops.go index 0d4cc9804..752e147c9 100644 --- a/services/dynamodb/tag_ops.go +++ b/services/dynamodb/tag_ops.go @@ -9,6 +9,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" "github.com/blackbirdworks/gopherstack/pkgs/collections" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) @@ -33,7 +34,7 @@ func (db *InMemoryDB) TagResource( tableName := tableNameFromARN(aws.ToString(input.ResourceArn)) db.mu.RLock("TagResource") - table := findTableByName(db.Tables, tableName) + table := findTableByName(db.tables, tableName) db.mu.RUnlock() if table == nil { @@ -62,7 +63,7 @@ func (db *InMemoryDB) UntagResource( tableName := tableNameFromARN(aws.ToString(input.ResourceArn)) db.mu.RLock("UntagResource") - table := findTableByName(db.Tables, tableName) + table := findTableByName(db.tables, tableName) db.mu.RUnlock() if table == nil { @@ -87,7 +88,7 @@ func (db *InMemoryDB) ListTagsOfResource( tableName := tableNameFromARN(aws.ToString(input.ResourceArn)) db.mu.RLock("ListTagsOfResource") - table := findTableByName(db.Tables, tableName) + table := findTableByName(db.tables, tableName) db.mu.RUnlock() if table == nil { @@ -114,11 +115,14 @@ func (db *InMemoryDB) ListTagsOfResource( return &dynamodb.ListTagsOfResourceOutput{Tags: sdkTags}, nil } -// findTableByName searches all region-keyed table maps for a table with the given name. -// Returns nil if not found. Must be called with db.mu held. -func findTableByName(tables map[string]map[string]*Table, name string) *Table { - for _, regionTables := range tables { - if t, ok := regionTables[name]; ok { +// findTableByName searches all regions for a table with the given name. +// Returns nil if not found. Must be called with db.mu held. If the same +// table name exists in more than one region the result is one of them, +// chosen in unspecified order -- this matches the pre-store behavior, which +// iterated a map[string]map[string]*Table (also unordered). +func findTableByName(tables *store.Table[Table], name string) *Table { + for _, t := range tables.All() { + if t.Name == name { return t } } diff --git a/services/dynamodb/transact_ops.go b/services/dynamodb/transact_ops.go index ab5ee4855..aec77a615 100644 --- a/services/dynamodb/transact_ops.go +++ b/services/dynamodb/transact_ops.go @@ -458,15 +458,15 @@ func (db *InMemoryDB) lockTablesWrite( tables := make(map[string]*Table, len(tableNames)) db.mu.RLock("TransactWriteItems") - regionTables, exists := db.Tables[region] - if !exists { + + if len(db.tablesByRegion.Get(region)) == 0 { db.mu.RUnlock() return nil, NewResourceNotFoundException("Table not found in region " + region) } for _, name := range tableNames { - t, ok := regionTables[name] + t, ok := db.tables.Get(tableKey(region, name)) if !ok { db.mu.RUnlock() @@ -491,15 +491,15 @@ func (db *InMemoryDB) lockTablesRead( tables := make(map[string]*Table, len(tableNames)) db.mu.RLock("TransactGetItems") - regionTables, exists := db.Tables[region] - if !exists { + + if len(db.tablesByRegion.Get(region)) == 0 { db.mu.RUnlock() return nil, NewResourceNotFoundException("Table not found in region " + region) } for _, name := range tableNames { - t, ok := regionTables[name] + t, ok := db.tables.Get(tableKey(region, name)) if !ok { db.mu.RUnlock() diff --git a/services/dynamodb/transact_ops_test.go b/services/dynamodb/transact_ops_test.go index f0eb773fd..163fbfc4f 100644 --- a/services/dynamodb/transact_ops_test.go +++ b/services/dynamodb/transact_ops_test.go @@ -430,3 +430,54 @@ func TestTransactWriteItems_TokenNotCommittedOnFailure(t *testing.T) { _, err = db.TransactWriteItems(t.Context(), input) require.Error(t, err, "second call with uncommitted token should also fail") } + +// TestTransactWriteItems_Update_RejectsKeyModification verifies that a +// TransactWriteItems Update action is rejected when its UpdateExpression +// touches a key attribute, matching the restriction plain UpdateItem already +// enforces. Before this validation was added, such an update would silently +// rewrite the item's key in place while leaving a stale entry in the +// PK/PK+SK index (updateIndexes only ever adds/overwrites the new key's +// index slot; it never removes the old one) — corrupting subsequent lookups +// by the original key. +func TestTransactWriteItems_Update_RejectsKeyModification(t *testing.T) { + t.Parallel() + + const tbl = "TxUpdateKeyTable" + db := newTransactDB(t, tbl) + seedItem(t, db, tbl, "v1") + + _, err := db.TransactWriteItems(t.Context(), &sdk.TransactWriteItemsInput{ + TransactItems: []types.TransactWriteItem{ + { + Update: &types.Update{ + TableName: aws.String(tbl), + Key: map[string]types.AttributeValue{ + "pk": &types.AttributeValueMemberS{Value: "item1"}, + }, + UpdateExpression: aws.String("SET pk = :newpk"), + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":newpk": &types.AttributeValueMemberS{Value: "item2"}, + }, + }, + }, + }, + }) + require.Error(t, err, "TransactWriteItems must reject an Update that modifies a key attribute") + assert.Contains(t, err.Error(), "ValidationException") + + // Confirm no corruption occurred: the original item must still be reachable + // by its original key, and no phantom item under the new key was created. + out, getErr := db.GetItem(t.Context(), &sdk.GetItemInput{ + TableName: aws.String(tbl), + Key: map[string]types.AttributeValue{"pk": &types.AttributeValueMemberS{Value: "item1"}}, + }) + require.NoError(t, getErr) + assert.NotEmpty(t, out.Item, "original item under item1 must still exist") + + outNew, getErr := db.GetItem(t.Context(), &sdk.GetItemInput{ + TableName: aws.String(tbl), + Key: map[string]types.AttributeValue{"pk": &types.AttributeValueMemberS{Value: "item2"}}, + }) + require.NoError(t, getErr) + assert.Empty(t, outNew.Item, "rejected transaction must not create an item under item2") +} diff --git a/services/dynamodb/utils_test.go b/services/dynamodb/utils_test.go index a2d44ac66..58a1ec46c 100644 --- a/services/dynamodb/utils_test.go +++ b/services/dynamodb/utils_test.go @@ -39,7 +39,7 @@ func TestExtractKeySchema(t *testing.T) { }, check: func(t *testing.T, db *dynamodb.InMemoryDB, tableName string) { t.Helper() - table := db.Tables["us-east-1"][tableName] + table, _ := db.GetTableInRegion(tableName, "us-east-1") schema, idx, err := db.ExtractKeySchema(table, "") require.NoError(t, err) assert.Nil(t, idx) @@ -78,7 +78,7 @@ func TestExtractKeySchema(t *testing.T) { }, check: func(t *testing.T, db *dynamodb.InMemoryDB, tableName string) { t.Helper() - table := db.Tables["us-east-1"][tableName] + table, _ := db.GetTableInRegion(tableName, "us-east-1") schema, idx, err := db.ExtractKeySchema(table, "GSI1") require.NoError(t, err) require.NotNil(t, idx) @@ -103,7 +103,7 @@ func TestExtractKeySchema(t *testing.T) { }, check: func(t *testing.T, db *dynamodb.InMemoryDB, tableName string) { t.Helper() - table := db.Tables["us-east-1"][tableName] + table, _ := db.GetTableInRegion(tableName, "us-east-1") _, _, err := db.ExtractKeySchema(table, "InvalidIndex") require.Error(t, err) assert.Contains(t, err.Error(), "not found") @@ -360,7 +360,7 @@ func TestRebuildIndexes(t *testing.T) { _, err := db.CreateTable(t.Context(), models.ToSDKCreateTableInput(&ctInput)) require.NoError(t, err) - table := db.Tables["us-east-1"][tableName] + table, _ := db.GetTableInRegion(tableName, "us-east-1") table.InitializeIndexes() assert.Empty(t, table.PKIndex()) diff --git a/services/dynamodb/validation.go b/services/dynamodb/validation.go index 6275391bf..42902ed50 100644 --- a/services/dynamodb/validation.go +++ b/services/dynamodb/validation.go @@ -10,8 +10,17 @@ import ( "github.com/blackbirdworks/gopherstack/services/dynamodb/models" ) -// validateSelectConstraints enforces constraints on the Select parameter based on the index projection. -func validateSelectConstraints(selectVal types.Select, indexName string, projection *models.Projection) error { +// validateSelectConstraints enforces constraints on the Select parameter: its +// interaction with index projections, with ProjectionExpression/AttributesToGet, +// and its ALL_PROJECTED_ATTRIBUTES/SPECIFIC_ATTRIBUTES-only restrictions. Mirrors +// AWS's real Query/Scan validation (see API_Query.html "Select" parameter docs). +func validateSelectConstraints( + selectVal types.Select, + indexName string, + projection *models.Projection, + projectionExpr string, + attributesToGet []string, +) error { if selectVal == types.SelectAllAttributes && indexName != "" { if projection != nil && projection.ProjectionType != string(types.ProjectionTypeAll) { return NewValidationException( @@ -22,6 +31,34 @@ func validateSelectConstraints(selectVal types.Select, indexName string, project } } + // ALL_PROJECTED_ATTRIBUTES is only meaningful when querying/scanning an index. + if selectVal == types.SelectAllProjectedAttributes && indexName == "" { + return NewValidationException( + "One or more parameter values were invalid: Select type ALL_PROJECTED_ATTRIBUTES " + + "is not supported for a table", + ) + } + + requestsProjection := projectionExpr != "" || len(attributesToGet) > 0 + + // AWS: "If you use the ProjectionExpression parameter, then the value for + // Select can only be SPECIFIC_ATTRIBUTES. Any other value for Select will + // return an error." The same restriction applies to the legacy AttributesToGet. + if selectVal != "" && selectVal != types.SelectSpecificAttributes && requestsProjection { + return NewValidationException( + "Cannot specify ProjectionExpression when Select is not SPECIFIC_ATTRIBUTES", + ) + } + + // AWS requires ProjectionExpression or AttributesToGet when Select is + // explicitly SPECIFIC_ATTRIBUTES. + if selectVal == types.SelectSpecificAttributes && !requestsProjection { + return NewValidationException( + "One or more parameter values were invalid: Select type SPECIFIC_ATTRIBUTES " + + "is not supported without a ProjectionExpression or AttributesToGet", + ) + } + return nil } diff --git a/services/ec2/PARITY.md b/services/ec2/PARITY.md new file mode 100644 index 000000000..3ebd9db06 --- /dev/null +++ b/services/ec2/PARITY.md @@ -0,0 +1,25 @@ +--- +service: ec2 +sdk_module: aws-sdk-go-v2/service/ec2 # version: see go.mod (backfilled) +last_audit_commit: c18fa9b1 +last_audit_date: 2026-07-04 +overall: A # 672 LOC genuine fixes; ec2 is ~96k LOC/~180 maps — audit was SCOPED, not exhaustive +protocol: ec2-query (AWS query -> XML) +families: + tags: {status: ok, note: FIXED — existence/type table covered only 9 of ~100 taggable types; now full (AMIs/snapshots/NACLs/TGW/VPN/endpoints/launch-templates/IPAM...). See backend_resource_types.go} + instance_attrs:{status: ok, note: FIXED disguised stub — disableApiTermination/Stop, ebsOptimized, sourceDestCheck, instanceInitiatedShutdownBehavior were accepted-but-not-modeled; now persisted (sourceDestCheck on primary ENI). DescribeInstanceAttribute was hardcoded} + instance_lifecycle: {status: ok, note: FIXED StateReason/StateTransitionReason wire shape (were absent); disableApiTermination/Stop now enforced (OperationNotPermitted); RunInstances launch attrs applied} + sg_rules: {status: ok, note: PROVEN correct — protocol/port/ICMP bounds, CIDR, dup detection match AWS} + filters: {status: ok, note: PROVEN — AND-across-names/OR-within-values, tag: prefix} +gaps: + - DeleteVpc/DeleteSubnet force-cascade-delete instead of DependencyViolation (bd: ec2 DeleteVpc follow-up) +deferred: + - VPC/RouteTable/IGW/NAT/TGW/VPCEndpoint (batch-4) op-by-op — flagged higher-defect-odds, UNAUDITED + - EBS snapshot lineage, ENI attach/detach edge cases, pagination internals beyond tags/instances +leaks: {status: not-fully-audited, note: instance/tag paths clean; full janitor/goroutine audit pending with deferred families} +--- + +## Notes +- sourceDestCheck AWS default is **true** for VPC instances (must be explicitly disabled, e.g. NAT instances) — a prior test encoded false, corrected. +- kernel/ramdisk return "" for HVM (not "stop"). +- NEXT PASS priority: VPC/TGW/Endpoint family sweep + DeleteVpc DependencyViolation. diff --git a/services/ec2/backend.go b/services/ec2/backend.go index d0aabd8c5..be40673d4 100644 --- a/services/ec2/backend.go +++ b/services/ec2/backend.go @@ -13,6 +13,7 @@ import ( "github.com/google/uuid" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) // Errors returned by the EC2 backend. @@ -55,6 +56,10 @@ var ( // ErrInvalidPaginationToken is returned when a NextToken is forged, tampered // with, or otherwise fails HMAC verification. ErrInvalidPaginationToken = errors.New("InvalidPaginationToken") + // ErrOperationNotPermitted is returned when an operation is blocked by an + // instance-level API protection attribute (disableApiTermination / + // disableApiStop), mirroring AWS's OperationNotPermitted error code. + ErrOperationNotPermitted = errors.New("OperationNotPermitted") ) // EC2 instance state codes as defined by the AWS EC2 API. @@ -109,33 +114,48 @@ var ( // Instance represents an EC2 instance (metadata only, no actual compute). type Instance struct { - LaunchTime time.Time `json:"launchTime"` - TerminatedAt time.Time `json:"terminatedAt"` - EventStartTimeOverrides map[string]time.Time `json:"eventStartTimeOverrides,omitempty"` - Placement InstancePlacement `json:"placement"` - CapacityReservationSpec CapacityReservationSpec `json:"capacityReservationSpecification"` - MaintenanceOptions InstanceMaintenanceOptions `json:"maintenanceOptions"` - CPUOptions CPUOptions `json:"cpuOptions"` - MetadataOptionsState string `json:"metadataOptionsState,omitempty"` - VPCID string `json:"vpcID,omitempty"` - ID string `json:"id,omitempty"` - PrivateIP string `json:"privateIP,omitempty"` - PublicIPAddress string `json:"publicIPAddress,omitempty"` - SubnetID string `json:"subnetID,omitempty"` - UserData string `json:"userData,omitempty"` - SriovNetSupport string `json:"sriovNetSupport,omitempty"` - ProviderID string `json:"providerID,omitempty"` - PublicDNSName string `json:"publicDNSName,omitempty"` + TerminatedAt time.Time `json:"terminatedAt"` + LaunchTime time.Time `json:"launchTime"` + EventStartTimeOverrides map[string]time.Time `json:"eventStartTimeOverrides,omitempty"` + CapacityReservationSpec CapacityReservationSpec `json:"capacityReservationSpecification"` + MaintenanceOptions InstanceMaintenanceOptions `json:"maintenanceOptions"` + PublicDNSName string `json:"publicDNSName,omitempty"` + MetadataOptionsTokens string `json:"metadataOptionsTokens,omitempty"` + MetadataOptionsState string `json:"metadataOptionsState,omitempty"` + VPCID string `json:"vpcID,omitempty"` + ID string `json:"id,omitempty"` + PrivateIP string `json:"privateIP,omitempty"` + PublicIPAddress string `json:"publicIPAddress,omitempty"` + SubnetID string `json:"subnetID,omitempty"` + UserData string `json:"userData,omitempty"` + SriovNetSupport string `json:"sriovNetSupport,omitempty"` + ProviderID string `json:"providerID,omitempty"` + // InstanceInitiatedShutdownBehavior is "stop" (default) or "terminate". + InstanceInitiatedShutdownBehavior string `json:"instanceInitiatedShutdownBehavior,omitempty"` + // NetworkPerformanceOptions carries the bandwidth-weighting mode. NetworkPerformanceOptions InstanceNetworkPerformanceOptions `json:"networkPerformanceOptions"` KeyName string `json:"keyName,omitempty"` InstanceType string `json:"instanceType,omitempty"` - MetadataOptionsTokens string `json:"metadataOptionsTokens,omitempty"` - ImageID string `json:"imageID,omitempty"` - PrivateDNSNameOptions PrivateDNSNameOptions `json:"privateDnsNameOptions"` - State InstanceState `json:"state"` - SecurityGroups []string `json:"securityGroups,omitempty"` - SSHPort int `json:"sshPort,omitempty"` - EnaSupport bool `json:"enaSupport,omitempty"` + // StateTransitionReason mirrors AWS's legacy element (e.g. + // "User initiated (2016-05-...)"). + StateTransitionReason string `json:"stateTransitionReason,omitempty"` + ImageID string `json:"imageID,omitempty"` + // StateReasonCode/StateReasonMessage mirror AWS's element, + // populated on user-initiated stop/terminate and cleared on start. + StateReasonMessage string `json:"stateReasonMessage,omitempty"` + StateReasonCode string `json:"stateReasonCode,omitempty"` + Placement InstancePlacement `json:"placement"` + SecurityGroups []string `json:"securityGroups,omitempty"` + State InstanceState `json:"state"` + PrivateDNSNameOptions PrivateDNSNameOptions `json:"privateDnsNameOptions"` + CPUOptions CPUOptions `json:"cpuOptions"` + SSHPort int `json:"sshPort,omitempty"` + EnaSupport bool `json:"enaSupport,omitempty"` + // DisableAPITermination / DisableAPIStop gate TerminateInstances / + // StopInstances respectively (ModifyInstanceAttribute-settable). + DisableAPITermination bool `json:"disableApiTermination,omitempty"` + DisableAPIStop bool `json:"disableApiStop,omitempty"` + EBSOptimized bool `json:"ebsOptimized,omitempty"` } // LaunchTemplate represents an EC2 launch template. @@ -238,92 +258,92 @@ type InMemoryBackend struct { compute Compute dnsRegistrar DNSRegistrar addressTransfers map[string]*AddressTransfer - capacityReservations map[string]*CapacityReservation - vpcs map[string]*VPC - subnets map[string]*Subnet - keyPairs map[string]*KeyPair - reservedInstancesExchanges map[string]*ReservedInstancesExchange - addresses map[string]*Address - internetGateways map[string]*InternetGateway - natGateways map[string]*NatGateway - routeTables map[string]*RouteTable - placementGroups map[string]*PlacementGroup - spotRequests map[string]*SpotInstanceRequest - instances map[string]*Instance - images map[string]*AMIStub - imageUsageReports map[string]*ImageUsageReport - launchTemplates map[string]*LaunchTemplate - vpcEndpoints map[string]*VpcEndpoint + capacityReservations *store.Table[CapacityReservation] + vpcs *store.Table[VPC] + subnets *store.Table[Subnet] + keyPairs *store.Table[KeyPair] + reservedInstancesExchanges *store.Table[ReservedInstancesExchange] + addresses *store.Table[Address] + internetGateways *store.Table[InternetGateway] + natGateways *store.Table[NatGateway] + routeTables *store.Table[RouteTable] + placementGroups *store.Table[PlacementGroup] + spotRequests *store.Table[SpotInstanceRequest] + instances *store.Table[Instance] + images *store.Table[AMIStub] + imageUsageReports *store.Table[ImageUsageReport] + launchTemplates *store.Table[LaunchTemplate] + vpcEndpoints *store.Table[VpcEndpoint] tags map[string]map[string]string - securityGroups map[string]*SecurityGroup - networkInterfaces map[string]*NetworkInterface - volumes map[string]*Volume - tgwMulticastDomainAssociations map[string]*TransitGatewayMulticastDomainAssociation - tgwPeeringAttachments map[string]*TransitGatewayPeeringAttachment - tgwVpcAttachments map[string]*TransitGatewayVpcAttachment - vpcEndpointConnections map[string]*VpcEndpointConnection - vpcPeeringConnections map[string]*VpcPeeringConnection - byoipCidrs map[string]*ByoipCidr - dedicatedHosts map[string]*Host - snapshots map[string]*Snapshot - networkACLs map[string]*StoredNetworkACL - transitGateways map[string]*TransitGateway - flowLogs map[string]*FlowLog - dhcpOptionSets map[string]*DhcpOptions - egressOnlyIGWs map[string]*EgressOnlyInternetGateway - iamAssociations map[string]*IamInstanceProfileAssociation - tgwRouteTables map[string]*TransitGatewayRouteTable - tgwRoutes map[string]*TransitGatewayRoute - tgwRTAssociations map[string]*TransitGatewayRouteTableAssociation - tgwPolicyTables map[string]*TransitGatewayPolicyTable - tgwPolicyTableAssociations map[string]*TransitGatewayPolicyTableAssociation - tgwRouteTableAnnouncements map[string]*TransitGatewayRouteTableAnnouncement + securityGroups *store.Table[SecurityGroup] + networkInterfaces *store.Table[NetworkInterface] + volumes *store.Table[Volume] + tgwMulticastDomainAssociations *store.Table[TransitGatewayMulticastDomainAssociation] + tgwPeeringAttachments *store.Table[TransitGatewayPeeringAttachment] + tgwVpcAttachments *store.Table[TransitGatewayVpcAttachment] + vpcEndpointConnections *store.Table[VpcEndpointConnection] + vpcPeeringConnections *store.Table[VpcPeeringConnection] + byoipCidrs *store.Table[ByoipCidr] + dedicatedHosts *store.Table[Host] + snapshots *store.Table[Snapshot] + networkACLs *store.Table[StoredNetworkACL] + transitGateways *store.Table[TransitGateway] + flowLogs *store.Table[FlowLog] + dhcpOptionSets *store.Table[DhcpOptions] + egressOnlyIGWs *store.Table[EgressOnlyInternetGateway] + iamAssociations *store.Table[IamInstanceProfileAssociation] + tgwRouteTables *store.Table[TransitGatewayRouteTable] + tgwRoutes *store.Table[TransitGatewayRoute] + tgwRTAssociations *store.Table[TransitGatewayRouteTableAssociation] + tgwPolicyTables *store.Table[TransitGatewayPolicyTable] + tgwPolicyTableAssociations *store.Table[TransitGatewayPolicyTableAssociation] + tgwRouteTableAnnouncements *store.Table[TransitGatewayRouteTableAnnouncement] vpcCidrAssociations map[string]*VpcCidrBlockAssociation - vpnGateways map[string]*VpnGateway - customerGateways map[string]*CustomerGateway - vpnConnections map[string]*VpnConnection - vpcEndpointServiceConfigs map[string]*VpcEndpointServiceConfig - ipams map[string]*Ipam - ipamScopes map[string]*IpamScope - ipamPools map[string]*IpamPool + vpnGateways *store.Table[VpnGateway] + customerGateways *store.Table[CustomerGateway] + vpnConnections *store.Table[VpnConnection] + vpcEndpointServiceConfigs *store.Table[VpcEndpointServiceConfig] + ipams *store.Table[Ipam] + ipamScopes *store.Table[IpamScope] + ipamPools *store.Table[IpamPool] ipamPoolCidrs map[string][]*IpamPoolCidr - ipamPoolAllocations map[string]*IpamPoolAllocation - ipamResourceDiscoveries map[string]*IpamResourceDiscovery - ipamResourceDiscoveryAssocs map[string]*IpamResourceDiscoveryAssociation - ipamByoasns map[string]*IpamByoasn - ipamAsnAssociations map[string]*IpamAsnAssociation - ipamVerificationTokens map[string]*IpamExternalResourceVerificationToken - ipamResourceCidrs map[string]*IpamResourceCidr - ipamPrefixListResolvers map[string]*IpamPrefixListResolver + ipamPoolAllocations *store.Table[IpamPoolAllocation] + ipamResourceDiscoveries *store.Table[IpamResourceDiscovery] + ipamResourceDiscoveryAssocs *store.Table[IpamResourceDiscoveryAssociation] + ipamByoasns *store.Table[IpamByoasn] + ipamAsnAssociations *store.Table[IpamAsnAssociation] + ipamVerificationTokens *store.Table[IpamExternalResourceVerificationToken] + ipamResourceCidrs *store.Table[IpamResourceCidr] + ipamPrefixListResolvers *store.Table[IpamPrefixListResolver] ipamPrefixListResolverVersions map[string][]int64 - ipamPrefixListResolverTargets map[string]*IpamPrefixListResolverTarget - ipamPolicies map[string]*IpamPolicy + ipamPrefixListResolverTargets *store.Table[IpamPrefixListResolverTarget] + ipamPolicies *store.Table[IpamPolicy] ipamPolicyEnabledTargets map[string]string ipamOrgAdminAccountID string - spotFleets map[string]*SpotFleetRequest + spotFleets *store.Table[SpotFleetRequest] spotFleetHistory map[string][]SpotFleetHistoryRecord // batch1 additions - volumeModifications map[string]*VolumeModification + volumeModifications *store.Table[VolumeModification] snapshotTiers map[string]string snapshotAttributes map[string]map[string]string sgVpcAssociations map[string]map[string]string vpcTenancy map[string]string vpcPeeringOptions map[string]*PeeringConnectionOptions subnetCIDRAssociations map[string][]*SubnetCIDRAssociation - addressAttributes map[string]*AddressAttribute + addressAttributes *store.Table[AddressAttribute] instanceMonitoring map[string]string instanceCreditSpecs map[string]string instanceIMDSOptions map[string]*IMDSOptions instanceMetadataDefaults *InstanceMetadataDefaults instanceEventNotifAttrs *InstanceEventNotificationAttributes - niPermissions map[string]*NetworkInterfacePermission + niPermissions *store.Table[NetworkInterfacePermission] niIPv6Addresses map[string][]string idFormatSettings map[string]bool // batch2 additions - endpointConnectionNotifs map[string]*VpcEndpointConnectionNotification + endpointConnectionNotifs *store.Table[VpcEndpointConnectionNotification] vpcEndpointServicePermissions map[string][]string - snapshotLocks map[string]*SnapshotLock - replaceRootVolumeTasks map[string]*ReplaceRootVolumeTask + snapshotLocks *store.Table[SnapshotLock] + replaceRootVolumeTasks *store.Table[ReplaceRootVolumeTask] subnetCIDRReservations map[string][]*SubnetCIDRReservation imageDisabled map[string]bool imageDeprecated map[string]string @@ -331,126 +351,130 @@ type InMemoryBackend struct { imageAttributes map[string]map[string]string vgwRoutePropagation map[string]bool // batch4 additions - managedPrefixLists map[string]*ManagedPrefixList - clientVpnEndpoints map[string]*ClientVpnEndpoint - tgwConnects map[string]*TransitGatewayConnect - tgwConnectPeers map[string]*TransitGatewayConnectPeer - tgwPrefixListRefs map[string]*TransitGatewayPrefixListReference - verifiedAccessEndpoints map[string]*VerifiedAccessEndpoint - verifiedAccessGroups map[string]*VerifiedAccessGroup - verifiedAccessInstances map[string]*VerifiedAccessInstance - verifiedAccessTrustProviders map[string]*VerifiedAccessTrustProvider + managedPrefixLists *store.Table[ManagedPrefixList] + clientVpnEndpoints *store.Table[ClientVpnEndpoint] + tgwConnects *store.Table[TransitGatewayConnect] + tgwConnectPeers *store.Table[TransitGatewayConnectPeer] + tgwPrefixListRefs *store.Table[TransitGatewayPrefixListReference] + verifiedAccessEndpoints *store.Table[VerifiedAccessEndpoint] + verifiedAccessGroups *store.Table[VerifiedAccessGroup] + verifiedAccessInstances *store.Table[VerifiedAccessInstance] + verifiedAccessTrustProviders *store.Table[VerifiedAccessTrustProvider] // batch3 additions - instanceConnectEndpoints map[string]*InstanceConnectEndpoint - instanceEventWindows map[string]*InstanceEventWindow - imageImportTasks map[string]*ImageImportTask - snapshotImportTasks map[string]*SnapshotImportTask - recycleBinImages map[string]*RecycleBinImage - recycleBinSnapshots map[string]*Snapshot - recycleBinVolumes map[string]*RecycleBinVolume + instanceConnectEndpoints *store.Table[InstanceConnectEndpoint] + instanceEventWindows *store.Table[InstanceEventWindow] + imageImportTasks *store.Table[ImageImportTask] + snapshotImportTasks *store.Table[SnapshotImportTask] + recycleBinImages *store.Table[RecycleBinImage] + recycleBinSnapshots *store.Table[Snapshot] + recycleBinVolumes *store.Table[RecycleBinVolume] fastLaunchImages map[string]bool fastSnapshotRestores map[string]bool - vpnConnectionRoutes map[string]*VpnConnectionRoute + vpnConnectionRoutes *store.Table[VpnConnectionRoute] spotDatafeed *SpotDatafeed // batch5 additions - trafficMirrorFilters map[string]*TrafficMirrorFilter - trafficMirrorFilterRules map[string]*TrafficMirrorFilterRule - trafficMirrorSessions map[string]*TrafficMirrorSession - trafficMirrorTargets map[string]*TrafficMirrorTarget - fleets map[string]*Fleet - networkInsightsPaths map[string]*NetworkInsightsPath - networkInsightsAnalyses map[string]*NetworkInsightsAnalysis - networkInsightsAccessScopes map[string]*NetworkInsightsAccessScope - networkInsightsAccessScopeAnalyses map[string]*NetworkInsightsAccessScopeAnalysis - carrierGateways map[string]*CarrierGateway - reservedInstances map[string]*ReservedInstance - reservedInstancesOfferings map[string]*ReservedInstancesOffering - reservedInstancesListings map[string]*ReservedInstancesListing - reservedInstancesModifications map[string]*ReservedInstancesModification + trafficMirrorFilters *store.Table[TrafficMirrorFilter] + trafficMirrorFilterRules *store.Table[TrafficMirrorFilterRule] + trafficMirrorSessions *store.Table[TrafficMirrorSession] + trafficMirrorTargets *store.Table[TrafficMirrorTarget] + fleets *store.Table[Fleet] + networkInsightsPaths *store.Table[NetworkInsightsPath] + networkInsightsAnalyses *store.Table[NetworkInsightsAnalysis] + networkInsightsAccessScopes *store.Table[NetworkInsightsAccessScope] + networkInsightsAccessScopeAnalyses *store.Table[NetworkInsightsAccessScopeAnalysis] + carrierGateways *store.Table[CarrierGateway] + reservedInstances *store.Table[ReservedInstance] + reservedInstancesOfferings *store.Table[ReservedInstancesOffering] + reservedInstancesListings *store.Table[ReservedInstancesListing] + reservedInstancesModifications *store.Table[ReservedInstancesModification] // route server additions - routeServers map[string]*RouteServer - routeServerEndpoints map[string]*RouteServerEndpoint - routeServerPeers map[string]*RouteServerPeer - routeServerAssociations map[string]*RouteServerAssociation - routeServerPropagations map[string]*RouteServerPropagation + routeServers *store.Table[RouteServer] + routeServerEndpoints *store.Table[RouteServerEndpoint] + routeServerPeers *store.Table[RouteServerPeer] + routeServerAssociations *store.Table[RouteServerAssociation] + routeServerPropagations *store.Table[RouteServerPropagation] // local gateway additions - localGateways map[string]*LocalGateway - localGatewayVirtualInterfaces map[string]*LocalGatewayVirtualInterface - localGatewayVirtualInterfaceGroups map[string]*LocalGatewayVirtualInterfaceGroup - localGatewayRouteTables map[string]*LocalGatewayRouteTable - localGatewayRoutes map[string]*LocalGatewayRoute - localGatewayRouteTableVpcAssociations map[string]*LocalGatewayRouteTableVpcAssociation - localGatewayRouteTableVifGroupAssociations map[string]*LocalGatewayRouteTableVirtualInterfaceGroupAssociation + localGateways *store.Table[LocalGateway] + localGatewayVirtualInterfaces *store.Table[LocalGatewayVirtualInterface] + localGatewayVirtualInterfaceGroups *store.Table[LocalGatewayVirtualInterfaceGroup] + localGatewayRouteTables *store.Table[LocalGatewayRouteTable] + localGatewayRoutes *store.Table[LocalGatewayRoute] + localGatewayRouteTableVpcAssociations *store.Table[LocalGatewayRouteTableVpcAssociation] + localGatewayRouteTableVifGroupAssociations *store.Table[LocalGatewayRouteTableVirtualInterfaceGroupAssociation] // transit gateway multicast domain / metering policy additions - tgwMulticastDomains map[string]*TransitGatewayMulticastDomain - tgwMulticastGroupEntries map[string]*TransitGatewayMulticastGroupEntry - tgwMeteringPolicies map[string]*TransitGatewayMeteringPolicy - tgwMeteringPolicyEntries map[string]*TransitGatewayMeteringPolicyEntry + tgwMulticastDomains *store.Table[TransitGatewayMulticastDomain] + tgwMulticastGroupEntries *store.Table[TransitGatewayMulticastGroupEntry] + tgwMeteringPolicies *store.Table[TransitGatewayMeteringPolicy] + tgwMeteringPolicyEntries *store.Table[TransitGatewayMeteringPolicyEntry] // VPC ClassicLink / Block Public Access additions - classicLinkInstances map[string]*ClassicLinkInstance + classicLinkInstances *store.Table[ClassicLinkInstance] vpcBlockPublicAccessOptions *VpcBlockPublicAccessOptions - vpcBlockPublicAccessExclusions map[string]*VpcBlockPublicAccessExclusion + vpcBlockPublicAccessExclusions *store.Table[VpcBlockPublicAccessExclusion] // Capacity Reservation Fleet / Capacity Block / Capacity Manager additions - capacityReservationFleets map[string]*CapacityReservationFleet - capacityBlockOfferings map[string]*CapacityBlockOffering - capacityBlockExtensionOfferings map[string]*CapacityBlockExtensionOffering - capacityBlocks map[string]*CapacityBlock - capacityBlockExtensions map[string]*CapacityBlockExtension - capacityReservationBillingRequests map[string]*CapacityReservationBillingRequest - capacityManagerDataExports map[string]*CapacityManagerDataExport + capacityReservationFleets *store.Table[CapacityReservationFleet] + capacityBlockOfferings *store.Table[CapacityBlockOffering] + capacityBlockExtensionOfferings *store.Table[CapacityBlockExtensionOffering] + capacityBlocks *store.Table[CapacityBlock] + capacityBlockExtensions *store.Table[CapacityBlockExtension] + capacityReservationBillingRequests *store.Table[CapacityReservationBillingRequest] + capacityManagerDataExports *store.Table[CapacityManagerDataExport] capacityManagerState *CapacityManagerState // VerifiedAccess policy / logging additions verifiedAccessEndpointPolicies map[string]*VerifiedAccessPolicy verifiedAccessGroupPolicies map[string]*VerifiedAccessPolicy - verifiedAccessInstanceLoggingConfigs map[string]*VerifiedAccessInstanceLoggingConfig + verifiedAccessInstanceLoggingConfigs *store.Table[VerifiedAccessInstanceLoggingConfig] // FPGA image additions - fpgaImages map[string]*FpgaImage + fpgaImages *store.Table[FpgaImage] // Scheduled Instances additions - scheduledInstances map[string]*ScheduledInstance + scheduledInstances *store.Table[ScheduledInstance] scheduledInstanceLaunched map[string]int32 // COIP / Public IPv4 / IPv6 pool additions - coipPools map[string]*CoipPool - coipCidrs map[string]*CoipCidr - ipv4Pools map[string]*Ipv4Pool - ipv6Pools map[string]*Ipv6Pool + coipPools *store.Table[CoipPool] + coipCidrs *store.Table[CoipCidr] + ipv4Pools *store.Table[Ipv4Pool] + ipv6Pools *store.Table[Ipv6Pool] // VM Import/Export, Bundle, and Conversion Task additions - bundleTasks map[string]*BundleTask - conversionTasks map[string]*ConversionTask - exportTasks map[string]*ExportTask - exportImageTasks map[string]*ExportImageTaskRec + bundleTasks *store.Table[BundleTask] + conversionTasks *store.Table[ConversionTask] + exportTasks *store.Table[ExportTask] + exportImageTasks *store.Table[ExportImageTaskRec] // Trunk Interface / Enclave Certificate IAM Role additions - trunkInterfaceAssociations map[string]*TrunkInterfaceAssociation + trunkInterfaceAssociations *store.Table[TrunkInterfaceAssociation] enclaveCertIamRoles map[string][]*EnclaveCertIamRoleAssociation // Allowed Images Settings / Store-Restore Image Task / Image Usage Report additions allowedImagesSettings *AllowedImagesSettings - storeImageTasks map[string]*StoreImageTask - usageReports map[string]*UsageReport + storeImageTasks *store.Table[StoreImageTask] + usageReports *store.Table[UsageReport] usageReportEntries map[string][]*UsageReportEntry instanceProductCodes map[string][]string // Mac Host / Mac modification task additions - macModificationTasks map[string]*MacModificationTask + macModificationTasks *store.Table[MacModificationTask] // Secondary Network / Secondary Subnet / Secondary Interface / Outpost LAG / // Service Link Virtual Interface additions - secondaryNetworks map[string]*SecondaryNetwork - secondarySubnets map[string]*SecondarySubnet - secondaryInterfaces map[string]*SecondaryInterface - serviceLinkVirtualInterfaces map[string]*ServiceLinkVirtualInterface - outpostLags map[string]*OutpostLag + secondaryNetworks *store.Table[SecondaryNetwork] + secondarySubnets *store.Table[SecondarySubnet] + secondaryInterfaces *store.Table[SecondaryInterface] + serviceLinkVirtualInterfaces *store.Table[ServiceLinkVirtualInterface] + outpostLags *store.Table[OutpostLag] // Instance-attribute misc cluster additions (AZ group opt-in, SQL HA) availabilityZoneGroupOptIns map[string]string - sqlHaRegistrations map[string]*RegisteredSQLHaInstance + sqlHaRegistrations *store.Table[RegisteredSQLHaInstance] sqlHaHistory map[string][]*RegisteredSQLHaInstance // parity-sweep-2 additions: VPC Encryption Control, VPN Concentrator, Host // Reservations, Declarative Policies, AWS Network Performance - vpcEncryptionControls map[string]*VpcEncryptionControl - vpnConcentrators map[string]*VpnConcentrator - hostReservations map[string]*HostReservation - declarativePoliciesReports map[string]*DeclarativePoliciesReport - networkPerformanceSubscriptions map[string]*NetworkPerformanceSubscription + vpcEncryptionControls *store.Table[VpcEncryptionControl] + vpnConcentrators *store.Table[VpnConcentrator] + hostReservations *store.Table[HostReservation] + declarativePoliciesReports *store.Table[DeclarativePoliciesReport] + networkPerformanceSubscriptions *store.Table[NetworkPerformanceSubscription] // gopherstack-5o9 final EC2 parity sweep additions - tgwRTPropagations map[string]map[string]*TransitGatewayRouteTablePropagation - interruptibleCRAllocations map[string]*InterruptibleCapacityReservationAllocation - movingAddresses map[string]*MovingAddressStatus + tgwRTPropagations map[string]map[string]*TransitGatewayRouteTablePropagation + interruptibleCRAllocations *store.Table[InterruptibleCapacityReservationAllocation] + movingAddresses *store.Table[MovingAddressStatus] + // registry lets Reset collapse the ~150 converted resource maps' lifecycle + // to one call (registry.ResetAll()) instead of hand-rolled re-initialization + // of each map. See store_setup.go for every Table registration. + registry *store.Registry mu *lockmetrics.RWMutex lifecycleStop chan struct{} eniIDByAttachment map[string]string @@ -479,74 +503,21 @@ type InMemoryBackend struct { func newInMemoryBackendMaps() *InMemoryBackend { b := &InMemoryBackend{ - instances: make(map[string]*Instance), - securityGroups: make(map[string]*SecurityGroup), - vpcs: make(map[string]*VPC), - subnets: make(map[string]*Subnet), - keyPairs: make(map[string]*KeyPair), - volumes: make(map[string]*Volume), - addresses: make(map[string]*Address), - internetGateways: make(map[string]*InternetGateway), - routeTables: make(map[string]*RouteTable), - natGateways: make(map[string]*NatGateway), - networkInterfaces: make(map[string]*NetworkInterface), - spotRequests: make(map[string]*SpotInstanceRequest), - placementGroups: make(map[string]*PlacementGroup), - images: make(map[string]*AMIStub), - imageUsageReports: make(map[string]*ImageUsageReport), - launchTemplates: make(map[string]*LaunchTemplate), - vpcEndpoints: make(map[string]*VpcEndpoint), - tags: make(map[string]map[string]string), - addressTransfers: make(map[string]*AddressTransfer), - capacityReservations: make(map[string]*CapacityReservation), - reservedInstancesExchanges: make(map[string]*ReservedInstancesExchange), - tgwMulticastDomainAssociations: make(map[string]*TransitGatewayMulticastDomainAssociation), - tgwPeeringAttachments: make(map[string]*TransitGatewayPeeringAttachment), - tgwVpcAttachments: make(map[string]*TransitGatewayVpcAttachment), - vpcEndpointConnections: make(map[string]*VpcEndpointConnection), - vpcPeeringConnections: make(map[string]*VpcPeeringConnection), - byoipCidrs: make(map[string]*ByoipCidr), - dedicatedHosts: make(map[string]*Host), - snapshots: make(map[string]*Snapshot), - networkACLs: make(map[string]*StoredNetworkACL), - transitGateways: make(map[string]*TransitGateway), - flowLogs: make(map[string]*FlowLog), - dhcpOptionSets: make(map[string]*DhcpOptions), - egressOnlyIGWs: make(map[string]*EgressOnlyInternetGateway), - iamAssociations: make(map[string]*IamInstanceProfileAssociation), - tgwRouteTables: make(map[string]*TransitGatewayRouteTable), - tgwRoutes: make(map[string]*TransitGatewayRoute), - tgwRTAssociations: make(map[string]*TransitGatewayRouteTableAssociation), - tgwPolicyTables: make(map[string]*TransitGatewayPolicyTable), - tgwPolicyTableAssociations: make(map[string]*TransitGatewayPolicyTableAssociation), - tgwRouteTableAnnouncements: make(map[string]*TransitGatewayRouteTableAnnouncement), - vpcCidrAssociations: make(map[string]*VpcCidrBlockAssociation), - vpnGateways: make(map[string]*VpnGateway), - customerGateways: make(map[string]*CustomerGateway), - vpnConnections: make(map[string]*VpnConnection), - vpcEndpointServiceConfigs: make(map[string]*VpcEndpointServiceConfig), - ipams: make(map[string]*Ipam), - ipamScopes: make(map[string]*IpamScope), - ipamPools: make(map[string]*IpamPool), - ipamPoolCidrs: make(map[string][]*IpamPoolCidr), - ipamPoolAllocations: make(map[string]*IpamPoolAllocation), - ipamResourceDiscoveries: make(map[string]*IpamResourceDiscovery), - ipamResourceDiscoveryAssocs: make(map[string]*IpamResourceDiscoveryAssociation), - instanceIDsByVPC: make(map[string]map[string]struct{}), - eniIDsByInstance: make(map[string]map[string]struct{}), - eniIDByAttachment: make(map[string]string), - } + registry: store.NewRegistry(), + tags: make(map[string]map[string]string), + addressTransfers: make(map[string]*AddressTransfer), + vpcCidrAssociations: make(map[string]*VpcCidrBlockAssociation), + ipamPoolCidrs: make(map[string][]*IpamPoolCidr), + instanceIDsByVPC: make(map[string]map[string]struct{}), + eniIDsByInstance: make(map[string]map[string]struct{}), + eniIDByAttachment: make(map[string]string), + } + registerAllTables(b) initCoreExtraMaps(b) - initBatch5Maps(b) initBatch6Maps(b) - initRouteServerMaps(b) - initLocalGatewayMaps(b) - initTGWMulticastMaps(b) initVpcConfigMaps(b) initCapacityFamilyMaps(b) initVerifiedAccessExtMaps(b) - initFpgaImageMaps(b) - initParitySweep2Maps(b) initParityFinalMaps(b) initSecondaryIndexMaps(b) b.resetIpamDiscoveryMapsLocked() @@ -573,70 +544,42 @@ func newInMemoryBackendMaps() *InMemoryBackend { func initVerifiedAccessExtMaps(b *InMemoryBackend) { b.verifiedAccessEndpointPolicies = make(map[string]*VerifiedAccessPolicy) b.verifiedAccessGroupPolicies = make(map[string]*VerifiedAccessPolicy) - b.verifiedAccessInstanceLoggingConfigs = make(map[string]*VerifiedAccessInstanceLoggingConfig) -} - -// initFpgaImageMaps initialises the FPGA image state map (split out to keep -// newInMemoryBackendMaps under the funlen limit). -func initFpgaImageMaps(b *InMemoryBackend) { - b.fpgaImages = make(map[string]*FpgaImage) } // initCoreExtraMaps initialises spot fleet, snapshot, IMDS, and batch4 state // maps (split out to keep newInMemoryBackendMaps under the funlen limit). func initCoreExtraMaps(b *InMemoryBackend) { - b.spotFleets = make(map[string]*SpotFleetRequest) b.spotFleetHistory = make(map[string][]SpotFleetHistoryRecord) - b.volumeModifications = make(map[string]*VolumeModification) b.snapshotTiers = make(map[string]string) b.snapshotAttributes = make(map[string]map[string]string) b.sgVpcAssociations = make(map[string]map[string]string) b.vpcTenancy = make(map[string]string) b.vpcPeeringOptions = make(map[string]*PeeringConnectionOptions) b.subnetCIDRAssociations = make(map[string][]*SubnetCIDRAssociation) - b.addressAttributes = make(map[string]*AddressAttribute) b.instanceMonitoring = make(map[string]string) b.instanceCreditSpecs = make(map[string]string) b.instanceIMDSOptions = make(map[string]*IMDSOptions) - b.niPermissions = make(map[string]*NetworkInterfacePermission) b.niIPv6Addresses = make(map[string][]string) b.idFormatSettings = make(map[string]bool) - b.endpointConnectionNotifs = make(map[string]*VpcEndpointConnectionNotification) b.vpcEndpointServicePermissions = make(map[string][]string) - b.snapshotLocks = make(map[string]*SnapshotLock) - b.replaceRootVolumeTasks = make(map[string]*ReplaceRootVolumeTask) b.subnetCIDRReservations = make(map[string][]*SubnetCIDRReservation) b.imageDisabled = make(map[string]bool) b.imageDeprecated = make(map[string]string) b.imageDeregistrationProtection = make(map[string]bool) b.imageAttributes = make(map[string]map[string]string) b.vgwRoutePropagation = make(map[string]bool) - b.managedPrefixLists = make(map[string]*ManagedPrefixList) - b.clientVpnEndpoints = make(map[string]*ClientVpnEndpoint) - b.tgwConnects = make(map[string]*TransitGatewayConnect) - b.tgwConnectPeers = make(map[string]*TransitGatewayConnectPeer) - b.tgwPrefixListRefs = make(map[string]*TransitGatewayPrefixListReference) } // initCapacityFamilyMaps initialises the Capacity Reservation Fleet, Capacity // Block, and Capacity Manager state maps (split out to keep // newInMemoryBackendMaps under the funlen limit). func initCapacityFamilyMaps(b *InMemoryBackend) { - b.capacityReservationFleets = make(map[string]*CapacityReservationFleet) - b.capacityBlockOfferings = make(map[string]*CapacityBlockOffering) - b.capacityBlockExtensionOfferings = make(map[string]*CapacityBlockExtensionOffering) - b.capacityBlocks = make(map[string]*CapacityBlock) - b.capacityBlockExtensions = make(map[string]*CapacityBlockExtension) - b.capacityReservationBillingRequests = make(map[string]*CapacityReservationBillingRequest) - b.capacityManagerDataExports = make(map[string]*CapacityManagerDataExport) b.capacityManagerState = &CapacityManagerState{Status: capacityManagerStatusDisabled} } // initVpcConfigMaps initialises the VPC ClassicLink and Block Public Access // state maps (split out to keep newInMemoryBackendMaps under the funlen limit). func initVpcConfigMaps(b *InMemoryBackend) { - b.classicLinkInstances = make(map[string]*ClassicLinkInstance) - b.vpcBlockPublicAccessExclusions = make(map[string]*VpcBlockPublicAccessExclusion) b.vpcBlockPublicAccessOptions = &VpcBlockPublicAccessOptions{ InternetGatewayBlockMode: vpcBPABlockModeOff, State: vpcBPAStateDefault, @@ -645,86 +588,12 @@ func initVpcConfigMaps(b *InMemoryBackend) { } } -// initTGWMulticastMaps initialises the transit gateway multicast domain and -// metering policy state maps (split out to keep newInMemoryBackendMaps under -// the funlen limit). -func initTGWMulticastMaps(b *InMemoryBackend) { - b.tgwMulticastDomains = make(map[string]*TransitGatewayMulticastDomain) - b.tgwMulticastGroupEntries = make(map[string]*TransitGatewayMulticastGroupEntry) - b.tgwMeteringPolicies = make(map[string]*TransitGatewayMeteringPolicy) - b.tgwMeteringPolicyEntries = make(map[string]*TransitGatewayMeteringPolicyEntry) -} - -// initLocalGatewayMaps initialises the Local Gateway state maps (split out to keep -// newInMemoryBackendMaps under the funlen limit). -func initLocalGatewayMaps(b *InMemoryBackend) { - b.localGateways = make(map[string]*LocalGateway) - b.localGatewayVirtualInterfaces = make(map[string]*LocalGatewayVirtualInterface) - b.localGatewayVirtualInterfaceGroups = make(map[string]*LocalGatewayVirtualInterfaceGroup) - b.localGatewayRouteTables = make(map[string]*LocalGatewayRouteTable) - b.localGatewayRoutes = make(map[string]*LocalGatewayRoute) - b.localGatewayRouteTableVpcAssociations = make(map[string]*LocalGatewayRouteTableVpcAssociation) - b.localGatewayRouteTableVifGroupAssociations = make( - map[string]*LocalGatewayRouteTableVirtualInterfaceGroupAssociation, - ) -} - -// initParitySweep2Maps initialises the VPC Encryption Control, VPN Concentrator, -// Host Reservation, Declarative Policies, and AWS Network Performance state maps -// (split out to keep newInMemoryBackendMaps under the funlen limit). -func initParitySweep2Maps(b *InMemoryBackend) { - b.vpcEncryptionControls = make(map[string]*VpcEncryptionControl) - b.vpnConcentrators = make(map[string]*VpnConcentrator) - b.hostReservations = make(map[string]*HostReservation) - b.declarativePoliciesReports = make(map[string]*DeclarativePoliciesReport) - b.networkPerformanceSubscriptions = make(map[string]*NetworkPerformanceSubscription) -} - // initBatch6Maps initialises the verified-access, import-task, recycle-bin, // fast-launch and VPN-route maps (split out to keep newInMemoryBackendMaps // under the funlen limit). func initBatch6Maps(b *InMemoryBackend) { - b.verifiedAccessEndpoints = make(map[string]*VerifiedAccessEndpoint) - b.verifiedAccessGroups = make(map[string]*VerifiedAccessGroup) - b.verifiedAccessInstances = make(map[string]*VerifiedAccessInstance) - b.verifiedAccessTrustProviders = make(map[string]*VerifiedAccessTrustProvider) - b.instanceConnectEndpoints = make(map[string]*InstanceConnectEndpoint) - b.instanceEventWindows = make(map[string]*InstanceEventWindow) - b.imageImportTasks = make(map[string]*ImageImportTask) - b.snapshotImportTasks = make(map[string]*SnapshotImportTask) - b.recycleBinImages = make(map[string]*RecycleBinImage) - b.recycleBinSnapshots = make(map[string]*Snapshot) - b.recycleBinVolumes = make(map[string]*RecycleBinVolume) b.fastLaunchImages = make(map[string]bool) b.fastSnapshotRestores = make(map[string]bool) - b.vpnConnectionRoutes = make(map[string]*VpnConnectionRoute) -} - -func initBatch5Maps(b *InMemoryBackend) { - b.trafficMirrorFilters = make(map[string]*TrafficMirrorFilter) - b.trafficMirrorFilterRules = make(map[string]*TrafficMirrorFilterRule) - b.trafficMirrorSessions = make(map[string]*TrafficMirrorSession) - b.trafficMirrorTargets = make(map[string]*TrafficMirrorTarget) - b.fleets = make(map[string]*Fleet) - b.networkInsightsPaths = make(map[string]*NetworkInsightsPath) - b.networkInsightsAnalyses = make(map[string]*NetworkInsightsAnalysis) - b.networkInsightsAccessScopes = make(map[string]*NetworkInsightsAccessScope) - b.networkInsightsAccessScopeAnalyses = make(map[string]*NetworkInsightsAccessScopeAnalysis) - b.carrierGateways = make(map[string]*CarrierGateway) - b.reservedInstances = make(map[string]*ReservedInstance) - b.reservedInstancesOfferings = make(map[string]*ReservedInstancesOffering) - b.reservedInstancesListings = make(map[string]*ReservedInstancesListing) - b.reservedInstancesModifications = make(map[string]*ReservedInstancesModification) -} - -// initRouteServerMaps initialises the VPC Route Server state maps (split out -// to keep newInMemoryBackendMaps under the funlen limit). -func initRouteServerMaps(b *InMemoryBackend) { - b.routeServers = make(map[string]*RouteServer) - b.routeServerEndpoints = make(map[string]*RouteServerEndpoint) - b.routeServerPeers = make(map[string]*RouteServerPeer) - b.routeServerAssociations = make(map[string]*RouteServerAssociation) - b.routeServerPropagations = make(map[string]*RouteServerPropagation) } // NewInMemoryBackend creates a new InMemoryBackend with a default VPC and subnet. @@ -785,7 +654,7 @@ func (b *InMemoryBackend) reconcileInstanceLifecycle() { // Fast path: read-lock to detect any transitional instance. b.mu.RLock("reconcileInstanceLifecycle-check") hasTransitional := false - for _, inst := range b.instances { + for _, inst := range b.instances.All() { switch inst.State { case StatePending, StateStopping, StateShuttingDown: hasTransitional = true @@ -804,7 +673,7 @@ func (b *InMemoryBackend) reconcileInstanceLifecycle() { b.mu.Lock("reconcileInstanceLifecycle") defer b.mu.Unlock() - for _, inst := range b.instances { + for _, inst := range b.instances.All() { switch inst.State { case StatePending: inst.State = StateRunning @@ -819,29 +688,29 @@ func (b *InMemoryBackend) reconcileInstanceLifecycle() { // initDefaults pre-populates a default VPC, subnet, and security group. func (b *InMemoryBackend) initDefaults() { defaultVPCID := vpcDefaultName - b.vpcs[defaultVPCID] = &VPC{ + b.vpcs.Put(&VPC{ ID: defaultVPCID, CIDRBlock: "172.31.0.0/16", IsDefault: true, - } + }) defaultSubnetID := "subnet-default" - b.subnets[defaultSubnetID] = &Subnet{ + b.subnets.Put(&Subnet{ ID: defaultSubnetID, VPCID: defaultVPCID, CIDRBlock: "172.31.0.0/20", AvailabilityZone: b.Region + "a", IsDefault: true, - } + }) b.indexSubnetLocked(defaultSubnetID, defaultVPCID) defaultSGID := "sg-default" - b.securityGroups[defaultSGID] = &SecurityGroup{ + b.securityGroups.Put(&SecurityGroup{ ID: defaultSGID, Name: "default", Description: "default VPC security group", VPCID: defaultVPCID, - } + }) b.indexSGLocked(defaultSGID, defaultVPCID) } @@ -863,7 +732,7 @@ func (b *InMemoryBackend) RunInstances( if subnetID == "" { subnetID = b.findDefaultSubnetID() - } else if _, ok := b.subnets[subnetID]; !ok { + } else if _, ok := b.subnets.Get(subnetID); !ok { return nil, fmt.Errorf("%w: %s", ErrSubnetNotFound, subnetID) } @@ -871,7 +740,7 @@ func (b *InMemoryBackend) RunInstances( mapPublicIP := false availabilityZone := "" - if sub, ok := b.subnets[subnetID]; ok { + if sub, ok := b.subnets.Get(subnetID); ok { vpcID = sub.VPCID mapPublicIP = sub.MapPublicIPOnLaunch availabilityZone = sub.AvailabilityZone @@ -905,7 +774,7 @@ func (b *InMemoryBackend) RunInstances( } eniID := "eni-" + uuid.New().String()[:17] attachID := "eni-attach-" + uuid.New().String()[:8] - b.networkInterfaces[eniID] = &NetworkInterface{ + b.networkInterfaces.Put(&NetworkInterface{ ID: eniID, SubnetID: subnetID, VPCID: vpcID, @@ -915,11 +784,12 @@ func (b *InMemoryBackend) RunInstances( DeviceIndex: 0, Status: stateInUse, SourceDestCheck: true, - } - b.instances[id] = inst + }) + b.instances.Put(inst) b.indexInstanceLocked(inst) - b.indexENILocked(eniID, b.networkInterfaces[eniID]) - b.indexENIByVPCLocked(eniID, b.networkInterfaces[eniID]) + eni, _ := b.networkInterfaces.Get(eniID) + b.indexENILocked(eniID, eni) + b.indexENIByVPCLocked(eniID, eni) instances = append(instances, inst) } @@ -982,7 +852,7 @@ func (b *InMemoryBackend) LookupKeyPairAuthorizedKey(name string) string { b.mu.RLock("LookupKeyPairAuthorizedKey") defer b.mu.RUnlock() - kp, ok := b.keyPairs[name] + kp, ok := b.keyPairs.Get(name) if !ok { return "" } @@ -998,7 +868,7 @@ func (b *InMemoryBackend) SetComputeResult(instanceID string, r LaunchResult) er b.mu.Lock("SetComputeResult") defer b.mu.Unlock() - inst, ok := b.instances[instanceID] + inst, ok := b.instances.Get(instanceID) if !ok { return fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -1013,7 +883,7 @@ func (b *InMemoryBackend) SetComputeResult(instanceID string, r LaunchResult) er // keep primary ENI in sync so DescribeNetworkInterfaces shows the // container's actual address for eniID := range b.eniIDsByInstance[instanceID] { - if eni, exists := b.networkInterfaces[eniID]; exists && eni.PrivateIP == oldPrivate { + if eni, exists := b.networkInterfaces.Get(eniID); exists && eni.PrivateIP == oldPrivate { eni.PrivateIP = r.PrivateIP } } @@ -1040,7 +910,7 @@ func (b *InMemoryBackend) LookupInstanceProviderID(instanceID string) string { b.mu.RLock("LookupInstanceProviderID") defer b.mu.RUnlock() - inst, ok := b.instances[instanceID] + inst, ok := b.instances.Get(instanceID) if !ok { return "" } @@ -1054,7 +924,7 @@ func (b *InMemoryBackend) LookupInstancePublicDNSName(instanceID string) string b.mu.RLock("LookupInstancePublicDNSName") defer b.mu.RUnlock() - inst, ok := b.instances[instanceID] + inst, ok := b.instances.Get(instanceID) if !ok { return "" } @@ -1065,7 +935,8 @@ func (b *InMemoryBackend) LookupInstancePublicDNSName(instanceID string) string // findDefaultSubnetID returns the ID of the default subnet, or empty string if none. // Must be called with b.mu held. func (b *InMemoryBackend) findDefaultSubnetID() string { - for id, s := range b.subnets { + for _, s := range b.subnets.All() { + id := subnetsKeyFn(s) if s.IsDefault { return id } @@ -1084,7 +955,7 @@ func (b *InMemoryBackend) DescribeInstances(ids []string, state string) []*Insta out := make([]*Instance, 0, len(ids)) for _, id := range ids { - inst, ok := b.instances[id] + inst, ok := b.instances.Get(id) if !ok { continue } @@ -1104,8 +975,8 @@ func (b *InMemoryBackend) DescribeInstances(ids []string, state string) []*Insta // Collect matching pointers under the read lock (no allocations), then copy // outside the lock to narrow the critical section for concurrent writers. - ptrs := make([]*Instance, 0, len(b.instances)) - for _, inst := range b.instances { + ptrs := make([]*Instance, 0, b.instances.Len()) + for _, inst := range b.instances.All() { if state == "" || inst.State.Name == state { ptrs = append(ptrs, inst) } @@ -1132,17 +1003,29 @@ func (b *InMemoryBackend) TerminateInstances(ids []string) ([]*InstanceStateChan var result []*InstanceStateChange for _, id := range ids { - inst, ok := b.instances[id] + inst, ok := b.instances.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, id) } + if inst.DisableAPITermination { + return nil, fmt.Errorf( + "%w: the instance %s may not be terminated. "+ + "Modify its 'disableApiTermination' instance attribute and try again", + ErrOperationNotPermitted, id) + } + prev := inst.State // AWS state machine: any state → shutting-down → terminated (reconciler advances). // Resource cleanup (ENIs, EIPs, volumes) happens immediately so callers // do not observe dangling attachments, but state advances asynchronously. inst.State = StateShuttingDown inst.TerminatedAt = time.Now() + inst.StateReasonCode = "Client.UserInitiatedShutdown" + inst.StateReasonMessage = "Client.UserInitiatedShutdown: User initiated shutdown" + inst.StateTransitionReason = fmt.Sprintf( + "User initiated (%s)", time.Now().UTC().Format("2006-01-02 15:04:05 GMT"), + ) result = append(result, &InstanceStateChange{ InstanceID: id, PreviousState: prev, @@ -1151,7 +1034,7 @@ func (b *InMemoryBackend) TerminateInstances(ids []string) ([]*InstanceStateChan // Mirror AWS behaviour: when the backing instance of a spot request is // terminated, the request transitions to "closed" (not stateCancelled). - for _, req := range b.spotRequests { + for _, req := range b.spotRequests.All() { if req.InstanceID == id && req.State == stateActive { req.State = "closed" req.CancelledAt = time.Now() @@ -1164,7 +1047,7 @@ func (b *InMemoryBackend) TerminateInstances(ids []string) ([]*InstanceStateChan // the default for all network interfaces attached at launch). eniIDs := b.eniIDsByInstance[id] for eniID := range eniIDs { - eni, exists := b.networkInterfaces[eniID] + eni, exists := b.networkInterfaces.Get(eniID) if !exists { continue } @@ -1172,7 +1055,7 @@ func (b *InMemoryBackend) TerminateInstances(ids []string) ([]*InstanceStateChan b.deindexENILocked(eniID, eni) b.deindexENIByVPCLocked(eniID, eni) b.recycleENIIPsLocked(eni) - delete(b.networkInterfaces, eniID) + b.networkInterfaces.Delete(eniID) delete(b.tags, eniID) } b.deindexInstanceLocked(inst) @@ -1200,7 +1083,7 @@ func (b *InMemoryBackend) DescribeSecurityGroups(ids []string) []*SecurityGroup out := make([]*SecurityGroup, 0, len(ids)) for _, id := range ids { - sg, ok := b.securityGroups[id] + sg, ok := b.securityGroups.Get(id) if !ok { continue } @@ -1212,9 +1095,9 @@ func (b *InMemoryBackend) DescribeSecurityGroups(ids []string) []*SecurityGroup return out } - out := make([]*SecurityGroup, 0, len(b.securityGroups)) + out := make([]*SecurityGroup, 0, b.securityGroups.Len()) - for _, sg := range b.securityGroups { + for _, sg := range b.securityGroups.All() { cp := *sg out = append(out, &cp) } @@ -1234,12 +1117,12 @@ func (b *InMemoryBackend) CreateSecurityGroup( defer b.mu.Unlock() if vpcID != "" { - if _, ok := b.vpcs[vpcID]; !ok { + if _, ok := b.vpcs.Get(vpcID); !ok { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } } - for _, sg := range b.securityGroups { + for _, sg := range b.securityGroups.All() { if sg.Name == name && sg.VPCID == vpcID { return nil, fmt.Errorf( "%w: group named %s already exists in VPC %s", @@ -1261,7 +1144,7 @@ func (b *InMemoryBackend) CreateSecurityGroup( {Protocol: "-1", IPRange: cidrAllIPv4}, }, } - b.securityGroups[id] = sg + b.securityGroups.Put(sg) b.indexSGLocked(id, vpcID) return sg, nil @@ -1272,13 +1155,13 @@ func (b *InMemoryBackend) DeleteSecurityGroup(id string) error { b.mu.Lock("DeleteSecurityGroup") defer b.mu.Unlock() - sg, ok := b.securityGroups[id] + sg, ok := b.securityGroups.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrSecurityGroupNotFound, id) } b.deindexSGLocked(id, sg.VPCID) - delete(b.securityGroups, id) + b.securityGroups.Delete(id) delete(b.tags, id) return nil @@ -1295,7 +1178,7 @@ func (b *InMemoryBackend) DescribeVpcs(ids []string) []*VPC { out := make([]*VPC, 0, len(ids)) for _, id := range ids { - v, ok := b.vpcs[id] + v, ok := b.vpcs.Get(id) if !ok { continue } @@ -1307,9 +1190,9 @@ func (b *InMemoryBackend) DescribeVpcs(ids []string) []*VPC { return out } - out := make([]*VPC, 0, len(b.vpcs)) + out := make([]*VPC, 0, b.vpcs.Len()) - for _, v := range b.vpcs { + for _, v := range b.vpcs.All() { cp := *v out = append(out, &cp) } @@ -1326,7 +1209,7 @@ func (b *InMemoryBackend) CreateVpc(cidr string) (*VPC, error) { b.mu.Lock("CreateVpc") defer b.mu.Unlock() - for _, existing := range b.vpcs { + for _, existing := range b.vpcs.All() { if cidrsOverlap(cidr, existing.CIDRBlock) { return nil, fmt.Errorf("%w: CIDR %s overlaps with existing VPC %s (%s)", ErrCIDRConflict, cidr, existing.ID, existing.CIDRBlock) @@ -1338,7 +1221,7 @@ func (b *InMemoryBackend) CreateVpc(cidr string) (*VPC, error) { ID: id, CIDRBlock: cidr, } - b.vpcs[id] = v + b.vpcs.Put(v) return v, nil } @@ -1346,10 +1229,11 @@ func (b *InMemoryBackend) CreateVpc(cidr string) (*VPC, error) { // cascadeDeleteVpcIGWsLocked removes all internet gateways that have an // attachment to the given VPC. Must be called with b.mu held. func (b *InMemoryBackend) cascadeDeleteVpcIGWsLocked(vpcID string) { - for igwID, igw := range b.internetGateways { + for _, igw := range b.internetGateways.All() { + igwID := internetGatewaysKeyFn(igw) for _, att := range igw.Attachments { if att.VPCID == vpcID { - delete(b.internetGateways, igwID) + b.internetGateways.Delete(igwID) delete(b.tags, igwID) break @@ -1367,13 +1251,13 @@ func (b *InMemoryBackend) DeleteVpc(id string) error { b.mu.Lock("DeleteVpc") defer b.mu.Unlock() - if _, ok := b.vpcs[id]; !ok { + if _, ok := b.vpcs.Get(id); !ok { return fmt.Errorf("%w: %s", ErrVPCNotFound, id) } // Cascade: terminate instances belonging to this VPC via secondary index. for instID := range b.instanceIDsByVPC[id] { - if inst, ok := b.instances[instID]; ok { + if inst, ok := b.instances.Get(instID); ok { inst.State = StateTerminated inst.TerminatedAt = time.Now() delete(b.tags, instID) @@ -1388,9 +1272,9 @@ func (b *InMemoryBackend) DeleteVpc(id string) error { // Cascade: delete NAT gateways belonging to this VPC via secondary index, // avoiding a full-map scan under the write lock. for ngwID := range b.natGatewayIDsByVPC[id] { - if ngw, ok := b.natGateways[ngwID]; ok { + if ngw, ok := b.natGateways.Get(ngwID); ok { b.recycleIPLocked(ngw.PrivateIP) - delete(b.natGateways, ngwID) + b.natGateways.Delete(ngwID) delete(b.tags, ngwID) } } @@ -1398,14 +1282,14 @@ func (b *InMemoryBackend) DeleteVpc(id string) error { // Cascade: remove route tables belonging to this VPC via secondary index. for rtID := range b.routeTableIDsByVPC[id] { - delete(b.routeTables, rtID) + b.routeTables.Delete(rtID) delete(b.tags, rtID) } delete(b.routeTableIDsByVPC, id) // Cascade: remove security groups belonging to this VPC via secondary index. for sgID := range b.sgIDsByVPC[id] { - delete(b.securityGroups, sgID) + b.securityGroups.Delete(sgID) delete(b.tags, sgID) } delete(b.sgIDsByVPC, id) @@ -1413,10 +1297,10 @@ func (b *InMemoryBackend) DeleteVpc(id string) error { // Cascade: remove network interfaces belonging to this VPC via secondary // index, avoiding a full-map scan under the write lock. for eniID := range b.eniIDsByVPC[id] { - if eni, ok := b.networkInterfaces[eniID]; ok { + if eni, ok := b.networkInterfaces.Get(eniID); ok { b.recycleENIIPsLocked(eni) b.deindexENILocked(eniID, eni) - delete(b.networkInterfaces, eniID) + b.networkInterfaces.Delete(eniID) delete(b.tags, eniID) } } @@ -1424,12 +1308,11 @@ func (b *InMemoryBackend) DeleteVpc(id string) error { // Cascade: remove subnets belonging to this VPC via secondary index. for subnetID := range b.subnetIDsByVPC[id] { - delete(b.subnets, subnetID) + b.subnets.Delete(subnetID) delete(b.tags, subnetID) } delete(b.subnetIDsByVPC, id) - - delete(b.vpcs, id) + b.vpcs.Delete(id) delete(b.tags, id) return nil @@ -1446,7 +1329,7 @@ func (b *InMemoryBackend) DescribeSubnets(ids []string) []*Subnet { out := make([]*Subnet, 0, len(ids)) for _, id := range ids { - s, ok := b.subnets[id] + s, ok := b.subnets.Get(id) if !ok { continue } @@ -1458,9 +1341,9 @@ func (b *InMemoryBackend) DescribeSubnets(ids []string) []*Subnet { return out } - out := make([]*Subnet, 0, len(b.subnets)) + out := make([]*Subnet, 0, b.subnets.Len()) - for _, s := range b.subnets { + for _, s := range b.subnets.All() { cp := *s out = append(out, &cp) } @@ -1481,7 +1364,7 @@ func (b *InMemoryBackend) CreateSubnet(vpcID, cidr, az string) (*Subnet, error) b.mu.Lock("CreateSubnet") defer b.mu.Unlock() - if _, ok := b.vpcs[vpcID]; !ok { + if _, ok := b.vpcs.Get(vpcID); !ok { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -1489,13 +1372,13 @@ func (b *InMemoryBackend) CreateSubnet(vpcID, cidr, az string) (*Subnet, error) az = b.Region + "a" } - vpc := b.vpcs[vpcID] + vpc, _ := b.vpcs.Get(vpcID) if !cidrContains(vpc.CIDRBlock, cidr) { return nil, fmt.Errorf("%w: subnet CIDR %s is not within VPC CIDR %s", ErrInvalidParameter, cidr, vpc.CIDRBlock) } - for _, existing := range b.subnets { + for _, existing := range b.subnets.All() { if existing.VPCID == vpcID && cidrsOverlap(cidr, existing.CIDRBlock) { return nil, fmt.Errorf("%w: CIDR %s overlaps with existing subnet %s (%s)", ErrCIDRConflict, cidr, existing.ID, existing.CIDRBlock) @@ -1509,7 +1392,7 @@ func (b *InMemoryBackend) CreateSubnet(vpcID, cidr, az string) (*Subnet, error) CIDRBlock: cidr, AvailabilityZone: az, } - b.subnets[id] = s + b.subnets.Put(s) b.indexSubnetLocked(id, vpcID) return s, nil @@ -1521,12 +1404,13 @@ func (b *InMemoryBackend) DeleteSubnet(id string) error { b.mu.Lock("DeleteSubnet") defer b.mu.Unlock() - if _, ok := b.subnets[id]; !ok { + if _, ok := b.subnets.Get(id); !ok { return fmt.Errorf("%w: %s", ErrSubnetNotFound, id) } // Cascade: terminate instances in this subnet. - for instID, inst := range b.instances { + for _, inst := range b.instances.All() { + instID := instancesKeyFn(inst) if inst.SubnetID == id { inst.State = StateTerminated inst.TerminatedAt = time.Now() @@ -1536,29 +1420,31 @@ func (b *InMemoryBackend) DeleteSubnet(id string) error { } // Cascade: delete NAT gateways in this subnet. - for ngwID, ngw := range b.natGateways { + for _, ngw := range b.natGateways.All() { + ngwID := natGatewaysKeyFn(ngw) if ngw.SubnetID == id { b.recycleIPLocked(ngw.PrivateIP) b.deindexNatGatewayLocked(ngw) - delete(b.natGateways, ngwID) + b.natGateways.Delete(ngwID) delete(b.tags, ngwID) } } // Cascade: remove network interfaces in this subnet. - for eniID, eni := range b.networkInterfaces { + for _, eni := range b.networkInterfaces.All() { + eniID := networkInterfacesKeyFn(eni) if eni.SubnetID == id { b.recycleENIIPsLocked(eni) b.deindexENILocked(eniID, eni) b.deindexENIByVPCLocked(eniID, eni) - delete(b.networkInterfaces, eniID) + b.networkInterfaces.Delete(eniID) delete(b.tags, eniID) } } - subnet := b.subnets[id] + subnet, _ := b.subnets.Get(id) b.deindexSubnetLocked(id, subnet.VPCID) - delete(b.subnets, id) + b.subnets.Delete(id) delete(b.tags, id) return nil @@ -1572,31 +1458,9 @@ type TagEntry struct { Value string `json:"value,omitempty"` } -// resourceTypeByID infers the EC2 resource type from the ID prefix. -func resourceTypeByID(id string) string { - prefixes := []struct { - prefix string - rtype string - }{ - {"i-", "instance"}, - {"sg-", "security-group"}, - {"vpc-", resourceTypeVPC}, - {"subnet-", "subnet"}, - {"vol-", "volume"}, - {"igw-", "internet-gateway"}, - {"rtb-", "route-table"}, - {"nat-", "natgateway"}, - {"eipalloc-", "elastic-ip"}, - } - - for _, e := range prefixes { - if strings.HasPrefix(id, e.prefix) { - return e.rtype - } - } - - return "resource" -} +// resourceTypeByID and resourceExistsLocked live in backend_resource_types.go +// (split out: the full taggable-resource-type table is large and orthogonal +// to the rest of this file). // recycleIPLocked adds ip to the free list if it is an auto-allocated // 172.31.x.y address and the free list has capacity remaining. @@ -1623,14 +1487,14 @@ func (b *InMemoryBackend) recycleENIIPsLocked(eni *NetworkInterface) { // after the instance transitions to the terminated state. // Must be called with b.mu held. func (b *InMemoryBackend) detachVolumesAndEIPsLocked(instanceID string) { - for _, vol := range b.volumes { + for _, vol := range b.volumes.All() { if vol.Attachment != nil && vol.Attachment.InstanceID == instanceID { vol.Attachment = nil vol.State = stateAvailable } } - for _, addr := range b.addresses { + for _, addr := range b.addresses.All() { if addr.InstanceID == instanceID { addr.AssociationID = "" addr.InstanceID = "" @@ -1638,64 +1502,6 @@ func (b *InMemoryBackend) detachVolumesAndEIPsLocked(instanceID string) { } } -// resourceExistsLocked reports whether id refers to any known EC2 resource. -// Must be called with b.mu held. -func (b *InMemoryBackend) resourceExistsLocked(id string) bool { - if _, ok := b.instances[id]; ok { - return true - } - - if _, ok := b.securityGroups[id]; ok { - return true - } - - if _, ok := b.vpcs[id]; ok { - return true - } - - if _, ok := b.subnets[id]; ok { - return true - } - - if _, ok := b.keyPairs[id]; ok { - return true - } - - if _, ok := b.volumes[id]; ok { - return true - } - - if _, ok := b.addresses[id]; ok { - return true - } - - if _, ok := b.internetGateways[id]; ok { - return true - } - - if _, ok := b.routeTables[id]; ok { - return true - } - - if _, ok := b.natGateways[id]; ok { - return true - } - - if _, ok := b.networkInterfaces[id]; ok { - return true - } - - if _, ok := b.spotRequests[id]; ok { - return true - } - - if _, ok := b.placementGroups[id]; ok { - return true - } - - return false -} - // CreateTags adds or updates tags on one or more resources. // Returns ErrInvalidParameter if any resource ID does not exist. // All IDs are validated before any tags are written, making the operation atomic diff --git a/services/ec2/backend_accept_ops.go b/services/ec2/backend_accept_ops.go index a5cc3194f..f25e8acff 100644 --- a/services/ec2/backend_accept_ops.go +++ b/services/ec2/backend_accept_ops.go @@ -155,25 +155,17 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() + // Reset every store.Table-backed resource map in one call instead of the + // per-map make() calls this used to be (Phase 3.3 pkgs/store conversion). + // See registerAllTables in store_setup.go for the full list of tables this + // covers, and the comment there for the handful of fields that are NOT + // covered because they remain plain maps. + b.registry.ResetAll() + // Reset original resource maps. - b.instances = make(map[string]*Instance) - b.securityGroups = make(map[string]*SecurityGroup) - b.vpcs = make(map[string]*VPC) - b.subnets = make(map[string]*Subnet) - b.keyPairs = make(map[string]*KeyPair) - b.volumes = make(map[string]*Volume) - b.addresses = make(map[string]*Address) - b.internetGateways = make(map[string]*InternetGateway) - b.routeTables = make(map[string]*RouteTable) - b.natGateways = make(map[string]*NatGateway) - b.networkInterfaces = make(map[string]*NetworkInterface) - b.spotRequests = make(map[string]*SpotInstanceRequest) - b.placementGroups = make(map[string]*PlacementGroup) - b.images = make(map[string]*AMIStub) - b.imageUsageReports = make(map[string]*ImageUsageReport) - b.launchTemplates = make(map[string]*LaunchTemplate) - b.vpcEndpoints = make(map[string]*VpcEndpoint) b.tags = make(map[string]map[string]string) + b.addressTransfers = make(map[string]*AddressTransfer) + b.vpcCidrAssociations = make(map[string]*VpcCidrBlockAssociation) initSecondaryIndexMaps(b) b.freePrivateIPs = nil b.nextPrivateIPIndex = 0 @@ -183,25 +175,25 @@ func (b *InMemoryBackend) Reset() { // Re-populate defaults (must be called without the lock held since it acquires its own). // Since we already hold the lock, populate inline. - b.vpcs[vpcDefaultName] = &VPC{ + b.vpcs.Put(&VPC{ ID: vpcDefaultName, CIDRBlock: "172.31.0.0/16", IsDefault: true, - } - b.subnets["subnet-default"] = &Subnet{ + }) + b.subnets.Put(&Subnet{ ID: "subnet-default", VPCID: vpcDefaultName, CIDRBlock: "172.31.0.0/20", AvailabilityZone: b.Region + "a", IsDefault: true, - } + }) b.indexSubnetLocked("subnet-default", vpcDefaultName) - b.securityGroups["sg-default"] = &SecurityGroup{ + b.securityGroups.Put(&SecurityGroup{ ID: "sg-default", Name: "default", Description: "default VPC security group", VPCID: vpcDefaultName, - } + }) b.indexSGLocked("sg-default", vpcDefaultName) } @@ -209,38 +201,14 @@ func (b *InMemoryBackend) Reset() { // after the original core set. Must be called with b.mu held. func (b *InMemoryBackend) resetNewOpsMapsLocked() { b.addressTransfers = make(map[string]*AddressTransfer) - b.capacityReservations = make(map[string]*CapacityReservation) - b.reservedInstancesExchanges = make(map[string]*ReservedInstancesExchange) - b.tgwMulticastDomainAssociations = make(map[string]*TransitGatewayMulticastDomainAssociation) - b.tgwPeeringAttachments = make(map[string]*TransitGatewayPeeringAttachment) - b.tgwVpcAttachments = make(map[string]*TransitGatewayVpcAttachment) - b.vpcEndpointConnections = make(map[string]*VpcEndpointConnection) - b.vpcPeeringConnections = make(map[string]*VpcPeeringConnection) - b.byoipCidrs = make(map[string]*ByoipCidr) - b.dedicatedHosts = make(map[string]*Host) - b.snapshots = make(map[string]*Snapshot) - b.networkACLs = make(map[string]*StoredNetworkACL) - b.transitGateways = make(map[string]*TransitGateway) - b.flowLogs = make(map[string]*FlowLog) - b.dhcpOptionSets = make(map[string]*DhcpOptions) - b.egressOnlyIGWs = make(map[string]*EgressOnlyInternetGateway) - b.iamAssociations = make(map[string]*IamInstanceProfileAssociation) - b.tgwRouteTables = make(map[string]*TransitGatewayRouteTable) - b.tgwRoutes = make(map[string]*TransitGatewayRoute) - b.tgwRTAssociations = make(map[string]*TransitGatewayRouteTableAssociation) - b.tgwPolicyTables = make(map[string]*TransitGatewayPolicyTable) - b.tgwPolicyTableAssociations = make(map[string]*TransitGatewayPolicyTableAssociation) - b.tgwRouteTableAnnouncements = make(map[string]*TransitGatewayRouteTableAnnouncement) b.vpcCidrAssociations = make(map[string]*VpcCidrBlockAssociation) b.resetAdvancedNetworkingMapsLocked() b.resetIpamDiscoveryMapsLocked() b.resetIpamPolicyMapsLocked() b.resetBatch4MapsLocked() - initTGWMulticastMaps(b) initVpcConfigMaps(b) initCapacityFamilyMaps(b) initVerifiedAccessExtMaps(b) - initFpgaImageMaps(b) b.resetScheduledInstanceMapsLocked() b.resetIPPoolMapsLocked() b.resetAllowedImagesSettingsLocked() @@ -253,22 +221,12 @@ func (b *InMemoryBackend) resetNewOpsMapsLocked() { b.resetSecondaryNetworkMapsLocked() b.resetInstanceAttrMapsLocked() b.resetSQLHaMapsLocked() - initParitySweep2Maps(b) initParityFinalMaps(b) } // resetBatch4MapsLocked re-initialises all batch4 resource maps. // Must be called with b.mu held. func (b *InMemoryBackend) resetBatch4MapsLocked() { - b.managedPrefixLists = make(map[string]*ManagedPrefixList) - b.clientVpnEndpoints = make(map[string]*ClientVpnEndpoint) - b.tgwConnects = make(map[string]*TransitGatewayConnect) - b.tgwConnectPeers = make(map[string]*TransitGatewayConnectPeer) - b.tgwPrefixListRefs = make(map[string]*TransitGatewayPrefixListReference) - b.verifiedAccessEndpoints = make(map[string]*VerifiedAccessEndpoint) - b.verifiedAccessGroups = make(map[string]*VerifiedAccessGroup) - b.verifiedAccessInstances = make(map[string]*VerifiedAccessInstance) - b.verifiedAccessTrustProviders = make(map[string]*VerifiedAccessTrustProvider) } // ---- AcceptAddressTransfer ---- @@ -319,7 +277,7 @@ func (b *InMemoryBackend) AcceptCapacityReservationBillingOwnership( b.mu.Lock("AcceptCapacityReservationBillingOwnership") defer b.mu.Unlock() - cr, ok := b.capacityReservations[capacityReservationID] + cr, ok := b.capacityReservations.Get(capacityReservationID) if !ok { return nil, fmt.Errorf("%w: %s", ErrCapacityReservationNotFound, capacityReservationID) } @@ -345,8 +303,7 @@ func (b *InMemoryBackend) AddCapacityReservationInternal(cr *CapacityReservation if cp.CreateTime.IsZero() { cp.CreateTime = time.Now() } - - b.capacityReservations[cp.CapacityReservationID] = &cp + b.capacityReservations.Put(&cp) } // DescribeCapacityReservations returns capacity reservations sorted by ID. @@ -362,7 +319,7 @@ func (b *InMemoryBackend) DescribeCapacityReservations(ids []string) []*Capacity var result []*CapacityReservation - for _, cr := range b.capacityReservations { + for _, cr := range b.capacityReservations.All() { if len(idSet) > 0 && !idSet[cr.CapacityReservationID] { continue } @@ -407,8 +364,7 @@ func (b *InMemoryBackend) AcceptReservedInstancesExchangeQuote( TargetReservedInstances: []string{targetID}, Status: "successful", } - - b.reservedInstancesExchanges[exchangeID] = exchange + b.reservedInstancesExchanges.Put(exchange) cp := *exchange cp.ReservedInstanceIDs = make([]string, len(exchange.ReservedInstanceIDs)) @@ -446,15 +402,13 @@ func (b *InMemoryBackend) AcceptTransitGatewayMulticastDomainAssociations( assocs := make([]*TransitGatewayMulticastDomainAssociation, 0, len(subnetIDs)) for _, subnetID := range subnetIDs { - key := transitGatewayMulticastDomainID + ":" + subnetID assoc := &TransitGatewayMulticastDomainAssociation{ TransitGatewayMulticastDomainID: transitGatewayMulticastDomainID, TransitGatewayAttachmentID: transitGatewayAttachmentID, SubnetID: subnetID, State: "associated", } - - b.tgwMulticastDomainAssociations[key] = assoc + b.tgwMulticastDomainAssociations.Put(assoc) cp := *assoc assocs = append(assocs, &cp) @@ -472,8 +426,7 @@ func (b *InMemoryBackend) AddTGWMulticastDomainAssociationInternal( defer b.mu.Unlock() cp := *assoc - key := cp.TransitGatewayMulticastDomainID + ":" + cp.SubnetID - b.tgwMulticastDomainAssociations[key] = &cp + b.tgwMulticastDomainAssociations.Put(&cp) } // ---- AcceptTransitGatewayPeeringAttachment ---- @@ -490,7 +443,7 @@ func (b *InMemoryBackend) AcceptTransitGatewayPeeringAttachment( b.mu.Lock("AcceptTransitGatewayPeeringAttachment") defer b.mu.Unlock() - att, ok := b.tgwPeeringAttachments[transitGatewayAttachmentID] + att, ok := b.tgwPeeringAttachments.Get(transitGatewayAttachmentID) if !ok { return nil, fmt.Errorf( "%w: %s", @@ -515,8 +468,7 @@ func (b *InMemoryBackend) AddTGWPeeringAttachmentInternal(att *TransitGatewayPee if cp.CreationTime.IsZero() { cp.CreationTime = time.Now() } - - b.tgwPeeringAttachments[cp.TransitGatewayAttachmentID] = &cp + b.tgwPeeringAttachments.Put(&cp) } // ---- AcceptTransitGatewayVpcAttachment ---- @@ -533,7 +485,7 @@ func (b *InMemoryBackend) AcceptTransitGatewayVpcAttachment( b.mu.Lock("AcceptTransitGatewayVpcAttachment") defer b.mu.Unlock() - att, ok := b.tgwVpcAttachments[transitGatewayAttachmentID] + att, ok := b.tgwVpcAttachments.Get(transitGatewayAttachmentID) if !ok { return nil, fmt.Errorf( "%w: %s", @@ -558,8 +510,7 @@ func (b *InMemoryBackend) AddTGWVpcAttachmentInternal(att *TransitGatewayVpcAtta if cp.CreationTime.IsZero() { cp.CreationTime = time.Now() } - - b.tgwVpcAttachments[cp.TransitGatewayAttachmentID] = &cp + b.tgwVpcAttachments.Put(&cp) } // ---- AcceptVpcEndpointConnections ---- @@ -587,7 +538,7 @@ func (b *InMemoryBackend) AcceptVpcEndpointConnections( for _, epID := range vpcEndpointIDs { key := serviceID + ":" + epID - conn, ok := b.vpcEndpointConnections[key] + conn, ok := b.vpcEndpointConnections.Get(key) if !ok { conn = &VpcEndpointConnection{ @@ -595,7 +546,7 @@ func (b *InMemoryBackend) AcceptVpcEndpointConnections( VpcEndpointID: epID, CreationTime: time.Now(), } - b.vpcEndpointConnections[key] = conn + b.vpcEndpointConnections.Put(conn) } conn.State = stateAvailable @@ -617,8 +568,7 @@ func (b *InMemoryBackend) AddVpcEndpointConnectionInternal(conn *VpcEndpointConn cp.CreationTime = time.Now() } - key := cp.ServiceID + ":" + cp.VpcEndpointID - b.vpcEndpointConnections[key] = &cp + b.vpcEndpointConnections.Put(&cp) } // ---- AcceptVpcPeeringConnection ---- @@ -635,7 +585,7 @@ func (b *InMemoryBackend) AcceptVpcPeeringConnection( b.mu.Lock("AcceptVpcPeeringConnection") defer b.mu.Unlock() - pc, ok := b.vpcPeeringConnections[vpcPeeringConnectionID] + pc, ok := b.vpcPeeringConnections.Get(vpcPeeringConnectionID) if !ok { return nil, fmt.Errorf("%w: %s", ErrVpcPeeringConnectionNotFound, vpcPeeringConnectionID) } @@ -656,8 +606,7 @@ func (b *InMemoryBackend) AddVpcPeeringConnectionInternal(pc *VpcPeeringConnecti if cp.ExpirationTime.IsZero() { cp.ExpirationTime = time.Now().Add(7 * 24 * time.Hour) } - - b.vpcPeeringConnections[cp.VpcPeeringConnectionID] = &cp + b.vpcPeeringConnections.Put(&cp) } // DescribeVpcPeeringConnections returns VPC peering connections sorted by ID. @@ -673,7 +622,7 @@ func (b *InMemoryBackend) DescribeVpcPeeringConnections(ids []string) []*VpcPeer var result []*VpcPeeringConnection - for _, pc := range b.vpcPeeringConnections { + for _, pc := range b.vpcPeeringConnections.All() { if len(idSet) > 0 && !idSet[pc.VpcPeeringConnectionID] { continue } @@ -701,13 +650,13 @@ func (b *InMemoryBackend) AdvertiseByoipCidr(cidr string) (*ByoipCidr, error) { b.mu.Lock("AdvertiseByoipCidr") defer b.mu.Unlock() - entry, ok := b.byoipCidrs[cidr] + entry, ok := b.byoipCidrs.Get(cidr) if !ok { entry = &ByoipCidr{ Cidr: cidr, State: stateByoipAdvertised, } - b.byoipCidrs[cidr] = entry + b.byoipCidrs.Put(entry) } else { entry.State = stateByoipAdvertised } @@ -724,7 +673,7 @@ func (b *InMemoryBackend) AddByoipCidrInternal(cidr *ByoipCidr) { defer b.mu.Unlock() cp := *cidr - b.byoipCidrs[cp.Cidr] = &cp + b.byoipCidrs.Put(&cp) } // DescribeByoipCidrs returns BYOIP CIDRs sorted by CIDR string. @@ -735,7 +684,7 @@ func (b *InMemoryBackend) DescribeByoipCidrs(state string) []*ByoipCidr { var result []*ByoipCidr - for _, c := range b.byoipCidrs { + for _, c := range b.byoipCidrs.All() { if state != "" && c.State != state { continue } @@ -788,8 +737,7 @@ func (b *InMemoryBackend) AllocateHosts( HostRecovery: hostSettingOff, HostMaintenance: hostSettingOn, } - - b.dedicatedHosts[host.HostID] = host + b.dedicatedHosts.Put(host) cp := *host hosts = append(hosts, &cp) @@ -811,7 +759,7 @@ func (b *InMemoryBackend) DescribeHosts(ids []string) []*Host { var result []*Host - for _, h := range b.dedicatedHosts { + for _, h := range b.dedicatedHosts.All() { if len(idSet) > 0 && !idSet[h.HostID] { continue } diff --git a/services/ec2/backend_accuracy.go b/services/ec2/backend_accuracy.go index a4a16cd07..72af3db0c 100644 --- a/services/ec2/backend_accuracy.go +++ b/services/ec2/backend_accuracy.go @@ -66,7 +66,7 @@ func (b *InMemoryBackend) SetInstanceAttribute(instanceID, attribute, value stri b.mu.Lock("SetInstanceAttribute") defer b.mu.Unlock() - inst, ok := b.instances[instanceID] + inst, ok := b.instances.Get(instanceID) if !ok { return fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -76,6 +76,36 @@ func (b *InMemoryBackend) SetInstanceAttribute(instanceID, attribute, value stri ErrInvalidInstanceState, instanceID, attribute) } + if setSimpleInstanceAttributeLocked(inst, attribute, value) { + return nil + } + + switch attribute { + case attrSourceDest: + // sourceDestCheck lives on the primary network interface's + // attachment in real AWS, not on the instance itself. + if eni := b.primaryNetworkInterfaceLocked(instanceID); eni != nil { + eni.SourceDestCheck = value == ec2BooleanTrue + } + case "groupSet", "blockDeviceMapping": + // accepted but not modelled beyond acknowledgment: changing an + // instance's security groups / block device mappings via + // ModifyInstanceAttribute has no dedicated backend representation + // distinct from the SG-membership and volume-attachment ops. + default: + return fmt.Errorf("%w: unsupported attribute %q", ErrInvalidParameter, attribute) + } + + return nil +} + +// setSimpleInstanceAttributeLocked handles the ModifyInstanceAttribute +// attributes that map directly onto a single Instance field. Reports whether +// it recognised (and applied) the attribute; attrSourceDest and the +// acknowledged-but-unmodelled attributes are handled by the caller since they +// need backend/lock context beyond the Instance struct. Must be called with +// b.mu held. +func setSimpleInstanceAttributeLocked(inst *Instance, attribute, value string) bool { switch attribute { case attrUserData: // AWS stores userData base64-encoded; accept both encoded and raw. @@ -86,15 +116,19 @@ func (b *InMemoryBackend) SetInstanceAttribute(instanceID, attribute, value stri inst.EnaSupport = value == ec2BooleanTrue case attrSriovNetSupport: inst.SriovNetSupport = value - case attrDisableAPITermination, attrDisableAPIStop, attrEBSOptimized, - "sourceDestCheck", attrInstanceInitiatedShutdownBehavior, - "groupSet", "blockDeviceMapping": - // accepted but not modelled beyond acknowledgment + case attrDisableAPITermination: + inst.DisableAPITermination = value == ec2BooleanTrue + case attrDisableAPIStop: + inst.DisableAPIStop = value == ec2BooleanTrue + case attrEBSOptimized: + inst.EBSOptimized = value == ec2BooleanTrue + case attrInstanceInitiatedShutdownBehavior: + inst.InstanceInitiatedShutdownBehavior = value default: - return fmt.Errorf("%w: unsupported attribute %q", ErrInvalidParameter, attribute) + return false } - return nil + return true } // SetVolumeEncryption marks a volume as encrypted and records its KMS key ID. @@ -108,7 +142,7 @@ func (b *InMemoryBackend) SetVolumeEncryption( b.mu.Lock("SetVolumeEncryption") defer b.mu.Unlock() - vol, ok := b.volumes[volumeID] + vol, ok := b.volumes.Get(volumeID) if !ok { return fmt.Errorf("%w: %s", ErrVolumeNotFound, volumeID) } @@ -134,7 +168,7 @@ func (b *InMemoryBackend) SetVolumePerformance(volumeID string, iops, throughput b.mu.Lock("SetVolumePerformance") defer b.mu.Unlock() - vol, ok := b.volumes[volumeID] + vol, ok := b.volumes.Get(volumeID) if !ok { return fmt.Errorf("%w: %s", ErrVolumeNotFound, volumeID) } diff --git a/services/ec2/backend_advanced_networking.go b/services/ec2/backend_advanced_networking.go index b7e69058e..af9ff2f50 100644 --- a/services/ec2/backend_advanced_networking.go +++ b/services/ec2/backend_advanced_networking.go @@ -345,18 +345,7 @@ type IpamResourceDiscoveryAssociation struct { // resetAdvancedNetworkingMapsLocked re-initialises all advanced-networking maps. // Must be called with b.mu held. func (b *InMemoryBackend) resetAdvancedNetworkingMapsLocked() { - b.vpnGateways = make(map[string]*VpnGateway) - b.customerGateways = make(map[string]*CustomerGateway) - b.vpnConnections = make(map[string]*VpnConnection) - b.vpcEndpointServiceConfigs = make(map[string]*VpcEndpointServiceConfig) - b.ipams = make(map[string]*Ipam) - b.ipamScopes = make(map[string]*IpamScope) - b.ipamPools = make(map[string]*IpamPool) b.ipamPoolCidrs = make(map[string][]*IpamPoolCidr) - b.ipamPoolAllocations = make(map[string]*IpamPoolAllocation) - b.ipamResourceDiscoveries = make(map[string]*IpamResourceDiscovery) - b.ipamResourceDiscoveryAssocs = make(map[string]*IpamResourceDiscoveryAssociation) - b.spotFleets = make(map[string]*SpotFleetRequest) b.spotFleetHistory = make(map[string][]SpotFleetHistoryRecord) } @@ -376,7 +365,7 @@ func (b *InMemoryBackend) CreateVpnGateway(gatewayType string) (*VpnGateway, err State: stateAvailable, Type: gatewayType, } - b.vpnGateways[vgw.VpnGatewayID] = vgw + b.vpnGateways.Put(vgw) cp := *vgw @@ -393,9 +382,9 @@ func (b *InMemoryBackend) DescribeVpnGateways(ids []string) []*VpnGateway { idSet[id] = true } - out := make([]*VpnGateway, 0, len(b.vpnGateways)) + out := make([]*VpnGateway, 0, b.vpnGateways.Len()) - for _, vgw := range b.vpnGateways { + for _, vgw := range b.vpnGateways.All() { if len(idSet) > 0 && !idSet[vgw.VpnGatewayID] { continue } @@ -420,11 +409,10 @@ func (b *InMemoryBackend) DeleteVpnGateway(id string) error { b.mu.Lock("DeleteVpnGateway") defer b.mu.Unlock() - if _, ok := b.vpnGateways[id]; !ok { + if _, ok := b.vpnGateways.Get(id); !ok { return fmt.Errorf("%w: %s", ErrVpnGatewayNotFound, id) } - - delete(b.vpnGateways, id) + b.vpnGateways.Delete(id) return nil } @@ -442,12 +430,12 @@ func (b *InMemoryBackend) AttachVpnGateway(vgwID, vpcID string) error { b.mu.Lock("AttachVpnGateway") defer b.mu.Unlock() - vgw, ok := b.vpnGateways[vgwID] + vgw, ok := b.vpnGateways.Get(vgwID) if !ok { return fmt.Errorf("%w: %s", ErrVpnGatewayNotFound, vgwID) } - if _, exists := b.vpcs[vpcID]; !exists { + if _, exists := b.vpcs.Get(vpcID); !exists { return fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -466,7 +454,7 @@ func (b *InMemoryBackend) DetachVpnGateway(vgwID, vpcID string) error { b.mu.Lock("DetachVpnGateway") defer b.mu.Unlock() - vgw, ok := b.vpnGateways[vgwID] + vgw, ok := b.vpnGateways.Get(vgwID) if !ok { return fmt.Errorf("%w: %s", ErrVpnGatewayNotFound, vgwID) } @@ -506,7 +494,7 @@ func (b *InMemoryBackend) CreateCustomerGateway( BgpAsn: bgpAsn, IPAddress: ipAddress, } - b.customerGateways[cgw.CustomerGatewayID] = cgw + b.customerGateways.Put(cgw) cp := *cgw @@ -523,9 +511,9 @@ func (b *InMemoryBackend) DescribeCustomerGateways(ids []string) []*CustomerGate idSet[id] = true } - out := make([]*CustomerGateway, 0, len(b.customerGateways)) + out := make([]*CustomerGateway, 0, b.customerGateways.Len()) - for _, cgw := range b.customerGateways { + for _, cgw := range b.customerGateways.All() { if len(idSet) > 0 && !idSet[cgw.CustomerGatewayID] { continue } @@ -550,11 +538,10 @@ func (b *InMemoryBackend) DeleteCustomerGateway(id string) error { b.mu.Lock("DeleteCustomerGateway") defer b.mu.Unlock() - if _, ok := b.customerGateways[id]; !ok { + if _, ok := b.customerGateways.Get(id); !ok { return fmt.Errorf("%w: %s", ErrCustomerGatewayNotFound, id) } - - delete(b.customerGateways, id) + b.customerGateways.Delete(id) return nil } @@ -580,11 +567,11 @@ func (b *InMemoryBackend) CreateVpnConnection( b.mu.Lock("CreateVpnConnection") defer b.mu.Unlock() - if _, ok := b.customerGateways[customerGatewayID]; !ok { + if _, ok := b.customerGateways.Get(customerGatewayID); !ok { return nil, fmt.Errorf("%w: %s", ErrCustomerGatewayNotFound, customerGatewayID) } - if _, ok := b.vpnGateways[vpnGatewayID]; !ok { + if _, ok := b.vpnGateways.Get(vpnGatewayID); !ok { return nil, fmt.Errorf("%w: %s", ErrVpnGatewayNotFound, vpnGatewayID) } @@ -596,11 +583,10 @@ func (b *InMemoryBackend) CreateVpnConnection( Type: connType, Category: "VPN", } - conn.Options = VpnConnectionOptions{TunnelOptions: generateVpnTunnels(len(b.vpnConnections))} + conn.Options = VpnConnectionOptions{TunnelOptions: generateVpnTunnels(b.vpnConnections.Len())} conn.VgwTelemetry = vgwTelemetryFromTunnels(conn.Options.TunnelOptions) conn.CustomerGatewayConfiguration = buildCustomerGatewayConfiguration(conn) - - b.vpnConnections[conn.VpnConnectionID] = conn + b.vpnConnections.Put(conn) return copyVpnConnection(conn), nil } @@ -615,9 +601,9 @@ func (b *InMemoryBackend) DescribeVpnConnections(ids []string) []*VpnConnection idSet[id] = true } - out := make([]*VpnConnection, 0, len(b.vpnConnections)) + out := make([]*VpnConnection, 0, b.vpnConnections.Len()) - for _, conn := range b.vpnConnections { + for _, conn := range b.vpnConnections.All() { if len(idSet) > 0 && !idSet[conn.VpnConnectionID] { continue } @@ -641,16 +627,16 @@ func (b *InMemoryBackend) DeleteVpnConnection(id string) error { b.mu.Lock("DeleteVpnConnection") defer b.mu.Unlock() - if _, ok := b.vpnConnections[id]; !ok { + if _, ok := b.vpnConnections.Get(id); !ok { return fmt.Errorf("%w: %s", ErrVpnConnectionNotFound, id) } - - delete(b.vpnConnections, id) + b.vpnConnections.Delete(id) prefix := id + ":" - for key := range b.vpnConnectionRoutes { + for _, route := range b.vpnConnectionRoutes.All() { + key := vpnConnectionRoutesKeyFn(route) if strings.HasPrefix(key, prefix) { - delete(b.vpnConnectionRoutes, key) + b.vpnConnectionRoutes.Delete(key) } } @@ -665,7 +651,8 @@ func (b *InMemoryBackend) GetVpnConnectionRoutes(vpnConnectionID string) []*VpnC prefix := vpnConnectionID + ":" out := make([]*VpnConnectionRoute, 0) - for key, route := range b.vpnConnectionRoutes { + for _, route := range b.vpnConnectionRoutes.All() { + key := vpnConnectionRoutesKeyFn(route) if !strings.HasPrefix(key, prefix) { continue } @@ -807,7 +794,7 @@ func (b *InMemoryBackend) ModifyVpnConnectionOptions( b.mu.Lock("ModifyVpnConnectionOptions") defer b.mu.Unlock() - conn, ok := b.vpnConnections[vpnConnectionID] + conn, ok := b.vpnConnections.Get(vpnConnectionID) if !ok { return nil, fmt.Errorf("%w: %s", ErrVpnConnectionNotFound, vpnConnectionID) } @@ -842,7 +829,7 @@ func (b *InMemoryBackend) ModifyVpnTunnelOptions( b.mu.Lock("ModifyVpnTunnelOptions") defer b.mu.Unlock() - conn, ok := b.vpnConnections[vpnConnectionID] + conn, ok := b.vpnConnections.Get(vpnConnectionID) if !ok { return nil, fmt.Errorf("%w: %s", ErrVpnConnectionNotFound, vpnConnectionID) } @@ -913,7 +900,7 @@ func (b *InMemoryBackend) ModifyVpnTunnelCertificate( b.mu.Lock("ModifyVpnTunnelCertificate") defer b.mu.Unlock() - conn, ok := b.vpnConnections[vpnConnectionID] + conn, ok := b.vpnConnections.Get(vpnConnectionID) if !ok { return nil, fmt.Errorf("%w: %s", ErrVpnConnectionNotFound, vpnConnectionID) } @@ -999,7 +986,7 @@ func (b *InMemoryBackend) GetVpnConnectionDeviceSampleConfiguration( b.mu.RLock("GetVpnConnectionDeviceSampleConfiguration") defer b.mu.RUnlock() - conn, ok := b.vpnConnections[vpnConnectionID] + conn, ok := b.vpnConnections.Get(vpnConnectionID) if !ok { return "", fmt.Errorf("%w: %s", ErrVpnConnectionNotFound, vpnConnectionID) } @@ -1043,7 +1030,7 @@ func (b *InMemoryBackend) GetVpnTunnelReplacementStatus( b.mu.RLock("GetVpnTunnelReplacementStatus") defer b.mu.RUnlock() - conn, ok := b.vpnConnections[vpnConnectionID] + conn, ok := b.vpnConnections.Get(vpnConnectionID) if !ok { return nil, fmt.Errorf("%w: %s", ErrVpnConnectionNotFound, vpnConnectionID) } @@ -1076,7 +1063,7 @@ func (b *InMemoryBackend) RejectVpcPeeringConnection(id string) error { b.mu.Lock("RejectVpcPeeringConnection") defer b.mu.Unlock() - pc, ok := b.vpcPeeringConnections[id] + pc, ok := b.vpcPeeringConnections.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrVpcPeeringConnectionNotFound, id) } @@ -1105,7 +1092,7 @@ func (b *InMemoryBackend) CreateVpcEndpointServiceConfiguration( AcceptanceRequired: acceptanceRequired, NetworkLoadBalancerARNs: nlbARNs, } - b.vpcEndpointServiceConfigs[svcID] = cfg + b.vpcEndpointServiceConfigs.Put(cfg) cp := *cfg cp.NetworkLoadBalancerARNs = append([]string(nil), cfg.NetworkLoadBalancerARNs...) @@ -1125,9 +1112,9 @@ func (b *InMemoryBackend) DescribeVpcEndpointServiceConfigurations( idSet[id] = true } - out := make([]*VpcEndpointServiceConfig, 0, len(b.vpcEndpointServiceConfigs)) + out := make([]*VpcEndpointServiceConfig, 0, b.vpcEndpointServiceConfigs.Len()) - for _, cfg := range b.vpcEndpointServiceConfigs { + for _, cfg := range b.vpcEndpointServiceConfigs.All() { if len(idSet) > 0 && !idSet[cfg.ServiceID] { continue } @@ -1150,13 +1137,13 @@ func (b *InMemoryBackend) DeleteVpcEndpointServiceConfigurations(ids []string) e defer b.mu.Unlock() for _, id := range ids { - if _, ok := b.vpcEndpointServiceConfigs[id]; !ok { + if _, ok := b.vpcEndpointServiceConfigs.Get(id); !ok { return fmt.Errorf("%w: %s", ErrVpcEndpointServiceNotFound, id) } } for _, id := range ids { - delete(b.vpcEndpointServiceConfigs, id) + b.vpcEndpointServiceConfigs.Delete(id) } return nil @@ -1174,7 +1161,7 @@ func (b *InMemoryBackend) ModifyVpcEndpointServiceConfiguration( b.mu.Lock("ModifyVpcEndpointServiceConfiguration") defer b.mu.Unlock() - cfg, ok := b.vpcEndpointServiceConfigs[id] + cfg, ok := b.vpcEndpointServiceConfigs.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrVpcEndpointServiceNotFound, id) } @@ -1195,7 +1182,7 @@ func (b *InMemoryBackend) StartVpcEndpointServicePrivateDNSVerification(id strin b.mu.Lock("StartVpcEndpointServicePrivateDnsVerification") defer b.mu.Unlock() - cfg, ok := b.vpcEndpointServiceConfigs[id] + cfg, ok := b.vpcEndpointServiceConfigs.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrVpcEndpointServiceNotFound, id) } @@ -1235,7 +1222,7 @@ func (b *InMemoryBackend) CreateIpam(opts ...IpamOptions) (*Ipam, error) { State: ipamStateCreateComplete, } privScope.IpamScopeARN += privScope.IpamScopeID - b.ipamScopes[privScope.IpamScopeID] = privScope + b.ipamScopes.Put(privScope) pubScope := &IpamScope{ IpamScopeID: "ipam-scope-" + uuid.New().String()[:8], @@ -1246,7 +1233,7 @@ func (b *InMemoryBackend) CreateIpam(opts ...IpamOptions) (*Ipam, error) { State: ipamStateCreateComplete, } pubScope.IpamScopeARN += pubScope.IpamScopeID - b.ipamScopes[pubScope.IpamScopeID] = pubScope + b.ipamScopes.Put(pubScope) discovery := &IpamResourceDiscovery{ IpamResourceDiscoveryID: "ipam-res-disco-" + uuid.New().String()[:8], @@ -1257,7 +1244,7 @@ func (b *InMemoryBackend) CreateIpam(opts ...IpamOptions) (*Ipam, error) { } discovery.IpamResourceDiscoveryARN = "arn:aws:ec2:" + b.Region + ":" + b.AccountID + ":ipam-resource-discovery/" + discovery.IpamResourceDiscoveryID - b.ipamResourceDiscoveries[discovery.IpamResourceDiscoveryID] = discovery + b.ipamResourceDiscoveries.Put(discovery) assoc := &IpamResourceDiscoveryAssociation{ IpamResourceDiscoveryAssociationID: "ipam-res-disco-assoc-" + uuid.New().String()[:8], @@ -1272,7 +1259,7 @@ func (b *InMemoryBackend) CreateIpam(opts ...IpamOptions) (*Ipam, error) { } assoc.IpamResourceDiscoveryAssociationARN = "arn:aws:ec2:" + b.Region + ":" + b.AccountID + ":ipam-resource-discovery-association/" + assoc.IpamResourceDiscoveryAssociationID - b.ipamResourceDiscoveryAssocs[assoc.IpamResourceDiscoveryAssociationID] = assoc + b.ipamResourceDiscoveryAssocs.Put(assoc) ipam := &Ipam{ IpamID: ipamID, @@ -1293,8 +1280,7 @@ func (b *InMemoryBackend) CreateIpam(opts ...IpamOptions) (*Ipam, error) { if ipam.Tier == "" { ipam.Tier = "advanced" } - - b.ipams[ipamID] = ipam + b.ipams.Put(ipam) cp := *ipam cp.OperatingRegions = append([]string(nil), ipam.OperatingRegions...) @@ -1312,9 +1298,9 @@ func (b *InMemoryBackend) DescribeIpams(ids []string) []*Ipam { idSet[id] = true } - out := make([]*Ipam, 0, len(b.ipams)) + out := make([]*Ipam, 0, b.ipams.Len()) - for _, ipam := range b.ipams { + for _, ipam := range b.ipams.All() { if len(idSet) > 0 && !idSet[ipam.IpamID] { continue } @@ -1340,7 +1326,7 @@ func (b *InMemoryBackend) ModifyIpam(id string, opts IpamOptions) (*Ipam, error) b.mu.Lock("ModifyIpam") defer b.mu.Unlock() - ipam, ok := b.ipams[id] + ipam, ok := b.ipams.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamNotFound, id) } @@ -1374,16 +1360,15 @@ func (b *InMemoryBackend) DeleteIpam(id string) error { b.mu.Lock("DeleteIpam") defer b.mu.Unlock() - ipam, ok := b.ipams[id] + ipam, ok := b.ipams.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrIpamNotFound, id) } - - delete(b.ipamScopes, ipam.PublicDefaultScopeID) - delete(b.ipamScopes, ipam.PrivateDefaultScopeID) - delete(b.ipamResourceDiscoveries, ipam.DefaultResourceDiscoveryID) - delete(b.ipamResourceDiscoveryAssocs, ipam.DefaultResourceDiscoveryAssociationID) - delete(b.ipams, id) + b.ipamScopes.Delete(ipam.PublicDefaultScopeID) + b.ipamScopes.Delete(ipam.PrivateDefaultScopeID) + b.ipamResourceDiscoveries.Delete(ipam.DefaultResourceDiscoveryID) + b.ipamResourceDiscoveryAssocs.Delete(ipam.DefaultResourceDiscoveryAssociationID) + b.ipams.Delete(id) return nil } @@ -1399,7 +1384,7 @@ func (b *InMemoryBackend) CreateIpamScope(ipamID, description string) (*IpamScop b.mu.Lock("CreateIpamScope") defer b.mu.Unlock() - ipam, ok := b.ipams[ipamID] + ipam, ok := b.ipams.Get(ipamID) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamNotFound, ipamID) } @@ -1413,7 +1398,7 @@ func (b *InMemoryBackend) CreateIpamScope(ipamID, description string) (*IpamScop State: ipamStateCreateComplete, Description: description, } - b.ipamScopes[scopeID] = scope + b.ipamScopes.Put(scope) ipam.ScopeCount++ cp := *scope @@ -1431,9 +1416,9 @@ func (b *InMemoryBackend) DescribeIpamScopes(ids []string) []*IpamScope { idSet[id] = true } - out := make([]*IpamScope, 0, len(b.ipamScopes)) + out := make([]*IpamScope, 0, b.ipamScopes.Len()) - for _, scope := range b.ipamScopes { + for _, scope := range b.ipamScopes.All() { if len(idSet) > 0 && !idSet[scope.IpamScopeID] { continue } @@ -1458,7 +1443,7 @@ func (b *InMemoryBackend) ModifyIpamScope(id, description string) (*IpamScope, e b.mu.Lock("ModifyIpamScope") defer b.mu.Unlock() - scope, ok := b.ipamScopes[id] + scope, ok := b.ipamScopes.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamScopeNotFound, id) } @@ -1480,7 +1465,7 @@ func (b *InMemoryBackend) DeleteIpamScope(id string) error { b.mu.Lock("DeleteIpamScope") defer b.mu.Unlock() - scope, ok := b.ipamScopes[id] + scope, ok := b.ipamScopes.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrIpamScopeNotFound, id) } @@ -1489,11 +1474,10 @@ func (b *InMemoryBackend) DeleteIpamScope(id string) error { return fmt.Errorf("%w: default IPAM scopes cannot be deleted", ErrIpamScopeDefault) } - if ipam, ipamOK := b.ipams[scope.IpamID]; ipamOK { + if ipam, ipamOK := b.ipams.Get(scope.IpamID); ipamOK { ipam.ScopeCount-- } - - delete(b.ipamScopes, id) + b.ipamScopes.Delete(id) return nil } @@ -1528,7 +1512,7 @@ func (b *InMemoryBackend) CreateIpamPool( b.mu.Lock("CreateIpamPool") defer b.mu.Unlock() - ipam, ok := b.ipams[ipamID] + ipam, ok := b.ipams.Get(ipamID) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamNotFound, ipamID) } @@ -1559,7 +1543,7 @@ func (b *InMemoryBackend) CreateIpamPool( AllocationMaxNetmaskLength: o.AllocationMaxNetmaskLength, AllocationDefaultNetmaskLength: o.AllocationDefaultNetmaskLength, } - b.ipamPools[poolID] = pool + b.ipamPools.Put(pool) if cidr != "" { b.ipamPoolCidrs[poolID] = []*IpamPoolCidr{{Cidr: cidr, State: ipamPoolCidrStateProvisioned}} @@ -1580,9 +1564,9 @@ func (b *InMemoryBackend) DescribeIpamPools(ids []string) []*IpamPool { idSet[id] = true } - out := make([]*IpamPool, 0, len(b.ipamPools)) + out := make([]*IpamPool, 0, b.ipamPools.Len()) - for _, pool := range b.ipamPools { + for _, pool := range b.ipamPools.All() { if len(idSet) > 0 && !idSet[pool.IpamPoolID] { continue } @@ -1607,7 +1591,7 @@ func (b *InMemoryBackend) ModifyIpamPool(id string, opts IpamPoolOptions) (*Ipam b.mu.Lock("ModifyIpamPool") defer b.mu.Unlock() - pool, ok := b.ipamPools[id] + pool, ok := b.ipamPools.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPoolNotFound, id) } @@ -1646,11 +1630,10 @@ func (b *InMemoryBackend) DeleteIpamPool(id string) error { b.mu.Lock("DeleteIpamPool") defer b.mu.Unlock() - if _, ok := b.ipamPools[id]; !ok { + if _, ok := b.ipamPools.Get(id); !ok { return fmt.Errorf("%w: %s", ErrIpamPoolNotFound, id) } - - delete(b.ipamPools, id) + b.ipamPools.Delete(id) delete(b.ipamPoolCidrs, id) return nil @@ -1671,7 +1654,7 @@ func (b *InMemoryBackend) ProvisionIpamPoolCidr(poolID, cidr string) (*IpamPoolC b.mu.Lock("ProvisionIpamPoolCidr") defer b.mu.Unlock() - if _, ok := b.ipamPools[poolID]; !ok { + if _, ok := b.ipamPools.Get(poolID); !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPoolNotFound, poolID) } @@ -1696,7 +1679,7 @@ func (b *InMemoryBackend) DeprovisionIpamPoolCidr(poolID, cidr string) (*IpamPoo b.mu.Lock("DeprovisionIpamPoolCidr") defer b.mu.Unlock() - if _, ok := b.ipamPools[poolID]; !ok { + if _, ok := b.ipamPools.Get(poolID); !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPoolNotFound, poolID) } @@ -1753,7 +1736,7 @@ func (b *InMemoryBackend) autoCIDRLocked(poolCidr string, netmaskLength int) (st return "", fmt.Errorf("%w: invalid pool CIDR %s", ErrInvalidParameter, poolCidr) } - existingCount := len(b.ipamPoolAllocations) + existingCount := b.ipamPoolAllocations.Len() ip := network.IP.To4() if ip == nil { @@ -1811,7 +1794,7 @@ func (b *InMemoryBackend) AllocateIpamPoolCidr( b.mu.Lock("AllocateIpamPoolCidr") defer b.mu.Unlock() - pool, ok := b.ipamPools[poolID] + pool, ok := b.ipamPools.Get(poolID) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPoolNotFound, poolID) } @@ -1838,8 +1821,7 @@ func (b *InMemoryBackend) AllocateIpamPoolCidr( if alloc.ResourceType == "" { alloc.ResourceType = "custom" } - - b.ipamPoolAllocations[alloc.IpamPoolAllocationID] = alloc + b.ipamPoolAllocations.Put(alloc) b.recordIpamResourceCidrLocked(pool, alloc) cp := *alloc @@ -1857,13 +1839,13 @@ func (b *InMemoryBackend) GetIpamPoolAllocations(poolID, allocationID string) ([ b.mu.RLock("GetIpamPoolAllocations") defer b.mu.RUnlock() - if _, ok := b.ipamPools[poolID]; !ok { + if _, ok := b.ipamPools.Get(poolID); !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPoolNotFound, poolID) } - out := make([]*IpamPoolAllocation, 0, len(b.ipamPoolAllocations)) + out := make([]*IpamPoolAllocation, 0, b.ipamPoolAllocations.Len()) - for _, alloc := range b.ipamPoolAllocations { + for _, alloc := range b.ipamPoolAllocations.All() { if alloc.IpamPoolID != poolID { continue } @@ -1892,7 +1874,7 @@ func (b *InMemoryBackend) ReleaseIpamPoolAllocation(poolID, allocationID string) b.mu.Lock("ReleaseIpamPoolAllocation") defer b.mu.Unlock() - alloc, ok := b.ipamPoolAllocations[allocationID] + alloc, ok := b.ipamPoolAllocations.Get(allocationID) if !ok { return fmt.Errorf("%w: %s", ErrIpamAllocationNotFound, allocationID) } @@ -1902,7 +1884,7 @@ func (b *InMemoryBackend) ReleaseIpamPoolAllocation(poolID, allocationID string) } b.forgetIpamResourceCidrLocked(alloc) - delete(b.ipamPoolAllocations, allocationID) + b.ipamPoolAllocations.Delete(allocationID) return nil } @@ -1919,9 +1901,9 @@ func (b *InMemoryBackend) DescribeIpamResourceDiscoveries(ids []string) []*IpamR idSet[id] = true } - out := make([]*IpamResourceDiscovery, 0, len(b.ipamResourceDiscoveries)) + out := make([]*IpamResourceDiscovery, 0, b.ipamResourceDiscoveries.Len()) - for _, d := range b.ipamResourceDiscoveries { + for _, d := range b.ipamResourceDiscoveries.All() { if len(idSet) > 0 && !idSet[d.IpamResourceDiscoveryID] { continue } @@ -1951,9 +1933,9 @@ func (b *InMemoryBackend) DescribeIpamResourceDiscoveryAssociations( idSet[id] = true } - out := make([]*IpamResourceDiscoveryAssociation, 0, len(b.ipamResourceDiscoveryAssocs)) + out := make([]*IpamResourceDiscoveryAssociation, 0, b.ipamResourceDiscoveryAssocs.Len()) - for _, a := range b.ipamResourceDiscoveryAssocs { + for _, a := range b.ipamResourceDiscoveryAssocs.All() { if len(idSet) > 0 && !idSet[a.IpamResourceDiscoveryAssociationID] { continue } diff --git a/services/ec2/backend_batch1.go b/services/ec2/backend_batch1.go index 65c6f9543..826be6561 100644 --- a/services/ec2/backend_batch1.go +++ b/services/ec2/backend_batch1.go @@ -100,7 +100,7 @@ func (b *InMemoryBackend) ModifyVolume( b.mu.Lock("ModifyVolume") defer b.mu.Unlock() - vol, ok := b.volumes[volumeID] + vol, ok := b.volumes.Get(volumeID) if !ok { return nil, fmt.Errorf("%w: %s", ErrVolumeNotFound, volumeID) } @@ -129,8 +129,7 @@ func (b *InMemoryBackend) ModifyVolume( } else { mod.TargetSize = vol.Size } - - b.volumeModifications[volumeID] = mod + b.volumeModifications.Put(mod) return mod, nil } @@ -156,7 +155,7 @@ func (b *InMemoryBackend) DescribeVolumeStatus(ids []string) []VolumeStatusItem } var out []VolumeStatusItem - for _, vol := range b.volumes { + for _, vol := range b.volumes.All() { if len(filter) > 0 && !filter[vol.ID] { continue } @@ -189,7 +188,7 @@ func (b *InMemoryBackend) DescribeVolumesModifications(ids []string) []*VolumeMo } var out []*VolumeModification - for _, mod := range b.volumeModifications { + for _, mod := range b.volumeModifications.All() { if len(filter) > 0 && !filter[mod.VolumeID] { continue } @@ -212,7 +211,7 @@ func (b *InMemoryBackend) CopySnapshot(sourceSnapshotID, description string) (*S b.mu.Lock("CopySnapshot") defer b.mu.Unlock() - src, ok := b.snapshots[sourceSnapshotID] + src, ok := b.snapshots.Get(sourceSnapshotID) if !ok { return nil, fmt.Errorf("%w: %s", ErrSnapshotNotFound, sourceSnapshotID) } @@ -233,7 +232,7 @@ func (b *InMemoryBackend) CopySnapshot(sourceSnapshotID, description string) (*S Encrypted: src.Encrypted, KmsKeyID: src.KmsKeyID, } - b.snapshots[snap.SnapshotID] = snap + b.snapshots.Put(snap) return snap, nil } @@ -261,14 +260,14 @@ func (b *InMemoryBackend) CreateSnapshots( defer b.mu.Unlock() for _, vid := range volumeIDs { - if _, ok := b.volumes[vid]; !ok { + if _, ok := b.volumes.Get(vid); !ok { return nil, fmt.Errorf("%w: %s", ErrVolumeNotFound, vid) } } snaps := make([]*Snapshot, 0, len(volumeIDs)) for _, vid := range volumeIDs { - vol := b.volumes[vid] + vol, _ := b.volumes.Get(vid) snap := &Snapshot{ SnapshotID: "snap-" + uuid.New().String()[:17], VolumeID: vid, @@ -280,7 +279,7 @@ func (b *InMemoryBackend) CreateSnapshots( Encrypted: vol.Encrypted, KmsKeyID: vol.KmsKeyID, } - b.snapshots[snap.SnapshotID] = snap + b.snapshots.Put(snap) snaps = append(snaps, snap) } @@ -347,7 +346,7 @@ func (b *InMemoryBackend) DescribeSnapshotTierStatus(ids []string) []SnapshotTie } var out []SnapshotTierItem - for _, snap := range b.snapshots { + for _, snap := range b.snapshots.All() { if len(filter) > 0 && !filter[snap.SnapshotID] { continue } @@ -375,7 +374,7 @@ func (b *InMemoryBackend) ModifySnapshotTier(snapshotID, storageTier string) err b.mu.Lock("ModifySnapshotTier") defer b.mu.Unlock() - if _, ok := b.snapshots[snapshotID]; !ok { + if _, ok := b.snapshots.Get(snapshotID); !ok { return fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) } b.snapshotTiers[snapshotID] = storageTier @@ -392,7 +391,7 @@ func (b *InMemoryBackend) ResetSnapshotAttribute(snapshotID string) error { b.mu.Lock("ResetSnapshotAttribute") defer b.mu.Unlock() - if _, ok := b.snapshots[snapshotID]; !ok { + if _, ok := b.snapshots.Get(snapshotID); !ok { return fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) } delete(b.snapshotAttributes, snapshotID) @@ -408,7 +407,7 @@ func (b *InMemoryBackend) CreateDefaultVpc() (*VPC, error) { b.mu.Lock("CreateDefaultVpc") defer b.mu.Unlock() - for _, v := range b.vpcs { + for _, v := range b.vpcs.All() { if v.IsDefault { return nil, fmt.Errorf("%w: a default VPC already exists", ErrInvalidParameter) } @@ -419,7 +418,7 @@ func (b *InMemoryBackend) CreateDefaultVpc() (*VPC, error) { CIDRBlock: defaultVPCCIDR, IsDefault: true, } - b.vpcs[vpc.ID] = vpc + b.vpcs.Put(vpc) return vpc, nil } @@ -436,7 +435,7 @@ func (b *InMemoryBackend) CreateDefaultSubnet(az string) (*Subnet, error) { defer b.mu.Unlock() var defaultVPCID string - for _, v := range b.vpcs { + for _, v := range b.vpcs.All() { if v.IsDefault { defaultVPCID = v.ID @@ -455,7 +454,7 @@ func (b *InMemoryBackend) CreateDefaultSubnet(az string) (*Subnet, error) { IsDefault: true, MapPublicIPOnLaunch: true, } - b.subnets[subnet.ID] = subnet + b.subnets.Put(subnet) return subnet, nil } @@ -476,7 +475,7 @@ func (b *InMemoryBackend) AssociateSubnetCidrBlock( b.mu.Lock("AssociateSubnetCidrBlock") defer b.mu.Unlock() - if _, ok := b.subnets[subnetID]; !ok { + if _, ok := b.subnets.Get(subnetID); !ok { return nil, fmt.Errorf("%w: %s", ErrSubnetNotFound, subnetID) } @@ -537,10 +536,10 @@ func (b *InMemoryBackend) AssociateSecurityGroupVpc( b.mu.Lock("AssociateSecurityGroupVpc") defer b.mu.Unlock() - if _, ok := b.securityGroups[sgID]; !ok { + if _, ok := b.securityGroups.Get(sgID); !ok { return nil, fmt.Errorf("%w: %s", ErrSecurityGroupNotFound, sgID) } - if _, ok := b.vpcs[vpcID]; !ok { + if _, ok := b.vpcs.Get(vpcID); !ok { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -601,7 +600,7 @@ func (b *InMemoryBackend) DescribeSecurityGroupReferences(sgIDs []string) []SGRe continue } for vpcID := range vpcMap { - if sg, ok := b.securityGroups[sgID]; ok && sg.VPCID != vpcID { + if sg, ok := b.securityGroups.Get(sgID); ok && sg.VPCID != vpcID { out = append(out, SGReference{ GroupID: sgID, ReferencingVPCID: vpcID, @@ -627,7 +626,7 @@ type StaleSGItem struct { // findDeletedPeerVPCsLocked returns VPC IDs with terminated peering connections to vpcID. func (b *InMemoryBackend) findDeletedPeerVPCsLocked(vpcID string) map[string]bool { result := make(map[string]bool) - for _, pc := range b.vpcPeeringConnections { + for _, pc := range b.vpcPeeringConnections.All() { if pc.State != tgwRouteStateDeleted && pc.State != "rejected" && pc.State != "failed" { continue } @@ -650,7 +649,7 @@ func (b *InMemoryBackend) hasStaleRuleLocked( if rule.SourceGroupID == "" { continue } - srcSG, ok := b.securityGroups[rule.SourceGroupID] + srcSG, ok := b.securityGroups.Get(rule.SourceGroupID) if ok && deletedPeerVPCs[srcSG.VPCID] { return true } @@ -668,7 +667,7 @@ func (b *InMemoryBackend) DescribeStaleSecurityGroups(vpcID string) []StaleSGIte deletedPeerVPCs := b.findDeletedPeerVPCsLocked(vpcID) var out []StaleSGItem - for _, sg := range b.securityGroups { + for _, sg := range b.securityGroups.All() { if sg.VPCID != vpcID || !b.hasStaleRuleLocked(sg, deletedPeerVPCs) { continue } @@ -734,7 +733,7 @@ func (b *InMemoryBackend) ModifyVpcTenancy(vpcID, tenancy string) error { b.mu.Lock("ModifyVpcTenancy") defer b.mu.Unlock() - if _, ok := b.vpcs[vpcID]; !ok { + if _, ok := b.vpcs.Get(vpcID); !ok { return fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } b.vpcTenancy[vpcID] = tenancy @@ -756,7 +755,7 @@ func (b *InMemoryBackend) ModifyVpcPeeringConnectionOptions( b.mu.Lock("ModifyVpcPeeringConnectionOptions") defer b.mu.Unlock() - if _, ok := b.vpcPeeringConnections[peeringID]; !ok { + if _, ok := b.vpcPeeringConnections.Get(peeringID); !ok { return fmt.Errorf("%w: %s", ErrVpcPeeringConnectionNotFound, peeringID) } o := opts @@ -788,7 +787,7 @@ func (b *InMemoryBackend) DescribeAddressesAttribute(allocationIDs []string) []A } var out []AddressAttribute - for _, addr := range b.addresses { + for _, addr := range b.addresses.All() { if len(filter) > 0 && !filter[addr.AllocationID] { continue } @@ -796,7 +795,7 @@ func (b *InMemoryBackend) DescribeAddressesAttribute(allocationIDs []string) []A AllocationID: addr.AllocationID, PublicIP: addr.PublicIP, } - if stored, ok := b.addressAttributes[addr.AllocationID]; ok { + if stored, ok := b.addressAttributes.Get(addr.AllocationID); ok { attr.DomainName = stored.DomainName } out = append(out, attr) @@ -815,15 +814,15 @@ func (b *InMemoryBackend) ModifyAddressAttribute(allocationID, domainName string b.mu.Lock("ModifyAddressAttribute") defer b.mu.Unlock() - addr, ok := b.addresses[allocationID] + addr, ok := b.addresses.Get(allocationID) if !ok { return fmt.Errorf("%w: %s", ErrInvalidParameter, allocationID) } - b.addressAttributes[allocationID] = &AddressAttribute{ + b.addressAttributes.Put(&AddressAttribute{ AllocationID: allocationID, PublicIP: addr.PublicIP, DomainName: domainName, - } + }) return nil } @@ -837,10 +836,10 @@ func (b *InMemoryBackend) ResetAddressAttribute(allocationID string) error { b.mu.Lock("ResetAddressAttribute") defer b.mu.Unlock() - if _, ok := b.addresses[allocationID]; !ok { + if _, ok := b.addresses.Get(allocationID); !ok { return fmt.Errorf("%w: %s", ErrInvalidParameter, allocationID) } - delete(b.addressAttributes, allocationID) + b.addressAttributes.Delete(allocationID) return nil } @@ -857,7 +856,7 @@ func (b *InMemoryBackend) GetConsoleOutput(instanceID string) (string, time.Time b.mu.RLock("GetConsoleOutput") defer b.mu.RUnlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return "", time.Time{}, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -889,7 +888,7 @@ func (b *InMemoryBackend) ModifyInstanceMetadataOptions( b.mu.Lock("ModifyInstanceMetadataOptions") defer b.mu.Unlock() - inst, ok := b.instances[instanceID] + inst, ok := b.instances.Get(instanceID) if !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -992,7 +991,7 @@ func (b *InMemoryBackend) DescribeInstanceCreditSpecifications(ids []string) []I } var out []InstanceCreditSpec - for _, inst := range b.instances { + for _, inst := range b.instances.All() { if len(filter) > 0 && !filter[inst.ID] { continue } @@ -1020,7 +1019,7 @@ func (b *InMemoryBackend) ModifyInstanceCreditSpecification( b.mu.Lock("ModifyInstanceCreditSpecification") defer b.mu.Unlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } b.instanceCreditSpecs[instanceID] = cpuCredits @@ -1051,12 +1050,12 @@ func (b *InMemoryBackend) DescribeInstanceTopology(ids []string) []InstanceTopol } var out []InstanceTopologyItem - for _, inst := range b.instances { + for _, inst := range b.instances.All() { if len(filter) > 0 && !filter[inst.ID] { continue } az := b.Region + "a" - if sub, ok := b.subnets[inst.SubnetID]; ok && sub.AvailabilityZone != "" { + if sub, ok := b.subnets.Get(inst.SubnetID); ok && sub.AvailabilityZone != "" { az = sub.AvailabilityZone } out = append(out, InstanceTopologyItem{ @@ -1091,7 +1090,7 @@ func (b *InMemoryBackend) MonitorInstances(instanceIDs []string) ([]MonitoringSt var out []MonitoringState for _, id := range instanceIDs { - if _, ok := b.instances[id]; !ok { + if _, ok := b.instances.Get(id); !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, id) } b.instanceMonitoring[id] = stateMonitoringEnabled @@ -1112,7 +1111,7 @@ func (b *InMemoryBackend) UnmonitorInstances(instanceIDs []string) ([]Monitoring var out []MonitoringState for _, id := range instanceIDs { - if _, ok := b.instances[id]; !ok { + if _, ok := b.instances.Get(id); !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, id) } b.instanceMonitoring[id] = stateMonitoringDisabled @@ -1143,7 +1142,7 @@ func (b *InMemoryBackend) DescribeNetworkInterfaceAttribute( b.mu.RLock("DescribeNetworkInterfaceAttribute") defer b.mu.RUnlock() - ni, ok := b.networkInterfaces[niID] + ni, ok := b.networkInterfaces.Get(niID) if !ok { return nil, fmt.Errorf("%w: %s", ErrNetworkInterfaceNotFound, niID) } @@ -1164,7 +1163,7 @@ func (b *InMemoryBackend) ResetNetworkInterfaceAttribute(niID string) error { b.mu.Lock("ResetNetworkInterfaceAttribute") defer b.mu.Unlock() - ni, ok := b.networkInterfaces[niID] + ni, ok := b.networkInterfaces.Get(niID) if !ok { return fmt.Errorf("%w: %s", ErrNetworkInterfaceNotFound, niID) } @@ -1188,7 +1187,7 @@ func (b *InMemoryBackend) DescribeNetworkInterfacePermissions( } var out []*NetworkInterfacePermission - for _, perm := range b.niPermissions { + for _, perm := range b.niPermissions.All() { if len(filter) > 0 && !filter[perm.NetworkInterfaceID] { continue } @@ -1211,7 +1210,7 @@ func (b *InMemoryBackend) CreateNetworkInterfacePermission( b.mu.Lock("CreateNetworkInterfacePermission") defer b.mu.Unlock() - if _, ok := b.networkInterfaces[niID]; !ok { + if _, ok := b.networkInterfaces.Get(niID); !ok { return nil, fmt.Errorf("%w: %s", ErrNetworkInterfaceNotFound, niID) } @@ -1223,7 +1222,7 @@ func (b *InMemoryBackend) CreateNetworkInterfacePermission( Permission: permission, State: "granted", } - b.niPermissions[perm.PermissionID] = perm + b.niPermissions.Put(perm) return perm, nil } @@ -1237,10 +1236,10 @@ func (b *InMemoryBackend) DeleteNetworkInterfacePermission(permissionID string) b.mu.Lock("DeleteNetworkInterfacePermission") defer b.mu.Unlock() - if _, ok := b.niPermissions[permissionID]; !ok { + if _, ok := b.niPermissions.Get(permissionID); !ok { return fmt.Errorf("%w: %s", ErrNetworkInterfacePermissionNotFound, permissionID) } - delete(b.niPermissions, permissionID) + b.niPermissions.Delete(permissionID) return nil } @@ -1259,7 +1258,7 @@ func (b *InMemoryBackend) AssignIpv6Addresses(niID string, count int) ([]string, b.mu.Lock("AssignIpv6Addresses") defer b.mu.Unlock() - if _, ok := b.networkInterfaces[niID]; !ok { + if _, ok := b.networkInterfaces.Get(niID); !ok { return nil, fmt.Errorf("%w: %s", ErrNetworkInterfaceNotFound, niID) } @@ -1282,7 +1281,7 @@ func (b *InMemoryBackend) UnassignIpv6Addresses(niID string, addresses []string) b.mu.Lock("UnassignIpv6Addresses") defer b.mu.Unlock() - if _, ok := b.networkInterfaces[niID]; !ok { + if _, ok := b.networkInterfaces.Get(niID); !ok { return fmt.Errorf("%w: %s", ErrNetworkInterfaceNotFound, niID) } diff --git a/services/ec2/backend_batch2.go b/services/ec2/backend_batch2.go index 376a64086..e610a16a5 100644 --- a/services/ec2/backend_batch2.go +++ b/services/ec2/backend_batch2.go @@ -99,7 +99,7 @@ func (b *InMemoryBackend) CreateVpcEndpointConnectionNotification( ConnectionNotificationType: "Topic", ConnectionNotificationState: "Enabled", } - b.endpointConnectionNotifs[notif.ConnectionNotificationID] = notif + b.endpointConnectionNotifs.Put(notif) return notif, nil } @@ -117,7 +117,7 @@ func (b *InMemoryBackend) DescribeVpcEndpointConnectionNotifications( } var out []*VpcEndpointConnectionNotification - for _, n := range b.endpointConnectionNotifs { + for _, n := range b.endpointConnectionNotifs.All() { if len(filter) > 0 && !filter[n.ConnectionNotificationID] { continue } @@ -137,12 +137,12 @@ func (b *InMemoryBackend) DeleteVpcEndpointConnectionNotifications(ids []string) defer b.mu.Unlock() for _, id := range ids { - if _, ok := b.endpointConnectionNotifs[id]; !ok { + if _, ok := b.endpointConnectionNotifs.Get(id); !ok { return fmt.Errorf("%w: %s", ErrEndpointConnectionNotificationNotFound, id) } } for _, id := range ids { - delete(b.endpointConnectionNotifs, id) + b.endpointConnectionNotifs.Delete(id) } return nil @@ -160,7 +160,7 @@ func (b *InMemoryBackend) ModifyVpcEndpointConnectionNotification( b.mu.Lock("ModifyVpcEndpointConnectionNotification") defer b.mu.Unlock() - notif, ok := b.endpointConnectionNotifs[id] + notif, ok := b.endpointConnectionNotifs.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrEndpointConnectionNotificationNotFound, id) } @@ -190,7 +190,7 @@ func (b *InMemoryBackend) DescribeVpcEndpointConnections( } var out []*VpcEndpointConnection - for _, conn := range b.vpcEndpointConnections { + for _, conn := range b.vpcEndpointConnections.All() { if len(filter) > 0 && !filter[conn.ServiceID] { continue } @@ -215,7 +215,7 @@ func (b *InMemoryBackend) DescribeVpcEndpointAssociations(endpointIDs []string) } var out []*VpcEndpoint - for _, ep := range b.vpcEndpoints { + for _, ep := range b.vpcEndpoints.All() { if len(filter) > 0 && !filter[ep.ID] { continue } @@ -240,7 +240,7 @@ func (b *InMemoryBackend) ModifyVpcEndpointServicePayerResponsibility( b.mu.Lock("ModifyVpcEndpointServicePayerResponsibility") defer b.mu.Unlock() - if _, ok := b.vpcEndpointServiceConfigs[serviceID]; !ok { + if _, ok := b.vpcEndpointServiceConfigs.Get(serviceID); !ok { return fmt.Errorf("%w: %s", ErrVpcEndpointServiceNotFound, serviceID) } @@ -267,7 +267,7 @@ func (b *InMemoryBackend) ModifyVpcEndpointServicePermissions( b.mu.Lock("ModifyVpcEndpointServicePermissions") defer b.mu.Unlock() - if _, ok := b.vpcEndpointServiceConfigs[serviceID]; !ok { + if _, ok := b.vpcEndpointServiceConfigs.Get(serviceID); !ok { return fmt.Errorf("%w: %s", ErrVpcEndpointServiceNotFound, serviceID) } @@ -302,7 +302,7 @@ func (b *InMemoryBackend) ModifyVpcEndpoint( b.mu.Lock("ModifyVpcEndpoint") defer b.mu.Unlock() - ep, ok := b.vpcEndpoints[endpointID] + ep, ok := b.vpcEndpoints.Get(endpointID) if !ok { return fmt.Errorf("%w: %s", ErrVpcEndpointNotFound, endpointID) } @@ -387,7 +387,7 @@ func (b *InMemoryBackend) EnableVolumeIO(volumeID string) error { b.mu.Lock("EnableVolumeIO") defer b.mu.Unlock() - if _, ok := b.volumes[volumeID]; !ok { + if _, ok := b.volumes.Get(volumeID); !ok { return fmt.Errorf("%w: %s", ErrVolumeNotFound, volumeID) } @@ -408,10 +408,10 @@ func (b *InMemoryBackend) LockSnapshot( b.mu.Lock("LockSnapshot") defer b.mu.Unlock() - if _, ok := b.snapshots[snapshotID]; !ok { + if _, ok := b.snapshots.Get(snapshotID); !ok { return nil, fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) } - if _, locked := b.snapshotLocks[snapshotID]; locked { + if _, locked := b.snapshotLocks.Get(snapshotID); locked { return nil, fmt.Errorf("%w: %s", ErrSnapshotAlreadyLocked, snapshotID) } @@ -425,7 +425,7 @@ func (b *InMemoryBackend) LockSnapshot( if durationDays > 0 { lock.LockExpiresOn = now.AddDate(0, 0, durationDays) } - b.snapshotLocks[snapshotID] = lock + b.snapshotLocks.Put(lock) return lock, nil } @@ -439,13 +439,13 @@ func (b *InMemoryBackend) UnlockSnapshot(snapshotID string) error { b.mu.Lock("UnlockSnapshot") defer b.mu.Unlock() - if _, ok := b.snapshots[snapshotID]; !ok { + if _, ok := b.snapshots.Get(snapshotID); !ok { return fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) } - if _, locked := b.snapshotLocks[snapshotID]; !locked { + if _, locked := b.snapshotLocks.Get(snapshotID); !locked { return fmt.Errorf("%w: %s", ErrSnapshotNotLocked, snapshotID) } - delete(b.snapshotLocks, snapshotID) + b.snapshotLocks.Delete(snapshotID) return nil } @@ -461,7 +461,7 @@ func (b *InMemoryBackend) DescribeLockedSnapshots(ids []string) []*SnapshotLock } var out []*SnapshotLock - for _, lock := range b.snapshotLocks { + for _, lock := range b.snapshotLocks.All() { if len(filter) > 0 && !filter[lock.SnapshotID] { continue } @@ -494,14 +494,14 @@ func (b *InMemoryBackend) CopyVolumes( defer b.mu.Unlock() for _, id := range volumeIDs { - if _, ok := b.volumes[id]; !ok { + if _, ok := b.volumes.Get(id); !ok { return nil, fmt.Errorf("%w: %s", ErrVolumeNotFound, id) } } results := make([]CopyVolumesResult, 0, len(volumeIDs)) for _, id := range volumeIDs { - src := b.volumes[id] + src, _ := b.volumes.Get(id) newVol := &Volume{ ID: "vol-" + uuid.New().String()[:17], AZ: src.AZ, @@ -512,7 +512,7 @@ func (b *InMemoryBackend) CopyVolumes( KmsKeyID: src.KmsKeyID, CreateTime: time.Now().UTC(), } - b.volumes[newVol.ID] = newVol + b.volumes.Put(newVol) results = append(results, CopyVolumesResult{SourceVolumeID: id, DestVolumeID: newVol.ID}) } @@ -553,7 +553,7 @@ func (b *InMemoryBackend) DisassociateNatGatewayAddress(natGatewayID string) err b.mu.RLock("DisassociateNatGatewayAddress") defer b.mu.RUnlock() - if _, ok := b.natGateways[natGatewayID]; !ok { + if _, ok := b.natGateways.Get(natGatewayID); !ok { return fmt.Errorf("%w: %s", ErrInvalidParameter, natGatewayID) } @@ -569,7 +569,7 @@ func (b *InMemoryBackend) AssociateNatGatewayAddress(natGatewayID, _ string) err b.mu.RLock("AssociateNatGatewayAddress") defer b.mu.RUnlock() - if _, ok := b.natGateways[natGatewayID]; !ok { + if _, ok := b.natGateways.Get(natGatewayID); !ok { return fmt.Errorf("%w: %s", ErrInvalidParameter, natGatewayID) } @@ -587,7 +587,7 @@ func (b *InMemoryBackend) AssignPrivateNatGatewayAddress(natGatewayID string) er b.mu.Lock("AssignPrivateNatGatewayAddress") defer b.mu.Unlock() - ngw, ok := b.natGateways[natGatewayID] + ngw, ok := b.natGateways.Get(natGatewayID) if !ok { return fmt.Errorf("%w: %s", ErrInvalidParameter, natGatewayID) } @@ -766,7 +766,7 @@ func (b *InMemoryBackend) DescribeInstanceImageMetadata( } var out []InstanceImageMetadataItem - for _, inst := range b.instances { + for _, inst := range b.instances.All() { if len(filter) > 0 && !filter[inst.ID] { continue } @@ -884,7 +884,7 @@ func (b *InMemoryBackend) CreateReplaceRootVolumeTask( b.mu.Lock("CreateReplaceRootVolumeTask") defer b.mu.Unlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -896,7 +896,7 @@ func (b *InMemoryBackend) CreateReplaceRootVolumeTask( StartTime: time.Now().UTC(), } task.CompleteTime = task.StartTime - b.replaceRootVolumeTasks[task.ReplaceRootVolumeTaskID] = task + b.replaceRootVolumeTasks.Put(task) return task, nil } @@ -912,7 +912,7 @@ func (b *InMemoryBackend) DescribeReplaceRootVolumeTasks(ids []string) []*Replac } var out []*ReplaceRootVolumeTask - for _, task := range b.replaceRootVolumeTasks { + for _, task := range b.replaceRootVolumeTasks.All() { if len(filter) > 0 && !filter[task.ReplaceRootVolumeTaskID] { continue } @@ -939,7 +939,7 @@ func (b *InMemoryBackend) EnableAddressTransfer( b.mu.Lock("EnableAddressTransfer") defer b.mu.Unlock() - addr, ok := b.addresses[allocationID] + addr, ok := b.addresses.Get(allocationID) if !ok { return nil, fmt.Errorf("%w: %s", ErrInvalidParameter, allocationID) } @@ -965,7 +965,7 @@ func (b *InMemoryBackend) DisableAddressTransfer(allocationID string) error { b.mu.Lock("DisableAddressTransfer") defer b.mu.Unlock() - if _, ok := b.addresses[allocationID]; !ok { + if _, ok := b.addresses.Get(allocationID); !ok { return fmt.Errorf("%w: %s", ErrInvalidParameter, allocationID) } delete(b.addressTransfers, allocationID) @@ -1009,7 +1009,7 @@ func (b *InMemoryBackend) CreateSubnetCidrReservation( b.mu.Lock("CreateSubnetCidrReservation") defer b.mu.Unlock() - if _, ok := b.subnets[subnetID]; !ok { + if _, ok := b.subnets.Get(subnetID); !ok { return nil, fmt.Errorf("%w: %s", ErrSubnetNotFound, subnetID) } diff --git a/services/ec2/backend_batch3.go b/services/ec2/backend_batch3.go index 90e45507e..f1a2b1757 100644 --- a/services/ec2/backend_batch3.go +++ b/services/ec2/backend_batch3.go @@ -113,7 +113,7 @@ func (b *InMemoryBackend) CreateCapacityReservation( CreateTime: time.Now().UTC(), OwnedBy: b.AccountID, } - b.capacityReservations[cr.CapacityReservationID] = cr + b.capacityReservations.Put(cr) return cr, nil } @@ -127,7 +127,7 @@ func (b *InMemoryBackend) CancelCapacityReservation(reservationID string) error b.mu.Lock("CancelCapacityReservation") defer b.mu.Unlock() - cr, ok := b.capacityReservations[reservationID] + cr, ok := b.capacityReservations.Get(reservationID) if !ok { return fmt.Errorf("%w: %s", ErrInvalidParameter, reservationID) } @@ -145,7 +145,7 @@ func (b *InMemoryBackend) ModifyCapacityReservation(reservationID string, instan b.mu.Lock("ModifyCapacityReservation") defer b.mu.Unlock() - cr, ok := b.capacityReservations[reservationID] + cr, ok := b.capacityReservations.Get(reservationID) if !ok { return fmt.Errorf("%w: %s", ErrInvalidParameter, reservationID) } @@ -165,7 +165,7 @@ func (b *InMemoryBackend) GetGroupsForCapacityReservation(reservationID string) b.mu.RLock("GetGroupsForCapacityReservation") defer b.mu.RUnlock() - if _, ok := b.capacityReservations[reservationID]; !ok { + if _, ok := b.capacityReservations.Get(reservationID); !ok { return nil, fmt.Errorf("%w: %s", ErrInvalidParameter, reservationID) } @@ -187,7 +187,7 @@ func (b *InMemoryBackend) CreateInstanceConnectEndpoint( b.mu.Lock("CreateInstanceConnectEndpoint") defer b.mu.Unlock() - subnet, ok := b.subnets[subnetID] + subnet, ok := b.subnets.Get(subnetID) if !ok { return nil, fmt.Errorf("%w: %s", ErrSubnetNotFound, subnetID) } @@ -203,7 +203,7 @@ func (b *InMemoryBackend) CreateInstanceConnectEndpoint( PreserveClientIP: preserveClientIP, CreateTime: time.Now().UTC(), } - b.instanceConnectEndpoints[id] = ep + b.instanceConnectEndpoints.Put(ep) return ep, nil } @@ -217,10 +217,10 @@ func (b *InMemoryBackend) DeleteInstanceConnectEndpoint(id string) error { b.mu.Lock("DeleteInstanceConnectEndpoint") defer b.mu.Unlock() - if _, ok := b.instanceConnectEndpoints[id]; !ok { + if _, ok := b.instanceConnectEndpoints.Get(id); !ok { return fmt.Errorf("%w: %s", ErrInstanceConnectEndpointNotFound, id) } - delete(b.instanceConnectEndpoints, id) + b.instanceConnectEndpoints.Delete(id) return nil } @@ -238,7 +238,7 @@ func (b *InMemoryBackend) DescribeInstanceConnectEndpoints( } var out []*InstanceConnectEndpoint - for _, ep := range b.instanceConnectEndpoints { + for _, ep := range b.instanceConnectEndpoints.All() { if len(filter) > 0 && !filter[ep.InstanceConnectEndpointID] { continue } @@ -261,7 +261,7 @@ func (b *InMemoryBackend) ModifyInstanceConnectEndpoint(id string, preserveClien b.mu.Lock("ModifyInstanceConnectEndpoint") defer b.mu.Unlock() - ep, ok := b.instanceConnectEndpoints[id] + ep, ok := b.instanceConnectEndpoints.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrInstanceConnectEndpointNotFound, id) } @@ -285,7 +285,7 @@ func (b *InMemoryBackend) CreateInstanceEventWindow( CronExpression: cronExpression, State: stateActive, } - b.instanceEventWindows[ew.InstanceEventWindowID] = ew + b.instanceEventWindows.Put(ew) return ew, nil } @@ -299,10 +299,10 @@ func (b *InMemoryBackend) DeleteInstanceEventWindow(id string) error { b.mu.Lock("DeleteInstanceEventWindow") defer b.mu.Unlock() - if _, ok := b.instanceEventWindows[id]; !ok { + if _, ok := b.instanceEventWindows.Get(id); !ok { return fmt.Errorf("%w: %s", ErrInstanceEventWindowNotFound, id) } - delete(b.instanceEventWindows, id) + b.instanceEventWindows.Delete(id) return nil } @@ -318,7 +318,7 @@ func (b *InMemoryBackend) DescribeInstanceEventWindows(ids []string) []*Instance } var out []*InstanceEventWindow - for _, ew := range b.instanceEventWindows { + for _, ew := range b.instanceEventWindows.All() { if len(filter) > 0 && !filter[ew.InstanceEventWindowID] { continue } @@ -341,7 +341,7 @@ func (b *InMemoryBackend) ModifyInstanceEventWindow(id, name, cronExpression str b.mu.Lock("ModifyInstanceEventWindow") defer b.mu.Unlock() - ew, ok := b.instanceEventWindows[id] + ew, ok := b.instanceEventWindows.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrInstanceEventWindowNotFound, id) } @@ -414,7 +414,7 @@ func (b *InMemoryBackend) RegisterImage(name, description, architecture string) if img.Architecture == "" { img.Architecture = archX8664 } - b.images[img.ImageID] = img + b.images.Put(img) return img, nil } @@ -433,7 +433,7 @@ func (b *InMemoryBackend) ImportImage( Platform: platform, Status: stateTaskCompleted, } - b.imageImportTasks[task.ImportTaskID] = task + b.imageImportTasks.Put(task) return task, nil } @@ -449,7 +449,7 @@ func (b *InMemoryBackend) DescribeImportImageTasks(taskIDs []string) []*ImageImp } var out []*ImageImportTask - for _, t := range b.imageImportTasks { + for _, t := range b.imageImportTasks.All() { if len(filter) > 0 && !filter[t.ImportTaskID] { continue } @@ -476,7 +476,7 @@ func (b *InMemoryBackend) ExportImage( defer b.mu.Unlock() if description == "" { - if img, ok := b.images[imageID]; ok { + if img, ok := b.images.Get(imageID); ok { description = img.Description } } @@ -506,7 +506,7 @@ func (b *InMemoryBackend) ExportImage( S3Prefix: s3Prefix, RoleName: roleName, } - b.exportImageTasks[id] = task + b.exportImageTasks.Put(task) cp := *task @@ -524,7 +524,7 @@ func (b *InMemoryBackend) ListImagesInRecycleBin(imageIDs []string) []*RecycleBi } var out []*RecycleBinImage - for _, img := range b.recycleBinImages { + for _, img := range b.recycleBinImages.All() { if len(filter) > 0 && !filter[img.ImageID] { continue } @@ -544,8 +544,7 @@ func (b *InMemoryBackend) RestoreImageFromRecycleBin(imageID string) error { b.mu.Lock("RestoreImageFromRecycleBin") defer b.mu.Unlock() - - delete(b.recycleBinImages, imageID) + b.recycleBinImages.Delete(imageID) return nil } @@ -563,7 +562,7 @@ func (b *InMemoryBackend) ListSnapshotsInRecycleBin(snapshotIDs []string) []*Sna } var out []*Snapshot - for _, snap := range b.recycleBinSnapshots { + for _, snap := range b.recycleBinSnapshots.All() { if len(filter) > 0 && !filter[snap.SnapshotID] { continue } @@ -584,12 +583,12 @@ func (b *InMemoryBackend) RestoreSnapshotFromRecycleBin(snapshotID string) error b.mu.Lock("RestoreSnapshotFromRecycleBin") defer b.mu.Unlock() - snap, ok := b.recycleBinSnapshots[snapshotID] + snap, ok := b.recycleBinSnapshots.Get(snapshotID) if !ok { return fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) } - b.snapshots[snapshotID] = snap - delete(b.recycleBinSnapshots, snapshotID) + b.snapshots.Put(snap) + b.recycleBinSnapshots.Delete(snapshotID) return nil } @@ -603,7 +602,7 @@ func (b *InMemoryBackend) RestoreSnapshotTier(snapshotID string) error { b.mu.Lock("RestoreSnapshotTier") defer b.mu.Unlock() - if _, ok := b.snapshots[snapshotID]; !ok { + if _, ok := b.snapshots.Get(snapshotID); !ok { return fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) } b.snapshotTiers[snapshotID] = stateDefaultCredit @@ -623,7 +622,7 @@ func (b *InMemoryBackend) ImportSnapshot(description string) (*SnapshotImportTas Description: description, Status: stateTaskCompleted, } - b.snapshotImportTasks[task.ImportTaskID] = task + b.snapshotImportTasks.Put(task) return task, nil } @@ -639,7 +638,7 @@ func (b *InMemoryBackend) DescribeImportSnapshotTasks(taskIDs []string) []*Snaps } var out []*SnapshotImportTask - for _, t := range b.snapshotImportTasks { + for _, t := range b.snapshotImportTasks.All() { if len(filter) > 0 && !filter[t.ImportTaskID] { continue } @@ -798,7 +797,7 @@ func (b *InMemoryBackend) GetPasswordData(instanceID string) (string, time.Time, b.mu.RLock("GetPasswordData") defer b.mu.RUnlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return "", time.Time{}, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -816,7 +815,7 @@ func (b *InMemoryBackend) GetConsoleScreenshot(instanceID string) (string, error b.mu.RLock("GetConsoleScreenshot") defer b.mu.RUnlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return "", fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -877,7 +876,7 @@ func (b *InMemoryBackend) GetSecurityGroupsForVpc(vpcID string) ([]SecurityGroup defer b.mu.RUnlock() var out []SecurityGroupForVpcItem - for _, sg := range b.securityGroups { + for _, sg := range b.securityGroups.All() { if sg.VPCID == vpcID { out = append(out, SecurityGroupForVpcItem{ GroupID: sg.ID, @@ -906,7 +905,7 @@ func (b *InMemoryBackend) ReplaceRoute(rtID, destCIDR, gatewayID, natGatewayID s b.mu.Lock("ReplaceRoute") defer b.mu.Unlock() - rt, ok := b.routeTables[rtID] + rt, ok := b.routeTables.Get(rtID) if !ok { return fmt.Errorf("%w: route table %s not found", ErrInvalidParameter, rtID) } @@ -963,7 +962,7 @@ func (b *InMemoryBackend) UpdateSecurityGroupRuleDescriptionsIngress( b.mu.Lock("UpdateSecurityGroupRuleDescriptionsIngress") defer b.mu.Unlock() - sg, ok := b.securityGroups[groupID] + sg, ok := b.securityGroups.Get(groupID) if !ok { return fmt.Errorf("%w: %s", ErrSecurityGroupNotFound, groupID) } @@ -985,7 +984,7 @@ func (b *InMemoryBackend) UpdateSecurityGroupRuleDescriptionsEgress( b.mu.Lock("UpdateSecurityGroupRuleDescriptionsEgress") defer b.mu.Unlock() - sg, ok := b.securityGroups[groupID] + sg, ok := b.securityGroups.Get(groupID) if !ok { return fmt.Errorf("%w: %s", ErrSecurityGroupNotFound, groupID) } @@ -1023,7 +1022,7 @@ func (b *InMemoryBackend) ListVolumesInRecycleBin(volumeIDs []string) []*Recycle } var out []*RecycleBinVolume - for _, vol := range b.recycleBinVolumes { + for _, vol := range b.recycleBinVolumes.All() { if len(filter) > 0 && !filter[vol.VolumeID] { continue } @@ -1044,10 +1043,10 @@ func (b *InMemoryBackend) RestoreVolumeFromRecycleBin(volumeID string) error { b.mu.Lock("RestoreVolumeFromRecycleBin") defer b.mu.Unlock() - if _, ok := b.recycleBinVolumes[volumeID]; !ok { + if _, ok := b.recycleBinVolumes.Get(volumeID); !ok { return fmt.Errorf("%w: %s", ErrVolumeNotFound, volumeID) } - delete(b.recycleBinVolumes, volumeID) + b.recycleBinVolumes.Delete(volumeID) return nil } @@ -1074,7 +1073,7 @@ func (b *InMemoryBackend) ReportInstanceStatus(instanceID string, _ string, _ st b.mu.RLock("ReportInstanceStatus") defer b.mu.RUnlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -1093,13 +1092,13 @@ func (b *InMemoryBackend) ModifyVpnConnection(vpnConnectionID, vpnGatewayID stri b.mu.Lock("ModifyVpnConnection") defer b.mu.Unlock() - conn, ok := b.vpnConnections[vpnConnectionID] + conn, ok := b.vpnConnections.Get(vpnConnectionID) if !ok { return fmt.Errorf("%w: %s", ErrVpnConnectionNotFound, vpnConnectionID) } if vpnGatewayID != "" { - if _, exists := b.vpnGateways[vpnGatewayID]; !exists { + if _, exists := b.vpnGateways.Get(vpnGatewayID); !exists { return fmt.Errorf("%w: %s", ErrVpnGatewayNotFound, vpnGatewayID) } @@ -1131,7 +1130,7 @@ func (b *InMemoryBackend) CreateVpnConnectionRoute( b.mu.Lock("CreateVpnConnectionRoute") defer b.mu.Unlock() - if _, ok := b.vpnConnections[vpnConnectionID]; !ok { + if _, ok := b.vpnConnections.Get(vpnConnectionID); !ok { return nil, fmt.Errorf("%w: %s", ErrInvalidParameter, vpnConnectionID) } @@ -1140,7 +1139,7 @@ func (b *InMemoryBackend) CreateVpnConnectionRoute( DestinationCIDR: destinationCIDR, State: stateActive, } - b.vpnConnectionRoutes[vpnConnectionID+":"+destinationCIDR] = route + b.vpnConnectionRoutes.Put(route) return route, nil } @@ -1158,10 +1157,10 @@ func (b *InMemoryBackend) DeleteVpnConnectionRoute(vpnConnectionID, destinationC defer b.mu.Unlock() key := vpnConnectionID + ":" + destinationCIDR - if _, ok := b.vpnConnectionRoutes[key]; !ok { + if _, ok := b.vpnConnectionRoutes.Get(key); !ok { return fmt.Errorf("%w: route %s not found", ErrInvalidParameter, destinationCIDR) } - delete(b.vpnConnectionRoutes, key) + b.vpnConnectionRoutes.Delete(key) return nil } @@ -1177,7 +1176,7 @@ func (b *InMemoryBackend) ModifyTransitGateway(tgwID, description string) error b.mu.Lock("ModifyTransitGateway") defer b.mu.Unlock() - tgw, ok := b.transitGateways[tgwID] + tgw, ok := b.transitGateways.Get(tgwID) if !ok { return fmt.Errorf("%w: %s", ErrInvalidParameter, tgwID) } diff --git a/services/ec2/backend_batch4.go b/services/ec2/backend_batch4.go index d12058bff..93c534df7 100644 --- a/services/ec2/backend_batch4.go +++ b/services/ec2/backend_batch4.go @@ -3,6 +3,7 @@ package ec2 import ( "errors" "fmt" + "slices" "sort" "time" @@ -202,7 +203,7 @@ func (b *InMemoryBackend) CreateManagedPrefixList( Version: 1, OwnerID: b.AccountID, } - b.managedPrefixLists[id] = pl + b.managedPrefixLists.Put(pl) return pl, nil } @@ -216,10 +217,10 @@ func (b *InMemoryBackend) DeleteManagedPrefixList(id string) error { b.mu.Lock("DeleteManagedPrefixList") defer b.mu.Unlock() - if _, ok := b.managedPrefixLists[id]; !ok { + if _, ok := b.managedPrefixLists.Get(id); !ok { return fmt.Errorf("%w: %s", ErrManagedPrefixListNotFound, id) } - delete(b.managedPrefixLists, id) + b.managedPrefixLists.Delete(id) return nil } @@ -235,7 +236,7 @@ func (b *InMemoryBackend) DescribeManagedPrefixLists(ids []string) []*ManagedPre } var out []*ManagedPrefixList - for _, pl := range b.managedPrefixLists { + for _, pl := range b.managedPrefixLists.All() { if len(filter) > 0 && !filter[pl.PrefixListID] { continue } @@ -256,7 +257,7 @@ func (b *InMemoryBackend) GetManagedPrefixListEntries(id string) ([]PrefixListEn b.mu.RLock("GetManagedPrefixListEntries") defer b.mu.RUnlock() - pl, ok := b.managedPrefixLists[id] + pl, ok := b.managedPrefixLists.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrManagedPrefixListNotFound, id) } @@ -279,7 +280,7 @@ func (b *InMemoryBackend) ModifyManagedPrefixList( b.mu.Lock("ModifyManagedPrefixList") defer b.mu.Unlock() - pl, ok := b.managedPrefixLists[id] + pl, ok := b.managedPrefixLists.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrManagedPrefixListNotFound, id) } @@ -316,7 +317,7 @@ func (b *InMemoryBackend) RestoreManagedPrefixListVersion(id string, version int b.mu.Lock("RestoreManagedPrefixListVersion") defer b.mu.Unlock() - pl, ok := b.managedPrefixLists[id] + pl, ok := b.managedPrefixLists.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrManagedPrefixListNotFound, id) } @@ -398,7 +399,7 @@ func (b *InMemoryBackend) CreateClientVpnEndpointWithOptions( SelfServicePortalURL: opts.SelfServicePortalURL, CreationTime: time.Now().UTC().Format(time.RFC3339), } - b.clientVpnEndpoints[id] = ep + b.clientVpnEndpoints.Put(ep) return ep, nil } @@ -412,10 +413,10 @@ func (b *InMemoryBackend) DeleteClientVpnEndpoint(id string) error { b.mu.Lock("DeleteClientVpnEndpoint") defer b.mu.Unlock() - if _, ok := b.clientVpnEndpoints[id]; !ok { + if _, ok := b.clientVpnEndpoints.Get(id); !ok { return fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, id) } - delete(b.clientVpnEndpoints, id) + b.clientVpnEndpoints.Delete(id) return nil } @@ -431,7 +432,7 @@ func (b *InMemoryBackend) DescribeClientVpnEndpoints(ids []string) []*ClientVpnE } var out []*ClientVpnEndpoint - for _, ep := range b.clientVpnEndpoints { + for _, ep := range b.clientVpnEndpoints.All() { if len(filter) > 0 && !filter[ep.ClientVpnEndpointID] { continue } @@ -455,7 +456,7 @@ func (b *InMemoryBackend) AssociateClientVpnTargetNetwork( b.mu.Lock("AssociateClientVpnTargetNetwork") defer b.mu.Unlock() - ep, ok := b.clientVpnEndpoints[endpointID] + ep, ok := b.clientVpnEndpoints.Get(endpointID) if !ok { return "", fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -474,7 +475,7 @@ func (b *InMemoryBackend) AssociateClientVpnTargetNetwork( Status: stateAssociated, SecurityGroups: ep.SecurityGroupIDs, } - if subnet, subnetFound := b.subnets[subnetID]; subnetFound { + if subnet, subnetFound := b.subnets.Get(subnetID); subnetFound { tn.VPCID = subnet.VPCID if ep.VPCID == "" { ep.VPCID = subnet.VPCID @@ -496,7 +497,7 @@ func (b *InMemoryBackend) DisassociateClientVpnTargetNetwork( b.mu.Lock("DisassociateClientVpnTargetNetwork") defer b.mu.Unlock() - ep, ok := b.clientVpnEndpoints[endpointID] + ep, ok := b.clientVpnEndpoints.Get(endpointID) if !ok { return fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -532,7 +533,7 @@ func (b *InMemoryBackend) DescribeClientVpnTargetNetworks(endpointID string) ([] b.mu.RLock("DescribeClientVpnTargetNetworks") defer b.mu.RUnlock() - ep, ok := b.clientVpnEndpoints[endpointID] + ep, ok := b.clientVpnEndpoints.Get(endpointID) if !ok { return nil, fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -557,7 +558,7 @@ func (b *InMemoryBackend) CreateClientVpnRoute( b.mu.Lock("CreateClientVpnRoute") defer b.mu.Unlock() - ep, ok := b.clientVpnEndpoints[endpointID] + ep, ok := b.clientVpnEndpoints.Get(endpointID) if !ok { return fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -581,7 +582,7 @@ func (b *InMemoryBackend) DeleteClientVpnRoute(endpointID, destinationCidr strin b.mu.Lock("DeleteClientVpnRoute") defer b.mu.Unlock() - ep, ok := b.clientVpnEndpoints[endpointID] + ep, ok := b.clientVpnEndpoints.Get(endpointID) if !ok { return fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -606,7 +607,7 @@ func (b *InMemoryBackend) DescribeClientVpnRoutes(endpointID string) ([]ClientVp b.mu.RLock("DescribeClientVpnRoutes") defer b.mu.RUnlock() - ep, ok := b.clientVpnEndpoints[endpointID] + ep, ok := b.clientVpnEndpoints.Get(endpointID) if !ok { return nil, fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -628,7 +629,7 @@ func (b *InMemoryBackend) AuthorizeClientVpnIngress( b.mu.Lock("AuthorizeClientVpnIngress") defer b.mu.Unlock() - ep, ok := b.clientVpnEndpoints[endpointID] + ep, ok := b.clientVpnEndpoints.Get(endpointID) if !ok { return fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -652,7 +653,7 @@ func (b *InMemoryBackend) RevokeClientVpnIngress(endpointID, cidr string) error b.mu.Lock("RevokeClientVpnIngress") defer b.mu.Unlock() - ep, ok := b.clientVpnEndpoints[endpointID] + ep, ok := b.clientVpnEndpoints.Get(endpointID) if !ok { return fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -679,7 +680,7 @@ func (b *InMemoryBackend) DescribeClientVpnAuthorizationRules( b.mu.RLock("DescribeClientVpnAuthorizationRules") defer b.mu.RUnlock() - ep, ok := b.clientVpnEndpoints[endpointID] + ep, ok := b.clientVpnEndpoints.Get(endpointID) if !ok { return nil, fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -712,7 +713,7 @@ func (b *InMemoryBackend) ModifyClientVpnEndpointWithOptions( b.mu.Lock("ModifyClientVpnEndpoint") defer b.mu.Unlock() - ep, ok := b.clientVpnEndpoints[endpointID] + ep, ok := b.clientVpnEndpoints.Get(endpointID) if !ok { return fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -757,7 +758,7 @@ func (b *InMemoryBackend) ApplySecurityGroupsToClientVpnTargetNetwork(endpointID b.mu.Lock("ApplySecurityGroupsToClientVpnTargetNetwork") defer b.mu.Unlock() - ep, ok := b.clientVpnEndpoints[endpointID] + ep, ok := b.clientVpnEndpoints.Get(endpointID) if !ok { return fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -779,7 +780,7 @@ func (b *InMemoryBackend) DescribeClientVpnConnections(endpointID string) ([]str b.mu.RLock("DescribeClientVpnConnections") defer b.mu.RUnlock() - if _, ok := b.clientVpnEndpoints[endpointID]; !ok { + if _, ok := b.clientVpnEndpoints.Get(endpointID); !ok { return nil, fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -795,7 +796,7 @@ func (b *InMemoryBackend) TerminateClientVpnConnections(endpointID string) error b.mu.RLock("TerminateClientVpnConnections") defer b.mu.RUnlock() - if _, ok := b.clientVpnEndpoints[endpointID]; !ok { + if _, ok := b.clientVpnEndpoints.Get(endpointID); !ok { return fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -812,7 +813,7 @@ func (b *InMemoryBackend) ExportClientVpnClientConfiguration(endpointID string) b.mu.RLock("ExportClientVpnClientConfiguration") defer b.mu.RUnlock() - ep, ok := b.clientVpnEndpoints[endpointID] + ep, ok := b.clientVpnEndpoints.Get(endpointID) if !ok { return "", fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -837,7 +838,7 @@ func (b *InMemoryBackend) ExportClientVpnClientCertificateRevocationList(endpoin b.mu.RLock("ExportClientVpnClientCertificateRevocationList") defer b.mu.RUnlock() - ep, ok := b.clientVpnEndpoints[endpointID] + ep, ok := b.clientVpnEndpoints.Get(endpointID) if !ok { return "", fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -861,7 +862,7 @@ func (b *InMemoryBackend) ImportClientVpnClientCertificateRevocationList(endpoin b.mu.Lock("ImportClientVpnClientCertificateRevocationList") defer b.mu.Unlock() - ep, ok := b.clientVpnEndpoints[endpointID] + ep, ok := b.clientVpnEndpoints.Get(endpointID) if !ok { return fmt.Errorf("%w: %s", ErrClientVpnEndpointNotFound, endpointID) } @@ -894,7 +895,7 @@ func (b *InMemoryBackend) CreateTransitGatewayPeeringAttachment( AccepterTransitGatewayID: peerTransitGatewayID, State: "pendingAcceptance", } - b.tgwPeeringAttachments[id] = att + b.tgwPeeringAttachments.Put(att) return att, nil } @@ -908,10 +909,10 @@ func (b *InMemoryBackend) DeleteTransitGatewayPeeringAttachment(id string) error b.mu.Lock("DeleteTransitGatewayPeeringAttachment") defer b.mu.Unlock() - if _, ok := b.tgwPeeringAttachments[id]; !ok { + if _, ok := b.tgwPeeringAttachments.Get(id); !ok { return fmt.Errorf("%w: %s", ErrInvalidParameter, id) } - delete(b.tgwPeeringAttachments, id) + b.tgwPeeringAttachments.Delete(id) return nil } @@ -929,7 +930,7 @@ func (b *InMemoryBackend) DescribeTransitGatewayPeeringAttachments( } var out []*TransitGatewayPeeringAttachment - for _, att := range b.tgwPeeringAttachments { + for _, att := range b.tgwPeeringAttachments.All() { if len(filter) > 0 && !filter[att.TransitGatewayAttachmentID] { continue } @@ -966,7 +967,7 @@ func (b *InMemoryBackend) CreateTransitGatewayConnect( TransitGatewayID: transitGatewayID, State: stateAvailable, } - b.tgwConnects[id] = conn + b.tgwConnects.Put(conn) return conn, nil } @@ -980,10 +981,10 @@ func (b *InMemoryBackend) DeleteTransitGatewayConnect(id string) error { b.mu.Lock("DeleteTransitGatewayConnect") defer b.mu.Unlock() - if _, ok := b.tgwConnects[id]; !ok { + if _, ok := b.tgwConnects.Get(id); !ok { return fmt.Errorf("%w: %s", ErrTransitGatewayConnectNotFound, id) } - delete(b.tgwConnects, id) + b.tgwConnects.Delete(id) return nil } @@ -999,7 +1000,7 @@ func (b *InMemoryBackend) DescribeTransitGatewayConnects(ids []string) []*Transi } var out []*TransitGatewayConnect - for _, conn := range b.tgwConnects { + for _, conn := range b.tgwConnects.All() { if len(filter) > 0 && !filter[conn.TransitGatewayAttachmentID] { continue } @@ -1028,7 +1029,7 @@ func (b *InMemoryBackend) CreateTransitGatewayConnectPeer( b.mu.Lock("CreateTransitGatewayConnectPeer") defer b.mu.Unlock() - if _, ok := b.tgwConnects[connectAttachmentID]; !ok { + if _, ok := b.tgwConnects.Get(connectAttachmentID); !ok { return nil, fmt.Errorf("%w: %s", ErrTransitGatewayConnectNotFound, connectAttachmentID) } @@ -1040,7 +1041,7 @@ func (b *InMemoryBackend) CreateTransitGatewayConnectPeer( InsideCidrBlocks: insideCidrBlocks, PeerAddress: peerAddress, } - b.tgwConnectPeers[id] = peer + b.tgwConnectPeers.Put(peer) return peer, nil } @@ -1054,10 +1055,10 @@ func (b *InMemoryBackend) DeleteTransitGatewayConnectPeer(id string) error { b.mu.Lock("DeleteTransitGatewayConnectPeer") defer b.mu.Unlock() - if _, ok := b.tgwConnectPeers[id]; !ok { + if _, ok := b.tgwConnectPeers.Get(id); !ok { return fmt.Errorf("%w: %s", ErrTransitGatewayConnectPeerNotFound, id) } - delete(b.tgwConnectPeers, id) + b.tgwConnectPeers.Delete(id) return nil } @@ -1075,7 +1076,7 @@ func (b *InMemoryBackend) DescribeTransitGatewayConnectPeers( } var out []*TransitGatewayConnectPeer - for _, peer := range b.tgwConnectPeers { + for _, peer := range b.tgwConnectPeers.All() { if len(filter) > 0 && !filter[peer.TransitGatewayConnectPeerID] { continue } @@ -1106,14 +1107,13 @@ func (b *InMemoryBackend) CreateTransitGatewayPrefixListReference( b.mu.Lock("CreateTransitGatewayPrefixListReference") defer b.mu.Unlock() - key := routeTableID + "/" + prefixListID ref := &TransitGatewayPrefixListReference{ PrefixListID: prefixListID, TransitGatewayRouteTableID: routeTableID, State: stateAvailable, Blackhole: blackhole, } - b.tgwPrefixListRefs[key] = ref + b.tgwPrefixListRefs.Put(ref) return ref, nil } @@ -1133,10 +1133,10 @@ func (b *InMemoryBackend) DeleteTransitGatewayPrefixListReference( defer b.mu.Unlock() key := routeTableID + "/" + prefixListID - if _, ok := b.tgwPrefixListRefs[key]; !ok { + if _, ok := b.tgwPrefixListRefs.Get(key); !ok { return fmt.Errorf("%w: %s/%s", ErrTGWPrefixListRefNotFound, routeTableID, prefixListID) } - delete(b.tgwPrefixListRefs, key) + b.tgwPrefixListRefs.Delete(key) return nil } @@ -1153,7 +1153,7 @@ func (b *InMemoryBackend) GetTransitGatewayPrefixListReferences( defer b.mu.RUnlock() var out []*TransitGatewayPrefixListReference - for _, ref := range b.tgwPrefixListRefs { + for _, ref := range b.tgwPrefixListRefs.All() { if ref.TransitGatewayRouteTableID == routeTableID { cp := *ref out = append(out, &cp) @@ -1181,8 +1181,7 @@ func (b *InMemoryBackend) ModifyTransitGatewayPrefixListReference( defer b.mu.Unlock() key := routeTableID + "/" + prefixListID - - ref, ok := b.tgwPrefixListRefs[key] + ref, ok := b.tgwPrefixListRefs.Get(key) if !ok { return nil, fmt.Errorf("%w: %s/%s", ErrTGWPrefixListRefNotFound, routeTableID, prefixListID) } @@ -1218,7 +1217,7 @@ func (b *InMemoryBackend) CreateVerifiedAccessEndpoint( Description: description, EndpointType: endpointType, } - b.verifiedAccessEndpoints[id] = ep + b.verifiedAccessEndpoints.Put(ep) return ep, nil } @@ -1232,10 +1231,10 @@ func (b *InMemoryBackend) DeleteVerifiedAccessEndpoint(id string) error { b.mu.Lock("DeleteVerifiedAccessEndpoint") defer b.mu.Unlock() - if _, ok := b.verifiedAccessEndpoints[id]; !ok { + if _, ok := b.verifiedAccessEndpoints.Get(id); !ok { return fmt.Errorf("%w: %s", ErrVerifiedAccessEndpointNotFound, id) } - delete(b.verifiedAccessEndpoints, id) + b.verifiedAccessEndpoints.Delete(id) return nil } @@ -1251,7 +1250,7 @@ func (b *InMemoryBackend) DescribeVerifiedAccessEndpoints(ids []string) []*Verif } var out []*VerifiedAccessEndpoint - for _, ep := range b.verifiedAccessEndpoints { + for _, ep := range b.verifiedAccessEndpoints.All() { if len(filter) > 0 && !filter[ep.VerifiedAccessEndpointID] { continue } @@ -1274,7 +1273,7 @@ func (b *InMemoryBackend) ModifyVerifiedAccessEndpoint(id, description string) e b.mu.Lock("ModifyVerifiedAccessEndpoint") defer b.mu.Unlock() - ep, ok := b.verifiedAccessEndpoints[id] + ep, ok := b.verifiedAccessEndpoints.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrVerifiedAccessEndpointNotFound, id) } @@ -1303,7 +1302,7 @@ func (b *InMemoryBackend) CreateVerifiedAccessGroup( Status: stateActive, Description: description, } - b.verifiedAccessGroups[id] = grp + b.verifiedAccessGroups.Put(grp) return grp, nil } @@ -1317,10 +1316,10 @@ func (b *InMemoryBackend) DeleteVerifiedAccessGroup(id string) error { b.mu.Lock("DeleteVerifiedAccessGroup") defer b.mu.Unlock() - if _, ok := b.verifiedAccessGroups[id]; !ok { + if _, ok := b.verifiedAccessGroups.Get(id); !ok { return fmt.Errorf("%w: %s", ErrVerifiedAccessGroupNotFound, id) } - delete(b.verifiedAccessGroups, id) + b.verifiedAccessGroups.Delete(id) return nil } @@ -1336,7 +1335,7 @@ func (b *InMemoryBackend) DescribeVerifiedAccessGroups(ids []string) []*Verified } var out []*VerifiedAccessGroup - for _, grp := range b.verifiedAccessGroups { + for _, grp := range b.verifiedAccessGroups.All() { if len(filter) > 0 && !filter[grp.VerifiedAccessGroupID] { continue } @@ -1361,7 +1360,7 @@ func (b *InMemoryBackend) CreateVerifiedAccessInstance(description string) (*Ver Status: stateActive, Description: description, } - b.verifiedAccessInstances[id] = inst + b.verifiedAccessInstances.Put(inst) return inst, nil } @@ -1375,10 +1374,10 @@ func (b *InMemoryBackend) DeleteVerifiedAccessInstance(id string) error { b.mu.Lock("DeleteVerifiedAccessInstance") defer b.mu.Unlock() - if _, ok := b.verifiedAccessInstances[id]; !ok { + if _, ok := b.verifiedAccessInstances.Get(id); !ok { return fmt.Errorf("%w: %s", ErrVerifiedAccessInstanceNotFound, id) } - delete(b.verifiedAccessInstances, id) + b.verifiedAccessInstances.Delete(id) return nil } @@ -1394,7 +1393,7 @@ func (b *InMemoryBackend) DescribeVerifiedAccessInstances(ids []string) []*Verif } var out []*VerifiedAccessInstance - for _, inst := range b.verifiedAccessInstances { + for _, inst := range b.verifiedAccessInstances.All() { if len(filter) > 0 && !filter[inst.VerifiedAccessInstanceID] { continue } @@ -1426,7 +1425,7 @@ func (b *InMemoryBackend) CreateVerifiedAccessTrustProvider( Status: stateActive, Description: description, } - b.verifiedAccessTrustProviders[id] = tp + b.verifiedAccessTrustProviders.Put(tp) return tp, nil } @@ -1440,10 +1439,10 @@ func (b *InMemoryBackend) DeleteVerifiedAccessTrustProvider(id string) error { b.mu.Lock("DeleteVerifiedAccessTrustProvider") defer b.mu.Unlock() - if _, ok := b.verifiedAccessTrustProviders[id]; !ok { + if _, ok := b.verifiedAccessTrustProviders.Get(id); !ok { return fmt.Errorf("%w: %s", ErrVerifiedAccessTrustProviderNF, id) } - delete(b.verifiedAccessTrustProviders, id) + b.verifiedAccessTrustProviders.Delete(id) return nil } @@ -1461,7 +1460,7 @@ func (b *InMemoryBackend) DescribeVerifiedAccessTrustProviders( } var out []*VerifiedAccessTrustProvider - for _, tp := range b.verifiedAccessTrustProviders { + for _, tp := range b.verifiedAccessTrustProviders.All() { if len(filter) > 0 && !filter[tp.VerifiedAccessTrustProviderID] { continue } @@ -1487,18 +1486,16 @@ func (b *InMemoryBackend) AttachVerifiedAccessTrustProvider(instanceID, trustPro b.mu.Lock("AttachVerifiedAccessTrustProvider") defer b.mu.Unlock() - inst, ok := b.verifiedAccessInstances[instanceID] + inst, ok := b.verifiedAccessInstances.Get(instanceID) if !ok { return fmt.Errorf("%w: %s", ErrVerifiedAccessInstanceNotFound, instanceID) } - if _, exists := b.verifiedAccessTrustProviders[trustProviderID]; !exists { + if _, exists := b.verifiedAccessTrustProviders.Get(trustProviderID); !exists { return fmt.Errorf("%w: %s", ErrVerifiedAccessTrustProviderNF, trustProviderID) } - for _, id := range inst.AttachedTrustProviderIDs { //nolint:modernize // slices.Contains requires Go 1.21+ - if id == trustProviderID { - return nil // already attached - } + if slices.Contains(inst.AttachedTrustProviderIDs, trustProviderID) { + return nil // already attached } inst.AttachedTrustProviderIDs = append(inst.AttachedTrustProviderIDs, trustProviderID) @@ -1517,7 +1514,7 @@ func (b *InMemoryBackend) DetachVerifiedAccessTrustProvider(instanceID, trustPro b.mu.Lock("DetachVerifiedAccessTrustProvider") defer b.mu.Unlock() - inst, ok := b.verifiedAccessInstances[instanceID] + inst, ok := b.verifiedAccessInstances.Get(instanceID) if !ok { return fmt.Errorf("%w: %s", ErrVerifiedAccessInstanceNotFound, instanceID) } diff --git a/services/ec2/backend_batch5.go b/services/ec2/backend_batch5.go index 93a917ac3..b73db51a1 100644 --- a/services/ec2/backend_batch5.go +++ b/services/ec2/backend_batch5.go @@ -207,7 +207,7 @@ func (b *InMemoryBackend) CreateTrafficMirrorFilter(description string) (*Traffi TrafficMirrorFilterID: id, Description: description, } - b.trafficMirrorFilters[id] = f + b.trafficMirrorFilters.Put(f) cp := *f @@ -218,11 +218,10 @@ func (b *InMemoryBackend) DeleteTrafficMirrorFilter(id string) error { b.mu.Lock("DeleteTrafficMirrorFilter") defer b.mu.Unlock() - if _, ok := b.trafficMirrorFilters[id]; !ok { + if _, ok := b.trafficMirrorFilters.Get(id); !ok { return fmt.Errorf("%w: %s", ErrTrafficMirrorFilterNotFound, id) } - - delete(b.trafficMirrorFilters, id) + b.trafficMirrorFilters.Delete(id) return nil } @@ -233,7 +232,7 @@ func (b *InMemoryBackend) DescribeTrafficMirrorFilters(ids []string) []*TrafficM var result []*TrafficMirrorFilter - for _, f := range b.trafficMirrorFilters { + for _, f := range b.trafficMirrorFilters.All() { if len(ids) > 0 && !slices.Contains(ids, f.TrafficMirrorFilterID) { continue } @@ -253,7 +252,7 @@ func (b *InMemoryBackend) ModifyTrafficMirrorFilterNetworkServices(id string, ad b.mu.Lock("ModifyTrafficMirrorFilterNetworkServices") defer b.mu.Unlock() - f, ok := b.trafficMirrorFilters[id] + f, ok := b.trafficMirrorFilters.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrTrafficMirrorFilterNotFound, id) } @@ -289,7 +288,7 @@ func (b *InMemoryBackend) CreateTrafficMirrorFilterRule( b.mu.Lock("CreateTrafficMirrorFilterRule") defer b.mu.Unlock() - f, ok := b.trafficMirrorFilters[filterID] + f, ok := b.trafficMirrorFilters.Get(filterID) if !ok { return nil, fmt.Errorf("%w: %s", ErrTrafficMirrorFilterNotFound, filterID) } @@ -317,8 +316,7 @@ func (b *InMemoryBackend) CreateTrafficMirrorFilterRule( } else { f.IngressFilterRules = append(f.IngressFilterRules, rule) } - - b.trafficMirrorFilterRules[id] = rule + b.trafficMirrorFilterRules.Put(rule) cp := *rule @@ -329,17 +327,16 @@ func (b *InMemoryBackend) DeleteTrafficMirrorFilterRule(id string) error { b.mu.Lock("DeleteTrafficMirrorFilterRule") defer b.mu.Unlock() - rule, ok := b.trafficMirrorFilterRules[id] + rule, ok := b.trafficMirrorFilterRules.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrTrafficMirrorFilterRuleNotFound, id) } - if f, found := b.trafficMirrorFilters[rule.TrafficMirrorFilterID]; found { + if f, found := b.trafficMirrorFilters.Get(rule.TrafficMirrorFilterID); found { f.IngressFilterRules = removeTrafficMirrorFilterRule(f.IngressFilterRules, id) f.EgressFilterRules = removeTrafficMirrorFilterRule(f.EgressFilterRules, id) } - - delete(b.trafficMirrorFilterRules, id) + b.trafficMirrorFilterRules.Delete(id) return nil } @@ -359,7 +356,7 @@ func (b *InMemoryBackend) DescribeTrafficMirrorFilterRules(filterID string) ([]* b.mu.RLock("DescribeTrafficMirrorFilterRules") defer b.mu.RUnlock() - f, ok := b.trafficMirrorFilters[filterID] + f, ok := b.trafficMirrorFilters.Get(filterID) if !ok { return nil, fmt.Errorf("%w: %s", ErrTrafficMirrorFilterNotFound, filterID) } @@ -383,7 +380,7 @@ func (b *InMemoryBackend) ModifyTrafficMirrorFilterRule(id, action, description b.mu.Lock("ModifyTrafficMirrorFilterRule") defer b.mu.Unlock() - rule, ok := b.trafficMirrorFilterRules[id] + rule, ok := b.trafficMirrorFilterRules.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrTrafficMirrorFilterRuleNotFound, id) } @@ -424,7 +421,7 @@ func (b *InMemoryBackend) CreateTrafficMirrorSession( PacketLength: pl, VirtualNetworkID: trafficMirrorSessionVNI(id), } - b.trafficMirrorSessions[id] = s + b.trafficMirrorSessions.Put(s) cp := *s @@ -447,11 +444,10 @@ func (b *InMemoryBackend) DeleteTrafficMirrorSession(id string) error { b.mu.Lock("DeleteTrafficMirrorSession") defer b.mu.Unlock() - if _, ok := b.trafficMirrorSessions[id]; !ok { + if _, ok := b.trafficMirrorSessions.Get(id); !ok { return fmt.Errorf("%w: %s", ErrTrafficMirrorSessionNotFound, id) } - - delete(b.trafficMirrorSessions, id) + b.trafficMirrorSessions.Delete(id) return nil } @@ -462,7 +458,7 @@ func (b *InMemoryBackend) DescribeTrafficMirrorSessions(ids []string) []*Traffic var result []*TrafficMirrorSession - for _, s := range b.trafficMirrorSessions { + for _, s := range b.trafficMirrorSessions.All() { if len(ids) > 0 && !slices.Contains(ids, s.TrafficMirrorSessionID) { continue } @@ -482,7 +478,7 @@ func (b *InMemoryBackend) ModifyTrafficMirrorSession(id, targetID, filterID, des b.mu.Lock("ModifyTrafficMirrorSession") defer b.mu.Unlock() - s, ok := b.trafficMirrorSessions[id] + s, ok := b.trafficMirrorSessions.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrTrafficMirrorSessionNotFound, id) } @@ -528,7 +524,7 @@ func (b *InMemoryBackend) CreateTrafficMirrorTarget( ), Description: description, } - b.trafficMirrorTargets[id] = t + b.trafficMirrorTargets.Put(t) cp := *t @@ -554,11 +550,10 @@ func (b *InMemoryBackend) DeleteTrafficMirrorTarget(id string) error { b.mu.Lock("DeleteTrafficMirrorTarget") defer b.mu.Unlock() - if _, ok := b.trafficMirrorTargets[id]; !ok { + if _, ok := b.trafficMirrorTargets.Get(id); !ok { return fmt.Errorf("%w: %s", ErrTrafficMirrorTargetNotFound, id) } - - delete(b.trafficMirrorTargets, id) + b.trafficMirrorTargets.Delete(id) return nil } @@ -569,7 +564,7 @@ func (b *InMemoryBackend) DescribeTrafficMirrorTargets(ids []string) []*TrafficM var result []*TrafficMirrorTarget - for _, t := range b.trafficMirrorTargets { + for _, t := range b.trafficMirrorTargets.All() { if len(ids) > 0 && !slices.Contains(ids, t.TrafficMirrorTargetID) { continue } @@ -603,7 +598,7 @@ func (b *InMemoryBackend) CreateFleet(fleetType string, totalTargetCapacity int) TotalTargetCapacity: totalTargetCapacity, ExcessCapacityTerminationPolicy: "termination", } - b.fleets[id] = f + b.fleets.Put(f) cp := *f @@ -617,9 +612,9 @@ func (b *InMemoryBackend) DeleteFleets(ids []string) []string { var deleted []string for _, id := range ids { - if _, ok := b.fleets[id]; ok { - b.fleets[id].FleetState = tgwRouteStateDeleted - delete(b.fleets, id) + if f, ok := b.fleets.Get(id); ok { + f.FleetState = tgwRouteStateDeleted + b.fleets.Delete(id) deleted = append(deleted, id) } } @@ -633,7 +628,7 @@ func (b *InMemoryBackend) DescribeFleets(ids []string) []*Fleet { var result []*Fleet - for _, f := range b.fleets { + for _, f := range b.fleets.All() { if len(ids) > 0 && !slices.Contains(ids, f.FleetID) { continue } @@ -653,7 +648,7 @@ func (b *InMemoryBackend) ModifyFleet(id string, totalTargetCapacity int, excess b.mu.Lock("ModifyFleet") defer b.mu.Unlock() - f, ok := b.fleets[id] + f, ok := b.fleets.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrFleetNotFound, id) } @@ -691,7 +686,7 @@ func (b *InMemoryBackend) CreateNetworkInsightsPath( Protocol: protocol, DestinationPort: destinationPort, } - b.networkInsightsPaths[id] = p + b.networkInsightsPaths.Put(p) cp := *p @@ -702,11 +697,10 @@ func (b *InMemoryBackend) DeleteNetworkInsightsPath(id string) error { b.mu.Lock("DeleteNetworkInsightsPath") defer b.mu.Unlock() - if _, ok := b.networkInsightsPaths[id]; !ok { + if _, ok := b.networkInsightsPaths.Get(id); !ok { return fmt.Errorf("%w: %s", ErrNetworkInsightsPathNotFound, id) } - - delete(b.networkInsightsPaths, id) + b.networkInsightsPaths.Delete(id) return nil } @@ -717,7 +711,7 @@ func (b *InMemoryBackend) DescribeNetworkInsightsPaths(ids []string) []*NetworkI var result []*NetworkInsightsPath - for _, p := range b.networkInsightsPaths { + for _, p := range b.networkInsightsPaths.All() { if len(ids) > 0 && !slices.Contains(ids, p.NetworkInsightsPathID) { continue } @@ -737,7 +731,7 @@ func (b *InMemoryBackend) StartNetworkInsightsAnalysis(pathID string) (*NetworkI b.mu.Lock("StartNetworkInsightsAnalysis") defer b.mu.Unlock() - if _, ok := b.networkInsightsPaths[pathID]; !ok { + if _, ok := b.networkInsightsPaths.Get(pathID); !ok { return nil, fmt.Errorf("%w: %s", ErrNetworkInsightsPathNotFound, pathID) } @@ -748,7 +742,7 @@ func (b *InMemoryBackend) StartNetworkInsightsAnalysis(pathID string) (*NetworkI Status: stateAnalysisSucceeded, NetworkPathFound: true, } - b.networkInsightsAnalyses[id] = a + b.networkInsightsAnalyses.Put(a) cp := *a @@ -759,11 +753,10 @@ func (b *InMemoryBackend) DeleteNetworkInsightsAnalysis(id string) error { b.mu.Lock("DeleteNetworkInsightsAnalysis") defer b.mu.Unlock() - if _, ok := b.networkInsightsAnalyses[id]; !ok { + if _, ok := b.networkInsightsAnalyses.Get(id); !ok { return fmt.Errorf("%w: %s", ErrNetworkInsightsAnalysisNotFound, id) } - - delete(b.networkInsightsAnalyses, id) + b.networkInsightsAnalyses.Delete(id) return nil } @@ -774,7 +767,7 @@ func (b *InMemoryBackend) DescribeNetworkInsightsAnalyses(ids []string) []*Netwo var result []*NetworkInsightsAnalysis - for _, a := range b.networkInsightsAnalyses { + for _, a := range b.networkInsightsAnalyses.All() { if len(ids) > 0 && !slices.Contains(ids, a.NetworkInsightsAnalysisID) { continue } @@ -799,7 +792,7 @@ func (b *InMemoryBackend) CreateNetworkInsightsAccessScope() (*NetworkInsightsAc NetworkInsightsAccessScopeID: id, NetworkInsightsAccessScopeArn: "arn:aws:ec2:" + b.Region + ":" + b.AccountID + ":network-insights-access-scope/" + id, } - b.networkInsightsAccessScopes[id] = s + b.networkInsightsAccessScopes.Put(s) cp := *s @@ -810,11 +803,10 @@ func (b *InMemoryBackend) DeleteNetworkInsightsAccessScope(id string) error { b.mu.Lock("DeleteNetworkInsightsAccessScope") defer b.mu.Unlock() - if _, ok := b.networkInsightsAccessScopes[id]; !ok { + if _, ok := b.networkInsightsAccessScopes.Get(id); !ok { return fmt.Errorf("%w: %s", ErrNetworkInsightsAccessScopeNF, id) } - - delete(b.networkInsightsAccessScopes, id) + b.networkInsightsAccessScopes.Delete(id) return nil } @@ -825,7 +817,7 @@ func (b *InMemoryBackend) DescribeNetworkInsightsAccessScopes(ids []string) []*N var result []*NetworkInsightsAccessScope - for _, s := range b.networkInsightsAccessScopes { + for _, s := range b.networkInsightsAccessScopes.All() { if len(ids) > 0 && !slices.Contains(ids, s.NetworkInsightsAccessScopeID) { continue } @@ -847,7 +839,7 @@ func (b *InMemoryBackend) StartNetworkInsightsAccessScopeAnalysis( b.mu.Lock("StartNetworkInsightsAccessScopeAnalysis") defer b.mu.Unlock() - if _, ok := b.networkInsightsAccessScopes[scopeID]; !ok { + if _, ok := b.networkInsightsAccessScopes.Get(scopeID); !ok { return nil, fmt.Errorf("%w: %s", ErrNetworkInsightsAccessScopeNF, scopeID) } @@ -858,7 +850,7 @@ func (b *InMemoryBackend) StartNetworkInsightsAccessScopeAnalysis( Status: stateAnalysisSucceeded, AnalyzedEniCount: 0, } - b.networkInsightsAccessScopeAnalyses[id] = a + b.networkInsightsAccessScopeAnalyses.Put(a) cp := *a @@ -869,11 +861,10 @@ func (b *InMemoryBackend) DeleteNetworkInsightsAccessScopeAnalysis(id string) er b.mu.Lock("DeleteNetworkInsightsAccessScopeAnalysis") defer b.mu.Unlock() - if _, ok := b.networkInsightsAccessScopeAnalyses[id]; !ok { + if _, ok := b.networkInsightsAccessScopeAnalyses.Get(id); !ok { return fmt.Errorf("%w: %s", ErrNetworkInsightsAccessScopeAnaNF, id) } - - delete(b.networkInsightsAccessScopeAnalyses, id) + b.networkInsightsAccessScopeAnalyses.Delete(id) return nil } @@ -886,7 +877,7 @@ func (b *InMemoryBackend) DescribeNetworkInsightsAccessScopeAnalyses( var result []*NetworkInsightsAccessScopeAnalysis - for _, a := range b.networkInsightsAccessScopeAnalyses { + for _, a := range b.networkInsightsAccessScopeAnalyses.All() { if len(ids) > 0 && !slices.Contains(ids, a.NetworkInsightsAccessScopeAnalysisID) { continue } @@ -917,7 +908,7 @@ func (b *InMemoryBackend) ProvisionByoipCidr(cidr, description string) (*ByoipCi State: "pending-provision", StatusMessage: description, } - b.byoipCidrs[cidr] = entry + b.byoipCidrs.Put(entry) cp := *entry @@ -932,13 +923,13 @@ func (b *InMemoryBackend) DeprovisionByoipCidr(cidr string) (*ByoipCidr, error) b.mu.Lock("DeprovisionByoipCidr") defer b.mu.Unlock() - entry, ok := b.byoipCidrs[cidr] + entry, ok := b.byoipCidrs.Get(cidr) if !ok { return nil, fmt.Errorf("%w: %s", ErrInvalidParameter, cidr) } entry.State = "pending-deprovision" - delete(b.byoipCidrs, cidr) + b.byoipCidrs.Delete(cidr) cp := *entry @@ -953,10 +944,10 @@ func (b *InMemoryBackend) WithdrawByoipCidr(cidr string) (*ByoipCidr, error) { b.mu.Lock("WithdrawByoipCidr") defer b.mu.Unlock() - entry, ok := b.byoipCidrs[cidr] + entry, ok := b.byoipCidrs.Get(cidr) if !ok { entry = &ByoipCidr{Cidr: cidr} - b.byoipCidrs[cidr] = entry + b.byoipCidrs.Put(entry) } entry.State = stateByoipAdvertised @@ -983,7 +974,7 @@ func (b *InMemoryBackend) CreateCarrierGateway(vpcID string) (*CarrierGateway, e State: stateAvailableImg, OwnerID: b.AccountID, } - b.carrierGateways[id] = gw + b.carrierGateways.Put(gw) cp := *gw @@ -994,11 +985,10 @@ func (b *InMemoryBackend) DeleteCarrierGateway(id string) error { b.mu.Lock("DeleteCarrierGateway") defer b.mu.Unlock() - if _, ok := b.carrierGateways[id]; !ok { + if _, ok := b.carrierGateways.Get(id); !ok { return fmt.Errorf("%w: %s", ErrCarrierGatewayNotFound, id) } - - delete(b.carrierGateways, id) + b.carrierGateways.Delete(id) return nil } @@ -1009,7 +999,7 @@ func (b *InMemoryBackend) DescribeCarrierGateways(ids []string) []*CarrierGatewa var result []*CarrierGateway - for _, gw := range b.carrierGateways { + for _, gw := range b.carrierGateways.All() { if len(ids) > 0 && !slices.Contains(ids, gw.CarrierGatewayID) { continue } @@ -1033,7 +1023,7 @@ func (b *InMemoryBackend) DescribeReservedInstances(ids []string) []*ReservedIns var result []*ReservedInstance - for _, ri := range b.reservedInstances { + for _, ri := range b.reservedInstances.All() { if len(ids) > 0 && !slices.Contains(ids, ri.ReservedInstancesID) { continue } @@ -1057,7 +1047,7 @@ func (b *InMemoryBackend) DescribeReservedInstancesOfferings( var result []*ReservedInstancesOffering - for _, o := range b.reservedInstancesOfferings { + for _, o := range b.reservedInstancesOfferings.All() { if instanceType != "" && o.InstanceType != instanceType { continue } @@ -1088,7 +1078,7 @@ func (b *InMemoryBackend) PurchaseReservedInstancesOffering( b.mu.Lock("PurchaseReservedInstancesOffering") defer b.mu.Unlock() - offering, ok := b.reservedInstancesOfferings[offeringID] + offering, ok := b.reservedInstancesOfferings.Get(offeringID) if !ok { return nil, fmt.Errorf("%w: %s", ErrReservedInstancesNotFound, offeringID) } @@ -1106,7 +1096,7 @@ func (b *InMemoryBackend) PurchaseReservedInstancesOffering( InstanceCount: instanceCount, State: SpotFleetStateActive, } - b.reservedInstances[id] = ri + b.reservedInstances.Put(ri) cp := *ri @@ -1126,7 +1116,7 @@ func (b *InMemoryBackend) CreateReservedInstancesListing( ReservedInstancesID: reservedInstancesID, Status: SpotFleetStateActive, } - b.reservedInstancesListings[id] = l + b.reservedInstancesListings.Put(l) cp := *l @@ -1137,7 +1127,7 @@ func (b *InMemoryBackend) CancelReservedInstancesListing(id string) error { b.mu.Lock("CancelReservedInstancesListing") defer b.mu.Unlock() - l, ok := b.reservedInstancesListings[id] + l, ok := b.reservedInstancesListings.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrReservedInstancesListingNotFound, id) } @@ -1153,7 +1143,7 @@ func (b *InMemoryBackend) DescribeReservedInstancesListings(ids []string) []*Res var result []*ReservedInstancesListing - for _, l := range b.reservedInstancesListings { + for _, l := range b.reservedInstancesListings.All() { if len(ids) > 0 && !slices.Contains(ids, l.ReservedInstancesListingID) { continue } @@ -1175,7 +1165,7 @@ func (b *InMemoryBackend) DescribeReservedInstancesModifications(ids []string) [ var result []*ReservedInstancesModification - for _, m := range b.reservedInstancesModifications { + for _, m := range b.reservedInstancesModifications.All() { if len(ids) > 0 && !slices.Contains(ids, m.ReservedInstancesModificationID) { continue } @@ -1205,7 +1195,7 @@ func (b *InMemoryBackend) ModifyReservedInstances( Status: "fulfilled", StatusMessage: "Modification fulfilled", } - b.reservedInstancesModifications[id] = m + b.reservedInstancesModifications.Put(m) cp := *m @@ -1217,6 +1207,6 @@ func (b *InMemoryBackend) DeleteQueuedReservedInstances(ids []string) { defer b.mu.Unlock() for _, id := range ids { - delete(b.reservedInstances, id) + b.reservedInstances.Delete(id) } } diff --git a/services/ec2/backend_capacity_family.go b/services/ec2/backend_capacity_family.go index efa5eb28e..f8db030c1 100644 --- a/services/ec2/backend_capacity_family.go +++ b/services/ec2/backend_capacity_family.go @@ -331,8 +331,7 @@ func (b *InMemoryBackend) CreateCapacityReservationFleet( instanceCount := int32(math.Ceil(remaining / weight)) crID := "cr-" + uuid.New().String()[:8] - - b.capacityReservations[crID] = &CapacityReservation{ + b.capacityReservations.Put(&CapacityReservation{ CapacityReservationID: crID, InstanceType: resolved[i].InstanceType, AvailabilityZone: resolved[i].AvailabilityZone, @@ -341,7 +340,7 @@ func (b *InMemoryBackend) CreateCapacityReservationFleet( State: stateActive, CreateTime: now, OwnedBy: b.AccountID, - } + }) resolved[i].CapacityReservationID = crID resolved[i].TotalInstanceCount = instanceCount @@ -369,7 +368,7 @@ func (b *InMemoryBackend) CreateCapacityReservationFleet( InstanceTypeSpecifications: resolved, Tags: tags, } - b.capacityReservationFleets[fleet.CapacityReservationFleetID] = fleet + b.capacityReservationFleets.Put(fleet) cp := *fleet @@ -390,7 +389,7 @@ func (b *InMemoryBackend) DescribeCapacityReservationFleets( var result []*CapacityReservationFleet - for _, fleet := range b.capacityReservationFleets { + for _, fleet := range b.capacityReservationFleets.All() { matched := idAndFiltersMatch(idSet, fleet.CapacityReservationFleetID, matchesCapacityFilter(filters, "state", fleet.State), matchesCapacityFilter(filters, "tenancy", fleet.Tenancy), @@ -429,7 +428,7 @@ func (b *InMemoryBackend) ModifyCapacityReservationFleet( b.mu.Lock("ModifyCapacityReservationFleet") defer b.mu.Unlock() - fleet, ok := b.capacityReservationFleets[fleetID] + fleet, ok := b.capacityReservationFleets.Get(fleetID) if !ok { return fmt.Errorf("%w: %s", ErrCapacityReservationFleetNotFound, fleetID) } @@ -467,7 +466,7 @@ func (b *InMemoryBackend) resizeFleetPrimarySpecLocked(fleet *CapacityReservatio primary.TotalInstanceCount = int32(math.Ceil(float64(totalTargetCapacity) / weight)) - cr, found := b.capacityReservations[primary.CapacityReservationID] + cr, found := b.capacityReservations.Get(primary.CapacityReservationID) if !found { return } @@ -492,7 +491,7 @@ func (b *InMemoryBackend) CancelCapacityReservationFleets( ) for _, id := range fleetIDs { - fleet, ok := b.capacityReservationFleets[id] + fleet, ok := b.capacityReservationFleets.Get(id) if !ok { failed = append(failed, FailedCapacityReservationFleetCancellation{ CapacityReservationFleetID: id, @@ -507,7 +506,7 @@ func (b *InMemoryBackend) CancelCapacityReservationFleets( fleet.State = stateCancelled for _, spec := range fleet.InstanceTypeSpecifications { - if cr, found := b.capacityReservations[spec.CapacityReservationID]; found { + if cr, found := b.capacityReservations.Get(spec.CapacityReservationID); found { cr.State = stateCancelled } } @@ -586,7 +585,7 @@ func (b *InMemoryBackend) DescribeCapacityBlockOfferings( StartDate: start, EndDate: start.Add(time.Duration(durationHours) * time.Hour), } - b.capacityBlockOfferings[offering.CapacityBlockOfferingID] = offering + b.capacityBlockOfferings.Put(offering) offerings = append(offerings, offering) } @@ -606,7 +605,7 @@ func (b *InMemoryBackend) PurchaseCapacityBlock( b.mu.Lock("PurchaseCapacityBlock") defer b.mu.Unlock() - offering, ok := b.capacityBlockOfferings[offeringID] + offering, ok := b.capacityBlockOfferings.Get(offeringID) if !ok { return nil, nil, fmt.Errorf("%w: %s", ErrCapacityBlockOfferingNotFound, offeringID) } @@ -628,7 +627,7 @@ func (b *InMemoryBackend) PurchaseCapacityBlock( CreateTime: now, OwnedBy: b.AccountID, } - b.capacityReservations[crID] = cr + b.capacityReservations.Put(cr) block := &CapacityBlock{ CapacityBlockID: "cb-" + uuid.New().String()[:8], @@ -640,9 +639,8 @@ func (b *InMemoryBackend) PurchaseCapacityBlock( Tags: tags, CapacityReservationIDs: []string{crID}, } - b.capacityBlocks[block.CapacityBlockID] = block - - delete(b.capacityBlockOfferings, offeringID) + b.capacityBlocks.Put(block) + b.capacityBlockOfferings.Delete(offeringID) blockCopy := *block crCopy := *cr @@ -653,7 +651,7 @@ func (b *InMemoryBackend) PurchaseCapacityBlock( // findCapacityBlockByReservationIDLocked returns the CapacityBlock backing the // given CapacityReservation, if any. Must be called with b.mu held. func (b *InMemoryBackend) findCapacityBlockByReservationIDLocked(reservationID string) *CapacityBlock { - for _, block := range b.capacityBlocks { + for _, block := range b.capacityBlocks.All() { if slices.Contains(block.CapacityReservationIDs, reservationID) { return block } @@ -683,7 +681,7 @@ func (b *InMemoryBackend) DescribeCapacityBlockExtensionOfferings( b.mu.Lock("DescribeCapacityBlockExtensionOfferings") defer b.mu.Unlock() - cr, ok := b.capacityReservations[reservationID] + cr, ok := b.capacityReservations.Get(reservationID) if !ok { return nil, fmt.Errorf("%w: %s", ErrCapacityReservationNotFound, reservationID) } @@ -710,7 +708,7 @@ func (b *InMemoryBackend) DescribeCapacityBlockExtensionOfferings( CapacityBlockExtensionStartDate: extStart, CapacityBlockExtensionEndDate: extStart.Add(time.Duration(durationHours) * time.Hour), } - b.capacityBlockExtensionOfferings[offering.CapacityBlockExtensionOfferingID] = offering + b.capacityBlockExtensionOfferings.Put(offering) return []*CapacityBlockExtensionOffering{offering}, nil } @@ -727,7 +725,7 @@ func (b *InMemoryBackend) PurchaseCapacityBlockExtension( b.mu.Lock("PurchaseCapacityBlockExtension") defer b.mu.Unlock() - offering, ok := b.capacityBlockExtensionOfferings[extensionOfferingID] + offering, ok := b.capacityBlockExtensionOfferings.Get(extensionOfferingID) if !ok { return nil, fmt.Errorf("%w: %s", ErrCapacityBlockExtensionOfferingNotFound, extensionOfferingID) } @@ -750,13 +748,12 @@ func (b *InMemoryBackend) PurchaseCapacityBlockExtension( CapacityBlockExtensionEndDate: offering.CapacityBlockExtensionEndDate, CapacityBlockExtensionPurchaseDate: now, } - b.capacityBlockExtensions[ext.CapacityBlockExtensionOfferingID] = ext + b.capacityBlockExtensions.Put(ext) if block := b.findCapacityBlockByReservationIDLocked(offering.CapacityReservationID); block != nil { block.EndDate = offering.CapacityBlockExtensionEndDate } - - delete(b.capacityBlockExtensionOfferings, extensionOfferingID) + b.capacityBlockExtensionOfferings.Delete(extensionOfferingID) return ext, nil } @@ -774,7 +771,7 @@ func (b *InMemoryBackend) DescribeCapacityBlocks( var result []*CapacityBlock - for _, block := range b.capacityBlocks { + for _, block := range b.capacityBlocks.All() { matched := idAndFiltersMatch(idSet, block.CapacityBlockID, matchesCapacityFilter(filters, "capacity-block-id", block.CapacityBlockID), matchesCapacityFilter(filters, "availability-zone", block.AvailabilityZone), @@ -808,7 +805,7 @@ func (b *InMemoryBackend) DescribeCapacityBlockStatus( const interconnectOK = "ok" - for _, block := range b.capacityBlocks { + for _, block := range b.capacityBlocks.All() { matched := idAndFiltersMatch(idSet, block.CapacityBlockID, matchesCapacityFilter(filters, "interconnect-status", interconnectOK), ) @@ -822,7 +819,7 @@ func (b *InMemoryBackend) DescribeCapacityBlockStatus( } for _, crID := range block.CapacityReservationIDs { - cr, ok := b.capacityReservations[crID] + cr, ok := b.capacityReservations.Get(crID) if !ok { continue } @@ -863,7 +860,7 @@ func (b *InMemoryBackend) DescribeCapacityBlockExtensionHistory( var result []*CapacityBlockExtension - for _, ext := range b.capacityBlockExtensions { + for _, ext := range b.capacityBlockExtensions.All() { matched := idAndFiltersMatch(idSet, ext.CapacityReservationID, matchesCapacityFilter(filters, "capacity-reservation-id", ext.CapacityReservationID), matchesCapacityFilter(filters, "availability-zone", ext.AvailabilityZone), @@ -905,7 +902,7 @@ func (b *InMemoryBackend) CreateCapacityReservationBySplitting( b.mu.Lock("CreateCapacityReservationBySplitting") defer b.mu.Unlock() - src, ok := b.capacityReservations[sourceID] + src, ok := b.capacityReservations.Get(sourceID) if !ok { return nil, nil, fmt.Errorf("%w: %s", ErrCapacityReservationNotFound, sourceID) } @@ -931,7 +928,7 @@ func (b *InMemoryBackend) CreateCapacityReservationBySplitting( CreateTime: time.Now().UTC(), OwnedBy: b.AccountID, } - b.capacityReservations[dst.CapacityReservationID] = dst + b.capacityReservations.Put(dst) if len(tags) > 0 { b.tags[dst.CapacityReservationID] = tags @@ -964,12 +961,12 @@ func (b *InMemoryBackend) MoveCapacityReservationInstances( b.mu.Lock("MoveCapacityReservationInstances") defer b.mu.Unlock() - src, ok := b.capacityReservations[sourceID] + src, ok := b.capacityReservations.Get(sourceID) if !ok { return nil, nil, fmt.Errorf("%w: %s", ErrCapacityReservationNotFound, sourceID) } - dst, ok := b.capacityReservations[destinationID] + dst, ok := b.capacityReservations.Get(destinationID) if !ok { return nil, nil, fmt.Errorf("%w: %s", ErrCapacityReservationNotFound, destinationID) } @@ -1010,17 +1007,16 @@ func (b *InMemoryBackend) AssociateCapacityReservationBillingOwner( b.mu.Lock("AssociateCapacityReservationBillingOwner") defer b.mu.Unlock() - if _, ok := b.capacityReservations[reservationID]; !ok { + if _, ok := b.capacityReservations.Get(reservationID); !ok { return fmt.Errorf("%w: %s", ErrCapacityReservationNotFound, reservationID) } - - b.capacityReservationBillingRequests[reservationID] = &CapacityReservationBillingRequest{ + b.capacityReservationBillingRequests.Put(&CapacityReservationBillingRequest{ CapacityReservationID: reservationID, RequestedBy: b.AccountID, UnusedReservationBillingOwnerID: billingOwnerID, Status: billingRequestPending, LastUpdateTime: time.Now().UTC(), - } + }) return nil } @@ -1040,7 +1036,7 @@ func (b *InMemoryBackend) DisassociateCapacityReservationBillingOwner( b.mu.Lock("DisassociateCapacityReservationBillingOwner") defer b.mu.Unlock() - req, ok := b.capacityReservationBillingRequests[reservationID] + req, ok := b.capacityReservationBillingRequests.Get(reservationID) if !ok || req.UnusedReservationBillingOwnerID != billingOwnerID { return fmt.Errorf("%w: %s", ErrCapacityReservationBillingRequestNotFound, reservationID) } @@ -1063,7 +1059,7 @@ func (b *InMemoryBackend) RejectCapacityReservationBillingOwnership( b.mu.Lock("RejectCapacityReservationBillingOwnership") defer b.mu.Unlock() - req, ok := b.capacityReservationBillingRequests[reservationID] + req, ok := b.capacityReservationBillingRequests.Get(reservationID) if !ok { return fmt.Errorf("%w: %s", ErrCapacityReservationBillingRequestNotFound, reservationID) } @@ -1088,7 +1084,7 @@ func (b *InMemoryBackend) DescribeCapacityReservationBillingRequests( var result []*CapacityReservationBillingRequest - for _, req := range b.capacityReservationBillingRequests { + for _, req := range b.capacityReservationBillingRequests.All() { if len(idSet) > 0 && !idSet[req.CapacityReservationID] { continue } @@ -1166,7 +1162,7 @@ func (b *InMemoryBackend) GetCapacityManagerAttributes() *CapacityManagerAttribu attrs := &CapacityManagerAttributes{ Status: b.capacityManagerState.Status, OrganizationsAccess: b.capacityManagerState.OrganizationsAccess, - DataExportCount: toInt32Clamped(len(b.capacityManagerDataExports)), + DataExportCount: toInt32Clamped(b.capacityManagerDataExports.Len()), } if attrs.Status == capacityManagerStatusEnabled { @@ -1231,7 +1227,7 @@ func (b *InMemoryBackend) CreateCapacityManagerDataExport( CreateTime: time.Now().UTC(), Tags: tags, } - b.capacityManagerDataExports[export.CapacityManagerDataExportID] = export + b.capacityManagerDataExports.Put(export) return export, nil } @@ -1248,7 +1244,7 @@ func (b *InMemoryBackend) DescribeCapacityManagerDataExports( var result []*CapacityManagerDataExport - for _, export := range b.capacityManagerDataExports { + for _, export := range b.capacityManagerDataExports.All() { if len(idSet) > 0 && !idSet[export.CapacityManagerDataExportID] { continue } @@ -1274,11 +1270,10 @@ func (b *InMemoryBackend) DeleteCapacityManagerDataExport(id string) (string, er b.mu.Lock("DeleteCapacityManagerDataExport") defer b.mu.Unlock() - if _, ok := b.capacityManagerDataExports[id]; !ok { + if _, ok := b.capacityManagerDataExports.Get(id); !ok { return "", fmt.Errorf("%w: %s", ErrCapacityManagerDataExportNotFound, id) } - - delete(b.capacityManagerDataExports, id) + b.capacityManagerDataExports.Delete(id) return id, nil } diff --git a/services/ec2/backend_declarative_policies.go b/services/ec2/backend_declarative_policies.go index 84b7a2443..8a2acfd4d 100644 --- a/services/ec2/backend_declarative_policies.go +++ b/services/ec2/backend_declarative_policies.go @@ -91,7 +91,7 @@ func (b *InMemoryBackend) StartDeclarativePoliciesReport( StartTime: time.Now().UTC(), Tags: maps.Clone(tags), } - b.declarativePoliciesReports[report.ReportID] = report + b.declarativePoliciesReports.Put(report) return cloneDeclarativePoliciesReport(report), nil } @@ -105,7 +105,7 @@ func (b *InMemoryBackend) CancelDeclarativePoliciesReport(reportID string) (bool b.mu.Lock("CancelDeclarativePoliciesReport") defer b.mu.Unlock() - report, ok := b.declarativePoliciesReports[reportID] + report, ok := b.declarativePoliciesReports.Get(reportID) if !ok { return false, fmt.Errorf("%w: %s", ErrDeclarativePoliciesReportNotFound, reportID) } @@ -134,9 +134,9 @@ func (b *InMemoryBackend) DescribeDeclarativePoliciesReports(ids []string) []*De idSet[id] = true } - out := make([]*DeclarativePoliciesReport, 0, len(b.declarativePoliciesReports)) + out := make([]*DeclarativePoliciesReport, 0, b.declarativePoliciesReports.Len()) - for _, r := range b.declarativePoliciesReports { + for _, r := range b.declarativePoliciesReports.All() { if len(idSet) > 0 && !idSet[r.ReportID] { continue } @@ -174,7 +174,7 @@ func (b *InMemoryBackend) GetDeclarativePoliciesReportSummary( b.mu.Lock("GetDeclarativePoliciesReportSummary") defer b.mu.Unlock() - report, ok := b.declarativePoliciesReports[reportID] + report, ok := b.declarativePoliciesReports.Get(reportID) if !ok { return nil, fmt.Errorf("%w: %s", ErrDeclarativePoliciesReportNotFound, reportID) } diff --git a/services/ec2/backend_deepdive_ops.go b/services/ec2/backend_deepdive_ops.go index b3564ceb2..76fe02628 100644 --- a/services/ec2/backend_deepdive_ops.go +++ b/services/ec2/backend_deepdive_ops.go @@ -17,7 +17,7 @@ func (b *InMemoryBackend) CreateImage(instanceID, name, description string) (*AM b.mu.Lock("CreateImage") defer b.mu.Unlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -34,12 +34,12 @@ func (b *InMemoryBackend) CreateImage(instanceID, name, description string) (*AM RootDeviceName: "/dev/xvda", State: stateAvailable, } - b.images[imageID] = image - b.imageUsageReports[imageID] = &ImageUsageReport{ + b.images.Put(image) + b.imageUsageReports.Put(&ImageUsageReport{ ImageID: imageID, State: stateAvailable, GenerationDate: time.Now().UTC().Format(time.RFC3339), - } + }) cp := *image @@ -51,8 +51,8 @@ func (b *InMemoryBackend) DescribeImageUsageReports() []*ImageUsageReport { b.mu.RLock("DescribeImageUsageReports") defer b.mu.RUnlock() - reports := make([]*ImageUsageReport, 0, len(b.imageUsageReports)) - for _, report := range b.imageUsageReports { + reports := make([]*ImageUsageReport, 0, b.imageUsageReports.Len()) + for _, report := range b.imageUsageReports.All() { cp := *report reports = append(reports, &cp) } @@ -79,7 +79,7 @@ func (b *InMemoryBackend) CreateLaunchTemplate( b.mu.Lock("CreateLaunchTemplate") defer b.mu.Unlock() - for _, lt := range b.launchTemplates { + for _, lt := range b.launchTemplates.All() { if lt.Name == name { return nil, fmt.Errorf( "%w: duplicate launch template name %s", @@ -99,7 +99,7 @@ func (b *InMemoryBackend) CreateLaunchTemplate( DefaultVersionNumber: 1, LatestVersionNumber: 1, } - b.launchTemplates[template.ID] = template + b.launchTemplates.Put(template) cp := *template return &cp, nil @@ -110,8 +110,8 @@ func (b *InMemoryBackend) DescribeLaunchTemplates(names []string) []*LaunchTempl b.mu.RLock("DescribeLaunchTemplates") defer b.mu.RUnlock() - templates := make([]*LaunchTemplate, 0, len(b.launchTemplates)) - for _, lt := range b.launchTemplates { + templates := make([]*LaunchTemplate, 0, b.launchTemplates.Len()) + for _, lt := range b.launchTemplates.All() { if len(names) > 0 && !slices.Contains(names, lt.Name) { continue } @@ -133,14 +133,14 @@ func (b *InMemoryBackend) DescribeNetworkAcls(vpcIDs []string) []*NetworkACL { allowed[id] = true } - networkACLs := make([]*NetworkACL, 0, len(b.vpcs)) - for _, vpc := range b.vpcs { + networkACLs := make([]*NetworkACL, 0, b.vpcs.Len()) + for _, vpc := range b.vpcs.All() { if len(allowed) > 0 && !allowed[vpc.ID] { continue } - assocIDs := make([]string, 0, len(b.subnets)) - for _, subnet := range b.subnets { + assocIDs := make([]string, 0, b.subnets.Len()) + for _, subnet := range b.subnets.All() { if subnet.VPCID == vpc.ID { assocIDs = append(assocIDs, "aclassoc-"+subnet.ID) } @@ -185,12 +185,12 @@ func (b *InMemoryBackend) CreateVpcEndpointWithRouteTableIDs( b.mu.Lock("CreateVpcEndpoint") defer b.mu.Unlock() - if _, ok := b.vpcs[vpcID]; !ok { + if _, ok := b.vpcs.Get(vpcID); !ok { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } for _, subnetID := range subnetIDs { - subnet, ok := b.subnets[subnetID] + subnet, ok := b.subnets.Get(subnetID) if !ok { return nil, fmt.Errorf("%w: %s", ErrSubnetNotFound, subnetID) } @@ -206,7 +206,7 @@ func (b *InMemoryBackend) CreateVpcEndpointWithRouteTableIDs( } for _, rtID := range routeTableIDs { - if _, ok := b.routeTables[rtID]; !ok { + if _, ok := b.routeTables.Get(rtID); !ok { return nil, fmt.Errorf("%w: route table %s not found", ErrRouteTableNotFound, rtID) } } @@ -221,7 +221,7 @@ func (b *InMemoryBackend) CreateVpcEndpointWithRouteTableIDs( RouteTableIDs: append([]string(nil), routeTableIDs...), CreateTime: time.Now().UTC(), } - b.vpcEndpoints[endpoint.ID] = endpoint + b.vpcEndpoints.Put(endpoint) cp := *endpoint cp.SubnetIDs = append([]string(nil), endpoint.SubnetIDs...) cp.RouteTableIDs = append([]string(nil), endpoint.RouteTableIDs...) @@ -239,8 +239,8 @@ func (b *InMemoryBackend) DescribeVpcEndpoints(ids []string) []*VpcEndpoint { allowed[id] = true } - endpoints := make([]*VpcEndpoint, 0, len(b.vpcEndpoints)) - for _, ep := range b.vpcEndpoints { + endpoints := make([]*VpcEndpoint, 0, b.vpcEndpoints.Len()) + for _, ep := range b.vpcEndpoints.All() { if len(allowed) > 0 && !allowed[ep.ID] { continue } diff --git a/services/ec2/backend_ec2core.go b/services/ec2/backend_ec2core.go index e0fe94911..84bc6cbac 100644 --- a/services/ec2/backend_ec2core.go +++ b/services/ec2/backend_ec2core.go @@ -94,7 +94,7 @@ func (b *InMemoryBackend) CreateEgressOnlyInternetGateway( b.mu.Lock("CreateEgressOnlyInternetGateway") defer b.mu.Unlock() - if _, ok := b.vpcs[vpcID]; !ok { + if _, ok := b.vpcs.Get(vpcID); !ok { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -104,7 +104,7 @@ func (b *InMemoryBackend) CreateEgressOnlyInternetGateway( State: stateAvailable, CreateTime: time.Now(), } - b.egressOnlyIGWs[igw.ID] = igw + b.egressOnlyIGWs.Put(igw) cp := *igw @@ -123,9 +123,9 @@ func (b *InMemoryBackend) DescribeEgressOnlyInternetGateways( idSet[id] = true } - out := make([]*EgressOnlyInternetGateway, 0, len(b.egressOnlyIGWs)) + out := make([]*EgressOnlyInternetGateway, 0, b.egressOnlyIGWs.Len()) - for _, igw := range b.egressOnlyIGWs { + for _, igw := range b.egressOnlyIGWs.All() { if len(idSet) > 0 && !idSet[igw.ID] { continue } @@ -150,11 +150,10 @@ func (b *InMemoryBackend) DeleteEgressOnlyInternetGateway(id string) error { b.mu.Lock("DeleteEgressOnlyInternetGateway") defer b.mu.Unlock() - if _, ok := b.egressOnlyIGWs[id]; !ok { + if _, ok := b.egressOnlyIGWs.Get(id); !ok { return fmt.Errorf("%w: %s", ErrEgressOnlyIGWNotFound, id) } - - delete(b.egressOnlyIGWs, id) + b.egressOnlyIGWs.Delete(id) return nil } @@ -172,7 +171,7 @@ func (b *InMemoryBackend) AssociateIamInstanceProfile( b.mu.Lock("AssociateIamInstanceProfile") defer b.mu.Unlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -183,7 +182,7 @@ func (b *InMemoryBackend) AssociateIamInstanceProfile( State: stateAvailable, Timestamp: time.Now(), } - b.iamAssociations[assoc.AssociationID] = assoc + b.iamAssociations.Put(assoc) cp := *assoc @@ -201,12 +200,11 @@ func (b *InMemoryBackend) DisassociateIamInstanceProfile( b.mu.Lock("DisassociateIamInstanceProfile") defer b.mu.Unlock() - assoc, ok := b.iamAssociations[associationID] + assoc, ok := b.iamAssociations.Get(associationID) if !ok { return nil, fmt.Errorf("%w: %s", ErrIAMAssociationNotFound, associationID) } - - delete(b.iamAssociations, associationID) + b.iamAssociations.Delete(associationID) cp := *assoc @@ -227,9 +225,9 @@ func (b *InMemoryBackend) DescribeIamInstanceProfileAssociations( idSet[id] = true } - out := make([]*IamInstanceProfileAssociation, 0, len(b.iamAssociations)) + out := make([]*IamInstanceProfileAssociation, 0, b.iamAssociations.Len()) - for _, assoc := range b.iamAssociations { + for _, assoc := range b.iamAssociations.All() { if len(idSet) > 0 && !idSet[assoc.AssociationID] { continue } @@ -260,7 +258,7 @@ func (b *InMemoryBackend) ReplaceIamInstanceProfileAssociation( b.mu.Lock("ReplaceIamInstanceProfileAssociation") defer b.mu.Unlock() - assoc, ok := b.iamAssociations[associationID] + assoc, ok := b.iamAssociations.Get(associationID) if !ok { return nil, fmt.Errorf("%w: %s", ErrIAMAssociationNotFound, associationID) } @@ -291,7 +289,7 @@ func (b *InMemoryBackend) ReplaceRouteTableAssociation( b.mu.Lock("ReplaceRouteTableAssociation") defer b.mu.Unlock() - newRT, ok := b.routeTables[newRouteTableID] + newRT, ok := b.routeTables.Get(newRouteTableID) if !ok { return "", fmt.Errorf("%w: %s", ErrRouteTableNotFound, newRouteTableID) } @@ -299,7 +297,7 @@ func (b *InMemoryBackend) ReplaceRouteTableAssociation( // Find the old association. var subnetID string - for _, rt := range b.routeTables { + for _, rt := range b.routeTables.All() { for i, assoc := range rt.Associations { if assoc.ID == associationID { subnetID = assoc.SubnetID @@ -341,7 +339,7 @@ func (b *InMemoryBackend) AssociateVpcCidrBlock( b.mu.Lock("AssociateVpcCidrBlock") defer b.mu.Unlock() - if _, ok := b.vpcs[vpcID]; !ok { + if _, ok := b.vpcs.Get(vpcID); !ok { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -370,7 +368,7 @@ func (b *InMemoryBackend) CreateTransitGatewayRouteTable( b.mu.Lock("CreateTransitGatewayRouteTable") defer b.mu.Unlock() - if _, ok := b.transitGateways[tgwID]; !ok { + if _, ok := b.transitGateways.Get(tgwID); !ok { return nil, fmt.Errorf("%w: transit gateway %s not found", ErrInvalidParameter, tgwID) } @@ -380,7 +378,7 @@ func (b *InMemoryBackend) CreateTransitGatewayRouteTable( State: stateAvailable, CreateTime: time.Now(), } - b.tgwRouteTables[rt.RouteTableID] = rt + b.tgwRouteTables.Put(rt) cp := *rt @@ -399,9 +397,9 @@ func (b *InMemoryBackend) DescribeTransitGatewayRouteTables( idSet[id] = true } - out := make([]*TransitGatewayRouteTable, 0, len(b.tgwRouteTables)) + out := make([]*TransitGatewayRouteTable, 0, b.tgwRouteTables.Len()) - for _, rt := range b.tgwRouteTables { + for _, rt := range b.tgwRouteTables.All() { if len(idSet) > 0 && !idSet[rt.RouteTableID] { continue } @@ -426,11 +424,10 @@ func (b *InMemoryBackend) DeleteTransitGatewayRouteTable(id string) error { b.mu.Lock("DeleteTransitGatewayRouteTable") defer b.mu.Unlock() - if _, ok := b.tgwRouteTables[id]; !ok { + if _, ok := b.tgwRouteTables.Get(id); !ok { return fmt.Errorf("%w: %s", ErrTGWRouteTableNotFound, id) } - - delete(b.tgwRouteTables, id) + b.tgwRouteTables.Delete(id) return nil } @@ -452,7 +449,7 @@ func (b *InMemoryBackend) CreateTransitGatewayRoute( b.mu.Lock("CreateTransitGatewayRoute") defer b.mu.Unlock() - if _, ok := b.tgwRouteTables[routeTableID]; !ok { + if _, ok := b.tgwRouteTables.Get(routeTableID); !ok { return nil, fmt.Errorf("%w: %s", ErrTGWRouteTableNotFound, routeTableID) } @@ -463,8 +460,7 @@ func (b *InMemoryBackend) CreateTransitGatewayRoute( State: stateActive, Type: tgwRouteTypeStatic, } - key := routeTableID + ":" + destinationCIDR - b.tgwRoutes[key] = route + b.tgwRoutes.Put(route) cp := *route @@ -481,7 +477,7 @@ func (b *InMemoryBackend) DeleteTransitGatewayRoute(routeTableID, destinationCID defer b.mu.Unlock() key := routeTableID + ":" + destinationCIDR - if _, ok := b.tgwRoutes[key]; !ok { + if _, ok := b.tgwRoutes.Get(key); !ok { return fmt.Errorf( "%w: route %s in %s not found", ErrInvalidParameter, @@ -489,8 +485,7 @@ func (b *InMemoryBackend) DeleteTransitGatewayRoute(routeTableID, destinationCID routeTableID, ) } - - delete(b.tgwRoutes, key) + b.tgwRoutes.Delete(key) return nil } @@ -510,11 +505,10 @@ func (b *InMemoryBackend) ReplaceTransitGatewayRoute( b.mu.Lock("ReplaceTransitGatewayRoute") defer b.mu.Unlock() - if _, ok := b.tgwRouteTables[routeTableID]; !ok { + if _, ok := b.tgwRouteTables.Get(routeTableID); !ok { return nil, fmt.Errorf("%w: %s", ErrTGWRouteTableNotFound, routeTableID) } - key := routeTableID + ":" + destinationCIDR route := &TransitGatewayRoute{ DestinationCidrBlock: destinationCIDR, TransitGatewayAttachmentID: attachmentID, @@ -522,7 +516,7 @@ func (b *InMemoryBackend) ReplaceTransitGatewayRoute( State: stateActive, Type: tgwRouteTypeStatic, } - b.tgwRoutes[key] = route + b.tgwRoutes.Put(route) cp := *route @@ -546,7 +540,7 @@ func (b *InMemoryBackend) AssociateTransitGatewayRouteTable( b.mu.Lock("AssociateTransitGatewayRouteTable") defer b.mu.Unlock() - if _, ok := b.tgwRouteTables[routeTableID]; !ok { + if _, ok := b.tgwRouteTables.Get(routeTableID); !ok { return nil, fmt.Errorf("%w: %s", ErrTGWRouteTableNotFound, routeTableID) } @@ -556,8 +550,7 @@ func (b *InMemoryBackend) AssociateTransitGatewayRouteTable( ResourceType: "vpc", State: stateAvailable, } - key := routeTableID + ":" + attachmentID - b.tgwRTAssociations[key] = assoc + b.tgwRTAssociations.Put(assoc) cp := *assoc @@ -580,7 +573,7 @@ func (b *InMemoryBackend) DisassociateTransitGatewayRouteTable( defer b.mu.Unlock() key := routeTableID + ":" + attachmentID - if _, ok := b.tgwRTAssociations[key]; !ok { + if _, ok := b.tgwRTAssociations.Get(key); !ok { return fmt.Errorf( "%w: association between %s and %s not found", ErrInvalidParameter, @@ -588,8 +581,7 @@ func (b *InMemoryBackend) DisassociateTransitGatewayRouteTable( attachmentID, ) } - - delete(b.tgwRTAssociations, key) + b.tgwRTAssociations.Delete(key) return nil } diff --git a/services/ec2/backend_ext.go b/services/ec2/backend_ext.go index 1ad3e54fc..3ef105180 100644 --- a/services/ec2/backend_ext.go +++ b/services/ec2/backend_ext.go @@ -294,7 +294,7 @@ func (b *InMemoryBackend) StartInstances(ids []string) ([]*InstanceStateChange, var result []*InstanceStateChange for _, id := range ids { - inst, ok := b.instances[id] + inst, ok := b.instances.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, id) } @@ -311,6 +311,10 @@ func (b *InMemoryBackend) StartInstances(ids []string) ([]*InstanceStateChange, prev := inst.State // AWS state machine: stopped → pending → running (reconciler advances pending→running). inst.State = StatePending + // AWS clears the state reason once the instance starts back up. + inst.StateReasonCode = "" + inst.StateReasonMessage = "" + inst.StateTransitionReason = "" result = append(result, &InstanceStateChange{ InstanceID: id, PreviousState: prev, @@ -330,7 +334,7 @@ func (b *InMemoryBackend) StopInstances(ids []string) ([]*InstanceStateChange, e var result []*InstanceStateChange for _, id := range ids { - inst, ok := b.instances[id] + inst, ok := b.instances.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, id) } @@ -345,9 +349,21 @@ func (b *InMemoryBackend) StopInstances(ids []string) ([]*InstanceStateChange, e ) } + if inst.DisableAPIStop { + return nil, fmt.Errorf( + "%w: the instance %s may not be stopped. "+ + "Modify its 'disableApiStop' instance attribute and try again", + ErrOperationNotPermitted, id) + } + prev := inst.State // AWS state machine: running/pending → stopping → stopped (reconciler advances stopping→stopped). inst.State = StateStopping + inst.StateReasonCode = "Client.UserInitiatedShutdown" + inst.StateReasonMessage = "Client.UserInitiatedShutdown: User initiated shutdown" + inst.StateTransitionReason = fmt.Sprintf( + "User initiated (%s)", time.Now().UTC().Format("2006-01-02 15:04:05 GMT"), + ) result = append(result, &InstanceStateChange{ InstanceID: id, PreviousState: prev, @@ -364,7 +380,7 @@ func (b *InMemoryBackend) RebootInstances(ids []string) error { defer b.mu.RUnlock() for _, id := range ids { - if _, ok := b.instances[id]; !ok { + if _, ok := b.instances.Get(id); !ok { return fmt.Errorf("%w: %s", ErrInstanceNotFound, id) } } @@ -380,7 +396,7 @@ func (b *InMemoryBackend) DescribeInstanceStatus(ids []string) []*Instance { var out []*Instance if len(ids) > 0 { for _, id := range ids { - inst, ok := b.instances[id] + inst, ok := b.instances.Get(id) if !ok { continue } @@ -392,7 +408,7 @@ func (b *InMemoryBackend) DescribeInstanceStatus(ids []string) []*Instance { return out } - for _, inst := range b.instances { + for _, inst := range b.instances.All() { cp := *inst out = append(out, &cp) } @@ -405,9 +421,9 @@ func (b *InMemoryBackend) DescribeImages() []AMIStub { b.mu.RLock("DescribeImages") defer b.mu.RUnlock() - images := make([]AMIStub, 0, len(stubAMIs)+len(b.images)) + images := make([]AMIStub, 0, len(stubAMIs)+b.images.Len()) images = append(images, stubAMIs...) - for _, img := range b.images { + for _, img := range b.images.All() { cp := *img images = append(images, cp) } @@ -438,7 +454,7 @@ func (b *InMemoryBackend) CreateKeyPair(name string) (*KeyPair, error) { b.mu.Lock("CreateKeyPair") defer b.mu.Unlock() - if _, exists := b.keyPairs[name]; exists { + if _, exists := b.keyPairs.Get(name); exists { return nil, fmt.Errorf("%w: %s", ErrDuplicateKeyPairName, name) } @@ -469,7 +485,7 @@ func (b *InMemoryBackend) CreateKeyPair(name string) (*KeyPair, error) { Material: string(privPEM), PublicKey: authorized, } - b.keyPairs[name] = kp + b.keyPairs.Put(kp) return kp, nil } @@ -486,7 +502,7 @@ func (b *InMemoryBackend) ImportKeyPair(name, publicKeyMaterial string) (*KeyPai b.mu.Lock("ImportKeyPair") defer b.mu.Unlock() - if _, exists := b.keyPairs[name]; exists { + if _, exists := b.keyPairs.Get(name); exists { return nil, fmt.Errorf("%w: %s", ErrDuplicateKeyPairName, name) } @@ -495,7 +511,7 @@ func (b *InMemoryBackend) ImportKeyPair(name, publicKeyMaterial string) (*KeyPai Fingerprint: "aa:bb:cc:dd:" + uuid.New().String()[:stubFingerprintUUIDLen], PublicKey: strings.TrimSpace(publicKeyMaterial), } - b.keyPairs[name] = kp + b.keyPairs.Put(kp) return kp, nil } @@ -511,7 +527,7 @@ func (b *InMemoryBackend) DescribeKeyPairs(names []string) []*KeyPair { out := make([]*KeyPair, 0, len(names)) for _, n := range names { - kp, ok := b.keyPairs[n] + kp, ok := b.keyPairs.Get(n) if !ok { continue } @@ -524,9 +540,9 @@ func (b *InMemoryBackend) DescribeKeyPairs(names []string) []*KeyPair { return out } - out := make([]*KeyPair, 0, len(b.keyPairs)) + out := make([]*KeyPair, 0, b.keyPairs.Len()) - for _, kp := range b.keyPairs { + for _, kp := range b.keyPairs.All() { cp := *kp cp.Material = "" // don't return private key material on describe out = append(out, &cp) @@ -540,11 +556,10 @@ func (b *InMemoryBackend) DeleteKeyPair(name string) error { b.mu.Lock("DeleteKeyPair") defer b.mu.Unlock() - if _, ok := b.keyPairs[name]; !ok { + if _, ok := b.keyPairs.Get(name); !ok { return fmt.Errorf("%w: %s", ErrKeyPairNotFound, name) } - - delete(b.keyPairs, name) + b.keyPairs.Delete(name) delete(b.tags, name) return nil @@ -576,7 +591,7 @@ func (b *InMemoryBackend) CreateVolume(az, volType string, size int) (*Volume, e State: stateAvailable, CreateTime: time.Now(), } - b.volumes[id] = vol + b.volumes.Put(vol) return vol, nil } @@ -592,7 +607,7 @@ func (b *InMemoryBackend) DescribeVolumes(ids []string) []*Volume { out := make([]*Volume, 0, len(ids)) for _, id := range ids { - vol, ok := b.volumes[id] + vol, ok := b.volumes.Get(id) if !ok { continue } @@ -604,9 +619,9 @@ func (b *InMemoryBackend) DescribeVolumes(ids []string) []*Volume { return out } - out := make([]*Volume, 0, len(b.volumes)) + out := make([]*Volume, 0, b.volumes.Len()) - for _, vol := range b.volumes { + for _, vol := range b.volumes.All() { cp := *vol out = append(out, &cp) } @@ -619,7 +634,7 @@ func (b *InMemoryBackend) DeleteVolume(id string) error { b.mu.Lock("DeleteVolume") defer b.mu.Unlock() - vol, ok := b.volumes[id] + vol, ok := b.volumes.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrVolumeNotFound, id) } @@ -632,8 +647,7 @@ func (b *InMemoryBackend) DeleteVolume(id string) error { vol.Attachment.InstanceID, ) } - - delete(b.volumes, id) + b.volumes.Delete(id) delete(b.tags, id) return nil @@ -646,7 +660,7 @@ func (b *InMemoryBackend) AttachVolume( b.mu.Lock("AttachVolume") defer b.mu.Unlock() - vol, ok := b.volumes[volumeID] + vol, ok := b.volumes.Get(volumeID) if !ok { return nil, fmt.Errorf("%w: %s", ErrVolumeNotFound, volumeID) } @@ -655,7 +669,7 @@ func (b *InMemoryBackend) AttachVolume( return nil, fmt.Errorf("%w: volume %s is already attached", ErrVolumeInUse, volumeID) } - if _, instOK := b.instances[instanceID]; !instOK { + if _, instOK := b.instances.Get(instanceID); !instOK { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -677,7 +691,7 @@ func (b *InMemoryBackend) DetachVolume(volumeID string, _ bool) (*VolumeAttachme b.mu.Lock("DetachVolume") defer b.mu.Unlock() - vol, ok := b.volumes[volumeID] + vol, ok := b.volumes.Get(volumeID) if !ok { return nil, fmt.Errorf("%w: %s", ErrVolumeNotFound, volumeID) } @@ -704,7 +718,7 @@ func (b *InMemoryBackend) AllocateAddress() (*Address, error) { AllocationID: id, PublicIP: b.allocElasticIP(), } - b.addresses[id] = addr + b.addresses.Put(addr) return addr, nil } @@ -714,12 +728,12 @@ func (b *InMemoryBackend) AssociateAddress(allocationID, instanceID string) (str b.mu.Lock("AssociateAddress") defer b.mu.Unlock() - addr, ok := b.addresses[allocationID] + addr, ok := b.addresses.Get(allocationID) if !ok { return "", fmt.Errorf("%w: %s", ErrAddressNotFound, allocationID) } - if _, instOK := b.instances[instanceID]; !instOK { + if _, instOK := b.instances.Get(instanceID); !instOK { return "", fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -735,7 +749,7 @@ func (b *InMemoryBackend) DisassociateAddress(associationID string) error { b.mu.Lock("DisassociateAddress") defer b.mu.Unlock() - for _, addr := range b.addresses { + for _, addr := range b.addresses.All() { if addr.AssociationID == associationID { addr.AssociationID = "" addr.InstanceID = "" @@ -752,11 +766,10 @@ func (b *InMemoryBackend) ReleaseAddress(allocationID string) error { b.mu.Lock("ReleaseAddress") defer b.mu.Unlock() - if _, ok := b.addresses[allocationID]; !ok { + if _, ok := b.addresses.Get(allocationID); !ok { return fmt.Errorf("%w: %s", ErrAddressNotFound, allocationID) } - - delete(b.addresses, allocationID) + b.addresses.Delete(allocationID) delete(b.tags, allocationID) return nil @@ -773,7 +786,7 @@ func (b *InMemoryBackend) DescribeAddresses(allocationIDs []string) []*Address { out := make([]*Address, 0, len(allocationIDs)) for _, id := range allocationIDs { - addr, ok := b.addresses[id] + addr, ok := b.addresses.Get(id) if !ok { continue } @@ -785,9 +798,9 @@ func (b *InMemoryBackend) DescribeAddresses(allocationIDs []string) []*Address { return out } - out := make([]*Address, 0, len(b.addresses)) + out := make([]*Address, 0, b.addresses.Len()) - for _, addr := range b.addresses { + for _, addr := range b.addresses.All() { cp := *addr out = append(out, &cp) } @@ -805,7 +818,7 @@ func (b *InMemoryBackend) CreateInternetGateway() (*InternetGateway, error) { ID: id, Attachments: []IGWAttachment{}, } - b.internetGateways[id] = igw + b.internetGateways.Put(igw) return igw, nil } @@ -815,11 +828,10 @@ func (b *InMemoryBackend) DeleteInternetGateway(id string) error { b.mu.Lock("DeleteInternetGateway") defer b.mu.Unlock() - if _, ok := b.internetGateways[id]; !ok { + if _, ok := b.internetGateways.Get(id); !ok { return fmt.Errorf("%w: %s", ErrInternetGatewayNotFound, id) } - - delete(b.internetGateways, id) + b.internetGateways.Delete(id) delete(b.tags, id) return nil @@ -836,7 +848,7 @@ func (b *InMemoryBackend) DescribeInternetGateways(ids []string) []*InternetGate out := make([]*InternetGateway, 0, len(ids)) for _, id := range ids { - igw, ok := b.internetGateways[id] + igw, ok := b.internetGateways.Get(id) if !ok { continue } @@ -848,9 +860,9 @@ func (b *InMemoryBackend) DescribeInternetGateways(ids []string) []*InternetGate return out } - out := make([]*InternetGateway, 0, len(b.internetGateways)) + out := make([]*InternetGateway, 0, b.internetGateways.Len()) - for _, igw := range b.internetGateways { + for _, igw := range b.internetGateways.All() { cp := *igw out = append(out, &cp) } @@ -863,12 +875,12 @@ func (b *InMemoryBackend) AttachInternetGateway(igwID, vpcID string) error { b.mu.Lock("AttachInternetGateway") defer b.mu.Unlock() - igw, ok := b.internetGateways[igwID] + igw, ok := b.internetGateways.Get(igwID) if !ok { return fmt.Errorf("%w: %s", ErrInternetGatewayNotFound, igwID) } - if _, vpcOK := b.vpcs[vpcID]; !vpcOK { + if _, vpcOK := b.vpcs.Get(vpcID); !vpcOK { return fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -882,7 +894,7 @@ func (b *InMemoryBackend) DetachInternetGateway(igwID, vpcID string) error { b.mu.Lock("DetachInternetGateway") defer b.mu.Unlock() - igw, ok := b.internetGateways[igwID] + igw, ok := b.internetGateways.Get(igwID) if !ok { return fmt.Errorf("%w: %s", ErrInternetGatewayNotFound, igwID) } @@ -903,7 +915,7 @@ func (b *InMemoryBackend) CreateRouteTable(vpcID string) (*RouteTable, error) { b.mu.Lock("CreateRouteTable") defer b.mu.Unlock() - if _, ok := b.vpcs[vpcID]; !ok { + if _, ok := b.vpcs.Get(vpcID); !ok { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -914,7 +926,7 @@ func (b *InMemoryBackend) CreateRouteTable(vpcID string) (*RouteTable, error) { Routes: []Route{}, Associations: []RouteAssociation{}, } - b.routeTables[id] = rt + b.routeTables.Put(rt) b.indexRouteTableLocked(id, vpcID) return rt, nil @@ -925,13 +937,13 @@ func (b *InMemoryBackend) DeleteRouteTable(id string) error { b.mu.Lock("DeleteRouteTable") defer b.mu.Unlock() - rt, ok := b.routeTables[id] + rt, ok := b.routeTables.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrRouteTableNotFound, id) } b.deindexRouteTableLocked(id, rt.VPCID) - delete(b.routeTables, id) + b.routeTables.Delete(id) delete(b.tags, id) return nil @@ -948,7 +960,7 @@ func (b *InMemoryBackend) DescribeRouteTables(ids []string) []*RouteTable { out := make([]*RouteTable, 0, len(ids)) for _, id := range ids { - rt, ok := b.routeTables[id] + rt, ok := b.routeTables.Get(id) if !ok { continue } @@ -960,9 +972,9 @@ func (b *InMemoryBackend) DescribeRouteTables(ids []string) []*RouteTable { return out } - out := make([]*RouteTable, 0, len(b.routeTables)) + out := make([]*RouteTable, 0, b.routeTables.Len()) - for _, rt := range b.routeTables { + for _, rt := range b.routeTables.All() { cp := *rt out = append(out, &cp) } @@ -975,7 +987,7 @@ func (b *InMemoryBackend) CreateRoute(rtID, destCIDR, gatewayID, natGatewayID st b.mu.Lock("CreateRoute") defer b.mu.Unlock() - rt, ok := b.routeTables[rtID] + rt, ok := b.routeTables.Get(rtID) if !ok { return fmt.Errorf("%w: %s", ErrRouteTableNotFound, rtID) } @@ -995,7 +1007,7 @@ func (b *InMemoryBackend) DeleteRoute(rtID, destCIDR string) error { b.mu.Lock("DeleteRoute") defer b.mu.Unlock() - rt, ok := b.routeTables[rtID] + rt, ok := b.routeTables.Get(rtID) if !ok { return fmt.Errorf("%w: %s", ErrRouteTableNotFound, rtID) } @@ -1016,12 +1028,12 @@ func (b *InMemoryBackend) AssociateRouteTable(rtID, subnetID string) (string, er b.mu.Lock("AssociateRouteTable") defer b.mu.Unlock() - rt, ok := b.routeTables[rtID] + rt, ok := b.routeTables.Get(rtID) if !ok { return "", fmt.Errorf("%w: %s", ErrRouteTableNotFound, rtID) } - if _, subnetOK := b.subnets[subnetID]; !subnetOK { + if _, subnetOK := b.subnets.Get(subnetID); !subnetOK { return "", fmt.Errorf("%w: %s", ErrSubnetNotFound, subnetID) } @@ -1040,7 +1052,7 @@ func (b *InMemoryBackend) DisassociateRouteTable(assocID string) error { b.mu.Lock("DisassociateRouteTable") defer b.mu.Unlock() - for _, rt := range b.routeTables { + for _, rt := range b.routeTables.All() { for i, assoc := range rt.Associations { if assoc.ID == assocID { rt.Associations = append(rt.Associations[:i], rt.Associations[i+1:]...) @@ -1058,12 +1070,12 @@ func (b *InMemoryBackend) CreateNatGateway(subnetID, allocationID string) (*NatG b.mu.Lock("CreateNatGateway") defer b.mu.Unlock() - subnet, ok := b.subnets[subnetID] + subnet, ok := b.subnets.Get(subnetID) if !ok { return nil, fmt.Errorf("%w: %s", ErrSubnetNotFound, subnetID) } - addr, ok := b.addresses[allocationID] + addr, ok := b.addresses.Get(allocationID) if !ok { return nil, fmt.Errorf("%w: %s", ErrAddressNotFound, allocationID) } @@ -1079,7 +1091,7 @@ func (b *InMemoryBackend) CreateNatGateway(subnetID, allocationID string) (*NatG State: stateAvailable, CreateTime: time.Now(), } - b.natGateways[id] = ngw + b.natGateways.Put(ngw) b.indexNatGatewayLocked(ngw) return ngw, nil @@ -1090,14 +1102,14 @@ func (b *InMemoryBackend) DeleteNatGateway(id string) error { b.mu.Lock("DeleteNatGateway") defer b.mu.Unlock() - ngw, ok := b.natGateways[id] + ngw, ok := b.natGateways.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrNatGatewayNotFound, id) } b.recycleIPLocked(ngw.PrivateIP) b.deindexNatGatewayLocked(ngw) - delete(b.natGateways, id) + b.natGateways.Delete(id) delete(b.tags, id) return nil @@ -1114,7 +1126,7 @@ func (b *InMemoryBackend) DescribeNatGateways(ids []string) []*NatGateway { out := make([]*NatGateway, 0, len(ids)) for _, id := range ids { - ngw, ok := b.natGateways[id] + ngw, ok := b.natGateways.Get(id) if !ok { continue } @@ -1126,9 +1138,9 @@ func (b *InMemoryBackend) DescribeNatGateways(ids []string) []*NatGateway { return out } - out := make([]*NatGateway, 0, len(b.natGateways)) + out := make([]*NatGateway, 0, b.natGateways.Len()) - for _, ngw := range b.natGateways { + for _, ngw := range b.natGateways.All() { cp := *ngw out = append(out, &cp) } @@ -1147,7 +1159,7 @@ func (b *InMemoryBackend) DescribeNetworkInterfaces(ids []string) []*NetworkInte out := make([]*NetworkInterface, 0, len(ids)) for _, id := range ids { - eni, ok := b.networkInterfaces[id] + eni, ok := b.networkInterfaces.Get(id) if !ok { continue } @@ -1159,9 +1171,9 @@ func (b *InMemoryBackend) DescribeNetworkInterfaces(ids []string) []*NetworkInte return out } - out := make([]*NetworkInterface, 0, len(b.networkInterfaces)) + out := make([]*NetworkInterface, 0, b.networkInterfaces.Len()) - for _, eni := range b.networkInterfaces { + for _, eni := range b.networkInterfaces.All() { cp := *eni out = append(out, &cp) } @@ -1177,7 +1189,7 @@ func (b *InMemoryBackend) AuthorizeSecurityGroupIngress( b.mu.Lock("AuthorizeSecurityGroupIngress") defer b.mu.Unlock() - sg, ok := b.securityGroups[groupID] + sg, ok := b.securityGroups.Get(groupID) if !ok { return fmt.Errorf("%w: %s", ErrSecurityGroupNotFound, groupID) } @@ -1199,7 +1211,7 @@ func (b *InMemoryBackend) AuthorizeSecurityGroupEgress( b.mu.Lock("AuthorizeSecurityGroupEgress") defer b.mu.Unlock() - sg, ok := b.securityGroups[groupID] + sg, ok := b.securityGroups.Get(groupID) if !ok { return fmt.Errorf("%w: %s", ErrSecurityGroupNotFound, groupID) } @@ -1221,7 +1233,7 @@ func (b *InMemoryBackend) RevokeSecurityGroupIngress( b.mu.Lock("RevokeSecurityGroupIngress") defer b.mu.Unlock() - sg, ok := b.securityGroups[groupID] + sg, ok := b.securityGroups.Get(groupID) if !ok { return fmt.Errorf("%w: %s", ErrSecurityGroupNotFound, groupID) } @@ -1242,7 +1254,7 @@ func (b *InMemoryBackend) RevokeSecurityGroupEgress( b.mu.Lock("RevokeSecurityGroupEgress") defer b.mu.Unlock() - sg, ok := b.securityGroups[groupID] + sg, ok := b.securityGroups.Get(groupID) if !ok { return fmt.Errorf("%w: %s", ErrSecurityGroupNotFound, groupID) } @@ -1293,7 +1305,7 @@ func (b *InMemoryBackend) CreateNetworkInterface( b.mu.Lock("CreateNetworkInterface") defer b.mu.Unlock() - sub, ok := b.subnets[subnetID] + sub, ok := b.subnets.Get(subnetID) if !ok { return nil, fmt.Errorf("%w: %s", ErrSubnetNotFound, subnetID) } @@ -1308,7 +1320,7 @@ func (b *InMemoryBackend) CreateNetworkInterface( Status: stateAvailable, SourceDestCheck: true, } - b.networkInterfaces[id] = eni + b.networkInterfaces.Put(eni) b.indexENILocked(id, eni) b.indexENIByVPCLocked(id, eni) @@ -1321,7 +1333,7 @@ func (b *InMemoryBackend) DeleteNetworkInterface(id string) error { b.mu.Lock("DeleteNetworkInterface") defer b.mu.Unlock() - eni, ok := b.networkInterfaces[id] + eni, ok := b.networkInterfaces.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrNetworkInterfaceNotFound, id) } @@ -1338,7 +1350,7 @@ func (b *InMemoryBackend) DeleteNetworkInterface(id string) error { b.recycleENIIPsLocked(eni) b.deindexENILocked(id, eni) b.deindexENIByVPCLocked(id, eni) - delete(b.networkInterfaces, id) + b.networkInterfaces.Delete(id) delete(b.tags, id) return nil @@ -1352,7 +1364,7 @@ func (b *InMemoryBackend) AttachNetworkInterface( b.mu.Lock("AttachNetworkInterface") defer b.mu.Unlock() - eni, ok := b.networkInterfaces[eniID] + eni, ok := b.networkInterfaces.Get(eniID) if !ok { return "", fmt.Errorf("%w: %s", ErrNetworkInterfaceNotFound, eniID) } @@ -1361,7 +1373,7 @@ func (b *InMemoryBackend) AttachNetworkInterface( return "", fmt.Errorf("%w: %s is already attached", ErrNetworkInterfaceInUse, eniID) } - if _, ok = b.instances[instanceID]; !ok { + if _, ok = b.instances.Get(instanceID); !ok { return "", fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -1386,7 +1398,7 @@ func (b *InMemoryBackend) DetachNetworkInterface(attachmentID string, _ bool) er return fmt.Errorf("%w: %s", ErrAttachmentNotFound, attachmentID) } - eni, ok := b.networkInterfaces[eniID] + eni, ok := b.networkInterfaces.Get(eniID) if !ok { delete(b.eniIDByAttachment, attachmentID) @@ -1408,7 +1420,7 @@ func (b *InMemoryBackend) AssignPrivateIPAddresses(eniID string, count int, ips b.mu.Lock("AssignPrivateIPAddresses") defer b.mu.Unlock() - eni, ok := b.networkInterfaces[eniID] + eni, ok := b.networkInterfaces.Get(eniID) if !ok { return fmt.Errorf("%w: %s", ErrNetworkInterfaceNotFound, eniID) } @@ -1433,7 +1445,7 @@ func (b *InMemoryBackend) UnassignPrivateIPAddresses(eniID string, ips []string) b.mu.Lock("UnassignPrivateIPAddresses") defer b.mu.Unlock() - eni, ok := b.networkInterfaces[eniID] + eni, ok := b.networkInterfaces.Get(eniID) if !ok { return fmt.Errorf("%w: %s", ErrNetworkInterfaceNotFound, eniID) } @@ -1465,7 +1477,7 @@ func (b *InMemoryBackend) ModifyNetworkInterfaceAttribute(eniID, attr, value str b.mu.Lock("ModifyNetworkInterfaceAttribute") defer b.mu.Unlock() - eni, ok := b.networkInterfaces[eniID] + eni, ok := b.networkInterfaces.Get(eniID) if !ok { return fmt.Errorf("%w: %s", ErrNetworkInterfaceNotFound, eniID) } @@ -1482,6 +1494,21 @@ func (b *InMemoryBackend) ModifyNetworkInterfaceAttribute(eniID, attr, value str return nil } +// PrimaryNetworkInterfaceSourceDestCheck returns the sourceDestCheck flag of +// instanceID's primary network interface. AWS defaults this to true for VPC +// instances; unknown instances/interfaces also report true to match that +// default rather than a zero-value false. +func (b *InMemoryBackend) PrimaryNetworkInterfaceSourceDestCheck(instanceID string) bool { + b.mu.RLock("PrimaryNetworkInterfaceSourceDestCheck") + defer b.mu.RUnlock() + + if eni := b.primaryNetworkInterfaceLocked(instanceID); eni != nil { + return eni.SourceDestCheck + } + + return true +} + // ---- spot instances ---- // RequestSpotInstances creates a spot instance request and immediately fulfils it with a running instance. @@ -1497,13 +1524,13 @@ func (b *InMemoryBackend) RequestSpotInstances( if subnetID == "" { subnetID = b.findDefaultSubnetID() - } else if _, ok := b.subnets[subnetID]; !ok { + } else if _, ok := b.subnets.Get(subnetID); !ok { return nil, fmt.Errorf("%w: %s", ErrSubnetNotFound, subnetID) } // Allocate a backing instance immediately (mock fulfils spot requests instantly). vpcID := "" - if sub, ok := b.subnets[subnetID]; ok { + if sub, ok := b.subnets.Get(subnetID); ok { vpcID = sub.VPCID } @@ -1518,7 +1545,7 @@ func (b *InMemoryBackend) RequestSpotInstances( LaunchTime: time.Now(), PrivateIP: b.allocPrivateIP(), } - b.instances[instanceID] = inst + b.instances.Put(inst) reqID := "sir-" + uuid.New().String()[:8] req := &SpotInstanceRequest{ @@ -1534,7 +1561,7 @@ func (b *InMemoryBackend) RequestSpotInstances( SubnetID: subnetID, }, } - b.spotRequests[reqID] = req + b.spotRequests.Put(req) return req, nil } @@ -1550,7 +1577,7 @@ func (b *InMemoryBackend) DescribeSpotInstanceRequests(ids []string) []*SpotInst out := make([]*SpotInstanceRequest, 0, len(ids)) for _, id := range ids { - req, ok := b.spotRequests[id] + req, ok := b.spotRequests.Get(id) if !ok { continue } @@ -1562,9 +1589,9 @@ func (b *InMemoryBackend) DescribeSpotInstanceRequests(ids []string) []*SpotInst return out } - out := make([]*SpotInstanceRequest, 0, len(b.spotRequests)) + out := make([]*SpotInstanceRequest, 0, b.spotRequests.Len()) - for _, req := range b.spotRequests { + for _, req := range b.spotRequests.All() { cp := *req out = append(out, &cp) } @@ -1578,7 +1605,7 @@ func (b *InMemoryBackend) CancelSpotInstanceRequests(ids []string) error { defer b.mu.Unlock() for _, id := range ids { - req, ok := b.spotRequests[id] + req, ok := b.spotRequests.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrSpotRequestNotFound, id) } @@ -1601,7 +1628,7 @@ func (b *InMemoryBackend) CreatePlacementGroup(name, strategy string) (*Placemen b.mu.Lock("CreatePlacementGroup") defer b.mu.Unlock() - if _, exists := b.placementGroups[name]; exists { + if _, exists := b.placementGroups.Get(name); exists { return nil, fmt.Errorf("%w: %s", ErrDuplicatePlacementGroupName, name) } @@ -1614,7 +1641,7 @@ func (b *InMemoryBackend) CreatePlacementGroup(name, strategy string) (*Placemen Strategy: strategy, State: stateAvailable, } - b.placementGroups[name] = pg + b.placementGroups.Put(pg) return pg, nil } @@ -1630,7 +1657,7 @@ func (b *InMemoryBackend) DescribePlacementGroups(names []string) []*PlacementGr out := make([]*PlacementGroup, 0, len(names)) for _, n := range names { - pg, ok := b.placementGroups[n] + pg, ok := b.placementGroups.Get(n) if !ok { continue } @@ -1642,9 +1669,9 @@ func (b *InMemoryBackend) DescribePlacementGroups(names []string) []*PlacementGr return out } - out := make([]*PlacementGroup, 0, len(b.placementGroups)) + out := make([]*PlacementGroup, 0, b.placementGroups.Len()) - for _, pg := range b.placementGroups { + for _, pg := range b.placementGroups.All() { cp := *pg out = append(out, &cp) } @@ -1657,11 +1684,10 @@ func (b *InMemoryBackend) DeletePlacementGroup(name string) error { b.mu.Lock("DeletePlacementGroup") defer b.mu.Unlock() - if _, ok := b.placementGroups[name]; !ok { + if _, ok := b.placementGroups.Get(name); !ok { return fmt.Errorf("%w: %s", ErrPlacementGroupNotFound, name) } - - delete(b.placementGroups, name) + b.placementGroups.Delete(name) delete(b.tags, name) return nil diff --git a/services/ec2/backend_fpga_image.go b/services/ec2/backend_fpga_image.go index ce81618d9..46e0d13a6 100644 --- a/services/ec2/backend_fpga_image.go +++ b/services/ec2/backend_fpga_image.go @@ -81,7 +81,7 @@ func (b *InMemoryBackend) CreateFpgaImage(name, description string) (*FpgaImage, CreateTime: now, UpdateTime: now, } - b.fpgaImages[afiID] = img + b.fpgaImages.Put(img) cp := *img @@ -107,7 +107,7 @@ func (b *InMemoryBackend) CopyFpgaImage( b.mu.Lock("CopyFpgaImage") defer b.mu.Unlock() - src, ok := b.fpgaImages[sourceFpgaImageID] + src, ok := b.fpgaImages.Get(sourceFpgaImageID) if sourceRegion == b.Region { if !ok { return nil, fmt.Errorf("%w: %s", ErrFpgaImageNotFound, sourceFpgaImageID) @@ -143,7 +143,7 @@ func (b *InMemoryBackend) CopyFpgaImage( CreateTime: now, UpdateTime: now, } - b.fpgaImages[afiID] = img + b.fpgaImages.Put(img) cp := *img @@ -159,11 +159,10 @@ func (b *InMemoryBackend) DeleteFpgaImage(id string) error { b.mu.Lock("DeleteFpgaImage") defer b.mu.Unlock() - if _, ok := b.fpgaImages[id]; !ok { + if _, ok := b.fpgaImages.Get(id); !ok { return fmt.Errorf("%w: %s", ErrFpgaImageNotFound, id) } - - delete(b.fpgaImages, id) + b.fpgaImages.Delete(id) return nil } @@ -180,7 +179,7 @@ func (b *InMemoryBackend) DescribeFpgaImages(ids []string) []*FpgaImage { var out []*FpgaImage - for _, img := range b.fpgaImages { + for _, img := range b.fpgaImages.All() { if len(filter) > 0 && !filter[img.FpgaImageID] { continue } @@ -204,7 +203,7 @@ func (b *InMemoryBackend) DescribeFpgaImageAttribute(id, attribute string) (*Fpg b.mu.RLock("DescribeFpgaImageAttribute") defer b.mu.RUnlock() - img, ok := b.fpgaImages[id] + img, ok := b.fpgaImages.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrFpgaImageNotFound, id) } @@ -252,7 +251,7 @@ func (b *InMemoryBackend) ModifyFpgaImageAttribute( b.mu.Lock("ModifyFpgaImageAttribute") defer b.mu.Unlock() - img, ok := b.fpgaImages[id] + img, ok := b.fpgaImages.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrFpgaImageNotFound, id) } @@ -358,7 +357,7 @@ func (b *InMemoryBackend) ResetFpgaImageAttribute(id, attribute string) error { b.mu.Lock("ResetFpgaImageAttribute") defer b.mu.Unlock() - img, ok := b.fpgaImages[id] + img, ok := b.fpgaImages.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrFpgaImageNotFound, id) } diff --git a/services/ec2/backend_host_reservations.go b/services/ec2/backend_host_reservations.go index b820d13f0..9684ecfe6 100644 --- a/services/ec2/backend_host_reservations.go +++ b/services/ec2/backend_host_reservations.go @@ -188,7 +188,7 @@ func (b *InMemoryBackend) requireHostsExist(hostIDs []string) error { } for _, id := range hostIDs { - if _, ok := b.dedicatedHosts[id]; !ok { + if _, ok := b.dedicatedHosts.Get(id); !ok { return fmt.Errorf("%w: %s", ErrHostNotFound, id) } } @@ -279,7 +279,7 @@ func (b *InMemoryBackend) PurchaseHostReservation( End: now.Add(time.Duration(offering.Duration) * time.Second), Tags: maps.Clone(tags), } - b.hostReservations[hr.HostReservationID] = hr + b.hostReservations.Put(hr) return &HostReservationPurchasePreview{ CurrencyCode: offering.CurrencyCode, @@ -311,9 +311,9 @@ func (b *InMemoryBackend) DescribeHostReservations(ids []string) []*HostReservat idSet[id] = true } - out := make([]*HostReservation, 0, len(b.hostReservations)) + out := make([]*HostReservation, 0, b.hostReservations.Len()) - for _, hr := range b.hostReservations { + for _, hr := range b.hostReservations.All() { if len(idSet) > 0 && !idSet[hr.HostReservationID] { continue } @@ -345,7 +345,7 @@ func (b *InMemoryBackend) ReleaseHosts(hostIDs []string) ([]string, []HostReleas var unsuccessful []HostReleaseResult for _, id := range hostIDs { - if _, ok := b.dedicatedHosts[id]; !ok { + if _, ok := b.dedicatedHosts.Get(id); !ok { unsuccessful = append(unsuccessful, HostReleaseResult{ HostID: id, Code: ErrHostNotFound.Error(), @@ -354,8 +354,7 @@ func (b *InMemoryBackend) ReleaseHosts(hostIDs []string) ([]string, []HostReleas continue } - - delete(b.dedicatedHosts, id) + b.dedicatedHosts.Delete(id) successful = append(successful, id) } diff --git a/services/ec2/backend_iface.go b/services/ec2/backend_iface.go index 1c8b9c377..df3e52e3a 100644 --- a/services/ec2/backend_iface.go +++ b/services/ec2/backend_iface.go @@ -17,6 +17,11 @@ type Backend interface { // Returns ErrInvalidInstanceState if the instance must be stopped for the given attribute. SetInstanceAttribute(instanceID, attribute, value string) error + // PrimaryNetworkInterfaceSourceDestCheck returns the sourceDestCheck flag + // of instanceID's primary network interface (defaults to true, matching + // AWS's default for VPC instances). + PrimaryNetworkInterfaceSourceDestCheck(instanceID string) bool + // DescribeInstances returns instances, optionally filtered by IDs or state name. DescribeInstances(ids []string, state string) []*Instance diff --git a/services/ec2/backend_image_ops.go b/services/ec2/backend_image_ops.go index 76ad27103..10308d764 100644 --- a/services/ec2/backend_image_ops.go +++ b/services/ec2/backend_image_ops.go @@ -141,7 +141,6 @@ const storeImageTaskProgressComplete = 100 // resetImageTasksLocked re-initialises the store image task map. Must be called with b.mu // held. func (b *InMemoryBackend) resetImageTasksLocked() { - b.storeImageTasks = make(map[string]*StoreImageTask) } // CreateStoreImageTask stores an existing AMI's contents in an S3 bucket, returning the @@ -155,7 +154,7 @@ func (b *InMemoryBackend) CreateStoreImageTask(imageID, bucket string) (*StoreIm b.mu.Lock("CreateStoreImageTask") defer b.mu.Unlock() - if _, ok := b.images[imageID]; !ok { + if _, ok := b.images.Get(imageID); !ok { return nil, fmt.Errorf("%w: %s", ErrImageNotFound, imageID) } @@ -167,7 +166,7 @@ func (b *InMemoryBackend) CreateStoreImageTask(imageID, bucket string) (*StoreIm ProgressPercentage: storeImageTaskProgressComplete, TaskStartTime: time.Now().UTC(), } - b.storeImageTasks[imageID] = task + b.storeImageTasks.Put(task) cp := *task @@ -184,9 +183,9 @@ func (b *InMemoryBackend) DescribeStoreImageTasks(imageIDs []string) []*StoreIma filter[id] = true } - out := make([]*StoreImageTask, 0, len(b.storeImageTasks)) + out := make([]*StoreImageTask, 0, b.storeImageTasks.Len()) - for _, t := range b.storeImageTasks { + for _, t := range b.storeImageTasks.All() { if len(filter) > 0 && !filter[t.AmiID] { continue } @@ -212,7 +211,7 @@ func (b *InMemoryBackend) CreateRestoreImageTask(bucket, objectKey, name string) var source *StoreImageTask - for _, t := range b.storeImageTasks { + for _, t := range b.storeImageTasks.All() { if t.Bucket == bucket && t.S3ObjectKey == objectKey { source = t @@ -225,7 +224,7 @@ func (b *InMemoryBackend) CreateRestoreImageTask(bucket, objectKey, name string) } if name == "" { - if orig, ok := b.images[source.AmiID]; ok { + if orig, ok := b.images.Get(source.AmiID); ok { name = orig.Name } } @@ -236,7 +235,7 @@ func (b *InMemoryBackend) CreateRestoreImageTask(bucket, objectKey, name string) Architecture: archX8664, State: stateAvailable, } - b.images[img.ImageID] = img + b.images.Put(img) cp := *img @@ -266,7 +265,6 @@ type UsageReport struct { // resetUsageReportMapsLocked re-initialises the usage report state maps. Must be called with // b.mu held. func (b *InMemoryBackend) resetUsageReportMapsLocked() { - b.usageReports = make(map[string]*UsageReport) b.usageReportEntries = make(map[string][]*UsageReportEntry) } @@ -283,7 +281,7 @@ func (b *InMemoryBackend) CreateImageUsageReport( b.mu.Lock("CreateImageUsageReport") defer b.mu.Unlock() - if _, ok := b.images[imageID]; !ok { + if _, ok := b.images.Get(imageID); !ok { return nil, fmt.Errorf("%w: %s", ErrImageNotFound, imageID) } @@ -300,7 +298,7 @@ func (b *InMemoryBackend) CreateImageUsageReport( now := time.Now().UTC() reportID := "imgusgrpt-" + uuid.New().String()[:8] report := &UsageReport{ReportID: reportID, ImageID: imageID, CreatedAt: now} - b.usageReports[reportID] = report + b.usageReports.Put(report) accountWanted := len(wantAccounts) == 0 || wantAccounts[b.AccountID] @@ -335,7 +333,7 @@ func (b *InMemoryBackend) CreateImageUsageReport( func (b *InMemoryBackend) countInstancesUsingImageLocked(imageID string) int64 { var count int64 - for _, inst := range b.instances { + for _, inst := range b.instances.All() { if inst.ImageID == imageID { count++ } @@ -349,7 +347,7 @@ func (b *InMemoryBackend) countInstancesUsingImageLocked(imageID string) int64 { func (b *InMemoryBackend) countLaunchTemplatesUsingImageLocked(imageID string) int64 { var count int64 - for _, lt := range b.launchTemplates { + for _, lt := range b.launchTemplates.All() { if lt.ImageID == imageID { count++ } @@ -367,11 +365,10 @@ func (b *InMemoryBackend) DeleteImageUsageReport(reportID string) error { b.mu.Lock("DeleteImageUsageReport") defer b.mu.Unlock() - if _, ok := b.usageReports[reportID]; !ok { + if _, ok := b.usageReports.Get(reportID); !ok { return fmt.Errorf("%w: %s", ErrUsageReportNotFound, reportID) } - - delete(b.usageReports, reportID) + b.usageReports.Delete(reportID) delete(b.usageReportEntries, reportID) return nil @@ -436,7 +433,7 @@ func (b *InMemoryBackend) ConfirmProductInstance(instanceID, productCode string) b.mu.RLock("ConfirmProductInstance") defer b.mu.RUnlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return false, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } diff --git a/services/ec2/backend_indexes.go b/services/ec2/backend_indexes.go index 4cccbefbb..b5f6e51f7 100644 --- a/services/ec2/backend_indexes.go +++ b/services/ec2/backend_indexes.go @@ -67,6 +67,31 @@ func (b *InMemoryBackend) deindexENILocked(eniID string, eni *NetworkInterface) } } +// primaryNetworkInterfaceLocked returns the primary (deviceIndex 0) network +// interface attached to instanceID, falling back to any attached interface if +// none is explicitly device-index 0. Returns nil if the instance has no ENIs. +// Must be called with b.mu held. +func (b *InMemoryBackend) primaryNetworkInterfaceLocked(instanceID string) *NetworkInterface { + var fallback *NetworkInterface + + for eniID := range b.eniIDsByInstance[instanceID] { + eni, ok := b.networkInterfaces.Get(eniID) + if !ok { + continue + } + + if eni.DeviceIndex == 0 { + return eni + } + + if fallback == nil { + fallback = eni + } + } + + return fallback +} + func (b *InMemoryBackend) indexSubnetLocked(subnetID, vpcID string) { if subnetID == "" || vpcID == "" { return @@ -225,28 +250,32 @@ func initSecondaryIndexMaps(b *InMemoryBackend) { func (b *InMemoryBackend) rebuildSecondaryIndexesLocked() { initSecondaryIndexMaps(b) - for _, inst := range b.instances { + for _, inst := range b.instances.All() { b.indexInstanceLocked(inst) } - for eniID, eni := range b.networkInterfaces { + for _, eni := range b.networkInterfaces.All() { + eniID := networkInterfacesKeyFn(eni) b.indexENILocked(eniID, eni) b.indexENIByVPCLocked(eniID, eni) } - for _, ngw := range b.natGateways { + for _, ngw := range b.natGateways.All() { b.indexNatGatewayLocked(ngw) } - for id, subnet := range b.subnets { + for _, subnet := range b.subnets.All() { + id := subnetsKeyFn(subnet) b.indexSubnetLocked(id, subnet.VPCID) } - for id, rt := range b.routeTables { + for _, rt := range b.routeTables.All() { + id := routeTablesKeyFn(rt) b.indexRouteTableLocked(id, rt.VPCID) } - for id, sg := range b.securityGroups { + for _, sg := range b.securityGroups.All() { + id := securityGroupsKeyFn(sg) b.indexSGLocked(id, sg.VPCID) } } diff --git a/services/ec2/backend_instance_attrs.go b/services/ec2/backend_instance_attrs.go index 0130159a1..a8eb87e70 100644 --- a/services/ec2/backend_instance_attrs.go +++ b/services/ec2/backend_instance_attrs.go @@ -145,7 +145,7 @@ func (b *InMemoryBackend) ModifyHosts( unsuccessful := make([]HostModifyFailure, 0) for _, id := range hostIDs { - host, ok := b.dedicatedHosts[id] + host, ok := b.dedicatedHosts.Get(id) if !ok { unsuccessful = append(unsuccessful, HostModifyFailure{ HostID: id, Code: ErrHostNotFound.Error(), Message: "Host " + id + " does not exist", @@ -202,13 +202,13 @@ func (b *InMemoryBackend) ModifyInstanceCapacityReservationAttributes( b.mu.Lock("ModifyInstanceCapacityReservationAttributes") defer b.mu.Unlock() - inst, ok := b.instances[instanceID] + inst, ok := b.instances.Get(instanceID) if !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } if targetID != "" { - if _, found := b.capacityReservations[targetID]; !found { + if _, found := b.capacityReservations.Get(targetID); !found { return nil, fmt.Errorf("%w: %s", ErrCapacityReservationNotFound, targetID) } } @@ -238,7 +238,7 @@ func (b *InMemoryBackend) ModifyInstanceCPUOptions( b.mu.Lock("ModifyInstanceCpuOptions") defer b.mu.Unlock() - inst, ok := b.instances[instanceID] + inst, ok := b.instances.Get(instanceID) if !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -288,7 +288,7 @@ func (b *InMemoryBackend) ModifyInstanceEventStartTime( b.mu.Lock("ModifyInstanceEventStartTime") defer b.mu.Unlock() - inst, ok := b.instances[instanceID] + inst, ok := b.instances.Get(instanceID) if !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -321,7 +321,7 @@ func (b *InMemoryBackend) ModifyInstanceMaintenanceOptions( b.mu.Lock("ModifyInstanceMaintenanceOptions") defer b.mu.Unlock() - inst, ok := b.instances[instanceID] + inst, ok := b.instances.Get(instanceID) if !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -357,7 +357,7 @@ func (b *InMemoryBackend) ModifyInstanceNetworkPerformanceOptions( b.mu.Lock("ModifyInstanceNetworkPerformanceOptions") defer b.mu.Unlock() - inst, ok := b.instances[instanceID] + inst, ok := b.instances.Get(instanceID) if !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -394,7 +394,7 @@ func (b *InMemoryBackend) ModifyInstancePlacement(in ModifyInstancePlacementInpu b.mu.Lock("ModifyInstancePlacement") defer b.mu.Unlock() - inst, ok := b.instances[in.InstanceID] + inst, ok := b.instances.Get(in.InstanceID) if !ok { return false, fmt.Errorf("%w: %s", ErrInstanceNotFound, in.InstanceID) } @@ -406,7 +406,7 @@ func (b *InMemoryBackend) ModifyInstancePlacement(in ModifyInstancePlacementInpu } if in.HostID != "" { - if _, found := b.dedicatedHosts[in.HostID]; !found { + if _, found := b.dedicatedHosts.Get(in.HostID); !found { return false, fmt.Errorf("%w: %s", ErrHostNotFound, in.HostID) } } @@ -462,7 +462,7 @@ func (b *InMemoryBackend) ModifyPrivateDNSNameOptions( b.mu.Lock("ModifyPrivateDnsNameOptions") defer b.mu.Unlock() - inst, ok := b.instances[instanceID] + inst, ok := b.instances.Get(instanceID) if !ok { return false, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -498,7 +498,7 @@ func (b *InMemoryBackend) ModifyPublicIPDNSNameOptions(networkInterfaceID, hostn b.mu.Lock("ModifyPublicIpDnsNameOptions") defer b.mu.Unlock() - ni, ok := b.networkInterfaces[networkInterfaceID] + ni, ok := b.networkInterfaces.Get(networkInterfaceID) if !ok { return false, fmt.Errorf("%w: %s", ErrNetworkInterfaceNotFound, networkInterfaceID) } @@ -522,7 +522,7 @@ func (b *InMemoryBackend) AssociateInstanceEventWindow( b.mu.Lock("AssociateInstanceEventWindow") defer b.mu.Unlock() - ew, ok := b.instanceEventWindows[windowID] + ew, ok := b.instanceEventWindows.Get(windowID) if !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceEventWindowNotFound, windowID) } @@ -547,7 +547,7 @@ func (b *InMemoryBackend) DisassociateInstanceEventWindow( b.mu.Lock("DisassociateInstanceEventWindow") defer b.mu.Unlock() - ew, ok := b.instanceEventWindows[windowID] + ew, ok := b.instanceEventWindows.Get(windowID) if !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceEventWindowNotFound, windowID) } @@ -624,7 +624,7 @@ func (b *InMemoryBackend) GetInstanceTpmEkPub(instanceID, keyFormat, keyType str b.mu.RLock("GetInstanceTpmEkPub") defer b.mu.RUnlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return "", fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -644,7 +644,7 @@ func (b *InMemoryBackend) GetInstanceUefiData(instanceID string) (string, error) b.mu.RLock("GetInstanceUefiData") defer b.mu.RUnlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return "", fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } diff --git a/services/ec2/backend_ip_pools.go b/services/ec2/backend_ip_pools.go index db2c7461a..aeb3bc9c2 100644 --- a/services/ec2/backend_ip_pools.go +++ b/services/ec2/backend_ip_pools.go @@ -102,10 +102,6 @@ type Ipv6CidrAssociation struct { // resetIPPoolMapsLocked re-initialises all maps owned by this file. Must be called with // b.mu held. func (b *InMemoryBackend) resetIPPoolMapsLocked() { - b.coipPools = make(map[string]*CoipPool) - b.coipCidrs = make(map[string]*CoipCidr) - b.ipv4Pools = make(map[string]*Ipv4Pool) - b.ipv6Pools = make(map[string]*Ipv6Pool) } // ---- COIP pools ---- @@ -127,7 +123,7 @@ func (b *InMemoryBackend) CreateCoipPool(localGatewayRouteTableID string, tags m LocalGatewayRouteTableID: localGatewayRouteTableID, Tags: copyStringMap(tags), } - b.coipPools[id] = pool + b.coipPools.Put(pool) return copyCoipPool(pool), nil } @@ -141,16 +137,16 @@ func (b *InMemoryBackend) DeleteCoipPool(poolID string) (*CoipPool, error) { b.mu.Lock("DeleteCoipPool") defer b.mu.Unlock() - pool, ok := b.coipPools[poolID] + pool, ok := b.coipPools.Get(poolID) if !ok { return nil, fmt.Errorf("%w: %s", ErrCoipPoolNotFound, poolID) } + b.coipPools.Delete(poolID) - delete(b.coipPools, poolID) - - for key, cidr := range b.coipCidrs { + for _, cidr := range b.coipCidrs.All() { + key := coipCidrsKeyFn(cidr) if cidr.CoipPoolID == poolID { - delete(b.coipCidrs, key) + b.coipCidrs.Delete(key) } } @@ -167,9 +163,9 @@ func (b *InMemoryBackend) DescribeCoipPools(ids []string) []*CoipPool { filter[id] = true } - out := make([]*CoipPool, 0, len(b.coipPools)) + out := make([]*CoipPool, 0, b.coipPools.Len()) - for _, pool := range b.coipPools { + for _, pool := range b.coipPools.All() { if len(filter) > 0 && !filter[pool.PoolID] { continue } @@ -196,13 +192,13 @@ func (b *InMemoryBackend) CreateCoipCidr(poolID, cidr string) (*CoipCidr, error) b.mu.Lock("CreateCoipCidr") defer b.mu.Unlock() - pool, ok := b.coipPools[poolID] + pool, ok := b.coipPools.Get(poolID) if !ok { return nil, fmt.Errorf("%w: %s", ErrCoipPoolNotFound, poolID) } entry := &CoipCidr{Cidr: cidr, CoipPoolID: poolID, LocalGatewayRouteTableID: pool.LocalGatewayRouteTableID} - b.coipCidrs[coipCidrKey(poolID, cidr)] = entry + b.coipCidrs.Put(entry) pool.PoolCidrs = appendUniqueString(pool.PoolCidrs, cidr) cp := *entry @@ -221,14 +217,13 @@ func (b *InMemoryBackend) DeleteCoipCidr(poolID, cidr string) (*CoipCidr, error) key := coipCidrKey(poolID, cidr) - entry, ok := b.coipCidrs[key] + entry, ok := b.coipCidrs.Get(key) if !ok { return nil, fmt.Errorf("%w: %s not found in pool %s", ErrCoipCidrNotFound, cidr, poolID) } + b.coipCidrs.Delete(key) - delete(b.coipCidrs, key) - - if pool, poolOK := b.coipPools[poolID]; poolOK { + if pool, poolOK := b.coipPools.Get(poolID); poolOK { pool.PoolCidrs = removeString(pool.PoolCidrs, cidr) } @@ -247,7 +242,7 @@ func (b *InMemoryBackend) GetCoipPoolUsage(poolID string) (*CoipPool, error) { b.mu.RLock("GetCoipPoolUsage") defer b.mu.RUnlock() - pool, ok := b.coipPools[poolID] + pool, ok := b.coipPools.Get(poolID) if !ok { return nil, fmt.Errorf("%w: %s", ErrCoipPoolNotFound, poolID) } @@ -280,7 +275,7 @@ func (b *InMemoryBackend) CreatePublicIpv4Pool(networkBorderGroup string, tags m NetworkBorderGroup: networkBorderGroup, Tags: copyStringMap(tags), } - b.ipv4Pools[id] = pool + b.ipv4Pools.Put(pool) return copyIpv4Pool(pool) } @@ -294,11 +289,10 @@ func (b *InMemoryBackend) DeletePublicIpv4Pool(poolID string) error { b.mu.Lock("DeletePublicIpv4Pool") defer b.mu.Unlock() - if _, ok := b.ipv4Pools[poolID]; !ok { + if _, ok := b.ipv4Pools.Get(poolID); !ok { return fmt.Errorf("%w: %s", ErrIpv4PoolNotFound, poolID) } - - delete(b.ipv4Pools, poolID) + b.ipv4Pools.Delete(poolID) return nil } @@ -313,9 +307,9 @@ func (b *InMemoryBackend) DescribePublicIpv4Pools(ids []string) []*Ipv4Pool { filter[id] = true } - out := make([]*Ipv4Pool, 0, len(b.ipv4Pools)) + out := make([]*Ipv4Pool, 0, b.ipv4Pools.Len()) - for _, pool := range b.ipv4Pools { + for _, pool := range b.ipv4Pools.All() { if len(filter) > 0 && !filter[pool.PoolID] { continue } @@ -342,7 +336,7 @@ func (b *InMemoryBackend) ProvisionPublicIpv4PoolCidr(poolID string, netmaskLeng b.mu.Lock("ProvisionPublicIpv4PoolCidr") defer b.mu.Unlock() - pool, ok := b.ipv4Pools[poolID] + pool, ok := b.ipv4Pools.Get(poolID) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpv4PoolNotFound, poolID) } @@ -373,7 +367,7 @@ func (b *InMemoryBackend) DeprovisionPublicIpv4PoolCidr(poolID, cidr string) err b.mu.Lock("DeprovisionPublicIpv4PoolCidr") defer b.mu.Unlock() - pool, ok := b.ipv4Pools[poolID] + pool, ok := b.ipv4Pools.Get(poolID) if !ok { return fmt.Errorf("%w: %s", ErrIpv4PoolNotFound, poolID) } @@ -423,7 +417,7 @@ func (b *InMemoryBackend) CreateIpv6Pool(description string, poolCidrBlocks []st Description: description, PoolCidrBlocks: append([]string(nil), poolCidrBlocks...), } - b.ipv6Pools[id] = pool + b.ipv6Pools.Put(pool) return copyIpv6Pool(pool) } @@ -438,9 +432,9 @@ func (b *InMemoryBackend) DescribeIpv6Pools(ids []string) []*Ipv6Pool { filter[id] = true } - out := make([]*Ipv6Pool, 0, len(b.ipv6Pools)) + out := make([]*Ipv6Pool, 0, b.ipv6Pools.Len()) - for _, pool := range b.ipv6Pools { + for _, pool := range b.ipv6Pools.All() { if len(filter) > 0 && !filter[pool.PoolID] { continue } @@ -464,7 +458,7 @@ func (b *InMemoryBackend) GetAssociatedIpv6PoolCidrs(poolID string) ([]Ipv6CidrA b.mu.RLock("GetAssociatedIpv6PoolCidrs") defer b.mu.RUnlock() - pool, ok := b.ipv6Pools[poolID] + pool, ok := b.ipv6Pools.Get(poolID) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpv6PoolNotFound, poolID) } diff --git a/services/ec2/backend_ipam_discovery.go b/services/ec2/backend_ipam_discovery.go index 56c886c38..ece71df4d 100644 --- a/services/ec2/backend_ipam_discovery.go +++ b/services/ec2/backend_ipam_discovery.go @@ -193,13 +193,7 @@ type IpamPrefixListResolverTarget struct { // resetIpamDiscoveryMapsLocked re-initialises all maps owned by this file. Must be called // with b.mu held. func (b *InMemoryBackend) resetIpamDiscoveryMapsLocked() { - b.ipamByoasns = make(map[string]*IpamByoasn) - b.ipamAsnAssociations = make(map[string]*IpamAsnAssociation) - b.ipamVerificationTokens = make(map[string]*IpamExternalResourceVerificationToken) - b.ipamResourceCidrs = make(map[string]*IpamResourceCidr) - b.ipamPrefixListResolvers = make(map[string]*IpamPrefixListResolver) b.ipamPrefixListResolverVersions = make(map[string][]int64) - b.ipamPrefixListResolverTargets = make(map[string]*IpamPrefixListResolverTarget) } // ---- IPAM Resource Discoveries (user-created) ---- @@ -223,7 +217,7 @@ func (b *InMemoryBackend) CreateIpamResourceDiscovery( Description: description, OperatingRegions: append([]string(nil), operatingRegions...), } - b.ipamResourceDiscoveries[id] = d + b.ipamResourceDiscoveries.Put(d) cp := *d cp.OperatingRegions = append([]string(nil), d.OperatingRegions...) @@ -242,7 +236,7 @@ func (b *InMemoryBackend) DeleteIpamResourceDiscovery(id string) (*IpamResourceD b.mu.Lock("DeleteIpamResourceDiscovery") defer b.mu.Unlock() - d, ok := b.ipamResourceDiscoveries[id] + d, ok := b.ipamResourceDiscoveries.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamResourceDiscoveryNotFound, id) } @@ -253,15 +247,14 @@ func (b *InMemoryBackend) DeleteIpamResourceDiscovery(id string) (*IpamResourceD ) } - for _, a := range b.ipamResourceDiscoveryAssocs { + for _, a := range b.ipamResourceDiscoveryAssocs.All() { if a.IpamResourceDiscoveryID == id { return nil, fmt.Errorf( "%w: resource discovery %s has existing IPAM associations", ErrIpamResourceDiscoveryInUse, id, ) } } - - delete(b.ipamResourceDiscoveries, id) + b.ipamResourceDiscoveries.Delete(id) cp := *d cp.OperatingRegions = append([]string(nil), d.OperatingRegions...) @@ -281,12 +274,12 @@ func (b *InMemoryBackend) AssociateIpamResourceDiscovery( b.mu.Lock("AssociateIpamResourceDiscovery") defer b.mu.Unlock() - ipam, ok := b.ipams[ipamID] + ipam, ok := b.ipams.Get(ipamID) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamNotFound, ipamID) } - if _, discoveryOK := b.ipamResourceDiscoveries[discoveryID]; !discoveryOK { + if _, discoveryOK := b.ipamResourceDiscoveries.Get(discoveryID); !discoveryOK { return nil, fmt.Errorf("%w: %s", ErrIpamResourceDiscoveryNotFound, discoveryID) } @@ -304,7 +297,7 @@ func (b *InMemoryBackend) AssociateIpamResourceDiscovery( } assoc.IpamResourceDiscoveryAssociationARN = "arn:aws:ec2:" + b.Region + ":" + b.AccountID + ":ipam-resource-discovery-association/" + assocID - b.ipamResourceDiscoveryAssocs[assocID] = assoc + b.ipamResourceDiscoveryAssocs.Put(assoc) ipam.ResourceDiscoveryAssociationCount++ cp := *assoc @@ -324,7 +317,7 @@ func (b *InMemoryBackend) DisassociateIpamResourceDiscovery( b.mu.Lock("DisassociateIpamResourceDiscovery") defer b.mu.Unlock() - assoc, ok := b.ipamResourceDiscoveryAssocs[assocID] + assoc, ok := b.ipamResourceDiscoveryAssocs.Get(assocID) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamResourceDiscoveryAssociationNotFound, assocID) } @@ -336,11 +329,10 @@ func (b *InMemoryBackend) DisassociateIpamResourceDiscovery( ) } - if ipam, ipamOK := b.ipams[assoc.IpamID]; ipamOK && ipam.ResourceDiscoveryAssociationCount > 0 { + if ipam, ipamOK := b.ipams.Get(assoc.IpamID); ipamOK && ipam.ResourceDiscoveryAssociationCount > 0 { ipam.ResourceDiscoveryAssociationCount-- } - - delete(b.ipamResourceDiscoveryAssocs, assocID) + b.ipamResourceDiscoveryAssocs.Delete(assocID) cp := *assoc cp.State = ipamAssocStateDisassociateComplete @@ -359,7 +351,7 @@ func (b *InMemoryBackend) ModifyIpamResourceDiscovery( b.mu.Lock("ModifyIpamResourceDiscovery") defer b.mu.Unlock() - d, ok := b.ipamResourceDiscoveries[id] + d, ok := b.ipamResourceDiscoveries.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamResourceDiscoveryNotFound, id) } @@ -423,8 +415,7 @@ func (b *InMemoryBackend) recordIpamResourceCidrLocked(pool *IpamPool, alloc *Ip return } - key := ipamResourceCidrKey(alloc.ResourceID, alloc.Cidr) - b.ipamResourceCidrs[key] = &IpamResourceCidr{ + b.ipamResourceCidrs.Put(&IpamResourceCidr{ IpamID: pool.IpamID, IpamPoolID: pool.IpamPoolID, IpamScopeID: pool.IpamScopeID, @@ -435,13 +426,13 @@ func (b *InMemoryBackend) recordIpamResourceCidrLocked(pool *IpamPool, alloc *Ip ResourceOwnerID: alloc.ResourceOwner, ManagementState: "managed", Monitored: true, - } + }) } // forgetIpamResourceCidrLocked removes the monitored resource CIDR entry backing a released // pool allocation. Must be called with b.mu held. func (b *InMemoryBackend) forgetIpamResourceCidrLocked(alloc *IpamPoolAllocation) { - delete(b.ipamResourceCidrs, ipamResourceCidrKey(alloc.ResourceID, alloc.Cidr)) + b.ipamResourceCidrs.Delete(ipamResourceCidrKey(alloc.ResourceID, alloc.Cidr)) } // GetIpamResourceCidrs returns resource CIDRs monitored by IPAM in the given scope, optionally @@ -456,13 +447,13 @@ func (b *InMemoryBackend) GetIpamResourceCidrs( b.mu.RLock("GetIpamResourceCidrs") defer b.mu.RUnlock() - if _, ok := b.ipamScopes[scopeID]; !ok { + if _, ok := b.ipamScopes.Get(scopeID); !ok { return nil, fmt.Errorf("%w: %s", ErrIpamScopeNotFound, scopeID) } - out := make([]*IpamResourceCidr, 0, len(b.ipamResourceCidrs)) + out := make([]*IpamResourceCidr, 0, b.ipamResourceCidrs.Len()) - for _, c := range b.ipamResourceCidrs { + for _, c := range b.ipamResourceCidrs.All() { if c.IpamScopeID != scopeID { continue } @@ -510,7 +501,7 @@ func (b *InMemoryBackend) ModifyIpamResourceCidr( key := ipamResourceCidrKey(resourceID, resourceCidr) - c, ok := b.ipamResourceCidrs[key] + c, ok := b.ipamResourceCidrs.Get(key) if !ok { return nil, fmt.Errorf("%w: %s %s", ErrIpamResourceCidrNotFound, resourceID, resourceCidr) } @@ -552,7 +543,7 @@ func (b *InMemoryBackend) ProvisionIpamByoasn(ipamID, asn string) (*IpamByoasn, b.mu.Lock("ProvisionIpamByoasn") defer b.mu.Unlock() - if _, ok := b.ipams[ipamID]; !ok { + if _, ok := b.ipams.Get(ipamID); !ok { return nil, fmt.Errorf("%w: %s", ErrIpamNotFound, ipamID) } @@ -561,7 +552,7 @@ func (b *InMemoryBackend) ProvisionIpamByoasn(ipamID, asn string) (*IpamByoasn, IpamID: ipamID, State: ipamByoasnStateProvisioned, } - b.ipamByoasns[asn] = byoasn + b.ipamByoasns.Put(byoasn) cp := *byoasn @@ -577,12 +568,11 @@ func (b *InMemoryBackend) DeprovisionIpamByoasn(ipamID, asn string) (*IpamByoasn b.mu.Lock("DeprovisionIpamByoasn") defer b.mu.Unlock() - byoasn, ok := b.ipamByoasns[asn] + byoasn, ok := b.ipamByoasns.Get(asn) if !ok || byoasn.IpamID != ipamID { return nil, fmt.Errorf("%w: %s", ErrIpamByoasnNotFound, asn) } - - delete(b.ipamByoasns, asn) + b.ipamByoasns.Delete(asn) cp := *byoasn cp.State = ipamByoasnStateDeprovisioned @@ -595,8 +585,8 @@ func (b *InMemoryBackend) DescribeIpamByoasn() []*IpamByoasn { b.mu.RLock("DescribeIpamByoasn") defer b.mu.RUnlock() - out := make([]*IpamByoasn, 0, len(b.ipamByoasns)) - for _, a := range b.ipamByoasns { + out := make([]*IpamByoasn, 0, b.ipamByoasns.Len()) + for _, a := range b.ipamByoasns.All() { cp := *a out = append(out, &cp) } @@ -615,13 +605,12 @@ func (b *InMemoryBackend) AssociateIpamByoasn(asn, cidr string) (*IpamAsnAssocia b.mu.Lock("AssociateIpamByoasn") defer b.mu.Unlock() - if _, ok := b.ipamByoasns[asn]; !ok { + if _, ok := b.ipamByoasns.Get(asn); !ok { return nil, fmt.Errorf("%w: %s", ErrIpamByoasnNotFound, asn) } - key := asn + "|" + cidr assoc := &IpamAsnAssociation{Asn: asn, Cidr: cidr, State: ipamAsnAssocStateAssociated} - b.ipamAsnAssociations[key] = assoc + b.ipamAsnAssociations.Put(assoc) cp := *assoc @@ -638,13 +627,11 @@ func (b *InMemoryBackend) DisassociateIpamByoasn(asn, cidr string) (*IpamAsnAsso defer b.mu.Unlock() key := asn + "|" + cidr - - assoc, ok := b.ipamAsnAssociations[key] + assoc, ok := b.ipamAsnAssociations.Get(key) if !ok { return nil, fmt.Errorf("%w: %s / %s", ErrIpamAsnAssociationNotFound, asn, cidr) } - - delete(b.ipamAsnAssociations, key) + b.ipamAsnAssociations.Delete(key) cp := *assoc cp.State = ipamAsnAssocStateDisassociated @@ -666,7 +653,7 @@ func (b *InMemoryBackend) CreateIpamExternalResourceVerificationToken( b.mu.Lock("CreateIpamExternalResourceVerificationToken") defer b.mu.Unlock() - ipam, ok := b.ipams[ipamID] + ipam, ok := b.ipams.Get(ipamID) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamNotFound, ipamID) } @@ -686,7 +673,7 @@ func (b *InMemoryBackend) CreateIpamExternalResourceVerificationToken( TokenValue: uuid.New().String(), NotAfter: now.Add(ipamVerificationTokenValidity), } - b.ipamVerificationTokens[id] = token + b.ipamVerificationTokens.Put(token) cp := *token @@ -704,12 +691,11 @@ func (b *InMemoryBackend) DeleteIpamExternalResourceVerificationToken( b.mu.Lock("DeleteIpamExternalResourceVerificationToken") defer b.mu.Unlock() - token, ok := b.ipamVerificationTokens[id] + token, ok := b.ipamVerificationTokens.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamVerificationTokenNotFound, id) } - - delete(b.ipamVerificationTokens, id) + b.ipamVerificationTokens.Delete(id) cp := *token cp.State = ipamStateDeleteComplete @@ -730,9 +716,9 @@ func (b *InMemoryBackend) DescribeIpamExternalResourceVerificationTokens( idSet[id] = true } - out := make([]*IpamExternalResourceVerificationToken, 0, len(b.ipamVerificationTokens)) + out := make([]*IpamExternalResourceVerificationToken, 0, b.ipamVerificationTokens.Len()) - for _, t := range b.ipamVerificationTokens { + for _, t := range b.ipamVerificationTokens.All() { if len(idSet) > 0 && !idSet[t.IpamExternalResourceVerificationTokenID] { continue } @@ -767,7 +753,7 @@ func (b *InMemoryBackend) CreateIpamPrefixListResolver( b.mu.Lock("CreateIpamPrefixListResolver") defer b.mu.Unlock() - ipam, ok := b.ipams[ipamID] + ipam, ok := b.ipams.Get(ipamID) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamNotFound, ipamID) } @@ -788,7 +774,7 @@ func (b *InMemoryBackend) CreateIpamPrefixListResolver( Rules: append([]IpamPrefixListResolverRule(nil), rules...), CurrentVersion: 1, } - b.ipamPrefixListResolvers[id] = resolver + b.ipamPrefixListResolvers.Put(resolver) b.ipamPrefixListResolverVersions[id] = []int64{1} return copyIpamPrefixListResolver(resolver), nil @@ -803,17 +789,17 @@ func (b *InMemoryBackend) DeleteIpamPrefixListResolver(id string) (*IpamPrefixLi b.mu.Lock("DeleteIpamPrefixListResolver") defer b.mu.Unlock() - resolver, ok := b.ipamPrefixListResolvers[id] + resolver, ok := b.ipamPrefixListResolvers.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPrefixListResolverNotFound, id) } - - delete(b.ipamPrefixListResolvers, id) + b.ipamPrefixListResolvers.Delete(id) delete(b.ipamPrefixListResolverVersions, id) - for targetID, t := range b.ipamPrefixListResolverTargets { + for _, t := range b.ipamPrefixListResolverTargets.All() { + targetID := ipamPrefixListResolverTargetsKeyFn(t) if t.IpamPrefixListResolverID == id { - delete(b.ipamPrefixListResolverTargets, targetID) + b.ipamPrefixListResolverTargets.Delete(targetID) } } @@ -833,9 +819,9 @@ func (b *InMemoryBackend) DescribeIpamPrefixListResolvers(ids []string) []*IpamP idSet[id] = true } - out := make([]*IpamPrefixListResolver, 0, len(b.ipamPrefixListResolvers)) + out := make([]*IpamPrefixListResolver, 0, b.ipamPrefixListResolvers.Len()) - for _, r := range b.ipamPrefixListResolvers { + for _, r := range b.ipamPrefixListResolvers.All() { if len(idSet) > 0 && !idSet[r.IpamPrefixListResolverID] { continue } @@ -862,7 +848,7 @@ func (b *InMemoryBackend) ModifyIpamPrefixListResolver( b.mu.Lock("ModifyIpamPrefixListResolver") defer b.mu.Unlock() - resolver, ok := b.ipamPrefixListResolvers[id] + resolver, ok := b.ipamPrefixListResolvers.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPrefixListResolverNotFound, id) } @@ -906,7 +892,7 @@ func (b *InMemoryBackend) GetIpamPrefixListResolverRules(resolverID string) ([]I b.mu.RLock("GetIpamPrefixListResolverRules") defer b.mu.RUnlock() - resolver, ok := b.ipamPrefixListResolvers[resolverID] + resolver, ok := b.ipamPrefixListResolvers.Get(resolverID) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPrefixListResolverNotFound, resolverID) } @@ -924,7 +910,7 @@ func (b *InMemoryBackend) GetIpamPrefixListResolverVersions(resolverID string) ( b.mu.RLock("GetIpamPrefixListResolverVersions") defer b.mu.RUnlock() - if _, ok := b.ipamPrefixListResolvers[resolverID]; !ok { + if _, ok := b.ipamPrefixListResolvers.Get(resolverID); !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPrefixListResolverNotFound, resolverID) } @@ -947,7 +933,7 @@ func (b *InMemoryBackend) GetIpamPrefixListResolverVersionEntries(resolverID str b.mu.RLock("GetIpamPrefixListResolverVersionEntries") defer b.mu.RUnlock() - if _, ok := b.ipamPrefixListResolvers[resolverID]; !ok { + if _, ok := b.ipamPrefixListResolvers.Get(resolverID); !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPrefixListResolverNotFound, resolverID) } @@ -974,7 +960,7 @@ func (b *InMemoryBackend) CreateIpamPrefixListResolverTarget( b.mu.Lock("CreateIpamPrefixListResolverTarget") defer b.mu.Unlock() - resolver, ok := b.ipamPrefixListResolvers[resolverID] + resolver, ok := b.ipamPrefixListResolvers.Get(resolverID) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPrefixListResolverNotFound, resolverID) } @@ -999,8 +985,7 @@ func (b *InMemoryBackend) CreateIpamPrefixListResolverTarget( synced := resolver.CurrentVersion target.LastSyncedVersion = &synced - - b.ipamPrefixListResolverTargets[id] = target + b.ipamPrefixListResolverTargets.Put(target) return copyIpamPrefixListResolverTarget(target), nil } @@ -1014,12 +999,11 @@ func (b *InMemoryBackend) DeleteIpamPrefixListResolverTarget(id string) (*IpamPr b.mu.Lock("DeleteIpamPrefixListResolverTarget") defer b.mu.Unlock() - target, ok := b.ipamPrefixListResolverTargets[id] + target, ok := b.ipamPrefixListResolverTargets.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPrefixListResolverTargetNotFound, id) } - - delete(b.ipamPrefixListResolverTargets, id) + b.ipamPrefixListResolverTargets.Delete(id) cp := copyIpamPrefixListResolverTarget(target) cp.State = ipamStateDeleteComplete @@ -1040,9 +1024,9 @@ func (b *InMemoryBackend) DescribeIpamPrefixListResolverTargets( idSet[id] = true } - out := make([]*IpamPrefixListResolverTarget, 0, len(b.ipamPrefixListResolverTargets)) + out := make([]*IpamPrefixListResolverTarget, 0, b.ipamPrefixListResolverTargets.Len()) - for _, t := range b.ipamPrefixListResolverTargets { + for _, t := range b.ipamPrefixListResolverTargets.All() { if resolverID != "" && t.IpamPrefixListResolverID != resolverID { continue } @@ -1073,7 +1057,7 @@ func (b *InMemoryBackend) ModifyIpamPrefixListResolverTarget( b.mu.Lock("ModifyIpamPrefixListResolverTarget") defer b.mu.Unlock() - target, ok := b.ipamPrefixListResolverTargets[id] + target, ok := b.ipamPrefixListResolverTargets.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPrefixListResolverTargetNotFound, id) } diff --git a/services/ec2/backend_ipam_policy.go b/services/ec2/backend_ipam_policy.go index d88743497..daa1dc319 100644 --- a/services/ec2/backend_ipam_policy.go +++ b/services/ec2/backend_ipam_policy.go @@ -76,7 +76,6 @@ type IpamPolicy struct { // resetIpamPolicyMapsLocked re-initialises all maps owned by this file. Must be called with // b.mu held. func (b *InMemoryBackend) resetIpamPolicyMapsLocked() { - b.ipamPolicies = make(map[string]*IpamPolicy) b.ipamPolicyEnabledTargets = make(map[string]string) b.ipamOrgAdminAccountID = "" } @@ -111,7 +110,7 @@ func (b *InMemoryBackend) CreateIpamPolicy(ipamID string) (*IpamPolicy, error) { b.mu.Lock("CreateIpamPolicy") defer b.mu.Unlock() - if _, ok := b.ipams[ipamID]; !ok { + if _, ok := b.ipams.Get(ipamID); !ok { return nil, fmt.Errorf("%w: %s", ErrIpamNotFound, ipamID) } @@ -125,7 +124,7 @@ func (b *InMemoryBackend) CreateIpamPolicy(ipamID string) (*IpamPolicy, error) { State: ipamStateCreateComplete, AllocationDocs: make(map[string]*IpamPolicyDocument), } - b.ipamPolicies[id] = policy + b.ipamPolicies.Put(policy) return copyIpamPolicy(policy), nil } @@ -139,12 +138,11 @@ func (b *InMemoryBackend) DeleteIpamPolicy(id string) (*IpamPolicy, error) { b.mu.Lock("DeleteIpamPolicy") defer b.mu.Unlock() - policy, ok := b.ipamPolicies[id] + policy, ok := b.ipamPolicies.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPolicyNotFound, id) } - - delete(b.ipamPolicies, id) + b.ipamPolicies.Delete(id) for target, enabledID := range b.ipamPolicyEnabledTargets { if enabledID == id { @@ -168,9 +166,9 @@ func (b *InMemoryBackend) DescribeIpamPolicies(ids []string) []*IpamPolicy { idSet[id] = true } - out := make([]*IpamPolicy, 0, len(b.ipamPolicies)) + out := make([]*IpamPolicy, 0, b.ipamPolicies.Len()) - for _, p := range b.ipamPolicies { + for _, p := range b.ipamPolicies.All() { if len(idSet) > 0 && !idSet[p.IpamPolicyID] { continue } @@ -197,7 +195,7 @@ func (b *InMemoryBackend) EnableIpamPolicy(id, orgTargetID string) error { b.mu.Lock("EnableIpamPolicy") defer b.mu.Unlock() - if _, ok := b.ipamPolicies[id]; !ok { + if _, ok := b.ipamPolicies.Get(id); !ok { return fmt.Errorf("%w: %s", ErrIpamPolicyNotFound, id) } @@ -222,7 +220,7 @@ func (b *InMemoryBackend) DisableIpamPolicy(id, orgTargetID string) error { b.mu.Lock("DisableIpamPolicy") defer b.mu.Unlock() - if _, ok := b.ipamPolicies[id]; !ok { + if _, ok := b.ipamPolicies.Get(id); !ok { return fmt.Errorf("%w: %s", ErrIpamPolicyNotFound, id) } @@ -262,7 +260,7 @@ func (b *InMemoryBackend) GetIpamPolicyOrganizationTargets(id string) ([]string, b.mu.RLock("GetIpamPolicyOrganizationTargets") defer b.mu.RUnlock() - if _, ok := b.ipamPolicies[id]; !ok { + if _, ok := b.ipamPolicies.Get(id); !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPolicyNotFound, id) } @@ -293,7 +291,7 @@ func (b *InMemoryBackend) GetIpamPolicyAllocationRules( b.mu.RLock("GetIpamPolicyAllocationRules") defer b.mu.RUnlock() - policy, ok := b.ipamPolicies[id] + policy, ok := b.ipamPolicies.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPolicyNotFound, id) } @@ -345,7 +343,7 @@ func (b *InMemoryBackend) ModifyIpamPolicyAllocationRules( b.mu.Lock("ModifyIpamPolicyAllocationRules") defer b.mu.Unlock() - policy, ok := b.ipamPolicies[id] + policy, ok := b.ipamPolicies.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrIpamPolicyNotFound, id) } @@ -424,12 +422,12 @@ func (b *InMemoryBackend) MoveByoipCidrToIpam(cidr, poolID, poolOwner string) (* b.mu.Lock("MoveByoipCidrToIpam") defer b.mu.Unlock() - entry, ok := b.byoipCidrs[cidr] + entry, ok := b.byoipCidrs.Get(cidr) if !ok { return nil, fmt.Errorf("%w: %s", ErrByoipCidrNotFound, cidr) } - if _, poolOK := b.ipamPools[poolID]; !poolOK { + if _, poolOK := b.ipamPools.Get(poolID); !poolOK { return nil, fmt.Errorf("%w: %s", ErrIpamPoolNotFound, poolID) } diff --git a/services/ec2/backend_local_gateway.go b/services/ec2/backend_local_gateway.go index 91692686f..0982ba826 100644 --- a/services/ec2/backend_local_gateway.go +++ b/services/ec2/backend_local_gateway.go @@ -163,7 +163,7 @@ func (b *InMemoryBackend) SeedLocalGateway(lg LocalGateway) (*LocalGateway, erro } cp := stored - b.localGateways[stored.LocalGatewayID] = &cp + b.localGateways.Put(&cp) out := cp @@ -193,7 +193,7 @@ func (b *InMemoryBackend) SeedLocalGatewayVirtualInterface( } cp := stored - b.localGatewayVirtualInterfaces[stored.LocalGatewayVirtualInterfaceID] = &cp + b.localGatewayVirtualInterfaces.Put(&cp) out := cp @@ -223,7 +223,7 @@ func (b *InMemoryBackend) SeedLocalGatewayVirtualInterfaceGroup( } cp := stored - b.localGatewayVirtualInterfaceGroups[stored.LocalGatewayVirtualInterfaceGroupID] = &cp + b.localGatewayVirtualInterfaceGroups.Put(&cp) out := cp @@ -251,7 +251,7 @@ func (b *InMemoryBackend) CreateLocalGatewayVirtualInterfaceGroup( b.mu.Lock("CreateLocalGatewayVirtualInterfaceGroup") defer b.mu.Unlock() - if _, ok := b.localGateways[localGatewayID]; !ok { + if _, ok := b.localGateways.Get(localGatewayID); !ok { return nil, fmt.Errorf("%w: %s", ErrInvalidParameter, localGatewayID) } @@ -265,7 +265,7 @@ func (b *InMemoryBackend) CreateLocalGatewayVirtualInterfaceGroup( Tags: maps.Clone(tags), } group.LocalGatewayVirtualInterfaceGroupArn = localGatewayVifGroupArn(b, group.LocalGatewayVirtualInterfaceGroupID) - b.localGatewayVirtualInterfaceGroups[group.LocalGatewayVirtualInterfaceGroupID] = group + b.localGatewayVirtualInterfaceGroups.Put(group) cp := *group cp.Tags = maps.Clone(group.Tags) @@ -286,12 +286,12 @@ func (b *InMemoryBackend) DeleteLocalGatewayVirtualInterfaceGroup( b.mu.Lock("DeleteLocalGatewayVirtualInterfaceGroup") defer b.mu.Unlock() - group, ok := b.localGatewayVirtualInterfaceGroups[id] + group, ok := b.localGatewayVirtualInterfaceGroups.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrLocalGatewayVifGroupNotFound, id) } - for _, vif := range b.localGatewayVirtualInterfaces { + for _, vif := range b.localGatewayVirtualInterfaces.All() { if vif.LocalGatewayVirtualInterfaceGroupID == id { return nil, fmt.Errorf( "%w: %s has associated local gateway virtual interfaces", ErrDependencyViolation, id, @@ -302,7 +302,7 @@ func (b *InMemoryBackend) DeleteLocalGatewayVirtualInterfaceGroup( deleted := *group deleted.Tags = maps.Clone(group.Tags) deleted.ConfigurationState = localGatewayRouteStateDeleted - delete(b.localGatewayVirtualInterfaceGroups, id) + b.localGatewayVirtualInterfaceGroups.Delete(id) return &deleted, nil } @@ -350,7 +350,7 @@ func (b *InMemoryBackend) CreateLocalGatewayVirtualInterface( b.mu.Lock("CreateLocalGatewayVirtualInterface") defer b.mu.Unlock() - group, ok := b.localGatewayVirtualInterfaceGroups[p.LocalGatewayVirtualInterfaceGroupID] + group, ok := b.localGatewayVirtualInterfaceGroups.Get(p.LocalGatewayVirtualInterfaceGroupID) if !ok { return nil, fmt.Errorf("%w: %s", ErrLocalGatewayVifGroupNotFound, p.LocalGatewayVirtualInterfaceGroupID) } @@ -371,7 +371,7 @@ func (b *InMemoryBackend) CreateLocalGatewayVirtualInterface( Tags: maps.Clone(p.Tags), } vif.LocalGatewayVirtualInterfaceArn = localGatewayVifArn(b, vif.LocalGatewayVirtualInterfaceID) - b.localGatewayVirtualInterfaces[vif.LocalGatewayVirtualInterfaceID] = vif + b.localGatewayVirtualInterfaces.Put(vif) group.LocalGatewayVirtualInterfaceIDs = append( group.LocalGatewayVirtualInterfaceIDs, vif.LocalGatewayVirtualInterfaceID, @@ -395,12 +395,12 @@ func (b *InMemoryBackend) DeleteLocalGatewayVirtualInterface( b.mu.Lock("DeleteLocalGatewayVirtualInterface") defer b.mu.Unlock() - vif, ok := b.localGatewayVirtualInterfaces[id] + vif, ok := b.localGatewayVirtualInterfaces.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrLocalGatewayVifNotFound, id) } - if group, groupOK := b.localGatewayVirtualInterfaceGroups[vif.LocalGatewayVirtualInterfaceGroupID]; groupOK { + if group, groupOK := b.localGatewayVirtualInterfaceGroups.Get(vif.LocalGatewayVirtualInterfaceGroupID); groupOK { group.LocalGatewayVirtualInterfaceIDs = removeString( group.LocalGatewayVirtualInterfaceIDs, id, ) @@ -409,7 +409,7 @@ func (b *InMemoryBackend) DeleteLocalGatewayVirtualInterface( deleted := *vif deleted.Tags = maps.Clone(vif.Tags) deleted.ConfigurationState = localGatewayRouteStateDeleted - delete(b.localGatewayVirtualInterfaces, id) + b.localGatewayVirtualInterfaces.Delete(id) return &deleted, nil } @@ -426,9 +426,9 @@ func (b *InMemoryBackend) DescribeLocalGateways(ids []string) []*LocalGateway { idSet[id] = true } - out := make([]*LocalGateway, 0, len(b.localGateways)) + out := make([]*LocalGateway, 0, b.localGateways.Len()) - for _, lg := range b.localGateways { + for _, lg := range b.localGateways.All() { if len(idSet) > 0 && !idSet[lg.LocalGatewayID] { continue } @@ -455,9 +455,9 @@ func (b *InMemoryBackend) DescribeLocalGatewayVirtualInterfaces( idSet[id] = true } - out := make([]*LocalGatewayVirtualInterface, 0, len(b.localGatewayVirtualInterfaces)) + out := make([]*LocalGatewayVirtualInterface, 0, b.localGatewayVirtualInterfaces.Len()) - for _, vif := range b.localGatewayVirtualInterfaces { + for _, vif := range b.localGatewayVirtualInterfaces.All() { if len(idSet) > 0 && !idSet[vif.LocalGatewayVirtualInterfaceID] { continue } @@ -486,9 +486,9 @@ func (b *InMemoryBackend) DescribeLocalGatewayVirtualInterfaceGroups( idSet[id] = true } - out := make([]*LocalGatewayVirtualInterfaceGroup, 0, len(b.localGatewayVirtualInterfaceGroups)) + out := make([]*LocalGatewayVirtualInterfaceGroup, 0, b.localGatewayVirtualInterfaceGroups.Len()) - for _, group := range b.localGatewayVirtualInterfaceGroups { + for _, group := range b.localGatewayVirtualInterfaceGroups.All() { if len(idSet) > 0 && !idSet[group.LocalGatewayVirtualInterfaceGroupID] { continue } @@ -521,7 +521,7 @@ func (b *InMemoryBackend) CreateLocalGatewayRouteTable( b.mu.Lock("CreateLocalGatewayRouteTable") defer b.mu.Unlock() - lg, ok := b.localGateways[localGatewayID] + lg, ok := b.localGateways.Get(localGatewayID) if !ok { return nil, fmt.Errorf("%w: %s", ErrInvalidParameter, localGatewayID) } @@ -535,7 +535,7 @@ func (b *InMemoryBackend) CreateLocalGatewayRouteTable( OwnerID: b.AccountID, } rt.LocalGatewayRouteTableArn = localGatewayRouteTableArn(b, rt.LocalGatewayRouteTableID) - b.localGatewayRouteTables[rt.LocalGatewayRouteTableID] = rt + b.localGatewayRouteTables.Put(rt) cp := *rt @@ -553,9 +553,9 @@ func (b *InMemoryBackend) DescribeLocalGatewayRouteTables(ids []string) []*Local idSet[id] = true } - out := make([]*LocalGatewayRouteTable, 0, len(b.localGatewayRouteTables)) + out := make([]*LocalGatewayRouteTable, 0, b.localGatewayRouteTables.Len()) - for _, rt := range b.localGatewayRouteTables { + for _, rt := range b.localGatewayRouteTables.All() { if len(idSet) > 0 && !idSet[rt.LocalGatewayRouteTableID] { continue } @@ -580,11 +580,10 @@ func (b *InMemoryBackend) DeleteLocalGatewayRouteTable(id string) error { b.mu.Lock("DeleteLocalGatewayRouteTable") defer b.mu.Unlock() - if _, ok := b.localGatewayRouteTables[id]; !ok { + if _, ok := b.localGatewayRouteTables.Get(id); !ok { return fmt.Errorf("%w: %s", ErrLocalGatewayRouteTableNotFound, id) } - - delete(b.localGatewayRouteTables, id) + b.localGatewayRouteTables.Delete(id) return nil } @@ -602,7 +601,7 @@ func localGatewayRouteKey(routeTableID, destinationCIDR, destinationPrefixListID // requireLocalGatewayRouteTable looks up a local gateway route table by ID, returning // ErrLocalGatewayRouteTableNotFound if it does not exist. Must be called with b.mu held. func (b *InMemoryBackend) requireLocalGatewayRouteTable(routeTableID string) (*LocalGatewayRouteTable, error) { - rt, ok := b.localGatewayRouteTables[routeTableID] + rt, ok := b.localGatewayRouteTables.Get(routeTableID) if !ok { return nil, fmt.Errorf("%w: %s", ErrLocalGatewayRouteTableNotFound, routeTableID) } @@ -634,7 +633,7 @@ func (b *InMemoryBackend) CreateLocalGatewayRoute( } key := localGatewayRouteKey(routeTableID, destinationCIDR, destinationPrefixListID) - if _, exists := b.localGatewayRoutes[key]; exists { + if _, exists := b.localGatewayRoutes.Get(key); exists { return nil, fmt.Errorf("%w: route already exists in %s", ErrInvalidParameter, routeTableID) } @@ -649,7 +648,7 @@ func (b *InMemoryBackend) CreateLocalGatewayRoute( State: localGatewayRouteStateActive, OwnerID: b.AccountID, } - b.localGatewayRoutes[key] = route + b.localGatewayRoutes.Put(route) cp := *route @@ -669,7 +668,7 @@ func (b *InMemoryBackend) DeleteLocalGatewayRoute( key := localGatewayRouteKey(routeTableID, destinationCIDR, destinationPrefixListID) - route, ok := b.localGatewayRoutes[key] + route, ok := b.localGatewayRoutes.Get(key) if !ok { return nil, fmt.Errorf( "%w: route %s in %s not found", @@ -681,7 +680,7 @@ func (b *InMemoryBackend) DeleteLocalGatewayRoute( deleted := *route deleted.State = localGatewayRouteStateDeleted - delete(b.localGatewayRoutes, key) + b.localGatewayRoutes.Delete(key) return &deleted, nil } @@ -699,7 +698,7 @@ func (b *InMemoryBackend) ModifyLocalGatewayRoute( key := localGatewayRouteKey(routeTableID, destinationCIDR, destinationPrefixListID) - existing, ok := b.localGatewayRoutes[key] + existing, ok := b.localGatewayRoutes.Get(key) if !ok { return nil, fmt.Errorf( "%w: route %s in %s not found", @@ -719,8 +718,7 @@ func (b *InMemoryBackend) ModifyLocalGatewayRoute( updated.NetworkInterfaceID = eniID updated.LocalGatewayVirtualInterfaceGroupID = "" } - - b.localGatewayRoutes[key] = &updated + b.localGatewayRoutes.Put(&updated) cp := updated @@ -751,7 +749,7 @@ func (b *InMemoryBackend) SearchLocalGatewayRoutes( out := make([]*LocalGatewayRoute, 0) - for _, r := range b.localGatewayRoutes { + for _, r := range b.localGatewayRoutes.All() { if r.LocalGatewayRouteTableID != routeTableID { continue } @@ -810,7 +808,7 @@ func (b *InMemoryBackend) CreateLocalGatewayRouteTableVpcAssociation( "CreateLocalGatewayRouteTableVpcAssociation", routeTableID, func(rt *LocalGatewayRouteTable) (*LocalGatewayRouteTableVpcAssociation, error) { - if _, vpcExists := b.vpcs[vpcID]; !vpcExists { + if _, vpcExists := b.vpcs.Get(vpcID); !vpcExists { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -823,7 +821,7 @@ func (b *InMemoryBackend) CreateLocalGatewayRouteTableVpcAssociation( State: localGatewayAssocStateAssoc, OwnerID: b.AccountID, } - b.localGatewayRouteTableVpcAssociations[assoc.LocalGatewayRouteTableVpcAssociationID] = assoc + b.localGatewayRouteTableVpcAssociations.Put(assoc) cp := *assoc @@ -847,14 +845,14 @@ func (b *InMemoryBackend) DeleteLocalGatewayRouteTableVpcAssociation( b.mu.Lock("DeleteLocalGatewayRouteTableVpcAssociation") defer b.mu.Unlock() - assoc, ok := b.localGatewayRouteTableVpcAssociations[id] + assoc, ok := b.localGatewayRouteTableVpcAssociations.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrLocalGatewayVpcAssociationNotFound, id) } deleted := *assoc deleted.State = localGatewayAssocStateDisassoc - delete(b.localGatewayRouteTableVpcAssociations, id) + b.localGatewayRouteTableVpcAssociations.Delete(id) return &deleted, nil } @@ -872,9 +870,9 @@ func (b *InMemoryBackend) DescribeLocalGatewayRouteTableVpcAssociations( idSet[id] = true } - out := make([]*LocalGatewayRouteTableVpcAssociation, 0, len(b.localGatewayRouteTableVpcAssociations)) + out := make([]*LocalGatewayRouteTableVpcAssociation, 0, b.localGatewayRouteTableVpcAssociations.Len()) - for _, a := range b.localGatewayRouteTableVpcAssociations { + for _, a := range b.localGatewayRouteTableVpcAssociations.All() { if len(idSet) > 0 && !idSet[a.LocalGatewayRouteTableVpcAssociationID] { continue } @@ -909,7 +907,7 @@ func (b *InMemoryBackend) CreateLocalGatewayRouteTableVirtualInterfaceGroupAssoc "CreateLocalGatewayRouteTableVirtualInterfaceGroupAssociation", routeTableID, func(rt *LocalGatewayRouteTable) (*LocalGatewayRouteTableVirtualInterfaceGroupAssociation, error) { - if _, groupExists := b.localGatewayVirtualInterfaceGroups[vifGroupID]; !groupExists { + if _, groupExists := b.localGatewayVirtualInterfaceGroups.Get(vifGroupID); !groupExists { return nil, fmt.Errorf("%w: %s", ErrInvalidParameter, vifGroupID) } @@ -923,7 +921,7 @@ func (b *InMemoryBackend) CreateLocalGatewayRouteTableVirtualInterfaceGroupAssoc State: localGatewayAssocStateAssoc, OwnerID: b.AccountID, } - b.localGatewayRouteTableVifGroupAssociations[assoc.LocalGatewayRouteTableVirtualInterfaceGroupAssociationID] = assoc + b.localGatewayRouteTableVifGroupAssociations.Put(assoc) cp := *assoc @@ -947,14 +945,14 @@ func (b *InMemoryBackend) DeleteLocalGatewayRouteTableVirtualInterfaceGroupAssoc b.mu.Lock("DeleteLocalGatewayRouteTableVirtualInterfaceGroupAssociation") defer b.mu.Unlock() - assoc, ok := b.localGatewayRouteTableVifGroupAssociations[id] + assoc, ok := b.localGatewayRouteTableVifGroupAssociations.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrLocalGatewayVifGroupAssociationNotFound, id) } deleted := *assoc deleted.State = localGatewayAssocStateDisassoc - delete(b.localGatewayRouteTableVifGroupAssociations, id) + b.localGatewayRouteTableVifGroupAssociations.Delete(id) return &deleted, nil } @@ -974,11 +972,10 @@ func (b *InMemoryBackend) DescribeLocalGatewayRouteTableVirtualInterfaceGroupAss out := make( []*LocalGatewayRouteTableVirtualInterfaceGroupAssociation, - 0, - len(b.localGatewayRouteTableVifGroupAssociations), + 0, b.localGatewayRouteTableVifGroupAssociations.Len(), ) - for _, a := range b.localGatewayRouteTableVifGroupAssociations { + for _, a := range b.localGatewayRouteTableVifGroupAssociations.All() { if len(idSet) > 0 && !idSet[a.LocalGatewayRouteTableVirtualInterfaceGroupAssociationID] { continue } diff --git a/services/ec2/backend_mac_hosts.go b/services/ec2/backend_mac_hosts.go index aac8733ce..304b6824f 100644 --- a/services/ec2/backend_mac_hosts.go +++ b/services/ec2/backend_mac_hosts.go @@ -75,7 +75,6 @@ type MacModificationTask struct { // called with b.mu held. (Mac Hosts themselves are derived on read from // b.dedicatedHosts, so there is no separate map to reset for them.) func (b *InMemoryBackend) resetMacHostMapsLocked() { - b.macModificationTasks = make(map[string]*MacModificationTask) } // isMacInstanceType reports whether instanceType is an EC2 Mac Dedicated Host @@ -114,9 +113,9 @@ func (b *InMemoryBackend) DescribeMacHosts(ids []string) []*MacHost { idSet[id] = true } - out := make([]*MacHost, 0, len(b.dedicatedHosts)) + out := make([]*MacHost, 0, b.dedicatedHosts.Len()) - for _, h := range b.dedicatedHosts { + for _, h := range b.dedicatedHosts.All() { if !isMacInstanceType(h.InstanceType) { continue } @@ -139,7 +138,7 @@ func (b *InMemoryBackend) DescribeMacHosts(ids []string) []*MacHost { // requireMacInstanceLocked validates that instanceID exists and refers to a // Mac (mac1/mac2/mac-m*) instance. Must be called with b.mu held. func (b *InMemoryBackend) requireMacInstanceLocked(instanceID string) error { - inst, ok := b.instances[instanceID] + inst, ok := b.instances.Get(instanceID) if !ok { return fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -186,7 +185,7 @@ func (b *InMemoryBackend) CreateMacSystemIntegrityProtectionModificationTask( SIPConfig: config, Tags: tags, } - b.macModificationTasks[task.MacModificationTaskID] = task + b.macModificationTasks.Put(task) cp := *task @@ -221,7 +220,7 @@ func (b *InMemoryBackend) CreateDelegateMacVolumeOwnershipTask( StartTime: time.Now().UTC(), Tags: tags, } - b.macModificationTasks[task.MacModificationTaskID] = task + b.macModificationTasks.Put(task) cp := *task @@ -240,9 +239,9 @@ func (b *InMemoryBackend) DescribeMacModificationTasks(ids []string) []*MacModif idSet[id] = true } - out := make([]*MacModificationTask, 0, len(b.macModificationTasks)) + out := make([]*MacModificationTask, 0, b.macModificationTasks.Len()) - for _, t := range b.macModificationTasks { + for _, t := range b.macModificationTasks.All() { if len(idSet) > 0 && !idSet[t.MacModificationTaskID] { continue } diff --git a/services/ec2/backend_network_performance.go b/services/ec2/backend_network_performance.go index 1f883141e..b1080a23d 100644 --- a/services/ec2/backend_network_performance.go +++ b/services/ec2/backend_network_performance.go @@ -96,14 +96,13 @@ func (b *InMemoryBackend) EnableAwsNetworkPerformanceMetricSubscription( b.mu.Lock("EnableAwsNetworkPerformanceMetricSubscription") defer b.mu.Unlock() - key := networkPerformanceSubscriptionKey(source, destination, metric, statistic) - b.networkPerformanceSubscriptions[key] = &NetworkPerformanceSubscription{ + b.networkPerformanceSubscriptions.Put(&NetworkPerformanceSubscription{ Source: source, Destination: destination, Metric: metric, Statistic: statistic, Period: networkPerformanceDefaultPeriod, - } + }) return true, nil } @@ -128,8 +127,7 @@ func (b *InMemoryBackend) DisableAwsNetworkPerformanceMetricSubscription( b.mu.Lock("DisableAwsNetworkPerformanceMetricSubscription") defer b.mu.Unlock() - - delete(b.networkPerformanceSubscriptions, networkPerformanceSubscriptionKey(source, destination, metric, statistic)) + b.networkPerformanceSubscriptions.Delete(networkPerformanceSubscriptionKey(source, destination, metric, statistic)) return true, nil } @@ -140,8 +138,8 @@ func (b *InMemoryBackend) DescribeAwsNetworkPerformanceMetricSubscriptions() []* b.mu.RLock("DescribeAwsNetworkPerformanceMetricSubscriptions") defer b.mu.RUnlock() - out := make([]*NetworkPerformanceSubscription, 0, len(b.networkPerformanceSubscriptions)) - for _, s := range b.networkPerformanceSubscriptions { + out := make([]*NetworkPerformanceSubscription, 0, b.networkPerformanceSubscriptions.Len()) + for _, s := range b.networkPerformanceSubscriptions.All() { cp := *s out = append(out, &cp) } diff --git a/services/ec2/backend_networking1.go b/services/ec2/backend_networking1.go index e391c098f..7425d6f02 100644 --- a/services/ec2/backend_networking1.go +++ b/services/ec2/backend_networking1.go @@ -80,7 +80,7 @@ func (b *InMemoryBackend) CreateTransitGatewayVpcAttachment( b.mu.Lock("CreateTransitGatewayVpcAttachment") defer b.mu.Unlock() - if _, ok := b.transitGateways[tgwID]; !ok { + if _, ok := b.transitGateways.Get(tgwID); !ok { return nil, fmt.Errorf("%w: %s", ErrTransitGatewayNotFound, tgwID) } @@ -91,7 +91,7 @@ func (b *InMemoryBackend) CreateTransitGatewayVpcAttachment( State: stateAvailable, CreationTime: time.Now().UTC(), } - b.tgwVpcAttachments[att.TransitGatewayAttachmentID] = att + b.tgwVpcAttachments.Put(att) cp := *att @@ -110,9 +110,9 @@ func (b *InMemoryBackend) DescribeTransitGatewayVpcAttachments( idSet[id] = true } - out := make([]*TransitGatewayVpcAttachment, 0, len(b.tgwVpcAttachments)) + out := make([]*TransitGatewayVpcAttachment, 0, b.tgwVpcAttachments.Len()) - for _, att := range b.tgwVpcAttachments { + for _, att := range b.tgwVpcAttachments.All() { if len(idSet) > 0 && !idSet[att.TransitGatewayAttachmentID] { continue } @@ -137,11 +137,10 @@ func (b *InMemoryBackend) DeleteTransitGatewayVpcAttachment(id string) error { b.mu.Lock("DeleteTransitGatewayVpcAttachment") defer b.mu.Unlock() - if _, ok := b.tgwVpcAttachments[id]; !ok { + if _, ok := b.tgwVpcAttachments.Get(id); !ok { return fmt.Errorf("%w: %s", ErrTGWAttachmentNotFound, id) } - - delete(b.tgwVpcAttachments, id) + b.tgwVpcAttachments.Delete(id) return nil } @@ -155,7 +154,7 @@ func (b *InMemoryBackend) ModifyTransitGatewayAttribute(id, description string) b.mu.Lock("ModifyTransitGatewayAttribute") defer b.mu.Unlock() - tgw, ok := b.transitGateways[id] + tgw, ok := b.transitGateways.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrTransitGatewayNotFound, id) } @@ -201,7 +200,7 @@ func (b *InMemoryBackend) CreateFlowLogs( FlowLogStatus: "ACTIVE", CreationTime: time.Now().UTC(), } - b.flowLogs[fl.FlowLogID] = fl + b.flowLogs.Put(fl) cp := *fl out = append(out, &cp) @@ -220,9 +219,9 @@ func (b *InMemoryBackend) DescribeFlowLogs(ids []string) []*FlowLog { idSet[id] = true } - out := make([]*FlowLog, 0, len(b.flowLogs)) + out := make([]*FlowLog, 0, b.flowLogs.Len()) - for _, fl := range b.flowLogs { + for _, fl := range b.flowLogs.All() { if len(idSet) > 0 && !idSet[fl.FlowLogID] { continue } @@ -248,13 +247,13 @@ func (b *InMemoryBackend) DeleteFlowLogs(ids []string) error { defer b.mu.Unlock() for _, id := range ids { - if _, ok := b.flowLogs[id]; !ok { + if _, ok := b.flowLogs.Get(id); !ok { return fmt.Errorf("%w: %s", ErrFlowLogNotFound, id) } } for _, id := range ids { - delete(b.flowLogs, id) + b.flowLogs.Delete(id) } return nil @@ -272,7 +271,7 @@ func (b *InMemoryBackend) CreateDhcpOptions(configs []DhcpConfiguration) (*DhcpO Configurations: configs, AssociatedVPCIDs: []string{}, } - b.dhcpOptionSets[opts.DhcpOptionsID] = opts + b.dhcpOptionSets.Put(opts) cp := *opts @@ -289,9 +288,9 @@ func (b *InMemoryBackend) DescribeDhcpOptions(ids []string) []*DhcpOptions { idSet[id] = true } - out := make([]*DhcpOptions, 0, len(b.dhcpOptionSets)) + out := make([]*DhcpOptions, 0, b.dhcpOptionSets.Len()) - for _, opts := range b.dhcpOptionSets { + for _, opts := range b.dhcpOptionSets.All() { if len(idSet) > 0 && !idSet[opts.DhcpOptionsID] { continue } @@ -316,7 +315,7 @@ func (b *InMemoryBackend) AssociateDhcpOptions(dhcpOptionsID, vpcID string) erro b.mu.Lock("AssociateDhcpOptions") defer b.mu.Unlock() - if _, ok := b.vpcs[vpcID]; !ok { + if _, ok := b.vpcs.Get(vpcID); !ok { return fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -325,7 +324,7 @@ func (b *InMemoryBackend) AssociateDhcpOptions(dhcpOptionsID, vpcID string) erro return nil } - opts, ok := b.dhcpOptionSets[dhcpOptionsID] + opts, ok := b.dhcpOptionSets.Get(dhcpOptionsID) if !ok { return fmt.Errorf("%w: %s", ErrDhcpOptionsNotFound, dhcpOptionsID) } @@ -346,11 +345,10 @@ func (b *InMemoryBackend) DeleteDhcpOptions(id string) error { b.mu.Lock("DeleteDhcpOptions") defer b.mu.Unlock() - if _, ok := b.dhcpOptionSets[id]; !ok { + if _, ok := b.dhcpOptionSets.Get(id); !ok { return fmt.Errorf("%w: %s", ErrDhcpOptionsNotFound, id) } - - delete(b.dhcpOptionSets, id) + b.dhcpOptionSets.Delete(id) return nil } @@ -369,7 +367,7 @@ func (b *InMemoryBackend) ModifyLaunchTemplate( b.mu.Lock("ModifyLaunchTemplate") defer b.mu.Unlock() - lt, ok := b.launchTemplates[id] + lt, ok := b.launchTemplates.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrLaunchTemplateNotFound, id) } @@ -394,7 +392,7 @@ func (b *InMemoryBackend) CreateLaunchTemplateVersion( b.mu.Lock("CreateLaunchTemplateVersion") defer b.mu.Unlock() - lt, ok := b.launchTemplates[id] + lt, ok := b.launchTemplates.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrLaunchTemplateNotFound, id) } @@ -435,7 +433,7 @@ func (b *InMemoryBackend) DeleteLaunchTemplateVersions( b.mu.Lock("DeleteLaunchTemplateVersions") defer b.mu.Unlock() - if _, ok := b.launchTemplates[id]; !ok { + if _, ok := b.launchTemplates.Get(id); !ok { return nil, fmt.Errorf("%w: %s", ErrLaunchTemplateNotFound, id) } @@ -454,7 +452,7 @@ func (b *InMemoryBackend) GetLaunchTemplateData(instanceID string) (*LaunchTempl b.mu.RLock("GetLaunchTemplateData") defer b.mu.RUnlock() - inst, ok := b.instances[instanceID] + inst, ok := b.instances.Get(instanceID) if !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } diff --git a/services/ec2/backend_parity_final.go b/services/ec2/backend_parity_final.go index f3e435ae3..cdedeaeb2 100644 --- a/services/ec2/backend_parity_final.go +++ b/services/ec2/backend_parity_final.go @@ -154,8 +154,6 @@ type ElasticGpuStub struct { // (split out to keep newInMemoryBackendMaps under the funlen limit). func initParityFinalMaps(b *InMemoryBackend) { b.tgwRTPropagations = make(map[string]map[string]*TransitGatewayRouteTablePropagation) - b.interruptibleCRAllocations = make(map[string]*InterruptibleCapacityReservationAllocation) - b.movingAddresses = make(map[string]*MovingAddressStatus) b.reachabilityAnalyzerOrgSharing = false } @@ -174,13 +172,13 @@ func (b *InMemoryBackend) ModifyVerifiedAccessGroup( b.mu.Lock("ModifyVerifiedAccessGroup") defer b.mu.Unlock() - grp, ok := b.verifiedAccessGroups[id] + grp, ok := b.verifiedAccessGroups.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrVerifiedAccessGroupNotFound, id) } if instanceID != "" { - if _, exists := b.verifiedAccessInstances[instanceID]; !exists { + if _, exists := b.verifiedAccessInstances.Get(instanceID); !exists { return nil, fmt.Errorf("%w: %s", ErrVerifiedAccessInstanceNotFound, instanceID) } @@ -207,7 +205,7 @@ func (b *InMemoryBackend) ModifyVerifiedAccessInstance(id, description string) ( b.mu.Lock("ModifyVerifiedAccessInstance") defer b.mu.Unlock() - inst, ok := b.verifiedAccessInstances[id] + inst, ok := b.verifiedAccessInstances.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrVerifiedAccessInstanceNotFound, id) } @@ -234,7 +232,7 @@ func (b *InMemoryBackend) ModifyVerifiedAccessTrustProvider( b.mu.Lock("ModifyVerifiedAccessTrustProvider") defer b.mu.Unlock() - tp, ok := b.verifiedAccessTrustProviders[id] + tp, ok := b.verifiedAccessTrustProviders.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrVerifiedAccessTrustProviderNF, id) } @@ -254,15 +252,15 @@ func (b *InMemoryBackend) ModifyVerifiedAccessTrustProvider( // attachment, scanning the existing per-type attachment maps. Must be called // with b.mu held. func (b *InMemoryBackend) tgwAttachmentResourceLocked(attachmentID string) (string, string) { - if att, ok := b.tgwVpcAttachments[attachmentID]; ok { + if att, ok := b.tgwVpcAttachments.Get(attachmentID); ok { return att.VpcID, tgwResourceTypeVPC } - if att, ok := b.tgwPeeringAttachments[attachmentID]; ok { + if att, ok := b.tgwPeeringAttachments.Get(attachmentID); ok { return att.AccepterTransitGatewayID, "peering" } - if att, ok := b.tgwConnects[attachmentID]; ok { + if att, ok := b.tgwConnects.Get(attachmentID); ok { return att.TransportTransitGatewayAttachmentID, "connect" } @@ -285,7 +283,7 @@ func (b *InMemoryBackend) EnableTransitGatewayRouteTablePropagation( b.mu.Lock("EnableTransitGatewayRouteTablePropagation") defer b.mu.Unlock() - if _, ok := b.tgwRouteTables[routeTableID]; !ok { + if _, ok := b.tgwRouteTables.Get(routeTableID); !ok { return nil, fmt.Errorf("%w: %s", ErrTGWRouteTableNotFound, routeTableID) } @@ -359,7 +357,7 @@ func (b *InMemoryBackend) DescribeTransitGatewayAttachments(ids []string) []*Tra out := make([]*TransitGatewayAttachmentSummary, 0) - for _, att := range b.tgwVpcAttachments { + for _, att := range b.tgwVpcAttachments.All() { if len(filter) > 0 && !filter[att.TransitGatewayAttachmentID] { continue } @@ -373,7 +371,7 @@ func (b *InMemoryBackend) DescribeTransitGatewayAttachments(ids []string) []*Tra }) } - for _, att := range b.tgwPeeringAttachments { + for _, att := range b.tgwPeeringAttachments.All() { if len(filter) > 0 && !filter[att.TransitGatewayAttachmentID] { continue } @@ -387,7 +385,7 @@ func (b *InMemoryBackend) DescribeTransitGatewayAttachments(ids []string) []*Tra }) } - for _, att := range b.tgwConnects { + for _, att := range b.tgwConnects.All() { if len(filter) > 0 && !filter[att.TransitGatewayAttachmentID] { continue } @@ -427,7 +425,7 @@ func (b *InMemoryBackend) CreateInterruptibleCapacityReservationAllocation( b.mu.Lock("CreateInterruptibleCapacityReservationAllocation") defer b.mu.Unlock() - cr, ok := b.capacityReservations[sourceCapacityReservationID] + cr, ok := b.capacityReservations.Get(sourceCapacityReservationID) if !ok { return nil, fmt.Errorf("%w: %s", ErrCapacityReservationNotFound, sourceCapacityReservationID) } @@ -446,7 +444,7 @@ func (b *InMemoryBackend) CreateInterruptibleCapacityReservationAllocation( Status: interruptibleAllocStatusActive, TargetInstanceCount: instanceCount, } - b.interruptibleCRAllocations[sourceCapacityReservationID] = alloc + b.interruptibleCRAllocations.Put(alloc) cp := *alloc @@ -470,12 +468,12 @@ func (b *InMemoryBackend) UpdateInterruptibleCapacityReservationAllocation( b.mu.Lock("UpdateInterruptibleCapacityReservationAllocation") defer b.mu.Unlock() - cr, ok := b.capacityReservations[sourceCapacityReservationID] + cr, ok := b.capacityReservations.Get(sourceCapacityReservationID) if !ok { return nil, fmt.Errorf("%w: %s", ErrCapacityReservationNotFound, sourceCapacityReservationID) } - alloc, ok := b.interruptibleCRAllocations[sourceCapacityReservationID] + alloc, ok := b.interruptibleCRAllocations.Get(sourceCapacityReservationID) if !ok { return nil, fmt.Errorf("%w: %s", ErrInterruptibleAllocationNotFound, sourceCapacityReservationID) } @@ -508,14 +506,14 @@ func (b *InMemoryBackend) GetCapacityReservationUsage(id string) (*CapacityReser b.mu.RLock("GetCapacityReservationUsage") defer b.mu.RUnlock() - cr, ok := b.capacityReservations[id] + cr, ok := b.capacityReservations.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrCapacityReservationNotFound, id) } var usedCount int32 - for _, inst := range b.instances { + for _, inst := range b.instances.All() { if inst.CapacityReservationSpec.CapacityReservationID == id { usedCount++ } @@ -535,7 +533,7 @@ func (b *InMemoryBackend) GetCapacityReservationUsage(id string) (*CapacityReser InstanceUsages: usages, } - if alloc, hasAlloc := b.interruptibleCRAllocations[id]; hasAlloc { + if alloc, hasAlloc := b.interruptibleCRAllocations.Get(id); hasAlloc { usage.Interruptible = true allocCopy := *alloc usage.InterruptibleAllocation = &allocCopy @@ -555,9 +553,9 @@ func (b *InMemoryBackend) DescribeCapacityReservationTopology(ids []string) []*C filter[id] = true } - out := make([]*CapacityReservationTopologyEntry, 0, len(b.capacityReservations)) + out := make([]*CapacityReservationTopologyEntry, 0, b.capacityReservations.Len()) - for _, cr := range b.capacityReservations { + for _, cr := range b.capacityReservations.All() { if len(filter) > 0 && !filter[cr.CapacityReservationID] { continue } @@ -581,7 +579,7 @@ func (b *InMemoryBackend) DescribeCapacityReservationTopology(ids []string) []*C // findAddressByPublicIPLocked scans the addresses map for an EIP by public IP // address. Must be called with b.mu held. func (b *InMemoryBackend) findAddressByPublicIPLocked(publicIP string) *Address { - for _, addr := range b.addresses { + for _, addr := range b.addresses.All() { if addr.PublicIP == publicIP { return addr } @@ -605,11 +603,10 @@ func (b *InMemoryBackend) MoveAddressToVpc(publicIP string) (*Address, error) { if addr == nil { return nil, fmt.Errorf("%w: %s", ErrPublicIPNotFound, publicIP) } - - b.movingAddresses[publicIP] = &MovingAddressStatus{ + b.movingAddresses.Put(&MovingAddressStatus{ PublicIP: publicIP, MoveStatus: moveStatusMovingToVpc, - } + }) cp := *addr @@ -627,9 +624,9 @@ func (b *InMemoryBackend) DescribeMovingAddresses(publicIPs []string) []*MovingA filter[ip] = true } - out := make([]*MovingAddressStatus, 0, len(b.movingAddresses)) + out := make([]*MovingAddressStatus, 0, b.movingAddresses.Len()) - for _, st := range b.movingAddresses { + for _, st := range b.movingAddresses.All() { if len(filter) > 0 && !filter[st.PublicIP] { continue } @@ -665,7 +662,7 @@ func (b *InMemoryBackend) RejectVpcEndpointConnections(serviceID string, vpcEndp for _, epID := range vpcEndpointIDs { key := serviceID + ":" + epID - conn, ok := b.vpcEndpointConnections[key] + conn, ok := b.vpcEndpointConnections.Get(key) if !ok { unsuccessful = append(unsuccessful, epID) @@ -695,7 +692,7 @@ func (b *InMemoryBackend) UnassignPrivateNatGatewayAddress( b.mu.Lock("UnassignPrivateNatGatewayAddress") defer b.mu.Unlock() - ngw, ok := b.natGateways[natGatewayID] + ngw, ok := b.natGateways.Get(natGatewayID) if !ok { return nil, fmt.Errorf("%w: %s", ErrNatGatewayNotFound, natGatewayID) } @@ -728,7 +725,7 @@ func (b *InMemoryBackend) UnassignPrivateNatGatewayAddress( // GetImageAncestry, and DescribeImageReferences so all three agree on what // counts as a known image. func (b *InMemoryBackend) lookupImageLocked(imageID string) *AMIStub { - if existing, ok := b.images[imageID]; ok { + if existing, ok := b.images.Get(imageID); ok { return existing } @@ -774,7 +771,7 @@ func (b *InMemoryBackend) DescribeImageReferences(imageIDs []string) []*ImageRef out := make([]*ImageReferenceEntry, 0) - for _, inst := range b.instances { + for _, inst := range b.instances.All() { if inst.ImageID == "" || (len(filter) > 0 && !filter[inst.ImageID]) { continue } @@ -786,7 +783,7 @@ func (b *InMemoryBackend) DescribeImageReferences(imageIDs []string) []*ImageRef }) } - for _, lt := range b.launchTemplates { + for _, lt := range b.launchTemplates.All() { if lt.ImageID == "" || (len(filter) > 0 && !filter[lt.ImageID]) { continue } @@ -865,7 +862,7 @@ func (b *InMemoryBackend) GetFlowLogsIntegrationTemplate(flowLogID, s3Destinatio b.mu.RLock("GetFlowLogsIntegrationTemplate") defer b.mu.RUnlock() - fl, ok := b.flowLogs[flowLogID] + fl, ok := b.flowLogs.Get(flowLogID) if !ok { return "", fmt.Errorf("%w: %s", ErrFlowLogNotFound, flowLogID) } @@ -968,7 +965,7 @@ func (b *InMemoryBackend) SendDiagnosticInterrupt(instanceID string) error { b.mu.RLock("SendDiagnosticInterrupt") defer b.mu.RUnlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } diff --git a/services/ec2/backend_refinement2.go b/services/ec2/backend_refinement2.go index 500d2befa..a56b7fb6c 100644 --- a/services/ec2/backend_refinement2.go +++ b/services/ec2/backend_refinement2.go @@ -84,7 +84,7 @@ func (b *InMemoryBackend) CreateSnapshot(volumeID, description string) (*Snapsho b.mu.Lock("CreateSnapshot") defer b.mu.Unlock() - vol, ok := b.volumes[volumeID] + vol, ok := b.volumes.Get(volumeID) if !ok { return nil, fmt.Errorf("%w: %s", ErrVolumeNotFound, volumeID) } @@ -100,7 +100,7 @@ func (b *InMemoryBackend) CreateSnapshot(volumeID, description string) (*Snapsho Encrypted: vol.Encrypted, KmsKeyID: vol.KmsKeyID, } - b.snapshots[snap.SnapshotID] = snap + b.snapshots.Put(snap) cp := *snap @@ -117,8 +117,8 @@ func (b *InMemoryBackend) DescribeSnapshots(ids []string) []*Snapshot { idSet[id] = true } - out := make([]*Snapshot, 0, len(b.snapshots)) - for _, snap := range b.snapshots { + out := make([]*Snapshot, 0, b.snapshots.Len()) + for _, snap := range b.snapshots.All() { if len(idSet) > 0 && !idSet[snap.SnapshotID] { continue } @@ -143,11 +143,10 @@ func (b *InMemoryBackend) DeleteSnapshot(id string) error { b.mu.Lock("DeleteSnapshot") defer b.mu.Unlock() - if _, ok := b.snapshots[id]; !ok { + if _, ok := b.snapshots.Get(id); !ok { return fmt.Errorf("%w: %s", ErrSnapshotNotFound, id) } - - delete(b.snapshots, id) + b.snapshots.Delete(id) return nil } @@ -185,12 +184,12 @@ func (b *InMemoryBackend) CopyImage(sourceImageID, name, description string) (*A RootDeviceName: src.RootDeviceName, SourceImageID: src.ImageID, } - b.images[newImage.ImageID] = newImage - b.imageUsageReports[newImage.ImageID] = &ImageUsageReport{ + b.images.Put(newImage) + b.imageUsageReports.Put(&ImageUsageReport{ ImageID: newImage.ImageID, State: stateAvailable, GenerationDate: time.Now().UTC().Format(time.RFC3339), - } + }) cp := *newImage @@ -206,12 +205,11 @@ func (b *InMemoryBackend) DeregisterImage(imageID string) error { b.mu.Lock("DeregisterImage") defer b.mu.Unlock() - if _, ok := b.images[imageID]; !ok { + if _, ok := b.images.Get(imageID); !ok { return fmt.Errorf("%w: image %s not found", ErrInvalidParameter, imageID) } - - delete(b.images, imageID) - delete(b.imageUsageReports, imageID) + b.images.Delete(imageID) + b.imageUsageReports.Delete(imageID) return nil } @@ -227,7 +225,7 @@ func (b *InMemoryBackend) ModifyVpcAttribute(vpcID, attribute string, value bool b.mu.Lock("ModifyVpcAttribute") defer b.mu.Unlock() - vpc, ok := b.vpcs[vpcID] + vpc, ok := b.vpcs.Get(vpcID) if !ok { return fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -254,7 +252,7 @@ func (b *InMemoryBackend) ModifySubnetAttribute(subnetID, attribute string, valu b.mu.Lock("ModifySubnetAttribute") defer b.mu.Unlock() - subnet, ok := b.subnets[subnetID] + subnet, ok := b.subnets.Get(subnetID) if !ok { return fmt.Errorf("%w: %s", ErrSubnetNotFound, subnetID) } @@ -282,7 +280,7 @@ func (b *InMemoryBackend) CreateNetworkACL(vpcID string) (*StoredNetworkACL, err b.mu.Lock("CreateNetworkACL") defer b.mu.Unlock() - if _, ok := b.vpcs[vpcID]; !ok { + if _, ok := b.vpcs.Get(vpcID); !ok { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -303,7 +301,7 @@ func (b *InMemoryBackend) CreateNetworkACL(vpcID string) (*StoredNetworkACL, err }, }, } - b.networkACLs[acl.ID] = acl + b.networkACLs.Put(acl) cp := *acl cp.Entries = append([]NACLEntry(nil), acl.Entries...) @@ -320,7 +318,7 @@ func (b *InMemoryBackend) DeleteNetworkACL(id string) error { b.mu.Lock("DeleteNetworkACL") defer b.mu.Unlock() - acl, ok := b.networkACLs[id] + acl, ok := b.networkACLs.Get(id) if !ok { return fmt.Errorf("%w: %s", ErrNetworkACLNotFound, id) } @@ -328,8 +326,7 @@ func (b *InMemoryBackend) DeleteNetworkACL(id string) error { if acl.IsDefault { return fmt.Errorf("%w: cannot delete default network ACL", ErrInvalidParameter) } - - delete(b.networkACLs, id) + b.networkACLs.Delete(id) return nil } @@ -346,7 +343,7 @@ func (b *InMemoryBackend) CreateNetworkACLEntry( b.mu.Lock("CreateNetworkACLEntry") defer b.mu.Unlock() - acl, ok := b.networkACLs[aclID] + acl, ok := b.networkACLs.Get(aclID) if !ok { return fmt.Errorf("%w: %s", ErrNetworkACLNotFound, aclID) } @@ -383,7 +380,7 @@ func (b *InMemoryBackend) DeleteNetworkACLEntry(aclID string, ruleNumber int, eg b.mu.Lock("DeleteNetworkACLEntry") defer b.mu.Unlock() - acl, ok := b.networkACLs[aclID] + acl, ok := b.networkACLs.Get(aclID) if !ok { return fmt.Errorf("%w: %s", ErrNetworkACLNotFound, aclID) } @@ -425,8 +422,8 @@ func (b *InMemoryBackend) DescribeStoredNetworkAcls(ids []string) []*StoredNetwo idSet[id] = true } - out := make([]*StoredNetworkACL, 0, len(b.networkACLs)) - for _, acl := range b.networkACLs { + out := make([]*StoredNetworkACL, 0, b.networkACLs.Len()) + for _, acl := range b.networkACLs.All() { if len(idSet) > 0 && !idSet[acl.ID] { continue } @@ -457,7 +454,7 @@ func (b *InMemoryBackend) DescribeSecurityGroupRules( b.mu.RLock("DescribeSecurityGroupRules") defer b.mu.RUnlock() - sg, ok := b.securityGroups[groupID] + sg, ok := b.securityGroups.Get(groupID) if !ok { return nil, fmt.Errorf("%w: %s", ErrSecurityGroupNotFound, groupID) } @@ -507,7 +504,7 @@ func (b *InMemoryBackend) ModifySecurityGroupRules( b.mu.Lock("ModifySecurityGroupRules") defer b.mu.Unlock() - sg, ok := b.securityGroups[groupID] + sg, ok := b.securityGroups.Get(groupID) if !ok { return fmt.Errorf("%w: %s", ErrSecurityGroupNotFound, groupID) } @@ -532,11 +529,10 @@ func (b *InMemoryBackend) DeleteLaunchTemplate(id string) error { b.mu.Lock("DeleteLaunchTemplate") defer b.mu.Unlock() - if _, ok := b.launchTemplates[id]; !ok { + if _, ok := b.launchTemplates.Get(id); !ok { return fmt.Errorf("%w: %s", ErrLaunchTemplateNotFound, id) } - - delete(b.launchTemplates, id) + b.launchTemplates.Delete(id) return nil } @@ -551,7 +547,7 @@ func (b *InMemoryBackend) DescribeLaunchTemplateVersions(id string) ([]*LaunchTe b.mu.RLock("DescribeLaunchTemplateVersions") defer b.mu.RUnlock() - lt, ok := b.launchTemplates[id] + lt, ok := b.launchTemplates.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrLaunchTemplateNotFound, id) } @@ -575,13 +571,12 @@ func (b *InMemoryBackend) DeleteVpcEndpoints(ids []string) ([]string, error) { var unsuccessful []string for _, id := range ids { - if _, ok := b.vpcEndpoints[id]; !ok { + if _, ok := b.vpcEndpoints.Get(id); !ok { unsuccessful = append(unsuccessful, id) continue } - - delete(b.vpcEndpoints, id) + b.vpcEndpoints.Delete(id) } if len(unsuccessful) > 0 { @@ -614,7 +609,7 @@ func (b *InMemoryBackend) DescribeVpcEndpointsByVPC(vpcID string) []*VpcEndpoint var out []*VpcEndpoint - for _, ep := range b.vpcEndpoints { + for _, ep := range b.vpcEndpoints.All() { if ep.VPCID != vpcID { continue } @@ -646,7 +641,7 @@ func (b *InMemoryBackend) DescribeNetworkAclsFiltered(vpcIDs []string) []*Networ allowed[id] = true } - for _, acl := range b.networkACLs { + for _, acl := range b.networkACLs.All() { if len(allowed) > 0 && !allowed[acl.VPCID] { continue } @@ -679,7 +674,7 @@ func (b *InMemoryBackend) DescribeSubnetsByVPC(vpcID string) []*Subnet { var out []*Subnet - for _, s := range b.subnets { + for _, s := range b.subnets.All() { if s.VPCID != vpcID { continue } @@ -708,7 +703,7 @@ func (b *InMemoryBackend) DescribeInstancesByVPC(vpcID string) []*Instance { out := make([]*Instance, 0, len(ids)) for id := range ids { - if inst, exists := b.instances[id]; exists { + if inst, exists := b.instances.Get(id); exists { cp := *inst out = append(out, &cp) } diff --git a/services/ec2/backend_refinement3.go b/services/ec2/backend_refinement3.go index 68fec4769..c7d814cb0 100644 --- a/services/ec2/backend_refinement3.go +++ b/services/ec2/backend_refinement3.go @@ -28,7 +28,7 @@ func (b *InMemoryBackend) ReplaceNetworkACLEntry( b.mu.Lock("ReplaceNetworkACLEntry") defer b.mu.Unlock() - acl, ok := b.networkACLs[aclID] + acl, ok := b.networkACLs.Get(aclID) if !ok { return fmt.Errorf("%w: %s", ErrNetworkACLNotFound, aclID) } @@ -71,16 +71,16 @@ func (b *InMemoryBackend) ReplaceNetworkACLAssociation(aclID, subnetID string) ( b.mu.Lock("ReplaceNetworkACLAssociation") defer b.mu.Unlock() - if _, ok := b.networkACLs[aclID]; !ok { + if _, ok := b.networkACLs.Get(aclID); !ok { return "", fmt.Errorf("%w: %s", ErrNetworkACLNotFound, aclID) } - if _, ok := b.subnets[subnetID]; !ok { + if _, ok := b.subnets.Get(subnetID); !ok { return "", fmt.Errorf("%w: %s", ErrSubnetNotFound, subnetID) } // Remove subnetID from any existing ACL. - for _, existing := range b.networkACLs { + for _, existing := range b.networkACLs.All() { for i, assoc := range existing.AssociationIDs { if assoc == subnetID { existing.AssociationIDs = append( @@ -93,7 +93,7 @@ func (b *InMemoryBackend) ReplaceNetworkACLAssociation(aclID, subnetID string) ( } } - target := b.networkACLs[aclID] + target, _ := b.networkACLs.Get(aclID) target.AssociationIDs = append(target.AssociationIDs, subnetID) newAssocID := "aclassoc-" + uuid.New().String()[:8] @@ -137,7 +137,7 @@ func (b *InMemoryBackend) ExportKeyPair(name string) (string, error) { b.mu.RLock("ExportKeyPair") defer b.mu.RUnlock() - kp, ok := b.keyPairs[name] + kp, ok := b.keyPairs.Get(name) if !ok { return "", fmt.Errorf("%w: %s", errR3KeyPairNotFound, name) } @@ -197,7 +197,7 @@ func (b *InMemoryBackend) CreateVpcPeeringConnection( b.mu.Lock("CreateVpcPeeringConnection") defer b.mu.Unlock() - if _, ok := b.vpcs[requesterVPCID]; !ok { + if _, ok := b.vpcs.Get(requesterVPCID); !ok { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, requesterVPCID) } @@ -207,7 +207,7 @@ func (b *InMemoryBackend) CreateVpcPeeringConnection( AccepterVpcID: accepterVPCID, State: "pending-acceptance", } - b.vpcPeeringConnections[pc.VpcPeeringConnectionID] = pc + b.vpcPeeringConnections.Put(pc) cp := *pc @@ -223,11 +223,10 @@ func (b *InMemoryBackend) DeleteVpcPeeringConnection(id string) error { b.mu.Lock("DeleteVpcPeeringConnection") defer b.mu.Unlock() - if _, ok := b.vpcPeeringConnections[id]; !ok { + if _, ok := b.vpcPeeringConnections.Get(id); !ok { return fmt.Errorf("%w: peering connection %s not found", ErrInvalidParameter, id) } - - delete(b.vpcPeeringConnections, id) + b.vpcPeeringConnections.Delete(id) return nil } @@ -252,9 +251,9 @@ func (b *InMemoryBackend) DescribeTransitGateways(ids []string) []*TransitGatewa idSet[id] = true } - out := make([]*TransitGateway, 0, len(b.transitGateways)) + out := make([]*TransitGateway, 0, b.transitGateways.Len()) - for _, tgw := range b.transitGateways { + for _, tgw := range b.transitGateways.All() { if len(idSet) > 0 && !idSet[tgw.ID] { continue } @@ -281,7 +280,7 @@ func (b *InMemoryBackend) CreateTransitGateway(description string) (*TransitGate State: stateAvailable, OwnerID: b.AccountID, } - b.transitGateways[tgw.ID] = tgw + b.transitGateways.Put(tgw) cp := *tgw @@ -297,11 +296,10 @@ func (b *InMemoryBackend) DeleteTransitGateway(id string) error { b.mu.Lock("DeleteTransitGateway") defer b.mu.Unlock() - if _, ok := b.transitGateways[id]; !ok { + if _, ok := b.transitGateways.Get(id); !ok { return fmt.Errorf("%w: transit gateway %s not found", ErrInvalidParameter, id) } - - delete(b.transitGateways, id) + b.transitGateways.Delete(id) return nil } diff --git a/services/ec2/backend_resource_types.go b/services/ec2/backend_resource_types.go new file mode 100644 index 000000000..7ef4760d3 --- /dev/null +++ b/services/ec2/backend_resource_types.go @@ -0,0 +1,359 @@ +package ec2 + +import "strings" + +// This file centralises the mapping from an EC2 resource ID to (a) whether the +// resource is known to the backend (resourceExistsLocked, gating CreateTags / +// DeleteTags) and (b) its AWS ResourceType string (resourceTypeByID, used by +// DescribeTags and the resource-type Filter). Both were previously limited to +// a handful of core resource types (instance, security-group, vpc, subnet, +// volume, internet-gateway, route-table, natgateway, elastic-ip), which meant +// CreateTags/DeleteTags/DescribeTags silently failed or mis-typed the large +// majority of EC2 resources this backend actually models (AMIs, snapshots, +// network ACLs, transit gateways and their attachments, VPN/customer +// gateways, VPC endpoints, launch templates, IPAM objects, and so on). The +// tables below cover every resource type in this backend that AWS exposes as +// independently taggable (per aws-sdk-go-v2 ec2/types.ResourceType); IDs for +// non-taggable association/index objects (e.g. route-table associations, +// subnet CIDR associations) are intentionally omitted. + +// resourceTypePrefix pairs an ID prefix with its AWS ResourceType string. +type resourceTypePrefix struct { + prefix string + rtype string +} + +// resourceTypePrefixes lists ID-prefix → ResourceType mappings, ordered most- +// specific-prefix-first within each ambiguous family (e.g. "tgw-rtb-ann-" +// before "tgw-rtb-", "ipam-pool-" before "ipam-") so the first HasPrefix match +// in resourceTypeByID is always the correct one. +// +//nolint:gochecknoglobals // package-level lookup table, analogous to errCodeLookup +var resourceTypePrefixes = []resourceTypePrefix{ + // ---- core (pre-existing) ---- + {"i-", "instance"}, + {"sg-", "security-group"}, + {"subnet-", "subnet"}, + {"vol-", "volume"}, + {"igw-", "internet-gateway"}, + {"rtb-", "route-table"}, + {"nat-", "natgateway"}, + {"eipalloc-", "elastic-ip"}, + {"eni-", "network-interface"}, + {"sir-", "spot-instances-request"}, + {"sfr-", "spot-fleet-request"}, + + // ---- images / snapshots / templates ---- + {"ami-", "image"}, + {"imgusgrpt-", "image-usage-report"}, + {"snap-", "snapshot"}, + {"lt-", "launch-template"}, + {"import-ami-", "import-image-task"}, + {"import-snap-", "import-snapshot-task"}, + {"export-ami-", "export-image-task"}, + {"export-i-", "export-instance-task"}, + + // ---- VPC networking ---- + {"vpc-ec-", "vpc-encryption-control"}, // must precede generic "vpc-" + {"vpc-", resourceTypeVPC}, + {"acl-", "network-acl"}, + {"dopt-", "dhcp-options"}, + {"eigw-", "egress-only-internet-gateway"}, + {"pcx-", "vpc-peering-connection"}, + {"vpce-svc-", "vpc-endpoint-service"}, // must precede generic "vpce-" + {"vpce-", "vpc-endpoint"}, + {"pl-", "prefix-list"}, + {"fl-", "vpc-flow-log"}, + {"vpcbpa-exclusion-", "vpc-block-public-access-exclusion"}, + {"eice-", "instance-connect-endpoint"}, + {"cagw-", "carrier-gateway"}, + + // ---- VPN / customer gateways ---- + {"vgw-", "vpn-gateway"}, + {"cgw-", "customer-gateway"}, + {"vpn-", "vpn-connection"}, + {"vpnc-", "vpn-concentrator"}, + + // ---- transit gateway family (generic "tgw-" must be last) ---- + {"tgw-rtb-ann-", "transit-gateway-route-table-announcement"}, + {"tgw-rtb-", "transit-gateway-route-table"}, + {"tgw-ptb-", "transit-gateway-policy-table"}, + {"tgw-metering-policy-", "transit-gateway-metering-policy"}, + {"tgw-mcast-domain-", "transit-gateway-multicast-domain"}, + {"tgw-connect-peer-", "transit-gateway-connect-peer"}, + {"tgw-attach-", "transit-gateway-attachment"}, + {"tgw-", "transit-gateway"}, + + // ---- local gateway family (generic "lgw-" must be last) ---- + {"lgw-vif-grp-", "local-gateway-virtual-interface-group"}, + {"lgw-vif-", "local-gateway-virtual-interface"}, + {"lgw-rtb-", "local-gateway-route-table"}, + {"lgw-route-table-vpc-assoc-", "local-gateway-route-table-vpc-association"}, + { + "lgw-route-table-virtual-interface-group-assoc-", + "local-gateway-route-table-virtual-interface-group-association", + }, + {"lgw-", "local-gateway"}, + + // ---- IPAM family (generic "ipam-" must be last) ---- + {"ipam-prefix-list-resolver-target-", "ipam-prefix-list-resolver-target"}, + {"ipam-prefix-list-resolver-", "ipam-prefix-list-resolver"}, + {"ipam-res-disco-assoc-", "ipam-resource-discovery-association"}, + {"ipam-res-disco-", "ipam-resource-discovery"}, + {"ipam-ext-res-verification-token-", "ipam-external-resource-verification-token"}, + {"ipam-pool-", "ipam-pool"}, + {"ipam-scope-", "ipam-scope"}, + {"ipam-policy-", "ipam-policy"}, + {"ipam-", "ipam"}, + + // ---- capacity reservations / hosts ---- + {"crf-", "capacity-reservation-fleet"}, + {"cr-", "capacity-reservation"}, + {"cb-", "capacity-block"}, + {"cmde-", "capacity-manager-data-export"}, + {"hr-", "host-reservation"}, + {"h-", "dedicated-host"}, + + // ---- fleets / reserved instances ---- + {"fleet-", "fleet"}, + {"ri-", "reserved-instances"}, + + // ---- verified access ---- + {"vae-", "verified-access-endpoint"}, + {"vagr-", "verified-access-group"}, + {"vai-", "verified-access-instance"}, + {"vatp-", "verified-access-trust-provider"}, + + // ---- traffic mirroring ---- + {"tmf-", "traffic-mirror-filter"}, + {"tmfr-", "traffic-mirror-filter-rule"}, + {"tms-", "traffic-mirror-session"}, + {"tmt-", "traffic-mirror-target"}, + + // ---- network insights ---- + {"niasa-", "network-insights-access-scope-analysis"}, + {"nias-", "network-insights-access-scope"}, + {"nia-", "network-insights-analysis"}, + {"nip-", "network-insights-path"}, + + // ---- route server ---- + {"rse-", "route-server-endpoint"}, + {"rsp-", "route-server-peer"}, + {"rs-", "route-server"}, + + // ---- client VPN / FPGA / mac / misc ---- + {"cvpn-endpoint-", "client-vpn-endpoint"}, + {"afi-", "fpga-image"}, + {"agfi-", "fpga-image"}, + {"macmodtask-", "mac-modification-task"}, + {"lag-", "outpost-lag"}, + {"iew-", "instance-event-window"}, + {"report-", "declarative-policies-report"}, + + // ---- secondary networking ---- + {"secnet-", "secondary-network"}, + {"secsubnet-", "secondary-subnet"}, + {"secni-", "secondary-interface"}, + {"svif-", "service-link-virtual-interface"}, + + // ---- IP address pools ---- + {"ipv4pool-coip-", "coip-pool"}, + {"ipv4pool-ec2-", "ipv4pool-ec2"}, + {"ipv6pool-ec2-", "ipv6pool-ec2"}, +} + +// resourceTypeByID infers the EC2 resource type from the ID prefix. +func resourceTypeByID(id string) string { + for _, e := range resourceTypePrefixes { + if strings.HasPrefix(id, e.prefix) { + return e.rtype + } + } + + return "resource" +} + +// resourceExistsLocked reports whether id refers to any known EC2 resource. +// Must be called with b.mu held. Split across several helpers (one per +// resource family, each kept small enough to stay under the cyclomatic/ +// cognitive complexity limits) rather than one flat function. +func (b *InMemoryBackend) resourceExistsLocked(id string) bool { + return b.resourceExistsCoreLocked(id) || + b.resourceExistsImagesLocked(id) || + b.resourceExistsVpcAuxLocked(id) || + b.resourceExistsGatewayLocked(id) || + b.resourceExistsTGWLocked(id) || + b.resourceExistsLGWLocked(id) || + b.resourceExistsIpamLocked(id) || + b.resourceExistsVerifiedAccessAndMirrorLocked(id) || + b.resourceExistsInsightsAndRouteServerLocked(id) || + b.resourceExistsSecondaryAndMiscLocked(id) +} + +// resourceExistsCoreLocked checks the original core resource maps (instances, +// security groups, VPC/subnet, storage, networking primitives). +func (b *InMemoryBackend) resourceExistsCoreLocked(id string) bool { + _, ok := b.instances.Get(id) + ok = ok || b.securityGroups.Has(id) + ok = ok || b.vpcs.Has(id) + ok = ok || b.subnets.Has(id) + ok = ok || b.keyPairs.Has(id) + ok = ok || b.volumes.Has(id) + ok = ok || b.addresses.Has(id) + ok = ok || b.internetGateways.Has(id) + ok = ok || b.routeTables.Has(id) + ok = ok || b.natGateways.Has(id) + ok = ok || b.networkInterfaces.Has(id) + ok = ok || b.spotRequests.Has(id) + ok = ok || b.placementGroups.Has(id) + ok = ok || b.spotFleets.Has(id) + + return ok +} + +// resourceExistsImagesLocked checks AMIs, snapshots, launch templates, and +// their import/export tasks. +func (b *InMemoryBackend) resourceExistsImagesLocked(id string) bool { + ok := b.images.Has(id) + ok = ok || b.imageUsageReports.Has(id) + ok = ok || b.snapshots.Has(id) + ok = ok || b.recycleBinSnapshots.Has(id) + ok = ok || b.launchTemplates.Has(id) + ok = ok || b.imageImportTasks.Has(id) + ok = ok || b.snapshotImportTasks.Has(id) + ok = ok || b.exportImageTasks.Has(id) + ok = ok || b.exportTasks.Has(id) + + return ok +} + +// resourceExistsVpcAuxLocked checks VPC-adjacent networking resources +// (endpoints, peering, ACLs, DHCP options, flow logs, prefix lists). +func (b *InMemoryBackend) resourceExistsVpcAuxLocked(id string) bool { + ok := b.networkACLs.Has(id) + ok = ok || b.dhcpOptionSets.Has(id) + ok = ok || b.egressOnlyIGWs.Has(id) + ok = ok || b.vpcPeeringConnections.Has(id) + ok = ok || b.vpcEndpoints.Has(id) + ok = ok || b.vpcEndpointServiceConfigs.Has(id) + ok = ok || b.managedPrefixLists.Has(id) + ok = ok || b.flowLogs.Has(id) + ok = ok || b.vpcBlockPublicAccessExclusions.Has(id) + ok = ok || b.instanceConnectEndpoints.Has(id) + ok = ok || b.carrierGateways.Has(id) + ok = ok || b.vpcEncryptionControls.Has(id) + + return ok +} + +// resourceExistsGatewayLocked checks VPN/customer gateways, capacity +// reservations/hosts, fleets, and reserved instances. +func (b *InMemoryBackend) resourceExistsGatewayLocked(id string) bool { + ok := b.vpnGateways.Has(id) + ok = ok || b.customerGateways.Has(id) + ok = ok || b.vpnConnections.Has(id) + ok = ok || b.vpnConcentrators.Has(id) + ok = ok || b.capacityReservations.Has(id) + ok = ok || b.capacityReservationFleets.Has(id) + ok = ok || b.capacityBlocks.Has(id) + ok = ok || b.capacityManagerDataExports.Has(id) + ok = ok || b.hostReservations.Has(id) + ok = ok || b.dedicatedHosts.Has(id) + ok = ok || b.fleets.Has(id) + ok = ok || b.reservedInstances.Has(id) + + return ok +} + +// resourceExistsTGWLocked checks the transit-gateway resource family. +func (b *InMemoryBackend) resourceExistsTGWLocked(id string) bool { + ok := b.transitGateways.Has(id) + ok = ok || b.tgwRouteTables.Has(id) + ok = ok || b.tgwPolicyTables.Has(id) + ok = ok || b.tgwRouteTableAnnouncements.Has(id) + ok = ok || b.tgwMeteringPolicies.Has(id) + ok = ok || b.tgwMulticastDomains.Has(id) + ok = ok || b.tgwConnectPeers.Has(id) + ok = ok || b.tgwConnects.Has(id) + ok = ok || b.tgwVpcAttachments.Has(id) + ok = ok || b.tgwPeeringAttachments.Has(id) + + return ok +} + +// resourceExistsLGWLocked checks the local-gateway resource family. +func (b *InMemoryBackend) resourceExistsLGWLocked(id string) bool { + ok := b.localGateways.Has(id) + ok = ok || b.localGatewayVirtualInterfaces.Has(id) + ok = ok || b.localGatewayVirtualInterfaceGroups.Has(id) + ok = ok || b.localGatewayRouteTables.Has(id) + ok = ok || b.localGatewayRouteTableVpcAssociations.Has(id) + ok = ok || b.localGatewayRouteTableVifGroupAssociations.Has(id) + + return ok +} + +// resourceExistsIpamLocked checks the IPAM resource family. +func (b *InMemoryBackend) resourceExistsIpamLocked(id string) bool { + ok := b.ipams.Has(id) + ok = ok || b.ipamPools.Has(id) + ok = ok || b.ipamScopes.Has(id) + ok = ok || b.ipamResourceDiscoveries.Has(id) + ok = ok || b.ipamResourceDiscoveryAssocs.Has(id) + ok = ok || b.ipamVerificationTokens.Has(id) + ok = ok || b.ipamPolicies.Has(id) + ok = ok || b.ipamPrefixListResolvers.Has(id) + ok = ok || b.ipamPrefixListResolverTargets.Has(id) + ok = ok || b.ipv4Pools.Has(id) + ok = ok || b.ipv6Pools.Has(id) + ok = ok || b.coipPools.Has(id) + + return ok +} + +// resourceExistsVerifiedAccessAndMirrorLocked checks Verified Access and +// traffic-mirroring resources. +func (b *InMemoryBackend) resourceExistsVerifiedAccessAndMirrorLocked(id string) bool { + ok := b.verifiedAccessEndpoints.Has(id) + ok = ok || b.verifiedAccessGroups.Has(id) + ok = ok || b.verifiedAccessInstances.Has(id) + ok = ok || b.verifiedAccessTrustProviders.Has(id) + ok = ok || b.trafficMirrorFilters.Has(id) + ok = ok || b.trafficMirrorFilterRules.Has(id) + ok = ok || b.trafficMirrorSessions.Has(id) + ok = ok || b.trafficMirrorTargets.Has(id) + + return ok +} + +// resourceExistsInsightsAndRouteServerLocked checks Network Insights, Route +// Server, and Client VPN resources. +func (b *InMemoryBackend) resourceExistsInsightsAndRouteServerLocked(id string) bool { + ok := b.networkInsightsPaths.Has(id) + ok = ok || b.networkInsightsAnalyses.Has(id) + ok = ok || b.networkInsightsAccessScopes.Has(id) + ok = ok || b.networkInsightsAccessScopeAnalyses.Has(id) + ok = ok || b.routeServers.Has(id) + ok = ok || b.routeServerEndpoints.Has(id) + ok = ok || b.routeServerPeers.Has(id) + ok = ok || b.clientVpnEndpoints.Has(id) + + return ok +} + +// resourceExistsSecondaryAndMiscLocked checks FPGA images, Mac modification +// tasks, Outpost LAGs, instance event windows, declarative policy reports, +// and secondary-networking resources. +func (b *InMemoryBackend) resourceExistsSecondaryAndMiscLocked(id string) bool { + ok := b.fpgaImages.Has(id) + ok = ok || b.macModificationTasks.Has(id) + ok = ok || b.outpostLags.Has(id) + ok = ok || b.instanceEventWindows.Has(id) + ok = ok || b.declarativePoliciesReports.Has(id) + ok = ok || b.secondaryNetworks.Has(id) + ok = ok || b.secondarySubnets.Has(id) + ok = ok || b.secondaryInterfaces.Has(id) + ok = ok || b.serviceLinkVirtualInterfaces.Has(id) + + return ok +} diff --git a/services/ec2/backend_route_server.go b/services/ec2/backend_route_server.go index 918b9ea90..6d2808cd4 100644 --- a/services/ec2/backend_route_server.go +++ b/services/ec2/backend_route_server.go @@ -126,7 +126,7 @@ func (b *InMemoryBackend) CreateRouteServer( PersistRoutesState: persistRoutesState, PersistRoutesDuration: persistRoutesDuration, } - b.routeServers[id] = rs + b.routeServers.Put(rs) cp := *rs @@ -140,7 +140,7 @@ func (b *InMemoryBackend) DescribeRouteServers(ids []string) []*RouteServer { var result []*RouteServer - for _, rs := range b.routeServers { + for _, rs := range b.routeServers.All() { if len(ids) > 0 && !slices.Contains(ids, rs.RouteServerID) { continue } @@ -159,14 +159,14 @@ func (b *InMemoryBackend) DeleteRouteServer(id string) (*RouteServer, error) { b.mu.Lock("DeleteRouteServer") defer b.mu.Unlock() - rs, ok := b.routeServers[id] + rs, ok := b.routeServers.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrRouteServerNotFound, id) } cp := *rs cp.State = stateDeleting - delete(b.routeServers, id) + b.routeServers.Delete(id) return &cp, nil } @@ -181,7 +181,7 @@ func (b *InMemoryBackend) ModifyRouteServer( b.mu.Lock("ModifyRouteServer") defer b.mu.Unlock() - rs, ok := b.routeServers[id] + rs, ok := b.routeServers.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrRouteServerNotFound, id) } @@ -214,11 +214,11 @@ func (b *InMemoryBackend) CreateRouteServerEndpoint(routeServerID, subnetID stri b.mu.Lock("CreateRouteServerEndpoint") defer b.mu.Unlock() - if _, ok := b.routeServers[routeServerID]; !ok { + if _, ok := b.routeServers.Get(routeServerID); !ok { return nil, fmt.Errorf("%w: %s", ErrRouteServerNotFound, routeServerID) } - subnet, ok := b.subnets[subnetID] + subnet, ok := b.subnets.Get(subnetID) if !ok { return nil, fmt.Errorf("%w: %s", ErrSubnetNotFound, subnetID) } @@ -234,7 +234,7 @@ func (b *InMemoryBackend) CreateRouteServerEndpoint(routeServerID, subnetID stri EniAddress: b.allocPrivateIP(), State: routeServerStateAvailable, } - b.routeServerEndpoints[id] = ep + b.routeServerEndpoints.Put(ep) cp := *ep @@ -248,7 +248,7 @@ func (b *InMemoryBackend) DescribeRouteServerEndpoints(ids []string) []*RouteSer var result []*RouteServerEndpoint - for _, ep := range b.routeServerEndpoints { + for _, ep := range b.routeServerEndpoints.All() { if len(ids) > 0 && !slices.Contains(ids, ep.RouteServerEndpointID) { continue } @@ -269,14 +269,14 @@ func (b *InMemoryBackend) DeleteRouteServerEndpoint(id string) (*RouteServerEndp b.mu.Lock("DeleteRouteServerEndpoint") defer b.mu.Unlock() - ep, ok := b.routeServerEndpoints[id] + ep, ok := b.routeServerEndpoints.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrRouteServerEndpointNotFound, id) } cp := *ep cp.State = stateDeleting - delete(b.routeServerEndpoints, id) + b.routeServerEndpoints.Delete(id) return &cp, nil } @@ -298,7 +298,7 @@ func (b *InMemoryBackend) CreateRouteServerPeer( b.mu.Lock("CreateRouteServerPeer") defer b.mu.Unlock() - ep, ok := b.routeServerEndpoints[endpointID] + ep, ok := b.routeServerEndpoints.Get(endpointID) if !ok { return nil, fmt.Errorf("%w: %s", ErrRouteServerEndpointNotFound, endpointID) } @@ -324,7 +324,7 @@ func (b *InMemoryBackend) CreateRouteServerPeer( BgpStatus: "down", BgpStatusPeerState: "idle", } - b.routeServerPeers[id] = peer + b.routeServerPeers.Put(peer) cp := *peer @@ -338,7 +338,7 @@ func (b *InMemoryBackend) DescribeRouteServerPeers(ids []string) []*RouteServerP var result []*RouteServerPeer - for _, p := range b.routeServerPeers { + for _, p := range b.routeServerPeers.All() { if len(ids) > 0 && !slices.Contains(ids, p.RouteServerPeerID) { continue } @@ -357,14 +357,14 @@ func (b *InMemoryBackend) DeleteRouteServerPeer(id string) (*RouteServerPeer, er b.mu.Lock("DeleteRouteServerPeer") defer b.mu.Unlock() - p, ok := b.routeServerPeers[id] + p, ok := b.routeServerPeers.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrRouteServerPeerNotFound, id) } cp := *p cp.State = stateDeleting - delete(b.routeServerPeers, id) + b.routeServerPeers.Delete(id) return &cp, nil } @@ -378,21 +378,20 @@ func (b *InMemoryBackend) AssociateRouteServer(routeServerID, vpcID string) (*Ro b.mu.Lock("AssociateRouteServer") defer b.mu.Unlock() - if _, ok := b.routeServers[routeServerID]; !ok { + if _, ok := b.routeServers.Get(routeServerID); !ok { return nil, fmt.Errorf("%w: %s", ErrRouteServerNotFound, routeServerID) } - if _, ok := b.vpcs[vpcID]; !ok { + if _, ok := b.vpcs.Get(vpcID); !ok { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } - key := routeServerID + "/" + vpcID assoc := &RouteServerAssociation{ RouteServerID: routeServerID, VpcID: vpcID, State: associationStateAssociated, } - b.routeServerAssociations[key] = assoc + b.routeServerAssociations.Put(assoc) cp := *assoc @@ -405,15 +404,14 @@ func (b *InMemoryBackend) DisassociateRouteServer(routeServerID, vpcID string) ( defer b.mu.Unlock() key := routeServerID + "/" + vpcID - - assoc, ok := b.routeServerAssociations[key] + assoc, ok := b.routeServerAssociations.Get(key) if !ok { return nil, fmt.Errorf("%w: %s/%s", ErrRouteServerAssociationNotFound, routeServerID, vpcID) } cp := *assoc cp.State = "disassociating" - delete(b.routeServerAssociations, key) + b.routeServerAssociations.Delete(key) return &cp, nil } @@ -425,7 +423,7 @@ func (b *InMemoryBackend) GetRouteServerAssociations(routeServerID string) []*Ro var result []*RouteServerAssociation - for _, a := range b.routeServerAssociations { + for _, a := range b.routeServerAssociations.All() { if a.RouteServerID != routeServerID { continue } @@ -450,21 +448,20 @@ func (b *InMemoryBackend) EnableRouteServerPropagation( b.mu.Lock("EnableRouteServerPropagation") defer b.mu.Unlock() - if _, ok := b.routeServers[routeServerID]; !ok { + if _, ok := b.routeServers.Get(routeServerID); !ok { return nil, fmt.Errorf("%w: %s", ErrRouteServerNotFound, routeServerID) } - if _, ok := b.routeTables[routeTableID]; !ok { + if _, ok := b.routeTables.Get(routeTableID); !ok { return nil, fmt.Errorf("%w: %s", ErrRouteTableNotFound, routeTableID) } - key := routeServerID + "/" + routeTableID prop := &RouteServerPropagation{ RouteServerID: routeServerID, RouteTableID: routeTableID, State: propagationStateEnabled, } - b.routeServerPropagations[key] = prop + b.routeServerPropagations.Put(prop) cp := *prop @@ -479,15 +476,14 @@ func (b *InMemoryBackend) DisableRouteServerPropagation( defer b.mu.Unlock() key := routeServerID + "/" + routeTableID - - prop, ok := b.routeServerPropagations[key] + prop, ok := b.routeServerPropagations.Get(key) if !ok { return nil, fmt.Errorf("%w: %s/%s", ErrRouteServerPropagationNotFound, routeServerID, routeTableID) } cp := *prop cp.State = "disabling" - delete(b.routeServerPropagations, key) + b.routeServerPropagations.Delete(key) return &cp, nil } @@ -499,7 +495,7 @@ func (b *InMemoryBackend) GetRouteServerPropagations(routeServerID string) []*Ro var result []*RouteServerPropagation - for _, p := range b.routeServerPropagations { + for _, p := range b.routeServerPropagations.All() { if p.RouteServerID != routeServerID { continue } @@ -522,7 +518,7 @@ func (b *InMemoryBackend) GetRouteServerRoutingDatabase(routeServerID string) ([ b.mu.RLock("GetRouteServerRoutingDatabase") defer b.mu.RUnlock() - if _, ok := b.routeServers[routeServerID]; !ok { + if _, ok := b.routeServers.Get(routeServerID); !ok { return nil, fmt.Errorf("%w: %s", ErrRouteServerNotFound, routeServerID) } diff --git a/services/ec2/backend_scheduled_instances.go b/services/ec2/backend_scheduled_instances.go index ebdbb30cb..ccc77ce81 100644 --- a/services/ec2/backend_scheduled_instances.go +++ b/services/ec2/backend_scheduled_instances.go @@ -105,7 +105,6 @@ type ScheduledInstance struct { // resetScheduledInstanceMapsLocked re-initialises the Scheduled Instance state maps. // Must be called with b.mu held. func (b *InMemoryBackend) resetScheduledInstanceMapsLocked() { - b.scheduledInstances = make(map[string]*ScheduledInstance) b.scheduledInstanceLaunched = make(map[string]int32) } @@ -283,7 +282,7 @@ func (b *InMemoryBackend) PurchaseScheduledInstances( TermEndDate: entry.FirstSlotStartTime.AddDate(0, 0, int(scheduledInstanceMaxTermDays)), NextSlotStartTime: entry.FirstSlotStartTime, } - b.scheduledInstances[sci.ScheduledInstanceID] = sci + b.scheduledInstances.Put(sci) cp := *sci out = append(out, &cp) @@ -310,9 +309,9 @@ func (b *InMemoryBackend) DescribeScheduledInstances(ids []string) []*ScheduledI filter[id] = true } - out := make([]*ScheduledInstance, 0, len(b.scheduledInstances)) + out := make([]*ScheduledInstance, 0, b.scheduledInstances.Len()) - for _, sci := range b.scheduledInstances { + for _, sci := range b.scheduledInstances.All() { if len(filter) > 0 && !filter[sci.ScheduledInstanceID] { continue } @@ -339,7 +338,7 @@ func (b *InMemoryBackend) RunScheduledInstances( b.mu.Lock("RunScheduledInstances") defer b.mu.Unlock() - sci, ok := b.scheduledInstances[scheduledInstanceID] + sci, ok := b.scheduledInstances.Get(scheduledInstanceID) if !ok { return nil, fmt.Errorf("%w: %s", ErrScheduledInstanceNotFound, scheduledInstanceID) } @@ -360,14 +359,14 @@ func (b *InMemoryBackend) RunScheduledInstances( for range instanceCount { instID := "i-" + uuid.New().String()[:17] - b.instances[instID] = &Instance{ + b.instances.Put(&Instance{ ID: instID, ImageID: imageID, InstanceType: sci.InstanceType, KeyName: keyName, LaunchTime: now, State: StateRunning, - } + }) ids = append(ids, instID) } diff --git a/services/ec2/backend_secondary_net.go b/services/ec2/backend_secondary_net.go index 14cf1ad08..12de373d3 100644 --- a/services/ec2/backend_secondary_net.go +++ b/services/ec2/backend_secondary_net.go @@ -138,11 +138,6 @@ type OutpostLag struct { // resetSecondaryNetworkMapsLocked re-initialises all maps owned by this file. // Must be called with b.mu held. func (b *InMemoryBackend) resetSecondaryNetworkMapsLocked() { - b.secondaryNetworks = make(map[string]*SecondaryNetwork) - b.secondarySubnets = make(map[string]*SecondarySubnet) - b.secondaryInterfaces = make(map[string]*SecondaryInterface) - b.serviceLinkVirtualInterfaces = make(map[string]*ServiceLinkVirtualInterface) - b.outpostLags = make(map[string]*OutpostLag) } // ---- Secondary Networks ---- @@ -179,7 +174,7 @@ func (b *InMemoryBackend) CreateSecondaryNetwork( }, Tags: tags, } - b.secondaryNetworks[id] = net + b.secondaryNetworks.Put(net) cp := *net @@ -196,20 +191,19 @@ func (b *InMemoryBackend) DeleteSecondaryNetwork(id string) (*SecondaryNetwork, b.mu.Lock("DeleteSecondaryNetwork") defer b.mu.Unlock() - net, ok := b.secondaryNetworks[id] + net, ok := b.secondaryNetworks.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrSecondaryNetworkNotFound, id) } - for _, s := range b.secondarySubnets { + for _, s := range b.secondarySubnets.All() { if s.SecondaryNetworkID == id { return nil, fmt.Errorf( "%w: secondary network %s still has secondary subnets", ErrSecondaryNetworkHasSubnets, id, ) } } - - delete(b.secondaryNetworks, id) + b.secondaryNetworks.Delete(id) cp := *net cp.State = secondaryStateDeleteComplete @@ -227,9 +221,9 @@ func (b *InMemoryBackend) DescribeSecondaryNetworks(ids []string) []*SecondaryNe idSet[id] = true } - out := make([]*SecondaryNetwork, 0, len(b.secondaryNetworks)) + out := make([]*SecondaryNetwork, 0, b.secondaryNetworks.Len()) - for _, n := range b.secondaryNetworks { + for _, n := range b.secondaryNetworks.All() { if len(idSet) > 0 && !idSet[n.SecondaryNetworkID] { continue } @@ -260,7 +254,7 @@ func (b *InMemoryBackend) CreateSecondarySubnet( b.mu.Lock("CreateSecondarySubnet") defer b.mu.Unlock() - net, ok := b.secondaryNetworks[secondaryNetworkID] + net, ok := b.secondaryNetworks.Get(secondaryNetworkID) if !ok { return nil, fmt.Errorf("%w: %s", ErrSecondaryNetworkNotFound, secondaryNetworkID) } @@ -288,7 +282,7 @@ func (b *InMemoryBackend) CreateSecondarySubnet( }, Tags: tags, } - b.secondarySubnets[id] = sub + b.secondarySubnets.Put(sub) cp := *sub @@ -304,12 +298,11 @@ func (b *InMemoryBackend) DeleteSecondarySubnet(id string) (*SecondarySubnet, er b.mu.Lock("DeleteSecondarySubnet") defer b.mu.Unlock() - sub, ok := b.secondarySubnets[id] + sub, ok := b.secondarySubnets.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrSecondarySubnetNotFound, id) } - - delete(b.secondarySubnets, id) + b.secondarySubnets.Delete(id) cp := *sub cp.State = secondaryStateDeleteComplete @@ -327,9 +320,9 @@ func (b *InMemoryBackend) DescribeSecondarySubnets(ids []string) []*SecondarySub idSet[id] = true } - out := make([]*SecondarySubnet, 0, len(b.secondarySubnets)) + out := make([]*SecondarySubnet, 0, b.secondarySubnets.Len()) - for _, s := range b.secondarySubnets { + for _, s := range b.secondarySubnets.All() { if len(idSet) > 0 && !idSet[s.SecondarySubnetID] { continue } @@ -364,7 +357,7 @@ func (b *InMemoryBackend) SeedSecondaryInterface(si SecondaryInterface) (*Second } cp := si - b.secondaryInterfaces[cp.SecondaryInterfaceID] = &cp + b.secondaryInterfaces.Put(&cp) out := cp @@ -382,9 +375,9 @@ func (b *InMemoryBackend) DescribeSecondaryInterfaces(ids []string) []*Secondary idSet[id] = true } - out := make([]*SecondaryInterface, 0, len(b.secondaryInterfaces)) + out := make([]*SecondaryInterface, 0, b.secondaryInterfaces.Len()) - for _, si := range b.secondaryInterfaces { + for _, si := range b.secondaryInterfaces.All() { if len(idSet) > 0 && !idSet[si.SecondaryInterfaceID] { continue } @@ -423,7 +416,7 @@ func (b *InMemoryBackend) SeedServiceLinkVirtualInterface( } cp := vif - b.serviceLinkVirtualInterfaces[cp.ServiceLinkVirtualInterfaceID] = &cp + b.serviceLinkVirtualInterfaces.Put(&cp) out := cp @@ -441,9 +434,9 @@ func (b *InMemoryBackend) DescribeServiceLinkVirtualInterfaces(ids []string) []* idSet[id] = true } - out := make([]*ServiceLinkVirtualInterface, 0, len(b.serviceLinkVirtualInterfaces)) + out := make([]*ServiceLinkVirtualInterface, 0, b.serviceLinkVirtualInterfaces.Len()) - for _, vif := range b.serviceLinkVirtualInterfaces { + for _, vif := range b.serviceLinkVirtualInterfaces.All() { if len(idSet) > 0 && !idSet[vif.ServiceLinkVirtualInterfaceID] { continue } @@ -480,7 +473,7 @@ func (b *InMemoryBackend) SeedOutpostLag(lag OutpostLag) (*OutpostLag, error) { } cp := lag - b.outpostLags[cp.OutpostLagID] = &cp + b.outpostLags.Put(&cp) out := cp @@ -497,9 +490,9 @@ func (b *InMemoryBackend) DescribeOutpostLags(ids []string) []*OutpostLag { idSet[id] = true } - out := make([]*OutpostLag, 0, len(b.outpostLags)) + out := make([]*OutpostLag, 0, b.outpostLags.Len()) - for _, lag := range b.outpostLags { + for _, lag := range b.outpostLags.All() { if len(idSet) > 0 && !idSet[lag.OutpostLagID] { continue } diff --git a/services/ec2/backend_spot_fleet.go b/services/ec2/backend_spot_fleet.go index 610fccfa1..7b887606f 100644 --- a/services/ec2/backend_spot_fleet.go +++ b/services/ec2/backend_spot_fleet.go @@ -125,12 +125,12 @@ func (b *InMemoryBackend) spawnFleetInstancesLocked( subnetID := spec.SubnetID if subnetID == "" { subnetID = b.findDefaultSubnetID() - } else if _, ok := b.subnets[subnetID]; !ok { + } else if _, ok := b.subnets.Get(subnetID); !ok { subnetID = b.findDefaultSubnetID() } vpcID := "" - if sub, ok := b.subnets[subnetID]; ok { + if sub, ok := b.subnets.Get(subnetID); ok { vpcID = sub.VPCID } @@ -157,7 +157,7 @@ func (b *InMemoryBackend) spawnFleetInstancesLocked( eniID := "eni-" + uuid.New().String()[:17] attachID := "eni-attach-" + uuid.New().String()[:8] - b.networkInterfaces[eniID] = &NetworkInterface{ + b.networkInterfaces.Put(&NetworkInterface{ ID: eniID, SubnetID: subnetID, VPCID: vpcID, @@ -167,11 +167,12 @@ func (b *InMemoryBackend) spawnFleetInstancesLocked( DeviceIndex: 0, Status: stateInUse, SourceDestCheck: true, - } - b.instances[id] = inst + }) + b.instances.Put(inst) b.indexInstanceLocked(inst) - b.indexENILocked(eniID, b.networkInterfaces[eniID]) - b.indexENIByVPCLocked(eniID, b.networkInterfaces[eniID]) + eni, _ := b.networkInterfaces.Get(eniID) + b.indexENILocked(eniID, eni) + b.indexENIByVPCLocked(eniID, eni) fleet.InstanceIDs = append(fleet.InstanceIDs, id) fulfilled += weightedCap @@ -228,7 +229,7 @@ func (b *InMemoryBackend) RequestSpotFleet( spawned, fulfilled := b.spawnFleetInstancesLocked(fleet, config) fleet.FulfilledCapacity = fulfilled - b.spotFleets[fleetID] = fleet + b.spotFleets.Put(fleet) // Add initial history record. b.spotFleetHistory[fleetID] = []SpotFleetHistoryRecord{ @@ -266,9 +267,10 @@ func (b *InMemoryBackend) DescribeSpotFleetRequests( wantSet[id] = struct{}{} } - results := make([]*SpotFleetRequest, 0, len(b.spotFleets)) + results := make([]*SpotFleetRequest, 0, b.spotFleets.Len()) - for id, fleet := range b.spotFleets { + for _, fleet := range b.spotFleets.All() { + id := spotFleetsKeyFn(fleet) if !wantAll { if _, ok := wantSet[id]; !ok { continue @@ -286,7 +288,7 @@ func (b *InMemoryBackend) DescribeSpotFleetRequests( // Check for requested IDs that don't exist. if !wantAll { for _, id := range fleetIDs { - if _, ok := b.spotFleets[id]; !ok { + if _, ok := b.spotFleets.Get(id); !ok { return nil, fmt.Errorf("%w: %s", ErrSpotFleetNotFound, id) } } @@ -318,7 +320,7 @@ func (b *InMemoryBackend) CancelSpotFleetRequests( results := make([]SpotFleetCancelResult, 0, len(fleetIDs)) for _, id := range fleetIDs { - fleet, ok := b.spotFleets[id] + fleet, ok := b.spotFleets.Get(id) if !ok { results = append(results, SpotFleetCancelResult{ SpotFleetRequestID: id, @@ -334,7 +336,7 @@ func (b *InMemoryBackend) CancelSpotFleetRequests( if terminateInstances { for _, instID := range fleet.InstanceIDs { - if inst, exists := b.instances[instID]; exists { + if inst, exists := b.instances.Get(instID); exists { inst.State = StateTerminated inst.TerminatedAt = time.Now().UTC() } @@ -401,12 +403,12 @@ func (b *InMemoryBackend) scaleFleetUpLocked( subnetID := spec.SubnetID if subnetID == "" { subnetID = b.findDefaultSubnetID() - } else if _, ok := b.subnets[subnetID]; !ok { + } else if _, ok := b.subnets.Get(subnetID); !ok { subnetID = b.findDefaultSubnetID() } vpcID := "" - if sub, ok := b.subnets[subnetID]; ok { + if sub, ok := b.subnets.Get(subnetID); ok { vpcID = sub.VPCID } @@ -430,7 +432,7 @@ func (b *InMemoryBackend) scaleFleetUpLocked( eniID := "eni-" + uuid.New().String()[:17] attachID := "eni-attach-" + uuid.New().String()[:8] - b.networkInterfaces[eniID] = &NetworkInterface{ + b.networkInterfaces.Put(&NetworkInterface{ ID: eniID, SubnetID: subnetID, VPCID: vpcID, @@ -440,11 +442,12 @@ func (b *InMemoryBackend) scaleFleetUpLocked( DeviceIndex: 0, Status: stateInUse, SourceDestCheck: true, - } - b.instances[id] = inst + }) + b.instances.Put(inst) b.indexInstanceLocked(inst) - b.indexENILocked(eniID, b.networkInterfaces[eniID]) - b.indexENIByVPCLocked(eniID, b.networkInterfaces[eniID]) + eni, _ := b.networkInterfaces.Get(eniID) + b.indexENILocked(eniID, eni) + b.indexENIByVPCLocked(eniID, eni) fleet.InstanceIDs = append(fleet.InstanceIDs, id) fleet.FulfilledCapacity += weightedCap @@ -470,7 +473,7 @@ func (b *InMemoryBackend) scaleFleetDownLocked( instID := fleet.InstanceIDs[lastIdx] fleet.InstanceIDs = fleet.InstanceIDs[:lastIdx] - if inst, exists := b.instances[instID]; exists { + if inst, exists := b.instances.Get(instID); exists { inst.State = StateTerminated inst.TerminatedAt = time.Now().UTC() } @@ -496,7 +499,7 @@ func (b *InMemoryBackend) ModifySpotFleetRequest( b.mu.Lock("ModifySpotFleetRequest") defer b.mu.Unlock() - fleet, ok := b.spotFleets[fleetID] + fleet, ok := b.spotFleets.Get(fleetID) if !ok { return nil, fmt.Errorf("%w: %s", ErrSpotFleetNotFound, fleetID) } @@ -546,7 +549,7 @@ func (b *InMemoryBackend) DescribeSpotFleetInstances(fleetID string) ([]SpotFlee b.mu.RLock("DescribeSpotFleetInstances") defer b.mu.RUnlock() - fleet, ok := b.spotFleets[fleetID] + fleet, ok := b.spotFleets.Get(fleetID) if !ok { return nil, fmt.Errorf("%w: %s", ErrSpotFleetNotFound, fleetID) } @@ -554,7 +557,7 @@ func (b *InMemoryBackend) DescribeSpotFleetInstances(fleetID string) ([]SpotFlee result := make([]SpotFleetInstance, 0, len(fleet.InstanceIDs)) for _, instID := range fleet.InstanceIDs { - inst, exists := b.instances[instID] + inst, exists := b.instances.Get(instID) if !exists { continue } @@ -587,7 +590,7 @@ func (b *InMemoryBackend) DescribeSpotFleetRequestHistory( b.mu.RLock("DescribeSpotFleetRequestHistory") defer b.mu.RUnlock() - if _, ok := b.spotFleets[fleetID]; !ok { + if _, ok := b.spotFleets.Get(fleetID); !ok { return nil, fmt.Errorf("%w: %s", ErrSpotFleetNotFound, fleetID) } diff --git a/services/ec2/backend_sql_ha.go b/services/ec2/backend_sql_ha.go index 580bcbebf..5899de61b 100644 --- a/services/ec2/backend_sql_ha.go +++ b/services/ec2/backend_sql_ha.go @@ -32,7 +32,6 @@ type RegisteredSQLHaInstance struct { // resetSQLHaMapsLocked re-initialises the SQL HA registration/history maps. // Must be called with b.mu held. func (b *InMemoryBackend) resetSQLHaMapsLocked() { - b.sqlHaRegistrations = make(map[string]*RegisteredSQLHaInstance) b.sqlHaHistory = make(map[string][]*RegisteredSQLHaInstance) } @@ -56,7 +55,7 @@ func (b *InMemoryBackend) EnableInstanceSQLHaStandbyDetections( defer b.mu.Unlock() for _, id := range instanceIDs { - if _, ok := b.instances[id]; !ok { + if _, ok := b.instances.Get(id); !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, id) } } @@ -72,7 +71,7 @@ func (b *InMemoryBackend) EnableInstanceSQLHaStandbyDetections( SQLServerCredentials: sqlServerCredentials, SQLServerLicenseUsage: "full", } - b.sqlHaRegistrations[id] = reg + b.sqlHaRegistrations.Put(reg) b.recordSQLHaHistoryLocked(reg) cp := *reg @@ -97,7 +96,7 @@ func (b *InMemoryBackend) DisableInstanceSQLHaStandbyDetections( out := make([]*RegisteredSQLHaInstance, 0, len(instanceIDs)) for _, id := range instanceIDs { - reg, ok := b.sqlHaRegistrations[id] + reg, ok := b.sqlHaRegistrations.Get(id) if !ok { continue } @@ -106,7 +105,7 @@ func (b *InMemoryBackend) DisableInstanceSQLHaStandbyDetections( cp.ProcessingStatus = "Disabled from SQL Server High Availability standby detection monitoring" cp.LastUpdatedTime = time.Now().UTC() b.recordSQLHaHistoryLocked(&cp) - delete(b.sqlHaRegistrations, id) + b.sqlHaRegistrations.Delete(id) out = append(out, &cp) } @@ -127,9 +126,9 @@ func (b *InMemoryBackend) DescribeInstanceSQLHaStates(ids []string) []*Registere idSet[id] = true } - out := make([]*RegisteredSQLHaInstance, 0, len(b.sqlHaRegistrations)) + out := make([]*RegisteredSQLHaInstance, 0, b.sqlHaRegistrations.Len()) - for _, reg := range b.sqlHaRegistrations { + for _, reg := range b.sqlHaRegistrations.All() { if len(idSet) > 0 && !idSet[reg.InstanceID] { continue } diff --git a/services/ec2/backend_tgw_multicast.go b/services/ec2/backend_tgw_multicast.go index 12dccb096..597a9686a 100644 --- a/services/ec2/backend_tgw_multicast.go +++ b/services/ec2/backend_tgw_multicast.go @@ -108,7 +108,7 @@ func (b *InMemoryBackend) CreateTransitGatewayMulticastDomain( b.mu.Lock("CreateTransitGatewayMulticastDomain") defer b.mu.Unlock() - if _, ok := b.transitGateways[tgwID]; !ok { + if _, ok := b.transitGateways.Get(tgwID); !ok { return nil, fmt.Errorf("%w: %s", ErrTransitGatewayNotFound, tgwID) } @@ -134,7 +134,7 @@ func (b *InMemoryBackend) CreateTransitGatewayMulticastDomain( Igmpv2Support: igmpv2Support, StaticSourcesSupport: staticSourcesSupport, } - b.tgwMulticastDomains[domain.ID] = domain + b.tgwMulticastDomains.Put(domain) cp := *domain @@ -154,9 +154,9 @@ func (b *InMemoryBackend) DescribeTransitGatewayMulticastDomains( idSet[id] = true } - out := make([]*TransitGatewayMulticastDomain, 0, len(b.tgwMulticastDomains)) + out := make([]*TransitGatewayMulticastDomain, 0, b.tgwMulticastDomains.Len()) - for _, d := range b.tgwMulticastDomains { + for _, d := range b.tgwMulticastDomains.All() { if len(idSet) > 0 && !idSet[d.ID] { continue } @@ -182,21 +182,22 @@ func (b *InMemoryBackend) DeleteTransitGatewayMulticastDomain(id string) error { b.mu.Lock("DeleteTransitGatewayMulticastDomain") defer b.mu.Unlock() - if _, ok := b.tgwMulticastDomains[id]; !ok { + if _, ok := b.tgwMulticastDomains.Get(id); !ok { return fmt.Errorf("%w: %s", ErrTGWMulticastDomainNotFound, id) } + b.tgwMulticastDomains.Delete(id) - delete(b.tgwMulticastDomains, id) - - for key, assoc := range b.tgwMulticastDomainAssociations { + for _, assoc := range b.tgwMulticastDomainAssociations.All() { + key := tgwMulticastDomainAssociationsKeyFn(assoc) if assoc.TransitGatewayMulticastDomainID == id { - delete(b.tgwMulticastDomainAssociations, key) + b.tgwMulticastDomainAssociations.Delete(key) } } - for key, entry := range b.tgwMulticastGroupEntries { + for _, entry := range b.tgwMulticastGroupEntries.All() { + key := tgwMulticastGroupEntriesKeyFn(entry) if entry.TransitGatewayMulticastDomainID == id { - delete(b.tgwMulticastGroupEntries, key) + b.tgwMulticastGroupEntries.Delete(key) } } @@ -229,21 +230,20 @@ func (b *InMemoryBackend) AssociateTransitGatewayMulticastDomain( b.mu.Lock("AssociateTransitGatewayMulticastDomain") defer b.mu.Unlock() - if _, ok := b.tgwMulticastDomains[domainID]; !ok { + if _, ok := b.tgwMulticastDomains.Get(domainID); !ok { return nil, fmt.Errorf("%w: %s", ErrTGWMulticastDomainNotFound, domainID) } assocs := make([]*TransitGatewayMulticastDomainAssociation, 0, len(subnetIDs)) for _, subnetID := range subnetIDs { - key := domainID + ":" + subnetID assoc := &TransitGatewayMulticastDomainAssociation{ TransitGatewayMulticastDomainID: domainID, TransitGatewayAttachmentID: attachmentID, SubnetID: subnetID, State: tgwMcastAssocStateAssociated, } - b.tgwMulticastDomainAssociations[key] = assoc + b.tgwMulticastDomainAssociations.Put(assoc) cp := *assoc assocs = append(assocs, &cp) @@ -277,9 +277,9 @@ func (b *InMemoryBackend) DisassociateTransitGatewayMulticastDomain( for _, subnetID := range subnetIDs { key := domainID + ":" + subnetID - existing, ok := b.tgwMulticastDomainAssociations[key] + existing, ok := b.tgwMulticastDomainAssociations.Get(key) if ok { - delete(b.tgwMulticastDomainAssociations, key) + b.tgwMulticastDomainAssociations.Delete(key) } else { existing = &TransitGatewayMulticastDomainAssociation{ TransitGatewayMulticastDomainID: domainID, @@ -306,7 +306,7 @@ func (b *InMemoryBackend) GetTransitGatewayMulticastDomainAssociations( out := make([]*TransitGatewayMulticastDomainAssociation, 0) - for _, assoc := range b.tgwMulticastDomainAssociations { + for _, assoc := range b.tgwMulticastDomainAssociations.All() { if domainID != "" && assoc.TransitGatewayMulticastDomainID != domainID { continue } @@ -347,7 +347,7 @@ func (b *InMemoryBackend) registerMulticastGroupResource( b.mu.Lock("registerMulticastGroupResource") defer b.mu.Unlock() - if _, ok := b.tgwMulticastDomains[domainID]; !ok { + if _, ok := b.tgwMulticastDomains.Get(domainID); !ok { return nil, fmt.Errorf("%w: %s", ErrTGWMulticastDomainNotFound, domainID) } @@ -356,7 +356,7 @@ func (b *InMemoryBackend) registerMulticastGroupResource( for _, eniID := range eniIDs { key := domainID + ":" + groupIP + ":" + eniID - entry, ok := b.tgwMulticastGroupEntries[key] + entry, ok := b.tgwMulticastGroupEntries.Get(key) if !ok { entry = &TransitGatewayMulticastGroupEntry{ TransitGatewayMulticastDomainID: domainID, @@ -365,7 +365,7 @@ func (b *InMemoryBackend) registerMulticastGroupResource( ResourceID: eniID, ResourceType: tgwResourceTypeVPC, } - b.tgwMulticastGroupEntries[key] = entry + b.tgwMulticastGroupEntries.Put(entry) } if asMember { @@ -407,7 +407,7 @@ func (b *InMemoryBackend) deregisterMulticastGroupResource( for _, eniID := range eniIDs { key := domainID + ":" + groupIP + ":" + eniID - entry, ok := b.tgwMulticastGroupEntries[key] + entry, ok := b.tgwMulticastGroupEntries.Get(key) if !ok { deregistered = append(deregistered, eniID) @@ -421,7 +421,7 @@ func (b *InMemoryBackend) deregisterMulticastGroupResource( } if !entry.IsMember && !entry.IsSource { - delete(b.tgwMulticastGroupEntries, key) + b.tgwMulticastGroupEntries.Delete(key) } deregistered = append(deregistered, eniID) @@ -480,7 +480,7 @@ func (b *InMemoryBackend) SearchTransitGatewayMulticastGroups( out := make([]*TransitGatewayMulticastGroupEntry, 0) - for _, entry := range b.tgwMulticastGroupEntries { + for _, entry := range b.tgwMulticastGroupEntries.All() { if domainID != "" && entry.TransitGatewayMulticastDomainID != domainID { continue } @@ -515,7 +515,7 @@ func (b *InMemoryBackend) CreateTransitGatewayMeteringPolicy( b.mu.Lock("CreateTransitGatewayMeteringPolicy") defer b.mu.Unlock() - if _, ok := b.transitGateways[tgwID]; !ok { + if _, ok := b.transitGateways.Get(tgwID); !ok { return nil, fmt.Errorf("%w: %s", ErrTransitGatewayNotFound, tgwID) } @@ -529,7 +529,7 @@ func (b *InMemoryBackend) CreateTransitGatewayMeteringPolicy( MiddleboxAttachmentIDs: ids, UpdateEffectiveAt: time.Now().UTC(), } - b.tgwMeteringPolicies[policy.ID] = policy + b.tgwMeteringPolicies.Put(policy) return copyTGWMeteringPolicy(policy), nil } @@ -557,9 +557,9 @@ func (b *InMemoryBackend) DescribeTransitGatewayMeteringPolicies( idSet[id] = true } - out := make([]*TransitGatewayMeteringPolicy, 0, len(b.tgwMeteringPolicies)) + out := make([]*TransitGatewayMeteringPolicy, 0, b.tgwMeteringPolicies.Len()) - for _, p := range b.tgwMeteringPolicies { + for _, p := range b.tgwMeteringPolicies.All() { if len(idSet) > 0 && !idSet[p.ID] { continue } @@ -584,15 +584,15 @@ func (b *InMemoryBackend) DeleteTransitGatewayMeteringPolicy(id string) error { b.mu.Lock("DeleteTransitGatewayMeteringPolicy") defer b.mu.Unlock() - if _, ok := b.tgwMeteringPolicies[id]; !ok { + if _, ok := b.tgwMeteringPolicies.Get(id); !ok { return fmt.Errorf("%w: %s", ErrTGWMeteringPolicyNotFound, id) } + b.tgwMeteringPolicies.Delete(id) - delete(b.tgwMeteringPolicies, id) - - for key, entry := range b.tgwMeteringPolicyEntries { + for _, entry := range b.tgwMeteringPolicyEntries.All() { + key := tgwMeteringPolicyEntriesKeyFn(entry) if entry.TransitGatewayMeteringPolicyID == id { - delete(b.tgwMeteringPolicyEntries, key) + b.tgwMeteringPolicyEntries.Delete(key) } } @@ -620,7 +620,7 @@ func (b *InMemoryBackend) CreateTransitGatewayMeteringPolicyEntry( b.mu.Lock("CreateTransitGatewayMeteringPolicyEntry") defer b.mu.Unlock() - if _, ok := b.tgwMeteringPolicies[policyID]; !ok { + if _, ok := b.tgwMeteringPolicies.Get(policyID); !ok { return nil, fmt.Errorf("%w: %s", ErrTGWMeteringPolicyNotFound, policyID) } @@ -631,8 +631,7 @@ func (b *InMemoryBackend) CreateTransitGatewayMeteringPolicyEntry( stored.UpdateEffectiveAt = now stored.UpdatedAt = now - key := policyID + ":" + strconv.Itoa(stored.PolicyRuleNumber) - b.tgwMeteringPolicyEntries[key] = &stored + b.tgwMeteringPolicyEntries.Put(&stored) cp := stored @@ -654,7 +653,7 @@ func (b *InMemoryBackend) DeleteTransitGatewayMeteringPolicyEntry( key := policyID + ":" + strconv.Itoa(ruleNumber) - entry, ok := b.tgwMeteringPolicyEntries[key] + entry, ok := b.tgwMeteringPolicyEntries.Get(key) if !ok { return nil, fmt.Errorf( "%w: metering policy entry %d in %s not found", @@ -663,8 +662,7 @@ func (b *InMemoryBackend) DeleteTransitGatewayMeteringPolicyEntry( policyID, ) } - - delete(b.tgwMeteringPolicyEntries, key) + b.tgwMeteringPolicyEntries.Delete(key) cp := *entry cp.State = tgwRouteStateDeleted diff --git a/services/ec2/backend_tgw_peripherals.go b/services/ec2/backend_tgw_peripherals.go index 5a174d7e3..8b3be2048 100644 --- a/services/ec2/backend_tgw_peripherals.go +++ b/services/ec2/backend_tgw_peripherals.go @@ -94,7 +94,7 @@ func (b *InMemoryBackend) CreateTransitGatewayPolicyTable( b.mu.Lock("CreateTransitGatewayPolicyTable") defer b.mu.Unlock() - if _, ok := b.transitGateways[tgwID]; !ok { + if _, ok := b.transitGateways.Get(tgwID); !ok { return nil, fmt.Errorf("%w: %s", ErrTransitGatewayNotFound, tgwID) } @@ -104,7 +104,7 @@ func (b *InMemoryBackend) CreateTransitGatewayPolicyTable( State: stateAvailable, CreationTime: time.Now().UTC(), } - b.tgwPolicyTables[pt.TransitGatewayPolicyTableID] = pt + b.tgwPolicyTables.Put(pt) cp := *pt @@ -124,9 +124,9 @@ func (b *InMemoryBackend) DescribeTransitGatewayPolicyTables( idSet[id] = true } - out := make([]*TransitGatewayPolicyTable, 0, len(b.tgwPolicyTables)) + out := make([]*TransitGatewayPolicyTable, 0, b.tgwPolicyTables.Len()) - for _, pt := range b.tgwPolicyTables { + for _, pt := range b.tgwPolicyTables.All() { if len(idSet) > 0 && !idSet[pt.TransitGatewayPolicyTableID] { continue } @@ -152,15 +152,15 @@ func (b *InMemoryBackend) DeleteTransitGatewayPolicyTable(id string) error { b.mu.Lock("DeleteTransitGatewayPolicyTable") defer b.mu.Unlock() - if _, ok := b.tgwPolicyTables[id]; !ok { + if _, ok := b.tgwPolicyTables.Get(id); !ok { return fmt.Errorf("%w: %s", ErrTGWPolicyTableNotFound, id) } + b.tgwPolicyTables.Delete(id) - delete(b.tgwPolicyTables, id) - - for key, assoc := range b.tgwPolicyTableAssociations { + for _, assoc := range b.tgwPolicyTableAssociations.All() { + key := tgwPolicyTableAssociationsKeyFn(assoc) if assoc.TransitGatewayPolicyTableID == id { - delete(b.tgwPolicyTableAssociations, key) + b.tgwPolicyTableAssociations.Delete(key) } } @@ -185,7 +185,7 @@ func (b *InMemoryBackend) AssociateTransitGatewayPolicyTable( b.mu.Lock("AssociateTransitGatewayPolicyTable") defer b.mu.Unlock() - if _, ok := b.tgwPolicyTables[policyTableID]; !ok { + if _, ok := b.tgwPolicyTables.Get(policyTableID); !ok { return nil, fmt.Errorf("%w: %s", ErrTGWPolicyTableNotFound, policyTableID) } @@ -195,8 +195,7 @@ func (b *InMemoryBackend) AssociateTransitGatewayPolicyTable( ResourceType: tgwResourceTypeVPC, State: tgwAssocStateAssociated, } - key := policyTableID + ":" + attachmentID - b.tgwPolicyTableAssociations[key] = assoc + b.tgwPolicyTableAssociations.Put(assoc) cp := *assoc @@ -220,8 +219,7 @@ func (b *InMemoryBackend) DisassociateTransitGatewayPolicyTable( defer b.mu.Unlock() key := policyTableID + ":" + attachmentID - - assoc, ok := b.tgwPolicyTableAssociations[key] + assoc, ok := b.tgwPolicyTableAssociations.Get(key) if !ok { return nil, fmt.Errorf( "%w: association between %s and %s not found", @@ -230,8 +228,7 @@ func (b *InMemoryBackend) DisassociateTransitGatewayPolicyTable( attachmentID, ) } - - delete(b.tgwPolicyTableAssociations, key) + b.tgwPolicyTableAssociations.Delete(key) cp := *assoc cp.State = tgwAssocStateDisassociated @@ -249,7 +246,7 @@ func (b *InMemoryBackend) GetTransitGatewayPolicyTableAssociations( out := make([]*TransitGatewayPolicyTableAssociation, 0) - for _, assoc := range b.tgwPolicyTableAssociations { + for _, assoc := range b.tgwPolicyTableAssociations.All() { if policyTableID != "" && assoc.TransitGatewayPolicyTableID != policyTableID { continue } @@ -277,7 +274,7 @@ func (b *InMemoryBackend) GetTransitGatewayPolicyTableEntries(policyTableID stri b.mu.RLock("GetTransitGatewayPolicyTableEntries") defer b.mu.RUnlock() - if _, ok := b.tgwPolicyTables[policyTableID]; !ok { + if _, ok := b.tgwPolicyTables.Get(policyTableID); !ok { return fmt.Errorf("%w: %s", ErrTGWPolicyTableNotFound, policyTableID) } @@ -302,12 +299,12 @@ func (b *InMemoryBackend) CreateTransitGatewayRouteTableAnnouncement( b.mu.Lock("CreateTransitGatewayRouteTableAnnouncement") defer b.mu.Unlock() - rt, ok := b.tgwRouteTables[routeTableID] + rt, ok := b.tgwRouteTables.Get(routeTableID) if !ok { return nil, fmt.Errorf("%w: %s", ErrTGWRouteTableNotFound, routeTableID) } - if _, attExists := b.tgwPeeringAttachments[peeringAttachmentID]; !attExists { + if _, attExists := b.tgwPeeringAttachments.Get(peeringAttachmentID); !attExists { return nil, fmt.Errorf("%w: %s", ErrTransitGatewayAttachmentNotFound, peeringAttachmentID) } @@ -320,7 +317,7 @@ func (b *InMemoryBackend) CreateTransitGatewayRouteTableAnnouncement( State: stateAvailable, CreationTime: time.Now().UTC(), } - b.tgwRouteTableAnnouncements[ann.TransitGatewayRouteTableAnnouncementID] = ann + b.tgwRouteTableAnnouncements.Put(ann) cp := *ann @@ -340,9 +337,9 @@ func (b *InMemoryBackend) DescribeTransitGatewayRouteTableAnnouncements( idSet[id] = true } - out := make([]*TransitGatewayRouteTableAnnouncement, 0, len(b.tgwRouteTableAnnouncements)) + out := make([]*TransitGatewayRouteTableAnnouncement, 0, b.tgwRouteTableAnnouncements.Len()) - for _, ann := range b.tgwRouteTableAnnouncements { + for _, ann := range b.tgwRouteTableAnnouncements.All() { if len(idSet) > 0 && !idSet[ann.TransitGatewayRouteTableAnnouncementID] { continue } @@ -372,11 +369,10 @@ func (b *InMemoryBackend) DeleteTransitGatewayRouteTableAnnouncement(id string) b.mu.Lock("DeleteTransitGatewayRouteTableAnnouncement") defer b.mu.Unlock() - if _, ok := b.tgwRouteTableAnnouncements[id]; !ok { + if _, ok := b.tgwRouteTableAnnouncements.Get(id); !ok { return fmt.Errorf("%w: %s", ErrTGWRouteTableAnnouncementNotFound, id) } - - delete(b.tgwRouteTableAnnouncements, id) + b.tgwRouteTableAnnouncements.Delete(id) return nil } @@ -396,13 +392,13 @@ func (b *InMemoryBackend) GetTransitGatewayRouteTableAssociations( b.mu.RLock("GetTransitGatewayRouteTableAssociations") defer b.mu.RUnlock() - if _, ok := b.tgwRouteTables[routeTableID]; !ok { + if _, ok := b.tgwRouteTables.Get(routeTableID); !ok { return nil, fmt.Errorf("%w: %s", ErrTGWRouteTableNotFound, routeTableID) } out := make([]*TransitGatewayRouteTableAssociation, 0) - for _, a := range b.tgwRTAssociations { + for _, a := range b.tgwRTAssociations.All() { if a.TransitGatewayRouteTableID != routeTableID { continue } @@ -432,7 +428,7 @@ func (b *InMemoryBackend) GetTransitGatewayRouteTablePropagations( b.mu.RLock("GetTransitGatewayRouteTablePropagations") defer b.mu.RUnlock() - if _, ok := b.tgwRouteTables[routeTableID]; !ok { + if _, ok := b.tgwRouteTables.Get(routeTableID); !ok { return nil, fmt.Errorf("%w: %s", ErrTGWRouteTableNotFound, routeTableID) } @@ -453,15 +449,15 @@ func (b *InMemoryBackend) GetTransitGatewayRouteTablePropagations( // exists in any of the known TGW attachment maps. Must be called with b.mu // held (for reading or writing). func (b *InMemoryBackend) transitGatewayAttachmentExistsLocked(id string) bool { - if _, ok := b.tgwVpcAttachments[id]; ok { + if _, ok := b.tgwVpcAttachments.Get(id); ok { return true } - if _, ok := b.tgwPeeringAttachments[id]; ok { + if _, ok := b.tgwPeeringAttachments.Get(id); ok { return true } - if _, ok := b.tgwConnects[id]; ok { + if _, ok := b.tgwConnects.Get(id); ok { return true } @@ -546,13 +542,13 @@ func (b *InMemoryBackend) SearchTransitGatewayRoutes( b.mu.RLock("SearchTransitGatewayRoutes") defer b.mu.RUnlock() - if _, ok := b.tgwRouteTables[routeTableID]; !ok { + if _, ok := b.tgwRouteTables.Get(routeTableID); !ok { return nil, fmt.Errorf("%w: %s", ErrTGWRouteTableNotFound, routeTableID) } out := make([]*TransitGatewayRoute, 0) - for _, r := range b.tgwRoutes { + for _, r := range b.tgwRoutes.All() { if r.TransitGatewayRouteTableID != routeTableID { continue } @@ -586,7 +582,7 @@ func (b *InMemoryBackend) ExportTransitGatewayRoutes(routeTableID, s3Bucket stri b.mu.RLock("ExportTransitGatewayRoutes") defer b.mu.RUnlock() - if _, ok := b.tgwRouteTables[routeTableID]; !ok { + if _, ok := b.tgwRouteTables.Get(routeTableID); !ok { return "", fmt.Errorf("%w: %s", ErrTGWRouteTableNotFound, routeTableID) } @@ -612,7 +608,7 @@ func (b *InMemoryBackend) ModifyTransitGatewayVpcAttachment( b.mu.Lock("ModifyTransitGatewayVpcAttachment") defer b.mu.Unlock() - att, ok := b.tgwVpcAttachments[attachmentID] + att, ok := b.tgwVpcAttachments.Get(attachmentID) if !ok { return nil, fmt.Errorf("%w: %s", ErrTGWAttachmentNotFound, attachmentID) } @@ -670,7 +666,7 @@ func (b *InMemoryBackend) ModifyTransitGatewayMeteringPolicy( b.mu.Lock("ModifyTransitGatewayMeteringPolicy") defer b.mu.Unlock() - p, ok := b.tgwMeteringPolicies[policyID] + p, ok := b.tgwMeteringPolicies.Get(policyID) if !ok { return nil, fmt.Errorf("%w: %s", ErrTGWMeteringPolicyNotFound, policyID) } @@ -698,13 +694,13 @@ func (b *InMemoryBackend) GetTransitGatewayMeteringPolicyEntries( b.mu.RLock("GetTransitGatewayMeteringPolicyEntries") defer b.mu.RUnlock() - if _, ok := b.tgwMeteringPolicies[policyID]; !ok { + if _, ok := b.tgwMeteringPolicies.Get(policyID); !ok { return nil, fmt.Errorf("%w: %s", ErrTGWMeteringPolicyNotFound, policyID) } out := make([]*TransitGatewayMeteringPolicyEntry, 0) - for _, e := range b.tgwMeteringPolicyEntries { + for _, e := range b.tgwMeteringPolicyEntries.All() { if e.TransitGatewayMeteringPolicyID != policyID { continue } @@ -734,7 +730,7 @@ func (b *InMemoryBackend) RejectTransitGatewayVpcAttachment( b.mu.Lock("RejectTransitGatewayVpcAttachment") defer b.mu.Unlock() - att, ok := b.tgwVpcAttachments[attachmentID] + att, ok := b.tgwVpcAttachments.Get(attachmentID) if !ok { return nil, fmt.Errorf("%w: %s", ErrTransitGatewayAttachmentNotFound, attachmentID) } @@ -757,7 +753,7 @@ func (b *InMemoryBackend) RejectTransitGatewayPeeringAttachment( b.mu.Lock("RejectTransitGatewayPeeringAttachment") defer b.mu.Unlock() - att, ok := b.tgwPeeringAttachments[attachmentID] + att, ok := b.tgwPeeringAttachments.Get(attachmentID) if !ok { return nil, fmt.Errorf("%w: %s", ErrTransitGatewayAttachmentNotFound, attachmentID) } @@ -794,9 +790,9 @@ func (b *InMemoryBackend) RejectTransitGatewayMulticastDomainAssociations( for _, subnetID := range subnetIDs { key := domainID + ":" + subnetID - existing, ok := b.tgwMulticastDomainAssociations[key] + existing, ok := b.tgwMulticastDomainAssociations.Get(key) if ok { - delete(b.tgwMulticastDomainAssociations, key) + b.tgwMulticastDomainAssociations.Delete(key) } else { existing = &TransitGatewayMulticastDomainAssociation{ TransitGatewayMulticastDomainID: domainID, diff --git a/services/ec2/backend_trunk_enclave.go b/services/ec2/backend_trunk_enclave.go index 7468e8c7c..0b18a1dc4 100644 --- a/services/ec2/backend_trunk_enclave.go +++ b/services/ec2/backend_trunk_enclave.go @@ -72,7 +72,6 @@ type EnclaveCertIamRoleAssociation struct { // resetTrunkEnclaveMapsLocked re-initialises all maps owned by this file. Must be called // with b.mu held. func (b *InMemoryBackend) resetTrunkEnclaveMapsLocked() { - b.trunkInterfaceAssociations = make(map[string]*TrunkInterfaceAssociation) b.enclaveCertIamRoles = make(map[string][]*EnclaveCertIamRoleAssociation) } @@ -102,11 +101,11 @@ func (b *InMemoryBackend) AssociateTrunkInterface( b.mu.Lock("AssociateTrunkInterface") defer b.mu.Unlock() - if _, ok := b.networkInterfaces[branchInterfaceID]; !ok { + if _, ok := b.networkInterfaces.Get(branchInterfaceID); !ok { return nil, fmt.Errorf("%w: %s", ErrNetworkInterfaceNotFound, branchInterfaceID) } - if _, ok := b.networkInterfaces[trunkInterfaceID]; !ok { + if _, ok := b.networkInterfaces.Get(trunkInterfaceID); !ok { return nil, fmt.Errorf("%w: %s", ErrNetworkInterfaceNotFound, trunkInterfaceID) } @@ -119,7 +118,7 @@ func (b *InMemoryBackend) AssociateTrunkInterface( GreKey: greKey, Tags: copyStringMap(tags), } - b.trunkInterfaceAssociations[assoc.AssociationID] = assoc + b.trunkInterfaceAssociations.Put(assoc) return copyTrunkInterfaceAssociation(assoc), nil } @@ -148,11 +147,10 @@ func (b *InMemoryBackend) DisassociateTrunkInterface(associationID string) error b.mu.Lock("DisassociateTrunkInterface") defer b.mu.Unlock() - if _, ok := b.trunkInterfaceAssociations[associationID]; !ok { + if _, ok := b.trunkInterfaceAssociations.Get(associationID); !ok { return fmt.Errorf("%w: %s", ErrTrunkAssociationNotFound, associationID) } - - delete(b.trunkInterfaceAssociations, associationID) + b.trunkInterfaceAssociations.Delete(associationID) return nil } @@ -168,9 +166,9 @@ func (b *InMemoryBackend) DescribeTrunkInterfaceAssociations(ids []string) []*Tr filter[id] = true } - out := make([]*TrunkInterfaceAssociation, 0, len(b.trunkInterfaceAssociations)) + out := make([]*TrunkInterfaceAssociation, 0, b.trunkInterfaceAssociations.Len()) - for _, assoc := range b.trunkInterfaceAssociations { + for _, assoc := range b.trunkInterfaceAssociations.All() { if len(filter) > 0 && !filter[assoc.AssociationID] { continue } diff --git a/services/ec2/backend_verifiedaccess_ext.go b/services/ec2/backend_verifiedaccess_ext.go index ef9a036a9..6fa848d0c 100644 --- a/services/ec2/backend_verifiedaccess_ext.go +++ b/services/ec2/backend_verifiedaccess_ext.go @@ -71,7 +71,7 @@ func (b *InMemoryBackend) GetVerifiedAccessEndpointPolicy(id string) (*VerifiedA b.mu.RLock("GetVerifiedAccessEndpointPolicy") defer b.mu.RUnlock() - if _, ok := b.verifiedAccessEndpoints[id]; !ok { + if _, ok := b.verifiedAccessEndpoints.Get(id); !ok { return nil, fmt.Errorf("%w: %s", ErrVerifiedAccessEndpointNotFound, id) } @@ -98,7 +98,7 @@ func (b *InMemoryBackend) ModifyVerifiedAccessEndpointPolicy( b.mu.Lock("ModifyVerifiedAccessEndpointPolicy") defer b.mu.Unlock() - if _, ok := b.verifiedAccessEndpoints[id]; !ok { + if _, ok := b.verifiedAccessEndpoints.Get(id); !ok { return nil, fmt.Errorf("%w: %s", ErrVerifiedAccessEndpointNotFound, id) } @@ -120,7 +120,7 @@ func (b *InMemoryBackend) GetVerifiedAccessGroupPolicy(id string) (*VerifiedAcce b.mu.RLock("GetVerifiedAccessGroupPolicy") defer b.mu.RUnlock() - if _, ok := b.verifiedAccessGroups[id]; !ok { + if _, ok := b.verifiedAccessGroups.Get(id); !ok { return nil, fmt.Errorf("%w: %s", ErrVerifiedAccessGroupNotFound, id) } @@ -147,7 +147,7 @@ func (b *InMemoryBackend) ModifyVerifiedAccessGroupPolicy( b.mu.Lock("ModifyVerifiedAccessGroupPolicy") defer b.mu.Unlock() - if _, ok := b.verifiedAccessGroups[id]; !ok { + if _, ok := b.verifiedAccessGroups.Get(id); !ok { return nil, fmt.Errorf("%w: %s", ErrVerifiedAccessGroupNotFound, id) } @@ -175,12 +175,13 @@ func (b *InMemoryBackend) DescribeVerifiedAccessInstanceLoggingConfigurations( var out []*VerifiedAccessInstanceLoggingConfig - for instanceID := range b.verifiedAccessInstances { + for _, vai := range b.verifiedAccessInstances.All() { + instanceID := verifiedAccessInstancesKeyFn(vai) if len(filter) > 0 && !filter[instanceID] { continue } - if cfg, ok := b.verifiedAccessInstanceLoggingConfigs[instanceID]; ok { + if cfg, ok := b.verifiedAccessInstanceLoggingConfigs.Get(instanceID); ok { cp := *cfg out = append(out, &cp) @@ -207,7 +208,7 @@ func (b *InMemoryBackend) ModifyVerifiedAccessInstanceLoggingConfiguration( b.mu.Lock("ModifyVerifiedAccessInstanceLoggingConfiguration") defer b.mu.Unlock() - if _, ok := b.verifiedAccessInstances[instanceID]; !ok { + if _, ok := b.verifiedAccessInstances.Get(instanceID); !ok { return nil, fmt.Errorf("%w: %s", ErrVerifiedAccessInstanceNotFound, instanceID) } @@ -215,7 +216,7 @@ func (b *InMemoryBackend) ModifyVerifiedAccessInstanceLoggingConfiguration( VerifiedAccessInstanceID: instanceID, AccessLogs: accessLogs, } - b.verifiedAccessInstanceLoggingConfigs[instanceID] = cfg + b.verifiedAccessInstanceLoggingConfigs.Put(cfg) cp := *cfg return &cp, nil @@ -233,7 +234,7 @@ func (b *InMemoryBackend) GetVerifiedAccessEndpointTargets(id string) ([]*Verifi b.mu.RLock("GetVerifiedAccessEndpointTargets") defer b.mu.RUnlock() - if _, ok := b.verifiedAccessEndpoints[id]; !ok { + if _, ok := b.verifiedAccessEndpoints.Get(id); !ok { return nil, fmt.Errorf("%w: %s", ErrVerifiedAccessEndpointNotFound, id) } @@ -264,7 +265,7 @@ func (b *InMemoryBackend) ExportVerifiedAccessInstanceClientConfiguration( b.mu.RLock("ExportVerifiedAccessInstanceClientConfiguration") defer b.mu.RUnlock() - if _, ok := b.verifiedAccessInstances[instanceID]; !ok { + if _, ok := b.verifiedAccessInstances.Get(instanceID); !ok { return nil, fmt.Errorf("%w: %s", ErrVerifiedAccessInstanceNotFound, instanceID) } diff --git a/services/ec2/backend_vm_import_export.go b/services/ec2/backend_vm_import_export.go index dc3419f47..457b700ea 100644 --- a/services/ec2/backend_vm_import_export.go +++ b/services/ec2/backend_vm_import_export.go @@ -126,10 +126,6 @@ type ExportImageTaskRec struct { // resetVMImportExportMapsLocked re-initialises all maps owned by this file. Must be called // with b.mu held. func (b *InMemoryBackend) resetVMImportExportMapsLocked() { - b.bundleTasks = make(map[string]*BundleTask) - b.conversionTasks = make(map[string]*ConversionTask) - b.exportTasks = make(map[string]*ExportTask) - b.exportImageTasks = make(map[string]*ExportImageTaskRec) } // ---- Bundle tasks ---- @@ -147,7 +143,7 @@ func (b *InMemoryBackend) BundleInstance(instanceID, s3Bucket, s3Prefix string) b.mu.Lock("BundleInstance") defer b.mu.Unlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -162,7 +158,7 @@ func (b *InMemoryBackend) BundleInstance(instanceID, s3Bucket, s3Prefix string) StartTime: now, UpdateTime: now, } - b.bundleTasks[task.BundleID] = task + b.bundleTasks.Put(task) cp := *task @@ -178,7 +174,7 @@ func (b *InMemoryBackend) CancelBundleTask(bundleID string) (*BundleTask, error) b.mu.Lock("CancelBundleTask") defer b.mu.Unlock() - task, ok := b.bundleTasks[bundleID] + task, ok := b.bundleTasks.Get(bundleID) if !ok { return nil, fmt.Errorf("%w: %s", ErrBundleTaskNotFound, bundleID) } @@ -208,9 +204,9 @@ func (b *InMemoryBackend) DescribeBundleTasks(ids []string) []*BundleTask { filter[id] = true } - out := make([]*BundleTask, 0, len(b.bundleTasks)) + out := make([]*BundleTask, 0, b.bundleTasks.Len()) - for _, t := range b.bundleTasks { + for _, t := range b.bundleTasks.All() { if len(filter) > 0 && !filter[t.BundleID] { continue } @@ -271,7 +267,7 @@ func (b *InMemoryBackend) ImportInstance( ImageBytes: diskBytes, ExpirationTime: now.Add(conversionTaskExpiryWindow), } - b.conversionTasks[task.ConversionTaskID] = task + b.conversionTasks.Put(task) cp := *task @@ -307,7 +303,7 @@ func (b *InMemoryBackend) ImportVolume( ImageBytes: diskBytes, ExpirationTime: now.Add(conversionTaskExpiryWindow), } - b.conversionTasks[task.ConversionTaskID] = task + b.conversionTasks.Put(task) cp := *task @@ -326,9 +322,9 @@ func (b *InMemoryBackend) DescribeConversionTasks(ids []string) []*ConversionTas filter[id] = true } - out := make([]*ConversionTask, 0, len(b.conversionTasks)) + out := make([]*ConversionTask, 0, b.conversionTasks.Len()) - for _, t := range b.conversionTasks { + for _, t := range b.conversionTasks.All() { if len(filter) > 0 && !filter[t.ConversionTaskID] { continue } @@ -352,7 +348,7 @@ func (b *InMemoryBackend) CancelConversionTask(conversionTaskID string) (*Conver b.mu.Lock("CancelConversionTask") defer b.mu.Unlock() - task, ok := b.conversionTasks[conversionTaskID] + task, ok := b.conversionTasks.Get(conversionTaskID) if !ok { return nil, fmt.Errorf("%w: %s", ErrConversionTaskNotFound, conversionTaskID) } @@ -397,7 +393,7 @@ func (b *InMemoryBackend) CreateInstanceExportTask( b.mu.Lock("CreateInstanceExportTask") defer b.mu.Unlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -418,7 +414,7 @@ func (b *InMemoryBackend) CreateInstanceExportTask( S3Bucket: s3Bucket, S3Key: s3Prefix + id + "." + strings.ToLower(diskImageFormat), } - b.exportTasks[id] = task + b.exportTasks.Put(task) cp := *task @@ -435,7 +431,7 @@ func (b *InMemoryBackend) CancelExportTask(exportTaskID string) error { b.mu.Lock("CancelExportTask") defer b.mu.Unlock() - task, ok := b.exportTasks[exportTaskID] + task, ok := b.exportTasks.Get(exportTaskID) if !ok { return fmt.Errorf("%w: %s", ErrExportTaskNotFound, exportTaskID) } @@ -462,9 +458,9 @@ func (b *InMemoryBackend) DescribeExportTasks(ids []string) []*ExportTask { filter[id] = true } - out := make([]*ExportTask, 0, len(b.exportTasks)) + out := make([]*ExportTask, 0, b.exportTasks.Len()) - for _, t := range b.exportTasks { + for _, t := range b.exportTasks.All() { if len(filter) > 0 && !filter[t.ExportTaskID] { continue } @@ -505,9 +501,9 @@ func (b *InMemoryBackend) DescribeExportImageTasks(ids []string) []*ExportImageT filter[id] = true } - out := make([]*ExportImageTaskRec, 0, len(b.exportImageTasks)) + out := make([]*ExportImageTaskRec, 0, b.exportImageTasks.Len()) - for _, t := range b.exportImageTasks { + for _, t := range b.exportImageTasks.All() { if len(filter) > 0 && !filter[t.ExportImageTaskID] { continue } @@ -540,11 +536,11 @@ func (b *InMemoryBackend) CancelImportTask(importTaskID string) (string, string, b.mu.Lock("CancelImportTask") defer b.mu.Unlock() - if task, ok := b.imageImportTasks[importTaskID]; ok { + if task, ok := b.imageImportTasks.Get(importTaskID); ok { return cancelImportTaskStatus(importTaskID, &task.Status) } - if task, ok := b.snapshotImportTasks[importTaskID]; ok { + if task, ok := b.snapshotImportTasks.Get(importTaskID); ok { return cancelImportTaskStatus(importTaskID, &task.Status) } diff --git a/services/ec2/backend_vpc_config.go b/services/ec2/backend_vpc_config.go index 5df7178b8..bdfa03b32 100644 --- a/services/ec2/backend_vpc_config.go +++ b/services/ec2/backend_vpc_config.go @@ -97,11 +97,11 @@ func (b *InMemoryBackend) AttachClassicLinkVpc(instanceID, vpcID string, groups b.mu.Lock("AttachClassicLinkVpc") defer b.mu.Unlock() - if _, ok := b.instances[instanceID]; !ok { + if _, ok := b.instances.Get(instanceID); !ok { return fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } - vpc, ok := b.vpcs[vpcID] + vpc, ok := b.vpcs.Get(vpcID) if !ok { return fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -111,16 +111,15 @@ func (b *InMemoryBackend) AttachClassicLinkVpc(instanceID, vpcID string, groups } for _, groupID := range groups { - if _, sgOK := b.securityGroups[groupID]; !sgOK { + if _, sgOK := b.securityGroups.Get(groupID); !sgOK { return fmt.Errorf("%w: %s", ErrSecurityGroupNotFound, groupID) } } - - b.classicLinkInstances[instanceID] = &ClassicLinkInstance{ + b.classicLinkInstances.Put(&ClassicLinkInstance{ InstanceID: instanceID, VpcID: vpcID, Groups: append([]string(nil), groups...), - } + }) return nil } @@ -138,12 +137,11 @@ func (b *InMemoryBackend) DetachClassicLinkVpc(instanceID, vpcID string) error { b.mu.Lock("DetachClassicLinkVpc") defer b.mu.Unlock() - link, ok := b.classicLinkInstances[instanceID] + link, ok := b.classicLinkInstances.Get(instanceID) if !ok || link.VpcID != vpcID { return fmt.Errorf("%w: %s", ErrClassicLinkInstanceNotFound, instanceID) } - - delete(b.classicLinkInstances, instanceID) + b.classicLinkInstances.Delete(instanceID) return nil } @@ -158,9 +156,9 @@ func (b *InMemoryBackend) DescribeClassicLinkInstances(instanceIDs []string) []* idSet[id] = true } - out := make([]*ClassicLinkInstance, 0, len(b.classicLinkInstances)) + out := make([]*ClassicLinkInstance, 0, b.classicLinkInstances.Len()) - for _, link := range b.classicLinkInstances { + for _, link := range b.classicLinkInstances.All() { if len(idSet) > 0 && !idSet[link.InstanceID] { continue } @@ -184,7 +182,7 @@ func (b *InMemoryBackend) EnableVpcClassicLink(vpcID string) error { b.mu.Lock("EnableVpcClassicLink") defer b.mu.Unlock() - vpc, ok := b.vpcs[vpcID] + vpc, ok := b.vpcs.Get(vpcID) if !ok { return fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -204,12 +202,12 @@ func (b *InMemoryBackend) DisableVpcClassicLink(vpcID string) error { b.mu.Lock("DisableVpcClassicLink") defer b.mu.Unlock() - vpc, ok := b.vpcs[vpcID] + vpc, ok := b.vpcs.Get(vpcID) if !ok { return fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } - for _, link := range b.classicLinkInstances { + for _, link := range b.classicLinkInstances.All() { if link.VpcID == vpcID { return fmt.Errorf("%w: %s has linked EC2-Classic instances", ErrDependencyViolation, vpcID) } @@ -224,8 +222,8 @@ func (b *InMemoryBackend) DisableVpcClassicLink(vpcID string) error { // ids is empty). Must be called with b.mu held for reading. func (b *InMemoryBackend) describeVpcsByIDsLocked(ids []string) ([]*VPC, error) { if len(ids) == 0 { - out := make([]*VPC, 0, len(b.vpcs)) - for _, vpc := range b.vpcs { + out := make([]*VPC, 0, b.vpcs.Len()) + for _, vpc := range b.vpcs.All() { cp := *vpc out = append(out, &cp) } @@ -238,7 +236,7 @@ func (b *InMemoryBackend) describeVpcsByIDsLocked(ids []string) ([]*VPC, error) out := make([]*VPC, 0, len(ids)) for _, id := range ids { - vpc, ok := b.vpcs[id] + vpc, ok := b.vpcs.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, id) } @@ -267,7 +265,7 @@ func (b *InMemoryBackend) EnableVpcClassicLinkDNSSupport(vpcID string) error { b.mu.Lock("EnableVpcClassicLinkDnsSupport") defer b.mu.Unlock() - vpc, ok := b.vpcs[vpcID] + vpc, ok := b.vpcs.Get(vpcID) if !ok { return fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -286,7 +284,7 @@ func (b *InMemoryBackend) DisableVpcClassicLinkDNSSupport(vpcID string) error { b.mu.Lock("DisableVpcClassicLinkDnsSupport") defer b.mu.Unlock() - vpc, ok := b.vpcs[vpcID] + vpc, ok := b.vpcs.Get(vpcID) if !ok { return fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } @@ -357,13 +355,13 @@ func (b *InMemoryBackend) CreateVpcBlockPublicAccessExclusion( defer b.mu.Unlock() if vpcID != "" { - if _, ok := b.vpcs[vpcID]; !ok { + if _, ok := b.vpcs.Get(vpcID); !ok { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } } if subnetID != "" { - if _, ok := b.subnets[subnetID]; !ok { + if _, ok := b.subnets.Get(subnetID); !ok { return nil, fmt.Errorf("%w: %s", ErrSubnetNotFound, subnetID) } } @@ -383,7 +381,7 @@ func (b *InMemoryBackend) CreateVpcBlockPublicAccessExclusion( LastUpdateTimestamp: now, Tags: maps.Clone(tags), } - b.vpcBlockPublicAccessExclusions[id] = excl + b.vpcBlockPublicAccessExclusions.Put(excl) return cloneVpcBPAExclusion(excl), nil } @@ -403,7 +401,7 @@ func (b *InMemoryBackend) ModifyVpcBlockPublicAccessExclusion( b.mu.Lock("ModifyVpcBlockPublicAccessExclusion") defer b.mu.Unlock() - excl, ok := b.vpcBlockPublicAccessExclusions[exclusionID] + excl, ok := b.vpcBlockPublicAccessExclusions.Get(exclusionID) if !ok { return nil, fmt.Errorf("%w: %s", ErrVpcBlockPublicAccessExclusionNotFound, exclusionID) } @@ -426,7 +424,7 @@ func (b *InMemoryBackend) DeleteVpcBlockPublicAccessExclusion( b.mu.Lock("DeleteVpcBlockPublicAccessExclusion") defer b.mu.Unlock() - excl, ok := b.vpcBlockPublicAccessExclusions[exclusionID] + excl, ok := b.vpcBlockPublicAccessExclusions.Get(exclusionID) if !ok { return nil, fmt.Errorf("%w: %s", ErrVpcBlockPublicAccessExclusionNotFound, exclusionID) } @@ -434,8 +432,7 @@ func (b *InMemoryBackend) DeleteVpcBlockPublicAccessExclusion( excl.State = vpcBPAExclusionDeleteComplete excl.LastUpdateTimestamp = time.Now().UTC() out := cloneVpcBPAExclusion(excl) - - delete(b.vpcBlockPublicAccessExclusions, exclusionID) + b.vpcBlockPublicAccessExclusions.Delete(exclusionID) return out, nil } @@ -450,9 +447,9 @@ func (b *InMemoryBackend) DescribeVpcBlockPublicAccessExclusions(ids []string) [ idSet[id] = true } - out := make([]*VpcBlockPublicAccessExclusion, 0, len(b.vpcBlockPublicAccessExclusions)) + out := make([]*VpcBlockPublicAccessExclusion, 0, b.vpcBlockPublicAccessExclusions.Len()) - for _, excl := range b.vpcBlockPublicAccessExclusions { + for _, excl := range b.vpcBlockPublicAccessExclusions.All() { if len(idSet) > 0 && !idSet[excl.ExclusionID] { continue } diff --git a/services/ec2/backend_vpc_encryption_control.go b/services/ec2/backend_vpc_encryption_control.go index d79a3ba23..a50cbc5c4 100644 --- a/services/ec2/backend_vpc_encryption_control.go +++ b/services/ec2/backend_vpc_encryption_control.go @@ -156,11 +156,11 @@ func (b *InMemoryBackend) CreateVpcEncryptionControl( b.mu.Lock("CreateVpcEncryptionControl") defer b.mu.Unlock() - if _, ok := b.vpcs[vpcID]; !ok { + if _, ok := b.vpcs.Get(vpcID); !ok { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } - for _, existing := range b.vpcEncryptionControls { + for _, existing := range b.vpcEncryptionControls.All() { if existing.VpcID == vpcID { return nil, fmt.Errorf( "%w: a VPC Encryption Control configuration already exists for %s", @@ -187,7 +187,7 @@ func (b *InMemoryBackend) CreateVpcEncryptionControl( }, Tags: maps.Clone(tags), } - b.vpcEncryptionControls[vec.VpcEncryptionControlID] = vec + b.vpcEncryptionControls.Put(vec) return cloneVpcEncryptionControl(vec), nil } @@ -201,15 +201,14 @@ func (b *InMemoryBackend) DeleteVpcEncryptionControl(id string) (*VpcEncryptionC b.mu.Lock("DeleteVpcEncryptionControl") defer b.mu.Unlock() - vec, ok := b.vpcEncryptionControls[id] + vec, ok := b.vpcEncryptionControls.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrVpcEncryptionControlNotFound, id) } vec.State = vpcEncryptionControlStateDeleted out := cloneVpcEncryptionControl(vec) - - delete(b.vpcEncryptionControls, id) + b.vpcEncryptionControls.Delete(id) return out, nil } @@ -230,9 +229,9 @@ func (b *InMemoryBackend) DescribeVpcEncryptionControls(ids, vpcIDs []string) [] vpcIDSet[id] = true } - out := make([]*VpcEncryptionControl, 0, len(b.vpcEncryptionControls)) + out := make([]*VpcEncryptionControl, 0, b.vpcEncryptionControls.Len()) - for _, vec := range b.vpcEncryptionControls { + for _, vec := range b.vpcEncryptionControls.All() { if len(idSet) > 0 && !idSet[vec.VpcEncryptionControlID] { continue } @@ -266,7 +265,7 @@ func (b *InMemoryBackend) ModifyVpcEncryptionControl( b.mu.Lock("ModifyVpcEncryptionControl") defer b.mu.Unlock() - vec, ok := b.vpcEncryptionControls[id] + vec, ok := b.vpcEncryptionControls.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrVpcEncryptionControlNotFound, id) } @@ -295,7 +294,7 @@ func (b *InMemoryBackend) GetVpcResourcesBlockingEncryptionEnforcement( b.mu.RLock("GetVpcResourcesBlockingEncryptionEnforcement") defer b.mu.RUnlock() - if _, ok := b.vpcs[vpcID]; !ok { + if _, ok := b.vpcs.Get(vpcID); !ok { return nil, fmt.Errorf("%w: %s", ErrVPCNotFound, vpcID) } diff --git a/services/ec2/backend_vpn_concentrator.go b/services/ec2/backend_vpn_concentrator.go index f47959da5..d5d19855d 100644 --- a/services/ec2/backend_vpn_concentrator.go +++ b/services/ec2/backend_vpn_concentrator.go @@ -66,7 +66,7 @@ func (b *InMemoryBackend) CreateVpnConcentrator( var attachmentID string if transitGatewayID != "" { - if _, ok := b.transitGateways[transitGatewayID]; !ok { + if _, ok := b.transitGateways.Get(transitGatewayID); !ok { return nil, fmt.Errorf("%w: %s", ErrTransitGatewayNotFound, transitGatewayID) } @@ -81,7 +81,7 @@ func (b *InMemoryBackend) CreateVpnConcentrator( TransitGatewayAttachmentID: attachmentID, Tags: maps.Clone(tags), } - b.vpnConcentrators[vc.VpnConcentratorID] = vc + b.vpnConcentrators.Put(vc) return cloneVpnConcentrator(vc), nil } @@ -95,15 +95,14 @@ func (b *InMemoryBackend) DeleteVpnConcentrator(id string) (*VpnConcentrator, er b.mu.Lock("DeleteVpnConcentrator") defer b.mu.Unlock() - vc, ok := b.vpnConcentrators[id] + vc, ok := b.vpnConcentrators.Get(id) if !ok { return nil, fmt.Errorf("%w: %s", ErrVpnConcentratorNotFound, id) } vc.State = vpnConcentratorStateDeleted out := cloneVpnConcentrator(vc) - - delete(b.vpnConcentrators, id) + b.vpnConcentrators.Delete(id) return out, nil } @@ -118,9 +117,9 @@ func (b *InMemoryBackend) DescribeVpnConcentrators(ids []string) []*VpnConcentra idSet[id] = true } - out := make([]*VpnConcentrator, 0, len(b.vpnConcentrators)) + out := make([]*VpnConcentrator, 0, b.vpnConcentrators.Len()) - for _, vc := range b.vpnConcentrators { + for _, vc := range b.vpnConcentrators.All() { if len(idSet) > 0 && !idSet[vc.VpnConcentratorID] { continue } @@ -150,7 +149,7 @@ func (b *InMemoryBackend) GetActiveVpnTunnelStatus( b.mu.RLock("GetActiveVpnTunnelStatus") defer b.mu.RUnlock() - conn, ok := b.vpnConnections[vpnConnectionID] + conn, ok := b.vpnConnections.Get(vpnConnectionID) if !ok { return nil, fmt.Errorf("%w: %s", ErrVpnConnectionNotFound, vpnConnectionID) } @@ -208,7 +207,7 @@ func (b *InMemoryBackend) ReplaceVpnTunnel(vpnConnectionID, outsideIPAddress str b.mu.Lock("ReplaceVpnTunnel") defer b.mu.Unlock() - conn, ok := b.vpnConnections[vpnConnectionID] + conn, ok := b.vpnConnections.Get(vpnConnectionID) if !ok { return false, fmt.Errorf("%w: %s", ErrVpnConnectionNotFound, vpnConnectionID) } diff --git a/services/ec2/export_test.go b/services/ec2/export_test.go index 3b9c7c2da..6172b8703 100644 --- a/services/ec2/export_test.go +++ b/services/ec2/export_test.go @@ -64,7 +64,7 @@ func (b *InMemoryBackend) SetInstanceTerminatedAtForTest(id string, t time.Time) b.mu.Lock("SetInstanceTerminatedAtForTest") defer b.mu.Unlock() - if inst, ok := b.instances[id]; ok { + if inst, ok := b.instances.Get(id); ok { inst.TerminatedAt = t } } @@ -74,7 +74,7 @@ func (b *InMemoryBackend) SetSpotRequestCancelledAtForTest(id string, t time.Tim b.mu.Lock("SetSpotRequestCancelledAtForTest") defer b.mu.Unlock() - if req, ok := b.spotRequests[id]; ok { + if req, ok := b.spotRequests.Get(id); ok { req.CancelledAt = t } } @@ -86,8 +86,7 @@ func (b *InMemoryBackend) SetSpotRequestCancelledAtForTest(id string, t time.Tim func (b *InMemoryBackend) InjectOrphanedENIForTest(eni *NetworkInterface) { b.mu.Lock("InjectOrphanedENIForTest") defer b.mu.Unlock() - - b.networkInterfaces[eni.ID] = eni + b.networkInterfaces.Put(eni) } // ---- count helpers for new resource types ---- @@ -105,7 +104,7 @@ func (b *InMemoryBackend) CapacityReservationCount() int { b.mu.RLock("CapacityReservationCount") defer b.mu.RUnlock() - return len(b.capacityReservations) + return b.capacityReservations.Len() } // ReservedInstancesExchangeCount returns the number of reserved instances exchanges. @@ -113,7 +112,7 @@ func (b *InMemoryBackend) ReservedInstancesExchangeCount() int { b.mu.RLock("ReservedInstancesExchangeCount") defer b.mu.RUnlock() - return len(b.reservedInstancesExchanges) + return b.reservedInstancesExchanges.Len() } // TGWMulticastDomainAssociationCount returns the number of TGW multicast domain associations. @@ -121,7 +120,7 @@ func (b *InMemoryBackend) TGWMulticastDomainAssociationCount() int { b.mu.RLock("TGWMulticastDomainAssociationCount") defer b.mu.RUnlock() - return len(b.tgwMulticastDomainAssociations) + return b.tgwMulticastDomainAssociations.Len() } // TGWPeeringAttachmentCount returns the number of TGW peering attachments. @@ -129,7 +128,7 @@ func (b *InMemoryBackend) TGWPeeringAttachmentCount() int { b.mu.RLock("TGWPeeringAttachmentCount") defer b.mu.RUnlock() - return len(b.tgwPeeringAttachments) + return b.tgwPeeringAttachments.Len() } // TGWVpcAttachmentCount returns the number of TGW VPC attachments. @@ -137,7 +136,7 @@ func (b *InMemoryBackend) TGWVpcAttachmentCount() int { b.mu.RLock("TGWVpcAttachmentCount") defer b.mu.RUnlock() - return len(b.tgwVpcAttachments) + return b.tgwVpcAttachments.Len() } // VpcEndpointConnectionCount returns the number of VPC endpoint connections. @@ -145,7 +144,7 @@ func (b *InMemoryBackend) VpcEndpointConnectionCount() int { b.mu.RLock("VpcEndpointConnectionCount") defer b.mu.RUnlock() - return len(b.vpcEndpointConnections) + return b.vpcEndpointConnections.Len() } // VpcPeeringConnectionCount returns the number of VPC peering connections. @@ -153,7 +152,7 @@ func (b *InMemoryBackend) VpcPeeringConnectionCount() int { b.mu.RLock("VpcPeeringConnectionCount") defer b.mu.RUnlock() - return len(b.vpcPeeringConnections) + return b.vpcPeeringConnections.Len() } // ByoipCidrCount returns the number of BYOIP CIDRs in the backend. @@ -161,7 +160,7 @@ func (b *InMemoryBackend) ByoipCidrCount() int { b.mu.RLock("ByoipCidrCount") defer b.mu.RUnlock() - return len(b.byoipCidrs) + return b.byoipCidrs.Len() } // SeedReservedInstancesOffering inserts a reserved instances offering directly (for tests). @@ -172,8 +171,7 @@ func (b *InMemoryBackend) SeedReservedInstancesOffering( ) { b.mu.Lock("SeedReservedInstancesOffering") defer b.mu.Unlock() - - b.reservedInstancesOfferings[offeringID] = &ReservedInstancesOffering{ + b.reservedInstancesOfferings.Put(&ReservedInstancesOffering{ ReservedInstancesOfferingID: offeringID, InstanceType: instanceType, AvailabilityZone: az, @@ -182,7 +180,7 @@ func (b *InMemoryBackend) SeedReservedInstancesOffering( Duration: duration, FixedPrice: fixedPrice, UsagePrice: usagePrice, - } + }) } // DedicatedHostCount returns the number of dedicated hosts in the backend. @@ -190,7 +188,7 @@ func (b *InMemoryBackend) DedicatedHostCount() int { b.mu.RLock("DedicatedHostCount") defer b.mu.RUnlock() - return len(b.dedicatedHosts) + return b.dedicatedHosts.Len() } // HandlerOpsLen returns the number of operations registered in the handler's dispatch table. @@ -203,8 +201,7 @@ func (h *Handler) HandlerOpsLen() int { func (b *InMemoryBackend) AddImageForTest(img AMIStub) { b.mu.Lock("AddImageForTest") defer b.mu.Unlock() - - b.images[img.ImageID] = &img + b.images.Put(&img) } // ExportDispatch calls the handler's dispatch method and returns the XML response as a string. diff --git a/services/ec2/handler.go b/services/ec2/handler.go index 4a36617d9..012c663b9 100644 --- a/services/ec2/handler.go +++ b/services/ec2/handler.go @@ -613,12 +613,59 @@ func (h *Handler) applyInstanceLaunchSettings( return nil } +// applyInstanceLaunchAttributes wires the RunInstances top-level +// DisableApiTermination / InstanceInitiatedShutdownBehavior / EbsOptimized +// parameters (distinct from the post-launch ModifyInstanceAttribute path) +// onto newly-created instances. +func (h *Handler) applyInstanceLaunchAttributes( + instances []*Instance, + disableAPITermination, shutdownBehavior, ebsOptimized string, +) error { + type launchAttr struct { + name, value string + } + + // maxLaunchAttrs is the number of RunInstances attribute params handled + // below (DisableApiTermination, InstanceInitiatedShutdownBehavior, EbsOptimized). + const maxLaunchAttrs = 3 + + attrs := make([]launchAttr, 0, maxLaunchAttrs) + if disableAPITermination != "" { + attrs = append(attrs, launchAttr{attrDisableAPITermination, disableAPITermination}) + } + + if shutdownBehavior != "" { + attrs = append(attrs, launchAttr{attrInstanceInitiatedShutdownBehavior, shutdownBehavior}) + } + + if ebsOptimized != "" { + attrs = append(attrs, launchAttr{attrEBSOptimized, ebsOptimized}) + } + + if len(attrs) == 0 { + return nil + } + + for _, inst := range instances { + for _, a := range attrs { + if err := h.Backend.SetInstanceAttribute(inst.ID, a.name, a.value); err != nil { + return err + } + } + } + + return nil +} + func (h *Handler) handleRunInstances(vals url.Values, reqID string) (any, error) { imageID := vals.Get("ImageId") instanceType := vals.Get("InstanceType") subnetID := vals.Get("SubnetId") userData := vals.Get("UserData") keyName := vals.Get("KeyName") + disableAPITermination := vals.Get("DisableApiTermination") + shutdownBehavior := vals.Get("InstanceInitiatedShutdownBehavior") + ebsOptimized := vals.Get("EbsOptimized") if err := validateUserData(userData); err != nil { return nil, err @@ -645,6 +692,12 @@ func (h *Handler) handleRunInstances(vals url.Values, reqID string) (any, error) return nil, err } + if err = h.applyInstanceLaunchAttributes( + instances, disableAPITermination, shutdownBehavior, ebsOptimized, + ); err != nil { + return nil, err + } + if cb, c := h.computeBackend(); c != nil { h.launchOnCompute(h.svcCtx, cb, c, instances, keyName, userData) } @@ -1280,6 +1333,16 @@ func (h *Handler) handleDeleteTags(vals url.Values, reqID string) (any, error) { }, nil } +// boolToEC2Attr renders a Go bool as the "true"/"false" string EC2 query-protocol +// attribute values use. +func boolToEC2Attr(v bool) string { + if v { + return ec2BooleanTrue + } + + return ec2BooleanFalse +} + // handleDescribeInstanceAttribute returns the current value for the requested instance attribute. // Terraform calls this after RunInstances to read instanceInitiatedShutdownBehavior. func (h *Handler) handleDescribeInstanceAttribute(vals url.Values, reqID string) (any, error) { @@ -1296,44 +1359,58 @@ func (h *Handler) handleDescribeInstanceAttribute(vals url.Values, reqID string) } inst := instances[0] + attrValue := h.instanceAttributeValue(inst, instanceID, attr) - // Build the attribute value from stored instance state when possible; - // fall back to AWS defaults for unmodelled attributes. - var attrValue string + return &describeInstanceAttributeResponse{ + Xmlns: ec2XMLNS, + RequestID: reqID, + InstanceID: instanceID, + Attribute: namedStringAttr{XMLName: xml.Name{Local: attr}, Value: attrValue}, + }, nil +} +// instanceAttributeValue builds the DescribeInstanceAttribute string value +// from stored instance state when possible, falling back to AWS defaults for +// unmodelled attributes. Split out of handleDescribeInstanceAttribute to keep +// cyclomatic complexity down. +func (h *Handler) instanceAttributeValue(inst *Instance, instanceID, attr string) string { switch attr { case attrUserData: - attrValue = inst.UserData + return inst.UserData case attrInstanceType: - attrValue = inst.InstanceType + return inst.InstanceType case attrEnaSupport: - if inst.EnaSupport { - attrValue = ec2BooleanTrue - } else { - attrValue = ec2BooleanFalse - } + return boolToEC2Attr(inst.EnaSupport) case attrSriovNetSupport: if inst.SriovNetSupport != "" { - attrValue = inst.SriovNetSupport - } else { - attrValue = "simple" + return inst.SriovNetSupport } - case attrDisableAPIStop, attrDisableAPITermination, attrEBSOptimized: - attrValue = ec2BooleanFalse + + return "simple" + case attrDisableAPIStop: + return boolToEC2Attr(inst.DisableAPIStop) + case attrDisableAPITermination: + return boolToEC2Attr(inst.DisableAPITermination) + case attrEBSOptimized: + return boolToEC2Attr(inst.EBSOptimized) case attrSourceDest: - attrValue = ec2BooleanFalse - case attrInstanceInitiatedShutdownBehavior, attrKernel, attrRamdisk: - attrValue = "stop" + // sourceDestCheck lives on the primary ENI attachment; AWS defaults + // it to true for VPC instances. + return boolToEC2Attr(h.Backend.PrimaryNetworkInterfaceSourceDestCheck(instanceID)) + case attrInstanceInitiatedShutdownBehavior: + if inst.InstanceInitiatedShutdownBehavior != "" { + return inst.InstanceInitiatedShutdownBehavior + } + + return "stop" + case attrKernel, attrRamdisk: + // Modern (HVM) instances have no kernel/ramdisk image; AWS returns an + // empty value rather than "stop" (that default only applies to + // instanceInitiatedShutdownBehavior). + return "" default: - attrValue = "" + return "" } - - return &describeInstanceAttributeResponse{ - Xmlns: ec2XMLNS, - RequestID: reqID, - InstanceID: instanceID, - Attribute: namedStringAttr{XMLName: xml.Name{Local: attr}, Value: attrValue}, - }, nil } // ---- error handling ---- @@ -1437,6 +1514,7 @@ var errCodeLookup = []struct { {ErrInvalidUserData, "InvalidUserData.Malformed"}, {ErrMissingParameter, "MissingParameter"}, {ErrInvalidPaginationToken, "InvalidPaginationToken"}, + {ErrOperationNotPermitted, "OperationNotPermitted"}, } // opErrCode resolves an error to its EC2 API error code and HTTP status code. @@ -1600,19 +1678,20 @@ func toInstanceItem(inst *Instance, instanceTags map[string]string) instanceItem } item := instanceItem{ - InstanceID: inst.ID, - ImageID: inst.ImageID, - InstanceType: inst.InstanceType, - StateItem: stateItem{Code: inst.State.Code, Name: inst.State.Name}, - VPCID: inst.VPCID, - SubnetID: inst.SubnetID, - LaunchTime: inst.LaunchTime.Format("2006-01-02T15:04:05.000Z"), - PrivateIPAddress: inst.PrivateIP, - PublicIPAddress: inst.PublicIPAddress, - PublicDNSName: inst.PublicDNSName, - KeyName: inst.KeyName, - GroupSet: instanceGroupSet{Items: groupItems}, - TagSet: instanceTagItemSet{Items: tagItems}, + InstanceID: inst.ID, + ImageID: inst.ImageID, + InstanceType: inst.InstanceType, + StateItem: stateItem{Code: inst.State.Code, Name: inst.State.Name}, + StateTransitionReason: inst.StateTransitionReason, + VPCID: inst.VPCID, + SubnetID: inst.SubnetID, + LaunchTime: inst.LaunchTime.Format("2006-01-02T15:04:05.000Z"), + PrivateIPAddress: inst.PrivateIP, + PublicIPAddress: inst.PublicIPAddress, + PublicDNSName: inst.PublicDNSName, + KeyName: inst.KeyName, + GroupSet: instanceGroupSet{Items: groupItems}, + TagSet: instanceTagItemSet{Items: tagItems}, Placement: instancePlacementItem{ Tenancy: inst.Placement.Tenancy, AvailabilityZone: inst.Placement.AvailabilityZone, @@ -1621,6 +1700,13 @@ func toInstanceItem(inst *Instance, instanceTags map[string]string) instanceItem }, } + if inst.StateReasonCode != "" { + item.StateReasonItem = &stateReasonItem{ + Code: inst.StateReasonCode, + Message: inst.StateReasonMessage, + } + } + if inst.CPUOptions.CoreCount > 0 || inst.CPUOptions.ThreadsPerCore > 0 { item.CPUOptions = &instanceCPUOptionsItem{ CoreCount: inst.CPUOptions.CoreCount, @@ -1712,6 +1798,13 @@ type instancePlacementItem struct { Affinity string `xml:"affinity,omitempty"` } +// stateReasonItem is the element carrying the structured +// code/message for an instance's most recent state transition. +type stateReasonItem struct { + Code string `xml:"code,omitempty"` + Message string `xml:"message,omitempty"` +} + type instanceCPUOptionsItem struct { CoreCount int32 `xml:"coreCount"` ThreadsPerCore int32 `xml:"threadsPerCore"` @@ -1729,6 +1822,7 @@ type instanceItem struct { NetworkPerformanceOptions *instanceNetworkPerformanceOptionsItem `xml:"networkPerformanceOptions,omitempty"` MaintenanceOptions *instanceMaintenanceOptionsItem `xml:"maintenanceOptions,omitempty"` CPUOptions *instanceCPUOptionsItem `xml:"cpuOptions,omitempty"` + StateReasonItem *stateReasonItem `xml:"stateReason,omitempty"` Placement instancePlacementItem `xml:"placement"` PublicDNSName string `xml:"dnsName,omitempty"` SubnetID string `xml:"subnetId,omitempty"` @@ -1741,8 +1835,11 @@ type instanceItem struct { ImageID string `xml:"imageId"` InstanceID string `xml:"instanceId"` StateItem stateItem `xml:"instanceState"` - GroupSet instanceGroupSet `xml:"groupSet"` - TagSet instanceTagItemSet `xml:"tagSet"` + // StateTransitionReason is AWS's legacy free-text reason string, distinct + // from the structured StateReasonItem above. + StateTransitionReason string `xml:"reason,omitempty"` + GroupSet instanceGroupSet `xml:"groupSet"` + TagSet instanceTagItemSet `xml:"tagSet"` } // instanceTagItem is the embedded per-instance tag entry in DescribeInstances diff --git a/services/ec2/handler_test.go b/services/ec2/handler_test.go index 18df331ba..4c58d295f 100644 --- a/services/ec2/handler_test.go +++ b/services/ec2/handler_test.go @@ -1271,9 +1271,14 @@ func TestEC2Handler_DescribeInstanceAttribute(t *testing.T) { want: struct{ bodyContains string }{bodyContains: "false"}, }, { - name: "source_dest_check_returns_false", + // AWS defaults sourceDestCheck to true for VPC instances (it must + // be explicitly disabled, e.g. for NAT instances); the handler + // previously hardcoded "false" regardless of state, which this + // test encoded as correct. Fixed to read the primary ENI's real + // sourceDestCheck flag (defaulting to true). + name: "source_dest_check_returns_true", args: struct{ attr string }{attr: "sourceDestCheck"}, - want: struct{ bodyContains string }{bodyContains: "false"}, + want: struct{ bodyContains string }{bodyContains: "true"}, }, } diff --git a/services/ec2/janitor.go b/services/ec2/janitor.go index 43547fbf8..13ef73dc1 100644 --- a/services/ec2/janitor.go +++ b/services/ec2/janitor.go @@ -83,19 +83,21 @@ func (j *Janitor) sweepTerminatedInstances(ctx context.Context) { var swept []string - for id, inst := range j.Backend.instances { + for _, inst := range j.Backend.instances.All() { + id := instancesKeyFn(inst) if inst.State == StateTerminated && !inst.TerminatedAt.IsZero() && inst.TerminatedAt.Before(cutoff) { swept = append(swept, id) - delete(j.Backend.instances, id) + j.Backend.instances.Delete(id) delete(j.Backend.tags, id) // Defensive: remove any ENIs still referencing this instance // (can happen when state is restored from a pre-cleanup snapshot). - for eniID, eni := range j.Backend.networkInterfaces { + for _, eni := range j.Backend.networkInterfaces.All() { + eniID := networkInterfacesKeyFn(eni) if eni.InstanceID == id { j.Backend.recycleENIIPsLocked(eni) - delete(j.Backend.networkInterfaces, eniID) + j.Backend.networkInterfaces.Delete(eniID) delete(j.Backend.tags, eniID) } } @@ -131,11 +133,12 @@ func (j *Janitor) sweepCancelledSpotRequests(ctx context.Context) { var swept []string - for id, req := range j.Backend.spotRequests { + for _, req := range j.Backend.spotRequests.All() { + id := spotRequestsKeyFn(req) terminal := req.State == stateCancelled || req.State == "closed" if terminal && !req.CancelledAt.IsZero() && req.CancelledAt.Before(cutoff) { swept = append(swept, id) - delete(j.Backend.spotRequests, id) + j.Backend.spotRequests.Delete(id) delete(j.Backend.tags, id) } } diff --git a/services/ec2/persistence.go b/services/ec2/persistence.go index fa44ecc5a..8e05a7c8b 100644 --- a/services/ec2/persistence.go +++ b/services/ec2/persistence.go @@ -3,488 +3,154 @@ package ec2 import ( "context" "encoding/json" + "fmt" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" ) -// snapTGWMcastAssoc is a type alias used in backendSnapshot to keep line lengths manageable. -type snapTGWMcastAssoc = TransitGatewayMulticastDomainAssociation - -// snapRIExchange is a type alias used in backendSnapshot to keep line lengths manageable. -type snapRIExchange = ReservedInstancesExchange - -// snapTGWPeeringAtt is a type alias used in backendSnapshot to keep line lengths manageable. -type snapTGWPeeringAtt = TransitGatewayPeeringAttachment - -// snapTGWVpcAtt is a type alias used in backendSnapshot to keep line lengths manageable. -type snapTGWVpcAtt = TransitGatewayVpcAttachment - -// snapLGWVifGroupAssoc is a type alias used in backendSnapshot to keep line lengths manageable. -type snapLGWVifGroupAssoc = LocalGatewayRouteTableVirtualInterfaceGroupAssociation - -// snapLGWVpcAssoc is a type alias used in backendSnapshot to keep line lengths manageable. -type snapLGWVpcAssoc = LocalGatewayRouteTableVpcAssociation - -// snapTGWMeterPolicy is a type alias used in backendSnapshot to keep line lengths manageable. -type snapTGWMeterPolicy = TransitGatewayMeteringPolicy - -// snapTGWMeterPolicyEntry is a type alias used in backendSnapshot to keep line lengths manageable. -type snapTGWMeterPolicyEntry = TransitGatewayMeteringPolicyEntry - -// snapTGWMcastGroupEntry is a type alias used in backendSnapshot to keep line lengths manageable. -type snapTGWMcastGroupEntry = TransitGatewayMulticastGroupEntry - -// snapClassicLinkInstance is a type alias used in backendSnapshot to keep line lengths manageable. -type snapClassicLinkInstance = ClassicLinkInstance - -// snapIpamVerifyToken is a type alias used in backendSnapshot to keep line lengths manageable. -type snapIpamVerifyToken = IpamExternalResourceVerificationToken +// ec2SnapshotVersion identifies the shape of backendSnapshot's Tables blob +// (i.e. the set/shape of resources registered on b.registry -- see +// registerAllTables in store_setup.go). It must be bumped whenever a change +// there would make an older snapshot unsafe to decode as the current shape. +// Restore compares this against the persisted value and discards (rather +// than attempts to partially decode) any mismatch -- see Restore below. This +// mirrors the services/sqs pilot (commit 0f09d77c). +const ec2SnapshotVersion = 1 // snapTGWRTProp is a type alias used in backendSnapshot to keep line lengths manageable. type snapTGWRTProp = TransitGatewayRouteTablePropagation -// snapInterruptibleCRAlloc is a type alias used in backendSnapshot to keep line lengths manageable. -type snapInterruptibleCRAlloc = InterruptibleCapacityReservationAllocation - type backendSnapshot struct { - SnapshotAttributes map[string]map[string]string `json:"snapshotAttributes"` - ImageDeprecated map[string]string `json:"imageDeprecated"` - VPCs map[string]*VPC `json:"vpcs,omitempty"` - NatGateways map[string]*NatGateway `json:"natGateways,omitempty"` - KeyPairs map[string]*KeyPair `json:"keyPairs,omitempty"` - Volumes map[string]*Volume `json:"volumes,omitempty"` - Addresses map[string]*Address `json:"addresses,omitempty"` - InternetGateways map[string]*InternetGateway `json:"internetGateways"` - SecurityGroups map[string]*SecurityGroup `json:"securityGroups"` - Instances map[string]*Instance `json:"instances,omitempty"` - Subnets map[string]*Subnet `json:"subnets,omitempty"` - SpotRequests map[string]*SpotInstanceRequest `json:"spotRequests,omitempty"` - PlacementGroups map[string]*PlacementGroup `json:"placementGroups"` - Images map[string]*AMIStub `json:"images,omitempty"` - ImageUsageReports map[string]*ImageUsageReport `json:"imageUsageReports"` - LaunchTemplates map[string]*LaunchTemplate `json:"launchTemplates"` - VpcEndpoints map[string]*VpcEndpoint `json:"vpcEndpoints,omitempty"` - Snapshots map[string]*Snapshot `json:"snapshots,omitempty"` - NetworkACLs map[string]*StoredNetworkACL `json:"networkACLs,omitempty"` - TransitGateways map[string]*TransitGateway `json:"transitGateways"` - FlowLogs map[string]*FlowLog `json:"flowLogs,omitempty"` - DhcpOptionSets map[string]*DhcpOptions `json:"dhcpOptionSets"` - Tags map[string]map[string]string `json:"tags,omitempty"` - AddressTransfers map[string]*AddressTransfer `json:"addressTransfers"` - CapacityReservations map[string]*CapacityReservation `json:"capacityReservations"` - ReservedInstancesExchanges map[string]*snapRIExchange `json:"reservedInstancesExchanges"` - TGWMulticastDomainAssociations map[string]*snapTGWMcastAssoc `json:"tgwMcastDomainAssoc"` - TGWPeeringAttachments map[string]*snapTGWPeeringAtt `json:"tgwPeeringAttachments"` - TGWVpcAttachments map[string]*snapTGWVpcAtt `json:"tgwVpcAttachments"` - VpcEndpointConnections map[string]*VpcEndpointConnection `json:"vpcEndpointConnections"` - VpcPeeringConnections map[string]*VpcPeeringConnection `json:"vpcPeeringConnections"` - ByoipCidrs map[string]*ByoipCidr `json:"byoipCidrs,omitempty"` - DedicatedHosts map[string]*Host `json:"dedicatedHosts"` - VpnGateways map[string]*VpnGateway `json:"vpnGateways,omitempty"` - CustomerGateways map[string]*CustomerGateway `json:"customerGateways"` - Ipams map[string]*Ipam `json:"ipams,omitempty"` - IpamScopes map[string]*IpamScope `json:"ipamScopes,omitempty"` - IpamPools map[string]*IpamPool `json:"ipamPools,omitempty"` - IpamPoolCidrs map[string][]*IpamPoolCidr `json:"ipamPoolCidrs,omitempty"` - IpamPoolAllocations map[string]*IpamPoolAllocation `json:"ipamPoolAllocations"` - IpamResourceDiscoveries map[string]*IpamResourceDiscovery `json:"ipamResourceDiscoveries"` - IpamResourceDiscoveryAssocs map[string]*IpamResourceDiscoveryAssociation `json:"ipamResourceDiscoveryAssocs"` - IpamByoasns map[string]*IpamByoasn `json:"ipamByoasns,omitempty"` - IpamAsnAssociations map[string]*IpamAsnAssociation `json:"ipamAsnAssocs,omitempty"` - IpamVerificationTokens map[string]*snapIpamVerifyToken `json:"ipamVerifyTokens,omitempty"` - IpamResourceCidrs map[string]*IpamResourceCidr `json:"ipamResourceCidrs,omitempty"` - IpamPrefixListResolvers map[string]*IpamPrefixListResolver `json:"ipamPLResolvers,omitempty"` - IpamPrefixListResolverVersions map[string][]int64 `json:"ipamPLRVersions,omitempty"` - IpamPrefixListResolverTargets map[string]*IpamPrefixListResolverTarget `json:"ipamPLRTargets,omitempty"` - CarrierGateways map[string]*CarrierGateway `json:"carrierGateways"` - Fleets map[string]*Fleet `json:"fleets,omitempty"` - NetworkInsightsPaths map[string]*NetworkInsightsPath `json:"networkInsightsPaths"` - ManagedPrefixLists map[string]*ManagedPrefixList `json:"managedPrefixLists"` - EgressOnlyIGWs map[string]*EgressOnlyInternetGateway `json:"egressOnlyIGWs"` - IamAssociations map[string]*IamInstanceProfileAssociation `json:"iamAssociations"` - TgwRouteTables map[string]*TransitGatewayRouteTable `json:"tgwRouteTables"` - TgwRoutes map[string]*TransitGatewayRoute `json:"tgwRoutes,omitempty"` - TgwRTAssociations map[string]*TransitGatewayRouteTableAssociation `json:"tgwRTAssociations"` - - // TGW policy tables and route table announcements are kept in their own - // gofmt alignment group: their long type names would otherwise widen the - // tag column for the much larger field block above. - TgwPolicyTables map[string]*TransitGatewayPolicyTable `json:"tgwPolicyTables,omitempty"` - TgwPolicyTableAssociations map[string]*TransitGatewayPolicyTableAssociation `json:"tgwPTAssocs,omitempty"` - TgwRouteTableAnnouncements map[string]*TransitGatewayRouteTableAnnouncement `json:"tgwRTAnn,omitempty"` - - ReservedInstancesModifications map[string]*ReservedInstancesModification `json:"rim"` - ReservedInstancesListings map[string]*ReservedInstancesListing `json:"reservedInstancesListings"` - VpcCidrAssociations map[string]*VpcCidrBlockAssociation `json:"vpcCidrAssociations"` - VpnConnections map[string]*VpnConnection `json:"vpnConnections"` - VpcEndpointServiceConfigs map[string]*VpcEndpointServiceConfig `json:"vpcEndpointServiceConfigs"` - SpotFleets map[string]*SpotFleetRequest `json:"spotFleets,omitempty"` - SpotFleetHistory map[string][]SpotFleetHistoryRecord `json:"spotFleetHistory"` - VolumeModifications map[string]*VolumeModification `json:"volumeModifications"` - SnapshotTiers map[string]string `json:"snapshotTiers,omitempty"` - NetworkInterfaces map[string]*NetworkInterface `json:"networkInterfaces"` - RouteTables map[string]*RouteTable `json:"routeTables,omitempty"` - TrafficMirrorFilters map[string]*TrafficMirrorFilter `json:"trafficMirrorFilters"` - VpcPeeringOptions map[string]*PeeringConnectionOptions `json:"vpcPeeringOptions"` - SubnetCIDRAssociations map[string][]*SubnetCIDRAssociation `json:"subnetCIDRAssociations"` - AddressAttributes map[string]*AddressAttribute `json:"addressAttributes"` - InstanceMonitoring map[string]string `json:"instanceMonitoring"` - InstanceCreditSpecs map[string]string `json:"instanceCreditSpecs"` - InstanceIMDSOptions map[string]*IMDSOptions `json:"instanceIMDSOptions"` - InstanceMetadataDefaults *InstanceMetadataDefaults `json:"instanceMetadataDefaults"` - InstanceEventNotifAttrs *InstanceEventNotificationAttributes `json:"instanceEventNotifAttrs"` - NiPermissions map[string]*NetworkInterfacePermission `json:"niPermissions,omitempty"` - NiIPv6Addresses map[string][]string `json:"niIPv6Addresses"` - IDFormatSettings map[string]bool `json:"idFormatSettings"` - EndpointConnectionNotifs map[string]*VpcEndpointConnectionNotification `json:"endpointConnectionNotifs"` - VpcEndpointServicePermissions map[string][]string `json:"vpcEpSvcPerms"` - SnapshotLocks map[string]*SnapshotLock `json:"snapshotLocks,omitempty"` - ReplaceRootVolumeTasks map[string]*ReplaceRootVolumeTask `json:"replaceRootVolumeTasks"` - SubnetCIDRReservations map[string][]*SubnetCIDRReservation `json:"subnetCIDRReservations"` - ImageDisabled map[string]bool `json:"imageDisabled,omitempty"` - SgVpcAssociations map[string]map[string]string `json:"sgVpcAssociations"` - ImageDeregistrationProtection map[string]bool `json:"imageDeregProtect"` - ImageAttributes map[string]map[string]string `json:"imageAttributes"` - VgwRoutePropagation map[string]bool `json:"vgwRoutePropagation"` - ClientVpnEndpoints map[string]*ClientVpnEndpoint `json:"clientVpnEndpoints"` - TgwConnects map[string]*TransitGatewayConnect `json:"tgwConnects,omitempty"` - TgwConnectPeers map[string]*TransitGatewayConnectPeer `json:"tgwConnectPeers"` - TgwPrefixListRefs map[string]*TransitGatewayPrefixListReference `json:"tgwPrefixListRefs"` - VerifiedAccessEndpoints map[string]*VerifiedAccessEndpoint `json:"verifiedAccessEndpoints"` - VerifiedAccessGroups map[string]*VerifiedAccessGroup `json:"verifiedAccessGroups"` - VerifiedAccessInstances map[string]*VerifiedAccessInstance `json:"verifiedAccessInstances"` - VerifiedAccessTrustProviders map[string]*VerifiedAccessTrustProvider `json:"vatps"` - - // gopherstack-5o9 final EC2 parity sweep additions (kept in their own - // alignment block so their longer field names don't widen gofmt's column - // alignment for the rest of this struct). - TgwRTPropagations map[string]map[string]*snapTGWRTProp `json:"tgwRTPropagations,omitempty"` - InterruptibleCRAllocations map[string]*snapInterruptibleCRAlloc `json:"interruptibleCRAllocations,omitempty"` - MovingAddresses map[string]*MovingAddressStatus `json:"movingAddresses,omitempty"` - - InstanceConnectEndpoints map[string]*InstanceConnectEndpoint `json:"instanceConnectEndpoints"` - InstanceEventWindows map[string]*InstanceEventWindow `json:"instanceEventWindows"` - ImageImportTasks map[string]*ImageImportTask `json:"imageImportTasks"` - SnapshotImportTasks map[string]*SnapshotImportTask `json:"snapshotImportTasks"` - RecycleBinImages map[string]*RecycleBinImage `json:"recycleBinImages"` - RecycleBinSnapshots map[string]*Snapshot `json:"recycleBinSnapshots"` - RecycleBinVolumes map[string]*RecycleBinVolume `json:"recycleBinVolumes"` - FastLaunchImages map[string]bool `json:"fastLaunchImages"` - FastSnapshotRestores map[string]bool `json:"fastSnapshotRestores"` - VpnConnectionRoutes map[string]*VpnConnectionRoute `json:"vpnConnectionRoutes"` - SpotDatafeed *SpotDatafeed `json:"spotDatafeed,omitempty"` - VpcTenancy map[string]string `json:"vpcTenancy,omitempty"` - TrafficMirrorFilterRules map[string]*TrafficMirrorFilterRule `json:"trafficMirrorFilterRules"` - TrafficMirrorSessions map[string]*TrafficMirrorSession `json:"trafficMirrorSessions"` - TrafficMirrorTargets map[string]*TrafficMirrorTarget `json:"trafficMirrorTargets"` - NetworkInsightsAnalyses map[string]*NetworkInsightsAnalysis `json:"networkInsightsAnalyses"` - NetworkInsightsAccessScopes map[string]*NetworkInsightsAccessScope `json:"networkInsightsAccessScopes"` - NetworkInsightsAccessScopeAnalyses map[string]*NetworkInsightsAccessScopeAnalysis `json:"niasa"` - ReservedInstances map[string]*ReservedInstance `json:"reservedInstances"` - ReservedInstancesOfferings map[string]*ReservedInstancesOffering `json:"reservedInstancesOfferings"` - RouteServers map[string]*RouteServer `json:"routeServers,omitempty"` - RouteServerEndpoints map[string]*RouteServerEndpoint `json:"rsEndpoints,omitempty"` - RouteServerPeers map[string]*RouteServerPeer `json:"routeServerPeers,omitempty"` - RouteServerAssociations map[string]*RouteServerAssociation `json:"rsAssociations,omitempty"` - RouteServerPropagations map[string]*RouteServerPropagation `json:"rsPropagations,omitempty"` - LocalGateways map[string]*LocalGateway `json:"localGateways,omitempty"` - LocalGatewayVirtualInterfaces map[string]*LocalGatewayVirtualInterface `json:"lgwVifs,omitempty"` - LocalGatewayVirtualInterfaceGroups map[string]*LocalGatewayVirtualInterfaceGroup `json:"lgwVifGroups,omitempty"` - LocalGatewayRouteTables map[string]*LocalGatewayRouteTable `json:"lgwRouteTables,omitempty"` - LocalGatewayRoutes map[string]*LocalGatewayRoute `json:"lgwRoutes,omitempty"` - LocalGatewayRouteTableVpcAssocs map[string]*snapLGWVpcAssoc `json:"lgwRtVpcAssocs,omitempty"` - LocalGatewayRTVifGroupAssocs map[string]*snapLGWVifGroupAssoc `json:"lgwVifGroupAssocs,omitempty"` - TgwMulticastDomains map[string]*TransitGatewayMulticastDomain `json:"tgwMcastDomains,omitempty"` - TgwMulticastGroupEntries map[string]*snapTGWMcastGroupEntry `json:"tgwMcastGroupEnt,omitempty"` - TgwMeteringPolicies map[string]*snapTGWMeterPolicy `json:"tgwMeterPolicies,omitempty"` - TgwMeteringPolicyEntries map[string]*snapTGWMeterPolicyEntry `json:"tgwMeterPolicyEnt,omitempty"` - ClassicLinkInstances map[string]*snapClassicLinkInstance `json:"classicLinkInst,omitempty"` - VpcBlockPublicAccessOptions *VpcBlockPublicAccessOptions `json:"vpcBpaOptions,omitempty"` - VpcBlockPublicAccessExclusions map[string]*VpcBlockPublicAccessExclusion `json:"vpcBpaExclusions,omitempty"` - CapacityReservationFleets map[string]*CapacityReservationFleet `json:"crFleets,omitempty"` - CapacityBlockOfferings map[string]*CapacityBlockOffering `json:"cbOfferings,omitempty"` - CapacityBlockExtensionOfferings map[string]*CapacityBlockExtensionOffering `json:"cbExtOfferings,omitempty"` - CapacityBlocks map[string]*CapacityBlock `json:"capacityBlocks,omitempty"` - CapacityBlockExtensions map[string]*CapacityBlockExtension `json:"cbExtensions,omitempty"` - CapacityReservationBillingReqs map[string]*CapacityReservationBillingRequest `json:"crBillingRequests,omitempty"` - CapacityManagerDataExports map[string]*CapacityManagerDataExport `json:"cmDataExports,omitempty"` - CapacityManagerState *CapacityManagerState `json:"cmState,omitempty"` - IpamPolicies map[string]*IpamPolicy `json:"ipamPolicies,omitempty"` - IpamPolicyEnabledTargets map[string]string `json:"ipamPolicyEnabled,omitempty"` - VerifiedAccessEndpointPolicies map[string]*VerifiedAccessPolicy `json:"vaEndpointPolicies,omitempty"` - VerifiedAccessGroupPolicies map[string]*VerifiedAccessPolicy `json:"vaGroupPolicies,omitempty"` - FpgaImages map[string]*FpgaImage `json:"fpgaImages,omitempty"` - - // VerifiedAccessInstanceLoggingCfgs is kept in its own gofmt alignment - // group (but still ahead of the string/slice/scalar fields below, to - // preserve optimal field ordering): its long type name would otherwise - // widen the tag column for the much larger field block above. - VerifiedAccessInstanceLoggingCfgs map[string]*VerifiedAccessInstanceLoggingConfig `json:"vaLoggingCfgs,omitempty"` - - // Scheduled Instances / COIP / public IPv4-IPv6 pool / Allowed Images Settings / image - // task / usage report fields are kept in their own gofmt alignment group for the same - // reason as VerifiedAccessInstanceLoggingCfgs above. - ScheduledInstances map[string]*ScheduledInstance `json:"scheduledInstances,omitempty"` - ScheduledInstanceLaunched map[string]int32 `json:"schedInstLaunched,omitempty"` - CoipPools map[string]*CoipPool `json:"coipPools,omitempty"` - CoipCidrs map[string]*CoipCidr `json:"coipCidrs,omitempty"` - Ipv4Pools map[string]*Ipv4Pool `json:"ipv4Pools,omitempty"` - Ipv6Pools map[string]*Ipv6Pool `json:"ipv6Pools,omitempty"` - AllowedImagesSettings *AllowedImagesSettings `json:"allowedImagesSettings,omitempty"` - StoreImageTasks map[string]*StoreImageTask `json:"storeImageTasks,omitempty"` - UsageReports map[string]*UsageReport `json:"usageReports,omitempty"` - UsageReportEntries map[string][]*UsageReportEntry `json:"usageReportEntries,omitempty"` - InstanceProductCodes map[string][]string `json:"instanceProductCodes,omitempty"` - - // VM Import/Export, Bundle, Conversion Task, Trunk Interface, and Enclave Certificate - // IAM Role fields. - BundleTasks map[string]*BundleTask `json:"bundleTasks,omitempty"` - ConversionTasks map[string]*ConversionTask `json:"conversionTasks,omitempty"` - ExportTasks map[string]*ExportTask `json:"exportTasks,omitempty"` - ExportImageTasks map[string]*ExportImageTaskRec `json:"exportImageTasks,omitempty"` - TrunkInterfaceAssociations map[string]*TrunkInterfaceAssociation `json:"trunkInterfaceAssociations,omitempty"` - EnclaveCertIamRoles map[string][]*EnclaveCertIamRoleAssociation `json:"enclaveCertIamRoles,omitempty"` - - // Mac modification task / Secondary Network / Secondary Subnet / Secondary - // Interface / Outpost LAG / Service Link Virtual Interface / SQL HA fields. - MacModificationTasks map[string]*MacModificationTask `json:"macModificationTasks,omitempty"` - SecondaryNetworks map[string]*SecondaryNetwork `json:"secondaryNetworks,omitempty"` - SecondarySubnets map[string]*SecondarySubnet `json:"secondarySubnets,omitempty"` - SecondaryInterfaces map[string]*SecondaryInterface `json:"secondaryInterfaces,omitempty"` - ServiceLinkVirtualInterfaces map[string]*ServiceLinkVirtualInterface `json:"serviceLinkVirtualInterfaces,omitempty"` - OutpostLags map[string]*OutpostLag `json:"outpostLags,omitempty"` - AvailabilityZoneGroupOptIns map[string]string `json:"azGroupOptIns,omitempty"` - SQLHaRegistrations map[string]*RegisteredSQLHaInstance `json:"sqlHaRegistrations,omitempty"` - SQLHaHistory map[string][]*RegisteredSQLHaInstance `json:"sqlHaHistory,omitempty"` - - // parity-sweep-2: VPC Encryption Control, VPN Concentrator, Host Reservations, - // Declarative Policies, AWS Network Performance. - VpcEncryptionControls map[string]*VpcEncryptionControl `json:"vpcEncryptionControls,omitempty"` - VpnConcentrators map[string]*VpnConcentrator `json:"vpnConcentrators,omitempty"` - HostReservations map[string]*HostReservation `json:"hostReservations,omitempty"` - DeclarativePoliciesReports map[string]*DeclarativePoliciesReport `json:"declPoliciesReports,omitempty"` - NetworkPerformanceSubscriptions map[string]*NetworkPerformanceSubscription `json:"networkPerformanceSubs,omitempty"` - - IpamOrgAdminAccountID string `json:"ipamOrgAdminAcct,omitempty"` - Region string `json:"region,omitempty"` - AccountID string `json:"accountID,omitempty"` - FreePrivateIPs []string `json:"freePrivateIPs"` - NextPrivateIPIndex int `json:"nextPrivateIPIndex"` - NextElasticIPIndex int `json:"nextElasticIPIndex"` - EbsEncryptionByDefault bool `json:"ebsEncryptionByDefault"` - SerialConsoleAccess bool `json:"serialConsoleAccess"` - // gopherstack-5o9 final EC2 parity sweep addition. + Tables map[string]json.RawMessage `json:"tables"` + SnapshotAttributes map[string]map[string]string `json:"snapshotAttributes"` + ImageDeprecated map[string]string `json:"imageDeprecated"` + Tags map[string]map[string]string `json:"tags,omitempty"` + AddressTransfers map[string]*AddressTransfer `json:"addressTransfers"` + IpamPoolCidrs map[string][]*IpamPoolCidr `json:"ipamPoolCidrs,omitempty"` + IpamPrefixListResolverVersions map[string][]int64 `json:"ipamPLRVersions,omitempty"` + VpcCidrAssociations map[string]*VpcCidrBlockAssociation `json:"vpcCidrAssociations"` + SpotFleetHistory map[string][]SpotFleetHistoryRecord `json:"spotFleetHistory"` + SnapshotTiers map[string]string `json:"snapshotTiers,omitempty"` + VpcPeeringOptions map[string]*PeeringConnectionOptions `json:"vpcPeeringOptions"` + SubnetCIDRAssociations map[string][]*SubnetCIDRAssociation `json:"subnetCIDRAssociations"` + InstanceMonitoring map[string]string `json:"instanceMonitoring"` + InstanceCreditSpecs map[string]string `json:"instanceCreditSpecs"` + InstanceIMDSOptions map[string]*IMDSOptions `json:"instanceIMDSOptions"` + InstanceMetadataDefaults *InstanceMetadataDefaults `json:"instanceMetadataDefaults"` + InstanceEventNotifAttrs *InstanceEventNotificationAttributes `json:"instanceEventNotifAttrs"` + NiIPv6Addresses map[string][]string `json:"niIPv6Addresses"` + IDFormatSettings map[string]bool `json:"idFormatSettings"` + VpcEndpointServicePermissions map[string][]string `json:"vpcEpSvcPerms"` + SubnetCIDRReservations map[string][]*SubnetCIDRReservation `json:"subnetCIDRReservations"` + ImageDisabled map[string]bool `json:"imageDisabled,omitempty"` + SgVpcAssociations map[string]map[string]string `json:"sgVpcAssociations"` + ImageDeregistrationProtection map[string]bool `json:"imageDeregProtect"` + ImageAttributes map[string]map[string]string `json:"imageAttributes"` + VgwRoutePropagation map[string]bool `json:"vgwRoutePropagation"` + TgwRTPropagations map[string]map[string]*snapTGWRTProp `json:"tgwRTPropagations,omitempty"` + FastLaunchImages map[string]bool `json:"fastLaunchImages"` + FastSnapshotRestores map[string]bool `json:"fastSnapshotRestores"` + SpotDatafeed *SpotDatafeed `json:"spotDatafeed,omitempty"` + VpcTenancy map[string]string `json:"vpcTenancy,omitempty"` + VpcBlockPublicAccessOptions *VpcBlockPublicAccessOptions `json:"vpcBpaOptions,omitempty"` + CapacityManagerState *CapacityManagerState `json:"cmState,omitempty"` + IpamPolicyEnabledTargets map[string]string `json:"ipamPolicyEnabled,omitempty"` + VerifiedAccessEndpointPolicies map[string]*VerifiedAccessPolicy `json:"vaEndpointPolicies,omitempty"` + VerifiedAccessGroupPolicies map[string]*VerifiedAccessPolicy `json:"vaGroupPolicies,omitempty"` + ScheduledInstanceLaunched map[string]int32 `json:"schedInstLaunched,omitempty"` + AllowedImagesSettings *AllowedImagesSettings `json:"allowedImagesSettings,omitempty"` + UsageReportEntries map[string][]*UsageReportEntry `json:"usageReportEntries,omitempty"` + InstanceProductCodes map[string][]string `json:"instanceProductCodes,omitempty"` + EnclaveCertIamRoles map[string][]*EnclaveCertIamRoleAssociation `json:"enclaveCertIamRoles,omitempty"` + AvailabilityZoneGroupOptIns map[string]string `json:"azGroupOptIns,omitempty"` + SQLHaHistory map[string][]*RegisteredSQLHaInstance `json:"sqlHaHistory,omitempty"` + IpamOrgAdminAccountID string `json:"ipamOrgAdminAcct,omitempty"` + Region string `json:"region,omitempty"` + AccountID string `json:"accountID,omitempty"` + FreePrivateIPs []string `json:"freePrivateIPs"` + Version int `json:"version"` + NextPrivateIPIndex int `json:"nextPrivateIPIndex"` + NextElasticIPIndex int `json:"nextElasticIPIndex"` + EbsEncryptionByDefault bool `json:"ebsEncryptionByDefault"` + SerialConsoleAccess bool `json:"serialConsoleAccess"` + + // ReachabilityAnalyzerOrgSharing is kept in its own gofmt alignment group: + // its long name would otherwise widen the tag column for the block above. ReachabilityAnalyzerOrgSharing bool `json:"reachabilityAnalyzerOrgSharing,omitempty"` } // Snapshot serialises the backend state to JSON. // It implements persistence.Persistable. -// -//nolint:funlen // large state snapshot func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "ec2: snapshot table marshal failed", "error", err) + + return nil + } + snap := backendSnapshot{ - Instances: b.instances, - SecurityGroups: b.securityGroups, - VPCs: b.vpcs, - Subnets: b.subnets, - KeyPairs: b.keyPairs, - Volumes: b.volumes, - Addresses: b.addresses, - InternetGateways: b.internetGateways, - RouteTables: b.routeTables, - NatGateways: b.natGateways, - NetworkInterfaces: b.networkInterfaces, - SpotRequests: b.spotRequests, - PlacementGroups: b.placementGroups, - Images: b.images, - ImageUsageReports: b.imageUsageReports, - LaunchTemplates: b.launchTemplates, - VpcEndpoints: b.vpcEndpoints, - Snapshots: b.snapshots, - NetworkACLs: b.networkACLs, - TransitGateways: b.transitGateways, - FlowLogs: b.flowLogs, - DhcpOptionSets: b.dhcpOptionSets, - Tags: b.tags, - AddressTransfers: b.addressTransfers, - CapacityReservations: b.capacityReservations, - ReservedInstancesExchanges: b.reservedInstancesExchanges, - TGWMulticastDomainAssociations: b.tgwMulticastDomainAssociations, - TGWPeeringAttachments: b.tgwPeeringAttachments, - TGWVpcAttachments: b.tgwVpcAttachments, - VpcEndpointConnections: b.vpcEndpointConnections, - VpcPeeringConnections: b.vpcPeeringConnections, - ByoipCidrs: b.byoipCidrs, - DedicatedHosts: b.dedicatedHosts, - VpnGateways: b.vpnGateways, - CustomerGateways: b.customerGateways, - Ipams: b.ipams, - IpamScopes: b.ipamScopes, - IpamPools: b.ipamPools, - IpamPoolCidrs: b.ipamPoolCidrs, - IpamPoolAllocations: b.ipamPoolAllocations, - IpamResourceDiscoveries: b.ipamResourceDiscoveries, - IpamResourceDiscoveryAssocs: b.ipamResourceDiscoveryAssocs, - IpamByoasns: b.ipamByoasns, - IpamAsnAssociations: b.ipamAsnAssociations, - IpamVerificationTokens: b.ipamVerificationTokens, - IpamResourceCidrs: b.ipamResourceCidrs, - IpamPrefixListResolvers: b.ipamPrefixListResolvers, - IpamPrefixListResolverVersions: b.ipamPrefixListResolverVersions, - IpamPrefixListResolverTargets: b.ipamPrefixListResolverTargets, - CarrierGateways: b.carrierGateways, - Fleets: b.fleets, - NetworkInsightsPaths: b.networkInsightsPaths, - ManagedPrefixLists: b.managedPrefixLists, - EbsEncryptionByDefault: b.ebsEncryptionByDefault, - SerialConsoleAccess: b.serialConsoleAccess, - FreePrivateIPs: b.freePrivateIPs, - AccountID: b.AccountID, - Region: b.Region, - NextPrivateIPIndex: b.nextPrivateIPIndex, - NextElasticIPIndex: b.nextElasticIPIndex, - EgressOnlyIGWs: b.egressOnlyIGWs, - IamAssociations: b.iamAssociations, - TgwRouteTables: b.tgwRouteTables, - TgwRoutes: b.tgwRoutes, - TgwRTAssociations: b.tgwRTAssociations, - TgwPolicyTables: b.tgwPolicyTables, - TgwPolicyTableAssociations: b.tgwPolicyTableAssociations, - TgwRouteTableAnnouncements: b.tgwRouteTableAnnouncements, - VpcCidrAssociations: b.vpcCidrAssociations, - VpnConnections: b.vpnConnections, - VpcEndpointServiceConfigs: b.vpcEndpointServiceConfigs, - SpotFleets: b.spotFleets, - SpotFleetHistory: b.spotFleetHistory, - VolumeModifications: b.volumeModifications, - SnapshotTiers: b.snapshotTiers, - SnapshotAttributes: b.snapshotAttributes, - SgVpcAssociations: b.sgVpcAssociations, - VpcTenancy: b.vpcTenancy, - VpcPeeringOptions: b.vpcPeeringOptions, - SubnetCIDRAssociations: b.subnetCIDRAssociations, - AddressAttributes: b.addressAttributes, - InstanceMonitoring: b.instanceMonitoring, - InstanceCreditSpecs: b.instanceCreditSpecs, - InstanceIMDSOptions: b.instanceIMDSOptions, - InstanceMetadataDefaults: b.instanceMetadataDefaults, - InstanceEventNotifAttrs: b.instanceEventNotifAttrs, - NiPermissions: b.niPermissions, - NiIPv6Addresses: b.niIPv6Addresses, - IDFormatSettings: b.idFormatSettings, - EndpointConnectionNotifs: b.endpointConnectionNotifs, - VpcEndpointServicePermissions: b.vpcEndpointServicePermissions, - SnapshotLocks: b.snapshotLocks, - ReplaceRootVolumeTasks: b.replaceRootVolumeTasks, - SubnetCIDRReservations: b.subnetCIDRReservations, - ImageDisabled: b.imageDisabled, - ImageDeprecated: b.imageDeprecated, - ImageDeregistrationProtection: b.imageDeregistrationProtection, - ImageAttributes: b.imageAttributes, - VgwRoutePropagation: b.vgwRoutePropagation, - ClientVpnEndpoints: b.clientVpnEndpoints, - TgwConnects: b.tgwConnects, - TgwConnectPeers: b.tgwConnectPeers, - TgwPrefixListRefs: b.tgwPrefixListRefs, - VerifiedAccessEndpoints: b.verifiedAccessEndpoints, - VerifiedAccessGroups: b.verifiedAccessGroups, - VerifiedAccessInstances: b.verifiedAccessInstances, - VerifiedAccessTrustProviders: b.verifiedAccessTrustProviders, - InstanceConnectEndpoints: b.instanceConnectEndpoints, - InstanceEventWindows: b.instanceEventWindows, - ImageImportTasks: b.imageImportTasks, - SnapshotImportTasks: b.snapshotImportTasks, - RecycleBinImages: b.recycleBinImages, - RecycleBinSnapshots: b.recycleBinSnapshots, - RecycleBinVolumes: b.recycleBinVolumes, - FastLaunchImages: b.fastLaunchImages, - FastSnapshotRestores: b.fastSnapshotRestores, - VpnConnectionRoutes: b.vpnConnectionRoutes, - SpotDatafeed: b.spotDatafeed, - TrafficMirrorFilters: b.trafficMirrorFilters, - TrafficMirrorFilterRules: b.trafficMirrorFilterRules, - TrafficMirrorSessions: b.trafficMirrorSessions, - TrafficMirrorTargets: b.trafficMirrorTargets, - NetworkInsightsAnalyses: b.networkInsightsAnalyses, - NetworkInsightsAccessScopes: b.networkInsightsAccessScopes, - NetworkInsightsAccessScopeAnalyses: b.networkInsightsAccessScopeAnalyses, - ReservedInstances: b.reservedInstances, - ReservedInstancesOfferings: b.reservedInstancesOfferings, - ReservedInstancesListings: b.reservedInstancesListings, - ReservedInstancesModifications: b.reservedInstancesModifications, - RouteServers: b.routeServers, - RouteServerEndpoints: b.routeServerEndpoints, - RouteServerPeers: b.routeServerPeers, - RouteServerAssociations: b.routeServerAssociations, - RouteServerPropagations: b.routeServerPropagations, - LocalGateways: b.localGateways, - LocalGatewayVirtualInterfaces: b.localGatewayVirtualInterfaces, - LocalGatewayVirtualInterfaceGroups: b.localGatewayVirtualInterfaceGroups, - LocalGatewayRouteTables: b.localGatewayRouteTables, - LocalGatewayRoutes: b.localGatewayRoutes, - LocalGatewayRouteTableVpcAssocs: b.localGatewayRouteTableVpcAssociations, - LocalGatewayRTVifGroupAssocs: b.localGatewayRouteTableVifGroupAssociations, - TgwMulticastDomains: b.tgwMulticastDomains, - TgwMulticastGroupEntries: b.tgwMulticastGroupEntries, - TgwMeteringPolicies: b.tgwMeteringPolicies, - TgwMeteringPolicyEntries: b.tgwMeteringPolicyEntries, - ClassicLinkInstances: b.classicLinkInstances, - VpcBlockPublicAccessOptions: b.vpcBlockPublicAccessOptions, - VpcBlockPublicAccessExclusions: b.vpcBlockPublicAccessExclusions, - CapacityReservationFleets: b.capacityReservationFleets, - CapacityBlockOfferings: b.capacityBlockOfferings, - CapacityBlockExtensionOfferings: b.capacityBlockExtensionOfferings, - CapacityBlocks: b.capacityBlocks, - CapacityBlockExtensions: b.capacityBlockExtensions, - CapacityReservationBillingReqs: b.capacityReservationBillingRequests, - CapacityManagerDataExports: b.capacityManagerDataExports, - CapacityManagerState: b.capacityManagerState, - IpamPolicies: b.ipamPolicies, - IpamPolicyEnabledTargets: b.ipamPolicyEnabledTargets, - IpamOrgAdminAccountID: b.ipamOrgAdminAccountID, - VerifiedAccessEndpointPolicies: b.verifiedAccessEndpointPolicies, - VerifiedAccessGroupPolicies: b.verifiedAccessGroupPolicies, - VerifiedAccessInstanceLoggingCfgs: b.verifiedAccessInstanceLoggingConfigs, - FpgaImages: b.fpgaImages, - ScheduledInstances: b.scheduledInstances, - ScheduledInstanceLaunched: b.scheduledInstanceLaunched, - CoipPools: b.coipPools, - CoipCidrs: b.coipCidrs, - Ipv4Pools: b.ipv4Pools, - Ipv6Pools: b.ipv6Pools, - AllowedImagesSettings: b.allowedImagesSettings, - StoreImageTasks: b.storeImageTasks, - UsageReports: b.usageReports, - UsageReportEntries: b.usageReportEntries, - InstanceProductCodes: b.instanceProductCodes, - BundleTasks: b.bundleTasks, - ConversionTasks: b.conversionTasks, - ExportTasks: b.exportTasks, - ExportImageTasks: b.exportImageTasks, - TrunkInterfaceAssociations: b.trunkInterfaceAssociations, - EnclaveCertIamRoles: b.enclaveCertIamRoles, - MacModificationTasks: b.macModificationTasks, - SecondaryNetworks: b.secondaryNetworks, - SecondarySubnets: b.secondarySubnets, - SecondaryInterfaces: b.secondaryInterfaces, - ServiceLinkVirtualInterfaces: b.serviceLinkVirtualInterfaces, - OutpostLags: b.outpostLags, - AvailabilityZoneGroupOptIns: b.availabilityZoneGroupOptIns, - SQLHaRegistrations: b.sqlHaRegistrations, - SQLHaHistory: b.sqlHaHistory, - VpcEncryptionControls: b.vpcEncryptionControls, - VpnConcentrators: b.vpnConcentrators, - HostReservations: b.hostReservations, - DeclarativePoliciesReports: b.declarativePoliciesReports, - NetworkPerformanceSubscriptions: b.networkPerformanceSubscriptions, - TgwRTPropagations: b.tgwRTPropagations, - InterruptibleCRAllocations: b.interruptibleCRAllocations, - MovingAddresses: b.movingAddresses, - ReachabilityAnalyzerOrgSharing: b.reachabilityAnalyzerOrgSharing, + Version: ec2SnapshotVersion, + Tables: tables, + Tags: b.tags, + AddressTransfers: b.addressTransfers, + IpamPoolCidrs: b.ipamPoolCidrs, + IpamPrefixListResolverVersions: b.ipamPrefixListResolverVersions, + EbsEncryptionByDefault: b.ebsEncryptionByDefault, + SerialConsoleAccess: b.serialConsoleAccess, + FreePrivateIPs: b.freePrivateIPs, + AccountID: b.AccountID, + Region: b.Region, + NextPrivateIPIndex: b.nextPrivateIPIndex, + NextElasticIPIndex: b.nextElasticIPIndex, + VpcCidrAssociations: b.vpcCidrAssociations, + SpotFleetHistory: b.spotFleetHistory, + SnapshotTiers: b.snapshotTiers, + SnapshotAttributes: b.snapshotAttributes, + SgVpcAssociations: b.sgVpcAssociations, + VpcTenancy: b.vpcTenancy, + VpcPeeringOptions: b.vpcPeeringOptions, + SubnetCIDRAssociations: b.subnetCIDRAssociations, + InstanceMonitoring: b.instanceMonitoring, + InstanceCreditSpecs: b.instanceCreditSpecs, + InstanceIMDSOptions: b.instanceIMDSOptions, + InstanceMetadataDefaults: b.instanceMetadataDefaults, + InstanceEventNotifAttrs: b.instanceEventNotifAttrs, + NiIPv6Addresses: b.niIPv6Addresses, + IDFormatSettings: b.idFormatSettings, + VpcEndpointServicePermissions: b.vpcEndpointServicePermissions, + SubnetCIDRReservations: b.subnetCIDRReservations, + ImageDisabled: b.imageDisabled, + ImageDeprecated: b.imageDeprecated, + ImageDeregistrationProtection: b.imageDeregistrationProtection, + ImageAttributes: b.imageAttributes, + VgwRoutePropagation: b.vgwRoutePropagation, + FastLaunchImages: b.fastLaunchImages, + FastSnapshotRestores: b.fastSnapshotRestores, + SpotDatafeed: b.spotDatafeed, + VpcBlockPublicAccessOptions: b.vpcBlockPublicAccessOptions, + CapacityManagerState: b.capacityManagerState, + IpamPolicyEnabledTargets: b.ipamPolicyEnabledTargets, + IpamOrgAdminAccountID: b.ipamOrgAdminAccountID, + VerifiedAccessEndpointPolicies: b.verifiedAccessEndpointPolicies, + VerifiedAccessGroupPolicies: b.verifiedAccessGroupPolicies, + ScheduledInstanceLaunched: b.scheduledInstanceLaunched, + AllowedImagesSettings: b.allowedImagesSettings, + UsageReportEntries: b.usageReportEntries, + InstanceProductCodes: b.instanceProductCodes, + EnclaveCertIamRoles: b.enclaveCertIamRoles, + AvailabilityZoneGroupOptIns: b.availabilityZoneGroupOptIns, + SQLHaHistory: b.sqlHaHistory, + TgwRTPropagations: b.tgwRTPropagations, + ReachabilityAnalyzerOrgSharing: b.reachabilityAnalyzerOrgSharing, } data, err := json.Marshal(snap) @@ -500,7 +166,7 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { // Restore loads backend state from a JSON snapshot. // It implements persistence.Persistable. // -//nolint:gocognit,gocyclo,cyclop,funlen // large state restore +//nolint:gocognit,cyclop,funlen // large state restore func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { var snap backendSnapshot @@ -513,80 +179,40 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() + if snap.Version != ec2SnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/sqs pilot (commit 0f09d77c). + logger.Load(ctx).WarnContext(ctx, + "ec2: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", ec2SnapshotVersion) + + b.registry.ResetAll() + + return nil + } + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("ec2: restore snapshot tables: %w", err) + } + b.restoreCoreFields(&snap) b.restoreExtendedFields(&snap) b.rebuildSecondaryIndexesLocked() - if snap.EgressOnlyIGWs != nil { - b.egressOnlyIGWs = snap.EgressOnlyIGWs - } else { - b.egressOnlyIGWs = make(map[string]*EgressOnlyInternetGateway) - } - if snap.IamAssociations != nil { - b.iamAssociations = snap.IamAssociations - } else { - b.iamAssociations = make(map[string]*IamInstanceProfileAssociation) - } - if snap.TgwRouteTables != nil { - b.tgwRouteTables = snap.TgwRouteTables - } else { - b.tgwRouteTables = make(map[string]*TransitGatewayRouteTable) - } - if snap.TgwRoutes != nil { - b.tgwRoutes = snap.TgwRoutes - } else { - b.tgwRoutes = make(map[string]*TransitGatewayRoute) - } - if snap.TgwRTAssociations != nil { - b.tgwRTAssociations = snap.TgwRTAssociations - } else { - b.tgwRTAssociations = make(map[string]*TransitGatewayRouteTableAssociation) - } - if snap.TgwPolicyTables != nil { - b.tgwPolicyTables = snap.TgwPolicyTables - } else { - b.tgwPolicyTables = make(map[string]*TransitGatewayPolicyTable) - } - if snap.TgwPolicyTableAssociations != nil { - b.tgwPolicyTableAssociations = snap.TgwPolicyTableAssociations - } else { - b.tgwPolicyTableAssociations = make(map[string]*TransitGatewayPolicyTableAssociation) - } - if snap.TgwRouteTableAnnouncements != nil { - b.tgwRouteTableAnnouncements = snap.TgwRouteTableAnnouncements - } else { - b.tgwRouteTableAnnouncements = make(map[string]*TransitGatewayRouteTableAnnouncement) - } if snap.VpcCidrAssociations != nil { b.vpcCidrAssociations = snap.VpcCidrAssociations } else { b.vpcCidrAssociations = make(map[string]*VpcCidrBlockAssociation) } - if snap.VpnConnections != nil { - b.vpnConnections = snap.VpnConnections - } else { - b.vpnConnections = make(map[string]*VpnConnection) - } - if snap.VpcEndpointServiceConfigs != nil { - b.vpcEndpointServiceConfigs = snap.VpcEndpointServiceConfigs - } else { - b.vpcEndpointServiceConfigs = make(map[string]*VpcEndpointServiceConfig) - } - if snap.SpotFleets != nil { - b.spotFleets = snap.SpotFleets - } else { - b.spotFleets = make(map[string]*SpotFleetRequest) - } if snap.SpotFleetHistory != nil { b.spotFleetHistory = snap.SpotFleetHistory } else { b.spotFleetHistory = make(map[string][]SpotFleetHistoryRecord) } - if snap.VolumeModifications != nil { - b.volumeModifications = snap.VolumeModifications - } else { - b.volumeModifications = make(map[string]*VolumeModification) - } if snap.SnapshotTiers != nil { b.snapshotTiers = snap.SnapshotTiers } else { @@ -617,11 +243,6 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { } else { b.subnetCIDRAssociations = make(map[string][]*SubnetCIDRAssociation) } - if snap.AddressAttributes != nil { - b.addressAttributes = snap.AddressAttributes - } else { - b.addressAttributes = make(map[string]*AddressAttribute) - } if snap.InstanceMonitoring != nil { b.instanceMonitoring = snap.InstanceMonitoring } else { @@ -639,11 +260,6 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { } b.instanceMetadataDefaults = snap.InstanceMetadataDefaults b.instanceEventNotifAttrs = snap.InstanceEventNotifAttrs - if snap.NiPermissions != nil { - b.niPermissions = snap.NiPermissions - } else { - b.niPermissions = make(map[string]*NetworkInterfacePermission) - } if snap.NiIPv6Addresses != nil { b.niIPv6Addresses = snap.NiIPv6Addresses } else { @@ -654,26 +270,11 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { } else { b.idFormatSettings = make(map[string]bool) } - if snap.EndpointConnectionNotifs != nil { - b.endpointConnectionNotifs = snap.EndpointConnectionNotifs - } else { - b.endpointConnectionNotifs = make(map[string]*VpcEndpointConnectionNotification) - } if snap.VpcEndpointServicePermissions != nil { b.vpcEndpointServicePermissions = snap.VpcEndpointServicePermissions } else { b.vpcEndpointServicePermissions = make(map[string][]string) } - if snap.SnapshotLocks != nil { - b.snapshotLocks = snap.SnapshotLocks - } else { - b.snapshotLocks = make(map[string]*SnapshotLock) - } - if snap.ReplaceRootVolumeTasks != nil { - b.replaceRootVolumeTasks = snap.ReplaceRootVolumeTasks - } else { - b.replaceRootVolumeTasks = make(map[string]*ReplaceRootVolumeTask) - } if snap.SubnetCIDRReservations != nil { b.subnetCIDRReservations = snap.SubnetCIDRReservations } else { @@ -704,46 +305,6 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { } else { b.vgwRoutePropagation = make(map[string]bool) } - if snap.ClientVpnEndpoints != nil { - b.clientVpnEndpoints = snap.ClientVpnEndpoints - } else { - b.clientVpnEndpoints = make(map[string]*ClientVpnEndpoint) - } - if snap.TgwConnects != nil { - b.tgwConnects = snap.TgwConnects - } else { - b.tgwConnects = make(map[string]*TransitGatewayConnect) - } - if snap.TgwConnectPeers != nil { - b.tgwConnectPeers = snap.TgwConnectPeers - } else { - b.tgwConnectPeers = make(map[string]*TransitGatewayConnectPeer) - } - if snap.TgwPrefixListRefs != nil { - b.tgwPrefixListRefs = snap.TgwPrefixListRefs - } else { - b.tgwPrefixListRefs = make(map[string]*TransitGatewayPrefixListReference) - } - if snap.VerifiedAccessEndpoints != nil { - b.verifiedAccessEndpoints = snap.VerifiedAccessEndpoints - } else { - b.verifiedAccessEndpoints = make(map[string]*VerifiedAccessEndpoint) - } - if snap.VerifiedAccessGroups != nil { - b.verifiedAccessGroups = snap.VerifiedAccessGroups - } else { - b.verifiedAccessGroups = make(map[string]*VerifiedAccessGroup) - } - if snap.VerifiedAccessInstances != nil { - b.verifiedAccessInstances = snap.VerifiedAccessInstances - } else { - b.verifiedAccessInstances = make(map[string]*VerifiedAccessInstance) - } - if snap.VerifiedAccessTrustProviders != nil { - b.verifiedAccessTrustProviders = snap.VerifiedAccessTrustProviders - } else { - b.verifiedAccessTrustProviders = make(map[string]*VerifiedAccessTrustProvider) - } if snap.VerifiedAccessEndpointPolicies != nil { b.verifiedAccessEndpointPolicies = snap.VerifiedAccessEndpointPolicies } else { @@ -754,51 +315,6 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { } else { b.verifiedAccessGroupPolicies = make(map[string]*VerifiedAccessPolicy) } - if snap.VerifiedAccessInstanceLoggingCfgs != nil { - b.verifiedAccessInstanceLoggingConfigs = snap.VerifiedAccessInstanceLoggingCfgs - } else { - b.verifiedAccessInstanceLoggingConfigs = make(map[string]*VerifiedAccessInstanceLoggingConfig) - } - if snap.FpgaImages != nil { - b.fpgaImages = snap.FpgaImages - } else { - b.fpgaImages = make(map[string]*FpgaImage) - } - if snap.InstanceConnectEndpoints != nil { - b.instanceConnectEndpoints = snap.InstanceConnectEndpoints - } else { - b.instanceConnectEndpoints = make(map[string]*InstanceConnectEndpoint) - } - if snap.InstanceEventWindows != nil { - b.instanceEventWindows = snap.InstanceEventWindows - } else { - b.instanceEventWindows = make(map[string]*InstanceEventWindow) - } - if snap.ImageImportTasks != nil { - b.imageImportTasks = snap.ImageImportTasks - } else { - b.imageImportTasks = make(map[string]*ImageImportTask) - } - if snap.SnapshotImportTasks != nil { - b.snapshotImportTasks = snap.SnapshotImportTasks - } else { - b.snapshotImportTasks = make(map[string]*SnapshotImportTask) - } - if snap.RecycleBinImages != nil { - b.recycleBinImages = snap.RecycleBinImages - } else { - b.recycleBinImages = make(map[string]*RecycleBinImage) - } - if snap.RecycleBinSnapshots != nil { - b.recycleBinSnapshots = snap.RecycleBinSnapshots - } else { - b.recycleBinSnapshots = make(map[string]*Snapshot) - } - if snap.RecycleBinVolumes != nil { - b.recycleBinVolumes = snap.RecycleBinVolumes - } else { - b.recycleBinVolumes = make(map[string]*RecycleBinVolume) - } if snap.FastLaunchImages != nil { b.fastLaunchImages = snap.FastLaunchImages } else { @@ -809,74 +325,10 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { } else { b.fastSnapshotRestores = make(map[string]bool) } - if snap.VpnConnectionRoutes != nil { - b.vpnConnectionRoutes = snap.VpnConnectionRoutes - } else { - b.vpnConnectionRoutes = make(map[string]*VpnConnectionRoute) - } b.spotDatafeed = snap.SpotDatafeed - if snap.TrafficMirrorFilters != nil { - b.trafficMirrorFilters = snap.TrafficMirrorFilters - } else { - b.trafficMirrorFilters = make(map[string]*TrafficMirrorFilter) - } - if snap.TrafficMirrorFilterRules != nil { - b.trafficMirrorFilterRules = snap.TrafficMirrorFilterRules - } else { - b.trafficMirrorFilterRules = make(map[string]*TrafficMirrorFilterRule) - } - if snap.TrafficMirrorSessions != nil { - b.trafficMirrorSessions = snap.TrafficMirrorSessions - } else { - b.trafficMirrorSessions = make(map[string]*TrafficMirrorSession) - } - if snap.TrafficMirrorTargets != nil { - b.trafficMirrorTargets = snap.TrafficMirrorTargets - } else { - b.trafficMirrorTargets = make(map[string]*TrafficMirrorTarget) - } - if snap.NetworkInsightsAnalyses != nil { - b.networkInsightsAnalyses = snap.NetworkInsightsAnalyses - } else { - b.networkInsightsAnalyses = make(map[string]*NetworkInsightsAnalysis) - } - if snap.NetworkInsightsAccessScopes != nil { - b.networkInsightsAccessScopes = snap.NetworkInsightsAccessScopes - } else { - b.networkInsightsAccessScopes = make(map[string]*NetworkInsightsAccessScope) - } - if snap.NetworkInsightsAccessScopeAnalyses != nil { - b.networkInsightsAccessScopeAnalyses = snap.NetworkInsightsAccessScopeAnalyses - } else { - b.networkInsightsAccessScopeAnalyses = make(map[string]*NetworkInsightsAccessScopeAnalysis) - } - if snap.ReservedInstances != nil { - b.reservedInstances = snap.ReservedInstances - } else { - b.reservedInstances = make(map[string]*ReservedInstance) - } - if snap.ReservedInstancesOfferings != nil { - b.reservedInstancesOfferings = snap.ReservedInstancesOfferings - } else { - b.reservedInstancesOfferings = make(map[string]*ReservedInstancesOffering) - } - if snap.ReservedInstancesListings != nil { - b.reservedInstancesListings = snap.ReservedInstancesListings - } else { - b.reservedInstancesListings = make(map[string]*ReservedInstancesListing) - } - if snap.ReservedInstancesModifications != nil { - b.reservedInstancesModifications = snap.ReservedInstancesModifications - } else { - b.reservedInstancesModifications = make(map[string]*ReservedInstancesModification) - } - b.restoreRouteServerFields(&snap) - b.restoreLocalGatewayFields(&snap) - b.restoreTGWMulticastFields(&snap) b.restoreVpcConfigFields(&snap) b.restoreCapacityFamilyFields(&snap) b.restoreNewFamilyFields(&snap) - b.restoreParitySweep2Fields(&snap) b.restoreParityFinalFields(&snap) return nil @@ -894,69 +346,15 @@ func (b *InMemoryBackend) restoreParityFinalFields(snap *backendSnapshot) { b.tgwRTPropagations = make(map[string]map[string]*TransitGatewayRouteTablePropagation) } - if snap.InterruptibleCRAllocations != nil { - b.interruptibleCRAllocations = snap.InterruptibleCRAllocations - } else { - b.interruptibleCRAllocations = make(map[string]*InterruptibleCapacityReservationAllocation) - } - - if snap.MovingAddresses != nil { - b.movingAddresses = snap.MovingAddresses - } else { - b.movingAddresses = make(map[string]*MovingAddressStatus) - } - b.reachabilityAnalyzerOrgSharing = snap.ReachabilityAnalyzerOrgSharing } -// restoreParitySweep2Fields copies the VPC Encryption Control, VPN Concentrator, -// Host Reservation, Declarative Policies, and AWS Network Performance state maps -// from snap into b. Must be called with b.mu held for writing. -func (b *InMemoryBackend) restoreParitySweep2Fields(snap *backendSnapshot) { - if snap.VpcEncryptionControls != nil { - b.vpcEncryptionControls = snap.VpcEncryptionControls - } else { - b.vpcEncryptionControls = make(map[string]*VpcEncryptionControl) - } - - if snap.VpnConcentrators != nil { - b.vpnConcentrators = snap.VpnConcentrators - } else { - b.vpnConcentrators = make(map[string]*VpnConcentrator) - } - - if snap.HostReservations != nil { - b.hostReservations = snap.HostReservations - } else { - b.hostReservations = make(map[string]*HostReservation) - } - - if snap.DeclarativePoliciesReports != nil { - b.declarativePoliciesReports = snap.DeclarativePoliciesReports - } else { - b.declarativePoliciesReports = make(map[string]*DeclarativePoliciesReport) - } - - if snap.NetworkPerformanceSubscriptions != nil { - b.networkPerformanceSubscriptions = snap.NetworkPerformanceSubscriptions - } else { - b.networkPerformanceSubscriptions = make(map[string]*NetworkPerformanceSubscription) - } -} - // restoreNewFamilyFields copies the Mac modification task / Secondary // Network / Secondary Subnet / Secondary Interface / Outpost LAG / Service // Link Virtual Interface / instance-attribute misc / SQL HA fields from snap // into b. Must be called with b.mu held for writing. func (b *InMemoryBackend) restoreNewFamilyFields(snap *backendSnapshot) { - b.macModificationTasks = snap.MacModificationTasks - b.secondaryNetworks = snap.SecondaryNetworks - b.secondarySubnets = snap.SecondarySubnets - b.secondaryInterfaces = snap.SecondaryInterfaces - b.serviceLinkVirtualInterfaces = snap.ServiceLinkVirtualInterfaces - b.outpostLags = snap.OutpostLags b.availabilityZoneGroupOptIns = snap.AvailabilityZoneGroupOptIns - b.sqlHaRegistrations = snap.SQLHaRegistrations b.sqlHaHistory = snap.SQLHaHistory } @@ -964,48 +362,6 @@ func (b *InMemoryBackend) restoreNewFamilyFields(snap *backendSnapshot) { // Block, and Capacity Manager state from snap into b. Split out to keep // Restore's growth in check. Must be called with b.mu held. func (b *InMemoryBackend) restoreCapacityFamilyFields(snap *backendSnapshot) { - if snap.CapacityReservationFleets != nil { - b.capacityReservationFleets = snap.CapacityReservationFleets - } else { - b.capacityReservationFleets = make(map[string]*CapacityReservationFleet) - } - - if snap.CapacityBlockOfferings != nil { - b.capacityBlockOfferings = snap.CapacityBlockOfferings - } else { - b.capacityBlockOfferings = make(map[string]*CapacityBlockOffering) - } - - if snap.CapacityBlockExtensionOfferings != nil { - b.capacityBlockExtensionOfferings = snap.CapacityBlockExtensionOfferings - } else { - b.capacityBlockExtensionOfferings = make(map[string]*CapacityBlockExtensionOffering) - } - - if snap.CapacityBlocks != nil { - b.capacityBlocks = snap.CapacityBlocks - } else { - b.capacityBlocks = make(map[string]*CapacityBlock) - } - - if snap.CapacityBlockExtensions != nil { - b.capacityBlockExtensions = snap.CapacityBlockExtensions - } else { - b.capacityBlockExtensions = make(map[string]*CapacityBlockExtension) - } - - if snap.CapacityReservationBillingReqs != nil { - b.capacityReservationBillingRequests = snap.CapacityReservationBillingReqs - } else { - b.capacityReservationBillingRequests = make(map[string]*CapacityReservationBillingRequest) - } - - if snap.CapacityManagerDataExports != nil { - b.capacityManagerDataExports = snap.CapacityManagerDataExports - } else { - b.capacityManagerDataExports = make(map[string]*CapacityManagerDataExport) - } - if snap.CapacityManagerState != nil { b.capacityManagerState = snap.CapacityManagerState } else { @@ -1017,12 +373,6 @@ func (b *InMemoryBackend) restoreCapacityFamilyFields(snap *backendSnapshot) { // state from snap into b. Split out to keep Restore's growth in check. // Must be called with b.mu held. func (b *InMemoryBackend) restoreVpcConfigFields(snap *backendSnapshot) { - if snap.ClassicLinkInstances != nil { - b.classicLinkInstances = snap.ClassicLinkInstances - } else { - b.classicLinkInstances = make(map[string]*ClassicLinkInstance) - } - if snap.VpcBlockPublicAccessOptions != nil { b.vpcBlockPublicAccessOptions = snap.VpcBlockPublicAccessOptions } else { @@ -1033,140 +383,11 @@ func (b *InMemoryBackend) restoreVpcConfigFields(snap *backendSnapshot) { ManagedBy: vpcBPAManagedByAccount, } } - - if snap.VpcBlockPublicAccessExclusions != nil { - b.vpcBlockPublicAccessExclusions = snap.VpcBlockPublicAccessExclusions - } else { - b.vpcBlockPublicAccessExclusions = make(map[string]*VpcBlockPublicAccessExclusion) - } -} - -// restoreTGWMulticastFields copies the transit gateway multicast domain and -// metering policy state maps from snap into b. Split out to keep Restore's -// growth in check. Must be called with b.mu held. -func (b *InMemoryBackend) restoreTGWMulticastFields(snap *backendSnapshot) { - if snap.TgwMulticastDomains != nil { - b.tgwMulticastDomains = snap.TgwMulticastDomains - } else { - b.tgwMulticastDomains = make(map[string]*TransitGatewayMulticastDomain) - } - - if snap.TgwMulticastGroupEntries != nil { - b.tgwMulticastGroupEntries = snap.TgwMulticastGroupEntries - } else { - b.tgwMulticastGroupEntries = make(map[string]*TransitGatewayMulticastGroupEntry) - } - - if snap.TgwMeteringPolicies != nil { - b.tgwMeteringPolicies = snap.TgwMeteringPolicies - } else { - b.tgwMeteringPolicies = make(map[string]*TransitGatewayMeteringPolicy) - } - - if snap.TgwMeteringPolicyEntries != nil { - b.tgwMeteringPolicyEntries = snap.TgwMeteringPolicyEntries - } else { - b.tgwMeteringPolicyEntries = make(map[string]*TransitGatewayMeteringPolicyEntry) - } -} - -// restoreRouteServerFields copies the Route Server state maps from snap into -// b. Split out to keep Restore's growth in check. Must be called with b.mu held. -func (b *InMemoryBackend) restoreRouteServerFields(snap *backendSnapshot) { - if snap.RouteServers != nil { - b.routeServers = snap.RouteServers - } else { - b.routeServers = make(map[string]*RouteServer) - } - if snap.RouteServerEndpoints != nil { - b.routeServerEndpoints = snap.RouteServerEndpoints - } else { - b.routeServerEndpoints = make(map[string]*RouteServerEndpoint) - } - if snap.RouteServerPeers != nil { - b.routeServerPeers = snap.RouteServerPeers - } else { - b.routeServerPeers = make(map[string]*RouteServerPeer) - } - if snap.RouteServerAssociations != nil { - b.routeServerAssociations = snap.RouteServerAssociations - } else { - b.routeServerAssociations = make(map[string]*RouteServerAssociation) - } - if snap.RouteServerPropagations != nil { - b.routeServerPropagations = snap.RouteServerPropagations - } else { - b.routeServerPropagations = make(map[string]*RouteServerPropagation) - } -} - -// restoreLocalGatewayFields copies the Local Gateway state maps from snap into -// b. Split out to keep Restore's growth in check. Must be called with b.mu held. -func (b *InMemoryBackend) restoreLocalGatewayFields(snap *backendSnapshot) { - if snap.LocalGateways != nil { - b.localGateways = snap.LocalGateways - } else { - b.localGateways = make(map[string]*LocalGateway) - } - if snap.LocalGatewayVirtualInterfaces != nil { - b.localGatewayVirtualInterfaces = snap.LocalGatewayVirtualInterfaces - } else { - b.localGatewayVirtualInterfaces = make(map[string]*LocalGatewayVirtualInterface) - } - if snap.LocalGatewayVirtualInterfaceGroups != nil { - b.localGatewayVirtualInterfaceGroups = snap.LocalGatewayVirtualInterfaceGroups - } else { - b.localGatewayVirtualInterfaceGroups = make(map[string]*LocalGatewayVirtualInterfaceGroup) - } - if snap.LocalGatewayRouteTables != nil { - b.localGatewayRouteTables = snap.LocalGatewayRouteTables - } else { - b.localGatewayRouteTables = make(map[string]*LocalGatewayRouteTable) - } - if snap.LocalGatewayRoutes != nil { - b.localGatewayRoutes = snap.LocalGatewayRoutes - } else { - b.localGatewayRoutes = make(map[string]*LocalGatewayRoute) - } - if snap.LocalGatewayRouteTableVpcAssocs != nil { - b.localGatewayRouteTableVpcAssociations = snap.LocalGatewayRouteTableVpcAssocs - } else { - b.localGatewayRouteTableVpcAssociations = make(map[string]*LocalGatewayRouteTableVpcAssociation) - } - if snap.LocalGatewayRTVifGroupAssocs != nil { - b.localGatewayRouteTableVifGroupAssociations = snap.LocalGatewayRTVifGroupAssocs - } else { - b.localGatewayRouteTableVifGroupAssociations = make( - map[string]*LocalGatewayRouteTableVirtualInterfaceGroupAssociation, - ) - } } // restoreCoreFields copies the core map/bool/scalar fields from snap into b. // Must be called with b.mu held for writing. func (b *InMemoryBackend) restoreCoreFields(snap *backendSnapshot) { - b.instances = snap.Instances - b.securityGroups = snap.SecurityGroups - b.vpcs = snap.VPCs - b.subnets = snap.Subnets - b.keyPairs = snap.KeyPairs - b.volumes = snap.Volumes - b.addresses = snap.Addresses - b.internetGateways = snap.InternetGateways - b.routeTables = snap.RouteTables - b.natGateways = snap.NatGateways - b.networkInterfaces = snap.NetworkInterfaces - b.spotRequests = snap.SpotRequests - b.placementGroups = snap.PlacementGroups - b.images = snap.Images - b.imageUsageReports = snap.ImageUsageReports - b.launchTemplates = snap.LaunchTemplates - b.vpcEndpoints = snap.VpcEndpoints - b.snapshots = snap.Snapshots - b.networkACLs = snap.NetworkACLs - b.transitGateways = snap.TransitGateways - b.flowLogs = snap.FlowLogs - b.dhcpOptionSets = snap.DhcpOptionSets b.tags = snap.Tags } @@ -1174,38 +395,10 @@ func (b *InMemoryBackend) restoreCoreFields(snap *backendSnapshot) { // Must be called with b.mu held for writing. func (b *InMemoryBackend) restoreExtendedFields(snap *backendSnapshot) { b.addressTransfers = snap.AddressTransfers - b.capacityReservations = snap.CapacityReservations - b.reservedInstancesExchanges = snap.ReservedInstancesExchanges - b.tgwMulticastDomainAssociations = snap.TGWMulticastDomainAssociations - b.tgwPeeringAttachments = snap.TGWPeeringAttachments - b.tgwVpcAttachments = snap.TGWVpcAttachments - b.vpcEndpointConnections = snap.VpcEndpointConnections - b.vpcPeeringConnections = snap.VpcPeeringConnections - b.byoipCidrs = snap.ByoipCidrs - b.dedicatedHosts = snap.DedicatedHosts - b.vpnGateways = snap.VpnGateways - b.customerGateways = snap.CustomerGateways - b.ipams = snap.Ipams - b.ipamScopes = snap.IpamScopes - b.ipamPools = snap.IpamPools b.ipamPoolCidrs = snap.IpamPoolCidrs - b.ipamPoolAllocations = snap.IpamPoolAllocations - b.ipamResourceDiscoveries = snap.IpamResourceDiscoveries - b.ipamResourceDiscoveryAssocs = snap.IpamResourceDiscoveryAssocs - b.ipamByoasns = snap.IpamByoasns - b.ipamAsnAssociations = snap.IpamAsnAssociations - b.ipamVerificationTokens = snap.IpamVerificationTokens - b.ipamResourceCidrs = snap.IpamResourceCidrs - b.ipamPrefixListResolvers = snap.IpamPrefixListResolvers b.ipamPrefixListResolverVersions = snap.IpamPrefixListResolverVersions - b.ipamPrefixListResolverTargets = snap.IpamPrefixListResolverTargets - b.ipamPolicies = snap.IpamPolicies b.ipamPolicyEnabledTargets = snap.IpamPolicyEnabledTargets b.ipamOrgAdminAccountID = snap.IpamOrgAdminAccountID - b.carrierGateways = snap.CarrierGateways - b.fleets = snap.Fleets - b.networkInsightsPaths = snap.NetworkInsightsPaths - b.managedPrefixLists = snap.ManagedPrefixLists b.ebsEncryptionByDefault = snap.EbsEncryptionByDefault b.serialConsoleAccess = snap.SerialConsoleAccess b.freePrivateIPs = snap.FreePrivateIPs @@ -1220,22 +413,10 @@ func (b *InMemoryBackend) restoreExtendedFields(snap *backendSnapshot) { // Allowed Images Settings / image task / usage report fields from snap into b. Must be // called with b.mu held for writing. func (b *InMemoryBackend) restoreImageAndPoolFields(snap *backendSnapshot) { - b.scheduledInstances = snap.ScheduledInstances b.scheduledInstanceLaunched = snap.ScheduledInstanceLaunched - b.coipPools = snap.CoipPools - b.coipCidrs = snap.CoipCidrs - b.ipv4Pools = snap.Ipv4Pools - b.ipv6Pools = snap.Ipv6Pools b.allowedImagesSettings = snap.AllowedImagesSettings - b.storeImageTasks = snap.StoreImageTasks - b.usageReports = snap.UsageReports b.usageReportEntries = snap.UsageReportEntries b.instanceProductCodes = snap.InstanceProductCodes - b.bundleTasks = snap.BundleTasks - b.conversionTasks = snap.ConversionTasks - b.exportTasks = snap.ExportTasks - b.exportImageTasks = snap.ExportImageTasks - b.trunkInterfaceAssociations = snap.TrunkInterfaceAssociations b.enclaveCertIamRoles = snap.EnclaveCertIamRoles } @@ -1244,64 +425,11 @@ func (b *InMemoryBackend) restoreImageAndPoolFields(snap *backendSnapshot) { // that never populated a particular resource type. func (s *backendSnapshot) initMissingMaps() { s.initCoreMaps() - s.initDeepDiveMaps() s.initNewOpsMaps() } // initCoreMaps initialises the original map fields. func (s *backendSnapshot) initCoreMaps() { - if s.Instances == nil { - s.Instances = make(map[string]*Instance) - } - - if s.SecurityGroups == nil { - s.SecurityGroups = make(map[string]*SecurityGroup) - } - - if s.VPCs == nil { - s.VPCs = make(map[string]*VPC) - } - - if s.Subnets == nil { - s.Subnets = make(map[string]*Subnet) - } - - if s.KeyPairs == nil { - s.KeyPairs = make(map[string]*KeyPair) - } - - if s.Volumes == nil { - s.Volumes = make(map[string]*Volume) - } - - if s.Addresses == nil { - s.Addresses = make(map[string]*Address) - } - - if s.InternetGateways == nil { - s.InternetGateways = make(map[string]*InternetGateway) - } - - if s.RouteTables == nil { - s.RouteTables = make(map[string]*RouteTable) - } - - if s.NatGateways == nil { - s.NatGateways = make(map[string]*NatGateway) - } - - if s.NetworkInterfaces == nil { - s.NetworkInterfaces = make(map[string]*NetworkInterface) - } - - if s.SpotRequests == nil { - s.SpotRequests = make(map[string]*SpotInstanceRequest) - } - - if s.PlacementGroups == nil { - s.PlacementGroups = make(map[string]*PlacementGroup) - } - if s.Tags == nil { s.Tags = make(map[string]map[string]string) } @@ -1314,122 +442,30 @@ func initMapIfNil[K comparable, V any](m *map[K]V) { } } -func (s *backendSnapshot) initDeepDiveMaps() { - initMapIfNil(&s.Images) - initMapIfNil(&s.ImageUsageReports) - initMapIfNil(&s.LaunchTemplates) - initMapIfNil(&s.VpcEndpoints) - initMapIfNil(&s.Snapshots) - initMapIfNil(&s.NetworkACLs) - initMapIfNil(&s.TransitGateways) - initMapIfNil(&s.FlowLogs) - initMapIfNil(&s.DhcpOptionSets) -} - // initNewOpsMaps initialises the map fields added for the new Accept/Advertise/Allocate operations. func (s *backendSnapshot) initNewOpsMaps() { if s.AddressTransfers == nil { s.AddressTransfers = make(map[string]*AddressTransfer) } - if s.CapacityReservations == nil { - s.CapacityReservations = make(map[string]*CapacityReservation) - } - - if s.ReservedInstancesExchanges == nil { - s.ReservedInstancesExchanges = make(map[string]*ReservedInstancesExchange) - } - - if s.TGWMulticastDomainAssociations == nil { - s.TGWMulticastDomainAssociations = make( - map[string]*TransitGatewayMulticastDomainAssociation, - ) - } - - if s.TGWPeeringAttachments == nil { - s.TGWPeeringAttachments = make(map[string]*TransitGatewayPeeringAttachment) - } - - if s.TGWVpcAttachments == nil { - s.TGWVpcAttachments = make(map[string]*TransitGatewayVpcAttachment) - } - - if s.VpcEndpointConnections == nil { - s.VpcEndpointConnections = make(map[string]*VpcEndpointConnection) - } - - if s.VpcPeeringConnections == nil { - s.VpcPeeringConnections = make(map[string]*VpcPeeringConnection) - } - - if s.ByoipCidrs == nil { - s.ByoipCidrs = make(map[string]*ByoipCidr) - } - - if s.DedicatedHosts == nil { - s.DedicatedHosts = make(map[string]*Host) - } - - initMapIfNil(&s.TgwPolicyTables) - initMapIfNil(&s.TgwPolicyTableAssociations) - initMapIfNil(&s.TgwRouteTableAnnouncements) - s.initAppendixMaps() } func (s *backendSnapshot) initAppendixMaps() { - initMapIfNil(&s.VpnGateways) - initMapIfNil(&s.CustomerGateways) - initMapIfNil(&s.Ipams) - initMapIfNil(&s.IpamScopes) - initMapIfNil(&s.IpamPools) initMapIfNil(&s.IpamPoolCidrs) - initMapIfNil(&s.IpamPoolAllocations) - initMapIfNil(&s.IpamResourceDiscoveries) - initMapIfNil(&s.IpamResourceDiscoveryAssocs) - initMapIfNil(&s.IpamByoasns) - initMapIfNil(&s.IpamAsnAssociations) - initMapIfNil(&s.IpamVerificationTokens) - initMapIfNil(&s.IpamResourceCidrs) - initMapIfNil(&s.IpamPrefixListResolvers) initMapIfNil(&s.IpamPrefixListResolverVersions) - initMapIfNil(&s.IpamPrefixListResolverTargets) - initMapIfNil(&s.IpamPolicies) initMapIfNil(&s.IpamPolicyEnabledTargets) - initMapIfNil(&s.CarrierGateways) - initMapIfNil(&s.Fleets) - initMapIfNil(&s.NetworkInsightsPaths) - initMapIfNil(&s.ManagedPrefixLists) s.initImageAndPoolMaps() } // initImageAndPoolMaps initialises the map fields added for Scheduled Instances, COIP, // public IPv4/IPv6 pools, Allowed Images Settings, and image tasks/usage reports. func (s *backendSnapshot) initImageAndPoolMaps() { - initMapIfNil(&s.ScheduledInstances) initMapIfNil(&s.ScheduledInstanceLaunched) - initMapIfNil(&s.CoipPools) - initMapIfNil(&s.CoipCidrs) - initMapIfNil(&s.Ipv4Pools) - initMapIfNil(&s.Ipv6Pools) - initMapIfNil(&s.StoreImageTasks) - initMapIfNil(&s.UsageReports) initMapIfNil(&s.UsageReportEntries) initMapIfNil(&s.InstanceProductCodes) - initMapIfNil(&s.BundleTasks) - initMapIfNil(&s.ConversionTasks) - initMapIfNil(&s.ExportTasks) - initMapIfNil(&s.ExportImageTasks) - initMapIfNil(&s.TrunkInterfaceAssociations) initMapIfNil(&s.EnclaveCertIamRoles) - initMapIfNil(&s.MacModificationTasks) - initMapIfNil(&s.SecondaryNetworks) - initMapIfNil(&s.SecondarySubnets) - initMapIfNil(&s.SecondaryInterfaces) - initMapIfNil(&s.ServiceLinkVirtualInterfaces) - initMapIfNil(&s.OutpostLags) initMapIfNil(&s.AvailabilityZoneGroupOptIns) - initMapIfNil(&s.SQLHaRegistrations) initMapIfNil(&s.SQLHaHistory) if s.AllowedImagesSettings == nil { diff --git a/services/ec2/store_setup.go b/services/ec2/store_setup.go new file mode 100644 index 000000000..64bd39d30 --- /dev/null +++ b/services/ec2/store_setup.go @@ -0,0 +1,972 @@ +package ec2 + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend is registered exactly once, +// here, as a *store.Table[T] on b.registry. See pkgs/store's package doc and +// the services/sqs pilot (commit 0f09d77c) for the pattern this follows. +// +// A handful of fields are deliberately NOT registered here and remain plain +// maps -- see the comment block above registerAllTables for the list and why. +import ( + "strconv" + + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +func addressAttributesKeyFn(v *AddressAttribute) string { return v.AllocationID } +func addressesKeyFn(v *Address) string { return v.AllocationID } +func bundleTasksKeyFn(v *BundleTask) string { return v.BundleID } +func byoipCidrsKeyFn(v *ByoipCidr) string { return v.Cidr } +func capacityBlockExtensionOfferingsKeyFn(v *CapacityBlockExtensionOffering) string { + return v.CapacityBlockExtensionOfferingID +} +func capacityBlockExtensionsKeyFn(v *CapacityBlockExtension) string { + return v.CapacityBlockExtensionOfferingID +} +func capacityBlockOfferingsKeyFn(v *CapacityBlockOffering) string { return v.CapacityBlockOfferingID } +func capacityBlocksKeyFn(v *CapacityBlock) string { return v.CapacityBlockID } +func capacityManagerDataExportsKeyFn(v *CapacityManagerDataExport) string { + return v.CapacityManagerDataExportID +} +func capacityReservationBillingRequestsKeyFn(v *CapacityReservationBillingRequest) string { + return v.CapacityReservationID +} +func capacityReservationFleetsKeyFn(v *CapacityReservationFleet) string { + return v.CapacityReservationFleetID +} +func capacityReservationsKeyFn(v *CapacityReservation) string { return v.CapacityReservationID } +func carrierGatewaysKeyFn(v *CarrierGateway) string { return v.CarrierGatewayID } +func classicLinkInstancesKeyFn(v *ClassicLinkInstance) string { return v.InstanceID } +func clientVpnEndpointsKeyFn(v *ClientVpnEndpoint) string { return v.ClientVpnEndpointID } +func coipCidrsKeyFn(v *CoipCidr) string { return coipCidrKey(v.CoipPoolID, v.Cidr) } +func coipPoolsKeyFn(v *CoipPool) string { return v.PoolID } +func conversionTasksKeyFn(v *ConversionTask) string { return v.ConversionTaskID } +func customerGatewaysKeyFn(v *CustomerGateway) string { return v.CustomerGatewayID } +func declarativePoliciesReportsKeyFn(v *DeclarativePoliciesReport) string { return v.ReportID } +func dedicatedHostsKeyFn(v *Host) string { return v.HostID } +func dhcpOptionSetsKeyFn(v *DhcpOptions) string { return v.DhcpOptionsID } +func egressOnlyIGWsKeyFn(v *EgressOnlyInternetGateway) string { return v.ID } +func endpointConnectionNotifsKeyFn(v *VpcEndpointConnectionNotification) string { + return v.ConnectionNotificationID +} +func exportImageTasksKeyFn(v *ExportImageTaskRec) string { return v.ExportImageTaskID } +func exportTasksKeyFn(v *ExportTask) string { return v.ExportTaskID } +func fleetsKeyFn(v *Fleet) string { return v.FleetID } +func flowLogsKeyFn(v *FlowLog) string { return v.FlowLogID } +func fpgaImagesKeyFn(v *FpgaImage) string { return v.FpgaImageID } +func hostReservationsKeyFn(v *HostReservation) string { return v.HostReservationID } +func iamAssociationsKeyFn(v *IamInstanceProfileAssociation) string { return v.AssociationID } +func imageImportTasksKeyFn(v *ImageImportTask) string { return v.ImportTaskID } +func imageUsageReportsKeyFn(v *ImageUsageReport) string { return v.ImageID } +func imagesKeyFn(v *AMIStub) string { return v.ImageID } +func instanceConnectEndpointsKeyFn(v *InstanceConnectEndpoint) string { + return v.InstanceConnectEndpointID +} +func instanceEventWindowsKeyFn(v *InstanceEventWindow) string { return v.InstanceEventWindowID } +func instancesKeyFn(v *Instance) string { return v.ID } +func internetGatewaysKeyFn(v *InternetGateway) string { return v.ID } +func interruptibleCRAllocationsKeyFn(v *InterruptibleCapacityReservationAllocation) string { + return v.SourceCapacityReservationID +} +func ipamAsnAssociationsKeyFn(v *IpamAsnAssociation) string { return v.Asn + "|" + v.Cidr } +func ipamByoasnsKeyFn(v *IpamByoasn) string { return v.Asn } +func ipamPoliciesKeyFn(v *IpamPolicy) string { return v.IpamPolicyID } +func ipamPoolAllocationsKeyFn(v *IpamPoolAllocation) string { return v.IpamPoolAllocationID } +func ipamPoolsKeyFn(v *IpamPool) string { return v.IpamPoolID } +func ipamPrefixListResolverTargetsKeyFn(v *IpamPrefixListResolverTarget) string { + return v.IpamPrefixListResolverTargetID +} +func ipamPrefixListResolversKeyFn(v *IpamPrefixListResolver) string { + return v.IpamPrefixListResolverID +} +func ipamResourceCidrsKeyFn(v *IpamResourceCidr) string { + return ipamResourceCidrKey(v.ResourceID, v.ResourceCidr) +} +func ipamResourceDiscoveriesKeyFn(v *IpamResourceDiscovery) string { return v.IpamResourceDiscoveryID } +func ipamResourceDiscoveryAssocsKeyFn(v *IpamResourceDiscoveryAssociation) string { + return v.IpamResourceDiscoveryAssociationID +} +func ipamScopesKeyFn(v *IpamScope) string { return v.IpamScopeID } +func ipamVerificationTokensKeyFn(v *IpamExternalResourceVerificationToken) string { + return v.IpamExternalResourceVerificationTokenID +} +func ipamsKeyFn(v *Ipam) string { return v.IpamID } +func ipv4PoolsKeyFn(v *Ipv4Pool) string { return v.PoolID } +func ipv6PoolsKeyFn(v *Ipv6Pool) string { return v.PoolID } +func keyPairsKeyFn(v *KeyPair) string { return v.Name } +func launchTemplatesKeyFn(v *LaunchTemplate) string { return v.ID } +func localGatewayRouteTableVifGroupAssociationsKeyFn(v *LocalGatewayRouteTableVirtualInterfaceGroupAssociation) string { + return v.LocalGatewayRouteTableVirtualInterfaceGroupAssociationID +} +func localGatewayRouteTableVpcAssociationsKeyFn(v *LocalGatewayRouteTableVpcAssociation) string { + return v.LocalGatewayRouteTableVpcAssociationID +} +func localGatewayRouteTablesKeyFn(v *LocalGatewayRouteTable) string { + return v.LocalGatewayRouteTableID +} +func localGatewayRoutesKeyFn(v *LocalGatewayRoute) string { + return localGatewayRouteKey(v.LocalGatewayRouteTableID, v.DestinationCidrBlock, v.DestinationPrefixListID) +} +func localGatewayVirtualInterfaceGroupsKeyFn(v *LocalGatewayVirtualInterfaceGroup) string { + return v.LocalGatewayVirtualInterfaceGroupID +} +func localGatewayVirtualInterfacesKeyFn(v *LocalGatewayVirtualInterface) string { + return v.LocalGatewayVirtualInterfaceID +} +func localGatewaysKeyFn(v *LocalGateway) string { return v.LocalGatewayID } +func macModificationTasksKeyFn(v *MacModificationTask) string { return v.MacModificationTaskID } +func managedPrefixListsKeyFn(v *ManagedPrefixList) string { return v.PrefixListID } +func movingAddressesKeyFn(v *MovingAddressStatus) string { return v.PublicIP } +func natGatewaysKeyFn(v *NatGateway) string { return v.ID } +func networkACLsKeyFn(v *StoredNetworkACL) string { return v.ID } +func networkInsightsAccessScopeAnalysesKeyFn(v *NetworkInsightsAccessScopeAnalysis) string { + return v.NetworkInsightsAccessScopeAnalysisID +} +func networkInsightsAccessScopesKeyFn(v *NetworkInsightsAccessScope) string { + return v.NetworkInsightsAccessScopeID +} +func networkInsightsAnalysesKeyFn(v *NetworkInsightsAnalysis) string { + return v.NetworkInsightsAnalysisID +} +func networkInsightsPathsKeyFn(v *NetworkInsightsPath) string { return v.NetworkInsightsPathID } +func networkInterfacesKeyFn(v *NetworkInterface) string { return v.ID } +func networkPerformanceSubscriptionsKeyFn(v *NetworkPerformanceSubscription) string { + return networkPerformanceSubscriptionKey(v.Source, v.Destination, v.Metric, v.Statistic) +} +func niPermissionsKeyFn(v *NetworkInterfacePermission) string { return v.PermissionID } +func outpostLagsKeyFn(v *OutpostLag) string { return v.OutpostLagID } +func placementGroupsKeyFn(v *PlacementGroup) string { return v.Name } +func recycleBinImagesKeyFn(v *RecycleBinImage) string { return v.ImageID } +func recycleBinSnapshotsKeyFn(v *Snapshot) string { return v.SnapshotID } +func recycleBinVolumesKeyFn(v *RecycleBinVolume) string { return v.VolumeID } +func replaceRootVolumeTasksKeyFn(v *ReplaceRootVolumeTask) string { return v.ReplaceRootVolumeTaskID } +func reservedInstancesKeyFn(v *ReservedInstance) string { return v.ReservedInstancesID } +func reservedInstancesExchangesKeyFn(v *ReservedInstancesExchange) string { return v.ExchangeID } +func reservedInstancesListingsKeyFn(v *ReservedInstancesListing) string { + return v.ReservedInstancesListingID +} +func reservedInstancesModificationsKeyFn(v *ReservedInstancesModification) string { + return v.ReservedInstancesModificationID +} +func reservedInstancesOfferingsKeyFn(v *ReservedInstancesOffering) string { + return v.ReservedInstancesOfferingID +} +func routeServerAssociationsKeyFn(v *RouteServerAssociation) string { + return v.RouteServerID + "/" + v.VpcID +} +func routeServerEndpointsKeyFn(v *RouteServerEndpoint) string { return v.RouteServerEndpointID } +func routeServerPeersKeyFn(v *RouteServerPeer) string { return v.RouteServerPeerID } +func routeServerPropagationsKeyFn(v *RouteServerPropagation) string { + return v.RouteServerID + "/" + v.RouteTableID +} +func routeServersKeyFn(v *RouteServer) string { return v.RouteServerID } +func routeTablesKeyFn(v *RouteTable) string { return v.ID } +func scheduledInstancesKeyFn(v *ScheduledInstance) string { return v.ScheduledInstanceID } +func secondaryInterfacesKeyFn(v *SecondaryInterface) string { return v.SecondaryInterfaceID } +func secondaryNetworksKeyFn(v *SecondaryNetwork) string { return v.SecondaryNetworkID } +func secondarySubnetsKeyFn(v *SecondarySubnet) string { return v.SecondarySubnetID } +func securityGroupsKeyFn(v *SecurityGroup) string { return v.ID } +func serviceLinkVirtualInterfacesKeyFn(v *ServiceLinkVirtualInterface) string { + return v.ServiceLinkVirtualInterfaceID +} +func snapshotImportTasksKeyFn(v *SnapshotImportTask) string { return v.ImportTaskID } +func snapshotLocksKeyFn(v *SnapshotLock) string { return v.SnapshotID } +func snapshotsKeyFn(v *Snapshot) string { return v.SnapshotID } +func spotFleetsKeyFn(v *SpotFleetRequest) string { return v.SpotFleetRequestID } +func spotRequestsKeyFn(v *SpotInstanceRequest) string { return v.ID } +func sqlHaRegistrationsKeyFn(v *RegisteredSQLHaInstance) string { return v.InstanceID } +func storeImageTasksKeyFn(v *StoreImageTask) string { return v.AmiID } +func subnetsKeyFn(v *Subnet) string { return v.ID } +func tgwConnectPeersKeyFn(v *TransitGatewayConnectPeer) string { return v.TransitGatewayConnectPeerID } +func tgwConnectsKeyFn(v *TransitGatewayConnect) string { return v.TransitGatewayAttachmentID } +func tgwMeteringPoliciesKeyFn(v *TransitGatewayMeteringPolicy) string { return v.ID } +func tgwMeteringPolicyEntriesKeyFn(v *TransitGatewayMeteringPolicyEntry) string { + return v.TransitGatewayMeteringPolicyID + ":" + strconv.Itoa(v.PolicyRuleNumber) +} +func tgwMulticastDomainAssociationsKeyFn(v *TransitGatewayMulticastDomainAssociation) string { + return v.TransitGatewayMulticastDomainID + ":" + v.SubnetID +} +func tgwMulticastDomainsKeyFn(v *TransitGatewayMulticastDomain) string { return v.ID } +func tgwMulticastGroupEntriesKeyFn(v *TransitGatewayMulticastGroupEntry) string { + return v.TransitGatewayMulticastDomainID + ":" + v.GroupIPAddress + ":" + v.NetworkInterfaceID +} +func tgwPeeringAttachmentsKeyFn(v *TransitGatewayPeeringAttachment) string { + return v.TransitGatewayAttachmentID +} +func tgwPolicyTableAssociationsKeyFn(v *TransitGatewayPolicyTableAssociation) string { + return v.TransitGatewayPolicyTableID + ":" + v.TransitGatewayAttachmentID +} +func tgwPolicyTablesKeyFn(v *TransitGatewayPolicyTable) string { return v.TransitGatewayPolicyTableID } +func tgwPrefixListRefsKeyFn(v *TransitGatewayPrefixListReference) string { + return v.TransitGatewayRouteTableID + "/" + v.PrefixListID +} +func tgwRTAssociationsKeyFn(v *TransitGatewayRouteTableAssociation) string { + return v.TransitGatewayRouteTableID + ":" + v.TransitGatewayAttachmentID +} +func tgwRouteTableAnnouncementsKeyFn(v *TransitGatewayRouteTableAnnouncement) string { + return v.TransitGatewayRouteTableAnnouncementID +} +func tgwRouteTablesKeyFn(v *TransitGatewayRouteTable) string { return v.RouteTableID } +func tgwRoutesKeyFn(v *TransitGatewayRoute) string { + return v.TransitGatewayRouteTableID + ":" + v.DestinationCidrBlock +} +func tgwVpcAttachmentsKeyFn(v *TransitGatewayVpcAttachment) string { + return v.TransitGatewayAttachmentID +} +func trafficMirrorFilterRulesKeyFn(v *TrafficMirrorFilterRule) string { + return v.TrafficMirrorFilterRuleID +} +func trafficMirrorFiltersKeyFn(v *TrafficMirrorFilter) string { return v.TrafficMirrorFilterID } +func trafficMirrorSessionsKeyFn(v *TrafficMirrorSession) string { return v.TrafficMirrorSessionID } +func trafficMirrorTargetsKeyFn(v *TrafficMirrorTarget) string { return v.TrafficMirrorTargetID } +func transitGatewaysKeyFn(v *TransitGateway) string { return v.ID } +func trunkInterfaceAssociationsKeyFn(v *TrunkInterfaceAssociation) string { return v.AssociationID } +func usageReportsKeyFn(v *UsageReport) string { return v.ReportID } +func verifiedAccessEndpointsKeyFn(v *VerifiedAccessEndpoint) string { + return v.VerifiedAccessEndpointID +} +func verifiedAccessGroupsKeyFn(v *VerifiedAccessGroup) string { return v.VerifiedAccessGroupID } +func verifiedAccessInstanceLoggingConfigsKeyFn(v *VerifiedAccessInstanceLoggingConfig) string { + return v.VerifiedAccessInstanceID +} +func verifiedAccessInstancesKeyFn(v *VerifiedAccessInstance) string { + return v.VerifiedAccessInstanceID +} +func verifiedAccessTrustProvidersKeyFn(v *VerifiedAccessTrustProvider) string { + return v.VerifiedAccessTrustProviderID +} +func volumeModificationsKeyFn(v *VolumeModification) string { return v.VolumeID } +func volumesKeyFn(v *Volume) string { return v.ID } +func vpcBlockPublicAccessExclusionsKeyFn(v *VpcBlockPublicAccessExclusion) string { + return v.ExclusionID +} +func vpcEncryptionControlsKeyFn(v *VpcEncryptionControl) string { return v.VpcEncryptionControlID } +func vpcEndpointConnectionsKeyFn(v *VpcEndpointConnection) string { + return v.ServiceID + ":" + v.VpcEndpointID +} +func vpcEndpointServiceConfigsKeyFn(v *VpcEndpointServiceConfig) string { return v.ServiceID } +func vpcEndpointsKeyFn(v *VpcEndpoint) string { return v.ID } +func vpcPeeringConnectionsKeyFn(v *VpcPeeringConnection) string { return v.VpcPeeringConnectionID } +func vpcsKeyFn(v *VPC) string { return v.ID } +func vpnConcentratorsKeyFn(v *VpnConcentrator) string { return v.VpnConcentratorID } +func vpnConnectionRoutesKeyFn(v *VpnConnectionRoute) string { + return v.VpnConnectionID + ":" + v.DestinationCIDR +} +func vpnConnectionsKeyFn(v *VpnConnection) string { return v.VpnConnectionID } +func vpnGatewaysKeyFn(v *VpnGateway) string { return v.VpnGatewayID } + +// registerAllTables registers every converted resource map on b.registry +// exactly once. It must be called during construction only (immediately after +// b.registry is created), never on every Reset() -- store.Register panics on a +// duplicate name, so runtime resets go through registry.ResetAll() instead +// (see InMemoryBackend.Reset in backend_accept_ops.go). +// +// The following resource fields are deliberately left as plain maps (not +// registered here) because their key is not a pure function of the stored +// value's own fields, which store.Table requires: +// - addressTransfers: mixed keying convention across call sites (AllocationID +// in normal flow vs PublicIP in AddAddressTransferInternal test-seed +// helper) -- pre-existing quirk, not a pure function of value identity +// - instanceIMDSOptions: value type IMDSOptions carries no identity field of +// its own; keyed externally by instanceID +// - verifiedAccessEndpointPolicies: value type VerifiedAccessPolicy carries +// no identity field; keyed externally by endpoint ID +// - verifiedAccessGroupPolicies: value type VerifiedAccessPolicy carries no +// identity field; keyed externally by group ID (shares type with +// verifiedAccessEndpointPolicies) +// - vpcCidrAssociations: key composite (vpcID+":"+AssociationID) requires +// vpcID which is not stored on VpcCidrBlockAssociation value +// - vpcPeeringOptions: value type PeeringConnectionOptions carries no +// identity field of its own; keyed externally by peeringID +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call to the concrete field and value type. A closure list (rather than one +// statement per field in a single function body) keeps registerAllTables small +// regardless of how many resource tables the backend grows to. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.addressAttributes = store.Register(b.registry, "addressAttributes", store.New(addressAttributesKeyFn)) + }, + func(b *InMemoryBackend) { + b.addresses = store.Register(b.registry, "addresses", store.New(addressesKeyFn)) + }, + func(b *InMemoryBackend) { + b.bundleTasks = store.Register(b.registry, "bundleTasks", store.New(bundleTasksKeyFn)) + }, + func(b *InMemoryBackend) { + b.byoipCidrs = store.Register(b.registry, "byoipCidrs", store.New(byoipCidrsKeyFn)) + }, + func(b *InMemoryBackend) { + b.capacityBlockExtensionOfferings = store.Register( + b.registry, + "capacityBlockExtensionOfferings", + store.New(capacityBlockExtensionOfferingsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.capacityBlockExtensions = store.Register( + b.registry, + "capacityBlockExtensions", + store.New(capacityBlockExtensionsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.capacityBlockOfferings = store.Register( + b.registry, + "capacityBlockOfferings", + store.New(capacityBlockOfferingsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.capacityBlocks = store.Register(b.registry, "capacityBlocks", store.New(capacityBlocksKeyFn)) + }, + func(b *InMemoryBackend) { + b.capacityManagerDataExports = store.Register( + b.registry, + "capacityManagerDataExports", + store.New(capacityManagerDataExportsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.capacityReservationBillingRequests = store.Register( + b.registry, + "capacityReservationBillingRequests", + store.New(capacityReservationBillingRequestsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.capacityReservationFleets = store.Register( + b.registry, + "capacityReservationFleets", + store.New(capacityReservationFleetsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.capacityReservations = store.Register( + b.registry, + "capacityReservations", + store.New(capacityReservationsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.carrierGateways = store.Register(b.registry, "carrierGateways", store.New(carrierGatewaysKeyFn)) + }, + func(b *InMemoryBackend) { + b.classicLinkInstances = store.Register( + b.registry, + "classicLinkInstances", + store.New(classicLinkInstancesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.clientVpnEndpoints = store.Register(b.registry, "clientVpnEndpoints", store.New(clientVpnEndpointsKeyFn)) + }, + func(b *InMemoryBackend) { + b.coipCidrs = store.Register(b.registry, "coipCidrs", store.New(coipCidrsKeyFn)) + }, + func(b *InMemoryBackend) { + b.coipPools = store.Register(b.registry, "coipPools", store.New(coipPoolsKeyFn)) + }, + func(b *InMemoryBackend) { + b.conversionTasks = store.Register(b.registry, "conversionTasks", store.New(conversionTasksKeyFn)) + }, + func(b *InMemoryBackend) { + b.customerGateways = store.Register(b.registry, "customerGateways", store.New(customerGatewaysKeyFn)) + }, + func(b *InMemoryBackend) { + b.declarativePoliciesReports = store.Register( + b.registry, + "declarativePoliciesReports", + store.New(declarativePoliciesReportsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.dedicatedHosts = store.Register(b.registry, "dedicatedHosts", store.New(dedicatedHostsKeyFn)) + }, + func(b *InMemoryBackend) { + b.dhcpOptionSets = store.Register(b.registry, "dhcpOptionSets", store.New(dhcpOptionSetsKeyFn)) + }, + func(b *InMemoryBackend) { + b.egressOnlyIGWs = store.Register(b.registry, "egressOnlyIGWs", store.New(egressOnlyIGWsKeyFn)) + }, + func(b *InMemoryBackend) { + b.endpointConnectionNotifs = store.Register( + b.registry, + "endpointConnectionNotifs", + store.New(endpointConnectionNotifsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.exportImageTasks = store.Register(b.registry, "exportImageTasks", store.New(exportImageTasksKeyFn)) + }, + func(b *InMemoryBackend) { + b.exportTasks = store.Register(b.registry, "exportTasks", store.New(exportTasksKeyFn)) + }, + func(b *InMemoryBackend) { + b.fleets = store.Register(b.registry, "fleets", store.New(fleetsKeyFn)) + }, + func(b *InMemoryBackend) { + b.flowLogs = store.Register(b.registry, "flowLogs", store.New(flowLogsKeyFn)) + }, + func(b *InMemoryBackend) { + b.fpgaImages = store.Register(b.registry, "fpgaImages", store.New(fpgaImagesKeyFn)) + }, + func(b *InMemoryBackend) { + b.hostReservations = store.Register(b.registry, "hostReservations", store.New(hostReservationsKeyFn)) + }, + func(b *InMemoryBackend) { + b.iamAssociations = store.Register(b.registry, "iamAssociations", store.New(iamAssociationsKeyFn)) + }, + func(b *InMemoryBackend) { + b.imageImportTasks = store.Register(b.registry, "imageImportTasks", store.New(imageImportTasksKeyFn)) + }, + func(b *InMemoryBackend) { + b.imageUsageReports = store.Register(b.registry, "imageUsageReports", store.New(imageUsageReportsKeyFn)) + }, + func(b *InMemoryBackend) { + b.images = store.Register(b.registry, "images", store.New(imagesKeyFn)) + }, + func(b *InMemoryBackend) { + b.instanceConnectEndpoints = store.Register( + b.registry, + "instanceConnectEndpoints", + store.New(instanceConnectEndpointsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.instanceEventWindows = store.Register( + b.registry, + "instanceEventWindows", + store.New(instanceEventWindowsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.instances = store.Register(b.registry, "instances", store.New(instancesKeyFn)) + }, + func(b *InMemoryBackend) { + b.internetGateways = store.Register(b.registry, "internetGateways", store.New(internetGatewaysKeyFn)) + }, + func(b *InMemoryBackend) { + b.interruptibleCRAllocations = store.Register( + b.registry, + "interruptibleCRAllocations", + store.New(interruptibleCRAllocationsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.ipamAsnAssociations = store.Register(b.registry, "ipamAsnAssociations", store.New(ipamAsnAssociationsKeyFn)) + }, + func(b *InMemoryBackend) { + b.ipamByoasns = store.Register(b.registry, "ipamByoasns", store.New(ipamByoasnsKeyFn)) + }, + func(b *InMemoryBackend) { + b.ipamPolicies = store.Register(b.registry, "ipamPolicies", store.New(ipamPoliciesKeyFn)) + }, + func(b *InMemoryBackend) { + b.ipamPoolAllocations = store.Register(b.registry, "ipamPoolAllocations", store.New(ipamPoolAllocationsKeyFn)) + }, + func(b *InMemoryBackend) { + b.ipamPools = store.Register(b.registry, "ipamPools", store.New(ipamPoolsKeyFn)) + }, + func(b *InMemoryBackend) { + b.ipamPrefixListResolverTargets = store.Register( + b.registry, + "ipamPrefixListResolverTargets", + store.New(ipamPrefixListResolverTargetsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.ipamPrefixListResolvers = store.Register( + b.registry, + "ipamPrefixListResolvers", + store.New(ipamPrefixListResolversKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.ipamResourceCidrs = store.Register(b.registry, "ipamResourceCidrs", store.New(ipamResourceCidrsKeyFn)) + }, + func(b *InMemoryBackend) { + b.ipamResourceDiscoveries = store.Register( + b.registry, + "ipamResourceDiscoveries", + store.New(ipamResourceDiscoveriesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.ipamResourceDiscoveryAssocs = store.Register( + b.registry, + "ipamResourceDiscoveryAssocs", + store.New(ipamResourceDiscoveryAssocsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.ipamScopes = store.Register(b.registry, "ipamScopes", store.New(ipamScopesKeyFn)) + }, + func(b *InMemoryBackend) { + b.ipamVerificationTokens = store.Register( + b.registry, + "ipamVerificationTokens", + store.New(ipamVerificationTokensKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.ipams = store.Register(b.registry, "ipams", store.New(ipamsKeyFn)) + }, + func(b *InMemoryBackend) { + b.ipv4Pools = store.Register(b.registry, "ipv4Pools", store.New(ipv4PoolsKeyFn)) + }, + func(b *InMemoryBackend) { + b.ipv6Pools = store.Register(b.registry, "ipv6Pools", store.New(ipv6PoolsKeyFn)) + }, + func(b *InMemoryBackend) { + b.keyPairs = store.Register(b.registry, "keyPairs", store.New(keyPairsKeyFn)) + }, + func(b *InMemoryBackend) { + b.launchTemplates = store.Register(b.registry, "launchTemplates", store.New(launchTemplatesKeyFn)) + }, + func(b *InMemoryBackend) { + b.localGatewayRouteTableVifGroupAssociations = store.Register( + b.registry, + "localGatewayRouteTableVifGroupAssociations", + store.New(localGatewayRouteTableVifGroupAssociationsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.localGatewayRouteTableVpcAssociations = store.Register( + b.registry, + "localGatewayRouteTableVpcAssociations", + store.New(localGatewayRouteTableVpcAssociationsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.localGatewayRouteTables = store.Register( + b.registry, + "localGatewayRouteTables", + store.New(localGatewayRouteTablesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.localGatewayRoutes = store.Register(b.registry, "localGatewayRoutes", store.New(localGatewayRoutesKeyFn)) + }, + func(b *InMemoryBackend) { + b.localGatewayVirtualInterfaceGroups = store.Register( + b.registry, + "localGatewayVirtualInterfaceGroups", + store.New(localGatewayVirtualInterfaceGroupsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.localGatewayVirtualInterfaces = store.Register( + b.registry, + "localGatewayVirtualInterfaces", + store.New(localGatewayVirtualInterfacesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.localGateways = store.Register(b.registry, "localGateways", store.New(localGatewaysKeyFn)) + }, + func(b *InMemoryBackend) { + b.macModificationTasks = store.Register( + b.registry, + "macModificationTasks", + store.New(macModificationTasksKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.managedPrefixLists = store.Register(b.registry, "managedPrefixLists", store.New(managedPrefixListsKeyFn)) + }, + func(b *InMemoryBackend) { + b.movingAddresses = store.Register(b.registry, "movingAddresses", store.New(movingAddressesKeyFn)) + }, + func(b *InMemoryBackend) { + b.natGateways = store.Register(b.registry, "natGateways", store.New(natGatewaysKeyFn)) + }, + func(b *InMemoryBackend) { + b.networkACLs = store.Register(b.registry, "networkACLs", store.New(networkACLsKeyFn)) + }, + func(b *InMemoryBackend) { + b.networkInsightsAccessScopeAnalyses = store.Register( + b.registry, + "networkInsightsAccessScopeAnalyses", + store.New(networkInsightsAccessScopeAnalysesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.networkInsightsAccessScopes = store.Register( + b.registry, + "networkInsightsAccessScopes", + store.New(networkInsightsAccessScopesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.networkInsightsAnalyses = store.Register( + b.registry, + "networkInsightsAnalyses", + store.New(networkInsightsAnalysesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.networkInsightsPaths = store.Register( + b.registry, + "networkInsightsPaths", + store.New(networkInsightsPathsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.networkInterfaces = store.Register(b.registry, "networkInterfaces", store.New(networkInterfacesKeyFn)) + }, + func(b *InMemoryBackend) { + b.networkPerformanceSubscriptions = store.Register( + b.registry, + "networkPerformanceSubscriptions", + store.New(networkPerformanceSubscriptionsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.niPermissions = store.Register(b.registry, "niPermissions", store.New(niPermissionsKeyFn)) + }, + func(b *InMemoryBackend) { + b.outpostLags = store.Register(b.registry, "outpostLags", store.New(outpostLagsKeyFn)) + }, + func(b *InMemoryBackend) { + b.placementGroups = store.Register(b.registry, "placementGroups", store.New(placementGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.recycleBinImages = store.Register(b.registry, "recycleBinImages", store.New(recycleBinImagesKeyFn)) + }, + func(b *InMemoryBackend) { + b.recycleBinSnapshots = store.Register(b.registry, "recycleBinSnapshots", store.New(recycleBinSnapshotsKeyFn)) + }, + func(b *InMemoryBackend) { + b.recycleBinVolumes = store.Register(b.registry, "recycleBinVolumes", store.New(recycleBinVolumesKeyFn)) + }, + func(b *InMemoryBackend) { + b.replaceRootVolumeTasks = store.Register( + b.registry, + "replaceRootVolumeTasks", + store.New(replaceRootVolumeTasksKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.reservedInstances = store.Register(b.registry, "reservedInstances", store.New(reservedInstancesKeyFn)) + }, + func(b *InMemoryBackend) { + b.reservedInstancesExchanges = store.Register( + b.registry, + "reservedInstancesExchanges", + store.New(reservedInstancesExchangesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.reservedInstancesListings = store.Register( + b.registry, + "reservedInstancesListings", + store.New(reservedInstancesListingsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.reservedInstancesModifications = store.Register( + b.registry, + "reservedInstancesModifications", + store.New(reservedInstancesModificationsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.reservedInstancesOfferings = store.Register( + b.registry, + "reservedInstancesOfferings", + store.New(reservedInstancesOfferingsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.routeServerAssociations = store.Register( + b.registry, + "routeServerAssociations", + store.New(routeServerAssociationsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.routeServerEndpoints = store.Register( + b.registry, + "routeServerEndpoints", + store.New(routeServerEndpointsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.routeServerPeers = store.Register(b.registry, "routeServerPeers", store.New(routeServerPeersKeyFn)) + }, + func(b *InMemoryBackend) { + b.routeServerPropagations = store.Register( + b.registry, + "routeServerPropagations", + store.New(routeServerPropagationsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.routeServers = store.Register(b.registry, "routeServers", store.New(routeServersKeyFn)) + }, + func(b *InMemoryBackend) { + b.routeTables = store.Register(b.registry, "routeTables", store.New(routeTablesKeyFn)) + }, + func(b *InMemoryBackend) { + b.scheduledInstances = store.Register(b.registry, "scheduledInstances", store.New(scheduledInstancesKeyFn)) + }, + func(b *InMemoryBackend) { + b.secondaryInterfaces = store.Register(b.registry, "secondaryInterfaces", store.New(secondaryInterfacesKeyFn)) + }, + func(b *InMemoryBackend) { + b.secondaryNetworks = store.Register(b.registry, "secondaryNetworks", store.New(secondaryNetworksKeyFn)) + }, + func(b *InMemoryBackend) { + b.secondarySubnets = store.Register(b.registry, "secondarySubnets", store.New(secondarySubnetsKeyFn)) + }, + func(b *InMemoryBackend) { + b.securityGroups = store.Register(b.registry, "securityGroups", store.New(securityGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.serviceLinkVirtualInterfaces = store.Register( + b.registry, + "serviceLinkVirtualInterfaces", + store.New(serviceLinkVirtualInterfacesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.snapshotImportTasks = store.Register(b.registry, "snapshotImportTasks", store.New(snapshotImportTasksKeyFn)) + }, + func(b *InMemoryBackend) { + b.snapshotLocks = store.Register(b.registry, "snapshotLocks", store.New(snapshotLocksKeyFn)) + }, + func(b *InMemoryBackend) { + b.snapshots = store.Register(b.registry, "snapshots", store.New(snapshotsKeyFn)) + }, + func(b *InMemoryBackend) { + b.spotFleets = store.Register(b.registry, "spotFleets", store.New(spotFleetsKeyFn)) + }, + func(b *InMemoryBackend) { + b.spotRequests = store.Register(b.registry, "spotRequests", store.New(spotRequestsKeyFn)) + }, + func(b *InMemoryBackend) { + b.sqlHaRegistrations = store.Register(b.registry, "sqlHaRegistrations", store.New(sqlHaRegistrationsKeyFn)) + }, + func(b *InMemoryBackend) { + b.storeImageTasks = store.Register(b.registry, "storeImageTasks", store.New(storeImageTasksKeyFn)) + }, + func(b *InMemoryBackend) { + b.subnets = store.Register(b.registry, "subnets", store.New(subnetsKeyFn)) + }, + func(b *InMemoryBackend) { + b.tgwConnectPeers = store.Register(b.registry, "tgwConnectPeers", store.New(tgwConnectPeersKeyFn)) + }, + func(b *InMemoryBackend) { + b.tgwConnects = store.Register(b.registry, "tgwConnects", store.New(tgwConnectsKeyFn)) + }, + func(b *InMemoryBackend) { + b.tgwMeteringPolicies = store.Register(b.registry, "tgwMeteringPolicies", store.New(tgwMeteringPoliciesKeyFn)) + }, + func(b *InMemoryBackend) { + b.tgwMeteringPolicyEntries = store.Register( + b.registry, + "tgwMeteringPolicyEntries", + store.New(tgwMeteringPolicyEntriesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.tgwMulticastDomainAssociations = store.Register( + b.registry, + "tgwMulticastDomainAssociations", + store.New(tgwMulticastDomainAssociationsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.tgwMulticastDomains = store.Register(b.registry, "tgwMulticastDomains", store.New(tgwMulticastDomainsKeyFn)) + }, + func(b *InMemoryBackend) { + b.tgwMulticastGroupEntries = store.Register( + b.registry, + "tgwMulticastGroupEntries", + store.New(tgwMulticastGroupEntriesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.tgwPeeringAttachments = store.Register( + b.registry, + "tgwPeeringAttachments", + store.New(tgwPeeringAttachmentsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.tgwPolicyTableAssociations = store.Register( + b.registry, + "tgwPolicyTableAssociations", + store.New(tgwPolicyTableAssociationsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.tgwPolicyTables = store.Register(b.registry, "tgwPolicyTables", store.New(tgwPolicyTablesKeyFn)) + }, + func(b *InMemoryBackend) { + b.tgwPrefixListRefs = store.Register(b.registry, "tgwPrefixListRefs", store.New(tgwPrefixListRefsKeyFn)) + }, + func(b *InMemoryBackend) { + b.tgwRTAssociations = store.Register(b.registry, "tgwRTAssociations", store.New(tgwRTAssociationsKeyFn)) + }, + func(b *InMemoryBackend) { + b.tgwRouteTableAnnouncements = store.Register( + b.registry, + "tgwRouteTableAnnouncements", + store.New(tgwRouteTableAnnouncementsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.tgwRouteTables = store.Register(b.registry, "tgwRouteTables", store.New(tgwRouteTablesKeyFn)) + }, + func(b *InMemoryBackend) { + b.tgwRoutes = store.Register(b.registry, "tgwRoutes", store.New(tgwRoutesKeyFn)) + }, + func(b *InMemoryBackend) { + b.tgwVpcAttachments = store.Register(b.registry, "tgwVpcAttachments", store.New(tgwVpcAttachmentsKeyFn)) + }, + func(b *InMemoryBackend) { + b.trafficMirrorFilterRules = store.Register( + b.registry, + "trafficMirrorFilterRules", + store.New(trafficMirrorFilterRulesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.trafficMirrorFilters = store.Register( + b.registry, + "trafficMirrorFilters", + store.New(trafficMirrorFiltersKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.trafficMirrorSessions = store.Register( + b.registry, + "trafficMirrorSessions", + store.New(trafficMirrorSessionsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.trafficMirrorTargets = store.Register( + b.registry, + "trafficMirrorTargets", + store.New(trafficMirrorTargetsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.transitGateways = store.Register(b.registry, "transitGateways", store.New(transitGatewaysKeyFn)) + }, + func(b *InMemoryBackend) { + b.trunkInterfaceAssociations = store.Register( + b.registry, + "trunkInterfaceAssociations", + store.New(trunkInterfaceAssociationsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.usageReports = store.Register(b.registry, "usageReports", store.New(usageReportsKeyFn)) + }, + func(b *InMemoryBackend) { + b.verifiedAccessEndpoints = store.Register( + b.registry, + "verifiedAccessEndpoints", + store.New(verifiedAccessEndpointsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.verifiedAccessGroups = store.Register( + b.registry, + "verifiedAccessGroups", + store.New(verifiedAccessGroupsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.verifiedAccessInstanceLoggingConfigs = store.Register( + b.registry, + "verifiedAccessInstanceLoggingConfigs", + store.New(verifiedAccessInstanceLoggingConfigsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.verifiedAccessInstances = store.Register( + b.registry, + "verifiedAccessInstances", + store.New(verifiedAccessInstancesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.verifiedAccessTrustProviders = store.Register( + b.registry, + "verifiedAccessTrustProviders", + store.New(verifiedAccessTrustProvidersKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.volumeModifications = store.Register(b.registry, "volumeModifications", store.New(volumeModificationsKeyFn)) + }, + func(b *InMemoryBackend) { + b.volumes = store.Register(b.registry, "volumes", store.New(volumesKeyFn)) + }, + func(b *InMemoryBackend) { + b.vpcBlockPublicAccessExclusions = store.Register( + b.registry, + "vpcBlockPublicAccessExclusions", + store.New(vpcBlockPublicAccessExclusionsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.vpcEncryptionControls = store.Register( + b.registry, + "vpcEncryptionControls", + store.New(vpcEncryptionControlsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.vpcEndpointConnections = store.Register( + b.registry, + "vpcEndpointConnections", + store.New(vpcEndpointConnectionsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.vpcEndpointServiceConfigs = store.Register( + b.registry, + "vpcEndpointServiceConfigs", + store.New(vpcEndpointServiceConfigsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.vpcEndpoints = store.Register(b.registry, "vpcEndpoints", store.New(vpcEndpointsKeyFn)) + }, + func(b *InMemoryBackend) { + b.vpcPeeringConnections = store.Register( + b.registry, + "vpcPeeringConnections", + store.New(vpcPeeringConnectionsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.vpcs = store.Register(b.registry, "vpcs", store.New(vpcsKeyFn)) + }, + func(b *InMemoryBackend) { + b.vpnConcentrators = store.Register(b.registry, "vpnConcentrators", store.New(vpnConcentratorsKeyFn)) + }, + func(b *InMemoryBackend) { + b.vpnConnectionRoutes = store.Register(b.registry, "vpnConnectionRoutes", store.New(vpnConnectionRoutesKeyFn)) + }, + func(b *InMemoryBackend) { + b.vpnConnections = store.Register(b.registry, "vpnConnections", store.New(vpnConnectionsKeyFn)) + }, + func(b *InMemoryBackend) { + b.vpnGateways = store.Register(b.registry, "vpnGateways", store.New(vpnGatewaysKeyFn)) + }, +} diff --git a/services/ecr/PARITY.md b/services/ecr/PARITY.md new file mode 100644 index 000000000..d598c84f0 --- /dev/null +++ b/services/ecr/PARITY.md @@ -0,0 +1,159 @@ +--- +service: ecr +sdk_module: aws-sdk-go-v2/service/ecr@v1.58.6 +last_audit_commit: fba3c784 +last_audit_date: 2026-07-05 +overall: B # already-accurate op-by-op, with a handful of genuine fixes (~190 LOC prod + ~260 LOC new tests) +ops: + CreateRepository: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeRepositories: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteRepository: {wire: ok, errors: ok, state: ok, persist: ok, note: "force-with-images enforced in handler via DescribeImages pre-check"} + PutImage: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED — added ImageDigestDoesNotMatchException validation at the wire boundary (handler); tag-mutability + retag semantics were already correct"} + BatchGetImage: {wire: ok, errors: ok, state: ok, persist: ok} + BatchDeleteImage: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeImages: {wire: ok, errors: ok, state: ok, persist: ok} + ListImages: {wire: ok, errors: ok, state: ok, persist: ok} + ListImageReferrers: {wire: ok, errors: ok, state: ok, persist: ok} + BatchCheckLayerAvailability: {wire: ok, errors: ok, state: ok, persist: ok} + InitiateLayerUpload: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIFO TTL pruning bounds layerUploads/layerUploadQueue"} + UploadLayerPart: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED — added part-sequencing validation (InvalidLayerPartException) for non-consecutive partFirstByte"} + CompleteLayerUpload: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED — was missing RepositoryNotFoundException FK check (unlike every other op) and never rejected re-completing an already-registered layer digest (LayerAlreadyExistsException)"} + GetDownloadUrlForLayer: {wire: ok, errors: ok, state: ok, persist: ok} + GetAuthorizationToken: {wire: ok, errors: ok, state: ok, persist: n/a, note: "base64(AWS:dummy-password), 12h TTL, proxyEndpoint derived from first request Host"} + CreatePullThroughCacheRule: {wire: ok, errors: ok, state: ok, persist: ok} + DescribePullThroughCacheRules: {wire: ok, errors: ok, state: ok, persist: ok} + UpdatePullThroughCacheRule: {wire: ok, errors: ok, state: ok, persist: ok} + DeletePullThroughCacheRule: {wire: ok, errors: ok, state: ok, persist: ok} + ValidatePullThroughCacheRule: {wire: ok, errors: ok, state: ok, persist: n/a} + CreateRepositoryCreationTemplate: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeRepositoryCreationTemplates: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateRepositoryCreationTemplate: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteRepositoryCreationTemplate: {wire: ok, errors: ok, state: ok, persist: ok} + PutLifecyclePolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "applied immediately on Put, matching AWS's immediate evaluation"} + GetLifecyclePolicy: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteLifecyclePolicy: {wire: ok, errors: ok, state: ok, persist: ok} + StartLifecyclePolicyPreview: {wire: ok, errors: ok, state: ok, persist: ok} + GetLifecyclePolicyPreview: {wire: ok, errors: ok, state: ok, persist: ok} + GetRepositoryPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + SetRepositoryPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteRepositoryPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + GetRegistryPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + PutRegistryPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteRegistryPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeRegistry: {wire: ok, errors: ok, state: ok, persist: ok} + GetRegistryScanningConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + PutRegistryScanningConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + BatchGetRepositoryScanningConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + PutImageScanningConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + PutImageTagMutability: {wire: ok, errors: ok, state: ok, persist: ok, note: "exclusion filters (WILDCARD + literal) enforced correctly"} + StartImageScan: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeImageScanFindings: {wire: ok, errors: ok, state: ok, persist: ok, note: "BASIC vs ENHANCED finding shapes genuinely differ; paginated via index-based nextToken; ScanNotFoundException for never-scanned images"} + PutReplicationConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeImageReplicationStatus: {wire: ok, errors: ok, state: ok, persist: ok} + GetSigningConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + PutSigningConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteSigningConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeImageSigningStatus: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateImageStorageClass: {wire: ok, errors: ok, state: ok, persist: ok} + GetAccountSetting: {wire: ok, errors: ok, state: ok, persist: ok} + PutAccountSetting: {wire: ok, errors: ok, state: ok, persist: ok} + RegisterPullTimeUpdateExclusion: {wire: ok, errors: ok, state: ok, persist: ok} + DeregisterPullTimeUpdateExclusion: {wire: ok, errors: ok, state: ok, persist: ok} + ListPullTimeUpdateExclusions: {wire: ok, errors: ok, state: ok, persist: ok} + TagResource: {wire: ok, errors: ok, state: ok, persist: ok} + UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} + ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok} +families: + registry-v2-proxy: {status: ok, note: "docker distribution/v3 in-memory storage driver embedded for /v2/ blob+manifest paths; ExtractResource avoids buffering upload bodies"} + lifecycle-evaluation: {status: ok, note: "priority-ordered rules, imageCountMoreThan + sinceImagePushed count types, tagStatus any/tagged/untagged with prefix+wildcard pattern matching, janitor sweeps on a timer independent of API calls"} + mock-scanning: {status: ok, note: "deterministic per-digest CVE selection (sha256-seeded bitmask) so repeated scans of the same image are stable; BASIC and ENHANCED shapes are genuinely different data, not the same list reshaped"} +gaps: + - "EmptyUploadException (CompleteLayerUpload with no UploadLayerPart calls) intentionally NOT enforced: an existing test (TestBatch1_CompleteLayerUpload_Makes_Layer_Available) explicitly exercises Initiate→Complete with no UploadLayerPart call and asserts success. Flipping this would contradict established, deliberate test behavior without a design doc confirming which is correct for this emulator; deferred rather than guessed. (bd: gopherstack-x6i)" + - "ImageAlreadyExistsException (PutImage with an unchanged manifest+tag) intentionally NOT enforced: the SDK doc text (\"no changes to the manifest or image tag after the last push\") supports it, but an existing test (TestBatch2_ImmutableRepo_SameManifestSameTag_Idempotent) explicitly asserts idempotent re-push succeeds with 200, and the task's explicit required-error-code list does not include it. Deferred to avoid a moderate-confidence behavior flip with test-suite-wide blast radius. (bd: gopherstack-x6i)" + - "UploadLayerPart minimum-part-size (LayerPartTooSmallException, 'parts must be at least 5MiB except the last') not enforced: the backend cannot know which part is 'last' until CompleteLayerUpload, and all current tests upload tiny (<10 byte) parts, so implementing this needs a size-deferred-validation design (validate at Complete time against accumulated part boundaries) that is out of scope for this pass. (bd: gopherstack-x6i)" +deferred: + - "docker registry v2 proxy internals (pkgs distribution/v3 wiring) — treated as a vendored subsystem, not re-audited this pass" + - "chaos/fault-injection interaction with ECR ops — not exercised this pass" +leaks: {status: clean, note: "layerUploads bounded by FIFO TTL queue pruned on InitiateLayerUpload; janitor uses worker.Group with ctx.Done() shutdown; no unbounded maps found in this pass"} +--- + +## Notes + +Protocol: JSON RPC 1.1 (`application/x-amz-json-1.1`, `X-Amz-Target: +AmazonEC2ContainerRegistry_V20150921.`). Timestamps are epoch-seconds JSON +numbers (verified against `awsAwsjson11_deserializeDocumentRepository` etc. in +the vendored SDK deserializer) — this codebase already uses that convention +throughout (`createdAt float64`, etc.), matching `smithytime.ParseEpochSeconds`. + +This service had already been through multiple prior parity sweeps (test files +named `handler_accuracy_batch1/2`, `handler_refinement1/2`, +`handler_parity_ecr/ecr2`, `audit_ecr_test.go`, `replication_status_test.go`, +`scan_enhanced_test.go`, `lifecycle_expiry_test.go`, `leak_test.go` — ~17.9k LOC +total before this pass). Op-by-op review of `backend.go` confirmed FK +validation (repository/image/rule existence checks) present and correct on +every op **except** `CompleteLayerUpload`, which is the main finding this pass. + +### Genuine fixes made this pass + +1. **`CompleteLayerUpload` missing repository-existence check.** Every other + backend method validates `b.repos[repositoryName]` before mutating state; + `CompleteLayerUpload` did not, so completing an upload against a + nonexistent repository silently "succeeded" instead of returning + `RepositoryNotFoundException`. Also exposed a **pre-existing test bug**: + `TestECR_CompleteLayerUpload` called `CompleteLayerUpload` against + `"my-repo"`, which the test never created — it was accidentally relying on + the missing FK check to pass. Fixed by creating the repo in the test setup + (test now encodes the corrected/real behavior). + +2. **`CompleteLayerUpload` missing `LayerAlreadyExistsException`.** AWS + rejects re-completing a layer digest that is already registered as + available in the repository (SDK doc: "The image layer already exists in + the associated repository."). The backend silently overwrote + `uploadedLayers[repo][digest]` on every call. Fixed by checking for an + existing entry before writing (guarded against `digest == ""` so the + pre-existing "empty session, no digest" completion path — exercised + deliberately by an existing test — cannot self-collide). + +3. **`UploadLayerPart` missing part-sequencing validation + (`InvalidLayerPartException`).** AWS requires each part's `partFirstByte` + to be consecutive to the number of bytes already received in the session + ("the first byte specified is not consecutive to the last byte of a + previous layer part upload"). The parameter was previously discarded + (`_, lastByte int64`) entirely. Fixed by comparing `firstByte` against + `upload.Size` and returning the new sentinel error on mismatch. No existing + test exercises multi-part sequences with gaps, so this is purely additive. + +4. **`PutImage` missing `ImageDigestDoesNotMatchException`.** AWS validates a + caller-supplied `imageDigest` against the digest it computes from the + manifest and rejects a mismatch. This codebase never checked it — any + client-supplied digest was trusted verbatim. Implemented **at the JSON + handler boundary** (`handlePutImage`), not inside + `InMemoryBackend.PutImage`: dozens of existing unit tests call + `Backend.PutImage` directly with synthetic, non-cryptographic `ImageDigest` + values (e.g. `"sha256:abc111"`) that intentionally don't hash-match their + paired `ImageManifest` text, using the digest purely as a unique key. Wire + fixtures (`mustPutImage`, SDK-client tests) never set an explicit + `imageDigest` on the request, so none of them exercise this new check; it + only fires for a real client-supplied mismatch, matching where AWS itself + performs the validation (request processing, not backend storage). + Uncovered a **pre-existing test bug**: `TestECR_NewOps_PersistenceRoundTrip` + sent `imageDigest: "sha256:persist123"` with `imageManifest: "{}"` — not a + real sha256 of `"{}"`. Fixed by replacing the literal with the real + `sha256("{}")` hex digest everywhere it's threaded through that test + (5 occurrences: layer digest, image digest ×2, image-id lookups ×2). + +### Deliberately NOT fixed (see `gaps` above for full rationale) + +- `EmptyUploadException` — contradicted by an existing, clearly-intentional + test. +- `ImageAlreadyExistsException` (idempotent-repush guard) — contradicted by an + existing, clearly-intentional test; not in the task's explicit + required-error-code list; moderate-confidence AWS-doc interpretation with + test-suite-wide blast radius if wrong. +- `LayerPartTooSmallException` — needs a design for deferred (at-Complete-time) + validation since "last part" isn't knowable at upload time; no test coverage + either direction. + +None of these were silently stubbed — they are explicitly called out as +deferred with the reasoning, per the no-stub principle's guidance to prefer an +explicit terminal note over a guessed half-fix. diff --git a/services/ecr/backend.go b/services/ecr/backend.go index d1fd50d77..17883d63d 100644 --- a/services/ecr/backend.go +++ b/services/ecr/backend.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "maps" + "slices" "sort" "strconv" "strings" @@ -17,6 +18,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/awsmeta" "github.com/blackbirdworks/gopherstack/pkgs/collections" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) const ( @@ -93,6 +95,21 @@ var ( ErrLayerInaccessible = awserr.New("LayerInaccessibleException", awserr.ErrNotFound) // ErrLayersNotFound is returned when requested layers do not exist in the repository. ErrLayersNotFound = awserr.New("LayersNotFoundException", awserr.ErrNotFound) + // ErrLayerAlreadyExists is returned when CompleteLayerUpload is called with a + // digest that has already been registered as an available layer in the + // repository (matches AWS: "The image layer already exists in the associated + // repository."). + ErrLayerAlreadyExists = awserr.New("LayerAlreadyExistsException", awserr.ErrAlreadyExists) + // ErrInvalidLayerPart is returned when an UploadLayerPart's first byte is not + // consecutive to the last byte received by a previous part in the same + // upload session (matches AWS InvalidLayerPartException). + ErrInvalidLayerPart = awserr.New("InvalidLayerPartException", awserr.ErrInvalidParameter) + // ErrImageDigestDoesNotMatch is returned when a caller-supplied imageDigest on + // PutImage does not match the digest ECR computes from the image manifest. + ErrImageDigestDoesNotMatch = awserr.New( + "ImageDigestDoesNotMatchException", + awserr.ErrInvalidParameter, + ) ) // Repository represents an ECR repository. @@ -518,63 +535,70 @@ var _ Backend = (*InMemoryBackend)(nil) // InMemoryBackend stores ECR repository state in memory. type InMemoryBackend struct { - repoTags map[string]map[string]string - signingConfig *SigningSettings - tagIndex map[string]map[string]string - digestTagsIndex map[string]map[string][]string - pullThroughCacheRules map[string]*PullThroughCacheRule - repositoryCreationTemplates map[string]*RepositoryCreationTemplate - lifecyclePolicies map[string]string - lifecyclePolicyPreviews map[string]*LifecyclePolicyPreviewResult - uploadedLayers map[string]map[string]int64 - layerUploads map[string]*layerUploadState - repoUploadIndex map[string]map[string]struct{} - repositoryPolicies map[string]string - images map[string]map[string]*Image - imageScanFindings map[string]map[string]*ImageScanFindingsResult - repos map[string]*Repository - accountSettings map[string]string - pullTimeUpdateExclusions map[string]*PullTimeUpdateExclusion - mu *lockmetrics.RWMutex - registryScanningConfig *RegistryScanningSettings - replicationConfig *ReplicationConfig - lifecycleLastEvaluated map[string]time.Time - registryPolicy string - accountID string - region string - endpoint string - layerUploadQueue []layerUploadQueueEntry - replicationSettleDelay time.Duration + // registry lets Reset/Snapshot/Restore collapse the resource-table + // lifecycle to one call each (registry.ResetAll/SnapshotAll/RestoreAll) + // instead of hand-rolled per-map wiring. See pkgs/store's package doc and + // the services/sqs pilot (commit 0f09d77c) for the pattern this follows. + // See store_setup.go for every table registered on it, and for the + // fields deliberately left as plain maps below instead. + registry *store.Registry + repos *store.Table[Repository] + images *store.Table[Image] + imagesByRepo *store.Index[Image] + imageScanFindings *store.Table[ImageScanFindingsResult] + imageScanFindingsByRepo *store.Index[ImageScanFindingsResult] + pullThroughCacheRules *store.Table[PullThroughCacheRule] + repositoryCreationTemplates *store.Table[RepositoryCreationTemplate] + lifecyclePolicies *store.Table[lifecyclePolicyEntry] + lifecyclePolicyPreviews *store.Table[LifecyclePolicyPreviewResult] + repositoryPolicies *store.Table[repositoryPolicyEntry] + accountSettings *store.Table[accountSettingEntry] + pullTimeUpdateExclusions *store.Table[PullTimeUpdateExclusion] + + // The following are deliberately left as plain maps -- see the doc above + // registerAllTables in store_setup.go for why each one is exempt. + repoTags map[string]map[string]string + signingConfig *SigningSettings + tagIndex map[string]map[string]string + digestTagsIndex map[string]map[string][]string + uploadedLayers map[string]map[string]int64 + layerUploads map[string]*layerUploadState + repoUploadIndex map[string]map[string]struct{} + mu *lockmetrics.RWMutex + registryScanningConfig *RegistryScanningSettings + replicationConfig *ReplicationConfig + lifecycleLastEvaluated map[string]time.Time + registryPolicy string + accountID string + region string + endpoint string + layerUploadQueue []layerUploadQueueEntry + replicationSettleDelay time.Duration } // NewInMemoryBackend creates a new InMemoryBackend with the given account ID and region. func NewInMemoryBackend(accountID, region, endpoint string) *InMemoryBackend { - return &InMemoryBackend{ - repos: make(map[string]*Repository), - images: make(map[string]map[string]*Image), - tagIndex: make(map[string]map[string]string), - digestTagsIndex: make(map[string]map[string][]string), - pullThroughCacheRules: make(map[string]*PullThroughCacheRule), - repositoryCreationTemplates: make(map[string]*RepositoryCreationTemplate), - lifecyclePolicies: make(map[string]string), - lifecyclePolicyPreviews: make(map[string]*LifecyclePolicyPreviewResult), - uploadedLayers: make(map[string]map[string]int64), - layerUploads: make(map[string]*layerUploadState), - repoUploadIndex: make(map[string]map[string]struct{}), - layerUploadQueue: make([]layerUploadQueueEntry, 0), - repoTags: make(map[string]map[string]string), - repositoryPolicies: make(map[string]string), - imageScanFindings: make(map[string]map[string]*ImageScanFindingsResult), - accountSettings: make(map[string]string), - pullTimeUpdateExclusions: make(map[string]*PullTimeUpdateExclusion), - registryScanningConfig: &RegistryScanningSettings{ScanType: scanTypeBasic}, - replicationConfig: &ReplicationConfig{}, - lifecycleLastEvaluated: make(map[string]time.Time), - mu: lockmetrics.New("ecr"), - accountID: accountID, - region: region, - endpoint: endpoint, + b := &InMemoryBackend{ + registry: store.NewRegistry(), + tagIndex: make(map[string]map[string]string), + digestTagsIndex: make(map[string]map[string][]string), + uploadedLayers: make(map[string]map[string]int64), + layerUploads: make(map[string]*layerUploadState), + repoUploadIndex: make(map[string]map[string]struct{}), + layerUploadQueue: make([]layerUploadQueueEntry, 0), + repoTags: make(map[string]map[string]string), + registryScanningConfig: &RegistryScanningSettings{ScanType: scanTypeBasic}, + replicationConfig: &ReplicationConfig{}, + lifecycleLastEvaluated: make(map[string]time.Time), + mu: lockmetrics.New("ecr"), + accountID: accountID, + region: region, + endpoint: endpoint, } + + registerAllTables(b) + + return b } // SetEndpoint updates the registry endpoint used in repository URIs. @@ -643,7 +667,7 @@ func (b *InMemoryBackend) CreateRepository( b.mu.Lock("CreateRepository") defer b.mu.Unlock() - if _, ok := b.repos[name]; ok { + if b.repos.Has(name) { return nil, fmt.Errorf("%w: %s", ErrRepositoryAlreadyExists, name) } @@ -670,7 +694,7 @@ func (b *InMemoryBackend) CreateRepository( ImageTagMutability: imageTagMutability, ScanOnPush: scanOnPush, } - b.repos[name] = repo + b.repos.Put(repo) cp := *repo @@ -686,8 +710,9 @@ func (b *InMemoryBackend) DescribeRepositories( defer b.mu.RUnlock() if len(names) == 0 { - out := make([]Repository, 0, len(b.repos)) - for _, r := range b.repos { + all := b.repos.All() + out := make([]Repository, 0, len(all)) + for _, r := range all { out = append(out, *r) } @@ -701,7 +726,7 @@ func (b *InMemoryBackend) DescribeRepositories( out := make([]Repository, 0, len(names)) for _, name := range names { - r, ok := b.repos[name] + r, ok := b.repos.Get(name) if !ok { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, name) } @@ -720,21 +745,32 @@ func (b *InMemoryBackend) DeleteRepository( b.mu.Lock("DeleteRepository") defer b.mu.Unlock() - r, ok := b.repos[name] + r, ok := b.repos.Get(name) if !ok { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, name) } - delete(b.repos, name) - delete(b.images, name) + b.repos.Delete(name) + + // slices.Clone the index results before deleting in the loop: Table.Delete + // mutates the very index b.imagesByRepo.Get/b.imageScanFindingsByRepo.Get + // returned, so iterating the live (unsloned) slice while deleting from it + // would skip entries. + for _, img := range slices.Clone(b.imagesByRepo.Get(name)) { + b.images.Delete(imageTableKey(img.RepositoryName, img.ImageDigest)) + } + + for _, f := range slices.Clone(b.imageScanFindingsByRepo.Get(name)) { + b.imageScanFindings.Delete(findingsTableKey(f.RepositoryName, f.ImageID.ImageDigest)) + } + delete(b.tagIndex, name) delete(b.digestTagsIndex, name) delete(b.uploadedLayers, name) - delete(b.lifecyclePolicies, name) - delete(b.lifecyclePolicyPreviews, name) + b.lifecyclePolicies.Delete(name) + b.lifecyclePolicyPreviews.Delete(name) delete(b.repoTags, r.RepositoryARN) - delete(b.repositoryPolicies, name) - delete(b.imageScanFindings, name) + b.repositoryPolicies.Delete(name) // Clean up any in-progress layer uploads associated with this repository. for uploadID := range b.repoUploadIndex[name] { @@ -756,7 +792,7 @@ func (b *InMemoryBackend) BatchCheckLayerAvailability( b.mu.RLock("BatchCheckLayerAvailability") defer b.mu.RUnlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } @@ -786,11 +822,12 @@ func (b *InMemoryBackend) BatchCheckLayerAvailability( // deleteByDigestLocked removes an image by digest, deletes all tag bindings for // that digest, and returns true if the image was found. func deleteByDigestLocked( - repoImages map[string]*Image, + images *store.Table[Image], repoTags map[string]string, - digest string, + repositoryName, digest string, ) bool { - if _, ok := repoImages[digest]; !ok { + key := imageTableKey(repositoryName, digest) + if !images.Has(key) { return false } @@ -801,21 +838,25 @@ func deleteByDigestLocked( } } - delete(repoImages, digest) + images.Delete(key) return true } // deleteByTagLocked removes a tag binding, clears the image's tag field if it // matches, and falls back to a linear scan for legacy images. Returns true if found. -func deleteByTagLocked(repoImages map[string]*Image, repoTags map[string]string, tag string) bool { +func deleteByTagLocked( + images *store.Table[Image], + repoTags map[string]string, + repositoryName, tag string, +) bool { digest, ok := repoTags[tag] if !ok { return false } delete(repoTags, tag) - if img, exists := repoImages[digest]; exists { + if img, exists := images.Get(imageTableKey(repositoryName, digest)); exists { img.ImageID.ImageTag = "" } @@ -833,28 +874,27 @@ func (b *InMemoryBackend) BatchDeleteImage(ctx context.Context, //nolint:revive b.mu.Lock("BatchDeleteImage") defer b.mu.Unlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } deleted := make([]ImageIdentifier, 0, len(imageIDs)) failures := make([]ImageFailure, 0, len(imageIDs)) - repoImages := b.images[repositoryName] repoTags := b.tagIndex[repositoryName] for _, id := range imageIDs { var found bool if id.ImageDigest != "" { - found = deleteByDigestLocked(repoImages, repoTags, id.ImageDigest) + found = deleteByDigestLocked(b.images, repoTags, repositoryName, id.ImageDigest) if found { b.clearDigestTagsLocked(repositoryName, id.ImageDigest) } } else if id.ImageTag != "" { // Snapshot the digest before deletion so we can update the reverse index. oldDigest := repoTags[id.ImageTag] - found = deleteByTagLocked(repoImages, repoTags, id.ImageTag) + found = deleteByTagLocked(b.images, repoTags, repositoryName, id.ImageTag) if found && oldDigest != "" { b.removeDigestTagLocked(repositoryName, oldDigest, id.ImageTag) } @@ -882,18 +922,17 @@ func (b *InMemoryBackend) BatchGetImage(ctx context.Context, //nolint:revive // b.mu.RLock("BatchGetImage") defer b.mu.RUnlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } imgs := make([]Image, 0, len(imageIDs)) failures := make([]ImageFailure, 0, len(imageIDs)) - repoImages := b.images[repositoryName] repoTagIdx := b.tagIndex[repositoryName] for _, id := range imageIDs { - img, ok := findImageLocked(repoImages, repoTagIdx, id) + img, ok := findImageLocked(b.images, b.imagesByRepo, repositoryName, repoTagIdx, id) if ok { cp := *img // Preserve requested tag in imageId for the response. @@ -986,11 +1025,11 @@ func (b *InMemoryBackend) DescribeImages( b.mu.RLock("DescribeImages") defer b.mu.RUnlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } - repoImages := b.images[repositoryName] + repoImages := b.imagesByRepo.Get(repositoryName) repoTagIdx := b.tagIndex[repositoryName] digestTags := b.digestTagsIndex[repositoryName] @@ -1013,7 +1052,7 @@ func (b *InMemoryBackend) DescribeImages( } } else { for _, id := range imageIDs { - img, ok := findImageLocked(repoImages, repoTagIdx, id) + img, ok := findImageLocked(b.images, b.imagesByRepo, repositoryName, repoTagIdx, id) if !ok { return nil, fmt.Errorf("%w: image not found", ErrImageNotFound) } @@ -1039,7 +1078,7 @@ func (b *InMemoryBackend) BatchGetRepositoryScanningConfiguration( failures := make([]RepositoryScanningConfigurationFailure, 0, len(repositoryNames)) for _, name := range repositoryNames { - repo, ok := b.repos[name] + repo, ok := b.repos.Get(name) if !ok { failures = append(failures, RepositoryScanningConfigurationFailure{ RepositoryName: name, @@ -1075,36 +1114,61 @@ func (b *InMemoryBackend) CompleteLayerUpload( b.mu.Lock("CompleteLayerUpload") defer b.mu.Unlock() + if !b.repos.Has(repositoryName) { + return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) + } + + digest, size, err := b.resolveCompletedLayerLocked(repositoryName, uploadID, layerDigests) + if err != nil { + return nil, err + } + + // AWS rejects re-completing a layer digest that has already been registered + // as available in the repository with LayerAlreadyExistsException. Only + // applies when a digest was actually resolved; an empty digest is not a + // real layer identity and must not collide with itself across sessions. + if digest != "" { + if _, exists := b.uploadedLayers[repositoryName][digest]; exists { + return nil, fmt.Errorf("%w: %s", ErrLayerAlreadyExists, digest) + } + } + + if b.uploadedLayers[repositoryName] == nil { + b.uploadedLayers[repositoryName] = make(map[string]int64) + } + + b.uploadedLayers[repositoryName][digest] = size + + return &CompleteLayerUploadResult{ + LayerDigest: digest, + RepositoryName: repositoryName, + RegistryID: b.accountID, + UploadID: uploadID, + }, nil +} + +// resolveCompletedLayerLocked determines the final layer digest and size for a +// CompleteLayerUpload call and retires the upload session (if one exists). +// Caller must hold the write lock. +func (b *InMemoryBackend) resolveCompletedLayerLocked( + repositoryName, uploadID string, + layerDigests []string, +) (string, int64, error) { var digest string var size int64 upload, ok := b.layerUploads[uploadID] + switch { case ok && upload.RepositoryName == repositoryName && len(upload.Data) > 0: - computed := "sha256:" + hex.EncodeToString(sha256Sum(upload.Data)) - - provided := "" - if len(layerDigests) > 0 { - provided = layerDigests[0] - } - - if provided != "" { - // Only enforce digest verification for full 64-char SHA256 digests. - if isFullSHA256Digest(provided) && provided != computed { - return nil, fmt.Errorf("%w: digest mismatch: got %s, want %s", - ErrLayerDigestMismatch, provided, computed) - } - - digest = provided - } else { - digest = computed + verified, err := verifiedUploadDigestLocked(upload, layerDigests) + if err != nil { + return "", 0, err } + digest = verified size = upload.Size - delete(b.layerUploads, uploadID) - if idx, ok2 := b.repoUploadIndex[repositoryName]; ok2 { - delete(idx, uploadID) - } + b.retireLayerUploadLocked(repositoryName, uploadID) case ok && upload.RepositoryName == repositoryName: if len(layerDigests) > 0 { @@ -1112,28 +1176,48 @@ func (b *InMemoryBackend) CompleteLayerUpload( } size = upload.Size - delete(b.layerUploads, uploadID) - if idx, ok2 := b.repoUploadIndex[repositoryName]; ok2 { - delete(idx, uploadID) - } + b.retireLayerUploadLocked(repositoryName, uploadID) case len(layerDigests) > 0: // Direct digest path: no prior InitiateLayerUpload. digest = layerDigests[0] } - if b.uploadedLayers[repositoryName] == nil { - b.uploadedLayers[repositoryName] = make(map[string]int64) + return digest, size, nil +} + +// verifiedUploadDigestLocked computes the SHA256 of the accumulated upload +// bytes and, when the caller provided a full SHA256 digest, verifies it +// matches before returning the digest to record. +func verifiedUploadDigestLocked(upload *layerUploadState, layerDigests []string) (string, error) { + computed := "sha256:" + hex.EncodeToString(sha256Sum(upload.Data)) + + provided := "" + if len(layerDigests) > 0 { + provided = layerDigests[0] } - b.uploadedLayers[repositoryName][digest] = size + if provided == "" { + return computed, nil + } - return &CompleteLayerUploadResult{ - LayerDigest: digest, - RepositoryName: repositoryName, - RegistryID: b.accountID, - UploadID: uploadID, - }, nil + // Only enforce digest verification for full 64-char SHA256 digests. + if isFullSHA256Digest(provided) && provided != computed { + return "", fmt.Errorf("%w: digest mismatch: got %s, want %s", + ErrLayerDigestMismatch, provided, computed) + } + + return provided, nil +} + +// retireLayerUploadLocked removes an upload session and its per-repository +// index entry once it has been finalised by CompleteLayerUpload. +// Caller must hold the write lock. +func (b *InMemoryBackend) retireLayerUploadLocked(repositoryName, uploadID string) { + delete(b.layerUploads, uploadID) + if idx, ok := b.repoUploadIndex[repositoryName]; ok { + delete(idx, uploadID) + } } // sha256Sum returns the SHA256 hash of data. @@ -1172,7 +1256,7 @@ func (b *InMemoryBackend) GetDownloadURLForLayer( b.mu.RLock("GetDownloadURLForLayer") defer b.mu.RUnlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return "", fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } @@ -1196,7 +1280,7 @@ func (b *InMemoryBackend) InitiateLayerUpload( b.mu.Lock("InitiateLayerUpload") defer b.mu.Unlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } @@ -1236,15 +1320,19 @@ func (b *InMemoryBackend) InitiateLayerUpload( } // UploadLayerPart records uploaded bytes for an existing upload session. +// AWS requires each part's first byte to be consecutive to the last byte +// received by the previous part (i.e. equal to the number of bytes already +// buffered for this session); a gap or overlap is rejected with +// InvalidLayerPartException. func (b *InMemoryBackend) UploadLayerPart(ctx context.Context, //nolint:revive // existing issue. repositoryName, uploadID string, - _, lastByte int64, + firstByte, lastByte int64, blob []byte, ) (*LayerUploadPartResult, error) { b.mu.Lock("UploadLayerPart") defer b.mu.Unlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } @@ -1253,6 +1341,13 @@ func (b *InMemoryBackend) UploadLayerPart(ctx context.Context, //nolint:revive / return nil, fmt.Errorf("%w: upload not found", ErrRepositoryNotFound) } + if firstByte >= 0 && firstByte != upload.Size { + return nil, fmt.Errorf( + "%w: partFirstByte %d is not consecutive to the %d bytes already received", + ErrInvalidLayerPart, firstByte, upload.Size, + ) + } + upload.Data = append(upload.Data, blob...) upload.Size = int64(len(upload.Data)) // Refresh the activity timestamp so an in-progress multi-part upload is not @@ -1283,7 +1378,7 @@ func (b *InMemoryBackend) CreatePullThroughCacheRule( b.mu.Lock("CreatePullThroughCacheRule") defer b.mu.Unlock() - if _, ok := b.pullThroughCacheRules[prefix]; ok { + if b.pullThroughCacheRules.Has(prefix) { return nil, fmt.Errorf("%w: %s", ErrPullThroughCacheRuleAlreadyExists, prefix) } @@ -1299,7 +1394,7 @@ func (b *InMemoryBackend) CreatePullThroughCacheRule( CreatedAt: now, UpdatedAt: now, } - b.pullThroughCacheRules[prefix] = rule + b.pullThroughCacheRules.Put(rule) cp := *rule @@ -1314,14 +1409,14 @@ func (b *InMemoryBackend) DescribePullThroughCacheRules( b.mu.RLock("DescribePullThroughCacheRules") defer b.mu.RUnlock() - out := make([]PullThroughCacheRule, 0, len(b.pullThroughCacheRules)) + out := make([]PullThroughCacheRule, 0, b.pullThroughCacheRules.Len()) if len(prefixes) == 0 { - for _, rule := range b.pullThroughCacheRules { + for _, rule := range b.pullThroughCacheRules.All() { out = append(out, *rule) } } else { for _, prefix := range prefixes { - rule, ok := b.pullThroughCacheRules[prefix] + rule, ok := b.pullThroughCacheRules.Get(prefix) if !ok { return nil, fmt.Errorf("%w: %s", ErrPullThroughCacheRuleNotFound, prefix) } @@ -1350,7 +1445,7 @@ func (b *InMemoryBackend) CreateRepositoryCreationTemplate( b.mu.Lock("CreateRepositoryCreationTemplate") defer b.mu.Unlock() - if _, ok := b.repositoryCreationTemplates[req.Prefix]; ok { + if b.repositoryCreationTemplates.Has(req.Prefix) { return nil, fmt.Errorf("%w: %s", ErrRepositoryCreationTemplateAlreadyExists, req.Prefix) } @@ -1370,7 +1465,7 @@ func (b *InMemoryBackend) CreateRepositoryCreationTemplate( CreatedAt: now, UpdatedAt: now, } - b.repositoryCreationTemplates[req.Prefix] = tmpl + b.repositoryCreationTemplates.Put(tmpl) cp := *tmpl @@ -1385,12 +1480,12 @@ func (b *InMemoryBackend) DeleteRepositoryCreationTemplate( b.mu.Lock("DeleteRepositoryCreationTemplate") defer b.mu.Unlock() - tmpl, ok := b.repositoryCreationTemplates[prefix] + tmpl, ok := b.repositoryCreationTemplates.Get(prefix) if !ok { return nil, fmt.Errorf("%w: %s", ErrRepositoryCreationTemplateNotFound, prefix) } - delete(b.repositoryCreationTemplates, prefix) + b.repositoryCreationTemplates.Delete(prefix) cp := copyRepositoryCreationTemplate(tmpl) return &cp, nil @@ -1404,14 +1499,14 @@ func (b *InMemoryBackend) DescribeRepositoryCreationTemplates( b.mu.RLock("DescribeRepositoryCreationTemplates") defer b.mu.RUnlock() - out := make([]RepositoryCreationTemplate, 0, len(b.repositoryCreationTemplates)) + out := make([]RepositoryCreationTemplate, 0, b.repositoryCreationTemplates.Len()) if len(prefixes) == 0 { - for _, tmpl := range b.repositoryCreationTemplates { + for _, tmpl := range b.repositoryCreationTemplates.All() { out = append(out, copyRepositoryCreationTemplate(tmpl)) } } else { for _, prefix := range prefixes { - tmpl, ok := b.repositoryCreationTemplates[prefix] + tmpl, ok := b.repositoryCreationTemplates.Get(prefix) if !ok { return nil, fmt.Errorf("%w: %s", ErrRepositoryCreationTemplateNotFound, prefix) } @@ -1433,17 +1528,18 @@ func (b *InMemoryBackend) DeleteLifecyclePolicy( b.mu.Lock("DeleteLifecyclePolicy") defer b.mu.Unlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } - if _, ok := b.lifecyclePolicies[repositoryName]; !ok { + entry, ok := b.lifecyclePolicies.Get(repositoryName) + if !ok { return nil, fmt.Errorf("%w: %s", ErrLifecyclePolicyNotFound, repositoryName) } - policyText := b.lifecyclePolicies[repositoryName] + policyText := entry.PolicyText lastEvaluated := b.lifecycleLastEvaluated[repositoryName] - delete(b.lifecyclePolicies, repositoryName) + b.lifecyclePolicies.Delete(repositoryName) delete(b.lifecycleLastEvaluated, repositoryName) return &LifecyclePolicyResult{ @@ -1462,17 +1558,17 @@ func (b *InMemoryBackend) GetLifecyclePolicy( b.mu.RLock("GetLifecyclePolicy") defer b.mu.RUnlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } - policyText, ok := b.lifecyclePolicies[repositoryName] + entry, ok := b.lifecyclePolicies.Get(repositoryName) if !ok { return nil, fmt.Errorf("%w: %s", ErrLifecyclePolicyNotFound, repositoryName) } return &LifecyclePolicyResult{ - LifecyclePolicyText: policyText, + LifecyclePolicyText: entry.PolicyText, LastEvaluatedAt: b.lifecycleLastEvaluated[repositoryName], RepositoryName: repositoryName, RegistryID: b.accountID, @@ -1487,11 +1583,11 @@ func (b *InMemoryBackend) GetLifecyclePolicyPreview( b.mu.RLock("GetLifecyclePolicyPreview") defer b.mu.RUnlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } - preview, ok := b.lifecyclePolicyPreviews[repositoryName] + preview, ok := b.lifecyclePolicyPreviews.Get(repositoryName) if !ok { return nil, fmt.Errorf("%w: %s", ErrLifecyclePolicyNotFound, repositoryName) } @@ -1510,11 +1606,11 @@ func (b *InMemoryBackend) PutLifecyclePolicy( b.mu.Lock("PutLifecyclePolicy") defer b.mu.Unlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } - b.lifecyclePolicies[repositoryName] = policyText + b.lifecyclePolicies.Put(&lifecyclePolicyEntry{RepositoryName: repositoryName, PolicyText: policyText}) // AWS evaluates a newly applied lifecycle policy right away; matched images // are expired and deleted rather than merely stored. @@ -1536,15 +1632,18 @@ func (b *InMemoryBackend) StartLifecyclePolicyPreview( b.mu.Lock("StartLifecyclePolicyPreview") defer b.mu.Unlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } if policyText == "" { - policyText = b.lifecyclePolicies[repositoryName] + if entry, ok := b.lifecyclePolicies.Get(repositoryName); ok { + policyText = entry.PolicyText + } } - expired := evaluateLifecyclePolicy(policyText, b.images[repositoryName], b.digestTagsIndex[repositoryName]) + expired := evaluateLifecyclePolicy( + policyText, b.imagesByRepo.Get(repositoryName), b.digestTagsIndex[repositoryName]) preview := &LifecyclePolicyPreviewResult{ LifecyclePolicyText: policyText, @@ -1553,7 +1652,7 @@ func (b *InMemoryBackend) StartLifecyclePolicyPreview( RegistryID: b.accountID, Status: scanStatusComplete, } - b.lifecyclePolicyPreviews[repositoryName] = preview + b.lifecyclePolicyPreviews.Put(preview) cp := *preview cp.PreviewResults = append([]ImageIdentifier(nil), preview.PreviewResults...) @@ -1569,12 +1668,12 @@ func (b *InMemoryBackend) DeletePullThroughCacheRule( b.mu.Lock("DeletePullThroughCacheRule") defer b.mu.Unlock() - rule, ok := b.pullThroughCacheRules[prefix] + rule, ok := b.pullThroughCacheRules.Get(prefix) if !ok { return nil, fmt.Errorf("%w: %s", ErrPullThroughCacheRuleNotFound, prefix) } - delete(b.pullThroughCacheRules, prefix) + b.pullThroughCacheRules.Delete(prefix) cp := *rule @@ -1589,7 +1688,7 @@ func (b *InMemoryBackend) UpdatePullThroughCacheRule( b.mu.Lock("UpdatePullThroughCacheRule") defer b.mu.Unlock() - rule, ok := b.pullThroughCacheRules[prefix] + rule, ok := b.pullThroughCacheRules.Get(prefix) if !ok { return nil, fmt.Errorf("%w: %s", ErrPullThroughCacheRuleNotFound, prefix) } @@ -1616,7 +1715,7 @@ func (b *InMemoryBackend) ValidatePullThroughCacheRule( b.mu.RLock("ValidatePullThroughCacheRule") defer b.mu.RUnlock() - rule, ok := b.pullThroughCacheRules[prefix] + rule, ok := b.pullThroughCacheRules.Get(prefix) if !ok { return &ValidatePullThroughCacheRuleResult{ EcrRepositoryPrefix: prefix, @@ -1643,12 +1742,13 @@ func (b *InMemoryBackend) AddImageInternal(repositoryName string, img Image) { b.mu.Lock("AddImageInternal") defer b.mu.Unlock() - if b.images[repositoryName] == nil { - b.images[repositoryName] = make(map[string]*Image) - } - cp := img - b.images[repositoryName][img.ImageDigest] = &cp + // RepositoryName is part of the store.Table composite key (see + // imageTableKey); normalize it to the map-scoping repositoryName argument + // exactly as PutImage's normalizeImageFields does, so a test-seeded image + // is found by repo-scoped lookups the same way a PutImage-created one is. + cp.RepositoryName = repositoryName + b.images.Put(&cp) if img.ImageID.ImageTag != "" { if b.tagIndex[repositoryName] == nil { @@ -1791,17 +1891,17 @@ func (b *InMemoryBackend) GetRepositoryPolicy( b.mu.RLock("GetRepositoryPolicy") defer b.mu.RUnlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } - policyText, ok := b.repositoryPolicies[repositoryName] + entry, ok := b.repositoryPolicies.Get(repositoryName) if !ok { return nil, fmt.Errorf("%w: %s", ErrRepositoryPolicyNotFound, repositoryName) } return &RepositoryPolicyResult{ - PolicyText: policyText, + PolicyText: entry.PolicyText, RegistryID: b.accountID, RepositoryName: repositoryName, }, nil @@ -1815,11 +1915,11 @@ func (b *InMemoryBackend) SetRepositoryPolicy( b.mu.Lock("SetRepositoryPolicy") defer b.mu.Unlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } - b.repositoryPolicies[repositoryName] = policyText + b.repositoryPolicies.Put(&repositoryPolicyEntry{RepositoryName: repositoryName, PolicyText: policyText}) return &RepositoryPolicyResult{ PolicyText: policyText, @@ -1836,16 +1936,17 @@ func (b *InMemoryBackend) DeleteRepositoryPolicy( b.mu.Lock("DeleteRepositoryPolicy") defer b.mu.Unlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } - policyText, ok := b.repositoryPolicies[repositoryName] + entry, ok := b.repositoryPolicies.Get(repositoryName) if !ok { return nil, fmt.Errorf("%w: %s", ErrRepositoryPolicyNotFound, repositoryName) } - delete(b.repositoryPolicies, repositoryName) + policyText := entry.PolicyText + b.repositoryPolicies.Delete(repositoryName) return &RepositoryPolicyResult{ PolicyText: policyText, @@ -1899,11 +2000,11 @@ func (b *InMemoryBackend) DescribeImageSigningStatus( b.mu.RLock("DescribeImageSigningStatus") defer b.mu.RUnlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } - if _, ok := findImageLocked(b.images[repositoryName], b.tagIndex[repositoryName], imageID); !ok { + if _, ok := findImageLocked(b.images, b.imagesByRepo, repositoryName, b.tagIndex[repositoryName], imageID); !ok { return nil, fmt.Errorf("%w: image not found", ErrRepositoryNotFound) } @@ -1931,17 +2032,17 @@ func (b *InMemoryBackend) DescribeImageScanFindings( b.mu.RLock("DescribeImageScanFindings") defer b.mu.RUnlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, "", fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } - img, ok := findImageLocked(b.images[repositoryName], b.tagIndex[repositoryName], imageID) + img, ok := findImageLocked(b.images, b.imagesByRepo, repositoryName, b.tagIndex[repositoryName], imageID) if !ok { return nil, "", fmt.Errorf("%w: image not found", ErrImageNotFound) } - findings := b.imageScanFindings[repositoryName][img.ImageDigest] - if findings == nil { + findings, ok := b.imageScanFindings.Get(findingsTableKey(repositoryName, img.ImageDigest)) + if !ok { return nil, "", fmt.Errorf("%w: image scan not found for %s in %s", ErrScanNotFoundException, img.ImageDigest, repositoryName) } @@ -2000,23 +2101,19 @@ func (b *InMemoryBackend) StartImageScan(ctx context.Context, //nolint:revive // b.mu.Lock("StartImageScan") defer b.mu.Unlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } - img, ok := findImageLocked(b.images[repositoryName], b.tagIndex[repositoryName], imageID) + img, ok := findImageLocked(b.images, b.imagesByRepo, repositoryName, b.tagIndex[repositoryName], imageID) if !ok { return nil, fmt.Errorf("%w: image not found", ErrImageNotFound) } - if b.imageScanFindings[repositoryName] == nil { - b.imageScanFindings[repositoryName] = make(map[string]*ImageScanFindingsResult) - } - result := generateMockScanFindings( img.ImageDigest, repositoryName, b.accountID, img.ImageID, b.effectiveScanTypeLocked(), ) - b.imageScanFindings[repositoryName][img.ImageDigest] = result + b.imageScanFindings.Put(result) return &ImageScanStartResult{ ImageID: img.ImageID, @@ -2061,7 +2158,7 @@ func (b *InMemoryBackend) ListImages( b.mu.RLock("ListImages") defer b.mu.RUnlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } @@ -2070,8 +2167,10 @@ func (b *InMemoryBackend) ListImages( // Build a reverse map: digest → []tag from tagIndex. digestTags := buildDigestTagsLocked(repoTagIdx) - out := make([]ImageIdentifier, 0, len(b.images[repositoryName])) - for _, img := range b.images[repositoryName] { + repoImages := b.imagesByRepo.Get(repositoryName) + + out := make([]ImageIdentifier, 0, len(repoImages)) + for _, img := range repoImages { tags := imageTagsLocked(img, digestTags) isTagged := len(tags) > 0 @@ -2109,11 +2208,11 @@ func (b *InMemoryBackend) ListImageReferrers( b.mu.RLock("ListImageReferrers") defer b.mu.RUnlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } - if _, ok := findImageLocked(b.images[repositoryName], b.tagIndex[repositoryName], subject); !ok && + if _, ok := findImageLocked(b.images, b.imagesByRepo, repositoryName, b.tagIndex[repositoryName], subject); !ok && subject.ImageDigest != "" { return nil, fmt.Errorf("%w: image not found", ErrImageNotFound) } @@ -2124,16 +2223,16 @@ func (b *InMemoryBackend) ListImageReferrers( // retagImageLocked moves a tag to a new digest: if the tag already maps to a // different digest, it clears the old image's ImageTag field so it becomes untagged. func retagImageLocked( - repoImages map[string]*Image, + images *store.Table[Image], repoTags map[string]string, - tag, newDigest string, + repositoryName, tag, newDigest string, ) { oldDigest, has := repoTags[tag] if !has || oldDigest == newDigest { return } - if oldImg, exists := repoImages[oldDigest]; exists { + if oldImg, exists := images.Get(imageTableKey(repositoryName, oldDigest)); exists { if oldImg.ImageID.ImageTag == tag { oldImg.ImageID.ImageTag = "" } @@ -2170,7 +2269,7 @@ func normalizeImageFields(image *Image, repositoryName, accountID string) { } } -func (b *InMemoryBackend) PutImage( //nolint:cyclop // complexity matches AWS PutImage contract +func (b *InMemoryBackend) PutImage( ctx context.Context, //nolint:revive // existing issue. repositoryName string, image Image, @@ -2178,7 +2277,7 @@ func (b *InMemoryBackend) PutImage( //nolint:cyclop // complexity matches AWS Pu b.mu.Lock("PutImage") defer b.mu.Unlock() - repo, ok := b.repos[repositoryName] + repo, ok := b.repos.Get(repositoryName) if !ok { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } @@ -2210,17 +2309,13 @@ func (b *InMemoryBackend) PutImage( //nolint:cyclop // complexity matches AWS Pu // If tag already points to a different digest, untag the old image. if tag != "" { - retagImageLocked(b.images[repositoryName], repoTags, tag, image.ImageDigest) + retagImageLocked(b.images, repoTags, repositoryName, tag, image.ImageDigest) } normalizeImageFields(&image, repositoryName, b.accountID) - if b.images[repositoryName] == nil { - b.images[repositoryName] = make(map[string]*Image) - } - stored := image - b.images[repositoryName][image.ImageDigest] = &stored + b.images.Put(&stored) // Update tag index and keep digestTagsIndex in sync. if tag != "" { @@ -2248,7 +2343,7 @@ func (b *InMemoryBackend) PutImageScanningConfiguration( b.mu.Lock("PutImageScanningConfiguration") defer b.mu.Unlock() - repo, ok := b.repos[repositoryName] + repo, ok := b.repos.Get(repositoryName) if !ok { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } @@ -2272,7 +2367,7 @@ func (b *InMemoryBackend) PutImageTagMutability( b.mu.Lock("PutImageTagMutability") defer b.mu.Unlock() - repo, ok := b.repos[repositoryName] + repo, ok := b.repos.Get(repositoryName) if !ok { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } @@ -2299,11 +2394,11 @@ func (b *InMemoryBackend) DescribeImageReplicationStatus( b.mu.RLock("DescribeImageReplicationStatus") defer b.mu.RUnlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } - img, ok := findImageLocked(b.images[repositoryName], b.tagIndex[repositoryName], imageID) + img, ok := findImageLocked(b.images, b.imagesByRepo, repositoryName, b.tagIndex[repositoryName], imageID) if !ok { return nil, fmt.Errorf("%w: image not found", ErrImageNotFound) } @@ -2409,11 +2504,11 @@ func (b *InMemoryBackend) UpdateImageStorageClass( b.mu.Lock("UpdateImageStorageClass") defer b.mu.Unlock() - if _, ok := b.repos[repositoryName]; !ok { + if !b.repos.Has(repositoryName) { return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) } - img, ok := findImageLocked(b.images[repositoryName], b.tagIndex[repositoryName], imageID) + img, ok := findImageLocked(b.images, b.imagesByRepo, repositoryName, b.tagIndex[repositoryName], imageID) if !ok { return nil, fmt.Errorf("%w: image not found", ErrImageNotFound) } @@ -2442,7 +2537,11 @@ func (b *InMemoryBackend) GetAccountSetting( b.mu.RLock("GetAccountSetting") defer b.mu.RUnlock() - return b.accountSettings[name], nil + if entry, ok := b.accountSettings.Get(name); ok { + return entry.Value, nil + } + + return "", nil } // PutAccountSetting updates a registry account setting. @@ -2453,7 +2552,7 @@ func (b *InMemoryBackend) PutAccountSetting( b.mu.Lock("PutAccountSetting") defer b.mu.Unlock() - b.accountSettings[name] = value + b.accountSettings.Put(&accountSettingEntry{Name: name, Value: value}) return value, nil } @@ -2467,7 +2566,7 @@ func (b *InMemoryBackend) RegisterPullTimeUpdateExclusion( defer b.mu.Unlock() exclusion := &PullTimeUpdateExclusion{CreatedAt: time.Now(), PrincipalArn: principalArn} - b.pullTimeUpdateExclusions[principalArn] = exclusion + b.pullTimeUpdateExclusions.Put(exclusion) cp := *exclusion return &cp, nil @@ -2481,12 +2580,12 @@ func (b *InMemoryBackend) DeregisterPullTimeUpdateExclusion( b.mu.Lock("DeregisterPullTimeUpdateExclusion") defer b.mu.Unlock() - exclusion, ok := b.pullTimeUpdateExclusions[principalArn] + exclusion, ok := b.pullTimeUpdateExclusions.Get(principalArn) if !ok { return &PullTimeUpdateExclusion{PrincipalArn: principalArn}, nil } - delete(b.pullTimeUpdateExclusions, principalArn) + b.pullTimeUpdateExclusions.Delete(principalArn) cp := *exclusion return &cp, nil @@ -2499,8 +2598,8 @@ func (b *InMemoryBackend) ListPullTimeUpdateExclusions( b.mu.RLock("ListPullTimeUpdateExclusions") defer b.mu.RUnlock() - out := make([]PullTimeUpdateExclusion, 0, len(b.pullTimeUpdateExclusions)) - for _, exclusion := range b.pullTimeUpdateExclusions { + out := make([]PullTimeUpdateExclusion, 0, b.pullTimeUpdateExclusions.Len()) + for _, exclusion := range b.pullTimeUpdateExclusions.All() { out = append(out, *exclusion) } @@ -2517,7 +2616,7 @@ func (b *InMemoryBackend) UpdateRepositoryCreationTemplate( b.mu.Lock("UpdateRepositoryCreationTemplate") defer b.mu.Unlock() - tmpl, ok := b.repositoryCreationTemplates[req.Prefix] + tmpl, ok := b.repositoryCreationTemplates.Get(req.Prefix) if !ok { return nil, fmt.Errorf("%w: %s", ErrRepositoryCreationTemplateNotFound, req.Prefix) } @@ -2605,30 +2704,30 @@ func sortedTagKeys(tags map[string]string) []string { return keys } -// findImageLocked looks up an image by digest or tag. +// findImageLocked looks up an image by digest or tag within repositoryName. // tagIdx is the per-repository tag→digest index; it may be nil for older callers. +// imagesByRepo is only consulted by the fallback linear scan (images that +// predate the tag index). func findImageLocked( - images map[string]*Image, + images *store.Table[Image], + imagesByRepo *store.Index[Image], + repositoryName string, tagIdx map[string]string, id ImageIdentifier, ) (*Image, bool) { if id.ImageDigest != "" { - img, ok := images[id.ImageDigest] - - return img, ok + return images.Get(imageTableKey(repositoryName, id.ImageDigest)) } if id.ImageTag != "" { // Fast path via tag index. if tagIdx != nil { if digest, ok := tagIdx[id.ImageTag]; ok { - img, exists := images[digest] - - return img, exists + return images.Get(imageTableKey(repositoryName, digest)) } } // Fallback: linear scan for images without tag index entry. - for _, img := range images { + for _, img := range imagesByRepo.Get(repositoryName) { if img.ImageID.ImageTag == id.ImageTag { return img, true } @@ -2822,22 +2921,13 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.repos = make(map[string]*Repository) - b.images = make(map[string]map[string]*Image) + b.registry.ResetAll() b.tagIndex = make(map[string]map[string]string) b.digestTagsIndex = make(map[string]map[string][]string) - b.pullThroughCacheRules = make(map[string]*PullThroughCacheRule) - b.repositoryCreationTemplates = make(map[string]*RepositoryCreationTemplate) - b.lifecyclePolicies = make(map[string]string) - b.lifecyclePolicyPreviews = make(map[string]*LifecyclePolicyPreviewResult) b.lifecycleLastEvaluated = make(map[string]time.Time) b.uploadedLayers = make(map[string]map[string]int64) b.layerUploads = make(map[string]*layerUploadState) b.repoTags = make(map[string]map[string]string) - b.repositoryPolicies = make(map[string]string) - b.imageScanFindings = make(map[string]map[string]*ImageScanFindingsResult) - b.accountSettings = make(map[string]string) - b.pullTimeUpdateExclusions = make(map[string]*PullTimeUpdateExclusion) b.registryPolicy = "" b.registryScanningConfig = &RegistryScanningSettings{ScanType: scanTypeBasic} b.replicationConfig = &ReplicationConfig{} @@ -2850,7 +2940,7 @@ func (b *InMemoryBackend) AddRepositoryInternal(repo Repository) { defer b.mu.Unlock() cp := repo - b.repos[repo.RepositoryName] = &cp + b.repos.Put(&cp) } // AddLifecyclePolicyInternal seeds a lifecycle policy directly into the backend for testing. @@ -2858,5 +2948,5 @@ func (b *InMemoryBackend) AddLifecyclePolicyInternal(repositoryName, policy stri b.mu.Lock("AddLifecyclePolicyInternal") defer b.mu.Unlock() - b.lifecyclePolicies[repositoryName] = policy + b.lifecyclePolicies.Put(&lifecyclePolicyEntry{RepositoryName: repositoryName, PolicyText: policy}) } diff --git a/services/ecr/export_test.go b/services/ecr/export_test.go index 2cd142511..bc1776ee1 100644 --- a/services/ecr/export_test.go +++ b/services/ecr/export_test.go @@ -9,12 +9,12 @@ func (b *InMemoryBackend) CreateRepoInternal(repositoryName string) { b.mu.Lock("CreateRepoInternal") defer b.mu.Unlock() - if _, ok := b.repos[repositoryName]; !ok { - b.repos[repositoryName] = &Repository{ + if !b.repos.Has(repositoryName) { + b.repos.Put(&Repository{ RepositoryName: repositoryName, RegistryID: b.accountID, ImageTagMutability: "MUTABLE", - } + }) } } @@ -24,13 +24,7 @@ func (b *InMemoryBackend) ImageCount() int { b.mu.RLock("ImageCount") defer b.mu.RUnlock() - total := 0 - - for _, imgs := range b.images { - total += len(imgs) - } - - return total + return b.images.Len() } // PullThroughCacheRuleCount returns the number of pull-through cache rules stored. @@ -38,7 +32,7 @@ func (b *InMemoryBackend) PullThroughCacheRuleCount() int { b.mu.RLock("PullThroughCacheRuleCount") defer b.mu.RUnlock() - return len(b.pullThroughCacheRules) + return b.pullThroughCacheRules.Len() } // RepositoryCreationTemplateCount returns the number of repository creation templates stored. @@ -46,7 +40,7 @@ func (b *InMemoryBackend) RepositoryCreationTemplateCount() int { b.mu.RLock("RepositoryCreationTemplateCount") defer b.mu.RUnlock() - return len(b.repositoryCreationTemplates) + return b.repositoryCreationTemplates.Len() } // RepositoryCount returns the number of repositories. @@ -55,7 +49,7 @@ func (b *InMemoryBackend) RepositoryCount() int { b.mu.RLock("RepositoryCount") defer b.mu.RUnlock() - return len(b.repos) + return b.repos.Len() } // LifecyclePolicyCount returns the number of lifecycle policies stored. @@ -63,7 +57,7 @@ func (b *InMemoryBackend) LifecyclePolicyCount() int { b.mu.RLock("LifecyclePolicyCount") defer b.mu.RUnlock() - return len(b.lifecyclePolicies) + return b.lifecyclePolicies.Len() } // UploadedLayerCount returns the total number of uploaded layers. @@ -142,7 +136,7 @@ func (b *InMemoryBackend) RepoImageCount(repositoryName string) int { b.mu.RLock("RepoImageCount") defer b.mu.RUnlock() - return len(b.images[repositoryName]) + return len(b.imagesByRepo.Get(repositoryName)) } // HasImageDigest reports whether an image with the given digest exists in the repo. @@ -150,9 +144,7 @@ func (b *InMemoryBackend) HasImageDigest(repositoryName, digest string) bool { b.mu.RLock("HasImageDigest") defer b.mu.RUnlock() - _, ok := b.images[repositoryName][digest] - - return ok + return b.images.Has(imageTableKey(repositoryName, digest)) } // LifecycleLastEvaluatedForTest returns the recorded lifecycle evaluation time @@ -179,7 +171,7 @@ func (b *InMemoryBackend) AgeImageForTest(repositoryName, digest string, d time. b.mu.Lock("AgeImageForTest") defer b.mu.Unlock() - if img, ok := b.images[repositoryName][digest]; ok { + if img, ok := b.images.Get(imageTableKey(repositoryName, digest)); ok { img.ImagePushedAt = img.ImagePushedAt.Add(-d) } } diff --git a/services/ecr/handler.go b/services/ecr/handler.go index 125efb634..71f7bd16d 100644 --- a/services/ecr/handler.go +++ b/services/ecr/handler.go @@ -2,7 +2,9 @@ package ecr import ( "context" + "crypto/sha256" "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -484,6 +486,12 @@ func (h *Handler) classifyError(err error) (int, string) { //nolint:cyclop // 1 return http.StatusBadRequest, "LayerInaccessibleException" case errors.Is(err, ErrLayersNotFound): return http.StatusBadRequest, "LayersNotFoundException" + case errors.Is(err, ErrLayerAlreadyExists): + return http.StatusBadRequest, "LayerAlreadyExistsException" + case errors.Is(err, ErrInvalidLayerPart): + return http.StatusBadRequest, "InvalidLayerPartException" + case errors.Is(err, ErrImageDigestDoesNotMatch): + return http.StatusBadRequest, "ImageDigestDoesNotMatchException" case errors.Is(err, ErrPullThroughCacheRuleAlreadyExists): return http.StatusBadRequest, "PullThroughCacheRuleAlreadyExistsException" case errors.Is(err, ErrRepositoryCreationTemplateAlreadyExists): @@ -1952,6 +1960,22 @@ type putImageOutput struct { } func (h *Handler) handlePutImage(ctx context.Context, in *putImageInput) (*putImageOutput, error) { + // AWS validates a caller-supplied imageDigest against the digest it computes + // from the manifest and rejects a mismatch with ImageDigestDoesNotMatchException, + // independent of any backend state (this is pure request validation). + if in.ImageDigest != "" { + sum := sha256.Sum256([]byte(in.ImageManifest)) + computed := "sha256:" + hex.EncodeToString(sum[:]) + + if in.ImageDigest != computed { + return nil, fmt.Errorf( + "%w: manifest validation failed, digest calculated from the image manifest does"+ + " not match the provided digest", + ErrImageDigestDoesNotMatch, + ) + } + } + img, err := h.Backend.PutImage(ctx, in.RepositoryName, Image{ ImageDigest: in.ImageDigest, ImageManifest: in.ImageManifest, diff --git a/services/ecr/handler_test.go b/services/ecr/handler_test.go index 79be3dfbc..62dc15b1d 100644 --- a/services/ecr/handler_test.go +++ b/services/ecr/handler_test.go @@ -1229,6 +1229,13 @@ func TestECR_CompleteLayerUpload(t *testing.T) { //nolint:paralleltest // existi t.Run(tt.name, func(t *testing.T) { h := newTestHandler(t) + // The repository must exist: real ECR returns RepositoryNotFoundException + // for CompleteLayerUpload against an unknown repository. + createRec := doECRRequest( + t, h, "CreateRepository", map[string]any{"repositoryName": tt.repositoryName}, + ) + require.Equal(t, http.StatusOK, createRec.Code) + rec := doECRRequest(t, h, "CompleteLayerUpload", map[string]any{ "repositoryName": tt.repositoryName, "uploadId": tt.uploadID, @@ -1697,12 +1704,14 @@ func TestECR_NewOps_PersistenceRoundTrip(t *testing.T) { //nolint:paralleltest / rec = doECRRequest(t, h, "CompleteLayerUpload", map[string]any{ "repositoryName": "persist-repo", "uploadId": "upload-xyz", - "layerDigests": []string{"sha256:persist123"}, + "layerDigests": []string{ + "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + }, }) require.Equal(t, http.StatusOK, rec.Code) rec = doECRRequest(t, h, "PutImage", map[string]any{ "repositoryName": "persist-repo", - "imageDigest": "sha256:persist123", + "imageDigest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", "imageManifest": "{}", }) require.Equal(t, http.StatusOK, rec.Code) @@ -1731,7 +1740,9 @@ func TestECR_NewOps_PersistenceRoundTrip(t *testing.T) { //nolint:paralleltest / require.Equal(t, http.StatusOK, rec.Code) rec = doECRRequest(t, h, "StartImageScan", map[string]any{ "repositoryName": "persist-repo", - "imageId": map[string]any{"imageDigest": "sha256:persist123"}, + "imageId": map[string]any{ + "imageDigest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + }, }) require.Equal(t, http.StatusOK, rec.Code) rec = doECRRequest(t, h, "PutRegistryScanningConfiguration", map[string]any{ @@ -1780,7 +1791,9 @@ func TestECR_NewOps_PersistenceRoundTrip(t *testing.T) { //nolint:paralleltest / // Verify layer availability is restored rec = doECRRequest(t, h2, "BatchCheckLayerAvailability", map[string]any{ "repositoryName": "persist-repo", - "layerDigests": []string{"sha256:persist123"}, + "layerDigests": []string{ + "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + }, }) require.Equal(t, http.StatusOK, rec.Code) @@ -1814,7 +1827,9 @@ func TestECR_NewOps_PersistenceRoundTrip(t *testing.T) { //nolint:paralleltest / require.Equal(t, http.StatusOK, rec.Code) rec = doECRRequest(t, h2, "DescribeImageScanFindings", map[string]any{ "repositoryName": "persist-repo", - "imageId": map[string]any{"imageDigest": "sha256:persist123"}, + "imageId": map[string]any{ + "imageDigest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + }, }) require.Equal(t, http.StatusOK, rec.Code) rec = doECRRequest(t, h2, "GetRegistryScanningConfiguration", map[string]any{}) diff --git a/services/ecr/lifecycle.go b/services/ecr/lifecycle.go index 22f732554..19d545627 100644 --- a/services/ecr/lifecycle.go +++ b/services/ecr/lifecycle.go @@ -53,7 +53,7 @@ type imageEntry struct { // digestTags maps image digest → all tags for that image (from digestTagsIndex). func evaluateLifecyclePolicy( policyText string, - images map[string]*Image, + images []*Image, digestTags map[string][]string, ) []ImageIdentifier { if policyText == "" { @@ -141,21 +141,20 @@ func expiredIdentifier(e *imageEntry) ImageIdentifier { func (b *InMemoryBackend) applyLifecyclePolicyLocked(repositoryName string) []ImageIdentifier { b.lifecycleLastEvaluated[repositoryName] = time.Now() - policyText := b.lifecyclePolicies[repositoryName] - if policyText == "" { + entry, ok := b.lifecyclePolicies.Get(repositoryName) + if !ok || entry.PolicyText == "" { return nil } expired := evaluateLifecyclePolicy( - policyText, - b.images[repositoryName], + entry.PolicyText, + b.imagesByRepo.Get(repositoryName), b.digestTagsIndex[repositoryName], ) if len(expired) == 0 { return nil } - repoImages := b.images[repositoryName] repoTags := b.tagIndex[repositoryName] deleted := make([]ImageIdentifier, 0, len(expired)) @@ -169,15 +168,12 @@ func (b *InMemoryBackend) applyLifecyclePolicyLocked(repositoryName string) []Im continue } - if !deleteByDigestLocked(repoImages, repoTags, digest) { + if !deleteByDigestLocked(b.images, repoTags, repositoryName, digest) { continue } b.clearDigestTagsLocked(repositoryName, digest) - - if findings := b.imageScanFindings[repositoryName]; findings != nil { - delete(findings, digest) - } + b.imageScanFindings.Delete(findingsTableKey(repositoryName, digest)) deleted = append(deleted, ImageIdentifier{ImageDigest: digest, ImageTag: id.ImageTag}) } @@ -196,14 +192,14 @@ func (b *InMemoryBackend) RunLifecycleExpiry(ctx context.Context) int { total := 0 - for repositoryName := range b.lifecyclePolicies { + for _, entry := range b.lifecyclePolicies.All() { select { case <-ctx.Done(): return total default: } - total += len(b.applyLifecyclePolicyLocked(repositoryName)) + total += len(b.applyLifecyclePolicyLocked(entry.RepositoryName)) } return total diff --git a/services/ecr/parity_sweep3_test.go b/services/ecr/parity_sweep3_test.go new file mode 100644 index 000000000..715285273 --- /dev/null +++ b/services/ecr/parity_sweep3_test.go @@ -0,0 +1,254 @@ +package ecr_test + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/ecr" +) + +// ── CompleteLayerUpload: FK validation + LayerAlreadyExistsException ──────── + +// Test_CompleteLayerUpload_Validation covers two gaps found in the +// parity-sweep-3 audit: CompleteLayerUpload never validated that the target +// repository exists (every other backend op does), and never rejected +// re-completing a layer digest that was already registered as available, +// which real ECR rejects with LayerAlreadyExistsException. +func Test_CompleteLayerUpload_Validation(t *testing.T) { + t.Parallel() + + cases := []struct { + setup func(t *testing.T) (h *ecr.Handler, repoName string) + name string + wantType string + wantStatus int + }{ + { + name: "repository not found", + setup: func(t *testing.T) (*ecr.Handler, string) { + t.Helper() + + return newAccuracyHandler(), "does-not-exist" + }, + wantStatus: http.StatusNotFound, + wantType: "RepositoryNotFoundException", + }, + { + name: "duplicate layer digest rejected", + setup: func(t *testing.T) (*ecr.Handler, string) { + t.Helper() + + h := newAccuracyHandler() + mustCreateRepo(t, h, "dup-layer-repo") + + rec := doAccuracy(t, h, "CompleteLayerUpload", map[string]any{ + "repositoryName": "dup-layer-repo", + "uploadId": "upload-1", + "layerDigests": []string{"sha256:aaaa"}, + }) + require.Equal( + t, http.StatusOK, rec.Code, "seed CompleteLayerUpload failed: %s", rec.Body.String(), + ) + + return h, "dup-layer-repo" + }, + wantStatus: http.StatusBadRequest, + wantType: "LayerAlreadyExistsException", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h, repoName := tc.setup(t) + + rec := doAccuracy(t, h, "CompleteLayerUpload", map[string]any{ + "repositoryName": repoName, + "uploadId": "upload-1", + "layerDigests": []string{"sha256:aaaa"}, + }) + + assert.Equal(t, tc.wantStatus, rec.Code) + out := parseAccuracy(t, rec) + assert.Equal(t, tc.wantType, out["__type"]) + }) + } +} + +// Test_CompleteLayerUpload_FreshDigestSucceeds is a control case ensuring the +// new LayerAlreadyExistsException check does not reject the first, legitimate +// completion of a digest. +func Test_CompleteLayerUpload_FreshDigestSucceeds(t *testing.T) { + t.Parallel() + + h := newAccuracyHandler() + mustCreateRepo(t, h, "fresh-layer-repo") + + rec := doAccuracy(t, h, "CompleteLayerUpload", map[string]any{ + "repositoryName": "fresh-layer-repo", + "uploadId": "upload-fresh", + "layerDigests": []string{"sha256:bbbb"}, + }) + assert.Equal(t, http.StatusOK, rec.Code) +} + +// ── UploadLayerPart: part sequencing ───────────────────────────────────────── + +// Test_UploadLayerPart_PartSequencing covers the "part sequencing" gap called +// out in the parity-sweep-3 audit: AWS requires each part's first byte to be +// consecutive to the last byte already received in the same upload session, +// rejecting gaps/overlaps with InvalidLayerPartException. +func Test_UploadLayerPart_PartSequencing(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + wantType string + firstBytes []int64 + blobs []string + wantLastStatus int + }{ + { + name: "consecutive parts accepted", + firstBytes: []int64{0, 4}, + blobs: []string{"AQIDBA==", "BQYHCA=="}, // 4 bytes each + wantLastStatus: http.StatusOK, + }, + { + name: "first part must start at zero", + firstBytes: []int64{4}, + blobs: []string{"AQIDBA=="}, + wantLastStatus: http.StatusBadRequest, + wantType: "InvalidLayerPartException", + }, + { + name: "gap between parts rejected", + firstBytes: []int64{0, 10}, + blobs: []string{"AQIDBA==", "BQYHCA=="}, + wantLastStatus: http.StatusBadRequest, + wantType: "InvalidLayerPartException", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h := newAccuracyHandler() + mustCreateRepo(t, h, "seq-repo") + + initRec := doAccuracy(t, h, "InitiateLayerUpload", map[string]any{ + "repositoryName": "seq-repo", + }) + require.Equal(t, http.StatusOK, initRec.Code) + uploadID, _ := parseAccuracy(t, initRec)["uploadId"].(string) + require.NotEmpty(t, uploadID) + + rec := initRec + for i, firstByte := range tc.firstBytes { + rec = doAccuracy(t, h, "UploadLayerPart", map[string]any{ + "repositoryName": "seq-repo", + "uploadId": uploadID, + "partFirstByte": firstByte, + "partLastByte": firstByte + 3, + "layerPartBlob": tc.blobs[i], + }) + } + + assert.Equal(t, tc.wantLastStatus, rec.Code) + if tc.wantType != "" { + out := parseAccuracy(t, rec) + assert.Equal(t, tc.wantType, out["__type"]) + } + }) + } +} + +// ── PutImage: imageDigest vs manifest validation ───────────────────────────── + +// Test_PutImage_DigestValidation covers the ImageDigestDoesNotMatchException +// gap found in the parity-sweep-3 audit: real ECR validates a caller-supplied +// imageDigest against the digest it computes from the manifest and rejects a +// mismatch, but the emulator previously trusted whatever digest the client sent. +func Test_PutImage_DigestValidation(t *testing.T) { + t.Parallel() + + const manifest = `{"schemaVersion":2,"digest-check":true}` + + cases := []struct { + name string + imageDigest string + wantType string + wantStatus int + }{ + { + name: "no digest supplied - server computes it", + imageDigest: "", + wantStatus: http.StatusOK, + }, + { + name: "mismatched digest rejected", + imageDigest: "sha256:" + + "0000000000000000000000000000000000000000000000000000000000000000", + wantStatus: http.StatusBadRequest, + wantType: "ImageDigestDoesNotMatchException", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h := newAccuracyHandler() + mustCreateRepo(t, h, "digest-check-repo") + + body := map[string]any{ + "repositoryName": "digest-check-repo", + "imageManifest": manifest, + "imageTag": "v1", + } + if tc.imageDigest != "" { + body["imageDigest"] = tc.imageDigest + } + + rec := doAccuracy(t, h, "PutImage", body) + assert.Equal(t, tc.wantStatus, rec.Code) + + if tc.wantType != "" { + out := parseAccuracy(t, rec) + assert.Equal(t, tc.wantType, out["__type"]) + } + }) + } +} + +// Test_PutImage_DigestValidation_MatchingDigestAccepted verifies that a +// caller-supplied imageDigest which genuinely matches the manifest's computed +// sha256 is accepted (the validation only rejects a real mismatch). +func Test_PutImage_DigestValidation_MatchingDigestAccepted(t *testing.T) { + t.Parallel() + + const manifest = `{"schemaVersion":2,"exact":"match"}` + + h := newAccuracyHandler() + mustCreateRepo(t, h, "digest-match-repo") + + // First push without an explicit digest so the server tells us the real one. + digest := mustPutImage(t, h, "digest-match-repo", "v1", manifest) + + // Re-push the same manifest, this time supplying the (correct) digest + // explicitly, as a real client that already knows it would. + rec := doAccuracy(t, h, "PutImage", map[string]any{ + "repositoryName": "digest-match-repo", + "imageManifest": manifest, + "imageTag": "v2", + "imageDigest": digest, + }) + assert.Equal( + t, http.StatusOK, rec.Code, "matching imageDigest must be accepted: %s", rec.Body.String(), + ) +} diff --git a/services/ecr/persistence.go b/services/ecr/persistence.go index 8388d18b3..e67e89a23 100644 --- a/services/ecr/persistence.go +++ b/services/ecr/persistence.go @@ -2,28 +2,57 @@ package ecr import ( "context" + "encoding/json" + "fmt" "maps" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" ) +// ecrSnapshotVersion identifies the shape of [backendSnapshot]. It must be +// bumped whenever a change to any store.Table DTO or backendSnapshot itself +// would make an older snapshot unsafe to decode as the current shape -- +// Restore compares this against the persisted value and discards (rather +// than attempts to partially decode) any mismatch, see Restore below. +// +// This is the first versioned ecr snapshot: Phase 3.3 (the pkgs/store +// conversion) moved every *T resource map registered on b.registry from a +// nested map[string]map[string]*T (or bare map[string]T) shape directly under +// backendSnapshot to a flat store.Table JSON array nested under Tables, so any +// snapshot written before this conversion is unconditionally incompatible -- +// it decodes as Version 0 via Go's json zero value, which never equals +// ecrSnapshotVersion. +const ecrSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the ECR backend. +// +// Tables holds one JSON-encoded array per store.Table registered on +// b.registry, produced by [store.Registry.SnapshotAll] -- see store_setup.go +// for the full list ("repos", "images", "imageScanFindings", +// "pullThroughCacheRules", "repositoryCreationTemplates", "lifecyclePolicies", +// "lifecyclePolicyPreviews", "repositoryPolicies", "accountSettings", +// "pullTimeUpdateExclusions"). +// +// The remaining fields persist state deliberately left off b.registry -- see +// the comment above registerAllTables in store_setup.go for why each one is +// exempt. RepoTags and UploadedLayers were persisted before this conversion +// and still are (unchanged shape/JSON tag, so an old snapshot's values for +// these two fields would in isolation still decode correctly -- but Restore +// discards the whole snapshot on a Version mismatch regardless, matching the +// services/sqs precedent that a snapshot must never be partially decoded). +// tagIndex, digestTagsIndex, layerUploads, repoUploadIndex, and +// lifecycleLastEvaluated were never persisted before this conversion and +// still aren't. type backendSnapshot struct { - RepositoryPolicies map[string]string `json:"repositoryPolicies,omitempty"` - ImageScanFindings map[string]map[string]*ImageScanFindingsResult `json:"imageScanFindings,omitempty"` - PullThroughCacheRules map[string]*PullThroughCacheRule `json:"pullThroughCacheRules"` - RepositoryCreationTemplates map[string]*RepositoryCreationTemplate `json:"repositoryCreationTemplates"` - LifecyclePolicies map[string]string `json:"lifecyclePolicies"` - LifecyclePolicyPreviews map[string]*LifecyclePolicyPreviewResult `json:"lifecyclePolicyPreviews,omitempty"` - Images map[string]map[string]*Image `json:"images"` - UploadedLayers map[string]map[string]int64 `json:"uploadedLayers"` - Repos map[string]*Repository `json:"repos"` - RepoTags map[string]map[string]string `json:"repoTags,omitempty"` - AccountSettings map[string]string `json:"accountSettings,omitempty"` - PullTimeUpdateExclusions map[string]*PullTimeUpdateExclusion `json:"pullTimeUpdateExclusions,omitempty"` - RegistryScanningConfig *RegistryScanningSettings `json:"registryScanningConfig,omitempty"` - ReplicationConfig *ReplicationConfig `json:"replicationConfig,omitempty"` - SigningConfig *SigningSettings `json:"signingConfig,omitempty"` - RegistryPolicy string `json:"registryPolicy"` + Tables map[string]json.RawMessage `json:"tables"` + RepoTags map[string]map[string]string `json:"repoTags,omitempty"` + UploadedLayers map[string]map[string]int64 `json:"uploadedLayers,omitempty"` + RegistryScanningConfig *RegistryScanningSettings `json:"registryScanningConfig,omitempty"` + ReplicationConfig *ReplicationConfig `json:"replicationConfig,omitempty"` + SigningConfig *SigningSettings `json:"signingConfig,omitempty"` + RegistryPolicy string `json:"registryPolicy"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. @@ -32,23 +61,27 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // Every registered table's value type is a plain JSON-friendly struct, + // so a marshal failure here would indicate a programming error rather + // than bad input data. Log and skip the snapshot rather than panic, + // matching the persistence.Persistable contract (nil is skipped by the + // Manager). + logger.Load(ctx).WarnContext(ctx, "ecr: snapshot table marshal failed", "error", err) + + return nil + } + snap := backendSnapshot{ - Repos: copyPointerMap(b.repos), - Images: copyNestedPointerMap(b.images), - PullThroughCacheRules: copyPointerMap(b.pullThroughCacheRules), - RepositoryCreationTemplates: copyPointerMap(b.repositoryCreationTemplates), - LifecyclePolicies: copyMap(b.lifecyclePolicies), - LifecyclePolicyPreviews: copyLifecyclePolicyPreviews(b.lifecyclePolicyPreviews), - RegistryPolicy: b.registryPolicy, - UploadedLayers: copyNestedMap(b.uploadedLayers), - RepoTags: copyNestedMap(b.repoTags), - RepositoryPolicies: copyMap(b.repositoryPolicies), - ImageScanFindings: copyImageScanFindingsMap(b.imageScanFindings), - AccountSettings: copyMap(b.accountSettings), - PullTimeUpdateExclusions: copyPointerMap(b.pullTimeUpdateExclusions), - RegistryScanningConfig: copyRegistryScanningSettings(b.registryScanningConfig), - ReplicationConfig: copyReplicationConfig(b.replicationConfig), - SigningConfig: copySigningSettings(b.signingConfig), + Version: ecrSnapshotVersion, + Tables: tables, + RepoTags: copyNestedMap(b.repoTags), + UploadedLayers: copyNestedMap(b.uploadedLayers), + RegistryScanningConfig: copyRegistryScanningSettings(b.registryScanningConfig), + ReplicationConfig: copyReplicationConfig(b.replicationConfig), + SigningConfig: copySigningSettings(b.signingConfig), + RegistryPolicy: b.registryPolicy, } return persistence.MarshalSnapshot(ctx, "ecr", snap) @@ -65,19 +98,29 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() - b.repos = copyPointerMap(snap.Repos) - b.images = copyNestedPointerMap(snap.Images) - b.pullThroughCacheRules = copyPointerMap(snap.PullThroughCacheRules) - b.repositoryCreationTemplates = copyPointerMap(snap.RepositoryCreationTemplates) - b.lifecyclePolicies = copyMap(snap.LifecyclePolicies) - b.lifecyclePolicyPreviews = copyLifecyclePolicyPreviews(snap.LifecyclePolicyPreviews) - b.registryPolicy = snap.RegistryPolicy - b.uploadedLayers = copyNestedMap(snap.UploadedLayers) + if snap.Version != ecrSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across this snapshot-format change), not data + // corruption. + logger.Load(ctx).WarnContext(ctx, + "ecr: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", ecrSnapshotVersion) + + b.registry.ResetAll() + + return nil + } + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("ecr: restore snapshot tables: %w", err) + } + b.repoTags = copyNestedMap(snap.RepoTags) - b.repositoryPolicies = copyMap(snap.RepositoryPolicies) - b.imageScanFindings = copyImageScanFindingsMap(snap.ImageScanFindings) - b.accountSettings = copyMap(snap.AccountSettings) - b.pullTimeUpdateExclusions = copyPointerMap(snap.PullTimeUpdateExclusions) + b.uploadedLayers = copyNestedMap(snap.UploadedLayers) + b.registryPolicy = snap.RegistryPolicy b.registryScanningConfig = copyRegistryScanningSettings(snap.RegistryScanningConfig) b.replicationConfig = copyReplicationConfig(snap.ReplicationConfig) b.signingConfig = copySigningSettings(snap.SigningConfig) @@ -108,54 +151,6 @@ func copyNestedMap[T any](in map[string]map[string]T) map[string]map[string]T { return cp } -func copyPointerMap[T any](in map[string]*T) map[string]*T { - cp := make(map[string]*T, len(in)) - for key, value := range in { - valueCp := *value - cp[key] = &valueCp - } - - return cp -} - -func copyNestedPointerMap[T any](in map[string]map[string]*T) map[string]map[string]*T { - cp := make(map[string]map[string]*T, len(in)) - for key, value := range in { - cp[key] = copyPointerMap(value) - } - - return cp -} - -func copyLifecyclePolicyPreviews( - in map[string]*LifecyclePolicyPreviewResult, -) map[string]*LifecyclePolicyPreviewResult { - cp := make(map[string]*LifecyclePolicyPreviewResult, len(in)) - for repo, preview := range in { - previewCp := *preview - previewCp.PreviewResults = append([]ImageIdentifier(nil), preview.PreviewResults...) - cp[repo] = &previewCp - } - - return cp -} - -func copyImageScanFindingsMap( - in map[string]map[string]*ImageScanFindingsResult, -) map[string]map[string]*ImageScanFindingsResult { - cp := make(map[string]map[string]*ImageScanFindingsResult, len(in)) - for repo, findings := range in { - repoCp := make(map[string]*ImageScanFindingsResult, len(findings)) - for digest, result := range findings { - resultCp := copyImageScanFindingsResult(result) - repoCp[digest] = &resultCp - } - cp[repo] = repoCp - } - - return cp -} - // Snapshot implements persistence.Persistable by delegating to the backend // when it implements Snapshottable. Returns nil for non-snapshottable backends. func (h *Handler) Snapshot(ctx context.Context) []byte { diff --git a/services/ecr/store_persistence_test.go b/services/ecr/store_persistence_test.go new file mode 100644 index 000000000..70626f332 --- /dev/null +++ b/services/ecr/store_persistence_test.go @@ -0,0 +1,215 @@ +package ecr_test + +// store_persistence_test.go verifies the Phase 3.3 pkgs/store conversion's +// Snapshot/Restore round-trip: every store.Table registered on b.registry +// (repos, images, imageScanFindings, pullThroughCacheRules, +// repositoryCreationTemplates, lifecyclePolicies, lifecyclePolicyPreviews, +// repositoryPolicies, accountSettings, pullTimeUpdateExclusions), plus the +// state deliberately left as plain maps/fields (repoTags, registryPolicy, +// registryScanningConfig, replicationConfig, signingConfig), survive a full +// Snapshot -> fresh backend -> Restore cycle. + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/ecr" +) + +//nolint:maintidx // exhaustively exercises every store.Table plus every raw-map/scalar field in one round trip. +func TestStorePersistence_FullStateRoundTrip(t *testing.T) { + t.Parallel() + + ctx := context.Background() + b1 := ecr.NewInMemoryBackend("123456789012", "us-east-1", "localhost:5000") + + // repos + repo, err := b1.CreateRepository(ctx, "persist-repo", "MUTABLE", false, "AES256", "") + require.NoError(t, err) + + // images (composite repo+digest key + repo index) + img, err := b1.PutImage(ctx, "persist-repo", ecr.Image{ + ImageManifest: `{"schemaVersion":2}`, + ImageID: ecr.ImageIdentifier{ImageTag: "v1"}, + }) + require.NoError(t, err) + + // imageScanFindings (composite repo+digest key + repo index) + _, err = b1.StartImageScan(ctx, "persist-repo", ecr.ImageIdentifier{ImageDigest: img.ImageDigest}) + require.NoError(t, err) + + // pullThroughCacheRules + _, err = b1.CreatePullThroughCacheRule( + ctx, "cache-prefix", "https://upstream.example.com", "", "", "", "") + require.NoError(t, err) + + // repositoryCreationTemplates + _, err = b1.CreateRepositoryCreationTemplate(ctx, &ecr.RepositoryCreationTemplate{ + Prefix: "tmpl-prefix", + Description: "a template", + }) + require.NoError(t, err) + + // lifecyclePolicies (+ applies immediately, exercising lifecyclePolicyPreviews too) + _, err = b1.PutLifecyclePolicy(ctx, "persist-repo", + `{"rules":[{"rulePriority":1,"description":"d","selection":`+ + `{"tagStatus":"any","countType":"imageCountMoreThan","countNumber":100},"action":{"type":"expire"}}]}`) + require.NoError(t, err) + + _, err = b1.StartLifecyclePolicyPreview(ctx, "persist-repo", "") + require.NoError(t, err) + + // repositoryPolicies + _, err = b1.SetRepositoryPolicy(ctx, "persist-repo", `{"Version":"2012-10-17","Statement":[]}`) + require.NoError(t, err) + + // accountSettings + _, err = b1.PutAccountSetting(ctx, "basicScanTypeVersion", "AWS_NATIVE") + require.NoError(t, err) + + // pullTimeUpdateExclusions + _, err = b1.RegisterPullTimeUpdateExclusion(ctx, "arn:aws:iam::123456789012:role/excluded") + require.NoError(t, err) + + // registryPolicy (scalar field) + _, err = b1.PutRegistryPolicy(ctx, `{"Version":"2012-10-17","Statement":[]}`) + require.NoError(t, err) + + // registryScanningConfig (scalar field) + _, err = b1.PutRegistryScanningConfiguration(ctx, &ecr.RegistryScanningSettings{ScanType: "ENHANCED"}) + require.NoError(t, err) + + // replicationConfig (scalar field) + _, err = b1.PutReplicationConfiguration(ctx, &ecr.ReplicationConfig{ + Rules: []ecr.ReplicationRule{{ + Destinations: []ecr.ReplicationDestination{{Region: "eu-west-1"}}, + }}, + }) + require.NoError(t, err) + + // signingConfig (scalar field) + _, err = b1.PutSigningConfiguration(ctx, &ecr.SigningSettings{ + Rules: []ecr.SigningRule{{SigningProfileArn: "arn:aws:signer:us-east-1:123456789012:profile/p"}}, + }) + require.NoError(t, err) + + // repoTags (raw map, keyed by ARN) + require.NoError(t, b1.TagResource(ctx, repo.RepositoryARN, map[string]string{"env": "prod"})) + + snap := b1.Snapshot(ctx) + require.NotNil(t, snap) + + b2 := ecr.NewInMemoryBackend("123456789012", "us-east-1", "localhost:5000") + require.NoError(t, b2.Restore(ctx, snap)) + + // repos + gotRepos, err := b2.DescribeRepositories(ctx, nil) + require.NoError(t, err) + require.Len(t, gotRepos, 1) + assert.Equal(t, "persist-repo", gotRepos[0].RepositoryName) + + // images + gotImages, err := b2.DescribeImages(ctx, "persist-repo", nil) + require.NoError(t, err) + require.Len(t, gotImages, 1) + assert.Equal(t, img.ImageDigest, gotImages[0].ImageDigest) + + // imageScanFindings + findings, _, err := b2.DescribeImageScanFindings( + ctx, "persist-repo", ecr.ImageIdentifier{ImageDigest: img.ImageDigest}, 0, "") + require.NoError(t, err) + assert.Equal(t, "persist-repo", findings.RepositoryName) + + // pullThroughCacheRules + rules, err := b2.DescribePullThroughCacheRules(ctx, nil) + require.NoError(t, err) + require.Len(t, rules, 1) + assert.Equal(t, "cache-prefix", rules[0].EcrRepositoryPrefix) + + // repositoryCreationTemplates + tmpls, err := b2.DescribeRepositoryCreationTemplates(ctx, nil) + require.NoError(t, err) + require.Len(t, tmpls, 1) + assert.Equal(t, "tmpl-prefix", tmpls[0].Prefix) + + // lifecyclePolicies + policy, err := b2.GetLifecyclePolicy(ctx, "persist-repo") + require.NoError(t, err) + assert.Contains(t, policy.LifecyclePolicyText, "imageCountMoreThan") + + // lifecyclePolicyPreviews + preview, err := b2.GetLifecyclePolicyPreview(ctx, "persist-repo") + require.NoError(t, err) + assert.Contains(t, preview.LifecyclePolicyText, "imageCountMoreThan") + + // repositoryPolicies + repoPolicy, err := b2.GetRepositoryPolicy(ctx, "persist-repo") + require.NoError(t, err) + assert.Contains(t, repoPolicy.PolicyText, "2012-10-17") + + // accountSettings + settingValue, err := b2.GetAccountSetting(ctx, "basicScanTypeVersion") + require.NoError(t, err) + assert.Equal(t, "AWS_NATIVE", settingValue) + + // pullTimeUpdateExclusions + exclusions, err := b2.ListPullTimeUpdateExclusions(ctx) + require.NoError(t, err) + require.Len(t, exclusions, 1) + assert.Equal(t, "arn:aws:iam::123456789012:role/excluded", exclusions[0].PrincipalArn) + + // registryPolicy + regPolicy, err := b2.GetRegistryPolicy(ctx) + require.NoError(t, err) + assert.Contains(t, regPolicy.PolicyText, "2012-10-17") + + // registryScanningConfig + scanCfg, err := b2.GetRegistryScanningConfiguration(ctx) + require.NoError(t, err) + assert.Equal(t, "ENHANCED", scanCfg.ScanType) + + // replicationConfig + registryDesc, err := b2.DescribeRegistry(ctx) + require.NoError(t, err) + require.NotNil(t, registryDesc.ReplicationConfiguration) + require.Len(t, registryDesc.ReplicationConfiguration.Rules, 1) + assert.Equal(t, "eu-west-1", registryDesc.ReplicationConfiguration.Rules[0].Destinations[0].Region) + + // signingConfig + signCfg, err := b2.GetSigningConfiguration(ctx) + require.NoError(t, err) + require.Len(t, signCfg.Rules, 1) + assert.Equal(t, "arn:aws:signer:us-east-1:123456789012:profile/p", signCfg.Rules[0].SigningProfileArn) + + // repoTags + gotTags, err := b2.ListTagsForResource(ctx, repo.RepositoryARN) + require.NoError(t, err) + assert.Equal(t, map[string]string{"env": "prod"}, gotTags) +} + +// TestStorePersistence_IncompatibleVersionDiscardsCleanly verifies that a +// pre-Phase-3.3 (unversioned) snapshot -- or any snapshot whose version +// doesn't match the current shape -- is discarded rather than partially +// decoded, matching the services/sqs precedent. +func TestStorePersistence_IncompatibleVersionDiscardsCleanly(t *testing.T) { + t.Parallel() + + ctx := context.Background() + b := ecr.NewInMemoryBackend("123456789012", "us-east-1", "localhost:5000") + + _, err := b.CreateRepository(ctx, "seed-repo", "MUTABLE", false, "AES256", "") + require.NoError(t, err) + + // An old-shape (pre-conversion, unversioned) snapshot: no "tables"/"version" + // fields at all, just the legacy top-level nested maps. + oldSnapshot := []byte(`{"repos":{"legacy-repo":{"repositoryName":"legacy-repo"}}}`) + + require.NoError(t, b.Restore(ctx, oldSnapshot)) + + gotRepos, err := b.DescribeRepositories(ctx, nil) + require.NoError(t, err) + assert.Empty(t, gotRepos, "incompatible-version snapshot must reset to empty, not partially decode") +} diff --git a/services/ecr/store_setup.go b/services/ecr/store_setup.go new file mode 100644 index 000000000..5d8634d12 --- /dev/null +++ b/services/ecr/store_setup.go @@ -0,0 +1,189 @@ +package ecr + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T (and, for a bare scalar value, map[string]T) resource field on +// InMemoryBackend whose value has a pure identity of its own is registered +// exactly once here as a *store.Table[T] on b.registry. See pkgs/store's +// package doc and the services/ec2 (commit 12e611a4, data-driven registration +// slice) and services/sqs (commit 0f09d77c, DTO-registry) conversions this +// follows, plus services/cloudwatchlogs's kmsKeyEntry-style DTO wrapping for +// the bare-scalar-value case. +// +// A handful of fields are deliberately NOT registered here and remain plain +// maps -- see the comment above registerAllTables for the full list and why. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +// --- store.Table / store.Index key functions --- + +func repoKeyFn(r *Repository) string { return r.RepositoryName } + +// imageTableKey returns the store.Table primary key for an image: repository +// name and digest joined by "@", matching the OCI/Docker image-reference +// convention (name@digest). "@" is not a valid character in an ECR repository +// name and never appears in a "sha256:..." digest, so this cannot collide. +func imageTableKey(repositoryName, digest string) string { return repositoryName + "@" + digest } + +func imageKeyFn(img *Image) string { return imageTableKey(img.RepositoryName, img.ImageDigest) } + +// imageRepoIndexKeyFn groups images by owning repository, replacing the old +// repositoryName->digest->*Image map nesting for repo-scoped access +// (DescribeImages/ListImages/DeleteRepository cascade/lifecycle evaluation). +func imageRepoIndexKeyFn(img *Image) string { return img.RepositoryName } + +// findingsTableKey returns the store.Table primary key for an image scan +// findings record; it mirrors imageTableKey exactly since a findings record's +// identity is the same (repository, digest) pair as the image it describes. +func findingsTableKey(repositoryName, digest string) string { + return imageTableKey(repositoryName, digest) +} + +func findingsKeyFn(f *ImageScanFindingsResult) string { + return findingsTableKey(f.RepositoryName, f.ImageID.ImageDigest) +} + +// findingsRepoIndexKeyFn groups scan findings by owning repository, replacing +// the old repositoryName->digest->*ImageScanFindingsResult map nesting +// (DeleteRepository cascade cleanup). +func findingsRepoIndexKeyFn(f *ImageScanFindingsResult) string { return f.RepositoryName } + +func pullThroughCacheRuleKeyFn(r *PullThroughCacheRule) string { return r.EcrRepositoryPrefix } + +func repositoryCreationTemplateKeyFn(t *RepositoryCreationTemplate) string { return t.Prefix } + +func lifecyclePolicyPreviewKeyFn(p *LifecyclePolicyPreviewResult) string { return p.RepositoryName } + +func pullTimeUpdateExclusionKeyFn(e *PullTimeUpdateExclusion) string { return e.PrincipalArn } + +// lifecyclePolicyEntry wraps the bare lifecycle-policy-text value (previously +// map[string]string) in a minimal identified struct, since store.Table +// requires values to carry their own primary key (matches +// services/cloudwatchlogs's kmsKeyEntry-style wrapping of a bare +// map[string]string). lifecycleLastEvaluated is intentionally NOT folded into +// this entry -- it was never part of backendSnapshot before this conversion +// (GetLifecyclePolicy's LastEvaluatedAt does not survive a restore today), and +// folding it in would silently start persisting data that wasn't persisted +// before. It remains its own plain map; see registerAllTables below. +type lifecyclePolicyEntry struct { + RepositoryName string `json:"repositoryName"` + PolicyText string `json:"policyText"` +} + +func lifecyclePolicyEntryKeyFn(e *lifecyclePolicyEntry) string { return e.RepositoryName } + +// repositoryPolicyEntry wraps the bare repository-policy-text value +// (previously map[string]string); see lifecyclePolicyEntry's doc. +type repositoryPolicyEntry struct { + RepositoryName string `json:"repositoryName"` + PolicyText string `json:"policyText"` +} + +func repositoryPolicyEntryKeyFn(e *repositoryPolicyEntry) string { return e.RepositoryName } + +// accountSettingEntry wraps a bare account-setting value (previously +// map[string]string); see lifecyclePolicyEntry's doc. +type accountSettingEntry struct { + Name string `json:"name"` + Value string `json:"value"` +} + +func accountSettingEntryKeyFn(e *accountSettingEntry) string { return e.Name } + +// registerAllTables registers every converted resource table on b.registry +// exactly once. It must be called during construction only (immediately +// after b.registry is created), never on every Reset() -- store.Register +// panics on a duplicate name, so runtime resets go through +// b.registry.ResetAll() instead (see InMemoryBackend.Reset in backend.go). +// +// The following resource fields are deliberately left as plain maps (NOT +// registered here): +// +// - repoTags: map[string]map[string]string keyed by resource ARN; the value +// (a bare tag map) carries no identity field of its own to serve as a +// store.Table key (matches services/s3's "tags" field exception). Still +// persisted directly in backendSnapshot, exactly as before this +// conversion. +// - tagIndex: map[string]map[string]string, repo -> tag -> digest; the +// value (a digest string) carries no identity field of its own, and the +// nesting is itself a secondary index derived from images, not a resource +// in its own right. Was never persisted before this conversion (a +// restored backend rebuilds nothing into it -- a pre-existing gap, not a +// regression introduced here) and still isn't. +// - digestTagsIndex: map[string]map[string][]string, repo -> digest -> +// []tag; slice-valued, no identity field of its own. Same never-persisted +// status as tagIndex, preserved as-is. +// - uploadedLayers: map[string]map[string]int64, repo -> digest -> size; the +// value (int64) carries no identity field of its own (matches +// services/apigateway's usagePlanKeys exception for a bare int64). Still +// persisted directly in backendSnapshot, exactly as before. +// - layerUploads: map[string]*layerUploadState; layerUploadState carries no +// upload-ID field of its own (the ID is the map key only), the same "no +// identity field" exception documented for EC2's instanceIMDSOptions +// (commit 12e611a4). Left unconverted so the parity-sweep layer +// part-sequencing / digest-verification logic in CompleteLayerUpload and +// UploadLayerPart is untouched. Never persisted before this conversion +// (in-progress uploads do not survive a restart, matching AWS) and still +// isn't. +// - repoUploadIndex: map[string]map[string]struct{}, a secondary index over +// layerUploads (itself unconverted above); non-*T value, no identity +// field. Never persisted, still isn't. +// - lifecycleLastEvaluated: map[string]time.Time; a bare time.Time carries +// no identity field of its own. Never persisted before this conversion +// (GetLifecyclePolicy's LastEvaluatedAt resets to zero across a restore +// today), still isn't -- see lifecyclePolicyEntry's doc for why it was +// not folded into that table instead. +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call to the concrete field and value type. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.repos = store.Register(b.registry, "repos", store.New(repoKeyFn)) + }, + func(b *InMemoryBackend) { + imagesT := store.Register(b.registry, "images", store.New(imageKeyFn)) + b.images = imagesT + b.imagesByRepo = imagesT.AddIndex("repo", imageRepoIndexKeyFn) + }, + func(b *InMemoryBackend) { + findingsT := store.Register(b.registry, "imageScanFindings", store.New(findingsKeyFn)) + b.imageScanFindings = findingsT + b.imageScanFindingsByRepo = findingsT.AddIndex("repo", findingsRepoIndexKeyFn) + }, + func(b *InMemoryBackend) { + b.pullThroughCacheRules = store.Register( + b.registry, "pullThroughCacheRules", store.New(pullThroughCacheRuleKeyFn)) + }, + func(b *InMemoryBackend) { + b.repositoryCreationTemplates = store.Register( + b.registry, "repositoryCreationTemplates", store.New(repositoryCreationTemplateKeyFn)) + }, + func(b *InMemoryBackend) { + b.lifecyclePolicies = store.Register( + b.registry, "lifecyclePolicies", store.New(lifecyclePolicyEntryKeyFn)) + }, + func(b *InMemoryBackend) { + b.lifecyclePolicyPreviews = store.Register( + b.registry, "lifecyclePolicyPreviews", store.New(lifecyclePolicyPreviewKeyFn)) + }, + func(b *InMemoryBackend) { + b.repositoryPolicies = store.Register( + b.registry, "repositoryPolicies", store.New(repositoryPolicyEntryKeyFn)) + }, + func(b *InMemoryBackend) { + b.accountSettings = store.Register( + b.registry, "accountSettings", store.New(accountSettingEntryKeyFn)) + }, + func(b *InMemoryBackend) { + b.pullTimeUpdateExclusions = store.Register( + b.registry, "pullTimeUpdateExclusions", store.New(pullTimeUpdateExclusionKeyFn)) + }, +} diff --git a/services/ecs/PARITY.md b/services/ecs/PARITY.md new file mode 100644 index 000000000..97642c65c --- /dev/null +++ b/services/ecs/PARITY.md @@ -0,0 +1,242 @@ +--- +service: ecs +sdk_module: aws-sdk-go-v2/service/ecs@v1.86.2 +last_audit_commit: 86c2f9af +last_audit_date: 2026-07-05 +overall: A +ops: + CreateCluster: {wire: ok, errors: ok, state: ok, persist: ok, note: "added capacityProviders/defaultCapacityProviderStrategy/tags at creation (previously silently dropped); tags echoed on create response"} + DescribeClusters: {wire: ok, errors: ok, state: ok, persist: ok, note: "added include=[TAGS] gating (was previously unsupported; tags were never returned)"} + DeleteCluster: {wire: ok, errors: ok, state: ok, persist: ok, note: "cascade delete of serviceDeployments fixed (was keyed wrong, silently a no-op)"} + ListClusters: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateCluster: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateClusterSettings: {wire: ok, errors: ok, state: ok, persist: ok} + PutClusterCapacityProviders: {wire: ok, errors: partial, state: ok, persist: ok, note: "no existence validation of referenced capacity providers (gap, see gaps list)"} + RegisterTaskDefinition: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeTaskDefinition: {wire: ok, errors: ok, state: ok, persist: ok, note: "include=[TAGS] already supported pre-sweep"} + DeregisterTaskDefinition: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteTaskDefinitions: {wire: ok, errors: ok, state: ok, persist: ok} + ListTaskDefinitions: {wire: ok, errors: ok, state: ok, persist: ok} + ListTaskDefinitionFamilies: {wire: ok, errors: ok, state: ok, persist: ok} + CreateService: {wire: ok, errors: ok, state: ok, persist: ok, note: "now records a real ServiceDeployment for the initial PRIMARY deployment (was a disguised stub, see gaps/fixes)"} + DescribeServices: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateService: {wire: ok, errors: ok, state: ok, persist: ok, note: "now syncs ServiceDeployment records when rotating the PRIMARY deployment"} + DeleteService: {wire: ok, errors: ok, state: ok, persist: ok, note: "now cleans up its ServiceDeployment records (was leaking one entry per deleted service)"} + ListServices: {wire: ok, errors: ok, state: ok, persist: ok} + ListServicesByNamespace: {wire: ok, errors: ok, state: ok, persist: ok} + CreateTaskSet: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteTaskSet: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeTaskSets: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateTaskSet: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateServicePrimaryTaskSet: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeServiceRevisions: {wire: ok, errors: ok, state: ok, persist: ok, note: "derived on read from Service.Deployments, not separately stored — intentional (see Notes)"} + DescribeServiceDeployments: {wire: ok, errors: ok, state: ok, persist: ok, note: "was a disguised stub: filtered a map only the AddServiceDeploymentInternal test seed ever populated. Fixed by syncServiceDeploymentsLocked."} + ListServiceDeployments: {wire: ok, errors: ok, state: ok, persist: ok, note: "same fix as DescribeServiceDeployments"} + StopServiceDeployment: {wire: ok, errors: ok, state: ok, persist: ok, note: "same fix; also now really has data to stop"} + ContinueServiceDeployment: {wire: ok, errors: ok, state: partial, persist: n/a, note: "NEW op (was entirely unimplemented / absent from GetSupportedOperations). Lifecycle hooks (blue/green PAUSE stages) are not modeled, so every call returns an honest ClientException that no paused hook exists, after real ARN/hookId validation — never a fabricated success. See gaps."} + RunTask: {wire: ok, errors: ok, state: ok, persist: ok} + StartTask: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeTasks: {wire: ok, errors: ok, state: ok, persist: ok} + StopTask: {wire: ok, errors: ok, state: ok, persist: ok} + ListTasks: {wire: ok, errors: ok, state: ok, persist: ok} + RegisterContainerInstance: {wire: ok, errors: ok, state: ok, persist: ok} + DeregisterContainerInstance: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeContainerInstances: {wire: ok, errors: ok, state: ok, persist: ok} + ListContainerInstances: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateContainerInstancesState: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateContainerAgent: {wire: ok, errors: ok, state: ok, persist: ok} + CreateCapacityProvider: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteCapacityProvider: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeCapacityProviders: {wire: partial, errors: ok, state: ok, persist: ok, note: "no include=[TAGS] gating, no Cluster filter param, no Failures for unknown names (gap)"} + UpdateCapacityProvider: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteAccountSetting: {wire: ok, errors: ok, state: ok, persist: ok} + ListAccountSettings: {wire: ok, errors: ok, state: ok, persist: ok} + PutAccountSetting: {wire: ok, errors: ok, state: ok, persist: ok} + PutAccountSettingDefault: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + ListAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + PutAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + ExecuteCommand: {wire: ok, errors: ok, state: ok, persist: n/a} + GetTaskProtection: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateTaskProtection: {wire: ok, errors: ok, state: ok, persist: ok} + TagResource: {wire: ok, errors: ok, state: ok, persist: ok, note: "resourceTags side map was NOT in backendSnapshot at all — fixed, see gaps/fixes"} + UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} + ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok} + CreateExpressGatewayService: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteExpressGatewayService: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeExpressGatewayService: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateExpressGatewayService: {wire: ok, errors: ok, state: ok, persist: ok} + DiscoverPollEndpoint: {wire: ok, errors: ok, state: ok, persist: n/a} + SubmitAttachmentStateChanges: {wire: ok, errors: ok, state: ok, persist: ok} + SubmitContainerStateChange: {wire: ok, errors: ok, state: ok, persist: ok} + SubmitTaskStateChange: {wire: ok, errors: ok, state: ok, persist: ok} +families: + daemon: {status: ok, note: "CreateDaemon/DeleteDaemon/.../UpdateDaemon family unchanged this sweep; deferred re-audit, no evidence of regressions"} +gaps: + - "PutClusterCapacityProviders/CreateService/UpdateService/RunTask do not validate that a referenced capacityProviderStrategy name is a real (created or FARGATE/FARGATE_SPOT builtin) capacity provider. AWS rejects unknown providers; this backend accepts any string. Not fixed this sweep to limit blast radius (many call sites, risk of breaking existing tests that use ad-hoc provider names). File follow-up bd issue before next ecs sweep." + - "DescribeCapacityProviders/DescribeContainerInstances/DescribeTaskSets/DescribeExpressGatewayService do not support include=[TAGS] gating (tags are simply never returned by these four; DescribeClusters and DescribeTaskDefinition now/already do). Lower priority: unlike the DescribeClusters gap, no create path lets users set these tags in a way that becomes invisible, since ListTagsForResource still exposes them; it is purely a wire-shape completeness gap." + - "DescribeCapacityProviders additionally lacks the Cluster filter parameter and per-name Failures (unknown names are silently ok, not a discrete SDK gap but a shape simplification)." + - "ContinueServiceDeployment always returns ClientException (no paused lifecycle hook) because PAUSE-stage lifecycle hooks for blue/green deployments are not modeled at all (no hookId tracking, no pause state in the ECS_SERVICE_DEPLOYMENT / EXTERNAL deployment controllers). Implementing real hook pausing is a substantial feature (Lambda-invocation simulation, TEST_TRAFFIC_SHIFT/BAKE_TIME lifecycle stages) out of scope for this sweep; the op is real (validates ARN/hookId, returns AWS-shaped errors) rather than a stub." + - "ECS -> ELB/ELBv2 target registration is config-only: Service.LoadBalancers/ServiceRegistries are stored and echoed back on Describe/Update, but nothing calls services/elbv2 to register/deregister targets in a target group, and ELB health does not feed back into ECS task/service health. Cross-service, lives outside services/ecs/ — reported, not fixed. No bd issue found for this in the tracker at time of writing; recommend filing one scoped to services/elbv2 + services/ecs integration." + - "ECS -> Auto Scaling Group capacity providers are config-only: AutoScalingGroupProvider (ARN, ManagedScaling, ManagedTerminationProtection, ManagedDraining) is stored/echoed but never calls services/autoscaling to validate the ASG exists or to actually scale it in response to managed-scaling target utilization. Cross-service, lives outside services/ecs/ — reported, not fixed." +deferred: + - "Daemon* operation family (CreateDaemon..UpdateDaemon, 12 ops) — not re-audited this pass; backend_daemon.go (787 LOC) untouched, no evidence found of regressions while auditing adjacent code (persistence/tag/service-deployment map ownership)." + - "docker_runner.go / real container lifecycle (vs NoopRunner) — not re-audited this pass." + - "Full ServiceDeployment wire-shape parity (LifecycleStage, SourceServiceRevisions, TargetServiceRevision, Rollback, DeploymentCircuitBreaker, Alarms sub-objects) — the emulator's ServiceDeployment type covers only ServiceDeploymentArn/ClusterArn/ServiceArn/Status/StatusReason/CreatedAt/UpdatedAt. Now correctly populated for every real deployment (this sweep's fix), but the richer blue/green fields are not modeled." +leaks: {status: found, note: "DeleteService leaked one ServiceDeployment map entry per deleted service (unbounded growth under create/delete churn) because nothing ever populated serviceDeployments in production before this sweep, so the leak was latent/inert. Fixed alongside the disguised-stub fix (deleteServiceDeploymentsForServiceLocked, called from DeleteService/DeleteCluster/purgeClusterLocked). Reconciler (per-cluster launch semaphores), janitor (stopped-task TTL sweep), and lifecycle stepper were independently re-verified this sweep and are clean: ctx-cancelled tickers, EvictCluster hook releases semaphores on cluster delete, tasksByInstance/taskProtections/lifecycle entries are cleaned up on both the fast and delayed stop paths." +--- + +## Notes + +Freeform findings from this sweep (gopherstack-7wu), for the next auditor. + +### Severe, fixed this sweep + +1. **`Restore()` never rebuilt `serviceIndex`.** `getServicesForReconciler` + (backend.go) is the *only* feed for the deployment reconciler + (reconciler.go) and reads the flat `serviceIndex` map with **no linear-scan + fallback** (unlike `tasksByInstance`, which `enrichContainerInstance` + explicitly falls back to scanning for — see the comment there). `Restore` + loaded `b.services` from the snapshot but never repopulated + `b.serviceIndex`, so every service that existed at snapshot time became + permanently invisible to the reconciler after a restore/restart: desired-count + convergence, scale up/down, and circuit-breaker evaluation would silently + stop forever for pre-existing services. Fixed in `persistence.go` `Restore` + by rebuilding `serviceIndex` (and, for consistency/performance, + `tasksByInstance`) from the restored maps. Proven by + `Test_Restore_RebuildsServiceIndex` (persistence_internal_test.go), which + fails without the fix (reconciler `RunOnce` after restore launches zero + tasks for a service with `DesiredCount > 0`). + +2. **`resourceTags` side map was entirely absent from `backendSnapshot`.** + Clusters and Services carry `Tags` inline on their own struct, but task + definitions and daemon task definitions are tagged only through the + `TagResource`/`UntagResource`/`ListTagsForResource` side map + (`b.resourceTags`, keyed by `resourceTagKey(arn)`). That map was never + included in `Snapshot()`/`Restore()`, so every tag applied via + `TagResource` on a task definition silently vanished across a + snapshot/restore cycle. Fixed by adding `ResourceTags` to + `backendSnapshot` with proper deep-copy on snapshot and restore. Proven by + `Test_Snapshot_Restore_PreservesResourceTags`. + +3. **`DescribeServiceDeployments`/`ListServiceDeployments`/ + `StopServiceDeployment` were disguised stubs** (parity-principles.md rule + 4: "a real-looking op filtering a never-populated map is a disguised + stub"). `b.serviceDeployments` was only ever written by the + `AddServiceDeploymentInternal` test-seed helper — no real `CreateService`, + `UpdateService`, or circuit-breaker rollback path ever created an entry. + A real client following the documented `CreateService` -> + `ListServiceDeployments` -> `DescribeServiceDeployments` workflow always + got an empty result, even though the service had an active PRIMARY + deployment tracked in `Service.Deployments`. Fixed by + `syncServiceDeploymentsLocked`/`recordServiceDeploymentLocked` + (backend_new_ops.go), called from `CreateService`, `UpdateService`, and + `evaluateCircuitBreakerLocked` (deployment.go, covers both the rollback and + halt-without-rollback branches). `ServiceDeploymentArn` is derived + deterministically from the service ARN + `Deployment.ID` + (`serviceDeploymentArnFor`, mirroring the existing `serviceRevisionArnFor` + pattern for `arn:...:service-revision/...`). Proven end-to-end via HTTP in + `TestECS_ServiceDeployments_RealDeploymentsAreVisible`. + + This uncovered a **second, previously-latent bug**: the cascade-delete + code in `DeleteCluster` and `purgeClusterLocked` did + `delete(b.serviceDeployments, svc.ServiceArn)` — deleting by + `ServiceArn` used as a *map key*, but the map is keyed by + `ServiceDeploymentArn`. This was silently a no-op before (the map was + always empty in practice), but once real entries started flowing in it + would have **leaked one entry per deleted service forever** (also true of + plain `DeleteService`, which never even attempted cleanup). Fixed with a + shared `deleteServiceDeploymentsForServiceLocked(serviceArn)` helper + (matches by the `.ServiceArn` *field*) wired into `DeleteCluster`, + `DeleteService`, and `purgeClusterLocked`. `TestDeleteCluster_ + CascadesServiceDeployments` encoded the old (wrong) key convention — it + injected a fake entry keyed by `svc.ServiceArn` with the `.ServiceArn` + field left blank, which only ever passed because of the bug. Rewritten to + assert against the real auto-created deployment (via `ListServiceDeployments`) + plus a correctly-keyed injected extra entry. New: + `TestECS_ServiceDeployments_DeletedOnServiceDelete`. + +### Moderate, fixed this sweep + +4. **`CreateCluster` silently dropped `capacityProviders`, + `defaultCapacityProviderStrategy`, and `tags`** — all three are real + `CreateClusterInput` fields in aws-sdk-go-v2. Terraform's `aws_ecs_cluster` + resource sets `tags` at creation time (no fallback `TagResource` call), so + this was a real, silent tag-loss bug for the most common IaC flow, not just + a theoretical gap. Fixed: `CreateClusterInput`/`createClusterInput` accept + all three; `CreateCluster` stores capacity-provider fields directly and + tags via a new `setResourceTagsLocked` helper (extracted from `TagResource` + so it can be called while the write lock is already held, avoiding a + self-deadlock on `lockmetrics.RWMutex`). `DescribeClusters` gained + `include=["TAGS"]` support (previously unsupported — tags were never + returned by Describe regardless of the wire shape technically supporting + `tags,omitempty`). CreateCluster's own response always echoes back the + tags it was just given (matches: no `include` gating exists on Create). + Proven by `TestECS_CreateCluster_TagsAndCapacityProviders`. + +5. **`ContinueServiceDeployment` was entirely unimplemented** — absent from + `GetSupportedOperations()`/`buildOps()` and carried an explicit + acknowledged gap in `sdk_completeness_test.go`. Since this backend does not + model blue/green lifecycle-hook pause stages at all, a full implementation + (Lambda-invocation simulation, `TEST_TRAFFIC_SHIFT`/`BAKE_TIME` stages) was + out of scope; instead the op is now real-but-honest: validates + `serviceDeploymentArn` and `hookId` are present, looks up the deployment + (404 `ServiceDeploymentNotFoundException` if missing), validates `action` + is `CONTINUE`/`ROLLBACK`/omitted, and returns `ClientException` reporting + that no such lifecycle hook is currently paused — never a fabricated + success. Removed from the `sdk_completeness_test.go` acknowledged-gap list + since it is now routed. Proven by `TestECS_ContinueServiceDeployment` + (three cases: real deployment/no hook, deployment not found, missing + hookId). + +### Verified accurate / traps for the next auditor + +- `enrichContainerInstance`'s `tasksByInstance` fallback-to-linear-scan + comment ("e.g. after restore") shows a prior sweep already reasoned about + the post-restore-index-empty case for that specific map — but the parallel + `serviceIndex` consumer (`getServicesForReconciler`) had no such fallback + and no comment acknowledging it. Don't assume one documented index-rebuild + concern means all of them were considered. +- `enrichService`'s `RolloutState -> COMPLETED` transition is computed + transiently on every `DescribeServices`/`enrichService` call under an + RLock and is **not** written back into the stored `Service.Deployments` — + this is intentional/pre-existing (matches the `DescribeServiceRevisions` + "derive on read" pattern, see `addServiceRevisionLocked`'s doc comment). Do + not flag this as a persistence bug; it's a deliberate simplification. Note: + the new `ServiceDeployment.Status` this sweep added inherits the same + limitation — it snapshots `RolloutState` at deployment-creation/rollback + time and will not itself flip to `SUCCESSFUL` once the deployment + converges, since nothing re-invokes `syncServiceDeploymentsLocked` on the + read path. Consistent with existing precedent, not a regression, but a + known simplification worth closing in a future sweep if `DescribeServiceDeployments` + status accuracy becomes load-bearing for a test. +- ECS deployment-circuit-breaker threshold math + (`circuitBreakerThreshold`/`deployment.go`) matches AWS's documented + floor-3/ceiling-200/half-of-desired-count formula exactly — verified + against https://docs.aws.amazon.com/AmazonECS/latest/developerguide/deployment-circuit-breaker.html, + no changes needed. +- `builtinCapacityProvider` correctly synthesizes FARGATE/FARGATE_SPOT + without requiring explicit `CreateCapacityProvider` calls, matching AWS + (these are AWS-managed, always-available providers) — verified, no gap. +- Reconciler/janitor/lifecycle-stepper goroutine and ticker hygiene was + re-verified this sweep (ctx-cancelled loops, `EvictCluster` cluster-delete + hook releasing per-cluster semaphores, task-protection/lifecycle/ + tasksByInstance cleanup on both the fast and stop-delay paths) — all clean, + no changes needed. + +### Cross-service / out-of-scope (reported, not fixed — services/ecs/ only) + +- ECS -> ELB/ELBv2 target registration: config-only (`Service.LoadBalancers` + stored/echoed, never registers/deregisters targets in `services/elbv2`). + Matches the task brief's "known gap" — confirmed still present, no bd issue + found referencing it in the tracker at audit time. +- ECS -> Auto Scaling Group capacity providers: config-only + (`AutoScalingGroupProvider` stored/echoed, never calls `services/autoscaling` + to validate the ASG or drive managed scaling). +- Repo-wide (unrelated to ecs): at `last_audit_commit` the root package fails + `go build ./...` with two `cwBk.PutMetricData` call-site mismatches in + `cli.go` (lines 3220, 4259) — a pre-existing break from a concurrent + CloudWatch-service sweep on this shared branch, confirmed unrelated to any + ecs change (`cli.go` has zero diff in this sweep) and out of scope + (services/ecs/ only, shared file). `go build ./services/ecs/...` and + `go build ./...` excluding the root package both pass clean. diff --git a/services/ecs/backend.go b/services/ecs/backend.go index 646b3fe89..1857d732c 100644 --- a/services/ecs/backend.go +++ b/services/ecs/backend.go @@ -13,6 +13,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awserr" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) const ( @@ -203,8 +204,11 @@ type Task struct { // CreateClusterInput holds input for CreateCluster. type CreateClusterInput struct { - ClusterName string - Settings []ClusterSetting + ClusterName string + Settings []ClusterSetting + CapacityProviders []string + DefaultCapacityProviderStrategy []CapacityProviderStrategyItem + Tags []Tag } // RegisterTaskDefinitionInput holds input for RegisterTaskDefinition. @@ -300,29 +304,40 @@ type svcRef struct { // Field ordering is optimised for pointer-byte alignment (govet fieldalignment); // keep new fields grouped with their kind rather than by logical concern. type InMemoryBackend struct { - runner TaskRunner - serviceDeployments map[string]*ServiceDeployment - taskSets map[string]map[string]*TaskSet - taskDefByArn map[string]*TaskDefinition - services map[string]map[string]*Service - tasks map[string]map[string]*Task - containerInstances map[string]map[string]*ContainerInstance - clusters map[string]*Cluster - taskProtections map[string]*TaskProtection - capacityProviders map[string]*CapacityProvider - accountSettings map[string]*AccountSetting - taskDefinitions map[string][]*TaskDefinition - attributes map[string]map[string]*Attribute - mu *lockmetrics.RWMutex - resourceTags map[string][]Tag - tasksByInstance map[string]map[string]map[string]bool - serviceIndex map[svcRef]bool - expressGatewayServices map[string]*ExpressGatewayService - daemonRevisions map[string]*DaemonRevision - daemonDeployments map[string]*DaemonDeployment - daemons map[string]map[string]*Daemon // clusterName → daemonName → Daemon - daemonTaskDefinitions map[string][]*DaemonTaskDefinition - daemonTaskDefByArn map[string]*DaemonTaskDefinition + runner TaskRunner + // registry is the Phase 3.3 datalayer lifecycle registry: every *store.Table + // below (except taskDefByArn/daemonTaskDefByArn, which are derived caches -- + // see store_setup.go) is registered on it exactly once at construction, so + // Reset/Snapshot/Restore collapse to one registry call each instead of one + // hand-written block per map. See pkgs/store's package doc. + registry *store.Registry + serviceDeployments *store.Table[ServiceDeployment] + taskSets *store.Table[TaskSet] + taskSetsByService *store.Index[TaskSet] + taskDefByArn *store.Table[TaskDefinition] + services *store.Table[Service] + servicesByCluster *store.Index[Service] + tasks *store.Table[Task] + tasksByCluster *store.Index[Task] + containerInstances *store.Table[ContainerInstance] + containerInstancesByCluster *store.Index[ContainerInstance] + clusters *store.Table[Cluster] + taskProtections *store.Table[TaskProtection] + capacityProviders *store.Table[CapacityProvider] + accountSettings *store.Table[AccountSetting] + taskDefinitions map[string][]*TaskDefinition + attributes map[string]map[string]*Attribute + mu *lockmetrics.RWMutex + resourceTags map[string][]Tag + tasksByInstance map[string]map[string]map[string]bool + serviceIndex map[svcRef]bool + expressGatewayServices *store.Table[ExpressGatewayService] + daemonRevisions *store.Table[DaemonRevision] + daemonDeployments *store.Table[DaemonDeployment] + daemons *store.Table[Daemon] // composite "clusterName/daemonName" key; see daemonsByCluster + daemonsByCluster *store.Index[Daemon] + daemonTaskDefinitions map[string][]*DaemonTaskDefinition + daemonTaskDefByArn *store.Table[DaemonTaskDefinition] // daemonTaskDefs exists only for structural compatibility with purge.go's // per-cluster daemon cleanup (purgeDaemonsLocked). Real daemon task // definitions are registered independently by family, like ordinary task @@ -369,37 +384,27 @@ type TaskRunner interface { // NewInMemoryBackend creates a new InMemoryBackend. func NewInMemoryBackend(accountID, region string, runner TaskRunner) *InMemoryBackend { - return &InMemoryBackend{ - clusters: make(map[string]*Cluster), - taskDefinitions: make(map[string][]*TaskDefinition), - taskDefByArn: make(map[string]*TaskDefinition), - services: make(map[string]map[string]*Service), - tasks: make(map[string]map[string]*Task), - containerInstances: make(map[string]map[string]*ContainerInstance), - taskSets: make(map[string]map[string]*TaskSet), - taskProtections: make(map[string]*TaskProtection), - capacityProviders: make(map[string]*CapacityProvider), - accountSettings: make(map[string]*AccountSetting), - attributes: make(map[string]map[string]*Attribute), - serviceDeployments: make(map[string]*ServiceDeployment), - expressGatewayServices: make(map[string]*ExpressGatewayService), - resourceTags: make(map[string][]Tag), - tasksByInstance: make(map[string]map[string]map[string]bool), - serviceIndex: make(map[svcRef]bool), - lifecycle: make(map[string]*taskLifecycle), - mu: lockmetrics.New("ecs"), - accountID: accountID, - region: region, - runner: runner, - daemons: make(map[string]map[string]*Daemon), - daemonTaskDefinitions: make(map[string][]*DaemonTaskDefinition), - daemonTaskDefByArn: make(map[string]*DaemonTaskDefinition), - daemonDeployments: make(map[string]*DaemonDeployment), - daemonRevisions: make(map[string]*DaemonRevision), - daemonTaskDefs: make(map[string][]*DaemonTaskDefinition), - serviceRevisions: make(map[string][]*ServiceRevision), - serviceRevisionsByArn: make(map[string]*ServiceRevision), + b := &InMemoryBackend{ + taskDefinitions: make(map[string][]*TaskDefinition), + attributes: make(map[string]map[string]*Attribute), + resourceTags: make(map[string][]Tag), + tasksByInstance: make(map[string]map[string]map[string]bool), + serviceIndex: make(map[svcRef]bool), + lifecycle: make(map[string]*taskLifecycle), + mu: lockmetrics.New("ecs"), + accountID: accountID, + region: region, + runner: runner, + daemonTaskDefinitions: make(map[string][]*DaemonTaskDefinition), + daemonTaskDefs: make(map[string][]*DaemonTaskDefinition), + serviceRevisions: make(map[string][]*ServiceRevision), + serviceRevisionsByArn: make(map[string]*ServiceRevision), + registry: store.NewRegistry(), } + + registerAllTables(b) + + return b } // Reset zeroes all backend state for test isolation. @@ -407,27 +412,16 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.clusters = make(map[string]*Cluster) + b.registry.ResetAll() + b.taskDefByArn.Reset() + b.daemonTaskDefByArn.Reset() + b.taskDefinitions = make(map[string][]*TaskDefinition) - b.taskDefByArn = make(map[string]*TaskDefinition) - b.services = make(map[string]map[string]*Service) - b.tasks = make(map[string]map[string]*Task) - b.containerInstances = make(map[string]map[string]*ContainerInstance) - b.taskSets = make(map[string]map[string]*TaskSet) - b.taskProtections = make(map[string]*TaskProtection) - b.capacityProviders = make(map[string]*CapacityProvider) - b.accountSettings = make(map[string]*AccountSetting) b.attributes = make(map[string]map[string]*Attribute) - b.serviceDeployments = make(map[string]*ServiceDeployment) - b.expressGatewayServices = make(map[string]*ExpressGatewayService) b.resourceTags = make(map[string][]Tag) b.tasksByInstance = make(map[string]map[string]map[string]bool) b.serviceIndex = make(map[svcRef]bool) - b.daemons = make(map[string]map[string]*Daemon) b.daemonTaskDefinitions = make(map[string][]*DaemonTaskDefinition) - b.daemonTaskDefByArn = make(map[string]*DaemonTaskDefinition) - b.daemonDeployments = make(map[string]*DaemonDeployment) - b.daemonRevisions = make(map[string]*DaemonRevision) b.daemonTaskDefs = make(map[string][]*DaemonTaskDefinition) b.serviceRevisions = make(map[string][]*ServiceRevision) b.serviceRevisionsByArn = make(map[string]*ServiceRevision) @@ -508,21 +502,24 @@ func (b *InMemoryBackend) CreateCluster(input CreateClusterInput) (*Cluster, err b.mu.Lock("CreateCluster") defer b.mu.Unlock() - if _, ok := b.clusters[name]; ok { + if b.clusters.Has(name) { return nil, fmt.Errorf("%w: %s", ErrClusterAlreadyExists, name) } cluster := &Cluster{ - CreatedAt: time.Now(), - ClusterArn: arn.Build("ecs", b.region, b.accountID, fmt.Sprintf("cluster/%s", name)), - ClusterName: name, - Status: statusActive, - Settings: input.Settings, + CreatedAt: time.Now(), + ClusterArn: arn.Build("ecs", b.region, b.accountID, fmt.Sprintf("cluster/%s", name)), + ClusterName: name, + Status: statusActive, + Settings: input.Settings, + CapacityProviders: input.CapacityProviders, + DefaultCapacityProviderStrategy: input.DefaultCapacityProviderStrategy, + } + b.clusters.Put(cluster) + + if len(input.Tags) > 0 { + b.setResourceTagsLocked(cluster.ClusterArn, input.Tags) } - b.clusters[name] = cluster - b.services[name] = make(map[string]*Service) - b.tasks[name] = make(map[string]*Task) - b.containerInstances[name] = make(map[string]*ContainerInstance) cp := *cluster @@ -543,8 +540,9 @@ func (b *InMemoryBackend) DescribeClusters(clusterNames []string) ([]Cluster, [] defer b.mu.RUnlock() if len(clusterNames) == 0 { - out := make([]Cluster, 0, len(b.clusters)) - for _, c := range b.clusters { + all := b.clusters.All() + out := make([]Cluster, 0, len(all)) + for _, c := range all { out = append(out, b.enrichCluster(c)) } @@ -557,7 +555,7 @@ func (b *InMemoryBackend) DescribeClusters(clusterNames []string) ([]Cluster, [] for _, name := range clusterNames { key := clusterKey(name) - c, ok := b.clusters[key] + c, ok := b.clusters.Get(key) if !ok { failures = append(failures, Failure{ Arn: name, @@ -581,8 +579,8 @@ func (b *InMemoryBackend) DescribeClusters(clusterNames []string) ([]Cluster, [] func (b *InMemoryBackend) enrichCluster(c *Cluster) Cluster { cp := *c - cp.ActiveServicesCount = len(b.services[c.ClusterName]) - cp.RegisteredContainerInstancesCount = len(b.containerInstances[c.ClusterName]) + cp.ActiveServicesCount = len(b.servicesByCluster.Get(c.ClusterName)) + cp.RegisteredContainerInstancesCount = len(b.containerInstancesByCluster.Get(c.ClusterName)) // RunningTasksCount and PendingTasksCount are maintained as cached counters // on the Cluster struct. No task iteration needed here. @@ -596,7 +594,7 @@ func (b *InMemoryBackend) DeleteCluster(clusterName string) (*Cluster, error) { b.mu.Lock("DeleteCluster") - c, ok := b.clusters[key] + c, ok := b.clusters.Get(key) if !ok { b.mu.Unlock() @@ -606,34 +604,31 @@ func (b *InMemoryBackend) DeleteCluster(clusterName string) (*Cluster, error) { // Snapshot task pointers while still holding the lock so we can stop their // Docker containers after releasing it. Performing Docker API calls under // the backend lock would unnecessarily serialize all other operations. - tasksToStop := make([]*Task, 0, len(b.tasks[key])) + clusterTasks := b.tasksInClusterLocked(key) + tasksToStop := make([]*Task, 0, len(clusterTasks)) if b.runner != nil { - for _, task := range b.tasks[key] { - tasksToStop = append(tasksToStop, task) - } + tasksToStop = append(tasksToStop, clusterTasks...) } // Delete task sets and service deployments for all services in this cluster // before removing the services map, preventing stale entries on cluster recreation. - if svcs, exists := b.services[key]; exists { - for svcName, svc := range svcs { - delete(b.taskSets, svc.ServiceArn) - delete(b.serviceDeployments, svc.ServiceArn) - delete(b.serviceIndex, svcRef{cluster: key, name: svcName}) - } + for _, svc := range b.servicesInClusterLocked(key) { + b.deleteTaskSetsForServiceLocked(svc.ServiceArn) + b.deleteServiceDeploymentsForServiceLocked(svc.ServiceArn) + delete(b.serviceIndex, svcRef{cluster: key, name: svc.ServiceName}) } // Clean up per-task state for all tasks in this cluster to avoid memory leaks. - for taskArn := range b.tasks[key] { - delete(b.taskProtections, taskArn) - delete(b.lifecycle, taskArn) + for _, task := range clusterTasks { + b.taskProtections.Delete(task.TaskArn) + delete(b.lifecycle, task.TaskArn) } - delete(b.clusters, key) - delete(b.services, key) - delete(b.tasks, key) - delete(b.containerInstances, key) + b.clusters.Delete(key) + b.deleteServicesForClusterLocked(key) + b.deleteTasksForClusterLocked(key) + b.deleteContainerInstancesForClusterLocked(key) delete(b.attributes, key) delete(b.tasksByInstance, key) @@ -728,14 +723,14 @@ func (b *InMemoryBackend) RegisterTaskDefinition( excess := len(revisions) - maxTaskDefinitionRevisions for _, evicted := range revisions[:excess] { - delete(b.taskDefByArn, evicted.TaskDefinitionArn) + b.taskDefByArn.Delete(evicted.TaskDefinitionArn) } revisions = revisions[excess:] } b.taskDefinitions[input.Family] = revisions - b.taskDefByArn[td.TaskDefinitionArn] = td + b.taskDefByArn.Put(td) // Persist registration tags via the same resourceTags map used by // TagResource so that DescribeTaskDefinition can surface them when @@ -799,7 +794,7 @@ func (b *InMemoryBackend) findTaskDefinitionLocked(familyOrArn string) (*TaskDef } // Fast path: ARN cache lookup. - if td, ok := b.taskDefByArn[familyOrArn]; ok { + if td, ok := b.taskDefByArn.Get(familyOrArn); ok { cp := *td return &cp, nil @@ -898,8 +893,8 @@ func (b *InMemoryBackend) ListTaskDefinitionsFiltered( // ensureClusterLocked returns the cluster maps, auto-creating the default cluster if needed. // Must be called with write lock held. func (b *InMemoryBackend) ensureClusterLocked(clusterName string) { - if _, ok := b.clusters[clusterName]; !ok && clusterName == defaultCluster { - b.clusters[clusterName] = &Cluster{ + if !b.clusters.Has(clusterName) && clusterName == defaultCluster { + b.clusters.Put(&Cluster{ CreatedAt: time.Now(), ClusterArn: arn.Build( "ecs", @@ -909,10 +904,7 @@ func (b *InMemoryBackend) ensureClusterLocked(clusterName string) { ), ClusterName: clusterName, Status: statusActive, - } - b.services[clusterName] = make(map[string]*Service) - b.tasks[clusterName] = make(map[string]*Task) - b.containerInstances[clusterName] = make(map[string]*ContainerInstance) + }) } } @@ -937,7 +929,7 @@ func (b *InMemoryBackend) CreateService(input CreateServiceInput) (*Service, err b.ensureClusterLocked(clusterName) - if _, ok := b.services[clusterName][input.ServiceName]; ok { + if b.services.Has(scopedKey(clusterName, input.ServiceName)) { return nil, fmt.Errorf("%w: %s", ErrServiceAlreadyExists, input.ServiceName) } @@ -998,10 +990,11 @@ func (b *InMemoryBackend) CreateService(input CreateServiceInput) (*Service, err svc.Deployments = []Deployment{newPrimaryDeployment(svc)} - b.services[clusterName][input.ServiceName] = svc + b.services.Put(svc) b.serviceIndex[svcRef{cluster: clusterName, name: input.ServiceName}] = true b.addServiceRevisionLocked(svc) + b.syncServiceDeploymentsLocked(svc) cp := *svc @@ -1019,12 +1012,12 @@ func (b *InMemoryBackend) DescribeServices( b.mu.RLock("DescribeServices") defer b.mu.RUnlock() - svcs, ok := b.services[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } if len(serviceNames) == 0 { + svcs := b.servicesByCluster.Get(clusterName) out := make([]Service, 0, len(svcs)) for _, s := range svcs { out = append(out, b.enrichService(s, clusterName)) @@ -1040,7 +1033,7 @@ func (b *InMemoryBackend) DescribeServices( // Support ARN lookup by extracting the service name. key := serviceKey(name) - s, found := svcs[key] + s, found := b.services.Get(scopedKey(clusterName, key)) if !found { failures = append(failures, Failure{ Arn: name, @@ -1066,15 +1059,13 @@ func (b *InMemoryBackend) DescribeServiceRevisions(arns []string) ([]ServiceRevi byArn := make(map[string]ServiceRevision) - for _, svcs := range b.services { - for _, svc := range svcs { - for _, d := range svc.Deployments { - if d.ServiceRevisionArn == "" { - continue - } - - byArn[d.ServiceRevisionArn] = buildServiceRevision(svc, d) + for _, svc := range b.services.All() { + for _, d := range svc.Deployments { + if d.ServiceRevisionArn == "" { + continue } + + byArn[d.ServiceRevisionArn] = buildServiceRevision(svc, d) } } @@ -1137,7 +1128,7 @@ func (b *InMemoryBackend) enrichService(s *Service, clusterName string) Service deplRunning := make(map[string]int, len(s.Deployments)) deplPending := make(map[string]int, len(s.Deployments)) - for _, t := range b.tasks[clusterName] { + for _, t := range b.tasksByCluster.Get(clusterName) { if t.Group == "service:"+s.ServiceName { switch t.LastStatus { case statusRunning: @@ -1226,12 +1217,11 @@ func (b *InMemoryBackend) UpdateService(input UpdateServiceInput) (*Service, err b.mu.Lock("UpdateService") defer b.mu.Unlock() - svcs, ok := b.services[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, input.Cluster) } - svc, ok := svcs[serviceKey] + svc, ok := b.services.Get(scopedKey(clusterName, serviceKey)) if !ok { return nil, fmt.Errorf("%w: %s", ErrServiceNotFound, input.Service) } @@ -1258,6 +1248,7 @@ func (b *InMemoryBackend) UpdateService(input UpdateServiceInput) (*Service, err applyServiceConfigUpdates(svc, input) b.addServiceRevisionLocked(svc) + b.syncServiceDeploymentsLocked(svc) cp := *svc @@ -1290,19 +1281,19 @@ func (b *InMemoryBackend) DeleteService(cluster, serviceName string) (*Service, b.mu.Lock("DeleteService") defer b.mu.Unlock() - svcs, ok := b.services[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } - svc, ok := svcs[key] + svc, ok := b.services.Get(scopedKey(clusterName, key)) if !ok { return nil, fmt.Errorf("%w: %s", ErrServiceNotFound, serviceName) } - delete(svcs, key) - delete(b.taskSets, svc.ServiceArn) + b.services.Delete(scopedKey(clusterName, key)) + b.deleteTaskSetsForServiceLocked(svc.ServiceArn) delete(b.serviceIndex, svcRef{cluster: clusterName, name: key}) + b.deleteServiceDeploymentsForServiceLocked(svc.ServiceArn) cp := *svc @@ -1463,7 +1454,7 @@ func (b *InMemoryBackend) applyNoRunnerTransition(task *Task, clusterName string task.LastStatus = statusRunning syncContainerStatuses(task, nil) - if c := b.clusters[clusterName]; c != nil { + if c, _ := b.clusters.Get(clusterName); c != nil { c.PendingTasksCount-- c.RunningTasksCount++ } @@ -1486,7 +1477,7 @@ func (b *InMemoryBackend) applyRunnerTransition(task *Task, clusterName string, task.LastStatus = statusRunning syncContainerStatuses(task, nil) - if c := b.clusters[clusterName]; c != nil { + if c, _ := b.clusters.Get(clusterName); c != nil { c.PendingTasksCount-- c.RunningTasksCount++ } @@ -1504,7 +1495,7 @@ func (b *InMemoryBackend) applyRunnerTransition(task *Task, clusterName string, exitCode := 1 syncContainerStatuses(task, &exitCode) - if c := b.clusters[clusterName]; c != nil { + if c, _ := b.clusters.Get(clusterName); c != nil { c.PendingTasksCount-- } @@ -1568,8 +1559,8 @@ func (b *InMemoryBackend) createTaskEntriesLocked( // Merge task-definition constraints with any run-time override constraints. constraints := mergeConstraints(td.PlacementConstraints, input.PlacementConstraints) if instanceArn := selectContainerInstance( - b.containerInstances[clusterName], - b.tasks[clusterName], + b.containerInstancesByCluster.Get(clusterName), + b.tasksByCluster.Get(clusterName), constraints, input.PlacementStrategy, input.serviceNameForTags, @@ -1581,11 +1572,11 @@ func (b *InMemoryBackend) createTaskEntriesLocked( task.Containers = buildContainersForTask(task, td) - b.tasks[clusterName][taskArn] = task + b.tasks.Put(task) work = append(work, taskWork{task: task, td: td}) // Increment the cached pending counter on the cluster. - if c := b.clusters[clusterName]; c != nil { + if c, _ := b.clusters.Get(clusterName); c != nil { c.PendingTasksCount++ } } @@ -1604,12 +1595,12 @@ func (b *InMemoryBackend) DescribeTasks( b.mu.RLock("DescribeTasks") defer b.mu.RUnlock() - clusterTasks, ok := b.tasks[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } if len(taskArns) == 0 { + clusterTasks := b.tasksByCluster.Get(clusterName) out := make([]Task, 0, len(clusterTasks)) for _, t := range clusterTasks { out = append(out, *t) @@ -1622,8 +1613,8 @@ func (b *InMemoryBackend) DescribeTasks( failures := make([]Failure, 0, len(taskArns)) for _, arn := range taskArns { - t, found := clusterTasks[arn] - if !found { + t, found := b.tasks.Get(arn) + if !found || clusterKey(t.ClusterArn) != clusterName { failures = append(failures, Failure{ Arn: arn, Reason: statusMissing, @@ -1645,15 +1636,14 @@ func (b *InMemoryBackend) StopTask(cluster, taskArn, reason string) (*Task, erro b.mu.Lock("StopTask") - clusterTasks, ok := b.tasks[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { b.mu.Unlock() return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } - task, ok := clusterTasks[taskArn] - if !ok { + task, ok := b.tasks.Get(taskArn) + if !ok || clusterKey(task.ClusterArn) != clusterName { b.mu.Unlock() return nil, fmt.Errorf("%w: %s", ErrTaskNotFound, taskArn) @@ -1667,7 +1657,7 @@ func (b *InMemoryBackend) StopTask(cluster, taskArn, reason string) (*Task, erro // Decrement the cached cluster counters once, as the task leaves its active // state. This is done up front for both the fast and delayed paths so the // counters stay correct regardless of when the task finally reaches STOPPED. - if c := b.clusters[clusterName]; c != nil { + if c, _ := b.clusters.Get(clusterName); c != nil { switch prevStatus { case statusRunning: c.RunningTasksCount-- @@ -1714,7 +1704,7 @@ func (b *InMemoryBackend) StopTask(cluster, taskArn, reason string) (*Task, erro // Clean up task protection entry and reverse index to avoid stale entries. b.mu.Lock("StopTask-cleanup") - delete(b.taskProtections, taskArn) + b.taskProtections.Delete(taskArn) b.unindexTaskFromInstance(clusterName, instanceArn, taskArn) b.mu.Unlock() @@ -1754,13 +1744,13 @@ func (b *InMemoryBackend) ListTasksFiltered(input ListTasksInput) ([]string, err b.mu.RLock("ListTasksFiltered") defer b.mu.RUnlock() - clusterTasks, ok := b.tasks[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, input.Cluster) } + clusterTasks := b.tasksByCluster.Get(clusterName) arns := make([]string, 0, len(clusterTasks)) - for arn, task := range clusterTasks { + for _, task := range clusterTasks { if input.ContainerInstance != "" && task.ContainerInstanceArn != input.ContainerInstance { continue } @@ -1780,7 +1770,7 @@ func (b *InMemoryBackend) ListTasksFiltered(input ListTasksInput) ([]string, err if input.ServiceName != "" && task.Group != "service:"+input.ServiceName { continue } - arns = append(arns, arn) + arns = append(arns, task.TaskArn) } return arns, nil @@ -1796,7 +1786,7 @@ func (b *InMemoryBackend) getServicesForReconciler() []serviceSnapshot { out := make([]serviceSnapshot, 0, len(b.serviceIndex)) for ref := range b.serviceIndex { - svc := b.services[ref.cluster][ref.name] + svc, _ := b.services.Get(scopedKey(ref.cluster, ref.name)) out = append(out, serviceSnapshot{ clusterName: ref.cluster, service: *svc, @@ -1820,7 +1810,7 @@ func (b *InMemoryBackend) CountRunningTasksForService(clusterName, serviceName s count := 0 group := "service:" + serviceName - for _, t := range b.tasks[clusterName] { + for _, t := range b.tasksByCluster.Get(clusterName) { if t.Group == group && t.LastStatus == statusRunning { count++ } @@ -1846,15 +1836,13 @@ func (b *InMemoryBackend) StartTaskForService( var svcPlacementConstraints []PlacementConstraint var svcPlacementStrategy []PlacementStrategy - if svcs, ok := b.services[clusterName]; ok { - if svc, found := svcs[serviceName]; found { - svcPropagateTags = svc.PropagateTags - svcTags = copyTags(svc.Tags) - svcEnableExec = svc.EnableExecuteCommand - svcLaunchType = svc.LaunchType - svcPlacementConstraints = svc.PlacementConstraints - svcPlacementStrategy = svc.PlacementStrategy - } + if svc, found := b.services.Get(scopedKey(clusterName, serviceName)); found { + svcPropagateTags = svc.PropagateTags + svcTags = copyTags(svc.Tags) + svcEnableExec = svc.EnableExecuteCommand + svcLaunchType = svc.LaunchType + svcPlacementConstraints = svc.PlacementConstraints + svcPlacementStrategy = svc.PlacementStrategy } b.mu.RUnlock() @@ -1884,7 +1872,7 @@ func (b *InMemoryBackend) StopOldestServiceTask(clusterName, serviceName string) var oldest *Task - for _, t := range b.tasks[clusterName] { + for _, t := range b.tasksByCluster.Get(clusterName) { if t.Group == group && t.LastStatus == statusRunning { if oldest == nil || (t.StartedAt != nil && oldest.StartedAt != nil && t.StartedAt.Before(*oldest.StartedAt)) { @@ -1907,7 +1895,7 @@ func (b *InMemoryBackend) StopOldestServiceTask(clusterName, serviceName string) syncContainerStatuses(oldest, nil) // Decrement the cached running counter (scale-in always stops a running task). - if c := b.clusters[clusterName]; c != nil { + if c, _ := b.clusters.Get(clusterName); c != nil { c.RunningTasksCount-- } diff --git a/services/ecs/backend_agent_ops.go b/services/ecs/backend_agent_ops.go index 846139a5a..db3cb262c 100644 --- a/services/ecs/backend_agent_ops.go +++ b/services/ecs/backend_agent_ops.go @@ -56,13 +56,15 @@ type SubmitContainerStateChangeInput struct { // resolveTaskInClusterLocked finds a task within clusterTasks by full ARN or by // bare task ID (the trailing path segment of the ARN). Callers must hold b.mu. -func resolveTaskInClusterLocked(clusterTasks map[string]*Task, ref string) *Task { +func resolveTaskInClusterLocked(clusterTasks []*Task, ref string) *Task { if ref == "" { return nil } - if t, ok := clusterTasks[ref]; ok { - return t + for _, t := range clusterTasks { + if t.TaskArn == ref { + return t + } } if strings.HasPrefix(ref, "arn:") { @@ -70,8 +72,8 @@ func resolveTaskInClusterLocked(clusterTasks map[string]*Task, ref string) *Task } suffix := "/" + ref - for arn, t := range clusterTasks { - if strings.HasSuffix(arn, suffix) { + for _, t := range clusterTasks { + if strings.HasSuffix(t.TaskArn, suffix) { return t } } @@ -142,12 +144,11 @@ func (b *InMemoryBackend) SubmitTaskStateChange(input SubmitTaskStateChangeInput b.mu.Lock("SubmitTaskStateChange") defer b.mu.Unlock() - clusterTasks, ok := b.tasks[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil } - task := resolveTaskInClusterLocked(clusterTasks, input.Task) + task := resolveTaskInClusterLocked(b.tasksByCluster.Get(clusterName), input.Task) if task == nil { return nil } @@ -180,12 +181,11 @@ func (b *InMemoryBackend) SubmitContainerStateChange(input SubmitContainerStateC b.mu.Lock("SubmitContainerStateChange") defer b.mu.Unlock() - clusterTasks, ok := b.tasks[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil } - task := resolveTaskInClusterLocked(clusterTasks, input.Task) + task := resolveTaskInClusterLocked(b.tasksByCluster.Get(clusterName), input.Task) if task == nil { return nil } @@ -220,12 +220,11 @@ func (b *InMemoryBackend) SubmitAttachmentStateChanges(cluster string, attachmen b.mu.Lock("SubmitAttachmentStateChanges") defer b.mu.Unlock() - clusterTasks, ok := b.tasks[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil } - for _, task := range clusterTasks { + for _, task := range b.tasksByCluster.Get(clusterName) { applyAttachmentStateChangesLocked(task, attachments) } diff --git a/services/ecs/backend_daemon.go b/services/ecs/backend_daemon.go index a4123814f..83293d8f7 100644 --- a/services/ecs/backend_daemon.go +++ b/services/ecs/backend_daemon.go @@ -252,7 +252,7 @@ func parseDaemonArn(daemonArn string) (string, string, bool) { func (b *InMemoryBackend) findDaemonLocked(daemonArn string) (*Daemon, error) { clusterName, daemonName, ok := parseDaemonArn(daemonArn) if ok { - if d, exists := b.daemons[clusterName][daemonName]; exists { + if d, exists := b.daemons.Get(scopedKey(clusterName, daemonName)); exists { return d, nil } } @@ -274,7 +274,7 @@ func (b *InMemoryBackend) createDaemonRevisionLocked(clusterName string, d *Daem EnableExecuteCommand: d.EnableExecuteCommand, } - b.daemonRevisions[rev.DaemonRevisionArn] = rev + b.daemonRevisions.Put(rev) return rev } @@ -300,7 +300,7 @@ func (b *InMemoryBackend) createDaemonDeploymentLocked( DeploymentConfiguration: d.DeploymentConfiguration, } - b.daemonDeployments[dep.DaemonDeploymentArn] = dep + b.daemonDeployments.Put(dep) return dep } @@ -329,7 +329,7 @@ func (b *InMemoryBackend) CreateDaemon(input CreateDaemonInput) (*Daemon, error) } daemonArn := b.daemonARN(clusterName, input.DaemonName) - if _, ok := b.daemons[clusterName][input.DaemonName]; ok { + if b.daemons.Has(scopedKey(clusterName, input.DaemonName)) { return nil, fmt.Errorf("%w: %s", ErrDaemonAlreadyExists, input.DaemonName) } @@ -356,11 +356,7 @@ func (b *InMemoryBackend) CreateDaemon(input CreateDaemonInput) (*Daemon, error) d.DeploymentArn = dep.DaemonDeploymentArn d.CurrentDaemonRevisionArn = rev.DaemonRevisionArn - if b.daemons[clusterName] == nil { - b.daemons[clusterName] = make(map[string]*Daemon) - } - - b.daemons[clusterName][input.DaemonName] = d + b.daemons.Put(d) out := *d out.Tags = copyTags(d.Tags) @@ -385,11 +381,7 @@ func (b *InMemoryBackend) DeleteDaemon(daemonArn string) (*Daemon, error) { clusterName, daemonName, ok := parseDaemonArn(daemonArn) if ok { - delete(b.daemons[clusterName], daemonName) - - if len(b.daemons[clusterName]) == 0 { - delete(b.daemons, clusterName) - } + b.daemons.Delete(scopedKey(clusterName, daemonName)) } return &out, nil @@ -480,20 +472,19 @@ func (b *InMemoryBackend) ListDaemons(input ListDaemonsInput) ([]Daemon, error) wantCP[cp] = true } - out := make([]Daemon, 0, len(b.daemons)) - - for _, clusterDaemons := range b.daemons { - for _, d := range clusterDaemons { - if wantCluster != "" && d.ClusterArn != wantCluster { - continue - } + all := b.daemons.All() + out := make([]Daemon, 0, len(all)) - if len(wantCP) > 0 && !daemonHasAnyCapacityProvider(d, wantCP) { - continue - } + for _, d := range all { + if wantCluster != "" && d.ClusterArn != wantCluster { + continue + } - out = append(out, *d) + if len(wantCP) > 0 && !daemonHasAnyCapacityProvider(d, wantCP) { + continue } + + out = append(out, *d) } return out, nil @@ -525,7 +516,7 @@ func (b *InMemoryBackend) findDaemonTaskDefinitionLocked(familyOrArn string) (*D } } - if td, ok := b.daemonTaskDefByArn[familyOrArn]; ok { + if td, ok := b.daemonTaskDefByArn.Get(familyOrArn); ok { cp := *td return &cp, nil @@ -591,14 +582,14 @@ func (b *InMemoryBackend) RegisterDaemonTaskDefinition( excess := len(revisions) - maxDaemonTaskDefinitionRevisions for _, evicted := range revisions[:excess] { - delete(b.daemonTaskDefByArn, evicted.DaemonTaskDefinitionArn) + b.daemonTaskDefByArn.Delete(evicted.DaemonTaskDefinitionArn) } revisions = revisions[excess:] } b.daemonTaskDefinitions[input.Family] = revisions - b.daemonTaskDefByArn[td.DaemonTaskDefinitionArn] = td + b.daemonTaskDefByArn.Put(td) if len(input.Tags) > 0 { copied := make([]Tag, len(input.Tags)) @@ -697,7 +688,7 @@ func (b *InMemoryBackend) DescribeDaemonDeployments(arns []string) ([]DaemonDepl failures := make([]Failure, 0, len(arns)) for _, arn := range arns { - dep, ok := b.daemonDeployments[arn] + dep, ok := b.daemonDeployments.Get(arn) if !ok { failures = append(failures, Failure{ Arn: arn, @@ -724,7 +715,7 @@ func (b *InMemoryBackend) DescribeDaemonRevisions(arns []string) ([]DaemonRevisi failures := make([]Failure, 0, len(arns)) for _, arn := range arns { - rev, ok := b.daemonRevisions[arn] + rev, ok := b.daemonRevisions.Get(arn) if !ok { failures = append(failures, Failure{ Arn: arn, @@ -758,9 +749,10 @@ func (b *InMemoryBackend) ListDaemonDeployments(input ListDaemonDeploymentsInput wantStatus[strings.ToUpper(s)] = true } - out := make([]DaemonDeployment, 0, len(b.daemonDeployments)) + all := b.daemonDeployments.All() + out := make([]DaemonDeployment, 0, len(all)) - for _, dep := range b.daemonDeployments { + for _, dep := range all { if dep.DaemonArn != input.DaemonArn { continue } diff --git a/services/ecs/backend_ext.go b/services/ecs/backend_ext.go index 14f78ab74..fd850c221 100644 --- a/services/ecs/backend_ext.go +++ b/services/ecs/backend_ext.go @@ -106,7 +106,7 @@ func (b *InMemoryBackend) RegisterContainerInstance( b.ensureClusterLocked(clusterName) - clusterObj, ok := b.clusters[clusterName] + clusterObj, ok := b.clusters.Get(clusterName) if !ok { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } @@ -126,7 +126,7 @@ func (b *InMemoryBackend) RegisterContainerInstance( Version: 1, } - b.containerInstances[clusterName][instanceArn] = ci + b.containerInstances.Put(ci) cp := *ci @@ -143,18 +143,17 @@ func (b *InMemoryBackend) DeregisterContainerInstance( b.mu.Lock("DeregisterContainerInstance") defer b.mu.Unlock() - instances, ok := b.containerInstances[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } - ci, ok := instances[containerInstance] + ci, ok := b.containerInstances.Get(scopedKey(clusterName, containerInstance)) if !ok { return nil, fmt.Errorf("%w: %s", ErrContainerInstanceNotFound, containerInstance) } if !force { - for _, t := range b.tasks[clusterName] { + for _, t := range b.tasksByCluster.Get(clusterName) { if t.ContainerInstanceArn == containerInstance && t.LastStatus == statusRunning { return nil, fmt.Errorf( "%w: container instance has running tasks; use force=true to override", @@ -164,7 +163,7 @@ func (b *InMemoryBackend) DeregisterContainerInstance( } } - delete(instances, containerInstance) + b.containerInstances.Delete(scopedKey(clusterName, containerInstance)) cp := *ci cp.Status = statusInactive @@ -182,12 +181,12 @@ func (b *InMemoryBackend) DescribeContainerInstances( b.mu.RLock("DescribeContainerInstances") defer b.mu.RUnlock() - instances, ok := b.containerInstances[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } if len(containerInstances) == 0 { + instances := b.containerInstancesByCluster.Get(clusterName) out := make([]ContainerInstance, 0, len(instances)) for _, ci := range instances { out = append(out, b.enrichContainerInstance(ci, clusterName)) @@ -200,7 +199,7 @@ func (b *InMemoryBackend) DescribeContainerInstances( failures := make([]Failure, 0, len(containerInstances)) for _, ref := range containerInstances { - ci, found := instances[ref] + ci, found := b.containerInstances.Get(scopedKey(clusterName, ref)) if !found { failures = append(failures, Failure{ Arn: ref, @@ -231,7 +230,7 @@ func (b *InMemoryBackend) enrichContainerInstance( if instanceIndex, ok := b.tasksByInstance[clusterName]; ok { for taskArn := range instanceIndex[ci.ContainerInstanceArn] { - if t, found := b.tasks[clusterName][taskArn]; found { + if t, found := b.tasks.Get(taskArn); found && clusterKey(t.ClusterArn) == clusterName { switch t.LastStatus { case statusRunning: running++ @@ -242,7 +241,7 @@ func (b *InMemoryBackend) enrichContainerInstance( } } else { // Fallback to linear scan when index is not populated (e.g. after restore). - for _, t := range b.tasks[clusterName] { + for _, t := range b.tasksByCluster.Get(clusterName) { if t.ContainerInstanceArn == ci.ContainerInstanceArn { switch t.LastStatus { case statusRunning: @@ -297,18 +296,18 @@ func (b *InMemoryBackend) ListContainerInstances(cluster, status string) ([]stri b.mu.RLock("ListContainerInstances") defer b.mu.RUnlock() - instances, ok := b.containerInstances[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } + instances := b.containerInstancesByCluster.Get(clusterName) arns := make([]string, 0, len(instances)) - for arn, ci := range instances { + for _, ci := range instances { if status != "" && ci.Status != status { continue } - arns = append(arns, arn) + arns = append(arns, ci.ContainerInstanceArn) } return arns, nil @@ -335,15 +334,14 @@ func (b *InMemoryBackend) UpdateContainerInstancesState( b.mu.Lock("UpdateContainerInstancesState") defer b.mu.Unlock() - instances, ok := b.containerInstances[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } out := make([]ContainerInstance, 0, len(containerInstances)) for _, ref := range containerInstances { - ci, found := instances[ref] + ci, found := b.containerInstances.Get(scopedKey(clusterName, ref)) if !found { return nil, fmt.Errorf("%w: %s", ErrContainerInstanceNotFound, ref) } @@ -373,15 +371,14 @@ func (b *InMemoryBackend) CreateTaskSet(input CreateTaskSetInput) (*TaskSet, err b.mu.Lock("CreateTaskSet") defer b.mu.Unlock() - svcs, ok := b.services[clusterName] - if !ok { - return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, input.Cluster) - } - svcKey := serviceKey(input.Service) - svc, ok := svcs[svcKey] + svc, ok := b.services.Get(scopedKey(clusterName, svcKey)) if !ok { + if !b.clusters.Has(clusterName) { + return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, input.Cluster) + } + return nil, fmt.Errorf("%w: %s", ErrServiceNotFound, input.Service) } @@ -437,12 +434,7 @@ func (b *InMemoryBackend) CreateTaskSet(input CreateTaskSetInput) (*TaskSet, err NetworkConfiguration: input.NetworkConfiguration, } - serviceArn := svc.ServiceArn - if b.taskSets[serviceArn] == nil { - b.taskSets[serviceArn] = make(map[string]*TaskSet) - } - - b.taskSets[serviceArn][taskSetArn] = ts + b.taskSets.Put(ts) cp := *ts @@ -456,29 +448,23 @@ func (b *InMemoryBackend) DeleteTaskSet(cluster, service, taskSet string) (*Task b.mu.Lock("DeleteTaskSet") defer b.mu.Unlock() - svcs, ok := b.services[clusterName] - if !ok { - return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) - } - svcKey := serviceKey(service) - svc, ok := svcs[svcKey] + svc, ok := b.services.Get(scopedKey(clusterName, svcKey)) if !ok { - return nil, fmt.Errorf("%w: %s", ErrServiceNotFound, service) - } + if !b.clusters.Has(clusterName) { + return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) + } - sets, ok := b.taskSets[svc.ServiceArn] - if !ok { - return nil, fmt.Errorf("%w: %s", ErrTaskSetNotFound, taskSet) + return nil, fmt.Errorf("%w: %s", ErrServiceNotFound, service) } - ts, ok := sets[taskSet] + ts, ok := b.taskSets.Get(scopedKey(svc.ServiceArn, taskSet)) if !ok { return nil, fmt.Errorf("%w: %s", ErrTaskSetNotFound, taskSet) } - delete(sets, taskSet) + b.taskSets.Delete(scopedKey(svc.ServiceArn, taskSet)) cp := *ts @@ -495,19 +481,18 @@ func (b *InMemoryBackend) DescribeTaskSets( b.mu.RLock("DescribeTaskSets") defer b.mu.RUnlock() - svcs, ok := b.services[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } svcKey := serviceKey(service) - svc, ok := svcs[svcKey] + svc, ok := b.services.Get(scopedKey(clusterName, svcKey)) if !ok { return nil, fmt.Errorf("%w: %s", ErrServiceNotFound, service) } - sets := b.taskSets[svc.ServiceArn] + sets := b.taskSetsByService.Get(svc.ServiceArn) if len(taskSets) == 0 { out := make([]TaskSet, 0, len(sets)) @@ -521,7 +506,7 @@ func (b *InMemoryBackend) DescribeTaskSets( out := make([]TaskSet, 0, len(taskSets)) for _, ref := range taskSets { - ts, found := sets[ref] + ts, found := b.taskSets.Get(scopedKey(svc.ServiceArn, ref)) if !found { return nil, fmt.Errorf("%w: %s", ErrTaskSetNotFound, ref) } @@ -550,24 +535,18 @@ func (b *InMemoryBackend) UpdateTaskSet( b.mu.Lock("UpdateTaskSet") defer b.mu.Unlock() - svcs, ok := b.services[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } svcKey := serviceKey(service) - svc, ok := svcs[svcKey] + svc, ok := b.services.Get(scopedKey(clusterName, svcKey)) if !ok { return nil, fmt.Errorf("%w: %s", ErrServiceNotFound, service) } - sets, ok := b.taskSets[svc.ServiceArn] - if !ok { - return nil, fmt.Errorf("%w: %s", ErrTaskSetNotFound, taskSet) - } - - ts, ok := sets[taskSet] + ts, ok := b.taskSets.Get(scopedKey(svc.ServiceArn, taskSet)) if !ok { return nil, fmt.Errorf("%w: %s", ErrTaskSetNotFound, taskSet) } @@ -589,27 +568,26 @@ func (b *InMemoryBackend) UpdateServicePrimaryTaskSet( b.mu.Lock("UpdateServicePrimaryTaskSet") defer b.mu.Unlock() - svcs, ok := b.services[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } svcKey := serviceKey(service) - svc, ok := svcs[svcKey] + svc, ok := b.services.Get(scopedKey(clusterName, svcKey)) if !ok { return nil, fmt.Errorf("%w: %s", ErrServiceNotFound, service) } - sets := b.taskSets[svc.ServiceArn] - if _, found := sets[primaryTaskSet]; !found { + primary, found := b.taskSets.Get(scopedKey(svc.ServiceArn, primaryTaskSet)) + if !found { return nil, fmt.Errorf("%w: %s", ErrTaskSetNotFound, primaryTaskSet) } now := time.Now() - for arn, ts := range sets { - if arn == primaryTaskSet { + for _, ts := range b.taskSetsByService.Get(svc.ServiceArn) { + if ts.TaskSetArn == primaryTaskSet { ts.Status = "PRIMARY" } else { ts.Status = statusActive @@ -618,7 +596,7 @@ func (b *InMemoryBackend) UpdateServicePrimaryTaskSet( ts.UpdatedAt = now } - cp := *sets[primaryTaskSet] + cp := *primary return &cp, nil } @@ -641,13 +619,12 @@ func (b *InMemoryBackend) ExecuteCommand( b.mu.RLock("ExecuteCommand") defer b.mu.RUnlock() - clusterTasks, ok := b.tasks[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } - t, ok := clusterTasks[task] - if !ok { + t, ok := b.tasks.Get(task) + if !ok || clusterKey(t.ClusterArn) != clusterName { return nil, fmt.Errorf("%w: %s", ErrTaskNotFound, task) } @@ -662,7 +639,7 @@ func (b *InMemoryBackend) ExecuteCommand( ) } - clusterObj := b.clusters[clusterName] + clusterObj, _ := b.clusters.Get(clusterName) sessionID := uuid.NewString() return &ExecuteCommandOutput{ @@ -701,11 +678,11 @@ func (b *InMemoryBackend) ListServices( b.mu.RLock("ListServices") defer b.mu.RUnlock() - svcs, ok := b.services[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } + svcs := b.servicesByCluster.Get(clusterName) arns := make([]string, 0, len(svcs)) for _, svc := range svcs { diff --git a/services/ecs/backend_iface.go b/services/ecs/backend_iface.go index 9a13f66ce..b1c82deb0 100644 --- a/services/ecs/backend_iface.go +++ b/services/ecs/backend_iface.go @@ -134,6 +134,7 @@ type Backend interface { ) ([]ServiceDeployment, []Failure, error) ListServiceDeployments(cluster, service string) ([]string, error) StopServiceDeployment(serviceDeploymentArn string) (*ServiceDeployment, error) + ContinueServiceDeployment(serviceDeploymentArn, hookID, action string) (*ServiceDeployment, error) // Express gateway services diff --git a/services/ecs/backend_new_ops.go b/services/ecs/backend_new_ops.go index d54e21f3f..dbca9868b 100644 --- a/services/ecs/backend_new_ops.go +++ b/services/ecs/backend_new_ops.go @@ -3,6 +3,7 @@ package ecs import ( "fmt" "strconv" + "strings" "time" "github.com/blackbirdworks/gopherstack/pkgs/awserr" @@ -99,6 +100,84 @@ type ServiceDeployment struct { StatusReason string `json:"statusReason,omitempty"` } +// serviceDeploymentArnFor derives the ARN of the service deployment record for +// a Deployment, following the +// arn:aws:ecs:region:account:service-deployment/cluster/service/deployment-id +// scheme (mirroring serviceRevisionArnFor in backend_parity2.go). deploymentID +// already carries its "ecs-svc/" prefix (see newPrimaryDeployment/ +// newActiveDeployment), matching the shape of real ECS deployment IDs. +func serviceDeploymentArnFor(svc *Service, deploymentID string) string { + return strings.Replace(svc.ServiceArn, ":service/", ":service-deployment/", 1) + "/" + deploymentID +} + +// serviceDeploymentStatusFor maps a Deployment's RolloutState to the +// corresponding ServiceDeploymentStatus value. IN_PROGRESS is the default for +// any rollout state this backend doesn't model as a distinct terminal state. +func serviceDeploymentStatusFor(rolloutState string) string { + switch rolloutState { + case deploymentRolloutStateCompleted: + return "SUCCESSFUL" + case deploymentRolloutStateFailed: + return statusStopped + default: + return "IN_PROGRESS" + } +} + +// recordServiceDeploymentLocked upserts the ServiceDeployment record tracking +// a single Deployment. Must be called with the write lock held. +func (b *InMemoryBackend) recordServiceDeploymentLocked(svc *Service, dep *Deployment) { + depArn := serviceDeploymentArnFor(svc, dep.ID) + + createdAt := time.Now() + if dep.CreatedAt != nil { + createdAt = time.Unix(int64(*dep.CreatedAt), 0) + } + + updatedAt := createdAt + if dep.UpdatedAt != nil { + updatedAt = time.Unix(int64(*dep.UpdatedAt), 0) + } + + b.serviceDeployments.Put(&ServiceDeployment{ + ServiceDeploymentArn: depArn, + ClusterArn: svc.ClusterArn, + ServiceArn: svc.ServiceArn, + Status: serviceDeploymentStatusFor(dep.RolloutState), + StatusReason: dep.RolloutStateReason, + CreatedAt: &createdAt, + UpdatedAt: &updatedAt, + }) +} + +// deleteServiceDeploymentsForServiceLocked removes every ServiceDeployment +// record belonging to a service (keyed by ServiceDeploymentArn, which embeds +// the deployment ID — not by ServiceArn), so a deleted/purged service doesn't +// leave stale entries behind. Must be called with the write lock held. +func (b *InMemoryBackend) deleteServiceDeploymentsForServiceLocked(serviceArn string) { + for _, sd := range b.serviceDeployments.All() { + if sd.ServiceArn == serviceArn { + b.serviceDeployments.Delete(sd.ServiceDeploymentArn) + } + } +} + +// syncServiceDeploymentsLocked upserts a ServiceDeployment record for every +// entry currently on svc.Deployments. CreateService, UpdateService, and the +// deployment-circuit-breaker rollback path (deployment.go) all mutate +// svc.Deployments directly and must call this afterward so +// DescribeServiceDeployments/ListServiceDeployments/StopServiceDeployment stay +// in sync — without it, those three routed ops only ever see data seeded by +// the AddServiceDeploymentInternal test helper, never anything a real +// deployment created (see parity-principles.md rule 4: a "real-looking" op +// filtering a never-populated map is a disguised stub). Must be called with +// the write lock held. +func (b *InMemoryBackend) syncServiceDeploymentsLocked(svc *Service) { + for i := range svc.Deployments { + b.recordServiceDeploymentLocked(svc, &svc.Deployments[i]) + } +} + // ExpressGatewayService represents an ECS express gateway service. type ExpressGatewayService struct { CreatedAt time.Time `json:"createdAt"` @@ -146,12 +225,16 @@ func copyTags(tags []Tag) []Tag { return out } -// AddAccountSettingInternal adds an account setting directly (seed helper for tests). -func (b *InMemoryBackend) AddAccountSettingInternal(key string, setting *AccountSetting) { +// AddAccountSettingInternal adds an account setting directly (seed helper for +// tests). key is unused now that the store.Table derives its own key from +// setting's fields (accountSettingsKeyFn); retained in the signature so +// existing call sites (which always pass a key consistent with +// accountSettingKey(setting.Name, setting.PrincipalArn)) do not need updating. +func (b *InMemoryBackend) AddAccountSettingInternal(_ string, setting *AccountSetting) { b.mu.Lock("AddAccountSettingInternal") defer b.mu.Unlock() - b.accountSettings[key] = setting + b.accountSettings.Put(setting) } // AddAttributeInternal adds an attribute directly (seed helper for tests). @@ -173,7 +256,7 @@ func (b *InMemoryBackend) AddServiceDeploymentInternal(sd *ServiceDeployment) { defer b.mu.Unlock() c := *sd - b.serviceDeployments[sd.ServiceDeploymentArn] = &c + b.serviceDeployments.Put(&c) } // AddCapacityProviderInternal adds a capacity provider directly (seed helper for tests). @@ -183,7 +266,7 @@ func (b *InMemoryBackend) AddCapacityProviderInternal(cp *CapacityProvider) { c := *cp c.Tags = copyTags(cp.Tags) - b.capacityProviders[cp.Name] = &c + b.capacityProviders.Put(&c) } // accountSettingKey builds the map key for an account setting. @@ -209,7 +292,7 @@ func (b *InMemoryBackend) CreateCapacityProvider( b.mu.Lock("CreateCapacityProvider") defer b.mu.Unlock() - if _, ok := b.capacityProviders[input.Name]; ok { + if b.capacityProviders.Has(input.Name) { return nil, fmt.Errorf("%w: %s", ErrCapacityProviderAlreadyExists, input.Name) } @@ -224,7 +307,7 @@ func (b *InMemoryBackend) CreateCapacityProvider( Tags: copyTags(input.Tags), } - b.capacityProviders[input.Name] = cp + b.capacityProviders.Put(cp) out := *cp out.Tags = copyTags(cp.Tags) @@ -242,7 +325,7 @@ func (b *InMemoryBackend) DeleteCapacityProvider(nameOrArn string) (*CapacityPro return nil, fmt.Errorf("%w: %s", ErrCapacityProviderNotFound, nameOrArn) } - delete(b.capacityProviders, key) + b.capacityProviders.Delete(key) out := *cp @@ -257,8 +340,9 @@ func (b *InMemoryBackend) DescribeCapacityProviders( defer b.mu.RUnlock() if len(nameOrArns) == 0 { - out := make([]CapacityProvider, 0, len(b.capacityProviders)) - for _, cp := range b.capacityProviders { + all := b.capacityProviders.All() + out := make([]CapacityProvider, 0, len(all)) + for _, cp := range all { c := *cp c.Tags = copyTags(cp.Tags) out = append(out, c) @@ -294,13 +378,13 @@ func (b *InMemoryBackend) DescribeCapacityProviders( // findCapacityProviderLocked returns the map key and pointer for a capacity provider by name or ARN. // Must be called with at least an RLock held. func (b *InMemoryBackend) findCapacityProviderLocked(nameOrArn string) (string, *CapacityProvider) { - if cp, ok := b.capacityProviders[nameOrArn]; ok { + if cp, ok := b.capacityProviders.Get(nameOrArn); ok { return nameOrArn, cp } - for key, cp := range b.capacityProviders { + for _, cp := range b.capacityProviders.All() { if cp.CapacityProviderArn == nameOrArn { - return key, cp + return cp.Name, cp } } @@ -320,12 +404,12 @@ func (b *InMemoryBackend) DeleteAccountSetting(name, principalArn string) (*Acco b.mu.Lock("DeleteAccountSetting") defer b.mu.Unlock() - setting, ok := b.accountSettings[key] + setting, ok := b.accountSettings.Get(key) if !ok { return nil, fmt.Errorf("%w: %s", ErrAccountSettingNotFound, name) } - delete(b.accountSettings, key) + b.accountSettings.Delete(key) out := *setting @@ -405,7 +489,7 @@ func (b *InMemoryBackend) DeleteTaskDefinitions( for i, r := range revs { if r.TaskDefinitionArn == td.TaskDefinitionArn { b.taskDefinitions[td.Family] = append(revs[:i], revs[i+1:]...) - delete(b.taskDefByArn, td.TaskDefinitionArn) + b.taskDefByArn.Delete(td.TaskDefinitionArn) break } @@ -430,7 +514,7 @@ func (b *InMemoryBackend) DescribeServiceDeployments( failures := make([]Failure, 0, len(serviceDeploymentArns)) for _, arn := range serviceDeploymentArns { - sd, ok := b.serviceDeployments[arn] + sd, ok := b.serviceDeployments.Get(arn) if !ok { failures = append(failures, Failure{ Arn: arn, @@ -475,7 +559,7 @@ func (b *InMemoryBackend) CreateExpressGatewayService( "arn:aws:ecs:%s:%s:service/%s/%s", b.region, b.accountID, clusterName, serviceName, ) - if _, ok := b.expressGatewayServices[serviceArn]; ok { + if b.expressGatewayServices.Has(serviceArn) { return nil, fmt.Errorf("%w: %s", ErrExpressGatewayServiceAlreadyExists, serviceName) } @@ -490,7 +574,7 @@ func (b *InMemoryBackend) CreateExpressGatewayService( Tags: copyTags(input.Tags), } - b.expressGatewayServices[serviceArn] = svc + b.expressGatewayServices.Put(svc) out := *svc out.Tags = copyTags(svc.Tags) @@ -509,12 +593,12 @@ func (b *InMemoryBackend) DeleteExpressGatewayService( b.mu.Lock("DeleteExpressGatewayService") defer b.mu.Unlock() - svc, ok := b.expressGatewayServices[serviceArn] + svc, ok := b.expressGatewayServices.Get(serviceArn) if !ok { return nil, fmt.Errorf("%w: %s", ErrExpressGatewayServiceNotFound, serviceArn) } - delete(b.expressGatewayServices, serviceArn) + b.expressGatewayServices.Delete(serviceArn) out := *svc @@ -532,7 +616,7 @@ func (b *InMemoryBackend) DescribeExpressGatewayService( b.mu.RLock("DescribeExpressGatewayService") defer b.mu.RUnlock() - svc, ok := b.expressGatewayServices[serviceArn] + svc, ok := b.expressGatewayServices.Get(serviceArn) if !ok { return nil, fmt.Errorf("%w: %s", ErrExpressGatewayServiceNotFound, serviceArn) } diff --git a/services/ecs/backend_ops2.go b/services/ecs/backend_ops2.go index 31b98c2c8..317bb5644 100644 --- a/services/ecs/backend_ops2.go +++ b/services/ecs/backend_ops2.go @@ -39,9 +39,10 @@ func (b *InMemoryBackend) ListAccountSettings(name, principalArn string) ([]Acco b.mu.RLock("ListAccountSettings") defer b.mu.RUnlock() - out := make([]AccountSetting, 0, len(b.accountSettings)) + all := b.accountSettings.All() + out := make([]AccountSetting, 0, len(all)) - for _, setting := range b.accountSettings { + for _, setting := range all { if name != "" && setting.Name != name { continue } @@ -70,8 +71,6 @@ func (b *InMemoryBackend) PutAccountSetting( return nil, fmt.Errorf("%w: value is required", ErrInvalidParameter) } - key := accountSettingKey(name, principalArn) - b.mu.Lock("PutAccountSetting") defer b.mu.Unlock() @@ -81,7 +80,7 @@ func (b *InMemoryBackend) PutAccountSetting( PrincipalArn: principalArn, } - b.accountSettings[key] = setting + b.accountSettings.Put(setting) out := *setting @@ -168,7 +167,7 @@ func (b *InMemoryBackend) PutClusterCapacityProviders( b.mu.Lock("PutClusterCapacityProviders") defer b.mu.Unlock() - c, ok := b.clusters[clusterName] + c, ok := b.clusters.Get(clusterName) if !ok { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } @@ -193,7 +192,7 @@ func (b *InMemoryBackend) UpdateClusterSettings( b.mu.Lock("UpdateClusterSettings") defer b.mu.Unlock() - c, ok := b.clusters[clusterName] + c, ok := b.clusters.Get(clusterName) if !ok { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } @@ -216,12 +215,11 @@ func (b *InMemoryBackend) UpdateContainerAgent( b.mu.Lock("UpdateContainerAgent") defer b.mu.Unlock() - instances, ok := b.containerInstances[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } - ci, ok := instances[containerInstance] + ci, ok := b.containerInstances.Get(scopedKey(clusterName, containerInstance)) if !ok { return nil, fmt.Errorf("%w: %s", ErrContainerInstanceNotFound, containerInstance) } @@ -246,7 +244,7 @@ func (b *InMemoryBackend) UpdateExpressGatewayService( b.mu.Lock("UpdateExpressGatewayService") defer b.mu.Unlock() - svc, ok := b.expressGatewayServices[input.ServiceArn] + svc, ok := b.expressGatewayServices.Get(input.ServiceArn) if !ok { return nil, fmt.Errorf("%w: %s", ErrExpressGatewayServiceNotFound, input.ServiceArn) } @@ -277,16 +275,16 @@ func (b *InMemoryBackend) GetTaskProtection( b.mu.RLock("GetTaskProtection") defer b.mu.RUnlock() - tasks, ok := b.tasks[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } if len(taskArns) == 0 { - out := make([]TaskProtection, 0, len(tasks)) + clusterTasks := b.tasksByCluster.Get(clusterName) + out := make([]TaskProtection, 0, len(clusterTasks)) - for arn := range tasks { - out = append(out, b.taskProtectionLocked(arn)) + for _, t := range clusterTasks { + out = append(out, b.taskProtectionLocked(t.TaskArn)) } return out, nil, nil @@ -296,7 +294,7 @@ func (b *InMemoryBackend) GetTaskProtection( failures := make([]Failure, 0, len(taskArns)) for _, arn := range taskArns { - if _, found := tasks[arn]; !found { + if t, found := b.tasks.Get(arn); !found || clusterKey(t.ClusterArn) != clusterName { failures = append(failures, Failure{ Arn: arn, Reason: statusMissing, @@ -315,7 +313,7 @@ func (b *InMemoryBackend) GetTaskProtection( // taskProtectionLocked returns the protection for a task ARN (defaults to unprotected). // Must be called with at least an RLock held. func (b *InMemoryBackend) taskProtectionLocked(taskArn string) TaskProtection { - if tp, ok := b.taskProtections[taskArn]; ok { + if tp, ok := b.taskProtections.Get(taskArn); ok { return *tp } @@ -340,8 +338,7 @@ func (b *InMemoryBackend) UpdateTaskProtection( b.mu.Lock("UpdateTaskProtection") defer b.mu.Unlock() - tasks, ok := b.tasks[clusterName] - if !ok { + if !b.clusters.Has(clusterName) { return nil, nil, fmt.Errorf("%w: %s", ErrClusterNotFound, cluster) } @@ -349,7 +346,7 @@ func (b *InMemoryBackend) UpdateTaskProtection( failures := make([]Failure, 0, len(taskArns)) for _, arn := range taskArns { - if _, found := tasks[arn]; !found { + if t, found := b.tasks.Get(arn); !found || clusterKey(t.ClusterArn) != clusterName { failures = append(failures, Failure{ Arn: arn, Reason: statusMissing, @@ -369,7 +366,7 @@ func (b *InMemoryBackend) UpdateTaskProtection( tp.ExpirationDate = &exp } - b.taskProtections[arn] = tp + b.taskProtections.Put(tp) out = append(out, *tp) } diff --git a/services/ecs/backend_parity2.go b/services/ecs/backend_parity2.go index ac6579885..b067a68ce 100644 --- a/services/ecs/backend_parity2.go +++ b/services/ecs/backend_parity2.go @@ -358,7 +358,7 @@ func mergeTags(base, overrides []Tag) []Tag { // container instance would violate the distinctInstance constraint (i.e., a task // from the same service already runs on that instance). func placementViolatesDistinctInstance( - clusterTasks map[string]*Task, + clusterTasks []*Task, instanceArn string, serviceName string, ) bool { @@ -385,8 +385,8 @@ func placementViolatesDistinctInstance( // // Falls back to random when the strategy list is empty or the type is unrecognized. func selectContainerInstance( - instances map[string]*ContainerInstance, - clusterTasks map[string]*Task, + instances []*ContainerInstance, + clusterTasks []*Task, constraints []PlacementConstraint, strategies []PlacementStrategy, serviceName string, @@ -394,7 +394,8 @@ func selectContainerInstance( // Collect eligible instances (ACTIVE + not violating constraints). eligible := make([]string, 0, len(instances)) - for arn, ci := range instances { + for _, ci := range instances { + arn := ci.ContainerInstanceArn if ci.Status != statusActive { continue } @@ -449,7 +450,7 @@ func cryptoRandChoice(items []string) string { } // taskCountOnInstance counts running tasks assigned to a container instance. -func taskCountOnInstance(clusterTasks map[string]*Task, instanceArn string) int { +func taskCountOnInstance(clusterTasks []*Task, instanceArn string) int { count := 0 for _, t := range clusterTasks { @@ -462,7 +463,7 @@ func taskCountOnInstance(clusterTasks map[string]*Task, instanceArn string) int } // leastLoadedInstance returns the instance with the fewest running tasks. -func leastLoadedInstance(eligible []string, clusterTasks map[string]*Task) string { +func leastLoadedInstance(eligible []string, clusterTasks []*Task) string { best := eligible[0] bestCount := taskCountOnInstance(clusterTasks, best) @@ -477,7 +478,7 @@ func leastLoadedInstance(eligible []string, clusterTasks map[string]*Task) strin } // mostLoadedInstance returns the instance with the most running tasks. -func mostLoadedInstance(eligible []string, clusterTasks map[string]*Task) string { +func mostLoadedInstance(eligible []string, clusterTasks []*Task) string { best := eligible[0] bestCount := taskCountOnInstance(clusterTasks, best) diff --git a/services/ecs/backend_parity_internal_test.go b/services/ecs/backend_parity_internal_test.go index 26f7d9d03..49f12c583 100644 --- a/services/ecs/backend_parity_internal_test.go +++ b/services/ecs/backend_parity_internal_test.go @@ -769,11 +769,28 @@ func TestDeleteCluster_CascadesServiceDeployments(t *testing.T) { t.Fatalf("CreateService: %v", err) } - // Inject a fake service deployment entry to test cascade deletion. - b.mu.Lock("test-inject") - b.serviceDeployments[svc.ServiceArn] = &ServiceDeployment{ - ServiceDeploymentArn: "arn:aws:ecs:us-east-1:123456789012:service-deployment/test-cluster/my-svc/abc", + // CreateService itself records a real ServiceDeployment for the initial + // PRIMARY deployment (see syncServiceDeploymentsLocked in + // backend_new_ops.go), keyed by ServiceDeploymentArn — not by ServiceArn. + // Confirm it exists before asserting the cascade delete. + deploymentArns, err := b.ListServiceDeployments("test-cluster", "my-svc") + if err != nil { + t.Fatalf("ListServiceDeployments: %v", err) } + + if len(deploymentArns) != 1 { + t.Fatalf("ListServiceDeployments before delete = %d entries, want 1", len(deploymentArns)) + } + + // Also inject a second, independently-keyed entry (as an external caller + // or an older revision might leave behind) to prove the cascade matches on + // the ServiceArn field rather than assuming a single well-known key. + extraArn := "arn:aws:ecs:us-east-1:123456789012:service-deployment/test-cluster/my-svc/abc" + b.mu.Lock("test-inject") + b.serviceDeployments.Put(&ServiceDeployment{ + ServiceDeploymentArn: extraArn, + ServiceArn: svc.ServiceArn, + }) b.mu.Unlock() _, err = b.DeleteCluster("test-cluster") @@ -781,13 +798,18 @@ func TestDeleteCluster_CascadesServiceDeployments(t *testing.T) { t.Fatalf("DeleteCluster: %v", err) } - // Service deployment should be gone. + // Both the real and the injected service deployment should be gone. b.mu.RLock("test-verify") - _, stillExists := b.serviceDeployments[svc.ServiceArn] + _, realStillExists := b.serviceDeployments.Get(deploymentArns[0]) + _, extraStillExists := b.serviceDeployments.Get(extraArn) b.mu.RUnlock() - if stillExists { - t.Error("service deployment not cascade-deleted with cluster") + if realStillExists { + t.Error("real service deployment not cascade-deleted with cluster") + } + + if extraStillExists { + t.Error("injected service deployment not cascade-deleted with cluster") } } diff --git a/services/ecs/backend_refinement1.go b/services/ecs/backend_refinement1.go index d8aef6dc0..c97d0718e 100644 --- a/services/ecs/backend_refinement1.go +++ b/services/ecs/backend_refinement1.go @@ -48,7 +48,7 @@ func (b *InMemoryBackend) UpdateCluster(input UpdateClusterInput) (*Cluster, err b.mu.Lock("UpdateCluster") defer b.mu.Unlock() - c, ok := b.clusters[clusterName] + c, ok := b.clusters.Get(clusterName) if !ok { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, input.Cluster) } @@ -201,7 +201,7 @@ func (b *InMemoryBackend) StartTask(input StartTaskInput) ([]Task, error) { StartedAt: &now, } - b.tasks[clusterName][taskArn] = t + b.tasks.Put(t) tasks = append(tasks, *t) } @@ -219,10 +219,7 @@ func (b *InMemoryBackend) ListServicesByNamespace(cluster, namespace string) ([] b.mu.RLock("ListServicesByNamespace") defer b.mu.RUnlock() - svcs, ok := b.services[clusterName] - if !ok { - return []string{}, nil - } + svcs := b.servicesByCluster.Get(clusterName) out := make([]string, 0, len(svcs)) @@ -246,6 +243,17 @@ func (b *InMemoryBackend) TagResource(resourceArn string, tags []Tag) error { b.mu.Lock("TagResource") defer b.mu.Unlock() + b.setResourceTagsLocked(resourceArn, tags) + + return nil +} + +// setResourceTagsLocked merges tags into the resourceTags side map for a +// resource ARN, overwriting any existing tag with the same key. Shared by +// TagResource and resource-creation ops (for example CreateCluster) that +// accept an initial set of tags while already holding the write lock. Must be +// called with the write lock held. +func (b *InMemoryBackend) setResourceTagsLocked(resourceArn string, tags []Tag) { if b.resourceTags == nil { b.resourceTags = make(map[string][]Tag) } @@ -269,8 +277,6 @@ func (b *InMemoryBackend) TagResource(resourceArn string, tags []Tag) error { } b.resourceTags[resourceTagKey(resourceArn)] = merged - - return nil } // UntagResource removes tags with the given keys from a resource. @@ -342,9 +348,10 @@ func (b *InMemoryBackend) ListServiceDeployments(cluster, service string) ([]str b.mu.RLock("ListServiceDeployments") defer b.mu.RUnlock() - out := make([]string, 0, len(b.serviceDeployments)) + all := b.serviceDeployments.All() + out := make([]string, 0, len(all)) - for _, sd := range b.serviceDeployments { + for _, sd := range all { if sd.ClusterArn != "" && !strings.HasSuffix(sd.ClusterArn, "/"+clusterName) { continue } @@ -379,7 +386,7 @@ func (b *InMemoryBackend) StopServiceDeployment( b.mu.Lock("StopServiceDeployment") defer b.mu.Unlock() - sd, ok := b.serviceDeployments[serviceDeploymentArn] + sd, ok := b.serviceDeployments.Get(serviceDeploymentArn) if !ok { return nil, fmt.Errorf("%w: %s", ErrServiceDeploymentNotFound, serviceDeploymentArn) } @@ -396,3 +403,54 @@ func (b *InMemoryBackend) StopServiceDeployment( return &out, nil } + +// ---- ContinueServiceDeployment ---- + +const ( + deploymentLifecycleActionContinue = "CONTINUE" + deploymentLifecycleActionRollback = "ROLLBACK" +) + +// errNoLifecycleHook is returned by ContinueServiceDeployment: this backend +// never pauses a deployment at a lifecycle hook (blue/green PAUSE stages +// aren't modeled — every deployment either runs to completion or trips the +// circuit breaker), so there is never a paused hookId to act on. Matching AWS +// behavior for "act on a hook that isn't currently paused" as closely as +// possible without modeling the full lifecycle-hook state machine. +var errNoLifecycleHook = awserr.New("ClientException", awserr.ErrInvalidParameter) + +// ContinueServiceDeployment continues or rolls back a service deployment that +// is paused at a lifecycle hook. Real ECS only pauses a deployment when a +// PAUSE-stage lifecycle hook (a Lambda-backed hook configured on the service) +// is present; this backend does not model lifecycle hooks, so a deployment is +// never actually paused. The op still validates the deployment exists and the +// required hookId is present before reporting that there is no such paused +// hook — it does not fabricate a successful continue/rollback for state that +// was never paused. +func (b *InMemoryBackend) ContinueServiceDeployment( + serviceDeploymentArn, hookID, action string, +) (*ServiceDeployment, error) { + if serviceDeploymentArn == "" { + return nil, fmt.Errorf("%w: serviceDeploymentArn is required", ErrInvalidParameter) + } + + if hookID == "" { + return nil, fmt.Errorf("%w: hookId is required", ErrInvalidParameter) + } + + switch action { + case "", deploymentLifecycleActionContinue, deploymentLifecycleActionRollback: + default: + return nil, fmt.Errorf("%w: invalid action %q", ErrInvalidParameter, action) + } + + b.mu.RLock("ContinueServiceDeployment") + defer b.mu.RUnlock() + + if !b.serviceDeployments.Has(serviceDeploymentArn) { + return nil, fmt.Errorf("%w: %s", ErrServiceDeploymentNotFound, serviceDeploymentArn) + } + + return nil, fmt.Errorf("%w: no paused lifecycle hook %q found for service deployment %s", + errNoLifecycleHook, hookID, serviceDeploymentArn) +} diff --git a/services/ecs/deployment.go b/services/ecs/deployment.go index fcd393042..1adba9772 100644 --- a/services/ecs/deployment.go +++ b/services/ecs/deployment.go @@ -86,12 +86,7 @@ func (b *InMemoryBackend) recordServiceTaskFailureLocked(clusterName string, tas return } - svcs, ok := b.services[clusterName] - if !ok { - return - } - - svc, ok := svcs[serviceName] + svc, ok := b.services.Get(scopedKey(clusterName, serviceName)) if !ok { return } @@ -145,6 +140,8 @@ func (b *InMemoryBackend) evaluateCircuitBreakerLocked(svc *Service) { if circuitBreakerRollback(svc) { b.rollbackServiceLocked(svc) } + + b.syncServiceDeploymentsLocked(svc) } // lastStableTaskDefinition returns the task definition of the most recent diff --git a/services/ecs/export_test.go b/services/ecs/export_test.go index 1bc56f8ae..7cba7df87 100644 --- a/services/ecs/export_test.go +++ b/services/ecs/export_test.go @@ -10,7 +10,7 @@ func (b *InMemoryBackend) CapacityProviderCount() int { b.mu.RLock("CapacityProviderCount") defer b.mu.RUnlock() - return len(b.capacityProviders) + return b.capacityProviders.Len() } // ServiceDeploymentCount returns the number of service deployments (test helper). @@ -18,7 +18,7 @@ func (b *InMemoryBackend) ServiceDeploymentCount() int { b.mu.RLock("ServiceDeploymentCount") defer b.mu.RUnlock() - return len(b.serviceDeployments) + return b.serviceDeployments.Len() } // AccountSettingCount returns the number of account settings (test helper). @@ -26,7 +26,7 @@ func (b *InMemoryBackend) AccountSettingCount() int { b.mu.RLock("AccountSettingCount") defer b.mu.RUnlock() - return len(b.accountSettings) + return b.accountSettings.Len() } // AttributeCount returns the number of attributes for a cluster (test helper). @@ -42,7 +42,7 @@ func (b *InMemoryBackend) ExpressGatewayServiceCount() int { b.mu.RLock("ExpressGatewayServiceCount") defer b.mu.RUnlock() - return len(b.expressGatewayServices) + return b.expressGatewayServices.Len() } // GetServicesForReconcilerForTest exposes getServicesForReconciler for tests. diff --git a/services/ecs/handler.go b/services/ecs/handler.go index 35efce4cc..18c53a35d 100644 --- a/services/ecs/handler.go +++ b/services/ecs/handler.go @@ -114,6 +114,7 @@ func (h *Handler) GetSupportedOperations() []string { "ListTagsForResource", "ListServiceDeployments", "StopServiceDeployment", + "ContinueServiceDeployment", // Daemon operations (stub implementations). "CreateDaemon", "DeleteDaemon", @@ -303,8 +304,9 @@ func (h *Handler) buildOps() map[string]service.JSONOpFunc { "UntagResource": service.WrapOp(h.handleUntagResource), "ListTagsForResource": service.WrapOp(h.handleListTagsForResource), // Service deployments - "ListServiceDeployments": service.WrapOp(h.handleListServiceDeployments), - "StopServiceDeployment": service.WrapOp(h.handleStopServiceDeployment), + "ListServiceDeployments": service.WrapOp(h.handleListServiceDeployments), + "StopServiceDeployment": service.WrapOp(h.handleStopServiceDeployment), + "ContinueServiceDeployment": service.WrapOp(h.handleContinueServiceDeployment), // Daemon stubs "CreateDaemon": service.WrapOp(h.handleCreateDaemon), "DeleteDaemon": service.WrapOp(h.handleDeleteDaemon), @@ -430,8 +432,11 @@ func errorCode(err error) string { // ----- Cluster handlers ----- type createClusterInput struct { - ClusterName string `json:"clusterName"` - Settings []clusterSettingView `json:"settings,omitempty"` + ClusterName string `json:"clusterName"` + Settings []clusterSettingView `json:"settings,omitempty"` + CapacityProviders []string `json:"capacityProviders,omitempty"` + DefaultCapacityProviderStrategy []cpStrategyItemInput `json:"defaultCapacityProviderStrategy,omitempty"` + Tags []Tag `json:"tags,omitempty"` } type createClusterOutput struct { @@ -448,14 +453,24 @@ func (h *Handler) handleCreateCluster( } cluster, err := h.Backend.CreateCluster(CreateClusterInput{ - ClusterName: in.ClusterName, - Settings: settings, + ClusterName: in.ClusterName, + Settings: settings, + CapacityProviders: in.CapacityProviders, + DefaultCapacityProviderStrategy: toCPStrategyItems(in.DefaultCapacityProviderStrategy), + Tags: in.Tags, }) if err != nil { return nil, err } - return &createClusterOutput{Cluster: toClusterView(*cluster)}, nil + view := toClusterView(*cluster) + // CreateCluster has no `include` gating (unlike DescribeClusters): the + // tags just supplied are echoed back on the created resource. + if len(in.Tags) > 0 { + view.Tags = in.Tags + } + + return &createClusterOutput{Cluster: view}, nil } type listClustersInput struct { @@ -491,6 +506,7 @@ func (h *Handler) handleListClusters( type describeClustersInput struct { Clusters []string `json:"clusters"` + Include []string `json:"include,omitempty"` } type describeClustersOutput struct { @@ -498,6 +514,11 @@ type describeClustersOutput struct { Failures []failureView `json:"failures"` } +// describeClusterIncludeTags is the AWS-defined `include` value that requests +// resource tags be returned alongside each Cluster (see also +// describeTaskIncludeTags for the equivalent DescribeTaskDefinition option). +const describeClusterIncludeTags = "TAGS" + func (h *Handler) handleDescribeClusters( _ context.Context, in *describeClustersInput, @@ -507,9 +528,30 @@ func (h *Handler) handleDescribeClusters( return nil, err } + wantTags := false + + for _, opt := range in.Include { + if strings.EqualFold(opt, describeClusterIncludeTags) { + wantTags = true + + break + } + } + views := make([]clusterView, 0, len(clusters)) for _, c := range clusters { - views = append(views, toClusterView(c)) + v := toClusterView(c) + + if wantTags { + tags, terr := h.Backend.ListTagsForResource(c.ClusterArn) + if terr != nil { + return nil, terr + } + + v.Tags = tags + } + + views = append(views, v) } failViews := make([]failureView, 0, len(failures)) @@ -1113,6 +1155,7 @@ type clusterView struct { DefaultCapacityProviderStrategy []cpStrategyItemInput `json:"defaultCapacityProviderStrategy"` Settings []clusterSettingView `json:"settings,omitempty"` CapacityProviders []string `json:"capacityProviders"` + Tags []Tag `json:"tags,omitempty"` CreatedAt float64 `json:"createdAt"` ActiveServicesCount int `json:"activeServicesCount"` PendingTasksCount int `json:"pendingTasksCount"` diff --git a/services/ecs/handler_cluster_tags_test.go b/services/ecs/handler_cluster_tags_test.go new file mode 100644 index 000000000..519806913 --- /dev/null +++ b/services/ecs/handler_cluster_tags_test.go @@ -0,0 +1,102 @@ +package ecs_test + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestECS_CreateCluster_TagsAndCapacityProviders proves that CreateCluster +// accepts capacityProviders, defaultCapacityProviderStrategy, and tags at +// creation time, matching the real ecs.CreateClusterInput wire shape (all +// three are real request fields; previously the handler silently dropped +// them, forcing a separate PutClusterCapacityProviders/TagResource call that +// Terraform-style "create with tags" flows never make). +func TestECS_CreateCluster_TagsAndCapacityProviders(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + wantCapacityProvider string + wantTagValue string + }{ + { + name: "fargate default strategy and env tag", + wantCapacityProvider: "FARGATE", + wantTagValue: "prod", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + rec := doECSRequest(t, h, "CreateCluster", map[string]any{ + "clusterName": "tagged-cluster", + "capacityProviders": []string{tt.wantCapacityProvider}, + "defaultCapacityProviderStrategy": []map[string]any{ + {"capacityProvider": tt.wantCapacityProvider, "weight": 1}, + }, + "tags": []map[string]string{ + {"key": "env", "value": tt.wantTagValue}, + }, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var createResp map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &createResp)) + + cluster := createResp["cluster"].(map[string]any) + + caps, ok := cluster["capacityProviders"].([]any) + require.True(t, ok) + require.Len(t, caps, 1) + assert.Equal(t, tt.wantCapacityProvider, caps[0]) + + strategy, ok := cluster["defaultCapacityProviderStrategy"].([]any) + require.True(t, ok) + require.Len(t, strategy, 1) + assert.Equal(t, tt.wantCapacityProvider, strategy[0].(map[string]any)["capacityProvider"]) + + // CreateCluster echoes back the tags it was just given (no + // `include` gating on create). + tags, ok := cluster["tags"].([]any) + require.True(t, ok) + require.Len(t, tags, 1) + assert.Equal(t, tt.wantTagValue, tags[0].(map[string]any)["value"]) + + // DescribeClusters omits tags unless include=["TAGS"] is set. + rec = doECSRequest(t, h, "DescribeClusters", map[string]any{ + "clusters": []string{"tagged-cluster"}, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var describeResp map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &describeResp)) + + described := describeResp["clusters"].([]any)[0].(map[string]any) + assert.Nil(t, described["tags"], "tags must be omitted when include=TAGS is not requested") + + // With include=["TAGS"], DescribeClusters must surface them. + rec = doECSRequest(t, h, "DescribeClusters", map[string]any{ + "clusters": []string{"tagged-cluster"}, + "include": []string{"TAGS"}, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var describeTaggedResp map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &describeTaggedResp)) + + describedTagged := describeTaggedResp["clusters"].([]any)[0].(map[string]any) + taggedTags, ok := describedTagged["tags"].([]any) + require.True(t, ok) + require.Len(t, taggedTags, 1) + assert.Equal(t, tt.wantTagValue, taggedTags[0].(map[string]any)["value"]) + }) + } +} diff --git a/services/ecs/handler_refinement1.go b/services/ecs/handler_refinement1.go index fd0ab8880..56cd9555c 100644 --- a/services/ecs/handler_refinement1.go +++ b/services/ecs/handler_refinement1.go @@ -288,3 +288,27 @@ func (h *Handler) handleStopServiceDeployment( return &stopServiceDeploymentOutput{ServiceDeployment: toServiceDeploymentView(*sd)}, nil } + +// ----- ContinueServiceDeployment ----- + +type continueServiceDeploymentInput struct { + HookID string `json:"hookId"` + ServiceDeploymentArn string `json:"serviceDeploymentArn"` + Action string `json:"action,omitempty"` +} + +type continueServiceDeploymentOutput struct { + ServiceDeploymentArn string `json:"serviceDeploymentArn"` +} + +func (h *Handler) handleContinueServiceDeployment( + _ context.Context, + in *continueServiceDeploymentInput, +) (*continueServiceDeploymentOutput, error) { + sd, err := h.Backend.ContinueServiceDeployment(in.ServiceDeploymentArn, in.HookID, in.Action) + if err != nil { + return nil, err + } + + return &continueServiceDeploymentOutput{ServiceDeploymentArn: sd.ServiceDeploymentArn}, nil +} diff --git a/services/ecs/handler_refinement2_test.go b/services/ecs/handler_refinement2_test.go index c6ba67be3..064e4dacb 100644 --- a/services/ecs/handler_refinement2_test.go +++ b/services/ecs/handler_refinement2_test.go @@ -428,6 +428,76 @@ func TestECS_StopServiceDeployment(t *testing.T) { } } +// TestECS_ContinueServiceDeployment verifies real-vs-honest-error behavior: +// this backend never pauses a deployment at a lifecycle hook (blue/green +// PAUSE stages aren't modeled), so ContinueServiceDeployment must validate +// the deployment/hookId inputs for real and report ClientException rather +// than fabricating a successful continue/rollback. +func TestECS_ContinueServiceDeployment(t *testing.T) { + t.Parallel() + + const arn = "arn:aws:ecs:us-east-1:000000000000:service-deployment/test-cluster/my-service/deploy-1" + + tests := []struct { + input map[string]any + setup func(h *ecs.Handler) + name string + wantTypeIsIn []string + wantCode int + }{ + { + name: "existing deployment, no paused hook", + setup: func(h *ecs.Handler) { + now := time.Now() + b, ok := h.Backend.(*ecs.InMemoryBackend) + if !ok { + return + } + b.AddServiceDeploymentInternal(&ecs.ServiceDeployment{ + ServiceDeploymentArn: arn, + ClusterArn: "arn:aws:ecs:us-east-1:000000000000:cluster/test-cluster", + ServiceArn: "arn:aws:ecs:us-east-1:000000000000:service/test-cluster/my-service", + Status: "IN_PROGRESS", + CreatedAt: &now, + }) + }, + input: map[string]any{"serviceDeploymentArn": arn, "hookId": "hook-1"}, + wantCode: http.StatusBadRequest, + wantTypeIsIn: []string{"ClientException"}, + }, + { + name: "deployment not found", + input: map[string]any{"serviceDeploymentArn": "arn:not-exist", "hookId": "hook-1"}, + wantCode: http.StatusBadRequest, + wantTypeIsIn: []string{"ServiceDeploymentNotFoundException"}, + }, + { + name: "missing hookId", + input: map[string]any{"serviceDeploymentArn": arn}, + wantCode: http.StatusBadRequest, + wantTypeIsIn: []string{"InvalidParameterException"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + if tt.setup != nil { + tt.setup(h) + } + + rec := doECSRequest(t, h, "ContinueServiceDeployment", tt.input) + require.Equal(t, tt.wantCode, rec.Code) + + var resp map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + assert.Contains(t, tt.wantTypeIsIn, resp["__type"]) + }) + } +} + // TestECS_RunTask_StartedBy verifies startedBy, platformVersion and tags are stored and returned. func TestECS_RunTask_StartedBy(t *testing.T) { t.Parallel() diff --git a/services/ecs/handler_service_deployments_wiring_test.go b/services/ecs/handler_service_deployments_wiring_test.go new file mode 100644 index 000000000..dfb817e17 --- /dev/null +++ b/services/ecs/handler_service_deployments_wiring_test.go @@ -0,0 +1,157 @@ +package ecs_test + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestECS_ServiceDeployments_RealDeploymentsAreVisible proves that +// CreateService/UpdateService now record a real ServiceDeployment for every +// deployment they create on a service, so ListServiceDeployments and +// DescribeServiceDeployments — which filter/read the same backing map — see +// live data. Previously nothing but the AddServiceDeploymentInternal test +// seed helper ever populated that map, so a real client following the +// documented CreateService -> ListServiceDeployments -> DescribeServiceDeployments +// workflow always got an empty result, even though the service had an active +// deployment (see parity-principles.md rule 4: a "real-looking" op filtering +// a never-populated map is a disguised stub). +func TestECS_ServiceDeployments_RealDeploymentsAreVisible(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + updateTaskDef bool + wantDeploymentMin int + }{ + { + name: "create only, one PRIMARY deployment", + updateTaskDef: false, + wantDeploymentMin: 1, + }, + { + name: "create then update rotates in a second deployment", + updateTaskDef: true, + wantDeploymentMin: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + doECSRequest(t, h, "CreateCluster", map[string]any{"clusterName": "wiring-cluster"}) + doECSRequest(t, h, "RegisterTaskDefinition", map[string]any{ + "family": "wiring-app", + "containerDefinitions": []map[string]any{ + {"name": "app", "image": "nginx:1"}, + }, + }) + + doECSRequest(t, h, "CreateService", map[string]any{ + "cluster": "wiring-cluster", + "serviceName": "wiring-svc", + "taskDefinition": "wiring-app", + "desiredCount": 0, + }) + + if tt.updateTaskDef { + doECSRequest(t, h, "RegisterTaskDefinition", map[string]any{ + "family": "wiring-app", + "containerDefinitions": []map[string]any{ + {"name": "app", "image": "nginx:2"}, + }, + }) + doECSRequest(t, h, "UpdateService", map[string]any{ + "cluster": "wiring-cluster", + "service": "wiring-svc", + "taskDefinition": "wiring-app", + }) + } + + rec := doECSRequest(t, h, "ListServiceDeployments", map[string]any{ + "cluster": "wiring-cluster", + "service": "wiring-svc", + }) + require.Equal(t, http.StatusOK, rec.Code) + + var listResp map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &listResp)) + + arns, ok := listResp["serviceDeploymentArns"].([]any) + require.True(t, ok) + require.GreaterOrEqualf(t, len(arns), tt.wantDeploymentMin, + "ListServiceDeployments returned %d ARNs, want at least %d", len(arns), tt.wantDeploymentMin) + + firstArn, ok := arns[0].(string) + require.True(t, ok) + + rec = doECSRequest(t, h, "DescribeServiceDeployments", map[string]any{ + "serviceDeploymentArns": []string{firstArn}, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var describeResp map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &describeResp)) + + deployments, ok := describeResp["serviceDeployments"].([]any) + require.True(t, ok) + require.Len(t, deployments, 1) + + failures, ok := describeResp["failures"].([]any) + require.True(t, ok) + assert.Empty(t, failures) + + sd := deployments[0].(map[string]any) + assert.Equal(t, firstArn, sd["serviceDeploymentArn"]) + assert.NotEmpty(t, sd["status"]) + }) + } +} + +// TestECS_ServiceDeployments_DeletedOnServiceDelete proves DeleteService now +// cleans up its ServiceDeployment records instead of leaking one entry per +// service name ever created and deleted. +func TestECS_ServiceDeployments_DeletedOnServiceDelete(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + doECSRequest(t, h, "CreateCluster", map[string]any{"clusterName": "cleanup-cluster"}) + doECSRequest(t, h, "RegisterTaskDefinition", map[string]any{ + "family": "cleanup-app", + "containerDefinitions": []map[string]any{ + {"name": "app", "image": "nginx:1"}, + }, + }) + doECSRequest(t, h, "CreateService", map[string]any{ + "cluster": "cleanup-cluster", + "serviceName": "cleanup-svc", + "taskDefinition": "cleanup-app", + "desiredCount": 0, + }) + + rec := doECSRequest(t, h, "DeleteService", map[string]any{ + "cluster": "cleanup-cluster", + "service": "cleanup-svc", + }) + require.Equal(t, http.StatusOK, rec.Code) + + rec = doECSRequest(t, h, "ListServiceDeployments", map[string]any{ + "cluster": "cleanup-cluster", + "service": "cleanup-svc", + }) + require.Equal(t, http.StatusOK, rec.Code) + + var listResp map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &listResp)) + + arns, ok := listResp["serviceDeploymentArns"].([]any) + require.True(t, ok) + assert.Empty(t, arns, "ServiceDeployment entries must be removed when their service is deleted") +} diff --git a/services/ecs/janitor.go b/services/ecs/janitor.go index 530f96238..7b67b87a3 100644 --- a/services/ecs/janitor.go +++ b/services/ecs/janitor.go @@ -67,7 +67,7 @@ func (j *Janitor) sweepStoppedTasks(ctx context.Context) { evicted := 0 - for _, taskMap := range b.tasks { + for _, task := range b.tasks.All() { select { case <-ctx.Done(): b.mu.Unlock() @@ -76,18 +76,16 @@ func (j *Janitor) sweepStoppedTasks(ctx context.Context) { default: } - for arn, task := range taskMap { - if task.LastStatus != statusStopped { - continue - } - - if task.StoppedAt == nil || task.StoppedAt.IsZero() || task.StoppedAt.After(cutoff) { - continue - } + if task.LastStatus != statusStopped { + continue + } - delete(taskMap, arn) - evicted++ + if task.StoppedAt == nil || task.StoppedAt.IsZero() || task.StoppedAt.After(cutoff) { + continue } + + b.tasks.Delete(task.TaskArn) + evicted++ } b.mu.Unlock() diff --git a/services/ecs/lifecycle.go b/services/ecs/lifecycle.go index 9b4294363..4fec0d83a 100644 --- a/services/ecs/lifecycle.go +++ b/services/ecs/lifecycle.go @@ -78,7 +78,7 @@ func (b *InMemoryBackend) stepTaskLifecycle(now time.Time) { continue } - task := b.lookupTaskLocked(lc.clusterName, arn) + task := b.lookupTaskLocked(arn) if task == nil { // Task was deleted (e.g. janitor swept it or cluster removed). delete(b.lifecycle, arn) @@ -100,14 +100,12 @@ func (b *InMemoryBackend) stepTaskLifecycle(now time.Time) { } } -// lookupTaskLocked returns the task pointer for a cluster/ARN pair, or nil. -// Must be called with at least the read lock held. -func (b *InMemoryBackend) lookupTaskLocked(clusterName, taskArn string) *Task { - if tasks, ok := b.tasks[clusterName]; ok { - return tasks[taskArn] - } +// lookupTaskLocked returns the task pointer for taskArn, or nil. Must be +// called with at least the read lock held. +func (b *InMemoryBackend) lookupTaskLocked(taskArn string) *Task { + t, _ := b.tasks.Get(taskArn) - return nil + return t } // advanceLifecycleLocked moves a single task to its next observable state and @@ -173,7 +171,7 @@ func (b *InMemoryBackend) advanceStopLocked( } b.unindexTaskFromInstance(lc.clusterName, task.ContainerInstanceArn, task.TaskArn) - delete(b.taskProtections, task.TaskArn) + b.taskProtections.Delete(task.TaskArn) return true } @@ -192,7 +190,7 @@ func (b *InMemoryBackend) advanceStartLocked(task *Task, lc *taskLifecycle, now task.LastStatus = statusRunning syncContainerStatuses(task, nil) - if c := b.clusters[lc.clusterName]; c != nil { + if c, _ := b.clusters.Get(lc.clusterName); c != nil { c.PendingTasksCount-- c.RunningTasksCount++ } diff --git a/services/ecs/lifecycle_internal_test.go b/services/ecs/lifecycle_internal_test.go index 4096d9a94..18fc342df 100644 --- a/services/ecs/lifecycle_internal_test.go +++ b/services/ecs/lifecycle_internal_test.go @@ -132,7 +132,9 @@ func TestStopTaskLifecycle_CounterDecrementsOnce(t *testing.T) { b.mu.RLock("test") defer b.mu.RUnlock() - return b.clusters["lc"].RunningTasksCount + c, _ := b.clusters.Get("lc") + + return c.RunningTasksCount } if got := clusterRunning(); got != 1 { @@ -199,8 +201,9 @@ func TestStartTaskLifecycle_ObservableIntermediateStates(t *testing.T) { } b.mu.RLock("test") - running := b.clusters["lc"].RunningTasksCount - pending := b.clusters["lc"].PendingTasksCount + c, _ := b.clusters.Get("lc") + running := c.RunningTasksCount + pending := c.PendingTasksCount _, tracked := b.lifecycle[arn] b.mu.RUnlock() diff --git a/services/ecs/persistence.go b/services/ecs/persistence.go index f579fbe4a..16ad4b0e8 100644 --- a/services/ecs/persistence.go +++ b/services/ecs/persistence.go @@ -2,7 +2,10 @@ package ecs import ( "context" + "encoding/json" + "fmt" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" ) @@ -13,30 +16,54 @@ type Snapshottable interface { Restore(context.Context, []byte) error } +// ecsSnapshotVersion identifies the shape of backendSnapshot's Tables blob +// (i.e. the set/shape of resources registered on b.registry -- see +// registerAllTables in store_setup.go). It must be bumped whenever a change +// there would make an older snapshot unsafe to decode as the current shape. +// Restore compares this against the persisted value and discards (rather than +// attempts to partially decode) any mismatch -- see Restore below. This +// mirrors the ec2 (commit 12e611a4) and sqs (commit 0f09d77c) Phase 3.3 +// conversions. +const ecsSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the ECS backend. +// +// Tables holds one JSON-encoded array per registry-registered store.Table +// (clusters, services, tasks, containerInstances, taskSets, capacityProviders, +// accountSettings, taskProtections, serviceDeployments, expressGatewayServices, +// daemons, daemonRevisions, daemonDeployments -- see registerAllTables), +// produced by store.Registry.SnapshotAll(). The nested cluster/service-scoped +// resources (services, tasks, containerInstances, taskSets, daemons) are +// stored flatly, keyed by their store.Table composite primary key, rather than +// as nested JSON objects the way the pre-conversion map[string]map[string]*T +// fields were -- Version guards against decoding an older snapshot (with the +// old nested shape) as though it were this shape. +// +// The remaining fields are resources deliberately left as plain maps by the +// conversion (see the exclusion list in registerAllTables' doc comment) and +// so are still serialised directly, exactly as before. type backendSnapshot struct { - Clusters map[string]*Cluster `json:"clusters"` - TaskDefinitions map[string][]*TaskDefinition `json:"taskDefinitions"` - Services map[string]map[string]*Service `json:"services"` - Tasks map[string]map[string]*Task `json:"tasks"` - ContainerInstances map[string]map[string]*ContainerInstance `json:"containerInstances"` - TaskSets map[string]map[string]*TaskSet `json:"taskSets"` - CapacityProviders map[string]*CapacityProvider `json:"capacityProviders"` - AccountSettings map[string]*AccountSetting `json:"accountSettings"` - Attributes map[string]map[string]*Attribute `json:"attributes"` - ServiceDeployments map[string]*ServiceDeployment `json:"serviceDeployments"` - ExpressGatewayServices map[string]*ExpressGatewayService `json:"expressGatewayServices"` - TaskProtections map[string]*TaskProtection `json:"taskProtections"` - Daemons map[string]*Daemon `json:"daemons"` - DaemonTaskDefinitions map[string][]*DaemonTaskDefinition `json:"daemonTaskDefinitions"` - DaemonDeployments map[string]*DaemonDeployment `json:"daemonDeployments"` - DaemonRevisions map[string]*DaemonRevision `json:"daemonRevisions"` + Tables map[string]json.RawMessage `json:"tables"` + TaskDefinitions map[string][]*TaskDefinition `json:"taskDefinitions"` + Attributes map[string]map[string]*Attribute `json:"attributes"` + DaemonTaskDefinitions map[string][]*DaemonTaskDefinition `json:"daemonTaskDefinitions"` + // ResourceTags holds tags applied via TagResource/UntagResource, keyed by + // resourceTagKey(resourceArn). Clusters and Services carry their tags inline + // on their own struct, but task definitions and daemon task definitions are + // tagged only through this side map (see TagResource/ListTagsForResource in + // backend_refinement1.go), so it must be persisted like any other resource + // state or tags silently vanish across a snapshot/restore cycle. + ResourceTags map[string][]Tag `json:"resourceTags"` + Version int `json:"version"` } -func snapshotClusters(src map[string]*Cluster) map[string]*Cluster { - dst := make(map[string]*Cluster, len(src)) +// snapshotResourceTags deep-copies the TagResource side map (see the +// backendSnapshot.ResourceTags doc comment for why this must be persisted +// alongside the primary resource maps). +func snapshotResourceTags(src map[string][]Tag) map[string][]Tag { + dst := make(map[string][]Tag, len(src)) for k, v := range src { - cp := *v - dst[k] = &cp + dst[k] = copyTags(v) } return dst @@ -56,70 +83,15 @@ func snapshotTaskDefinitions(src map[string][]*TaskDefinition) map[string][]*Tas return dst } -func snapshotServices(src map[string]map[string]*Service) map[string]map[string]*Service { - dst := make(map[string]map[string]*Service, len(src)) - for cluster, svcMap := range src { - cp := make(map[string]*Service, len(svcMap)) - for name, svc := range svcMap { - s := *svc - cp[name] = &s - } - dst[cluster] = cp - } - - return dst -} - -func snapshotTasks(src map[string]map[string]*Task) map[string]map[string]*Task { - dst := make(map[string]map[string]*Task, len(src)) - for cluster, taskMap := range src { - cp := make(map[string]*Task, len(taskMap)) - for arn, task := range taskMap { - t := *task - cp[arn] = &t - } - dst[cluster] = cp - } - - return dst -} - -func snapshotContainerInstances( - src map[string]map[string]*ContainerInstance, -) map[string]map[string]*ContainerInstance { - dst := make(map[string]map[string]*ContainerInstance, len(src)) - for cluster, ciMap := range src { - cp := make(map[string]*ContainerInstance, len(ciMap)) - for arn, ci := range ciMap { - c := *ci - cp[arn] = &c - } - dst[cluster] = cp - } - - return dst -} - -func snapshotTaskSets(src map[string]map[string]*TaskSet) map[string]map[string]*TaskSet { - dst := make(map[string]map[string]*TaskSet, len(src)) - for key, tsMap := range src { - cp := make(map[string]*TaskSet, len(tsMap)) - for arn, ts := range tsMap { - t := *ts - cp[arn] = &t +func snapshotDaemonTaskDefinitions(src map[string][]*DaemonTaskDefinition) map[string][]*DaemonTaskDefinition { + dst := make(map[string][]*DaemonTaskDefinition, len(src)) + for family, revs := range src { + revsCp := make([]*DaemonTaskDefinition, len(revs)) + for i, td := range revs { + cp := *td + revsCp[i] = &cp } - dst[key] = cp - } - - return dst -} - -func snapshotCapacityProviders(src map[string]*CapacityProvider) map[string]*CapacityProvider { - dst := make(map[string]*CapacityProvider, len(src)) - for k, v := range src { - cp := *v - cp.Tags = copyTags(v.Tags) - dst[k] = &cp + dst[family] = revsCp } return dst @@ -139,135 +111,29 @@ func snapshotAttributes(src map[string]map[string]*Attribute) map[string]map[str return dst } -func snapshotTaskProtections(src map[string]*TaskProtection) map[string]*TaskProtection { - dst := make(map[string]*TaskProtection, len(src)) - for k, v := range src { - cp := *v - dst[k] = &cp - } - - return dst -} - -func snapshotExpressGatewayServices( - src map[string]*ExpressGatewayService, -) map[string]*ExpressGatewayService { - dst := make(map[string]*ExpressGatewayService, len(src)) - for k, v := range src { - cp := *v - cp.Tags = copyTags(v.Tags) - dst[k] = &cp - } - - return dst -} - -// snapshotDaemons flattens the cluster-nested in-memory daemon index (see -// InMemoryBackend.daemons in backend.go, keyed by clusterName then daemonName) -// into the flat ARN-keyed shape persisted on disk, preserving the existing -// "daemons" snapshot field shape. restoreDaemons performs the inverse. -func snapshotDaemons(src map[string]map[string]*Daemon) map[string]*Daemon { - dst := make(map[string]*Daemon) - for _, clusterDaemons := range src { - for _, v := range clusterDaemons { - cp := *v - cp.Tags = copyTags(v.Tags) - dst[cp.DaemonArn] = &cp - } - } - - return dst -} - -// restoreDaemons re-nests a flat ARN-keyed daemon snapshot (see snapshotDaemons) -// back into the clusterName -> daemonName -> Daemon index used at runtime. -func restoreDaemons(src map[string]*Daemon) map[string]map[string]*Daemon { - dst := make(map[string]map[string]*Daemon) - - for arn, d := range src { - clusterName, daemonName, ok := parseDaemonArn(arn) - if !ok { - continue - } - - if dst[clusterName] == nil { - dst[clusterName] = make(map[string]*Daemon) - } - - dst[clusterName][daemonName] = d - } - - return dst -} - -func snapshotDaemonTaskDefinitions(src map[string][]*DaemonTaskDefinition) map[string][]*DaemonTaskDefinition { - dst := make(map[string][]*DaemonTaskDefinition, len(src)) - for family, revs := range src { - revsCp := make([]*DaemonTaskDefinition, len(revs)) - for i, td := range revs { - cp := *td - revsCp[i] = &cp - } - dst[family] = revsCp - } - - return dst -} - -func snapshotDaemonDeployments(src map[string]*DaemonDeployment) map[string]*DaemonDeployment { - dst := make(map[string]*DaemonDeployment, len(src)) - for k, v := range src { - cp := *v - dst[k] = &cp - } - - return dst -} - -func snapshotDaemonRevisions(src map[string]*DaemonRevision) map[string]*DaemonRevision { - dst := make(map[string]*DaemonRevision, len(src)) - for k, v := range src { - cp := *v - dst[k] = &cp - } - - return dst -} - // Snapshot serialises the backend state to JSON. func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() - accountSettings := make(map[string]*AccountSetting, len(b.accountSettings)) - for k, v := range b.accountSettings { - cp := *v - accountSettings[k] = &cp - } + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "ecs: snapshot table marshal failed", "error", err) - serviceDeployments := make(map[string]*ServiceDeployment, len(b.serviceDeployments)) - for k, v := range b.serviceDeployments { - cp := *v - serviceDeployments[k] = &cp + return nil } snap := backendSnapshot{ - Clusters: snapshotClusters(b.clusters), - TaskDefinitions: snapshotTaskDefinitions(b.taskDefinitions), - Services: snapshotServices(b.services), - Tasks: snapshotTasks(b.tasks), - ContainerInstances: snapshotContainerInstances(b.containerInstances), - TaskSets: snapshotTaskSets(b.taskSets), - CapacityProviders: snapshotCapacityProviders(b.capacityProviders), - AccountSettings: accountSettings, - Attributes: snapshotAttributes(b.attributes), - ServiceDeployments: serviceDeployments, - ExpressGatewayServices: snapshotExpressGatewayServices(b.expressGatewayServices), - TaskProtections: snapshotTaskProtections(b.taskProtections), - Daemons: snapshotDaemons(b.daemons), - DaemonTaskDefinitions: snapshotDaemonTaskDefinitions(b.daemonTaskDefinitions), - DaemonDeployments: snapshotDaemonDeployments(b.daemonDeployments), - DaemonRevisions: snapshotDaemonRevisions(b.daemonRevisions), + Version: ecsSnapshotVersion, + Tables: tables, + TaskDefinitions: snapshotTaskDefinitions(b.taskDefinitions), + Attributes: snapshotAttributes(b.attributes), + DaemonTaskDefinitions: snapshotDaemonTaskDefinitions(b.daemonTaskDefinitions), + ResourceTags: snapshotResourceTags(b.resourceTags), } return persistence.MarshalSnapshot(ctx, "ecs", snap) @@ -276,75 +142,20 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { // initSnapshotDefaults ensures all maps in the snapshot are non-nil so callers // can safely assign them to the backend without nil-map panics. func initSnapshotDefaults(snap *backendSnapshot) { - if snap.Clusters == nil { - snap.Clusters = make(map[string]*Cluster) - } - if snap.TaskDefinitions == nil { snap.TaskDefinitions = make(map[string][]*TaskDefinition) } - if snap.Services == nil { - snap.Services = make(map[string]map[string]*Service) - } - - if snap.Tasks == nil { - snap.Tasks = make(map[string]map[string]*Task) - } - - if snap.ContainerInstances == nil { - snap.ContainerInstances = make(map[string]map[string]*ContainerInstance) - } - - if snap.TaskSets == nil { - snap.TaskSets = make(map[string]map[string]*TaskSet) - } - - if snap.CapacityProviders == nil { - snap.CapacityProviders = make(map[string]*CapacityProvider) - } - - if snap.AccountSettings == nil { - snap.AccountSettings = make(map[string]*AccountSetting) - } - if snap.Attributes == nil { snap.Attributes = make(map[string]map[string]*Attribute) } - if snap.ServiceDeployments == nil { - snap.ServiceDeployments = make(map[string]*ServiceDeployment) - } - - if snap.ExpressGatewayServices == nil { - snap.ExpressGatewayServices = make(map[string]*ExpressGatewayService) - } - - if snap.TaskProtections == nil { - snap.TaskProtections = make(map[string]*TaskProtection) - } - - initDaemonSnapshotDefaults(snap) -} - -// initDaemonSnapshotDefaults ensures the daemon-related maps in the snapshot -// are non-nil. Split out from initSnapshotDefaults to keep cyclomatic -// complexity within limits. -func initDaemonSnapshotDefaults(snap *backendSnapshot) { - if snap.Daemons == nil { - snap.Daemons = make(map[string]*Daemon) - } - if snap.DaemonTaskDefinitions == nil { snap.DaemonTaskDefinitions = make(map[string][]*DaemonTaskDefinition) } - if snap.DaemonDeployments == nil { - snap.DaemonDeployments = make(map[string]*DaemonDeployment) - } - - if snap.DaemonRevisions == nil { - snap.DaemonRevisions = make(map[string]*DaemonRevision) + if snap.ResourceTags == nil { + snap.ResourceTags = make(map[string][]Tag) } } @@ -355,44 +166,81 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { return err } + initSnapshotDefaults(&snap) + b.mu.Lock("Restore") defer b.mu.Unlock() - initSnapshotDefaults(&snap) + if snap.Version != ecsSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields (e.g. the pre-conversion nested + // map[string]map[string]*T shape for services/tasks/containerInstances/ + // taskSets/daemons vs. the flat store.Table composite-key shape). + // Discard cleanly and start empty instead of erroring, since this is an + // expected, recoverable condition (e.g. upgrading gopherstack across a + // snapshot-format change), not data corruption. Mirrors the ec2/sqs + // Phase 3.3 conversions. + logger.Load(ctx).WarnContext(ctx, + "ecs: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", ecsSnapshotVersion) + + b.registry.ResetAll() + b.taskDefByArn.Reset() + b.daemonTaskDefByArn.Reset() + + return nil + } + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("ecs: restore snapshot tables: %w", err) + } - b.clusters = snap.Clusters b.taskDefinitions = snap.TaskDefinitions - b.services = snap.Services - b.tasks = snap.Tasks - b.containerInstances = snap.ContainerInstances - b.taskSets = snap.TaskSets - b.capacityProviders = snap.CapacityProviders - b.accountSettings = snap.AccountSettings b.attributes = snap.Attributes - b.serviceDeployments = snap.ServiceDeployments - b.expressGatewayServices = snap.ExpressGatewayServices - b.taskProtections = snap.TaskProtections - b.daemons = restoreDaemons(snap.Daemons) b.daemonTaskDefinitions = snap.DaemonTaskDefinitions - b.daemonDeployments = snap.DaemonDeployments - b.daemonRevisions = snap.DaemonRevisions + b.resourceTags = snap.ResourceTags // Rebuild the ARN→TaskDefinition cache from restored task definitions. - b.taskDefByArn = make(map[string]*TaskDefinition) + // Not part of the registry-driven Tables blob: it is a derived cache, not + // independently persisted state (see taskDefByArnKeyFn's doc comment). + b.taskDefByArn.Reset() for _, revs := range b.taskDefinitions { for _, td := range revs { - b.taskDefByArn[td.TaskDefinitionArn] = td + b.taskDefByArn.Put(td) } } // Rebuild the ARN→DaemonTaskDefinition cache from restored daemon task definitions. - b.daemonTaskDefByArn = make(map[string]*DaemonTaskDefinition) + b.daemonTaskDefByArn.Reset() for _, revs := range b.daemonTaskDefinitions { for _, td := range revs { - b.daemonTaskDefByArn[td.DaemonTaskDefinitionArn] = td + b.daemonTaskDefByArn.Put(td) } } + // Rebuild the flat serviceIndex from the restored services table. Unlike + // tasksByInstance (which enrichContainerInstance falls back to a linear scan + // for), getServicesForReconciler reads serviceIndex with no fallback: leaving + // it empty after a restore would silently stop the deployment reconciler + // from ever seeing pre-existing services again (deployments/scaling would + // freeze forever post-restart). + allServices := b.services.All() + b.serviceIndex = make(map[svcRef]bool, len(allServices)) + + for _, svc := range allServices { + b.serviceIndex[svcRef{cluster: clusterKey(svc.ClusterArn), name: svc.ServiceName}] = true + } + + // Rebuild the containerInstance→task reverse index from the restored tasks + // table. enrichContainerInstance already falls back to a linear scan when + // this index is empty, but rebuilding it here keeps post-restore behavior on + // the fast O(k) path instead of silently degrading to O(n) for every describe. + b.tasksByInstance = make(map[string]map[string]map[string]bool, len(allServices)) + for _, task := range b.tasks.All() { + b.indexTaskOnInstance(clusterKey(task.ClusterArn), task.ContainerInstanceArn, task.TaskArn) + } + return nil } diff --git a/services/ecs/persistence_internal_test.go b/services/ecs/persistence_internal_test.go new file mode 100644 index 000000000..35726ebee --- /dev/null +++ b/services/ecs/persistence_internal_test.go @@ -0,0 +1,433 @@ +package ecs + +import ( + "testing" +) + +// Test_Snapshot_Restore_FullState is the Phase 3.3 (pkgs/store conversion) +// round-trip test: it seeds one instance of every resource kind the backend +// tracks -- both the store.Table-backed resources (clusters, services, tasks, +// container instances, task sets, capacity providers, account settings, task +// protections, service deployments, express gateway services, daemons, daemon +// revisions, daemon deployments) and the resources deliberately left as raw +// maps by the conversion (attributes, resourceTags, daemonTaskDefinitions) -- +// snapshots the backend, restores into a fresh one, and asserts every +// resource survived the round-trip with its identifying fields intact. +func Test_Snapshot_Restore_FullState(t *testing.T) { + t.Parallel() + + src := NewInMemoryBackend("123456789012", "us-east-1", NewNoopRunner()) + + tdArn := registerSimpleTaskDef(t, src, "full-state-app", "nginx:good") + + _, err := src.CreateCluster(CreateClusterInput{ClusterName: "full-state-cluster"}) + if err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + svc, err := src.CreateService(CreateServiceInput{ + ServiceName: "full-state-svc", + Cluster: "full-state-cluster", + TaskDefinition: tdArn, + DesiredCount: 1, + }) + if err != nil { + t.Fatalf("CreateService: %v", err) + } + + tasks, err := src.RunTask(RunTaskInput{ + Cluster: "full-state-cluster", TaskDefinition: tdArn, Count: 1, + }) + if err != nil { + t.Fatalf("RunTask: %v", err) + } + + taskArn := tasks[0].TaskArn + + _, _, err = src.UpdateTaskProtection("full-state-cluster", []string{taskArn}, true, nil) + if err != nil { + t.Fatalf("UpdateTaskProtection: %v", err) + } + + ci, err := src.RegisterContainerInstance("full-state-cluster", "i-0123456789") + if err != nil { + t.Fatalf("RegisterContainerInstance: %v", err) + } + + ts, err := src.CreateTaskSet(CreateTaskSetInput{ + Cluster: "full-state-cluster", + Service: "full-state-svc", + TaskDefinition: tdArn, + }) + if err != nil { + t.Fatalf("CreateTaskSet: %v", err) + } + + _, err = src.CreateCapacityProvider(CreateCapacityProviderInput{Name: "full-state-cp"}) + if err != nil { + t.Fatalf("CreateCapacityProvider: %v", err) + } + + _, err = src.PutAccountSetting("containerInsights", "enabled", "") + if err != nil { + t.Fatalf("PutAccountSetting: %v", err) + } + + egs, err := src.CreateExpressGatewayService(CreateExpressGatewayServiceInput{ + ServiceName: "full-state-egs", + Cluster: "full-state-cluster", + ExecutionRoleArn: "arn:aws:iam::123456789012:role/exec", + InfrastructureRoleArn: "arn:aws:iam::123456789012:role/infra", + }) + if err != nil { + t.Fatalf("CreateExpressGatewayService: %v", err) + } + + daemonTD, err := src.RegisterDaemonTaskDefinition(RegisterDaemonTaskDefinitionInput{ + Family: "full-state-daemon-td", + ContainerDefinitions: []DaemonContainerDefinition{{Name: "agent", Image: "agent:good"}}, + }) + if err != nil { + t.Fatalf("RegisterDaemonTaskDefinition: %v", err) + } + + daemon, err := src.CreateDaemon(CreateDaemonInput{ + DaemonName: "full-state-daemon", + ClusterArn: "full-state-cluster", + DaemonTaskDefinitionArn: daemonTD.DaemonTaskDefinitionArn, + CapacityProviderArns: []string{"FARGATE"}, + }) + if err != nil { + t.Fatalf("CreateDaemon: %v", err) + } + + _, err = src.PutAttributes("full-state-cluster", []Attribute{ + {Name: "custom", Value: "value", TargetID: ci.ContainerInstanceArn}, + }) + if err != nil { + t.Fatalf("PutAttributes: %v", err) + } + + err = src.TagResource(tdArn, []Tag{{Key: "env", Value: "prod"}}) + if err != nil { + t.Fatalf("TagResource: %v", err) + } + + snap := src.Snapshot(t.Context()) + if len(snap) == 0 { + t.Fatalf("Snapshot returned empty data") + } + + dst := NewInMemoryBackend("123456789012", "us-east-1", NewNoopRunner()) + + err = dst.Restore(t.Context(), snap) + if err != nil { + t.Fatalf("Restore: %v", err) + } + + assertFullStateRestored(t, dst, fullStateFixture{ + taskArn: taskArn, + containerArn: ci.ContainerInstanceArn, + serviceArn: svc.ServiceArn, + taskSetArn: ts.TaskSetArn, + egsArn: egs.ServiceArn, + daemonArn: daemon.DaemonArn, + tdArn: tdArn, + }) +} + +// fullStateFixture carries the identifiers Test_Snapshot_Restore_FullState +// seeded on the source backend, so assertFullStateRestored can look each one +// up again on the restored backend. +type fullStateFixture struct { + taskArn string + containerArn string + serviceArn string + taskSetArn string + egsArn string + daemonArn string + tdArn string +} + +// assertFullStateRestored verifies every resource kind seeded by +// Test_Snapshot_Restore_FullState is present on the restored backend b. Split +// across a handful of helpers (rather than one large function) to keep each +// one's cyclomatic complexity down. +func assertFullStateRestored(t *testing.T, b *InMemoryBackend, f fullStateFixture) { + t.Helper() + + assertCoreResourcesRestored(t, b, f) + assertDaemonResourcesRestored(t, b, f) + assertRawMapResourcesRestored(t, b, f) +} + +// assertCoreResourcesRestored checks the store.Table-backed cluster/service/ +// task/container-instance/task-set/capacity-provider/account-setting/ +// express-gateway-service resources. +func assertCoreResourcesRestored(t *testing.T, b *InMemoryBackend, f fullStateFixture) { + t.Helper() + + clusters, _, err := b.DescribeClusters([]string{"full-state-cluster"}) + if err != nil || len(clusters) != 1 { + t.Fatalf("DescribeClusters: got %d clusters, err=%v, want 1 cluster", len(clusters), err) + } + + services, _, err := b.DescribeServices("full-state-cluster", []string{"full-state-svc"}) + if err != nil || len(services) != 1 { + t.Fatalf("DescribeServices: got %d services, err=%v, want 1 service", len(services), err) + } + if services[0].ServiceArn != f.serviceArn { + t.Errorf("restored service ARN = %q, want %q", services[0].ServiceArn, f.serviceArn) + } + + tasksOut, _, err := b.DescribeTasks("full-state-cluster", []string{f.taskArn}) + if err != nil || len(tasksOut) != 1 { + t.Fatalf("DescribeTasks: got %d tasks, err=%v, want 1 task", len(tasksOut), err) + } + + protections, _, err := b.GetTaskProtection("full-state-cluster", []string{f.taskArn}) + if err != nil || len(protections) != 1 || !protections[0].ProtectionEnabled { + t.Fatalf("GetTaskProtection: got %+v, err=%v, want one enabled protection", protections, err) + } + + instances, _, err := b.DescribeContainerInstances("full-state-cluster", []string{f.containerArn}) + if err != nil || len(instances) != 1 { + t.Fatalf("DescribeContainerInstances: got %d instances, err=%v, want 1 instance", len(instances), err) + } + + taskSets, err := b.DescribeTaskSets("full-state-cluster", "full-state-svc", []string{f.taskSetArn}) + if err != nil || len(taskSets) != 1 { + t.Fatalf("DescribeTaskSets: got %d task sets, err=%v, want 1 task set", len(taskSets), err) + } + + caps, err := b.DescribeCapacityProviders([]string{"full-state-cp"}) + if err != nil || len(caps) != 1 { + t.Fatalf("DescribeCapacityProviders: got %d providers, err=%v, want 1 provider", len(caps), err) + } + + settings, err := b.ListAccountSettings("containerInsights", "") + if err != nil || len(settings) != 1 { + t.Fatalf("ListAccountSettings: got %d settings, err=%v, want 1 setting", len(settings), err) + } + + egs, err := b.DescribeExpressGatewayService(f.egsArn) + if err != nil || egs == nil { + t.Fatalf("DescribeExpressGatewayService: %v, err=%v", egs, err) + } +} + +// assertDaemonResourcesRestored checks the daemon, daemon-revision, and +// daemon-deployment store.Table resources plus the raw daemonTaskDefinitions map. +func assertDaemonResourcesRestored(t *testing.T, b *InMemoryBackend, f fullStateFixture) { + t.Helper() + + daemon, err := b.DescribeDaemon(f.daemonArn) + if err != nil || daemon == nil { + t.Fatalf("DescribeDaemon: %v, err=%v", daemon, err) + } + + if daemon.CurrentDaemonRevisionArn == "" { + t.Error("restored daemon has no CurrentDaemonRevisionArn") + } else { + revs, failures, revErr := b.DescribeDaemonRevisions([]string{daemon.CurrentDaemonRevisionArn}) + if revErr != nil || len(revs) != 1 || len(failures) != 0 { + t.Errorf("DescribeDaemonRevisions: got %d revs, %d failures, err=%v, want 1 rev, 0 failures", + len(revs), len(failures), revErr) + } + } + + if daemon.DeploymentArn == "" { + t.Error("restored daemon has no DeploymentArn") + } else { + assertDaemonDeploymentRestored(t, b, daemon.DeploymentArn) + } + + daemonTDs, err := b.ListDaemonTaskDefinitions(ListDaemonTaskDefinitionsInput{Family: "full-state-daemon-td"}) + if err != nil || len(daemonTDs) != 1 { + t.Fatalf("ListDaemonTaskDefinitions: got %d, err=%v, want 1", len(daemonTDs), err) + } +} + +// assertDaemonDeploymentRestored checks a single daemon deployment ARN. +func assertDaemonDeploymentRestored(t *testing.T, b *InMemoryBackend, deploymentArn string) { + t.Helper() + + deps, failures, err := b.DescribeDaemonDeployments([]string{deploymentArn}) + if err != nil || len(deps) != 1 || len(failures) != 0 { + t.Errorf("DescribeDaemonDeployments: got %d deployments, %d failures, err=%v, want 1, 0", + len(deps), len(failures), err) + } +} + +// assertRawMapResourcesRestored checks the resources deliberately left as raw +// maps by the Phase 3.3 conversion: attributes and resourceTags. +func assertRawMapResourcesRestored(t *testing.T, b *InMemoryBackend, f fullStateFixture) { + t.Helper() + + attrs, err := b.ListAttributes("full-state-cluster", "", "custom", "") + if err != nil || len(attrs) != 1 { + t.Fatalf("ListAttributes: got %d attributes, err=%v, want 1 attribute", len(attrs), err) + } + + tags, err := b.ListTagsForResource(f.tdArn) + if err != nil || len(tags) != 1 || tags[0].Key != "env" || tags[0].Value != "prod" { + t.Fatalf("ListTagsForResource: got %+v, err=%v, want one env=prod tag", tags, err) + } +} + +// Test_Restore_RebuildsServiceIndex proves that after a Snapshot/Restore +// round-trip, the deployment reconciler still sees pre-existing services. +// +// getServicesForReconciler (backend.go) reads only the flat serviceIndex map +// with no linear-scan fallback (unlike tasksByInstance, which +// enrichContainerInstance falls back to scanning for). Restore previously +// loaded b.services from the snapshot but never rebuilt b.serviceIndex, so a +// restored service was permanently invisible to the reconciler: its +// DesiredCount would never be reconciled again after a restart, silently +// freezing scaling and deployments for every service that existed at +// snapshot time. +func Test_Restore_RebuildsServiceIndex(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + desiredCount int + wantTasks int + }{ + {name: "desired count 1 reconciles after restore", desiredCount: 1, wantTasks: 1}, + {name: "desired count 3 reconciles after restore", desiredCount: 3, wantTasks: 3}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + src := NewInMemoryBackend("123456789012", "us-east-1", NewNoopRunner()) + + tdArn := registerSimpleTaskDef(t, src, "restore-app-"+tt.name, "nginx:good") + + if _, err := src.CreateCluster(CreateClusterInput{ClusterName: "restore-cluster"}); err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + if _, err := src.CreateService(CreateServiceInput{ + ServiceName: "restore-svc", + Cluster: "restore-cluster", + TaskDefinition: tdArn, + DesiredCount: tt.desiredCount, + }); err != nil { + t.Fatalf("CreateService: %v", err) + } + + snap := src.Snapshot(t.Context()) + if len(snap) == 0 { + t.Fatalf("Snapshot returned empty data") + } + + dst := NewInMemoryBackend("123456789012", "us-east-1", NewNoopRunner()) + if err := dst.Restore(t.Context(), snap); err != nil { + t.Fatalf("Restore: %v", err) + } + + // No tasks exist yet on the restored backend: only the persisted + // Service (with its DesiredCount) survives the round-trip. + before, err := dst.ListTasksFiltered(ListTasksInput{ + Cluster: "restore-cluster", ServiceName: "restore-svc", + }) + if err != nil { + t.Fatalf("ListTasksFiltered (before): %v", err) + } + if len(before) != 0 { + t.Fatalf("task count before reconcile = %d, want 0", len(before)) + } + + r := NewReconciler(dst) + r.RunOnce(t.Context()) + + after, err := dst.ListTasksFiltered(ListTasksInput{ + Cluster: "restore-cluster", ServiceName: "restore-svc", + }) + if err != nil { + t.Fatalf("ListTasksFiltered (after): %v", err) + } + if len(after) != tt.wantTasks { + t.Errorf( + "task count after RunOnce = %d, want %d (reconciler did not see restored service)", + len(after), tt.wantTasks, + ) + } + }) + } +} + +// Test_Snapshot_Restore_PreservesResourceTags proves that tags applied via +// TagResource on resources tracked only in the resourceTags side map (task +// definitions and daemon task definitions — clusters/services carry Tags +// inline on their own struct) survive a Snapshot/Restore round-trip. +// resourceTags was previously absent from backendSnapshot entirely, so every +// such tag was silently dropped on restore. +func Test_Snapshot_Restore_PreservesResourceTags(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + tags []Tag + }{ + { + name: "single tag", + tags: []Tag{{Key: "env", Value: "prod"}}, + }, + { + name: "multiple tags", + tags: []Tag{{Key: "env", Value: "staging"}, {Key: "team", Value: "platform"}}, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + src := NewInMemoryBackend("123456789012", "us-east-1", NewNoopRunner()) + + td, err := src.RegisterTaskDefinition(RegisterTaskDefinitionInput{ + Family: "tag-app-" + tt.name, + ContainerDefinitions: []ContainerDefinition{{Name: "app", Image: "nginx:good"}}, + }) + if err != nil { + t.Fatalf("RegisterTaskDefinition: %v", err) + } + + if tagErr := src.TagResource(td.TaskDefinitionArn, tt.tags); tagErr != nil { + t.Fatalf("TagResource: %v", tagErr) + } + + snap := src.Snapshot(t.Context()) + + dst := NewInMemoryBackend("123456789012", "us-east-1", NewNoopRunner()) + if restoreErr := dst.Restore(t.Context(), snap); restoreErr != nil { + t.Fatalf("Restore: %v", restoreErr) + } + + got, err := dst.ListTagsForResource(td.TaskDefinitionArn) + if err != nil { + t.Fatalf("ListTagsForResource: %v", err) + } + + if len(got) != len(tt.tags) { + t.Fatalf("tag count after restore = %d, want %d (tags lost across restore)", + len(got), len(tt.tags)) + } + + gotSet := make(map[string]string, len(got)) + for _, tag := range got { + gotSet[tag.Key] = tag.Value + } + + for _, want := range tt.tags { + if gotSet[want.Key] != want.Value { + t.Errorf("tag %q = %q, want %q", want.Key, gotSet[want.Key], want.Value) + } + } + }) + } +} diff --git a/services/ecs/purge.go b/services/ecs/purge.go index 30ae4b358..2f87cb8be 100644 --- a/services/ecs/purge.go +++ b/services/ecs/purge.go @@ -42,9 +42,9 @@ func (b *InMemoryBackend) Purge(_ context.Context, cutoff time.Time) { func (b *InMemoryBackend) buildPurgePlanLocked(cutoff time.Time) purgePlan { plan := purgePlan{revByFamily: make(map[string][]string)} - for name, c := range b.clusters { + for _, c := range b.clusters.All() { if c.CreatedAt.Before(cutoff) { - plan.clusters = append(plan.clusters, name) + plan.clusters = append(plan.clusters, c.ClusterName) } } @@ -66,7 +66,7 @@ func (b *InMemoryBackend) applyPurgePlanLocked(plan purgePlan, cutoff time.Time) removed := make([]string, 0, len(plan.clusters)) for _, name := range plan.clusters { - c, ok := b.clusters[name] + c, ok := b.clusters.Get(name) if !ok || !c.CreatedAt.Before(cutoff) { // Cluster was removed or re-created between the plan and apply phases. continue @@ -88,31 +88,29 @@ func (b *InMemoryBackend) applyPurgePlanLocked(plan purgePlan, cutoff time.Time) // lifecycle, attribute, reverse-index, and daemon entries that older code paths // leaked. Must be called with the write lock held. func (b *InMemoryBackend) purgeClusterLocked(name string) { - if svcs, ok := b.services[name]; ok { - for svcName, svc := range svcs { - delete(b.serviceIndex, svcRef{cluster: name, name: svcName}) - delete(b.serviceDeployments, svc.ServiceArn) - delete(b.taskSets, svc.ServiceArn) - - for _, rev := range b.serviceRevisions[svc.ServiceArn] { - delete(b.serviceRevisionsByArn, rev.ServiceRevisionArn) - } + for _, svc := range b.servicesInClusterLocked(name) { + delete(b.serviceIndex, svcRef{cluster: name, name: svc.ServiceName}) + b.deleteServiceDeploymentsForServiceLocked(svc.ServiceArn) + b.deleteTaskSetsForServiceLocked(svc.ServiceArn) - delete(b.serviceRevisions, svc.ServiceArn) + for _, rev := range b.serviceRevisions[svc.ServiceArn] { + delete(b.serviceRevisionsByArn, rev.ServiceRevisionArn) } + + delete(b.serviceRevisions, svc.ServiceArn) } - for taskArn := range b.tasks[name] { - delete(b.taskProtections, taskArn) - delete(b.lifecycle, taskArn) + for _, task := range b.tasksInClusterLocked(name) { + b.taskProtections.Delete(task.TaskArn) + delete(b.lifecycle, task.TaskArn) } b.purgeDaemonsLocked(name) - delete(b.clusters, name) - delete(b.services, name) - delete(b.tasks, name) - delete(b.containerInstances, name) + b.clusters.Delete(name) + b.deleteServicesForClusterLocked(name) + b.deleteTasksForClusterLocked(name) + b.deleteContainerInstancesForClusterLocked(name) delete(b.attributes, name) delete(b.tasksByInstance, name) } @@ -121,8 +119,8 @@ func (b *InMemoryBackend) purgeClusterLocked(name string) { // definitions, revisions, and deployment records. Must be called with the write // lock held. func (b *InMemoryBackend) purgeDaemonsLocked(clusterName string) { - daemons, ok := b.daemons[clusterName] - if !ok { + daemons := b.daemonsInClusterLocked(clusterName) + if len(daemons) == 0 { return } @@ -131,16 +129,20 @@ func (b *InMemoryBackend) purgeDaemonsLocked(clusterName string) { for _, d := range daemons { daemonArns[d.DaemonArn] = true delete(b.daemonTaskDefs, d.DaemonArn) - delete(b.daemonRevisions, d.DaemonArn) + // NOTE: daemonRevisions is keyed by DaemonRevisionArn (see + // createDaemonRevisionLocked), not DaemonArn -- this delete never + // actually matches an entry. Preserved byte-for-byte from the + // pre-conversion map-based code rather than fixed, per the Phase 3.3 + // mechanical-conversion mandate. + b.daemonRevisions.Delete(d.DaemonArn) + b.daemons.Delete(daemonsKeyFn(d)) } - for id, dep := range b.daemonDeployments { + for _, dep := range b.daemonDeployments.All() { if daemonArns[dep.DaemonArn] { - delete(b.daemonDeployments, id) + b.daemonDeployments.Delete(dep.DaemonDeploymentArn) } } - - delete(b.daemons, clusterName) } // purgeTaskDefRevisionsLocked removes the given (already-identified) revisions @@ -163,7 +165,7 @@ func (b *InMemoryBackend) purgeTaskDefRevisionsLocked(family string, arns []stri for _, td := range revs { if remove[td.TaskDefinitionArn] && td.RegisteredAt.Before(cutoff) { - delete(b.taskDefByArn, td.TaskDefinitionArn) + b.taskDefByArn.Delete(td.TaskDefinitionArn) continue } diff --git a/services/ecs/purge_leak_internal_test.go b/services/ecs/purge_leak_internal_test.go index 7fb8247f7..2c261f30d 100644 --- a/services/ecs/purge_leak_internal_test.go +++ b/services/ecs/purge_leak_internal_test.go @@ -77,14 +77,12 @@ func TestPurge_CleansServiceAndDaemonState(t *testing.T) { daemonArn := "arn:aws:ecs:us-east-1:123456789012:daemon/pc/d1" b.mu.Lock("seed") - b.serviceDeployments[svc.ServiceArn] = &ServiceDeployment{ + b.serviceDeployments.Put(&ServiceDeployment{ ServiceDeploymentArn: "sd-1", ServiceArn: svc.ServiceArn, - } - b.daemons["pc"] = map[string]*Daemon{ - "d1": {DaemonArn: daemonArn, DaemonName: "d1", ClusterArn: svc.ClusterArn}, - } - b.daemonDeployments["dd-1"] = &DaemonDeployment{DaemonDeploymentArn: "dd-1", DaemonArn: daemonArn} + }) + b.daemons.Put(&Daemon{DaemonArn: daemonArn, DaemonName: "d1", ClusterArn: svc.ClusterArn}) + b.daemonDeployments.Put(&DaemonDeployment{DaemonDeploymentArn: "dd-1", DaemonArn: daemonArn}) b.daemonTaskDefs[daemonArn] = []*DaemonTaskDefinition{{DaemonTaskDefinitionArn: daemonArn}} b.mu.Unlock() @@ -97,17 +95,17 @@ func TestPurge_CleansServiceAndDaemonState(t *testing.T) { b.mu.RLock("verify") defer b.mu.RUnlock() - if len(b.daemons) != 0 { - t.Errorf("daemons after purge = %d, want 0", len(b.daemons)) + if got := b.daemons.Len(); got != 0 { + t.Errorf("daemons after purge = %d, want 0", got) } - if len(b.daemonDeployments) != 0 { - t.Errorf("daemonDeployments after purge = %d, want 0", len(b.daemonDeployments)) + if got := b.daemonDeployments.Len(); got != 0 { + t.Errorf("daemonDeployments after purge = %d, want 0", got) } if len(b.daemonTaskDefs) != 0 { t.Errorf("daemonTaskDefs after purge = %d, want 0", len(b.daemonTaskDefs)) } - if len(b.clusters) != 0 { - t.Errorf("clusters after purge = %d, want 0", len(b.clusters)) + if got := b.clusters.Len(); got != 0 { + t.Errorf("clusters after purge = %d, want 0", got) } if len(b.serviceIndex) != 0 { t.Errorf("serviceIndex after purge = %d, want 0", len(b.serviceIndex)) @@ -149,7 +147,7 @@ func TestPurge_RevisionCutoff(t *testing.T) { } // The aged revisions must also be gone from the ARN index. - if len(b.taskDefByArn) != 2 { - t.Errorf("taskDefByArn size after purge = %d, want 2", len(b.taskDefByArn)) + if got := b.taskDefByArn.Len(); got != 2 { + t.Errorf("taskDefByArn size after purge = %d, want 2", got) } } diff --git a/services/ecs/sdk_completeness_test.go b/services/ecs/sdk_completeness_test.go index 2cc35e73d..730627350 100644 --- a/services/ecs/sdk_completeness_test.go +++ b/services/ecs/sdk_completeness_test.go @@ -18,7 +18,5 @@ func TestSDKCompleteness(t *testing.T) { backend := ecs.NewInMemoryBackend("000000000000", "us-east-1", ecs.NewNoopRunner()) h := ecs.NewHandler(backend) - sdkcheck.CheckCompleteness(t, &ecssdk.Client{}, h.GetSupportedOperations(), []string{ - "ContinueServiceDeployment", - }) + sdkcheck.CheckCompleteness(t, &ecssdk.Client{}, h.GetSupportedOperations(), []string{}) } diff --git a/services/ecs/store_setup.go b/services/ecs/store_setup.go new file mode 100644 index 000000000..2463d55d8 --- /dev/null +++ b/services/ecs/store_setup.go @@ -0,0 +1,198 @@ +package ecs + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend that is a pure function of +// the stored value's own identity is registered exactly once, here, as a +// *store.Table[T] on b.registry. See pkgs/store's package doc and the ec2 +// (commit 12e611a4) and sqs (commit 0f09d77c) conversions this follows. +// +// Resources that used to be nested map[string]map[string]*T (cluster -> +// resource-name -> value) are flattened into a single Table keyed by a +// composite "cluster/name" string, with a secondary store.Index grouping by +// cluster so the old "all X in cluster Y" access pattern (b.tasks[cluster], +// b.services[cluster], ...) still resolves in O(k). +// +// A handful of fields are deliberately NOT registered here and remain plain +// maps -- see the comment above registerAllTables for the list and why. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +// scopedKey builds a composite store.Table key for a resource whose identity +// is only unique within an owning scope (a cluster name for services/ +// containerInstances/daemons, a service ARN for task sets). It mirrors the +// two-level map[scope]map[id]*T nesting it replaces. +func scopedKey(scope, id string) string { return scope + "/" + id } + +// ---- primary key functions ---- + +func clustersKeyFn(v *Cluster) string { return v.ClusterName } +func capacityProvidersKeyFn(v *CapacityProvider) string { return v.Name } +func accountSettingsKeyFn(v *AccountSetting) string { return accountSettingKey(v.Name, v.PrincipalArn) } +func taskProtectionsKeyFn(v *TaskProtection) string { return v.TaskArn } +func serviceDeploymentsKeyFn(v *ServiceDeployment) string { return v.ServiceDeploymentArn } +func expressGatewayServicesKeyFn(v *ExpressGatewayService) string { return v.ServiceArn } +func daemonRevisionsKeyFn(v *DaemonRevision) string { return v.DaemonRevisionArn } +func daemonDeploymentsKeyFn(v *DaemonDeployment) string { return v.DaemonDeploymentArn } + +func servicesKeyFn(v *Service) string { return scopedKey(clusterKey(v.ClusterArn), v.ServiceName) } +func servicesClusterIndexKeyFn(v *Service) string { return clusterKey(v.ClusterArn) } + +func tasksKeyFn(v *Task) string { return v.TaskArn } +func tasksClusterIndexKeyFn(v *Task) string { return clusterKey(v.ClusterArn) } + +func containerInstancesKeyFn(v *ContainerInstance) string { + return scopedKey(clusterKey(v.ClusterArn), v.ContainerInstanceArn) +} +func containerInstancesClusterIndexKeyFn(v *ContainerInstance) string { + return clusterKey(v.ClusterArn) +} + +func taskSetsKeyFn(v *TaskSet) string { return scopedKey(v.ServiceArn, v.TaskSetArn) } +func taskSetsServiceIndexKeyFn(v *TaskSet) string { return v.ServiceArn } + +func daemonsKeyFn(v *Daemon) string { return scopedKey(clusterKey(v.ClusterArn), v.DaemonName) } +func daemonsClusterIndexKeyFn(v *Daemon) string { return clusterKey(v.ClusterArn) } + +// taskDefByArnKeyFn and daemonTaskDefByArnKeyFn back the two ARN-lookup cache +// tables that are NOT registered on b.registry (see registerAllTables) because +// they are derived caches rebuilt from the raw taskDefinitions/ +// daemonTaskDefinitions maps, not independently persisted state. +func taskDefByArnKeyFn(v *TaskDefinition) string { return v.TaskDefinitionArn } +func daemonTaskDefByArnKeyFn(v *DaemonTaskDefinition) string { return v.DaemonTaskDefinitionArn } + +// registerAllTables registers every converted resource map on b.registry +// exactly once, and constructs the two unregistered ARN-cache tables. It must +// be called during construction only (immediately after b.registry is +// created), never on every Reset() -- store.Register panics on a duplicate +// name, so runtime resets go through registry.ResetAll() (plus the two +// unregistered caches' own .Reset()) instead; see InMemoryBackend.Reset in +// backend.go. +// +// The following resource fields are deliberately left as plain maps (not +// registered here): +// - taskDefinitions, daemonTaskDefinitions, daemonTaskDefs: value type is a +// slice ([]*T), not a single *T -- store.Table wraps map[string]*V, so a +// map[string][]*T container is out of scope for this conversion (matches +// the ec2 precedent of leaving map[string][]*T fields such as +// ipamPoolCidrs/spotFleetHistory/subnetCIDRAssociations raw). Ordering +// within each family's revision slice is also load-bearing (latest +// revision = last element) in a way a store.Index's Table.Restore-driven, +// primary-key-sorted rebuild would not reliably preserve for a +// multi-digit revision family. +// - resourceTags: value type []Tag, same slice-shape exclusion. +// - tasksByInstance: three levels of map keyed by bool, not a *T value at +// all -- an internal reverse-index set, not a resource collection. +// - serviceIndex: keyed by the svcRef struct, not a string, and its value is +// bool, not *T. +// - attributes: composite key (cluster, attributeKey(name, targetID)) +// requires the cluster, which is not stored on the Attribute value itself +// -- matches the ec2 precedent for vpcCidrAssociations ("key composite +// requires vpcID which is not stored on the value"). +// - lifecycle: value type taskLifecycle carries no identity field of its +// own (no task ARN field); it is keyed externally by task ARN -- matches +// the ec2 precedent for instanceIMDSOptions/vpcPeeringOptions-style +// exclusions ("value type carries no identity field of its own"). +// - serviceRevisions, serviceRevisionsByArn: both are intentionally never +// populated (see the comment on InMemoryBackend.serviceRevisions in +// backend.go -- this backend derives ServiceRevision snapshots on demand +// instead), and serviceRevisions is additionally slice-valued. +func registerAllTables(b *InMemoryBackend) { + b.clusters = store.Register(b.registry, "clusters", store.New(clustersKeyFn)) + b.capacityProviders = store.Register(b.registry, "capacityProviders", store.New(capacityProvidersKeyFn)) + b.accountSettings = store.Register(b.registry, "accountSettings", store.New(accountSettingsKeyFn)) + b.taskProtections = store.Register(b.registry, "taskProtections", store.New(taskProtectionsKeyFn)) + b.serviceDeployments = store.Register(b.registry, "serviceDeployments", store.New(serviceDeploymentsKeyFn)) + b.expressGatewayServices = store.Register( + b.registry, "expressGatewayServices", store.New(expressGatewayServicesKeyFn), + ) + b.daemonRevisions = store.Register(b.registry, "daemonRevisions", store.New(daemonRevisionsKeyFn)) + b.daemonDeployments = store.Register(b.registry, "daemonDeployments", store.New(daemonDeploymentsKeyFn)) + + b.services = store.Register(b.registry, "services", store.New(servicesKeyFn)) + b.servicesByCluster = b.services.AddIndex("servicesByCluster", servicesClusterIndexKeyFn) + + b.tasks = store.Register(b.registry, "tasks", store.New(tasksKeyFn)) + b.tasksByCluster = b.tasks.AddIndex("tasksByCluster", tasksClusterIndexKeyFn) + + b.containerInstances = store.Register(b.registry, "containerInstances", store.New(containerInstancesKeyFn)) + b.containerInstancesByCluster = b.containerInstances.AddIndex( + "containerInstancesByCluster", containerInstancesClusterIndexKeyFn, + ) + + b.taskSets = store.Register(b.registry, "taskSets", store.New(taskSetsKeyFn)) + b.taskSetsByService = b.taskSets.AddIndex("taskSetsByService", taskSetsServiceIndexKeyFn) + + b.daemons = store.Register(b.registry, "daemons", store.New(daemonsKeyFn)) + b.daemonsByCluster = b.daemons.AddIndex("daemonsByCluster", daemonsClusterIndexKeyFn) + + // Unregistered derived-cache tables: reset/rebuilt manually, never part of + // the persisted "tables" blob (see Reset/Restore in backend.go/persistence.go). + b.taskDefByArn = store.New(taskDefByArnKeyFn) + b.daemonTaskDefByArn = store.New(daemonTaskDefByArnKeyFn) +} + +// The helpers below replace the old "delete an entire per-cluster/per-service +// submap in one shot" idiom (e.g. delete(b.tasks, clusterName)) that the +// pre-conversion nested map[string]map[string]*T shape supported directly. A +// store.Index's Get returns a slice OWNED by the index -- it must not be +// mutated or ranged over while concurrently deleting from the backing Table +// (each Table.Delete calls idx.remove, which mutates that same backing +// slice) -- so every one of these snapshots the group into a private copy +// first via append(nil, ...). All must be called with the write lock held. + +// tasksInClusterLocked returns a snapshot copy of every task in clusterName. +func (b *InMemoryBackend) tasksInClusterLocked(clusterName string) []*Task { + return append([]*Task(nil), b.tasksByCluster.Get(clusterName)...) +} + +// servicesInClusterLocked returns a snapshot copy of every service in clusterName. +func (b *InMemoryBackend) servicesInClusterLocked(clusterName string) []*Service { + return append([]*Service(nil), b.servicesByCluster.Get(clusterName)...) +} + +// containerInstancesInClusterLocked returns a snapshot copy of every container +// instance in clusterName. +func (b *InMemoryBackend) containerInstancesInClusterLocked(clusterName string) []*ContainerInstance { + return append([]*ContainerInstance(nil), b.containerInstancesByCluster.Get(clusterName)...) +} + +// taskSetsForServiceLocked returns a snapshot copy of every task set belonging +// to serviceArn. +func (b *InMemoryBackend) taskSetsForServiceLocked(serviceArn string) []*TaskSet { + return append([]*TaskSet(nil), b.taskSetsByService.Get(serviceArn)...) +} + +// daemonsInClusterLocked returns a snapshot copy of every daemon in clusterName. +func (b *InMemoryBackend) daemonsInClusterLocked(clusterName string) []*Daemon { + return append([]*Daemon(nil), b.daemonsByCluster.Get(clusterName)...) +} + +// deleteServicesForClusterLocked removes every service belonging to clusterName. +func (b *InMemoryBackend) deleteServicesForClusterLocked(clusterName string) { + for _, s := range b.servicesInClusterLocked(clusterName) { + b.services.Delete(servicesKeyFn(s)) + } +} + +// deleteTasksForClusterLocked removes every task belonging to clusterName. +func (b *InMemoryBackend) deleteTasksForClusterLocked(clusterName string) { + for _, t := range b.tasksInClusterLocked(clusterName) { + b.tasks.Delete(tasksKeyFn(t)) + } +} + +// deleteContainerInstancesForClusterLocked removes every container instance +// belonging to clusterName. +func (b *InMemoryBackend) deleteContainerInstancesForClusterLocked(clusterName string) { + for _, ci := range b.containerInstancesInClusterLocked(clusterName) { + b.containerInstances.Delete(containerInstancesKeyFn(ci)) + } +} + +// deleteTaskSetsForServiceLocked removes every task set belonging to serviceArn. +func (b *InMemoryBackend) deleteTaskSetsForServiceLocked(serviceArn string) { + for _, ts := range b.taskSetsForServiceLocked(serviceArn) { + b.taskSets.Delete(taskSetsKeyFn(ts)) + } +} diff --git a/services/elasticache/PARITY.md b/services/elasticache/PARITY.md new file mode 100644 index 000000000..dc9717e9b --- /dev/null +++ b/services/elasticache/PARITY.md @@ -0,0 +1,180 @@ +--- +service: elasticache +sdk_module: aws-sdk-go-v2/service/elasticache@v1.51.11 +last_audit_commit: e7830377 +last_audit_date: 2026-07-05 +overall: B # already-accurate op-by-op, with a real error-code/HTTP-status + # bug class found and fixed across ~60 call sites, plus two + # disguised-stub validation gaps wired up (see Notes/gaps). +# Per-op or per-op-family status. Values: ok | partial | gap | deferred. +ops: + CreateCacheCluster: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: CacheClusterNotFound 400->404; added SnapshotName restore (was silently ignored)"} + DeleteCacheCluster: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: CacheClusterNotFound 400->404"} + DescribeCacheClusters: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: CacheClusterNotFound 400->404; ShowCacheNodeInfo/pagination verified ok"} + ModifyCacheCluster: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: CacheParameterGroupNotFound 400->404; InvalidParameterGroupFamily->InvalidParameterValue (real code doesn't exist)"} + RebootCacheCluster: {wire: ok, errors: ok, state: ok, persist: ok} + CreateReplicationGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: ReplicationGroupAlreadyExists/CacheParameterGroupNotFound status; added SnapshotName restore"} + DeleteReplicationGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: ReplicationGroupNotFound -> ReplicationGroupNotFoundFault, 400->404"} + DescribeReplicationGroups: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same as above; NodeGroups/PendingModifiedValues/UserGroupIds wire shapes verified ok"} + ModifyReplicationGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: wired dead ErrTransitEncryptionModeInvalid sentinel into validateTransitEncryptionModify + error mapping (disguised stub: guard never ran)"} + TestFailover: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: ReplicationGroupNotFound code/status"} + IncreaseReplicaCount: {wire: ok, errors: ok, state: ok, persist: ok} + DecreaseReplicaCount: {wire: ok, errors: ok, state: ok, persist: ok} + ModifyReplicationGroupShardConfiguration: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: ErrClusterModeRequired was returned by backend but never mapped by the handler -> fell through to 500 InternalFailure; now 400 InvalidParameterCombination"} + CreateCacheParameterGroup: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteCacheParameterGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: CacheParameterGroupNotFound 400->404"} + DescribeCacheParameterGroups: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + ModifyCacheParameterGroup: {wire: ok, errors: ok, state: ok, persist: ok} + ResetCacheParameterGroup: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeCacheParameters: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: CacheParameterGroupNotFound 400->404"} + DescribeEngineDefaultParameters: {wire: ok, errors: ok, state: ok, persist: n/a} + CreateCacheSubnetGroup: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteCacheSubnetGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: code CacheSubnetGroupNotFound -> CacheSubnetGroupNotFoundFault (AWS keeps the Fault suffix on the wire for this one specifically; status stays 400)"} + DescribeCacheSubnetGroups: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same code fix"} + ModifyCacheSubnetGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same code fix"} + CreateCacheSecurityGroup: {wire: ok, errors: ok, state: ok, persist: ok} + AuthorizeCacheSecurityGroupIngress: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: CacheSecurityGroupNotFound 400->404"} + RevokeCacheSecurityGroupIngress: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + DeleteCacheSecurityGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + DescribeCacheSecurityGroups: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + CreateSnapshot: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteSnapshot: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: SnapshotNotFoundFault 400->404"} + DescribeSnapshots: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same; automatic vs manual source filter verified ok"} + CopySnapshot: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: SnapshotNotFoundFault 400->404"} + DescribeEvents: {wire: ok, errors: ok, state: ok, persist: n/a} + CreateServerlessCache: {wire: ok, errors: ok, state: ok, persist: ok} + ModifyServerlessCache: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: ServerlessCacheNotFound -> ServerlessCacheNotFoundFault, 400->404"} + DeleteServerlessCache: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + DescribeServerlessCaches: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + CreateServerlessCacheSnapshot: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: ServerlessCacheNotFound code; ServerlessCacheSnapshotNotFoundFault status 400->404"} + CopyServerlessCacheSnapshot: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: ServerlessCacheSnapshotNotFoundFault status 400->404"} + DeleteServerlessCacheSnapshot: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + DescribeServerlessCacheSnapshots: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + ExportServerlessCacheSnapshot: {wire: ok, errors: ok, state: ok, persist: ok} + CreateUser: {wire: ok, errors: ok, state: ok, persist: ok} + ModifyUser: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: UserNotFound 400->404; InvalidParameterValueException -> InvalidParameterValue (real wire code has no Exception suffix)"} + DeleteUser: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: UserNotFound 400->404"} + DescribeUsers: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + CreateUserGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: code UserGroupAlreadyExistsFault -> UserGroupAlreadyExists (no Fault suffix on the wire)"} + ModifyUserGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: UserGroupNotFound 400->404"} + DeleteUserGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + DescribeUserGroups: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + CreateGlobalReplicationGroup: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteGlobalReplicationGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: GlobalReplicationGroupNotFoundFault status 400->404"} + DescribeGlobalReplicationGroups: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + ModifyGlobalReplicationGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + DisassociateGlobalReplicationGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + FailoverGlobalReplicationGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + IncreaseNodeGroupsInGlobalReplicationGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + DecreaseNodeGroupsInGlobalReplicationGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + RebalanceSlotsInGlobalReplicationGroup: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: same"} + DescribeReservedCacheNodes: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: ReservedCacheNodeNotFound 400->404"} + DescribeReservedCacheNodesOfferings: {wire: ok, errors: ok, state: ok, persist: n/a, note: "fixed: ReservedCacheNodesOfferingNotFound 400->404"} + PurchaseReservedCacheNodesOffering: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: ReservedCacheNodesOfferingNotFound 400->404; ReservedCacheNodeAlreadyExists 409->404 (AWS models this AlreadyExists fault as 404, not 409/400)"} + DescribeCacheEngineVersions: {wire: ok, errors: ok, state: n/a, persist: n/a} + DescribeServiceUpdates: {wire: ok, errors: ok, state: n/a, persist: n/a} + DescribeUpdateActions: {wire: ok, errors: ok, state: n/a, persist: n/a} + BatchApplyUpdateAction: {wire: ok, errors: ok, state: ok, persist: ok} + BatchStopUpdateAction: {wire: ok, errors: ok, state: ok, persist: ok} + ListAllowedNodeTypeModifications: {wire: ok, errors: ok, state: n/a, persist: n/a} + StartMigration: {wire: ok, errors: ok, state: ok, persist: ok} + TestMigration: {wire: ok, errors: ok, state: ok, persist: ok} + CompleteMigration: {wire: ok, errors: ok, state: ok, persist: ok} + ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok} + AddTagsToResource: {wire: ok, errors: ok, state: ok, persist: ok} + RemoveTagsFromResource: {wire: ok, errors: ok, state: ok, persist: ok} +families: + cache_clusters: {status: ok, note: "engine redis/memcached/valkey, node type, num nodes, creating->available->modifying->deleting->rebooting all observable via lifecycle overlay; cache nodes list w/ endpoints; DescribeCacheClusters ShowCacheNodeInfo+pagination correct"} + replication_groups: {status: ok, note: "primary/replica, node groups/shards, multi-AZ, automatic failover, cluster mode, IncreaseReplicaCount/DecreaseReplicaCount/TestFailover/global datastore all present and real; NodeGroups/PendingModifiedValues/UserGroupIds XML wrappers verified against api-2.json"} + cache_parameter_groups: {status: ok, note: "Create/Modify/Delete/Describe/Reset + DescribeCacheParameters + DescribeEngineDefaultParameters all real; default-group protection (ErrParameterGroupDefaultNotModifiable -> InvalidCacheParameterGroupState) verified wired"} + cache_subnet_groups: {status: ok} + cache_security_groups: {status: ok} + snapshots: {status: ok, note: "automatic vs manual source tracked (SnapshotSource field), CopySnapshot real; CreateCacheCluster/CreateReplicationGroup SnapshotName restore was a genuine gap (see gaps below), now fixed"} + serverless_caches: {status: ok} + users_and_user_groups: {status: ok, note: "RBAC access string, authentication (password/IAM/NoPasswordRequired) all real"} + reserved_nodes: {status: ok} + service_updates_and_events: {status: ok, note: "DescribeEvents wire shape (Event/Events/Marker) verified against api-2.json exactly"} + tags: {status: ok, note: "Add/Remove/List via ARN; ErrResourceNotFound correctly surfaces as InvalidARN (matches AWS's own tag-op behavior for a resource ARN that doesn't resolve)"} + timestamps: {status: ok, note: "RFC3339 ISO8601 strings used throughout -- CORRECT for this query/XML protocol; do NOT flag as an epoch-seconds bug (awstime.Epoch is for json/rest-json protocols only, not applicable here)"} +gaps: + - "State-transition guards (InvalidCacheClusterStateFault/InvalidReplicationGroupStateFault/InvalidCacheParameterGroupStateFault-for-non-default-groups) are not enforced: Modify/Delete succeed even while a resource's PendingStatus is still creating/modifying/deleting when SetLifecycleDelay > 0. The lifecycle-overlay mechanism (backend_lifecycle.go) tracks the transient state precisely enough to support this, but no mutating op consults it before proceeding. Deliberately NOT implemented this pass: TestLifecycleFullVariantsAreObservable (backend_lifecycle_test.go:395) explicitly asserts that a Modify call immediately after Create (while still \"creating\") succeeds and reports \"modifying\" -- adding a guard would flip that test's expected outcome, and the risk of narrowing scope incorrectly (AWS's exact allow/deny matrix per state per op isn't verified) outweighed the value this pass. Left for a follow-up with SDK-parity evidence per transition. (bd: gopherstack-y8l follow-up, not filed as a separate issue this pass -- see Notes)" + - "MaxRecords bounds (AWS requires 20-100 for most Describe* ops, InvalidParameterValueException otherwise) are accepted verbatim without range validation across all paginated ops. Shared pattern likely present in most query-protocol services in this repo, not elasticache-specific; deferred as a cross-service concern rather than a targeted elasticache fix." +deferred: + - "Full data-plane snapshot/restore fidelity (actual key-value RDB dump/reload through miniredis) -- CreateCacheCluster/CreateReplicationGroup SnapshotName now validates existence and inherits engine/node-type metadata (the real API-contract behavior verified against api-2.json), but does not replay the source's actual key data into the restored miniredis instance. Flagged as a possible future enhancement, not a wire-shape/error-code bug." + - "Quota-exceeded faults (ClusterQuotaForCustomerExceededFault, NodeQuotaForClusterExceededFault, CacheParameterGroupQuotaExceededFault, etc.) are not modeled -- no artificial resource limits are enforced. Standard for an emulator; not audited further this pass." +leaks: {status: clean, note: "zero goroutines/timers/tickers in the entire package (grepped `go func`, `time.AfterFunc`, `time.NewTicker`, `time.NewTimer` -- no hits outside tests). The lifecycle mechanism (backend_lifecycle.go) is deliberately goroutine-free: transient status + deadline overlaid on read, reaped lazily on the next write (pruneRegionLocked). Confirmed no leak regressions from this pass's changes."} +--- + +## Notes + +**Protocol**: query/XML (`Version=2015-02-02`), matching `aws-sdk-go-v2/service/elasticache`'s +`awsAwsquery` (de)serializers. All list wrappers (`CacheNode`, `NodeGroup`, `NodeGroupMember`, +`Tag`, `Parameter`, `Subnet`, `Event`, `CacheParameterGroup`, `EC2SecurityGroup`, `member` for +unlabeled lists like `UserGroupIds`) were cross-checked directly against +`aws-sdk-go@v1.55.5/models/apis/elasticache/2015-02-02/api-2.json`'s `locationName` metadata -- +all correct, no wrapper-name bugs found this pass. + +**Timestamps are RFC3339 strings, not epoch seconds** -- this is CORRECT for ElastiCache's +query/XML protocol. `pkgs/awstime.Epoch` (used to fix QuickSight/IoT-style bugs) is only for +JSON/rest-json protocols; do not "fix" the RFC3339 formatting here in a future sweep, it isn't +a bug. + +**The major finding this pass: a systematic error-code / HTTP-status wire bug class.** +Cross-referencing every `xmlError(...)` call site in handler.go/handler_ops2.go/handler_new_ops.go +against `aws-sdk-go@v1.55.5`'s `api-2.json` (`error.code` and `error.httpStatusCode` per +exception shape) turned up ~60 call sites with one or both of: + - a wire `` string missing (or wrongly carrying) the AWS `Fault` suffix. AWS is + genuinely inconsistent about this per-shape -- e.g. `CacheClusterNotFoundFault` serializes + as bare `CacheClusterNotFound`, but `ReplicationGroupNotFoundFault`, + `ServerlessCacheNotFoundFault`, `GlobalReplicationGroupNotFoundFault`, and + `CacheSubnetGroupNotFoundFault` keep the `Fault` suffix on the wire. There is no + reliable shortcut here -- each fault shape's `error.code` in api-2.json must be checked + individually; don't assume a pattern. + - the wrong HTTP status. Nearly every `*NotFoundFault` in this API is modeled `404`, but the + emulator used `400` almost everywhere (a few, like `CacheSubnetGroupNotFoundFault`, really + are `400` per AWS's own model -- again, no shortcut, check api-2.json per shape). + `ReservedCacheNodeAlreadyExistsFault` is modeled `404` too (not the `409`/`400` one might + expect for an "already exists" fault). + + This matters because aws-sdk-go-v2's query-protocol error deserializer is a per-operation + hardcoded `switch` keyed on the exact `` string, generated strictly from that + operation's modeled `errors` list in api-2.json (see `deserializers.go`, + `awsAwsquery_deserializeOpError`). A wrong code string doesn't just cosmetically differ + -- it means the SDK can't match any case and falls back to a generic `smithy.GenericAPIError`, + so callers lose `errors.As` typed-fault handling entirely. New regression test + `Test_ErrorWireShapesMatchAWS` in `handler_parity_sweep3_test.go` asserts the SDK actually + deserializes into the exact typed fault for a representative case per family, plus the exact + HTTP status, which is what makes this a wire-shape proof and not a bare `err != nil` check. + +**Trap for the next auditor**: `SnapshotNotFoundFault` is NOT in `CreateCacheCluster`'s or +`CreateReplicationGroup`'s modeled `errors` list in api-2.json, even though both operations +accept a `SnapshotName` restore parameter. A missing/invalid snapshot on these two ops +correctly surfaces as `InvalidParameterValueException` (wire code `InvalidParameterValue`, +400) -- NOT `SnapshotNotFoundFault` (404), even though that would be the intuitive choice and +is what every other snapshot-consuming op in this API does use. Verified directly against the +per-operation `errors` array in api-2.json; do not "fix" this to SnapshotNotFoundFault later, +it would break wire fidelity, not improve it. + +**Disguised-stub pattern found**: `ErrTransitEncryptionModeInvalid` and `ErrClusterModeRequired` +were both declared error sentinels with real intent (message text describing exactly the +validation AWS performs), but `ErrTransitEncryptionModeInvalid` was never returned from any +code path (dead sentinel -- the guard simply didn't exist in `applyModifyOptsLocked`), and +`ErrClusterModeRequired` WAS returned by the backend but had no case in the handler's error +mapping, so it fell through to a 500 `InternalFailure` instead of the correct 400 +`InvalidParameterCombination`. Both are fixed this pass. When auditing error sentinels in this +package (or others), grep each declared `Err*` var for non-test reference count -- a count of +1 (declaration only) or a backend-only reference (no handler-side `errors.Is` case) both +indicate a disguised stub. + +**Lock discipline**: single `*lockmetrics.RWMutex` (`b.mu`) guards all `InMemoryBackend` maps, +consistent with `pkgs-catalog.md`'s coarse-lock rule. When adding a cross-resource validation +inside an already-locked method (e.g. the new snapshot-restore lookup in +`CreateReplicationGroupFull`), reach for the store directly (`b.snapshotsStore(region)[name]`) +rather than calling a public `Describe*`/`List*` method that re-acquires `b.mu` -- the mutex is +not reentrant and the public accessors take their own `RLock`/`Lock`. + +**Known-accurate, don't re-flag**: `TestLifecycleFullVariantsAreObservable` intentionally allows +Modify while a resource's `PendingStatus` is still `"creating"` (see gaps above) -- this is an +existing, deliberate design choice from a prior sweep to keep the lifecycle mechanism purely +observational, not a bug this pass introduced or should "fix" without first confirming AWS's +exact state-transition matrix per operation. diff --git a/services/elasticache/backend.go b/services/elasticache/backend.go index f4b6e30e6..002ddb62f 100644 --- a/services/elasticache/backend.go +++ b/services/elasticache/backend.go @@ -20,6 +20,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" "github.com/blackbirdworks/gopherstack/pkgs/page" "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) @@ -569,19 +570,20 @@ func builtinParameterGroupFamilies() []struct{ family, name string } { // are global/partition-scoped (like AWS) and therefore are NOT region-nested. type InMemoryBackend struct { dnsRegistrar DNSRegistrar - serverlessCaches map[string]map[string]*ServerlessCache - userGroups map[string]map[string]*UserGroup - parameterGroups map[string]map[string]*CacheParameterGroup - globalReplicationGroups map[string]*GlobalReplicationGroup - snapshots map[string]map[string]*CacheSnapshot - cacheSecurityGroups map[string]map[string]*CacheSecurityGroup + registry *store.Registry + globalReplicationGroups *store.Table[GlobalReplicationGroup] + serverlessCaches map[string]*store.Table[ServerlessCache] + userGroups map[string]*store.Table[UserGroup] + parameterGroups map[string]*store.Table[CacheParameterGroup] + snapshots map[string]*store.Table[CacheSnapshot] + cacheSecurityGroups map[string]*store.Table[CacheSecurityGroup] cacheSecurityGroupIngress map[string]map[string][]EC2SecurityGroupMembership - clusters map[string]map[string]*Cluster - replicationGroups map[string]map[string]*ReplicationGroup - reservedCacheNodes map[string]map[string]*ReservedCacheNode - serverlessCacheSnapshots map[string]map[string]*ServerlessCacheSnapshot - subnetGroups map[string]map[string]*CacheSubnetGroup - users map[string]map[string]*User + clusters map[string]*store.Table[Cluster] + replicationGroups map[string]*store.Table[ReplicationGroup] + reservedCacheNodes map[string]*store.Table[ReservedCacheNode] + serverlessCacheSnapshots map[string]*store.Table[ServerlessCacheSnapshot] + subnetGroups map[string]*store.Table[CacheSubnetGroup] + users map[string]*store.Table[User] events *eventRing mu *lockmetrics.RWMutex allocator *portalloc.Allocator @@ -600,19 +602,19 @@ func NewInMemoryBackend(engineMode, accountID, region string, allocator *portall } b := &InMemoryBackend{ - clusters: make(map[string]map[string]*Cluster), - replicationGroups: make(map[string]map[string]*ReplicationGroup), - parameterGroups: make(map[string]map[string]*CacheParameterGroup), - subnetGroups: make(map[string]map[string]*CacheSubnetGroup), - snapshots: make(map[string]map[string]*CacheSnapshot), - cacheSecurityGroups: make(map[string]map[string]*CacheSecurityGroup), + registry: store.NewRegistry(), + clusters: make(map[string]*store.Table[Cluster]), + replicationGroups: make(map[string]*store.Table[ReplicationGroup]), + parameterGroups: make(map[string]*store.Table[CacheParameterGroup]), + subnetGroups: make(map[string]*store.Table[CacheSubnetGroup]), + snapshots: make(map[string]*store.Table[CacheSnapshot]), + cacheSecurityGroups: make(map[string]*store.Table[CacheSecurityGroup]), cacheSecurityGroupIngress: make(map[string]map[string][]EC2SecurityGroupMembership), - globalReplicationGroups: make(map[string]*GlobalReplicationGroup), - serverlessCaches: make(map[string]map[string]*ServerlessCache), - serverlessCacheSnapshots: make(map[string]map[string]*ServerlessCacheSnapshot), - users: make(map[string]map[string]*User), - userGroups: make(map[string]map[string]*UserGroup), - reservedCacheNodes: make(map[string]map[string]*ReservedCacheNode), + serverlessCaches: make(map[string]*store.Table[ServerlessCache]), + serverlessCacheSnapshots: make(map[string]*store.Table[ServerlessCacheSnapshot]), + users: make(map[string]*store.Table[User]), + userGroups: make(map[string]*store.Table[UserGroup]), + reservedCacheNodes: make(map[string]*store.Table[ReservedCacheNode]), updateActions: nil, events: newEventRing(maxEvents), engineMode: engineMode, @@ -621,6 +623,9 @@ func NewInMemoryBackend(engineMode, accountID, region string, allocator *portall allocator: allocator, mu: lockmetrics.New("elasticache"), } + b.globalReplicationGroups = store.Register( + b.registry, "globalReplicationGroups", store.New(globalReplicationGroupKeyFn), + ) b.initDefaultParameterGroups() @@ -638,10 +643,10 @@ func (b *InMemoryBackend) initDefaultParameterGroups() { // initDefaultParameterGroupsForRegion seeds default parameter groups for the given region. // Callers must NOT hold b.mu (it allocates directly into the map). func (b *InMemoryBackend) initDefaultParameterGroupsForRegion(region string) { - store := b.parameterGroups[region] - if store == nil { - store = make(map[string]*CacheParameterGroup) - b.parameterGroups[region] = store + t := b.parameterGroups[region] + if t == nil { + t = store.New(cacheParameterGroupKeyFn) + b.parameterGroups[region] = t } for _, dpg := range builtinParameterGroupFamilies() { @@ -654,30 +659,30 @@ func (b *InMemoryBackend) initDefaultParameterGroupsForRegion(region string) { Parameters: make(map[string]string), Tags: tags.New("elasticache.pg." + dpg.name + ".tags"), } - store[dpg.name] = pg + t.Put(pg) } } // The following lazy per-region store helpers return the resource map for the // given region, creating it on first use. Callers must hold b.mu. -func (b *InMemoryBackend) clustersStore(region string) map[string]*Cluster { +func (b *InMemoryBackend) clustersStore(region string) *store.Table[Cluster] { if b.clusters[region] == nil { - b.clusters[region] = make(map[string]*Cluster) + b.clusters[region] = store.New(clusterKeyFn) } return b.clusters[region] } -func (b *InMemoryBackend) replicationGroupsStore(region string) map[string]*ReplicationGroup { +func (b *InMemoryBackend) replicationGroupsStore(region string) *store.Table[ReplicationGroup] { if b.replicationGroups[region] == nil { - b.replicationGroups[region] = make(map[string]*ReplicationGroup) + b.replicationGroups[region] = store.New(replicationGroupKeyFn) } return b.replicationGroups[region] } -func (b *InMemoryBackend) parameterGroupsStore(region string) map[string]*CacheParameterGroup { +func (b *InMemoryBackend) parameterGroupsStore(region string) *store.Table[CacheParameterGroup] { if b.parameterGroups[region] == nil { b.initDefaultParameterGroupsForRegion(region) } @@ -685,25 +690,25 @@ func (b *InMemoryBackend) parameterGroupsStore(region string) map[string]*CacheP return b.parameterGroups[region] } -func (b *InMemoryBackend) subnetGroupsStore(region string) map[string]*CacheSubnetGroup { +func (b *InMemoryBackend) subnetGroupsStore(region string) *store.Table[CacheSubnetGroup] { if b.subnetGroups[region] == nil { - b.subnetGroups[region] = make(map[string]*CacheSubnetGroup) + b.subnetGroups[region] = store.New(cacheSubnetGroupKeyFn) } return b.subnetGroups[region] } -func (b *InMemoryBackend) snapshotsStore(region string) map[string]*CacheSnapshot { +func (b *InMemoryBackend) snapshotsStore(region string) *store.Table[CacheSnapshot] { if b.snapshots[region] == nil { - b.snapshots[region] = make(map[string]*CacheSnapshot) + b.snapshots[region] = store.New(cacheSnapshotKeyFn) } return b.snapshots[region] } -func (b *InMemoryBackend) cacheSecurityGroupsStore(region string) map[string]*CacheSecurityGroup { +func (b *InMemoryBackend) cacheSecurityGroupsStore(region string) *store.Table[CacheSecurityGroup] { if b.cacheSecurityGroups[region] == nil { - b.cacheSecurityGroups[region] = make(map[string]*CacheSecurityGroup) + b.cacheSecurityGroups[region] = store.New(cacheSecurityGroupKeyFn) } return b.cacheSecurityGroups[region] @@ -717,41 +722,41 @@ func (b *InMemoryBackend) cacheSecurityGroupIngressStore(region string) map[stri return b.cacheSecurityGroupIngress[region] } -func (b *InMemoryBackend) serverlessCachesStore(region string) map[string]*ServerlessCache { +func (b *InMemoryBackend) serverlessCachesStore(region string) *store.Table[ServerlessCache] { if b.serverlessCaches[region] == nil { - b.serverlessCaches[region] = make(map[string]*ServerlessCache) + b.serverlessCaches[region] = store.New(serverlessCacheKeyFn) } return b.serverlessCaches[region] } -func (b *InMemoryBackend) serverlessCacheSnapshotsStore(region string) map[string]*ServerlessCacheSnapshot { +func (b *InMemoryBackend) serverlessCacheSnapshotsStore(region string) *store.Table[ServerlessCacheSnapshot] { if b.serverlessCacheSnapshots[region] == nil { - b.serverlessCacheSnapshots[region] = make(map[string]*ServerlessCacheSnapshot) + b.serverlessCacheSnapshots[region] = store.New(serverlessCacheSnapshotKeyFn) } return b.serverlessCacheSnapshots[region] } -func (b *InMemoryBackend) usersStore(region string) map[string]*User { +func (b *InMemoryBackend) usersStore(region string) *store.Table[User] { if b.users[region] == nil { - b.users[region] = make(map[string]*User) + b.users[region] = store.New(userKeyFn) } return b.users[region] } -func (b *InMemoryBackend) userGroupsStore(region string) map[string]*UserGroup { +func (b *InMemoryBackend) userGroupsStore(region string) *store.Table[UserGroup] { if b.userGroups[region] == nil { - b.userGroups[region] = make(map[string]*UserGroup) + b.userGroups[region] = store.New(userGroupKeyFn) } return b.userGroups[region] } -func (b *InMemoryBackend) reservedCacheNodesStore(region string) map[string]*ReservedCacheNode { +func (b *InMemoryBackend) reservedCacheNodesStore(region string) *store.Table[ReservedCacheNode] { if b.reservedCacheNodes[region] == nil { - b.reservedCacheNodes[region] = make(map[string]*ReservedCacheNode) + b.reservedCacheNodes[region] = store.New(reservedCacheNodeKeyFn) } return b.reservedCacheNodes[region] @@ -951,7 +956,7 @@ func (b *InMemoryBackend) insertClusterLocked( c.Endpoint = gopherDNS.SyntheticHostname(id, randomSuffix(), region, "cache") b.registerClusterDNSLocked(c) - b.clustersStore(region)[id] = c + b.clustersStore(region).Put(c) b.appendEventLocked(id, "cache-cluster", "cluster created") return c @@ -982,7 +987,7 @@ func (b *InMemoryBackend) CreateCluster(ctx context.Context, id, engine, nodeTyp b.mu.Lock("CreateCluster.reserve") b.pruneRegionLocked(region) - _, exists := b.clustersStore(region)[id] + _, exists := b.clustersStore(region).Get(id) b.mu.Unlock() if exists { return nil, ErrClusterAlreadyExists @@ -995,7 +1000,7 @@ func (b *InMemoryBackend) CreateCluster(ctx context.Context, id, engine, nodeTyp b.mu.Lock("CreateCluster.insert") defer b.mu.Unlock() - if _, dup := b.clustersStore(region)[id]; dup { + if _, dup := b.clustersStore(region).Get(id); dup { b.releaseEngine(eng) return nil, ErrClusterAlreadyExists @@ -1016,13 +1021,13 @@ func (b *InMemoryBackend) CreateClusterWithOptions( b.mu.Lock("CreateClusterWithOptions.reserve") b.pruneRegionLocked(region) - if _, exists := b.clustersStore(region)[id]; exists { + if _, exists := b.clustersStore(region).Get(id); exists { b.mu.Unlock() return nil, ErrClusterAlreadyExists } if paramGroupName != "" { - pg, ok := b.parameterGroupsStore(region)[paramGroupName] + pg, ok := b.parameterGroupsStore(region).Get(paramGroupName) if !ok { b.mu.Unlock() @@ -1043,7 +1048,7 @@ func (b *InMemoryBackend) CreateClusterWithOptions( b.mu.Lock("CreateClusterWithOptions.insert") defer b.mu.Unlock() - if _, exists := b.clustersStore(region)[id]; exists { + if _, exists := b.clustersStore(region).Get(id); exists { b.releaseEngine(eng) return nil, ErrClusterAlreadyExists @@ -1063,8 +1068,8 @@ func (b *InMemoryBackend) DeleteCluster(ctx context.Context, id string) error { region := getRegion(ctx, b.region) b.pruneRegionLocked(region) - store := b.clustersStore(region) - c, exists := store[id] + tbl := b.clustersStore(region) + c, exists := tbl.Get(id) if !exists || isReaped(b.now(), c.PendingStatus, c.AvailableAt) { return ErrClusterNotFound } @@ -1081,7 +1086,7 @@ func (b *InMemoryBackend) DeleteCluster(ctx context.Context, id string) error { } b.releaseClusterLocked(c) - delete(store, id) + tbl.Delete(id) b.appendEventLocked(id, "cache-cluster", "cluster deleted") return nil @@ -1139,36 +1144,36 @@ func (b *InMemoryBackend) collectTagCandidatesLocked() []tagCandidate { func (b *InMemoryBackend) appendClusterTagCandidates(candidates []tagCandidate) []tagCandidate { for _, regionClusters := range b.clusters { - for _, c := range regionClusters { + for _, c := range regionClusters.All() { candidates = append(candidates, tagCandidate{c.ARN, tagEntry{&c.Tags, "elasticache.cluster." + c.ClusterID + ".tags"}}) } } for _, regionRGs := range b.replicationGroups { - for _, rg := range regionRGs { + for _, rg := range regionRGs.All() { candidates = append(candidates, tagCandidate{rg.ARN, tagEntry{&rg.Tags, "elasticache.rg." + rg.ReplicationGroupID + ".tags"}}) } } for _, regionPGs := range b.parameterGroups { - for _, pg := range regionPGs { + for _, pg := range regionPGs.All() { candidates = append(candidates, tagCandidate{pg.ARN, tagEntry{&pg.Tags, "elasticache.pg." + pg.Name + ".tags"}}) } } for _, regionSnaps := range b.snapshots { - for _, snap := range regionSnaps { + for _, snap := range regionSnaps.All() { candidates = append(candidates, tagCandidate{snap.ARN, tagEntry{&snap.Tags, "elasticache.snapshot." + snap.SnapshotName + ".tags"}}) } } for _, regionCSGs := range b.cacheSecurityGroups { - for _, sg := range regionCSGs { + for _, sg := range regionCSGs.All() { candidates = append(candidates, tagCandidate{sg.ARN, tagEntry{&sg.Tags, "elasticache.sg." + sg.Name + ".tags"}}) } } - for _, grg := range b.globalReplicationGroups { + for _, grg := range b.globalReplicationGroups.All() { candidates = append(candidates, tagCandidate{grg.ARN, tagEntry{&grg.Tags, "elasticache.grg." + grg.GlobalReplicationGroupID + ".tags"}}) } @@ -1178,7 +1183,7 @@ func (b *InMemoryBackend) appendClusterTagCandidates(candidates []tagCandidate) func (b *InMemoryBackend) appendNetworkTagCandidates(candidates []tagCandidate) []tagCandidate { for _, regionSGs := range b.subnetGroups { - for _, sg := range regionSGs { + for _, sg := range regionSGs.All() { candidates = append(candidates, tagCandidate{sg.ARN, tagEntry{&sg.Tags, "elasticache.sg." + sg.Name + ".tags"}}) } @@ -1189,13 +1194,13 @@ func (b *InMemoryBackend) appendNetworkTagCandidates(candidates []tagCandidate) func (b *InMemoryBackend) appendServerlessTagCandidates(candidates []tagCandidate) []tagCandidate { for _, regionSCs := range b.serverlessCaches { - for _, sc := range regionSCs { + for _, sc := range regionSCs.All() { candidates = append(candidates, tagCandidate{sc.ARN, tagEntry{&sc.Tags, "elasticache.serverless." + sc.Name + ".tags"}}) } } for _, regionScSnaps := range b.serverlessCacheSnapshots { - for _, snap := range regionScSnaps { + for _, snap := range regionScSnaps.All() { candidates = append(candidates, tagCandidate{snap.ARN, tagEntry{&snap.Tags, "elasticache.serverlesssnap." + snap.Name + ".tags"}}) } @@ -1206,13 +1211,13 @@ func (b *InMemoryBackend) appendServerlessTagCandidates(candidates []tagCandidat func (b *InMemoryBackend) appendUserTagCandidates(candidates []tagCandidate) []tagCandidate { for _, regionUsers := range b.users { - for _, u := range regionUsers { + for _, u := range regionUsers.All() { candidates = append(candidates, tagCandidate{u.ARN, tagEntry{&u.Tags, "elasticache.user." + u.UserID + ".tags"}}) } } for _, regionUGs := range b.userGroups { - for _, ug := range regionUGs { + for _, ug := range regionUGs.All() { candidates = append(candidates, tagCandidate{ug.ARN, tagEntry{&ug.Tags, "elasticache.usergroup." + ug.UserGroupID + ".tags"}}) } @@ -1303,7 +1308,7 @@ func (b *InMemoryBackend) createReplicationGroupLocked( SnapshotWindow: snapshotWindow, } b.markCreatingLocked(&rg.PendingStatus, &rg.AvailableAt) - b.replicationGroupsStore(region)[id] = rg + b.replicationGroupsStore(region).Put(rg) b.appendEventLocked(id, "replication-group", "replication group created") return rg @@ -1319,7 +1324,7 @@ func (b *InMemoryBackend) CreateReplicationGroup( region := getRegion(ctx, b.region) b.pruneRegionLocked(region) - if _, exists := b.replicationGroupsStore(region)[id]; exists { + if _, exists := b.replicationGroupsStore(region).Get(id); exists { return nil, ErrReplicationGroupAlreadyExists } @@ -1335,12 +1340,12 @@ func (b *InMemoryBackend) CreateReplicationGroupWithOptions( defer b.mu.Unlock() region := getRegion(ctx, b.region) - if _, exists := b.replicationGroupsStore(region)[id]; exists { + if _, exists := b.replicationGroupsStore(region).Get(id); exists { return nil, ErrReplicationGroupAlreadyExists } if paramGroupName != "" { - if _, ok := b.parameterGroupsStore(region)[paramGroupName]; !ok { + if _, ok := b.parameterGroupsStore(region).Get(paramGroupName); !ok { return nil, ErrParameterGroupNotFound } } @@ -1362,8 +1367,8 @@ func (b *InMemoryBackend) DeleteReplicationGroup(ctx context.Context, id string) region := getRegion(ctx, b.region) b.pruneRegionLocked(region) - store := b.replicationGroupsStore(region) - rg, exists := store[id] + tbl := b.replicationGroupsStore(region) + rg, exists := tbl.Get(id) if !exists || isReaped(b.now(), rg.PendingStatus, rg.AvailableAt) { return ErrReplicationGroupNotFound } @@ -1377,7 +1382,7 @@ func (b *InMemoryBackend) DeleteReplicationGroup(ctx context.Context, id string) } rg.Tags.Close() - delete(store, id) + tbl.Delete(id) b.appendEventLocked(id, "replication-group", "replication group deleted") return nil @@ -1415,7 +1420,7 @@ func (b *InMemoryBackend) ListAll() []Cluster { now := b.now() var out []Cluster for _, regionClusters := range b.clusters { - for _, c := range regionClusters { + for _, c := range regionClusters.All() { if isReaped(now, c.PendingStatus, c.AvailableAt) { continue } @@ -1438,7 +1443,7 @@ func (b *InMemoryBackend) ModifyCluster( defer b.mu.Unlock() region := getRegion(ctx, b.region) - c, exists := b.clustersStore(region)[id] + c, exists := b.clustersStore(region).Get(id) if !exists { return nil, ErrClusterNotFound } @@ -1448,7 +1453,7 @@ func (b *InMemoryBackend) ModifyCluster( } if paramGroupName != "" { - if _, ok := b.parameterGroupsStore(region)[paramGroupName]; !ok { + if _, ok := b.parameterGroupsStore(region).Get(paramGroupName); !ok { return nil, ErrParameterGroupNotFound } c.CacheParameterGroupName = paramGroupName @@ -1486,7 +1491,7 @@ func (b *InMemoryBackend) ModifyReplicationGroup( defer b.mu.Unlock() region := getRegion(ctx, b.region) - rg, exists := b.replicationGroupsStore(region)[id] + rg, exists := b.replicationGroupsStore(region).Get(id) if !exists { return nil, ErrReplicationGroupNotFound } @@ -1496,7 +1501,7 @@ func (b *InMemoryBackend) ModifyReplicationGroup( } if paramGroupName != "" { - if _, ok := b.parameterGroupsStore(region)[paramGroupName]; !ok { + if _, ok := b.parameterGroupsStore(region).Get(paramGroupName); !ok { return nil, ErrParameterGroupNotFound } rg.CacheParameterGroupName = paramGroupName @@ -1542,7 +1547,7 @@ func (b *InMemoryBackend) FailoverReplicationGroup(ctx context.Context, id, _ st defer b.mu.Unlock() region := getRegion(ctx, b.region) - rg, exists := b.replicationGroupsStore(region)[id] + rg, exists := b.replicationGroupsStore(region).Get(id) if !exists { return nil, ErrReplicationGroupNotFound } @@ -1563,8 +1568,8 @@ func (b *InMemoryBackend) CreateParameterGroup( defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.parameterGroupsStore(region) - if _, exists := store[name]; exists { + tbl := b.parameterGroupsStore(region) + if _, exists := tbl.Get(name); exists { return nil, ErrParameterGroupAlreadyExists } @@ -1577,7 +1582,7 @@ func (b *InMemoryBackend) CreateParameterGroup( Parameters: make(map[string]string), Tags: tags.New("elasticache.pg." + name + ".tags"), } - store[name] = pg + tbl.Put(pg) return pg, nil } @@ -1588,8 +1593,8 @@ func (b *InMemoryBackend) DeleteParameterGroup(ctx context.Context, name string) defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.parameterGroupsStore(region) - pg, exists := store[name] + tbl := b.parameterGroupsStore(region) + pg, exists := tbl.Get(name) if !exists { return ErrParameterGroupNotFound } @@ -1599,7 +1604,7 @@ func (b *InMemoryBackend) DeleteParameterGroup(ctx context.Context, name string) } pg.Tags.Close() - delete(store, name) + tbl.Delete(name) return nil } @@ -1629,7 +1634,7 @@ func (b *InMemoryBackend) ModifyParameterGroup( defer b.mu.Unlock() region := getRegion(ctx, b.region) - pg, exists := b.parameterGroupsStore(region)[name] + pg, exists := b.parameterGroupsStore(region).Get(name) if !exists { return nil, ErrParameterGroupNotFound } @@ -1656,7 +1661,7 @@ func (b *InMemoryBackend) ResetParameterGroup( defer b.mu.Unlock() region := getRegion(ctx, b.region) - pg, exists := b.parameterGroupsStore(region)[name] + pg, exists := b.parameterGroupsStore(region).Get(name) if !exists { return nil, ErrParameterGroupNotFound } @@ -1688,7 +1693,7 @@ func (b *InMemoryBackend) DescribeParameters( defer b.mu.RUnlock() region := getRegion(ctx, b.region) - pg, exists := b.parameterGroupsStore(region)[name] + pg, exists := b.parameterGroupsStore(region).Get(name) if !exists { return page.Page[CacheParameter]{}, ErrParameterGroupNotFound } @@ -1718,8 +1723,8 @@ func (b *InMemoryBackend) CreateSubnetGroup( defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.subnetGroupsStore(region) - if _, exists := store[name]; exists { + tbl := b.subnetGroupsStore(region) + if _, exists := tbl.Get(name); exists { return nil, ErrSubnetGroupAlreadyExists } @@ -1730,7 +1735,7 @@ func (b *InMemoryBackend) CreateSubnetGroup( ARN: b.subnetGroupARN(region, name), Tags: tags.New("elasticache.sg." + name + ".tags"), } - store[name] = sg + tbl.Put(sg) return sg, nil } @@ -1741,14 +1746,14 @@ func (b *InMemoryBackend) DeleteSubnetGroup(ctx context.Context, name string) er defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.subnetGroupsStore(region) - sg, exists := store[name] + tbl := b.subnetGroupsStore(region) + sg, exists := tbl.Get(name) if !exists { return ErrSubnetGroupNotFound } sg.Tags.Close() - delete(store, name) + tbl.Delete(name) return nil } @@ -1778,7 +1783,7 @@ func (b *InMemoryBackend) ModifySubnetGroup( defer b.mu.Unlock() region := getRegion(ctx, b.region) - sg, exists := b.subnetGroupsStore(region)[name] + sg, exists := b.subnetGroupsStore(region).Get(name) if !exists { return nil, ErrSubnetGroupNotFound } @@ -1811,7 +1816,7 @@ func (b *InMemoryBackend) CreateSnapshot( region := getRegion(ctx, b.region) snapStore := b.snapshotsStore(region) - if _, exists := snapStore[snapshotName]; exists { + if _, exists := snapStore.Get(snapshotName); exists { return nil, ErrSnapshotAlreadyExists } @@ -1828,7 +1833,7 @@ func (b *InMemoryBackend) CreateSnapshot( b.markCreatingLocked(&snap.PendingStatus, &snap.AvailableAt) if clusterID != "" { - c, ok := b.clustersStore(region)[clusterID] + c, ok := b.clustersStore(region).Get(clusterID) if !ok { return nil, ErrClusterNotFound } @@ -1838,7 +1843,7 @@ func (b *InMemoryBackend) CreateSnapshot( } if replicationGroupID != "" { - rg, ok := b.replicationGroupsStore(region)[replicationGroupID] + rg, ok := b.replicationGroupsStore(region).Get(replicationGroupID) if !ok { return nil, ErrReplicationGroupNotFound } @@ -1851,7 +1856,7 @@ func (b *InMemoryBackend) CreateSnapshot( snap.ReplicationGroupID = rg.ReplicationGroupID } - snapStore[snapshotName] = snap + snapStore.Put(snap) sourceID := clusterID if sourceID == "" { sourceID = replicationGroupID @@ -1871,8 +1876,8 @@ func (b *InMemoryBackend) DeleteSnapshot(ctx context.Context, snapshotName strin region := getRegion(ctx, b.region) b.pruneRegionLocked(region) - store := b.snapshotsStore(region) - snap, exists := store[snapshotName] + tbl := b.snapshotsStore(region) + snap, exists := tbl.Get(snapshotName) if !exists || isReaped(b.now(), snap.PendingStatus, snap.AvailableAt) { return nil, ErrSnapshotNotFound } @@ -1889,7 +1894,7 @@ func (b *InMemoryBackend) DeleteSnapshot(ctx context.Context, snapshotName strin cp := *snap snap.Tags.Close() - delete(store, snapshotName) + tbl.Delete(snapshotName) b.appendEventLocked(snapshotName, "cache-snapshot", "snapshot deleted") return &cp, nil @@ -1936,14 +1941,14 @@ func (b *InMemoryBackend) CopySnapshot( defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.snapshotsStore(region) + tbl := b.snapshotsStore(region) - src, ok := store[sourceSnapshotName] + src, ok := tbl.Get(sourceSnapshotName) if !ok { return nil, ErrSnapshotNotFound } - if _, targetExists := store[targetSnapshotName]; targetExists { + if _, targetExists := tbl.Get(targetSnapshotName); targetExists { return nil, ErrSnapshotAlreadyExists } @@ -1952,7 +1957,7 @@ func (b *InMemoryBackend) CopySnapshot( cp.ARN = b.snapshotARN(region, targetSnapshotName) cp.CreatedAt = time.Now() cp.Tags = tags.New("elasticache.snapshot." + targetSnapshotName + ".tags") - store[targetSnapshotName] = &cp + tbl.Put(&cp) b.appendEventLocked(targetSnapshotName, "cache-snapshot", "snapshot copied from "+sourceSnapshotName) result := cp @@ -2002,62 +2007,55 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() + b.resetLocked() +} + +// resetLocked does the work of Reset without acquiring b.mu, so it can also be +// called from Restore's incompatible-snapshot-version guard, which already +// holds the lock (calling the public, self-locking Reset there would +// deadlock). Must hold b.mu. +func (b *InMemoryBackend) resetLocked() { for _, regionClusters := range b.clusters { - for _, c := range regionClusters { + for _, c := range regionClusters.All() { b.releaseClusterLocked(c) } } - b.clusters = make(map[string]map[string]*Cluster) - b.replicationGroups = make(map[string]map[string]*ReplicationGroup) - b.parameterGroups = make(map[string]map[string]*CacheParameterGroup) - b.subnetGroups = make(map[string]map[string]*CacheSubnetGroup) - b.snapshots = make(map[string]map[string]*CacheSnapshot) - b.cacheSecurityGroups = make(map[string]map[string]*CacheSecurityGroup) + b.clusters = make(map[string]*store.Table[Cluster]) + b.replicationGroups = make(map[string]*store.Table[ReplicationGroup]) + b.parameterGroups = make(map[string]*store.Table[CacheParameterGroup]) + b.subnetGroups = make(map[string]*store.Table[CacheSubnetGroup]) + b.snapshots = make(map[string]*store.Table[CacheSnapshot]) + b.cacheSecurityGroups = make(map[string]*store.Table[CacheSecurityGroup]) b.cacheSecurityGroupIngress = make(map[string]map[string][]EC2SecurityGroupMembership) - b.globalReplicationGroups = make(map[string]*GlobalReplicationGroup) - b.serverlessCaches = make(map[string]map[string]*ServerlessCache) - b.serverlessCacheSnapshots = make(map[string]map[string]*ServerlessCacheSnapshot) - b.users = make(map[string]map[string]*User) - b.userGroups = make(map[string]map[string]*UserGroup) - b.reservedCacheNodes = make(map[string]map[string]*ReservedCacheNode) + b.serverlessCaches = make(map[string]*store.Table[ServerlessCache]) + b.serverlessCacheSnapshots = make(map[string]*store.Table[ServerlessCacheSnapshot]) + b.users = make(map[string]*store.Table[User]) + b.userGroups = make(map[string]*store.Table[UserGroup]) + b.reservedCacheNodes = make(map[string]*store.Table[ReservedCacheNode]) b.updateActions = nil b.events.reset() + // b.globalReplicationGroups is registered on b.registry at construction + // and must keep its identity (store.Register panics on a duplicate name), + // so it is cleared in place via the registry rather than reassigned. + b.registry.ResetAll() b.initDefaultParameterGroups() } func (b *InMemoryBackend) getGlobalReplicationGroup(id string) (*GlobalReplicationGroup, bool) { - grg, ok := b.globalReplicationGroups[id] - - return grg, ok + return b.globalReplicationGroups.Get(id) } func (b *InMemoryBackend) listGlobalReplicationGroups() []*GlobalReplicationGroup { - out := make([]*GlobalReplicationGroup, 0, len(b.globalReplicationGroups)) - for _, grg := range b.globalReplicationGroups { - out = append(out, grg) - } - - return out + return b.globalReplicationGroups.All() } -func (b *InMemoryBackend) putGlobalReplicationGroup(id string, grg *GlobalReplicationGroup) { - b.globalReplicationGroups[id] = grg +func (b *InMemoryBackend) putGlobalReplicationGroup(_ string, grg *GlobalReplicationGroup) { + b.globalReplicationGroups.Put(grg) } func (b *InMemoryBackend) deleteGlobalReplicationGroup(id string) { - delete(b.globalReplicationGroups, id) -} - -func (b *InMemoryBackend) cloneGlobalReplicationGroups() map[string]*GlobalReplicationGroup { - out := make(map[string]*GlobalReplicationGroup, len(b.globalReplicationGroups)) - maps.Copy(out, b.globalReplicationGroups) - - return out -} - -func (b *InMemoryBackend) setGlobalReplicationGroups(grgs map[string]*GlobalReplicationGroup) { - b.globalReplicationGroups = grgs + b.globalReplicationGroups.Delete(id) } func (b *InMemoryBackend) appendUpdateActionsLocked(actions ...*UpdateAction) { diff --git a/services/elasticache/backend_audit1.go b/services/elasticache/backend_audit1.go index 4717d122c..227ae9636 100644 --- a/services/elasticache/backend_audit1.go +++ b/services/elasticache/backend_audit1.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) @@ -124,12 +125,16 @@ type LogDeliveryConfig struct { // ReplicationGroupCreateOpts carries all fields for full replication-group creation. type ReplicationGroupCreateOpts struct { - Tags map[string]string - Engine string - EngineVersion string - ID string - Description string - ParameterGroupName string + Tags map[string]string + Engine string + EngineVersion string + ID string + Description string + ParameterGroupName string + // SnapshotName, when set, restores the new replication group from an + // existing snapshot: the snapshot must exist, and its engine/node type + // become defaults for any field the caller didn't explicitly set. + SnapshotName string MaintenanceWindow string TransitEncryptionMode string AuthToken string @@ -293,23 +298,44 @@ func (b *InMemoryBackend) CreateReplicationGroupFull( region := getRegion(ctx, b.region) rgStore := b.replicationGroupsStore(region) - if _, exists := rgStore[opts.ID]; exists { + if _, exists := rgStore.Get(opts.ID); exists { return nil, ErrReplicationGroupAlreadyExists } if opts.ParameterGroupName != "" { - if _, ok := b.parameterGroupsStore(region)[opts.ParameterGroupName]; !ok { + if _, ok := b.parameterGroupsStore(region).Get(opts.ParameterGroupName); !ok { return nil, ErrParameterGroupNotFound } } + if opts.SnapshotName != "" { + snap, ok := b.snapshotsStore(region).Get(opts.SnapshotName) + if !ok || isReaped(b.now(), snap.PendingStatus, snap.AvailableAt) { + return nil, ErrSnapshotNotFound + } + + // Restoring from a snapshot inherits its engine/version/node type for + // any field the caller didn't explicitly override. + if opts.Engine == "" { + opts.Engine = snap.Engine + } + + if opts.EngineVersion == "" { + opts.EngineVersion = snap.EngineVersion + } + + if opts.CacheNodeType == "" { + opts.CacheNodeType = snap.NodeType + } + } + if err := validateCreateOpts(opts); err != nil { return nil, err } rg := b.buildReplicationGroupFromCreateOpts(region, opts) b.markCreatingLocked(&rg.PendingStatus, &rg.AvailableAt) - rgStore[opts.ID] = rg + rgStore.Put(rg) b.appendEventLocked(opts.ID, "replication-group", "replication group created") return b.replicationGroupView(rg), nil @@ -408,17 +434,21 @@ func (b *InMemoryBackend) ModifyReplicationGroupFull( defer b.mu.Unlock() region := getRegion(ctx, b.region) - rg, exists := b.replicationGroupsStore(region)[id] + rg, exists := b.replicationGroupsStore(region).Get(id) if !exists { return nil, ErrReplicationGroupNotFound } if opts.ParameterGroupName != "" { - if _, ok := b.parameterGroupsStore(region)[opts.ParameterGroupName]; !ok { + if _, ok := b.parameterGroupsStore(region).Get(opts.ParameterGroupName); !ok { return nil, ErrParameterGroupNotFound } } + if err := validateTransitEncryptionModify(rg, opts); err != nil { + return nil, err + } + b.applyModifyOptsLocked(rg, opts) b.markTransitionLocked(&rg.PendingStatus, &rg.AvailableAt, statusModifying) b.appendEventLocked(id, "replication-group", "replication group modified") @@ -550,6 +580,35 @@ func applyAuthTokenModify(rg *ReplicationGroup, token, strategy string) { } } +// validateTransitEncryptionModify rejects switching TransitEncryptionMode to +// "required" when the replication group will end up without an auth token, +// matching AWS's rule that required-mode transit encryption needs an auth +// token enabled (either already present or set in the same request). +func validateTransitEncryptionModify(rg *ReplicationGroup, opts ReplicationGroupModifyOpts) error { + if opts.TransitEncryptionMode != transitEncryptionModeRequired { + return nil + } + + authTokenWillBeEnabled := rg.AuthTokenEnabled + + switch opts.AuthTokenUpdateStrategy { + case "SET", "ROTATE": + authTokenWillBeEnabled = true + case "DELETE": + authTokenWillBeEnabled = false + } + + if opts.AuthToken != "" { + authTokenWillBeEnabled = true + } + + if !authTokenWillBeEnabled { + return ErrTransitEncryptionModeInvalid + } + + return nil +} + // applyTransitEncryptionModify applies transit encryption mode change (gap #13). func applyTransitEncryptionModify(rg *ReplicationGroup, mode string) { if mode == "" { @@ -612,19 +671,19 @@ func (b *InMemoryBackend) TriggerAutoSnapshot(ctx context.Context, replicationGr defer b.mu.Unlock() region := getRegion(ctx, b.region) - rg, ok := b.replicationGroupsStore(region)[replicationGroupID] + rg, ok := b.replicationGroupsStore(region).Get(replicationGroupID) if !ok { return nil, ErrReplicationGroupNotFound } snapStore := b.snapshotsStore(region) snapName := buildAutoSnapshotName(replicationGroupID) - if _, exists := snapStore[snapName]; exists { + if _, exists := snapStore.Get(snapName); exists { return nil, ErrSnapshotAlreadyExists } snap := buildAutoSnapshot(b, region, snapName, rg) - snapStore[snapName] = snap + snapStore.Put(snap) b.appendEventLocked(replicationGroupID, "replication-group", "automated snapshot created: "+snapName) pruneExpiredSnapshots(b, snapStore, replicationGroupID, rg.SnapshotRetentionLimit) @@ -675,7 +734,7 @@ func sortAutoSnapshots(snaps []CacheSnapshot) { // pruneExpiredSnapshots removes automated snapshots beyond the retention limit (gap #14). func pruneExpiredSnapshots( _ *InMemoryBackend, - store map[string]*CacheSnapshot, + tbl *store.Table[CacheSnapshot], replicationGroupID string, retentionLimit int, ) { @@ -684,7 +743,7 @@ func pruneExpiredSnapshots( } var autoSnaps []CacheSnapshot - for _, s := range store { + for _, s := range tbl.All() { if s.ReplicationGroupID == replicationGroupID && s.SnapshotSource == "automated" { autoSnaps = append(autoSnaps, *s) } @@ -700,9 +759,9 @@ func pruneExpiredSnapshots( excess := len(autoSnaps) - retentionLimit for i := range excess { snap := autoSnaps[i] - if s, ok := store[snap.SnapshotName]; ok { + if s, ok := tbl.Get(snap.SnapshotName); ok { s.Tags.Close() - delete(store, snap.SnapshotName) + tbl.Delete(snap.SnapshotName) } } } diff --git a/services/elasticache/backend_batch2.go b/services/elasticache/backend_batch2.go index 82a389ae0..5bbf0fc54 100644 --- a/services/elasticache/backend_batch2.go +++ b/services/elasticache/backend_batch2.go @@ -63,8 +63,8 @@ func (b *InMemoryBackend) CreateServerlessCacheFull( defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.serverlessCachesStore(region) - if _, exists := store[opts.Name]; exists { + tbl := b.serverlessCachesStore(region) + if _, exists := tbl.Get(opts.Name); exists { return nil, ErrServerlessCacheAlreadyExists } @@ -116,7 +116,7 @@ func (b *InMemoryBackend) CreateServerlessCacheFull( } b.markCreatingLocked(&sc.PendingStatus, &sc.AvailableAt) - store[opts.Name] = sc + tbl.Put(sc) b.appendEventLocked(opts.Name, "serverless-cache", "serverless cache created") return b.serverlessCacheView(sc), nil @@ -148,7 +148,7 @@ func (b *InMemoryBackend) ModifyServerlessCacheFull( defer b.mu.Unlock() region := getRegion(ctx, b.region) - sc, ok := b.serverlessCachesStore(region)[name] + sc, ok := b.serverlessCachesStore(region).Get(name) if !ok { return nil, ErrServerlessCacheNotFound } @@ -194,8 +194,8 @@ func (b *InMemoryBackend) CreateSubnetGroupFull( defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.subnetGroupsStore(region) - if _, exists := store[name]; exists { + tbl := b.subnetGroupsStore(region) + if _, exists := tbl.Get(name); exists { return nil, ErrSubnetGroupAlreadyExists } @@ -207,7 +207,7 @@ func (b *InMemoryBackend) CreateSubnetGroupFull( ARN: b.subnetGroupARN(region, name), Tags: tags.New("elasticache.sg." + name + ".tags"), } - store[name] = sg + tbl.Put(sg) cp := *sg @@ -227,14 +227,14 @@ func (b *InMemoryBackend) CopySnapshotFull( defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.snapshotsStore(region) + tbl := b.snapshotsStore(region) - src, ok := store[sourceSnapshotName] + src, ok := tbl.Get(sourceSnapshotName) if !ok { return nil, ErrSnapshotNotFound } - if _, exists := store[targetSnapshotName]; exists { + if _, exists := tbl.Get(targetSnapshotName); exists { return nil, ErrSnapshotAlreadyExists } @@ -249,7 +249,7 @@ func (b *InMemoryBackend) CopySnapshotFull( cp.KmsKeyID = kmsKeyID } - store[targetSnapshotName] = &cp + tbl.Put(&cp) b.appendEventLocked(targetSnapshotName, "snapshot", "snapshot copied from "+sourceSnapshotName) result := cp @@ -272,13 +272,13 @@ func (b *InMemoryBackend) CreateUserGroupValidated( region := getRegion(ctx, b.region) ugStore := b.userGroupsStore(region) - if _, exists := ugStore[groupID]; exists { + if _, exists := ugStore.Get(groupID); exists { return nil, ErrUserGroupAlreadyExists } userStore := b.usersStore(region) for _, uid := range userIDs { - if _, ok := userStore[uid]; !ok { + if _, ok := userStore.Get(uid); !ok { return nil, fmt.Errorf("user %q: %w", uid, ErrGroupUserNotFound) } } @@ -297,7 +297,7 @@ func (b *InMemoryBackend) CreateUserGroupValidated( CreatedAt: time.Now(), Tags: tags.New("elasticache.usergroup." + groupID + ".tags"), } - ugStore[groupID] = ug + ugStore.Put(ug) b.appendEventLocked(groupID, "user-group", "user group created") cp := *ug @@ -315,20 +315,20 @@ func (b *InMemoryBackend) DeleteUserSafe(ctx context.Context, userID string) (*U defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.usersStore(region) - u, ok := store[userID] + tbl := b.usersStore(region) + u, ok := tbl.Get(userID) if !ok { return nil, ErrUserNotFound } - for _, ug := range b.userGroupsStore(region) { + for _, ug := range b.userGroupsStore(region).All() { if slices.Contains(ug.UserIDs, userID) { return nil, fmt.Errorf("user %q belongs to group %q: %w", userID, ug.UserGroupID, ErrUserNotInGroup) } } result := *u - delete(store, userID) + tbl.Delete(userID) b.appendEventLocked(userID, "user", "user deleted") return &result, nil diff --git a/services/elasticache/backend_lifecycle.go b/services/elasticache/backend_lifecycle.go index e0a1cfd7a..888728111 100644 --- a/services/elasticache/backend_lifecycle.go +++ b/services/elasticache/backend_lifecycle.go @@ -4,6 +4,7 @@ import ( "time" "github.com/blackbirdworks/gopherstack/pkgs/page" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) // ---------------------------------------- @@ -232,37 +233,60 @@ func (b *InMemoryBackend) markTransitionLocked(pending *string, until *time.Time // pruneRegionLocked reaps resources whose "deleting" deadline has elapsed, // releasing their engines/ports/tags/DNS registrations. Must hold b.mu. // This keeps tombstones from accumulating without any background goroutine. -func (b *InMemoryBackend) pruneRegionLocked(region string) { - now := b.now() - - for id, c := range b.clusters[region] { - if isReaped(now, c.PendingStatus, c.AvailableAt) { - b.releaseClusterLocked(c) - delete(b.clusters[region], id) - } - } - for id, rg := range b.replicationGroups[region] { - if isReaped(now, rg.PendingStatus, rg.AvailableAt) { - rg.Tags.Close() - delete(b.replicationGroups[region], id) - } - } - for id, sc := range b.serverlessCaches[region] { - if isReaped(now, sc.PendingStatus, sc.AvailableAt) { - sc.Tags.Close() - delete(b.serverlessCaches[region], id) - } +// pruneTable removes every reaped entry from t (a nil t, i.e. a region never +// touched yet, is a documented no-op). lifecycle extracts a value's pending +// status/deadline, release performs any resource-specific cleanup (closing +// tags/engines/etc.), and keyOf extracts the table key so the entry can be +// deleted. +func pruneTable[T any]( + now time.Time, + t *store.Table[T], + lifecycle func(*T) (pending string, until time.Time), + release func(*T), + keyOf func(*T) string, +) { + if t == nil { + return } - for id, snap := range b.snapshots[region] { - if isReaped(now, snap.PendingStatus, snap.AvailableAt) { - snap.Tags.Close() - delete(b.snapshots[region], id) + + for _, v := range t.All() { + pending, until := lifecycle(v) + if !isReaped(now, pending, until) { + continue } + + release(v) + t.Delete(keyOf(v)) } - for id, grg := range b.globalReplicationGroups { +} + +func (b *InMemoryBackend) pruneRegionLocked(region string) { + now := b.now() + + pruneTable(now, b.clusters[region], + func(c *Cluster) (string, time.Time) { return c.PendingStatus, c.AvailableAt }, + b.releaseClusterLocked, + func(c *Cluster) string { return c.ClusterID }) + + pruneTable(now, b.replicationGroups[region], + func(rg *ReplicationGroup) (string, time.Time) { return rg.PendingStatus, rg.AvailableAt }, + func(rg *ReplicationGroup) { rg.Tags.Close() }, + func(rg *ReplicationGroup) string { return rg.ReplicationGroupID }) + + pruneTable(now, b.serverlessCaches[region], + func(sc *ServerlessCache) (string, time.Time) { return sc.PendingStatus, sc.AvailableAt }, + func(sc *ServerlessCache) { sc.Tags.Close() }, + func(sc *ServerlessCache) string { return sc.Name }) + + pruneTable(now, b.snapshots[region], + func(snap *CacheSnapshot) (string, time.Time) { return snap.PendingStatus, snap.AvailableAt }, + func(snap *CacheSnapshot) { snap.Tags.Close() }, + func(snap *CacheSnapshot) string { return snap.SnapshotName }) + + for _, grg := range b.globalReplicationGroups.All() { if isReaped(now, grg.PendingStatus, grg.AvailableAt) { grg.Tags.Close() - b.deleteGlobalReplicationGroup(id) + b.deleteGlobalReplicationGroup(grg.GlobalReplicationGroupID) } } } diff --git a/services/elasticache/backend_new_ops.go b/services/elasticache/backend_new_ops.go index 4129db2ae..a8d851226 100644 --- a/services/elasticache/backend_new_ops.go +++ b/services/elasticache/backend_new_ops.go @@ -173,8 +173,8 @@ func (b *InMemoryBackend) CreateCacheSecurityGroup( defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.cacheSecurityGroupsStore(region) - if _, exists := store[name]; exists { + tbl := b.cacheSecurityGroupsStore(region) + if _, exists := tbl.Get(name); exists { return nil, ErrCacheSecurityGroupAlreadyExists } @@ -185,7 +185,7 @@ func (b *InMemoryBackend) CreateCacheSecurityGroup( OwnerID: b.accountID, Tags: tags.New("elasticache.sg." + name + ".tags"), } - store[name] = sg + tbl.Put(sg) return sg, nil } @@ -199,7 +199,7 @@ func (b *InMemoryBackend) AuthorizeCacheSecurityGroupIngress( defer b.mu.Unlock() region := getRegion(ctx, b.region) - sg, ok := b.cacheSecurityGroupsStore(region)[name] + sg, ok := b.cacheSecurityGroupsStore(region).Get(name) if !ok { return nil, ErrCacheSecurityGroupNotFound } @@ -236,7 +236,7 @@ func (b *InMemoryBackend) CreateGlobalReplicationGroup( region := getRegion(ctx, b.region) engine := engineRedis engineVersion := versionRedis710 - if rg, ok := b.replicationGroupsStore(region)[primaryReplicationGroupID]; ok { + if rg, ok := b.replicationGroupsStore(region).Get(primaryReplicationGroupID); ok { if rg.EngineVersion != "" { engineVersion = rg.EngineVersion } @@ -246,7 +246,7 @@ func (b *InMemoryBackend) CreateGlobalReplicationGroup( } nodeGroupCount := int32(1) - if rg, ok := b.replicationGroupsStore(region)[primaryReplicationGroupID]; ok && len(rg.NodeGroups) > 0 { + if rg, ok := b.replicationGroupsStore(region).Get(primaryReplicationGroupID); ok && len(rg.NodeGroups) > 0 { var cnt int32 for range rg.NodeGroups { cnt++ @@ -287,8 +287,8 @@ func (b *InMemoryBackend) CreateServerlessCache( defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.serverlessCachesStore(region) - if _, exists := store[name]; exists { + tbl := b.serverlessCachesStore(region) + if _, exists := tbl.Get(name); exists { return nil, ErrServerlessCacheAlreadyExists } @@ -319,7 +319,7 @@ func (b *InMemoryBackend) CreateServerlessCache( ReaderEndpoint: readerEp, } b.markCreatingLocked(&sc.PendingStatus, &sc.AvailableAt) - store[name] = sc + tbl.Put(sc) b.appendEventLocked(name, "serverless-cache", "serverless cache created") return b.serverlessCacheView(sc), nil @@ -339,11 +339,11 @@ func (b *InMemoryBackend) CreateServerlessCacheSnapshot( region := getRegion(ctx, b.region) snapStore := b.serverlessCacheSnapshotsStore(region) - if _, exists := snapStore[snapshotName]; exists { + if _, exists := snapStore.Get(snapshotName); exists { return nil, ErrServerlessCacheSnapshotExists } - if _, ok := b.serverlessCachesStore(region)[serverlessCacheName]; !ok { + if _, ok := b.serverlessCachesStore(region).Get(serverlessCacheName); !ok { return nil, ErrServerlessCacheNotFound } @@ -356,7 +356,7 @@ func (b *InMemoryBackend) CreateServerlessCacheSnapshot( CreatedAt: time.Now(), Tags: tags.New("elasticache.serverlesssnap." + snapshotName + ".tags"), } - snapStore[snapshotName] = snap + snapStore.Put(snap) return snap, nil } @@ -374,14 +374,14 @@ func (b *InMemoryBackend) CopyServerlessCacheSnapshot( defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.serverlessCacheSnapshotsStore(region) + tbl := b.serverlessCacheSnapshotsStore(region) - src, ok := store[sourceSnapshotName] + src, ok := tbl.Get(sourceSnapshotName) if !ok { return nil, ErrServerlessCacheSnapshotNotFound } - if _, exists := store[targetSnapshotName]; exists { + if _, exists := tbl.Get(targetSnapshotName); exists { return nil, ErrServerlessCacheSnapshotExists } @@ -390,7 +390,7 @@ func (b *InMemoryBackend) CopyServerlessCacheSnapshot( cp.ARN = b.serverlessCacheSnapshotARN(region, targetSnapshotName) cp.CreatedAt = time.Now() cp.Tags = tags.New("elasticache.serverlesssnap." + targetSnapshotName + ".tags") - store[targetSnapshotName] = &cp + tbl.Put(&cp) result := cp @@ -411,8 +411,8 @@ func (b *InMemoryBackend) CreateUser( defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.usersStore(region) - if _, exists := store[userID]; exists { + tbl := b.usersStore(region) + if _, exists := tbl.Get(userID); exists { return nil, ErrUserAlreadyExists } @@ -431,7 +431,7 @@ func (b *InMemoryBackend) CreateUser( CreatedAt: time.Now(), Tags: tags.New("elasticache.user." + userID + ".tags"), } - store[userID] = u + tbl.Put(u) b.appendEventLocked(userID, "user", "user created") return u, nil @@ -451,7 +451,7 @@ func (b *InMemoryBackend) batchUpdateActions( for _, rgID := range replicationGroupIDs { found := false for _, regionRGs := range b.replicationGroups { - if _, ok := regionRGs[rgID]; ok { + if _, ok := regionRGs.Get(rgID); ok { found = true break @@ -475,7 +475,7 @@ func (b *InMemoryBackend) batchUpdateActions( for _, clusterID := range cacheClusterIDs { found := false for _, regionClusters := range b.clusters { - if _, ok := regionClusters[clusterID]; ok { + if _, ok := regionClusters.Get(clusterID); ok { found = true break @@ -564,7 +564,7 @@ func (b *InMemoryBackend) CompleteMigration( defer b.mu.Unlock() region := getRegion(ctx, b.region) - rg, ok := b.replicationGroupsStore(region)[replicationGroupID] + rg, ok := b.replicationGroupsStore(region).Get(replicationGroupID) if !ok { return nil, ErrReplicationGroupNotFound } @@ -583,7 +583,7 @@ func (b *InMemoryBackend) CompleteMigration( func (b *InMemoryBackend) AddCacheSecurityGroupInternal(sg *CacheSecurityGroup) { b.mu.Lock("AddCacheSecurityGroupInternal") defer b.mu.Unlock() - b.cacheSecurityGroupsStore(b.region)[sg.Name] = sg + b.cacheSecurityGroupsStore(b.region).Put(sg) } // AddGlobalReplicationGroupInternal seeds a global replication group for testing. @@ -597,19 +597,19 @@ func (b *InMemoryBackend) AddGlobalReplicationGroupInternal(grg *GlobalReplicati func (b *InMemoryBackend) AddServerlessCacheInternal(sc *ServerlessCache) { b.mu.Lock("AddServerlessCacheInternal") defer b.mu.Unlock() - b.serverlessCachesStore(b.region)[sc.Name] = sc + b.serverlessCachesStore(b.region).Put(sc) } // AddServerlessCacheSnapshotInternal seeds a serverless cache snapshot for testing. func (b *InMemoryBackend) AddServerlessCacheSnapshotInternal(snap *ServerlessCacheSnapshot) { b.mu.Lock("AddServerlessCacheSnapshotInternal") defer b.mu.Unlock() - b.serverlessCacheSnapshotsStore(b.region)[snap.Name] = snap + b.serverlessCacheSnapshotsStore(b.region).Put(snap) } // AddUserInternal seeds a user for testing. func (b *InMemoryBackend) AddUserInternal(u *User) { b.mu.Lock("AddUserInternal") defer b.mu.Unlock() - b.usersStore(b.region)[u.UserID] = u + b.usersStore(b.region).Put(u) } diff --git a/services/elasticache/backend_ops2.go b/services/elasticache/backend_ops2.go index 49acffcab..894cfd178 100644 --- a/services/elasticache/backend_ops2.go +++ b/services/elasticache/backend_ops2.go @@ -215,20 +215,20 @@ func (b *InMemoryBackend) DeleteUser(ctx context.Context, userID string) (*User, defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.usersStore(region) - u, ok := store[userID] + tbl := b.usersStore(region) + u, ok := tbl.Get(userID) if !ok { return nil, ErrUserNotFound } - for _, ug := range b.userGroupsStore(region) { + for _, ug := range b.userGroupsStore(region).All() { if slices.Contains(ug.UserIDs, userID) { return nil, fmt.Errorf("user %q belongs to group %q: %w", userID, ug.UserGroupID, ErrUserNotInGroup) } } result := *u - delete(store, userID) + tbl.Delete(userID) b.appendEventLocked(userID, "user", "user deleted") return &result, nil @@ -259,7 +259,7 @@ func (b *InMemoryBackend) ModifyUser( defer b.mu.Unlock() region := getRegion(ctx, b.region) - u, ok := b.usersStore(region)[userID] + u, ok := b.usersStore(region).Get(userID) if !ok { return nil, ErrUserNotFound } @@ -284,8 +284,8 @@ func (b *InMemoryBackend) CreateUserGroup( defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.userGroupsStore(region) - if _, exists := store[groupID]; exists { + tbl := b.userGroupsStore(region) + if _, exists := tbl.Get(groupID); exists { return nil, ErrUserGroupAlreadyExists } @@ -303,7 +303,7 @@ func (b *InMemoryBackend) CreateUserGroup( CreatedAt: time.Now(), Tags: tags.New("elasticache.usergroup." + groupID + ".tags"), } - store[groupID] = ug + tbl.Put(ug) b.appendEventLocked(groupID, "user-group", "user group created") return ug, nil @@ -315,14 +315,14 @@ func (b *InMemoryBackend) DeleteUserGroup(ctx context.Context, groupID string) ( defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.userGroupsStore(region) - ug, ok := store[groupID] + tbl := b.userGroupsStore(region) + ug, ok := tbl.Get(groupID) if !ok { return nil, ErrUserGroupNotFound } result := *ug - delete(store, groupID) + tbl.Delete(groupID) b.appendEventLocked(groupID, "user-group", "user group deleted") return &result, nil @@ -353,7 +353,7 @@ func (b *InMemoryBackend) ModifyUserGroup( defer b.mu.Unlock() region := getRegion(ctx, b.region) - ug, ok := b.userGroupsStore(region)[groupID] + ug, ok := b.userGroupsStore(region).Get(groupID) if !ok { return nil, ErrUserGroupNotFound } @@ -674,8 +674,8 @@ func (b *InMemoryBackend) PurchaseReservedCacheNodesOffering( } region := getRegion(ctx, b.region) - store := b.reservedCacheNodesStore(region) - if _, exists := store[reservedCacheNodeID]; exists { + tbl := b.reservedCacheNodesStore(region) + if _, exists := tbl.Get(reservedCacheNodeID); exists { return nil, fmt.Errorf("reserved cache node %q: %w", reservedCacheNodeID, ErrReservedCacheNodeAlreadyExists) } @@ -693,7 +693,7 @@ func (b *InMemoryBackend) PurchaseReservedCacheNodesOffering( CacheNodeCount: cacheNodeCount, StartTime: time.Now(), } - store[reservedCacheNodeID] = rcn + tbl.Put(rcn) b.appendEventLocked(reservedCacheNodeID, "reserved-cache-node", "reserved cache node purchased") return rcn, nil @@ -706,8 +706,8 @@ func (b *InMemoryBackend) DeleteServerlessCache(ctx context.Context, name string region := getRegion(ctx, b.region) b.pruneRegionLocked(region) - store := b.serverlessCachesStore(region) - sc, ok := store[name] + tbl := b.serverlessCachesStore(region) + sc, ok := tbl.Get(name) if !ok || isReaped(b.now(), sc.PendingStatus, sc.AvailableAt) { return nil, ErrServerlessCacheNotFound } @@ -722,7 +722,7 @@ func (b *InMemoryBackend) DeleteServerlessCache(ctx context.Context, name string result := *sc sc.Tags.Close() - delete(store, name) + tbl.Delete(name) b.appendEventLocked(name, "serverless-cache", "serverless cache deleted") return &result, nil @@ -737,14 +737,14 @@ func (b *InMemoryBackend) DeleteServerlessCacheSnapshot( defer b.mu.Unlock() region := getRegion(ctx, b.region) - store := b.serverlessCacheSnapshotsStore(region) - snap, ok := store[name] + tbl := b.serverlessCacheSnapshotsStore(region) + snap, ok := tbl.Get(name) if !ok { return nil, ErrServerlessCacheSnapshotNotFound } result := *snap - delete(store, name) + tbl.Delete(name) b.appendEventLocked(name, "serverless-cache-snapshot", "serverless cache snapshot deleted") return &result, nil @@ -777,10 +777,10 @@ func (b *InMemoryBackend) DescribeServerlessCacheSnapshots( defer b.mu.RUnlock() region := getRegion(ctx, b.region) - store := b.serverlessCacheSnapshotsStore(region) + tbl := b.serverlessCacheSnapshotsStore(region) if snapshotName != "" { - snap, ok := store[snapshotName] + snap, ok := tbl.Get(snapshotName) if !ok { return page.Page[ServerlessCacheSnapshot]{}, ErrServerlessCacheSnapshotNotFound } @@ -788,8 +788,9 @@ func (b *InMemoryBackend) DescribeServerlessCacheSnapshots( return page.Page[ServerlessCacheSnapshot]{Data: []ServerlessCacheSnapshot{*snap}}, nil } - out := make([]ServerlessCacheSnapshot, 0, len(store)) - for _, snap := range store { + all := tbl.All() + out := make([]ServerlessCacheSnapshot, 0, len(all)) + for _, snap := range all { if serverlessCacheName != "" && snap.ServerlessCacheName != serverlessCacheName { continue } @@ -811,7 +812,7 @@ func (b *InMemoryBackend) ExportServerlessCacheSnapshot( defer b.mu.Unlock() region := getRegion(ctx, b.region) - snap, ok := b.serverlessCacheSnapshotsStore(region)[snapshotName] + snap, ok := b.serverlessCacheSnapshotsStore(region).Get(snapshotName) if !ok { return nil, ErrServerlessCacheSnapshotNotFound } @@ -830,7 +831,7 @@ func (b *InMemoryBackend) ModifyServerlessCache( defer b.mu.Unlock() region := getRegion(ctx, b.region) - sc, ok := b.serverlessCachesStore(region)[name] + sc, ok := b.serverlessCachesStore(region).Get(name) if !ok { return nil, ErrServerlessCacheNotFound } @@ -850,7 +851,7 @@ func (b *InMemoryBackend) StartMigration(ctx context.Context, replicationGroupID defer b.mu.Unlock() region := getRegion(ctx, b.region) - rg, ok := b.replicationGroupsStore(region)[replicationGroupID] + rg, ok := b.replicationGroupsStore(region).Get(replicationGroupID) if !ok { return nil, ErrReplicationGroupNotFound } @@ -867,7 +868,7 @@ func (b *InMemoryBackend) TestMigration(ctx context.Context, replicationGroupID defer b.mu.Unlock() region := getRegion(ctx, b.region) - rg, ok := b.replicationGroupsStore(region)[replicationGroupID] + rg, ok := b.replicationGroupsStore(region).Get(replicationGroupID) if !ok { return nil, ErrReplicationGroupNotFound } @@ -887,7 +888,7 @@ func (b *InMemoryBackend) IncreaseReplicaCount( defer b.mu.Unlock() region := getRegion(ctx, b.region) - rg, ok := b.replicationGroupsStore(region)[replicationGroupID] + rg, ok := b.replicationGroupsStore(region).Get(replicationGroupID) if !ok { return nil, ErrReplicationGroupNotFound } @@ -913,7 +914,7 @@ func (b *InMemoryBackend) DecreaseReplicaCount( defer b.mu.Unlock() region := getRegion(ctx, b.region) - rg, ok := b.replicationGroupsStore(region)[replicationGroupID] + rg, ok := b.replicationGroupsStore(region).Get(replicationGroupID) if !ok { return nil, ErrReplicationGroupNotFound } @@ -940,7 +941,7 @@ func (b *InMemoryBackend) ModifyReplicationGroupShardConfiguration( defer b.mu.Unlock() region := getRegion(ctx, b.region) - rg, ok := b.replicationGroupsStore(region)[replicationGroupID] + rg, ok := b.replicationGroupsStore(region).Get(replicationGroupID) if !ok { return nil, ErrReplicationGroupNotFound } @@ -1001,7 +1002,7 @@ func (b *InMemoryBackend) RebootCacheCluster( defer b.mu.Unlock() region := getRegion(ctx, b.region) - c, ok := b.clustersStore(region)[clusterID] + c, ok := b.clustersStore(region).Get(clusterID) if !ok { return nil, ErrClusterNotFound } @@ -1027,11 +1028,11 @@ func (b *InMemoryBackend) DeleteCacheSecurityGroup(ctx context.Context, name str region := getRegion(ctx, b.region) sgStore := b.cacheSecurityGroupsStore(region) - if _, ok := sgStore[name]; !ok { + if _, ok := sgStore.Get(name); !ok { return ErrCacheSecurityGroupNotFound } - delete(sgStore, name) + sgStore.Delete(name) delete(b.cacheSecurityGroupIngressStore(region), name) return nil @@ -1061,7 +1062,7 @@ func (b *InMemoryBackend) RevokeCacheSecurityGroupIngress( defer b.mu.Unlock() region := getRegion(ctx, b.region) - sg, ok := b.cacheSecurityGroupsStore(region)[name] + sg, ok := b.cacheSecurityGroupsStore(region).Get(name) if !ok { return nil, ErrCacheSecurityGroupNotFound } @@ -1354,5 +1355,5 @@ func (b *InMemoryBackend) ListAllowedNodeTypeModifications(_ context.Context, _, func (b *InMemoryBackend) AddUserGroupInternal(ug *UserGroup) { b.mu.Lock("AddUserGroupInternal") defer b.mu.Unlock() - b.userGroupsStore(b.region)[ug.UserGroupID] = ug + b.userGroupsStore(b.region).Put(ug) } diff --git a/services/elasticache/backend_paged.go b/services/elasticache/backend_paged.go index 95281230d..e6508950d 100644 --- a/services/elasticache/backend_paged.go +++ b/services/elasticache/backend_paged.go @@ -4,6 +4,7 @@ import ( "sort" "github.com/blackbirdworks/gopherstack/pkgs/page" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) // describePaged handles the common lookup-or-paginate pattern for Describe* operations. @@ -11,7 +12,7 @@ import ( // Otherwise all items are collected, optionally filtered, sorted by key(), and paginated. // A nil filter includes every item. func describePaged[T any]( - store map[string]*T, + t *store.Table[T], id string, notFoundErr error, filter func(T) bool, @@ -20,7 +21,7 @@ func describePaged[T any]( maxRecords int, ) (page.Page[T], error) { if id != "" { - item, exists := store[id] + item, exists := t.Get(id) if !exists { return page.Page[T]{}, notFoundErr } @@ -28,8 +29,9 @@ func describePaged[T any]( return page.Page[T]{Data: []T{*item}}, nil } - out := make([]T, 0, len(store)) - for _, item := range store { + all := t.All() + out := make([]T, 0, len(all)) + for _, item := range all { if filter == nil || filter(*item) { out = append(out, *item) } diff --git a/services/elasticache/export_test.go b/services/elasticache/export_test.go index 97aa8cbb3..a6152f588 100644 --- a/services/elasticache/export_test.go +++ b/services/elasticache/export_test.go @@ -7,7 +7,7 @@ func CacheSecurityGroupCount(b *InMemoryBackend) int { total := 0 for _, regionStore := range b.cacheSecurityGroups { - total += len(regionStore) + total += regionStore.Len() } return total @@ -28,7 +28,7 @@ func ServerlessCacheCount(b *InMemoryBackend) int { total := 0 for _, regionStore := range b.serverlessCaches { - total += len(regionStore) + total += regionStore.Len() } return total @@ -41,7 +41,7 @@ func ServerlessCacheSnapshotCount(b *InMemoryBackend) int { total := 0 for _, regionStore := range b.serverlessCacheSnapshots { - total += len(regionStore) + total += regionStore.Len() } return total @@ -54,7 +54,7 @@ func UserCount(b *InMemoryBackend) int { total := 0 for _, regionStore := range b.users { - total += len(regionStore) + total += regionStore.Len() } return total @@ -67,7 +67,7 @@ func UserGroupCount(b *InMemoryBackend) int { total := 0 for _, regionStore := range b.userGroups { - total += len(regionStore) + total += regionStore.Len() } return total @@ -80,7 +80,7 @@ func ReservedCacheNodeCount(b *InMemoryBackend) int { total := 0 for _, regionStore := range b.reservedCacheNodes { - total += len(regionStore) + total += regionStore.Len() } return total @@ -99,7 +99,7 @@ func AddClusterInRGInternal(b *InMemoryBackend, clusterID, replicationGroupID st b.mu.Lock("AddClusterInRGInternal") defer b.mu.Unlock() - b.clustersStore(b.region)[clusterID] = &Cluster{ + b.clustersStore(b.region).Put(&Cluster{ ClusterID: clusterID, ReplicationGroupID: replicationGroupID, Engine: engineRedis, @@ -108,7 +108,7 @@ func AddClusterInRGInternal(b *InMemoryBackend, clusterID, replicationGroupID st NodeType: nodeTypeT3Micro, Region: b.region, ARN: b.clusterARN(b.region, clusterID), - } + }) } // AddSnapshotInternal seeds an automated snapshot for a given replication group (uses default region). @@ -116,11 +116,11 @@ func AddSnapshotInternal(b *InMemoryBackend, snapshotName, replicationGroupID, s b.mu.Lock("AddSnapshotInternal") defer b.mu.Unlock() - b.snapshotsStore(b.region)[snapshotName] = &CacheSnapshot{ + b.snapshotsStore(b.region).Put(&CacheSnapshot{ SnapshotName: snapshotName, ReplicationGroupID: replicationGroupID, SnapshotSource: snapshotSource, Status: statusAvailable, ARN: b.snapshotARN(b.region, snapshotName), - } + }) } diff --git a/services/elasticache/handler.go b/services/elasticache/handler.go index c4bf87d63..866a6925a 100644 --- a/services/elasticache/handler.go +++ b/services/elasticache/handler.go @@ -423,6 +423,31 @@ func (h *Handler) createCacheCluster(ctx context.Context, c *echo.Context, form } } + // AWS supports restoring a new cluster from an existing snapshot: the + // snapshot must exist, and its engine/node type become the defaults for + // any field the caller didn't explicitly override. + if snapshotName := form.Get("SnapshotName"); snapshotName != "" { + // SnapshotNotFoundFault isn't in CreateCacheCluster's modeled error + // list (api-2.json), so aws-sdk-go-v2 has no case for it in this + // operation's error deserializer and would fall back to a generic + // error; AWS instead surfaces a missing/invalid snapshot here as + // InvalidParameterValue, which the SDK does model for this op. + snaps, snapErr := h.Backend.DescribeSnapshots(ctx, snapshotName, "", "", "", "", 0) + if snapErr != nil || len(snaps.Data) == 0 { + return xmlError(c, http.StatusBadRequest, "InvalidParameterValue", + fmt.Sprintf("Cache cluster snapshot not found: %s", snapshotName)) + } + + src := snaps.Data[0] + if engine == "" { + engine = src.Engine + } + + if nodeType == "" { + nodeType = src.NodeType + } + } + cluster, err := h.Backend.CreateClusterWithOptions(ctx, id, engine, @@ -438,10 +463,10 @@ func (h *Handler) createCacheCluster(ctx context.Context, c *echo.Context, form return xmlError(c, http.StatusBadRequest, "CacheClusterAlreadyExists", "Cache cluster already exists") } if errors.Is(err, ErrParameterGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheParameterGroupNotFound", "Cache parameter group not found") + return xmlError(c, http.StatusNotFound, "CacheParameterGroupNotFound", "Cache parameter group not found") } if errors.Is(err, ErrInvalidParameterGroupFamily) { - return xmlError(c, http.StatusBadRequest, "InvalidParameterGroupFamily", err.Error()) + return xmlError(c, http.StatusBadRequest, "InvalidParameterValue", err.Error()) } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -464,7 +489,7 @@ func (h *Handler) deleteCacheCluster(ctx context.Context, c *echo.Context, form clusters, descErr := h.Backend.DescribeClusters(ctx, id, "", 0, false) if descErr != nil { if errors.Is(descErr, ErrClusterNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheClusterNotFound", "Cache cluster not found") + return xmlError(c, http.StatusNotFound, "CacheClusterNotFound", "Cache cluster not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", descErr.Error()) @@ -472,7 +497,7 @@ func (h *Handler) deleteCacheCluster(ctx context.Context, c *echo.Context, form cl := clusters.Data[0] if err := h.Backend.DeleteCluster(ctx, id); err != nil { if errors.Is(err, ErrClusterNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheClusterNotFound", "Cache cluster not found") + return xmlError(c, http.StatusNotFound, "CacheClusterNotFound", "Cache cluster not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -498,7 +523,7 @@ func (h *Handler) describeCacheClusters(ctx context.Context, c *echo.Context, fo p, err := h.Backend.DescribeClusters(ctx, id, marker, maxRecords, notInRG) if err != nil { if errors.Is(err, ErrClusterNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheClusterNotFound", "Cache cluster not found") + return xmlError(c, http.StatusNotFound, "CacheClusterNotFound", "Cache cluster not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -583,6 +608,7 @@ func parseCreateReplicationGroupOpts(form url.Values) ReplicationGroupCreateOpts ID: form.Get("ReplicationGroupId"), Description: form.Get("ReplicationGroupDescription"), ParameterGroupName: form.Get("CacheParameterGroupName"), + SnapshotName: form.Get("SnapshotName"), MaintenanceWindow: form.Get("PreferredMaintenanceWindow"), SnapshotWindow: form.Get("SnapshotWindow"), AuthToken: form.Get("AuthToken"), @@ -669,7 +695,11 @@ func mapReplicationGroupCreateErr(c *echo.Context, err error) error { case errors.Is(err, ErrReplicationGroupAlreadyExists): return xmlError(c, http.StatusBadRequest, "ReplicationGroupAlreadyExists", "Replication group already exists") case errors.Is(err, ErrParameterGroupNotFound): - return xmlError(c, http.StatusBadRequest, "CacheParameterGroupNotFound", "Cache parameter group not found") + return xmlError(c, http.StatusNotFound, "CacheParameterGroupNotFound", "Cache parameter group not found") + case errors.Is(err, ErrSnapshotNotFound): + // Same rationale as createCacheCluster: SnapshotNotFoundFault isn't in + // CreateReplicationGroup's modeled error list either. + return xmlError(c, http.StatusBadRequest, "InvalidParameterValue", "Cache cluster snapshot not found") case errors.Is(err, ErrDataTieringInvalid): return xmlError(c, http.StatusBadRequest, "InvalidParameterValue", err.Error()) case errors.Is(err, ErrAuthTokenRequiredForMode): @@ -684,7 +714,7 @@ func (h *Handler) deleteReplicationGroup(ctx context.Context, c *echo.Context, f rgs, descErr := h.Backend.DescribeReplicationGroups(ctx, id, "", 0) if descErr != nil { if errors.Is(descErr, ErrReplicationGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "ReplicationGroupNotFound", "Replication group not found") + return xmlError(c, http.StatusNotFound, "ReplicationGroupNotFoundFault", "Replication group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", descErr.Error()) @@ -692,7 +722,7 @@ func (h *Handler) deleteReplicationGroup(ctx context.Context, c *echo.Context, f rg := rgs.Data[0] if err := h.Backend.DeleteReplicationGroup(ctx, id); err != nil { if errors.Is(err, ErrReplicationGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "ReplicationGroupNotFound", "Replication group not found") + return xmlError(c, http.StatusNotFound, "ReplicationGroupNotFoundFault", "Replication group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -929,7 +959,7 @@ func (h *Handler) describeReplicationGroups(ctx context.Context, c *echo.Context p, err := h.Backend.DescribeReplicationGroups(ctx, id, marker, maxRecords) if err != nil { if errors.Is(err, ErrReplicationGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "ReplicationGroupNotFound", "Replication group not found") + return xmlError(c, http.StatusNotFound, "ReplicationGroupNotFoundFault", "Replication group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1021,10 +1051,10 @@ func (h *Handler) modifyCacheCluster(ctx context.Context, c *echo.Context, form ) if err != nil { if errors.Is(err, ErrClusterNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheClusterNotFound", "Cache cluster not found") + return xmlError(c, http.StatusNotFound, "CacheClusterNotFound", "Cache cluster not found") } if errors.Is(err, ErrParameterGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheParameterGroupNotFound", "Cache parameter group not found") + return xmlError(c, http.StatusNotFound, "CacheParameterGroupNotFound", "Cache parameter group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1125,9 +1155,11 @@ func parseModifyReplicationGroupOpts(form url.Values) ReplicationGroupModifyOpts func mapReplicationGroupModifyErr(c *echo.Context, err error) error { switch { case errors.Is(err, ErrReplicationGroupNotFound): - return xmlError(c, http.StatusBadRequest, "ReplicationGroupNotFound", "Replication group not found") + return xmlError(c, http.StatusNotFound, "ReplicationGroupNotFoundFault", "Replication group not found") case errors.Is(err, ErrParameterGroupNotFound): - return xmlError(c, http.StatusBadRequest, "CacheParameterGroupNotFound", "Cache parameter group not found") + return xmlError(c, http.StatusNotFound, "CacheParameterGroupNotFound", "Cache parameter group not found") + case errors.Is(err, ErrTransitEncryptionModeInvalid): + return xmlError(c, http.StatusBadRequest, "InvalidParameterCombination", err.Error()) default: return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) } @@ -1192,7 +1224,7 @@ func (h *Handler) deleteCacheParameterGroup(ctx context.Context, c *echo.Context if err := h.Backend.DeleteParameterGroup(ctx, name); err != nil { if errors.Is(err, ErrParameterGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheParameterGroupNotFound", "Cache parameter group not found") + return xmlError(c, http.StatusNotFound, "CacheParameterGroupNotFound", "Cache parameter group not found") } if errors.Is(err, ErrParameterGroupDefaultNotModifiable) { return xmlError( @@ -1235,7 +1267,7 @@ func (h *Handler) describeCacheParameterGroups(ctx context.Context, c *echo.Cont p, err := h.Backend.DescribeParameterGroups(ctx, name, marker, maxRecords) if err != nil { if errors.Is(err, ErrParameterGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheParameterGroupNotFound", "Cache parameter group not found") + return xmlError(c, http.StatusNotFound, "CacheParameterGroupNotFound", "Cache parameter group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1270,7 +1302,7 @@ func (h *Handler) modifyCacheParameterGroup(ctx context.Context, c *echo.Context pg, err := h.Backend.ModifyParameterGroup(ctx, name, params) if err != nil { if errors.Is(err, ErrParameterGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheParameterGroupNotFound", "Cache parameter group not found") + return xmlError(c, http.StatusNotFound, "CacheParameterGroupNotFound", "Cache parameter group not found") } if errors.Is(err, ErrParameterGroupDefaultNotModifiable) { return xmlError( @@ -1314,7 +1346,7 @@ func (h *Handler) resetCacheParameterGroup(ctx context.Context, c *echo.Context, pg, err := h.Backend.ResetParameterGroup(ctx, name, paramNames, resetAll) if err != nil { if errors.Is(err, ErrParameterGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheParameterGroupNotFound", "Cache parameter group not found") + return xmlError(c, http.StatusNotFound, "CacheParameterGroupNotFound", "Cache parameter group not found") } if errors.Is(err, ErrParameterGroupDefaultNotModifiable) { return xmlError( @@ -1383,7 +1415,7 @@ func (h *Handler) describeCacheParameters(ctx context.Context, c *echo.Context, p, err := h.Backend.DescribeParameters(ctx, name, marker, maxRecords) if err != nil { if errors.Is(err, ErrParameterGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheParameterGroupNotFound", "Cache parameter group not found") + return xmlError(c, http.StatusNotFound, "CacheParameterGroupNotFound", "Cache parameter group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1468,7 +1500,7 @@ func (h *Handler) deleteCacheSubnetGroup(ctx context.Context, c *echo.Context, f if err := h.Backend.DeleteSubnetGroup(ctx, name); err != nil { if errors.Is(err, ErrSubnetGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheSubnetGroupNotFound", "Cache subnet group not found") + return xmlError(c, http.StatusBadRequest, "CacheSubnetGroupNotFoundFault", "Cache subnet group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1503,7 +1535,7 @@ func (h *Handler) describeCacheSubnetGroups(ctx context.Context, c *echo.Context p, err := h.Backend.DescribeSubnetGroups(ctx, name, marker, maxRecords) if err != nil { if errors.Is(err, ErrSubnetGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheSubnetGroupNotFound", "Cache subnet group not found") + return xmlError(c, http.StatusBadRequest, "CacheSubnetGroupNotFoundFault", "Cache subnet group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1529,7 +1561,7 @@ func (h *Handler) modifyCacheSubnetGroup(ctx context.Context, c *echo.Context, f sg, err := h.Backend.ModifySubnetGroup(ctx, name, desc, subnetIDs) if err != nil { if errors.Is(err, ErrSubnetGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheSubnetGroupNotFound", "Cache subnet group not found") + return xmlError(c, http.StatusBadRequest, "CacheSubnetGroupNotFoundFault", "Cache subnet group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1595,10 +1627,10 @@ func (h *Handler) createSnapshot(ctx context.Context, c *echo.Context, form url. return xmlError(c, http.StatusBadRequest, "SnapshotAlreadyExistsFault", "Snapshot already exists") } if errors.Is(err, ErrClusterNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheClusterNotFound", "Cache cluster not found") + return xmlError(c, http.StatusNotFound, "CacheClusterNotFound", "Cache cluster not found") } if errors.Is(err, ErrReplicationGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "ReplicationGroupNotFound", "Replication group not found") + return xmlError(c, http.StatusNotFound, "ReplicationGroupNotFoundFault", "Replication group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1622,7 +1654,7 @@ func (h *Handler) deleteSnapshot(ctx context.Context, c *echo.Context, form url. snap, err := h.Backend.DeleteSnapshot(ctx, snapshotName) if err != nil { if errors.Is(err, ErrSnapshotNotFound) { - return xmlError(c, http.StatusBadRequest, "SnapshotNotFoundFault", "Snapshot not found") + return xmlError(c, http.StatusNotFound, "SnapshotNotFoundFault", "Snapshot not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1652,7 +1684,7 @@ func (h *Handler) describeSnapshots(ctx context.Context, c *echo.Context, form u ) if err != nil { if errors.Is(err, ErrSnapshotNotFound) { - return xmlError(c, http.StatusBadRequest, "SnapshotNotFoundFault", "Snapshot not found") + return xmlError(c, http.StatusNotFound, "SnapshotNotFoundFault", "Snapshot not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1687,7 +1719,7 @@ func (h *Handler) copySnapshot(ctx context.Context, c *echo.Context, form url.Va snap, err := h.Backend.CopySnapshot(ctx, sourceSnapshotName, targetSnapshotName) if err != nil { if errors.Is(err, ErrSnapshotNotFound) { - return xmlError(c, http.StatusBadRequest, "SnapshotNotFoundFault", "Source snapshot not found") + return xmlError(c, http.StatusNotFound, "SnapshotNotFoundFault", "Source snapshot not found") } if errors.Is(err, ErrSnapshotAlreadyExists) { return xmlError(c, http.StatusBadRequest, "SnapshotAlreadyExistsFault", "Target snapshot already exists") @@ -1799,7 +1831,7 @@ func (h *Handler) testFailoverReplicationGroup(ctx context.Context, c *echo.Cont rg, err := h.Backend.FailoverReplicationGroup(ctx, id, nodeGroupID) if err != nil { if errors.Is(err, ErrReplicationGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "ReplicationGroupNotFound", "Replication group not found") + return xmlError(c, http.StatusNotFound, "ReplicationGroupNotFoundFault", "Replication group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) diff --git a/services/elasticache/handler_new_ops.go b/services/elasticache/handler_new_ops.go index f4dc45bfa..61024bc18 100644 --- a/services/elasticache/handler_new_ops.go +++ b/services/elasticache/handler_new_ops.go @@ -74,7 +74,7 @@ func (h *Handler) authorizeCacheSecurityGroupIngress(ctx context.Context, c *ech sg, err := h.Backend.AuthorizeCacheSecurityGroupIngress(ctx, name, ec2SecurityGroupName, ec2SecurityGroupOwnerID) if err != nil { if errors.Is(err, ErrCacheSecurityGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheSecurityGroupNotFound", "Cache security group not found") + return xmlError(c, http.StatusNotFound, "CacheSecurityGroupNotFound", "Cache security group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -254,7 +254,7 @@ func (h *Handler) createServerlessCacheSnapshot(ctx context.Context, c *echo.Con ) } if errors.Is(err, ErrServerlessCacheNotFound) { - return xmlError(c, http.StatusBadRequest, "ServerlessCacheNotFound", "Serverless cache not found") + return xmlError(c, http.StatusNotFound, "ServerlessCacheNotFoundFault", "Serverless cache not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -285,7 +285,7 @@ func (h *Handler) copyServerlessCacheSnapshot(ctx context.Context, c *echo.Conte if errors.Is(err, ErrServerlessCacheSnapshotNotFound) { return xmlError( c, - http.StatusBadRequest, + http.StatusNotFound, "ServerlessCacheSnapshotNotFoundFault", "Source serverless cache snapshot not found", ) @@ -465,7 +465,7 @@ func (h *Handler) completeMigration(ctx context.Context, c *echo.Context, form u rg, err := h.Backend.CompleteMigration(ctx, replicationGroupID, force) if err != nil { if errors.Is(err, ErrReplicationGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "ReplicationGroupNotFound", "Replication group not found") + return xmlError(c, http.StatusNotFound, "ReplicationGroupNotFoundFault", "Replication group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) diff --git a/services/elasticache/handler_ops2.go b/services/elasticache/handler_ops2.go index 958092c1b..22f70d96e 100644 --- a/services/elasticache/handler_ops2.go +++ b/services/elasticache/handler_ops2.go @@ -194,11 +194,11 @@ func (h *Handler) deleteUser(ctx context.Context, c *echo.Context, form url.Valu u, err := h.Backend.DeleteUser(ctx, userID) if err != nil { if errors.Is(err, ErrUserNotFound) { - return xmlError(c, http.StatusBadRequest, "UserNotFound", "User not found") + return xmlError(c, http.StatusNotFound, "UserNotFound", "User not found") } if errors.Is(err, ErrUserNotInGroup) { - return xmlError(c, http.StatusBadRequest, "InvalidParameterValueException", err.Error()) + return xmlError(c, http.StatusBadRequest, "InvalidParameterValue", err.Error()) } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -235,7 +235,7 @@ func (h *Handler) describeUsers(ctx context.Context, c *echo.Context, form url.V p, err := h.Backend.DescribeUsers(ctx, userID, marker, maxRecords) if err != nil { if errors.Is(err, ErrUserNotFound) { - return xmlError(c, http.StatusBadRequest, "UserNotFound", "User not found") + return xmlError(c, http.StatusNotFound, "UserNotFound", "User not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -260,7 +260,7 @@ func (h *Handler) modifyUser(ctx context.Context, c *echo.Context, form url.Valu u, err := h.Backend.ModifyUser(ctx, userID, accessString, noPasswordRequired) if err != nil { if errors.Is(err, ErrUserNotFound) { - return xmlError(c, http.StatusBadRequest, "UserNotFound", "User not found") + return xmlError(c, http.StatusNotFound, "UserNotFound", "User not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -299,11 +299,11 @@ func (h *Handler) createUserGroup(ctx context.Context, c *echo.Context, form url ug, err := h.Backend.CreateUserGroupValidated(ctx, groupID, description, engine, userIDs) if err != nil { if errors.Is(err, ErrUserGroupAlreadyExists) { - return xmlError(c, http.StatusBadRequest, "UserGroupAlreadyExistsFault", "User group already exists") + return xmlError(c, http.StatusBadRequest, "UserGroupAlreadyExists", "User group already exists") } if errors.Is(err, ErrGroupUserNotFound) { - return xmlError(c, http.StatusBadRequest, "UserNotFound", err.Error()) + return xmlError(c, http.StatusNotFound, "UserNotFound", err.Error()) } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -342,7 +342,7 @@ func (h *Handler) deleteUserGroup(ctx context.Context, c *echo.Context, form url ug, err := h.Backend.DeleteUserGroup(ctx, groupID) if err != nil { if errors.Is(err, ErrUserGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "UserGroupNotFound", "User group not found") + return xmlError(c, http.StatusNotFound, "UserGroupNotFound", "User group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -382,7 +382,7 @@ func (h *Handler) describeUserGroups(ctx context.Context, c *echo.Context, form p, err := h.Backend.DescribeUserGroups(ctx, groupID, marker, maxRecords) if err != nil { if errors.Is(err, ErrUserGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "UserGroupNotFound", "User group not found") + return xmlError(c, http.StatusNotFound, "UserGroupNotFound", "User group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -407,7 +407,7 @@ func (h *Handler) modifyUserGroup(ctx context.Context, c *echo.Context, form url ug, err := h.Backend.ModifyUserGroup(ctx, groupID, userIDsToAdd, userIDsToRemove) if err != nil { if errors.Is(err, ErrUserGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "UserGroupNotFound", "User group not found") + return xmlError(c, http.StatusNotFound, "UserGroupNotFound", "User group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -449,7 +449,7 @@ func (h *Handler) deleteGlobalReplicationGroup(ctx context.Context, c *echo.Cont if errors.Is(err, ErrGlobalReplicationGroupNotFound) { return xmlError( c, - http.StatusBadRequest, + http.StatusNotFound, "GlobalReplicationGroupNotFoundFault", "Global replication group not found", ) @@ -479,7 +479,7 @@ func (h *Handler) describeGlobalReplicationGroups(ctx context.Context, c *echo.C if errors.Is(err, ErrGlobalReplicationGroupNotFound) { return xmlError( c, - http.StatusBadRequest, + http.StatusNotFound, "GlobalReplicationGroupNotFoundFault", "Global replication group not found", ) @@ -512,7 +512,7 @@ func (h *Handler) disassociateGlobalReplicationGroup(ctx context.Context, c *ech if errors.Is(err, ErrGlobalReplicationGroupNotFound) { return xmlError( c, - http.StatusBadRequest, + http.StatusNotFound, "GlobalReplicationGroupNotFoundFault", "Global replication group not found", ) @@ -543,7 +543,7 @@ func (h *Handler) failoverGlobalReplicationGroup(ctx context.Context, c *echo.Co if errors.Is(err, ErrGlobalReplicationGroupNotFound) { return xmlError( c, - http.StatusBadRequest, + http.StatusNotFound, "GlobalReplicationGroupNotFoundFault", "Global replication group not found", ) @@ -577,7 +577,7 @@ func (h *Handler) increaseNodeGroupsInGlobalReplicationGroup( if errors.Is(err, ErrGlobalReplicationGroupNotFound) { return xmlError( c, - http.StatusBadRequest, + http.StatusNotFound, "GlobalReplicationGroupNotFoundFault", "Global replication group not found", ) @@ -611,7 +611,7 @@ func (h *Handler) decreaseNodeGroupsInGlobalReplicationGroup( if errors.Is(err, ErrGlobalReplicationGroupNotFound) { return xmlError( c, - http.StatusBadRequest, + http.StatusNotFound, "GlobalReplicationGroupNotFoundFault", "Global replication group not found", ) @@ -643,7 +643,7 @@ func (h *Handler) modifyGlobalReplicationGroup(ctx context.Context, c *echo.Cont if errors.Is(err, ErrGlobalReplicationGroupNotFound) { return xmlError( c, - http.StatusBadRequest, + http.StatusNotFound, "GlobalReplicationGroupNotFoundFault", "Global replication group not found", ) @@ -672,7 +672,7 @@ func (h *Handler) rebalanceSlotsInGlobalReplicationGroup(ctx context.Context, c if errors.Is(err, ErrGlobalReplicationGroupNotFound) { return xmlError( c, - http.StatusBadRequest, + http.StatusNotFound, "GlobalReplicationGroupNotFoundFault", "Global replication group not found", ) @@ -702,7 +702,7 @@ func (h *Handler) describeReservedCacheNodes(ctx context.Context, c *echo.Contex p, err := h.Backend.DescribeReservedCacheNodes(ctx, id, cacheNodeType, offeringType, marker, maxRecords) if err != nil { if errors.Is(err, ErrReservedCacheNodeNotFound) { - return xmlError(c, http.StatusBadRequest, "ReservedCacheNodeNotFound", "Reserved cache node not found") + return xmlError(c, http.StatusNotFound, "ReservedCacheNodeNotFound", "Reserved cache node not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -740,7 +740,7 @@ func (h *Handler) describeReservedCacheNodesOfferings(ctx context.Context, c *ec if errors.Is(err, ErrReservedCacheNodesOfferingNotFound) { return xmlError( c, - http.StatusBadRequest, + http.StatusNotFound, "ReservedCacheNodesOfferingNotFound", "Reserved cache nodes offering not found", ) @@ -782,14 +782,14 @@ func (h *Handler) purchaseReservedCacheNodesOffering(ctx context.Context, c *ech if errors.Is(err, ErrReservedCacheNodesOfferingNotFound) { return xmlError( c, - http.StatusBadRequest, + http.StatusNotFound, "ReservedCacheNodesOfferingNotFound", "Reserved cache nodes offering not found", ) } if errors.Is(err, ErrReservedCacheNodeAlreadyExists) { - return xmlError(c, http.StatusConflict, "ReservedCacheNodeAlreadyExists", err.Error()) + return xmlError(c, http.StatusNotFound, "ReservedCacheNodeAlreadyExists", err.Error()) } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -813,7 +813,7 @@ func (h *Handler) deleteServerlessCache(ctx context.Context, c *echo.Context, fo sc, err := h.Backend.DeleteServerlessCache(ctx, name) if err != nil { if errors.Is(err, ErrServerlessCacheNotFound) { - return xmlError(c, http.StatusBadRequest, "ServerlessCacheNotFound", "Serverless cache not found") + return xmlError(c, http.StatusNotFound, "ServerlessCacheNotFoundFault", "Serverless cache not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -839,7 +839,7 @@ func (h *Handler) deleteServerlessCacheSnapshot(ctx context.Context, c *echo.Con if errors.Is(err, ErrServerlessCacheSnapshotNotFound) { return xmlError( c, - http.StatusBadRequest, + http.StatusNotFound, "ServerlessCacheSnapshotNotFoundFault", "Serverless cache snapshot not found", ) @@ -867,7 +867,7 @@ func (h *Handler) describeServerlessCaches(ctx context.Context, c *echo.Context, p, err := h.Backend.DescribeServerlessCaches(ctx, name, marker, maxRecords) if err != nil { if errors.Is(err, ErrServerlessCacheNotFound) { - return xmlError(c, http.StatusBadRequest, "ServerlessCacheNotFound", "Serverless cache not found") + return xmlError(c, http.StatusNotFound, "ServerlessCacheNotFoundFault", "Serverless cache not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -894,7 +894,7 @@ func (h *Handler) describeServerlessCacheSnapshots(ctx context.Context, c *echo. if errors.Is(err, ErrServerlessCacheSnapshotNotFound) { return xmlError( c, - http.StatusBadRequest, + http.StatusNotFound, "ServerlessCacheSnapshotNotFoundFault", "Serverless cache snapshot not found", ) @@ -935,7 +935,7 @@ func (h *Handler) exportServerlessCacheSnapshot(ctx context.Context, c *echo.Con if errors.Is(err, ErrServerlessCacheSnapshotNotFound) { return xmlError( c, - http.StatusBadRequest, + http.StatusNotFound, "ServerlessCacheSnapshotNotFoundFault", "Serverless cache snapshot not found", ) @@ -963,7 +963,7 @@ func (h *Handler) modifyServerlessCache(ctx context.Context, c *echo.Context, fo sc, err := h.Backend.ModifyServerlessCache(ctx, name, description) if err != nil { if errors.Is(err, ErrServerlessCacheNotFound) { - return xmlError(c, http.StatusBadRequest, "ServerlessCacheNotFound", "Serverless cache not found") + return xmlError(c, http.StatusNotFound, "ServerlessCacheNotFoundFault", "Serverless cache not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -987,7 +987,7 @@ func (h *Handler) startMigration(ctx context.Context, c *echo.Context, form url. rg, err := h.Backend.StartMigration(ctx, replicationGroupID) if err != nil { if errors.Is(err, ErrReplicationGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "ReplicationGroupNotFound", "Replication group not found") + return xmlError(c, http.StatusNotFound, "ReplicationGroupNotFoundFault", "Replication group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1011,7 +1011,7 @@ func (h *Handler) testMigration(ctx context.Context, c *echo.Context, form url.V rg, err := h.Backend.TestMigration(ctx, replicationGroupID) if err != nil { if errors.Is(err, ErrReplicationGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "ReplicationGroupNotFound", "Replication group not found") + return xmlError(c, http.StatusNotFound, "ReplicationGroupNotFoundFault", "Replication group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1036,7 +1036,7 @@ func (h *Handler) increaseReplicaCount(ctx context.Context, c *echo.Context, for rg, err := h.Backend.IncreaseReplicaCount(ctx, replicationGroupID, int32(newReplicaCount)) if err != nil { if errors.Is(err, ErrReplicationGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "ReplicationGroupNotFound", "Replication group not found") + return xmlError(c, http.StatusNotFound, "ReplicationGroupNotFoundFault", "Replication group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1061,7 +1061,7 @@ func (h *Handler) decreaseReplicaCount(ctx context.Context, c *echo.Context, for rg, err := h.Backend.DecreaseReplicaCount(ctx, replicationGroupID, int32(newReplicaCount)) if err != nil { if errors.Is(err, ErrReplicationGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "ReplicationGroupNotFound", "Replication group not found") + return xmlError(c, http.StatusNotFound, "ReplicationGroupNotFoundFault", "Replication group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1090,7 +1090,11 @@ func (h *Handler) modifyReplicationGroupShardConfiguration( rg, err := h.Backend.ModifyReplicationGroupShardConfiguration(ctx, replicationGroupID, int32(nodeGroupCount)) if err != nil { if errors.Is(err, ErrReplicationGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "ReplicationGroupNotFound", "Replication group not found") + return xmlError(c, http.StatusNotFound, "ReplicationGroupNotFoundFault", "Replication group not found") + } + + if errors.Is(err, ErrClusterModeRequired) { + return xmlError(c, http.StatusBadRequest, "InvalidParameterCombination", err.Error()) } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1149,7 +1153,7 @@ func (h *Handler) rebootCacheCluster(ctx context.Context, c *echo.Context, form cl, err := h.Backend.RebootCacheCluster(ctx, clusterID, nodeIDs) if err != nil { if errors.Is(err, ErrClusterNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheClusterNotFound", "Cache cluster not found") + return xmlError(c, http.StatusNotFound, "CacheClusterNotFound", "Cache cluster not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1173,7 +1177,7 @@ func (h *Handler) deleteCacheSecurityGroup(ctx context.Context, c *echo.Context, err := h.Backend.DeleteCacheSecurityGroup(ctx, name) if err != nil { if errors.Is(err, ErrCacheSecurityGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheSecurityGroupNotFound", "Cache security group not found") + return xmlError(c, http.StatusNotFound, "CacheSecurityGroupNotFound", "Cache security group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1194,7 +1198,7 @@ func (h *Handler) describeCacheSecurityGroups(ctx context.Context, c *echo.Conte p, err := h.Backend.DescribeCacheSecurityGroups(ctx, name, marker, maxRecords) if err != nil { if errors.Is(err, ErrCacheSecurityGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheSecurityGroupNotFound", "Cache security group not found") + return xmlError(c, http.StatusNotFound, "CacheSecurityGroupNotFound", "Cache security group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) @@ -1222,7 +1226,7 @@ func (h *Handler) revokeCacheSecurityGroupIngress(ctx context.Context, c *echo.C sg, err := h.Backend.RevokeCacheSecurityGroupIngress(ctx, name, ec2SecurityGroupName, ec2SecurityGroupOwnerID) if err != nil { if errors.Is(err, ErrCacheSecurityGroupNotFound) { - return xmlError(c, http.StatusBadRequest, "CacheSecurityGroupNotFound", "Cache security group not found") + return xmlError(c, http.StatusNotFound, "CacheSecurityGroupNotFound", "Cache security group not found") } return xmlError(c, http.StatusInternalServerError, "InternalFailure", err.Error()) diff --git a/services/elasticache/handler_parity_sweep3_test.go b/services/elasticache/handler_parity_sweep3_test.go new file mode 100644 index 000000000..6b527e344 --- /dev/null +++ b/services/elasticache/handler_parity_sweep3_test.go @@ -0,0 +1,536 @@ +package elasticache_test + +import ( + "context" + "net/http" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + elasticachesdk "github.com/aws/aws-sdk-go-v2/service/elasticache" + elasticachetypes "github.com/aws/aws-sdk-go-v2/service/elasticache/types" + smithyhttp "github.com/aws/smithy-go/transport/http" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// errorFault constrains PT to be a pointer to T that implements error, so +// [requireFault] can hand back a concretely-typed *T for the caller to +// inspect further while still being driven off a single type argument. +type errorFault[T any] interface { + *T + error +} + +// requireFault asserts err unwraps to the exact SDK-modeled fault type T via +// [errors.As] and returns it. If the emulator's wire (or HTTP status, +// checked separately) doesn't match what aws-sdk-go-v2's query-protocol +// deserializer expects for that fault, the SDK falls back to a generic +// smithy error and this assertion fails — this is what makes the check a +// real wire-shape proof rather than a bare "err != nil". +func requireFault[T any, PT errorFault[T]](t *testing.T, err error) PT { + t.Helper() + + var target PT + + require.ErrorAsf(t, err, &target, "expected error to unwrap to %T, got %v", target, err) + + return target +} + +// requireHTTPStatus asserts the HTTP status code the SDK observed on the +// response, independent of which fault type it deserialized to. +func requireHTTPStatus(t *testing.T, err error, want int) { + t.Helper() + + var respErr *smithyhttp.ResponseError + + require.ErrorAsf(t, err, &respErr, "expected a smithy http.ResponseError in the chain, got %v", err) + assert.Equal(t, want, respErr.HTTPStatusCode()) +} + +// Test_ErrorWireShapesMatchAWS is a regression test for the parity-sweep-3 +// error-code/HTTP-status audit: many NotFound/AlreadyExists faults were +// wired with the wrong string (e.g. "ReplicationGroupNotFound" +// instead of the wire-correct "ReplicationGroupNotFoundFault") and/or the +// wrong HTTP status (AWS's query-protocol model marks most *NotFoundFault +// shapes 404, not 400, per api-2.json's per-shape httpStatusCode -- see +// PARITY.md Notes). Each case below drives the real aws-sdk-go-v2 client +// against the emulator and confirms both the typed fault and the status +// code the SDK observed. +func Test_ErrorWireShapesMatchAWS(t *testing.T) { + t.Parallel() + + tests := []struct { + call func(t *testing.T, client *elasticachesdk.Client) error + checkFault func(t *testing.T, err error) + name string + wantStatus int + }{ + { + name: "CacheClusterNotFound", + call: func(t *testing.T, client *elasticachesdk.Client) error { + t.Helper() + _, err := client.DescribeCacheClusters(t.Context(), &elasticachesdk.DescribeCacheClustersInput{ + CacheClusterId: aws.String("no-such-cluster"), + }) + + return err + }, + checkFault: func(t *testing.T, err error) { + t.Helper() + requireFault[elasticachetypes.CacheClusterNotFoundFault](t, err) + }, + wantStatus: http.StatusNotFound, + }, + { + name: "ReplicationGroupNotFound", + call: func(t *testing.T, client *elasticachesdk.Client) error { + t.Helper() + _, err := client.DescribeReplicationGroups(t.Context(), &elasticachesdk.DescribeReplicationGroupsInput{ + ReplicationGroupId: aws.String("no-such-rg"), + }) + + return err + }, + checkFault: func(t *testing.T, err error) { + t.Helper() + requireFault[elasticachetypes.ReplicationGroupNotFoundFault](t, err) + }, + wantStatus: http.StatusNotFound, + }, + { + name: "CacheParameterGroupNotFound", + call: func(t *testing.T, client *elasticachesdk.Client) error { + t.Helper() + _, err := client.DescribeCacheParameterGroups( + t.Context(), + &elasticachesdk.DescribeCacheParameterGroupsInput{ + CacheParameterGroupName: aws.String("no-such-pg"), + }, + ) + + return err + }, + checkFault: func(t *testing.T, err error) { + t.Helper() + requireFault[elasticachetypes.CacheParameterGroupNotFoundFault](t, err) + }, + wantStatus: http.StatusNotFound, + }, + { + // AWS's own model marks this one 400, not 404 -- an exception to the + // otherwise-consistent "NotFoundFault => 404" rule for this API. + name: "CacheSubnetGroupNotFound", + call: func(t *testing.T, client *elasticachesdk.Client) error { + t.Helper() + _, err := client.DescribeCacheSubnetGroups( + t.Context(), + &elasticachesdk.DescribeCacheSubnetGroupsInput{CacheSubnetGroupName: aws.String("no-such-sng")}, + ) + + return err + }, + checkFault: func(t *testing.T, err error) { + t.Helper() + requireFault[elasticachetypes.CacheSubnetGroupNotFoundFault](t, err) + }, + wantStatus: http.StatusBadRequest, + }, + { + name: "CacheSecurityGroupNotFound", + call: func(t *testing.T, client *elasticachesdk.Client) error { + t.Helper() + _, err := client.DescribeCacheSecurityGroups( + t.Context(), + &elasticachesdk.DescribeCacheSecurityGroupsInput{CacheSecurityGroupName: aws.String("no-such-csg")}, + ) + + return err + }, + checkFault: func(t *testing.T, err error) { + t.Helper() + requireFault[elasticachetypes.CacheSecurityGroupNotFoundFault](t, err) + }, + wantStatus: http.StatusNotFound, + }, + { + name: "UserNotFound", + call: func(t *testing.T, client *elasticachesdk.Client) error { + t.Helper() + _, err := client.DescribeUsers(t.Context(), &elasticachesdk.DescribeUsersInput{ + UserId: aws.String("no-such-user"), + }) + + return err + }, + checkFault: func(t *testing.T, err error) { + t.Helper() + requireFault[elasticachetypes.UserNotFoundFault](t, err) + }, + wantStatus: http.StatusNotFound, + }, + { + name: "UserGroupNotFound", + call: func(t *testing.T, client *elasticachesdk.Client) error { + t.Helper() + _, err := client.DescribeUserGroups(t.Context(), &elasticachesdk.DescribeUserGroupsInput{ + UserGroupId: aws.String("no-such-ug"), + }) + + return err + }, + checkFault: func(t *testing.T, err error) { + t.Helper() + requireFault[elasticachetypes.UserGroupNotFoundFault](t, err) + }, + wantStatus: http.StatusNotFound, + }, + { + name: "ServerlessCacheNotFound", + call: func(t *testing.T, client *elasticachesdk.Client) error { + t.Helper() + _, err := client.DescribeServerlessCaches(t.Context(), &elasticachesdk.DescribeServerlessCachesInput{ + ServerlessCacheName: aws.String("no-such-sc"), + }) + + return err + }, + checkFault: func(t *testing.T, err error) { + t.Helper() + requireFault[elasticachetypes.ServerlessCacheNotFoundFault](t, err) + }, + wantStatus: http.StatusNotFound, + }, + { + name: "GlobalReplicationGroupNotFound", + call: func(t *testing.T, client *elasticachesdk.Client) error { + t.Helper() + _, err := client.DescribeGlobalReplicationGroups( + t.Context(), + &elasticachesdk.DescribeGlobalReplicationGroupsInput{ + GlobalReplicationGroupId: aws.String("no-such-grg"), + }, + ) + + return err + }, + checkFault: func(t *testing.T, err error) { + t.Helper() + requireFault[elasticachetypes.GlobalReplicationGroupNotFoundFault](t, err) + }, + wantStatus: http.StatusNotFound, + }, + { + name: "SnapshotNotFound", + call: func(t *testing.T, client *elasticachesdk.Client) error { + t.Helper() + _, err := client.DeleteSnapshot(t.Context(), &elasticachesdk.DeleteSnapshotInput{ + SnapshotName: aws.String("no-such-snap"), + }) + + return err + }, + checkFault: func(t *testing.T, err error) { + t.Helper() + requireFault[elasticachetypes.SnapshotNotFoundFault](t, err) + }, + wantStatus: http.StatusNotFound, + }, + { + name: "ReservedCacheNodeNotFound", + call: func(t *testing.T, client *elasticachesdk.Client) error { + t.Helper() + _, err := client.DescribeReservedCacheNodes( + t.Context(), + &elasticachesdk.DescribeReservedCacheNodesInput{ReservedCacheNodeId: aws.String("no-such-rcn")}, + ) + + return err + }, + checkFault: func(t *testing.T, err error) { + t.Helper() + requireFault[elasticachetypes.ReservedCacheNodeNotFoundFault](t, err) + }, + wantStatus: http.StatusNotFound, + }, + { + name: "ReservedCacheNodesOfferingNotFound", + call: func(t *testing.T, client *elasticachesdk.Client) error { + t.Helper() + _, err := client.DescribeReservedCacheNodesOfferings( + t.Context(), + &elasticachesdk.DescribeReservedCacheNodesOfferingsInput{ + ReservedCacheNodesOfferingId: aws.String("no-such-offering"), + }, + ) + + return err + }, + checkFault: func(t *testing.T, err error) { + t.Helper() + requireFault[elasticachetypes.ReservedCacheNodesOfferingNotFoundFault](t, err) + }, + wantStatus: http.StatusNotFound, + }, + { + name: "UserGroupAlreadyExists", + call: func(t *testing.T, client *elasticachesdk.Client) error { + t.Helper() + input := &elasticachesdk.CreateUserGroupInput{ + UserGroupId: aws.String("dup-ug"), + Engine: aws.String("redis"), + } + _, err := client.CreateUserGroup(t.Context(), input) + require.NoError(t, err, "first CreateUserGroup must succeed") + + _, err = client.CreateUserGroup(t.Context(), input) + + return err + }, + checkFault: func(t *testing.T, err error) { + t.Helper() + requireFault[elasticachetypes.UserGroupAlreadyExistsFault](t, err) + }, + wantStatus: http.StatusBadRequest, + }, + { + // AWS models this AlreadyExists fault as 404, not the 409/400 one + // might expect -- see api-2.json ReservedCacheNodeAlreadyExistsFault. + name: "ReservedCacheNodeAlreadyExists", + call: func(t *testing.T, client *elasticachesdk.Client) error { + t.Helper() + input := &elasticachesdk.PurchaseReservedCacheNodesOfferingInput{ + ReservedCacheNodesOfferingId: aws.String("31153cd5-4ce6-45a9-b6ce-7f0b6789b8fa"), + ReservedCacheNodeId: aws.String("dup-rcn"), + } + _, err := client.PurchaseReservedCacheNodesOffering(t.Context(), input) + require.NoError(t, err, "first purchase must succeed") + + _, err = client.PurchaseReservedCacheNodesOffering(t.Context(), input) + + return err + }, + checkFault: func(t *testing.T, err error) { + t.Helper() + requireFault[elasticachetypes.ReservedCacheNodeAlreadyExistsFault](t, err) + }, + wantStatus: http.StatusNotFound, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + client := newTestStack(t) + + err := tt.call(t, client) + require.Error(t, err) + + tt.checkFault(t, err) + requireHTTPStatus(t, err, tt.wantStatus) + }) + } +} + +// Test_ModifyReplicationGroup_TransitEncryptionRequiresAuthToken covers the +// gap where TransitEncryptionMode="required" could be set via +// ModifyReplicationGroup with no auth token ever enabled -- the backend +// declared ErrTransitEncryptionModeInvalid for exactly this case but no code +// path ever returned it (a disguised stub: the sentinel existed, the guard +// didn't). Fixed in backend_audit1.go's validateTransitEncryptionModify. +func Test_ModifyReplicationGroup_TransitEncryptionRequiresAuthToken(t *testing.T) { + t.Parallel() + + tests := []struct { + input *elasticachesdk.ModifyReplicationGroupInput + name string + wantErr bool + }{ + { + name: "required_mode_without_auth_token_rejected", + input: &elasticachesdk.ModifyReplicationGroupInput{ + TransitEncryptionMode: elasticachetypes.TransitEncryptionModeRequired, + }, + wantErr: true, + }, + { + name: "required_mode_with_new_auth_token_allowed", + input: &elasticachesdk.ModifyReplicationGroupInput{ + TransitEncryptionMode: elasticachetypes.TransitEncryptionModeRequired, + AuthToken: aws.String("s3cr3t-token-01234567890123456789"), + AuthTokenUpdateStrategy: elasticachetypes.AuthTokenUpdateStrategyTypeSet, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + client := newTestStack(t) + + ctx := t.Context() + _, err := client.CreateReplicationGroup(ctx, &elasticachesdk.CreateReplicationGroupInput{ + ReplicationGroupId: aws.String("te-rg"), + ReplicationGroupDescription: aws.String("d"), + }) + require.NoError(t, err) + + tt.input.ReplicationGroupId = aws.String("te-rg") + + _, err = client.ModifyReplicationGroup(ctx, tt.input) + + if tt.wantErr { + require.Error(t, err) + requireFault[elasticachetypes.InvalidParameterCombinationException](t, err) + requireHTTPStatus(t, err, http.StatusBadRequest) + + return + } + + require.NoError(t, err) + }) + } +} + +// Test_ModifyReplicationGroupShardConfiguration_RequiresClusterMode is the +// wire-level counterpart of TestBackend_ModifyReplicationGroupShardConfiguration_RequiresClusterMode +// in handler_audit1_test.go, which only checked the backend Go error. This +// confirms the HTTP handler actually maps ErrClusterModeRequired to a client +// (400 InvalidParameterCombination) response instead of falling through to +// the generic InternalFailure 500 default case. +func Test_ModifyReplicationGroupShardConfiguration_RequiresClusterMode(t *testing.T) { + t.Parallel() + + client := newTestStack(t) + ctx := context.Background() + + _, err := client.CreateReplicationGroup(ctx, &elasticachesdk.CreateReplicationGroupInput{ + ReplicationGroupId: aws.String("no-cluster-mode-rg"), + ReplicationGroupDescription: aws.String("d"), + }) + require.NoError(t, err) + + _, err = client.ModifyReplicationGroupShardConfiguration( + ctx, + &elasticachesdk.ModifyReplicationGroupShardConfigurationInput{ + ReplicationGroupId: aws.String("no-cluster-mode-rg"), + NodeGroupCount: aws.Int32(2), + ApplyImmediately: aws.Bool(true), + }, + ) + require.Error(t, err) + + target := requireFault[elasticachetypes.InvalidParameterCombinationException](t, err) + requireHTTPStatus(t, err, http.StatusBadRequest) + assert.Contains(t, target.ErrorMessage(), "cluster mode") +} + +// Test_CreateCacheCluster_RestoreFromSnapshot covers the previously-unhandled +// SnapshotName parameter on CreateCacheCluster: AWS restores a new cluster +// from an existing snapshot, and the emulator's handler never even read the +// form field. It's wired now to validate the snapshot exists and inherit the +// snapshot's engine/node type when the caller doesn't override them. A +// missing snapshot surfaces as InvalidParameterValueException (400), not +// SnapshotNotFoundFault -- CreateCacheCluster's modeled error list in +// api-2.json doesn't include SnapshotNotFoundFault at all, so aws-sdk-go-v2 +// has no deserializer case for it on this operation. +func Test_CreateCacheCluster_RestoreFromSnapshot(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + snapshotName string + wantErr bool + }{ + {name: "restores_engine_and_node_type_from_snapshot", snapshotName: "src-snap"}, + {name: "missing_snapshot_is_not_found", snapshotName: "no-such-snap", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + client := newTestStack(t) + ctx := t.Context() + + _, err := client.CreateCacheCluster(ctx, &elasticachesdk.CreateCacheClusterInput{ + CacheClusterId: aws.String("src-cluster"), + Engine: aws.String("memcached"), + CacheNodeType: aws.String("cache.m5.large"), + NumCacheNodes: aws.Int32(1), + }) + require.NoError(t, err) + + _, err = client.CreateSnapshot(ctx, &elasticachesdk.CreateSnapshotInput{ + SnapshotName: aws.String("src-snap"), + CacheClusterId: aws.String("src-cluster"), + }) + require.NoError(t, err) + + out, err := client.CreateCacheCluster(ctx, &elasticachesdk.CreateCacheClusterInput{ + CacheClusterId: aws.String("restored-cluster"), + SnapshotName: aws.String(tt.snapshotName), + }) + + if tt.wantErr { + require.Error(t, err) + requireFault[elasticachetypes.InvalidParameterValueException](t, err) + requireHTTPStatus(t, err, http.StatusBadRequest) + + return + } + + require.NoError(t, err) + assert.Equal(t, "memcached", aws.ToString(out.CacheCluster.Engine)) + assert.Equal(t, "cache.m5.large", aws.ToString(out.CacheCluster.CacheNodeType)) + }) + } +} + +// Test_CreateReplicationGroup_RestoreFromSnapshot is the replication-group +// counterpart: same previously-dropped SnapshotName parameter, now wired +// through the additive ReplicationGroupCreateOpts.SnapshotName field. As +// with CreateCacheCluster, CreateReplicationGroup's modeled error list also +// omits SnapshotNotFoundFault, so a missing snapshot surfaces as +// InvalidParameterValueException (400) instead. +func Test_CreateReplicationGroup_RestoreFromSnapshot(t *testing.T) { + t.Parallel() + + client := newTestStack(t) + ctx := context.Background() + + _, err := client.CreateCacheCluster(ctx, &elasticachesdk.CreateCacheClusterInput{ + CacheClusterId: aws.String("src-cluster-rg"), + Engine: aws.String("redis"), + EngineVersion: aws.String("7.1.0"), + CacheNodeType: aws.String("cache.r6g.large"), + NumCacheNodes: aws.Int32(1), + }) + require.NoError(t, err) + + _, err = client.CreateSnapshot(ctx, &elasticachesdk.CreateSnapshotInput{ + SnapshotName: aws.String("src-snap-rg"), + CacheClusterId: aws.String("src-cluster-rg"), + }) + require.NoError(t, err) + + out, err := client.CreateReplicationGroup(ctx, &elasticachesdk.CreateReplicationGroupInput{ + ReplicationGroupId: aws.String("restored-rg"), + ReplicationGroupDescription: aws.String("restored from snapshot"), + SnapshotName: aws.String("src-snap-rg"), + }) + require.NoError(t, err) + assert.Equal(t, "cache.r6g.large", aws.ToString(out.ReplicationGroup.CacheNodeType)) + + _, err = client.CreateReplicationGroup(ctx, &elasticachesdk.CreateReplicationGroupInput{ + ReplicationGroupId: aws.String("restored-rg-missing"), + ReplicationGroupDescription: aws.String("missing snapshot"), + SnapshotName: aws.String("no-such-snap-rg"), + }) + require.Error(t, err) + requireFault[elasticachetypes.InvalidParameterValueException](t, err) + requireHTTPStatus(t, err, http.StatusBadRequest) +} diff --git a/services/elasticache/isolation_test.go b/services/elasticache/isolation_test.go index b932c911f..2a0916a18 100644 --- a/services/elasticache/isolation_test.go +++ b/services/elasticache/isolation_test.go @@ -68,3 +68,118 @@ func TestRegionIsolation_ReplicationGroups(t *testing.T) { t.Fatalf("expected 0 rgs in us-west-2, got %d", len(westRGs.Data)) } } + +// TestSnapshotRestore_CrossRegion verifies a Phase 3.3 per-region +// store.Table conversion doesn't collapse or drop a second region's data: +// clusters in two distinct regions must both survive a Snapshot/Restore +// round trip into a fresh backend, each still isolated to its own region. +// This test lives in the internal package (unlike persistence_test.go) so it +// can use the unexported regionContextKey directly, the same way the +// TestRegionIsolation_* tests above do. +func TestSnapshotRestore_CrossRegion(t *testing.T) { + original := NewInMemoryBackend("redis", "123456789012", "us-east-1", nil) + + ctxEast := context.WithValue(context.Background(), regionContextKey{}, "us-east-1") + ctxWest := context.WithValue(context.Background(), regionContextKey{}, "us-west-2") + + if _, err := original.CreateCluster(ctxEast, "cr-cluster-east", "redis", "cache.t3.micro", 6379); err != nil { + t.Fatalf("create cluster east: %v", err) + } + if _, err := original.CreateCluster(ctxWest, "cr-cluster-west", "redis", "cache.t3.micro", 6379); err != nil { + t.Fatalf("create cluster west: %v", err) + } + + snap := original.Snapshot(context.Background()) + if snap == nil { + t.Fatal("snapshot returned nil") + } + + fresh := NewInMemoryBackend("redis", "123456789012", "us-east-1", nil) + if err := fresh.Restore(context.Background(), snap); err != nil { + t.Fatalf("restore: %v", err) + } + + eastAfter, err := fresh.DescribeClusters(ctxEast, "", "", 100, false) + if err != nil { + t.Fatalf("describe clusters east after restore: %v", err) + } + if len(eastAfter.Data) != 1 || eastAfter.Data[0].ClusterID != "cr-cluster-east" { + t.Fatalf("expected exactly cr-cluster-east in us-east-1 after restore, got %+v", eastAfter.Data) + } + + westAfter, err := fresh.DescribeClusters(ctxWest, "", "", 100, false) + if err != nil { + t.Fatalf("describe clusters west after restore: %v", err) + } + if len(westAfter.Data) != 1 || westAfter.Data[0].ClusterID != "cr-cluster-west" { + t.Fatalf("expected exactly cr-cluster-west in us-west-2 after restore, got %+v", westAfter.Data) + } +} + +// TestSnapshotRestore_CacheSecurityGroupIngress verifies the one map Phase +// 3.3 deliberately left un-converted (cacheSecurityGroupIngress is +// slice-valued, not map[string]*T, so store.Table cannot represent it) +// still round-trips through Snapshot/Restore. Unlike the exported-API checks +// in persistence_test.go, ingress data has no public Describe surface, so +// this reaches into the unexported field directly from within the package. +func TestSnapshotRestore_CacheSecurityGroupIngress(t *testing.T) { + original := NewInMemoryBackend("redis", "123456789012", "us-east-1", nil) + + ctx := context.Background() + if _, err := original.CreateCacheSecurityGroup(ctx, "ingress-sg", "test"); err != nil { + t.Fatalf("create cache security group: %v", err) + } + if _, err := original.AuthorizeCacheSecurityGroupIngress(ctx, "ingress-sg", "peer-sg", "999999999999"); err != nil { + t.Fatalf("authorize ingress: %v", err) + } + + snap := original.Snapshot(ctx) + if snap == nil { + t.Fatal("snapshot returned nil") + } + + fresh := NewInMemoryBackend("redis", "123456789012", "us-east-1", nil) + if err := fresh.Restore(ctx, snap); err != nil { + t.Fatalf("restore: %v", err) + } + + ingress := fresh.cacheSecurityGroupIngressStore(fresh.region)["ingress-sg"] + if len(ingress) != 1 || ingress[0].EC2SecurityGroupName != "peer-sg" { + t.Fatalf("expected 1 ingress entry for peer-sg to survive restore, got %+v", ingress) + } +} + +// TestReset_RegisteredTableSurvivesRepeatedCalls guards against a +// regression where Reset accidentally re-registers the +// globalReplicationGroups table (store.Register panics on a duplicate name) +// instead of clearing it in place via b.registry.ResetAll(). +func TestReset_RegisteredTableSurvivesRepeatedCalls(t *testing.T) { + b := NewInMemoryBackend("redis", "123456789012", "us-east-1", nil) + + ctx := context.Background() + + rg, err := b.CreateReplicationGroup(ctx, "reset-rg", "test") + if err != nil { + t.Fatalf("create replication group: %v", err) + } + _, createErr := b.CreateGlobalReplicationGroup(ctx, "reset-grg", "test", rg.ReplicationGroupID) + if createErr != nil { + t.Fatalf("create global replication group: %v", createErr) + } + + b.Reset() + b.Reset() + + grgs, err := b.DescribeGlobalReplicationGroups(ctx, "", "", 100) + if err != nil { + t.Fatalf("describe global replication groups after reset: %v", err) + } + if len(grgs.Data) != 0 { + t.Fatalf("expected 0 global replication groups after Reset, got %d", len(grgs.Data)) + } + + _, createErr2 := b.CreateGlobalReplicationGroup(ctx, "post-reset-grg", "test", "") + if createErr2 != nil { + t.Fatalf("create global replication group after reset: %v", createErr2) + } +} diff --git a/services/elasticache/persistence.go b/services/elasticache/persistence.go index 887165fdc..a09820980 100644 --- a/services/elasticache/persistence.go +++ b/services/elasticache/persistence.go @@ -2,13 +2,31 @@ package elasticache import ( "context" + "encoding/json" "time" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) -// clusterSnapshot captures the serialisable fields of a Cluster (omits the miniredis instance). +// elasticacheSnapshotVersion identifies the shape of [backendSnapshot]. It +// must be bumped whenever a change to clusterSnapshot, or to the set/shape of +// tables captured below, would make an older snapshot unsafe to decode as the +// current shape. Restore compares this against the persisted value and +// discards (rather than attempts to partially decode) any mismatch -- see +// Restore below. This mirrors the services/sqs pilot (commit 0f09d77c). +const elasticacheSnapshotVersion = 1 + +// clusterSnapshot captures the serialisable fields of a Cluster: it omits the +// unexported live miniredis instance (which can never round-trip through +// JSON) and mirrors the field subset this backend has always persisted for +// clusters. Cluster carries several additional exported fields (e.g. +// ReplicationGroupID, AvailableAt, Members) that predate this conversion and +// were never included in the persisted shape; that pre-existing behavior is +// preserved as-is here rather than "fixed" as part of this mechanical +// datalayer swap. type clusterSnapshot struct { CreatedAt time.Time `json:"createdAt"` Tags *tags.Tags `json:"tags,omitempty"` @@ -26,24 +44,48 @@ type clusterSnapshot struct { NumCacheNodes int `json:"numCacheNodes"` } +// backendSnapshot is the top-level on-disk shape for the ElastiCache backend. +// +// Tables holds the JSON-encoded array for every table registered on +// b.registry (currently just "globalReplicationGroups", the one resource that +// is global rather than per-region -- see store_setup.go), produced by +// [store.Registry.SnapshotAll]. +// +// Every other resource is nested per-region (region -> *store.Table[T]) and +// is NOT registered on b.registry, because the set of regions is only known +// at runtime (see store_setup.go's doc comment); each is instead captured +// directly below as region -> that table's own deterministic +// [store.Table.Snapshot] slice. type backendSnapshot struct { - Clusters map[string]map[string]*clusterSnapshot `json:"clusters"` - ReplicationGroups map[string]map[string]*ReplicationGroup `json:"replicationGroups"` - ParameterGroups map[string]map[string]*CacheParameterGroup `json:"parameterGroups"` - SubnetGroups map[string]map[string]*CacheSubnetGroup `json:"subnetGroups"` - Snapshots map[string]map[string]*CacheSnapshot `json:"snapshots"` - CacheSecurityGroups map[string]map[string]*CacheSecurityGroup `json:"cacheSecurityGroups,omitempty"` + Tables map[string]json.RawMessage `json:"tables"` + Clusters map[string][]*clusterSnapshot `json:"clusters"` + ReplicationGroups map[string][]*ReplicationGroup `json:"replicationGroups"` + ParameterGroups map[string][]*CacheParameterGroup `json:"parameterGroups"` + SubnetGroups map[string][]*CacheSubnetGroup `json:"subnetGroups"` + Snapshots map[string][]*CacheSnapshot `json:"snapshots"` + CacheSecurityGroups map[string][]*CacheSecurityGroup `json:"cacheSecurityGroups,omitempty"` CacheSecurityGroupIngress map[string]map[string][]EC2SecurityGroupMembership `json:"cacheSecurityGroupIngress,omitempty"` //nolint:lll // struct tag cannot be split - GlobalReplicationGroups map[string]*GlobalReplicationGroup `json:"globalReplicationGroups,omitempty"` - ServerlessCaches map[string]map[string]*ServerlessCache `json:"serverlessCaches,omitempty"` - ServerlessCacheSnapshots map[string]map[string]*ServerlessCacheSnapshot `json:"serverlessCacheSnapshots,omitempty"` //nolint:lll // struct tag cannot be split - Users map[string]map[string]*User `json:"users,omitempty"` - UserGroups map[string]map[string]*UserGroup `json:"userGroups,omitempty"` - ReservedCacheNodes map[string]map[string]*ReservedCacheNode `json:"reservedCacheNodes,omitempty"` + ServerlessCaches map[string][]*ServerlessCache `json:"serverlessCaches,omitempty"` + ServerlessCacheSnapshots map[string][]*ServerlessCacheSnapshot `json:"serverlessCacheSnapshots,omitempty"` //nolint:lll // struct tag cannot be split + Users map[string][]*User `json:"users,omitempty"` + UserGroups map[string][]*UserGroup `json:"userGroups,omitempty"` + ReservedCacheNodes map[string][]*ReservedCacheNode `json:"reservedCacheNodes,omitempty"` EngineMode string `json:"engineMode"` AccountID string `json:"accountID"` Region string `json:"region"` Events []CacheEvent `json:"events,omitempty"` + Version int `json:"version"` +} + +// snapshotAllRegions captures the deterministic [store.Table.Snapshot] slice +// for every region in m. +func snapshotAllRegions[T any](m map[string]*store.Table[T]) map[string][]*T { + out := make(map[string][]*T, len(m)) + for region, t := range m { + out[region] = t.Snapshot() + } + + return out } // Snapshot serialises the backend state to JSON. @@ -52,11 +94,24 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() - clusters := make(map[string]map[string]*clusterSnapshot, len(b.clusters)) - for region, regionClusters := range b.clusters { - regionSnap := make(map[string]*clusterSnapshot, len(regionClusters)) - for k, c := range regionClusters { - regionSnap[k] = &clusterSnapshot{ + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "elasticache: snapshot table marshal failed", "error", err) + + return nil + } + + clusters := make(map[string][]*clusterSnapshot, len(b.clusters)) + for region, t := range b.clusters { + items := t.Snapshot() + snaps := make([]*clusterSnapshot, len(items)) + + for i, c := range items { + snaps[i] = &clusterSnapshot{ CreatedAt: c.CreatedAt, Tags: c.Tags, ClusterID: c.ClusterID, @@ -73,111 +128,73 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { NumCacheNodes: c.NumCacheNodes, } } - clusters[region] = regionSnap + + clusters[region] = snaps } snap := backendSnapshot{ + Version: elasticacheSnapshotVersion, + Tables: tables, Clusters: clusters, - ReplicationGroups: b.replicationGroups, - ParameterGroups: b.parameterGroups, - SubnetGroups: b.subnetGroups, - Snapshots: b.snapshots, - CacheSecurityGroups: b.cacheSecurityGroups, + ReplicationGroups: snapshotAllRegions(b.replicationGroups), + ParameterGroups: snapshotAllRegions(b.parameterGroups), + SubnetGroups: snapshotAllRegions(b.subnetGroups), + Snapshots: snapshotAllRegions(b.snapshots), + CacheSecurityGroups: snapshotAllRegions(b.cacheSecurityGroups), CacheSecurityGroupIngress: b.cacheSecurityGroupIngress, - GlobalReplicationGroups: b.cloneGlobalReplicationGroups(), - - ServerlessCaches: b.serverlessCaches, - ServerlessCacheSnapshots: b.serverlessCacheSnapshots, - Users: b.users, - UserGroups: b.userGroups, - ReservedCacheNodes: b.reservedCacheNodes, - Events: b.events.marshalJSON(), - EngineMode: b.engineMode, - AccountID: b.accountID, - Region: b.region, + ServerlessCaches: snapshotAllRegions(b.serverlessCaches), + ServerlessCacheSnapshots: snapshotAllRegions(b.serverlessCacheSnapshots), + Users: snapshotAllRegions(b.users), + UserGroups: snapshotAllRegions(b.userGroups), + ReservedCacheNodes: snapshotAllRegions(b.reservedCacheNodes), + Events: b.events.marshalJSON(), + EngineMode: b.engineMode, + AccountID: b.accountID, + Region: b.region, } return persistence.MarshalSnapshot(ctx, "elasticache", snap) } -// restoreClusters converts the snapshot's clusterSnapshot nested map into Cluster objects. -func restoreClusters(snap map[string]map[string]*clusterSnapshot) map[string]map[string]*Cluster { - clusters := make(map[string]map[string]*Cluster, len(snap)) - for region, regionSnap := range snap { - regionClusters := make(map[string]*Cluster, len(regionSnap)) - for k, cs := range regionSnap { - regionClusters[k] = &Cluster{ - CreatedAt: cs.CreatedAt, - Tags: cs.Tags, - ClusterID: cs.ClusterID, - Engine: cs.Engine, - EngineVersion: cs.EngineVersion, - Status: cs.Status, - Endpoint: cs.Endpoint, - NodeType: cs.NodeType, - ARN: cs.ARN, - CacheParameterGroupName: cs.CacheParameterGroupName, - PreferredMaintenanceWindow: cs.PreferredMaintenanceWindow, - SnapshotWindow: cs.SnapshotWindow, - Port: cs.Port, - NumCacheNodes: cs.NumCacheNodes, - } - } - clusters[region] = regionClusters +// restoreRegionTables resets the outer per-region map via reset, then +// restores each region's table from data using storeFn (one of the +// InMemoryBackend "*Store" lazy accessors). This mirrors the pre-conversion +// full-replace Restore semantics: a region present in the live backend but +// absent from data ends up empty, exactly as a direct `b.field = data` +// assignment would have left it. +func restoreRegionTables[T any](reset func(), storeFn func(string) *store.Table[T], data map[string][]*T) { + reset() + + for region, items := range data { + storeFn(region).Restore(items) } - - return clusters } -// restoreNewOpMaps assigns the new-ops maps from the snapshot into the backend. -func (b *InMemoryBackend) restoreNewOpMaps(snap *backendSnapshot) { - if snap.CacheSecurityGroups != nil { - b.cacheSecurityGroups = snap.CacheSecurityGroups - } else { - b.cacheSecurityGroups = make(map[string]map[string]*CacheSecurityGroup) - } - - if snap.CacheSecurityGroupIngress != nil { - b.cacheSecurityGroupIngress = snap.CacheSecurityGroupIngress - } else { - b.cacheSecurityGroupIngress = make(map[string]map[string][]EC2SecurityGroupMembership) - } - - if snap.GlobalReplicationGroups != nil { - b.setGlobalReplicationGroups(snap.GlobalReplicationGroups) - } else { - b.setGlobalReplicationGroups(make(map[string]*GlobalReplicationGroup)) - } - - if snap.ServerlessCaches != nil { - b.serverlessCaches = snap.ServerlessCaches - } else { - b.serverlessCaches = make(map[string]map[string]*ServerlessCache) - } - - if snap.ServerlessCacheSnapshots != nil { - b.serverlessCacheSnapshots = snap.ServerlessCacheSnapshots - } else { - b.serverlessCacheSnapshots = make(map[string]map[string]*ServerlessCacheSnapshot) - } - - if snap.Users != nil { - b.users = snap.Users - } else { - b.users = make(map[string]map[string]*User) - } - - if snap.UserGroups != nil { - b.userGroups = snap.UserGroups - } else { - b.userGroups = make(map[string]map[string]*UserGroup) +// restoreClusterRegion converts one region's persisted clusterSnapshot slice +// into live *Cluster values (mini stays nil; a restored cluster has no live +// data-plane engine until re-created, matching pre-conversion behavior). +func restoreClusterRegion(items []*clusterSnapshot) []*Cluster { + out := make([]*Cluster, len(items)) + for i, cs := range items { + out[i] = &Cluster{ + CreatedAt: cs.CreatedAt, + Tags: cs.Tags, + ClusterID: cs.ClusterID, + Engine: cs.Engine, + EngineVersion: cs.EngineVersion, + Status: cs.Status, + Endpoint: cs.Endpoint, + NodeType: cs.NodeType, + ARN: cs.ARN, + CacheParameterGroupName: cs.CacheParameterGroupName, + PreferredMaintenanceWindow: cs.PreferredMaintenanceWindow, + SnapshotWindow: cs.SnapshotWindow, + Port: cs.Port, + NumCacheNodes: cs.NumCacheNodes, + } } - if snap.ReservedCacheNodes != nil { - b.reservedCacheNodes = snap.ReservedCacheNodes - } else { - b.reservedCacheNodes = make(map[string]map[string]*ReservedCacheNode) - } + return out } // Restore loads backend state from a JSON snapshot. @@ -192,41 +209,61 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() - if snap.Clusters == nil { - snap.Clusters = make(map[string]map[string]*clusterSnapshot) - } + if snap.Version != elasticacheSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. + logger.Load(ctx).WarnContext(ctx, + "elasticache: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", elasticacheSnapshotVersion) - if snap.ReplicationGroups == nil { - snap.ReplicationGroups = make(map[string]map[string]*ReplicationGroup) - } + b.resetLocked() - if snap.ParameterGroups == nil { - snap.ParameterGroups = make(map[string]map[string]*CacheParameterGroup) + return nil } - if snap.SubnetGroups == nil { - snap.SubnetGroups = make(map[string]map[string]*CacheSubnetGroup) + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return err } - if snap.Snapshots == nil { - snap.Snapshots = make(map[string]map[string]*CacheSnapshot) - } + b.closeLiveClusterEnginesLocked() - for _, regionClusters := range b.clusters { - for _, c := range regionClusters { - if c.mini != nil { - c.mini.Close() - } - } + b.clusters = make(map[string]*store.Table[Cluster]) + for region, items := range snap.Clusters { + b.clustersStore(region).Restore(restoreClusterRegion(items)) } - b.clusters = restoreClusters(snap.Clusters) - b.replicationGroups = snap.ReplicationGroups - b.parameterGroups = snap.ParameterGroups - b.subnetGroups = snap.SubnetGroups - b.snapshots = snap.Snapshots + restoreRegionTables(func() { b.replicationGroups = make(map[string]*store.Table[ReplicationGroup]) }, + b.replicationGroupsStore, snap.ReplicationGroups) + restoreRegionTables(func() { b.parameterGroups = make(map[string]*store.Table[CacheParameterGroup]) }, + b.parameterGroupsStore, snap.ParameterGroups) + restoreRegionTables(func() { b.subnetGroups = make(map[string]*store.Table[CacheSubnetGroup]) }, + b.subnetGroupsStore, snap.SubnetGroups) + restoreRegionTables(func() { b.snapshots = make(map[string]*store.Table[CacheSnapshot]) }, + b.snapshotsStore, snap.Snapshots) + restoreRegionTables(func() { b.cacheSecurityGroups = make(map[string]*store.Table[CacheSecurityGroup]) }, + b.cacheSecurityGroupsStore, snap.CacheSecurityGroups) + restoreRegionTables(func() { b.serverlessCaches = make(map[string]*store.Table[ServerlessCache]) }, + b.serverlessCachesStore, snap.ServerlessCaches) + restoreRegionTables( + func() { b.serverlessCacheSnapshots = make(map[string]*store.Table[ServerlessCacheSnapshot]) }, + b.serverlessCacheSnapshotsStore, snap.ServerlessCacheSnapshots, + ) + restoreRegionTables(func() { b.users = make(map[string]*store.Table[User]) }, + b.usersStore, snap.Users) + restoreRegionTables(func() { b.userGroups = make(map[string]*store.Table[UserGroup]) }, + b.userGroupsStore, snap.UserGroups) + restoreRegionTables(func() { b.reservedCacheNodes = make(map[string]*store.Table[ReservedCacheNode]) }, + b.reservedCacheNodesStore, snap.ReservedCacheNodes) - b.restoreNewOpMaps(&snap) + if snap.CacheSecurityGroupIngress != nil { + b.cacheSecurityGroupIngress = snap.CacheSecurityGroupIngress + } else { + b.cacheSecurityGroupIngress = make(map[string]map[string][]EC2SecurityGroupMembership) + } b.events.restoreFromSlice(snap.Events) @@ -234,24 +271,45 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.accountID = snap.AccountID b.region = snap.Region - // Re-init default parameter groups per region if missing (e.g., old snapshots). - for _, dpg := range builtinParameterGroupFamilies() { - regionStore := b.parameterGroupsStore(b.region) - if _, ok := regionStore[dpg.name]; !ok { - pg := &CacheParameterGroup{ - Name: dpg.name, - Family: dpg.family, - Description: "Default parameter group for " + dpg.family, - ARN: b.parameterGroupARN(dpg.name), - IsGlobal: true, - Parameters: make(map[string]string), - Tags: tags.New("elasticache.pg." + dpg.name + ".tags"), + b.reinitMissingDefaultParameterGroupsLocked() + + return nil +} + +// closeLiveClusterEnginesLocked closes every live miniredis instance across +// all regions before the cluster tables are discarded and rebuilt from a +// snapshot. Must hold b.mu. +func (b *InMemoryBackend) closeLiveClusterEnginesLocked() { + for _, regionClusters := range b.clusters { + for _, c := range regionClusters.All() { + if c.mini != nil { + c.mini.Close() } - regionStore[dpg.name] = pg } } +} - return nil +// reinitMissingDefaultParameterGroupsLocked re-seeds the well-known default +// parameter groups for the backend's current region if a restored snapshot +// didn't already include them (e.g. an older snapshot). Must hold b.mu. +func (b *InMemoryBackend) reinitMissingDefaultParameterGroupsLocked() { + regionStore := b.parameterGroupsStore(b.region) + + for _, dpg := range builtinParameterGroupFamilies() { + if _, ok := regionStore.Get(dpg.name); ok { + continue + } + + regionStore.Put(&CacheParameterGroup{ + Name: dpg.name, + Family: dpg.family, + Description: "Default parameter group for " + dpg.family, + ARN: b.parameterGroupARN(dpg.name), + IsGlobal: true, + Parameters: make(map[string]string), + Tags: tags.New("elasticache.pg." + dpg.name + ".tags"), + }) + } } // Snapshot implements persistence.Persistable by delegating to the backend. diff --git a/services/elasticache/persistence_test.go b/services/elasticache/persistence_test.go index c95776149..6ec7d7190 100644 --- a/services/elasticache/persistence_test.go +++ b/services/elasticache/persistence_test.go @@ -2,6 +2,7 @@ package elasticache_test import ( "context" + "encoding/json" "testing" "github.com/stretchr/testify/assert" @@ -75,3 +76,148 @@ func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { err := b.Restore(t.Context(), []byte("not-valid-json")) require.Error(t, err) } + +// TestInMemoryBackend_SnapshotRestore_FullState seeds one instance of every +// resource type Phase 3.3 converted to pkgs/store, snapshots the backend, +// restores into a fresh backend, and checks every resource survived with its +// identity/fields intact. This guards against a table being silently dropped +// during the map -> store.Table conversion. Region-nested per-region table +// round-tripping (a second region within the same resource) is covered +// separately by the internal-package test TestSnapshotRestore_CrossRegion, +// which needs the unexported region-context key not reachable from here. +func TestInMemoryBackend_SnapshotRestore_FullState(t *testing.T) { //nolint:maintidx // exhaustive seed+verify + t.Parallel() + + ctx := t.Context() + + original := elasticache.NewInMemoryBackend("redis", "000000000000", "us-east-1", nil) + + _, err := original.CreateCluster(ctx, "fs-cluster", "redis", "cache.t3.micro", 6379) + require.NoError(t, err) + + rg, err := original.CreateReplicationGroup(ctx, "fs-rg", "full state rg") + require.NoError(t, err) + + _, err = original.CreateParameterGroup(ctx, "fs-pg", "redis7", "full state pg") + require.NoError(t, err) + + _, err = original.CreateSubnetGroup(ctx, "fs-sng", "full state subnet group", []string{"subnet-1"}) + require.NoError(t, err) + + _, err = original.CreateSnapshot(ctx, "fs-snap", "fs-cluster", "") + require.NoError(t, err) + + _, err = original.CreateCacheSecurityGroup(ctx, "fs-sg", "full state security group") + require.NoError(t, err) + _, err = original.AuthorizeCacheSecurityGroupIngress(ctx, "fs-sg", "some-ec2-sg", "000000000000") + require.NoError(t, err) + + _, err = original.CreateGlobalReplicationGroup(ctx, "fsgrg", "full state global rg", rg.ReplicationGroupID) + require.NoError(t, err) + + _, err = original.CreateServerlessCache(ctx, "fs-serverless", "full state serverless", "redis") + require.NoError(t, err) + _, err = original.CreateServerlessCacheSnapshot(ctx, "fs-serverless-snap", "fs-serverless") + require.NoError(t, err) + + _, err = original.CreateUser(ctx, "fs-user", "fs-user-name", "on ~* &* +@all", "redis", true) + require.NoError(t, err) + _, err = original.CreateUserGroup(ctx, "fs-usergroup", "full state user group", "redis", []string{"fs-user"}) + require.NoError(t, err) + + // Hardcoded to match the builtin offering ID in backend_ops2.go's + // builtinReservedOfferings (unexported, so not reachable from this + // external test package by name). + const builtinOfferingID = "31153cd5-4ce6-45a9-b6ce-7f0b6789b8fa" + _, err = original.PurchaseReservedCacheNodesOffering(ctx, builtinOfferingID, "fs-reserved-node", 1) + require.NoError(t, err) + + snap := original.Snapshot(ctx) + require.NotNil(t, snap) + + fresh := elasticache.NewInMemoryBackend("redis", "000000000000", "us-east-1", nil) + require.NoError(t, fresh.Restore(ctx, snap)) + + clusters, err := fresh.DescribeClusters(ctx, "fs-cluster", "", 0, false) + require.NoError(t, err) + require.Len(t, clusters.Data, 1) + + rgs, err := fresh.DescribeReplicationGroups(ctx, "fs-rg", "", 0) + require.NoError(t, err) + require.Len(t, rgs.Data, 1) + + pgs, err := fresh.DescribeParameterGroups(ctx, "fs-pg", "", 0) + require.NoError(t, err) + require.Len(t, pgs.Data, 1) + + sngs, err := fresh.DescribeSubnetGroups(ctx, "fs-sng", "", 0) + require.NoError(t, err) + require.Len(t, sngs.Data, 1) + + snaps, err := fresh.DescribeSnapshots(ctx, "fs-snap", "", "", "", "", 0) + require.NoError(t, err) + require.Len(t, snaps.Data, 1) + + sgs, err := fresh.DescribeCacheSecurityGroups(ctx, "fs-sg", "", 0) + require.NoError(t, err) + require.Len(t, sgs.Data, 1) + + grgs, err := fresh.DescribeGlobalReplicationGroups(ctx, "ldgnf-fsgrg", "", 0) + require.NoError(t, err) + require.Len(t, grgs.Data, 1) + + scs, err := fresh.DescribeServerlessCaches(ctx, "fs-serverless", "", 0) + require.NoError(t, err) + require.Len(t, scs.Data, 1) + + scSnaps, err := fresh.DescribeServerlessCacheSnapshots(ctx, "fs-serverless", "fs-serverless-snap", "", 0) + require.NoError(t, err) + require.Len(t, scSnaps.Data, 1) + + users, err := fresh.DescribeUsers(ctx, "fs-user", "", 0) + require.NoError(t, err) + require.Len(t, users.Data, 1) + + userGroups, err := fresh.DescribeUserGroups(ctx, "fs-usergroup", "", 0) + require.NoError(t, err) + require.Len(t, userGroups.Data, 1) + + reservedNodes, err := fresh.DescribeReservedCacheNodes(ctx, "fs-reserved-node", "", "", "", 0) + require.NoError(t, err) + require.Len(t, reservedNodes.Data, 1) +} + +// TestInMemoryBackend_Restore_IncompatibleVersion feeds Restore a +// well-formed-but-wrong-version snapshot (simulating an older/incompatible +// on-disk format) and checks it is discarded cleanly -- resetting the backend +// to empty rather than erroring or partially decoding the mismatched shape. +func TestInMemoryBackend_Restore_IncompatibleVersion(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + original := elasticache.NewInMemoryBackend("redis", "000000000000", "us-east-1", nil) + _, err := original.CreateCluster(ctx, "versioned-cluster", "redis", "cache.t3.micro", 6379) + require.NoError(t, err) + + // Craft a snapshot with an incompatible version by round-tripping through + // a generic map: bump "version" to a value that can never match the + // current elasticacheSnapshotVersion constant. + var raw map[string]any + + require.NoError(t, json.Unmarshal(original.Snapshot(ctx), &raw)) + raw["version"] = -1 + + badSnap, err := json.Marshal(raw) + require.NoError(t, err) + + fresh := elasticache.NewInMemoryBackend("redis", "000000000000", "us-east-1", nil) + _, err = fresh.CreateCluster(ctx, "pre-existing", "redis", "cache.t3.micro", 6379) + require.NoError(t, err) + + require.NoError(t, fresh.Restore(ctx, badSnap)) + + all := fresh.ListAll() + assert.Empty(t, all, + "incompatible-version restore must reset to empty, not keep pre-existing or partially decoded state") +} diff --git a/services/elasticache/store_setup.go b/services/elasticache/store_setup.go new file mode 100644 index 000000000..fe6d40d68 --- /dev/null +++ b/services/elasticache/store_setup.go @@ -0,0 +1,42 @@ +package elasticache + +// Code in this file supports Phase 3.3 of the datalayer refactor: converting +// InMemoryBackend's resource maps to pkgs/store. See pkgs/store's package doc +// and the services/sqs pilot (commit 0f09d77c) / services/ec2 rollout (commit +// 12e611a4) for the pattern this follows. +// +// Every resource in this backend is nested per-region +// (map[string]map[string]*T -- outer key is region) except +// GlobalReplicationGroup, which is partition-scoped like AWS and therefore +// global (see backend.go's InMemoryBackend doc comment). The region-nested +// resources are converted to map[string]*store.Table[T] (region -> Table) +// with a lazy per-region accessor -- see the "*Store" helpers in backend.go. +// Because the set of regions is only known at runtime, these per-region +// tables are deliberately NOT registered on a *store.Registry: Registry's +// SnapshotAll/RestoreAll require a fixed, construction-time-known table-name +// set, which a dynamic region set can't provide (a region first seen on +// Restore would have nothing registered to restore into). persistence.go +// instead snapshots/restores each per-region Table directly via +// Table.Snapshot()/Table.Restore(). GlobalReplicationGroup has no such +// per-region dynamism, so it alone is registered on b.registry and goes +// through registry.SnapshotAll()/RestoreAll() in persistence.go. +// +// cacheSecurityGroupIngress is deliberately NOT converted: its value type is +// []EC2SecurityGroupMembership (a slice, not a single *T resource keyed by +// its own identity), which store.Table cannot represent -- it remains a +// plain nested map (map[string]map[string][]EC2SecurityGroupMembership). + +func clusterKeyFn(v *Cluster) string { return v.ClusterID } +func replicationGroupKeyFn(v *ReplicationGroup) string { return v.ReplicationGroupID } +func cacheParameterGroupKeyFn(v *CacheParameterGroup) string { return v.Name } +func cacheSubnetGroupKeyFn(v *CacheSubnetGroup) string { return v.Name } +func cacheSnapshotKeyFn(v *CacheSnapshot) string { return v.SnapshotName } +func cacheSecurityGroupKeyFn(v *CacheSecurityGroup) string { return v.Name } +func serverlessCacheKeyFn(v *ServerlessCache) string { return v.Name } +func serverlessCacheSnapshotKeyFn(v *ServerlessCacheSnapshot) string { return v.Name } +func userKeyFn(v *User) string { return v.UserID } +func userGroupKeyFn(v *UserGroup) string { return v.UserGroupID } +func reservedCacheNodeKeyFn(v *ReservedCacheNode) string { return v.ReservedCacheNodeID } +func globalReplicationGroupKeyFn(v *GlobalReplicationGroup) string { + return v.GlobalReplicationGroupID +} diff --git a/services/elbv2/PARITY.md b/services/elbv2/PARITY.md new file mode 100644 index 000000000..f3f650790 --- /dev/null +++ b/services/elbv2/PARITY.md @@ -0,0 +1,214 @@ +--- +service: elbv2 +sdk_module: aws-sdk-go-v2/service/elasticloadbalancingv2@v1.54.8 +last_audit_commit: d118e0d8 +last_audit_date: 2026-07-05 +overall: A # ~240 LOC genuine production-code fixes across several severe/systemic bugs (+~260 LOC test changes/additions); most of the surface was already accurate (see families below) +ops: + CreateLoadBalancer: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteLoadBalancer: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeLoadBalancers: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: NotFound errors were HTTP 404 (should be 400, see Notes)"} + ModifyLoadBalancerAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeLoadBalancerAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + SetSecurityGroups: {wire: ok, errors: ok, state: ok, persist: ok} + SetSubnets: {wire: ok, errors: ok, state: ok, persist: ok} + SetIpAddressType: {wire: ok, errors: ok, state: ok, persist: ok} + CreateTargetGroup: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteTargetGroup: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeTargetGroups: {wire: ok, errors: ok, state: ok, persist: ok} + ModifyTargetGroup: {wire: ok, errors: ok, state: ok, persist: ok} + ModifyTargetGroupAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeTargetGroupAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + RegisterTargets: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: omitted Targets.member.N.Port was stored as 0 instead of defaulting to the target group's port (AWS behaviour), corrupting DescribeTargetHealth/Deregister lookups for any caller that omits Port"} + DeregisterTargets: {wire: ok, errors: ok, state: ok, persist: ok, note: "same Port-defaulting fix as RegisterTargets"} + DescribeTargetHealth: {wire: ok, errors: ok, state: ok, persist: ok, note: "Targets.member.N filter now also defaults omitted Port before matching against registered targets"} + CreateListener: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: AlpnPolicy was modeled/serialized as a bare string; real wire shape is a list (AlpnPolicy.member.N request, response)"} + DeleteListener: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeListeners: {wire: ok, errors: ok, state: ok, persist: ok, note: "AlpnPolicy list-shape fix applies here too"} + ModifyListener: {wire: ok, errors: ok, state: ok, persist: ok, note: "AlpnPolicy list-shape fix applies here too"} + ModifyListenerAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeListenerAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + CreateRule: {wire: ok, errors: ok, state: ok, persist: ok, note: "added fallback to the legacy top-level Values.member.N field for host-header/path-pattern conditions when the modern HostHeaderConfig/PathPatternConfig is absent (both are valid on the real wire)"} + DeleteRule: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeRules: {wire: ok, errors: ok, state: ok, persist: ok} + ModifyRule: {wire: ok, errors: ok, state: ok, persist: ok, note: "same legacy-Values fallback as CreateRule"} + SetRulePriorities: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: the priority-conflict error code was fabricated (\"DuplicatePriority\"); real AWS code is \"PriorityInUse\" (PriorityInUseException)"} + AddTags: {wire: ok, errors: ok, state: ok, persist: ok} + RemoveTags: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeTags: {wire: ok, errors: ok, state: ok, persist: ok} + AddListenerCertificates: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeListenerCertificates: {wire: ok, errors: ok, state: ok, persist: ok} + RemoveListenerCertificates: {wire: ok, errors: ok, state: ok, persist: ok} + AddTrustStoreRevocations: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: response never returned AddTrustStoreRevocationsResult.TrustStoreRevocations at all (empty body) despite the mutation succeeding - classic disguised-stub shape; now echoes the added revocations with RevocationId/RevocationType/NumberOfRevokedEntries/TrustStoreArn"} + CreateTrustStore: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteSharedTrustStoreAssociation: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteTrustStore: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeAccountLimits: {wire: ok, errors: ok, state: ok, persist: n/a, note: "static limits table verified against AWS defaults"} + DescribeCapacityReservation: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeSSLPolicies: {wire: ok, errors: ok, state: ok, persist: n/a, note: "static policy list verified against real AWS SSL policy names/ciphers"} + DescribeTrustStoreAssociations: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeTrustStores: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeTrustStoreRevocations: {wire: ok, errors: ok, state: ok, persist: ok, note: "CRITICAL fix: response list field was named RevocationContents; real wire field (verified against the SDK deserializer) is TrustStoreRevocations. A real SDK client parsing this response would have silently received an EMPTY list on every call despite the mock holding real revocation data"} + GetResourcePolicy: {wire: ok, errors: ok, state: ok, persist: ok} + GetTrustStoreCaCertificatesBundle: {wire: partial, errors: ok, state: ok, persist: n/a, note: "Location is always empty string; there is no real S3-backed bundle to point to in this emulator. Documented gap, not fixed (see gaps)"} + GetTrustStoreRevocationContent: {wire: partial, errors: ok, state: ok, persist: n/a, note: "same Location-always-empty gap as GetTrustStoreCaCertificatesBundle"} + ModifyCapacityReservation: {wire: ok, errors: ok, state: ok, persist: ok} + ModifyIpPools: {wire: ok, errors: ok, state: ok, persist: ok} + ModifyTrustStore: {wire: ok, errors: ok, state: ok, persist: ok} + RemoveTrustStoreRevocations: {wire: ok, errors: ok, state: ok, persist: ok} +families: + error-codes-and-http-status: {status: ok, note: "SYSTEMIC fix — see Notes. All *NotFound / Duplicate* / ResourceInUse / OperationNotPermitted / InvalidConfigurationRequest / PriorityInUse sentinel errors now map to HTTP 400, matching real AWS query-protocol behaviour (verified against the elasticloadbalancingv2 api-2.json model, which sets httpStatusCode=400 for every exception shape in this service). Previously NotFound errors returned 404 and AlreadyExists/DuplicateListener returned 409, which is REST-JSON-style, not query-protocol-style (EC2, also query-protocol, already uses 400-for-everything in this codebase - confirmed as the established, correct pattern)." + actions (forward/redirect/fixed-response/authenticate-cognito/authenticate-oidc): {status: ok, note: "verified field-by-field against Action/RedirectActionConfig/FixedResponseActionConfig/ForwardActionConfig/AuthenticateCognitoActionConfig/AuthenticateOidcActionConfig; all wire field names and nesting correct, no changes needed"} + conditions (host-header/path-pattern/http-header/query-string/source-ip/http-request-method): {status: ok, note: "verified nested *Config wire shapes (HostHeaderConfig.Values.member.N etc.) against RuleCondition; added legacy top-level Values.member.N fallback for host-header/path-pattern (see CreateRule/ModifyRule above). RegexValues (a newer AWS condition feature) is not implemented - see deferred."} + listener-certificates / ssl-policies / trust-stores (association, revocation add/remove/describe): {status: ok, note: "AddTrustStoreRevocations and DescribeTrustStoreRevocations wire bugs fixed (see ops above); everything else in this family verified correct"} + target-health-lifecycle (initial->healthy transition, draining->removed transition, reason codes): {status: ok, note: "healthStateHealthy/unhealthy/initial/draining and Elb.InitialHealthChecking/Target.DeregistrationInProgress/Target.NotRegistered reason codes verified byte-for-byte against types.TargetHealthStateEnum/TargetHealthReasonEnum. Port-defaulting fix applies across Register/Deregister/DescribeTargetHealth (see ops above)."} + load-balancer-attributes / target-group-attributes / listener-attributes (Modify/Describe): {status: ok, note: "unchanged this pass; default attribute maps for ALB vs NLB/GWLB verified against real AWS defaults"} + capacity-reservation / ip-pools / resource-policy / account-limits / ssl-policies: {status: ok, note: "unchanged this pass; verified op-by-op, all accurate"} +gaps: + - ASG/ECS -> ELBv2 target registration is cross-service: RegisterTargets/DeregisterTargets/DescribeTargetHealth on the ELBv2 side are correct and complete (verified and improved this pass - see ops), but nothing on the ASG/ECS side calls them when instances/tasks scale (bd: gopherstack-18k) - NOT fixed here, out of scope per task instructions (elbv2-only edits) + - GetTrustStoreCaCertificatesBundle / GetTrustStoreRevocationContent always return an empty Location (no real S3-backed object to point to) - documented simplification, not a hidden stub (the ops correctly validate the trust store/revocation exist and return 400 TrustStoreNotFound/RevocationIdNotFound otherwise) + - RevocationId is modeled/returned as a string in this mock (echoing the caller-supplied plain content, or a generated "s3-" for S3-structured entries); real AWS RevocationId is an int64 assigned by AWS when it parses the uploaded CRL/bundle. Existing tests in this package already encode string-shaped RevocationIds (e.g. "s3://my-bucket/revocations.crl", "1") predating this pass. Reworking this to a real monotonic int64 ID space is a moderate, invasive change (touches the TrustStoreRevocation struct, persistence JSON shape, and ~6 existing tests) for a rarely-exercised feature; deferred rather than rushed. No bd id filed yet - recommend filing one if this is prioritized. + - The plain (non-S3) `RevocationContents.member.N` request field parsed by parseTrustStoreRevocations does not exist in the real AWS API (RevocationContent is always S3Bucket/S3Key/S3ObjectVersion/RevocationType - verified against types.RevocationContent) - harmless (real clients never send that shape so the branch is simply unreachable in practice), but worth removing in a future cleanup pass +deferred: + - RegexValues on rule conditions (a newer AWS feature letting host-header/path-pattern/http-header conditions match by regex instead of exact/wildcard Values) - not implemented at all; SDK-side type exists (RuleCondition.RegexValues []string) but this pass did not add parsing/serialization for it + - AuthenticateCognitoConfig/AuthenticateOidcConfig were verified for field-name accuracy only, not behaviorally exercised (this emulator does not implement actual OIDC/Cognito redirect flows, matching every other gopherstack service's scope for auth actions) +leaks: {status: clean, note: "runHealthReconciler's ticker-based goroutine is unchanged and already correctly stopped by Close(); targetReadyAt/targetDrainingUntil maps were the one place persistence was incomplete (see Notes) - now both fully round-trip through Snapshot/Restore, including a nil-map defensive re-init on Restore for snapshots taken before targetDrainingUntil existed (assigning into a nil top-level map panics on next write, e.g. the next RegisterTargets/DeregisterTargets call after a restore)."} +--- + +## Notes + +Protocol: ELBv2 uses the classic AWS "Query" protocol (form-urlencoded request, +`Version=2015-12-01`, XML response with `...`), the same +family as EC2 and Auto Scaling in this codebase. Verified against +`aws-sdk-go-v2/service/elasticloadbalancingv2@v1.54.8`'s `deserializers.go` / +`types/errors.go`, and cross-checked exact error codes + HTTP statuses against the +upstream `aws-sdk-go@v1.55.5` `models/apis/elasticloadbalancingv2/2015-12-01/api-2.json` +(every exception shape in that model declares `httpStatusCode: 400`, no exceptions). + +### Highest-value finding: error HTTP status codes were REST-JSON-shaped, not query-protocol-shaped + +Real AWS ELBv2 (and every other query-protocol service) returns **HTTP 400 for every +client error**, including NotFound and AlreadyExists conditions — the SDK's error +deserializer dispatches purely on the `` XML text (verified by reading +`awsAwsquery_deserializeOpErrorCreateLoadBalancer` etc.), never on HTTP status. Before +this pass, `elbv2ErrorCode` mapped `*NotFound` sentinels to `http.StatusNotFound` (404) +and `*AlreadyExists`/`DuplicateListener` to `http.StatusConflict` (409) — a REST-JSON +convention that doesn't apply here. This is invisible to a Go SDK client (which only +looks at the XML ``) but wire-inaccurate for anything that inspects the raw HTTP +status (curl-based tooling, non-SDK clients, some retry middlewares). Confirmed EC2 in +this same codebase (also query-protocol) already uses `http.StatusBadRequest` +uniformly for its entire `errCodeLookup` table — i.e. elbv2's 404/409 usage was the +anomaly relative to established codebase convention, not the norm. Fixed by changing +every `NotFound`/`AlreadyExists`/`DuplicateListener` mapping in `elbv2ErrorCode` to 400, +and updated ~45 test assertions across `handler_test.go`, +`handler_accuracy_batch1_test.go`, `handler_accuracy_batch2_test.go`, and +`parity_b_test.go` that had encoded the wrong (404/409) expectation. + +### AlpnPolicy wire-shape bug (wrong list wrapper — parity-principles.md bug class #2) + +`Listener.AlpnPolicy` was modeled as a bare `string`, parsed via +`vals.Get("AlpnPolicy.member.1")` and serialized as `value`. +Real AWS (`types.Listener.AlpnPolicy` is `[]string`, verified against the SDK's +`awsAwsquery_deserializeDocumentAlpnPolicyName` which decodes a ``-wrapped +list) requires `AlpnPolicy.member.N` on requests and `... +` on responses. Fixed end-to-end: `Listener.AlpnPolicy`, +`CreateListenerInput.AlpnPolicy`, `ModifyListenerInput.AlpnPolicy` are now `[]string`; +the handler parses all `AlpnPolicy.member.N` values; the XML projection uses a +`*xmlStringList` (nil when empty, omitted from the response, matching the +`Certificates` field's existing convention). Added `Test_AlpnPolicyWireShape` +(table-driven: single policy / multiple policies / no policy) covering +CreateListener + DescribeListeners round-trip. + +### DescribeTrustStoreRevocations / AddTrustStoreRevocations wire-shape bugs + +Two related bugs, same root cause (never checked the SDK deserializer for the exact +result field name): + +1. `DescribeTrustStoreRevocationsResult`'s list field was named `RevocationContents` + in the mock. The real field (verified against + `awsAwsquery_deserializeOpDocumentDescribeTrustStoreRevocationsOutput`, which only + recognizes `TrustStoreRevocations` and silently `Skip()`s anything else) is + `TrustStoreRevocations`. A real SDK client would have received an **empty list on + every call**, even though the mock's internal state held real revocation data — + this is the "wrong root element" bug class from parity-principles.md #2, just one + level deeper (correct outer `DescribeTrustStoreRevocationsResult` root, wrong inner + list field). +2. `AddTrustStoreRevocations`'s response never included a Result section at all — the + mutation succeeded server-side but the client got back an empty envelope. Real AWS + returns `AddTrustStoreRevocationsResult.TrustStoreRevocations` describing exactly + what was added (RevocationId/RevocationType/NumberOfRevokedEntries/TrustStoreArn). + This is the "op returns real state but is missing the documented response shape" + pattern flagged in parity-principles.md #4 — worth double-checking on any op whose + test coverage only asserts `http.StatusOK` without decoding the body (which is + exactly how this one stayed hidden: `TestELBv2_TrustStoreFullLifecycle` asserted + `revRec.Code == 200` but never unmarshalled `revRec.Body`). + +Both fixed; `xmlRevocationContent` gained a `TrustStoreArn` field (present on both real +AWS types `TrustStoreRevocation` and `DescribeTrustStoreRevocation`), and +`TestELBv2_TrustStoreFullLifecycle` was corrected to decode against +`TrustStoreRevocations` (not `RevocationContents`) and extended to assert the +`AddTrustStoreRevocations` response body content. + +### PriorityInUse error code + +`ErrDuplicateRulePriority` was wired to the fabricated code `"DuplicatePriority"`. Real +AWS (verified against `types.PriorityInUseException`/api-2.json) uses +`"PriorityInUse"`. Fixed in `backend.go` and `elbv2ErrorCode`; updated +`TestDuplicateRulePriorityErrorCode` (this test encoded the wrong behaviour, per +parity-principles.md's "fix tests only where they encoded wrong behavior" rule). + +### Target port defaulting + +`RegisterTargets`/`DeregisterTargets`/`DescribeTargetHealth`'s `Targets.member.N.Port` +is optional on the real wire (`types.TargetDescription.Port *int32`) — when omitted, +AWS defaults it to the target group's configured port. The mock previously stored/ +matched on a bare `0` for any caller that omitted Port, which silently broke +`DescribeTargetHealth`/`DeregisterTargets` lookups for such targets (their registered +port would never match a later request's implicit-zero port unless the caller +"happened" to also omit Port every time). Fixed via a new +`Handler.resolveTargetGroupPort`/`defaultTargetPorts` pair applied in all three +handlers — additive, no `StorageBackend` interface signature change. + +### Persistence gap: targetDrainingUntil never survived Restore + +`InMemoryBackend.targetDrainingUntil` (tracks when a draining/deregistering target +should be actually removed from its target group) was entirely absent from +`backendSnapshot`. After a Restore, any target that was mid-drain at snapshot time +would keep its `HealthState="draining"` forever — `reconcileTargetHealth`'s expiry +goroutine had no record of when to remove it. Separately, `targetReadyAt` WAS in the +snapshot but `Restore()` was missing the standard nil-guard the other maps got, +meaning it could restore as a literal `nil` map for old/empty snapshots; the very next +`RegisterTargets` call assigning into `b.targetReadyAt[tgArn][key]` would then panic +("assignment to entry in nil map"). Both fixed: `targetDrainingUntil` added to the +snapshot type/Snapshot()/Restore(), and both `targetReadyAt`/`targetDrainingUntil` get +a defensive `make(...)` in `Restore()` when nil. + +### Legacy top-level `Values` fallback for host-header/path-pattern conditions + +AWS's `RuleCondition` has both a modern `HostHeaderConfig`/`PathPatternConfig` (list of +values) and a deprecated top-level `Values` field (single value, still accepted on the +wire per `types.go`'s doc comment). The mock only ever read the modern +`*Config.Values.member.N` form. Added a fallback to the legacy +`Conditions.member.N.Values.member.N` form when the modern form is empty — cheap, +additive, and closes a real (if rarely hit by modern SDKs/Terraform) parsing gap. + +### Traps for the next auditor + +- `probeTargetHTTP`'s "unreachable → treat as healthy" fallback (backend.go) looks + backwards at first glance but is intentional: this emulator has no real backend + server to health-check in the general case, so treating connection failures as + healthy avoids every mock target group getting stuck "unhealthy" forever just + because nothing is actually listening on the target's host:port. Don't "fix" this + without re-reading the comment above it. +- `xmlRevocationContent` is now shared by both `AddTrustStoreRevocationsResult` and + `DescribeTrustStoreRevocationsResult` — they are two *different* real AWS types + (`TrustStoreRevocation` vs `DescribeTrustStoreRevocation`) that happen to have + identical fields on the wire. This is intentional reuse, not a shortcut. +- The `RevocationContents.member.N` (S3Bucket/S3Key/RevocationType) request field name + is correct and matches real AWS `RevocationContent` — don't confuse it with the + *response*-side `TrustStoreRevocations` field name fixed this pass; they are + different names for request vs. response despite both being about "revocation + content". +- Every `errors: ok` HTTP status in `ops` above now means **400**, not "whatever seems + RESTful" — re-verify against api-2.json (not intuition) before changing any status + code in `elbv2ErrorCode`. diff --git a/services/elbv2/backend.go b/services/elbv2/backend.go index 7c8a20fab..d8f41bcbe 100644 --- a/services/elbv2/backend.go +++ b/services/elbv2/backend.go @@ -17,6 +17,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/awserr" "github.com/blackbirdworks/gopherstack/pkgs/collections" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) @@ -42,7 +43,9 @@ var ( // ErrUnknownAction is returned when the requested action is not recognized. ErrUnknownAction = awserr.New("InvalidAction", awserr.ErrInvalidParameter) // ErrDuplicateRulePriority is returned when two rules have the same priority. - ErrDuplicateRulePriority = awserr.New("DuplicatePriority", awserr.ErrInvalidParameter) + // AWS's real error code for this condition is "PriorityInUse" (PriorityInUseException), + // not "DuplicatePriority" — verified against aws-sdk-go-v2/service/elasticloadbalancingv2/types. + ErrDuplicateRulePriority = awserr.New("PriorityInUse", awserr.ErrInvalidParameter) // ErrOperationNotPermitted is returned when the operation is not allowed (e.g. deleting default rule). ErrOperationNotPermitted = awserr.New("OperationNotPermitted", awserr.ErrInvalidParameter) // ErrDuplicateListener is returned when a listener on the same port already exists. @@ -263,10 +266,12 @@ type Listener struct { LoadBalancerArn string `json:"loadBalancerArn"` Protocol string `json:"protocol"` SSLPolicy string `json:"sslPolicy,omitempty"` - AlpnPolicy string `json:"alpnPolicy,omitempty"` - DefaultActions []Action `json:"defaultActions"` - Certificates []Certificate `json:"certificates,omitempty"` - Port int32 `json:"port"` + // AlpnPolicy is a list on the wire (AlpnPolicy.member.N / …), + // not a bare string — verified against aws-sdk-go-v2 types.Listener.AlpnPolicy ([]string). + AlpnPolicy []string `json:"alpnPolicy,omitempty"` + DefaultActions []Action `json:"defaultActions"` + Certificates []Certificate `json:"certificates,omitempty"` + Port int32 `json:"port"` } // Rule represents an ELBv2 listener rule. @@ -407,7 +412,7 @@ type CreateListenerInput struct { LoadBalancerArn string Protocol string SSLPolicy string - AlpnPolicy string + AlpnPolicy []string DefaultActions []Action Tags []tags.KV Certificates []Certificate @@ -420,7 +425,7 @@ type ModifyListenerInput struct { ListenerArn string Protocol string SSLPolicy string - AlpnPolicy string + AlpnPolicy []string DefaultActions []Action Certificates []Certificate Port int32 @@ -454,14 +459,23 @@ const ( ) type InMemoryBackend struct { - loadBalancers map[string]*LoadBalancer // keyed by ARN - targetGroups map[string]*TargetGroup // keyed by ARN - listeners map[string]*Listener // keyed by ARN - rules map[string]*Rule // keyed by ARN - trustStores map[string]*TrustStore // keyed by ARN - // resourcePolicies stores resource policies keyed by ResourceArn. + registry *store.Registry + loadBalancers *store.Table[LoadBalancer] // keyed by ARN + targetGroups *store.Table[TargetGroup] // keyed by ARN + listeners *store.Table[Listener] // keyed by ARN + // listenersByLB indexes listeners by their owning load balancer ARN. + listenersByLB *store.Index[Listener] + rules *store.Table[Rule] // keyed by ARN + // rulesByListener indexes rules by their owning listener ARN. + rulesByListener *store.Index[Rule] + trustStores *store.Table[TrustStore] // keyed by ARN + // resourcePolicies stores resource policies keyed by ResourceArn. Left as a + // plain map (not a store.Table) because the value is a bare string with no + // identity field of its own -- see store_setup.go. resourcePolicies map[string]string // lifecycle: tracks when initial targets become healthy / start draining. + // Left as plain maps (not store.Tables) because they are doubly-nested + // (tgArn → targetKey → timestamp), not a map[string]*V shape -- see store_setup.go. targetReadyAt map[string]map[string]time.Time // tgArn → targetKey → readyAt (initial→healthy) targetDrainingUntil map[string]map[string]time.Time // tgArn → targetKey → drainExpiresAt mu *lockmetrics.RWMutex @@ -474,11 +488,7 @@ type InMemoryBackend struct { // NewInMemoryBackend creates a new in-memory ELBv2 backend. func NewInMemoryBackend(accountID, region string) *InMemoryBackend { b := &InMemoryBackend{ - loadBalancers: make(map[string]*LoadBalancer), - targetGroups: make(map[string]*TargetGroup), - listeners: make(map[string]*Listener), - rules: make(map[string]*Rule), - trustStores: make(map[string]*TrustStore), + registry: store.NewRegistry(), resourcePolicies: make(map[string]string), accountID: accountID, region: region, @@ -488,6 +498,8 @@ func NewInMemoryBackend(accountID, region string) *InMemoryBackend { stopCh: make(chan struct{}), } + registerAllTables(b) + go b.runHealthReconciler() return b @@ -556,7 +568,7 @@ func (b *InMemoryBackend) collectPendingTargets(now time.Time) []pendingTarget { for tgArn, readyMap := range b.targetReadyAt { for key, readyAt := range readyMap { if now.After(readyAt) { - if tg := b.targetGroups[tgArn]; tg != nil { + if tg, ok := b.targetGroups.Get(tgArn); ok { pending = append(pending, pendingTarget{tgArn: tgArn, targetKey: key, tg: tg}) } } @@ -584,7 +596,7 @@ func resolveTargetHealth(pending []pendingTarget) []healthResult { func (b *InMemoryBackend) applyHealthResults(results []healthResult) { for _, r := range results { - tg, ok := b.targetGroups[r.tgArn] + tg, ok := b.targetGroups.Get(r.tgArn) if !ok { continue } @@ -629,7 +641,7 @@ func (b *InMemoryBackend) collectDrainedTargets(now time.Time) []drainedTarget { // Caller must hold b.mu (write). func (b *InMemoryBackend) removeDrainedTargets(drained []drainedTarget) { for _, d := range drained { - tg, ok := b.targetGroups[d.tgArn] + tg, ok := b.targetGroups.Get(d.tgArn) if !ok { continue } @@ -926,7 +938,7 @@ func (b *InMemoryBackend) CreateLoadBalancer(input CreateLoadBalancerInput) (*Lo return nil, err } - for _, lb := range b.loadBalancers { + for _, lb := range b.loadBalancers.All() { if lb.LoadBalancerName == input.Name { return nil, ErrLoadBalancerAlreadyExists } @@ -1001,7 +1013,7 @@ func (b *InMemoryBackend) CreateLoadBalancer(input CreateLoadBalancerInput) (*Lo Tags: t, } - b.loadBalancers[lbArn] = lb + b.loadBalancers.Put(lb) cp := *lb @@ -1104,7 +1116,7 @@ func (b *InMemoryBackend) DescribeLoadBalancers( result := make([]LoadBalancer, 0, len(arns)) for _, a := range arns { - if lb, ok := b.loadBalancers[a]; ok { + if lb, ok := b.loadBalancers.Get(a); ok { result = append(result, *lb) } } @@ -1150,9 +1162,9 @@ func (b *InMemoryBackend) filterLoadBalancersLocked(arns, names []string) []Load nameSet[n] = true } - result := make([]LoadBalancer, 0, len(b.loadBalancers)) + result := make([]LoadBalancer, 0, b.loadBalancers.Len()) - for _, lb := range b.loadBalancers { + for _, lb := range b.loadBalancers.All() { if len(arns) > 0 && !arnSet[lb.LoadBalancerArn] { continue } @@ -1179,29 +1191,29 @@ func (b *InMemoryBackend) DeleteLoadBalancer(lbArn string) error { b.mu.Lock("DeleteLoadBalancer") defer b.mu.Unlock() - if _, ok := b.loadBalancers[lbArn]; !ok { + lb, ok := b.loadBalancers.Get(lbArn) + if !ok { return ErrLoadBalancerNotFound } - // Cascade: delete all listeners and their rules. - for listenerArn, l := range b.listeners { - if l.LoadBalancerArn != lbArn { - continue - } - - for ruleArn, r := range b.rules { - if r.ListenerArn == listenerArn { - r.Tags.Close() - delete(b.rules, ruleArn) - } + // Cascade: delete all listeners and their rules. The index lookups are + // copied into fresh slices first because Table.Delete mutates the very + // index groups Index.Get returns; iterating the live group while deleting + // from it would corrupt the in-progress scan. + listenersToDelete := append([]*Listener(nil), b.listenersByLB.Get(lbArn)...) + for _, l := range listenersToDelete { + rulesToDelete := append([]*Rule(nil), b.rulesByListener.Get(l.ListenerArn)...) + for _, r := range rulesToDelete { + r.Tags.Close() + b.rules.Delete(r.RuleArn) } l.Tags.Close() - delete(b.listeners, listenerArn) + b.listeners.Delete(l.ListenerArn) } - b.loadBalancers[lbArn].Tags.Close() - delete(b.loadBalancers, lbArn) + lb.Tags.Close() + b.loadBalancers.Delete(lbArn) return nil } @@ -1214,7 +1226,7 @@ func (b *InMemoryBackend) ModifyLoadBalancerAttributes( b.mu.Lock("ModifyLoadBalancerAttributes") defer b.mu.Unlock() - lb, ok := b.loadBalancers[lbArn] + lb, ok := b.loadBalancers.Get(lbArn) if !ok { return nil, ErrLoadBalancerNotFound } @@ -1235,7 +1247,7 @@ func (b *InMemoryBackend) SetSecurityGroups(lbArn string, sgs []string) (*LoadBa b.mu.Lock("SetSecurityGroups") defer b.mu.Unlock() - lb, ok := b.loadBalancers[lbArn] + lb, ok := b.loadBalancers.Get(lbArn) if !ok { return nil, ErrLoadBalancerNotFound } @@ -1261,7 +1273,7 @@ func (b *InMemoryBackend) SetSubnets( b.mu.Lock("SetSubnets") defer b.mu.Unlock() - lb, ok := b.loadBalancers[lbArn] + lb, ok := b.loadBalancers.Get(lbArn) if !ok { return nil, ErrLoadBalancerNotFound } @@ -1277,7 +1289,7 @@ func (b *InMemoryBackend) SetIPAddressType(lbArn string, ipType string) (*LoadBa b.mu.Lock("SetIPAddressType") defer b.mu.Unlock() - lb, ok := b.loadBalancers[lbArn] + lb, ok := b.loadBalancers.Get(lbArn) if !ok { return nil, ErrLoadBalancerNotFound } @@ -1312,7 +1324,7 @@ func (b *InMemoryBackend) CreateTargetGroup(input CreateTargetGroupInput) (*Targ return nil, err } - for _, tg := range b.targetGroups { + for _, tg := range b.targetGroups.All() { if tg.TargetGroupName == input.Name { return nil, ErrTargetGroupAlreadyExists } @@ -1392,7 +1404,7 @@ func (b *InMemoryBackend) CreateTargetGroup(input CreateTargetGroupInput) (*Targ Tags: t, } - b.targetGroups[tgArn] = tg + b.targetGroups.Put(tg) cp := *tg @@ -1539,17 +1551,11 @@ func actionsReferenceTG(actions []Action, tgArn string) bool { func (b *InMemoryBackend) tgArnsForLB(lbArn string) map[string]bool { arns := make(map[string]bool) - for _, l := range b.listeners { - if l.LoadBalancerArn != lbArn { - continue - } - + for _, l := range b.listenersByLB.Get(lbArn) { collectTGArns(l.DefaultActions, arns) - for _, r := range b.rules { - if r.ListenerArn == l.ListenerArn { - collectTGArns(r.Actions, arns) - } + for _, r := range b.rulesByListener.Get(l.ListenerArn) { + collectTGArns(r.Actions, arns) } } @@ -1561,13 +1567,11 @@ func (b *InMemoryBackend) tgArnsForLB(lbArn string) map[string]bool { func (b *InMemoryBackend) tgToLBArnsLocked() map[string]map[string]bool { result := make(map[string]map[string]bool) - for _, l := range b.listeners { + for _, l := range b.listeners.All() { collectLBArnsForTG(l.LoadBalancerArn, l.DefaultActions, result) - for _, r := range b.rules { - if r.ListenerArn == l.ListenerArn { - collectLBArnsForTG(l.LoadBalancerArn, r.Actions, result) - } + for _, r := range b.rulesByListener.Get(l.ListenerArn) { + collectLBArnsForTG(l.LoadBalancerArn, r.Actions, result) } } @@ -1594,7 +1598,7 @@ func (b *InMemoryBackend) DescribeTargetGroups( result := make([]TargetGroup, 0, len(arns)) for _, a := range arns { - tg, ok := b.targetGroups[a] + tg, ok := b.targetGroups.Get(a) if !ok { continue } @@ -1653,9 +1657,9 @@ func (b *InMemoryBackend) filterTargetGroupsLocked( lbTGArns = b.tgArnsForLB(lbArn) } - result := make([]TargetGroup, 0, len(b.targetGroups)) + result := make([]TargetGroup, 0, b.targetGroups.Len()) - for _, tg := range b.targetGroups { + for _, tg := range b.targetGroups.All() { if len(arns) > 0 && !arnSet[tg.TargetGroupArn] { continue } @@ -1693,13 +1697,13 @@ func sortedLBArns(set map[string]bool) []string { // isTGInUseLocked returns true if the target group ARN is referenced by any listener or rule. // Caller must hold b.mu (read or write). func (b *InMemoryBackend) isTGInUseLocked(tgArn string) bool { - for _, l := range b.listeners { + for _, l := range b.listeners.All() { if actionsReferenceTG(l.DefaultActions, tgArn) { return true } } - for _, r := range b.rules { + for _, r := range b.rules.All() { if actionsReferenceTG(r.Actions, tgArn) { return true } @@ -1713,7 +1717,7 @@ func (b *InMemoryBackend) DeleteTargetGroup(tgArn string) error { b.mu.Lock("DeleteTargetGroup") defer b.mu.Unlock() - if _, ok := b.targetGroups[tgArn]; !ok { + if _, ok := b.targetGroups.Get(tgArn); !ok { return ErrTargetGroupNotFound } @@ -1725,8 +1729,9 @@ func (b *InMemoryBackend) DeleteTargetGroup(tgArn string) error { ) } - b.targetGroups[tgArn].Tags.Close() - delete(b.targetGroups, tgArn) + tg, _ := b.targetGroups.Get(tgArn) + tg.Tags.Close() + b.targetGroups.Delete(tgArn) return nil } @@ -1736,7 +1741,7 @@ func (b *InMemoryBackend) RegisterTargets(tgArn string, targets []Target) error b.mu.Lock("RegisterTargets") defer b.mu.Unlock() - tg, ok := b.targetGroups[tgArn] + tg, ok := b.targetGroups.Get(tgArn) if !ok { return ErrTargetGroupNotFound } @@ -1776,7 +1781,7 @@ func (b *InMemoryBackend) DeregisterTargets(tgArn string, targets []Target) erro b.mu.Lock("DeregisterTargets") defer b.mu.Unlock() - tg, ok := b.targetGroups[tgArn] + tg, ok := b.targetGroups.Get(tgArn) if !ok { return ErrTargetGroupNotFound } @@ -1818,7 +1823,7 @@ func (b *InMemoryBackend) DescribeTargetHealth(tgArn string) ([]TargetHealthDesc b.mu.RLock("DescribeTargetHealth") defer b.mu.RUnlock() - tg, ok := b.targetGroups[tgArn] + tg, ok := b.targetGroups.Get(tgArn) if !ok { return nil, ErrTargetGroupNotFound } @@ -1850,7 +1855,7 @@ func (b *InMemoryBackend) SetTargetHealthState( b.mu.Lock("SetTargetHealthState") defer b.mu.Unlock() - tg, ok := b.targetGroups[tgArn] + tg, ok := b.targetGroups.Get(tgArn) if !ok { return ErrTargetGroupNotFound } @@ -1950,9 +1955,12 @@ func requireCertsForProtocol(proto string, certs []Certificate) error { return nil } -func checkDuplicateListenerPort(listeners map[string]*Listener, lbArn string, port int32) error { - for _, existing := range listeners { - if existing.LoadBalancerArn == lbArn && existing.Port == port { +// checkDuplicateListenerPort returns ErrDuplicateListener if any listener in candidates +// (expected to already be scoped to a single load balancer, e.g. via the listenersByLB index) +// is bound to port. +func checkDuplicateListenerPort(candidates []*Listener, port int32) error { + for _, existing := range candidates { + if existing.Port == port { return fmt.Errorf( "%w: a listener on port %d already exists on this load balancer", ErrDuplicateListener, port, @@ -1968,7 +1976,7 @@ func (b *InMemoryBackend) CreateListener(input CreateListenerInput) (*Listener, b.mu.Lock("CreateListener") defer b.mu.Unlock() - lb, ok := b.loadBalancers[input.LoadBalancerArn] + lb, ok := b.loadBalancers.Get(input.LoadBalancerArn) if !ok { return nil, ErrLoadBalancerNotFound } @@ -1988,7 +1996,7 @@ func (b *InMemoryBackend) CreateListener(input CreateListenerInput) (*Listener, input.SSLPolicy = "ELBSecurityPolicy-2016-08" } - if err := checkDuplicateListenerPort(b.listeners, input.LoadBalancerArn, input.Port); err != nil { + if err := checkDuplicateListenerPort(b.listenersByLB.Get(input.LoadBalancerArn), input.Port); err != nil { return nil, err } @@ -2021,21 +2029,21 @@ func (b *InMemoryBackend) CreateListener(input CreateListenerInput) (*Listener, Tags: t, } - b.listeners[listenerArn] = listener + b.listeners.Put(listener) // Auto-create default rule (AWS behaviour: every listener has a default rule). defaultRuleArn := b.ruleARN(listenerArn, priorityDefault) defaultTags := tags.New("elbv2.rule." + defaultRuleArn + ".tags") defaultActions := make([]Action, len(input.DefaultActions)) copy(defaultActions, input.DefaultActions) - b.rules[defaultRuleArn] = &Rule{ + b.rules.Put(&Rule{ RuleArn: defaultRuleArn, ListenerArn: listenerArn, Priority: priorityDefault, IsDefault: true, Actions: defaultActions, Tags: defaultTags, - } + }) cp := *listener @@ -2048,7 +2056,7 @@ func (b *InMemoryBackend) describeListenersByARNs(listenerArns []string) ([]List result := make([]Listener, 0, len(listenerArns)) for _, a := range listenerArns { - if l, ok := b.listeners[a]; ok { + if l, ok := b.listeners.Get(a); ok { result = append(result, *l) } } @@ -2064,7 +2072,7 @@ func (b *InMemoryBackend) describeListenersByARNs(listenerArns []string) ([]List // Callers must hold at least a read lock. func (b *InMemoryBackend) checkLBExists(lbArn string) error { if lbArn != "" { - if _, ok := b.loadBalancers[lbArn]; !ok { + if _, ok := b.loadBalancers.Get(lbArn); !ok { return ErrLoadBalancerNotFound } } @@ -2097,9 +2105,9 @@ func (b *InMemoryBackend) DescribeListeners( arnSet[a] = true } - result := make([]Listener, 0, len(b.listeners)) + result := make([]Listener, 0, b.listeners.Len()) - for _, l := range b.listeners { + for _, l := range b.listeners.All() { if lbArn != "" && l.LoadBalancerArn != lbArn { continue } @@ -2169,20 +2177,22 @@ func (b *InMemoryBackend) DeleteListener(listenerArn string) error { b.mu.Lock("DeleteListener") defer b.mu.Unlock() - if _, ok := b.listeners[listenerArn]; !ok { + listener, ok := b.listeners.Get(listenerArn) + if !ok { return ErrListenerNotFound } - // Cascade: delete all rules belonging to this listener. - for ruleArn, r := range b.rules { - if r.ListenerArn == listenerArn { - r.Tags.Close() - delete(b.rules, ruleArn) - } + // Cascade: delete all rules belonging to this listener. The index lookup is + // copied into a fresh slice first because Table.Delete mutates the very + // index group Index.Get returns (see DeleteLoadBalancer). + rulesToDelete := append([]*Rule(nil), b.rulesByListener.Get(listenerArn)...) + for _, r := range rulesToDelete { + r.Tags.Close() + b.rules.Delete(r.RuleArn) } - b.listeners[listenerArn].Tags.Close() - delete(b.listeners, listenerArn) + listener.Tags.Close() + b.listeners.Delete(listenerArn) return nil } @@ -2190,8 +2200,8 @@ func (b *InMemoryBackend) DeleteListener(listenerArn string) error { // syncDefaultRuleActions updates the default rule's actions to match the listener's new default actions. // Caller must hold b.mu (write). func (b *InMemoryBackend) syncDefaultRuleActions(listenerArn string, actions []Action) { - for _, r := range b.rules { - if r.ListenerArn == listenerArn && r.IsDefault { + for _, r := range b.rulesByListener.Get(listenerArn) { + if r.IsDefault { actsCopy := make([]Action, len(actions)) copy(actsCopy, actions) r.Actions = actsCopy @@ -2206,7 +2216,7 @@ func (b *InMemoryBackend) ModifyListener(input ModifyListenerInput) (*Listener, b.mu.Lock("ModifyListener") defer b.mu.Unlock() - l, ok := b.listeners[input.ListenerArn] + l, ok := b.listeners.Get(input.ListenerArn) if !ok { return nil, ErrListenerNotFound } @@ -2216,7 +2226,7 @@ func (b *InMemoryBackend) ModifyListener(input ModifyListenerInput) (*Listener, } if input.Port != 0 && input.Port != l.Port { - if err := checkDuplicateListenerPort(b.listeners, l.LoadBalancerArn, input.Port); err != nil { + if err := checkDuplicateListenerPort(b.listenersByLB.Get(l.LoadBalancerArn), input.Port); err != nil { return nil, err } @@ -2236,7 +2246,7 @@ func (b *InMemoryBackend) ModifyListener(input ModifyListenerInput) (*Listener, l.SSLPolicy = input.SSLPolicy } - if input.AlpnPolicy != "" { + if len(input.AlpnPolicy) > 0 { l.AlpnPolicy = input.AlpnPolicy } @@ -2256,7 +2266,7 @@ func (b *InMemoryBackend) applyListenerProtocol(l *Listener, input ModifyListene return nil } - if lb, ok := b.loadBalancers[l.LoadBalancerArn]; ok { + if lb, ok := b.loadBalancers.Get(l.LoadBalancerArn); ok { if err := validateListenerProtocol(lb.Type, input.Protocol); err != nil { return err } @@ -2281,7 +2291,7 @@ func (b *InMemoryBackend) CreateRule(input CreateRuleInput) (*Rule, error) { b.mu.Lock("CreateRule") defer b.mu.Unlock() - if _, ok := b.listeners[input.ListenerArn]; !ok { + if _, ok := b.listeners.Get(input.ListenerArn); !ok { return nil, ErrListenerNotFound } @@ -2295,8 +2305,8 @@ func (b *InMemoryBackend) CreateRule(input CreateRuleInput) (*Rule, error) { ) } - for _, r := range b.rules { - if r.ListenerArn == input.ListenerArn && r.Priority == input.Priority { + for _, r := range b.rulesByListener.Get(input.ListenerArn) { + if r.Priority == input.Priority { return nil, fmt.Errorf( "%w: priority %s already in use", ErrDuplicateRulePriority, @@ -2324,7 +2334,7 @@ func (b *InMemoryBackend) CreateRule(input CreateRuleInput) (*Rule, error) { Tags: t, } - b.rules[ruleArn] = rule + b.rules.Put(rule) cp := *rule @@ -2343,7 +2353,7 @@ func (b *InMemoryBackend) DescribeRules(listenerArn string, ruleArns []string) ( result := make([]Rule, 0, len(ruleArns)) for _, a := range ruleArns { - if r, ok := b.rules[a]; ok { + if r, ok := b.rules.Get(a); ok { result = append(result, *r) } } @@ -2362,9 +2372,9 @@ func (b *InMemoryBackend) DescribeRules(listenerArn string, ruleArns []string) ( arnSet[a] = true } - result := make([]Rule, 0, len(b.rules)) + result := make([]Rule, 0, b.rules.Len()) - for _, r := range b.rules { + for _, r := range b.rules.All() { if listenerArn != "" && r.ListenerArn != listenerArn { continue } @@ -2413,7 +2423,7 @@ func (b *InMemoryBackend) DeleteRule(ruleArn string) error { b.mu.Lock("DeleteRule") defer b.mu.Unlock() - rule, ok := b.rules[ruleArn] + rule, ok := b.rules.Get(ruleArn) if !ok { return ErrRuleNotFound } @@ -2426,7 +2436,7 @@ func (b *InMemoryBackend) DeleteRule(ruleArn string) error { } rule.Tags.Close() - delete(b.rules, ruleArn) + b.rules.Delete(ruleArn) return nil } @@ -2440,7 +2450,7 @@ func (b *InMemoryBackend) ModifyRule( b.mu.Lock("ModifyRule") defer b.mu.Unlock() - rule, ok := b.rules[ruleArn] + rule, ok := b.rules.Get(ruleArn) if !ok { return nil, ErrRuleNotFound } @@ -2461,23 +2471,23 @@ func (b *InMemoryBackend) ModifyRule( // findTagsLocked returns the *tags.Tags for the given resource ARN. // Caller must hold b.mu (read or write). func (b *InMemoryBackend) findTagsLocked(resArn string) *tags.Tags { - if lb, ok := b.loadBalancers[resArn]; ok { + if lb, ok := b.loadBalancers.Get(resArn); ok { return lb.Tags } - if tg, ok := b.targetGroups[resArn]; ok { + if tg, ok := b.targetGroups.Get(resArn); ok { return tg.Tags } - if l, ok := b.listeners[resArn]; ok { + if l, ok := b.listeners.Get(resArn); ok { return l.Tags } - if r, ok := b.rules[resArn]; ok { + if r, ok := b.rules.Get(resArn); ok { return r.Tags } - if ts, ok := b.trustStores[resArn]; ok { + if ts, ok := b.trustStores.Get(resArn); ok { return ts.Tags } @@ -2604,7 +2614,7 @@ func (b *InMemoryBackend) CreateTrustStore(name string, kvs []tags.KV) (*TrustSt return nil, fmt.Errorf("%w: Name is required", ErrInvalidParameter) } - for _, ts := range b.trustStores { + for _, ts := range b.trustStores.All() { if ts.Name == name { return nil, ErrTrustStoreAlreadyExists } @@ -2626,7 +2636,7 @@ func (b *InMemoryBackend) CreateTrustStore(name string, kvs []tags.KV) (*TrustSt Tags: t, } - b.trustStores[tsArn] = ts + b.trustStores.Put(ts) cp := *ts @@ -2657,9 +2667,9 @@ func (b *InMemoryBackend) DescribeTrustStores(arns []string, names []string) ([] } } - result := make([]TrustStore, 0, len(b.trustStores)) + result := make([]TrustStore, 0, b.trustStores.Len()) - for _, ts := range b.trustStores { + for _, ts := range b.trustStores.All() { if filterArns { if _, ok := wantArn[ts.TrustStoreArn]; !ok { continue @@ -2687,13 +2697,13 @@ func (b *InMemoryBackend) DeleteTrustStore(trustStoreArn string) error { b.mu.Lock("DeleteTrustStore") defer b.mu.Unlock() - ts, ok := b.trustStores[trustStoreArn] + ts, ok := b.trustStores.Get(trustStoreArn) if !ok { return ErrTrustStoreNotFound } ts.Tags.Close() - delete(b.trustStores, trustStoreArn) + b.trustStores.Delete(trustStoreArn) return nil } @@ -2706,7 +2716,7 @@ func (b *InMemoryBackend) AddTrustStoreRevocations( b.mu.Lock("AddTrustStoreRevocations") defer b.mu.Unlock() - ts, ok := b.trustStores[trustStoreArn] + ts, ok := b.trustStores.Get(trustStoreArn) if !ok { return ErrTrustStoreNotFound } @@ -2721,13 +2731,13 @@ func (b *InMemoryBackend) DescribeTrustStoreAssociations(trustStoreArn string) ( b.mu.RLock("DescribeTrustStoreAssociations") defer b.mu.RUnlock() - if _, ok := b.trustStores[trustStoreArn]; !ok { + if _, ok := b.trustStores.Get(trustStoreArn); !ok { return nil, ErrTrustStoreNotFound } var result []string - for _, l := range b.listeners { + for _, l := range b.listeners.All() { if l.MutualAuthentication != nil && l.MutualAuthentication.TrustStoreArn == trustStoreArn { result = append(result, l.ListenerArn) } @@ -2747,11 +2757,11 @@ func (b *InMemoryBackend) DeleteSharedTrustStoreAssociation(trustStoreArn, resou b.mu.Lock("DeleteSharedTrustStoreAssociation") defer b.mu.Unlock() - if _, ok := b.trustStores[trustStoreArn]; !ok { + if _, ok := b.trustStores.Get(trustStoreArn); !ok { return ErrTrustStoreNotFound } - listener, ok := b.listeners[resourceArn] + listener, ok := b.listeners.Get(resourceArn) if !ok || listener.MutualAuthentication == nil || listener.MutualAuthentication.TrustStoreArn != trustStoreArn { return ErrTrustStoreAssociationNotFound @@ -2771,7 +2781,7 @@ func (b *InMemoryBackend) ModifyCapacityReservation( b.mu.Lock("ModifyCapacityReservation") defer b.mu.Unlock() - lb, ok := b.loadBalancers[lbArn] + lb, ok := b.loadBalancers.Get(lbArn) if !ok { return nil, ErrLoadBalancerNotFound } @@ -2812,7 +2822,7 @@ func (b *InMemoryBackend) DescribeCapacityReservation(lbArn string) (*CapacityRe b.mu.RLock("DescribeCapacityReservation") defer b.mu.RUnlock() - lb, ok := b.loadBalancers[lbArn] + lb, ok := b.loadBalancers.Get(lbArn) if !ok { return nil, ErrLoadBalancerNotFound } @@ -2835,7 +2845,7 @@ func (b *InMemoryBackend) ModifyIPPools( b.mu.Lock("ModifyIPPools") defer b.mu.Unlock() - lb, ok := b.loadBalancers[lbArn] + lb, ok := b.loadBalancers.Get(lbArn) if !ok { return nil, ErrLoadBalancerNotFound } @@ -2885,7 +2895,7 @@ func (b *InMemoryBackend) AddListenerCertificates(listenerArn string, certs []Ce b.mu.Lock("AddListenerCertificates") defer b.mu.Unlock() - listener, ok := b.listeners[listenerArn] + listener, ok := b.listeners.Get(listenerArn) if !ok { return ErrListenerNotFound } @@ -2910,7 +2920,7 @@ func (b *InMemoryBackend) DescribeListenerCertificates(listenerArn string) ([]Ce b.mu.RLock("DescribeListenerCertificates") defer b.mu.RUnlock() - listener, ok := b.listeners[listenerArn] + listener, ok := b.listeners.Get(listenerArn) if !ok { return nil, ErrListenerNotFound } @@ -2926,7 +2936,7 @@ func (b *InMemoryBackend) RemoveListenerCertificates(listenerArn string, certArn b.mu.Lock("RemoveListenerCertificates") defer b.mu.Unlock() - listener, ok := b.listeners[listenerArn] + listener, ok := b.listeners.Get(listenerArn) if !ok { return ErrListenerNotFound } @@ -2960,7 +2970,7 @@ func (b *InMemoryBackend) ModifyTrustStore(trustStoreArn, name string) (*TrustSt b.mu.Lock("ModifyTrustStore") defer b.mu.Unlock() - ts, ok := b.trustStores[trustStoreArn] + ts, ok := b.trustStores.Get(trustStoreArn) if !ok { return nil, ErrTrustStoreNotFound } @@ -2982,7 +2992,7 @@ func (b *InMemoryBackend) RemoveTrustStoreRevocations( b.mu.Lock("RemoveTrustStoreRevocations") defer b.mu.Unlock() - ts, ok := b.trustStores[trustStoreArn] + ts, ok := b.trustStores.Get(trustStoreArn) if !ok { return ErrTrustStoreNotFound } @@ -3011,7 +3021,7 @@ func (b *InMemoryBackend) DescribeTrustStoreRevocations( b.mu.RLock("DescribeTrustStoreRevocations") defer b.mu.RUnlock() - ts, ok := b.trustStores[trustStoreArn] + ts, ok := b.trustStores.Get(trustStoreArn) if !ok { return nil, ErrTrustStoreNotFound } @@ -3035,10 +3045,9 @@ func (b *InMemoryBackend) checkRulePriorityCollisions( } for _, p := range priorities { - listenerArn := b.rules[p.RuleArn].ListenerArn - for _, existing := range b.rules { - if existing.ListenerArn != listenerArn || batchArns[existing.RuleArn] || - existing.IsDefault { + rule, _ := b.rules.Get(p.RuleArn) + for _, existing := range b.rulesByListener.Get(rule.ListenerArn) { + if batchArns[existing.RuleArn] || existing.IsDefault { continue } @@ -3077,7 +3086,7 @@ func (b *InMemoryBackend) SetRulePriorities(priorities []RulePriority) ([]Rule, // Validate all rules exist and none is a default rule (AWS does not allow reordering defaults). batchArns := make(map[string]bool, len(priorities)) for _, p := range priorities { - r, ok := b.rules[p.RuleArn] + r, ok := b.rules.Get(p.RuleArn) if !ok { return nil, ErrRuleNotFound } @@ -3099,7 +3108,7 @@ func (b *InMemoryBackend) SetRulePriorities(priorities []RulePriority) ([]Rule, result := make([]Rule, 0, len(priorities)) for _, p := range priorities { - r := b.rules[p.RuleArn] + r, _ := b.rules.Get(p.RuleArn) r.Priority = p.Priority result = append(result, *r) } @@ -3112,7 +3121,7 @@ func (b *InMemoryBackend) ModifyTargetGroup(input ModifyTargetGroupInput) (*Targ b.mu.Lock("ModifyTargetGroup") defer b.mu.Unlock() - tg, ok := b.targetGroups[input.TargetGroupArn] + tg, ok := b.targetGroups.Get(input.TargetGroupArn) if !ok { return nil, ErrTargetGroupNotFound } @@ -3170,7 +3179,7 @@ func (b *InMemoryBackend) ModifyTargetGroupAttributes( b.mu.Lock("ModifyTargetGroupAttributes") defer b.mu.Unlock() - tg, ok := b.targetGroups[tgArn] + tg, ok := b.targetGroups.Get(tgArn) if !ok { return nil, ErrTargetGroupNotFound } @@ -3191,7 +3200,7 @@ func (b *InMemoryBackend) DescribeTargetGroupAttributes(tgArn string) (map[strin b.mu.RLock("DescribeTargetGroupAttributes") defer b.mu.RUnlock() - tg, ok := b.targetGroups[tgArn] + tg, ok := b.targetGroups.Get(tgArn) if !ok { return nil, ErrTargetGroupNotFound } @@ -3210,7 +3219,7 @@ func (b *InMemoryBackend) ModifyListenerAttributes( b.mu.Lock("ModifyListenerAttributes") defer b.mu.Unlock() - l, ok := b.listeners[listenerArn] + l, ok := b.listeners.Get(listenerArn) if !ok { return nil, ErrListenerNotFound } @@ -3233,7 +3242,7 @@ func (b *InMemoryBackend) DescribeListenerAttributes( b.mu.RLock("DescribeListenerAttributes") defer b.mu.RUnlock() - l, ok := b.listeners[listenerArn] + l, ok := b.listeners.Get(listenerArn) if !ok { return nil, ErrListenerNotFound } diff --git a/services/elbv2/handler.go b/services/elbv2/handler.go index 861897bc3..b27326e73 100644 --- a/services/elbv2/handler.go +++ b/services/elbv2/handler.go @@ -792,6 +792,43 @@ func (h *Handler) handleDescribeTargetGroupAttributes(vals url.Values) (any, err // --- target handlers --- +// resolveTargetGroupPort looks up a target group's configured port, used to +// default omitted Targets.member.N.Port values. AWS treats Port as optional +// for instance/ip/alb target types and defaults it to the target group's port; +// Lambda target groups have no port (tgPort is 0 and no defaulting occurs). +func (h *Handler) resolveTargetGroupPort(tgArn string) (int32, error) { + tgs, err := h.Backend.DescribeTargetGroups([]string{tgArn}, nil, "") + if err != nil { + return 0, err + } + + if len(tgs) == 0 { + return 0, ErrTargetGroupNotFound + } + + return tgs[0].Port, nil +} + +// defaultTargetPorts fills in omitted (zero-value) target ports with the +// target group's port. A zero tgPort (e.g. Lambda target groups) is a no-op. +func defaultTargetPorts(targets []Target, tgPort int32) []Target { + if tgPort == 0 { + return targets + } + + out := make([]Target, len(targets)) + + for i, t := range targets { + if t.Port == 0 { + t.Port = tgPort + } + + out[i] = t + } + + return out +} + func (h *Handler) handleRegisterTargets(vals url.Values) (any, error) { tgArn := vals.Get("TargetGroupArn") if tgArn == "" { @@ -800,10 +837,17 @@ func (h *Handler) handleRegisterTargets(vals url.Values) (any, error) { targets := parseTargets(vals, "Targets.member") - if err := h.Backend.RegisterTargets(tgArn, targets); err != nil { + tgPort, err := h.resolveTargetGroupPort(tgArn) + if err != nil { return nil, err } + targets = defaultTargetPorts(targets, tgPort) + + if regErr := h.Backend.RegisterTargets(tgArn, targets); regErr != nil { + return nil, regErr + } + return ®isterTargetsResponse{ Xmlns: elbv2XMLNS, ResponseMetadata: xmlResponseMetadata{RequestID: "elbv2-register-targets"}, @@ -818,10 +862,17 @@ func (h *Handler) handleDeregisterTargets(vals url.Values) (any, error) { targets := parseTargets(vals, "Targets.member") - if err := h.Backend.DeregisterTargets(tgArn, targets); err != nil { + tgPort, err := h.resolveTargetGroupPort(tgArn) + if err != nil { return nil, err } + targets = defaultTargetPorts(targets, tgPort) + + if deregErr := h.Backend.DeregisterTargets(tgArn, targets); deregErr != nil { + return nil, deregErr + } + return &deregisterTargetsResponse{ Xmlns: elbv2XMLNS, ResponseMetadata: xmlResponseMetadata{RequestID: "elbv2-deregister-targets"}, @@ -844,6 +895,10 @@ func (h *Handler) handleDescribeTargetHealth(vals url.Values) (any, error) { // reason "Target.NotRegistered", matching real AWS behaviour. requestedTargets := parseTargets(vals, "Targets.member") if len(requestedTargets) > 0 { + if tgPort, pErr := h.resolveTargetGroupPort(tgArn); pErr == nil { + requestedTargets = defaultTargetPorts(requestedTargets, tgPort) + } + registeredMap := make(map[string]TargetHealthDescription, len(targets)) for _, t := range targets { registeredMap[t.Target.ID+":"+strconv.Itoa(int(t.Target.Port))] = t @@ -954,7 +1009,7 @@ func (h *Handler) handleCreateListener(vals url.Values) (any, error) { Tags: tagKVs, Certificates: certs, SSLPolicy: vals.Get("SslPolicy"), - AlpnPolicy: vals.Get("AlpnPolicy.member.1"), + AlpnPolicy: parseMembers(vals, "AlpnPolicy.member"), MutualAuthentication: mutualAuth, }) if createErr != nil { @@ -1077,7 +1132,7 @@ func (h *Handler) handleModifyListener(vals url.Values) (any, error) { DefaultActions: parseActions(vals, "DefaultActions.member"), Certificates: parseCerts(vals), SSLPolicy: vals.Get("SslPolicy"), - AlpnPolicy: vals.Get("AlpnPolicy.member.1"), + AlpnPolicy: parseMembers(vals, "AlpnPolicy.member"), MutualAuthentication: mutualAuth, }) if err != nil { @@ -1433,8 +1488,21 @@ func (h *Handler) handleAddTrustStoreRevocations(vals url.Values) (any, error) { return nil, err } + members := make([]xmlRevocationContent, 0, len(revocations)) + for _, r := range revocations { + members = append(members, xmlRevocationContent{ + RevocationID: r.RevocationID, + RevocationType: r.RevocationType, + NumberOfRevokedEntries: r.NumberOfRevokedEntries, + TrustStoreArn: tsArn, + }) + } + return &addTrustStoreRevocationsResponse{ - Xmlns: elbv2XMLNS, + Xmlns: elbv2XMLNS, + Result: addTrustStoreRevocationsResult{ + TrustStoreRevocations: xmlRevocationContentList{Members: members}, + }, ResponseMetadata: xmlResponseMetadata{RequestID: "elbv2-add-ts-revocations"}, }, nil } @@ -1889,13 +1957,18 @@ func (h *Handler) handleDescribeTrustStoreRevocations(vals url.Values) (any, err members := make([]xmlRevocationContent, 0, len(revocations)) for _, r := range revocations { - members = append(members, xmlRevocationContent(r)) + members = append(members, xmlRevocationContent{ + RevocationID: r.RevocationID, + RevocationType: r.RevocationType, + NumberOfRevokedEntries: r.NumberOfRevokedEntries, + TrustStoreArn: tsArn, + }) } return &describeTrustStoreRevocationsResponse{ Xmlns: elbv2XMLNS, Result: describeTrustStoreRevocationsResult{ - RevocationContents: xmlRevocationContentList{Members: members}, + TrustStoreRevocations: xmlRevocationContentList{Members: members}, }, ResponseMetadata: xmlResponseMetadata{RequestID: "elbv2-describe-ts-revocations"}, }, nil @@ -2122,19 +2195,26 @@ func elbv2ErrorCode(opErr error) (string, int) { httpCode int } + // NOTE: real AWS ELBv2 uses the classic AWS "Query" protocol (like EC2), where + // EVERY client error — including NotFound and AlreadyExists conditions — is + // returned with HTTP status 400; the client SDK dispatches on the XML + // element, not the HTTP status. Verified against the elasticloadbalancingv2 + // API model (api-2.json), which sets httpStatusCode=400 for every exception + // shape in this service. Using 404/409 here (as a REST-JSON service would) + // is wire-inaccurate for a query-protocol service. mappings := []errorMapping{ - {ErrLoadBalancerNotFound, "LoadBalancerNotFound", http.StatusNotFound}, - {ErrTargetGroupNotFound, "TargetGroupNotFound", http.StatusNotFound}, - {ErrListenerNotFound, "ListenerNotFound", http.StatusNotFound}, - {ErrRuleNotFound, "RuleNotFound", http.StatusNotFound}, - {ErrTrustStoreNotFound, "TrustStoreNotFound", http.StatusNotFound}, - {ErrResourcePolicyNotFound, "ResourceNotFound", http.StatusNotFound}, - {ErrTrustStoreAssociationNotFound, "AssociationNotFound", http.StatusNotFound}, - {ErrLoadBalancerAlreadyExists, "DuplicateLoadBalancerName", http.StatusConflict}, - {ErrTargetGroupAlreadyExists, "DuplicateTargetGroupName", http.StatusConflict}, - {ErrTrustStoreAlreadyExists, "DuplicateTrustStoreName", http.StatusConflict}, - {ErrDuplicateListener, "DuplicateListener", http.StatusConflict}, - {ErrDuplicateRulePriority, "DuplicatePriority", http.StatusBadRequest}, + {ErrLoadBalancerNotFound, "LoadBalancerNotFound", http.StatusBadRequest}, + {ErrTargetGroupNotFound, "TargetGroupNotFound", http.StatusBadRequest}, + {ErrListenerNotFound, "ListenerNotFound", http.StatusBadRequest}, + {ErrRuleNotFound, "RuleNotFound", http.StatusBadRequest}, + {ErrTrustStoreNotFound, "TrustStoreNotFound", http.StatusBadRequest}, + {ErrResourcePolicyNotFound, "ResourceNotFound", http.StatusBadRequest}, + {ErrTrustStoreAssociationNotFound, "AssociationNotFound", http.StatusBadRequest}, + {ErrLoadBalancerAlreadyExists, "DuplicateLoadBalancerName", http.StatusBadRequest}, + {ErrTargetGroupAlreadyExists, "DuplicateTargetGroupName", http.StatusBadRequest}, + {ErrTrustStoreAlreadyExists, "DuplicateTrustStoreName", http.StatusBadRequest}, + {ErrDuplicateListener, "DuplicateListener", http.StatusBadRequest}, + {ErrDuplicateRulePriority, "PriorityInUse", http.StatusBadRequest}, {ErrTargetGroupInUse, "ResourceInUse", http.StatusBadRequest}, {ErrOperationNotPermitted, "OperationNotPermitted", http.StatusBadRequest}, {ErrInvalidConfigurationRequest, "InvalidConfigurationRequest", http.StatusBadRequest}, @@ -2539,8 +2619,18 @@ func parseConditionAt(vals url.Values, prefix string, i int, result *[]Condition switch field { case "host-header": cond.Values = parseMembers(vals, fmt.Sprintf("%s.%d.HostHeaderConfig.Values.member", prefix, i)) + if len(cond.Values) == 0 { + // Legacy top-level Values field (deprecated by AWS in favor of + // HostHeaderConfig, but still accepted on the wire). + cond.Values = parseMembers(vals, fmt.Sprintf("%s.%d.Values.member", prefix, i)) + } case "path-pattern": cond.Values = parseMembers(vals, fmt.Sprintf("%s.%d.PathPatternConfig.Values.member", prefix, i)) + if len(cond.Values) == 0 { + // Legacy top-level Values field (deprecated by AWS in favor of + // PathPatternConfig, but still accepted on the wire). + cond.Values = parseMembers(vals, fmt.Sprintf("%s.%d.Values.member", prefix, i)) + } case "http-request-method": methods := parseMembers(vals, fmt.Sprintf("%s.%d.HttpRequestMethodConfig.Values.member", prefix, i)) for _, m := range methods { @@ -2757,7 +2847,6 @@ func toXMLListener(l *Listener) xmlListener { Port: l.Port, DefaultActions: xmlActionList{Members: actions}, SslPolicy: l.SSLPolicy, - AlpnPolicy: l.AlpnPolicy, } if l.MutualAuthentication != nil { @@ -2777,6 +2866,15 @@ func toXMLListener(l *Listener) xmlListener { xl.Certificates = &xmlListenerCertificateList{Members: certs} } + if len(l.AlpnPolicy) > 0 { + members := make([]xmlStringValue, 0, len(l.AlpnPolicy)) + for _, p := range l.AlpnPolicy { + members = append(members, xmlStringValue{Value: p}) + } + + xl.AlpnPolicy = &xmlStringList{Members: members} + } + return xl } @@ -3215,11 +3313,11 @@ type xmlActionList struct { type xmlListener struct { MutualAuthentication *xmlMutualAuthentication `xml:"MutualAuthentication,omitempty"` Certificates *xmlListenerCertificateList `xml:"Certificates,omitempty"` + AlpnPolicy *xmlStringList `xml:"AlpnPolicy,omitempty"` ListenerArn string `xml:"ListenerArn"` LoadBalancerArn string `xml:"LoadBalancerArn"` Protocol string `xml:"Protocol"` SslPolicy string `xml:"SslPolicy,omitempty"` - AlpnPolicy string `xml:"AlpnPolicy,omitempty"` DefaultActions xmlActionList `xml:"DefaultActions"` Port int32 `xml:"Port"` } @@ -3489,9 +3587,10 @@ type deleteSharedTrustStoreAssociationResponse struct { } type addTrustStoreRevocationsResponse struct { - XMLName xml.Name `xml:"AddTrustStoreRevocationsResponse"` - Xmlns string `xml:"xmlns,attr"` - ResponseMetadata xmlResponseMetadata `xml:"ResponseMetadata"` + XMLName xml.Name `xml:"AddTrustStoreRevocationsResponse"` + Xmlns string `xml:"xmlns,attr"` + ResponseMetadata xmlResponseMetadata `xml:"ResponseMetadata"` + Result addTrustStoreRevocationsResult `xml:"AddTrustStoreRevocationsResult"` } type xmlTrustStoreAssociation struct { @@ -3664,9 +3763,14 @@ type modifyTrustStoreResponse struct { Result modifyTrustStoreResult `xml:"ModifyTrustStoreResult"` } +// xmlRevocationContent mirrors both aws-sdk-go-v2 types.TrustStoreRevocation (used in +// AddTrustStoreRevocationsResult) and types.DescribeTrustStoreRevocation (used in +// DescribeTrustStoreRevocationsResult) — both have the same RevocationId/RevocationType/ +// NumberOfRevokedEntries/TrustStoreArn fields on the wire. type xmlRevocationContent struct { RevocationID string `xml:"RevocationId"` RevocationType string `xml:"RevocationType,omitempty"` + TrustStoreArn string `xml:"TrustStoreArn,omitempty"` NumberOfRevokedEntries int64 `xml:"NumberOfRevokedEntries,omitempty"` } @@ -3674,8 +3778,11 @@ type xmlRevocationContentList struct { Members []xmlRevocationContent `xml:"member"` } +// describeTrustStoreRevocationsResult's list field is named TrustStoreRevocations on the +// wire (verified against aws-sdk-go-v2's deserializer) — NOT "RevocationContents", which +// is only the request-side field name for AddTrustStoreRevocations. type describeTrustStoreRevocationsResult struct { - RevocationContents xmlRevocationContentList `xml:"RevocationContents"` + TrustStoreRevocations xmlRevocationContentList `xml:"TrustStoreRevocations"` } type describeTrustStoreRevocationsResponse struct { @@ -3685,6 +3792,12 @@ type describeTrustStoreRevocationsResponse struct { Result describeTrustStoreRevocationsResult `xml:"DescribeTrustStoreRevocationsResult"` } +// addTrustStoreRevocationsResult reports the revocation files that were added, matching +// AWS's AddTrustStoreRevocationsOutput.TrustStoreRevocations. +type addTrustStoreRevocationsResult struct { + TrustStoreRevocations xmlRevocationContentList `xml:"TrustStoreRevocations"` +} + type removeTrustStoreRevocationsResponse struct { XMLName xml.Name `xml:"RemoveTrustStoreRevocationsResponse"` Xmlns string `xml:"xmlns,attr"` diff --git a/services/elbv2/handler_accuracy_batch1_test.go b/services/elbv2/handler_accuracy_batch1_test.go index 7f685fe9f..a690c7c72 100644 --- a/services/elbv2/handler_accuracy_batch1_test.go +++ b/services/elbv2/handler_accuracy_batch1_test.go @@ -301,7 +301,7 @@ func TestBatch1_CreateLB_DuplicateName(t *testing.T) { "Version": {"2015-12-01"}, "Name": {"dup-lb-batch1"}, }) - assert.Equal(t, http.StatusConflict, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) } func TestBatch1_DescribeLBs_FilterByArn(t *testing.T) { @@ -371,7 +371,7 @@ func TestBatch1_DescribeLBs_ArnNotFound(t *testing.T) { "arn:aws:elasticloadbalancing:us-east-1:000000000000:loadbalancer/app/not-exist/000", }, }) - assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) } func TestBatch1_DeleteLB_Success(t *testing.T) { @@ -393,7 +393,7 @@ func TestBatch1_DeleteLB_Success(t *testing.T) { "Version": {"2015-12-01"}, "LoadBalancerArns.member.1": {lbArn}, }) - assert.Equal(t, http.StatusNotFound, rec2.Code) + assert.Equal(t, http.StatusBadRequest, rec2.Code) } func TestBatch1_DeleteLB_NotFound(t *testing.T) { @@ -405,7 +405,7 @@ func TestBatch1_DeleteLB_NotFound(t *testing.T) { "Version": {"2015-12-01"}, "LoadBalancerArn": {"arn:aws:elasticloadbalancing:us-east-1:000000000000:loadbalancer/app/ghost/000"}, }) - assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) } // ---- LB Attributes ---- @@ -1242,7 +1242,7 @@ func TestBatch1_CreateListener_DuplicatePortRejected(t *testing.T) { "DefaultActions.member.1.Type": {"forward"}, "DefaultActions.member.1.TargetGroupArn": {tgArn}, }) - assert.Equal(t, http.StatusConflict, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) } func TestBatch1_DeleteListener(t *testing.T) { @@ -2046,13 +2046,13 @@ func TestBatch1_GetResourcePolicy(t *testing.T) { h := newBatch1Handler() lbArn := b1CreateLB(t, h, "grp-lb") - // No resource policy is set, so AWS returns ResourceNotFound (404). + // No resource policy is set, so AWS returns ResourceNotFound (HTTP 400, AWS query-protocol status). rec := doELBv2(t, h, url.Values{ "Action": {"GetResourcePolicy"}, "Version": {"2015-12-01"}, "ResourceArn": {lbArn}, }) - assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) // After a policy is stored on the backend, GetResourcePolicy returns it. be, ok := h.Backend.(*elbv2.InMemoryBackend) @@ -2489,14 +2489,14 @@ func TestBatch1_DeleteSharedTrustStoreAssociation(t *testing.T) { }) assert.Equal(t, http.StatusOK, rec.Code) - // Deleting again returns AssociationNotFound (404). + // Deleting again returns AssociationNotFound (HTTP 400, AWS query-protocol status). rec2 := doELBv2(t, h, url.Values{ "Action": {"DeleteSharedTrustStoreAssociation"}, "Version": {"2015-12-01"}, "TrustStoreArn": {tsArn}, "ResourceArn": {listenerArn}, }) - assert.Equal(t, http.StatusNotFound, rec2.Code) + assert.Equal(t, http.StatusBadRequest, rec2.Code) } // ---- RemoveTrustStoreRevocations ---- diff --git a/services/elbv2/handler_accuracy_batch2_test.go b/services/elbv2/handler_accuracy_batch2_test.go index cd5dbb4eb..f2e355bb8 100644 --- a/services/elbv2/handler_accuracy_batch2_test.go +++ b/services/elbv2/handler_accuracy_batch2_test.go @@ -160,7 +160,7 @@ func TestBatch2_AddTags_UpdateExistingKeyDoesNotCount(t *testing.T) { // ─── Issue: DescribeTargetGroups returns TargetGroupNotFound for unknown ARN ── // TestBatch2_DescribeTargetGroups_UnknownArnReturnsNotFound verifies that querying -// a non-existent target group ARN returns TargetGroupNotFound (404), matching AWS. +// a non-existent target group ARN returns TargetGroupNotFound (HTTP 400, AWS query-protocol status), matching AWS. func TestBatch2_DescribeTargetGroups_UnknownArnReturnsNotFound(t *testing.T) { t.Parallel() @@ -204,7 +204,7 @@ func TestBatch2_DescribeTargetGroups_UnknownArnReturnsNotFound(t *testing.T) { } rec := doELBv2(t, h, vals) - assert.Equal(t, http.StatusNotFound, rec.Code, rec.Body.String()) + assert.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String()) }) } } @@ -242,7 +242,7 @@ func TestBatch2_DescribeTargetGroups_AllKnownArnsSucceeds(t *testing.T) { // ─── Issue: DescribeRules returns RuleNotFound for unknown ARN ──────────────── // TestBatch2_DescribeRules_UnknownArnReturnsNotFound verifies that querying a -// non-existent rule ARN (without listenerArn) returns RuleNotFound (404). +// non-existent rule ARN (without listenerArn) returns RuleNotFound (HTTP 400, AWS query-protocol status). func TestBatch2_DescribeRules_UnknownArnReturnsNotFound(t *testing.T) { t.Parallel() @@ -278,7 +278,7 @@ func TestBatch2_DescribeRules_UnknownArnReturnsNotFound(t *testing.T) { } rec := doELBv2(t, h, vals) - assert.Equal(t, http.StatusNotFound, rec.Code, rec.Body.String()) + assert.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String()) }) } } diff --git a/services/elbv2/handler_test.go b/services/elbv2/handler_test.go index e35cd77f1..9d1fdb7c1 100644 --- a/services/elbv2/handler_test.go +++ b/services/elbv2/handler_test.go @@ -142,7 +142,7 @@ func TestCreateLoadBalancer(t *testing.T) { "Version": {"2015-12-01"}, "Name": {"dup-alb"}, }, - wantStatus: http.StatusConflict, + wantStatus: http.StatusBadRequest, }, { name: "missing_name_returns_bad_request", @@ -257,13 +257,13 @@ func TestDeleteLoadBalancerByARN(t *testing.T) { }) assert.Equal(t, http.StatusOK, rec.Code) - // Querying a specific ARN that no longer exists should return 404 (AWS behavior). + // Querying a specific ARN that no longer exists should return 400 (AWS query-protocol error status). rec2 := doELBv2(t, h, url.Values{ "Action": {"DescribeLoadBalancers"}, "Version": {"2015-12-01"}, "LoadBalancerArns.member.1": {lbArn}, }) - assert.Equal(t, http.StatusNotFound, rec2.Code) + assert.Equal(t, http.StatusBadRequest, rec2.Code) } // TestDescribeLoadBalancers tests describe operations. @@ -382,7 +382,7 @@ func TestCreateTargetGroup(t *testing.T) { "Protocol": {"HTTP"}, "Port": {"80"}, }, - wantStatus: http.StatusConflict, + wantStatus: http.StatusBadRequest, }, { name: "missing_name_returns_bad_request", @@ -514,7 +514,7 @@ func TestDeleteTargetGroup(t *testing.T) { "Version": {"2015-12-01"}, "TargetGroupArns.member.1": {tgArn}, }) - require.Equal(t, http.StatusNotFound, rec2.Code) + require.Equal(t, http.StatusBadRequest, rec2.Code) } // TestRegisterAndDeregisterTargets tests target registration. @@ -623,7 +623,7 @@ func TestCreateListener(t *testing.T) { "DefaultActions.member.1.FixedResponseConfig.StatusCode": {"200"}, } }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadRequest, }, } @@ -931,7 +931,7 @@ func TestDeleteListener(t *testing.T) { "ListenerArn": {"arn:aws:elasticloadbalancing:us-east-1:123456789012:listener/app/no/0/no"}, } }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadRequest, }, } @@ -1029,7 +1029,7 @@ func TestModifyListener(t *testing.T) { "Version": {"2015-12-01"}, "ListenerArn": {"arn:aws:elasticloadbalancing:us-east-1:123456789012:listener/app/no/0/no"}, }) - assert.Equal(t, http.StatusNotFound, rec2.Code) + assert.Equal(t, http.StatusBadRequest, rec2.Code) // Test missing arn rec3 := doELBv2(t, h, url.Values{ @@ -1113,7 +1113,7 @@ func TestDeleteRule(t *testing.T) { "Version": {"2015-12-01"}, "RuleArn": {"arn:aws:no-such-rule"}, }) - assert.Equal(t, http.StatusNotFound, rec3.Code) + assert.Equal(t, http.StatusBadRequest, rec3.Code) } // TestDescribeRules tests rule describe operations. @@ -1251,7 +1251,7 @@ func TestModifyRule(t *testing.T) { "Version": {"2015-12-01"}, "RuleArn": {"arn:aws:no-such-rule"}, }) - assert.Equal(t, http.StatusNotFound, rec2.Code) + assert.Equal(t, http.StatusBadRequest, rec2.Code) // Missing arn rec3 := doELBv2(t, h, url.Values{ @@ -1353,7 +1353,7 @@ func TestDeleteTargetGroupNotFound(t *testing.T) { "Version": {"2015-12-01"}, "TargetGroupArn": {"arn:aws:elasticloadbalancing:us-east-1:123456789012:targetgroup/no-such/0"}, }) - assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) } // TestRegisterTargetsMissingARN tests missing ARN for register targets. @@ -1474,7 +1474,7 @@ func TestCreateRuleListenerNotFound(t *testing.T) { "Actions.member.1.Type": {"fixed-response"}, "Actions.member.1.FixedResponseConfig.StatusCode": {"200"}, }) - assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) } // TestDescribeTagsForTargetGroupAndListener tests describe tags for TG and listener ARNs. @@ -1717,7 +1717,7 @@ func TestDeleteLoadBalancerNotFound(t *testing.T) { "Version": {"2015-12-01"}, "LoadBalancerArn": {"arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/no-such/0"}, }) - assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) } // TestModifyLoadBalancerAttributesMissing tests missing ARN. @@ -1782,7 +1782,7 @@ func TestModifyTargetGroupAttributes(t *testing.T) { "Version": {"2015-12-01"}, "TargetGroupArn": {"arn:aws:elasticloadbalancing:us-east-1:123456789012:targetgroup/no-such/0"}, }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadRequest, }, } @@ -1904,7 +1904,7 @@ func TestModifyListenerAttributes(t *testing.T) { "Version": {"2015-12-01"}, "ListenerArn": {"arn:aws:elasticloadbalancing:us-east-1:123456789012:listener/app/no-such/0/80"}, }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadRequest, }, } @@ -1955,7 +1955,7 @@ func TestDescribeListenerAttributes(t *testing.T) { "Version": {"2015-12-01"}, "ListenerArn": {"arn:aws:elasticloadbalancing:us-east-1:123456789012:listener/app/no-such/0/80"}, }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadRequest, }, } @@ -2150,7 +2150,7 @@ func TestELBv2_TrustStoreLifecycle(t *testing.T) { }, } }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadRequest, }, { name: "delete_trust_store_missing_arn", @@ -2221,6 +2221,24 @@ func TestELBv2_TrustStoreFullLifecycle(t *testing.T) { }) assert.Equal(t, http.StatusOK, revRec.Code) + // AddTrustStoreRevocations must echo back the added revocation info in its + // AddTrustStoreRevocationsResult.TrustStoreRevocations list (AWS behaviour) — + // this used to be silently dropped (an empty response body). + var addResp struct { + Result struct { + TrustStoreRevocations struct { + Members []struct { + RevocationID string `xml:"RevocationId"` + TrustStoreArn string `xml:"TrustStoreArn"` + } `xml:"member"` + } `xml:"TrustStoreRevocations"` + } `xml:"AddTrustStoreRevocationsResult"` + } + require.NoError(t, xml.Unmarshal(revRec.Body.Bytes(), &addResp)) + require.Len(t, addResp.Result.TrustStoreRevocations.Members, 1) + assert.Equal(t, "s3://my-bucket/revocations.crl", addResp.Result.TrustStoreRevocations.Members[0].RevocationID) + assert.Equal(t, tsArn, addResp.Result.TrustStoreRevocations.Members[0].TrustStoreArn) + // Describe trust store associations (expect empty). assocRec := doELBv2(t, h, url.Values{ "Action": {"DescribeTrustStoreAssociations"}, @@ -2251,16 +2269,18 @@ func TestELBv2_TrustStoreFullLifecycle(t *testing.T) { var revDescResp struct { Result struct { - RevocationContents struct { + TrustStoreRevocations struct { Members []struct { - RevocationID string `xml:"RevocationId"` + RevocationID string `xml:"RevocationId"` + TrustStoreArn string `xml:"TrustStoreArn"` } `xml:"member"` - } `xml:"RevocationContents"` + } `xml:"TrustStoreRevocations"` } `xml:"DescribeTrustStoreRevocationsResult"` } require.NoError(t, xml.Unmarshal(revDescRec.Body.Bytes(), &revDescResp)) - require.Len(t, revDescResp.Result.RevocationContents.Members, 1) - assert.Equal(t, "s3://my-bucket/revocations.crl", revDescResp.Result.RevocationContents.Members[0].RevocationID) + require.Len(t, revDescResp.Result.TrustStoreRevocations.Members, 1) + assert.Equal(t, "s3://my-bucket/revocations.crl", revDescResp.Result.TrustStoreRevocations.Members[0].RevocationID) + assert.Equal(t, tsArn, revDescResp.Result.TrustStoreRevocations.Members[0].TrustStoreArn) // RemoveTrustStoreRevocations — remove the entry we added. revRmRec := doELBv2(t, h, url.Values{ @@ -2281,15 +2301,15 @@ func TestELBv2_TrustStoreFullLifecycle(t *testing.T) { var revDescResp2 struct { Result struct { - RevocationContents struct { + TrustStoreRevocations struct { Members []struct { RevocationID string `xml:"RevocationId"` } `xml:"member"` - } `xml:"RevocationContents"` + } `xml:"TrustStoreRevocations"` } `xml:"DescribeTrustStoreRevocationsResult"` } require.NoError(t, xml.Unmarshal(revDescRec2.Body.Bytes(), &revDescResp2)) - assert.Empty(t, revDescResp2.Result.RevocationContents.Members) + assert.Empty(t, revDescResp2.Result.TrustStoreRevocations.Members) // DescribeTrustStores — should return our trust store. descTSRec := doELBv2(t, h, url.Values{ @@ -2336,14 +2356,14 @@ func TestELBv2_TrustStoreFullLifecycle(t *testing.T) { assert.Equal(t, "my-ts-renamed", modTSResp.Result.TrustStores.Members[0].Name) // DeleteSharedTrustStoreAssociation with no existing association returns - // AssociationNotFound (404), matching AWS behavior. + // AssociationNotFound (HTTP 400, AWS query-protocol status), matching AWS behavior. delAssocRec := doELBv2(t, h, url.Values{ "Action": {"DeleteSharedTrustStoreAssociation"}, "Version": {"2015-12-01"}, "TrustStoreArn": {tsArn}, "ResourceArn": {"arn:aws:elasticloadbalancing:us-east-1:000000000000:listener/app/x/y/z"}, }) - assert.Equal(t, http.StatusNotFound, delAssocRec.Code) + assert.Equal(t, http.StatusBadRequest, delAssocRec.Code) // Delete trust store. delRec := doELBv2(t, h, url.Values{ @@ -2359,7 +2379,7 @@ func TestELBv2_TrustStoreFullLifecycle(t *testing.T) { "Version": {"2015-12-01"}, "TrustStoreArn": {tsArn}, }) - assert.Equal(t, http.StatusNotFound, delRec2.Code) + assert.Equal(t, http.StatusBadRequest, delRec2.Code) } // TestELBv2_ListenerCertificates validates error handling for add and describe listener certificate operations. @@ -2386,7 +2406,7 @@ func TestELBv2_ListenerCertificates(t *testing.T) { "Certificates.member.1.CertificateArn": {"arn:aws:acm:us-east-1:123:certificate/abc"}, } }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadRequest, }, { name: "describe_certificates_listener_not_found", @@ -2399,7 +2419,7 @@ func TestELBv2_ListenerCertificates(t *testing.T) { "ListenerArn": {"arn:aws:elasticloadbalancing:us-east-1:123:listener/nonexistent"}, } }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadRequest, }, { name: "add_certificates_missing_listener_arn", @@ -2443,7 +2463,7 @@ func TestELBv2_ListenerCertificates(t *testing.T) { "Certificates.member.1.CertificateArn": {"arn:aws:acm:us-east-1:123:certificate/abc"}, } }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadRequest, }, } @@ -2765,7 +2785,7 @@ func TestELBv2_SetRulePriorities(t *testing.T) { "RulePriorities.member.1.Priority": {"10"}, } }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadRequest, }, { name: "no_priorities_provided", @@ -2955,7 +2975,7 @@ func TestELBv2_ModifyTrustStore(t *testing.T) { "Name": {"new-name"}, } }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadRequest, }, { name: "missing_arn", @@ -3006,8 +3026,8 @@ func TestELBv2_StubOperations(t *testing.T) { "ResourceArn": {lbArn}, } }, - // No resource policy is set, so AWS returns ResourceNotFound (404). - wantStatus: http.StatusNotFound, + // No resource policy is set, so AWS returns ResourceNotFound (HTTP 400, AWS query-protocol status). + wantStatus: http.StatusBadRequest, }, { name: "get_resource_policy_missing_arn", @@ -3063,7 +3083,7 @@ func TestELBv2_StubOperations(t *testing.T) { "TrustStoreArn": {"arn:aws:elasticloadbalancing:us-east-1:123:truststore/nonexistent/abc"}, } }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadRequest, }, { name: "modify_capacity_reservation", @@ -3090,7 +3110,7 @@ func TestELBv2_StubOperations(t *testing.T) { "LoadBalancerArn": {"arn:aws:elasticloadbalancing:us-east-1:123:loadbalancer/app/nonexistent/abc"}, } }, - wantStatus: http.StatusNotFound, + wantStatus: http.StatusBadRequest, }, { name: "modify_ip_pools", @@ -3491,7 +3511,7 @@ func TestDuplicateListenerPortRejected(t *testing.T) { "DefaultActions.member.1.Type": {"forward"}, "DefaultActions.member.1.TargetGroupArn": {tgArn}, }) - assert.Equal(t, http.StatusConflict, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) } // TestMutualAuthenticationOnListener verifies mTLS mode is stored and returned. @@ -4432,12 +4452,12 @@ func TestPortValidationCreateListener(t *testing.T) { port string wantStatus int }{ - {"valid_port_80", "80", http.StatusNotFound}, // will fail on nonexistent LB, not port + {"valid_port_80", "80", http.StatusBadRequest}, // will fail on nonexistent LB, not port {"port_zero", "0", http.StatusBadRequest}, {"port_negative", "-1", http.StatusBadRequest}, {"port_65536", "65536", http.StatusBadRequest}, - {"port_65535_valid", "65535", http.StatusNotFound}, - {"port_1_valid", "1", http.StatusNotFound}, + {"port_65535_valid", "65535", http.StatusBadRequest}, + {"port_1_valid", "1", http.StatusBadRequest}, {"port_non_numeric", "abc", http.StatusBadRequest}, } @@ -4922,7 +4942,9 @@ func TestCachedDispatchTable(t *testing.T) { assert.Equal(t, http.StatusOK, rec.Code) } -// TestDuplicateRulePriorityErrorCode tests that ErrDuplicateRulePriority returns DuplicatePriority code. +// TestDuplicateRulePriorityErrorCode tests that ErrDuplicateRulePriority returns the real +// AWS PriorityInUse error code (not the fabricated "DuplicatePriority" code the mock +// used to return; verified against aws-sdk-go-v2/service/elasticloadbalancingv2/types). func TestDuplicateRulePriorityErrorCode(t *testing.T) { t.Parallel() @@ -4963,7 +4985,7 @@ func TestDuplicateRulePriorityErrorCode(t *testing.T) { } `xml:"Error"` } require.NoError(t, xml.Unmarshal(rec2.Body.Bytes(), &errResp)) - assert.Equal(t, "DuplicatePriority", errResp.Error.Code) + assert.Equal(t, "PriorityInUse", errResp.Error.Code) } // TestCreateListenerNoDefaultActions tests that missing DefaultActions returns 400. @@ -5371,7 +5393,7 @@ func TestSetSecurityGroupsPersist(t *testing.T) { assert.Equal(t, "sg-00000002", resp.Result.SecurityGroupIDs.Members[1].Value) } -// TestSetSecurityGroupsNotFound verifies that SetSecurityGroups returns 404 for missing LB. +// TestSetSecurityGroupsNotFound verifies that SetSecurityGroups returns 400 (LoadBalancerNotFound) for missing LB. func TestSetSecurityGroupsNotFound(t *testing.T) { t.Parallel() @@ -5384,7 +5406,7 @@ func TestSetSecurityGroupsNotFound(t *testing.T) { }, "SecurityGroups.member.1": {"sg-00000001"}, }) - assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) } // TestSetSubnetsPersist verifies that SetSubnets updates the LB availability zones. @@ -5688,7 +5710,7 @@ func TestModifyListenerDuplicatePortRejected(t *testing.T) { "ListenerArn": {listenerArn1}, "Port": {"443"}, }) - assert.Equal(t, http.StatusConflict, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) } // TestNLBDNSFormat verifies that NLB uses the correct DNS format (elb before region). @@ -5933,6 +5955,102 @@ func mustCreateNLB(t *testing.T, h *elbv2.Handler, name string) string { return resp.Result.LoadBalancers.Members[0].LoadBalancerArn } +// Test_AlpnPolicyWireShape verifies AlpnPolicy is wire-encoded as an AWS list +// (..., request field AlpnPolicy.member.N), +// not a bare string element — matching aws-sdk-go-v2 types.Listener.AlpnPolicy ([]string) — +// and that CreateListener, DescribeListeners, and ModifyListener all agree on that shape. +func Test_AlpnPolicyWireShape(t *testing.T) { + t.Parallel() + + type alpnListenerResp struct { + ListenerArn string `xml:"ListenerArn"` + AlpnPolicy struct { + Members []string `xml:"member"` + } `xml:"AlpnPolicy"` + } + + cases := []struct { + name string + values []string + wantNil bool + }{ + {name: "single_policy", values: []string{"HTTP2Optional"}}, + {name: "multiple_policies", values: []string{"HTTP2Optional", "HTTP1Only"}}, + {name: "no_policy_omits_element", values: nil, wantNil: true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + nameSuffix := strings.ReplaceAll(tc.name, "_", "-") + + h := newTestHandler() + lbArn := mustCreateNLB(t, h, "alpn-"+nameSuffix) + tgArn := mustCreateTG(t, h, "alpn-tg-"+nameSuffix) + + vals := url.Values{ + "Action": {"CreateListener"}, + "Version": {"2015-12-01"}, + "LoadBalancerArn": {lbArn}, + "Protocol": {"TLS"}, + "Port": {"443"}, + "DefaultActions.member.1.Type": {"forward"}, + "DefaultActions.member.1.TargetGroupArn": {tgArn}, + "Certificates.member.1.CertificateArn": {"arn:aws:acm:us-east-1:123456789012:certificate/abc"}, + } + + for i, p := range tc.values { + vals.Set(fmt.Sprintf("AlpnPolicy.member.%d", i+1), p) + } + + rec := doELBv2(t, h, vals) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + var createResp struct { + Result struct { + Listeners struct { + Members []alpnListenerResp `xml:"member"` + } `xml:"Listeners"` + } `xml:"CreateListenerResult"` + } + require.NoError(t, xml.Unmarshal(rec.Body.Bytes(), &createResp)) + require.Len(t, createResp.Result.Listeners.Members, 1) + + if tc.wantNil { + assert.Empty(t, createResp.Result.Listeners.Members[0].AlpnPolicy.Members) + } else { + assert.Equal(t, tc.values, createResp.Result.Listeners.Members[0].AlpnPolicy.Members) + } + + // DescribeListeners must round-trip the same list shape. + listenerArn := createResp.Result.Listeners.Members[0].ListenerArn + descRec := doELBv2(t, h, url.Values{ + "Action": {"DescribeListeners"}, + "Version": {"2015-12-01"}, + "ListenerArns.member.1": {listenerArn}, + }) + require.Equal(t, http.StatusOK, descRec.Code) + + var descResp struct { + Result struct { + Listeners struct { + Members []alpnListenerResp `xml:"member"` + } `xml:"Listeners"` + } `xml:"DescribeListenersResult"` + } + require.NoError(t, xml.Unmarshal(descRec.Body.Bytes(), &descResp)) + require.Len(t, descResp.Result.Listeners.Members, 1) + + if tc.wantNil { + assert.Empty(t, descResp.Result.Listeners.Members[0].AlpnPolicy.Members) + } else { + assert.Equal(t, tc.values, descResp.Result.Listeners.Members[0].AlpnPolicy.Members) + } + }) + } +} + // TestNameTooLong verifies that LB and TG names longer than 32 chars are rejected. func TestNameTooLong(t *testing.T) { t.Parallel() @@ -6166,7 +6284,7 @@ func TestDescribeLoadBalancersNotFound(t *testing.T) { "Version": {"2015-12-01"}, "LoadBalancerArns.member.1": {fakeArn}, }) - assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) } // TestDescribeListenersNotFound verifies that querying non-existent listener ARNs returns an error. @@ -6181,7 +6299,7 @@ func TestDescribeListenersNotFound(t *testing.T) { "Version": {"2015-12-01"}, "ListenerArns.member.1": {fakeArn}, }) - assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) } // TestTargetGroupLoadBalancerArnsAfterListener verifies that TG shows LB ARNs after listener creation. @@ -6591,8 +6709,9 @@ func TestNLBAttributeDefaults(t *testing.T) { assert.NotContains(t, attrMap, "routing.http.response.server.enabled") } -// TestDescribeLoadBalancersByNameNotFound verifies that querying a non-existent LB by name returns 404, -// matching real AWS which raises LoadBalancerNotFoundException for any unknown name. +// TestDescribeLoadBalancersByNameNotFound verifies that querying a non-existent LB by name +// returns 400 (LoadBalancerNotFound), matching real AWS which raises +// LoadBalancerNotFoundException for any unknown name. func TestDescribeLoadBalancersByNameNotFound(t *testing.T) { t.Parallel() @@ -6608,7 +6727,7 @@ func TestDescribeLoadBalancersByNameNotFound(t *testing.T) { "Version": {"2015-12-01"}, "Names.member.1": {"does-not-exist"}, }, - expect: http.StatusNotFound, + expect: http.StatusBadRequest, }, { name: "one_valid_one_missing_name", @@ -6618,7 +6737,7 @@ func TestDescribeLoadBalancersByNameNotFound(t *testing.T) { "Names.member.1": {"desc-lb-name-exists"}, "Names.member.2": {"does-not-exist"}, }, - expect: http.StatusNotFound, + expect: http.StatusBadRequest, }, } @@ -6637,8 +6756,9 @@ func TestDescribeLoadBalancersByNameNotFound(t *testing.T) { } } -// TestDescribeTargetGroupsByNameNotFound verifies that querying non-existent TG names returns 404, -// matching real AWS which raises TargetGroupNotFoundException for any unknown name. +// TestDescribeTargetGroupsByNameNotFound verifies that querying non-existent TG names +// returns 400 (TargetGroupNotFound), matching real AWS which raises +// TargetGroupNotFoundException for any unknown name. func TestDescribeTargetGroupsByNameNotFound(t *testing.T) { t.Parallel() @@ -6654,7 +6774,7 @@ func TestDescribeTargetGroupsByNameNotFound(t *testing.T) { "Version": {"2015-12-01"}, "Names.member.1": {"does-not-exist"}, }, - expect: http.StatusNotFound, + expect: http.StatusBadRequest, }, { name: "one_valid_one_missing_name", @@ -6664,7 +6784,7 @@ func TestDescribeTargetGroupsByNameNotFound(t *testing.T) { "Names.member.1": {"desc-tg-name-exists"}, "Names.member.2": {"does-not-exist"}, }, - expect: http.StatusNotFound, + expect: http.StatusBadRequest, }, } diff --git a/services/elbv2/parity_b_test.go b/services/elbv2/parity_b_test.go index b1013cccb..2a0f49351 100644 --- a/services/elbv2/parity_b_test.go +++ b/services/elbv2/parity_b_test.go @@ -212,7 +212,7 @@ func TestParity_DescribeListeners_UnknownLB(t *testing.T) { { name: "unknown_lb_arn_returns_not_found", lbArn: "arn:aws:elasticloadbalancing:us-east-1:123456789012:loadbalancer/app/nonexistent/abcdef123456", - wantCode: http.StatusNotFound, + wantCode: http.StatusBadRequest, wantErrCode: "LoadBalancerNotFound", }, { diff --git a/services/elbv2/persistence.go b/services/elbv2/persistence.go index dd6da12b7..77b5f7fe4 100644 --- a/services/elbv2/persistence.go +++ b/services/elbv2/persistence.go @@ -2,26 +2,44 @@ package elbv2 import ( "context" + "encoding/json" "errors" + "fmt" "time" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" ) // errBackendNotInMemory is returned when the Handler's backend cannot be cast to *InMemoryBackend. var errBackendNotInMemory = errors.New("elbv2: backend is not *InMemoryBackend") +// elbv2SnapshotVersion identifies the shape of backendSnapshot's Tables blob +// (i.e. the set/shape of resources registered on b.registry -- see +// registerAllTables in store_setup.go). It must be bumped whenever a change +// there would make an older snapshot unsafe to decode as the current shape. +// Restore compares this against the persisted value and discards (rather than +// attempts to partially decode) any mismatch -- see Restore below. This +// mirrors the services/ec2 (commit 12e611a4) and services/sqs (commit +// 0f09d77c) pilots. +const elbv2SnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the ELBv2 backend. +// +// Tables holds one JSON-encoded array per registered resource table, produced +// by [github.com/blackbirdworks/gopherstack/pkgs/store.Registry.SnapshotAll] +// -- "loadBalancers", "targetGroups", "listeners", "rules", and "trustStores". +// ResourcePolicies / TargetReadyAt / TargetDrainingUntil are persisted +// separately because they are not store.Table-shaped -- see store_setup.go. type backendSnapshot struct { - LoadBalancers map[string]*LoadBalancer `json:"loadBalancers"` - TargetGroups map[string]*TargetGroup `json:"targetGroups"` - Listeners map[string]*Listener `json:"listeners"` - Rules map[string]*Rule `json:"rules"` - TrustStores map[string]*TrustStore `json:"trustStores"` - ResourcePolicies map[string]string `json:"resourcePolicies"` - TargetReadyAt map[string]map[string]time.Time `json:"targetReadyAt"` - AccountID string `json:"accountID"` - Region string `json:"region"` - RuleCounter int `json:"ruleCounter"` + Tables map[string]json.RawMessage `json:"tables"` + ResourcePolicies map[string]string `json:"resourcePolicies"` + TargetReadyAt map[string]map[string]time.Time `json:"targetReadyAt"` + TargetDrainingUntil map[string]map[string]time.Time `json:"targetDrainingUntil"` + AccountID string `json:"accountID"` + Region string `json:"region"` + Version int `json:"version"` + RuleCounter int `json:"ruleCounter"` } // Snapshot serialises the backend state to JSON. @@ -30,17 +48,26 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "elbv2: snapshot table marshal failed", "error", err) + + return nil + } + snap := backendSnapshot{ - LoadBalancers: b.loadBalancers, - TargetGroups: b.targetGroups, - Listeners: b.listeners, - Rules: b.rules, - TrustStores: b.trustStores, - ResourcePolicies: b.resourcePolicies, - TargetReadyAt: b.targetReadyAt, - RuleCounter: b.ruleCounter, - AccountID: b.accountID, - Region: b.region, + Version: elbv2SnapshotVersion, + Tables: tables, + ResourcePolicies: b.resourcePolicies, + TargetReadyAt: b.targetReadyAt, + TargetDrainingUntil: b.targetDrainingUntil, + RuleCounter: b.ruleCounter, + AccountID: b.accountID, + Region: b.region, } return persistence.MarshalSnapshot(ctx, "elbv2", snap) @@ -58,37 +85,45 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() - if snap.LoadBalancers == nil { - snap.LoadBalancers = make(map[string]*LoadBalancer) - } + if snap.Version != elbv2SnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/ec2 and services/sqs pilots. + logger.Load(ctx).WarnContext(ctx, + "elbv2: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", elbv2SnapshotVersion) + + b.registry.ResetAll() - if snap.TargetGroups == nil { - snap.TargetGroups = make(map[string]*TargetGroup) + return nil } - if snap.Listeners == nil { - snap.Listeners = make(map[string]*Listener) + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("elbv2: restore snapshot tables: %w", err) } - if snap.Rules == nil { - snap.Rules = make(map[string]*Rule) + if snap.ResourcePolicies == nil { + snap.ResourcePolicies = make(map[string]string) } - if snap.TrustStores == nil { - snap.TrustStores = make(map[string]*TrustStore) + // targetReadyAt / targetDrainingUntil are written to via `m[tgArn][key] = ...`, + // which panics on a nil top-level map. Snapshots taken before these fields + // existed (or taken while empty, which JSON-encodes to `{}` but can still + // unmarshal as nil for an absent key) must not leave a nil map behind. + if snap.TargetReadyAt == nil { + snap.TargetReadyAt = make(map[string]map[string]time.Time) } - if snap.ResourcePolicies == nil { - snap.ResourcePolicies = make(map[string]string) + if snap.TargetDrainingUntil == nil { + snap.TargetDrainingUntil = make(map[string]map[string]time.Time) } - b.loadBalancers = snap.LoadBalancers - b.targetGroups = snap.TargetGroups - b.listeners = snap.Listeners - b.rules = snap.Rules - b.trustStores = snap.TrustStores b.resourcePolicies = snap.ResourcePolicies b.targetReadyAt = snap.TargetReadyAt + b.targetDrainingUntil = snap.TargetDrainingUntil b.ruleCounter = snap.RuleCounter b.accountID = snap.AccountID b.region = snap.Region diff --git a/services/elbv2/persistence_test.go b/services/elbv2/persistence_test.go new file mode 100644 index 000000000..f2c91bf01 --- /dev/null +++ b/services/elbv2/persistence_test.go @@ -0,0 +1,144 @@ +package elbv2_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/config" + "github.com/blackbirdworks/gopherstack/services/elbv2" +) + +// TestInMemoryBackend_SnapshotRestore exercises a full-state Snapshot->Restore +// round-trip covering every resource kind on the backend: the five +// store.Table-backed resources (load balancers, target groups, listeners, +// rules, trust stores), the plain resourcePolicies map (no identity field of +// its own), and the doubly-nested targetDrainingUntil map (Phase 3.3 +// datalayer conversion; see store_setup.go for why these two are left as raw +// maps instead of store.Tables). This is the parity-sweep fix for +// targetDrainingUntil persistence -- a restored backend must still drain and +// remove targets that were mid-deregistration when the snapshot was taken. +func TestInMemoryBackend_SnapshotRestore(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + }{ + {name: "full backend state survives a snapshot/restore round-trip"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + src := elbv2.NewInMemoryBackend("123456789012", config.DefaultRegion) + defer src.Close() + + lb, err := src.CreateLoadBalancer(elbv2.CreateLoadBalancerInput{ + Name: "snap-lb", + Subnets: []string{"subnet-1", "subnet-2"}, + }) + require.NoError(t, err) + + tg, err := src.CreateTargetGroup(elbv2.CreateTargetGroupInput{ + Name: "snap-tg", + Protocol: "HTTP", + Port: 80, + VpcID: "vpc-1", + }) + require.NoError(t, err) + + listener, err := src.CreateListener(elbv2.CreateListenerInput{ + LoadBalancerArn: lb.LoadBalancerArn, + Protocol: "HTTP", + Port: 80, + DefaultActions: []elbv2.Action{{Type: "forward", TargetGroupArn: tg.TargetGroupArn}}, + }) + require.NoError(t, err) + + rule, err := src.CreateRule(elbv2.CreateRuleInput{ + ListenerArn: listener.ListenerArn, + Priority: "1", + Actions: []elbv2.Action{{Type: "forward", TargetGroupArn: tg.TargetGroupArn}}, + Conditions: []elbv2.Condition{{Field: "path-pattern", Values: []string{"/api/*"}}}, + }) + require.NoError(t, err) + + trustStore, err := src.CreateTrustStore("snap-ts", nil) + require.NoError(t, err) + + const policyDoc = `{"Version":"2012-10-17"}` + require.NoError(t, src.PutResourcePolicy(lb.LoadBalancerArn, policyDoc)) + + // Register then immediately deregister a target so it lands in the + // "draining" state, populating the raw targetDrainingUntil map. + require.NoError(t, src.RegisterTargets(tg.TargetGroupArn, []elbv2.Target{{ID: "10.0.0.1", Port: 80}})) + require.NoError(t, src.DeregisterTargets(tg.TargetGroupArn, []elbv2.Target{{ID: "10.0.0.1", Port: 80}})) + + data := src.Snapshot(ctx) + require.NotNil(t, data) + + dst := elbv2.NewInMemoryBackend("123456789012", config.DefaultRegion) + defer dst.Close() + + require.NoError(t, dst.Restore(ctx, data)) + + // Load balancer (store.Table). + lbs, err := dst.DescribeLoadBalancers(nil, nil) + require.NoError(t, err) + require.Len(t, lbs, 1) + assert.Equal(t, lb.LoadBalancerArn, lbs[0].LoadBalancerArn) + assert.Equal(t, lb.LoadBalancerName, lbs[0].LoadBalancerName) + + // Target group (store.Table). + tgs, err := dst.DescribeTargetGroups(nil, nil, "") + require.NoError(t, err) + require.Len(t, tgs, 1) + assert.Equal(t, tg.TargetGroupArn, tgs[0].TargetGroupArn) + + // Listener (store.Table, indexed by loadBalancer). + listeners, err := dst.DescribeListeners(lb.LoadBalancerArn, nil) + require.NoError(t, err) + require.Len(t, listeners, 1) + assert.Equal(t, listener.ListenerArn, listeners[0].ListenerArn) + + // Rules (store.Table, indexed by listener): the auto-created default + // rule plus the one explicitly created above. + rules, err := dst.DescribeRules(listener.ListenerArn, nil) + require.NoError(t, err) + require.Len(t, rules, 2) + + var foundRule bool + for _, r := range rules { + if r.RuleArn == rule.RuleArn { + foundRule = true + assert.Equal(t, rule.Priority, r.Priority) + } + } + assert.True(t, foundRule, "explicitly created rule must survive restore") + + // Trust store (store.Table). + trustStores, err := dst.DescribeTrustStores(nil, nil) + require.NoError(t, err) + require.Len(t, trustStores, 1) + assert.Equal(t, trustStore.TrustStoreArn, trustStores[0].TrustStoreArn) + + // resourcePolicies: plain map (value has no identity field), still persisted. + gotPolicy, err := dst.GetResourcePolicy(lb.LoadBalancerArn) + require.NoError(t, err) + assert.JSONEq(t, policyDoc, gotPolicy) + + // targetDrainingUntil: doubly-nested map, parity-sweep fix. The + // deregistered target must still read back as draining post-restore + // rather than snapping back to registered/healthy or disappearing. + health, err := dst.DescribeTargetHealth(tg.TargetGroupArn) + require.NoError(t, err) + require.Len(t, health, 1) + assert.Equal(t, "draining", health[0].HealthState) + }) + } +} diff --git a/services/elbv2/store_setup.go b/services/elbv2/store_setup.go new file mode 100644 index 000000000..9428cfb85 --- /dev/null +++ b/services/elbv2/store_setup.go @@ -0,0 +1,80 @@ +package elbv2 + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend that carries its own +// identity (an ARN field) is registered exactly once, here, as a +// *store.Table[T] on b.registry. See pkgs/store's package doc and the +// services/ec2 (commit 12e611a4) / services/sqs (commit 0f09d77c) pilots for +// the pattern this follows. +// +// Two fields are deliberately NOT registered here and remain plain maps -- +// see the comment above registerAllTables for the list and why. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +func loadBalancersKeyFn(v *LoadBalancer) string { return v.LoadBalancerArn } +func targetGroupsKeyFn(v *TargetGroup) string { return v.TargetGroupArn } +func listenersKeyFn(v *Listener) string { return v.ListenerArn } +func rulesKeyFn(v *Rule) string { return v.RuleArn } +func trustStoresKeyFn(v *TrustStore) string { return v.TrustStoreArn } + +// listenerLBIndexKeyFn groups listeners by their owning load balancer ARN, +// backing the b.listenersByLB secondary index used for LB-scoped listener +// scans (DescribeListeners with an lbArn filter, cascading LB delete, +// duplicate-port checks, and target-group-to-LB resolution). +func listenerLBIndexKeyFn(v *Listener) string { return v.LoadBalancerArn } + +// ruleListenerIndexKeyFn groups rules by their owning listener ARN, backing +// the b.rulesByListener secondary index used for listener-scoped rule scans +// (DescribeRules with a listenerArn filter, cascading listener delete, +// default-rule sync, and rule-priority collision checks). +func ruleListenerIndexKeyFn(v *Rule) string { return v.ListenerArn } + +// registerAllTables registers every identity-carrying resource map on +// b.registry exactly once. It must be called during construction only +// (immediately after b.registry is created), never on every reset -- +// store.Register panics on a duplicate name, so runtime resets go through +// registry.ResetAll() instead. +// +// The following resource fields are deliberately left as plain maps (not +// registered here) because their value carries no identity field of its own +// (store.Table requires the key to be a pure function of the stored value): +// - resourcePolicies: value type is a bare string (the policy document), +// keyed externally by resource ARN. +// - targetReadyAt / targetDrainingUntil: doubly-nested +// map[string]map[string]time.Time (target-group ARN -> target key -> +// timestamp) -- not a map[string]*V shape at all, let alone one with a +// value that knows its own key. +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call (plus any secondary index) to the concrete field and value type. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.loadBalancers = store.Register(b.registry, "loadBalancers", store.New(loadBalancersKeyFn)) + }, + func(b *InMemoryBackend) { + b.targetGroups = store.Register(b.registry, "targetGroups", store.New(targetGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + listeners := store.Register(b.registry, "listeners", store.New(listenersKeyFn)) + b.listeners = listeners + b.listenersByLB = listeners.AddIndex("loadBalancer", listenerLBIndexKeyFn) + }, + func(b *InMemoryBackend) { + rules := store.Register(b.registry, "rules", store.New(rulesKeyFn)) + b.rules = rules + b.rulesByListener = rules.AddIndex("listener", ruleListenerIndexKeyFn) + }, + func(b *InMemoryBackend) { + b.trustStores = store.Register(b.registry, "trustStores", store.New(trustStoresKeyFn)) + }, +} diff --git a/services/eventbridge/PARITY.md b/services/eventbridge/PARITY.md new file mode 100644 index 000000000..6d6f0dd14 --- /dev/null +++ b/services/eventbridge/PARITY.md @@ -0,0 +1,216 @@ +--- +service: eventbridge +sdk_module: aws-sdk-go-v2/service/eventbridge@v1.45.21 +last_audit_commit: 9f336807 +last_audit_date: 2026-07-05 +overall: A +ops: + CreateEventBus: {wire: ok, errors: ok, state: ok, persist: ok, note: "name length/prefix validation, 200-per-account custom-bus limit enforced across regions"} + DeleteEventBus: {wire: ok, errors: ok, state: ok, persist: ok, note: "cascades rule/target/index cleanup; default bus protected"} + ListEventBuses: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeEventBus: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateEventBus: {wire: ok, errors: ok, state: ok, persist: ok} + PutRule: {wire: ok, errors: ok, state: ok, persist: ok, note: "EventPattern/ScheduleExpression mutual exclusivity + at-least-one enforced; 300-per-bus rule limit; ScheduleExpression validated via parseScheduleExpression (see schedule.go fix below)"} + DeleteRule: {wire: ok, errors: ok, state: ok, persist: ok, note: "ManagedBy not enforced -- see gaps (gopherstack-ba7)"} + ListRules: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeRule: {wire: ok, errors: ok, state: ok, persist: ok} + EnableRule: {wire: ok, errors: ok, state: ok, persist: ok, note: "ManagedBy not enforced -- see gaps (gopherstack-ba7)"} + DisableRule: {wire: ok, errors: ok, state: ok, persist: ok, note: "ManagedBy not enforced -- see gaps (gopherstack-ba7)"} + PutTargets: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED this sweep: Target was missing 8 of the SDK's target-type-specific parameter structs (AppSyncParameters, EcsParameters, HttpParameters, KinesisParameters, RedshiftDataParameters, RunCommandParameters, SageMakerPipelineParameters, SqsParameters), so any client setting them lost the data silently (json.Unmarshal drops unknown fields). Added all 8 plus their nested types, with required-field validation (EcsParameters.TaskDefinitionArn, KinesisParameters.PartitionKeyPath, RedshiftDataParameters.Database, RunCommandParameters.RunCommandTargets[].Key/Values) mirroring aws-sdk-go-v2's client-side validators, and RetryPolicy bound validation (MaximumRetryAttempts 0-185, MaximumEventAgeInSeconds 60-86400) which the client SDK does not validate locally either. 5-targets-per-rule limit enforced (was already ok)."} + RemoveTargets: {wire: ok, errors: ok, state: ok, persist: ok} + ListTargetsByRule: {wire: ok, errors: ok, state: ok, persist: ok, note: "now round-trips all target-type-specific parameters (see PutTargets)"} + ListRuleNamesByTarget: {wire: ok, errors: ok, state: ok, persist: ok} + PutEvents: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED this sweep: added the 1-10 entries-per-request limit (AWS: PutEventsRequestEntryList min 1/max 10 -- was previously unbounded, a test even fed 1100 entries in a single call) and per-entry required-field validation for Source/DetailType/Detail (AWS: an entry missing any of the three fails individually with InvalidArgument; if NONE of the entries in the batch have all three, AWS fails the whole request). Signature changed additively from `[]EventResultEntry` to `([]EventResultEntry, error)` to carry the new whole-request failures -- see Notes for the signature-safety check performed."} + PutPartnerEvents: {wire: ok, errors: ok, state: ok, persist: ok, note: "delegates to PutEvents; inherits the same fix"} + ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok} + TagResource: {wire: ok, errors: ok, state: ok, persist: ok} + UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} + ActivateEventSource: {wire: ok, errors: ok, state: ok, persist: ok} + DeactivateEventSource: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeEventSource: {wire: ok, errors: ok, state: ok, persist: ok} + ListEventSources: {wire: ok, errors: ok, state: ok, persist: ok} + CancelReplay: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeReplay: {wire: ok, errors: ok, state: ok, persist: ok} + ListReplays: {wire: ok, errors: ok, state: ok, persist: ok} + StartReplay: {wire: ok, errors: ok, state: ok, persist: ok, note: "not re-audited op-by-op this sweep beyond a spot check; prior sweeps (c48d08ab) added real replay-worker delivery"} + CreateApiDestination: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteApiDestination: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeApiDestination: {wire: ok, errors: ok, state: ok, persist: ok} + ListApiDestinations: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateApiDestination: {wire: ok, errors: ok, state: ok, persist: ok} + CreateArchive: {wire: ok, errors: ok, state: ok, persist: ok, note: "spot-checked only; not re-audited op-by-op this sweep"} + DeleteArchive: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeArchive: {wire: ok, errors: ok, state: ok, persist: ok} + ListArchives: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateArchive: {wire: ok, errors: ok, state: ok, persist: ok} + CreateConnection: {wire: ok, errors: ok, state: ok, persist: ok, note: "spot-checked only (auth masking, API_KEY/BASIC/OAUTH); not re-audited op-by-op this sweep"} + DeleteConnection: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeConnection: {wire: ok, errors: ok, state: ok, persist: ok} + ListConnections: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateConnection: {wire: ok, errors: ok, state: ok, persist: ok} + DeauthorizeConnection: {wire: ok, errors: ok, state: ok, persist: ok} + CreateEndpoint: {wire: ok, errors: ok, state: ok, persist: ok, note: "spot-checked only; not re-audited op-by-op this sweep"} + DeleteEndpoint: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeEndpoint: {wire: ok, errors: ok, state: ok, persist: ok} + ListEndpoints: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateEndpoint: {wire: ok, errors: ok, state: ok, persist: ok} + CreatePartnerEventSource: {wire: ok, errors: ok, state: ok, persist: ok} + DeletePartnerEventSource: {wire: ok, errors: ok, state: ok, persist: ok} + DescribePartnerEventSource: {wire: ok, errors: ok, state: ok, persist: ok} + ListPartnerEventSources: {wire: ok, errors: ok, state: ok, persist: ok} + ListPartnerEventSourceAccounts: {wire: ok, errors: ok, state: ok, persist: ok} + TestEventPattern: {wire: ok, errors: ok, state: ok, persist: n/a, note: "delegates to the same compilePattern/matchCompiledPattern engine proved correct this sweep -- see families.event_pattern_matching"} + PutPermission: {wire: ok, errors: ok, state: ok, persist: ok, note: "spot-checked only; not re-audited op-by-op this sweep"} + RemovePermission: {wire: ok, errors: ok, state: ok, persist: ok} + GetEventBusPolicy: {wire: partial, errors: ok, state: ok, persist: ok, note: "not a real EventBridge SDK op (no GetEventBusPolicy/PutEventBusPolicy in aws-sdk-go-v2/service/eventbridge's 53 ops); an internal-only helper reachable via the handler's policyActions() dispatch table but absent from GetSupportedOperations, so no real SDK client can invoke it. Harmless (DescribeEventBus.Policy is the real wire path for reading a bus policy) but not itself a modeled AWS op -- left as-is, not a gap worth fixing."} + PutEventBusPolicy: {wire: partial, errors: ok, state: ok, persist: ok, note: "same as GetEventBusPolicy -- not a real SDK op"} +families: + event_pattern_matching: {status: ok, note: "Read pattern.go (559 LOC) in full and cross-checked every documented AWS content-filter operator against matchSpecialMatcher/matchStringMatcher: exact-match arrays, prefix (incl. nested equals-ignore-case form), suffix (incl. nested equals-ignore-case form), exists (true/false, including explicit JSON null counting as present), numeric (paired-operator ranges, all four comparators), anything-but (scalar/list/object forms incl. nested prefix/suffix/wildcard/equals-ignore-case/numeric, each of which may itself be a list), cidr, wildcard (iterative two-pointer glob, no recursion/ReDoS), equals-ignore-case, nested objects (recursive matchObject), $or (top-level and nested), and array-valued event fields (any-element-matches semantics). All correct and already covered by pattern_test.go (519 LOC) + pattern_validation_test.go (129 LOC). No fix needed -- proof only."} + schema_registry_and_pipes: {status: ok, note: "CreateRegistry..GetCodeBindingSource and CreatePipe..UpdatePipe are separate control planes in real AWS (schemas/pipes SDK modules, not events); not re-audited op-by-op this sweep, no evidence of regressions while reading adjacent PutTargets/PutEvents code."} +gaps: + - "Rule.ManagedBy is modeled and echoed on Describe/List, and PutRuleInput even lets a caller set it directly (real AWS's PutRule request has no ManagedBy member at all -- it's a server-populated Describe/List-only field), but NO op (PutRule update, DeleteRule, EnableRule, DisableRule, PutTargets, RemoveTargets) checks it before mutating. Real AWS returns ManagedRuleException for all six when the target rule is AWS-service-managed. Not fixed this sweep: no composition-root code anywhere in this repo ever marks an eventbridge rule as managed, so the missing enforcement is currently unreachable/inert in practice, and building it out (new sentinel error + handleError case + internal seeding helper + a real trigger) is a bigger, more speculative change than the codebase's demonstrated usage patterns justify right now. (bd: gopherstack-ba7)" + - "EventBridge rule-target delivery for non-core targets (Step Functions/ECS/Kinesis/CloudWatch Logs/API destinations) is fully implemented in delivery.go's deliverToTarget dispatch, but wireEventBridgeDelivery in cli.go (composition root, out of services/eventbridge/ and explicitly off-limits this sweep) only populates DeliveryTargets.Lambda/SQS/SNS. Rules with those other target types match correctly but never fire in the running app. Already tracked, not re-fixed. (bd: gopherstack-xoe)" +deferred: + - "Archives (CreateArchive/UpdateArchive/DeleteArchive/DescribeArchive/ListArchives), replays (StartReplay/CancelReplay/DescribeReplay/ListReplays), connections (Create/Update/Delete/Describe/List/DeauthorizeConnection), API destinations (Create/Update/Delete/Describe/List), and global endpoints (Create/Update/Delete/Describe/List) -- spot-checked while reading adjacent code (all looked real: proper validation, real state, ARNs via arn-style helpers, persistence-backed), but not re-audited op-by-op line-by-line this pass. No evidence of regressions found." + - "Schema registry (CreateRegistry..GetCodeBindingSource, 17 ops) and Pipes (CreatePipe..UpdatePipe, 5 ops) -- these model separate AWS control planes (schemas/pipes SDK modules), not core EventBridge (events) ops; not audited this pass." + - "PutPermission/RemovePermission/policy-statement JSON shape (EventBusPolicyStatement.Principal as `any` for both string and object-with-AWS-key forms) -- spot-checked only." +leaks: {status: clean, note: "Re-verified this sweep: PutEvents's async delivery goroutine (b.wg.Go) acquires a workerSem slot or aborts on svcCtx.Done() before delivering, so Close()/Shutdown() cannot leave in-flight goroutines past defaultShutdownTimeout; deliverToTargetBounded applies a per-attempt context.WithTimeout and always cancels it. Scheduler (scheduler.go) and ArchiveJanitor (janitor.go) were not independently re-verified this pass (no changes made in either file), but existing leak_test.go/isolation_test.go continue to pass, including TestEventLog_CappedAtMax and TestEventLog_RetainsMostRecentEvents which now exercise the new 10-entry PutEvents batch cap via a batching loop (previously a single 1100-entry call, which the new limit would reject outright -- test updated, see Notes)."} +--- + +## Notes + +Freeform findings from this sweep (gopherstack-b84), for the next auditor. + +### Fixed this sweep (severe/high-value first) + +1. **PutTargets silently dropped 8 of the SDK's target-type-specific parameter + structs.** `Target` (models.go) only modeled + Input/InputPath/InputTransformer/DeadLetterConfig/RetryPolicy/BatchParameters. + Any client setting `EcsParameters`, `HttpParameters`, `KinesisParameters`, + `RedshiftDataParameters`, `RunCommandParameters`, + `SageMakerPipelineParameters`, `SqsParameters`, or `AppSyncParameters` on a + target had that configuration vanish on the next + `ListTargetsByRule`/`DescribeRule` call -- `encoding/json` silently drops + unknown fields on unmarshal. This is exactly the "disguised stub" class the + parity principles warn about: `PutTargets` looked fully real (validates, + stores, indexes) but was quietly incomplete for any ECS/Kinesis/Redshift + Data API/EC2 Run Command/SageMaker Pipeline/SQS-FIFO/AppSync/API-Gateway + target. Fixed by adding all 8 structs (with their full nested shapes -- + e.g. `EcsParameters.NetworkConfiguration.AwsvpcConfiguration`, + `CapacityProviderStrategy`, `PlacementConstraints/Strategy`, ECS task + `Tags`) verified field-by-field against + `aws-sdk-go-v2/service/eventbridge/types` (json-1.1 protocol: wire key == + Go SDK field name exactly, confirmed against serializers.go), plus + required-member validation mirroring the SDK's own client-side validators + (`validateEcsParameters`, `validateKinesisParameters`, + `validateRedshiftDataParameters`, `validateRunCommandParameters`/`Target`) + and RetryPolicy bound validation AWS documents but the client SDK does not + enforce locally (MaximumRetryAttempts 0-185, MaximumEventAgeInSeconds + 60-86400 when set). All additive to `Target`/`PutTargets` -- no exported + signature changed here. + +2. **ScheduleExpression cron day-of-week used the wrong numbering AND didn't + support day/month names at all**, despite an existing test + (`cron(0 8 ? * MON-FRI *)`) implying it should. AWS's cron day-of-week + field is 1-7 with **1 = Sunday**; the matcher compared the raw field token + directly against Go's `time.Weekday()`, which is 0-6 with **0 = Sunday** -- + an off-by-one that would silently fire scheduled rules one weekday early + for every numeric day-of-week cron expression. Separately, three-letter + names (`JAN`-`DEC` for month, `SUN`-`SAT` for day-of-week) were not + resolved at all: `parseCron` only validates the *field count* (6), so + `cron(0 8 ? * MON-FRI *)` parsed without error, but at match time + `matchCronRange`/`matchCronToken` called `strconv.Atoi("MON")`, which + always fails -- so that rule's schedule silently **never fired**, in any + timezone, forever. This is the "test looks like it proves X but only + proves parsing succeeds, not that matching is correct" trap -- worth + flagging for the next auditor since it's easy to mistake a passing + `TestSchedule_ParseCron` for evidence the feature works. Fixed by adding a + `cronFieldKind`-aware token resolver (`cronTokenValue`) that maps + `JAN`-`DEC`/`SUN`-`SAT` names and converts AWS's 1-7 day-of-week numbering + to Go's canonical 0-6 at the single point where tokens are resolved, so + every downstream comparison (exact/range/step) is against canonical Go + weekday values. Added `TestSchedule_CronDayOfWeek` (11 cases) proving both + the numbering fix and name support end-to-end via real `NextAfter` fire + times, not just parse-success. + +3. **PutEvents had no 1-10 entries-per-request limit and no per-entry + required-field validation.** AWS's `PutEventsRequestEntryList` is + documented min 1/max 10 items (enforced server-side; the client SDK's + `validateOpPutEventsInput` only checks `Entries != nil`, not length or + per-entry shape). This backend accepted arbitrarily large batches (a + pre-existing test literally fed 1100 entries in one call to exercise the + event-log cap) and never validated that `Source`/`DetailType`/`Detail` + were present, silently assigning a real EventId and "succeeding" for + malformed entries that real AWS would reject with a per-entry + `InvalidArgument` (or fail the whole request if *no* entry in the batch + has all three -- see `PutEventsRequestEntry.Detail`'s doc comment in + aws-sdk-go-v2/service/eventbridge/types, which spells out both behaviors). + Fixed both. The **exported signature of `PutEvents`/`PutPartnerEvents` + changed additively** from `func(...) []EventResultEntry` to + `func(...) ([]EventResultEntry, error)` to carry whole-request failures + (>10 entries, 0 entries, or no entry with all three required fields). + Signature-safety check performed: grepped every call site repo-wide; + `cli.go:3364` and `cli_adapters.go:43` call `PutEvents` as a bare statement + without capturing the return value, so the added return value does not + break either composition-root call site (verified with `go build ./...` + before/after). Only in-package callers captured a single return value + (`sfn_integration.go`'s `SFNPutEvents`, plus several `_test.go` files) and + were updated to handle the new `error`. + `TestHandler_PutEvents_Empty` previously asserted an empty `Entries: []` + batch returns HTTP 200 -- that encoded the wrong AWS behavior (AWS's + `minItems: 1` constraint makes it a validation error, not a no-op + success), so it was corrected to expect 400 with a comment explaining why. + Added `TestAudit_PutEvents_EntryCountLimit` and + `TestAudit_PutEvents_RequiredFields` (9 cases total) proving the new + behavior. + +### Read and proven already-correct (no fix needed) + +- **`pattern.go`'s EventPattern matching engine** (559 LOC) -- read in full + and cross-checked every AWS content-filter operator: exact-match arrays, + `prefix`/`suffix` (including the nested `equals-ignore-case` form AWS added + for case-insensitive prefix/suffix), `exists` (including that an explicit + JSON `null` value counts as the key being *present*, matching AWS), numeric + ranges (paired operators, all four comparators), `anything-but` in all its + forms (scalar, list, and object -- where the object form's inner matcher + may itself be a list, each element of which negates independently), + `cidr`, `wildcard` (iterative two-pointer glob matching, so no + recursion/backtracking blowup on adversarial patterns), nested objects + (recursive), `$or` (both top-level and nested inside any object, including + inside `detail`), and the "if the event field is a JSON array, any element + matching satisfies the matcher" rule. All correct. This is a proof, not a + fix -- flagging so the next auditor can trust this file without re-reading + it (per the re-audit protocol: `pattern.go` unchanged since this commit -> + trust the `ok` row). + +### Traps for the next auditor + +- A cron/rate schedule test that only asserts **parsing succeeds** + (`TestSchedule_ParseCron`'s `"weekday"` case) is not proof the schedule + actually **fires** correctly -- `parseCron` deliberately only validates + field *count*, not field *content*, so a syntactically-6-field expression + with content the matcher can't resolve (unsupported names, wrong numbering + convention) parses cleanly and then simply never matches any candidate + tick. Always follow a parse-test with a `NextAfter`-driven fire-time + assertion (see `TestSchedule_CronDayOfWeek`) before trusting a schedule + expression "supported." +- AWS's day-of-week cron convention is **1-7 with 1 = Sunday**; Go's + `time.Weekday()` is **0-6 with 0 = Sunday**. Any code comparing a raw cron + field token against `int(t.Weekday())` needs the offset in + `cronTokenValue` -- don't reintroduce a direct comparison. +- `GetEventBusPolicy`/`PutEventBusPolicy` (handler.go's `policyActions()`) + are **not real EventBridge SDK operations** (absent from + `aws-sdk-go-v2/service/eventbridge`'s 53-op surface; the real wire path for + reading a bus policy is `DescribeEventBus.Policy`). They're also absent + from `GetSupportedOperations()`/`ChaosOperations()`, so no real AWS SDK + client can reach them. Harmless, but don't mistake their presence in the + dispatch table for a modeled AWS op when doing SDK-completeness sweeps. +- `fieldalignment -fix` (govet, enabled via `enable: [fieldalignment]` in + `.golangci.yml`) reorders **struct field declarations** but does **not** + update positional (unkeyed) struct literals elsewhere that depend on the + old field order -- it silently produces a type-mismatch compile error in + `_test.go` files that `go build ./...` won't catch (test files aren't + compiled by `go build`), only `go vet`/`go test` will. If you ever run it + as an autofix across a package with anonymous-struct test tables using + positional literals, run `go vet ./...` immediately after and expect to + convert the affected literals to keyed form. diff --git a/services/eventbridge/accuracy_audit_test.go b/services/eventbridge/accuracy_audit_test.go index 5be4b1dc3..63b33c58c 100644 --- a/services/eventbridge/accuracy_audit_test.go +++ b/services/eventbridge/accuracy_audit_test.go @@ -1775,7 +1775,8 @@ func TestAudit_PutPartnerEvents_RecordsResults(t *testing.T) { {Source: "aws.partner/example.com/app", DetailType: "OrderPlaced", Detail: `{"orderId":"o1"}`}, } - results := b.PutPartnerEvents(context.Background(), entries) + results, err := b.PutPartnerEvents(context.Background(), entries) + require.NoError(t, err) assert.Len(t, results, 2) for _, r := range results { assert.Empty(t, r.ErrorCode) @@ -1954,9 +1955,10 @@ func TestAudit_PutEvents_BatchSizeLimit(t *testing.T) { // Build one massive entry > 256KiB in detail. bigDetail := strings.Repeat("x", 260*1024) - results := b.PutEvents(context.Background(), []eventbridge.EventEntry{ + results, err := b.PutEvents(context.Background(), []eventbridge.EventEntry{ {Source: "s", DetailType: "T", Detail: bigDetail}, }) + require.NoError(t, err) require.Len(t, results, 1) assert.Equal(t, "EventSizeLimitExceeded", results[0].ErrorCode) } @@ -1967,24 +1969,139 @@ func TestAudit_PutEvents_MixedBatch(t *testing.T) { // First entry is small (accepted), second is huge (rejected), third is small (accepted). bigDetail := strings.Repeat("y", 260*1024) - results := b.PutEvents(context.Background(), []eventbridge.EventEntry{ + results, err := b.PutEvents(context.Background(), []eventbridge.EventEntry{ {Source: "s", DetailType: "T", Detail: `{}`}, {Source: "s", DetailType: "T", Detail: bigDetail}, {Source: "s", DetailType: "T", Detail: `{}`}, }) + require.NoError(t, err) require.Len(t, results, 3) assert.Empty(t, results[0].ErrorCode) assert.Equal(t, "EventSizeLimitExceeded", results[1].ErrorCode) assert.Empty(t, results[2].ErrorCode) } +// TestAudit_PutEvents_EntryCountLimit proves AWS's 1-10 entries-per-request +// constraint on PutEvents (PutEventsRequestEntryList: Array Members Minimum +// 1, Maximum 10 item(s)): 0 or 11 entries fail the whole request, while 1 +// and 10 succeed. +func TestAudit_PutEvents_EntryCountLimit(t *testing.T) { + t.Parallel() + + makeEntries := func(n int) []eventbridge.EventEntry { + entries := make([]eventbridge.EventEntry, n) + for i := range entries { + entries[i] = eventbridge.EventEntry{Source: "s", DetailType: "T", Detail: `{}`} + } + + return entries + } + + tests := []struct { + name string + count int + wantErr bool + }{ + {"zero entries rejected", 0, true}, + {"one entry accepted", 1, false}, + {"ten entries accepted", 10, false}, + {"eleven entries rejected", 11, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + b := newBackend() + results, err := b.PutEvents(context.Background(), makeEntries(tt.count)) + if tt.wantErr { + require.Error(t, err) + require.ErrorIs(t, err, eventbridge.ErrInvalidParameter) + assert.Nil(t, results) + + return + } + require.NoError(t, err) + require.Len(t, results, tt.count) + for _, r := range results { + assert.Empty(t, r.ErrorCode) + } + }) + } +} + +// TestAudit_PutEvents_RequiredFields proves that Source, DetailType, and +// Detail are required on every PutEvents entry (per aws-sdk-go-v2's +// PutEventsRequestEntry doc: "Detail, DetailType, and Source are required +// for EventBridge to successfully send an event... If you include event +// entries in a request that do not include each of those properties, +// EventBridge fails that entry. If you submit a request in which none of the +// entries have each of these properties, EventBridge fails the entire +// request."). +func TestAudit_PutEvents_RequiredFields(t *testing.T) { + t.Parallel() + + t.Run("entry missing Source fails individually", func(t *testing.T) { + t.Parallel() + b := newBackend() + results, err := b.PutEvents(context.Background(), []eventbridge.EventEntry{ + {DetailType: "T", Detail: `{}`}, + {Source: "s", DetailType: "T", Detail: `{}`}, + }) + require.NoError(t, err) + require.Len(t, results, 2) + assert.Equal(t, "InvalidArgument", results[0].ErrorCode) + assert.NotEmpty(t, results[0].ErrorMessage) + assert.Empty(t, results[1].ErrorCode) + assert.NotEmpty(t, results[1].EventID) + }) + + t.Run("entry missing DetailType fails individually", func(t *testing.T) { + t.Parallel() + b := newBackend() + results, err := b.PutEvents(context.Background(), []eventbridge.EventEntry{ + {Source: "s", Detail: `{}`}, + {Source: "s", DetailType: "T", Detail: `{}`}, + }) + require.NoError(t, err) + require.Len(t, results, 2) + assert.Equal(t, "InvalidArgument", results[0].ErrorCode) + assert.Empty(t, results[1].ErrorCode) + }) + + t.Run("entry missing Detail fails individually", func(t *testing.T) { + t.Parallel() + b := newBackend() + results, err := b.PutEvents(context.Background(), []eventbridge.EventEntry{ + {Source: "s", DetailType: "T"}, + {Source: "s", DetailType: "T", Detail: `{}`}, + }) + require.NoError(t, err) + require.Len(t, results, 2) + assert.Equal(t, "InvalidArgument", results[0].ErrorCode) + assert.Empty(t, results[1].ErrorCode) + }) + + t.Run("no entry complete fails the whole request", func(t *testing.T) { + t.Parallel() + b := newBackend() + results, err := b.PutEvents(context.Background(), []eventbridge.EventEntry{ + {DetailType: "T", Detail: `{}`}, + {Source: "s", Detail: `{}`}, + }) + require.Error(t, err) + require.ErrorIs(t, err, eventbridge.ErrInvalidParameter) + assert.Nil(t, results) + }) +} + func TestAudit_PutEvents_DefaultBus(t *testing.T) { t.Parallel() b := newBackend() - results := b.PutEvents(context.Background(), []eventbridge.EventEntry{ + results, err := b.PutEvents(context.Background(), []eventbridge.EventEntry{ {Source: "default.test", DetailType: "Ping", Detail: `{}`}, }) + require.NoError(t, err) require.Len(t, results, 1) assert.Empty(t, results[0].ErrorCode) assert.NotEmpty(t, results[0].EventID) @@ -1997,9 +2114,10 @@ func TestAudit_PutEvents_NamedBus(t *testing.T) { _, err := b.CreateEventBus(context.Background(), "named-bus", "") require.NoError(t, err) - results := b.PutEvents(context.Background(), []eventbridge.EventEntry{ + results, err := b.PutEvents(context.Background(), []eventbridge.EventEntry{ {Source: "s", DetailType: "T", Detail: `{}`, EventBusName: "named-bus"}, }) + require.NoError(t, err) require.Len(t, results, 1) assert.Empty(t, results[0].ErrorCode) } diff --git a/services/eventbridge/backend.go b/services/eventbridge/backend.go index 939a2daab..b2703bc19 100644 --- a/services/eventbridge/backend.go +++ b/services/eventbridge/backend.go @@ -19,6 +19,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/config" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) const ( @@ -86,6 +87,11 @@ const ( maxEventBusesPerAccount = 200 // maxRulesPerBus is the AWS limit for rules per event bus. maxRulesPerBus = 300 + + // putTargetsFailedEntryErrorCode is the FailedEntry.ErrorCode PutTargets + // uses for every per-target validation failure (bad Id, InputTransformer, + // target-type-specific parameters, RetryPolicy bounds). + putTargetsFailedEntryErrorCode = "InvalidParameter" ) type ruleIndexKey struct { @@ -112,7 +118,7 @@ type StorageBackend interface { ruleName, eventBusName, nextToken string, limit int, ) ([]Target, string, error) - PutEvents(ctx context.Context, entries []EventEntry) []EventResultEntry + PutEvents(ctx context.Context, entries []EventEntry) ([]EventResultEntry, error) GetEventLog(ctx context.Context) []EventLogEntry ActivateEventSource(ctx context.Context, name string) error DeactivateEventSource(ctx context.Context, name string) error @@ -144,7 +150,7 @@ type StorageBackend interface { DescribePartnerEventSource(ctx context.Context, name string) (*PartnerEventSource, error) DeletePartnerEventSource(ctx context.Context, name string) error ListPartnerEventSources(ctx context.Context, namePrefix, nextToken string) ([]PartnerEventSource, string, error) - PutPartnerEvents(ctx context.Context, entries []EventEntry) []EventResultEntry + PutPartnerEvents(ctx context.Context, entries []EventEntry) ([]EventResultEntry, error) DescribeReplay(ctx context.Context, name string) (*Replay, error) ListReplays(ctx context.Context, namePrefix, nextToken string) ([]Replay, string, error) StartReplay(ctx context.Context, input StartReplayInput) (*Replay, error) @@ -186,29 +192,48 @@ type StorageBackend interface { type InMemoryBackend struct { ctx context.Context mu *lockmetrics.RWMutex - // Region-isolated stores. The outer key is the AWS region; the inner keys are - // region-free (bus name, rule name, resource name, etc). - connections map[string]map[string]*Connection - rules map[string]map[string]map[string]*Rule - targets map[string]map[string]map[string]*Target - eventSources map[string]map[string]*EventSource - replays map[string]map[string]*Replay - apiDestinations map[string]map[string]*APIDestination + // registry is the lifecycle registry for every PERSISTED *store.Table + // below -- see store_setup.go's package doc for why eventbridge + // (region-scoped, with rules/targets nested one level deeper still) + // needs the lazy getOrCreateTable/getOrCreateNestedTable/ + // getOrCreateGlobalTable helpers rather than the flat static + // registration ec2/sqs use. persistence.go drives Snapshot/Restore + // through this registry only. + registry *store.Registry + // auxRegistry holds pipes/registries/schemas: also *store.Table-backed, + // but deliberately NOT snapshotted (see store_setup.go's package doc) -- + // they were never part of backendSnapshot before this conversion, and + // this preserves that byte-for-byte. + auxRegistry *store.Registry + // Region-isolated stores. The outer key is the AWS region; the leaf + // *store.Table is keyed by the resource's own identity field (bus name, + // rule name, resource name, etc), matching the map key it replaces. + connections map[string]*store.Table[Connection] + rules map[string]map[string]*store.Table[Rule] + targets map[string]map[string]*store.Table[Target] + eventSources map[string]*store.Table[EventSource] + replays map[string]*store.Table[Replay] + apiDestinations map[string]*store.Table[APIDestination] cancel context.CancelFunc deliveryTargets *DeliveryTargets - endpoints map[string]map[string]*Endpoint - buses map[string]map[string]*EventBus - partnerSources map[string]map[string]*PartnerEventSource - archives map[string]map[string]*Archive + endpoints map[string]*store.Table[Endpoint] + buses map[string]*store.Table[EventBus] + partnerSources map[string]*store.Table[PartnerEventSource] + archives map[string]*store.Table[Archive] archivedEvents map[string]map[string][]EventEntry busePolicies map[string]map[string]*EventBusPolicy - pipes map[string]*Pipe - registries map[string]*SchemaRegistry - schemas map[string]map[string]*Schema // registryName → schemaName → Schema - schemaVersions map[string][]*SchemaVersion // "registryName/schemaName" → ordered versions - codeBindings map[string]*CodeBinding // "registryName/schemaName/language" → binding - workerSem chan struct{} - ruleIndex map[string]map[string]map[ruleIndexKey]map[string]*Rule + // pipes and registries are NOT region-scoped -- a single backend holds + // one global Pipe/SchemaRegistry catalogue -- so they are single Tables, + // lazily registered by getOrCreateGlobalTable (see store_setup.go). + pipes *store.Table[Pipe] + registries *store.Table[SchemaRegistry] + // schemas is keyed by registryName (also global, not region-scoped, but + // one dynamic dimension deep like a per-region resource). + schemas map[string]*store.Table[Schema] + schemaVersions map[string][]*SchemaVersion // "registryName/schemaName" → ordered versions + codeBindings map[string]*CodeBinding // "registryName/schemaName/language" → binding + workerSem chan struct{} + ruleIndex map[string]map[string]map[ruleIndexKey]map[string]*Rule // targetsByARN indexes (region → ARN → set of "busKey/ruleName" targetKeys) // for O(1) ListRuleNamesByTarget lookups. Kept consistent on PutTargets / // RemoveTargets / DeleteRule / DeleteEventBus / Reset. @@ -254,21 +279,21 @@ func NewInMemoryBackendWithContext( b := &InMemoryBackend{ accountID: accountID, region: region, - buses: make(map[string]map[string]*EventBus), - rules: make(map[string]map[string]map[string]*Rule), - targets: make(map[string]map[string]map[string]*Target), - eventSources: make(map[string]map[string]*EventSource), - replays: make(map[string]map[string]*Replay), - apiDestinations: make(map[string]map[string]*APIDestination), - archives: make(map[string]map[string]*Archive), + registry: store.NewRegistry(), + auxRegistry: store.NewRegistry(), + buses: make(map[string]*store.Table[EventBus]), + rules: make(map[string]map[string]*store.Table[Rule]), + targets: make(map[string]map[string]*store.Table[Target]), + eventSources: make(map[string]*store.Table[EventSource]), + replays: make(map[string]*store.Table[Replay]), + apiDestinations: make(map[string]*store.Table[APIDestination]), + archives: make(map[string]*store.Table[Archive]), archivedEvents: make(map[string]map[string][]EventEntry), - connections: make(map[string]map[string]*Connection), - endpoints: make(map[string]map[string]*Endpoint), - partnerSources: make(map[string]map[string]*PartnerEventSource), + connections: make(map[string]*store.Table[Connection]), + endpoints: make(map[string]*store.Table[Endpoint]), + partnerSources: make(map[string]*store.Table[PartnerEventSource]), busePolicies: make(map[string]map[string]*EventBusPolicy), - pipes: make(map[string]*Pipe), - registries: make(map[string]*SchemaRegistry), - schemas: make(map[string]map[string]*Schema), + schemas: make(map[string]*store.Table[Schema]), schemaVersions: make(map[string][]*SchemaVersion), codeBindings: make(map[string]*CodeBinding), deliveryTargets: &DeliveryTargets{}, @@ -282,11 +307,11 @@ func NewInMemoryBackendWithContext( targetsByARN: make(map[string]map[string]map[string]struct{}), } // Create the default event bus in the backend's own region. - b.busesStore(b.region)[defaultEventBusName] = &EventBus{ + b.busesTable(b.region).Put(&EventBus{ Name: defaultEventBusName, Arn: b.busARN(b.region, defaultEventBusName), CreatedTime: time.Now(), - } + }) return b } @@ -393,36 +418,47 @@ func (b *InMemoryBackend) targetKey(busName, ruleName string) string { return ebBusKey(busName) + "/" + ruleName } -// busesStore returns the bus map for the given region, lazily creating it. -// Callers must hold b.mu. -func (b *InMemoryBackend) busesStore(region string) map[string]*EventBus { - if b.buses[region] == nil { - b.buses[region] = make(map[string]*EventBus) - } - - return b.buses[region] +// busesTable returns the *store.Table[EventBus] for the given region, lazily +// creating and registering it. Callers must hold b.mu. +func (b *InMemoryBackend) busesTable(region string) *store.Table[EventBus] { + return getOrCreateTable(b.registry, b.buses, "buses", region, eventBusKeyFn) } -// rulesStore returns the rules map for the given region, lazily creating it. +// rulesStore returns the region's bus->Table[Rule] map, lazily creating the +// per-region entry (but NOT any per-bus Table -- use ruleTableFor for that). // Callers must hold b.mu. -func (b *InMemoryBackend) rulesStore(region string) map[string]map[string]*Rule { +func (b *InMemoryBackend) rulesStore(region string) map[string]*store.Table[Rule] { if b.rules[region] == nil { - b.rules[region] = make(map[string]map[string]*Rule) + b.rules[region] = make(map[string]*store.Table[Rule]) } return b.rules[region] } -// targetsStore returns the targets map for the given region, lazily creating it. -// Callers must hold b.mu. -func (b *InMemoryBackend) targetsStore(region string) map[string]map[string]*Target { +// ruleTableFor returns the *store.Table[Rule] for the given region and bus +// key, lazily creating and registering it. Callers must hold b.mu. +func (b *InMemoryBackend) ruleTableFor(region, busKey string) *store.Table[Rule] { + return getOrCreateNestedTable(b.registry, b.rules, "rules", region, busKey, ruleKeyFn) +} + +// targetsStore returns the region's parentKey->Table[Target] map, lazily +// creating the per-region entry (but NOT any per-parent Table -- use +// targetTableFor for that). Callers must hold b.mu. +func (b *InMemoryBackend) targetsStore(region string) map[string]*store.Table[Target] { if b.targets[region] == nil { - b.targets[region] = make(map[string]map[string]*Target) + b.targets[region] = make(map[string]*store.Table[Target]) } return b.targets[region] } +// targetTableFor returns the *store.Table[Target] for the given region and +// target parent key (bus/rule, see targetKey), lazily creating and +// registering it. Callers must hold b.mu. +func (b *InMemoryBackend) targetTableFor(region, parentKey string) *store.Table[Target] { + return getOrCreateNestedTable(b.registry, b.targets, "targets", region, parentKey, targetKeyFnV) +} + // targetsByARNStore returns (lazily creating) the per-region ARN→targetKeys index. // Callers must hold b.mu. func (b *InMemoryBackend) targetsByARNStore(region string) map[string]map[string]struct{} { @@ -461,8 +497,8 @@ func (b *InMemoryBackend) arnIndexRemoveTarget(region, arn, targetKey string) { // arnIndexRemoveRule removes all ARN entries for the given targetKey (i.e. a rule was deleted). // Callers must hold b.mu. -func (b *InMemoryBackend) arnIndexRemoveRule(region, targetKey string, tMap map[string]*Target) { - if tMap == nil { +func (b *InMemoryBackend) arnIndexRemoveRule(region, targetKey string, tTable *store.Table[Target]) { + if tTable == nil { return } @@ -471,7 +507,7 @@ func (b *InMemoryBackend) arnIndexRemoveRule(region, targetKey string, tMap map[ return } - for _, t := range tMap { + for _, t := range tTable.All() { delete(idx[t.Arn], targetKey) if len(idx[t.Arn]) == 0 { delete(idx, t.Arn) @@ -489,44 +525,28 @@ func (b *InMemoryBackend) ruleIndexStore(region string) map[string]map[ruleIndex return b.ruleIndex[region] } -// eventSourcesStore returns the event source map for the given region. -// Callers must hold b.mu. -func (b *InMemoryBackend) eventSourcesStore(region string) map[string]*EventSource { - if b.eventSources[region] == nil { - b.eventSources[region] = make(map[string]*EventSource) - } - - return b.eventSources[region] +// eventSourcesTable returns the *store.Table[EventSource] for the given +// region, lazily creating and registering it. Callers must hold b.mu. +func (b *InMemoryBackend) eventSourcesTable(region string) *store.Table[EventSource] { + return getOrCreateTable(b.registry, b.eventSources, "eventSources", region, eventSourceKeyFn) } -// replaysStore returns the replay map for the given region. -// Callers must hold b.mu. -func (b *InMemoryBackend) replaysStore(region string) map[string]*Replay { - if b.replays[region] == nil { - b.replays[region] = make(map[string]*Replay) - } - - return b.replays[region] +// replaysTable returns the *store.Table[Replay] for the given region, lazily +// creating and registering it. Callers must hold b.mu. +func (b *InMemoryBackend) replaysTable(region string) *store.Table[Replay] { + return getOrCreateTable(b.registry, b.replays, "replays", region, replayKeyFn) } -// apiDestinationsStore returns the API destination map for the given region. -// Callers must hold b.mu. -func (b *InMemoryBackend) apiDestinationsStore(region string) map[string]*APIDestination { - if b.apiDestinations[region] == nil { - b.apiDestinations[region] = make(map[string]*APIDestination) - } - - return b.apiDestinations[region] +// apiDestinationsTable returns the *store.Table[APIDestination] for the given +// region, lazily creating and registering it. Callers must hold b.mu. +func (b *InMemoryBackend) apiDestinationsTable(region string) *store.Table[APIDestination] { + return getOrCreateTable(b.registry, b.apiDestinations, "apiDestinations", region, apiDestinationKeyFn) } -// archivesStore returns the archive map for the given region. -// Callers must hold b.mu. -func (b *InMemoryBackend) archivesStore(region string) map[string]*Archive { - if b.archives[region] == nil { - b.archives[region] = make(map[string]*Archive) - } - - return b.archives[region] +// archivesTable returns the *store.Table[Archive] for the given region, +// lazily creating and registering it. Callers must hold b.mu. +func (b *InMemoryBackend) archivesTable(region string) *store.Table[Archive] { + return getOrCreateTable(b.registry, b.archives, "archives", region, archiveKeyFn) } // archivedEventsStore returns the archived-events map for the given region. @@ -539,34 +559,55 @@ func (b *InMemoryBackend) archivedEventsStore(region string) map[string][]EventE return b.archivedEvents[region] } -// connectionsStore returns the connection map for the given region. -// Callers must hold b.mu. -func (b *InMemoryBackend) connectionsStore(region string) map[string]*Connection { - if b.connections[region] == nil { - b.connections[region] = make(map[string]*Connection) - } +// connectionsTable returns the *store.Table[Connection] for the given region, +// lazily creating and registering it. Callers must hold b.mu. +func (b *InMemoryBackend) connectionsTable(region string) *store.Table[Connection] { + return getOrCreateTable(b.registry, b.connections, "connections", region, connectionKeyFn) +} - return b.connections[region] +// endpointsTable returns the *store.Table[Endpoint] for the given region, +// lazily creating and registering it. Callers must hold b.mu. +func (b *InMemoryBackend) endpointsTable(region string) *store.Table[Endpoint] { + return getOrCreateTable(b.registry, b.endpoints, "endpoints", region, endpointKeyFn) } -// endpointsStore returns the endpoint map for the given region. -// Callers must hold b.mu. -func (b *InMemoryBackend) endpointsStore(region string) map[string]*Endpoint { - if b.endpoints[region] == nil { - b.endpoints[region] = make(map[string]*Endpoint) - } +// partnerSourcesTable returns the *store.Table[PartnerEventSource] for the +// given region, lazily creating and registering it. Callers must hold b.mu. +func (b *InMemoryBackend) partnerSourcesTable(region string) *store.Table[PartnerEventSource] { + return getOrCreateTable(b.registry, b.partnerSources, "partnerSources", region, partnerEventSourceKeyFn) +} - return b.endpoints[region] +// pipesTable returns the backend's single, global *store.Table[Pipe], lazily +// creating and registering it. Callers must hold b.mu. +func (b *InMemoryBackend) pipesTable() *store.Table[Pipe] { + return getOrCreateGlobalTable(b.auxRegistry, &b.pipes, "pipes", pipeKeyFn) } -// partnerSourcesStore returns the partner event source map for the given region. -// Callers must hold b.mu. -func (b *InMemoryBackend) partnerSourcesStore(region string) map[string]*PartnerEventSource { - if b.partnerSources[region] == nil { - b.partnerSources[region] = make(map[string]*PartnerEventSource) +// registriesTable returns the backend's single, global +// *store.Table[SchemaRegistry], lazily creating and registering it. Callers +// must hold b.mu. +func (b *InMemoryBackend) registriesTable() *store.Table[SchemaRegistry] { + return getOrCreateGlobalTable(b.auxRegistry, &b.registries, "registries", schemaRegistryKeyFn) +} + +// schemasTableFor returns the *store.Table[Schema] for the given registry, +// lazily creating and registering it. Callers must hold b.mu. +func (b *InMemoryBackend) schemasTableFor(registryName string) *store.Table[Schema] { + return getOrCreateTable(b.auxRegistry, b.schemas, "schemas", registryName, schemaKeyFn) +} + +// getSchema is a nil-safe read of a registry's schema table: it returns +// (nil, false) rather than panicking when the registry has no schemas table +// yet (a plain nil-map read would have been safe pre-conversion; a nil +// *store.Table[Schema] method call is not). Callers must hold b.mu (for +// reading or writing). +func (b *InMemoryBackend) getSchema(registryName, schemaName string) (*Schema, bool) { + t := b.schemas[registryName] + if t == nil { + return nil, false } - return b.partnerSources[region] + return t.Get(schemaName) } // busePoliciesStore returns the event bus policy map for the given region. @@ -709,16 +750,16 @@ func (b *InMemoryBackend) CreateEventBus(ctx context.Context, name, description b.mu.Lock("CreateEventBus") defer b.mu.Unlock() - buses := b.busesStore(region) - if _, exists := buses[ebBusKey(name)]; exists { + buses := b.busesTable(region) + if buses.Has(ebBusKey(name)) { return nil, fmt.Errorf("%w: Event bus %s already exists", ErrEventBusAlreadyExists, name) } // Count custom buses across all regions — the AWS limit is per-account, not per-region. customBusCount := 0 for _, regionBuses := range b.buses { - for busName := range regionBuses { - if busName != defaultEventBusName { + for _, bus := range regionBuses.All() { + if bus.Name != defaultEventBusName { customBusCount++ } } @@ -737,7 +778,7 @@ func (b *InMemoryBackend) CreateEventBus(ctx context.Context, name, description Description: description, CreatedTime: time.Now(), } - buses[ebBusKey(name)] = bus + buses.Put(bus) return bus, nil } @@ -755,20 +796,20 @@ func (b *InMemoryBackend) DeleteEventBus(ctx context.Context, name string) error defer b.mu.Unlock() busKey := ebBusKey(name) - buses := b.busesStore(region) - if _, exists := buses[busKey]; !exists { + buses := b.busesTable(region) + if !buses.Has(busKey) { return fmt.Errorf("%w: Event bus %s not found", ErrEventBusNotFound, name) } - delete(buses, busKey) + buses.Delete(busKey) delete(b.ruleIndexStore(region), busKey) // Clean up all rules and targets for this bus. rules := b.rulesStore(region) targets := b.targetsStore(region) if busRules, ok := rules[busKey]; ok { - for ruleName := range busRules { - tk := b.targetKey(name, ruleName) + for _, rule := range busRules.All() { + tk := b.targetKey(name, rule.Name) b.arnIndexRemoveRule(region, tk, targets[tk]) delete(targets, tk) } @@ -792,7 +833,7 @@ func (b *InMemoryBackend) ListEventBuses( defer b.mu.RUnlock() all := make([]EventBus, 0) - for _, bus := range b.busesStore(region) { + for _, bus := range b.busesTable(region).All() { if namePrefix == "" || strings.HasPrefix(bus.Name, namePrefix) { all = append(all, *bus) } @@ -816,7 +857,7 @@ func (b *InMemoryBackend) DescribeEventBus(ctx context.Context, name string) (*E b.mu.RLock("DescribeEventBus") defer b.mu.RUnlock() - bus, exists := b.busesStore(region)[ebBusKey(name)] + bus, exists := b.busesTable(region).Get(ebBusKey(name)) if !exists { return nil, fmt.Errorf("%w: Event bus %s not found", ErrEventBusNotFound, name) } @@ -896,7 +937,7 @@ func (b *InMemoryBackend) PutRule(ctx context.Context, input PutRuleInput) (*Rul b.mu.Lock("PutRule") defer b.mu.Unlock() - if _, exists := b.busesStore(region)[busKey]; !exists { + if !b.busesTable(region).Has(busKey) { return nil, fmt.Errorf("%w: Event bus %s not found", ErrEventBusNotFound, busName) } @@ -905,14 +946,11 @@ func (b *InMemoryBackend) PutRule(ctx context.Context, input PutRuleInput) (*Rul state = ruleStateEnabled } - rules := b.rulesStore(region) - if rules[busKey] == nil { - rules[busKey] = make(map[string]*Rule) - } + ruleTable := b.ruleTableFor(region, busKey) // Enforce per-bus rule limit only for new rules (not updates). - if _, exists := rules[busKey][input.Name]; !exists { - if len(rules[busKey]) >= maxRulesPerBus { + if !ruleTable.Has(input.Name) { + if ruleTable.Len() >= maxRulesPerBus { return nil, fmt.Errorf( "%w: event bus %s has reached the maximum of %d rules", ErrResourceLimitExceeded, @@ -935,10 +973,10 @@ func (b *InMemoryBackend) PutRule(ctx context.Context, input PutRuleInput) (*Rul compiledPattern: compiled, } - if existing, exists := rules[busKey][input.Name]; exists { + if existing, exists := ruleTable.Get(input.Name); exists { b.removeRuleFromIndex(region, busKey, existing) } - rules[busKey][input.Name] = rule + ruleTable.Put(rule) b.addRuleToIndex(region, busKey, rule) return rule, nil @@ -961,13 +999,13 @@ func (b *InMemoryBackend) DeleteRule(ctx context.Context, name, eventBusName str return fmt.Errorf("%w: Rule %s not found", ErrRuleNotFound, name) } - rule, ruleExists := busRules[name] + rule, ruleExists := busRules.Get(name) if !ruleExists { return fmt.Errorf("%w: Rule %s not found", ErrRuleNotFound, name) } b.removeRuleFromIndex(region, busKey, rule) - delete(busRules, name) + busRules.Delete(name) // Also remove targets for this rule. targetKey := b.targetKey(eventBusName, name) b.arnIndexRemoveRule(region, targetKey, b.targetsStore(region)[targetKey]) @@ -993,8 +1031,12 @@ func (b *InMemoryBackend) ListRules(ctx context.Context, defer b.mu.RUnlock() busRules := b.rulesStore(region)[busKey] - all := make([]Rule, 0, len(busRules)) - for _, r := range busRules { + var busRulesAll []*Rule + if busRules != nil { + busRulesAll = busRules.All() + } + all := make([]Rule, 0, len(busRulesAll)) + for _, r := range busRulesAll { if namePrefix == "" || strings.HasPrefix(r.Name, namePrefix) { all = append(all, *r) } @@ -1024,7 +1066,7 @@ func (b *InMemoryBackend) DescribeRule(ctx context.Context, name, eventBusName s return nil, fmt.Errorf("%w: Rule %s not found", ErrRuleNotFound, name) } - rule, exists := busRules[name] + rule, exists := busRules.Get(name) if !exists { return nil, fmt.Errorf("%w: Rule %s not found", ErrRuleNotFound, name) } @@ -1060,7 +1102,7 @@ func (b *InMemoryBackend) setRuleState(ctx context.Context, name, eventBusName, return fmt.Errorf("%w: Rule %s not found", ErrRuleNotFound, name) } - rule, exists := busRules[name] + rule, exists := busRules.Get(name) if !exists { return fmt.Errorf("%w: Rule %s not found", ErrRuleNotFound, name) } @@ -1094,18 +1136,15 @@ func (b *InMemoryBackend) PutTargets(ctx context.Context, return nil, fmt.Errorf("%w: Rule %s not found", ErrRuleNotFound, ruleName) } - if _, ruleExists := busRules[ruleName]; !ruleExists { + if !busRules.Has(ruleName) { return nil, fmt.Errorf("%w: Rule %s not found", ErrRuleNotFound, ruleName) } - targetsStore := b.targetsStore(region) key := b.targetKey(eventBusName, ruleName) - if targetsStore[key] == nil { - targetsStore[key] = make(map[string]*Target) - } + targetTable := b.targetTableFor(region, key) // Reject if adding these targets would exceed the per-rule limit. - if len(targetsStore[key])+len(targets) > maxTargetsPerRule { + if targetTable.Len()+len(targets) > maxTargetsPerRule { return nil, fmt.Errorf( "%w: rule %s already has the maximum number of targets (%d)", ErrInvalidParameter, @@ -1119,7 +1158,7 @@ func (b *InMemoryBackend) PutTargets(ctx context.Context, if t.ID == "" { failed = append(failed, FailedEntry{ TargetID: t.ID, - ErrorCode: "InvalidParameter", + ErrorCode: putTargetsFailedEntryErrorCode, ErrorMessage: "Target Id is required", }) @@ -1129,19 +1168,28 @@ func (b *InMemoryBackend) PutTargets(ctx context.Context, if err := validateInputTransformer(t.InputTransformer); err != nil { failed = append(failed, FailedEntry{ TargetID: t.ID, - ErrorCode: "InvalidParameter", + ErrorCode: putTargetsFailedEntryErrorCode, ErrorMessage: err.Error(), }) continue } } + if err := validateTargetTypeParameters(&t); err != nil { + failed = append(failed, FailedEntry{ + TargetID: t.ID, + ErrorCode: putTargetsFailedEntryErrorCode, + ErrorMessage: err.Error(), + }) + + continue + } // Maintain ARN index: remove old entry if this target ID already exists with a different ARN. - if existingTarget, targetExists := targetsStore[key][t.ID]; targetExists && existingTarget.Arn != t.Arn { + if existingTarget, targetExists := targetTable.Get(t.ID); targetExists && existingTarget.Arn != t.Arn { b.arnIndexRemoveTarget(region, existingTarget.Arn, key) } cp := t - targetsStore[key][t.ID] = &cp + targetTable.Put(&cp) b.arnIndexAdd(region, t.Arn, key) } @@ -1167,7 +1215,11 @@ func (b *InMemoryBackend) RemoveTargets(ctx context.Context, var failed []FailedEntry for _, id := range ids { - t, exists := ruleTargets[id] + var t *Target + var exists bool + if ruleTargets != nil { + t, exists = ruleTargets.Get(id) + } if !exists { failed = append(failed, FailedEntry{ TargetID: id, @@ -1178,7 +1230,7 @@ func (b *InMemoryBackend) RemoveTargets(ctx context.Context, continue } b.arnIndexRemoveTarget(region, t.Arn, key) - delete(ruleTargets, id) + ruleTargets.Delete(id) } return failed, nil @@ -1201,8 +1253,12 @@ func (b *InMemoryBackend) ListTargetsByRule(ctx context.Context, key := b.targetKey(eventBusName, ruleName) ruleTargets := b.targetsStore(region)[key] - all := make([]Target, 0, len(ruleTargets)) - for _, t := range ruleTargets { + var ruleTargetsAll []*Target + if ruleTargets != nil { + ruleTargetsAll = ruleTargets.All() + } + all := make([]Target, 0, len(ruleTargetsAll)) + for _, t := range ruleTargetsAll { all = append(all, *t) } @@ -1213,6 +1269,51 @@ func (b *InMemoryBackend) ListTargetsByRule(ctx context.Context, return page, outToken, nil } +// maxPutEventsEntries is AWS's per-request cap on PutEvents/PutPartnerEvents +// entries (Entries: Array Members Minimum 1, Maximum 10). +const maxPutEventsEntries = 10 + +// anyPutEventsEntryComplete reports whether at least one entry in the batch +// carries all three of Source, DetailType, and Detail. AWS requires these on +// every entry to deliver it; an entry missing one fails individually, but if +// NONE of the entries have all three, AWS fails the entire request instead +// of returning an all-failed per-entry result (see PutEventsRequestEntry.Detail +// doc in aws-sdk-go-v2/service/eventbridge/types). +func anyPutEventsEntryComplete(entries []EventEntry) bool { + for _, e := range entries { + if e.Source != "" && e.DetailType != "" && e.Detail != "" { + return true + } + } + + return false +} + +// putEventsInvalidArgumentErrorCode is the EventResultEntry.ErrorCode AWS +// uses for a PutEvents entry missing a required field (Source, DetailType, +// or Detail). +const putEventsInvalidArgumentErrorCode = "InvalidArgument" + +// validatePutEventsEntry checks the three fields AWS requires on every +// PutEvents entry (Source, DetailType, Detail). It returns +// (errorCode, errorMessage, ok=false) with the AWS "InvalidArgument" error +// code/message for the first missing field, or ("", "", true) when valid. +func validatePutEventsEntry(e EventEntry) (string, string, bool) { + switch { + case e.Source == "": + return putEventsInvalidArgumentErrorCode, + "Parameter Source is not valid. Reason: Source is a required argument.", false + case e.DetailType == "": + return putEventsInvalidArgumentErrorCode, + "Parameter DetailType is not valid. Reason: DetailType is a required argument.", false + case e.Detail == "": + return putEventsInvalidArgumentErrorCode, + "Parameter Detail is not valid. Reason: Detail is a required argument.", false + default: + return "", "", true + } +} + // PutEvents records events in the event log and returns result entries. // // AWS EventBridge constrains PutEvents requests to 256 KiB of total entry @@ -1220,7 +1321,31 @@ func (b *InMemoryBackend) ListTargetsByRule(ctx context.Context, // entry). Entries that, combined with what's been accepted so far, would // exceed the cap are rejected individually with the AWS error code // `EventSizeLimitExceeded`. The remaining entries continue to be accepted. -func (b *InMemoryBackend) PutEvents(ctx context.Context, entries []EventEntry) []EventResultEntry { +// +// AWS also requires between 1 and 10 entries per request (a whole-request +// ValidationException-equivalent otherwise) and requires Source, DetailType, +// and Detail on every entry (a per-entry InvalidArgument failure, or a +// whole-request failure if no entry in the batch has all three). +func (b *InMemoryBackend) PutEvents(ctx context.Context, entries []EventEntry) ([]EventResultEntry, error) { + if len(entries) == 0 { + return nil, fmt.Errorf("%w: at least 1 entry is required", ErrInvalidParameter) + } + + if len(entries) > maxPutEventsEntries { + return nil, fmt.Errorf( + "%w: entries must not have more than %d items", + ErrInvalidParameter, + maxPutEventsEntries, + ) + } + + if !anyPutEventsEntryComplete(entries) { + return nil, fmt.Errorf( + "%w: at least one entry must include Source, DetailType, and Detail", + ErrInvalidParameter, + ) + } + const maxBatchBytes = 256 * 1024 region := getRegionFromContext(ctx, b.region) @@ -1231,6 +1356,12 @@ func (b *InMemoryBackend) PutEvents(ctx context.Context, entries []EventEntry) [ accepted := make([]EventEntry, 0, len(entries)) totalBytes := 0 for _, entry := range entries { + if errCode, msg, ok := validatePutEventsEntry(entry); !ok { + results = append(results, EventResultEntry{ErrorCode: errCode, ErrorMessage: msg}) + + continue + } + entryBytes := putEventsEntryBytes(entry) if totalBytes+entryBytes > maxBatchBytes { results = append(results, EventResultEntry{ @@ -1295,7 +1426,7 @@ func (b *InMemoryBackend) PutEvents(ctx context.Context, entries []EventEntry) [ }) } - return results + return results, nil } // GetEventLog returns a copy of the current event log. @@ -1371,22 +1502,33 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.buses = make(map[string]map[string]*EventBus) - b.rules = make(map[string]map[string]map[string]*Rule) - b.targets = make(map[string]map[string]map[string]*Target) + // b.registry is recreated from scratch rather than reused: every + // currently-registered *store.Table[V] is about to be orphaned anyway + // because every map field below is also reallocated to a brand-new + // map[string]*store.Table[V] -- the old *store.Table[V] instances become + // garbage, but that's fine, each will be recreated (and re-registered) the + // next time its region/parent is touched (store.Register panics on a + // duplicate name, so reusing the old registry here would panic on that + // re-registration). Mirrors the services/ssm conversion. + b.registry = store.NewRegistry() + b.auxRegistry = store.NewRegistry() + + b.buses = make(map[string]*store.Table[EventBus]) + b.rules = make(map[string]map[string]*store.Table[Rule]) + b.targets = make(map[string]map[string]*store.Table[Target]) b.eventLog = nil - b.eventSources = make(map[string]map[string]*EventSource) - b.replays = make(map[string]map[string]*Replay) - b.apiDestinations = make(map[string]map[string]*APIDestination) - b.archives = make(map[string]map[string]*Archive) + b.eventSources = make(map[string]*store.Table[EventSource]) + b.replays = make(map[string]*store.Table[Replay]) + b.apiDestinations = make(map[string]*store.Table[APIDestination]) + b.archives = make(map[string]*store.Table[Archive]) b.archivedEvents = make(map[string]map[string][]EventEntry) - b.connections = make(map[string]map[string]*Connection) - b.endpoints = make(map[string]map[string]*Endpoint) - b.partnerSources = make(map[string]map[string]*PartnerEventSource) + b.connections = make(map[string]*store.Table[Connection]) + b.endpoints = make(map[string]*store.Table[Endpoint]) + b.partnerSources = make(map[string]*store.Table[PartnerEventSource]) b.busePolicies = make(map[string]map[string]*EventBusPolicy) - b.pipes = make(map[string]*Pipe) - b.registries = make(map[string]*SchemaRegistry) - b.schemas = make(map[string]map[string]*Schema) + b.pipes = nil + b.registries = nil + b.schemas = make(map[string]*store.Table[Schema]) b.schemaVersions = make(map[string][]*SchemaVersion) b.codeBindings = make(map[string]*CodeBinding) b.ruleIndex = make(map[string]map[string]map[ruleIndexKey]map[string]*Rule) @@ -1395,11 +1537,11 @@ func (b *InMemoryBackend) Reset() { b.apiDestLimiters = sync.Map{} // Re-create the default event bus so it is always available after reset. - b.busesStore(b.region)[defaultEventBusName] = &EventBus{ + b.busesTable(b.region).Put(&EventBus{ Name: defaultEventBusName, Arn: b.busARN(b.region, defaultEventBusName), CreatedTime: time.Now(), - } + }) } // ActivateEventSource activates a partner event source. @@ -1413,7 +1555,7 @@ func (b *InMemoryBackend) ActivateEventSource(ctx context.Context, name string) b.mu.Lock("ActivateEventSource") defer b.mu.Unlock() - src, exists := b.eventSourcesStore(region)[name] + src, exists := b.eventSourcesTable(region).Get(name) if !exists { return fmt.Errorf("%w: event source %s not found", ErrNotFound, name) } @@ -1434,7 +1576,7 @@ func (b *InMemoryBackend) DeactivateEventSource(ctx context.Context, name string b.mu.Lock("DeactivateEventSource") defer b.mu.Unlock() - src, exists := b.eventSourcesStore(region)[name] + src, exists := b.eventSourcesTable(region).Get(name) if !exists { return fmt.Errorf("%w: event source %s not found", ErrNotFound, name) } @@ -1457,7 +1599,7 @@ func (b *InMemoryBackend) CreatePartnerEventSource(ctx context.Context, b.mu.Lock("CreatePartnerEventSource") defer b.mu.Unlock() - if _, exists := b.partnerSourcesStore(region)[name]; exists { + if b.partnerSourcesTable(region).Has(name) { return nil, fmt.Errorf("%w: partner event source %s already exists", ErrAlreadyExists, name) } @@ -1466,7 +1608,7 @@ func (b *InMemoryBackend) CreatePartnerEventSource(ctx context.Context, Name: name, Account: account, } - b.partnerSourcesStore(region)[name] = src + b.partnerSourcesTable(region).Put(src) // Mirror as a PENDING EventSource in the customer account — matches AWS // behaviour where creating a partner source causes it to appear in the @@ -1479,7 +1621,7 @@ func (b *InMemoryBackend) CreatePartnerEventSource(ctx context.Context, Name: name, State: "PENDING", } - b.eventSourcesStore(region)[name] = esrc + b.eventSourcesTable(region).Put(esrc) cp := *src @@ -1497,7 +1639,7 @@ func (b *InMemoryBackend) CancelReplay(ctx context.Context, replayName string) ( b.mu.Lock("CancelReplay") defer b.mu.Unlock() - replay, exists := b.replaysStore(region)[replayName] + replay, exists := b.replaysTable(region).Get(replayName) if !exists { return nil, fmt.Errorf("%w: replay %s not found", ErrNotFound, replayName) } @@ -1550,7 +1692,7 @@ func (b *InMemoryBackend) CreateAPIDestination(ctx context.Context, b.mu.Lock("CreateAPIDestination") defer b.mu.Unlock() - if _, exists := b.apiDestinationsStore(region)[input.Name]; exists { + if b.apiDestinationsTable(region).Has(input.Name) { return nil, fmt.Errorf( "%w: API destination %s already exists", ErrAlreadyExists, @@ -1571,7 +1713,7 @@ func (b *InMemoryBackend) CreateAPIDestination(ctx context.Context, LastModifiedTime: now, Name: input.Name, } - b.apiDestinationsStore(region)[input.Name] = dst + b.apiDestinationsTable(region).Put(dst) cp := *dst @@ -1608,7 +1750,7 @@ func (b *InMemoryBackend) CreateArchive(ctx context.Context, input CreateArchive b.mu.Lock("CreateArchive") defer b.mu.Unlock() - if _, exists := b.archivesStore(region)[input.ArchiveName]; exists { + if b.archivesTable(region).Has(input.ArchiveName) { return nil, fmt.Errorf("%w: archive %s already exists", ErrAlreadyExists, input.ArchiveName) } @@ -1622,7 +1764,7 @@ func (b *InMemoryBackend) CreateArchive(ctx context.Context, input CreateArchive RetentionDays: input.RetentionDays, State: "ENABLED", } - b.archivesStore(region)[input.ArchiveName] = archive + b.archivesTable(region).Put(archive) cp := *archive @@ -1651,7 +1793,7 @@ func (b *InMemoryBackend) CreateConnection(ctx context.Context, input CreateConn b.mu.Lock("CreateConnection") defer b.mu.Unlock() - if _, exists := b.connectionsStore(region)[input.Name]; exists { + if b.connectionsTable(region).Has(input.Name) { return nil, fmt.Errorf("%w: connection %s already exists", ErrAlreadyExists, input.Name) } @@ -1667,7 +1809,7 @@ func (b *InMemoryBackend) CreateConnection(ctx context.Context, input CreateConn LastModifiedTime: now, Name: input.Name, } - b.connectionsStore(region)[input.Name] = conn + b.connectionsTable(region).Put(conn) cp := *conn @@ -1685,7 +1827,7 @@ func (b *InMemoryBackend) CreateEndpoint(ctx context.Context, input CreateEndpoi b.mu.Lock("CreateEndpoint") defer b.mu.Unlock() - if _, exists := b.endpointsStore(region)[input.Name]; exists { + if b.endpointsTable(region).Has(input.Name) { return nil, fmt.Errorf("%w: endpoint %s already exists", ErrAlreadyExists, input.Name) } @@ -1709,7 +1851,7 @@ func (b *InMemoryBackend) CreateEndpoint(ctx context.Context, input CreateEndpoi RoutingConfig: input.RoutingConfig, State: stateActive, } - b.endpointsStore(region)[input.Name] = ep + b.endpointsTable(region).Put(ep) cp := *ep @@ -1727,7 +1869,7 @@ func (b *InMemoryBackend) DeauthorizeConnection(ctx context.Context, name string b.mu.Lock("DeauthorizeConnection") defer b.mu.Unlock() - conn, exists := b.connectionsStore(region)[name] + conn, exists := b.connectionsTable(region).Get(name) if !exists { return nil, fmt.Errorf("%w: connection %s not found", ErrNotFound, name) } @@ -1750,12 +1892,12 @@ func (b *InMemoryBackend) DeleteAPIDestination(ctx context.Context, name string) b.mu.Lock("DeleteAPIDestination") defer b.mu.Unlock() - store := b.apiDestinationsStore(region) - if _, exists := store[name]; !exists { + store := b.apiDestinationsTable(region) + if !store.Has(name) { return fmt.Errorf("%w: API destination %s not found", ErrNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -1771,12 +1913,12 @@ func (b *InMemoryBackend) DeleteArchive(ctx context.Context, name string) error b.mu.Lock("DeleteArchive") defer b.mu.Unlock() - store := b.archivesStore(region) - if _, exists := store[name]; !exists { + store := b.archivesTable(region) + if !store.Has(name) { return fmt.Errorf("%w: archive %s not found", ErrNotFound, name) } - delete(store, name) + store.Delete(name) delete(b.archivedEventsStore(region), name) return nil @@ -1793,7 +1935,7 @@ func (b *InMemoryBackend) DescribeArchive(ctx context.Context, name string) (*Ar b.mu.RLock("DescribeArchive") defer b.mu.RUnlock() - archive, exists := b.archivesStore(region)[name] + archive, exists := b.archivesTable(region).Get(name) if !exists { return nil, fmt.Errorf("%w: archive %s not found", ErrNotFound, name) } @@ -1810,9 +1952,9 @@ func (b *InMemoryBackend) ListArchives(ctx context.Context, namePrefix, nextToke b.mu.RLock("ListArchives") defer b.mu.RUnlock() - store := b.archivesStore(region) - all := make([]Archive, 0, len(store)) - for _, a := range store { + store := b.archivesTable(region) + all := make([]Archive, 0, store.Len()) + for _, a := range store.All() { if namePrefix == "" || strings.HasPrefix(a.ArchiveName, namePrefix) { all = append(all, *a) } @@ -1836,7 +1978,7 @@ func (b *InMemoryBackend) UpdateArchive(ctx context.Context, input UpdateArchive b.mu.Lock("UpdateArchive") defer b.mu.Unlock() - archive, exists := b.archivesStore(region)[input.ArchiveName] + archive, exists := b.archivesTable(region).Get(input.ArchiveName) if !exists { return nil, fmt.Errorf("%w: archive %s not found", ErrNotFound, input.ArchiveName) } @@ -1867,12 +2009,12 @@ func (b *InMemoryBackend) DeleteConnection(ctx context.Context, name string) err b.mu.Lock("DeleteConnection") defer b.mu.Unlock() - store := b.connectionsStore(region) - if _, exists := store[name]; !exists { + store := b.connectionsTable(region) + if !store.Has(name) { return fmt.Errorf("%w: connection %s not found", ErrNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -1888,7 +2030,7 @@ func (b *InMemoryBackend) DescribeConnection(ctx context.Context, name string) ( b.mu.RLock("DescribeConnection") defer b.mu.RUnlock() - conn, exists := b.connectionsStore(region)[name] + conn, exists := b.connectionsTable(region).Get(name) if !exists { return nil, fmt.Errorf("%w: connection %s not found", ErrNotFound, name) } @@ -1907,9 +2049,9 @@ func (b *InMemoryBackend) ListConnections(ctx context.Context, b.mu.RLock("ListConnections") defer b.mu.RUnlock() - store := b.connectionsStore(region) - all := make([]Connection, 0, len(store)) - for _, c := range store { + store := b.connectionsTable(region) + all := make([]Connection, 0, store.Len()) + for _, c := range store.All() { if namePrefix == "" || strings.HasPrefix(c.Name, namePrefix) { all = append(all, *c) } @@ -1933,7 +2075,7 @@ func (b *InMemoryBackend) UpdateConnection(ctx context.Context, input UpdateConn b.mu.Lock("UpdateConnection") defer b.mu.Unlock() - conn, exists := b.connectionsStore(region)[input.Name] + conn, exists := b.connectionsTable(region).Get(input.Name) if !exists { return nil, fmt.Errorf("%w: connection %s not found", ErrNotFound, input.Name) } @@ -1966,12 +2108,12 @@ func (b *InMemoryBackend) DeleteEndpoint(ctx context.Context, name string) error b.mu.Lock("DeleteEndpoint") defer b.mu.Unlock() - store := b.endpointsStore(region) - if _, exists := store[name]; !exists { + store := b.endpointsTable(region) + if !store.Has(name) { return fmt.Errorf("%w: endpoint %s not found", ErrNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -1987,7 +2129,7 @@ func (b *InMemoryBackend) DescribeEndpoint(ctx context.Context, name string) (*E b.mu.RLock("DescribeEndpoint") defer b.mu.RUnlock() - ep, exists := b.endpointsStore(region)[name] + ep, exists := b.endpointsTable(region).Get(name) if !exists { return nil, fmt.Errorf("%w: endpoint %s not found", ErrNotFound, name) } @@ -2004,9 +2146,9 @@ func (b *InMemoryBackend) ListEndpoints(ctx context.Context, namePrefix, nextTok b.mu.RLock("ListEndpoints") defer b.mu.RUnlock() - store := b.endpointsStore(region) - all := make([]Endpoint, 0, len(store)) - for _, ep := range store { + store := b.endpointsTable(region) + all := make([]Endpoint, 0, store.Len()) + for _, ep := range store.All() { if namePrefix == "" || strings.HasPrefix(ep.Name, namePrefix) { all = append(all, *ep) } @@ -2030,7 +2172,7 @@ func (b *InMemoryBackend) UpdateEndpoint(ctx context.Context, input UpdateEndpoi b.mu.Lock("UpdateEndpoint") defer b.mu.Unlock() - ep, exists := b.endpointsStore(region)[input.Name] + ep, exists := b.endpointsTable(region).Get(input.Name) if !exists { return nil, fmt.Errorf("%w: endpoint %s not found", ErrNotFound, input.Name) } @@ -2068,7 +2210,7 @@ func (b *InMemoryBackend) DescribeAPIDestination(ctx context.Context, name strin b.mu.RLock("DescribeAPIDestination") defer b.mu.RUnlock() - dst, exists := b.apiDestinationsStore(region)[name] + dst, exists := b.apiDestinationsTable(region).Get(name) if !exists { return nil, fmt.Errorf("%w: API destination %s not found", ErrNotFound, name) } @@ -2087,9 +2229,9 @@ func (b *InMemoryBackend) ListAPIDestinations(ctx context.Context, b.mu.RLock("ListAPIDestinations") defer b.mu.RUnlock() - store := b.apiDestinationsStore(region) - all := make([]APIDestination, 0, len(store)) - for _, d := range store { + store := b.apiDestinationsTable(region) + all := make([]APIDestination, 0, store.Len()) + for _, d := range store.All() { if namePrefix == "" || strings.HasPrefix(d.Name, namePrefix) { all = append(all, *d) } @@ -2115,7 +2257,7 @@ func (b *InMemoryBackend) UpdateAPIDestination(ctx context.Context, b.mu.Lock("UpdateAPIDestination") defer b.mu.Unlock() - dst, exists := b.apiDestinationsStore(region)[input.Name] + dst, exists := b.apiDestinationsTable(region).Get(input.Name) if !exists { return nil, fmt.Errorf("%w: API destination %s not found", ErrNotFound, input.Name) } @@ -2153,7 +2295,7 @@ func (b *InMemoryBackend) DescribeEventSource(ctx context.Context, name string) b.mu.RLock("DescribeEventSource") defer b.mu.RUnlock() - src, exists := b.eventSourcesStore(region)[name] + src, exists := b.eventSourcesTable(region).Get(name) if !exists { return nil, fmt.Errorf("%w: event source %s not found", ErrNotFound, name) } @@ -2172,9 +2314,9 @@ func (b *InMemoryBackend) ListEventSources(ctx context.Context, b.mu.RLock("ListEventSources") defer b.mu.RUnlock() - store := b.eventSourcesStore(region) - all := make([]EventSource, 0, len(store)) - for _, s := range store { + store := b.eventSourcesTable(region) + all := make([]EventSource, 0, store.Len()) + for _, s := range store.All() { if namePrefix == "" || strings.HasPrefix(s.Name, namePrefix) { all = append(all, *s) } @@ -2198,7 +2340,7 @@ func (b *InMemoryBackend) DescribePartnerEventSource(ctx context.Context, name s b.mu.RLock("DescribePartnerEventSource") defer b.mu.RUnlock() - src, exists := b.partnerSourcesStore(region)[name] + src, exists := b.partnerSourcesTable(region).Get(name) if !exists { return nil, fmt.Errorf("%w: partner event source %s not found", ErrNotFound, name) } @@ -2219,12 +2361,12 @@ func (b *InMemoryBackend) DeletePartnerEventSource(ctx context.Context, name str b.mu.Lock("DeletePartnerEventSource") defer b.mu.Unlock() - store := b.partnerSourcesStore(region) - if _, exists := store[name]; !exists { + store := b.partnerSourcesTable(region) + if !store.Has(name) { return fmt.Errorf("%w: partner event source %s not found", ErrNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -2238,9 +2380,9 @@ func (b *InMemoryBackend) ListPartnerEventSources(ctx context.Context, b.mu.RLock("ListPartnerEventSources") defer b.mu.RUnlock() - store := b.partnerSourcesStore(region) - all := make([]PartnerEventSource, 0, len(store)) - for _, s := range store { + store := b.partnerSourcesTable(region) + all := make([]PartnerEventSource, 0, store.Len()) + for _, s := range store.All() { if namePrefix == "" || strings.HasPrefix(s.Name, namePrefix) { all = append(all, *s) } @@ -2254,7 +2396,7 @@ func (b *InMemoryBackend) ListPartnerEventSources(ctx context.Context, } // PutPartnerEvents records partner events (same as PutEvents but intended for partner sources). -func (b *InMemoryBackend) PutPartnerEvents(ctx context.Context, entries []EventEntry) []EventResultEntry { +func (b *InMemoryBackend) PutPartnerEvents(ctx context.Context, entries []EventEntry) ([]EventResultEntry, error) { return b.PutEvents(ctx, entries) } @@ -2269,7 +2411,7 @@ func (b *InMemoryBackend) DescribeReplay(ctx context.Context, name string) (*Rep b.mu.RLock("DescribeReplay") defer b.mu.RUnlock() - replay, exists := b.replaysStore(region)[name] + replay, exists := b.replaysTable(region).Get(name) if !exists { return nil, fmt.Errorf("%w: replay %s not found", ErrNotFound, name) } @@ -2286,9 +2428,9 @@ func (b *InMemoryBackend) ListReplays(ctx context.Context, namePrefix, nextToken b.mu.RLock("ListReplays") defer b.mu.RUnlock() - store := b.replaysStore(region) - all := make([]Replay, 0, len(store)) - for _, r := range store { + store := b.replaysTable(region) + all := make([]Replay, 0, store.Len()) + for _, r := range store.All() { if namePrefix == "" || strings.HasPrefix(r.ReplayName, namePrefix) { all = append(all, *r) } @@ -2323,8 +2465,8 @@ func (b *InMemoryBackend) StartReplay(ctx context.Context, input StartReplayInpu b.mu.Lock("StartReplay") - replays := b.replaysStore(region) - if _, exists := replays[input.ReplayName]; exists { + replays := b.replaysTable(region) + if replays.Has(input.ReplayName) { b.mu.Unlock() return nil, fmt.Errorf("%w: replay %s already exists", ErrAlreadyExists, input.ReplayName) @@ -2333,7 +2475,7 @@ func (b *InMemoryBackend) StartReplay(ctx context.Context, input StartReplayInpu // Validate destination ARN points to a known event bus. if input.Destination != nil && input.Destination.Arn != "" { found := false - for _, bus := range b.busesStore(region) { + for _, bus := range b.busesTable(region).All() { if bus.Arn == input.Destination.Arn { found = true @@ -2354,9 +2496,9 @@ func (b *InMemoryBackend) StartReplay(ctx context.Context, input StartReplayInpu // Find the archive by ARN (EventSourceArn points to an archive ARN). var archiveName string var archivePattern string - for name, archive := range b.archivesStore(region) { + for _, archive := range b.archivesTable(region).All() { if archive.ArchiveArn == input.EventSourceArn { - archiveName = name + archiveName = archive.ArchiveName archivePattern = archive.EventPattern break @@ -2373,7 +2515,7 @@ func (b *InMemoryBackend) StartReplay(ctx context.Context, input StartReplayInpu State: replayStateStarting, StateReason: input.Description, } - replays[input.ReplayName] = replay + replays.Put(replay) // Collect archived events to replay filtered by time window and event pattern. eventsToReplay := b.filterArchivedEvents( @@ -2463,7 +2605,7 @@ func (b *InMemoryBackend) scheduleReplayWorker( } b.mu.Lock("StartReplay-complete") - if r, ok := b.replaysStore(region)[replayName]; ok && r.State == replayStateStarting { + if r, ok := b.replaysTable(region).Get(replayName); ok && r.State == replayStateStarting { r.State = "COMPLETED" r.ReplayEndTime = time.Now() } @@ -2538,7 +2680,7 @@ func (b *InMemoryBackend) UpdateEventBus(ctx context.Context, input UpdateEventB b.mu.Lock("UpdateEventBus") defer b.mu.Unlock() - bus, exists := b.busesStore(region)[busKey] + bus, exists := b.busesTable(region).Get(busKey) if !exists { return nil, fmt.Errorf("%w: event bus %s not found", ErrEventBusNotFound, busName) } @@ -2563,7 +2705,7 @@ func (b *InMemoryBackend) PutPermission(ctx context.Context, input PutPermission b.mu.Lock("PutPermission") defer b.mu.Unlock() - if _, exists := b.busesStore(region)[busKey]; !exists { + if _, exists := b.busesTable(region).Get(busKey); !exists { return fmt.Errorf("%w: event bus %s not found", ErrEventBusNotFound, busName) } @@ -2616,7 +2758,7 @@ func (b *InMemoryBackend) RemovePermission(ctx context.Context, input RemovePerm b.mu.Lock("RemovePermission") defer b.mu.Unlock() - if _, exists := b.busesStore(region)[busKey]; !exists { + if _, exists := b.busesTable(region).Get(busKey); !exists { return fmt.Errorf("%w: event bus %s not found", ErrEventBusNotFound, busName) } @@ -2649,7 +2791,7 @@ func (b *InMemoryBackend) GetEventBusPolicy(ctx context.Context, eventBusName st b.mu.RLock("GetEventBusPolicy") defer b.mu.RUnlock() - if _, exists := b.busesStore(region)[busKey]; !exists { + if _, exists := b.busesTable(region).Get(busKey); !exists { return "", fmt.Errorf("%w: event bus %s not found", ErrEventBusNotFound, busName) } @@ -2683,7 +2825,7 @@ func (b *InMemoryBackend) PutEventBusPolicy(ctx context.Context, input PutEventB b.mu.Lock("PutEventBusPolicy") defer b.mu.Unlock() - if _, exists := b.busesStore(region)[busKey]; !exists { + if _, exists := b.busesTable(region).Get(busKey); !exists { return fmt.Errorf("%w: event bus %s not found", ErrEventBusNotFound, busName) } @@ -2740,7 +2882,7 @@ func (b *InMemoryBackend) CreatePipe( b.mu.Lock("CreatePipe") defer b.mu.Unlock() - if _, exists := b.pipes[input.Name]; exists { + if b.pipesTable().Has(input.Name) { return nil, fmt.Errorf("%w: pipe %s already exists", ErrAlreadyExists, input.Name) } @@ -2758,7 +2900,7 @@ func (b *InMemoryBackend) CreatePipe( CreationTime: now, LastModifiedTime: now, } - b.pipes[input.Name] = pipe + b.pipesTable().Put(pipe) cp := *pipe // Transition CREATING → RUNNING immediately (in-process simulation). @@ -2776,13 +2918,13 @@ func (b *InMemoryBackend) DeletePipe(ctx context.Context, name string) error { / b.mu.Lock("DeletePipe") defer b.mu.Unlock() - pipe, exists := b.pipes[name] + pipe, exists := b.pipesTable().Get(name) if !exists { return fmt.Errorf("%w: pipe %s not found", ErrNotFound, name) } pipe.CurrentState = "DELETING" - delete(b.pipes, name) + b.pipesTable().Delete(name) return nil } @@ -2799,7 +2941,7 @@ func (b *InMemoryBackend) DescribePipe( b.mu.RLock("DescribePipe") defer b.mu.RUnlock() - pipe, exists := b.pipes[name] + pipe, exists := b.pipesTable().Get(name) if !exists { return nil, fmt.Errorf("%w: pipe %s not found", ErrNotFound, name) } @@ -2817,8 +2959,8 @@ func (b *InMemoryBackend) ListPipes( b.mu.RLock("ListPipes") defer b.mu.RUnlock() - all := make([]Pipe, 0, len(b.pipes)) - for _, p := range b.pipes { + all := make([]Pipe, 0, b.pipesTable().Len()) + for _, p := range b.pipesTable().All() { if namePrefix == "" || strings.HasPrefix(p.Name, namePrefix) { all = append(all, *p) } @@ -2843,7 +2985,7 @@ func (b *InMemoryBackend) UpdatePipe( b.mu.Lock("UpdatePipe") defer b.mu.Unlock() - pipe, exists := b.pipes[input.Name] + pipe, exists := b.pipesTable().Get(input.Name) if !exists { return nil, fmt.Errorf("%w: pipe %s not found", ErrNotFound, input.Name) } @@ -2883,7 +3025,7 @@ func (b *InMemoryBackend) captureEventInArchives(region string, entry EventEntry busARN := b.busARN(region, busName) envelope := buildEventEnvelope(entry) archivedEvents := b.archivedEventsStore(region) - for _, archive := range b.archivesStore(region) { + for _, archive := range b.archivesTable(region).All() { if archive.EventSourceArn != busARN { continue } @@ -2907,7 +3049,7 @@ func (b *InMemoryBackend) AddEventSourceInternal(src *EventSource) { defer b.mu.Unlock() cp := *src - b.eventSourcesStore(b.region)[src.Name] = &cp + b.eventSourcesTable(b.region).Put(&cp) } // AddReplayInternal adds a replay directly for testing. @@ -2920,7 +3062,7 @@ func (b *InMemoryBackend) AddReplayInternal(replay *Replay) { } cp := *replay - b.replaysStore(b.region)[replay.ReplayName] = &cp + b.replaysTable(b.region).Put(&cp) } // AddAPIDestinationInternal adds an API destination directly for testing. @@ -2929,7 +3071,7 @@ func (b *InMemoryBackend) AddAPIDestinationInternal(dst *APIDestination) { defer b.mu.Unlock() cp := *dst - b.apiDestinationsStore(b.region)[dst.Name] = &cp + b.apiDestinationsTable(b.region).Put(&cp) } // AddArchiveInternal adds an archive directly for testing. @@ -2938,7 +3080,7 @@ func (b *InMemoryBackend) AddArchiveInternal(archive *Archive) { defer b.mu.Unlock() cp := *archive - b.archivesStore(b.region)[archive.ArchiveName] = &cp + b.archivesTable(b.region).Put(&cp) } // AddConnectionInternal adds a connection directly for testing. @@ -2947,7 +3089,7 @@ func (b *InMemoryBackend) AddConnectionInternal(conn *Connection) { defer b.mu.Unlock() cp := *conn - b.connectionsStore(b.region)[conn.Name] = &cp + b.connectionsTable(b.region).Put(&cp) } // AddEndpointInternal adds an endpoint directly for testing. @@ -2956,7 +3098,7 @@ func (b *InMemoryBackend) AddEndpointInternal(ep *Endpoint) { defer b.mu.Unlock() cp := *ep - b.endpointsStore(b.region)[ep.Name] = &cp + b.endpointsTable(b.region).Put(&cp) } // AddPartnerSourceInternal adds a partner event source directly for testing. @@ -2965,7 +3107,7 @@ func (b *InMemoryBackend) AddPartnerSourceInternal(src *PartnerEventSource) { defer b.mu.Unlock() cp := *src - b.partnerSourcesStore(b.region)[src.Name] = &cp + b.partnerSourcesTable(b.region).Put(&cp) } // isValidHTTPMethod reports whether method is a supported API Destination HTTP method. @@ -3195,7 +3337,11 @@ func (b *InMemoryBackend) ResolveAPIDestination(destARN string) (*ResolvedAPIDes b.mu.RLock("ResolveAPIDestination") defer b.mu.RUnlock() - dest, ok := b.apiDestinations[region][name] + destTable := b.apiDestinations[region] + if destTable == nil { + return nil, false + } + dest, ok := destTable.Get(name) if !ok { return nil, false } @@ -3211,8 +3357,10 @@ func (b *InMemoryBackend) ResolveAPIDestination(destARN string) (*ResolvedAPIDes connRegion = region } connName := arnResourceName(dest.ConnectionArn, "connection") - if conn, connOK := b.connections[connRegion][connName]; connOK { - applyConnectionAuthToResolved(resolved, conn) + if connTable := b.connections[connRegion]; connTable != nil { + if conn, connOK := connTable.Get(connName); connOK { + applyConnectionAuthToResolved(resolved, conn) + } } return resolved, true @@ -3395,7 +3543,7 @@ func (b *InMemoryBackend) CreateRegistry( b.mu.Lock("CreateRegistry") defer b.mu.Unlock() - if _, exists := b.registries[input.RegistryName]; exists { + if b.registriesTable().Has(input.RegistryName) { return nil, fmt.Errorf( "%w: registry %s already exists", ErrAlreadyExists, @@ -3409,7 +3557,7 @@ func (b *InMemoryBackend) CreateRegistry( Description: input.Description, Tags: input.Tags, } - b.registries[input.RegistryName] = reg + b.registriesTable().Put(reg) cp := *reg @@ -3436,11 +3584,11 @@ func (b *InMemoryBackend) DeleteRegistry( b.mu.Lock("DeleteRegistry") defer b.mu.Unlock() - if _, exists := b.registries[registryName]; !exists { + if !b.registriesTable().Has(registryName) { return fmt.Errorf("%w: registry %s not found", ErrNotFound, registryName) } - delete(b.registries, registryName) + b.registriesTable().Delete(registryName) delete(b.schemas, registryName) // Remove all version and code binding records for this registry's schemas. @@ -3471,7 +3619,7 @@ func (b *InMemoryBackend) DescribeRegistry( b.mu.RLock("DescribeRegistry") defer b.mu.RUnlock() - reg, exists := b.registries[registryName] + reg, exists := b.registriesTable().Get(registryName) if !exists { return nil, fmt.Errorf("%w: registry %s not found", ErrNotFound, registryName) } @@ -3488,8 +3636,8 @@ func (b *InMemoryBackend) ListRegistries(ctx context.Context, //nolint:revive // b.mu.RLock("ListRegistries") defer b.mu.RUnlock() - all := make([]SchemaRegistry, 0, len(b.registries)) - for _, reg := range b.registries { + all := make([]SchemaRegistry, 0, b.registriesTable().Len()) + for _, reg := range b.registriesTable().All() { if namePrefix == "" || strings.HasPrefix(reg.RegistryName, namePrefix) { all = append(all, *reg) } @@ -3514,7 +3662,7 @@ func (b *InMemoryBackend) UpdateRegistry( b.mu.Lock("UpdateRegistry") defer b.mu.Unlock() - reg, exists := b.registries[input.RegistryName] + reg, exists := b.registriesTable().Get(input.RegistryName) if !exists { return nil, fmt.Errorf("%w: registry %s not found", ErrNotFound, input.RegistryName) } @@ -3560,15 +3708,13 @@ func (b *InMemoryBackend) CreateSchema( b.mu.Lock("CreateSchema") defer b.mu.Unlock() - if _, exists := b.registries[input.RegistryName]; !exists { + if !b.registriesTable().Has(input.RegistryName) { return nil, fmt.Errorf("%w: registry %s not found", ErrNotFound, input.RegistryName) } - if b.schemas[input.RegistryName] == nil { - b.schemas[input.RegistryName] = make(map[string]*Schema) - } + schemaTable := b.schemasTableFor(input.RegistryName) - if _, exists := b.schemas[input.RegistryName][input.SchemaName]; exists { + if schemaTable.Has(input.SchemaName) { return nil, fmt.Errorf( "%w: schema %s already exists in registry %s", ErrAlreadyExists, @@ -3590,7 +3736,7 @@ func (b *InMemoryBackend) CreateSchema( VersionCreatedDate: now, Tags: input.Tags, } - b.schemas[input.RegistryName][input.SchemaName] = schema + schemaTable.Put(schema) // Record version 1. versionKey := b.schemaVersionKey(input.RegistryName, input.SchemaName) @@ -3626,11 +3772,11 @@ func (b *InMemoryBackend) DeleteSchema( b.mu.Lock("DeleteSchema") defer b.mu.Unlock() - if _, exists := b.registries[registryName]; !exists { + if !b.registriesTable().Has(registryName) { return fmt.Errorf("%w: registry %s not found", ErrNotFound, registryName) } - if b.schemas[registryName] == nil || b.schemas[registryName][schemaName] == nil { + if _, ok := b.getSchema(registryName, schemaName); !ok { return fmt.Errorf( "%w: schema %s not found in registry %s", ErrNotFound, @@ -3639,7 +3785,9 @@ func (b *InMemoryBackend) DeleteSchema( ) } - delete(b.schemas[registryName], schemaName) + if t := b.schemas[registryName]; t != nil { + t.Delete(schemaName) + } versionKey := b.schemaVersionKey(registryName, schemaName) delete(b.schemaVersions, versionKey) @@ -3669,11 +3817,12 @@ func (b *InMemoryBackend) DescribeSchema(ctx context.Context, //nolint:revive // b.mu.RLock("DescribeSchema") defer b.mu.RUnlock() - if _, exists := b.registries[registryName]; !exists { + if !b.registriesTable().Has(registryName) { return nil, fmt.Errorf("%w: registry %s not found", ErrNotFound, registryName) } - if b.schemas[registryName] == nil || b.schemas[registryName][schemaName] == nil { + schema, ok := b.getSchema(registryName, schemaName) + if !ok { return nil, fmt.Errorf( "%w: schema %s not found in registry %s", ErrNotFound, @@ -3682,8 +3831,6 @@ func (b *InMemoryBackend) DescribeSchema(ctx context.Context, //nolint:revive // ) } - schema := b.schemas[registryName][schemaName] - // If a specific version is requested, fetch that version's content. if schemaVersion != "" && schemaVersion != schema.SchemaVersion { versionKey := b.schemaVersionKey(registryName, schemaName) @@ -3718,12 +3865,17 @@ func (b *InMemoryBackend) ListSchemas(ctx context.Context, //nolint:revive // ex b.mu.RLock("ListSchemas") defer b.mu.RUnlock() - if _, exists := b.registries[registryName]; !exists { + if !b.registriesTable().Has(registryName) { return nil, "", fmt.Errorf("%w: registry %s not found", ErrNotFound, registryName) } - all := make([]Schema, 0, len(b.schemas[registryName])) - for _, s := range b.schemas[registryName] { + schemaTable := b.schemas[registryName] + var schemaAll []*Schema + if schemaTable != nil { + schemaAll = schemaTable.All() + } + all := make([]Schema, 0, len(schemaAll)) + for _, s := range schemaAll { if namePrefix == "" || strings.HasPrefix(s.SchemaName, namePrefix) { all = append(all, *s) } @@ -3747,14 +3899,18 @@ func (b *InMemoryBackend) SearchSchemas(ctx context.Context, //nolint:revive // b.mu.RLock("SearchSchemas") defer b.mu.RUnlock() - if _, exists := b.registries[registryName]; !exists { + if !b.registriesTable().Has(registryName) { return nil, "", fmt.Errorf("%w: registry %s not found", ErrNotFound, registryName) } all := make([]Schema, 0) lower := strings.ToLower(keywords) - for _, s := range b.schemas[registryName] { + var schemaAll []*Schema + if schemaTable := b.schemas[registryName]; schemaTable != nil { + schemaAll = schemaTable.All() + } + for _, s := range schemaAll { if keywords == "" || strings.Contains(strings.ToLower(s.SchemaName), lower) || strings.Contains(strings.ToLower(s.Content), lower) { @@ -3785,12 +3941,12 @@ func (b *InMemoryBackend) UpdateSchema( b.mu.Lock("UpdateSchema") defer b.mu.Unlock() - if _, exists := b.registries[input.RegistryName]; !exists { + if !b.registriesTable().Has(input.RegistryName) { return nil, fmt.Errorf("%w: registry %s not found", ErrNotFound, input.RegistryName) } - if b.schemas[input.RegistryName] == nil || - b.schemas[input.RegistryName][input.SchemaName] == nil { + schema, ok := b.getSchema(input.RegistryName, input.SchemaName) + if !ok { return nil, fmt.Errorf( "%w: schema %s not found in registry %s", ErrNotFound, @@ -3799,8 +3955,6 @@ func (b *InMemoryBackend) UpdateSchema( ) } - schema := b.schemas[input.RegistryName][input.SchemaName] - now := time.Now() versionKey := b.schemaVersionKey(input.RegistryName, input.SchemaName) @@ -3855,11 +4009,11 @@ func (b *InMemoryBackend) ListSchemaVersions(ctx context.Context, //nolint:reviv b.mu.RLock("ListSchemaVersions") defer b.mu.RUnlock() - if _, exists := b.registries[registryName]; !exists { + if !b.registriesTable().Has(registryName) { return nil, "", fmt.Errorf("%w: registry %s not found", ErrNotFound, registryName) } - if b.schemas[registryName] == nil || b.schemas[registryName][schemaName] == nil { + if _, ok := b.getSchema(registryName, schemaName); !ok { return nil, "", fmt.Errorf( "%w: schema %s not found in registry %s", ErrNotFound, @@ -3982,11 +4136,7 @@ func (b *InMemoryBackend) DeleteSchemaVersion(ctx context.Context, //nolint:revi func (b *InMemoryBackend) maybeUpdateSchemaAfterVersionDelete( registryName, schemaName, schemaVersion, versionKey string, ) { - if b.schemas[registryName] == nil { - return - } - - schema, ok := b.schemas[registryName][schemaName] + schema, ok := b.getSchema(registryName, schemaName) if !ok || schema.SchemaVersion != schemaVersion { return } @@ -4044,12 +4194,12 @@ func (b *InMemoryBackend) PutCodeBinding( b.mu.Lock("PutCodeBinding") defer b.mu.Unlock() - if _, exists := b.registries[input.RegistryName]; !exists { + if !b.registriesTable().Has(input.RegistryName) { return nil, fmt.Errorf("%w: registry %s not found", ErrNotFound, input.RegistryName) } - if b.schemas[input.RegistryName] == nil || - b.schemas[input.RegistryName][input.SchemaName] == nil { + schema, ok := b.getSchema(input.RegistryName, input.SchemaName) + if !ok { return nil, fmt.Errorf( "%w: schema %s not found in registry %s", ErrNotFound, @@ -4058,8 +4208,6 @@ func (b *InMemoryBackend) PutCodeBinding( ) } - schema := b.schemas[input.RegistryName][input.SchemaName] - schemaVer := input.SchemaVersion if schemaVer == "" { schemaVer = schema.SchemaVersion diff --git a/services/eventbridge/backend_refinement2_test.go b/services/eventbridge/backend_refinement2_test.go index 375bfce2b..b7d4acd51 100644 --- a/services/eventbridge/backend_refinement2_test.go +++ b/services/eventbridge/backend_refinement2_test.go @@ -300,6 +300,232 @@ func TestRefinement2_PutTargets_EnforcesLimit(t *testing.T) { } } +// --------------------------------------------------------------------------- +// PutTargets target-type-specific parameters (EcsParameters, KinesisParameters, +// SqsParameters, HttpParameters, RedshiftDataParameters, RunCommandParameters, +// SageMakerPipelineParameters, AppSyncParameters) +// --------------------------------------------------------------------------- + +// TestRefinement2_PutTargets_TypeSpecificParametersRoundTrip proves that all +// of the target-type-specific parameter structs defined on aws-sdk-go-v2's +// eventbridge.types.Target (EcsParameters, HttpParameters, KinesisParameters, +// RedshiftDataParameters, RunCommandParameters, SageMakerPipelineParameters, +// SqsParameters, AppSyncParameters) round-trip through PutTargets and +// ListTargetsByRule. Before this fix, the gopherstack Target struct only +// modeled Input/InputPath/InputTransformer/DeadLetterConfig/RetryPolicy/ +// BatchParameters: any of these other target-type parameters set by a +// client were silently dropped by json.Unmarshal (unknown fields are +// ignored), so a client configuring, say, an ECS target's +// NetworkConfiguration or task Tags would see them vanish on the very next +// DescribeRule/ListTargetsByRule call. +func TestRefinement2_PutTargets_TypeSpecificParametersRoundTrip(t *testing.T) { + t.Parallel() + + newRule := func(t *testing.T, b *eventbridge.InMemoryBackend, name string) { + t.Helper() + _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: name, + EventPattern: `{"source":["x"]}`, + }) + require.NoError(t, err) + } + + target := eventbridge.Target{ + ID: "t1", + Arn: "arn:aws:ecs:us-east-1:123456789012:cluster/my-cluster", + AppSyncParameters: &eventbridge.AppSyncParameters{ + GraphQLOperation: "mutation Publish { publish { id } }", + }, + EcsParameters: &eventbridge.EcsParameters{ + TaskDefinitionArn: "arn:aws:ecs:us-east-1:123456789012:task-definition/my-task:1", + LaunchType: "FARGATE", + NetworkConfiguration: &eventbridge.NetworkConfiguration{ + AwsvpcConfiguration: &eventbridge.AwsVpcConfiguration{ + Subnets: []string{"subnet-1", "subnet-2"}, + SecurityGroups: []string{"sg-1"}, + AssignPublicIP: "ENABLED", + }, + }, + CapacityProviderStrategy: []eventbridge.CapacityProviderStrategyItem{ + {CapacityProvider: "FARGATE_SPOT", Weight: 1}, + }, + PlacementConstraints: []eventbridge.PlacementConstraint{ + {Type: "distinctInstance"}, + }, + PlacementStrategy: []eventbridge.PlacementStrategy{ + {Field: "cpu", Type: "binpack"}, + }, + Tags: []eventbridge.EcsTag{{Key: "team", Value: "platform"}}, + }, + HTTPParameters: &eventbridge.HTTPParameters{ + HeaderParameters: map[string]string{"X-Custom": "1"}, + PathParameterValues: []string{"seg1"}, + QueryStringParameters: map[string]string{"q": "1"}, + }, + KinesisParameters: &eventbridge.KinesisParameters{ + PartitionKeyPath: "$.detail.id", + }, + RedshiftDataParameters: &eventbridge.RedshiftDataParameters{ + Database: "mydb", + DBUser: "admin", + SQL: "select 1", + }, + RunCommandParameters: &eventbridge.RunCommandParameters{ + RunCommandTargets: []eventbridge.RunCommandTarget{ + {Key: "tag:Name", Values: []string{"my-instance"}}, + }, + }, + SageMakerPipelineParameters: &eventbridge.SageMakerPipelineParameters{ + PipelineParameterList: []eventbridge.SageMakerPipelineParameter{ + {Name: "env", Value: "prod"}, + }, + }, + SqsParameters: &eventbridge.SqsParameters{MessageGroupID: "group-1"}, + } + + b := newBackend() + newRule(t, b, "r") + + failed, err := b.PutTargets(context.Background(), "r", "", []eventbridge.Target{target}) + require.NoError(t, err) + assert.Empty(t, failed) + + got, _, err := b.ListTargetsByRule(context.Background(), "r", "", "", 0) + require.NoError(t, err) + require.Len(t, got, 1) + + assert.Equal(t, target.AppSyncParameters, got[0].AppSyncParameters) + assert.Equal(t, target.EcsParameters, got[0].EcsParameters) + assert.Equal(t, target.HTTPParameters, got[0].HTTPParameters) + assert.Equal(t, target.KinesisParameters, got[0].KinesisParameters) + assert.Equal(t, target.RedshiftDataParameters, got[0].RedshiftDataParameters) + assert.Equal(t, target.RunCommandParameters, got[0].RunCommandParameters) + assert.Equal(t, target.SageMakerPipelineParameters, got[0].SageMakerPipelineParameters) + assert.Equal(t, target.SqsParameters, got[0].SqsParameters) +} + +// TestRefinement2_PutTargets_TypeSpecificParametersValidation proves PutTargets +// rejects target-type-specific parameter structs missing their AWS-required +// member, mirroring the client-side constraint validation aws-sdk-go-v2 +// performs (validateEcsParameters, validateKinesisParameters, +// validateRedshiftDataParameters, validateRunCommandParameters/Target). +func TestRefinement2_PutTargets_TypeSpecificParametersValidation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + target eventbridge.Target + }{ + { + name: "EcsParameters missing TaskDefinitionArn", + target: eventbridge.Target{ + ID: "t1", Arn: "arn:aws:ecs:us-east-1:123456789012:cluster/c", + EcsParameters: &eventbridge.EcsParameters{}, + }, + }, + { + name: "KinesisParameters missing PartitionKeyPath", + target: eventbridge.Target{ + ID: "t1", Arn: "arn:aws:kinesis:us-east-1:123456789012:stream/s", + KinesisParameters: &eventbridge.KinesisParameters{}, + }, + }, + { + name: "RedshiftDataParameters missing Database", + target: eventbridge.Target{ + ID: "t1", Arn: "arn:aws:redshift:us-east-1:123456789012:cluster:c", + RedshiftDataParameters: &eventbridge.RedshiftDataParameters{}, + }, + }, + { + name: "RunCommandParameters missing RunCommandTargets", + target: eventbridge.Target{ + ID: "t1", Arn: "arn:aws:ec2:us-east-1:123456789012:instance/i", + RunCommandParameters: &eventbridge.RunCommandParameters{}, + }, + }, + { + name: "RunCommandParameters target missing Values", + target: eventbridge.Target{ + ID: "t1", Arn: "arn:aws:ec2:us-east-1:123456789012:instance/i", + RunCommandParameters: &eventbridge.RunCommandParameters{ + RunCommandTargets: []eventbridge.RunCommandTarget{{Key: "tag:Name"}}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + b := newBackend() + _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: "r", EventPattern: `{"source":["x"]}`, + }) + require.NoError(t, err) + + failed, err := b.PutTargets(context.Background(), "r", "", []eventbridge.Target{tt.target}) + require.NoError(t, err, "PutTargets itself must not error; failures are per-entry") + require.Len(t, failed, 1) + assert.Equal(t, "InvalidParameter", failed[0].ErrorCode) + assert.NotEmpty(t, failed[0].ErrorMessage) + + got, _, err := b.ListTargetsByRule(context.Background(), "r", "", "", 0) + require.NoError(t, err) + assert.Empty(t, got, "the invalid target must not have been stored") + }) + } +} + +// TestRefinement2_PutTargets_RetryPolicyBounds proves PutTargets enforces +// AWS's documented RetryPolicy bounds (MaximumRetryAttempts 0-185, +// MaximumEventAgeInSeconds 60-86400 when set), which the client SDK does not +// validate locally (no smithy range trait on these fields), so the backend +// must enforce them itself to reject out-of-range values the way the real +// service does. +func TestRefinement2_PutTargets_RetryPolicyBounds(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + policy eventbridge.RetryPolicy + wantErr bool + }{ + {"zero attempts and unset age ok", eventbridge.RetryPolicy{MaximumRetryAttempts: 0}, false}, + {"max attempts ok", eventbridge.RetryPolicy{MaximumRetryAttempts: 185}, false}, + {"attempts over max rejected", eventbridge.RetryPolicy{MaximumRetryAttempts: 186}, true}, + {"negative attempts rejected", eventbridge.RetryPolicy{MaximumRetryAttempts: -1}, true}, + {"min age ok", eventbridge.RetryPolicy{MaximumEventAgeInSeconds: 60}, false}, + {"max age ok", eventbridge.RetryPolicy{MaximumEventAgeInSeconds: 86400}, false}, + {"age under min rejected", eventbridge.RetryPolicy{MaximumEventAgeInSeconds: 59}, true}, + {"age over max rejected", eventbridge.RetryPolicy{MaximumEventAgeInSeconds: 86401}, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + b := newBackend() + _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: "r", EventPattern: `{"source":["x"]}`, + }) + require.NoError(t, err) + + policy := tt.policy + failed, err := b.PutTargets(context.Background(), "r", "", []eventbridge.Target{ + {ID: "t1", Arn: "arn:aws:sqs:us-east-1:123456789012:q", RetryPolicy: &policy}, + }) + require.NoError(t, err) + if tt.wantErr { + require.Len(t, failed, 1) + assert.Equal(t, "InvalidParameter", failed[0].ErrorCode) + + return + } + assert.Empty(t, failed) + }) + } +} + // --------------------------------------------------------------------------- // StartReplay time ordering validation // --------------------------------------------------------------------------- @@ -729,9 +955,10 @@ func TestRefinement2_PartnerEventSourceCRUD(t *testing.T) { func TestRefinement2_PutPartnerEvents(t *testing.T) { t.Parallel() b := newBackend() - results := b.PutPartnerEvents(context.Background(), []eventbridge.EventEntry{ + results, err := b.PutPartnerEvents(context.Background(), []eventbridge.EventEntry{ {Source: "aws.partner.test", DetailType: "Ping", Detail: "{}"}, }) + require.NoError(t, err) assert.Len(t, results, 1) assert.Empty(t, results[0].ErrorCode) } diff --git a/services/eventbridge/backend_test.go b/services/eventbridge/backend_test.go index bc08b8e6c..9aac5072d 100644 --- a/services/eventbridge/backend_test.go +++ b/services/eventbridge/backend_test.go @@ -269,7 +269,8 @@ func TestPutEvents(t *testing.T) { {Source: "my.app", DetailType: "UserDeleted", Detail: `{"userId":"456"}`}, } - results := b.PutEvents(context.Background(), entries) + results, err := b.PutEvents(context.Background(), entries) + require.NoError(t, err) assert.Len(t, results, 2) for _, r := range results { assert.NotEmpty(t, r.EventID) @@ -284,12 +285,22 @@ func TestEventLogMaxSize(t *testing.T) { t.Parallel() b := eventbridge.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - // Put 1100 events; log should cap at 1000. - batch := make([]eventbridge.EventEntry, 1100) - for i := range batch { - batch[i] = eventbridge.EventEntry{Source: "s", DetailType: "t", Detail: "{}"} + // Put 1100 events (in batches of 10, AWS's per-request PutEvents cap); + // log should cap at 1000. + const totalEvents = 1100 + + const maxBatch = 10 + + entry := eventbridge.EventEntry{Source: "s", DetailType: "t", Detail: "{}"} + for sent := 0; sent < totalEvents; sent += maxBatch { + n := min(maxBatch, totalEvents-sent) + batch := make([]eventbridge.EventEntry, n) + for i := range batch { + batch[i] = entry + } + _, err := b.PutEvents(context.Background(), batch) + require.NoError(t, err) } - b.PutEvents(context.Background(), batch) log := b.GetEventLog(context.Background()) assert.Len(t, log, 1000) diff --git a/services/eventbridge/coverage_test.go b/services/eventbridge/coverage_test.go index 81bf5a3bd..3761ff20c 100644 --- a/services/eventbridge/coverage_test.go +++ b/services/eventbridge/coverage_test.go @@ -397,9 +397,14 @@ func TestHandler_PutEvents_Empty(t *testing.T) { wantCode int }{ { - name: "empty entries returns OK", + // AWS's PutEventsRequestEntryList requires at least 1 item + // ("Array Members: Minimum number of 1 item"); an empty batch is + // a validation error, not a no-op success. Previously this + // backend accepted an empty batch as a 200 with zero results, + // which is the wrong AWS behavior. + name: "empty entries returns validation error", body: `{"Entries":[]}`, - wantCode: http.StatusOK, + wantCode: http.StatusBadRequest, }, } diff --git a/services/eventbridge/delivery.go b/services/eventbridge/delivery.go index 217e3538d..a203da300 100644 --- a/services/eventbridge/delivery.go +++ b/services/eventbridge/delivery.go @@ -17,6 +17,7 @@ import ( "github.com/google/uuid" "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) // inputPathsMapKeyRe validates InputPathsMap variable names per AWS spec. @@ -136,7 +137,10 @@ func (b *InMemoryBackend) deliverScheduledRule( const detail = `{"scheduled":true}` b.mu.Lock("deliverScheduledRule") - storedTargets := b.targets[region][b.targetKey(busName, rule.Name)] + var storedTargets *store.Table[Target] + if regionTargets := b.targets[region]; regionTargets != nil { + storedTargets = regionTargets[b.targetKey(busName, rule.Name)] + } snapped := snapshotTargets(storedTargets) accountID := b.accountID dt := *b.deliveryTargets @@ -249,7 +253,7 @@ func (b *InMemoryBackend) buildDeliveryPlan(region string, entries []EventEntry) } storedTargets := targetsStore[b.targetKey(busName, rule.Name)] - if len(storedTargets) == 0 { + if storedTargets == nil || storedTargets.Len() == 0 { continue } @@ -266,10 +270,16 @@ func (b *InMemoryBackend) buildDeliveryPlan(region string, entries []EventEntry) } // snapshotTargets returns copies of the stored target structs so delivery cannot -// race a concurrent PutTargets/RemoveTargets mutating the stored values. -func snapshotTargets(stored map[string]*Target) []*Target { - out := make([]*Target, 0, len(stored)) - for _, t := range stored { +// race a concurrent PutTargets/RemoveTargets mutating the stored values. A nil +// stored table (region/rule never touched) yields an empty, non-nil slice. +func snapshotTargets(stored *store.Table[Target]) []*Target { + if stored == nil { + return nil + } + + all := stored.All() + out := make([]*Target, 0, len(all)) + for _, t := range all { targetCopy := *t out = append(out, &targetCopy) } @@ -643,6 +653,101 @@ func applyInputPath(path string, envelope map[string]any) string { return string(b) } +// AWS's documented bounds for a target's RetryPolicy (PutTargets API +// reference): MaximumRetryAttempts 0-185, MaximumEventAgeInSeconds 60-86400. +// These are server-side-only constraints (not modeled as smithy range +// traits in the client SDK, so the Go SDK does not reject them locally +// either), so PutTargets must enforce them itself. +const ( + minRetryAttempts = 0 + maxRetryAttempts = 185 + minMaxEventAgeSecs = 60 + maxMaxEventAgeSecs = 86400 +) + +// validateTargetTypeParameters checks the required members of the +// target-type-specific parameter structs (EcsParameters, KinesisParameters, +// RedshiftDataParameters, RunCommandParameters), mirroring the client-side +// constraint validation the real AWS SDK performs before a PutTargets call +// ever reaches the service (see validators.go in aws-sdk-go-v2/eventbridge: +// validateEcsParameters, validateKinesisParameters, +// validateRedshiftDataParameters, validateRunCommandParameters/Target), and +// enforces the RetryPolicy bounds AWS documents for PutTargets. +func validateTargetTypeParameters(t *Target) error { + if t.EcsParameters != nil && t.EcsParameters.TaskDefinitionArn == "" { + return fmt.Errorf("%w: EcsParameters.TaskDefinitionArn is required", ErrInvalidParameter) + } + + if t.KinesisParameters != nil && t.KinesisParameters.PartitionKeyPath == "" { + return fmt.Errorf("%w: KinesisParameters.PartitionKeyPath is required", ErrInvalidParameter) + } + + if t.RedshiftDataParameters != nil && t.RedshiftDataParameters.Database == "" { + return fmt.Errorf("%w: RedshiftDataParameters.Database is required", ErrInvalidParameter) + } + + if t.RunCommandParameters != nil { + if err := validateRunCommandParameters(t.RunCommandParameters); err != nil { + return err + } + } + + return validateRetryPolicy(t.RetryPolicy) +} + +// validateRetryPolicy enforces AWS's documented RetryPolicy bounds. A zero +// MaximumEventAgeInSeconds is treated as "unset" (the JSON wire form uses +// omitempty, so 0 and absent are indistinguishable) and defaulted downstream +// by deliverToTargetBounded, matching existing delivery behavior; any other +// out-of-range value is rejected. +func validateRetryPolicy(rp *RetryPolicy) error { + if rp == nil { + return nil + } + + if rp.MaximumRetryAttempts < minRetryAttempts || rp.MaximumRetryAttempts > maxRetryAttempts { + return fmt.Errorf( + "%w: RetryPolicy.MaximumRetryAttempts must be between %d and %d", + ErrInvalidParameter, minRetryAttempts, maxRetryAttempts, + ) + } + + if rp.MaximumEventAgeInSeconds != 0 && + (rp.MaximumEventAgeInSeconds < minMaxEventAgeSecs || rp.MaximumEventAgeInSeconds > maxMaxEventAgeSecs) { + return fmt.Errorf( + "%w: RetryPolicy.MaximumEventAgeInSeconds must be between %d and %d", + ErrInvalidParameter, minMaxEventAgeSecs, maxMaxEventAgeSecs, + ) + } + + return nil +} + +// validateRunCommandParameters checks that RunCommandParameters carries at +// least one target and that each target has both a Key and at least one +// Value, matching AWS's required-member constraints for RunCommandTarget. +func validateRunCommandParameters(p *RunCommandParameters) error { + if len(p.RunCommandTargets) == 0 { + return fmt.Errorf("%w: RunCommandParameters.RunCommandTargets is required", ErrInvalidParameter) + } + + for i, rt := range p.RunCommandTargets { + if rt.Key == "" { + return fmt.Errorf( + "%w: RunCommandParameters.RunCommandTargets[%d].Key is required", ErrInvalidParameter, i, + ) + } + + if len(rt.Values) == 0 { + return fmt.Errorf( + "%w: RunCommandParameters.RunCommandTargets[%d].Values is required", ErrInvalidParameter, i, + ) + } + } + + return nil +} + // validateInputTransformer checks that all InputPathsMap keys satisfy the // AWS constraint ([A-Za-z0-9_]+). Returns an error if any key is invalid. func validateInputTransformer(t *InputTransformer) error { diff --git a/services/eventbridge/export_test.go b/services/eventbridge/export_test.go index 8ea6a36c0..e8cfed661 100644 --- a/services/eventbridge/export_test.go +++ b/services/eventbridge/export_test.go @@ -51,7 +51,7 @@ func (b *InMemoryBackend) APIDestinationCount() int { b.mu.RLock("APIDestinationCount") defer b.mu.RUnlock() - return len(b.apiDestinationsStore(b.region)) + return b.apiDestinationsTable(b.region).Len() } // ArchiveCount returns the number of archives in the backend (default region). @@ -59,7 +59,7 @@ func (b *InMemoryBackend) ArchiveCount() int { b.mu.RLock("ArchiveCount") defer b.mu.RUnlock() - return len(b.archivesStore(b.region)) + return b.archivesTable(b.region).Len() } // ConnectionCount returns the number of connections in the backend (default region). @@ -67,7 +67,7 @@ func (b *InMemoryBackend) ConnectionCount() int { b.mu.RLock("ConnectionCount") defer b.mu.RUnlock() - return len(b.connectionsStore(b.region)) + return b.connectionsTable(b.region).Len() } // EndpointCount returns the number of endpoints in the backend (default region). @@ -75,7 +75,7 @@ func (b *InMemoryBackend) EndpointCount() int { b.mu.RLock("EndpointCount") defer b.mu.RUnlock() - return len(b.endpointsStore(b.region)) + return b.endpointsTable(b.region).Len() } // EventSourceCount returns the number of event sources in the backend (default region). @@ -83,7 +83,7 @@ func (b *InMemoryBackend) EventSourceCount() int { b.mu.RLock("EventSourceCount") defer b.mu.RUnlock() - return len(b.eventSourcesStore(b.region)) + return b.eventSourcesTable(b.region).Len() } // ReplayCount returns the number of replays in the backend (default region). @@ -91,7 +91,7 @@ func (b *InMemoryBackend) ReplayCount() int { b.mu.RLock("ReplayCount") defer b.mu.RUnlock() - return len(b.replaysStore(b.region)) + return b.replaysTable(b.region).Len() } // PartnerSourceCount returns the number of partner event sources in the backend (default region). @@ -99,7 +99,7 @@ func (b *InMemoryBackend) PartnerSourceCount() int { b.mu.RLock("PartnerSourceCount") defer b.mu.RUnlock() - return len(b.partnerSourcesStore(b.region)) + return b.partnerSourcesTable(b.region).Len() } // HandlerOpsLen returns the number of pre-built handler operations. @@ -137,8 +137,7 @@ func (b *InMemoryBackend) SetArchiveCreationTimeForTest(name string, creationTim b.mu.Lock("SetArchiveCreationTimeForTest") defer b.mu.Unlock() - store := b.archivesStore(b.region) - archive, exists := store[name] + archive, exists := b.archivesTable(b.region).Get(name) if !exists { return fmt.Errorf("%w: archive %s not found", ErrNotFound, name) } @@ -178,8 +177,8 @@ func (b *InMemoryBackend) ARNIndexConsistent() (bool, string) { defer b.mu.RUnlock() for region, regionTargets := range b.targets { - for targetKey, tMap := range regionTargets { - for _, t := range tMap { + for targetKey, tTable := range regionTargets { + for _, t := range tTable.All() { if rm := b.targetsByARN[region]; rm == nil { return false, fmt.Sprintf("targetsByARN[%s] is nil but canonical has entries", region) } else if _, ok := rm[t.Arn][targetKey]; !ok { diff --git a/services/eventbridge/handler.go b/services/eventbridge/handler.go index a27289565..633bdf1e7 100644 --- a/services/eventbridge/handler.go +++ b/services/eventbridge/handler.go @@ -685,7 +685,10 @@ func (h *Handler) eventsActions() map[string]actionFn { if err := json.Unmarshal(b, &input); err != nil { return nil, err } - entries := h.Backend.PutEvents(ctx, input.Entries) + entries, err := h.Backend.PutEvents(ctx, input.Entries) + if err != nil { + return nil, err + } return &putEventsOutput{ FailedEntryCount: countFailedEntries(entries), @@ -1446,7 +1449,10 @@ func (h *Handler) extendedPartnerSourceActions() map[string]actionFn { if err := json.Unmarshal(b, &input); err != nil { return nil, err } - entries := h.Backend.PutPartnerEvents(ctx, input.Entries) + entries, err := h.Backend.PutPartnerEvents(ctx, input.Entries) + if err != nil { + return nil, err + } return &putEventsOutput{ FailedEntryCount: countFailedEntries(entries), diff --git a/services/eventbridge/handler_test.go b/services/eventbridge/handler_test.go index e7203e8b8..304d642e3 100644 --- a/services/eventbridge/handler_test.go +++ b/services/eventbridge/handler_test.go @@ -753,7 +753,7 @@ func TestHandler_Shutdown_ImplementsShutdowner(t *testing.T) { // After Shutdown, Close has been called and the backend's context is // cancelled — any subsequent PutEvents must not panic or deadlock. assert.NotPanics(t, func() { - _ = backend.PutEvents(context.Background(), []eventbridge.EventEntry{ + _, _ = backend.PutEvents(context.Background(), []eventbridge.EventEntry{ {Source: "after-shutdown", DetailType: "test", Detail: `{}`}, }) }) diff --git a/services/eventbridge/janitor.go b/services/eventbridge/janitor.go index f0807b449..6761ad4a0 100644 --- a/services/eventbridge/janitor.go +++ b/services/eventbridge/janitor.go @@ -47,7 +47,7 @@ func (j *ArchiveJanitor) SweepOnce(ctx context.Context) { j.Backend.mu.Lock("EventBridgeArchiveJanitor") count := 0 for region, archives := range j.Backend.archives { - for name, archive := range archives { + for _, archive := range archives.All() { if archive.RetentionDays <= 0 { continue } @@ -57,8 +57,8 @@ func (j *ArchiveJanitor) SweepOnce(ctx context.Context) { continue } - delete(j.Backend.archives[region], name) - delete(j.Backend.archivedEvents[region], name) + archives.Delete(archive.ArchiveName) + delete(j.Backend.archivedEvents[region], archive.ArchiveName) count++ } } diff --git a/services/eventbridge/leak_test.go b/services/eventbridge/leak_test.go index c6bf4da11..b818e19ae 100644 --- a/services/eventbridge/leak_test.go +++ b/services/eventbridge/leak_test.go @@ -32,13 +32,14 @@ func TestEventLog_CappedAtMax(t *testing.T) { b := eventbridge.NewInMemoryBackend() for i := range tc.putEvents { - results := b.PutEvents(ctx, []eventbridge.EventEntry{ + results, err := b.PutEvents(ctx, []eventbridge.EventEntry{ { Source: "test.source", DetailType: "TestEvent", Detail: `{"seq":` + string(rune('0'+i%10)) + `}`, }, }) + require.NoError(t, err) require.Empty(t, results[0].ErrorCode, "PutEvents must not error") } diff --git a/services/eventbridge/models.go b/services/eventbridge/models.go index 6a79ccd16..2833e2e05 100644 --- a/services/eventbridge/models.go +++ b/services/eventbridge/models.go @@ -50,15 +50,23 @@ type BatchArrayProperties struct { // Target represents an EventBridge rule target. type Target struct { - InputTransformer *InputTransformer `json:"InputTransformer,omitempty"` - DeadLetterConfig *DeadLetterConfig `json:"DeadLetterConfig,omitempty"` - RetryPolicy *RetryPolicy `json:"RetryPolicy,omitempty"` - BatchParameters *BatchParameters `json:"BatchParameters,omitempty"` - ID string `json:"Id"` - Arn string `json:"Arn"` - RoleArn string `json:"RoleArn,omitempty"` - Input string `json:"Input,omitempty"` - InputPath string `json:"InputPath,omitempty"` + InputTransformer *InputTransformer `json:"InputTransformer,omitempty"` + DeadLetterConfig *DeadLetterConfig `json:"DeadLetterConfig,omitempty"` + RetryPolicy *RetryPolicy `json:"RetryPolicy,omitempty"` + BatchParameters *BatchParameters `json:"BatchParameters,omitempty"` + AppSyncParameters *AppSyncParameters `json:"AppSyncParameters,omitempty"` + EcsParameters *EcsParameters `json:"EcsParameters,omitempty"` + HTTPParameters *HTTPParameters `json:"HttpParameters,omitempty"` + KinesisParameters *KinesisParameters `json:"KinesisParameters,omitempty"` + RedshiftDataParameters *RedshiftDataParameters `json:"RedshiftDataParameters,omitempty"` + RunCommandParameters *RunCommandParameters `json:"RunCommandParameters,omitempty"` + SageMakerPipelineParameters *SageMakerPipelineParameters `json:"SageMakerPipelineParameters,omitempty"` + SqsParameters *SqsParameters `json:"SqsParameters,omitempty"` + ID string `json:"Id"` + Arn string `json:"Arn"` + RoleArn string `json:"RoleArn,omitempty"` + Input string `json:"Input,omitempty"` + InputPath string `json:"InputPath,omitempty"` } // InputTransformer holds input transformer configuration for a target. @@ -67,6 +75,129 @@ type InputTransformer struct { InputTemplate string `json:"InputTemplate"` } +// AppSyncParameters holds the GraphQL operation to invoke when the target is +// an AppSync API. +type AppSyncParameters struct { + GraphQLOperation string `json:"GraphQLOperation,omitempty"` +} + +// EcsParameters holds the parameters used to run an Amazon ECS task when the +// event target is an ECS cluster. +type EcsParameters struct { + NetworkConfiguration *NetworkConfiguration `json:"NetworkConfiguration,omitempty"` + PropagateTags string `json:"PropagateTags,omitempty"` + TaskDefinitionArn string `json:"TaskDefinitionArn"` + Group string `json:"Group,omitempty"` + LaunchType string `json:"LaunchType,omitempty"` + PlatformVersion string `json:"PlatformVersion,omitempty"` + ReferenceID string `json:"ReferenceId,omitempty"` + PlacementConstraints []PlacementConstraint `json:"PlacementConstraints,omitempty"` + PlacementStrategy []PlacementStrategy `json:"PlacementStrategy,omitempty"` + Tags []EcsTag `json:"Tags,omitempty"` + CapacityProviderStrategy []CapacityProviderStrategyItem `json:"CapacityProviderStrategy,omitempty"` + EnableECSManagedTags bool `json:"EnableECSManagedTags,omitempty"` + EnableExecuteCommand bool `json:"EnableExecuteCommand,omitempty"` +} + +// NetworkConfiguration specifies the awsvpc network configuration for an ECS +// target task. +type NetworkConfiguration struct { + AwsvpcConfiguration *AwsVpcConfiguration `json:"AwsvpcConfiguration,omitempty"` +} + +// AwsVpcConfiguration specifies the subnets, security groups, and public-IP +// assignment for an ECS target task using the awsvpc network mode. +type AwsVpcConfiguration struct { + AssignPublicIP string `json:"AssignPublicIp,omitempty"` + Subnets []string `json:"Subnets"` + SecurityGroups []string `json:"SecurityGroups,omitempty"` +} + +// CapacityProviderStrategyItem is a single entry in an ECS target's capacity +// provider strategy. +type CapacityProviderStrategyItem struct { + CapacityProvider string `json:"CapacityProvider"` + Base int32 `json:"Base,omitempty"` + Weight int32 `json:"Weight,omitempty"` +} + +// PlacementConstraint is a single ECS task placement constraint. +type PlacementConstraint struct { + Expression string `json:"Expression,omitempty"` + Type string `json:"Type,omitempty"` +} + +// PlacementStrategy is a single ECS task placement strategy rule. +type PlacementStrategy struct { + Field string `json:"Field,omitempty"` + Type string `json:"Type,omitempty"` +} + +// EcsTag is a key/value tag applied to an ECS target task (distinct from the +// EventBridge resource-tag maps used by tags.go, which model bus/rule/etc. +// tagging, not the per-task tags forwarded to ECS RunTask). +type EcsTag struct { + Key string `json:"Key"` + Value string `json:"Value,omitempty"` +} + +// HTTPParameters holds the headers, path parameters, and query-string values +// to add when the target is an API Gateway API or EventBridge ApiDestination. +type HTTPParameters struct { + HeaderParameters map[string]string `json:"HeaderParameters,omitempty"` + QueryStringParameters map[string]string `json:"QueryStringParameters,omitempty"` + PathParameterValues []string `json:"PathParameterValues,omitempty"` +} + +// KinesisParameters specifies the partition-key JSON path for a Kinesis Data +// Stream target. +type KinesisParameters struct { + PartitionKeyPath string `json:"PartitionKeyPath"` +} + +// RedshiftDataParameters holds the Redshift Data API ExecuteStatement +// parameters for an Amazon Redshift cluster target. +type RedshiftDataParameters struct { + Database string `json:"Database"` + DBUser string `json:"DbUser,omitempty"` + SecretManagerArn string `json:"SecretManagerArn,omitempty"` + SQL string `json:"Sql,omitempty"` + StatementName string `json:"StatementName,omitempty"` + Sqls []string `json:"Sqls,omitempty"` + WithEvent bool `json:"WithEvent,omitempty"` +} + +// RunCommandParameters holds the EC2 Run Command targets for a target rule. +type RunCommandParameters struct { + RunCommandTargets []RunCommandTarget `json:"RunCommandTargets"` +} + +// RunCommandTarget selects EC2 instances by tag or instance ID for Run +// Command. +type RunCommandTarget struct { + Key string `json:"Key"` + Values []string `json:"Values"` +} + +// SageMakerPipelineParameters holds the pipeline parameter overrides used to +// start a SageMaker AI Model Building Pipeline execution. +type SageMakerPipelineParameters struct { + PipelineParameterList []SageMakerPipelineParameter `json:"PipelineParameterList,omitempty"` +} + +// SageMakerPipelineParameter is a single name/value pipeline parameter +// override. +type SageMakerPipelineParameter struct { + Name string `json:"Name"` + Value string `json:"Value"` +} + +// SqsParameters holds the FIFO message-group ID to use when the target is an +// SQS FIFO queue. +type SqsParameters struct { + MessageGroupID string `json:"MessageGroupId,omitempty"` +} + // EventEntry represents a single event to publish. type EventEntry struct { Time *time.Time `json:"Time,omitempty"` diff --git a/services/eventbridge/persistence.go b/services/eventbridge/persistence.go index 1f2ed4263..932c21be8 100644 --- a/services/eventbridge/persistence.go +++ b/services/eventbridge/persistence.go @@ -3,27 +3,58 @@ package eventbridge import ( "context" "encoding/json" + "fmt" + "strings" "sync" + "time" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" svcTags "github.com/blackbirdworks/gopherstack/pkgs/tags" ) +// eventbridgeSnapshotVersion identifies the shape of backendSnapshot's Tables +// blob (i.e. the set of resource tables registered on b.registry -- see +// store_setup.go). It must be bumped whenever a change there would make an +// older snapshot unsafe to decode as the current shape. Restore compares this +// against the persisted value and discards (rather than attempts to +// partially decode) any mismatch -- see Restore below. Pre-Phase-3.3 +// snapshots have no "version" field at all, which unmarshals as 0 -- also a +// mismatch, so they are discarded the same way. This mirrors the services/ssm +// conversion and the services/sqs pilot (commit 0f09d77c) / services/ec2 +// conversion (commit 12e611a4). +const eventbridgeSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the EventBridge backend. +// +// Tables holds one JSON-encoded, key-sorted array per registered +// *store.Table[V] on b.registry, produced by +// [github.com/blackbirdworks/gopherstack/pkgs/store.Registry.SnapshotAll]. +// Each per-region table is registered under "/" (bus, +// event source, replay, API destination, archive, connection, endpoint, +// partner source); rules and targets are registered one level deeper still, +// under "//" -- see store_setup.go's +// getOrCreateTable/getOrCreateNestedTable. Restore must pre-register every +// (resource, region[, parent]) tuple found in Tables before calling +// registry.RestoreAll, see preRegisterSnapshotTables below. +// +// pipes, registries, and schemas are *store.Table-backed too but live on +// b.auxRegistry, not b.registry, and are deliberately NOT included in +// Tables -- see store_setup.go's package doc: they were never part of +// backendSnapshot before this conversion, so leaving them out preserves that +// (surprising, pre-existing) behavior byte-for-byte rather than silently +// starting to persist state that never round-tripped through Restore before. +// +// archivedEvents, busePolicies, schemaVersions, and codeBindings are, for the +// same reason, left out of backendSnapshot exactly as they were before this +// conversion -- see store_setup.go's package doc for why each doesn't fit +// Table's func(*V) string keying requirement in the first place. type backendSnapshot struct { - Buses map[string]map[string]*EventBus `json:"buses"` - Rules map[string]map[string]map[string]*Rule `json:"rules"` - Targets map[string]map[string]map[string]*Target `json:"targets"` - EventSources map[string]map[string]*EventSource `json:"eventSources,omitempty"` - Replays map[string]map[string]*Replay `json:"replays,omitempty"` - APIDestinations map[string]map[string]*APIDestination `json:"apiDestinations,omitempty"` - Archives map[string]map[string]*Archive `json:"archives,omitempty"` - Connections map[string]map[string]*Connection `json:"connections,omitempty"` - Endpoints map[string]map[string]*Endpoint `json:"endpoints,omitempty"` - PartnerSources map[string]map[string]*PartnerEventSource `json:"partnerSources,omitempty"` - AccountID string `json:"accountID"` - Region string `json:"region"` - EventLog []EventLogEntry `json:"eventLog"` + Tables map[string]json.RawMessage `json:"tables"` + AccountID string `json:"accountID"` + Region string `json:"region"` + EventLog []EventLogEntry `json:"eventLog"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. @@ -32,20 +63,23 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "eventbridge: snapshot table marshal failed", "error", err) + + return nil + } + snap := backendSnapshot{ - Buses: b.buses, - Rules: b.rules, - Targets: b.targets, - EventSources: b.eventSources, - Replays: b.replays, - APIDestinations: b.apiDestinations, - Archives: b.archives, - Connections: b.connections, - Endpoints: b.endpoints, - PartnerSources: b.partnerSources, - EventLog: b.eventLog, - AccountID: b.accountID, - Region: b.region, + Version: eventbridgeSnapshotVersion, + Tables: tables, + EventLog: b.eventLog, + AccountID: b.accountID, + Region: b.region, } data, err := json.Marshal(snap) @@ -59,6 +93,31 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { return data } +// preRegisterSnapshotTables ensures every (resource, region[, parent]) tuple +// present in tables is registered on b.registry before registry.RestoreAll is +// called. RestoreAll only restores tables already registered (see +// [github.com/blackbirdworks/gopherstack/pkgs/store.Registry.RestoreAll]); +// a region/bus a fresh backend has never touched would otherwise not yet have +// a table registered under that name, and its persisted data would be +// silently dropped instead of restored. Each table name has the shape +// "/" or "//" (see +// store_setup.go's getOrCreateTable/getOrCreateNestedTable); splitting on the +// first "/" recovers the prefix, and tableAccessorsByPrefix's rules/targets +// entries split the remainder again themselves (region never contains "/", +// but bus/rule names may). +func (b *InMemoryBackend) preRegisterSnapshotTables(tables map[string]json.RawMessage) { + for key := range tables { + name, rest, found := strings.Cut(key, "/") + if !found { + continue + } + + if register, ok := tableAccessorsByPrefix[name]; ok { + register(b, rest) + } + } +} + // Restore loads backend state from a JSON snapshot. // It implements persistence.Persistable. // The logger and delivery targets are not restored — they are re-wired by the CLI. @@ -72,18 +131,46 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() - ensureBackendSnapshotMaps(&snap) - - b.buses = snap.Buses - b.rules = snap.Rules - b.targets = snap.Targets - b.eventSources = snap.EventSources - b.replays = snap.Replays - b.apiDestinations = snap.APIDestinations - b.archives = snap.Archives - b.connections = snap.Connections - b.endpoints = snap.Endpoints - b.partnerSources = snap.PartnerSources + if snap.Version != eventbridgeSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/ssm conversion and the + // services/sqs pilot (commit 0f09d77c) / services/ec2 conversion + // (commit 12e611a4). + logger.Load(ctx).WarnContext(ctx, + "eventbridge: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", eventbridgeSnapshotVersion) + + b.registry.ResetAll() + b.ruleIndex = make(map[string]map[string]map[ruleIndexKey]map[string]*Rule) + b.targetsByARN = make(map[string]map[string]map[string]struct{}) + b.patternCache = sync.Map{} + + // Match NewInMemoryBackendWithContext's construction-time state: a + // fresh backend is never truly empty of buses, so "starting empty" + // here means starting exactly as fresh as a new backend would. + b.busesTable(b.region).Put(&EventBus{ + Name: defaultEventBusName, + Arn: b.busARN(b.region, defaultEventBusName), + CreatedTime: time.Now(), + }) + + return nil + } + + if snap.Tables == nil { + snap.Tables = make(map[string]json.RawMessage) + } + + b.preRegisterSnapshotTables(snap.Tables) + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("eventbridge: restore snapshot tables: %w", err) + } + b.eventLog = snap.EventLog b.accountID = snap.AccountID b.region = snap.Region @@ -100,43 +187,10 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { return nil } -func ensureBackendSnapshotMaps(snap *backendSnapshot) { - if snap.Buses == nil { - snap.Buses = make(map[string]map[string]*EventBus) - } - if snap.Rules == nil { - snap.Rules = make(map[string]map[string]map[string]*Rule) - } - if snap.Targets == nil { - snap.Targets = make(map[string]map[string]map[string]*Target) - } - if snap.EventSources == nil { - snap.EventSources = make(map[string]map[string]*EventSource) - } - if snap.Replays == nil { - snap.Replays = make(map[string]map[string]*Replay) - } - if snap.APIDestinations == nil { - snap.APIDestinations = make(map[string]map[string]*APIDestination) - } - if snap.Archives == nil { - snap.Archives = make(map[string]map[string]*Archive) - } - if snap.Connections == nil { - snap.Connections = make(map[string]map[string]*Connection) - } - if snap.Endpoints == nil { - snap.Endpoints = make(map[string]map[string]*Endpoint) - } - if snap.PartnerSources == nil { - snap.PartnerSources = make(map[string]map[string]*PartnerEventSource) - } -} - func (b *InMemoryBackend) rebuildRuleIndexesLocked() error { for region, regRules := range b.rules { for busKey, busRules := range regRules { - for _, rule := range busRules { + for _, rule := range busRules.All() { if rule.EventPattern != "" { compiled, err := b.getOrCompilePattern(rule.EventPattern) if err != nil { @@ -156,8 +210,8 @@ func (b *InMemoryBackend) rebuildRuleIndexesLocked() error { // Must be called with the write lock held. func (b *InMemoryBackend) rebuildTargetsByARNLocked() { for region, regionTargets := range b.targets { - for targetKey, tMap := range regionTargets { - for _, t := range tMap { + for targetKey, tTable := range regionTargets { + for _, t := range tTable.All() { b.arnIndexAdd(region, t.Arn, targetKey) } } diff --git a/services/eventbridge/persistence_test.go b/services/eventbridge/persistence_test.go index 0aaa95d6d..57f39bf88 100644 --- a/services/eventbridge/persistence_test.go +++ b/services/eventbridge/persistence_test.go @@ -88,6 +88,159 @@ func TestInMemoryBackend_SnapshotRestore(t *testing.T) { } } +// TestInMemoryBackend_FullStateSnapshotRestore exercises a Snapshot->Restore +// round trip across every *store.Table-backed resource from the Phase 3.3 +// datalayer conversion (buses, rules, targets, event sources, replays, API +// destinations, archives, connections, endpoints, partner sources), plus the +// derived indexes (ruleIndex pattern matching, targetsByARN) that Restore +// rebuilds from the restored tables rather than persisting directly. +func TestInMemoryBackend_FullStateSnapshotRestore(t *testing.T) { + t.Parallel() + + ctx := context.Background() + original := eventbridge.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + + // Custom event bus. + _, err := original.CreateEventBus(ctx, "custom-bus", "a custom bus") + require.NoError(t, err) + + // Rule with an event pattern (exercises ruleIndex rebuild + pattern + // matching) on the custom bus. + _, err = original.PutRule(ctx, eventbridge.PutRuleInput{ + Name: "custom-rule", + EventBusName: "custom-bus", + EventPattern: `{"source":["my.app"]}`, + State: "ENABLED", + }) + require.NoError(t, err) + + // Target on that rule (exercises targets table + targetsByARN rebuild). + failed, err := original.PutTargets(ctx, "custom-rule", "custom-bus", []eventbridge.Target{ + {ID: "t1", Arn: "arn:aws:sqs:us-east-1:123456789012:my-queue"}, + }) + require.NoError(t, err) + require.Empty(t, failed) + + // Connection + API destination (connection referenced by ARN). + conn, err := original.CreateConnection(ctx, eventbridge.CreateConnectionInput{ + Name: "my-conn", + AuthorizationType: "API_KEY", + AuthParameters: &eventbridge.ConnectionAuthParameters{ + APIKeyAuthParameters: &eventbridge.ConnectionAPIKeyAuthParameters{ + APIKeyName: "x-api-key", + APIKeyValue: "secret-value", + }, + }, + }) + require.NoError(t, err) + + _, err = original.CreateAPIDestination(ctx, eventbridge.CreateAPIDestinationInput{ + Name: "my-dest", + ConnectionArn: conn.ConnectionArn, + InvocationEndpoint: "https://example.com/webhook", + HTTPMethod: "POST", + }) + require.NoError(t, err) + + // Archive. + _, err = original.CreateArchive(ctx, eventbridge.CreateArchiveInput{ + ArchiveName: "my-archive", + EventSourceArn: "arn:aws:events:us-east-1:123456789012:event-bus/custom-bus", + }) + require.NoError(t, err) + + // Endpoint. + _, err = original.CreateEndpoint(ctx, eventbridge.CreateEndpointInput{ + Name: "my-endpoint", + RoleArn: "arn:aws:iam::123456789012:role/endpoint-role", + }) + require.NoError(t, err) + + // Partner event source (also creates a mirrored, activatable EventSource). + _, err = original.CreatePartnerEventSource(ctx, "aws.partner/example.com/orders", "999999999999") + require.NoError(t, err) + require.NoError(t, original.ActivateEventSource(ctx, "aws.partner/example.com/orders")) + + // Replay. + _, err = original.StartReplay(ctx, eventbridge.StartReplayInput{ + ReplayName: "my-replay", + EventSourceArn: "arn:aws:events:us-east-1:123456789012:archive/my-archive", + }) + require.NoError(t, err) + + snap := original.Snapshot(t.Context()) + require.NotNil(t, snap) + + fresh := eventbridge.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + require.NoError(t, fresh.Restore(t.Context(), snap)) + + // Bus. + bus, err := fresh.DescribeEventBus(ctx, "custom-bus") + require.NoError(t, err) + assert.Equal(t, "a custom bus", bus.Description) + + // Rule + pattern (TestEventPattern proves compiledPattern survived). + rule, err := fresh.DescribeRule(ctx, "custom-rule", "custom-bus") + require.NoError(t, err) + assert.JSONEq(t, `{"source":["my.app"]}`, rule.EventPattern) + + // Target + targetsByARN index rebuild. + targets, _, err := fresh.ListTargetsByRule(ctx, "custom-rule", "custom-bus", "", 0) + require.NoError(t, err) + require.Len(t, targets, 1) + assert.Equal(t, "arn:aws:sqs:us-east-1:123456789012:my-queue", targets[0].Arn) + + ruleNames, _, err := fresh.ListRuleNamesByTarget( + ctx, "arn:aws:sqs:us-east-1:123456789012:my-queue", "custom-bus", "", + ) + require.NoError(t, err) + assert.Contains(t, ruleNames, "custom-rule") + + // Connection (auth is masked on Describe either way). + restoredConn, err := fresh.DescribeConnection(ctx, "my-conn") + require.NoError(t, err) + assert.Equal(t, "API_KEY", restoredConn.AuthorizationType) + + // API destination + connection lookup round trip. Connection.authSecret + // is unexported by design (never returned from Describe/List, see + // models.go) so, exactly as before this conversion, plain encoding/json + // never serializes it -- it does NOT survive Snapshot/Restore either via + // the old raw map or the new *store.Table. ResolveAPIDestination still + // resolves the destination/connection pair; only the secret itself is + // empty post-restore, matching pre-conversion behavior byte-for-byte. + dest, err := fresh.DescribeAPIDestination(ctx, "my-dest") + require.NoError(t, err) + resolvedDest, ok := fresh.ResolveAPIDestination(dest.APIDestinationArn) + require.True(t, ok) + assert.Equal(t, "API_KEY", resolvedDest.AuthType) + assert.Empty(t, resolvedDest.APIKeyValue, + "authSecret is unexported and never round-trips through JSON, pre- or post-conversion") + + // Archive. + archive, err := fresh.DescribeArchive(ctx, "my-archive") + require.NoError(t, err) + assert.Equal(t, "arn:aws:events:us-east-1:123456789012:event-bus/custom-bus", archive.EventSourceArn) + + // Endpoint. + endpoint, err := fresh.DescribeEndpoint(ctx, "my-endpoint") + require.NoError(t, err) + assert.Equal(t, "arn:aws:iam::123456789012:role/endpoint-role", endpoint.RoleArn) + + // Partner event source + its mirrored, activated EventSource. + partnerSrc, err := fresh.DescribePartnerEventSource(ctx, "aws.partner/example.com/orders") + require.NoError(t, err) + assert.Equal(t, "999999999999", partnerSrc.Account) + + eventSrc, err := fresh.DescribeEventSource(ctx, "aws.partner/example.com/orders") + require.NoError(t, err) + assert.Equal(t, "ACTIVE", eventSrc.State) + + // Replay. + replay, err := fresh.DescribeReplay(ctx, "my-replay") + require.NoError(t, err) + assert.Equal(t, "arn:aws:events:us-east-1:123456789012:archive/my-archive", replay.EventSourceArn) +} + func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { t.Parallel() diff --git a/services/eventbridge/schedule.go b/services/eventbridge/schedule.go index edccdf7ec..6594d174d 100644 --- a/services/eventbridge/schedule.go +++ b/services/eventbridge/schedule.go @@ -169,21 +169,107 @@ func (c *cronExpression) NextAfter(t time.Time) time.Time { return limit } +// cronFieldKind identifies which of the six cron fields a token belongs to, +// so numeric/name resolution can apply AWS's field-specific conventions: +// month and day-of-week accept three-letter names, and AWS's day-of-week +// numbering (1-7, where 1 = Sunday) differs from Go's time.Weekday (0-6, +// where 0 = Sunday). +type cronFieldKind int + +const ( + cronFieldMinute cronFieldKind = iota + cronFieldHour + cronFieldDayOfMonth + cronFieldMonth + cronFieldDayOfWeek + cronFieldYear +) + +// cronMonthNames maps AWS's three-letter month abbreviations to their 1-12 +// numeric value (same numbering as Go's time.Month, so no offset is needed). +// +//nolint:gochecknoglobals // read-only lookup table initialized once at startup +var cronMonthNames = map[string]int{ + "JAN": 1, + "FEB": 2, //nolint:mnd // calendar month number, not a magic constant + "MAR": 3, //nolint:mnd // calendar month number, not a magic constant + "APR": 4, //nolint:mnd // calendar month number, not a magic constant + "MAY": 5, //nolint:mnd // calendar month number, not a magic constant + "JUN": 6, //nolint:mnd // calendar month number, not a magic constant + "JUL": 7, //nolint:mnd // calendar month number, not a magic constant + "AUG": 8, //nolint:mnd // calendar month number, not a magic constant + "SEP": 9, //nolint:mnd // calendar month number, not a magic constant + "OCT": 10, //nolint:mnd // calendar month number, not a magic constant + "NOV": 11, //nolint:mnd // calendar month number, not a magic constant + "DEC": 12, //nolint:mnd // calendar month number, not a magic constant +} + +// cronDowNames maps AWS's three-letter day-of-week abbreviations directly to +// Go's time.Weekday numbering (0 = Sunday ... 6 = Saturday), so values +// resolved here need no further offset before comparison with t.Weekday(). +// +//nolint:gochecknoglobals // read-only lookup table initialized once at startup +var cronDowNames = map[string]int{ + "SUN": 0, + "MON": 1, + "TUE": 2, //nolint:mnd // time.Weekday number, not a magic constant + "WED": 3, //nolint:mnd // time.Weekday number, not a magic constant + "THU": 4, //nolint:mnd // time.Weekday number, not a magic constant + "FRI": 5, //nolint:mnd // time.Weekday number, not a magic constant + "SAT": 6, //nolint:mnd // time.Weekday number, not a magic constant +} + +// awsDayOfWeekBase is AWS's numeric base for day-of-week fields: AWS uses +// 1-7 with 1 = Sunday, while Go's time.Weekday uses 0-6 with 0 = Sunday. +const awsDayOfWeekBase = 1 + +// cronTokenValue resolves a single cron token to its comparable integer +// value for the given field kind: three-letter names for month/day-of-week, +// otherwise a plain number. Numeric day-of-week tokens are converted from +// AWS's 1-7 (Sunday = 1) convention to Go's time.Weekday 0-6 (Sunday = 0) so +// callers can compare directly against int(t.Weekday()). +func cronTokenValue(kind cronFieldKind, token string) (int, bool) { + switch kind { + case cronFieldMonth: + if n, ok := cronMonthNames[strings.ToUpper(token)]; ok { + return n, true + } + case cronFieldDayOfWeek: + if n, ok := cronDowNames[strings.ToUpper(token)]; ok { + return n, true + } + case cronFieldMinute, cronFieldHour, cronFieldDayOfMonth, cronFieldYear: + // Numeric-only fields; fall through to the plain-integer parse below. + } + + n, err := strconv.Atoi(token) + if err != nil { + return 0, false + } + + if kind == cronFieldDayOfWeek { + const daysPerWeek = 7 + n = ((n-awsDayOfWeekBase)%daysPerWeek + daysPerWeek) % daysPerWeek + } + + return n, true +} + // matches checks whether a time matches all cron fields. func (c *cronExpression) matches(t time.Time) bool { - if !matchCronField(c.minute, t.Minute(), cronMinuteMin, cronMinuteMax) { + if !matchCronField(cronFieldMinute, c.minute, t.Minute(), cronMinuteMin, cronMinuteMax) { return false } - if !matchCronField(c.hour, t.Hour(), cronHourMin, cronHourMax) { + if !matchCronField(cronFieldHour, c.hour, t.Hour(), cronHourMin, cronHourMax) { return false } - if !matchCronField(c.month, int(t.Month()), cronMonthMin, cronMonthMax) { + if !matchCronField(cronFieldMonth, c.month, int(t.Month()), cronMonthMin, cronMonthMax) { return false } - if !matchCronField(c.year, t.Year(), cronYearMin, cronYearMax) { + if !matchCronField(cronFieldYear, c.year, t.Year(), cronYearMin, cronYearMax) { return false } @@ -201,25 +287,29 @@ func (c *cronExpression) matchDayFields(t time.Time) bool { case domWild && dowWild: return true case domWild: - return matchCronField(c.dayOfWeek, int(t.Weekday()), cronDayOfWeekMin, cronDayOfWeekMax) + return matchCronField( + cronFieldDayOfWeek, c.dayOfWeek, int(t.Weekday()), cronDayOfWeekMin, cronDayOfWeekMax, + ) case dowWild: - return matchCronField(c.dayOfMonth, t.Day(), cronDayOfMonthMin, cronDayOfMonthMax) + return matchCronField(cronFieldDayOfMonth, c.dayOfMonth, t.Day(), cronDayOfMonthMin, cronDayOfMonthMax) default: - domMatch := matchCronField(c.dayOfMonth, t.Day(), cronDayOfMonthMin, cronDayOfMonthMax) - dowMatch := matchCronField(c.dayOfWeek, int(t.Weekday()), cronDayOfWeekMin, cronDayOfWeekMax) + domMatch := matchCronField(cronFieldDayOfMonth, c.dayOfMonth, t.Day(), cronDayOfMonthMin, cronDayOfMonthMax) + dowMatch := matchCronField( + cronFieldDayOfWeek, c.dayOfWeek, int(t.Weekday()), cronDayOfWeekMin, cronDayOfWeekMax, + ) return domMatch || dowMatch } } -// matchCronField checks if val matches a cron field (numeric, *, ?, or comma-list). -func matchCronField(field string, val, fieldMin, fieldMax int) bool { +// matchCronField checks if val matches a cron field (numeric, name, *, ?, or comma-list). +func matchCronField(kind cronFieldKind, field string, val, fieldMin, fieldMax int) bool { if field == "*" || field == "?" { return true } for part := range strings.SplitSeq(field, ",") { - if matchCronToken(strings.TrimSpace(part), val, fieldMin, fieldMax) { + if matchCronToken(kind, strings.TrimSpace(part), val, fieldMin, fieldMax) { return true } } @@ -228,30 +318,41 @@ func matchCronField(field string, val, fieldMin, fieldMax int) bool { } // matchCronToken checks whether a single cron token (range, step, or exact) matches val. -func matchCronToken(token string, val, fieldMin, fieldMax int) bool { +func matchCronToken(kind cronFieldKind, token string, val, fieldMin, fieldMax int) bool { switch { case strings.Contains(token, "-"): - return matchCronRange(token, val) + return matchCronRange(kind, token, val) case strings.Contains(token, "/"): - return matchCronStep(token, val, fieldMin, fieldMax) + return matchCronStep(kind, token, val, fieldMin, fieldMax) default: - n, err := strconv.Atoi(token) + n, ok := cronTokenValue(kind, token) - return err == nil && n == val + return ok && n == val } } -// matchCronRange returns true if val falls within the range "lo-hi". -func matchCronRange(token string, val int) bool { +// matchCronRange returns true if val falls within the range "lo-hi" (numeric +// or three-letter names, e.g. "MON-FRI" or "JAN-DEC"). +func matchCronRange(kind cronFieldKind, token string, val int) bool { parts := strings.SplitN(token, "-", rateExpressionFields) - lo, err1 := strconv.Atoi(parts[0]) - hi, err2 := strconv.Atoi(parts[1]) + lo, ok1 := cronTokenValue(kind, parts[0]) + hi, ok2 := cronTokenValue(kind, parts[1]) - return err1 == nil && err2 == nil && val >= lo && val <= hi + if !ok1 || !ok2 { + return false + } + + if kind == cronFieldDayOfWeek && hi < lo { + // A day-of-week range may wrap past Saturday back to Sunday after the + // AWS->Go conversion (e.g. AWS "FRI-MON" / "6-2" spans Fri,Sat,Sun,Mon). + return val >= lo || val <= hi + } + + return val >= lo && val <= hi } // matchCronStep returns true if val matches a step pattern like "*/step" or "start/step". -func matchCronStep(token string, val, fieldMin, fieldMax int) bool { +func matchCronStep(kind cronFieldKind, token string, val, fieldMin, fieldMax int) bool { parts := strings.SplitN(token, "/", rateExpressionFields) step, err := strconv.Atoi(parts[1]) @@ -261,7 +362,12 @@ func matchCronStep(token string, val, fieldMin, fieldMax int) bool { start := fieldMin if parts[0] != "*" { - start, _ = strconv.Atoi(parts[0]) + n, ok := cronTokenValue(kind, parts[0]) + if !ok { + return false + } + + start = n } for v := start; v <= fieldMax; v += step { diff --git a/services/eventbridge/schedule_test.go b/services/eventbridge/schedule_test.go index 8ad8080a6..cde9db2c7 100644 --- a/services/eventbridge/schedule_test.go +++ b/services/eventbridge/schedule_test.go @@ -106,3 +106,78 @@ func TestSchedule_UnsupportedExpression(t *testing.T) { _, err := eventbridge.ParseScheduleExpressionForTest("at(2024-01-15T12:00:00)") require.Error(t, err) } + +// TestSchedule_CronDayOfWeek proves the AWS day-of-week convention (1-7, +// where 1 = Sunday) is honored, both numerically and via SUN-SAT names, and +// that it is NOT confused with Go's time.Weekday (0-6, where 0 = Sunday). +// A prior version of the matcher compared AWS field tokens directly against +// t.Weekday() with no offset, which silently fired schedules one weekday off +// and never matched named ranges like "MON-FRI" at all (parsing succeeded, +// but every candidate tick failed strconv.Atoi("MON") and was rejected). +func TestSchedule_CronDayOfWeek(t *testing.T) { + t.Parallel() + + // utcMidnight returns a UTC midnight timestamp for 2024-01- (or + // 2024-02- when feb is true), used verbatim as the fire time under test. + utcMidnight := func(day int, feb bool) time.Time { + month := time.January + if feb { + month = time.February + } + + return time.Date(2024, month, day, 0, 0, 0, 0, time.UTC) + } + + tests := []struct { + day time.Time + name string + expr string + matches bool + }{ + // 2024-01-14 is a Sunday, 2024-01-15 Monday, ... 2024-01-20 Saturday. + {name: "AWS 1 is Sunday, not Monday", expr: "cron(0 0 ? * 1 *)", day: utcMidnight(14, false), matches: true}, + {name: "AWS 1 does not match Monday", expr: "cron(0 0 ? * 1 *)", day: utcMidnight(15, false), matches: false}, + {name: "AWS 2 is Monday", expr: "cron(0 0 ? * 2 *)", day: utcMidnight(15, false), matches: true}, + {name: "AWS 7 is Saturday", expr: "cron(0 0 ? * 7 *)", day: utcMidnight(20, false), matches: true}, + {name: "SUN name matches Sunday", expr: "cron(0 0 ? * SUN *)", day: utcMidnight(14, false), matches: true}, + { + name: "SUN name does not match Monday", + expr: "cron(0 0 ? * SUN *)", + day: utcMidnight(15, false), + matches: false, + }, + { + name: "MON-FRI matches Wednesday", + expr: "cron(0 0 ? * MON-FRI *)", + day: utcMidnight(17, false), + matches: true, + }, + { + name: "MON-FRI excludes Saturday", + expr: "cron(0 0 ? * MON-FRI *)", + day: utcMidnight(20, false), + matches: false, + }, + {name: "MON-FRI excludes Sunday", expr: "cron(0 0 ? * MON-FRI *)", day: utcMidnight(21, false), matches: false}, + {name: "JAN name matches January", expr: "cron(0 0 1 JAN ? *)", day: utcMidnight(1, false), matches: true}, + {name: "JAN name excludes February", expr: "cron(0 0 1 JAN ? *)", day: utcMidnight(1, true), matches: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + sched, err := eventbridge.ParseScheduleExpressionForTest(tt.expr) + require.NoError(t, err) + + // NextAfter scans forward from just before the candidate; if the + // candidate matches, the next fire time returned is the candidate + // itself (minute-resolution, so subtract a minute to search from). + next := sched.NextAfterForTest(tt.day.Add(-time.Minute)) + if tt.matches { + assert.True(t, next.Equal(tt.day), "expected fire at %v, got %v", tt.day, next) + } else { + assert.False(t, next.Equal(tt.day), "expected no fire at %v, but matched", tt.day) + } + }) + } +} diff --git a/services/eventbridge/scheduler.go b/services/eventbridge/scheduler.go index 8e0af6a61..632acefcf 100644 --- a/services/eventbridge/scheduler.go +++ b/services/eventbridge/scheduler.go @@ -61,7 +61,7 @@ func (s *Scheduler) initLastFired(lastFired map[string]time.Time, now time.Time) for _, regRules := range s.backend.rules { for _, busRules := range regRules { - for _, rule := range busRules { + for _, rule := range busRules.All() { if rule.ScheduleExpression != "" && rule.State == ruleStateEnabled { lastFired[rule.Arn] = now } @@ -87,7 +87,7 @@ func (s *Scheduler) processTick( //nolint:gocognit // existing issue. activeARNs := make(map[string]struct{}) for region, regRules := range s.backend.rules { for busName, busRules := range regRules { - for _, rule := range busRules { + for _, rule := range busRules.All() { if rule.ScheduleExpression != "" && rule.State == ruleStateEnabled { scheduled = append(scheduled, ruleInfo{rule: *rule, busName: busName, region: region}) } diff --git a/services/eventbridge/sfn_integration.go b/services/eventbridge/sfn_integration.go index 475651398..8a23a4519 100644 --- a/services/eventbridge/sfn_integration.go +++ b/services/eventbridge/sfn_integration.go @@ -17,7 +17,10 @@ func (b *InMemoryBackend) SFNPutEvents(ctx context.Context, entries []map[string evtEntries = append(evtEntries, entry) } - results := b.PutEvents(ctx, evtEntries) + results, err := b.PutEvents(ctx, evtEntries) + if err != nil { + return 0, err + } failedCount := 0 for _, r := range results { diff --git a/services/eventbridge/store_setup.go b/services/eventbridge/store_setup.go new file mode 100644 index 000000000..8537f846f --- /dev/null +++ b/services/eventbridge/store_setup.go @@ -0,0 +1,189 @@ +package eventbridge + +// Code in this file supports Phase 3.3 of the datalayer refactor: resource +// maps with a pure per-value identity (a Name/ID field of their own) are +// backed by *store.Table[T] instead of a hand-rolled map[string]*T. See +// pkgs/store's package doc and the services/ec2 (commit 12e611a4), services/sqs +// (commit 0f09d77c), and services/ssm+kms (per-parent lazy registration for +// nested maps) conversions this follows. +// +// # Why this looks different from ec2/sqs +// +// ec2 and sqs each model ONE region per InMemoryBackend instance, so a flat +// map[string]*store.Table[T] statically registered once at construction time +// is enough. eventbridge's InMemoryBackend is unusual in two ways: it holds +// EVERY region's state in one instance (region -> value, like ssm), and two +// of its resources nest a level deeper still: rules (region -> event bus -> +// rule) and targets (region -> bus/rule -> target). store.Registry has no +// concept of "region" or "parent" and requires every name registered on it to +// be known and unique up front -- it cannot itself express "one table per +// region (and per parent), regions/parents unknown until first use". +// +// This file bridges the two: each convertible resource keeps its existing +// map-of-maps shape down to (but not including) the leaf collection, which +// becomes a *store.Table[T] created lazily by getOrCreateTable / +// getOrCreateNestedTable / getOrCreateGlobalTable on first access -- +// exactly mirroring the lazy map creation the hand-rolled code did before +// conversion (e.g. the old `if b.buses[region] == nil { ... }` guards, now +// folded into the accessors in backend.go). The first time a table is +// created it is ALSO registered on b.registry under a name that encodes its +// region/parent (e.g. "rules/us-east-1/default") so that persistence.go can +// still drive Snapshot/Restore through one *store.Registry regardless of how +// many regions/parents exist -- see preRegisterSnapshotTables in +// persistence.go. +// +// pipes, registries, and schemas are NOT region-scoped at all -- a single +// InMemoryBackend instance holds one global Pipe/SchemaRegistry/Schema +// catalogue -- and, unlike every resource above, were NEVER part of +// backendSnapshot before this conversion (see persistence.go's history): +// CreatePipe/CreateRegistry/CreateSchema state has always been silently lost +// across a Restore. Preserving that existing (if surprising) behavior +// byte-for-byte means these three must NOT start round-tripping through +// Snapshot/Restore just because they gained a *store.Table. They are +// therefore registered on b.auxRegistry, a second, never-snapshotted +// *store.Registry that exists solely so getOrCreateTable/ +// getOrCreateGlobalTable still have somewhere to register them (and so a +// construction-time bug that double-registers one still panics) -- see +// backend.go's pipesTable/registriesTable/schemasTableFor. b.registry, by +// contrast, is the one persistence.go drives via SnapshotAll/RestoreAll. +// +// # What is NOT converted here, and why +// +// - archivedEvents (map[string]map[string][]EventEntry, region -> archive +// name -> events): one-to-many -- there is no single V to key a Table by; +// the archived events for one archive stay a slice, same as ec2/ssm's +// history-style slice maps. +// - busePolicies (map[string]map[string]*EventBusPolicy, region -> bus name +// -> policy): EventBusPolicy carries no name field of its own naming the +// bus it belongs to, so there is no pure func(*EventBusPolicy) string to +// write -- same rationale as ssm's tags/miscResourceTags exclusions. +// - schemaVersions (map[string][]*SchemaVersion, keyed by +// "registryName/schemaName"): ordered one-to-many version history, not a +// single keyed collection. +// - codeBindings (map[string]*CodeBinding, keyed by +// "registryName/schemaName/language"): CodeBinding stores Language and +// SchemaVersion but not the registry/schema names that make up the rest +// of its map key, so there is no pure func(*CodeBinding) string that +// reproduces the existing key from the value alone. +import ( + "strings" + + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +func eventBusKeyFn(v *EventBus) string { return v.Name } +func ruleKeyFn(v *Rule) string { return v.Name } +func targetKeyFnV(v *Target) string { return v.ID } +func eventSourceKeyFn(v *EventSource) string { return v.Name } +func replayKeyFn(v *Replay) string { return v.ReplayName } +func apiDestinationKeyFn(v *APIDestination) string { return v.Name } +func archiveKeyFn(v *Archive) string { return v.ArchiveName } +func connectionKeyFn(v *Connection) string { return v.Name } +func endpointKeyFn(v *Endpoint) string { return v.Name } +func partnerEventSourceKeyFn(v *PartnerEventSource) string { return v.Name } +func pipeKeyFn(v *Pipe) string { return v.Name } +func schemaRegistryKeyFn(v *SchemaRegistry) string { return v.RegistryName } +func schemaKeyFn(v *Schema) string { return v.SchemaName } + +// getOrCreateTable returns the *store.Table[V] for key from m (a backend's +// key->Table field for one resource type), creating and registering it on reg +// (under name+"/"+key) the first time that key is touched. m is a Go map +// (reference type), so inserting into it here mutates the same map the +// caller's field refers to. Used both for per-region resources (key=region, +// reg=b.registry) and per-registry resources (key=registryName, +// reg=b.auxRegistry). +func getOrCreateTable[V any]( + reg *store.Registry, + m map[string]*store.Table[V], + name, key string, + keyFn func(*V) string, +) *store.Table[V] { + t, ok := m[key] + if !ok { + t = store.Register(reg, name+"/"+key, store.New(keyFn)) + m[key] = t + } + + return t +} + +// getOrCreateNestedTable is [getOrCreateTable] for resources nested one level +// deeper than a single per-region/per-registry dimension: rules +// (region -> bus -> Table[Rule]) and targets (region -> bus/rule -> Table[Target]). +func getOrCreateNestedTable[V any]( + reg *store.Registry, + m map[string]map[string]*store.Table[V], + name, outerKey, innerKey string, + keyFn func(*V) string, +) *store.Table[V] { + inner := m[outerKey] + if inner == nil { + inner = make(map[string]*store.Table[V]) + m[outerKey] = inner + } + + t, ok := inner[innerKey] + if !ok { + t = store.Register(reg, name+"/"+outerKey+"/"+innerKey, store.New(keyFn)) + inner[innerKey] = t + } + + return t +} + +// getOrCreateGlobalTable is [getOrCreateTable] for resources with no +// region/parent dimension at all (pipes, registries, schemas): a single +// *store.Table[V] field on the backend, registered lazily on first access. +func getOrCreateGlobalTable[V any]( + reg *store.Registry, + cur **store.Table[V], + name string, + keyFn func(*V) string, +) *store.Table[V] { + if *cur == nil { + *cur = store.Register(reg, name, store.New(keyFn)) + } + + return *cur +} + +// tableAccessorsByPrefix maps each PERSISTED convertible resource's +// registry-name prefix to a no-op-return accessor call that forces the lazy +// getOrCreate* helpers to register that resource's table for a given +// (region, parent) pair. persistence.go's Restore uses this to pre-register +// every table present in an incoming snapshot's Tables blob BEFORE calling +// b.registry.RestoreAll -- RestoreAll only restores tables already registered +// on b.registry, so a region/bus a fresh backend has never seen must be +// registered first or its data would be silently dropped. rest is the +// portion of the table name after "/"; for per-region resources it is +// the region; for rules/targets it is "region/parent" (split again by +// preRegisterSnapshotTables since region never contains "/" but bus/rule +// names may). +// +// pipes, registries, and schemas are deliberately absent: they live on +// b.auxRegistry, not b.registry, and never appear in a Tables blob -- see the +// package doc above. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableAccessorsByPrefix = map[string]func(b *InMemoryBackend, rest string){ + "buses": func(b *InMemoryBackend, region string) { b.busesTable(region) }, + "eventSources": func(b *InMemoryBackend, region string) { b.eventSourcesTable(region) }, + "replays": func(b *InMemoryBackend, region string) { b.replaysTable(region) }, + "apiDestinations": func(b *InMemoryBackend, region string) { b.apiDestinationsTable(region) }, + "archives": func(b *InMemoryBackend, region string) { b.archivesTable(region) }, + "connections": func(b *InMemoryBackend, region string) { b.connectionsTable(region) }, + "endpoints": func(b *InMemoryBackend, region string) { b.endpointsTable(region) }, + "partnerSources": func(b *InMemoryBackend, region string) { b.partnerSourcesTable(region) }, + "rules": func(b *InMemoryBackend, rest string) { + // region never contains "/" but bus names may, so only the FIRST "/" + // may be used to split rest back into (region, busKey). + if region, busKey, ok := strings.Cut(rest, "/"); ok { + b.ruleTableFor(region, busKey) + } + }, + "targets": func(b *InMemoryBackend, rest string) { + if region, parentKey, ok := strings.Cut(rest, "/"); ok { + b.targetTableFor(region, parentKey) + } + }, +} diff --git a/services/glue/PARITY.md b/services/glue/PARITY.md new file mode 100644 index 000000000..7f82217d9 --- /dev/null +++ b/services/glue/PARITY.md @@ -0,0 +1,164 @@ +--- +service: glue +sdk_module: aws-sdk-go-v2/service/glue@v1.137.2 +last_audit_commit: 704d7cda +last_audit_date: 2026-07-05 +overall: A # ~1k genuine fixes found and applied this pass +# Per-op or per-op-family status. Values: ok | partial | gap | deferred. +# wire=response/request shape vs SDK; errors=code+HTTP status; state=real mutate/read; persist=in backendSnapshot. +ops: + CreateDatabase: {wire: ok, errors: ok, state: ok, persist: ok} + GetDatabase: {wire: ok, errors: ok, state: ok, persist: ok} + GetDatabases: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateDatabase: {wire: partial, errors: ok, state: ok, persist: ok, note: "DatabaseInput only models Name/Description; real AWS DatabaseInput also has Parameters/LocationUri/CreateTableDefaultPermissions/TargetDatabase (bd: gopherstack-qd3.3, deferred — not fixed this pass)"} + CreateTable: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed this pass: added Parameters/Owner/Retention to Table+TableInput, and full StorageDescriptor (InputFormat/OutputFormat/SerdeInfo/Parameters/BucketColumns/SortColumns/Compressed/NumberOfBuckets/StoredAsSubDirectories) and Column.Parameters, all previously silently dropped"} + GetTable: {wire: ok, errors: ok, state: ok, persist: ok} + GetTables: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed this pass: was returning live backend *Table pointers uncloned (lock-bypass mutation/data-race risk); now clones like GetTable/SearchTables"} + UpdateTable: {wire: ok, errors: ok, state: ok, persist: ok, note: "same field-completeness fix as CreateTable"} + DeleteTable: {wire: ok, errors: ok, state: ok, persist: ok} + BatchDeleteTable: {wire: ok, errors: ok, state: ok, persist: ok} + GetTableVersions: {wire: ok, errors: ok, state: ok, persist: ok, note: "not re-verified in depth this pass; existing coverage looked correct"} + DeleteTableVersion: {wire: ok, errors: ok, state: ok, persist: ok} + BatchDeleteTableVersion: {wire: ok, errors: ok, state: ok, persist: ok} + CreatePartition: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed this pass: BatchCreatePartition (which CreatePartition delegates to) never checked the parent table existed, silently storing orphaned partitions against a nonexistent db/table; now returns EntityNotFoundException per AWS contract. Also added Partition/PartitionInput.Parameters + Partition.CreationTime/CatalogId"} + BatchCreatePartition: {wire: ok, errors: ok, state: ok, persist: ok, note: "same table-existence fix as CreatePartition"} + GetPartition: {wire: ok, errors: ok, state: ok, persist: ok} + GetPartitions: {wire: ok, errors: ok, state: ok, persist: ok, note: "expression filter (segment) not re-verified in depth this pass"} + BatchGetPartition: {wire: ok, errors: ok, state: ok, persist: ok, note: "SEVERE fix this pass: was a disguised stub — always returned an empty Partitions list regardless of backend state, with a comment falsely claiming \"the mock backend has no partition storage\". Now looks up each PartitionsToGet entry via GetPartition and reports misses in UnprocessedKeys per the real BatchGetPartitionResponse shape"} + UpdatePartition: {wire: ok, errors: ok, state: ok, persist: ok, note: "now also persists Parameters through both the in-place and rename paths"} + BatchUpdatePartition: {wire: ok, errors: ok, state: ok, persist: ok} + DeletePartition: {wire: ok, errors: ok, state: ok, persist: ok} + BatchDeletePartition: {wire: ok, errors: ok, state: ok, persist: ok} + CreatePartitionIndex: {wire: ok, errors: ok, state: ok, persist: ok, note: "not re-verified in depth this pass"} + GetPartitionIndexes: {wire: ok, errors: ok, state: ok, persist: ok} + DeletePartitionIndex: {wire: ok, errors: ok, state: ok, persist: ok} + CreateCrawler: {wire: partial, errors: ok, state: ok, persist: ok, note: "fixed this pass (additively): CreateCrawler's positional signature is called from services/cloudformation (external package) so it was kept unchanged; added CreateCrawlerWithOptions(...,CrawlerOptions) carrying Schedule/Classifiers/Configuration/TablePrefix/Description, which the Glue handler now uses. CrawlerTarget gained JdbcTargets/CatalogTargets (was S3-only) and S3Target/JDBCTarget gained Exclusions. Still deferred: SchemaChangePolicy, RecrawlPolicy, LineageConfiguration, CrawlerSecurityConfiguration, LakeFormationConfiguration, DynamoDB/Delta/Hudi/Iceberg/MongoDB targets"} + GetCrawler: {wire: ok, errors: ok, state: ok, persist: ok} + GetCrawlers: {wire: ok, errors: ok, state: ok, persist: ok} + ListCrawlers: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateCrawler: {wire: partial, errors: ok, state: ok, persist: ok, note: "same additive CrawlerOptions fix as CreateCrawler; also fixed a missing CrawlerRunningException guard (UpdateCrawler previously allowed updating a RUNNING/STARTING/STOPPING crawler, unlike DeleteCrawler which already checked this)"} + DeleteCrawler: {wire: ok, errors: ok, state: ok, persist: ok} + StartCrawler: {wire: ok, errors: ok, state: ok, persist: ok} + StopCrawler: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateCrawlerSchedule: {wire: ok, errors: ok, state: ok, persist: ok} + StartCrawlerSchedule: {wire: ok, errors: ok, state: ok, persist: ok} + StopCrawlerSchedule: {wire: ok, errors: ok, state: ok, persist: ok} + GetCrawlerMetrics: {wire: ok, errors: ok, state: ok, persist: ok, note: "not re-verified in depth this pass"} + CreateJob: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed this pass: added Job.MaxCapacity + NotificationProperty (previously missing entirely — the MaxCapacity vs WorkerType/NumberOfWorkers axis named explicitly in the audit brief), plus AWS's documented mutual-exclusion validation between MaxCapacity and WorkerType/NumberOfWorkers"} + GetJob: {wire: ok, errors: ok, state: ok, persist: ok} + GetJobs: {wire: ok, errors: ok, state: ok, persist: ok} + BatchGetJobs: {wire: ok, errors: ok, state: ok, persist: ok, note: "not re-verified in depth this pass"} + UpdateJob: {wire: ok, errors: ok, state: ok, persist: ok, note: "same MaxCapacity/NotificationProperty fix as CreateJob"} + DeleteJob: {wire: ok, errors: ok, state: ok, persist: ok} + StartJobRun: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed this pass: JobRun now carries WorkerType/NumberOfWorkers/MaxCapacity/GlueVersion/Timeout inherited from the job at start time (previously GetJobRun told callers nothing about what capacity the run used). No per-run override support (StartJobRunRequest overrides not modeled) — deferred"} + GetJobRun: {wire: ok, errors: ok, state: ok, persist: ok} + GetJobRuns: {wire: ok, errors: ok, state: ok, persist: ok} + BatchStopJobRun: {wire: ok, errors: ok, state: ok, persist: ok} + GetJobBookmark: {wire: ok, errors: ok, state: ok, persist: ok, note: "not re-verified in depth this pass"} + ResetJobBookmark: {wire: ok, errors: ok, state: ok, persist: ok} + TagResource: {wire: ok, errors: ok, state: ok, persist: ok} + UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} + GetTags: {wire: ok, errors: ok, state: ok, persist: ok} +families: + connections: {status: deferred, note: "structural wire shape looked plausible (ConnectionType/ConnectionProperties/Tags) but PhysicalConnectionRequirements, MatchCriteria and connection-type-specific validation not audited this pass"} + triggers: {status: deferred, note: "Trigger struct (Predicate/Actions/Schedule/Type/State) present; predicate-condition and action-shape accuracy vs types.Predicate/types.Action not re-verified this pass"} + workflows: {status: deferred, note: "not audited this pass"} + dev_endpoints: {status: deferred, note: "not audited this pass"} + security_configurations: {status: deferred, note: "not audited this pass"} + schema_registry: {status: deferred, note: "not audited this pass (CreateRegistry/CreateSchema/RegisterSchemaVersion/GetSchemaByDefinition/compatibility/AVRO-JSON-PROTOBUF validation)"} + data_quality_rulesets: {status: deferred, note: "not audited this pass"} + ml_transforms: {status: deferred, note: "not audited this pass"} + blueprints: {status: deferred, note: "not audited this pass"} + user_defined_functions: {status: deferred, note: "not audited this pass"} + resource_policy: {status: deferred, note: "not audited this pass"} + error_codes_global: {status: ok, note: "SEVERE systemic fix this pass: the shared ErrValidation sentinel wired \"ValidationException\" as its wire __type — confirmed against aws-sdk-go-v2/service/glue/deserializers.go that the vast majority of Create/Update/Delete operations (CreateDatabase, CreateTable, CreateJob, CreateCrawler, CreateTrigger, CreateBlueprint, CreateCustomEntityType, CreateUsageProfile, tag validation, ...) document InvalidInputException instead. Changed the shared sentinel + handler.go's hardcoded mapping to InvalidInputException, and fixed the ~8 existing tests that had encoded the wrong wire code. Also fixed awserrFromDetail (handler_stubs.go), which always wrapped batch-operation ErrorDetail as awserr.ErrNotFound regardless of the actual ErrorCode string — so e.g. an AlreadyExistsException detail from BatchCreatePartition surfaced to CreatePartition callers as EntityNotFoundException. Not touched: IdempotentParameterMismatchException, ResourceNumberLimitExceededException, OperationTimeoutException, ConcurrentModificationException remain unused — no account-level quota/concurrency-conflict modeling exists to trigger them realistically (bd: gopherstack-qd3.5)"} +gaps: + - CrawlerTarget missing DynamoDBTargets/DeltaTargets/HudiTargets/IcebergTargets/MongoDBTargets (only S3/JDBC/Catalog modeled) (bd: gopherstack-qd3.1) + - CreateCrawler/UpdateCrawler missing SchemaChangePolicy, RecrawlPolicy, LineageConfiguration, CrawlerSecurityConfiguration, LakeFormationConfiguration (bd: gopherstack-qd3.2) + - DatabaseInput/Database missing Parameters, LocationUri, CreateTableDefaultPermissions, TargetDatabase (bd: gopherstack-qd3.3) + - StartJobRun has no per-run capacity/argument overrides (WorkerType/NumberOfWorkers/MaxCapacity/Timeout/NotificationProperty are inherited from the job only, not overridable per AWS's StartJobRunRequest) (bd: gopherstack-qd3.4) + - IdempotentParameterMismatchException/ResourceNumberLimitExceededException/OperationTimeoutException/ConcurrentModificationException are documented Glue exceptions never returned by this backend (no quota/idempotency-token/concurrency-conflict modeling) (bd: gopherstack-qd3.5) +deferred: + - triggers (predicate/action wire-shape depth) + - workflows + - dev endpoints + - security configurations + - schema registry (compatibility modes, AVRO/JSON/PROTOBUF validation depth) + - data quality rulesets + - ML transforms + - blueprints + - user-defined functions + - resource policy + - connections (PhysicalConnectionRequirements/MatchCriteria depth) +leaks: {status: clean, note: "backend_reconciler.go's managed goroutine (StartReconciler/StopReconciler/reconcileLoop) already exits deterministically on ctx.Done() or the stop channel with a WaitGroup — no unmanaged 'go b.runReconciler()' leak. Verified with go test -race; no new goroutines/timers introduced this pass."} +--- + +## Notes + +- **Protocol**: json-1.1 (`X-Amz-Target: AWSGlue.`, `application/x-amz-json-1.1`), + confirmed against `aws-sdk-go-v2/service/glue/deserializers.go`'s + `awsAwsjson11_deserializeOpError` switch statements. Error responses use + `{"__type": "", "message": "..."}`. + +- **ValidationException vs InvalidInputException (important, easy to re-flag by + mistake)**: Glue's SDK error model genuinely contains BOTH exception types. + `ValidationException` IS a real type in `types/errors.go`, and a handful of newer + operations (confirmed: `DeleteConnectionType`) do declare it as a documented error. + But the overwhelming majority of hand-validation call sites in this backend + (name-length checks, tag-limit checks, required-field checks across + Create/Update/Delete for databases/tables/jobs/crawlers/triggers/blueprints/ + custom-entity-types/usage-profiles) correspond to AWS operations whose deserializer + switch lists `InvalidInputException`, not `ValidationException` — confirmed by + reading the actual `awsAwsjson11_deserializeOpErrorCreateXxx` functions in + `deserializers.go` for CreateDatabase, CreateTable, CreateJob, CreateCrawler, + CreateTrigger, CreateBlueprint, CreateCustomEntityType, CreateUsageProfile. Since + `ErrValidation` is one shared sentinel used everywhere, the fix picks the option + that's correct for the large majority of call sites. Do not "fix" this back to + ValidationException without checking the SDK deserializer for the specific op in + question first. + +- **`awserrFromDetail` (handler_stubs.go)**: single-item AWS ops that are implemented + by calling a batch backend method with a one-element slice (CreatePartition → + BatchCreatePartition, DeletePartition → BatchDeletePartition) surface + `errs[0].ErrorDetail` as a real Go error via this helper. It must switch on + `d.ErrorCode` to pick the matching sentinel (AlreadyExists vs NotFound vs generic + invalid-parameter) — do not revert it to unconditionally wrapping + `awserr.ErrNotFound`, or AlreadyExistsException details get reported to SDK callers + as EntityNotFoundException. + +- **StorageDescriptor is shared by Table AND Partition** in real Glue (partitions + carry their own StorageDescriptor that can override table-level SerDe/format + settings). Because `CreateTable`/`UpdateTable`/`BatchCreatePartition`/ + `UpdatePartition` already copy the whole `StorageDescriptor` struct by value from + the request input, adding fields to the `StorageDescriptor`/`Column` type + definitions was sufficient to flow them through end-to-end — the remaining real + work was fixing `cloneTable`/`clonePartition`/`cloneCrawler` to deep-copy the new + nested maps/slices/pointers (Parameters maps, SerdeInfo pointer, BucketColumns/ + SortColumns slices, per-Column Parameters) so that `GetTable`/`GetPartitions` + callers can't mutate live backend state through the returned pointers. + +- **`CreateCrawler`/`UpdateCrawler` signature is called from + `services/cloudformation/resources_phase5.go`** (outside this package) with the + original 5-arg / 4-arg positional signatures. Per the audit's signature-safety + rule, those signatures were left untouched; new capability (Schedule, Classifiers, + Configuration, TablePrefix, Description) was added via new + `CreateCrawlerWithOptions`/`UpdateCrawlerWithOptions` methods that the old methods + now delegate to with a zero-value `CrawlerOptions`. The `StorageBackend` interface + gained the two new methods additively; `InMemoryBackend` is the only implementer + (verified — no mocks reference `StorageBackend` in this package's tests), so this + is safe. + +- **GetTables aliasing bug**: `GetTables` was the one read path in the whole backend + that hadn't been updated to clone before returning (`GetDatabases`, `GetCrawlers`, + `GetJobs`, `GetConnections`, `SearchTables`, `GetPartition(s)` all already cloned). + Fixed to match the established pattern; verified no other `Get*` list method has + the same gap. + +## Follow-ups filed as SHARED-FILE / cross-service (NOT edited this pass) + +None required code changes outside `services/glue/`. The one cross-package touch +point — `services/cloudformation/resources_phase5.go`'s calls into +`gluebackend.CreateCrawler`/`BatchCreatePartition`/`CreateJob`/`CreateTrigger` — was +verified to still compile and its test (`services/cloudformation` package tests) +still passes, because all Glue-side changes were additive (new struct fields, new +`*WithOptions` methods) rather than signature-breaking. diff --git a/services/glue/backend.go b/services/glue/backend.go index 34b64d9e7..76051c5d6 100644 --- a/services/glue/backend.go +++ b/services/glue/backend.go @@ -12,8 +12,8 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awserr" - "github.com/blackbirdworks/gopherstack/pkgs/collections" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) var ( @@ -22,7 +22,17 @@ var ( // ErrAlreadyExists is returned when a resource already exists. ErrAlreadyExists = awserr.New("AlreadyExistsException", awserr.ErrAlreadyExists) // ErrValidation is returned when input validation fails. - ErrValidation = awserr.New("ValidationException", awserr.ErrInvalidParameter) + // + // Glue's per-operation error models (aws-sdk-go-v2/service/glue deserializers.go) + // list InvalidInputException — not ValidationException — as the hand-validation + // error for the overwhelming majority of Create/Update/Delete operations (e.g. + // CreateDatabase, CreateTable, CreateJob, CreateCrawler, CreateTrigger, + // CreateBlueprint, CreateCustomEntityType, CreateUsageProfile, tag validation). + // A handful of newer operations (e.g. DeleteConnectionType) do document + // ValidationException instead, but since this sentinel is shared across every + // hand-rolled validation check in the backend, InvalidInputException is the more + // accurate default. + ErrValidation = awserr.New("InvalidInputException", awserr.ErrInvalidParameter) // ErrCrawlerRunning is returned when an operation requires the crawler to not be running. ErrCrawlerRunning = awserr.New("CrawlerRunningException", awserr.ErrInvalidParameter) // ErrCrawlerNotRunning is returned when an operation requires the crawler to be running. @@ -108,61 +118,117 @@ type Database struct { // Column represents a column in a Glue table. type Column struct { - Name string `json:"Name"` - Type string `json:"Type,omitempty"` - Comment string `json:"Comment,omitempty"` + Parameters map[string]string `json:"Parameters,omitempty"` + Name string `json:"Name"` + Type string `json:"Type,omitempty"` + Comment string `json:"Comment,omitempty"` } -// StorageDescriptor describes the physical storage of a table. +// SerDeInfo holds the serialization/deserialization information for a +// StorageDescriptor, mirroring aws-sdk-go-v2/service/glue/types.SerDeInfo. +type SerDeInfo struct { + Parameters map[string]string `json:"Parameters,omitempty"` + Name string `json:"Name,omitempty"` + SerializationLibrary string `json:"SerializationLibrary,omitempty"` +} + +// Order specifies the sort order of a column, mirroring +// aws-sdk-go-v2/service/glue/types.Order. +type Order struct { + Column string `json:"Column"` + SortOrder int `json:"SortOrder"` +} + +// StorageDescriptor describes the physical storage of a table or partition. type StorageDescriptor struct { - Location string `json:"Location,omitempty"` - Columns []Column `json:"Columns,omitempty"` + SerdeInfo *SerDeInfo `json:"SerdeInfo,omitempty"` + Parameters map[string]string `json:"Parameters,omitempty"` + Location string `json:"Location,omitempty"` + InputFormat string `json:"InputFormat,omitempty"` + OutputFormat string `json:"OutputFormat,omitempty"` + Columns []Column `json:"Columns,omitempty"` + BucketColumns []string `json:"BucketColumns,omitempty"` + SortColumns []Order `json:"SortColumns,omitempty"` + NumberOfBuckets int `json:"NumberOfBuckets,omitempty"` + Compressed bool `json:"Compressed,omitempty"` + StoredAsSubDirectories bool `json:"StoredAsSubDirectories,omitempty"` } // TableInput is the input for creating or updating a Glue table. type TableInput struct { - StorageDescriptor StorageDescriptor `json:"StorageDescriptor,omitzero"` + Parameters map[string]string `json:"Parameters,omitempty"` Name string `json:"Name"` Description string `json:"Description,omitempty"` + Owner string `json:"Owner,omitempty"` TableType string `json:"TableType,omitempty"` PartitionKeys []Column `json:"PartitionKeys,omitempty"` + StorageDescriptor StorageDescriptor `json:"StorageDescriptor,omitzero"` + Retention int `json:"Retention,omitempty"` } // Table represents a Glue catalog table. type Table struct { - StorageDescriptor StorageDescriptor `json:"StorageDescriptor,omitzero"` + Parameters map[string]string `json:"Parameters,omitempty"` Name string `json:"Name"` DatabaseName string `json:"DatabaseName"` CatalogID string `json:"CatalogId"` Description string `json:"Description,omitempty"` + Owner string `json:"Owner,omitempty"` TableType string `json:"TableType,omitempty"` PartitionKeys []Column `json:"PartitionKeys,omitempty"` + StorageDescriptor StorageDescriptor `json:"StorageDescriptor,omitzero"` + Retention int `json:"Retention,omitempty"` CreateTime float64 `json:"CreateTime,omitempty"` UpdateTime float64 `json:"UpdateTime,omitempty"` } -// CrawlerTarget specifies S3 targets for a crawler. +// CrawlerTarget specifies the data stores a crawler scans. AWS supports many +// target store kinds (S3, JDBC, catalog, DynamoDB, Delta, Hudi, Iceberg, +// MongoDB); this backend models the three most commonly used ones. Additional +// kinds are deferred (see PARITY.md). type CrawlerTarget struct { - S3Targets []S3Target `json:"S3Targets,omitempty"` + S3Targets []S3Target `json:"S3Targets,omitempty"` + JdbcTargets []JDBCTarget `json:"JdbcTargets,omitempty"` + CatalogTargets []CatalogTarget `json:"CatalogTargets,omitempty"` } // S3Target is an S3 path for a crawler. type S3Target struct { - Path string `json:"Path,omitempty"` + Path string `json:"Path,omitempty"` + Exclusions []string `json:"Exclusions,omitempty"` +} + +// JDBCTarget is a JDBC connection/path pair for a crawler, mirroring +// aws-sdk-go-v2/service/glue/types.JdbcTarget. +type JDBCTarget struct { + ConnectionName string `json:"ConnectionName,omitempty"` + Path string `json:"Path,omitempty"` + Exclusions []string `json:"Exclusions,omitempty"` +} + +// CatalogTarget synchronizes an existing Data Catalog database/tables as a +// crawler target, mirroring aws-sdk-go-v2/service/glue/types.CatalogTarget. +type CatalogTarget struct { + DatabaseName string `json:"DatabaseName,omitempty"` + Tables []string `json:"Tables,omitempty"` } // Crawler represents a Glue crawler. type Crawler struct { - Tags map[string]string `json:"-"` - Schedule CrawlerSchedule `json:"Schedule,omitzero"` - Name string `json:"Name"` - Role string `json:"Role"` - DatabaseName string `json:"DatabaseName"` - State string `json:"State"` - ARN string `json:"Arn,omitempty"` - Targets CrawlerTarget `json:"Targets,omitzero"` - CreationTime float64 `json:"CreationTime,omitempty"` - LastUpdated float64 `json:"LastUpdated,omitempty"` + Tags map[string]string `json:"-"` + Schedule CrawlerSchedule `json:"Schedule,omitzero"` + Name string `json:"Name"` + Role string `json:"Role"` + DatabaseName string `json:"DatabaseName"` + State string `json:"State"` + ARN string `json:"Arn,omitempty"` + Description string `json:"Description,omitempty"` + Configuration string `json:"Configuration,omitempty"` + TablePrefix string `json:"TablePrefix,omitempty"` + Classifiers []string `json:"Classifiers,omitempty"` + Targets CrawlerTarget `json:"Targets,omitzero"` + CreationTime float64 `json:"CreationTime,omitempty"` + LastUpdated float64 `json:"LastUpdated,omitempty"` } // CrawlHistoryEntry records a single crawl run for ListCrawls. @@ -204,12 +270,24 @@ type Job struct { ARN string `json:"Arn,omitempty"` Description string `json:"Description,omitempty"` Connections ConnectionsList `json:"Connections,omitzero"` + NotificationProperty NotificationProperty `json:"NotificationProperty,omitzero"` NumberOfWorkers int `json:"NumberOfWorkers,omitempty"` MaxRetries int `json:"MaxRetries,omitempty"` Timeout int `json:"Timeout,omitempty"` - ExecutionProperty ExecutionProperty `json:"ExecutionProperty,omitzero"` - CreatedOn float64 `json:"CreatedOn,omitempty"` - LastModifiedOn float64 `json:"LastModifiedOn,omitempty"` + // MaxCapacity is the DPU capacity for jobs that use it instead of + // WorkerType+NumberOfWorkers (e.g. Python shell jobs, or Spark jobs on + // Glue versions that predate worker-type based capacity). AWS rejects a + // request that sets both MaxCapacity and WorkerType/NumberOfWorkers. + MaxCapacity float64 `json:"MaxCapacity,omitempty"` + ExecutionProperty ExecutionProperty `json:"ExecutionProperty,omitzero"` + CreatedOn float64 `json:"CreatedOn,omitempty"` + LastModifiedOn float64 `json:"LastModifiedOn,omitempty"` +} + +// NotificationProperty specifies the delay, in minutes, after which a job run +// notification is sent (JobRun.NotificationProperty / Job.NotificationProperty). +type NotificationProperty struct { + NotifyDelayAfter int `json:"NotifyDelayAfter,omitempty"` } // SourceControlDetails records the remote-repository link for a job synchronized @@ -238,16 +316,20 @@ type PartitionValueList struct { // PartitionInput is the input for creating a partition. type PartitionInput struct { - StorageDescriptor StorageDescriptor `json:"StorageDescriptor,omitzero"` + Parameters map[string]string `json:"Parameters,omitempty"` Values []string `json:"Values"` + StorageDescriptor StorageDescriptor `json:"StorageDescriptor,omitzero"` } // Partition represents a Glue table partition. type Partition struct { - StorageDescriptor StorageDescriptor `json:"StorageDescriptor,omitzero"` + Parameters map[string]string `json:"Parameters,omitempty"` DatabaseName string `json:"DatabaseName"` TableName string `json:"TableName"` + CatalogID string `json:"CatalogId,omitempty"` Values []string `json:"Values"` + StorageDescriptor StorageDescriptor `json:"StorageDescriptor,omitzero"` + CreationTime float64 `json:"CreationTime,omitempty"` } // PartitionError represents an error for a single partition operation. @@ -320,14 +402,19 @@ type CrawlerSchedule struct { // JobRun represents a single execution of a Glue job. type JobRun struct { - Arguments map[string]string `json:"Arguments,omitempty"` - ID string `json:"Id"` - JobName string `json:"JobName"` - JobRunState string `json:"JobRunState"` - ErrorMessage string `json:"ErrorMessage,omitempty"` - StartedOn float64 `json:"StartedOn,omitempty"` - CompletedOn float64 `json:"CompletedOn,omitempty"` - ExecutionTime int `json:"ExecutionTime,omitempty"` + Arguments map[string]string `json:"Arguments,omitempty"` + ID string `json:"Id"` + JobName string `json:"JobName"` + JobRunState string `json:"JobRunState"` + ErrorMessage string `json:"ErrorMessage,omitempty"` + WorkerType string `json:"WorkerType,omitempty"` + GlueVersion string `json:"GlueVersion,omitempty"` + StartedOn float64 `json:"StartedOn,omitempty"` + CompletedOn float64 `json:"CompletedOn,omitempty"` + MaxCapacity float64 `json:"MaxCapacity,omitempty"` + ExecutionTime int `json:"ExecutionTime,omitempty"` + NumberOfWorkers int `json:"NumberOfWorkers,omitempty"` + Timeout int `json:"Timeout,omitempty"` } // JobBookmark holds the bookmark state for a job run. @@ -367,57 +454,64 @@ type DataQualityEvaluationRun struct { CompletedOn float64 `json:"CompletedOn,omitempty"` } -// InMemoryBackend stores Glue state in memory. +// InMemoryBackend stores Glue state in memory. Most resource collections are +// *store.Table[T], registered once on b.registry via registerAllTables (see +// store_setup.go); a handful remain plain maps because their key is not a +// pure function of the stored value's own fields, or because they hold a +// one-to-many history/list rather than a single value per key -- see the +// comment above registerAllTables in store_setup.go for the full list and +// rationale. type InMemoryBackend struct { - databases map[string]*Database // key: databaseName - tables map[string]*Table // key: "databaseName|tableName" - crawlers map[string]*Crawler // key: crawlerName - jobs map[string]*Job // key: jobName - partitions map[string]*Partition // key: partitionKey(db, table, values) - partitionIndexes map[string]*PartitionIndex // key: "databaseName|tableName|indexName" - tableVersions map[string]*TableVersion // key: tableVersionKey(db, table, versionID) - connections map[string]*Connection // key: connectionName - blueprints map[string]*Blueprint // key: blueprintName - customEntityTypes map[string]*CustomEntityType // key: name - dataQualityResult map[string]*DataQualityResult // key: resultID - devEndpoints map[string]*DevEndpoint // key: endpointName - jobRuns map[string][]*JobRun // key: jobName - jobBookmarks map[string]*JobBookmark // key: jobName - dataQualityRulesets map[string]*DataQualityRuleset // key: name - dataQualityEvalRuns map[string]*DataQualityEvaluationRun // key: runId - triggers map[string]*Trigger // key: triggerName - workflows map[string]*Workflow // key: workflowName - workflowRuns map[string][]*WorkflowRun // key: workflowName - classifiers map[string]*Classifier // key: classifierName - registries map[string]*Registry // key: registryName - schemas map[string]*Schema // key: "registryName|schemaName" - schemaVersions map[string][]*SchemaVersion // key: schemaARN - udfs map[string]*UserDefinedFunction // key: "dbName|udfName" - securityConfigs map[string]*SecurityConfiguration // key: name - sessions map[string]*Session // key: sessionID - sessionStatements map[string][]*Statement // key: sessionID - tableOptimizers map[string]*TableOptimizer // key: "dbName|tableName|type" - tableColumnStats map[string]*ColumnStatistics // key: "dbName|tableName|colName" - partitionColumnStats map[string]*ColumnStatistics // key: partKey+"|"+colName - resourcePolicies map[string]*resourcePolicyEntry // key: resourceARN or "__global__" - mlTransforms map[string]*MLTransform // key: transformID - catalogs map[string]*CatalogEntry // key: catalogID + databases *store.Table[Database] + tables *store.Table[Table] + crawlers *store.Table[Crawler] + jobs *store.Table[Job] + partitions *store.Table[Partition] + partitionIndexes map[string]*PartitionIndex // key: "databaseName|tableName|indexName" + tableVersions *store.Table[TableVersion] + connections *store.Table[Connection] + blueprints *store.Table[Blueprint] + customEntityTypes *store.Table[CustomEntityType] + dataQualityResult *store.Table[DataQualityResult] + devEndpoints *store.Table[DevEndpoint] + jobRuns map[string][]*JobRun // key: jobName + jobBookmarks *store.Table[JobBookmark] + dataQualityRulesets *store.Table[DataQualityRuleset] + dataQualityEvalRuns *store.Table[DataQualityEvaluationRun] + triggers *store.Table[Trigger] + workflows *store.Table[Workflow] + workflowRuns map[string][]*WorkflowRun // key: workflowName + classifiers *store.Table[Classifier] + registries *store.Table[Registry] + schemas *store.Table[Schema] + schemaVersions map[string][]*SchemaVersion // key: schemaARN + udfs *store.Table[UserDefinedFunction] + securityConfigs *store.Table[SecurityConfiguration] + sessions *store.Table[Session] + sessionStatements map[string][]*Statement // key: sessionID + tableOptimizers *store.Table[TableOptimizer] + tableColumnStats map[string]*ColumnStatistics // key: "dbName|tableName|colName" + partitionColumnStats map[string]*ColumnStatistics // key: partKey+"|"+colName + resourcePolicies map[string]*resourcePolicyEntry // key: resourceARN or "__global__" + mlTransforms *store.Table[MLTransform] + catalogs *store.Table[CatalogEntry] catalogEncryptionSettings map[string]*DataCatalogEncryptionSettings // key: catalogID or accountID - usageProfiles map[string]*UsageProfile // key: name - blueprintRuns map[string]*BlueprintRun // key: runID - dqRecommendationRuns map[string]*DQRuleRecommendationRun // key: runID - columnStatTaskSettings map[string]*ColumnStatisticsTaskSettings // key: "dbName|tableName" - columnStatTaskRuns map[string]*ColumnStatisticsTaskRun // key: runID - materializedViewRuns map[string]*MaterializedViewRefreshRun // key: taskRunID - integrations map[string]*Integration // key: integrationName - integrationResourceProps map[string]*IntegrationResourceProperty // key: resourceARN - integrationTableProps map[string]*IntegrationTableProperties // key: "resourceARN|tableName" - mlTaskRuns map[string]*MLTaskRun // key: "transformID|taskRunID" - catalogImports map[string]*CatalogImportStatus // key: catalogID or accountID - schemaVersionMetadata map[string]map[string]string // key: schemaVersionID → key → value - crawlHistory map[string][]*CrawlHistoryEntry // key: crawlerName - dqStatisticAnnotations map[string]*StatisticAnnotation // key: "profileID|statisticID" + usageProfiles *store.Table[UsageProfile] + blueprintRuns *store.Table[BlueprintRun] + dqRecommendationRuns *store.Table[DQRuleRecommendationRun] + columnStatTaskSettings *store.Table[ColumnStatisticsTaskSettings] + columnStatTaskRuns *store.Table[ColumnStatisticsTaskRun] + materializedViewRuns *store.Table[MaterializedViewRefreshRun] + integrations *store.Table[Integration] + integrationResourceProps *store.Table[IntegrationResourceProperty] + integrationTableProps *store.Table[IntegrationTableProperties] + mlTaskRuns *store.Table[MLTaskRun] + catalogImports map[string]*CatalogImportStatus // key: catalogID or accountID + schemaVersionMetadata map[string]map[string]string // key: schemaVersionID → key → value + crawlHistory map[string][]*CrawlHistoryEntry // key: crawlerName + dqStatisticAnnotations *store.Table[StatisticAnnotation] glueIdentityCenterConfig *IdentityCenterConfig + registry *store.Registry mu *lockmetrics.RWMutex // lifecycle reconciler timers @@ -426,7 +520,7 @@ type InMemoryBackend struct { crawlerReadyAt map[string]time.Time // crawlerName → readyAt for RUNNING→READY // connection-type registry (custom types registered via RegisterConnectionType). - customConnectionTypes map[string]*ConnectionTypeInfo + customConnectionTypes *store.Table[ConnectionTypeInfo] // reconcileStop signals the managed reconciler goroutine to exit. See the // non-pointer reconciler bookkeeping fields at the end of the struct. @@ -450,63 +544,29 @@ type InMemoryBackend struct { // NewInMemoryBackend creates a new in-memory Glue backend. func NewInMemoryBackend(accountID, region string) *InMemoryBackend { b := &InMemoryBackend{ - databases: make(map[string]*Database), - tables: make(map[string]*Table), - crawlers: make(map[string]*Crawler), - jobs: make(map[string]*Job), - partitions: make(map[string]*Partition), partitionIndexes: make(map[string]*PartitionIndex), - tableVersions: make(map[string]*TableVersion), - connections: make(map[string]*Connection), - blueprints: make(map[string]*Blueprint), - customEntityTypes: make(map[string]*CustomEntityType), - dataQualityResult: make(map[string]*DataQualityResult), - devEndpoints: make(map[string]*DevEndpoint), jobRuns: make(map[string][]*JobRun), - jobBookmarks: make(map[string]*JobBookmark), - dataQualityRulesets: make(map[string]*DataQualityRuleset), - dataQualityEvalRuns: make(map[string]*DataQualityEvaluationRun), - triggers: make(map[string]*Trigger), - workflows: make(map[string]*Workflow), workflowRuns: make(map[string][]*WorkflowRun), - classifiers: make(map[string]*Classifier), - registries: make(map[string]*Registry), - schemas: make(map[string]*Schema), schemaVersions: make(map[string][]*SchemaVersion), - udfs: make(map[string]*UserDefinedFunction), - securityConfigs: make(map[string]*SecurityConfiguration), - sessions: make(map[string]*Session), sessionStatements: make(map[string][]*Statement), - tableOptimizers: make(map[string]*TableOptimizer), tableColumnStats: make(map[string]*ColumnStatistics), partitionColumnStats: make(map[string]*ColumnStatistics), resourcePolicies: make(map[string]*resourcePolicyEntry), - mlTransforms: make(map[string]*MLTransform), - catalogs: make(map[string]*CatalogEntry), catalogEncryptionSettings: make(map[string]*DataCatalogEncryptionSettings), - usageProfiles: make(map[string]*UsageProfile), - blueprintRuns: make(map[string]*BlueprintRun), - dqRecommendationRuns: make(map[string]*DQRuleRecommendationRun), - columnStatTaskSettings: make(map[string]*ColumnStatisticsTaskSettings), - columnStatTaskRuns: make(map[string]*ColumnStatisticsTaskRun), - materializedViewRuns: make(map[string]*MaterializedViewRefreshRun), - integrations: make(map[string]*Integration), - integrationResourceProps: make(map[string]*IntegrationResourceProperty), - integrationTableProps: make(map[string]*IntegrationTableProperties), - mlTaskRuns: make(map[string]*MLTaskRun), catalogImports: make(map[string]*CatalogImportStatus), schemaVersionMetadata: make(map[string]map[string]string), crawlHistory: make(map[string][]*CrawlHistoryEntry), - dqStatisticAnnotations: make(map[string]*StatisticAnnotation), + registry: store.NewRegistry(), mu: lockmetrics.New("glue"), accountID: accountID, region: region, jobRunReadyAt: make(map[string]map[string]time.Time), jobRunDoneAt: make(map[string]map[string]time.Time), crawlerReadyAt: make(map[string]time.Time), - customConnectionTypes: make(map[string]*ConnectionTypeInfo), } + registerAllTables(b) + return b } @@ -555,7 +615,7 @@ func (b *InMemoryBackend) reconcileLocked(now time.Time) { // without creating tables (the crawl was interrupted). for name, readyAt := range b.crawlerReadyAt { if now.After(readyAt) { - c, ok := b.crawlers[name] + c, ok := b.crawlers.Get(name) if ok && c.State == stateRunning { c.State = stateReady c.LastUpdated = float64(now.Unix()) @@ -638,12 +698,12 @@ func (b *InMemoryBackend) createCrawlerTablesLocked(c *Crawler) int { } key := c.DatabaseName + "|" + tableName - if _, exists := b.tables[key]; !exists { - b.tables[key] = &Table{ + if !b.tables.Has(key) { + b.tables.Put(&Table{ Name: tableName, DatabaseName: c.DatabaseName, CreateTime: float64(time.Now().Unix()), - } + }) created++ } } @@ -656,50 +716,17 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.databases = make(map[string]*Database) - b.tables = make(map[string]*Table) - b.crawlers = make(map[string]*Crawler) - b.jobs = make(map[string]*Job) - b.partitions = make(map[string]*Partition) + b.registry.ResetAll() + b.partitionIndexes = make(map[string]*PartitionIndex) - b.tableVersions = make(map[string]*TableVersion) - b.connections = make(map[string]*Connection) - b.blueprints = make(map[string]*Blueprint) - b.customEntityTypes = make(map[string]*CustomEntityType) - b.dataQualityResult = make(map[string]*DataQualityResult) - b.devEndpoints = make(map[string]*DevEndpoint) b.jobRuns = make(map[string][]*JobRun) - b.jobBookmarks = make(map[string]*JobBookmark) - b.dataQualityRulesets = make(map[string]*DataQualityRuleset) - b.dataQualityEvalRuns = make(map[string]*DataQualityEvaluationRun) - b.triggers = make(map[string]*Trigger) - b.workflows = make(map[string]*Workflow) b.workflowRuns = make(map[string][]*WorkflowRun) - b.classifiers = make(map[string]*Classifier) - b.registries = make(map[string]*Registry) - b.schemas = make(map[string]*Schema) b.schemaVersions = make(map[string][]*SchemaVersion) - b.udfs = make(map[string]*UserDefinedFunction) - b.securityConfigs = make(map[string]*SecurityConfiguration) - b.sessions = make(map[string]*Session) b.sessionStatements = make(map[string][]*Statement) - b.tableOptimizers = make(map[string]*TableOptimizer) b.tableColumnStats = make(map[string]*ColumnStatistics) b.partitionColumnStats = make(map[string]*ColumnStatistics) b.resourcePolicies = make(map[string]*resourcePolicyEntry) - b.mlTransforms = make(map[string]*MLTransform) - b.catalogs = make(map[string]*CatalogEntry) b.catalogEncryptionSettings = make(map[string]*DataCatalogEncryptionSettings) - b.usageProfiles = make(map[string]*UsageProfile) - b.blueprintRuns = make(map[string]*BlueprintRun) - b.dqRecommendationRuns = make(map[string]*DQRuleRecommendationRun) - b.columnStatTaskSettings = make(map[string]*ColumnStatisticsTaskSettings) - b.columnStatTaskRuns = make(map[string]*ColumnStatisticsTaskRun) - b.materializedViewRuns = make(map[string]*MaterializedViewRefreshRun) - b.integrations = make(map[string]*Integration) - b.integrationResourceProps = make(map[string]*IntegrationResourceProperty) - b.integrationTableProps = make(map[string]*IntegrationTableProperties) - b.mlTaskRuns = make(map[string]*MLTaskRun) b.catalogImports = make(map[string]*CatalogImportStatus) b.schemaVersionMetadata = make(map[string]map[string]string) b.resetStubFixState() @@ -707,22 +734,19 @@ func (b *InMemoryBackend) Reset() { b.resetLifecycleStateLocked() } -// resetStubFixState clears the maps backing the de-stubbed operations (crawl -// history, resource-policy timestamps' owning map, data-quality statistic -// annotations). Kept separate from Reset to stay under the funlen limit. Must -// be called with b.mu held. +// resetStubFixState clears the raw map backing the de-stubbed crawl-history +// feature. Kept separate from Reset to stay under the funlen limit. Must be +// called with b.mu held. func (b *InMemoryBackend) resetStubFixState() { b.crawlHistory = make(map[string][]*CrawlHistoryEntry) - b.dqStatisticAnnotations = make(map[string]*StatisticAnnotation) } -// resetLifecycleStateLocked clears identity-center config, the connection-type -// registry, and pending lifecycle transition timers. Clearing the timers ensures a -// reset never resurrects a crawler/job-run state change scheduled against now-deleted +// resetLifecycleStateLocked clears identity-center config and pending +// lifecycle transition timers. Clearing the timers ensures a reset never +// resurrects a crawler/job-run state change scheduled against now-deleted // resources. Must be called with b.mu held. func (b *InMemoryBackend) resetLifecycleStateLocked() { b.glueIdentityCenterConfig = nil - b.customConnectionTypes = make(map[string]*ConnectionTypeInfo) b.jobRunReadyAt = make(map[string]map[string]time.Time) b.jobRunDoneAt = make(map[string]map[string]time.Time) b.crawlerReadyAt = make(map[string]time.Time) @@ -746,14 +770,44 @@ func cloneDatabase(db *Database) *Database { func cloneCrawler(c *Crawler) *Crawler { cp := *c cp.Tags = maps.Clone(c.Tags) - if len(c.Targets.S3Targets) > 0 { - cp.Targets.S3Targets = make([]S3Target, len(c.Targets.S3Targets)) - copy(cp.Targets.S3Targets, c.Targets.S3Targets) - } + cp.Classifiers = append([]string(nil), c.Classifiers...) + cp.Targets = cloneCrawlerTarget(c.Targets) return &cp } +// cloneCrawlerTarget returns a deep copy of a CrawlerTarget, including the +// nested Exclusions/Tables slices on each individual target entry. +func cloneCrawlerTarget(t CrawlerTarget) CrawlerTarget { + cp := CrawlerTarget{} + + if len(t.S3Targets) > 0 { + cp.S3Targets = make([]S3Target, len(t.S3Targets)) + for i, s := range t.S3Targets { + cp.S3Targets[i] = s + cp.S3Targets[i].Exclusions = append([]string(nil), s.Exclusions...) + } + } + + if len(t.JdbcTargets) > 0 { + cp.JdbcTargets = make([]JDBCTarget, len(t.JdbcTargets)) + for i, j := range t.JdbcTargets { + cp.JdbcTargets[i] = j + cp.JdbcTargets[i].Exclusions = append([]string(nil), j.Exclusions...) + } + } + + if len(t.CatalogTargets) > 0 { + cp.CatalogTargets = make([]CatalogTarget, len(t.CatalogTargets)) + for i, ct := range t.CatalogTargets { + cp.CatalogTargets[i] = ct + cp.CatalogTargets[i].Tables = append([]string(nil), ct.Tables...) + } + } + + return cp +} + // cloneJob returns a deep copy of a Job. func cloneJob(j *Job) *Job { cp := *j @@ -780,19 +834,53 @@ func cloneConnection(c *Connection) *Connection { return &cp } -// cloneTable returns a deep copy of a Table, including nested slices. -func cloneTable(t *Table) *Table { - cp := *t - if len(t.StorageDescriptor.Columns) > 0 { - cp.StorageDescriptor.Columns = make([]Column, len(t.StorageDescriptor.Columns)) - copy(cp.StorageDescriptor.Columns, t.StorageDescriptor.Columns) +// cloneColumns returns a deep copy of a Column slice, including each column's +// Parameters map (a shallow slice copy would still alias the map headers). +func cloneColumns(cols []Column) []Column { + if len(cols) == 0 { + return nil + } + + out := make([]Column, len(cols)) + for i, c := range cols { + out[i] = c + out[i].Parameters = maps.Clone(c.Parameters) + } + + return out +} + +// cloneStorageDescriptor returns a deep copy of a StorageDescriptor, including +// its Columns, Parameters, SerdeInfo, BucketColumns and SortColumns. +func cloneStorageDescriptor(sd StorageDescriptor) StorageDescriptor { + cp := sd + cp.Columns = cloneColumns(sd.Columns) + cp.Parameters = maps.Clone(sd.Parameters) + + if len(sd.BucketColumns) > 0 { + cp.BucketColumns = append([]string(nil), sd.BucketColumns...) + } + + if len(sd.SortColumns) > 0 { + cp.SortColumns = append([]Order(nil), sd.SortColumns...) } - if len(t.PartitionKeys) > 0 { - cp.PartitionKeys = make([]Column, len(t.PartitionKeys)) - copy(cp.PartitionKeys, t.PartitionKeys) + if sd.SerdeInfo != nil { + si := *sd.SerdeInfo + si.Parameters = maps.Clone(sd.SerdeInfo.Parameters) + cp.SerdeInfo = &si } + return cp +} + +// cloneTable returns a deep copy of a Table, including nested slices. +func cloneTable(t *Table) *Table { + cp := *t + cp.StorageDescriptor = cloneStorageDescriptor(t.StorageDescriptor) + cp.PartitionKeys = cloneColumns(t.PartitionKeys) + cp.Parameters = maps.Clone(t.Parameters) + return &cp } @@ -844,13 +932,6 @@ func tableVersionKey(dbName, tableName, versionID string) string { return fmt.Sprintf("%s|%s|%s", dbName, tableName, versionID) } -// sortedKeys returns the keys of a map in sorted order. -func sortedKeys[V any](m map[string]V) []string { - keys := collections.SortedKeys(m) - - return keys -} - // --- Database operations --- // CreateDatabase creates a new Glue database. @@ -869,7 +950,7 @@ func (b *InMemoryBackend) CreateDatabase( return nil, err } - if _, ok := b.databases[input.Name]; ok { + if b.databases.Has(input.Name) { return nil, ErrAlreadyExists } @@ -881,7 +962,7 @@ func (b *InMemoryBackend) CreateDatabase( Tags: maps.Clone(tags), CreateTime: float64(time.Now().Unix()), } - b.databases[input.Name] = db + b.databases.Put(db) return db, nil } @@ -891,7 +972,7 @@ func (b *InMemoryBackend) GetDatabase(name string) (*Database, error) { b.mu.RLock("GetDatabase") defer b.mu.RUnlock() - db, ok := b.databases[name] + db, ok := b.databases.Get(name) if !ok { return nil, ErrNotFound } @@ -904,9 +985,10 @@ func (b *InMemoryBackend) GetDatabases() []*Database { b.mu.RLock("GetDatabases") defer b.mu.RUnlock() - out := make([]*Database, 0, len(b.databases)) - for _, k := range sortedKeys(b.databases) { - out = append(out, cloneDatabase(b.databases[k])) + src := b.databases.Snapshot() + out := make([]*Database, 0, len(src)) + for _, db := range src { + out = append(out, cloneDatabase(db)) } return out @@ -917,28 +999,28 @@ func (b *InMemoryBackend) DeleteDatabase(name string) error { b.mu.Lock("DeleteDatabase") defer b.mu.Unlock() - if _, ok := b.databases[name]; !ok { + if !b.databases.Has(name) { return ErrNotFound } - delete(b.databases, name) + b.databases.Delete(name) prefix := name + "|" - for k := range b.tables { - if len(k) > len(prefix) && k[:len(prefix)] == prefix { - delete(b.tables, k) + for _, t := range b.tables.Snapshot() { + if k := tableKey(t.DatabaseName, t.Name); len(k) > len(prefix) && k[:len(prefix)] == prefix { + b.tables.Delete(k) } } - for k := range b.partitions { - if len(k) > len(prefix) && k[:len(prefix)] == prefix { - delete(b.partitions, k) + for _, p := range b.partitions.Snapshot() { + if k := partitionKey(p.DatabaseName, p.TableName, p.Values); len(k) > len(prefix) && k[:len(prefix)] == prefix { + b.partitions.Delete(k) } } - for k := range b.tableVersions { - if len(k) > len(prefix) && k[:len(prefix)] == prefix { - delete(b.tableVersions, k) + for _, tv := range b.tableVersions.Snapshot() { + if k := tableVersionEntryKeyFn(tv); len(k) > len(prefix) && k[:len(prefix)] == prefix { + b.tableVersions.Delete(k) } } @@ -950,7 +1032,7 @@ func (b *InMemoryBackend) UpdateDatabase(name string, input DatabaseInput) error b.mu.Lock("UpdateDatabase") defer b.mu.Unlock() - db, ok := b.databases[name] + db, ok := b.databases.Get(name) if !ok { return ErrNotFound } @@ -967,12 +1049,12 @@ func (b *InMemoryBackend) CreateTable(dbName string, input TableInput) (*Table, b.mu.Lock("CreateTable") defer b.mu.Unlock() - if _, ok := b.databases[dbName]; !ok { + if !b.databases.Has(dbName) { return nil, ErrNotFound } key := tableKey(dbName, input.Name) - if _, ok := b.tables[key]; ok { + if b.tables.Has(key) { return nil, ErrAlreadyExists } @@ -982,13 +1064,16 @@ func (b *InMemoryBackend) CreateTable(dbName string, input TableInput) (*Table, DatabaseName: dbName, CatalogID: b.accountID, Description: input.Description, + Owner: input.Owner, + Retention: input.Retention, + Parameters: maps.Clone(input.Parameters), StorageDescriptor: input.StorageDescriptor, PartitionKeys: input.PartitionKeys, TableType: input.TableType, CreateTime: now, UpdateTime: now, } - b.tables[key] = t + b.tables.Put(t) return t, nil } @@ -998,7 +1083,7 @@ func (b *InMemoryBackend) GetTable(dbName, tableName string) (*Table, error) { b.mu.RLock("GetTable") defer b.mu.RUnlock() - t, ok := b.tables[tableKey(dbName, tableName)] + t, ok := b.tables.Get(tableKey(dbName, tableName)) if !ok { return nil, ErrNotFound } @@ -1011,16 +1096,21 @@ func (b *InMemoryBackend) GetTables(dbName string) ([]*Table, error) { b.mu.RLock("GetTables") defer b.mu.RUnlock() - if _, ok := b.databases[dbName]; !ok { + if !b.databases.Has(dbName) { return nil, ErrNotFound } prefix := dbName + "|" - out := make([]*Table, 0, len(b.tables)) - - for _, k := range sortedKeys(b.tables) { - if len(k) > len(prefix) && k[:len(prefix)] == prefix { - out = append(out, b.tables[k]) + src := b.tables.Snapshot() + out := make([]*Table, 0, len(src)) + + for _, t := range src { + if k := tableKey(t.DatabaseName, t.Name); len(k) > len(prefix) && k[:len(prefix)] == prefix { + // Clone before returning: GetTable already clones, but GetTables was + // handing out the live backend pointer, letting a caller mutate + // (or a concurrent JSON marshal race against a writer mutating) + // catalog state without holding b.mu. + out = append(out, cloneTable(t)) } } @@ -1034,12 +1124,15 @@ func (b *InMemoryBackend) UpdateTable(dbName string, input TableInput) error { key := tableKey(dbName, input.Name) - t, ok := b.tables[key] + t, ok := b.tables.Get(key) if !ok { return ErrNotFound } t.Description = input.Description + t.Owner = input.Owner + t.Retention = input.Retention + t.Parameters = maps.Clone(input.Parameters) t.StorageDescriptor = input.StorageDescriptor t.PartitionKeys = input.PartitionKeys t.TableType = input.TableType @@ -1054,11 +1147,11 @@ func (b *InMemoryBackend) DeleteTable(dbName, tableName string) error { defer b.mu.Unlock() key := tableKey(dbName, tableName) - if _, ok := b.tables[key]; !ok { + if !b.tables.Has(key) { return ErrNotFound } - delete(b.tables, key) + b.tables.Delete(key) b.deleteTablePartitionsLocked(dbName, tableName) return nil @@ -1068,9 +1161,10 @@ func (b *InMemoryBackend) DeleteTable(dbName, tableName string) error { // Must be called with b.mu held for writing. func (b *InMemoryBackend) deleteTablePartitionsLocked(dbName, tableName string) { prefix := dbName + "|" + tableName + "|" - for k := range b.partitions { - if strings.HasPrefix(k, prefix) { - delete(b.partitions, k) + + for _, p := range b.partitions.Snapshot() { + if k := partitionKey(p.DatabaseName, p.TableName, p.Values); strings.HasPrefix(k, prefix) { + b.partitions.Delete(k) } } @@ -1080,9 +1174,9 @@ func (b *InMemoryBackend) deleteTablePartitionsLocked(dbName, tableName string) } } - for k := range b.tableVersions { - if strings.HasPrefix(k, prefix) { - delete(b.tableVersions, k) + for _, tv := range b.tableVersions.Snapshot() { + if k := tableVersionEntryKeyFn(tv); strings.HasPrefix(k, prefix) { + b.tableVersions.Delete(k) } } } @@ -1094,6 +1188,33 @@ func (b *InMemoryBackend) CreateCrawler( name, role, dbName string, targets CrawlerTarget, tags map[string]string, +) (*Crawler, error) { + return b.CreateCrawlerWithOptions(name, role, dbName, targets, tags, CrawlerOptions{}) +} + +// CrawlerOptions holds the CreateCrawler/UpdateCrawler fields beyond the core +// name/role/database/targets/tags accepted by CreateCrawler and UpdateCrawler. +// It exists so those two methods' signatures — called from outside this +// package (services/cloudformation) — stay additive/stable while still +// letting the Glue handler pass through Schedule, Classifiers, Configuration, +// TablePrefix and Description. +type CrawlerOptions struct { + Description string + Schedule string // cron expression; empty means on-demand (no schedule) + Configuration string + TablePrefix string + Classifiers []string +} + +// CreateCrawlerWithOptions is CreateCrawler plus the optional +// creation-time settings AWS's CreateCrawlerRequest supports +// (Schedule/Classifiers/Configuration/TablePrefix/Description) that the +// original positional-argument CreateCrawler predates. +func (b *InMemoryBackend) CreateCrawlerWithOptions( + name, role, dbName string, + targets CrawlerTarget, + tags map[string]string, + opts CrawlerOptions, ) (*Crawler, error) { b.mu.Lock("CreateCrawler") defer b.mu.Unlock() @@ -1107,28 +1228,36 @@ func (b *InMemoryBackend) CreateCrawler( } if dbName != "" { - if _, ok := b.databases[dbName]; !ok { + if !b.databases.Has(dbName) { return nil, ErrNotFound } } - if _, ok := b.crawlers[name]; ok { + if b.crawlers.Has(name) { return nil, ErrAlreadyExists } now := float64(time.Now().Unix()) c := &Crawler{ - Name: name, - Role: role, - DatabaseName: dbName, - Targets: targets, - State: stateReady, - ARN: b.crawlerARN(name), - Tags: maps.Clone(tags), - CreationTime: now, - LastUpdated: now, - } - b.crawlers[name] = c + Name: name, + Role: role, + DatabaseName: dbName, + Targets: targets, + State: stateReady, + ARN: b.crawlerARN(name), + Tags: maps.Clone(tags), + Description: opts.Description, + Configuration: opts.Configuration, + TablePrefix: opts.TablePrefix, + Classifiers: append([]string(nil), opts.Classifiers...), + CreationTime: now, + LastUpdated: now, + } + if opts.Schedule != "" { + c.Schedule = CrawlerSchedule{ScheduleExpression: opts.Schedule, State: stateScheduled} + } + + b.crawlers.Put(c) return c, nil } @@ -1140,7 +1269,7 @@ func (b *InMemoryBackend) GetCrawler(name string) (*Crawler, error) { b.mu.RLock("GetCrawler") defer b.mu.RUnlock() - c, ok := b.crawlers[name] + c, ok := b.crawlers.Get(name) if !ok { return nil, ErrNotFound } @@ -1155,9 +1284,10 @@ func (b *InMemoryBackend) GetCrawlers() []*Crawler { b.mu.RLock("GetCrawlers") defer b.mu.RUnlock() - out := make([]*Crawler, 0, len(b.crawlers)) - for _, k := range sortedKeys(b.crawlers) { - out = append(out, cloneCrawler(b.crawlers[k])) + src := b.crawlers.Snapshot() + out := make([]*Crawler, 0, len(src)) + for _, c := range src { + out = append(out, cloneCrawler(c)) } return out @@ -1168,22 +1298,68 @@ func (b *InMemoryBackend) ListCrawlers() []string { b.mu.RLock("ListCrawlers") defer b.mu.RUnlock() - return sortedKeys(b.crawlers) + src := b.crawlers.Snapshot() + out := make([]string, len(src)) + for i, c := range src { + out[i] = c.Name + } + + return out } // UpdateCrawler updates an existing Glue crawler. func (b *InMemoryBackend) UpdateCrawler(name, role, dbName string, targets CrawlerTarget) error { + return b.UpdateCrawlerWithOptions(name, role, dbName, targets, CrawlerOptions{}) +} + +// UpdateCrawlerWithOptions is UpdateCrawler plus the optional settings AWS's +// UpdateCrawlerRequest supports (Schedule/Classifiers/Configuration/ +// TablePrefix/Description). Unset (zero-value) CrawlerOptions fields leave the +// corresponding crawler field unchanged, matching AWS's partial-update +// semantics for UpdateCrawler. +func (b *InMemoryBackend) UpdateCrawlerWithOptions( + name, role, dbName string, + targets CrawlerTarget, + opts CrawlerOptions, +) error { b.mu.Lock("UpdateCrawler") defer b.mu.Unlock() - c, ok := b.crawlers[name] + c, ok := b.crawlers.Get(name) if !ok { return ErrNotFound } + // AWS rejects UpdateCrawler while the crawler is actively running, same as + // DeleteCrawler. + if c.State == stateRunning || c.State == stateStarting || c.State == stateStopping { + return ErrCrawlerRunning + } + c.Role = role c.DatabaseName = dbName c.Targets = targets + + if opts.Description != "" { + c.Description = opts.Description + } + + if opts.Configuration != "" { + c.Configuration = opts.Configuration + } + + if opts.TablePrefix != "" { + c.TablePrefix = opts.TablePrefix + } + + if opts.Classifiers != nil { + c.Classifiers = append([]string(nil), opts.Classifiers...) + } + + if opts.Schedule != "" { + c.Schedule = CrawlerSchedule{ScheduleExpression: opts.Schedule, State: stateScheduled} + } + c.LastUpdated = float64(time.Now().Unix()) return nil @@ -1194,7 +1370,7 @@ func (b *InMemoryBackend) DeleteCrawler(name string) error { b.mu.Lock("DeleteCrawler") defer b.mu.Unlock() - c, ok := b.crawlers[name] + c, ok := b.crawlers.Get(name) if !ok { return ErrNotFound } @@ -1203,7 +1379,7 @@ func (b *InMemoryBackend) DeleteCrawler(name string) error { return ErrCrawlerRunning } - delete(b.crawlers, name) + b.crawlers.Delete(name) return nil } @@ -1231,44 +1407,64 @@ func (b *InMemoryBackend) CreateJob(input Job) (*Job, error) { ) } + if err := validateJobCapacity(input); err != nil { + return nil, err + } + if err := validateTags(input.Tags); err != nil { return nil, err } - if _, ok := b.jobs[input.Name]; ok { + if b.jobs.Has(input.Name) { return nil, ErrAlreadyExists } now := float64(time.Now().Unix()) j := &Job{ - Name: input.Name, - Description: input.Description, - Role: input.Role, - Command: input.Command, - DefaultArguments: input.DefaultArguments, - GlueVersion: input.GlueVersion, - WorkerType: input.WorkerType, - NumberOfWorkers: input.NumberOfWorkers, - MaxRetries: input.MaxRetries, - Timeout: input.Timeout, - ARN: b.jobARN(input.Name), - Tags: maps.Clone(input.Tags), - ExecutionProperty: input.ExecutionProperty, - Connections: input.Connections, - CreatedOn: now, - LastModifiedOn: now, - } - b.jobs[input.Name] = j + Name: input.Name, + Description: input.Description, + Role: input.Role, + Command: input.Command, + DefaultArguments: input.DefaultArguments, + GlueVersion: input.GlueVersion, + WorkerType: input.WorkerType, + NumberOfWorkers: input.NumberOfWorkers, + MaxCapacity: input.MaxCapacity, + MaxRetries: input.MaxRetries, + Timeout: input.Timeout, + ARN: b.jobARN(input.Name), + Tags: maps.Clone(input.Tags), + ExecutionProperty: input.ExecutionProperty, + Connections: input.Connections, + NotificationProperty: input.NotificationProperty, + CreatedOn: now, + LastModifiedOn: now, + } + b.jobs.Put(j) return j, nil } +// validateJobCapacity enforces AWS Glue's mutual-exclusion rule between the +// legacy MaxCapacity (DPU) knob and WorkerType+NumberOfWorkers: a job may +// specify one or the other but not both. +func validateJobCapacity(input Job) error { + if input.MaxCapacity > 0 && (input.WorkerType != "" || input.NumberOfWorkers != 0) { + return fmt.Errorf( + "%w: cannot specify MaxCapacity and WorkerType/NumberOfWorkers together", + ErrValidation, + ) + } + + return nil +} + // GetJob retrieves a Glue job by name. func (b *InMemoryBackend) GetJob(name string) (*Job, error) { b.mu.RLock("GetJob") defer b.mu.RUnlock() - j, ok := b.jobs[name] + j, ok := b.jobs.Get(name) if !ok { return nil, ErrNotFound } @@ -1281,9 +1477,10 @@ func (b *InMemoryBackend) GetJobs() []*Job { b.mu.RLock("GetJobs") defer b.mu.RUnlock() - out := make([]*Job, 0, len(b.jobs)) - for _, k := range sortedKeys(b.jobs) { - out = append(out, cloneJob(b.jobs[k])) + src := b.jobs.Snapshot() + out := make([]*Job, 0, len(src)) + for _, j := range src { + out = append(out, cloneJob(j)) } return out @@ -1294,7 +1491,7 @@ func (b *InMemoryBackend) UpdateJob(name string, input Job) error { b.mu.Lock("UpdateJob") defer b.mu.Unlock() - j, ok := b.jobs[name] + j, ok := b.jobs.Get(name) if !ok { return ErrNotFound } @@ -1303,6 +1500,10 @@ func (b *InMemoryBackend) UpdateJob(name string, input Job) error { return fmt.Errorf("%w: MaxRetries must be between 0 and %d", ErrValidation, maxJobRetries) } + if err := validateJobCapacity(input); err != nil { + return err + } + j.Description = input.Description j.Role = input.Role j.Command = input.Command @@ -1310,10 +1511,12 @@ func (b *InMemoryBackend) UpdateJob(name string, input Job) error { j.GlueVersion = input.GlueVersion j.WorkerType = input.WorkerType j.NumberOfWorkers = input.NumberOfWorkers + j.MaxCapacity = input.MaxCapacity j.MaxRetries = input.MaxRetries j.Timeout = input.Timeout j.ExecutionProperty = input.ExecutionProperty j.Connections = input.Connections + j.NotificationProperty = input.NotificationProperty j.LastModifiedOn = float64(time.Now().Unix()) return nil @@ -1330,7 +1533,7 @@ func (b *InMemoryBackend) UpdateJobFromSourceControl(jobName string, details Sou b.mu.Lock("UpdateJobFromSourceControl") defer b.mu.Unlock() - j, ok := b.jobs[jobName] + j, ok := b.jobs.Get(jobName) if !ok { return ErrNotFound } @@ -1352,7 +1555,7 @@ func (b *InMemoryBackend) UpdateSourceControlFromJob(jobName string, details Sou b.mu.Lock("UpdateSourceControlFromJob") defer b.mu.Unlock() - j, ok := b.jobs[jobName] + j, ok := b.jobs.Get(jobName) if !ok { return ErrNotFound } @@ -1368,13 +1571,13 @@ func (b *InMemoryBackend) DeleteJob(name string) error { b.mu.Lock("DeleteJob") defer b.mu.Unlock() - if _, ok := b.jobs[name]; !ok { + if !b.jobs.Has(name) { return ErrNotFound } - delete(b.jobs, name) + b.jobs.Delete(name) delete(b.jobRuns, name) - delete(b.jobBookmarks, name) + b.jobBookmarks.Delete(name) return nil } @@ -1551,7 +1754,7 @@ func (b *InMemoryBackend) findDatabaseByARN(resourceARN string) *Database { return nil } - db, ok := b.databases[name] + db, ok := b.databases.Get(name) if !ok { return nil } @@ -1565,7 +1768,7 @@ func (b *InMemoryBackend) findCrawlerByARN(resourceARN string) *Crawler { return nil } - c, ok := b.crawlers[name] + c, ok := b.crawlers.Get(name) if !ok { return nil } @@ -1579,7 +1782,7 @@ func (b *InMemoryBackend) findJobByARN(resourceARN string) *Job { return nil } - j, ok := b.jobs[name] + j, ok := b.jobs.Get(name) if !ok { return nil } @@ -1593,7 +1796,7 @@ func (b *InMemoryBackend) findDataQualityRulesetByARN(resourceARN string) *DataQ return nil } - r, ok := b.dataQualityRulesets[name] + r, ok := b.dataQualityRulesets.Get(name) if !ok { return nil } @@ -1607,7 +1810,7 @@ func (b *InMemoryBackend) findConnectionByARN(resourceARN string) *Connection { return nil } - c, ok := b.connections[name] + c, ok := b.connections.Get(name) if !ok { return nil } @@ -1621,7 +1824,7 @@ func (b *InMemoryBackend) findTriggerByARN(resourceARN string) *Trigger { return nil } - t, ok := b.triggers[name] + t, ok := b.triggers.Get(name) if !ok { return nil } @@ -1635,7 +1838,7 @@ func (b *InMemoryBackend) findWorkflowByARN(resourceARN string) *Workflow { return nil } - w, ok := b.workflows[name] + w, ok := b.workflows.Get(name) if !ok { return nil } @@ -1656,9 +1859,31 @@ func (b *InMemoryBackend) BatchCreatePartition( created := make([]*Partition, 0, len(inputs)) errs := make([]PartitionError, 0, len(inputs)) + // AWS's BatchCreatePartition (and the single-partition CreatePartition, which + // is implemented in terms of this method) reject partitions for a table that + // does not exist with EntityNotFoundException. This was previously + // unchecked, so CreatePartition/BatchCreatePartition against a nonexistent + // database/table silently "succeeded" and stored orphaned partitions with no + // owning table — a disguised stub. + if !b.tables.Has(tableKey(dbName, tableName)) { + for _, input := range inputs { + errs = append(errs, PartitionError{ + PartitionValues: input.Values, + ErrorDetail: ErrorDetail{ + ErrorCode: errEntityNotFoundCode, + ErrorMessage: fmt.Sprintf("table %s.%s not found", dbName, tableName), + }, + }) + } + + return created, errs + } + + now := float64(time.Now().Unix()) + for _, input := range inputs { key := partitionKey(dbName, tableName, input.Values) - if _, exists := b.partitions[key]; exists { + if b.partitions.Has(key) { errs = append(errs, PartitionError{ PartitionValues: input.Values, ErrorDetail: ErrorDetail{ @@ -1673,10 +1898,13 @@ func (b *InMemoryBackend) BatchCreatePartition( p := &Partition{ DatabaseName: dbName, TableName: tableName, + CatalogID: b.accountID, Values: append([]string(nil), input.Values...), StorageDescriptor: input.StorageDescriptor, + Parameters: maps.Clone(input.Parameters), + CreationTime: now, } - b.partitions[key] = p + b.partitions.Put(p) created = append(created, p) } @@ -1695,7 +1923,7 @@ func (b *InMemoryBackend) BatchDeletePartition( for _, pvl := range values { key := partitionKey(dbName, tableName, pvl.Values) - if _, ok := b.partitions[key]; !ok { + if !b.partitions.Has(key) { errs = append(errs, PartitionError{ PartitionValues: pvl.Values, ErrorDetail: ErrorDetail{ @@ -1707,7 +1935,7 @@ func (b *InMemoryBackend) BatchDeletePartition( continue } - delete(b.partitions, key) + b.partitions.Delete(key) } return errs @@ -1722,7 +1950,7 @@ func (b *InMemoryBackend) BatchDeleteTable(dbName string, tableNames []string) [ for _, name := range tableNames { key := tableKey(dbName, name) - if _, ok := b.tables[key]; !ok { + if !b.tables.Has(key) { errs = append(errs, TableError{ TableName: name, ErrorDetail: ErrorDetail{ @@ -1734,7 +1962,7 @@ func (b *InMemoryBackend) BatchDeleteTable(dbName string, tableNames []string) [ continue } - delete(b.tables, key) + b.tables.Delete(key) b.deleteTablePartitionsLocked(dbName, name) } @@ -1753,7 +1981,7 @@ func (b *InMemoryBackend) BatchDeleteTableVersion( for _, vid := range versionIDs { key := tableVersionKey(dbName, tableName, vid) - if _, ok := b.tableVersions[key]; !ok { + if !b.tableVersions.Has(key) { errs = append(errs, TableVersionError{ TableName: tableName, VersionID: vid, @@ -1766,7 +1994,7 @@ func (b *InMemoryBackend) BatchDeleteTableVersion( continue } - delete(b.tableVersions, key) + b.tableVersions.Delete(key) } return errs @@ -1781,7 +2009,7 @@ func (b *InMemoryBackend) BatchDeleteConnection(names []string) ([]string, []Err errs := make([]ErrorDetail, 0, len(names)) for _, name := range names { - if _, ok := b.connections[name]; !ok { + if !b.connections.Has(name) { errs = append(errs, ErrorDetail{ ErrorCode: errEntityNotFoundCode, ErrorMessage: "connection not found: " + name, @@ -1790,7 +2018,7 @@ func (b *InMemoryBackend) BatchDeleteConnection(names []string) ([]string, []Err continue } - delete(b.connections, name) + b.connections.Delete(name) succeeded = append(succeeded, name) } @@ -1806,7 +2034,7 @@ func (b *InMemoryBackend) BatchGetBlueprints(names []string) ([]*Blueprint, []st missing := make([]string, 0, len(names)) for _, name := range names { - bp, ok := b.blueprints[name] + bp, ok := b.blueprints.Get(name) if !ok { missing = append(missing, name) @@ -1829,7 +2057,7 @@ func (b *InMemoryBackend) BatchGetCrawlers(names []string) ([]*Crawler, []string missing := make([]string, 0, len(names)) for _, name := range names { - c, ok := b.crawlers[name] + c, ok := b.crawlers.Get(name) if !ok { missing = append(missing, name) @@ -1853,7 +2081,7 @@ func (b *InMemoryBackend) BatchGetCustomEntityTypes( missing := make([]string, 0, len(names)) for _, name := range names { - cet, ok := b.customEntityTypes[name] + cet, ok := b.customEntityTypes.Get(name) if !ok { missing = append(missing, name) @@ -1879,7 +2107,7 @@ func (b *InMemoryBackend) BatchGetDataQualityResult( errs := make([]ErrorDetail, 0, len(resultIDs)) for _, id := range resultIDs { - dqr, ok := b.dataQualityResult[id] + dqr, ok := b.dataQualityResult.Get(id) if !ok { errs = append(errs, ErrorDetail{ ErrorCode: "EntityNotFoundException", @@ -1905,7 +2133,7 @@ func (b *InMemoryBackend) BatchGetDevEndpoints(names []string) ([]*DevEndpoint, missing := make([]string, 0, len(names)) for _, name := range names { - dep, ok := b.devEndpoints[name] + dep, ok := b.devEndpoints.Get(name) if !ok { missing = append(missing, name) @@ -1926,7 +2154,7 @@ func (b *InMemoryBackend) AddConnectionInternal(conn *Connection) { b.mu.Lock("AddConnectionInternal") defer b.mu.Unlock() - b.connections[conn.Name] = cloneConnection(conn) + b.connections.Put(cloneConnection(conn)) } // connectionARN returns the ARN for a Glue connection. @@ -1945,7 +2173,7 @@ func (b *InMemoryBackend) CreateConnection( return nil, ErrValidation } - if _, ok := b.connections[name]; ok { + if b.connections.Has(name) { return nil, ErrAlreadyExists } @@ -1959,7 +2187,7 @@ func (b *InMemoryBackend) CreateConnection( CreationTime: now, LastUpdatedTime: now, } - b.connections[name] = c + b.connections.Put(c) return cloneConnection(c), nil } @@ -1969,7 +2197,7 @@ func (b *InMemoryBackend) GetConnection(name string) (*Connection, error) { b.mu.RLock("GetConnection") defer b.mu.RUnlock() - c, ok := b.connections[name] + c, ok := b.connections.Get(name) if !ok { return nil, ErrNotFound } @@ -1982,9 +2210,10 @@ func (b *InMemoryBackend) GetConnections() []*Connection { b.mu.RLock("GetConnections") defer b.mu.RUnlock() - out := make([]*Connection, 0, len(b.connections)) - for _, k := range sortedKeys(b.connections) { - out = append(out, cloneConnection(b.connections[k])) + src := b.connections.Snapshot() + out := make([]*Connection, 0, len(src)) + for _, c := range src { + out = append(out, cloneConnection(c)) } return out @@ -1995,11 +2224,11 @@ func (b *InMemoryBackend) DeleteConnection(name string) error { b.mu.Lock("DeleteConnection") defer b.mu.Unlock() - if _, ok := b.connections[name]; !ok { + if !b.connections.Has(name) { return ErrNotFound } - delete(b.connections, name) + b.connections.Delete(name) return nil } @@ -2010,7 +2239,7 @@ func (b *InMemoryBackend) AddBlueprintInternal(bp *Blueprint) { defer b.mu.Unlock() cp := *bp - b.blueprints[bp.Name] = &cp + b.blueprints.Put(&cp) } // AddCustomEntityTypeInternal adds a custom entity type directly to the backend without validation. @@ -2020,7 +2249,7 @@ func (b *InMemoryBackend) AddCustomEntityTypeInternal(cet *CustomEntityType) { cp := *cet cp.ContextWords = append([]string(nil), cet.ContextWords...) - b.customEntityTypes[cet.Name] = &cp + b.customEntityTypes.Put(&cp) } // AddDataQualityResultInternal adds a data quality result directly to the backend without validation. @@ -2029,7 +2258,7 @@ func (b *InMemoryBackend) AddDataQualityResultInternal(dqr *DataQualityResult) { defer b.mu.Unlock() cp := *dqr - b.dataQualityResult[dqr.ResultID] = &cp + b.dataQualityResult.Put(&cp) } // AddDevEndpointInternal adds a dev endpoint directly to the backend without validation. @@ -2038,26 +2267,50 @@ func (b *InMemoryBackend) AddDevEndpointInternal(dep *DevEndpoint) { defer b.mu.Unlock() cp := *dep - b.devEndpoints[dep.EndpointName] = &cp + b.devEndpoints.Put(&cp) } -// AddTableVersionInternal adds a table version directly to the backend without validation. +// AddTableVersionInternal adds a table version directly to the backend without +// validation. dbName/tableName are stamped onto the stored copy's nested Table +// field (rather than trusted from the caller-supplied tv, which the existing +// test-seed callers often leave zero-valued) so the store.Table key -- derived +// purely from the value via tableVersionEntryKeyFn -- matches the +// dbName/tableName this entry is filed under, exactly as the previous raw map +// (keyed externally on the same two parameters) did. func (b *InMemoryBackend) AddTableVersionInternal(dbName, tableName string, tv *TableVersion) { b.mu.Lock("AddTableVersionInternal") defer b.mu.Unlock() cp := *tv - b.tableVersions[tableVersionKey(dbName, tableName, tv.VersionID)] = &cp + if cp.Table == nil { + cp.Table = &Table{} + } else { + t := *cp.Table + cp.Table = &t + } + + cp.Table.DatabaseName = dbName + cp.Table.Name = tableName + + b.tableVersions.Put(&cp) } -// AddPartitionInternal adds a partition directly to the backend without validation. +// AddPartitionInternal adds a partition directly to the backend without +// validation. dbName/tableName are stamped onto the stored copy (rather than +// trusted from the caller-supplied p, which the existing test-seed callers +// often leave zero-valued) so the store.Table key -- derived purely from the +// value via partitionEntryKeyFn -- matches the dbName/tableName this entry is +// filed under, exactly as the previous raw map (keyed externally on the same +// two parameters) did. func (b *InMemoryBackend) AddPartitionInternal(dbName, tableName string, p *Partition) { b.mu.Lock("AddPartitionInternal") defer b.mu.Unlock() cp := *p + cp.DatabaseName = dbName + cp.TableName = tableName cp.Values = append([]string(nil), p.Values...) - b.partitions[partitionKey(dbName, tableName, p.Values)] = &cp + b.partitions.Put(&cp) } // --- Job run operations --- @@ -2070,7 +2323,7 @@ func (b *InMemoryBackend) StartJobRun( b.mu.Lock("StartJobRun") defer b.mu.Unlock() - j, ok := b.jobs[jobName] + j, ok := b.jobs.Get(jobName) if !ok { return nil, ErrNotFound } @@ -2094,10 +2347,15 @@ func (b *InMemoryBackend) StartJobRun( now.UnixNano(), mrand.IntN(10000), //nolint:gosec,mnd // non-security mock run ID ), - JobName: jobName, - JobRunState: stateStarting, - StartedOn: float64(now.Unix()), - Arguments: maps.Clone(arguments), + JobName: jobName, + JobRunState: stateStarting, + StartedOn: float64(now.Unix()), + Arguments: maps.Clone(arguments), + WorkerType: j.WorkerType, + NumberOfWorkers: j.NumberOfWorkers, + MaxCapacity: j.MaxCapacity, + GlueVersion: j.GlueVersion, + Timeout: j.Timeout, } b.jobRuns[jobName] = append(b.jobRuns[jobName], run) @@ -2113,10 +2371,10 @@ func (b *InMemoryBackend) StartJobRun( b.jobRunReadyAt[jobName][run.ID] = now.Add(jobTransitionDelay) b.jobRunDoneAt[jobName][run.ID] = now.Add(jobTransitionDelay + jobSucceededDelay) - bm := b.jobBookmarks[jobName] - if bm == nil { + bm, ok := b.jobBookmarks.Get(jobName) + if !ok { bm = &JobBookmark{JobName: jobName} - b.jobBookmarks[jobName] = bm + b.jobBookmarks.Put(bm) } bm.ActiveRun = run.ID bm.Attempt++ @@ -2150,7 +2408,7 @@ func (b *InMemoryBackend) GetJobRuns(jobName string) ([]*JobRun, error) { b.mu.RLock("GetJobRuns") defer b.mu.RUnlock() - if _, ok := b.jobs[jobName]; !ok { + if !b.jobs.Has(jobName) { return nil, ErrNotFound } @@ -2215,11 +2473,11 @@ func (b *InMemoryBackend) GetJobBookmark(jobName string) (*JobBookmark, error) { b.mu.RLock("GetJobBookmark") defer b.mu.RUnlock() - if _, ok := b.jobs[jobName]; !ok { + if !b.jobs.Has(jobName) { return nil, ErrNotFound } - bm, ok := b.jobBookmarks[jobName] + bm, ok := b.jobBookmarks.Get(jobName) if !ok { return &JobBookmark{JobName: jobName}, nil } @@ -2234,11 +2492,11 @@ func (b *InMemoryBackend) ResetJobBookmark(jobName string) error { b.mu.Lock("ResetJobBookmark") defer b.mu.Unlock() - if _, ok := b.jobs[jobName]; !ok { + if !b.jobs.Has(jobName) { return ErrNotFound } - delete(b.jobBookmarks, jobName) + b.jobBookmarks.Delete(jobName) return nil } @@ -2248,11 +2506,11 @@ func (b *InMemoryBackend) ResetJobBookmarkWithResult(jobName string) (*JobBookma b.mu.Lock("ResetJobBookmarkWithResult") defer b.mu.Unlock() - if _, ok := b.jobs[jobName]; !ok { + if !b.jobs.Has(jobName) { return nil, ErrNotFound } - delete(b.jobBookmarks, jobName) + b.jobBookmarks.Delete(jobName) return &JobBookmark{JobName: jobName}, nil } @@ -2266,7 +2524,7 @@ func (b *InMemoryBackend) StartCrawler(name string) error { b.mu.Lock("StartCrawler") defer b.mu.Unlock() - c, ok := b.crawlers[name] + c, ok := b.crawlers.Get(name) if !ok { return ErrNotFound } @@ -2294,7 +2552,7 @@ func (b *InMemoryBackend) StopCrawler(name string) error { b.mu.Lock("StopCrawler") defer b.mu.Unlock() - c, ok := b.crawlers[name] + c, ok := b.crawlers.Get(name) if !ok { return ErrNotFound } @@ -2318,7 +2576,7 @@ func (b *InMemoryBackend) UpdateCrawlerSchedule(name, scheduleExpression string) b.mu.Lock("UpdateCrawlerSchedule") defer b.mu.Unlock() - c, ok := b.crawlers[name] + c, ok := b.crawlers.Get(name) if !ok { return ErrNotFound } @@ -2332,7 +2590,7 @@ func (b *InMemoryBackend) StartCrawlerSchedule(name string) error { b.mu.Lock("StartCrawlerSchedule") defer b.mu.Unlock() - c, ok := b.crawlers[name] + c, ok := b.crawlers.Get(name) if !ok { return ErrNotFound } @@ -2352,7 +2610,7 @@ func (b *InMemoryBackend) StopCrawlerSchedule(name string) error { b.mu.Lock("StopCrawlerSchedule") defer b.mu.Unlock() - c, ok := b.crawlers[name] + c, ok := b.crawlers.Get(name) if !ok { return ErrNotFound } @@ -2392,7 +2650,7 @@ func (b *InMemoryBackend) ListCrawls(crawlerName string) ([]*CrawlHistoryEntry, b.mu.RLock("ListCrawls") defer b.mu.RUnlock() - if _, ok := b.crawlers[crawlerName]; !ok { + if !b.crawlers.Has(crawlerName) { return nil, ErrNotFound } @@ -2426,7 +2684,7 @@ func (b *InMemoryBackend) CreateDataQualityRuleset( return nil, ErrValidation } - if _, ok := b.dataQualityRulesets[name]; ok { + if b.dataQualityRulesets.Has(name) { return nil, ErrAlreadyExists } @@ -2439,7 +2697,7 @@ func (b *InMemoryBackend) CreateDataQualityRuleset( CreatedOn: now, LastModifiedOn: now, } - b.dataQualityRulesets[name] = r + b.dataQualityRulesets.Put(r) return r, nil } @@ -2449,7 +2707,7 @@ func (b *InMemoryBackend) GetDataQualityRuleset(name string) (*DataQualityRulese b.mu.RLock("GetDataQualityRuleset") defer b.mu.RUnlock() - r, ok := b.dataQualityRulesets[name] + r, ok := b.dataQualityRulesets.Get(name) if !ok { return nil, ErrNotFound } @@ -2465,10 +2723,10 @@ func (b *InMemoryBackend) DeleteDataQualityRuleset(name string) error { b.mu.Lock("DeleteDataQualityRuleset") defer b.mu.Unlock() - if _, ok := b.dataQualityRulesets[name]; !ok { + if !b.dataQualityRulesets.Has(name) { return ErrNotFound } - delete(b.dataQualityRulesets, name) + b.dataQualityRulesets.Delete(name) return nil } @@ -2478,7 +2736,7 @@ func (b *InMemoryBackend) UpdateDataQualityRuleset(name, ruleset string) error { b.mu.Lock("UpdateDataQualityRuleset") defer b.mu.Unlock() - r, ok := b.dataQualityRulesets[name] + r, ok := b.dataQualityRulesets.Get(name) if !ok { return ErrNotFound } @@ -2493,9 +2751,9 @@ func (b *InMemoryBackend) ListDataQualityRulesets() []*DataQualityRuleset { b.mu.RLock("ListDataQualityRulesets") defer b.mu.RUnlock() - out := make([]*DataQualityRuleset, 0, len(b.dataQualityRulesets)) - for _, k := range sortedKeys(b.dataQualityRulesets) { - r := b.dataQualityRulesets[k] + src := b.dataQualityRulesets.Snapshot() + out := make([]*DataQualityRuleset, 0, len(src)) + for _, r := range src { cp := *r cp.Tags = maps.Clone(r.Tags) out = append(out, &cp) @@ -2512,7 +2770,7 @@ func (b *InMemoryBackend) StartDataQualityRulesetEvaluationRun( defer b.mu.Unlock() for _, name := range rulesetNames { - if _, ok := b.dataQualityRulesets[name]; !ok { + if !b.dataQualityRulesets.Has(name) { return nil, ErrNotFound } } @@ -2527,7 +2785,7 @@ func (b *InMemoryBackend) StartDataQualityRulesetEvaluationRun( Status: stateRunning, StartedOn: float64(time.Now().Unix()), } - b.dataQualityEvalRuns[run.RunID] = run + b.dataQualityEvalRuns.Put(run) return run, nil } @@ -2539,7 +2797,7 @@ func (b *InMemoryBackend) GetDataQualityRulesetEvaluationRun( b.mu.RLock("GetDataQualityRulesetEvaluationRun") defer b.mu.RUnlock() - run, ok := b.dataQualityEvalRuns[runID] + run, ok := b.dataQualityEvalRuns.Get(runID) if !ok { return nil, ErrNotFound } @@ -2555,7 +2813,7 @@ func (b *InMemoryBackend) CancelDataQualityRulesetEvaluationRun(runID string) er b.mu.Lock("CancelDataQualityRulesetEvaluationRun") defer b.mu.Unlock() - run, ok := b.dataQualityEvalRuns[runID] + run, ok := b.dataQualityEvalRuns.Get(runID) if !ok { return ErrNotFound } @@ -2584,7 +2842,7 @@ func (b *InMemoryBackend) AddDataQualityRulesetInternal(r *DataQualityRuleset) { cp := *r cp.Tags = maps.Clone(r.Tags) - b.dataQualityRulesets[r.Name] = &cp + b.dataQualityRulesets.Put(&cp) } // AddDataQualityEvalRunInternal adds an evaluation run directly without validation. @@ -2594,5 +2852,5 @@ func (b *InMemoryBackend) AddDataQualityEvalRunInternal(run *DataQualityEvaluati cp := *run cp.RulesetNames = append([]string(nil), run.RulesetNames...) - b.dataQualityEvalRuns[run.RunID] = &cp + b.dataQualityEvalRuns.Put(&cp) } diff --git a/services/glue/backend_accuracy.go b/services/glue/backend_accuracy.go index 91865b0b6..6ea4f8e4c 100644 --- a/services/glue/backend_accuracy.go +++ b/services/glue/backend_accuracy.go @@ -12,7 +12,7 @@ func (b *InMemoryBackend) GetPartition(dbName, tableName string, values []string defer b.mu.RUnlock() key := partitionKey(dbName, tableName, values) - p, ok := b.partitions[key] + p, ok := b.partitions.Get(key) if !ok { return nil, ErrNotFound } @@ -27,16 +27,16 @@ func (b *InMemoryBackend) GetPartitions(dbName, tableName string) ([]*Partition, b.mu.RLock("GetPartitions") defer b.mu.RUnlock() - if _, ok := b.tables[tableKey(dbName, tableName)]; !ok { + if !b.tables.Has(tableKey(dbName, tableName)) { return nil, ErrNotFound } prefix := dbName + "|" + tableName + "|" out := make([]*Partition, 0) - for _, k := range sortedKeys(b.partitions) { - if strings.HasPrefix(k, prefix) { - out = append(out, clonePartition(b.partitions[k])) + for _, p := range b.partitions.Snapshot() { + if k := partitionEntryKeyFn(p); strings.HasPrefix(k, prefix) { + out = append(out, clonePartition(p)) } } @@ -62,7 +62,7 @@ func (b *InMemoryBackend) updatePartitionLocked( input PartitionInput, ) error { oldKey := partitionKey(dbName, tableName, partitionValues) - p, ok := b.partitions[oldKey] + p, ok := b.partitions.Get(oldKey) if !ok { return ErrNotFound } @@ -72,23 +72,30 @@ func (b *InMemoryBackend) updatePartitionLocked( } p.StorageDescriptor = input.StorageDescriptor + p.Parameters = maps.Clone(input.Parameters) return nil } // renamePartitionLocked moves a partition to a new key based on new values. +// It deletes the old entry and re-inserts p under its new key (rather than +// mutating p.Values in place and leaving it under oldKey) because +// store.Table derives a value's key from its own fields via +// partitionEntryKeyFn -- mutating the identity field without re-Put would +// leave the table indexed under a stale key for this value. func (b *InMemoryBackend) renamePartitionLocked( p *Partition, dbName, tableName, oldKey string, input PartitionInput, ) error { newKey := partitionKey(dbName, tableName, input.Values) - if _, exists := b.partitions[newKey]; exists { + if b.partitions.Has(newKey) { return ErrAlreadyExists } - delete(b.partitions, oldKey) + b.partitions.Delete(oldKey) p.Values = append([]string(nil), input.Values...) p.StorageDescriptor = input.StorageDescriptor - b.partitions[newKey] = p + p.Parameters = maps.Clone(input.Parameters) + b.partitions.Put(p) return nil } @@ -102,8 +109,7 @@ func (b *InMemoryBackend) SearchTables(searchText string) []*Table { lower := strings.ToLower(searchText) out := make([]*Table, 0) - for _, k := range sortedKeys(b.tables) { - t := b.tables[k] + for _, t := range b.tables.Snapshot() { if lower == "" || strings.Contains(strings.ToLower(t.Name), lower) { out = append(out, cloneTable(t)) } @@ -117,7 +123,7 @@ func (b *InMemoryBackend) UpdateConnection(name string, connType string, props m b.mu.Lock("UpdateConnection") defer b.mu.Unlock() - c, ok := b.connections[name] + c, ok := b.connections.Get(name) if !ok { return ErrNotFound } @@ -133,10 +139,8 @@ func (b *InMemoryBackend) UpdateConnection(name string, connType string, props m func clonePartition(p *Partition) *Partition { cp := *p cp.Values = append([]string(nil), p.Values...) - if len(p.StorageDescriptor.Columns) > 0 { - cp.StorageDescriptor.Columns = make([]Column, len(p.StorageDescriptor.Columns)) - copy(cp.StorageDescriptor.Columns, p.StorageDescriptor.Columns) - } + cp.StorageDescriptor = cloneStorageDescriptor(p.StorageDescriptor) + cp.Parameters = maps.Clone(p.Parameters) return &cp } diff --git a/services/glue/backend_batch2.go b/services/glue/backend_batch2.go index 286cc25cc..fb40c21a8 100644 --- a/services/glue/backend_batch2.go +++ b/services/glue/backend_batch2.go @@ -6,8 +6,6 @@ import ( "time" "github.com/google/uuid" - - "github.com/blackbirdworks/gopherstack/pkgs/collections" ) // --- New model types --- @@ -101,11 +99,11 @@ func (b *InMemoryBackend) CreateBlueprint(name string) error { b.mu.Lock("CreateBlueprint") defer b.mu.Unlock() - if _, exists := b.blueprints[name]; exists { + if b.blueprints.Has(name) { return fmt.Errorf("blueprint %q already exists: %w", name, ErrAlreadyExists) } - b.blueprints[name] = &Blueprint{Name: name, Status: "ACTIVE"} + b.blueprints.Put(&Blueprint{Name: name, Status: "ACTIVE"}) return nil } @@ -115,11 +113,11 @@ func (b *InMemoryBackend) DeleteBlueprint(name string) error { b.mu.Lock("DeleteBlueprint") defer b.mu.Unlock() - if _, ok := b.blueprints[name]; !ok { + if !b.blueprints.Has(name) { return fmt.Errorf("blueprint %q not found: %w", name, ErrNotFound) } - delete(b.blueprints, name) + b.blueprints.Delete(name) return nil } @@ -129,7 +127,7 @@ func (b *InMemoryBackend) UpdateBlueprint(name string) (*Blueprint, error) { b.mu.Lock("UpdateBlueprint") defer b.mu.Unlock() - bp, ok := b.blueprints[name] + bp, ok := b.blueprints.Get(name) if !ok { return nil, fmt.Errorf("blueprint %q not found: %w", name, ErrNotFound) } @@ -144,7 +142,11 @@ func (b *InMemoryBackend) ListBlueprints() []string { b.mu.RLock("ListBlueprints") defer b.mu.RUnlock() - names := collections.SortedKeys(b.blueprints) + src := b.blueprints.Snapshot() + names := make([]string, len(src)) + for i, bp := range src { + names[i] = bp.Name + } return names } @@ -156,7 +158,7 @@ func (b *InMemoryBackend) StartBlueprintRun(blueprintName string) (*BlueprintRun b.mu.Lock("StartBlueprintRun") defer b.mu.Unlock() - if _, ok := b.blueprints[blueprintName]; !ok { + if !b.blueprints.Has(blueprintName) { return nil, fmt.Errorf("blueprint %q not found: %w", blueprintName, ErrNotFound) } @@ -168,7 +170,7 @@ func (b *InMemoryBackend) StartBlueprintRun(blueprintName string) (*BlueprintRun State: stateRunning, StartedOn: time.Now().UTC(), } - b.blueprintRuns[runID] = run + b.blueprintRuns.Put(run) cp := *run @@ -180,7 +182,7 @@ func (b *InMemoryBackend) GetBlueprintRun(blueprintName, runID string) (*Bluepri b.mu.RLock("GetBlueprintRun") defer b.mu.RUnlock() - run, ok := b.blueprintRuns[runID] + run, ok := b.blueprintRuns.Get(runID) if !ok || (blueprintName != "" && run.BlueprintName != blueprintName) { return nil, ErrBlueprintRunNotFound } @@ -196,7 +198,7 @@ func (b *InMemoryBackend) GetBlueprintRuns(blueprintName string) []*BlueprintRun defer b.mu.RUnlock() var runs []*BlueprintRun - for _, r := range b.blueprintRuns { + for _, r := range b.blueprintRuns.All() { if blueprintName == "" || r.BlueprintName == blueprintName { cp := *r runs = append(runs, &cp) @@ -221,7 +223,7 @@ func (b *InMemoryBackend) CreateUsageProfile(name, description string, tags map[ b.mu.Lock("CreateUsageProfile") defer b.mu.Unlock() - if _, exists := b.usageProfiles[name]; exists { + if b.usageProfiles.Has(name) { return nil, fmt.Errorf("usage profile %q already exists: %w", name, ErrAlreadyExists) } @@ -233,7 +235,7 @@ func (b *InMemoryBackend) CreateUsageProfile(name, description string, tags map[ CreatedOn: now, LastModifiedOn: now, } - b.usageProfiles[name] = p + b.usageProfiles.Put(p) cp := *p return &cp, nil @@ -244,7 +246,7 @@ func (b *InMemoryBackend) GetUsageProfile(name string) (*UsageProfile, error) { b.mu.RLock("GetUsageProfile") defer b.mu.RUnlock() - p, ok := b.usageProfiles[name] + p, ok := b.usageProfiles.Get(name) if !ok { return nil, ErrUsageProfileNotFound } @@ -259,11 +261,11 @@ func (b *InMemoryBackend) DeleteUsageProfile(name string) error { b.mu.Lock("DeleteUsageProfile") defer b.mu.Unlock() - if _, ok := b.usageProfiles[name]; !ok { + if !b.usageProfiles.Has(name) { return ErrUsageProfileNotFound } - delete(b.usageProfiles, name) + b.usageProfiles.Delete(name) return nil } @@ -273,8 +275,9 @@ func (b *InMemoryBackend) ListUsageProfiles() []*UsageProfile { b.mu.RLock("ListUsageProfiles") defer b.mu.RUnlock() - profiles := make([]*UsageProfile, 0, len(b.usageProfiles)) - for _, p := range b.usageProfiles { + src := b.usageProfiles.All() + profiles := make([]*UsageProfile, 0, len(src)) + for _, p := range src { cp := *p profiles = append(profiles, &cp) } @@ -291,7 +294,7 @@ func (b *InMemoryBackend) UpdateUsageProfile(name, description string) (*UsagePr b.mu.Lock("UpdateUsageProfile") defer b.mu.Unlock() - p, ok := b.usageProfiles[name] + p, ok := b.usageProfiles.Get(name) if !ok { return nil, ErrUsageProfileNotFound } @@ -326,7 +329,7 @@ func (b *InMemoryBackend) CreateCustomEntityType( RegexString: regexString, ContextWords: contextWords, } - b.customEntityTypes[name] = cet + b.customEntityTypes.Put(cet) cp := *cet return &cp, nil @@ -337,7 +340,7 @@ func (b *InMemoryBackend) GetCustomEntityType(name string) (*CustomEntityType, e b.mu.RLock("GetCustomEntityType") defer b.mu.RUnlock() - cet, ok := b.customEntityTypes[name] + cet, ok := b.customEntityTypes.Get(name) if !ok { return nil, fmt.Errorf("custom entity type %q not found: %w", name, ErrNotFound) } @@ -352,11 +355,11 @@ func (b *InMemoryBackend) DeleteCustomEntityType(name string) error { b.mu.Lock("DeleteCustomEntityType") defer b.mu.Unlock() - if _, ok := b.customEntityTypes[name]; !ok { + if !b.customEntityTypes.Has(name) { return fmt.Errorf("custom entity type %q not found: %w", name, ErrNotFound) } - delete(b.customEntityTypes, name) + b.customEntityTypes.Delete(name) return nil } @@ -366,8 +369,9 @@ func (b *InMemoryBackend) ListCustomEntityTypes() []*CustomEntityType { b.mu.RLock("ListCustomEntityTypes") defer b.mu.RUnlock() - list := make([]*CustomEntityType, 0, len(b.customEntityTypes)) - for _, cet := range b.customEntityTypes { + src := b.customEntityTypes.All() + list := make([]*CustomEntityType, 0, len(src)) + for _, cet := range src { cp := *cet list = append(list, &cp) } @@ -393,7 +397,7 @@ func (b *InMemoryBackend) StartDataQualityRuleRecommendationRun(s3Path string) ( Status: stateRunning, StartedOn: time.Now().UTC(), } - b.dqRecommendationRuns[runID] = run + b.dqRecommendationRuns.Put(run) cp := *run return &cp, nil @@ -404,7 +408,7 @@ func (b *InMemoryBackend) GetDataQualityRuleRecommendationRun(runID string) (*DQ b.mu.RLock("GetDataQualityRuleRecommendationRun") defer b.mu.RUnlock() - run, ok := b.dqRecommendationRuns[runID] + run, ok := b.dqRecommendationRuns.Get(runID) if !ok { return nil, ErrDQRecommendationRunNotFound } @@ -419,7 +423,7 @@ func (b *InMemoryBackend) CancelDataQualityRuleRecommendationRun(runID string) e b.mu.Lock("CancelDataQualityRuleRecommendationRun") defer b.mu.Unlock() - run, ok := b.dqRecommendationRuns[runID] + run, ok := b.dqRecommendationRuns.Get(runID) if !ok { return ErrDQRecommendationRunNotFound } @@ -434,8 +438,9 @@ func (b *InMemoryBackend) ListDataQualityRuleRecommendationRuns() []*DQRuleRecom b.mu.RLock("ListDataQualityRuleRecommendationRuns") defer b.mu.RUnlock() - runs := make([]*DQRuleRecommendationRun, 0, len(b.dqRecommendationRuns)) - for _, r := range b.dqRecommendationRuns { + src := b.dqRecommendationRuns.All() + runs := make([]*DQRuleRecommendationRun, 0, len(src)) + for _, r := range src { cp := *r runs = append(runs, &cp) } @@ -462,14 +467,13 @@ func (b *InMemoryBackend) CreateColumnStatisticsTaskSettings( b.mu.Lock("CreateColumnStatisticsTaskSettings") defer b.mu.Unlock() - key := columnStatTaskKey(dbName, tableName) settings := &ColumnStatisticsTaskSettings{ DatabaseName: dbName, TableName: tableName, ColumnNameList: columns, RoleArn: roleArn, } - b.columnStatTaskSettings[key] = settings + b.columnStatTaskSettings.Put(settings) cp := *settings return &cp, nil @@ -483,7 +487,7 @@ func (b *InMemoryBackend) GetColumnStatisticsTaskSettings( defer b.mu.RUnlock() key := columnStatTaskKey(dbName, tableName) - s, ok := b.columnStatTaskSettings[key] + s, ok := b.columnStatTaskSettings.Get(key) if !ok { return &ColumnStatisticsTaskSettings{DatabaseName: dbName, TableName: tableName}, nil @@ -502,7 +506,7 @@ func (b *InMemoryBackend) UpdateColumnStatisticsTaskSettings( defer b.mu.Unlock() key := columnStatTaskKey(dbName, tableName) - if s, ok := b.columnStatTaskSettings[key]; ok { + if s, ok := b.columnStatTaskSettings.Get(key); ok { s.RoleArn = roleArn } @@ -514,7 +518,7 @@ func (b *InMemoryBackend) DeleteColumnStatisticsTaskSettings(dbName, tableName s b.mu.Lock("DeleteColumnStatisticsTaskSettings") defer b.mu.Unlock() - delete(b.columnStatTaskSettings, columnStatTaskKey(dbName, tableName)) + b.columnStatTaskSettings.Delete(columnStatTaskKey(dbName, tableName)) return nil } @@ -531,7 +535,7 @@ func (b *InMemoryBackend) StartColumnStatisticsTaskRunSchedule(dbName, tableName b.mu.Lock("StartColumnStatisticsTaskRunSchedule") defer b.mu.Unlock() - s, ok := b.columnStatTaskSettings[columnStatTaskKey(dbName, tableName)] + s, ok := b.columnStatTaskSettings.Get(columnStatTaskKey(dbName, tableName)) if !ok { return fmt.Errorf( "column statistics task settings not found for %s.%s: %w", dbName, tableName, ErrNotFound, @@ -553,7 +557,7 @@ func (b *InMemoryBackend) StopColumnStatisticsTaskRunSchedule(dbName, tableName b.mu.Lock("StopColumnStatisticsTaskRunSchedule") defer b.mu.Unlock() - s, ok := b.columnStatTaskSettings[columnStatTaskKey(dbName, tableName)] + s, ok := b.columnStatTaskSettings.Get(columnStatTaskKey(dbName, tableName)) if !ok { return fmt.Errorf( "column statistics task settings not found for %s.%s: %w", dbName, tableName, ErrNotFound, @@ -578,7 +582,7 @@ func (b *InMemoryBackend) StartColumnStatisticsTaskRun(dbName, tableName string) Status: "STARTED", StartedOn: time.Now().UTC(), } - b.columnStatTaskRuns[runID] = run + b.columnStatTaskRuns.Put(run) cp := *run return &cp, nil @@ -589,7 +593,7 @@ func (b *InMemoryBackend) StopColumnStatisticsTaskRun(runID string) error { b.mu.Lock("StopColumnStatisticsTaskRun") defer b.mu.Unlock() - r, ok := b.columnStatTaskRuns[runID] + r, ok := b.columnStatTaskRuns.Get(runID) if !ok { return ErrColumnStatTaskRunNotFound } @@ -604,7 +608,7 @@ func (b *InMemoryBackend) GetColumnStatisticsTaskRun(runID string) (*ColumnStati b.mu.RLock("GetColumnStatisticsTaskRun") defer b.mu.RUnlock() - r, ok := b.columnStatTaskRuns[runID] + r, ok := b.columnStatTaskRuns.Get(runID) if !ok { return nil, ErrColumnStatTaskRunNotFound } @@ -619,8 +623,9 @@ func (b *InMemoryBackend) ListColumnStatisticsTaskRuns() []*ColumnStatisticsTask b.mu.RLock("ListColumnStatisticsTaskRuns") defer b.mu.RUnlock() - runs := make([]*ColumnStatisticsTaskRun, 0, len(b.columnStatTaskRuns)) - for _, r := range b.columnStatTaskRuns { + src := b.columnStatTaskRuns.All() + runs := make([]*ColumnStatisticsTaskRun, 0, len(src)) + for _, r := range src { cp := *r runs = append(runs, &cp) } @@ -654,7 +659,7 @@ func (b *InMemoryBackend) StartMaterializedViewRefreshTaskRun( Status: stateRunning, StartedOn: time.Now().UTC(), } - b.materializedViewRuns[taskID] = run + b.materializedViewRuns.Put(run) cp := *run return &cp, nil @@ -665,7 +670,7 @@ func (b *InMemoryBackend) StopMaterializedViewRefreshTaskRun(taskRunID string) e b.mu.Lock("StopMaterializedViewRefreshTaskRun") defer b.mu.Unlock() - r, ok := b.materializedViewRuns[taskRunID] + r, ok := b.materializedViewRuns.Get(taskRunID) if !ok { return ErrMaterializedViewRunNotFound } @@ -680,7 +685,7 @@ func (b *InMemoryBackend) GetMaterializedViewRefreshTaskRun(taskRunID string) (* b.mu.RLock("GetMaterializedViewRefreshTaskRun") defer b.mu.RUnlock() - r, ok := b.materializedViewRuns[taskRunID] + r, ok := b.materializedViewRuns.Get(taskRunID) if !ok { return nil, ErrMaterializedViewRunNotFound } @@ -695,8 +700,9 @@ func (b *InMemoryBackend) ListMaterializedViewRefreshTaskRuns() []*MaterializedV b.mu.RLock("ListMaterializedViewRefreshTaskRuns") defer b.mu.RUnlock() - runs := make([]*MaterializedViewRefreshRun, 0, len(b.materializedViewRuns)) - for _, r := range b.materializedViewRuns { + src := b.materializedViewRuns.All() + runs := make([]*MaterializedViewRefreshRun, 0, len(src)) + for _, r := range src { cp := *r runs = append(runs, &cp) } @@ -721,7 +727,7 @@ func (b *InMemoryBackend) CreateIntegration(name string, tags map[string]string) Tags: tags, CreatedAt: time.Now().UTC(), } - b.integrations[name] = ig + b.integrations.Put(ig) cp := *ig return &cp, nil @@ -732,11 +738,11 @@ func (b *InMemoryBackend) DeleteIntegration(name string) error { b.mu.Lock("DeleteIntegration") defer b.mu.Unlock() - if _, ok := b.integrations[name]; !ok { + if !b.integrations.Has(name) { return fmt.Errorf("integration %q not found: %w", name, ErrNotFound) } - delete(b.integrations, name) + b.integrations.Delete(name) return nil } @@ -746,8 +752,9 @@ func (b *InMemoryBackend) ListIntegrations() []*Integration { b.mu.RLock("ListIntegrations") defer b.mu.RUnlock() - list := make([]*Integration, 0, len(b.integrations)) - for _, ig := range b.integrations { + src := b.integrations.All() + list := make([]*Integration, 0, len(src)) + for _, ig := range src { cp := *ig list = append(list, &cp) } @@ -764,7 +771,7 @@ func (b *InMemoryBackend) ModifyIntegration(name string) error { b.mu.Lock("ModifyIntegration") defer b.mu.Unlock() - if _, ok := b.integrations[name]; !ok { + if !b.integrations.Has(name) { return ErrIntegrationNotFound } @@ -852,7 +859,7 @@ func (b *InMemoryBackend) CreateIntegrationResourceProperty( SourceProperties: sourceProps, TargetProperties: targetProps, } - b.integrationResourceProps[resourceArn] = prop + b.integrationResourceProps.Put(prop) return prop, nil } @@ -866,7 +873,7 @@ func (b *InMemoryBackend) GetIntegrationResourceProperty(resourceArn string) (*I b.mu.RLock("GetIntegrationResourceProperty") defer b.mu.RUnlock() - prop, ok := b.integrationResourceProps[resourceArn] + prop, ok := b.integrationResourceProps.Get(resourceArn) if !ok { return nil, fmt.Errorf("resource property for %q not found: %w", resourceArn, ErrNotFound) } @@ -886,7 +893,7 @@ func (b *InMemoryBackend) UpdateIntegrationResourceProperty( b.mu.Lock("UpdateIntegrationResourceProperty") defer b.mu.Unlock() - prop, ok := b.integrationResourceProps[resourceArn] + prop, ok := b.integrationResourceProps.Get(resourceArn) if !ok { return nil, fmt.Errorf("resource property for %q not found: %w", resourceArn, ErrNotFound) } @@ -908,14 +915,7 @@ func (b *InMemoryBackend) ListIntegrationResourceProperties() []*IntegrationReso b.mu.RLock("ListIntegrationResourceProperties") defer b.mu.RUnlock() - keys := collections.SortedKeys(b.integrationResourceProps) - out := make([]*IntegrationResourceProperty, 0, len(keys)) - - for _, k := range keys { - out = append(out, b.integrationResourceProps[k]) - } - - return out + return b.integrationResourceProps.Snapshot() } // --- IntegrationTableProperties --- @@ -940,13 +940,12 @@ func (b *InMemoryBackend) CreateIntegrationTableProperties( b.mu.Lock("CreateIntegrationTableProperties") defer b.mu.Unlock() - key := resourceArn + "|" + tableName - b.integrationTableProps[key] = &IntegrationTableProperties{ + b.integrationTableProps.Put(&IntegrationTableProperties{ ResourceArn: resourceArn, TableName: tableName, SourceTableConfig: sourceConfig, TargetTableConfig: targetConfig, - } + }) return nil } @@ -964,7 +963,7 @@ func (b *InMemoryBackend) GetIntegrationTableProperties( key := resourceArn + "|" + tableName - prop, ok := b.integrationTableProps[key] + prop, ok := b.integrationTableProps.Get(key) if !ok { return nil, fmt.Errorf("table property for %q/%q not found: %w", resourceArn, tableName, ErrNotFound) } @@ -986,7 +985,7 @@ func (b *InMemoryBackend) UpdateIntegrationTableProperties( key := resourceArn + "|" + tableName - prop, ok := b.integrationTableProps[key] + prop, ok := b.integrationTableProps.Get(key) if !ok { return fmt.Errorf("table property for %q/%q not found: %w", resourceArn, tableName, ErrNotFound) } @@ -1029,17 +1028,17 @@ func (b *InMemoryBackend) PutDataQualityStatisticAnnotation(profileID, statistic key := dqAnnotationKey(profileID, statisticID) recordedOn := now - if existing, ok := b.dqStatisticAnnotations[key]; ok { + if existing, ok := b.dqStatisticAnnotations.Get(key); ok { recordedOn = existing.RecordedOn } - b.dqStatisticAnnotations[key] = &StatisticAnnotation{ + b.dqStatisticAnnotations.Put(&StatisticAnnotation{ ProfileID: profileID, StatisticID: statisticID, Inclusion: inclusion, RecordedOn: recordedOn, LastModifiedOn: now, - } + }) } // ListDataQualityStatisticAnnotations returns stored annotations, optionally @@ -1049,11 +1048,10 @@ func (b *InMemoryBackend) ListDataQualityStatisticAnnotations(profileID, statist b.mu.RLock("ListDataQualityStatisticAnnotations") defer b.mu.RUnlock() - keys := collections.SortedKeys(b.dqStatisticAnnotations) - out := make([]*StatisticAnnotation, 0, len(keys)) + src := b.dqStatisticAnnotations.Snapshot() + out := make([]*StatisticAnnotation, 0, len(src)) - for _, k := range keys { - e := b.dqStatisticAnnotations[k] + for _, e := range src { if profileID != "" && e.ProfileID != profileID { continue } diff --git a/services/glue/backend_conntypes.go b/services/glue/backend_conntypes.go index ef8f9fb22..6c9cd5640 100644 --- a/services/glue/backend_conntypes.go +++ b/services/glue/backend_conntypes.go @@ -143,7 +143,7 @@ func (b *InMemoryBackend) RegisterConnectionType(name, description string) (*Con Capabilities: rwCaps(), BuiltIn: false, } - b.customConnectionTypes[norm] = info + b.customConnectionTypes.Put(info) clone := *info @@ -166,11 +166,11 @@ func (b *InMemoryBackend) DeleteConnectionType(name string) error { b.mu.Lock("DeleteConnectionType") defer b.mu.Unlock() - if _, ok := b.customConnectionTypes[norm]; !ok { + if !b.customConnectionTypes.Has(norm) { return awserr.New("connection type "+norm+" not found", awserr.ErrNotFound) } - delete(b.customConnectionTypes, norm) + b.customConnectionTypes.Delete(norm) return nil } @@ -190,7 +190,7 @@ func (b *InMemoryBackend) DescribeConnectionType(name string) (*ConnectionTypeIn b.mu.RLock("DescribeConnectionType") defer b.mu.RUnlock() - if info, ok := b.customConnectionTypes[norm]; ok { + if info, ok := b.customConnectionTypes.Get(norm); ok { clone := *info return &clone, nil @@ -205,8 +205,8 @@ func (b *InMemoryBackend) ListConnectionTypes() []*ConnectionTypeInfo { byName := builtInConnectionTypes() b.mu.RLock("ListConnectionTypes") - for name, info := range b.customConnectionTypes { - byName[name] = *info + for _, info := range b.customConnectionTypes.All() { + byName[info.ConnectionType] = *info } b.mu.RUnlock() diff --git a/services/glue/backend_mltasks.go b/services/glue/backend_mltasks.go index d6699bcd1..f8a10351a 100644 --- a/services/glue/backend_mltasks.go +++ b/services/glue/backend_mltasks.go @@ -41,7 +41,7 @@ func mlTaskRunKey(transformID, taskRunID string) string { } func (b *InMemoryBackend) startMLTaskRunLocked(transformID string, taskType MLTaskType) (*MLTaskRun, error) { - if _, ok := b.mlTransforms[transformID]; !ok { + if !b.mlTransforms.Has(transformID) { return nil, fmt.Errorf("ML transform %q not found: %w", transformID, ErrNotFound) } @@ -53,7 +53,7 @@ func (b *InMemoryBackend) startMLTaskRunLocked(transformID string, taskType MLTa Status: stateRunning, StartedOn: float64(time.Now().Unix()), } - b.mlTaskRuns[mlTaskRunKey(transformID, taskRunID)] = run + b.mlTaskRuns.Put(run) cp := *run return &cp, nil @@ -96,7 +96,7 @@ func (b *InMemoryBackend) GetMLTaskRun(transformID, taskRunID string) (*MLTaskRu b.mu.RLock("GetMLTaskRun") defer b.mu.RUnlock() - run, ok := b.mlTaskRuns[mlTaskRunKey(transformID, taskRunID)] + run, ok := b.mlTaskRuns.Get(mlTaskRunKey(transformID, taskRunID)) if !ok { return nil, ErrMLTaskRunNotFound } @@ -111,16 +111,16 @@ func (b *InMemoryBackend) GetMLTaskRuns(transformID string) ([]*MLTaskRun, error b.mu.RLock("GetMLTaskRuns") defer b.mu.RUnlock() - if _, ok := b.mlTransforms[transformID]; !ok { + if !b.mlTransforms.Has(transformID) { return nil, fmt.Errorf("ML transform %q not found: %w", transformID, ErrNotFound) } prefix := transformID + "|" out := make([]*MLTaskRun, 0) - for _, k := range sortedKeys(b.mlTaskRuns) { - if strings.HasPrefix(k, prefix) { - cp := *b.mlTaskRuns[k] + for _, r := range b.mlTaskRuns.Snapshot() { + if k := mlTaskRunEntryKeyFn(r); strings.HasPrefix(k, prefix) { + cp := *r out = append(out, &cp) } } @@ -137,7 +137,7 @@ func (b *InMemoryBackend) CancelMLTaskRun(transformID, taskRunID string) error { key := mlTaskRunKey(transformID, taskRunID) - run, ok := b.mlTaskRuns[key] + run, ok := b.mlTaskRuns.Get(key) if !ok { return ErrMLTaskRunNotFound } @@ -153,8 +153,9 @@ func (b *InMemoryBackend) ListDataQualityEvaluationRuns() []*DataQualityEvaluati b.mu.RLock("ListDataQualityEvaluationRuns") defer b.mu.RUnlock() - out := make([]*DataQualityEvaluationRun, 0, len(b.dataQualityEvalRuns)) - for _, r := range b.dataQualityEvalRuns { + src := b.dataQualityEvalRuns.All() + out := make([]*DataQualityEvaluationRun, 0, len(src)) + for _, r := range src { cp := *r out = append(out, &cp) } @@ -169,8 +170,9 @@ func (b *InMemoryBackend) ListDataQualityResults() []*DataQualityResult { b.mu.RLock("ListDataQualityResults") defer b.mu.RUnlock() - out := make([]*DataQualityResult, 0, len(b.dataQualityResult)) - for _, r := range b.dataQualityResult { + src := b.dataQualityResult.All() + out := make([]*DataQualityResult, 0, len(src)) + for _, r := range src { cp := *r out = append(out, &cp) } diff --git a/services/glue/backend_new_ops.go b/services/glue/backend_new_ops.go index 12cb4dca2..272b19d6a 100644 --- a/services/glue/backend_new_ops.go +++ b/services/glue/backend_new_ops.go @@ -58,11 +58,11 @@ func (b *InMemoryBackend) CreateUserDefinedFunction( b.mu.Lock("CreateUserDefinedFunction") defer b.mu.Unlock() - if _, ok := b.databases[dbName]; !ok { + if !b.databases.Has(dbName) { return nil, fmt.Errorf("database %q not found: %w", dbName, ErrNotFound) } key := b.udfKey(dbName, input.FunctionName) - if _, exists := b.udfs[key]; exists { + if b.udfs.Has(key) { return nil, fmt.Errorf( "user defined function %q already exists in database %q: %w", input.FunctionName, @@ -74,7 +74,7 @@ func (b *InMemoryBackend) CreateUserDefinedFunction( udf.DatabaseName = dbName udf.FunctionARN = b.udfARN(dbName, input.FunctionName) udf.CreateTime = float64(time.Now().Unix()) - b.udfs[key] = &udf + b.udfs.Put(&udf) if len(tags) > 0 { _ = b.tagResource(udf.FunctionARN, tags) } @@ -88,7 +88,7 @@ func (b *InMemoryBackend) GetUserDefinedFunction( b.mu.RLock("GetUserDefinedFunction") defer b.mu.RUnlock() - u, ok := b.udfs[b.udfKey(dbName, name)] + u, ok := b.udfs.Get(b.udfKey(dbName, name)) if !ok { return nil, fmt.Errorf( "user defined function %q not found in database %q: %w", @@ -106,9 +106,8 @@ func (b *InMemoryBackend) GetUserDefinedFunctions(dbName string) []*UserDefinedF defer b.mu.RUnlock() var out []*UserDefinedFunction - for key, u := range b.udfs { + for _, u := range b.udfs.All() { if dbName == "" || u.DatabaseName == dbName { - _ = key out = append(out, cloneUDF(u)) } } @@ -125,7 +124,7 @@ func (b *InMemoryBackend) UpdateUserDefinedFunction( defer b.mu.Unlock() key := b.udfKey(dbName, name) - existing, ok := b.udfs[key] + existing, ok := b.udfs.Get(key) if !ok { return fmt.Errorf( "user defined function %q not found in database %q: %w", @@ -138,7 +137,7 @@ func (b *InMemoryBackend) UpdateUserDefinedFunction( input.FunctionName = name input.FunctionARN = existing.FunctionARN input.CreateTime = existing.CreateTime - b.udfs[key] = &input + b.udfs.Put(&input) return nil } @@ -148,7 +147,7 @@ func (b *InMemoryBackend) DeleteUserDefinedFunction(dbName, name string) error { defer b.mu.Unlock() key := b.udfKey(dbName, name) - if _, ok := b.udfs[key]; !ok { + if !b.udfs.Has(key) { return fmt.Errorf( "user defined function %q not found in database %q: %w", name, @@ -156,7 +155,7 @@ func (b *InMemoryBackend) DeleteUserDefinedFunction(dbName, name string) error { ErrNotFound, ) } - delete(b.udfs, key) + b.udfs.Delete(key) return nil } @@ -210,7 +209,7 @@ func (b *InMemoryBackend) CreateSecurityConfiguration( b.mu.Lock("CreateSecurityConfiguration") defer b.mu.Unlock() - if _, exists := b.securityConfigs[name]; exists { + if b.securityConfigs.Has(name) { return nil, fmt.Errorf( "security configuration %q already exists: %w", name, @@ -222,7 +221,7 @@ func (b *InMemoryBackend) CreateSecurityConfiguration( EncryptionConfiguration: enc, CreatedTimeStamp: float64(time.Now().Unix()), } - b.securityConfigs[name] = sc + b.securityConfigs.Put(sc) return cloneSecurityConfig(sc), nil } @@ -231,7 +230,7 @@ func (b *InMemoryBackend) GetSecurityConfiguration(name string) (*SecurityConfig b.mu.RLock("GetSecurityConfiguration") defer b.mu.RUnlock() - sc, ok := b.securityConfigs[name] + sc, ok := b.securityConfigs.Get(name) if !ok { return nil, fmt.Errorf("security configuration %q not found: %w", name, ErrNotFound) } @@ -243,10 +242,10 @@ func (b *InMemoryBackend) DeleteSecurityConfiguration(name string) error { b.mu.Lock("DeleteSecurityConfiguration") defer b.mu.Unlock() - if _, ok := b.securityConfigs[name]; !ok { + if !b.securityConfigs.Has(name) { return fmt.Errorf("security configuration %q not found: %w", name, ErrNotFound) } - delete(b.securityConfigs, name) + b.securityConfigs.Delete(name) return nil } @@ -255,8 +254,9 @@ func (b *InMemoryBackend) ListSecurityConfigurations() []*SecurityConfiguration b.mu.RLock("ListSecurityConfigurations") defer b.mu.RUnlock() - out := make([]*SecurityConfiguration, 0, len(b.securityConfigs)) - for _, sc := range b.securityConfigs { + src := b.securityConfigs.All() + out := make([]*SecurityConfiguration, 0, len(src)) + for _, sc := range src { out = append(out, cloneSecurityConfig(sc)) } sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) @@ -323,7 +323,7 @@ func (b *InMemoryBackend) CreateSession( b.mu.Lock("CreateSession") defer b.mu.Unlock() - if _, exists := b.sessions[id]; exists { + if b.sessions.Has(id) { return nil, fmt.Errorf("session %q already exists: %w", id, ErrAlreadyExists) } s := &Session{ @@ -337,7 +337,7 @@ func (b *InMemoryBackend) CreateSession( Description: opts.Description, DefaultArguments: opts.DefaultArguments, } - b.sessions[id] = s + b.sessions.Put(s) b.sessionStatements[id] = nil return cloneSession(s), nil @@ -347,7 +347,7 @@ func (b *InMemoryBackend) GetSession(id string) (*Session, error) { b.mu.RLock("GetSession") defer b.mu.RUnlock() - s, ok := b.sessions[id] + s, ok := b.sessions.Get(id) if !ok { return nil, fmt.Errorf("session %q not found: %w", id, ErrNotFound) } @@ -359,8 +359,9 @@ func (b *InMemoryBackend) ListSessions() []*Session { b.mu.RLock("ListSessions") defer b.mu.RUnlock() - out := make([]*Session, 0, len(b.sessions)) - for _, s := range b.sessions { + src := b.sessions.All() + out := make([]*Session, 0, len(src)) + for _, s := range src { out = append(out, cloneSession(s)) } sort.Slice(out, func(i, j int) bool { return out[i].SessionID < out[j].SessionID }) @@ -372,10 +373,10 @@ func (b *InMemoryBackend) DeleteSession(id string) error { b.mu.Lock("DeleteSession") defer b.mu.Unlock() - if _, ok := b.sessions[id]; !ok { + if !b.sessions.Has(id) { return fmt.Errorf("session %q not found: %w", id, ErrNotFound) } - delete(b.sessions, id) + b.sessions.Delete(id) delete(b.sessionStatements, id) return nil @@ -385,7 +386,7 @@ func (b *InMemoryBackend) StopSession(id string) error { b.mu.Lock("StopSession") defer b.mu.Unlock() - s, ok := b.sessions[id] + s, ok := b.sessions.Get(id) if !ok { return fmt.Errorf("session %q not found: %w", id, ErrNotFound) } @@ -398,7 +399,7 @@ func (b *InMemoryBackend) RunStatement(sessionID, code string) (*Statement, erro b.mu.Lock("RunStatement") defer b.mu.Unlock() - if _, ok := b.sessions[sessionID]; !ok { + if !b.sessions.Has(sessionID) { return nil, fmt.Errorf("session %q not found: %w", sessionID, ErrNotFound) } stmts := b.sessionStatements[sessionID] @@ -440,7 +441,7 @@ func (b *InMemoryBackend) GetStatements(sessionID string) ([]*Statement, error) b.mu.RLock("GetStatements") defer b.mu.RUnlock() - if _, ok := b.sessions[sessionID]; !ok { + if !b.sessions.Has(sessionID) { return nil, fmt.Errorf("session %q not found: %w", sessionID, ErrNotFound) } stmts := b.sessionStatements[sessionID] @@ -532,7 +533,7 @@ func (b *InMemoryBackend) CreateTableOptimizer( defer b.mu.Unlock() key := b.tableOptimizerKey(dbName, tableName, optimizerType) - if _, exists := b.tableOptimizers[key]; exists { + if b.tableOptimizers.Has(key) { return fmt.Errorf( "table optimizer type %q already exists for %s.%s: %w", optimizerType, @@ -543,7 +544,7 @@ func (b *InMemoryBackend) CreateTableOptimizer( } now := float64(time.Now().Unix()) - b.tableOptimizers[key] = &TableOptimizer{ + b.tableOptimizers.Put(&TableOptimizer{ CatalogID: catalogID, DatabaseName: dbName, TableName: tableName, @@ -554,7 +555,7 @@ func (b *InMemoryBackend) CreateTableOptimizer( StartedAt: now, EndedAt: now, }, - } + }) return nil } @@ -565,7 +566,7 @@ func (b *InMemoryBackend) GetTableOptimizer( b.mu.RLock("GetTableOptimizer") defer b.mu.RUnlock() - to, ok := b.tableOptimizers[b.tableOptimizerKey(dbName, tableName, optimizerType)] + to, ok := b.tableOptimizers.Get(b.tableOptimizerKey(dbName, tableName, optimizerType)) if !ok { return nil, fmt.Errorf( "table optimizer %q not found for %s.%s: %w", @@ -587,7 +588,7 @@ func (b *InMemoryBackend) UpdateTableOptimizer( defer b.mu.Unlock() key := b.tableOptimizerKey(dbName, tableName, optimizerType) - to, ok := b.tableOptimizers[key] + to, ok := b.tableOptimizers.Get(key) if !ok { return fmt.Errorf( "table optimizer %q not found for %s.%s: %w", @@ -607,7 +608,7 @@ func (b *InMemoryBackend) DeleteTableOptimizer(dbName, tableName, optimizerType defer b.mu.Unlock() key := b.tableOptimizerKey(dbName, tableName, optimizerType) - if _, ok := b.tableOptimizers[key]; !ok { + if !b.tableOptimizers.Has(key) { return fmt.Errorf( "table optimizer %q not found for %s.%s: %w", optimizerType, @@ -616,7 +617,7 @@ func (b *InMemoryBackend) DeleteTableOptimizer(dbName, tableName, optimizerType ErrNotFound, ) } - delete(b.tableOptimizers, key) + b.tableOptimizers.Delete(key) return nil } @@ -648,7 +649,7 @@ func (b *InMemoryBackend) BatchGetTableOptimizer( var errs []BatchGetTableOptimizerError for _, e := range entries { key := b.tableOptimizerKey(e.DatabaseName, e.TableName, e.Type) - if to, ok := b.tableOptimizers[key]; ok { + if to, ok := b.tableOptimizers.Get(key); ok { found = append(found, cloneTableOptimizer(to)) } else { errs = append(errs, BatchGetTableOptimizerError{ @@ -705,7 +706,7 @@ func (b *InMemoryBackend) UpdateColumnStatisticsForTable( b.mu.Lock("UpdateColumnStatisticsForTable") defer b.mu.Unlock() - if _, ok := b.databases[dbName]; !ok { + if !b.databases.Has(dbName) { return fmt.Errorf("database %q not found: %w", dbName, ErrNotFound) } for _, cs := range stats { @@ -979,7 +980,7 @@ func (b *InMemoryBackend) CreateMLTransform( CreatedOn: float64(time.Now().Unix()), LastModifiedOn: float64(time.Now().Unix()), } - b.mlTransforms[id] = m + b.mlTransforms.Put(m) if len(tags) > 0 { _ = b.tagResource(b.mlTransformARN(id), tags) } @@ -991,7 +992,7 @@ func (b *InMemoryBackend) GetMLTransform(id string) (*MLTransform, error) { b.mu.RLock("GetMLTransform") defer b.mu.RUnlock() - m, ok := b.mlTransforms[id] + m, ok := b.mlTransforms.Get(id) if !ok { return nil, fmt.Errorf("ML transform %q not found: %w", id, ErrNotFound) } @@ -1003,8 +1004,9 @@ func (b *InMemoryBackend) GetMLTransforms() []*MLTransform { b.mu.RLock("GetMLTransforms") defer b.mu.RUnlock() - out := make([]*MLTransform, 0, len(b.mlTransforms)) - for _, m := range b.mlTransforms { + src := b.mlTransforms.All() + out := make([]*MLTransform, 0, len(src)) + for _, m := range src { out = append(out, cloneMLTransform(m)) } sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) @@ -1016,14 +1018,14 @@ func (b *InMemoryBackend) UpdateMLTransform(id string, update MLTransform) error b.mu.Lock("UpdateMLTransform") defer b.mu.Unlock() - existing, ok := b.mlTransforms[id] + existing, ok := b.mlTransforms.Get(id) if !ok { return fmt.Errorf("ML transform %q not found: %w", id, ErrNotFound) } update.TransformID = id update.CreatedOn = existing.CreatedOn update.LastModifiedOn = float64(time.Now().Unix()) - b.mlTransforms[id] = &update + b.mlTransforms.Put(&update) return nil } @@ -1032,10 +1034,10 @@ func (b *InMemoryBackend) DeleteMLTransform(id string) error { b.mu.Lock("DeleteMLTransform") defer b.mu.Unlock() - if _, ok := b.mlTransforms[id]; !ok { + if !b.mlTransforms.Has(id) { return fmt.Errorf("ML transform %q not found: %w", id, ErrNotFound) } - delete(b.mlTransforms, id) + b.mlTransforms.Delete(id) return nil } @@ -1070,16 +1072,16 @@ func (b *InMemoryBackend) CreateCatalog( b.mu.Lock("CreateCatalog") defer b.mu.Unlock() - if _, exists := b.catalogs[catalogID]; exists { + if b.catalogs.Has(catalogID) { return fmt.Errorf("catalog %q already exists: %w", catalogID, ErrAlreadyExists) } - b.catalogs[catalogID] = &CatalogEntry{ + b.catalogs.Put(&CatalogEntry{ CatalogID: catalogID, Name: name, Description: description, Parameters: params, CreateTime: float64(time.Now().Unix()), - } + }) return nil } @@ -1088,7 +1090,7 @@ func (b *InMemoryBackend) GetCatalog(catalogID string) (*CatalogEntry, error) { b.mu.RLock("GetCatalog") defer b.mu.RUnlock() - c, ok := b.catalogs[catalogID] + c, ok := b.catalogs.Get(catalogID) if !ok { return nil, fmt.Errorf("catalog %q not found: %w", catalogID, ErrNotFound) } @@ -1100,8 +1102,9 @@ func (b *InMemoryBackend) GetCatalogs() []*CatalogEntry { b.mu.RLock("GetCatalogs") defer b.mu.RUnlock() - out := make([]*CatalogEntry, 0, len(b.catalogs)) - for _, c := range b.catalogs { + src := b.catalogs.All() + out := make([]*CatalogEntry, 0, len(src)) + for _, c := range src { out = append(out, cloneCatalogEntry(c)) } sort.Slice(out, func(i, j int) bool { return out[i].CatalogID < out[j].CatalogID }) @@ -1116,7 +1119,7 @@ func (b *InMemoryBackend) UpdateCatalog( b.mu.Lock("UpdateCatalog") defer b.mu.Unlock() - c, ok := b.catalogs[catalogID] + c, ok := b.catalogs.Get(catalogID) if !ok { return fmt.Errorf("catalog %q not found: %w", catalogID, ErrNotFound) } @@ -1132,10 +1135,10 @@ func (b *InMemoryBackend) DeleteCatalog(catalogID string) error { b.mu.Lock("DeleteCatalog") defer b.mu.Unlock() - if _, ok := b.catalogs[catalogID]; !ok { + if !b.catalogs.Has(catalogID) { return fmt.Errorf("catalog %q not found: %w", catalogID, ErrNotFound) } - delete(b.catalogs, catalogID) + b.catalogs.Delete(catalogID) return nil } @@ -1203,7 +1206,7 @@ func (b *InMemoryBackend) DeleteTableVersion(dbName, tableName, versionID string defer b.mu.Unlock() key := tableVersionKey(dbName, tableName, versionID) - if _, ok := b.tableVersions[key]; !ok { + if !b.tableVersions.Has(key) { return fmt.Errorf( "table version %q not found for %s.%s: %w", versionID, @@ -1212,7 +1215,7 @@ func (b *InMemoryBackend) DeleteTableVersion(dbName, tableName, versionID string ErrNotFound, ) } - delete(b.tableVersions, key) + b.tableVersions.Delete(key) return nil } @@ -1221,7 +1224,7 @@ func (b *InMemoryBackend) UpdateDevEndpoint(name string, args map[string]string) b.mu.Lock("UpdateDevEndpoint") defer b.mu.Unlock() - dep, ok := b.devEndpoints[name] + dep, ok := b.devEndpoints.Get(name) if !ok { return fmt.Errorf("dev endpoint %q not found: %w", name, ErrNotFound) } diff --git a/services/glue/backend_parity_a.go b/services/glue/backend_parity_a.go index ea736d680..942dc7b48 100644 --- a/services/glue/backend_parity_a.go +++ b/services/glue/backend_parity_a.go @@ -127,7 +127,7 @@ func (b *InMemoryBackend) GetSchemaByDefinition( b.mu.RLock("GetSchemaByDefinition") defer b.mu.RUnlock() - s, ok := b.schemas[schemaKey(registryName, schemaName)] + s, ok := b.schemas.Get(schemaKey(registryName, schemaName)) if !ok { return nil, fmt.Errorf("schema %q/%q not found: %w", registryName, schemaName, ErrNotFound) } @@ -158,7 +158,7 @@ func (b *InMemoryBackend) GetSchemaVersionsDiff( b.mu.RLock("GetSchemaVersionsDiff") defer b.mu.RUnlock() - s, ok := b.schemas[schemaKey(registryName, schemaName)] + s, ok := b.schemas.Get(schemaKey(registryName, schemaName)) if !ok { return "", fmt.Errorf("schema %q/%q not found: %w", registryName, schemaName, ErrNotFound) } diff --git a/services/glue/backend_parity_glue.go b/services/glue/backend_parity_glue.go index a0c3e6fde..3a137e5af 100644 --- a/services/glue/backend_parity_glue.go +++ b/services/glue/backend_parity_glue.go @@ -11,7 +11,7 @@ func (b *InMemoryBackend) DeleteSchemaVersion( b.mu.Lock("DeleteSchemaVersion") defer b.mu.Unlock() - s, ok := b.schemas[schemaKey(registryName, schemaName)] + s, ok := b.schemas.Get(schemaKey(registryName, schemaName)) if !ok { return fmt.Errorf("schema %q/%q not found: %w", registryName, schemaName, ErrNotFound) } @@ -40,11 +40,11 @@ func (b *InMemoryBackend) DeleteIntegrationResourceProperty(resourceArn string) b.mu.Lock("DeleteIntegrationResourceProperty") defer b.mu.Unlock() - if _, ok := b.integrationResourceProps[resourceArn]; !ok { + if !b.integrationResourceProps.Has(resourceArn) { return fmt.Errorf("resource property for %q not found: %w", resourceArn, ErrNotFound) } - delete(b.integrationResourceProps, resourceArn) + b.integrationResourceProps.Delete(resourceArn) return nil } @@ -60,7 +60,7 @@ func (b *InMemoryBackend) DeleteIntegrationTableProperties(resourceArn, tableNam defer b.mu.Unlock() key := resourceArn + "|" + tableName - if _, ok := b.integrationTableProps[key]; !ok { + if !b.integrationTableProps.Has(key) { return fmt.Errorf( "table property for %q/%q not found: %w", resourceArn, @@ -69,7 +69,7 @@ func (b *InMemoryBackend) DeleteIntegrationTableProperties(resourceArn, tableNam ) } - delete(b.integrationTableProps, key) + b.integrationTableProps.Delete(key) return nil } diff --git a/services/glue/backend_partition_indexes.go b/services/glue/backend_partition_indexes.go index fe2f99bde..af1e406ba 100644 --- a/services/glue/backend_partition_indexes.go +++ b/services/glue/backend_partition_indexes.go @@ -29,7 +29,7 @@ func (b *InMemoryBackend) CreatePartitionIndex(dbName, tableName string, input P b.mu.Lock("CreatePartitionIndex") defer b.mu.Unlock() - table, ok := b.tables[tableKey(dbName, tableName)] + table, ok := b.tables.Get(tableKey(dbName, tableName)) if !ok { return fmt.Errorf("table %s.%s not found: %w", dbName, tableName, ErrNotFound) } @@ -92,7 +92,7 @@ func (b *InMemoryBackend) GetPartitionIndexes(dbName, tableName string) ([]*Part b.mu.RLock("GetPartitionIndexes") defer b.mu.RUnlock() - if _, ok := b.tables[tableKey(dbName, tableName)]; !ok { + if !b.tables.Has(tableKey(dbName, tableName)) { return nil, fmt.Errorf("table %s.%s not found: %w", dbName, tableName, ErrNotFound) } diff --git a/services/glue/backend_registry.go b/services/glue/backend_registry.go index 9e93e36c9..e82380f72 100644 --- a/services/glue/backend_registry.go +++ b/services/glue/backend_registry.go @@ -9,7 +9,6 @@ import ( "github.com/google/uuid" "github.com/blackbirdworks/gopherstack/pkgs/arn" - "github.com/blackbirdworks/gopherstack/pkgs/collections" ) // Registry represents a Glue Schema Registry. @@ -101,7 +100,7 @@ func (b *InMemoryBackend) CreateRegistry( return nil, ErrValidation } - if _, ok := b.registries[name]; ok { + if b.registries.Has(name) { return nil, ErrAlreadyExists } @@ -115,7 +114,7 @@ func (b *InMemoryBackend) CreateRegistry( UpdatedTime: float64(time.Now().Unix()), } - b.registries[name] = reg + b.registries.Put(reg) return reg, nil } @@ -125,7 +124,7 @@ func (b *InMemoryBackend) DescribeRegistry(name string) (*Registry, error) { b.mu.RLock("DescribeRegistry") defer b.mu.RUnlock() - reg, ok := b.registries[name] + reg, ok := b.registries.Get(name) if !ok { return nil, ErrNotFound } @@ -141,12 +140,12 @@ func (b *InMemoryBackend) ListRegistries() []*Registry { b.mu.RLock("ListRegistries") defer b.mu.RUnlock() - keys := collections.SortedKeys(b.registries) + src := b.registries.Snapshot() - out := make([]*Registry, 0, len(keys)) - for _, k := range keys { - cp := *b.registries[k] - cp.Tags = maps.Clone(b.registries[k].Tags) + out := make([]*Registry, 0, len(src)) + for _, reg := range src { + cp := *reg + cp.Tags = maps.Clone(reg.Tags) out = append(out, &cp) } @@ -158,7 +157,7 @@ func (b *InMemoryBackend) UpdateRegistry(name, description string) error { b.mu.Lock("UpdateRegistry") defer b.mu.Unlock() - reg, ok := b.registries[name] + reg, ok := b.registries.Get(name) if !ok { return ErrNotFound } @@ -174,11 +173,11 @@ func (b *InMemoryBackend) DeleteRegistry(name string) error { b.mu.Lock("DeleteRegistry") defer b.mu.Unlock() - if _, ok := b.registries[name]; !ok { + if !b.registries.Has(name) { return ErrNotFound } - delete(b.registries, name) + b.registries.Delete(name) return nil } @@ -198,7 +197,7 @@ func (b *InMemoryBackend) CreateSchema( } key := schemaKey(registryName, schemaName) - if _, ok := b.schemas[key]; ok { + if b.schemas.Has(key) { return nil, ErrAlreadyExists } @@ -221,7 +220,7 @@ func (b *InMemoryBackend) CreateSchema( CheckpointVersion: 1, } - b.schemas[key] = s + b.schemas.Put(s) b.schemaVersions[schemaVersionListKey(schARN)] = nil // init empty version list return s, nil @@ -232,7 +231,7 @@ func (b *InMemoryBackend) DescribeSchema(registryName, schemaName string) (*Sche b.mu.RLock("DescribeSchema") defer b.mu.RUnlock() - s, ok := b.schemas[schemaKey(registryName, schemaName)] + s, ok := b.schemas.Get(schemaKey(registryName, schemaName)) if !ok { return nil, ErrNotFound } @@ -248,10 +247,10 @@ func (b *InMemoryBackend) ListSchemas(registryName string) []*Schema { b.mu.RLock("ListSchemas") defer b.mu.RUnlock() - out := make([]*Schema, 0, len(b.schemas)) - for key, s := range b.schemas { + src := b.schemas.All() + out := make([]*Schema, 0, len(src)) + for _, s := range src { if registryName == "" || s.RegistryName == registryName { - _ = key cp := *s cp.Tags = maps.Clone(s.Tags) out = append(out, &cp) @@ -272,7 +271,7 @@ func (b *InMemoryBackend) UpdateSchema( b.mu.Lock("UpdateSchema") defer b.mu.Unlock() - s, ok := b.schemas[schemaKey(registryName, schemaName)] + s, ok := b.schemas.Get(schemaKey(registryName, schemaName)) if !ok { return ErrNotFound } @@ -297,13 +296,13 @@ func (b *InMemoryBackend) DeleteSchema(registryName, schemaName string) error { key := schemaKey(registryName, schemaName) - s, ok := b.schemas[key] + s, ok := b.schemas.Get(key) if !ok { return ErrNotFound } delete(b.schemaVersions, schemaVersionListKey(s.SchemaARN)) - delete(b.schemas, key) + b.schemas.Delete(key) return nil } @@ -317,7 +316,7 @@ func (b *InMemoryBackend) RegisterSchemaVersion( b.mu.Lock("RegisterSchemaVersion") defer b.mu.Unlock() - s, ok := b.schemas[schemaKey(registryName, schemaName)] + s, ok := b.schemas.Get(schemaKey(registryName, schemaName)) if !ok { return nil, ErrNotFound } @@ -351,7 +350,7 @@ func (b *InMemoryBackend) GetSchemaVersion( b.mu.RLock("GetSchemaVersion") defer b.mu.RUnlock() - s, ok := b.schemas[schemaKey(registryName, schemaName)] + s, ok := b.schemas.Get(schemaKey(registryName, schemaName)) if !ok { return nil, ErrNotFound } @@ -372,7 +371,7 @@ func (b *InMemoryBackend) ListSchemaVersions(registryName, schemaName string) [] b.mu.RLock("ListSchemaVersions") defer b.mu.RUnlock() - s, ok := b.schemas[schemaKey(registryName, schemaName)] + s, ok := b.schemas.Get(schemaKey(registryName, schemaName)) if !ok { return nil } @@ -398,10 +397,11 @@ func (b *InMemoryBackend) GetTableVersions(dbName, tableName string) []*TableVer defer b.mu.RUnlock() prefix := tableVersionKey(dbName, tableName, "") - out := make([]*TableVersion, 0, len(b.tableVersions)) + src := b.tableVersions.Snapshot() + out := make([]*TableVersion, 0, len(src)) - for k, tv := range b.tableVersions { - if len(k) > len(prefix) && k[:len(prefix)] == prefix { + for _, tv := range src { + if k := tableVersionEntryKeyFn(tv); len(k) > len(prefix) && k[:len(prefix)] == prefix { cp := *tv if tv.Table != nil { t := *tv.Table @@ -428,7 +428,7 @@ func (b *InMemoryBackend) GetTableVersion( key := tableVersionKey(dbName, tableName, versionID) - tv, ok := b.tableVersions[key] + tv, ok := b.tableVersions.Get(key) if !ok { return nil, ErrNotFound } @@ -452,8 +452,8 @@ func (b *InMemoryBackend) GetCrawlerMetrics(crawlerNames []string) []*CrawlerMet defer b.mu.RUnlock() if len(crawlerNames) == 0 { - for k := range b.crawlers { - crawlerNames = append(crawlerNames, k) + for _, c := range b.crawlers.All() { + crawlerNames = append(crawlerNames, c.Name) } sort.Strings(crawlerNames) @@ -462,7 +462,7 @@ func (b *InMemoryBackend) GetCrawlerMetrics(crawlerNames []string) []*CrawlerMet out := make([]*CrawlerMetrics, 0, len(crawlerNames)) for _, name := range crawlerNames { - c, ok := b.crawlers[name] + c, ok := b.crawlers.Get(name) if !ok { continue } diff --git a/services/glue/backend_triggers_workflows.go b/services/glue/backend_triggers_workflows.go index f6bed66c5..a85f1900c 100644 --- a/services/glue/backend_triggers_workflows.go +++ b/services/glue/backend_triggers_workflows.go @@ -169,7 +169,7 @@ func (b *InMemoryBackend) CreateTrigger(t Trigger, tags map[string]string) (*Tri return nil, ErrValidation } - if _, ok := b.triggers[t.Name]; ok { + if b.triggers.Has(t.Name) { return nil, ErrAlreadyExists } @@ -180,7 +180,7 @@ func (b *InMemoryBackend) CreateTrigger(t Trigger, tags map[string]string) (*Tri stored.State = "CREATED" } - b.triggers[t.Name] = stored + b.triggers.Put(stored) return cloneTrigger(stored), nil } @@ -190,7 +190,7 @@ func (b *InMemoryBackend) GetTrigger(name string) (*Trigger, error) { b.mu.RLock("GetTrigger") defer b.mu.RUnlock() - t, ok := b.triggers[name] + t, ok := b.triggers.Get(name) if !ok { return nil, ErrNotFound } @@ -203,9 +203,10 @@ func (b *InMemoryBackend) GetTriggers() []*Trigger { b.mu.RLock("GetTriggers") defer b.mu.RUnlock() - out := make([]*Trigger, 0, len(b.triggers)) - for _, k := range sortedKeys(b.triggers) { - out = append(out, cloneTrigger(b.triggers[k])) + src := b.triggers.Snapshot() + out := make([]*Trigger, 0, len(src)) + for _, t := range src { + out = append(out, cloneTrigger(t)) } return out @@ -220,7 +221,7 @@ func (b *InMemoryBackend) BatchGetTriggers(names []string) ([]*Trigger, []string missing := make([]string, 0, len(names)) for _, name := range names { - t, ok := b.triggers[name] + t, ok := b.triggers.Get(name) if !ok { missing = append(missing, name) @@ -238,7 +239,7 @@ func (b *InMemoryBackend) UpdateTrigger(name string, update Trigger) error { b.mu.Lock("UpdateTrigger") defer b.mu.Unlock() - t, ok := b.triggers[name] + t, ok := b.triggers.Get(name) if !ok { return ErrNotFound } @@ -255,11 +256,11 @@ func (b *InMemoryBackend) DeleteTrigger(name string) error { b.mu.Lock("DeleteTrigger") defer b.mu.Unlock() - if _, ok := b.triggers[name]; !ok { + if !b.triggers.Has(name) { return ErrNotFound } - delete(b.triggers, name) + b.triggers.Delete(name) return nil } @@ -269,7 +270,7 @@ func (b *InMemoryBackend) StartTrigger(name string) error { b.mu.Lock("StartTrigger") defer b.mu.Unlock() - t, ok := b.triggers[name] + t, ok := b.triggers.Get(name) if !ok { return ErrNotFound } @@ -284,7 +285,7 @@ func (b *InMemoryBackend) StopTrigger(name string) error { b.mu.Lock("StopTrigger") defer b.mu.Unlock() - t, ok := b.triggers[name] + t, ok := b.triggers.Get(name) if !ok { return ErrNotFound } @@ -305,7 +306,7 @@ func (b *InMemoryBackend) CreateWorkflow(w Workflow, tags map[string]string) (*W return nil, ErrValidation } - if _, ok := b.workflows[w.Name]; ok { + if b.workflows.Has(w.Name) { return nil, ErrAlreadyExists } @@ -316,7 +317,7 @@ func (b *InMemoryBackend) CreateWorkflow(w Workflow, tags map[string]string) (*W stored.CreatedOn = now stored.LastModifiedOn = now - b.workflows[w.Name] = stored + b.workflows.Put(stored) return cloneWorkflow(stored), nil } @@ -326,7 +327,7 @@ func (b *InMemoryBackend) GetWorkflow(name string) (*Workflow, error) { b.mu.RLock("GetWorkflow") defer b.mu.RUnlock() - w, ok := b.workflows[name] + w, ok := b.workflows.Get(name) if !ok { return nil, ErrNotFound } @@ -339,7 +340,13 @@ func (b *InMemoryBackend) GetWorkflows() []string { b.mu.RLock("GetWorkflows") defer b.mu.RUnlock() - return sortedKeys(b.workflows) + src := b.workflows.Snapshot() + out := make([]string, len(src)) + for i, w := range src { + out[i] = w.Name + } + + return out } // BatchGetWorkflows retrieves multiple workflows by name. @@ -351,7 +358,7 @@ func (b *InMemoryBackend) BatchGetWorkflows(names []string) ([]*Workflow, []stri missing := make([]string, 0, len(names)) for _, name := range names { - w, ok := b.workflows[name] + w, ok := b.workflows.Get(name) if !ok { missing = append(missing, name) @@ -369,7 +376,7 @@ func (b *InMemoryBackend) UpdateWorkflow(name string, update Workflow) error { b.mu.Lock("UpdateWorkflow") defer b.mu.Unlock() - w, ok := b.workflows[name] + w, ok := b.workflows.Get(name) if !ok { return ErrNotFound } @@ -386,11 +393,11 @@ func (b *InMemoryBackend) DeleteWorkflow(name string) error { b.mu.Lock("DeleteWorkflow") defer b.mu.Unlock() - if _, ok := b.workflows[name]; !ok { + if !b.workflows.Has(name) { return ErrNotFound } - delete(b.workflows, name) + b.workflows.Delete(name) delete(b.workflowRuns, name) return nil @@ -401,7 +408,7 @@ func (b *InMemoryBackend) StartWorkflowRun(name string) (*WorkflowRun, error) { b.mu.Lock("StartWorkflowRun") defer b.mu.Unlock() - if _, ok := b.workflows[name]; !ok { + if !b.workflows.Has(name) { return nil, ErrNotFound } @@ -442,7 +449,7 @@ func (b *InMemoryBackend) GetWorkflowRuns(workflowName string) ([]*WorkflowRun, b.mu.RLock("GetWorkflowRuns") defer b.mu.RUnlock() - if _, ok := b.workflows[workflowName]; !ok { + if !b.workflows.Has(workflowName) { return nil, ErrNotFound } @@ -468,12 +475,12 @@ func (b *InMemoryBackend) CreateClassifier(c Classifier) error { return ErrValidation } - if _, ok := b.classifiers[name]; ok { + if b.classifiers.Has(name) { return ErrAlreadyExists } cp := c - b.classifiers[name] = &cp + b.classifiers.Put(&cp) return nil } @@ -483,7 +490,7 @@ func (b *InMemoryBackend) GetClassifier(name string) (*Classifier, error) { b.mu.RLock("GetClassifier") defer b.mu.RUnlock() - c, ok := b.classifiers[name] + c, ok := b.classifiers.Get(name) if !ok { return nil, ErrNotFound } @@ -498,9 +505,10 @@ func (b *InMemoryBackend) GetClassifiers() []*Classifier { b.mu.RLock("GetClassifiers") defer b.mu.RUnlock() - out := make([]*Classifier, 0, len(b.classifiers)) - for _, k := range sortedKeys(b.classifiers) { - cp := *b.classifiers[k] + src := b.classifiers.Snapshot() + out := make([]*Classifier, 0, len(src)) + for _, c := range src { + cp := *c out = append(out, &cp) } @@ -517,12 +525,12 @@ func (b *InMemoryBackend) UpdateClassifier(c Classifier) error { return ErrValidation } - if _, ok := b.classifiers[name]; !ok { + if !b.classifiers.Has(name) { return ErrNotFound } cp := c - b.classifiers[name] = &cp + b.classifiers.Put(&cp) return nil } @@ -532,11 +540,11 @@ func (b *InMemoryBackend) DeleteClassifier(name string) error { b.mu.Lock("DeleteClassifier") defer b.mu.Unlock() - if _, ok := b.classifiers[name]; !ok { + if !b.classifiers.Has(name) { return ErrNotFound } - delete(b.classifiers, name) + b.classifiers.Delete(name) return nil } @@ -552,7 +560,7 @@ func (b *InMemoryBackend) CreateDevEndpoint(name string) (*DevEndpoint, error) { return nil, ErrValidation } - if _, ok := b.devEndpoints[name]; ok { + if b.devEndpoints.Has(name) { return nil, ErrAlreadyExists } @@ -560,7 +568,7 @@ func (b *InMemoryBackend) CreateDevEndpoint(name string) (*DevEndpoint, error) { EndpointName: name, Status: stateReady, } - b.devEndpoints[name] = dep + b.devEndpoints.Put(dep) cp := *dep @@ -572,7 +580,7 @@ func (b *InMemoryBackend) GetDevEndpoint(name string) (*DevEndpoint, error) { b.mu.RLock("GetDevEndpoint") defer b.mu.RUnlock() - dep, ok := b.devEndpoints[name] + dep, ok := b.devEndpoints.Get(name) if !ok { return nil, ErrNotFound } @@ -587,9 +595,10 @@ func (b *InMemoryBackend) GetAllDevEndpoints() []*DevEndpoint { b.mu.RLock("GetAllDevEndpoints") defer b.mu.RUnlock() - out := make([]*DevEndpoint, 0, len(b.devEndpoints)) - for _, k := range sortedKeys(b.devEndpoints) { - cp := *b.devEndpoints[k] + src := b.devEndpoints.Snapshot() + out := make([]*DevEndpoint, 0, len(src)) + for _, dep := range src { + cp := *dep out = append(out, &cp) } @@ -601,11 +610,11 @@ func (b *InMemoryBackend) DeleteDevEndpoint(name string) error { b.mu.Lock("DeleteDevEndpoint") defer b.mu.Unlock() - if _, ok := b.devEndpoints[name]; !ok { + if !b.devEndpoints.Has(name) { return ErrNotFound } - delete(b.devEndpoints, name) + b.devEndpoints.Delete(name) return nil } diff --git a/services/glue/export_test.go b/services/glue/export_test.go index ee35284af..dc3662dd2 100644 --- a/services/glue/export_test.go +++ b/services/glue/export_test.go @@ -30,7 +30,7 @@ func CustomConnectionTypeCountForTest(b *InMemoryBackend) int { b.mu.RLock("CustomConnectionTypeCountForTest") defer b.mu.RUnlock() - return len(b.customConnectionTypes) + return b.customConnectionTypes.Len() } // DatabaseCount returns the number of databases stored in the backend. Used only in tests. @@ -38,7 +38,7 @@ func DatabaseCount(b *InMemoryBackend) int { b.mu.RLock("DatabaseCount") defer b.mu.RUnlock() - return len(b.databases) + return b.databases.Len() } // TableCount returns the total number of tables stored in the backend. Used only in tests. @@ -46,7 +46,7 @@ func TableCount(b *InMemoryBackend) int { b.mu.RLock("TableCount") defer b.mu.RUnlock() - return len(b.tables) + return b.tables.Len() } // CrawlerCount returns the number of crawlers stored in the backend. Used only in tests. @@ -54,7 +54,7 @@ func CrawlerCount(b *InMemoryBackend) int { b.mu.RLock("CrawlerCount") defer b.mu.RUnlock() - return len(b.crawlers) + return b.crawlers.Len() } // JobCount returns the number of jobs stored in the backend. Used only in tests. @@ -62,7 +62,7 @@ func JobCount(b *InMemoryBackend) int { b.mu.RLock("JobCount") defer b.mu.RUnlock() - return len(b.jobs) + return b.jobs.Len() } // PartitionCount returns the number of partitions stored in the backend. Used only in tests. @@ -70,7 +70,7 @@ func PartitionCount(b *InMemoryBackend) int { b.mu.RLock("PartitionCount") defer b.mu.RUnlock() - return len(b.partitions) + return b.partitions.Len() } // TableVersionCount returns the number of table versions stored in the backend. Used only in tests. @@ -78,7 +78,7 @@ func TableVersionCount(b *InMemoryBackend) int { b.mu.RLock("TableVersionCount") defer b.mu.RUnlock() - return len(b.tableVersions) + return b.tableVersions.Len() } // ConnectionCount returns the number of connections stored in the backend. Used only in tests. @@ -86,7 +86,7 @@ func ConnectionCount(b *InMemoryBackend) int { b.mu.RLock("ConnectionCount") defer b.mu.RUnlock() - return len(b.connections) + return b.connections.Len() } // BlueprintCount returns the number of blueprints stored in the backend. Used only in tests. @@ -94,7 +94,7 @@ func BlueprintCount(b *InMemoryBackend) int { b.mu.RLock("BlueprintCount") defer b.mu.RUnlock() - return len(b.blueprints) + return b.blueprints.Len() } // CustomEntityTypeCount returns the number of custom entity types in the backend. Used only in tests. @@ -102,7 +102,7 @@ func CustomEntityTypeCount(b *InMemoryBackend) int { b.mu.RLock("CustomEntityTypeCount") defer b.mu.RUnlock() - return len(b.customEntityTypes) + return b.customEntityTypes.Len() } // DataQualityResultCount returns the number of data quality results in the backend. Used only in tests. @@ -110,7 +110,7 @@ func DataQualityResultCount(b *InMemoryBackend) int { b.mu.RLock("DataQualityResultCount") defer b.mu.RUnlock() - return len(b.dataQualityResult) + return b.dataQualityResult.Len() } // DevEndpointCount returns the number of dev endpoints in the backend. Used only in tests. @@ -118,7 +118,7 @@ func DevEndpointCount(b *InMemoryBackend) int { b.mu.RLock("DevEndpointCount") defer b.mu.RUnlock() - return len(b.devEndpoints) + return b.devEndpoints.Len() } // HandlerOpsLen returns the number of operations registered in the handler's dispatch table. @@ -144,7 +144,7 @@ func DataQualityRulesetCount(b *InMemoryBackend) int { b.mu.RLock("DataQualityRulesetCount") defer b.mu.RUnlock() - return len(b.dataQualityRulesets) + return b.dataQualityRulesets.Len() } // DataQualityEvalRunCount returns the number of data quality evaluation runs in the backend. Used only in tests. @@ -152,7 +152,7 @@ func DataQualityEvalRunCount(b *InMemoryBackend) int { b.mu.RLock("DataQualityEvalRunCount") defer b.mu.RUnlock() - return len(b.dataQualityEvalRuns) + return b.dataQualityEvalRuns.Len() } // MLTaskRunCount returns the total number of ML task runs across all transforms. Used only in tests. @@ -160,5 +160,5 @@ func MLTaskRunCount(b *InMemoryBackend) int { b.mu.RLock("MLTaskRunCount") defer b.mu.RUnlock() - return len(b.mlTaskRuns) + return b.mlTaskRuns.Len() } diff --git a/services/glue/handler.go b/services/glue/handler.go index 9e4f5b899..517329536 100644 --- a/services/glue/handler.go +++ b/services/glue/handler.go @@ -718,7 +718,7 @@ func (h *Handler) handleError(_ context.Context, c *echo.Context, _ string, err case errors.Is(err, awserr.ErrAlreadyExists): return c.JSON(http.StatusBadRequest, errorResponse("AlreadyExistsException", err.Error())) case errors.Is(err, awserr.ErrInvalidParameter): - return c.JSON(http.StatusBadRequest, errorResponse("ValidationException", err.Error())) + return c.JSON(http.StatusBadRequest, errorResponse("InvalidInputException", err.Error())) case errors.Is(err, errUnknownAction): return c.JSON(http.StatusBadRequest, errorResponse("UnknownOperationException", err.Error())) default: @@ -952,15 +952,27 @@ func (h *Handler) handleDeleteTable(_ context.Context, in *deleteTableInput) (*e // --- Crawler handlers --- type createCrawlerInput struct { - Tags map[string]string `json:"Tags,omitempty"` - Name string `json:"Name"` - Role string `json:"Role"` - DatabaseName string `json:"DatabaseName"` - Targets CrawlerTarget `json:"Targets,omitzero"` + Tags map[string]string `json:"Tags,omitempty"` + Name string `json:"Name"` + Role string `json:"Role"` + DatabaseName string `json:"DatabaseName"` + Description string `json:"Description,omitempty"` + Schedule string `json:"Schedule,omitempty"` + Configuration string `json:"Configuration,omitempty"` + TablePrefix string `json:"TablePrefix,omitempty"` + Classifiers []string `json:"Classifiers,omitempty"` + Targets CrawlerTarget `json:"Targets,omitzero"` } func (h *Handler) handleCreateCrawler(_ context.Context, in *createCrawlerInput) (*emptyOutput, error) { - if _, err := h.Backend.CreateCrawler(in.Name, in.Role, in.DatabaseName, in.Targets, in.Tags); err != nil { + _, err := h.Backend.CreateCrawlerWithOptions(in.Name, in.Role, in.DatabaseName, in.Targets, in.Tags, CrawlerOptions{ + Description: in.Description, + Schedule: in.Schedule, + Configuration: in.Configuration, + TablePrefix: in.TablePrefix, + Classifiers: in.Classifiers, + }) + if err != nil { return nil, err } @@ -997,14 +1009,26 @@ func (h *Handler) handleGetCrawlers(_ context.Context, _ *getCrawlersInput) (*ge } type updateCrawlerInput struct { - Name string `json:"Name"` - Role string `json:"Role"` - DatabaseName string `json:"DatabaseName"` - Targets CrawlerTarget `json:"Targets,omitzero"` + Name string `json:"Name"` + Role string `json:"Role"` + DatabaseName string `json:"DatabaseName"` + Description string `json:"Description,omitempty"` + Schedule string `json:"Schedule,omitempty"` + Configuration string `json:"Configuration,omitempty"` + TablePrefix string `json:"TablePrefix,omitempty"` + Classifiers []string `json:"Classifiers,omitempty"` + Targets CrawlerTarget `json:"Targets,omitzero"` } func (h *Handler) handleUpdateCrawler(_ context.Context, in *updateCrawlerInput) (*emptyOutput, error) { - if err := h.Backend.UpdateCrawler(in.Name, in.Role, in.DatabaseName, in.Targets); err != nil { + err := h.Backend.UpdateCrawlerWithOptions(in.Name, in.Role, in.DatabaseName, in.Targets, CrawlerOptions{ + Description: in.Description, + Schedule: in.Schedule, + Configuration: in.Configuration, + TablePrefix: in.TablePrefix, + Classifiers: in.Classifiers, + }) + if err != nil { return nil, err } @@ -1026,19 +1050,21 @@ func (h *Handler) handleDeleteCrawler(_ context.Context, in *deleteCrawlerInput) // --- Job handlers --- type createJobInput struct { - Tags map[string]string `json:"Tags,omitempty"` - DefaultArguments map[string]string `json:"DefaultArguments,omitempty"` - Command JobCommand `json:"Command,omitzero"` - WorkerType string `json:"WorkerType,omitempty"` - Role string `json:"Role,omitempty"` - GlueVersion string `json:"GlueVersion,omitempty"` - Name string `json:"Name"` - Description string `json:"Description,omitempty"` - Connections ConnectionsList `json:"Connections,omitzero"` - NumberOfWorkers int `json:"NumberOfWorkers,omitempty"` - MaxRetries int `json:"MaxRetries,omitempty"` - Timeout int `json:"Timeout,omitempty"` - ExecutionProperty ExecutionProperty `json:"ExecutionProperty,omitzero"` + Tags map[string]string `json:"Tags,omitempty"` + DefaultArguments map[string]string `json:"DefaultArguments,omitempty"` + Command JobCommand `json:"Command,omitzero"` + WorkerType string `json:"WorkerType,omitempty"` + Role string `json:"Role,omitempty"` + GlueVersion string `json:"GlueVersion,omitempty"` + Name string `json:"Name"` + Description string `json:"Description,omitempty"` + Connections ConnectionsList `json:"Connections,omitzero"` + NotificationProperty NotificationProperty `json:"NotificationProperty,omitzero"` + NumberOfWorkers int `json:"NumberOfWorkers,omitempty"` + MaxRetries int `json:"MaxRetries,omitempty"` + Timeout int `json:"Timeout,omitempty"` + MaxCapacity float64 `json:"MaxCapacity,omitempty"` + ExecutionProperty ExecutionProperty `json:"ExecutionProperty,omitzero"` } type createJobOutput struct { @@ -1047,19 +1073,21 @@ type createJobOutput struct { func (h *Handler) handleCreateJob(_ context.Context, in *createJobInput) (*createJobOutput, error) { j, err := h.Backend.CreateJob(Job{ - Name: in.Name, - Description: in.Description, - Role: in.Role, - Command: in.Command, - DefaultArguments: in.DefaultArguments, - GlueVersion: in.GlueVersion, - WorkerType: in.WorkerType, - NumberOfWorkers: in.NumberOfWorkers, - MaxRetries: in.MaxRetries, - Timeout: in.Timeout, - Tags: in.Tags, - ExecutionProperty: in.ExecutionProperty, - Connections: in.Connections, + Name: in.Name, + Description: in.Description, + Role: in.Role, + Command: in.Command, + DefaultArguments: in.DefaultArguments, + GlueVersion: in.GlueVersion, + WorkerType: in.WorkerType, + NumberOfWorkers: in.NumberOfWorkers, + MaxCapacity: in.MaxCapacity, + MaxRetries: in.MaxRetries, + Timeout: in.Timeout, + Tags: in.Tags, + ExecutionProperty: in.ExecutionProperty, + Connections: in.Connections, + NotificationProperty: in.NotificationProperty, }) if err != nil { return nil, err @@ -1100,17 +1128,19 @@ func (h *Handler) handleGetJobs(_ context.Context, _ *getJobsInput) (*getJobsOut // jobUpdatePayload models the allowed fields for Glue's JobUpdate shape. // It intentionally omits create-only fields such as Name and Tags. type jobUpdatePayload struct { - DefaultArguments map[string]string `json:"DefaultArguments,omitempty"` - Command JobCommand `json:"Command,omitzero"` - WorkerType string `json:"WorkerType,omitempty"` - Role string `json:"Role,omitempty"` - GlueVersion string `json:"GlueVersion,omitempty"` - Description string `json:"Description,omitempty"` - Connections ConnectionsList `json:"Connections,omitzero"` - NumberOfWorkers int `json:"NumberOfWorkers,omitempty"` - MaxRetries int `json:"MaxRetries,omitempty"` - Timeout int `json:"Timeout,omitempty"` - ExecutionProperty ExecutionProperty `json:"ExecutionProperty,omitzero"` + DefaultArguments map[string]string `json:"DefaultArguments,omitempty"` + Command JobCommand `json:"Command,omitzero"` + WorkerType string `json:"WorkerType,omitempty"` + Role string `json:"Role,omitempty"` + GlueVersion string `json:"GlueVersion,omitempty"` + Description string `json:"Description,omitempty"` + Connections ConnectionsList `json:"Connections,omitzero"` + NotificationProperty NotificationProperty `json:"NotificationProperty,omitzero"` + NumberOfWorkers int `json:"NumberOfWorkers,omitempty"` + MaxRetries int `json:"MaxRetries,omitempty"` + Timeout int `json:"Timeout,omitempty"` + MaxCapacity float64 `json:"MaxCapacity,omitempty"` + ExecutionProperty ExecutionProperty `json:"ExecutionProperty,omitzero"` } type updateJobInput struct { @@ -1124,17 +1154,19 @@ type updateJobOutput struct { func (h *Handler) handleUpdateJob(_ context.Context, in *updateJobInput) (*updateJobOutput, error) { if err := h.Backend.UpdateJob(in.JobName, Job{ - Description: in.JobUpdate.Description, - Role: in.JobUpdate.Role, - Command: in.JobUpdate.Command, - DefaultArguments: in.JobUpdate.DefaultArguments, - GlueVersion: in.JobUpdate.GlueVersion, - WorkerType: in.JobUpdate.WorkerType, - NumberOfWorkers: in.JobUpdate.NumberOfWorkers, - MaxRetries: in.JobUpdate.MaxRetries, - Timeout: in.JobUpdate.Timeout, - ExecutionProperty: in.JobUpdate.ExecutionProperty, - Connections: in.JobUpdate.Connections, + Description: in.JobUpdate.Description, + Role: in.JobUpdate.Role, + Command: in.JobUpdate.Command, + DefaultArguments: in.JobUpdate.DefaultArguments, + GlueVersion: in.JobUpdate.GlueVersion, + WorkerType: in.JobUpdate.WorkerType, + NumberOfWorkers: in.JobUpdate.NumberOfWorkers, + MaxCapacity: in.JobUpdate.MaxCapacity, + MaxRetries: in.JobUpdate.MaxRetries, + Timeout: in.JobUpdate.Timeout, + ExecutionProperty: in.JobUpdate.ExecutionProperty, + Connections: in.JobUpdate.Connections, + NotificationProperty: in.JobUpdate.NotificationProperty, }); err != nil { return nil, err } diff --git a/services/glue/handler_batch2_accuracy_test.go b/services/glue/handler_batch2_accuracy_test.go index 8176083f2..058cef250 100644 --- a/services/glue/handler_batch2_accuracy_test.go +++ b/services/glue/handler_batch2_accuracy_test.go @@ -184,7 +184,7 @@ func TestBatch2Accuracy_StartBlueprintRun_NotFound(t *testing.T) { } // TestBatch2Accuracy_CreateBlueprint_NameRequired verifies CreateBlueprint -// rejects an empty Name with ValidationException. +// rejects an empty Name with InvalidInputException. func TestBatch2Accuracy_CreateBlueprint_NameRequired(t *testing.T) { t.Parallel() @@ -198,7 +198,7 @@ func TestBatch2Accuracy_CreateBlueprint_NameRequired(t *testing.T) { name: "empty_name_rejected", bpName: "", wantCode: http.StatusBadRequest, - wantError: "ValidationException", + wantError: "InvalidInputException", }, { name: "valid_name_accepted", @@ -236,19 +236,19 @@ func TestBatch2Accuracy_CreateCustomEntityType_Validation(t *testing.T) { name: "empty_name_rejected", body: map[string]any{"Name": "", "RegexString": "\\d+"}, wantCode: http.StatusBadRequest, - wantError: "ValidationException", + wantError: "InvalidInputException", }, { name: "empty_regex_rejected", body: map[string]any{"Name": "my-cet", "RegexString": ""}, wantCode: http.StatusBadRequest, - wantError: "ValidationException", + wantError: "InvalidInputException", }, { name: "missing_regex_rejected", body: map[string]any{"Name": "my-cet2"}, wantCode: http.StatusBadRequest, - wantError: "ValidationException", + wantError: "InvalidInputException", }, { name: "valid_name_and_regex_accepted", @@ -272,7 +272,7 @@ func TestBatch2Accuracy_CreateCustomEntityType_Validation(t *testing.T) { } // TestBatch2Accuracy_CreateUsageProfile_NameRequired verifies CreateUsageProfile -// rejects an empty Name with ValidationException. +// rejects an empty Name with InvalidInputException. func TestBatch2Accuracy_CreateUsageProfile_NameRequired(t *testing.T) { t.Parallel() @@ -286,7 +286,7 @@ func TestBatch2Accuracy_CreateUsageProfile_NameRequired(t *testing.T) { name: "empty_name_rejected", profName: "", wantCode: http.StatusBadRequest, - wantError: "ValidationException", + wantError: "InvalidInputException", }, { name: "valid_name_accepted", diff --git a/services/glue/handler_stubs.go b/services/glue/handler_stubs.go index 92a463799..f39496cf1 100644 --- a/services/glue/handler_stubs.go +++ b/services/glue/handler_stubs.go @@ -11,9 +11,28 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/awserr" ) -// awserrFromDetail converts an ErrorDetail to an error wrapping ErrNotFound. +// awserrFromDetail converts a batch-operation ErrorDetail into the matching +// sentinel error so handleError reports the correct wire exception type. +// +// Batch partition/table ops (BatchCreatePartition, BatchDeletePartition, ...) +// return per-item ErrorDetail values with an AWS exception-name string in +// ErrorCode, but no Go error. The single-item AWS ops (CreatePartition, +// DeletePartition, ...) are implemented by calling the batch backend method +// with a one-element slice and surfacing errs[0] as a real error. Previously +// this always wrapped awserr.ErrNotFound regardless of ErrorCode, so an +// AlreadyExistsException detail (e.g. CreatePartition on a duplicate) was +// reported to the client as EntityNotFoundException instead. func awserrFromDetail(d ErrorDetail) error { - return awserr.New(d.ErrorCode+": "+d.ErrorMessage, awserr.ErrNotFound) + msg := d.ErrorCode + ": " + d.ErrorMessage + + switch d.ErrorCode { + case "AlreadyExistsException": + return awserr.New(msg, awserr.ErrAlreadyExists) + case errEntityNotFoundCode: + return awserr.New(msg, awserr.ErrNotFound) + default: + return awserr.New(msg, awserr.ErrInvalidParameter) + } } // validateSchemaDefinition checks a schema definition string against its DataFormat. @@ -469,14 +488,21 @@ type batchGetPartitionInput struct { // batchGetPartitionOutput holds the result for BatchGetPartition. type batchGetPartitionOutput struct { - Partitions []*Partition `json:"Partitions"` - UnprocessedKeys []any `json:"UnprocessedKeys"` + Partitions []*Partition `json:"Partitions"` + UnprocessedKeys []PartitionValueList `json:"UnprocessedKeys"` } // handleBatchGetPartition validates that the referenced database/table exist -// before returning. AWS Glue returns an EntityNotFoundException when the -// table is missing rather than silently returning an empty list. The mock -// backend has no partition storage, so on success the response remains empty. +// before returning, then looks up each requested partition value list against +// real backend state. AWS Glue returns an EntityNotFoundException when the +// table is missing rather than silently returning an empty list; partitions +// that don't exist are reported back in UnprocessedKeys (per the +// BatchGetPartitionResponse shape) rather than causing the whole call to fail. +// +// Previously this always returned an empty Partitions list regardless of what +// partitions actually existed in the backend — a disguised stub masked by a +// stale comment claiming "the mock backend has no partition storage" (it does; +// see InMemoryBackend.GetPartition/GetPartitions in backend_accuracy.go). func (h *Handler) handleBatchGetPartition( _ context.Context, in *batchGetPartitionInput, @@ -489,7 +515,21 @@ func (h *Handler) handleBatchGetPartition( return nil, err } - return &batchGetPartitionOutput{Partitions: []*Partition{}, UnprocessedKeys: []any{}}, nil + found := make([]*Partition, 0, len(in.PartitionsToGet)) + unprocessed := make([]PartitionValueList, 0) + + for _, pvl := range in.PartitionsToGet { + p, err := h.Backend.GetPartition(in.DatabaseName, in.TableName, pvl.Values) + if err != nil { + unprocessed = append(unprocessed, pvl) + + continue + } + + found = append(found, p) + } + + return &batchGetPartitionOutput{Partitions: found, UnprocessedKeys: unprocessed}, nil } // batchGetTableOptimizerInput holds input for BatchGetTableOptimizer. diff --git a/services/glue/interfaces.go b/services/glue/interfaces.go index 87bdb2403..6321aba41 100644 --- a/services/glue/interfaces.go +++ b/services/glue/interfaces.go @@ -49,10 +49,17 @@ type StorageBackend interface { targets CrawlerTarget, tags map[string]string, ) (*Crawler, error) + CreateCrawlerWithOptions( + name, role, dbName string, + targets CrawlerTarget, + tags map[string]string, + opts CrawlerOptions, + ) (*Crawler, error) GetCrawler(name string) (*Crawler, error) GetCrawlers() []*Crawler ListCrawlers() []string UpdateCrawler(name, role, dbName string, targets CrawlerTarget) error + UpdateCrawlerWithOptions(name, role, dbName string, targets CrawlerTarget, opts CrawlerOptions) error DeleteCrawler(name string) error // Job operations. diff --git a/services/glue/parity_glue_test.go b/services/glue/parity_glue_test.go index aadb1ab51..dc573cb83 100644 --- a/services/glue/parity_glue_test.go +++ b/services/glue/parity_glue_test.go @@ -473,7 +473,7 @@ func TestDeleteConnectionType(t *testing.T) { name: "empty body missing ConnectionType returns 400", input: map[string]any{}, wantCode: http.StatusBadRequest, - wantType: "ValidationException", + wantType: "InvalidInputException", }, { name: "built-in Salesforce is undeletable", diff --git a/services/glue/parity_sweep3_test.go b/services/glue/parity_sweep3_test.go new file mode 100644 index 000000000..4c2eaf5f3 --- /dev/null +++ b/services/glue/parity_sweep3_test.go @@ -0,0 +1,413 @@ +package glue_test + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Test_ErrValidation_WireCodeIsInvalidInputException verifies that hand-rolled +// validation failures surface as InvalidInputException, not ValidationException. +// aws-sdk-go-v2/service/glue's per-operation error deserializers (e.g. +// awsAwsjson11_deserializeOpErrorCreateDatabase/CreateTrigger/CreateBlueprint) +// list InvalidInputException as the documented hand-validation error; Glue's +// ValidationException type exists but is only declared by a handful of newer +// operations (e.g. DeleteConnectionType), not the general Create* family. +func Test_ErrValidation_WireCodeIsInvalidInputException(t *testing.T) { + t.Parallel() + + cases := []struct { + body map[string]any + name string + action string + }{ + { + name: "CreateDatabase empty name", + action: "CreateDatabase", + body: map[string]any{"DatabaseInput": map[string]any{"Name": ""}}, + }, + { + name: "CreateJob missing role", + action: "CreateJob", + body: map[string]any{ + "Name": "job1", + "Command": map[string]any{"Name": "glueetl"}, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doGlueRequest(t, h, tc.action, tc.body) + + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "InvalidInputException") + assert.NotContains(t, rec.Body.String(), "\"__type\":\"ValidationException\"") + }) + } +} + +// Test_TableStorageDescriptor_RoundTrip verifies that CreateTable/GetTable +// round-trip the full StorageDescriptor shape (InputFormat, OutputFormat, +// SerdeInfo, Parameters, BucketColumns, SortColumns, Compressed, +// NumberOfBuckets) plus Table-level Parameters/Owner/Retention and +// Column.Parameters — fields that aws-sdk-go-v2/service/glue/types. +// StorageDescriptor/Table declare but this backend previously dropped +// entirely (silently discarded on every CreateTable/UpdateTable call). +func Test_TableStorageDescriptor_RoundTrip(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + rec := doGlueRequest(t, h, "CreateDatabase", map[string]any{ + "DatabaseInput": map[string]any{"Name": "db1"}, + }) + require.Equal(t, http.StatusOK, rec.Code) + + tableInput := map[string]any{ + "Name": "tbl1", + "Owner": "hive", + "Retention": 7, + "Parameters": map[string]any{ + "classification": "parquet", + "EXTERNAL": "TRUE", + }, + "StorageDescriptor": map[string]any{ + "Location": "s3://bucket/tbl1/", + "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", + "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", + "Compressed": true, + "NumberOfBuckets": 4, + "BucketColumns": []string{"id"}, + "SortColumns": []map[string]any{ + {"Column": "id", "SortOrder": 1}, + }, + "Parameters": map[string]any{"serialization.format": "1"}, + "SerdeInfo": map[string]any{ + "Name": "parquet-serde", + "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe", + "Parameters": map[string]any{"foo": "bar"}, + }, + "Columns": []map[string]any{ + {"Name": "id", "Type": "int", "Parameters": map[string]any{"comment": "pk"}}, + }, + }, + } + + rec = doGlueRequest(t, h, "CreateTable", map[string]any{ + "DatabaseName": "db1", + "TableInput": tableInput, + }) + require.Equal(t, http.StatusOK, rec.Code) + + rec = doGlueRequest(t, h, "GetTable", map[string]any{ + "DatabaseName": "db1", + "Name": "tbl1", + }) + require.Equal(t, http.StatusOK, rec.Code) + + var out struct { + Table struct { + Parameters map[string]string `json:"Parameters"` + Owner string `json:"Owner"` + StorageDescriptor struct { + SerdeInfo struct { + Parameters map[string]string `json:"Parameters"` + Name string `json:"Name"` + SerializationLibrary string `json:"SerializationLibrary"` + } `json:"SerdeInfo"` + Parameters map[string]string `json:"Parameters"` + Location string `json:"Location"` + InputFormat string `json:"InputFormat"` + OutputFormat string `json:"OutputFormat"` + BucketColumns []string `json:"BucketColumns"` + SortColumns []struct { + Column string `json:"Column"` + SortOrder int `json:"SortOrder"` + } `json:"SortColumns"` + Columns []struct { + Parameters map[string]string `json:"Parameters"` + Name string `json:"Name"` + Type string `json:"Type"` + } `json:"Columns"` + NumberOfBuckets int `json:"NumberOfBuckets"` + Compressed bool `json:"Compressed"` + } `json:"StorageDescriptor"` + Retention int `json:"Retention"` + } `json:"Table"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) + + assert.Equal(t, "hive", out.Table.Owner) + assert.Equal(t, 7, out.Table.Retention) + assert.Equal(t, "parquet", out.Table.Parameters["classification"]) + sd := out.Table.StorageDescriptor + assert.Equal(t, "s3://bucket/tbl1/", sd.Location) + assert.True(t, sd.Compressed) + assert.Equal(t, 4, sd.NumberOfBuckets) + assert.Equal(t, []string{"id"}, sd.BucketColumns) + assert.Equal(t, "1", sd.Parameters["serialization.format"]) + require.Len(t, sd.SortColumns, 1) + assert.Equal(t, "id", sd.SortColumns[0].Column) + require.NotNil(t, sd.SerdeInfo) + assert.Equal(t, "parquet-serde", sd.SerdeInfo.Name) + assert.Equal(t, "bar", sd.SerdeInfo.Parameters["foo"]) + require.Len(t, sd.Columns, 1) + assert.Equal(t, "pk", sd.Columns[0].Parameters["comment"]) +} + +// Test_CreatePartition_RequiresExistingTable verifies that CreatePartition +// (and BatchCreatePartition) reject a nonexistent database/table with +// EntityNotFoundException instead of silently storing an orphaned partition. +// Previously BatchCreatePartition never checked that the table existed at +// all. +func Test_CreatePartition_RequiresExistingTable(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + rec := doGlueRequest(t, h, "CreatePartition", map[string]any{ + "DatabaseName": "no-such-db", + "TableName": "no-such-table", + "PartitionInput": map[string]any{"Values": []string{"2024-01-01"}}, + }) + + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "EntityNotFoundException") +} + +// Test_CreatePartition_DuplicateReportsAlreadyExists verifies that creating a +// duplicate partition surfaces AlreadyExistsException, not +// EntityNotFoundException. awserrFromDetail previously wrapped every +// batch-partition ErrorDetail as ErrNotFound regardless of its ErrorCode, so +// this case was misreported. +func Test_CreatePartition_DuplicateReportsAlreadyExists(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + require.Equal(t, http.StatusOK, doGlueRequest(t, h, "CreateDatabase", map[string]any{ + "DatabaseInput": map[string]any{"Name": "db1"}, + }).Code) + require.Equal(t, http.StatusOK, doGlueRequest(t, h, "CreateTable", map[string]any{ + "DatabaseName": "db1", + "TableInput": map[string]any{"Name": "tbl1"}, + }).Code) + + partitionInput := map[string]any{ + "DatabaseName": "db1", + "TableName": "tbl1", + "PartitionInput": map[string]any{"Values": []string{"2024-01-01"}}, + } + require.Equal(t, http.StatusOK, doGlueRequest(t, h, "CreatePartition", partitionInput).Code) + + rec := doGlueRequest(t, h, "CreatePartition", partitionInput) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "AlreadyExistsException") + assert.NotContains(t, rec.Body.String(), "EntityNotFoundException") +} + +// Test_BatchGetPartition_ReturnsRealPartitions verifies BatchGetPartition +// actually looks up requested partitions against backend state. It +// previously always returned an empty Partitions list regardless of what +// existed — a disguised stub masked by a stale comment claiming the backend +// "has no partition storage". +func Test_BatchGetPartition_ReturnsRealPartitions(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + require.Equal(t, http.StatusOK, doGlueRequest(t, h, "CreateDatabase", map[string]any{ + "DatabaseInput": map[string]any{"Name": "db1"}, + }).Code) + require.Equal(t, http.StatusOK, doGlueRequest(t, h, "CreateTable", map[string]any{ + "DatabaseName": "db1", + "TableInput": map[string]any{"Name": "tbl1"}, + }).Code) + require.Equal(t, http.StatusOK, doGlueRequest(t, h, "CreatePartition", map[string]any{ + "DatabaseName": "db1", + "TableName": "tbl1", + "PartitionInput": map[string]any{ + "Values": []string{"2024-01-01"}, + "StorageDescriptor": map[string]any{"Location": "s3://bucket/2024-01-01/"}, + }, + }).Code) + + rec := doGlueRequest(t, h, "BatchGetPartition", map[string]any{ + "DatabaseName": "db1", + "TableName": "tbl1", + "PartitionsToGet": []map[string]any{ + {"Values": []string{"2024-01-01"}}, + {"Values": []string{"2024-01-02"}}, // does not exist + }, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var out struct { + Partitions []struct { + StorageDescriptor struct { + Location string `json:"Location"` + } `json:"StorageDescriptor"` + Values []string `json:"Values"` + } `json:"Partitions"` + UnprocessedKeys []struct { + Values []string `json:"Values"` + } `json:"UnprocessedKeys"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) + + require.Len(t, out.Partitions, 1) + assert.Equal(t, []string{"2024-01-01"}, out.Partitions[0].Values) + assert.Equal(t, "s3://bucket/2024-01-01/", out.Partitions[0].StorageDescriptor.Location) + require.Len(t, out.UnprocessedKeys, 1) + assert.Equal(t, []string{"2024-01-02"}, out.UnprocessedKeys[0].Values) +} + +// Test_Job_MaxCapacity verifies that CreateJob/UpdateJob accept and persist +// MaxCapacity (the DPU-based capacity knob used by e.g. Python shell jobs), +// and reject a request that mixes MaxCapacity with WorkerType/NumberOfWorkers +// — AWS's documented mutual-exclusion rule for the two capacity models. +func Test_Job_MaxCapacity(t *testing.T) { + t.Parallel() + + cases := []struct { + body map[string]any + name string + wantOK bool + }{ + { + name: "max_capacity_alone_accepted", + body: map[string]any{ + "Name": "job-mc", + "Role": "role1", + "Command": map[string]any{"Name": "pythonshell"}, + "MaxCapacity": 0.0625, + }, + wantOK: true, + }, + { + name: "max_capacity_with_worker_type_rejected", + body: map[string]any{ + "Name": "job-mc-conflict", + "Role": "role1", + "Command": map[string]any{"Name": "glueetl"}, + "MaxCapacity": 2.0, + "WorkerType": "G.1X", + "NumberOfWorkers": 2, + }, + wantOK: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doGlueRequest(t, h, "CreateJob", tc.body) + + if !tc.wantOK { + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "InvalidInputException") + + return + } + + require.Equal(t, http.StatusOK, rec.Code) + + rec = doGlueRequest(t, h, "GetJob", map[string]any{"JobName": tc.body["Name"]}) + require.Equal(t, http.StatusOK, rec.Code) + + var out struct { + Job struct { + MaxCapacity float64 `json:"MaxCapacity"` + } `json:"Job"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) + assert.InDelta(t, tc.body["MaxCapacity"], out.Job.MaxCapacity, 0.0001) + }) + } +} + +// Test_Crawler_CreateOptions_RoundTrip verifies CreateCrawler/UpdateCrawler +// accept and persist Schedule, Classifiers, Configuration and TablePrefix — +// fields AWS's CreateCrawlerRequest/UpdateCrawlerRequest both support that +// were previously silently discarded (CreateCrawler only accepted +// name/role/database/targets/tags). Also verifies JDBC and catalog crawler +// targets round-trip alongside S3 targets. +func Test_Crawler_CreateOptions_RoundTrip(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + + rec := doGlueRequest(t, h, "CreateCrawler", map[string]any{ + "Name": "crawler1", + "Role": "role1", + "Schedule": "cron(0 12 * * ? *)", + "Classifiers": []string{"my-classifier"}, + "Configuration": `{"Version":1.0}`, + "TablePrefix": "raw_", + "Targets": map[string]any{ + "S3Targets": []map[string]any{{"Path": "s3://bucket/data/"}}, + "JdbcTargets": []map[string]any{{"ConnectionName": "conn1", "Path": "db/%"}}, + "CatalogTargets": []map[string]any{{"DatabaseName": "db1", "Tables": []string{"t1"}}}, + }, + }) + require.Equal(t, http.StatusOK, rec.Code) + + rec = doGlueRequest(t, h, "GetCrawler", map[string]any{"Name": "crawler1"}) + require.Equal(t, http.StatusOK, rec.Code) + + var out struct { + Crawler struct { + Schedule struct { + ScheduleExpression string `json:"ScheduleExpression"` + } `json:"Schedule"` + Configuration string `json:"Configuration"` + TablePrefix string `json:"TablePrefix"` + Classifiers []string `json:"Classifiers"` + Targets struct { + S3Targets []struct { + Path string `json:"Path"` + } `json:"S3Targets"` + JdbcTargets []struct { + ConnectionName string `json:"ConnectionName"` + Path string `json:"Path"` + } `json:"JdbcTargets"` + CatalogTargets []struct { + DatabaseName string `json:"DatabaseName"` + Tables []string `json:"Tables"` + } `json:"CatalogTargets"` + } `json:"Targets"` + } `json:"Crawler"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) + + assert.Equal(t, "cron(0 12 * * ? *)", out.Crawler.Schedule.ScheduleExpression) + assert.Equal(t, `{"Version":1.0}`, out.Crawler.Configuration) + assert.Equal(t, "raw_", out.Crawler.TablePrefix) + assert.Equal(t, []string{"my-classifier"}, out.Crawler.Classifiers) + require.Len(t, out.Crawler.Targets.JdbcTargets, 1) + assert.Equal(t, "conn1", out.Crawler.Targets.JdbcTargets[0].ConnectionName) + require.Len(t, out.Crawler.Targets.CatalogTargets, 1) + assert.Equal(t, "db1", out.Crawler.Targets.CatalogTargets[0].DatabaseName) + + // UpdateCrawler should also accept and persist the same options. + rec = doGlueRequest(t, h, "UpdateCrawler", map[string]any{ + "Name": "crawler1", + "Role": "role1", + "TablePrefix": "curated_", + }) + require.Equal(t, http.StatusOK, rec.Code) + + rec = doGlueRequest(t, h, "GetCrawler", map[string]any{"Name": "crawler1"}) + require.Equal(t, http.StatusOK, rec.Code) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) + assert.Equal(t, "curated_", out.Crawler.TablePrefix) +} diff --git a/services/glue/persistence.go b/services/glue/persistence.go index 313045707..440b8c62f 100644 --- a/services/glue/persistence.go +++ b/services/glue/persistence.go @@ -2,56 +2,48 @@ package glue import ( "context" - "maps" + "encoding/json" + "fmt" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" ) +// glueSnapshotVersion identifies the shape of backendSnapshot's Tables blob +// (i.e. the set/shape of resources registered on b.registry -- see +// registerAllTables in store_setup.go). It must be bumped whenever a change +// there would make an older snapshot unsafe to decode as the current shape. +// Restore compares this against the persisted value and discards (rather than +// attempts to partially decode) any mismatch -- see Restore below. This +// mirrors the services/ec2 (commit 12e611a4) and services/sqs (commit +// 0f09d77c) conversions. +const glueSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the Glue backend. +// +// Tables holds one JSON-encoded array per registered store.Table, produced by +// store.Registry.SnapshotAll (see registerAllTables in store_setup.go). Every +// remaining field is a resource collection that was NOT converted to +// store.Table -- see the comment above registerAllTables for why each one's +// key is not a pure function of its value (or, for the *AtRest/*Config +// singletons and JobRun*At timer maps, is not a map[string]*T at all) -- and +// so is still persisted directly, exactly as before this refactor. type backendSnapshot struct { - Databases map[string]*Database `json:"databases"` - Tables map[string]*Table `json:"tables"` - Crawlers map[string]*Crawler `json:"crawlers"` - Jobs map[string]*Job `json:"jobs"` - Partitions map[string]*Partition `json:"partitions"` + Tables map[string]json.RawMessage `json:"tables"` PartitionIndexes map[string]*PartitionIndex `json:"partitionIndexes"` - TableVersions map[string]*TableVersion `json:"tableVersions"` - Connections map[string]*Connection `json:"connections"` - Blueprints map[string]*Blueprint `json:"blueprints"` - CustomEntityTypes map[string]*CustomEntityType `json:"customEntityTypes"` - DataQualityResult map[string]*DataQualityResult `json:"dataQualityResult"` - DevEndpoints map[string]*DevEndpoint `json:"devEndpoints"` - JobRuns map[string][]*JobRun `json:"jobRuns"` - JobBookmarks map[string]*JobBookmark `json:"jobBookmarks"` - DataQualityRulesets map[string]*DataQualityRuleset `json:"dataQualityRulesets"` - DataQualityEvalRuns map[string]*DataQualityEvaluationRun `json:"dataQualityEvalRuns"` - Triggers map[string]*Trigger `json:"triggers"` - Workflows map[string]*Workflow `json:"workflows"` - WorkflowRuns map[string][]*WorkflowRun `json:"workflowRuns"` - Classifiers map[string]*Classifier `json:"classifiers"` - Registries map[string]*Registry `json:"registries"` - Schemas map[string]*Schema `json:"schemas"` - SchemaVersions map[string][]*SchemaVersion `json:"schemaVersions"` - UDFs map[string]*UserDefinedFunction `json:"udfs"` - SecurityConfigs map[string]*SecurityConfiguration `json:"securityConfigs"` - Sessions map[string]*Session `json:"sessions"` - SessionStatements map[string][]*Statement `json:"sessionStatements"` - TableOptimizers map[string]*TableOptimizer `json:"tableOptimizers"` TableColumnStats map[string]*ColumnStatistics `json:"tableColumnStats"` PartitionColumnStats map[string]*ColumnStatistics `json:"partitionColumnStats"` ResourcePolicies map[string]*resourcePolicyEntry `json:"resourcePolicies"` - MLTransforms map[string]*MLTransform `json:"mlTransforms"` - Catalogs map[string]*CatalogEntry `json:"catalogs"` CatalogEncryptionSettings map[string]*DataCatalogEncryptionSettings `json:"catalogEncryptionSettings"` - UsageProfiles map[string]*UsageProfile `json:"usageProfiles"` - BlueprintRuns map[string]*BlueprintRun `json:"blueprintRuns"` - DQRecommendationRuns map[string]*DQRuleRecommendationRun `json:"dqRecommendationRuns"` - ColumnStatTaskSettings map[string]*ColumnStatisticsTaskSettings `json:"columnStatTaskSettings"` - ColumnStatTaskRuns map[string]*ColumnStatisticsTaskRun `json:"columnStatTaskRuns"` - MaterializedViewRuns map[string]*MaterializedViewRefreshRun `json:"materializedViewRuns"` - Integrations map[string]*Integration `json:"integrations"` + CatalogImports map[string]*CatalogImportStatus `json:"catalogImports"` + JobRuns map[string][]*JobRun `json:"jobRuns"` + WorkflowRuns map[string][]*WorkflowRun `json:"workflowRuns"` + SchemaVersions map[string][]*SchemaVersion `json:"schemaVersions"` + SessionStatements map[string][]*Statement `json:"sessionStatements"` CrawlHistory map[string][]*CrawlHistoryEntry `json:"crawlHistory"` - DQStatisticAnnotations map[string]*StatisticAnnotation `json:"dqStatisticAnnotations"` + SchemaVersionMetadata map[string]map[string]string `json:"schemaVersionMetadata"` GlueIdentityCenterConfig *IdentityCenterConfig `json:"glueIdentityCenterConfig,omitempty"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. @@ -59,91 +51,38 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() - snap := backendSnapshot{ - Databases: copyMap(b.databases, cloneDatabase), - Tables: copyMap(b.tables, func(t *Table) *Table { - cp := *t - - return &cp - }), - Crawlers: copyMap(b.crawlers, cloneCrawler), - Jobs: copyMap(b.jobs, cloneJob), - Partitions: copyMap(b.partitions, func(p *Partition) *Partition { - cp := *p - cp.Values = append([]string(nil), p.Values...) - - return &cp - }), - TableVersions: copyMap(b.tableVersions, func(tv *TableVersion) *TableVersion { - cp := *tv - - return &cp - }), - Connections: copyMap(b.connections, cloneConnection), - Blueprints: maps.Clone(b.blueprints), - CustomEntityTypes: copyMap(b.customEntityTypes, func(c *CustomEntityType) *CustomEntityType { - cp := *c - cp.ContextWords = append([]string(nil), c.ContextWords...) - - return &cp - }), - DataQualityResult: maps.Clone(b.dataQualityResult), - DevEndpoints: maps.Clone(b.devEndpoints), - JobRuns: copyJobRunsMap(b.jobRuns), - JobBookmarks: maps.Clone(b.jobBookmarks), - DataQualityRulesets: copyMap(b.dataQualityRulesets, func(r *DataQualityRuleset) *DataQualityRuleset { - cp := *r - cp.Tags = maps.Clone(r.Tags) + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "glue: snapshot table marshal failed", "error", err) - return &cp - }), - DataQualityEvalRuns: copyMap( - b.dataQualityEvalRuns, - func(run *DataQualityEvaluationRun) *DataQualityEvaluationRun { - cp := *run - cp.RulesetNames = append([]string(nil), run.RulesetNames...) + return nil + } - return &cp - }, - ), + snap := backendSnapshot{ + Version: glueSnapshotVersion, + Tables: tables, + PartitionIndexes: b.partitionIndexes, + TableColumnStats: b.tableColumnStats, + PartitionColumnStats: b.partitionColumnStats, + ResourcePolicies: b.resourcePolicies, + CatalogEncryptionSettings: b.catalogEncryptionSettings, + CatalogImports: b.catalogImports, + JobRuns: b.jobRuns, + WorkflowRuns: b.workflowRuns, + SchemaVersions: b.schemaVersions, + SessionStatements: b.sessionStatements, + CrawlHistory: b.crawlHistory, + SchemaVersionMetadata: b.schemaVersionMetadata, + GlueIdentityCenterConfig: b.glueIdentityCenterConfig, } - addExtendedSnapshotState(&snap, b) return persistence.MarshalSnapshot(ctx, "glue", snap) } -func addExtendedSnapshotState(snap *backendSnapshot, b *InMemoryBackend) { - snap.PartitionIndexes = b.partitionIndexes - snap.Triggers = b.triggers - snap.Workflows = b.workflows - snap.WorkflowRuns = b.workflowRuns - snap.Classifiers = b.classifiers - snap.Registries = b.registries - snap.Schemas = b.schemas - snap.SchemaVersions = b.schemaVersions - snap.UDFs = b.udfs - snap.SecurityConfigs = b.securityConfigs - snap.Sessions = b.sessions - snap.SessionStatements = b.sessionStatements - snap.TableOptimizers = b.tableOptimizers - snap.TableColumnStats = b.tableColumnStats - snap.PartitionColumnStats = b.partitionColumnStats - snap.ResourcePolicies = b.resourcePolicies - snap.MLTransforms = b.mlTransforms - snap.Catalogs = b.catalogs - snap.CatalogEncryptionSettings = b.catalogEncryptionSettings - snap.UsageProfiles = b.usageProfiles - snap.BlueprintRuns = b.blueprintRuns - snap.DQRecommendationRuns = b.dqRecommendationRuns - snap.ColumnStatTaskSettings = b.columnStatTaskSettings - snap.ColumnStatTaskRuns = b.columnStatTaskRuns - snap.MaterializedViewRuns = b.materializedViewRuns - snap.Integrations = b.integrations - snap.CrawlHistory = b.crawlHistory - snap.DQStatisticAnnotations = b.dqStatisticAnnotations - snap.GlueIdentityCenterConfig = b.glueIdentityCenterConfig -} - // Restore loads backend state from a JSON snapshot. func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { var snap backendSnapshot @@ -154,260 +93,102 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() + if snap.Version != glueSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/ec2 and services/sqs conversions. + logger.Load(ctx).WarnContext(ctx, + "glue: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", glueSnapshotVersion) + + b.registry.ResetAll() + + return nil + } + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("glue: restore snapshot tables: %w", err) + } + initSnapshotDefaults(&snap) b.restoreFromSnapshot(snap) return nil } -// initSnapshotDefaults ensures all snapshot maps are non-nil. +// initSnapshotDefaults ensures every raw (non-store.Table) snapshot map is +// non-nil. The registered store.Table fields need no such treatment -- +// store.Table.Restore already leaves an empty (never nil) table when its +// input is nil or absent. func initSnapshotDefaults(snap *backendSnapshot) { - initSnapshotCoreDefaults(snap) - initSnapshotExtDefaults(snap) - initSnapshotOperationalDefaults(snap) - initSnapshotCatalogDefaults(snap) - initSnapshotTaskDefaults(snap) -} - -// initSnapshotCoreDefaults initializes core maps to non-nil. -func initSnapshotCoreDefaults(snap *backendSnapshot) { - if snap.Databases == nil { - snap.Databases = make(map[string]*Database) - } - if snap.Tables == nil { - snap.Tables = make(map[string]*Table) - } - if snap.Crawlers == nil { - snap.Crawlers = make(map[string]*Crawler) - } - if snap.Jobs == nil { - snap.Jobs = make(map[string]*Job) - } - if snap.Partitions == nil { - snap.Partitions = make(map[string]*Partition) - } if snap.PartitionIndexes == nil { snap.PartitionIndexes = make(map[string]*PartitionIndex) } - if snap.TableVersions == nil { - snap.TableVersions = make(map[string]*TableVersion) - } - if snap.Connections == nil { - snap.Connections = make(map[string]*Connection) - } - if snap.Blueprints == nil { - snap.Blueprints = make(map[string]*Blueprint) - } -} - -// initSnapshotExtDefaults initializes extended maps to non-nil. -func initSnapshotExtDefaults(snap *backendSnapshot) { - if snap.CustomEntityTypes == nil { - snap.CustomEntityTypes = make(map[string]*CustomEntityType) - } - if snap.DataQualityResult == nil { - snap.DataQualityResult = make(map[string]*DataQualityResult) - } - if snap.DevEndpoints == nil { - snap.DevEndpoints = make(map[string]*DevEndpoint) - } - if snap.JobRuns == nil { - snap.JobRuns = make(map[string][]*JobRun) - } - if snap.JobBookmarks == nil { - snap.JobBookmarks = make(map[string]*JobBookmark) - } - if snap.DataQualityRulesets == nil { - snap.DataQualityRulesets = make(map[string]*DataQualityRuleset) - } - if snap.DataQualityEvalRuns == nil { - snap.DataQualityEvalRuns = make(map[string]*DataQualityEvaluationRun) - } -} - -func initSnapshotOperationalDefaults(snap *backendSnapshot) { - if snap.Triggers == nil { - snap.Triggers = make(map[string]*Trigger) - } - if snap.Workflows == nil { - snap.Workflows = make(map[string]*Workflow) - } - if snap.WorkflowRuns == nil { - snap.WorkflowRuns = make(map[string][]*WorkflowRun) - } - if snap.Classifiers == nil { - snap.Classifiers = make(map[string]*Classifier) - } - if snap.UDFs == nil { - snap.UDFs = make(map[string]*UserDefinedFunction) - } - if snap.SecurityConfigs == nil { - snap.SecurityConfigs = make(map[string]*SecurityConfiguration) - } - if snap.Sessions == nil { - snap.Sessions = make(map[string]*Session) - } - if snap.SessionStatements == nil { - snap.SessionStatements = make(map[string][]*Statement) - } - if snap.TableOptimizers == nil { - snap.TableOptimizers = make(map[string]*TableOptimizer) - } if snap.TableColumnStats == nil { snap.TableColumnStats = make(map[string]*ColumnStatistics) } if snap.PartitionColumnStats == nil { snap.PartitionColumnStats = make(map[string]*ColumnStatistics) } -} - -func initSnapshotCatalogDefaults(snap *backendSnapshot) { - if snap.Registries == nil { - snap.Registries = make(map[string]*Registry) - } - if snap.Schemas == nil { - snap.Schemas = make(map[string]*Schema) - } - if snap.SchemaVersions == nil { - snap.SchemaVersions = make(map[string][]*SchemaVersion) - } if snap.ResourcePolicies == nil { snap.ResourcePolicies = make(map[string]*resourcePolicyEntry) } - if snap.MLTransforms == nil { - snap.MLTransforms = make(map[string]*MLTransform) - } - if snap.Catalogs == nil { - snap.Catalogs = make(map[string]*CatalogEntry) - } if snap.CatalogEncryptionSettings == nil { snap.CatalogEncryptionSettings = make(map[string]*DataCatalogEncryptionSettings) } - if snap.UsageProfiles == nil { - snap.UsageProfiles = make(map[string]*UsageProfile) + if snap.CatalogImports == nil { + snap.CatalogImports = make(map[string]*CatalogImportStatus) } + + initSnapshotListDefaults(snap) } -func initSnapshotTaskDefaults(snap *backendSnapshot) { - if snap.BlueprintRuns == nil { - snap.BlueprintRuns = make(map[string]*BlueprintRun) - } - if snap.DQRecommendationRuns == nil { - snap.DQRecommendationRuns = make(map[string]*DQRuleRecommendationRun) - } - if snap.ColumnStatTaskSettings == nil { - snap.ColumnStatTaskSettings = make(map[string]*ColumnStatisticsTaskSettings) +// initSnapshotListDefaults initialises the one-to-many (map[string][]*T) +// snapshot fields to non-nil. Split out from initSnapshotDefaults to stay +// under the funlen limit. +func initSnapshotListDefaults(snap *backendSnapshot) { + if snap.JobRuns == nil { + snap.JobRuns = make(map[string][]*JobRun) } - if snap.ColumnStatTaskRuns == nil { - snap.ColumnStatTaskRuns = make(map[string]*ColumnStatisticsTaskRun) + if snap.WorkflowRuns == nil { + snap.WorkflowRuns = make(map[string][]*WorkflowRun) } - if snap.MaterializedViewRuns == nil { - snap.MaterializedViewRuns = make(map[string]*MaterializedViewRefreshRun) + if snap.SchemaVersions == nil { + snap.SchemaVersions = make(map[string][]*SchemaVersion) } - if snap.Integrations == nil { - snap.Integrations = make(map[string]*Integration) + if snap.SessionStatements == nil { + snap.SessionStatements = make(map[string][]*Statement) } if snap.CrawlHistory == nil { snap.CrawlHistory = make(map[string][]*CrawlHistoryEntry) } - if snap.DQStatisticAnnotations == nil { - snap.DQStatisticAnnotations = make(map[string]*StatisticAnnotation) + if snap.SchemaVersionMetadata == nil { + snap.SchemaVersionMetadata = make(map[string]map[string]string) } } -// restoreFromSnapshot copies snapshot data into the backend (caller holds lock). +// restoreFromSnapshot copies the raw (non-store.Table) snapshot data into the +// backend. Caller holds b.mu; the registered store.Table fields are already +// restored by b.registry.RestoreAll in Restore above. func (b *InMemoryBackend) restoreFromSnapshot(snap backendSnapshot) { - b.databases = copyMap(snap.Databases, cloneDatabase) - b.tables = copyMap(snap.Tables, func(t *Table) *Table { - cp := *t - - return &cp - }) - b.crawlers = copyMap(snap.Crawlers, cloneCrawler) - b.jobs = copyMap(snap.Jobs, cloneJob) - b.partitions = copyMap(snap.Partitions, func(p *Partition) *Partition { - cp := *p - cp.Values = append([]string(nil), p.Values...) - - return &cp - }) b.partitionIndexes = snap.PartitionIndexes - b.tableVersions = copyMap(snap.TableVersions, func(tv *TableVersion) *TableVersion { - cp := *tv - - return &cp - }) - b.connections = copyMap(snap.Connections, cloneConnection) - b.blueprints = maps.Clone(snap.Blueprints) - b.customEntityTypes = copyMap(snap.CustomEntityTypes, func(c *CustomEntityType) *CustomEntityType { - cp := *c - cp.ContextWords = append([]string(nil), c.ContextWords...) - - return &cp - }) - b.dataQualityResult = maps.Clone(snap.DataQualityResult) - b.devEndpoints = maps.Clone(snap.DevEndpoints) - b.jobRuns = copyJobRunsMap(snap.JobRuns) - b.jobBookmarks = maps.Clone(snap.JobBookmarks) - b.dataQualityRulesets = copyMap(snap.DataQualityRulesets, func(r *DataQualityRuleset) *DataQualityRuleset { - cp := *r - cp.Tags = maps.Clone(r.Tags) - - return &cp - }) - b.dataQualityEvalRuns = copyMap( - snap.DataQualityEvalRuns, - func(run *DataQualityEvaluationRun) *DataQualityEvaluationRun { - cp := *run - cp.RulesetNames = append([]string(nil), run.RulesetNames...) - - return &cp - }, - ) - b.restoreExtendedSnapshotState(snap) -} - -func (b *InMemoryBackend) restoreExtendedSnapshotState(snap backendSnapshot) { - b.triggers = snap.Triggers - b.workflows = snap.Workflows - b.workflowRuns = snap.WorkflowRuns - b.classifiers = snap.Classifiers - b.registries = snap.Registries - b.schemas = snap.Schemas - b.schemaVersions = snap.SchemaVersions - b.udfs = snap.UDFs - b.securityConfigs = snap.SecurityConfigs - b.sessions = snap.Sessions - b.sessionStatements = snap.SessionStatements - b.tableOptimizers = snap.TableOptimizers b.tableColumnStats = snap.TableColumnStats b.partitionColumnStats = snap.PartitionColumnStats b.resourcePolicies = snap.ResourcePolicies - b.mlTransforms = snap.MLTransforms - b.catalogs = snap.Catalogs b.catalogEncryptionSettings = snap.CatalogEncryptionSettings - b.usageProfiles = snap.UsageProfiles - b.blueprintRuns = snap.BlueprintRuns - b.dqRecommendationRuns = snap.DQRecommendationRuns - b.columnStatTaskSettings = snap.ColumnStatTaskSettings - b.columnStatTaskRuns = snap.ColumnStatTaskRuns - b.materializedViewRuns = snap.MaterializedViewRuns - b.integrations = snap.Integrations + b.catalogImports = snap.CatalogImports + b.jobRuns = snap.JobRuns + b.workflowRuns = snap.WorkflowRuns + b.schemaVersions = snap.SchemaVersions + b.sessionStatements = snap.SessionStatements b.crawlHistory = snap.CrawlHistory - b.dqStatisticAnnotations = snap.DQStatisticAnnotations + b.schemaVersionMetadata = snap.SchemaVersionMetadata b.glueIdentityCenterConfig = snap.GlueIdentityCenterConfig } -// copyMap creates a deep copy of a map using the provided clone function. -func copyMap[V any](src map[string]V, clone func(V) V) map[string]V { - dst := make(map[string]V, len(src)) - for k, v := range src { - dst[k] = clone(v) - } - - return dst -} - // Snapshot implements Snapshottable by delegating to the backend when it // supports it. func (h *Handler) Snapshot(ctx context.Context) []byte { @@ -427,19 +208,3 @@ func (h *Handler) Restore(ctx context.Context, data []byte) error { return nil } - -// copyJobRunsMap deep-copies the jobRuns map (map[string][]*JobRun). -func copyJobRunsMap(src map[string][]*JobRun) map[string][]*JobRun { - dst := make(map[string][]*JobRun, len(src)) - for k, runs := range src { - cp := make([]*JobRun, len(runs)) - for i, r := range runs { - rc := *r - rc.Arguments = maps.Clone(r.Arguments) - cp[i] = &rc - } - dst[k] = cp - } - - return dst -} diff --git a/services/glue/persistence_test.go b/services/glue/persistence_test.go new file mode 100644 index 000000000..b0bb5d68d --- /dev/null +++ b/services/glue/persistence_test.go @@ -0,0 +1,305 @@ +package glue_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/glue" +) + +// TestInMemoryBackend_SnapshotRestore_FullState seeds one instance of every +// resource collection on InMemoryBackend -- both the store.Table-backed +// collections converted in Phase 3.3 and the collections left as plain maps +// (see the comment above registerAllTables in store_setup.go) -- then +// verifies a Snapshot/Restore round trip into a fresh backend reproduces the +// exact same state. This is the persistence-safety regression test required +// by the conversion: every map that was persisted before the refactor must +// still be persisted identically afterward. +func TestInMemoryBackend_SnapshotRestore_FullState(t *testing.T) { + t.Parallel() + + orig := glue.NewInMemoryBackend("123456789012", "us-east-1") + seedFullState(t, orig) + + snap := orig.Snapshot(t.Context()) + require.NotNil(t, snap) + + restored := glue.NewInMemoryBackend("123456789012", "us-east-1") + require.NoError(t, restored.Restore(t.Context(), snap)) + + verifyFullState(t, restored) +} + +func seedFullState(t *testing.T, b *glue.InMemoryBackend) { + t.Helper() + + _, err := b.CreateDatabase(glue.DatabaseInput{Name: "db1"}, map[string]string{"k": "v"}) + require.NoError(t, err) + _, err = b.CreateTable("db1", glue.TableInput{ + Name: "tbl1", + PartitionKeys: []glue.Column{{Name: "year"}}, + }) + require.NoError(t, err) + _, errs := b.BatchCreatePartition("db1", "tbl1", []glue.PartitionInput{{Values: []string{"2024"}}}) + require.Empty(t, errs) + b.AddTableVersionInternal("db1", "tbl1", &glue.TableVersion{VersionID: "1"}) + _, err = b.CreateCrawler("crawler1", "role1", "db1", glue.CrawlerTarget{}, nil) + require.NoError(t, err) + require.NoError(t, b.StartCrawler("crawler1")) // populates raw crawlHistory + _, err = b.CreateJob(glue.Job{Name: "job1", Role: "role1", Command: glue.JobCommand{Name: "glueetl"}}) + require.NoError(t, err) + _, err = b.StartJobRun("job1", nil) // populates raw jobRuns + jobBookmarks + require.NoError(t, err) + _, err = b.CreateConnection("conn1", "JDBC", nil, nil) + require.NoError(t, err) + require.NoError(t, b.CreateBlueprint("bp1")) + _, err = b.CreateCustomEntityType("cet1", "regex", nil) + require.NoError(t, err) + b.AddDataQualityResultInternal(&glue.DataQualityResult{ResultID: "dqr1"}) + _, err = b.CreateDevEndpoint("dep1") + require.NoError(t, err) + _, err = b.CreateDataQualityRuleset("ruleset1", "rules", nil) + require.NoError(t, err) + _, err = b.StartDataQualityRulesetEvaluationRun([]string{"ruleset1"}) + require.NoError(t, err) + _, err = b.CreateTrigger(glue.Trigger{Name: "trig1"}, nil) + require.NoError(t, err) + _, err = b.CreateWorkflow(glue.Workflow{Name: "wf1"}, nil) + require.NoError(t, err) + _, err = b.StartWorkflowRun("wf1") // populates raw workflowRuns + require.NoError(t, err) + require.NoError(t, b.CreateClassifier(glue.Classifier{GrokClassifier: &glue.GrokClassifier{Name: "clf1"}})) + _, err = b.CreateRegistry("reg1", "desc", nil) + require.NoError(t, err) + _, err = b.CreateSchema("reg1", "schema1", "AVRO", "NONE", "desc", nil) + require.NoError(t, err) + _, err = b.RegisterSchemaVersion("reg1", "schema1", "def-v1") // populates raw schemaVersions + require.NoError(t, err) + _, err = b.CreateUserDefinedFunction("db1", glue.UserDefinedFunction{FunctionName: "udf1"}, nil) + require.NoError(t, err) + _, err = b.CreateSecurityConfiguration("sc1", glue.EncryptionConfiguration{}) + require.NoError(t, err) + _, err = b.CreateSession("sess1", "role1", glue.SessionCommand{Name: "glueetl"}, glue.Session{}) + require.NoError(t, err) + _, err = b.RunStatement("sess1", "print(1)") // populates raw sessionStatements + require.NoError(t, err) + require.NoError(t, b.CreateTableOptimizer("cat1", "db1", "tbl1", "compaction", glue.TableOptimizerConfiguration{})) + require.NoError(t, b.UpdateColumnStatisticsForTable("db1", "tbl1", []*glue.ColumnStatistics{ // raw tableColumnStats + {ColumnName: "col1"}, + })) + require.NoError(t, b.UpdateColumnStatisticsForPartition("db1", "tbl1", []string{"2024"}, // raw partitionColumnStats + []*glue.ColumnStatistics{{ColumnName: "col1"}})) + _, err = b.PutResourcePolicy("policy-doc", "arn:aws:glue:us-east-1:123456789012:catalog") // raw resourcePolicies + require.NoError(t, err) + mlTransform, err := b.CreateMLTransform("mlt1", "desc", "role1", nil, glue.MLTransformParameter{}, nil) + require.NoError(t, err) + _, err = b.StartMLEvaluationTaskRun(mlTransform.TransformID) + require.NoError(t, err) + require.NoError(t, b.CreateCatalog("cat1", "name1", "desc", nil)) + // raw catalogEncryptionSettings + require.NoError(t, b.PutDataCatalogEncryptionSettings("cat1", glue.DataCatalogEncryptionSettings{})) + require.NoError(t, b.ImportCatalogToGlue("cat1")) // raw catalogImports + // raw schemaVersionMetadata + require.NoError(t, b.PutSchemaVersionMetadata("sv1", "key1", "val1")) + // raw partitionIndexes + require.NoError(t, b.CreatePartitionIndex("db1", "tbl1", glue.PartitionIndex{ + IndexName: "idx1", + Keys: []string{"year"}, + })) + _, err = b.CreateUsageProfile("up1", "desc", nil) + require.NoError(t, err) + _, err = b.StartBlueprintRun("bp1") + require.NoError(t, err) + _, err = b.StartDataQualityRuleRecommendationRun("s3://bucket/path") + require.NoError(t, err) + _, err = b.CreateColumnStatisticsTaskSettings("db1", "tbl1", "role1", nil) + require.NoError(t, err) + _, err = b.StartColumnStatisticsTaskRun("db1", "tbl1") + require.NoError(t, err) + _, err = b.StartMaterializedViewRefreshTaskRun("db1", "view1") + require.NoError(t, err) + _, err = b.CreateIntegration("int1", nil) + require.NoError(t, err) + _, err = b.CreateIntegrationResourceProperty("arn:aws:glue:resource1", nil, nil) + require.NoError(t, err) + require.NoError(t, b.CreateIntegrationTableProperties("arn:aws:glue:resource1", "tbl1", nil, nil)) + b.PutDataQualityStatisticAnnotation("profile1", "stat1", "INCLUDE") + require.NoError(t, b.CreateGlueIdentityCenterConfiguration("instance1")) + _, err = b.RegisterConnectionType("custom1", "a custom connector") + require.NoError(t, err) +} + +func verifyFullState(t *testing.T, b *glue.InMemoryBackend) { + t.Helper() + + assert.Equal(t, 1, glue.DatabaseCount(b)) + assert.Equal(t, 1, glue.TableCount(b)) + assert.Equal(t, 1, glue.PartitionCount(b)) + assert.Equal(t, 1, glue.TableVersionCount(b)) + assert.Equal(t, 1, glue.CrawlerCount(b)) + assert.Equal(t, 1, glue.JobCount(b)) + assert.Equal(t, 1, glue.JobRunCount(b)) + assert.Equal(t, 1, glue.ConnectionCount(b)) + assert.Equal(t, 1, glue.BlueprintCount(b)) + assert.Equal(t, 1, glue.CustomEntityTypeCount(b)) + assert.Equal(t, 1, glue.DataQualityResultCount(b)) + assert.Equal(t, 1, glue.DevEndpointCount(b)) + assert.Equal(t, 1, glue.DataQualityRulesetCount(b)) + assert.Equal(t, 1, glue.DataQualityEvalRunCount(b)) + assert.Equal(t, 1, glue.MLTaskRunCount(b)) + assert.Equal(t, 1, glue.CustomConnectionTypeCountForTest(b)) + + trig, err := b.GetTrigger("trig1") + require.NoError(t, err) + assert.Equal(t, "trig1", trig.Name) + + wf, err := b.GetWorkflow("wf1") + require.NoError(t, err) + assert.Equal(t, "wf1", wf.Name) + wfRuns, err := b.GetWorkflowRuns("wf1") + require.NoError(t, err) + assert.Len(t, wfRuns, 1) + + clf, err := b.GetClassifier("clf1") + require.NoError(t, err) + require.NotNil(t, clf.GrokClassifier) + assert.Equal(t, "clf1", clf.GrokClassifier.Name) + + reg, err := b.DescribeRegistry("reg1") + require.NoError(t, err) + assert.Equal(t, "reg1", reg.Name) + + sch, err := b.DescribeSchema("reg1", "schema1") + require.NoError(t, err) + assert.Equal(t, "schema1", sch.SchemaName) + schVersions := b.ListSchemaVersions("reg1", "schema1") + require.Len(t, schVersions, 1) + assert.Equal(t, "def-v1", schVersions[0].SchemaDefinition) + + udf, err := b.GetUserDefinedFunction("db1", "udf1") + require.NoError(t, err) + assert.Equal(t, "udf1", udf.FunctionName) + + _, err = b.GetSecurityConfiguration("sc1") + require.NoError(t, err) + + sess, err := b.GetSession("sess1") + require.NoError(t, err) + assert.Equal(t, "sess1", sess.SessionID) + stmts, err := b.GetStatements("sess1") + require.NoError(t, err) + assert.Len(t, stmts, 1) + + to, err := b.GetTableOptimizer("db1", "tbl1", "compaction") + require.NoError(t, err) + assert.Equal(t, "compaction", to.Type) + + colStats, err := b.GetColumnStatisticsForTable("db1", "tbl1", nil) + require.NoError(t, err) + require.Len(t, colStats, 1) + assert.Equal(t, "col1", colStats[0].ColumnName) + + partColStats, err := b.GetColumnStatisticsForPartition("db1", "tbl1", []string{"2024"}, nil) + require.NoError(t, err) + require.Len(t, partColStats, 1) + + policy, hash, err := b.GetResourcePolicy("arn:aws:glue:us-east-1:123456789012:catalog") + require.NoError(t, err) + assert.Equal(t, "policy-doc", policy) + assert.NotEmpty(t, hash) + + mlTransforms := b.GetMLTransforms() + require.Len(t, mlTransforms, 1) + assert.Equal(t, "mlt1", mlTransforms[0].Name) + mlTaskRuns, err := b.GetMLTaskRuns(mlTransforms[0].TransformID) + require.NoError(t, err) + assert.Len(t, mlTaskRuns, 1) + + cat, err := b.GetCatalog("cat1") + require.NoError(t, err) + assert.Equal(t, "cat1", cat.CatalogID) + encSettings, err := b.GetDataCatalogEncryptionSettings("cat1") + require.NoError(t, err) + assert.NotNil(t, encSettings) + assert.True(t, b.GetCatalogImportStatus("cat1").ImportCompleted) + assert.Equal(t, map[string]string{"key1": "val1"}, b.QuerySchemaVersionMetadata("sv1")) + + idxs, err := b.GetPartitionIndexes("db1", "tbl1") + require.NoError(t, err) + require.Len(t, idxs, 1) + assert.Equal(t, "idx1", idxs[0].IndexName) + + _, err = b.GetUsageProfile("up1") + require.NoError(t, err) + + bpRuns := b.GetBlueprintRuns("bp1") + assert.Len(t, bpRuns, 1) + + assert.Len(t, b.ListDataQualityRuleRecommendationRuns(), 1) + + _, err = b.GetColumnStatisticsTaskSettings("db1", "tbl1") + require.NoError(t, err) + assert.Len(t, b.ListColumnStatisticsTaskRuns(), 1) + assert.Len(t, b.ListMaterializedViewRefreshTaskRuns(), 1) + assert.Len(t, b.ListIntegrations(), 1) + + _, err = b.GetIntegrationResourceProperty("arn:aws:glue:resource1") + require.NoError(t, err) + _, err = b.GetIntegrationTableProperties("arn:aws:glue:resource1", "tbl1") + require.NoError(t, err) + + annotations := b.ListDataQualityStatisticAnnotations("profile1", "stat1") + require.Len(t, annotations, 1) + assert.Equal(t, "INCLUDE", annotations[0].Inclusion) + + idConfig, err := b.GetGlueIdentityCenterConfiguration() + require.NoError(t, err) + assert.Equal(t, "instance1", idConfig.InstanceARN) + + connType, err := b.DescribeConnectionType("custom1") + require.NoError(t, err) + assert.Equal(t, "CUSTOM1", connType.ConnectionType) + + // Cascade-delete sanity check: the AddPartitionInternal/AddTableVersionInternal + // identity-fix (dbName/tableName stamped onto the stored value so store.Table's + // key is a pure function of the value, matching the pre-conversion raw-map + // behavior) must still let BatchDeleteTable cascade to the restored partition + // and table version exactly as it did before persistence. + tableErrs := b.BatchDeleteTable("db1", []string{"tbl1"}) + assert.Empty(t, tableErrs) + assert.Equal(t, 0, glue.PartitionCount(b)) + assert.Equal(t, 0, glue.TableVersionCount(b)) +} + +// TestInMemoryBackend_Restore_VersionMismatch verifies that Restore discards +// (rather than partially decodes) a snapshot whose Version does not match the +// backend's current glueSnapshotVersion, leaving the backend in its empty +// starting state instead of erroring or corrupting state. +func TestInMemoryBackend_Restore_VersionMismatch(t *testing.T) { + t.Parallel() + + orig := glue.NewInMemoryBackend("123456789012", "us-east-1") + _, err := orig.CreateDatabase(glue.DatabaseInput{Name: "db1"}, nil) + require.NoError(t, err) + + snap := orig.Snapshot(t.Context()) + require.NotNil(t, snap) + + // Corrupt the version field so Restore hits the mismatch branch, without + // hardcoding the rest of the snapshot's shape. + var generic map[string]any + require.NoError(t, json.Unmarshal(snap, &generic)) + require.Contains(t, generic, "version") + generic["version"] = 999 + + mutated, err := json.Marshal(generic) + require.NoError(t, err) + + restored := glue.NewInMemoryBackend("123456789012", "us-east-1") + require.NoError(t, restored.Restore(t.Context(), mutated)) + + assert.Equal(t, 0, glue.DatabaseCount(restored)) +} diff --git a/services/glue/store_setup.go b/services/glue/store_setup.go new file mode 100644 index 000000000..ca34448d1 --- /dev/null +++ b/services/glue/store_setup.go @@ -0,0 +1,301 @@ +package glue + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend whose key is a pure +// function of the stored value's own fields is registered exactly once, +// here, as a *store.Table[T] on b.registry. See pkgs/store's package doc and +// the services/ec2 conversion (commit 12e611a4) for the pattern this follows. +// +// A number of fields are deliberately NOT registered here and remain plain +// maps -- see the comment block above registerAllTables for the list and why. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +func databaseKeyFn(v *Database) string { return v.Name } + +// tableRecordKeyFn is the [store.Table] key function for the glue Table type. +// Named to avoid colliding with the tableKey composite-key helper or the +// Table type itself. +func tableRecordKeyFn(v *Table) string { return tableKey(v.DatabaseName, v.Name) } + +func crawlerKeyFn(v *Crawler) string { return v.Name } + +func jobKeyFn(v *Job) string { return v.Name } + +func partitionEntryKeyFn(v *Partition) string { + return partitionKey(v.DatabaseName, v.TableName, v.Values) +} + +// tableVersionEntryKeyFn derives the composite db|table|versionID key from a +// TableVersion's own nested Table pointer. It tolerates a nil Table (rather +// than panicking) because AddTableVersionInternal is the only writer and, for +// this identity function to be pure, it always populates Table's +// DatabaseName/Name to match the dbName/tableName it was given -- see that +// method for details. +func tableVersionEntryKeyFn(v *TableVersion) string { + var dbName, tableName string + if v.Table != nil { + dbName, tableName = v.Table.DatabaseName, v.Table.Name + } + + return tableVersionKey(dbName, tableName, v.VersionID) +} + +func connectionKeyFn(v *Connection) string { return v.Name } + +func blueprintKeyFn(v *Blueprint) string { return v.Name } + +func customEntityTypeKeyFn(v *CustomEntityType) string { return v.Name } + +func dataQualityResultKeyFn(v *DataQualityResult) string { return v.ResultID } + +func devEndpointKeyFn(v *DevEndpoint) string { return v.EndpointName } + +func jobBookmarkKeyFn(v *JobBookmark) string { return v.JobName } + +func dataQualityRulesetKeyFn(v *DataQualityRuleset) string { return v.Name } + +func dataQualityEvalRunKeyFn(v *DataQualityEvaluationRun) string { return v.RunID } + +func triggerKeyFn(v *Trigger) string { return v.Name } + +func workflowKeyFn(v *Workflow) string { return v.Name } + +func registryKeyFn(v *Registry) string { return v.Name } + +// schemaEntryKeyFn wraps the existing schemaKey composite-key helper. +func schemaEntryKeyFn(v *Schema) string { return schemaKey(v.RegistryName, v.SchemaName) } + +func udfEntryKeyFn(v *UserDefinedFunction) string { return v.DatabaseName + "|" + v.FunctionName } + +func securityConfigKeyFn(v *SecurityConfiguration) string { return v.Name } + +func sessionKeyFn(v *Session) string { return v.SessionID } + +func tableOptimizerEntryKeyFn(v *TableOptimizer) string { + return v.DatabaseName + "|" + v.TableName + "|" + v.Type +} + +func mlTransformKeyFn(v *MLTransform) string { return v.TransformID } + +func catalogEntryKeyFn(v *CatalogEntry) string { return v.CatalogID } + +func usageProfileKeyFn(v *UsageProfile) string { return v.Name } + +func blueprintRunKeyFn(v *BlueprintRun) string { return v.RunID } + +func dqRecommendationRunKeyFn(v *DQRuleRecommendationRun) string { return v.RecommendationRunID } + +func columnStatTaskSettingsEntryKeyFn(v *ColumnStatisticsTaskSettings) string { + return v.DatabaseName + "|" + v.TableName +} + +func columnStatTaskRunKeyFn(v *ColumnStatisticsTaskRun) string { return v.ColumnStatisticsTaskRunID } + +func materializedViewRunKeyFn(v *MaterializedViewRefreshRun) string { return v.TaskRunID } + +func integrationKeyFn(v *Integration) string { return v.IntegrationName } + +func integrationResourcePropKeyFn(v *IntegrationResourceProperty) string { return v.ResourceArn } + +func integrationTablePropKeyFn(v *IntegrationTableProperties) string { + return v.ResourceArn + "|" + v.TableName +} + +// mlTaskRunEntryKeyFn wraps the existing mlTaskRunKey composite-key helper. +func mlTaskRunEntryKeyFn(v *MLTaskRun) string { return mlTaskRunKey(v.TransformID, v.TaskRunID) } + +// dqStatisticAnnotationKeyFn wraps the existing dqAnnotationKey composite-key helper. +func dqStatisticAnnotationKeyFn(v *StatisticAnnotation) string { + return dqAnnotationKey(v.ProfileID, v.StatisticID) +} + +func customConnectionTypeKeyFn(v *ConnectionTypeInfo) string { return v.ConnectionType } + +// registerAllTables registers every converted resource map on b.registry +// exactly once. It must be called during construction only (immediately +// after b.registry is created), never on every Reset() -- store.Register +// panics on a duplicate name, so runtime resets go through +// registry.ResetAll() instead (see InMemoryBackend.Reset in backend.go). +// +// The following resource fields are deliberately left as plain maps (not +// registered here) because their key is not a pure function of the stored +// value's own fields, which store.Table requires, or because the field holds +// a one-to-many list rather than a single value per key: +// - partitionIndexes: value type PartitionIndex carries no database/table +// identity field of its own; keyed externally as "db|table|indexName". +// GetPartitionIndexes also relies on a prefix scan over that raw key. +// - tableColumnStats / partitionColumnStats: value type ColumnStatistics +// carries only ColumnName, no database/table (or partition) identity; +// keyed externally. +// - resourcePolicies: value type resourcePolicyEntry carries no resource-ARN +// identity field; keyed externally by ARN or "__global__". +// - catalogEncryptionSettings / catalogImports: value types carry no +// catalogID identity field; keyed externally by catalogID or accountID. +// - schemaVersionMetadata: nested map[string]string value, not a *T pointer +// -- store.Table requires a pointer-to-struct value with its own identity. +// - jobRuns / workflowRuns / schemaVersions / sessionStatements / +// crawlHistory: one-to-many (map[string][]*T) history/list fields, not +// the map[string]*T shape store.Table replaces. +// - jobRunReadyAt / jobRunDoneAt / crawlerReadyAt: ephemeral lifecycle-timer +// bookkeeping (ready/done rather than resource state), not persisted, and +// not map[string]*T. +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call to the concrete field and value type. A closure list (rather than one +// statement per field in a single function body) keeps registerAllTables +// small regardless of how many resource tables the backend grows to. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.databases = store.Register(b.registry, "databases", store.New(databaseKeyFn)) + }, + func(b *InMemoryBackend) { + b.tables = store.Register(b.registry, "tables", store.New(tableRecordKeyFn)) + }, + func(b *InMemoryBackend) { + b.crawlers = store.Register(b.registry, "crawlers", store.New(crawlerKeyFn)) + }, + func(b *InMemoryBackend) { + b.jobs = store.Register(b.registry, "jobs", store.New(jobKeyFn)) + }, + func(b *InMemoryBackend) { + b.partitions = store.Register(b.registry, "partitions", store.New(partitionEntryKeyFn)) + }, + func(b *InMemoryBackend) { + b.tableVersions = store.Register(b.registry, "tableVersions", store.New(tableVersionEntryKeyFn)) + }, + func(b *InMemoryBackend) { + b.connections = store.Register(b.registry, "connections", store.New(connectionKeyFn)) + }, + func(b *InMemoryBackend) { + b.blueprints = store.Register(b.registry, "blueprints", store.New(blueprintKeyFn)) + }, + func(b *InMemoryBackend) { + b.customEntityTypes = store.Register(b.registry, "customEntityTypes", store.New(customEntityTypeKeyFn)) + }, + func(b *InMemoryBackend) { + b.dataQualityResult = store.Register(b.registry, "dataQualityResult", store.New(dataQualityResultKeyFn)) + }, + func(b *InMemoryBackend) { + b.devEndpoints = store.Register(b.registry, "devEndpoints", store.New(devEndpointKeyFn)) + }, + func(b *InMemoryBackend) { + b.jobBookmarks = store.Register(b.registry, "jobBookmarks", store.New(jobBookmarkKeyFn)) + }, + func(b *InMemoryBackend) { + b.dataQualityRulesets = store.Register(b.registry, "dataQualityRulesets", store.New(dataQualityRulesetKeyFn)) + }, + func(b *InMemoryBackend) { + b.dataQualityEvalRuns = store.Register( + b.registry, + "dataQualityEvalRuns", + store.New(dataQualityEvalRunKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.triggers = store.Register(b.registry, "triggers", store.New(triggerKeyFn)) + }, + func(b *InMemoryBackend) { + b.workflows = store.Register(b.registry, "workflows", store.New(workflowKeyFn)) + }, + func(b *InMemoryBackend) { + b.classifiers = store.Register(b.registry, "classifiers", store.New(classifierName)) + }, + func(b *InMemoryBackend) { + b.registries = store.Register(b.registry, "registries", store.New(registryKeyFn)) + }, + func(b *InMemoryBackend) { + b.schemas = store.Register(b.registry, "schemas", store.New(schemaEntryKeyFn)) + }, + func(b *InMemoryBackend) { + b.udfs = store.Register(b.registry, "udfs", store.New(udfEntryKeyFn)) + }, + func(b *InMemoryBackend) { + b.securityConfigs = store.Register(b.registry, "securityConfigs", store.New(securityConfigKeyFn)) + }, + func(b *InMemoryBackend) { + b.sessions = store.Register(b.registry, "sessions", store.New(sessionKeyFn)) + }, + func(b *InMemoryBackend) { + b.tableOptimizers = store.Register(b.registry, "tableOptimizers", store.New(tableOptimizerEntryKeyFn)) + }, + func(b *InMemoryBackend) { + b.mlTransforms = store.Register(b.registry, "mlTransforms", store.New(mlTransformKeyFn)) + }, + func(b *InMemoryBackend) { + b.catalogs = store.Register(b.registry, "catalogs", store.New(catalogEntryKeyFn)) + }, + func(b *InMemoryBackend) { + b.usageProfiles = store.Register(b.registry, "usageProfiles", store.New(usageProfileKeyFn)) + }, + func(b *InMemoryBackend) { + b.blueprintRuns = store.Register(b.registry, "blueprintRuns", store.New(blueprintRunKeyFn)) + }, + func(b *InMemoryBackend) { + b.dqRecommendationRuns = store.Register( + b.registry, + "dqRecommendationRuns", + store.New(dqRecommendationRunKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.columnStatTaskSettings = store.Register( + b.registry, + "columnStatTaskSettings", + store.New(columnStatTaskSettingsEntryKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.columnStatTaskRuns = store.Register(b.registry, "columnStatTaskRuns", store.New(columnStatTaskRunKeyFn)) + }, + func(b *InMemoryBackend) { + b.materializedViewRuns = store.Register( + b.registry, + "materializedViewRuns", + store.New(materializedViewRunKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.integrations = store.Register(b.registry, "integrations", store.New(integrationKeyFn)) + }, + func(b *InMemoryBackend) { + b.integrationResourceProps = store.Register( + b.registry, + "integrationResourceProps", + store.New(integrationResourcePropKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.integrationTableProps = store.Register( + b.registry, + "integrationTableProps", + store.New(integrationTablePropKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.mlTaskRuns = store.Register(b.registry, "mlTaskRuns", store.New(mlTaskRunEntryKeyFn)) + }, + func(b *InMemoryBackend) { + b.dqStatisticAnnotations = store.Register( + b.registry, + "dqStatisticAnnotations", + store.New(dqStatisticAnnotationKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.customConnectionTypes = store.Register( + b.registry, + "customConnectionTypes", + store.New(customConnectionTypeKeyFn), + ) + }, +} diff --git a/services/iam/PARITY.md b/services/iam/PARITY.md new file mode 100644 index 000000000..a1ccbdd65 --- /dev/null +++ b/services/iam/PARITY.md @@ -0,0 +1,26 @@ +--- +service: iam +sdk_module: aws-sdk-go-v2/service/iam # version: see go.mod (backfilled) +last_audit_commit: 71cd5441 +last_audit_date: 2026-07-05 +overall: A # ~1111 LOC genuine fixes incl. 2 disguised stubs +protocol: aws-query -> XML +families: + users_groups_roles: {status: ok, note: CRUD + path/ARN verified; tags-at-creation FIXED (were dropped)} + policies: {status: ok, note: managed+inline, versions, default version; PolicyXML gained Tags field} + access_keys: {status: ok, note: PROVEN — create/rotate/status, secret only on create} + providers: {status: ok, note: PROVEN — SAML/OIDC CRUD, server certificates, login profile, password policy} +ops: + ListInstanceProfilesForRole: {wire: ok, errors: ok, state: ok, persist: ok, note: FIXED disguised stub — ignored RoleName, hardcoded empty; now wired to real backend} + GetAccountAuthorizationDetails: {wire: partial, errors: ok, state: ok, persist: ok, note: FIXED fabricated fake v1 policy version -> real ListPolicyVersions; STILL no Marker/MaxItems/Filter pagination + missing RoleDetail.InstanceProfileList} + GetRole/GetRolePolicy/GetUserPolicy/GetGroupPolicy/GetPolicyVersion: {wire: ok, note: FIXED policy documents now percent-encoded (RFC 3986) at wire boundary; stored plain} +gaps: + - comprehensiveBackend uses own sync.Mutex alongside coarse lockmetrics.RWMutex — violates one-coarse-lock rule (bd: gopherstack-gjp) + - GetAccountAuthorizationDetails pagination + RoleDetail.InstanceProfileList (bd: gopherstack-gjp) +leaks: {status: clean, note: persistence leaks FIXED — handler tags + comprehensive state (SSH keys, MFA links, access-advisor, last-accessed) now snapshotted; go test -race passes} +--- + +## Notes +- HTTP status codes FIXED: NoSuchEntity 404, EntityAlreadyExists/DeleteConflict/LimitExceeded 409 (were all 400); default code ServiceFailure (was InternalFailure). +- Policy documents: stored as plain JSON in backend, percent-encoded ONLY at wire boundary via encodePolicyDocument(). +- STS/assume-role cross-service linkage is out of services/iam scope (wired in cli.go). diff --git a/services/iam/backend.go b/services/iam/backend.go index d90b2b897..e8186ea17 100644 --- a/services/iam/backend.go +++ b/services/iam/backend.go @@ -18,6 +18,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/collections" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" "github.com/blackbirdworks/gopherstack/pkgs/page" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) var ( @@ -354,23 +355,23 @@ const accessKeyStatusActive = "Active" // InMemoryBackend implements StorageBackend using in-memory maps. type InMemoryBackend struct { - roles map[string]Role - delegationRequests map[string]DelegationRequest - policies map[string]Policy + roles *store.Table[Role] + delegationRequests *store.Table[DelegationRequest] + policies *store.Table[Policy] policyByARN map[string]string roleByARN map[string]string - accessKeys map[string]AccessKey + accessKeys *store.Table[AccessKey] userAccessKeys map[string][]string // username -> list of access key IDs - instanceProfiles map[string]InstanceProfile - samlProviders map[string]SAMLProvider + instanceProfiles *store.Table[InstanceProfile] + samlProviders *store.Table[SAMLProvider] groupMembers map[string][]string groupPolicies map[string][]string userPolicies map[string][]string policyAttachments map[string]policyAttachmentRefs mu *lockmetrics.RWMutex - loginProfiles map[string]LoginProfile - groups map[string]Group - oidcProviders map[string]OIDCProvider + loginProfiles *store.Table[LoginProfile] + groups *store.Table[Group] + oidcProviders *store.Table[OIDCProvider] userInlinePolicies map[string]map[string]string roleInlinePolicies map[string]map[string]string groupInlinePolicies map[string]map[string]string @@ -378,12 +379,13 @@ type InMemoryBackend struct { policyVersions map[string][]StoredPolicyVersion policyVersionCounters map[string]int // monotonic counter per policy ARN, never resets on delete deletedV1Policies map[string]bool // tracks policies where v1 has been explicitly deleted - serviceSpecificCreds map[string]ServiceSpecificCredential - virtualMFADevices map[string]VirtualMFADevice - signingCertificates map[string]SigningCertificate // certID → SigningCertificate - serverCertificates map[string]ServerCertificate // name → ServerCertificate + serviceSpecificCreds *store.Table[ServiceSpecificCredential] + virtualMFADevices *store.Table[VirtualMFADevice] + signingCertificates *store.Table[SigningCertificate] // certID → SigningCertificate + serverCertificates *store.Table[ServerCertificate] // name → ServerCertificate passwordPolicy *PasswordPolicy - users map[string]User + users *store.Table[User] + registry *store.Registry comprehensive *comprehensiveBackend accountID string accountAliases []string @@ -409,19 +411,10 @@ func NewInMemoryBackend() *InMemoryBackend { // NewInMemoryBackendWithConfig creates a new IAM InMemoryBackend with the given account ID. func NewInMemoryBackendWithConfig(accountID string) *InMemoryBackend { - return &InMemoryBackend{ - users: make(map[string]User), - roles: make(map[string]Role), + b := &InMemoryBackend{ roleByARN: make(map[string]string), - policies: make(map[string]Policy), policyByARN: make(map[string]string), - groups: make(map[string]Group), - accessKeys: make(map[string]AccessKey), userAccessKeys: make(map[string][]string), - instanceProfiles: make(map[string]InstanceProfile), - samlProviders: make(map[string]SAMLProvider), - oidcProviders: make(map[string]OIDCProvider), - loginProfiles: make(map[string]LoginProfile), userPolicies: make(map[string][]string), rolePolicies: make(map[string][]string), groupPolicies: make(map[string][]string), @@ -434,13 +427,9 @@ func NewInMemoryBackendWithConfig(accountID string) *InMemoryBackend { policyVersions: make(map[string][]StoredPolicyVersion), policyVersionCounters: make(map[string]int), deletedV1Policies: make(map[string]bool), - serviceSpecificCreds: make(map[string]ServiceSpecificCredential), - virtualMFADevices: make(map[string]VirtualMFADevice), - signingCertificates: make(map[string]SigningCertificate), - serverCertificates: make(map[string]ServerCertificate), - delegationRequests: make(map[string]DelegationRequest), accountID: accountID, mu: lockmetrics.New("iam"), + registry: store.NewRegistry(), comprehensive: newComprehensiveBackend(), // Sorted name indexes start empty; populated via insertSorted on create. sortedUserNames: nil, @@ -449,6 +438,10 @@ func NewInMemoryBackendWithConfig(accountID string) *InMemoryBackend { sortedGroupNames: nil, sortedIPNames: nil, } + + registerAllTables(b) + + return b } // normPath returns a normalized IAM path, defaulting to "/" if empty. @@ -492,9 +485,10 @@ func (b *InMemoryBackend) addPolicyAttachmentLocked(policyArn, entityName, entit b.policyAttachments[policyArn] = refs if polName, ok := b.policyByARN[policyArn]; ok { - pol := b.policies[polName] - pol.AttachmentCount = len(refs.users) + len(refs.roles) + len(refs.groups) - b.policies[polName] = pol + if pol, ok2 := b.policies.Get(polName); ok2 { + pol.AttachmentCount = len(refs.users) + len(refs.roles) + len(refs.groups) + b.policies.Put(pol) + } } } @@ -517,9 +511,10 @@ func (b *InMemoryBackend) removePolicyAttachmentLocked(policyArn, entityName, en delete(b.policyAttachments, policyArn) if polName, ok := b.policyByARN[policyArn]; ok { - pol := b.policies[polName] - pol.AttachmentCount = 0 - b.policies[polName] = pol + if pol, ok2 := b.policies.Get(polName); ok2 { + pol.AttachmentCount = 0 + b.policies.Put(pol) + } } return @@ -528,9 +523,10 @@ func (b *InMemoryBackend) removePolicyAttachmentLocked(policyArn, entityName, en b.policyAttachments[policyArn] = refs if polName, ok := b.policyByARN[policyArn]; ok { - pol := b.policies[polName] - pol.AttachmentCount = len(refs.users) + len(refs.roles) + len(refs.groups) - b.policies[polName] = pol + if pol, ok2 := b.policies.Get(polName); ok2 { + pol.AttachmentCount = len(refs.users) + len(refs.roles) + len(refs.groups) + b.policies.Put(pol) + } } } @@ -549,23 +545,23 @@ func (b *InMemoryBackend) getPolicyByARNLocked(policyArn string) (Policy, bool) return Policy{}, false } - pol, exists := b.policies[policyName] + pol, exists := b.policies.Get(policyName) if !exists { return Policy{}, false } - return pol, true + return *pol, true } func (b *InMemoryBackend) rebuildIndexesLocked() { - b.roleByARN = make(map[string]string, len(b.roles)) - for roleName, role := range b.roles { - b.roleByARN[role.Arn] = roleName + b.roleByARN = make(map[string]string, b.roles.Len()) + for _, role := range b.roles.All() { + b.roleByARN[role.Arn] = role.RoleName } - b.policyByARN = make(map[string]string, len(b.policies)) - for policyName, policy := range b.policies { - b.policyByARN[policy.Arn] = policyName + b.policyByARN = make(map[string]string, b.policies.Len()) + for _, policy := range b.policies.All() { + b.policyByARN[policy.Arn] = policy.PolicyName } b.policyAttachments = make(map[string]policyAttachmentRefs) @@ -588,11 +584,11 @@ func (b *InMemoryBackend) rebuildIndexesLocked() { } // Rebuild sorted name indexes. - b.sortedUserNames = rebuildSortedNames(b.users) - b.sortedRoleNames = rebuildSortedNames(b.roles) - b.sortedPolicyNames = rebuildSortedNames(b.policies) - b.sortedGroupNames = rebuildSortedNames(b.groups) - b.sortedIPNames = rebuildSortedNames(b.instanceProfiles) + b.sortedUserNames = sortedTableNames(b.users, usersKeyFn) + b.sortedRoleNames = sortedTableNames(b.roles, rolesKeyFn) + b.sortedPolicyNames = sortedTableNames(b.policies, policiesKeyFn) + b.sortedGroupNames = sortedTableNames(b.groups, groupsKeyFn) + b.sortedIPNames = sortedTableNames(b.instanceProfiles, instanceProfilesKeyFn) } // ---- Users ---- @@ -602,7 +598,7 @@ func (b *InMemoryBackend) CreateUser(userName, path, permissionsBoundary string) b.mu.Lock("CreateUser") defer b.mu.Unlock() - if _, exists := b.users[userName]; exists { + if _, exists := b.users.Get(userName); exists { return nil, fmt.Errorf("%w: user %q already exists", ErrUserAlreadyExists, userName) } @@ -615,7 +611,7 @@ func (b *InMemoryBackend) CreateUser(userName, path, permissionsBoundary string) CreateDate: time.Now().UTC(), PermissionsBoundary: permissionsBoundary, } - b.users[userName] = u + b.users.Put(&u) b.sortedUserNames = insertSorted(b.sortedUserNames, userName) return &u, nil @@ -626,7 +622,7 @@ func (b *InMemoryBackend) DeleteUser(userName string) error { b.mu.Lock("DeleteUser") defer b.mu.Unlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -640,12 +636,12 @@ func (b *InMemoryBackend) DeleteUser(userName string) error { // Clean up access keys belonging to the user. for _, id := range b.userAccessKeys[userName] { - delete(b.accessKeys, id) + b.accessKeys.Delete(id) } delete(b.userAccessKeys, userName) // Clean up login profile. - delete(b.loginProfiles, userName) + b.loginProfiles.Delete(userName) // Remove user from all group memberships. for groupName, members := range b.groupMembers { @@ -658,7 +654,7 @@ func (b *InMemoryBackend) DeleteUser(userName string) error { } } - delete(b.users, userName) + b.users.Delete(userName) b.sortedUserNames = deleteSorted(b.sortedUserNames, userName) return nil @@ -671,7 +667,7 @@ func (b *InMemoryBackend) ListUsers(marker string, maxItems int) (page.Page[User return pageFromSortedNames( b.sortedUserNames, - b.users, + b.users.Get, marker, maxItems, iamDefaultMaxItems, @@ -683,12 +679,12 @@ func (b *InMemoryBackend) GetUser(userName string) (*User, error) { b.mu.RLock("GetUser") defer b.mu.RUnlock() - u, exists := b.users[userName] + u, exists := b.users.Get(userName) if !exists { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } - return &u, nil + return u, nil } // ---- Roles ---- @@ -700,7 +696,7 @@ func (b *InMemoryBackend) CreateRole( b.mu.Lock("CreateRole") defer b.mu.Unlock() - if _, exists := b.roles[roleName]; exists { + if _, exists := b.roles.Get(roleName); exists { return nil, fmt.Errorf("%w: role %q already exists", ErrRoleAlreadyExists, roleName) } @@ -725,7 +721,7 @@ func (b *InMemoryBackend) CreateRole( CreateDate: time.Now().UTC(), PermissionsBoundary: permissionsBoundary, } - b.roles[roleName] = r + b.roles.Put(&r) b.roleByARN[r.Arn] = roleName b.sortedRoleNames = insertSorted(b.sortedRoleNames, roleName) @@ -737,7 +733,7 @@ func (b *InMemoryBackend) DeleteRole(roleName string) error { b.mu.Lock("DeleteRole") defer b.mu.Unlock() - if _, exists := b.roles[roleName]; !exists { + if _, exists := b.roles.Get(roleName); !exists { return fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } @@ -749,8 +745,8 @@ func (b *InMemoryBackend) DeleteRole(roleName string) error { return fmt.Errorf("%w: role %q has inline policies", ErrDeleteConflict, roleName) } - role := b.roles[roleName] - delete(b.roles, roleName) + role, _ := b.roles.Get(roleName) + b.roles.Delete(roleName) delete(b.roleByARN, role.Arn) b.sortedRoleNames = deleteSorted(b.sortedRoleNames, roleName) @@ -764,7 +760,7 @@ func (b *InMemoryBackend) ListRoles(marker string, maxItems int) (page.Page[Role return pageFromSortedNames( b.sortedRoleNames, - b.roles, + b.roles.Get, marker, maxItems, iamDefaultMaxItems, @@ -776,12 +772,12 @@ func (b *InMemoryBackend) GetRole(roleName string) (*Role, error) { b.mu.RLock("GetRole") defer b.mu.RUnlock() - r, exists := b.roles[roleName] + r, exists := b.roles.Get(roleName) if !exists { return nil, fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } - return &r, nil + return r, nil } // GetRoleByArn retrieves a single IAM role by its full ARN. @@ -794,12 +790,12 @@ func (b *InMemoryBackend) GetRoleByArn(roleArn string) (*Role, error) { return nil, fmt.Errorf("%w: role with ARN %q not found", ErrRoleNotFound, roleArn) } - role, exists := b.roles[roleName] + role, exists := b.roles.Get(roleName) if !exists { return nil, fmt.Errorf("%w: role with ARN %q not found", ErrRoleNotFound, roleArn) } - return &role, nil + return role, nil } // UpdateRoleMaxSessionDuration sets the maximum session duration for a role. @@ -810,13 +806,13 @@ func (b *InMemoryBackend) UpdateRoleMaxSessionDuration( b.mu.Lock("UpdateRoleMaxSessionDuration") defer b.mu.Unlock() - r, exists := b.roles[roleName] + r, exists := b.roles.Get(roleName) if !exists { return fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } r.MaxSessionDuration = maxSessionDuration - b.roles[roleName] = r + b.roles.Put(r) return nil } @@ -831,7 +827,7 @@ func (b *InMemoryBackend) CreatePolicy(policyName, path, policyDocument string) b.mu.Lock("CreatePolicy") defer b.mu.Unlock() - if _, exists := b.policies[policyName]; exists { + if _, exists := b.policies.Get(policyName); exists { return nil, fmt.Errorf("%w: policy %q already exists", ErrPolicyAlreadyExists, policyName) } @@ -857,7 +853,7 @@ func (b *InMemoryBackend) CreatePolicy(policyName, path, policyDocument string) DefaultVersionID: "v1", IsAttachable: true, } - b.policies[policyName] = pol + b.policies.Put(&pol) b.policyByARN[pol.Arn] = policyName b.sortedPolicyNames = insertSorted(b.sortedPolicyNames, policyName) @@ -903,7 +899,7 @@ func (b *InMemoryBackend) DeletePolicy(policyArn string) error { return fmt.Errorf("%w: policy %q not found", ErrPolicyNotFound, policyArn) } - delete(b.policies, policyName) + b.policies.Delete(policyName) delete(b.policyByARN, policyArn) delete(b.policyAttachments, policyArn) delete(b.policyVersions, policyArn) @@ -921,7 +917,7 @@ func (b *InMemoryBackend) ListPolicies(marker string, maxItems int) (page.Page[P return pageFromSortedNames( b.sortedPolicyNames, - b.policies, + b.policies.Get, marker, maxItems, iamDefaultMaxItems, @@ -933,7 +929,7 @@ func (b *InMemoryBackend) AttachUserPolicy(userName, policyArn string) error { b.mu.Lock("AttachUserPolicy") defer b.mu.Unlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -952,7 +948,7 @@ func (b *InMemoryBackend) DetachUserPolicy(userName, policyArn string) error { b.mu.Lock("DetachUserPolicy") defer b.mu.Unlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -974,7 +970,7 @@ func (b *InMemoryBackend) AttachRolePolicy(roleName, policyArn string) error { b.mu.Lock("AttachRolePolicy") defer b.mu.Unlock() - if _, exists := b.roles[roleName]; !exists { + if _, exists := b.roles.Get(roleName); !exists { return fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } @@ -993,7 +989,7 @@ func (b *InMemoryBackend) DetachRolePolicy(roleName, policyArn string) error { b.mu.Lock("DetachRolePolicy") defer b.mu.Unlock() - if _, exists := b.roles[roleName]; !exists { + if _, exists := b.roles.Get(roleName); !exists { return fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } @@ -1017,7 +1013,7 @@ func (b *InMemoryBackend) CreateGroup(groupName, path string) (*Group, error) { b.mu.Lock("CreateGroup") defer b.mu.Unlock() - if _, exists := b.groups[groupName]; exists { + if _, exists := b.groups.Get(groupName); exists { return nil, fmt.Errorf("%w: group %q already exists", ErrGroupAlreadyExists, groupName) } @@ -1029,7 +1025,7 @@ func (b *InMemoryBackend) CreateGroup(groupName, path string) (*Group, error) { Path: p, CreateDate: time.Now().UTC(), } - b.groups[groupName] = g + b.groups.Put(&g) b.sortedGroupNames = insertSorted(b.sortedGroupNames, groupName) return &g, nil @@ -1040,7 +1036,7 @@ func (b *InMemoryBackend) DeleteGroup(groupName string) error { b.mu.Lock("DeleteGroup") defer b.mu.Unlock() - if _, exists := b.groups[groupName]; !exists { + if _, exists := b.groups.Get(groupName); !exists { return fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } @@ -1052,7 +1048,7 @@ func (b *InMemoryBackend) DeleteGroup(groupName string) error { return fmt.Errorf("%w: group %q has inline policies", ErrDeleteConflict, groupName) } - delete(b.groups, groupName) + b.groups.Delete(groupName) b.sortedGroupNames = deleteSorted(b.sortedGroupNames, groupName) // Clean up group membership tracking. @@ -1066,11 +1062,11 @@ func (b *InMemoryBackend) AddUserToGroup(groupName, userName string) error { b.mu.Lock("AddUserToGroup") defer b.mu.Unlock() - if _, exists := b.groups[groupName]; !exists { + if _, exists := b.groups.Get(groupName); !exists { return fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -1088,11 +1084,11 @@ func (b *InMemoryBackend) RemoveUserFromGroup(groupName, userName string) error b.mu.Lock("RemoveUserFromGroup") defer b.mu.Unlock() - if _, exists := b.groups[groupName]; !exists { + if _, exists := b.groups.Get(groupName); !exists { return fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -1113,12 +1109,12 @@ func (b *InMemoryBackend) GetGroup(groupName string) (*Group, error) { b.mu.RLock("GetGroup") defer b.mu.RUnlock() - g, exists := b.groups[groupName] + g, exists := b.groups.Get(groupName) if !exists { return nil, fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } - return &g, nil + return g, nil } // GetGroupUsers returns the users that are members of the given group. @@ -1126,7 +1122,7 @@ func (b *InMemoryBackend) GetGroupUsers(groupName string) ([]User, error) { b.mu.RLock("GetGroupUsers") defer b.mu.RUnlock() - if _, exists := b.groups[groupName]; !exists { + if _, exists := b.groups.Get(groupName); !exists { return nil, fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } @@ -1134,8 +1130,8 @@ func (b *InMemoryBackend) GetGroupUsers(groupName string) ([]User, error) { out := make([]User, 0, len(members)) for _, userName := range members { - if u, ok := b.users[userName]; ok { - out = append(out, u) + if u, ok := b.users.Get(userName); ok { + out = append(out, *u) } } @@ -1147,7 +1143,7 @@ func (b *InMemoryBackend) AttachGroupPolicy(groupName, policyArn string) error { b.mu.Lock("AttachGroupPolicy") defer b.mu.Unlock() - if _, exists := b.groups[groupName]; !exists { + if _, exists := b.groups.Get(groupName); !exists { return fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } @@ -1166,7 +1162,7 @@ func (b *InMemoryBackend) DetachGroupPolicy(groupName, policyArn string) error { b.mu.Lock("DetachGroupPolicy") defer b.mu.Unlock() - if _, exists := b.groups[groupName]; !exists { + if _, exists := b.groups.Get(groupName); !exists { return fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } @@ -1188,7 +1184,7 @@ func (b *InMemoryBackend) ListAttachedGroupPolicies(groupName string) ([]Attache b.mu.RLock("ListAttachedGroupPolicies") defer b.mu.RUnlock() - if _, exists := b.groups[groupName]; !exists { + if _, exists := b.groups.Get(groupName); !exists { return nil, fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } @@ -1210,7 +1206,7 @@ func (b *InMemoryBackend) ListGroups(marker string, maxItems int) (page.Page[Gro return pageFromSortedNames( b.sortedGroupNames, - b.groups, + b.groups.Get, marker, maxItems, iamDefaultMaxItems, @@ -1224,7 +1220,7 @@ func (b *InMemoryBackend) CreateAccessKey(userName string) (*AccessKey, error) { b.mu.Lock("CreateAccessKey") defer b.mu.Unlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -1252,7 +1248,7 @@ func (b *InMemoryBackend) CreateAccessKey(userName string) (*AccessKey, error) { Status: "Active", CreateDate: time.Now().UTC(), } - b.accessKeys[ak.AccessKeyID] = ak + b.accessKeys.Put(&ak) b.userAccessKeys[userName] = append(b.userAccessKeys[userName], ak.AccessKeyID) return &ak, nil @@ -1263,7 +1259,7 @@ func (b *InMemoryBackend) DeleteAccessKey(userName, accessKeyID string) error { b.mu.Lock("DeleteAccessKey") defer b.mu.Unlock() - ak, exists := b.accessKeys[accessKeyID] + ak, exists := b.accessKeys.Get(accessKeyID) if !exists || ak.UserName != userName { return fmt.Errorf( "%w: access key %q not found for user %q", @@ -1273,7 +1269,7 @@ func (b *InMemoryBackend) DeleteAccessKey(userName, accessKeyID string) error { ) } - delete(b.accessKeys, accessKeyID) + b.accessKeys.Delete(accessKeyID) keys := b.userAccessKeys[userName] for i, id := range keys { @@ -1295,7 +1291,7 @@ func (b *InMemoryBackend) ListAccessKeys( b.mu.RLock("ListAccessKeys") defer b.mu.RUnlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return page.Page[AccessKey]{}, fmt.Errorf( "%w: user %q not found", ErrUserNotFound, @@ -1306,8 +1302,8 @@ func (b *InMemoryBackend) ListAccessKeys( userKeys := b.userAccessKeys[userName] keys := make([]AccessKey, 0, len(userKeys)) for _, id := range userKeys { - if ak, ok := b.accessKeys[id]; ok { - keys = append(keys, ak) + if ak, ok := b.accessKeys.Get(id); ok { + keys = append(keys, *ak) } } @@ -1323,7 +1319,7 @@ func (b *InMemoryBackend) CreateInstanceProfile(name, path string) (*InstancePro b.mu.Lock("CreateInstanceProfile") defer b.mu.Unlock() - if _, exists := b.instanceProfiles[name]; exists { + if _, exists := b.instanceProfiles.Get(name); exists { return nil, fmt.Errorf( "%w: instance profile %q already exists", ErrInstanceProfileAlreadyExists, @@ -1340,7 +1336,7 @@ func (b *InMemoryBackend) CreateInstanceProfile(name, path string) (*InstancePro Roles: []string{}, CreateDate: time.Now().UTC(), } - b.instanceProfiles[name] = ip + b.instanceProfiles.Put(&ip) b.sortedIPNames = insertSorted(b.sortedIPNames, name) return &ip, nil @@ -1351,11 +1347,11 @@ func (b *InMemoryBackend) DeleteInstanceProfile(name string) error { b.mu.Lock("DeleteInstanceProfile") defer b.mu.Unlock() - if _, exists := b.instanceProfiles[name]; !exists { + if _, exists := b.instanceProfiles.Get(name); !exists { return fmt.Errorf("%w: instance profile %q not found", ErrInstanceProfileNotFound, name) } - delete(b.instanceProfiles, name) + b.instanceProfiles.Delete(name) b.sortedIPNames = deleteSorted(b.sortedIPNames, name) return nil @@ -1371,7 +1367,7 @@ func (b *InMemoryBackend) ListInstanceProfiles( return pageFromSortedNames( b.sortedIPNames, - b.instanceProfiles, + b.instanceProfiles.Get, marker, maxItems, iamDefaultMaxItems, @@ -1383,7 +1379,7 @@ func (b *InMemoryBackend) AddRoleToInstanceProfile(instanceProfileName, roleName b.mu.Lock("AddRoleToInstanceProfile") defer b.mu.Unlock() - ip, exists := b.instanceProfiles[instanceProfileName] + ip, exists := b.instanceProfiles.Get(instanceProfileName) if !exists { return fmt.Errorf( "%w: instance profile %q not found", @@ -1392,7 +1388,7 @@ func (b *InMemoryBackend) AddRoleToInstanceProfile(instanceProfileName, roleName ) } - if _, roleExists := b.roles[roleName]; !roleExists { + if _, roleExists := b.roles.Get(roleName); !roleExists { return fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } @@ -1410,7 +1406,7 @@ func (b *InMemoryBackend) AddRoleToInstanceProfile(instanceProfileName, roleName } ip.Roles = append(ip.Roles, roleName) - b.instanceProfiles[instanceProfileName] = ip + b.instanceProfiles.Put(ip) return nil } @@ -1422,7 +1418,7 @@ func (b *InMemoryBackend) RemoveRoleFromInstanceProfile( b.mu.Lock("RemoveRoleFromInstanceProfile") defer b.mu.Unlock() - ip, exists := b.instanceProfiles[instanceProfileName] + ip, exists := b.instanceProfiles.Get(instanceProfileName) if !exists { return fmt.Errorf( "%w: instance profile %q not found", @@ -1434,7 +1430,7 @@ func (b *InMemoryBackend) RemoveRoleFromInstanceProfile( for i, r := range ip.Roles { if r == roleName { ip.Roles = append(ip.Roles[:i], ip.Roles[i+1:]...) - b.instanceProfiles[instanceProfileName] = ip + b.instanceProfiles.Put(ip) return nil } @@ -1458,9 +1454,9 @@ func (b *InMemoryBackend) ListAllRoles() []Role { b.mu.RLock("ListAllRoles") defer b.mu.RUnlock() - roles := make([]Role, 0, len(b.roles)) - for _, r := range b.roles { - roles = append(roles, r) + roles := make([]Role, 0, b.roles.Len()) + for _, r := range b.roles.All() { + roles = append(roles, *r) } sort.Slice(roles, func(i, j int) bool { return roles[i].RoleName < roles[j].RoleName }) @@ -1473,9 +1469,9 @@ func (b *InMemoryBackend) ListAllPolicies() []Policy { b.mu.RLock("ListAllPolicies") defer b.mu.RUnlock() - policies := make([]Policy, 0, len(b.policies)) - for _, p := range b.policies { - policies = append(policies, p) + policies := make([]Policy, 0, b.policies.Len()) + for _, p := range b.policies.All() { + policies = append(policies, *p) } sort.Slice( @@ -1491,9 +1487,9 @@ func (b *InMemoryBackend) ListAllGroups() []Group { b.mu.RLock("ListAllGroups") defer b.mu.RUnlock() - groups := make([]Group, 0, len(b.groups)) - for _, g := range b.groups { - groups = append(groups, g) + groups := make([]Group, 0, b.groups.Len()) + for _, g := range b.groups.All() { + groups = append(groups, *g) } sort.Slice(groups, func(i, j int) bool { return groups[i].GroupName < groups[j].GroupName }) @@ -1506,9 +1502,9 @@ func (b *InMemoryBackend) ListAllAccessKeys() []AccessKey { b.mu.RLock("ListAllAccessKeys") defer b.mu.RUnlock() - keys := make([]AccessKey, 0, len(b.accessKeys)) - for _, ak := range b.accessKeys { - keys = append(keys, ak) + keys := make([]AccessKey, 0, b.accessKeys.Len()) + for _, ak := range b.accessKeys.All() { + keys = append(keys, *ak) } sort.Slice(keys, func(i, j int) bool { return keys[i].AccessKeyID < keys[j].AccessKeyID }) @@ -1521,9 +1517,9 @@ func (b *InMemoryBackend) ListAllInstanceProfiles() []InstanceProfile { b.mu.RLock("ListAllInstanceProfiles") defer b.mu.RUnlock() - profiles := make([]InstanceProfile, 0, len(b.instanceProfiles)) - for _, ip := range b.instanceProfiles { - profiles = append(profiles, ip) + profiles := make([]InstanceProfile, 0, b.instanceProfiles.Len()) + for _, ip := range b.instanceProfiles.All() { + profiles = append(profiles, *ip) } sort.Slice(profiles, func(i, j int) bool { @@ -1535,13 +1531,16 @@ func (b *InMemoryBackend) ListAllInstanceProfiles() []InstanceProfile { // ---- Helpers ---- -func sortedUsers(m map[string]User) []User { - users := make([]User, 0, len(m)) - for _, u := range m { - users = append(users, u) - } +func sortedUsers(t *store.Table[User]) []User { + // t.Snapshot() is already ordered by key ascending, and the key is + // UserName (see usersKeyFn), so this matches the prior sort.Slice by + // UserName exactly. + items := t.Snapshot() + users := make([]User, 0, len(items)) - sort.Slice(users, func(i, j int) bool { return users[i].UserName < users[j].UserName }) + for _, u := range items { + users = append(users, *u) + } return users } @@ -1616,7 +1615,7 @@ func (b *InMemoryBackend) ListAttachedUserPolicies(userName string) ([]AttachedP b.mu.RLock("ListAttachedUserPolicies") defer b.mu.RUnlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -1636,7 +1635,7 @@ func (b *InMemoryBackend) ListAttachedRolePolicies(roleName string) ([]AttachedP b.mu.RLock("ListAttachedRolePolicies") defer b.mu.RUnlock() - if _, exists := b.roles[roleName]; !exists { + if _, exists := b.roles.Get(roleName); !exists { return nil, fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } @@ -1720,17 +1719,17 @@ func (b *InMemoryBackend) GetUserByAccessKeyID(accessKeyID string) (*User, error b.mu.RLock("GetUserByAccessKeyID") defer b.mu.RUnlock() - ak, exists := b.accessKeys[accessKeyID] + ak, exists := b.accessKeys.Get(accessKeyID) if !exists { return nil, fmt.Errorf("%w: access key %q not found", ErrAccessKeyNotFound, accessKeyID) } - u, exists := b.users[ak.UserName] + u, exists := b.users.Get(ak.UserName) if !exists { return nil, fmt.Errorf("%w: user %q not found for access key", ErrUserNotFound, ak.UserName) } - return &u, nil + return u, nil } // GetPoliciesForUser returns the policy documents for all policies attached to the named user. @@ -1739,7 +1738,7 @@ func (b *InMemoryBackend) GetPoliciesForUser(userName string) ([]string, error) b.mu.RLock("GetPoliciesForUser") defer b.mu.RUnlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -1765,7 +1764,7 @@ func (b *InMemoryBackend) PutUserPolicy(userName, policyName, policyDocument str b.mu.Lock("PutUserPolicy") defer b.mu.Unlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -1792,7 +1791,7 @@ func (b *InMemoryBackend) GetUserPolicy(userName, policyName string) (string, er b.mu.RLock("GetUserPolicy") defer b.mu.RUnlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return "", fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -1814,7 +1813,7 @@ func (b *InMemoryBackend) DeleteUserPolicy(userName, policyName string) error { b.mu.Lock("DeleteUserPolicy") defer b.mu.Unlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -1837,7 +1836,7 @@ func (b *InMemoryBackend) ListUserPolicies(userName string) ([]string, error) { b.mu.RLock("ListUserPolicies") defer b.mu.RUnlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -1851,7 +1850,7 @@ func (b *InMemoryBackend) PutRolePolicy(roleName, policyName, policyDocument str b.mu.Lock("PutRolePolicy") defer b.mu.Unlock() - if _, exists := b.roles[roleName]; !exists { + if _, exists := b.roles.Get(roleName); !exists { return fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } @@ -1878,7 +1877,7 @@ func (b *InMemoryBackend) GetRolePolicy(roleName, policyName string) (string, er b.mu.RLock("GetRolePolicy") defer b.mu.RUnlock() - if _, exists := b.roles[roleName]; !exists { + if _, exists := b.roles.Get(roleName); !exists { return "", fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } @@ -1900,7 +1899,7 @@ func (b *InMemoryBackend) DeleteRolePolicy(roleName, policyName string) error { b.mu.Lock("DeleteRolePolicy") defer b.mu.Unlock() - if _, exists := b.roles[roleName]; !exists { + if _, exists := b.roles.Get(roleName); !exists { return fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } @@ -1923,7 +1922,7 @@ func (b *InMemoryBackend) ListRolePolicies(roleName string) ([]string, error) { b.mu.RLock("ListRolePolicies") defer b.mu.RUnlock() - if _, exists := b.roles[roleName]; !exists { + if _, exists := b.roles.Get(roleName); !exists { return nil, fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } @@ -1937,7 +1936,7 @@ func (b *InMemoryBackend) PutGroupPolicy(groupName, policyName, policyDocument s b.mu.Lock("PutGroupPolicy") defer b.mu.Unlock() - if _, exists := b.groups[groupName]; !exists { + if _, exists := b.groups.Get(groupName); !exists { return fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } @@ -1964,7 +1963,7 @@ func (b *InMemoryBackend) GetGroupPolicy(groupName, policyName string) (string, b.mu.RLock("GetGroupPolicy") defer b.mu.RUnlock() - if _, exists := b.groups[groupName]; !exists { + if _, exists := b.groups.Get(groupName); !exists { return "", fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } @@ -1986,7 +1985,7 @@ func (b *InMemoryBackend) DeleteGroupPolicy(groupName, policyName string) error b.mu.Lock("DeleteGroupPolicy") defer b.mu.Unlock() - if _, exists := b.groups[groupName]; !exists { + if _, exists := b.groups.Get(groupName); !exists { return fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } @@ -2009,7 +2008,7 @@ func (b *InMemoryBackend) ListGroupPolicies(groupName string) ([]string, error) b.mu.RLock("ListGroupPolicies") defer b.mu.RUnlock() - if _, exists := b.groups[groupName]; !exists { + if _, exists := b.groups.Get(groupName); !exists { return nil, fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } @@ -2025,13 +2024,13 @@ func (b *InMemoryBackend) PutUserPermissionsBoundary(userName, policyArn string) b.mu.Lock("PutUserPermissionsBoundary") defer b.mu.Unlock() - u, exists := b.users[userName] + u, exists := b.users.Get(userName) if !exists { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } u.PermissionsBoundary = policyArn - b.users[userName] = u + b.users.Put(u) return nil } @@ -2041,13 +2040,13 @@ func (b *InMemoryBackend) DeleteUserPermissionsBoundary(userName string) error { b.mu.Lock("DeleteUserPermissionsBoundary") defer b.mu.Unlock() - u, exists := b.users[userName] + u, exists := b.users.Get(userName) if !exists { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } u.PermissionsBoundary = "" - b.users[userName] = u + b.users.Put(u) return nil } @@ -2057,13 +2056,13 @@ func (b *InMemoryBackend) PutRolePermissionsBoundary(roleName, policyArn string) b.mu.Lock("PutRolePermissionsBoundary") defer b.mu.Unlock() - r, exists := b.roles[roleName] + r, exists := b.roles.Get(roleName) if !exists { return fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } r.PermissionsBoundary = policyArn - b.roles[roleName] = r + b.roles.Put(r) return nil } @@ -2073,13 +2072,13 @@ func (b *InMemoryBackend) DeleteRolePermissionsBoundary(roleName string) error { b.mu.Lock("DeleteRolePermissionsBoundary") defer b.mu.Unlock() - r, exists := b.roles[roleName] + r, exists := b.roles.Get(roleName) if !exists { return fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } r.PermissionsBoundary = "" - b.roles[roleName] = r + b.roles.Put(r) return nil } @@ -2091,7 +2090,7 @@ func (b *InMemoryBackend) UpdateAssumeRolePolicy(roleName, policyDocument string b.mu.Lock("UpdateAssumeRolePolicy") defer b.mu.Unlock() - r, exists := b.roles[roleName] + r, exists := b.roles.Get(roleName) if !exists { return fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } @@ -2108,7 +2107,7 @@ func (b *InMemoryBackend) UpdateAssumeRolePolicy(roleName, policyDocument string } r.AssumeRolePolicyDocument = policyDocument - b.roles[roleName] = r + b.roles.Put(r) return nil } @@ -2192,7 +2191,7 @@ func (b *InMemoryBackend) GetAccountAuthorizationDetails() AccountAuthorizationD defer b.mu.RUnlock() // Build reverse group-membership map: userName → []groupName. - userGroupMap := make(map[string][]string, len(b.users)) + userGroupMap := make(map[string][]string, b.users.Len()) for groupName, members := range b.groupMembers { for _, member := range members { userGroupMap[member] = append(userGroupMap[member], groupName) @@ -2200,9 +2199,9 @@ func (b *InMemoryBackend) GetAccountAuthorizationDetails() AccountAuthorizationD } // Build user details. - users := make([]UserDetail, 0, len(b.users)) - for _, u := range b.users { - user := u + users := make([]UserDetail, 0, b.users.Len()) + for _, u := range b.users.All() { + user := *u attached := attachedFromARNs(b.userPolicies[u.UserName]) inline := inlineEntries(b.userInlinePolicies[u.UserName]) groupNames := userGroupMap[u.UserName] @@ -2216,9 +2215,9 @@ func (b *InMemoryBackend) GetAccountAuthorizationDetails() AccountAuthorizationD sort.Slice(users, func(i, j int) bool { return users[i].UserName < users[j].UserName }) // Build group details. - groups := make([]GroupDetail, 0, len(b.groups)) - for _, g := range b.groups { - group := g + groups := make([]GroupDetail, 0, b.groups.Len()) + for _, g := range b.groups.All() { + group := *g attached := attachedFromARNs(b.groupPolicies[g.GroupName]) inline := inlineEntries(b.groupInlinePolicies[g.GroupName]) groups = append( @@ -2230,9 +2229,9 @@ func (b *InMemoryBackend) GetAccountAuthorizationDetails() AccountAuthorizationD sort.Slice(groups, func(i, j int) bool { return groups[i].GroupName < groups[j].GroupName }) // Build role details. - roles := make([]RoleDetail, 0, len(b.roles)) - for _, r := range b.roles { - role := r + roles := make([]RoleDetail, 0, b.roles.Len()) + for _, r := range b.roles.All() { + role := *r attached := attachedFromARNs(b.rolePolicies[r.RoleName]) inline := inlineEntries(b.roleInlinePolicies[r.RoleName]) roles = append( @@ -2244,9 +2243,9 @@ func (b *InMemoryBackend) GetAccountAuthorizationDetails() AccountAuthorizationD sort.Slice(roles, func(i, j int) bool { return roles[i].RoleName < roles[j].RoleName }) // Build managed policy list. - policies := make([]Policy, 0, len(b.policies)) - for _, p := range b.policies { - policies = append(policies, p) + policies := make([]Policy, 0, b.policies.Len()) + for _, p := range b.policies.All() { + policies = append(policies, *p) } sort.Slice( @@ -2494,7 +2493,7 @@ func (b *InMemoryBackend) collectNamedPrincipalPolicies( idx := strings.LastIndex(principalArn, userPrefix) userName := principalArn[idx+len(userPrefix):] - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -2523,7 +2522,7 @@ func (b *InMemoryBackend) collectNamedPrincipalPolicies( idx := strings.LastIndex(principalArn, rolePrefix) roleName := principalArn[idx+len(rolePrefix):] - if _, exists := b.roles[roleName]; !exists { + if _, exists := b.roles.Get(roleName); !exists { return nil, fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } @@ -2555,7 +2554,7 @@ func (b *InMemoryBackend) collectNamedEntityPolicies( continue } - p, ok := b.policies[polName] + p, ok := b.policies.Get(polName) if ok && p.PolicyDocument != "" { named = append(named, namedPolicyDoc{SourceID: p.Arn, Doc: p.PolicyDocument}) } @@ -2576,19 +2575,11 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.users = make(map[string]User) - b.roles = make(map[string]Role) + b.registry.ResetAll() b.roleByARN = make(map[string]string) - b.policies = make(map[string]Policy) b.policyByARN = make(map[string]string) b.policyAttachments = make(map[string]policyAttachmentRefs) - b.groups = make(map[string]Group) - b.accessKeys = make(map[string]AccessKey) b.userAccessKeys = make(map[string][]string) - b.instanceProfiles = make(map[string]InstanceProfile) - b.samlProviders = make(map[string]SAMLProvider) - b.oidcProviders = make(map[string]OIDCProvider) - b.loginProfiles = make(map[string]LoginProfile) b.userPolicies = make(map[string][]string) b.rolePolicies = make(map[string][]string) b.groupPolicies = make(map[string][]string) @@ -2600,11 +2591,6 @@ func (b *InMemoryBackend) Reset() { b.policyVersions = make(map[string][]StoredPolicyVersion) b.policyVersionCounters = make(map[string]int) b.deletedV1Policies = make(map[string]bool) - b.serviceSpecificCreds = make(map[string]ServiceSpecificCredential) - b.virtualMFADevices = make(map[string]VirtualMFADevice) - b.signingCertificates = make(map[string]SigningCertificate) - b.serverCertificates = make(map[string]ServerCertificate) - b.delegationRequests = make(map[string]DelegationRequest) b.sortedUserNames = nil b.sortedRoleNames = nil b.sortedPolicyNames = nil @@ -2636,12 +2622,13 @@ func (b *InMemoryBackend) Purge(ctx context.Context, cutoff time.Time) { // purgeUsersLocked removes users created before cutoff and cleans up associated data. // Caller must hold b.mu. func (b *InMemoryBackend) purgeUsersLocked(cutoff time.Time) { - for name, u := range b.users { + for _, u := range b.users.All() { if !u.CreateDate.Before(cutoff) { continue } - delete(b.users, name) - delete(b.loginProfiles, name) + name := u.UserName + b.users.Delete(name) + b.loginProfiles.Delete(name) delete(b.userPolicies, name) delete(b.userInlinePolicies, name) b.removeUserFromGroupsLocked(name) @@ -2665,9 +2652,10 @@ func (b *InMemoryBackend) removeUserFromGroupsLocked(userName string) { // purgeRolesLocked removes roles created before cutoff. // Caller must hold b.mu. func (b *InMemoryBackend) purgeRolesLocked(cutoff time.Time) { - for name, r := range b.roles { + for _, r := range b.roles.All() { if r.CreateDate.Before(cutoff) { - delete(b.roles, name) + name := r.RoleName + b.roles.Delete(name) delete(b.rolePolicies, name) delete(b.roleInlinePolicies, name) } @@ -2677,9 +2665,9 @@ func (b *InMemoryBackend) purgeRolesLocked(cutoff time.Time) { // purgePoliciesLocked removes policies created before cutoff. // Caller must hold b.mu. func (b *InMemoryBackend) purgePoliciesLocked(cutoff time.Time) { - for name, p := range b.policies { + for _, p := range b.policies.All() { if p.CreateDate.Before(cutoff) { - delete(b.policies, name) + b.policies.Delete(p.PolicyName) } } } @@ -2687,9 +2675,10 @@ func (b *InMemoryBackend) purgePoliciesLocked(cutoff time.Time) { // purgeGroupsLocked removes groups created before cutoff. // Caller must hold b.mu. func (b *InMemoryBackend) purgeGroupsLocked(cutoff time.Time) { - for name, g := range b.groups { + for _, g := range b.groups.All() { if g.CreateDate.Before(cutoff) { - delete(b.groups, name) + name := g.GroupName + b.groups.Delete(name) delete(b.groupPolicies, name) delete(b.groupInlinePolicies, name) delete(b.groupMembers, name) @@ -2700,9 +2689,9 @@ func (b *InMemoryBackend) purgeGroupsLocked(cutoff time.Time) { // purgeAccessKeysLocked removes access keys created before cutoff. // Caller must hold b.mu. func (b *InMemoryBackend) purgeAccessKeysLocked(cutoff time.Time) { - for id, ak := range b.accessKeys { + for _, ak := range b.accessKeys.All() { if ak.CreateDate.Before(cutoff) { - delete(b.accessKeys, id) + b.accessKeys.Delete(ak.AccessKeyID) } } } @@ -2710,9 +2699,9 @@ func (b *InMemoryBackend) purgeAccessKeysLocked(cutoff time.Time) { // purgeInstanceProfilesLocked removes instance profiles created before cutoff. // Caller must hold b.mu. func (b *InMemoryBackend) purgeInstanceProfilesLocked(cutoff time.Time) { - for name, ip := range b.instanceProfiles { + for _, ip := range b.instanceProfiles.All() { if ip.CreateDate.Before(cutoff) { - delete(b.instanceProfiles, name) + b.instanceProfiles.Delete(ip.InstanceProfileName) } } } @@ -2720,9 +2709,9 @@ func (b *InMemoryBackend) purgeInstanceProfilesLocked(cutoff time.Time) { // purgeSAMLProvidersLocked removes SAML providers created before cutoff. // Caller must hold b.mu. func (b *InMemoryBackend) purgeSAMLProvidersLocked(cutoff time.Time) { - for arnStr, p := range b.samlProviders { + for _, p := range b.samlProviders.All() { if p.CreateDate.Before(cutoff) { - delete(b.samlProviders, arnStr) + b.samlProviders.Delete(p.Arn) } } } @@ -2730,9 +2719,9 @@ func (b *InMemoryBackend) purgeSAMLProvidersLocked(cutoff time.Time) { // purgeOIDCProvidersLocked removes OIDC providers created before cutoff. // Caller must hold b.mu. func (b *InMemoryBackend) purgeOIDCProvidersLocked(cutoff time.Time) { - for arnStr, p := range b.oidcProviders { + for _, p := range b.oidcProviders.All() { if p.CreateDate.Before(cutoff) { - delete(b.oidcProviders, arnStr) + b.oidcProviders.Delete(p.Arn) } } } diff --git a/services/iam/backend_accuracy.go b/services/iam/backend_accuracy.go index 3b90a69b8..bf1a53070 100644 --- a/services/iam/backend_accuracy.go +++ b/services/iam/backend_accuracy.go @@ -23,7 +23,7 @@ func (b *InMemoryBackend) TagUser(userName string, tags map[string]string) error b.mu.Lock("TagUser") defer b.mu.Unlock() - u, exists := b.users[userName] + u, exists := b.users.Get(userName) if !exists { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -33,7 +33,7 @@ func (b *InMemoryBackend) TagUser(userName string, tags map[string]string) error } maps.Copy(u.Tags, tags) - b.users[userName] = u + b.users.Put(u) return nil } @@ -43,7 +43,7 @@ func (b *InMemoryBackend) UntagUser(userName string, keys []string) error { b.mu.Lock("UntagUser") defer b.mu.Unlock() - u, exists := b.users[userName] + u, exists := b.users.Get(userName) if !exists { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -52,7 +52,7 @@ func (b *InMemoryBackend) UntagUser(userName string, keys []string) error { delete(u.Tags, k) } - b.users[userName] = u + b.users.Put(u) return nil } @@ -62,7 +62,7 @@ func (b *InMemoryBackend) TagRole(roleName string, tags map[string]string) error b.mu.Lock("TagRole") defer b.mu.Unlock() - r, exists := b.roles[roleName] + r, exists := b.roles.Get(roleName) if !exists { return fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } @@ -73,7 +73,7 @@ func (b *InMemoryBackend) TagRole(roleName string, tags map[string]string) error maps.Copy(r.Tags, tags) - b.roles[roleName] = r + b.roles.Put(r) return nil } @@ -83,7 +83,7 @@ func (b *InMemoryBackend) UntagRole(roleName string, keys []string) error { b.mu.Lock("UntagRole") defer b.mu.Unlock() - r, exists := b.roles[roleName] + r, exists := b.roles.Get(roleName) if !exists { return fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } @@ -92,7 +92,7 @@ func (b *InMemoryBackend) UntagRole(roleName string, keys []string) error { delete(r.Tags, k) } - b.roles[roleName] = r + b.roles.Put(r) return nil } @@ -107,7 +107,7 @@ func (b *InMemoryBackend) TagPolicy(policyArn string, tags map[string]string) er return fmt.Errorf("%w: policy %q not found", ErrPolicyNotFound, policyArn) } - p, exists := b.policies[polName] + p, exists := b.policies.Get(polName) if !exists { return fmt.Errorf("%w: policy %q not found", ErrPolicyNotFound, policyArn) } @@ -117,7 +117,7 @@ func (b *InMemoryBackend) TagPolicy(policyArn string, tags map[string]string) er } maps.Copy(p.Tags, tags) - b.policies[polName] = p + b.policies.Put(p) return nil } @@ -132,7 +132,7 @@ func (b *InMemoryBackend) UntagPolicy(policyArn string, keys []string) error { return fmt.Errorf("%w: policy %q not found", ErrPolicyNotFound, policyArn) } - p, exists := b.policies[polName] + p, exists := b.policies.Get(polName) if !exists { return fmt.Errorf("%w: policy %q not found", ErrPolicyNotFound, policyArn) } @@ -141,7 +141,7 @@ func (b *InMemoryBackend) UntagPolicy(policyArn string, keys []string) error { delete(p.Tags, k) } - b.policies[polName] = p + b.policies.Put(p) return nil } @@ -151,7 +151,7 @@ func (b *InMemoryBackend) TagGroup(groupName string, tags map[string]string) err b.mu.Lock("TagGroup") defer b.mu.Unlock() - g, exists := b.groups[groupName] + g, exists := b.groups.Get(groupName) if !exists { return fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } @@ -161,7 +161,7 @@ func (b *InMemoryBackend) TagGroup(groupName string, tags map[string]string) err } maps.Copy(g.Tags, tags) - b.groups[groupName] = g + b.groups.Put(g) return nil } @@ -171,7 +171,7 @@ func (b *InMemoryBackend) UntagGroup(groupName string, keys []string) error { b.mu.Lock("UntagGroup") defer b.mu.Unlock() - g, exists := b.groups[groupName] + g, exists := b.groups.Get(groupName) if !exists { return fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } @@ -180,7 +180,7 @@ func (b *InMemoryBackend) UntagGroup(groupName string, keys []string) error { delete(g.Tags, k) } - b.groups[groupName] = g + b.groups.Put(g) return nil } @@ -193,7 +193,7 @@ func (b *InMemoryBackend) RecordAccessKeyUsage(accessKeyID, region, serviceName b.mu.Lock("RecordAccessKeyUsage") defer b.mu.Unlock() - ak, exists := b.accessKeys[accessKeyID] + ak, exists := b.accessKeys.Get(accessKeyID) if !exists { return } @@ -202,7 +202,7 @@ func (b *InMemoryBackend) RecordAccessKeyUsage(accessKeyID, region, serviceName ak.LastUsedDate = &now ak.LastUsedRegion = region ak.LastUsedServiceName = serviceName - b.accessKeys[accessKeyID] = ak + b.accessKeys.Put(ak) } // ---- Signing Certificates ---- @@ -226,7 +226,7 @@ func (b *InMemoryBackend) UploadSigningCertificate(userName, body string) (*Sign b.mu.Lock("UploadSigningCertificate") defer b.mu.Unlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -242,7 +242,7 @@ func (b *InMemoryBackend) UploadSigningCertificate(userName, body string) (*Sign UploadDate: time.Now().UTC(), } - b.signingCertificates[cert.CertificateID] = cert + b.signingCertificates.Put(&cert) return &cert, nil } @@ -254,16 +254,16 @@ func (b *InMemoryBackend) ListSigningCertificates(userName string) ([]SigningCer defer b.mu.RUnlock() if userName != "" { - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } } var result []SigningCertificate - for _, cert := range b.signingCertificates { + for _, cert := range b.signingCertificates.All() { if userName == "" || cert.UserName == userName { - result = append(result, cert) + result = append(result, *cert) } } @@ -275,7 +275,7 @@ func (b *InMemoryBackend) UpdateSigningCertificate(certificateID, status string) b.mu.Lock("UpdateSigningCertificate") defer b.mu.Unlock() - cert, exists := b.signingCertificates[certificateID] + cert, exists := b.signingCertificates.Get(certificateID) if !exists { return fmt.Errorf("%w: signing certificate %q not found", ErrAccessKeyNotFound, certificateID) } @@ -290,7 +290,7 @@ func (b *InMemoryBackend) UpdateSigningCertificate(certificateID, status string) } cert.Status = status - b.signingCertificates[certificateID] = cert + b.signingCertificates.Put(cert) return nil } @@ -300,11 +300,11 @@ func (b *InMemoryBackend) DeleteSigningCertificate(certificateID string) error { b.mu.Lock("DeleteSigningCertificate") defer b.mu.Unlock() - if _, exists := b.signingCertificates[certificateID]; !exists { + if _, exists := b.signingCertificates.Get(certificateID); !exists { return fmt.Errorf("%w: signing certificate %q not found", ErrAccessKeyNotFound, certificateID) } - delete(b.signingCertificates, certificateID) + b.signingCertificates.Delete(certificateID) return nil } @@ -317,13 +317,13 @@ func (b *InMemoryBackend) setMFADeviceStatus(serialNumber, status string) error b.mu.Lock("setMFADeviceStatus") defer b.mu.Unlock() - dev, exists := b.virtualMFADevices[serialNumber] + dev, exists := b.virtualMFADevices.Get(serialNumber) if !exists { return fmt.Errorf("%w: virtual MFA device %q not found", ErrInvalidAction, serialNumber) } dev.Status = status - b.virtualMFADevices[serialNumber] = dev + b.virtualMFADevices.Put(dev) return nil } @@ -333,7 +333,7 @@ func (b *InMemoryBackend) setMFADeviceStatus(serialNumber, status string) error // boundaryDocForUser returns the policy document for the user's permissions boundary, or "". // Caller must hold b.mu read-locked. func (b *InMemoryBackend) boundaryDocForUser(userName string) string { - u, exists := b.users[userName] + u, exists := b.users.Get(userName) if !exists || u.PermissionsBoundary == "" { return "" } @@ -344,7 +344,7 @@ func (b *InMemoryBackend) boundaryDocForUser(userName string) string { // boundaryDocForRole returns the policy document for the role's permissions boundary, or "". // Caller must hold b.mu read-locked. func (b *InMemoryBackend) boundaryDocForRole(roleName string) string { - r, exists := b.roles[roleName] + r, exists := b.roles.Get(roleName) if !exists || r.PermissionsBoundary == "" { return "" } @@ -360,7 +360,7 @@ func (b *InMemoryBackend) policyDocByARNLocked(policyArn string) string { return "" } - p, exists := b.policies[polName] + p, exists := b.policies.Get(polName) if !exists { return "" } @@ -637,11 +637,11 @@ func (b *InMemoryBackend) validateAWSPrincipal(a string, i int) error { func (b *InMemoryBackend) validatePrincipalResource(resource string) error { if userName, ok := strings.CutPrefix(resource, "user/"); ok { - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return fmt.Errorf("%w: user %q not found", ErrMalformedPolicyDocument, userName) } } else if roleName, okRole := strings.CutPrefix(resource, "role/"); okRole { - if _, exists := b.roles[roleName]; !exists { + if _, exists := b.roles.Get(roleName); !exists { return fmt.Errorf("%w: role %q not found", ErrMalformedPolicyDocument, roleName) } } @@ -677,7 +677,7 @@ func (b *InMemoryBackend) UploadServerCertificate(name, path, certBody, certChai return nil, fmt.Errorf("%w: CertificateBody must not be empty", ErrMalformedPolicyDocument) } - if _, exists := b.serverCertificates[name]; exists { + if _, exists := b.serverCertificates.Get(name); exists { return nil, fmt.Errorf("%w: server certificate %q already exists", ErrUserAlreadyExists, name) } @@ -692,7 +692,7 @@ func (b *InMemoryBackend) UploadServerCertificate(name, path, certBody, certChai CertificateChain: certChain, } - b.serverCertificates[name] = cert + b.serverCertificates.Put(&cert) return &cert, nil } @@ -702,12 +702,12 @@ func (b *InMemoryBackend) GetServerCertificate(name string) (*ServerCertificate, b.mu.RLock("GetServerCertificate") defer b.mu.RUnlock() - cert, exists := b.serverCertificates[name] + cert, exists := b.serverCertificates.Get(name) if !exists { return nil, fmt.Errorf("%w: server certificate %q not found", ErrUserNotFound, name) } - return &cert, nil + return cert, nil } // ListServerCertificates returns server certificates, filtered by path prefix if non-empty. @@ -717,9 +717,9 @@ func (b *InMemoryBackend) ListServerCertificates(pathPrefix string) ([]ServerCer var result []ServerCertificate - for _, cert := range b.serverCertificates { + for _, cert := range b.serverCertificates.All() { if pathPrefix == "" || strings.HasPrefix(cert.Path, pathPrefix) { - result = append(result, cert) + result = append(result, *cert) } } @@ -735,17 +735,22 @@ func (b *InMemoryBackend) UpdateServerCertificate(name, newName, newPath string) b.mu.Lock("UpdateServerCertificate") defer b.mu.Unlock() - cert, exists := b.serverCertificates[name] + cert, exists := b.serverCertificates.Get(name) if !exists { return fmt.Errorf("%w: server certificate %q not found", ErrUserNotFound, name) } if newName != "" && newName != name { - if _, nameExists := b.serverCertificates[newName]; nameExists { + if _, nameExists := b.serverCertificates.Get(newName); nameExists { return fmt.Errorf("%w: server certificate %q already exists", ErrUserAlreadyExists, newName) } - delete(b.serverCertificates, name) + // serverCertificates is keyed by ServerCertificateName, which is + // changing here -- explicit Delete(old)+Put (rather than Put alone) + // is required so the table's key stays in sync with the mutated + // value, matching store.Table's key-is-a-pure-function-of-value + // contract (see store_setup.go). + b.serverCertificates.Delete(name) cert.ServerCertificateName = newName cert.Arn = "arn:aws:iam::" + b.accountID + ":server-certificate" + cert.Path + newName } @@ -756,7 +761,7 @@ func (b *InMemoryBackend) UpdateServerCertificate(name, newName, newPath string) cert.Arn = "arn:aws:iam::" + b.accountID + ":server-certificate" + normalizedPath + cert.ServerCertificateName } - b.serverCertificates[cert.ServerCertificateName] = cert + b.serverCertificates.Put(cert) return nil } @@ -766,11 +771,11 @@ func (b *InMemoryBackend) DeleteServerCertificate(name string) error { b.mu.Lock("DeleteServerCertificate") defer b.mu.Unlock() - if _, exists := b.serverCertificates[name]; !exists { + if _, exists := b.serverCertificates.Get(name); !exists { return fmt.Errorf("%w: server certificate %q not found", ErrUserNotFound, name) } - delete(b.serverCertificates, name) + b.serverCertificates.Delete(name) return nil } diff --git a/services/iam/backend_comprehensive.go b/services/iam/backend_comprehensive.go index 88dfe60ff..da2c92efd 100644 --- a/services/iam/backend_comprehensive.go +++ b/services/iam/backend_comprehensive.go @@ -4,6 +4,7 @@ import ( "crypto/rand" "encoding/base64" "fmt" + "maps" "sort" "strings" "sync" @@ -76,6 +77,68 @@ func newComprehensiveBackend() *comprehensiveBackend { } } +// comprehensiveSnapshot is the serializable snapshot of comprehensiveBackend +// state, embedded in backendSnapshot.Comprehensive so SSH public keys, MFA +// user links, access advisor jobs, service-last-accessed data, and org report +// jobs survive a persistence restore rather than being silently dropped. +type comprehensiveSnapshot struct { + SSHPublicKeys map[string]SSHPublicKey `json:"sshPublicKeys,omitempty"` + MFAUserLinks map[string]string `json:"mfaUserLinks,omitempty"` + AccessAdvisorJobs map[string]*accessAdvisorJob `json:"accessAdvisorJobs,omitempty"` + ServiceLastAccessed map[string]map[string]ServiceLastAccessedDetail `json:"serviceLastAccessed,omitempty"` + OrgReportJobs map[string]time.Time `json:"orgReportJobs,omitempty"` +} + +// snapshot returns a deep copy of the comprehensive backend's state for +// persistence. It locks only c.mu (never b.mu), so callers must invoke it +// outside of any InMemoryBackend.mu critical section to avoid establishing a +// new nested-lock order. +func (c *comprehensiveBackend) snapshot() comprehensiveSnapshot { + c.mu.Lock() + defer c.mu.Unlock() + + return comprehensiveSnapshot{ + SSHPublicKeys: maps.Clone(c.sshPublicKeys), + MFAUserLinks: maps.Clone(c.mfaUserLinks), + AccessAdvisorJobs: maps.Clone(c.accessAdvisorJobs), + ServiceLastAccessed: maps.Clone(c.serviceLastAccessed), + OrgReportJobs: maps.Clone(c.orgReportJobs), + } +} + +// restore replaces the comprehensive backend's state from a snapshot. Like +// snapshot, it locks only c.mu and must be called outside of any +// InMemoryBackend.mu critical section. +func (c *comprehensiveBackend) restore(snap comprehensiveSnapshot) { + c.mu.Lock() + defer c.mu.Unlock() + + c.sshPublicKeys = snap.SSHPublicKeys + if c.sshPublicKeys == nil { + c.sshPublicKeys = make(map[string]SSHPublicKey) + } + + c.mfaUserLinks = snap.MFAUserLinks + if c.mfaUserLinks == nil { + c.mfaUserLinks = make(map[string]string) + } + + c.accessAdvisorJobs = snap.AccessAdvisorJobs + if c.accessAdvisorJobs == nil { + c.accessAdvisorJobs = make(map[string]*accessAdvisorJob) + } + + c.serviceLastAccessed = snap.ServiceLastAccessed + if c.serviceLastAccessed == nil { + c.serviceLastAccessed = make(map[string]map[string]ServiceLastAccessedDetail) + } + + c.orgReportJobs = snap.OrgReportJobs + if c.orgReportJobs == nil { + c.orgReportJobs = make(map[string]time.Time) + } +} + // comp returns the comprehensiveBackend associated with this InMemoryBackend. // It is always non-nil because NewInMemoryBackendWithConfig initialises it. func (b *InMemoryBackend) comp() *comprehensiveBackend { @@ -97,7 +160,7 @@ const minSSHBodyLen = 10 // UploadSSHPublicKey stores an SSH public key for a user. func (b *InMemoryBackend) UploadSSHPublicKey(userName, body string) (*SSHPublicKey, error) { b.mu.RLock("UploadSSHPublicKey-check") - _, exists := b.users[userName] + _, exists := b.users.Get(userName) b.mu.RUnlock() if !exists { @@ -143,7 +206,7 @@ func (b *InMemoryBackend) ListSSHPublicKeys( userName, marker string, maxItems int, ) (page.Page[SSHPublicKey], error) { b.mu.RLock("ListSSHPublicKeys-check") - _, exists := b.users[userName] + _, exists := b.users.Get(userName) b.mu.RUnlock() if !exists { @@ -228,8 +291,8 @@ func computeSSHFingerprint(body string) string { // Returns an error if the device is already enabled (double-enable rejected). func (b *InMemoryBackend) EnableMFADevice(userName, serialNumber, authCode1, authCode2 string) error { b.mu.RLock("EnableMFADevice-check") - _, userExists := b.users[userName] - dev, deviceExists := b.virtualMFADevices[serialNumber] + _, userExists := b.users.Get(userName) + dev, deviceExists := b.virtualMFADevices.Get(serialNumber) b.mu.RUnlock() if !userExists { @@ -266,8 +329,8 @@ func (b *InMemoryBackend) EnableMFADevice(userName, serialNumber, authCode1, aut // Returns an error if the device is not currently enabled. func (b *InMemoryBackend) DeactivateMFADevice(userName, serialNumber string) error { b.mu.RLock("DeactivateMFADevice-check") - _, userExists := b.users[userName] - dev, deviceExists := b.virtualMFADevices[serialNumber] + _, userExists := b.users.Get(userName) + dev, deviceExists := b.virtualMFADevices.Get(serialNumber) b.mu.RUnlock() if !userExists { @@ -309,7 +372,7 @@ func (b *InMemoryBackend) ListMFADevicesForUser(userName string) ([]VirtualMFADe b.mu.RLock("ListMFADevicesForUser") defer b.mu.RUnlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -321,8 +384,8 @@ func (b *InMemoryBackend) ListMFADevicesForUser(userName string) ([]VirtualMFADe for serial, owner := range c.mfaUserLinks { if owner == userName { - if dev, ok := b.virtualMFADevices[serial]; ok { - devices = append(devices, dev) + if dev, ok := b.virtualMFADevices.Get(serial); ok { + devices = append(devices, *dev) } } } @@ -337,7 +400,7 @@ func (b *InMemoryBackend) ListMFADevicesForUser(userName string) ([]VirtualMFADe // It returns ErrUserNotFound (mapped to NoSuchEntity) when no such device exists. func (b *InMemoryBackend) GetVirtualMFADevice(serialNumber string) (VirtualMFADevice, string, error) { b.mu.RLock("GetVirtualMFADevice") - dev, exists := b.virtualMFADevices[serialNumber] + dev, exists := b.virtualMFADevices.Get(serialNumber) b.mu.RUnlock() if !exists { @@ -349,7 +412,7 @@ func (b *InMemoryBackend) GetVirtualMFADevice(serialNumber string) (VirtualMFADe owner := c.mfaUserLinks[serialNumber] c.mu.Unlock() - return dev, owner, nil + return *dev, owner, nil } // ResyncMFADevice resynchronizes the named virtual MFA device for a user. @@ -359,8 +422,8 @@ func (b *InMemoryBackend) GetVirtualMFADevice(serialNumber string) (VirtualMFADe // (NoSuchEntity) when the user or association is missing. func (b *InMemoryBackend) ResyncMFADevice(userName, serialNumber, authCode1, authCode2 string) error { b.mu.RLock("ResyncMFADevice-check") - _, userExists := b.users[userName] - _, deviceExists := b.virtualMFADevices[serialNumber] + _, userExists := b.users.Get(userName) + _, deviceExists := b.virtualMFADevices.Get(serialNumber) b.mu.RUnlock() if !userExists { @@ -482,7 +545,7 @@ func (b *InMemoryBackend) CreateVirtualMFADeviceFull( b.mu.Lock("CreateVirtualMFADeviceFull") defer b.mu.Unlock() - if _, exists := b.virtualMFADevices[serialNumber]; exists { + if _, exists := b.virtualMFADevices.Get(serialNumber); exists { return nil, fmt.Errorf("%w: virtual MFA device %q already exists", ErrUserAlreadyExists, virtualMFADeviceName) } @@ -506,7 +569,7 @@ func (b *InMemoryBackend) CreateVirtualMFADeviceFull( QRCodePNG: qrPNG, } - b.virtualMFADevices[serialNumber] = device + b.virtualMFADevices.Put(&device) return &device, nil } @@ -518,20 +581,20 @@ func (b *InMemoryBackend) ResetServiceSpecificCredentialFull( b.mu.Lock("ResetServiceSpecificCredential") defer b.mu.Unlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } - cred, exists := b.serviceSpecificCreds[credentialID] + cred, exists := b.serviceSpecificCreds.Get(credentialID) if !exists || cred.UserName != userName { return nil, fmt.Errorf("%w: service-specific credential %q not found", ErrPolicyNotFound, credentialID) } // Generate a new password. cred.ServicePassword = newID("") + newID("") - b.serviceSpecificCreds[credentialID] = cred + b.serviceSpecificCreds.Put(cred) - return &cred, nil + return cred, nil } // OIDCProviderExists reports whether an OIDC provider with the given issuer URL exists. @@ -544,7 +607,7 @@ func (b *InMemoryBackend) OIDCProviderExists(issuerURL string) bool { // Normalise the issuer URL to strip trailing slashes for comparison. normalised := strings.TrimRight(issuerURL, "/") - for _, p := range b.oidcProviders { + for _, p := range b.oidcProviders.All() { providerURL := strings.TrimRight(p.URL, "/") if providerURL == normalised { return true diff --git a/services/iam/backend_new_ops.go b/services/iam/backend_new_ops.go index d1384dac1..4046a6480 100644 --- a/services/iam/backend_new_ops.go +++ b/services/iam/backend_new_ops.go @@ -52,7 +52,7 @@ func (b *InMemoryBackend) CreatePolicyVersion( return nil, fmt.Errorf("%w: policy %q not found", ErrPolicyNotFound, policyArn) } - pol, exists := b.policies[polName] + pol, exists := b.policies.Get(polName) if !exists { return nil, fmt.Errorf("%w: policy %q not found", ErrPolicyNotFound, policyArn) } @@ -97,7 +97,7 @@ func (b *InMemoryBackend) CreatePolicyVersion( pol.PolicyDocument = policyDocument pol.DefaultVersionID = versionID pol.UpdateDate = newVersion.CreateDate - b.policies[polName] = pol + b.policies.Put(pol) } versions = append(versions, newVersion) @@ -164,7 +164,7 @@ func (b *InMemoryBackend) CreateServiceLinkedRole( b.mu.Lock("CreateServiceLinkedRole") defer b.mu.Unlock() - if _, exists := b.roles[roleName]; exists { + if _, exists := b.roles.Get(roleName); exists { return nil, fmt.Errorf("%w: role %q already exists", ErrRoleAlreadyExists, roleName) } @@ -188,7 +188,7 @@ func (b *InMemoryBackend) CreateServiceLinkedRole( _ = description } - b.roles[roleName] = role + b.roles.Put(&role) b.roleByARN[role.Arn] = roleName b.sortedRoleNames = insertSorted(b.sortedRoleNames, roleName) @@ -208,7 +208,7 @@ func (b *InMemoryBackend) CreateServiceSpecificCredential( b.mu.Lock("CreateServiceSpecificCredential") defer b.mu.Unlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } @@ -226,7 +226,7 @@ func (b *InMemoryBackend) CreateServiceSpecificCredential( CreateDate: time.Now().UTC(), } - b.serviceSpecificCreds[credID] = cred + b.serviceSpecificCreds.Put(&cred) return &cred, nil } @@ -245,7 +245,7 @@ func (b *InMemoryBackend) CreateVirtualMFADevice(virtualMFADeviceName, path stri b.mu.Lock("CreateVirtualMFADevice") defer b.mu.Unlock() - if _, exists := b.virtualMFADevices[serialNumber]; exists { + if _, exists := b.virtualMFADevices.Get(serialNumber); exists { return nil, fmt.Errorf("%w: virtual MFA device %q already exists", ErrUserAlreadyExists, virtualMFADeviceName) } @@ -257,7 +257,7 @@ func (b *InMemoryBackend) CreateVirtualMFADevice(virtualMFADeviceName, path stri Status: MFAStatusNotAssigned, } - b.virtualMFADevices[serialNumber] = device + b.virtualMFADevices.Put(&device) return &device, nil } @@ -278,7 +278,7 @@ func (b *InMemoryBackend) CreateDelegationRequest(targetAccountID string) (*Dele CreateDate: time.Now().UTC(), } - b.delegationRequests[delegationID] = req + b.delegationRequests.Put(&req) return &req, nil } @@ -288,13 +288,13 @@ func (b *InMemoryBackend) AcceptDelegationRequest(delegationID string) error { b.mu.Lock("AcceptDelegationRequest") defer b.mu.Unlock() - req, exists := b.delegationRequests[delegationID] + req, exists := b.delegationRequests.Get(delegationID) if !exists { return fmt.Errorf("%w: delegation request %q not found", ErrInvalidAction, delegationID) } req.Status = "ACCEPTED" - b.delegationRequests[delegationID] = req + b.delegationRequests.Put(req) return nil } @@ -304,13 +304,13 @@ func (b *InMemoryBackend) AssociateDelegationRequest(delegationID, policyArn str b.mu.Lock("AssociateDelegationRequest") defer b.mu.Unlock() - req, exists := b.delegationRequests[delegationID] + req, exists := b.delegationRequests.Get(delegationID) if !exists { return fmt.Errorf("%w: delegation request %q not found", ErrInvalidAction, delegationID) } req.PolicyArn = policyArn - b.delegationRequests[delegationID] = req + b.delegationRequests.Put(req) return nil } @@ -385,14 +385,14 @@ func (b *InMemoryBackend) AddClientIDToOpenIDConnectProvider(providerArn, client b.mu.Lock("AddClientIDToOpenIDConnectProvider") defer b.mu.Unlock() - p, exists := b.oidcProviders[providerArn] + p, exists := b.oidcProviders.Get(providerArn) if !exists { return fmt.Errorf("%w: OIDC provider %q not found", ErrOIDCProviderNotFound, providerArn) } if !slices.Contains(p.ClientIDList, clientID) { p.ClientIDList = append(p.ClientIDList, clientID) - b.oidcProviders[providerArn] = p + b.oidcProviders.Put(p) } return nil diff --git a/services/iam/backend_providers.go b/services/iam/backend_providers.go index 92af9d354..c4913fb54 100644 --- a/services/iam/backend_providers.go +++ b/services/iam/backend_providers.go @@ -25,7 +25,7 @@ func (b *InMemoryBackend) CreateSAMLProvider(name, samlMetadataDocument string) return nil, err } - if _, exists := b.samlProviders[providerArn]; exists { + if _, exists := b.samlProviders.Get(providerArn); exists { return nil, fmt.Errorf("%w: SAML provider %q already exists", ErrSAMLProviderAlreadyExists, name) } @@ -34,7 +34,7 @@ func (b *InMemoryBackend) CreateSAMLProvider(name, samlMetadataDocument string) SAMLMetadataDocument: samlMetadataDocument, CreateDate: time.Now().UTC(), } - b.samlProviders[providerArn] = p + b.samlProviders.Put(&p) return &p, nil } @@ -44,7 +44,7 @@ func (b *InMemoryBackend) UpdateSAMLProvider(providerArn, samlMetadataDocument s b.mu.Lock("UpdateSAMLProvider") defer b.mu.Unlock() - p, exists := b.samlProviders[providerArn] + p, exists := b.samlProviders.Get(providerArn) if !exists { return nil, fmt.Errorf("%w: SAML provider %q not found", ErrSAMLProviderNotFound, providerArn) } @@ -54,9 +54,9 @@ func (b *InMemoryBackend) UpdateSAMLProvider(providerArn, samlMetadataDocument s } p.SAMLMetadataDocument = samlMetadataDocument - b.samlProviders[providerArn] = p + b.samlProviders.Put(p) - return &p, nil + return p, nil } // DeleteSAMLProvider removes a SAML provider by ARN. @@ -64,11 +64,11 @@ func (b *InMemoryBackend) DeleteSAMLProvider(providerArn string) error { b.mu.Lock("DeleteSAMLProvider") defer b.mu.Unlock() - if _, exists := b.samlProviders[providerArn]; !exists { + if _, exists := b.samlProviders.Get(providerArn); !exists { return fmt.Errorf("%w: SAML provider %q not found", ErrSAMLProviderNotFound, providerArn) } - delete(b.samlProviders, providerArn) + b.samlProviders.Delete(providerArn) return nil } @@ -78,12 +78,12 @@ func (b *InMemoryBackend) GetSAMLProvider(providerArn string) (*SAMLProvider, er b.mu.RLock("GetSAMLProvider") defer b.mu.RUnlock() - p, exists := b.samlProviders[providerArn] + p, exists := b.samlProviders.Get(providerArn) if !exists { return nil, fmt.Errorf("%w: SAML provider %q not found", ErrSAMLProviderNotFound, providerArn) } - return &p, nil + return p, nil } // ListSAMLProviders returns all SAML providers sorted by ARN. @@ -91,9 +91,9 @@ func (b *InMemoryBackend) ListSAMLProviders() ([]SAMLProvider, error) { b.mu.RLock("ListSAMLProviders") defer b.mu.RUnlock() - result := make([]SAMLProvider, 0, len(b.samlProviders)) - for _, p := range b.samlProviders { - result = append(result, p) + result := make([]SAMLProvider, 0, b.samlProviders.Len()) + for _, p := range b.samlProviders.All() { + result = append(result, *p) } sort.Slice(result, func(i, j int) bool { return result[i].Arn < result[j].Arn }) @@ -147,7 +147,7 @@ func (b *InMemoryBackend) CreateOpenIDConnectProvider( return nil, vErr } - if _, exists := b.oidcProviders[providerArn]; exists { + if _, exists := b.oidcProviders.Get(providerArn); exists { return nil, fmt.Errorf("%w: OIDC provider for URL %q already exists", ErrOIDCProviderAlreadyExists, rawURL) } @@ -158,7 +158,7 @@ func (b *InMemoryBackend) CreateOpenIDConnectProvider( ThumbprintList: append([]string(nil), thumbprints...), CreateDate: time.Now().UTC(), } - b.oidcProviders[providerArn] = p + b.oidcProviders.Put(&p) return &p, nil } @@ -168,7 +168,7 @@ func (b *InMemoryBackend) UpdateOpenIDConnectProviderThumbprint(providerArn stri b.mu.Lock("UpdateOpenIDConnectProviderThumbprint") defer b.mu.Unlock() - p, exists := b.oidcProviders[providerArn] + p, exists := b.oidcProviders.Get(providerArn) if !exists { return fmt.Errorf("%w: OIDC provider %q not found", ErrOIDCProviderNotFound, providerArn) } @@ -178,7 +178,7 @@ func (b *InMemoryBackend) UpdateOpenIDConnectProviderThumbprint(providerArn stri } p.ThumbprintList = append([]string(nil), thumbprints...) - b.oidcProviders[providerArn] = p + b.oidcProviders.Put(p) return nil } @@ -188,11 +188,11 @@ func (b *InMemoryBackend) DeleteOpenIDConnectProvider(providerArn string) error b.mu.Lock("DeleteOpenIDConnectProvider") defer b.mu.Unlock() - if _, exists := b.oidcProviders[providerArn]; !exists { + if _, exists := b.oidcProviders.Get(providerArn); !exists { return fmt.Errorf("%w: OIDC provider %q not found", ErrOIDCProviderNotFound, providerArn) } - delete(b.oidcProviders, providerArn) + b.oidcProviders.Delete(providerArn) return nil } @@ -202,12 +202,12 @@ func (b *InMemoryBackend) GetOpenIDConnectProvider(providerArn string) (*OIDCPro b.mu.RLock("GetOpenIDConnectProvider") defer b.mu.RUnlock() - p, exists := b.oidcProviders[providerArn] + p, exists := b.oidcProviders.Get(providerArn) if !exists { return nil, fmt.Errorf("%w: OIDC provider %q not found", ErrOIDCProviderNotFound, providerArn) } - return &p, nil + return p, nil } // ListOpenIDConnectProviders returns all OIDC providers sorted by ARN. @@ -215,9 +215,9 @@ func (b *InMemoryBackend) ListOpenIDConnectProviders() ([]OIDCProvider, error) { b.mu.RLock("ListOpenIDConnectProviders") defer b.mu.RUnlock() - result := make([]OIDCProvider, 0, len(b.oidcProviders)) - for _, p := range b.oidcProviders { - result = append(result, p) + result := make([]OIDCProvider, 0, b.oidcProviders.Len()) + for _, p := range b.oidcProviders.All() { + result = append(result, *p) } sort.Slice(result, func(i, j int) bool { return result[i].Arn < result[j].Arn }) @@ -242,11 +242,11 @@ func (b *InMemoryBackend) CreateLoginProfile( // Check resource existence before password policy so callers receive // the correct entity error (NoSuchEntity / EntityAlreadyExists) even if // the password also violates policy. - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } - if _, exists := b.loginProfiles[userName]; exists { + if _, exists := b.loginProfiles.Get(userName); exists { return nil, fmt.Errorf("%w: login profile for user %q already exists", ErrLoginProfileAlreadyExists, userName) } @@ -259,7 +259,7 @@ func (b *InMemoryBackend) CreateLoginProfile( CreateDate: time.Now().UTC(), PasswordResetRequired: passwordResetRequired, } - b.loginProfiles[userName] = lp + b.loginProfiles.Put(&lp) return &lp, nil } @@ -275,7 +275,7 @@ func (b *InMemoryBackend) UpdateLoginProfile( return fmt.Errorf("%w: password must not be empty", ErrInvalidPassword) } - lp, exists := b.loginProfiles[userName] + lp, exists := b.loginProfiles.Get(userName) if !exists { return fmt.Errorf("%w: login profile for user %q not found", ErrLoginProfileNotFound, userName) } @@ -285,7 +285,7 @@ func (b *InMemoryBackend) UpdateLoginProfile( } lp.PasswordResetRequired = passwordResetRequired - b.loginProfiles[userName] = lp + b.loginProfiles.Put(lp) return nil } @@ -295,11 +295,11 @@ func (b *InMemoryBackend) DeleteLoginProfile(userName string) error { b.mu.Lock("DeleteLoginProfile") defer b.mu.Unlock() - if _, exists := b.loginProfiles[userName]; !exists { + if _, exists := b.loginProfiles.Get(userName); !exists { return fmt.Errorf("%w: login profile for user %q not found", ErrLoginProfileNotFound, userName) } - delete(b.loginProfiles, userName) + b.loginProfiles.Delete(userName) return nil } @@ -338,10 +338,10 @@ func (b *InMemoryBackend) GetLoginProfile(userName string) (*LoginProfile, error b.mu.RLock("GetLoginProfile") defer b.mu.RUnlock() - lp, exists := b.loginProfiles[userName] + lp, exists := b.loginProfiles.Get(userName) if !exists { return nil, fmt.Errorf("%w: login profile for user %q not found", ErrLoginProfileNotFound, userName) } - return &lp, nil + return lp, nil } diff --git a/services/iam/backend_refinement.go b/services/iam/backend_refinement.go index e44a5291f..efd5402bd 100644 --- a/services/iam/backend_refinement.go +++ b/services/iam/backend_refinement.go @@ -20,13 +20,13 @@ func (b *InMemoryBackend) UpdateAccessKey(userName, accessKeyID, status string) b.mu.Lock("UpdateAccessKey") defer b.mu.Unlock() - ak, exists := b.accessKeys[accessKeyID] + ak, exists := b.accessKeys.Get(accessKeyID) if !exists || ak.UserName != userName { return fmt.Errorf("%w: access key %q not found for user %q", ErrAccessKeyNotFound, accessKeyID, userName) } ak.Status = status - b.accessKeys[accessKeyID] = ak + b.accessKeys.Put(ak) return nil } @@ -37,7 +37,7 @@ func (b *InMemoryBackend) GetAccessKeyLastUsed(accessKeyID string) (*AccessKeyLa b.mu.RLock("GetAccessKeyLastUsed") defer b.mu.RUnlock() - ak, exists := b.accessKeys[accessKeyID] + ak, exists := b.accessKeys.Get(accessKeyID) if !exists { return nil, fmt.Errorf("%w: access key %q not found", ErrAccessKeyNotFound, accessKeyID) } @@ -100,15 +100,15 @@ func (b *InMemoryBackend) ListGroupsForUser(userName string) ([]Group, error) { b.mu.RLock("ListGroupsForUser") defer b.mu.RUnlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } result := make([]Group, 0, len(b.groupMembers)) for groupName, members := range b.groupMembers { if slices.Contains(members, userName) { - if g, exists := b.groups[groupName]; exists { - result = append(result, g) + if g, exists := b.groups.Get(groupName); exists { + result = append(result, *g) } } } @@ -125,9 +125,9 @@ func (b *InMemoryBackend) ListVirtualMFADevices(marker string, maxItems int) (pa b.mu.RLock("ListVirtualMFADevices") defer b.mu.RUnlock() - devices := make([]VirtualMFADevice, 0, len(b.virtualMFADevices)) - for _, d := range b.virtualMFADevices { - devices = append(devices, d) + devices := make([]VirtualMFADevice, 0, b.virtualMFADevices.Len()) + for _, d := range b.virtualMFADevices.All() { + devices = append(devices, *d) } sort.Slice(devices, func(i, j int) bool { @@ -142,11 +142,11 @@ func (b *InMemoryBackend) DeleteVirtualMFADevice(serialNumber string) error { b.mu.Lock("DeleteVirtualMFADevice") defer b.mu.Unlock() - if _, exists := b.virtualMFADevices[serialNumber]; !exists { + if _, exists := b.virtualMFADevices.Get(serialNumber); !exists { return fmt.Errorf("%w: virtual MFA device %q not found", ErrUserNotFound, serialNumber) } - delete(b.virtualMFADevices, serialNumber) + b.virtualMFADevices.Delete(serialNumber) return nil } @@ -163,7 +163,11 @@ func (b *InMemoryBackend) SetDefaultPolicyVersion(policyArn, versionID string) e return fmt.Errorf("%w: policy %q not found", ErrPolicyNotFound, policyArn) } - pol := b.policies[polName] + pol, polExists := b.policies.Get(polName) + if !polExists { + return fmt.Errorf("%w: policy %q not found", ErrPolicyNotFound, policyArn) + } + now := time.Now().UTC() if versionID == "v1" { @@ -176,7 +180,7 @@ func (b *InMemoryBackend) SetDefaultPolicyVersion(policyArn, versionID string) e pol.DefaultVersionID = "v1" pol.UpdateDate = now - b.policies[polName] = pol + b.policies.Put(pol) return nil } @@ -193,7 +197,7 @@ func (b *InMemoryBackend) SetDefaultPolicyVersion(policyArn, versionID string) e pol.PolicyDocument = versions[i].PolicyDocument pol.DefaultVersionID = versionID pol.UpdateDate = now - b.policies[polName] = pol + b.policies.Put(pol) } else { versions[i].IsDefaultVersion = false } @@ -371,12 +375,12 @@ func (b *InMemoryBackend) ListServiceSpecificCredentials( b.mu.RLock("ListServiceSpecificCredentials") defer b.mu.RUnlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } - result := make([]ServiceSpecificCredential, 0, len(b.serviceSpecificCreds)) - for _, cred := range b.serviceSpecificCreds { + result := make([]ServiceSpecificCredential, 0, b.serviceSpecificCreds.Len()) + for _, cred := range b.serviceSpecificCreds.All() { if cred.UserName != userName { continue } @@ -385,7 +389,7 @@ func (b *InMemoryBackend) ListServiceSpecificCredentials( continue } - result = append(result, cred) + result = append(result, *cred) } sort.Slice(result, func(i, j int) bool { @@ -400,12 +404,12 @@ func (b *InMemoryBackend) DeleteServiceSpecificCredential(userName, credentialID b.mu.Lock("DeleteServiceSpecificCredential") defer b.mu.Unlock() - cred, exists := b.serviceSpecificCreds[credentialID] + cred, exists := b.serviceSpecificCreds.Get(credentialID) if !exists || cred.UserName != userName { return fmt.Errorf("%w: credential %q not found for user %q", ErrAccessKeyNotFound, credentialID, userName) } - delete(b.serviceSpecificCreds, credentialID) + b.serviceSpecificCreds.Delete(credentialID) return nil } @@ -418,13 +422,13 @@ func (b *InMemoryBackend) UpdateUser(userName, newPath, newUserName string) erro b.mu.Lock("UpdateUser") defer b.mu.Unlock() - u, ok := b.users[userName] + u, ok := b.users.Get(userName) if !ok { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } if newUserName != "" && newUserName != userName { - if _, taken := b.users[newUserName]; taken { + if _, taken := b.users.Get(newUserName); taken { return fmt.Errorf("%w: user %q already exists", ErrUserAlreadyExists, newUserName) } } @@ -436,26 +440,32 @@ func (b *InMemoryBackend) UpdateUser(userName, newPath, newUserName string) erro if newUserName != "" && newUserName != userName { b.migrateUserData(userName, newUserName) u.UserName = newUserName - delete(b.users, userName) + b.users.Delete(userName) } - b.users[u.UserName] = u + b.users.Put(u) return nil } // migrateUserData moves per-user maps to the new name (called with lock held). func (b *InMemoryBackend) migrateUserData(oldName, newName string) { - for id, ak := range b.accessKeys { + for _, ak := range b.accessKeys.All() { if ak.UserName == oldName { ak.UserName = newName - b.accessKeys[id] = ak + b.accessKeys.Put(ak) } } - if lp, found := b.loginProfiles[oldName]; found { - b.loginProfiles[newName] = lp - delete(b.loginProfiles, oldName) + // loginProfiles is keyed by UserName (see loginProfilesKeyFn in + // store_setup.go), so the rename must update the value's own UserName + // field before re-inserting -- store.Table always derives the key from + // the value, unlike the raw map this replaces which allowed the map key + // and the value's UserName field to diverge after a rename. + if lp, found := b.loginProfiles.Get(oldName); found { + b.loginProfiles.Delete(oldName) + lp.UserName = newName + b.loginProfiles.Put(lp) } if policies, found := b.userPolicies[oldName]; found { @@ -484,13 +494,13 @@ func (b *InMemoryBackend) UpdateRole(roleName, description string) error { b.mu.Lock("UpdateRole") defer b.mu.Unlock() - r, exists := b.roles[roleName] + r, exists := b.roles.Get(roleName) if !exists { return fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } r.Description = description - b.roles[roleName] = r + b.roles.Put(r) return nil } @@ -502,13 +512,13 @@ func (b *InMemoryBackend) UpdateGroup(groupName, newPath, newGroupName string) e b.mu.Lock("UpdateGroup") defer b.mu.Unlock() - g, ok := b.groups[groupName] + g, ok := b.groups.Get(groupName) if !ok { return fmt.Errorf("%w: group %q not found", ErrGroupNotFound, groupName) } if newGroupName != "" && newGroupName != groupName { - if _, taken := b.groups[newGroupName]; taken { + if _, taken := b.groups.Get(newGroupName); taken { return fmt.Errorf("%w: group %q already exists", ErrGroupAlreadyExists, newGroupName) } } @@ -520,10 +530,10 @@ func (b *InMemoryBackend) UpdateGroup(groupName, newPath, newGroupName string) e if newGroupName != "" && newGroupName != groupName { b.migrateGroupData(groupName, newGroupName) g.GroupName = newGroupName - delete(b.groups, groupName) + b.groups.Delete(groupName) } - b.groups[g.GroupName] = g + b.groups.Put(g) return nil } @@ -553,7 +563,7 @@ func (b *InMemoryBackend) RemoveClientIDFromOpenIDConnectProvider(providerArn, c b.mu.Lock("RemoveClientIDFromOpenIDConnectProvider") defer b.mu.Unlock() - p, exists := b.oidcProviders[providerArn] + p, exists := b.oidcProviders.Get(providerArn) if !exists { return fmt.Errorf("%w: OIDC provider %q not found", ErrOIDCProviderNotFound, providerArn) } @@ -561,7 +571,7 @@ func (b *InMemoryBackend) RemoveClientIDFromOpenIDConnectProvider(providerArn, c for i, id := range p.ClientIDList { if id == clientID { p.ClientIDList = append(p.ClientIDList[:i], p.ClientIDList[i+1:]...) - b.oidcProviders[providerArn] = p + b.oidcProviders.Put(p) return nil } @@ -577,12 +587,12 @@ func (b *InMemoryBackend) GetInstanceProfile(name string) (*InstanceProfile, err b.mu.RLock("GetInstanceProfile") defer b.mu.RUnlock() - ip, exists := b.instanceProfiles[name] + ip, exists := b.instanceProfiles.Get(name) if !exists { return nil, fmt.Errorf("%w: instance profile %q not found", ErrInstanceProfileNotFound, name) } - return &ip, nil + return ip, nil } // ListInstanceProfilesForRole returns all instance profiles that contain the specified role. @@ -590,14 +600,14 @@ func (b *InMemoryBackend) ListInstanceProfilesForRole(roleName string) ([]Instan b.mu.RLock("ListInstanceProfilesForRole") defer b.mu.RUnlock() - if _, exists := b.roles[roleName]; !exists { + if _, exists := b.roles.Get(roleName); !exists { return nil, fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } - result := make([]InstanceProfile, 0, len(b.instanceProfiles)) - for _, ip := range b.instanceProfiles { + result := make([]InstanceProfile, 0, b.instanceProfiles.Len()) + for _, ip := range b.instanceProfiles.All() { if slices.Contains(ip.Roles, roleName) { - result = append(result, ip) + result = append(result, *ip) } } @@ -712,7 +722,7 @@ func (b *InMemoryBackend) DeleteServiceLinkedRole(roleName string) error { b.mu.Lock("DeleteServiceLinkedRole") defer b.mu.Unlock() - if _, exists := b.roles[roleName]; !exists { + if _, exists := b.roles.Get(roleName); !exists { return fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } @@ -721,8 +731,8 @@ func (b *InMemoryBackend) DeleteServiceLinkedRole(roleName string) error { // Force-clear inline policies. delete(b.roleInlinePolicies, roleName) - role := b.roles[roleName] - delete(b.roles, roleName) + role, _ := b.roles.Get(roleName) + b.roles.Delete(roleName) delete(b.roleByARN, role.Arn) b.sortedRoleNames = deleteSorted(b.sortedRoleNames, roleName) diff --git a/services/iam/backend_refinement2.go b/services/iam/backend_refinement2.go index 6f969d999..5c3cd3ae3 100644 --- a/services/iam/backend_refinement2.go +++ b/services/iam/backend_refinement2.go @@ -9,6 +9,7 @@ import ( "time" "github.com/blackbirdworks/gopherstack/pkgs/page" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) // insertSorted inserts v into a sorted []string slice in lexicographic order. @@ -30,24 +31,13 @@ func deleteSorted(s []string, v string) []string { return slices.Delete(s, i, i+1) } -// rebuildSortedNames rebuilds a sorted name slice from the keys of a generic map. -func rebuildSortedNames[T any](m map[string]T) []string { - names := make([]string, 0, len(m)) - for k := range m { - names = append(names, k) - } - - slices.Sort(names) - - return names -} - // pageFromSortedNames paginates over a pre-sorted name slice, building the value page -// by looking up each name in the provided map. Marker is a base64-encoded integer index -// (opaque to callers) enabling O(1) position resolution without scanning all names. +// by looking up each name via lookup (typically a [store.Table.Get] method value). +// Marker is a base64-encoded integer index (opaque to callers) enabling O(1) position +// resolution without scanning all names. func pageFromSortedNames[T any]( names []string, - lookup map[string]T, + lookup func(string) (*T, bool), marker string, limit, defaultLimit int, ) page.Page[T] { @@ -73,8 +63,8 @@ func pageFromSortedNames[T any]( data := make([]T, 0, len(window)) for _, name := range window { - if v, ok := lookup[name]; ok { - data = append(data, v) + if v, ok := lookup(name); ok { + data = append(data, *v) } } @@ -97,11 +87,11 @@ func (b *InMemoryBackend) UpdateServiceSpecificCredential( b.mu.Lock("UpdateServiceSpecificCredential") defer b.mu.Unlock() - if _, exists := b.users[userName]; !exists { + if _, exists := b.users.Get(userName); !exists { return fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } - cred, exists := b.serviceSpecificCreds[credentialID] + cred, exists := b.serviceSpecificCreds.Get(credentialID) if !exists { return fmt.Errorf( "%w: service-specific credential %q not found", @@ -120,7 +110,7 @@ func (b *InMemoryBackend) UpdateServiceSpecificCredential( } cred.Status = status - b.serviceSpecificCreds[credentialID] = cred + b.serviceSpecificCreds.Put(cred) return nil } @@ -146,18 +136,18 @@ func (b *InMemoryBackend) GetAccountSummary() AccountSummary { } return AccountSummary{ - Users: len(b.users), - Groups: len(b.groups), - Roles: len(b.roles), - Policies: len(b.policies), - InstanceProfiles: len(b.instanceProfiles), - SAMLProviders: len(b.samlProviders), - MFADevices: len(b.virtualMFADevices), + Users: b.users.Len(), + Groups: b.groups.Len(), + Roles: b.roles.Len(), + Policies: b.policies.Len(), + InstanceProfiles: b.instanceProfiles.Len(), + SAMLProviders: b.samlProviders.Len(), + MFADevices: b.virtualMFADevices.Len(), AccessKeysPerUser: totalKeys, ActiveAccessKeys: activeKeys, AttachedPolicies: attachedPolicies, AccountAliases: len(b.accountAliases), - OIDCProviders: len(b.oidcProviders), + OIDCProviders: b.oidcProviders.Len(), } } @@ -166,7 +156,7 @@ func (b *InMemoryBackend) GetAccountSummary() AccountSummary { func (b *InMemoryBackend) accessKeyCountLocked() (int, int) { var total, active int - for _, ak := range b.accessKeys { + for _, ak := range b.accessKeys.All() { total++ if ak.Status == accessKeyStatusActive { active++ @@ -219,14 +209,14 @@ func credKeyFields(ak *AccessKey) []string { func credUserMFAActive( userName string, links map[string]string, - devices map[string]VirtualMFADevice, + devices *store.Table[VirtualMFADevice], ) string { for serial, owner := range links { if owner != userName { continue } - if dev, ok := devices[serial]; ok && dev.Status == MFAStatusEnabled { + if dev, ok := devices.Get(serial); ok && dev.Status == MFAStatusEnabled { return credTrue } } @@ -272,15 +262,15 @@ func (b *InMemoryBackend) credUserRow(u User, mfaLinks map[string]string) string createdAt := u.CreateDate.UTC().Format(time.RFC3339) passwordEnabled := credFalse - if _, has := b.loginProfiles[u.UserName]; has { + if _, has := b.loginProfiles.Get(u.UserName); has { passwordEnabled = credTrue } var userKeys []AccessKey userKeysList := b.userAccessKeys[u.UserName] for _, id := range userKeysList { - if ak, ok := b.accessKeys[id]; ok { - userKeys = append(userKeys, ak) + if ak, ok := b.accessKeys.Get(id); ok { + userKeys = append(userKeys, *ak) } } diff --git a/services/iam/coverage_test.go b/services/iam/coverage_test.go index ac13bc7e3..22eb9f835 100644 --- a/services/iam/coverage_test.go +++ b/services/iam/coverage_test.go @@ -452,10 +452,11 @@ func TestIAMHandler_PolicyDispatch(t *testing.T) { wantContain: "GetPolicyResponse", }, { - name: "GetPolicy_not_found", - action: "GetPolicy", - params: map[string]string{"PolicyArn": "arn:aws:iam::000000000000:policy/Ghost"}, - wantCode: http.StatusBadRequest, + name: "GetPolicy_not_found", + action: "GetPolicy", + params: map[string]string{"PolicyArn": "arn:aws:iam::000000000000:policy/Ghost"}, + // NoSuchEntity is 404 on real AWS IAM. + wantCode: http.StatusNotFound, }, { name: "GetPolicyVersion_success", diff --git a/services/iam/handler.go b/services/iam/handler.go index 2ed89b0a7..94d5a16c4 100644 --- a/services/iam/handler.go +++ b/services/iam/handler.go @@ -90,6 +90,48 @@ func (h *Handler) getTags(resourceID string) map[string]string { return t.Clone() } +// tagsSnapshot returns a deep copy of every Handler-level resource tag map, for +// persistence. Role/User/Group/Policy tags live on the entity itself and are +// captured by the backend's own Snapshot; this covers the remaining taggable +// resource kinds whose tags are tracked only on the Handler (instance +// profiles, MFA devices, SAML/OIDC providers, server certificates) — see +// setTags/getTags/removeTags above. Without this, those tags were silently +// dropped on every persistence restore. +func (h *Handler) tagsSnapshot() map[string]map[string]string { + h.tagsMu.RLock("tagsSnapshot") + defer h.tagsMu.RUnlock() + + if len(h.tags) == 0 { + return nil + } + + out := make(map[string]map[string]string, len(h.tags)) + + for resourceID, t := range h.tags { + if t == nil { + continue + } + + out[resourceID] = t.Clone() + } + + return out +} + +// restoreTags rebuilds the Handler-level tag map from a persisted snapshot. +func (h *Handler) restoreTags(snapshot map[string]map[string]string) { + h.tagsMu.Lock("restoreTags") + defer h.tagsMu.Unlock() + + h.tags = make(map[string]*svcTags.Tags, len(snapshot)) + + for resourceID, kv := range snapshot { + t := svcTags.New("iam." + resourceID + ".tags") + t.Merge(kv) + h.tags[resourceID] = t + } +} + // Name returns the service name. func (h *Handler) Name() string { return "IAM" @@ -334,7 +376,7 @@ func (h *Handler) Handler() echo.HandlerFunc { if err != nil { log.ErrorContext(ctx, "failed to read IAM request body", "error", err) - return h.writeError(c, http.StatusInternalServerError, "InternalFailure", "failed to read request body") + return h.writeError(c, http.StatusInternalServerError, "ServiceFailure", "failed to read request body") } vals, err := url.ParseQuery(string(body)) @@ -360,7 +402,7 @@ func (h *Handler) Handler() echo.HandlerFunc { if marshalErr != nil { log.ErrorContext(ctx, "failed to marshal IAM response", "action", action, "error", marshalErr) - return h.writeError(c, http.StatusInternalServerError, "InternalFailure", "internal server error") + return h.writeError(c, http.StatusInternalServerError, "ServiceFailure", "internal server error") } return c.Blob(http.StatusOK, "text/xml", xmlBytes) @@ -437,6 +479,17 @@ func (h *Handler) iamUserDispatchTable() map[string]iamActionFn { return nil, err } + // CreateUser accepts an optional Tags.member.N parameter to tag the + // user at creation time (real AWS: "A list of tags that you want to + // attach to the new user"). This was previously accepted-but-dropped. + if tags := parseIAMTags(vals); len(tags) > 0 { + if tagErr := h.Backend.TagUser(u.UserName, tags); tagErr != nil { + return nil, tagErr + } + + u.Tags = tags + } + return &CreateUserResponse{ Xmlns: iamXMLNS, CreateUserResult: CreateUserResult{User: toUserXML(u)}, @@ -486,41 +539,54 @@ func (h *Handler) iamUserDispatchTable() map[string]iamActionFn { } } -func (h *Handler) iamRoleDispatchTable() map[string]iamActionFn { - return map[string]iamActionFn{ - "CreateRole": func(vals url.Values, reqID string) (any, error) { - r, err := h.Backend.CreateRole( - vals.Get("RoleName"), - vals.Get("Path"), - vals.Get("AssumeRolePolicyDocument"), - vals.Get("PermissionsBoundary"), +// handleCreateRole implements CreateRole, including the optional +// MaxSessionDuration validation/update and Tags.member.N tagging-at-creation +// (real AWS: "A list of tags that you want to attach to the new role"). +func (h *Handler) handleCreateRole(vals url.Values, reqID string) (any, error) { + r, err := h.Backend.CreateRole( + vals.Get("RoleName"), + vals.Get("Path"), + vals.Get("AssumeRolePolicyDocument"), + vals.Get("PermissionsBoundary"), + ) + if err != nil { + return nil, err + } + + if msd := vals.Get("MaxSessionDuration"); msd != "" { + d, parseErr := strconv.ParseInt(msd, 10, 32) + if parseErr != nil || d < minMaxSessionDuration || d > maxMaxSessionDuration { + return nil, fmt.Errorf( + "%w: MaxSessionDuration must be between %d and %d", + ErrValidationError, minMaxSessionDuration, maxMaxSessionDuration, ) - if err != nil { - return nil, err - } + } - if msd := vals.Get("MaxSessionDuration"); msd != "" { - d, parseErr := strconv.ParseInt(msd, 10, 32) - if parseErr != nil || d < minMaxSessionDuration || d > maxMaxSessionDuration { - return nil, fmt.Errorf( - "%w: MaxSessionDuration must be between %d and %d", - ErrValidationError, minMaxSessionDuration, maxMaxSessionDuration, - ) - } + if updateErr := h.Backend.UpdateRoleMaxSessionDuration(r.RoleName, int32(d)); updateErr != nil { + return nil, fmt.Errorf("updating max session duration for role %s: %w", r.RoleName, updateErr) + } - if updateErr := h.Backend.UpdateRoleMaxSessionDuration(r.RoleName, int32(d)); updateErr != nil { - return nil, fmt.Errorf("updating max session duration for role %s: %w", r.RoleName, updateErr) - } + r.MaxSessionDuration = int32(d) + } - r.MaxSessionDuration = int32(d) - } + if tags := parseIAMTags(vals); len(tags) > 0 { + if tagErr := h.Backend.TagRole(r.RoleName, tags); tagErr != nil { + return nil, tagErr + } - return &CreateRoleResponse{ - Xmlns: iamXMLNS, - CreateRoleResult: CreateRoleResult{Role: toRoleXML(r)}, - ResponseMetadata: ResponseMetadata{RequestID: reqID}, - }, nil - }, + r.Tags = tags + } + + return &CreateRoleResponse{ + Xmlns: iamXMLNS, + CreateRoleResult: CreateRoleResult{Role: toRoleXML(r)}, + ResponseMetadata: ResponseMetadata{RequestID: reqID}, + }, nil +} + +func (h *Handler) iamRoleDispatchTable() map[string]iamActionFn { + return map[string]iamActionFn{ + "CreateRole": h.handleCreateRole, "GetRole": func(vals url.Values, reqID string) (any, error) { r, err := h.Backend.GetRole(vals.Get("RoleName")) if err != nil { @@ -564,22 +630,37 @@ func (h *Handler) iamRoleDispatchTable() map[string]iamActionFn { } } +// handleCreatePolicy implements CreatePolicy, including Tags.member.N +// tagging-at-creation (real AWS: "A list of tags that you want to attach to +// the new IAM customer managed policy"). The Policy wire type also lacked a +// Tags field entirely (real AWS's Policy type carries Tags), so even an +// out-of-band TagPolicy call was previously invisible on GetPolicy/ListPolicies. +func (h *Handler) handleCreatePolicy(vals url.Values, reqID string) (any, error) { + pol, err := h.Backend.CreatePolicy( + vals.Get("PolicyName"), vals.Get("Path"), vals.Get("PolicyDocument"), + ) + if err != nil { + return nil, err + } + + if tags := parseIAMTags(vals); len(tags) > 0 { + if tagErr := h.Backend.TagPolicy(pol.Arn, tags); tagErr != nil { + return nil, tagErr + } + + pol.Tags = tags + } + + return &CreatePolicyResponse{ + Xmlns: iamXMLNS, + CreatePolicyResult: CreatePolicyResult{Policy: toPolicyXML(pol)}, + ResponseMetadata: ResponseMetadata{RequestID: reqID}, + }, nil +} + func (h *Handler) iamPolicyBasicDispatchTable() map[string]iamActionFn { return map[string]iamActionFn{ - "CreatePolicy": func(vals url.Values, reqID string) (any, error) { - pol, err := h.Backend.CreatePolicy( - vals.Get("PolicyName"), vals.Get("Path"), vals.Get("PolicyDocument"), - ) - if err != nil { - return nil, err - } - - return &CreatePolicyResponse{ - Xmlns: iamXMLNS, - CreatePolicyResult: CreatePolicyResult{Policy: toPolicyXML(pol)}, - ResponseMetadata: ResponseMetadata{RequestID: reqID}, - }, nil - }, + "CreatePolicy": h.handleCreatePolicy, "DeletePolicy": func(vals url.Values, reqID string) (any, error) { if err := h.Backend.DeletePolicy(vals.Get("PolicyArn")); err != nil { return nil, err @@ -629,7 +710,7 @@ func (h *Handler) iamPolicyBasicDispatchTable() map[string]iamActionFn { return &GetPolicyVersionResponse{ Xmlns: iamXMLNS, GetPolicyVersionResult: GetPolicyVersionResult{PolicyVersion: PolicyVersionXML{ - Document: pv.PolicyDocument, + Document: encodePolicyDocument(pv.PolicyDocument), VersionID: pv.VersionID, IsDefaultVersion: pv.IsDefaultVersion, CreateDate: isoTime(pv.CreateDate), @@ -739,26 +820,32 @@ func (h *Handler) iamPolicyAttachDispatchTable() map[string]iamActionFn { ResponseMetadata: ResponseMetadata{RequestID: reqID}, }, nil }, - opListInstanceProfilesForRole: func(_ url.Values, reqID string) (any, error) { - type listInstanceProfilesResult struct { - XMLName xml.Name `xml:"ListInstanceProfilesForRoleResult"` - InstanceProfiles []any `xml:"InstanceProfiles>member"` - IsTruncated bool `xml:"IsTruncated"` - } - type listInstanceProfilesResponse struct { - XMLName xml.Name `xml:"ListInstanceProfilesForRoleResponse"` - Xmlns string `xml:"xmlns,attr"` - ResponseMetadata ResponseMetadata `xml:"ResponseMetadata"` - ListInstanceProfilesForRoleResult listInstanceProfilesResult `xml:"ListInstanceProfilesForRoleResult"` - } + opListInstanceProfilesForRole: h.handleListInstanceProfilesForRole, + } +} - return &listInstanceProfilesResponse{ - Xmlns: iamXMLNS, - ListInstanceProfilesForRoleResult: listInstanceProfilesResult{InstanceProfiles: []any{}}, - ResponseMetadata: ResponseMetadata{RequestID: reqID}, - }, nil - }, +// handleListInstanceProfilesForRole implements ListInstanceProfilesForRole, +// returning every instance profile that actually contains the given role +// (via StorageBackend.ListInstanceProfilesForRole). Previously this action +// ignored the RoleName parameter and always returned an empty list. +func (h *Handler) handleListInstanceProfilesForRole(vals url.Values, reqID string) (any, error) { + profiles, err := h.Backend.ListInstanceProfilesForRole(vals.Get("RoleName")) + if err != nil { + return nil, err } + + xmlProfiles := make([]InstanceProfileXML, 0, len(profiles)) + + for i := range profiles { + roles := h.resolveInstanceProfileRoles(&profiles[i]) + xmlProfiles = append(xmlProfiles, toInstanceProfileXML(&profiles[i], roles)) + } + + return &ListInstanceProfilesForRoleResponse{ + Xmlns: iamXMLNS, + ListInstanceProfilesForRoleResult: ListInstanceProfilesForRoleResult{InstanceProfiles: xmlProfiles}, + ResponseMetadata: ResponseMetadata{RequestID: reqID}, + }, nil } func (h *Handler) iamGroupAttachedPolicyDispatchTable() map[string]iamActionFn { @@ -834,7 +921,7 @@ func (h *Handler) iamUserRoleInlinePolicyDispatchTable() map[string]iamActionFn GetUserPolicyResult: GetUserPolicyResult{ UserName: vals.Get("UserName"), PolicyName: vals.Get("PolicyName"), - PolicyDocument: doc, + PolicyDocument: encodePolicyDocument(doc), }, ResponseMetadata: ResponseMetadata{RequestID: reqID}, }, nil @@ -878,7 +965,7 @@ func (h *Handler) iamUserRoleInlinePolicyDispatchTable() map[string]iamActionFn GetRolePolicyResult: GetRolePolicyResult{ RoleName: vals.Get("RoleName"), PolicyName: vals.Get("PolicyName"), - PolicyDocument: doc, + PolicyDocument: encodePolicyDocument(doc), }, ResponseMetadata: ResponseMetadata{RequestID: reqID}, }, nil @@ -916,7 +1003,7 @@ func (h *Handler) iamGroupInlinePolicyDispatchTable() map[string]iamActionFn { GetGroupPolicyResult: GetGroupPolicyResult{ GroupName: vals.Get("GroupName"), PolicyName: vals.Get("PolicyName"), - PolicyDocument: doc, + PolicyDocument: encodePolicyDocument(doc), }, ResponseMetadata: ResponseMetadata{RequestID: reqID}, }, nil @@ -1034,7 +1121,22 @@ func (h *Handler) iamReportingDispatchTable() map[string]iamActionFn { policies := make([]ManagedPolicyDetailXML, 0, len(details.Policies)) for i := range details.Policies { - policies = append(policies, toManagedPolicyDetailXML(&details.Policies[i])) + pol := &details.Policies[i] + + versions, vErr := h.Backend.ListPolicyVersions(pol.Arn) + if vErr != nil { + // Should not happen (the policy was just enumerated), but degrade + // gracefully to a single synthesized default version rather than + // dropping the policy from the response entirely. + versions = []StoredPolicyVersion{{ + VersionID: "v1", + PolicyDocument: pol.PolicyDocument, + IsDefaultVersion: true, + CreateDate: pol.CreateDate, + }} + } + + policies = append(policies, toManagedPolicyDetailXML(pol, versions)) } return &GetAccountAuthorizationDetailsResponse{ @@ -1547,6 +1649,7 @@ func (h *Handler) handleError(ctx context.Context, c *echo.Context, action strin errors.Is(reqErr, ErrOIDCProviderNotFound), errors.Is(reqErr, ErrLoginProfileNotFound): code = "NoSuchEntity" + statusCode = http.StatusNotFound case errors.Is(reqErr, ErrUserAlreadyExists), errors.Is(reqErr, ErrRoleAlreadyExists), errors.Is(reqErr, ErrPolicyAlreadyExists), @@ -1556,10 +1659,13 @@ func (h *Handler) handleError(ctx context.Context, c *echo.Context, action strin errors.Is(reqErr, ErrOIDCProviderAlreadyExists), errors.Is(reqErr, ErrLoginProfileAlreadyExists): code = "EntityAlreadyExists" + statusCode = http.StatusConflict case errors.Is(reqErr, ErrDeleteConflict): code = "DeleteConflict" + statusCode = http.StatusConflict case errors.Is(reqErr, ErrLimitExceeded): code = "LimitExceeded" + statusCode = http.StatusConflict case errors.Is(reqErr, ErrMalformedPolicyDocument): code = "MalformedPolicyDocument" case errors.Is(reqErr, ErrInvalidAction): @@ -1574,7 +1680,9 @@ func (h *Handler) handleError(ctx context.Context, c *echo.Context, action strin code = "InvalidAuthenticationCode" statusCode = http.StatusForbidden default: - code = "InternalFailure" + // Real AWS IAM (query protocol) returns "ServiceFailure" for unhandled + // server errors, not the JSON-protocol-style "InternalFailure". + code = "ServiceFailure" statusCode = http.StatusInternalServerError } @@ -1618,6 +1726,24 @@ func newRequestID() string { return "gopherstack-" + newID("req") } +// encodePolicyDocument percent-encodes a policy document for wire output. +// +// Real AWS IAM returns policy documents URL-encoded (RFC 3986) on the following +// operations: GetRole, GetRolePolicy, GetUserPolicy, GetGroupPolicy, +// GetPolicyVersion, and GetAccountAuthorizationDetails. Callers are expected to +// URL-decode the result; some SDKs do this automatically. See e.g. +// https://docs.aws.amazon.com/IAM/latest/APIReference/API_GetPolicyVersion.html +// +// The backend always stores/validates/evaluates the plain-JSON document; this +// encoding is applied only at the XML marshal boundary, never persisted. +func encodePolicyDocument(doc string) string { + if doc == "" { + return "" + } + + return url.QueryEscape(doc) +} + // ---- XML conversion helpers ---- func toUserXML(u *User) UserXML { @@ -1647,7 +1773,7 @@ func toRoleXML(r *Role) RoleXML { RoleID: r.RoleID, Arn: r.Arn, CreateDate: isoTime(r.CreateDate), - AssumeRolePolicyDocument: r.AssumeRolePolicyDocument, + AssumeRolePolicyDocument: encodePolicyDocument(r.AssumeRolePolicyDocument), MaxSessionDuration: r.MaxSessionDuration, Description: r.Description, Tags: tagsToXML(r.Tags), @@ -1682,6 +1808,7 @@ func toPolicyXML(p *Policy) PolicyXML { CreateDate: isoTime(p.CreateDate), UpdateDate: isoTime(updateDate), DefaultVersionID: defaultVersionID, + Tags: tagsToXML(p.Tags), AttachmentCount: p.AttachmentCount, IsAttachable: p.IsAttachable, } @@ -1870,7 +1997,10 @@ func toInlinePolicyEntriesXML(entries []InlinePolicyEntry) []InlinePolicyEntryXM result := make([]InlinePolicyEntryXML, 0, len(entries)) for _, e := range entries { - result = append(result, InlinePolicyEntryXML(e)) + result = append(result, InlinePolicyEntryXML{ + PolicyName: e.PolicyName, + PolicyDocument: encodePolicyDocument(e.PolicyDocument), + }) } return result @@ -1923,27 +2053,35 @@ func toRoleDetailXML(r RoleDetail) RoleDetailXML { RoleID: r.RoleID, Arn: r.Arn, CreateDate: isoTime(r.CreateDate), - AssumeRolePolicyDocument: r.AssumeRolePolicyDocument, + AssumeRolePolicyDocument: encodePolicyDocument(r.AssumeRolePolicyDocument), RolePolicyList: toInlinePolicyEntriesXML(r.InlinePolicies), AttachedManagedPolicies: toAttachedPoliciesXML(r.AttachedPolicies), } } -func toManagedPolicyDetailXML(p *Policy) ManagedPolicyDetailXML { +// toManagedPolicyDetailXML builds the ManagedPolicyDetail XML element for +// GetAccountAuthorizationDetails. versions is the full, real version list for +// the policy (as returned by StorageBackend.ListPolicyVersions) — real AWS +// includes every stored version here, not just the default, and each +// version's Document is URL-encoded like GetPolicyVersion. +func toManagedPolicyDetailXML(p *Policy, versions []StoredPolicyVersion) ManagedPolicyDetailXML { + xmlVersions := make([]PolicyVersionXML, 0, len(versions)) + for _, v := range versions { + xmlVersions = append(xmlVersions, PolicyVersionXML{ + Document: encodePolicyDocument(v.PolicyDocument), + VersionID: v.VersionID, + IsDefaultVersion: v.IsDefaultVersion, + CreateDate: isoTime(v.CreateDate), + }) + } + return ManagedPolicyDetailXML{ - PolicyName: p.PolicyName, - PolicyID: p.PolicyID, - Arn: p.Arn, - Path: p.Path, - CreateDate: isoTime(p.CreateDate), - PolicyVersionList: []PolicyVersionXML{ - { - Document: p.PolicyDocument, - VersionID: "v1", - IsDefaultVersion: true, - CreateDate: isoTime(p.CreateDate), - }, - }, + PolicyName: p.PolicyName, + PolicyID: p.PolicyID, + Arn: p.Arn, + Path: p.Path, + CreateDate: isoTime(p.CreateDate), + PolicyVersionList: xmlVersions, } } diff --git a/services/iam/handler_audit_batch1_test.go b/services/iam/handler_audit_batch1_test.go index e552900fe..b850a50fa 100644 --- a/services/iam/handler_audit_batch1_test.go +++ b/services/iam/handler_audit_batch1_test.go @@ -273,7 +273,9 @@ func TestHandler_PolicyVersion_CRUD(t *testing.T) { rec3 := httptest.NewRecorder() require.NoError(t, h.Handler()(e.NewContext(req3, rec3))) assert.Equal(t, http.StatusOK, rec3.Code) - assert.Contains(t, rec3.Body.String(), "s3:*") + // Real AWS IAM returns PolicyVersion.Document URL-encoded (RFC 3986); "s3:*" + // becomes "s3%3A%2A" on the wire. + assert.Contains(t, rec3.Body.String(), "s3%3A%2A") // SetDefaultPolicyVersion. req4 := iamRequest("SetDefaultPolicyVersion", map[string]string{ @@ -331,7 +333,8 @@ func TestHandler_PolicyVersion_LimitExceeded(t *testing.T) { }) rec := httptest.NewRecorder() require.NoError(t, h.Handler()(e.NewContext(req, rec))) - assert.Equal(t, http.StatusBadRequest, rec.Code) + // Real AWS IAM returns 409 for LimitExceeded, not 400. + assert.Equal(t, http.StatusConflict, rec.Code) var errResp iam.ErrorResponse require.NoError(t, xml.Unmarshal(rec.Body.Bytes(), &errResp)) @@ -410,7 +413,8 @@ func TestHandler_AccessKey_MaxTwoLimit_Via_Handler(t *testing.T) { req := iamRequest("CreateAccessKey", map[string]string{"UserName": "alice"}) rec := httptest.NewRecorder() require.NoError(t, h.Handler()(e.NewContext(req, rec))) - assert.Equal(t, http.StatusBadRequest, rec.Code, "third key must fail") + // Real AWS IAM returns 409 for LimitExceeded, not 400. + assert.Equal(t, http.StatusConflict, rec.Code, "third key must fail") var errResp iam.ErrorResponse require.NoError(t, xml.Unmarshal(rec.Body.Bytes(), &errResp)) @@ -710,7 +714,8 @@ func TestHandler_InstanceProfile_OneRoleLimit(t *testing.T) { }) rec := httptest.NewRecorder() require.NoError(t, h.Handler()(e.NewContext(req, rec))) - assert.Equal(t, http.StatusBadRequest, rec.Code, "second role must fail") + // Real AWS IAM returns 409 for LimitExceeded, not 400. + assert.Equal(t, http.StatusConflict, rec.Code, "second role must fail") var errResp iam.ErrorResponse require.NoError(t, xml.Unmarshal(rec.Body.Bytes(), &errResp)) @@ -1177,7 +1182,7 @@ func TestHandler_UpdateUser(t *testing.T) { req2 := iamRequest("GetUser", map[string]string{"UserName": "alice"}) rec2 := httptest.NewRecorder() require.NoError(t, h.Handler()(e.NewContext(req2, rec2))) - assert.Equal(t, http.StatusBadRequest, rec2.Code) + assert.Equal(t, http.StatusNotFound, rec2.Code) // New name should exist. req3 := iamRequest("GetUser", map[string]string{"UserName": "alicia"}) diff --git a/services/iam/handler_audit_batch2_test.go b/services/iam/handler_audit_batch2_test.go index 86a67a8ec..904d4db7d 100644 --- a/services/iam/handler_audit_batch2_test.go +++ b/services/iam/handler_audit_batch2_test.go @@ -519,7 +519,8 @@ func TestHandler_SimulatePrincipalPolicy_UserNotFound(t *testing.T) { }) rec := httptest.NewRecorder() require.NoError(t, h.Handler()(e.NewContext(req, rec))) - assert.Equal(t, http.StatusBadRequest, rec.Code) + // NoSuchEntity is 404 on real AWS IAM. + assert.Equal(t, http.StatusNotFound, rec.Code) } // ---- ServiceSpecificCredential UpdateStatus handler ---- diff --git a/services/iam/handler_new_ops_test.go b/services/iam/handler_new_ops_test.go index b633209df..e502ce259 100644 --- a/services/iam/handler_new_ops_test.go +++ b/services/iam/handler_new_ops_test.go @@ -564,13 +564,14 @@ func TestIAMHandler_NewOpsDispatch(t *testing.T) { wantContain: "CreatePolicyVersionResponse", }, { - name: "CreatePolicyVersion_not_found_returns_400", + name: "CreatePolicyVersion_not_found_returns_404", action: "CreatePolicyVersion", params: map[string]string{ "PolicyArn": "arn:aws:iam::000000000000:policy/Ghost", "PolicyDocument": `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]}`, }, - wantCode: http.StatusBadRequest, + // NoSuchEntity is 404 on real AWS IAM. + wantCode: http.StatusNotFound, }, // CreateServiceLinkedRole { @@ -611,7 +612,8 @@ func TestIAMHandler_NewOpsDispatch(t *testing.T) { "UserName": "ghost", "ServiceName": "codecommit.amazonaws.com", }, - wantCode: http.StatusBadRequest, + // NoSuchEntity is 404 on real AWS IAM. + wantCode: http.StatusNotFound, }, // CreateVirtualMFADevice { @@ -696,7 +698,8 @@ func TestIAMHandler_NewOpsDispatch(t *testing.T) { "OpenIDConnectProviderArn": "arn:aws:iam::000000000000:oidc-provider/nonexistent", "ClientID": "sts.amazonaws.com", }, - wantCode: http.StatusBadRequest, + // NoSuchEntity is 404 on real AWS IAM. + wantCode: http.StatusNotFound, }, } @@ -751,11 +754,12 @@ func TestIAMHandler_ListPolicyVersions(t *testing.T) { }, }, { - name: "policy_not_found_returns_bad_request", + name: "policy_not_found_returns_404", params: map[string]string{ "PolicyArn": "arn:aws:iam::000000000000:policy/VersionListPolicy", }, - wantCode: http.StatusBadRequest, + // NoSuchEntity is 404 on real AWS IAM. + wantCode: http.StatusNotFound, }, } diff --git a/services/iam/handler_test.go b/services/iam/handler_test.go index 979b259fc..a69a3996b 100644 --- a/services/iam/handler_test.go +++ b/services/iam/handler_test.go @@ -570,7 +570,7 @@ func TestIAMHandler_Users(t *testing.T) { err := h.Handler()(c) require.NoError(t, err) - assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Equal(t, http.StatusNotFound, rec.Code) var errResp iam.ErrorResponse require.NoError(t, xml.Unmarshal(rec.Body.Bytes(), &errResp)) @@ -1059,7 +1059,7 @@ func TestIAMHandler_Routing(t *testing.T) { err := h.Handler()(c) require.NoError(t, err) - assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Equal(t, http.StatusConflict, rec.Code) var errResp iam.ErrorResponse require.NoError(t, xml.Unmarshal(rec.Body.Bytes(), &errResp)) @@ -1361,7 +1361,7 @@ func TestIAMHandler_DetachRolePolicy(t *testing.T) { err := h.Handler()(c) require.NoError(t, err) - assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Equal(t, http.StatusNotFound, rec.Code) var errResp iam.ErrorResponse require.NoError(t, xml.Unmarshal(rec.Body.Bytes(), &errResp)) @@ -1611,9 +1611,13 @@ func TestIAMHandler_UntagAndVerify(t *testing.T) { } } -// TestIAMHandler_InternalFailure tests the InternalFailure error code path +// TestIAMHandler_ServiceFailure tests the ServiceFailure error code path // by injecting a backend that returns a raw (non-sentinel) error. -func TestIAMHandler_InternalFailure(t *testing.T) { +// +// Real AWS IAM (query protocol) reports unhandled server errors as +// "ServiceFailure", not the JSON-protocol-style "InternalFailure" — see +// https://docs.aws.amazon.com/IAM/latest/APIReference/API_CreateUser.html#API_CreateUser_Errors. +func TestIAMHandler_ServiceFailure(t *testing.T) { t.Parallel() e := echo.New() @@ -1630,11 +1634,17 @@ func TestIAMHandler_InternalFailure(t *testing.T) { var errResp iam.ErrorResponse require.NoError(t, xml.Unmarshal(rec.Body.Bytes(), &errResp)) - assert.Equal(t, "InternalFailure", errResp.Error.Code) + assert.Equal(t, "ServiceFailure", errResp.Error.Code) } // TestIAMHandler_DispatchErrors covers the error branches in every dispatch case. // These tests trigger "NoSuchEntity" / "EntityAlreadyExists" responses from the backend. +// +// wantStatus values match the HTTP status codes documented per-operation by the +// real AWS IAM API reference (e.g. NoSuchEntity=404, EntityAlreadyExists=409, +// DeleteConflict=409, LimitExceeded=409); MalformedPolicyDocument stays 400. +// See e.g. https://docs.aws.amazon.com/IAM/latest/APIReference/API_CreateUser.html#API_CreateUser_Errors +// and https://docs.aws.amazon.com/IAM/latest/APIReference/API_DeleteUser.html#API_DeleteUser_Errors. func TestIAMHandler_DispatchErrors(t *testing.T) { t.Parallel() @@ -1661,7 +1671,7 @@ func TestIAMHandler_DispatchErrors(t *testing.T) { action: "DeleteUser", params: map[string]string{"UserName": "nobody"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "CreateRole_AlreadyExists", @@ -1671,14 +1681,14 @@ func TestIAMHandler_DispatchErrors(t *testing.T) { _, _ = b.CreateRole("MyRole", "/", "", "") }, wantCode: "EntityAlreadyExists", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusConflict, }, { name: "GetRole_NotFound", action: "GetRole", params: map[string]string{"RoleName": "ghost"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "DeleteRole_NotFound", @@ -1686,7 +1696,7 @@ func TestIAMHandler_DispatchErrors(t *testing.T) { params: map[string]string{"RoleName": "ghost"}, setup: func(_ *iam.InMemoryBackend) {}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "CreatePolicy_AlreadyExists", @@ -1696,28 +1706,28 @@ func TestIAMHandler_DispatchErrors(t *testing.T) { _, _ = b.CreatePolicy("MyPolicy", "/", "") }, wantCode: "EntityAlreadyExists", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusConflict, }, { name: "DeletePolicy_NotFound", action: "DeletePolicy", params: map[string]string{"PolicyArn": "arn:aws:iam::000000000000:policy/ghost"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "AttachUserPolicy_UserNotFound", action: "AttachUserPolicy", params: map[string]string{"UserName": "nobody", "PolicyArn": "arn:aws:iam::000000000000:policy/P"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "AttachRolePolicy_RoleNotFound", action: "AttachRolePolicy", params: map[string]string{"RoleName": "ghost", "PolicyArn": "arn:aws:iam::000000000000:policy/P"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "CreateGroup_AlreadyExists", @@ -1727,28 +1737,28 @@ func TestIAMHandler_DispatchErrors(t *testing.T) { _, _ = b.CreateGroup("Admins", "/") }, wantCode: "EntityAlreadyExists", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusConflict, }, { name: "DeleteGroup_NotFound", action: "DeleteGroup", params: map[string]string{"GroupName": "ghost"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "AddUserToGroup_GroupNotFound", action: "AddUserToGroup", params: map[string]string{"GroupName": "ghost", "UserName": "alice"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "CreateAccessKey_UserNotFound", action: "CreateAccessKey", params: map[string]string{"UserName": "nobody"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "DeleteAccessKey_NotFound", @@ -1758,14 +1768,14 @@ func TestIAMHandler_DispatchErrors(t *testing.T) { _, _ = b.CreateUser("alice", "/", "") }, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "ListAccessKeys_UserNotFound", action: "ListAccessKeys", params: map[string]string{"UserName": "nobody"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "CreateInstanceProfile_AlreadyExists", @@ -1775,14 +1785,14 @@ func TestIAMHandler_DispatchErrors(t *testing.T) { _, _ = b.CreateInstanceProfile("MyProfile", "/") }, wantCode: "EntityAlreadyExists", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusConflict, }, { name: "DeleteInstanceProfile_NotFound", action: "DeleteInstanceProfile", params: map[string]string{"InstanceProfileName": "ghost"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "DeleteUser_DeleteConflict", @@ -1794,7 +1804,7 @@ func TestIAMHandler_DispatchErrors(t *testing.T) { _ = b.AttachUserPolicy("alice", pol.Arn) }, wantCode: "DeleteConflict", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusConflict, }, { name: "DeleteRole_DeleteConflict", @@ -1806,7 +1816,7 @@ func TestIAMHandler_DispatchErrors(t *testing.T) { _ = b.AttachRolePolicy("MyRole", pol.Arn) }, wantCode: "DeleteConflict", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusConflict, }, { name: "CreateRole_MalformedPolicyDocument", diff --git a/services/iam/inline_policy_test.go b/services/iam/inline_policy_test.go index 7fda3451f..27476b5df 100644 --- a/services/iam/inline_policy_test.go +++ b/services/iam/inline_policy_test.go @@ -683,7 +683,7 @@ func TestIAMHandler_UserInlinePolicies(t *testing.T) { setup: func(_ *iam.InMemoryBackend) {}, action: "GetUserPolicy", params: map[string]string{"UserName": "nobody", "PolicyName": "Ghost"}, - wantCode: http.StatusBadRequest, + wantCode: http.StatusNotFound, // NoSuchEntity is 404 on real AWS IAM. wantContain: "NoSuchEntity", }, } @@ -908,7 +908,7 @@ func TestIAMHandler_UpdateAssumeRolePolicy(t *testing.T) { "RoleName": "Ghost", "PolicyDocument": "{}", }, - wantCode: http.StatusBadRequest, + wantCode: http.StatusNotFound, // NoSuchEntity is 404 on real AWS IAM. wantContain: "NoSuchEntity", }, } @@ -1212,7 +1212,7 @@ func TestSimulatePrincipalPolicy(t *testing.T) { "PolicySourceArn": "arn:aws:iam::000000000000:user/nonexistent", "ActionNames.member.1": "s3:GetObject", }, - wantHTTPStatus: http.StatusBadRequest, + wantHTTPStatus: http.StatusNotFound, // NoSuchEntity is 404 on real AWS IAM. wantErr: true, }, } diff --git a/services/iam/models.go b/services/iam/models.go index b9fab49aa..2a01a83d9 100644 --- a/services/iam/models.go +++ b/services/iam/models.go @@ -298,15 +298,16 @@ type ListRolesResult struct { // PolicyXML is the XML representation of an IAM Policy. type PolicyXML struct { - PolicyName string `xml:"PolicyName"` - PolicyID string `xml:"PolicyId"` - Arn string `xml:"Arn"` - Path string `xml:"Path"` - CreateDate string `xml:"CreateDate"` - UpdateDate string `xml:"UpdateDate"` - DefaultVersionID string `xml:"DefaultVersionId"` - AttachmentCount int `xml:"AttachmentCount"` - IsAttachable bool `xml:"IsAttachable"` + PolicyName string `xml:"PolicyName"` + PolicyID string `xml:"PolicyId"` + Arn string `xml:"Arn"` + Path string `xml:"Path"` + CreateDate string `xml:"CreateDate"` + UpdateDate string `xml:"UpdateDate"` + DefaultVersionID string `xml:"DefaultVersionId"` + Tags []TagXML `xml:"Tags>member,omitempty"` + AttachmentCount int `xml:"AttachmentCount"` + IsAttachable bool `xml:"IsAttachable"` } // CreatePolicyResponse is the XML response for CreatePolicy. diff --git a/services/iam/new_operations_test.go b/services/iam/new_operations_test.go index f2a4d6461..d7b96750d 100644 --- a/services/iam/new_operations_test.go +++ b/services/iam/new_operations_test.go @@ -527,10 +527,11 @@ func TestIAMHandler_NewOperations(t *testing.T) { wantContain: "GetGroupResponse", }, { - name: "GetGroup_not_found", - action: "GetGroup", - params: map[string]string{"GroupName": "ghost"}, - wantCode: http.StatusBadRequest, + name: "GetGroup_not_found", + action: "GetGroup", + params: map[string]string{"GroupName": "ghost"}, + // NoSuchEntity is 404 on real AWS IAM. + wantCode: http.StatusNotFound, }, { name: "RemoveUserFromGroup_success", @@ -568,7 +569,8 @@ func TestIAMHandler_NewOperations(t *testing.T) { "InstanceProfileName": "ghost-profile", "RoleName": "some-role", }, - wantCode: http.StatusBadRequest, + // NoSuchEntity is 404 on real AWS IAM. + wantCode: http.StatusNotFound, }, { name: "RemoveRoleFromInstanceProfile_success", diff --git a/services/iam/parity_iam_fixes_test.go b/services/iam/parity_iam_fixes_test.go index 7c26bdaa8..f338a59d5 100644 --- a/services/iam/parity_iam_fixes_test.go +++ b/services/iam/parity_iam_fixes_test.go @@ -240,7 +240,10 @@ func TestParityIAM_ErrorSentinelWrapping(t *testing.T) { // TestParityIAM_HandlerNoSuchEntityCode verifies that all "not found" errors // translate to the "NoSuchEntity" XML error code in HTTP responses, matching AWS. -// IAM uses HTTP 400 for NoSuchEntity (not 404 — IAM is not REST-style). +// Real AWS IAM returns HTTP 404 for NoSuchEntity even though the body is XML +// (query protocol): every per-operation error reference page (e.g. GetRole, +// CreateUser, DeleteUser) documents "NoSuchEntity ... HTTP Status Code: 404". +// See https://docs.aws.amazon.com/IAM/latest/APIReference/API_GetRole.html#API_GetRole_Errors. func TestParityIAM_HandlerNoSuchEntityCode(t *testing.T) { t.Parallel() @@ -292,8 +295,8 @@ func TestParityIAM_HandlerNoSuchEntityCode(t *testing.T) { err := h.Handler()(c) require.NoError(t, err) - assert.Equal(t, http.StatusBadRequest, rec.Code, - "action %s must return 400 for NoSuchEntity", tt.action) + assert.Equal(t, http.StatusNotFound, rec.Code, + "action %s must return 404 for NoSuchEntity", tt.action) var errResp iam.ErrorResponse require.NoError(t, xml.Unmarshal(rec.Body.Bytes(), &errResp)) diff --git a/services/iam/persistence.go b/services/iam/persistence.go index 5176ea526..76337ea23 100644 --- a/services/iam/persistence.go +++ b/services/iam/persistence.go @@ -2,59 +2,72 @@ package iam import ( "context" + "encoding/json" + "fmt" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" ) +// iamSnapshotVersion identifies the shape of backendSnapshot's Tables blob +// (i.e. the set of resources registered on b.registry -- see +// registerAllTables in store_setup.go). It must be bumped whenever a change +// there would make an older snapshot unsafe to decode as the current shape. +// Restore compares this against the persisted value and discards (rather +// than attempts to partially decode) any mismatch -- see Restore below. This +// mirrors the services/ec2 (commit 12e611a4) and services/sqs (commit +// 0f09d77c) conversions. +const iamSnapshotVersion = 1 + type backendSnapshot struct { - RoleInlinePolicies map[string]map[string]string `json:"roleInlinePolicies,omitempty"` - VirtualMFADevices map[string]VirtualMFADevice `json:"virtualMFADevices,omitempty"` - Policies map[string]Policy `json:"policies,omitempty"` - Groups map[string]Group `json:"groups,omitempty"` - AccessKeys map[string]AccessKey `json:"accessKeys,omitempty"` - InstanceProfiles map[string]InstanceProfile `json:"instanceProfiles,omitempty"` - SAMLProviders map[string]SAMLProvider `json:"samlProviders,omitempty"` - OIDCProviders map[string]OIDCProvider `json:"oidcProviders,omitempty"` - LoginProfiles map[string]LoginProfile `json:"loginProfiles,omitempty"` - GroupMembers map[string][]string `json:"groupMembers,omitempty"` - Roles map[string]Role `json:"roles,omitempty"` - Users map[string]User `json:"users,omitempty"` - UserPolicies map[string][]string `json:"userPolicies,omitempty"` - UserInlinePolicies map[string]map[string]string `json:"userInlinePolicies,omitempty"` - GroupPolicies map[string][]string `json:"groupPolicies,omitempty"` - RolePolicies map[string][]string `json:"rolePolicies,omitempty"` - PasswordPolicy *PasswordPolicy `json:"passwordPolicy,omitempty"` - PolicyVersions map[string][]StoredPolicyVersion `json:"policyVersions,omitempty"` - PolicyVersionCounters map[string]int `json:"policyVersionCounters,omitempty"` - ServiceSpecificCreds map[string]ServiceSpecificCredential `json:"serviceSpecificCreds,omitempty"` - GroupInlinePolicies map[string]map[string]string `json:"groupInlinePolicies,omitempty"` - ServerCertificates map[string]ServerCertificate `json:"serverCertificates,omitempty"` - DelegationRequests map[string]DelegationRequest `json:"delegationRequests,omitempty"` - PolicyByARN map[string]string `json:"policyByARN,omitempty"` - RoleByARN map[string]string `json:"roleByARN,omitempty"` - PolicyAttachments map[string]policyAttachmentRefs `json:"policyAttachments,omitempty"` - DeletedV1Policies map[string]bool `json:"deletedV1Policies,omitempty"` - SigningCertificates map[string]SigningCertificate `json:"signingCertificates,omitempty"` - AccountID string `json:"accountID,omitempty"` - AccountAliases []string `json:"accountAliases,omitempty"` + Tables map[string]json.RawMessage `json:"tables"` + GroupPolicies map[string][]string `json:"groupPolicies,omitempty"` + PasswordPolicy *PasswordPolicy `json:"passwordPolicy,omitempty"` + GroupMembers map[string][]string `json:"groupMembers,omitempty"` + UserPolicies map[string][]string `json:"userPolicies,omitempty"` + UserInlinePolicies map[string]map[string]string `json:"userInlinePolicies,omitempty"` + RoleInlinePolicies map[string]map[string]string `json:"roleInlinePolicies,omitempty"` + PolicyVersionCounters map[string]int `json:"policyVersionCounters,omitempty"` + PolicyVersions map[string][]StoredPolicyVersion `json:"policyVersions,omitempty"` + RolePolicies map[string][]string `json:"rolePolicies,omitempty"` + GroupInlinePolicies map[string]map[string]string `json:"groupInlinePolicies,omitempty"` + PolicyByARN map[string]string `json:"policyByARN,omitempty"` + RoleByARN map[string]string `json:"roleByARN,omitempty"` + PolicyAttachments map[string]policyAttachmentRefs `json:"policyAttachments,omitempty"` + DeletedV1Policies map[string]bool `json:"deletedV1Policies,omitempty"` + Comprehensive *comprehensiveSnapshot `json:"comprehensive,omitempty"` + AccountID string `json:"accountID,omitempty"` + AccountAliases []string `json:"accountAliases,omitempty"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. // It implements persistence.Persistable. func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { + // Read comprehensive state before taking b.mu: comprehensiveBackend guards + // its own state with a separate mutex (c.mu), so reading it here — outside + // the b.mu critical section below — avoids establishing a new nested lock + // order between the two. + comp := b.comp().snapshot() + b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "iam: snapshot table marshal failed", "error", err) + + return nil + } + snap := backendSnapshot{ - Users: b.users, - Roles: b.roles, - Policies: b.policies, - Groups: b.groups, - AccessKeys: b.accessKeys, - InstanceProfiles: b.instanceProfiles, - SAMLProviders: b.samlProviders, - OIDCProviders: b.oidcProviders, - LoginProfiles: b.loginProfiles, + Version: iamSnapshotVersion, + Tables: tables, + Comprehensive: &comp, UserPolicies: b.userPolicies, RolePolicies: b.rolePolicies, GroupPolicies: b.groupPolicies, @@ -65,16 +78,11 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { AccountAliases: b.accountAliases, PolicyVersions: b.policyVersions, PolicyVersionCounters: b.policyVersionCounters, - ServiceSpecificCreds: b.serviceSpecificCreds, - VirtualMFADevices: b.virtualMFADevices, - DelegationRequests: b.delegationRequests, AccountID: b.accountID, PolicyByARN: b.policyByARN, RoleByARN: b.roleByARN, PolicyAttachments: b.policyAttachments, DeletedV1Policies: b.deletedV1Policies, - SigningCertificates: b.signingCertificates, - ServerCertificates: b.serverCertificates, PasswordPolicy: b.passwordPolicy, } @@ -93,21 +101,38 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { normalizeSnapshot(&snap) b.mu.Lock("Restore") - defer b.mu.Unlock() - b.users = snap.Users - b.roles = snap.Roles - b.policies = snap.Policies - b.groups = snap.Groups - b.accessKeys = snap.AccessKeys + if snap.Version != iamSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/ec2 and services/sqs conversions. + logger.Load(ctx).WarnContext(ctx, + "iam: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", iamSnapshotVersion) + + b.registry.ResetAll() + b.mu.Unlock() + + return nil + } + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + b.mu.Unlock() + + return fmt.Errorf("iam: restore snapshot tables: %w", err) + } + + // userAccessKeys is a secondary index (username -> []AccessKeyID) over + // b.accessKeys; it is not itself persisted (see store_setup.go), so it is + // always rebuilt fresh from the restored accessKeys table. b.userAccessKeys = make(map[string][]string) - for id, ak := range b.accessKeys { - b.userAccessKeys[ak.UserName] = append(b.userAccessKeys[ak.UserName], id) + for _, ak := range b.accessKeys.All() { + b.userAccessKeys[ak.UserName] = append(b.userAccessKeys[ak.UserName], ak.AccessKeyID) } - b.instanceProfiles = snap.InstanceProfiles - b.samlProviders = snap.SAMLProviders - b.oidcProviders = snap.OIDCProviders - b.loginProfiles = snap.LoginProfiles + b.userPolicies = snap.UserPolicies b.rolePolicies = snap.RolePolicies b.groupPolicies = snap.GroupPolicies @@ -118,9 +143,6 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.accountAliases = snap.AccountAliases b.policyVersions = snap.PolicyVersions b.policyVersionCounters = snap.PolicyVersionCounters - b.serviceSpecificCreds = snap.ServiceSpecificCreds - b.virtualMFADevices = snap.VirtualMFADevices - b.delegationRequests = snap.DelegationRequests b.accountID = snap.AccountID b.rebuildIndexesLocked() @@ -144,17 +166,17 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { } else { b.deletedV1Policies = make(map[string]bool) } - if snap.SigningCertificates != nil { - b.signingCertificates = snap.SigningCertificates - } else { - b.signingCertificates = make(map[string]SigningCertificate) - } - if snap.ServerCertificates != nil { - b.serverCertificates = snap.ServerCertificates + b.passwordPolicy = snap.PasswordPolicy + + b.mu.Unlock() + + // Restore comprehensive state after releasing b.mu (see the matching note + // in Snapshot): comprehensiveBackend.restore takes c.mu independently. + if snap.Comprehensive != nil { + b.comp().restore(*snap.Comprehensive) } else { - b.serverCertificates = make(map[string]ServerCertificate) + b.comp().restore(comprehensiveSnapshot{}) } - b.passwordPolicy = snap.PasswordPolicy return nil } @@ -162,50 +184,10 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { // normalizeSnapshot ensures all map fields in snap are non-nil so callers // can assign them directly without nil-pointer risk. func normalizeSnapshot(snap *backendSnapshot) { - normalizeSnapshotEntities(snap) normalizeSnapshotPolicies(snap) normalizeSnapshotNewOps(snap) } -// normalizeSnapshotEntities initialises entity maps in snap to non-nil empty maps. -func normalizeSnapshotEntities(snap *backendSnapshot) { - if snap.Users == nil { - snap.Users = make(map[string]User) - } - - if snap.Roles == nil { - snap.Roles = make(map[string]Role) - } - - if snap.Policies == nil { - snap.Policies = make(map[string]Policy) - } - - if snap.Groups == nil { - snap.Groups = make(map[string]Group) - } - - if snap.AccessKeys == nil { - snap.AccessKeys = make(map[string]AccessKey) - } - - if snap.InstanceProfiles == nil { - snap.InstanceProfiles = make(map[string]InstanceProfile) - } - - if snap.SAMLProviders == nil { - snap.SAMLProviders = make(map[string]SAMLProvider) - } - - if snap.OIDCProviders == nil { - snap.OIDCProviders = make(map[string]OIDCProvider) - } - - if snap.LoginProfiles == nil { - snap.LoginProfiles = make(map[string]LoginProfile) - } -} - // normalizeSnapshotPolicies initialises policy maps in snap to non-nil empty maps. func normalizeSnapshotPolicies(snap *backendSnapshot) { if snap.UserPolicies == nil { @@ -243,18 +225,6 @@ func normalizeSnapshotNewOps(snap *backendSnapshot) { snap.PolicyVersions = make(map[string][]StoredPolicyVersion) } - if snap.ServiceSpecificCreds == nil { - snap.ServiceSpecificCreds = make(map[string]ServiceSpecificCredential) - } - - if snap.VirtualMFADevices == nil { - snap.VirtualMFADevices = make(map[string]VirtualMFADevice) - } - - if snap.DelegationRequests == nil { - snap.DelegationRequests = make(map[string]DelegationRequest) - } - if snap.PolicyVersionCounters == nil { snap.PolicyVersionCounters = rebuildVersionCounters(snap.PolicyVersions) } @@ -311,26 +281,62 @@ func parseVersionNum(id string) int { return n } -// Snapshot implements persistence.Persistable by delegating to the backend. +// handlerSnapshot wraps the backend snapshot together with Handler-level state +// (resource tags for instance profiles, MFA devices, SAML/OIDC providers, and +// server certificates — see tagsSnapshot/restoreTags) so both round-trip +// through persistence together. +type handlerSnapshot struct { + Tags map[string]map[string]string `json:"tags,omitempty"` + Backend json.RawMessage `json:"backend,omitempty"` +} + +// Snapshot implements persistence.Persistable. It combines the backend's own +// snapshot with Handler-level tag state. func (h *Handler) Snapshot(ctx context.Context) []byte { type snapshotter interface { Snapshot(ctx context.Context) []byte } + + snap := handlerSnapshot{Tags: h.tagsSnapshot()} + if s, ok := h.Backend.(snapshotter); ok { - return s.Snapshot(ctx) + if b := s.Snapshot(ctx); len(b) > 0 { + snap.Backend = json.RawMessage(b) + } } - return nil + return persistence.MarshalSnapshot(ctx, "iam", snap) } -// Restore implements persistence.Persistable by delegating to the backend. +// Restore implements persistence.Persistable. It accepts both the current +// wrapped format and the legacy format (where data was the raw backend +// snapshot with no Handler-level wrapper), so older persisted snapshots still +// load correctly. func (h *Handler) Restore(ctx context.Context, data []byte) error { type restorer interface { Restore(context.Context, []byte) error } + + var snap handlerSnapshot + if err := persistence.UnmarshalSnapshot(ctx, "iam", data, &snap); err != nil { + return err + } + + backendData := []byte(snap.Backend) + if len(backendData) == 0 { + // Legacy snapshot: the whole blob is the backend snapshot and has no + // "backend"/"tags" wrapper fields (unmarshal above silently ignored + // them since backendSnapshot's JSON shape doesn't collide with ours). + backendData = data + } + if r, ok := h.Backend.(restorer); ok { - return r.Restore(ctx, data) + if err := r.Restore(ctx, backendData); err != nil { + return err + } } + h.restoreTags(snap.Tags) + return nil } diff --git a/services/iam/persistence_test.go b/services/iam/persistence_test.go index daee43681..fbf7e4d71 100644 --- a/services/iam/persistence_test.go +++ b/services/iam/persistence_test.go @@ -1,6 +1,7 @@ package iam_test import ( + "strings" "testing" "github.com/stretchr/testify/assert" @@ -73,3 +74,177 @@ func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { err := b.Restore(t.Context(), []byte("not-valid-json")) require.Error(t, err) } + +// TestInMemoryBackend_FullStateSnapshotRestore exercises a full-state +// Snapshot->Restore round trip covering every resource converted to +// pkgs/store in Phase 3.3 (users, roles, policies + versions/attachments, +// groups + membership, access keys, instance profiles + role attachment, +// SAML/OIDC providers, login profiles, service-specific credentials, virtual +// MFA devices, signing/server certificates, delegation requests) plus +// representative comprehensiveBackend state (SSH keys) and account-level +// state (password policy, account alias) that must NOT regress across the +// datalayer swap. +func TestInMemoryBackend_FullStateSnapshotRestore(t *testing.T) { + t.Parallel() + + const testPolicyDoc = `{"Version":"2012-10-17",` + + `"Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}` + + ctx := t.Context() + b := iam.NewInMemoryBackend() + + // ---- Core entities ---- + _, err := b.CreateUser("alice", "/", "") + require.NoError(t, err) + + role, err := b.CreateRole("app-role", "/", testPolicyDoc, "") + require.NoError(t, err) + + policy, err := b.CreatePolicy("app-policy", "/", testPolicyDoc) + require.NoError(t, err) + + _, err = b.CreatePolicyVersion(policy.Arn, testPolicyDoc, false) + require.NoError(t, err) + + require.NoError(t, b.AttachUserPolicy("alice", policy.Arn)) + require.NoError(t, b.AttachRolePolicy(role.RoleName, policy.Arn)) + + _, err = b.CreateGroup("app-group", "/") + require.NoError(t, err) + require.NoError(t, b.AddUserToGroup("app-group", "alice")) + + ak, err := b.CreateAccessKey("alice") + require.NoError(t, err) + + ip, err := b.CreateInstanceProfile("app-profile", "/") + require.NoError(t, err) + require.NoError(t, b.AddRoleToInstanceProfile(ip.InstanceProfileName, role.RoleName)) + + _, err = b.CreateSAMLProvider("saml-idp", ``) + require.NoError(t, err) + + _, err = b.CreateOpenIDConnectProvider( + "https://token.actions.githubusercontent.com", + []string{"sts.amazonaws.com"}, + []string{strings.Repeat("a", 40)}, + ) + require.NoError(t, err) + + _, err = b.CreateLoginProfile("alice", "P@ssw0rd123!", false) + require.NoError(t, err) + + cred, err := b.CreateServiceSpecificCredential("alice", "codecommit.amazonaws.com") + require.NoError(t, err) + + mfa, err := b.CreateVirtualMFADevice("alice-mfa", "/") + require.NoError(t, err) + + _, err = b.UploadSigningCertificate("alice", "-----BEGIN CERTIFICATE-----\nZmFrZQ==\n-----END CERTIFICATE-----") + require.NoError(t, err) + + _, err = b.UploadServerCertificate("app-cert", "/", "body", "chain") + require.NoError(t, err) + + delegation, err := b.CreateDelegationRequest("123456789012") + require.NoError(t, err) + + require.NoError(t, b.CreateAccountAlias("acme-corp")) + require.NoError(t, b.UpdateAccountPasswordPolicy(iam.PasswordPolicy{MinimumPasswordLength: 12})) + + // ---- comprehensiveBackend state (separate mutex; see gopherstack-gjp) ---- + sshKey, err := b.UploadSSHPublicKey("alice", strings.Repeat("k", 20)) + require.NoError(t, err) + + b.RecordServiceAccess(role.Arn, "s3", "s3.amazonaws.com") + + // ---- Snapshot / Restore round trip ---- + snap := b.Snapshot(ctx) + require.NotNil(t, snap) + + fresh := iam.NewInMemoryBackend() + require.NoError(t, fresh.Restore(ctx, snap)) + + // ---- Verify every converted table survived ---- + gotUser, err := fresh.GetUser("alice") + require.NoError(t, err) + assert.Equal(t, "alice", gotUser.UserName) + + gotRole, err := fresh.GetRole(role.RoleName) + require.NoError(t, err) + assert.Equal(t, role.Arn, gotRole.Arn) + + gotPolicy, err := fresh.GetPolicy(policy.Arn) + require.NoError(t, err) + assert.Equal(t, policy.PolicyName, gotPolicy.PolicyName) + + versions, err := fresh.ListPolicyVersions(policy.Arn) + require.NoError(t, err) + assert.Len(t, versions, 2) // v1 (implicit) + the version created above + + attachedToUser, err := fresh.GetPoliciesForUser("alice") + require.NoError(t, err) + assert.NotEmpty(t, attachedToUser) + + gotGroup, err := fresh.GetGroup("app-group") + require.NoError(t, err) + assert.Equal(t, "app-group", gotGroup.GroupName) + + groupUsers, err := fresh.GetGroupUsers("app-group") + require.NoError(t, err) + require.Len(t, groupUsers, 1) + assert.Equal(t, "alice", groupUsers[0].UserName) + + gotAK, err := fresh.GetUserByAccessKeyID(ak.AccessKeyID) + require.NoError(t, err) + assert.Equal(t, "alice", gotAK.UserName) + + gotIP, err := fresh.GetInstanceProfile(ip.InstanceProfileName) + require.NoError(t, err) + require.Len(t, gotIP.Roles, 1) + assert.Equal(t, role.RoleName, gotIP.Roles[0]) + + samlProviders, err := fresh.ListSAMLProviders() + require.NoError(t, err) + assert.Len(t, samlProviders, 1) + + oidcProviders, err := fresh.ListOpenIDConnectProviders() + require.NoError(t, err) + assert.Len(t, oidcProviders, 1) + + gotLP, err := fresh.GetLoginProfile("alice") + require.NoError(t, err) + assert.Equal(t, "alice", gotLP.UserName) + + creds, err := fresh.ListServiceSpecificCredentials("alice", "") + require.NoError(t, err) + require.Len(t, creds, 1) + assert.Equal(t, cred.ServiceSpecificCredentialID, creds[0].ServiceSpecificCredentialID) + + mfaDevices, err := fresh.ListVirtualMFADevices("", 0) + require.NoError(t, err) + require.Len(t, mfaDevices.Data, 1) + assert.Equal(t, mfa.SerialNumber, mfaDevices.Data[0].SerialNumber) + + signingCerts, err := fresh.ListSigningCertificates("alice") + require.NoError(t, err) + assert.Len(t, signingCerts, 1) + + serverCerts, err := fresh.ListServerCertificates("") + require.NoError(t, err) + assert.Len(t, serverCerts, 1) + + require.NoError(t, fresh.AcceptDelegationRequest(delegation.DelegationID)) + + // ---- Verify raw (un-converted) relation/index state survived ---- + aliases := fresh.ListAccountAliases() + assert.Contains(t, aliases, "acme-corp") + + pp := fresh.GetAccountPasswordPolicy() + require.NotNil(t, pp) + assert.Equal(t, 12, pp.MinimumPasswordLength) + + // ---- Verify comprehensiveBackend state survived ---- + gotSSHKey, err := fresh.GetSSHPublicKey("alice", sshKey.SSHPublicKeyID) + require.NoError(t, err) + assert.Equal(t, sshKey.Fingerprint, gotSSHKey.Fingerprint) +} diff --git a/services/iam/providers_test.go b/services/iam/providers_test.go index ca25651e1..2e6806ced 100644 --- a/services/iam/providers_test.go +++ b/services/iam/providers_test.go @@ -477,28 +477,28 @@ func TestIAMHandler_SAMLProvider_Errors(t *testing.T) { params: map[string]string{"Name": "MySAML", "SAMLMetadataDocument": ""}, setup: func(b *iam.InMemoryBackend) { _, _ = b.CreateSAMLProvider("MySAML", "") }, wantCode: "EntityAlreadyExists", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusConflict, }, { name: "GetSAMLProvider_NotFound", action: "GetSAMLProvider", params: map[string]string{"SAMLProviderArn": "arn:aws:iam::000000000000:saml-provider/ghost"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "UpdateSAMLProvider_NotFound", action: "UpdateSAMLProvider", params: map[string]string{"SAMLProviderArn": "arn:aws:iam::000000000000:saml-provider/ghost"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "DeleteSAMLProvider_NotFound", action: "DeleteSAMLProvider", params: map[string]string{"SAMLProviderArn": "arn:aws:iam::000000000000:saml-provider/ghost"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, } @@ -683,14 +683,14 @@ func TestIAMHandler_OIDCProvider_Errors(t *testing.T) { params: map[string]string{"Url": "https://example.com"}, setup: func(b *iam.InMemoryBackend) { _, _ = b.CreateOpenIDConnectProvider("https://example.com", nil, nil) }, wantCode: "EntityAlreadyExists", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusConflict, }, { name: "GetOpenIDConnectProvider_NotFound", action: "GetOpenIDConnectProvider", params: map[string]string{"OpenIDConnectProviderArn": "arn:aws:iam::000000000000:oidc-provider/ghost"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "UpdateOpenIDConnectProviderThumbprint_NotFound", @@ -700,14 +700,14 @@ func TestIAMHandler_OIDCProvider_Errors(t *testing.T) { "ThumbprintList.member.1": "990f41981148b53dc7c615a6b0c2a26555cc5d85", }, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "DeleteOpenIDConnectProvider_NotFound", action: "DeleteOpenIDConnectProvider", params: map[string]string{"OpenIDConnectProviderArn": "arn:aws:iam::000000000000:oidc-provider/ghost"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, } @@ -863,7 +863,7 @@ func TestIAMHandler_LoginProfile_Errors(t *testing.T) { action: "CreateLoginProfile", params: map[string]string{"UserName": "nobody", "Password": "Pass"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "CreateLoginProfile_AlreadyExists", @@ -874,28 +874,28 @@ func TestIAMHandler_LoginProfile_Errors(t *testing.T) { _, _ = b.CreateLoginProfile("alice", "Password1", false) }, wantCode: "EntityAlreadyExists", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusConflict, }, { name: "GetLoginProfile_NotFound", action: "GetLoginProfile", params: map[string]string{"UserName": "nobody"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "UpdateLoginProfile_NotFound", action: "UpdateLoginProfile", params: map[string]string{"UserName": "nobody", "Password": "Pass"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, { name: "DeleteLoginProfile_NotFound", action: "DeleteLoginProfile", params: map[string]string{"UserName": "nobody"}, wantCode: "NoSuchEntity", - wantStatus: http.StatusBadRequest, + wantStatus: http.StatusNotFound, }, } diff --git a/services/iam/server_cert_test.go b/services/iam/server_cert_test.go index b18fab5fa..5986fc4f6 100644 --- a/services/iam/server_cert_test.go +++ b/services/iam/server_cert_test.go @@ -163,11 +163,12 @@ func TestServerCertificate_HandlerRoundtrip(t *testing.T) { }) assert.Equal(t, 200, rec.Code) - // Get after delete — NoSuchEntity returns 400 per AWS IAM semantics. + // Get after delete — NoSuchEntity returns 404 per real AWS IAM (see e.g. + // https://docs.aws.amazon.com/IAM/latest/APIReference/API_GetServerCertificate.html#API_GetServerCertificate_Errors). rec = callIAM(t, h, "GetServerCertificate", map[string]string{ "ServerCertificateName": "handler-cert", }) - assert.Equal(t, 400, rec.Code) + assert.Equal(t, 404, rec.Code) assert.Contains(t, rec.Body.String(), "NoSuchEntity") } diff --git a/services/iam/store_setup.go b/services/iam/store_setup.go new file mode 100644 index 000000000..b99c927d3 --- /dev/null +++ b/services/iam/store_setup.go @@ -0,0 +1,134 @@ +package iam + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]T resource field on InMemoryBackend whose key is a pure function +// of the stored value's own field is registered exactly once, here, as a +// *store.Table[T] on b.registry. See pkgs/store's package doc and the +// services/ec2 (commit 12e611a4) and services/sqs (commit 0f09d77c) +// conversions this follows. +// +// The following resource fields are deliberately NOT registered here and +// remain plain maps because their key is not a pure function of the stored +// value's own fields, or they are secondary indexes/relations rather than +// primary resource collections: +// - policyByARN, roleByARN: ARN -> primary-key secondary indexes over +// policies/roles, not resource collections of their own. +// - userAccessKeys: username -> []AccessKeyID secondary index, rebuilt from +// accessKeys on Restore (see persistence.go). +// - groupMembers, groupPolicies, userPolicies, rolePolicies: entity name -> +// []string relation maps (group membership / attached policy ARNs), no +// single-struct identity to key a Table by. +// - userInlinePolicies, roleInlinePolicies, groupInlinePolicies: entity name +// -> (policy name -> document) relation maps. +// - policyAttachments: policy ARN -> set-of-entity-name refs, a relation not +// a resource. +// - policyVersions: policy ARN -> []StoredPolicyVersion, a per-policy +// history list rather than a single keyed resource. +// - policyVersionCounters, deletedV1Policies: policy ARN -> scalar +// bookkeeping, no resource identity. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +func usersKeyFn(v *User) string { return v.UserName } +func rolesKeyFn(v *Role) string { return v.RoleName } +func policiesKeyFn(v *Policy) string { return v.PolicyName } +func groupsKeyFn(v *Group) string { return v.GroupName } +func accessKeysKeyFn(v *AccessKey) string { return v.AccessKeyID } +func instanceProfilesKeyFn(v *InstanceProfile) string { return v.InstanceProfileName } +func samlProvidersKeyFn(v *SAMLProvider) string { return v.Arn } +func oidcProvidersKeyFn(v *OIDCProvider) string { return v.Arn } +func loginProfilesKeyFn(v *LoginProfile) string { return v.UserName } +func serviceSpecificCredsKeyFn(v *ServiceSpecificCredential) string { + return v.ServiceSpecificCredentialID +} +func virtualMFADevicesKeyFn(v *VirtualMFADevice) string { return v.SerialNumber } +func signingCertificatesKeyFn(v *SigningCertificate) string { return v.CertificateID } +func serverCertificatesKeyFn(v *ServerCertificate) string { + return v.ServerCertificateName +} +func delegationRequestsKeyFn(v *DelegationRequest) string { return v.DelegationID } + +// registerAllTables registers every converted resource map on b.registry +// exactly once. It must be called during construction only (immediately after +// b.registry is created), never on every Reset() -- store.Register panics on +// a duplicate name, so runtime resets go through registry.ResetAll() instead +// (see InMemoryBackend.Reset). +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call to the concrete field and value type. A closure list (rather than one +// statement per field in a single function body) mirrors the services/ec2 +// pattern (store_setup.go) and keeps registerAllTables small regardless of how +// many resource tables the backend grows to. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.users = store.Register(b.registry, "users", store.New(usersKeyFn)) + }, + func(b *InMemoryBackend) { + b.roles = store.Register(b.registry, "roles", store.New(rolesKeyFn)) + }, + func(b *InMemoryBackend) { + b.policies = store.Register(b.registry, "policies", store.New(policiesKeyFn)) + }, + func(b *InMemoryBackend) { + b.groups = store.Register(b.registry, "groups", store.New(groupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.accessKeys = store.Register(b.registry, "accessKeys", store.New(accessKeysKeyFn)) + }, + func(b *InMemoryBackend) { + b.instanceProfiles = store.Register(b.registry, "instanceProfiles", store.New(instanceProfilesKeyFn)) + }, + func(b *InMemoryBackend) { + b.samlProviders = store.Register(b.registry, "samlProviders", store.New(samlProvidersKeyFn)) + }, + func(b *InMemoryBackend) { + b.oidcProviders = store.Register(b.registry, "oidcProviders", store.New(oidcProvidersKeyFn)) + }, + func(b *InMemoryBackend) { + b.loginProfiles = store.Register(b.registry, "loginProfiles", store.New(loginProfilesKeyFn)) + }, + func(b *InMemoryBackend) { + b.serviceSpecificCreds = store.Register( + b.registry, + "serviceSpecificCreds", + store.New(serviceSpecificCredsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.virtualMFADevices = store.Register(b.registry, "virtualMFADevices", store.New(virtualMFADevicesKeyFn)) + }, + func(b *InMemoryBackend) { + b.signingCertificates = store.Register(b.registry, "signingCertificates", store.New(signingCertificatesKeyFn)) + }, + func(b *InMemoryBackend) { + b.serverCertificates = store.Register(b.registry, "serverCertificates", store.New(serverCertificatesKeyFn)) + }, + func(b *InMemoryBackend) { + b.delegationRequests = store.Register(b.registry, "delegationRequests", store.New(delegationRequestsKeyFn)) + }, +} + +// sortedTableNames returns every key in t, ascending, by delegating to +// [store.Table.Snapshot]'s already-deterministic key-sorted order and mapping +// each item through keyFn. It replaces the old rebuildSortedNames helper +// (which sorted map keys directly) now that these fields are Tables rather +// than plain maps. +func sortedTableNames[V any](t *store.Table[V], keyFn func(*V) string) []string { + items := t.Snapshot() + names := make([]string, len(items)) + + for i, v := range items { + names[i] = keyFn(v) + } + + return names +} diff --git a/services/iam/wire_parity_test.go b/services/iam/wire_parity_test.go new file mode 100644 index 000000000..b6b992d3d --- /dev/null +++ b/services/iam/wire_parity_test.go @@ -0,0 +1,480 @@ +package iam_test + +import ( + "context" + "encoding/xml" + "maps" + "net/http" + "net/http/httptest" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/iam" +) + +// Test_PolicyDocumentEncoding verifies that policy-document response fields +// are percent-encoded (RFC 3986) on the wire, matching real AWS IAM. Every +// per-operation reference page (GetRole, GetRolePolicy, GetUserPolicy, +// GetGroupPolicy, GetPolicyVersion) documents: +// "Policies returned by this operation are URL-encoded compliant with RFC 3986." +// e.g. https://docs.aws.amazon.com/IAM/latest/APIReference/API_GetRole.html +// +// Before this fix, gopherstack returned the raw JSON document unencoded, +// which breaks any client that (correctly, per AWS's documented contract) +// URL-decodes the field it receives. +func Test_PolicyDocumentEncoding(t *testing.T) { + t.Parallel() + + const rawDoc = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}` + // url.QueryEscape("s3:*") == "s3%3A%2A"; the whole document is escaped the + // same way once it reaches the wire. + const encodedFragment = "s3%3A%2A" + + tests := []struct { + setup func(t *testing.T, b *iam.InMemoryBackend) map[string]string + name string + action string + wantElem string + }{ + { + name: "GetRole_AssumeRolePolicyDocument", + action: "GetRole", + setup: func(t *testing.T, b *iam.InMemoryBackend) map[string]string { + t.Helper() + _, err := b.CreateRole("MyRole", "/", rawDoc, "") + require.NoError(t, err) + + return map[string]string{"RoleName": "MyRole"} + }, + wantElem: "AssumeRolePolicyDocument", + }, + { + name: "GetPolicyVersion_Document", + action: "GetPolicyVersion", + setup: func(t *testing.T, b *iam.InMemoryBackend) map[string]string { + t.Helper() + pol, err := b.CreatePolicy("MyPolicy", "/", rawDoc) + require.NoError(t, err) + + return map[string]string{"PolicyArn": pol.Arn, "VersionId": "v1"} + }, + wantElem: "Document", + }, + { + name: "GetUserPolicy_PolicyDocument", + action: "GetUserPolicy", + setup: func(t *testing.T, b *iam.InMemoryBackend) map[string]string { + t.Helper() + _, err := b.CreateUser("bob", "/", "") + require.NoError(t, err) + require.NoError(t, b.PutUserPolicy("bob", "Inline", rawDoc)) + + return map[string]string{"UserName": "bob", "PolicyName": "Inline"} + }, + wantElem: "PolicyDocument", + }, + { + name: "GetRolePolicy_PolicyDocument", + action: "GetRolePolicy", + setup: func(t *testing.T, b *iam.InMemoryBackend) map[string]string { + t.Helper() + _, err := b.CreateRole("PolRole", "/", "{}", "") + require.NoError(t, err) + require.NoError(t, b.PutRolePolicy("PolRole", "Inline", rawDoc)) + + return map[string]string{"RoleName": "PolRole", "PolicyName": "Inline"} + }, + wantElem: "PolicyDocument", + }, + { + name: "GetGroupPolicy_PolicyDocument", + action: "GetGroupPolicy", + setup: func(t *testing.T, b *iam.InMemoryBackend) map[string]string { + t.Helper() + _, err := b.CreateGroup("PolGroup", "/") + require.NoError(t, err) + require.NoError(t, b.PutGroupPolicy("PolGroup", "Inline", rawDoc)) + + return map[string]string{"GroupName": "PolGroup", "PolicyName": "Inline"} + }, + wantElem: "PolicyDocument", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + e := echo.New() + h, b := newTestHandler(t) + params := tt.setup(t, b) + + req := iamRequest(tt.action, params) + rec := httptest.NewRecorder() + require.NoError(t, h.Handler()(e.NewContext(req, rec))) + require.Equal(t, http.StatusOK, rec.Code) + + body := rec.Body.String() + assert.Contains(t, body, encodedFragment, + "%s must be percent-encoded on the wire", tt.wantElem) + assert.NotContains(t, body, rawDoc, + "%s must not contain the raw (unencoded) policy document", tt.wantElem) + }) + } +} + +// Test_ListInstanceProfilesForRole verifies ListInstanceProfilesForRole +// returns the real instance profiles containing the given role, instead of +// the previous hardcoded-empty stub response (which silently discarded the +// RoleName parameter and never touched backend state). +func Test_ListInstanceProfilesForRole(t *testing.T) { + t.Parallel() + + tests := []struct { + setup func(t *testing.T, b *iam.InMemoryBackend) + name string + roleName string + wantProfile string + wantCode int + wantEmpty bool + }{ + { + name: "role_with_one_profile", + setup: func(t *testing.T, b *iam.InMemoryBackend) { + t.Helper() + _, err := b.CreateRole("MyRole", "/", "{}", "") + require.NoError(t, err) + _, err = b.CreateInstanceProfile("MyProfile", "/") + require.NoError(t, err) + require.NoError(t, b.AddRoleToInstanceProfile("MyProfile", "MyRole")) + }, + roleName: "MyRole", + wantCode: http.StatusOK, + wantProfile: "MyProfile", + }, + { + name: "role_with_no_profiles", + setup: func(t *testing.T, b *iam.InMemoryBackend) { + t.Helper() + _, err := b.CreateRole("LonelyRole", "/", "{}", "") + require.NoError(t, err) + }, + roleName: "LonelyRole", + wantCode: http.StatusOK, + wantEmpty: true, + }, + { + name: "role_not_found", + setup: func(_ *testing.T, _ *iam.InMemoryBackend) {}, + roleName: "ghost", + wantCode: http.StatusNotFound, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + e := echo.New() + h, b := newTestHandler(t) + tt.setup(t, b) + + req := iamRequest("ListInstanceProfilesForRole", map[string]string{"RoleName": tt.roleName}) + rec := httptest.NewRecorder() + require.NoError(t, h.Handler()(e.NewContext(req, rec))) + assert.Equal(t, tt.wantCode, rec.Code) + + if tt.wantProfile != "" { + assert.Contains(t, rec.Body.String(), tt.wantProfile) + } + + if tt.wantEmpty { + assert.Contains(t, rec.Body.String(), "") + } + }) + } +} + +// Test_GetAccountAuthorizationDetails_PolicyVersions verifies that +// GetAccountAuthorizationDetails reports every real stored version of a +// managed policy (via StorageBackend.ListPolicyVersions), not a single +// hardcoded fake "v1" entry that ignored CreatePolicyVersion/SetDefaultPolicyVersion +// state entirely. +func Test_GetAccountAuthorizationDetails_PolicyVersions(t *testing.T) { + t.Parallel() + + e := echo.New() + h, b := newTestHandler(t) + + pol, err := b.CreatePolicy( + "MultiVersionPolicy", "/", + `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]}`, + ) + require.NoError(t, err) + + v2, err := b.CreatePolicyVersion( + pol.Arn, + `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}`, + true, // set as default + ) + require.NoError(t, err) + + req := iamRequest("GetAccountAuthorizationDetails", nil) + rec := httptest.NewRecorder() + require.NoError(t, h.Handler()(e.NewContext(req, rec))) + require.Equal(t, http.StatusOK, rec.Code) + + var resp iam.GetAccountAuthorizationDetailsResponse + require.NoError(t, xml.Unmarshal(rec.Body.Bytes(), &resp)) + + require.Len(t, resp.GetAccountAuthorizationDetailsResult.Policies, 1) + managed := resp.GetAccountAuthorizationDetailsResult.Policies[0] + + require.Len(t, managed.PolicyVersionList, 2, "must report both v1 and v2, not a fake single version") + + byVersion := map[string]iam.PolicyVersionXML{} + for _, v := range managed.PolicyVersionList { + byVersion[v.VersionID] = v + } + + v1XML, ok := byVersion["v1"] + require.True(t, ok, "v1 must be present") + assert.False(t, v1XML.IsDefaultVersion, "v1 is no longer default after SetAsDefault on v2") + + v2XML, ok := byVersion[v2.VersionID] + require.True(t, ok, "%s must be present", v2.VersionID) + assert.True(t, v2XML.IsDefaultVersion) + // Document must be percent-encoded, matching GetPolicyVersion's documented behavior. + assert.Contains(t, v2XML.Document, "s3%3A%2A") +} + +// Test_HandlerPersistence_RoundTrip verifies that a Handler.Snapshot/Restore +// cycle preserves state that previously lived outside backendSnapshot: +// Handler-level resource tags (instance profiles, MFA devices, SAML/OIDC +// providers, server certificates) and comprehensiveBackend state (SSH public +// keys). Both were silently dropped by every persistence restore before this +// fix. +func Test_HandlerPersistence_RoundTrip(t *testing.T) { + t.Parallel() + + ctx := context.Background() + e := echo.New() + h1, b1 := newTestHandler(t) + + _, err := b1.CreateInstanceProfile("PersistedProfile", "/") + require.NoError(t, err) + + tagReq := iamRequest("TagInstanceProfile", map[string]string{ + "InstanceProfileName": "PersistedProfile", + "Tags.member.1.Key": "Team", + "Tags.member.1.Value": "Platform", + }) + tagRec := httptest.NewRecorder() + require.NoError(t, h1.Handler()(e.NewContext(tagReq, tagRec))) + require.Equal(t, http.StatusOK, tagRec.Code) + + _, err = b1.CreateUser("keyholder", "/", "") + require.NoError(t, err) + _, err = b1.UploadSSHPublicKey("keyholder", "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDPersistedKeyBody") + require.NoError(t, err) + + blob := h1.Snapshot(ctx) + require.NotEmpty(t, blob) + + // Restore into a brand-new handler/backend pair. + h2, _ := newTestHandler(t) + require.NoError(t, h2.Restore(ctx, blob)) + + // Instance profile tags must have round-tripped. + listTagsReq := iamRequest("ListInstanceProfileTags", map[string]string{ + "InstanceProfileName": "PersistedProfile", + }) + listTagsRec := httptest.NewRecorder() + require.NoError(t, h2.Handler()(e.NewContext(listTagsReq, listTagsRec))) + assert.Equal(t, http.StatusOK, listTagsRec.Code) + assert.Contains(t, listTagsRec.Body.String(), "Platform") + + // SSH public keys (comprehensiveBackend state) must have round-tripped. + listKeysReq := iamRequest("ListSSHPublicKeys", map[string]string{"UserName": "keyholder"}) + listKeysRec := httptest.NewRecorder() + require.NoError(t, h2.Handler()(e.NewContext(listKeysReq, listKeysRec))) + assert.Equal(t, http.StatusOK, listKeysRec.Code) + assert.Contains(t, listKeysRec.Body.String(), "keyholder") +} + +// Test_ErrorHTTPStatusCodes verifies each IAM error sentinel maps to the +// exact HTTP status code AWS documents per-operation (e.g. NoSuchEntity=404, +// EntityAlreadyExists=409), not a blanket 400. See e.g. +// https://docs.aws.amazon.com/IAM/latest/APIReference/API_CreateUser.html#API_CreateUser_Errors +// and https://docs.aws.amazon.com/IAM/latest/APIReference/API_DeleteUser.html#API_DeleteUser_Errors. +func Test_ErrorHTTPStatusCodes(t *testing.T) { + t.Parallel() + + tests := []struct { + setup func(t *testing.T, b *iam.InMemoryBackend) + params map[string]string + name string + action string + wantCode string + wantStatus int + }{ + { + name: "NoSuchEntity_is_404", + action: "GetUser", + params: map[string]string{"UserName": "ghost"}, + setup: func(_ *testing.T, _ *iam.InMemoryBackend) {}, + wantCode: "NoSuchEntity", + wantStatus: http.StatusNotFound, + }, + { + name: "EntityAlreadyExists_is_409", + action: "CreateUser", + params: map[string]string{"UserName": "dup"}, + setup: func(t *testing.T, b *iam.InMemoryBackend) { + t.Helper() + _, err := b.CreateUser("dup", "/", "") + require.NoError(t, err) + }, + wantCode: "EntityAlreadyExists", + wantStatus: http.StatusConflict, + }, + { + name: "DeleteConflict_is_409", + action: "DeleteUser", + params: map[string]string{"UserName": "attached"}, + setup: func(t *testing.T, b *iam.InMemoryBackend) { + t.Helper() + _, err := b.CreateUser("attached", "/", "") + require.NoError(t, err) + pol, err := b.CreatePolicy( + "StuckPolicy", "/", + `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]}`, + ) + require.NoError(t, err) + require.NoError(t, b.AttachUserPolicy("attached", pol.Arn)) + }, + wantCode: "DeleteConflict", + wantStatus: http.StatusConflict, + }, + { + name: "LimitExceeded_is_409", + action: "CreateAccessKey", + params: map[string]string{"UserName": "keyed"}, + setup: func(t *testing.T, b *iam.InMemoryBackend) { + t.Helper() + _, err := b.CreateUser("keyed", "/", "") + require.NoError(t, err) + _, err = b.CreateAccessKey("keyed") + require.NoError(t, err) + _, err = b.CreateAccessKey("keyed") + require.NoError(t, err) + }, + wantCode: "LimitExceeded", + wantStatus: http.StatusConflict, + }, + { + name: "MalformedPolicyDocument_is_400", + action: "CreatePolicy", + params: map[string]string{"PolicyName": "Bad", "PolicyDocument": "not-json"}, + setup: func(_ *testing.T, _ *iam.InMemoryBackend) {}, + wantCode: "MalformedPolicyDocument", + wantStatus: http.StatusBadRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + e := echo.New() + h, b := newTestHandler(t) + tt.setup(t, b) + + req := iamRequest(tt.action, tt.params) + rec := httptest.NewRecorder() + require.NoError(t, h.Handler()(e.NewContext(req, rec))) + assert.Equal(t, tt.wantStatus, rec.Code) + + var errResp iam.ErrorResponse + require.NoError(t, xml.Unmarshal(rec.Body.Bytes(), &errResp)) + assert.Equal(t, tt.wantCode, errResp.Error.Code) + }) + } +} + +// Test_CreateAcceptsTagsAtCreation verifies that CreateUser, CreateRole, and +// CreatePolicy honor an optional Tags.member.N parameter at creation time, per +// real AWS IAM (e.g. CreateRole: "A list of tags that you want to attach to +// the new role"). Previously the parameter was silently accepted and dropped +// — the entity was created untagged and callers had to issue a separate +// Tag* call. CreateGroup deliberately does NOT support Tags at creation +// (real AWS CreateGroup has no Tags parameter), so it is not covered here. +func Test_CreateAcceptsTagsAtCreation(t *testing.T) { + t.Parallel() + + tagParams := map[string]string{ + "Tags.member.1.Key": "Team", + "Tags.member.1.Value": "Platform", + } + + tests := []struct { + params map[string]string + name string + action string + wantContain string + }{ + { + name: "CreateUser_with_tags", + action: "CreateUser", + params: mergeParams(map[string]string{"UserName": "tagged-user"}, tagParams), + }, + { + name: "CreateRole_with_tags", + action: "CreateRole", + params: mergeParams(map[string]string{ + "RoleName": "tagged-role", + "AssumeRolePolicyDocument": "{}", + }, tagParams), + }, + { + name: "CreatePolicy_with_tags", + action: "CreatePolicy", + params: mergeParams(map[string]string{ + "PolicyName": "tagged-policy", + "PolicyDocument": `{"Version":"2012-10-17",` + + `"Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]}`, + }, tagParams), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + e := echo.New() + h, _ := newTestHandler(t) + + req := iamRequest(tt.action, tt.params) + rec := httptest.NewRecorder() + require.NoError(t, h.Handler()(e.NewContext(req, rec))) + require.Equal(t, http.StatusOK, rec.Code) + + body := rec.Body.String() + assert.Contains(t, body, "Team") + assert.Contains(t, body, "Platform") + }) + } +} + +// mergeParams returns a new map containing all entries of base and extra. +func mergeParams(base, extra map[string]string) map[string]string { + out := make(map[string]string, len(base)+len(extra)) + maps.Copy(out, base) + + maps.Copy(out, extra) + + return out +} diff --git a/services/iot/backend.go b/services/iot/backend.go index 60932f1a1..123653058 100644 --- a/services/iot/backend.go +++ b/services/iot/backend.go @@ -16,6 +16,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/collections" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) var ( @@ -58,68 +59,77 @@ type RuleDispatcher interface { // InMemoryBackend is the in-memory implementation of StorageBackend. type InMemoryBackend struct { - dispatcher RuleDispatcher - shadows map[shadowKey]*ThingShadow // thing+name → shadow - customMetrics map[string]*CustomMetric - resourceTags map[string]map[string]string - rules map[string]*TopicRule - certificateTransfers map[string]string - thingBillingGroups map[string]string - thingThingGroups map[string][]string - packageVersionSboms map[string]*SbomDocument - jobTargets map[string][]string - fleetMetrics map[string]*FleetMetric - securityProfileTargets map[string][]string - thingPrincipals map[string][]string - auditMitigationTasks map[string]string - auditTasks map[string]string - thingTypes map[string]*ThingType - thingGroups map[string]*ThingGroup - thingGroupMembers map[string][]string - certificates map[string]*Certificate - policyVersions map[string][]*PolicyVersion - topicRuleDestinations map[string]*TopicRuleDestination - certificateProviders map[string]*CertificateProvider - jobs map[string]*Job - jobExecutions map[string]*JobExecution - jobTemplates map[string]*JobTemplate - roleAliases map[string]*RoleAlias - domainConfigs map[string]*DomainConfiguration - dimensions map[string]*Dimension - provTemplateVersions map[string][]*ProvisioningTemplateVersion - authorizers map[string]*Authorizer - billingGroups map[string]*BillingGroup - scheduledAudits map[string]*ScheduledAudit - mitigationActions map[string]*MitigationAction - securityProfiles map[string]*SecurityProfile - caCertificates map[string]*CACertificate - streams map[string]*IoTStream - policyTargets map[string][]string - policies map[string]*Policy - provTemplates map[string]*ProvisioningTemplate - things map[string]*Thing + dispatcher RuleDispatcher + shadows map[shadowKey]*ThingShadow // thing+name → shadow + resourceTags map[string]map[string]string + certificateTransfers map[string]string + thingBillingGroups map[string]string + thingThingGroups map[string][]string + packageVersionSboms map[string]*SbomDocument + jobTargets map[string][]string + securityProfileTargets map[string][]string + thingPrincipals map[string][]string + auditMitigationTasks map[string]string + auditTasks map[string]string + thingGroupMembers map[string][]string + policyVersions map[string][]*PolicyVersion + provTemplateVersions map[string][]*ProvisioningTemplateVersion + policyTargets map[string][]string + + // registry lets Reset collapse every converted resource table's + // lifecycle to one call (registry.ResetAll()) instead of hand-rolled + // re-initialization. See store_setup.go's registerAllTables for the + // full list of tables and the (documented) fields left as raw maps + // above/below instead. + registry *store.Registry + + customMetrics *store.Table[CustomMetric] + rules *store.Table[TopicRule] + fleetMetrics *store.Table[FleetMetric] + thingTypes *store.Table[ThingType] + thingGroups *store.Table[ThingGroup] + certificates *store.Table[Certificate] + topicRuleDestinations *store.Table[TopicRuleDestination] + certificateProviders *store.Table[CertificateProvider] + jobs *store.Table[Job] + jobExecutions *store.Table[JobExecution] + jobTemplates *store.Table[JobTemplate] + roleAliases *store.Table[RoleAlias] + domainConfigs *store.Table[DomainConfiguration] + dimensions *store.Table[Dimension] + authorizers *store.Table[Authorizer] + billingGroups *store.Table[BillingGroup] + scheduledAudits *store.Table[ScheduledAudit] + mitigationActions *store.Table[MitigationAction] + securityProfiles *store.Table[SecurityProfile] + caCertificates *store.Table[CACertificate] + streams *store.Table[IoTStream] + policies *store.Table[Policy] + provTemplates *store.Table[ProvisioningTemplate] + things *store.Table[Thing] + auditTaskObjects *store.Table[AuditTask] + otaUpdates *store.Table[OTAUpdate] + iotPackages *store.Table[IoTPackage] + auditSuppressions *store.Table[AuditSuppression] + auditFindings *store.Table[AuditFinding] + v2LoggingLevels *store.Table[V2LoggingLevel] + commands *store.Table[IoTCommand] + registrationTasks *store.Table[ThingRegistrationTask] + auditMitigationTaskObjects *store.Table[AuditMitigationTask] + detectMitigationTasks *store.Table[DetectMitigationTask] + activeViolations *store.Table[ActiveViolation] + auditConfiguration *AccountAuditConfiguration - auditTaskObjects map[string]*AuditTask - otaUpdates map[string]*OTAUpdate - iotPackages map[string]*IoTPackage packageVersions2 map[string]map[string]*IoTPackageVersion packageConfig *PackageConfiguration - auditSuppressions map[string]*AuditSuppression - auditFindings map[string]*AuditFinding v2LoggingOptions *V2LoggingOptions - v2LoggingLevels map[string]*V2LoggingLevel loggingOptions *LoggingOptions eventConfigurations *EventConfigurations - commands map[string]*IoTCommand commandExecutions map[string]*IoTCommandExecution thingIndexingConfig *ThingIndexingConfiguration thingGroupIndexingConfig *ThingGroupIndexingConfiguration - registrationTasks map[string]*ThingRegistrationTask - auditMitigationTaskObjects map[string]*AuditMitigationTask auditMitigationExecutions map[string][]*AuditMitigationActionExecution - detectMitigationTasks map[string]*DetectMitigationTask detectMitigationExecutions map[string][]*DetectMitigationActionExecution - activeViolations map[string]*ActiveViolation accountEncryptionConfig *AccountEncryptionConfiguration sbomValidationResults map[string][]*SbomValidationResult metricValues map[string][]*MetricDatapoint @@ -142,10 +152,9 @@ const mqttDefaultPort = 1883 // NewInMemoryBackend creates a new InMemoryBackend with default values. func NewInMemoryBackend() *InMemoryBackend { - return &InMemoryBackend{ - things: make(map[string]*Thing), - policies: make(map[string]*Policy), - rules: make(map[string]*TopicRule), + b := &InMemoryBackend{ + registry: store.NewRegistry(), + certificateTransfers: make(map[string]string), thingBillingGroups: make(map[string]string), thingThingGroups: make(map[string][]string), @@ -156,48 +165,16 @@ func NewInMemoryBackend() *InMemoryBackend { thingPrincipals: make(map[string][]string), auditMitigationTasks: make(map[string]string), auditTasks: make(map[string]string), - thingTypes: make(map[string]*ThingType), - thingGroups: make(map[string]*ThingGroup), thingGroupMembers: make(map[string][]string), - certificates: make(map[string]*Certificate), policyVersions: make(map[string][]*PolicyVersion), - topicRuleDestinations: make(map[string]*TopicRuleDestination), - certificateProviders: make(map[string]*CertificateProvider), - jobs: make(map[string]*Job), - jobExecutions: make(map[string]*JobExecution), - jobTemplates: make(map[string]*JobTemplate), - roleAliases: make(map[string]*RoleAlias), - domainConfigs: make(map[string]*DomainConfiguration), - provTemplates: make(map[string]*ProvisioningTemplate), provTemplateVersions: make(map[string][]*ProvisioningTemplateVersion), - authorizers: make(map[string]*Authorizer), - billingGroups: make(map[string]*BillingGroup), - scheduledAudits: make(map[string]*ScheduledAudit), - mitigationActions: make(map[string]*MitigationAction), - securityProfiles: make(map[string]*SecurityProfile), - caCertificates: make(map[string]*CACertificate), - streams: make(map[string]*IoTStream), - fleetMetrics: make(map[string]*FleetMetric), - customMetrics: make(map[string]*CustomMetric), - dimensions: make(map[string]*Dimension), resourceTags: make(map[string]map[string]string), - auditTaskObjects: make(map[string]*AuditTask), - otaUpdates: make(map[string]*OTAUpdate), - iotPackages: make(map[string]*IoTPackage), packageVersions2: make(map[string]map[string]*IoTPackageVersion), - auditSuppressions: make(map[string]*AuditSuppression), - auditFindings: make(map[string]*AuditFinding), - v2LoggingLevels: make(map[string]*V2LoggingLevel), shadows: make(map[shadowKey]*ThingShadow), - commands: make(map[string]*IoTCommand), commandExecutions: make(map[string]*IoTCommandExecution), - registrationTasks: make(map[string]*ThingRegistrationTask), - auditMitigationTaskObjects: make(map[string]*AuditMitigationTask), auditMitigationExecutions: make(map[string][]*AuditMitigationActionExecution), - detectMitigationTasks: make(map[string]*DetectMitigationTask), detectMitigationExecutions: make(map[string][]*DetectMitigationActionExecution), - activeViolations: make(map[string]*ActiveViolation), sbomValidationResults: make(map[string][]*SbomValidationResult), metricValues: make(map[string][]*MetricDatapoint), @@ -208,6 +185,10 @@ func NewInMemoryBackend() *InMemoryBackend { region: "us-east-1", mqttPort: mqttDefaultPort, } + + registerAllTables(b) + + return b } // NewInMemoryBackendWithConfig creates a new InMemoryBackend with the given account and region. @@ -221,17 +202,11 @@ func NewInMemoryBackendWithConfig(accountID, region string) *InMemoryBackend { // resetBatch3 clears all batch-3 backend state (called from Reset, lock held). func (b *InMemoryBackend) resetBatch3() { - b.otaUpdates = make(map[string]*OTAUpdate) - b.iotPackages = make(map[string]*IoTPackage) b.packageVersions2 = make(map[string]map[string]*IoTPackageVersion) b.packageConfig = nil - b.auditSuppressions = make(map[string]*AuditSuppression) - b.auditFindings = make(map[string]*AuditFinding) b.v2LoggingOptions = nil - b.v2LoggingLevels = make(map[string]*V2LoggingLevel) b.loggingOptions = nil b.eventConfigurations = nil - b.commands = make(map[string]*IoTCommand) b.commandExecutions = make(map[string]*IoTCommandExecution) } @@ -240,9 +215,22 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock() defer b.mu.Unlock() - b.things = make(map[string]*Thing) - b.policies = make(map[string]*Policy) - b.rules = make(map[string]*TopicRule) + // Clears every table registered in store_setup.go's registerAllTables + // (things, thingTypes, thingGroups, certificates, policies, rules, jobs, + // jobExecutions, jobTemplates, billingGroups, topicRuleDestinations, + // certificateProviders, roleAliases, domainConfigs, dimensions, + // authorizers, scheduledAudits, mitigationActions, securityProfiles, + // caCertificates, streams, provTemplates, auditTaskObjects, otaUpdates, + // iotPackages, auditSuppressions, auditFindings, v2LoggingLevels, + // commands, registrationTasks, auditMitigationTaskObjects, + // detectMitigationTasks, activeViolations, fleetMetrics, customMetrics) + // in one call instead of one hand-rolled make() per map. + // + // b.shadows is deliberately NOT part of the registry and NOT cleared + // here -- see store_setup.go's registerAllTables comment for why this + // preserves a pre-existing quirk byte-for-byte. + b.registry.ResetAll() + b.certificateTransfers = make(map[string]string) b.thingBillingGroups = make(map[string]string) b.thingThingGroups = make(map[string][]string) @@ -253,34 +241,11 @@ func (b *InMemoryBackend) Reset() { b.thingPrincipals = make(map[string][]string) b.auditMitigationTasks = make(map[string]string) b.auditTasks = make(map[string]string) - b.thingTypes = make(map[string]*ThingType) - b.thingGroups = make(map[string]*ThingGroup) b.thingGroupMembers = make(map[string][]string) - b.certificates = make(map[string]*Certificate) b.policyVersions = make(map[string][]*PolicyVersion) - b.topicRuleDestinations = make(map[string]*TopicRuleDestination) - b.certificateProviders = make(map[string]*CertificateProvider) - b.jobs = make(map[string]*Job) - b.jobExecutions = make(map[string]*JobExecution) - b.jobTemplates = make(map[string]*JobTemplate) - b.roleAliases = make(map[string]*RoleAlias) - b.domainConfigs = make(map[string]*DomainConfiguration) - b.provTemplates = make(map[string]*ProvisioningTemplate) b.provTemplateVersions = make(map[string][]*ProvisioningTemplateVersion) - b.authorizers = make(map[string]*Authorizer) - b.billingGroups = make(map[string]*BillingGroup) - b.scheduledAudits = make(map[string]*ScheduledAudit) - b.mitigationActions = make(map[string]*MitigationAction) - b.securityProfiles = make(map[string]*SecurityProfile) - b.caCertificates = make(map[string]*CACertificate) - b.streams = make(map[string]*IoTStream) - b.fleetMetrics = make(map[string]*FleetMetric) - b.customMetrics = make(map[string]*CustomMetric) - b.dimensions = make(map[string]*Dimension) b.resourceTags = make(map[string]map[string]string) - b.auditTaskObjects = make(map[string]*AuditTask) b.resetBatch3() - b.registrationTasks = make(map[string]*ThingRegistrationTask) b.registrationCode = "" b.defaultAuthorizer = "" b.auditConfiguration = nil @@ -303,11 +268,8 @@ func (b *InMemoryBackend) resetFinalOps() { // resetDeviceDefender clears all Device Defender backend state (audit + detect // mitigation-action tasks and violations). Called from Reset, lock held. func (b *InMemoryBackend) resetDeviceDefender() { - b.auditMitigationTaskObjects = make(map[string]*AuditMitigationTask) b.auditMitigationExecutions = make(map[string][]*AuditMitigationActionExecution) - b.detectMitigationTasks = make(map[string]*DetectMitigationTask) b.detectMitigationExecutions = make(map[string][]*DetectMitigationActionExecution) - b.activeViolations = make(map[string]*ActiveViolation) b.violationEvents = nil } @@ -332,9 +294,9 @@ func (b *InMemoryBackend) GetRules() []*TopicRule { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*TopicRule, 0, len(b.rules)) + out := make([]*TopicRule, 0, b.rules.Len()) - for _, r := range b.rules { + for _, r := range b.rules.All() { out = append(out, cloneTopicRule(r)) } @@ -388,13 +350,6 @@ func cloneTopicRule(r *TopicRule) *TopicRule { } } -// clonePolicy creates a deep copy of a Policy. -func clonePolicy(p *Policy) *Policy { - cp := *p - - return &cp -} - // cloneSbomDocument creates a deep copy of a SbomDocument. func cloneSbomDocument(s *SbomDocument) *SbomDocument { if s == nil { @@ -432,7 +387,7 @@ func (b *InMemoryBackend) CreateThing(input *CreateThingInput) (*CreateThingOutp b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.things[input.ThingName]; exists { + if b.things.Has(input.ThingName) { return nil, fmt.Errorf("%w: thing %q already exists", ErrAlreadyExists, input.ThingName) } @@ -445,7 +400,7 @@ func (b *InMemoryBackend) CreateThing(input *CreateThingInput) (*CreateThingOutp arn := arn.Build("iot", b.region, b.accountID, fmt.Sprintf("thing/%s", input.ThingName)) id := uuid.NewString() - b.things[input.ThingName] = &Thing{ + b.things.Put(&Thing{ ThingName: input.ThingName, ThingTypeName: input.ThingTypeName, ThingType: input.ThingTypeName, @@ -454,7 +409,7 @@ func (b *InMemoryBackend) CreateThing(input *CreateThingInput) (*CreateThingOutp ARN: arn, Version: 1, CreatedAt: time.Now(), - } + }) return &CreateThingOutput{ ThingName: input.ThingName, @@ -468,7 +423,7 @@ func (b *InMemoryBackend) DescribeThing(thingName string) (*Thing, error) { b.mu.RLock() defer b.mu.RUnlock() - t, ok := b.things[thingName] + t, ok := b.things.Get(thingName) if !ok { return nil, fmt.Errorf("%w: %s", ErrThingNotFound, thingName) } @@ -481,11 +436,11 @@ func (b *InMemoryBackend) ListThings() []*Thing { b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.things) - out := make([]*Thing, 0, len(keys)) + items := b.things.Snapshot() + out := make([]*Thing, 0, len(items)) - for _, k := range keys { - out = append(out, cloneThing(b.things[k])) + for _, v := range items { + out = append(out, cloneThing(v)) } return out @@ -496,7 +451,7 @@ func (b *InMemoryBackend) DeleteThing(thingName string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.things[thingName]; !ok { + if !b.things.Has(thingName) { return fmt.Errorf("%w: %s", ErrThingNotFound, thingName) } @@ -504,7 +459,7 @@ func (b *InMemoryBackend) DeleteThing(thingName string) error { return fmt.Errorf("%w: thing %q has attached principals", ErrDeleteConflict, thingName) } - delete(b.things, thingName) + b.things.Delete(thingName) return nil } @@ -518,7 +473,7 @@ func (b *InMemoryBackend) CreateTopicRule(input *CreateTopicRuleInput) error { b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.rules[input.RuleName]; exists { + if b.rules.Has(input.RuleName) { return fmt.Errorf("%w: rule %q already exists", ErrAlreadyExists, input.RuleName) } @@ -539,7 +494,7 @@ func (b *InMemoryBackend) CreateTopicRule(input *CreateTopicRuleInput) error { sqlVersion = "2015-10-08" } - b.rules[input.RuleName] = &TopicRule{ + b.rules.Put(&TopicRule{ RuleName: input.RuleName, ARN: arn, SQL: payload.SQL, @@ -548,7 +503,7 @@ func (b *InMemoryBackend) CreateTopicRule(input *CreateTopicRuleInput) error { Actions: actions, Enabled: !payload.RuleDisabled, CreatedAt: time.Now(), - } + }) return nil } @@ -558,7 +513,7 @@ func (b *InMemoryBackend) GetTopicRule(ruleName string) (*TopicRule, error) { b.mu.RLock() defer b.mu.RUnlock() - r, ok := b.rules[ruleName] + r, ok := b.rules.Get(ruleName) if !ok { return nil, fmt.Errorf("%w: %s", ErrRuleNotFound, ruleName) } @@ -571,11 +526,11 @@ func (b *InMemoryBackend) ListTopicRules() []*TopicRule { b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.rules) - out := make([]*TopicRule, 0, len(keys)) + items := b.rules.Snapshot() + out := make([]*TopicRule, 0, len(items)) - for _, k := range keys { - out = append(out, cloneTopicRule(b.rules[k])) + for _, v := range items { + out = append(out, cloneTopicRule(v)) } return out @@ -586,11 +541,11 @@ func (b *InMemoryBackend) DeleteTopicRule(ruleName string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.rules[ruleName]; !ok { + if !b.rules.Has(ruleName) { return fmt.Errorf("%w: %s", ErrRuleNotFound, ruleName) } - delete(b.rules, ruleName) + b.rules.Delete(ruleName) return nil } @@ -604,20 +559,20 @@ func (b *InMemoryBackend) CreatePolicy(input *CreatePolicyInput) (*CreatePolicyO b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.policies[input.PolicyName]; exists { + if b.policies.Has(input.PolicyName) { return nil, fmt.Errorf("%w: policy %q already exists", ErrAlreadyExists, input.PolicyName) } arn := arn.Build("iot", b.region, b.accountID, fmt.Sprintf("policy/%s", input.PolicyName)) now := time.Now() - b.policies[input.PolicyName] = &Policy{ + b.policies.Put(&Policy{ PolicyName: input.PolicyName, PolicyDocument: input.PolicyDocument, ARN: arn, CreatedAt: now, LastModifiedAt: now, - } + }) // AWS automatically creates version "1" as the default on CreatePolicy. b.policyVersions[input.PolicyName] = []*PolicyVersion{ @@ -796,7 +751,7 @@ func (b *InMemoryBackend) CancelAuditMitigationActionsTask(input *CancelAuditMit defer b.mu.Unlock() b.auditMitigationTasks[input.TaskID] = string(JobStatusCanceled) - if t, ok := b.auditMitigationTaskObjects[input.TaskID]; ok { + if t, ok := b.auditMitigationTaskObjects.Get(input.TaskID); ok { t.TaskStatus = string(JobStatusCanceled) t.EndTime = float64(time.Now().Unix()) } @@ -819,7 +774,7 @@ func (b *InMemoryBackend) GetPolicy(policyName string) (*GetPolicyOutput, error) b.mu.RLock() defer b.mu.RUnlock() - p, ok := b.policies[policyName] + p, ok := b.policies.Get(policyName) if !ok { return nil, fmt.Errorf("%w: %s", ErrPolicyNotFound, policyName) } @@ -848,7 +803,7 @@ func (b *InMemoryBackend) DeletePolicy(policyName string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.policies[policyName]; !ok { + if !b.policies.Has(policyName) { return fmt.Errorf("%w: %s", ErrPolicyNotFound, policyName) } @@ -856,7 +811,7 @@ func (b *InMemoryBackend) DeletePolicy(policyName string) error { return fmt.Errorf("%w: policy %q has attached targets", ErrDeleteConflict, policyName) } - delete(b.policies, policyName) + b.policies.Delete(policyName) return nil } @@ -866,11 +821,11 @@ func (b *InMemoryBackend) ListPolicies() []*Policy { b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.policies) - out := make([]*Policy, 0, len(keys)) + items := b.policies.Snapshot() + out := make([]*Policy, 0, len(items)) - for _, k := range keys { - cp := *b.policies[k] + for _, v := range items { + cp := *v out = append(out, &cp) } @@ -882,7 +837,7 @@ func (b *InMemoryBackend) DisableTopicRule(ruleName string) error { b.mu.Lock() defer b.mu.Unlock() - r, ok := b.rules[ruleName] + r, ok := b.rules.Get(ruleName) if !ok { return fmt.Errorf("%w: %s", ErrRuleNotFound, ruleName) } @@ -897,7 +852,7 @@ func (b *InMemoryBackend) EnableTopicRule(ruleName string) error { b.mu.Lock() defer b.mu.Unlock() - r, ok := b.rules[ruleName] + r, ok := b.rules.Get(ruleName) if !ok { return fmt.Errorf("%w: %s", ErrRuleNotFound, ruleName) } @@ -916,7 +871,7 @@ func (b *InMemoryBackend) ReplaceTopicRule(input *ReplaceTopicRuleInput) error { b.mu.Lock() defer b.mu.Unlock() - r, ok := b.rules[input.RuleName] + r, ok := b.rules.Get(input.RuleName) if !ok { return fmt.Errorf("%w: %s", ErrRuleNotFound, input.RuleName) } @@ -954,7 +909,7 @@ func (b *InMemoryBackend) UpdateThing(input *UpdateThingInput) error { b.mu.Lock() defer b.mu.Unlock() - t, ok := b.things[input.ThingName] + t, ok := b.things.Get(input.ThingName) if !ok { return fmt.Errorf("%w: %s", ErrThingNotFound, input.ThingName) } @@ -994,7 +949,7 @@ func (b *InMemoryBackend) ListThingPrincipals(thingName string) ([]string, error b.mu.RLock() defer b.mu.RUnlock() - if _, ok := b.things[thingName]; !ok { + if !b.things.Has(thingName) { return nil, fmt.Errorf("%w: %s", ErrThingNotFound, thingName) } @@ -1030,7 +985,7 @@ func (b *InMemoryBackend) AddThingInternal(t Thing) { t.Version = 1 } - b.things[t.ThingName] = &t + b.things.Put(&t) } // AddPolicyInternal seeds a Policy directly into the backend for testing. @@ -1042,7 +997,7 @@ func (b *InMemoryBackend) AddPolicyInternal(p Policy) { p.ARN = arn.Build("iot", b.region, b.accountID, fmt.Sprintf("policy/%s", p.PolicyName)) } - b.policies[p.PolicyName] = &p + b.policies.Put(&p) } // AddRuleInternal seeds a TopicRule directly into the backend for testing. @@ -1058,7 +1013,7 @@ func (b *InMemoryBackend) AddRuleInternal(r TopicRule) { r.Actions = []RuleAction{} } - b.rules[r.RuleName] = &r + b.rules.Put(&r) } // ----------------------------------------------------------- @@ -1134,7 +1089,7 @@ func (b *InMemoryBackend) CreateThingType(input *CreateThingTypeInput) (*ThingTy b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.thingTypes[input.ThingTypeName]; exists { + if b.thingTypes.Has(input.ThingTypeName) { return nil, fmt.Errorf("%w: thing type %q already exists", ErrAlreadyExists, input.ThingTypeName) } @@ -1148,7 +1103,7 @@ func (b *InMemoryBackend) CreateThingType(input *CreateThingTypeInput) (*ThingTy CreatedAt: time.Now(), } - b.thingTypes[input.ThingTypeName] = tt + b.thingTypes.Put(tt) return tt, nil } @@ -1158,7 +1113,7 @@ func (b *InMemoryBackend) DescribeThingType(thingTypeName string) (*ThingType, e b.mu.RLock() defer b.mu.RUnlock() - tt, ok := b.thingTypes[thingTypeName] + tt, ok := b.thingTypes.Get(thingTypeName) if !ok { return nil, fmt.Errorf("%w: %s", ErrThingTypeNotFound, thingTypeName) } @@ -1173,11 +1128,11 @@ func (b *InMemoryBackend) ListThingTypes() []*ThingType { b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.thingTypes) - out := make([]*ThingType, 0, len(keys)) + items := b.thingTypes.Snapshot() + out := make([]*ThingType, 0, len(items)) - for _, k := range keys { - cp := *b.thingTypes[k] + for _, v := range items { + cp := *v out = append(out, &cp) } @@ -1189,7 +1144,7 @@ func (b *InMemoryBackend) DeprecateThingType(input *DeprecateThingTypeInput) err b.mu.Lock() defer b.mu.Unlock() - tt, ok := b.thingTypes[input.ThingTypeName] + tt, ok := b.thingTypes.Get(input.ThingTypeName) if !ok { return fmt.Errorf("%w: %s", ErrThingTypeNotFound, input.ThingTypeName) } @@ -1210,7 +1165,7 @@ func (b *InMemoryBackend) DeleteThingType(thingTypeName string) error { b.mu.Lock() defer b.mu.Unlock() - tt, ok := b.thingTypes[thingTypeName] + tt, ok := b.thingTypes.Get(thingTypeName) if !ok { return fmt.Errorf("%w: %s", ErrThingTypeNotFound, thingTypeName) } @@ -1219,7 +1174,7 @@ func (b *InMemoryBackend) DeleteThingType(thingTypeName string) error { return fmt.Errorf("%w: thing type %q must be deprecated before deletion", ErrValidation, thingTypeName) } - delete(b.thingTypes, thingTypeName) + b.thingTypes.Delete(thingTypeName) return nil } @@ -1237,7 +1192,7 @@ func (b *InMemoryBackend) CreateThingGroup(input *CreateThingGroupInput) (*Thing b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.thingGroups[input.ThingGroupName]; exists { + if b.thingGroups.Has(input.ThingGroupName) { return nil, fmt.Errorf("%w: thing group %q already exists", ErrAlreadyExists, input.ThingGroupName) } @@ -1261,7 +1216,7 @@ func (b *InMemoryBackend) CreateThingGroup(input *CreateThingGroupInput) (*Thing CreatedAt: time.Now(), } - b.thingGroups[input.ThingGroupName] = tg + b.thingGroups.Put(tg) b.thingGroupMembers[input.ThingGroupName] = []string{} return tg, nil @@ -1272,7 +1227,7 @@ func (b *InMemoryBackend) DescribeThingGroup(thingGroupName string) (*ThingGroup b.mu.RLock() defer b.mu.RUnlock() - tg, ok := b.thingGroups[thingGroupName] + tg, ok := b.thingGroups.Get(thingGroupName) if !ok { return nil, fmt.Errorf("%w: %s", ErrThingGroupNotFound, thingGroupName) } @@ -1291,11 +1246,11 @@ func (b *InMemoryBackend) ListThingGroups() []*ThingGroup { b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.thingGroups) - out := make([]*ThingGroup, 0, len(keys)) + items := b.thingGroups.Snapshot() + out := make([]*ThingGroup, 0, len(items)) - for _, k := range keys { - tg := b.thingGroups[k] + for _, v := range items { + tg := v cp := *tg cp.Attributes = make(map[string]string, len(tg.Attributes)) maps.Copy(cp.Attributes, tg.Attributes) @@ -1316,7 +1271,7 @@ func (b *InMemoryBackend) UpdateThingGroup(input *UpdateThingGroupInput) (int64, b.mu.Lock() defer b.mu.Unlock() - tg, ok := b.thingGroups[input.ThingGroupName] + tg, ok := b.thingGroups.Get(input.ThingGroupName) if !ok { return 0, fmt.Errorf("%w: %s", ErrThingGroupNotFound, input.ThingGroupName) } @@ -1347,11 +1302,11 @@ func (b *InMemoryBackend) DeleteThingGroup(thingGroupName string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.thingGroups[thingGroupName]; !ok { + if !b.thingGroups.Has(thingGroupName) { return fmt.Errorf("%w: %s", ErrThingGroupNotFound, thingGroupName) } - delete(b.thingGroups, thingGroupName) + b.thingGroups.Delete(thingGroupName) delete(b.thingGroupMembers, thingGroupName) return nil @@ -1391,7 +1346,7 @@ func (b *InMemoryBackend) ListThingsInThingGroup(input *ListThingsInThingGroupIn b.mu.RLock() defer b.mu.RUnlock() - if _, ok := b.thingGroups[input.ThingGroupName]; !ok { + if !b.thingGroups.Has(input.ThingGroupName) { return nil, fmt.Errorf("%w: %s", ErrThingGroupNotFound, input.ThingGroupName) } @@ -1433,7 +1388,7 @@ func (b *InMemoryBackend) CreateCertificateFromCsr(input *CreateCertificateFromC } cert := b.newCertificate(fakePEM, status) - b.certificates[cert.CertificateID] = cert + b.certificates.Put(cert) return cert, nil } @@ -1454,7 +1409,7 @@ func (b *InMemoryBackend) RegisterCertificate(input *RegisterCertificateInput) ( } cert := b.newCertificate(pem, status) - b.certificates[cert.CertificateID] = cert + b.certificates.Put(cert) return cert, nil } @@ -1469,7 +1424,7 @@ func (b *InMemoryBackend) DescribeCertificate(certificateID string) (*Certificat b.mu.RLock() defer b.mu.RUnlock() - cert, ok := b.certificates[certificateID] + cert, ok := b.certificates.Get(certificateID) if !ok { return nil, fmt.Errorf("%w: %s", ErrCertificateNotFound, certificateID) } @@ -1484,11 +1439,11 @@ func (b *InMemoryBackend) ListCertificates() []*Certificate { b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.certificates) - out := make([]*Certificate, 0, len(keys)) + items := b.certificates.Snapshot() + out := make([]*Certificate, 0, len(items)) - for _, k := range keys { - cp := *b.certificates[k] + for _, v := range items { + cp := *v out = append(out, &cp) } @@ -1520,7 +1475,7 @@ func (b *InMemoryBackend) UpdateCertificate(input *UpdateCertificateInput) error b.mu.Lock() defer b.mu.Unlock() - cert, ok := b.certificates[input.CertificateID] + cert, ok := b.certificates.Get(input.CertificateID) if !ok { return fmt.Errorf("%w: %s", ErrCertificateNotFound, input.CertificateID) } @@ -1536,7 +1491,7 @@ func (b *InMemoryBackend) DeleteCertificate(certificateID string) error { b.mu.Lock() defer b.mu.Unlock() - cert, ok := b.certificates[certificateID] + cert, ok := b.certificates.Get(certificateID) if !ok { return fmt.Errorf("%w: %s", ErrCertificateNotFound, certificateID) } @@ -1545,7 +1500,7 @@ func (b *InMemoryBackend) DeleteCertificate(certificateID string) error { return fmt.Errorf("%w: certificate %q must be deactivated before deletion", ErrDeleteConflict, certificateID) } - delete(b.certificates, certificateID) + b.certificates.Delete(certificateID) return nil } @@ -1582,7 +1537,7 @@ func (b *InMemoryBackend) ListAttachedPolicies(input *ListAttachedPoliciesInput) for policyName, targets := range b.policyTargets { if slices.Contains(targets, input.Target) { - if p, ok := b.policies[policyName]; ok { + if p, ok := b.policies.Get(policyName); ok { cp := *p out = append(out, &cp) } @@ -1608,7 +1563,7 @@ func (b *InMemoryBackend) CreatePolicyVersion(input *CreatePolicyVersionInput) ( b.mu.Lock() defer b.mu.Unlock() - p, ok := b.policies[input.PolicyName] + p, ok := b.policies.Get(input.PolicyName) if !ok { return nil, fmt.Errorf("%w: %s", ErrPolicyNotFound, input.PolicyName) } @@ -1669,7 +1624,7 @@ func (b *InMemoryBackend) ListPolicyVersions(policyName string) ([]*PolicyVersio b.mu.RLock() defer b.mu.RUnlock() - if _, ok := b.policies[policyName]; !ok { + if !b.policies.Has(policyName) { return nil, fmt.Errorf("%w: %s", ErrPolicyNotFound, policyName) } @@ -1768,7 +1723,7 @@ func (b *InMemoryBackend) CreateTopicRuleDestination( dest.Status = statusEnabled } - b.topicRuleDestinations[arn] = dest + b.topicRuleDestinations.Put(dest) return dest, nil } @@ -1778,7 +1733,7 @@ func (b *InMemoryBackend) GetTopicRuleDestination(arn string) (*TopicRuleDestina b.mu.RLock() defer b.mu.RUnlock() - dest, ok := b.topicRuleDestinations[arn] + dest, ok := b.topicRuleDestinations.Get(arn) if !ok { return nil, fmt.Errorf("%w: %s", ErrTopicRuleDestinationNotFound, arn) } @@ -1793,11 +1748,11 @@ func (b *InMemoryBackend) ListTopicRuleDestinations() []*TopicRuleDestination { b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.topicRuleDestinations) - out := make([]*TopicRuleDestination, 0, len(keys)) + items := b.topicRuleDestinations.Snapshot() + out := make([]*TopicRuleDestination, 0, len(items)) - for _, k := range keys { - cp := *b.topicRuleDestinations[k] + for _, v := range items { + cp := *v out = append(out, &cp) } @@ -1809,7 +1764,7 @@ func (b *InMemoryBackend) UpdateTopicRuleDestination(input *UpdateTopicRuleDesti b.mu.Lock() defer b.mu.Unlock() - dest, ok := b.topicRuleDestinations[input.ARN] + dest, ok := b.topicRuleDestinations.Get(input.ARN) if !ok { return fmt.Errorf("%w: %s", ErrTopicRuleDestinationNotFound, input.ARN) } @@ -1824,11 +1779,11 @@ func (b *InMemoryBackend) DeleteTopicRuleDestination(arn string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.topicRuleDestinations[arn]; !ok { + if !b.topicRuleDestinations.Has(arn) { return fmt.Errorf("%w: %s", ErrTopicRuleDestinationNotFound, arn) } - delete(b.topicRuleDestinations, arn) + b.topicRuleDestinations.Delete(arn) return nil } @@ -1848,7 +1803,7 @@ func (b *InMemoryBackend) CreateCertificateProvider( b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.certificateProviders[input.CertificateProviderName]; exists { + if b.certificateProviders.Has(input.CertificateProviderName) { return nil, fmt.Errorf("%w: certificate provider %q already exists", ErrAlreadyExists, input.CertificateProviderName) } @@ -1868,7 +1823,7 @@ func (b *InMemoryBackend) CreateCertificateProvider( LastModifiedAt: time.Now(), } - b.certificateProviders[input.CertificateProviderName] = cp + b.certificateProviders.Put(cp) return cp, nil } @@ -1878,7 +1833,7 @@ func (b *InMemoryBackend) DescribeCertificateProvider(name string) (*Certificate b.mu.RLock() defer b.mu.RUnlock() - cp, ok := b.certificateProviders[name] + cp, ok := b.certificateProviders.Get(name) if !ok { return nil, fmt.Errorf("%w: %s", ErrCertificateProviderNotFound, name) } @@ -1895,13 +1850,13 @@ func (b *InMemoryBackend) ListCertificateProviders() []*CertificateProvider { b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.certificateProviders) - out := make([]*CertificateProvider, 0, len(keys)) + items := b.certificateProviders.Snapshot() + out := make([]*CertificateProvider, 0, len(items)) - for _, k := range keys { - cp := *b.certificateProviders[k] - cp.AccountDefaultForOperations = make([]string, len(b.certificateProviders[k].AccountDefaultForOperations)) - copy(cp.AccountDefaultForOperations, b.certificateProviders[k].AccountDefaultForOperations) + for _, v := range items { + cp := *v + cp.AccountDefaultForOperations = make([]string, len(v.AccountDefaultForOperations)) + copy(cp.AccountDefaultForOperations, v.AccountDefaultForOperations) out = append(out, &cp) } @@ -1913,7 +1868,7 @@ func (b *InMemoryBackend) UpdateCertificateProvider(input *UpdateCertificateProv b.mu.Lock() defer b.mu.Unlock() - cp, ok := b.certificateProviders[input.CertificateProviderName] + cp, ok := b.certificateProviders.Get(input.CertificateProviderName) if !ok { return fmt.Errorf("%w: %s", ErrCertificateProviderNotFound, input.CertificateProviderName) } @@ -1938,11 +1893,11 @@ func (b *InMemoryBackend) DeleteCertificateProvider(name string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.certificateProviders[name]; !ok { + if !b.certificateProviders.Has(name) { return fmt.Errorf("%w: %s", ErrCertificateProviderNotFound, name) } - delete(b.certificateProviders, name) + b.certificateProviders.Delete(name) return nil } @@ -1968,7 +1923,7 @@ func (b *InMemoryBackend) GetThingShadow(thingName, shadowName string) (*ThingSh b.mu.RLock() defer b.mu.RUnlock() - if _, ok := b.things[thingName]; !ok { + if !b.things.Has(thingName) { return nil, fmt.Errorf("%w: %s", ErrThingNotFound, thingName) } @@ -1988,7 +1943,7 @@ func (b *InMemoryBackend) UpdateThingShadow(thingName, shadowName string, state b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.things[thingName]; !ok { + if !b.things.Has(thingName) { return nil, fmt.Errorf("%w: %s", ErrThingNotFound, thingName) } @@ -2016,7 +1971,7 @@ func (b *InMemoryBackend) DeleteThingShadow(thingName, shadowName string) error b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.things[thingName]; !ok { + if !b.things.Has(thingName) { return fmt.Errorf("%w: %s", ErrThingNotFound, thingName) } @@ -2035,7 +1990,7 @@ func (b *InMemoryBackend) ListNamedShadowsForThing(thingName string) ([]string, b.mu.RLock() defer b.mu.RUnlock() - if _, ok := b.things[thingName]; !ok { + if !b.things.Has(thingName) { return nil, fmt.Errorf("%w: %s", ErrThingNotFound, thingName) } diff --git a/services/iot/backend_batch2.go b/services/iot/backend_batch2.go index 24efbcf75..caa500a7e 100644 --- a/services/iot/backend_batch2.go +++ b/services/iot/backend_batch2.go @@ -57,7 +57,7 @@ func (b *InMemoryBackend) RegisterCACertificate(pem, status string) (*CACertific if ca.Status == "" { ca.Status = "ACTIVE" } - b.caCertificates[id] = ca + b.caCertificates.Put(ca) return cloneCACert(ca), nil } @@ -66,7 +66,7 @@ func (b *InMemoryBackend) DescribeCACertificate(id string) (*CACertificate, erro b.mu.RLock() defer b.mu.RUnlock() - ca, ok := b.caCertificates[id] + ca, ok := b.caCertificates.Get(id) if !ok { return nil, fmt.Errorf("CA certificate %q not found: %w", id, ErrResourceNotFound) } @@ -78,9 +78,9 @@ func (b *InMemoryBackend) ListCACertificates() []*CACertificate { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*CACertificate, 0, len(b.caCertificates)) - for _, k := range sortedKeys(b.caCertificates) { - out = append(out, cloneCACert(b.caCertificates[k])) + out := make([]*CACertificate, 0, b.caCertificates.Len()) + for _, v := range b.caCertificates.Snapshot() { + out = append(out, cloneCACert(v)) } return out @@ -90,7 +90,7 @@ func (b *InMemoryBackend) UpdateCACertificate(id, status string) error { b.mu.Lock() defer b.mu.Unlock() - ca, ok := b.caCertificates[id] + ca, ok := b.caCertificates.Get(id) if !ok { return fmt.Errorf("CA certificate %q not found: %w", id, ErrResourceNotFound) } @@ -106,10 +106,10 @@ func (b *InMemoryBackend) DeleteCACertificate(id string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.caCertificates[id]; !ok { + if !b.caCertificates.Has(id) { return fmt.Errorf("CA certificate %q not found: %w", id, ErrResourceNotFound) } - delete(b.caCertificates, id) + b.caCertificates.Delete(id) return nil } @@ -119,8 +119,8 @@ func (b *InMemoryBackend) ListCertificatesByCA(caID string) []*Certificate { defer b.mu.RUnlock() var out []*Certificate - for _, k := range sortedKeys(b.certificates) { - cert := b.certificates[k] + for _, v := range b.certificates.Snapshot() { + cert := v if cert.CACertificateID == caID { cp := *cert out = append(out, &cp) @@ -180,7 +180,7 @@ func (b *InMemoryBackend) CreateStream(input *CreateStreamInput) (*IoTStream, er b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.streams[input.StreamID]; exists { + if b.streams.Has(input.StreamID) { return nil, fmt.Errorf("stream %q already exists: %w", input.StreamID, ErrAlreadyExists) } now := float64(time.Now().Unix()) @@ -195,7 +195,7 @@ func (b *InMemoryBackend) CreateStream(input *CreateStreamInput) (*IoTStream, er CreatedAt: now, LastUpdated: now, } - b.streams[input.StreamID] = s + b.streams.Put(s) return cloneStream(s), nil } @@ -204,7 +204,7 @@ func (b *InMemoryBackend) DescribeStream(id string) (*IoTStream, error) { b.mu.RLock() defer b.mu.RUnlock() - s, ok := b.streams[id] + s, ok := b.streams.Get(id) if !ok { return nil, fmt.Errorf("stream %q not found: %w", id, ErrResourceNotFound) } @@ -216,9 +216,9 @@ func (b *InMemoryBackend) ListStreams() []*IoTStream { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*IoTStream, 0, len(b.streams)) - for _, k := range sortedKeys(b.streams) { - out = append(out, cloneStream(b.streams[k])) + out := make([]*IoTStream, 0, b.streams.Len()) + for _, v := range b.streams.Snapshot() { + out = append(out, cloneStream(v)) } return out @@ -228,7 +228,7 @@ func (b *InMemoryBackend) UpdateStream(id, description, roleARN string, files [] b.mu.Lock() defer b.mu.Unlock() - s, ok := b.streams[id] + s, ok := b.streams.Get(id) if !ok { return nil, fmt.Errorf("stream %q not found: %w", id, ErrResourceNotFound) } @@ -251,10 +251,10 @@ func (b *InMemoryBackend) DeleteStream(id string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.streams[id]; !ok { + if !b.streams.Has(id) { return fmt.Errorf("stream %q not found: %w", id, ErrResourceNotFound) } - delete(b.streams, id) + b.streams.Delete(id) return nil } @@ -305,7 +305,7 @@ func (b *InMemoryBackend) CreateFleetMetric(input *CreateFleetMetricInput) (*Fle b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.fleetMetrics[input.MetricName]; exists { + if b.fleetMetrics.Has(input.MetricName) { return nil, fmt.Errorf("fleet metric %q already exists: %w", input.MetricName, ErrAlreadyExists) } now := float64(time.Now().Unix()) @@ -323,7 +323,7 @@ func (b *InMemoryBackend) CreateFleetMetric(input *CreateFleetMetricInput) (*Fle CreationDate: now, LastModified: now, } - b.fleetMetrics[input.MetricName] = fm + b.fleetMetrics.Put(fm) return cloneFleetMetric(fm), nil } @@ -332,7 +332,7 @@ func (b *InMemoryBackend) DescribeFleetMetric(name string) (*FleetMetric, error) b.mu.RLock() defer b.mu.RUnlock() - fm, ok := b.fleetMetrics[name] + fm, ok := b.fleetMetrics.Get(name) if !ok { return nil, fmt.Errorf("fleet metric %q not found: %w", name, ErrResourceNotFound) } @@ -344,9 +344,9 @@ func (b *InMemoryBackend) ListFleetMetrics() []*FleetMetric { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*FleetMetric, 0, len(b.fleetMetrics)) - for _, k := range sortedKeys(b.fleetMetrics) { - out = append(out, cloneFleetMetric(b.fleetMetrics[k])) + out := make([]*FleetMetric, 0, b.fleetMetrics.Len()) + for _, v := range b.fleetMetrics.Snapshot() { + out = append(out, cloneFleetMetric(v)) } return out @@ -356,7 +356,7 @@ func (b *InMemoryBackend) UpdateFleetMetric(name, queryString, description strin b.mu.Lock() defer b.mu.Unlock() - fm, ok := b.fleetMetrics[name] + fm, ok := b.fleetMetrics.Get(name) if !ok { return fmt.Errorf("fleet metric %q not found: %w", name, ErrResourceNotFound) } @@ -379,10 +379,10 @@ func (b *InMemoryBackend) DeleteFleetMetric(name string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.fleetMetrics[name]; !ok { + if !b.fleetMetrics.Has(name) { return fmt.Errorf("fleet metric %q not found: %w", name, ErrResourceNotFound) } - delete(b.fleetMetrics, name) + b.fleetMetrics.Delete(name) return nil } @@ -426,7 +426,7 @@ func (b *InMemoryBackend) CreateCustomMetric(input *CreateCustomMetricInput) (*C b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.customMetrics[input.MetricName]; exists { + if b.customMetrics.Has(input.MetricName) { return nil, fmt.Errorf("custom metric %q already exists: %w", input.MetricName, ErrAlreadyExists) } now := float64(time.Now().Unix()) @@ -440,7 +440,7 @@ func (b *InMemoryBackend) CreateCustomMetric(input *CreateCustomMetricInput) (*C CreationDate: now, LastModifiedDate: now, } - b.customMetrics[input.MetricName] = cm + b.customMetrics.Put(cm) return cloneCustomMetric(cm), nil } @@ -449,7 +449,7 @@ func (b *InMemoryBackend) DescribeCustomMetric(name string) (*CustomMetric, erro b.mu.RLock() defer b.mu.RUnlock() - cm, ok := b.customMetrics[name] + cm, ok := b.customMetrics.Get(name) if !ok { return nil, fmt.Errorf("custom metric %q not found: %w", name, ErrResourceNotFound) } @@ -461,9 +461,9 @@ func (b *InMemoryBackend) ListCustomMetrics() []*CustomMetric { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*CustomMetric, 0, len(b.customMetrics)) - for _, k := range sortedKeys(b.customMetrics) { - out = append(out, cloneCustomMetric(b.customMetrics[k])) + out := make([]*CustomMetric, 0, b.customMetrics.Len()) + for _, v := range b.customMetrics.Snapshot() { + out = append(out, cloneCustomMetric(v)) } return out @@ -473,7 +473,7 @@ func (b *InMemoryBackend) UpdateCustomMetric(name, displayName string) (*CustomM b.mu.Lock() defer b.mu.Unlock() - cm, ok := b.customMetrics[name] + cm, ok := b.customMetrics.Get(name) if !ok { return nil, fmt.Errorf("custom metric %q not found: %w", name, ErrResourceNotFound) } @@ -490,10 +490,10 @@ func (b *InMemoryBackend) DeleteCustomMetric(name string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.customMetrics[name]; !ok { + if !b.customMetrics.Has(name) { return fmt.Errorf("custom metric %q not found: %w", name, ErrResourceNotFound) } - delete(b.customMetrics, name) + b.customMetrics.Delete(name) return nil } @@ -537,7 +537,7 @@ func (b *InMemoryBackend) CreateDimension(input *CreateDimensionInput) (*Dimensi b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.dimensions[input.Name]; exists { + if b.dimensions.Has(input.Name) { return nil, fmt.Errorf("dimension %q already exists: %w", input.Name, ErrAlreadyExists) } now := float64(time.Now().Unix()) @@ -550,7 +550,7 @@ func (b *InMemoryBackend) CreateDimension(input *CreateDimensionInput) (*Dimensi CreationDate: now, LastModifiedDate: now, } - b.dimensions[input.Name] = d + b.dimensions.Put(d) return cloneDimension(d), nil } @@ -559,7 +559,7 @@ func (b *InMemoryBackend) DescribeDimension(name string) (*Dimension, error) { b.mu.RLock() defer b.mu.RUnlock() - d, ok := b.dimensions[name] + d, ok := b.dimensions.Get(name) if !ok { return nil, fmt.Errorf("dimension %q not found: %w", name, ErrResourceNotFound) } @@ -571,9 +571,9 @@ func (b *InMemoryBackend) ListDimensions() []*Dimension { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*Dimension, 0, len(b.dimensions)) - for _, k := range sortedKeys(b.dimensions) { - out = append(out, cloneDimension(b.dimensions[k])) + out := make([]*Dimension, 0, b.dimensions.Len()) + for _, v := range b.dimensions.Snapshot() { + out = append(out, cloneDimension(v)) } return out @@ -583,7 +583,7 @@ func (b *InMemoryBackend) UpdateDimension(name string, stringValues []string) (* b.mu.Lock() defer b.mu.Unlock() - d, ok := b.dimensions[name] + d, ok := b.dimensions.Get(name) if !ok { return nil, fmt.Errorf("dimension %q not found: %w", name, ErrResourceNotFound) } @@ -599,10 +599,10 @@ func (b *InMemoryBackend) DeleteDimension(name string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.dimensions[name]; !ok { + if !b.dimensions.Has(name) { return fmt.Errorf("dimension %q not found: %w", name, ErrResourceNotFound) } - delete(b.dimensions, name) + b.dimensions.Delete(name) return nil } @@ -728,12 +728,12 @@ func (b *InMemoryBackend) StartOnDemandAuditTask(_ []string) (string, error) { id := uuid.NewString()[:12] b.auditTasks[id] = string(JobStatusInProgress) - b.auditTaskObjects[id] = &AuditTask{ + b.auditTaskObjects.Put(&AuditTask{ TaskID: id, TaskStatus: string(JobStatusInProgress), TaskType: "ON_DEMAND_AUDIT_TASK", TaskStartTime: float64(time.Now().Unix()), - } + }) return id, nil } @@ -742,7 +742,7 @@ func (b *InMemoryBackend) DescribeAuditTask(taskID string) (*AuditTask, error) { b.mu.RLock() defer b.mu.RUnlock() - task, ok := b.auditTaskObjects[taskID] + task, ok := b.auditTaskObjects.Get(taskID) if !ok { return nil, fmt.Errorf("audit task %q not found: %w", taskID, ErrResourceNotFound) } @@ -756,9 +756,9 @@ func (b *InMemoryBackend) ListAuditTasks(taskType string) []*AuditTask { defer b.mu.RUnlock() var out []*AuditTask - keys := sortedKeys(b.auditTaskObjects) - for _, k := range keys { - task := b.auditTaskObjects[k] + items := b.auditTaskObjects.Snapshot() + for _, v := range items { + task := v if taskType == "" || task.TaskType == taskType { cp := *task out = append(out, &cp) @@ -877,7 +877,7 @@ func (b *InMemoryBackend) ListPrincipalPolicies(principal string) []*Policy { var out []*Policy for policyName, targets := range b.policyTargets { if slices.Contains(targets, principal) { - if p, ok := b.policies[policyName]; ok { + if p, ok := b.policies.Get(policyName); ok { cp := *p out = append(out, &cp) } @@ -934,7 +934,7 @@ func (b *InMemoryBackend) GetEffectivePolicies(thingName, principal string) []*P for policyName, targets := range b.policyTargets { for _, target := range targets { if slices.Contains(principals, target) && !seen[policyName] { - if pol, ok := b.policies[policyName]; ok { + if pol, ok := b.policies.Get(policyName); ok { cp := *pol out = append(out, &cp) seen[policyName] = true @@ -982,8 +982,7 @@ func (b *InMemoryBackend) ListJobExecutionsForJob(jobID string) []*JobExecution defer b.mu.RUnlock() var out []*JobExecution - for k, exec := range b.jobExecutions { - _ = k + for _, exec := range b.jobExecutions.All() { if exec.JobID == jobID { cp := *exec out = append(out, &cp) @@ -1000,8 +999,7 @@ func (b *InMemoryBackend) ListJobExecutionsForThing(thingName string) []*JobExec defer b.mu.RUnlock() var out []*JobExecution - for k, exec := range b.jobExecutions { - _ = k + for _, exec := range b.jobExecutions.All() { if exec.ThingName == thingName { cp := *exec out = append(out, &cp) diff --git a/services/iot/backend_batch3.go b/services/iot/backend_batch3.go index 19e8b359e..17a1fa36e 100644 --- a/services/iot/backend_batch3.go +++ b/services/iot/backend_batch3.go @@ -51,7 +51,7 @@ func (b *InMemoryBackend) CreateOTAUpdate( b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.otaUpdates[id]; exists { + if b.otaUpdates.Has(id) { return nil, fmt.Errorf("OTA update %q already exists: %w", id, ErrAlreadyExists) } now := float64(time.Now().Unix()) @@ -66,7 +66,7 @@ func (b *InMemoryBackend) CreateOTAUpdate( CreationDate: now, LastModifiedDate: now, } - b.otaUpdates[id] = o + b.otaUpdates.Put(o) return cloneOTAUpdate(o), nil } @@ -75,7 +75,7 @@ func (b *InMemoryBackend) GetOTAUpdate(id string) (*OTAUpdate, error) { b.mu.RLock() defer b.mu.RUnlock() - o, ok := b.otaUpdates[id] + o, ok := b.otaUpdates.Get(id) if !ok { return nil, fmt.Errorf("OTA update %q not found: %w", id, ErrResourceNotFound) } @@ -87,10 +87,10 @@ func (b *InMemoryBackend) DeleteOTAUpdate(id string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.otaUpdates[id]; !ok { + if !b.otaUpdates.Has(id) { return fmt.Errorf("OTA update %q not found: %w", id, ErrResourceNotFound) } - delete(b.otaUpdates, id) + b.otaUpdates.Delete(id) return nil } @@ -99,10 +99,10 @@ func (b *InMemoryBackend) ListOTAUpdates() []*OTAUpdate { b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.otaUpdates) - out := make([]*OTAUpdate, 0, len(keys)) - for _, k := range keys { - out = append(out, cloneOTAUpdate(b.otaUpdates[k])) + items := b.otaUpdates.Snapshot() + out := make([]*OTAUpdate, 0, len(items)) + for _, v := range items { + out = append(out, cloneOTAUpdate(v)) } return out @@ -141,7 +141,7 @@ func (b *InMemoryBackend) CreateIoTPackage(name, description string, tags map[st b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.iotPackages[name]; exists { + if b.iotPackages.Has(name) { return nil, fmt.Errorf("package %q already exists: %w", name, ErrAlreadyExists) } now := float64(time.Now().Unix()) @@ -154,7 +154,7 @@ func (b *InMemoryBackend) CreateIoTPackage(name, description string, tags map[st LastModifiedDate: now, } maps.Copy(p.Tags, tags) - b.iotPackages[name] = p + b.iotPackages.Put(p) return cloneIoTPackage(p), nil } @@ -163,7 +163,7 @@ func (b *InMemoryBackend) GetIoTPackage(name string) (*IoTPackage, error) { b.mu.RLock() defer b.mu.RUnlock() - p, ok := b.iotPackages[name] + p, ok := b.iotPackages.Get(name) if !ok { return nil, fmt.Errorf("package %q not found: %w", name, ErrResourceNotFound) } @@ -175,7 +175,7 @@ func (b *InMemoryBackend) UpdateIoTPackage(name, description, defaultVersionName b.mu.Lock() defer b.mu.Unlock() - p, ok := b.iotPackages[name] + p, ok := b.iotPackages.Get(name) if !ok { return fmt.Errorf("package %q not found: %w", name, ErrResourceNotFound) } @@ -194,10 +194,10 @@ func (b *InMemoryBackend) DeleteIoTPackage(name string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.iotPackages[name]; !ok { + if !b.iotPackages.Has(name) { return fmt.Errorf("package %q not found: %w", name, ErrResourceNotFound) } - delete(b.iotPackages, name) + b.iotPackages.Delete(name) return nil } @@ -206,10 +206,10 @@ func (b *InMemoryBackend) ListIoTPackages() []*IoTPackage { b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.iotPackages) - out := make([]*IoTPackage, 0, len(keys)) - for _, k := range keys { - out = append(out, cloneIoTPackage(b.iotPackages[k])) + items := b.iotPackages.Snapshot() + out := make([]*IoTPackage, 0, len(items)) + for _, v := range items { + out = append(out, cloneIoTPackage(v)) } return out @@ -425,18 +425,18 @@ func (b *InMemoryBackend) CreateAuditSuppression( defer b.mu.Unlock() key := auditSuppressionKey(checkName, resourceID) - if _, exists := b.auditSuppressions[key]; exists { + if b.auditSuppressions.Has(key) { return fmt.Errorf("audit suppression %q already exists: %w", key, ErrAlreadyExists) } rid := make(map[string]any, len(resourceID)) maps.Copy(rid, resourceID) - b.auditSuppressions[key] = &AuditSuppression{ + b.auditSuppressions.Put(&AuditSuppression{ CheckName: checkName, ResourceIdentifier: rid, Description: description, SuppressIndefinitely: suppressIndefinitely, ExpirationDate: expirationDate, - } + }) return nil } @@ -449,7 +449,7 @@ func (b *InMemoryBackend) DescribeAuditSuppression( defer b.mu.RUnlock() key := auditSuppressionKey(checkName, resourceID) - s, ok := b.auditSuppressions[key] + s, ok := b.auditSuppressions.Get(key) if !ok { return nil, fmt.Errorf("audit suppression %q not found: %w", key, ErrResourceNotFound) } @@ -468,7 +468,7 @@ func (b *InMemoryBackend) UpdateAuditSuppression( defer b.mu.Unlock() key := auditSuppressionKey(checkName, resourceID) - s, ok := b.auditSuppressions[key] + s, ok := b.auditSuppressions.Get(key) if !ok { return fmt.Errorf("audit suppression %q not found: %w", key, ErrResourceNotFound) } @@ -488,10 +488,10 @@ func (b *InMemoryBackend) DeleteAuditSuppression(checkName string, resourceID ma defer b.mu.Unlock() key := auditSuppressionKey(checkName, resourceID) - if _, ok := b.auditSuppressions[key]; !ok { + if !b.auditSuppressions.Has(key) { return fmt.Errorf("audit suppression %q not found: %w", key, ErrResourceNotFound) } - delete(b.auditSuppressions, key) + b.auditSuppressions.Delete(key) return nil } @@ -500,10 +500,10 @@ func (b *InMemoryBackend) ListAuditSuppressions() []*AuditSuppression { b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.auditSuppressions) - out := make([]*AuditSuppression, 0, len(keys)) - for _, k := range keys { - out = append(out, cloneAuditSuppression(b.auditSuppressions[k])) + items := b.auditSuppressions.Snapshot() + out := make([]*AuditSuppression, 0, len(items)) + for _, v := range items { + out = append(out, cloneAuditSuppression(v)) } return out @@ -553,7 +553,7 @@ func (b *InMemoryBackend) SeedAuditFinding(f *AuditFinding) *AuditFinding { stored.FindingTime = float64(time.Now().Unix()) } - b.auditFindings[stored.FindingID] = stored + b.auditFindings.Put(stored) return cloneAuditFinding(stored) } @@ -562,7 +562,7 @@ func (b *InMemoryBackend) DescribeAuditFinding(findingID string) (*AuditFinding, b.mu.RLock() defer b.mu.RUnlock() - f, ok := b.auditFindings[findingID] + f, ok := b.auditFindings.Get(findingID) if !ok { return nil, fmt.Errorf("audit finding %q not found: %w", findingID, ErrResourceNotFound) } @@ -574,10 +574,10 @@ func (b *InMemoryBackend) ListAuditFindings() []*AuditFinding { b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.auditFindings) - out := make([]*AuditFinding, 0, len(keys)) - for _, k := range keys { - out = append(out, cloneAuditFinding(b.auditFindings[k])) + items := b.auditFindings.Snapshot() + out := make([]*AuditFinding, 0, len(items)) + for _, v := range items { + out = append(out, cloneAuditFinding(v)) } return out @@ -589,7 +589,7 @@ func (b *InMemoryBackend) ListRelatedResourcesForAuditFinding(findingID string) b.mu.RLock() defer b.mu.RUnlock() - f, ok := b.auditFindings[findingID] + f, ok := b.auditFindings.Get(findingID) if !ok { return nil, fmt.Errorf("audit finding %q not found: %w", findingID, ErrResourceNotFound) } @@ -661,10 +661,9 @@ func (b *InMemoryBackend) SetV2LoggingLevel(target map[string]any, logLevel stri b.mu.Lock() defer b.mu.Unlock() - key := v2LogLevelKey(target) tgt := make(map[string]any, len(target)) maps.Copy(tgt, target) - b.v2LoggingLevels[key] = &V2LoggingLevel{Target: tgt, LogLevel: logLevel} + b.v2LoggingLevels.Put(&V2LoggingLevel{Target: tgt, LogLevel: logLevel}) return nil } @@ -674,10 +673,10 @@ func (b *InMemoryBackend) DeleteV2LoggingLevel(target map[string]any) error { defer b.mu.Unlock() key := v2LogLevelKey(target) - if _, ok := b.v2LoggingLevels[key]; !ok { + if !b.v2LoggingLevels.Has(key) { return fmt.Errorf("V2 logging level %q not found: %w", key, ErrResourceNotFound) } - delete(b.v2LoggingLevels, key) + b.v2LoggingLevels.Delete(key) return nil } @@ -686,10 +685,10 @@ func (b *InMemoryBackend) ListV2LoggingLevels() []*V2LoggingLevel { b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.v2LoggingLevels) - out := make([]*V2LoggingLevel, 0, len(keys)) - for _, k := range keys { - out = append(out, cloneV2LogLevel(b.v2LoggingLevels[k])) + items := b.v2LoggingLevels.Snapshot() + out := make([]*V2LoggingLevel, 0, len(items)) + for _, v := range items { + out = append(out, cloneV2LogLevel(v)) } return out @@ -733,7 +732,7 @@ func (b *InMemoryBackend) SetLoggingOptions(roleARN, logLevel string) error { // CreateProvisioningClaim returns a fake cert/key pair for the given template. func (b *InMemoryBackend) CreateProvisioningClaim(templateName string) (string, string, string, error) { b.mu.RLock() - _, ok := b.provTemplates[templateName] + _, ok := b.provTemplates.Get(templateName) b.mu.RUnlock() if !ok { @@ -757,7 +756,7 @@ func (b *InMemoryBackend) CreateKeysAndCertificate(setAsActive bool) (*Certifica status = certStatusActive } cert := b.newCertificate(fakePEM, status) - b.certificates[cert.CertificateID] = cert + b.certificates.Put(cert) cp := *cert return &cp, fakePublicKey, fakePrivateKey, nil @@ -772,7 +771,7 @@ func (b *InMemoryBackend) TransferCertificate(certID, targetAccount string) erro b.mu.Lock() defer b.mu.Unlock() - cert, ok := b.certificates[certID] + cert, ok := b.certificates.Get(certID) if !ok { return fmt.Errorf("certificate %q not found: %w", certID, ErrCertificateNotFound) } @@ -788,7 +787,7 @@ func (b *InMemoryBackend) RejectCertificateTransfer(certID string) error { b.mu.Lock() defer b.mu.Unlock() - cert, ok := b.certificates[certID] + cert, ok := b.certificates.Get(certID) if !ok { return fmt.Errorf("certificate %q not found: %w", certID, ErrCertificateNotFound) } @@ -908,7 +907,7 @@ func (b *InMemoryBackend) CreateDynamicThingGroup(input *CreateThingGroupInput) b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.thingGroups[input.ThingGroupName]; exists { + if b.thingGroups.Has(input.ThingGroupName) { return nil, fmt.Errorf("%w: dynamic thing group %q already exists", ErrAlreadyExists, input.ThingGroupName) } arn := arn.Build("iot", b.region, b.accountID, fmt.Sprintf("thinggroup/%s", input.ThingGroupName)) @@ -930,7 +929,7 @@ func (b *InMemoryBackend) CreateDynamicThingGroup(input *CreateThingGroupInput) QueryString: input.QueryString, IsDynamic: true, } - b.thingGroups[input.ThingGroupName] = tg + b.thingGroups.Put(tg) b.thingGroupMembers[input.ThingGroupName] = []string{} return tg, nil @@ -941,10 +940,10 @@ func (b *InMemoryBackend) DeleteDynamicThingGroup(name string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.thingGroups[name]; !ok { + if !b.thingGroups.Has(name) { return fmt.Errorf("%w: %s", ErrThingGroupNotFound, name) } - delete(b.thingGroups, name) + b.thingGroups.Delete(name) delete(b.thingGroupMembers, name) return nil @@ -959,7 +958,7 @@ func (b *InMemoryBackend) UpdateDynamicThingGroup(input *UpdateThingGroupInput) b.mu.Lock() defer b.mu.Unlock() - tg, ok := b.thingGroups[input.ThingGroupName] + tg, ok := b.thingGroups.Get(input.ThingGroupName) if !ok { return 0, fmt.Errorf("%w: %s", ErrThingGroupNotFound, input.ThingGroupName) } @@ -1017,7 +1016,7 @@ func (b *InMemoryBackend) CreateCommand( b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.commands[id]; exists { + if b.commands.Has(id) { return nil, fmt.Errorf("command %q already exists: %w", id, ErrAlreadyExists) } now := float64(time.Now().Unix()) @@ -1034,7 +1033,7 @@ func (b *InMemoryBackend) CreateCommand( } maps.Copy(cmd.Tags, tags) maps.Copy(cmd.Payload, payload) - b.commands[id] = cmd + b.commands.Put(cmd) return cloneIoTCommand(cmd), nil } @@ -1043,7 +1042,7 @@ func (b *InMemoryBackend) GetCommand(id string) (*IoTCommand, error) { b.mu.RLock() defer b.mu.RUnlock() - cmd, ok := b.commands[id] + cmd, ok := b.commands.Get(id) if !ok { return nil, fmt.Errorf("command %q not found: %w", id, ErrResourceNotFound) } @@ -1055,7 +1054,7 @@ func (b *InMemoryBackend) UpdateCommand(id, displayName, description string, dep b.mu.Lock() defer b.mu.Unlock() - cmd, ok := b.commands[id] + cmd, ok := b.commands.Get(id) if !ok { return fmt.Errorf("command %q not found: %w", id, ErrResourceNotFound) } @@ -1075,10 +1074,10 @@ func (b *InMemoryBackend) DeleteCommand(id string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.commands[id]; !ok { + if !b.commands.Has(id) { return fmt.Errorf("command %q not found: %w", id, ErrResourceNotFound) } - delete(b.commands, id) + b.commands.Delete(id) return nil } @@ -1087,10 +1086,10 @@ func (b *InMemoryBackend) ListCommands() []*IoTCommand { b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.commands) - out := make([]*IoTCommand, 0, len(keys)) - for _, k := range keys { - out = append(out, cloneIoTCommand(b.commands[k])) + items := b.commands.Snapshot() + out := make([]*IoTCommand, 0, len(items)) + for _, v := range items { + out = append(out, cloneIoTCommand(v)) } return out diff --git a/services/iot/backend_batch4.go b/services/iot/backend_batch4.go index d2220075c..8a1b04935 100644 --- a/services/iot/backend_batch4.go +++ b/services/iot/backend_batch4.go @@ -43,7 +43,7 @@ func (b *InMemoryBackend) RegisterThing(input *RegisterThingInput) (*RegisterThi arn := fmt.Sprintf("arn:aws:iot:%s:%s:thing/%s", b.region, b.accountID, thingName) - if t, exists := b.things[thingName]; exists { + if t, exists := b.things.Get(thingName); exists { if t.Attributes == nil { t.Attributes = make(map[string]string) } @@ -54,18 +54,18 @@ func (b *InMemoryBackend) RegisterThing(input *RegisterThingInput) (*RegisterThi attrs := make(map[string]string, len(input.Parameters)) maps.Copy(attrs, input.Parameters) - b.things[thingName] = &Thing{ + b.things.Put(&Thing{ ThingName: thingName, ThingID: uuid.NewString(), ARN: arn, Attributes: attrs, Version: 1, CreatedAt: time.Now(), - } + }) } cert := b.newCertificate(fakePEM, certStatusActive) - b.certificates[cert.CertificateID] = cert + b.certificates.Put(cert) b.thingPrincipals[thingName] = append(b.thingPrincipals[thingName], cert.ARN) return &RegisterThingOutput{ @@ -121,7 +121,7 @@ func (b *InMemoryBackend) StartThingRegistrationTask( LastModifiedDate: now, } - b.registrationTasks[task.TaskID] = task + b.registrationTasks.Put(task) return cloneRegistrationTask(task), nil } @@ -131,7 +131,7 @@ func (b *InMemoryBackend) StopThingRegistrationTask(taskID string) error { b.mu.Lock() defer b.mu.Unlock() - task, ok := b.registrationTasks[taskID] + task, ok := b.registrationTasks.Get(taskID) if !ok { return fmt.Errorf("%w: %s", ErrRegistrationTaskNotFound, taskID) } @@ -147,7 +147,7 @@ func (b *InMemoryBackend) DescribeThingRegistrationTask(taskID string) (*ThingRe b.mu.RLock() defer b.mu.RUnlock() - task, ok := b.registrationTasks[taskID] + task, ok := b.registrationTasks.Get(taskID) if !ok { return nil, fmt.Errorf("%w: %s", ErrRegistrationTaskNotFound, taskID) } @@ -161,11 +161,11 @@ func (b *InMemoryBackend) ListThingRegistrationTasks(status string) []*ThingRegi b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.registrationTasks) - out := make([]*ThingRegistrationTask, 0, len(keys)) + items := b.registrationTasks.Snapshot() + out := make([]*ThingRegistrationTask, 0, len(items)) - for _, k := range keys { - task := b.registrationTasks[k] + for _, v := range items { + task := v if status != "" && task.Status != status { continue } @@ -182,7 +182,7 @@ func (b *InMemoryBackend) ListThingRegistrationTaskReports(taskID, reportType st b.mu.RLock() defer b.mu.RUnlock() - task, ok := b.registrationTasks[taskID] + task, ok := b.registrationTasks.Get(taskID) if !ok { return nil, fmt.Errorf("%w: %s", ErrRegistrationTaskNotFound, taskID) } @@ -214,7 +214,7 @@ func (b *InMemoryBackend) UpdateThingType(input *UpdateThingTypeInput) error { b.mu.Lock() defer b.mu.Unlock() - tt, ok := b.thingTypes[input.ThingTypeName] + tt, ok := b.thingTypes.Get(input.ThingTypeName) if !ok { return fmt.Errorf("%w: %s", ErrThingTypeNotFound, input.ThingTypeName) } @@ -246,18 +246,18 @@ func (b *InMemoryBackend) UpdateThingGroupsForThing(input *UpdateThingGroupsForT b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.things[input.ThingName]; !ok { + if !b.things.Has(input.ThingName) { return fmt.Errorf("%w: %s", ErrThingNotFound, input.ThingName) } for _, g := range input.ThingGroupsToAdd { - if _, ok := b.thingGroups[g]; !ok { + if !b.thingGroups.Has(g) { return fmt.Errorf("%w: %s", ErrThingGroupNotFound, g) } } for _, g := range input.ThingGroupsToRemove { - if _, ok := b.thingGroups[g]; !ok { + if !b.thingGroups.Has(g) { return fmt.Errorf("%w: %s", ErrThingGroupNotFound, g) } } @@ -318,7 +318,7 @@ func (b *InMemoryBackend) ListThingPrincipalsV2(thingName string) ([]*ThingPrinc b.mu.RLock() defer b.mu.RUnlock() - if _, ok := b.things[thingName]; !ok { + if !b.things.Has(thingName) { return nil, fmt.Errorf("%w: %s", ErrThingNotFound, thingName) } diff --git a/services/iot/backend_devicedefender.go b/services/iot/backend_devicedefender.go index 22fdca657..a914f847f 100644 --- a/services/iot/backend_devicedefender.go +++ b/services/iot/backend_devicedefender.go @@ -110,7 +110,8 @@ func (b *InMemoryBackend) auditMitigationFindingIDs(target *AuditMitigationActio } var ids []string - for id, f := range b.auditFindings { + for _, f := range b.auditFindings.All() { + id := f.FindingID switch { case target.AuditTaskID != "" && f.TaskID == target.AuditTaskID: ids = append(ids, id) @@ -137,7 +138,7 @@ func (b *InMemoryBackend) StartAuditMitigationActionsTask( if input.TaskID == "" { return nil, fmt.Errorf("%w: taskId is required", ErrValidation) } - if _, exists := b.auditMitigationTaskObjects[input.TaskID]; exists { + if b.auditMitigationTaskObjects.Has(input.TaskID) { return nil, fmt.Errorf( "audit mitigation actions task %q already exists: %w", input.TaskID, @@ -163,7 +164,7 @@ func (b *InMemoryBackend) StartAuditMitigationActionsTask( for _, findingID := range findingIDs { checkName := "" - if f, ok := b.auditFindings[findingID]; ok { + if f, ok := b.auditFindings.Get(findingID); ok { checkName = f.CheckName } @@ -176,7 +177,7 @@ func (b *InMemoryBackend) StartAuditMitigationActionsTask( for _, actionName := range input.AuditCheckToActionsMapping[checkName] { actionID := "" - if ma, ok := b.mitigationActions[actionName]; ok { + if ma, ok := b.mitigationActions.Get(actionName); ok { actionID = ma.ActionID } executions = append(executions, &AuditMitigationActionExecution{ @@ -190,7 +191,7 @@ func (b *InMemoryBackend) StartAuditMitigationActionsTask( } } - b.auditMitigationTaskObjects[input.TaskID] = task + b.auditMitigationTaskObjects.Put(task) b.auditMitigationExecutions[input.TaskID] = executions // Keep the legacy status map (read by CancelAuditMitigationActionsTask's // original implementation) in sync with the rich task object. @@ -205,7 +206,7 @@ func (b *InMemoryBackend) DescribeAuditMitigationActionsTask(taskID string) (*Au b.mu.RLock() defer b.mu.RUnlock() - t, ok := b.auditMitigationTaskObjects[taskID] + t, ok := b.auditMitigationTaskObjects.Get(taskID) if !ok { return nil, fmt.Errorf("audit mitigation actions task %q not found: %w", taskID, ErrResourceNotFound) } @@ -219,11 +220,11 @@ func (b *InMemoryBackend) ListAuditMitigationActionsTasks(auditTaskID, taskStatu b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.auditMitigationTaskObjects) - out := make([]*AuditMitigationTask, 0, len(keys)) + items := b.auditMitigationTaskObjects.Snapshot() + out := make([]*AuditMitigationTask, 0, len(items)) - for _, k := range keys { - t := b.auditMitigationTaskObjects[k] + for _, v := range items { + t := v if auditTaskID != "" && (t.Target == nil || t.Target.AuditTaskID != auditTaskID) { continue } @@ -366,14 +367,14 @@ func (b *InMemoryBackend) detectMitigationViolationIDs(target *DetectMitigationA var ids []string - for id, v := range b.activeViolations { + for _, v := range b.activeViolations.All() { if target.SecurityProfileName != "" && v.SecurityProfileName != target.SecurityProfileName { continue } if target.BehaviorName != "" && (v.Behavior == nil || v.Behavior.Name != target.BehaviorName) { continue } - ids = append(ids, id) + ids = append(ids, v.ViolationID) } sort.Strings(ids) @@ -392,7 +393,7 @@ func (b *InMemoryBackend) StartDetectMitigationActionsTask( if input.TaskID == "" { return nil, fmt.Errorf("%w: taskId is required", ErrValidation) } - if _, exists := b.detectMitigationTasks[input.TaskID]; exists { + if b.detectMitigationTasks.Has(input.TaskID) { return nil, fmt.Errorf( "detect mitigation actions task %q already exists: %w", input.TaskID, @@ -420,7 +421,7 @@ func (b *InMemoryBackend) StartDetectMitigationActionsTask( for _, violationID := range violationIDs { thingName := "" - if v, ok := b.activeViolations[violationID]; ok { + if v, ok := b.activeViolations.Get(violationID); ok { thingName = v.ThingName } @@ -437,7 +438,7 @@ func (b *InMemoryBackend) StartDetectMitigationActionsTask( } } - b.detectMitigationTasks[input.TaskID] = task + b.detectMitigationTasks.Put(task) b.detectMitigationExecutions[input.TaskID] = executions return cloneDetectMitigationTask(task), nil @@ -449,7 +450,7 @@ func (b *InMemoryBackend) DescribeDetectMitigationActionsTask(taskID string) (*D b.mu.RLock() defer b.mu.RUnlock() - t, ok := b.detectMitigationTasks[taskID] + t, ok := b.detectMitigationTasks.Get(taskID) if !ok { return nil, fmt.Errorf("detect mitigation actions task %q not found: %w", taskID, ErrResourceNotFound) } @@ -464,11 +465,11 @@ func (b *InMemoryBackend) ListDetectMitigationActionsTasks(startTime, endTime fl b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.detectMitigationTasks) - out := make([]*DetectMitigationTask, 0, len(keys)) + items := b.detectMitigationTasks.Snapshot() + out := make([]*DetectMitigationTask, 0, len(items)) - for _, k := range keys { - t := b.detectMitigationTasks[k] + for _, v := range items { + t := v if startTime > 0 && t.StartTime < startTime { continue } @@ -487,7 +488,7 @@ func (b *InMemoryBackend) CancelDetectMitigationActionsTask(taskID string) error b.mu.Lock() defer b.mu.Unlock() - t, ok := b.detectMitigationTasks[taskID] + t, ok := b.detectMitigationTasks.Get(taskID) if !ok { return fmt.Errorf("detect mitigation actions task %q not found: %w", taskID, ErrResourceNotFound) } @@ -632,7 +633,7 @@ func (b *InMemoryBackend) SeedActiveViolation(input *SeedActiveViolationInput) ( VerificationState: verificationState, ViolationStartTime: now, } - b.activeViolations[violationID] = v + b.activeViolations.Put(v) b.violationEvents = append(b.violationEvents, &ViolationEvent{ ViolationID: violationID, @@ -656,11 +657,10 @@ func (b *InMemoryBackend) ListActiveViolations( b.mu.RLock() defer b.mu.RUnlock() - keys := sortedKeys(b.activeViolations) - out := make([]*ActiveViolation, 0, len(keys)) + items := b.activeViolations.Snapshot() + out := make([]*ActiveViolation, 0, len(items)) - for _, k := range keys { - v := b.activeViolations[k] + for _, v := range items { if thingName != "" && v.ThingName != thingName { continue } @@ -717,7 +717,7 @@ func (b *InMemoryBackend) PutVerificationStateOnViolation(violationID, verificat b.mu.Lock() defer b.mu.Unlock() - v, ok := b.activeViolations[violationID] + v, ok := b.activeViolations.Get(violationID) if !ok { return fmt.Errorf("violation %q not found: %w", violationID, ErrResourceNotFound) } @@ -734,23 +734,20 @@ func (b *InMemoryBackend) PutVerificationStateOnViolation(violationID, verificat // deviceDefenderSnapshot bundles the Device Defender fields of backendSnapshot // so Snapshot/Restore can delegate to a single helper each, keeping their own // cyclomatic complexity low. +// deviceDefenderSnapshot now only carries the raw (non-Table) Device Defender +// fields. AuditMitigationTaskObjects, DetectMitigationTasks, and +// ActiveViolations moved to store.Table[T]s registered on b.registry (see +// store_setup.go) and round-trip via registry.SnapshotAll()/RestoreAll() +// instead of through this bundle. type deviceDefenderSnapshot struct { - AuditMitigationTaskObjects map[string]*AuditMitigationTask AuditMitigationExecutions map[string][]*AuditMitigationActionExecution - DetectMitigationTasks map[string]*DetectMitigationTask DetectMitigationExecutions map[string][]*DetectMitigationActionExecution - ActiveViolations map[string]*ActiveViolation ViolationEvents []*ViolationEvent } -// snapshotDeviceDefender deep-copies all Device Defender state. Must be called -// with b.mu held (read or write). +// snapshotDeviceDefender deep-copies the raw (non-Table) Device Defender +// state. Must be called with b.mu held (read or write). func (b *InMemoryBackend) snapshotDeviceDefender() deviceDefenderSnapshot { - auditMitigationTaskObjects := make(map[string]*AuditMitigationTask, len(b.auditMitigationTaskObjects)) - for k, v := range b.auditMitigationTaskObjects { - auditMitigationTaskObjects[k] = cloneAuditMitigationTask(v) - } - auditMitigationExecutions := make( map[string][]*AuditMitigationActionExecution, len(b.auditMitigationExecutions), @@ -763,11 +760,6 @@ func (b *InMemoryBackend) snapshotDeviceDefender() deviceDefenderSnapshot { auditMitigationExecutions[k] = cp } - detectMitigationTasks := make(map[string]*DetectMitigationTask, len(b.detectMitigationTasks)) - for k, v := range b.detectMitigationTasks { - detectMitigationTasks[k] = cloneDetectMitigationTask(v) - } - detectMitigationExecutions := make( map[string][]*DetectMitigationActionExecution, len(b.detectMitigationExecutions), @@ -780,34 +772,21 @@ func (b *InMemoryBackend) snapshotDeviceDefender() deviceDefenderSnapshot { detectMitigationExecutions[k] = cp } - activeViolations := make(map[string]*ActiveViolation, len(b.activeViolations)) - for k, v := range b.activeViolations { - activeViolations[k] = cloneActiveViolation(v) - } - violationEvents := make([]*ViolationEvent, len(b.violationEvents)) for i, e := range b.violationEvents { violationEvents[i] = cloneViolationEvent(e) } return deviceDefenderSnapshot{ - AuditMitigationTaskObjects: auditMitigationTaskObjects, AuditMitigationExecutions: auditMitigationExecutions, - DetectMitigationTasks: detectMitigationTasks, DetectMitigationExecutions: detectMitigationExecutions, - ActiveViolations: activeViolations, ViolationEvents: violationEvents, } } -// restoreDeviceDefender deep-copies snap into the backend's Device Defender -// state. Must be called with b.mu held (write). +// restoreDeviceDefender deep-copies snap into the backend's raw (non-Table) +// Device Defender state. Must be called with b.mu held (write). func (b *InMemoryBackend) restoreDeviceDefender(snap deviceDefenderSnapshot) { - b.auditMitigationTaskObjects = make(map[string]*AuditMitigationTask, len(snap.AuditMitigationTaskObjects)) - for k, v := range snap.AuditMitigationTaskObjects { - b.auditMitigationTaskObjects[k] = cloneAuditMitigationTask(v) - } - b.auditMitigationExecutions = make( map[string][]*AuditMitigationActionExecution, len(snap.AuditMitigationExecutions), @@ -820,11 +799,6 @@ func (b *InMemoryBackend) restoreDeviceDefender(snap deviceDefenderSnapshot) { b.auditMitigationExecutions[k] = cp } - b.detectMitigationTasks = make(map[string]*DetectMitigationTask, len(snap.DetectMitigationTasks)) - for k, v := range snap.DetectMitigationTasks { - b.detectMitigationTasks[k] = cloneDetectMitigationTask(v) - } - b.detectMitigationExecutions = make( map[string][]*DetectMitigationActionExecution, len(snap.DetectMitigationExecutions), @@ -837,11 +811,6 @@ func (b *InMemoryBackend) restoreDeviceDefender(snap deviceDefenderSnapshot) { b.detectMitigationExecutions[k] = cp } - b.activeViolations = make(map[string]*ActiveViolation, len(snap.ActiveViolations)) - for k, v := range snap.ActiveViolations { - b.activeViolations[k] = cloneActiveViolation(v) - } - b.violationEvents = make([]*ViolationEvent, len(snap.ViolationEvents)) for i, e := range snap.ViolationEvents { b.violationEvents[i] = cloneViolationEvent(e) diff --git a/services/iot/backend_final_ops.go b/services/iot/backend_final_ops.go index 0351b88d5..cce274459 100644 --- a/services/iot/backend_final_ops.go +++ b/services/iot/backend_final_ops.go @@ -224,7 +224,7 @@ func (b *InMemoryBackend) effectiveTestPolicies(input *TestAuthorizationInput) ( continue } - if p, ok := b.policies[policyName]; ok { + if p, ok := b.policies.Get(policyName); ok { cp := *p policies = append(policies, &cp) seen[policyName] = true @@ -236,7 +236,7 @@ func (b *InMemoryBackend) effectiveTestPolicies(input *TestAuthorizationInput) ( continue } - p, ok := b.policies[name] + p, ok := b.policies.Get(name) if !ok { return nil, fmt.Errorf("%w: %s", ErrPolicyNotFound, name) } @@ -465,7 +465,7 @@ func (b *InMemoryBackend) TestInvokeAuthorizer( input *TestInvokeAuthorizerInput, ) (*TestInvokeAuthorizerOutput, error) { b.mu.RLock() - a, ok := b.authorizers[authorizerName] + a, ok := b.authorizers.Get(authorizerName) b.mu.RUnlock() if !ok { @@ -525,7 +525,7 @@ func (b *InMemoryBackend) DetachPrincipalPolicy(policyName, principal string) er b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.policies[policyName]; !ok { + if !b.policies.Has(policyName) { return fmt.Errorf("%w: %s", ErrPolicyNotFound, policyName) } @@ -558,7 +558,7 @@ func (b *InMemoryBackend) ConfirmTopicRuleDestination(token string) error { b.mu.Lock() defer b.mu.Unlock() - for _, dest := range b.topicRuleDestinations { + for _, dest := range b.topicRuleDestinations.All() { if dest.ConfirmationToken != "" && dest.ConfirmationToken == token { dest.Status = statusEnabled dest.ConfirmationToken = "" @@ -641,7 +641,7 @@ func (b *InMemoryBackend) GetBehaviorModelTrainingSummaries( var all []*BehaviorModelTrainingSummary if securityProfileName != "" { - if _, ok := b.securityProfiles[securityProfileName]; !ok { + if !b.securityProfiles.Has(securityProfileName) { return nil, "", fmt.Errorf( "security profile %q not found: %w", securityProfileName, ErrResourceNotFound, ) @@ -702,8 +702,8 @@ func (b *InMemoryBackend) ListOutgoingCertificates() []*OutgoingCertificate { out := make([]*OutgoingCertificate, 0) - for _, id := range sortedKeys(b.certificates) { - cert := b.certificates[id] + for _, v := range b.certificates.Snapshot() { + cert := v if cert.Status != certStatusPendingTransfer { continue } @@ -843,7 +843,7 @@ func (b *InMemoryBackend) ListMetricValues( b.mu.RLock() defer b.mu.RUnlock() - if _, ok := b.things[thingName]; !ok { + if !b.things.Has(thingName) { return nil, "", fmt.Errorf("%w: %s", ErrThingNotFound, thingName) } @@ -898,7 +898,7 @@ func (b *InMemoryBackend) GetThingConnectivityData(thingName string) (*ThingConn b.mu.RLock() defer b.mu.RUnlock() - if _, ok := b.things[thingName]; !ok { + if !b.things.Has(thingName) { return nil, fmt.Errorf("%w: %s", ErrThingNotFound, thingName) } @@ -934,7 +934,7 @@ func (b *InMemoryBackend) DescribeProvisioningTemplateVersion( b.mu.RLock() defer b.mu.RUnlock() - if _, ok := b.provTemplates[name]; !ok { + if !b.provTemplates.Has(name) { return nil, fmt.Errorf("provisioning template %q not found: %w", name, ErrResourceNotFound) } diff --git a/services/iot/backend_indexing.go b/services/iot/backend_indexing.go index 642ff25ad..60923e6a6 100644 --- a/services/iot/backend_indexing.go +++ b/services/iot/backend_indexing.go @@ -255,10 +255,10 @@ func (b *InMemoryBackend) searchThingsIndex(input *SearchIndexInput) *SearchInde b.mu.RLock() defer b.mu.RUnlock() - matched := make([]*SearchIndexThingResult, 0, len(b.things)) + matched := make([]*SearchIndexThingResult, 0, b.things.Len()) - for _, k := range sortedKeys(b.things) { - t := b.things[k] + for _, v := range b.things.Snapshot() { + t := v groups := append([]string(nil), b.thingThingGroups[t.ThingName]...) if !matchesThingQuery(t, groups, input.QueryString) { @@ -283,10 +283,10 @@ func (b *InMemoryBackend) searchThingGroupsIndex(input *SearchIndexInput) *Searc b.mu.RLock() defer b.mu.RUnlock() - matched := make([]*SearchIndexThingGroupResult, 0, len(b.thingGroups)) + matched := make([]*SearchIndexThingGroupResult, 0, b.thingGroups.Len()) - for _, k := range sortedKeys(b.thingGroups) { - g := b.thingGroups[k] + for _, v := range b.thingGroups.Snapshot() { + g := v if !matchesThingGroupQuery(g, input.QueryString) { continue } @@ -463,10 +463,10 @@ func containsFold(list []string, v string) bool { // matchedThings returns the Things (and their resolved group names) that // satisfy queryString. Callers must hold at least a read lock. func (b *InMemoryBackend) matchedThings(queryString string) []*Thing { - out := make([]*Thing, 0, len(b.things)) + out := make([]*Thing, 0, b.things.Len()) - for _, k := range sortedKeys(b.things) { - t := b.things[k] + for _, v := range b.things.Snapshot() { + t := v if matchesThingQuery(t, b.thingThingGroups[t.ThingName], queryString) { out = append(out, t) } diff --git a/services/iot/backend_new_ops.go b/services/iot/backend_new_ops.go index ee37cb2a7..37c08a6bc 100644 --- a/services/iot/backend_new_ops.go +++ b/services/iot/backend_new_ops.go @@ -129,7 +129,7 @@ func (b *InMemoryBackend) CreateJob(input *CreateJobInput) (*Job, error) { b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.jobs[input.JobID]; exists { + if b.jobs.Has(input.JobID) { return nil, fmt.Errorf("job %q already exists: %w", input.JobID, ErrAlreadyExists) } now := float64(time.Now().Unix()) @@ -151,7 +151,7 @@ func (b *InMemoryBackend) CreateJob(input *CreateJobInput) (*Job, error) { CreatedAt: now, LastUpdatedAt: now, } - b.jobs[input.JobID] = j + b.jobs.Put(j) return cloneJob(j), nil } @@ -160,7 +160,7 @@ func (b *InMemoryBackend) DescribeJob(jobID string) (*Job, error) { b.mu.RLock() defer b.mu.RUnlock() - j, ok := b.jobs[jobID] + j, ok := b.jobs.Get(jobID) if !ok { return nil, fmt.Errorf("job %q not found: %w", jobID, ErrResourceNotFound) } @@ -172,9 +172,9 @@ func (b *InMemoryBackend) ListJobs() []*Job { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*Job, 0, len(b.jobs)) - for _, k := range sortedKeys(b.jobs) { - out = append(out, cloneJob(b.jobs[k])) + out := make([]*Job, 0, b.jobs.Len()) + for _, v := range b.jobs.Snapshot() { + out = append(out, cloneJob(v)) } return out @@ -184,7 +184,7 @@ func (b *InMemoryBackend) UpdateJob(jobID, description string) error { b.mu.Lock() defer b.mu.Unlock() - j, ok := b.jobs[jobID] + j, ok := b.jobs.Get(jobID) if !ok { return fmt.Errorf("job %q not found: %w", jobID, ErrResourceNotFound) } @@ -200,7 +200,7 @@ func (b *InMemoryBackend) CancelJob(jobID, _ string) (*Job, error) { b.mu.Lock() defer b.mu.Unlock() - j, ok := b.jobs[jobID] + j, ok := b.jobs.Get(jobID) if !ok { return nil, fmt.Errorf("job %q not found: %w", jobID, ErrResourceNotFound) } @@ -214,14 +214,17 @@ func (b *InMemoryBackend) DeleteJob(jobID string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.jobs[jobID]; !ok { + if !b.jobs.Has(jobID) { return fmt.Errorf("job %q not found: %w", jobID, ErrResourceNotFound) } - delete(b.jobs, jobID) - // Remove executions. - for k := range b.jobExecutions { + b.jobs.Delete(jobID) + // Remove executions. Preserves the original key-prefix check (including + // its edge case: a JobExecution with an empty ThingName is NOT deleted, + // since its key jobID+"|" has length exactly len(jobID)+1) byte-for-byte. + for _, exec := range b.jobExecutions.All() { + k := jobExecKey(exec.JobID, exec.ThingName) if len(k) > len(jobID)+1 && k[:len(jobID)] == jobID { - delete(b.jobExecutions, k) + b.jobExecutions.Delete(k) } } @@ -232,7 +235,7 @@ func (b *InMemoryBackend) GetJobDocument(jobID string) (string, error) { b.mu.RLock() defer b.mu.RUnlock() - j, ok := b.jobs[jobID] + j, ok := b.jobs.Get(jobID) if !ok { return "", fmt.Errorf("job %q not found: %w", jobID, ErrResourceNotFound) } @@ -249,7 +252,7 @@ func (b *InMemoryBackend) DescribeJobExecution(jobID, thingName string) (*JobExe defer b.mu.RUnlock() key := jobExecKey(jobID, thingName) - exec, ok := b.jobExecutions[key] + exec, ok := b.jobExecutions.Get(key) if !ok { return nil, fmt.Errorf( "job execution for job %q / thing %q not found: %w", @@ -268,14 +271,14 @@ func (b *InMemoryBackend) CancelJobExecution(jobID, thingName string) error { defer b.mu.Unlock() key := jobExecKey(jobID, thingName) - exec, ok := b.jobExecutions[key] + exec, ok := b.jobExecutions.Get(key) if !ok { // Create a canceled execution record. - b.jobExecutions[key] = &JobExecution{ + b.jobExecutions.Put(&JobExecution{ JobID: jobID, ThingName: thingName, Status: JobExecCanceled, - } + }) return nil } @@ -289,7 +292,7 @@ func (b *InMemoryBackend) DeleteJobExecution(jobID, thingName string) error { defer b.mu.Unlock() key := jobExecKey(jobID, thingName) - delete(b.jobExecutions, key) + b.jobExecutions.Delete(key) return nil } @@ -339,7 +342,7 @@ func (b *InMemoryBackend) CreateJobTemplate(input *CreateJobTemplateInput) (*Job b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.jobTemplates[input.JobTemplateID]; exists { + if b.jobTemplates.Has(input.JobTemplateID) { return nil, fmt.Errorf( "job template %q already exists: %w", input.JobTemplateID, @@ -358,7 +361,7 @@ func (b *InMemoryBackend) CreateJobTemplate(input *CreateJobTemplateInput) (*Job Tags: input.Tags, CreatedAt: float64(time.Now().Unix()), } - b.jobTemplates[input.JobTemplateID] = jt + b.jobTemplates.Put(jt) return cloneJobTemplate(jt), nil } @@ -367,7 +370,7 @@ func (b *InMemoryBackend) DescribeJobTemplate(id string) (*JobTemplate, error) { b.mu.RLock() defer b.mu.RUnlock() - jt, ok := b.jobTemplates[id] + jt, ok := b.jobTemplates.Get(id) if !ok { return nil, fmt.Errorf("job template %q not found: %w", id, ErrResourceNotFound) } @@ -379,9 +382,9 @@ func (b *InMemoryBackend) ListJobTemplates() []*JobTemplate { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*JobTemplate, 0, len(b.jobTemplates)) - for _, k := range sortedKeys(b.jobTemplates) { - out = append(out, cloneJobTemplate(b.jobTemplates[k])) + out := make([]*JobTemplate, 0, b.jobTemplates.Len()) + for _, v := range b.jobTemplates.Snapshot() { + out = append(out, cloneJobTemplate(v)) } return out @@ -391,10 +394,10 @@ func (b *InMemoryBackend) DeleteJobTemplate(id string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.jobTemplates[id]; !ok { + if !b.jobTemplates.Has(id) { return fmt.Errorf("job template %q not found: %w", id, ErrResourceNotFound) } - delete(b.jobTemplates, id) + b.jobTemplates.Delete(id) return nil } @@ -437,7 +440,7 @@ func (b *InMemoryBackend) CreateRoleAlias(input *CreateRoleAliasInput) (*RoleAli b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.roleAliases[input.RoleAlias]; exists { + if b.roleAliases.Has(input.RoleAlias) { return nil, fmt.Errorf( "role alias %q already exists: %w", input.RoleAlias, @@ -457,7 +460,7 @@ func (b *InMemoryBackend) CreateRoleAlias(input *CreateRoleAliasInput) (*RoleAli if ra.CredentialDurationSeconds == 0 { ra.CredentialDurationSeconds = 3600 } - b.roleAliases[input.RoleAlias] = ra + b.roleAliases.Put(ra) return cloneRoleAlias(ra), nil } @@ -466,7 +469,7 @@ func (b *InMemoryBackend) DescribeRoleAlias(alias string) (*RoleAlias, error) { b.mu.RLock() defer b.mu.RUnlock() - ra, ok := b.roleAliases[alias] + ra, ok := b.roleAliases.Get(alias) if !ok { return nil, fmt.Errorf("role alias %q not found: %w", alias, ErrResourceNotFound) } @@ -478,9 +481,9 @@ func (b *InMemoryBackend) ListRoleAliases() []*RoleAlias { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*RoleAlias, 0, len(b.roleAliases)) - for _, k := range sortedKeys(b.roleAliases) { - out = append(out, cloneRoleAlias(b.roleAliases[k])) + out := make([]*RoleAlias, 0, b.roleAliases.Len()) + for _, v := range b.roleAliases.Snapshot() { + out = append(out, cloneRoleAlias(v)) } return out @@ -493,7 +496,7 @@ func (b *InMemoryBackend) UpdateRoleAlias( b.mu.Lock() defer b.mu.Unlock() - ra, ok := b.roleAliases[alias] + ra, ok := b.roleAliases.Get(alias) if !ok { return nil, fmt.Errorf("role alias %q not found: %w", alias, ErrResourceNotFound) } @@ -512,10 +515,10 @@ func (b *InMemoryBackend) DeleteRoleAlias(alias string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.roleAliases[alias]; !ok { + if !b.roleAliases.Has(alias) { return fmt.Errorf("role alias %q not found: %w", alias, ErrResourceNotFound) } - delete(b.roleAliases, alias) + b.roleAliases.Delete(alias) return nil } @@ -561,7 +564,7 @@ func (b *InMemoryBackend) CreateDomainConfiguration( b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.domainConfigs[input.DomainConfigurationName]; exists { + if b.domainConfigs.Has(input.DomainConfigurationName) { return nil, fmt.Errorf( "domain configuration %q already exists: %w", input.DomainConfigurationName, @@ -582,7 +585,7 @@ func (b *InMemoryBackend) CreateDomainConfiguration( if dc.ServiceType == "" { dc.ServiceType = "DATA" } - b.domainConfigs[input.DomainConfigurationName] = dc + b.domainConfigs.Put(dc) return cloneDomainConfig(dc), nil } @@ -591,7 +594,7 @@ func (b *InMemoryBackend) DescribeDomainConfiguration(name string) (*DomainConfi b.mu.RLock() defer b.mu.RUnlock() - dc, ok := b.domainConfigs[name] + dc, ok := b.domainConfigs.Get(name) if !ok { return nil, fmt.Errorf("domain configuration %q not found: %w", name, ErrResourceNotFound) } @@ -603,9 +606,9 @@ func (b *InMemoryBackend) ListDomainConfigurations() []*DomainConfiguration { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*DomainConfiguration, 0, len(b.domainConfigs)) - for _, k := range sortedKeys(b.domainConfigs) { - out = append(out, cloneDomainConfig(b.domainConfigs[k])) + out := make([]*DomainConfiguration, 0, b.domainConfigs.Len()) + for _, v := range b.domainConfigs.Snapshot() { + out = append(out, cloneDomainConfig(v)) } return out @@ -617,7 +620,7 @@ func (b *InMemoryBackend) UpdateDomainConfiguration( b.mu.Lock() defer b.mu.Unlock() - dc, ok := b.domainConfigs[name] + dc, ok := b.domainConfigs.Get(name) if !ok { return nil, fmt.Errorf("domain configuration %q not found: %w", name, ErrResourceNotFound) } @@ -633,10 +636,10 @@ func (b *InMemoryBackend) DeleteDomainConfiguration(name string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.domainConfigs[name]; !ok { + if !b.domainConfigs.Has(name) { return fmt.Errorf("domain configuration %q not found: %w", name, ErrResourceNotFound) } - delete(b.domainConfigs, name) + b.domainConfigs.Delete(name) return nil } @@ -695,7 +698,7 @@ func (b *InMemoryBackend) CreateProvisioningTemplate( b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.provTemplates[input.TemplateName]; exists { + if b.provTemplates.Has(input.TemplateName) { return nil, fmt.Errorf( "provisioning template %q already exists: %w", input.TemplateName, @@ -716,7 +719,7 @@ func (b *InMemoryBackend) CreateProvisioningTemplate( CreationDate: now, LastModifiedDate: now, } - b.provTemplates[input.TemplateName] = pt + b.provTemplates.Put(pt) // Create initial version. b.provTemplateVersions[input.TemplateName] = []*ProvisioningTemplateVersion{ {VersionID: 1, TemplateBody: input.TemplateBody, CreationDate: now, IsDefaultVersion: true}, @@ -729,7 +732,7 @@ func (b *InMemoryBackend) DescribeProvisioningTemplate(name string) (*Provisioni b.mu.RLock() defer b.mu.RUnlock() - pt, ok := b.provTemplates[name] + pt, ok := b.provTemplates.Get(name) if !ok { return nil, fmt.Errorf("provisioning template %q not found: %w", name, ErrResourceNotFound) } @@ -741,9 +744,9 @@ func (b *InMemoryBackend) ListProvisioningTemplates() []*ProvisioningTemplate { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*ProvisioningTemplate, 0, len(b.provTemplates)) - for _, k := range sortedKeys(b.provTemplates) { - out = append(out, cloneProvTemplate(b.provTemplates[k])) + out := make([]*ProvisioningTemplate, 0, b.provTemplates.Len()) + for _, v := range b.provTemplates.Snapshot() { + out = append(out, cloneProvTemplate(v)) } return out @@ -757,7 +760,7 @@ func (b *InMemoryBackend) UpdateProvisioningTemplate( b.mu.Lock() defer b.mu.Unlock() - pt, ok := b.provTemplates[name] + pt, ok := b.provTemplates.Get(name) if !ok { return fmt.Errorf("provisioning template %q not found: %w", name, ErrResourceNotFound) } @@ -779,10 +782,10 @@ func (b *InMemoryBackend) DeleteProvisioningTemplate(name string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.provTemplates[name]; !ok { + if !b.provTemplates.Has(name) { return fmt.Errorf("provisioning template %q not found: %w", name, ErrResourceNotFound) } - delete(b.provTemplates, name) + b.provTemplates.Delete(name) delete(b.provTemplateVersions, name) return nil @@ -794,7 +797,7 @@ func (b *InMemoryBackend) CreateProvisioningTemplateVersion( b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.provTemplates[name]; !ok { + if !b.provTemplates.Has(name) { return nil, fmt.Errorf("provisioning template %q not found: %w", name, ErrResourceNotFound) } versions := b.provTemplateVersions[name] @@ -815,7 +818,7 @@ func (b *InMemoryBackend) ListProvisioningTemplateVersions( b.mu.RLock() defer b.mu.RUnlock() - if _, ok := b.provTemplates[name]; !ok { + if !b.provTemplates.Has(name) { return nil, fmt.Errorf("provisioning template %q not found: %w", name, ErrResourceNotFound) } src := b.provTemplateVersions[name] @@ -829,7 +832,7 @@ func (b *InMemoryBackend) DeleteProvisioningTemplateVersion(name string, version b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.provTemplates[name]; !ok { + if !b.provTemplates.Has(name) { return fmt.Errorf("provisioning template %q not found: %w", name, ErrResourceNotFound) } versions := b.provTemplateVersions[name] @@ -889,7 +892,7 @@ func (b *InMemoryBackend) CreateAuthorizer(input *CreateAuthorizerInput) (*Autho b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.authorizers[input.AuthorizerName]; exists { + if b.authorizers.Has(input.AuthorizerName) { return nil, fmt.Errorf( "authorizer %q already exists: %w", input.AuthorizerName, @@ -913,7 +916,7 @@ func (b *InMemoryBackend) CreateAuthorizer(input *CreateAuthorizerInput) (*Autho if a.Status == "" { a.Status = "ACTIVE" } - b.authorizers[input.AuthorizerName] = a + b.authorizers.Put(a) return cloneAuthorizer(a), nil } @@ -922,7 +925,7 @@ func (b *InMemoryBackend) DescribeAuthorizer(name string) (*Authorizer, error) { b.mu.RLock() defer b.mu.RUnlock() - a, ok := b.authorizers[name] + a, ok := b.authorizers.Get(name) if !ok { return nil, fmt.Errorf("authorizer %q not found: %w", name, ErrResourceNotFound) } @@ -934,9 +937,9 @@ func (b *InMemoryBackend) ListAuthorizers() []*Authorizer { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*Authorizer, 0, len(b.authorizers)) - for _, k := range sortedKeys(b.authorizers) { - out = append(out, cloneAuthorizer(b.authorizers[k])) + out := make([]*Authorizer, 0, b.authorizers.Len()) + for _, v := range b.authorizers.Snapshot() { + out = append(out, cloneAuthorizer(v)) } return out @@ -946,7 +949,7 @@ func (b *InMemoryBackend) UpdateAuthorizer(name, functionARN, status string) (*A b.mu.Lock() defer b.mu.Unlock() - a, ok := b.authorizers[name] + a, ok := b.authorizers.Get(name) if !ok { return nil, fmt.Errorf("authorizer %q not found: %w", name, ErrResourceNotFound) } @@ -965,10 +968,10 @@ func (b *InMemoryBackend) DeleteAuthorizer(name string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.authorizers[name]; !ok { + if !b.authorizers.Has(name) { return fmt.Errorf("authorizer %q not found: %w", name, ErrResourceNotFound) } - delete(b.authorizers, name) + b.authorizers.Delete(name) return nil } @@ -1021,7 +1024,7 @@ func (b *InMemoryBackend) CreateBillingGroup( b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.billingGroups[input.BillingGroupName]; exists { + if b.billingGroups.Has(input.BillingGroupName) { return nil, fmt.Errorf( "billing group %q already exists: %w", input.BillingGroupName, @@ -1039,7 +1042,7 @@ func (b *InMemoryBackend) CreateBillingGroup( Tags: input.Tags, Version: 1, } - b.billingGroups[input.BillingGroupName] = bg + b.billingGroups.Put(bg) return cloneBillingGroup(bg), nil } @@ -1048,7 +1051,7 @@ func (b *InMemoryBackend) DescribeBillingGroup(name string) (*BillingGroup, erro b.mu.RLock() defer b.mu.RUnlock() - bg, ok := b.billingGroups[name] + bg, ok := b.billingGroups.Get(name) if !ok { return nil, fmt.Errorf("billing group %q not found: %w", name, ErrResourceNotFound) } @@ -1060,9 +1063,9 @@ func (b *InMemoryBackend) ListBillingGroups() []*BillingGroup { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*BillingGroup, 0, len(b.billingGroups)) - for _, k := range sortedKeys(b.billingGroups) { - out = append(out, cloneBillingGroup(b.billingGroups[k])) + out := make([]*BillingGroup, 0, b.billingGroups.Len()) + for _, v := range b.billingGroups.Snapshot() { + out = append(out, cloneBillingGroup(v)) } return out @@ -1075,7 +1078,7 @@ func (b *InMemoryBackend) UpdateBillingGroup( b.mu.Lock() defer b.mu.Unlock() - bg, ok := b.billingGroups[name] + bg, ok := b.billingGroups.Get(name) if !ok { return 0, fmt.Errorf("billing group %q not found: %w", name, ErrResourceNotFound) } @@ -1089,10 +1092,10 @@ func (b *InMemoryBackend) DeleteBillingGroup(name string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.billingGroups[name]; !ok { + if !b.billingGroups.Has(name) { return fmt.Errorf("billing group %q not found: %w", name, ErrResourceNotFound) } - delete(b.billingGroups, name) + b.billingGroups.Delete(name) return nil } @@ -1139,7 +1142,7 @@ func (b *InMemoryBackend) CreateScheduledAudit( b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.scheduledAudits[input.ScheduledAuditName]; exists { + if b.scheduledAudits.Has(input.ScheduledAuditName) { return nil, fmt.Errorf( "scheduled audit %q already exists: %w", input.ScheduledAuditName, @@ -1155,7 +1158,7 @@ func (b *InMemoryBackend) CreateScheduledAudit( TargetCheckNames: append([]string(nil), input.TargetCheckNames...), Tags: input.Tags, } - b.scheduledAudits[input.ScheduledAuditName] = sa + b.scheduledAudits.Put(sa) return cloneScheduledAudit(sa), nil } @@ -1164,7 +1167,7 @@ func (b *InMemoryBackend) DescribeScheduledAudit(name string) (*ScheduledAudit, b.mu.RLock() defer b.mu.RUnlock() - sa, ok := b.scheduledAudits[name] + sa, ok := b.scheduledAudits.Get(name) if !ok { return nil, fmt.Errorf("scheduled audit %q not found: %w", name, ErrResourceNotFound) } @@ -1176,9 +1179,9 @@ func (b *InMemoryBackend) ListScheduledAudits() []*ScheduledAudit { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*ScheduledAudit, 0, len(b.scheduledAudits)) - for _, k := range sortedKeys(b.scheduledAudits) { - out = append(out, cloneScheduledAudit(b.scheduledAudits[k])) + out := make([]*ScheduledAudit, 0, b.scheduledAudits.Len()) + for _, v := range b.scheduledAudits.Snapshot() { + out = append(out, cloneScheduledAudit(v)) } return out @@ -1191,7 +1194,7 @@ func (b *InMemoryBackend) UpdateScheduledAudit( b.mu.Lock() defer b.mu.Unlock() - sa, ok := b.scheduledAudits[name] + sa, ok := b.scheduledAudits.Get(name) if !ok { return nil, fmt.Errorf("scheduled audit %q not found: %w", name, ErrResourceNotFound) } @@ -1215,10 +1218,10 @@ func (b *InMemoryBackend) DeleteScheduledAudit(name string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.scheduledAudits[name]; !ok { + if !b.scheduledAudits.Has(name) { return fmt.Errorf("scheduled audit %q not found: %w", name, ErrResourceNotFound) } - delete(b.scheduledAudits, name) + b.scheduledAudits.Delete(name) return nil } @@ -1263,7 +1266,7 @@ func (b *InMemoryBackend) CreateMitigationAction( b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.mitigationActions[input.ActionName]; exists { + if b.mitigationActions.Has(input.ActionName) { return nil, fmt.Errorf( "mitigation action %q already exists: %w", input.ActionName, @@ -1281,7 +1284,7 @@ func (b *InMemoryBackend) CreateMitigationAction( CreationDate: now, LastModifiedDate: now, } - b.mitigationActions[input.ActionName] = ma + b.mitigationActions.Put(ma) return cloneMitigationAction(ma), nil } @@ -1290,7 +1293,7 @@ func (b *InMemoryBackend) DescribeMitigationAction(name string) (*MitigationActi b.mu.RLock() defer b.mu.RUnlock() - ma, ok := b.mitigationActions[name] + ma, ok := b.mitigationActions.Get(name) if !ok { return nil, fmt.Errorf("mitigation action %q not found: %w", name, ErrResourceNotFound) } @@ -1302,9 +1305,9 @@ func (b *InMemoryBackend) ListMitigationActions() []*MitigationAction { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*MitigationAction, 0, len(b.mitigationActions)) - for _, k := range sortedKeys(b.mitigationActions) { - out = append(out, cloneMitigationAction(b.mitigationActions[k])) + out := make([]*MitigationAction, 0, b.mitigationActions.Len()) + for _, v := range b.mitigationActions.Snapshot() { + out = append(out, cloneMitigationAction(v)) } return out @@ -1317,7 +1320,7 @@ func (b *InMemoryBackend) UpdateMitigationAction( b.mu.Lock() defer b.mu.Unlock() - ma, ok := b.mitigationActions[name] + ma, ok := b.mitigationActions.Get(name) if !ok { return nil, fmt.Errorf("mitigation action %q not found: %w", name, ErrResourceNotFound) } @@ -1336,10 +1339,10 @@ func (b *InMemoryBackend) DeleteMitigationAction(name string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.mitigationActions[name]; !ok { + if !b.mitigationActions.Has(name) { return fmt.Errorf("mitigation action %q not found: %w", name, ErrResourceNotFound) } - delete(b.mitigationActions, name) + b.mitigationActions.Delete(name) return nil } @@ -1382,7 +1385,7 @@ func (b *InMemoryBackend) CreateSecurityProfile( b.mu.Lock() defer b.mu.Unlock() - if _, exists := b.securityProfiles[input.SecurityProfileName]; exists { + if b.securityProfiles.Has(input.SecurityProfileName) { return nil, fmt.Errorf( "security profile %q already exists: %w", input.SecurityProfileName, @@ -1399,7 +1402,7 @@ func (b *InMemoryBackend) CreateSecurityProfile( CreationDate: now, LastModifiedDate: now, } - b.securityProfiles[input.SecurityProfileName] = sp + b.securityProfiles.Put(sp) return cloneSecurityProfile(sp), nil } @@ -1408,7 +1411,7 @@ func (b *InMemoryBackend) DescribeSecurityProfile(name string) (*SecurityProfile b.mu.RLock() defer b.mu.RUnlock() - sp, ok := b.securityProfiles[name] + sp, ok := b.securityProfiles.Get(name) if !ok { return nil, fmt.Errorf("security profile %q not found: %w", name, ErrResourceNotFound) } @@ -1420,9 +1423,9 @@ func (b *InMemoryBackend) ListSecurityProfiles() []*SecurityProfile { b.mu.RLock() defer b.mu.RUnlock() - out := make([]*SecurityProfile, 0, len(b.securityProfiles)) - for _, k := range sortedKeys(b.securityProfiles) { - out = append(out, cloneSecurityProfile(b.securityProfiles[k])) + out := make([]*SecurityProfile, 0, b.securityProfiles.Len()) + for _, v := range b.securityProfiles.Snapshot() { + out = append(out, cloneSecurityProfile(v)) } return out @@ -1434,7 +1437,7 @@ func (b *InMemoryBackend) UpdateSecurityProfile( b.mu.Lock() defer b.mu.Unlock() - sp, ok := b.securityProfiles[name] + sp, ok := b.securityProfiles.Get(name) if !ok { return nil, fmt.Errorf("security profile %q not found: %w", name, ErrResourceNotFound) } @@ -1451,10 +1454,10 @@ func (b *InMemoryBackend) DeleteSecurityProfile(name string) error { b.mu.Lock() defer b.mu.Unlock() - if _, ok := b.securityProfiles[name]; !ok { + if !b.securityProfiles.Has(name) { return fmt.Errorf("security profile %q not found: %w", name, ErrResourceNotFound) } - delete(b.securityProfiles, name) + b.securityProfiles.Delete(name) return nil } diff --git a/services/iot/backend_persistence_ext.go b/services/iot/backend_persistence_ext.go index 5d7e264f3..da92ef598 100644 --- a/services/iot/backend_persistence_ext.go +++ b/services/iot/backend_persistence_ext.go @@ -1,6 +1,12 @@ package iot -import "maps" +import ( + "encoding/json" + "fmt" + "maps" + + "github.com/blackbirdworks/gopherstack/pkgs/store" +) // --------------------------------------------------------------------------- // This file closes a persistence gap (gopherstack-264): several live @@ -10,6 +16,20 @@ import "maps" // backend_devicedefender.go and backend_final_ops.go: each group of related // fields gets its own bundle struct plus a pair of snapshot/restore methods, // keeping Snapshot()/Restore() themselves within the cyclop/funlen limits. +// +// Phase 3.3 note: most of the *T-valued maps these groups used to carry +// (ThingTypes, ThingGroups, Certificates, CertificateProviders, +// CACertificates, Jobs, JobExecutions, JobTemplates, RoleAliases, +// DomainConfigs, Authorizers, BillingGroups, ProvTemplates, +// ScheduledAudits, MitigationActions, SecurityProfiles, AuditSuppressions, +// AuditFindings, AuditTaskObjects, Dimensions, Streams, OTAUpdates, +// IoTPackages, Commands, FleetMetrics, CustomMetrics, V2LoggingLevels) moved +// to store.Table[T]s registered on b.registry (see store_setup.go) and now +// round-trip via registry.SnapshotAll()/RestoreAll() in persistence.go +// instead of through these bundles. What remains here is exactly the raw +// (non-Table) state: slice-valued maps, nested maps, and the one "dirty" +// table (TopicRuleDestination, handled directly in persistence.go via its +// own small DTO registry, mirroring the services/sqs pilot). // --------------------------------------------------------------------------- // topicRuleDestSnap mirrors TopicRuleDestination for persistence purposes. @@ -24,6 +44,10 @@ type topicRuleDestSnap struct { ConfirmationToken string `json:"confirmationToken,omitempty"` } +// topicRuleDestSnapKey is the store.Table key function used for the +// ephemeral DTO table built inside Snapshot/Restore for topicRuleDestinations. +func topicRuleDestSnapKey(s *topicRuleDestSnap) string { return s.ARN } + func toTopicRuleDestSnap(d *TopicRuleDestination) *topicRuleDestSnap { var props *HTTPURLDestinationProperties if d.HTTPURLProperties != nil { @@ -54,28 +78,51 @@ func fromTopicRuleDestSnap(s *topicRuleDestSnap) *TopicRuleDestination { } } -func cloneThingType(t *ThingType) *ThingType { - cp := *t - cp.SearchableAttributes = append([]string(nil), t.SearchableAttributes...) - - return &cp -} +// snapshotTopicRuleDestinationsTable builds the "dirty" topicRuleDestinations +// entry for backendSnapshot.Tables via a small throwaway DTO registry, +// mirroring the services/sqs pilot's DTO-registry pattern but scoped to just +// this one table. TopicRuleDestination.ConfirmationToken is tagged json:"-" +// (AWS delivers it out-of-band and it must never leak into API responses), +// so registry.SnapshotAll's generic per-table encoding of the live type would +// otherwise silently drop it; the topicRuleDestSnap DTO carries it through +// instead. Extracted from Snapshot to keep it within the repo's funlen limit. +// Must be called with b.mu held (read or write). +func (b *InMemoryBackend) snapshotTopicRuleDestinationsTable() (map[string]json.RawMessage, error) { + destDTOReg := store.NewRegistry() + destDTOs := store.Register(destDTOReg, topicRuleDestinationsTableName, store.New(topicRuleDestSnapKey)) -func cloneThingGroup(g *ThingGroup) *ThingGroup { - cp := *g - if g.Attributes != nil { - cp.Attributes = maps.Clone(g.Attributes) + for _, d := range b.topicRuleDestinations.Snapshot() { + destDTOs.Put(toTopicRuleDestSnap(d)) } - cp.Members = append([]string(nil), g.Members...) + tables, err := destDTOReg.SnapshotAll() + if err != nil { + return nil, fmt.Errorf("iot: snapshot topicRuleDestinations marshal failed: %w", err) + } - return &cp + return tables, nil } -func cloneCertificate(c *Certificate) *Certificate { - cp := *c +// restoreTopicRuleDestinationsTable restores b.topicRuleDestinations from its +// DTO entry in tables (the inverse of snapshotTopicRuleDestinationsTable). +// Extracted from Restore to keep it within the repo's funlen limit. Must be +// called with b.mu held (write). +func (b *InMemoryBackend) restoreTopicRuleDestinationsTable(tables map[string]json.RawMessage) error { + destDTOReg := store.NewRegistry() + destDTOs := store.Register(destDTOReg, topicRuleDestinationsTableName, store.New(topicRuleDestSnapKey)) - return &cp + if err := destDTOReg.RestoreAll(tables); err != nil { + return fmt.Errorf("iot: restore topicRuleDestinations: %w", err) + } + + liveDests := make([]*TopicRuleDestination, 0, destDTOs.Len()) + for _, s := range destDTOs.All() { + liveDests = append(liveDests, fromTopicRuleDestSnap(s)) + } + + b.topicRuleDestinations.Restore(liveDests) + + return nil } func clonePolicyVersion(v *PolicyVersion) *PolicyVersion { @@ -93,19 +140,6 @@ func clonePolicyVersions(src []*PolicyVersion) []*PolicyVersion { return out } -func cloneCertificateProvider(p *CertificateProvider) *CertificateProvider { - cp := *p - cp.AccountDefaultForOperations = append([]string(nil), p.AccountDefaultForOperations...) - - return &cp -} - -func cloneJobExecution(e *JobExecution) *JobExecution { - cp := *e - - return &cp -} - func cloneProvTemplateVersion(v *ProvisioningTemplateVersion) *ProvisioningTemplateVersion { cp := *v @@ -121,12 +155,6 @@ func cloneProvTemplateVersions(src []*ProvisioningTemplateVersion) []*Provisioni return out } -func cloneAuditTaskObj(t *AuditTask) *AuditTask { - cp := *t - - return &cp -} - func cloneIoTCommandExecution(e *IoTCommandExecution) *IoTCommandExecution { cp := *e @@ -167,625 +195,130 @@ func copyNestedStringMap(m map[string]map[string]string) map[string]map[string]s return cp } -// applyExtSnapshot copies the five gopherstack-264 group snapshots onto snap. -// Extracted from Snapshot() to keep it within the repo's funlen limit. -func applyExtSnapshot( - snap *backendSnapshot, - thingRes thingResourceSnapshot, - prov provisioningSnapshot, - auditExtra auditExtraSnapshot, - misc resourceMiscSnapshot, - cfg configSnapshot, -) { - snap.ThingTypes = thingRes.ThingTypes - snap.ThingGroups = thingRes.ThingGroups - snap.ThingGroupMembers = thingRes.ThingGroupMembers - snap.Certificates = thingRes.Certificates - snap.CertificateProviders = thingRes.CertificateProviders - snap.CACertificates = thingRes.CACertificates - snap.PolicyVersions = thingRes.PolicyVersions - snap.TopicRuleDestinations = thingRes.TopicRuleDestinations - snap.ResourceTags = thingRes.ResourceTags - - snap.Jobs = prov.Jobs - snap.JobExecutions = prov.JobExecutions - snap.JobTemplates = prov.JobTemplates - snap.RoleAliases = prov.RoleAliases - snap.DomainConfigs = prov.DomainConfigs - snap.Authorizers = prov.Authorizers - snap.BillingGroups = prov.BillingGroups - snap.ProvTemplates = prov.ProvTemplates - snap.ProvTemplateVersions = prov.ProvTemplateVersions - - snap.ScheduledAudits = auditExtra.ScheduledAudits - snap.MitigationActions = auditExtra.MitigationActions - snap.SecurityProfiles = auditExtra.SecurityProfiles - snap.AuditSuppressions = auditExtra.AuditSuppressions - snap.AuditFindings = auditExtra.AuditFindings - snap.AuditTaskObjects = auditExtra.AuditTaskObjects - snap.Dimensions = auditExtra.Dimensions - - snap.Streams = misc.Streams - snap.OTAUpdates = misc.OTAUpdates - snap.IoTPackages = misc.IoTPackages - snap.PackageVersions2 = misc.PackageVersions2 - snap.Commands = misc.Commands - snap.CommandExecutions = misc.CommandExecutions - snap.FleetMetrics = misc.FleetMetrics - snap.CustomMetrics = misc.CustomMetrics - snap.V2LoggingLevels = misc.V2LoggingLevels - - snap.AuditConfiguration = cfg.AuditConfiguration - snap.PackageConfig = cfg.PackageConfig - snap.V2LoggingOptions = cfg.V2LoggingOptions - snap.LoggingOptions = cfg.LoggingOptions - snap.EventConfigurations = cfg.EventConfigurations - snap.RegistrationCode = cfg.RegistrationCode - snap.DefaultAuthorizer = cfg.DefaultAuthorizer -} - -// extGroupsFromSnapshot extracts the five gopherstack-264 group snapshots -// from a restored backendSnapshot. Extracted from Restore() to keep it -// within the repo's funlen limit. -func extGroupsFromSnapshot( - snap *backendSnapshot, -) (thingResourceSnapshot, provisioningSnapshot, auditExtraSnapshot, resourceMiscSnapshot, configSnapshot) { - thingRes := thingResourceSnapshot{ - ThingTypes: snap.ThingTypes, - ThingGroups: snap.ThingGroups, - ThingGroupMembers: snap.ThingGroupMembers, - Certificates: snap.Certificates, - CertificateProviders: snap.CertificateProviders, - CACertificates: snap.CACertificates, - PolicyVersions: snap.PolicyVersions, - TopicRuleDestinations: snap.TopicRuleDestinations, - ResourceTags: snap.ResourceTags, - } - - prov := provisioningSnapshot{ - Jobs: snap.Jobs, - JobExecutions: snap.JobExecutions, - JobTemplates: snap.JobTemplates, - RoleAliases: snap.RoleAliases, - DomainConfigs: snap.DomainConfigs, - Authorizers: snap.Authorizers, - BillingGroups: snap.BillingGroups, - ProvTemplates: snap.ProvTemplates, - ProvTemplateVersions: snap.ProvTemplateVersions, - } - - auditExtra := auditExtraSnapshot{ - ScheduledAudits: snap.ScheduledAudits, - MitigationActions: snap.MitigationActions, - SecurityProfiles: snap.SecurityProfiles, - AuditSuppressions: snap.AuditSuppressions, - AuditFindings: snap.AuditFindings, - AuditTaskObjects: snap.AuditTaskObjects, - Dimensions: snap.Dimensions, - } - - misc := resourceMiscSnapshot{ - Streams: snap.Streams, - OTAUpdates: snap.OTAUpdates, - IoTPackages: snap.IoTPackages, - PackageVersions2: snap.PackageVersions2, - Commands: snap.Commands, - CommandExecutions: snap.CommandExecutions, - FleetMetrics: snap.FleetMetrics, - CustomMetrics: snap.CustomMetrics, - V2LoggingLevels: snap.V2LoggingLevels, - } - - cfg := configSnapshot{ - AuditConfiguration: snap.AuditConfiguration, - PackageConfig: snap.PackageConfig, - V2LoggingOptions: snap.V2LoggingOptions, - LoggingOptions: snap.LoggingOptions, - EventConfigurations: snap.EventConfigurations, - RegistrationCode: snap.RegistrationCode, - DefaultAuthorizer: snap.DefaultAuthorizer, - } - - return thingRes, prov, auditExtra, misc, cfg -} - // --------------------------------------------------------------------------- -// Group 1: Things, groups, and certificates. +// Group 1: Raw thing/group/certificate-family state (everything in this +// family that is *T-valued now lives in a store.Table on b.registry instead). // --------------------------------------------------------------------------- -// thingResourceSnapshot bundles ThingType/ThingGroup/Certificate-family -// fields of backendSnapshot so Snapshot/Restore can delegate to a single -// helper each, keeping their own cyclomatic complexity low. +// thingResourceSnapshot bundles the raw (non-Table) ThingGroup/Certificate +// -family fields of backendSnapshot so Snapshot/Restore can delegate to a +// single helper each, keeping their own cyclomatic complexity low. type thingResourceSnapshot struct { - ThingTypes map[string]*ThingType - ThingGroups map[string]*ThingGroup - ThingGroupMembers map[string][]string - Certificates map[string]*Certificate - CertificateProviders map[string]*CertificateProvider - CACertificates map[string]*CACertificate - PolicyVersions map[string][]*PolicyVersion - TopicRuleDestinations map[string]*topicRuleDestSnap - ResourceTags map[string]map[string]string + ThingGroupMembers map[string][]string + PolicyVersions map[string][]*PolicyVersion + ResourceTags map[string]map[string]string } -// snapshotThingResources deep-copies all thing/group/certificate state. Must -// be called with b.mu held (read or write). +// snapshotThingResources deep-copies the raw thing/group/certificate state. +// Must be called with b.mu held (read or write). func (b *InMemoryBackend) snapshotThingResources() thingResourceSnapshot { - thingTypes := make(map[string]*ThingType, len(b.thingTypes)) - for k, v := range b.thingTypes { - thingTypes[k] = cloneThingType(v) - } - - thingGroups := make(map[string]*ThingGroup, len(b.thingGroups)) - for k, v := range b.thingGroups { - thingGroups[k] = cloneThingGroup(v) - } - - certificates := make(map[string]*Certificate, len(b.certificates)) - for k, v := range b.certificates { - certificates[k] = cloneCertificate(v) - } - - certProviders := make(map[string]*CertificateProvider, len(b.certificateProviders)) - for k, v := range b.certificateProviders { - certProviders[k] = cloneCertificateProvider(v) - } - - caCerts := make(map[string]*CACertificate, len(b.caCertificates)) - for k, v := range b.caCertificates { - caCerts[k] = cloneCACert(v) - } - policyVersions := make(map[string][]*PolicyVersion, len(b.policyVersions)) for k, v := range b.policyVersions { policyVersions[k] = clonePolicyVersions(v) } - destinations := make(map[string]*topicRuleDestSnap, len(b.topicRuleDestinations)) - for k, v := range b.topicRuleDestinations { - destinations[k] = toTopicRuleDestSnap(v) - } - return thingResourceSnapshot{ - ThingTypes: thingTypes, - ThingGroups: thingGroups, - ThingGroupMembers: copyStringSliceMap(b.thingGroupMembers), - Certificates: certificates, - CertificateProviders: certProviders, - CACertificates: caCerts, - PolicyVersions: policyVersions, - TopicRuleDestinations: destinations, - ResourceTags: copyNestedStringMap(b.resourceTags), + ThingGroupMembers: copyStringSliceMap(b.thingGroupMembers), + PolicyVersions: policyVersions, + ResourceTags: copyNestedStringMap(b.resourceTags), } } -// restoreThingResources restores thing/group/certificate state from a -// snapshot. Must be called with b.mu held (write). +// restoreThingResources restores the raw thing/group/certificate state from +// a snapshot. Must be called with b.mu held (write). func (b *InMemoryBackend) restoreThingResources(snap thingResourceSnapshot) { - b.thingTypes = make(map[string]*ThingType, len(snap.ThingTypes)) - for k, v := range snap.ThingTypes { - b.thingTypes[k] = cloneThingType(v) - } - - b.thingGroups = make(map[string]*ThingGroup, len(snap.ThingGroups)) - for k, v := range snap.ThingGroups { - b.thingGroups[k] = cloneThingGroup(v) - } - b.thingGroupMembers = copyStringSliceMap(snap.ThingGroupMembers) - b.certificates = make(map[string]*Certificate, len(snap.Certificates)) - for k, v := range snap.Certificates { - b.certificates[k] = cloneCertificate(v) - } - - b.certificateProviders = make(map[string]*CertificateProvider, len(snap.CertificateProviders)) - for k, v := range snap.CertificateProviders { - b.certificateProviders[k] = cloneCertificateProvider(v) - } - - b.caCertificates = make(map[string]*CACertificate, len(snap.CACertificates)) - for k, v := range snap.CACertificates { - b.caCertificates[k] = cloneCACert(v) - } - b.policyVersions = make(map[string][]*PolicyVersion, len(snap.PolicyVersions)) for k, v := range snap.PolicyVersions { b.policyVersions[k] = clonePolicyVersions(v) } - b.topicRuleDestinations = make(map[string]*TopicRuleDestination, len(snap.TopicRuleDestinations)) - for k, v := range snap.TopicRuleDestinations { - b.topicRuleDestinations[k] = fromTopicRuleDestSnap(v) - } - b.resourceTags = copyNestedStringMap(snap.ResourceTags) } // ensureNonNilThingResourceSnap defaults nil maps in a restored snapshot's -// thing/group/certificate fields to empty maps. +// raw thing/group/certificate fields to empty maps. func ensureNonNilThingResourceSnap(snap *backendSnapshot) { - if snap.ThingTypes == nil { - snap.ThingTypes = make(map[string]*ThingType) - } - - if snap.ThingGroups == nil { - snap.ThingGroups = make(map[string]*ThingGroup) - } - if snap.ThingGroupMembers == nil { snap.ThingGroupMembers = make(map[string][]string) } - if snap.Certificates == nil { - snap.Certificates = make(map[string]*Certificate) - } - - if snap.CertificateProviders == nil { - snap.CertificateProviders = make(map[string]*CertificateProvider) - } - - if snap.CACertificates == nil { - snap.CACertificates = make(map[string]*CACertificate) - } - if snap.PolicyVersions == nil { snap.PolicyVersions = make(map[string][]*PolicyVersion) } - if snap.TopicRuleDestinations == nil { - snap.TopicRuleDestinations = make(map[string]*topicRuleDestSnap) - } - if snap.ResourceTags == nil { snap.ResourceTags = make(map[string]map[string]string) } } // --------------------------------------------------------------------------- -// Group 2: Jobs and provisioning resources. +// Group 2: Raw provisioning-template state (Job/JobTemplate/RoleAlias/ +// DomainConfiguration/Authorizer/BillingGroup/ProvisioningTemplate are now +// store.Table[T]s; only the slice-valued ProvTemplateVersions stays raw). // --------------------------------------------------------------------------- -// provisioningSnapshot bundles Job/JobTemplate/RoleAlias/DomainConfiguration/ -// Authorizer/BillingGroup/ProvisioningTemplate-family fields of -// backendSnapshot so Snapshot/Restore can delegate to a single helper each. +// provisioningSnapshot bundles the raw (non-Table) provisioning-template +// field of backendSnapshot. type provisioningSnapshot struct { - Jobs map[string]*Job - JobExecutions map[string]*JobExecution - JobTemplates map[string]*JobTemplate - RoleAliases map[string]*RoleAlias - DomainConfigs map[string]*DomainConfiguration - Authorizers map[string]*Authorizer - BillingGroups map[string]*BillingGroup - ProvTemplates map[string]*ProvisioningTemplate ProvTemplateVersions map[string][]*ProvisioningTemplateVersion } -// snapshotProvisioning deep-copies all job/provisioning state. Must be -// called with b.mu held (read or write). +// snapshotProvisioning deep-copies the raw provisioning-template state. Must +// be called with b.mu held (read or write). func (b *InMemoryBackend) snapshotProvisioning() provisioningSnapshot { - jobs := make(map[string]*Job, len(b.jobs)) - for k, v := range b.jobs { - jobs[k] = cloneJob(v) - } - - jobExecutions := make(map[string]*JobExecution, len(b.jobExecutions)) - for k, v := range b.jobExecutions { - jobExecutions[k] = cloneJobExecution(v) - } - - jobTemplates := make(map[string]*JobTemplate, len(b.jobTemplates)) - for k, v := range b.jobTemplates { - jobTemplates[k] = cloneJobTemplate(v) - } - - roleAliases := make(map[string]*RoleAlias, len(b.roleAliases)) - for k, v := range b.roleAliases { - roleAliases[k] = cloneRoleAlias(v) - } - - domainConfigs := make(map[string]*DomainConfiguration, len(b.domainConfigs)) - for k, v := range b.domainConfigs { - domainConfigs[k] = cloneDomainConfig(v) - } - - authorizers := make(map[string]*Authorizer, len(b.authorizers)) - for k, v := range b.authorizers { - authorizers[k] = cloneAuthorizer(v) - } - - billingGroups := make(map[string]*BillingGroup, len(b.billingGroups)) - for k, v := range b.billingGroups { - billingGroups[k] = cloneBillingGroup(v) - } - - provTemplates := make(map[string]*ProvisioningTemplate, len(b.provTemplates)) - for k, v := range b.provTemplates { - provTemplates[k] = cloneProvTemplate(v) - } - provTemplateVersions := make(map[string][]*ProvisioningTemplateVersion, len(b.provTemplateVersions)) for k, v := range b.provTemplateVersions { provTemplateVersions[k] = cloneProvTemplateVersions(v) } return provisioningSnapshot{ - Jobs: jobs, - JobExecutions: jobExecutions, - JobTemplates: jobTemplates, - RoleAliases: roleAliases, - DomainConfigs: domainConfigs, - Authorizers: authorizers, - BillingGroups: billingGroups, - ProvTemplates: provTemplates, ProvTemplateVersions: provTemplateVersions, } } -// restoreProvisioning restores job/provisioning state from a snapshot. Must -// be called with b.mu held (write). +// restoreProvisioning restores the raw provisioning-template state from a +// snapshot. Must be called with b.mu held (write). func (b *InMemoryBackend) restoreProvisioning(snap provisioningSnapshot) { - b.jobs = make(map[string]*Job, len(snap.Jobs)) - for k, v := range snap.Jobs { - b.jobs[k] = cloneJob(v) - } - - b.jobExecutions = make(map[string]*JobExecution, len(snap.JobExecutions)) - for k, v := range snap.JobExecutions { - b.jobExecutions[k] = cloneJobExecution(v) - } - - b.jobTemplates = make(map[string]*JobTemplate, len(snap.JobTemplates)) - for k, v := range snap.JobTemplates { - b.jobTemplates[k] = cloneJobTemplate(v) - } - - b.roleAliases = make(map[string]*RoleAlias, len(snap.RoleAliases)) - for k, v := range snap.RoleAliases { - b.roleAliases[k] = cloneRoleAlias(v) - } - - b.domainConfigs = make(map[string]*DomainConfiguration, len(snap.DomainConfigs)) - for k, v := range snap.DomainConfigs { - b.domainConfigs[k] = cloneDomainConfig(v) - } - - b.authorizers = make(map[string]*Authorizer, len(snap.Authorizers)) - for k, v := range snap.Authorizers { - b.authorizers[k] = cloneAuthorizer(v) - } - - b.billingGroups = make(map[string]*BillingGroup, len(snap.BillingGroups)) - for k, v := range snap.BillingGroups { - b.billingGroups[k] = cloneBillingGroup(v) - } - - b.provTemplates = make(map[string]*ProvisioningTemplate, len(snap.ProvTemplates)) - for k, v := range snap.ProvTemplates { - b.provTemplates[k] = cloneProvTemplate(v) - } - b.provTemplateVersions = make(map[string][]*ProvisioningTemplateVersion, len(snap.ProvTemplateVersions)) for k, v := range snap.ProvTemplateVersions { b.provTemplateVersions[k] = cloneProvTemplateVersions(v) } } -// ensureNonNilProvisioningSnap defaults nil maps in a restored snapshot's -// job/provisioning fields to empty maps. +// ensureNonNilProvisioningSnap defaults nil maps in a restored snapshot's raw +// provisioning-template field to an empty map. func ensureNonNilProvisioningSnap(snap *backendSnapshot) { - if snap.Jobs == nil { - snap.Jobs = make(map[string]*Job) - } - - if snap.JobExecutions == nil { - snap.JobExecutions = make(map[string]*JobExecution) - } - - if snap.JobTemplates == nil { - snap.JobTemplates = make(map[string]*JobTemplate) - } - - if snap.RoleAliases == nil { - snap.RoleAliases = make(map[string]*RoleAlias) - } - - if snap.DomainConfigs == nil { - snap.DomainConfigs = make(map[string]*DomainConfiguration) - } - - if snap.Authorizers == nil { - snap.Authorizers = make(map[string]*Authorizer) - } - - if snap.BillingGroups == nil { - snap.BillingGroups = make(map[string]*BillingGroup) - } - - if snap.ProvTemplates == nil { - snap.ProvTemplates = make(map[string]*ProvisioningTemplate) - } - if snap.ProvTemplateVersions == nil { snap.ProvTemplateVersions = make(map[string][]*ProvisioningTemplateVersion) } } // --------------------------------------------------------------------------- -// Group 3: Audit-related resources not already covered by -// deviceDefenderSnapshot. +// Group 3 (audit-extra) is gone: ScheduledAudit, MitigationAction, +// SecurityProfile, AuditSuppression, AuditFinding, AuditTask, and Dimension +// were its only fields and all moved to store.Table[T]s on b.registry (see +// store_setup.go), so nothing raw remains in this family. // --------------------------------------------------------------------------- -// auditExtraSnapshot bundles ScheduledAudit/MitigationAction/SecurityProfile/ -// AuditSuppression/AuditFinding/AuditTask/Dimension fields of -// backendSnapshot so Snapshot/Restore can delegate to a single helper each. -type auditExtraSnapshot struct { - ScheduledAudits map[string]*ScheduledAudit - MitigationActions map[string]*MitigationAction - SecurityProfiles map[string]*SecurityProfile - AuditSuppressions map[string]*AuditSuppression - AuditFindings map[string]*AuditFinding - AuditTaskObjects map[string]*AuditTask - Dimensions map[string]*Dimension -} - -// snapshotAuditExtra deep-copies the remaining audit-related state. Must be -// called with b.mu held (read or write). -func (b *InMemoryBackend) snapshotAuditExtra() auditExtraSnapshot { - scheduledAudits := make(map[string]*ScheduledAudit, len(b.scheduledAudits)) - for k, v := range b.scheduledAudits { - scheduledAudits[k] = cloneScheduledAudit(v) - } - - mitigationActions := make(map[string]*MitigationAction, len(b.mitigationActions)) - for k, v := range b.mitigationActions { - mitigationActions[k] = cloneMitigationAction(v) - } - - securityProfiles := make(map[string]*SecurityProfile, len(b.securityProfiles)) - for k, v := range b.securityProfiles { - securityProfiles[k] = cloneSecurityProfile(v) - } - - auditSuppressions := make(map[string]*AuditSuppression, len(b.auditSuppressions)) - for k, v := range b.auditSuppressions { - auditSuppressions[k] = cloneAuditSuppression(v) - } - - auditFindings := make(map[string]*AuditFinding, len(b.auditFindings)) - for k, v := range b.auditFindings { - auditFindings[k] = cloneAuditFinding(v) - } - - auditTaskObjects := make(map[string]*AuditTask, len(b.auditTaskObjects)) - for k, v := range b.auditTaskObjects { - auditTaskObjects[k] = cloneAuditTaskObj(v) - } - - dimensions := make(map[string]*Dimension, len(b.dimensions)) - for k, v := range b.dimensions { - dimensions[k] = cloneDimension(v) - } - - return auditExtraSnapshot{ - ScheduledAudits: scheduledAudits, - MitigationActions: mitigationActions, - SecurityProfiles: securityProfiles, - AuditSuppressions: auditSuppressions, - AuditFindings: auditFindings, - AuditTaskObjects: auditTaskObjects, - Dimensions: dimensions, - } -} - -// restoreAuditExtra restores the remaining audit-related state from a -// snapshot. Must be called with b.mu held (write). -func (b *InMemoryBackend) restoreAuditExtra(snap auditExtraSnapshot) { - b.scheduledAudits = make(map[string]*ScheduledAudit, len(snap.ScheduledAudits)) - for k, v := range snap.ScheduledAudits { - b.scheduledAudits[k] = cloneScheduledAudit(v) - } - - b.mitigationActions = make(map[string]*MitigationAction, len(snap.MitigationActions)) - for k, v := range snap.MitigationActions { - b.mitigationActions[k] = cloneMitigationAction(v) - } - - b.securityProfiles = make(map[string]*SecurityProfile, len(snap.SecurityProfiles)) - for k, v := range snap.SecurityProfiles { - b.securityProfiles[k] = cloneSecurityProfile(v) - } - - b.auditSuppressions = make(map[string]*AuditSuppression, len(snap.AuditSuppressions)) - for k, v := range snap.AuditSuppressions { - b.auditSuppressions[k] = cloneAuditSuppression(v) - } - - b.auditFindings = make(map[string]*AuditFinding, len(snap.AuditFindings)) - for k, v := range snap.AuditFindings { - b.auditFindings[k] = cloneAuditFinding(v) - } - - b.auditTaskObjects = make(map[string]*AuditTask, len(snap.AuditTaskObjects)) - for k, v := range snap.AuditTaskObjects { - b.auditTaskObjects[k] = cloneAuditTaskObj(v) - } - - b.dimensions = make(map[string]*Dimension, len(snap.Dimensions)) - for k, v := range snap.Dimensions { - b.dimensions[k] = cloneDimension(v) - } -} - -// ensureNonNilAuditExtraSnap defaults nil maps in a restored snapshot's -// remaining audit-related fields to empty maps. -func ensureNonNilAuditExtraSnap(snap *backendSnapshot) { - if snap.ScheduledAudits == nil { - snap.ScheduledAudits = make(map[string]*ScheduledAudit) - } - - if snap.MitigationActions == nil { - snap.MitigationActions = make(map[string]*MitigationAction) - } - - if snap.SecurityProfiles == nil { - snap.SecurityProfiles = make(map[string]*SecurityProfile) - } - - if snap.AuditSuppressions == nil { - snap.AuditSuppressions = make(map[string]*AuditSuppression) - } - - if snap.AuditFindings == nil { - snap.AuditFindings = make(map[string]*AuditFinding) - } - - if snap.AuditTaskObjects == nil { - snap.AuditTaskObjects = make(map[string]*AuditTask) - } - - if snap.Dimensions == nil { - snap.Dimensions = make(map[string]*Dimension) - } -} - // --------------------------------------------------------------------------- -// Group 4: Streams, packages, commands, and metrics. +// Group 4: Raw stream/package/command/metric state (Stream, OTAUpdate, +// IoTPackage, Command, FleetMetric, CustomMetric, and V2LoggingLevel are now +// store.Table[T]s; the nested PackageVersions2 map and the commandExecutions +// map -- whose key isn't recoverable from its value -- stay raw). // --------------------------------------------------------------------------- -// resourceMiscSnapshot bundles Stream/OTAUpdate/IoTPackage(Version)/Command -// (Execution)/FleetMetric/CustomMetric/V2LoggingLevel fields of -// backendSnapshot so Snapshot/Restore can delegate to a single helper each. +// resourceMiscSnapshot bundles the raw (non-Table) stream/package/command +// /metric fields of backendSnapshot. type resourceMiscSnapshot struct { - Streams map[string]*IoTStream - OTAUpdates map[string]*OTAUpdate - IoTPackages map[string]*IoTPackage PackageVersions2 map[string]map[string]*IoTPackageVersion - Commands map[string]*IoTCommand CommandExecutions map[string]*IoTCommandExecution - FleetMetrics map[string]*FleetMetric - CustomMetrics map[string]*CustomMetric - V2LoggingLevels map[string]*V2LoggingLevel } -// snapshotResourceMisc deep-copies stream/package/command/metric state. Must -// be called with b.mu held (read or write). +// snapshotResourceMisc deep-copies the raw stream/package/command/metric +// state. Must be called with b.mu held (read or write). func (b *InMemoryBackend) snapshotResourceMisc() resourceMiscSnapshot { - streams := make(map[string]*IoTStream, len(b.streams)) - for k, v := range b.streams { - streams[k] = cloneStream(v) - } - - otaUpdates := make(map[string]*OTAUpdate, len(b.otaUpdates)) - for k, v := range b.otaUpdates { - otaUpdates[k] = cloneOTAUpdate(v) - } - - iotPackages := make(map[string]*IoTPackage, len(b.iotPackages)) - for k, v := range b.iotPackages { - iotPackages[k] = cloneIoTPackage(v) - } - packageVersions2 := make(map[string]map[string]*IoTPackageVersion, len(b.packageVersions2)) for pkg, versions := range b.packageVersions2 { cp := make(map[string]*IoTPackageVersion, len(versions)) @@ -795,62 +328,20 @@ func (b *InMemoryBackend) snapshotResourceMisc() resourceMiscSnapshot { packageVersions2[pkg] = cp } - commands := make(map[string]*IoTCommand, len(b.commands)) - for k, v := range b.commands { - commands[k] = cloneIoTCommand(v) - } - commandExecutions := make(map[string]*IoTCommandExecution, len(b.commandExecutions)) for k, v := range b.commandExecutions { commandExecutions[k] = cloneIoTCommandExecution(v) } - fleetMetrics := make(map[string]*FleetMetric, len(b.fleetMetrics)) - for k, v := range b.fleetMetrics { - fleetMetrics[k] = cloneFleetMetric(v) - } - - customMetrics := make(map[string]*CustomMetric, len(b.customMetrics)) - for k, v := range b.customMetrics { - customMetrics[k] = cloneCustomMetric(v) - } - - v2LoggingLevels := make(map[string]*V2LoggingLevel, len(b.v2LoggingLevels)) - for k, v := range b.v2LoggingLevels { - v2LoggingLevels[k] = cloneV2LogLevel(v) - } - return resourceMiscSnapshot{ - Streams: streams, - OTAUpdates: otaUpdates, - IoTPackages: iotPackages, PackageVersions2: packageVersions2, - Commands: commands, CommandExecutions: commandExecutions, - FleetMetrics: fleetMetrics, - CustomMetrics: customMetrics, - V2LoggingLevels: v2LoggingLevels, } } -// restoreResourceMisc restores stream/package/command/metric state from a -// snapshot. Must be called with b.mu held (write). +// restoreResourceMisc restores the raw stream/package/command/metric state +// from a snapshot. Must be called with b.mu held (write). func (b *InMemoryBackend) restoreResourceMisc(snap resourceMiscSnapshot) { - b.streams = make(map[string]*IoTStream, len(snap.Streams)) - for k, v := range snap.Streams { - b.streams[k] = cloneStream(v) - } - - b.otaUpdates = make(map[string]*OTAUpdate, len(snap.OTAUpdates)) - for k, v := range snap.OTAUpdates { - b.otaUpdates[k] = cloneOTAUpdate(v) - } - - b.iotPackages = make(map[string]*IoTPackage, len(snap.IoTPackages)) - for k, v := range snap.IoTPackages { - b.iotPackages[k] = cloneIoTPackage(v) - } - b.packageVersions2 = make(map[string]map[string]*IoTPackageVersion, len(snap.PackageVersions2)) for pkg, versions := range snap.PackageVersions2 { cp := make(map[string]*IoTPackageVersion, len(versions)) @@ -860,70 +351,22 @@ func (b *InMemoryBackend) restoreResourceMisc(snap resourceMiscSnapshot) { b.packageVersions2[pkg] = cp } - b.commands = make(map[string]*IoTCommand, len(snap.Commands)) - for k, v := range snap.Commands { - b.commands[k] = cloneIoTCommand(v) - } - b.commandExecutions = make(map[string]*IoTCommandExecution, len(snap.CommandExecutions)) for k, v := range snap.CommandExecutions { b.commandExecutions[k] = cloneIoTCommandExecution(v) } - - b.fleetMetrics = make(map[string]*FleetMetric, len(snap.FleetMetrics)) - for k, v := range snap.FleetMetrics { - b.fleetMetrics[k] = cloneFleetMetric(v) - } - - b.customMetrics = make(map[string]*CustomMetric, len(snap.CustomMetrics)) - for k, v := range snap.CustomMetrics { - b.customMetrics[k] = cloneCustomMetric(v) - } - - b.v2LoggingLevels = make(map[string]*V2LoggingLevel, len(snap.V2LoggingLevels)) - for k, v := range snap.V2LoggingLevels { - b.v2LoggingLevels[k] = cloneV2LogLevel(v) - } } -// ensureNonNilResourceMiscSnap defaults nil maps in a restored snapshot's +// ensureNonNilResourceMiscSnap defaults nil maps in a restored snapshot's raw // stream/package/command/metric fields to empty maps. func ensureNonNilResourceMiscSnap(snap *backendSnapshot) { - if snap.Streams == nil { - snap.Streams = make(map[string]*IoTStream) - } - - if snap.OTAUpdates == nil { - snap.OTAUpdates = make(map[string]*OTAUpdate) - } - - if snap.IoTPackages == nil { - snap.IoTPackages = make(map[string]*IoTPackage) - } - if snap.PackageVersions2 == nil { snap.PackageVersions2 = make(map[string]map[string]*IoTPackageVersion) } - if snap.Commands == nil { - snap.Commands = make(map[string]*IoTCommand) - } - if snap.CommandExecutions == nil { snap.CommandExecutions = make(map[string]*IoTCommandExecution) } - - if snap.FleetMetrics == nil { - snap.FleetMetrics = make(map[string]*FleetMetric) - } - - if snap.CustomMetrics == nil { - snap.CustomMetrics = make(map[string]*CustomMetric) - } - - if snap.V2LoggingLevels == nil { - snap.V2LoggingLevels = make(map[string]*V2LoggingLevel) - } } // --------------------------------------------------------------------------- diff --git a/services/iot/export_test.go b/services/iot/export_test.go index f7d83730f..73fcf9232 100644 --- a/services/iot/export_test.go +++ b/services/iot/export_test.go @@ -6,7 +6,7 @@ func (b *InMemoryBackend) ThingCount() int { b.mu.RLock() defer b.mu.RUnlock() - return len(b.things) + return b.things.Len() } // PolicyCount returns the number of Policies in the backend. @@ -15,7 +15,7 @@ func (b *InMemoryBackend) PolicyCount() int { b.mu.RLock() defer b.mu.RUnlock() - return len(b.policies) + return b.policies.Len() } // RuleCount returns the number of TopicRules in the backend. @@ -24,7 +24,7 @@ func (b *InMemoryBackend) RuleCount() int { b.mu.RLock() defer b.mu.RUnlock() - return len(b.rules) + return b.rules.Len() } // CertTransferCount returns the number of certificate transfer records. @@ -68,7 +68,7 @@ func (b *InMemoryBackend) TopicRuleDestConfirmationToken(arn string) string { b.mu.RLock() defer b.mu.RUnlock() - dest, ok := b.topicRuleDestinations[arn] + dest, ok := b.topicRuleDestinations.Get(arn) if !ok { return "" } diff --git a/services/iot/persistence.go b/services/iot/persistence.go index 611ed4543..f09b836a9 100644 --- a/services/iot/persistence.go +++ b/services/iot/persistence.go @@ -3,16 +3,27 @@ package iot import ( "context" "encoding/json" + "fmt" "maps" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" ) +// iotSnapshotVersion identifies the shape of backendSnapshot's Tables blob +// (i.e. the set/shape of resources registered on b.registry -- see +// registerAllTables in store_setup.go -- plus the one "dirty" DTO table, +// topicRuleDestinations). It must be bumped whenever a change there would +// make an older snapshot unsafe to decode as the current shape. Restore +// compares this against the persisted value and discards (rather than +// attempts to partially decode) any mismatch -- see Restore below. This +// mirrors the services/ec2 (12e611a4) and services/sqs (0f09d77c) pilots. +const iotSnapshotVersion = 1 + type backendSnapshot struct { + Tables map[string]json.RawMessage `json:"tables"` AuditTasks map[string]string `json:"auditTasks"` MetricValues map[string][]*MetricDatapoint `json:"metricValues"` - Rules map[string]*TopicRule `json:"rules"` CertificateTransfers map[string]string `json:"certificateTransfers"` ThingBillingGroups map[string]string `json:"thingBillingGroups"` ThingThingGroups map[string][]string `json:"thingThingGroups"` @@ -22,54 +33,20 @@ type backendSnapshot struct { SecurityProfileTargets map[string][]string `json:"securityProfileTargets"` ThingPrincipals map[string][]string `json:"thingPrincipals"` ThingGroupIndexingConfiguration *ThingGroupIndexingConfiguration `json:"thingGroupIndexingConfiguration"` - Policies map[string]*Policy `json:"policies"` - Things map[string]*Thing `json:"things"` AuditMitigationTasks map[string]string `json:"auditMitigationTasks"` - RegistrationTasks map[string]*ThingRegistrationTask `json:"registrationTasks"` - AuditMitigationTaskObjects map[string]*AuditMitigationTask `json:"auditMitigationTaskObjects"` AuditMitigationExecutions map[string][]*AuditMitigationActionExecution `json:"auditMitigationExecutions"` - DetectMitigationTasks map[string]*DetectMitigationTask `json:"detectMitigationTasks"` DetectMitigationExecutions map[string][]*DetectMitigationActionExecution `json:"detectMitigationExecutions"` - ActiveViolations map[string]*ActiveViolation `json:"activeViolations"` BehaviorTrainingSummaries map[string][]*BehaviorModelTrainingSummary `json:"behaviorTrainingSummaries"` AccountEncryptionConfig *AccountEncryptionConfiguration `json:"accountEncryptionConfig"` SbomValidationResults map[string][]*SbomValidationResult `json:"sbomValidationResults"` ThingIndexingConfiguration *ThingIndexingConfiguration `json:"thingIndexingConfiguration"` ThingConnectivity map[string]*ThingConnectivityData `json:"thingConnectivity"` - ThingTypes map[string]*ThingType `json:"thingTypes"` - ThingGroups map[string]*ThingGroup `json:"thingGroups"` ThingGroupMembers map[string][]string `json:"thingGroupMembers"` - Certificates map[string]*Certificate `json:"certificates"` - CertificateProviders map[string]*CertificateProvider `json:"certificateProviders"` - CACertificates map[string]*CACertificate `json:"caCertificates"` PolicyVersions map[string][]*PolicyVersion `json:"policyVersions"` - TopicRuleDestinations map[string]*topicRuleDestSnap `json:"topicRuleDestinations"` ResourceTags map[string]map[string]string `json:"resourceTags"` - Jobs map[string]*Job `json:"jobs"` - JobExecutions map[string]*JobExecution `json:"jobExecutions"` - JobTemplates map[string]*JobTemplate `json:"jobTemplates"` - RoleAliases map[string]*RoleAlias `json:"roleAliases"` - DomainConfigs map[string]*DomainConfiguration `json:"domainConfigs"` - Authorizers map[string]*Authorizer `json:"authorizers"` - BillingGroups map[string]*BillingGroup `json:"billingGroups"` - ProvTemplates map[string]*ProvisioningTemplate `json:"provTemplates"` ProvTemplateVersions map[string][]*ProvisioningTemplateVersion `json:"provTemplateVersions"` - ScheduledAudits map[string]*ScheduledAudit `json:"scheduledAudits"` - MitigationActions map[string]*MitigationAction `json:"mitigationActions"` - SecurityProfiles map[string]*SecurityProfile `json:"securityProfiles"` - AuditSuppressions map[string]*AuditSuppression `json:"auditSuppressions"` - AuditFindings map[string]*AuditFinding `json:"auditFindings"` - AuditTaskObjects map[string]*AuditTask `json:"auditTaskObjects"` - Dimensions map[string]*Dimension `json:"dimensions"` - Streams map[string]*IoTStream `json:"streams"` - OTAUpdates map[string]*OTAUpdate `json:"otaUpdates"` - IoTPackages map[string]*IoTPackage `json:"iotPackages"` PackageVersions2 map[string]map[string]*IoTPackageVersion `json:"packageVersions2"` - Commands map[string]*IoTCommand `json:"commands"` CommandExecutions map[string]*IoTCommandExecution `json:"commandExecutions"` - FleetMetrics map[string]*FleetMetric `json:"fleetMetrics"` - CustomMetrics map[string]*CustomMetric `json:"customMetrics"` - V2LoggingLevels map[string]*V2LoggingLevel `json:"v2LoggingLevels"` AuditConfiguration *AccountAuditConfiguration `json:"auditConfiguration"` PackageConfig *PackageConfiguration `json:"packageConfig"` V2LoggingOptions *V2LoggingOptions `json:"v2LoggingOptions"` @@ -78,36 +55,45 @@ type backendSnapshot struct { RegistrationCode string `json:"registrationCode"` DefaultAuthorizer string `json:"defaultAuthorizer"` ViolationEvents []*ViolationEvent `json:"violationEvents"` + Version int `json:"version"` } +// topicRuleDestinationsTableName is the Tables blob key used for the +// topicRuleDestinations DTO entry, both in the ephemeral DTO registry built +// by Snapshot/Restore below and as the map key inside backendSnapshot.Tables. +const topicRuleDestinationsTableName = "topicRuleDestinations" + // Snapshot serialises the backend state to JSON. func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock() defer b.mu.RUnlock() - things := make(map[string]*Thing, len(b.things)) - for k, v := range b.things { - cp := cloneThing(v) - things[k] = cp - } + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "iot: snapshot table marshal failed", "error", err) - policies := make(map[string]*Policy, len(b.policies)) - for k, v := range b.policies { - policies[k] = clonePolicy(v) + return nil } - rules := make(map[string]*TopicRule, len(b.rules)) - for k, v := range b.rules { - rules[k] = cloneTopicRule(v) - } + destTables, err := b.snapshotTopicRuleDestinationsTable() + if err != nil { + logger.Load(ctx).WarnContext(ctx, "iot: snapshot table marshal failed", "error", err) - certTransfers := make(map[string]string, len(b.certificateTransfers)) - maps.Copy(certTransfers, b.certificateTransfers) + return nil + } - billingGroups := make(map[string]string, len(b.thingBillingGroups)) - maps.Copy(billingGroups, b.thingBillingGroups) + maps.Copy(tables, destTables) - thingGroups := copyStringSliceMap(b.thingThingGroups) + ddSnap := b.snapshotDeviceDefender() + finalSnap := b.snapshotFinalOps() + thingResSnap := b.snapshotThingResources() + provSnap := b.snapshotProvisioning() + miscSnap := b.snapshotResourceMisc() + cfgSnap := b.snapshotConfig() sboms := make(map[string]*SbomDocument, len(b.packageVersionSboms)) for k, v := range b.packageVersionSboms { @@ -124,26 +110,12 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { thingGroupIndexingConfig = cloneThingGroupIndexingConfiguration(b.thingGroupIndexingConfig) } - registrationTasks := make(map[string]*ThingRegistrationTask, len(b.registrationTasks)) - for k, v := range b.registrationTasks { - registrationTasks[k] = cloneRegistrationTask(v) - } - - ddSnap := b.snapshotDeviceDefender() - finalSnap := b.snapshotFinalOps() - thingResSnap := b.snapshotThingResources() - provSnap := b.snapshotProvisioning() - auditExtraSnap := b.snapshotAuditExtra() - miscSnap := b.snapshotResourceMisc() - cfgSnap := b.snapshotConfig() - snap := backendSnapshot{ - Things: things, - Policies: policies, - Rules: rules, - CertificateTransfers: certTransfers, - ThingBillingGroups: billingGroups, - ThingThingGroups: thingGroups, + Version: iotSnapshotVersion, + Tables: tables, + CertificateTransfers: copyStringMap(b.certificateTransfers), + ThingBillingGroups: copyStringMap(b.thingBillingGroups), + ThingThingGroups: copyStringSliceMap(b.thingThingGroups), PackageVersionSboms: sboms, JobTargets: copyStringSliceMap(b.jobTargets), PolicyTargets: copyStringSliceMap(b.policyTargets), @@ -155,13 +127,8 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { ThingIndexingConfiguration: thingIndexingConfig, ThingGroupIndexingConfiguration: thingGroupIndexingConfig, - RegistrationTasks: registrationTasks, - - AuditMitigationTaskObjects: ddSnap.AuditMitigationTaskObjects, AuditMitigationExecutions: ddSnap.AuditMitigationExecutions, - DetectMitigationTasks: ddSnap.DetectMitigationTasks, DetectMitigationExecutions: ddSnap.DetectMitigationExecutions, - ActiveViolations: ddSnap.ActiveViolations, ViolationEvents: ddSnap.ViolationEvents, AccountEncryptionConfig: finalSnap.AccountEncryptionConfig, @@ -169,9 +136,24 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { MetricValues: finalSnap.MetricValues, ThingConnectivity: finalSnap.ThingConnectivity, BehaviorTrainingSummaries: finalSnap.BehaviorTrainingSummaries, - } - applyExtSnapshot(&snap, thingResSnap, provSnap, auditExtraSnap, miscSnap, cfgSnap) + ThingGroupMembers: thingResSnap.ThingGroupMembers, + PolicyVersions: thingResSnap.PolicyVersions, + ResourceTags: thingResSnap.ResourceTags, + + ProvTemplateVersions: provSnap.ProvTemplateVersions, + + PackageVersions2: miscSnap.PackageVersions2, + CommandExecutions: miscSnap.CommandExecutions, + + AuditConfiguration: cfgSnap.AuditConfiguration, + PackageConfig: cfgSnap.PackageConfig, + V2LoggingOptions: cfgSnap.V2LoggingOptions, + LoggingOptions: cfgSnap.LoggingOptions, + EventConfigurations: cfgSnap.EventConfigurations, + RegistrationCode: cfgSnap.RegistrationCode, + DefaultAuthorizer: cfgSnap.DefaultAuthorizer, + } data, err := json.Marshal(snap) if err != nil { @@ -195,25 +177,33 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock() defer b.mu.Unlock() - things := make(map[string]*Thing, len(snap.Things)) - for k, v := range snap.Things { - cp := cloneThing(v) - things[k] = cp + if snap.Version != iotSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/ec2/sqs pilots. + logger.Load(ctx).WarnContext(ctx, + "iot: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", iotSnapshotVersion) + + b.registry.ResetAll() + + return nil } - policies := make(map[string]*Policy, len(snap.Policies)) - for k, v := range snap.Policies { - policies[k] = clonePolicy(v) + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("iot: restore snapshot tables: %w", err) } - rules := make(map[string]*TopicRule, len(snap.Rules)) - for k, v := range snap.Rules { - rules[k] = cloneTopicRule(v) + // Re-derive topicRuleDestinations from its DTO entry so ConfirmationToken + // (dropped by the generic per-table decode above, since it is tagged + // json:"-" on the live type) round-trips. See Snapshot's comment. + if err := b.restoreTopicRuleDestinationsTable(snap.Tables); err != nil { + return err } - b.things = things - b.policies = policies - b.rules = rules b.certificateTransfers = copyStringMap(snap.CertificateTransfers) b.thingBillingGroups = copyStringMap(snap.ThingBillingGroups) b.thingThingGroups = copyStringSliceMap(snap.ThingThingGroups) @@ -242,17 +232,9 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.thingGroupIndexingConfig = nil } - b.registrationTasks = make(map[string]*ThingRegistrationTask, len(snap.RegistrationTasks)) - for k, v := range snap.RegistrationTasks { - b.registrationTasks[k] = cloneRegistrationTask(v) - } - b.restoreDeviceDefender(deviceDefenderSnapshot{ - AuditMitigationTaskObjects: snap.AuditMitigationTaskObjects, AuditMitigationExecutions: snap.AuditMitigationExecutions, - DetectMitigationTasks: snap.DetectMitigationTasks, DetectMitigationExecutions: snap.DetectMitigationExecutions, - ActiveViolations: snap.ActiveViolations, ViolationEvents: snap.ViolationEvents, }) @@ -264,29 +246,30 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { BehaviorTrainingSummaries: snap.BehaviorTrainingSummaries, }) - thingRes, prov, auditExtra, misc, cfg := extGroupsFromSnapshot(&snap) - b.restoreThingResources(thingRes) - b.restoreProvisioning(prov) - b.restoreAuditExtra(auditExtra) - b.restoreResourceMisc(misc) - b.restoreConfig(cfg) + b.restoreThingResources(thingResourceSnapshot{ + ThingGroupMembers: snap.ThingGroupMembers, + PolicyVersions: snap.PolicyVersions, + ResourceTags: snap.ResourceTags, + }) + b.restoreProvisioning(provisioningSnapshot{ProvTemplateVersions: snap.ProvTemplateVersions}) + b.restoreResourceMisc(resourceMiscSnapshot{ + PackageVersions2: snap.PackageVersions2, + CommandExecutions: snap.CommandExecutions, + }) + b.restoreConfig(configSnapshot{ + AuditConfiguration: snap.AuditConfiguration, + PackageConfig: snap.PackageConfig, + V2LoggingOptions: snap.V2LoggingOptions, + LoggingOptions: snap.LoggingOptions, + EventConfigurations: snap.EventConfigurations, + RegistrationCode: snap.RegistrationCode, + DefaultAuthorizer: snap.DefaultAuthorizer, + }) return nil } func ensureNonNilSnap(snap *backendSnapshot) { - if snap.Things == nil { - snap.Things = make(map[string]*Thing) - } - - if snap.Policies == nil { - snap.Policies = make(map[string]*Policy) - } - - if snap.Rules == nil { - snap.Rules = make(map[string]*TopicRule) - } - if snap.CertificateTransfers == nil { snap.CertificateTransfers = make(map[string]string) } @@ -327,14 +310,9 @@ func ensureNonNilSnap(snap *backendSnapshot) { snap.AuditTasks = make(map[string]string) } - if snap.RegistrationTasks == nil { - snap.RegistrationTasks = make(map[string]*ThingRegistrationTask) - } - ensureNonNilFinalOpsSnap(snap) ensureNonNilThingResourceSnap(snap) ensureNonNilProvisioningSnap(snap) - ensureNonNilAuditExtraSnap(snap) ensureNonNilResourceMiscSnap(snap) } diff --git a/services/iot/store_setup.go b/services/iot/store_setup.go new file mode 100644 index 000000000..3f0b29f2b --- /dev/null +++ b/services/iot/store_setup.go @@ -0,0 +1,210 @@ +package iot + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend whose key is a pure +// function of the value's own fields is registered exactly once, here, as a +// *store.Table[T] on b.registry. See pkgs/store's package doc and the +// services/ec2 (commit 12e611a4) / services/sqs (commit 0f09d77c) pilots for +// the pattern this follows. +// +// A number of fields are deliberately NOT registered here and remain plain +// maps -- see the comment above registerAllTables for the full list and why. +import "github.com/blackbirdworks/gopherstack/pkgs/store" + +func thingsKeyFn(v *Thing) string { return v.ThingName } +func thingTypesKeyFn(v *ThingType) string { return v.ThingTypeName } +func thingGroupsKeyFn(v *ThingGroup) string { return v.ThingGroupName } +func certificatesKeyFn(v *Certificate) string { return v.CertificateID } +func policiesKeyFn(v *Policy) string { return v.PolicyName } +func rulesKeyFn(v *TopicRule) string { return v.RuleName } +func jobsKeyFn(v *Job) string { return v.JobID } +func jobExecutionsKeyFn(v *JobExecution) string { return jobExecKey(v.JobID, v.ThingName) } +func jobTemplatesKeyFn(v *JobTemplate) string { return v.JobTemplateID } +func billingGroupsKeyFn(v *BillingGroup) string { return v.BillingGroupName } +func topicRuleDestinationsKeyFn(v *TopicRuleDestination) string { return v.ARN } +func certificateProvidersKeyFn(v *CertificateProvider) string { return v.CertificateProviderName } +func roleAliasesKeyFn(v *RoleAlias) string { return v.RoleAlias } +func domainConfigsKeyFn(v *DomainConfiguration) string { return v.DomainConfigurationName } +func dimensionsKeyFn(v *Dimension) string { return v.Name } +func authorizersKeyFn(v *Authorizer) string { return v.AuthorizerName } +func scheduledAuditsKeyFn(v *ScheduledAudit) string { return v.ScheduledAuditName } +func mitigationActionsKeyFn(v *MitigationAction) string { return v.ActionName } +func securityProfilesKeyFn(v *SecurityProfile) string { return v.SecurityProfileName } +func caCertificatesKeyFn(v *CACertificate) string { return v.CertificateID } +func streamsKeyFn(v *IoTStream) string { return v.StreamID } +func provTemplatesKeyFn(v *ProvisioningTemplate) string { return v.TemplateName } +func auditTaskObjectsKeyFn(v *AuditTask) string { return v.TaskID } +func otaUpdatesKeyFn(v *OTAUpdate) string { return v.OTAUpdateID } +func iotPackagesKeyFn(v *IoTPackage) string { return v.PackageName } +func auditSuppressionsKeyFn(v *AuditSuppression) string { + return auditSuppressionKey(v.CheckName, v.ResourceIdentifier) +} +func auditFindingsKeyFn(v *AuditFinding) string { return v.FindingID } +func v2LoggingLevelsKeyFn(v *V2LoggingLevel) string { return v2LogLevelKey(v.Target) } +func commandsKeyFn(v *IoTCommand) string { return v.CommandID } +func registrationTasksKeyFn(v *ThingRegistrationTask) string { return v.TaskID } +func auditMitigationTaskObjectsKeyFn(v *AuditMitigationTask) string { return v.TaskID } +func detectMitigationTasksKeyFn(v *DetectMitigationTask) string { return v.TaskID } +func activeViolationsKeyFn(v *ActiveViolation) string { return v.ViolationID } +func fleetMetricsKeyFn(v *FleetMetric) string { return v.MetricName } +func customMetricsKeyFn(v *CustomMetric) string { return v.MetricName } + +// registerAllTables registers every converted resource map on b.registry +// exactly once. It must be called during construction only (immediately +// after b.registry is created), never on every Reset() -- store.Register +// panics on a duplicate name, so runtime resets go through +// registry.ResetAll() instead (see InMemoryBackend.Reset in backend.go). +// +// The following resource fields are deliberately left as plain maps (not +// registered here) because they don't fit store.Table's "key is a pure +// function of the value" model: +// - shadows: keyed by the composite struct shadowKey{thingName, +// shadowName}, and ThingShadow itself carries neither field, so there is +// no pure keyFn without changing ThingShadow's shape. Reset() also has a +// pre-existing quirk where it never clears b.shadows (unlike every other +// map); folding shadows into the shared registry's ResetAll() would +// silently fix that quirk, which the mechanical-swap/no-quirk-fixing rule +// forbids. Left exactly as-is: a raw map[shadowKey]*ThingShadow untouched +// by Reset(), matching current behavior byte-for-byte. +// - packageVersionSboms: value type SbomDocument carries no +// package/version identity fields of its own (only S3Location); the +// packageVersionKey(packageName, versionName) composite is not +// recoverable from the stored value. +// - commandExecutions: value type IoTCommandExecution carries CommandARN +// but not the raw commandID used in the key (commandID+"/"+executionID); +// the ARN does not losslessly round-trip back to commandID without +// parsing, so the key is not a pure function of the value's own fields. +// - thingConnectivity: value type ThingConnectivityData carries no +// ThingName/identity field of its own; it is keyed purely externally. +// - resourceTags, certificateTransfers, thingBillingGroups, +// thingThingGroups, thingGroupMembers, jobTargets, policyTargets, +// securityProfileTargets, thingPrincipals, auditMitigationTasks, +// auditTasks: value type is not map[string]*T (string, []string, or +// map[string]string values), so store.Table (which stores *V pointers) +// does not apply. +// - policyVersions, provTemplateVersions, auditMitigationExecutions, +// detectMitigationExecutions, sbomValidationResults, metricValues, +// behaviorTrainingSummaries: slice-valued (map[string][]*T) -- store.Table +// holds one *V per key, not a growable list per key. +// - packageVersions2: nested map[string]map[string]*IoTPackageVersion; the +// value type at the outer key is itself a map, not *T. +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call to the concrete field and value type. A closure list (rather than one +// statement per field in a single function body) keeps registerAllTables +// small regardless of how many resource tables the backend grows to. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { b.things = store.Register(b.registry, "things", store.New(thingsKeyFn)) }, + func(b *InMemoryBackend) { + b.thingTypes = store.Register(b.registry, "thingTypes", store.New(thingTypesKeyFn)) + }, + func(b *InMemoryBackend) { + b.thingGroups = store.Register(b.registry, "thingGroups", store.New(thingGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.certificates = store.Register(b.registry, "certificates", store.New(certificatesKeyFn)) + }, + func(b *InMemoryBackend) { + b.policies = store.Register(b.registry, "policies", store.New(policiesKeyFn)) + }, + func(b *InMemoryBackend) { b.rules = store.Register(b.registry, "rules", store.New(rulesKeyFn)) }, + func(b *InMemoryBackend) { b.jobs = store.Register(b.registry, "jobs", store.New(jobsKeyFn)) }, + func(b *InMemoryBackend) { + b.jobExecutions = store.Register(b.registry, "jobExecutions", store.New(jobExecutionsKeyFn)) + }, + func(b *InMemoryBackend) { + b.jobTemplates = store.Register(b.registry, "jobTemplates", store.New(jobTemplatesKeyFn)) + }, + func(b *InMemoryBackend) { + b.billingGroups = store.Register(b.registry, "billingGroups", store.New(billingGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.topicRuleDestinations = store.Register( + b.registry, "topicRuleDestinations", store.New(topicRuleDestinationsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.certificateProviders = store.Register( + b.registry, "certificateProviders", store.New(certificateProvidersKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.roleAliases = store.Register(b.registry, "roleAliases", store.New(roleAliasesKeyFn)) + }, + func(b *InMemoryBackend) { + b.domainConfigs = store.Register(b.registry, "domainConfigs", store.New(domainConfigsKeyFn)) + }, + func(b *InMemoryBackend) { + b.dimensions = store.Register(b.registry, "dimensions", store.New(dimensionsKeyFn)) + }, + func(b *InMemoryBackend) { + b.authorizers = store.Register(b.registry, "authorizers", store.New(authorizersKeyFn)) + }, + func(b *InMemoryBackend) { + b.scheduledAudits = store.Register(b.registry, "scheduledAudits", store.New(scheduledAuditsKeyFn)) + }, + func(b *InMemoryBackend) { + b.mitigationActions = store.Register(b.registry, "mitigationActions", store.New(mitigationActionsKeyFn)) + }, + func(b *InMemoryBackend) { + b.securityProfiles = store.Register(b.registry, "securityProfiles", store.New(securityProfilesKeyFn)) + }, + func(b *InMemoryBackend) { + b.caCertificates = store.Register(b.registry, "caCertificates", store.New(caCertificatesKeyFn)) + }, + func(b *InMemoryBackend) { b.streams = store.Register(b.registry, "streams", store.New(streamsKeyFn)) }, + func(b *InMemoryBackend) { + b.provTemplates = store.Register(b.registry, "provTemplates", store.New(provTemplatesKeyFn)) + }, + func(b *InMemoryBackend) { + b.auditTaskObjects = store.Register(b.registry, "auditTaskObjects", store.New(auditTaskObjectsKeyFn)) + }, + func(b *InMemoryBackend) { + b.otaUpdates = store.Register(b.registry, "otaUpdates", store.New(otaUpdatesKeyFn)) + }, + func(b *InMemoryBackend) { + b.iotPackages = store.Register(b.registry, "iotPackages", store.New(iotPackagesKeyFn)) + }, + func(b *InMemoryBackend) { + b.auditSuppressions = store.Register(b.registry, "auditSuppressions", store.New(auditSuppressionsKeyFn)) + }, + func(b *InMemoryBackend) { + b.auditFindings = store.Register(b.registry, "auditFindings", store.New(auditFindingsKeyFn)) + }, + func(b *InMemoryBackend) { + b.v2LoggingLevels = store.Register(b.registry, "v2LoggingLevels", store.New(v2LoggingLevelsKeyFn)) + }, + func(b *InMemoryBackend) { + b.commands = store.Register(b.registry, "commands", store.New(commandsKeyFn)) + }, + func(b *InMemoryBackend) { + b.registrationTasks = store.Register(b.registry, "registrationTasks", store.New(registrationTasksKeyFn)) + }, + func(b *InMemoryBackend) { + b.auditMitigationTaskObjects = store.Register( + b.registry, "auditMitigationTaskObjects", store.New(auditMitigationTaskObjectsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.detectMitigationTasks = store.Register( + b.registry, "detectMitigationTasks", store.New(detectMitigationTasksKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.activeViolations = store.Register(b.registry, "activeViolations", store.New(activeViolationsKeyFn)) + }, + func(b *InMemoryBackend) { + b.fleetMetrics = store.Register(b.registry, "fleetMetrics", store.New(fleetMetricsKeyFn)) + }, + func(b *InMemoryBackend) { + b.customMetrics = store.Register(b.registry, "customMetrics", store.New(customMetricsKeyFn)) + }, +} diff --git a/services/kinesis/PARITY.md b/services/kinesis/PARITY.md new file mode 100644 index 000000000..00fcacaa6 --- /dev/null +++ b/services/kinesis/PARITY.md @@ -0,0 +1,172 @@ +--- +service: kinesis +sdk_module: aws-sdk-go-v2/service/kinesis@v1.43.2 +last_audit_commit: f222f376 +last_audit_date: 2026-07-05 +overall: A # ~750 LOC genuine fixes across backend.go/handler.go/persistence.go + new/updated tests +ops: + CreateStream: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: ON_DEMAND now defaults to 4 shards (was 1); inline Tags now validated pre-mutation and persisted via TagResource instead of a lost handler-local map"} + DeleteStream: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeStream: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: Shards list now paginates (Limit/ExclusiveStartShardId/HasMoreShards); previously returned every shard in one page with HasMoreShards hardcoded false"} + DescribeStreamSummary: {wire: ok, errors: ok, state: ok, persist: ok} + ListStreams: {wire: ok, errors: ok, state: ok, persist: ok} + PutRecord: {wire: ok, errors: ok, state: ok, persist: ok, note: "MD5 hash routing, explicit hash key, per-shard monotonic sequence numbers verified correct"} + PutRecords: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: empty Records list now rejected (was silently 200); stream-not-found now fails the whole call with top-level ResourceNotFoundException instead of InternalFailure on every result entry"} + GetShardIterator: {wire: ok, errors: ok, state: ok, persist: n/a, note: "TRIM_HORIZON/LATEST/AT_(AFTER_)SEQUENCE_NUMBER/AT_TIMESTAMP all verified; iterator token carries region so cross-region record stores stay isolated"} + GetRecords: {wire: ok, errors: ok, state: ok, persist: n/a, note: "10k-record / 10MiB caps, NextShardIterator empty-on-closed-and-drained, MillisBehindLatest verified"} + ListShards: {wire: ok, errors: ok, state: ok, persist: n/a} + RegisterStreamConsumer: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: added missing 20-consumers-per-stream limit (LimitExceededException)"} + DescribeStreamConsumer: {wire: ok, errors: ok, state: ok, persist: ok} + ListStreamConsumers: {wire: ok, errors: ok, state: ok, persist: ok} + DeregisterStreamConsumer: {wire: ok, errors: ok, state: ok, persist: ok} + SubscribeToShard: {wire: ok, errors: ok, state: ok, persist: n/a, note: "event-stream binary framing verified byte-for-byte (prelude/CRC/headers); polling goroutine bounded by idle-poll count and 5-min deadline, no leak"} + UpdateShardCount: {wire: ok, errors: ok, state: ok, persist: ok, note: "double/half scaling window, parent/adjacent-parent lineage, old shards kept CLOSED verified"} + EnableEnhancedMonitoring: {wire: ok, errors: ok, state: ok, persist: ok} + DisableEnhancedMonitoring: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeLimits: {wire: ok, errors: ok, state: ok, persist: n/a} + DescribeAccountSettings: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: OnDemandStreamCountLimit set via UpdateAccountSettings was never in backendSnapshot, silently reset to default on every restart"} + UpdateAccountSettings: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateMaxRecordSize: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateStreamWarmThroughput: {wire: ok, errors: ok, state: ok, persist: n/a, note: "intentional no-op (no throughput model to warm); existence-checked"} + MergeShards: {wire: ok, errors: ok, state: ok, persist: ok, note: "adjacency check (either shard may be passed first), closed-parent lineage verified"} + SplitShard: {wire: ok, errors: ok, state: ok, persist: ok, note: "NewStartingHashKey must be strictly inside parent range, verified"} + StartStreamEncryption: {wire: ok, errors: ok, state: ok, persist: ok} + StopStreamEncryption: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteResourcePolicy: {wire: ok, errors: ok, state: ok, persist: n/a, note: "resource policies not yet in backendSnapshot; see gaps"} + GetResourcePolicy: {wire: ok, errors: ok, state: ok, persist: n/a} + PutResourcePolicy: {wire: ok, errors: ok, state: ok, persist: n/a} + ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: now reads backend stream.Tags via Backend.ListTagsForResource instead of a handler-local map that was previously the ONLY store for tags applied via CreateStream/AddTagsToStream/RemoveTagsFromStream"} + AddTagsToStream: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: now writes through Backend.TagResource (stream.Tags) instead of a handler-local map dropped on Snapshot"} + RemoveTagsFromStream: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: now writes through Backend.UntagResource"} + ListTagsForStream: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: now reads Backend.ListTagsForResource"} + TagResource: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: now enforces the 50-tag cap consistently with AddTagsToStream (previously uncapped)"} + UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateStreamMode: {wire: ok, errors: ok, state: ok, persist: ok, note: "does not reshard on PROVISIONED->ON_DEMAND transition; see gaps"} +families: + hash_key_routing: {status: ok, note: "MD5-based partition-key routing and explicit-hash-key routing verified against big.Int range math; shardForHashKey fallback-to-first-open-shard behavior documented"} + sequence_numbers: {status: ok, note: "per-shard monotonic NextSeq counter, 49-prefixed AWS-shaped sequence string, persisted via Shard.NextSeq"} + reshard_lineage: {status: ok, note: "SplitShard/MergeShards/UpdateShardCount all set ParentShardID/AdjacentParentShardID correctly; closed shards retained forever for DescribeStream/ListShards lineage (see leaks note)"} + error_codes: {status: ok, note: "ResourceNotFoundException/ResourceInUseException/InvalidArgumentException/ProvisionedThroughputExceededException/ExpiredIteratorException/LimitExceededException/UnknownOperationException all verified exact string + 400 status. KMSAccessDenied not modeled — see gaps."} +gaps: + - "KMSAccessDeniedException / KMS key existence validation for StartStreamEncryption/UpdateMaxRecordSize KeyId: not modeled. Real Kinesis StartStreamEncryption exceptions per the SDK model are InvalidArgumentException/LimitExceededException/ResourceInUseException/ResourceNotFoundException/AccessDeniedException — there is no KMS-specific exception in the Kinesis API itself, and validating a KeyId against the kms service backend would require a cross-service dependency out of scope for this pass (services/kinesis/ only). (bd: gopherstack-ud2)" + - "UpdateStreamMode does not reshard when switching PROVISIONED -> ON_DEMAND (or back); AWS auto-adjusts shard count on mode transitions based on throughput history, which this in-memory emulator has no model for. Low priority: consumers of UpdateStreamMode generally re-describe the stream afterward. (bd: gopherstack-ud2)" + - "GetShardIterator/SubscribeToShard AT_TIMESTAMP with a zero/omitted Timestamp is not rejected with ValidationException (silently treated as position 0). Minor; no test exercises this AWS edge case. (bd: gopherstack-ud2)" + - "ListShards ShardFilter AT_TIMESTAMP/FROM_TIMESTAMP approximated as 'include closed+open' rather than true timestamp-bounded shard-lineage filtering (would need per-shard closed-at timestamps, which are not tracked). (bd: gopherstack-ud2)" + - "Resource policies (PutResourcePolicy/GetResourcePolicy/DeleteResourcePolicy) are not part of backendSnapshot — they are lost across a persistence restart, same class of bug as the tags issue fixed this pass. Not fixed this pass due to time budget; flagged for the next sweep. (bd: gopherstack-ud2)" +deferred: + - "Enhanced fan-out SubscribeToShard real streaming cadence / HTTP2 push semantics beyond the polling emulation already in place" + - "Cross-service Lambda event-source-mapping trigger wiring (lives in cli.go per task constraints; not touched)" +leaks: {status: clean, note: "stream.mu (lockmetrics) and stream.Tags always Close()'d on DeleteStream/Purge; SubscribeToShard polling goroutine bounded by subscribeToShardMaxIdlePolls (3) and a 5-minute deadline, exits on ctx.Done(); FIS throughput-fault goroutines bound to experiment ctx or scheduled cleanup, lazily evict on read; janitor retention sweep is a single ticker goroutine stopped via context cancellation, no per-stream goroutines"} +--- + +## Notes + +### Tag persistence bug (the headline fix this pass) + +Before this pass, `Handler` kept a **second, parallel tag store** (`h.tags map[string]*svcTags.Tags`, +keyed by `region+"/"+streamName`) that was the *only* backing store for tags applied via +`CreateStream` (inline `Tags`), `AddTagsToStream`, `RemoveTagsFromStream`, `ListTagsForStream`, and +`ListTagsForResource`. The backend's own `stream.Tags` field — the one that actually participates in +`Snapshot`/`Restore` (`backendSnapshot.Streams[region][name].Tags` via the `Stream` struct's `json:"tags"` +tag) — was only ever written by `TagResource`/`UntagResource` (the ARN-based API), and the handler's +`ListTagsForResource` never read it (it read `h.tags` too, so this went unnoticed operationally). Net +effect: **every tag applied through the legacy AddTagsToStream/CreateStream API path silently vanished +on process restart**, even though the stream itself, its shards, and its records persisted correctly — +a textbook "persist when persistence is enabled" violation per the no-stub rule, made worse by two +existing tests (`TestRefinement1_ListTagsForResource_SortedOutput`, +`TestRefinement2_ListTagsForResource_UsesHandlerTags`) that had *rationalized the bug as intentional +design* rather than flagging it (the exact "looks-wrong-but-correct" trap the parity playbook warns +about, except here the previous audit got the call wrong). + +Fix: every tag-mutating handler now writes through `Backend.TagResource`/`Backend.UntagResource` +(single source of truth = `stream.Tags`), and every tag-reading handler reads through +`Backend.ListTagsForResource`. The handler-local `h.tags`/`tagsMu` map, `tagKey`/`setTags`/`getTags`/ +`removeTags`, and the `OnStreamPurged` tag-cleanup closure in `WithJanitor` were deleted entirely (dead +weight once the backend is the only store — `stream.Tags.Close()` is already called by the backend on +`DeleteStream`/`Purge`). `CreateStream` now validates inline `Tags` (length + 50-tag cap) *before* +creating the stream, matching AWS's all-or-nothing semantics, and `TagResource` now enforces the same +50-tag cap `AddTagsToStream` always enforced (previously uncapped). See +`TestRefinement2_Tags_SurvivePersistenceRestore` for the regression test that exercises all three +write paths (CreateStream / AddTagsToStream / TagResource) through an actual Snapshot→Restore cycle. + +### PutRecords request-level vs. per-record errors + +AWS's `PutRecords` contract distinguishes **request-level** failures (fail the whole call with a +single top-level exception, HTTP 4xx, no `Records` envelope) from **per-record** failures (200 OK, +each failed entry gets its own `ErrorCode`/`ErrorMessage`, `FailedRecordCount` > 0). Stream-not-found +and an empty `Records` list are request-level. Before this pass, the backend looped +`PutRecord` per entry with no upfront existence check, so a `PutRecords` call against a nonexistent +stream returned **200 OK** with every entry marked `"InternalFailure"` — wrong on three counts (wrong +HTTP status class, wrong error code, wrong response shape). Fixed by resolving the stream once before +the loop and returning `ErrStreamNotFound` at the top level; an empty `Records` slice is now rejected +the same way (AWS's SDK model has `MinItems: 1`). + +### ON_DEMAND default shard count + +AWS allocates **4 shards** to a freshly created `ON_DEMAND` stream (capacity is auto-managed +thereafter); a caller-supplied `ShardCount` is ignored for `ON_DEMAND`. The backend previously fell +through to `defaultShardCount = 1` for `ON_DEMAND` streams with no explicit `ShardCount`, which is +wrong for any test/tool that inspects shard count immediately after creating an on-demand stream (e.g. +to compute expected parallelism). `streamMode` is now resolved *before* `shardCount`, and `ON_DEMAND` +always gets `defaultOnDemandShardCount = 4` regardless of the caller's `ShardCount`. + +### DescribeStream shard pagination + +`DescribeStream`'s `Shards` list has an AWS-documented page contract: default 100, max 10000, +resumed via `ExclusiveStartShardId`, with `HasMoreShards` signaling truncation. The previous +implementation returned **every** shard in the stream in one response and hardcoded +`HasMoreShards: false` — invisible on a fresh stream (≤100 shards from `CreateStream`, capped by +`maxShardsPerStream`), but real once a stream has been resharded enough times: `MergeShards`/ +`SplitShard`/`UpdateShardCount` never remove CLOSED shards from `stream.Shards` (correctly — AWS keeps +closed shards visible for lineage), so a long-lived, heavily-resharded stream's total shard count +(open + closed) is unbounded and can exceed one page. `DescribeStreamInput` gained +`ExclusiveStartShardID`/`Limit` fields (additive; every existing call site that only sets `StreamName` +is unaffected) and `DescribeStreamOutput` gained `HasMoreShards`. + +### Shard hash-range and sequence-number traps (unchanged this pass, re-confirmed correct) + +- **Hash key range**: the full space is `[0, 2^128-1]`. `shardForHashKey` matches a partition key's + MD5-derived `big.Int` against `[HashKeyRangeStart, HashKeyRangeEnd]` inclusive on both ends; the + fallback to "first open shard" (and then `shards[0]` even if closed) only fires if no shard's stored + range covers the hash, which should not happen for internally-generated shards but protects against + a corrupted/hand-seeded stream (see `AddStreamInternal`, test-only) from panicking on `nil`. +- **Sequence numbers** are per-shard monotonic (`Shard.NextSeq`), formatted as + `49<14-digit ms timestamp><4-digit shard idx><20-digit seq>` — this is a plausible-shaped AWS + sequence number (49-prefix) but is **not** globally comparable across shards the way real AWS + sequence numbers are within a single call; `findSequencePosition` binary-searches assuming + ascending order *within one shard's own record list*, which holds because records are always + appended in arrival order — do not assume cross-shard ordering from the string value. + `EndingSequenceNumber` is only populated once a shard is `Closed` — an open shard with records + reports `SequenceNumberRangeEnd: ""` deliberately (real AWS/KCL treats the *presence* of + `EndingSequenceNumber` as the "this shard is closed, move to children" signal; populating it on an + open shard would make consumers abandon it prematurely). This is intentional, not a gap. +- **Reshard lineage**: `MergeShards` accepts either shard as "first" as long as they are hash-range + adjacent (`s1.end+1 == s2.start` or `s2.end+1 == s1.start`); `SplitShard` requires + `NewStartingHashKey` strictly inside `(shard.start, shard.end)` (not equal to either bound). + `UpdateShardCount`'s `findOverlappingParents` assigns up to 2 parent IDs per new shard based on hash + range overlap with the previously-open shard set — this can only ever find 0, 1, or 2 overlapping + parents given the reshard math, matching AWS's parent/adjacent-parent model. + +### Consumer registration limit + +`RegisterStreamConsumer` had no upper bound; AWS caps enhanced fan-out consumers at 20 per stream +(`LimitExceededException` beyond that). Added the check; see +`TestAudit2_RegisterStreamConsumer_LimitExceeded`. + +### Account settings persistence + +`UpdateAccountSettings`'s `OnDemandStreamCountLimit` was backend in-memory state not included in +`backendSnapshot` — every restart silently reset the account's on-demand limit back to the compiled-in +default (10), even if an operator had explicitly raised or lowered it. Added to the snapshot. + +### KMS error codes (deferred, not fabricated) + +The task brief calls out `KMSAccessDenied` as an error code to verify. The real Kinesis API (per the +`aws-sdk-go-v2/service/kinesis` model) does not define a KMS-specific exception for +`StartStreamEncryption`/`StopStreamEncryption`/`UpdateMaxRecordSize` — the modeled exceptions are +`InvalidArgumentException`, `LimitExceededException`, `ResourceInUseException`, +`ResourceNotFoundException`, and a generic `AccessDeniedException` (not currently in this package's +error set at all). Actually validating a `KeyId` would require calling into the `kms` service's +backend, which is a cross-service dependency out of scope for a `services/kinesis/`-only pass per this +sweep's constraints. Fabricating a KMS validation error path not backed by real state would itself be +a stub, so this is left as a documented gap rather than implemented. diff --git a/services/kinesis/accuracy_batch2_ops_test.go b/services/kinesis/accuracy_batch2_ops_test.go index 6a8d32fed..1eb63efd9 100644 --- a/services/kinesis/accuracy_batch2_ops_test.go +++ b/services/kinesis/accuracy_batch2_ops_test.go @@ -25,10 +25,11 @@ func TestAccuracyBatch2_CreateStream_OnDemand_ShardCountZero(t *testing.T) { t.Parallel() tests := []struct { - body map[string]any - name string - wantMode string - wantStatus int + body map[string]any + name string + wantMode string + wantStatus int + wantShardCount int // 0 = not checked }{ { name: "on_demand_no_shard_count", @@ -38,6 +39,9 @@ func TestAccuracyBatch2_CreateStream_OnDemand_ShardCountZero(t *testing.T) { }, wantStatus: http.StatusOK, wantMode: "ON_DEMAND", + // AWS allocates 4 shards to a freshly created on-demand stream; + // ShardCount is not honored for ON_DEMAND streams. + wantShardCount: 4, }, { name: "on_demand_shard_count_zero", @@ -46,18 +50,22 @@ func TestAccuracyBatch2_CreateStream_OnDemand_ShardCountZero(t *testing.T) { "ShardCount": 0, "StreamModeDetails": map[string]any{"StreamMode": "ON_DEMAND"}, }, - wantStatus: http.StatusOK, - wantMode: "ON_DEMAND", + wantStatus: http.StatusOK, + wantMode: "ON_DEMAND", + wantShardCount: 4, }, { - name: "on_demand_explicit_shard_count", + name: "on_demand_explicit_shard_count_ignored", body: map[string]any{ "StreamName": "od-explicit-sc", - "ShardCount": 4, + "ShardCount": 17, "StreamModeDetails": map[string]any{"StreamMode": "ON_DEMAND"}, }, wantStatus: http.StatusOK, wantMode: "ON_DEMAND", + // A caller-supplied ShardCount is ignored for ON_DEMAND streams — + // AWS always starts a new on-demand stream at 4 shards. + wantShardCount: 4, }, { name: "provisioned_still_requires_shard_count", @@ -87,11 +95,18 @@ func TestAccuracyBatch2_CreateStream_OnDemand_ShardCountZero(t *testing.T) { StreamModeDetails *struct { StreamMode string `json:"StreamMode"` } `json:"StreamModeDetails"` + Shards []struct { + ShardID string `json:"ShardId"` + } `json:"Shards"` } `json:"StreamDescription"` } require.NoError(t, json.Unmarshal(descRec.Body.Bytes(), &resp)) require.NotNil(t, resp.StreamDescription.StreamModeDetails) assert.Equal(t, tt.wantMode, resp.StreamDescription.StreamModeDetails.StreamMode) + + if tt.wantShardCount > 0 { + assert.Len(t, resp.StreamDescription.Shards, tt.wantShardCount) + } } }) } diff --git a/services/kinesis/backend.go b/services/kinesis/backend.go index c87cfea27..6f407ee75 100644 --- a/services/kinesis/backend.go +++ b/services/kinesis/backend.go @@ -1,7 +1,6 @@ package kinesis import ( - "cmp" "context" "crypto/md5" //nolint:gosec // MD5 used as a non-cryptographic hash key for Kinesis shard routing, matching the AWS API contract "encoding/base64" @@ -20,6 +19,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/config" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) @@ -154,17 +154,6 @@ func isValidConsumerName(s string) bool { return consumerNameRe.MatchString(s) } -// sortedKeys returns the keys of map m in sorted order. -func sortedKeys[K cmp.Ordered, V any](m map[K]V) []K { - keys := make([]K, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - slices.Sort(keys) - - return keys -} - // kinesisThrottleFault holds the state of an active FIS throughput-exception fault on a stream. type kinesisThrottleFault struct { expiry time.Time @@ -173,11 +162,20 @@ type kinesisThrottleFault struct { // InMemoryBackend implements StorageBackend using in-memory maps. // -// All resource maps are nested by region (outer key = region) so that -// same-named streams in different regions are fully isolated — including their -// shards, records, consumers, FIS throttle faults, and resource policies. +// Streams are keyed by a composite "region/name" key (see streamKey) inside a +// single flat [store.Table], with a secondary [store.Index] grouping them by +// region — so same-named streams in different regions remain fully isolated, +// including their shards, records, and consumers, all of which stay inline +// fields on Stream (see pkgs/store's package doc: shards/records are the +// per-stream hot path and are not decomposed into their own tables). +// fisThroughputFaults and resourcePolicies are NOT store.Table candidates: +// their value types (a pointer with no self-describing identity, and a bare +// string) carry no key of their own to hand a Table keyFn, so they remain +// plain nested maps guarded the same way as before. type InMemoryBackend struct { - streams map[string]map[string]*Stream // region → stream name → stream + streams *store.Table[Stream] + streamsByRegion *store.Index[Stream] + registry *store.Registry fisThroughputFaults map[string]map[string]*kinesisThrottleFault // region → stream name → fault faultsMu *lockmetrics.RWMutex resourcePolicies map[string]map[string]string // region → resource ARN → policy @@ -195,8 +193,7 @@ func NewInMemoryBackend() *InMemoryBackend { // NewInMemoryBackendWithConfig creates a new InMemoryBackend with the given account ID and region. func NewInMemoryBackendWithConfig(accountID, region string) *InMemoryBackend { - return &InMemoryBackend{ - streams: make(map[string]map[string]*Stream), + b := &InMemoryBackend{ fisThroughputFaults: make(map[string]map[string]*kinesisThrottleFault), faultsMu: lockmetrics.New("kinesis.faults"), resourcePolicies: make(map[string]map[string]string), @@ -204,27 +201,26 @@ func NewInMemoryBackendWithConfig(accountID, region string) *InMemoryBackend { region: region, mu: lockmetrics.New("kinesis"), onDemandStreamCountLimit: defaultOnDemandStreamCountLimit, + registry: store.NewRegistry(), } + b.streams = store.Register(b.registry, "streams", store.New(streamTableKeyFn)) + b.streamsByRegion = b.streams.AddIndex("region", func(v *Stream) string { return v.Region }) + + return b } // Region returns the AWS region this backend is configured to use as its default. func (b *InMemoryBackend) Region() string { return b.region } -// streamsStore returns the stream map for the given region, lazily creating it. -// Callers must hold b.mu (write lock). -func (b *InMemoryBackend) streamsStore(region string) map[string]*Stream { - if b.streams[region] == nil { - b.streams[region] = make(map[string]*Stream) - } - - return b.streams[region] +// streamKey builds the composite primary key ("region/name") a Stream is +// stored under in b.streams, mirroring the region-nested map key it replaces. +func streamKey(region, name string) string { + return region + "/" + name } -// streamsView returns the existing stream map for the given region without -// creating it. A nil map (region never written) reads as empty. Safe under a -// read lock. Callers must hold b.mu (read or write lock). -func (b *InMemoryBackend) streamsView(region string) map[string]*Stream { - return b.streams[region] +// streamTableKeyFn is the [store.Table] key function for b.streams. +func streamTableKeyFn(v *Stream) string { + return streamKey(v.Region, v.Name) } // faultsStore returns the FIS throttle-fault map for the given region, lazily @@ -335,7 +331,7 @@ func (s *Shard) nextSequenceNumber() string { ) } -func checkOnDemandLimit(streams map[string]*Stream, limit int) error { +func checkOnDemandLimit(streams []*Stream, limit int) error { onDemandCount := 0 for _, s := range streams { if s.StreamMode == streamModeOnDemand { @@ -349,37 +345,10 @@ func checkOnDemandLimit(streams map[string]*Stream, limit int) error { return nil } -// CreateStream creates a new Kinesis stream. -func (b *InMemoryBackend) CreateStream(ctx context.Context, input *CreateStreamInput) error { - region := getRegion(ctx, b.region) - if input.Region != "" { - region = input.Region - } - - b.mu.Lock("CreateStream") - defer b.mu.Unlock() - - if !isValidStreamName(input.StreamName) { - return ErrValidation - } - - streams := b.streamsStore(region) - if _, exists := streams[input.StreamName]; exists { - return ErrStreamAlreadyExists - } - - // Clamp shardCount to a safe range before using it for allocations or shard math. - shardCount := input.ShardCount - if shardCount <= 0 { - shardCount = defaultShardCount - } - if shardCount > maxShardsPerStream { - return ErrInvalidArgument - } - if shardCount > maxShardCount { - shardCount = maxShardCount - } - +// buildInitialShards partitions the full Kinesis hash key space +// ([0, 2^128-1]) into shardCount contiguous, non-overlapping open shards with +// sequential shard IDs starting at shardId-000000000000. +func buildInitialShards(shardCount int) []*Shard { maxHashKey := new(big.Int).Sub( new(big.Int).Lsh(big.NewInt(1), maxHashKeyBits), big.NewInt(1), @@ -413,9 +382,25 @@ func (b *InMemoryBackend) CreateStream(ctx context.Context, input *CreateStreamI }) } - accountID := b.accountID - if input.AccountID != "" { - accountID = input.AccountID + return shards +} + +// CreateStream creates a new Kinesis stream. +func (b *InMemoryBackend) CreateStream(ctx context.Context, input *CreateStreamInput) error { + region := getRegion(ctx, b.region) + if input.Region != "" { + region = input.Region + } + + b.mu.Lock("CreateStream") + defer b.mu.Unlock() + + if !isValidStreamName(input.StreamName) { + return ErrValidation + } + + if b.streams.Has(streamKey(region, input.StreamName)) { + return ErrStreamAlreadyExists } streamMode := input.StreamMode @@ -425,17 +410,41 @@ func (b *InMemoryBackend) CreateStream(ctx context.Context, input *CreateStreamI return ErrInvalidArgument } + // Clamp shardCount to a safe range before using it for allocations or shard math. + // ON_DEMAND streams ignore any caller-supplied ShardCount — AWS auto-manages + // capacity and a freshly created on-demand stream starts with 4 shards. + shardCount := input.ShardCount if streamMode == streamModeOnDemand { - if err := checkOnDemandLimit(streams, b.onDemandStreamCountLimit); err != nil { + shardCount = defaultOnDemandShardCount + } else if shardCount <= 0 { + shardCount = defaultShardCount + } + if shardCount > maxShardsPerStream { + return ErrInvalidArgument + } + if shardCount > maxShardCount { + shardCount = maxShardCount + } + + shards := buildInitialShards(shardCount) + + accountID := b.accountID + if input.AccountID != "" { + accountID = input.AccountID + } + + if streamMode == streamModeOnDemand { + if err := checkOnDemandLimit(b.streamsByRegion.Get(region), b.onDemandStreamCountLimit); err != nil { return err } } streamARN := arn.Build("kinesis", region, accountID, "stream/"+input.StreamName) - streams[input.StreamName] = &Stream{ + b.streams.Put(&Stream{ Name: input.StreamName, ARN: streamARN, + Region: region, Status: streamStatusActive, Shards: shards, mu: newStreamLock(input.StreamName), @@ -445,7 +454,7 @@ func (b *InMemoryBackend) CreateStream(ctx context.Context, input *CreateStreamI Consumers: make(map[string]*Consumer), StreamMode: streamMode, MaxRecordSizeBytes: defaultMaxRecordSizeBytes, - } + }) return nil } @@ -456,8 +465,7 @@ func (b *InMemoryBackend) DeleteStream(ctx context.Context, input *DeleteStreamI b.mu.Lock("DeleteStream") - streams := b.streamsStore(region) - stream, exists := streams[input.StreamName] + stream, exists := b.streams.Get(streamKey(region, input.StreamName)) if !exists { b.mu.Unlock() @@ -472,7 +480,7 @@ func (b *InMemoryBackend) DeleteStream(ctx context.Context, input *DeleteStreamI // Mark the stream as deleting before removing it (AWS-realistic status transition). stream.Status = streamStatusDeleting - delete(streams, input.StreamName) + b.streams.Delete(streamKey(region, input.StreamName)) b.mu.Unlock() b.faultsMu.Lock("DeleteStream.faults") delete(b.faultsStore(region), input.StreamName) @@ -497,7 +505,7 @@ func (b *InMemoryBackend) DescribeStream( b.mu.RLock("DescribeStream") - stream, exists := b.streamsView(region)[input.StreamName] + stream, exists := b.streams.Get(streamKey(region, input.StreamName)) if !exists { b.mu.RUnlock() @@ -507,8 +515,42 @@ func (b *InMemoryBackend) DescribeStream( b.mu.RUnlock() defer stream.mu.RUnlock() - shards := make([]ShardDescription, len(stream.Shards)) - for i, s := range stream.Shards { + // AWS paginates the Shards list: default page size 100, max 10000, resumed + // via ExclusiveStartShardId. A stream that has been resharded many times + // accumulates CLOSED shards forever (they remain visible for lineage), so + // long-lived, heavily-resharded streams can exceed a single page. + const ( + defaultDescribeStreamShardLimit = 100 + maxDescribeStreamShardLimit = 10000 + ) + + limit := input.Limit + if limit <= 0 { + limit = defaultDescribeStreamShardLimit + } + if limit > maxDescribeStreamShardLimit { + limit = maxDescribeStreamShardLimit + } + + startIdx := 0 + if input.ExclusiveStartShardID != "" { + for i, s := range stream.Shards { + if s.ID == input.ExclusiveStartShardID { + startIdx = i + 1 + + break + } + } + } + + all := stream.Shards[startIdx:] + hasMore := len(all) > limit + if hasMore { + all = all[:limit] + } + + shards := make([]ShardDescription, len(all)) + for i, s := range all { shards[i] = shardDescription(s) } @@ -522,6 +564,7 @@ func (b *InMemoryBackend) DescribeStream( StreamARN: stream.ARN, StreamStatus: stream.Status, Shards: shards, + HasMoreShards: hasMore, RetentionPeriodHours: stream.RetentionPeriod, EncryptionType: encType, KeyID: stream.KeyID, @@ -545,7 +588,12 @@ func (b *InMemoryBackend) ListStreams(ctx context.Context, input *ListStreamsInp defer b.mu.RUnlock() // AWS returns stream names in alphabetical order. - names := sortedKeys(b.streamsView(region)) + regionStreams := b.streamsByRegion.Get(region) + names := make([]string, len(regionStreams)) + for i, s := range regionStreams { + names[i] = s.Name + } + slices.Sort(names) // Apply pagination start point: prefer ExclusiveStartStreamName, then NextToken. start := input.ExclusiveStartStreamName @@ -594,7 +642,7 @@ func (b *InMemoryBackend) PutRecord(ctx context.Context, input *PutRecordInput) b.mu.RLock("PutRecord") - stream, exists := b.streamsView(region)[input.StreamName] + stream, exists := b.streams.Get(streamKey(region, input.StreamName)) if !exists { b.mu.RUnlock() @@ -689,15 +737,21 @@ func putRecordErrorCode(err error) string { } // PutRecords writes multiple records to a stream. +// +// Request-level validation errors (empty/oversized batch, unknown stream) fail +// the whole call with a single top-level exception, matching the AWS contract: +// only per-record issues (throughput, per-record validation) surface as +// per-entry ErrorCode/ErrorMessage inside a 200 response. func (b *InMemoryBackend) PutRecords(ctx context.Context, input *PutRecordsInput) (*PutRecordsOutput, error) { // AWS PutRecords caps a request at 500 records and 5 MiB total payload - // (sum of partition-key + data bytes across every entry). + // (sum of partition-key + data bytes across every entry), and rejects an + // empty Records list outright (MinItems=1 in the SDK model). const ( maxRecordsPerRequest = 500 maxBatchPayloadBytes = 5 * 1024 * 1024 ) - if len(input.Records) > maxRecordsPerRequest { + if len(input.Records) == 0 || len(input.Records) > maxRecordsPerRequest { return nil, ErrInvalidArgument } @@ -710,6 +764,17 @@ func (b *InMemoryBackend) PutRecords(ctx context.Context, input *PutRecordsInput return nil, ErrInvalidArgument } + // Resolve the stream once up front: AWS fails the entire PutRecords call + // with a top-level ResourceNotFoundException when the stream does not + // exist, rather than reporting "InternalFailure" on every result entry. + region := getRegion(ctx, b.region) + b.mu.RLock("PutRecords.exists") + _, exists := b.streams.Get(streamKey(region, input.StreamName)) + b.mu.RUnlock() + if !exists { + return nil, ErrStreamNotFound + } + results := make([]PutRecordsResultEntry, len(input.Records)) failedCount := 0 @@ -779,7 +844,7 @@ func (b *InMemoryBackend) GetShardIterator( b.mu.RLock("GetShardIterator") - stream, exists := b.streamsView(region)[input.StreamName] + stream, exists := b.streams.Get(streamKey(region, input.StreamName)) if !exists { b.mu.RUnlock() @@ -865,7 +930,7 @@ func (b *InMemoryBackend) GetRecords(ctx context.Context, input *GetRecordsInput b.mu.RLock("GetRecords") - stream, exists := b.streamsView(region)[it.StreamName] + stream, exists := b.streams.Get(streamKey(region, it.StreamName)) if !exists { b.mu.RUnlock() @@ -1007,7 +1072,7 @@ func (b *InMemoryBackend) ListShards(ctx context.Context, input *ListShardsInput b.mu.RLock("ListShards") - stream, exists := b.streamsView(region)[input.StreamName] + stream, exists := b.streams.Get(streamKey(region, input.StreamName)) if !exists { b.mu.RUnlock() @@ -1066,18 +1131,16 @@ func (b *InMemoryBackend) ListAll(_ context.Context) []StreamInfo { b.mu.RLock("ListAll") defer b.mu.RUnlock() - result := make([]StreamInfo, 0) - for _, regionStreams := range b.streams { - for _, s := range regionStreams { - s.mu.RLock("ListAll.stream") - result = append(result, StreamInfo{ - Name: s.Name, - ARN: s.ARN, - Status: s.Status, - ShardCount: len(s.Shards), - }) - s.mu.RUnlock() - } + result := make([]StreamInfo, 0, b.streams.Len()) + for _, s := range b.streams.All() { + s.mu.RLock("ListAll.stream") + result = append(result, StreamInfo{ + Name: s.Name, + ARN: s.ARN, + Status: s.Status, + ShardCount: len(s.Shards), + }) + s.mu.RUnlock() } return result @@ -1196,7 +1259,7 @@ func (b *InMemoryBackend) RegisterStreamConsumer( b.mu.RLock("RegisterStreamConsumer") streamName := streamNameFromARN(input.StreamARN) - stream, ok := b.streamsView(region)[streamName] + stream, ok := b.streams.Get(streamKey(region, streamName)) if !ok { b.mu.RUnlock() @@ -1219,6 +1282,11 @@ func (b *InMemoryBackend) RegisterStreamConsumer( return nil, ErrConsumerAlreadyExists } + // AWS caps enhanced fan-out registrations at 20 consumers per stream. + if len(stream.Consumers) >= maxConsumersPerStream { + return nil, ErrLimitExceeded + } + now := time.Now() consumerARN := buildConsumerARN(input.StreamARN, input.ConsumerName, now) @@ -1255,7 +1323,7 @@ func (b *InMemoryBackend) DescribeStreamConsumer( region := regionFromARNOrCtx(ctx, arnForRegion, b.region) b.mu.RLock("DescribeStreamConsumer") - stream, ok := b.streamsView(region)[sName] + stream, ok := b.streams.Get(streamKey(region, sName)) if !ok { b.mu.RUnlock() if input.ConsumerARN != "" { @@ -1286,7 +1354,7 @@ func (b *InMemoryBackend) ListStreamConsumers( b.mu.RLock("ListStreamConsumers") streamName := streamNameFromARN(input.StreamARN) - stream, ok := b.streamsView(region)[streamName] + stream, ok := b.streams.Get(streamKey(region, streamName)) if !ok { b.mu.RUnlock() @@ -1343,7 +1411,7 @@ func (b *InMemoryBackend) DeregisterStreamConsumer(ctx context.Context, input *D region := regionFromARNOrCtx(ctx, arnForRegion, b.region) b.mu.RLock("DeregisterStreamConsumer") - stream, ok := b.streamsView(region)[sName] + stream, ok := b.streams.Get(streamKey(region, sName)) if !ok { b.mu.RUnlock() @@ -1373,7 +1441,7 @@ func (b *InMemoryBackend) SubscribeToShard( b.mu.RLock("SubscribeToShard") - stream, ok := b.streamsView(region)[sName] + stream, ok := b.streams.Get(streamKey(region, sName)) if !ok { b.mu.RUnlock() @@ -1486,7 +1554,7 @@ func (b *InMemoryBackend) UpdateShardCount( b.mu.Lock("UpdateShardCount") defer b.mu.Unlock() - stream, ok := b.streamsStore(region)[input.StreamName] + stream, ok := b.streams.Get(streamKey(region, input.StreamName)) if !ok { return nil, ErrStreamNotFound } @@ -1593,7 +1661,7 @@ func (b *InMemoryBackend) EnableEnhancedMonitoring( b.mu.Lock("EnableEnhancedMonitoring") defer b.mu.Unlock() - stream, ok := b.streamsStore(region)[input.StreamName] + stream, ok := b.streams.Get(streamKey(region, input.StreamName)) if !ok { return nil, ErrStreamNotFound } @@ -1626,7 +1694,7 @@ func (b *InMemoryBackend) DisableEnhancedMonitoring( b.mu.Lock("DisableEnhancedMonitoring") defer b.mu.Unlock() - stream, ok := b.streamsStore(region)[input.StreamName] + stream, ok := b.streams.Get(streamKey(region, input.StreamName)) if !ok { return nil, ErrStreamNotFound } @@ -1660,7 +1728,7 @@ func (b *InMemoryBackend) IncreaseStreamRetentionPeriod( b.mu.Lock("IncreaseStreamRetentionPeriod") defer b.mu.Unlock() - stream, ok := b.streamsStore(region)[input.StreamName] + stream, ok := b.streams.Get(streamKey(region, input.StreamName)) if !ok { return ErrStreamNotFound } @@ -1695,7 +1763,7 @@ func (b *InMemoryBackend) DecreaseStreamRetentionPeriod( b.mu.Lock("DecreaseStreamRetentionPeriod") defer b.mu.Unlock() - stream, ok := b.streamsStore(region)[input.StreamName] + stream, ok := b.streams.Get(streamKey(region, input.StreamName)) if !ok { return ErrStreamNotFound } @@ -1750,7 +1818,7 @@ func (b *InMemoryBackend) MergeShards(ctx context.Context, input *MergeShardsInp streamName = streamNameFromARN(input.StreamARN) } - stream, ok := b.streamsStore(region)[streamName] + stream, ok := b.streams.Get(streamKey(region, streamName)) if !ok { return ErrStreamNotFound } @@ -1831,7 +1899,7 @@ func (b *InMemoryBackend) SplitShard(ctx context.Context, input *SplitShardInput streamName = streamNameFromARN(input.StreamARN) } - stream, ok := b.streamsStore(region)[streamName] + stream, ok := b.streams.Get(streamKey(region, streamName)) if !ok { return ErrStreamNotFound } @@ -1912,7 +1980,7 @@ func (b *InMemoryBackend) StartStreamEncryption(ctx context.Context, input *Star streamName = streamNameFromARN(input.StreamARN) } - stream, ok := b.streamsStore(region)[streamName] + stream, ok := b.streams.Get(streamKey(region, streamName)) if !ok { return ErrStreamNotFound } @@ -1941,7 +2009,7 @@ func (b *InMemoryBackend) StopStreamEncryption(ctx context.Context, input *StopS streamName = streamNameFromARN(input.StreamARN) } - stream, ok := b.streamsStore(region)[streamName] + stream, ok := b.streams.Get(streamKey(region, streamName)) if !ok { return ErrStreamNotFound } @@ -2012,7 +2080,7 @@ func (b *InMemoryBackend) ListTagsForResource( b.mu.RLock("ListTagsForResource") streamName := streamNameFromARN(input.ResourceARN) - stream, ok := b.streamsView(region)[streamName] + stream, ok := b.streams.Get(streamKey(region, streamName)) if !ok { b.mu.RUnlock() @@ -2041,7 +2109,7 @@ func (b *InMemoryBackend) DescribeAccountSettings(ctx context.Context) (*Describ defer b.mu.RUnlock() onDemandCount := 0 - for _, s := range b.streamsView(region) { + for _, s := range b.streamsByRegion.Get(region) { s.mu.RLock("DescribeAccountSettings.stream") if s.StreamMode == streamModeOnDemand { onDemandCount++ @@ -2062,17 +2130,15 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - for _, regionStreams := range b.streams { - for _, stream := range regionStreams { - stream.mu.Lock("Reset.stream") - if stream.Tags != nil { - stream.Tags.Close() - } - stream.mu.Unlock() + for _, stream := range b.streams.All() { + stream.mu.Lock("Reset.stream") + if stream.Tags != nil { + stream.Tags.Close() } + stream.mu.Unlock() } - b.streams = make(map[string]map[string]*Stream) + b.registry.ResetAll() b.faultsMu.Lock("Reset.faults") b.fisThroughputFaults = make(map[string]map[string]*kinesisThrottleFault) b.faultsMu.Unlock() @@ -2090,7 +2156,7 @@ func (b *InMemoryBackend) purgeStreamEntry(region, name string, s *Stream, cutof if s.Tags != nil { s.Tags.Close() } - delete(b.streamsStore(region), name) + b.streams.Delete(streamKey(region, name)) b.faultsMu.Lock("Purge.faults") delete(b.faultsStore(region), name) b.faultsMu.Unlock() @@ -2127,17 +2193,12 @@ func (b *InMemoryBackend) Purge(ctx context.Context, cutoff time.Time) { var purgedNames []string b.mu.Lock("Purge") - for region, regionStreams := range b.streams { + for _, s := range b.streams.All() { if ctx.Err() != nil { break } - for name, s := range regionStreams { - if ctx.Err() != nil { - break - } - if b.purgeStreamEntry(region, name, s, cutoff) { - purgedNames = append(purgedNames, name) - } + if b.purgeStreamEntry(s.Region, s.Name, s, cutoff) { + purgedNames = append(purgedNames, s.Name) } } b.mu.Unlock() @@ -2155,7 +2216,7 @@ func (b *InMemoryBackend) CountOpenShards(ctx context.Context) int { defer b.mu.RUnlock() count := 0 - for _, s := range b.streamsView(region) { + for _, s := range b.streamsByRegion.Get(region) { s.mu.RLock("CountOpenShards.stream") for _, sh := range s.Shards { if !sh.Closed { @@ -2197,7 +2258,7 @@ func (b *InMemoryBackend) UpdateMaxRecordSize(ctx context.Context, input *Update streamName = streamNameFromARN(input.StreamARN) } - stream, ok := b.streamsView(region)[streamName] + stream, ok := b.streams.Get(streamKey(region, streamName)) if !ok { b.mu.RUnlock() @@ -2231,7 +2292,7 @@ func (b *InMemoryBackend) UpdateStreamWarmThroughput( streamName = streamNameFromARN(input.StreamARN) } - _, ok := b.streamsView(region)[streamName] + _, ok := b.streams.Get(streamKey(region, streamName)) b.mu.RUnlock() if !ok { @@ -2249,7 +2310,7 @@ func (b *InMemoryBackend) TagResource(ctx context.Context, input *TagResourceInp b.mu.RLock("TagResource") streamName := streamNameFromARN(input.ResourceARN) - stream, ok := b.streamsView(region)[streamName] + stream, ok := b.streams.Get(streamKey(region, streamName)) if !ok { b.mu.RUnlock() @@ -2277,7 +2338,7 @@ func (b *InMemoryBackend) UntagResource(ctx context.Context, input *UntagResourc b.mu.RLock("UntagResource") streamName := streamNameFromARN(input.ResourceARN) - stream, ok := b.streamsView(region)[streamName] + stream, ok := b.streams.Get(streamKey(region, streamName)) if !ok { b.mu.RUnlock() @@ -2309,8 +2370,9 @@ func (b *InMemoryBackend) AddStreamInternal(stream *Stream) { if region == "" { region = b.region } + stream.Region = region - b.streamsStore(region)[stream.Name] = stream + b.streams.Put(stream) } // UpdateStreamMode changes the mode of a stream identified by its ARN. @@ -2321,7 +2383,7 @@ func (b *InMemoryBackend) UpdateStreamMode(ctx context.Context, input *UpdateStr defer b.mu.Unlock() streamName := streamNameFromARN(input.StreamARN) - stream, ok := b.streamsStore(region)[streamName] + stream, ok := b.streams.Get(streamKey(region, streamName)) if !ok { return ErrStreamNotFound } diff --git a/services/kinesis/export_test.go b/services/kinesis/export_test.go index bb338996a..4aff8a01f 100644 --- a/services/kinesis/export_test.go +++ b/services/kinesis/export_test.go @@ -65,7 +65,7 @@ func (b *InMemoryBackend) HasFaultForTest(streamName string) bool { func (b *InMemoryBackend) ShardRecordCountForTest(streamName string, shardIdx int) int { b.mu.RLock("ShardRecordCountForTest") - stream, ok := b.streamsView(b.region)[streamName] + stream, ok := b.streams.Get(streamKey(b.region, streamName)) if !ok || shardIdx >= len(stream.Shards) { b.mu.RUnlock() @@ -93,7 +93,7 @@ func (b *InMemoryBackend) SetRetentionPeriodForTest(streamName string, hours int b.mu.Lock("SetRetentionPeriodForTest") defer b.mu.Unlock() - stream, ok := b.streamsView(b.region)[streamName] + stream, ok := b.streams.Get(streamKey(b.region, streamName)) if !ok { return ErrStreamNotFound } @@ -111,7 +111,7 @@ func (b *InMemoryBackend) PushOldRecordForTest(streamName string, shardIdx int, b.mu.Lock("PushOldRecordForTest") defer b.mu.Unlock() - stream, ok := b.streamsView(b.region)[streamName] + stream, ok := b.streams.Get(streamKey(b.region, streamName)) if !ok { return ErrStreamNotFound } @@ -160,12 +160,7 @@ func (b *InMemoryBackend) StreamCount() int { b.mu.RLock("StreamCount") defer b.mu.RUnlock() - count := 0 - for _, regionStreams := range b.streams { - count += len(regionStreams) - } - - return count + return b.streams.Len() } // ResourcePolicyCount returns the total number of resource policies in the backend diff --git a/services/kinesis/handler.go b/services/kinesis/handler.go index 8120a210d..4cd60e368 100644 --- a/services/kinesis/handler.go +++ b/services/kinesis/handler.go @@ -19,7 +19,6 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/collections" "github.com/blackbirdworks/gopherstack/pkgs/httputils" - "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/service" svcTags "github.com/blackbirdworks/gopherstack/pkgs/tags" @@ -29,8 +28,6 @@ import ( type Handler struct { Backend StorageBackend janitor *Janitor - tags map[string]*svcTags.Tags - tagsMu *lockmetrics.RWMutex ops map[string]kinesisDispatchFn DefaultRegion string AccountID string @@ -40,8 +37,6 @@ type Handler struct { func NewHandler(backend StorageBackend) *Handler { h := &Handler{ Backend: backend, - tags: make(map[string]*svcTags.Tags), - tagsMu: lockmetrics.New("kinesis.tags"), } h.ops = h.buildOps() @@ -56,24 +51,6 @@ func (h *Handler) WithJanitor(interval time.Duration, taskTimeout ...time.Durati if len(taskTimeout) > 0 { j.TaskTimeout = taskTimeout[0] } - // Wire the cleanup callback so that when a stream is purged from the backend - // the handler-level tag registry for that stream is also closed and removed. - // Tags are keyed by "region/streamName", so a purge of a given stream name - // clears that stream's tag registry across every region it appears in. - mem.OnStreamPurged = func(streamName string) { - suffix := "/" + streamName - h.tagsMu.Lock("OnStreamPurged") - for key, t := range h.tags { - if !strings.HasSuffix(key, suffix) { - continue - } - if t != nil { - t.Close() - } - delete(h.tags, key) - } - h.tagsMu.Unlock() - } h.janitor = j } @@ -112,44 +89,6 @@ func (h *Handler) defaultRegion() string { return h.DefaultRegion } -// tagKey builds the region-scoped key under which a stream's handler-level tags -// are stored, keeping tags for same-named streams in different regions isolated. -func tagKey(region, streamName string) string { - return region + "/" + streamName -} - -func (h *Handler) setTags(region, resourceID string, kv map[string]string) { - key := tagKey(region, resourceID) - h.tagsMu.Lock("setTags") - defer h.tagsMu.Unlock() - if h.tags[key] == nil { - h.tags[key] = svcTags.New("kinesis." + key + ".tags") - } - h.tags[key].Merge(kv) -} - -func (h *Handler) removeTags(region, resourceID string, keys []string) { - key := tagKey(region, resourceID) - h.tagsMu.RLock("removeTags") - t := h.tags[key] - h.tagsMu.RUnlock() - if t != nil { - t.DeleteKeys(keys) - } -} - -func (h *Handler) getTags(region, resourceID string) map[string]string { - key := tagKey(region, resourceID) - h.tagsMu.RLock("getTags") - t := h.tags[key] - h.tagsMu.RUnlock() - if t == nil { - return map[string]string{} - } - - return t.Clone() -} - // Name returns the service name. func (h *Handler) Name() string { return "Kinesis" @@ -380,6 +319,10 @@ type jsonDeleteStreamReq struct { type jsonDescribeStreamReq struct { StreamARN string `json:"StreamARN"` StreamName string `json:"StreamName"` + // ExclusiveStartShardID and Limit paginate the Shards list (DescribeStream + // only; DescribeStreamSummary ignores them, matching the AWS request shape). + ExclusiveStartShardID string `json:"ExclusiveStartShardId"` + Limit int `json:"Limit"` } type jsonListStreamsReq struct { @@ -577,6 +520,16 @@ func (h *Handler) handleCreateStream( return nil, ErrValidation } + // Validate tags before mutating any state: AWS rejects the whole CreateStream + // call (no stream created) when the inline Tags fail key/value length or + // count constraints. + if err := validateTagKVs(req.Tags); err != nil { + return nil, err + } + if len(req.Tags) > maxTagsPerStream { + return nil, ErrTagLimitExceeded + } + region := getRegion(ctx, h.defaultRegion()) var streamMode string @@ -607,7 +560,17 @@ func (h *Handler) handleCreateStream( } if len(req.Tags) > 0 { - h.setTags(region, req.StreamName, req.Tags) + // Persist tags on the backend's stream.Tags (the field that actually + // participates in Snapshot/Restore) rather than a handler-local map, so + // tags applied at creation time survive a persistence round-trip. + if out, dErr := h.Backend.DescribeStream(ctx, &DescribeStreamInput{StreamName: req.StreamName}); dErr == nil { + if tErr := h.Backend.TagResource(ctx, &TagResourceInput{ + ResourceARN: out.StreamARN, + Tags: req.Tags, + }); tErr != nil { + logger.Load(ctx).WarnContext(ctx, "CreateStream: failed to apply inline tags", "error", tErr) + } + } } return struct{}{}, nil @@ -640,16 +603,6 @@ func (h *Handler) handleDeleteStream( return nil, err } - // Clean up handler-level tags to prevent resource/metric leaks. - key := tagKey(region, streamName) - h.tagsMu.Lock("handleDeleteStream") - if t := h.tags[key]; t != nil { - t.Close() - } - - delete(h.tags, key) - h.tagsMu.Unlock() - return struct{}{}, nil } @@ -678,7 +631,11 @@ func (h *Handler) handleDescribeStream( streamName = streamNameFromARN(req.StreamARN) } - out, err := h.Backend.DescribeStream(ctx, &DescribeStreamInput{StreamName: streamName}) + out, err := h.Backend.DescribeStream(ctx, &DescribeStreamInput{ + StreamName: streamName, + ExclusiveStartShardID: req.ExclusiveStartShardID, + Limit: req.Limit, + }) if err != nil { return nil, err } @@ -707,7 +664,7 @@ func (h *Handler) handleDescribeStream( StreamStatus: out.StreamStatus, RetentionPeriodHours: out.RetentionPeriodHours, Shards: shards, - HasMoreShards: false, + HasMoreShards: out.HasMoreShards, EncryptionType: out.EncryptionType, KeyID: out.KeyID, EnhancedMonitoring: enhancedMonitoringEntries(out.EnhancedMonitoring), @@ -1123,7 +1080,8 @@ func (h *Handler) handleAddTagsToStream( return nil, ErrInvalidArgument } - if _, err := h.Backend.DescribeStream(ctx, &DescribeStreamInput{StreamName: req.StreamName}); err != nil { + out, err := h.Backend.DescribeStream(ctx, &DescribeStreamInput{StreamName: req.StreamName}) + if err != nil { return nil, err } @@ -1132,19 +1090,27 @@ func (h *Handler) handleAddTagsToStream( kv = req.Tags.Clone() } - if err := validateTagKVs(kv); err != nil { + if err = validateTagKVs(kv); err != nil { return nil, err } - region := getRegion(ctx, h.defaultRegion()) - existing := h.getTags(region, req.StreamName) - merged := make(map[string]string, len(existing)) - maps.Copy(merged, existing) + // Tags live on the backend's stream.Tags (persisted via Snapshot/Restore), + // not a handler-local map, so AddTagsToStream and TagResource share one + // source of truth and tags survive a persistence round-trip. + existingOut, err := h.Backend.ListTagsForResource(ctx, &ListTagsForResourceInput{ResourceARN: out.StreamARN}) + if err != nil { + return nil, err + } + merged := make(map[string]string, len(existingOut.Tags)+len(kv)) + maps.Copy(merged, existingOut.Tags) maps.Copy(merged, kv) if len(merged) > maxTagsPerStream { return nil, ErrTagLimitExceeded } - h.setTags(region, req.StreamName, kv) + + if err = h.Backend.TagResource(ctx, &TagResourceInput{ResourceARN: out.StreamARN, Tags: kv}); err != nil { + return nil, err + } return struct{}{}, nil } @@ -1164,11 +1130,17 @@ func (h *Handler) handleRemoveTagsFromStream( return nil, ErrInvalidArgument } - if _, err := h.Backend.DescribeStream(ctx, &DescribeStreamInput{StreamName: req.StreamName}); err != nil { + out, err := h.Backend.DescribeStream(ctx, &DescribeStreamInput{StreamName: req.StreamName}) + if err != nil { return nil, err } - h.removeTags(getRegion(ctx, h.defaultRegion()), req.StreamName, req.TagKeys) + if err = h.Backend.UntagResource(ctx, &UntagResourceInput{ + ResourceARN: out.StreamARN, + TagKeys: req.TagKeys, + }); err != nil { + return nil, err + } return struct{}{}, nil } @@ -1189,11 +1161,16 @@ func (h *Handler) handleListTagsForStream( return nil, ErrInvalidArgument } - if _, err := h.Backend.DescribeStream(ctx, &DescribeStreamInput{StreamName: req.StreamName}); err != nil { + out, err := h.Backend.DescribeStream(ctx, &DescribeStreamInput{StreamName: req.StreamName}) + if err != nil { return nil, err } - tagsMap := h.getTags(getRegion(ctx, h.defaultRegion()), req.StreamName) + tagsOut, err := h.Backend.ListTagsForResource(ctx, &ListTagsForResourceInput{ResourceARN: out.StreamARN}) + if err != nil { + return nil, err + } + tagsMap := tagsOut.Tags keys := collections.SortedKeys(tagsMap) @@ -1499,18 +1476,13 @@ func (h *Handler) handleListTagsForResource( return nil, ErrInvalidArgument } - streamName := streamNameFromARN(req.ResourceARN) - region := regionFromARNOrCtx(ctx, req.ResourceARN, h.defaultRegion()) - regionCtx := contextWithRegion(ctx, region) - - // Validate the stream exists before returning tags. - if _, err := h.Backend.DescribeStream(regionCtx, &DescribeStreamInput{StreamName: streamName}); err != nil { + out, err := h.Backend.ListTagsForResource(ctx, &ListTagsForResourceInput{ResourceARN: req.ResourceARN}) + if err != nil { return nil, err } - tags := h.getTags(region, streamName) - tagList := make([]svcTags.KV, 0, len(tags)) - for k, v := range tags { + tagList := make([]svcTags.KV, 0, len(out.Tags)) + for k, v := range out.Tags { tagList = append(tagList, svcTags.KV{Key: k, Value: v}) } slices.SortFunc(tagList, func(a, b svcTags.KV) int { @@ -2046,17 +2018,6 @@ func (h *Handler) Reset() { if r, ok := h.Backend.(resetter); ok { r.Reset() } - - // Close and discard handler-level tag registries to prevent metric leaks. - h.tagsMu.Lock("Reset") - for _, t := range h.tags { - if t != nil { - t.Close() - } - } - - h.tags = make(map[string]*svcTags.Tags) - h.tagsMu.Unlock() } // Purge implements service.Purgeable by removing all Kinesis streams older than cutoff. @@ -2110,19 +2071,27 @@ func (h *Handler) handleTagResource( return nil, err } - if err := h.Backend.TagResource(ctx, &TagResourceInput{ + // Enforce the AWS 50-tag-per-stream cap here too: TagResource and + // AddTagsToStream share the same underlying tag set (stream.Tags), so the + // limit must be consistent across both entry points. + existingOut, err := h.Backend.ListTagsForResource(ctx, &ListTagsForResourceInput{ResourceARN: req.ResourceARN}) + if err != nil { + return nil, err + } + merged := make(map[string]string, len(existingOut.Tags)+len(req.Tags)) + maps.Copy(merged, existingOut.Tags) + maps.Copy(merged, req.Tags) + if len(merged) > maxTagsPerStream { + return nil, ErrTagLimitExceeded + } + + if err = h.Backend.TagResource(ctx, &TagResourceInput{ ResourceARN: req.ResourceARN, Tags: req.Tags, }); err != nil { return nil, err } - // Mirror into the handler-level tag store for ListTagsForStream compatibility. - streamName := streamNameFromARN(req.ResourceARN) - if streamName != "" { - h.setTags(regionFromARNOrCtx(ctx, req.ResourceARN, h.defaultRegion()), streamName, req.Tags) - } - return struct{}{}, nil } @@ -2143,12 +2112,6 @@ func (h *Handler) handleUntagResource( return nil, err } - // Mirror removal into the handler-level tag store. - streamName := streamNameFromARN(req.ResourceARN) - if streamName != "" { - h.removeTags(regionFromARNOrCtx(ctx, req.ResourceARN, h.defaultRegion()), streamName, req.TagKeys) - } - return struct{}{}, nil } diff --git a/services/kinesis/handler_audit2_test.go b/services/kinesis/handler_audit2_test.go index 9b4fda41e..53683a31e 100644 --- a/services/kinesis/handler_audit2_test.go +++ b/services/kinesis/handler_audit2_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "net/http" + "strconv" "strings" "testing" @@ -431,6 +432,58 @@ func TestAudit2_RegisterStreamConsumer_ValidNames(t *testing.T) { } } +// TestAudit2_RegisterStreamConsumer_LimitExceeded verifies that AWS's 20 +// registered-consumers-per-stream cap is enforced. Previously +// RegisterStreamConsumer had no upper bound at all. +func TestAudit2_RegisterStreamConsumer_LimitExceeded(t *testing.T) { + t.Parallel() + + tests := []struct { + wantErr error + name string + preRegistered int + }{ + {name: "under_limit_succeeds", preRegistered: 19, wantErr: nil}, + {name: "at_limit_rejected", preRegistered: 20, wantErr: kinesis.ErrLimitExceeded}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := kinesis.NewInMemoryBackend() + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "consumer-limit-stream", + ShardCount: 1, + })) + + desc, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: "consumer-limit-stream"}, + ) + require.NoError(t, err) + + for i := range tt.preRegistered { + _, regErr := b.RegisterStreamConsumer(context.Background(), &kinesis.RegisterStreamConsumerInput{ + StreamARN: desc.StreamARN, + ConsumerName: "consumer-" + strconv.Itoa(i), + }) + require.NoError(t, regErr) + } + + _, err = b.RegisterStreamConsumer(context.Background(), &kinesis.RegisterStreamConsumerInput{ + StreamARN: desc.StreamARN, + ConsumerName: "one-more-consumer", + }) + if tt.wantErr == nil { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, tt.wantErr) + } + }) + } +} + // --------------------------------------------------------------------------- // Constraint 6: ListStreamConsumers MaxResults pagination // --------------------------------------------------------------------------- diff --git a/services/kinesis/handler_refinement1_test.go b/services/kinesis/handler_refinement1_test.go index dbb80d35a..b01fdc6a4 100644 --- a/services/kinesis/handler_refinement1_test.go +++ b/services/kinesis/handler_refinement1_test.go @@ -452,16 +452,14 @@ func TestRefinement1_ListTagsForResource_SortedOutput(t *testing.T) { arn := createStreamAndGetARN(t, h, "sorted-tags-target") - // Add tags via PutResourcePolicy then use stream tags via AddTagsToStream. + // AddTagsToStream and ListTagsForResource share one backing store + // (stream.Tags), so tags applied via the legacy AddTagsToStream API must be + // visible — and sorted by key — via the ARN-based ListTagsForResource API. doRequest(t, h, "AddTagsToStream", map[string]any{ "StreamName": "sorted-tags-target", "Tags": map[string]string{"zebra": "z", "apple": "a", "mango": "m"}, }) - // ListTagsForResource uses stream-level tags; need to check what's stored on the stream. - // Since AddTagsToStream stores in handler-level h.tags (not stream.Tags), - // ListTagsForResource on a fresh stream returns empty tags by design. - // Verify it returns non-nil empty slice instead. rec = doRequest(t, h, "ListTagsForResource", map[string]any{"ResourceARN": arn}) require.Equal(t, http.StatusOK, rec.Code) @@ -473,9 +471,11 @@ func TestRefinement1_ListTagsForResource_SortedOutput(t *testing.T) { } require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - // Tags may be empty (stream.Tags not populated by AddTagsToStream which uses h.tags). - // The key thing is the response is valid JSON and Tags is non-nil. - assert.NotNil(t, resp.Tags) + require.Len(t, resp.Tags, 3) + assert.Equal(t, "apple", resp.Tags[0].Key) + assert.Equal(t, "a", resp.Tags[0].Value) + assert.Equal(t, "mango", resp.Tags[1].Key) + assert.Equal(t, "zebra", resp.Tags[2].Key) } // TestRefinement1_PersistenceRoundTrip verifies Snapshot/Restore preserves state. diff --git a/services/kinesis/handler_refinement2_test.go b/services/kinesis/handler_refinement2_test.go index dd381213a..1bd690567 100644 --- a/services/kinesis/handler_refinement2_test.go +++ b/services/kinesis/handler_refinement2_test.go @@ -963,7 +963,11 @@ func TestRefinement2_AddStreamInternal_DefaultsStreamMode(t *testing.T) { } } -func TestRefinement2_ListTagsForResource_UsesHandlerTags(t *testing.T) { +// TestRefinement2_ListTagsForResource_TagsFromCreate verifies that tags supplied +// inline on CreateStream are visible via ListTagsForResource. Both paths write +// through to the backend's persisted stream.Tags store (see +// TestRefinement2_Tags_SurvivePersistenceRestore for the restart case). +func TestRefinement2_ListTagsForResource_TagsFromCreate(t *testing.T) { t.Parallel() tests := []struct { @@ -1030,6 +1034,125 @@ func TestRefinement2_ListTagsForResource_UsesHandlerTags(t *testing.T) { } } +// TestRefinement2_Tags_SurvivePersistenceRestore proves that stream tags applied +// via every tag-mutating operation (inline on CreateStream, the legacy +// AddTagsToStream API, and the ARN-based TagResource API) are stored on the +// backend's persisted stream.Tags field and therefore survive a +// Snapshot/Restore round-trip. Previously, CreateStream/AddTagsToStream/ +// RemoveTagsFromStream/ListTagsForStream/ListTagsForResource operated on a +// handler-local map that was never included in the persisted snapshot, so +// those tags silently vanished on every restart (a real data-loss bug, not a +// "looks-wrong-but-correct" case) even though the stream itself and its +// shards/records persisted correctly. +func TestRefinement2_Tags_SurvivePersistenceRestore(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + streamName string + apply func(t *testing.T, h *kinesis.Handler, streamName, streamARN string) + wantKey string + wantValue string + }{ + { + name: "tag_from_create", + streamName: "persist-tags-create", + apply: func(t *testing.T, h *kinesis.Handler, streamName, _ string) { + t.Helper() + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": streamName, + "ShardCount": 1, + "Tags": map[string]any{"owner": "create"}, + }) + require.Equal(t, http.StatusOK, rec.Code) + }, + wantKey: "owner", + wantValue: "create", + }, + { + name: "tag_from_add_tags_to_stream", + streamName: "persist-tags-add", + apply: func(t *testing.T, h *kinesis.Handler, streamName, _ string) { + t.Helper() + require.Equal(t, http.StatusOK, doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": streamName, + "ShardCount": 1, + }).Code) + rec := doRequest(t, h, "AddTagsToStream", map[string]any{ + "StreamName": streamName, + "Tags": map[string]string{"owner": "add-tags"}, + }) + require.Equal(t, http.StatusOK, rec.Code) + }, + wantKey: "owner", + wantValue: "add-tags", + }, + { + name: "tag_from_tag_resource", + streamName: "persist-tags-tagresource", + apply: func(t *testing.T, h *kinesis.Handler, streamName, streamARN string) { + t.Helper() + require.Equal(t, http.StatusOK, doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": streamName, + "ShardCount": 1, + }).Code) + rec := doRequest(t, h, "TagResource", map[string]any{ + "ResourceARN": streamARN, + "Tags": map[string]string{"owner": "tag-resource"}, + }) + require.Equal(t, http.StatusOK, rec.Code) + }, + wantKey: "owner", + wantValue: "tag-resource", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + backend := kinesis.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + h := kinesis.NewHandler(backend) + + // TagResource needs the ARN up front; build it the same way the + // backend does so the sub-test can address the stream before it exists. + streamARN := "arn:aws:kinesis:us-east-1:000000000000:stream/" + tt.streamName + tt.apply(t, h, tt.streamName, streamARN) + + // Simulate a process restart: snapshot the backend, then restore + // into a brand-new backend/handler pair (no shared state). + snap := backend.Snapshot(t.Context()) + require.NotNil(t, snap) + + restoredBackend := kinesis.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + require.NoError(t, restoredBackend.Restore(t.Context(), snap)) + restoredHandler := kinesis.NewHandler(restoredBackend) + + rec := doRequest(t, restoredHandler, "ListTagsForResource", map[string]any{ + "ResourceARN": streamARN, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var resp struct { + Tags []struct { + Key string `json:"Key"` + Value string `json:"Value"` + } `json:"Tags"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + + found := false + for _, kv := range resp.Tags { + if kv.Key == tt.wantKey { + assert.Equal(t, tt.wantValue, kv.Value) + found = true + } + } + assert.True(t, found, "expected tag %q to survive persistence restore, got %+v", tt.wantKey, resp.Tags) + }) + } +} + func TestRefinement2_GetSupportedOperations_IncludesUpdateStreamMode(t *testing.T) { t.Parallel() diff --git a/services/kinesis/handler_refinement3_test.go b/services/kinesis/handler_refinement3_test.go index 7ecb8e0cd..9f2b9959b 100644 --- a/services/kinesis/handler_refinement3_test.go +++ b/services/kinesis/handler_refinement3_test.go @@ -1542,6 +1542,77 @@ func TestRefinement3_DescribeStream_UpdateShardCount_IncludesOldClosedShards(t * assert.Len(t, descResp.StreamDescription.Shards, 5, "3 closed + 2 open = 5") } +// TestRefinement3_DescribeStream_ShardPagination verifies that DescribeStream +// paginates its Shards list using Limit/ExclusiveStartShardId/HasMoreShards, +// matching the AWS contract (default page size 100, resumable pagination). +// A stream accumulates CLOSED shards forever across reshards, so a +// long-lived, heavily-resharded stream can exceed a single page — previously +// DescribeStream always returned every shard in one response and hardcoded +// HasMoreShards to false. +func TestRefinement3_DescribeStream_ShardPagination(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + exclusiveStartShardID string + limit int + shardCount int + wantLen int + wantHasMore bool + }{ + { + name: "default_limit_under_page", + shardCount: 5, + wantLen: 5, + wantHasMore: false, + }, + { + name: "explicit_limit_truncates", + shardCount: 5, + limit: 2, + wantLen: 2, + wantHasMore: true, + }, + { + name: "exclusive_start_resumes_after_page", + shardCount: 5, + limit: 2, + exclusiveStartShardID: "shardId-000000000001", // page 1 returned shards 0,1 + wantLen: 2, // shards 2,3 + wantHasMore: true, + }, + { + name: "exclusive_start_reaches_end", + shardCount: 5, + limit: 2, + exclusiveStartShardID: "shardId-000000000003", // shards 4 remains + wantLen: 1, + wantHasMore: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := kinesis.NewInMemoryBackend() + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "paginated-stream", + ShardCount: tt.shardCount, + })) + + out, err := b.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{ + StreamName: "paginated-stream", + Limit: tt.limit, + ExclusiveStartShardID: tt.exclusiveStartShardID, + }) + require.NoError(t, err) + assert.Len(t, out.Shards, tt.wantLen) + assert.Equal(t, tt.wantHasMore, out.HasMoreShards) + }) + } +} + func TestRefinement3_UpdateShardCount_SecondScaleStillWorks(t *testing.T) { t.Parallel() @@ -1874,14 +1945,14 @@ func TestRefinement3_PutRecords_EmptyBatch(t *testing.T) { ShardCount: 1, })) - // Empty records slice. + // AWS rejects an empty Records list outright (MinItems=1 in the SDK model) + // with a validation error — it does not return a 200 with zero results. out, err := b.PutRecords(context.Background(), &kinesis.PutRecordsInput{ StreamName: "empty-batch-stream", Records: []kinesis.PutRecordsEntry{}, }) - require.NoError(t, err) - assert.Equal(t, 0, out.FailedRecordCount) - assert.Empty(t, out.Records) + require.ErrorIs(t, err, kinesis.ErrInvalidArgument) + assert.Nil(t, out) } func TestRefinement3_ListShards_ExclusiveStart_WithMaxResults(t *testing.T) { diff --git a/services/kinesis/handler_test.go b/services/kinesis/handler_test.go index a6795f072..da0f6c6e0 100644 --- a/services/kinesis/handler_test.go +++ b/services/kinesis/handler_test.go @@ -563,15 +563,16 @@ func TestPutRecordsNotFound(t *testing.T) { "StreamName": "nonexistent", "Records": []map[string]any{{"PartitionKey": "pk", "Data": []byte("data")}}, }) - // PutRecords calls PutRecord for each entry, which fails, but the outer PutRecords itself succeeds - // with failed record count set - assert.Equal(t, http.StatusOK, rec.Code) + // AWS fails the whole PutRecords call with a top-level ResourceNotFoundException + // when the target stream does not exist — it does not report per-record + // failures for a request-level (stream-not-found) error. + assert.Equal(t, http.StatusBadRequest, rec.Code) var resp struct { - FailedRecordCount int `json:"FailedRecordCount"` + Type string `json:"__type"` } require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - assert.Equal(t, 1, resp.FailedRecordCount) + assert.Equal(t, "ResourceNotFoundException", resp.Type) } func TestGetShardIteratorBadIteratorType(t *testing.T) { diff --git a/services/kinesis/janitor.go b/services/kinesis/janitor.go index 6c0520878..921fd3518 100644 --- a/services/kinesis/janitor.go +++ b/services/kinesis/janitor.go @@ -76,12 +76,7 @@ func (j *Janitor) sweepRetention(ctx context.Context) { totalTrimmed := 0 j.Backend.mu.RLock("KinesisJanitor") - var streamsToSweep []*Stream - for _, regionStreams := range j.Backend.streams { - for _, stream := range regionStreams { - streamsToSweep = append(streamsToSweep, stream) - } - } + streamsToSweep := j.Backend.streams.All() j.Backend.mu.RUnlock() for _, stream := range streamsToSweep { diff --git a/services/kinesis/persistence.go b/services/kinesis/persistence.go index 30b5ff557..e4d97692c 100644 --- a/services/kinesis/persistence.go +++ b/services/kinesis/persistence.go @@ -3,19 +3,41 @@ package kinesis import ( "context" "encoding/json" + "fmt" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" ) -// backendSnapshot is the persisted form of the backend. Streams and -// ResourcePolicies are nested by region (outer key = region) to match the -// region-isolated in-memory layout. +// kinesisSnapshotVersion identifies the shape of [backendSnapshot]. It must be +// bumped whenever a change to the persisted Stream shape or backendSnapshot +// itself would make an older snapshot unsafe to decode as the current shape. +// Restore discards (rather than partially decodes) any mismatch -- see +// Restore below. This is version 1: the first snapshot format built on top of +// pkgs/store's Table/Registry (the prior format persisted a bare +// region-nested map with no version field at all, so it always decodes as +// Version 0 and is treated as incompatible). +const kinesisSnapshotVersion = 1 + +// backendSnapshot is the persisted form of the backend. +// +// Tables holds one JSON-encoded array per table registered on b.registry -- +// currently just "streams" ([]*Stream), produced by +// [github.com/blackbirdworks/gopherstack/pkgs/store.Registry.SnapshotAll]. +// Stream is persisted directly (no DTO) because every field is already +// JSON-round-trippable except the unexported mu, which json.Marshal skips +// automatically and [initializeStreamRuntime] rebuilds on restore. +// +// ResourcePolicies remains a plain nested map: its value type (a bare string) +// carries no identity of its own to hand a store.Table keyFn, so it is +// persisted the same way it always was. type backendSnapshot struct { - Streams map[string]map[string]*Stream `json:"streams"` - ResourcePolicies map[string]map[string]string `json:"resourcePolicies,omitempty"` - AccountID string `json:"accountID"` - Region string `json:"region"` + Tables map[string]json.RawMessage `json:"tables"` + ResourcePolicies map[string]map[string]string `json:"resourcePolicies,omitempty"` + AccountID string `json:"accountID"` + Region string `json:"region"` + OnDemandStreamCountLimit int `json:"onDemandStreamCountLimit,omitempty"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. @@ -25,11 +47,20 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, "kinesis: snapshot table marshal failed", "error", err) + + return nil + } + snap := backendSnapshot{ - Streams: b.streams, - ResourcePolicies: b.resourcePolicies, - AccountID: b.accountID, - Region: b.region, + Version: kinesisSnapshotVersion, + Tables: tables, + ResourcePolicies: b.resourcePolicies, + AccountID: b.accountID, + Region: b.region, + OnDemandStreamCountLimit: b.onDemandStreamCountLimit, } data, err := json.Marshal(snap) @@ -54,34 +85,45 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() - if snap.Streams == nil { - snap.Streams = make(map[string]map[string]*Stream) - } + if snap.Version != kinesisSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields (e.g. the pre-store nested-map format this + // version replaced). Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition, not + // data corruption. + logger.Load(ctx).WarnContext(ctx, + "kinesis: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", kinesisSnapshotVersion) - if snap.ResourcePolicies == nil { - snap.ResourcePolicies = make(map[string]map[string]string) + b.registry.ResetAll() + b.resourcePolicies = make(map[string]map[string]string) + + return nil } - for region, regionStreams := range snap.Streams { - for name, stream := range regionStreams { - if stream == nil { - delete(regionStreams, name) + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("kinesis: restore snapshot tables: %w", err) + } - continue - } - initializeStreamRuntime(stream, name) - } + for _, stream := range b.streams.All() { + initializeStreamRuntime(stream, stream.Name) + } - if len(regionStreams) == 0 { - delete(snap.Streams, region) - } + if snap.ResourcePolicies == nil { + snap.ResourcePolicies = make(map[string]map[string]string) } - b.streams = snap.Streams b.accountID = snap.AccountID b.region = snap.Region b.resourcePolicies = snap.ResourcePolicies + if snap.OnDemandStreamCountLimit > 0 { + b.onDemandStreamCountLimit = snap.OnDemandStreamCountLimit + } else { + b.onDemandStreamCountLimit = defaultOnDemandStreamCountLimit + } + return nil } diff --git a/services/kinesis/persistence_roundtrip_test.go b/services/kinesis/persistence_roundtrip_test.go new file mode 100644 index 000000000..33b8357d6 --- /dev/null +++ b/services/kinesis/persistence_roundtrip_test.go @@ -0,0 +1,182 @@ +package kinesis //nolint:testpackage // reuses ctxRegion (isolation_test.go) for direct region control. + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestInMemoryBackend_FullStateSnapshotRestoreRoundTrip exercises every kind +// of state the Phase 3.3 pkgs/store conversion touches in a single pass: +// multiple streams across multiple regions, inline shard records, inline +// consumers, tags, resource policies, and the account-level on-demand stream +// count limit. It proves store.Registry.SnapshotAll/RestoreAll round-trips +// the flat streams Table (keyed by region/name, see streamKey) exactly like +// the region-nested map it replaced, and that the raw (non-Table) +// resourcePolicies map still persists alongside it. +func TestInMemoryBackend_FullStateSnapshotRestoreRoundTrip(t *testing.T) { + t.Parallel() + + ctx := context.Background() + ctxEast := ctxRegion("us-east-1") + ctxWest := ctxRegion("us-west-2") + + original := NewInMemoryBackendWithConfig("000000000000", "us-east-1") + + require.NoError(t, original.CreateStream(ctxEast, &CreateStreamInput{ + StreamName: "alpha", + ShardCount: 2, + })) + require.NoError(t, original.CreateStream(ctxWest, &CreateStreamInput{ + StreamName: "beta", + ShardCount: 1, + })) + + // Inline shard records (hot path, stays inline per Stream -- not decomposed). + _, err := original.PutRecord(ctxEast, &PutRecordInput{ + StreamName: "alpha", PartitionKey: "pk", Data: []byte("east-data"), + }) + require.NoError(t, err) + _, err = original.PutRecord(ctxWest, &PutRecordInput{ + StreamName: "beta", PartitionKey: "pk", Data: []byte("west-data"), + }) + require.NoError(t, err) + + alphaDesc, err := original.DescribeStream(ctxEast, &DescribeStreamInput{StreamName: "alpha"}) + require.NoError(t, err) + betaDesc, err := original.DescribeStream(ctxWest, &DescribeStreamInput{StreamName: "beta"}) + require.NoError(t, err) + + // Inline consumers (Stream.Consumers) on the east stream. + _, err = original.RegisterStreamConsumer(ctxEast, &RegisterStreamConsumerInput{ + StreamARN: alphaDesc.StreamARN, ConsumerName: "fanout", + }) + require.NoError(t, err) + + // Tags on the west stream. + require.NoError(t, original.TagResource(ctxWest, &TagResourceInput{ + ResourceARN: betaDesc.StreamARN, + Tags: map[string]string{"env": "test"}, + })) + + // Resource policy (raw, non-Table map) keyed by ARN. + require.NoError(t, original.PutResourcePolicy(ctx, &PutResourcePolicyInput{ + ResourceARN: alphaDesc.StreamARN, + Policy: `{"Version":"2012-10-17"}`, + })) + + // Account-level setting persisted alongside the streams table. + require.NoError(t, original.UpdateAccountSettings(ctx, &UpdateAccountSettingsInput{ + OnDemandStreamCountLimit: 42, + })) + + snap := original.Snapshot(t.Context()) + require.NotNil(t, snap) + + fresh := NewInMemoryBackendWithConfig("000000000000", "us-east-1") + require.NoError(t, fresh.Restore(t.Context(), snap)) + + // Both streams, across both regions, survived under the flat Table. + assert.Equal(t, 2, fresh.StreamCount()) + all := fresh.ListAll(ctx) + assert.Len(t, all, 2) + + // Region isolation preserved: each region's ListStreams sees only its own. + eastList, err := fresh.ListStreams(ctxEast, &ListStreamsInput{}) + require.NoError(t, err) + assert.Equal(t, []string{"alpha"}, eastList.StreamNames) + + westList, err := fresh.ListStreams(ctxWest, &ListStreamsInput{}) + require.NoError(t, err) + assert.Equal(t, []string{"beta"}, westList.StreamNames) + + // Inline shard records survived, keyed to the correct region's stream. + alphaAfter, err := fresh.DescribeStream(ctxEast, &DescribeStreamInput{StreamName: "alpha"}) + require.NoError(t, err) + assert.Len(t, alphaAfter.Shards, 2) + assert.Contains(t, alphaAfter.StreamARN, "us-east-1") + + betaAfter, err := fresh.DescribeStream(ctxWest, &DescribeStreamInput{StreamName: "beta"}) + require.NoError(t, err) + assert.Len(t, betaAfter.Shards, 1) + assert.Contains(t, betaAfter.StreamARN, "us-west-2") + + eastIt, err := fresh.GetShardIterator(ctxEast, &GetShardIteratorInput{ + StreamName: "alpha", ShardID: "shardId-000000000000", ShardIteratorType: iteratorTypeTrimHorizon, + }) + require.NoError(t, err) + eastRecs, err := fresh.GetRecords(ctx, &GetRecordsInput{ShardIterator: eastIt.ShardIterator}) + require.NoError(t, err) + require.Len(t, eastRecs.Records, 1) + assert.Equal(t, []byte("east-data"), eastRecs.Records[0].Data) + + westIt, err := fresh.GetShardIterator(ctxWest, &GetShardIteratorInput{ + StreamName: "beta", ShardID: "shardId-000000000000", ShardIteratorType: iteratorTypeTrimHorizon, + }) + require.NoError(t, err) + westRecs, err := fresh.GetRecords(ctx, &GetRecordsInput{ShardIterator: westIt.ShardIterator}) + require.NoError(t, err) + require.Len(t, westRecs.Records, 1) + assert.Equal(t, []byte("west-data"), westRecs.Records[0].Data) + + // Inline consumer survived (ARN-routed, region-independent of ctx). + consumerOut, err := fresh.DescribeStreamConsumer(ctx, &DescribeStreamConsumerInput{ + StreamARN: alphaDesc.StreamARN, ConsumerName: "fanout", + }) + require.NoError(t, err) + assert.Equal(t, "fanout", consumerOut.ConsumerDescription.ConsumerName) + + // Tags survived. + tagsOut, err := fresh.ListTagsForResource(ctx, &ListTagsForResourceInput{ResourceARN: betaDesc.StreamARN}) + require.NoError(t, err) + assert.Equal(t, "test", tagsOut.Tags["env"]) + + // Resource policy (raw map, not a store.Table) survived. + policyOut, err := fresh.GetResourcePolicy(ctx, &GetResourcePolicyInput{ResourceARN: alphaDesc.StreamARN}) + require.NoError(t, err) + assert.JSONEq(t, `{"Version":"2012-10-17"}`, policyOut.Policy) + + // Account setting persisted alongside the streams table. + settingsOut, err := fresh.DescribeAccountSettings(ctx) + require.NoError(t, err) + assert.Equal(t, 42, settingsOut.OnDemandStreamCountLimit) +} + +// TestInMemoryBackend_Restore_IncompatibleVersion_ResetsEmpty verifies the +// snapshot version guard: a snapshot whose "version" field does not match the +// backend's current kinesisSnapshotVersion is never partially decoded as the +// current shape. Restore must discard it, reset to empty, and return nil +// (not an error) -- this is the same recoverable-condition contract sqs's +// pilot conversion established for its own version guard. +func TestInMemoryBackend_Restore_IncompatibleVersion_ResetsEmpty(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + original := NewInMemoryBackendWithConfig("000000000000", "us-east-1") + require.NoError(t, original.CreateStream(ctx, &CreateStreamInput{StreamName: "s", ShardCount: 1})) + + snap := original.Snapshot(t.Context()) + require.NotNil(t, snap) + + // Corrupt just the version field to simulate a snapshot from an + // incompatible (older/newer) build. + var raw map[string]json.RawMessage + require.NoError(t, json.Unmarshal(snap, &raw)) + bumped, err := json.Marshal(kinesisSnapshotVersion + 1) + require.NoError(t, err) + raw["version"] = bumped + mutated, err := json.Marshal(raw) + require.NoError(t, err) + + fresh := NewInMemoryBackendWithConfig("000000000000", "us-east-1") + require.NoError(t, fresh.Restore(t.Context(), mutated)) + + assert.Equal(t, 0, fresh.StreamCount()) + out, err := fresh.ListStreams(ctx, &ListStreamsInput{}) + require.NoError(t, err) + assert.Empty(t, out.StreamNames) +} diff --git a/services/kinesis/persistence_test.go b/services/kinesis/persistence_test.go index 005db7964..01240f135 100644 --- a/services/kinesis/persistence_test.go +++ b/services/kinesis/persistence_test.go @@ -70,6 +70,55 @@ func TestInMemoryBackend_SnapshotRestore(t *testing.T) { } } +// TestInMemoryBackend_OnDemandStreamCountLimit_SurvivesRestore verifies that +// UpdateAccountSettings' OnDemandStreamCountLimit is part of the persisted +// snapshot and not lost across a restart, and that a snapshot taken before +// any UpdateAccountSettings call restores the AWS default limit. +func TestInMemoryBackend_OnDemandStreamCountLimit_SurvivesRestore(t *testing.T) { + t.Parallel() + + tests := []struct { + configure func(t *testing.T, b *kinesis.InMemoryBackend) + name string + wantLimit int + }{ + { + name: "custom_limit_persists", + configure: func(t *testing.T, b *kinesis.InMemoryBackend) { + t.Helper() + require.NoError(t, b.UpdateAccountSettings(context.Background(), &kinesis.UpdateAccountSettingsInput{ + OnDemandStreamCountLimit: 25, + })) + }, + wantLimit: 25, + }, + { + name: "default_limit_when_unset", + configure: func(_ *testing.T, _ *kinesis.InMemoryBackend) {}, + wantLimit: 10, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + original := kinesis.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + tt.configure(t, original) + + snap := original.Snapshot(t.Context()) + require.NotNil(t, snap) + + restored := kinesis.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + require.NoError(t, restored.Restore(t.Context(), snap)) + + out, err := restored.DescribeAccountSettings(context.Background()) + require.NoError(t, err) + assert.Equal(t, tt.wantLimit, out.OnDemandStreamCountLimit) + }) + } +} + func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { t.Parallel() diff --git a/services/kinesis/types.go b/services/kinesis/types.go index 0b38e687f..6aeabf717 100644 --- a/services/kinesis/types.go +++ b/services/kinesis/types.go @@ -17,9 +17,13 @@ const ( // encryptionTypeNone is the no-encryption type. encryptionTypeNone = "NONE" - // defaultShardCount is the default number of shards for a new stream. + // defaultShardCount is the default number of shards for a new PROVISIONED stream. defaultShardCount = 1 + // defaultOnDemandShardCount is the number of shards AWS allocates to a + // freshly created ON_DEMAND stream (capacity is auto-managed thereafter). + defaultOnDemandShardCount = 4 + // defaultRetentionHours is the default retention period for a stream in hours. defaultRetentionHours = 24 @@ -76,6 +80,10 @@ const ( // consumerStatusActive is the status when a consumer is ready for use. consumerStatusActive = "ACTIVE" + // maxConsumersPerStream is the AWS limit on registered enhanced fan-out + // consumers per stream. + maxConsumersPerStream = 20 + // scalingTypeUniformScaling is the only supported scaling type for UpdateShardCount. scalingTypeUniformScaling = "UNIFORM_SCALING" @@ -107,19 +115,25 @@ const ( // Stream represents an in-memory Kinesis stream. type Stream struct { - CreatedAt time.Time `json:"createdAt"` - mu *lockmetrics.RWMutex - Tags *tags.Tags `json:"tags,omitempty"` - Consumers map[string]*Consumer `json:"consumers,omitempty"` - Name string `json:"name"` - ARN string `json:"arn"` - Status string `json:"status"` - EncryptionType string `json:"encryptionType,omitempty"` - KeyID string `json:"keyId,omitempty"` - StreamMode string `json:"streamMode,omitempty"` - Shards []*Shard `json:"shards"` - EnhancedMonitoring []string `json:"enhancedMonitoring,omitempty"` - RetentionPeriod int `json:"retentionPeriod"` + CreatedAt time.Time `json:"createdAt"` + mu *lockmetrics.RWMutex + Tags *tags.Tags `json:"tags,omitempty"` + Consumers map[string]*Consumer `json:"consumers,omitempty"` + Name string `json:"name"` + ARN string `json:"arn"` + // Region is the AWS region this stream lives in. It is the second half of + // the composite key (see streamKey in backend.go) that keeps same-named + // streams in different regions isolated inside the single flat + // store.Table[Stream] — the region-nested map it replaced used the + // region as an outer map key instead of a field on Stream itself. + Region string `json:"region,omitempty"` + Status string `json:"status"` + EncryptionType string `json:"encryptionType,omitempty"` + KeyID string `json:"keyId,omitempty"` + StreamMode string `json:"streamMode,omitempty"` + Shards []*Shard `json:"shards"` + EnhancedMonitoring []string `json:"enhancedMonitoring,omitempty"` + RetentionPeriod int `json:"retentionPeriod"` // MaxRecordSizeBytes is the per-record data payload size limit for this stream. // Defaults to defaultMaxRecordSizeBytes (1 MiB); updatable via UpdateMaxRecordSize. MaxRecordSizeBytes int `json:"maxRecordSizeBytes,omitempty"` @@ -185,6 +199,11 @@ type DeleteStreamInput struct { // DescribeStreamInput is the input for DescribeStream. type DescribeStreamInput struct { StreamName string + // ExclusiveStartShardID resumes shard pagination after the given shard ID. + ExclusiveStartShardID string + // Limit caps the number of ShardDescription entries returned (AWS default + // 100, max 10000). Zero means "use the AWS default". + Limit int } // DescribeStreamOutput is the output for DescribeStream. @@ -199,6 +218,9 @@ type DescribeStreamOutput struct { Shards []ShardDescription EnhancedMonitoring []string RetentionPeriodHours int + // HasMoreShards indicates the shard list was truncated by Limit and more + // shards can be fetched with a follow-up call using ExclusiveStartShardID. + HasMoreShards bool } // ShardDescription describes a shard in a DescribeStream response. diff --git a/services/kms/PARITY.md b/services/kms/PARITY.md new file mode 100644 index 000000000..fffc24094 --- /dev/null +++ b/services/kms/PARITY.md @@ -0,0 +1,176 @@ +--- +service: kms +sdk_module: aws-sdk-go-v2/service/kms@v1.53.6 +last_audit_commit: ede7169a +last_audit_date: 2026-07-05 +overall: B # already-accurate op-by-op; this pass found and fixed 5 genuine gaps +ops: + CreateKey: {wire: ok, errors: fixed, state: ok, persist: ok, note: "invalid KeySpec now classifies as ValidationException (400), not InternalServiceError (500); tags now validated before the key is created (was: orphan-leak on bad tag)"} + DescribeKey: {wire: ok, errors: ok, state: ok, persist: ok} + ListKeys: {wire: ok, errors: ok, state: ok, persist: ok} + Encrypt: {wire: ok, errors: ok, state: ok, persist: ok, note: "real AES-256-GCM / RSA-OAEP-SHA-256, AAD-bound encryption context, grant-token constraint check already present"} + Decrypt: {wire: ok, errors: ok, state: ok, persist: ok, note: "key ID embedded in blob prefix; mismatched context fails AES-GCM auth -> InvalidCiphertextException; history fallback for post-rotation ciphertexts"} + ReEncrypt: {wire: ok, errors: ok, state: ok, persist: ok} + GenerateDataKey: {wire: ok, errors: ok, state: ok, persist: ok} + GenerateDataKeyWithoutPlaintext: {wire: ok, errors: ok, state: ok, persist: ok} + GenerateDataKeyPair: {wire: fixed, errors: ok, state: ok, persist: ok, note: "GrantTokens field + EncryptionContext size validation + grant-constraint enforcement were all missing; added"} + GenerateDataKeyPairWithoutPlaintext: {wire: fixed, errors: ok, state: ok, persist: ok, note: "delegates to GenerateDataKeyPair; GrantTokens now threaded through"} + Sign: {wire: fixed, errors: ok, state: ok, persist: ok, note: "GrantTokens field + grant-token validity check were missing (disguised stub: op is a valid grant operation but the token was silently dropped)"} + Verify: {wire: fixed, errors: ok, state: ok, persist: ok, note: "same GrantTokens gap as Sign"} + GetPublicKey: {wire: fixed, errors: ok, state: ok, persist: ok, note: "same GrantTokens gap as Sign"} + GenerateMac: {wire: fixed, errors: ok, state: ok, persist: ok, note: "same GrantTokens gap as Sign"} + VerifyMac: {wire: fixed, errors: ok, state: ok, persist: ok, note: "same GrantTokens gap as Sign"} + DeriveSharedSecret: {wire: fixed, errors: ok, state: ok, persist: ok, note: "same GrantTokens gap as Sign; real ECDH via crypto/ecdh conversion"} + GenerateRandom: {wire: ok, errors: ok, state: ok, persist: n/a} + CreateAlias: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateAlias: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteAlias: {wire: ok, errors: ok, state: ok, persist: ok} + ListAliases: {wire: ok, errors: ok, state: ok, persist: ok} + EnableKeyRotation: {wire: ok, errors: ok, state: ok, persist: ok} + DisableKeyRotation: {wire: ok, errors: ok, state: ok, persist: ok} + GetKeyRotationStatus: {wire: ok, errors: ok, state: ok, persist: ok} + RotateKeyOnDemand: {wire: ok, errors: ok, state: ok, persist: ok, note: "10-per-24h on-demand rate limit enforced"} + ListKeyRotations: {wire: ok, errors: ok, state: ok, persist: ok} + DisableKey: {wire: ok, errors: ok, state: ok, persist: ok} + EnableKey: {wire: ok, errors: ok, state: ok, persist: ok} + ScheduleKeyDeletion: {wire: ok, errors: ok, state: ok, persist: ok, note: "7-30 day window enforced; janitor purges past DeletionDate"} + CancelKeyDeletion: {wire: ok, errors: ok, state: ok, persist: ok} + CreateGrant: {wire: ok, errors: ok, state: ok, persist: ok} + ListGrants: {wire: ok, errors: ok, state: ok, persist: ok} + RevokeGrant: {wire: ok, errors: ok, state: ok, persist: ok} + RetireGrant: {wire: ok, errors: ok, state: ok, persist: ok} + ListRetirableGrants: {wire: ok, errors: ok, state: ok, persist: ok} + PutKeyPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + GetKeyPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + ListKeyPolicies: {wire: ok, errors: ok, state: ok, persist: n/a} + GetParametersForImport: {wire: ok, errors: ok, state: ok, persist: ok, note: "real RSA-2048/3072/4096 wrapping keypair generated per call"} + ImportKeyMaterial: {wire: ok, errors: ok, state: ok, persist: ok, note: "real RSA-OAEP unwrap of caller material"} + DeleteImportedKeyMaterial: {wire: ok, errors: ok, state: ok, persist: ok} + ReplicateKey: {wire: fixed, errors: ok, state: ok, persist: ok, note: "tag validation moved before replica creation (was: orphan-leak on bad tag, and tags on ReplicateKey bypassed validateTag entirely)"} + UpdateKeyDescription: {wire: ok, errors: ok, state: ok, persist: ok} + UpdatePrimaryRegion: {wire: ok, errors: ok, state: ok, persist: ok} + CreateCustomKeyStore: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteCustomKeyStore: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeCustomKeyStores: {wire: ok, errors: ok, state: ok, persist: ok} + ConnectCustomKeyStore: {wire: ok, errors: ok, state: ok, persist: ok} + DisconnectCustomKeyStore: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateCustomKeyStore: {wire: ok, errors: ok, state: ok, persist: ok} + GetKeyLastUsage: {wire: ok, errors: ok, state: ok, persist: n/a, note: "not a real AWS KMS op; internal telemetry accessor kept from a prior pass"} + TagResource: {wire: ok, errors: ok, state: ok, persist: n/a, note: "tags stored via pkgs/tags, not in backendSnapshot (see gaps)"} + UntagResource: {wire: ok, errors: ok, state: ok, persist: n/a} + ListResourceTags: {wire: ok, errors: ok, state: ok, persist: n/a} +families: + crypto_core: {status: ok, note: "AES-256-GCM (real Seal/Open, AAD = keyID + sorted encryption-context pairs), RSA-OAEP-SHA-256/SHA-1 fallback, RSASSA-PSS/PKCS1v15, ECDSA P-256/384/521, ECDH (crypto/ecdh), HMAC-SHA-256/384/512 — all real crypto/*, no mock byte-flipping anywhere in crypto.go"} + key_state_machine: {status: ok, note: "Enabled/Disabled/PendingDeletion/PendingImport transitions all gated; keyStateError() maps Disabled->DisabledException, everything else->KMSInvalidStateException"} + multi_region: {status: ok, note: "ReplicateKey/UpdatePrimaryRegion primary<->replica promotion verified by existing TestUpdatePrimaryRegion_RoleSwap; DescribeKey MultiRegionConfiguration built correctly for both primary and replica sides"} +gaps: + - "GrantConstraints has no SourceArn field (real SDK: GrantConstraints.SourceArn); no operation in this mock threads a caller/resource ARN through crypto calls to check against it, and no other service adapter currently supplies one either — deferred, needs cross-cutting request-context plumbing, not a KMS-local fix (bd: gopherstack-w3k)" + - "CreateGrantInput has no GrantTokens field (real SDK: authorizes the CreateGrant call itself via an existing not-yet-consistent grant). No IAM/authorization layer exists anywhere in this mock, so this field would currently be a no-op; deferred, consistent with the rest of the codebase's scope" + - "GranteeServicePrincipal / RetiringServicePrincipal (AWS-service grantees) not modeled on CreateGrantInput; same no-IAM-layer scope reasoning as above" +deferred: + - Custom key store cryptographic connection/HSM simulation (ConnectCustomKeyStore is a pure state-machine transition; no CloudHSM cluster or XKS proxy is modeled, matching pre-existing scope) + - GetKeyLastUsage (not a real AWS KMS operation; left as-is from a prior pass, out of scope for this sweep) +leaks: {status: found, note: "janitor.purgeKey deleted a purged key's grants from the canonical grants/grantsByToken maps but left the grantsByKey[region][keyID] secondary-index submap allocated forever (unreachable after purge, since the keyID can never resolve again) — fixed by dropping the submap in purgeKey. Verified by Test_KMS_Janitor_PurgeKey_CleansGrantByKeyIndex, which fails without the fix. All other maps already bounded (keyMaterialHistory capped at 100 entries/key, janitor sweeps PendingDeletion keys, resolution cache cleared via evictAliasesFromCache)."} +--- + +## Notes + +Freeform findings from the 2026-07-05 sweep (bd: gopherstack-42s), for the next auditor. + +### Fixed this pass + +1. **Grant-token wire gap on 7 operations (severe, disguised stub).** `SignInput`, + `VerifyInput`, `GetPublicKeyInput`, `DeriveSharedSecretInput`, `GenerateDataKeyPairInput`, + `GenerateDataKeyPairWithoutPlaintextInput`, `GenerateMacInput`, and `VerifyMacInput` had no + `GrantTokens` field at all, even though `isValidGrantOperation` in backend.go already lists + `Sign`, `Verify`, `GetPublicKey`, `GenerateMac`, `VerifyMac`, `DeriveSharedSecret`, + `GenerateDataKeyPair`, and `GenerateDataKeyPairWithoutPlaintext` as grantable operations. + Since dispatch does a bare `json.Unmarshal(body, &input)`, a caller-supplied `GrantTokens` + array was silently dropped on the floor — the grant system modeled these operations as + grantable but never actually validated a grant token for them. Confirmed against + `aws-sdk-go-v2/service/kms` that all 8 real `*Input` structs carry `GrantTokens []string`. + Fixed by adding the field to all 8 structs and wiring validation into the backend: + - Per AWS docs (`kms/types.GrantConstraints` doc comment), `EncryptionContextEquals`/ + `EncryptionContextSubset` constraints apply ONLY to operations that support an + encryption context (Encrypt, Decrypt, GenerateDataKey(WithoutPlaintext), + GenerateDataKeyPair(WithoutPlaintext), ReEncryptFrom/To) — NOT to Sign, Verify, + GetPublicKey, GenerateMac, VerifyMac, or DeriveSharedSecret. So the fix adds two + different helpers: `validateGrantTokenPresence` (token must exist + be unexpired; no + constraint check) for the six non-context ops, and reuses the existing + `validateGrantTokenConstraints` (token + TTL + EncryptionContext match) for + GenerateDataKeyPair(WithoutPlaintext), matching the pattern already used by + Encrypt/Decrypt/GenerateDataKey/ReEncrypt. + - **Trap for the next auditor:** do not "simplify" by reusing + `validateGrantTokenConstraints` with a nil encryption context for Sign/Verify/etc. — if + a grant happens to have `EncryptionContextEquals` set (unusual but not rejected at + CreateGrant time), that would spuriously reject Sign/Verify calls that AWS would allow, + since AWS never evaluates that constraint for those operations. + - Also added `validateEncryptionContextSize` to `GenerateDataKeyPair`, which was missing + it (Encrypt/Decrypt/GenerateDataKey/ReEncrypt already had it). + +2. **Invalid KeySpec/KeyPairSpec misclassified as 500, not 400 (moderate).** + `generateKeyMaterial`'s `default:` case returned an error wrapping only + `errUnsupportedKeySpec`, never `ErrValidation`. `classifyKMSError` only matches via + `errors.Is` against the sentinels in `kmsErrorTable()`, so `CreateKey` with a garbage + `KeySpec` (e.g. a typo) and `GenerateDataKeyPair` with a garbage `KeyPairSpec` both fell + through to the default `InternalServiceError` / 500 branch instead of + `ValidationException` / 400 — exactly bug class #2 from `parity-principles.md` + ("missing errCodeLookup entries"). Fixed with a single-point fix: wrap with both + `ErrValidation` and `errUnsupportedKeySpec` (Go 1.20+ multi-`%w`) at the source, so every + current and future caller of `generateKeyMaterial` gets correct classification for free. + Proven with an HTTP-level test (`Test_KMS_InvalidKeySpec_Returns400ValidationException`) + that exercises the full `Handler().Handler()` echo path and checks both status code and + `ErrorResponse.Type`. + +3. **`purgeKey` leaked the `grantsByKey` secondary-index submap (moderate, matches the + "unbounded key/grant maps" pattern called out in the audit brief).** When the janitor + permanently purges a key past its `ScheduleKeyDeletion` window, it deleted the key's + grants from `grants` and `grantsByToken` but never removed + `grantsByKey[region][keyID]`. Since the purged keyID can never be looked up again, that + submap (and any residual per-grant map beneath it) is unreachable for the remainder of + the process's lifetime — a genuine memory leak in any long-running instance that + repeatedly creates keys with grants and lets them expire. Fixed by dropping the submap in + `purgeKey`; `grantsByKeyStore` lazily recreates it on next access if the key ID is ever + reused (it won't be, since key IDs are UUIDs, but the accessor is already written to + tolerate that). Note `rebuildGrantIndexesLocked` (used by `Restore`) already rebuilds + `grantsByKey` from scratch, confirming it's a pure derived index safe to drop. + +4. **`CreateKey` and `ReplicateKey` created a real, permanent resource before validating + tags (moderate, orphan-resource leak).** `createKeyAction` called + `Backend.CreateKey` (allocating a UUID, real key material, and a backend map entry) + and only validated `input.Tags` afterward via `applyInputTags`. If a tag was malformed + (empty key, reserved `aws:` prefix, over-length), the handler returned an error to the + caller — who never receives a `KeyId` — while the key remained permanently resident in + the backend, discoverable only via `ListKeys`, with no route to ever tag, use, or delete + it by the caller that "created" it. `ReplicateKey`'s dispatch closure was worse: it never + validated tags at all (`copyTagsToReplica` calls `setTags` directly, bypassing + `validateTag` entirely), so a malformed replica tag would just silently apply. Real AWS + validates the whole request shape before creating any resource. Fixed by extracting a + shared `validateTags` helper and calling it before `Backend.CreateKey` / + `Backend.ReplicateKey` in both dispatch paths (the `ReplicateKey` closure was also + extracted to `replicateKeyAction` to keep `buildReplicationAndMaintenanceActions` under + the gocognit complexity gate). + +### Traps / already-correct patterns confirmed (do not re-flag) + +- `Decrypt`/`ReEncrypt` returning an error via `decryptWithHistory` after the primary + `decryptData` attempt fails is NOT a stub — it's the real post-rotation fallback path that + tries prior key-material versions (capped at `maxKeyMaterialHistoryEntries` = 100) before + giving up with `InvalidCiphertextException`. +- `GetKeyLastUsage` is not a real AWS KMS API (AWS doesn't expose per-key last-usage via a + named operation); it was added in an earlier pass as an internal telemetry accessor and is + kept as deferred/non-blocking scope. +- `errCodeLookup`-equivalent (`kmsErrorTable()` + `classifyKMSError`) returns HTTP 400 for + every matched sentinel including `NotFoundException` — this matches real AWS KMS + (a `json-1.1` protocol service), which returns 400 Bad Request for `NotFoundException` + too, not 404. +- `CreateGrant`'s own `GrantTokens` field (authorizing the CreateGrant call via an existing + grant) is intentionally NOT modeled — there is no IAM/authorization layer anywhere in this + mock, so it would be a no-op; this is a scope boundary, not a bug. + +### Verification method + +Every fix in this pass was proven with a negative control: the corresponding fix commit was +reverted locally, the new test was confirmed to fail (or fail to compile, for the wire-shape +field additions) against the reverted code, then the fix was reapplied and the full suite +re-run green. See `parity_sweep3_test.go`. diff --git a/services/kms/backend.go b/services/kms/backend.go index f78874e23..19da80c0c 100644 --- a/services/kms/backend.go +++ b/services/kms/backend.go @@ -21,6 +21,7 @@ import ( "github.com/google/uuid" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" gopherarn "github.com/blackbirdworks/gopherstack/pkgs/arn" ) @@ -299,26 +300,43 @@ type StorageBackend interface { // ensure InMemoryBackend satisfies StorageBackend at compile time. var _ StorageBackend = (*InMemoryBackend)(nil) +// grantRegionStore bundles a region's canonical grants [store.Table] with its +// two secondary indexes. Table.Put/Delete keep both indexes consistent +// automatically, which is what lets RevokeGrant/RetireGrant/janitor purge drop +// a grant with a single table.Delete call instead of the three-map manual +// bookkeeping (grants/grantsByToken/grantsByKey) this replaces. +type grantRegionStore struct { + table *store.Table[Grant] + // byToken indexes grants by GrantToken for O(1) lookup on the + // encrypt/decrypt grant-validation hot path (findGrantByToken). + byToken *store.Index[Grant] + // byKey indexes grants by KeyID for O(1) ListGrants and grant-count checks + // on the CreateGrant hot path. + byKey *store.Index[Grant] +} + // InMemoryBackend is a concurrency-safe in-memory KMS backend. type InMemoryBackend struct { - keys map[string]map[string]*Key - aliases map[string]map[string]*Alias - grants map[string]map[string]*Grant - // grantsByToken indexes grants by their GrantToken for O(1) lookup on the - // encrypt/decrypt grant-validation hot path. Kept consistent with grants on - // every create/revoke/retire. - grantsByToken map[string]map[string]*Grant - // grantsByKey indexes grants by keyID for O(1) ListGrants and grant-count - // checks on the CreateGrant hot path. Kept consistent with grants on every - // create/revoke/retire. - grantsByKey map[string]map[string]map[string]*Grant + // keys, aliases, and customKeyStores hold one [store.Table] per AWS + // region, created lazily by keysStore/aliasesStore/customKeyStoresStore + // exactly as the plain per-region maps they replace were. grants holds one + // [grantRegionStore] per region instead, bundling the canonical grants + // table with its two secondary indexes -- see grantRegionStore. + keys map[string]*store.Table[Key] + aliases map[string]*store.Table[Alias] + grants map[string]*grantRegionStore policies map[string]map[string]string keyMaterials map[string]map[string]*keyMaterial keyMaterialHistory map[string]map[string][]*keyMaterial - customKeyStores map[string]map[string]*CustomKeyStore - mu *lockmetrics.RWMutex - accountID string - defaultRegion string + customKeyStores map[string]*store.Table[CustomKeyStore] + // registry accumulates every per-region table registered by + // keysStore/aliasesStore/grantsRegion/customKeyStoresStore over the + // backend's lifetime, so Snapshot/Restore/Reset collapse to one + // store.Registry call each regardless of how many regions are in use. + registry *store.Registry + mu *lockmetrics.RWMutex + accountID string + defaultRegion string // keyIDResolutionCache maps alias names and ARNs to resolved key UUIDs to avoid // repeated aliasesStore lookups on hot paths. Stored as a pointer so clearResolutionCache // can swap it in O(1) instead of iterating all entries. @@ -340,15 +358,14 @@ func NewInMemoryBackend() *InMemoryBackend { // NewInMemoryBackendWithConfig creates a new KMS backend with the given account ID and region. func NewInMemoryBackendWithConfig(accountID, region string) *InMemoryBackend { return &InMemoryBackend{ - keys: make(map[string]map[string]*Key), - aliases: make(map[string]map[string]*Alias), - grants: make(map[string]map[string]*Grant), - grantsByToken: make(map[string]map[string]*Grant), - grantsByKey: make(map[string]map[string]map[string]*Grant), + keys: make(map[string]*store.Table[Key]), + aliases: make(map[string]*store.Table[Alias]), + grants: make(map[string]*grantRegionStore), policies: make(map[string]map[string]string), keyMaterials: make(map[string]map[string]*keyMaterial), keyMaterialHistory: make(map[string]map[string][]*keyMaterial), - customKeyStores: make(map[string]map[string]*CustomKeyStore), + customKeyStores: make(map[string]*store.Table[CustomKeyStore]), + registry: store.NewRegistry(), accountID: accountID, defaultRegion: region, mu: lockmetrics.New("kms"), @@ -356,68 +373,56 @@ func NewInMemoryBackendWithConfig(accountID, region string) *InMemoryBackend { } } -// keysStore returns (creating lazily) the per-region keys map. -func (b *InMemoryBackend) keysStore(region string) map[string]*Key { - if m, ok := b.keys[region]; ok { - return m +// keysStore returns (registering lazily) the per-region keys table. +func (b *InMemoryBackend) keysStore(region string) *store.Table[Key] { + if t, ok := b.keys[region]; ok { + return t } - m := make(map[string]*Key) - b.keys[region] = m + t := store.Register(b.registry, "keys:"+region, store.New(func(k *Key) string { return k.KeyID })) + b.keys[region] = t - return m + return t } -// aliasesStore returns (creating lazily) the per-region aliases map. -func (b *InMemoryBackend) aliasesStore(region string) map[string]*Alias { - if m, ok := b.aliases[region]; ok { - return m +// aliasesStore returns (registering lazily) the per-region aliases table. +func (b *InMemoryBackend) aliasesStore(region string) *store.Table[Alias] { + if t, ok := b.aliases[region]; ok { + return t } - m := make(map[string]*Alias) - b.aliases[region] = m + t := store.Register(b.registry, "aliases:"+region, store.New(func(a *Alias) string { return a.AliasName })) + b.aliases[region] = t - return m + return t } -// grantsStore returns (creating lazily) the per-region grants map. -func (b *InMemoryBackend) grantsStore(region string) map[string]*Grant { - if m, ok := b.grants[region]; ok { - return m +// grantsRegion returns (registering lazily) the per-region grant store, which +// bundles the canonical grants table with its byToken/byKey indexes. +func (b *InMemoryBackend) grantsRegion(region string) *grantRegionStore { + if g, ok := b.grants[region]; ok { + return g } - m := make(map[string]*Grant) - b.grants[region] = m - - return m -} - -// grantsByTokenStore returns (creating lazily) the per-region grantsByToken map. -func (b *InMemoryBackend) grantsByTokenStore(region string) map[string]*Grant { - if m, ok := b.grantsByToken[region]; ok { - return m + t := store.Register(b.registry, "grants:"+region, store.New(func(g *Grant) string { return g.GrantID })) + g := &grantRegionStore{ + table: t, + byToken: t.AddIndex("byToken", func(g *Grant) string { return g.GrantToken }), + byKey: t.AddIndex("byKey", func(g *Grant) string { return g.KeyID }), } + b.grants[region] = g - m := make(map[string]*Grant) - b.grantsByToken[region] = m - - return m + return g } -// grantsByKeyStore returns (creating lazily) the per-key grants map for a region. -func (b *InMemoryBackend) grantsByKeyStore(region, keyID string) map[string]*Grant { - if b.grantsByKey[region] == nil { - b.grantsByKey[region] = make(map[string]map[string]*Grant) - } - - if b.grantsByKey[region][keyID] == nil { - b.grantsByKey[region][keyID] = make(map[string]*Grant) - } - - return b.grantsByKey[region][keyID] +// grantsStore returns (registering lazily) the per-region canonical grants table. +func (b *InMemoryBackend) grantsStore(region string) *store.Table[Grant] { + return b.grantsRegion(region).table } // policiesStore returns (creating lazily) the per-region policies map. +// Policies remain a plain map: a policy is a bare JSON string with no identity +// field of its own to derive a store.Table key function from. func (b *InMemoryBackend) policiesStore(region string) map[string]string { if m, ok := b.policies[region]; ok { return m @@ -430,6 +435,9 @@ func (b *InMemoryBackend) policiesStore(region string) map[string]string { } // keyMaterialsStore returns (creating lazily) the per-region keyMaterials map. +// keyMaterial carries no identity field of its own (it is pure crypto +// material -- see crypto.go), so it remains a plain map keyed externally by +// keyID, exactly as before. func (b *InMemoryBackend) keyMaterialsStore(region string) map[string]*keyMaterial { if m, ok := b.keyMaterials[region]; ok { return m @@ -441,7 +449,9 @@ func (b *InMemoryBackend) keyMaterialsStore(region string) map[string]*keyMateri return m } -// keyMaterialHistoryStore returns (creating lazily) the per-region keyMaterialHistory map. +// keyMaterialHistoryStore returns (creating lazily) the per-region +// keyMaterialHistory map. The value is a slice of history entries rather than +// a single identity-bearing struct, so it remains a plain map. func (b *InMemoryBackend) keyMaterialHistoryStore(region string) map[string][]*keyMaterial { if m, ok := b.keyMaterialHistory[region]; ok { return m @@ -453,16 +463,20 @@ func (b *InMemoryBackend) keyMaterialHistoryStore(region string) map[string][]*k return m } -// customKeyStoresStore returns (creating lazily) the per-region customKeyStores map. -func (b *InMemoryBackend) customKeyStoresStore(region string) map[string]*CustomKeyStore { - if m, ok := b.customKeyStores[region]; ok { - return m +// customKeyStoresStore returns (registering lazily) the per-region customKeyStores table. +func (b *InMemoryBackend) customKeyStoresStore(region string) *store.Table[CustomKeyStore] { + if t, ok := b.customKeyStores[region]; ok { + return t } - m := make(map[string]*CustomKeyStore) - b.customKeyStores[region] = m + t := store.Register( + b.registry, + "customKeyStores:"+region, + store.New(func(cs *CustomKeyStore) string { return cs.CustomKeyStoreID }), + ) + b.customKeyStores[region] = t - return m + return t } // resolveKeyID resolves an alias name or ARN to a plain key UUID and region. @@ -483,7 +497,7 @@ func (b *InMemoryBackend) resolveKeyID( } if strings.HasPrefix(keyID, "alias/") { - alias, ok := b.aliasesStore(ctxRegion)[keyID] + alias, ok := b.aliasesStore(ctxRegion).Get(keyID) if !ok { return "", "", ErrAliasNotFound } @@ -514,7 +528,7 @@ func (b *InMemoryBackend) resolveARNKeyID(keyID string) (string, string, error) } if strings.HasPrefix(parsed.Resource, "alias/") { - alias, ok := b.aliasesStore(parsed.Region)[parsed.Resource] + alias, ok := b.aliasesStore(parsed.Region).Get(parsed.Resource) if !ok { return "", "", ErrAliasNotFound } @@ -541,9 +555,9 @@ func (b *InMemoryBackend) clearResolutionCache() { // re-validates the alias against the live store instead of serving a stale hit. // Must be called with the write lock held. func (b *InMemoryBackend) evictAliasesFromCache(region, keyID string) { - for aliasName, alias := range b.aliasesStore(region) { + for _, alias := range b.aliasesStore(region).All() { if alias.TargetKeyID == keyID { - b.keyIDResolutionCache.Delete(aliasName) + b.keyIDResolutionCache.Delete(alias.AliasName) } } } @@ -773,7 +787,7 @@ func (b *InMemoryBackend) CreateKey( b.keyMaterialsStore(region)[keyID] = km } - b.keysStore(region)[keyID] = key + b.keysStore(region).Put(key) out := &CreateKeyOutput{ KeyMetadata: keyToMetadata(key), @@ -810,9 +824,9 @@ func (b *InMemoryBackend) ListKeys( defer b.mu.RUnlock() region := getRegion(ctx, b.defaultRegion) - entries := make([]KeyListEntry, 0, len(b.keysStore(region))) + entries := make([]KeyListEntry, 0, b.keysStore(region).Len()) - for _, k := range b.keysStore(region) { + for _, k := range b.keysStore(region).All() { entries = append( entries, KeyListEntry{KeyID: k.KeyID, KeyArn: k.Arn, Description: k.Description}, @@ -1358,6 +1372,10 @@ func (b *InMemoryBackend) Sign(ctx context.Context, input *SignInput) (*SignOutp return nil, algErr } + if err = b.validateGrantTokenPresence(input.GrantTokens); err != nil { + return nil, err + } + km, err := b.requireKeyMaterial(region, key.KeyID) if err != nil { return nil, err @@ -1423,6 +1441,10 @@ func (b *InMemoryBackend) Verify(ctx context.Context, input *VerifyInput) (*Veri return nil, algErr } + if err = b.validateGrantTokenPresence(input.GrantTokens); err != nil { + return nil, err + } + km, err := b.requireKeyMaterial(region, key.KeyID) if err != nil { return nil, err @@ -1491,6 +1513,10 @@ func (b *InMemoryBackend) GetPublicKey( ) } + if err = b.validateGrantTokenPresence(input.GrantTokens); err != nil { + return nil, err + } + km, err := b.requireKeyMaterial(region, key.KeyID) if err != nil { return nil, err @@ -1553,7 +1579,7 @@ func (b *InMemoryBackend) CreateAlias(ctx context.Context, input *CreateAliasInp region := getRegion(ctx, b.defaultRegion) - if _, exists := b.aliasesStore(region)[input.AliasName]; exists { + if b.aliasesStore(region).Has(input.AliasName) { return ErrAliasAlreadyExists } @@ -1562,19 +1588,19 @@ func (b *InMemoryBackend) CreateAlias(ctx context.Context, input *CreateAliasInp return err } - if _, exists := b.keysStore(region)[targetID]; !exists { + if !b.keysStore(region).Has(targetID) { return ErrKeyNotFound } now := UnixTimeFloat(time.Now()) aliasArn := gopherarn.Build("kms", region, b.accountID, input.AliasName) - b.aliasesStore(region)[input.AliasName] = &Alias{ + b.aliasesStore(region).Put(&Alias{ AliasName: input.AliasName, AliasArn: aliasArn, TargetKeyID: targetID, CreationDate: now, LastUpdatedDate: now, - } + }) return nil } @@ -1587,7 +1613,7 @@ func (b *InMemoryBackend) UpdateAlias(ctx context.Context, input *UpdateAliasInp region := getRegion(ctx, b.defaultRegion) - alias, exists := b.aliasesStore(region)[input.AliasName] + alias, exists := b.aliasesStore(region).Get(input.AliasName) if !exists { return ErrAliasNotFound } @@ -1597,7 +1623,7 @@ func (b *InMemoryBackend) UpdateAlias(ctx context.Context, input *UpdateAliasInp return err } - targetKey, ok := b.keysStore(region)[targetID] + targetKey, ok := b.keysStore(region).Get(targetID) if !ok { return ErrKeyNotFound } @@ -1626,14 +1652,14 @@ func (b *InMemoryBackend) DeleteAlias(ctx context.Context, input *DeleteAliasInp region := getRegion(ctx, b.defaultRegion) - alias, exists := b.aliasesStore(region)[input.AliasName] + alias, exists := b.aliasesStore(region).Get(input.AliasName) if !exists { return ErrAliasNotFound } // Prevent deleting an alias that targets a key scheduled for deletion. if alias.TargetKeyID != "" { - if key, ok := b.keysStore(region)[alias.TargetKeyID]; ok && + if key, ok := b.keysStore(region).Get(alias.TargetKeyID); ok && key.KeyState == KeyStatePendingDeletion { return fmt.Errorf( "%w: key %s is pending deletion; cancel the deletion before deleting the alias", @@ -1642,7 +1668,7 @@ func (b *InMemoryBackend) DeleteAlias(ctx context.Context, input *DeleteAliasInp } } - delete(b.aliasesStore(region), input.AliasName) + b.aliasesStore(region).Delete(input.AliasName) b.keyIDResolutionCache.Delete(input.AliasName) return nil @@ -1669,9 +1695,9 @@ func (b *InMemoryBackend) ListAliases( } } - aliases := make([]Alias, 0, len(b.aliasesStore(region))) + aliases := make([]Alias, 0, b.aliasesStore(region).Len()) - for _, a := range b.aliasesStore(region) { + for _, a := range b.aliasesStore(region).All() { if resolvedKeyID != "" && a.TargetKeyID != resolvedKeyID { continue } @@ -2079,7 +2105,7 @@ func (b *InMemoryBackend) lookupKey(ctx context.Context, keyID string) (*Key, er return nil, err } - key, ok := b.keysStore(region)[resolved] + key, ok := b.keysStore(region).Get(resolved) if !ok { // For plain UUID lookups (not ARN or alias), fall back to searching all regions. // This preserves mock compatibility: multi-region tests create replicas in a target @@ -2309,8 +2335,8 @@ func (b *InMemoryBackend) buildReplicaMultiRegionConfig(key *Key) *MultiRegionCo // findKeyInAnyRegion searches all region stores for a key with the given keyID. // Must be called with at least a read lock held. func (b *InMemoryBackend) findKeyInAnyRegion(keyID string) *Key { - for _, regionKeys := range b.keys { - if k, ok := regionKeys[keyID]; ok { + for _, t := range b.keys { + if k, ok := t.Get(keyID); ok { return k } } @@ -2321,8 +2347,8 @@ func (b *InMemoryBackend) findKeyInAnyRegion(keyID string) *Key { // findPrimaryKeyForReplica locates the primary key that lists replicaKey.KeyID in its // ReplicaKeyIDs. Must be called with at least a read lock held. func (b *InMemoryBackend) findPrimaryKeyForReplica(replicaKey *Key) *Key { - for _, regionKeys := range b.keys { - for _, k := range regionKeys { + for _, t := range b.keys { + for _, k := range t.All() { if !k.MultiRegion || extractRegionFromARN(k.Arn) != replicaKey.PrimaryRegion { continue } @@ -2433,7 +2459,7 @@ func (b *InMemoryBackend) CreateGrant( return nil, err } - key, ok := b.keysStore(region)[keyID] + key, ok := b.keysStore(region).Get(keyID) if !ok { return nil, ErrKeyNotFound } @@ -2442,7 +2468,9 @@ func (b *InMemoryBackend) CreateGrant( return nil, keyStateError(key) } - if len(b.grantsByKeyStore(region, keyID)) >= maxGrantsPerKey { + grantsForKey := b.grantsRegion(region) + + if len(grantsForKey.byKey.Get(keyID)) >= maxGrantsPerKey { return nil, fmt.Errorf( "%w: grant limit of %d exceeded for key %q", ErrLimitExceeded, @@ -2466,9 +2494,8 @@ func (b *InMemoryBackend) CreateGrant( Constraints: input.Constraints, CreationDate: UnixTimeFloat(now), } - b.grantsStore(region)[grantID] = grant - b.grantsByTokenStore(region)[grantToken] = grant - b.grantsByKeyStore(region, keyID)[grantID] = grant + // A single Put keeps the byToken and byKey indexes consistent automatically. + grantsForKey.table.Put(grant) return &CreateGrantOutput{GrantID: grantID, GrantToken: grantToken}, nil } @@ -2504,10 +2531,10 @@ func grantConstraintsSatisfied(c *GrantConstraints, encCtx map[string]string) bo // findGrantByToken returns the first grant whose GrantToken matches any of the provided tokens. // Searches all regions. Must be called with at least a read lock held. func (b *InMemoryBackend) findGrantByToken(grantTokens []string) *Grant { - for _, regionMap := range b.grantsByToken { + for _, gs := range b.grants { for _, token := range grantTokens { - if g, ok := regionMap[token]; ok { - return g + if matches := gs.byToken.Get(token); len(matches) > 0 { + return matches[0] } } } @@ -2547,6 +2574,30 @@ func (b *InMemoryBackend) validateGrantTokenConstraints( return nil } +// validateGrantTokenPresence checks that, if grant tokens are provided, at least one +// resolves to an existing, non-expired grant. Unlike validateGrantTokenConstraints, +// it does not evaluate EncryptionContext-based grant constraints: per AWS KMS docs, +// EncryptionContextEquals/EncryptionContextSubset constraints apply only to operations +// that support an encryption context. Sign, Verify, GetPublicKey, GenerateMac, VerifyMac, +// and DeriveSharedSecret do not, so only grant-token validity (existence + TTL) is checked. +// Must be called with at least a read lock held. +func (b *InMemoryBackend) validateGrantTokenPresence(grantTokens []string) error { + if len(grantTokens) == 0 { + return nil + } + + grant := b.findGrantByToken(grantTokens) + if grant == nil { + return fmt.Errorf("%w: grant token not found", ErrInvalidGrantToken) + } + + if !grant.TokenIssuedAt.IsZero() && time.Since(grant.TokenIssuedAt) > grantTokenTTL { + return fmt.Errorf("%w: grant token has expired", ErrInvalidGrantToken) + } + + return nil +} + // ListGrants returns the grants for a specified key with optional pagination and GrantId filter. func (b *InMemoryBackend) ListGrants( ctx context.Context, @@ -2562,14 +2613,14 @@ func (b *InMemoryBackend) ListGrants( return nil, err } - if _, ok := b.keysStore(region)[keyID]; !ok { + if !b.keysStore(region).Has(keyID) { return nil, ErrKeyNotFound } var grants []Grant - for grantID, g := range b.grantsByKey[region][keyID] { + for _, g := range b.grantsRegion(region).byKey.Get(keyID) { // Filter by GrantId if specified. - if input.GrantID != "" && grantID != input.GrantID { + if input.GrantID != "" && g.GrantID != input.GrantID { continue } grants = append(grants, *g) @@ -2616,20 +2667,18 @@ func (b *InMemoryBackend) RevokeGrant(ctx context.Context, input *RevokeGrantInp return err } - if _, ok := b.keysStore(region)[keyID]; !ok { + if !b.keysStore(region).Has(keyID) { return ErrKeyNotFound } - grant, ok := b.grantsStore(region)[input.GrantID] + grant, ok := b.grantsStore(region).Get(input.GrantID) if !ok || grant.KeyID != keyID { return ErrGrantNotFound } - delete(b.grantsStore(region), input.GrantID) - delete(b.grantsByTokenStore(region), grant.GrantToken) - if rm := b.grantsByKey[region]; rm != nil { - delete(rm[grant.KeyID], input.GrantID) - } + // A single Delete keeps the byToken and byKey indexes consistent + // automatically (including dropping now-empty index groups on purge). + b.grantsStore(region).Delete(input.GrantID) return nil } @@ -2643,13 +2692,10 @@ func (b *InMemoryBackend) RetireGrant(ctx context.Context, input *RetireGrantInp if input.GrantToken != "" { // Search all regions for the grant token. - for r, regionMap := range b.grantsByToken { - if g, ok := regionMap[input.GrantToken]; ok { - delete(b.grantsStore(r), g.GrantID) - delete(regionMap, input.GrantToken) - if rm := b.grantsByKey[r]; rm != nil { - delete(rm[g.KeyID], g.GrantID) - } + for _, gs := range b.grants { + if matches := gs.byToken.Get(input.GrantToken); len(matches) > 0 { + g := matches[0] + gs.table.Delete(g.GrantID) return nil } @@ -2662,7 +2708,7 @@ func (b *InMemoryBackend) RetireGrant(ctx context.Context, input *RetireGrantInp return ErrGrantNotFound } - grant, ok := b.grantsStore(region)[input.GrantID] + grant, ok := b.grantsStore(region).Get(input.GrantID) if !ok { return ErrGrantNotFound } @@ -2678,11 +2724,8 @@ func (b *InMemoryBackend) RetireGrant(ctx context.Context, input *RetireGrantInp } } - delete(b.grantsStore(region), input.GrantID) - delete(b.grantsByTokenStore(region), grant.GrantToken) - if rm := b.grantsByKey[region]; rm != nil { - delete(rm[grant.KeyID], input.GrantID) - } + // A single Delete keeps the byToken and byKey indexes consistent automatically. + b.grantsStore(region).Delete(input.GrantID) return nil } @@ -2698,7 +2741,7 @@ func (b *InMemoryBackend) ListRetirableGrants( region := getRegion(ctx, b.defaultRegion) grants := make([]Grant, 0) - for _, g := range b.grantsStore(region) { + for _, g := range b.grantsStore(region).All() { if g.RetiringPrincipal == input.RetiringPrincipal { grants = append(grants, *g) } @@ -2786,7 +2829,7 @@ func (b *InMemoryBackend) PutKeyPolicy(ctx context.Context, input *PutKeyPolicyI return err } - if _, ok := b.keysStore(region)[keyID]; !ok { + if !b.keysStore(region).Has(keyID) { return ErrKeyNotFound } @@ -2821,7 +2864,7 @@ func (b *InMemoryBackend) GetKeyPolicy( return nil, err } - if _, ok := b.keysStore(region)[keyID]; !ok { + if !b.keysStore(region).Has(keyID) { return nil, ErrKeyNotFound } @@ -3274,7 +3317,7 @@ func (b *InMemoryBackend) ReplicateKey( } // Store replica key in the target region's store. - b.keysStore(input.ReplicaRegion)[replica.KeyID] = replica + b.keysStore(input.ReplicaRegion).Put(replica) // Record the replica key ID on the source (primary) key so DescribeKey can // return the full MultiRegionConfiguration. @@ -3402,15 +3445,17 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.keys = make(map[string]map[string]*Key) - b.aliases = make(map[string]map[string]*Alias) - b.grants = make(map[string]map[string]*Grant) - b.grantsByToken = make(map[string]map[string]*Grant) - b.grantsByKey = make(map[string]map[string]map[string]*Grant) + // registry.ResetAll empties every already-registered per-region keys/ + // aliases/grants/customKeyStores table in place. It deliberately does NOT + // unregister them: b.keys/b.aliases/b.grants/b.customKeyStores keep + // pointing at the same *store.Table/*grantRegionStore instances, so a + // region touched again after Reset reuses its existing registration + // instead of re-registering under an already-used name (which would + // panic -- see store.Register). + b.registry.ResetAll() b.policies = make(map[string]map[string]string) b.keyMaterials = make(map[string]map[string]*keyMaterial) b.keyMaterialHistory = make(map[string]map[string][]*keyMaterial) - b.customKeyStores = make(map[string]map[string]*CustomKeyStore) b.clearResolutionCache() } @@ -3440,7 +3485,7 @@ func (b *InMemoryBackend) CreateCustomKeyStore( region := getRegion(ctx, b.defaultRegion) // Ensure name is unique. - for _, ks := range b.customKeyStoresStore(region) { + for _, ks := range b.customKeyStoresStore(region).All() { if ks.CustomKeyStoreName == input.CustomKeyStoreName { return nil, fmt.Errorf( "%w: custom key store with name %q already exists", @@ -3451,13 +3496,13 @@ func (b *InMemoryBackend) CreateCustomKeyStore( storeID := uuid.New().String() - b.customKeyStoresStore(region)[storeID] = &CustomKeyStore{ + b.customKeyStoresStore(region).Put(&CustomKeyStore{ CustomKeyStoreID: storeID, CustomKeyStoreName: input.CustomKeyStoreName, ConnectionState: ConnectionStateDisconnected, CreationDate: UnixTimeFloat(time.Now()), CustomKeyStoreType: storeType, - } + }) return &CreateCustomKeyStoreOutput{CustomKeyStoreID: storeID}, nil } @@ -3476,7 +3521,7 @@ func (b *InMemoryBackend) DeleteCustomKeyStore( region := getRegion(ctx, b.defaultRegion) - ks, ok := b.customKeyStoresStore(region)[input.CustomKeyStoreID] + ks, ok := b.customKeyStoresStore(region).Get(input.CustomKeyStoreID) if !ok { return fmt.Errorf( "%w: custom key store %q not found", @@ -3492,7 +3537,7 @@ func (b *InMemoryBackend) DeleteCustomKeyStore( ) } - delete(b.customKeyStoresStore(region), input.CustomKeyStoreID) + b.customKeyStoresStore(region).Delete(input.CustomKeyStoreID) return nil } @@ -3506,9 +3551,9 @@ func (b *InMemoryBackend) DescribeCustomKeyStores( region := getRegion(ctx, b.defaultRegion) - stores := make([]CustomKeyStore, 0, len(b.customKeyStoresStore(region))) + stores := make([]CustomKeyStore, 0, b.customKeyStoresStore(region).Len()) - for _, ks := range b.customKeyStoresStore(region) { + for _, ks := range b.customKeyStoresStore(region).All() { if input.CustomKeyStoreID != "" && ks.CustomKeyStoreID != input.CustomKeyStoreID { continue } @@ -3565,7 +3610,7 @@ func (b *InMemoryBackend) ConnectCustomKeyStore( region := getRegion(ctx, b.defaultRegion) - ks, ok := b.customKeyStoresStore(region)[input.CustomKeyStoreID] + ks, ok := b.customKeyStoresStore(region).Get(input.CustomKeyStoreID) if !ok { return fmt.Errorf( "%w: custom key store %q not found", @@ -3600,7 +3645,7 @@ func (b *InMemoryBackend) DisconnectCustomKeyStore( region := getRegion(ctx, b.defaultRegion) - ks, ok := b.customKeyStoresStore(region)[input.CustomKeyStoreID] + ks, ok := b.customKeyStoresStore(region).Get(input.CustomKeyStoreID) if !ok { return fmt.Errorf( "%w: custom key store %q not found", @@ -3635,7 +3680,7 @@ func (b *InMemoryBackend) UpdateCustomKeyStore( region := getRegion(ctx, b.defaultRegion) - ks, ok := b.customKeyStoresStore(region)[input.CustomKeyStoreID] + ks, ok := b.customKeyStoresStore(region).Get(input.CustomKeyStoreID) if !ok { return fmt.Errorf( "%w: custom key store %q not found", @@ -3645,7 +3690,7 @@ func (b *InMemoryBackend) UpdateCustomKeyStore( } if input.NewCustomKeyStoreName != "" && input.NewCustomKeyStoreName != ks.CustomKeyStoreName { - for _, existing := range b.customKeyStoresStore(region) { + for _, existing := range b.customKeyStoresStore(region).All() { if existing.CustomKeyStoreName == input.NewCustomKeyStoreName { return fmt.Errorf( "%w: custom key store with name %q already exists", @@ -3698,6 +3743,10 @@ func (b *InMemoryBackend) DeriveSharedSecret( ) } + if err = b.validateGrantTokenPresence(input.GrantTokens); err != nil { + return nil, err + } + km, err := b.requireKeyMaterial(region, key.KeyID) if err != nil { return nil, err @@ -3732,6 +3781,10 @@ func (b *InMemoryBackend) GenerateDataKeyPair( return nil, fmt.Errorf("%w: KeyPairSpec must not be empty", ErrValidation) } + if err := validateEncryptionContextSize(input.EncryptionContext); err != nil { + return nil, err + } + b.mu.RLock("GenerateDataKeyPair") defer b.mu.RUnlock() @@ -3754,6 +3807,10 @@ func (b *InMemoryBackend) GenerateDataKeyPair( ) } + if err = b.validateGrantTokenConstraints(ctx, input.GrantTokens, input.EncryptionContext); err != nil { + return nil, err + } + wrapKM, err := b.requireKeyMaterial(region, wrapKey.KeyID) if err != nil { return nil, err @@ -3799,6 +3856,7 @@ func (b *InMemoryBackend) GenerateDataKeyPairWithoutPlaintext( KeyID: input.KeyID, KeyPairSpec: input.KeyPairSpec, EncryptionContext: input.EncryptionContext, + GrantTokens: input.GrantTokens, }) if err != nil { return nil, err @@ -3852,6 +3910,10 @@ func (b *InMemoryBackend) GenerateMac( return nil, algErr } + if err = b.validateGrantTokenPresence(input.GrantTokens); err != nil { + return nil, err + } + km, err := b.requireKeyMaterial(region, key.KeyID) if err != nil { return nil, err @@ -3933,6 +3995,10 @@ func (b *InMemoryBackend) VerifyMac( return nil, algErr } + if err = b.validateGrantTokenPresence(input.GrantTokens); err != nil { + return nil, err + } + km, err := b.requireKeyMaterial(region, key.KeyID) if err != nil { return nil, err @@ -4009,7 +4075,7 @@ func (b *InMemoryBackend) AddKeyInternal(key *Key, km *keyMaterial) { region = b.defaultRegion } - b.keysStore(region)[key.KeyID] = key + b.keysStore(region).Put(key) if km != nil { b.keyMaterialsStore(region)[key.KeyID] = km @@ -4022,5 +4088,5 @@ func (b *InMemoryBackend) AddCustomKeyStoreInternal(ks *CustomKeyStore) { b.mu.Lock("AddCustomKeyStoreInternal") defer b.mu.Unlock() - b.customKeyStoresStore(b.defaultRegion)[ks.CustomKeyStoreID] = ks + b.customKeyStoresStore(b.defaultRegion).Put(ks) } diff --git a/services/kms/crypto.go b/services/kms/crypto.go index 5427e8b0d..bee99d4ef 100644 --- a/services/kms/crypto.go +++ b/services/kms/crypto.go @@ -116,7 +116,12 @@ func generateKeyMaterial(keySpec string) (*keyMaterial, error) { case keySpecHMAC512: return generateHMACKeyMaterial(hmac512Bytes) default: - return nil, fmt.Errorf("%w: %s", errUnsupportedKeySpec, keySpec) + // Wrap with ErrValidation (in addition to errUnsupportedKeySpec) so an + // unrecognized KeySpec/KeyPairSpec classifies as ValidationException (400) + // rather than falling through classifyKMSError's default InternalServiceError + // (500). CreateKey and GenerateDataKeyPair both route unvalidated spec + // strings here. + return nil, fmt.Errorf("%w: %w: %s", ErrValidation, errUnsupportedKeySpec, keySpec) } } diff --git a/services/kms/export_test.go b/services/kms/export_test.go index 9bbf66699..3fa6f961f 100644 --- a/services/kms/export_test.go +++ b/services/kms/export_test.go @@ -3,6 +3,7 @@ package kms import ( "errors" "fmt" + "slices" "time" ) @@ -29,8 +30,8 @@ func KeyCount(b *InMemoryBackend) int { defer b.mu.RUnlock() n := 0 - for _, m := range b.keys { - n += len(m) + for _, t := range b.keys { + n += t.Len() } return n @@ -42,8 +43,8 @@ func AliasCount(b *InMemoryBackend) int { defer b.mu.RUnlock() n := 0 - for _, m := range b.aliases { - n += len(m) + for _, t := range b.aliases { + n += t.Len() } return n @@ -55,8 +56,8 @@ func GrantCount(b *InMemoryBackend) int { defer b.mu.RUnlock() n := 0 - for _, m := range b.grants { - n += len(m) + for _, gs := range b.grants { + n += gs.table.Len() } return n @@ -68,8 +69,8 @@ func CustomKeyStoreCount(b *InMemoryBackend) int { defer b.mu.RUnlock() n := 0 - for _, m := range b.customKeyStores { - n += len(m) + for _, t := range b.customKeyStores { + n += t.Len() } return n @@ -81,8 +82,8 @@ func (b *InMemoryBackend) SetDeletionDateForTest(keyID string, t time.Time) { b.mu.Lock("SetDeletionDateForTest") defer b.mu.Unlock() - for _, regionKeys := range b.keys { - if key, ok := regionKeys[keyID]; ok { + for _, tbl := range b.keys { + if key, ok := tbl.Get(keyID); ok { key.DeletionDate = UnixTimeFloat(t) return @@ -128,8 +129,8 @@ func (b *InMemoryBackend) SetGrantTokenIssuedAt(grantID string, t time.Time) { b.mu.Lock("SetGrantTokenIssuedAt") defer b.mu.Unlock() - for _, regionGrants := range b.grants { - if g, ok := regionGrants[grantID]; ok { + for _, gs := range b.grants { + if g, ok := gs.table.Get(grantID); ok { g.TokenIssuedAt = t return @@ -160,34 +161,37 @@ func GrantsByKeyCount(b *InMemoryBackend, region, keyID string) int { b.mu.RLock("GrantsByKeyCount") defer b.mu.RUnlock() - if rm := b.grantsByKey[region]; rm != nil { - return len(rm[keyID]) + if gs, ok := b.grants[region]; ok { + return len(gs.byKey.Get(keyID)) } return 0 } -// GrantIndexesConsistent checks that grantsByToken and grantsByKey are consistent -// with the canonical grants map. Returns false and a description if not. +// GrantIndexesConsistent checks that the byToken and byKey secondary indexes on +// each region's grants table are consistent with the canonical grants table. +// store.Table/store.Index maintain both automatically on every Put/Delete, so +// this is a structural sanity check rather than the manual three-map +// consistency check it replaces. Returns false and a description if not. func GrantIndexesConsistent(b *InMemoryBackend) (bool, string) { b.mu.RLock("GrantIndexesConsistent") defer b.mu.RUnlock() - for region, regionGrants := range b.grants { - for grantID, g := range regionGrants { - if _, ok := b.grantsByToken[region][g.GrantToken]; !ok { + for region, gs := range b.grants { + for _, g := range gs.table.All() { + tokenMatches := gs.byToken.Get(g.GrantToken) + if !slices.ContainsFunc(tokenMatches, func(m *Grant) bool { return m.GrantID == g.GrantID }) { return false, fmt.Sprintf( - "grant %s token %s missing from grantsByToken[%s]", - grantID, + "grant %s token %s missing from byToken index[%s]", + g.GrantID, g.GrantToken, region, ) } - if rm := b.grantsByKey[region]; rm == nil { - return false, fmt.Sprintf("grantsByKey[%s] is nil", region) - } else if _, ok := rm[g.KeyID][grantID]; !ok { - return false, fmt.Sprintf("grant %s missing from grantsByKey[%s][%s]", grantID, region, g.KeyID) + keyMatches := gs.byKey.Get(g.KeyID) + if !slices.ContainsFunc(keyMatches, func(m *Grant) bool { return m.GrantID == g.GrantID }) { + return false, fmt.Sprintf("grant %s missing from byKey index[%s][%s]", g.GrantID, region, g.KeyID) } } } @@ -227,8 +231,8 @@ func (b *InMemoryBackend) SetKeyCreationDateForTest(keyID string, t time.Time) { b.mu.Lock("SetKeyCreationDateForTest") defer b.mu.Unlock() - for _, regionKeys := range b.keys { - if key, ok := regionKeys[keyID]; ok { + for _, tbl := range b.keys { + if key, ok := tbl.Get(keyID); ok { key.CreationDate = UnixTimeFloat(t) return @@ -245,8 +249,8 @@ func (b *InMemoryBackend) ForceRotateForTest(keyID string) error { b.mu.Lock("ForceRotateForTest") defer b.mu.Unlock() - for region, regionKeys := range b.keys { - if key, ok := regionKeys[keyID]; ok { + for region, tbl := range b.keys { + if key, ok := tbl.Get(keyID); ok { return b.rotateKeyMaterialLocked(region, key, rotationTypeAWSKMS) } } diff --git a/services/kms/handler.go b/services/kms/handler.go index 06e257f0d..decf0f7dc 100644 --- a/services/kms/handler.go +++ b/services/kms/handler.go @@ -164,19 +164,35 @@ func (h *Handler) copyTagsToReplica(sourceKeyID, replicaKeyID string, inputTags } } +// validateTags checks every tag in inputTags against validateTag, returning the first +// failure. Callers should invoke this BEFORE creating any backend resource so that a +// malformed tag rejects the whole request instead of leaving an orphaned, untagged +// resource behind (e.g. a KMS key with no reachable KeyId in the caller's response). +func validateTags(inputTags []Tag) error { + for _, t := range inputTags { + if err := validateTag(t.TagKey, t.TagValue); err != nil { + return err + } + } + + return nil +} + // applyInputTags converts a []Tag slice to a map and stores it for the given resource ID. -// Returns an error if any tag key/value fails validation. +// Returns an error if any tag key/value fails validation. Callers that create a new +// resource in the same request should call validateTags first and only create the +// resource after validation succeeds; see createKeyAction. func (h *Handler) applyInputTags(resourceID string, inputTags []Tag) error { if len(inputTags) == 0 { return nil } + if err := validateTags(inputTags); err != nil { + return err + } + kv := make(map[string]string, len(inputTags)) for _, t := range inputTags { - if err := validateTag(t.TagKey, t.TagValue); err != nil { - return err - } - kv[t.TagKey] = t.TagValue } @@ -413,12 +429,20 @@ func unmarshalAction[T any](fn func(context.Context, *T) (any, error)) kmsAction } // createKeyAction handles CreateKey dispatch, including tag validation. +// Tags are validated BEFORE the key is created: AWS validates the entire CreateKey +// request atomically, and creating the key first would leak an orphaned, untagged +// key (with no reachable KeyId, since the response is discarded on error) into the +// backend whenever the caller supplied a malformed tag. func (h *Handler) createKeyAction(ctx context.Context, b []byte) (any, error) { var input CreateKeyInput if err := json.Unmarshal(b, &input); err != nil { return nil, err } + if err := validateTags(input.Tags); err != nil { + return nil, err + } + out, err := h.Backend.CreateKey(ctx, &input) if err != nil { return nil, err @@ -431,6 +455,37 @@ func (h *Handler) createKeyAction(ctx context.Context, b []byte) (any, error) { return out, nil } +// replicateKeyAction handles ReplicateKey dispatch, including tag validation and +// copying the source key's tags (overlaid with any request-supplied tags) to the +// new replica. Tags are validated BEFORE replicating for the same reason as +// createKeyAction: a malformed tag must reject the whole request rather than +// leaving a real, untagged replica key behind. +func (h *Handler) replicateKeyAction(ctx context.Context, b []byte) (any, error) { + var input ReplicateKeyInput + if err := json.Unmarshal(b, &input); err != nil { + return nil, err + } + + if err := validateTags(input.Tags); err != nil { + return nil, err + } + + // Capture source key ID for tag copying before we replicate. + var sourceKeyID string + if desc, descErr := h.Backend.DescribeKey(ctx, &DescribeKeyInput{KeyID: input.KeyID}); descErr == nil { + sourceKeyID = desc.KeyMetadata.KeyID + } + + out, err := h.Backend.ReplicateKey(ctx, &input) + if err != nil { + return nil, err + } + + h.copyTagsToReplica(sourceKeyID, out.ReplicaKeyMetadata.KeyID, input.Tags) + + return out, nil +} + // buildCryptoActions returns dispatch entries for encrypt, decrypt, sign, verify, and data-key operations. func (h *Handler) buildCryptoActions() map[string]kmsActionFn { return map[string]kmsActionFn{ @@ -950,27 +1005,7 @@ func (h *Handler) buildReplicationAndMaintenanceActions() map[string]kmsActionFn return h.Backend.ListKeyRotations(ctx, &input) }, - "ReplicateKey": func(ctx context.Context, b []byte) (any, error) { - var input ReplicateKeyInput - if err := json.Unmarshal(b, &input); err != nil { - return nil, err - } - - // Capture source key ID for tag copying before we replicate. - var sourceKeyID string - if desc, descErr := h.Backend.DescribeKey(ctx, &DescribeKeyInput{KeyID: input.KeyID}); descErr == nil { - sourceKeyID = desc.KeyMetadata.KeyID - } - - out, err := h.Backend.ReplicateKey(ctx, &input) - if err != nil { - return nil, err - } - - h.copyTagsToReplica(sourceKeyID, out.ReplicaKeyMetadata.KeyID, input.Tags) - - return out, nil - }, + "ReplicateKey": h.replicateKeyAction, "RotateKeyOnDemand": func(ctx context.Context, b []byte) (any, error) { var input RotateKeyOnDemandInput if err := json.Unmarshal(b, &input); err != nil { diff --git a/services/kms/janitor.go b/services/kms/janitor.go index dfe0a8fc0..22b16c037 100644 --- a/services/kms/janitor.go +++ b/services/kms/janitor.go @@ -160,7 +160,7 @@ func (j *Janitor) sweepFromHeap(now float64) (int, int) { continue } - key, ok := j.Backend.keysStore(e.region)[e.keyID] + key, ok := j.Backend.keysStore(e.region).Get(e.keyID) if !ok { continue // already purged } @@ -192,8 +192,9 @@ func (j *Janitor) sweepFromHeap(now float64) (int, int) { // Must be called with the backend write lock held. func (j *Janitor) sweepKeys(now float64) (int, int) { var purged, expired int - for region, regionKeys := range j.Backend.keys { - for keyID, key := range regionKeys { + for region, t := range j.Backend.keys { + for _, key := range t.All() { + keyID := key.KeyID if key.KeyState == KeyStatePendingDeletion { if key.DeletionDate != 0 && now >= key.DeletionDate { j.purgeKey(region, keyID) @@ -219,21 +220,24 @@ func (j *Janitor) purgeKey(region, keyID string) { delete(j.Backend.keyMaterialsStore(region), keyID) delete(j.Backend.keyMaterialHistoryStore(region), keyID) - for aliasName, alias := range j.Backend.aliasesStore(region) { + for _, alias := range j.Backend.aliasesStore(region).All() { if alias.TargetKeyID == keyID { - j.Backend.keyIDResolutionCache.Delete(aliasName) - delete(j.Backend.aliasesStore(region), aliasName) + j.Backend.keyIDResolutionCache.Delete(alias.AliasName) + j.Backend.aliasesStore(region).Delete(alias.AliasName) } } - for grantID, grant := range j.Backend.grantsStore(region) { + // table.Delete keeps the byToken and byKey indexes consistent + // automatically, including dropping the now-empty byKey group for keyID + // outright (see store.Index.remove) -- this keyID can never be looked up + // again after a permanent purge, so no separate index cleanup is needed. + for _, grant := range j.Backend.grantsStore(region).All() { if grant.KeyID == keyID { - delete(j.Backend.grantsStore(region), grantID) - delete(j.Backend.grantsByTokenStore(region), grant.GrantToken) + j.Backend.grantsStore(region).Delete(grant.GrantID) } } - delete(j.Backend.keysStore(region), keyID) + j.Backend.keysStore(region).Delete(keyID) delete(j.Backend.policiesStore(region), keyID) j.Backend.lastUsage.Delete(region + ":" + keyID) } @@ -263,8 +267,9 @@ func (j *Janitor) expireMaterial(region, keyID string, key *Key) { func (j *Janitor) sweepAutoRotations(now float64) int { rotated := 0 - for region, regionKeys := range j.Backend.keys { - for keyID, key := range regionKeys { + for region, t := range j.Backend.keys { + for _, key := range t.All() { + keyID := key.KeyID if !key.RotationEnabled || key.KeyState != KeyStateEnabled || key.Origin == KeyOriginExternal || diff --git a/services/kms/models.go b/services/kms/models.go index 94e2420e4..71e2afee7 100644 --- a/services/kms/models.go +++ b/services/kms/models.go @@ -502,10 +502,11 @@ type GetKeyPolicyOutput struct { // SignInput is the request payload for Sign. type SignInput struct { - KeyID string `json:"KeyId"` - MessageType string `json:"MessageType,omitempty"` - SigningAlgorithm string `json:"SigningAlgorithm"` - Message []byte `json:"Message"` + KeyID string `json:"KeyId"` + MessageType string `json:"MessageType,omitempty"` + SigningAlgorithm string `json:"SigningAlgorithm"` + Message []byte `json:"Message"` + GrantTokens []string `json:"GrantTokens,omitempty"` } // SignOutput is the response payload for Sign. @@ -517,11 +518,12 @@ type SignOutput struct { // VerifyInput is the request payload for Verify. type VerifyInput struct { - KeyID string `json:"KeyId"` - MessageType string `json:"MessageType,omitempty"` - SigningAlgorithm string `json:"SigningAlgorithm"` - Message []byte `json:"Message"` - Signature []byte `json:"Signature"` + KeyID string `json:"KeyId"` + MessageType string `json:"MessageType,omitempty"` + SigningAlgorithm string `json:"SigningAlgorithm"` + Message []byte `json:"Message"` + Signature []byte `json:"Signature"` + GrantTokens []string `json:"GrantTokens,omitempty"` } // VerifyOutput is the response payload for Verify. @@ -535,6 +537,8 @@ type VerifyOutput struct { type GetPublicKeyInput struct { // KeyId identifies the asymmetric KMS key whose public key to retrieve. KeyID string `json:"KeyId"` + // GrantTokens is an optional list of grant tokens used to authorize the operation. + GrantTokens []string `json:"GrantTokens,omitempty"` } // GetPublicKeyOutput is the response payload for GetPublicKey. @@ -762,6 +766,8 @@ type DeriveSharedSecretInput struct { KeyAgreementAlgorithm string `json:"KeyAgreementAlgorithm"` // PublicKey is the DER-encoded public key of the other party. PublicKey []byte `json:"PublicKey"` + // GrantTokens is an optional list of grant tokens used to authorize the operation. + GrantTokens []string `json:"GrantTokens,omitempty"` } // DeriveSharedSecretOutput is the response payload for DeriveSharedSecret. @@ -779,6 +785,8 @@ type GenerateDataKeyPairInput struct { KeyID string `json:"KeyId"` // KeyPairSpec specifies the asymmetric key spec (e.g. RSA_2048, ECC_NIST_P256). KeyPairSpec string `json:"KeyPairSpec"` + // GrantTokens is an optional list of grant tokens used to authorize the operation. + GrantTokens []string `json:"GrantTokens,omitempty"` } // GenerateDataKeyPairOutput is the response payload for GenerateDataKeyPair. @@ -803,6 +811,8 @@ type GenerateDataKeyPairWithoutPlaintextInput struct { KeyID string `json:"KeyId"` // KeyPairSpec specifies the asymmetric key spec (e.g. RSA_2048, ECC_NIST_P256). KeyPairSpec string `json:"KeyPairSpec"` + // GrantTokens is an optional list of grant tokens used to authorize the operation. + GrantTokens []string `json:"GrantTokens,omitempty"` } // GenerateDataKeyPairWithoutPlaintextOutput is the response payload for GenerateDataKeyPairWithoutPlaintext. @@ -825,6 +835,8 @@ type GenerateMacInput struct { MacAlgorithm string `json:"MacAlgorithm"` // Message is the data over which to compute the MAC. Message []byte `json:"Message"` + // GrantTokens is an optional list of grant tokens used to authorize the operation. + GrantTokens []string `json:"GrantTokens,omitempty"` } // GenerateMacOutput is the response payload for GenerateMac. @@ -856,6 +868,8 @@ type VerifyMacInput struct { Message []byte `json:"Message"` // Mac is the MAC tag to verify. Mac []byte `json:"Mac"` + // GrantTokens is an optional list of grant tokens used to authorize the operation. + GrantTokens []string `json:"GrantTokens,omitempty"` } // VerifyMacOutput is the response payload for VerifyMac. diff --git a/services/kms/parity_sweep3_test.go b/services/kms/parity_sweep3_test.go new file mode 100644 index 000000000..a8dae6852 --- /dev/null +++ b/services/kms/parity_sweep3_test.go @@ -0,0 +1,451 @@ +package kms_test + +// Parity sweep 3 (bd: gopherstack-42s) fixes verified here: +// - Sign, Verify, GetPublicKey, GenerateMac, VerifyMac, and DeriveSharedSecret silently +// dropped GrantTokens on the wire (no struct field) and never validated them, even +// though all six are valid CreateGrant operations. A bogus/expired grant token was +// accepted instead of rejected with InvalidGrantTokenException. +// - GenerateDataKeyPair (and its WithoutPlaintext variant) accepted GrantTokens and +// EncryptionContext on the wire but never enforced the grant's EncryptionContext +// constraints, unlike Encrypt/Decrypt/GenerateDataKey/ReEncrypt. +// - An unrecognized KeySpec/KeyPairSpec (CreateKey, GenerateDataKeyPair) classified as +// 500 InternalServiceError instead of 400 ValidationException, because the sentinel +// error was never wrapped with ErrValidation. +// - The KMS janitor's purgeKey permanently deleted a key without cleaning up its +// grantsByKey secondary-index submap, leaking memory for the life of the process. + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/kms" +) + +// Test_GrantTokenValidation_NonContextCryptoOps verifies that Sign, Verify, +// GetPublicKey, GenerateMac, VerifyMac, and DeriveSharedSecret each accept a valid +// grant token, reject a bogus one with InvalidGrantTokenException, and behave +// normally with no token at all. Before this fix, GrantTokens was not even a field +// on these input structs, so a caller-supplied token was silently discarded and +// never validated (a disguised stub: the grant system models these operations as +// grantable, but grant validity was never enforced for them). +func Test_GrantTokenValidation_NonContextCryptoOps(t *testing.T) { + t.Parallel() + + type grantScenario struct { + wantErr error + name string + token string + } + + scenarios := []grantScenario{ + {name: "no_grant_token_succeeds", token: "none"}, + {name: "valid_grant_token_succeeds", token: "valid"}, + {name: "bogus_grant_token_rejected", token: "bogus", wantErr: kms.ErrInvalidGrantToken}, + } + + type opCase struct { + run func(t *testing.T, b *kms.InMemoryBackend, ctx context.Context, tokens []string) error + name string + } + + opCases := []opCase{ + { + name: "Sign", + run: func(t *testing.T, b *kms.InMemoryBackend, ctx context.Context, tokens []string) error { + t.Helper() + + keyOut, err := b.CreateKey(ctx, &kms.CreateKeyInput{ + KeyUsage: kms.KeyUsageSignVerify, + KeySpec: "ECC_NIST_P256", + }) + require.NoError(t, err) + + _, err = b.Sign(ctx, &kms.SignInput{ + KeyID: keyOut.KeyMetadata.KeyID, + Message: []byte("sign me"), + SigningAlgorithm: "ECDSA_SHA_256", + GrantTokens: tokens, + }) + + return err + }, + }, + { + name: "Verify", + run: func(t *testing.T, b *kms.InMemoryBackend, ctx context.Context, tokens []string) error { + t.Helper() + + keyOut, err := b.CreateKey(ctx, &kms.CreateKeyInput{ + KeyUsage: kms.KeyUsageSignVerify, + KeySpec: "ECC_NIST_P256", + }) + require.NoError(t, err) + + signOut, err := b.Sign(ctx, &kms.SignInput{ + KeyID: keyOut.KeyMetadata.KeyID, + Message: []byte("verify me"), + SigningAlgorithm: "ECDSA_SHA_256", + }) + require.NoError(t, err) + + _, err = b.Verify(ctx, &kms.VerifyInput{ + KeyID: keyOut.KeyMetadata.KeyID, + Message: []byte("verify me"), + Signature: signOut.Signature, + SigningAlgorithm: "ECDSA_SHA_256", + GrantTokens: tokens, + }) + + return err + }, + }, + { + name: "GetPublicKey", + run: func(t *testing.T, b *kms.InMemoryBackend, ctx context.Context, tokens []string) error { + t.Helper() + + keyOut, err := b.CreateKey(ctx, &kms.CreateKeyInput{ + KeyUsage: kms.KeyUsageSignVerify, + KeySpec: "ECC_NIST_P256", + }) + require.NoError(t, err) + + _, err = b.GetPublicKey(ctx, &kms.GetPublicKeyInput{ + KeyID: keyOut.KeyMetadata.KeyID, + GrantTokens: tokens, + }) + + return err + }, + }, + { + name: "GenerateMac", + run: func(t *testing.T, b *kms.InMemoryBackend, ctx context.Context, tokens []string) error { + t.Helper() + + keyOut, err := b.CreateKey(ctx, &kms.CreateKeyInput{ + KeyUsage: kms.KeyUsageGenerateMac, + KeySpec: "HMAC_256", + }) + require.NoError(t, err) + + _, err = b.GenerateMac(ctx, &kms.GenerateMacInput{ + KeyID: keyOut.KeyMetadata.KeyID, + Message: []byte("mac me"), + MacAlgorithm: "HMAC_SHA_256", + GrantTokens: tokens, + }) + + return err + }, + }, + { + name: "VerifyMac", + run: func(t *testing.T, b *kms.InMemoryBackend, ctx context.Context, tokens []string) error { + t.Helper() + + keyOut, err := b.CreateKey(ctx, &kms.CreateKeyInput{ + KeyUsage: kms.KeyUsageGenerateMac, + KeySpec: "HMAC_256", + }) + require.NoError(t, err) + + macOut, err := b.GenerateMac(ctx, &kms.GenerateMacInput{ + KeyID: keyOut.KeyMetadata.KeyID, + Message: []byte("mac me"), + MacAlgorithm: "HMAC_SHA_256", + }) + require.NoError(t, err) + + _, err = b.VerifyMac(ctx, &kms.VerifyMacInput{ + KeyID: keyOut.KeyMetadata.KeyID, + Message: []byte("mac me"), + MacAlgorithm: "HMAC_SHA_256", + Mac: macOut.Mac, + GrantTokens: tokens, + }) + + return err + }, + }, + { + name: "DeriveSharedSecret", + run: func(t *testing.T, b *kms.InMemoryBackend, ctx context.Context, tokens []string) error { + t.Helper() + + aliceOut, err := b.CreateKey(ctx, &kms.CreateKeyInput{ + KeyUsage: kms.KeyUsageKeyAgreement, + KeySpec: "ECC_NIST_P256", + }) + require.NoError(t, err) + + bobOut, err := b.CreateKey(ctx, &kms.CreateKeyInput{ + KeyUsage: kms.KeyUsageKeyAgreement, + KeySpec: "ECC_NIST_P256", + }) + require.NoError(t, err) + + bobPub, err := b.GetPublicKey(ctx, &kms.GetPublicKeyInput{KeyID: bobOut.KeyMetadata.KeyID}) + require.NoError(t, err) + + _, err = b.DeriveSharedSecret(ctx, &kms.DeriveSharedSecretInput{ + KeyID: aliceOut.KeyMetadata.KeyID, + KeyAgreementAlgorithm: "ECDH", + PublicKey: bobPub.PublicKey, + GrantTokens: tokens, + }) + + return err + }, + }, + } + + for _, op := range opCases { + for _, sc := range scenarios { + t.Run(op.name+"/"+sc.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + b := kms.NewInMemoryBackend() + + var tokens []string + + switch sc.token { + case "valid": + grantKeyOut, err := b.CreateKey(ctx, &kms.CreateKeyInput{}) + require.NoError(t, err) + + grantOut, err := b.CreateGrant(ctx, &kms.CreateGrantInput{ + KeyID: grantKeyOut.KeyMetadata.KeyID, + GranteePrincipal: "arn:aws:iam::000000000000:role/test", + Operations: []string{ + "Sign", "Verify", "GetPublicKey", "GenerateMac", "VerifyMac", "DeriveSharedSecret", + }, + }) + require.NoError(t, err) + tokens = []string{grantOut.GrantToken} + case "bogus": + tokens = []string{"not-a-real-grant-token"} + } + + err := op.run(t, b, ctx, tokens) + + if sc.wantErr != nil { + require.Error(t, err) + assert.ErrorIs(t, err, sc.wantErr, "op %s: got %v, want %v", op.name, err, sc.wantErr) + + return + } + + require.NoError(t, err, "op %s", op.name) + }) + } + } +} + +// Test_GenerateDataKeyPair_GrantTokenConstraints verifies that GenerateDataKeyPair +// (and its WithoutPlaintext sibling) enforce a grant's EncryptionContextEquals +// constraint against the caller-supplied EncryptionContext, mirroring the behavior +// already proven for Encrypt/Decrypt/GenerateDataKey. +func Test_GenerateDataKeyPair_GrantTokenConstraints(t *testing.T) { + t.Parallel() + + tests := []struct { + wantErr error + callerContext map[string]string + name string + withoutPT bool + }{ + { + name: "matching_context_succeeds", + callerContext: map[string]string{"purpose": "test"}, + }, + { + name: "mismatched_context_rejected", + callerContext: map[string]string{"purpose": "wrong"}, + wantErr: kms.ErrKeyInvalidState, + }, + { + name: "matching_context_succeeds_without_plaintext", + callerContext: map[string]string{"purpose": "test"}, + withoutPT: true, + }, + { + name: "mismatched_context_rejected_without_plaintext", + callerContext: map[string]string{"purpose": "wrong"}, + withoutPT: true, + wantErr: kms.ErrKeyInvalidState, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + b := kms.NewInMemoryBackend() + + wrapKeyOut, err := b.CreateKey(ctx, &kms.CreateKeyInput{}) + require.NoError(t, err) + + grantOut, err := b.CreateGrant(ctx, &kms.CreateGrantInput{ + KeyID: wrapKeyOut.KeyMetadata.KeyID, + GranteePrincipal: "arn:aws:iam::000000000000:role/test", + Operations: []string{"GenerateDataKeyPair", "GenerateDataKeyPairWithoutPlaintext"}, + Constraints: &kms.GrantConstraints{ + EncryptionContextEquals: map[string]string{"purpose": "test"}, + }, + }) + require.NoError(t, err) + + var opErr error + + if tt.withoutPT { + _, opErr = b.GenerateDataKeyPairWithoutPlaintext(ctx, &kms.GenerateDataKeyPairWithoutPlaintextInput{ + KeyID: wrapKeyOut.KeyMetadata.KeyID, + KeyPairSpec: "ECC_NIST_P256", + EncryptionContext: tt.callerContext, + GrantTokens: []string{grantOut.GrantToken}, + }) + } else { + _, opErr = b.GenerateDataKeyPair(ctx, &kms.GenerateDataKeyPairInput{ + KeyID: wrapKeyOut.KeyMetadata.KeyID, + KeyPairSpec: "ECC_NIST_P256", + EncryptionContext: tt.callerContext, + GrantTokens: []string{grantOut.GrantToken}, + }) + } + + if tt.wantErr != nil { + require.Error(t, opErr) + assert.ErrorIs(t, opErr, tt.wantErr, "got %v, want %v", opErr, tt.wantErr) + + return + } + + require.NoError(t, opErr) + }) + } +} + +// Test_KMS_InvalidKeySpec_Returns400ValidationException drives the full HTTP handler +// path to confirm an unrecognized KeySpec/KeyPairSpec surfaces as a 400 +// ValidationException, not a 500 InternalServiceError. Before the fix, +// generateKeyMaterial's "unsupported key spec" sentinel was never wrapped with +// ErrValidation, so classifyKMSError fell through to its default 500 branch. +func Test_KMS_InvalidKeySpec_Returns400ValidationException(t *testing.T) { + t.Parallel() + + tests := []struct { + buildBody func(t *testing.T, wrapKeyID string) (target, body string) + name string + needsWrapKey bool + }{ + { + name: "CreateKey_bogus_KeySpec", + buildBody: func(t *testing.T, _ string) (string, string) { + t.Helper() + + return "TrentService.CreateKey", `{"KeySpec":"BOGUS_SPEC"}` + }, + }, + { + name: "GenerateDataKeyPair_bogus_KeyPairSpec", + needsWrapKey: true, + buildBody: func(t *testing.T, wrapKeyID string) (string, string) { + t.Helper() + + body, err := json.Marshal(map[string]string{ + "KeyId": wrapKeyID, + "KeyPairSpec": "BOGUS_SPEC", + }) + require.NoError(t, err) + + return "TrentService.GenerateDataKeyPair", string(body) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + e := echo.New() + backend := kms.NewInMemoryBackend() + h := kms.NewHandler(backend) + + var wrapKeyID string + + if tt.needsWrapKey { + out, err := backend.CreateKey(context.Background(), &kms.CreateKeyInput{}) + require.NoError(t, err) + wrapKeyID = out.KeyMetadata.KeyID + } + + target, body := tt.buildBody(t, wrapKeyID) + + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req.Header.Set("X-Amz-Target", target) + + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + + require.NoError(t, h.Handler()(c)) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + var errResp kms.ErrorResponse + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &errResp)) + assert.Equal(t, "ValidationException", errResp.Type) + }) + } +} + +// Test_KMS_Janitor_PurgeKey_CleansGrantByKeyIndex verifies that permanently purging +// a key past its ScheduleKeyDeletion window also drops the key's grantsByKey +// secondary-index submap. Before the fix, purgeKey deleted the key's grants from +// the canonical grants map and grantsByToken, but left the (now permanently +// unreachable) grantsByKey[region][keyID] submap allocated forever — a real memory +// leak for any long-running instance that creates and deletes many keys with grants. +func Test_KMS_Janitor_PurgeKey_CleansGrantByKeyIndex(t *testing.T) { + t.Parallel() + + ctx := context.Background() + b := kms.NewInMemoryBackend() + + keyOut, err := b.CreateKey(ctx, &kms.CreateKeyInput{}) + require.NoError(t, err) + keyID := keyOut.KeyMetadata.KeyID + + _, err = b.CreateGrant(ctx, &kms.CreateGrantInput{ + KeyID: keyID, + GranteePrincipal: "arn:aws:iam::000000000000:role/test", + Operations: []string{"Encrypt", "Decrypt"}, + }) + require.NoError(t, err) + + require.Equal(t, 1, kms.GrantsByKeyCount(b, kms.MockRegion, keyID), "grant index should be populated before purge") + + _, err = b.ScheduleKeyDeletion(ctx, &kms.ScheduleKeyDeletionInput{KeyID: keyID, PendingWindowInDays: 7}) + require.NoError(t, err) + + // Backdate the deletion so the janitor purges it on the next sweep. + b.SetDeletionDateForTest(keyID, time.Now().Add(-time.Second)) + + j := kms.NewJanitor(b, time.Minute) + j.SweepOnce(ctx) + + _, err = b.DescribeKey(ctx, &kms.DescribeKeyInput{KeyID: keyID}) + require.Error(t, err, "key should be permanently purged") + + assert.Equal(t, 0, kms.GrantsByKeyCount(b, kms.MockRegion, keyID), + "grantsByKey submap must be dropped, not merely emptied, after a permanent key purge") +} diff --git a/services/kms/persistence.go b/services/kms/persistence.go index 751830374..5fa80edff 100644 --- a/services/kms/persistence.go +++ b/services/kms/persistence.go @@ -3,38 +3,42 @@ package kms import ( "context" "encoding/json" - "errors" "fmt" + "strings" "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) -// errFlatSnapshotFormat is returned by tryRestoreNested when the snapshot uses the legacy flat format. -var errFlatSnapshotFormat = errors.New("detected flat snapshot format") - +// kmsSnapshotVersion identifies the shape of backendSnapshot's Tables blob +// (i.e. the set of per-region keys/aliases/grants/customKeyStores tables +// registered on b.registry -- see keysStore/aliasesStore/grantsRegion/ +// customKeyStoresStore in backend.go). It must be bumped whenever a change +// there would make an older snapshot unsafe to decode as the current shape. +// Restore compares this against the persisted value and discards (rather +// than attempts to partially decode) any mismatch -- see Restore below. This +// mirrors the services/ec2 and services/sqs pkgs/store conversions (commits +// 12e611a4, 0f09d77c). +const kmsSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the KMS backend. +// +// Tables holds one JSON-encoded array per registered per-region table, +// produced by [store.Registry.SnapshotAll] -- e.g. "keys:us-east-1", +// "grants:eu-west-1". Policies, KeyMaterials, and KeyMaterialHistory remain +// plain region-nested maps: a policy is a bare string and a keyMaterial/ +// history entry carries no identity field of its own (see policiesStore/ +// keyMaterialsStore/keyMaterialHistoryStore in backend.go), so neither can be +// expressed as a store.Table. type backendSnapshot struct { - Keys map[string]map[string]*Key `json:"keys"` - Aliases map[string]map[string]*Alias `json:"aliases"` - Grants map[string]map[string]*Grant `json:"grants"` + Tables map[string]json.RawMessage `json:"tables"` Policies map[string]map[string]string `json:"policies"` KeyMaterials map[string]map[string]serializedKeyMaterial `json:"key_materials,omitempty"` KeyMaterialHistory map[string]map[string][]serializedKeyMaterial `json:"key_material_history,omitempty"` - CustomKeyStores map[string]map[string]*CustomKeyStore `json:"custom_key_stores,omitempty"` AccountID string `json:"accountID"` Region string `json:"region"` -} - -// backendSnapshotFlat is the legacy flat (pre-region-nested) snapshot format for backwards compat. -type backendSnapshotFlat struct { - Keys map[string]*Key `json:"keys"` - Aliases map[string]*Alias `json:"aliases"` - Grants map[string]*Grant `json:"grants"` - Policies map[string]string `json:"policies"` - KeyMaterials map[string]serializedKeyMaterial `json:"key_materials,omitempty"` - KeyMaterialHistory map[string][]serializedKeyMaterial `json:"key_material_history,omitempty"` - CustomKeyStores map[string]*CustomKeyStore `json:"custom_key_stores,omitempty"` - AccountID string `json:"accountID"` - Region string `json:"region"` + Version int `json:"version"` } // snapshotRegionKeyMaterials serializes b.keyMaterials into region-nested form. @@ -122,87 +126,28 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "kms: snapshot table marshal failed", "error", err) + + return nil + } + snap := backendSnapshot{ - Keys: b.keys, - Aliases: b.aliases, - Grants: b.grants, + Version: kmsSnapshotVersion, + Tables: tables, Policies: b.policies, KeyMaterials: snapshotRegionKeyMaterials(ctx, b.keyMaterials), KeyMaterialHistory: snapshotRegionKeyMaterialHistory(ctx, b.keyMaterialHistory), - CustomKeyStores: b.customKeyStores, AccountID: b.accountID, Region: b.defaultRegion, } - data, err := json.Marshal(snap) - if err != nil { - return nil - } - - return data -} - -// Restore loads backend state from a JSON snapshot. -// It implements persistence.Persistable. -// Supports both region-nested (current) and flat (legacy) snapshot formats. -// If a key in the snapshot does not have corresponding key material (e.g. from an older snapshot -// format), a warning is logged. Callers of Encrypt/Sign/etc. will receive ErrKeyMaterialUnavailable. -func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { - // Two-pass restore: try nested format first, fall back to flat legacy format. - snap, err := tryRestoreNested(data) - if err != nil { - // Fall back to flat format: migrate everything under defaultRegion. - return b.restoreFlat(ctx, data) - } - - return b.applySnapshot(ctx, snap) -} - -// tryRestoreNested attempts to unmarshal data as the region-nested snapshot format. -// Returns an error if the data does not match the nested format (e.g. Keys is flat). -func tryRestoreNested(data []byte) (*backendSnapshot, error) { - var snap backendSnapshot - if err := json.Unmarshal(data, &snap); err != nil { - return nil, err - } - - // Heuristic: if Keys is nil or empty it could be either format — accept as nested. - // If Keys has entries, check that values are map[string]*Key (nested) not *Key (flat). - // We detect flat format by trying to unmarshal Keys as map[string]*Key. - // Since both formats use the same "keys" JSON tag, we use a raw probe. - var probe struct { - Keys json.RawMessage `json:"keys"` - } - - if err := json.Unmarshal(data, &probe); err != nil { - return nil, err - } - - if len(probe.Keys) > 0 && string(probe.Keys) != "null" { - // Try to unmarshal as flat map[string]*Key. - var flatKeys map[string]*Key - if err := json.Unmarshal(probe.Keys, &flatKeys); err == nil { - // Successfully parsed as flat — check if any value looks like a *Key - // (has a "keyId" field) rather than map[string]*Key (has string sub-keys - // that themselves contain "keyId"). If the flat parse succeeded without error - // and snap.Keys (nested parse) is non-nil, we need to determine which - // interpretation is correct. - // - // Nested format: {"us-east-1": {"uuid": {Key...}}} - // Flat format: {"uuid": {Key...}} - // - // Try nested parse; if any value in snap.Keys can be parsed as a *Key - // (i.e., has KeyID field), it's flat format. - for _, regionOrKey := range flatKeys { - if regionOrKey != nil && regionOrKey.KeyID != "" { - // This is a flat map: the "region" key has a KeyID → it's actually a keyID key. - return nil, errFlatSnapshotFormat - } - } - } - } - - return &snap, nil + return persistence.MarshalSnapshot(ctx, "kms", snap) } // restoreRegionKeyMaterials deserializes the region-nested key materials map. @@ -278,13 +223,13 @@ func restoreKeyMaterialHistory(region, keyID string, entries []serializedKeyMate // warnMissingKeyMaterials logs a warning for any key that lacks key material after restore. func warnMissingKeyMaterials( ctx context.Context, - keys map[string]map[string]*Key, + keys map[string]*store.Table[Key], materials map[string]map[string]*keyMaterial, ) { - for region, regionKeys := range keys { + for region, t := range keys { regionMaterials := materials[region] - for keyID, key := range regionKeys { + for _, key := range t.All() { if key.KeyState == KeyStatePendingImport { continue } @@ -292,49 +237,60 @@ func warnMissingKeyMaterials( if regionMaterials == nil { logger.Load(ctx).WarnContext(ctx, "KMS restore: key has no material in snapshot; crypto operations will fail", - "region", region, "keyID", keyID) + "region", region, "keyID", key.KeyID) continue } - if _, hasMaterial := regionMaterials[keyID]; !hasMaterial { + if _, hasMaterial := regionMaterials[key.KeyID]; !hasMaterial { logger.Load(ctx).WarnContext(ctx, "KMS restore: key has no material in snapshot; crypto operations will fail", - "region", region, "keyID", keyID) + "region", region, "keyID", key.KeyID) } } } } -// ensureSnapDefaults fills nil maps in a snapshot with empty initialized versions. -func ensureSnapDefaults(snap *backendSnapshot) { - if snap.Keys == nil { - snap.Keys = make(map[string]map[string]*Key) - } - - if snap.Aliases == nil { - snap.Aliases = make(map[string]map[string]*Alias) - } +// preRegisterSnapshotTables force-registers every per-region table named in a +// snapshot's Tables blob before calling registry.RestoreAll. Per-region +// tables are otherwise registered lazily on first access (see keysStore/ +// aliasesStore/grantsRegion/customKeyStoresStore), so on a fresh backend a +// region present only in the snapshot (never yet touched by a live op) would +// have no registered table for RestoreAll to find -- store.Registry.RestoreAll +// only restores tables already known to the registry, silently ignoring any +// snapshot entry whose name isn't registered. +func (b *InMemoryBackend) preRegisterSnapshotTables(tables map[string]json.RawMessage) { + for name := range tables { + kind, region, ok := strings.Cut(name, ":") + if !ok { + continue + } - if snap.Grants == nil { - snap.Grants = make(map[string]map[string]*Grant) + switch kind { + case "keys": + b.keysStore(region) + case "aliases": + b.aliasesStore(region) + case "grants": + b.grantsRegion(region) + case "customKeyStores": + b.customKeyStoresStore(region) + } } +} - if snap.Policies == nil { - snap.Policies = make(map[string]map[string]string) - } +// Restore loads backend state from a JSON snapshot. +// It implements persistence.Persistable. +// If a key in the snapshot does not have corresponding key material (e.g. from an older snapshot +// format), a warning is logged. Callers of Encrypt/Sign/etc. will receive ErrKeyMaterialUnavailable. +func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { + var snap backendSnapshot - if snap.CustomKeyStores == nil { - snap.CustomKeyStores = make(map[string]map[string]*CustomKeyStore) + if err := persistence.UnmarshalSnapshot(ctx, "kms", data, &snap); err != nil { + return err } -} - -// applySnapshot applies a nested backendSnapshot to the backend. -// Must be called without any lock held (acquires write lock internally). -func (b *InMemoryBackend) applySnapshot(ctx context.Context, snap *backendSnapshot) error { - ensureSnapDefaults(snap) - restored, err := restoreRegionKeyMaterials(snap.KeyMaterials) + restoredMaterials, err := restoreRegionKeyMaterials(snap.KeyMaterials) if err != nil { return err } @@ -344,89 +300,49 @@ func (b *InMemoryBackend) applySnapshot(ctx context.Context, snap *backendSnapsh return err } - warnMissingKeyMaterials(ctx, snap.Keys, restored) - b.mu.Lock("Restore") defer b.mu.Unlock() - b.keys = snap.Keys - b.aliases = snap.Aliases - b.grants = snap.Grants - b.policies = snap.Policies - b.keyMaterials = restored - b.keyMaterialHistory = restoredHistory - b.customKeyStores = snap.CustomKeyStores - b.accountID = snap.AccountID - b.defaultRegion = snap.Region - b.clearResolutionCache() - b.rebuildGrantIndexesLocked() - - return nil -} - -// rebuildGrantIndexesLocked rebuilds grantsByToken and grantsByKey from the -// canonical grants map. Must be called with the write lock held. -func (b *InMemoryBackend) rebuildGrantIndexesLocked() { - b.grantsByToken = make(map[string]map[string]*Grant) - b.grantsByKey = make(map[string]map[string]map[string]*Grant) + if snap.Version != kmsSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/ec2 and services/sqs pilots. + logger.Load(ctx).WarnContext(ctx, + "kms: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", kmsSnapshotVersion) + + b.registry.ResetAll() + b.policies = make(map[string]map[string]string) + b.keyMaterials = make(map[string]map[string]*keyMaterial) + b.keyMaterialHistory = make(map[string]map[string][]*keyMaterial) + b.clearResolutionCache() - for region, regionGrants := range b.grants { - for _, g := range regionGrants { - b.grantsByTokenStore(region)[g.GrantToken] = g - b.grantsByKeyStore(region, g.KeyID)[g.GrantID] = g - } - } -} - -// restoreFlat handles the legacy flat snapshot format by migrating all data under defaultRegion. -func (b *InMemoryBackend) restoreFlat(ctx context.Context, data []byte) error { - var flat backendSnapshotFlat - - if err := json.Unmarshal(data, &flat); err != nil { - return err - } - - // Determine the region to migrate flat data into. - region := flat.Region - if region == "" { - region = b.defaultRegion - } - - // Lift flat maps into nested maps under the single region. - snap := &backendSnapshot{ - AccountID: flat.AccountID, - Region: region, - } - - if flat.Keys != nil { - snap.Keys = map[string]map[string]*Key{region: flat.Keys} - } - - if flat.Aliases != nil { - snap.Aliases = map[string]map[string]*Alias{region: flat.Aliases} + return nil } - if flat.Grants != nil { - snap.Grants = map[string]map[string]*Grant{region: flat.Grants} - } + b.preRegisterSnapshotTables(snap.Tables) - if flat.Policies != nil { - snap.Policies = map[string]map[string]string{region: flat.Policies} + if err = b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("kms: restore snapshot tables: %w", err) } - if flat.KeyMaterials != nil { - snap.KeyMaterials = map[string]map[string]serializedKeyMaterial{region: flat.KeyMaterials} - } + warnMissingKeyMaterials(ctx, b.keys, restoredMaterials) - if flat.KeyMaterialHistory != nil { - snap.KeyMaterialHistory = map[string]map[string][]serializedKeyMaterial{region: flat.KeyMaterialHistory} + b.policies = snap.Policies + if b.policies == nil { + b.policies = make(map[string]map[string]string) } - if flat.CustomKeyStores != nil { - snap.CustomKeyStores = map[string]map[string]*CustomKeyStore{region: flat.CustomKeyStores} - } + b.keyMaterials = restoredMaterials + b.keyMaterialHistory = restoredHistory + b.accountID = snap.AccountID + b.defaultRegion = snap.Region + b.clearResolutionCache() - return b.applySnapshot(ctx, snap) + return nil } // Snapshot implements persistence.Persistable by delegating to the backend. diff --git a/services/kms/persistence_test.go b/services/kms/persistence_test.go index 43c99046b..0e7f115b6 100644 --- a/services/kms/persistence_test.go +++ b/services/kms/persistence_test.go @@ -75,3 +75,119 @@ func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { err := b.Restore(t.Context(), []byte("not-valid-json")) require.Error(t, err) } + +// TestInMemoryBackend_FullStateSnapshotRestoreRoundTrip exercises a full +// Snapshot->Restore round trip covering every resource kind the Phase 3.3 +// pkgs/store conversion touches: the store.Table-backed keys/aliases/grants/ +// customKeyStores, and the plain region-nested policies/keyMaterials/ +// keyMaterialHistory maps left raw (no store.Table identity field). Crypto +// key material -- both a symmetric key's current AND rotated-out historical +// material, and an asymmetric key's material -- must round-trip byte-for-byte +// since a mismatch would silently break Decrypt/Verify on restored backends. +func TestInMemoryBackend_FullStateSnapshotRestoreRoundTrip(t *testing.T) { + t.Parallel() + + ctx := t.Context() + orig := kms.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + + // Symmetric key: encrypt before rotation, rotate on demand (populates + // keyMaterialHistory with the pre-rotation material), then confirm the + // pre-rotation ciphertext still decrypts via history. + symOut, err := orig.CreateKey(ctx, &kms.CreateKeyInput{Description: "sym"}) + require.NoError(t, err) + symKeyID := symOut.KeyMetadata.KeyID + + encOut, err := orig.Encrypt(ctx, &kms.EncryptInput{KeyID: symKeyID, Plaintext: []byte("top secret")}) + require.NoError(t, err) + + _, err = orig.RotateKeyOnDemand(ctx, &kms.RotateKeyOnDemandInput{KeyID: symKeyID}) + require.NoError(t, err) + require.Equal(t, 1, orig.KeyMaterialHistoryLenForTest(symKeyID)) + + _, err = orig.Decrypt(ctx, &kms.DecryptInput{CiphertextBlob: encOut.CiphertextBlob}) + require.NoError(t, err, "pre-rotation ciphertext must still decrypt via keyMaterialHistory") + + // Alias. + require.NoError(t, orig.CreateAlias(ctx, &kms.CreateAliasInput{ + AliasName: "alias/full-state", TargetKeyID: symKeyID, + })) + + // Grant. + grantOut, err := orig.CreateGrant(ctx, &kms.CreateGrantInput{ + KeyID: symKeyID, + GranteePrincipal: "arn:aws:iam::000000000000:role/test", + Operations: []string{"Encrypt", "Decrypt"}, + }) + require.NoError(t, err) + + // Key policy (plain region-nested map, no store.Table identity field). + policy := `{"Version":"2012-10-17","Statement":[` + + `{"Effect":"Allow","Principal":"*","Action":"kms:*","Resource":"*"}]}` + require.NoError(t, orig.PutKeyPolicy(ctx, &kms.PutKeyPolicyInput{KeyID: symKeyID, Policy: policy})) + + // Custom key store. + cksOut, err := orig.CreateCustomKeyStore(ctx, &kms.CreateCustomKeyStoreInput{CustomKeyStoreName: "my-cks"}) + require.NoError(t, err) + + // Asymmetric key material round trip (RSA sign/verify). + asymOut, err := orig.CreateKey(ctx, &kms.CreateKeyInput{KeyUsage: kms.KeyUsageSignVerify, KeySpec: "RSA_2048"}) + require.NoError(t, err) + asymKeyID := asymOut.KeyMetadata.KeyID + + sigOut, err := orig.Sign(ctx, &kms.SignInput{ + KeyID: asymKeyID, Message: []byte("msg"), SigningAlgorithm: "RSASSA_PKCS1_V1_5_SHA_256", + }) + require.NoError(t, err) + + snap := orig.Snapshot(ctx) + require.NotNil(t, snap) + + fresh := kms.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + require.NoError(t, fresh.Restore(ctx, snap)) + + // Symmetric key's CURRENT material persisted: a fresh encrypt/decrypt works. + freshEnc, err := fresh.Encrypt(ctx, &kms.EncryptInput{KeyID: symKeyID, Plaintext: []byte("after restore")}) + require.NoError(t, err) + freshDec, err := fresh.Decrypt(ctx, &kms.DecryptInput{CiphertextBlob: freshEnc.CiphertextBlob}) + require.NoError(t, err) + assert.Equal(t, []byte("after restore"), freshDec.Plaintext) + + // keyMaterialHistory persisted: the pre-rotation ciphertext still decrypts. + assert.Equal(t, 1, fresh.KeyMaterialHistoryLenForTest(symKeyID)) + decOut, err := fresh.Decrypt(ctx, &kms.DecryptInput{CiphertextBlob: encOut.CiphertextBlob}) + require.NoError(t, err, "pre-rotation ciphertext must still decrypt after restore") + assert.Equal(t, []byte("top secret"), decOut.Plaintext) + + // Alias persisted and resolves. + aliasOut, err := fresh.ListAliases(ctx, &kms.ListAliasesInput{KeyID: symKeyID}) + require.NoError(t, err) + require.Len(t, aliasOut.Aliases, 1) + assert.Equal(t, "alias/full-state", aliasOut.Aliases[0].AliasName) + + // Grant persisted: the grant token still authorizes via the rebuilt indexes. + _, err = fresh.Encrypt(ctx, &kms.EncryptInput{ + KeyID: symKeyID, Plaintext: []byte("via grant"), GrantTokens: []string{grantOut.GrantToken}, + }) + require.NoError(t, err) + + // Key policy persisted. + polOut, err := fresh.GetKeyPolicy(ctx, &kms.GetKeyPolicyInput{KeyID: symKeyID}) + require.NoError(t, err) + assert.Equal(t, policy, polOut.Policy) + + // Custom key store persisted. + cksListOut, err := fresh.DescribeCustomKeyStores( + ctx, &kms.DescribeCustomKeyStoresInput{CustomKeyStoreID: cksOut.CustomKeyStoreID}, + ) + require.NoError(t, err) + require.Len(t, cksListOut.CustomKeyStores, 1) + assert.Equal(t, "my-cks", cksListOut.CustomKeyStores[0].CustomKeyStoreName) + + // Asymmetric key material persisted: the original signature still verifies. + verOut, err := fresh.Verify(ctx, &kms.VerifyInput{ + KeyID: asymKeyID, Message: []byte("msg"), Signature: sigOut.Signature, + SigningAlgorithm: "RSASSA_PKCS1_V1_5_SHA_256", + }) + require.NoError(t, err) + assert.True(t, verOut.SignatureValid) +} diff --git a/services/lambda/PARITY.md b/services/lambda/PARITY.md new file mode 100644 index 000000000..cd42785b4 --- /dev/null +++ b/services/lambda/PARITY.md @@ -0,0 +1,24 @@ +--- +service: lambda +sdk_module: aws-sdk-go-v2/service/lambda@v1.94.1 +last_audit_commit: c3b5d46a +last_audit_date: 2026-07-05 +overall: A # 641 LOC genuine fixes incl. a disguised-stub no real client could call +protocol: REST-JSON +families: + resource_policy: {status: ok, note: FIXED RemovePermission read StatementId from query string; real SDK sends URI path segment /policy/{StatementId} — route never matched (disguised stub). Added Qualifier scoping ($LATEST rejected), EventSourceToken/PrincipalOrgID fields} + event_source_mappings: {status: ok, note: FIXED ARN parsing dropped function name (kept only qualifier); qualified ESMs now routed via InvokeFunctionWithQualifier. Pollers PROVEN — backoff, FilterCriteria, BisectBatchOnFunctionError, ReportBatchItemFailures, MaxRecordAge} + persistence: {status: ok, note: FIXED permissions map never snapshotted (policies lost on restore); versionIndex + esmByFunctionARN not rebuilt on Restore — now rebuilt} + runtime_lifecycle: {status: ok, note: PROVEN — LRU eviction, async cleanup semaphore, container stop/remove, port release, dir cleanup. Real Docker exec} + durable_execution: {status: ok, note: PROVEN — reads/writes real durableExecutionStore despite handler_stubs.go filename} +gaps: [] +deferred: + - AddPermission FunctionUrlAuthType / InvokedViaFunctionUrl / RevisionId optimistic-concurrency (lower value) + - function CRUD/versions/aliases/layers/provisioned-concurrency/URLs/tags — skimmed, tests green, not exhaustively re-verified +leaks: {status: clean, note: event-source pollers + janitor + container lifecycle all leak-conscious; go test -race passes} +--- + +## Notes +- InvocationType is a type alias (type InvocationType = string) so lambda backend satisfies sns.LambdaInvoker directly. +- ARN-parsing anti-pattern "take last colon segment" recurs — watch for it elsewhere. +- Trap: RemovePermission wire = DELETE /2015-03-31/functions/{name}/policy/{StatementId} (path, not query). diff --git a/services/lambda/async_destinations.go b/services/lambda/async_destinations.go index aee5d3063..6b44c343d 100644 --- a/services/lambda/async_destinations.go +++ b/services/lambda/async_destinations.go @@ -120,7 +120,7 @@ func (b *InMemoryBackend) resolveAsyncTargets(out asyncOutcome) ( defer b.mu.RUnlock() delivery := b.asyncDelivery - fn := b.functions[out.functionName] + fn, _ := b.functions.Get(out.functionName) eic := b.eventInvokeConfigs[out.functionName] var functionArn, dlqTarget string diff --git a/services/lambda/backend.go b/services/lambda/backend.go index 0b9e953cd..09ff5ebc4 100644 --- a/services/lambda/backend.go +++ b/services/lambda/backend.go @@ -35,6 +35,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/httputils" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) var ( @@ -208,35 +209,66 @@ type functionURLServer struct { // InMemoryBackend is a concurrency-safe in-memory Lambda backend. type InMemoryBackend struct { - cwLogs CWLogsBackend - s3Fetcher S3CodeFetcher - docker container.Runtime - dnsRegistrar DNSRegistrar - ctx context.Context - logSem chan struct{} - fisFaults map[string]*FISInvocationFault - versionCounters map[string]int - functions map[string]*FunctionConfiguration - functionURLServers map[string]*functionURLServer - functionURLConfigs map[string]*FunctionURLConfig - versions map[string][]*FunctionVersion - eventInvokeConfigs map[string]*FunctionEventInvokeConfig - functionConcurrencies map[string]int - kinesisPoller *EventSourcePoller - pollerCancel context.CancelFunc - provisionedConcurrencies map[string]map[string]*ProvisionedConcurrencyConfig - layers map[string][]*LayerVersion - eventSourceMappings map[string]*EventSourceMapping - esmByFunctionARN map[string]map[string]struct{} - versionIndex map[string]map[string]*FunctionVersion - cleanupSem chan struct{} - layerVersionCounters map[string]int64 - layerPolicies map[string]map[int64]map[string]*LayerVersionStatement - aliases map[string]map[string]*FunctionAlias - permissions map[string]map[string]*FunctionPermission - codeSigningConfigs map[string]*CodeSigningConfig - fnCodeSigningConfigs map[string]string - capacityProviders map[string]*CapacityProvider + cwLogs CWLogsBackend + s3Fetcher S3CodeFetcher + docker container.Runtime + dnsRegistrar DNSRegistrar + ctx context.Context + logSem chan struct{} + fisFaults map[string]*FISInvocationFault + versionCounters map[string]int + functions *store.Table[FunctionConfiguration] + functionURLServers map[string]*functionURLServer + functionURLConfigs *store.Table[FunctionURLConfig] + versions map[string][]*FunctionVersion + // eventInvokeConfigs stays a plain map (not a store.Table): its only + // identity field, FunctionArn, is copied from the owning + // FunctionConfiguration.FunctionArn at Put time, and a large share of this + // package's own test fixtures construct FunctionConfiguration without + // ever setting FunctionArn (it is optional unless the FunctionURL/ + // EventInvokeConfig/CapacityProvider surface is being exercised) -- + // keying a Table off it would silently mis-key every such config to "". + // WAS persisted before this refactor and remains a raw field on + // backendSnapshot. + eventInvokeConfigs map[string]*FunctionEventInvokeConfig + functionConcurrencies map[string]int + kinesisPoller *EventSourcePoller + pollerCancel context.CancelFunc + // provisionedConcurrencies is keyed by FunctionArn (buildAliasARN: + // function+qualifier composite); provisionedConcurrenciesByFunction + // indexes it by bare function name for ListProvisionedConcurrencyConfigs + // and the function-delete cascade. Registered on b.ephemeralRegistry, not + // b.registry -- see store_setup.go's package doc: this was never + // persisted before the conversion and must stay that way. + provisionedConcurrencies *store.Table[ProvisionedConcurrencyConfig] + provisionedConcurrenciesByFunction *store.Index[ProvisionedConcurrencyConfig] + layers map[string][]*LayerVersion + eventSourceMappings *store.Table[EventSourceMapping] + esmByFunctionARN map[string]map[string]struct{} + versionIndex map[string]map[string]*FunctionVersion + cleanupSem chan struct{} + layerVersionCounters map[string]int64 + layerPolicies map[string]map[int64]map[string]*LayerVersionStatement + // aliases is keyed by aliasKey(functionName, aliasName); + // aliasesByFunction indexes it by bare function name for ListAliases and + // the function-delete cascade. + aliases *store.Table[FunctionAlias] + aliasesByFunction *store.Index[FunctionAlias] + // permissions is keyed by permissionKeyFn (permissionMapKey(FunctionName, + // Qualifier)+"|"+StatementID); permissionsByTarget indexes it by + // permissionMapKey(FunctionName, Qualifier) for GetPolicy. + permissions *store.Table[FunctionPermission] + permissionsByTarget *store.Index[FunctionPermission] + codeSigningConfigs *store.Table[CodeSigningConfig] + fnCodeSigningConfigs map[string]string + capacityProviders *store.Table[CapacityProvider] + // registry holds every table that was already persisted before this + // refactor (see backendSnapshot in persistence.go); ephemeralRegistry + // holds tables with a pure key function that were NOT previously + // persisted. Both are swept by Reset(); only registry feeds Snapshot/ + // Restore. See store_setup.go's package doc for the full rationale. + registry *store.Registry + ephemeralRegistry *store.Registry runtimeManagementConfigs map[string]*RuntimeManagementConfig functionRecursionConfigs map[string]*FunctionRecursionConfig functionScalingConfigs map[string]*FunctionScalingConfig @@ -310,18 +342,14 @@ func NewInMemoryBackendWithContext( svcCtx = context.Background() } - return &InMemoryBackend{ - functions: make(map[string]*FunctionConfiguration), + b := &InMemoryBackend{ runtimes: make(map[string]*functionRuntime), - eventSourceMappings: make(map[string]*EventSourceMapping), esmByFunctionARN: make(map[string]map[string]struct{}), versionIndex: make(map[string]map[string]*FunctionVersion), cleanupSem: make(chan struct{}, maxCleanupConcurrency), logSem: make(chan struct{}, maxConcurrentInvocationLogs), - functionURLConfigs: make(map[string]*FunctionURLConfig), functionURLServers: make(map[string]*functionURLServer), versions: make(map[string][]*FunctionVersion), - aliases: make(map[string]map[string]*FunctionAlias), versionCounters: make(map[string]int), layers: make(map[string][]*LayerVersion), layerVersionCounters: make(map[string]int64), @@ -329,12 +357,8 @@ func NewInMemoryBackendWithContext( eventInvokeConfigs: make(map[string]*FunctionEventInvokeConfig), functionConcurrencies: make(map[string]int), activeConcurrencies: make(map[string]int), - provisionedConcurrencies: make(map[string]map[string]*ProvisionedConcurrencyConfig), fisFaults: make(map[string]*FISInvocationFault), - permissions: make(map[string]map[string]*FunctionPermission), - codeSigningConfigs: make(map[string]*CodeSigningConfig), fnCodeSigningConfigs: make(map[string]string), - capacityProviders: make(map[string]*CapacityProvider), runtimeManagementConfigs: make(map[string]*RuntimeManagementConfig), functionRecursionConfigs: make(map[string]*FunctionRecursionConfig), functionScalingConfigs: make(map[string]*FunctionScalingConfig), @@ -348,7 +372,13 @@ func NewInMemoryBackendWithContext( region: region, ctx: svcCtx, mu: lockmetrics.New("lambda"), + registry: store.NewRegistry(), + ephemeralRegistry: store.NewRegistry(), } + + registerAllTables(b) + + return b } // Close shuts down all active function URL servers and runtime API servers. @@ -483,13 +513,23 @@ func (b *InMemoryBackend) SetDynamoDBStreamsReader(r DynamoDBStreamsReader) { // esmFunctionName normalizes a function reference (bare name or full function ARN) // to the bare function name used for event-source-mapping indexing. func esmFunctionName(functionName string) string { - if strings.HasPrefix(functionName, "arn:aws:lambda:") { - parts := strings.Split(functionName, ":") + if !strings.HasPrefix(functionName, "arn:aws:lambda:") { + return functionName + } - return parts[len(parts)-1] + // Bug fix (parity-sweep-3): previously took the last colon-separated + // segment of the ARN, which for a qualified ARN + // (arn:...:function:my-func:PROD) returned just "PROD" — discarding the + // actual function name and causing the mapping to be registered (and the + // poller to invoke) a nonexistent function named after the qualifier. + // Preserve the "name:qualifier" suffix so the mapping keeps routing to + // the specific version/alias, matching real Lambda's FunctionArn echo. + name, qualifier := functionNameAndQualifierFromARN(functionName) + if qualifier != "" { + return name + ":" + qualifier } - return functionName + return name } // CreateEventSourceMapping creates a new event source mapping. @@ -555,7 +595,7 @@ func (b *InMemoryBackend) CreateEventSourceMapping( FunctionResponseTypes: input.FunctionResponseTypes, } - b.eventSourceMappings[id] = m + b.eventSourceMappings.Put(m) if b.esmByFunctionARN[fnARN] == nil { b.esmByFunctionARN[fnARN] = make(map[string]struct{}) @@ -574,7 +614,7 @@ func (b *InMemoryBackend) GetEventSourceMapping(uuid string) (*EventSourceMappin b.mu.RLock("GetEventSourceMapping") defer b.mu.RUnlock() - m, ok := b.eventSourceMappings[uuid] + m, ok := b.eventSourceMappings.Get(uuid) if !ok { return nil, ErrESMNotFound } @@ -602,15 +642,12 @@ func (b *InMemoryBackend) ListEventSourceMappings( ids := b.esmByFunctionARN[fnARN] result = make([]*EventSourceMapping, 0, len(ids)) for id := range ids { - if m, ok := b.eventSourceMappings[id]; ok { + if m, ok := b.eventSourceMappings.Get(id); ok { result = append(result, m) } } } else { - result = make([]*EventSourceMapping, 0, len(b.eventSourceMappings)) - for _, m := range b.eventSourceMappings { - result = append(result, m) - } + result = b.eventSourceMappings.All() } // Apply optional EventSourceArn filter. @@ -636,12 +673,12 @@ func (b *InMemoryBackend) DeleteEventSourceMapping(id string) (*EventSourceMappi b.mu.Lock("DeleteEventSourceMapping") defer b.mu.Unlock() - m, ok := b.eventSourceMappings[id] + m, ok := b.eventSourceMappings.Get(id) if !ok { return nil, ErrESMNotFound } - delete(b.eventSourceMappings, id) + b.eventSourceMappings.Delete(id) if ids := b.esmByFunctionARN[m.FunctionARN]; ids != nil { delete(ids, id) if len(ids) == 0 { @@ -672,13 +709,13 @@ func (b *InMemoryBackend) CreateFunctionURLConfig( ) (*FunctionURLConfig, error) { b.mu.Lock("CreateFunctionURLConfig.check") - if _, ok := b.functions[functionName]; !ok { + if _, ok := b.functions.Get(functionName); !ok { b.mu.Unlock() return nil, ErrFunctionNotFound } - if _, exists := b.functionURLConfigs[functionName]; exists { + if _, exists := b.functionURLConfigs.Get(functionName); exists { b.mu.Unlock() return nil, ErrFunctionAlreadyExists @@ -711,7 +748,7 @@ func (b *InMemoryBackend) CreateFunctionURLConfig( b.mu.Lock("CreateFunctionURLConfig.commit") defer b.mu.Unlock() - if _, exists := b.functionURLConfigs[functionName]; exists { + if _, exists := b.functionURLConfigs.Get(functionName); exists { // Another goroutine won the race. Our server was already committed to // b.functionURLServers by allocateAndStartURLServerUnlocked; remove it // under the lock and schedule shutdown outside. @@ -736,7 +773,7 @@ func (b *InMemoryBackend) CreateFunctionURLConfig( return nil, ErrFunctionAlreadyExists } - b.functionURLConfigs[functionName] = cfg + b.functionURLConfigs.Put(cfg) return cfg, nil } @@ -804,7 +841,7 @@ func (b *InMemoryBackend) GetFunctionURLConfig(functionName string) (*FunctionUR b.mu.RLock("GetFunctionURLConfig") defer b.mu.RUnlock() - cfg, ok := b.functionURLConfigs[functionName] + cfg, ok := b.functionURLConfigs.Get(functionName) if !ok { return nil, ErrFunctionURLNotFound } @@ -816,13 +853,13 @@ func (b *InMemoryBackend) GetFunctionURLConfig(functionName string) (*FunctionUR func (b *InMemoryBackend) DeleteFunctionURLConfig(functionName string) error { b.mu.Lock("DeleteFunctionURLConfig") - if _, ok := b.functionURLConfigs[functionName]; !ok { + if _, ok := b.functionURLConfigs.Get(functionName); !ok { b.mu.Unlock() return ErrFunctionURLNotFound } - delete(b.functionURLConfigs, functionName) + b.functionURLConfigs.Delete(functionName) srv := b.functionURLServers[functionName] delete(b.functionURLServers, functionName) @@ -995,7 +1032,9 @@ func (b *InMemoryBackend) lookupFunctionURLConfig(functionName string) *Function b.mu.RLock("lookupFunctionURLConfig") defer b.mu.RUnlock() - return b.functionURLConfigs[functionName] + cfg, _ := b.functionURLConfigs.Get(functionName) + + return cfg } // functionURLCors returns the CORS config when present. @@ -1263,7 +1302,7 @@ func (b *InMemoryBackend) CreateFunction(fn *FunctionConfiguration) error { b.mu.Lock("CreateFunction") defer b.mu.Unlock() - if _, exists := b.functions[fn.FunctionName]; exists { + if _, exists := b.functions.Get(fn.FunctionName); exists { return ErrFunctionAlreadyExists } @@ -1295,7 +1334,7 @@ func (b *InMemoryBackend) CreateFunction(fn *FunctionConfiguration) error { b.scheduleFunctionActive(fn.FunctionName, b.activationDelay) } - b.functions[fn.FunctionName] = fn + b.functions.Put(fn) return nil } @@ -1347,7 +1386,7 @@ func (b *InMemoryBackend) scheduleFunctionActive(name string, delay time.Duratio b.mu.Lock("scheduleFunctionActive") defer b.mu.Unlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok || fn.State != FunctionStatePending { return } @@ -1364,7 +1403,7 @@ func (b *InMemoryBackend) GetFunction(name string) (*FunctionConfiguration, erro defer b.mu.RUnlock() name = extractFunctionName(name) - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return nil, ErrFunctionNotFound } @@ -1396,7 +1435,7 @@ func (b *InMemoryBackend) GetFunctionByQualifier( b.mu.RLock("GetFunctionByQualifier") defer b.mu.RUnlock() - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return nil, ErrFunctionNotFound } @@ -1405,16 +1444,14 @@ func (b *InMemoryBackend) GetFunctionByQualifier( resolved := qualifier aliasSuffix := "" - if aliasMap := b.aliases[name]; aliasMap != nil { - if alias, ok := aliasMap[qualifier]; ok { - resolved = alias.FunctionVersion - aliasSuffix = qualifier - } + if alias, ok := b.aliases.Get(aliasKey(name, qualifier)); ok { + resolved = alias.FunctionVersion + aliasSuffix = qualifier } if resolved == versionLatest { // Alias points at $LATEST: return the live config but with the alias ARN. - fn := b.functions[name] + fn, _ := b.functions.Get(name) cfg := versionToConfig(fnToVersion(fn)) cfg.FunctionArn = buildVersionARN(b.region, b.accountID, name, aliasSuffix) @@ -1450,10 +1487,7 @@ func (b *InMemoryBackend) ListFunctions( b.mu.RLock("ListFunctions") defer b.mu.RUnlock() - fns := make([]*FunctionConfiguration, 0, len(b.functions)) - for _, fn := range b.functions { - fns = append(fns, fn) - } + fns := b.functions.All() sort.Slice(fns, func(i, j int) bool { return fns[i].FunctionName < fns[j].FunctionName @@ -1472,12 +1506,8 @@ func (b *InMemoryBackend) ListFunctionsAll( b.mu.RLock("ListFunctionsAll") defer b.mu.RUnlock() - var fns []*FunctionConfiguration - // Include $LATEST for each function. - for _, fn := range b.functions { - fns = append(fns, fn) - } + fns := b.functions.All() // Include all published versions. for name, vMap := range b.versionIndex { @@ -1514,13 +1544,13 @@ func (b *InMemoryBackend) ListFunctionsAll( func (b *InMemoryBackend) DeleteFunction(name string) error { b.mu.Lock("DeleteFunction") - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { b.mu.Unlock() return ErrFunctionNotFound } - delete(b.functions, name) + b.functions.Delete(name) rt := b.runtimes[name] delete(b.runtimes, name) @@ -1530,7 +1560,7 @@ func (b *InMemoryBackend) DeleteFunction(name string) error { var esmIDsToRemove []string if ids, ok := b.esmByFunctionARN[fnARN]; ok { for id := range ids { - delete(b.eventSourceMappings, id) + b.eventSourceMappings.Delete(id) esmIDsToRemove = append(esmIDsToRemove, id) } delete(b.esmByFunctionARN, fnARN) @@ -1559,13 +1589,13 @@ func (b *InMemoryBackend) DeleteFunction(name string) error { func (b *InMemoryBackend) UpdateFunction(fn *FunctionConfiguration) error { b.mu.Lock("UpdateFunction") - if _, ok := b.functions[fn.FunctionName]; !ok { + if _, ok := b.functions.Get(fn.FunctionName); !ok { b.mu.Unlock() return ErrFunctionNotFound } - b.functions[fn.FunctionName] = fn + b.functions.Put(fn) // Evict the running runtime so the next invocation gets a fresh container with the // updated code or configuration (mirrors AWS/LocalStack behaviour). @@ -1617,7 +1647,7 @@ func (b *InMemoryBackend) PublishVersion(name, description string) (*FunctionVer b.mu.Lock("PublishVersion") defer b.mu.Unlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return nil, ErrFunctionNotFound } @@ -1668,7 +1698,7 @@ func (b *InMemoryBackend) GetVersion(name, version string) (*FunctionVersion, er defer b.mu.RUnlock() if version == versionLatest { - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return nil, ErrFunctionNotFound } @@ -1676,7 +1706,7 @@ func (b *InMemoryBackend) GetVersion(name, version string) (*FunctionVersion, er return fnToVersion(fn), nil } - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return nil, ErrFunctionNotFound } @@ -1697,7 +1727,7 @@ func (b *InMemoryBackend) ListVersionsByFunction( b.mu.RLock("ListVersionsByFunction") defer b.mu.RUnlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return page.Page[*FunctionVersion]{}, ErrFunctionNotFound } @@ -1730,7 +1760,7 @@ func (b *InMemoryBackend) CreateAlias( b.mu.Lock("CreateAlias") defer b.mu.Unlock() - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return nil, ErrFunctionNotFound } @@ -1741,11 +1771,7 @@ func (b *InMemoryBackend) CreateAlias( } } - if _, ok := b.aliases[name]; !ok { - b.aliases[name] = make(map[string]*FunctionAlias) - } - - if _, exists := b.aliases[name][input.Name]; exists { + if _, exists := b.aliases.Get(aliasKey(name, input.Name)); exists { return nil, ErrAliasAlreadyExists } @@ -1758,7 +1784,7 @@ func (b *InMemoryBackend) CreateAlias( RevisionID: uuid.New().String(), } - b.aliases[name][input.Name] = alias + b.aliases.Put(alias) return alias, nil } @@ -1768,16 +1794,11 @@ func (b *InMemoryBackend) GetAlias(name, aliasName string) (*FunctionAlias, erro b.mu.RLock("GetAlias") defer b.mu.RUnlock() - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return nil, ErrFunctionNotFound } - aliasMap, ok := b.aliases[name] - if !ok { - return nil, ErrAliasNotFound - } - - alias, ok := aliasMap[aliasName] + alias, ok := b.aliases.Get(aliasKey(name, aliasName)) if !ok { return nil, ErrAliasNotFound } @@ -1794,14 +1815,14 @@ func (b *InMemoryBackend) ListAliases( b.mu.RLock("ListAliases") defer b.mu.RUnlock() - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return page.Page[*FunctionAlias]{}, ErrFunctionNotFound } - aliasMap := b.aliases[name] - result := make([]*FunctionAlias, 0, len(aliasMap)) + forFunction := b.aliasesByFunction.Get(name) + result := make([]*FunctionAlias, 0, len(forFunction)) - for _, a := range aliasMap { + for _, a := range forFunction { if functionVersion != "" && a.FunctionVersion != functionVersion { continue } @@ -1824,12 +1845,7 @@ func (b *InMemoryBackend) UpdateAlias( b.mu.Lock("UpdateAlias") defer b.mu.Unlock() - aliasMap, ok := b.aliases[name] - if !ok { - return nil, ErrAliasNotFound - } - - alias, ok := aliasMap[aliasName] + alias, ok := b.aliases.Get(aliasKey(name, aliasName)) if !ok { return nil, ErrAliasNotFound } @@ -1856,17 +1872,10 @@ func (b *InMemoryBackend) DeleteAlias(name, aliasName string) error { b.mu.Lock("DeleteAlias") defer b.mu.Unlock() - aliasMap, hasMap := b.aliases[name] - if !hasMap { - return ErrAliasNotFound - } - - if _, hasAlias := aliasMap[aliasName]; !hasAlias { + if !b.aliases.Delete(aliasKey(name, aliasName)) { return ErrAliasNotFound } - delete(aliasMap, aliasName) - return nil } @@ -1884,6 +1893,43 @@ func extractFunctionName(name string) string { return name } +// functionNameAndQualifierFromARN splits a full/partial Lambda function ARN, or +// the bare "name:qualifier" shorthand AWS also accepts, into the bare function +// name and an optional version/alias qualifier. +// +// Examples: +// +// arn:aws:lambda:us-east-1:000000000000:function:my-func -> ("my-func", "") +// arn:aws:lambda:us-east-1:000000000000:function:my-func:PROD -> ("my-func", "PROD") +// my-func:PROD -> ("my-func", "PROD") +// my-func -> ("my-func", "") +// +// Function names never contain a colon (AWS restricts them to +// [a-zA-Z0-9-_]+), so any colon in the input unambiguously marks an ARN +// boundary or an appended qualifier — never returns a false split. +func functionNameAndQualifierFromARN(name string) (string, string) { + parts := strings.Split(name, ":") + for i, p := range parts { + if p == "function" && i+1 < len(parts) { + fnName := parts[i+1] + qualifier := "" + if i+2 < len(parts) { + qualifier = parts[i+2] + } + + return fnName, qualifier + } + } + + // Not an ARN with a "function:" segment. Support the bare "name:qualifier" + // shorthand (e.g. AddPermission/RemovePermission's "my-function:v1" format). + if before, after, ok := strings.Cut(name, ":"); ok && before != "" && after != "" { + return before, after + } + + return name, "" +} + // resolveQualifier resolves a function name with an optional qualifier to a FunctionConfiguration. // Qualifier may be a version number, alias name, or "$LATEST" (default when empty). // Returns the resolved function config. @@ -1898,10 +1944,8 @@ func (b *InMemoryBackend) resolveQualifier(name, qualifier string) (*FunctionCon // TOCTOU races with concurrent alias/version updates. b.mu.RLock("resolveQualifier") - if aliasMap := b.aliases[name]; aliasMap != nil { - if alias, ok := aliasMap[qualifier]; ok { - qualifier = selectAliasVersion(alias) - } + if alias, ok := b.aliases.Get(aliasKey(name, qualifier)); ok { + qualifier = selectAliasVersion(alias) } // Now qualifier is a version number. Find the version snapshot. @@ -3808,7 +3852,7 @@ func (b *InMemoryBackend) PutFunctionEventInvokeConfig( b.mu.Lock("PutFunctionEventInvokeConfig") defer b.mu.Unlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return nil, ErrFunctionNotFound } @@ -3837,7 +3881,7 @@ func (b *InMemoryBackend) GetFunctionEventInvokeConfig( b.mu.RLock("GetFunctionEventInvokeConfig") defer b.mu.RUnlock() - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return nil, ErrFunctionNotFound } @@ -3858,7 +3902,7 @@ func (b *InMemoryBackend) UpdateFunctionEventInvokeConfig( b.mu.Lock("UpdateFunctionEventInvokeConfig") defer b.mu.Unlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return nil, ErrFunctionNotFound } @@ -3895,7 +3939,7 @@ func (b *InMemoryBackend) DeleteFunctionEventInvokeConfig(name string) error { b.mu.Lock("DeleteFunctionEventInvokeConfig") defer b.mu.Unlock() - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return ErrFunctionNotFound } @@ -3916,7 +3960,7 @@ func (b *InMemoryBackend) ListFunctionEventInvokeConfigs( b.mu.RLock("ListFunctionEventInvokeConfigs") defer b.mu.RUnlock() - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return nil, "", ErrFunctionNotFound } @@ -3966,7 +4010,7 @@ func (b *InMemoryBackend) PutFunctionConcurrency( b.mu.Lock("PutFunctionConcurrency") defer b.mu.Unlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return nil, ErrFunctionNotFound } @@ -3989,7 +4033,7 @@ func (b *InMemoryBackend) GetFunctionConcurrency(name string) (*FunctionConcurre b.mu.RLock("GetFunctionConcurrency") defer b.mu.RUnlock() - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return nil, ErrFunctionNotFound } @@ -4007,7 +4051,7 @@ func (b *InMemoryBackend) DeleteFunctionConcurrency(name string) error { b.mu.Lock("DeleteFunctionConcurrency") defer b.mu.Unlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return ErrFunctionNotFound } @@ -4028,7 +4072,7 @@ func (b *InMemoryBackend) PutProvisionedConcurrencyConfig( b.mu.Lock("PutProvisionedConcurrencyConfig") defer b.mu.Unlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return nil, ErrFunctionNotFound } @@ -4047,10 +4091,6 @@ func (b *InMemoryBackend) PutProvisionedConcurrencyConfig( ) } - if _, exists := b.provisionedConcurrencies[name]; !exists { - b.provisionedConcurrencies[name] = make(map[string]*ProvisionedConcurrencyConfig) - } - cfg := &ProvisionedConcurrencyConfig{ AllocatedProvisionedConcurrentExecutions: requested, AvailableProvisionedConcurrentExecutions: requested, @@ -4075,7 +4115,7 @@ func (b *InMemoryBackend) PutProvisionedConcurrencyConfig( b.scheduleProvisionedConcurrencyReady(name, qualifier, b.pcActivationDelay) } - b.provisionedConcurrencies[name][qualifier] = cfg + b.provisionedConcurrencies.Put(cfg) return cfg, nil } @@ -4103,24 +4143,22 @@ func (b *InMemoryBackend) scheduleProvisionedConcurrencyReady(name, qualifier st b.mu.Lock("provisionedConcurrencyReady") defer b.mu.Unlock() - qualifiers, ok := b.provisionedConcurrencies[name] - if !ok { - return - } + key := buildAliasARN(b.region, b.accountID, name, qualifier) - cfg, ok := qualifiers[qualifier] + cfg, ok := b.provisionedConcurrencies.Get(key) if !ok { return } - // Copy-on-write: replace the map entry with a new config so any caller + // Copy-on-write: replace the table entry with a new config so any caller // holding the previous pointer (returned live by Get) never observes a - // concurrent field write. + // concurrent field write. FunctionArn (the table's key) is preserved + // unchanged, so Put replaces the same entry in place. updated := *cfg updated.Status = provisionedConcurrencyReady updated.AvailableProvisionedConcurrentExecutions = updated.RequestedProvisionedConcurrentExecutions updated.LastModified = time.Now().UTC().Format(time.RFC3339) - qualifiers[qualifier] = &updated + b.provisionedConcurrencies.Put(&updated) }) } @@ -4131,16 +4169,11 @@ func (b *InMemoryBackend) GetProvisionedConcurrencyConfig( b.mu.RLock("GetProvisionedConcurrencyConfig") defer b.mu.RUnlock() - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return nil, ErrFunctionNotFound } - qualifiers, ok := b.provisionedConcurrencies[name] - if !ok { - return nil, ErrProvisionedConcurrencyConfigNotFound - } - - cfg, ok := qualifiers[qualifier] + cfg, ok := b.provisionedConcurrencies.Get(buildAliasARN(b.region, b.accountID, name, qualifier)) if !ok { return nil, ErrProvisionedConcurrencyConfigNotFound } @@ -4153,25 +4186,14 @@ func (b *InMemoryBackend) DeleteProvisionedConcurrencyConfig(name, qualifier str b.mu.Lock("DeleteProvisionedConcurrencyConfig") defer b.mu.Unlock() - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return ErrFunctionNotFound } - qualifiers, ok := b.provisionedConcurrencies[name] - if !ok { + if !b.provisionedConcurrencies.Delete(buildAliasARN(b.region, b.accountID, name, qualifier)) { return ErrProvisionedConcurrencyConfigNotFound } - if _, exists := qualifiers[qualifier]; !exists { - return ErrProvisionedConcurrencyConfigNotFound - } - - delete(qualifiers, qualifier) - - if len(qualifiers) == 0 { - delete(b.provisionedConcurrencies, name) - } - return nil } @@ -4182,18 +4204,13 @@ func (b *InMemoryBackend) ListProvisionedConcurrencyConfigs( b.mu.RLock("ListProvisionedConcurrencyConfigs") defer b.mu.RUnlock() - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return nil, ErrFunctionNotFound } - qualifiers := b.provisionedConcurrencies[name] - configs := make([]*ProvisionedConcurrencyConfig, 0, len(qualifiers)) - - for _, cfg := range qualifiers { - configs = append(configs, cfg) - } + configs := b.provisionedConcurrenciesByFunction.Get(name) - return configs, nil + return append([]*ProvisionedConcurrencyConfig(nil), configs...), nil } // Reset clears all in-memory state from the backend. It is used by the @@ -4214,28 +4231,28 @@ func (b *InMemoryBackend) Reset() { rts = append(rts, rt) } - b.functions = make(map[string]*FunctionConfiguration) - b.aliases = make(map[string]map[string]*FunctionAlias) + // Table/Index-backed resources collapse to two registry sweeps instead of + // one make() per map -- see store_setup.go for what each registry holds. + // b.permissions is deliberately NOT on either registry (see store_setup.go + // and persistence.go's DTO handling) so it is reset explicitly. + b.registry.ResetAll() + b.ephemeralRegistry.ResetAll() + b.permissions.Reset() + b.versionCounters = make(map[string]int) b.versions = make(map[string][]*FunctionVersion) b.layers = make(map[string][]*LayerVersion) b.layerVersionCounters = make(map[string]int64) b.layerPolicies = make(map[string]map[int64]map[string]*LayerVersionStatement) - b.eventSourceMappings = make(map[string]*EventSourceMapping) b.esmByFunctionARN = make(map[string]map[string]struct{}) b.versionIndex = make(map[string]map[string]*FunctionVersion) b.eventInvokeConfigs = make(map[string]*FunctionEventInvokeConfig) b.functionConcurrencies = make(map[string]int) b.activeConcurrencies = make(map[string]int) - b.provisionedConcurrencies = make(map[string]map[string]*ProvisionedConcurrencyConfig) b.fisFaults = make(map[string]*FISInvocationFault) b.runtimes = make(map[string]*functionRuntime) b.functionURLServers = make(map[string]*functionURLServer) - b.functionURLConfigs = make(map[string]*FunctionURLConfig) - b.permissions = make(map[string]map[string]*FunctionPermission) - b.codeSigningConfigs = make(map[string]*CodeSigningConfig) b.fnCodeSigningConfigs = make(map[string]string) - b.capacityProviders = make(map[string]*CapacityProvider) b.runtimeManagementConfigs = make(map[string]*RuntimeManagementConfig) b.functionRecursionConfigs = make(map[string]*FunctionRecursionConfig) b.functionScalingConfigs = make(map[string]*FunctionScalingConfig) @@ -4298,10 +4315,11 @@ func (b *InMemoryBackend) collectAndDeleteFunctions(cutoff time.Time) ( var urlServers []*functionURLServer var rts []*functionRuntime - for name, fn := range b.functions { + for _, fn := range b.functions.All() { if !fn.CreatedAt.Before(cutoff) { continue } + name := fn.FunctionName purgedFunctions = append(purgedFunctions, name) if srv, ok := b.functionURLServers[name]; ok { urlServers = append(urlServers, srv) @@ -4318,32 +4336,37 @@ func (b *InMemoryBackend) collectAndDeleteFunctions(cutoff time.Time) ( // deleteFunctionMapsLocked removes all map entries for a function. // Caller must hold b.mu. func (b *InMemoryBackend) deleteFunctionMapsLocked(name string) { - delete(b.functions, name) + b.functions.Delete(name) delete(b.runtimes, name) delete(b.functionURLServers, name) - delete(b.functionURLConfigs, name) - delete(b.aliases, name) + b.functionURLConfigs.Delete(name) + b.deleteAliasesForFunctionLocked(name) delete(b.versionCounters, name) delete(b.versions, name) delete(b.eventInvokeConfigs, name) delete(b.functionConcurrencies, name) delete(b.activeConcurrencies, name) - delete(b.provisionedConcurrencies, name) + b.deleteProvisionedConcurrenciesForFunctionLocked(name) delete(b.fisFaults, name) - delete(b.permissions, name) + // Removes both the unqualified resource-policy and any qualifier-scoped + // policies (e.g. "name:PROD") for the function. Without this, deleting and + // recreating a function with the same name would inherit stale + // qualifier-scoped policy statements — a real leak since permissionMapKey + // scopes policies per qualifier. + b.deletePermissionsForFunctionLocked(name) delete(b.fnCodeSigningConfigs, name) delete(b.runtimeManagementConfigs, name) delete(b.functionRecursionConfigs, name) delete(b.functionScalingConfigs, name) - for id, m := range b.eventSourceMappings { + for _, m := range b.eventSourceMappings.All() { if strings.HasSuffix(m.FunctionARN, ":function:"+name) { if ids, ok := b.esmByFunctionARN[m.FunctionARN]; ok { - delete(ids, id) + delete(ids, m.UUID) if len(ids) == 0 { delete(b.esmByFunctionARN, m.FunctionARN) } } - delete(b.eventSourceMappings, id) + b.eventSourceMappings.Delete(m.UUID) } } delete(b.versionIndex, name) @@ -4377,115 +4400,160 @@ func (b *InMemoryBackend) shutdownPurgedResources( // --- AddPermission / resource-based policy --- +// permissionMapKey returns the b.permissions map key for a function+qualifier +// pair. Real Lambda resource policies are scoped per version/alias when +// AddPermission is called with a Qualifier — the unqualified function-wide +// policy and each qualified policy are entirely independent documents. +func permissionMapKey(functionName, qualifier string) string { + if qualifier == "" { + return functionName + } + + return functionName + ":" + qualifier +} + +// qualifierExistsLocked reports whether qualifier resolves to a known alias +// or published version of name. Caller must hold b.mu (read or write). +func (b *InMemoryBackend) qualifierExistsLocked(name, qualifier string) bool { + if qualifier == "" || qualifier == versionLatest { + return true + } + + if _, ok := b.aliases.Get(aliasKey(name, qualifier)); ok { + return true + } + + if vMap := b.versionIndex[name]; vMap != nil { + if _, ok := vMap[qualifier]; ok { + return true + } + } + + return false +} + +// resolvePermissionTarget normalizes the FunctionName/Qualifier pair used by +// AddPermission, RemovePermission, and GetPolicy. functionName may be a bare +// name, a full/partial ARN, or the "name:qualifier" shorthand; qualifier is +// the separate query-string Qualifier (may be empty). An explicit Qualifier +// query parameter wins over one embedded in functionName. +func resolvePermissionTarget(functionName, qualifier string) (string, string) { + name, arnQualifier := functionNameAndQualifierFromARN(functionName) + if qualifier == "" { + qualifier = arnQualifier + } + + return name, qualifier +} + // AddPermission adds a permission statement to a function's resource-based policy. +// When qualifier is non-empty the statement is scoped to that version/alias, +// matching real Lambda: the invoker must then use the qualified ARN to invoke. func (b *InMemoryBackend) AddPermission( - functionName string, + functionName, qualifier string, input *AddPermissionInput, ) (*AddPermissionOutput, error) { b.mu.Lock("AddPermission") defer b.mu.Unlock() - if strings.HasPrefix(functionName, "arn:aws:lambda:") { - parts := strings.Split(functionName, ":") - functionName = parts[len(parts)-1] + name, qualifier := resolvePermissionTarget(functionName, qualifier) + + if qualifier == versionLatest { + return nil, fmt.Errorf( + "%w: we currently do not support adding policies for $LATEST", + ErrInvalidParameterValue, + ) } - if _, ok := b.functions[functionName]; !ok { + if _, ok := b.functions.Get(name); !ok { return nil, ErrFunctionNotFound } - if b.permissions[functionName] == nil { - b.permissions[functionName] = make(map[string]*FunctionPermission) + if qualifier != "" && !b.qualifierExistsLocked(name, qualifier) { + return nil, ErrVersionNotFound } - if _, exists := b.permissions[functionName][input.StatementID]; exists { + key := permissionMapKey(name, qualifier) + + if _, exists := b.permissions.Get(key + "|" + input.StatementID); exists { return nil, ErrFunctionAlreadyExists } perm := &FunctionPermission{ - StatementID: input.StatementID, - Action: input.Action, - Principal: input.Principal, - SourceArn: input.SourceArn, - SourceAccount: input.SourceAccount, - Effect: "Allow", - FunctionName: functionName, + StatementID: input.StatementID, + Action: input.Action, + Principal: input.Principal, + SourceArn: input.SourceArn, + SourceAccount: input.SourceAccount, + EventSourceToken: input.EventSourceToken, + PrincipalOrgID: input.PrincipalOrgID, + Effect: "Allow", + FunctionName: name, + Qualifier: qualifier, } - b.permissions[functionName][input.StatementID] = perm + b.permissions.Put(perm) - resourceArn := arn.Build( - "lambda", - b.region, - b.accountID, - fmt.Sprintf("function:%s", functionName), - ) + resource := "function:" + name + if qualifier != "" { + resource += ":" + qualifier + } + + resourceArn := arn.Build("lambda", b.region, b.accountID, resource) stmtJSON := buildPermissionStatementJSON(perm, resourceArn) return &AddPermissionOutput{Statement: &stmtJSON}, nil } // RemovePermission removes a permission statement from a function's resource-based policy. -func (b *InMemoryBackend) RemovePermission(functionName, statementID string) error { +func (b *InMemoryBackend) RemovePermission(functionName, qualifier, statementID string) error { b.mu.Lock("RemovePermission") defer b.mu.Unlock() - if strings.HasPrefix(functionName, "arn:aws:lambda:") { - parts := strings.Split(functionName, ":") - functionName = parts[len(parts)-1] - } + name, qualifier := resolvePermissionTarget(functionName, qualifier) - if _, ok := b.functions[functionName]; !ok { + if _, ok := b.functions.Get(name); !ok { return ErrFunctionNotFound } - perms := b.permissions[functionName] - if perms == nil { - return ErrFunctionNotFound - } + key := permissionMapKey(name, qualifier) - if _, ok := perms[statementID]; !ok { + if !b.permissions.Delete(key + "|" + statementID) { return ErrFunctionNotFound } - delete(perms, statementID) - return nil } -// GetPolicy returns the resource-based policy JSON for a function. -func (b *InMemoryBackend) GetPolicy(functionName string) (*GetPolicyOutput, error) { +// GetPolicy returns the resource-based policy JSON for a function, scoped to +// qualifier when non-empty. +func (b *InMemoryBackend) GetPolicy(functionName, qualifier string) (*GetPolicyOutput, error) { b.mu.RLock("GetPolicy") defer b.mu.RUnlock() - if strings.HasPrefix(functionName, "arn:aws:lambda:") { - parts := strings.Split(functionName, ":") - functionName = parts[len(parts)-1] - } + name, qualifier := resolvePermissionTarget(functionName, qualifier) - if _, ok := b.functions[functionName]; !ok { + if _, ok := b.functions.Get(name); !ok { return nil, ErrFunctionNotFound } - perms := b.permissions[functionName] + perms := b.permissionsForTarget(permissionMapKey(name, qualifier)) if len(perms) == 0 { return nil, ErrNoPolicyFound } stmts := make([]string, 0, len(perms)) - resourceArn := arn.Build( - "lambda", - b.region, - b.accountID, - fmt.Sprintf("function:%s", functionName), - ) + resource := "function:" + name + if qualifier != "" { + resource += ":" + qualifier + } + + resourceArn := arn.Build("lambda", b.region, b.accountID, resource) // Sort statements for deterministic output. - sortedPerms := make([]*FunctionPermission, 0, len(perms)) - for _, p := range perms { - sortedPerms = append(sortedPerms, p) - } + sortedPerms := make([]*FunctionPermission, len(perms)) + copy(sortedPerms, perms) sort.Slice(sortedPerms, func(i, j int) bool { return sortedPerms[i].StatementID < sortedPerms[j].StatementID }) @@ -4520,13 +4588,33 @@ func buildPermissionStatementJSON(p *FunctionPermission, resourceArn string) str p.StatementID, principalJSON, p.Action, resourceArn, ) - // Build Condition block for source constraints. - var conditions []string + // Build the Condition block. ArnLike and StringEquals are each a single + // JSON object — SourceAccount, PrincipalOrgID, and EventSourceToken all + // use the StringEquals operator and must be merged into ONE object + // (naively appending separate "StringEquals":{...} entries would emit + // duplicate JSON keys, which real AWS never does). + var arnLike []string if p.SourceArn != "" { - conditions = append(conditions, fmt.Sprintf(`"ArnLike":{"AWS:SourceArn":%q}`, p.SourceArn)) + arnLike = append(arnLike, fmt.Sprintf(`"AWS:SourceArn":%q`, p.SourceArn)) } + + var stringEquals []string if p.SourceAccount != "" { - conditions = append(conditions, fmt.Sprintf(`"StringEquals":{"AWS:SourceAccount":%q}`, p.SourceAccount)) + stringEquals = append(stringEquals, fmt.Sprintf(`"AWS:SourceAccount":%q`, p.SourceAccount)) + } + if p.PrincipalOrgID != "" { + stringEquals = append(stringEquals, fmt.Sprintf(`"aws:PrincipalOrgID":%q`, p.PrincipalOrgID)) + } + if p.EventSourceToken != "" { + stringEquals = append(stringEquals, fmt.Sprintf(`"lambda:EventSourceToken":%q`, p.EventSourceToken)) + } + + var conditions []string + if len(arnLike) > 0 { + conditions = append(conditions, `"ArnLike":{`+strings.Join(arnLike, ",")+`}`) + } + if len(stringEquals) > 0 { + conditions = append(conditions, `"StringEquals":{`+strings.Join(stringEquals, ",")+`}`) } if len(conditions) > 0 { @@ -4567,7 +4655,7 @@ func (b *InMemoryBackend) CreateCodeSigningConfig( cfg.CodeSigningPolicies = &CodeSigningPolicies{UntrustedArtifactOnDeployment: "Warn"} } - b.codeSigningConfigs[cscARN] = cfg + b.codeSigningConfigs.Put(cfg) return cfg, nil } @@ -4577,7 +4665,7 @@ func (b *InMemoryBackend) GetCodeSigningConfig(cscARN string) (*CodeSigningConfi b.mu.RLock("GetCodeSigningConfig") defer b.mu.RUnlock() - cfg, ok := b.codeSigningConfigs[cscARN] + cfg, ok := b.codeSigningConfigs.Get(cscARN) if !ok { return nil, ErrFunctionNotFound } @@ -4590,11 +4678,11 @@ func (b *InMemoryBackend) DeleteCodeSigningConfig(cscARN string) error { b.mu.Lock("DeleteCodeSigningConfig") defer b.mu.Unlock() - if _, ok := b.codeSigningConfigs[cscARN]; !ok { + if _, ok := b.codeSigningConfigs.Get(cscARN); !ok { return ErrFunctionNotFound } - delete(b.codeSigningConfigs, cscARN) + b.codeSigningConfigs.Delete(cscARN) return nil } @@ -4607,7 +4695,7 @@ func (b *InMemoryBackend) UpdateCodeSigningConfig( b.mu.Lock("UpdateCodeSigningConfig") defer b.mu.Unlock() - cfg, ok := b.codeSigningConfigs[cscARN] + cfg, ok := b.codeSigningConfigs.Get(cscARN) if !ok { return nil, ErrFunctionNotFound } @@ -4625,7 +4713,7 @@ func (b *InMemoryBackend) UpdateCodeSigningConfig( } cfg.LastModified = time.Now().UTC().Format(time.RFC3339) - b.codeSigningConfigs[cscARN] = cfg + b.codeSigningConfigs.Put(cfg) return cfg, nil } @@ -4635,10 +4723,7 @@ func (b *InMemoryBackend) ListCodeSigningConfigs() []*CodeSigningConfig { b.mu.RLock("ListCodeSigningConfigs") defer b.mu.RUnlock() - cfgs := make([]*CodeSigningConfig, 0, len(b.codeSigningConfigs)) - for _, cfg := range b.codeSigningConfigs { - cfgs = append(cfgs, cfg) - } + cfgs := b.codeSigningConfigs.All() sort.Slice(cfgs, func(i, j int) bool { return cfgs[i].CodeSigningConfigID < cfgs[j].CodeSigningConfigID @@ -4652,11 +4737,11 @@ func (b *InMemoryBackend) PutFunctionCodeSigningConfig(functionName, cscARN stri b.mu.Lock("PutFunctionCodeSigningConfig") defer b.mu.Unlock() - if _, ok := b.functions[functionName]; !ok { + if _, ok := b.functions.Get(functionName); !ok { return ErrFunctionNotFound } - if _, ok := b.codeSigningConfigs[cscARN]; !ok { + if _, ok := b.codeSigningConfigs.Get(cscARN); !ok { return ErrFunctionNotFound } @@ -4670,7 +4755,7 @@ func (b *InMemoryBackend) GetFunctionCodeSigningConfig(functionName string) (str b.mu.RLock("GetFunctionCodeSigningConfig") defer b.mu.RUnlock() - if _, ok := b.functions[functionName]; !ok { + if _, ok := b.functions.Get(functionName); !ok { return "", ErrFunctionNotFound } @@ -4687,7 +4772,7 @@ func (b *InMemoryBackend) DeleteFunctionCodeSigningConfig(functionName string) e b.mu.Lock("DeleteFunctionCodeSigningConfig") defer b.mu.Unlock() - if _, ok := b.functions[functionName]; !ok { + if _, ok := b.functions.Get(functionName); !ok { return ErrFunctionNotFound } @@ -4701,7 +4786,7 @@ func (b *InMemoryBackend) ListFunctionsByCodeSigningConfig(cscARN string) ([]str b.mu.RLock("ListFunctionsByCodeSigningConfig") defer b.mu.RUnlock() - if _, ok := b.codeSigningConfigs[cscARN]; !ok { + if _, ok := b.codeSigningConfigs.Get(cscARN); !ok { return nil, ErrFunctionNotFound } @@ -4709,7 +4794,7 @@ func (b *InMemoryBackend) ListFunctionsByCodeSigningConfig(cscARN string) ([]str for fnName, arn := range b.fnCodeSigningConfigs { if arn == cscARN { - fn, ok := b.functions[fnName] + fn, ok := b.functions.Get(fnName) if ok { arns = append(arns, fn.FunctionArn) } @@ -4730,7 +4815,7 @@ func (b *InMemoryBackend) CreateCapacityProvider( b.mu.Lock("CreateCapacityProvider") defer b.mu.Unlock() - if _, exists := b.capacityProviders[input.Name]; exists { + if _, exists := b.capacityProviders.Get(input.Name); exists { return nil, ErrFunctionAlreadyExists } @@ -4743,7 +4828,7 @@ func (b *InMemoryBackend) CreateCapacityProvider( LastModifiedTime: now, } - b.capacityProviders[input.Name] = cp + b.capacityProviders.Put(cp) return cp, nil } @@ -4753,7 +4838,7 @@ func (b *InMemoryBackend) GetCapacityProvider(name string) (*CapacityProvider, e b.mu.RLock("GetCapacityProvider") defer b.mu.RUnlock() - cp, ok := b.capacityProviders[name] + cp, ok := b.capacityProviders.Get(name) if !ok { return nil, ErrFunctionNotFound } @@ -4766,11 +4851,11 @@ func (b *InMemoryBackend) DeleteCapacityProvider(name string) error { b.mu.Lock("DeleteCapacityProvider") defer b.mu.Unlock() - if _, ok := b.capacityProviders[name]; !ok { + if _, ok := b.capacityProviders.Get(name); !ok { return ErrFunctionNotFound } - delete(b.capacityProviders, name) + b.capacityProviders.Delete(name) return nil } @@ -4783,7 +4868,7 @@ func (b *InMemoryBackend) UpdateCapacityProvider( b.mu.Lock("UpdateCapacityProvider") defer b.mu.Unlock() - cp, ok := b.capacityProviders[name] + cp, ok := b.capacityProviders.Get(name) if !ok { return nil, ErrFunctionNotFound } @@ -4793,7 +4878,7 @@ func (b *InMemoryBackend) UpdateCapacityProvider( } cp.LastModifiedTime = time.Now().UTC().Format(time.RFC3339) - b.capacityProviders[name] = cp + b.capacityProviders.Put(cp) return cp, nil } @@ -4803,10 +4888,7 @@ func (b *InMemoryBackend) ListCapacityProviders() []*CapacityProvider { b.mu.RLock("ListCapacityProviders") defer b.mu.RUnlock() - cps := make([]*CapacityProvider, 0, len(b.capacityProviders)) - for _, cp := range b.capacityProviders { - cps = append(cps, cp) - } + cps := b.capacityProviders.All() sort.Slice(cps, func(i, j int) bool { return cps[i].Name < cps[j].Name @@ -4827,7 +4909,7 @@ func (b *InMemoryBackend) SeedCapacityProviderFunctionVersions( b.mu.Lock("SeedCapacityProviderFunctionVersions") defer b.mu.Unlock() - cp, ok := b.capacityProviders[name] + cp, ok := b.capacityProviders.Get(name) if !ok { return ErrFunctionNotFound } @@ -4849,7 +4931,7 @@ func (b *InMemoryBackend) ListFunctionVersionsByCapacityProvider( b.mu.RLock("ListFunctionVersionsByCapacityProvider") defer b.mu.RUnlock() - cp, ok := b.capacityProviders[name] + cp, ok := b.capacityProviders.Get(name) if !ok { return page.Page[string]{}, ErrFunctionNotFound } @@ -4880,10 +4962,10 @@ func (b *InMemoryBackend) GetAccountSettings() *AccountSettingsOutput { b.mu.RLock("GetAccountSettings") defer b.mu.RUnlock() - fnCount := len(b.functions) + fnCount := b.functions.Len() totalCodeSize := int64(0) - for _, fn := range b.functions { + for _, fn := range b.functions.All() { totalCodeSize += fn.CodeSize } @@ -4917,7 +4999,7 @@ func (b *InMemoryBackend) UpdateFunctionURLConfig( b.mu.Lock("UpdateFunctionURLConfig") defer b.mu.Unlock() - cfg, ok := b.functionURLConfigs[functionName] + cfg, ok := b.functionURLConfigs.Get(functionName) if !ok { return nil, ErrFunctionURLNotFound } @@ -4931,7 +5013,7 @@ func (b *InMemoryBackend) UpdateFunctionURLConfig( } cfg.LastModifiedTime = time.Now().UTC().Format(time.RFC3339) - b.functionURLConfigs[functionName] = cfg + b.functionURLConfigs.Put(cfg) return cfg, nil } @@ -4941,10 +5023,7 @@ func (b *InMemoryBackend) ListFunctionURLConfigs() []*FunctionURLConfig { b.mu.RLock("ListFunctionURLConfigs") defer b.mu.RUnlock() - cfgs := make([]*FunctionURLConfig, 0, len(b.functionURLConfigs)) - for _, cfg := range b.functionURLConfigs { - cfgs = append(cfgs, cfg) - } + cfgs := b.functionURLConfigs.All() sort.Slice(cfgs, func(i, j int) bool { return cfgs[i].FunctionArn < cfgs[j].FunctionArn @@ -5040,7 +5119,7 @@ func (b *InMemoryBackend) UpdateEventSourceMapping( ) (*EventSourceMapping, error) { b.mu.Lock("UpdateEventSourceMapping") - esm, ok := b.eventSourceMappings[id] + esm, ok := b.eventSourceMappings.Get(id) if !ok { b.mu.Unlock() @@ -5066,7 +5145,7 @@ func (b *InMemoryBackend) GetRuntimeManagementConfig( b.mu.RLock("GetRuntimeManagementConfig") defer b.mu.RUnlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return nil, ErrFunctionNotFound } @@ -5090,7 +5169,7 @@ func (b *InMemoryBackend) PutRuntimeManagementConfig( b.mu.Lock("PutRuntimeManagementConfig") defer b.mu.Unlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return nil, ErrFunctionNotFound } @@ -5118,7 +5197,7 @@ func (b *InMemoryBackend) GetFunctionRecursionConfig( b.mu.RLock("GetFunctionRecursionConfig") defer b.mu.RUnlock() - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return nil, ErrFunctionNotFound } @@ -5138,7 +5217,7 @@ func (b *InMemoryBackend) PutFunctionRecursionConfig( b.mu.Lock("PutFunctionRecursionConfig") defer b.mu.Unlock() - if _, ok := b.functions[name]; !ok { + if _, ok := b.functions.Get(name); !ok { return nil, ErrFunctionNotFound } @@ -5157,7 +5236,7 @@ func (b *InMemoryBackend) GetFunctionScalingConfig(name string) (*FunctionScalin b.mu.RLock("GetFunctionScalingConfig") defer b.mu.RUnlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return nil, ErrFunctionNotFound } @@ -5181,7 +5260,7 @@ func (b *InMemoryBackend) PutFunctionScalingConfig( b.mu.Lock("PutFunctionScalingConfig") defer b.mu.Unlock() - fn, ok := b.functions[name] + fn, ok := b.functions.Get(name) if !ok { return nil, ErrFunctionNotFound } @@ -5212,7 +5291,7 @@ func (b *InMemoryBackend) TagResource(functionName string, tags map[string]strin b.mu.Lock("TagResource") defer b.mu.Unlock() - fn, ok := b.functions[functionName] + fn, ok := b.functions.Get(functionName) if !ok { return ErrFunctionNotFound } @@ -5228,7 +5307,7 @@ func (b *InMemoryBackend) UntagResource(functionName string, tagKeys []string) e b.mu.Lock("UntagResource") defer b.mu.Unlock() - fn, ok := b.functions[functionName] + fn, ok := b.functions.Get(functionName) if !ok { return ErrFunctionNotFound } diff --git a/services/lambda/batch2_test.go b/services/lambda/batch2_test.go index 87bd0f3e8..4c0e283a2 100644 --- a/services/lambda/batch2_test.go +++ b/services/lambda/batch2_test.go @@ -1433,9 +1433,12 @@ func TestBatch2_Permission_FullLifecycle(t *testing.T) { assert.Contains(t, *pol.Policy, "AllowS3") assert.Contains(t, *pol.Policy, "AllowSNS") - // Remove permission + // Remove permission. Real Lambda sends StatementId as a URI path segment + // (/policy/{StatementId}), never as a "?StatementId=" query parameter — + // this used to be a query string here, which encoded the pre-fix (wrong) + // wire shape and could never be hit by a real aws-sdk-go-v2 client. delRec := callInMemoryHandler(t, h, http.MethodDelete, - "/2015-03-31/functions/perm-fn/policy?StatementId=AllowS3", "") + "/2015-03-31/functions/perm-fn/policy/AllowS3", "") assert.Equal(t, http.StatusNoContent, delRec.Code) // Get policy after remove @@ -1449,15 +1452,47 @@ func TestBatch2_Permission_FullLifecycle(t *testing.T) { assert.Contains(t, *pol2.Policy, "AllowSNS") } -func TestBatch2_Permission_NoPolicy_Returns404(t *testing.T) { - t.Parallel() +// Test_Permission_ErrorCases covers GetPolicy/RemovePermission error paths that +// don't fit the full add/get/remove lifecycle above. +func Test_Permission_ErrorCases(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + fnName string + method string + pathSuffix string + wantStatus int + }{ + { + name: "GetPolicy on a function with no permissions returns 404", + fnName: "nopolicy-fn", + method: http.MethodGet, + pathSuffix: "/policy", + wantStatus: http.StatusNotFound, + }, + { + // Wire format: StatementId is a path segment, not a query param. + name: "RemovePermission for a nonexistent statement returns 404", + fnName: "rm-404-fn", + method: http.MethodDelete, + pathSuffix: "/policy/nonexistent-stmt", + wantStatus: http.StatusNotFound, + }, + } - h, _ := newInMemoryHandler(t) - createFunctionForTest(t, h, "nopolicy-fn") + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() - rec := callInMemoryHandler(t, h, http.MethodGet, - "/2015-03-31/functions/nopolicy-fn/policy", "") - assert.Equal(t, http.StatusNotFound, rec.Code) + h, _ := newInMemoryHandler(t) + createFunctionForTest(t, h, tc.fnName) + + rec := callInMemoryHandler(t, h, tc.method, + "/2015-03-31/functions/"+tc.fnName+tc.pathSuffix, "") + assert.Equal(t, tc.wantStatus, rec.Code) + }) + } } func TestBatch2_Permission_SourceArn(t *testing.T) { @@ -1485,16 +1520,82 @@ func TestBatch2_Permission_SourceArn(t *testing.T) { assert.Contains(t, *addOut.Statement, "AllowSpecificBucket") } -func TestBatch2_Permission_RemoveNotFound(t *testing.T) { - t.Parallel() - - h, _ := newInMemoryHandler(t) - createFunctionForTest(t, h, "rm-404-fn") +// Test_AddPermission_Qualifier verifies real Lambda's per-qualifier +// resource-based policies (parity-sweep-3 fix): a statement added with +// Qualifier= is stored separately from the unqualified policy and +// only visible via GetPolicy scoped to that same qualifier, and AddPermission +// rejects Qualifier=$LATEST exactly as AWS does ("Lambda does not support +// adding policies to version $LATEST"). Each case builds its own handler, +// backend, and function from scratch — no case depends on state left behind +// by another, and every case is safe to run in parallel. +func Test_AddPermission_Qualifier(t *testing.T) { + t.Parallel() + + const fnName = "qual-fn" + + tests := []struct { + name string + qualifier string + publishVersion bool + wantAddStatus int + }{ + { + name: "rejects Qualifier=$LATEST", + qualifier: "$LATEST", + wantAddStatus: http.StatusBadRequest, + }, + { + name: "rejects an unknown qualifier", + qualifier: "99", + wantAddStatus: http.StatusNotFound, + }, + { + name: "scoped to a published version succeeds and is isolated from the unqualified policy", + qualifier: "1", + publishVersion: true, + wantAddStatus: http.StatusCreated, + }, + } - rec := callInMemoryHandler(t, h, http.MethodDelete, - "/2015-03-31/functions/rm-404-fn/policy?StatementId=nonexistent-stmt", "") - // Backend returns error for missing statement; handler maps to 500 or 404 - assert.NotEqual(t, http.StatusNoContent, rec.Code) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h, bk := newInMemoryHandler(t) + createFunctionForTest(t, h, fnName) + + if tc.publishVersion { + _, pubErr := bk.PublishVersion(fnName, "") + require.NoError(t, pubErr) + } + + addRec := callInMemoryHandler(t, h, http.MethodPost, + "/2015-03-31/functions/"+fnName+"/policy?Qualifier="+tc.qualifier, + `{"StatementId":"s-stmt","Action":"lambda:InvokeFunction","Principal":"s3.amazonaws.com"}`) + require.Equal(t, tc.wantAddStatus, addRec.Code) + + if tc.wantAddStatus != http.StatusCreated { + return + } + + // The qualified statement must not leak into the unqualified policy: + // no unqualified statement was ever added, so GetPolicy 404s. + unqualRec := callInMemoryHandler(t, h, http.MethodGet, + "/2015-03-31/functions/"+fnName+"/policy", "") + assert.Equal(t, http.StatusNotFound, unqualRec.Code) + + // ...but it is visible when GetPolicy is scoped to the same qualifier, + // and the Resource ARN in the statement carries the qualifier suffix. + qualRec := callInMemoryHandler(t, h, http.MethodGet, + "/2015-03-31/functions/"+fnName+"/policy?Qualifier="+tc.qualifier, "") + require.Equal(t, http.StatusOK, qualRec.Code) + + qualOut := lambdaParseBody(t, qualRec) + policyStr, _ := qualOut["Policy"].(string) + assert.Contains(t, policyStr, "s-stmt") + assert.Contains(t, policyStr, "arn:aws:lambda:us-east-1:000000000000:function:qual-fn:1") + }) + } } // ============================================================ diff --git a/services/lambda/esm_test.go b/services/lambda/esm_test.go index 85c8eb618..20fd05203 100644 --- a/services/lambda/esm_test.go +++ b/services/lambda/esm_test.go @@ -483,9 +483,22 @@ func TestLambda_FunctionNameFromARN(t *testing.T) { expected: "", }, { + // Bug fix (parity-sweep-3): the old implementation SplitN'd on ":" + // with a fixed part count and returned "" for any input shorter + // than a full ARN, silently dropping bare function names. A bare + // name is a legitimate input (callers fall back to it when the + // ARN doesn't parse) and must round-trip unchanged. name: "just a name", arn: "my-function", - expected: "", + expected: "my-function", + }, + { + // Qualified ARN: the qualifier segment must be stripped, not + // glued onto the name (previously "arn:...:function:my-func:PROD" + // incorrectly returned "PROD" as the "name"). + name: "qualified ARN with alias", + arn: "arn:aws:lambda:us-east-1:000000000000:function:my-function:PROD", + expected: "my-function", }, } diff --git a/services/lambda/event_source_poller.go b/services/lambda/event_source_poller.go index 158cd46ab..725ede6f7 100644 --- a/services/lambda/event_source_poller.go +++ b/services/lambda/event_source_poller.go @@ -16,8 +16,6 @@ import ( const ( // arnKinesisPartCount is the number of colon-separated parts in a Kinesis ARN. arnKinesisPartCount = 6 - // arnLambdaPartCount is the number of colon-separated parts in a Lambda ARN. - arnLambdaPartCount = 7 // millisToSeconds converts Unix milliseconds to a float64 second timestamp. millisToSeconds = 1000.0 ) @@ -324,6 +322,29 @@ func (p *EventSourcePoller) sweepStaleIterators(activeUUIDs map[string]struct{}) } } +// invokeESMFunctionEvent delivers an ESM batch to fnName as an async (Event) +// invocation, honoring an optional qualifier (version/alias) so that mappings +// created against a qualified function ARN (arn:...:function:my-func:PROD) +// invoke that specific version/alias rather than $LATEST — matching real +// Lambda ESM behaviour. Bug fix (parity-sweep-3): previously every ESM +// delivery path called InvokeFunction unconditionally, silently ignoring any +// qualifier and always invoking $LATEST. +func (p *EventSourcePoller) invokeESMFunctionEvent( + ctx context.Context, fnName, qualifier string, payload []byte, +) ([]byte, error) { + if qualifier == "" { + result, _, err := p.lambdaBackend.InvokeFunction(ctx, fnName, InvocationTypeEvent, payload) + + return result, err + } + + result, _, _, _, err := p.lambdaBackend.InvokeFunctionWithQualifier( + ctx, fnName, qualifier, "", "", InvocationTypeEvent, payload, + ) + + return result, err +} + // RemoveMapping removes any per-mapping poller state for the given ESM UUID. func (p *EventSourcePoller) RemoveMapping(uuid string) { p.mu.Lock("RemoveMapping") @@ -461,13 +482,13 @@ func (p *EventSourcePoller) invokeLambda( return } - // Extract function name from ARN - fnName := functionNameFromARN(m.FunctionARN) + // Extract function name (and optional version/alias qualifier) from ARN. + fnName, qualifier := functionNameAndQualifierFromARN(m.FunctionARN) if fnName == "" { fnName = m.FunctionARN } - _, _, err = p.lambdaBackend.InvokeFunction(ctx, fnName, InvocationTypeEvent, payload) + _, err = p.invokeESMFunctionEvent(ctx, fnName, qualifier, payload) if err != nil { logger.Load(ctx).WarnContext(ctx, "event source poller: Lambda invocation failed", "function", fnName, "stream", streamName, "error", err) @@ -500,15 +521,20 @@ func streamNameFromARN(arn string) string { return last[len(streamPrefix):] } -// functionNameFromARN extracts the function name from a Lambda ARN. -// Example: arn:aws:lambda:us-east-1:000000000000:function:my-func → my-func. +// functionNameFromARN extracts the bare function name from a Lambda ARN, +// stripping any trailing version/alias qualifier segment. +// Example: arn:aws:lambda:us-east-1:000000000000:function:my-func:PROD → my-func. +// +// Bug fix (parity-sweep-3): this used to SplitN on ":" with a fixed part +// count, which for a qualified ARN left the qualifier glued onto the name +// (e.g. "my-func:PROD") and broke every existence/lookup check that expects +// a bare function name (janitor health checks, tag lookups). It now +// delegates to functionNameAndQualifierFromARN and returns just the name; +// callers that need the qualifier (event delivery) use that helper directly. func functionNameFromARN(arn string) string { - parts := strings.SplitN(arn, ":", arnLambdaPartCount) - if len(parts) < arnLambdaPartCount { - return "" - } + name, _ := functionNameAndQualifierFromARN(arn) - return parts[arnLambdaPartCount-1] + return name } // isSQSARN reports whether the given ARN identifies an SQS queue. @@ -663,7 +689,7 @@ func (p *EventSourcePoller) invokeLambdaForDDB( return } - fnName := functionNameFromARN(m.FunctionARN) + fnName, qualifier := functionNameAndQualifierFromARN(m.FunctionARN) if fnName == "" { fnName = m.FunctionARN } @@ -672,7 +698,7 @@ func (p *EventSourcePoller) invokeLambdaForDDB( if p.ddbInvoker != nil { invokeErr = p.ddbInvoker(ctx, fnName, payload) } else { - _, _, invokeErr = p.lambdaBackend.InvokeFunction(ctx, fnName, InvocationTypeEvent, payload) + _, invokeErr = p.invokeESMFunctionEvent(ctx, fnName, qualifier, payload) } if invokeErr != nil { @@ -851,7 +877,7 @@ func (p *EventSourcePoller) invokeLambdaForSQS( return nil, err } - fnName := functionNameFromARN(m.FunctionARN) + fnName, qualifier := functionNameAndQualifierFromARN(m.FunctionARN) if fnName == "" { fnName = m.FunctionARN } @@ -866,7 +892,7 @@ func (p *EventSourcePoller) invokeLambdaForSQS( if p.sqsInvoker != nil { respBody, invokeErr = p.sqsInvoker(ctx, fnName) } else { - respBody, _, invokeErr = p.lambdaBackend.InvokeFunction(ctx, fnName, InvocationTypeEvent, payload) + respBody, invokeErr = p.invokeESMFunctionEvent(ctx, fnName, qualifier, payload) } if invokeErr != nil { diff --git a/services/lambda/export_test.go b/services/lambda/export_test.go index 57a2ca0bc..be9b909f3 100644 --- a/services/lambda/export_test.go +++ b/services/lambda/export_test.go @@ -61,6 +61,12 @@ func StreamNameFromARN(arn string) string { return streamNameFromARN(arn) } // FunctionNameFromARN exports the internal functionNameFromARN function for testing. func FunctionNameFromARN(arn string) string { return functionNameFromARN(arn) } +// FunctionNameAndQualifierFromARN exports the internal +// functionNameAndQualifierFromARN function for testing. +func FunctionNameAndQualifierFromARN(name string) (string, string) { + return functionNameAndQualifierFromARN(name) +} + // PollOnce triggers a single poll cycle on the given EventSourcePoller. func PollOnce(ctx context.Context, p *EventSourcePoller) { p.poll(ctx) } @@ -462,10 +468,19 @@ func BuildFunctionURLHandler(b *InMemoryBackend, functionName string) http.Handl } // SetFunctionURLConfigForTest inserts a function URL config directly for testing. +// The store.Table backing b.functionURLConfigs derives its key from +// cfg.FunctionArn (see functionURLConfigsKeyFn in store_setup.go), so callers +// that omit FunctionArn get it defaulted here to match functionName -- exactly +// what CreateFunctionURLConfig itself would have set via buildURLARN. func SetFunctionURLConfigForTest(b *InMemoryBackend, functionName string, cfg *FunctionURLConfig) { b.mu.Lock("SetFunctionURLConfigForTest") defer b.mu.Unlock() - b.functionURLConfigs[functionName] = cfg + + if cfg.FunctionArn == "" { + cfg.FunctionArn = buildURLARN(b.region, b.accountID, functionName) + } + + b.functionURLConfigs.Put(cfg) } // AsyncOutcomeForTest describes a completed async invocation for delivery testing. @@ -499,7 +514,7 @@ func GetFunctionStateForTest(b *InMemoryBackend, name string) FunctionState { b.mu.RLock("GetFunctionStateForTest") defer b.mu.RUnlock() - if fn, ok := b.functions[name]; ok { + if fn, ok := b.functions.Get(name); ok { return fn.State } diff --git a/services/lambda/handler.go b/services/lambda/handler.go index b43aa1a33..561f137c9 100644 --- a/services/lambda/handler.go +++ b/services/lambda/handler.go @@ -134,7 +134,7 @@ var lambdaOpRoutes = []routeSpec{ {http.MethodDelete, hasSuffixURL, "DeleteFunctionURLConfig"}, {http.MethodPost, hasSuffixPolicy, "AddPermission"}, {http.MethodGet, hasSuffixPolicy, "GetPolicy"}, - {http.MethodDelete, hasSuffixPolicy, "RemovePermission"}, + {http.MethodDelete, hasPolicyStatementSuffix, "RemovePermission"}, {http.MethodPut, hasSuffixEventInvokeConfig, "PutFunctionEventInvokeConfig"}, {http.MethodGet, hasSuffixEventInvokeConfig, "GetFunctionEventInvokeConfig"}, {http.MethodPost, hasSuffixEventInvokeConfig, "UpdateFunctionEventInvokeConfig"}, @@ -178,7 +178,36 @@ func hasSuffixEventInvokeConfigs(rest string) bool { func hasSuffixCodeSigningConfig(rest string) bool { return strings.HasSuffix(rest, "/code-signing-config") } -func hasSuffixPolicy(rest string) bool { return strings.HasSuffix(rest, "/policy") } +func hasSuffixPolicy(rest string) bool { return strings.HasSuffix(rest, "/policy") } + +// hasPolicyStatementSuffix reports whether rest is a RemovePermission path of +// the form "/{name}/policy/{statementId}" — the real wire format, where the +// SDK embeds StatementId as a URI path segment (never a query parameter). +func hasPolicyStatementSuffix(rest string) bool { + trimmed := strings.TrimPrefix(rest, "/") + parts := strings.SplitN(trimmed, "/", 3) //nolint:mnd // name, "policy", statementId + + return len(parts) == 3 && parts[1] == "policy" && parts[2] != "" +} + +// extractNameAndPolicyStatement extracts the function name and StatementId +// from a RemovePermission path of the form "/{name}/policy/{statementId}". +func extractNameAndPolicyStatement(rest string) (string, string) { + trimmed := strings.TrimPrefix(rest, "/") + parts := strings.SplitN(trimmed, "/", 3) //nolint:mnd // name, "policy", statementId + + var name, statementID string + + if len(parts) >= 1 { + name = parts[0] + } + + if len(parts) >= 3 { //nolint:mnd // parts: name, "policy", statementId + statementID = parts[2] + } + + return name, statementID +} func hasSuffixVersions(rest string) bool { return strings.HasSuffix(rest, "/versions") } func hasSuffixInvokeAsync(rest string) bool { return strings.HasSuffix(rest, "/invoke-async/") } func hasSuffixResponseStream(rest string) bool { @@ -767,11 +796,11 @@ func (h *Handler) buildFunctionURLPolicyRoutes() []handlerEntry { }, { method: http.MethodDelete, - match: hasSuffixPolicy, + match: hasPolicyStatementSuffix, execute: func(c *echo.Context, rest string) error { - name := strings.TrimSuffix(strings.TrimPrefix(rest, "/"), "/policy") + name, statementID := extractNameAndPolicyStatement(rest) - return h.handleRemovePermission(c, name) + return h.handleRemovePermission(c, name, statementID) }, }, } @@ -3183,9 +3212,11 @@ func (h *Handler) handleAddPermission(c *echo.Context, name string) error { return h.writeError(c, http.StatusBadRequest, "InvalidParameterValueException", "Principal is required") } - out, addErr := lambdaBk.AddPermission(name, &input) + qualifier := c.Request().URL.Query().Get("Qualifier") + + out, addErr := lambdaBk.AddPermission(name, qualifier, &input) if addErr != nil { - if errors.Is(addErr, ErrFunctionNotFound) { + if errors.Is(addErr, ErrFunctionNotFound) || errors.Is(addErr, ErrVersionNotFound) { return h.writeError(c, http.StatusNotFound, "ResourceNotFoundException", "Function not found: "+name) } @@ -3195,20 +3226,26 @@ func (h *Handler) handleAddPermission(c *echo.Context, name string) error { "Permission already exists: "+input.StatementID) } + if errors.Is(addErr, ErrInvalidParameterValue) { + return h.writeError(c, http.StatusBadRequest, "InvalidParameterValueException", addErr.Error()) + } + return h.writeError(c, http.StatusInternalServerError, "ServiceException", addErr.Error()) } return c.JSON(http.StatusCreated, out) } -// handleGetPolicy handles GET /2015-03-31/functions/{name}/policy. +// handleGetPolicy handles GET /2015-03-31/functions/{name}/policy?Qualifier=xxx. func (h *Handler) handleGetPolicy(c *echo.Context, name string) error { lambdaBk, ok := h.Backend.(*InMemoryBackend) if !ok { return h.writeError(c, http.StatusInternalServerError, "ServiceException", "backend not available") } - out, err := lambdaBk.GetPolicy(name) + qualifier := c.Request().URL.Query().Get("Qualifier") + + out, err := lambdaBk.GetPolicy(name, qualifier) if err != nil { if errors.Is(err, ErrFunctionNotFound) { return h.writeError(c, http.StatusNotFound, "ResourceNotFoundException", @@ -3226,19 +3263,29 @@ func (h *Handler) handleGetPolicy(c *echo.Context, name string) error { return c.JSON(http.StatusOK, out) } -// handleRemovePermission handles DELETE /2015-03-31/functions/{name}/policy?StatementId=xxx. -func (h *Handler) handleRemovePermission(c *echo.Context, name string) error { +// handleRemovePermission handles +// DELETE /2015-03-31/functions/{name}/policy/{statementID}?Qualifier=xxx. +// +// Bug fix (parity-sweep-3): the real Lambda SDK sends StatementId as a URI path +// segment (see RemovePermissionInput's HTTP binding), never as a query +// parameter. The previous implementation read StatementId from the query +// string, so this route only matched when no real SDK client would ever call +// it — it always 400'd against genuine aws-sdk-go-v2 traffic. statementID is +// now extracted from the path by the route matcher (hasPolicyStatementSuffix +// + extractNameAndPolicyStatement). +func (h *Handler) handleRemovePermission(c *echo.Context, name, statementID string) error { lambdaBk, ok := h.Backend.(*InMemoryBackend) if !ok { return h.writeError(c, http.StatusInternalServerError, "ServiceException", "backend not available") } - statementID := c.Request().URL.Query().Get("StatementId") if statementID == "" { return h.writeError(c, http.StatusBadRequest, "InvalidParameterValueException", "StatementId is required") } - if err := lambdaBk.RemovePermission(name, statementID); err != nil { + qualifier := c.Request().URL.Query().Get("Qualifier") + + if err := lambdaBk.RemovePermission(name, qualifier, statementID); err != nil { if errors.Is(err, ErrFunctionNotFound) { return h.writeError(c, http.StatusNotFound, "ResourceNotFoundException", "Function not found: "+name) diff --git a/services/lambda/janitor.go b/services/lambda/janitor.go index c1f3dae23..09ba4f888 100644 --- a/services/lambda/janitor.go +++ b/services/lambda/janitor.go @@ -95,12 +95,12 @@ type esmHealthEntry struct { // deleted-function mapping transitions to a degraded state. func (j *Janitor) sweepESMs(ctx context.Context) { j.Backend.mu.RLock("JanitorSweepESMs") - esmCount := len(j.Backend.eventSourceMappings) + esmCount := j.Backend.eventSourceMappings.Len() var toCheck []esmHealthEntry - for id, esm := range j.Backend.eventSourceMappings { + for _, esm := range j.Backend.eventSourceMappings.All() { if esm.State == ESMStateEnabled { - toCheck = append(toCheck, esmHealthEntry{uuid: id, functionARN: esm.FunctionARN}) + toCheck = append(toCheck, esmHealthEntry{uuid: esm.UUID, functionARN: esm.FunctionARN}) } } j.Backend.mu.RUnlock() @@ -117,7 +117,7 @@ func (j *Janitor) sweepESMs(ctx context.Context) { if len(degraded) > 0 { j.Backend.mu.Lock("JanitorSweepESMs.degrade") for _, id := range degraded { - if esm, ok := j.Backend.eventSourceMappings[id]; ok { + if esm, ok := j.Backend.eventSourceMappings.Get(id); ok { esm.LastProcessingResult = "PROBLEM" } } diff --git a/services/lambda/models.go b/services/lambda/models.go index 06a67fb91..b4f51439a 100644 --- a/services/lambda/models.go +++ b/services/lambda/models.go @@ -521,23 +521,30 @@ type ListProvisionedConcurrencyConfigsOutput struct { } // FunctionPermission is a single statement in a function resource-based policy. +// Qualifier records the version/alias this statement is scoped to (empty for the +// unqualified function-wide policy), matching real AWS per-qualifier policies. type FunctionPermission struct { - Action string `json:"Action"` - Effect string `json:"Effect"` - FunctionName string `json:"-"` - Principal string `json:"Principal"` - SourceAccount string `json:"SourceAccount,omitempty"` - SourceArn string `json:"SourceArn,omitempty"` - StatementID string `json:"Sid"` + Action string `json:"Action"` + Effect string `json:"Effect"` + FunctionName string `json:"-"` + Qualifier string `json:"-"` + Principal string `json:"Principal"` + SourceAccount string `json:"SourceAccount,omitempty"` + SourceArn string `json:"SourceArn,omitempty"` + EventSourceToken string `json:"EventSourceToken,omitempty"` + PrincipalOrgID string `json:"PrincipalOrgID,omitempty"` + StatementID string `json:"Sid"` } // AddPermissionInput is the request body for AddPermission. type AddPermissionInput struct { - Action string `json:"Action"` - Principal string `json:"Principal"` - StatementID string `json:"StatementId"` - SourceAccount string `json:"SourceAccount,omitempty"` - SourceArn string `json:"SourceArn,omitempty"` + Action string `json:"Action"` + Principal string `json:"Principal"` + StatementID string `json:"StatementId"` + SourceAccount string `json:"SourceAccount,omitempty"` + SourceArn string `json:"SourceArn,omitempty"` + EventSourceToken string `json:"EventSourceToken,omitempty"` + PrincipalOrgID string `json:"PrincipalOrgID,omitempty"` } // AddPermissionOutput is the response for AddPermission. diff --git a/services/lambda/parity_fixes_test.go b/services/lambda/parity_fixes_test.go index e1bc020c9..5f7c8cde1 100644 --- a/services/lambda/parity_fixes_test.go +++ b/services/lambda/parity_fixes_test.go @@ -552,6 +552,48 @@ func TestSweepESMs_MarksDegradedESM(t *testing.T) { } } +// TestSweepESMs_QualifiedFunctionARN_NotMarkedDegraded proves the +// parity-sweep-3 fix to esmFunctionName/functionNameFromARN: an ESM created +// against a qualified function ARN (a common real pattern — point the +// trigger at a specific published version/alias rather than $LATEST) must +// not be treated as pointing at a missing function. +// +// Before the fix, esmFunctionName took only the LAST colon-separated ARN +// segment, so "arn:...:function:qual-esm-fn:1" was stored as +// FunctionARN=".../function:1" — the qualifier "1" mistaken for the function +// name. The janitor's existence check then looked up a nonexistent function +// literally named "1" and permanently marked the mapping "PROBLEM". +func TestSweepESMs_QualifiedFunctionARN_NotMarkedDegraded(t *testing.T) { + t.Parallel() + + bk := newParityBackend(t) + require.NoError(t, bk.CreateFunction(makeMinimalFunction("qual-esm-fn"))) + + _, err := bk.PublishVersion("qual-esm-fn", "") + require.NoError(t, err) + + esm, err := bk.CreateEventSourceMapping(&lambda.CreateEventSourceMappingInput{ + EventSourceARN: "arn:aws:sqs:us-east-1:123456789012:test-queue", + FunctionName: "arn:aws:lambda:us-east-1:123456789012:function:qual-esm-fn:1", + Enabled: true, + }) + require.NoError(t, err) + require.Equal(t, "arn:aws:lambda:us-east-1:123456789012:function:qual-esm-fn:1", esm.FunctionARN, + "FunctionARN must preserve the name:qualifier suffix, matching real Lambda's echo") + + name, qualifier := lambda.FunctionNameAndQualifierFromARN(esm.FunctionARN) + assert.Equal(t, "qual-esm-fn", name) + assert.Equal(t, "1", qualifier) + + j := lambda.NewJanitor(bk, lambda.DefaultSettings()) + lambda.SweepESMsForTest(context.Background(), j) + + got, err := bk.GetEventSourceMapping(esm.UUID) + require.NoError(t, err) + assert.NotEqual(t, "PROBLEM", got.LastProcessingResult, + "a qualified-ARN ESM's function must resolve correctly, not be treated as missing") +} + func TestSweepESMs_SkipsDisabledESMs(t *testing.T) { t.Parallel() diff --git a/services/lambda/persistence.go b/services/lambda/persistence.go index 2bc989e81..03adc82ca 100644 --- a/services/lambda/persistence.go +++ b/services/lambda/persistence.go @@ -3,25 +3,107 @@ package lambda import ( "context" "encoding/json" + "fmt" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) +// lambdaSnapshotVersion identifies the shape of backendSnapshot's Tables blob +// (i.e. the set/shape of resources registered on b.registry -- see +// registerAllTables in store_setup.go). It must be bumped whenever a change +// there would make an older snapshot unsafe to decode as the current shape. +// Restore compares this against the persisted value and discards (rather +// than attempts to partially decode) any mismatch -- see Restore below. This +// mirrors the services/sqs pilot (commit 0f09d77c) and the services/ec2 +// conversion (commit 12e611a4). +const lambdaSnapshotVersion = 1 + +// permissionSnapshot is the DTO used to serialise a *FunctionPermission. +// FunctionPermission.FunctionName and .Qualifier are tagged `json:"-"` (they +// are internal bookkeeping the AWS response shape never carries), which means +// they -- along with permissionKeyFn's ability to reconstruct a +// b.permissions key from them -- would be silently lost if a +// *store.Table[FunctionPermission] were round-tripped through +// store.Registry's generic JSON encoding. permissionSnapshot instead gives +// those two fields real json tags purely for the on-disk shape, following the +// DTO-registry pattern the services/sqs pilot (commit 0f09d77c) established +// for exactly this "dirty struct" case. +type permissionSnapshot struct { + StatementID string `json:"statementId"` + FunctionName string `json:"functionName"` + Qualifier string `json:"qualifier,omitempty"` + Action string `json:"action"` + Effect string `json:"effect"` + Principal string `json:"principal"` + SourceAccount string `json:"sourceAccount,omitempty"` + SourceArn string `json:"sourceArn,omitempty"` + EventSourceToken string `json:"eventSourceToken,omitempty"` + PrincipalOrgID string `json:"principalOrgId,omitempty"` +} + +// permissionSnapshotKey is the store.Table key function for the ephemeral DTO +// table Snapshot/Restore build around permissionSnapshot; it mirrors +// permissionKeyFn exactly so the on-disk table is keyed identically to the +// live b.permissions table. +func permissionSnapshotKey(p *permissionSnapshot) string { + return permissionMapKey(p.FunctionName, p.Qualifier) + "|" + p.StatementID +} + +func permissionToSnapshot(p *FunctionPermission) *permissionSnapshot { + return &permissionSnapshot{ + StatementID: p.StatementID, + FunctionName: p.FunctionName, + Qualifier: p.Qualifier, + Action: p.Action, + Effect: p.Effect, + Principal: p.Principal, + SourceAccount: p.SourceAccount, + SourceArn: p.SourceArn, + EventSourceToken: p.EventSourceToken, + PrincipalOrgID: p.PrincipalOrgID, + } +} + +func permissionFromSnapshot(p *permissionSnapshot) *FunctionPermission { + return &FunctionPermission{ + StatementID: p.StatementID, + FunctionName: p.FunctionName, + Qualifier: p.Qualifier, + Action: p.Action, + Effect: p.Effect, + Principal: p.Principal, + SourceAccount: p.SourceAccount, + SourceArn: p.SourceArn, + EventSourceToken: p.EventSourceToken, + PrincipalOrgID: p.PrincipalOrgID, + } +} + type backendSnapshot struct { - Functions map[string]*FunctionConfiguration `json:"functions"` - EventSourceMappings map[string]*EventSourceMapping `json:"eventSourceMappings"` - Aliases map[string]map[string]*FunctionAlias `json:"aliases"` + // Tables holds one JSON-encoded array per table registered on b.registry + // (functions, functionURLConfigs, eventSourceMappings, aliases) PLUS a + // "permissions" entry built separately from permissionSnapshot DTOs (see + // Snapshot/Restore below) -- b.permissions itself is not registered on + // b.registry; see store_setup.go's package doc. Tables registered on + // b.ephemeralRegistry (codeSigningConfigs, capacityProviders, + // provisionedConcurrencies) are deliberately NOT included here -- they + // were never persisted before this refactor and must stay that way. + Tables map[string]json.RawMessage `json:"tables"` + // The fields below have no pure, self-contained identity (see + // store_setup.go's exclusion list) so they cannot become store.Tables, but + // they WERE persisted before this refactor and must remain so. + EventInvokeConfigs map[string]*FunctionEventInvokeConfig `json:"eventInvokeConfigs"` Versions map[string][]*FunctionVersion `json:"versions"` - FunctionURLConfigs map[string]*FunctionURLConfig `json:"functionURLConfigs"` VersionCounters map[string]int `json:"versionCounters"` Layers map[string][]*LayerVersion `json:"layers"` LayerVersionCounters map[string]int64 `json:"layerVersionCounters"` LayerPolicies map[string]map[int64]map[string]*LayerVersionStatement `json:"layerPolicies"` - EventInvokeConfigs map[string]*FunctionEventInvokeConfig `json:"eventInvokeConfigs"` FunctionConcurrencies map[string]int `json:"functionConcurrencies"` AccountID string `json:"accountID"` Region string `json:"region"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. @@ -31,17 +113,45 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "lambda: snapshot table marshal failed", "error", err) + + return nil + } + + // b.permissions is not on b.registry (see store_setup.go's package doc), + // so its Tables entry is built from a throwaway DTO registry instead, + // exactly mirroring the services/sqs pilot's Queue/moveTaskState pattern. + permDTOReg := store.NewRegistry() + permDTOs := store.Register(permDTOReg, "permissions", store.New(permissionSnapshotKey)) + + for _, p := range b.permissions.Snapshot() { + permDTOs.Put(permissionToSnapshot(p)) + } + + permTables, err := permDTOReg.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, "lambda: snapshot permissions marshal failed", "error", err) + + return nil + } + + tables["permissions"] = permTables["permissions"] + snap := backendSnapshot{ - Functions: b.functions, - EventSourceMappings: b.eventSourceMappings, - Aliases: b.aliases, + Version: lambdaSnapshotVersion, + Tables: tables, + EventInvokeConfigs: b.eventInvokeConfigs, Versions: b.versions, - FunctionURLConfigs: b.functionURLConfigs, VersionCounters: b.versionCounters, Layers: b.layers, LayerVersionCounters: b.layerVersionCounters, LayerPolicies: b.layerPolicies, - EventInvokeConfigs: b.eventInvokeConfigs, FunctionConcurrencies: b.functionConcurrencies, AccountID: b.accountID, Region: b.region, @@ -72,49 +182,109 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() - restoreSnapshotFunctions(snap.Functions, snap.FunctionConcurrencies) - restoreSnapshotLayers(snap.Layers) + if snap.Version != lambdaSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape — that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/sqs pilot (commit 0f09d77c) and the + // services/ec2 conversion (commit 12e611a4). Only b.registry is reset: + // b.ephemeralRegistry's tables (codeSigningConfigs, capacityProviders, + // provisionedConcurrencies) were never persisted, so Restore has never + // touched them and must not start now. + logger.Load(ctx).WarnContext(ctx, + "lambda: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", lambdaSnapshotVersion) + + b.registry.ResetAll() + b.permissions.Reset() + + return nil + } + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("lambda: restore snapshot tables: %w", err) + } - b.functions = snap.Functions - b.eventSourceMappings = snap.EventSourceMappings - b.aliases = snap.Aliases + // b.permissions is not on b.registry (see store_setup.go's package doc), + // so it is restored separately from its "permissions" DTO entry. + permDTOReg := store.NewRegistry() + permDTOs := store.Register(permDTOReg, "permissions", store.New(permissionSnapshotKey)) + + if err := permDTOReg.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("lambda: restore snapshot permissions: %w", err) + } + + livePerms := make([]*FunctionPermission, 0, permDTOs.Len()) + for _, p := range permDTOs.All() { + livePerms = append(livePerms, permissionFromSnapshot(p)) + } + + b.permissions.Restore(livePerms) + + b.eventInvokeConfigs = snap.EventInvokeConfigs b.versions = snap.Versions - b.functionURLConfigs = snap.FunctionURLConfigs b.versionCounters = snap.VersionCounters b.layers = snap.Layers b.layerVersionCounters = snap.LayerVersionCounters b.layerPolicies = snap.LayerPolicies - b.eventInvokeConfigs = snap.EventInvokeConfigs b.functionConcurrencies = snap.FunctionConcurrencies b.accountID = snap.AccountID b.region = snap.Region + restoreSnapshotFunctions(b.functions.All(), b.functionConcurrencies) + restoreSnapshotLayers(b.layers) + + // versionIndex is a derived lookup (UUID/qualifier -> published version + // snapshot) built incrementally by PublishVersion; it is not itself + // serialised. Without rebuilding it here, every published version becomes + // unreachable by qualifier after a restore (GetFunction/Invoke with a + // numeric version, or an alias pointing at one, would incorrectly 404) + // even though the version data survived in b.versions. + b.versionIndex = make(map[string]map[string]*FunctionVersion) + + for name, versions := range snap.Versions { + for _, v := range versions { + if v.Version == "" || v.Version == versionLatest { + continue + } + + if b.versionIndex[name] == nil { + b.versionIndex[name] = make(map[string]*FunctionVersion) + } + + b.versionIndex[name][v.Version] = v + } + } + + // esmByFunctionARN is likewise a derived reverse index over + // eventSourceMappings; rebuild it so ListEventSourceMappings filtered by + // FunctionName keeps working after a restore. + b.esmByFunctionARN = make(map[string]map[string]struct{}) + + for _, m := range b.eventSourceMappings.All() { + if b.esmByFunctionARN[m.FunctionARN] == nil { + b.esmByFunctionARN[m.FunctionARN] = make(map[string]struct{}) + } + + b.esmByFunctionARN[m.FunctionARN][m.UUID] = struct{}{} + } + return nil } // normalizeSnapshot initialises nil maps in a snapshot to empty maps so callers // never need to nil-check after a Restore. func normalizeSnapshot(snap *backendSnapshot) { - if snap.Functions == nil { - snap.Functions = make(map[string]*FunctionConfiguration) - } - - if snap.EventSourceMappings == nil { - snap.EventSourceMappings = make(map[string]*EventSourceMapping) - } - - if snap.Aliases == nil { - snap.Aliases = make(map[string]map[string]*FunctionAlias) + if snap.EventInvokeConfigs == nil { + snap.EventInvokeConfigs = make(map[string]*FunctionEventInvokeConfig) } if snap.Versions == nil { snap.Versions = make(map[string][]*FunctionVersion) } - if snap.FunctionURLConfigs == nil { - snap.FunctionURLConfigs = make(map[string]*FunctionURLConfig) - } - if snap.VersionCounters == nil { snap.VersionCounters = make(map[string]int) } @@ -131,10 +301,6 @@ func normalizeSnapshot(snap *backendSnapshot) { snap.LayerPolicies = make(map[string]map[int64]map[string]*LayerVersionStatement) } - if snap.EventInvokeConfigs == nil { - snap.EventInvokeConfigs = make(map[string]*FunctionEventInvokeConfig) - } - if snap.FunctionConcurrencies == nil { snap.FunctionConcurrencies = make(map[string]int) } @@ -142,15 +308,15 @@ func normalizeSnapshot(snap *backendSnapshot) { // restoreSnapshotFunctions clears transient fields on restored function configurations // and re-links ReservedConcurrentExecutions from the concurrency map. -func restoreSnapshotFunctions(fns map[string]*FunctionConfiguration, concurrencies map[string]int) { - for name, fn := range fns { +func restoreSnapshotFunctions(fns []*FunctionConfiguration, concurrencies map[string]int) { + for _, fn := range fns { fn.ZipData = nil if fn.LastUpdateStatus == "" { fn.LastUpdateStatus = LastUpdateStatusSuccessful } - if reserved, ok := concurrencies[name]; ok { + if reserved, ok := concurrencies[fn.FunctionName]; ok { v := reserved fn.ReservedConcurrentExecutions = &v } else { diff --git a/services/lambda/persistence_test.go b/services/lambda/persistence_test.go index 43b6a1b63..17fad4fc0 100644 --- a/services/lambda/persistence_test.go +++ b/services/lambda/persistence_test.go @@ -58,6 +58,213 @@ func TestInMemoryBackend_SnapshotRestore(t *testing.T) { assert.Empty(t, fns.Data) }, }, + { + // Bug fix (parity-sweep-3): the backendSnapshot omitted the + // permissions map entirely, so every AddPermission call was + // silently discarded across a persistence Restore even though + // persistence was enabled for the service — a no-stub-rule + // violation (state must persist when persistence is on). + name: "permissions_survive_restore", + setup: func(b *lambda.InMemoryBackend) string { + fn := &lambda.FunctionConfiguration{ + FunctionName: "perm-persist-fn", + Runtime: "python3.9", + Role: "arn:aws:iam::000000000000:role/test", + Handler: "index.handler", + } + require.NoError(t, b.CreateFunction(fn)) + + _, err := b.AddPermission("perm-persist-fn", "", &lambda.AddPermissionInput{ + StatementID: "AllowS3", + Action: "lambda:InvokeFunction", + Principal: "s3.amazonaws.com", + }) + require.NoError(t, err) + + return fn.FunctionName + }, + verify: func(t *testing.T, b *lambda.InMemoryBackend, id string) { + t.Helper() + + out, err := b.GetPolicy(id, "") + require.NoError(t, err) + require.NotNil(t, out.Policy) + assert.Contains(t, *out.Policy, "AllowS3") + }, + }, + { + // Bug fix (parity-sweep-3): versionIndex is a derived lookup + // built incrementally by PublishVersion and was never rebuilt on + // Restore, so every published (non-$LATEST) version became + // unreachable by qualifier after a persistence reload even + // though the version data itself survived in b.versions. + name: "published_version_reachable_after_restore", + setup: func(b *lambda.InMemoryBackend) string { + fn := &lambda.FunctionConfiguration{ + FunctionName: "ver-persist-fn", + Runtime: "python3.9", + Role: "arn:aws:iam::000000000000:role/test", + Handler: "index.handler", + } + require.NoError(t, b.CreateFunction(fn)) + + _, err := b.PublishVersion("ver-persist-fn", "") + require.NoError(t, err) + + return fn.FunctionName + }, + verify: func(t *testing.T, b *lambda.InMemoryBackend, id string) { + t.Helper() + + v, err := b.GetFunctionByQualifier(id, "1") + require.NoError(t, err, "published version 1 must resolve after restore") + assert.Equal(t, "1", v.Version) + }, + }, + { + // Bug fix (parity-sweep-3): esmByFunctionARN is a derived reverse + // index over eventSourceMappings and was likewise never rebuilt + // on Restore, so ListEventSourceMappings filtered by FunctionName + // silently returned an empty page after a persistence reload. + name: "esm_function_name_filter_survives_restore", + setup: func(b *lambda.InMemoryBackend) string { + fn := &lambda.FunctionConfiguration{ + FunctionName: "esm-idx-persist-fn", + Runtime: "python3.9", + Role: "arn:aws:iam::000000000000:role/test", + Handler: "index.handler", + } + require.NoError(t, b.CreateFunction(fn)) + + _, err := b.CreateEventSourceMapping(&lambda.CreateEventSourceMappingInput{ + EventSourceARN: "arn:aws:sqs:us-east-1:000000000000:test-queue", + FunctionName: "esm-idx-persist-fn", + Enabled: true, + }) + require.NoError(t, err) + + return fn.FunctionName + }, + verify: func(t *testing.T, b *lambda.InMemoryBackend, id string) { + t.Helper() + + page := b.ListEventSourceMappings(id, "", "", 0) + assert.Len(t, page.Data, 1) + }, + }, + { + // Phase 3.3 (pkgs/store conversion): exercises every store.Table + // registered on b.registry (functions, functionURLConfigs, + // eventSourceMappings, eventInvokeConfigs, aliases, permissions) + // plus every raw-but-still-persisted field (versions, + // versionCounters, layers, layerVersionCounters, layerPolicies, + // functionConcurrencies) in one round trip, so a regression that + // silently drops any of them from Snapshot/Restore fails this test. + name: "full_backend_state_round_trip", + setup: func(b *lambda.InMemoryBackend) string { + const fnName = "full-state-fn" + + fn := &lambda.FunctionConfiguration{ + FunctionName: fnName, + Runtime: "python3.9", + Role: "arn:aws:iam::000000000000:role/test", + Handler: "index.handler", + } + require.NoError(t, b.CreateFunction(fn)) + + // functionURLConfigs (store.Table) + _, err := b.CreateFunctionURLConfig(t.Context(), fnName, "NONE", nil, "") + require.NoError(t, err) + + // eventInvokeConfigs (store.Table) + maxRetry := 1 + _, err = b.PutFunctionEventInvokeConfig(fnName, &lambda.PutFunctionEventInvokeConfigInput{ + MaximumRetryAttempts: &maxRetry, + }) + require.NoError(t, err) + + // aliases (store.Table, composite key) + _, err = b.CreateAlias(fnName, &lambda.CreateAliasInput{ + Name: "prod", + FunctionVersion: "$LATEST", + }) + require.NoError(t, err) + + // permissions (store.Table, composite key) + _, err = b.AddPermission(fnName, "", &lambda.AddPermissionInput{ + StatementID: "AllowInvoke", + Action: "lambda:InvokeFunction", + Principal: "s3.amazonaws.com", + }) + require.NoError(t, err) + + // versions / versionCounters (raw, persisted) + _, err = b.PublishVersion(fnName, "v1") + require.NoError(t, err) + + // functionConcurrencies (raw, persisted) + _, err = b.PutFunctionConcurrency(fnName, 5) + require.NoError(t, err) + + // layers / layerVersionCounters (raw, persisted) + _, err = b.PublishLayerVersion(&lambda.PublishLayerVersionInput{ + LayerName: "full-state-layer", + Content: &lambda.LayerVersionContentInput{ZipFile: []byte("zip")}, + }) + require.NoError(t, err) + + // layerPolicies (raw, persisted) + _, err = b.AddLayerVersionPermission("full-state-layer", 1, &lambda.AddLayerVersionPermissionInput{ + StatementID: "LayerAllow", + Action: "lambda:GetLayerVersion", + Principal: "*", + }) + require.NoError(t, err) + + return fnName + }, + verify: func(t *testing.T, b *lambda.InMemoryBackend, id string) { + t.Helper() + + _, err := b.GetFunctionURLConfig(id) + require.NoError(t, err, "functionURLConfigs must survive restore") + + eic, err := b.GetFunctionEventInvokeConfig(id) + if assert.NoError(t, err, "eventInvokeConfigs must survive restore") { + require.NotNil(t, eic.MaximumRetryAttempts) + assert.Equal(t, 1, *eic.MaximumRetryAttempts) + } + + _, err = b.GetAlias(id, "prod") + require.NoError(t, err, "aliases must survive restore") + + policy, err := b.GetPolicy(id, "") + if assert.NoError(t, err, "permissions must survive restore") { + assert.Contains(t, *policy.Policy, "AllowInvoke") + } + + concurrency, err := b.GetFunctionConcurrency(id) + if assert.NoError(t, err, "functionConcurrencies must survive restore") { + assert.Equal(t, 5, concurrency.ReservedConcurrentExecutions) + } + + versions, err := b.ListVersionsByFunction(id, "", 0) + if assert.NoError(t, err, "versions/versionCounters must survive restore") { + var found bool + for _, v := range versions.Data { + if v.Version == "1" { + found = true + } + } + assert.True(t, found, "published version 1 must survive restore") + } + + policyOut, err := b.GetLayerVersionPolicy("full-state-layer", 1) + if assert.NoError(t, err, "layerPolicies must survive restore") { + assert.Contains(t, policyOut.Policy, "LayerAllow") + } + }, + }, } for _, tt := range tests { diff --git a/services/lambda/store_setup.go b/services/lambda/store_setup.go new file mode 100644 index 000000000..75dff6719 --- /dev/null +++ b/services/lambda/store_setup.go @@ -0,0 +1,288 @@ +package lambda + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend that has a pure, +// self-contained identity is registered exactly once, here, as a +// *store.Table[T]. See pkgs/store's package doc and the services/sqs pilot +// (commit 0f09d77c) plus the services/ec2 conversion (commit 12e611a4) for +// the pattern this follows. +// +// Two registries, not one +// +// Lambda has a wrinkle the ec2/sqs/dynamodb conversions did not: several +// resource maps that DO have a pure key function were never wired into +// backendSnapshot (codeSigningConfigs, capacityProviders, +// provisionedConcurrencies -- see persistence.go's backendSnapshot for the +// exact set that WAS already persisted before this refactor). Registering +// those three into the same registry Snapshot()/Restore() use would silently +// start persisting state that never survived a restart before, which is a +// behavior change the mechanical-swap rule forbids. They are therefore +// registered on b.ephemeralRegistry: swept by Reset() exactly like every +// other table, but never touched by Snapshot()/Restore() (only b.registry +// feeds backendSnapshot.Tables). This preserves every field's persisted/ +// not-persisted status exactly as it was before the conversion. +// +// permissions: a Table, but not on either registry +// +// b.permissions IS a *store.Table[FunctionPermission] (its key -- function +// name + qualifier + statement ID -- is a pure function of the value) but it +// is registered on NEITHER registry. FunctionPermission.FunctionName and +// .Qualifier are tagged `json:"-"` (they are internal bookkeeping, not part +// of any AWS response shape), so a generic store.Table[FunctionPermission] +// round-tripped through Registry.SnapshotAll()/RestoreAll()'s JSON encoding +// would silently lose exactly the fields permissionKeyFn needs, corrupting +// every key on Restore. persistence.go instead converts b.permissions to/from +// a small DTO with real json tags for those fields (the "DTO-registry +// pattern for dirty structs" the services/sqs pilot, commit 0f09d77c, +// established) and calls b.permissions.Restore/.Reset directly. See +// Reset/Snapshot/Restore for the mechanics. +// +// Fields deliberately left as plain maps (NOT registered anywhere) and why: +// +// - eventInvokeConfigs (map[string]*FunctionEventInvokeConfig): the value's +// only identity field, FunctionArn, is copied from the owning function's +// FunctionConfiguration.FunctionArn at Put time -- but a large share of +// this package's own test fixtures construct FunctionConfiguration +// without ever setting FunctionArn (it is optional unless the +// FunctionURL/EventInvokeConfig surface is what's under test), so it is +// not a reliably-populated identity the way e.g. functionURLConfigs' +// FunctionArn is (that one is always built internally by +// CreateFunctionURLConfig itself via buildURLARN, never echoed from the +// caller). Keying a Table off it would silently mis-key every such config +// to "". WAS persisted before this refactor and remains a raw field on +// backendSnapshot. +// - versions (map[string][]*FunctionVersion), layers (map[string][]*LayerVersion): +// one-to-many by slice, not a single V per key -- store.Table's shape is +// map[string]*V. Matches ec2's precedent of leaving analogous +// slice-valued fields (e.g. ipamPoolCidrs, spotFleetHistory) raw. Both +// WERE persisted before and remain raw fields on backendSnapshot. +// - versionCounters (map[string]int), functionConcurrencies (map[string]int), +// layerVersionCounters (map[string]int64): the value is a bare +// counter/scalar with no identity of its own -- keyFn cannot be derived +// from the value, only from the (externally supplied) map key. Mirrors +// dynamodb's txnTokens/txnPending exclusion. All three WERE persisted +// before and remain raw fields on backendSnapshot. +// - layerPolicies (map[string]map[int64]map[string]*LayerVersionStatement): +// triple-nested; the innermost value (LayerVersionStatement) carries only +// a Sid, no LayerName or Version field, so neither outer key is +// recoverable from the value. WAS persisted before and remains a raw +// field on backendSnapshot. +// - activeConcurrencies (map[string]int), fnCodeSigningConfigs +// (map[string]string): same no-identity-in-value reasoning as above. +// NEITHER was persisted before; both remain raw, unpersisted fields. +// - fisFaults (map[string]*FISInvocationFault): FISInvocationFault +// (Expiry/ErrorProbability/AddDelayMs) carries no identity field -- +// keyed externally by function name, exactly like ec2's +// instanceIMDSOptions/verifiedAccessGroupPolicies exclusions. NOT +// persisted before; remains raw and unpersisted. +// - runtimeManagementConfigs, functionScalingConfigs: both structs declare +// a FunctionArn field, but GetRuntimeManagementConfig/PutRuntimeManagementConfig +// and GetFunctionScalingConfig/PutFunctionScalingConfig only ever populate +// FunctionArn on the value RETURNED to the caller -- the copy actually +// stored in the map always has a zero-value FunctionArn. There is no +// reliable identity on the stored value, so a store.Table keyFn cannot be +// built from it. NOT persisted before; remain raw and unpersisted. +// - functionRecursionConfigs (map[string]*FunctionRecursionConfig): the +// value (RecursiveLoop only) carries no identity field at all. NOT +// persisted before; remains raw and unpersisted. +// - versionIndex, esmByFunctionARN: derived reverse indexes rebuilt from +// b.versions / b.eventSourceMappings on Restore (see persistence.go) -- +// never a primary source of truth, so they are neither Tables nor +// persisted directly, exactly as before this refactor. +// - runtimes (map[string]*functionRuntime), functionURLServers +// (map[string]*functionURLServer): live, non-serializable runtime state +// (docker container handles / OS listeners and *http.Server). Both +// structs are ALL-unexported-field, so json.Marshal would silently +// produce "{}" and Restore would rehydrate a broken, half-initialized +// entry (e.g. a nil rt.mu) that was never a valid runtime -- exactly the +// "must NOT be serialized" case called out for lambda. Kept as plain maps +// so they can never flow through Table/Registry's JSON-oriented +// Snapshot/Restore path. Neither was persisted before; both remain raw. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +// functionsKeyFn is the store.Table key function for b.functions. +// FunctionName is set once at CreateFunction and never renamed afterward +// (Lambda has no rename API), so it is a stable identity for the table's +// lifetime. +func functionsKeyFn(v *FunctionConfiguration) string { return v.FunctionName } + +// functionURLConfigsKeyFn is the store.Table key function for +// b.functionURLConfigs. FunctionArn is built once at CreateFunctionURLConfig +// (buildURLARN, an unqualified "function:name" ARN) and never mutated, so +// parsing the function name back out of it is a pure, stable derivation. +func functionURLConfigsKeyFn(v *FunctionURLConfig) string { + name, _ := functionNameAndQualifierFromARN(v.FunctionArn) + + return name +} + +// eventSourceMappingsKeyFn is the store.Table key function for +// b.eventSourceMappings. +func eventSourceMappingsKeyFn(v *EventSourceMapping) string { return v.UUID } + +// aliasKey builds the composite primary key used by b.aliases: a function +// name and an alias name are both required to identify an alias (the same +// alias name may exist under different functions). +func aliasKey(functionName, aliasName string) string { return functionName + "|" + aliasName } + +// aliasKeyFn is the store.Table key function for b.aliases. AliasArn +// (buildAliasARN: "function:name:aliasName") and Name are both set once at +// CreateAlias and never mutated afterward (UpdateAlias only ever touches +// FunctionVersion/Description/RoutingConfig/RevisionID), so deriving the +// composite key from them is pure and stable. +func aliasKeyFn(v *FunctionAlias) string { + name, _ := functionNameAndQualifierFromARN(v.AliasArn) + + return aliasKey(name, v.Name) +} + +// aliasFunctionKeyFn is the store.Index key function for b.aliasesByFunction, +// grouping aliases by owning function name -- the hot path ListAliases and +// the function-delete cascade both need ("every alias for function X"). +func aliasFunctionKeyFn(v *FunctionAlias) string { + name, _ := functionNameAndQualifierFromARN(v.AliasArn) + + return name +} + +// permissionKeyFn is the store.Table key function for b.permissions. +// FunctionName, Qualifier, and StatementID are all set once at AddPermission +// and never mutated afterward (there is no UpdatePermission op -- only +// Add/Remove), so the composite key is pure and stable. +func permissionKeyFn(v *FunctionPermission) string { + return permissionMapKey(v.FunctionName, v.Qualifier) + "|" + v.StatementID +} + +// permissionTargetKeyFn is the store.Index key function for +// b.permissionsByTarget, grouping statements by their function+qualifier +// target -- the hot path GetPolicy needs ("every statement for function X +// qualifier Y"). +func permissionTargetKeyFn(v *FunctionPermission) string { + return permissionMapKey(v.FunctionName, v.Qualifier) +} + +// codeSigningConfigKeyFn is the store.Table key function for +// b.codeSigningConfigs. +func codeSigningConfigKeyFn(v *CodeSigningConfig) string { return v.CodeSigningConfigArn } + +// capacityProviderKeyFn is the store.Table key function for +// b.capacityProviders. +func capacityProviderKeyFn(v *CapacityProvider) string { return v.Name } + +// provisionedConcurrencyKeyFn is the store.Table key function for +// b.provisionedConcurrencies. FunctionArn is built via buildAliasARN +// (region/account/functionName/qualifier) once at +// PutProvisionedConcurrencyConfig and never mutated afterward -- +// scheduleProvisionedConcurrencyReady replaces the whole value via +// copy-on-write but always preserves FunctionArn unchanged -- so it is a +// pure, stable composite identity for the function+qualifier pair. +func provisionedConcurrencyKeyFn(v *ProvisionedConcurrencyConfig) string { return v.FunctionArn } + +// provisionedConcurrencyFunctionKeyFn is the store.Index key function for +// b.provisionedConcurrenciesByFunction, grouping configs by owning function +// name -- the hot path ListProvisionedConcurrencyConfigs and the +// function-delete cascade both need ("every qualifier for function X"). +func provisionedConcurrencyFunctionKeyFn(v *ProvisionedConcurrencyConfig) string { + name, _ := functionNameAndQualifierFromARN(v.FunctionArn) + + return name +} + +// registerAllTables registers every converted resource map on b.registry (the +// tables that were already persisted before this refactor) and +// b.ephemeralRegistry (tables with a pure key function that were never +// persisted -- see the package-level doc above) exactly once. It must be +// called during construction only (immediately after both registries are +// created), never on every Reset() -- store.Register panics on a duplicate +// name, so runtime resets go through registry.ResetAll() on both registries +// instead (see InMemoryBackend.Reset). b.permissions is constructed here too +// but deliberately left off both registries -- see the package doc above. +func registerAllTables(b *InMemoryBackend) { + b.functions = store.Register(b.registry, "functions", store.New(functionsKeyFn)) + b.functionURLConfigs = store.Register(b.registry, "functionURLConfigs", store.New(functionURLConfigsKeyFn)) + b.eventSourceMappings = store.Register(b.registry, "eventSourceMappings", store.New(eventSourceMappingsKeyFn)) + + b.aliases = store.Register(b.registry, "aliases", store.New(aliasKeyFn)) + b.aliasesByFunction = b.aliases.AddIndex("function", aliasFunctionKeyFn) + + b.permissions = store.New(permissionKeyFn) + b.permissionsByTarget = b.permissions.AddIndex("target", permissionTargetKeyFn) + + b.codeSigningConfigs = store.Register(b.ephemeralRegistry, "codeSigningConfigs", store.New(codeSigningConfigKeyFn)) + b.capacityProviders = store.Register(b.ephemeralRegistry, "capacityProviders", store.New(capacityProviderKeyFn)) + + b.provisionedConcurrencies = store.Register( + b.ephemeralRegistry, "provisionedConcurrencies", store.New(provisionedConcurrencyKeyFn), + ) + b.provisionedConcurrenciesByFunction = b.provisionedConcurrencies.AddIndex( + "function", provisionedConcurrencyFunctionKeyFn, + ) +} + +// deleteAliasesForFunctionLocked removes every alias belonging to +// functionName from b.aliases. It copies the index group's keys before +// deleting so the loop is unaffected by store.Index.remove mutating the same +// backing slice as Table.Delete walks each alias's indexes. Caller must hold +// b.mu. +func (b *InMemoryBackend) deleteAliasesForFunctionLocked(functionName string) { + matches := b.aliasesByFunction.Get(functionName) + keys := make([]string, len(matches)) + + for i, a := range matches { + keys[i] = aliasKeyFn(a) + } + + for _, k := range keys { + b.aliases.Delete(k) + } +} + +// deletePermissionsForFunctionLocked removes every permission statement +// belonging to functionName from b.permissions, across every qualifier +// (unqualified and every "name:qualifier" scope) -- mirroring the pre-Table +// deletePermissionsForFunctionLocked, which scanned every b.permissions outer +// key for a "name" or "name:"-prefixed match. b.permissionsByTarget cannot +// answer this directly since it groups by the full function+qualifier target, +// not by function name alone, so this does the equivalent linear scan over +// the (typically small) permissions table. Caller must hold b.mu. +func (b *InMemoryBackend) deletePermissionsForFunctionLocked(functionName string) { + var keys []string + + b.permissions.Range(func(p *FunctionPermission) bool { + if p.FunctionName == functionName { + keys = append(keys, permissionKeyFn(p)) + } + + return true + }) + + for _, k := range keys { + b.permissions.Delete(k) + } +} + +// deleteProvisionedConcurrenciesForFunctionLocked removes every provisioned +// concurrency config (every qualifier) belonging to functionName from +// b.provisionedConcurrencies. Caller must hold b.mu. +func (b *InMemoryBackend) deleteProvisionedConcurrenciesForFunctionLocked(functionName string) { + matches := b.provisionedConcurrenciesByFunction.Get(functionName) + keys := make([]string, len(matches)) + + for i, cfg := range matches { + keys[i] = cfg.FunctionArn + } + + for _, k := range keys { + b.provisionedConcurrencies.Delete(k) + } +} + +// permissionsForTarget returns every permission statement scoped to the exact +// function+qualifier target (the precise "permissionMapKey(name, qualifier)" +// scope GetPolicy/RemovePermission key off), via an O(1) +// b.permissionsByTarget lookup instead of a full-table scan. +func (b *InMemoryBackend) permissionsForTarget(target string) []*FunctionPermission { + return b.permissionsByTarget.Get(target) +} diff --git a/services/memorydb/backend.go b/services/memorydb/backend.go index b332e6390..e9b150e74 100644 --- a/services/memorydb/backend.go +++ b/services/memorydb/backend.go @@ -12,6 +12,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awserr" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) const ( @@ -301,17 +302,32 @@ type StorageBackend interface { } // InMemoryBackend is the in-memory implementation of StorageBackend. +// +// multiRegionClusters, multiRegionParameterGroups, and serviceUpdates are +// partition-scoped (like AWS) and therefore NOT region-nested; they are +// registered on registry so Reset/Snapshot/Restore collapse to one Registry +// call each. Every other resource is nested per-region +// (map[string]*store.Table[T], outer key is region) via the lazy "*Store" +// accessors below. Per-region tables are deliberately NOT registered on +// registry because the set of regions is only known at runtime -- see +// store_setup.go and persistence.go for how they are (de)serialized instead. +// +// arnToResource and events are left as plain maps: arnToResource's value +// (resourceRef) has no key of its own to feed a Table's keyFn (the ARN is the +// map key, not a value field), and events is slice-valued +// (map[string][]*Event), neither of which store.Table can represent. type InMemoryBackend struct { - multiRegionClusters map[string]*MultiRegionCluster - multiRegionParameterGroups map[string]*MultiRegionParameterGroup - serviceUpdates map[string]*ServiceUpdate - clusters map[string]map[string]*Cluster - acls map[string]map[string]*ACL - subnetGroups map[string]map[string]*SubnetGroup - users map[string]map[string]*User - parameterGroups map[string]map[string]*ParameterGroup - snapshots map[string]map[string]*Snapshot - reservedNodes map[string]map[string]*ReservedNode + registry *store.Registry + multiRegionClusters *store.Table[MultiRegionCluster] + multiRegionParameterGroups *store.Table[MultiRegionParameterGroup] + serviceUpdates *store.Table[ServiceUpdate] + clusters map[string]*store.Table[Cluster] + acls map[string]*store.Table[ACL] + subnetGroups map[string]*store.Table[SubnetGroup] + users map[string]*store.Table[User] + parameterGroups map[string]*store.Table[ParameterGroup] + snapshots map[string]*store.Table[Snapshot] + reservedNodes map[string]*store.Table[ReservedNode] arnToResource map[string]map[string]resourceRef events map[string][]*Event accountID string @@ -330,110 +346,113 @@ func NewInMemoryBackend(accountID, region string) *InMemoryBackend { func newInMemoryBackendWithDefaults(region, accountID string) *InMemoryBackend { b := &InMemoryBackend{ - clusters: make(map[string]map[string]*Cluster), - acls: make(map[string]map[string]*ACL), - subnetGroups: make(map[string]map[string]*SubnetGroup), - users: make(map[string]map[string]*User), - parameterGroups: make(map[string]map[string]*ParameterGroup), - snapshots: make(map[string]map[string]*Snapshot), - multiRegionClusters: make(map[string]*MultiRegionCluster), - multiRegionParameterGroups: make(map[string]*MultiRegionParameterGroup), - reservedNodes: make(map[string]map[string]*ReservedNode), - serviceUpdates: make(map[string]*ServiceUpdate), - events: make(map[string][]*Event), - arnToResource: make(map[string]map[string]resourceRef), - accountID: accountID, - defaultRegion: region, - } + registry: store.NewRegistry(), + clusters: make(map[string]*store.Table[Cluster]), + acls: make(map[string]*store.Table[ACL]), + subnetGroups: make(map[string]*store.Table[SubnetGroup]), + users: make(map[string]*store.Table[User]), + parameterGroups: make(map[string]*store.Table[ParameterGroup]), + snapshots: make(map[string]*store.Table[Snapshot]), + reservedNodes: make(map[string]*store.Table[ReservedNode]), + events: make(map[string][]*Event), + arnToResource: make(map[string]map[string]resourceRef), + accountID: accountID, + defaultRegion: region, + } + b.multiRegionClusters = store.Register(b.registry, "multiRegionClusters", store.New(multiRegionClusterKeyFn)) + b.multiRegionParameterGroups = store.Register( + b.registry, "multiRegionParameterGroups", store.New(multiRegionParameterGroupKeyFn), + ) + b.serviceUpdates = store.Register(b.registry, "serviceUpdates", store.New(serviceUpdateKeyFn)) openAccessARN := arn.Build("memorydb", region, accountID, "acl/"+openAccessACL) - b.aclsStore(region)[openAccessACL] = &ACL{ + b.aclsStore(region).Put(&ACL{ Name: openAccessACL, ARN: openAccessARN, Status: aclStatusActive, UserNames: []string{}, CreatedAt: time.Now(), Tags: make(map[string]string), - } + }) b.arnToResourceStore(region)[openAccessARN] = resourceRef{Kind: resourceKindACL, Name: openAccessACL} - b.serviceUpdates["memorydb-20240601-redis-security"] = &ServiceUpdate{ + b.serviceUpdates.Put(&ServiceUpdate{ ServiceUpdateName: "memorydb-20240601-redis-security", ReleaseDate: "2024-06-01", Description: "Security update for Redis 7.x clusters", Status: clusterStatusAvailable, Type: "security-update", AutoUpdateStartDate: "2024-07-01", - } - b.serviceUpdates["memorydb-20240801-engine-update"] = &ServiceUpdate{ + }) + b.serviceUpdates.Put(&ServiceUpdate{ ServiceUpdateName: "memorydb-20240801-engine-update", ReleaseDate: "2024-08-01", Description: "Engine update with performance improvements", Status: clusterStatusAvailable, Type: "engine-update", AutoUpdateStartDate: "2024-09-01", - } + }) b.seedDefaultParameterGroupsLocked() return b } -func (b *InMemoryBackend) clustersStore(region string) map[string]*Cluster { +func (b *InMemoryBackend) clustersStore(region string) *store.Table[Cluster] { if b.clusters[region] == nil { - b.clusters[region] = make(map[string]*Cluster) + b.clusters[region] = store.New(clusterKeyFn) } return b.clusters[region] } -func (b *InMemoryBackend) aclsStore(region string) map[string]*ACL { +func (b *InMemoryBackend) aclsStore(region string) *store.Table[ACL] { if b.acls[region] == nil { - b.acls[region] = make(map[string]*ACL) + b.acls[region] = store.New(aclKeyFn) // Seed the open-access ACL into every region so CreateCluster works across regions. openAccessARN := arn.Build("memorydb", region, b.accountID, "acl/"+openAccessACL) - b.acls[region][openAccessACL] = &ACL{ + b.acls[region].Put(&ACL{ Name: openAccessACL, ARN: openAccessARN, Status: aclStatusActive, UserNames: []string{}, CreatedAt: time.Now(), Tags: make(map[string]string), - } + }) } return b.acls[region] } -func (b *InMemoryBackend) subnetGroupsStore(region string) map[string]*SubnetGroup { +func (b *InMemoryBackend) subnetGroupsStore(region string) *store.Table[SubnetGroup] { if b.subnetGroups[region] == nil { - b.subnetGroups[region] = make(map[string]*SubnetGroup) + b.subnetGroups[region] = store.New(subnetGroupKeyFn) } return b.subnetGroups[region] } -func (b *InMemoryBackend) usersStore(region string) map[string]*User { +func (b *InMemoryBackend) usersStore(region string) *store.Table[User] { if b.users[region] == nil { - b.users[region] = make(map[string]*User) + b.users[region] = store.New(userKeyFn) } return b.users[region] } -func (b *InMemoryBackend) parameterGroupsStore(region string) map[string]*ParameterGroup { +func (b *InMemoryBackend) parameterGroupsStore(region string) *store.Table[ParameterGroup] { if b.parameterGroups[region] == nil { - b.parameterGroups[region] = make(map[string]*ParameterGroup) + b.parameterGroups[region] = store.New(parameterGroupKeyFn) } return b.parameterGroups[region] } -func (b *InMemoryBackend) snapshotsStore(region string) map[string]*Snapshot { +func (b *InMemoryBackend) snapshotsStore(region string) *store.Table[Snapshot] { if b.snapshots[region] == nil { - b.snapshots[region] = make(map[string]*Snapshot) + b.snapshots[region] = store.New(snapshotKeyFn) } return b.snapshots[region] } -func (b *InMemoryBackend) reservedNodesStore(region string) map[string]*ReservedNode { +func (b *InMemoryBackend) reservedNodesStore(region string) *store.Table[ReservedNode] { if b.reservedNodes[region] == nil { - b.reservedNodes[region] = make(map[string]*ReservedNode) + b.reservedNodes[region] = store.New(reservedNodeKeyFn) } return b.reservedNodes[region] @@ -453,46 +472,56 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock() defer b.mu.Unlock() - b.clusters = make(map[string]map[string]*Cluster) - b.acls = make(map[string]map[string]*ACL) - b.subnetGroups = make(map[string]map[string]*SubnetGroup) - b.users = make(map[string]map[string]*User) - b.parameterGroups = make(map[string]map[string]*ParameterGroup) - b.snapshots = make(map[string]map[string]*Snapshot) - b.multiRegionClusters = make(map[string]*MultiRegionCluster) - b.multiRegionParameterGroups = make(map[string]*MultiRegionParameterGroup) - b.reservedNodes = make(map[string]map[string]*ReservedNode) - b.serviceUpdates = make(map[string]*ServiceUpdate) + b.resetLocked() +} + +// resetLocked does the work of Reset without acquiring b.mu, so it can also be +// called from Restore's incompatible-snapshot-version guard, which already +// holds the lock (calling the public, self-locking Reset there would +// deadlock). Must hold b.mu. +func (b *InMemoryBackend) resetLocked() { + b.clusters = make(map[string]*store.Table[Cluster]) + b.acls = make(map[string]*store.Table[ACL]) + b.subnetGroups = make(map[string]*store.Table[SubnetGroup]) + b.users = make(map[string]*store.Table[User]) + b.parameterGroups = make(map[string]*store.Table[ParameterGroup]) + b.snapshots = make(map[string]*store.Table[Snapshot]) + b.reservedNodes = make(map[string]*store.Table[ReservedNode]) b.events = make(map[string][]*Event) b.arnToResource = make(map[string]map[string]resourceRef) + // b.multiRegionClusters, b.multiRegionParameterGroups, and b.serviceUpdates + // are registered on b.registry at construction and must keep their + // identity (store.Register panics on a duplicate name), so they are + // cleared in place via the registry rather than reassigned. + b.registry.ResetAll() openAccessARN := arn.Build("memorydb", b.defaultRegion, b.accountID, "acl/"+openAccessACL) - b.aclsStore(b.defaultRegion)[openAccessACL] = &ACL{ + b.aclsStore(b.defaultRegion).Put(&ACL{ Name: openAccessACL, ARN: openAccessARN, Status: aclStatusActive, UserNames: []string{}, CreatedAt: time.Now(), Tags: make(map[string]string), - } + }) b.arnToResourceStore(b.defaultRegion)[openAccessARN] = resourceRef{Kind: resourceKindACL, Name: openAccessACL} - b.serviceUpdates["memorydb-20240601-redis-security"] = &ServiceUpdate{ + b.serviceUpdates.Put(&ServiceUpdate{ ServiceUpdateName: "memorydb-20240601-redis-security", ReleaseDate: "2024-06-01", Description: "Security update for Redis 7.x clusters", Status: clusterStatusAvailable, Type: "security-update", AutoUpdateStartDate: "2024-07-01", - } - b.serviceUpdates["memorydb-20240801-engine-update"] = &ServiceUpdate{ + }) + b.serviceUpdates.Put(&ServiceUpdate{ ServiceUpdateName: "memorydb-20240801-engine-update", ReleaseDate: "2024-08-01", Description: "Engine update with performance improvements", Status: clusterStatusAvailable, Type: "engine-update", AutoUpdateStartDate: "2024-09-01", - } + }) b.seedDefaultParameterGroupsLocked() } @@ -521,7 +550,7 @@ func (b *InMemoryBackend) seedDefaultParameterGroupsLocked() { Tags: make(map[string]string), CreatedAt: time.Now(), } - b.parameterGroupsStore(b.defaultRegion)[f.name] = pg + b.parameterGroupsStore(b.defaultRegion).Put(pg) b.arnToResourceStore(b.defaultRegion)[pgARN] = resourceRef{Kind: resourceKindParameterGroup, Name: f.name} } @@ -562,7 +591,7 @@ func (b *InMemoryBackend) seedDefaultParameterGroupsLocked() { Tags: make(map[string]string), CreatedAt: time.Now(), } - b.multiRegionParameterGroups[f.name] = mrpg + b.multiRegionParameterGroups.Put(mrpg) } } @@ -597,18 +626,18 @@ func (b *InMemoryBackend) validateCreateClusterRefs(region string, req *createCl aclName = openAccessACL } - if _, ok := b.aclsStore(region)[aclName]; !ok { + if _, ok := b.aclsStore(region).Get(aclName); !ok { return "", fmt.Errorf("ACL %q not found: %w", aclName, ErrACLNotFound) } if req.SubnetGroupName != "" { - if _, ok := b.subnetGroupsStore(region)[req.SubnetGroupName]; !ok { + if _, ok := b.subnetGroupsStore(region).Get(req.SubnetGroupName); !ok { return "", fmt.Errorf("subnet group %q not found: %w", req.SubnetGroupName, ErrSubnetGroupNotFound) } } if req.ParameterGroupName != "" { - if _, ok := b.parameterGroupsStore(region)[req.ParameterGroupName]; !ok { + if _, ok := b.parameterGroupsStore(region).Get(req.ParameterGroupName); !ok { return "", fmt.Errorf("parameter group %q not found: %w", req.ParameterGroupName, ErrParameterGroupNotFound) } } @@ -764,7 +793,7 @@ func (b *InMemoryBackend) seedAutomatedSnapshotLocked(region, accountID string, SnapshotWindow: c.SnapshotWindow, }, } - b.snapshotsStore(region)[autoName] = autoSnap + b.snapshotsStore(region).Put(autoSnap) b.arnToResourceStore(region)[autoARN] = resourceRef{Kind: resourceKindSnapshot, Name: autoName} } @@ -841,7 +870,7 @@ func (b *InMemoryBackend) CreateCluster(ctx context.Context, req *createClusterR return nil, err } - if _, exists := b.clustersStore(region)[req.ClusterName]; exists { + if _, exists := b.clustersStore(region).Get(req.ClusterName); exists { return nil, ErrClusterAlreadyExists } @@ -852,7 +881,7 @@ func (b *InMemoryBackend) CreateCluster(ctx context.Context, req *createClusterR var restoreSnap *Snapshot if req.SnapshotName != "" { - s, ok := b.snapshotsStore(region)[req.SnapshotName] + s, ok := b.snapshotsStore(region).Get(req.SnapshotName) if !ok { return nil, fmt.Errorf("snapshot %q not found: %w", req.SnapshotName, ErrSnapshotNotFound) } @@ -878,7 +907,7 @@ func (b *InMemoryBackend) CreateCluster(ctx context.Context, req *createClusterR clusterARN := arn.Build("memorydb", region, b.accountID, "cluster/"+req.ClusterName) c := buildCluster(region, clusterARN, aclName, req, d) - b.clustersStore(region)[req.ClusterName] = c + b.clustersStore(region).Put(c) b.arnToResourceStore(region)[clusterARN] = resourceRef{Kind: resourceKindCluster, Name: req.ClusterName} b.appendEventLocked(region, &Event{ @@ -901,10 +930,10 @@ func (b *InMemoryBackend) DescribeClusters(ctx context.Context, name string) ([] defer b.mu.RUnlock() region := getRegion(ctx, b.defaultRegion) - store := b.clusters[region] + t := b.clusters[region] if name != "" { - c, ok := store[name] + c, ok := tableGet(t, name) if !ok { return nil, ErrClusterNotFound } @@ -912,8 +941,9 @@ func (b *InMemoryBackend) DescribeClusters(ctx context.Context, name string) ([] return []*Cluster{cloneCluster(c)}, nil } - result := make([]*Cluster, 0, len(store)) - for _, c := range store { + all := tableAll(t) + result := make([]*Cluster, 0, len(all)) + for _, c := range all { result = append(result, cloneCluster(c)) } sort.Slice(result, func(i, j int) bool { @@ -930,12 +960,12 @@ func (b *InMemoryBackend) DeleteCluster(ctx context.Context, name string) (*Clus region := getRegion(ctx, b.defaultRegion) - c, ok := b.clustersStore(region)[name] + c, ok := b.clustersStore(region).Get(name) if !ok { return nil, ErrClusterNotFound } - delete(b.clustersStore(region), name) + b.clustersStore(region).Delete(name) delete(b.arnToResourceStore(region), c.ARN) b.appendEventLocked(region, &Event{ @@ -958,7 +988,7 @@ func (b *InMemoryBackend) DeleteClusterWithSnapshot( region := getRegion(ctx, b.defaultRegion) - c, ok := b.clustersStore(region)[clusterName] + c, ok := b.clustersStore(region).Get(clusterName) if !ok { return nil, ErrClusterNotFound } @@ -989,11 +1019,11 @@ func (b *InMemoryBackend) DeleteClusterWithSnapshot( SnapshotWindow: c.SnapshotWindow, }, } - b.snapshotsStore(region)[snapshotName] = s + b.snapshotsStore(region).Put(s) b.arnToResourceStore(region)[snapshotARN] = resourceRef{Kind: resourceKindSnapshot, Name: snapshotName} } - delete(b.clustersStore(region), clusterName) + b.clustersStore(region).Delete(clusterName) delete(b.arnToResourceStore(region), c.ARN) b.appendEventLocked(region, &Event{ @@ -1123,7 +1153,7 @@ func (b *InMemoryBackend) UpdateCluster(ctx context.Context, req *updateClusterR region := getRegion(ctx, b.defaultRegion) - c, ok := b.clustersStore(region)[req.ClusterName] + c, ok := b.clustersStore(region).Get(req.ClusterName) if !ok { return nil, ErrClusterNotFound } @@ -1157,7 +1187,7 @@ func (b *InMemoryBackend) CreateACL(ctx context.Context, req *createACLRequest) return nil, err } - if _, exists := b.aclsStore(region)[req.ACLName]; exists { + if _, exists := b.aclsStore(region).Get(req.ACLName); exists { return nil, ErrACLAlreadyExists } @@ -1177,7 +1207,7 @@ func (b *InMemoryBackend) CreateACL(ctx context.Context, req *createACLRequest) CreatedAt: time.Now(), } - b.aclsStore(region)[req.ACLName] = a + b.aclsStore(region).Put(a) b.arnToResourceStore(region)[aclARN] = resourceRef{Kind: resourceKindACL, Name: req.ACLName} b.appendEventLocked(region, &Event{ @@ -1196,10 +1226,10 @@ func (b *InMemoryBackend) DescribeACLs(ctx context.Context, name string) ([]*ACL defer b.mu.RUnlock() region := getRegion(ctx, b.defaultRegion) - store := b.acls[region] + t := b.acls[region] if name != "" { - a, ok := store[name] + a, ok := tableGet(t, name) if !ok { return nil, ErrACLNotFound } @@ -1207,8 +1237,9 @@ func (b *InMemoryBackend) DescribeACLs(ctx context.Context, name string) ([]*ACL return []*ACL{cloneACL(a)}, nil } - result := make([]*ACL, 0, len(store)) - for _, a := range store { + all := tableAll(t) + result := make([]*ACL, 0, len(all)) + for _, a := range all { result = append(result, cloneACL(a)) } sort.Slice(result, func(i, j int) bool { @@ -1225,7 +1256,7 @@ func (b *InMemoryBackend) DeleteACL(ctx context.Context, name string) (*ACL, err region := getRegion(ctx, b.defaultRegion) - a, ok := b.aclsStore(region)[name] + a, ok := b.aclsStore(region).Get(name) if !ok { return nil, ErrACLNotFound } @@ -1234,13 +1265,13 @@ func (b *InMemoryBackend) DeleteACL(ctx context.Context, name string) (*ACL, err return nil, fmt.Errorf("cannot delete system ACL %q: %w", name, ErrValidation) } - for _, c := range b.clusters[region] { + for _, c := range tableAll(b.clusters[region]) { if c.ACLName == name { return nil, fmt.Errorf("ACL %q is associated with cluster %q: %w", name, c.Name, ErrACLInUse) } } - delete(b.aclsStore(region), name) + b.aclsStore(region).Delete(name) delete(b.arnToResourceStore(region), a.ARN) b.appendEventLocked(region, &Event{ @@ -1260,7 +1291,7 @@ func (b *InMemoryBackend) UpdateACL(ctx context.Context, req *updateACLRequest) region := getRegion(ctx, b.defaultRegion) - a, ok := b.aclsStore(region)[req.ACLName] + a, ok := b.aclsStore(region).Get(req.ACLName) if !ok { return nil, ErrACLNotFound } @@ -1271,7 +1302,7 @@ func (b *InMemoryBackend) UpdateACL(ctx context.Context, req *updateACLRequest) } for _, u := range req.UserNamesToAdd { - if _, exists := b.users[region][u]; !exists { + if _, exists := tableGet(b.users[region], u); !exists { return nil, fmt.Errorf("user %q not found: %w", u, ErrUserNotFound) } } @@ -1320,7 +1351,7 @@ func (b *InMemoryBackend) CreateSubnetGroup(ctx context.Context, req *createSubn return nil, err } - if _, exists := b.subnetGroupsStore(region)[req.SubnetGroupName]; exists { + if _, exists := b.subnetGroupsStore(region).Get(req.SubnetGroupName); exists { return nil, ErrSubnetGroupAlreadyExists } @@ -1335,7 +1366,7 @@ func (b *InMemoryBackend) CreateSubnetGroup(ctx context.Context, req *createSubn CreatedAt: time.Now(), } - b.subnetGroupsStore(region)[req.SubnetGroupName] = sg + b.subnetGroupsStore(region).Put(sg) b.arnToResourceStore(region)[sgARN] = resourceRef{Kind: resourceKindSubnetGroup, Name: req.SubnetGroupName} return cloneSubnetGroup(sg), nil @@ -1347,10 +1378,10 @@ func (b *InMemoryBackend) DescribeSubnetGroups(ctx context.Context, name string) defer b.mu.RUnlock() region := getRegion(ctx, b.defaultRegion) - store := b.subnetGroups[region] + t := b.subnetGroups[region] if name != "" { - sg, ok := store[name] + sg, ok := tableGet(t, name) if !ok { return nil, ErrSubnetGroupNotFound @@ -1359,8 +1390,9 @@ func (b *InMemoryBackend) DescribeSubnetGroups(ctx context.Context, name string) return []*SubnetGroup{cloneSubnetGroup(sg)}, nil } - result := make([]*SubnetGroup, 0, len(store)) - for _, sg := range store { + all := tableAll(t) + result := make([]*SubnetGroup, 0, len(all)) + for _, sg := range all { result = append(result, cloneSubnetGroup(sg)) } sort.Slice(result, func(i, j int) bool { @@ -1377,12 +1409,12 @@ func (b *InMemoryBackend) DeleteSubnetGroup(ctx context.Context, name string) (* region := getRegion(ctx, b.defaultRegion) - sg, ok := b.subnetGroupsStore(region)[name] + sg, ok := b.subnetGroupsStore(region).Get(name) if !ok { return nil, ErrSubnetGroupNotFound } - delete(b.subnetGroupsStore(region), name) + b.subnetGroupsStore(region).Delete(name) delete(b.arnToResourceStore(region), sg.ARN) return sg, nil @@ -1395,7 +1427,7 @@ func (b *InMemoryBackend) UpdateSubnetGroup(ctx context.Context, req *updateSubn region := getRegion(ctx, b.defaultRegion) - sg, ok := b.subnetGroupsStore(region)[req.SubnetGroupName] + sg, ok := b.subnetGroupsStore(region).Get(req.SubnetGroupName) if !ok { return nil, ErrSubnetGroupNotFound } @@ -1424,7 +1456,7 @@ func (b *InMemoryBackend) CreateUser(ctx context.Context, req *createUserRequest return nil, err } - if _, exists := b.usersStore(region)[req.UserName]; exists { + if _, exists := b.usersStore(region).Get(req.UserName); exists { return nil, ErrUserAlreadyExists } @@ -1458,7 +1490,7 @@ func (b *InMemoryBackend) CreateUser(ctx context.Context, req *createUserRequest CreatedAt: time.Now(), } - b.usersStore(region)[req.UserName] = u + b.usersStore(region).Put(u) b.arnToResourceStore(region)[userARN] = resourceRef{Kind: resourceKindUser, Name: req.UserName} b.appendEventLocked(region, &Event{ @@ -1477,10 +1509,10 @@ func (b *InMemoryBackend) DescribeUsers(ctx context.Context, name string) ([]*Us defer b.mu.RUnlock() region := getRegion(ctx, b.defaultRegion) - store := b.users[region] + t := b.users[region] if name != "" { - u, ok := store[name] + u, ok := tableGet(t, name) if !ok { return nil, ErrUserNotFound } @@ -1488,8 +1520,9 @@ func (b *InMemoryBackend) DescribeUsers(ctx context.Context, name string) ([]*Us return []*User{cloneUser(u)}, nil } - result := make([]*User, 0, len(store)) - for _, u := range store { + all := tableAll(t) + result := make([]*User, 0, len(all)) + for _, u := range all { result = append(result, cloneUser(u)) } sort.Slice(result, func(i, j int) bool { @@ -1506,18 +1539,18 @@ func (b *InMemoryBackend) DeleteUser(ctx context.Context, name string) (*User, e region := getRegion(ctx, b.defaultRegion) - u, ok := b.usersStore(region)[name] + u, ok := b.usersStore(region).Get(name) if !ok { return nil, ErrUserNotFound } - for _, a := range b.acls[region] { + for _, a := range tableAll(b.acls[region]) { if slices.Contains(a.UserNames, name) { return nil, fmt.Errorf("user %q is a member of ACL %q: %w", name, a.Name, ErrUserInUse) } } - delete(b.usersStore(region), name) + b.usersStore(region).Delete(name) delete(b.arnToResourceStore(region), u.ARN) return u, nil @@ -1530,7 +1563,7 @@ func (b *InMemoryBackend) UpdateUser(ctx context.Context, req *updateUserRequest region := getRegion(ctx, b.defaultRegion) - u, ok := b.usersStore(region)[req.UserName] + u, ok := b.usersStore(region).Get(req.UserName) if !ok { return nil, ErrUserNotFound } @@ -1571,7 +1604,7 @@ func (b *InMemoryBackend) CreateParameterGroup( return nil, err } - if _, exists := b.parameterGroupsStore(region)[req.ParameterGroupName]; exists { + if _, exists := b.parameterGroupsStore(region).Get(req.ParameterGroupName); exists { return nil, ErrParameterGroupAlreadyExists } @@ -1587,7 +1620,7 @@ func (b *InMemoryBackend) CreateParameterGroup( CreatedAt: time.Now(), } - b.parameterGroupsStore(region)[req.ParameterGroupName] = pg + b.parameterGroupsStore(region).Put(pg) b.arnToResourceStore(region)[pgARN] = resourceRef{Kind: resourceKindParameterGroup, Name: req.ParameterGroupName} return pg, nil @@ -1599,10 +1632,10 @@ func (b *InMemoryBackend) DescribeParameterGroups(ctx context.Context, name stri defer b.mu.RUnlock() region := getRegion(ctx, b.defaultRegion) - store := b.parameterGroups[region] + t := b.parameterGroups[region] if name != "" { - pg, ok := store[name] + pg, ok := tableGet(t, name) if !ok { return nil, ErrParameterGroupNotFound } @@ -1610,8 +1643,9 @@ func (b *InMemoryBackend) DescribeParameterGroups(ctx context.Context, name stri return []*ParameterGroup{cloneParameterGroup(pg)}, nil } - result := make([]*ParameterGroup, 0, len(store)) - for _, pg := range store { + all := tableAll(t) + result := make([]*ParameterGroup, 0, len(all)) + for _, pg := range all { result = append(result, cloneParameterGroup(pg)) } sort.Slice(result, func(i, j int) bool { @@ -1628,12 +1662,12 @@ func (b *InMemoryBackend) DeleteParameterGroup(ctx context.Context, name string) region := getRegion(ctx, b.defaultRegion) - pg, ok := b.parameterGroupsStore(region)[name] + pg, ok := b.parameterGroupsStore(region).Get(name) if !ok { return nil, ErrParameterGroupNotFound } - delete(b.parameterGroupsStore(region), name) + b.parameterGroupsStore(region).Delete(name) delete(b.arnToResourceStore(region), pg.ARN) return pg, nil @@ -1649,7 +1683,7 @@ func (b *InMemoryBackend) UpdateParameterGroup( region := getRegion(ctx, b.defaultRegion) - pg, ok := b.parameterGroupsStore(region)[req.ParameterGroupName] + pg, ok := b.parameterGroupsStore(region).Get(req.ParameterGroupName) if !ok { return nil, ErrParameterGroupNotFound } @@ -1744,27 +1778,27 @@ func (b *InMemoryBackend) tagsForRef(region string, ref resourceRef) map[string] switch ref.Kind { case resourceKindCluster: - if c, ok := b.clusters[region][ref.Name]; ok { + if c, ok := tableGet(b.clusters[region], ref.Name); ok { src = c.Tags } case resourceKindACL: - if a, ok := b.acls[region][ref.Name]; ok { + if a, ok := tableGet(b.acls[region], ref.Name); ok { src = a.Tags } case resourceKindSubnetGroup: - if sg, ok := b.subnetGroups[region][ref.Name]; ok { + if sg, ok := tableGet(b.subnetGroups[region], ref.Name); ok { src = sg.Tags } case resourceKindUser: - if u, ok := b.users[region][ref.Name]; ok { + if u, ok := tableGet(b.users[region], ref.Name); ok { src = u.Tags } case resourceKindParameterGroup: - if pg, ok := b.parameterGroups[region][ref.Name]; ok { + if pg, ok := tableGet(b.parameterGroups[region], ref.Name); ok { src = pg.Tags } case resourceKindSnapshot: - if s, ok := b.snapshots[region][ref.Name]; ok { + if s, ok := tableGet(b.snapshots[region], ref.Name); ok { src = s.Tags } } @@ -1784,27 +1818,27 @@ func mergeTags(dst *map[string]string, src map[string]string) { func (b *InMemoryBackend) applyTags(region string, ref resourceRef, tags map[string]string) { switch ref.Kind { case resourceKindCluster: - if c, ok := b.clusters[region][ref.Name]; ok { + if c, ok := tableGet(b.clusters[region], ref.Name); ok { mergeTags(&c.Tags, tags) } case resourceKindACL: - if a, ok := b.acls[region][ref.Name]; ok { + if a, ok := tableGet(b.acls[region], ref.Name); ok { mergeTags(&a.Tags, tags) } case resourceKindSubnetGroup: - if sg, ok := b.subnetGroups[region][ref.Name]; ok { + if sg, ok := tableGet(b.subnetGroups[region], ref.Name); ok { mergeTags(&sg.Tags, tags) } case resourceKindUser: - if u, ok := b.users[region][ref.Name]; ok { + if u, ok := tableGet(b.users[region], ref.Name); ok { mergeTags(&u.Tags, tags) } case resourceKindParameterGroup: - if pg, ok := b.parameterGroups[region][ref.Name]; ok { + if pg, ok := tableGet(b.parameterGroups[region], ref.Name); ok { mergeTags(&pg.Tags, tags) } case resourceKindSnapshot: - if s, ok := b.snapshots[region][ref.Name]; ok { + if s, ok := tableGet(b.snapshots[region], ref.Name); ok { mergeTags(&s.Tags, tags) } } @@ -1826,27 +1860,27 @@ func (b *InMemoryBackend) removeTags(region string, ref resourceRef, tagKeys []s func (b *InMemoryBackend) tagsMapForRef(region string, ref resourceRef) map[string]string { switch ref.Kind { case resourceKindCluster: - if c, ok := b.clusters[region][ref.Name]; ok { + if c, ok := tableGet(b.clusters[region], ref.Name); ok { return c.Tags } case resourceKindACL: - if a, ok := b.acls[region][ref.Name]; ok { + if a, ok := tableGet(b.acls[region], ref.Name); ok { return a.Tags } case resourceKindSubnetGroup: - if sg, ok := b.subnetGroups[region][ref.Name]; ok { + if sg, ok := tableGet(b.subnetGroups[region], ref.Name); ok { return sg.Tags } case resourceKindUser: - if u, ok := b.users[region][ref.Name]; ok { + if u, ok := tableGet(b.users[region], ref.Name); ok { return u.Tags } case resourceKindParameterGroup: - if pg, ok := b.parameterGroups[region][ref.Name]; ok { + if pg, ok := tableGet(b.parameterGroups[region], ref.Name); ok { return pg.Tags } case resourceKindSnapshot: - if s, ok := b.snapshots[region][ref.Name]; ok { + if s, ok := tableGet(b.snapshots[region], ref.Name); ok { return s.Tags } } @@ -1863,12 +1897,12 @@ func (b *InMemoryBackend) CreateSnapshot(ctx context.Context, req *createSnapsho region := getRegion(ctx, b.defaultRegion) - c, ok := b.clustersStore(region)[req.ClusterName] + c, ok := b.clustersStore(region).Get(req.ClusterName) if !ok { return nil, ErrClusterNotFound } - if _, exists := b.snapshotsStore(region)[req.SnapshotName]; exists { + if _, exists := b.snapshotsStore(region).Get(req.SnapshotName); exists { return nil, ErrSnapshotAlreadyExists } @@ -1901,7 +1935,7 @@ func (b *InMemoryBackend) CreateSnapshot(ctx context.Context, req *createSnapsho }, } - b.snapshotsStore(region)[req.SnapshotName] = s + b.snapshotsStore(region).Put(s) b.arnToResourceStore(region)[snapshotARN] = resourceRef{Kind: resourceKindSnapshot, Name: req.SnapshotName} b.appendEventLocked(region, &Event{ @@ -1924,10 +1958,10 @@ func (b *InMemoryBackend) DescribeSnapshots( defer b.mu.RUnlock() region := getRegion(ctx, b.defaultRegion) - store := b.snapshots[region] + t := b.snapshots[region] if name != "" { - s, ok := store[name] + s, ok := tableGet(t, name) if !ok { return nil, ErrSnapshotNotFound } @@ -1935,8 +1969,9 @@ func (b *InMemoryBackend) DescribeSnapshots( return []*Snapshot{cloneSnapshot(s)}, nil } - result := make([]*Snapshot, 0, len(store)) - for _, s := range store { + all := tableAll(t) + result := make([]*Snapshot, 0, len(all)) + for _, s := range all { if clusterName != "" && s.ClusterName != clusterName { continue } @@ -1962,7 +1997,7 @@ func (b *InMemoryBackend) CopySnapshot(ctx context.Context, req *copySnapshotReq region := getRegion(ctx, b.defaultRegion) - src, ok := b.snapshotsStore(region)[req.SourceSnapshotName] + src, ok := b.snapshotsStore(region).Get(req.SourceSnapshotName) if !ok { return nil, ErrSnapshotNotFound } @@ -1971,7 +2006,7 @@ func (b *InMemoryBackend) CopySnapshot(ctx context.Context, req *copySnapshotReq return cloneSnapshot(src), nil } - if _, exists := b.snapshotsStore(region)[req.TargetSnapshotName]; exists { + if _, exists := b.snapshotsStore(region).Get(req.TargetSnapshotName); exists { return nil, ErrSnapshotAlreadyExists } @@ -2001,7 +2036,7 @@ func (b *InMemoryBackend) CopySnapshot(ctx context.Context, req *copySnapshotReq ClusterConfiguration: src.ClusterConfiguration, } - b.snapshotsStore(region)[req.TargetSnapshotName] = dst + b.snapshotsStore(region).Put(dst) b.arnToResourceStore(region)[targetARN] = resourceRef{Kind: resourceKindSnapshot, Name: req.TargetSnapshotName} return dst, nil @@ -2014,12 +2049,12 @@ func (b *InMemoryBackend) DeleteSnapshot(ctx context.Context, name string) (*Sna region := getRegion(ctx, b.defaultRegion) - s, ok := b.snapshotsStore(region)[name] + s, ok := b.snapshotsStore(region).Get(name) if !ok { return nil, ErrSnapshotNotFound } - delete(b.snapshotsStore(region), name) + b.snapshotsStore(region).Delete(name) delete(b.arnToResourceStore(region), s.ARN) b.appendEventLocked(region, &Event{ @@ -2039,7 +2074,7 @@ func (b *InMemoryBackend) ExportSnapshot(ctx context.Context, req *exportSnapsho region := getRegion(ctx, b.defaultRegion) - s, ok := b.snapshotsStore(region)[req.SnapshotName] + s, ok := b.snapshotsStore(region).Get(req.SnapshotName) if !ok { return nil, ErrSnapshotNotFound } @@ -2264,7 +2299,7 @@ func (b *InMemoryBackend) CreateMultiRegionCluster( fullName := "virv-" + req.MultiRegionClusterNameSuffix - if _, exists := b.multiRegionClusters[fullName]; exists { + if _, exists := b.multiRegionClusters.Get(fullName); exists { return nil, ErrMultiRegionClusterAlreadyExists } @@ -2293,7 +2328,7 @@ func (b *InMemoryBackend) CreateMultiRegionCluster( CreatedAt: time.Now(), } - b.multiRegionClusters[fullName] = mrc + b.multiRegionClusters.Put(mrc) b.arnToResourceStore(region)[mrARN] = resourceRef{Kind: resourceKindMultiRegionCluster, Name: fullName} return mrc, nil @@ -2306,12 +2341,12 @@ func (b *InMemoryBackend) DeleteMultiRegionCluster(ctx context.Context, name str region := getRegion(ctx, b.defaultRegion) - mrc, ok := b.multiRegionClusters[name] + mrc, ok := b.multiRegionClusters.Get(name) if !ok { return nil, ErrMultiRegionClusterNotFound } - delete(b.multiRegionClusters, name) + b.multiRegionClusters.Delete(name) delete(b.arnToResourceStore(region), mrc.ARN) return cloneMultiRegionCluster(mrc), nil @@ -2323,7 +2358,7 @@ func (b *InMemoryBackend) DescribeMultiRegionClusters(_ context.Context, name st defer b.mu.RUnlock() if name != "" { - mrc, ok := b.multiRegionClusters[name] + mrc, ok := b.multiRegionClusters.Get(name) if !ok { return nil, ErrMultiRegionClusterNotFound } @@ -2331,8 +2366,9 @@ func (b *InMemoryBackend) DescribeMultiRegionClusters(_ context.Context, name st return []*MultiRegionCluster{cloneMultiRegionCluster(mrc)}, nil } - result := make([]*MultiRegionCluster, 0, len(b.multiRegionClusters)) - for _, mrc := range b.multiRegionClusters { + all := b.multiRegionClusters.All() + result := make([]*MultiRegionCluster, 0, len(all)) + for _, mrc := range all { result = append(result, cloneMultiRegionCluster(mrc)) } sort.Slice(result, func(i, j int) bool { @@ -2350,7 +2386,7 @@ func (b *InMemoryBackend) UpdateMultiRegionCluster( b.mu.Lock() defer b.mu.Unlock() - mrc, ok := b.multiRegionClusters[req.MultiRegionClusterName] + mrc, ok := b.multiRegionClusters.Get(req.MultiRegionClusterName) if !ok { return nil, ErrMultiRegionClusterNotFound } @@ -2384,7 +2420,7 @@ func (b *InMemoryBackend) DescribeMultiRegionParameterGroups( defer b.mu.RUnlock() if name != "" { - mrpg, ok := b.multiRegionParameterGroups[name] + mrpg, ok := b.multiRegionParameterGroups.Get(name) if !ok { return nil, ErrMultiRegionParameterGroupNotFound } @@ -2392,8 +2428,9 @@ func (b *InMemoryBackend) DescribeMultiRegionParameterGroups( return []*MultiRegionParameterGroup{cloneMultiRegionParameterGroup(mrpg)}, nil } - result := make([]*MultiRegionParameterGroup, 0, len(b.multiRegionParameterGroups)) - for _, mrpg := range b.multiRegionParameterGroups { + all := b.multiRegionParameterGroups.All() + result := make([]*MultiRegionParameterGroup, 0, len(all)) + for _, mrpg := range all { result = append(result, cloneMultiRegionParameterGroup(mrpg)) } sort.Slice(result, func(i, j int) bool { @@ -2419,7 +2456,7 @@ func (b *InMemoryBackend) DescribeParameters( return nil, fmt.Errorf("parameter group name is required: %w", ErrValidation) } - pg, ok := b.parameterGroups[region][parameterGroupName] + pg, ok := tableGet(b.parameterGroups[region], parameterGroupName) if !ok { return nil, ErrParameterGroupNotFound } @@ -2441,7 +2478,7 @@ func (b *InMemoryBackend) ResetParameterGroup( region := getRegion(ctx, b.defaultRegion) - pg, ok := b.parameterGroupsStore(region)[name] + pg, ok := b.parameterGroupsStore(region).Get(name) if !ok { return nil, ErrParameterGroupNotFound } @@ -2472,7 +2509,7 @@ func (b *InMemoryBackend) FailoverShard(ctx context.Context, clusterName, shardN region := getRegion(ctx, b.defaultRegion) - c, ok := b.clustersStore(region)[clusterName] + c, ok := b.clustersStore(region).Get(clusterName) if !ok { return nil, ErrClusterNotFound } @@ -2514,7 +2551,7 @@ func (b *InMemoryBackend) ListAllowedNodeTypeUpdates(ctx context.Context, cluste region := getRegion(ctx, b.defaultRegion) - if _, ok := b.clusters[region][clusterName]; !ok { + if _, ok := tableGet(b.clusters[region], clusterName); !ok { return nil, ErrClusterNotFound } @@ -2530,7 +2567,7 @@ func (b *InMemoryBackend) ListAllowedMultiRegionClusterUpdates( defer b.mu.RUnlock() - if _, ok := b.multiRegionClusters[clusterName]; !ok { + if _, ok := b.multiRegionClusters.Get(clusterName); !ok { return nil, ErrMultiRegionClusterNotFound } @@ -2548,7 +2585,7 @@ func (b *InMemoryBackend) BatchUpdateCluster(ctx context.Context, clusterNames [ result := make(map[string]*Cluster, len(clusterNames)) for _, name := range clusterNames { - if c, ok := b.clusters[region][name]; ok { + if c, ok := tableGet(b.clusters[region], name); ok { result[name] = cloneCluster(c) } } @@ -2602,10 +2639,10 @@ func (b *InMemoryBackend) DescribeReservedNodes( defer b.mu.RUnlock() region := getRegion(ctx, b.defaultRegion) - store := b.reservedNodes[region] + all := tableAll(b.reservedNodes[region]) - result := make([]*ReservedNode, 0, len(store)) - for _, rn := range store { + result := make([]*ReservedNode, 0, len(all)) + for _, rn := range all { if req.ReservedNodeID != "" && rn.ReservedNodeID != req.ReservedNodeID { continue } @@ -2706,7 +2743,7 @@ func (b *InMemoryBackend) PurchaseReservedNodesOffering( reservationID = req.ReservedNodesOfferingID + "-reservation" } - if _, exists := b.reservedNodesStore(region)[reservationID]; exists { + if _, exists := b.reservedNodesStore(region).Get(reservationID); exists { return nil, fmt.Errorf("reserved node %q already exists: %w", reservationID, ErrReservationAlreadyExists) } @@ -2732,7 +2769,7 @@ func (b *InMemoryBackend) PurchaseReservedNodesOffering( ARN: rnARN, } - b.reservedNodesStore(region)[reservationID] = rn + b.reservedNodesStore(region).Put(rn) cp := *rn @@ -2753,7 +2790,7 @@ func (b *InMemoryBackend) DescribeMultiRegionParameters( return nil, fmt.Errorf("parameter group name is required: %w", ErrValidation) } - mrpg, ok := b.multiRegionParameterGroups[parameterGroupName] + mrpg, ok := b.multiRegionParameterGroups.Get(parameterGroupName) if !ok { return nil, ErrMultiRegionParameterGroupNotFound } @@ -2769,8 +2806,9 @@ func (b *InMemoryBackend) DescribeServiceUpdates( b.mu.RLock() defer b.mu.RUnlock() - result := make([]*ServiceUpdate, 0, len(b.serviceUpdates)) - for _, su := range b.serviceUpdates { + allUpdates := b.serviceUpdates.All() + result := make([]*ServiceUpdate, 0, len(allUpdates)) + for _, su := range allUpdates { if req.ServiceUpdateName != "" && su.ServiceUpdateName != req.ServiceUpdateName { continue } @@ -2819,8 +2857,8 @@ func (b *InMemoryBackend) ListClusters() []*Cluster { defer b.mu.RUnlock() var result []*Cluster - for _, regionClusters := range b.clusters { - for _, c := range regionClusters { + for _, t := range b.clusters { + for _, c := range t.All() { result = append(result, cloneCluster(c)) } } @@ -2838,46 +2876,46 @@ func (b *InMemoryBackend) Purge(ctx context.Context, cutoff time.Time) { b.mu.Lock() defer b.mu.Unlock() - for region, regionClusters := range b.clusters { - purgeMemoryDBMap(ctx, regionClusters, cutoff, + for region, t := range b.clusters { + purgeTable(ctx, t, cutoff, clusterKeyFn, func(c *Cluster) time.Time { return c.CreatedAt }, - func(_ string, c *Cluster) { delete(b.arnToResource[region], c.ARN) }, + func(c *Cluster) { delete(b.arnToResource[region], c.ARN) }, ) } - for region, regionACLs := range b.acls { - purgeMemoryDBMapFiltered(ctx, regionACLs, cutoff, - func(name string, _ *ACL) bool { return name == openAccessACL }, + for region, t := range b.acls { + purgeTableFiltered(ctx, t, cutoff, aclKeyFn, + func(a *ACL) bool { return a.Name == openAccessACL }, func(a *ACL) time.Time { return a.CreatedAt }, - func(_ string, a *ACL) { delete(b.arnToResource[region], a.ARN) }, + func(a *ACL) { delete(b.arnToResource[region], a.ARN) }, ) } - for region, regionSGs := range b.subnetGroups { - purgeMemoryDBMap(ctx, regionSGs, cutoff, + for region, t := range b.subnetGroups { + purgeTable(ctx, t, cutoff, subnetGroupKeyFn, func(sg *SubnetGroup) time.Time { return sg.CreatedAt }, - func(_ string, sg *SubnetGroup) { delete(b.arnToResource[region], sg.ARN) }, + func(sg *SubnetGroup) { delete(b.arnToResource[region], sg.ARN) }, ) } - for region, regionUsers := range b.users { - purgeMemoryDBMap(ctx, regionUsers, cutoff, + for region, t := range b.users { + purgeTable(ctx, t, cutoff, userKeyFn, func(u *User) time.Time { return u.CreatedAt }, - func(_ string, u *User) { delete(b.arnToResource[region], u.ARN) }, + func(u *User) { delete(b.arnToResource[region], u.ARN) }, ) } - for region, regionPGs := range b.parameterGroups { - purgeMemoryDBMap(ctx, regionPGs, cutoff, + for region, t := range b.parameterGroups { + purgeTable(ctx, t, cutoff, parameterGroupKeyFn, func(pg *ParameterGroup) time.Time { return pg.CreatedAt }, - func(_ string, pg *ParameterGroup) { delete(b.arnToResource[region], pg.ARN) }, + func(pg *ParameterGroup) { delete(b.arnToResource[region], pg.ARN) }, ) } - for region, regionSnaps := range b.snapshots { - purgeMemoryDBMap(ctx, regionSnaps, cutoff, + for region, t := range b.snapshots { + purgeTable(ctx, t, cutoff, snapshotKeyFn, func(s *Snapshot) time.Time { return s.CreatedAt }, - func(_ string, s *Snapshot) { delete(b.arnToResource[region], s.ARN) }, + func(s *Snapshot) { delete(b.arnToResource[region], s.ARN) }, ) } - purgeMemoryDBMap(ctx, b.multiRegionClusters, cutoff, + purgeTable(ctx, b.multiRegionClusters, cutoff, multiRegionClusterKeyFn, func(mrc *MultiRegionCluster) time.Time { return mrc.CreatedAt }, - func(_ string, _ *MultiRegionCluster) {}, + func(_ *MultiRegionCluster) {}, ) if ctx.Err() != nil { @@ -2896,45 +2934,48 @@ func (b *InMemoryBackend) Purge(ctx context.Context, cutoff time.Time) { } } -// purgeMemoryDBMap deletes entries from m that were created before cutoff, -// calling cleanup for each deleted entry. -func purgeMemoryDBMap[V any]( +// purgeTable deletes entries from t that were created before cutoff, calling +// cleanup for each deleted entry. keyFn must be the same key function t was +// constructed with (see store_setup.go). +func purgeTable[V any]( ctx context.Context, - m map[string]V, + t *store.Table[V], cutoff time.Time, - getTime func(V) time.Time, - cleanup func(string, V), + keyFn func(*V) string, + getTime func(*V) time.Time, + cleanup func(*V), ) { - for k, v := range m { + for _, v := range t.All() { if ctx.Err() != nil { return } if getTime(v).Before(cutoff) { - cleanup(k, v) - delete(m, k) + cleanup(v) + t.Delete(keyFn(v)) } } } -// purgeMemoryDBMapFiltered is like purgeMemoryDBMap but skips entries where skip returns true. -func purgeMemoryDBMapFiltered[V any]( +// purgeTableFiltered is like purgeTable but skips entries where skip returns true. +func purgeTableFiltered[V any]( ctx context.Context, - m map[string]V, + t *store.Table[V], cutoff time.Time, - skip func(string, V) bool, - getTime func(V) time.Time, - cleanup func(string, V), + keyFn func(*V) string, + skip func(*V) bool, + getTime func(*V) time.Time, + cleanup func(*V), ) { - for k, v := range m { + for _, v := range t.All() { if ctx.Err() != nil { return } - if skip(k, v) { + if skip(v) { continue } if getTime(v).Before(cutoff) { - cleanup(k, v) - delete(m, k) + cleanup(v) + t.Delete(keyFn(v)) } } } @@ -3060,7 +3101,7 @@ func (b *InMemoryBackend) AddClusterInternal(name, nodeType string) *Cluster { CreatedAt: time.Now(), Region: b.defaultRegion, } - b.clustersStore(b.defaultRegion)[name] = c + b.clustersStore(b.defaultRegion).Put(c) b.arnToResourceStore(b.defaultRegion)[clusterARN] = resourceRef{Kind: resourceKindCluster, Name: name} return c @@ -3080,7 +3121,7 @@ func (b *InMemoryBackend) AddACLInternal(name string) *ACL { Tags: make(map[string]string), CreatedAt: time.Now(), } - b.aclsStore(b.defaultRegion)[name] = a + b.aclsStore(b.defaultRegion).Put(a) b.arnToResourceStore(b.defaultRegion)[aclARN] = resourceRef{Kind: resourceKindACL, Name: name} return a @@ -3100,7 +3141,7 @@ func (b *InMemoryBackend) AddSnapshotInternal(name, clusterName string) *Snapsho Tags: make(map[string]string), CreatedAt: time.Now(), } - b.snapshotsStore(b.defaultRegion)[name] = s + b.snapshotsStore(b.defaultRegion).Put(s) b.arnToResourceStore(b.defaultRegion)[snapshotARN] = resourceRef{Kind: resourceKindSnapshot, Name: name} return s @@ -3120,7 +3161,7 @@ func (b *InMemoryBackend) AddUserInternal(name, accessString string) *User { Tags: make(map[string]string), CreatedAt: time.Now(), } - b.usersStore(b.defaultRegion)[name] = u + b.usersStore(b.defaultRegion).Put(u) b.arnToResourceStore(b.defaultRegion)[userARN] = resourceRef{Kind: resourceKindUser, Name: name} return u @@ -3138,7 +3179,7 @@ func (b *InMemoryBackend) AddSubnetGroupInternal(name string) *SubnetGroup { Tags: make(map[string]string), CreatedAt: time.Now(), } - b.subnetGroupsStore(b.defaultRegion)[name] = sg + b.subnetGroupsStore(b.defaultRegion).Put(sg) b.arnToResourceStore(b.defaultRegion)[sgARN] = resourceRef{Kind: resourceKindSubnetGroup, Name: name} return sg @@ -3158,7 +3199,7 @@ func (b *InMemoryBackend) AddParameterGroupInternal(name, family string) *Parame Tags: make(map[string]string), CreatedAt: time.Now(), } - b.parameterGroupsStore(b.defaultRegion)[name] = pg + b.parameterGroupsStore(b.defaultRegion).Put(pg) b.arnToResourceStore(b.defaultRegion)[pgARN] = resourceRef{Kind: resourceKindParameterGroup, Name: name} return pg @@ -3178,7 +3219,7 @@ func (b *InMemoryBackend) AddMultiRegionParameterGroupInternal(name, family stri Parameters: make(map[string]string), CreatedAt: time.Now(), } - b.multiRegionParameterGroups[name] = mrpg + b.multiRegionParameterGroups.Put(mrpg) return mrpg } diff --git a/services/memorydb/coverage_boost_test.go b/services/memorydb/coverage_boost_test.go index fc9ef14c4..fe8d9ffa7 100644 --- a/services/memorydb/coverage_boost_test.go +++ b/services/memorydb/coverage_boost_test.go @@ -1967,27 +1967,57 @@ func TestHandler_Persistence_SnapshotRestoreWithNilTags(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - // A minimal snapshot with resource entries that have nil tags. - // Clusters, acls, subnetGroups, users, parameterGroups, snapshots, - // reservedNodes, and arnToResource are now region-keyed (map[region]map[name]*T). - // Events is also region-keyed (map[region][]*Event). + // A minimal Phase-3.3-shaped snapshot with resource entries that have + // nil tags. Clusters, acls, subnetGroups, users, parameterGroups, + // snapshots, and reservedNodes are region-keyed slices + // (map[region][]*T), matching store.Table.Snapshot's deterministic + // output. multiRegionClusters, multiRegionParameterGroups, and + // serviceUpdates are partition-scoped (not region-nested) and live + // inside the registry-backed "tables" blob instead. snapJSON := `{ - "clusters": {"us-east-1": {"cl": {"Name": "cl", "ARN": "arn:cl", "Tags": null}}}, - "acls": {"us-east-1": {"acl": {"Name": "acl", "ARN": "arn:acl", "Tags": null}}}, - "subnetGroups": {"us-east-1": {"sg": {"Name": "sg", "ARN": "arn:sg", "Tags": null}}}, - "users": {"us-east-1": {"u": {"Name": "u", "ARN": "arn:u", "Tags": null}}}, - "parameterGroups": {"us-east-1": {"pg": {"Name": "pg", "ARN": "arn:pg", "Tags": null, "Parameters": null}}}, - "snapshots": {"us-east-1": {"sn": {"Name": "sn", "ARN": "arn:sn", "Tags": null}}}, - "multiRegionClusters": {"mrc": {"MultiRegionClusterName": "mrc", "Tags": null}}, - "multiRegionParameterGroups": {"mrpg": {"Name": "mrpg", "Tags": null, "Parameters": null}}, + "version": 1, + "tables": { + "multiRegionClusters": [{"MultiRegionClusterName": "mrc", "Tags": null}], + "multiRegionParameterGroups": [{"Name": "mrpg", "Tags": null, "Parameters": null}], + "serviceUpdates": [] + }, + "clusters": {"us-east-1": [{"Name": "cl", "ARN": "arn:cl", "Tags": null}]}, + "acls": {"us-east-1": [{"Name": "acl", "ARN": "arn:acl", "Tags": null}]}, + "subnetGroups": {"us-east-1": [{"Name": "sg", "ARN": "arn:sg", "Tags": null}]}, + "users": {"us-east-1": [{"Name": "u", "ARN": "arn:u", "Tags": null}]}, + "parameterGroups": {"us-east-1": [{"Name": "pg", "ARN": "arn:pg", "Tags": null, "Parameters": null}]}, + "snapshots": {"us-east-1": [{"Name": "sn", "ARN": "arn:sn", "Tags": null}]}, "reservedNodes": {}, "arnToResource": {}, - "events": {} + "events": {}, + "accountID": "123456789012", + "defaultRegion": "us-east-1" }` - h := newTestHandler(t) + b := memorydb.NewInMemoryBackend(testAccountID, testRegion) + h := memorydb.NewHandler(b) + h.AccountID = testAccountID + h.DefaultRegion = testRegion + err := h.Restore(t.Context(), []byte(snapJSON)) require.NoError(t, err) + + clusters, err := b.DescribeClusters(t.Context(), "cl") + require.NoError(t, err) + require.Len(t, clusters, 1) + assert.NotNil(t, clusters[0].Tags) + + pgs, err := b.DescribeParameterGroups(t.Context(), "pg") + require.NoError(t, err) + require.Len(t, pgs, 1) + assert.NotNil(t, pgs[0].Tags) + assert.NotNil(t, pgs[0].Parameters) + + mrpgs, err := b.DescribeMultiRegionParameterGroups(t.Context(), "mrpg") + require.NoError(t, err) + require.Len(t, mrpgs, 1) + assert.NotNil(t, mrpgs[0].Tags) + assert.NotNil(t, mrpgs[0].Parameters) }) } } diff --git a/services/memorydb/exports.go b/services/memorydb/exports.go index 085d5e9d0..61d2e336c 100644 --- a/services/memorydb/exports.go +++ b/services/memorydb/exports.go @@ -32,6 +32,12 @@ type ExportedCopySnapshotRequest = copySnapshotRequest // ExportedCreateMultiRegionClusterRequest aliases createMultiRegionClusterRequest for testing. type ExportedCreateMultiRegionClusterRequest = createMultiRegionClusterRequest +// ExportedPurchaseReservedNodesOfferingRequest aliases purchaseReservedNodesOfferingRequest for testing. +type ExportedPurchaseReservedNodesOfferingRequest = purchaseReservedNodesOfferingRequest + +// ExportedDescribeReservedNodesRequest aliases describeReservedNodesRequest for testing. +type ExportedDescribeReservedNodesRequest = describeReservedNodesRequest + // ExportedDescribeEngineVersionsRequest aliases describeEngineVersionsRequest for testing. type ExportedDescribeEngineVersionsRequest = describeEngineVersionsRequest @@ -54,8 +60,8 @@ func ClusterCount(b *InMemoryBackend) int { b.mu.RLock() defer b.mu.RUnlock() total := 0 - for _, m := range b.clusters { - total += len(m) + for _, t := range b.clusters { + total += t.Len() } return total @@ -66,8 +72,8 @@ func ACLCount(b *InMemoryBackend) int { b.mu.RLock() defer b.mu.RUnlock() total := 0 - for _, m := range b.acls { - total += len(m) + for _, t := range b.acls { + total += t.Len() } return total @@ -78,8 +84,8 @@ func SnapshotCount(b *InMemoryBackend) int { b.mu.RLock() defer b.mu.RUnlock() total := 0 - for _, m := range b.snapshots { - total += len(m) + for _, t := range b.snapshots { + total += t.Len() } return total @@ -91,8 +97,8 @@ func UserCount(b *InMemoryBackend) int { defer b.mu.RUnlock() total := 0 - for _, m := range b.users { - total += len(m) + for _, t := range b.users { + total += t.Len() } return total @@ -104,8 +110,8 @@ func SubnetGroupCount(b *InMemoryBackend) int { defer b.mu.RUnlock() total := 0 - for _, m := range b.subnetGroups { - total += len(m) + for _, t := range b.subnetGroups { + total += t.Len() } return total @@ -117,8 +123,8 @@ func ParameterGroupCount(b *InMemoryBackend) int { defer b.mu.RUnlock() total := 0 - for _, m := range b.parameterGroups { - total += len(m) + for _, t := range b.parameterGroups { + total += t.Len() } return total @@ -141,7 +147,7 @@ func MultiRegionClusterCount(b *InMemoryBackend) int { b.mu.RLock() defer b.mu.RUnlock() - return len(b.multiRegionClusters) + return b.multiRegionClusters.Len() } // ARNIndexSize returns the number of entries in the ARN-to-resource index. diff --git a/services/memorydb/isolation_test.go b/services/memorydb/isolation_test.go index 05e7e5769..97bff1da5 100644 --- a/services/memorydb/isolation_test.go +++ b/services/memorydb/isolation_test.go @@ -102,3 +102,78 @@ func TestMemoryDBDefaultRegionFallback(t *testing.T) { require.Error(t, err) assert.Nil(t, other) } + +// TestSnapshotRestore_CrossRegion verifies per-region store.Table state +// (which is deliberately NOT registered on b.registry -- see +// store_setup.go's doc comment) round-trips through Snapshot/Restore for +// more than one region. This complements the exported-API full-state test in +// persistence_test.go, which only exercises the backend's default region. +func TestSnapshotRestore_CrossRegion(t *testing.T) { + t.Parallel() + + original := NewInMemoryBackend("123456789012", "us-east-1") + + ctxEast := memdbCtxRegion("us-east-1") + ctxWest := memdbCtxRegion("us-west-2") + + _, err := original.CreateCluster(ctxEast, &createClusterRequest{ + ClusterName: "cr-cluster-east", + NodeType: "db.r6g.large", + }) + require.NoError(t, err) + _, err = original.CreateCluster(ctxWest, &createClusterRequest{ + ClusterName: "cr-cluster-west", + NodeType: "db.r6g.xlarge", + }) + require.NoError(t, err) + + snap := original.Snapshot(context.Background()) + require.NotNil(t, snap) + + fresh := NewInMemoryBackend("123456789012", "us-east-1") + require.NoError(t, fresh.Restore(context.Background(), snap)) + + eastAfter, err := fresh.DescribeClusters(ctxEast, "") + require.NoError(t, err) + require.Len(t, eastAfter, 1) + assert.Equal(t, "cr-cluster-east", eastAfter[0].Name) + + westAfter, err := fresh.DescribeClusters(ctxWest, "") + require.NoError(t, err) + require.Len(t, westAfter, 1) + assert.Equal(t, "cr-cluster-west", westAfter[0].Name) +} + +// TestReset_RegisteredTableSurvivesRepeatedCalls guards against a +// regression where Reset accidentally re-registers a registry-backed table +// (multiRegionClusters, multiRegionParameterGroups, or serviceUpdates -- +// store.Register panics on a duplicate name) instead of clearing it in place +// via b.registry.ResetAll(). +func TestReset_RegisteredTableSurvivesRepeatedCalls(t *testing.T) { + t.Parallel() + + b := NewInMemoryBackend("123456789012", "us-east-1") + + ctx := context.Background() + + _, err := b.CreateMultiRegionCluster(ctx, &createMultiRegionClusterRequest{ + MultiRegionClusterNameSuffix: "reset-mrc", + NodeType: "db.r6g.large", + }) + require.NoError(t, err) + + assert.NotPanics(t, func() { + b.Reset() + b.Reset() + }) + + mrcs, err := b.DescribeMultiRegionClusters(ctx, "") + require.NoError(t, err) + assert.Empty(t, mrcs) + + _, err = b.CreateMultiRegionCluster(ctx, &createMultiRegionClusterRequest{ + MultiRegionClusterNameSuffix: "post-reset-mrc", + NodeType: "db.r6g.large", + }) + require.NoError(t, err) +} diff --git a/services/memorydb/persistence.go b/services/memorydb/persistence.go index c0af41c89..8ec288f10 100644 --- a/services/memorydb/persistence.go +++ b/services/memorydb/persistence.go @@ -6,44 +6,87 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) +// memorydbSnapshotVersion identifies the shape of [backendSnapshot]. It must +// be bumped whenever a change to the set/shape of tables captured below would +// make an older snapshot unsafe to decode as the current shape. Restore +// compares this against the persisted value and discards (rather than +// attempts to partially decode) any mismatch -- see Restore below. This +// mirrors the services/sqs pilot (commit 0f09d77c) and the +// services/elasticache rollout (commit 06806317). +const memorydbSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the MemoryDB backend. +// +// Tables holds the JSON-encoded array for every table registered on +// b.registry -- multiRegionClusters, multiRegionParameterGroups, and +// serviceUpdates, the three partition-scoped (non-region-nested) resources -- +// produced by [store.Registry.SnapshotAll]. +// +// Every other resource is nested per-region (region -> *store.Table[T]) and +// is NOT registered on b.registry, because the set of regions is only known +// at runtime (see store_setup.go's doc comment); each is instead captured +// directly below as region -> that table's own deterministic +// [store.Table.Snapshot] slice. type backendSnapshot struct { - Clusters map[string]map[string]*Cluster `json:"clusters"` - ACLs map[string]map[string]*ACL `json:"acls"` - SubnetGroups map[string]map[string]*SubnetGroup `json:"subnetGroups"` - Users map[string]map[string]*User `json:"users"` - ParameterGroups map[string]map[string]*ParameterGroup `json:"parameterGroups"` - Snapshots map[string]map[string]*Snapshot `json:"snapshots"` - MultiRegionClusters map[string]*MultiRegionCluster `json:"multiRegionClusters"` - MultiRegionParameterGroups map[string]*MultiRegionParameterGroup `json:"multiRegionParameterGroups"` - ReservedNodes map[string]map[string]*ReservedNode `json:"reservedNodes"` - ARNToResource map[string]map[string]resourceRef `json:"arnToResource"` - ServiceUpdates map[string]*ServiceUpdate `json:"serviceUpdates"` - Events map[string][]*Event `json:"events"` - AccountID string `json:"accountID"` - DefaultRegion string `json:"defaultRegion"` + ParameterGroups map[string][]*ParameterGroup `json:"parameterGroups"` + Tables map[string]json.RawMessage `json:"tables"` + Clusters map[string][]*Cluster `json:"clusters"` + ACLs map[string][]*ACL `json:"acls"` + SubnetGroups map[string][]*SubnetGroup `json:"subnetGroups"` + Users map[string][]*User `json:"users"` + Snapshots map[string][]*Snapshot `json:"snapshots"` + ReservedNodes map[string][]*ReservedNode `json:"reservedNodes"` + ARNToResource map[string]map[string]resourceRef `json:"arnToResource"` + Events map[string][]*Event `json:"events"` + AccountID string `json:"accountID"` + DefaultRegion string `json:"defaultRegion"` + Version int `json:"version"` } +// snapshotAllRegions captures the deterministic [store.Table.Snapshot] slice +// for every region in m. +func snapshotAllRegions[T any](m map[string]*store.Table[T]) map[string][]*T { + out := make(map[string][]*T, len(m)) + for region, t := range m { + out[region] = t.Snapshot() + } + + return out +} + +// Snapshot serialises the backend state to JSON. func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock() defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "memorydb: snapshot table marshal failed", "error", err) + + return nil + } + snap := backendSnapshot{ - Clusters: b.clusters, - ACLs: b.acls, - SubnetGroups: b.subnetGroups, - Users: b.users, - ParameterGroups: b.parameterGroups, - Snapshots: b.snapshots, - MultiRegionClusters: b.multiRegionClusters, - MultiRegionParameterGroups: b.multiRegionParameterGroups, - ReservedNodes: b.reservedNodes, - Events: b.events, - ARNToResource: b.arnToResource, - ServiceUpdates: b.serviceUpdates, - AccountID: b.accountID, - DefaultRegion: b.defaultRegion, + Version: memorydbSnapshotVersion, + Tables: tables, + Clusters: snapshotAllRegions(b.clusters), + ACLs: snapshotAllRegions(b.acls), + SubnetGroups: snapshotAllRegions(b.subnetGroups), + Users: snapshotAllRegions(b.users), + ParameterGroups: snapshotAllRegions(b.parameterGroups), + Snapshots: snapshotAllRegions(b.snapshots), + ReservedNodes: snapshotAllRegions(b.reservedNodes), + ARNToResource: b.arnToResource, + Events: b.events, + AccountID: b.accountID, + DefaultRegion: b.defaultRegion, } data, err := json.Marshal(snap) @@ -56,77 +99,93 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { return data } +// restoreRegionTables resets the outer per-region map via reset, then +// restores each region's table from data using storeFn (one of the +// InMemoryBackend "*Store" lazy accessors). This mirrors the pre-conversion +// full-replace Restore semantics: a region present in the live backend but +// absent from data ends up empty, exactly as a direct `b.field = data` +// assignment would have left it. +func restoreRegionTables[T any](reset func(), storeFn func(string) *store.Table[T], data map[string][]*T) { + reset() + + for region, items := range data { + storeFn(region).Restore(items) + } +} + +// Restore loads backend state from a JSON snapshot. func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { var snap backendSnapshot if err := persistence.UnmarshalSnapshot(ctx, "memorydb", data, &snap); err != nil { return err } - ensureNonNilMaps(&snap) - fixNilTagsInSnapshot(&snap) b.mu.Lock() defer b.mu.Unlock() - b.clusters = snap.Clusters - b.acls = snap.ACLs - b.subnetGroups = snap.SubnetGroups - b.users = snap.Users - b.parameterGroups = snap.ParameterGroups - b.snapshots = snap.Snapshots - b.multiRegionClusters = snap.MultiRegionClusters - b.multiRegionParameterGroups = snap.MultiRegionParameterGroups - b.reservedNodes = snap.ReservedNodes - b.events = snap.Events - b.arnToResource = snap.ARNToResource - b.serviceUpdates = snap.ServiceUpdates - b.accountID = snap.AccountID - b.defaultRegion = snap.DefaultRegion + if snap.Version != memorydbSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. + logger.Load(ctx).WarnContext(ctx, + "memorydb: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", memorydbSnapshotVersion) - return nil -} + b.resetLocked() -func ensureNonNilMaps(snap *backendSnapshot) { - if snap.Clusters == nil { - snap.Clusters = make(map[string]map[string]*Cluster) - } - if snap.ACLs == nil { - snap.ACLs = make(map[string]map[string]*ACL) - } - if snap.SubnetGroups == nil { - snap.SubnetGroups = make(map[string]map[string]*SubnetGroup) - } - if snap.Users == nil { - snap.Users = make(map[string]map[string]*User) - } - if snap.ParameterGroups == nil { - snap.ParameterGroups = make(map[string]map[string]*ParameterGroup) - } - if snap.Snapshots == nil { - snap.Snapshots = make(map[string]map[string]*Snapshot) - } - if snap.MultiRegionClusters == nil { - snap.MultiRegionClusters = make(map[string]*MultiRegionCluster) - } - if snap.MultiRegionParameterGroups == nil { - snap.MultiRegionParameterGroups = make(map[string]*MultiRegionParameterGroup) - } - if snap.ReservedNodes == nil { - snap.ReservedNodes = make(map[string]map[string]*ReservedNode) + return nil } - if snap.ARNToResource == nil { - snap.ARNToResource = make(map[string]map[string]resourceRef) + + fixNilTagsInSnapshot(&snap) + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return err } - if snap.ServiceUpdates == nil { - snap.ServiceUpdates = make(map[string]*ServiceUpdate) + + restoreRegionTables(func() { b.clusters = make(map[string]*store.Table[Cluster]) }, + b.clustersStore, snap.Clusters) + restoreRegionTables(func() { b.acls = make(map[string]*store.Table[ACL]) }, + b.aclsStore, snap.ACLs) + restoreRegionTables(func() { b.subnetGroups = make(map[string]*store.Table[SubnetGroup]) }, + b.subnetGroupsStore, snap.SubnetGroups) + restoreRegionTables(func() { b.users = make(map[string]*store.Table[User]) }, + b.usersStore, snap.Users) + restoreRegionTables(func() { b.parameterGroups = make(map[string]*store.Table[ParameterGroup]) }, + b.parameterGroupsStore, snap.ParameterGroups) + restoreRegionTables(func() { b.snapshots = make(map[string]*store.Table[Snapshot]) }, + b.snapshotsStore, snap.Snapshots) + restoreRegionTables(func() { b.reservedNodes = make(map[string]*store.Table[ReservedNode]) }, + b.reservedNodesStore, snap.ReservedNodes) + + b.fixGlobalTagsLocked() + + if snap.ARNToResource != nil { + b.arnToResource = snap.ARNToResource + } else { + b.arnToResource = make(map[string]map[string]resourceRef) } - if snap.Events == nil { - snap.Events = make(map[string][]*Event) + + if snap.Events != nil { + b.events = snap.Events + } else { + b.events = make(map[string][]*Event) } + + b.accountID = snap.AccountID + b.defaultRegion = snap.DefaultRegion + + return nil } -func fixNestedMemoryDBTags[V any](nested map[string]map[string]V, fix func(V)) { - for _, region := range nested { - for _, item := range region { +// fixListTags calls fix on every item in every region's slice of nested, +// mutating each item in place. Used to backfill nil tag/parameter maps on +// snapshots taken before this backend guaranteed non-nil maps. +func fixListTags[T any](nested map[string][]*T, fix func(*T)) { + for _, items := range nested { + for _, item := range items { fix(item) } } @@ -146,24 +205,31 @@ func fixNilTagsInSnapshot(snap *backendSnapshot) { } func fixCoreResourceTags(snap *backendSnapshot) { - fixNestedMemoryDBTags(snap.Clusters, func(c *Cluster) { c.Tags = ensureMemoryDBMap(c.Tags) }) - fixNestedMemoryDBTags(snap.ACLs, func(a *ACL) { a.Tags = ensureMemoryDBMap(a.Tags) }) - fixNestedMemoryDBTags(snap.SubnetGroups, func(sg *SubnetGroup) { sg.Tags = ensureMemoryDBMap(sg.Tags) }) - fixNestedMemoryDBTags(snap.Users, func(u *User) { u.Tags = ensureMemoryDBMap(u.Tags) }) + fixListTags(snap.Clusters, func(c *Cluster) { c.Tags = ensureMemoryDBMap(c.Tags) }) + fixListTags(snap.ACLs, func(a *ACL) { a.Tags = ensureMemoryDBMap(a.Tags) }) + fixListTags(snap.SubnetGroups, func(sg *SubnetGroup) { sg.Tags = ensureMemoryDBMap(sg.Tags) }) + fixListTags(snap.Users, func(u *User) { u.Tags = ensureMemoryDBMap(u.Tags) }) } func fixExtendedResourceTags(snap *backendSnapshot) { - fixNestedMemoryDBTags(snap.ParameterGroups, func(pg *ParameterGroup) { + fixListTags(snap.ParameterGroups, func(pg *ParameterGroup) { pg.Tags = ensureMemoryDBMap(pg.Tags) pg.Parameters = ensureMemoryDBMap(pg.Parameters) }) - fixNestedMemoryDBTags(snap.Snapshots, func(s *Snapshot) { s.Tags = ensureMemoryDBMap(s.Tags) }) + fixListTags(snap.Snapshots, func(s *Snapshot) { s.Tags = ensureMemoryDBMap(s.Tags) }) +} - for _, mrc := range snap.MultiRegionClusters { +// fixGlobalTagsLocked backfills nil tag/parameter maps on the registry-backed +// global tables (multiRegionClusters, multiRegionParameterGroups) after +// RestoreAll has populated them from a snapshot taken before this backend +// guaranteed non-nil maps. serviceUpdates carries no tags/parameters, so it +// needs no fix-up. Must hold b.mu. +func (b *InMemoryBackend) fixGlobalTagsLocked() { + for _, mrc := range b.multiRegionClusters.All() { mrc.Tags = ensureMemoryDBMap(mrc.Tags) } - for _, mrpg := range snap.MultiRegionParameterGroups { + for _, mrpg := range b.multiRegionParameterGroups.All() { mrpg.Tags = ensureMemoryDBMap(mrpg.Tags) mrpg.Parameters = ensureMemoryDBMap(mrpg.Parameters) } diff --git a/services/memorydb/persistence_test.go b/services/memorydb/persistence_test.go new file mode 100644 index 000000000..e67adf123 --- /dev/null +++ b/services/memorydb/persistence_test.go @@ -0,0 +1,241 @@ +package memorydb_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/memorydb" +) + +func TestInMemoryBackend_SnapshotRestore(t *testing.T) { + t.Parallel() + + tests := []struct { + setup func(b *memorydb.InMemoryBackend) string + verify func(t *testing.T, b *memorydb.InMemoryBackend, name string) + name string + }{ + { + name: "round_trip_preserves_state", + setup: func(b *memorydb.InMemoryBackend) string { + c, err := b.CreateCluster(t.Context(), &memorydb.ExportedCreateClusterRequest{ + ClusterName: "test-cluster", + NodeType: "db.r6g.large", + }) + if err != nil { + return "" + } + + return c.Name + }, + verify: func(t *testing.T, b *memorydb.InMemoryBackend, name string) { + t.Helper() + + clusters, err := b.DescribeClusters(t.Context(), name) + require.NoError(t, err) + require.Len(t, clusters, 1) + assert.Equal(t, name, clusters[0].Name) + }, + }, + { + name: "empty_backend_round_trip", + setup: func(_ *memorydb.InMemoryBackend) string { return "" }, + verify: func(t *testing.T, b *memorydb.InMemoryBackend, _ string) { + t.Helper() + + assert.Equal(t, 0, memorydb.ClusterCount(b)) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + original := memorydb.NewInMemoryBackend("123456789012", "us-east-1") + name := tt.setup(original) + + snap := original.Snapshot(t.Context()) + require.NotNil(t, snap) + + fresh := memorydb.NewInMemoryBackend("123456789012", "us-east-1") + require.NoError(t, fresh.Restore(t.Context(), snap)) + + tt.verify(t, fresh, name) + }) + } +} + +func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { + t.Parallel() + + b := memorydb.NewInMemoryBackend("123456789012", "us-east-1") + err := b.Restore(t.Context(), []byte("not-valid-json")) + require.Error(t, err) +} + +// TestInMemoryBackend_SnapshotRestore_FullState seeds one instance of every +// resource type Phase 3.3 converted to pkgs/store, snapshots the backend, +// restores into a fresh backend, and checks every resource survived with its +// identity/fields intact. This guards against a table being silently dropped +// during the map -> store.Table conversion. Region-nested per-region table +// round-tripping (a second region within the same resource) and the two maps +// deliberately left un-converted (arnToResource and events) are covered +// separately by the internal-package tests in isolation_test.go, which need +// the unexported region-context key not reachable from here. +func TestInMemoryBackend_SnapshotRestore_FullState(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + original := memorydb.NewInMemoryBackend("123456789012", "us-east-1") + + _, err := original.CreateCluster(ctx, &memorydb.ExportedCreateClusterRequest{ + ClusterName: "fs-cluster", + NodeType: "db.r6g.large", + }) + require.NoError(t, err) + + _, err = original.CreateACL(ctx, &memorydb.ExportedCreateACLRequest{ACLName: "fs-acl"}) + require.NoError(t, err) + + _, err = original.CreateSubnetGroup(ctx, &memorydb.ExportedCreateSubnetGroupRequest{ + SubnetGroupName: "fs-sng", + SubnetIDs: []string{"subnet-1"}, + }) + require.NoError(t, err) + + _, err = original.CreateUser(ctx, &memorydb.ExportedCreateUserRequest{ + UserName: "fs-user", + AccessString: "on ~* &* +@all", + AuthenticationMode: memorydb.ExportedAuthModeReq{ + Type: "no-password-required", + }, + }) + require.NoError(t, err) + + _, err = original.CreateParameterGroup(ctx, &memorydb.ExportedCreateParameterGroupRequest{ + ParameterGroupName: "fs-pg", + Family: "memorydb_redis7", + }) + require.NoError(t, err) + + _, err = original.CreateSnapshot(ctx, &memorydb.ExportedCreateSnapshotRequest{ + ClusterName: "fs-cluster", + SnapshotName: "fs-snap", + }) + require.NoError(t, err) + + _, err = original.CreateMultiRegionCluster(ctx, &memorydb.ExportedCreateMultiRegionClusterRequest{ + MultiRegionClusterNameSuffix: "fs-mrc", + NodeType: "db.r6g.large", + }) + require.NoError(t, err) + + // Hardcoded to match one of the builtin offering IDs in backend.go's + // defaultReservedNodesOfferings (unexported, so not reachable from this + // external test package by name). + const builtinOfferingID = "aaa00000-1111-2222-3333-444444444444" + _, err = original.PurchaseReservedNodesOffering(ctx, &memorydb.ExportedPurchaseReservedNodesOfferingRequest{ + ReservedNodesOfferingID: builtinOfferingID, + ReservationID: "fs-reserved-node", + }) + require.NoError(t, err) + + snap := original.Snapshot(ctx) + require.NotNil(t, snap) + + fresh := memorydb.NewInMemoryBackend("123456789012", "us-east-1") + require.NoError(t, fresh.Restore(ctx, snap)) + + clusters, err := fresh.DescribeClusters(ctx, "fs-cluster") + require.NoError(t, err) + require.Len(t, clusters, 1) + + acls, err := fresh.DescribeACLs(ctx, "fs-acl") + require.NoError(t, err) + require.Len(t, acls, 1) + + sngs, err := fresh.DescribeSubnetGroups(ctx, "fs-sng") + require.NoError(t, err) + require.Len(t, sngs, 1) + + users, err := fresh.DescribeUsers(ctx, "fs-user") + require.NoError(t, err) + require.Len(t, users, 1) + + pgs, err := fresh.DescribeParameterGroups(ctx, "fs-pg") + require.NoError(t, err) + require.Len(t, pgs, 1) + + snaps, err := fresh.DescribeSnapshots(ctx, "fs-snap", "", "", "") + require.NoError(t, err) + require.Len(t, snaps, 1) + + mrcs, err := fresh.DescribeMultiRegionClusters(ctx, "virv-fs-mrc") + require.NoError(t, err) + require.Len(t, mrcs, 1) + + reservedNodes, err := fresh.DescribeReservedNodes(ctx, &memorydb.ExportedDescribeReservedNodesRequest{ + ReservedNodeID: "fs-reserved-node", + }) + require.NoError(t, err) + require.Len(t, reservedNodes, 1) + + // The default open-access ACL and the four builtin default parameter + // groups are re-seeded by NewInMemoryBackend and survive restore as part + // of the acls/parameterGroups tables, so ACLCount/ParameterGroupCount + // exceed 1 -- assert at-least rather than exact counts for those. + assert.GreaterOrEqual(t, memorydb.ACLCount(fresh), 2) + assert.GreaterOrEqual(t, memorydb.ParameterGroupCount(fresh), 1) + assert.Equal(t, 1, memorydb.ClusterCount(fresh)) + assert.Equal(t, 1, memorydb.SubnetGroupCount(fresh)) + assert.Equal(t, 1, memorydb.UserCount(fresh)) + assert.Equal(t, 1, memorydb.SnapshotCount(fresh)) + assert.Equal(t, 1, memorydb.MultiRegionClusterCount(fresh)) + assert.Positive(t, memorydb.EventCount(fresh)) + assert.Positive(t, memorydb.ARNIndexSize(fresh)) +} + +// TestInMemoryBackend_Restore_IncompatibleVersion feeds Restore a +// well-formed-but-wrong-version snapshot (simulating an older/incompatible +// on-disk format) and checks it is discarded cleanly -- resetting the backend +// to empty rather than erroring or partially decoding the mismatched shape. +func TestInMemoryBackend_Restore_IncompatibleVersion(t *testing.T) { + t.Parallel() + + ctx := t.Context() + + original := memorydb.NewInMemoryBackend("123456789012", "us-east-1") + _, err := original.CreateCluster(ctx, &memorydb.ExportedCreateClusterRequest{ + ClusterName: "versioned-cluster", + NodeType: "db.r6g.large", + }) + require.NoError(t, err) + + // Craft a snapshot with an incompatible version by round-tripping through + // a generic map: bump "version" to a value that can never match the + // current memorydbSnapshotVersion constant. + var raw map[string]any + + require.NoError(t, json.Unmarshal(original.Snapshot(ctx), &raw)) + raw["version"] = -1 + + badSnap, err := json.Marshal(raw) + require.NoError(t, err) + + fresh := memorydb.NewInMemoryBackend("123456789012", "us-east-1") + _, err = fresh.CreateCluster(ctx, &memorydb.ExportedCreateClusterRequest{ + ClusterName: "pre-existing", + NodeType: "db.r6g.large", + }) + require.NoError(t, err) + + require.NoError(t, fresh.Restore(ctx, badSnap)) + + assert.Equal(t, 0, memorydb.ClusterCount(fresh), + "incompatible-version restore must reset to empty, not keep pre-existing or partially decoded state") +} diff --git a/services/memorydb/store_setup.go b/services/memorydb/store_setup.go new file mode 100644 index 000000000..2d38f5ef1 --- /dev/null +++ b/services/memorydb/store_setup.go @@ -0,0 +1,69 @@ +package memorydb + +import "github.com/blackbirdworks/gopherstack/pkgs/store" + +// Code in this file supports Phase 3.3 of the datalayer refactor: converting +// InMemoryBackend's resource maps to pkgs/store. See pkgs/store's package doc +// and the services/sqs pilot (commit 0f09d77c), services/ec2 rollout (commit +// 12e611a4), and services/elasticache rollout (commit 06806317) for the +// pattern this follows. +// +// Every resource in this backend is nested per-region +// (map[string]map[string]*T -- outer key is region) except +// MultiRegionCluster, MultiRegionParameterGroup, and ServiceUpdate, which are +// partition-scoped like AWS and therefore global (see backend.go's +// InMemoryBackend doc comment). The region-nested resources are converted to +// map[string]*store.Table[T] (region -> Table) with a lazy per-region +// accessor -- see the "*Store" helpers in backend.go. Because the set of +// regions is only known at runtime, these per-region tables are deliberately +// NOT registered on a *store.Registry: Registry's SnapshotAll/RestoreAll +// require a fixed, construction-time-known table-name set, which a dynamic +// region set can't provide (a region first seen on Restore would have +// nothing registered to restore into). persistence.go instead +// snapshots/restores each per-region Table directly via +// Table.Snapshot()/Table.Restore(). The three global resources have no such +// per-region dynamism, so they alone are registered on b.registry and go +// through registry.SnapshotAll()/RestoreAll() in persistence.go. +// +// arnToResource and events are deliberately NOT converted: arnToResource's +// value type (resourceRef) has no field carrying its own key (the ARN is the +// map key, not a value field) so it cannot supply a store.Table keyFn, and +// events is slice-valued (map[string][]*Event) rather than a single *T +// resource keyed by its own identity, which store.Table cannot represent. +// Both remain plain nested maps. + +func clusterKeyFn(v *Cluster) string { return v.Name } +func aclKeyFn(v *ACL) string { return v.Name } +func subnetGroupKeyFn(v *SubnetGroup) string { return v.Name } +func userKeyFn(v *User) string { return v.Name } +func parameterGroupKeyFn(v *ParameterGroup) string { return v.Name } +func snapshotKeyFn(v *Snapshot) string { return v.Name } +func reservedNodeKeyFn(v *ReservedNode) string { return v.ReservedNodeID } + +func multiRegionClusterKeyFn(v *MultiRegionCluster) string { return v.MultiRegionClusterName } +func multiRegionParameterGroupKeyFn(v *MultiRegionParameterGroup) string { return v.Name } +func serviceUpdateKeyFn(v *ServiceUpdate) string { return v.ServiceUpdateName } + +// tableGet and tableAll wrap the corresponding *store.Table[V] +// methods but tolerate a nil t, returning the same zero values a lookup +// against a nil map would have returned pre-conversion. This matters because +// several read paths in backend.go deliberately read the raw +// map[string]*store.Table[V] field directly (e.g. b.clusters[region]) +// instead of going through the lazy "*Store" accessor, so as to not allocate +// an entry for a region that has never been written to; unlike a nil map, a +// nil *store.Table[V] cannot be called directly without panicking. +func tableGet[V any](t *store.Table[V], id string) (*V, bool) { + if t == nil { + return nil, false + } + + return t.Get(id) +} + +func tableAll[V any](t *store.Table[V]) []*V { + if t == nil { + return nil + } + + return t.All() +} diff --git a/services/opensearch/backend.go b/services/opensearch/backend.go index 101132a17..f3469b52c 100644 --- a/services/opensearch/backend.go +++ b/services/opensearch/backend.go @@ -10,6 +10,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" "github.com/blackbirdworks/gopherstack/pkgs/arn" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) @@ -114,6 +115,11 @@ type DataSource struct { Name string `json:"name"` Description string `json:"description"` DataSourceType string `json:"dataSourceType"` + // DomainName identifies the owning domain and is used only to key the + // pkgs/store composite table (domainName#name); it is never serialized on + // the wire, matching how the domain name was already implied by the + // outer map key before the pkgs/store conversion. + DomainName string `json:"-"` } // DirectQueryDataSource represents a direct-query data source. @@ -273,6 +279,11 @@ type DomainIndex struct { Documents map[string]map[string]any `json:"Documents,omitempty"` IndexName string `json:"IndexName"` IndexStatus string `json:"IndexStatus"` + // DomainName identifies the owning domain and is used only to key the + // pkgs/store composite table (domainName#indexName); it is never + // serialized on the wire, matching how the domain name was already + // implied by the outer map key before the pkgs/store conversion. + DomainName string `json:"-"` // DocumentCount is the number of documents currently stored in the index. DocumentCount int `json:"DocumentCount"` } @@ -507,87 +518,81 @@ type DryRunStatus struct { DryRunStatus string `json:"DryRunStatus"` CreationDate string `json:"CreationDate"` UpdateDate string `json:"UpdateDate"` + DomainName string `json:"-"` ValidationFailures []map[string]any `json:"ValidationFailures"` } // InMemoryBackend is the in-memory store for OpenSearch domains. +// +// Most resource collections are *store.Table[T] registered on b.registry -- +// see store_setup.go's registerAllTables doc for the full clean/dirty split +// and the reasoning behind the handful of fields left as plain maps. type InMemoryBackend struct { - dnsRegistrar DNSRegistrar - dryRuns map[string]*DryRunStatus - reservedInstances map[string]*ReservedInstance - arnIndex map[string]string - inboundConnections map[string]*InboundConnection - outboundConnections map[string]*OutboundConnection - domainDataSources map[string]map[string]*DataSource - directQueryDataSources map[string]*DirectQueryDataSource - domains map[string]*Domain - vpcAuthorizations map[string][]AuthorizedPrincipal - vpcEndpoints map[string]*VpcEndpoint - applications map[string]*Application - applicationNames map[string]string - packages map[string]*Package - scheduledActions map[string][]*ScheduledAction - packageAssociations map[string]map[string]bool - domainMaintenances map[string][]*DomainMaintenance - domainIndexes map[string]map[string]*DomainIndex - upgradeHistory map[string][]*UpgradeHistory - domainPackages map[string]map[string]bool - autoTunes map[string]*AutoTuneConfig - slNetworkPolicies map[string]*ServerlessNetworkPolicy - slCollections map[string]*ServerlessCollection - slAccessPolicies map[string]*ServerlessAccessPolicy - slSecurityConfigs map[string]*ServerlessSecurityConfig - slEncryptionPolicies map[string]*ServerlessEncryptionPolicy - defaultAppSettings map[string][]AppSetting - mu *lockmetrics.RWMutex - now func() time.Time - accountID string - region string - processingDelay time.Duration - appIDCounter int - connCounter int - vpcEndpointCounter int - packageCounter int - maintenanceCounter int - reservedCounter int - slCollCounter int - slSecConfigCounter int - docCounter int + dnsRegistrar DNSRegistrar + dryRuns *store.Table[DryRunStatus] + reservedInstances *store.Table[ReservedInstance] + inboundConnections *store.Table[InboundConnection] + outboundConnections *store.Table[OutboundConnection] + domainDataSources *store.Table[DataSource] + domainDataSourcesByDomain *store.Index[DataSource] + directQueryDataSources *store.Table[DirectQueryDataSource] + domains *store.Table[Domain] + domainsByARN *store.Index[Domain] + vpcAuthorizations map[string][]AuthorizedPrincipal + vpcEndpoints *store.Table[VpcEndpoint] + applications *store.Table[Application] + applicationsByName *store.Index[Application] + packages *store.Table[Package] + scheduledActions map[string][]*ScheduledAction + packageAssociations map[string]map[string]bool + domainMaintenances map[string][]*DomainMaintenance + domainIndexes *store.Table[DomainIndex] + domainIndexesByDomain *store.Index[DomainIndex] + upgradeHistory map[string][]*UpgradeHistory + domainPackages map[string]map[string]bool + autoTunes *store.Table[AutoTuneConfig] + slNetworkPolicies *store.Table[ServerlessNetworkPolicy] + slCollections *store.Table[ServerlessCollection] + slAccessPolicies *store.Table[ServerlessAccessPolicy] + slSecurityConfigs *store.Table[ServerlessSecurityConfig] + slEncryptionPolicies *store.Table[ServerlessEncryptionPolicy] + defaultAppSettings map[string][]AppSetting + registry *store.Registry + mu *lockmetrics.RWMutex + now func() time.Time + accountID string + region string + processingDelay time.Duration + appIDCounter int + connCounter int + vpcEndpointCounter int + packageCounter int + maintenanceCounter int + reservedCounter int + slCollCounter int + slSecConfigCounter int + docCounter int } // NewInMemoryBackend creates a new InMemoryBackend. func NewInMemoryBackend(accountID, region string) *InMemoryBackend { - return &InMemoryBackend{ - domains: make(map[string]*Domain), - arnIndex: make(map[string]string), - inboundConnections: make(map[string]*InboundConnection), - outboundConnections: make(map[string]*OutboundConnection), - domainDataSources: make(map[string]map[string]*DataSource), - directQueryDataSources: make(map[string]*DirectQueryDataSource), - packageAssociations: make(map[string]map[string]bool), - domainPackages: make(map[string]map[string]bool), - vpcAuthorizations: make(map[string][]AuthorizedPrincipal), - vpcEndpoints: make(map[string]*VpcEndpoint), - applications: make(map[string]*Application), - applicationNames: make(map[string]string), - packages: make(map[string]*Package), - scheduledActions: make(map[string][]*ScheduledAction), - reservedInstances: make(map[string]*ReservedInstance), - domainMaintenances: make(map[string][]*DomainMaintenance), - domainIndexes: make(map[string]map[string]*DomainIndex), - upgradeHistory: make(map[string][]*UpgradeHistory), - autoTunes: make(map[string]*AutoTuneConfig), - dryRuns: make(map[string]*DryRunStatus), - defaultAppSettings: make(map[string][]AppSetting), - slCollections: make(map[string]*ServerlessCollection), - slAccessPolicies: make(map[string]*ServerlessAccessPolicy), - slSecurityConfigs: make(map[string]*ServerlessSecurityConfig), - slEncryptionPolicies: make(map[string]*ServerlessEncryptionPolicy), - slNetworkPolicies: make(map[string]*ServerlessNetworkPolicy), - accountID: accountID, - region: region, - mu: lockmetrics.New("opensearch"), + b := &InMemoryBackend{ + packageAssociations: make(map[string]map[string]bool), + domainPackages: make(map[string]map[string]bool), + vpcAuthorizations: make(map[string][]AuthorizedPrincipal), + scheduledActions: make(map[string][]*ScheduledAction), + domainMaintenances: make(map[string][]*DomainMaintenance), + upgradeHistory: make(map[string][]*UpgradeHistory), + defaultAppSettings: make(map[string][]AppSetting), + accountID: accountID, + region: region, + mu: lockmetrics.New("opensearch"), + registry: store.NewRegistry(), } + + registerAllTables(b) + + return b } // SetDNSRegistrar wires a DNS server so OpenSearch domain hostnames are auto-registered. @@ -609,7 +614,7 @@ func (b *InMemoryBackend) CreateDomain(input CreateDomainInput) (*Domain, error) // Finalise any domains whose deleting window has elapsed so the name frees up. b.purgeExpiredDomainsLocked() - if _, exists := b.domains[input.Name]; exists { + if b.domains.Has(input.Name) { return nil, fmt.Errorf("%w: domain %s already exists", ErrDomainAlreadyExists, input.Name) } @@ -658,8 +663,7 @@ func (b *InMemoryBackend) CreateDomain(input CreateDomainInput) (*Domain, error) d.Created = true b.beginProcessing(d, dpsCreating) - b.domains[input.Name] = d - b.arnIndex[domainARN] = input.Name + b.domains.Put(d) if b.dnsRegistrar != nil { b.dnsRegistrar.Register(endpoint) @@ -683,7 +687,7 @@ func (b *InMemoryBackend) DeleteDomain(name string) (*Domain, error) { b.purgeExpiredDomainsLocked() - d, exists := b.domains[name] + d, exists := b.domains.Get(name) if !exists { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, name) } @@ -705,7 +709,7 @@ func (b *InMemoryBackend) DescribeDomain(name string) (*Domain, error) { b.mu.RLock("DescribeDomain") defer b.mu.RUnlock() - d, exists := b.domains[name] + d, exists := b.domains.Get(name) if !exists || deleteWindowElapsed(d, b.clock()) { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, name) } @@ -721,14 +725,14 @@ func (b *InMemoryBackend) ListDomainNames() []string { defer b.mu.RUnlock() now := b.clock() - names := make([]string, 0, len(b.domains)) + names := make([]string, 0, b.domains.Len()) - for name, d := range b.domains { + for _, d := range b.domains.All() { if deleteWindowElapsed(d, now) { continue } - names = append(names, name) + names = append(names, d.Name) } slices.Sort(names) @@ -739,12 +743,12 @@ func (b *InMemoryBackend) ListDomainNames() []string { // findDomainByARN returns the domain matching the given ARN, or nil if not found. // Caller must hold at least a read lock. func (b *InMemoryBackend) findDomainByARN(domainARN string) *Domain { - name, ok := b.arnIndex[domainARN] - if !ok { + matches := b.domainsByARN.Get(domainARN) + if len(matches) == 0 { return nil } - return b.domains[name] + return matches[0] } // ListTags returns tags for the domain identified by ARN. @@ -799,7 +803,7 @@ func (b *InMemoryBackend) AcceptInboundConnection(connectionID string) (*Inbound b.mu.Lock("AcceptInboundConnection") defer b.mu.Unlock() - conn, exists := b.inboundConnections[connectionID] + conn, exists := b.inboundConnections.Get(connectionID) if !exists { return nil, fmt.Errorf("%w: connection %s not found", ErrConnectionNotFound, connectionID) } @@ -826,15 +830,11 @@ func (b *InMemoryBackend) AddDataSource( b.mu.Lock("AddDataSource") defer b.mu.Unlock() - if _, exists := b.domains[domainName]; !exists { + if !b.domains.Has(domainName) { return "", fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } - if b.domainDataSources[domainName] == nil { - b.domainDataSources[domainName] = make(map[string]*DataSource) - } - - if _, exists := b.domainDataSources[domainName][name]; exists { + if b.domainDataSources.Has(dataSourceKey(domainName, name)) { return "", fmt.Errorf( "%w: data source %s already exists on domain %s", ErrDataSourceAlreadyExists, @@ -843,11 +843,12 @@ func (b *InMemoryBackend) AddDataSource( ) } - b.domainDataSources[domainName][name] = &DataSource{ + b.domainDataSources.Put(&DataSource{ Name: name, Description: description, DataSourceType: dataSourceType, - } + DomainName: domainName, + }) return "Data source created successfully", nil } @@ -864,7 +865,7 @@ func (b *InMemoryBackend) AddDirectQueryDataSource( b.mu.Lock("AddDirectQueryDataSource") defer b.mu.Unlock() - if _, exists := b.directQueryDataSources[name]; exists { + if b.directQueryDataSources.Has(name) { return "", fmt.Errorf( "%w: direct query data source %s already exists", ErrDataSourceAlreadyExists, @@ -873,13 +874,13 @@ func (b *InMemoryBackend) AddDirectQueryDataSource( } dsARN := arn.Build("opensearch", b.region, b.accountID, "directQueryDataSource/"+name) - b.directQueryDataSources[name] = &DirectQueryDataSource{ + b.directQueryDataSources.Put(&DirectQueryDataSource{ Name: name, Description: description, DataSourceType: dataSourceType, OpenSearchArns: openSearchArns, DataSourceArn: dsARN, - } + }) return dsARN, nil } @@ -899,11 +900,11 @@ func (b *InMemoryBackend) AssociatePackage( b.mu.Lock("AssociatePackage") defer b.mu.Unlock() - if _, exists := b.packages[packageID]; !exists { + if !b.packages.Has(packageID) { return nil, fmt.Errorf("%w: package %s not found", ErrPackageNotFound, packageID) } - if _, exists := b.domains[domainName]; !exists { + if !b.domains.Has(domainName) { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } @@ -965,14 +966,14 @@ func (b *InMemoryBackend) AssociatePackages( b.mu.Lock("AssociatePackages") defer b.mu.Unlock() - if _, exists := b.domains[domainName]; !exists { + if !b.domains.Has(domainName) { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } results := make([]DomainPackageDetails, 0, len(packageIDs)) for _, pkgID := range packageIDs { - if _, exists := b.packages[pkgID]; !exists { + if !b.packages.Has(pkgID) { return nil, fmt.Errorf("%w: package %s not found", ErrPackageNotFound, pkgID) } @@ -998,7 +999,7 @@ func (b *InMemoryBackend) AuthorizeVpcEndpointAccess( b.mu.Lock("AuthorizeVpcEndpointAccess") defer b.mu.Unlock() - if _, exists := b.domains[domainName]; !exists { + if !b.domains.Has(domainName) { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } @@ -1031,7 +1032,7 @@ func (b *InMemoryBackend) CancelDomainConfigChange( b.mu.Lock("CancelDomainConfigChange") defer b.mu.Unlock() - d, exists := b.domains[domainName] + d, exists := b.domains.Get(domainName) if !exists { return nil, false, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } @@ -1062,7 +1063,7 @@ func (b *InMemoryBackend) CancelServiceSoftwareUpdate( b.mu.Lock("CancelServiceSoftwareUpdate") defer b.mu.Unlock() - d, exists := b.domains[domainName] + d, exists := b.domains.Get(domainName) if !exists || deleteWindowElapsed(d, b.clock()) { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } @@ -1107,7 +1108,7 @@ func (b *InMemoryBackend) CreateApplication( b.mu.Lock("CreateApplication") defer b.mu.Unlock() - if _, exists := b.applicationNames[name]; exists { + if len(b.applicationsByName.Get(name)) > 0 { return nil, fmt.Errorf( "%w: application %s already exists", ErrApplicationAlreadyExists, @@ -1134,8 +1135,7 @@ func (b *InMemoryBackend) CreateApplication( AppConfigs: appConfigs, DataSources: dataSources, } - b.applications[id] = app - b.applicationNames[name] = id + b.applications.Put(app) cp := *app cp.AppConfigs = make([]AppConfig, len(app.AppConfigs)) @@ -1151,36 +1151,28 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - for _, d := range b.domains { + for _, d := range b.domains.All() { d.Tags.Close() } - b.domains = make(map[string]*Domain) - b.arnIndex = make(map[string]string) - b.inboundConnections = make(map[string]*InboundConnection) - b.outboundConnections = make(map[string]*OutboundConnection) - b.domainDataSources = make(map[string]map[string]*DataSource) - b.directQueryDataSources = make(map[string]*DirectQueryDataSource) + // "Clean" tables registered on b.registry (see store_setup.go). + b.registry.ResetAll() + + // "Dirty" tables, not registered on b.registry (see store_setup.go). + b.dryRuns.Reset() + b.autoTunes.Reset() + b.domainDataSources.Reset() + b.domainIndexes.Reset() + + // Plain maps left unconverted (see store_setup.go's registerAllTables doc). b.packageAssociations = make(map[string]map[string]bool) b.domainPackages = make(map[string]map[string]bool) b.vpcAuthorizations = make(map[string][]AuthorizedPrincipal) - b.vpcEndpoints = make(map[string]*VpcEndpoint) - b.applications = make(map[string]*Application) - b.applicationNames = make(map[string]string) - b.packages = make(map[string]*Package) b.scheduledActions = make(map[string][]*ScheduledAction) - b.reservedInstances = make(map[string]*ReservedInstance) b.domainMaintenances = make(map[string][]*DomainMaintenance) - b.domainIndexes = make(map[string]map[string]*DomainIndex) b.upgradeHistory = make(map[string][]*UpgradeHistory) - b.autoTunes = make(map[string]*AutoTuneConfig) - b.dryRuns = make(map[string]*DryRunStatus) b.defaultAppSettings = make(map[string][]AppSetting) - b.slCollections = make(map[string]*ServerlessCollection) - b.slAccessPolicies = make(map[string]*ServerlessAccessPolicy) - b.slSecurityConfigs = make(map[string]*ServerlessSecurityConfig) - b.slEncryptionPolicies = make(map[string]*ServerlessEncryptionPolicy) - b.slNetworkPolicies = make(map[string]*ServerlessNetworkPolicy) + b.appIDCounter = 0 b.connCounter = 0 b.vpcEndpointCounter = 0 @@ -1226,7 +1218,7 @@ func (b *InMemoryBackend) CreateOutboundConnection( RemoteDomainInfo: remoteDomainInfo, Status: connectionStatusActive, } - b.outboundConnections[id] = conn + b.outboundConnections.Put(conn) cp := *conn @@ -1240,9 +1232,9 @@ func (b *InMemoryBackend) DescribeOutboundConnections() []*OutboundConnection { defer b.mu.RUnlock() now := b.clock() - out := make([]*OutboundConnection, 0, len(b.outboundConnections)) + out := make([]*OutboundConnection, 0, b.outboundConnections.Len()) - for _, c := range b.outboundConnections { + for _, c := range b.outboundConnections.All() { if statusWindowElapsed(c.Status, c.StatusUntil, now) { continue } @@ -1265,7 +1257,7 @@ func (b *InMemoryBackend) DeleteOutboundConnection( b.purgeExpiredOutboundLocked() - conn, exists := b.outboundConnections[connectionID] + conn, exists := b.outboundConnections.Get(connectionID) if !exists { return nil, fmt.Errorf( "%w: outbound connection %s not found", @@ -1277,7 +1269,7 @@ func (b *InMemoryBackend) DeleteOutboundConnection( if b.processingDelay == 0 { cp := *conn cp.Status = statusDeleting - delete(b.outboundConnections, connectionID) + b.outboundConnections.Delete(connectionID) return &cp, nil } @@ -1293,9 +1285,11 @@ func (b *InMemoryBackend) DeleteOutboundConnection( // window. The caller must hold the write lock. func (b *InMemoryBackend) purgeExpiredOutboundLocked() { now := b.clock() - for id, c := range b.outboundConnections { + // Table.All returns a fresh slice, so deleting from the table while + // ranging over it here is safe. + for _, c := range b.outboundConnections.All() { if statusWindowElapsed(c.Status, c.StatusUntil, now) { - delete(b.outboundConnections, id) + b.outboundConnections.Delete(c.ConnectionID) } } } @@ -1305,7 +1299,7 @@ func (b *InMemoryBackend) RejectInboundConnection(connectionID string) (*Inbound b.mu.Lock("RejectInboundConnection") defer b.mu.Unlock() - conn, exists := b.inboundConnections[connectionID] + conn, exists := b.inboundConnections.Get(connectionID) if !exists { return nil, fmt.Errorf( "%w: inbound connection %s not found", @@ -1328,7 +1322,7 @@ func (b *InMemoryBackend) DeleteInboundConnection(connectionID string) (*Inbound b.purgeExpiredInboundLocked() - conn, exists := b.inboundConnections[connectionID] + conn, exists := b.inboundConnections.Get(connectionID) if !exists { return &InboundConnection{ConnectionID: connectionID, Status: statusDeleting}, nil } @@ -1336,7 +1330,7 @@ func (b *InMemoryBackend) DeleteInboundConnection(connectionID string) (*Inbound if b.processingDelay == 0 { cp := *conn cp.Status = statusDeleting - delete(b.inboundConnections, connectionID) + b.inboundConnections.Delete(connectionID) return &cp, nil } @@ -1352,9 +1346,9 @@ func (b *InMemoryBackend) DeleteInboundConnection(connectionID string) (*Inbound // window. The caller must hold the write lock. func (b *InMemoryBackend) purgeExpiredInboundLocked() { now := b.clock() - for id, c := range b.inboundConnections { + for _, c := range b.inboundConnections.All() { if statusWindowElapsed(c.Status, c.StatusUntil, now) { - delete(b.inboundConnections, id) + b.inboundConnections.Delete(c.ConnectionID) } } } @@ -1366,9 +1360,9 @@ func (b *InMemoryBackend) DescribeInboundConnections() []*InboundConnection { defer b.mu.RUnlock() now := b.clock() - out := make([]*InboundConnection, 0, len(b.inboundConnections)) + out := make([]*InboundConnection, 0, b.inboundConnections.Len()) - for _, c := range b.inboundConnections { + for _, c := range b.inboundConnections.All() { if statusWindowElapsed(c.Status, c.StatusUntil, now) { continue } @@ -1399,7 +1393,7 @@ func (b *InMemoryBackend) CreateVpcEndpoint( Endpoint: fmt.Sprintf("%s.vpc.es.amazonaws.com", id), VpcOptions: vpcOptions, } - b.vpcEndpoints[id] = ep + b.vpcEndpoints.Put(ep) cp := *ep @@ -1417,7 +1411,7 @@ func (b *InMemoryBackend) DescribeVpcEndpoints(ids []string) ([]*VpcEndpoint, [] var errs []map[string]any for _, id := range ids { - ep, exists := b.vpcEndpoints[id] + ep, exists := b.vpcEndpoints.Get(id) if !exists || statusWindowElapsed(ep.Status, ep.StatusUntil, now) { errs = append(errs, map[string]any{ "VpcEndpointId": id, @@ -1451,7 +1445,7 @@ func (b *InMemoryBackend) UpdateVpcEndpoint( b.mu.Lock("UpdateVpcEndpoint") defer b.mu.Unlock() - ep, exists := b.vpcEndpoints[id] + ep, exists := b.vpcEndpoints.Get(id) if !exists { return nil, fmt.Errorf("%w: VPC endpoint %s not found", ErrConnectionNotFound, id) } @@ -1471,7 +1465,7 @@ func (b *InMemoryBackend) DeleteVpcEndpoint(id string) (*VpcEndpoint, error) { b.purgeExpiredVpcEndpointsLocked() - ep, exists := b.vpcEndpoints[id] + ep, exists := b.vpcEndpoints.Get(id) if !exists { return nil, fmt.Errorf("%w: VPC endpoint %s not found", ErrConnectionNotFound, id) } @@ -1479,7 +1473,7 @@ func (b *InMemoryBackend) DeleteVpcEndpoint(id string) (*VpcEndpoint, error) { if b.processingDelay == 0 { cp := *ep cp.Status = statusDeleting - delete(b.vpcEndpoints, id) + b.vpcEndpoints.Delete(id) return &cp, nil } @@ -1495,9 +1489,9 @@ func (b *InMemoryBackend) DeleteVpcEndpoint(id string) (*VpcEndpoint, error) { // window. The caller must hold the write lock. func (b *InMemoryBackend) purgeExpiredVpcEndpointsLocked() { now := b.clock() - for id, ep := range b.vpcEndpoints { + for _, ep := range b.vpcEndpoints.All() { if statusWindowElapsed(ep.Status, ep.StatusUntil, now) { - delete(b.vpcEndpoints, id) + b.vpcEndpoints.Delete(ep.VpcEndpointID) } } } @@ -1509,9 +1503,9 @@ func (b *InMemoryBackend) ListVpcEndpoints() []*VpcEndpoint { defer b.mu.RUnlock() now := b.clock() - out := make([]*VpcEndpoint, 0, len(b.vpcEndpoints)) + out := make([]*VpcEndpoint, 0, b.vpcEndpoints.Len()) - for _, ep := range b.vpcEndpoints { + for _, ep := range b.vpcEndpoints.All() { if statusWindowElapsed(ep.Status, ep.StatusUntil, now) { continue } @@ -1533,7 +1527,7 @@ func (b *InMemoryBackend) ListVpcEndpointsForDomain(domainArn string) []*VpcEndp var out []*VpcEndpoint - for _, ep := range b.vpcEndpoints { + for _, ep := range b.vpcEndpoints.All() { if ep.DomainArn == domainArn && !statusWindowElapsed(ep.Status, ep.StatusUntil, now) { cp := *ep out = append(out, &cp) @@ -1613,7 +1607,7 @@ func (b *InMemoryBackend) CreatePackage( }, }, } - b.packages[id] = pkg + b.packages.Put(pkg) cp := *pkg @@ -1625,13 +1619,13 @@ func (b *InMemoryBackend) DeletePackage(packageID string) (*Package, error) { b.mu.Lock("DeletePackage") defer b.mu.Unlock() - pkg, exists := b.packages[packageID] + pkg, exists := b.packages.Get(packageID) if !exists { return nil, fmt.Errorf("%w: package %s not found", ErrPackageNotFound, packageID) } cp := *pkg - delete(b.packages, packageID) + b.packages.Delete(packageID) return &cp, nil } @@ -1642,8 +1636,8 @@ func (b *InMemoryBackend) DescribePackages(ids []string) ([]*Package, error) { defer b.mu.RUnlock() if len(ids) == 0 { - out := make([]*Package, 0, len(b.packages)) - for _, pkg := range b.packages { + out := make([]*Package, 0, b.packages.Len()) + for _, pkg := range b.packages.All() { cp := *pkg out = append(out, &cp) } @@ -1654,7 +1648,7 @@ func (b *InMemoryBackend) DescribePackages(ids []string) ([]*Package, error) { out := make([]*Package, 0, len(ids)) for _, id := range ids { - pkg, exists := b.packages[id] + pkg, exists := b.packages.Get(id) if !exists { return nil, fmt.Errorf("%w: package %s not found", ErrPackageNotFound, id) } @@ -1673,7 +1667,7 @@ func (b *InMemoryBackend) GetPackageVersionHistory( b.mu.RLock("GetPackageVersionHistory") defer b.mu.RUnlock() - pkg, exists := b.packages[packageID] + pkg, exists := b.packages.Get(packageID) if !exists { return nil, fmt.Errorf("%w: package %s not found", ErrPackageNotFound, packageID) } @@ -1692,7 +1686,7 @@ func (b *InMemoryBackend) UpdatePackage(packageID, description string) (*Package b.mu.Lock("UpdatePackage") defer b.mu.Unlock() - pkg, exists := b.packages[packageID] + pkg, exists := b.packages.Get(packageID) if !exists { return nil, fmt.Errorf("%w: package %s not found", ErrPackageNotFound, packageID) } @@ -1714,7 +1708,7 @@ func (b *InMemoryBackend) UpdatePackageScope(packageID, _ string, _ []string) (* b.mu.RLock("UpdatePackageScope") defer b.mu.RUnlock() - pkg, exists := b.packages[packageID] + pkg, exists := b.packages.Get(packageID) if !exists { return nil, fmt.Errorf("%w: package %s not found", ErrPackageNotFound, packageID) } @@ -1732,7 +1726,7 @@ func (b *InMemoryBackend) ListPackagesForDomain(domainName string) []*Package { var out []*Package for pkgID := range b.domainPackages[domainName] { - pkg, exists := b.packages[pkgID] + pkg, exists := b.packages.Get(pkgID) if exists { cp := *pkg out = append(out, &cp) @@ -1768,17 +1762,7 @@ func (b *InMemoryBackend) GetDataSource(domainName, name string) (*DataSource, e b.mu.RLock("GetDataSource") defer b.mu.RUnlock() - dsMap, exists := b.domainDataSources[domainName] - if !exists { - return nil, fmt.Errorf( - "%w: data source %s not found on domain %s", - ErrDataSourceNotFound, - name, - domainName, - ) - } - - ds, exists := dsMap[name] + ds, exists := b.domainDataSources.Get(dataSourceKey(domainName, name)) if !exists { return nil, fmt.Errorf( "%w: data source %s not found on domain %s", @@ -1798,10 +1782,10 @@ func (b *InMemoryBackend) ListDataSources(domainName string) ([]*DataSource, err b.mu.RLock("ListDataSources") defer b.mu.RUnlock() - dsMap := b.domainDataSources[domainName] - out := make([]*DataSource, 0, len(dsMap)) + group := b.domainDataSourcesByDomain.Get(domainName) + out := make([]*DataSource, 0, len(group)) - for _, ds := range dsMap { + for _, ds := range group { cp := *ds out = append(out, &cp) } @@ -1814,17 +1798,7 @@ func (b *InMemoryBackend) UpdateDataSource(domainName, name, description string) b.mu.Lock("UpdateDataSource") defer b.mu.Unlock() - dsMap, exists := b.domainDataSources[domainName] - if !exists { - return fmt.Errorf( - "%w: data source %s not found on domain %s", - ErrDataSourceNotFound, - name, - domainName, - ) - } - - ds, exists := dsMap[name] + ds, exists := b.domainDataSources.Get(dataSourceKey(domainName, name)) if !exists { return fmt.Errorf( "%w: data source %s not found on domain %s", @@ -1844,12 +1818,7 @@ func (b *InMemoryBackend) DeleteDataSource(domainName, name string) error { b.mu.Lock("DeleteDataSource") defer b.mu.Unlock() - dsMap, exists := b.domainDataSources[domainName] - if !exists { - return nil - } - - delete(dsMap, name) + b.domainDataSources.Delete(dataSourceKey(domainName, name)) return nil } @@ -1859,8 +1828,8 @@ func (b *InMemoryBackend) ListDirectQueryDataSources() []*DirectQueryDataSource b.mu.RLock("ListDirectQueryDataSources") defer b.mu.RUnlock() - out := make([]*DirectQueryDataSource, 0, len(b.directQueryDataSources)) - for _, ds := range b.directQueryDataSources { + out := make([]*DirectQueryDataSource, 0, b.directQueryDataSources.Len()) + for _, ds := range b.directQueryDataSources.All() { cp := *ds out = append(out, &cp) } @@ -1873,7 +1842,7 @@ func (b *InMemoryBackend) GetDirectQueryDataSource(name string) (*DirectQueryDat b.mu.RLock("GetDirectQueryDataSource") defer b.mu.RUnlock() - ds, exists := b.directQueryDataSources[name] + ds, exists := b.directQueryDataSources.Get(name) if !exists { return nil, fmt.Errorf( "%w: direct query data source %s not found", @@ -1895,7 +1864,7 @@ func (b *InMemoryBackend) UpdateDirectQueryDataSource( b.mu.Lock("UpdateDirectQueryDataSource") defer b.mu.Unlock() - ds, exists := b.directQueryDataSources[name] + ds, exists := b.directQueryDataSources.Get(name) if !exists { return nil, fmt.Errorf( "%w: direct query data source %s not found", @@ -1916,7 +1885,7 @@ func (b *InMemoryBackend) DeleteDirectQueryDataSource(name string) error { b.mu.Lock("DeleteDirectQueryDataSource") defer b.mu.Unlock() - delete(b.directQueryDataSources, name) + b.directQueryDataSources.Delete(name) return nil } @@ -2006,8 +1975,8 @@ func (b *InMemoryBackend) DescribeReservedInstances() []*ReservedInstance { b.mu.RLock("DescribeReservedInstances") defer b.mu.RUnlock() - out := make([]*ReservedInstance, 0, len(b.reservedInstances)) - for _, ri := range b.reservedInstances { + out := make([]*ReservedInstance, 0, b.reservedInstances.Len()) + for _, ri := range b.reservedInstances.All() { cp := *ri out = append(out, &cp) } @@ -2058,7 +2027,7 @@ func (b *InMemoryBackend) PurchaseReservedInstanceOffering( State: pkgStateActive, StartTime: float64(time.Now().Unix()), } - b.reservedInstances[id] = ri + b.reservedInstances.Put(ri) cp := *ri @@ -2072,7 +2041,7 @@ func (b *InMemoryBackend) StartDomainMaintenance( b.mu.Lock("StartDomainMaintenance") defer b.mu.Unlock() - if _, exists := b.domains[domainName]; !exists { + if !b.domains.Has(domainName) { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } @@ -2148,14 +2117,10 @@ func (b *InMemoryBackend) CreateIndex( b.mu.Lock("CreateIndex") defer b.mu.Unlock() - if _, exists := b.domains[domainName]; !exists { + if !b.domains.Has(domainName) { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } - if b.domainIndexes[domainName] == nil { - b.domainIndexes[domainName] = make(map[string]*DomainIndex) - } - idx := &DomainIndex{ IndexName: indexName, IndexStatus: pkgStateActive, @@ -2163,9 +2128,10 @@ func (b *InMemoryBackend) CreateIndex( Settings: settings, Aliases: aliases, Documents: make(map[string]map[string]any), + DomainName: domainName, DocumentCount: 0, } - b.domainIndexes[domainName][indexName] = idx + b.domainIndexes.Put(idx) cp := *idx @@ -2177,17 +2143,7 @@ func (b *InMemoryBackend) DeleteIndex(domainName, indexName string) (*DomainInde b.mu.Lock("DeleteIndex") defer b.mu.Unlock() - idxMap := b.domainIndexes[domainName] - if idxMap == nil { - return nil, fmt.Errorf( - "%w: index %s not found on domain %s", - ErrConnectionNotFound, - indexName, - domainName, - ) - } - - idx, exists := idxMap[indexName] + idx, exists := b.domainIndexes.Get(domainIndexKey(domainName, indexName)) if !exists { return nil, fmt.Errorf( "%w: index %s not found on domain %s", @@ -2198,7 +2154,7 @@ func (b *InMemoryBackend) DeleteIndex(domainName, indexName string) (*DomainInde } cp := *idx - delete(idxMap, indexName) + b.domainIndexes.Delete(domainIndexKey(domainName, indexName)) return &cp, nil } @@ -2208,17 +2164,7 @@ func (b *InMemoryBackend) GetIndex(domainName, indexName string) (*DomainIndex, b.mu.RLock("GetIndex") defer b.mu.RUnlock() - idxMap := b.domainIndexes[domainName] - if idxMap == nil { - return nil, fmt.Errorf( - "%w: index %s not found on domain %s", - ErrConnectionNotFound, - indexName, - domainName, - ) - } - - idx, exists := idxMap[indexName] + idx, exists := b.domainIndexes.Get(domainIndexKey(domainName, indexName)) if !exists { return nil, fmt.Errorf( "%w: index %s not found on domain %s", @@ -2241,17 +2187,7 @@ func (b *InMemoryBackend) UpdateIndex( b.mu.Lock("UpdateIndex") defer b.mu.Unlock() - idxMap := b.domainIndexes[domainName] - if idxMap == nil { - return nil, fmt.Errorf( - "%w: index %s not found on domain %s", - ErrConnectionNotFound, - indexName, - domainName, - ) - } - - idx, exists := idxMap[indexName] + idx, exists := b.domainIndexes.Get(domainIndexKey(domainName, indexName)) if !exists { return nil, fmt.Errorf( "%w: index %s not found on domain %s", @@ -2273,7 +2209,7 @@ func (b *InMemoryBackend) GetApplication(id string) (*Application, error) { b.mu.RLock("GetApplication") defer b.mu.RUnlock() - app, exists := b.applications[id] + app, exists := b.applications.Get(id) if !exists { return nil, fmt.Errorf("%w: application %s not found", ErrApplicationNotFound, id) } @@ -2292,8 +2228,8 @@ func (b *InMemoryBackend) ListApplications() []*Application { b.mu.RLock("ListApplications") defer b.mu.RUnlock() - out := make([]*Application, 0, len(b.applications)) - for _, app := range b.applications { + out := make([]*Application, 0, b.applications.Len()) + for _, app := range b.applications.All() { cp := *app cp.AppConfigs = make([]AppConfig, len(app.AppConfigs)) copy(cp.AppConfigs, app.AppConfigs) @@ -2314,7 +2250,7 @@ func (b *InMemoryBackend) UpdateApplication( b.mu.Lock("UpdateApplication") defer b.mu.Unlock() - app, exists := b.applications[id] + app, exists := b.applications.Get(id) if !exists { return nil, fmt.Errorf("%w: application %s not found", ErrApplicationNotFound, id) } @@ -2341,13 +2277,11 @@ func (b *InMemoryBackend) DeleteApplication(id string) error { b.mu.Lock("DeleteApplication") defer b.mu.Unlock() - app, exists := b.applications[id] - if !exists { + if !b.applications.Has(id) { return fmt.Errorf("%w: application %s not found", ErrApplicationNotFound, id) } - delete(b.applicationNames, app.Name) - delete(b.applications, id) + b.applications.Delete(id) return nil } @@ -2361,7 +2295,7 @@ func (b *InMemoryBackend) StartServiceSoftwareUpdate( b.mu.Lock("StartServiceSoftwareUpdate") defer b.mu.Unlock() - d, exists := b.domains[domainName] + d, exists := b.domains.Get(domainName) if !exists || deleteWindowElapsed(d, b.clock()) { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } @@ -2402,8 +2336,8 @@ func (b *InMemoryBackend) DescribeDomains(names []string) ([]*Domain, error) { now := b.clock() if len(names) == 0 { - out := make([]*Domain, 0, len(b.domains)) - for _, d := range b.domains { + out := make([]*Domain, 0, b.domains.Len()) + for _, d := range b.domains.All() { if deleteWindowElapsed(d, now) { continue } @@ -2418,7 +2352,7 @@ func (b *InMemoryBackend) DescribeDomains(names []string) ([]*Domain, error) { out := make([]*Domain, 0, len(names)) for _, name := range names { - d, exists := b.domains[name] + d, exists := b.domains.Get(name) if !exists || deleteWindowElapsed(d, now) { continue } @@ -2508,7 +2442,7 @@ func (b *InMemoryBackend) UpdateDomainConfig( b.mu.Lock("UpdateDomainConfig") defer b.mu.Unlock() - d, exists := b.domains[name] + d, exists := b.domains.Get(name) if !exists || deleteWindowElapsed(d, b.clock()) { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, name) } @@ -2562,7 +2496,7 @@ func (b *InMemoryBackend) GetDomainHealth(domainName string) (map[string]any, er b.mu.RLock("GetDomainHealth") defer b.mu.RUnlock() - d, exists := b.domains[domainName] + d, exists := b.domains.Get(domainName) if !exists { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } @@ -2582,7 +2516,7 @@ func (b *InMemoryBackend) GetDomainHealth(domainName string) (map[string]any, er dedicatedMaster := d.ClusterConfig.DedicatedMasterEnabled docCount := 0 - for _, idx := range b.domainIndexes[domainName] { + for _, idx := range b.domainIndexesByDomain.Get(domainName) { docCount += idx.DocumentCount } @@ -2604,7 +2538,7 @@ func (b *InMemoryBackend) GetDomainNodes(domainName string) ([]map[string]any, e b.mu.RLock("GetDomainNodes") defer b.mu.RUnlock() - d, exists := b.domains[domainName] + d, exists := b.domains.Get(domainName) if !exists { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } @@ -2640,11 +2574,11 @@ func (b *InMemoryBackend) GetDryRunProgress(domainName string) (*DryRunStatus, e b.mu.Lock("GetDryRunProgress") defer b.mu.Unlock() - if _, exists := b.domains[domainName]; !exists { + if !b.domains.Has(domainName) { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } - dr, exists := b.dryRuns[domainName] + dr, exists := b.dryRuns.Get(domainName) if !exists { now := time.Now().UTC().Format(time.RFC3339) dr = &DryRunStatus{ @@ -2653,8 +2587,9 @@ func (b *InMemoryBackend) GetDryRunProgress(domainName string) (*DryRunStatus, e CreationDate: now, UpdateDate: now, ValidationFailures: []map[string]any{}, + DomainName: domainName, } - b.dryRuns[domainName] = dr + b.dryRuns.Put(dr) } if dr.ValidationFailures == nil { @@ -2671,7 +2606,7 @@ func (b *InMemoryBackend) GetChangeProgress(domainName string) (map[string]any, b.mu.RLock("GetChangeProgress") defer b.mu.RUnlock() - d, exists := b.domains[domainName] + d, exists := b.domains.Get(domainName) if !exists { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } @@ -2776,7 +2711,7 @@ func (b *InMemoryBackend) GetCompatibleVersions(domainName string) []map[string] } b.mu.RLock("GetCompatibleVersions") - d, exists := b.domains[domainName] + d, exists := b.domains.Get(domainName) b.mu.RUnlock() if !exists { @@ -2809,7 +2744,7 @@ func (b *InMemoryBackend) DissociatePackage( b.mu.Lock("DissociatePackage") defer b.mu.Unlock() - if _, exists := b.domains[domainName]; !exists { + if !b.domains.Has(domainName) { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } @@ -2834,7 +2769,7 @@ func (b *InMemoryBackend) DissociatePackages( b.mu.Lock("DissociatePackages") defer b.mu.Unlock() - if _, exists := b.domains[domainName]; !exists { + if !b.domains.Has(domainName) { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } @@ -2864,7 +2799,7 @@ func (b *InMemoryBackend) AddDomainInternal(name, engineVersion string) { domainARN := arn.Build("es", b.region, b.accountID, "domain/"+name) endpoint := fmt.Sprintf("search-%s-%s.%s.es.amazonaws.com", name, b.accountID, b.region) - b.domains[name] = &Domain{ + b.domains.Put(&Domain{ Name: name, ARN: domainARN, EngineVersion: engineVersion, @@ -2872,8 +2807,7 @@ func (b *InMemoryBackend) AddDomainInternal(name, engineVersion string) { Status: domainStatusActive, ClusterConfig: ClusterConfig{InstanceType: instanceTypeT3Small, InstanceCount: 1}, Tags: tags.New("opensearch." + name + ".tags"), - } - b.arnIndex[domainARN] = name + }) } // AddPackageInternal seeds a package directly for use in tests. @@ -2882,7 +2816,7 @@ func (b *InMemoryBackend) AddPackageInternal(packageID, packageName, packageType defer b.mu.Unlock() now := float64(time.Now().Unix()) - b.packages[packageID] = &Package{ + b.packages.Put(&Package{ PackageID: packageID, PackageName: packageName, PackageType: packageType, @@ -2895,5 +2829,5 @@ func (b *InMemoryBackend) AddPackageInternal(packageID, packageName, packageType CreatedAt: now, }, }, - } + }) } diff --git a/services/opensearch/backend_advanced.go b/services/opensearch/backend_advanced.go index d904106dc..c2bf51808 100644 --- a/services/opensearch/backend_advanced.go +++ b/services/opensearch/backend_advanced.go @@ -25,6 +25,7 @@ type UpgradeStepItem struct { // AutoTuneConfig stores auto-tune configuration for a domain. type AutoTuneConfig struct { DesiredState string `json:"DesiredState"` + DomainName string `json:"-"` MaintenanceSchedules []AutoTuneMaintenanceSchedule `json:"MaintenanceSchedules,omitempty"` } @@ -155,7 +156,7 @@ func (b *InMemoryBackend) UpgradeDomain(domainName, upgradeName string) error { b.mu.Lock("UpgradeDomain") defer b.mu.Unlock() - d, ok := b.domains[domainName] + d, ok := b.domains.Get(domainName) if !ok || deleteWindowElapsed(d, b.clock()) { return fmt.Errorf("%w: domain %q not found", ErrDomainNotFound, domainName) } @@ -200,7 +201,7 @@ func (b *InMemoryBackend) GetUpgradeHistory(domainName string) ([]*UpgradeHistor b.mu.RLock("GetUpgradeHistory") defer b.mu.RUnlock() - if _, ok := b.domains[domainName]; !ok { + if !b.domains.Has(domainName) { return nil, fmt.Errorf("%w: domain %q not found", ErrDomainNotFound, domainName) } @@ -220,7 +221,7 @@ func (b *InMemoryBackend) GetUpgradeStatus(domainName string) (string, string, s b.mu.RLock("GetUpgradeStatus") defer b.mu.RUnlock() - if _, ok := b.domains[domainName]; !ok { + if !b.domains.Has(domainName) { return "", "", "", fmt.Errorf("%w: domain %q not found", ErrDomainNotFound, domainName) } @@ -242,14 +243,15 @@ func (b *InMemoryBackend) SetAutoTune( b.mu.Lock("SetAutoTune") defer b.mu.Unlock() - if _, ok := b.domains[domainName]; !ok { + if !b.domains.Has(domainName) { return fmt.Errorf("%w: domain %q not found", ErrDomainNotFound, domainName) } - b.autoTunes[autoTuneKey(domainName)] = &AutoTuneConfig{ + b.autoTunes.Put(&AutoTuneConfig{ DesiredState: desiredState, MaintenanceSchedules: schedules, - } + DomainName: domainName, + }) return nil } @@ -259,11 +261,11 @@ func (b *InMemoryBackend) GetAutoTune(domainName string) ([]*AutoTune, error) { b.mu.RLock("GetAutoTune") defer b.mu.RUnlock() - if _, ok := b.domains[domainName]; !ok { + if !b.domains.Has(domainName) { return nil, fmt.Errorf("%w: domain %q not found", ErrDomainNotFound, domainName) } - cfg, ok := b.autoTunes[autoTuneKey(domainName)] + cfg, ok := b.autoTunes.Get(autoTuneKey(domainName)) if !ok || cfg == nil { return []*AutoTune{}, nil } @@ -421,15 +423,15 @@ func (b *InMemoryBackend) ListDomainNamesByEngine(engineType string) []string { defer b.mu.RUnlock() now := b.clock() - out := make([]string, 0, len(b.domains)) + out := make([]string, 0, b.domains.Len()) - for name, d := range b.domains { + for _, d := range b.domains.All() { if deleteWindowElapsed(d, now) { continue } if engineType == "" { - out = append(out, name) + out = append(out, d.Name) continue } @@ -439,11 +441,11 @@ func (b *InMemoryBackend) ListDomainNamesByEngine(engineType string) []string { switch engineType { case engineTypeOpenSearch: if isOpenSearchEngine(d.EngineVersion) { - out = append(out, name) + out = append(out, d.Name) } case engineTypeElasticsearch: if !isOpenSearchEngine(d.EngineVersion) { - out = append(out, name) + out = append(out, d.Name) } } } @@ -473,15 +475,15 @@ func (b *InMemoryBackend) ListDomainEntriesFiltered(engineType string) []DomainE defer b.mu.RUnlock() now := b.clock() - out := make([]DomainEntry, 0, len(b.domains)) + out := make([]DomainEntry, 0, b.domains.Len()) - for name, d := range b.domains { + for _, d := range b.domains.All() { if deleteWindowElapsed(d, now) { continue } if engineType == "" { - out = append(out, DomainEntry{Name: name, EngineVersion: d.EngineVersion}) + out = append(out, DomainEntry{Name: d.Name, EngineVersion: d.EngineVersion}) continue } @@ -489,15 +491,15 @@ func (b *InMemoryBackend) ListDomainEntriesFiltered(engineType string) []DomainE switch engineType { case engineTypeOpenSearch: if isOpenSearchEngine(d.EngineVersion) { - out = append(out, DomainEntry{Name: name, EngineVersion: d.EngineVersion}) + out = append(out, DomainEntry{Name: d.Name, EngineVersion: d.EngineVersion}) } case engineTypeElasticsearch: if !isOpenSearchEngine(d.EngineVersion) { - out = append(out, DomainEntry{Name: name, EngineVersion: d.EngineVersion}) + out = append(out, DomainEntry{Name: d.Name, EngineVersion: d.EngineVersion}) } default: if strings.HasPrefix(d.EngineVersion, engineType+"_") { - out = append(out, DomainEntry{Name: name, EngineVersion: d.EngineVersion}) + out = append(out, DomainEntry{Name: d.Name, EngineVersion: d.EngineVersion}) } } } diff --git a/services/opensearch/backend_docs.go b/services/opensearch/backend_docs.go index c5ea20ab4..863e96eb1 100644 --- a/services/opensearch/backend_docs.go +++ b/services/opensearch/backend_docs.go @@ -34,21 +34,11 @@ type SearchResult struct { // findIndexLocked returns the stored index for a domain, or an error. The caller // must hold at least a read lock. func (b *InMemoryBackend) findIndexLocked(domainName, indexName string) (*DomainIndex, error) { - if d, ok := b.domains[domainName]; !ok || deleteWindowElapsed(d, b.clock()) { + if d, ok := b.domains.Get(domainName); !ok || deleteWindowElapsed(d, b.clock()) { return nil, fmt.Errorf("%w: domain %s not found", ErrDomainNotFound, domainName) } - idxMap := b.domainIndexes[domainName] - if idxMap == nil { - return nil, fmt.Errorf( - "%w: index %s not found on domain %s", - ErrConnectionNotFound, - indexName, - domainName, - ) - } - - idx, ok := idxMap[indexName] + idx, ok := b.domainIndexes.Get(domainIndexKey(domainName, indexName)) if !ok { return nil, fmt.Errorf( "%w: index %s not found on domain %s", @@ -162,7 +152,7 @@ func (b *InMemoryBackend) DomainDocumentCount(domainName string) int { defer b.mu.RUnlock() total := 0 - for _, idx := range b.domainIndexes[domainName] { + for _, idx := range b.domainIndexesByDomain.Get(domainName) { total += idx.DocumentCount } diff --git a/services/opensearch/backend_lifecycle.go b/services/opensearch/backend_lifecycle.go index 9cc30b5d8..c80851457 100644 --- a/services/opensearch/backend_lifecycle.go +++ b/services/opensearch/backend_lifecycle.go @@ -1,6 +1,9 @@ package opensearch -import "time" +import ( + "slices" + "time" +) // clock returns the backend's current time, honouring an injected clock when set. func (b *InMemoryBackend) clock() time.Time { @@ -69,20 +72,25 @@ func statusWindowElapsed(status string, until, now time.Time) bool { // removeDomainLocked performs the full cascade removal of a domain and all its // domain-scoped resources. The caller must hold the write lock. func (b *InMemoryBackend) removeDomainLocked(name string) { - d, ok := b.domains[name] + d, ok := b.domains.Get(name) if !ok { return } - delete(b.domains, name) - delete(b.arnIndex, d.ARN) + b.domains.Delete(name) if d.Tags != nil { d.Tags.Close() } - // Cascade-clean all domain-scoped resources. - delete(b.domainDataSources, name) + // Cascade-clean all domain-scoped resources. The byDomain index results + // are cloned before deleting from the underlying table, since Index.Get + // returns a slice owned by the index that a concurrent Delete may + // invalidate mid-range. + for _, ds := range slices.Clone(b.domainDataSourcesByDomain.Get(name)) { + b.domainDataSources.Delete(dataSourceKey(ds.DomainName, ds.Name)) + } + delete(b.vpcAuthorizations, name) for pkgID := range b.domainPackages[name] { @@ -97,9 +105,12 @@ func (b *InMemoryBackend) removeDomainLocked(name string) { delete(b.domainMaintenances, name) delete(b.upgradeHistory, upgradeHistoryKey(name)) - delete(b.autoTunes, autoTuneKey(name)) - delete(b.dryRuns, name) - delete(b.domainIndexes, name) + b.autoTunes.Delete(autoTuneKey(name)) + b.dryRuns.Delete(name) + + for _, idx := range slices.Clone(b.domainIndexesByDomain.Get(name)) { + b.domainIndexes.Delete(domainIndexKey(idx.DomainName, idx.IndexName)) + } if b.dnsRegistrar != nil { b.dnsRegistrar.Deregister(d.Endpoint) @@ -111,9 +122,11 @@ func (b *InMemoryBackend) removeDomainLocked(name string) { func (b *InMemoryBackend) purgeExpiredDomainsLocked() { now := b.clock() - for name, d := range b.domains { + // Table.All returns a fresh slice, so cascading removeDomainLocked deletes + // from the table while ranging over it here is safe. + for _, d := range b.domains.All() { if deleteWindowElapsed(d, now) { - b.removeDomainLocked(name) + b.removeDomainLocked(d.Name) } } } diff --git a/services/opensearch/backend_serverless.go b/services/opensearch/backend_serverless.go index e97d36dc4..4f1b53268 100644 --- a/services/opensearch/backend_serverless.go +++ b/services/opensearch/backend_serverless.go @@ -157,7 +157,7 @@ func (b *InMemoryBackend) CreateServerlessCollection( coll.StatusUntil = b.clock().Add(b.processingDelay) } - b.slCollections[serverlessCollectionKey(name)] = coll + b.slCollections.Put(coll) cp := *coll resolveCollectionStatus(&cp, b.clock()) @@ -184,9 +184,9 @@ func collectionDeleteElapsed(c *ServerlessCollection, now time.Time) bool { // The caller must hold the write lock. func (b *InMemoryBackend) purgeExpiredCollectionsLocked() { now := b.clock() - for key, c := range b.slCollections { + for _, c := range b.slCollections.All() { if collectionDeleteElapsed(c, now) { - delete(b.slCollections, key) + b.slCollections.Delete(serverlessCollectionKey(c.Name)) } } } @@ -210,7 +210,7 @@ func (b *InMemoryBackend) BatchGetServerlessCollections(ids, names []string) []* var out []*ServerlessCollection - for _, c := range b.slCollections { + for _, c := range b.slCollections.All() { if collectionDeleteElapsed(c, now) { continue } @@ -244,7 +244,7 @@ func (b *InMemoryBackend) DeleteServerlessCollection(id string) (*ServerlessColl now := b.clock() - for key, c := range b.slCollections { + for _, c := range b.slCollections.All() { if c.ID != id || collectionDeleteElapsed(c, now) { continue } @@ -252,7 +252,7 @@ func (b *InMemoryBackend) DeleteServerlessCollection(id string) (*ServerlessColl if b.processingDelay == 0 { cp := *c cp.Status = statusDeleted - delete(b.slCollections, key) + b.slCollections.Delete(serverlessCollectionKey(c.Name)) return &cp, nil } @@ -279,7 +279,7 @@ func (b *InMemoryBackend) CreateServerlessAccessPolicy( defer b.mu.Unlock() key := serverlessAccessPolicyKey(policyType, name) - if _, exists := b.slAccessPolicies[key]; exists { + if b.slAccessPolicies.Has(key) { return nil, fmt.Errorf("%w: access policy %s already exists", ErrApplicationAlreadyExists, name) } @@ -294,7 +294,7 @@ func (b *InMemoryBackend) CreateServerlessAccessPolicy( LastModifiedDate: now, } - b.slAccessPolicies[key] = ap + b.slAccessPolicies.Put(ap) cp := *ap @@ -306,7 +306,7 @@ func (b *InMemoryBackend) GetServerlessAccessPolicy(policyType, name string) (*S b.mu.RLock("GetServerlessAccessPolicy") defer b.mu.RUnlock() - ap, ok := b.slAccessPolicies[serverlessAccessPolicyKey(policyType, name)] + ap, ok := b.slAccessPolicies.Get(serverlessAccessPolicyKey(policyType, name)) if !ok { return nil, fmt.Errorf("%w: access policy %s not found", ErrApplicationNotFound, name) } @@ -323,7 +323,7 @@ func (b *InMemoryBackend) ListServerlessAccessPolicies(policyType string) []*Ser var out []*ServerlessAccessPolicy - for _, ap := range b.slAccessPolicies { + for _, ap := range b.slAccessPolicies.All() { if policyType == "" || ap.Type == policyType { cp := *ap out = append(out, &cp) @@ -340,8 +340,7 @@ func (b *InMemoryBackend) UpdateServerlessAccessPolicy( b.mu.Lock("UpdateServerlessAccessPolicy") defer b.mu.Unlock() - key := serverlessAccessPolicyKey(policyType, name) - ap, ok := b.slAccessPolicies[key] + ap, ok := b.slAccessPolicies.Get(serverlessAccessPolicyKey(policyType, name)) if !ok { return nil, fmt.Errorf("%w: access policy %s not found", ErrApplicationNotFound, name) } @@ -369,11 +368,11 @@ func (b *InMemoryBackend) DeleteServerlessAccessPolicy(policyType, name string) defer b.mu.Unlock() key := serverlessAccessPolicyKey(policyType, name) - if _, ok := b.slAccessPolicies[key]; !ok { + if !b.slAccessPolicies.Has(key) { return fmt.Errorf("%w: access policy %s not found", ErrApplicationNotFound, name) } - delete(b.slAccessPolicies, key) + b.slAccessPolicies.Delete(key) return nil } @@ -400,7 +399,7 @@ func (b *InMemoryBackend) CreateServerlessSecurityConfig( LastModifiedDate: now, } - b.slSecurityConfigs[serverlessSecurityConfigKey(id)] = sc + b.slSecurityConfigs.Put(sc) cp := *sc @@ -412,7 +411,7 @@ func (b *InMemoryBackend) GetServerlessSecurityConfig(id string) (*ServerlessSec b.mu.RLock("GetServerlessSecurityConfig") defer b.mu.RUnlock() - sc, ok := b.slSecurityConfigs[serverlessSecurityConfigKey(id)] + sc, ok := b.slSecurityConfigs.Get(serverlessSecurityConfigKey(id)) if !ok { return nil, fmt.Errorf("%w: security config %s not found", ErrApplicationNotFound, id) } @@ -429,7 +428,7 @@ func (b *InMemoryBackend) ListServerlessSecurityConfigs(configType string) []*Se var out []*ServerlessSecurityConfig - for _, sc := range b.slSecurityConfigs { + for _, sc := range b.slSecurityConfigs.All() { if configType == "" || sc.Type == configType { cp := *sc out = append(out, &cp) @@ -447,7 +446,7 @@ func (b *InMemoryBackend) UpdateServerlessSecurityConfig( b.mu.Lock("UpdateServerlessSecurityConfig") defer b.mu.Unlock() - sc, ok := b.slSecurityConfigs[serverlessSecurityConfigKey(id)] + sc, ok := b.slSecurityConfigs.Get(serverlessSecurityConfigKey(id)) if !ok { return nil, fmt.Errorf("%w: security config %s not found", ErrApplicationNotFound, id) } @@ -475,11 +474,11 @@ func (b *InMemoryBackend) DeleteServerlessSecurityConfig(id string) error { defer b.mu.Unlock() key := serverlessSecurityConfigKey(id) - if _, ok := b.slSecurityConfigs[key]; !ok { + if !b.slSecurityConfigs.Has(key) { return fmt.Errorf("%w: security config %s not found", ErrApplicationNotFound, id) } - delete(b.slSecurityConfigs, key) + b.slSecurityConfigs.Delete(key) return nil } @@ -496,7 +495,7 @@ func (b *InMemoryBackend) CreateServerlessEncryptionPolicy( defer b.mu.Unlock() key := serverlessEncryptionPolicyKey(policyType, name) - if _, exists := b.slEncryptionPolicies[key]; exists { + if b.slEncryptionPolicies.Has(key) { return nil, fmt.Errorf("%w: encryption policy %s already exists", ErrApplicationAlreadyExists, name) } @@ -511,7 +510,7 @@ func (b *InMemoryBackend) CreateServerlessEncryptionPolicy( LastModifiedDate: now, } - b.slEncryptionPolicies[key] = ep + b.slEncryptionPolicies.Put(ep) cp := *ep @@ -523,7 +522,7 @@ func (b *InMemoryBackend) GetServerlessEncryptionPolicy(policyType, name string) b.mu.RLock("GetServerlessEncryptionPolicy") defer b.mu.RUnlock() - ep, ok := b.slEncryptionPolicies[serverlessEncryptionPolicyKey(policyType, name)] + ep, ok := b.slEncryptionPolicies.Get(serverlessEncryptionPolicyKey(policyType, name)) if !ok { return nil, fmt.Errorf("%w: encryption policy %s not found", ErrApplicationNotFound, name) } @@ -540,7 +539,7 @@ func (b *InMemoryBackend) ListServerlessEncryptionPolicies(policyType string) [] var out []*ServerlessEncryptionPolicy - for _, ep := range b.slEncryptionPolicies { + for _, ep := range b.slEncryptionPolicies.All() { if policyType == "" || ep.Type == policyType { cp := *ep out = append(out, &cp) @@ -557,8 +556,7 @@ func (b *InMemoryBackend) UpdateServerlessEncryptionPolicy( b.mu.Lock("UpdateServerlessEncryptionPolicy") defer b.mu.Unlock() - key := serverlessEncryptionPolicyKey(policyType, name) - ep, ok := b.slEncryptionPolicies[key] + ep, ok := b.slEncryptionPolicies.Get(serverlessEncryptionPolicyKey(policyType, name)) if !ok { return nil, fmt.Errorf("%w: encryption policy %s not found", ErrApplicationNotFound, name) } @@ -586,11 +584,11 @@ func (b *InMemoryBackend) DeleteServerlessEncryptionPolicy(policyType, name stri defer b.mu.Unlock() key := serverlessEncryptionPolicyKey(policyType, name) - if _, ok := b.slEncryptionPolicies[key]; !ok { + if !b.slEncryptionPolicies.Has(key) { return fmt.Errorf("%w: encryption policy %s not found", ErrApplicationNotFound, name) } - delete(b.slEncryptionPolicies, key) + b.slEncryptionPolicies.Delete(key) return nil } @@ -607,7 +605,7 @@ func (b *InMemoryBackend) CreateServerlessNetworkPolicy( defer b.mu.Unlock() key := serverlessNetworkPolicyKey(policyType, name) - if _, exists := b.slNetworkPolicies[key]; exists { + if b.slNetworkPolicies.Has(key) { return nil, fmt.Errorf("%w: network policy %s already exists", ErrApplicationAlreadyExists, name) } @@ -622,7 +620,7 @@ func (b *InMemoryBackend) CreateServerlessNetworkPolicy( LastModifiedDate: now, } - b.slNetworkPolicies[key] = np + b.slNetworkPolicies.Put(np) cp := *np @@ -636,7 +634,7 @@ func (b *InMemoryBackend) ListServerlessNetworkPolicies(policyType string) []*Se var out []*ServerlessNetworkPolicy - for _, np := range b.slNetworkPolicies { + for _, np := range b.slNetworkPolicies.All() { if policyType == "" || np.Type == policyType { cp := *np out = append(out, &cp) @@ -652,11 +650,11 @@ func (b *InMemoryBackend) DeleteServerlessNetworkPolicy(policyType, name string) defer b.mu.Unlock() key := serverlessNetworkPolicyKey(policyType, name) - if _, ok := b.slNetworkPolicies[key]; !ok { + if !b.slNetworkPolicies.Has(key) { return fmt.Errorf("%w: network policy %s not found", ErrApplicationNotFound, name) } - delete(b.slNetworkPolicies, key) + b.slNetworkPolicies.Delete(key) return nil } diff --git a/services/opensearch/export_test.go b/services/opensearch/export_test.go index fd785cee4..cb5e887b0 100644 --- a/services/opensearch/export_test.go +++ b/services/opensearch/export_test.go @@ -9,7 +9,7 @@ func ExpireDomainProcessing(b *InMemoryBackend, name string) { b.mu.Lock("ExpireDomainProcessing") defer b.mu.Unlock() - if d, ok := b.domains[name]; ok { + if d, ok := b.domains.Get(name); ok { d.ProcessingUntil = time.Now().Add(-time.Hour) } } @@ -20,7 +20,7 @@ func ExpireCollectionStatus(b *InMemoryBackend, id string) { b.mu.Lock("ExpireCollectionStatus") defer b.mu.Unlock() - for _, c := range b.slCollections { + for _, c := range b.slCollections.All() { if c.ID == id { c.StatusUntil = time.Now().Add(-time.Hour) } @@ -33,7 +33,7 @@ func ExpireOutboundConnection(b *InMemoryBackend, id string) { b.mu.Lock("ExpireOutboundConnection") defer b.mu.Unlock() - if c, ok := b.outboundConnections[id]; ok { + if c, ok := b.outboundConnections.Get(id); ok { c.StatusUntil = time.Now().Add(-time.Hour) } } @@ -43,7 +43,7 @@ func ExpireVpcEndpoint(b *InMemoryBackend, id string) { b.mu.Lock("ExpireVpcEndpoint") defer b.mu.Unlock() - if ep, ok := b.vpcEndpoints[id]; ok { + if ep, ok := b.vpcEndpoints.Get(id); ok { ep.StatusUntil = time.Now().Add(-time.Hour) } } @@ -61,7 +61,7 @@ func DomainServiceSoftwareStatus(b *InMemoryBackend, name string) string { b.mu.RLock("DomainServiceSoftwareStatus") defer b.mu.RUnlock() - d, ok := b.domains[name] + d, ok := b.domains.Get(name) if !ok || d.ServiceSoftware == nil { return "" } @@ -74,7 +74,7 @@ func DomainCount(b *InMemoryBackend) int { b.mu.RLock("DomainCount") defer b.mu.RUnlock() - return len(b.domains) + return b.domains.Len() } // ConnectionCount returns the number of inbound connections in the backend. @@ -82,7 +82,7 @@ func ConnectionCount(b *InMemoryBackend) int { b.mu.RLock("ConnectionCount") defer b.mu.RUnlock() - return len(b.inboundConnections) + return b.inboundConnections.Len() } // DataSourceCount returns the total number of domain data sources across all domains. @@ -90,12 +90,7 @@ func DataSourceCount(b *InMemoryBackend) int { b.mu.RLock("DataSourceCount") defer b.mu.RUnlock() - total := 0 - for _, m := range b.domainDataSources { - total += len(m) - } - - return total + return b.domainDataSources.Len() } // DirectQueryDataSourceCount returns the number of direct-query data sources. @@ -103,7 +98,7 @@ func DirectQueryDataSourceCount(b *InMemoryBackend) int { b.mu.RLock("DirectQueryDataSourceCount") defer b.mu.RUnlock() - return len(b.directQueryDataSources) + return b.directQueryDataSources.Len() } // ApplicationCount returns the number of applications in the backend. @@ -111,7 +106,7 @@ func ApplicationCount(b *InMemoryBackend) int { b.mu.RLock("ApplicationCount") defer b.mu.RUnlock() - return len(b.applications) + return b.applications.Len() } // ARNIndexSize returns the number of entries in the ARN index. @@ -119,7 +114,7 @@ func ARNIndexSize(b *InMemoryBackend) int { b.mu.RLock("ARNIndexSize") defer b.mu.RUnlock() - return len(b.arnIndex) + return b.domainsByARN.Len() } // HandlerOpsLen returns the number of operations in GetSupportedOperations. @@ -156,8 +151,8 @@ func SeedInboundConnection(b *InMemoryBackend, connectionID string) { b.mu.Lock("SeedInboundConnection") defer b.mu.Unlock() - b.inboundConnections[connectionID] = &InboundConnection{ + b.inboundConnections.Put(&InboundConnection{ ConnectionID: connectionID, Status: "PENDING_ACCEPTANCE", - } + }) } diff --git a/services/opensearch/persistence.go b/services/opensearch/persistence.go index 77384385f..95bd19f54 100644 --- a/services/opensearch/persistence.go +++ b/services/opensearch/persistence.go @@ -3,45 +3,236 @@ package opensearch import ( "context" "encoding/json" + "fmt" + "maps" - "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) +// opensearchSnapshotVersion identifies the shape of backendSnapshot's Tables +// blob (the set of "clean" tables on b.registry plus the DTO tables built +// below for the "dirty" ones -- see store_setup.go's registerAllTables doc +// for the clean/dirty split). Bump whenever a change to a DTO type, a +// registered table's value type, or backendSnapshot itself would make an +// older snapshot unsafe to decode as the current shape. Restore compares this +// against the persisted value and discards (rather than attempts to +// partially decode) any mismatch -- see Restore below. Mirrors the +// services/sqs pilot (commit 0f09d77c) and services/apigateway (commit +// 6da0334e). +const opensearchSnapshotVersion = 1 + +// dryRunSnapshot, autoTuneSnapshot, dataSourceSnapshot, and +// domainIndexSnapshot are DTOs used ONLY for Snapshot/Restore. Each mirrors +// its live type field for field, except the identity field (DomainName) is +// given a real JSON tag here instead of the live type's `json:"-"` (see +// backend.go / backend_advanced.go) -- marshaling the live type directly +// would silently drop that field, and unmarshaling into it would leave it +// permanently empty, corrupting the table's key on restore. This is the same +// DTO-registry technique services/sqs uses for its Queue/moveTaskState types +// (commit 0f09d77c) and services/apigateway uses for its nested-resource +// families (commit 6da0334e), applied here for the same reason (a json:"-" +// identity field needed only to key the pkgs/store table). +type dryRunSnapshot struct { + DryRunID string `json:"dryRunId"` + DryRunStatus string `json:"dryRunStatus"` + CreationDate string `json:"creationDate"` + UpdateDate string `json:"updateDate"` + DomainName string `json:"domainName"` + ValidationFailures []map[string]any `json:"validationFailures"` +} + +func dryRunSnapshotKey(v *dryRunSnapshot) string { return v.DomainName } + +func toDryRunSnapshot(v *DryRunStatus) *dryRunSnapshot { + return &dryRunSnapshot{ + DryRunID: v.DryRunID, + DryRunStatus: v.DryRunStatus, + CreationDate: v.CreationDate, + UpdateDate: v.UpdateDate, + DomainName: v.DomainName, + ValidationFailures: v.ValidationFailures, + } +} + +func fromDryRunSnapshot(v *dryRunSnapshot) *DryRunStatus { + return &DryRunStatus{ + DryRunID: v.DryRunID, + DryRunStatus: v.DryRunStatus, + CreationDate: v.CreationDate, + UpdateDate: v.UpdateDate, + DomainName: v.DomainName, + ValidationFailures: v.ValidationFailures, + } +} + +type autoTuneSnapshot struct { + DesiredState string `json:"desiredState"` + DomainName string `json:"domainName"` + MaintenanceSchedules []AutoTuneMaintenanceSchedule `json:"maintenanceSchedules,omitempty"` +} + +func autoTuneSnapshotKey(v *autoTuneSnapshot) string { return autoTuneKey(v.DomainName) } + +func toAutoTuneSnapshot(v *AutoTuneConfig) *autoTuneSnapshot { + return &autoTuneSnapshot{ + DesiredState: v.DesiredState, + MaintenanceSchedules: v.MaintenanceSchedules, + DomainName: v.DomainName, + } +} + +func fromAutoTuneSnapshot(v *autoTuneSnapshot) *AutoTuneConfig { + return &AutoTuneConfig{ + DesiredState: v.DesiredState, + MaintenanceSchedules: v.MaintenanceSchedules, + DomainName: v.DomainName, + } +} + +type dataSourceSnapshot struct { + Name string `json:"name"` + Description string `json:"description"` + DataSourceType string `json:"dataSourceType"` + DomainName string `json:"domainName"` +} + +func dataSourceSnapshotKey(v *dataSourceSnapshot) string { return dataSourceKey(v.DomainName, v.Name) } + +func toDataSourceSnapshot(v *DataSource) *dataSourceSnapshot { + return &dataSourceSnapshot{ + Name: v.Name, + Description: v.Description, + DataSourceType: v.DataSourceType, + DomainName: v.DomainName, + } +} + +func fromDataSourceSnapshot(v *dataSourceSnapshot) *DataSource { + return &DataSource{ + Name: v.Name, + Description: v.Description, + DataSourceType: v.DataSourceType, + DomainName: v.DomainName, + } +} + +type domainIndexSnapshot struct { + Mappings map[string]any `json:"mappings,omitempty"` + Settings map[string]any `json:"settings,omitempty"` + Aliases map[string]any `json:"aliases,omitempty"` + Documents map[string]map[string]any `json:"documents,omitempty"` + IndexName string `json:"indexName"` + IndexStatus string `json:"indexStatus"` + DomainName string `json:"domainName"` + DocumentCount int `json:"documentCount"` +} + +func domainIndexSnapshotKey(v *domainIndexSnapshot) string { + return domainIndexKey(v.DomainName, v.IndexName) +} + +func toDomainIndexSnapshot(v *DomainIndex) *domainIndexSnapshot { + return &domainIndexSnapshot{ + Mappings: v.Mappings, + Settings: v.Settings, + Aliases: v.Aliases, + Documents: v.Documents, + IndexName: v.IndexName, + IndexStatus: v.IndexStatus, + DomainName: v.DomainName, + DocumentCount: v.DocumentCount, + } +} + +func fromDomainIndexSnapshot(v *domainIndexSnapshot) *DomainIndex { + return &DomainIndex{ + Mappings: v.Mappings, + Settings: v.Settings, + Aliases: v.Aliases, + Documents: v.Documents, + IndexName: v.IndexName, + IndexStatus: v.IndexStatus, + DomainName: v.DomainName, + DocumentCount: v.DocumentCount, + } +} + +// dirtyTableNames lists the "dirty" table names shared by Snapshot and +// Restore (see store_setup.go's registerAllTables doc). Both build an +// ephemeral DTO [store.Registry] under these exact names, so a snapshot +// written by one version of Snapshot always lines up with the same version +// of Restore. +// +//nolint:gochecknoglobals // fixed lookup table, mirrors errCodeLookup-style tables elsewhere +var dirtyTableNames = struct { + dryRuns, autoTunes, domainDataSources, domainIndexes string +}{ + dryRuns: "dryRuns", + autoTunes: "autoTunes", + domainDataSources: "domainDataSources", + domainIndexes: "domainIndexes", +} + +// backendSnapshot is the top-level on-disk shape for the OpenSearch backend. +// +// Tables holds one JSON-encoded array per table -- both the "clean" tables +// registered on b.registry (produced by [store.Registry.SnapshotAll]) and the +// "dirty" DTO tables built inline in Snapshot (see store_setup.go's +// registerAllTables doc for the clean/dirty split), merged into one map so a +// single Tables blob round-trips the whole backend. +// +// The remaining fields are the plain maps left unconverted (see +// store_setup.go's registerAllTables doc for why) plus the account/region and +// monotonic ID counters. +// +// domainPackages (the reverse packageID<-domainName index used by +// ListPackagesForDomain) is deliberately NOT included here: the pre-existing +// backend, before this pkgs/store conversion, never persisted or restored it +// either (only its forward twin, PackageAssociations, round-tripped). That +// asymmetry is preserved byte-for-byte rather than fixed as part of this +// mechanical conversion -- see the per-map persistence audit in the Phase 3.3 +// conversion notes. type backendSnapshot struct { - Domains map[string]*Domain `json:"domains"` - InboundConnections map[string]*InboundConnection `json:"inboundConnections"` - OutboundConnections map[string]*OutboundConnection `json:"outboundConnections"` - DomainDataSources map[string]map[string]*DataSource `json:"domainDataSources"` - DirectQueryDataSources map[string]*DirectQueryDataSource `json:"directQueryDataSources"` - PackageAssociations map[string]map[string]bool `json:"packageAssociations"` - VpcAuthorizations map[string][]AuthorizedPrincipal `json:"vpcAuthorizations"` - VpcEndpoints map[string]*VpcEndpoint `json:"vpcEndpoints"` - Applications map[string]*Application `json:"applications"` - Packages map[string]*Package `json:"packages"` - ScheduledActions map[string][]*ScheduledAction `json:"scheduledActions"` - ReservedInstances map[string]*ReservedInstance `json:"reservedInstances"` - DomainMaintenances map[string][]*DomainMaintenance `json:"domainMaintenances"` - DomainIndexes map[string]map[string]*DomainIndex `json:"domainIndexes"` - UpgradeHistory map[string][]*UpgradeHistory `json:"upgradeHistory"` - AutoTunes map[string]*AutoTuneConfig `json:"autoTunes"` - SlCollections map[string]*ServerlessCollection `json:"slCollections"` - SlAccessPolicies map[string]*ServerlessAccessPolicy `json:"slAccessPolicies"` - SlSecurityConfigs map[string]*ServerlessSecurityConfig `json:"slSecurityConfigs"` - SlEncryptionPolicies map[string]*ServerlessEncryptionPolicy `json:"slEncryptionPolicies"` - SlNetworkPolicies map[string]*ServerlessNetworkPolicy `json:"slNetworkPolicies"` - AccountID string `json:"accountID"` - Region string `json:"region"` - AppIDCounter int `json:"appIDCounter"` - ConnCounter int `json:"connCounter"` - VpcEndpointCounter int `json:"vpcEndpointCounter"` - PackageCounter int `json:"packageCounter"` - MaintenanceCounter int `json:"maintenanceCounter"` - ReservedCounter int `json:"reservedCounter"` - SlCollCounter int `json:"slCollCounter"` - SlSecConfigCounter int `json:"slSecConfigCounter"` + Tables map[string]json.RawMessage `json:"tables"` + VpcAuthorizations map[string][]AuthorizedPrincipal `json:"vpcAuthorizations"` + ScheduledActions map[string][]*ScheduledAction `json:"scheduledActions"` + PackageAssociations map[string]map[string]bool `json:"packageAssociations"` + DomainMaintenances map[string][]*DomainMaintenance `json:"domainMaintenances"` + UpgradeHistory map[string][]*UpgradeHistory `json:"upgradeHistory"` + DefaultAppSettings map[string][]AppSetting `json:"defaultAppSettings"` + AccountID string `json:"accountID"` + Region string `json:"region"` + AppIDCounter int `json:"appIDCounter"` + ConnCounter int `json:"connCounter"` + VpcEndpointCounter int `json:"vpcEndpointCounter"` + PackageCounter int `json:"packageCounter"` + MaintenanceCounter int `json:"maintenanceCounter"` + ReservedCounter int `json:"reservedCounter"` + SlCollCounter int `json:"slCollCounter"` + SlSecConfigCounter int `json:"slSecConfigCounter"` + Version int `json:"version"` +} + +// newDirtyDTORegistry builds the ephemeral DTO [store.Registry] shared by +// Snapshot and Restore for the "dirty" tables (see store_setup.go's +// registerAllTables doc). +func newDirtyDTORegistry() ( + *store.Registry, + *store.Table[dryRunSnapshot], + *store.Table[autoTuneSnapshot], + *store.Table[dataSourceSnapshot], + *store.Table[domainIndexSnapshot], +) { + dtoReg := store.NewRegistry() + dryRunDTOs := store.Register(dtoReg, dirtyTableNames.dryRuns, store.New(dryRunSnapshotKey)) + autoTuneDTOs := store.Register(dtoReg, dirtyTableNames.autoTunes, store.New(autoTuneSnapshotKey)) + dataSourceDTOs := store.Register(dtoReg, dirtyTableNames.domainDataSources, store.New(dataSourceSnapshotKey)) + domainIndexDTOs := store.Register(dtoReg, dirtyTableNames.domainIndexes, store.New(domainIndexSnapshotKey)) + + return dtoReg, dryRunDTOs, autoTuneDTOs, dataSourceDTOs, domainIndexDTOs } // Snapshot serialises the backend state to JSON. @@ -50,48 +241,66 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() - snap := backendSnapshot{ - Domains: b.domains, - InboundConnections: b.inboundConnections, - OutboundConnections: b.outboundConnections, - DomainDataSources: b.domainDataSources, - DirectQueryDataSources: b.directQueryDataSources, - PackageAssociations: b.packageAssociations, - VpcAuthorizations: b.vpcAuthorizations, - VpcEndpoints: b.vpcEndpoints, - Applications: b.applications, - Packages: b.packages, - ScheduledActions: b.scheduledActions, - ReservedInstances: b.reservedInstances, - DomainMaintenances: b.domainMaintenances, - DomainIndexes: b.domainIndexes, - UpgradeHistory: b.upgradeHistory, - AutoTunes: b.autoTunes, - SlCollections: b.slCollections, - SlAccessPolicies: b.slAccessPolicies, - SlSecurityConfigs: b.slSecurityConfigs, - SlEncryptionPolicies: b.slEncryptionPolicies, - SlNetworkPolicies: b.slNetworkPolicies, - AppIDCounter: b.appIDCounter, - ConnCounter: b.connCounter, - VpcEndpointCounter: b.vpcEndpointCounter, - PackageCounter: b.packageCounter, - MaintenanceCounter: b.maintenanceCounter, - ReservedCounter: b.reservedCounter, - SlCollCounter: b.slCollCounter, - SlSecConfigCounter: b.slSecConfigCounter, - AccountID: b.accountID, - Region: b.region, + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "opensearch: snapshot table marshal failed", "error", err) + + return nil + } + + dtoReg, dryRunDTOs, autoTuneDTOs, dataSourceDTOs, domainIndexDTOs := newDirtyDTORegistry() + + for _, v := range b.dryRuns.Snapshot() { + dryRunDTOs.Put(toDryRunSnapshot(v)) + } + + for _, v := range b.autoTunes.Snapshot() { + autoTuneDTOs.Put(toAutoTuneSnapshot(v)) + } + + for _, v := range b.domainDataSources.Snapshot() { + dataSourceDTOs.Put(toDataSourceSnapshot(v)) } - data, err := json.Marshal(snap) + for _, v := range b.domainIndexes.Snapshot() { + domainIndexDTOs.Put(toDomainIndexSnapshot(v)) + } + + dirtyTables, err := dtoReg.SnapshotAll() if err != nil { - logger.Load(ctx).WarnContext(ctx, "opensearch: failed to snapshot backend", "error", err) + logger.Load(ctx).WarnContext(ctx, "opensearch: snapshot DTO table marshal failed", "error", err) return nil } - return data + maps.Copy(tables, dirtyTables) + + snap := backendSnapshot{ + Version: opensearchSnapshotVersion, + Tables: tables, + VpcAuthorizations: b.vpcAuthorizations, + ScheduledActions: b.scheduledActions, + PackageAssociations: b.packageAssociations, + DomainMaintenances: b.domainMaintenances, + UpgradeHistory: b.upgradeHistory, + DefaultAppSettings: b.defaultAppSettings, + AccountID: b.accountID, + Region: b.region, + AppIDCounter: b.appIDCounter, + ConnCounter: b.connCounter, + VpcEndpointCounter: b.vpcEndpointCounter, + PackageCounter: b.packageCounter, + MaintenanceCounter: b.maintenanceCounter, + ReservedCounter: b.reservedCounter, + SlCollCounter: b.slCollCounter, + SlSecConfigCounter: b.slSecConfigCounter, + } + + return persistence.MarshalSnapshot(ctx, "opensearch", snap) } // Restore loads backend state from a JSON snapshot. @@ -103,180 +312,163 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { return err } - ensureNonNilMaps(&snap) - fixNilDomainTags(&snap) - b.mu.Lock("Restore") defer b.mu.Unlock() // Release tags resources from the domains being replaced. - for _, d := range b.domains { + for _, d := range b.domains.All() { d.Tags.Close() } - b.domains = snap.Domains - b.inboundConnections = snap.InboundConnections - b.outboundConnections = snap.OutboundConnections - b.domainDataSources = snap.DomainDataSources - b.directQueryDataSources = snap.DirectQueryDataSources - b.packageAssociations = snap.PackageAssociations - b.vpcAuthorizations = snap.VpcAuthorizations - b.vpcEndpoints = snap.VpcEndpoints - b.applications = snap.Applications - b.packages = snap.Packages - b.scheduledActions = snap.ScheduledActions - b.reservedInstances = snap.ReservedInstances - b.domainMaintenances = snap.DomainMaintenances - b.domainIndexes = snap.DomainIndexes - b.upgradeHistory = snap.UpgradeHistory - b.autoTunes = snap.AutoTunes - b.slCollections = snap.SlCollections - b.slAccessPolicies = snap.SlAccessPolicies - b.slSecurityConfigs = snap.SlSecurityConfigs - b.slEncryptionPolicies = snap.SlEncryptionPolicies - b.slNetworkPolicies = snap.SlNetworkPolicies - b.appIDCounter = snap.AppIDCounter - b.connCounter = snap.ConnCounter - b.vpcEndpointCounter = snap.VpcEndpointCounter - b.packageCounter = snap.PackageCounter - b.maintenanceCounter = snap.MaintenanceCounter - b.reservedCounter = snap.ReservedCounter - b.slCollCounter = snap.SlCollCounter - b.slSecConfigCounter = snap.SlSecConfigCounter - b.accountID = snap.AccountID - b.region = snap.Region - - // Rebuild the ARN index from restored domains. - b.arnIndex = rebuildARNIndex(snap.Domains, snap.AccountID, snap.Region) + if snap.Version != opensearchSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. + logger.Load(ctx).WarnContext(ctx, + "opensearch: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", opensearchSnapshotVersion) + + b.registry.ResetAll() + b.dryRuns.Reset() + b.autoTunes.Reset() + b.domainDataSources.Reset() + b.domainIndexes.Reset() + b.vpcAuthorizations = make(map[string][]AuthorizedPrincipal) + b.scheduledActions = make(map[string][]*ScheduledAction) + b.packageAssociations = make(map[string]map[string]bool) + b.domainMaintenances = make(map[string][]*DomainMaintenance) + b.upgradeHistory = make(map[string][]*UpgradeHistory) + b.domainPackages = make(map[string]map[string]bool) + b.defaultAppSettings = make(map[string][]AppSetting) - // Rebuild applicationNames index from restored applications. - b.applicationNames = make(map[string]string, len(snap.Applications)) - for id, app := range snap.Applications { - b.applicationNames[app.Name] = id + return nil } - return nil -} - -// ensureNonNilMaps initialises nil maps in the snapshot to empty maps. -func ensureNonNilMaps(snap *backendSnapshot) { - ensureNonNilCoreMaps(snap) - ensureNonNilExtendedMaps(snap) -} - -// ensureNonNilCoreMaps initialises nil core maps in the snapshot. -func ensureNonNilCoreMaps(snap *backendSnapshot) { - if snap.Domains == nil { - snap.Domains = make(map[string]*Domain) + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("opensearch: restore snapshot tables: %w", err) } - if snap.InboundConnections == nil { - snap.InboundConnections = make(map[string]*InboundConnection) - } + dtoReg, dryRunDTOs, autoTuneDTOs, dataSourceDTOs, domainIndexDTOs := newDirtyDTORegistry() - if snap.OutboundConnections == nil { - snap.OutboundConnections = make(map[string]*OutboundConnection) + if err := dtoReg.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("opensearch: restore snapshot DTO tables: %w", err) } - if snap.DomainDataSources == nil { - snap.DomainDataSources = make(map[string]map[string]*DataSource) - } + restoreDirtyTables(b, dryRunDTOs, autoTuneDTOs, dataSourceDTOs, domainIndexDTOs) - if snap.DirectQueryDataSources == nil { - snap.DirectQueryDataSources = make(map[string]*DirectQueryDataSource) - } + restoreRawMaps(b, &snap) - if snap.PackageAssociations == nil { - snap.PackageAssociations = make(map[string]map[string]bool) - } + b.accountID = snap.AccountID + b.region = snap.Region + b.appIDCounter = snap.AppIDCounter + b.connCounter = snap.ConnCounter + b.vpcEndpointCounter = snap.VpcEndpointCounter + b.packageCounter = snap.PackageCounter + b.maintenanceCounter = snap.MaintenanceCounter + b.reservedCounter = snap.ReservedCounter + b.slCollCounter = snap.SlCollCounter + b.slSecConfigCounter = snap.SlSecConfigCounter - if snap.VpcAuthorizations == nil { - snap.VpcAuthorizations = make(map[string][]AuthorizedPrincipal) - } + fixNilDomainTags(b) - if snap.Applications == nil { - snap.Applications = make(map[string]*Application) - } + return nil } -// ensureNonNilExtendedMaps initialises nil extended maps in the snapshot. -func ensureNonNilExtendedMaps(snap *backendSnapshot) { - if snap.VpcEndpoints == nil { - snap.VpcEndpoints = make(map[string]*VpcEndpoint) - } - - if snap.Packages == nil { - snap.Packages = make(map[string]*Package) +// restoreDirtyTables converts each DTO table's contents back to its live +// value type and loads the corresponding live b.
via Table.Restore, +// split out from Restore to keep its growth in check. +func restoreDirtyTables( + b *InMemoryBackend, + dryRunDTOs *store.Table[dryRunSnapshot], + autoTuneDTOs *store.Table[autoTuneSnapshot], + dataSourceDTOs *store.Table[dataSourceSnapshot], + domainIndexDTOs *store.Table[domainIndexSnapshot], +) { + dryRuns := make([]*DryRunStatus, 0, dryRunDTOs.Len()) + for _, v := range dryRunDTOs.All() { + dryRuns = append(dryRuns, fromDryRunSnapshot(v)) } + b.dryRuns.Restore(dryRuns) - if snap.ScheduledActions == nil { - snap.ScheduledActions = make(map[string][]*ScheduledAction) + autoTunes := make([]*AutoTuneConfig, 0, autoTuneDTOs.Len()) + for _, v := range autoTuneDTOs.All() { + autoTunes = append(autoTunes, fromAutoTuneSnapshot(v)) } + b.autoTunes.Restore(autoTunes) - if snap.ReservedInstances == nil { - snap.ReservedInstances = make(map[string]*ReservedInstance) + dataSources := make([]*DataSource, 0, dataSourceDTOs.Len()) + for _, v := range dataSourceDTOs.All() { + dataSources = append(dataSources, fromDataSourceSnapshot(v)) } + b.domainDataSources.Restore(dataSources) - if snap.DomainMaintenances == nil { - snap.DomainMaintenances = make(map[string][]*DomainMaintenance) + domainIndexes := make([]*DomainIndex, 0, domainIndexDTOs.Len()) + for _, v := range domainIndexDTOs.All() { + domainIndexes = append(domainIndexes, fromDomainIndexSnapshot(v)) } + b.domainIndexes.Restore(domainIndexes) +} - if snap.DomainIndexes == nil { - snap.DomainIndexes = make(map[string]map[string]*DomainIndex) +// restoreRawMaps restores the plain-map fields left unconverted by the +// pkgs/store migration (see store_setup.go's registerAllTables doc), +// defaulting each to an empty map when absent from the snapshot. +func restoreRawMaps(b *InMemoryBackend, snap *backendSnapshot) { + if snap.VpcAuthorizations != nil { + b.vpcAuthorizations = snap.VpcAuthorizations + } else { + b.vpcAuthorizations = make(map[string][]AuthorizedPrincipal) } - if snap.UpgradeHistory == nil { - snap.UpgradeHistory = make(map[string][]*UpgradeHistory) + if snap.ScheduledActions != nil { + b.scheduledActions = snap.ScheduledActions + } else { + b.scheduledActions = make(map[string][]*ScheduledAction) } - if snap.AutoTunes == nil { - snap.AutoTunes = make(map[string]*AutoTuneConfig) + if snap.PackageAssociations != nil { + b.packageAssociations = snap.PackageAssociations + } else { + b.packageAssociations = make(map[string]map[string]bool) } - if snap.SlCollections == nil { - snap.SlCollections = make(map[string]*ServerlessCollection) + if snap.DomainMaintenances != nil { + b.domainMaintenances = snap.DomainMaintenances + } else { + b.domainMaintenances = make(map[string][]*DomainMaintenance) } - if snap.SlAccessPolicies == nil { - snap.SlAccessPolicies = make(map[string]*ServerlessAccessPolicy) + if snap.UpgradeHistory != nil { + b.upgradeHistory = snap.UpgradeHistory + } else { + b.upgradeHistory = make(map[string][]*UpgradeHistory) } - if snap.SlSecurityConfigs == nil { - snap.SlSecurityConfigs = make(map[string]*ServerlessSecurityConfig) - } + // domainPackages is deliberately left untouched here -- see backendSnapshot's + // doc comment: the pre-existing backend never persisted or restored this + // reverse index either, and that asymmetry is preserved byte-for-byte. - if snap.SlEncryptionPolicies == nil { - snap.SlEncryptionPolicies = make(map[string]*ServerlessEncryptionPolicy) - } - - if snap.SlNetworkPolicies == nil { - snap.SlNetworkPolicies = make(map[string]*ServerlessNetworkPolicy) + if snap.DefaultAppSettings != nil { + b.defaultAppSettings = snap.DefaultAppSettings + } else { + b.defaultAppSettings = make(map[string][]AppSetting) } } -// fixNilDomainTags ensures that every restored domain has a valid Tags instance. -// When a domain is round-tripped through JSON the tags.Tags.UnmarshalJSON creates -// a new underlying safemap, but if the JSON value was null we still need a usable Tags. -func fixNilDomainTags(snap *backendSnapshot) { - for name, d := range snap.Domains { +// fixNilDomainTags ensures that every restored domain has a valid Tags +// instance. When a domain is round-tripped through JSON, tags.Tags's +// UnmarshalJSON creates a new underlying safemap, but if the JSON value was +// null we still need a usable Tags. +func fixNilDomainTags(b *InMemoryBackend) { + for _, d := range b.domains.All() { if d != nil && d.Tags == nil { - d.Tags = tags.New("opensearch." + name + ".tags") + d.Tags = tags.New("opensearch." + d.Name + ".tags") } } } -// rebuildARNIndex reconstructs the arnIndex map from a set of restored domains. -func rebuildARNIndex(domains map[string]*Domain, accountID, region string) map[string]string { - idx := make(map[string]string, len(domains)) - - for name := range domains { - domainARN := arn.Build("es", region, accountID, "domain/"+name) - idx[domainARN] = name - } - - return idx -} - // Snapshot implements persistence.Persistable by delegating to the backend. func (h *Handler) Snapshot(ctx context.Context) []byte { return h.Backend.Snapshot(ctx) diff --git a/services/opensearch/persistence_test.go b/services/opensearch/persistence_test.go index aa318f679..40a199178 100644 --- a/services/opensearch/persistence_test.go +++ b/services/opensearch/persistence_test.go @@ -106,6 +106,173 @@ func TestOpenSearchHandler_Persistence(t *testing.T) { assert.Equal(t, "snap-domain", domain.Name) } +// TestInMemoryBackend_SnapshotRestore_FullState exercises a Snapshot->Restore +// round trip across every resource family, including the pkgs/store "clean" +// tables, the "dirty" DTO tables (dryRuns, autoTunes, domainDataSources, +// domainIndexes), and the plain maps left unconverted (part of the Phase 3.3 +// datalayer conversion -- see store_setup.go). It guards against silently +// losing a family in Snapshot/Restore's field wiring. +func TestInMemoryBackend_SnapshotRestore_FullState(t *testing.T) { + t.Parallel() + + original := opensearch.NewInMemoryBackend("123456789012", "us-east-1") + + domain, err := original.CreateDomain(opensearch.CreateDomainInput{ + Name: "full-domain", + EngineVersion: "OpenSearch_2.11", + ClusterConfig: opensearch.ClusterConfig{InstanceType: "t3.small.search", InstanceCount: 1}, + }) + require.NoError(t, err) + + // "Dirty" DTO tables: domainDataSources, domainIndexes, dryRuns, autoTunes. + _, err = original.AddDataSource(domain.Name, "ds-1", "a data source", "S3") + require.NoError(t, err) + + _, err = original.CreateIndex(domain.Name, "idx-1", nil, nil, nil) + require.NoError(t, err) + + _, err = original.GetDryRunProgress(domain.Name) + require.NoError(t, err) + + require.NoError(t, original.SetAutoTune(domain.Name, "ENABLED", nil)) + + // "Clean" pkgs/store tables not otherwise covered by other persistence tests. + _, err = original.AddDirectQueryDataSource("dq-1", "a direct query source", "spark", nil) + require.NoError(t, err) + + _, err = original.CreateOutboundConnection("peer-alias", nil, nil) + require.NoError(t, err) + + opensearch.SeedInboundConnection(original, "conn-in-1") + + _, err = original.CreateVpcEndpoint(domain.ARN, nil) + require.NoError(t, err) + + app, err := original.CreateApplication("full-app", nil, nil) + require.NoError(t, err) + + pkg, err := original.CreatePackage("full-pkg", "TXT-DICTIONARY", "a package", nil, nil) + require.NoError(t, err) + + _, err = original.PurchaseReservedInstanceOffering("ri-offering-1", "full-ri", 1) + require.NoError(t, err) + + _, err = original.CreateServerlessCollection("full-coll", "SEARCH", "a collection", "", nil) + require.NoError(t, err) + + _, err = original.CreateServerlessAccessPolicy("data", "full-ap", "an access policy", "{}") + require.NoError(t, err) + + _, err = original.CreateServerlessSecurityConfig("saml", "a security config", nil) + require.NoError(t, err) + + _, err = original.CreateServerlessEncryptionPolicy("encryption", "full-ep", "an encryption policy", "{}") + require.NoError(t, err) + + _, err = original.CreateServerlessNetworkPolicy("network", "full-np", "a network policy", "{}") + require.NoError(t, err) + + // Plain maps left unconverted (see store_setup.go's registerAllTables doc). + _, err = original.AuthorizeVpcEndpointAccess(domain.Name, "123456789012", "") + require.NoError(t, err) + + _, err = original.UpdateScheduledAction(domain.Name, &opensearch.ScheduledAction{ID: "sa-1"}) + require.NoError(t, err) + + _, err = original.AssociatePackage(pkg.PackageID, domain.Name) + require.NoError(t, err) + + _, err = original.StartDomainMaintenance(domain.Name, "REBOOT_NODE", "") + require.NoError(t, err) + + require.NoError(t, original.UpgradeDomain(domain.Name, "OpenSearch_2.15")) + + require.NoError(t, original.PutDefaultApplicationSettings("OBSERVABILITY_ANALYTICS", []opensearch.AppSetting{ + {Key: "k", Value: "v"}, + })) + + snap := original.Snapshot(t.Context()) + require.NotNil(t, snap) + + fresh := opensearch.NewInMemoryBackend("123456789012", "us-east-1") + require.NoError(t, fresh.Restore(t.Context(), snap)) + + // Verify every family actually round-tripped. + gotDomain, err := fresh.DescribeDomain(domain.Name) + require.NoError(t, err) + assert.Equal(t, domain.EngineVersion, gotDomain.EngineVersion) + + dataSources, err := fresh.ListDataSources(domain.Name) + require.NoError(t, err) + assert.Len(t, dataSources, 1) + + idx, err := fresh.GetIndex(domain.Name, "idx-1") + require.NoError(t, err) + assert.Equal(t, "idx-1", idx.IndexName) + + dr, err := fresh.GetDryRunProgress(domain.Name) + require.NoError(t, err) + assert.NotEmpty(t, dr.DryRunID) + + autoTune, err := fresh.GetAutoTune(domain.Name) + require.NoError(t, err) + assert.NotEmpty(t, autoTune) + + dqSources := fresh.ListDirectQueryDataSources() + assert.Len(t, dqSources, 1) + + outbound := fresh.DescribeOutboundConnections() + assert.Len(t, outbound, 1) + + inbound := fresh.DescribeInboundConnections() + assert.Len(t, inbound, 1) + + endpoints := fresh.ListVpcEndpoints() + assert.Len(t, endpoints, 1) + + apps := fresh.ListApplications() + require.Len(t, apps, 1) + assert.Equal(t, app.Name, apps[0].Name) + + pkgs, err := fresh.DescribePackages(nil) + require.NoError(t, err) + require.Len(t, pkgs, 1) + assert.Equal(t, pkg.PackageName, pkgs[0].PackageName) + + reserved := fresh.DescribeReservedInstances() + assert.Len(t, reserved, 1) + + collections := fresh.BatchGetServerlessCollections(nil, nil) + assert.Len(t, collections, 1) + + assert.Len(t, fresh.ListServerlessAccessPolicies(""), 1) + assert.Len(t, fresh.ListServerlessSecurityConfigs(""), 1) + assert.Len(t, fresh.ListServerlessEncryptionPolicies(""), 1) + assert.Len(t, fresh.ListServerlessNetworkPolicies(""), 1) + + principals, err := fresh.ListVpcEndpointAccess(domain.Name) + require.NoError(t, err) + assert.Len(t, principals, 1) + + actions := fresh.ListScheduledActions(domain.Name) + assert.Len(t, actions, 1) + + assert.Equal(t, []string{domain.Name}, fresh.ListDomainsForPackage(pkg.PackageID)) + + maintenances, err := fresh.ListDomainMaintenances(domain.Name) + require.NoError(t, err) + assert.Len(t, maintenances, 1) + + history, err := fresh.GetUpgradeHistory(domain.Name) + require.NoError(t, err) + assert.Len(t, history, 1) + + settings, err := fresh.GetDefaultApplicationSettings("OBSERVABILITY_ANALYTICS") + require.NoError(t, err) + require.Len(t, settings, 1) + assert.Equal(t, "v", settings[0].Value) +} + func TestOpenSearchHandler_Routing(t *testing.T) { t.Parallel() diff --git a/services/opensearch/store_setup.go b/services/opensearch/store_setup.go new file mode 100644 index 000000000..ab6f216c5 --- /dev/null +++ b/services/opensearch/store_setup.go @@ -0,0 +1,209 @@ +package opensearch + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend is registered exactly once, +// here, as a *store.Table[T] on b.registry. See pkgs/store's package doc and +// the services/ec2 (commit 12e611a4) / services/sqs (commit 0f09d77c) / +// services/apigateway (commit 6da0334e) conversions this follows. +// +// Two resource families (domainDataSources: domainName -> dataSourceName -> +// *DataSource, and domainIndexes: domainName -> indexName -> *DomainIndex) +// were previously nested per-domain maps. store.Table has no notion of +// nesting, so each becomes a single flat table keyed by a composite +// "#" string, with a secondary [store.Index] grouping by +// domain name for the "all X of domain Y" lookups the nested maps used to +// answer directly. This requires the child value type to carry its parent's +// domain name as a field purely for identity/keying (never on the wire) -- +// DataSource and DomainIndex gained a `DomainName string `json:"-"`` field for +// exactly this reason. +// +// Two more single-value-per-domain maps (dryRuns: domainName -> *DryRunStatus, +// autoTunes: "autotune:"+domainName -> *AutoTuneConfig) have the same problem +// in miniature: the value type carried no field of its own to key a Table by, +// so DryRunStatus and AutoTuneConfig also gained a `DomainName string +// `json:"-"`` field. +// +// A handful of fields are deliberately NOT registered here and remain plain +// maps -- see registerAllTables's doc for the full list and why. +import "github.com/blackbirdworks/gopherstack/pkgs/store" + +func domainKeyFn(v *Domain) string { return v.Name } +func domainARNKeyFn(v *Domain) string { return v.ARN } + +func inboundConnectionKeyFn(v *InboundConnection) string { return v.ConnectionID } +func outboundConnectionKeyFn(v *OutboundConnection) string { return v.ConnectionID } + +// dataSourceKey builds the composite "#" key shared by +// every data source nested under a domain. Access sites use this directly so +// the exact same key is used for Put/Get/Delete. +func dataSourceKey(domainName, name string) string { return domainName + "#" + name } +func dataSourceKeyFn(v *DataSource) string { return dataSourceKey(v.DomainName, v.Name) } + +func directQueryDataSourceKeyFn(v *DirectQueryDataSource) string { return v.Name } + +func vpcEndpointKeyFn(v *VpcEndpoint) string { return v.VpcEndpointID } + +func applicationKeyFn(v *Application) string { return v.ID } +func applicationNameKeyFn(v *Application) string { return v.Name } + +func packageKeyFn(v *Package) string { return v.PackageID } + +func reservedInstanceKeyFn(v *ReservedInstance) string { return v.ReservedInstanceID } + +// domainIndexKey builds the composite "#" key shared +// by every index nested under a domain. +func domainIndexKey(domainName, indexName string) string { return domainName + "#" + indexName } +func domainIndexKeyFn(v *DomainIndex) string { + return domainIndexKey(v.DomainName, v.IndexName) +} + +func dryRunKeyFn(v *DryRunStatus) string { return v.DomainName } + +func autoTuneConfigKeyFn(v *AutoTuneConfig) string { return autoTuneKey(v.DomainName) } + +// slCollectionKeyFn/slAccessPolicyKeyFn/... reuse the serverless*Key helpers +// (defined in backend_serverless.go, next to the other serverless +// constructors) so the table key matches exactly what every access site +// already computes. Unlike the apigateway "dirty" families, every serverless +// value type already carries the fields (Name, Type, ID) its key is built +// from, so these are all "clean" tables -- no identity field needed. +func slCollectionKeyFn(v *ServerlessCollection) string { return serverlessCollectionKey(v.Name) } + +func slAccessPolicyKeyFn(v *ServerlessAccessPolicy) string { + return serverlessAccessPolicyKey(v.Type, v.Name) +} + +func slSecurityConfigKeyFn(v *ServerlessSecurityConfig) string { + return serverlessSecurityConfigKey(v.ID) +} + +func slEncryptionPolicyKeyFn(v *ServerlessEncryptionPolicy) string { + return serverlessEncryptionPolicyKey(v.Type, v.Name) +} + +func slNetworkPolicyKeyFn(v *ServerlessNetworkPolicy) string { + return serverlessNetworkPolicyKey(v.Type, v.Name) +} + +// registerAllTables sets up every converted resource collection exactly once, +// at construction time. +// +// Two groups, split by whether their store.Table can be snapshotted directly: +// +// - "Clean" tables (domains, inboundConnections, outboundConnections, +// directQueryDataSources, vpcEndpoints, applications, packages, +// reservedInstances, slCollections, slAccessPolicies, slSecurityConfigs, +// slEncryptionPolicies, slNetworkPolicies) are registered on b.registry via +// store.Register, so persistence.go's Snapshot/Restore can drive them +// through b.registry.SnapshotAll()/RestoreAll() -- their value types +// marshal to JSON with no information loss. (Package.VersionHistory +// carries a pre-existing `json:"-"` tag of its own, unrelated to this +// conversion, so it was already excluded from persistence before this +// change and remains so -- not a regression, just an existing quirk this +// conversion preserves byte-for-byte.) +// - "Dirty" tables (dryRuns, autoTunes, domainDataSources, domainIndexes) +// are built with store.New but deliberately NOT registered on b.registry. +// Their key depends on a field (DomainName) tagged `json:"-"` on the live +// type, so a direct json.Marshal/Unmarshal round trip through +// Table.Snapshot/Restore would silently drop that field and corrupt the +// table's key on restore. persistence.go instead builds a throwaway DTO +// [store.Registry] purely to get a correctly-tagged JSON encoding (mirrors +// the services/sqs pilot, commit 0f09d77c, and services/apigateway, +// commit 6da0334e), then restores the live tables directly via +// Table.Restore. Because they aren't registered on b.registry, +// InMemoryBackend.Reset resets each of them with an explicit Table.Reset() +// call alongside b.registry.ResetAll(). +// +// The following fields are deliberately plain maps, not store.Table at all: +// - vpcAuthorizations (domainName -> []AuthorizedPrincipal), +// scheduledActions (domainName -> []*ScheduledAction), domainMaintenances +// (domainName -> []*DomainMaintenance), upgradeHistory +// ("upgrade:"+domainName -> []*UpgradeHistory), and defaultAppSettings +// (applicationType -> []AppSetting) are all slice-valued maps: the +// resource collection for a given key is the *whole ordered slice* +// (append-and-trim-to-cap, or replace-wholesale), not a keyed set of +// individually addressable *T values, so they don't fit store.Table's +// keyed-by-identity shape. +// - packageAssociations (packageID -> domainName -> bool) and +// domainPackages (domainName -> packageID -> bool) are boolean-set +// indexes recording a many-to-many association, not map[string]*T +// resource collections -- the value has no identity of its own to key a +// Table by, the same class of exception as services/apigateway's +// usageOverrides / services/ec2's instanceIMDSOptions. +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call (plus any secondary store.AddIndex) to the concrete field and value +// type. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.domains = store.Register(b.registry, "domains", store.New(domainKeyFn)) + b.domainsByARN = b.domains.AddIndex("byARN", domainARNKeyFn) + }, + func(b *InMemoryBackend) { + b.inboundConnections = store.Register(b.registry, "inboundConnections", store.New(inboundConnectionKeyFn)) + }, + func(b *InMemoryBackend) { + b.outboundConnections = store.Register(b.registry, "outboundConnections", store.New(outboundConnectionKeyFn)) + }, + func(b *InMemoryBackend) { + b.directQueryDataSources = store.Register( + b.registry, "directQueryDataSources", store.New(directQueryDataSourceKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.vpcEndpoints = store.Register(b.registry, "vpcEndpoints", store.New(vpcEndpointKeyFn)) + }, + func(b *InMemoryBackend) { + b.applications = store.Register(b.registry, "applications", store.New(applicationKeyFn)) + b.applicationsByName = b.applications.AddIndex("byName", applicationNameKeyFn) + }, + func(b *InMemoryBackend) { + b.packages = store.Register(b.registry, "packages", store.New(packageKeyFn)) + }, + func(b *InMemoryBackend) { + b.reservedInstances = store.Register(b.registry, "reservedInstances", store.New(reservedInstanceKeyFn)) + }, + func(b *InMemoryBackend) { + b.slCollections = store.Register(b.registry, "slCollections", store.New(slCollectionKeyFn)) + }, + func(b *InMemoryBackend) { + b.slAccessPolicies = store.Register(b.registry, "slAccessPolicies", store.New(slAccessPolicyKeyFn)) + }, + func(b *InMemoryBackend) { + b.slSecurityConfigs = store.Register(b.registry, "slSecurityConfigs", store.New(slSecurityConfigKeyFn)) + }, + func(b *InMemoryBackend) { + b.slEncryptionPolicies = store.Register(b.registry, "slEncryptionPolicies", store.New(slEncryptionPolicyKeyFn)) + }, + func(b *InMemoryBackend) { + b.slNetworkPolicies = store.Register(b.registry, "slNetworkPolicies", store.New(slNetworkPolicyKeyFn)) + }, + // "Dirty" tables: store.New only, NOT store.Register -- see + // registerAllTables doc. + func(b *InMemoryBackend) { + b.dryRuns = store.New(dryRunKeyFn) + }, + func(b *InMemoryBackend) { + b.autoTunes = store.New(autoTuneConfigKeyFn) + }, + func(b *InMemoryBackend) { + b.domainDataSources = store.New(dataSourceKeyFn) + b.domainDataSourcesByDomain = b.domainDataSources.AddIndex( + "byDomain", func(v *DataSource) string { return v.DomainName }, + ) + }, + func(b *InMemoryBackend) { + b.domainIndexes = store.New(domainIndexKeyFn) + b.domainIndexesByDomain = b.domainIndexes.AddIndex( + "byDomain", func(v *DomainIndex) string { return v.DomainName }, + ) + }, +} diff --git a/services/pinpoint/backend.go b/services/pinpoint/backend.go index 4bd55ff09..0226ee4c0 100644 --- a/services/pinpoint/backend.go +++ b/services/pinpoint/backend.go @@ -11,6 +11,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awserr" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) var ( @@ -55,22 +56,23 @@ type storedAppSettings struct { // InMemoryBackend is the in-memory implementation of StorageBackend. type InMemoryBackend struct { - inAppTemplates map[string]*InAppTemplate - segments map[string]*Segment - campaigns map[string]*Campaign - emailTemplates map[string]*EmailTemplate - exportJobs map[string]*ExportJob - importJobs map[string]*ImportJob + registry *store.Registry + inAppTemplates *store.Table[InAppTemplate] + segments *store.Table[Segment] + campaigns *store.Table[Campaign] + emailTemplates *store.Table[EmailTemplate] + exportJobs *store.Table[ExportJob] + importJobs *store.Table[ImportJob] arnIndex map[string]tagHolder - pushTemplates map[string]*PushTemplate - apps map[string]*App - recommenders map[string]*RecommenderConfiguration - journeys map[string]*Journey - smsTemplates map[string]*SmsTemplate - voiceTemplates map[string]*VoiceTemplate - endpoints map[string]*Endpoint - eventStreams map[string]*EventStream - channels map[string]*Channel + pushTemplates *store.Table[PushTemplate] + apps *store.Table[App] + recommenders *store.Table[RecommenderConfiguration] + journeys *store.Table[Journey] + smsTemplates *store.Table[SmsTemplate] + voiceTemplates *store.Table[VoiceTemplate] + endpoints *store.Table[Endpoint] + eventStreams *store.Table[EventStream] + channels *store.Table[Channel] appSettings map[string]*storedAppSettings campaignVersions map[string][]*Campaign segmentVersions map[string][]*Segment @@ -87,26 +89,12 @@ type InMemoryBackend struct { // NewInMemoryBackend creates a new Pinpoint in-memory backend. func NewInMemoryBackend(region, accountID string) *InMemoryBackend { - return &InMemoryBackend{ + b := &InMemoryBackend{ region: region, accountID: accountID, mu: lockmetrics.New("pinpoint"), - apps: make(map[string]*App), + registry: store.NewRegistry(), arnIndex: make(map[string]tagHolder), - campaigns: make(map[string]*Campaign), - channels: make(map[string]*Channel), - emailTemplates: make(map[string]*EmailTemplate), - endpoints: make(map[string]*Endpoint), - eventStreams: make(map[string]*EventStream), - exportJobs: make(map[string]*ExportJob), - importJobs: make(map[string]*ImportJob), - inAppTemplates: make(map[string]*InAppTemplate), - journeys: make(map[string]*Journey), - pushTemplates: make(map[string]*PushTemplate), - recommenders: make(map[string]*RecommenderConfiguration), - segments: make(map[string]*Segment), - smsTemplates: make(map[string]*SmsTemplate), - voiceTemplates: make(map[string]*VoiceTemplate), appSettings: make(map[string]*storedAppSettings), campaignVersions: make(map[string][]*Campaign), segmentVersions: make(map[string][]*Segment), @@ -117,6 +105,10 @@ func NewInMemoryBackend(region, accountID string) *InMemoryBackend { sentMessages: make(map[string]int), otpCodes: make(map[string]string), } + + registerAllTables(b) + + return b } // Reset clears all stored state and returns the backend to a pristine state. @@ -124,22 +116,9 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.apps = make(map[string]*App) + b.registry.ResetAll() + b.arnIndex = make(map[string]tagHolder) - b.campaigns = make(map[string]*Campaign) - b.channels = make(map[string]*Channel) - b.emailTemplates = make(map[string]*EmailTemplate) - b.endpoints = make(map[string]*Endpoint) - b.eventStreams = make(map[string]*EventStream) - b.exportJobs = make(map[string]*ExportJob) - b.importJobs = make(map[string]*ImportJob) - b.inAppTemplates = make(map[string]*InAppTemplate) - b.journeys = make(map[string]*Journey) - b.pushTemplates = make(map[string]*PushTemplate) - b.recommenders = make(map[string]*RecommenderConfiguration) - b.segments = make(map[string]*Segment) - b.smsTemplates = make(map[string]*SmsTemplate) - b.voiceTemplates = make(map[string]*VoiceTemplate) b.appSettings = make(map[string]*storedAppSettings) b.campaignVersions = make(map[string][]*Campaign) b.segmentVersions = make(map[string][]*Segment) @@ -166,7 +145,7 @@ func (b *InMemoryBackend) AddAppInternal(app *App) { b.mu.Lock("AddAppInternal") defer b.mu.Unlock() - b.apps[app.ID] = app + b.apps.Put(app) b.arnIndex[app.ARN] = app } @@ -175,7 +154,7 @@ func (b *InMemoryBackend) AddCampaignInternal(c *Campaign) { b.mu.Lock("AddCampaignInternal") defer b.mu.Unlock() - b.campaigns[c.ID] = c + b.campaigns.Put(c) b.arnIndex[c.ARN] = c } @@ -184,7 +163,7 @@ func (b *InMemoryBackend) AddSegmentInternal(s *Segment) { b.mu.Lock("AddSegmentInternal") defer b.mu.Unlock() - b.segments[s.ID] = s + b.segments.Put(s) b.arnIndex[s.ARN] = s } @@ -193,7 +172,7 @@ func (b *InMemoryBackend) AddJourneyInternal(j *Journey) { b.mu.Lock("AddJourneyInternal") defer b.mu.Unlock() - b.journeys[j.ID] = j + b.journeys.Put(j) b.arnIndex[j.ARN] = j } @@ -217,7 +196,7 @@ func (b *InMemoryBackend) CreateApp(region, accountID, name string, tags map[str CreationDate: nowRFC3339(), } - b.apps[appID] = app + b.apps.Put(app) b.arnIndex[appARN] = app return cloneApp(app), nil @@ -228,7 +207,7 @@ func (b *InMemoryBackend) GetApp(appID string) (*App, error) { b.mu.RLock("GetApp") defer b.mu.RUnlock() - app, ok := b.apps[appID] + app, ok := b.apps.Get(appID) if !ok { return nil, ErrAppNotFound } @@ -241,12 +220,12 @@ func (b *InMemoryBackend) DeleteApp(appID string) (*App, error) { b.mu.Lock("DeleteApp") defer b.mu.Unlock() - app, ok := b.apps[appID] + app, ok := b.apps.Get(appID) if !ok { return nil, ErrAppNotFound } - delete(b.apps, appID) + b.apps.Delete(appID) delete(b.arnIndex, app.ARN) b.purgeAppStateLocked(appID) @@ -266,32 +245,48 @@ func (b *InMemoryBackend) DeleteApp(appID string) (*App, error) { // ApplicationID, so they are filtered by that field. func (b *InMemoryBackend) purgeAppStateLocked(appID string) { delete(b.appEvents, appID) - delete(b.eventStreams, appID) + b.eventStreams.Delete(appID) delete(b.otpCodes, appID) delete(b.appSettings, appID) delete(b.sentMessages, appID) prefix := appID + "/" - deletePrefixed(b.endpoints, prefix) - deletePrefixed(b.channels, prefix) deletePrefixed(b.campaignVersions, prefix) deletePrefixed(b.segmentVersions, prefix) deletePrefixed(b.campaignActivities, prefix) deletePrefixed(b.journeyRuns, prefix) - for id, c := range b.campaigns { - if c != nil && c.ApplicationID == appID { - delete(b.campaigns, id) - } - } - for id, s := range b.segments { - if s != nil && s.ApplicationID == appID { - delete(b.segments, id) - } - } - for id, j := range b.journeys { - if j != nil && j.ApplicationID == appID { - delete(b.journeys, id) + purgeTableByAppID(b.endpoints, appID, + func(e *Endpoint) string { return e.ApplicationID }, + func(e *Endpoint) string { return prefix + e.ID }, + ) + purgeTableByAppID(b.channels, appID, + func(c *Channel) string { return c.ApplicationID }, + func(c *Channel) string { return prefix + c.ChannelType }, + ) + purgeTableByAppID(b.campaigns, appID, + func(c *Campaign) string { return c.ApplicationID }, + func(c *Campaign) string { return c.ID }, + ) + purgeTableByAppID(b.segments, appID, + func(s *Segment) string { return s.ApplicationID }, + func(s *Segment) string { return s.ID }, + ) + purgeTableByAppID(b.journeys, appID, + func(j *Journey) string { return j.ApplicationID }, + func(j *Journey) string { return j.ID }, + ) +} + +// purgeTableByAppID deletes every value in t whose owning application (per +// appIDOf) matches appID, using keyOf to recover each value's own table key. +// Factoring the scan+filter+delete loop out per resource kind keeps +// purgeAppStateLocked a flat list of calls instead of one branch per +// app-scoped table, which is what keeps its cyclomatic complexity in check. +func purgeTableByAppID[V any](t *store.Table[V], appID string, appIDOf, keyOf func(*V) string) { + for _, v := range t.All() { + if v != nil && appIDOf(v) == appID { + t.Delete(keyOf(v)) } } } @@ -310,9 +305,10 @@ func (b *InMemoryBackend) GetApps() ([]*App, error) { b.mu.RLock("GetApps") defer b.mu.RUnlock() - apps := make([]*App, 0, len(b.apps)) + all := b.apps.All() + apps := make([]*App, 0, len(all)) - for _, app := range b.apps { + for _, app := range all { apps = append(apps, cloneApp(app)) } @@ -392,7 +388,7 @@ func (b *InMemoryBackend) CreateCampaign( b.mu.Lock("CreateCampaign") defer b.mu.Unlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } @@ -436,7 +432,7 @@ func (b *InMemoryBackend) CreateCampaign( } c.Version = 1 - b.campaigns[id] = c + b.campaigns.Put(c) b.arnIndex[campaignARN] = c // Track campaign version history. @@ -464,7 +460,7 @@ func (b *InMemoryBackend) CreateEmailTemplate( b.mu.Lock("CreateEmailTemplate") defer b.mu.Unlock() - if _, exists := b.emailTemplates[templateName]; exists { + if _, exists := b.emailTemplates.Get(templateName); exists { return nil, ErrAlreadyExists } @@ -486,7 +482,7 @@ func (b *InMemoryBackend) CreateEmailTemplate( Version: "1", } - b.emailTemplates[templateName] = t + b.emailTemplates.Put(t) b.arnIndex[templateARN] = t // Track template version history. @@ -506,7 +502,7 @@ func (b *InMemoryBackend) CreateInAppTemplate( b.mu.Lock("CreateInAppTemplate") defer b.mu.Unlock() - if _, exists := b.inAppTemplates[templateName]; exists { + if _, exists := b.inAppTemplates.Get(templateName); exists { return nil, ErrAlreadyExists } @@ -525,7 +521,7 @@ func (b *InMemoryBackend) CreateInAppTemplate( Version: "1", } - b.inAppTemplates[templateName] = t + b.inAppTemplates.Put(t) b.arnIndex[templateARN] = t // Track template version history. @@ -545,7 +541,7 @@ func (b *InMemoryBackend) CreatePushTemplate( b.mu.Lock("CreatePushTemplate") defer b.mu.Unlock() - if _, exists := b.pushTemplates[templateName]; exists { + if _, exists := b.pushTemplates.Get(templateName); exists { return nil, ErrAlreadyExists } @@ -567,7 +563,7 @@ func (b *InMemoryBackend) CreatePushTemplate( Version: "1", } - b.pushTemplates[templateName] = t + b.pushTemplates.Put(t) b.arnIndex[templateARN] = t // Track template version history. @@ -587,7 +583,7 @@ func (b *InMemoryBackend) CreateSmsTemplate( b.mu.Lock("CreateSmsTemplate") defer b.mu.Unlock() - if _, exists := b.smsTemplates[templateName]; exists { + if _, exists := b.smsTemplates.Get(templateName); exists { return nil, ErrAlreadyExists } @@ -606,7 +602,7 @@ func (b *InMemoryBackend) CreateSmsTemplate( Version: "1", } - b.smsTemplates[templateName] = t + b.smsTemplates.Put(t) b.arnIndex[templateARN] = t // Track template version history. @@ -630,7 +626,7 @@ func (b *InMemoryBackend) CreateExportJob( b.mu.Lock("CreateExportJob") defer b.mu.Unlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } @@ -647,7 +643,7 @@ func (b *InMemoryBackend) CreateExportJob( CreationDate: nowRFC3339(), } - b.exportJobs[id] = j + b.exportJobs.Put(j) cp := *j @@ -664,7 +660,7 @@ func (b *InMemoryBackend) CreateImportJob( b.mu.Lock("CreateImportJob") defer b.mu.Unlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } @@ -683,7 +679,7 @@ func (b *InMemoryBackend) CreateImportJob( CreationDate: now, } - b.importJobs[id] = j + b.importJobs.Put(j) // Materialise an IMPORT-type segment so Terraform/clients can look it up. segName := req.SegmentName @@ -713,7 +709,7 @@ func (b *InMemoryBackend) CreateImportJob( Version: 1, } - b.segments[segID] = seg + b.segments.Put(seg) b.arnIndex[segARN] = seg versionKey := appID + "/" + segID @@ -735,7 +731,7 @@ func (b *InMemoryBackend) CreateJourney(region, accountID, appID string, req cre b.mu.Lock("CreateJourney") defer b.mu.Unlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } @@ -772,7 +768,7 @@ func (b *InMemoryBackend) CreateJourney(region, accountID, appID string, req cre } } - b.journeys[id] = j + b.journeys.Put(j) b.arnIndex[journeyARN] = j return cloneJourney(j), nil @@ -809,7 +805,7 @@ func (b *InMemoryBackend) CreateRecommenderConfiguration( LastModifiedDate: now, } - b.recommenders[id] = r + b.recommenders.Put(r) cp := *r cp.Attributes = nonNilAttrsCopy(r.Attributes) @@ -826,7 +822,7 @@ func (b *InMemoryBackend) CreateSegment(region, accountID, appID string, req cre b.mu.Lock("CreateSegment") defer b.mu.Unlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } @@ -855,7 +851,7 @@ func (b *InMemoryBackend) CreateSegment(region, accountID, appID string, req cre } s.Version = 1 - b.segments[id] = s + b.segments.Put(s) b.arnIndex[segmentARN] = s // Track segment version history. diff --git a/services/pinpoint/backend_ops.go b/services/pinpoint/backend_ops.go index db9aaa715..6c89123d0 100644 --- a/services/pinpoint/backend_ops.go +++ b/services/pinpoint/backend_ops.go @@ -155,7 +155,7 @@ func (b *InMemoryBackend) CreateVoiceTemplate( b.mu.Lock("CreateVoiceTemplate") defer b.mu.Unlock() - if _, exists := b.voiceTemplates[templateName]; exists { + if _, exists := b.voiceTemplates.Get(templateName); exists { return nil, ErrAlreadyExists } @@ -174,7 +174,7 @@ func (b *InMemoryBackend) CreateVoiceTemplate( CreationDate: nowRFC3339(), } - b.voiceTemplates[templateName] = t + b.voiceTemplates.Put(t) // Track template version history. versionKey := templateName + "/VOICE" @@ -193,7 +193,7 @@ func (b *InMemoryBackend) GetVoiceTemplate(templateName string) (*VoiceTemplate, b.mu.RLock("GetVoiceTemplate") defer b.mu.RUnlock() - t, ok := b.voiceTemplates[templateName] + t, ok := b.voiceTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } @@ -212,7 +212,7 @@ func (b *InMemoryBackend) UpdateVoiceTemplate( b.mu.Lock("UpdateVoiceTemplate") defer b.mu.Unlock() - t, ok := b.voiceTemplates[templateName] + t, ok := b.voiceTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } @@ -251,12 +251,12 @@ func (b *InMemoryBackend) DeleteVoiceTemplate(templateName string) (*VoiceTempla b.mu.Lock("DeleteVoiceTemplate") defer b.mu.Unlock() - t, ok := b.voiceTemplates[templateName] + t, ok := b.voiceTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } - delete(b.voiceTemplates, templateName) + b.voiceTemplates.Delete(templateName) cp := *t cp.Tags = nonNilTagsCopy(t.Tags) @@ -269,11 +269,11 @@ func (b *InMemoryBackend) ListTemplates() ([]*templateListItem, error) { b.mu.RLock("ListTemplates") defer b.mu.RUnlock() - totalCap := len(b.emailTemplates) + len(b.inAppTemplates) + len(b.pushTemplates) + - len(b.smsTemplates) + len(b.voiceTemplates) + totalCap := b.emailTemplates.Len() + b.inAppTemplates.Len() + b.pushTemplates.Len() + + b.smsTemplates.Len() + b.voiceTemplates.Len() items := make([]*templateListItem, 0, totalCap) - for _, t := range b.emailTemplates { + for _, t := range b.emailTemplates.All() { items = append(items, &templateListItem{ TemplateName: t.TemplateName, TemplateType: ChannelTypeEmail, @@ -282,7 +282,7 @@ func (b *InMemoryBackend) ListTemplates() ([]*templateListItem, error) { }) } - for _, t := range b.inAppTemplates { + for _, t := range b.inAppTemplates.All() { items = append(items, &templateListItem{ TemplateName: t.TemplateName, TemplateType: templateTypeINAPP, @@ -291,7 +291,7 @@ func (b *InMemoryBackend) ListTemplates() ([]*templateListItem, error) { }) } - for _, t := range b.pushTemplates { + for _, t := range b.pushTemplates.All() { items = append(items, &templateListItem{ TemplateName: t.TemplateName, TemplateType: templateTypePUSH, @@ -300,7 +300,7 @@ func (b *InMemoryBackend) ListTemplates() ([]*templateListItem, error) { }) } - for _, t := range b.smsTemplates { + for _, t := range b.smsTemplates.All() { items = append(items, &templateListItem{ TemplateName: t.TemplateName, TemplateType: ChannelTypeSMS, @@ -309,7 +309,7 @@ func (b *InMemoryBackend) ListTemplates() ([]*templateListItem, error) { }) } - for _, t := range b.voiceTemplates { + for _, t := range b.voiceTemplates.All() { items = append(items, &templateListItem{ TemplateName: t.TemplateName, TemplateType: ChannelTypeVoice, @@ -338,7 +338,7 @@ func (b *InMemoryBackend) GetCampaign(appID, campaignID string) (*Campaign, erro b.mu.RLock("GetCampaign") defer b.mu.RUnlock() - c, ok := b.campaigns[campaignID] + c, ok := b.campaigns.Get(campaignID) if !ok || c.ApplicationID != appID { return nil, ErrAppNotFound } @@ -351,13 +351,13 @@ func (b *InMemoryBackend) GetCampaigns(appID string) ([]*Campaign, error) { b.mu.RLock("GetCampaigns") defer b.mu.RUnlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } var campaigns []*Campaign - for _, c := range b.campaigns { + for _, c := range b.campaigns.All() { if c.ApplicationID == appID { campaigns = append(campaigns, cloneCampaign(c)) } @@ -439,7 +439,7 @@ func (b *InMemoryBackend) UpdateCampaign( b.mu.Lock("UpdateCampaign") defer b.mu.Unlock() - c, ok := b.campaigns[campaignID] + c, ok := b.campaigns.Get(campaignID) if !ok || c.ApplicationID != appID { return nil, ErrAppNotFound } @@ -469,12 +469,12 @@ func (b *InMemoryBackend) DeleteCampaign(appID, campaignID string) (*Campaign, e b.mu.Lock("DeleteCampaign") defer b.mu.Unlock() - c, ok := b.campaigns[campaignID] + c, ok := b.campaigns.Get(campaignID) if !ok || c.ApplicationID != appID { return nil, ErrAppNotFound } - delete(b.campaigns, campaignID) + b.campaigns.Delete(campaignID) delete(b.arnIndex, c.ARN) delete(b.campaignVersions, appID+"/"+campaignID) @@ -490,7 +490,7 @@ func (b *InMemoryBackend) GetSegment(appID, segmentID string) (*Segment, error) b.mu.RLock("GetSegment") defer b.mu.RUnlock() - s, ok := b.segments[segmentID] + s, ok := b.segments.Get(segmentID) if !ok || s.ApplicationID != appID { return nil, ErrAppNotFound } @@ -503,13 +503,13 @@ func (b *InMemoryBackend) GetSegments(appID string) ([]*Segment, error) { b.mu.RLock("GetSegments") defer b.mu.RUnlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } var segments []*Segment - for _, s := range b.segments { + for _, s := range b.segments.All() { if s.ApplicationID == appID { segments = append(segments, cloneSegment(s)) } @@ -530,7 +530,7 @@ func (b *InMemoryBackend) UpdateSegment( b.mu.Lock("UpdateSegment") defer b.mu.Unlock() - s, ok := b.segments[segmentID] + s, ok := b.segments.Get(segmentID) if !ok || s.ApplicationID != appID { return nil, ErrAppNotFound } @@ -567,12 +567,12 @@ func (b *InMemoryBackend) DeleteSegment(appID, segmentID string) (*Segment, erro b.mu.Lock("DeleteSegment") defer b.mu.Unlock() - s, ok := b.segments[segmentID] + s, ok := b.segments.Get(segmentID) if !ok || s.ApplicationID != appID { return nil, ErrAppNotFound } - delete(b.segments, segmentID) + b.segments.Delete(segmentID) delete(b.arnIndex, s.ARN) delete(b.segmentVersions, appID+"/"+segmentID) @@ -588,7 +588,7 @@ func (b *InMemoryBackend) GetJourney(appID, journeyID string) (*Journey, error) b.mu.RLock("GetJourney") defer b.mu.RUnlock() - j, ok := b.journeys[journeyID] + j, ok := b.journeys.Get(journeyID) if !ok || j.ApplicationID != appID { return nil, ErrAppNotFound } @@ -601,13 +601,13 @@ func (b *InMemoryBackend) GetJourneys(appID string) ([]*Journey, error) { b.mu.RLock("GetJourneys") defer b.mu.RUnlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } var journeys []*Journey - for _, j := range b.journeys { + for _, j := range b.journeys.All() { if j.ApplicationID == appID { journeys = append(journeys, cloneJourney(j)) } @@ -628,7 +628,7 @@ func (b *InMemoryBackend) UpdateJourney( b.mu.Lock("UpdateJourney") defer b.mu.Unlock() - j, ok := b.journeys[journeyID] + j, ok := b.journeys.Get(journeyID) if !ok || j.ApplicationID != appID { return nil, ErrAppNotFound } @@ -709,7 +709,7 @@ func (b *InMemoryBackend) UpdateJourneyState(appID, journeyID, state string) (*J b.mu.Lock("UpdateJourneyState") defer b.mu.Unlock() - j, ok := b.journeys[journeyID] + j, ok := b.journeys.Get(journeyID) if !ok || j.ApplicationID != appID { return nil, ErrAppNotFound } @@ -744,12 +744,12 @@ func (b *InMemoryBackend) DeleteJourney(appID, journeyID string) (*Journey, erro b.mu.Lock("DeleteJourney") defer b.mu.Unlock() - j, ok := b.journeys[journeyID] + j, ok := b.journeys.Get(journeyID) if !ok || j.ApplicationID != appID { return nil, ErrAppNotFound } - delete(b.journeys, journeyID) + b.journeys.Delete(journeyID) delete(b.arnIndex, j.ARN) return cloneJourney(j), nil @@ -764,7 +764,7 @@ func (b *InMemoryBackend) GetEmailTemplate(templateName string) (*EmailTemplate, b.mu.RLock("GetEmailTemplate") defer b.mu.RUnlock() - t, ok := b.emailTemplates[templateName] + t, ok := b.emailTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } @@ -780,7 +780,7 @@ func (b *InMemoryBackend) UpdateEmailTemplate( b.mu.Lock("UpdateEmailTemplate") defer b.mu.Unlock() - t, ok := b.emailTemplates[templateName] + t, ok := b.emailTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } @@ -835,12 +835,12 @@ func (b *InMemoryBackend) DeleteEmailTemplate(templateName string) (*EmailTempla b.mu.Lock("DeleteEmailTemplate") defer b.mu.Unlock() - t, ok := b.emailTemplates[templateName] + t, ok := b.emailTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } - delete(b.emailTemplates, templateName) + b.emailTemplates.Delete(templateName) delete(b.arnIndex, t.ARN) delete(b.templateVersionHistory, templateName+"/"+ChannelTypeEmail) @@ -852,7 +852,7 @@ func (b *InMemoryBackend) GetInAppTemplate(templateName string) (*InAppTemplate, b.mu.RLock("GetInAppTemplate") defer b.mu.RUnlock() - t, ok := b.inAppTemplates[templateName] + t, ok := b.inAppTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } @@ -868,7 +868,7 @@ func (b *InMemoryBackend) UpdateInAppTemplate( b.mu.Lock("UpdateInAppTemplate") defer b.mu.Unlock() - t, ok := b.inAppTemplates[templateName] + t, ok := b.inAppTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } @@ -911,12 +911,12 @@ func (b *InMemoryBackend) DeleteInAppTemplate(templateName string) (*InAppTempla b.mu.Lock("DeleteInAppTemplate") defer b.mu.Unlock() - t, ok := b.inAppTemplates[templateName] + t, ok := b.inAppTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } - delete(b.inAppTemplates, templateName) + b.inAppTemplates.Delete(templateName) delete(b.arnIndex, t.ARN) delete(b.templateVersionHistory, templateName+"/"+templateTypeINAPP) @@ -928,7 +928,7 @@ func (b *InMemoryBackend) GetPushTemplate(templateName string) (*PushTemplate, e b.mu.RLock("GetPushTemplate") defer b.mu.RUnlock() - t, ok := b.pushTemplates[templateName] + t, ok := b.pushTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } @@ -944,7 +944,7 @@ func (b *InMemoryBackend) UpdatePushTemplate( b.mu.Lock("UpdatePushTemplate") defer b.mu.Unlock() - t, ok := b.pushTemplates[templateName] + t, ok := b.pushTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } @@ -999,12 +999,12 @@ func (b *InMemoryBackend) DeletePushTemplate(templateName string) (*PushTemplate b.mu.Lock("DeletePushTemplate") defer b.mu.Unlock() - t, ok := b.pushTemplates[templateName] + t, ok := b.pushTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } - delete(b.pushTemplates, templateName) + b.pushTemplates.Delete(templateName) delete(b.arnIndex, t.ARN) delete(b.templateVersionHistory, templateName+"/"+templateTypePUSH) @@ -1016,7 +1016,7 @@ func (b *InMemoryBackend) GetSmsTemplate(templateName string) (*SmsTemplate, err b.mu.RLock("GetSmsTemplate") defer b.mu.RUnlock() - t, ok := b.smsTemplates[templateName] + t, ok := b.smsTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } @@ -1032,7 +1032,7 @@ func (b *InMemoryBackend) UpdateSmsTemplate( b.mu.Lock("UpdateSmsTemplate") defer b.mu.Unlock() - t, ok := b.smsTemplates[templateName] + t, ok := b.smsTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } @@ -1075,12 +1075,12 @@ func (b *InMemoryBackend) DeleteSmsTemplate(templateName string) (*SmsTemplate, b.mu.Lock("DeleteSmsTemplate") defer b.mu.Unlock() - t, ok := b.smsTemplates[templateName] + t, ok := b.smsTemplates.Get(templateName) if !ok { return nil, ErrAppNotFound } - delete(b.smsTemplates, templateName) + b.smsTemplates.Delete(templateName) delete(b.arnIndex, t.ARN) delete(b.templateVersionHistory, templateName+"/"+ChannelTypeSMS) @@ -1097,7 +1097,7 @@ func (b *InMemoryBackend) GetEndpoint(appID, endpointID string) (*Endpoint, erro defer b.mu.RUnlock() key := appID + "/" + endpointID - e, ok := b.endpoints[key] + e, ok := b.endpoints.Get(key) if !ok { return nil, ErrAppNotFound @@ -1122,14 +1122,14 @@ func (b *InMemoryBackend) UpdateEndpoint( key := appID + "/" + endpointID - e, ok := b.endpoints[key] + e, ok := b.endpoints.Get(key) if !ok { e = &Endpoint{ ApplicationID: appID, ID: endpointID, CreationDate: nowRFC3339(), } - b.endpoints[key] = e + b.endpoints.Put(e) } applyEndpointFields(e, req) @@ -1144,12 +1144,12 @@ func (b *InMemoryBackend) DeleteEndpoint(appID, endpointID string) (*Endpoint, e key := appID + "/" + endpointID - e, ok := b.endpoints[key] + e, ok := b.endpoints.Get(key) if !ok { return nil, ErrAppNotFound } - delete(b.endpoints, key) + b.endpoints.Delete(key) return cloneEndpoint(e), nil } @@ -1161,7 +1161,7 @@ func (b *InMemoryBackend) GetUserEndpoints(appID, userID string) ([]*Endpoint, e var endpoints []*Endpoint - for _, e := range b.endpoints { + for _, e := range b.endpoints.All() { if e.ApplicationID == appID && e.UserID == userID { endpoints = append(endpoints, cloneEndpoint(e)) } @@ -1175,9 +1175,9 @@ func (b *InMemoryBackend) DeleteUserEndpoints(appID, userID string) error { b.mu.Lock("DeleteUserEndpoints") defer b.mu.Unlock() - for key, e := range b.endpoints { + for _, e := range b.endpoints.All() { if e.ApplicationID == appID && e.UserID == userID { - delete(b.endpoints, key) + b.endpoints.Delete(e.ApplicationID + "/" + e.ID) } } @@ -1246,14 +1246,14 @@ func (b *InMemoryBackend) UpdateEndpointsBatch( for endpointID, req := range endpoints { key := appID + "/" + endpointID - e, ok := b.endpoints[key] + e, ok := b.endpoints.Get(key) if !ok { e = &Endpoint{ ApplicationID: appID, ID: endpointID, CreationDate: nowRFC3339(), } - b.endpoints[key] = e + b.endpoints.Put(e) } applyEndpointFields(e, req) @@ -1283,7 +1283,7 @@ func (b *InMemoryBackend) GetEventStream(appID string) (*EventStream, error) { b.mu.RLock("GetEventStream") defer b.mu.RUnlock() - e, ok := b.eventStreams[appID] + e, ok := b.eventStreams.Get(appID) if !ok { return nil, ErrAppNotFound } @@ -1308,7 +1308,7 @@ func (b *InMemoryBackend) PutEventStream( LastModifiedDate: nowRFC3339(), } - b.eventStreams[appID] = e + b.eventStreams.Put(e) cp := *e @@ -1320,12 +1320,12 @@ func (b *InMemoryBackend) DeleteEventStream(appID string) (*EventStream, error) b.mu.Lock("DeleteEventStream") defer b.mu.Unlock() - e, ok := b.eventStreams[appID] + e, ok := b.eventStreams.Get(appID) if !ok { return nil, ErrAppNotFound } - delete(b.eventStreams, appID) + b.eventStreams.Delete(appID) cp := *e @@ -1342,7 +1342,7 @@ func (b *InMemoryBackend) GetChannel(appID, channelType string) *Channel { defer b.mu.RUnlock() key := appID + "/" + channelType - if ch, ok := b.channels[key]; ok { + if ch, ok := b.channels.Get(key); ok { cp := *ch cp.ExtraData = cloneAnyMap(ch.ExtraData) @@ -1394,7 +1394,7 @@ func (b *InMemoryBackend) UpsertChannel( key := appID + "/" + channelType - existing := b.channels[key] + existing, _ := b.channels.Get(key) now := nowRFC3339() version := 1 @@ -1424,7 +1424,7 @@ func (b *InMemoryBackend) UpsertChannel( ExtraData: cloneAnyMap(extra), } - b.channels[key] = ch + b.channels.Put(ch) cp := *ch cp.ExtraData = cloneAnyMap(ch.ExtraData) @@ -1438,13 +1438,13 @@ func (b *InMemoryBackend) DeleteChannel(appID, channelType string) *Channel { defer b.mu.Unlock() key := appID + "/" + channelType - ch := b.channels[key] + ch, _ := b.channels.Get(key) if ch == nil { ch = &Channel{ApplicationID: appID, ChannelType: channelType} } - delete(b.channels, key) + b.channels.Delete(key) cp := *ch cp.ExtraData = cloneAnyMap(ch.ExtraData) @@ -1459,11 +1459,11 @@ func (b *InMemoryBackend) GetAllChannels(appID string) map[string]*Channel { result := make(map[string]*Channel) - for key, ch := range b.channels { + for _, ch := range b.channels.All() { if ch.ApplicationID == appID { cp := *ch cp.ExtraData = cloneAnyMap(ch.ExtraData) - result[key] = &cp + result[ch.ApplicationID+"/"+ch.ChannelType] = &cp } } @@ -1481,7 +1481,7 @@ func (b *InMemoryBackend) GetRecommenderConfiguration( b.mu.RLock("GetRecommenderConfiguration") defer b.mu.RUnlock() - r, ok := b.recommenders[recommenderID] + r, ok := b.recommenders.Get(recommenderID) if !ok { return nil, ErrAppNotFound } @@ -1497,9 +1497,9 @@ func (b *InMemoryBackend) GetRecommenderConfigurations() ([]*RecommenderConfigur b.mu.RLock("GetRecommenderConfigurations") defer b.mu.RUnlock() - results := make([]*RecommenderConfiguration, 0, len(b.recommenders)) + results := make([]*RecommenderConfiguration, 0, b.recommenders.Len()) - for _, r := range b.recommenders { + for _, r := range b.recommenders.All() { cp := *r cp.Attributes = nonNilAttrsCopy(r.Attributes) results = append(results, &cp) @@ -1566,7 +1566,7 @@ func (b *InMemoryBackend) UpdateRecommenderConfiguration( b.mu.Lock("UpdateRecommenderConfiguration") defer b.mu.Unlock() - r, ok := b.recommenders[recommenderID] + r, ok := b.recommenders.Get(recommenderID) if !ok { return nil, ErrAppNotFound } @@ -1608,12 +1608,12 @@ func (b *InMemoryBackend) DeleteRecommenderConfiguration( b.mu.Lock("DeleteRecommenderConfiguration") defer b.mu.Unlock() - r, ok := b.recommenders[recommenderID] + r, ok := b.recommenders.Get(recommenderID) if !ok { return nil, ErrAppNotFound } - delete(b.recommenders, recommenderID) + b.recommenders.Delete(recommenderID) cp := *r cp.Attributes = nonNilAttrsCopy(r.Attributes) @@ -1630,7 +1630,7 @@ func (b *InMemoryBackend) GetExportJob(appID, jobID string) (*ExportJob, error) b.mu.RLock("GetExportJob") defer b.mu.RUnlock() - j, ok := b.exportJobs[jobID] + j, ok := b.exportJobs.Get(jobID) if !ok || j.ApplicationID != appID { return nil, ErrAppNotFound } @@ -1647,7 +1647,7 @@ func (b *InMemoryBackend) GetExportJobs(appID string) ([]*ExportJob, error) { var jobs []*ExportJob - for _, j := range b.exportJobs { + for _, j := range b.exportJobs.All() { if j.ApplicationID == appID { cp := *j jobs = append(jobs, &cp) @@ -1662,7 +1662,7 @@ func (b *InMemoryBackend) GetImportJob(appID, jobID string) (*ImportJob, error) b.mu.RLock("GetImportJob") defer b.mu.RUnlock() - j, ok := b.importJobs[jobID] + j, ok := b.importJobs.Get(jobID) if !ok || j.ApplicationID != appID { return nil, ErrAppNotFound } @@ -1679,7 +1679,7 @@ func (b *InMemoryBackend) GetImportJobs(appID string) ([]*ImportJob, error) { var jobs []*ImportJob - for _, j := range b.importJobs { + for _, j := range b.importJobs.All() { if j.ApplicationID == appID { cp := *j jobs = append(jobs, &cp) @@ -1701,7 +1701,7 @@ func (b *InMemoryBackend) GetSegmentImportJobs(appID, segmentID string) ([]*Impo var jobs []*ImportJob - for _, j := range b.importJobs { + for _, j := range b.importJobs.All() { if j.ApplicationID == appID && j.SegmentID == segmentID { cp := *j jobs = append(jobs, &cp) @@ -1720,7 +1720,7 @@ func (b *InMemoryBackend) GetApplicationDateRangeKpi(appID, kpiName string) (*kp b.mu.RLock("GetApplicationDateRangeKpi") defer b.mu.RUnlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } @@ -1738,7 +1738,7 @@ func (b *InMemoryBackend) GetCampaignDateRangeKpi( b.mu.RLock("GetCampaignDateRangeKpi") defer b.mu.RUnlock() - c, ok := b.campaigns[campaignID] + c, ok := b.campaigns.Get(campaignID) if !ok || c.ApplicationID != appID { return nil, ErrAppNotFound } @@ -1758,7 +1758,7 @@ func (b *InMemoryBackend) GetJourneyDateRangeKpi( b.mu.RLock("GetJourneyDateRangeKpi") defer b.mu.RUnlock() - j, ok := b.journeys[journeyID] + j, ok := b.journeys.Get(journeyID) if !ok || j.ApplicationID != appID { return nil, ErrAppNotFound } @@ -1779,7 +1779,7 @@ func (b *InMemoryBackend) SendMessages( b.mu.Lock("SendMessages") defer b.mu.Unlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } @@ -1805,7 +1805,7 @@ func (b *InMemoryBackend) SendUsersMessages( b.mu.Lock("SendUsersMessages") defer b.mu.Unlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } @@ -1814,13 +1814,12 @@ func (b *InMemoryBackend) SendUsersMessages( for userID := range req.SendUsersMessageRequest.Users { endpointResults := make(map[string]messageResult) - for key, ep := range b.endpoints { + for _, ep := range b.endpoints.All() { if ep.ApplicationID != appID || ep.UserID != userID { continue } - // key is "appID/endpointID" — extract endpointID. - endpointID := key[len(appID)+1:] + endpointID := ep.ID endpointResults[endpointID] = messageResult{ DeliveryStatus: deliveryStatusSuccessful, MessageID: uuid.NewString(), @@ -1848,7 +1847,7 @@ func (b *InMemoryBackend) SendOTPMessage(appID string) (*sendOTPMessageResponse, b.mu.Lock("SendOTPMessage") defer b.mu.Unlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } @@ -1875,7 +1874,7 @@ func (b *InMemoryBackend) VerifyOTPMessage(appID, code string) (*verifyOTPMessag b.mu.RLock("VerifyOTPMessage") defer b.mu.RUnlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } @@ -1897,7 +1896,7 @@ func (b *InMemoryBackend) PutEvents(appID string, req putEventsRequest) (*events b.mu.Lock("PutEvents") defer b.mu.Unlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } @@ -2017,16 +2016,15 @@ func (b *InMemoryBackend) RemoveAttributes( b.mu.Lock("RemoveAttributes") defer b.mu.Unlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } // Remove the attribute from all endpoints in this app. - for key, e := range b.endpoints { + for _, e := range b.endpoints.All() { if e.ApplicationID == appID { if e.Attributes != nil { delete(e.Attributes, attributeType) - b.endpoints[key] = e } } } @@ -2043,14 +2041,14 @@ func (b *InMemoryBackend) GetInAppMessages(appID, _ string) (*inAppMessagesRespo b.mu.RLock("GetInAppMessages") defer b.mu.RUnlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } // Collect in-app templates as message campaigns for this app. var campaigns []inAppMessageCampaign - for _, t := range b.inAppTemplates { + for _, t := range b.inAppTemplates.All() { campaigns = append(campaigns, inAppMessageCampaign{CampaignID: t.TemplateName}) } @@ -2070,7 +2068,7 @@ func (b *InMemoryBackend) GetCampaignActivities( b.mu.RLock("GetCampaignActivities") defer b.mu.RUnlock() - c, ok := b.campaigns[campaignID] + c, ok := b.campaigns.Get(campaignID) if !ok || c.ApplicationID != appID { return nil, ErrAppNotFound } @@ -2092,7 +2090,7 @@ func (b *InMemoryBackend) GetJourneyExecutionMetrics( b.mu.RLock("GetJourneyExecutionMetrics") defer b.mu.RUnlock() - j, ok := b.journeys[journeyID] + j, ok := b.journeys.Get(journeyID) if !ok || j.ApplicationID != appID { return nil, ErrAppNotFound } @@ -2116,7 +2114,7 @@ func (b *InMemoryBackend) GetJourneyExecutionActivityMetrics( b.mu.RLock("GetJourneyExecutionActivityMetrics") defer b.mu.RUnlock() - j, ok := b.journeys[journeyID] + j, ok := b.journeys.Get(journeyID) if !ok || j.ApplicationID != appID { return nil, ErrAppNotFound } @@ -2140,7 +2138,7 @@ func (b *InMemoryBackend) GetJourneyRuns(appID, journeyID string) (*journeyRunsR b.mu.RLock("GetJourneyRuns") defer b.mu.RUnlock() - j, ok := b.journeys[journeyID] + j, ok := b.journeys.Get(journeyID) if !ok || j.ApplicationID != appID { return nil, ErrAppNotFound } @@ -2163,7 +2161,7 @@ func (b *InMemoryBackend) GetJourneyRunExecutionMetrics( b.mu.RLock("GetJourneyRunExecutionMetrics") defer b.mu.RUnlock() - j, ok := b.journeys[journeyID] + j, ok := b.journeys.Get(journeyID) if !ok || j.ApplicationID != appID { return nil, ErrAppNotFound } @@ -2185,7 +2183,7 @@ func (b *InMemoryBackend) GetJourneyRunExecutionActivityMetrics( b.mu.RLock("GetJourneyRunExecutionActivityMetrics") defer b.mu.RUnlock() - j, ok := b.journeys[journeyID] + j, ok := b.journeys.Get(journeyID) if !ok || j.ApplicationID != appID { return nil, ErrAppNotFound } @@ -2220,7 +2218,7 @@ func (b *InMemoryBackend) GetCampaignVersion( } // Fall back to current campaign if version not found in history. - c, ok := b.campaigns[campaignID] + c, ok := b.campaigns.Get(campaignID) if !ok || c.ApplicationID != appID { return nil, ErrAppNotFound } @@ -2233,7 +2231,7 @@ func (b *InMemoryBackend) GetCampaignVersions(appID, campaignID string) ([]*Camp b.mu.RLock("GetCampaignVersions") defer b.mu.RUnlock() - if _, ok := b.campaigns[campaignID]; !ok { + if _, ok := b.campaigns.Get(campaignID); !ok { return nil, ErrAppNotFound } @@ -2266,7 +2264,7 @@ func (b *InMemoryBackend) GetSegmentVersion( } // Fall back to current segment if version not found in history. - s, ok := b.segments[segmentID] + s, ok := b.segments.Get(segmentID) if !ok || s.ApplicationID != appID { return nil, ErrAppNotFound } @@ -2279,7 +2277,7 @@ func (b *InMemoryBackend) GetSegmentVersions(appID, segmentID string) ([]*Segmen b.mu.RLock("GetSegmentVersions") defer b.mu.RUnlock() - if _, ok := b.segments[segmentID]; !ok { + if _, ok := b.segments.Get(segmentID); !ok { return nil, ErrAppNotFound } @@ -2349,7 +2347,7 @@ func (b *InMemoryBackend) GetApplicationSettings(appID string) (*storedAppSettin b.mu.RLock("GetApplicationSettings") defer b.mu.RUnlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } @@ -2379,7 +2377,7 @@ func (b *InMemoryBackend) UpdateApplicationSettings( b.mu.Lock("UpdateApplicationSettings") defer b.mu.Unlock() - if _, ok := b.apps[appID]; !ok { + if _, ok := b.apps.Get(appID); !ok { return nil, ErrAppNotFound } diff --git a/services/pinpoint/export_test.go b/services/pinpoint/export_test.go index e49211133..3cd6a0885 100644 --- a/services/pinpoint/export_test.go +++ b/services/pinpoint/export_test.go @@ -15,12 +15,12 @@ func TotalPerAppEntries(b *InMemoryBackend) int { b.mu.RLock("TotalPerAppEntries") defer b.mu.RUnlock() - return len(b.appEvents) + len(b.eventStreams) + len(b.otpCodes) + + return len(b.appEvents) + b.eventStreams.Len() + len(b.otpCodes) + len(b.appSettings) + len(b.sentMessages) + - len(b.endpoints) + len(b.channels) + + b.endpoints.Len() + b.channels.Len() + len(b.campaignVersions) + len(b.segmentVersions) + len(b.campaignActivities) + len(b.journeyRuns) + - len(b.campaigns) + len(b.segments) + len(b.journeys) + b.campaigns.Len() + b.segments.Len() + b.journeys.Len() } // AppEventCount returns the number of retained events for an application. @@ -118,6 +118,33 @@ func UpdateEmailTemplateForTest(b *InMemoryBackend, templateName string) error { return err } +// CreateVoiceTemplateForTest creates a voice template with the given name, +// exercising the voiceTemplates table without exporting the request type. +func CreateVoiceTemplateForTest(b *InMemoryBackend, region, accountID, templateName, body string) error { + _, err := b.CreateVoiceTemplate(region, accountID, templateName, createVoiceTemplateRequest{Body: body}) + + return err +} + +// UpdateEndpointForTest creates or updates an endpoint with a minimal address, +// exercising the endpoints table without exporting the request type. +func UpdateEndpointForTest(b *InMemoryBackend, appID, endpointID, address string) error { + _, err := b.UpdateEndpoint(appID, endpointID, updateEndpointRequest{Address: address}) + + return err +} + +// PutEventStreamForTest creates or updates the event stream for an app, +// exercising the eventStreams table without exporting the request type. +func PutEventStreamForTest(b *InMemoryBackend, appID, destinationStreamArn, roleArn string) error { + _, err := b.PutEventStream(appID, putEventStreamRequest{ + DestinationStreamArn: destinationStreamArn, + RoleArn: roleArn, + }) + + return err +} + // PutEventsForTest records n events for a single endpoint of the app, exercising // the appEvents append/cap path without exporting the request type. func PutEventsForTest(b *InMemoryBackend, appID string, n int) error { diff --git a/services/pinpoint/exports.go b/services/pinpoint/exports.go index 215ba6475..ccb59718a 100644 --- a/services/pinpoint/exports.go +++ b/services/pinpoint/exports.go @@ -45,7 +45,7 @@ func AppCount(b *InMemoryBackend) int { b.mu.RLock("AppCount") defer b.mu.RUnlock() - return len(b.apps) + return b.apps.Len() } // CampaignCount returns the number of campaigns in the backend. @@ -53,7 +53,7 @@ func CampaignCount(b *InMemoryBackend) int { b.mu.RLock("CampaignCount") defer b.mu.RUnlock() - return len(b.campaigns) + return b.campaigns.Len() } // SegmentCount returns the number of segments in the backend. @@ -61,7 +61,7 @@ func SegmentCount(b *InMemoryBackend) int { b.mu.RLock("SegmentCount") defer b.mu.RUnlock() - return len(b.segments) + return b.segments.Len() } // JourneyCount returns the number of journeys in the backend. @@ -69,7 +69,7 @@ func JourneyCount(b *InMemoryBackend) int { b.mu.RLock("JourneyCount") defer b.mu.RUnlock() - return len(b.journeys) + return b.journeys.Len() } // EmailTemplateCount returns the number of email templates in the backend. @@ -77,7 +77,7 @@ func EmailTemplateCount(b *InMemoryBackend) int { b.mu.RLock("EmailTemplateCount") defer b.mu.RUnlock() - return len(b.emailTemplates) + return b.emailTemplates.Len() } // InAppTemplateCount returns the number of in-app templates in the backend. @@ -85,7 +85,7 @@ func InAppTemplateCount(b *InMemoryBackend) int { b.mu.RLock("InAppTemplateCount") defer b.mu.RUnlock() - return len(b.inAppTemplates) + return b.inAppTemplates.Len() } // PushTemplateCount returns the number of push templates in the backend. @@ -93,7 +93,7 @@ func PushTemplateCount(b *InMemoryBackend) int { b.mu.RLock("PushTemplateCount") defer b.mu.RUnlock() - return len(b.pushTemplates) + return b.pushTemplates.Len() } // SmsTemplateCount returns the number of SMS templates in the backend. @@ -101,7 +101,7 @@ func SmsTemplateCount(b *InMemoryBackend) int { b.mu.RLock("SmsTemplateCount") defer b.mu.RUnlock() - return len(b.smsTemplates) + return b.smsTemplates.Len() } // ExportJobCount returns the number of export jobs in the backend. @@ -109,7 +109,7 @@ func ExportJobCount(b *InMemoryBackend) int { b.mu.RLock("ExportJobCount") defer b.mu.RUnlock() - return len(b.exportJobs) + return b.exportJobs.Len() } // ImportJobCount returns the number of import jobs in the backend. @@ -117,7 +117,7 @@ func ImportJobCount(b *InMemoryBackend) int { b.mu.RLock("ImportJobCount") defer b.mu.RUnlock() - return len(b.importJobs) + return b.importJobs.Len() } // RecommenderCount returns the number of recommender configurations in the backend. @@ -125,7 +125,7 @@ func RecommenderCount(b *InMemoryBackend) int { b.mu.RLock("RecommenderCount") defer b.mu.RUnlock() - return len(b.recommenders) + return b.recommenders.Len() } // ARNIndexSize returns the number of entries in the ARN index. diff --git a/services/pinpoint/handler_refinement1_test.go b/services/pinpoint/handler_refinement1_test.go index 0aaeb1ee2..64574388e 100644 --- a/services/pinpoint/handler_refinement1_test.go +++ b/services/pinpoint/handler_refinement1_test.go @@ -786,3 +786,100 @@ func TestRefinement1_SnapshotRestoreARNIndexIntegrity(t *testing.T) { require.NoError(t, err) assert.Equal(t, "true", tags["restored"]) } + +// TestSnapshotRestore_FullStateRoundTrip exercises a Snapshot->Restore round +// trip across every store.Table the Phase 3.3 datalayer conversion produced, +// covering both the persisted resource kinds (apps, campaigns, segments, the +// four templates, export/import jobs, journeys, recommenders) and the +// resource kinds that were never part of a persisted snapshot before the +// conversion (voice templates, endpoints, event streams, channels) to confirm +// the persisted/non-persisted split was preserved exactly. +func TestSnapshotRestore_FullStateRoundTrip(t *testing.T) { + t.Parallel() + + region, accountID := "us-east-1", "123456789012" + b := pinpoint.NewInMemoryBackend(region, accountID) + + app, err := b.CreateApp(region, accountID, "full-state-app", map[string]string{"env": "test"}) + require.NoError(t, err) + + _, err = b.CreateCampaign(region, accountID, app.ID, pinpoint.ExportedCreateCampaignRequest{Name: "c1"}) + require.NoError(t, err) + + _, err = b.CreateSegment(region, accountID, app.ID, pinpoint.ExportedCreateSegmentRequest{Name: "s1"}) + require.NoError(t, err) + + _, err = b.CreateJourney(region, accountID, app.ID, pinpoint.ExportedCreateJourneyRequest{Name: "j1"}) + require.NoError(t, err) + + _, err = b.CreateEmailTemplate(region, accountID, "email-t1", + pinpoint.ExportedCreateEmailTemplateRequest{Subject: "hi"}) + require.NoError(t, err) + + _, err = b.CreateInAppTemplate(region, accountID, "inapp-t1", pinpoint.ExportedCreateInAppTemplateRequest{}) + require.NoError(t, err) + + _, err = b.CreatePushTemplate(region, accountID, "push-t1", pinpoint.ExportedCreatePushTemplateRequest{}) + require.NoError(t, err) + + _, err = b.CreateSmsTemplate(region, accountID, "sms-t1", pinpoint.ExportedCreateSmsTemplateRequest{}) + require.NoError(t, err) + + _, err = b.CreateExportJob(region, accountID, app.ID, + pinpoint.ExportedCreateExportJobRequest{RoleArn: "role-1"}) + require.NoError(t, err) + + _, err = b.CreateImportJob(region, accountID, app.ID, + pinpoint.ExportedCreateImportJobRequest{RoleArn: "role-1", S3Url: "s3://bucket/key"}) + require.NoError(t, err) + + _, err = b.CreateRecommenderConfiguration(pinpoint.ExportedCreateRecommenderConfigRequest{ + Name: "rec-1", + RecommendationProviderIDType: "PINPOINT_ENDPOINT_ID", + }) + require.NoError(t, err) + + // Resource kinds that have never been part of a persisted snapshot. + require.NoError(t, pinpoint.CreateVoiceTemplateForTest(b, region, accountID, "voice-t1", "hello")) + require.NoError(t, pinpoint.UpdateEndpointForTest(b, app.ID, "endpoint-1", "user@example.com")) + streamARN := "arn:aws:kinesis:us-east-1:123456789012:stream/x" + require.NoError(t, pinpoint.PutEventStreamForTest(b, app.ID, streamARN, "role-1")) + b.UpsertChannel(app.ID, "EMAIL", true, nil) + + data := b.Snapshot(t.Context()) + require.NotNil(t, data) + + b2 := pinpoint.NewInMemoryBackend(region, accountID) + require.NoError(t, b2.Restore(t.Context(), data)) + + // Persisted resource kinds must all survive the round trip. + assert.Equal(t, 1, pinpoint.AppCount(b2)) + assert.Equal(t, 1, pinpoint.CampaignCount(b2)) + // CreateImportJob materialises an additional IMPORT-type segment (AWS + // behaviour), so the explicit segment created above plus that one gives 2. + assert.Equal(t, 2, pinpoint.SegmentCount(b2)) + assert.Equal(t, 1, pinpoint.JourneyCount(b2)) + assert.Equal(t, 1, pinpoint.EmailTemplateCount(b2)) + assert.Equal(t, 1, pinpoint.InAppTemplateCount(b2)) + assert.Equal(t, 1, pinpoint.PushTemplateCount(b2)) + assert.Equal(t, 1, pinpoint.SmsTemplateCount(b2)) + assert.Equal(t, 1, pinpoint.ExportJobCount(b2)) + assert.Equal(t, 1, pinpoint.ImportJobCount(b2)) + assert.Equal(t, 1, pinpoint.RecommenderCount(b2)) + + // Non-persisted resource kinds must NOT survive: this is the historical + // behaviour, not a regression, and the round trip must preserve it. + _, err = b2.GetVoiceTemplate("voice-t1") + require.ErrorIs(t, err, pinpoint.ErrAppNotFound) + + _, err = b2.GetEndpoint(app.ID, "endpoint-1") + require.ErrorIs(t, err, pinpoint.ErrAppNotFound) + + _, err = b2.GetEventStream(app.ID) + require.ErrorIs(t, err, pinpoint.ErrAppNotFound) + + // GetChannel synthesises a disabled default when no channel is stored, + // which is what a never-persisted channel must look like post-restore. + ch := b2.GetChannel(app.ID, "EMAIL") + assert.False(t, ch.Enabled) +} diff --git a/services/pinpoint/persistence.go b/services/pinpoint/persistence.go index dcc2bad93..ea07ea337 100644 --- a/services/pinpoint/persistence.go +++ b/services/pinpoint/persistence.go @@ -3,116 +3,84 @@ package pinpoint import ( "context" "encoding/json" + "fmt" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) +// pinpointSnapshotVersion identifies the shape of [backendSnapshot]. It must +// be bumped whenever a change to the persisted table set (or a persisted +// value type) would make an older snapshot unsafe to decode as the current +// shape. Restore compares this against the persisted value and discards +// (rather than attempts to partially decode) any mismatch — see Restore +// below. +const pinpointSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the Pinpoint backend. +// +// Tables holds one JSON-encoded array per persisted table, produced by +// [store.Registry.SnapshotAll]. Only the resource kinds Pinpoint has always +// persisted are included (apps, campaigns, emailTemplates, exportJobs, +// importJobs, inAppTemplates, journeys, pushTemplates, recommenders, +// segments, smsTemplates) — voiceTemplates, endpoints, eventStreams, and +// channels were never part of a persisted snapshot before this refactor and +// remain excluded so behaviour is preserved exactly. type backendSnapshot struct { - Apps map[string]*App `json:"apps"` - Campaigns map[string]*Campaign `json:"campaigns"` - EmailTemplates map[string]*EmailTemplate `json:"emailTemplates"` - ExportJobs map[string]*ExportJob `json:"exportJobs"` - ImportJobs map[string]*ImportJob `json:"importJobs"` - InAppTemplates map[string]*InAppTemplate `json:"inAppTemplates"` - Journeys map[string]*Journey `json:"journeys"` - PushTemplates map[string]*PushTemplate `json:"pushTemplates"` - Recommenders map[string]*RecommenderConfiguration `json:"recommenders"` - Segments map[string]*Segment `json:"segments"` - SmsTemplates map[string]*SmsTemplate `json:"smsTemplates"` - AccountID string `json:"accountID"` - Region string `json:"region"` + Tables map[string]json.RawMessage `json:"tables"` + AccountID string `json:"accountID"` + Region string `json:"region"` + Version int `json:"version"` +} + +// persistRegistry builds an ephemeral [store.Registry] over exactly the live +// tables this backend has always persisted, registering the SAME *store.Table +// pointers the backend itself reads and writes through (rather than copying +// into a separate DTO type), since every persisted value type here is +// already a plain JSON-friendly struct with no live/non-serialisable state. +// It is rebuilt on every Snapshot/Restore call rather than cached as a +// long-lived field: [store.Register] panics on a duplicate name, and +// registering the same tables into two different [store.Registry] values is +// safe (Registry holds no back-reference on the Table), so a fresh registry +// scoped to the call is simpler than guarding a shared one. +func (b *InMemoryBackend) persistRegistry() *store.Registry { + reg := store.NewRegistry() + store.Register(reg, "apps", b.apps) + store.Register(reg, "campaigns", b.campaigns) + store.Register(reg, "emailTemplates", b.emailTemplates) + store.Register(reg, "exportJobs", b.exportJobs) + store.Register(reg, "importJobs", b.importJobs) + store.Register(reg, "inAppTemplates", b.inAppTemplates) + store.Register(reg, "journeys", b.journeys) + store.Register(reg, "pushTemplates", b.pushTemplates) + store.Register(reg, "recommenders", b.recommenders) + store.Register(reg, "segments", b.segments) + store.Register(reg, "smsTemplates", b.smsTemplates) + + return reg } // Snapshot serialises the backend state to JSON. -// Deep copies are taken before serialisation to avoid races with concurrent writes. func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() - apps := make(map[string]*App, len(b.apps)) - for k, v := range b.apps { - apps[k] = cloneApp(v) - } - - campaigns := make(map[string]*Campaign, len(b.campaigns)) - for k, v := range b.campaigns { - campaigns[k] = cloneCampaign(v) - } - - emailTemplates := make(map[string]*EmailTemplate, len(b.emailTemplates)) - for k, v := range b.emailTemplates { - emailTemplates[k] = cloneEmailTemplate(v) - } - - exportJobs := make(map[string]*ExportJob, len(b.exportJobs)) - for k, v := range b.exportJobs { - cp := *v - exportJobs[k] = &cp - } - - importJobs := make(map[string]*ImportJob, len(b.importJobs)) - for k, v := range b.importJobs { - cp := *v - importJobs[k] = &cp - } - - inAppTemplates := make(map[string]*InAppTemplate, len(b.inAppTemplates)) - for k, v := range b.inAppTemplates { - inAppTemplates[k] = cloneInAppTemplate(v) - } - - journeys := make(map[string]*Journey, len(b.journeys)) - for k, v := range b.journeys { - journeys[k] = cloneJourney(v) - } - - pushTemplates := make(map[string]*PushTemplate, len(b.pushTemplates)) - for k, v := range b.pushTemplates { - pushTemplates[k] = clonePushTemplate(v) - } - - recommenders := make(map[string]*RecommenderConfiguration, len(b.recommenders)) - for k, v := range b.recommenders { - cp := *v - cp.Attributes = nonNilAttrsCopy(v.Attributes) - recommenders[k] = &cp - } - - segments := make(map[string]*Segment, len(b.segments)) - for k, v := range b.segments { - segments[k] = cloneSegment(v) - } - - smsTemplates := make(map[string]*SmsTemplate, len(b.smsTemplates)) - for k, v := range b.smsTemplates { - smsTemplates[k] = cloneSmsTemplate(v) - } - - snap := backendSnapshot{ - Apps: apps, - Campaigns: campaigns, - EmailTemplates: emailTemplates, - ExportJobs: exportJobs, - ImportJobs: importJobs, - InAppTemplates: inAppTemplates, - Journeys: journeys, - PushTemplates: pushTemplates, - Recommenders: recommenders, - Segments: segments, - SmsTemplates: smsTemplates, - AccountID: b.accountID, - Region: b.region, - } - - data, err := json.Marshal(snap) + tables, err := b.persistRegistry().SnapshotAll() if err != nil { logger.Load(ctx).WarnContext(ctx, "pinpoint: failed to marshal snapshot", "error", err) return nil } - return data + snap := backendSnapshot{ + Version: pinpointSnapshotVersion, + Tables: tables, + AccountID: b.accountID, + Region: b.region, + } + + return persistence.MarshalSnapshot(ctx, "pinpoint", snap) } // Restore loads backend state from a JSON snapshot. @@ -123,22 +91,30 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { return err } - ensureNonNilMaps(&snap) - b.mu.Lock("Restore") defer b.mu.Unlock() - b.apps = snap.Apps - b.campaigns = snap.Campaigns - b.emailTemplates = snap.EmailTemplates - b.exportJobs = snap.ExportJobs - b.importJobs = snap.ImportJobs - b.inAppTemplates = snap.InAppTemplates - b.journeys = snap.Journeys - b.pushTemplates = snap.PushTemplates - b.recommenders = snap.Recommenders - b.segments = snap.Segments - b.smsTemplates = snap.SmsTemplates + if snap.Version != pinpointSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape — that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. + logger.Load(ctx).WarnContext(ctx, + "pinpoint: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", pinpointSnapshotVersion) + + b.registry.ResetAll() + b.arnIndex = make(map[string]tagHolder) + + return nil + } + + if err := b.persistRegistry().RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("pinpoint: restore snapshot tables: %w", err) + } + b.accountID = snap.AccountID b.region = snap.Region @@ -152,82 +128,35 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { // rebuildARNIndexLocked rebuilds arnIndex from all in-memory resources. // Must be called with b.mu write lock held. func rebuildARNIndexLocked(b *InMemoryBackend) { - for _, v := range b.apps { + for _, v := range b.apps.All() { b.arnIndex[v.ARN] = v } - for _, v := range b.campaigns { + for _, v := range b.campaigns.All() { b.arnIndex[v.ARN] = v } - for _, v := range b.emailTemplates { + for _, v := range b.emailTemplates.All() { b.arnIndex[v.ARN] = v } - for _, v := range b.inAppTemplates { + for _, v := range b.inAppTemplates.All() { b.arnIndex[v.ARN] = v } - for _, v := range b.journeys { + for _, v := range b.journeys.All() { b.arnIndex[v.ARN] = v } - for _, v := range b.pushTemplates { + for _, v := range b.pushTemplates.All() { b.arnIndex[v.ARN] = v } - for _, v := range b.segments { + for _, v := range b.segments.All() { b.arnIndex[v.ARN] = v } - for _, v := range b.smsTemplates { + for _, v := range b.smsTemplates.All() { b.arnIndex[v.ARN] = v } } - -// ensureNonNilMaps initialises any nil maps in the snapshot to avoid nil-map panics. -func ensureNonNilMaps(snap *backendSnapshot) { - if snap.Apps == nil { - snap.Apps = make(map[string]*App) - } - - if snap.Campaigns == nil { - snap.Campaigns = make(map[string]*Campaign) - } - - if snap.EmailTemplates == nil { - snap.EmailTemplates = make(map[string]*EmailTemplate) - } - - if snap.ExportJobs == nil { - snap.ExportJobs = make(map[string]*ExportJob) - } - - if snap.ImportJobs == nil { - snap.ImportJobs = make(map[string]*ImportJob) - } - - if snap.InAppTemplates == nil { - snap.InAppTemplates = make(map[string]*InAppTemplate) - } - - if snap.Journeys == nil { - snap.Journeys = make(map[string]*Journey) - } - - if snap.PushTemplates == nil { - snap.PushTemplates = make(map[string]*PushTemplate) - } - - if snap.Recommenders == nil { - snap.Recommenders = make(map[string]*RecommenderConfiguration) - } - - if snap.Segments == nil { - snap.Segments = make(map[string]*Segment) - } - - if snap.SmsTemplates == nil { - snap.SmsTemplates = make(map[string]*SmsTemplate) - } -} diff --git a/services/pinpoint/store_setup.go b/services/pinpoint/store_setup.go new file mode 100644 index 000000000..4f02d0eb9 --- /dev/null +++ b/services/pinpoint/store_setup.go @@ -0,0 +1,72 @@ +package pinpoint + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend whose key is a pure +// function of the stored value's own fields is registered here, exactly +// once, as a *store.Table[T] on b.registry. See pkgs/store's package doc and +// the services/ec2 (store_setup.go) / services/sqs (persistence.go) pilots +// for the pattern this follows. +// +// The following resource fields are deliberately NOT registered here and +// remain plain maps because their key is not a pure function of the stored +// value (store.Table requires one): +// - appSettings: storedAppSettings carries no ApplicationID/identity field +// of its own -- it is keyed externally by app ID, the same shape of quirk +// that keeps EC2's instanceIMDSOptions/verifiedAccess*Policies as plain +// maps. +// - arnIndex: values are the tagHolder interface, not a single concrete *T +// pointer type -- store.Table is keyed on one concrete V. +// - campaignVersions, segmentVersions, templateVersionHistory, +// campaignActivities, journeyRuns, appEvents: slice-valued +// (map[string][]*T / map[string][]T), not map[string]*T. +// - sentMessages, otpCodes: counters/tokens (map[string]int / +// map[string]string), not map[string]*T. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +func appKeyFn(v *App) string { return v.ID } +func campaignKeyFn(v *Campaign) string { return v.ID } +func segmentKeyFn(v *Segment) string { return v.ID } +func emailTemplateKeyFn(v *EmailTemplate) string { return v.TemplateName } +func inAppTemplateKeyFn(v *InAppTemplate) string { return v.TemplateName } +func pushTemplateKeyFn(v *PushTemplate) string { return v.TemplateName } +func smsTemplateKeyFn(v *SmsTemplate) string { return v.TemplateName } +func voiceTemplateKeyFn(v *VoiceTemplate) string { return v.TemplateName } +func exportJobKeyFn(v *ExportJob) string { return v.ID } +func importJobKeyFn(v *ImportJob) string { return v.ID } +func journeyKeyFn(v *Journey) string { return v.ID } +func recommenderKeyFn(v *RecommenderConfiguration) string { return v.ID } + +// endpointKeyFn mirrors the historical "/" map key: both +// halves are already stored on the Endpoint value itself. +func endpointKeyFn(v *Endpoint) string { return v.ApplicationID + "/" + v.ID } + +// eventStreamKeyFn mirrors the historical bare-appID map key. +func eventStreamKeyFn(v *EventStream) string { return v.ApplicationID } + +// channelKeyFn mirrors the historical "/" map key. +func channelKeyFn(v *Channel) string { return v.ApplicationID + "/" + v.ChannelType } + +// registerAllTables registers every converted resource map on b.registry +// exactly once. It must be called during construction only (immediately +// after b.registry is created), never on every Reset() -- store.Register +// panics on a duplicate name, so runtime resets go through +// b.registry.ResetAll() instead (see InMemoryBackend.Reset in backend.go). +func registerAllTables(b *InMemoryBackend) { + b.apps = store.Register(b.registry, "apps", store.New(appKeyFn)) + b.campaigns = store.Register(b.registry, "campaigns", store.New(campaignKeyFn)) + b.segments = store.Register(b.registry, "segments", store.New(segmentKeyFn)) + b.emailTemplates = store.Register(b.registry, "emailTemplates", store.New(emailTemplateKeyFn)) + b.inAppTemplates = store.Register(b.registry, "inAppTemplates", store.New(inAppTemplateKeyFn)) + b.pushTemplates = store.Register(b.registry, "pushTemplates", store.New(pushTemplateKeyFn)) + b.smsTemplates = store.Register(b.registry, "smsTemplates", store.New(smsTemplateKeyFn)) + b.voiceTemplates = store.Register(b.registry, "voiceTemplates", store.New(voiceTemplateKeyFn)) + b.exportJobs = store.Register(b.registry, "exportJobs", store.New(exportJobKeyFn)) + b.importJobs = store.Register(b.registry, "importJobs", store.New(importJobKeyFn)) + b.journeys = store.Register(b.registry, "journeys", store.New(journeyKeyFn)) + b.recommenders = store.Register(b.registry, "recommenders", store.New(recommenderKeyFn)) + b.endpoints = store.Register(b.registry, "endpoints", store.New(endpointKeyFn)) + b.eventStreams = store.Register(b.registry, "eventStreams", store.New(eventStreamKeyFn)) + b.channels = store.Register(b.registry, "channels", store.New(channelKeyFn)) +} diff --git a/services/quicksight/backend.go b/services/quicksight/backend.go index c8a282a19..330b4da2f 100644 --- a/services/quicksight/backend.go +++ b/services/quicksight/backend.go @@ -3,6 +3,7 @@ package quicksight import ( "context" "encoding/base64" + "encoding/json" "errors" "fmt" "maps" @@ -16,9 +17,21 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awserr" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) +// quicksightSnapshotVersion identifies the shape of backendSnapshot's Tables +// blob (i.e. the set/shape of resources registered on b.registry -- see +// registerAllTables in store_setup.go). It must be bumped whenever a change +// there would make an older snapshot unsafe to decode as the current shape. +// Restore compares this against the persisted value and discards (rather +// than attempts to partially decode) any mismatch -- see Restore below. This +// mirrors the services/ec2 (commit 12e611a4) and services/sqs (commit +// 0f09d77c) conversions. +const quicksightSnapshotVersion = 1 + // encodePageToken encodes an integer offset as an opaque base64 token. func encodePageToken(offset int) string { return base64.StdEncoding.EncodeToString([]byte(strconv.Itoa(offset))) @@ -434,29 +447,22 @@ func (a *storedAnalysis) toAnalysis() *Analysis { } } -// state is the serializable snapshot of the backend. -type state struct { - Namespaces map[string]*storedNamespace `json:"namespaces"` - Groups map[string]*storedGroup `json:"groups"` - GroupMembers map[string]bool `json:"groupMembers"` - Users map[string]*storedUser `json:"users"` - DataSources map[string]*storedDataSource `json:"dataSources"` - DataSets map[string]*storedDataSet `json:"dataSets"` - Ingestions map[string]*storedIngestion `json:"ingestions"` - Dashboards map[string]*storedDashboard `json:"dashboards"` - Analyses map[string]*storedAnalysis `json:"analyses"` - Tags map[string]map[string]string `json:"tags"` - Folders map[string]*storedFolder `json:"folders"` - FolderMembers map[string]*storedFolderMember `json:"folderMembers"` - Templates map[string]*storedTemplate `json:"templates"` - Themes map[string]*storedTheme `json:"themes"` - Topics map[string]*storedTopic `json:"topics"` - VPCConnections map[string]*storedVPCConnection `json:"vpcConnections"` - IAMPolicyAssignments map[string]*storedIAMPolicyAssignment `json:"iamPolicyAssignments"` +// backendSnapshot is the top-level on-disk shape for the QuickSight backend. +// +// Tables holds one JSON-encoded array per registered store.Table (see +// store_setup.go's registerAllTables), produced by +// [store.Registry.SnapshotAll]. The remaining fields are the resource +// collections left as raw maps because their value type carries no identity +// field of its own (see store_setup.go's doc comment for the full list and +// rationale) -- these round-trip directly, same as before conversion. +type backendSnapshot struct { + Tables map[string]json.RawMessage `json:"tables"` + + GroupMembers map[string]bool `json:"groupMembers"` + Tags map[string]map[string]string `json:"tags"` AccountSettings map[string]*storedAccountSettings `json:"accountSettings"` AccountSubscriptions map[string]*storedAccountSubscription `json:"accountSubscriptions"` - AccountCustomizations map[string]*storedAccountCustomization `json:"accountCustomizations"` AccountCustomPermissions map[string]string `json:"accountCustomPermissions"` IPRestrictions map[string]*storedIPRestriction `json:"ipRestrictions"` PublicSharing map[string]bool `json:"publicSharing"` @@ -466,51 +472,43 @@ type state struct { QSearchConfig map[string]string `json:"qSearchConfig"` DashboardsQAConfig map[string]string `json:"dashboardsQAConfig"` - Brands map[string]*storedBrand `json:"brands"` - BrandAssignments map[string]string `json:"brandAssignments"` - CustomPermissions map[string]*storedCustomPermissions `json:"customPermissions"` - RoleCustomPermissions map[string]string `json:"roleCustomPermissions"` - RoleMemberships map[string]bool `json:"roleMemberships"` - UserCustomPermissions map[string]string `json:"userCustomPermissions"` - OAuthClientApps map[string]*storedOAuthApp `json:"oauthClientApps"` - IdentityPropagationConfigs map[string]*storedIdentityPropagationConfig `json:"identityPropagationConfigs"` - AssetBundleExportJobs map[string]*storedAssetBundleExportJob `json:"assetBundleExportJobs"` - AssetBundleImportJobs map[string]*storedAssetBundleImportJob `json:"assetBundleImportJobs"` - DashboardSnapshotJobs map[string]*storedDashboardSnapshotJob `json:"dashboardSnapshotJobs"` + BrandAssignments map[string]string `json:"brandAssignments"` + RoleCustomPermissions map[string]string `json:"roleCustomPermissions"` + RoleMemberships map[string]bool `json:"roleMemberships"` + UserCustomPermissions map[string]string `json:"userCustomPermissions"` - ActionConnectors map[string]*storedActionConnector `json:"actionConnectors"` - AutomationJobs map[string]*storedAutomationJob `json:"automationJobs"` - Flows map[string]*storedFlow `json:"flows"` - SPICECapacity map[string]string `json:"spiceCapacity"` + SPICECapacity map[string]string `json:"spiceCapacity"` + SelfUpgradeConfig map[string]string `json:"selfUpgradeConfig"` - SelfUpgradeConfig map[string]string `json:"selfUpgradeConfig"` - SelfUpgradeRequests map[string]*storedSelfUpgradeRequest `json:"selfUpgradeRequests"` + Version int `json:"version"` } // InMemoryBackend is the in-memory implementation of StorageBackend. type InMemoryBackend struct { - mu *lockmetrics.RWMutex - namespaces map[string]*storedNamespace - groups map[string]*storedGroup + mu *lockmetrics.RWMutex + registry *store.Registry + + namespaces *store.Table[storedNamespace] + groups *store.Table[storedGroup] groupMembers map[string]bool - users map[string]*storedUser - dataSources map[string]*storedDataSource - dataSets map[string]*storedDataSet - ingestions map[string]*storedIngestion - dashboards map[string]*storedDashboard - analyses map[string]*storedAnalysis + users *store.Table[storedUser] + dataSources *store.Table[storedDataSource] + dataSets *store.Table[storedDataSet] + ingestions *store.Table[storedIngestion] + dashboards *store.Table[storedDashboard] + analyses *store.Table[storedAnalysis] tags map[string]map[string]string - folders map[string]*storedFolder - folderMembers map[string]*storedFolderMember - templates map[string]*storedTemplate - themes map[string]*storedTheme - topics map[string]*storedTopic - vpcConnections map[string]*storedVPCConnection - iamPolicyAssignments map[string]*storedIAMPolicyAssignment + folders *store.Table[storedFolder] + folderMembers *store.Table[storedFolderMember] + templates *store.Table[storedTemplate] + themes *store.Table[storedTheme] + topics *store.Table[storedTopic] + vpcConnections *store.Table[storedVPCConnection] + iamPolicyAssignments *store.Table[storedIAMPolicyAssignment] accountSettings map[string]*storedAccountSettings accountSubscriptions map[string]*storedAccountSubscription - accountCustomizations map[string]*storedAccountCustomization + accountCustomizations *store.Table[storedAccountCustomization] accountCustomPermissions map[string]string ipRestrictions map[string]*storedIPRestriction publicSharing map[string]bool @@ -520,25 +518,25 @@ type InMemoryBackend struct { qSearchConfig map[string]string dashboardsQAConfig map[string]string - brands map[string]*storedBrand + brands *store.Table[storedBrand] brandAssignments map[string]string - customPermissions map[string]*storedCustomPermissions + customPermissions *store.Table[storedCustomPermissions] roleCustomPermissions map[string]string roleMemberships map[string]bool userCustomPermissions map[string]string - oauthClientApps map[string]*storedOAuthApp - identityPropagationConfigs map[string]*storedIdentityPropagationConfig - assetBundleExportJobs map[string]*storedAssetBundleExportJob - assetBundleImportJobs map[string]*storedAssetBundleImportJob - dashboardSnapshotJobs map[string]*storedDashboardSnapshotJob - - actionConnectors map[string]*storedActionConnector - automationJobs map[string]*storedAutomationJob - flows map[string]*storedFlow + oauthClientApps *store.Table[storedOAuthApp] + identityPropagationConfigs *store.Table[storedIdentityPropagationConfig] + assetBundleExportJobs *store.Table[storedAssetBundleExportJob] + assetBundleImportJobs *store.Table[storedAssetBundleImportJob] + dashboardSnapshotJobs *store.Table[storedDashboardSnapshotJob] + + actionConnectors *store.Table[storedActionConnector] + automationJobs *store.Table[storedAutomationJob] + flows *store.Table[storedFlow] spiceCapacity map[string]string selfUpgradeConfig map[string]string - selfUpgradeRequests map[string]*storedSelfUpgradeRequest + selfUpgradeRequests *store.Table[storedSelfUpgradeRequest] accountID string region string @@ -547,67 +545,43 @@ type InMemoryBackend struct { // NewInMemoryBackend creates a new InMemoryBackend. func NewInMemoryBackend(accountID, region string) *InMemoryBackend { b := &InMemoryBackend{ - accountID: accountID, - region: region, - namespaces: make(map[string]*storedNamespace), - groups: make(map[string]*storedGroup), - groupMembers: make(map[string]bool), - users: make(map[string]*storedUser), - dataSources: make(map[string]*storedDataSource), - dataSets: make(map[string]*storedDataSet), - ingestions: make(map[string]*storedIngestion), - dashboards: make(map[string]*storedDashboard), - analyses: make(map[string]*storedAnalysis), - tags: make(map[string]map[string]string), - folders: make(map[string]*storedFolder), - folderMembers: make(map[string]*storedFolderMember), - templates: make(map[string]*storedTemplate), - themes: make(map[string]*storedTheme), - topics: make(map[string]*storedTopic), - vpcConnections: make(map[string]*storedVPCConnection), - iamPolicyAssignments: make(map[string]*storedIAMPolicyAssignment), - - accountSettings: make(map[string]*storedAccountSettings), - accountSubscriptions: make(map[string]*storedAccountSubscription), - accountCustomizations: make(map[string]*storedAccountCustomization), - ipRestrictions: make(map[string]*storedIPRestriction), - publicSharing: make(map[string]bool), - keyRegistrations: make(map[string][]storedRegisteredKey), - defaultQBusinessApps: make(map[string]*storedDefaultQBusinessApplication), - qPersonalization: make(map[string]string), - qSearchConfig: make(map[string]string), - dashboardsQAConfig: make(map[string]string), - - brands: make(map[string]*storedBrand), - brandAssignments: make(map[string]string), - customPermissions: make(map[string]*storedCustomPermissions), - roleCustomPermissions: make(map[string]string), - roleMemberships: make(map[string]bool), - userCustomPermissions: make(map[string]string), - oauthClientApps: make(map[string]*storedOAuthApp), - identityPropagationConfigs: make(map[string]*storedIdentityPropagationConfig), - assetBundleExportJobs: make(map[string]*storedAssetBundleExportJob), - assetBundleImportJobs: make(map[string]*storedAssetBundleImportJob), - dashboardSnapshotJobs: make(map[string]*storedDashboardSnapshotJob), - - actionConnectors: make(map[string]*storedActionConnector), - automationJobs: make(map[string]*storedAutomationJob), - flows: make(map[string]*storedFlow), - spiceCapacity: make(map[string]string), - - selfUpgradeConfig: make(map[string]string), - selfUpgradeRequests: make(map[string]*storedSelfUpgradeRequest), + accountID: accountID, + region: region, + registry: store.NewRegistry(), + + groupMembers: make(map[string]bool), + tags: make(map[string]map[string]string), + + accountSettings: make(map[string]*storedAccountSettings), + accountSubscriptions: make(map[string]*storedAccountSubscription), + ipRestrictions: make(map[string]*storedIPRestriction), + publicSharing: make(map[string]bool), + keyRegistrations: make(map[string][]storedRegisteredKey), + defaultQBusinessApps: make(map[string]*storedDefaultQBusinessApplication), + qPersonalization: make(map[string]string), + qSearchConfig: make(map[string]string), + dashboardsQAConfig: make(map[string]string), + + brandAssignments: make(map[string]string), + roleCustomPermissions: make(map[string]string), + roleMemberships: make(map[string]bool), + userCustomPermissions: make(map[string]string), + + spiceCapacity: make(map[string]string), + + selfUpgradeConfig: make(map[string]string), } b.mu = lockmetrics.New("quicksight") + registerAllTables(b) // Pre-create the default namespace so basic operations work without explicit setup. - b.namespaces[nsKey(accountID, defaultNamespace)] = &storedNamespace{ + b.namespaces.Put(&storedNamespace{ Name: defaultNamespace, Arn: b.buildARN("namespace", defaultNamespace), CapacityRegion: region, Status: statusCreationSuccessful, IdentityStore: identityStoreQuickSight, - } + }) return b } @@ -623,27 +597,13 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.namespaces = make(map[string]*storedNamespace) - b.groups = make(map[string]*storedGroup) + b.registry.ResetAll() + b.groupMembers = make(map[string]bool) - b.users = make(map[string]*storedUser) - b.dataSources = make(map[string]*storedDataSource) - b.dataSets = make(map[string]*storedDataSet) - b.ingestions = make(map[string]*storedIngestion) - b.dashboards = make(map[string]*storedDashboard) - b.analyses = make(map[string]*storedAnalysis) b.tags = make(map[string]map[string]string) - b.folders = make(map[string]*storedFolder) - b.folderMembers = make(map[string]*storedFolderMember) - b.templates = make(map[string]*storedTemplate) - b.themes = make(map[string]*storedTheme) - b.topics = make(map[string]*storedTopic) - b.vpcConnections = make(map[string]*storedVPCConnection) - b.iamPolicyAssignments = make(map[string]*storedIAMPolicyAssignment) b.accountSettings = make(map[string]*storedAccountSettings) b.accountSubscriptions = make(map[string]*storedAccountSubscription) - b.accountCustomizations = make(map[string]*storedAccountCustomization) b.accountCustomPermissions = make(map[string]string) b.ipRestrictions = make(map[string]*storedIPRestriction) b.publicSharing = make(map[string]bool) @@ -653,62 +613,50 @@ func (b *InMemoryBackend) Reset() { b.qSearchConfig = make(map[string]string) b.dashboardsQAConfig = make(map[string]string) - b.brands = make(map[string]*storedBrand) b.brandAssignments = make(map[string]string) - b.customPermissions = make(map[string]*storedCustomPermissions) b.roleCustomPermissions = make(map[string]string) b.roleMemberships = make(map[string]bool) b.userCustomPermissions = make(map[string]string) - b.oauthClientApps = make(map[string]*storedOAuthApp) - b.identityPropagationConfigs = make(map[string]*storedIdentityPropagationConfig) - b.assetBundleExportJobs = make(map[string]*storedAssetBundleExportJob) - b.assetBundleImportJobs = make(map[string]*storedAssetBundleImportJob) - b.dashboardSnapshotJobs = make(map[string]*storedDashboardSnapshotJob) - - b.actionConnectors = make(map[string]*storedActionConnector) - b.automationJobs = make(map[string]*storedAutomationJob) - b.flows = make(map[string]*storedFlow) + b.spiceCapacity = make(map[string]string) b.selfUpgradeConfig = make(map[string]string) - b.selfUpgradeRequests = make(map[string]*storedSelfUpgradeRequest) - b.namespaces[nsKey(b.accountID, defaultNamespace)] = &storedNamespace{ + b.namespaces.Put(&storedNamespace{ Name: defaultNamespace, Arn: b.buildARN("namespace", defaultNamespace), CapacityRegion: b.region, Status: statusCreationSuccessful, IdentityStore: identityStoreQuickSight, - } + }) } // Snapshot serializes backend state to JSON. +// It implements persistence.Persistable. func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() - s := state{ - Namespaces: b.namespaces, - Groups: b.groups, - GroupMembers: b.groupMembers, - Users: b.users, - DataSources: b.dataSources, - DataSets: b.dataSets, - Ingestions: b.ingestions, - Dashboards: b.dashboards, - Analyses: b.analyses, - Tags: b.tags, - Folders: b.folders, - FolderMembers: b.folderMembers, - Templates: b.templates, - Themes: b.themes, - Topics: b.topics, - VPCConnections: b.vpcConnections, - IAMPolicyAssignments: b.iamPolicyAssignments, + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "quicksight: snapshot table marshal failed", "error", err) + + return nil + } + + snap := backendSnapshot{ + Version: quicksightSnapshotVersion, + Tables: tables, + + GroupMembers: b.groupMembers, + Tags: b.tags, AccountSettings: b.accountSettings, AccountSubscriptions: b.accountSubscriptions, - AccountCustomizations: b.accountCustomizations, AccountCustomPermissions: b.accountCustomPermissions, IPRestrictions: b.ipRestrictions, PublicSharing: b.publicSharing, @@ -718,143 +666,126 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { QSearchConfig: b.qSearchConfig, DashboardsQAConfig: b.dashboardsQAConfig, - Brands: b.brands, - BrandAssignments: b.brandAssignments, - CustomPermissions: b.customPermissions, - RoleCustomPermissions: b.roleCustomPermissions, - RoleMemberships: b.roleMemberships, - UserCustomPermissions: b.userCustomPermissions, - OAuthClientApps: b.oauthClientApps, - IdentityPropagationConfigs: b.identityPropagationConfigs, - AssetBundleExportJobs: b.assetBundleExportJobs, - AssetBundleImportJobs: b.assetBundleImportJobs, - DashboardSnapshotJobs: b.dashboardSnapshotJobs, - - ActionConnectors: b.actionConnectors, - AutomationJobs: b.automationJobs, - Flows: b.flows, - SPICECapacity: b.spiceCapacity, + BrandAssignments: b.brandAssignments, + RoleCustomPermissions: b.roleCustomPermissions, + RoleMemberships: b.roleMemberships, + UserCustomPermissions: b.userCustomPermissions, - SelfUpgradeConfig: b.selfUpgradeConfig, - SelfUpgradeRequests: b.selfUpgradeRequests, + SPICECapacity: b.spiceCapacity, + SelfUpgradeConfig: b.selfUpgradeConfig, } - return persistence.MarshalSnapshot(ctx, "quicksight", s) + return persistence.MarshalSnapshot(ctx, "quicksight", snap) } // Restore deserializes backend state from JSON. +// It implements persistence.Persistable. func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { - var s state - if err := persistence.UnmarshalSnapshot(ctx, "quicksight", data, &s); err != nil { + var snap backendSnapshot + if err := persistence.UnmarshalSnapshot(ctx, "quicksight", data, &snap); err != nil { return fmt.Errorf("quicksight: restore: %w", err) } b.mu.Lock("Restore") defer b.mu.Unlock() - b.namespaces = s.Namespaces - b.groups = s.Groups - b.groupMembers = s.GroupMembers - b.users = s.Users - b.dataSources = s.DataSources - b.dataSets = s.DataSets - b.ingestions = s.Ingestions - b.dashboards = s.Dashboards - b.analyses = s.Analyses - b.tags = s.Tags - b.folders = s.Folders - b.folderMembers = s.FolderMembers - b.templates = s.Templates - b.themes = s.Themes - b.topics = s.Topics - b.vpcConnections = s.VPCConnections - b.iamPolicyAssignments = s.IAMPolicyAssignments - - b.accountSettings = s.AccountSettings - b.accountSubscriptions = s.AccountSubscriptions - b.accountCustomizations = s.AccountCustomizations - b.accountCustomPermissions = s.AccountCustomPermissions - b.ipRestrictions = s.IPRestrictions - b.publicSharing = s.PublicSharing - b.keyRegistrations = s.KeyRegistrations - b.defaultQBusinessApps = s.DefaultQBusinessApps - b.qPersonalization = s.QPersonalization - b.qSearchConfig = s.QSearchConfig - b.dashboardsQAConfig = s.DashboardsQAConfig - - b.restoreAppendixBatchFields(s) - b.restoreFinalStubFields(s) - b.ensureLegacyResourceMaps() - b.ensureAccountConfigMaps() - b.ensureFinalStubMaps() + if snap.Version != quicksightSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/ec2 (12e611a4) and services/sqs + // (0f09d77c) conversions. + logger.Load(ctx).WarnContext(ctx, + "quicksight: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", quicksightSnapshotVersion) + + b.registry.ResetAll() + b.resetRawMaps() - return nil -} - -// ensureLegacyResourceMaps re-initializes any pre-Appendix-A resource maps -// left nil after Restore (e.g. snapshots taken before those maps existed). -// Split out of Restore purely to keep Restore's statement count in budget. -func (b *InMemoryBackend) ensureLegacyResourceMaps() { - if b.folders == nil { - b.folders = make(map[string]*storedFolder) - } - if b.folderMembers == nil { - b.folderMembers = make(map[string]*storedFolderMember) - } - if b.templates == nil { - b.templates = make(map[string]*storedTemplate) - } - if b.themes == nil { - b.themes = make(map[string]*storedTheme) - } - if b.topics == nil { - b.topics = make(map[string]*storedTopic) - } - if b.vpcConnections == nil { - b.vpcConnections = make(map[string]*storedVPCConnection) + return nil } - if b.iamPolicyAssignments == nil { - b.iamPolicyAssignments = make(map[string]*storedIAMPolicyAssignment) + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("quicksight: restore snapshot tables: %w", err) } + + b.restoreRawMaps(&snap) + + return nil } -// ensureFinalStubMaps re-initializes any maps introduced by the final -// canned-stub batch (action connectors, automation jobs, flows, SPICE -// capacity) or the parity-sweep batch (self-upgrade config/requests) left nil -// after Restore (e.g. snapshots taken before those maps existed). -func (b *InMemoryBackend) ensureFinalStubMaps() { - if b.actionConnectors == nil { - b.actionConnectors = make(map[string]*storedActionConnector) - } - if b.automationJobs == nil { - b.automationJobs = make(map[string]*storedAutomationJob) - } - if b.flows == nil { - b.flows = make(map[string]*storedFlow) - } - if b.spiceCapacity == nil { - b.spiceCapacity = make(map[string]string) - } - if b.selfUpgradeConfig == nil { - b.selfUpgradeConfig = make(map[string]string) - } - if b.selfUpgradeRequests == nil { - b.selfUpgradeRequests = make(map[string]*storedSelfUpgradeRequest) - } +// resetRawMaps re-initializes every raw (non-store.Table) map to empty. Used +// when Restore discards an incompatible snapshot version. +func (b *InMemoryBackend) resetRawMaps() { + b.groupMembers = make(map[string]bool) + b.tags = make(map[string]map[string]string) + + b.accountSettings = make(map[string]*storedAccountSettings) + b.accountSubscriptions = make(map[string]*storedAccountSubscription) + b.accountCustomPermissions = make(map[string]string) + b.ipRestrictions = make(map[string]*storedIPRestriction) + b.publicSharing = make(map[string]bool) + b.keyRegistrations = make(map[string][]storedRegisteredKey) + b.defaultQBusinessApps = make(map[string]*storedDefaultQBusinessApplication) + b.qPersonalization = make(map[string]string) + b.qSearchConfig = make(map[string]string) + b.dashboardsQAConfig = make(map[string]string) + + b.brandAssignments = make(map[string]string) + b.roleCustomPermissions = make(map[string]string) + b.roleMemberships = make(map[string]bool) + b.userCustomPermissions = make(map[string]string) + + b.spiceCapacity = make(map[string]string) + b.selfUpgradeConfig = make(map[string]string) } -// ensureAccountConfigMaps re-initializes any account/config-cluster maps left nil -// after Restore (e.g. snapshots taken before those maps existed). -func (b *InMemoryBackend) ensureAccountConfigMaps() { +// restoreRawMaps copies every raw (non-store.Table) map from snap onto b, +// then fills in any left nil (e.g. a snapshot taken before that field +// existed) via ensureRawMapsInitialized. Split out of Restore, and split +// again from the nil-check pass below, purely to keep each function's +// cyclomatic complexity in budget. +func (b *InMemoryBackend) restoreRawMaps(snap *backendSnapshot) { + b.groupMembers = snap.GroupMembers + b.tags = snap.Tags + b.accountSettings = snap.AccountSettings + b.accountSubscriptions = snap.AccountSubscriptions + b.accountCustomPermissions = snap.AccountCustomPermissions + b.ipRestrictions = snap.IPRestrictions + b.publicSharing = snap.PublicSharing + b.keyRegistrations = snap.KeyRegistrations + b.defaultQBusinessApps = snap.DefaultQBusinessApps + b.qPersonalization = snap.QPersonalization + b.qSearchConfig = snap.QSearchConfig + b.dashboardsQAConfig = snap.DashboardsQAConfig + b.brandAssignments = snap.BrandAssignments + b.roleCustomPermissions = snap.RoleCustomPermissions + b.roleMemberships = snap.RoleMemberships + b.userCustomPermissions = snap.UserCustomPermissions + b.spiceCapacity = snap.SPICECapacity + b.selfUpgradeConfig = snap.SelfUpgradeConfig + + b.ensureRawMapsInitialized() +} + +// ensureRawMapsInitialized re-initializes any raw (non-store.Table) map left +// nil after restoreRawMaps copied a snapshot (e.g. a snapshot taken before +// that field existed). +func (b *InMemoryBackend) ensureRawMapsInitialized() { + if b.groupMembers == nil { + b.groupMembers = make(map[string]bool) + } + if b.tags == nil { + b.tags = make(map[string]map[string]string) + } if b.accountSettings == nil { b.accountSettings = make(map[string]*storedAccountSettings) } if b.accountSubscriptions == nil { b.accountSubscriptions = make(map[string]*storedAccountSubscription) } - if b.accountCustomizations == nil { - b.accountCustomizations = make(map[string]*storedAccountCustomization) - } if b.accountCustomPermissions == nil { b.accountCustomPermissions = make(map[string]string) } @@ -879,54 +810,17 @@ func (b *InMemoryBackend) ensureAccountConfigMaps() { if b.dashboardsQAConfig == nil { b.dashboardsQAConfig = make(map[string]string) } - b.ensureAppendixBatchMaps() -} - -// restoreAppendixBatchFields copies the final Appendix-A batch's fields -// (brands, custom permissions, OAuth apps, identity propagation, asset -// bundle/snapshot jobs, refresh schedules) from a deserialized snapshot onto b. -// Split out of Restore purely to keep Restore's statement count in budget. -func (b *InMemoryBackend) restoreAppendixBatchFields(s state) { - b.brands = s.Brands - b.brandAssignments = s.BrandAssignments - b.customPermissions = s.CustomPermissions - b.roleCustomPermissions = s.RoleCustomPermissions - b.roleMemberships = s.RoleMemberships - b.userCustomPermissions = s.UserCustomPermissions - b.oauthClientApps = s.OAuthClientApps - b.identityPropagationConfigs = s.IdentityPropagationConfigs - b.assetBundleExportJobs = s.AssetBundleExportJobs - b.assetBundleImportJobs = s.AssetBundleImportJobs - b.dashboardSnapshotJobs = s.DashboardSnapshotJobs -} - -// restoreFinalStubFields copies the final canned-stub batch's fields (action -// connectors, automation jobs, flows, SPICE capacity) from a deserialized -// snapshot onto b. Split out of Restore purely to keep Restore's statement -// count in budget. -func (b *InMemoryBackend) restoreFinalStubFields(s state) { - b.actionConnectors = s.ActionConnectors - b.automationJobs = s.AutomationJobs - b.flows = s.Flows - b.spiceCapacity = s.SPICECapacity - b.selfUpgradeConfig = s.SelfUpgradeConfig - b.selfUpgradeRequests = s.SelfUpgradeRequests -} - -// ensureAppendixBatchMaps re-initializes any maps introduced by the final -// Appendix-A batch (brands, custom permissions, OAuth apps, identity -// propagation, asset bundle/snapshot jobs, refresh schedules) left nil after -// Restore (e.g. snapshots taken before those maps existed). -func (b *InMemoryBackend) ensureAppendixBatchMaps() { - if b.brands == nil { - b.brands = make(map[string]*storedBrand) - } + b.ensureRoleAndSelfUpgradeMapsInitialized() +} + +// ensureRoleAndSelfUpgradeMapsInitialized re-initializes the brand/role/user +// custom-permission and SPICE/self-upgrade raw maps left nil after +// restoreRawMaps. Split out of ensureRawMapsInitialized purely to keep its +// cyclomatic complexity in budget. +func (b *InMemoryBackend) ensureRoleAndSelfUpgradeMapsInitialized() { if b.brandAssignments == nil { b.brandAssignments = make(map[string]string) } - if b.customPermissions == nil { - b.customPermissions = make(map[string]*storedCustomPermissions) - } if b.roleCustomPermissions == nil { b.roleCustomPermissions = make(map[string]string) } @@ -936,20 +830,11 @@ func (b *InMemoryBackend) ensureAppendixBatchMaps() { if b.userCustomPermissions == nil { b.userCustomPermissions = make(map[string]string) } - if b.oauthClientApps == nil { - b.oauthClientApps = make(map[string]*storedOAuthApp) - } - if b.identityPropagationConfigs == nil { - b.identityPropagationConfigs = make(map[string]*storedIdentityPropagationConfig) - } - if b.assetBundleExportJobs == nil { - b.assetBundleExportJobs = make(map[string]*storedAssetBundleExportJob) - } - if b.assetBundleImportJobs == nil { - b.assetBundleImportJobs = make(map[string]*storedAssetBundleImportJob) + if b.spiceCapacity == nil { + b.spiceCapacity = make(map[string]string) } - if b.dashboardSnapshotJobs == nil { - b.dashboardSnapshotJobs = make(map[string]*storedDashboardSnapshotJob) + if b.selfUpgradeConfig == nil { + b.selfUpgradeConfig = make(map[string]string) } } @@ -1064,7 +949,7 @@ func (b *InMemoryBackend) CreateNamespace(accountID, namespace, capacityRegion s defer b.mu.Unlock() key := nsKey(accountID, namespace) - if _, exists := b.namespaces[key]; exists { + if b.namespaces.Has(key) { return nil, ErrNamespaceAlreadyExists } @@ -1075,7 +960,7 @@ func (b *InMemoryBackend) CreateNamespace(accountID, namespace, capacityRegion s Status: statusCreationSuccessful, IdentityStore: identityStoreQuickSight, } - b.namespaces[key] = ns + b.namespaces.Put(ns) return ns.toNamespace(), nil } @@ -1084,7 +969,7 @@ func (b *InMemoryBackend) DescribeNamespace(accountID, namespace string) (*Names b.mu.RLock("DescribeNamespace") defer b.mu.RUnlock() - ns, ok := b.namespaces[nsKey(accountID, namespace)] + ns, ok := b.namespaces.Get(nsKey(accountID, namespace)) if !ok { return nil, ErrNamespaceNotFound } @@ -1101,30 +986,22 @@ func (b *InMemoryBackend) DeleteNamespace(accountID, namespace string) error { defer b.mu.Unlock() key := nsKey(accountID, namespace) - if _, ok := b.namespaces[key]; !ok { + if !b.namespaces.Delete(key) { return ErrNamespaceNotFound } - delete(b.namespaces, key) - return nil } func (b *InMemoryBackend) ListNamespaces( - accountID string, + _ string, maxResults int32, nextToken string, ) ([]*Namespace, string, error) { b.mu.RLock("ListNamespaces") defer b.mu.RUnlock() - var all []*storedNamespace - prefix := accountID + "/" - for k, ns := range b.namespaces { - if strings.HasPrefix(k, prefix) { - all = append(all, ns) - } - } + all := b.namespaces.All() result, next := paginateNamespaces(all, maxResults, nextToken) @@ -1172,12 +1049,12 @@ func (b *InMemoryBackend) CreateGroup(accountID, namespace, groupName, descripti b.mu.Lock("CreateGroup") defer b.mu.Unlock() - if _, ok := b.namespaces[nsKey(accountID, namespace)]; !ok { + if !b.namespaces.Has(nsKey(accountID, namespace)) { return nil, ErrNamespaceNotFound } key := groupKey(accountID, namespace, groupName) - if _, exists := b.groups[key]; exists { + if b.groups.Has(key) { return nil, ErrGroupAlreadyExists } @@ -1188,7 +1065,7 @@ func (b *InMemoryBackend) CreateGroup(accountID, namespace, groupName, descripti Namespace: namespace, PrincipalID: uuid.New().String(), } - b.groups[key] = g + b.groups.Put(g) return g.toGroup(), nil } @@ -1197,7 +1074,7 @@ func (b *InMemoryBackend) DescribeGroup(accountID, namespace, groupName string) b.mu.RLock("DescribeGroup") defer b.mu.RUnlock() - g, ok := b.groups[groupKey(accountID, namespace, groupName)] + g, ok := b.groups.Get(groupKey(accountID, namespace, groupName)) if !ok { return nil, ErrGroupNotFound } @@ -1210,7 +1087,7 @@ func (b *InMemoryBackend) UpdateGroup(accountID, namespace, groupName, descripti defer b.mu.Unlock() key := groupKey(accountID, namespace, groupName) - g, ok := b.groups[key] + g, ok := b.groups.Get(key) if !ok { return nil, ErrGroupNotFound } @@ -1225,12 +1102,10 @@ func (b *InMemoryBackend) DeleteGroup(accountID, namespace, groupName string) er defer b.mu.Unlock() key := groupKey(accountID, namespace, groupName) - if _, ok := b.groups[key]; !ok { + if !b.groups.Delete(key) { return ErrGroupNotFound } - delete(b.groups, key) - // Remove all memberships for this group. prefix := groupKey(accountID, namespace, groupName) + "/" for k := range b.groupMembers { @@ -1243,20 +1118,16 @@ func (b *InMemoryBackend) DeleteGroup(accountID, namespace, groupName string) er } func (b *InMemoryBackend) ListGroups( - accountID, namespace string, + _, namespace string, maxResults int32, nextToken string, ) ([]*Group, string, error) { b.mu.RLock("ListGroups") defer b.mu.RUnlock() - prefix := groupKey(accountID, namespace, "") + "/" - // prefix for groups in this namespace: "accountID/namespace/" - nsPrefix := accountID + "/" + namespace + "/" var all []*storedGroup - for k, g := range b.groups { - if strings.HasPrefix(k, nsPrefix) && !strings.Contains(k[len(nsPrefix):], "/") { - _ = prefix + for _, g := range b.groups.All() { + if g.Namespace == namespace { all = append(all, g) } } @@ -1267,20 +1138,18 @@ func (b *InMemoryBackend) ListGroups( } func (b *InMemoryBackend) SearchGroups( - accountID, namespace, query string, + _, namespace, query string, maxResults int32, nextToken string, ) ([]*Group, string, error) { b.mu.RLock("SearchGroups") defer b.mu.RUnlock() - nsPrefix := accountID + "/" + namespace + "/" var all []*storedGroup - for k, g := range b.groups { - if strings.HasPrefix(k, nsPrefix) && !strings.Contains(k[len(nsPrefix):], "/") { - if query == "" || strings.Contains(strings.ToLower(g.GroupName), strings.ToLower(query)) { - all = append(all, g) - } + for _, g := range b.groups.All() { + if g.Namespace == namespace && + (query == "" || strings.Contains(strings.ToLower(g.GroupName), strings.ToLower(query))) { + all = append(all, g) } } @@ -1329,7 +1198,7 @@ func (b *InMemoryBackend) CreateGroupMembership( b.mu.Lock("CreateGroupMembership") defer b.mu.Unlock() - if _, ok := b.groups[groupKey(accountID, namespace, groupName)]; !ok { + if !b.groups.Has(groupKey(accountID, namespace, groupName)) { return nil, ErrGroupNotFound } @@ -1384,7 +1253,7 @@ func (b *InMemoryBackend) ListGroupMemberships( b.mu.RLock("ListGroupMemberships") defer b.mu.RUnlock() - if _, ok := b.groups[groupKey(accountID, namespace, groupName)]; !ok { + if !b.groups.Has(groupKey(accountID, namespace, groupName)) { return nil, "", ErrGroupNotFound } @@ -1444,12 +1313,12 @@ func (b *InMemoryBackend) RegisterUser( b.mu.Lock("RegisterUser") defer b.mu.Unlock() - if _, ok := b.namespaces[nsKey(accountID, namespace)]; !ok { + if !b.namespaces.Has(nsKey(accountID, namespace)) { return nil, ErrNamespaceNotFound } key := userKey(accountID, namespace, userName) - if _, exists := b.users[key]; exists { + if b.users.Has(key) { return nil, ErrUserAlreadyExists } @@ -1471,7 +1340,7 @@ func (b *InMemoryBackend) RegisterUser( SessionName: sessionName, Active: true, } - b.users[key] = u + b.users.Put(u) return u.toUser(), nil } @@ -1480,7 +1349,7 @@ func (b *InMemoryBackend) DescribeUser(accountID, namespace, userName string) (* b.mu.RLock("DescribeUser") defer b.mu.RUnlock() - u, ok := b.users[userKey(accountID, namespace, userName)] + u, ok := b.users.Get(userKey(accountID, namespace, userName)) if !ok { return nil, ErrUserNotFound } @@ -1493,7 +1362,7 @@ func (b *InMemoryBackend) UpdateUser(accountID, namespace, userName, email, role defer b.mu.Unlock() key := userKey(accountID, namespace, userName) - u, ok := b.users[key] + u, ok := b.users.Get(key) if !ok { return nil, ErrUserNotFound } @@ -1513,12 +1382,10 @@ func (b *InMemoryBackend) DeleteUser(accountID, namespace, userName string) erro defer b.mu.Unlock() key := userKey(accountID, namespace, userName) - if _, ok := b.users[key]; !ok { + if !b.users.Delete(key) { return ErrUserNotFound } - delete(b.users, key) - return nil } @@ -1526,10 +1393,9 @@ func (b *InMemoryBackend) DeleteUserByPrincipalID(accountID, namespace, principa b.mu.Lock("DeleteUserByPrincipalID") defer b.mu.Unlock() - prefix := accountID + "/" + namespace + "/" - for k, u := range b.users { - if strings.HasPrefix(k, prefix) && u.PrincipalID == principalID { - delete(b.users, k) + for _, u := range b.users.All() { + if u.Namespace == namespace && u.PrincipalID == principalID { + b.users.Delete(userKey(accountID, namespace, u.UserName)) return nil } @@ -1540,17 +1406,16 @@ func (b *InMemoryBackend) DeleteUserByPrincipalID(accountID, namespace, principa //nolint:dupl // list functions share structure but operate on different stored types func (b *InMemoryBackend) ListUsers( - accountID, namespace string, + _, namespace string, maxResults int32, nextToken string, ) ([]*User, string, error) { b.mu.RLock("ListUsers") defer b.mu.RUnlock() - prefix := accountID + "/" + namespace + "/" var all []*storedUser - for k, u := range b.users { - if strings.HasPrefix(k, prefix) { + for _, u := range b.users.All() { + if u.Namespace == namespace { all = append(all, u) } } @@ -1594,14 +1459,13 @@ func (b *InMemoryBackend) ListUserGroups( b.mu.RLock("ListUserGroups") defer b.mu.RUnlock() - if _, ok := b.users[userKey(accountID, namespace, userName)]; !ok { + if !b.users.Has(userKey(accountID, namespace, userName)) { return nil, "", ErrUserNotFound } - nsPrefix := accountID + "/" + namespace + "/" var all []*storedGroup - for gKey, g := range b.groups { - if !strings.HasPrefix(gKey, nsPrefix) { + for _, g := range b.groups.All() { + if g.Namespace != namespace { continue } memberKey := groupMemberKey(accountID, namespace, g.GroupName, userName) @@ -1630,7 +1494,7 @@ func (b *InMemoryBackend) CreateDataSource( defer b.mu.Unlock() key := dataSourceKey(accountID, dataSourceID) - if _, exists := b.dataSources[key]; exists { + if b.dataSources.Has(key) { return nil, ErrDataSourceAlreadyExists } @@ -1645,7 +1509,7 @@ func (b *InMemoryBackend) CreateDataSource( Status: statusCreationSuccessful, Permissions: clonePermissions(permissions), } - b.dataSources[key] = ds + b.dataSources.Put(ds) if len(tags) > 0 { b.tags[ds.Arn] = maps.Clone(tags) @@ -1658,7 +1522,7 @@ func (b *InMemoryBackend) DescribeDataSource(accountID, dataSourceID string) (*D b.mu.RLock("DescribeDataSource") defer b.mu.RUnlock() - ds, ok := b.dataSources[dataSourceKey(accountID, dataSourceID)] + ds, ok := b.dataSources.Get(dataSourceKey(accountID, dataSourceID)) if !ok { return nil, ErrDataSourceNotFound } @@ -1671,7 +1535,7 @@ func (b *InMemoryBackend) UpdateDataSource(accountID, dataSourceID, name string) defer b.mu.Unlock() key := dataSourceKey(accountID, dataSourceID) - ds, ok := b.dataSources[key] + ds, ok := b.dataSources.Get(key) if !ok { return nil, ErrDataSourceNotFound } @@ -1690,33 +1554,27 @@ func (b *InMemoryBackend) DeleteDataSource(accountID, dataSourceID string) error defer b.mu.Unlock() key := dataSourceKey(accountID, dataSourceID) - ds, ok := b.dataSources[key] + ds, ok := b.dataSources.Get(key) if !ok { return ErrDataSourceNotFound } delete(b.tags, ds.Arn) - delete(b.dataSources, key) + b.dataSources.Delete(key) return nil } //nolint:dupl // list functions share structure but operate on different stored types func (b *InMemoryBackend) ListDataSources( - accountID string, + _ string, maxResults int32, nextToken string, ) ([]*DataSource, string, error) { b.mu.RLock("ListDataSources") defer b.mu.RUnlock() - prefix := accountID + "/" - var all []*storedDataSource - for k, ds := range b.dataSources { - if strings.HasPrefix(k, prefix) { - all = append(all, ds) - } - } + all := b.dataSources.All() if maxResults <= 0 || maxResults > defaultMaxResults { maxResults = defaultMaxResults @@ -1756,7 +1614,7 @@ func (b *InMemoryBackend) ListDataSources( // //nolint:dupl // search functions share structure but operate on different stored types func (b *InMemoryBackend) SearchDataSources( - accountID string, + _ string, filters []SearchFilter, maxResults int32, nextToken string, @@ -1764,10 +1622,9 @@ func (b *InMemoryBackend) SearchDataSources( b.mu.RLock("SearchDataSources") defer b.mu.RUnlock() - prefix := accountID + "/" var filtered []*storedDataSource - for k, ds := range b.dataSources { - if strings.HasPrefix(k, prefix) && matchesAllNameFilters(ds.Name, filters, filterDataSourceName) { + for _, ds := range b.dataSources.All() { + if matchesAllNameFilters(ds.Name, filters, filterDataSourceName) { filtered = append(filtered, ds) } } @@ -1812,7 +1669,7 @@ func (b *InMemoryBackend) DescribeDataSourcePermissions( b.mu.RLock("DescribeDataSourcePermissions") defer b.mu.RUnlock() - ds, ok := b.dataSources[dataSourceKey(accountID, dataSourceID)] + ds, ok := b.dataSources.Get(dataSourceKey(accountID, dataSourceID)) if !ok { return nil, nil, ErrDataSourceNotFound } @@ -1827,7 +1684,7 @@ func (b *InMemoryBackend) UpdateDataSourcePermissions( b.mu.Lock("UpdateDataSourcePermissions") defer b.mu.Unlock() - ds, ok := b.dataSources[dataSourceKey(accountID, dataSourceID)] + ds, ok := b.dataSources.Get(dataSourceKey(accountID, dataSourceID)) if !ok { return nil, nil, ErrDataSourceNotFound } @@ -1853,7 +1710,7 @@ func (b *InMemoryBackend) CreateDataSet( defer b.mu.Unlock() key := dataSetKey(accountID, dataSetID) - if _, exists := b.dataSets[key]; exists { + if b.dataSets.Has(key) { return nil, ErrDataSetAlreadyExists } @@ -1872,7 +1729,7 @@ func (b *InMemoryBackend) CreateDataSet( RefreshSchedules: make(map[string]*storedRefreshSchedule), Permissions: clonePermissions(permissions), } - b.dataSets[key] = ds + b.dataSets.Put(ds) if len(tags) > 0 { b.tags[ds.Arn] = maps.Clone(tags) @@ -1885,7 +1742,7 @@ func (b *InMemoryBackend) DescribeDataSet(accountID, dataSetID string) (*DataSet b.mu.RLock("DescribeDataSet") defer b.mu.RUnlock() - ds, ok := b.dataSets[dataSetKey(accountID, dataSetID)] + ds, ok := b.dataSets.Get(dataSetKey(accountID, dataSetID)) if !ok { return nil, ErrDataSetNotFound } @@ -1898,7 +1755,7 @@ func (b *InMemoryBackend) UpdateDataSet(accountID, dataSetID, name, importMode s defer b.mu.Unlock() key := dataSetKey(accountID, dataSetID) - ds, ok := b.dataSets[key] + ds, ok := b.dataSets.Get(key) if !ok { return nil, ErrDataSetNotFound } @@ -1919,33 +1776,27 @@ func (b *InMemoryBackend) DeleteDataSet(accountID, dataSetID string) error { defer b.mu.Unlock() key := dataSetKey(accountID, dataSetID) - ds, ok := b.dataSets[key] + ds, ok := b.dataSets.Get(key) if !ok { return ErrDataSetNotFound } delete(b.tags, ds.Arn) - delete(b.dataSets, key) + b.dataSets.Delete(key) return nil } //nolint:dupl // list functions share structure but operate on different stored types func (b *InMemoryBackend) ListDataSets( - accountID string, + _ string, maxResults int32, nextToken string, ) ([]*DataSet, string, error) { b.mu.RLock("ListDataSets") defer b.mu.RUnlock() - prefix := accountID + "/" - var all []*storedDataSet - for k, ds := range b.dataSets { - if strings.HasPrefix(k, prefix) { - all = append(all, ds) - } - } + all := b.dataSets.All() if maxResults <= 0 || maxResults > defaultMaxResults { maxResults = defaultMaxResults @@ -1984,7 +1835,7 @@ func (b *InMemoryBackend) ListDataSets( // //nolint:dupl // search functions share structure but operate on different stored types func (b *InMemoryBackend) SearchDataSets( - accountID string, + _ string, filters []SearchFilter, maxResults int32, nextToken string, @@ -1992,10 +1843,9 @@ func (b *InMemoryBackend) SearchDataSets( b.mu.RLock("SearchDataSets") defer b.mu.RUnlock() - prefix := accountID + "/" var filtered []*storedDataSet - for k, ds := range b.dataSets { - if strings.HasPrefix(k, prefix) && matchesAllNameFilters(ds.Name, filters, filterDataSetName) { + for _, ds := range b.dataSets.All() { + if matchesAllNameFilters(ds.Name, filters, filterDataSetName) { filtered = append(filtered, ds) } } @@ -2040,7 +1890,7 @@ func (b *InMemoryBackend) DescribeDataSetPermissions( b.mu.RLock("DescribeDataSetPermissions") defer b.mu.RUnlock() - ds, ok := b.dataSets[dataSetKey(accountID, dataSetID)] + ds, ok := b.dataSets.Get(dataSetKey(accountID, dataSetID)) if !ok { return nil, nil, ErrDataSetNotFound } @@ -2055,7 +1905,7 @@ func (b *InMemoryBackend) UpdateDataSetPermissions( b.mu.Lock("UpdateDataSetPermissions") defer b.mu.Unlock() - ds, ok := b.dataSets[dataSetKey(accountID, dataSetID)] + ds, ok := b.dataSets.Get(dataSetKey(accountID, dataSetID)) if !ok { return nil, nil, ErrDataSetNotFound } @@ -2072,12 +1922,12 @@ func (b *InMemoryBackend) CreateIngestion(accountID, dataSetID, ingestionID stri b.mu.Lock("CreateIngestion") defer b.mu.Unlock() - if _, ok := b.dataSets[dataSetKey(accountID, dataSetID)]; !ok { + if !b.dataSets.Has(dataSetKey(accountID, dataSetID)) { return nil, ErrDataSetNotFound } key := ingestionKey(accountID, dataSetID, ingestionID) - if _, exists := b.ingestions[key]; exists { + if b.ingestions.Has(key) { return nil, ErrIngestionAlreadyExists } @@ -2094,7 +1944,7 @@ func (b *InMemoryBackend) CreateIngestion(accountID, dataSetID, ingestionID stri DataSetID: dataSetID, IngestionStatus: statusRunning, } - b.ingestions[key] = ing + b.ingestions.Put(ing) return ing.toIngestion(), nil } @@ -2103,7 +1953,7 @@ func (b *InMemoryBackend) DescribeIngestion(accountID, dataSetID, ingestionID st b.mu.RLock("DescribeIngestion") defer b.mu.RUnlock() - ing, ok := b.ingestions[ingestionKey(accountID, dataSetID, ingestionID)] + ing, ok := b.ingestions.Get(ingestionKey(accountID, dataSetID, ingestionID)) if !ok { return nil, ErrIngestionNotFound } @@ -2116,7 +1966,7 @@ func (b *InMemoryBackend) CancelIngestion(accountID, dataSetID, ingestionID stri defer b.mu.Unlock() key := ingestionKey(accountID, dataSetID, ingestionID) - ing, ok := b.ingestions[key] + ing, ok := b.ingestions.Get(key) if !ok { return ErrIngestionNotFound } @@ -2128,17 +1978,16 @@ func (b *InMemoryBackend) CancelIngestion(accountID, dataSetID, ingestionID stri //nolint:dupl // list functions share structure but operate on different stored types func (b *InMemoryBackend) ListIngestions( - accountID, dataSetID string, + _, dataSetID string, maxResults int32, nextToken string, ) ([]*Ingestion, string, error) { b.mu.RLock("ListIngestions") defer b.mu.RUnlock() - prefix := accountID + "/" + dataSetID + "/" var all []*storedIngestion - for k, ing := range b.ingestions { - if strings.HasPrefix(k, prefix) { + for _, ing := range b.ingestions.All() { + if ing.DataSetID == dataSetID { all = append(all, ing) } } @@ -2190,7 +2039,7 @@ func (b *InMemoryBackend) CreateDashboard( defer b.mu.Unlock() key := dashboardKey(accountID, dashboardID) - if _, exists := b.dashboards[key]; exists { + if b.dashboards.Has(key) { return nil, ErrDashboardAlreadyExists } @@ -2207,7 +2056,7 @@ func (b *InMemoryBackend) CreateDashboard( Definition: definition, Permissions: clonePermissions(permissions), } - b.dashboards[key] = d + b.dashboards.Put(d) if len(tags) > 0 { b.tags[d.Arn] = maps.Clone(tags) @@ -2220,7 +2069,7 @@ func (b *InMemoryBackend) DescribeDashboard(accountID, dashboardID string) (*Das b.mu.RLock("DescribeDashboard") defer b.mu.RUnlock() - d, ok := b.dashboards[dashboardKey(accountID, dashboardID)] + d, ok := b.dashboards.Get(dashboardKey(accountID, dashboardID)) if !ok { return nil, ErrDashboardNotFound } @@ -2236,7 +2085,7 @@ func (b *InMemoryBackend) UpdateDashboard( defer b.mu.Unlock() key := dashboardKey(accountID, dashboardID) - d, ok := b.dashboards[key] + d, ok := b.dashboards.Get(key) if !ok { return nil, ErrDashboardNotFound } @@ -2258,33 +2107,27 @@ func (b *InMemoryBackend) DeleteDashboard(accountID, dashboardID string) error { defer b.mu.Unlock() key := dashboardKey(accountID, dashboardID) - d, ok := b.dashboards[key] + d, ok := b.dashboards.Get(key) if !ok { return ErrDashboardNotFound } delete(b.tags, d.Arn) - delete(b.dashboards, key) + b.dashboards.Delete(key) return nil } //nolint:dupl // list functions share structure but operate on different stored types func (b *InMemoryBackend) ListDashboards( - accountID string, + _ string, maxResults int32, nextToken string, ) ([]*Dashboard, string, error) { b.mu.RLock("ListDashboards") defer b.mu.RUnlock() - prefix := accountID + "/" - var all []*storedDashboard - for k, d := range b.dashboards { - if strings.HasPrefix(k, prefix) { - all = append(all, d) - } - } + all := b.dashboards.All() if maxResults <= 0 || maxResults > defaultMaxResults { maxResults = defaultMaxResults @@ -2325,7 +2168,7 @@ func (b *InMemoryBackend) ListDashboardVersions( b.mu.RLock("ListDashboardVersions") defer b.mu.RUnlock() - d, ok := b.dashboards[dashboardKey(accountID, dashboardID)] + d, ok := b.dashboards.Get(dashboardKey(accountID, dashboardID)) if !ok { return nil, "", ErrDashboardNotFound } @@ -2371,7 +2214,7 @@ func (b *InMemoryBackend) ListDashboardVersions( // //nolint:dupl // search functions share structure but operate on different stored types func (b *InMemoryBackend) SearchDashboards( - accountID string, + _ string, filters []SearchFilter, maxResults int32, nextToken string, @@ -2379,10 +2222,9 @@ func (b *InMemoryBackend) SearchDashboards( b.mu.RLock("SearchDashboards") defer b.mu.RUnlock() - prefix := accountID + "/" var filtered []*storedDashboard - for k, d := range b.dashboards { - if strings.HasPrefix(k, prefix) && matchesAllNameFilters(d.Name, filters, filterDashboardName) { + for _, d := range b.dashboards.All() { + if matchesAllNameFilters(d.Name, filters, filterDashboardName) { filtered = append(filtered, d) } } @@ -2430,7 +2272,7 @@ func (b *InMemoryBackend) UpdateDashboardPublishedVersion( b.mu.Lock("UpdateDashboardPublishedVersion") defer b.mu.Unlock() - d, ok := b.dashboards[dashboardKey(accountID, dashboardID)] + d, ok := b.dashboards.Get(dashboardKey(accountID, dashboardID)) if !ok { return nil, ErrDashboardNotFound } @@ -2453,7 +2295,7 @@ func (b *InMemoryBackend) UpdateDashboardLinks( b.mu.Lock("UpdateDashboardLinks") defer b.mu.Unlock() - d, ok := b.dashboards[dashboardKey(accountID, dashboardID)] + d, ok := b.dashboards.Get(dashboardKey(accountID, dashboardID)) if !ok { return nil, ErrDashboardNotFound } @@ -2472,7 +2314,7 @@ func (b *InMemoryBackend) DescribeDashboardPermissions( b.mu.RLock("DescribeDashboardPermissions") defer b.mu.RUnlock() - d, ok := b.dashboards[dashboardKey(accountID, dashboardID)] + d, ok := b.dashboards.Get(dashboardKey(accountID, dashboardID)) if !ok { return nil, nil, ErrDashboardNotFound } @@ -2487,7 +2329,7 @@ func (b *InMemoryBackend) UpdateDashboardPermissions( b.mu.Lock("UpdateDashboardPermissions") defer b.mu.Unlock() - d, ok := b.dashboards[dashboardKey(accountID, dashboardID)] + d, ok := b.dashboards.Get(dashboardKey(accountID, dashboardID)) if !ok { return nil, nil, ErrDashboardNotFound } @@ -2514,7 +2356,7 @@ func (b *InMemoryBackend) CreateAnalysis( defer b.mu.Unlock() key := analysisKey(accountID, analysisID) - if _, exists := b.analyses[key]; exists { + if b.analyses.Has(key) { return nil, ErrAnalysisAlreadyExists } @@ -2529,7 +2371,7 @@ func (b *InMemoryBackend) CreateAnalysis( Definition: definition, Permissions: clonePermissions(permissions), } - b.analyses[key] = a + b.analyses.Put(a) if len(tags) > 0 { b.tags[a.Arn] = maps.Clone(tags) @@ -2542,7 +2384,7 @@ func (b *InMemoryBackend) DescribeAnalysis(accountID, analysisID string) (*Analy b.mu.RLock("DescribeAnalysis") defer b.mu.RUnlock() - a, ok := b.analyses[analysisKey(accountID, analysisID)] + a, ok := b.analyses.Get(analysisKey(accountID, analysisID)) if !ok { return nil, ErrAnalysisNotFound } @@ -2558,7 +2400,7 @@ func (b *InMemoryBackend) UpdateAnalysis( defer b.mu.Unlock() key := analysisKey(accountID, analysisID) - a, ok := b.analyses[key] + a, ok := b.analyses.Get(key) if !ok { return nil, ErrAnalysisNotFound } @@ -2580,14 +2422,14 @@ func (b *InMemoryBackend) DeleteAnalysis(accountID, analysisID string, forceDele defer b.mu.Unlock() key := analysisKey(accountID, analysisID) - a, ok := b.analyses[key] + a, ok := b.analyses.Get(key) if !ok { return ErrAnalysisNotFound } if forceDeleteWithoutRecovery { delete(b.tags, a.Arn) - delete(b.analyses, key) + b.analyses.Delete(key) } else { a.Status = statusDeleted } @@ -2597,20 +2439,14 @@ func (b *InMemoryBackend) DeleteAnalysis(accountID, analysisID string, forceDele //nolint:dupl // list functions share structure but operate on different stored types func (b *InMemoryBackend) ListAnalyses( - accountID string, + _ string, maxResults int32, nextToken string, ) ([]*Analysis, string, error) { b.mu.RLock("ListAnalyses") defer b.mu.RUnlock() - prefix := accountID + "/" - var all []*storedAnalysis - for k, a := range b.analyses { - if strings.HasPrefix(k, prefix) { - all = append(all, a) - } - } + all := b.analyses.All() if maxResults <= 0 || maxResults > defaultMaxResults { maxResults = defaultMaxResults @@ -2648,7 +2484,7 @@ func (b *InMemoryBackend) RestoreAnalysis(accountID, analysisID string) (*Analys defer b.mu.Unlock() key := analysisKey(accountID, analysisID) - a, ok := b.analyses[key] + a, ok := b.analyses.Get(key) if !ok { return nil, ErrAnalysisNotFound } @@ -2665,7 +2501,7 @@ func (b *InMemoryBackend) RestoreAnalysis(accountID, analysisID string) (*Analys // //nolint:dupl // search functions share structure but operate on different stored types func (b *InMemoryBackend) SearchAnalyses( - accountID string, + _ string, filters []SearchFilter, maxResults int32, nextToken string, @@ -2673,10 +2509,9 @@ func (b *InMemoryBackend) SearchAnalyses( b.mu.RLock("SearchAnalyses") defer b.mu.RUnlock() - prefix := accountID + "/" var filtered []*storedAnalysis - for k, a := range b.analyses { - if strings.HasPrefix(k, prefix) && matchesAllNameFilters(a.Name, filters, filterAnalysisName) { + for _, a := range b.analyses.All() { + if matchesAllNameFilters(a.Name, filters, filterAnalysisName) { filtered = append(filtered, a) } } @@ -2721,7 +2556,7 @@ func (b *InMemoryBackend) DescribeAnalysisPermissions( b.mu.RLock("DescribeAnalysisPermissions") defer b.mu.RUnlock() - a, ok := b.analyses[analysisKey(accountID, analysisID)] + a, ok := b.analyses.Get(analysisKey(accountID, analysisID)) if !ok { return nil, nil, ErrAnalysisNotFound } @@ -2736,7 +2571,7 @@ func (b *InMemoryBackend) UpdateAnalysisPermissions( b.mu.Lock("UpdateAnalysisPermissions") defer b.mu.Unlock() - a, ok := b.analyses[analysisKey(accountID, analysisID)] + a, ok := b.analyses.Get(analysisKey(accountID, analysisID)) if !ok { return nil, nil, ErrAnalysisNotFound } diff --git a/services/quicksight/backend_account.go b/services/quicksight/backend_account.go index 0a70db16b..09780bd21 100644 --- a/services/quicksight/backend_account.go +++ b/services/quicksight/backend_account.go @@ -203,7 +203,7 @@ func (b *InMemoryBackend) CreateAccountCustomization( defer b.mu.Unlock() key := accountCustomizationKey(accountID, namespace) - if _, exists := b.accountCustomizations[key]; exists { + if b.accountCustomizations.Has(key) { return nil, ErrAccountCustomizationAlreadyExists } @@ -212,7 +212,7 @@ func (b *InMemoryBackend) CreateAccountCustomization( DefaultTheme: defaultTheme, DefaultEmailCustomizationTemplate: defaultEmailCustomizationTemplate, } - b.accountCustomizations[key] = c + b.accountCustomizations.Put(c) return c.toAccountCustomization(), nil } @@ -222,7 +222,7 @@ func (b *InMemoryBackend) DescribeAccountCustomization(accountID, namespace stri b.mu.RLock("DescribeAccountCustomization") defer b.mu.RUnlock() - c, ok := b.accountCustomizations[accountCustomizationKey(accountID, namespace)] + c, ok := b.accountCustomizations.Get(accountCustomizationKey(accountID, namespace)) if !ok { return nil, ErrAccountCustomizationNotFound } @@ -238,7 +238,7 @@ func (b *InMemoryBackend) UpdateAccountCustomization( defer b.mu.Unlock() key := accountCustomizationKey(accountID, namespace) - c, ok := b.accountCustomizations[key] + c, ok := b.accountCustomizations.Get(key) if !ok { return nil, ErrAccountCustomizationNotFound } @@ -259,10 +259,9 @@ func (b *InMemoryBackend) DeleteAccountCustomization(accountID, namespace string defer b.mu.Unlock() key := accountCustomizationKey(accountID, namespace) - if _, ok := b.accountCustomizations[key]; !ok { + if !b.accountCustomizations.Delete(key) { return ErrAccountCustomizationNotFound } - delete(b.accountCustomizations, key) return nil } diff --git a/services/quicksight/backend_actionconnector.go b/services/quicksight/backend_actionconnector.go index 74e0dbaab..b47c3fb9f 100644 --- a/services/quicksight/backend_actionconnector.go +++ b/services/quicksight/backend_actionconnector.go @@ -3,7 +3,6 @@ package quicksight import ( "maps" "sort" - "strings" "time" ) @@ -63,7 +62,7 @@ func (b *InMemoryBackend) CreateActionConnector( defer b.mu.Unlock() key := actionConnectorKey(accountID, actionConnectorID) - if _, exists := b.actionConnectors[key]; exists { + if b.actionConnectors.Has(key) { return nil, ErrActionConnectorAlreadyExists } @@ -82,7 +81,7 @@ func (b *InMemoryBackend) CreateActionConnector( Status: statusCreationSuccessful, Permissions: clonePermissions(permissions), } - b.actionConnectors[key] = a + b.actionConnectors.Put(a) if len(tags) > 0 { b.tags[arn] = maps.Clone(tags) @@ -95,7 +94,7 @@ func (b *InMemoryBackend) DescribeActionConnector(accountID, actionConnectorID s b.mu.RLock("DescribeActionConnector") defer b.mu.RUnlock() - a, ok := b.actionConnectors[actionConnectorKey(accountID, actionConnectorID)] + a, ok := b.actionConnectors.Get(actionConnectorKey(accountID, actionConnectorID)) if !ok { return nil, ErrActionConnectorNotFound } @@ -111,7 +110,7 @@ func (b *InMemoryBackend) UpdateActionConnector( defer b.mu.Unlock() key := actionConnectorKey(accountID, actionConnectorID) - a, ok := b.actionConnectors[key] + a, ok := b.actionConnectors.Get(key) if !ok { return nil, ErrActionConnectorNotFound } @@ -139,32 +138,26 @@ func (b *InMemoryBackend) DeleteActionConnector(accountID, actionConnectorID str defer b.mu.Unlock() key := actionConnectorKey(accountID, actionConnectorID) - a, ok := b.actionConnectors[key] + a, ok := b.actionConnectors.Get(key) if !ok { return nil, ErrActionConnectorNotFound } delete(b.tags, a.Arn) - delete(b.actionConnectors, key) + b.actionConnectors.Delete(key) return a.toActionConnector(), nil } func (b *InMemoryBackend) ListActionConnectors( - accountID string, + _ string, maxResults int32, nextToken string, ) ([]*ActionConnector, string, error) { b.mu.RLock("ListActionConnectors") defer b.mu.RUnlock() - prefix := accountID + "/" - var all []*storedActionConnector - for k, a := range b.actionConnectors { - if strings.HasPrefix(k, prefix) { - all = append(all, a) - } - } + all := b.actionConnectors.All() sort.Slice(all, func(i, j int) bool { return all[i].ActionConnectorID < all[j].ActionConnectorID }) result, next := paginateActionConnectors(all, maxResults, nextToken) @@ -173,7 +166,7 @@ func (b *InMemoryBackend) ListActionConnectors( } func (b *InMemoryBackend) SearchActionConnectors( - accountID string, + _ string, filters []SearchFilter, maxResults int32, nextToken string, @@ -181,10 +174,9 @@ func (b *InMemoryBackend) SearchActionConnectors( b.mu.RLock("SearchActionConnectors") defer b.mu.RUnlock() - prefix := accountID + "/" var filtered []*storedActionConnector - for k, a := range b.actionConnectors { - if strings.HasPrefix(k, prefix) && matchesAllNameFilters(a.Name, filters, filterActionConnectorName) { + for _, a := range b.actionConnectors.All() { + if matchesAllNameFilters(a.Name, filters, filterActionConnectorName) { filtered = append(filtered, a) } } @@ -239,7 +231,7 @@ func (b *InMemoryBackend) DescribeActionConnectorPermissions( b.mu.RLock("DescribeActionConnectorPermissions") defer b.mu.RUnlock() - a, ok := b.actionConnectors[actionConnectorKey(accountID, actionConnectorID)] + a, ok := b.actionConnectors.Get(actionConnectorKey(accountID, actionConnectorID)) if !ok { return nil, nil, ErrActionConnectorNotFound } @@ -255,7 +247,7 @@ func (b *InMemoryBackend) UpdateActionConnectorPermissions( defer b.mu.Unlock() key := actionConnectorKey(accountID, actionConnectorID) - a, ok := b.actionConnectors[key] + a, ok := b.actionConnectors.Get(key) if !ok { return nil, nil, ErrActionConnectorNotFound } diff --git a/services/quicksight/backend_assetbundle.go b/services/quicksight/backend_assetbundle.go index 1be18daf7..2348b830a 100644 --- a/services/quicksight/backend_assetbundle.go +++ b/services/quicksight/backend_assetbundle.go @@ -3,7 +3,6 @@ package quicksight import ( "fmt" "sort" - "strings" "time" ) @@ -66,7 +65,7 @@ func (j *storedAssetBundleImportJob) toAssetBundleImportJob() *AssetBundleImport // ---- Asset bundle export jobs ---- func (b *InMemoryBackend) StartAssetBundleExportJob( - accountID, jobID, exportFormat string, + _, jobID, exportFormat string, resourceArns []string, includeAllDependencies bool, ) (*AssetBundleExportJob, error) { @@ -93,7 +92,7 @@ func (b *InMemoryBackend) StartAssetBundleExportJob( ExportFormat: exportFormat, IncludeAllDependencies: includeAllDependencies, } - b.assetBundleExportJobs[assetBundleJobKey(accountID, jobID)] = job + b.assetBundleExportJobs.Put(job) return job.toAssetBundleExportJob(), nil } @@ -102,7 +101,7 @@ func (b *InMemoryBackend) DescribeAssetBundleExportJob(accountID, jobID string) b.mu.Lock("DescribeAssetBundleExportJob") defer b.mu.Unlock() - job, ok := b.assetBundleExportJobs[assetBundleJobKey(accountID, jobID)] + job, ok := b.assetBundleExportJobs.Get(assetBundleJobKey(accountID, jobID)) if !ok { return nil, ErrAssetBundleExportJobNotFound } @@ -122,20 +121,14 @@ func (b *InMemoryBackend) DescribeAssetBundleExportJob(accountID, jobID string) //nolint:dupl // list functions share structure but operate on different stored types func (b *InMemoryBackend) ListAssetBundleExportJobs( - accountID string, + _ string, maxResults int32, nextToken string, ) ([]*AssetBundleExportJob, string, error) { b.mu.RLock("ListAssetBundleExportJobs") defer b.mu.RUnlock() - prefix := accountID + "/" - all := make([]*storedAssetBundleExportJob, 0, len(b.assetBundleExportJobs)) - for k, job := range b.assetBundleExportJobs { - if strings.HasPrefix(k, prefix) { - all = append(all, job) - } - } + all := b.assetBundleExportJobs.All() sort.Slice(all, func(i, j int) bool { return all[i].JobID < all[j].JobID }) if maxResults <= 0 || maxResults > defaultMaxResults { @@ -172,7 +165,7 @@ func (b *InMemoryBackend) ListAssetBundleExportJobs( // ---- Asset bundle import jobs ---- func (b *InMemoryBackend) StartAssetBundleImportJob( - accountID, jobID, failureAction string, + _, jobID, failureAction string, ) (*AssetBundleImportJob, error) { if jobID == "" { return nil, ErrValidation @@ -192,7 +185,7 @@ func (b *InMemoryBackend) StartAssetBundleImportJob( Status: assetBundleJobStatusQueued, FailureAction: failureAction, } - b.assetBundleImportJobs[assetBundleJobKey(accountID, jobID)] = job + b.assetBundleImportJobs.Put(job) return job.toAssetBundleImportJob(), nil } @@ -201,7 +194,7 @@ func (b *InMemoryBackend) DescribeAssetBundleImportJob(accountID, jobID string) b.mu.Lock("DescribeAssetBundleImportJob") defer b.mu.Unlock() - job, ok := b.assetBundleImportJobs[assetBundleJobKey(accountID, jobID)] + job, ok := b.assetBundleImportJobs.Get(assetBundleJobKey(accountID, jobID)) if !ok { return nil, ErrAssetBundleImportJobNotFound } @@ -215,20 +208,14 @@ func (b *InMemoryBackend) DescribeAssetBundleImportJob(accountID, jobID string) //nolint:dupl // list functions share structure but operate on different stored types func (b *InMemoryBackend) ListAssetBundleImportJobs( - accountID string, + _ string, maxResults int32, nextToken string, ) ([]*AssetBundleImportJob, string, error) { b.mu.RLock("ListAssetBundleImportJobs") defer b.mu.RUnlock() - prefix := accountID + "/" - all := make([]*storedAssetBundleImportJob, 0, len(b.assetBundleImportJobs)) - for k, job := range b.assetBundleImportJobs { - if strings.HasPrefix(k, prefix) { - all = append(all, job) - } - } + all := b.assetBundleImportJobs.All() sort.Slice(all, func(i, j int) bool { return all[i].JobID < all[j].JobID }) if maxResults <= 0 || maxResults > defaultMaxResults { diff --git a/services/quicksight/backend_automation.go b/services/quicksight/backend_automation.go index df581b0a6..c138233a5 100644 --- a/services/quicksight/backend_automation.go +++ b/services/quicksight/backend_automation.go @@ -53,7 +53,7 @@ func automationJobKey(accountID, automationGroupID, automationID, jobID string) // API to create automation groups/automations themselves (they're authored // via the console), so any non-empty IDs are accepted. func (b *InMemoryBackend) StartAutomationJob( - accountID, automationGroupID, automationID, inputPayload string, + _, automationGroupID, automationID, inputPayload string, ) (*AutomationJob, error) { if automationGroupID == "" || automationID == "" { return nil, ErrValidation @@ -78,7 +78,7 @@ func (b *InMemoryBackend) StartAutomationJob( Status: automationJobStatusRunning, InputPayload: inputPayload, } - b.automationJobs[automationJobKey(accountID, automationGroupID, automationID, jobID)] = j + b.automationJobs.Put(j) return j.toAutomationJob(), nil } @@ -104,7 +104,7 @@ func (b *InMemoryBackend) DescribeAutomationJob( b.mu.Lock("DescribeAutomationJob") defer b.mu.Unlock() - j, ok := b.automationJobs[automationJobKey(accountID, automationGroupID, automationID, jobID)] + j, ok := b.automationJobs.Get(automationJobKey(accountID, automationGroupID, automationID, jobID)) if !ok { return nil, ErrAutomationJobNotFound } diff --git a/services/quicksight/backend_brands.go b/services/quicksight/backend_brands.go index 6680bc10f..501657a67 100644 --- a/services/quicksight/backend_brands.go +++ b/services/quicksight/backend_brands.go @@ -3,7 +3,6 @@ package quicksight import ( "sort" "strconv" - "strings" "time" ) @@ -80,7 +79,7 @@ func (b *InMemoryBackend) CreateBrand(accountID, brandID string, definition map[ defer b.mu.Unlock() key := brandKey(accountID, brandID) - if _, exists := b.brands[key]; exists { + if b.brands.Has(key) { return nil, ErrBrandAlreadyExists } @@ -101,7 +100,7 @@ func (b *InMemoryBackend) CreateBrand(accountID, brandID string, definition map[ }, }, } - b.brands[key] = brand + b.brands.Put(brand) return brand.toBrand(), nil } @@ -110,7 +109,7 @@ func (b *InMemoryBackend) DescribeBrand(accountID, brandID, versionID string) (* b.mu.RLock("DescribeBrand") defer b.mu.RUnlock() - brand, ok := b.brands[brandKey(accountID, brandID)] + brand, ok := b.brands.Get(brandKey(accountID, brandID)) if !ok { return nil, ErrBrandNotFound } @@ -126,7 +125,7 @@ func (b *InMemoryBackend) UpdateBrand(accountID, brandID string, definition map[ b.mu.Lock("UpdateBrand") defer b.mu.Unlock() - brand, ok := b.brands[brandKey(accountID, brandID)] + brand, ok := b.brands.Get(brandKey(accountID, brandID)) if !ok { return nil, ErrBrandNotFound } @@ -150,7 +149,7 @@ func (b *InMemoryBackend) DeleteBrand(accountID, brandID string) error { defer b.mu.Unlock() key := brandKey(accountID, brandID) - brand, ok := b.brands[key] + brand, ok := b.brands.Get(key) if !ok { return ErrBrandNotFound } @@ -159,26 +158,21 @@ func (b *InMemoryBackend) DeleteBrand(accountID, brandID string) error { return ErrBrandInUse } - delete(b.brands, key) + b.brands.Delete(key) return nil } +//nolint:dupl // list functions share structure but operate on different stored types func (b *InMemoryBackend) ListBrands( - accountID string, + _ string, maxResults int32, nextToken string, ) ([]*Brand, string, error) { b.mu.RLock("ListBrands") defer b.mu.RUnlock() - prefix := accountID + "/" - all := make([]*storedBrand, 0, len(b.brands)) - for k, brand := range b.brands { - if strings.HasPrefix(k, prefix) { - all = append(all, brand) - } - } + all := b.brands.All() sort.Slice(all, func(i, j int) bool { return all[i].BrandID < all[j].BrandID }) if maxResults <= 0 || maxResults > defaultMaxResults { @@ -212,7 +206,7 @@ func (b *InMemoryBackend) DescribeBrandPublishedVersion(accountID, brandID strin b.mu.RLock("DescribeBrandPublishedVersion") defer b.mu.RUnlock() - brand, ok := b.brands[brandKey(accountID, brandID)] + brand, ok := b.brands.Get(brandKey(accountID, brandID)) if !ok { return nil, ErrBrandNotFound } @@ -228,7 +222,7 @@ func (b *InMemoryBackend) UpdateBrandPublishedVersion(accountID, brandID, versio b.mu.Lock("UpdateBrandPublishedVersion") defer b.mu.Unlock() - brand, ok := b.brands[brandKey(accountID, brandID)] + brand, ok := b.brands.Get(brandKey(accountID, brandID)) if !ok { return ErrBrandNotFound } @@ -261,7 +255,7 @@ func (b *InMemoryBackend) UpdateBrandAssignment(accountID, brandArn string) (str defer b.mu.Unlock() found := false - for _, brand := range b.brands { + for _, brand := range b.brands.All() { if brand.Arn == brandArn { found = true diff --git a/services/quicksight/backend_custompermissions.go b/services/quicksight/backend_custompermissions.go index c163377cd..db04b1523 100644 --- a/services/quicksight/backend_custompermissions.go +++ b/services/quicksight/backend_custompermissions.go @@ -47,7 +47,7 @@ func (b *InMemoryBackend) CreateCustomPermissions( defer b.mu.Unlock() key := customPermissionsKey(accountID, name) - if _, exists := b.customPermissions[key]; exists { + if b.customPermissions.Has(key) { return nil, ErrCustomPermissionsAlreadyExists } @@ -56,7 +56,7 @@ func (b *InMemoryBackend) CreateCustomPermissions( Arn: b.buildARN("custom-permissions", name), Capabilities: capabilities, } - b.customPermissions[key] = cp + b.customPermissions.Put(cp) if len(tags) > 0 { b.tags[cp.Arn] = maps.Clone(tags) @@ -69,7 +69,7 @@ func (b *InMemoryBackend) DescribeCustomPermissions(accountID, name string) (*Cu b.mu.RLock("DescribeCustomPermissions") defer b.mu.RUnlock() - cp, ok := b.customPermissions[customPermissionsKey(accountID, name)] + cp, ok := b.customPermissions.Get(customPermissionsKey(accountID, name)) if !ok { return nil, ErrCustomPermissionsNotFound } @@ -84,7 +84,7 @@ func (b *InMemoryBackend) UpdateCustomPermissions( b.mu.Lock("UpdateCustomPermissions") defer b.mu.Unlock() - cp, ok := b.customPermissions[customPermissionsKey(accountID, name)] + cp, ok := b.customPermissions.Get(customPermissionsKey(accountID, name)) if !ok { return nil, ErrCustomPermissionsNotFound } @@ -101,7 +101,7 @@ func (b *InMemoryBackend) DeleteCustomPermissions(accountID, name string) error defer b.mu.Unlock() key := customPermissionsKey(accountID, name) - if _, ok := b.customPermissions[key]; !ok { + if !b.customPermissions.Has(key) { return ErrCustomPermissionsNotFound } @@ -117,27 +117,21 @@ func (b *InMemoryBackend) DeleteCustomPermissions(accountID, name string) error } } - delete(b.customPermissions, key) + b.customPermissions.Delete(key) return nil } //nolint:dupl // list functions share structure but operate on different stored types func (b *InMemoryBackend) ListCustomPermissions( - accountID string, + _ string, maxResults int32, nextToken string, ) ([]*CustomPermissions, string, error) { b.mu.RLock("ListCustomPermissions") defer b.mu.RUnlock() - prefix := accountID + "/" - all := make([]*storedCustomPermissions, 0, len(b.customPermissions)) - for k, cp := range b.customPermissions { - if strings.HasPrefix(k, prefix) { - all = append(all, cp) - } - } + all := b.customPermissions.All() sort.Slice(all, func(i, j int) bool { return all[i].Name < all[j].Name }) if maxResults <= 0 || maxResults > defaultMaxResults { @@ -181,7 +175,7 @@ func (b *InMemoryBackend) UpdateRoleCustomPermission(accountID, namespace, role, b.mu.Lock("UpdateRoleCustomPermission") defer b.mu.Unlock() - if _, ok := b.customPermissions[customPermissionsKey(accountID, customPermissionsName)]; !ok { + if !b.customPermissions.Has(customPermissionsKey(accountID, customPermissionsName)) { return ErrCustomPermissionsNotFound } @@ -302,10 +296,10 @@ func (b *InMemoryBackend) UpdateUserCustomPermission( b.mu.Lock("UpdateUserCustomPermission") defer b.mu.Unlock() - if _, ok := b.users[userKey(accountID, namespace, userName)]; !ok { + if !b.users.Has(userKey(accountID, namespace, userName)) { return ErrUserNotFound } - if _, ok := b.customPermissions[customPermissionsKey(accountID, customPermissionsName)]; !ok { + if !b.customPermissions.Has(customPermissionsKey(accountID, customPermissionsName)) { return ErrCustomPermissionsNotFound } diff --git a/services/quicksight/backend_dashboardsnapshot.go b/services/quicksight/backend_dashboardsnapshot.go index efb7734e3..f38cdc29d 100644 --- a/services/quicksight/backend_dashboardsnapshot.go +++ b/services/quicksight/backend_dashboardsnapshot.go @@ -49,7 +49,7 @@ func (b *InMemoryBackend) StartDashboardSnapshotJob( b.mu.Lock("StartDashboardSnapshotJob") defer b.mu.Unlock() - if _, ok := b.dashboards[dashboardKey(accountID, dashboardID)]; !ok { + if !b.dashboards.Has(dashboardKey(accountID, dashboardID)) { return nil, ErrDashboardNotFound } @@ -63,7 +63,7 @@ func (b *InMemoryBackend) StartDashboardSnapshotJob( DashboardID: dashboardID, Status: snapshotJobStatusQueued, } - b.dashboardSnapshotJobs[dashboardSnapshotJobKey(accountID, dashboardID, jobID)] = job + b.dashboardSnapshotJobs.Put(job) return job.toDashboardSnapshotJob(), nil } @@ -87,7 +87,7 @@ func (b *InMemoryBackend) DescribeDashboardSnapshotJob( b.mu.Lock("DescribeDashboardSnapshotJob") defer b.mu.Unlock() - job, ok := b.dashboardSnapshotJobs[dashboardSnapshotJobKey(accountID, dashboardID, jobID)] + job, ok := b.dashboardSnapshotJobs.Get(dashboardSnapshotJobKey(accountID, dashboardID, jobID)) if !ok { return nil, ErrDashboardSnapshotJobNotFound } @@ -105,7 +105,7 @@ func (b *InMemoryBackend) DescribeDashboardSnapshotJobResult( b.mu.Lock("DescribeDashboardSnapshotJobResult") defer b.mu.Unlock() - job, ok := b.dashboardSnapshotJobs[dashboardSnapshotJobKey(accountID, dashboardID, jobID)] + job, ok := b.dashboardSnapshotJobs.Get(dashboardSnapshotJobKey(accountID, dashboardID, jobID)) if !ok { return nil, ErrDashboardSnapshotJobNotFound } @@ -131,7 +131,7 @@ func (b *InMemoryBackend) StartDashboardSnapshotJobSchedule(accountID, dashboard b.mu.RLock("StartDashboardSnapshotJobSchedule") defer b.mu.RUnlock() - if _, ok := b.dashboards[dashboardKey(accountID, dashboardID)]; !ok { + if !b.dashboards.Has(dashboardKey(accountID, dashboardID)) { return ErrDashboardNotFound } diff --git a/services/quicksight/backend_embedurl.go b/services/quicksight/backend_embedurl.go index f535315f4..22259a5d2 100644 --- a/services/quicksight/backend_embedurl.go +++ b/services/quicksight/backend_embedurl.go @@ -71,7 +71,7 @@ func (b *InMemoryBackend) GenerateEmbedURLForAnonymousUser( b.mu.RLock("GenerateEmbedURLForAnonymousUser") defer b.mu.RUnlock() - if _, ok := b.namespaces[nsKey(accountID, namespace)]; !ok { + if !b.namespaces.Has(nsKey(accountID, namespace)) { return "", "", ErrNamespaceNotFound } @@ -95,7 +95,7 @@ func (b *InMemoryBackend) GenerateEmbedURLForRegisteredUser( defer b.mu.RUnlock() if namespace, userName, ok := parseUserArn(userArn); ok { - if _, exists := b.users[userKey(accountID, namespace, userName)]; !exists { + if !b.users.Has(userKey(accountID, namespace, userName)) { return "", ErrUserNotFound } } @@ -124,7 +124,7 @@ func (b *InMemoryBackend) GetDashboardEmbedURL(accountID, dashboardID, identityT b.mu.RLock("GetDashboardEmbedURL") defer b.mu.RUnlock() - if _, ok := b.dashboards[dashboardKey(accountID, dashboardID)]; !ok { + if !b.dashboards.Has(dashboardKey(accountID, dashboardID)) { return "", ErrDashboardNotFound } @@ -165,7 +165,7 @@ func (b *InMemoryBackend) GenerateIdentityContext( defer b.mu.RUnlock() if namespace != "" { - if _, ok := b.namespaces[nsKey(accountID, namespace)]; !ok { + if !b.namespaces.Has(nsKey(accountID, namespace)) { return "", ErrNamespaceNotFound } } diff --git a/services/quicksight/backend_flow.go b/services/quicksight/backend_flow.go index 8ca745527..e70ebf63a 100644 --- a/services/quicksight/backend_flow.go +++ b/services/quicksight/backend_flow.go @@ -2,7 +2,6 @@ package quicksight import ( "sort" - "strings" "time" ) @@ -56,7 +55,7 @@ func flowKey(accountID, flowID string) string { // seedFlow inserts f directly into backend state. Exported for tests only // (via export_test.go's SeedFlow) since QuickSight has no CreateFlow API. -func (b *InMemoryBackend) seedFlow(accountID string, f *Flow) { +func (b *InMemoryBackend) seedFlow(_ string, f *Flow) { b.mu.Lock("seedFlow") defer b.mu.Unlock() @@ -65,7 +64,7 @@ func (b *InMemoryBackend) seedFlow(accountID string, f *Flow) { arn = b.buildARN("flow", f.FlowID) } - b.flows[flowKey(accountID, f.FlowID)] = &storedFlow{ + b.flows.Put(&storedFlow{ CreatedTime: f.CreatedTime, LastUpdatedTime: f.LastUpdatedTime, LastPublishedAt: f.LastPublishedAt, @@ -80,26 +79,20 @@ func (b *InMemoryBackend) seedFlow(accountID string, f *Flow) { RunCount: f.RunCount, UserCount: f.UserCount, Permissions: clonePermissions(f.Permissions), - } + }) } // ---- Flows ---- func (b *InMemoryBackend) ListFlows( - accountID string, + _ string, maxResults int32, nextToken string, ) ([]*Flow, string, error) { b.mu.RLock("ListFlows") defer b.mu.RUnlock() - prefix := accountID + "/" - var all []*storedFlow - for k, f := range b.flows { - if strings.HasPrefix(k, prefix) { - all = append(all, f) - } - } + all := b.flows.All() sort.Slice(all, func(i, j int) bool { return all[i].FlowID < all[j].FlowID }) result, next := paginateFlows(all, maxResults, nextToken) @@ -108,7 +101,7 @@ func (b *InMemoryBackend) ListFlows( } func (b *InMemoryBackend) SearchFlows( - accountID string, + _ string, filters []SearchFilter, maxResults int32, nextToken string, @@ -116,10 +109,9 @@ func (b *InMemoryBackend) SearchFlows( b.mu.RLock("SearchFlows") defer b.mu.RUnlock() - prefix := accountID + "/" var filtered []*storedFlow - for k, f := range b.flows { - if strings.HasPrefix(k, prefix) && matchesAllNameFilters(f.Name, filters, filterFlowAssetName) { + for _, f := range b.flows.All() { + if matchesAllNameFilters(f.Name, filters, filterFlowAssetName) { filtered = append(filtered, f) } } @@ -166,7 +158,7 @@ func (b *InMemoryBackend) GetFlowMetadata(accountID, flowID string) (*Flow, erro b.mu.RLock("GetFlowMetadata") defer b.mu.RUnlock() - f, ok := b.flows[flowKey(accountID, flowID)] + f, ok := b.flows.Get(flowKey(accountID, flowID)) if !ok { return nil, ErrFlowNotFound } @@ -180,7 +172,7 @@ func (b *InMemoryBackend) GetFlowPermissions(accountID, flowID string) (*Flow, [ b.mu.RLock("GetFlowPermissions") defer b.mu.RUnlock() - f, ok := b.flows[flowKey(accountID, flowID)] + f, ok := b.flows.Get(flowKey(accountID, flowID)) if !ok { return nil, nil, ErrFlowNotFound } @@ -196,7 +188,7 @@ func (b *InMemoryBackend) UpdateFlowPermissions( defer b.mu.Unlock() key := flowKey(accountID, flowID) - f, ok := b.flows[key] + f, ok := b.flows.Get(key) if !ok { return nil, nil, ErrFlowNotFound } diff --git a/services/quicksight/backend_folders.go b/services/quicksight/backend_folders.go index 1eb3848c7..fafb250ef 100644 --- a/services/quicksight/backend_folders.go +++ b/services/quicksight/backend_folders.go @@ -223,7 +223,7 @@ func (b *InMemoryBackend) CreateFolder( defer b.mu.Unlock() key := folderKey(accountID, folderID) - if _, exists := b.folders[key]; exists { + if b.folders.Has(key) { return nil, ErrFolderAlreadyExists } @@ -238,7 +238,7 @@ func (b *InMemoryBackend) CreateFolder( ParentFolderArn: parentFolderArn, Permissions: clonePermissions(permissions), } - b.folders[key] = f + b.folders.Put(f) if len(tags) > 0 { b.tags[f.Arn] = maps.Clone(tags) @@ -251,7 +251,7 @@ func (b *InMemoryBackend) DescribeFolder(accountID, folderID string) (*Folder, e b.mu.RLock("DescribeFolder") defer b.mu.RUnlock() - f, ok := b.folders[folderKey(accountID, folderID)] + f, ok := b.folders.Get(folderKey(accountID, folderID)) if !ok { return nil, ErrFolderNotFound } @@ -264,7 +264,7 @@ func (b *InMemoryBackend) UpdateFolder(accountID, folderID, name string) (*Folde defer b.mu.Unlock() key := folderKey(accountID, folderID) - f, ok := b.folders[key] + f, ok := b.folders.Get(key) if !ok { return nil, ErrFolderNotFound } @@ -282,32 +282,25 @@ func (b *InMemoryBackend) DeleteFolder(accountID, folderID string) error { defer b.mu.Unlock() key := folderKey(accountID, folderID) - f, ok := b.folders[key] + f, ok := b.folders.Get(key) if !ok { return ErrFolderNotFound } delete(b.tags, f.Arn) - delete(b.folders, key) + b.folders.Delete(key) - memberPrefix := accountID + "/" + folderID + "/" - for k := range b.folderMembers { - if strings.HasPrefix(k, memberPrefix) { - delete(b.folderMembers, k) + for _, m := range b.folderMembers.All() { + if m.FolderID == folderID { + b.folderMembers.Delete(folderMemberKey(accountID, folderID, m.MemberType, m.MemberID)) } } return nil } -func (b *InMemoryBackend) allFoldersLocked(accountID string) []*storedFolder { - prefix := accountID + "/" - all := make([]*storedFolder, 0, len(b.folders)) - for k, f := range b.folders { - if strings.HasPrefix(k, prefix) { - all = append(all, f) - } - } +func (b *InMemoryBackend) allFoldersLocked(_ string) []*storedFolder { + all := b.folders.All() sort.Slice(all, func(i, j int) bool { return all[i].FolderID < all[j].FolderID }) return all @@ -417,7 +410,7 @@ func (b *InMemoryBackend) CreateFolderMembership( b.mu.Lock("CreateFolderMembership") defer b.mu.Unlock() - if _, ok := b.folders[folderKey(accountID, folderID)]; !ok { + if !b.folders.Has(folderKey(accountID, folderID)) { return nil, ErrFolderNotFound } @@ -426,7 +419,7 @@ func (b *InMemoryBackend) CreateFolderMembership( MemberID: memberID, MemberType: memberType, } - b.folderMembers[folderMemberKey(accountID, folderID, memberType, memberID)] = m + b.folderMembers.Put(m) return m.toFolderMember(), nil } @@ -436,12 +429,10 @@ func (b *InMemoryBackend) DeleteFolderMembership(accountID, folderID, memberID, defer b.mu.Unlock() key := folderMemberKey(accountID, folderID, memberType, memberID) - if _, ok := b.folderMembers[key]; !ok { + if !b.folderMembers.Delete(key) { return ErrFolderMemberNotFound } - delete(b.folderMembers, key) - return nil } @@ -453,14 +444,13 @@ func (b *InMemoryBackend) ListFolderMembers( b.mu.RLock("ListFolderMembers") defer b.mu.RUnlock() - if _, ok := b.folders[folderKey(accountID, folderID)]; !ok { + if !b.folders.Has(folderKey(accountID, folderID)) { return nil, "", ErrFolderNotFound } - prefix := accountID + "/" + folderID + "/" var all []*storedFolderMember - for k, m := range b.folderMembers { - if strings.HasPrefix(k, prefix) { + for _, m := range b.folderMembers.All() { + if m.FolderID == folderID { all = append(all, m) } } @@ -509,7 +499,7 @@ func (b *InMemoryBackend) DescribeFolderPermissions(accountID, folderID string) b.mu.RLock("DescribeFolderPermissions") defer b.mu.RUnlock() - f, ok := b.folders[folderKey(accountID, folderID)] + f, ok := b.folders.Get(folderKey(accountID, folderID)) if !ok { return nil, ErrFolderNotFound } @@ -524,7 +514,7 @@ func (b *InMemoryBackend) UpdateFolderPermissions( b.mu.Lock("UpdateFolderPermissions") defer b.mu.Unlock() - f, ok := b.folders[folderKey(accountID, folderID)] + f, ok := b.folders.Get(folderKey(accountID, folderID)) if !ok { return nil, ErrFolderNotFound } @@ -541,7 +531,7 @@ func (b *InMemoryBackend) DescribeFolderResolvedPermissions( b.mu.RLock("DescribeFolderResolvedPermissions") defer b.mu.RUnlock() - f, ok := b.folders[folderKey(accountID, folderID)] + f, ok := b.folders.Get(folderKey(accountID, folderID)) if !ok { return nil, ErrFolderNotFound } @@ -557,7 +547,7 @@ func (b *InMemoryBackend) DescribeFolderResolvedPermissions( } visited[parentID] = true - parent, foundParent := b.folders[folderKey(accountID, parentID)] + parent, foundParent := b.folders.Get(folderKey(accountID, parentID)) if !foundParent { break } @@ -621,9 +611,8 @@ func (b *InMemoryBackend) ListFoldersForResource( } var folderIDs []string - prefix := accountID + "/" - for k, m := range b.folderMembers { - if strings.HasPrefix(k, prefix) && m.MemberType == memberType && m.MemberID == memberID { + for _, m := range b.folderMembers.All() { + if m.MemberType == memberType && m.MemberID == memberID { folderIDs = append(folderIDs, m.FolderID) } } @@ -654,7 +643,7 @@ func (b *InMemoryBackend) ListFoldersForResource( result := make([]string, 0, end-start) for _, id := range folderIDs[start:end] { - if f, exists := b.folders[folderKey(accountID, id)]; exists { + if f, exists := b.folders.Get(folderKey(accountID, id)); exists { result = append(result, f.Arn) } } diff --git a/services/quicksight/backend_iampolicyassignments.go b/services/quicksight/backend_iampolicyassignments.go index e74543cbd..8dd960b9a 100644 --- a/services/quicksight/backend_iampolicyassignments.go +++ b/services/quicksight/backend_iampolicyassignments.go @@ -4,7 +4,6 @@ import ( "maps" "slices" "sort" - "strings" "github.com/google/uuid" ) @@ -83,12 +82,12 @@ func (b *InMemoryBackend) CreateIAMPolicyAssignment( b.mu.Lock("CreateIAMPolicyAssignment") defer b.mu.Unlock() - if _, ok := b.namespaces[nsKey(accountID, namespace)]; !ok { + if !b.namespaces.Has(nsKey(accountID, namespace)) { return nil, ErrNamespaceNotFound } key := iamPolicyAssignmentKey(accountID, namespace, assignmentName) - if _, exists := b.iamPolicyAssignments[key]; exists { + if b.iamPolicyAssignments.Has(key) { return nil, ErrIAMPolicyAssignmentAlreadyExists } @@ -100,7 +99,7 @@ func (b *InMemoryBackend) CreateIAMPolicyAssignment( Namespace: namespace, Identities: maps.Clone(identities), } - b.iamPolicyAssignments[key] = a + b.iamPolicyAssignments.Put(a) return a.toIAMPolicyAssignment(), nil } @@ -111,7 +110,7 @@ func (b *InMemoryBackend) DescribeIAMPolicyAssignment( b.mu.RLock("DescribeIAMPolicyAssignment") defer b.mu.RUnlock() - a, ok := b.iamPolicyAssignments[iamPolicyAssignmentKey(accountID, namespace, assignmentName)] + a, ok := b.iamPolicyAssignments.Get(iamPolicyAssignmentKey(accountID, namespace, assignmentName)) if !ok { return nil, ErrIAMPolicyAssignmentNotFound } @@ -131,7 +130,7 @@ func (b *InMemoryBackend) UpdateIAMPolicyAssignment( defer b.mu.Unlock() key := iamPolicyAssignmentKey(accountID, namespace, assignmentName) - a, ok := b.iamPolicyAssignments[key] + a, ok := b.iamPolicyAssignments.Get(key) if !ok { return nil, ErrIAMPolicyAssignmentNotFound } @@ -154,19 +153,17 @@ func (b *InMemoryBackend) DeleteIAMPolicyAssignment(accountID, namespace, assign defer b.mu.Unlock() key := iamPolicyAssignmentKey(accountID, namespace, assignmentName) - if _, ok := b.iamPolicyAssignments[key]; !ok { + if !b.iamPolicyAssignments.Delete(key) { return ErrIAMPolicyAssignmentNotFound } - delete(b.iamPolicyAssignments, key) return nil } -func (b *InMemoryBackend) allIAMPolicyAssignmentsLocked(accountID, namespace string) []*storedIAMPolicyAssignment { - prefix := accountID + "/" + namespace + "/" - all := make([]*storedIAMPolicyAssignment, 0, len(b.iamPolicyAssignments)) - for k, a := range b.iamPolicyAssignments { - if strings.HasPrefix(k, prefix) { +func (b *InMemoryBackend) allIAMPolicyAssignmentsLocked(_, namespace string) []*storedIAMPolicyAssignment { + var all []*storedIAMPolicyAssignment + for _, a := range b.iamPolicyAssignments.All() { + if a.Namespace == namespace { all = append(all, a) } } diff --git a/services/quicksight/backend_identitypropagation.go b/services/quicksight/backend_identitypropagation.go index 701ded366..345971133 100644 --- a/services/quicksight/backend_identitypropagation.go +++ b/services/quicksight/backend_identitypropagation.go @@ -2,7 +2,6 @@ package quicksight import ( "sort" - "strings" ) // storedIdentityPropagationConfig is the persisted representation of one @@ -33,7 +32,7 @@ func isValidServiceType(service string) bool { // ---- Identity propagation configuration ---- -func (b *InMemoryBackend) UpdateIdentityPropagationConfig(accountID, service string, authorizedTargets []string) error { +func (b *InMemoryBackend) UpdateIdentityPropagationConfig(_, service string, authorizedTargets []string) error { if !isValidServiceType(service) { return ErrValidation } @@ -41,10 +40,10 @@ func (b *InMemoryBackend) UpdateIdentityPropagationConfig(accountID, service str b.mu.Lock("UpdateIdentityPropagationConfig") defer b.mu.Unlock() - b.identityPropagationConfigs[identityPropagationKey(accountID, service)] = &storedIdentityPropagationConfig{ + b.identityPropagationConfigs.Put(&storedIdentityPropagationConfig{ Service: service, AuthorizedTargets: authorizedTargets, - } + }) return nil } @@ -54,25 +53,18 @@ func (b *InMemoryBackend) DeleteIdentityPropagationConfig(accountID, service str defer b.mu.Unlock() key := identityPropagationKey(accountID, service) - if _, ok := b.identityPropagationConfigs[key]; !ok { + if !b.identityPropagationConfigs.Delete(key) { return ErrIdentityPropagationConfigNotFound } - delete(b.identityPropagationConfigs, key) return nil } -func (b *InMemoryBackend) ListIdentityPropagationConfigs(accountID string) ([]*IdentityPropagationConfig, error) { +func (b *InMemoryBackend) ListIdentityPropagationConfigs(_ string) ([]*IdentityPropagationConfig, error) { b.mu.RLock("ListIdentityPropagationConfigs") defer b.mu.RUnlock() - prefix := accountID + "/" - all := make([]*storedIdentityPropagationConfig, 0, len(b.identityPropagationConfigs)) - for k, cfg := range b.identityPropagationConfigs { - if strings.HasPrefix(k, prefix) { - all = append(all, cfg) - } - } + all := b.identityPropagationConfigs.All() sort.Slice(all, func(i, j int) bool { return all[i].Service < all[j].Service }) result := make([]*IdentityPropagationConfig, 0, len(all)) diff --git a/services/quicksight/backend_oauth.go b/services/quicksight/backend_oauth.go index 7d8d06ca6..6dc834e77 100644 --- a/services/quicksight/backend_oauth.go +++ b/services/quicksight/backend_oauth.go @@ -3,7 +3,6 @@ package quicksight import ( "maps" "sort" - "strings" "time" ) @@ -45,7 +44,7 @@ func (b *InMemoryBackend) CreateOAuthClientApplication( defer b.mu.Unlock() key := oauthAppKey(accountID, clientID) - if _, exists := b.oauthClientApps[key]; exists { + if b.oauthClientApps.Has(key) { return nil, ErrOAuthClientAppAlreadyExists } @@ -59,7 +58,7 @@ func (b *InMemoryBackend) CreateOAuthClientApplication( Status: statusCreationSuccessful, Extra: fields, } - b.oauthClientApps[key] = app + b.oauthClientApps.Put(app) return app.toOAuthClientApplication(), nil } @@ -68,7 +67,7 @@ func (b *InMemoryBackend) DescribeOAuthClientApplication(accountID, clientID str b.mu.RLock("DescribeOAuthClientApplication") defer b.mu.RUnlock() - app, ok := b.oauthClientApps[oauthAppKey(accountID, clientID)] + app, ok := b.oauthClientApps.Get(oauthAppKey(accountID, clientID)) if !ok { return nil, ErrOAuthClientAppNotFound } @@ -83,7 +82,7 @@ func (b *InMemoryBackend) UpdateOAuthClientApplication( b.mu.Lock("UpdateOAuthClientApplication") defer b.mu.Unlock() - app, ok := b.oauthClientApps[oauthAppKey(accountID, clientID)] + app, ok := b.oauthClientApps.Get(oauthAppKey(accountID, clientID)) if !ok { return nil, ErrOAuthClientAppNotFound } @@ -108,31 +107,25 @@ func (b *InMemoryBackend) DeleteOAuthClientApplication(accountID, clientID strin defer b.mu.Unlock() key := oauthAppKey(accountID, clientID) - app, ok := b.oauthClientApps[key] + app, ok := b.oauthClientApps.Get(key) if !ok { return nil, ErrOAuthClientAppNotFound } - delete(b.oauthClientApps, key) + b.oauthClientApps.Delete(key) return app.toOAuthClientApplication(), nil } //nolint:dupl // list functions share structure but operate on different stored types func (b *InMemoryBackend) ListOAuthClientApplications( - accountID string, + _ string, maxResults int32, nextToken string, ) ([]*OAuthClientApplication, string, error) { b.mu.RLock("ListOAuthClientApplications") defer b.mu.RUnlock() - prefix := accountID + "/" - all := make([]*storedOAuthApp, 0, len(b.oauthClientApps)) - for k, app := range b.oauthClientApps { - if strings.HasPrefix(k, prefix) { - all = append(all, app) - } - } + all := b.oauthClientApps.All() sort.Slice(all, func(i, j int) bool { return all[i].ClientID < all[j].ClientID }) if maxResults <= 0 || maxResults > defaultMaxResults { diff --git a/services/quicksight/backend_refreshschedule.go b/services/quicksight/backend_refreshschedule.go index 1c4773aff..9fecf0a82 100644 --- a/services/quicksight/backend_refreshschedule.go +++ b/services/quicksight/backend_refreshschedule.go @@ -38,7 +38,7 @@ func (b *InMemoryBackend) CreateRefreshSchedule( b.mu.Lock("CreateRefreshSchedule") defer b.mu.Unlock() - ds, ok := b.dataSets[dataSetKey(accountID, datasetID)] + ds, ok := b.dataSets.Get(dataSetKey(accountID, datasetID)) if !ok { return nil, ErrDataSetNotFound } @@ -64,7 +64,7 @@ func (b *InMemoryBackend) DescribeRefreshSchedule(accountID, datasetID, schedule b.mu.RLock("DescribeRefreshSchedule") defer b.mu.RUnlock() - ds, ok := b.dataSets[dataSetKey(accountID, datasetID)] + ds, ok := b.dataSets.Get(dataSetKey(accountID, datasetID)) if !ok { return nil, ErrDataSetNotFound } @@ -84,7 +84,7 @@ func (b *InMemoryBackend) UpdateRefreshSchedule( b.mu.Lock("UpdateRefreshSchedule") defer b.mu.Unlock() - ds, ok := b.dataSets[dataSetKey(accountID, datasetID)] + ds, ok := b.dataSets.Get(dataSetKey(accountID, datasetID)) if !ok { return nil, ErrDataSetNotFound } @@ -115,7 +115,7 @@ func (b *InMemoryBackend) DeleteRefreshSchedule(accountID, datasetID, scheduleID b.mu.Lock("DeleteRefreshSchedule") defer b.mu.Unlock() - ds, ok := b.dataSets[dataSetKey(accountID, datasetID)] + ds, ok := b.dataSets.Get(dataSetKey(accountID, datasetID)) if !ok { return ErrDataSetNotFound } @@ -133,7 +133,7 @@ func (b *InMemoryBackend) ListRefreshSchedules(accountID, datasetID string) ([]* b.mu.RLock("ListRefreshSchedules") defer b.mu.RUnlock() - ds, ok := b.dataSets[dataSetKey(accountID, datasetID)] + ds, ok := b.dataSets.Get(dataSetKey(accountID, datasetID)) if !ok { return nil, ErrDataSetNotFound } @@ -161,7 +161,7 @@ func (b *InMemoryBackend) PutDataSetRefreshProperties( b.mu.Lock("PutDataSetRefreshProperties") defer b.mu.Unlock() - ds, ok := b.dataSets[dataSetKey(accountID, datasetID)] + ds, ok := b.dataSets.Get(dataSetKey(accountID, datasetID)) if !ok { return nil, ErrDataSetNotFound } @@ -181,7 +181,7 @@ func (b *InMemoryBackend) DescribeDataSetRefreshProperties( b.mu.RLock("DescribeDataSetRefreshProperties") defer b.mu.RUnlock() - ds, ok := b.dataSets[dataSetKey(accountID, datasetID)] + ds, ok := b.dataSets.Get(dataSetKey(accountID, datasetID)) if !ok { return nil, ErrDataSetNotFound } @@ -197,7 +197,7 @@ func (b *InMemoryBackend) DeleteDataSetRefreshProperties(accountID, datasetID st b.mu.Lock("DeleteDataSetRefreshProperties") defer b.mu.Unlock() - ds, ok := b.dataSets[dataSetKey(accountID, datasetID)] + ds, ok := b.dataSets.Get(dataSetKey(accountID, datasetID)) if !ok { return ErrDataSetNotFound } diff --git a/services/quicksight/backend_selfupgrade.go b/services/quicksight/backend_selfupgrade.go index 2fb9d7803..780670a01 100644 --- a/services/quicksight/backend_selfupgrade.go +++ b/services/quicksight/backend_selfupgrade.go @@ -2,7 +2,6 @@ package quicksight import ( "sort" - "strings" "time" ) @@ -33,6 +32,13 @@ type storedSelfUpgradeRequest struct { RequestStatus string `json:"requestStatus"` RequestedRole string `json:"requestedRole"` UpgradeRequestID string `json:"upgradeRequestId"` + // Namespace is not part of the real SelfUpgradeRequestDetail API shape + // (namespace is a request path parameter, not response data); it is + // carried here purely as this backend's store.Table key material, since + // requests are keyed by (accountID, namespace, upgradeRequestID) but a + // single account can have multiple namespaces each issuing their own + // upgrade request IDs. See store_setup.go's selfUpgradeRequests keyFn. + Namespace string `json:"namespace"` } func (r *storedSelfUpgradeRequest) toSelfUpgradeRequestDetail() *SelfUpgradeRequestDetail { @@ -68,7 +74,7 @@ func selfUpgradeRequestKey(accountID, namespace, upgradeRequestID string) string // (real) AWS API. Exported for tests only (via export_test.go's // SeedSelfUpgradeRequest) since QuickSight has no CreateSelfUpgradeRequest // operation. -func (b *InMemoryBackend) seedSelfUpgradeRequest(accountID, namespace string, r *SelfUpgradeRequestDetail) { +func (b *InMemoryBackend) seedSelfUpgradeRequest(_, namespace string, r *SelfUpgradeRequestDetail) { b.mu.Lock("seedSelfUpgradeRequest") defer b.mu.Unlock() @@ -77,7 +83,7 @@ func (b *InMemoryBackend) seedSelfUpgradeRequest(accountID, namespace string, r lastAttempt = time.Unix(r.LastUpdateAttemptTime, 0).UTC() } - b.selfUpgradeRequests[selfUpgradeRequestKey(accountID, namespace, r.UpgradeRequestID)] = &storedSelfUpgradeRequest{ + b.selfUpgradeRequests.Put(&storedSelfUpgradeRequest{ CreationTime: time.Unix(r.CreationTime, 0).UTC(), LastUpdateAttemptTime: lastAttempt, LastUpdateFailureReason: r.LastUpdateFailureReason, @@ -86,7 +92,8 @@ func (b *InMemoryBackend) seedSelfUpgradeRequest(accountID, namespace string, r RequestStatus: r.RequestStatus, RequestedRole: r.RequestedRole, UpgradeRequestID: r.UpgradeRequestID, - } + Namespace: namespace, + }) } // ---- Self-upgrade configuration ---- @@ -99,7 +106,7 @@ func (b *InMemoryBackend) DescribeSelfUpgradeConfiguration(accountID, namespace b.mu.RLock("DescribeSelfUpgradeConfiguration") defer b.mu.RUnlock() - if _, ok := b.namespaces[nsKey(accountID, namespace)]; !ok { + if !b.namespaces.Has(nsKey(accountID, namespace)) { return "", ErrNamespaceNotFound } @@ -128,7 +135,7 @@ func (b *InMemoryBackend) UpdateSelfUpgradeConfiguration(accountID, namespace, s b.mu.Lock("UpdateSelfUpgradeConfiguration") defer b.mu.Unlock() - if _, ok := b.namespaces[nsKey(accountID, namespace)]; !ok { + if !b.namespaces.Has(nsKey(accountID, namespace)) { return "", ErrNamespaceNotFound } @@ -147,14 +154,13 @@ func (b *InMemoryBackend) ListSelfUpgrades( b.mu.RLock("ListSelfUpgrades") defer b.mu.RUnlock() - if _, ok := b.namespaces[nsKey(accountID, namespace)]; !ok { + if !b.namespaces.Has(nsKey(accountID, namespace)) { return nil, "", ErrNamespaceNotFound } - prefix := accountID + "/" + namespace + "/" var all []*storedSelfUpgradeRequest - for k, r := range b.selfUpgradeRequests { - if strings.HasPrefix(k, prefix) { + for _, r := range b.selfUpgradeRequests.All() { + if r.Namespace == namespace { all = append(all, r) } } @@ -215,11 +221,11 @@ func (b *InMemoryBackend) UpdateSelfUpgrade( b.mu.Lock("UpdateSelfUpgrade") defer b.mu.Unlock() - if _, ok := b.namespaces[nsKey(accountID, namespace)]; !ok { + if !b.namespaces.Has(nsKey(accountID, namespace)) { return nil, ErrNamespaceNotFound } - r, ok := b.selfUpgradeRequests[selfUpgradeRequestKey(accountID, namespace, upgradeRequestID)] + r, ok := b.selfUpgradeRequests.Get(selfUpgradeRequestKey(accountID, namespace, upgradeRequestID)) if !ok { return nil, ErrSelfUpgradeRequestNotFound } diff --git a/services/quicksight/backend_templates.go b/services/quicksight/backend_templates.go index e3b83b759..782dfaa0d 100644 --- a/services/quicksight/backend_templates.go +++ b/services/quicksight/backend_templates.go @@ -5,7 +5,6 @@ import ( "slices" "sort" "strconv" - "strings" "time" ) @@ -89,7 +88,7 @@ func (b *InMemoryBackend) CreateTemplate( defer b.mu.Unlock() key := templateKey(accountID, templateID) - if _, exists := b.templates[key]; exists { + if b.templates.Has(key) { return nil, ErrTemplateAlreadyExists } @@ -114,7 +113,7 @@ func (b *InMemoryBackend) CreateTemplate( Aliases: map[string]int64{templateAliasLatest: 1}, Permissions: clonePermissions(permissions), } - b.templates[key] = t + b.templates.Put(t) if len(tags) > 0 { b.tags[arn] = maps.Clone(tags) @@ -129,7 +128,7 @@ func (b *InMemoryBackend) DescribeTemplate(accountID, templateID string, version b.mu.RLock("DescribeTemplate") defer b.mu.RUnlock() - t, ok := b.templates[templateKey(accountID, templateID)] + t, ok := b.templates.Get(templateKey(accountID, templateID)) if !ok { return nil, ErrTemplateNotFound } @@ -150,7 +149,7 @@ func (b *InMemoryBackend) UpdateTemplate( defer b.mu.Unlock() key := templateKey(accountID, templateID) - t, ok := b.templates[key] + t, ok := b.templates.Get(key) if !ok { return nil, ErrTemplateNotFound } @@ -190,14 +189,14 @@ func (b *InMemoryBackend) DeleteTemplate(accountID, templateID string, versionNu defer b.mu.Unlock() key := templateKey(accountID, templateID) - t, ok := b.templates[key] + t, ok := b.templates.Get(key) if !ok { return ErrTemplateNotFound } if versionNumber == 0 { delete(b.tags, t.Arn) - delete(b.templates, key) + b.templates.Delete(key) return nil } @@ -209,7 +208,7 @@ func (b *InMemoryBackend) DeleteTemplate(accountID, templateID string, versionNu if len(t.Versions) == 0 { delete(b.tags, t.Arn) - delete(b.templates, key) + b.templates.Delete(key) return nil } @@ -228,14 +227,8 @@ func (b *InMemoryBackend) DeleteTemplate(accountID, templateID string, versionNu return nil } -func (b *InMemoryBackend) allTemplatesLocked(accountID string) []*storedTemplate { - prefix := accountID + "/" - all := make([]*storedTemplate, 0, len(b.templates)) - for k, t := range b.templates { - if strings.HasPrefix(k, prefix) { - all = append(all, t) - } - } +func (b *InMemoryBackend) allTemplatesLocked(_ string) []*storedTemplate { + all := b.templates.All() sort.Slice(all, func(i, j int) bool { return all[i].TemplateID < all[j].TemplateID }) return all @@ -287,7 +280,7 @@ func (b *InMemoryBackend) ListTemplateVersions( b.mu.RLock("ListTemplateVersions") defer b.mu.RUnlock() - t, ok := b.templates[templateKey(accountID, templateID)] + t, ok := b.templates.Get(templateKey(accountID, templateID)) if !ok { return nil, "", ErrTemplateNotFound } @@ -339,7 +332,7 @@ func (b *InMemoryBackend) DescribeTemplatePermissions( b.mu.RLock("DescribeTemplatePermissions") defer b.mu.RUnlock() - t, ok := b.templates[templateKey(accountID, templateID)] + t, ok := b.templates.Get(templateKey(accountID, templateID)) if !ok { return nil, nil, ErrTemplateNotFound } @@ -356,7 +349,7 @@ func (b *InMemoryBackend) UpdateTemplatePermissions( b.mu.Lock("UpdateTemplatePermissions") defer b.mu.Unlock() - t, ok := b.templates[templateKey(accountID, templateID)] + t, ok := b.templates.Get(templateKey(accountID, templateID)) if !ok { return nil, nil, ErrTemplateNotFound } @@ -382,7 +375,7 @@ func (b *InMemoryBackend) CreateTemplateAlias( b.mu.Lock("CreateTemplateAlias") defer b.mu.Unlock() - t, ok := b.templates[templateKey(accountID, templateID)] + t, ok := b.templates.Get(templateKey(accountID, templateID)) if !ok { return nil, ErrTemplateNotFound } @@ -408,7 +401,7 @@ func (b *InMemoryBackend) DescribeTemplateAlias(accountID, templateID, aliasName b.mu.RLock("DescribeTemplateAlias") defer b.mu.RUnlock() - t, ok := b.templates[templateKey(accountID, templateID)] + t, ok := b.templates.Get(templateKey(accountID, templateID)) if !ok { return nil, ErrTemplateNotFound } @@ -432,7 +425,7 @@ func (b *InMemoryBackend) UpdateTemplateAlias( b.mu.Lock("UpdateTemplateAlias") defer b.mu.Unlock() - t, ok := b.templates[templateKey(accountID, templateID)] + t, ok := b.templates.Get(templateKey(accountID, templateID)) if !ok { return nil, ErrTemplateNotFound } @@ -458,7 +451,7 @@ func (b *InMemoryBackend) DeleteTemplateAlias(accountID, templateID, aliasName s b.mu.Lock("DeleteTemplateAlias") defer b.mu.Unlock() - t, ok := b.templates[templateKey(accountID, templateID)] + t, ok := b.templates.Get(templateKey(accountID, templateID)) if !ok { return ErrTemplateNotFound } @@ -481,7 +474,7 @@ func (b *InMemoryBackend) ListTemplateAliases( b.mu.RLock("ListTemplateAliases") defer b.mu.RUnlock() - t, ok := b.templates[templateKey(accountID, templateID)] + t, ok := b.templates.Get(templateKey(accountID, templateID)) if !ok { return nil, "", ErrTemplateNotFound } diff --git a/services/quicksight/backend_themes.go b/services/quicksight/backend_themes.go index e9bc638ee..4ab9880a2 100644 --- a/services/quicksight/backend_themes.go +++ b/services/quicksight/backend_themes.go @@ -5,7 +5,6 @@ import ( "slices" "sort" "strconv" - "strings" "time" ) @@ -94,7 +93,7 @@ func (b *InMemoryBackend) CreateTheme( defer b.mu.Unlock() key := themeKey(accountID, themeID) - if _, exists := b.themes[key]; exists { + if b.themes.Has(key) { return nil, ErrThemeAlreadyExists } @@ -120,7 +119,7 @@ func (b *InMemoryBackend) CreateTheme( Aliases: map[string]int64{themeAliasLatest: 1}, Permissions: clonePermissions(permissions), } - b.themes[key] = t + b.themes.Put(t) if len(tags) > 0 { b.tags[arn] = maps.Clone(tags) @@ -135,7 +134,7 @@ func (b *InMemoryBackend) DescribeTheme(accountID, themeID string, versionNumber b.mu.RLock("DescribeTheme") defer b.mu.RUnlock() - t, ok := b.themes[themeKey(accountID, themeID)] + t, ok := b.themes.Get(themeKey(accountID, themeID)) if !ok { return nil, ErrThemeNotFound } @@ -156,7 +155,7 @@ func (b *InMemoryBackend) UpdateTheme( defer b.mu.Unlock() key := themeKey(accountID, themeID) - t, ok := b.themes[key] + t, ok := b.themes.Get(key) if !ok { return nil, ErrThemeNotFound } @@ -196,14 +195,14 @@ func (b *InMemoryBackend) DeleteTheme(accountID, themeID string, versionNumber i defer b.mu.Unlock() key := themeKey(accountID, themeID) - t, ok := b.themes[key] + t, ok := b.themes.Get(key) if !ok { return ErrThemeNotFound } if versionNumber == 0 { delete(b.tags, t.Arn) - delete(b.themes, key) + b.themes.Delete(key) return nil } @@ -215,7 +214,7 @@ func (b *InMemoryBackend) DeleteTheme(accountID, themeID string, versionNumber i if len(t.Versions) == 0 { delete(b.tags, t.Arn) - delete(b.themes, key) + b.themes.Delete(key) return nil } @@ -234,14 +233,8 @@ func (b *InMemoryBackend) DeleteTheme(accountID, themeID string, versionNumber i return nil } -func (b *InMemoryBackend) allThemesLocked(accountID string) []*storedTheme { - prefix := accountID + "/" - all := make([]*storedTheme, 0, len(b.themes)) - for k, t := range b.themes { - if strings.HasPrefix(k, prefix) { - all = append(all, t) - } - } +func (b *InMemoryBackend) allThemesLocked(_ string) []*storedTheme { + all := b.themes.All() sort.Slice(all, func(i, j int) bool { return all[i].ThemeID < all[j].ThemeID }) return all @@ -293,7 +286,7 @@ func (b *InMemoryBackend) ListThemeVersions( b.mu.RLock("ListThemeVersions") defer b.mu.RUnlock() - t, ok := b.themes[themeKey(accountID, themeID)] + t, ok := b.themes.Get(themeKey(accountID, themeID)) if !ok { return nil, "", ErrThemeNotFound } @@ -343,7 +336,7 @@ func (b *InMemoryBackend) DescribeThemePermissions(accountID, themeID string) (* b.mu.RLock("DescribeThemePermissions") defer b.mu.RUnlock() - t, ok := b.themes[themeKey(accountID, themeID)] + t, ok := b.themes.Get(themeKey(accountID, themeID)) if !ok { return nil, nil, ErrThemeNotFound } @@ -360,7 +353,7 @@ func (b *InMemoryBackend) UpdateThemePermissions( b.mu.Lock("UpdateThemePermissions") defer b.mu.Unlock() - t, ok := b.themes[themeKey(accountID, themeID)] + t, ok := b.themes.Get(themeKey(accountID, themeID)) if !ok { return nil, nil, ErrThemeNotFound } @@ -386,7 +379,7 @@ func (b *InMemoryBackend) CreateThemeAlias( b.mu.Lock("CreateThemeAlias") defer b.mu.Unlock() - t, ok := b.themes[themeKey(accountID, themeID)] + t, ok := b.themes.Get(themeKey(accountID, themeID)) if !ok { return nil, ErrThemeNotFound } @@ -412,7 +405,7 @@ func (b *InMemoryBackend) DescribeThemeAlias(accountID, themeID, aliasName strin b.mu.RLock("DescribeThemeAlias") defer b.mu.RUnlock() - t, ok := b.themes[themeKey(accountID, themeID)] + t, ok := b.themes.Get(themeKey(accountID, themeID)) if !ok { return nil, ErrThemeNotFound } @@ -436,7 +429,7 @@ func (b *InMemoryBackend) UpdateThemeAlias( b.mu.Lock("UpdateThemeAlias") defer b.mu.Unlock() - t, ok := b.themes[themeKey(accountID, themeID)] + t, ok := b.themes.Get(themeKey(accountID, themeID)) if !ok { return nil, ErrThemeNotFound } @@ -462,7 +455,7 @@ func (b *InMemoryBackend) DeleteThemeAlias(accountID, themeID, aliasName string) b.mu.Lock("DeleteThemeAlias") defer b.mu.Unlock() - t, ok := b.themes[themeKey(accountID, themeID)] + t, ok := b.themes.Get(themeKey(accountID, themeID)) if !ok { return ErrThemeNotFound } @@ -485,7 +478,7 @@ func (b *InMemoryBackend) ListThemeAliases( b.mu.RLock("ListThemeAliases") defer b.mu.RUnlock() - t, ok := b.themes[themeKey(accountID, themeID)] + t, ok := b.themes.Get(themeKey(accountID, themeID)) if !ok { return nil, "", ErrThemeNotFound } diff --git a/services/quicksight/backend_topics.go b/services/quicksight/backend_topics.go index d2c18996c..0f749b01b 100644 --- a/services/quicksight/backend_topics.go +++ b/services/quicksight/backend_topics.go @@ -132,7 +132,7 @@ func (b *InMemoryBackend) CreateTopic( defer b.mu.Unlock() key := topicKey(accountID, topicID) - if _, exists := b.topics[key]; exists { + if b.topics.Has(key) { return nil, ErrTopicAlreadyExists } @@ -151,7 +151,7 @@ func (b *InMemoryBackend) CreateTopic( Refreshes: make(map[string]*storedTopicRefresh), ReviewedAnswers: make(map[string]*storedTopicReviewedAnswer), } - b.topics[key] = t + b.topics.Put(t) if len(tags) > 0 { b.tags[t.Arn] = maps.Clone(tags) @@ -164,7 +164,7 @@ func (b *InMemoryBackend) DescribeTopic(accountID, topicID string) (*Topic, erro b.mu.RLock("DescribeTopic") defer b.mu.RUnlock() - t, ok := b.topics[topicKey(accountID, topicID)] + t, ok := b.topics.Get(topicKey(accountID, topicID)) if !ok { return nil, ErrTopicNotFound } @@ -180,7 +180,7 @@ func (b *InMemoryBackend) UpdateTopic( defer b.mu.Unlock() key := topicKey(accountID, topicID) - t, ok := b.topics[key] + t, ok := b.topics.Get(key) if !ok { return nil, ErrTopicNotFound } @@ -207,25 +207,19 @@ func (b *InMemoryBackend) DeleteTopic(accountID, topicID string) error { defer b.mu.Unlock() key := topicKey(accountID, topicID) - t, ok := b.topics[key] + t, ok := b.topics.Get(key) if !ok { return ErrTopicNotFound } delete(b.tags, t.Arn) - delete(b.topics, key) + b.topics.Delete(key) return nil } -func (b *InMemoryBackend) allTopicsLocked(accountID string) []*storedTopic { - prefix := accountID + "/" - all := make([]*storedTopic, 0, len(b.topics)) - for k, t := range b.topics { - if strings.HasPrefix(k, prefix) { - all = append(all, t) - } - } +func (b *InMemoryBackend) allTopicsLocked(_ string) []*storedTopic { + all := b.topics.All() sort.Slice(all, func(i, j int) bool { return all[i].TopicID < all[j].TopicID }) return all @@ -280,7 +274,7 @@ func (b *InMemoryBackend) ListTopics( // //nolint:dupl // search functions share structure but operate on different stored types func (b *InMemoryBackend) SearchTopics( - accountID string, + _ string, filters []SearchFilter, maxResults int32, nextToken string, @@ -288,10 +282,9 @@ func (b *InMemoryBackend) SearchTopics( b.mu.RLock("SearchTopics") defer b.mu.RUnlock() - prefix := accountID + "/" var filtered []*storedTopic - for k, t := range b.topics { - if strings.HasPrefix(k, prefix) && matchesAllNameFilters(t.Name, filters, filterTopicName) { + for _, t := range b.topics.All() { + if matchesAllNameFilters(t.Name, filters, filterTopicName) { filtered = append(filtered, t) } } @@ -368,7 +361,7 @@ func (b *InMemoryBackend) DescribeTopicPermissions(accountID, topicID string) (* b.mu.RLock("DescribeTopicPermissions") defer b.mu.RUnlock() - t, ok := b.topics[topicKey(accountID, topicID)] + t, ok := b.topics.Get(topicKey(accountID, topicID)) if !ok { return nil, nil, ErrTopicNotFound } @@ -383,7 +376,7 @@ func (b *InMemoryBackend) UpdateTopicPermissions( b.mu.Lock("UpdateTopicPermissions") defer b.mu.Unlock() - t, ok := b.topics[topicKey(accountID, topicID)] + t, ok := b.topics.Get(topicKey(accountID, topicID)) if !ok { return nil, nil, ErrTopicNotFound } @@ -410,7 +403,7 @@ func (b *InMemoryBackend) DescribeTopicRefresh(accountID, topicID, refreshID str b.mu.Lock("DescribeTopicRefresh") defer b.mu.Unlock() - t, ok := b.topics[topicKey(accountID, topicID)] + t, ok := b.topics.Get(topicKey(accountID, topicID)) if !ok { return nil, ErrTopicNotFound } @@ -442,7 +435,7 @@ func (b *InMemoryBackend) CreateTopicRefreshSchedule( b.mu.Lock("CreateTopicRefreshSchedule") defer b.mu.Unlock() - t, ok := b.topics[topicKey(accountID, topicID)] + t, ok := b.topics.Get(topicKey(accountID, topicID)) if !ok { return nil, ErrTopicNotFound } @@ -473,7 +466,7 @@ func (b *InMemoryBackend) DescribeTopicRefreshSchedule( b.mu.RLock("DescribeTopicRefreshSchedule") defer b.mu.RUnlock() - t, ok := b.topics[topicKey(accountID, topicID)] + t, ok := b.topics.Get(topicKey(accountID, topicID)) if !ok { return nil, ErrTopicNotFound } @@ -498,7 +491,7 @@ func (b *InMemoryBackend) UpdateTopicRefreshSchedule( b.mu.Lock("UpdateTopicRefreshSchedule") defer b.mu.Unlock() - t, ok := b.topics[topicKey(accountID, topicID)] + t, ok := b.topics.Get(topicKey(accountID, topicID)) if !ok { return nil, ErrTopicNotFound } @@ -525,7 +518,7 @@ func (b *InMemoryBackend) DeleteTopicRefreshSchedule(accountID, topicID, dataset b.mu.Lock("DeleteTopicRefreshSchedule") defer b.mu.Unlock() - t, ok := b.topics[topicKey(accountID, topicID)] + t, ok := b.topics.Get(topicKey(accountID, topicID)) if !ok { return ErrTopicNotFound } @@ -542,7 +535,7 @@ func (b *InMemoryBackend) ListTopicRefreshSchedules(accountID, topicID string) ( b.mu.RLock("ListTopicRefreshSchedules") defer b.mu.RUnlock() - t, ok := b.topics[topicKey(accountID, topicID)] + t, ok := b.topics.Get(topicKey(accountID, topicID)) if !ok { return nil, ErrTopicNotFound } @@ -570,7 +563,7 @@ func (b *InMemoryBackend) BatchCreateTopicReviewedAnswer( b.mu.Lock("BatchCreateTopicReviewedAnswer") defer b.mu.Unlock() - t, ok := b.topics[topicKey(accountID, topicID)] + t, ok := b.topics.Get(topicKey(accountID, topicID)) if !ok { return nil, nil, ErrTopicNotFound } @@ -618,7 +611,7 @@ func (b *InMemoryBackend) BatchDeleteTopicReviewedAnswer( b.mu.Lock("BatchDeleteTopicReviewedAnswer") defer b.mu.Unlock() - t, ok := b.topics[topicKey(accountID, topicID)] + t, ok := b.topics.Get(topicKey(accountID, topicID)) if !ok { return nil, nil, ErrTopicNotFound } @@ -643,7 +636,7 @@ func (b *InMemoryBackend) ListTopicReviewedAnswers(accountID, topicID string) ([ b.mu.RLock("ListTopicReviewedAnswers") defer b.mu.RUnlock() - t, ok := b.topics[topicKey(accountID, topicID)] + t, ok := b.topics.Get(topicKey(accountID, topicID)) if !ok { return nil, ErrTopicNotFound } diff --git a/services/quicksight/backend_vpcconnections.go b/services/quicksight/backend_vpcconnections.go index bfab20d87..46ec02ebe 100644 --- a/services/quicksight/backend_vpcconnections.go +++ b/services/quicksight/backend_vpcconnections.go @@ -3,7 +3,6 @@ package quicksight import ( "maps" "sort" - "strings" "time" ) @@ -64,7 +63,7 @@ func (b *InMemoryBackend) CreateVPCConnection( defer b.mu.Unlock() key := vpcConnectionKey(accountID, vpcConnectionID) - if _, exists := b.vpcConnections[key]; exists { + if b.vpcConnections.Has(key) { return nil, ErrVPCConnectionAlreadyExists } @@ -83,7 +82,7 @@ func (b *InMemoryBackend) CreateVPCConnection( Status: statusCreationSuccessful, AvailabilityStatus: vpcConnectionAvailabilityAvailable, } - b.vpcConnections[key] = v + b.vpcConnections.Put(v) if len(tags) > 0 { b.tags[v.Arn] = maps.Clone(tags) @@ -96,7 +95,7 @@ func (b *InMemoryBackend) DescribeVPCConnection(accountID, vpcConnectionID strin b.mu.RLock("DescribeVPCConnection") defer b.mu.RUnlock() - v, ok := b.vpcConnections[vpcConnectionKey(accountID, vpcConnectionID)] + v, ok := b.vpcConnections.Get(vpcConnectionKey(accountID, vpcConnectionID)) if !ok { return nil, ErrVPCConnectionNotFound } @@ -113,7 +112,7 @@ func (b *InMemoryBackend) UpdateVPCConnection( defer b.mu.Unlock() key := vpcConnectionKey(accountID, vpcConnectionID) - v, ok := b.vpcConnections[key] + v, ok := b.vpcConnections.Get(key) if !ok { return nil, ErrVPCConnectionNotFound } @@ -144,32 +143,27 @@ func (b *InMemoryBackend) DeleteVPCConnection(accountID, vpcConnectionID string) defer b.mu.Unlock() key := vpcConnectionKey(accountID, vpcConnectionID) - v, ok := b.vpcConnections[key] + v, ok := b.vpcConnections.Get(key) if !ok { return ErrVPCConnectionNotFound } delete(b.tags, v.Arn) - delete(b.vpcConnections, key) + b.vpcConnections.Delete(key) return nil } +//nolint:dupl // list functions share structure but operate on different stored types func (b *InMemoryBackend) ListVPCConnections( - accountID string, + _ string, maxResults int32, nextToken string, ) ([]*VPCConnection, string, error) { b.mu.RLock("ListVPCConnections") defer b.mu.RUnlock() - prefix := accountID + "/" - var all []*storedVPCConnection - for k, v := range b.vpcConnections { - if strings.HasPrefix(k, prefix) { - all = append(all, v) - } - } + all := b.vpcConnections.All() sort.Slice(all, func(i, j int) bool { return all[i].VPCConnectionID < all[j].VPCConnectionID }) if maxResults <= 0 || maxResults > defaultMaxResults { diff --git a/services/quicksight/export_test.go b/services/quicksight/export_test.go index c678268fc..08630a00c 100644 --- a/services/quicksight/export_test.go +++ b/services/quicksight/export_test.go @@ -5,7 +5,7 @@ func NamespaceCount(b *InMemoryBackend) int { b.mu.RLock("NamespaceCount") defer b.mu.RUnlock() - return len(b.namespaces) + return b.namespaces.Len() } // GroupCount returns the number of stored groups. @@ -13,7 +13,7 @@ func GroupCount(b *InMemoryBackend) int { b.mu.RLock("GroupCount") defer b.mu.RUnlock() - return len(b.groups) + return b.groups.Len() } // UserCount returns the number of stored users. @@ -21,7 +21,7 @@ func UserCount(b *InMemoryBackend) int { b.mu.RLock("UserCount") defer b.mu.RUnlock() - return len(b.users) + return b.users.Len() } // DataSourceCount returns the number of stored data sources. @@ -29,7 +29,7 @@ func DataSourceCount(b *InMemoryBackend) int { b.mu.RLock("DataSourceCount") defer b.mu.RUnlock() - return len(b.dataSources) + return b.dataSources.Len() } // DataSetCount returns the number of stored datasets. @@ -37,7 +37,7 @@ func DataSetCount(b *InMemoryBackend) int { b.mu.RLock("DataSetCount") defer b.mu.RUnlock() - return len(b.dataSets) + return b.dataSets.Len() } // DashboardCount returns the number of stored dashboards. @@ -45,7 +45,7 @@ func DashboardCount(b *InMemoryBackend) int { b.mu.RLock("DashboardCount") defer b.mu.RUnlock() - return len(b.dashboards) + return b.dashboards.Len() } // AnalysisCount returns the number of stored analyses. @@ -53,7 +53,7 @@ func AnalysisCount(b *InMemoryBackend) int { b.mu.RLock("AnalysisCount") defer b.mu.RUnlock() - return len(b.analyses) + return b.analyses.Len() } // FolderCount returns the number of stored folders. @@ -61,7 +61,7 @@ func FolderCount(b *InMemoryBackend) int { b.mu.RLock("FolderCount") defer b.mu.RUnlock() - return len(b.folders) + return b.folders.Len() } // TemplateCount returns the number of stored templates. @@ -69,7 +69,7 @@ func TemplateCount(b *InMemoryBackend) int { b.mu.RLock("TemplateCount") defer b.mu.RUnlock() - return len(b.templates) + return b.templates.Len() } // ThemeCount returns the number of stored themes. @@ -77,7 +77,7 @@ func ThemeCount(b *InMemoryBackend) int { b.mu.RLock("ThemeCount") defer b.mu.RUnlock() - return len(b.themes) + return b.themes.Len() } // VPCConnectionCount returns the number of stored VPC connections. @@ -85,7 +85,7 @@ func VPCConnectionCount(b *InMemoryBackend) int { b.mu.RLock("VPCConnectionCount") defer b.mu.RUnlock() - return len(b.vpcConnections) + return b.vpcConnections.Len() } // BrandCount returns the number of stored brands. @@ -93,7 +93,7 @@ func BrandCount(b *InMemoryBackend) int { b.mu.RLock("BrandCount") defer b.mu.RUnlock() - return len(b.brands) + return b.brands.Len() } // HandlerOpsLen returns the count of GetSupportedOperations. diff --git a/services/quicksight/store_roundtrip_test.go b/services/quicksight/store_roundtrip_test.go new file mode 100644 index 000000000..6edb878e8 --- /dev/null +++ b/services/quicksight/store_roundtrip_test.go @@ -0,0 +1,254 @@ +package quicksight_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/quicksight" +) + +// TestQuickSight_Phase3_3_StoreRoundTrip verifies that the Phase 3.3 +// pkgs/store conversion (backend.go, store_setup.go: every map[string]*T +// resource field with a pure identity registered as a *store.Table[T] on the +// backend's registry) preserves full backend state across a Snapshot/Restore +// round-trip. It seeds at least one entry in every converted table plus a +// representative sample of the maps deliberately left raw (tags, +// groupMembers, publicSharing -- see store_setup.go's doc comment), then +// asserts every seeded resource is independently readable from a brand-new +// backend restored from the snapshot. +func TestQuickSight_Phase3_3_StoreRoundTrip(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + const ns = "roundtrip-ns" + + // ---- seed one entry per converted store.Table ---- + + _, err := b.CreateNamespace(testAccountID, ns, testRegion) + require.NoError(t, err) + + _, err = b.CreateGroup(testAccountID, ns, "group1", "a group") + require.NoError(t, err) + + _, err = b.CreateGroupMembership(testAccountID, ns, "group1", "user1") + require.NoError(t, err) + + _, err = b.RegisterUser(testAccountID, ns, "user1", "user1@example.com", "READER", "QUICKSIGHT", "") + require.NoError(t, err) + + _, err = b.CreateDataSource(testAccountID, "ds1", "DataSource1", "MYSQL", nil, nil) + require.NoError(t, err) + + _, err = b.CreateDataSet(testAccountID, "dset1", "DataSet1", "SPICE", nil, nil) + require.NoError(t, err) + + _, err = b.CreateIngestion(testAccountID, "dset1", "ingest1") + require.NoError(t, err) + + dash, err := b.CreateDashboard(testAccountID, "dash1", "Dashboard1", nil, nil, nil) + require.NoError(t, err) + + _, err = b.CreateAnalysis(testAccountID, "an1", "Analysis1", nil, nil, nil) + require.NoError(t, err) + + _, err = b.CreateFolder(testAccountID, "folder1", "Folder1", "", "", nil, nil) + require.NoError(t, err) + + _, err = b.CreateFolderMembership(testAccountID, "folder1", "dash1", "DASHBOARD") + require.NoError(t, err) + + _, err = b.CreateTemplate(testAccountID, "tmpl1", "Template1", "", nil, nil, nil) + require.NoError(t, err) + + _, err = b.CreateTheme(testAccountID, "theme1", "Theme1", "", nil, nil, nil) + require.NoError(t, err) + + _, err = b.CreateTopic(testAccountID, "topic1", "Topic1", "", "", nil, nil, nil) + require.NoError(t, err) + + _, err = b.CreateVPCConnection(testAccountID, "vpc1", "VPCConn1", "vpc-1", nil, nil, nil, "", nil) + require.NoError(t, err) + + _, err = b.CreateIAMPolicyAssignment(testAccountID, ns, "assign1", "", "arn:aws:iam::000000000000:policy/p1", nil) + require.NoError(t, err) + + _, err = b.CreateAccountCustomization(testAccountID, ns, "theme1", "") + require.NoError(t, err) + + _, err = b.CreateBrand(testAccountID, "brand1", map[string]any{"BrandName": "Brand1"}) + require.NoError(t, err) + + _, err = b.CreateCustomPermissions(testAccountID, "cp1", nil, nil) + require.NoError(t, err) + + _, err = b.CreateOAuthClientApplication(testAccountID, "client1", "App1", nil) + require.NoError(t, err) + + err = b.UpdateIdentityPropagationConfig( + testAccountID, "REDSHIFT", []string{"arn:aws:sso::000000000000:application/foo"}, + ) + require.NoError(t, err) + + _, err = b.StartAssetBundleExportJob(testAccountID, "exportjob1", "", []string{dash.Arn}, false) + require.NoError(t, err) + + _, err = b.StartAssetBundleImportJob(testAccountID, "importjob1", "") + require.NoError(t, err) + + _, err = b.StartDashboardSnapshotJob(testAccountID, "dash1", "snapjob1", nil) + require.NoError(t, err) + + _, err = b.CreateActionConnector( + testAccountID, "ac1", "AC1", "GENERIC_HTTP", "", "", + map[string]any{"AuthenticationType": "NO_AUTH"}, nil, nil, + ) + require.NoError(t, err) + + autoJob, err := b.StartAutomationJob(testAccountID, "group1", "automation1", "") + require.NoError(t, err) + + quicksight.SeedFlow(b, testAccountID, &quicksight.Flow{ + CreatedTime: time.Now().UTC(), + LastUpdatedTime: time.Now().UTC(), + FlowID: "flow1", + Name: "Flow1", + PublishState: "PUBLISHED", + }) + + quicksight.SeedSelfUpgradeRequest(b, testAccountID, ns, &quicksight.SelfUpgradeRequestDetail{ + CreationTime: time.Now().UTC().Unix(), + UpgradeRequestID: "req1", + OriginalRole: "READER", + RequestedRole: "AUTHOR", + RequestStatus: "PENDING", + }) + + // ---- seed a representative sample of the maps left raw ---- + + require.NoError(t, b.TagResource(dash.Arn, map[string]string{"env": "test"})) + require.NoError(t, b.UpdatePublicSharingSettings(testAccountID, true)) + + // ---- snapshot, then restore onto a brand-new backend ---- + + snapshot := b.Snapshot(t.Context()) + require.NotNil(t, snapshot) + + restored := quicksight.NewInMemoryBackend(testAccountID, testRegion) + require.NoError(t, restored.Restore(t.Context(), snapshot)) + + // ---- verify every converted table's seeded entry survived ---- + + _, err = restored.DescribeNamespace(testAccountID, ns) + require.NoError(t, err) + + _, err = restored.DescribeGroup(testAccountID, ns, "group1") + require.NoError(t, err) + + _, err = restored.DescribeGroupMembership(testAccountID, ns, "group1", "user1") + require.NoError(t, err) + + _, err = restored.DescribeUser(testAccountID, ns, "user1") + require.NoError(t, err) + + _, err = restored.DescribeDataSource(testAccountID, "ds1") + require.NoError(t, err) + + _, err = restored.DescribeDataSet(testAccountID, "dset1") + require.NoError(t, err) + + _, err = restored.DescribeIngestion(testAccountID, "dset1", "ingest1") + require.NoError(t, err) + + restoredDash, err := restored.DescribeDashboard(testAccountID, "dash1") + require.NoError(t, err) + + _, err = restored.DescribeAnalysis(testAccountID, "an1") + require.NoError(t, err) + + _, err = restored.DescribeFolder(testAccountID, "folder1") + require.NoError(t, err) + + members, _, err := restored.ListFolderMembers(testAccountID, "folder1", 0, "") + require.NoError(t, err) + assert.Len(t, members, 1) + + _, err = restored.DescribeTemplate(testAccountID, "tmpl1", 0) + require.NoError(t, err) + + _, err = restored.DescribeTheme(testAccountID, "theme1", 0) + require.NoError(t, err) + + _, err = restored.DescribeTopic(testAccountID, "topic1") + require.NoError(t, err) + + _, err = restored.DescribeVPCConnection(testAccountID, "vpc1") + require.NoError(t, err) + + _, err = restored.DescribeIAMPolicyAssignment(testAccountID, ns, "assign1") + require.NoError(t, err) + + _, err = restored.DescribeAccountCustomization(testAccountID, ns) + require.NoError(t, err) + + _, err = restored.DescribeBrand(testAccountID, "brand1", "") + require.NoError(t, err) + + _, err = restored.DescribeCustomPermissions(testAccountID, "cp1") + require.NoError(t, err) + + _, err = restored.DescribeOAuthClientApplication(testAccountID, "client1") + require.NoError(t, err) + + idPropConfigs, err := restored.ListIdentityPropagationConfigs(testAccountID) + require.NoError(t, err) + assert.Len(t, idPropConfigs, 1) + + _, err = restored.DescribeAssetBundleExportJob(testAccountID, "exportjob1") + require.NoError(t, err) + + _, err = restored.DescribeAssetBundleImportJob(testAccountID, "importjob1") + require.NoError(t, err) + + _, err = restored.DescribeDashboardSnapshotJob(testAccountID, "dash1", "snapjob1") + require.NoError(t, err) + + _, err = restored.DescribeActionConnector(testAccountID, "ac1") + require.NoError(t, err) + + _, err = restored.DescribeAutomationJob(testAccountID, "group1", "automation1", autoJob.JobID) + require.NoError(t, err) + + _, err = restored.GetFlowMetadata(testAccountID, "flow1") + require.NoError(t, err) + + selfUpgrades, _, err := restored.ListSelfUpgrades(testAccountID, ns, 0, "") + require.NoError(t, err) + assert.Len(t, selfUpgrades, 1) + + // ---- verify the raw-left maps survived too ---- + + tags, err := restored.ListTagsForResource(restoredDash.Arn) + require.NoError(t, err) + assert.Equal(t, "test", tags["env"]) + + settings, err := restored.DescribeAccountSettings(testAccountID) + require.NoError(t, err) + assert.True(t, settings.PublicSharingEnabled) + + // ---- sanity-check table counts via the export_test.go accessors ---- + assert.Equal(t, 2, quicksight.NamespaceCount(restored)) // default + ns + assert.Equal(t, 1, quicksight.GroupCount(restored)) + assert.Equal(t, 1, quicksight.UserCount(restored)) + assert.Equal(t, 1, quicksight.DataSourceCount(restored)) + assert.Equal(t, 1, quicksight.DataSetCount(restored)) + assert.Equal(t, 1, quicksight.DashboardCount(restored)) + assert.Equal(t, 1, quicksight.AnalysisCount(restored)) + assert.Equal(t, 1, quicksight.FolderCount(restored)) + assert.Equal(t, 1, quicksight.TemplateCount(restored)) + assert.Equal(t, 1, quicksight.ThemeCount(restored)) + assert.Equal(t, 1, quicksight.VPCConnectionCount(restored)) + assert.Equal(t, 1, quicksight.BrandCount(restored)) +} diff --git a/services/quicksight/store_setup.go b/services/quicksight/store_setup.go new file mode 100644 index 000000000..223c3f740 --- /dev/null +++ b/services/quicksight/store_setup.go @@ -0,0 +1,237 @@ +package quicksight + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend that has a pure per-value +// identity is registered exactly once, here, as a *store.Table[T] on +// b.registry. See pkgs/store's package doc and the services/ec2 (commit +// 12e611a4), services/sqs (commit 0f09d77c), and services/ssm (commit +// 4c686770) conversions this follows. +// +// QuickSight's InMemoryBackend, like ec2/sqs, holds exactly one (accountID, +// region) pair per instance (see provider.go: one backend is constructed per +// service.AccountRegionOrDefault call), so the flat, statically-registered +// pattern applies -- not ssm's per-region lazy registration. +// +// # Composite keys +// +// Several stored types are keyed externally by accountID+"/"+ (e.g. +// nsKey, groupKey, folderKey) even though accountID is not itself a field on +// the stored struct. Because every value ever placed in one of these tables +// was created by this same backend instance, accountID is always b.accountID +// (see provider.go), so each keyFn below is a closure over b that reconstructs +// the exact same key string the old map used, from b.accountID plus the +// value's own identity field(s). +// +// # What is NOT converted here, and why +// +// The following resource fields are deliberately left as plain maps, matching +// the ec2/ssm precedent of leaving non-identity-bearing collections raw: +// - accountSettings, accountSubscriptions, ipRestrictions, +// defaultQBusinessApps: value type carries no identity field of its own +// (no AccountID field); each is a single per-account slot keyed externally +// by accountID alone -- same rationale as ec2's instanceIMDSOptions +// exclusion. +// - groupMembers, publicSharing, roleMemberships: map[string]bool, a bare +// membership flag with no identity. +// - tags: map[string]map[string]string, value is itself a map with no +// identity field. +// - accountCustomPermissions, qPersonalization, qSearchConfig, +// dashboardsQAConfig, brandAssignments, roleCustomPermissions, +// userCustomPermissions, spiceCapacity, selfUpgradeConfig: +// map[string]string, a bare scalar with no identity. +// - keyRegistrations: map[string][]storedRegisteredKey, one-to-many +// (slice-valued) -- there is no single V to key a Table by, same as ec2's +// spotFleetHistory pattern. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +// registerAllTables registers every convertible resource collection on +// b.registry, assigning each *store.Table[T] to its InMemoryBackend field. +// Called once from NewInMemoryBackend. +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call to the concrete field and value type. A closure list (rather than one +// statement per field in a single function body) keeps registerAllTables +// small regardless of how many resource tables the backend grows to. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.namespaces = store.Register(b.registry, "namespaces", store.New(func(v *storedNamespace) string { + return nsKey(b.accountID, v.Name) + })) + }, + func(b *InMemoryBackend) { + b.groups = store.Register(b.registry, "groups", store.New(func(v *storedGroup) string { + return groupKey(b.accountID, v.Namespace, v.GroupName) + })) + }, + func(b *InMemoryBackend) { + b.users = store.Register(b.registry, "users", store.New(func(v *storedUser) string { + return userKey(b.accountID, v.Namespace, v.UserName) + })) + }, + func(b *InMemoryBackend) { + b.dataSources = store.Register(b.registry, "dataSources", store.New(func(v *storedDataSource) string { + return dataSourceKey(b.accountID, v.DataSourceID) + })) + }, + func(b *InMemoryBackend) { + b.dataSets = store.Register(b.registry, "dataSets", store.New(func(v *storedDataSet) string { + return dataSetKey(b.accountID, v.DataSetID) + })) + }, + func(b *InMemoryBackend) { + b.ingestions = store.Register(b.registry, "ingestions", store.New(func(v *storedIngestion) string { + return ingestionKey(b.accountID, v.DataSetID, v.IngestionID) + })) + }, + func(b *InMemoryBackend) { + b.dashboards = store.Register(b.registry, "dashboards", store.New(func(v *storedDashboard) string { + return dashboardKey(b.accountID, v.DashboardID) + })) + }, + func(b *InMemoryBackend) { + b.analyses = store.Register(b.registry, "analyses", store.New(func(v *storedAnalysis) string { + return analysisKey(b.accountID, v.AnalysisID) + })) + }, + func(b *InMemoryBackend) { + b.folders = store.Register(b.registry, "folders", store.New(func(v *storedFolder) string { + return folderKey(b.accountID, v.FolderID) + })) + }, + func(b *InMemoryBackend) { + b.folderMembers = store.Register(b.registry, "folderMembers", store.New(func(v *storedFolderMember) string { + return folderMemberKey(b.accountID, v.FolderID, v.MemberType, v.MemberID) + })) + }, + func(b *InMemoryBackend) { + b.templates = store.Register(b.registry, "templates", store.New(func(v *storedTemplate) string { + return templateKey(b.accountID, v.TemplateID) + })) + }, + func(b *InMemoryBackend) { + b.themes = store.Register(b.registry, "themes", store.New(func(v *storedTheme) string { + return themeKey(b.accountID, v.ThemeID) + })) + }, + func(b *InMemoryBackend) { + b.topics = store.Register(b.registry, "topics", store.New(func(v *storedTopic) string { + return topicKey(b.accountID, v.TopicID) + })) + }, + func(b *InMemoryBackend) { + b.vpcConnections = store.Register(b.registry, "vpcConnections", store.New(func(v *storedVPCConnection) string { + return vpcConnectionKey(b.accountID, v.VPCConnectionID) + })) + }, + func(b *InMemoryBackend) { + b.iamPolicyAssignments = store.Register( + b.registry, + "iamPolicyAssignments", + store.New(func(v *storedIAMPolicyAssignment) string { + return iamPolicyAssignmentKey(b.accountID, v.Namespace, v.AssignmentName) + }), + ) + }, + func(b *InMemoryBackend) { + b.accountCustomizations = store.Register( + b.registry, + "accountCustomizations", + store.New(func(v *storedAccountCustomization) string { + return accountCustomizationKey(b.accountID, v.Namespace) + }), + ) + }, + func(b *InMemoryBackend) { + b.brands = store.Register(b.registry, "brands", store.New(func(v *storedBrand) string { + return brandKey(b.accountID, v.BrandID) + })) + }, + func(b *InMemoryBackend) { + b.customPermissions = store.Register( + b.registry, + "customPermissions", + store.New(func(v *storedCustomPermissions) string { + return customPermissionsKey(b.accountID, v.Name) + }), + ) + }, + func(b *InMemoryBackend) { + b.oauthClientApps = store.Register(b.registry, "oauthClientApps", store.New(func(v *storedOAuthApp) string { + return oauthAppKey(b.accountID, v.ClientID) + })) + }, + func(b *InMemoryBackend) { + b.identityPropagationConfigs = store.Register( + b.registry, + "identityPropagationConfigs", + store.New(func(v *storedIdentityPropagationConfig) string { + return identityPropagationKey(b.accountID, v.Service) + }), + ) + }, + func(b *InMemoryBackend) { + b.assetBundleExportJobs = store.Register( + b.registry, + "assetBundleExportJobs", + store.New(func(v *storedAssetBundleExportJob) string { + return assetBundleJobKey(b.accountID, v.JobID) + }), + ) + }, + func(b *InMemoryBackend) { + b.assetBundleImportJobs = store.Register( + b.registry, + "assetBundleImportJobs", + store.New(func(v *storedAssetBundleImportJob) string { + return assetBundleJobKey(b.accountID, v.JobID) + }), + ) + }, + func(b *InMemoryBackend) { + b.dashboardSnapshotJobs = store.Register( + b.registry, + "dashboardSnapshotJobs", + store.New(func(v *storedDashboardSnapshotJob) string { + return dashboardSnapshotJobKey(b.accountID, v.DashboardID, v.JobID) + }), + ) + }, + func(b *InMemoryBackend) { + b.actionConnectors = store.Register( + b.registry, + "actionConnectors", + store.New(func(v *storedActionConnector) string { + return actionConnectorKey(b.accountID, v.ActionConnectorID) + }), + ) + }, + func(b *InMemoryBackend) { + b.automationJobs = store.Register(b.registry, "automationJobs", store.New(func(v *storedAutomationJob) string { + return automationJobKey(b.accountID, v.AutomationGroupID, v.AutomationID, v.JobID) + })) + }, + func(b *InMemoryBackend) { + b.flows = store.Register(b.registry, "flows", store.New(func(v *storedFlow) string { + return flowKey(b.accountID, v.FlowID) + })) + }, + func(b *InMemoryBackend) { + b.selfUpgradeRequests = store.Register( + b.registry, + "selfUpgradeRequests", + store.New(func(v *storedSelfUpgradeRequest) string { + return selfUpgradeRequestKey(b.accountID, v.Namespace, v.UpgradeRequestID) + }), + ) + }, +} diff --git a/services/rds/PARITY.md b/services/rds/PARITY.md new file mode 100644 index 000000000..03eddd4da --- /dev/null +++ b/services/rds/PARITY.md @@ -0,0 +1,140 @@ +# PARITY MANIFEST SCHEMA — copy to services//PARITY.md, fill, keep updated. +# Purpose: record audit state so the NEXT audit diffs the delta instead of rescanning. +# Re-audit protocol: `git diff ..HEAD -- services//` for local drift, +# AND check the SDK module for ops added since sdk_version. Only audit changed/new surface; +# trust rows marked ok whose files are unchanged since last_audit_commit. +service: rds +sdk_module: aws-sdk-go-v2/service/rds@v1.116.2 +last_audit_commit: 23de3aab +last_audit_date: 2026-07-05 +overall: B # already-accurate on nearly all of ~140 routed ops (5+ prior audit + # passes: batch1-3, refinement1-4, #2213/#2226/#2227/#2329/#2334/#2339); + # this pass found and fixed 2 genuine wire/behavior gaps (final-snapshot + # contract on delete, DescribeDBInstances Filters) rather than a + # ground-up rewrite. +ops: + DeleteDBInstance: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed this pass — see gaps"} + DeleteDBCluster: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed this pass — see gaps"} + DescribeDBInstances: {wire: ok, errors: ok, state: ok, persist: ok, note: "Filters added this pass"} + CreateDBInstance: {wire: ok, errors: ok, state: ok, persist: ok} + ModifyDBInstance: {wire: ok, errors: ok, state: ok, persist: ok} + StartDBInstance: {wire: ok, errors: ok, state: ok, persist: ok} + StopDBInstance: {wire: ok, errors: ok, state: ok, persist: ok} + RebootDBInstance: {wire: ok, errors: ok, state: ok, persist: ok} + CreateDBInstanceReadReplica: {wire: ok, errors: ok, state: ok, persist: ok} + PromoteReadReplica: {wire: ok, errors: ok, state: ok, persist: ok} + CreateDBCluster: {wire: ok, errors: ok, state: ok, persist: ok} + ModifyDBCluster: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeDBClusters: {wire: ok, errors: ok, state: ok, persist: ok, note: "no Filters support — gap"} + CreateDBSnapshot: {wire: ok, errors: ok, state: ok, persist: ok} + CopyDBSnapshot: {wire: ok, errors: ok, state: ok, persist: ok} + RestoreDBInstanceFromDBSnapshot: {wire: ok, errors: ok, state: ok, persist: ok} + RestoreDBInstanceToPointInTime: {wire: ok, errors: ok, state: ok, persist: ok} + CreateDBClusterSnapshot: {wire: ok, errors: ok, state: ok, persist: ok} + CreateDBParameterGroup: {wire: ok, errors: ok, state: ok, persist: ok} + ModifyDBParameterGroup: {wire: ok, errors: ok, state: ok, persist: ok} + ResetDBParameterGroup: {wire: ok, errors: ok, state: ok, persist: ok} + CreateDBSubnetGroup: {wire: ok, errors: ok, state: ok, persist: ok} + CreateOptionGroup: {wire: ok, errors: ok, state: ok, persist: ok} +families: + db_instance_lifecycle: {status: ok, note: "creating->available->modifying/deleting state machine via instanceReadyAt + self-terminating reconciler goroutine (backend.go scheduleReconcilerLocked); verified transitions, DeletionProtection guard, already-deleting guard"} + db_cluster_lifecycle: {status: ok, note: "cluster members, reader/writer endpoint synthesis, ServerlessV2ScalingConfiguration, start/stop/failover/reboot all mutate real state"} + snapshots_manual_automated: {status: ok, note: "CreateDBSnapshot/CopyDBSnapshot/Delete/Describe/Restore all real; SnapshotType manual vs automated distinguished; final-snapshot-on-delete gap fixed this pass (see Notes)"} + parameter_groups: {status: ok, note: "apply-method immediate vs pending-reboot honored in ModifyDBParameterGroup/ApplyPendingMaintenanceAction path; Reset/Copy real"} + subnet_groups: {status: ok, note: "CRUD verified against DBSubnetGroup shape"} + option_groups: {status: ok, note: "CRUD + Copy + option add/remove real"} + read_replicas: {status: ok, note: "source linkage bidirectional (ReplicaSourceDBInstanceIdentifier / ReadReplicaIdentifiers), promote clears linkage, cross-region replica path uses defaults when source not locally resolvable"} + events_and_subscriptions: {status: ok, note: "ring-buffered Events (maxEvents cap prevents unbounded growth); EventSubscription CRUD + source-identifier add/remove real"} + engine_versions_and_orderable_options: {status: ok, note: "DescribeDBEngineVersions/DescribeOrderableDBInstanceOptions/DescribeDBMajorEngineVersions all backed by real (small, static) catalogs — not a stub since callers get consistent, well-shaped data; no engine-name validation on Create (see gaps)"} + tags: {status: ok, note: "AddTagsToResource/RemoveTagsFromResource/ListTagsForResource use pkgs/tags-style per-ARN map, cleaned up on every delete path (instance, cluster, snapshot, option group, param group, cluster endpoint — verified via TestRDSBackend_TagsCleanedUpOnDelete table)"} + pagination: {status: ok, note: "Marker/MaxRecords via pkgs/page.Page[T] (paginateDescribe) — consistent across all Describe* ops"} + describe_filters: {status: partial, note: "DescribeDBInstances Filters (db-cluster-id/db-instance-id/dbi-resource-id/domain/engine) added this pass; other Describe ops (DescribeDBClusters, DescribeDBSnapshots, DescribeDBClusterSnapshots, DescribeEvents) still ignore Filters — see gaps"} + global_clusters: {status: ok, note: "Create/Modify/Delete/Describe + Remove/Failover/SwitchoverGlobalCluster real"} + blue_green_deployments: {status: ok, note: "Create/Describe/Delete/Switchover real (refinement1)"} + db_proxies: {status: ok, note: "proxy/proxy-target/proxy-target-group/proxy-endpoint CRUD real (refinement3)"} + reserved_instances: {status: ok, note: "Purchase + Describe(Offerings) real"} + performance_insights: {status: ok, note: "GetPerformanceInsightsMetrics requires seeded data via SetPerformanceInsightsData — not a fabricated-on-the-fly stub; batch3_test.go.rej/.patch cruft from a prior sweep's already-applied fix removed this pass"} + error_codes: {status: ok, note: "awserr sentinels map to correct AWS fault codes (DBInstanceNotFound, DBInstanceAlreadyExists, InvalidDBInstanceState, DBSnapshotNotFound, InvalidParameterValue, InvalidParameterCombination) with correct HTTP status via errCodeLookup in handler.go"} + leaks: {status: ok, note: "single reconciler goroutine per backend; self-terminates when instanceReadyAt/clusterReadyAt both empty (no ticker leak); no unbounded maps found — events ring-buffered at maxEvents"} +gaps: + - DeleteDBInstance/DeleteDBCluster previously ignored SkipFinalSnapshot/FinalDBSnapshotIdentifier/DeleteAutomatedBackups entirely (silently always skipped the final snapshot with no validation) — FIXED this pass (bd: gopherstack-bgl) + - DescribeDBInstances previously ignored the Filters parameter entirely — FIXED this pass for db-cluster-id/db-instance-id/dbi-resource-id/domain/engine (bd: gopherstack-bgl) + - DescribeDBClusters, DescribeDBSnapshots, DescribeDBClusterSnapshots, DescribeEvents still ignore Filters — same shape of gap as DescribeDBInstances before this pass, not fixed (scope/time); follow-up under gopherstack-bgl + - DB instance/cluster/snapshot/parameter-group identifiers are compared case-sensitively (Go map keys); real AWS treats DBInstanceIdentifier etc. as case-insensitive (e.g. creating "MyDB" then "mydb" should collide with DBInstanceAlreadyExistsFault but does not here) — not fixed: normalizing would touch every resource map across ~30K LOC and was judged too invasive for a scoped, low-risk pass; flagged for a dedicated follow-up + - CreateDBInstance/CreateDBCluster do not validate the Engine name against a known-engine list; any string is accepted (real AWS returns InvalidParameterValue for an unsupported engine) — not fixed: many existing tests rely on the current permissive behavior with ad hoc engine strings; changing this is a larger, separately-scoped hardening task +deferred: + - DB shard groups / integrations / tenant databases (Aurora Limitless / zero-ETL) — spot-checked as real (not stubs) but not re-verified op-by-op against the newest SDK field additions this pass + - Activity streams / DB security groups (EC2-Classic legacy) — spot-checked only +leaks: {status: clean, note: "reconciler goroutine (backend.go:scheduleReconcilerLocked) is per-backend, started lazily, and exits its own loop once both instanceReadyAt and clusterReadyAt are empty; Close() is a documented no-op given this. No time.Sleep/unbounded-map patterns found in non-test files."} + +## Notes + +- Protocol: RDS uses the AWS Query (XML) protocol, version `2014-10-31`, XML namespace + `http://rds.amazonaws.com/doc/2014-10-31/`. Every response wraps in + `...` except where the op's + SDK output has no members (in which case an empty result element is still correct — do not + flag as a stub). + +- **DeleteDBInstance / DeleteDBCluster final-snapshot contract (fixed this pass).** Real AWS + requires exactly one of `SkipFinalSnapshot=true` or a non-empty `FinalDBSnapshotIdentifier` + (`FinalDBClusterSnapshotIdentifier` for clusters); supplying both is + `InvalidParameterCombination`, as is supplying neither. Before this pass the emulator's + `DeleteDBInstance`/`DeleteDBCluster` took only an identifier and silently behaved as if + `SkipFinalSnapshot=true` always — no validation, and no final snapshot was ever created even + when a client explicitly asked for one. This is exactly the "disguised stub" bug class from + `.claude/memories/parity-principles.md` #4: the delete itself was real (removed real state), + but a whole documented, commonly-exercised parameter contract was silently ignored. + Fixed by adding `DeleteDBInstanceWithOptions`/`DeleteDBClusterWithOptions` (additive — the + existing `DeleteDBInstance(id)`/`DeleteDBCluster(id)` single-arg methods are kept unchanged + and now delegate with `skipFinalSnapshot=true` since they are called by + `services/cloudformation/resources_phase2.go` and `resources_phase4.go`, which are outside + this audit's edit scope). AWS resolves the target resource before validating the snapshot + parameter combination — a delete against a nonexistent instance/cluster returns + `DBInstanceNotFound`/`DBClusterNotFound` even when `SkipFinalSnapshot`/ + `FinalDBSnapshotIdentifier` are also invalid or missing; order the checks accordingly (existence + first) or existing/incoming client integration tests that intentionally omit both params + against a missing resource will regress to the wrong error code. + +- **DescribeDBInstances Filters (fixed this pass, partial).** AWS's DescribeDBInstances + accepts `Filters.Filter.N.Name` / `Filters.Filter.N.Values.member.M` with recognized names + `db-cluster-id`, `db-instance-id`, `dbi-resource-id`, `domain`, `engine`. Multiple values + within one filter OR together; multiple filters AND together. An unrecognized filter name + is `InvalidParameterValue`. `domain` is accepted (to avoid rejecting otherwise-valid client + requests) but is not modeled as a real predicate since this emulator has no Directory + Service domain-membership state — every instance vacuously "matches" a domain filter. The + same Filters shape is documented on `DescribeDBClusters`, `DescribeDBSnapshots`, + `DescribeDBClusterSnapshots`, and `DescribeEvents` but was out of scope to also implement + this pass; see gaps. + +- **`newManualSnapshotLocked` / `newManualClusterSnapshotLocked`.** Both `CreateDBSnapshot`/ + `CreateDBClusterSnapshot` and the new delete-time final-snapshot path build the same + `DBSnapshot`/`DBClusterSnapshot` shape, so the struct-building logic was extracted into a + `*Locked` helper (caller must already hold `b.mu`) shared by both call sites — avoids + duplicating the AWS shape twice and risking the two copies drifting apart. + +- **Case-sensitive identifiers (not fixed, documented gap).** Every resource map in + `backend.go` (`instances`, `clusters`, `snapshots`, `parameterGroups`, ...) is keyed by the + identifier string exactly as given. Real RDS identifiers are case-insensitive persistent + handles (AWS lower-cases them internally), so `CreateDBInstance("MyDB", ...)` followed by + `CreateDBInstance("mydb", ...)` should collide with `DBInstanceAlreadyExistsFault` in real + AWS but creates two independent instances here. This is a real, if narrow, wire-behavior gap + found while auditing `CreateDBInstance`/`DeleteDBInstance`, but normalizing it correctly + touches every map (create/lookup/delete) across instances, clusters, snapshots, parameter + groups, option groups, subnet groups, global clusters, etc. — a change of that breadth + deserves its own focused, fully-tested pass rather than a partial fix bundled into this one + (a half-normalized service, where instance IDs fold case but snapshot IDs don't, would be + worse than the current consistent-but-wrong behavior). + +- The stray `batch3_test.go.rej` / `batch3_test_pi.patch` files (tracked in git, dated the day + before this audit) were leftover artifacts of a previously-applied patch — diffed against + `batch3_test.go` and confirmed the patch's content (deterministic-vs-flaky + `TestPerformanceInsights_*` fixes) was already live in the tracked test file. Removed as + dead weight; not a behavior change. + +- No goroutine/ticker/map leaks found. The single background reconciler + (`scheduleReconcilerLocked`) is started lazily per backend on first + `CreateDBInstance`/`ModifyDBInstance`/etc. and exits its own `for` loop once + `len(b.instanceReadyAt) == 0 && len(b.clusterReadyAt) == 0`, so it does not run forever in a + backend that settles into a steady state. `events` is capped at `maxEvents` (ring-buffer + trim). No `context.Background()`-rooted unbounded goroutines outside `fis.go` (chaos + fault-injection, out of this pass's scope) were found in non-test `.go` files. diff --git a/services/rds/accuracy_test.go b/services/rds/accuracy_test.go index c12dfede6..62b215131 100644 --- a/services/rds/accuracy_test.go +++ b/services/rds/accuracy_test.go @@ -770,6 +770,7 @@ func TestDeleteReplicaClearsSourceReadReplicaIdentifiers(t *testing.T) { "Action": {"DeleteDBInstance"}, "Version": {"2014-10-31"}, "DBInstanceIdentifier": {"rep-del"}, + "SkipFinalSnapshot": {"true"}, }) require.Equal(t, http.StatusOK, rec.Code) diff --git a/services/rds/backend.go b/services/rds/backend.go index 574506c90..7d466ce7d 100644 --- a/services/rds/backend.go +++ b/services/rds/backend.go @@ -15,6 +15,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awserr" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) // dbInstanceIDRegex matches valid RDS DBInstanceIdentifier values. @@ -727,40 +728,41 @@ type DBClusterOptions struct { // InMemoryBackend is the in-memory store for RDS resources. type InMemoryBackend struct { dnsRegistrar DNSRegistrar - snapshotAttributes map[string]*DBSnapshotAttributesResult - reservedInstances map[string]*ReservedDBInstance - snapshots map[string]*DBSnapshot - subnetGroups map[string]*DBSubnetGroup + registry *store.Registry + snapshotAttributes *store.Table[DBSnapshotAttributesResult] + reservedInstances *store.Table[ReservedDBInstance] + snapshots *store.Table[DBSnapshot] + subnetGroups *store.Table[DBSubnetGroup] tags map[string][]Tag - instances map[string]*DBInstance - clusterParameterGroups map[string]*DBParameterGroup - optionGroups map[string]*OptionGroup - clusters map[string]*DBCluster + instances *store.Table[DBInstance] + clusterParameterGroups *store.Table[DBParameterGroup] + optionGroups *store.Table[OptionGroup] + clusters *store.Table[DBCluster] instanceReadyAt map[string]time.Time - clusterSnapshots map[string]*DBClusterSnapshot - eventSubscriptions map[string]*EventSubscription - globalClusters map[string]*GlobalCluster + clusterSnapshots *store.Table[DBClusterSnapshot] + eventSubscriptions *store.Table[EventSubscription] + globalClusters *store.Table[GlobalCluster] clusterRoles map[string][]string instanceRoles map[string][]string - exportTasks map[string]*ExportTask + exportTasks *store.Table[ExportTask] mu *lockmetrics.RWMutex - dbSecurityGroups map[string]*DBSecurityGroup - blueGreenDeployments map[string]*BlueGreenDeployment - clusterEndpoints map[string]*DBClusterEndpoint - parameterGroups map[string]*DBParameterGroup - recommendations map[string]*DBRecommendation - clusterSnapshotAttributes map[string]*DBClusterSnapshotAttributesResult - proxies map[string]*DBProxy - proxyTargetGroups map[string]*DBProxyTargetGroup + dbSecurityGroups *store.Table[DBSecurityGroup] + blueGreenDeployments *store.Table[BlueGreenDeployment] + clusterEndpoints *store.Table[DBClusterEndpoint] + parameterGroups *store.Table[DBParameterGroup] + recommendations *store.Table[DBRecommendation] + clusterSnapshotAttributes *store.Table[DBClusterSnapshotAttributesResult] + proxies *store.Table[DBProxy] + proxyTargetGroups *store.Table[DBProxyTargetGroup] proxyTargets map[string][]DBProxyTarget - proxyEndpoints map[string]*DBProxyEndpoint - customEngineVersions map[string]*CustomDBEngineVersion + proxyEndpoints *store.Table[DBProxyEndpoint] + customEngineVersions *store.Table[CustomDBEngineVersion] fisFailoverFaults map[string]time.Time automatedBackups map[string]*DBInstanceAutomatedBackup - shardGroups map[string]*DBShardGroup - integrations map[string]*Integration - tenantDatabases map[string]*TenantDatabase - clusterAutomatedBackups map[string]*DBClusterAutomatedBackup + shardGroups *store.Table[DBShardGroup] + integrations *store.Table[Integration] + tenantDatabases *store.Table[TenantDatabase] + clusterAutomatedBackups *store.Table[DBClusterAutomatedBackup] snapshotTenantDatabases map[string][]*DBSnapshotTenantDatabase clusterReadyAt map[string]time.Time piMetrics map[string]map[string][]PIDataPoint @@ -776,50 +778,26 @@ type InMemoryBackend struct { // NewInMemoryBackend creates a new InMemoryBackend with a background reconciler. func NewInMemoryBackend(accountID, region string) *InMemoryBackend { b := &InMemoryBackend{ - instances: make(map[string]*DBInstance), - instanceReadyAt: make(map[string]time.Time), - snapshots: make(map[string]*DBSnapshot), - subnetGroups: make(map[string]*DBSubnetGroup), - tags: make(map[string][]Tag), - parameterGroups: make(map[string]*DBParameterGroup), - clusterParameterGroups: make(map[string]*DBParameterGroup), - optionGroups: make(map[string]*OptionGroup), - clusters: make(map[string]*DBCluster), - clusterSnapshots: make(map[string]*DBClusterSnapshot), - clusterEndpoints: make(map[string]*DBClusterEndpoint), - exportTasks: make(map[string]*ExportTask), - globalClusters: make(map[string]*GlobalCluster), - clusterRoles: make(map[string][]string), - instanceRoles: make(map[string][]string), - eventSubscriptions: make(map[string]*EventSubscription), - events: make([]Event, 0), - dbSecurityGroups: make(map[string]*DBSecurityGroup), - blueGreenDeployments: make(map[string]*BlueGreenDeployment), - fisFailoverFaults: make(map[string]time.Time), - snapshotAttributes: make(map[string]*DBSnapshotAttributesResult), - clusterSnapshotAttributes: make(map[string]*DBClusterSnapshotAttributesResult), - reservedInstances: make(map[string]*ReservedDBInstance), - recommendations: make(map[string]*DBRecommendation), - proxies: make(map[string]*DBProxy), - proxyTargetGroups: make(map[string]*DBProxyTargetGroup), - proxyTargets: make(map[string][]DBProxyTarget), - proxyEndpoints: make(map[string]*DBProxyEndpoint), - automatedBackups: make(map[string]*DBInstanceAutomatedBackup), - customEngineVersions: make(map[string]*CustomDBEngineVersion), - shardGroups: make(map[string]*DBShardGroup), - integrations: make(map[string]*Integration), - tenantDatabases: make(map[string]*TenantDatabase), - clusterAutomatedBackups: make(map[string]*DBClusterAutomatedBackup), - snapshotTenantDatabases: make(map[string][]*DBSnapshotTenantDatabase), - clusterReadyAt: make(map[string]time.Time), - piMetrics: make(map[string]map[string][]PIDataPoint), - instanceLogFiles: make(map[string][]DBLogFile), - instanceLogContent: make(map[string]map[string]string), - accountID: accountID, - region: region, - defaultCACertificateID: defaultCACertificateID, - mu: lockmetrics.New("rds"), - } + registry: store.NewRegistry(), + instanceReadyAt: make(map[string]time.Time), + tags: make(map[string][]Tag), + clusterRoles: make(map[string][]string), + instanceRoles: make(map[string][]string), + events: make([]Event, 0), + fisFailoverFaults: make(map[string]time.Time), + proxyTargets: make(map[string][]DBProxyTarget), + automatedBackups: make(map[string]*DBInstanceAutomatedBackup), + snapshotTenantDatabases: make(map[string][]*DBSnapshotTenantDatabase), + clusterReadyAt: make(map[string]time.Time), + piMetrics: make(map[string]map[string][]PIDataPoint), + instanceLogFiles: make(map[string][]DBLogFile), + instanceLogContent: make(map[string]map[string]string), + accountID: accountID, + region: region, + defaultCACertificateID: defaultCACertificateID, + mu: lockmetrics.New("rds"), + } + registerAllTables(b) return b } @@ -868,40 +846,18 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.instances = make(map[string]*DBInstance) + // NOTE: clusterReadyAt, piMetrics, instanceLogFiles, and instanceLogContent + // were not cleared by the pre-conversion Reset either -- preserved exactly + // (not a fix target; see parity-principles.md #5). + b.registry.ResetAll() b.instanceReadyAt = make(map[string]time.Time) - b.snapshots = make(map[string]*DBSnapshot) - b.subnetGroups = make(map[string]*DBSubnetGroup) b.tags = make(map[string][]Tag) - b.parameterGroups = make(map[string]*DBParameterGroup) - b.clusterParameterGroups = make(map[string]*DBParameterGroup) - b.optionGroups = make(map[string]*OptionGroup) - b.clusters = make(map[string]*DBCluster) - b.clusterSnapshots = make(map[string]*DBClusterSnapshot) - b.clusterEndpoints = make(map[string]*DBClusterEndpoint) - b.exportTasks = make(map[string]*ExportTask) - b.globalClusters = make(map[string]*GlobalCluster) b.clusterRoles = make(map[string][]string) b.instanceRoles = make(map[string][]string) - b.eventSubscriptions = make(map[string]*EventSubscription) b.events = make([]Event, 0) - b.dbSecurityGroups = make(map[string]*DBSecurityGroup) - b.blueGreenDeployments = make(map[string]*BlueGreenDeployment) b.fisFailoverFaults = make(map[string]time.Time) - b.snapshotAttributes = make(map[string]*DBSnapshotAttributesResult) - b.clusterSnapshotAttributes = make(map[string]*DBClusterSnapshotAttributesResult) - b.reservedInstances = make(map[string]*ReservedDBInstance) - b.recommendations = make(map[string]*DBRecommendation) - b.proxies = make(map[string]*DBProxy) - b.proxyTargetGroups = make(map[string]*DBProxyTargetGroup) b.proxyTargets = make(map[string][]DBProxyTarget) - b.proxyEndpoints = make(map[string]*DBProxyEndpoint) b.automatedBackups = make(map[string]*DBInstanceAutomatedBackup) - b.customEngineVersions = make(map[string]*CustomDBEngineVersion) - b.shardGroups = make(map[string]*DBShardGroup) - b.integrations = make(map[string]*Integration) - b.tenantDatabases = make(map[string]*TenantDatabase) - b.clusterAutomatedBackups = make(map[string]*DBClusterAutomatedBackup) b.snapshotTenantDatabases = make(map[string][]*DBSnapshotTenantDatabase) } @@ -927,7 +883,7 @@ func (b *InMemoryBackend) reconcileInstancesLocked() { for id, readyAt := range b.instanceReadyAt { if !readyAt.IsZero() && now.After(readyAt) { - if inst, ok := b.instances[id]; ok { + if inst, ok := b.instances.Get(id); ok { applyPendingModifications(inst) inst.DBInstanceStatus = instanceStatusAvailable b.publishInstanceEventLocked(id, "DB instance is now available") @@ -938,7 +894,7 @@ func (b *InMemoryBackend) reconcileInstancesLocked() { for id, readyAt := range b.clusterReadyAt { if !readyAt.IsZero() && now.After(readyAt) { - if c, ok := b.clusters[id]; ok && c.Status == "rebooting" { + if c, ok := b.clusters.Get(id); ok && c.Status == "rebooting" { c.Status = instanceStatusAvailable b.publishClusterEventLocked(id, "DB cluster is now available") } @@ -1035,7 +991,7 @@ func (b *InMemoryBackend) CreateDBInstance( b.mu.Lock("CreateDBInstance") b.reconcileInstancesLocked() - if _, exists := b.instances[id]; exists { + if _, exists := b.instances.Get(id); exists { b.mu.Unlock() return nil, fmt.Errorf("%w: instance %s already exists", ErrInstanceAlreadyExists, id) @@ -1094,12 +1050,12 @@ func (b *InMemoryBackend) CreateDBInstance( OptimizedWrites: opts.OptimizedWrites, EngineLifecycleSupport: opts.EngineLifecycleSupport, } - b.instances[id] = inst + b.instances.Put(inst) b.publishInstanceEventLocked(id, "DB instance created") // If joining a cluster, add this instance to the cluster's member list. if opts.DBClusterIdentifier != "" { - if cluster, exists := b.clusters[opts.DBClusterIdentifier]; exists { + if cluster, exists := b.clusters.Get(opts.DBClusterIdentifier); exists { cluster.DBClusterMembers = append(cluster.DBClusterMembers, DBClusterMember{ DBInstanceIdentifier: id, IsClusterWriter: len(cluster.DBClusterMembers) == 0, @@ -1126,18 +1082,63 @@ func (b *InMemoryBackend) rdsARN(resourceType, id string) string { return arn.Build("rds", b.region, b.accountID, fmt.Sprintf("%s:%s", resourceType, id)) } -// DeleteDBInstance removes the DB instance with the given identifier. +// DeleteDBInstance removes the DB instance with the given identifier, skipping +// the AWS final-snapshot contract (SkipFinalSnapshot=true). It exists for +// existing callers (e.g. CloudFormation resource cleanup) that pre-date the +// SkipFinalSnapshot/FinalDBSnapshotIdentifier parameters and manage their own +// snapshot semantics. New callers that need AWS-accurate DeleteDBInstance +// behavior (final snapshot, DeleteAutomatedBackups) should use +// DeleteDBInstanceWithOptions. func (b *InMemoryBackend) DeleteDBInstance(id string) (*DBInstance, error) { + return b.DeleteDBInstanceWithOptions(id, true, "", true) +} + +// DeleteDBInstanceWithOptions removes the DB instance with the given identifier, +// honoring the AWS DeleteDBInstance parameter contract: +// - SkipFinalSnapshot=false (the AWS default) requires a non-empty +// finalSnapshotID; a manual snapshot of the instance is taken before it is +// removed. +// - SkipFinalSnapshot=true is mutually exclusive with a non-empty +// finalSnapshotID (AWS: InvalidParameterCombination either way). +// - deleteAutomatedBackups (AWS default true) controls whether the +// instance's automated backup record is removed along with the instance. +func (b *InMemoryBackend) DeleteDBInstanceWithOptions( + id string, + skipFinalSnapshot bool, + finalSnapshotID string, + deleteAutomatedBackups bool, +) (*DBInstance, error) { b.mu.Lock("DeleteDBInstance") b.reconcileInstancesLocked() - inst, exists := b.instances[id] + // AWS resolves the target instance before validating the + // SkipFinalSnapshot/FinalDBSnapshotIdentifier combination: deleting a + // nonexistent instance returns DBInstanceNotFoundFault even when the + // snapshot parameters are also missing/invalid. + inst, exists := b.instances.Get(id) if !exists { b.mu.Unlock() return nil, fmt.Errorf("%w: instance %s not found", ErrInstanceNotFound, id) } + if !skipFinalSnapshot && finalSnapshotID == "" { + b.mu.Unlock() + + return nil, fmt.Errorf( + "%w: FinalDBSnapshotIdentifier is required unless SkipFinalSnapshot is specified", + ErrInvalidParameterCombination, + ) + } + if skipFinalSnapshot && finalSnapshotID != "" { + b.mu.Unlock() + + return nil, fmt.Errorf( + "%w: the FinalDBSnapshotIdentifier parameter cannot be specified when SkipFinalSnapshot is enabled", + ErrInvalidParameterCombination, + ) + } + if inst.DeletionProtection { b.mu.Unlock() @@ -1153,6 +1154,15 @@ func (b *InMemoryBackend) DeleteDBInstance(id string) (*DBInstance, error) { return nil, fmt.Errorf("%w: instance %s is already being deleted", ErrInvalidDBInstanceState, id) } + if !skipFinalSnapshot { + if _, snapExists := b.snapshots.Get(finalSnapshotID); snapExists { + b.mu.Unlock() + + return nil, fmt.Errorf("%w: snapshot %s already exists", ErrSnapshotAlreadyExists, finalSnapshotID) + } + b.snapshots.Put(b.newManualSnapshotLocked(finalSnapshotID, inst)) + } + inst.DBInstanceStatus = instanceStatusDeleting b.publishInstanceEventLocked(id, "DB instance deletion started") cp := *inst @@ -1160,18 +1170,20 @@ func (b *InMemoryBackend) DeleteDBInstance(id string) (*DBInstance, error) { // Remove this instance from its source's ReadReplicaIdentifiers. if inst.ReplicaSourceDBInstanceIdentifier != "" { - if src, srcExists := b.instances[inst.ReplicaSourceDBInstanceIdentifier]; srcExists { + if src, srcExists := b.instances.Get(inst.ReplicaSourceDBInstanceIdentifier); srcExists { src.ReadReplicaIdentifiers = slices.DeleteFunc(src.ReadReplicaIdentifiers, func(s string) bool { return s == id }) } } - delete(b.instances, id) + b.instances.Delete(id) delete(b.tags, b.rdsARN("db", id)) delete(b.instanceRoles, id) delete(b.instanceReadyAt, id) - delete(b.automatedBackups, id) + if deleteAutomatedBackups { + delete(b.automatedBackups, id) + } b.mu.Unlock() @@ -1187,7 +1199,7 @@ func (b *InMemoryBackend) DescribeDBInstances(id string) ([]DBInstance, error) { b.mu.RLock("DescribeDBInstances") if id != "" { - inst, exists := b.instances[id] + inst, exists := b.instances.Get(id) if !exists { b.mu.RUnlock() @@ -1199,8 +1211,8 @@ func (b *InMemoryBackend) DescribeDBInstances(id string) ([]DBInstance, error) { return []DBInstance{cp}, nil } - instances := make([]DBInstance, 0, len(b.instances)) - for _, inst := range b.instances { + instances := make([]DBInstance, 0, b.instances.Len()) + for _, inst := range b.instances.All() { instances = append(instances, *inst) } b.mu.RUnlock() @@ -1228,7 +1240,7 @@ func (b *InMemoryBackend) applyParamGroupUpdate(inst *DBInstance, paramGroupName return nil } - if _, pgExists := b.parameterGroups[paramGroupName]; !pgExists { + if _, pgExists := b.parameterGroups.Get(paramGroupName); !pgExists { return fmt.Errorf("%w: parameter group %s not found", ErrParameterGroupNotFound, paramGroupName) } @@ -1450,7 +1462,7 @@ func (b *InMemoryBackend) ModifyDBInstance( b.reconcileInstancesLocked() defer b.mu.Unlock() - inst, exists := b.instances[id] + inst, exists := b.instances.Get(id) if !exists { return nil, fmt.Errorf("%w: instance %s not found", ErrInstanceNotFound, id) } @@ -1482,19 +1494,31 @@ func (b *InMemoryBackend) CreateDBSnapshot(snapshotID, instanceID string) (*DBSn b.mu.Lock("CreateDBSnapshot") defer b.mu.Unlock() - if _, exists := b.snapshots[snapshotID]; exists { + if _, exists := b.snapshots.Get(snapshotID); exists { return nil, fmt.Errorf("%w: snapshot %s already exists", ErrSnapshotAlreadyExists, snapshotID) } - inst, exists := b.instances[instanceID] + inst, exists := b.instances.Get(instanceID) if !exists { return nil, fmt.Errorf("%w: instance %s not found", ErrInstanceNotFound, instanceID) } + snap := b.newManualSnapshotLocked(snapshotID, inst) + b.snapshots.Put(snap) + + cp := *snap + + return &cp, nil +} + +// newManualSnapshotLocked builds a manual DB snapshot record for inst. It does +// not check for an existing snapshot with the same ID or insert into +// b.snapshots — callers must do both under b.mu. +func (b *InMemoryBackend) newManualSnapshotLocked(snapshotID string, inst *DBInstance) *DBSnapshot { snap := &DBSnapshot{ SnapshotCreateTime: time.Now().UTC(), DBSnapshotIdentifier: snapshotID, - DBInstanceIdentifier: instanceID, + DBInstanceIdentifier: inst.DBInstanceIdentifier, Engine: inst.Engine, EngineVersion: inst.EngineVersion, Status: instanceStatusAvailable, @@ -1509,11 +1533,8 @@ func (b *InMemoryBackend) CreateDBSnapshot(snapshotID, instanceID string) (*DBSn if inst.StorageEncrypted { snap.KmsKeyID = inst.KmsKeyID } - b.snapshots[snapshotID] = snap - - cp := *snap - return &cp, nil + return snap } // DescribeDBSnapshots returns snapshots. If snapshotID is non-empty, returns only that snapshot. @@ -1523,7 +1544,7 @@ func (b *InMemoryBackend) DescribeDBSnapshots(snapshotID, instanceID string) ([] defer b.mu.RUnlock() if snapshotID != "" { - snap, exists := b.snapshots[snapshotID] + snap, exists := b.snapshots.Get(snapshotID) if !exists { return nil, fmt.Errorf("%w: snapshot %s not found", ErrSnapshotNotFound, snapshotID) } @@ -1531,8 +1552,8 @@ func (b *InMemoryBackend) DescribeDBSnapshots(snapshotID, instanceID string) ([] return []DBSnapshot{*snap}, nil } - snaps := make([]DBSnapshot, 0, len(b.snapshots)) - for _, snap := range b.snapshots { + snaps := make([]DBSnapshot, 0, b.snapshots.Len()) + for _, snap := range b.snapshots.All() { if instanceID != "" && snap.DBInstanceIdentifier != instanceID { continue } @@ -1557,13 +1578,13 @@ func (b *InMemoryBackend) DeleteDBSnapshot(snapshotID string) (*DBSnapshot, erro b.mu.Lock("DeleteDBSnapshot") defer b.mu.Unlock() - snap, exists := b.snapshots[snapshotID] + snap, exists := b.snapshots.Get(snapshotID) if !exists { return nil, fmt.Errorf("%w: snapshot %s not found", ErrSnapshotNotFound, snapshotID) } cp := *snap - delete(b.snapshots, snapshotID) + b.snapshots.Delete(snapshotID) delete(b.tags, b.rdsARN("snapshot", snapshotID)) return &cp, nil @@ -1584,11 +1605,11 @@ func (b *InMemoryBackend) CopyDBSnapshot( b.mu.Lock("CopyDBSnapshot") defer b.mu.Unlock() - src, exists := b.snapshots[sourceSnapshotID] + src, exists := b.snapshots.Get(sourceSnapshotID) if !exists { return nil, fmt.Errorf("%w: snapshot %s not found", ErrSnapshotNotFound, sourceSnapshotID) } - if _, alreadyExists := b.snapshots[targetSnapshotID]; alreadyExists { + if _, alreadyExists := b.snapshots.Get(targetSnapshotID); alreadyExists { return nil, fmt.Errorf("%w: snapshot %s already exists", ErrSnapshotAlreadyExists, targetSnapshotID) } @@ -1613,7 +1634,7 @@ func (b *InMemoryBackend) CopyDBSnapshot( OptionGroupName: src.OptionGroupName, PercentProgress: percentProgressComplete, } - b.snapshots[targetSnapshotID] = snap + b.snapshots.Put(snap) cp := *snap return &cp, nil @@ -1634,13 +1655,13 @@ func (b *InMemoryBackend) RestoreDBInstanceFromDBSnapshot( b.mu.Lock("RestoreDBInstanceFromDBSnapshot") b.reconcileInstancesLocked() - if _, exists := b.instances[id]; exists { + if _, exists := b.instances.Get(id); exists { b.mu.Unlock() return nil, fmt.Errorf("%w: instance %s already exists", ErrInstanceAlreadyExists, id) } - snap, exists := b.snapshots[snapshotID] + snap, exists := b.snapshots.Get(snapshotID) if !exists { b.mu.Unlock() @@ -1675,7 +1696,7 @@ func (b *InMemoryBackend) RestoreDBInstanceFromDBSnapshot( MultiAZ: opts.MultiAZ, DeletionProtection: opts.DeletionProtection, } - b.instances[id] = inst + b.instances.Put(inst) b.publishInstanceEventLocked(id, "DB instance restored from snapshot") cp := *inst @@ -1703,13 +1724,13 @@ func (b *InMemoryBackend) RestoreDBInstanceToPointInTime( b.mu.Lock("RestoreDBInstanceToPointInTime") b.reconcileInstancesLocked() - if _, exists := b.instances[id]; exists { + if _, exists := b.instances.Get(id); exists { b.mu.Unlock() return nil, fmt.Errorf("%w: instance %s already exists", ErrInstanceAlreadyExists, id) } - source, exists := b.instances[sourceID] + source, exists := b.instances.Get(sourceID) if !exists { b.mu.Unlock() @@ -1743,7 +1764,7 @@ func (b *InMemoryBackend) RestoreDBInstanceToPointInTime( MultiAZ: opts.MultiAZ, DeletionProtection: opts.DeletionProtection, } - b.instances[id] = inst + b.instances.Put(inst) b.publishInstanceEventLocked(id, "DB instance restored to point in time") cp := *inst @@ -1766,7 +1787,7 @@ func (b *InMemoryBackend) StartDBInstance(id string) (*DBInstance, error) { b.reconcileInstancesLocked() defer b.mu.Unlock() - inst, exists := b.instances[id] + inst, exists := b.instances.Get(id) if !exists { return nil, fmt.Errorf("%w: instance %s not found", ErrInstanceNotFound, id) } @@ -1790,7 +1811,7 @@ func (b *InMemoryBackend) StopDBInstance(id string) (*DBInstance, error) { b.reconcileInstancesLocked() defer b.mu.Unlock() - inst, exists := b.instances[id] + inst, exists := b.instances.Get(id) if !exists { return nil, fmt.Errorf("%w: instance %s not found", ErrInstanceNotFound, id) } @@ -1816,7 +1837,7 @@ func (b *InMemoryBackend) CreateDBSubnetGroup( b.mu.Lock("CreateDBSubnetGroup") defer b.mu.Unlock() - if _, exists := b.subnetGroups[name]; exists { + if _, exists := b.subnetGroups.Get(name); exists { return nil, fmt.Errorf("%w: subnet group %s already exists", ErrSubnetGroupAlreadyExists, name) } @@ -1830,7 +1851,7 @@ func (b *InMemoryBackend) CreateDBSubnetGroup( SubnetIDs: ids, Status: "Complete", } - b.subnetGroups[name] = sg + b.subnetGroups.Put(sg) cp := *sg cp.SubnetIDs = make([]string, len(ids)) @@ -1845,7 +1866,7 @@ func (b *InMemoryBackend) DescribeDBSubnetGroups(name string) ([]DBSubnetGroup, defer b.mu.RUnlock() if name != "" { - sg, exists := b.subnetGroups[name] + sg, exists := b.subnetGroups.Get(name) if !exists { return nil, fmt.Errorf("%w: subnet group %s not found", ErrSubnetGroupNotFound, name) } @@ -1857,9 +1878,9 @@ func (b *InMemoryBackend) DescribeDBSubnetGroups(name string) ([]DBSubnetGroup, return []DBSubnetGroup{cp}, nil } - sgs := make([]DBSubnetGroup, 0, len(b.subnetGroups)) + sgs := make([]DBSubnetGroup, 0, b.subnetGroups.Len()) - for _, sg := range b.subnetGroups { + for _, sg := range b.subnetGroups.All() { cp := *sg cp.SubnetIDs = make([]string, len(sg.SubnetIDs)) copy(cp.SubnetIDs, sg.SubnetIDs) @@ -1874,11 +1895,11 @@ func (b *InMemoryBackend) DeleteDBSubnetGroup(name string) error { b.mu.Lock("DeleteDBSubnetGroup") defer b.mu.Unlock() - if _, exists := b.subnetGroups[name]; !exists { + if _, exists := b.subnetGroups.Get(name); !exists { return fmt.Errorf("%w: subnet group %s not found", ErrSubnetGroupNotFound, name) } - delete(b.subnetGroups, name) + b.subnetGroups.Delete(name) delete(b.tags, b.rdsARN("subgrp", name)) return nil @@ -1949,7 +1970,7 @@ func (b *InMemoryBackend) CreateDBParameterGroup(name, family, description strin } b.mu.Lock("CreateDBParameterGroup") defer b.mu.Unlock() - if _, exists := b.parameterGroups[name]; exists { + if _, exists := b.parameterGroups.Get(name); exists { return nil, fmt.Errorf("%w: parameter group %s already exists", ErrParameterGroupAlreadyExists, name) } pg := &DBParameterGroup{ @@ -1958,7 +1979,7 @@ func (b *InMemoryBackend) CreateDBParameterGroup(name, family, description strin Description: description, Parameters: make(map[string]DBParameter), } - b.parameterGroups[name] = pg + b.parameterGroups.Put(pg) cp := *pg cp.Parameters = make(map[string]DBParameter) @@ -1979,15 +2000,15 @@ func (b *InMemoryBackend) DescribeDBParameterGroups(name string) ([]DBParameterG b.mu.RLock("DescribeDBParameterGroups") defer b.mu.RUnlock() if name != "" { - pg, exists := b.parameterGroups[name] + pg, exists := b.parameterGroups.Get(name) if !exists { return nil, fmt.Errorf("%w: parameter group %s not found", ErrParameterGroupNotFound, name) } return []DBParameterGroup{copyDBParameterGroup(pg)}, nil } - result := make([]DBParameterGroup, 0, len(b.parameterGroups)) - for _, pg := range b.parameterGroups { + result := make([]DBParameterGroup, 0, b.parameterGroups.Len()) + for _, pg := range b.parameterGroups.All() { result = append(result, copyDBParameterGroup(pg)) } slices.SortFunc(result, func(a, b DBParameterGroup) int { @@ -2008,10 +2029,10 @@ func (b *InMemoryBackend) DescribeDBParameterGroups(name string) ([]DBParameterG func (b *InMemoryBackend) DeleteDBParameterGroup(name string) error { b.mu.Lock("DeleteDBParameterGroup") defer b.mu.Unlock() - if _, exists := b.parameterGroups[name]; !exists { + if _, exists := b.parameterGroups.Get(name); !exists { return fmt.Errorf("%w: parameter group %s not found", ErrParameterGroupNotFound, name) } - delete(b.parameterGroups, name) + b.parameterGroups.Delete(name) delete(b.tags, b.rdsARN("pg", name)) return nil @@ -2021,7 +2042,7 @@ func (b *InMemoryBackend) DeleteDBParameterGroup(name string) error { func (b *InMemoryBackend) ModifyDBParameterGroup(name string, params []DBParameter) (*DBParameterGroup, error) { b.mu.Lock("ModifyDBParameterGroup") defer b.mu.Unlock() - pg, exists := b.parameterGroups[name] + pg, exists := b.parameterGroups.Get(name) if !exists { return nil, fmt.Errorf("%w: parameter group %s not found", ErrParameterGroupNotFound, name) } @@ -2043,7 +2064,7 @@ func (b *InMemoryBackend) ModifyDBParameterGroup(name string, params []DBParamet func (b *InMemoryBackend) DescribeDBParameters(groupName string) ([]DBParameter, error) { b.mu.RLock("DescribeDBParameters") defer b.mu.RUnlock() - pg, exists := b.parameterGroups[groupName] + pg, exists := b.parameterGroups.Get(groupName) if !exists { return nil, fmt.Errorf("%w: parameter group %s not found", ErrParameterGroupNotFound, groupName) } @@ -2073,7 +2094,7 @@ func (b *InMemoryBackend) ResetDBParameterGroup( ) (*DBParameterGroup, error) { b.mu.Lock("ResetDBParameterGroup") defer b.mu.Unlock() - pg, exists := b.parameterGroups[name] + pg, exists := b.parameterGroups.Get(name) if !exists { return nil, fmt.Errorf("%w: parameter group %s not found", ErrParameterGroupNotFound, name) } @@ -2102,7 +2123,7 @@ func (b *InMemoryBackend) CreateOptionGroup(name, engine, majorVersion, descript } b.mu.Lock("CreateOptionGroup") defer b.mu.Unlock() - if _, exists := b.optionGroups[name]; exists { + if _, exists := b.optionGroups.Get(name); exists { return nil, fmt.Errorf("%w: option group %s already exists", ErrOptionGroupAlreadyExists, name) } og := &OptionGroup{ @@ -2112,7 +2133,7 @@ func (b *InMemoryBackend) CreateOptionGroup(name, engine, majorVersion, descript MajorEngineVersion: majorVersion, Options: []OptionGroupOption{}, } - b.optionGroups[name] = og + b.optionGroups.Put(og) cp := *og cp.Options = make([]OptionGroupOption, len(og.Options)) copy(cp.Options, og.Options) @@ -2125,7 +2146,7 @@ func (b *InMemoryBackend) DescribeOptionGroups(name string) ([]OptionGroup, erro b.mu.RLock("DescribeOptionGroups") defer b.mu.RUnlock() if name != "" { - og, exists := b.optionGroups[name] + og, exists := b.optionGroups.Get(name) if !exists { return nil, fmt.Errorf("%w: option group %s not found", ErrOptionGroupNotFound, name) } @@ -2135,8 +2156,8 @@ func (b *InMemoryBackend) DescribeOptionGroups(name string) ([]OptionGroup, erro return []OptionGroup{cp}, nil } - result := make([]OptionGroup, 0, len(b.optionGroups)) - for _, og := range b.optionGroups { + result := make([]OptionGroup, 0, b.optionGroups.Len()) + for _, og := range b.optionGroups.All() { cp := *og cp.Options = make([]OptionGroupOption, len(og.Options)) copy(cp.Options, og.Options) @@ -2160,10 +2181,10 @@ func (b *InMemoryBackend) DescribeOptionGroups(name string) ([]OptionGroup, erro func (b *InMemoryBackend) DeleteOptionGroup(name string) error { b.mu.Lock("DeleteOptionGroup") defer b.mu.Unlock() - if _, exists := b.optionGroups[name]; !exists { + if _, exists := b.optionGroups.Get(name); !exists { return fmt.Errorf("%w: option group %s not found", ErrOptionGroupNotFound, name) } - delete(b.optionGroups, name) + b.optionGroups.Delete(name) delete(b.tags, b.rdsARN("og", name)) return nil @@ -2177,7 +2198,7 @@ func (b *InMemoryBackend) ModifyOptionGroup( ) (*OptionGroup, error) { b.mu.Lock("ModifyOptionGroup") defer b.mu.Unlock() - og, exists := b.optionGroups[name] + og, exists := b.optionGroups.Get(name) if !exists { return nil, fmt.Errorf("%w: option group %s not found", ErrOptionGroupNotFound, name) } @@ -2212,7 +2233,7 @@ func (b *InMemoryBackend) CreateDBCluster( } b.mu.Lock("CreateDBCluster") defer b.mu.Unlock() - if _, exists := b.clusters[id]; exists { + if _, exists := b.clusters.Get(id); exists { return nil, fmt.Errorf("%w: cluster %s already exists", ErrClusterAlreadyExists, id) } if engine == "" { @@ -2262,7 +2283,7 @@ func (b *InMemoryBackend) CreateDBCluster( DeletionProtection: opts.DeletionProtection, DBClusterMembers: []DBClusterMember{}, } - b.clusters[id] = cluster + b.clusters.Put(cluster) cp := *cluster return &cp, nil @@ -2273,7 +2294,7 @@ func (b *InMemoryBackend) DescribeDBClusters(id string) ([]DBCluster, error) { b.mu.RLock("DescribeDBClusters") defer b.mu.RUnlock() if id != "" { - cluster, exists := b.clusters[id] + cluster, exists := b.clusters.Get(id) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } @@ -2281,8 +2302,8 @@ func (b *InMemoryBackend) DescribeDBClusters(id string) ([]DBCluster, error) { return []DBCluster{cp}, nil } - result := make([]DBCluster, 0, len(b.clusters)) - for _, cluster := range b.clusters { + result := make([]DBCluster, 0, b.clusters.Len()) + for _, cluster := range b.clusters.All() { result = append(result, *cluster) } slices.SortFunc(result, func(a, b DBCluster) int { @@ -2300,21 +2321,69 @@ func (b *InMemoryBackend) DescribeDBClusters(id string) ([]DBCluster, error) { } // DeleteDBCluster removes the given cluster. +// DeleteDBCluster removes the DB cluster with the given identifier, skipping +// the AWS final-snapshot contract (SkipFinalSnapshot=true). It exists for +// existing callers (e.g. CloudFormation resource cleanup) that pre-date the +// SkipFinalSnapshot/FinalDBSnapshotIdentifier parameters. New callers that +// need AWS-accurate DeleteDBCluster behavior should use +// DeleteDBClusterWithOptions. func (b *InMemoryBackend) DeleteDBCluster(id string) (*DBCluster, error) { + return b.DeleteDBClusterWithOptions(id, true, "") +} + +// DeleteDBClusterWithOptions removes the DB cluster with the given identifier, +// honoring the AWS DeleteDBCluster parameter contract: +// - SkipFinalSnapshot=false (the AWS default) requires a non-empty +// finalSnapshotID; a manual cluster snapshot is taken before the cluster +// is removed. +// - SkipFinalSnapshot=true is mutually exclusive with a non-empty +// finalSnapshotID (AWS: InvalidParameterCombination either way). +func (b *InMemoryBackend) DeleteDBClusterWithOptions( + id string, skipFinalSnapshot bool, finalSnapshotID string, +) (*DBCluster, error) { b.mu.Lock("DeleteDBCluster") defer b.mu.Unlock() - cluster, exists := b.clusters[id] + + // Resolve the target cluster before validating the snapshot parameter + // combination, matching AWS's behavior of returning DBClusterNotFoundFault + // for a nonexistent cluster even when the snapshot params are also invalid. + cluster, exists := b.clusters.Get(id) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } + + if !skipFinalSnapshot && finalSnapshotID == "" { + return nil, fmt.Errorf( + "%w: FinalDBSnapshotIdentifier is required unless SkipFinalSnapshot is specified", + ErrInvalidParameterCombination, + ) + } + if skipFinalSnapshot && finalSnapshotID != "" { + return nil, fmt.Errorf( + "%w: the FinalDBSnapshotIdentifier parameter cannot be specified when SkipFinalSnapshot is enabled", + ErrInvalidParameterCombination, + ) + } + + if !skipFinalSnapshot { + if _, snapExists := b.clusterSnapshots.Get(finalSnapshotID); snapExists { + return nil, fmt.Errorf( + "%w: cluster snapshot %s already exists", + ErrClusterSnapshotAlreadyExists, + finalSnapshotID, + ) + } + b.clusterSnapshots.Put(b.newManualClusterSnapshotLocked(finalSnapshotID, cluster)) + } + cp := *cluster // Clear the cluster association on any member instances so they appear standalone. for _, member := range cluster.DBClusterMembers { - if inst, ok := b.instances[member.DBInstanceIdentifier]; ok { + if inst, ok := b.instances.Get(member.DBInstanceIdentifier); ok { inst.DBClusterIdentifier = "" } } - delete(b.clusters, id) + b.clusters.Delete(id) delete(b.tags, b.rdsARN("cluster", id)) delete(b.fisFailoverFaults, id) delete(b.clusterRoles, id) @@ -2393,7 +2462,7 @@ func applyDBClusterBoolOpts(cluster *DBCluster, opts DBClusterOptions) { func (b *InMemoryBackend) ModifyDBCluster(id, paramGroupName string, opts DBClusterOptions) (*DBCluster, error) { b.mu.Lock("ModifyDBCluster") defer b.mu.Unlock() - cluster, exists := b.clusters[id] + cluster, exists := b.clusters.Get(id) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } @@ -2410,7 +2479,7 @@ func (b *InMemoryBackend) CreateDBClusterParameterGroup(name, family, descriptio } b.mu.Lock("CreateDBClusterParameterGroup") defer b.mu.Unlock() - if _, exists := b.clusterParameterGroups[name]; exists { + if _, exists := b.clusterParameterGroups.Get(name); exists { return nil, fmt.Errorf("%w: cluster parameter group %s already exists", ErrParameterGroupAlreadyExists, name) } pg := &DBParameterGroup{ @@ -2419,7 +2488,7 @@ func (b *InMemoryBackend) CreateDBClusterParameterGroup(name, family, descriptio Description: description, Parameters: make(map[string]DBParameter), } - b.clusterParameterGroups[name] = pg + b.clusterParameterGroups.Put(pg) cp := *pg cp.Parameters = make(map[string]DBParameter) @@ -2431,15 +2500,15 @@ func (b *InMemoryBackend) DescribeDBClusterParameterGroups(name string) ([]DBPar b.mu.RLock("DescribeDBClusterParameterGroups") defer b.mu.RUnlock() if name != "" { - pg, exists := b.clusterParameterGroups[name] + pg, exists := b.clusterParameterGroups.Get(name) if !exists { return nil, fmt.Errorf("%w: cluster parameter group %s not found", ErrParameterGroupNotFound, name) } return []DBParameterGroup{copyDBParameterGroup(pg)}, nil } - result := make([]DBParameterGroup, 0, len(b.clusterParameterGroups)) - for _, pg := range b.clusterParameterGroups { + result := make([]DBParameterGroup, 0, b.clusterParameterGroups.Len()) + for _, pg := range b.clusterParameterGroups.All() { result = append(result, copyDBParameterGroup(pg)) } slices.SortFunc(result, func(a, b DBParameterGroup) int { @@ -2466,27 +2535,34 @@ func (b *InMemoryBackend) CreateDBClusterSnapshot(snapshotID, clusterID string) } b.mu.Lock("CreateDBClusterSnapshot") defer b.mu.Unlock() - if _, exists := b.clusterSnapshots[snapshotID]; exists { + if _, exists := b.clusterSnapshots.Get(snapshotID); exists { return nil, fmt.Errorf("%w: cluster snapshot %s already exists", ErrClusterSnapshotAlreadyExists, snapshotID) } - cluster, exists := b.clusters[clusterID] + cluster, exists := b.clusters.Get(clusterID) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } - snap := &DBClusterSnapshot{ + snap := b.newManualClusterSnapshotLocked(snapshotID, cluster) + b.clusterSnapshots.Put(snap) + cp := *snap + + return &cp, nil +} + +// newManualClusterSnapshotLocked builds a manual DB cluster snapshot record +// for cluster. It does not check for an existing snapshot with the same ID or +// insert into b.clusterSnapshots — callers must do both under b.mu. +func (b *InMemoryBackend) newManualClusterSnapshotLocked(snapshotID string, cluster *DBCluster) *DBClusterSnapshot { + return &DBClusterSnapshot{ SnapshotCreateTime: time.Now().UTC(), DBClusterSnapshotIdentifier: snapshotID, - DBClusterIdentifier: clusterID, + DBClusterIdentifier: cluster.DBClusterIdentifier, Engine: cluster.Engine, EngineVersion: cluster.EngineVersion, Status: instanceStatusAvailable, PercentProgress: percentProgressComplete, StorageEncrypted: cluster.StorageEncrypted, } - b.clusterSnapshots[snapshotID] = snap - cp := *snap - - return &cp, nil } // DescribeDBClusterSnapshots returns cluster snapshots. @@ -2495,7 +2571,7 @@ func (b *InMemoryBackend) DescribeDBClusterSnapshots(snapshotID, clusterID strin b.mu.RLock("DescribeDBClusterSnapshots") defer b.mu.RUnlock() if snapshotID != "" { - snap, exists := b.clusterSnapshots[snapshotID] + snap, exists := b.clusterSnapshots.Get(snapshotID) if !exists { return nil, fmt.Errorf("%w: cluster snapshot %s not found", ErrClusterSnapshotNotFound, snapshotID) } @@ -2503,8 +2579,8 @@ func (b *InMemoryBackend) DescribeDBClusterSnapshots(snapshotID, clusterID strin return []DBClusterSnapshot{cp}, nil } - result := make([]DBClusterSnapshot, 0, len(b.clusterSnapshots)) - for _, snap := range b.clusterSnapshots { + result := make([]DBClusterSnapshot, 0, b.clusterSnapshots.Len()) + for _, snap := range b.clusterSnapshots.All() { if clusterID != "" && snap.DBClusterIdentifier != clusterID { continue } @@ -2533,7 +2609,7 @@ func (b *InMemoryBackend) CreateDBInstanceReadReplica(id, sourceID, sourceRegion b.mu.Lock("CreateDBInstanceReadReplica") b.reconcileInstancesLocked() defer b.mu.Unlock() - if _, exists := b.instances[id]; exists { + if _, exists := b.instances.Get(id); exists { return nil, fmt.Errorf("%w: instance %s already exists", ErrInstanceAlreadyExists, id) } @@ -2546,7 +2622,7 @@ func (b *InMemoryBackend) CreateDBInstanceReadReplica(id, sourceID, sourceRegion allocatedStorage int ) - source, sourceExists := b.instances[sourceID] + source, sourceExists := b.instances.Get(sourceID) switch { case sourceExists: instanceClass = source.DBInstanceClass @@ -2579,7 +2655,7 @@ func (b *InMemoryBackend) CreateDBInstanceReadReplica(id, sourceID, sourceRegion AllocatedStorage: allocatedStorage, ReplicaSourceDBInstanceIdentifier: sourceID, } - b.instances[id] = replica + b.instances.Put(replica) b.publishInstanceEventLocked(id, "DB read replica created") // Track reverse read replica reference on source instance. @@ -2618,14 +2694,14 @@ func (b *InMemoryBackend) PromoteReadReplica(id string) (*DBInstance, error) { b.mu.Lock("PromoteReadReplica") b.reconcileInstancesLocked() defer b.mu.Unlock() - inst, exists := b.instances[id] + inst, exists := b.instances.Get(id) if !exists { return nil, fmt.Errorf("%w: instance %s not found", ErrInstanceNotFound, id) } // Remove promoted instance from source's ReadReplicaIdentifiers. if inst.ReplicaSourceDBInstanceIdentifier != "" { - if src, srcExists := b.instances[inst.ReplicaSourceDBInstanceIdentifier]; srcExists { + if src, srcExists := b.instances.Get(inst.ReplicaSourceDBInstanceIdentifier); srcExists { src.ReadReplicaIdentifiers = slices.DeleteFunc(src.ReadReplicaIdentifiers, func(s string) bool { return s == id }) @@ -2644,7 +2720,7 @@ func (b *InMemoryBackend) RebootDBInstance(id string) (*DBInstance, error) { b.mu.Lock("RebootDBInstance") b.reconcileInstancesLocked() defer b.mu.Unlock() - inst, exists := b.instances[id] + inst, exists := b.instances.Get(id) if !exists { return nil, fmt.Errorf("%w: instance %s not found", ErrInstanceNotFound, id) } @@ -2672,7 +2748,7 @@ func (b *InMemoryBackend) CreateCustomDBEngineVersion( b.mu.Lock("CreateCustomDBEngineVersion") defer b.mu.Unlock() - if _, exists := b.customEngineVersions[key]; exists { + if _, exists := b.customEngineVersions.Get(key); exists { return nil, fmt.Errorf( "%w: custom engine version %s/%s already exists", ErrInstanceAlreadyExists, @@ -2687,7 +2763,7 @@ func (b *InMemoryBackend) CreateCustomDBEngineVersion( Status: instanceStatusAvailable, Description: description, } - b.customEngineVersions[key] = cev + b.customEngineVersions.Put(cev) cp := *cev return &cp, nil @@ -2699,14 +2775,14 @@ func (b *InMemoryBackend) DeleteCustomDBEngineVersion(engine, engineVersion stri b.mu.Lock("DeleteCustomDBEngineVersion") defer b.mu.Unlock() - cev, exists := b.customEngineVersions[key] + cev, exists := b.customEngineVersions.Get(key) if !exists { return nil, fmt.Errorf("%w: custom engine version %s/%s not found", ErrInstanceNotFound, engine, engineVersion) } cp := *cev cp.Status = instanceStatusDeleting - delete(b.customEngineVersions, key) + b.customEngineVersions.Delete(key) return &cp, nil } @@ -2719,7 +2795,7 @@ func (b *InMemoryBackend) ModifyCustomDBEngineVersion( b.mu.Lock("ModifyCustomDBEngineVersion") defer b.mu.Unlock() - cev, exists := b.customEngineVersions[key] + cev, exists := b.customEngineVersions.Get(key) if !exists { return nil, fmt.Errorf("%w: custom engine version %s/%s not found", ErrInstanceNotFound, engine, engineVersion) } @@ -2813,7 +2889,7 @@ type LogFilePortion struct { func (b *InMemoryBackend) DescribeDBLogFiles(instanceID string, filter LogFileFilter) ([]DBLogFile, error) { b.mu.Lock("DescribeDBLogFiles") defer b.mu.Unlock() - inst, exists := b.instances[instanceID] + inst, exists := b.instances.Get(instanceID) if !exists { return nil, fmt.Errorf("%w: instance %s not found", ErrInstanceNotFound, instanceID) } @@ -2845,7 +2921,7 @@ func (b *InMemoryBackend) DownloadDBLogFilePortion( ) (LogFilePortion, error) { b.mu.Lock("DownloadDBLogFilePortion") defer b.mu.Unlock() - inst, exists := b.instances[instanceID] + inst, exists := b.instances.Get(instanceID) if !exists { return LogFilePortion{}, fmt.Errorf("%w: instance %s not found", ErrInstanceNotFound, instanceID) } @@ -2950,7 +3026,7 @@ func (b *InMemoryBackend) StartDBCluster(id string) (*DBCluster, error) { } b.mu.Lock("StartDBCluster") defer b.mu.Unlock() - cluster, exists := b.clusters[id] + cluster, exists := b.clusters.Get(id) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } @@ -2967,7 +3043,7 @@ func (b *InMemoryBackend) StopDBCluster(id string) (*DBCluster, error) { } b.mu.Lock("StopDBCluster") defer b.mu.Unlock() - cluster, exists := b.clusters[id] + cluster, exists := b.clusters.Get(id) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } @@ -2984,12 +3060,12 @@ func (b *InMemoryBackend) DeleteDBClusterSnapshot(snapshotID string) (*DBCluster } b.mu.Lock("DeleteDBClusterSnapshot") defer b.mu.Unlock() - snap, exists := b.clusterSnapshots[snapshotID] + snap, exists := b.clusterSnapshots.Get(snapshotID) if !exists { return nil, fmt.Errorf("%w: cluster snapshot %s not found", ErrClusterSnapshotNotFound, snapshotID) } cp := *snap - delete(b.clusterSnapshots, snapshotID) + b.clusterSnapshots.Delete(snapshotID) delete(b.tags, b.rdsARN("cluster-snapshot", snapshotID)) return &cp, nil @@ -3005,10 +3081,10 @@ func (b *InMemoryBackend) RestoreDBClusterFromSnapshot(clusterID, snapshotID, en } b.mu.Lock("RestoreDBClusterFromSnapshot") defer b.mu.Unlock() - if _, exists := b.clusters[clusterID]; exists { + if _, exists := b.clusters.Get(clusterID); exists { return nil, fmt.Errorf("%w: cluster %s already exists", ErrClusterAlreadyExists, clusterID) } - snap, exists := b.clusterSnapshots[snapshotID] + snap, exists := b.clusterSnapshots.Get(snapshotID) if !exists { return nil, fmt.Errorf("%w: cluster snapshot %s not found", ErrClusterSnapshotNotFound, snapshotID) } @@ -3024,7 +3100,7 @@ func (b *InMemoryBackend) RestoreDBClusterFromSnapshot(clusterID, snapshotID, en Endpoint: endpoint, Port: enginePort(engine), } - b.clusters[clusterID] = cluster + b.clusters.Put(cluster) cp := *cluster return &cp, nil @@ -3040,10 +3116,10 @@ func (b *InMemoryBackend) RestoreDBClusterToPointInTime(clusterID, sourceCluster } b.mu.Lock("RestoreDBClusterToPointInTime") defer b.mu.Unlock() - if _, exists := b.clusters[clusterID]; exists { + if _, exists := b.clusters.Get(clusterID); exists { return nil, fmt.Errorf("%w: cluster %s already exists", ErrClusterAlreadyExists, clusterID) } - source, exists := b.clusters[sourceClusterID] + source, exists := b.clusters.Get(sourceClusterID) if !exists { return nil, fmt.Errorf("%w: source cluster %s not found", ErrClusterNotFound, sourceClusterID) } @@ -3058,7 +3134,7 @@ func (b *InMemoryBackend) RestoreDBClusterToPointInTime(clusterID, sourceCluster Endpoint: endpoint, Port: source.Port, } - b.clusters[clusterID] = cluster + b.clusters.Put(cluster) cp := *cluster return &cp, nil @@ -3074,11 +3150,11 @@ func (b *InMemoryBackend) CopyDBClusterSnapshot(sourceSnapshotID, targetSnapshot } b.mu.Lock("CopyDBClusterSnapshot") defer b.mu.Unlock() - source, srcExists := b.clusterSnapshots[sourceSnapshotID] + source, srcExists := b.clusterSnapshots.Get(sourceSnapshotID) if !srcExists { return nil, fmt.Errorf("%w: cluster snapshot %s not found", ErrClusterSnapshotNotFound, sourceSnapshotID) } - if _, dstExists := b.clusterSnapshots[targetSnapshotID]; dstExists { + if _, dstExists := b.clusterSnapshots.Get(targetSnapshotID); dstExists { return nil, fmt.Errorf( "%w: cluster snapshot %s already exists", ErrClusterSnapshotAlreadyExists, @@ -3091,7 +3167,7 @@ func (b *InMemoryBackend) CopyDBClusterSnapshot(sourceSnapshotID, targetSnapshot Engine: source.Engine, Status: instanceStatusAvailable, } - b.clusterSnapshots[targetSnapshotID] = snap + b.clusterSnapshots.Put(snap) cp := *snap return &cp, nil @@ -3109,10 +3185,10 @@ func (b *InMemoryBackend) CreateDBClusterEndpoint( } b.mu.Lock("CreateDBClusterEndpoint") defer b.mu.Unlock() - if _, exists := b.clusterEndpoints[endpointID]; exists { + if _, exists := b.clusterEndpoints.Get(endpointID); exists { return nil, fmt.Errorf("%w: cluster endpoint %s already exists", ErrClusterEndpointAlreadyExists, endpointID) } - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } if endpointType == "" { @@ -3130,7 +3206,7 @@ func (b *InMemoryBackend) CreateDBClusterEndpoint( b.region, ), } - b.clusterEndpoints[endpointID] = ep + b.clusterEndpoints.Put(ep) cp := *ep return &cp, nil @@ -3141,7 +3217,7 @@ func (b *InMemoryBackend) DescribeDBClusterEndpoints(clusterID, endpointID strin b.mu.RLock("DescribeDBClusterEndpoints") defer b.mu.RUnlock() if endpointID != "" { - ep, exists := b.clusterEndpoints[endpointID] + ep, exists := b.clusterEndpoints.Get(endpointID) if !exists { return nil, fmt.Errorf("%w: cluster endpoint %s not found", ErrClusterEndpointNotFound, endpointID) } @@ -3149,8 +3225,8 @@ func (b *InMemoryBackend) DescribeDBClusterEndpoints(clusterID, endpointID strin return []DBClusterEndpoint{cp}, nil } - result := make([]DBClusterEndpoint, 0, len(b.clusterEndpoints)) - for _, ep := range b.clusterEndpoints { + result := make([]DBClusterEndpoint, 0, b.clusterEndpoints.Len()) + for _, ep := range b.clusterEndpoints.All() { if clusterID != "" && ep.DBClusterIdentifier != clusterID { continue } @@ -3164,12 +3240,12 @@ func (b *InMemoryBackend) DescribeDBClusterEndpoints(clusterID, endpointID strin func (b *InMemoryBackend) DeleteDBClusterEndpoint(endpointID string) (*DBClusterEndpoint, error) { b.mu.Lock("DeleteDBClusterEndpoint") defer b.mu.Unlock() - ep, exists := b.clusterEndpoints[endpointID] + ep, exists := b.clusterEndpoints.Get(endpointID) if !exists { return nil, fmt.Errorf("%w: cluster endpoint %s not found", ErrClusterEndpointNotFound, endpointID) } cp := *ep - delete(b.clusterEndpoints, endpointID) + b.clusterEndpoints.Delete(endpointID) delete(b.tags, b.rdsARN("cluster-endpoint", endpointID)) return &cp, nil @@ -3180,7 +3256,7 @@ func (b *InMemoryBackend) DeleteDBClusterEndpoint(endpointID string) (*DBCluster func (b *InMemoryBackend) DescribeValidDBInstanceModifications(id string) (*DBInstance, error) { b.mu.RLock("DescribeValidDBInstanceModifications") defer b.mu.RUnlock() - inst, exists := b.instances[id] + inst, exists := b.instances.Get(id) if !exists { return nil, fmt.Errorf("%w: instance %s not found", ErrInstanceNotFound, id) } @@ -3196,7 +3272,7 @@ func (b *InMemoryBackend) StartExportTask(taskID, sourceARN, s3Bucket string) (* } b.mu.Lock("StartExportTask") defer b.mu.Unlock() - if _, exists := b.exportTasks[taskID]; exists { + if _, exists := b.exportTasks.Get(taskID); exists { return nil, fmt.Errorf("%w: export task %s already exists", ErrExportTaskAlreadyExists, taskID) } task := &ExportTask{ @@ -3205,7 +3281,7 @@ func (b *InMemoryBackend) StartExportTask(taskID, sourceARN, s3Bucket string) (* Status: "complete", S3Bucket: s3Bucket, } - b.exportTasks[taskID] = task + b.exportTasks.Put(task) cp := *task return &cp, nil @@ -3216,7 +3292,7 @@ func (b *InMemoryBackend) DescribeExportTasks(taskID string) ([]ExportTask, erro b.mu.RLock("DescribeExportTasks") defer b.mu.RUnlock() if taskID != "" { - task, exists := b.exportTasks[taskID] + task, exists := b.exportTasks.Get(taskID) if !exists { return nil, fmt.Errorf("%w: export task %s not found", ErrExportTaskNotFound, taskID) } @@ -3224,8 +3300,8 @@ func (b *InMemoryBackend) DescribeExportTasks(taskID string) ([]ExportTask, erro return []ExportTask{cp}, nil } - result := make([]ExportTask, 0, len(b.exportTasks)) - for _, task := range b.exportTasks { + result := make([]ExportTask, 0, b.exportTasks.Len()) + for _, task := range b.exportTasks.All() { result = append(result, *task) } @@ -3239,13 +3315,13 @@ func (b *InMemoryBackend) CancelExportTask(taskID string) (*ExportTask, error) { } b.mu.Lock("CancelExportTask") defer b.mu.Unlock() - task, exists := b.exportTasks[taskID] + task, exists := b.exportTasks.Get(taskID) if !exists { return nil, fmt.Errorf("%w: export task %s not found", ErrExportTaskNotFound, taskID) } task.Status = "canceled" cp := *task - delete(b.exportTasks, taskID) + b.exportTasks.Delete(taskID) return &cp, nil } @@ -3262,7 +3338,7 @@ func (b *InMemoryBackend) CreateGlobalCluster( b.mu.Lock("CreateGlobalCluster") defer b.mu.Unlock() - if _, exists := b.globalClusters[id]; exists { + if _, exists := b.globalClusters.Get(id); exists { return nil, fmt.Errorf("%w: global cluster %s already exists", ErrGlobalClusterAlreadyExists, id) } @@ -3278,7 +3354,7 @@ func (b *InMemoryBackend) CreateGlobalCluster( StorageEncrypted: storageEncrypted, DeletionProtection: deletionProtection, } - b.globalClusters[id] = gc + b.globalClusters.Put(gc) cp := *gc return &cp, nil @@ -3290,7 +3366,7 @@ func (b *InMemoryBackend) DescribeGlobalClusters(id string) ([]GlobalCluster, er defer b.mu.RUnlock() if id != "" { - gc, exists := b.globalClusters[id] + gc, exists := b.globalClusters.Get(id) if !exists { return nil, fmt.Errorf("%w: global cluster %s not found", ErrGlobalClusterNotFound, id) } @@ -3299,8 +3375,8 @@ func (b *InMemoryBackend) DescribeGlobalClusters(id string) ([]GlobalCluster, er return []GlobalCluster{cp}, nil } - result := make([]GlobalCluster, 0, len(b.globalClusters)) - for _, gc := range b.globalClusters { + result := make([]GlobalCluster, 0, b.globalClusters.Len()) + for _, gc := range b.globalClusters.All() { result = append(result, *gc) } @@ -3316,7 +3392,7 @@ func (b *InMemoryBackend) DeleteGlobalCluster(id string) (*GlobalCluster, error) b.mu.Lock("DeleteGlobalCluster") defer b.mu.Unlock() - gc, exists := b.globalClusters[id] + gc, exists := b.globalClusters.Get(id) if !exists { return nil, fmt.Errorf("%w: global cluster %s not found", ErrGlobalClusterNotFound, id) } @@ -3329,7 +3405,7 @@ func (b *InMemoryBackend) DeleteGlobalCluster(id string) (*GlobalCluster, error) } cp := *gc - delete(b.globalClusters, id) + b.globalClusters.Delete(id) return &cp, nil } @@ -3346,22 +3422,22 @@ func (b *InMemoryBackend) ModifyGlobalCluster( b.mu.Lock("ModifyGlobalCluster") defer b.mu.Unlock() - gc, exists := b.globalClusters[id] + gc, exists := b.globalClusters.Get(id) if !exists { return nil, fmt.Errorf("%w: global cluster %s not found", ErrGlobalClusterNotFound, id) } if newGlobalClusterID != "" && newGlobalClusterID != id { - if _, alreadyExists := b.globalClusters[newGlobalClusterID]; alreadyExists { + if _, alreadyExists := b.globalClusters.Get(newGlobalClusterID); alreadyExists { return nil, fmt.Errorf( "%w: global cluster %s already exists", ErrGlobalClusterAlreadyExists, newGlobalClusterID, ) } - delete(b.globalClusters, id) + b.globalClusters.Delete(id) gc.GlobalClusterIdentifier = newGlobalClusterID - b.globalClusters[newGlobalClusterID] = gc + b.globalClusters.Put(gc) } if engineVersion != "" { gc.EngineVersion = engineVersion @@ -3387,7 +3463,7 @@ func (b *InMemoryBackend) AddRoleToDBCluster(clusterID, roleARN string) error { b.mu.Lock("AddRoleToDBCluster") defer b.mu.Unlock() - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -3412,7 +3488,7 @@ func (b *InMemoryBackend) AddRoleToDBInstance(instanceID, roleARN string) error b.mu.Lock("AddRoleToDBInstance") defer b.mu.Unlock() - if _, exists := b.instances[instanceID]; !exists { + if _, exists := b.instances.Get(instanceID); !exists { return fmt.Errorf("%w: instance %s not found", ErrInstanceNotFound, instanceID) } @@ -3440,14 +3516,14 @@ func (b *InMemoryBackend) AddSourceIdentifierToSubscription( b.mu.Lock("AddSourceIdentifierToSubscription") defer b.mu.Unlock() - sub, exists := b.eventSubscriptions[subscriptionName] + sub, exists := b.eventSubscriptions.Get(subscriptionName) if !exists { sub = &EventSubscription{ SubscriptionName: subscriptionName, Status: subscriptionStatusActive, SourceIDs: []string{}, } - b.eventSubscriptions[subscriptionName] = sub + b.eventSubscriptions.Put(sub) } if !slices.Contains(sub.SourceIDs, sourceIdentifier) { @@ -3480,8 +3556,8 @@ func (b *InMemoryBackend) ApplyPendingMaintenanceAction( id := rdsIDFromARN(resourceID) // Validate that the referenced resource exists (instance or cluster). - if _, ok := b.instances[id]; !ok { - if _, ok2 := b.clusters[id]; !ok2 { + if _, ok := b.instances.Get(id); !ok { + if _, ok2 := b.clusters.Get(id); !ok2 { return "", fmt.Errorf("%w: resource %s not found", ErrInstanceNotFound, resourceID) } } @@ -3504,13 +3580,13 @@ func (b *InMemoryBackend) AuthorizeDBSecurityGroupIngress( b.mu.Lock("AuthorizeDBSecurityGroupIngress") defer b.mu.Unlock() - sg, exists := b.dbSecurityGroups[groupName] + sg, exists := b.dbSecurityGroups.Get(groupName) if !exists { sg = &DBSecurityGroup{ DBSecurityGroupName: groupName, IPRanges: []IPRange{}, } - b.dbSecurityGroups[groupName] = sg + b.dbSecurityGroups.Put(sg) } for _, r := range sg.IPRanges { @@ -3546,7 +3622,7 @@ func (b *InMemoryBackend) BacktrackDBCluster( b.mu.RLock("BacktrackDBCluster") defer b.mu.RUnlock() - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -3580,7 +3656,7 @@ func (b *InMemoryBackend) CopyDBClusterParameterGroup( b.mu.Lock("CopyDBClusterParameterGroup") defer b.mu.Unlock() - src, exists := b.clusterParameterGroups[sourceGroupName] + src, exists := b.clusterParameterGroups.Get(sourceGroupName) if !exists { return nil, fmt.Errorf( "%w: cluster parameter group %s not found", @@ -3589,7 +3665,7 @@ func (b *InMemoryBackend) CopyDBClusterParameterGroup( ) } - if _, alreadyExists := b.clusterParameterGroups[targetGroupName]; alreadyExists { + if _, alreadyExists := b.clusterParameterGroups.Get(targetGroupName); alreadyExists { return nil, fmt.Errorf( "%w: cluster parameter group %s already exists", ErrParameterGroupAlreadyExists, @@ -3598,7 +3674,7 @@ func (b *InMemoryBackend) CopyDBClusterParameterGroup( } pg := copyParameterGroupTo(src, targetGroupName, targetDescription) - b.clusterParameterGroups[targetGroupName] = pg + b.clusterParameterGroups.Put(pg) cp := copyDBParameterGroup(pg) @@ -3625,7 +3701,7 @@ func (b *InMemoryBackend) CopyDBParameterGroup( b.mu.Lock("CopyDBParameterGroup") defer b.mu.Unlock() - src, exists := b.parameterGroups[sourceGroupName] + src, exists := b.parameterGroups.Get(sourceGroupName) if !exists { return nil, fmt.Errorf( "%w: parameter group %s not found", @@ -3634,7 +3710,7 @@ func (b *InMemoryBackend) CopyDBParameterGroup( ) } - if _, alreadyExists := b.parameterGroups[targetGroupName]; alreadyExists { + if _, alreadyExists := b.parameterGroups.Get(targetGroupName); alreadyExists { return nil, fmt.Errorf( "%w: parameter group %s already exists", ErrParameterGroupAlreadyExists, @@ -3643,7 +3719,7 @@ func (b *InMemoryBackend) CopyDBParameterGroup( } pg := copyParameterGroupTo(src, targetGroupName, targetDescription) - b.parameterGroups[targetGroupName] = pg + b.parameterGroups.Put(pg) cp := copyDBParameterGroup(pg) @@ -3688,12 +3764,12 @@ func (b *InMemoryBackend) CopyOptionGroup( b.mu.Lock("CopyOptionGroup") defer b.mu.Unlock() - src, exists := b.optionGroups[sourceGroupName] + src, exists := b.optionGroups.Get(sourceGroupName) if !exists { return nil, fmt.Errorf("%w: option group %s not found", ErrOptionGroupNotFound, sourceGroupName) } - if _, alreadyExists := b.optionGroups[targetGroupName]; alreadyExists { + if _, alreadyExists := b.optionGroups.Get(targetGroupName); alreadyExists { return nil, fmt.Errorf( "%w: option group %s already exists", ErrOptionGroupAlreadyExists, @@ -3715,7 +3791,7 @@ func (b *InMemoryBackend) CopyOptionGroup( MajorEngineVersion: src.MajorEngineVersion, Options: opts, } - b.optionGroups[targetGroupName] = og + b.optionGroups.Put(og) cp := *og cp.Options = make([]OptionGroupOption, len(og.Options)) @@ -3740,7 +3816,7 @@ func (b *InMemoryBackend) CreateBlueGreenDeployment( id := "bgd-" + name - if _, exists := b.blueGreenDeployments[id]; exists { + if _, exists := b.blueGreenDeployments.Get(id); exists { return nil, fmt.Errorf( "%w: Blue/Green Deployment %s already exists", ErrBlueGreenDeploymentAlreadyExists, @@ -3756,7 +3832,7 @@ func (b *InMemoryBackend) CreateBlueGreenDeployment( Target: target, Status: blueGreenDeploymentStatusAvailable, } - b.blueGreenDeployments[id] = deployment + b.blueGreenDeployments.Put(deployment) cp := *deployment @@ -3776,7 +3852,7 @@ func (b *InMemoryBackend) RemoveRoleFromDBCluster(clusterID, roleARN string) err b.mu.Lock("RemoveRoleFromDBCluster") defer b.mu.Unlock() - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -3802,7 +3878,7 @@ func (b *InMemoryBackend) RemoveRoleFromDBInstance(instanceID, roleARN string) e b.mu.Lock("RemoveRoleFromDBInstance") defer b.mu.Unlock() - if _, exists := b.instances[instanceID]; !exists { + if _, exists := b.instances.Get(instanceID); !exists { return fmt.Errorf("%w: instance %s not found", ErrInstanceNotFound, instanceID) } @@ -3830,7 +3906,7 @@ func (b *InMemoryBackend) RemoveSourceIdentifierFromSubscription( b.mu.Lock("RemoveSourceIdentifierFromSubscription") defer b.mu.Unlock() - sub, exists := b.eventSubscriptions[subscriptionName] + sub, exists := b.eventSubscriptions.Get(subscriptionName) if !exists { return nil, fmt.Errorf("%w: subscription %s not found", ErrEventSubscriptionNotFound, subscriptionName) } @@ -3871,7 +3947,7 @@ func (b *InMemoryBackend) AddClusterInternal(id, engine string) *DBCluster { Engine: engine, Status: instanceStatusAvailable, } - b.clusters[id] = c + b.clusters.Put(c) cp := *c return &cp @@ -3889,7 +3965,7 @@ func (b *InMemoryBackend) AddInstanceInternal(id, engine string) *DBInstance { DBInstanceClass: defaultInstanceClass, AllocatedStorage: defaultAllocatedStorage, } - b.instances[id] = inst + b.instances.Put(inst) cp := *inst return &cp @@ -3906,7 +3982,7 @@ func (b *InMemoryBackend) AddEventSubscriptionInternal(name, snsTopicArn string) Status: subscriptionStatusActive, SourceIDs: []string{}, } - b.eventSubscriptions[name] = sub + b.eventSubscriptions.Put(sub) cp := *sub cp.SourceIDs = make([]string, 0) @@ -3925,7 +4001,7 @@ func (b *InMemoryBackend) AddBlueGreenDeploymentInternal(name, source string) *B Source: source, Status: blueGreenDeploymentStatusAvailable, } - b.blueGreenDeployments[id] = d + b.blueGreenDeployments.Put(d) cp := *d return &cp @@ -3941,7 +4017,7 @@ func (b *InMemoryBackend) AddSecurityGroupInternal(name, description string) *DB DBSecurityGroupDescription: description, IPRanges: []IPRange{}, } - b.dbSecurityGroups[name] = sg + b.dbSecurityGroups.Put(sg) cp := *sg cp.IPRanges = make([]IPRange, 0) diff --git a/services/rds/batch1.go b/services/rds/batch1.go index 8ec8f0496..605754fb3 100644 --- a/services/rds/batch1.go +++ b/services/rds/batch1.go @@ -44,7 +44,7 @@ func (b *InMemoryBackend) CreateDBShardGroup( if id == "" { return nil, fmt.Errorf("%w: DBShardGroupIdentifier is required", ErrInvalidParameter) } - if _, exists := b.shardGroups[id]; exists { + if _, exists := b.shardGroups.Get(id); exists { return nil, fmt.Errorf("%w: %s", ErrDBShardGroupAlreadyExists, id) } if clusterID == "" { @@ -61,7 +61,7 @@ func (b *InMemoryBackend) CreateDBShardGroup( Status: shardGroupStatusAvailableInternal, Endpoint: id + ".limitless." + clusterID + ".rds.amazonaws.com", } - b.shardGroups[id] = sg + b.shardGroups.Put(sg) cp := *sg return &cp, nil @@ -72,14 +72,14 @@ func (b *InMemoryBackend) DeleteDBShardGroup(id string) (*DBShardGroup, error) { b.mu.Lock("DeleteDBShardGroup") defer b.mu.Unlock() - sg, exists := b.shardGroups[id] + sg, exists := b.shardGroups.Get(id) if !exists { return nil, fmt.Errorf("%w: %s", ErrDBShardGroupNotFound, id) } cp := *sg cp.Status = shardGroupStatusDeletingInternal - delete(b.shardGroups, id) + b.shardGroups.Delete(id) return &cp, nil } @@ -89,8 +89,8 @@ func (b *InMemoryBackend) DescribeDBShardGroups(id string) ([]DBShardGroup, erro b.mu.RLock("DescribeDBShardGroups") defer b.mu.RUnlock() - result := make([]DBShardGroup, 0, len(b.shardGroups)) - for _, sg := range b.shardGroups { + result := make([]DBShardGroup, 0, b.shardGroups.Len()) + for _, sg := range b.shardGroups.All() { if id != "" && sg.DBShardGroupIdentifier != id { continue } @@ -124,7 +124,7 @@ func (b *InMemoryBackend) ModifyDBShardGroup( b.mu.Lock("ModifyDBShardGroup") defer b.mu.Unlock() - sg, exists := b.shardGroups[id] + sg, exists := b.shardGroups.Get(id) if !exists { return nil, fmt.Errorf("%w: %s", ErrDBShardGroupNotFound, id) } @@ -145,7 +145,7 @@ func (b *InMemoryBackend) RebootDBShardGroup(id string) (*DBShardGroup, error) { b.mu.Lock("RebootDBShardGroup") defer b.mu.Unlock() - sg, exists := b.shardGroups[id] + sg, exists := b.shardGroups.Get(id) if !exists { return nil, fmt.Errorf("%w: %s", ErrDBShardGroupNotFound, id) } @@ -169,7 +169,7 @@ func (b *InMemoryBackend) CreateIntegration( if name == "" { return nil, fmt.Errorf("%w: IntegrationName is required", ErrInvalidParameter) } - if _, exists := b.integrations[name]; exists { + if _, exists := b.integrations.Get(name); exists { return nil, fmt.Errorf("%w: %s", ErrIntegrationAlreadyExists, name) } @@ -189,7 +189,7 @@ func (b *InMemoryBackend) CreateIntegration( Status: integrationStatusActive, CreatedAt: time.Now(), } - b.integrations[name] = intg + b.integrations.Put(intg) cp := *intg return &cp, nil @@ -200,11 +200,11 @@ func (b *InMemoryBackend) DeleteIntegration(identifier string) (*Integration, er b.mu.Lock("DeleteIntegration") defer b.mu.Unlock() - for _, intg := range b.integrations { + for _, intg := range b.integrations.All() { if intg.IntegrationName == identifier || intg.IntegrationArn == identifier { cp := *intg cp.Status = integrationStatusDeleting - delete(b.integrations, intg.IntegrationName) + b.integrations.Delete(intg.IntegrationName) return &cp, nil } @@ -218,8 +218,8 @@ func (b *InMemoryBackend) DescribeIntegrations(identifier string) ([]Integration b.mu.RLock("DescribeIntegrations") defer b.mu.RUnlock() - result := make([]Integration, 0, len(b.integrations)) - for _, intg := range b.integrations { + result := make([]Integration, 0, b.integrations.Len()) + for _, intg := range b.integrations.All() { if identifier != "" && intg.IntegrationName != identifier && intg.IntegrationArn != identifier { continue @@ -250,7 +250,7 @@ func (b *InMemoryBackend) ModifyIntegration(identifier, dataFilter, description b.mu.Lock("ModifyIntegration") defer b.mu.Unlock() - for _, intg := range b.integrations { + for _, intg := range b.integrations.All() { if intg.IntegrationName == identifier || intg.IntegrationArn == identifier { if dataFilter != "" { intg.DataFilter = dataFilter @@ -288,7 +288,7 @@ func (b *InMemoryBackend) CreateTenantDatabase( } key := tenantKey(instanceID, tenantDBName) - if _, exists := b.tenantDatabases[key]; exists { + if _, exists := b.tenantDatabases.Get(key); exists { return nil, fmt.Errorf( "%w: %s/%s", ErrTenantDatabaseAlreadyExists, @@ -309,7 +309,7 @@ func (b *InMemoryBackend) CreateTenantDatabase( Status: tenantStatusAvailableInternal, CreatedAt: time.Now(), } - b.tenantDatabases[key] = tdb + b.tenantDatabases.Put(tdb) cp := *tdb return &cp, nil @@ -323,14 +323,14 @@ func (b *InMemoryBackend) DeleteTenantDatabase( defer b.mu.Unlock() key := tenantKey(instanceID, tenantDBName) - tdb, exists := b.tenantDatabases[key] + tdb, exists := b.tenantDatabases.Get(key) if !exists { return nil, fmt.Errorf("%w: %s/%s", ErrTenantDatabaseNotFound, instanceID, tenantDBName) } cp := *tdb cp.Status = tenantStatusDeletingInternal - delete(b.tenantDatabases, key) + b.tenantDatabases.Delete(key) return &cp, nil } @@ -342,8 +342,8 @@ func (b *InMemoryBackend) DescribeTenantDatabases( b.mu.RLock("DescribeTenantDatabases") defer b.mu.RUnlock() - result := make([]TenantDatabase, 0, len(b.tenantDatabases)) - for _, tdb := range b.tenantDatabases { + result := make([]TenantDatabase, 0, b.tenantDatabases.Len()) + for _, tdb := range b.tenantDatabases.All() { if instanceID != "" && tdb.DBInstanceIdentifier != instanceID { continue } @@ -377,7 +377,7 @@ func (b *InMemoryBackend) ModifyTenantDatabase( defer b.mu.Unlock() key := tenantKey(instanceID, tenantDBName) - tdb, exists := b.tenantDatabases[key] + tdb, exists := b.tenantDatabases.Get(key) if !exists { return nil, fmt.Errorf("%w: %s/%s", ErrTenantDatabaseNotFound, instanceID, tenantDBName) } @@ -397,7 +397,7 @@ func (b *InMemoryBackend) CreateDBClusterAutomatedBackup( b.mu.Lock("CreateDBClusterAutomatedBackup") defer b.mu.Unlock() - cluster, exists := b.clusters[clusterID] + cluster, exists := b.clusters.Get(clusterID) if !exists { return nil } @@ -411,7 +411,7 @@ func (b *InMemoryBackend) CreateDBClusterAutomatedBackup( Status: clusterBackupStatusAvailable, StorageEncrypted: cluster.StorageEncrypted, } - b.clusterAutomatedBackups[clusterID] = backup + b.clusterAutomatedBackups.Put(backup) return backup } @@ -423,11 +423,11 @@ func (b *InMemoryBackend) DeleteDBClusterAutomatedBackup( b.mu.Lock("DeleteDBClusterAutomatedBackup") defer b.mu.Unlock() - for key, backup := range b.clusterAutomatedBackups { + for _, backup := range b.clusterAutomatedBackups.All() { if backup.DBClusterResourceID == resourceID || backup.DBClusterIdentifier == resourceID { cp := *backup cp.Status = clusterBackupStatusDeleted - delete(b.clusterAutomatedBackups, key) + b.clusterAutomatedBackups.Delete(clusterAutomatedBackupsKeyFn(backup)) return &cp, nil } @@ -443,8 +443,8 @@ func (b *InMemoryBackend) DescribeDBClusterAutomatedBackups( b.mu.RLock("DescribeDBClusterAutomatedBackups") defer b.mu.RUnlock() - result := make([]DBClusterAutomatedBackup, 0, len(b.clusterAutomatedBackups)) - for _, backup := range b.clusterAutomatedBackups { + result := make([]DBClusterAutomatedBackup, 0, b.clusterAutomatedBackups.Len()) + for _, backup := range b.clusterAutomatedBackups.All() { if clusterID != "" && backup.DBClusterIdentifier != clusterID { continue } diff --git a/services/rds/batch2_test.go b/services/rds/batch2_test.go index 5a69b6c94..fe0533b38 100644 --- a/services/rds/batch2_test.go +++ b/services/rds/batch2_test.go @@ -1885,13 +1885,29 @@ func TestBatch2_Persistence_JSONContainsBatch1Keys(t *testing.T) { var raw map[string]json.RawMessage require.NoError(t, json.Unmarshal(snap, &raw)) + // As of the Phase 3.3 pkgs/store conversion, resource collections backed by + // a store.Table are serialized under the nested "tables" blob (produced by + // store.Registry.SnapshotAll) rather than as top-level snapshot keys. + require.Contains(t, raw, "tables") + + var tables map[string]json.RawMessage + require.NoError(t, json.Unmarshal(raw["tables"], &tables)) + for _, key := range []string{ "customEngineVersions", - "automatedBackups", "shardGroups", "integrations", "tenantDatabases", "clusterAutomatedBackups", + } { + assert.Contains(t, tables, key, "snapshot tables JSON missing key: %s", key) + } + + // automatedBackups and snapshotTenantDatabases remain plain maps (not + // store.Table-backed -- see registerAllTables's exclusion comment in + // store_setup.go) and so still serialize as top-level snapshot keys. + for _, key := range []string{ + "automatedBackups", "snapshotTenantDatabases", } { assert.Contains(t, raw, key, "snapshot JSON missing key: %s", key) diff --git a/services/rds/batch3.go b/services/rds/batch3.go index 22d72f36c..8c6dcb42f 100644 --- a/services/rds/batch3.go +++ b/services/rds/batch3.go @@ -31,9 +31,9 @@ func (b *InMemoryBackend) DescribeCustomDBEngineVersions(engine, engineVersion s b.mu.RLock("DescribeCustomDBEngineVersions") defer b.mu.RUnlock() - result := make([]CustomDBEngineVersion, 0, len(b.customEngineVersions)) + result := make([]CustomDBEngineVersion, 0, b.customEngineVersions.Len()) - for _, cev := range b.customEngineVersions { + for _, cev := range b.customEngineVersions.All() { if engine != "" && cev.Engine != engine { continue } @@ -69,7 +69,7 @@ func (b *InMemoryBackend) AddDBRecommendation(rec DBRecommendation) { defer b.mu.Unlock() cp := rec - b.recommendations[rec.RecommendationID] = &cp + b.recommendations.Put(&cp) } // GetPerformanceInsightsData returns synthetic Performance Insights metric data points @@ -86,7 +86,7 @@ func (b *InMemoryBackend) GetPerformanceInsightsData( // Validate that the instance exists and has Performance Insights enabled. var found *DBInstance - for _, inst := range b.instances { + for _, inst := range b.instances.All() { if inst.DbiResourceID == resourceID || inst.DBInstanceIdentifier == resourceID { found = inst diff --git a/services/rds/batch3_test.go.rej b/services/rds/batch3_test.go.rej deleted file mode 100644 index 132ffbf6a..000000000 --- a/services/rds/batch3_test.go.rej +++ /dev/null @@ -1,27 +0,0 @@ ---- batch3_test.go -+++ batch3_test.go -@@ -531,18 +531,21 @@ - func TestPerformanceInsights_ReturnsDataPoints(t *testing.T) { - t.Parallel() - - b := newBatch3Backend() -- now := time.Now().UTC() -+ now := time.Now().UTC() -+ b.SetPerformanceInsightsData("db-instance-1", "db.load.avg", []rds.PIDataPoint{{Timestamp: now.Format(time.RFC3339), Value: 1.0}}) -+ b.CreateDBInstance("db-instance-1", "mysql", "db.t3.micro", 20, map[string]string{"PerformanceInsightsEnabled": "true"}) - start := now.Add(-time.Hour) - - points, _ := b.GetPerformanceInsightsData("db-instance-1", "db.load.avg", start, now, 60) - -- assert.NotEmpty(t, points, "should return data points for a 1-hour window at 60s intervals") -- assert.Greater(t, len(points), 50, "should have at least 50 data points for a 1-hour window") -+ assert.NotEmpty(t, points, "should return data points") - } - - func TestPerformanceInsights_ViaHandler(t *testing.T) { - t.Parallel() - - h := newBatch3Handler() -+ h.Backend.SetPerformanceInsightsData("my-db-instance", "db.load.avg", []rds.PIDataPoint{{Timestamp: time.Now().Format(time.RFC3339), Value: 1.0}}) -+ h.Backend.CreateDBInstance("my-db-instance", "mysql", "db.t3.micro", 20, map[string]string{"PerformanceInsightsEnabled": "true"}) - start := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC) diff --git a/services/rds/batch3_test_pi.patch b/services/rds/batch3_test_pi.patch deleted file mode 100644 index f46e9f555..000000000 --- a/services/rds/batch3_test_pi.patch +++ /dev/null @@ -1,64 +0,0 @@ ---- services/rds/batch3_test.go -+++ services/rds/batch3_test.go -@@ -535,27 +535,28 @@ - - b := newBatch3Backend() - now := time.Now().UTC() -+ b.SetPerformanceInsightsData("db-instance-1", "db.load.avg", []rds.PIDataPoint{{Timestamp: now.Format(time.RFC3339), Value: 1.0}}) -+ b.CreateDBInstance("db-instance-1", "mysql", "db.t3.micro", 20, map[string]string{"PerformanceInsightsEnabled": "true"}) - start := now.Add(-time.Hour) - - points, _ := b.GetPerformanceInsightsData("db-instance-1", "db.load.avg", start, now, 60) - -- assert.NotEmpty(t, points, "should return data points for a 1-hour window at 60s intervals") -- assert.Greater(t, len(points), 50, "should have at least 50 data points for a 1-hour window") -+ assert.NotEmpty(t, points, "should return data points") - } - --func TestPerformanceInsights_Deterministic(t *testing.T) { -- t.Parallel() -- -- b := newBatch3Backend() -- start := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC) -- end := time.Date(2025, 1, 15, 11, 0, 0, 0, time.UTC) -- -- points1, _ := b.GetPerformanceInsightsData("db-1", "db.load.avg", start, end, 60) -- points2, _ := b.GetPerformanceInsightsData("db-1", "db.load.avg", start, end, 60) -- -- require.Len(t, points2, len(points1)) -- for i := range points1 { -- assert.Equal(t, points1[i].Timestamp, points2[i].Timestamp) -- assert.InDelta(t, points1[i].Value, points2[i].Value, 0) -- } --} -- --func TestPerformanceInsights_DifferentResourcesDifferentData(t *testing.T) { -- t.Parallel() -- -- b := newBatch3Backend() -- start := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC) -- end := time.Date(2025, 1, 15, 10, 10, 0, 0, time.UTC) -- -- points1, _ := b.GetPerformanceInsightsData("db-resource-A", "db.load.avg", start, end, 60) -- points2, _ := b.GetPerformanceInsightsData("db-resource-B", "db.load.avg", start, end, 60) -- -- require.Len(t, points2, len(points1), "same time window should yield same number of points") -- -- differentCount := 0 -- for i := range points1 { -- if points1[i].Value != points2[i].Value { -- differentCount++ -- } -- } -- -- assert.Positive(t, differentCount, "different resources should produce different values") --} - - func TestPerformanceInsights_ViaHandler(t *testing.T) { - t.Parallel() - - h := newBatch3Handler() -+ h.Backend.SetPerformanceInsightsData("my-db-instance", "db.load.avg", []rds.PIDataPoint{{Timestamp: time.Now().Format(time.RFC3339), Value: 1.0}}) -+ h.Backend.CreateDBInstance("my-db-instance", "mysql", "db.t3.micro", 20, map[string]string{"PerformanceInsightsEnabled": "true"}) - start := time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC) - end := time.Date(2025, 1, 15, 10, 10, 0, 0, time.UTC) diff --git a/services/rds/delete_db_cluster_test.go b/services/rds/delete_db_cluster_test.go new file mode 100644 index 000000000..9aedaa7c3 --- /dev/null +++ b/services/rds/delete_db_cluster_test.go @@ -0,0 +1,103 @@ +package rds_test + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/rds" +) + +// Test_DeleteDBCluster_FinalSnapshotContract verifies AWS's +// SkipFinalSnapshot/FinalDBSnapshotIdentifier contract for DeleteDBCluster: +// a final snapshot must be requested unless explicitly skipped, the two +// parameters are mutually exclusive, and a real manual cluster snapshot is +// created and persisted before the cluster is removed. +func Test_DeleteDBCluster_FinalSnapshotContract(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + finalSnapshotID string + wantErrContains string + skipFinalSnapshot bool + }{ + { + name: "missing both SkipFinalSnapshot and FinalDBSnapshotIdentifier is rejected", + wantErrContains: "FinalDBSnapshotIdentifier", + }, + { + name: "SkipFinalSnapshot with FinalDBSnapshotIdentifier is rejected", + skipFinalSnapshot: true, + finalSnapshotID: "final-csnap", + wantErrContains: "FinalDBSnapshotIdentifier", + }, + { + name: "SkipFinalSnapshot alone deletes without a snapshot", + skipFinalSnapshot: true, + }, + { + name: "FinalDBSnapshotIdentifier alone takes a final snapshot", + finalSnapshotID: "final-csnap", + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := rds.NewInMemoryBackend("000000000000", "us-east-1") + _, err := b.CreateDBCluster( + "del-cluster", "aurora-postgresql", "admin", "", "", 0, nil, rds.DBClusterOptions{}, + ) + require.NoError(t, err) + + _, err = b.DeleteDBClusterWithOptions("del-cluster", tt.skipFinalSnapshot, tt.finalSnapshotID) + + if tt.wantErrContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErrContains) + assert.Contains(t, err.Error(), "InvalidParameterCombination") + // The cluster must NOT have been deleted on a rejected request. + _, describeErr := b.DescribeDBClusters("del-cluster") + assert.NoError(t, describeErr, "cluster should still exist after a rejected delete") + + return + } + + require.NoError(t, err) + + _, describeErr := b.DescribeDBClusters("del-cluster") + require.Error(t, describeErr, "cluster should be removed after delete") + + snaps, _ := b.DescribeDBClusterSnapshots("", "del-cluster") + if tt.finalSnapshotID == "" { + assert.Empty(t, snaps, "no final snapshot should be created when SkipFinalSnapshot is set") + + return + } + + require.Len(t, snaps, 1, "exactly one final cluster snapshot should be created") + snap := snaps[0] + assert.Equal(t, tt.finalSnapshotID, snap.DBClusterSnapshotIdentifier) + assert.Equal(t, "del-cluster", snap.DBClusterIdentifier) + assert.Equal(t, "aurora-postgresql", snap.Engine) + }) + } +} + +// Test_DeleteDBCluster_NotFoundBeforeParamValidation verifies that a +// nonexistent cluster yields DBClusterNotFound even when the +// SkipFinalSnapshot/FinalDBSnapshotIdentifier combination is also invalid. +func Test_DeleteDBCluster_NotFoundBeforeParamValidation(t *testing.T) { + t.Parallel() + + h := newRDSHandler() + rec := postRDSForm(t, h, "Action=DeleteDBCluster&Version=2014-10-31&DBClusterIdentifier=missing-cluster") + + require.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "DBClusterNotFound") + assert.NotContains(t, rec.Body.String(), "InvalidParameterCombination") +} diff --git a/services/rds/delete_db_instance_test.go b/services/rds/delete_db_instance_test.go new file mode 100644 index 000000000..3dfc39f48 --- /dev/null +++ b/services/rds/delete_db_instance_test.go @@ -0,0 +1,147 @@ +package rds_test + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/rds" +) + +// Test_DeleteDBInstance_FinalSnapshotContract verifies AWS's +// SkipFinalSnapshot/FinalDBSnapshotIdentifier/DeleteAutomatedBackups contract +// for DeleteDBInstance: a final snapshot must be requested unless explicitly +// skipped, the two snapshot parameters are mutually exclusive, and a real +// manual snapshot is created and persisted before the instance is removed. +func Test_DeleteDBInstance_FinalSnapshotContract(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + finalSnapshotID string + wantErrContains string + skipFinalSnapshot bool + }{ + { + name: "missing both SkipFinalSnapshot and FinalDBSnapshotIdentifier is rejected", + wantErrContains: "FinalDBSnapshotIdentifier", + }, + { + name: "SkipFinalSnapshot with FinalDBSnapshotIdentifier is rejected", + skipFinalSnapshot: true, + finalSnapshotID: "final-snap", + wantErrContains: "FinalDBSnapshotIdentifier", + }, + { + name: "SkipFinalSnapshot alone deletes without a snapshot", + skipFinalSnapshot: true, + }, + { + name: "FinalDBSnapshotIdentifier alone takes a final snapshot", + finalSnapshotID: "final-snap", + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := rds.NewInMemoryBackend("000000000000", "us-east-1") + _, err := b.CreateDBInstance( + "del-inst", "postgres", "db.t3.micro", "", "admin", "", + 20, rds.DBInstanceOptions{}, + ) + require.NoError(t, err) + + _, err = b.DeleteDBInstanceWithOptions("del-inst", tt.skipFinalSnapshot, tt.finalSnapshotID, true) + + if tt.wantErrContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErrContains) + assert.Contains(t, err.Error(), "InvalidParameterCombination") + // The instance must NOT have been deleted on a rejected request. + _, describeErr := b.DescribeDBInstances("del-inst") + assert.NoError(t, describeErr, "instance should still exist after a rejected delete") + + return + } + + require.NoError(t, err) + + // Instance is gone either way. + _, describeErr := b.DescribeDBInstances("del-inst") + require.Error(t, describeErr, "instance should be removed after delete") + + snaps, _ := b.DescribeDBSnapshots("", "del-inst") + if tt.finalSnapshotID == "" { + assert.Empty(t, snaps, "no final snapshot should be created when SkipFinalSnapshot is set") + + return + } + + require.Len(t, snaps, 1, "exactly one final snapshot should be created") + snap := snaps[0] + assert.Equal(t, tt.finalSnapshotID, snap.DBSnapshotIdentifier) + assert.Equal(t, "del-inst", snap.DBInstanceIdentifier) + assert.Equal(t, "manual", snap.SnapshotType) + assert.Equal(t, "postgres", snap.Engine) + }) + } +} + +// Test_DeleteDBInstance_DeleteAutomatedBackups verifies that +// DeleteAutomatedBackups=false retains the instance's automated backup record +// after deletion, while the AWS default (true) removes it. +func Test_DeleteDBInstance_DeleteAutomatedBackups(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + deleteAutomatedBackups bool + wantBackupsRemain bool + }{ + {name: "default true removes the automated backup", deleteAutomatedBackups: true}, + {name: "false retains the automated backup", deleteAutomatedBackups: false, wantBackupsRemain: true}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := rds.NewInMemoryBackend("000000000000", "us-east-1") + _, err := b.CreateDBInstance( + "del-inst-backup", "postgres", "db.t3.micro", "", "admin", "", + 20, rds.DBInstanceOptions{BackupRetentionPeriod: 7}, + ) + require.NoError(t, err) + require.Len(t, b.DescribeDBInstanceAutomatedBackups("del-inst-backup"), 1) + + _, err = b.DeleteDBInstanceWithOptions("del-inst-backup", true, "", tt.deleteAutomatedBackups) + require.NoError(t, err) + + backups := b.DescribeDBInstanceAutomatedBackups("del-inst-backup") + if tt.wantBackupsRemain { + assert.Len(t, backups, 1) + } else { + assert.Empty(t, backups) + } + }) + } +} + +// Test_DeleteDBInstance_NotFoundBeforeParamValidation verifies that a +// nonexistent instance yields DBInstanceNotFound even when the +// SkipFinalSnapshot/FinalDBSnapshotIdentifier combination is also invalid — +// matching AWS's behavior of resolving the target resource first. +func Test_DeleteDBInstance_NotFoundBeforeParamValidation(t *testing.T) { + t.Parallel() + + h := newRDSHandler() + rec := postRDSForm(t, h, "Action=DeleteDBInstance&Version=2014-10-31&DBInstanceIdentifier=missing-inst") + + require.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "DBInstanceNotFound") + assert.NotContains(t, rec.Body.String(), "InvalidParameterCombination") +} diff --git a/services/rds/describe_db_instances_filters_test.go b/services/rds/describe_db_instances_filters_test.go new file mode 100644 index 000000000..b1b644840 --- /dev/null +++ b/services/rds/describe_db_instances_filters_test.go @@ -0,0 +1,108 @@ +package rds_test + +import ( + "encoding/xml" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Test_DescribeDBInstances_Filters verifies AWS's DescribeDBInstances +// Filters.Filter.N.Name/Values.member.M contract: db-instance-id, engine, +// db-cluster-id, and dbi-resource-id narrow the result set (OR within a +// filter's Values, AND across filters), and an unrecognized filter name +// returns InvalidParameterValue. +func Test_DescribeDBInstances_Filters(t *testing.T) { + t.Parallel() + + type describeResp struct { + XMLName xml.Name `xml:"DescribeDBInstancesResponse"` + Result struct { + DBInstances struct { + Members []struct { + DBInstanceIdentifier string `xml:"DBInstanceIdentifier"` + } `xml:"DBInstance"` + } `xml:"DBInstances"` + } `xml:"DescribeDBInstancesResult"` + } + + cases := []struct { + name string + query string + wantErrText string + wantIDs []string + wantCode int + }{ + { + name: "engine filter matches only mysql instances", + query: "Filters.Filter.1.Name=engine&Filters.Filter.1.Values.member.1=mysql", + wantCode: http.StatusOK, + wantIDs: []string{"filt-mysql-1"}, + }, + { + name: "db-instance-id filter with multiple values ORs together", + query: "Filters.Filter.1.Name=db-instance-id" + + "&Filters.Filter.1.Values.member.1=filt-mysql-1" + + "&Filters.Filter.1.Values.member.2=filt-postgres-1", + wantCode: http.StatusOK, + wantIDs: []string{"filt-mysql-1", "filt-postgres-1"}, + }, + { + name: "two filters AND together", + query: "Filters.Filter.1.Name=engine&Filters.Filter.1.Values.member.1=postgres" + + "&Filters.Filter.2.Name=db-instance-id&Filters.Filter.2.Values.member.1=filt-mysql-1", + wantCode: http.StatusOK, + wantIDs: nil, + }, + { + name: "unrecognized filter name is rejected", + query: "Filters.Filter.1.Name=bogus-filter&Filters.Filter.1.Values.member.1=x", + wantCode: http.StatusBadRequest, + wantErrText: "InvalidParameterValue", + }, + { + name: "no filters returns everything", + query: "", + wantCode: http.StatusOK, + wantIDs: []string{"filt-mysql-1", "filt-postgres-1"}, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newRDSHandler() + postRDSForm(t, h, + "Action=CreateDBInstance&Version=2014-10-31"+ + "&DBInstanceIdentifier=filt-mysql-1&Engine=mysql") + postRDSForm(t, h, + "Action=CreateDBInstance&Version=2014-10-31"+ + "&DBInstanceIdentifier=filt-postgres-1&Engine=postgres") + + body := "Action=DescribeDBInstances&Version=2014-10-31" + if tt.query != "" { + body += "&" + tt.query + } + rec := postRDSForm(t, h, body) + + require.Equal(t, tt.wantCode, rec.Code) + if tt.wantErrText != "" { + assert.Contains(t, rec.Body.String(), tt.wantErrText) + + return + } + + var resp describeResp + require.NoError(t, xml.Unmarshal(rec.Body.Bytes(), &resp)) + + gotIDs := make([]string, 0, len(resp.Result.DBInstances.Members)) + for _, m := range resp.Result.DBInstances.Members { + gotIDs = append(gotIDs, m.DBInstanceIdentifier) + } + assert.ElementsMatch(t, tt.wantIDs, gotIDs) + }) + } +} diff --git a/services/rds/export_test.go b/services/rds/export_test.go index 13b05d042..4bd6986de 100644 --- a/services/rds/export_test.go +++ b/services/rds/export_test.go @@ -11,13 +11,13 @@ func FlushInstanceLifecycle(b *InMemoryBackend) { b.mu.Lock("FlushInstanceLifecycle") defer b.mu.Unlock() - for id, inst := range b.instances { + for _, inst := range b.instances.All() { if inst.DBInstanceStatus == instanceStatusCreating || inst.DBInstanceStatus == instanceStatusModifying { applyPendingModifications(inst) inst.DBInstanceStatus = instanceStatusAvailable } - delete(b.instanceReadyAt, id) + delete(b.instanceReadyAt, inst.DBInstanceIdentifier) } } @@ -46,7 +46,7 @@ func InstanceCount(b *InMemoryBackend) int { b.mu.RLock("InstanceCount") defer b.mu.RUnlock() - return len(b.instances) + return b.instances.Len() } // ClusterCount returns the number of DB clusters in the backend. @@ -54,7 +54,7 @@ func ClusterCount(b *InMemoryBackend) int { b.mu.RLock("ClusterCount") defer b.mu.RUnlock() - return len(b.clusters) + return b.clusters.Len() } // SnapshotCount returns the number of DB snapshots in the backend. @@ -62,7 +62,7 @@ func SnapshotCount(b *InMemoryBackend) int { b.mu.RLock("SnapshotCount") defer b.mu.RUnlock() - return len(b.snapshots) + return b.snapshots.Len() } // EventSubscriptionCount returns the number of event subscriptions in the backend. @@ -70,7 +70,7 @@ func EventSubscriptionCount(b *InMemoryBackend) int { b.mu.RLock("EventSubscriptionCount") defer b.mu.RUnlock() - return len(b.eventSubscriptions) + return b.eventSubscriptions.Len() } // EventMessagesForSource returns published event messages for a source identifier. @@ -95,7 +95,7 @@ func SecurityGroupCount(b *InMemoryBackend) int { b.mu.RLock("SecurityGroupCount") defer b.mu.RUnlock() - return len(b.dbSecurityGroups) + return b.dbSecurityGroups.Len() } // BlueGreenDeploymentCount returns the number of Blue/Green Deployments in the backend. @@ -103,7 +103,7 @@ func BlueGreenDeploymentCount(b *InMemoryBackend) int { b.mu.RLock("BlueGreenDeploymentCount") defer b.mu.RUnlock() - return len(b.blueGreenDeployments) + return b.blueGreenDeployments.Len() } // ClusterRoleCount returns the number of IAM roles associated with a cluster. @@ -132,7 +132,7 @@ func RecommendationCount(b *InMemoryBackend) int { b.mu.RLock("RecommendationCount") defer b.mu.RUnlock() - return len(b.recommendations) + return b.recommendations.Len() } // IntegrationCount returns the number of integrations in the backend. @@ -140,7 +140,7 @@ func IntegrationCount(b *InMemoryBackend) int { b.mu.RLock("IntegrationCount") defer b.mu.RUnlock() - return len(b.integrations) + return b.integrations.Len() } // ShardGroupCount returns the number of shard groups in the backend. @@ -148,5 +148,5 @@ func ShardGroupCount(b *InMemoryBackend) int { b.mu.RLock("ShardGroupCount") defer b.mu.RUnlock() - return len(b.shardGroups) + return b.shardGroups.Len() } diff --git a/services/rds/handler.go b/services/rds/handler.go index f7190fbba..d45ef83fb 100644 --- a/services/rds/handler.go +++ b/services/rds/handler.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/url" + "slices" "sort" "strconv" "strings" @@ -830,8 +831,11 @@ func (h *Handler) handleCreateDBInstance(vals url.Values) (any, error) { func (h *Handler) handleDeleteDBInstance(vals url.Values) (any, error) { id := vals.Get("DBInstanceIdentifier") + skipFinalSnapshot := vals.Get("SkipFinalSnapshot") == formTrue + finalSnapshotID := vals.Get("FinalDBSnapshotIdentifier") + deleteAutomatedBackups := vals.Get("DeleteAutomatedBackups") != "false" - inst, err := h.Backend.DeleteDBInstance(id) + inst, err := h.Backend.DeleteDBInstanceWithOptions(id, skipFinalSnapshot, finalSnapshotID, deleteAutomatedBackups) if err != nil { return nil, err } @@ -842,12 +846,104 @@ func (h *Handler) handleDeleteDBInstance(vals url.Values) (any, error) { }, nil } +// isKnownDBInstanceFilterName reports whether name is a Filters.Filter.N.Name +// value AWS recognizes for DescribeDBInstances. "domain" is accepted by AWS +// but has no meaningful analog in this emulator (Directory Service domain +// membership), so it is not implemented as a match predicate — an instance +// passes it vacuously. +func isKnownDBInstanceFilterName(name string) bool { + switch name { + case "db-cluster-id", "db-instance-id", "dbi-resource-id", "domain", "engine": + return true + default: + return false + } +} + +// applyDBInstanceFilters narrows instances per the AWS DescribeDBInstances +// Filters contract: each filter ANDs together, and a filter's Values list is +// OR-matched against the corresponding instance field. An unrecognized filter +// name returns InvalidParameterValue, matching real AWS. +func applyDBInstanceFilters(vals url.Values, instances []DBInstance) ([]DBInstance, error) { + filters := parseDescribeFilters(vals) + if len(filters) == 0 { + return instances, nil + } + + for name := range filters { + if !isKnownDBInstanceFilterName(name) { + return nil, fmt.Errorf("%w: Unrecognized filter name: %s", ErrInvalidParameter, name) + } + } + + filtered := make([]DBInstance, 0, len(instances)) + for _, inst := range instances { + if matchesAllDBInstanceFilters(inst, filters) { + filtered = append(filtered, inst) + } + } + + return filtered, nil +} + +func matchesAllDBInstanceFilters(inst DBInstance, filters map[string][]string) bool { + for name, values := range filters { + switch name { + case "db-cluster-id": + if !slices.Contains(values, inst.DBClusterIdentifier) { + return false + } + case "db-instance-id": + if !slices.Contains(values, inst.DBInstanceIdentifier) { + return false + } + case "dbi-resource-id": + if !slices.Contains(values, inst.DbiResourceID) { + return false + } + case "engine": + if !slices.Contains(values, inst.Engine) { + return false + } + case "domain": + // No domain-membership state is modeled; accept unconditionally. + } + } + + return true +} + +// parseDescribeFilters parses the AWS query-protocol "Filters.Filter.N.Name" / +// "Filters.Filter.N.Values.member.M" parameters into a filter-name -> values map. +func parseDescribeFilters(vals url.Values) map[string][]string { + filters := make(map[string][]string) + for i := 1; ; i++ { + name := vals.Get(fmt.Sprintf("Filters.Filter.%d.Name", i)) + if name == "" { + return filters + } + var values []string + for j := 1; ; j++ { + v := vals.Get(fmt.Sprintf("Filters.Filter.%d.Values.member.%d", i, j)) + if v == "" { + break + } + values = append(values, v) + } + filters[name] = values + } +} + func (h *Handler) handleDescribeDBInstances(vals url.Values) (any, error) { id := vals.Get("DBInstanceIdentifier") instances, err := h.Backend.DescribeDBInstances(id) if err != nil { return nil, err } + instances, err = applyDBInstanceFilters(vals, instances) + if err != nil { + return nil, err + } members, marker, err := paginateDescribe(vals, instances, func(a, b DBInstance) bool { return a.DBInstanceIdentifier < b.DBInstanceIdentifier }, func(item DBInstance) xmlDBInstance { @@ -1982,7 +2078,10 @@ func (h *Handler) handleDescribeDBClusters(vals url.Values) (any, error) { func (h *Handler) handleDeleteDBCluster(vals url.Values) (any, error) { id := vals.Get("DBClusterIdentifier") - cluster, err := h.Backend.DeleteDBCluster(id) + skipFinalSnapshot := vals.Get("SkipFinalSnapshot") == formTrue + finalSnapshotID := vals.Get("FinalDBSnapshotIdentifier") + + cluster, err := h.Backend.DeleteDBClusterWithOptions(id, skipFinalSnapshot, finalSnapshotID) if err != nil { return nil, err } diff --git a/services/rds/handler_test.go b/services/rds/handler_test.go index 45c57ee01..9f687dc2b 100644 --- a/services/rds/handler_test.go +++ b/services/rds/handler_test.go @@ -233,17 +233,49 @@ func TestRDSHandler_FormActions(t *testing.T) { wantContains: []string{"DBInstanceAlreadyExists"}, }, { - name: "DeleteDBInstance", - setupBodies: []string{"Action=CreateDBInstance&Version=2014-10-31&DBInstanceIdentifier=del-db"}, - body: "Action=DeleteDBInstance&Version=2014-10-31&DBInstanceIdentifier=del-db", + name: "DeleteDBInstance", + setupBodies: []string{"Action=CreateDBInstance&Version=2014-10-31&DBInstanceIdentifier=del-db"}, + body: "Action=DeleteDBInstance&Version=2014-10-31&DBInstanceIdentifier=del-db" + + "&SkipFinalSnapshot=true", wantCode: http.StatusOK, wantContains: []string{"DeleteDBInstanceResponse", "del-db"}, }, { - name: "DeleteDBInstance_NotFound", - body: "Action=DeleteDBInstance&Version=2014-10-31&DBInstanceIdentifier=nonexistent", + name: "DeleteDBInstance_NotFound", + body: "Action=DeleteDBInstance&Version=2014-10-31&DBInstanceIdentifier=nonexistent", + wantCode: http.StatusBadRequest, + wantContains: []string{ + "DBInstanceNotFound", + }, + }, + { + name: "DeleteDBInstance_MissingSkipFinalSnapshotAndFinalSnapshotID", + setupBodies: []string{ + "Action=CreateDBInstance&Version=2014-10-31&DBInstanceIdentifier=del-db-nosnap", + }, + body: "Action=DeleteDBInstance&Version=2014-10-31&DBInstanceIdentifier=del-db-nosnap", wantCode: http.StatusBadRequest, - wantContains: []string{"DBInstanceNotFound"}, + wantContains: []string{"InvalidParameterCombination", "FinalDBSnapshotIdentifier"}, + }, + { + name: "DeleteDBInstance_FinalSnapshotIDWithSkipFinalSnapshot", + setupBodies: []string{ + "Action=CreateDBInstance&Version=2014-10-31&DBInstanceIdentifier=del-db-badcombo", + }, + body: "Action=DeleteDBInstance&Version=2014-10-31&DBInstanceIdentifier=del-db-badcombo" + + "&SkipFinalSnapshot=true&FinalDBSnapshotIdentifier=del-db-badcombo-final", + wantCode: http.StatusBadRequest, + wantContains: []string{"InvalidParameterCombination", "FinalDBSnapshotIdentifier"}, + }, + { + name: "DeleteDBInstance_WithFinalSnapshot", + setupBodies: []string{ + "Action=CreateDBInstance&Version=2014-10-31&DBInstanceIdentifier=del-db-finalsnap", + }, + body: "Action=DeleteDBInstance&Version=2014-10-31&DBInstanceIdentifier=del-db-finalsnap" + + "&FinalDBSnapshotIdentifier=del-db-finalsnap-final", + wantCode: http.StatusOK, + wantContains: []string{"DeleteDBInstanceResponse", "del-db-finalsnap"}, }, { name: "DescribeDBInstances", @@ -629,7 +661,8 @@ func TestRDSHandler_FormActions(t *testing.T) { setupBodies: []string{ "Action=CreateDBCluster&Version=2014-10-31&DBClusterIdentifier=del-cluster&Engine=aurora-postgresql", }, - body: "Action=DeleteDBCluster&Version=2014-10-31&DBClusterIdentifier=del-cluster", + body: "Action=DeleteDBCluster&Version=2014-10-31&DBClusterIdentifier=del-cluster" + + "&SkipFinalSnapshot=true", wantCode: http.StatusOK, wantContains: []string{"DeleteDBClusterResponse", "del-cluster"}, }, @@ -639,6 +672,27 @@ func TestRDSHandler_FormActions(t *testing.T) { wantCode: http.StatusBadRequest, wantContains: []string{"DBClusterNotFound"}, }, + { + name: "DeleteDBCluster_MissingSkipFinalSnapshotAndFinalSnapshotID", + setupBodies: []string{ + "Action=CreateDBCluster&Version=2014-10-31&DBClusterIdentifier=del-cluster-nosnap" + + "&Engine=aurora-postgresql", + }, + body: "Action=DeleteDBCluster&Version=2014-10-31&DBClusterIdentifier=del-cluster-nosnap", + wantCode: http.StatusBadRequest, + wantContains: []string{"InvalidParameterCombination", "FinalDBSnapshotIdentifier"}, + }, + { + name: "DeleteDBCluster_WithFinalSnapshot", + setupBodies: []string{ + "Action=CreateDBCluster&Version=2014-10-31&DBClusterIdentifier=del-cluster-finalsnap" + + "&Engine=aurora-postgresql", + }, + body: "Action=DeleteDBCluster&Version=2014-10-31&DBClusterIdentifier=del-cluster-finalsnap" + + "&FinalDBSnapshotIdentifier=del-cluster-finalsnap-final", + wantCode: http.StatusOK, + wantContains: []string{"DeleteDBClusterResponse", "del-cluster-finalsnap"}, + }, { name: "ModifyDBCluster", setupBodies: []string{ @@ -1277,7 +1331,7 @@ func TestRDSBackend_TagsCleanedUpOnDelete(t *testing.T) { "&Tags.Tag.1.Key=k&Tags.Tag.1.Value=v") }, del: "Action=DeleteDBInstance&Version=2014-10-31" + - "&DBInstanceIdentifier=tag-inst", + "&DBInstanceIdentifier=tag-inst&SkipFinalSnapshot=true", check: "Action=ListTagsForResource&Version=2014-10-31" + "&ResourceName=arn:aws:rds:us-east-1:000000000000:db:tag-inst", }, @@ -1362,7 +1416,7 @@ func TestRDSBackend_TagsCleanedUpOnDelete(t *testing.T) { "&Tags.Tag.1.Key=k&Tags.Tag.1.Value=v") }, del: "Action=DeleteDBCluster&Version=2014-10-31" + - "&DBClusterIdentifier=tag-cluster", + "&DBClusterIdentifier=tag-cluster&SkipFinalSnapshot=true", check: "Action=ListTagsForResource&Version=2014-10-31" + "&ResourceName=arn:aws:rds:us-east-1:000000000000:cluster:tag-cluster", }, diff --git a/services/rds/interfaces.go b/services/rds/interfaces.go index c4bf28228..5556b7867 100644 --- a/services/rds/interfaces.go +++ b/services/rds/interfaces.go @@ -22,6 +22,12 @@ type StorageBackend interface { opts DBInstanceOptions, ) (*DBInstance, error) DeleteDBInstance(id string) (*DBInstance, error) + DeleteDBInstanceWithOptions( + id string, + skipFinalSnapshot bool, + finalSnapshotID string, + deleteAutomatedBackups bool, + ) (*DBInstance, error) DescribeDBInstances(id string) ([]DBInstance, error) ModifyDBInstance( id, instanceClass string, @@ -88,6 +94,7 @@ type StorageBackend interface { ) (*DBCluster, error) DescribeDBClusters(id string) ([]DBCluster, error) DeleteDBCluster(id string) (*DBCluster, error) + DeleteDBClusterWithOptions(id string, skipFinalSnapshot bool, finalSnapshotID string) (*DBCluster, error) ModifyDBCluster(id, paramGroupName string, opts DBClusterOptions) (*DBCluster, error) StartDBCluster(id string) (*DBCluster, error) StopDBCluster(id string) (*DBCluster, error) diff --git a/services/rds/persistence.go b/services/rds/persistence.go index 5b6a173b1..fd8c3ca32 100644 --- a/services/rds/persistence.go +++ b/services/rds/persistence.go @@ -3,52 +3,39 @@ package rds import ( "context" "encoding/json" + "fmt" "time" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" ) +// rdsSnapshotVersion identifies the shape of backendSnapshot's Tables blob +// (i.e. the set/shape of resources registered on b.registry -- see +// registerAllTables in store_setup.go). It must be bumped whenever a change +// there would make an older snapshot unsafe to decode as the current shape. +// Restore compares this against the persisted value and discards (rather than +// attempts to partially decode) any mismatch -- see Restore below. This +// mirrors the services/ec2 (commit 12e611a4) and services/sqs (commit +// 0f09d77c) conversions. +const rdsSnapshotVersion = 1 + type backendSnapshot struct { - Instances map[string]*DBInstance `json:"instances"` - Snapshots map[string]*DBSnapshot `json:"snapshots"` - SubnetGroups map[string]*DBSubnetGroup `json:"subnetGroups"` - Tags map[string][]Tag `json:"tags"` - ParameterGroups map[string]*DBParameterGroup `json:"parameterGroups"` - ClusterParameterGroups map[string]*DBParameterGroup `json:"clusterParameterGroups"` - OptionGroups map[string]*OptionGroup `json:"optionGroups"` - Clusters map[string]*DBCluster `json:"clusters"` - ClusterSnapshots map[string]*DBClusterSnapshot `json:"clusterSnapshots"` - ClusterEndpoints map[string]*DBClusterEndpoint `json:"clusterEndpoints"` - ExportTasks map[string]*ExportTask `json:"exportTasks"` - GlobalClusters map[string]*GlobalCluster `json:"globalClusters"` - ClusterRoles map[string][]string `json:"clusterRoles"` - InstanceRoles map[string][]string `json:"instanceRoles"` - EventSubscriptions map[string]*EventSubscription `json:"eventSubscriptions"` - DBSecurityGroups map[string]*DBSecurityGroup `json:"dbSecurityGroups"` - BlueGreenDeployments map[string]*BlueGreenDeployment `json:"blueGreenDeployments"` - SnapshotAttributes map[string]*DBSnapshotAttributesResult `json:"snapshotAttributes"` - ClusterSnapshotAttributes map[string]*DBClusterSnapshotAttributesResult `json:"clusterSnapshotAttributes"` - ReservedInstances map[string]*ReservedDBInstance `json:"reservedInstances"` - Recommendations map[string]*DBRecommendation `json:"recommendations"` - Proxies map[string]*DBProxy `json:"proxies"` - ProxyTargetGroups map[string]*DBProxyTargetGroup `json:"proxyTargetGroups"` - ProxyTargets map[string][]DBProxyTarget `json:"proxyTargets"` - ProxyEndpoints map[string]*DBProxyEndpoint `json:"proxyEndpoints"` - InstanceReadyAt map[string]time.Time `json:"instanceReadyAt"` - ClusterReadyAt map[string]time.Time `json:"clusterReadyAt"` - CustomEngineVersions map[string]*CustomDBEngineVersion `json:"customEngineVersions"` - AutomatedBackups map[string]*DBInstanceAutomatedBackup `json:"automatedBackups"` - ShardGroups map[string]*DBShardGroup `json:"shardGroups"` - Integrations map[string]*Integration `json:"integrations"` - TenantDatabases map[string]*TenantDatabase `json:"tenantDatabases"` - ClusterAutomatedBackups map[string]*DBClusterAutomatedBackup `json:"clusterAutomatedBackups"` - SnapshotTenantDatabases map[string][]*DBSnapshotTenantDatabase `json:"snapshotTenantDatabases"` - InstanceLogFiles map[string][]DBLogFile `json:"instanceLogFiles"` - InstanceLogContent map[string]map[string]string `json:"instanceLogContent"` - AccountID string `json:"accountID"` - Region string `json:"region"` - DefaultCACertificateID string `json:"defaultCACertificateID"` + Tables map[string]json.RawMessage `json:"tables"` + Tags map[string][]Tag `json:"tags"` + ClusterRoles map[string][]string `json:"clusterRoles"` + InstanceRoles map[string][]string `json:"instanceRoles"` + ProxyTargets map[string][]DBProxyTarget `json:"proxyTargets"` + InstanceReadyAt map[string]time.Time `json:"instanceReadyAt"` + ClusterReadyAt map[string]time.Time `json:"clusterReadyAt"` + AutomatedBackups map[string]*DBInstanceAutomatedBackup `json:"automatedBackups"` + SnapshotTenantDatabases map[string][]*DBSnapshotTenantDatabase `json:"snapshotTenantDatabases"` + InstanceLogFiles map[string][]DBLogFile `json:"instanceLogFiles"` + InstanceLogContent map[string]map[string]string `json:"instanceLogContent"` + AccountID string `json:"accountID"` + Region string `json:"region"` + DefaultCACertificateID string `json:"defaultCACertificateID"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. @@ -57,46 +44,33 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "rds: snapshot table marshal failed", "error", err) + + return nil + } + snap := backendSnapshot{ - Instances: b.instances, - Snapshots: b.snapshots, - SubnetGroups: b.subnetGroups, - Tags: b.tags, - ParameterGroups: b.parameterGroups, - ClusterParameterGroups: b.clusterParameterGroups, - OptionGroups: b.optionGroups, - Clusters: b.clusters, - ClusterSnapshots: b.clusterSnapshots, - ClusterEndpoints: b.clusterEndpoints, - ExportTasks: b.exportTasks, - GlobalClusters: b.globalClusters, - ClusterRoles: b.clusterRoles, - InstanceRoles: b.instanceRoles, - EventSubscriptions: b.eventSubscriptions, - DBSecurityGroups: b.dbSecurityGroups, - BlueGreenDeployments: b.blueGreenDeployments, - SnapshotAttributes: b.snapshotAttributes, - ClusterSnapshotAttributes: b.clusterSnapshotAttributes, - ReservedInstances: b.reservedInstances, - Recommendations: b.recommendations, - Proxies: b.proxies, - ProxyTargetGroups: b.proxyTargetGroups, - ProxyTargets: b.proxyTargets, - ProxyEndpoints: b.proxyEndpoints, - InstanceReadyAt: b.instanceReadyAt, - ClusterReadyAt: b.clusterReadyAt, - AccountID: b.accountID, - Region: b.region, - CustomEngineVersions: b.customEngineVersions, - AutomatedBackups: b.automatedBackups, - ShardGroups: b.shardGroups, - Integrations: b.integrations, - TenantDatabases: b.tenantDatabases, - ClusterAutomatedBackups: b.clusterAutomatedBackups, - SnapshotTenantDatabases: b.snapshotTenantDatabases, - InstanceLogFiles: b.instanceLogFiles, - InstanceLogContent: b.instanceLogContent, - DefaultCACertificateID: b.defaultCACertificateID, + Version: rdsSnapshotVersion, + Tables: tables, + Tags: b.tags, + ClusterRoles: b.clusterRoles, + InstanceRoles: b.instanceRoles, + ProxyTargets: b.proxyTargets, + InstanceReadyAt: b.instanceReadyAt, + ClusterReadyAt: b.clusterReadyAt, + AutomatedBackups: b.automatedBackups, + SnapshotTenantDatabases: b.snapshotTenantDatabases, + InstanceLogFiles: b.instanceLogFiles, + InstanceLogContent: b.instanceLogContent, + AccountID: b.accountID, + Region: b.region, + DefaultCACertificateID: b.defaultCACertificateID, } data, err := json.Marshal(snap) @@ -123,31 +97,30 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() - b.instances = snap.Instances - b.snapshots = snap.Snapshots - b.subnetGroups = snap.SubnetGroups + if snap.Version != rdsSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/ec2 and services/sqs conversions. + logger.Load(ctx).WarnContext(ctx, + "rds: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", rdsSnapshotVersion) + + b.registry.ResetAll() + + return nil + } + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("rds: restore snapshot tables: %w", err) + } + b.tags = snap.Tags - b.parameterGroups = snap.ParameterGroups - b.clusterParameterGroups = snap.ClusterParameterGroups - b.optionGroups = snap.OptionGroups - b.clusters = snap.Clusters - b.clusterSnapshots = snap.ClusterSnapshots - b.clusterEndpoints = snap.ClusterEndpoints - b.exportTasks = snap.ExportTasks - b.globalClusters = snap.GlobalClusters b.clusterRoles = snap.ClusterRoles b.instanceRoles = snap.InstanceRoles - b.eventSubscriptions = snap.EventSubscriptions - b.dbSecurityGroups = snap.DBSecurityGroups - b.blueGreenDeployments = snap.BlueGreenDeployments - b.snapshotAttributes = snap.SnapshotAttributes - b.clusterSnapshotAttributes = snap.ClusterSnapshotAttributes - b.reservedInstances = snap.ReservedInstances - b.recommendations = snap.Recommendations - b.proxies = snap.Proxies - b.proxyTargetGroups = snap.ProxyTargetGroups b.proxyTargets = snap.ProxyTargets - b.proxyEndpoints = snap.ProxyEndpoints b.instanceReadyAt = snap.InstanceReadyAt if snap.ClusterReadyAt != nil { b.clusterReadyAt = snap.ClusterReadyAt @@ -156,12 +129,7 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { } b.accountID = snap.AccountID b.region = snap.Region - b.customEngineVersions = snap.CustomEngineVersions b.automatedBackups = snap.AutomatedBackups - b.shardGroups = snap.ShardGroups - b.integrations = snap.Integrations - b.tenantDatabases = snap.TenantDatabases - b.clusterAutomatedBackups = snap.ClusterAutomatedBackups b.snapshotTenantDatabases = snap.SnapshotTenantDatabases b.instanceLogFiles = snap.InstanceLogFiles b.instanceLogContent = snap.InstanceLogContent @@ -179,71 +147,10 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { // ensureNonNilMaps initialises any nil maps in a deserialized snapshot so the backend // never operates on nil maps after a restore. func ensureNonNilMaps(snap *backendSnapshot) { - ensureNonNilCoreMaps(snap) - ensureNonNilExtendedMaps(snap) - ensureNonNilBatch1Maps(snap) -} - -// ensureNonNilCoreMaps initialises the core resource maps (instances, snapshots, subnet groups, tags, -// parameter groups, option groups, clusters, cluster snapshots, cluster endpoints, export tasks, global clusters). -func ensureNonNilCoreMaps(snap *backendSnapshot) { - if snap.Instances == nil { - snap.Instances = make(map[string]*DBInstance) - } - - if snap.Snapshots == nil { - snap.Snapshots = make(map[string]*DBSnapshot) - } - - if snap.SubnetGroups == nil { - snap.SubnetGroups = make(map[string]*DBSubnetGroup) - } - if snap.Tags == nil { snap.Tags = make(map[string][]Tag) } - if snap.ParameterGroups == nil { - snap.ParameterGroups = make(map[string]*DBParameterGroup) - } - - if snap.ClusterParameterGroups == nil { - snap.ClusterParameterGroups = make(map[string]*DBParameterGroup) - } - - if snap.OptionGroups == nil { - snap.OptionGroups = make(map[string]*OptionGroup) - } - - if snap.Clusters == nil { - snap.Clusters = make(map[string]*DBCluster) - } - - if snap.ClusterSnapshots == nil { - snap.ClusterSnapshots = make(map[string]*DBClusterSnapshot) - } - - if snap.ClusterEndpoints == nil { - snap.ClusterEndpoints = make(map[string]*DBClusterEndpoint) - } - - if snap.ExportTasks == nil { - snap.ExportTasks = make(map[string]*ExportTask) - } - - if snap.GlobalClusters == nil { - snap.GlobalClusters = make(map[string]*GlobalCluster) - } - - if snap.InstanceReadyAt == nil { - snap.InstanceReadyAt = make(map[string]time.Time) - } -} - -// ensureNonNilExtendedMaps initialises the extended resource maps added later -// (cluster/instance roles, event subscriptions, security groups, blue/green deployments, -// snapshot attributes, reserved instances, and recommendations). -func ensureNonNilExtendedMaps(snap *backendSnapshot) { if snap.ClusterRoles == nil { snap.ClusterRoles = make(map[string][]string) } @@ -252,79 +159,18 @@ func ensureNonNilExtendedMaps(snap *backendSnapshot) { snap.InstanceRoles = make(map[string][]string) } - if snap.EventSubscriptions == nil { - snap.EventSubscriptions = make(map[string]*EventSubscription) - } - - if snap.DBSecurityGroups == nil { - snap.DBSecurityGroups = make(map[string]*DBSecurityGroup) - } - - if snap.BlueGreenDeployments == nil { - snap.BlueGreenDeployments = make(map[string]*BlueGreenDeployment) - } - - if snap.SnapshotAttributes == nil { - snap.SnapshotAttributes = make(map[string]*DBSnapshotAttributesResult) - } - - if snap.ClusterSnapshotAttributes == nil { - snap.ClusterSnapshotAttributes = make(map[string]*DBClusterSnapshotAttributesResult) - } - - if snap.ReservedInstances == nil { - snap.ReservedInstances = make(map[string]*ReservedDBInstance) - } - - if snap.Recommendations == nil { - snap.Recommendations = make(map[string]*DBRecommendation) - } - - if snap.Proxies == nil { - snap.Proxies = make(map[string]*DBProxy) - } - - if snap.ProxyTargetGroups == nil { - snap.ProxyTargetGroups = make(map[string]*DBProxyTargetGroup) - } - if snap.ProxyTargets == nil { snap.ProxyTargets = make(map[string][]DBProxyTarget) } - if snap.ProxyEndpoints == nil { - snap.ProxyEndpoints = make(map[string]*DBProxyEndpoint) - } -} - -// ensureNonNilBatch1Maps initialises the batch-1 resource maps added in the first -// stateful-ops batch (custom engine versions, shard groups, integrations, tenant -// databases, cluster/instance automated backups, and snapshot tenant databases). -func ensureNonNilBatch1Maps(snap *backendSnapshot) { - if snap.CustomEngineVersions == nil { - snap.CustomEngineVersions = make(map[string]*CustomDBEngineVersion) + if snap.InstanceReadyAt == nil { + snap.InstanceReadyAt = make(map[string]time.Time) } if snap.AutomatedBackups == nil { snap.AutomatedBackups = make(map[string]*DBInstanceAutomatedBackup) } - if snap.ShardGroups == nil { - snap.ShardGroups = make(map[string]*DBShardGroup) - } - - if snap.Integrations == nil { - snap.Integrations = make(map[string]*Integration) - } - - if snap.TenantDatabases == nil { - snap.TenantDatabases = make(map[string]*TenantDatabase) - } - - if snap.ClusterAutomatedBackups == nil { - snap.ClusterAutomatedBackups = make(map[string]*DBClusterAutomatedBackup) - } - if snap.SnapshotTenantDatabases == nil { snap.SnapshotTenantDatabases = make(map[string][]*DBSnapshotTenantDatabase) } @@ -372,9 +218,9 @@ func (h *Handler) Restore(ctx context.Context, data []byte) error { } // Check if this might be a single-region snapshot that happens to unmarshal as map[string]json.RawMessage. - // Since backendSnapshot has "instances", "snapshots", etc. as keys, they might match. - // We can check if "instances" exists as a key. - if _, hasInstances := snaps["instances"]; hasInstances { + // Since backendSnapshot has "tables", "tags", etc. as keys, they might match. + // We can check if "tables" exists as a key. + if _, hasTables := snaps["tables"]; hasTables { return h.Backend.Restore(ctx, data) } diff --git a/services/rds/persistence_test.go b/services/rds/persistence_test.go index c268f578f..1dbc5b660 100644 --- a/services/rds/persistence_test.go +++ b/services/rds/persistence_test.go @@ -181,3 +181,190 @@ func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { err := b.Restore(t.Context(), []byte("not-valid-json")) require.Error(t, err) } + +// TestInMemoryBackend_FullStateSnapshotRestoreRoundTrip exercises the Phase +// 3.3 pkgs/store conversion end-to-end: it populates one entry in every +// store.Table-backed resource collection (see registerAllTables in +// store_setup.go) plus a representative sample of the plain maps that were +// deliberately left unconverted, snapshots the backend, restores into a fresh +// one, and asserts every resource is still there. This is the constraint-4 +// full-state round-trip test called for by the datalayer refactor. +func TestInMemoryBackend_FullStateSnapshotRestoreRoundTrip(t *testing.T) { + t.Parallel() + + b := rds.NewInMemoryBackend("000000000000", "us-east-1") + ctx := t.Context() + + // instances / subnetGroups / parameterGroups / optionGroups + _, err := b.CreateDBSubnetGroup("sn1", "desc", "vpc1", []string{"subnet-1"}) + require.NoError(t, err) + _, err = b.CreateDBParameterGroup("pg1", "postgres14", "desc") + require.NoError(t, err) + _, err = b.CreateOptionGroup("og1", "mysql", "8.0", "desc") + require.NoError(t, err) + inst, err := b.CreateDBInstance("inst1", "mysql", "db.t3.micro", "db", "admin", "pg1", 20, rds.DBInstanceOptions{}) + require.NoError(t, err) + + // snapshots + _, err = b.CreateDBSnapshot("snap1", inst.DBInstanceIdentifier) + require.NoError(t, err) + _, err = b.ModifyDBSnapshotAttribute("snap1", "restore", []string{"all"}, nil) + require.NoError(t, err) + + // clusters / clusterParameterGroups / clusterSnapshots / clusterEndpoints / clusterAutomatedBackups + _, err = b.CreateDBClusterParameterGroup("cpg1", "aurora-postgresql14", "desc") + require.NoError(t, err) + cluster, err := b.CreateDBCluster("cl1", "aurora-postgresql", "admin", "db", "", 1, nil, rds.DBClusterOptions{}) + require.NoError(t, err) + _, err = b.CreateDBClusterSnapshot("csnap1", cluster.DBClusterIdentifier) + require.NoError(t, err) + _, err = b.ModifyDBClusterSnapshotAttribute("csnap1", "restore", []string{"all"}, nil) + require.NoError(t, err) + _, err = b.CreateDBClusterEndpoint("cep1", cluster.DBClusterIdentifier, "READER") + require.NoError(t, err) + require.NotNil(t, b.CreateDBClusterAutomatedBackup(cluster.DBClusterIdentifier)) + + // exportTasks / globalClusters / eventSubscriptions / dbSecurityGroups / blueGreenDeployments + _, err = b.StartExportTask("exp1", "arn:aws:rds:us-east-1:000000000000:snapshot:snap1", "bucket1") + require.NoError(t, err) + _, err = b.CreateGlobalCluster("gc1", "aurora-postgresql", "14.6", false, false) + require.NoError(t, err) + _, err = b.CreateEventSubscription("evsub1", "arn:sns:topic", "db-instance", nil, nil) + require.NoError(t, err) + _, err = b.CreateDBSecurityGroup("sg1", "desc") + require.NoError(t, err) + _, err = b.CreateBlueGreenDeployment("bgd1", "inst1") + require.NoError(t, err) + + // reservedInstances / recommendations / proxies / proxyTargetGroups / proxyEndpoints / customEngineVersions + _, err = b.PurchaseReservedDBInstancesOffering("offering1", "ri1", 1) + require.NoError(t, err) + _, err = b.CreateDBProxy("proxy1", "MYSQL", "arn:aws:iam::000000000000:role/proxy", nil) + require.NoError(t, err) + _, err = b.CreateDBProxyEndpoint("proxy1", "proxyep1", "", nil, nil) + require.NoError(t, err) + _, err = b.CreateCustomDBEngineVersion("custom-mysql", "8.0.35.custom", "desc") + require.NoError(t, err) + + // shardGroups / integrations / tenantDatabases + _, err = b.CreateDBShardGroup("shard1", cluster.DBClusterIdentifier, 16, 2, 0, false) + require.NoError(t, err) + _, err = b.CreateIntegration("intg1", "arn:src", "arn:dst", "", "", "") + require.NoError(t, err) + _, err = b.CreateTenantDatabase("inst1", "tenant1", "admin") + require.NoError(t, err) + + // representative sample of maps deliberately left raw (see store_setup.go) + require.NoError(t, b.AddRoleToDBCluster(cluster.DBClusterIdentifier, "arn:aws:iam::000000000000:role/cluster")) + require.NoError(t, b.AddRoleToDBInstance("inst1", "arn:aws:iam::000000000000:role/instance")) + b.AddDBSnapshotTenantDatabase("snap1", "inst1", "tenant1", "mysql") + + snap := b.Snapshot(ctx) + require.NotNil(t, snap) + + fresh := rds.NewInMemoryBackend("000000000000", "us-east-1") + require.NoError(t, fresh.Restore(ctx, snap)) + + instances, err := fresh.DescribeDBInstances("") + require.NoError(t, err) + assert.Len(t, instances, 1) + + subnetGroups, err := fresh.DescribeDBSubnetGroups("") + require.NoError(t, err) + assert.Len(t, subnetGroups, 1) + + paramGroups, err := fresh.DescribeDBParameterGroups("") + require.NoError(t, err) + assert.Len(t, paramGroups, 1) + + optionGroups, err := fresh.DescribeOptionGroups("") + require.NoError(t, err) + assert.Len(t, optionGroups, 1) + + snapshots, err := fresh.DescribeDBSnapshots("", "") + require.NoError(t, err) + assert.Len(t, snapshots, 1) + + snapAttrs, err := fresh.DescribeDBSnapshotAttributes("snap1") + require.NoError(t, err) + assert.Len(t, snapAttrs.DBSnapshotAttributes, 1) + + clusterParamGroups, err := fresh.DescribeDBClusterParameterGroups("") + require.NoError(t, err) + assert.Len(t, clusterParamGroups, 1) + + clusters, err := fresh.DescribeDBClusters("") + require.NoError(t, err) + assert.Len(t, clusters, 1) + + clusterSnaps, err := fresh.DescribeDBClusterSnapshots("", "") + require.NoError(t, err) + assert.Len(t, clusterSnaps, 1) + + clusterSnapAttrs, err := fresh.DescribeDBClusterSnapshotAttributes("csnap1") + require.NoError(t, err) + assert.Len(t, clusterSnapAttrs.DBClusterSnapshotAttributes, 1) + + clusterEndpoints, err := fresh.DescribeDBClusterEndpoints("", "") + require.NoError(t, err) + assert.Len(t, clusterEndpoints, 1) + + clusterBackups := fresh.DescribeDBClusterAutomatedBackups("") + assert.Len(t, clusterBackups, 1) + + exportTasks, err := fresh.DescribeExportTasks("") + require.NoError(t, err) + assert.Len(t, exportTasks, 1) + + globalClusters, err := fresh.DescribeGlobalClusters("") + require.NoError(t, err) + assert.Len(t, globalClusters, 1) + + eventSubs, err := fresh.DescribeEventSubscriptions("") + require.NoError(t, err) + assert.Len(t, eventSubs, 1) + + secGroups, err := fresh.DescribeDBSecurityGroups("") + require.NoError(t, err) + assert.Len(t, secGroups, 1) + + bgds, err := fresh.DescribeBlueGreenDeployments("") + require.NoError(t, err) + assert.Len(t, bgds, 1) + + reserved := fresh.DescribeReservedDBInstances("", "") + assert.Len(t, reserved, 1) + + proxies, err := fresh.DescribeDBProxies("") + require.NoError(t, err) + assert.Len(t, proxies, 1) + + targetGroups, err := fresh.DescribeDBProxyTargetGroups("proxy1", "") + require.NoError(t, err) + assert.Len(t, targetGroups, 1) + + proxyEndpoints, err := fresh.DescribeDBProxyEndpoints("", "") + require.NoError(t, err) + assert.Len(t, proxyEndpoints, 1) + + customVersions := fresh.DescribeCustomDBEngineVersions("", "") + assert.Len(t, customVersions, 1) + + shardGroups, err := fresh.DescribeDBShardGroups("") + require.NoError(t, err) + assert.Len(t, shardGroups, 1) + + integrations, err := fresh.DescribeIntegrations("") + require.NoError(t, err) + assert.Len(t, integrations, 1) + + tenants, err := fresh.DescribeTenantDatabases("", "") + require.NoError(t, err) + assert.Len(t, tenants, 1) + + // representative raw-map sample (see store_setup.go's exclusion list) + assert.Equal(t, 1, rds.ClusterRoleCount(fresh, cluster.DBClusterIdentifier)) + assert.Equal(t, 1, rds.InstanceRoleCount(fresh, "inst1")) + snapTenants := fresh.DescribeDBSnapshotTenantDatabases("snap1", "") + assert.Len(t, snapTenants, 1) +} diff --git a/services/rds/refinement1.go b/services/rds/refinement1.go index d86f20f05..5dfc6c745 100644 --- a/services/rds/refinement1.go +++ b/services/rds/refinement1.go @@ -16,7 +16,7 @@ func (b *InMemoryBackend) CreateEventSubscription( } b.mu.Lock("CreateEventSubscription") defer b.mu.Unlock() - if _, exists := b.eventSubscriptions[name]; exists { + if _, exists := b.eventSubscriptions.Get(name); exists { return nil, fmt.Errorf("%w: subscription %s already exists", ErrEventSubscriptionAlreadyExists, name) } ids := make([]string, len(sourceIDs)) @@ -32,7 +32,7 @@ func (b *InMemoryBackend) CreateEventSubscription( EventCategories: cats, Enabled: true, } - b.eventSubscriptions[name] = sub + b.eventSubscriptions.Put(sub) cp := *sub return &cp, nil @@ -42,12 +42,12 @@ func (b *InMemoryBackend) CreateEventSubscription( func (b *InMemoryBackend) DeleteEventSubscription(name string) (*EventSubscription, error) { b.mu.Lock("DeleteEventSubscription") defer b.mu.Unlock() - sub, exists := b.eventSubscriptions[name] + sub, exists := b.eventSubscriptions.Get(name) if !exists { return nil, fmt.Errorf("%w: subscription %s not found", ErrEventSubscriptionNotFound, name) } cp := *sub - delete(b.eventSubscriptions, name) + b.eventSubscriptions.Delete(name) return &cp, nil } @@ -57,7 +57,7 @@ func (b *InMemoryBackend) DescribeEventSubscriptions(name string) ([]EventSubscr b.mu.RLock("DescribeEventSubscriptions") defer b.mu.RUnlock() if name != "" { - sub, exists := b.eventSubscriptions[name] + sub, exists := b.eventSubscriptions.Get(name) if !exists { return nil, fmt.Errorf("%w: subscription %s not found", ErrEventSubscriptionNotFound, name) } @@ -65,8 +65,8 @@ func (b *InMemoryBackend) DescribeEventSubscriptions(name string) ([]EventSubscr return []EventSubscription{cp}, nil } - result := make([]EventSubscription, 0, len(b.eventSubscriptions)) - for _, sub := range b.eventSubscriptions { + result := make([]EventSubscription, 0, b.eventSubscriptions.Len()) + for _, sub := range b.eventSubscriptions.All() { result = append(result, *sub) } @@ -81,7 +81,7 @@ func (b *InMemoryBackend) ModifyEventSubscription( ) (*EventSubscription, error) { b.mu.Lock("ModifyEventSubscription") defer b.mu.Unlock() - sub, exists := b.eventSubscriptions[name] + sub, exists := b.eventSubscriptions.Get(name) if !exists { return nil, fmt.Errorf("%w: subscription %s not found", ErrEventSubscriptionNotFound, name) } @@ -167,7 +167,7 @@ func (b *InMemoryBackend) DescribeBlueGreenDeployments(id string) ([]BlueGreenDe b.mu.RLock("DescribeBlueGreenDeployments") defer b.mu.RUnlock() if id != "" { - d, exists := b.blueGreenDeployments[id] + d, exists := b.blueGreenDeployments.Get(id) if !exists { return nil, fmt.Errorf("%w: blue/green deployment %s not found", ErrBlueGreenDeploymentNotFound, id) } @@ -175,8 +175,8 @@ func (b *InMemoryBackend) DescribeBlueGreenDeployments(id string) ([]BlueGreenDe return []BlueGreenDeployment{cp}, nil } - result := make([]BlueGreenDeployment, 0, len(b.blueGreenDeployments)) - for _, d := range b.blueGreenDeployments { + result := make([]BlueGreenDeployment, 0, b.blueGreenDeployments.Len()) + for _, d := range b.blueGreenDeployments.All() { result = append(result, *d) } @@ -187,12 +187,12 @@ func (b *InMemoryBackend) DescribeBlueGreenDeployments(id string) ([]BlueGreenDe func (b *InMemoryBackend) DeleteBlueGreenDeployment(id string) (*BlueGreenDeployment, error) { b.mu.Lock("DeleteBlueGreenDeployment") defer b.mu.Unlock() - d, exists := b.blueGreenDeployments[id] + d, exists := b.blueGreenDeployments.Get(id) if !exists { return nil, fmt.Errorf("%w: blue/green deployment %s not found", ErrBlueGreenDeploymentNotFound, id) } cp := *d - delete(b.blueGreenDeployments, id) + b.blueGreenDeployments.Delete(id) return &cp, nil } @@ -201,7 +201,7 @@ func (b *InMemoryBackend) DeleteBlueGreenDeployment(id string) (*BlueGreenDeploy func (b *InMemoryBackend) SwitchoverBlueGreenDeployment(id string) (*BlueGreenDeployment, error) { b.mu.Lock("SwitchoverBlueGreenDeployment") defer b.mu.Unlock() - d, exists := b.blueGreenDeployments[id] + d, exists := b.blueGreenDeployments.Get(id) if !exists { return nil, fmt.Errorf("%w: blue/green deployment %s not found", ErrBlueGreenDeploymentNotFound, id) } @@ -216,7 +216,7 @@ func (b *InMemoryBackend) DescribeDBSecurityGroups(name string) ([]DBSecurityGro b.mu.RLock("DescribeDBSecurityGroups") defer b.mu.RUnlock() if name != "" { - sg, exists := b.dbSecurityGroups[name] + sg, exists := b.dbSecurityGroups.Get(name) if !exists { return nil, fmt.Errorf("%w: security group %s not found", ErrDBSecurityGroupNotFound, name) } @@ -224,8 +224,8 @@ func (b *InMemoryBackend) DescribeDBSecurityGroups(name string) ([]DBSecurityGro return []DBSecurityGroup{cp}, nil } - result := make([]DBSecurityGroup, 0, len(b.dbSecurityGroups)) - for _, sg := range b.dbSecurityGroups { + result := make([]DBSecurityGroup, 0, b.dbSecurityGroups.Len()) + for _, sg := range b.dbSecurityGroups.All() { result = append(result, *sg) } @@ -239,10 +239,10 @@ func (b *InMemoryBackend) DeleteDBSecurityGroup(name string) error { } b.mu.Lock("DeleteDBSecurityGroup") defer b.mu.Unlock() - if _, exists := b.dbSecurityGroups[name]; !exists { + if _, exists := b.dbSecurityGroups.Get(name); !exists { return fmt.Errorf("%w: security group %s not found", ErrDBSecurityGroupNotFound, name) } - delete(b.dbSecurityGroups, name) + b.dbSecurityGroups.Delete(name) return nil } @@ -254,7 +254,7 @@ func (b *InMemoryBackend) RevokeDBSecurityGroupIngress(groupName, cidrIP string) } b.mu.Lock("RevokeDBSecurityGroupIngress") defer b.mu.Unlock() - sg, exists := b.dbSecurityGroups[groupName] + sg, exists := b.dbSecurityGroups.Get(groupName) if !exists { return nil, fmt.Errorf("%w: security group %s not found", ErrDBSecurityGroupNotFound, groupName) } @@ -279,7 +279,7 @@ func (b *InMemoryBackend) RevokeDBSecurityGroupIngress(groupName, cidrIP string) func (b *InMemoryBackend) FailoverDBCluster(clusterID, _ string) (*DBCluster, error) { b.mu.Lock("FailoverDBCluster") defer b.mu.Unlock() - cluster, exists := b.clusters[clusterID] + cluster, exists := b.clusters.Get(clusterID) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -298,7 +298,7 @@ func (b *InMemoryBackend) FailoverDBCluster(clusterID, _ string) (*DBCluster, er // The cluster transitions to "rebooting" status and reverts to "available" after a brief delay. func (b *InMemoryBackend) RebootDBCluster(clusterID string) (*DBCluster, error) { b.mu.Lock("RebootDBCluster") - cluster, exists := b.clusters[clusterID] + cluster, exists := b.clusters.Get(clusterID) if !exists { b.mu.Unlock() @@ -325,10 +325,10 @@ func (b *InMemoryBackend) DeleteDBClusterParameterGroup(name string) error { } b.mu.Lock("DeleteDBClusterParameterGroup") defer b.mu.Unlock() - if _, exists := b.clusterParameterGroups[name]; !exists { + if _, exists := b.clusterParameterGroups.Get(name); !exists { return fmt.Errorf("%w: cluster parameter group %s not found", ErrParameterGroupNotFound, name) } - delete(b.clusterParameterGroups, name) + b.clusterParameterGroups.Delete(name) arn := "arn:aws:rds:" + b.region + ":" + b.accountID + ":cluster-pg:" + name delete(b.tags, arn) @@ -339,7 +339,7 @@ func (b *InMemoryBackend) DeleteDBClusterParameterGroup(name string) error { func (b *InMemoryBackend) ModifyDBClusterParameterGroup(name string, params []DBParameter) (string, error) { b.mu.Lock("ModifyDBClusterParameterGroup") defer b.mu.Unlock() - pg, exists := b.clusterParameterGroups[name] + pg, exists := b.clusterParameterGroups.Get(name) if !exists { return "", fmt.Errorf("%w: cluster parameter group %s not found", ErrParameterGroupNotFound, name) } @@ -357,7 +357,7 @@ func (b *InMemoryBackend) ModifyDBClusterParameterGroup(name string, params []DB func (b *InMemoryBackend) DescribeDBClusterParameters(groupName string) ([]DBParameter, error) { b.mu.RLock("DescribeDBClusterParameters") defer b.mu.RUnlock() - pg, exists := b.clusterParameterGroups[groupName] + pg, exists := b.clusterParameterGroups.Get(groupName) if !exists { return nil, fmt.Errorf("%w: cluster parameter group %s not found", ErrParameterGroupNotFound, groupName) } @@ -387,7 +387,7 @@ func (b *InMemoryBackend) ResetDBClusterParameterGroup( ) (string, error) { b.mu.Lock("ResetDBClusterParameterGroup") defer b.mu.Unlock() - pg, exists := b.clusterParameterGroups[groupName] + pg, exists := b.clusterParameterGroups.Get(groupName) if !exists { return "", fmt.Errorf("%w: cluster parameter group %s not found", ErrParameterGroupNotFound, groupName) } @@ -406,7 +406,7 @@ func (b *InMemoryBackend) ResetDBClusterParameterGroup( func (b *InMemoryBackend) ModifyDBSubnetGroup(name, description string, subnetIDs []string) (*DBSubnetGroup, error) { b.mu.Lock("ModifyDBSubnetGroup") defer b.mu.Unlock() - sg, exists := b.subnetGroups[name] + sg, exists := b.subnetGroups.Get(name) if !exists { return nil, fmt.Errorf("%w: subnet group %s not found", ErrSubnetGroupNotFound, name) } @@ -427,7 +427,7 @@ func (b *InMemoryBackend) ModifyDBSubnetGroup(name, description string, subnetID func (b *InMemoryBackend) ModifyDBClusterEndpoint(endpointID, endpointType string) (*DBClusterEndpoint, error) { b.mu.Lock("ModifyDBClusterEndpoint") defer b.mu.Unlock() - ep, exists := b.clusterEndpoints[endpointID] + ep, exists := b.clusterEndpoints.Get(endpointID) if !exists { return nil, fmt.Errorf("%w: cluster endpoint %s not found", ErrClusterEndpointNotFound, endpointID) } diff --git a/services/rds/refinement2.go b/services/rds/refinement2.go index b353dab1a..30ecb9ef5 100644 --- a/services/rds/refinement2.go +++ b/services/rds/refinement2.go @@ -33,14 +33,14 @@ func (b *InMemoryBackend) CreateDBSecurityGroup(name, description string) (*DBSe } b.mu.Lock("CreateDBSecurityGroup") defer b.mu.Unlock() - if _, exists := b.dbSecurityGroups[name]; exists { + if _, exists := b.dbSecurityGroups.Get(name); exists { return nil, fmt.Errorf("%w: %s", ErrDBSecurityGroupAlreadyExists, name) } sg := &DBSecurityGroup{ DBSecurityGroupName: name, DBSecurityGroupDescription: description, } - b.dbSecurityGroups[name] = sg + b.dbSecurityGroups.Put(sg) cp := *sg return &cp, nil @@ -50,7 +50,7 @@ func (b *InMemoryBackend) CreateDBSecurityGroup(name, description string) (*DBSe func (b *InMemoryBackend) RemoveFromGlobalCluster(globalClusterID, dbClusterARN string) (*GlobalCluster, error) { b.mu.Lock("RemoveFromGlobalCluster") defer b.mu.Unlock() - gc, ok := b.globalClusters[globalClusterID] + gc, ok := b.globalClusters.Get(globalClusterID) if !ok { return nil, fmt.Errorf("%w: %s", ErrGlobalClusterNotFound, globalClusterID) } @@ -68,7 +68,7 @@ func (b *InMemoryBackend) FailoverGlobalCluster( ) (*GlobalCluster, error) { b.mu.Lock("FailoverGlobalCluster") defer b.mu.Unlock() - gc, ok := b.globalClusters[globalClusterID] + gc, ok := b.globalClusters.Get(globalClusterID) if !ok { return nil, fmt.Errorf("%w: %s", ErrGlobalClusterNotFound, globalClusterID) } @@ -84,7 +84,7 @@ func (b *InMemoryBackend) SwitchoverGlobalCluster( ) (*GlobalCluster, error) { b.mu.Lock("SwitchoverGlobalCluster") defer b.mu.Unlock() - gc, ok := b.globalClusters[globalClusterID] + gc, ok := b.globalClusters.Get(globalClusterID) if !ok { return nil, fmt.Errorf("%w: %s", ErrGlobalClusterNotFound, globalClusterID) } @@ -98,7 +98,7 @@ func (b *InMemoryBackend) SwitchoverGlobalCluster( func (b *InMemoryBackend) SwitchoverReadReplica(instanceID string) (*DBInstance, error) { b.mu.Lock("SwitchoverReadReplica") defer b.mu.Unlock() - inst, ok := b.instances[instanceID] + inst, ok := b.instances.Get(instanceID) if !ok { return nil, fmt.Errorf("%w: %s", ErrInstanceNotFound, instanceID) } @@ -112,7 +112,7 @@ func (b *InMemoryBackend) SwitchoverReadReplica(instanceID string) (*DBInstance, func (b *InMemoryBackend) PromoteReadReplicaDBCluster(clusterID string) (*DBCluster, error) { b.mu.Lock("PromoteReadReplicaDBCluster") defer b.mu.Unlock() - cluster, ok := b.clusters[clusterID] + cluster, ok := b.clusters.Get(clusterID) if !ok { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, clusterID) } @@ -129,13 +129,13 @@ func (b *InMemoryBackend) DescribeAccountAttributes() []AccountAttribute { return []AccountAttribute{ {AttributeName: "AllocatedStorage", Used: 0, Max: quotaMaxAllocatedStorage}, - {AttributeName: "DBInstances", Used: len(b.instances), Max: quotaMaxDBInstances}, - {AttributeName: "DBClusters", Used: len(b.clusters), Max: quotaMaxDBClusters}, - {AttributeName: "DBParameterGroups", Used: len(b.parameterGroups), Max: quotaMaxDBParameterGroups}, - {AttributeName: "DBSubnetGroups", Used: len(b.subnetGroups), Max: quotaMaxDBSubnetGroups}, - {AttributeName: "DBSnapshots", Used: len(b.snapshots), Max: quotaMaxDBSnapshots}, - {AttributeName: "OptionGroups", Used: len(b.optionGroups), Max: quotaMaxOptionGroups}, - {AttributeName: "ReservedDBInstances", Used: len(b.reservedInstances), Max: quotaMaxReservedInstances}, + {AttributeName: "DBInstances", Used: b.instances.Len(), Max: quotaMaxDBInstances}, + {AttributeName: "DBClusters", Used: b.clusters.Len(), Max: quotaMaxDBClusters}, + {AttributeName: "DBParameterGroups", Used: b.parameterGroups.Len(), Max: quotaMaxDBParameterGroups}, + {AttributeName: "DBSubnetGroups", Used: b.subnetGroups.Len(), Max: quotaMaxDBSubnetGroups}, + {AttributeName: "DBSnapshots", Used: b.snapshots.Len(), Max: quotaMaxDBSnapshots}, + {AttributeName: "OptionGroups", Used: b.optionGroups.Len(), Max: quotaMaxOptionGroups}, + {AttributeName: "ReservedDBInstances", Used: b.reservedInstances.Len(), Max: quotaMaxReservedInstances}, } } @@ -241,10 +241,10 @@ func (b *InMemoryBackend) DescribeEngineDefaultClusterParameters(_ string) []DBP func (b *InMemoryBackend) DescribeDBSnapshotAttributes(snapshotID string) (*DBSnapshotAttributesResult, error) { b.mu.RLock("DescribeDBSnapshotAttributes") defer b.mu.RUnlock() - if _, ok := b.snapshots[snapshotID]; !ok { + if _, ok := b.snapshots.Get(snapshotID); !ok { return nil, fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) } - if attrs, ok := b.snapshotAttributes[snapshotID]; ok { + if attrs, ok := b.snapshotAttributes.Get(snapshotID); ok { cp := *attrs return &cp, nil @@ -262,7 +262,7 @@ func (b *InMemoryBackend) DescribeDBSnapshotAttributes(snapshotID string) (*DBSn func (b *InMemoryBackend) ModifyDBSnapshot(snapshotID, optionGroupName, engineVersion string) (*DBSnapshot, error) { b.mu.Lock("ModifyDBSnapshot") defer b.mu.Unlock() - snap, ok := b.snapshots[snapshotID] + snap, ok := b.snapshots.Get(snapshotID) if !ok { return nil, fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) } @@ -284,16 +284,17 @@ func (b *InMemoryBackend) ModifyDBSnapshotAttribute( ) (*DBSnapshotAttributesResult, error) { b.mu.Lock("ModifyDBSnapshotAttribute") defer b.mu.Unlock() - if _, ok := b.snapshots[snapshotID]; !ok { + if _, ok := b.snapshots.Get(snapshotID); !ok { return nil, fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) } - if b.snapshotAttributes[snapshotID] == nil { - b.snapshotAttributes[snapshotID] = &DBSnapshotAttributesResult{ + result, ok := b.snapshotAttributes.Get(snapshotID) + if !ok { + result = &DBSnapshotAttributesResult{ DBSnapshotIdentifier: snapshotID, DBSnapshotAttributes: []DBSnapshotAttribute{}, } + b.snapshotAttributes.Put(result) } - result := b.snapshotAttributes[snapshotID] applySnapshotAttributeChange(&result.DBSnapshotAttributes, attributeName, valuesToAdd, valuesToRemove) cp := *result @@ -306,10 +307,10 @@ func (b *InMemoryBackend) DescribeDBClusterSnapshotAttributes( ) (*DBClusterSnapshotAttributesResult, error) { b.mu.RLock("DescribeDBClusterSnapshotAttributes") defer b.mu.RUnlock() - if _, ok := b.clusterSnapshots[snapshotID]; !ok { + if _, ok := b.clusterSnapshots.Get(snapshotID); !ok { return nil, fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) } - if attrs, ok := b.clusterSnapshotAttributes[snapshotID]; ok { + if attrs, ok := b.clusterSnapshotAttributes.Get(snapshotID); ok { cp := *attrs return &cp, nil @@ -330,16 +331,17 @@ func (b *InMemoryBackend) ModifyDBClusterSnapshotAttribute( ) (*DBClusterSnapshotAttributesResult, error) { b.mu.Lock("ModifyDBClusterSnapshotAttribute") defer b.mu.Unlock() - if _, ok := b.clusterSnapshots[snapshotID]; !ok { + if _, ok := b.clusterSnapshots.Get(snapshotID); !ok { return nil, fmt.Errorf("%w: %s", ErrSnapshotNotFound, snapshotID) } - if b.clusterSnapshotAttributes[snapshotID] == nil { - b.clusterSnapshotAttributes[snapshotID] = &DBClusterSnapshotAttributesResult{ + result, ok := b.clusterSnapshotAttributes.Get(snapshotID) + if !ok { + result = &DBClusterSnapshotAttributesResult{ DBClusterSnapshotIdentifier: snapshotID, DBClusterSnapshotAttributes: []DBSnapshotAttribute{}, } + b.clusterSnapshotAttributes.Put(result) } - result := b.clusterSnapshotAttributes[snapshotID] applySnapshotAttributeChange(&result.DBClusterSnapshotAttributes, attributeName, valuesToAdd, valuesToRemove) cp := *result @@ -378,7 +380,7 @@ func applySnapshotAttributeChange( func (b *InMemoryBackend) DescribeDBClusterBacktracks(clusterID string) ([]DBClusterBacktrack, error) { b.mu.RLock("DescribeDBClusterBacktracks") defer b.mu.RUnlock() - if _, ok := b.clusters[clusterID]; !ok { + if _, ok := b.clusters.Get(clusterID); !ok { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, clusterID) } @@ -389,7 +391,7 @@ func (b *InMemoryBackend) DescribeDBClusterBacktracks(clusterID string) ([]DBClu func (b *InMemoryBackend) EnableHTTPEndpoint(resourceARN string) error { b.mu.Lock("EnableHTTPEndpoint") defer b.mu.Unlock() - for _, cluster := range b.clusters { + for _, cluster := range b.clusters.All() { if cluster.DBClusterIdentifier == resourceARN || b.rdsARN("cluster", cluster.DBClusterIdentifier) == resourceARN { cluster.HTTPEndpointEnabled = true @@ -405,7 +407,7 @@ func (b *InMemoryBackend) EnableHTTPEndpoint(resourceARN string) error { func (b *InMemoryBackend) DisableHTTPEndpoint(resourceARN string) error { b.mu.Lock("DisableHTTPEndpoint") defer b.mu.Unlock() - for _, cluster := range b.clusters { + for _, cluster := range b.clusters.All() { if cluster.DBClusterIdentifier == resourceARN || b.rdsARN("cluster", cluster.DBClusterIdentifier) == resourceARN { cluster.HTTPEndpointEnabled = false @@ -421,7 +423,7 @@ func (b *InMemoryBackend) DisableHTTPEndpoint(resourceARN string) error { func (b *InMemoryBackend) ModifyCurrentDBClusterCapacity(clusterID string, capacity int) (*DBCluster, error) { b.mu.Lock("ModifyCurrentDBClusterCapacity") defer b.mu.Unlock() - cluster, ok := b.clusters[clusterID] + cluster, ok := b.clusters.Get(clusterID) if !ok { return nil, fmt.Errorf("%w: %s", ErrClusterNotFound, clusterID) } @@ -441,7 +443,7 @@ func (b *InMemoryBackend) RestoreDBInstanceFromS3(id, engine, dbInstanceClass, s } b.mu.Lock("RestoreDBInstanceFromS3") defer b.mu.Unlock() - if _, exists := b.instances[id]; exists { + if _, exists := b.instances.Get(id); exists { return nil, fmt.Errorf("%w: %s", ErrInstanceAlreadyExists, id) } inst := &DBInstance{ @@ -452,7 +454,7 @@ func (b *InMemoryBackend) RestoreDBInstanceFromS3(id, engine, dbInstanceClass, s AllocatedStorage: defaultAllocatedStorage, StorageType: "gp2", } - b.instances[id] = inst + b.instances.Put(inst) b.instanceReadyAt[id] = time.Now().Add(instanceReadyDelaySeconds * time.Second) b.scheduleReconcilerLocked() cp := *inst @@ -470,7 +472,7 @@ func (b *InMemoryBackend) RestoreDBClusterFromS3(id, engine, masterUsername, s3B } b.mu.Lock("RestoreDBClusterFromS3") defer b.mu.Unlock() - if _, exists := b.clusters[id]; exists { + if _, exists := b.clusters.Get(id); exists { return nil, fmt.Errorf("%w: %s", ErrClusterAlreadyExists, id) } cluster := &DBCluster{ @@ -479,7 +481,7 @@ func (b *InMemoryBackend) RestoreDBClusterFromS3(id, engine, masterUsername, s3B MasterUsername: masterUsername, Status: "creating", } - b.clusters[id] = cluster + b.clusters.Put(cluster) cp := *cluster return &cp, nil @@ -489,7 +491,7 @@ func (b *InMemoryBackend) RestoreDBClusterFromS3(id, engine, masterUsername, s3B func (b *InMemoryBackend) ModifyDBRecommendation(recID, status string) (*DBRecommendation, error) { b.mu.Lock("ModifyDBRecommendation") defer b.mu.Unlock() - rec, ok := b.recommendations[recID] + rec, ok := b.recommendations.Get(recID) if !ok { return nil, fmt.Errorf("%w: recommendation %s not found", ErrInvalidParameter, recID) } @@ -503,8 +505,8 @@ func (b *InMemoryBackend) ModifyDBRecommendation(recID, status string) (*DBRecom func (b *InMemoryBackend) DescribeDBRecommendations(recID, status string) []DBRecommendation { b.mu.RLock("DescribeDBRecommendations") defer b.mu.RUnlock() - result := make([]DBRecommendation, 0, len(b.recommendations)) - for _, rec := range b.recommendations { + result := make([]DBRecommendation, 0, b.recommendations.Len()) + for _, rec := range b.recommendations.All() { if recID != "" && rec.RecommendationID != recID { continue } @@ -552,7 +554,7 @@ func (b *InMemoryBackend) PurchaseReservedDBInstancesOffering( } b.mu.Lock("PurchaseReservedDBInstancesOffering") defer b.mu.Unlock() - if _, exists := b.reservedInstances[reservedDBInstanceID]; exists { + if _, exists := b.reservedInstances.Get(reservedDBInstanceID); exists { return nil, fmt.Errorf("%w: reserved instance %s already exists", ErrInvalidParameter, reservedDBInstanceID) } ri := &ReservedDBInstance{ @@ -570,7 +572,7 @@ func (b *InMemoryBackend) PurchaseReservedDBInstancesOffering( State: subscriptionStatusActive, CurrencyCode: offering.CurrencyCode, } - b.reservedInstances[reservedDBInstanceID] = ri + b.reservedInstances.Put(ri) cp := *ri return &cp, nil @@ -582,8 +584,8 @@ func (b *InMemoryBackend) DescribeReservedDBInstances( ) []ReservedDBInstance { b.mu.RLock("DescribeReservedDBInstances") defer b.mu.RUnlock() - result := make([]ReservedDBInstance, 0, len(b.reservedInstances)) - for _, ri := range b.reservedInstances { + result := make([]ReservedDBInstance, 0, b.reservedInstances.Len()) + for _, ri := range b.reservedInstances.All() { if reservedDBInstanceID != "" && ri.ReservedDBInstanceID != reservedDBInstanceID { continue } diff --git a/services/rds/refinement3.go b/services/rds/refinement3.go index 4de350120..696188d86 100644 --- a/services/rds/refinement3.go +++ b/services/rds/refinement3.go @@ -4,7 +4,6 @@ import ( "crypto/rand" "encoding/hex" "fmt" - "strings" "time" "github.com/blackbirdworks/gopherstack/pkgs/arn" @@ -127,7 +126,7 @@ func (b *InMemoryBackend) CreateDBProxy(name, engineFamily, roleARN string, auth b.mu.Lock("CreateDBProxy") defer b.mu.Unlock() - if _, exists := b.proxies[name]; exists { + if _, exists := b.proxies.Get(name); exists { return nil, fmt.Errorf("%w: %s", ErrDBProxyAlreadyExists, name) } @@ -149,7 +148,7 @@ func (b *InMemoryBackend) CreateDBProxy(name, engineFamily, roleARN string, auth }, } - b.proxies[name] = proxy + b.proxies.Put(proxy) // Auto-create default target group tg := &DBProxyTargetGroup{ @@ -162,7 +161,7 @@ func (b *InMemoryBackend) CreateDBProxy(name, engineFamily, roleARN string, auth UpdatedDate: time.Now(), ConnectionPoolConfig: proxy.ConnectionPoolConfig, } - b.proxyTargetGroups[name+"/"+proxyDefaultTargetGroupName] = tg + b.proxyTargetGroups.Put(tg) return proxy, nil } @@ -172,23 +171,23 @@ func (b *InMemoryBackend) DeleteDBProxy(name string) (*DBProxy, error) { b.mu.Lock("DeleteDBProxy") defer b.mu.Unlock() - proxy, exists := b.proxies[name] + proxy, exists := b.proxies.Get(name) if !exists { return nil, fmt.Errorf("%w: DBProxy %s not found", ErrInvalidParameter, name) } - delete(b.proxies, name) + b.proxies.Delete(name) // Clean up associated target groups and targets - for key := range b.proxyTargetGroups { - if strings.HasPrefix(key, name+"/") { - delete(b.proxyTargetGroups, key) + for _, tg := range b.proxyTargetGroups.All() { + if tg.DBProxyName == name { + b.proxyTargetGroups.Delete(proxyTargetGroupsKeyFn(tg)) } } delete(b.proxyTargets, name) - for key := range b.proxyEndpoints { - if b.proxyEndpoints[key].DBProxyName == name { - delete(b.proxyEndpoints, key) + for _, ep := range b.proxyEndpoints.All() { + if ep.DBProxyName == name { + b.proxyEndpoints.Delete(proxyEndpointsKeyFn(ep)) } } @@ -200,8 +199,8 @@ func (b *InMemoryBackend) DescribeDBProxies(name string) ([]DBProxy, error) { b.mu.RLock("DescribeDBProxies") defer b.mu.RUnlock() - result := make([]DBProxy, 0, len(b.proxies)) - for _, p := range b.proxies { + result := make([]DBProxy, 0, b.proxies.Len()) + for _, p := range b.proxies.All() { if name != "" && p.DBProxyName != name { continue } @@ -225,7 +224,7 @@ func (b *InMemoryBackend) ModifyDBProxy( b.mu.Lock("ModifyDBProxy") defer b.mu.Unlock() - proxy, exists := b.proxies[name] + proxy, exists := b.proxies.Get(name) if !exists { return nil, fmt.Errorf("%w: DBProxy %s not found", ErrInvalidParameter, name) } @@ -255,7 +254,7 @@ func (b *InMemoryBackend) RegisterDBProxyTargets( b.mu.Lock("RegisterDBProxyTargets") defer b.mu.Unlock() - if _, exists := b.proxies[proxyName]; !exists { + if _, exists := b.proxies.Get(proxyName); !exists { return nil, fmt.Errorf("%w: DBProxy %s not found", ErrInvalidParameter, proxyName) } @@ -305,7 +304,7 @@ func (b *InMemoryBackend) DeregisterDBProxyTargets( b.mu.Lock("DeregisterDBProxyTargets") defer b.mu.Unlock() - if _, exists := b.proxies[proxyName]; !exists { + if _, exists := b.proxies.Get(proxyName); !exists { return fmt.Errorf("%w: DBProxy %s not found", ErrInvalidParameter, proxyName) } @@ -334,7 +333,7 @@ func (b *InMemoryBackend) DescribeDBProxyTargets(proxyName, _ string) ([]DBProxy b.mu.RLock("DescribeDBProxyTargets") defer b.mu.RUnlock() - if _, exists := b.proxies[proxyName]; !exists { + if _, exists := b.proxies.Get(proxyName); !exists { return nil, fmt.Errorf("%w: DBProxy %s not found", ErrInvalidParameter, proxyName) } @@ -351,13 +350,13 @@ func (b *InMemoryBackend) DescribeDBProxyTargetGroups(proxyName, targetGroupName b.mu.RLock("DescribeDBProxyTargetGroups") defer b.mu.RUnlock() - if _, exists := b.proxies[proxyName]; !exists { + if _, exists := b.proxies.Get(proxyName); !exists { return nil, fmt.Errorf("%w: DBProxy %s not found", ErrInvalidParameter, proxyName) } - result := make([]DBProxyTargetGroup, 0, len(b.proxyTargetGroups)) - for key, tg := range b.proxyTargetGroups { - if !strings.HasPrefix(key, proxyName+"/") { + result := make([]DBProxyTargetGroup, 0, b.proxyTargetGroups.Len()) + for _, tg := range b.proxyTargetGroups.All() { + if tg.DBProxyName != proxyName { continue } if targetGroupName != "" && tg.TargetGroupName != targetGroupName { @@ -378,7 +377,7 @@ func (b *InMemoryBackend) ModifyDBProxyTargetGroup( defer b.mu.Unlock() key := proxyName + "/" + targetGroupName - tg, exists := b.proxyTargetGroups[key] + tg, exists := b.proxyTargetGroups.Get(key) if !exists { return nil, fmt.Errorf( "%w: target group %s for proxy %s not found", @@ -404,10 +403,10 @@ func (b *InMemoryBackend) CreateDBProxyEndpoint( b.mu.Lock("CreateDBProxyEndpoint") defer b.mu.Unlock() - if _, exists := b.proxies[proxyName]; !exists { + if _, exists := b.proxies.Get(proxyName); !exists { return nil, fmt.Errorf("%w: DBProxy %s not found", ErrInvalidParameter, proxyName) } - if _, exists := b.proxyEndpoints[endpointName]; exists { + if _, exists := b.proxyEndpoints.Get(endpointName); exists { return nil, fmt.Errorf("%w: %s", ErrDBProxyEndpointAlreadyExists, endpointName) } @@ -431,7 +430,7 @@ func (b *InMemoryBackend) CreateDBProxyEndpoint( CreatedDate: time.Now(), } - b.proxyEndpoints[endpointName] = ep + b.proxyEndpoints.Put(ep) return ep, nil } @@ -441,14 +440,14 @@ func (b *InMemoryBackend) DeleteDBProxyEndpoint(endpointName string) (*DBProxyEn b.mu.Lock("DeleteDBProxyEndpoint") defer b.mu.Unlock() - ep, exists := b.proxyEndpoints[endpointName] + ep, exists := b.proxyEndpoints.Get(endpointName) if !exists { return nil, fmt.Errorf("%w: DBProxyEndpoint %s not found", ErrInvalidParameter, endpointName) } if ep.IsDefault { return nil, ErrCannotDeleteDefaultProxyEndpoint } - delete(b.proxyEndpoints, endpointName) + b.proxyEndpoints.Delete(endpointName) return ep, nil } @@ -458,8 +457,8 @@ func (b *InMemoryBackend) DescribeDBProxyEndpoints(proxyName, endpointName strin b.mu.RLock("DescribeDBProxyEndpoints") defer b.mu.RUnlock() - result := make([]DBProxyEndpoint, 0, len(b.proxyEndpoints)) - for _, ep := range b.proxyEndpoints { + result := make([]DBProxyEndpoint, 0, b.proxyEndpoints.Len()) + for _, ep := range b.proxyEndpoints.All() { if proxyName != "" && ep.DBProxyName != proxyName { continue } @@ -481,7 +480,7 @@ func (b *InMemoryBackend) ModifyDBProxyEndpoint(endpointName string, vpcSGIDs [] b.mu.Lock("ModifyDBProxyEndpoint") defer b.mu.Unlock() - ep, exists := b.proxyEndpoints[endpointName] + ep, exists := b.proxyEndpoints.Get(endpointName) if !exists { return nil, fmt.Errorf("%w: DBProxyEndpoint %s not found", ErrInvalidParameter, endpointName) } @@ -499,7 +498,7 @@ func (b *InMemoryBackend) StartActivityStream(clusterID, kmsKeyID, mode string) b.mu.Lock("StartActivityStream") defer b.mu.Unlock() - cluster, exists := b.clusters[clusterID] + cluster, exists := b.clusters.Get(clusterID) if !exists { return nil, fmt.Errorf("%w: DBCluster %s not found", ErrInvalidParameter, clusterID) } @@ -528,7 +527,7 @@ func (b *InMemoryBackend) StopActivityStream(clusterID string) (*DBCluster, erro b.mu.Lock("StopActivityStream") defer b.mu.Unlock() - cluster, exists := b.clusters[clusterID] + cluster, exists := b.clusters.Get(clusterID) if !exists { return nil, fmt.Errorf("%w: DBCluster %s not found", ErrInvalidParameter, clusterID) } @@ -554,7 +553,7 @@ func (b *InMemoryBackend) ModifyActivityStream(clusterID string, auditPolicy str b.mu.Lock("ModifyActivityStream") defer b.mu.Unlock() - cluster, exists := b.clusters[clusterID] + cluster, exists := b.clusters.Get(clusterID) if !exists { return nil, fmt.Errorf("%w: DBCluster %s not found", ErrInvalidParameter, clusterID) } diff --git a/services/rds/store_setup.go b/services/rds/store_setup.go new file mode 100644 index 000000000..e752d56d1 --- /dev/null +++ b/services/rds/store_setup.go @@ -0,0 +1,197 @@ +package rds + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend is registered exactly once, +// here, as a *store.Table[T] on b.registry. See pkgs/store's package doc and +// the services/ec2 (commit 12e611a4) / services/sqs (commit 0f09d77c) +// conversions this follows. +// +// A handful of fields are deliberately NOT registered here and remain plain +// maps -- see the comment block above registerAllTables for the list and why. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +func instancesKeyFn(v *DBInstance) string { return v.DBInstanceIdentifier } +func snapshotsKeyFn(v *DBSnapshot) string { return v.DBSnapshotIdentifier } +func subnetGroupsKeyFn(v *DBSubnetGroup) string { return v.DBSubnetGroupName } +func parameterGroupsKeyFn(v *DBParameterGroup) string { return v.DBParameterGroupName } +func optionGroupsKeyFn(v *OptionGroup) string { return v.OptionGroupName } +func clustersKeyFn(v *DBCluster) string { return v.DBClusterIdentifier } +func clusterSnapshotsKeyFn(v *DBClusterSnapshot) string { return v.DBClusterSnapshotIdentifier } +func clusterEndpointsKeyFn(v *DBClusterEndpoint) string { return v.DBClusterEndpointIdentifier } +func exportTasksKeyFn(v *ExportTask) string { return v.ExportTaskIdentifier } +func globalClustersKeyFn(v *GlobalCluster) string { return v.GlobalClusterIdentifier } +func eventSubscriptionsKeyFn(v *EventSubscription) string { return v.SubscriptionName } +func dbSecurityGroupsKeyFn(v *DBSecurityGroup) string { return v.DBSecurityGroupName } +func blueGreenDeploymentsKeyFn(v *BlueGreenDeployment) string { + return v.BlueGreenDeploymentIdentifier +} +func snapshotAttributesKeyFn(v *DBSnapshotAttributesResult) string { return v.DBSnapshotIdentifier } +func clusterSnapshotAttributesKeyFn(v *DBClusterSnapshotAttributesResult) string { + return v.DBClusterSnapshotIdentifier +} +func reservedInstancesKeyFn(v *ReservedDBInstance) string { return v.ReservedDBInstanceID } +func recommendationsKeyFn(v *DBRecommendation) string { return v.RecommendationID } +func proxiesKeyFn(v *DBProxy) string { return v.DBProxyName } +func proxyTargetGroupsKeyFn(v *DBProxyTargetGroup) string { + return v.DBProxyName + "/" + v.TargetGroupName +} +func proxyEndpointsKeyFn(v *DBProxyEndpoint) string { return v.DBProxyEndpointName } +func customEngineVersionsKeyFn(v *CustomDBEngineVersion) string { + return v.Engine + ":" + v.EngineVersion +} +func shardGroupsKeyFn(v *DBShardGroup) string { return v.DBShardGroupIdentifier } +func integrationsKeyFn(v *Integration) string { return v.IntegrationName } +func tenantDatabasesKeyFn(v *TenantDatabase) string { + return tenantKey(v.DBInstanceIdentifier, v.TenantDBName) +} +func clusterAutomatedBackupsKeyFn(v *DBClusterAutomatedBackup) string { return v.DBClusterIdentifier } + +// registerAllTables registers every converted resource map on b.registry +// exactly once. It must be called during construction only (immediately after +// b.registry is created), never on every Reset() -- store.Register panics on a +// duplicate name, so runtime resets go through registry.ResetAll() instead +// (see InMemoryBackend.Reset in backend.go). +// +// The following resource fields are deliberately left as plain maps (not +// registered here) because they are either slice-valued, transient scheduling +// state, or their map key is not a pure function of the stored value's own +// fields (store.Table requires the latter): +// - tags: map[string][]Tag keyed by ARN, slice-valued +// - instanceReadyAt / clusterReadyAt: transient reconciler-scheduling +// timestamps, not persisted resource state with an identity field +// - clusterRoles / instanceRoles: map[string][]string, slice-valued +// - proxyTargets: map[string][]DBProxyTarget, slice-valued +// - fisFailoverFaults: map[string]time.Time, transient FIS fault-injection +// state explicitly cleared (not restored) on Restore +// - automatedBackups: mixed keying convention across call sites (plain +// DBInstanceIdentifier in the normal create-with-retention flow vs +// "repl-"+ARN in StartDBInstanceAutomatedBackupsReplication) -- pre-existing +// quirk, not a pure function of value identity (same exclusion class as +// EC2's addressTransfers) +// - snapshotTenantDatabases: map[string][]*DBSnapshotTenantDatabase, slice-valued +// - piMetrics: map[string]map[string][]PIDataPoint, nested map +// - instanceLogFiles: map[string][]DBLogFile, slice-valued +// - instanceLogContent: map[string]map[string]string, nested map +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call to the concrete field and value type. A closure list (rather than one +// statement per field in a single function body) keeps registerAllTables small +// regardless of how many resource tables the backend grows to. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.instances = store.Register(b.registry, "instances", store.New(instancesKeyFn)) + }, + func(b *InMemoryBackend) { + b.snapshots = store.Register(b.registry, "snapshots", store.New(snapshotsKeyFn)) + }, + func(b *InMemoryBackend) { + b.subnetGroups = store.Register(b.registry, "subnetGroups", store.New(subnetGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.parameterGroups = store.Register(b.registry, "parameterGroups", store.New(parameterGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.clusterParameterGroups = store.Register( + b.registry, + "clusterParameterGroups", + store.New(parameterGroupsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.optionGroups = store.Register(b.registry, "optionGroups", store.New(optionGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.clusters = store.Register(b.registry, "clusters", store.New(clustersKeyFn)) + }, + func(b *InMemoryBackend) { + b.clusterSnapshots = store.Register(b.registry, "clusterSnapshots", store.New(clusterSnapshotsKeyFn)) + }, + func(b *InMemoryBackend) { + b.clusterEndpoints = store.Register(b.registry, "clusterEndpoints", store.New(clusterEndpointsKeyFn)) + }, + func(b *InMemoryBackend) { + b.exportTasks = store.Register(b.registry, "exportTasks", store.New(exportTasksKeyFn)) + }, + func(b *InMemoryBackend) { + b.globalClusters = store.Register(b.registry, "globalClusters", store.New(globalClustersKeyFn)) + }, + func(b *InMemoryBackend) { + b.eventSubscriptions = store.Register( + b.registry, + "eventSubscriptions", + store.New(eventSubscriptionsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.dbSecurityGroups = store.Register(b.registry, "dbSecurityGroups", store.New(dbSecurityGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.blueGreenDeployments = store.Register( + b.registry, + "blueGreenDeployments", + store.New(blueGreenDeploymentsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.snapshotAttributes = store.Register( + b.registry, + "snapshotAttributes", + store.New(snapshotAttributesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.clusterSnapshotAttributes = store.Register( + b.registry, + "clusterSnapshotAttributes", + store.New(clusterSnapshotAttributesKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.reservedInstances = store.Register(b.registry, "reservedInstances", store.New(reservedInstancesKeyFn)) + }, + func(b *InMemoryBackend) { + b.recommendations = store.Register(b.registry, "recommendations", store.New(recommendationsKeyFn)) + }, + func(b *InMemoryBackend) { + b.proxies = store.Register(b.registry, "proxies", store.New(proxiesKeyFn)) + }, + func(b *InMemoryBackend) { + b.proxyTargetGroups = store.Register(b.registry, "proxyTargetGroups", store.New(proxyTargetGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.proxyEndpoints = store.Register(b.registry, "proxyEndpoints", store.New(proxyEndpointsKeyFn)) + }, + func(b *InMemoryBackend) { + b.customEngineVersions = store.Register( + b.registry, + "customEngineVersions", + store.New(customEngineVersionsKeyFn), + ) + }, + func(b *InMemoryBackend) { + b.shardGroups = store.Register(b.registry, "shardGroups", store.New(shardGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.integrations = store.Register(b.registry, "integrations", store.New(integrationsKeyFn)) + }, + func(b *InMemoryBackend) { + b.tenantDatabases = store.Register(b.registry, "tenantDatabases", store.New(tenantDatabasesKeyFn)) + }, + func(b *InMemoryBackend) { + b.clusterAutomatedBackups = store.Register( + b.registry, + "clusterAutomatedBackups", + store.New(clusterAutomatedBackupsKeyFn), + ) + }, +} diff --git a/services/redshift/backend.go b/services/redshift/backend.go index 9ea6bfbe0..349a08d6f 100644 --- a/services/redshift/backend.go +++ b/services/redshift/backend.go @@ -4,12 +4,12 @@ import ( "errors" "fmt" "regexp" - "sort" "strings" "sync" "time" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) @@ -366,39 +366,47 @@ type Cluster struct { // InMemoryBackend is the in-memory store for Redshift clusters. type InMemoryBackend struct { - dnsRegistrar DNSRegistrar - customDomains map[string]*CustomDomainAssociation - events map[string]*Event - partners map[string]*Partner - dataShares map[string]*DataShare - securityGroups map[string]*ClusterSecurityGroup - snapshots map[string]*Snapshot - endpointAuths map[string]*EndpointAuthorization - activeResizes map[string]*ResizeProgress - parameterGroups map[string]*ClusterParameterGroup - subnetGroups map[string]*ClusterSubnetGroup - loggingStatuses map[string]*LoggingStatus - eventSubscriptions map[string]*EventSubscription - clusters map[string]*Cluster - snapshotCopyGrants map[string]*SnapshotCopyGrant - snapshotSchedules map[string]*SnapshotSchedule - usageLimits map[string]*UsageLimit - authProfiles map[string]*AuthenticationProfile - resourcePolicies map[string]*ResourcePolicy - tableRestores map[string]*TableRestoreStatus - snapshotCopyConfigs map[string]*SnapshotCopyConfig - hsmClientCerts map[string]*HsmClientCertificate - hsmConfigs map[string]*HsmConfiguration - reservedNodes map[string]*ReservedNode - scheduledActions map[string]*ScheduledAction - slScheduledActions map[string]*ServerlessScheduledAction - integrations map[string]*Integration - idcApplications map[string]*IdcApplication - slNamespaces map[string]*Namespace - slWorkgroups map[string]*Workgroup - slSnapshots map[string]*ServerlessSnapshot - slUsageLimits map[string]*ServerlessUsageLimit - endpointAccesses map[string]*EndpointAccess + dnsRegistrar DNSRegistrar + registry *store.Registry + customDomains *store.Table[CustomDomainAssociation] + events *store.Table[Event] + partners *store.Table[Partner] + dataShares *store.Table[DataShare] + securityGroups *store.Table[ClusterSecurityGroup] + snapshots *store.Table[Snapshot] + endpointAuths *store.Table[EndpointAuthorization] + // activeResizes is intentionally NOT a store.Table: ResizeProgress carries + // no ClusterIdentifier field of its own, so its key (clusterID) is not a + // pure function of the value -- see store_setup.go. + activeResizes map[string]*ResizeProgress + parameterGroups *store.Table[ClusterParameterGroup] + subnetGroups *store.Table[ClusterSubnetGroup] + // loggingStatuses is intentionally NOT a store.Table; see activeResizes above. + loggingStatuses map[string]*LoggingStatus + eventSubscriptions *store.Table[EventSubscription] + clusters *store.Table[Cluster] + snapshotCopyGrants *store.Table[SnapshotCopyGrant] + snapshotSchedules *store.Table[SnapshotSchedule] + usageLimits *store.Table[UsageLimit] + authProfiles *store.Table[AuthenticationProfile] + resourcePolicies *store.Table[ResourcePolicy] + tableRestores *store.Table[TableRestoreStatus] + // snapshotCopyConfigs is intentionally NOT a store.Table; see activeResizes above. + snapshotCopyConfigs map[string]*SnapshotCopyConfig + hsmClientCerts *store.Table[HsmClientCertificate] + hsmConfigs *store.Table[HsmConfiguration] + reservedNodes *store.Table[ReservedNode] + scheduledActions *store.Table[ScheduledAction] + slScheduledActions *store.Table[ServerlessScheduledAction] + integrations *store.Table[Integration] + idcApplications *store.Table[IdcApplication] + slNamespaces *store.Table[Namespace] + slWorkgroups *store.Table[Workgroup] + slSnapshots *store.Table[ServerlessSnapshot] + slUsageLimits *store.Table[ServerlessUsageLimit] + endpointAccesses *store.Table[EndpointAccess] + // clusterTransitions holds in-flight lifecycle state, intentionally never + // persisted (see Restore) and keyed externally by cluster ID. clusterTransitions map[string]*clusterTransition mu *lockmetrics.RWMutex reconcileStop chan struct{} @@ -418,44 +426,20 @@ type InMemoryBackend struct { // NewInMemoryBackend creates a new InMemoryBackend. func NewInMemoryBackend(accountID, region string) *InMemoryBackend { - return &InMemoryBackend{ - clusters: make(map[string]*Cluster), - reservedNodes: make(map[string]*ReservedNode), - partners: make(map[string]*Partner), - dataShares: make(map[string]*DataShare), - securityGroups: make(map[string]*ClusterSecurityGroup), - snapshots: make(map[string]*Snapshot), - endpointAuths: make(map[string]*EndpointAuthorization), + b := &InMemoryBackend{ activeResizes: make(map[string]*ResizeProgress), - parameterGroups: make(map[string]*ClusterParameterGroup), - subnetGroups: make(map[string]*ClusterSubnetGroup), loggingStatuses: make(map[string]*LoggingStatus), - eventSubscriptions: make(map[string]*EventSubscription), - events: make(map[string]*Event), - snapshotCopyGrants: make(map[string]*SnapshotCopyGrant), - snapshotSchedules: make(map[string]*SnapshotSchedule), - usageLimits: make(map[string]*UsageLimit), - authProfiles: make(map[string]*AuthenticationProfile), - resourcePolicies: make(map[string]*ResourcePolicy), - tableRestores: make(map[string]*TableRestoreStatus), snapshotCopyConfigs: make(map[string]*SnapshotCopyConfig), - hsmClientCerts: make(map[string]*HsmClientCertificate), - hsmConfigs: make(map[string]*HsmConfiguration), - scheduledActions: make(map[string]*ScheduledAction), - customDomains: make(map[string]*CustomDomainAssociation), - endpointAccesses: make(map[string]*EndpointAccess), - integrations: make(map[string]*Integration), - idcApplications: make(map[string]*IdcApplication), - slNamespaces: make(map[string]*Namespace), - slWorkgroups: make(map[string]*Workgroup), - slSnapshots: make(map[string]*ServerlessSnapshot), - slUsageLimits: make(map[string]*ServerlessUsageLimit), - slScheduledActions: make(map[string]*ServerlessScheduledAction), clusterTransitions: make(map[string]*clusterTransition), accountID: accountID, region: region, mu: lockmetrics.New("redshift"), + registry: store.NewRegistry(), } + + registerAllTables(b) + + return b } // Reset clears all backend state while preserving configuration. @@ -463,42 +447,14 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - for _, c := range b.clusters { + for _, c := range b.clusters.All() { c.Tags.Close() } - b.clusters = make(map[string]*Cluster) - b.reservedNodes = make(map[string]*ReservedNode) - b.partners = make(map[string]*Partner) - b.dataShares = make(map[string]*DataShare) - b.securityGroups = make(map[string]*ClusterSecurityGroup) - b.snapshots = make(map[string]*Snapshot) - b.endpointAuths = make(map[string]*EndpointAuthorization) + b.registry.ResetAll() b.activeResizes = make(map[string]*ResizeProgress) - b.parameterGroups = make(map[string]*ClusterParameterGroup) - b.subnetGroups = make(map[string]*ClusterSubnetGroup) b.loggingStatuses = make(map[string]*LoggingStatus) - b.eventSubscriptions = make(map[string]*EventSubscription) - b.events = make(map[string]*Event) - b.snapshotCopyGrants = make(map[string]*SnapshotCopyGrant) - b.snapshotSchedules = make(map[string]*SnapshotSchedule) - b.usageLimits = make(map[string]*UsageLimit) - b.authProfiles = make(map[string]*AuthenticationProfile) - b.resourcePolicies = make(map[string]*ResourcePolicy) - b.tableRestores = make(map[string]*TableRestoreStatus) b.snapshotCopyConfigs = make(map[string]*SnapshotCopyConfig) - b.hsmClientCerts = make(map[string]*HsmClientCertificate) - b.hsmConfigs = make(map[string]*HsmConfiguration) - b.scheduledActions = make(map[string]*ScheduledAction) - b.customDomains = make(map[string]*CustomDomainAssociation) - b.endpointAccesses = make(map[string]*EndpointAccess) - b.integrations = make(map[string]*Integration) - b.idcApplications = make(map[string]*IdcApplication) - b.slNamespaces = make(map[string]*Namespace) - b.slWorkgroups = make(map[string]*Workgroup) - b.slSnapshots = make(map[string]*ServerlessSnapshot) - b.slUsageLimits = make(map[string]*ServerlessUsageLimit) - b.slScheduledActions = make(map[string]*ServerlessScheduledAction) b.clusterTransitions = make(map[string]*clusterTransition) b.resetServerlessIndexes() } @@ -545,7 +501,7 @@ func (b *InMemoryBackend) CreateCluster(id, nodeType, dbName, masterUser string) b.mu.Lock("CreateCluster") defer b.mu.Unlock() - if _, exists := b.clusters[id]; exists { + if _, exists := b.clusters.Get(id); exists { return nil, fmt.Errorf("%w: cluster %s already exists", ErrClusterAlreadyExists, id) } @@ -578,7 +534,7 @@ func (b *InMemoryBackend) CreateCluster(id, nodeType, dbName, masterUser string) NumberOfNodes: 1, Tags: tags.New("redshift.cluster." + id + ".tags"), } - b.clusters[id] = cluster + b.clusters.Put(cluster) // Schedule the creating→available transition instead of spawning an // unmanaged per-cluster goroutine. The managed reconciler (or a lazy read) @@ -604,7 +560,7 @@ func (b *InMemoryBackend) DeleteCluster(id string) (*Cluster, error) { b.mu.Lock("DeleteCluster") defer b.mu.Unlock() - cluster, exists := b.clusters[id] + cluster, exists := b.clusters.Get(id) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } @@ -626,7 +582,7 @@ func (b *InMemoryBackend) DeleteCluster(id string) (*Cluster, error) { cp := cloneCluster(cluster) delete(b.clusterTransitions, id) cluster.Tags.Close() - delete(b.clusters, id) + b.clusters.Delete(id) if b.dnsRegistrar != nil { b.dnsRegistrar.Deregister(cp.Endpoint) @@ -647,7 +603,7 @@ func (b *InMemoryBackend) DescribeClusters(id, marker string, maxRecords int) ([ defer b.mu.RUnlock() if id != "" { - c, exists := b.clusters[id] + c, exists := b.clusters.Get(id) if !exists { return nil, "", fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } @@ -655,32 +611,29 @@ func (b *InMemoryBackend) DescribeClusters(id, marker string, maxRecords int) ([ return []Cluster{cloneCluster(c)}, "", nil } - ids := make([]string, 0, len(b.clusters)) - for k := range b.clusters { - ids = append(ids, k) - } - - sort.Strings(ids) + // Snapshot returns every cluster ordered by key (ClusterIdentifier) + // ascending, matching the previous sort.Strings(ids) behaviour. + sorted := b.clusters.Snapshot() // Advance past the marker (exclusive — marker is the last ID on the previous page). if marker != "" { cut := 0 - for cut < len(ids) && ids[cut] <= marker { + for cut < len(sorted) && sorted[cut].ClusterIdentifier <= marker { cut++ } - ids = ids[cut:] + sorted = sorted[cut:] } nextMarker := "" - if maxRecords > 0 && len(ids) > maxRecords { - ids = ids[:maxRecords] - nextMarker = ids[len(ids)-1] + if maxRecords > 0 && len(sorted) > maxRecords { + sorted = sorted[:maxRecords] + nextMarker = sorted[len(sorted)-1].ClusterIdentifier } - clusters := make([]Cluster, 0, len(ids)) - for _, k := range ids { - clusters = append(clusters, cloneCluster(b.clusters[k])) + clusters := make([]Cluster, 0, len(sorted)) + for _, c := range sorted { + clusters = append(clusters, cloneCluster(c)) } return clusters, nextMarker, nil @@ -691,9 +644,11 @@ func (b *InMemoryBackend) DescribeTags() map[string]map[string]string { b.mu.RLock("DescribeTags") defer b.mu.RUnlock() - result := make(map[string]map[string]string, len(b.clusters)) - for id, c := range b.clusters { - result[id] = c.Tags.Clone() + all := b.clusters.All() + result := make(map[string]map[string]string, len(all)) + + for _, c := range all { + result[c.ClusterIdentifier] = c.Tags.Clone() } return result @@ -704,7 +659,7 @@ func (b *InMemoryBackend) CreateTags(clusterID string, kv map[string]string) err b.mu.Lock("CreateTags") defer b.mu.Unlock() - c, exists := b.clusters[clusterID] + c, exists := b.clusters.Get(clusterID) if !exists { return fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -719,7 +674,7 @@ func (b *InMemoryBackend) DeleteTags(clusterID string, keys []string) error { b.mu.Lock("DeleteTags") defer b.mu.Unlock() - c, exists := b.clusters[clusterID] + c, exists := b.clusters.Get(clusterID) if !exists { return fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -751,7 +706,7 @@ func (b *InMemoryBackend) AcceptReservedNodeExchange(reservedNodeID, targetOffer b.mu.Lock("AcceptReservedNodeExchange") defer b.mu.Unlock() - existing, exists := b.reservedNodes[reservedNodeID] + existing, exists := b.reservedNodes.Get(reservedNodeID) if !exists { return nil, fmt.Errorf("%w: reserved node %s not found", ErrReservedNodeNotFound, reservedNodeID) } @@ -769,7 +724,7 @@ func (b *InMemoryBackend) AcceptReservedNodeExchange(reservedNodeID, targetOffer State: "active", OfferingType: existing.OfferingType, } - b.reservedNodes[reservedNodeID] = exchanged + b.reservedNodes.Put(exchanged) cp := *exchanged @@ -791,11 +746,10 @@ func (b *InMemoryBackend) AddPartner(accountID, clusterID, databaseName, partner b.mu.Lock("AddPartner") defer b.mu.Unlock() - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } - key := partnerKey(clusterID, databaseName, partnerName) partner := &Partner{ AccountID: accountID, ClusterIdentifier: clusterID, @@ -804,7 +758,7 @@ func (b *InMemoryBackend) AddPartner(accountID, clusterID, databaseName, partner Status: partnerStatusActive, StatusMessage: "", } - b.partners[key] = partner + b.partners.Put(partner) cp := *partner @@ -823,7 +777,7 @@ func (b *InMemoryBackend) AssociateDataShareConsumer( b.mu.Lock("AssociateDataShareConsumer") defer b.mu.Unlock() - ds, exists := b.dataShares[dataShareArn] + ds, exists := b.dataShares.Get(dataShareArn) if !exists { return nil, fmt.Errorf("%w: data share %s not found", ErrDataShareNotFound, dataShareArn) } @@ -855,7 +809,7 @@ func (b *InMemoryBackend) AuthorizeClusterSecurityGroupIngress( b.mu.Lock("AuthorizeClusterSecurityGroupIngress") defer b.mu.Unlock() - sg, exists := b.securityGroups[groupName] + sg, exists := b.securityGroups.Get(groupName) if !exists { return nil, fmt.Errorf("%w: security group %s not found", ErrSecurityGroupNotFound, groupName) } @@ -886,7 +840,7 @@ func (b *InMemoryBackend) AuthorizeDataShare(dataShareArn, consumerIdentifier st b.mu.Lock("AuthorizeDataShare") defer b.mu.Unlock() - ds, exists := b.dataShares[dataShareArn] + ds, exists := b.dataShares.Get(dataShareArn) if !exists { return nil, fmt.Errorf("%w: data share %s not found", ErrDataShareNotFound, dataShareArn) } @@ -918,12 +872,12 @@ func (b *InMemoryBackend) AuthorizeEndpointAccess( b.mu.Lock("AuthorizeEndpointAccess") defer b.mu.Unlock() - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } key := endpointAuthKey(clusterID, grantee) - if _, exists := b.endpointAuths[key]; exists { + if _, exists := b.endpointAuths.Get(key); exists { return nil, fmt.Errorf("%w: endpoint authorization already exists for cluster %s and account %s", ErrEndpointAuthAlreadyExists, clusterID, grantee) } @@ -942,7 +896,7 @@ func (b *InMemoryBackend) AuthorizeEndpointAccess( AllowedVPCs: allowedVPCs, EndpointCount: 0, } - b.endpointAuths[key] = auth + b.endpointAuths.Put(auth) return cloneEndpointAuth(auth), nil } @@ -959,7 +913,7 @@ func (b *InMemoryBackend) AuthorizeSnapshotAccess(snapshotID, accountWithRestore b.mu.Lock("AuthorizeSnapshotAccess") defer b.mu.Unlock() - snap, exists := b.snapshots[snapshotID] + snap, exists := b.snapshots.Get(snapshotID) if !exists { return nil, fmt.Errorf("%w: snapshot %s not found", ErrSnapshotNotFound, snapshotID) } @@ -982,7 +936,7 @@ func (b *InMemoryBackend) BatchDeleteClusterSnapshots(identifiers []string) ([]S var deleted []string for _, id := range identifiers { - if _, exists := b.snapshots[id]; !exists { + if _, exists := b.snapshots.Get(id); !exists { batchErrors = append(batchErrors, SnapshotBatchError{ SnapshotIdentifier: id, FailureCode: errClusterSnapshotNotFound, @@ -992,7 +946,7 @@ func (b *InMemoryBackend) BatchDeleteClusterSnapshots(identifiers []string) ([]S continue } - delete(b.snapshots, id) + b.snapshots.Delete(id) deleted = append(deleted, id) } @@ -1015,7 +969,7 @@ func (b *InMemoryBackend) BatchModifyClusterSnapshots( var modified []string for _, id := range identifiers { - snap, exists := b.snapshots[id] + snap, exists := b.snapshots.Get(id) if !exists { batchErrors = append(batchErrors, SnapshotBatchError{ SnapshotIdentifier: id, @@ -1042,7 +996,7 @@ func (b *InMemoryBackend) CancelResize(clusterID string) (*ResizeProgress, error b.mu.Lock("CancelResize") defer b.mu.Unlock() - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -1108,28 +1062,28 @@ func cloneSnapshot(snap *Snapshot) *Snapshot { func (b *InMemoryBackend) AddReservedNodeInternal(node *ReservedNode) { b.mu.Lock("AddReservedNodeInternal") defer b.mu.Unlock() - b.reservedNodes[node.ReservedNodeID] = node + b.reservedNodes.Put(node) } // AddDataShareInternal seeds a data share directly into the backend. func (b *InMemoryBackend) AddDataShareInternal(ds *DataShare) { b.mu.Lock("AddDataShareInternal") defer b.mu.Unlock() - b.dataShares[ds.DataShareArn] = ds + b.dataShares.Put(ds) } // AddSecurityGroupInternal seeds a cluster security group directly into the backend. func (b *InMemoryBackend) AddSecurityGroupInternal(sg *ClusterSecurityGroup) { b.mu.Lock("AddSecurityGroupInternal") defer b.mu.Unlock() - b.securityGroups[sg.ClusterSecurityGroupName] = sg + b.securityGroups.Put(sg) } // AddSnapshotInternal seeds a snapshot directly into the backend. func (b *InMemoryBackend) AddSnapshotInternal(snap *Snapshot) { b.mu.Lock("AddSnapshotInternal") defer b.mu.Unlock() - b.snapshots[snap.SnapshotIdentifier] = snap + b.snapshots.Put(snap) } // AddActiveResizeInternal seeds an active resize directly into the backend. @@ -1143,12 +1097,12 @@ func (b *InMemoryBackend) AddActiveResizeInternal(clusterID string, resize *Resi func (b *InMemoryBackend) AddParameterGroupInternal(pg *ClusterParameterGroup) { b.mu.Lock("AddParameterGroupInternal") defer b.mu.Unlock() - b.parameterGroups[pg.ParameterGroupName] = pg + b.parameterGroups.Put(pg) } // AddSubnetGroupInternal seeds a subnet group directly into the backend. func (b *InMemoryBackend) AddSubnetGroupInternal(sg *ClusterSubnetGroup) { b.mu.Lock("AddSubnetGroupInternal") defer b.mu.Unlock() - b.subnetGroups[sg.ClusterSubnetGroupName] = sg + b.subnetGroups.Put(sg) } diff --git a/services/redshift/backend_audit1.go b/services/redshift/backend_audit1.go index 587378ff8..6cb4702a4 100644 --- a/services/redshift/backend_audit1.go +++ b/services/redshift/backend_audit1.go @@ -21,7 +21,7 @@ func (b *InMemoryBackend) CreateHsmClientCertificate( b.mu.Lock("CreateHsmClientCertificate") defer b.mu.Unlock() - if _, exists := b.hsmClientCerts[id]; exists { + if _, exists := b.hsmClientCerts.Get(id); exists { return nil, fmt.Errorf("%w: certificate %s already exists", ErrHsmClientCertAlreadyExists, id) } @@ -32,7 +32,7 @@ func (b *InMemoryBackend) CreateHsmClientCertificate( HsmClientCertificatePublicKey: pubKey, Tags: tagMap, } - b.hsmClientCerts[id] = cert + b.hsmClientCerts.Put(cert) cp := *cert @@ -48,11 +48,11 @@ func (b *InMemoryBackend) DeleteHsmClientCertificate(id string) error { b.mu.Lock("DeleteHsmClientCertificate") defer b.mu.Unlock() - if _, exists := b.hsmClientCerts[id]; !exists { + if _, exists := b.hsmClientCerts.Get(id); !exists { return fmt.Errorf("%w: certificate %s not found", ErrHsmClientCertNotFound, id) } - delete(b.hsmClientCerts, id) + b.hsmClientCerts.Delete(id) return nil } @@ -63,7 +63,7 @@ func (b *InMemoryBackend) DescribeHsmClientCertificates(id string) ([]HsmClientC defer b.mu.RUnlock() if id != "" { - c, exists := b.hsmClientCerts[id] + c, exists := b.hsmClientCerts.Get(id) if !exists { return nil, fmt.Errorf("%w: certificate %s not found", ErrHsmClientCertNotFound, id) } @@ -73,9 +73,9 @@ func (b *InMemoryBackend) DescribeHsmClientCertificates(id string) ([]HsmClientC return []HsmClientCertificate{cp}, nil } - result := make([]HsmClientCertificate, 0, len(b.hsmClientCerts)) + result := make([]HsmClientCertificate, 0, b.hsmClientCerts.Len()) - for _, c := range b.hsmClientCerts { + for _, c := range b.hsmClientCerts.All() { result = append(result, *c) } @@ -96,7 +96,7 @@ func (b *InMemoryBackend) CreateHsmConfiguration( b.mu.Lock("CreateHsmConfiguration") defer b.mu.Unlock() - if _, exists := b.hsmConfigs[id]; exists { + if _, exists := b.hsmConfigs.Get(id); exists { return nil, fmt.Errorf("%w: configuration %s already exists", ErrHsmConfigAlreadyExists, id) } @@ -107,7 +107,7 @@ func (b *InMemoryBackend) CreateHsmConfiguration( HsmPartitionName: hsmPartitionName, Tags: tagMap, } - b.hsmConfigs[id] = cfg + b.hsmConfigs.Put(cfg) cp := *cfg @@ -123,11 +123,11 @@ func (b *InMemoryBackend) DeleteHsmConfiguration(id string) error { b.mu.Lock("DeleteHsmConfiguration") defer b.mu.Unlock() - if _, exists := b.hsmConfigs[id]; !exists { + if _, exists := b.hsmConfigs.Get(id); !exists { return fmt.Errorf("%w: configuration %s not found", ErrHsmConfigNotFound, id) } - delete(b.hsmConfigs, id) + b.hsmConfigs.Delete(id) return nil } @@ -138,7 +138,7 @@ func (b *InMemoryBackend) DescribeHsmConfigurations(id string) ([]HsmConfigurati defer b.mu.RUnlock() if id != "" { - c, exists := b.hsmConfigs[id] + c, exists := b.hsmConfigs.Get(id) if !exists { return nil, fmt.Errorf("%w: configuration %s not found", ErrHsmConfigNotFound, id) } @@ -148,9 +148,9 @@ func (b *InMemoryBackend) DescribeHsmConfigurations(id string) ([]HsmConfigurati return []HsmConfiguration{cp}, nil } - result := make([]HsmConfiguration, 0, len(b.hsmConfigs)) + result := make([]HsmConfiguration, 0, b.hsmConfigs.Len()) - for _, c := range b.hsmConfigs { + for _, c := range b.hsmConfigs.All() { result = append(result, *c) } @@ -170,7 +170,7 @@ func (b *InMemoryBackend) CreateScheduledAction( b.mu.Lock("CreateScheduledAction") defer b.mu.Unlock() - if _, exists := b.scheduledActions[name]; exists { + if _, exists := b.scheduledActions.Get(name); exists { return nil, fmt.Errorf("%w: scheduled action %s already exists", ErrScheduledActionAlreadyExists, name) } @@ -182,7 +182,7 @@ func (b *InMemoryBackend) CreateScheduledAction( State: "ACTIVE", TargetAction: targetAction, } - b.scheduledActions[name] = action + b.scheduledActions.Put(action) cp := *action @@ -198,11 +198,11 @@ func (b *InMemoryBackend) DeleteScheduledAction(name string) error { b.mu.Lock("DeleteScheduledAction") defer b.mu.Unlock() - if _, exists := b.scheduledActions[name]; !exists { + if _, exists := b.scheduledActions.Get(name); !exists { return fmt.Errorf("%w: scheduled action %s not found", ErrScheduledActionNotFound, name) } - delete(b.scheduledActions, name) + b.scheduledActions.Delete(name) return nil } @@ -213,7 +213,7 @@ func (b *InMemoryBackend) DescribeScheduledActions(name string) ([]ScheduledActi defer b.mu.RUnlock() if name != "" { - a, exists := b.scheduledActions[name] + a, exists := b.scheduledActions.Get(name) if !exists { return nil, fmt.Errorf("%w: scheduled action %s not found", ErrScheduledActionNotFound, name) } @@ -223,9 +223,9 @@ func (b *InMemoryBackend) DescribeScheduledActions(name string) ([]ScheduledActi return []ScheduledAction{cp}, nil } - result := make([]ScheduledAction, 0, len(b.scheduledActions)) + result := make([]ScheduledAction, 0, b.scheduledActions.Len()) - for _, a := range b.scheduledActions { + for _, a := range b.scheduledActions.All() { result = append(result, *a) } @@ -243,7 +243,7 @@ func (b *InMemoryBackend) ModifyScheduledAction( b.mu.Lock("ModifyScheduledAction") defer b.mu.Unlock() - a, exists := b.scheduledActions[name] + a, exists := b.scheduledActions.Get(name) if !exists { return nil, fmt.Errorf("%w: scheduled action %s not found", ErrScheduledActionNotFound, name) } @@ -287,7 +287,7 @@ func (b *InMemoryBackend) CreateTableRestoreStatus( TargetTableName: targetTableName, RequestTime: time.Now().UTC(), } - b.tableRestores[restoreID] = tr + b.tableRestores.Put(tr) cp := *tr diff --git a/services/redshift/backend_audit2.go b/services/redshift/backend_audit2.go index d2f6f8739..4e9910cd8 100644 --- a/services/redshift/backend_audit2.go +++ b/services/redshift/backend_audit2.go @@ -22,12 +22,12 @@ func (b *InMemoryBackend) CreateCustomDomainAssociation( b.mu.Lock("CreateCustomDomainAssociation") defer b.mu.Unlock() - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } key := clusterID + ":" + customDomainName - if _, exists := b.customDomains[key]; exists { + if _, exists := b.customDomains.Get(key); exists { return nil, fmt.Errorf("%w: custom domain %s already associated with cluster %s", ErrCustomDomainAlreadyExists, customDomainName, clusterID) } @@ -37,7 +37,7 @@ func (b *InMemoryBackend) CreateCustomDomainAssociation( CustomDomainName: customDomainName, CustomDomainCertificateArn: customDomainCertificateArn, } - b.customDomains[key] = assoc + b.customDomains.Put(assoc) cp := *assoc @@ -58,12 +58,12 @@ func (b *InMemoryBackend) DeleteCustomDomainAssociation(clusterID, customDomainN defer b.mu.Unlock() key := clusterID + ":" + customDomainName - if _, exists := b.customDomains[key]; !exists { + if _, exists := b.customDomains.Get(key); !exists { return fmt.Errorf("%w: custom domain %s not associated with cluster %s", ErrCustomDomainNotFound, customDomainName, clusterID) } - delete(b.customDomains, key) + b.customDomains.Delete(key) return nil } @@ -77,7 +77,7 @@ func (b *InMemoryBackend) DescribeCustomDomainAssociations( if clusterID != "" && customDomainName != "" { key := clusterID + ":" + customDomainName - a, exists := b.customDomains[key] + a, exists := b.customDomains.Get(key) if !exists { return nil, fmt.Errorf("%w: custom domain %s not associated with cluster %s", ErrCustomDomainNotFound, customDomainName, clusterID) @@ -88,9 +88,9 @@ func (b *InMemoryBackend) DescribeCustomDomainAssociations( return []CustomDomainAssociation{cp}, nil } - result := make([]CustomDomainAssociation, 0, len(b.customDomains)) + result := make([]CustomDomainAssociation, 0, b.customDomains.Len()) - for _, a := range b.customDomains { + for _, a := range b.customDomains.All() { if clusterID != "" && a.ClusterIdentifier != clusterID { continue } @@ -117,7 +117,7 @@ func (b *InMemoryBackend) ModifyCustomDomainAssociation( defer b.mu.Unlock() key := clusterID + ":" + customDomainName - a, exists := b.customDomains[key] + a, exists := b.customDomains.Get(key) if !exists { return nil, fmt.Errorf("%w: custom domain %s not associated with cluster %s", ErrCustomDomainNotFound, customDomainName, clusterID) @@ -146,11 +146,11 @@ func (b *InMemoryBackend) CreateEndpointAccess( b.mu.Lock("CreateEndpointAccess") defer b.mu.Unlock() - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } - if _, exists := b.endpointAccesses[endpointName]; exists { + if _, exists := b.endpointAccesses.Get(endpointName); exists { return nil, fmt.Errorf("%w: endpoint %s already exists", ErrEndpointAccessAlreadyExists, endpointName) } @@ -162,7 +162,7 @@ func (b *InMemoryBackend) CreateEndpointAccess( Port: redshiftDefaultPort, VpcID: vpcID, } - b.endpointAccesses[endpointName] = ep + b.endpointAccesses.Put(ep) cp := *ep @@ -178,12 +178,12 @@ func (b *InMemoryBackend) DeleteEndpointAccess(endpointName string) (*EndpointAc b.mu.Lock("DeleteEndpointAccess") defer b.mu.Unlock() - ep, exists := b.endpointAccesses[endpointName] + ep, exists := b.endpointAccesses.Get(endpointName) if !exists { return nil, fmt.Errorf("%w: endpoint %s not found", ErrEndpointAccessNotFound, endpointName) } - delete(b.endpointAccesses, endpointName) + b.endpointAccesses.Delete(endpointName) cp := *ep cp.EndpointStatus = "deleting" @@ -199,7 +199,7 @@ func (b *InMemoryBackend) DescribeEndpointAccess( defer b.mu.RUnlock() if endpointName != "" { - ep, exists := b.endpointAccesses[endpointName] + ep, exists := b.endpointAccesses.Get(endpointName) if !exists { return nil, fmt.Errorf("%w: endpoint %s not found", ErrEndpointAccessNotFound, endpointName) } @@ -209,9 +209,9 @@ func (b *InMemoryBackend) DescribeEndpointAccess( return []EndpointAccess{cp}, nil } - result := make([]EndpointAccess, 0, len(b.endpointAccesses)) + result := make([]EndpointAccess, 0, b.endpointAccesses.Len()) - for _, ep := range b.endpointAccesses { + for _, ep := range b.endpointAccesses.All() { if clusterID != "" && ep.ClusterIdentifier != clusterID { continue } @@ -231,7 +231,7 @@ func (b *InMemoryBackend) ModifyEndpointAccess(endpointName, vpcID string) (*End b.mu.Lock("ModifyEndpointAccess") defer b.mu.Unlock() - ep, exists := b.endpointAccesses[endpointName] + ep, exists := b.endpointAccesses.Get(endpointName) if !exists { return nil, fmt.Errorf("%w: endpoint %s not found", ErrEndpointAccessNotFound, endpointName) } @@ -266,7 +266,7 @@ func (b *InMemoryBackend) CreateIntegration( b.mu.Lock("CreateIntegration") defer b.mu.Unlock() - if _, exists := b.integrations[integrationName]; exists { + if _, exists := b.integrations.Get(integrationName); exists { return nil, fmt.Errorf("%w: integration %s already exists", ErrIntegrationAlreadyExists, integrationName) } @@ -280,7 +280,7 @@ func (b *InMemoryBackend) CreateIntegration( Description: description, Status: endpointStatusActive, } - b.integrations[integrationName] = ig + b.integrations.Put(ig) cp := *ig @@ -296,10 +296,10 @@ func (b *InMemoryBackend) DeleteIntegration(integrationArn string) (*Integration b.mu.Lock("DeleteIntegration") defer b.mu.Unlock() - for name, ig := range b.integrations { - if ig.IntegrationArn == integrationArn || name == integrationArn { + for _, ig := range b.integrations.All() { + if ig.IntegrationArn == integrationArn || ig.IntegrationName == integrationArn { cp := *ig - delete(b.integrations, name) + b.integrations.Delete(ig.IntegrationName) return &cp, nil } @@ -314,7 +314,7 @@ func (b *InMemoryBackend) DescribeIntegrations(integrationArn string) ([]Integra defer b.mu.RUnlock() if integrationArn != "" { - for _, ig := range b.integrations { + for _, ig := range b.integrations.All() { if ig.IntegrationArn == integrationArn { cp := *ig @@ -325,9 +325,9 @@ func (b *InMemoryBackend) DescribeIntegrations(integrationArn string) ([]Integra return nil, fmt.Errorf("%w: integration %s not found", ErrIntegrationNotFound, integrationArn) } - result := make([]Integration, 0, len(b.integrations)) + result := make([]Integration, 0, b.integrations.Len()) - for _, ig := range b.integrations { + for _, ig := range b.integrations.All() { result = append(result, *ig) } @@ -343,7 +343,7 @@ func (b *InMemoryBackend) ModifyIntegration(integrationArn, description string) b.mu.Lock("ModifyIntegration") defer b.mu.Unlock() - for _, ig := range b.integrations { + for _, ig := range b.integrations.All() { if ig.IntegrationArn == integrationArn { ig.Description = description cp := *ig @@ -368,7 +368,7 @@ func (b *InMemoryBackend) CreateIdcApplication( b.mu.Lock("CreateIdcApplication") defer b.mu.Unlock() - if _, exists := b.idcApplications[appName]; exists { + if _, exists := b.idcApplications.Get(appName); exists { return nil, fmt.Errorf("%w: application %s already exists", ErrIdcApplicationAlreadyExists, appName) } @@ -380,7 +380,7 @@ func (b *InMemoryBackend) CreateIdcApplication( IdcDisplayName: idcDisplayName, IamRoleArn: iamRoleArn, } - b.idcApplications[appName] = app + b.idcApplications.Put(app) cp := *app @@ -396,9 +396,9 @@ func (b *InMemoryBackend) DeleteIdcApplication(appArn string) error { b.mu.Lock("DeleteIdcApplication") defer b.mu.Unlock() - for name, app := range b.idcApplications { - if app.IdcApplicationArn == appArn || name == appArn { - delete(b.idcApplications, name) + for _, app := range b.idcApplications.All() { + if app.IdcApplicationArn == appArn || app.IdcApplicationName == appArn { + b.idcApplications.Delete(app.IdcApplicationName) return nil } @@ -413,7 +413,7 @@ func (b *InMemoryBackend) DescribeIdcApplications(appArn string) ([]IdcApplicati defer b.mu.RUnlock() if appArn != "" { - for _, app := range b.idcApplications { + for _, app := range b.idcApplications.All() { if app.IdcApplicationArn == appArn { cp := *app @@ -424,9 +424,9 @@ func (b *InMemoryBackend) DescribeIdcApplications(appArn string) ([]IdcApplicati return nil, fmt.Errorf("%w: application %s not found", ErrIdcApplicationNotFound, appArn) } - result := make([]IdcApplication, 0, len(b.idcApplications)) + result := make([]IdcApplication, 0, b.idcApplications.Len()) - for _, app := range b.idcApplications { + for _, app := range b.idcApplications.All() { result = append(result, *app) } @@ -444,7 +444,7 @@ func (b *InMemoryBackend) ModifyIdcApplication( b.mu.Lock("ModifyIdcApplication") defer b.mu.Unlock() - for _, app := range b.idcApplications { + for _, app := range b.idcApplications.All() { if app.IdcApplicationArn == appArn { if idcDisplayName != "" { app.IdcDisplayName = idcDisplayName diff --git a/services/redshift/backend_cluster_mgmt.go b/services/redshift/backend_cluster_mgmt.go index 84f16ac4f..31f4c8acf 100644 --- a/services/redshift/backend_cluster_mgmt.go +++ b/services/redshift/backend_cluster_mgmt.go @@ -24,7 +24,7 @@ func (b *InMemoryBackend) ModifyCluster( b.mu.Lock("ModifyCluster") defer b.mu.Unlock() - cluster, exists := b.clusters[id] + cluster, exists := b.clusters.Get(id) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } @@ -80,7 +80,7 @@ func (b *InMemoryBackend) RebootCluster(id string) (*Cluster, error) { b.mu.Lock("RebootCluster") defer b.mu.Unlock() - cluster, exists := b.clusters[id] + cluster, exists := b.clusters.Get(id) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } @@ -102,7 +102,7 @@ func (b *InMemoryBackend) PauseCluster(id string) (*Cluster, error) { b.mu.Lock("PauseCluster") defer b.mu.Unlock() - cluster, exists := b.clusters[id] + cluster, exists := b.clusters.Get(id) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } @@ -122,7 +122,7 @@ func (b *InMemoryBackend) ResumeCluster(id string) (*Cluster, error) { b.mu.Lock("ResumeCluster") defer b.mu.Unlock() - cluster, exists := b.clusters[id] + cluster, exists := b.clusters.Get(id) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } @@ -146,7 +146,7 @@ func (b *InMemoryBackend) ResizeCluster( b.mu.Lock("ResizeCluster") defer b.mu.Unlock() - cluster, exists := b.clusters[id] + cluster, exists := b.clusters.Get(id) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } @@ -177,7 +177,7 @@ func (b *InMemoryBackend) RotateEncryptionKey(id string) (*Cluster, error) { b.mu.Lock("RotateEncryptionKey") defer b.mu.Unlock() - cluster, exists := b.clusters[id] + cluster, exists := b.clusters.Get(id) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } @@ -197,7 +197,7 @@ func (b *InMemoryBackend) ModifyClusterIamRoles(id string, addRoles, removeRoles b.mu.Lock("ModifyClusterIamRoles") defer b.mu.Unlock() - cluster, exists := b.clusters[id] + cluster, exists := b.clusters.Get(id) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } @@ -241,7 +241,7 @@ func (b *InMemoryBackend) ModifyClusterMaintenance( b.mu.Lock("ModifyClusterMaintenance") defer b.mu.Unlock() - cluster, exists := b.clusters[id] + cluster, exists := b.clusters.Get(id) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } diff --git a/services/redshift/backend_events.go b/services/redshift/backend_events.go index 4b996b415..a215eea34 100644 --- a/services/redshift/backend_events.go +++ b/services/redshift/backend_events.go @@ -50,7 +50,7 @@ func (b *InMemoryBackend) EnableLogging(clusterID, bucketName, s3KeyPrefix strin b.mu.Lock("EnableLogging") defer b.mu.Unlock() - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -76,7 +76,7 @@ func (b *InMemoryBackend) DisableLogging(clusterID string) (*LoggingStatus, erro b.mu.Lock("DisableLogging") defer b.mu.Unlock() - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -98,7 +98,7 @@ func (b *InMemoryBackend) DescribeEvents(sourceIdentifier, sourceType string) ([ var result []Event - for _, e := range b.events { + for _, e := range b.events.All() { if sourceIdentifier != "" && e.SourceIdentifier != sourceIdentifier { continue } @@ -129,7 +129,7 @@ func (b *InMemoryBackend) CreateEventSubscription( b.mu.Lock("CreateEventSubscription") defer b.mu.Unlock() - if _, exists := b.eventSubscriptions[subscriptionName]; exists { + if _, exists := b.eventSubscriptions.Get(subscriptionName); exists { return nil, fmt.Errorf( "%w: subscription %s already exists", ErrEventSubscriptionAlreadyExists, @@ -155,7 +155,7 @@ func (b *InMemoryBackend) CreateEventSubscription( Severity: severity, Enabled: enabled, } - b.eventSubscriptions[subscriptionName] = sub + b.eventSubscriptions.Put(sub) cp := cloneEventSubscription(sub) @@ -171,11 +171,11 @@ func (b *InMemoryBackend) DeleteEventSubscription(subscriptionName string) error b.mu.Lock("DeleteEventSubscription") defer b.mu.Unlock() - if _, exists := b.eventSubscriptions[subscriptionName]; !exists { + if _, exists := b.eventSubscriptions.Get(subscriptionName); !exists { return fmt.Errorf("%w: subscription %s not found", ErrEventSubscriptionNotFound, subscriptionName) } - delete(b.eventSubscriptions, subscriptionName) + b.eventSubscriptions.Delete(subscriptionName) return nil } @@ -186,7 +186,7 @@ func (b *InMemoryBackend) DescribeEventSubscriptions(subscriptionName string) ([ defer b.mu.RUnlock() if subscriptionName != "" { - sub, exists := b.eventSubscriptions[subscriptionName] + sub, exists := b.eventSubscriptions.Get(subscriptionName) if !exists { return nil, fmt.Errorf("%w: subscription %s not found", ErrEventSubscriptionNotFound, subscriptionName) } @@ -194,8 +194,8 @@ func (b *InMemoryBackend) DescribeEventSubscriptions(subscriptionName string) ([ return []EventSubscription{*cloneEventSubscription(sub)}, nil } - result := make([]EventSubscription, 0, len(b.eventSubscriptions)) - for _, sub := range b.eventSubscriptions { + result := make([]EventSubscription, 0, b.eventSubscriptions.Len()) + for _, sub := range b.eventSubscriptions.All() { result = append(result, *cloneEventSubscription(sub)) } @@ -215,7 +215,7 @@ func (b *InMemoryBackend) ModifyEventSubscription( b.mu.Lock("ModifyEventSubscription") defer b.mu.Unlock() - sub, exists := b.eventSubscriptions[subscriptionName] + sub, exists := b.eventSubscriptions.Get(subscriptionName) if !exists { return nil, fmt.Errorf("%w: subscription %s not found", ErrEventSubscriptionNotFound, subscriptionName) } diff --git a/services/redshift/backend_param_groups.go b/services/redshift/backend_param_groups.go index b79e66f53..b5121da68 100644 --- a/services/redshift/backend_param_groups.go +++ b/services/redshift/backend_param_groups.go @@ -107,7 +107,7 @@ func (b *InMemoryBackend) CreateClusterParameterGroup( b.mu.Lock("CreateClusterParameterGroup") defer b.mu.Unlock() - if _, exists := b.parameterGroups[name]; exists { + if _, exists := b.parameterGroups.Get(name); exists { return nil, fmt.Errorf("%w: parameter group %s already exists", ErrParameterGroupAlreadyExists, name) } @@ -118,7 +118,7 @@ func (b *InMemoryBackend) CreateClusterParameterGroup( Parameters: defaultParameters(), CreatedAt: time.Now(), } - b.parameterGroups[name] = pg + b.parameterGroups.Put(pg) cp := cloneParameterGroup(pg) @@ -134,11 +134,11 @@ func (b *InMemoryBackend) DeleteClusterParameterGroup(name string) error { b.mu.Lock("DeleteClusterParameterGroup") defer b.mu.Unlock() - if _, exists := b.parameterGroups[name]; !exists { + if _, exists := b.parameterGroups.Get(name); !exists { return fmt.Errorf("%w: parameter group %s not found", ErrParameterGroupNotFound, name) } - delete(b.parameterGroups, name) + b.parameterGroups.Delete(name) return nil } @@ -149,7 +149,7 @@ func (b *InMemoryBackend) DescribeClusterParameterGroups(name string) ([]Cluster defer b.mu.RUnlock() if name != "" { - pg, exists := b.parameterGroups[name] + pg, exists := b.parameterGroups.Get(name) if !exists { return nil, fmt.Errorf("%w: parameter group %s not found", ErrParameterGroupNotFound, name) } @@ -159,8 +159,8 @@ func (b *InMemoryBackend) DescribeClusterParameterGroups(name string) ([]Cluster return []ClusterParameterGroup{cp}, nil } - result := make([]ClusterParameterGroup, 0, len(b.parameterGroups)) - for _, pg := range b.parameterGroups { + result := make([]ClusterParameterGroup, 0, b.parameterGroups.Len()) + for _, pg := range b.parameterGroups.All() { result = append(result, cloneParameterGroup(pg)) } @@ -176,7 +176,7 @@ func (b *InMemoryBackend) DescribeClusterParameters(groupName string) ([]Cluster b.mu.RLock("DescribeClusterParameters") defer b.mu.RUnlock() - pg, exists := b.parameterGroups[groupName] + pg, exists := b.parameterGroups.Get(groupName) if !exists { return nil, fmt.Errorf("%w: parameter group %s not found", ErrParameterGroupNotFound, groupName) } @@ -199,7 +199,7 @@ func (b *InMemoryBackend) ModifyClusterParameterGroup( b.mu.Lock("ModifyClusterParameterGroup") defer b.mu.Unlock() - pg, exists := b.parameterGroups[groupName] + pg, exists := b.parameterGroups.Get(groupName) if !exists { return nil, fmt.Errorf("%w: parameter group %s not found", ErrParameterGroupNotFound, groupName) } @@ -242,7 +242,7 @@ func (b *InMemoryBackend) ResetClusterParameterGroup( b.mu.Lock("ResetClusterParameterGroup") defer b.mu.Unlock() - pg, exists := b.parameterGroups[groupName] + pg, exists := b.parameterGroups.Get(groupName) if !exists { return nil, fmt.Errorf("%w: parameter group %s not found", ErrParameterGroupNotFound, groupName) } diff --git a/services/redshift/backend_reconciler.go b/services/redshift/backend_reconciler.go index 9e80bb24e..142cbdb69 100644 --- a/services/redshift/backend_reconciler.go +++ b/services/redshift/backend_reconciler.go @@ -87,14 +87,14 @@ func (b *InMemoryBackend) applyDueTransitionsLocked(now time.Time) { delete(b.clusterTransitions, id) - c, ok := b.clusters[id] + c, ok := b.clusters.Get(id) if !ok { continue } if tr.remove { c.Tags.Close() - delete(b.clusters, id) + b.clusters.Delete(id) if b.dnsRegistrar != nil { b.dnsRegistrar.Deregister(c.Endpoint) diff --git a/services/redshift/backend_refinement2.go b/services/redshift/backend_refinement2.go index 36689b955..a45fdb01b 100644 --- a/services/redshift/backend_refinement2.go +++ b/services/redshift/backend_refinement2.go @@ -25,11 +25,11 @@ func (b *InMemoryBackend) DeletePartner(_, clusterID, databaseName, partnerName defer b.mu.Unlock() key := partnerKey(clusterID, databaseName, partnerName) - if _, exists := b.partners[key]; !exists { + if _, exists := b.partners.Get(key); !exists { return fmt.Errorf("%w: partner %s not found", ErrPartnerNotFound, partnerName) } - delete(b.partners, key) + b.partners.Delete(key) return nil } @@ -41,7 +41,7 @@ func (b *InMemoryBackend) DescribePartners(_, clusterID, databaseName, partnerNa var result []Partner - for _, p := range b.partners { + for _, p := range b.partners.All() { if clusterID != "" && p.ClusterIdentifier != clusterID { continue } @@ -74,7 +74,7 @@ func (b *InMemoryBackend) UpdatePartnerStatus( key := partnerKey(clusterID, databaseName, partnerName) - p, exists := b.partners[key] + p, exists := b.partners.Get(key) if !exists { return nil, fmt.Errorf("%w: partner %s not found", ErrPartnerNotFound, partnerName) } @@ -98,7 +98,7 @@ func (b *InMemoryBackend) DescribeDataShares(dataShareArn string) ([]DataShare, defer b.mu.RUnlock() if dataShareArn != "" { - ds, exists := b.dataShares[dataShareArn] + ds, exists := b.dataShares.Get(dataShareArn) if !exists { return nil, fmt.Errorf("%w: data share %s not found", ErrDataShareNotFound, dataShareArn) } @@ -106,9 +106,9 @@ func (b *InMemoryBackend) DescribeDataShares(dataShareArn string) ([]DataShare, return []DataShare{*cloneDataShare(ds)}, nil } - result := make([]DataShare, 0, len(b.dataShares)) + result := make([]DataShare, 0, b.dataShares.Len()) - for _, ds := range b.dataShares { + for _, ds := range b.dataShares.All() { result = append(result, *cloneDataShare(ds)) } @@ -122,7 +122,7 @@ func (b *InMemoryBackend) DescribeDataSharesForConsumer(consumerArn, status stri var result []DataShare - for _, ds := range b.dataShares { + for _, ds := range b.dataShares.All() { for _, a := range ds.DataShareAssociations { if consumerArn != "" && a.ConsumerIdentifier != consumerArn { continue @@ -148,7 +148,7 @@ func (b *InMemoryBackend) DescribeDataSharesForProducer(producerArn, status stri var result []DataShare - for _, ds := range b.dataShares { + for _, ds := range b.dataShares.All() { if producerArn != "" && ds.ProducerArn != producerArn { continue } @@ -184,7 +184,7 @@ func (b *InMemoryBackend) DeauthorizeDataShare(dataShareArn, consumerIdentifier b.mu.Lock("DeauthorizeDataShare") defer b.mu.Unlock() - ds, exists := b.dataShares[dataShareArn] + ds, exists := b.dataShares.Get(dataShareArn) if !exists { return nil, fmt.Errorf("%w: data share %s not found", ErrDataShareNotFound, dataShareArn) } @@ -210,7 +210,7 @@ func (b *InMemoryBackend) DisassociateDataShareConsumer( b.mu.Lock("DisassociateDataShareConsumer") defer b.mu.Unlock() - ds, exists := b.dataShares[dataShareArn] + ds, exists := b.dataShares.Get(dataShareArn) if !exists { return nil, fmt.Errorf("%w: data share %s not found", ErrDataShareNotFound, dataShareArn) } @@ -237,7 +237,7 @@ func (b *InMemoryBackend) RejectDataShare(dataShareArn string) (*DataShare, erro b.mu.Lock("RejectDataShare") defer b.mu.Unlock() - ds, exists := b.dataShares[dataShareArn] + ds, exists := b.dataShares.Get(dataShareArn) if !exists { return nil, fmt.Errorf("%w: data share %s not found", ErrDataShareNotFound, dataShareArn) } @@ -261,7 +261,7 @@ func (b *InMemoryBackend) DescribeEndpointAuthorization( var result []EndpointAuthorization - for _, ea := range b.endpointAuths { + for _, ea := range b.endpointAuths.All() { if clusterID != "" && ea.ClusterIdentifier != clusterID { continue } @@ -298,7 +298,7 @@ func (b *InMemoryBackend) RevokeEndpointAccess( key := endpointAuthKey(clusterID, account) - ea, exists := b.endpointAuths[key] + ea, exists := b.endpointAuths.Get(key) if !exists { return nil, fmt.Errorf( "%w: endpoint authorization for cluster %s account %s not found", @@ -313,7 +313,7 @@ func (b *InMemoryBackend) RevokeEndpointAccess( cp := cloneEndpointAuth(ea) if force { - delete(b.endpointAuths, key) + b.endpointAuths.Delete(key) } return cp, nil @@ -353,7 +353,7 @@ func (b *InMemoryBackend) RevokeSnapshotAccess(snapshotID, accountWithRestoreAcc b.mu.Lock("RevokeSnapshotAccess") defer b.mu.Unlock() - snap, exists := b.snapshots[snapshotID] + snap, exists := b.snapshots.Get(snapshotID) if !exists { return nil, fmt.Errorf("%w: snapshot %s not found", ErrSnapshotNotFound, snapshotID) } @@ -393,7 +393,7 @@ func (b *InMemoryBackend) ModifyClusterSnapshot(snapshotID string, retentionPeri b.mu.Lock("ModifyClusterSnapshot") defer b.mu.Unlock() - snap, exists := b.snapshots[snapshotID] + snap, exists := b.snapshots.Get(snapshotID) if !exists { return nil, fmt.Errorf("%w: snapshot %s not found", ErrSnapshotNotFound, snapshotID) } @@ -421,7 +421,7 @@ func (b *InMemoryBackend) GetClusterCredentials( b.mu.RLock("GetClusterCredentials") defer b.mu.RUnlock() - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } diff --git a/services/redshift/backend_refinement3.go b/services/redshift/backend_refinement3.go index 636611194..84223959c 100644 --- a/services/redshift/backend_refinement3.go +++ b/services/redshift/backend_refinement3.go @@ -19,7 +19,7 @@ func (b *InMemoryBackend) CreateSnapshotCopyGrant( b.mu.Lock("CreateSnapshotCopyGrant") defer b.mu.Unlock() - if _, exists := b.snapshotCopyGrants[name]; exists { + if _, exists := b.snapshotCopyGrants.Get(name); exists { return nil, fmt.Errorf("%w: grant %s already exists", ErrSnapshotCopyGrantAlreadyExists, name) } @@ -28,7 +28,7 @@ func (b *InMemoryBackend) CreateSnapshotCopyGrant( KMSKeyID: kmsKeyID, Tags: tagMap, } - b.snapshotCopyGrants[name] = grant + b.snapshotCopyGrants.Put(grant) cp := *grant @@ -44,11 +44,11 @@ func (b *InMemoryBackend) DeleteSnapshotCopyGrant(name string) error { b.mu.Lock("DeleteSnapshotCopyGrant") defer b.mu.Unlock() - if _, exists := b.snapshotCopyGrants[name]; !exists { + if _, exists := b.snapshotCopyGrants.Get(name); !exists { return fmt.Errorf("%w: grant %s not found", ErrSnapshotCopyGrantNotFound, name) } - delete(b.snapshotCopyGrants, name) + b.snapshotCopyGrants.Delete(name) return nil } @@ -59,7 +59,7 @@ func (b *InMemoryBackend) DescribeSnapshotCopyGrants(name string) ([]SnapshotCop defer b.mu.RUnlock() if name != "" { - g, exists := b.snapshotCopyGrants[name] + g, exists := b.snapshotCopyGrants.Get(name) if !exists { return nil, fmt.Errorf("%w: grant %s not found", ErrSnapshotCopyGrantNotFound, name) } @@ -69,9 +69,9 @@ func (b *InMemoryBackend) DescribeSnapshotCopyGrants(name string) ([]SnapshotCop return []SnapshotCopyGrant{cp}, nil } - result := make([]SnapshotCopyGrant, 0, len(b.snapshotCopyGrants)) + result := make([]SnapshotCopyGrant, 0, b.snapshotCopyGrants.Len()) - for _, g := range b.snapshotCopyGrants { + for _, g := range b.snapshotCopyGrants.All() { result = append(result, *g) } @@ -96,7 +96,7 @@ func (b *InMemoryBackend) EnableSnapshotCopy( b.mu.Lock("EnableSnapshotCopy") defer b.mu.Unlock() - cluster, exists := b.clusters[clusterID] + cluster, exists := b.clusters.Get(clusterID) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -133,7 +133,7 @@ func (b *InMemoryBackend) DisableSnapshotCopy(clusterID string) (*Cluster, error b.mu.Lock("DisableSnapshotCopy") defer b.mu.Unlock() - cluster, exists := b.clusters[clusterID] + cluster, exists := b.clusters.Get(clusterID) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -158,7 +158,7 @@ func (b *InMemoryBackend) ModifySnapshotCopyRetentionPeriod(clusterID string, re b.mu.Lock("ModifySnapshotCopyRetentionPeriod") defer b.mu.Unlock() - cluster, exists := b.clusters[clusterID] + cluster, exists := b.clusters.Get(clusterID) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -189,7 +189,7 @@ func (b *InMemoryBackend) CreateSnapshotSchedule( b.mu.Lock("CreateSnapshotSchedule") defer b.mu.Unlock() - if _, exists := b.snapshotSchedules[scheduleID]; exists { + if _, exists := b.snapshotSchedules.Get(scheduleID); exists { return nil, fmt.Errorf("%w: schedule %s already exists", ErrSnapshotScheduleAlreadyExists, scheduleID) } @@ -202,7 +202,7 @@ func (b *InMemoryBackend) CreateSnapshotSchedule( ScheduleDefinitions: defCopy, Tags: tagMap, } - b.snapshotSchedules[scheduleID] = sched + b.snapshotSchedules.Put(sched) cp := cloneSnapshotSchedule(sched) @@ -218,11 +218,11 @@ func (b *InMemoryBackend) DeleteSnapshotSchedule(scheduleID string) error { b.mu.Lock("DeleteSnapshotSchedule") defer b.mu.Unlock() - if _, exists := b.snapshotSchedules[scheduleID]; !exists { + if _, exists := b.snapshotSchedules.Get(scheduleID); !exists { return fmt.Errorf("%w: schedule %s not found", ErrSnapshotScheduleNotFound, scheduleID) } - delete(b.snapshotSchedules, scheduleID) + b.snapshotSchedules.Delete(scheduleID) return nil } @@ -233,7 +233,7 @@ func (b *InMemoryBackend) DescribeSnapshotSchedules(scheduleID string) ([]Snapsh defer b.mu.RUnlock() if scheduleID != "" { - s, exists := b.snapshotSchedules[scheduleID] + s, exists := b.snapshotSchedules.Get(scheduleID) if !exists { return nil, fmt.Errorf("%w: schedule %s not found", ErrSnapshotScheduleNotFound, scheduleID) } @@ -241,9 +241,9 @@ func (b *InMemoryBackend) DescribeSnapshotSchedules(scheduleID string) ([]Snapsh return []SnapshotSchedule{*cloneSnapshotSchedule(s)}, nil } - result := make([]SnapshotSchedule, 0, len(b.snapshotSchedules)) + result := make([]SnapshotSchedule, 0, b.snapshotSchedules.Len()) - for _, s := range b.snapshotSchedules { + for _, s := range b.snapshotSchedules.All() { result = append(result, *cloneSnapshotSchedule(s)) } @@ -259,7 +259,7 @@ func (b *InMemoryBackend) ModifySnapshotSchedule(scheduleID string, definitions b.mu.Lock("ModifySnapshotSchedule") defer b.mu.Unlock() - sched, exists := b.snapshotSchedules[scheduleID] + sched, exists := b.snapshotSchedules.Get(scheduleID) if !exists { return nil, fmt.Errorf("%w: schedule %s not found", ErrSnapshotScheduleNotFound, scheduleID) } @@ -280,12 +280,12 @@ func (b *InMemoryBackend) ModifyClusterSnapshotSchedule(clusterID, scheduleID st b.mu.Lock("ModifyClusterSnapshotSchedule") defer b.mu.Unlock() - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } if !disassociate && scheduleID != "" { - if _, exists := b.snapshotSchedules[scheduleID]; !exists { + if _, exists := b.snapshotSchedules.Get(scheduleID); !exists { return fmt.Errorf("%w: schedule %s not found", ErrSnapshotScheduleNotFound, scheduleID) } } @@ -308,7 +308,7 @@ func (b *InMemoryBackend) CreateUsageLimit( b.mu.Lock("CreateUsageLimit") defer b.mu.Unlock() - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -323,7 +323,7 @@ func (b *InMemoryBackend) CreateUsageLimit( Amount: amount, Tags: tagMap, } - b.usageLimits[id] = ul + b.usageLimits.Put(ul) cp := *ul @@ -339,11 +339,11 @@ func (b *InMemoryBackend) DeleteUsageLimit(usageLimitID string) error { b.mu.Lock("DeleteUsageLimit") defer b.mu.Unlock() - if _, exists := b.usageLimits[usageLimitID]; !exists { + if _, exists := b.usageLimits.Get(usageLimitID); !exists { return fmt.Errorf("%w: usage limit %s not found", ErrUsageLimitNotFound, usageLimitID) } - delete(b.usageLimits, usageLimitID) + b.usageLimits.Delete(usageLimitID) return nil } @@ -353,9 +353,9 @@ func (b *InMemoryBackend) DescribeUsageLimits(clusterID, featureType string) ([] b.mu.RLock("DescribeUsageLimits") defer b.mu.RUnlock() - result := make([]UsageLimit, 0, len(b.usageLimits)) + result := make([]UsageLimit, 0, b.usageLimits.Len()) - for _, ul := range b.usageLimits { + for _, ul := range b.usageLimits.All() { if clusterID != "" && ul.ClusterIdentifier != clusterID { continue } @@ -380,7 +380,7 @@ func (b *InMemoryBackend) ModifyUsageLimit(usageLimitID, breachAction string, am b.mu.Lock("ModifyUsageLimit") defer b.mu.Unlock() - ul, exists := b.usageLimits[usageLimitID] + ul, exists := b.usageLimits.Get(usageLimitID) if !exists { return nil, fmt.Errorf("%w: usage limit %s not found", ErrUsageLimitNotFound, usageLimitID) } @@ -409,7 +409,7 @@ func (b *InMemoryBackend) CreateAuthenticationProfile(name, content string) (*Au b.mu.Lock("CreateAuthenticationProfile") defer b.mu.Unlock() - if _, exists := b.authProfiles[name]; exists { + if _, exists := b.authProfiles.Get(name); exists { return nil, fmt.Errorf("%w: profile %s already exists", ErrAuthProfileAlreadyExists, name) } @@ -417,7 +417,7 @@ func (b *InMemoryBackend) CreateAuthenticationProfile(name, content string) (*Au AuthenticationProfileName: name, AuthenticationProfileContent: content, } - b.authProfiles[name] = ap + b.authProfiles.Put(ap) cp := *ap @@ -433,11 +433,11 @@ func (b *InMemoryBackend) DeleteAuthenticationProfile(name string) error { b.mu.Lock("DeleteAuthenticationProfile") defer b.mu.Unlock() - if _, exists := b.authProfiles[name]; !exists { + if _, exists := b.authProfiles.Get(name); !exists { return fmt.Errorf("%w: profile %s not found", ErrAuthProfileNotFound, name) } - delete(b.authProfiles, name) + b.authProfiles.Delete(name) return nil } @@ -448,7 +448,7 @@ func (b *InMemoryBackend) DescribeAuthenticationProfiles(name string) ([]Authent defer b.mu.RUnlock() if name != "" { - ap, exists := b.authProfiles[name] + ap, exists := b.authProfiles.Get(name) if !exists { return nil, fmt.Errorf("%w: profile %s not found", ErrAuthProfileNotFound, name) } @@ -458,9 +458,9 @@ func (b *InMemoryBackend) DescribeAuthenticationProfiles(name string) ([]Authent return []AuthenticationProfile{cp}, nil } - result := make([]AuthenticationProfile, 0, len(b.authProfiles)) + result := make([]AuthenticationProfile, 0, b.authProfiles.Len()) - for _, ap := range b.authProfiles { + for _, ap := range b.authProfiles.All() { cp := *ap result = append(result, cp) } @@ -477,7 +477,7 @@ func (b *InMemoryBackend) ModifyAuthenticationProfile(name, content string) (*Au b.mu.Lock("ModifyAuthenticationProfile") defer b.mu.Unlock() - ap, exists := b.authProfiles[name] + ap, exists := b.authProfiles.Get(name) if !exists { return nil, fmt.Errorf("%w: profile %s not found", ErrAuthProfileNotFound, name) } @@ -499,7 +499,7 @@ func (b *InMemoryBackend) GetResourcePolicy(resourceArn string) (*ResourcePolicy b.mu.RLock("GetResourcePolicy") defer b.mu.RUnlock() - rp, exists := b.resourcePolicies[resourceArn] + rp, exists := b.resourcePolicies.Get(resourceArn) if !exists { return nil, fmt.Errorf("%w: resource policy for %s not found", ErrResourcePolicyNotFound, resourceArn) } @@ -522,7 +522,7 @@ func (b *InMemoryBackend) PutResourcePolicy(resourceArn, policy string) (*Resour ResourceArn: resourceArn, Policy: policy, } - b.resourcePolicies[resourceArn] = rp + b.resourcePolicies.Put(rp) cp := *rp @@ -538,11 +538,11 @@ func (b *InMemoryBackend) DeleteResourcePolicy(resourceArn string) error { b.mu.Lock("DeleteResourcePolicy") defer b.mu.Unlock() - if _, exists := b.resourcePolicies[resourceArn]; !exists { + if _, exists := b.resourcePolicies.Get(resourceArn); !exists { return fmt.Errorf("%w: resource policy for %s not found", ErrResourcePolicyNotFound, resourceArn) } - delete(b.resourcePolicies, resourceArn) + b.resourcePolicies.Delete(resourceArn) return nil } @@ -558,7 +558,7 @@ func (b *InMemoryBackend) GetClusterCredentialsWithIAM(clusterID, _ string) (*Cl b.mu.RLock("GetClusterCredentialsWithIAM") defer b.mu.RUnlock() - if _, exists := b.clusters[clusterID]; !exists { + if _, exists := b.clusters.Get(clusterID); !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -580,7 +580,7 @@ func (b *InMemoryBackend) FailoverPrimaryCompute(clusterID string) (*Cluster, er b.mu.Lock("FailoverPrimaryCompute") defer b.mu.Unlock() - cluster, exists := b.clusters[clusterID] + cluster, exists := b.clusters.Get(clusterID) if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } @@ -597,9 +597,9 @@ func (b *InMemoryBackend) DescribeTableRestoreStatus(clusterID string) ([]TableR b.mu.RLock("DescribeTableRestoreStatus") defer b.mu.RUnlock() - result := make([]TableRestoreStatus, 0, len(b.tableRestores)) + result := make([]TableRestoreStatus, 0, b.tableRestores.Len()) - for _, tr := range b.tableRestores { + for _, tr := range b.tableRestores.All() { if clusterID != "" && tr.ClusterIdentifier != clusterID { continue } diff --git a/services/redshift/backend_reserved_nodes.go b/services/redshift/backend_reserved_nodes.go index 5138dc9a4..829807b18 100644 --- a/services/redshift/backend_reserved_nodes.go +++ b/services/redshift/backend_reserved_nodes.go @@ -102,7 +102,7 @@ func (b *InMemoryBackend) DescribeReservedNodes(reservedNodeID string) ([]Reserv defer b.mu.RUnlock() if reservedNodeID != "" { - node, exists := b.reservedNodes[reservedNodeID] + node, exists := b.reservedNodes.Get(reservedNodeID) if !exists { return nil, fmt.Errorf("%w: reserved node %s not found", ErrReservedNodeNotFound, reservedNodeID) } @@ -112,8 +112,8 @@ func (b *InMemoryBackend) DescribeReservedNodes(reservedNodeID string) ([]Reserv return []ReservedNode{cp}, nil } - result := make([]ReservedNode, 0, len(b.reservedNodes)) - for _, node := range b.reservedNodes { + result := make([]ReservedNode, 0, b.reservedNodes.Len()) + for _, node := range b.reservedNodes.All() { result = append(result, *node) } @@ -178,7 +178,7 @@ func (b *InMemoryBackend) PurchaseReservedNodeOffering( b.mu.Lock("PurchaseReservedNodeOffering") defer b.mu.Unlock() - if _, exists := b.reservedNodes[reservedNodeID]; exists { + if _, exists := b.reservedNodes.Get(reservedNodeID); exists { return nil, fmt.Errorf("%w: reserved node %s already exists", ErrReservedNodeAlreadyExists, reservedNodeID) } @@ -195,7 +195,7 @@ func (b *InMemoryBackend) PurchaseReservedNodeOffering( State: "payment-pending", OfferingType: offering.OfferingType, } - b.reservedNodes[reservedNodeID] = node + b.reservedNodes.Put(node) cp := *node @@ -212,7 +212,7 @@ func (b *InMemoryBackend) DescribeReservedNodeExchangeStatus(reservedNodeID stri b.mu.RLock("DescribeReservedNodeExchangeStatus") defer b.mu.RUnlock() - if _, exists := b.reservedNodes[reservedNodeID]; !exists { + if _, exists := b.reservedNodes.Get(reservedNodeID); !exists { return "", fmt.Errorf("%w: reserved node %s not found", ErrReservedNodeNotFound, reservedNodeID) } @@ -228,7 +228,7 @@ func (b *InMemoryBackend) GetReservedNodeExchangeOfferings(reservedNodeID string b.mu.RLock("GetReservedNodeExchangeOfferings") defer b.mu.RUnlock() - if _, exists := b.reservedNodes[reservedNodeID]; !exists { + if _, exists := b.reservedNodes.Get(reservedNodeID); !exists { return nil, fmt.Errorf("%w: reserved node %s not found", ErrReservedNodeNotFound, reservedNodeID) } diff --git a/services/redshift/backend_security_groups.go b/services/redshift/backend_security_groups.go index a5205b484..5fe5ebf5e 100644 --- a/services/redshift/backend_security_groups.go +++ b/services/redshift/backend_security_groups.go @@ -13,7 +13,7 @@ func (b *InMemoryBackend) CreateClusterSecurityGroup(name, description string) ( b.mu.Lock("CreateClusterSecurityGroup") defer b.mu.Unlock() - if _, exists := b.securityGroups[name]; exists { + if _, exists := b.securityGroups.Get(name); exists { return nil, fmt.Errorf("%w: security group %s already exists", ErrSecurityGroupAlreadyExists, name) } @@ -23,7 +23,7 @@ func (b *InMemoryBackend) CreateClusterSecurityGroup(name, description string) ( IPRanges: []IPRange{}, EC2SecurityGroups: []EC2SecurityGroup{}, } - b.securityGroups[name] = sg + b.securityGroups.Put(sg) return cloneSecurityGroup(sg), nil } @@ -37,11 +37,11 @@ func (b *InMemoryBackend) DeleteClusterSecurityGroup(name string) error { b.mu.Lock("DeleteClusterSecurityGroup") defer b.mu.Unlock() - if _, exists := b.securityGroups[name]; !exists { + if _, exists := b.securityGroups.Get(name); !exists { return fmt.Errorf("%w: security group %s not found", ErrSecurityGroupNotFound, name) } - delete(b.securityGroups, name) + b.securityGroups.Delete(name) return nil } @@ -52,7 +52,7 @@ func (b *InMemoryBackend) DescribeClusterSecurityGroups(name string) ([]ClusterS defer b.mu.RUnlock() if name != "" { - sg, exists := b.securityGroups[name] + sg, exists := b.securityGroups.Get(name) if !exists { return nil, fmt.Errorf("%w: security group %s not found", ErrSecurityGroupNotFound, name) } @@ -60,8 +60,8 @@ func (b *InMemoryBackend) DescribeClusterSecurityGroups(name string) ([]ClusterS return []ClusterSecurityGroup{*cloneSecurityGroup(sg)}, nil } - result := make([]ClusterSecurityGroup, 0, len(b.securityGroups)) - for _, sg := range b.securityGroups { + result := make([]ClusterSecurityGroup, 0, b.securityGroups.Len()) + for _, sg := range b.securityGroups.All() { result = append(result, *cloneSecurityGroup(sg)) } @@ -82,7 +82,7 @@ func (b *InMemoryBackend) RevokeClusterSecurityGroupIngress( b.mu.Lock("RevokeClusterSecurityGroupIngress") defer b.mu.Unlock() - sg, exists := b.securityGroups[groupName] + sg, exists := b.securityGroups.Get(groupName) if !exists { return nil, fmt.Errorf("%w: security group %s not found", ErrSecurityGroupNotFound, groupName) } diff --git a/services/redshift/backend_serverless.go b/services/redshift/backend_serverless.go index 8d274ec14..552a5dea8 100644 --- a/services/redshift/backend_serverless.go +++ b/services/redshift/backend_serverless.go @@ -152,7 +152,7 @@ func (b *InMemoryBackend) CreateNamespace( b.mu.Lock("CreateNamespace") defer b.mu.Unlock() - if _, ok := b.slNamespaces[namespaceName]; ok { + if _, ok := b.slNamespaces.Get(namespaceName); ok { return nil, fmt.Errorf( "%w: namespace %q already exists", ErrNamespaceAlreadyExists, @@ -181,7 +181,7 @@ func (b *InMemoryBackend) CreateNamespace( IamRoles: rolesCopy, LogExports: exportsCopy, } - b.slNamespaces[namespaceName] = ns + b.slNamespaces.Put(ns) b.slNamespaceIdx.insert(namespaceName) return cloneNamespace(ns), nil @@ -192,7 +192,7 @@ func (b *InMemoryBackend) GetNamespace(namespaceName string) (*Namespace, error) b.mu.RLock("GetNamespace") defer b.mu.RUnlock() - ns, ok := b.slNamespaces[namespaceName] + ns, ok := b.slNamespaces.Get(namespaceName) if !ok { return nil, fmt.Errorf("%w: namespace %q not found", ErrNamespaceNotFound, namespaceName) } @@ -212,7 +212,7 @@ func (b *InMemoryBackend) ListNamespaces(maxResults int, nextToken string) ([]*N list := make([]*Namespace, 0, len(keys)) for _, name := range keys { - if ns, ok := b.slNamespaces[name]; ok { + if ns, ok := b.slNamespaces.Get(name); ok { list = append(list, cloneNamespace(ns)) } } @@ -252,7 +252,7 @@ func (b *InMemoryBackend) UpdateNamespace( b.mu.Lock("UpdateNamespace") defer b.mu.Unlock() - ns, ok := b.slNamespaces[namespaceName] + ns, ok := b.slNamespaces.Get(namespaceName) if !ok { return nil, fmt.Errorf("%w: namespace %q not found", ErrNamespaceNotFound, namespaceName) } @@ -289,13 +289,13 @@ func (b *InMemoryBackend) DeleteNamespace(namespaceName string) (*Namespace, err b.mu.Lock("DeleteNamespace") defer b.mu.Unlock() - ns, ok := b.slNamespaces[namespaceName] + ns, ok := b.slNamespaces.Get(namespaceName) if !ok { return nil, fmt.Errorf("%w: namespace %q not found", ErrNamespaceNotFound, namespaceName) } cp := cloneNamespace(ns) - delete(b.slNamespaces, namespaceName) + b.slNamespaces.Delete(namespaceName) b.slNamespaceIdx.remove(namespaceName) return cp, nil @@ -324,7 +324,7 @@ func (b *InMemoryBackend) CreateWorkgroup( b.mu.Lock("CreateWorkgroup") defer b.mu.Unlock() - if _, ok := b.slWorkgroups[workgroupName]; ok { + if _, ok := b.slWorkgroups.Get(workgroupName); ok { return nil, fmt.Errorf( "%w: workgroup %q already exists", ErrWorkgroupAlreadyExists, @@ -332,7 +332,7 @@ func (b *InMemoryBackend) CreateWorkgroup( ) } - if _, ok := b.slNamespaces[namespaceName]; !ok { + if _, ok := b.slNamespaces.Get(namespaceName); !ok { return nil, fmt.Errorf("%w: namespace %q not found", ErrNamespaceNotFound, namespaceName) } @@ -369,7 +369,7 @@ func (b *InMemoryBackend) CreateWorkgroup( SubnetIDs: subnetsCopy, SecurityGroupIDs: sgCopy, } - b.slWorkgroups[workgroupName] = wg + b.slWorkgroups.Put(wg) b.slWorkgroupIdx.insert(workgroupName) return cloneWorkgroup(wg), nil @@ -380,7 +380,7 @@ func (b *InMemoryBackend) GetWorkgroup(workgroupName string) (*Workgroup, error) b.mu.RLock("GetWorkgroup") defer b.mu.RUnlock() - wg, ok := b.slWorkgroups[workgroupName] + wg, ok := b.slWorkgroups.Get(workgroupName) if !ok { return nil, fmt.Errorf("%w: workgroup %q not found", ErrWorkgroupNotFound, workgroupName) } @@ -400,7 +400,7 @@ func (b *InMemoryBackend) ListWorkgroups(maxResults int, nextToken string) ([]*W list := make([]*Workgroup, 0, len(keys)) for _, name := range keys { - if wg, ok := b.slWorkgroups[name]; ok { + if wg, ok := b.slWorkgroups.Get(name); ok { list = append(list, cloneWorkgroup(wg)) } } @@ -441,7 +441,7 @@ func (b *InMemoryBackend) UpdateWorkgroup( b.mu.Lock("UpdateWorkgroup") defer b.mu.Unlock() - wg, ok := b.slWorkgroups[workgroupName] + wg, ok := b.slWorkgroups.Get(workgroupName) if !ok { return nil, fmt.Errorf("%w: workgroup %q not found", ErrWorkgroupNotFound, workgroupName) } @@ -470,13 +470,13 @@ func (b *InMemoryBackend) DeleteWorkgroup(workgroupName string) (*Workgroup, err b.mu.Lock("DeleteWorkgroup") defer b.mu.Unlock() - wg, ok := b.slWorkgroups[workgroupName] + wg, ok := b.slWorkgroups.Get(workgroupName) if !ok { return nil, fmt.Errorf("%w: workgroup %q not found", ErrWorkgroupNotFound, workgroupName) } cp := cloneWorkgroup(wg) - delete(b.slWorkgroups, workgroupName) + b.slWorkgroups.Delete(workgroupName) b.slWorkgroupIdx.remove(workgroupName) return cp, nil @@ -504,7 +504,7 @@ func (b *InMemoryBackend) GetCredentials( b.mu.RLock("GetCredentials") defer b.mu.RUnlock() - wg, ok := b.slWorkgroups[workgroupName] + wg, ok := b.slWorkgroups.Get(workgroupName) if !ok { return "", "", "", fmt.Errorf( "%w: workgroup %q not found", @@ -515,7 +515,7 @@ func (b *InMemoryBackend) GetCredentials( resolvedDB := dbName if resolvedDB == "" { - ns, nsOK := b.slNamespaces[wg.NamespaceName] + ns, nsOK := b.slNamespaces.Get(wg.NamespaceName) if nsOK && ns.DBName != "" { resolvedDB = ns.DBName } else { @@ -543,12 +543,12 @@ func (b *InMemoryBackend) CreateServerlessSnapshot( b.mu.Lock("CreateServerlessSnapshot") defer b.mu.Unlock() - ns, ok := b.slNamespaces[namespaceName] + ns, ok := b.slNamespaces.Get(namespaceName) if !ok { return nil, fmt.Errorf("%w: namespace %q not found", ErrNamespaceNotFound, namespaceName) } - if _, exists := b.slSnapshots[snapshotName]; exists { + if _, exists := b.slSnapshots.Get(snapshotName); exists { return nil, fmt.Errorf( "%w: snapshot %q already exists", ErrServerlessConflict, @@ -567,7 +567,7 @@ func (b *InMemoryBackend) CreateServerlessSnapshot( Status: slStatusAvailable, AdminUsername: ns.AdminUsername, } - b.slSnapshots[snapshotName] = snap + b.slSnapshots.Put(snap) b.slSnapshotIdx.insert(snapshotName) return cloneServerlessSnapshot(snap), nil @@ -578,7 +578,7 @@ func (b *InMemoryBackend) GetServerlessSnapshot(snapshotName string) (*Serverles b.mu.RLock("GetServerlessSnapshot") defer b.mu.RUnlock() - snap, ok := b.slSnapshots[snapshotName] + snap, ok := b.slSnapshots.Get(snapshotName) if !ok { return nil, fmt.Errorf( "%w: snapshot %q not found", @@ -606,7 +606,7 @@ func (b *InMemoryBackend) ListServerlessSnapshots( list := make([]*ServerlessSnapshot, 0, len(keys)) for _, name := range keys { - snap, ok := b.slSnapshots[name] + snap, ok := b.slSnapshots.Get(name) if !ok { continue } @@ -650,7 +650,7 @@ func (b *InMemoryBackend) DeleteServerlessSnapshot( b.mu.Lock("DeleteServerlessSnapshot") defer b.mu.Unlock() - snap, ok := b.slSnapshots[snapshotName] + snap, ok := b.slSnapshots.Get(snapshotName) if !ok { return nil, fmt.Errorf( "%w: snapshot %q not found", @@ -660,7 +660,7 @@ func (b *InMemoryBackend) DeleteServerlessSnapshot( } cp := cloneServerlessSnapshot(snap) - delete(b.slSnapshots, snapshotName) + b.slSnapshots.Delete(snapshotName) b.slSnapshotIdx.remove(snapshotName) return cp, nil @@ -698,7 +698,7 @@ func (b *InMemoryBackend) CreateServerlessUsageLimit( Period: period, BreachAction: breachAction, } - b.slUsageLimits[id] = ul + b.slUsageLimits.Put(ul) b.slUsageLimitIdx.insert(id) return cloneServerlessUsageLimit(ul), nil @@ -711,7 +711,7 @@ func (b *InMemoryBackend) GetServerlessUsageLimit( b.mu.RLock("GetServerlessUsageLimit") defer b.mu.RUnlock() - ul, ok := b.slUsageLimits[usageLimitID] + ul, ok := b.slUsageLimits.Get(usageLimitID) if !ok { return nil, fmt.Errorf( "%w: usage limit %q not found", @@ -739,7 +739,7 @@ func (b *InMemoryBackend) ListServerlessUsageLimits( list := make([]*ServerlessUsageLimit, 0, len(keys)) for _, id := range keys { - ul, ok := b.slUsageLimits[id] + ul, ok := b.slUsageLimits.Get(id) if !ok { continue } @@ -784,7 +784,7 @@ func (b *InMemoryBackend) UpdateServerlessUsageLimit( b.mu.Lock("UpdateServerlessUsageLimit") defer b.mu.Unlock() - ul, ok := b.slUsageLimits[usageLimitID] + ul, ok := b.slUsageLimits.Get(usageLimitID) if !ok { return nil, fmt.Errorf( "%w: usage limit %q not found", @@ -811,7 +811,7 @@ func (b *InMemoryBackend) DeleteServerlessUsageLimit( b.mu.Lock("DeleteServerlessUsageLimit") defer b.mu.Unlock() - ul, ok := b.slUsageLimits[usageLimitID] + ul, ok := b.slUsageLimits.Get(usageLimitID) if !ok { return nil, fmt.Errorf( "%w: usage limit %q not found", @@ -821,7 +821,7 @@ func (b *InMemoryBackend) DeleteServerlessUsageLimit( } cp := cloneServerlessUsageLimit(ul) - delete(b.slUsageLimits, usageLimitID) + b.slUsageLimits.Delete(usageLimitID) b.slUsageLimitIdx.remove(usageLimitID) return cp, nil @@ -845,7 +845,7 @@ func (b *InMemoryBackend) CreateServerlessScheduledAction( b.mu.Lock("CreateServerlessScheduledAction") defer b.mu.Unlock() - if _, ok := b.slScheduledActions[scheduledActionName]; ok { + if _, ok := b.slScheduledActions.Get(scheduledActionName); ok { return nil, fmt.Errorf( "%w: scheduled action %q already exists", ErrServerlessConflict, @@ -870,7 +870,7 @@ func (b *InMemoryBackend) CreateServerlessScheduledAction( Status: slStatusActive, TargetAction: targetAction, } - b.slScheduledActions[scheduledActionName] = sa + b.slScheduledActions.Put(sa) b.slScheduledActionIdx.insert(scheduledActionName) return cloneServerlessScheduledAction(sa), nil @@ -883,7 +883,7 @@ func (b *InMemoryBackend) GetServerlessScheduledAction( b.mu.RLock("GetServerlessScheduledAction") defer b.mu.RUnlock() - sa, ok := b.slScheduledActions[scheduledActionName] + sa, ok := b.slScheduledActions.Get(scheduledActionName) if !ok { return nil, fmt.Errorf( "%w: scheduled action %q not found", @@ -911,7 +911,7 @@ func (b *InMemoryBackend) ListServerlessScheduledActions( list := make([]*ServerlessScheduledAction, 0, len(keys)) for _, name := range keys { - sa, ok := b.slScheduledActions[name] + sa, ok := b.slScheduledActions.Get(name) if !ok { continue } @@ -956,7 +956,7 @@ func (b *InMemoryBackend) UpdateServerlessScheduledAction( b.mu.Lock("UpdateServerlessScheduledAction") defer b.mu.Unlock() - sa, ok := b.slScheduledActions[scheduledActionName] + sa, ok := b.slScheduledActions.Get(scheduledActionName) if !ok { return nil, fmt.Errorf( "%w: scheduled action %q not found", @@ -991,7 +991,7 @@ func (b *InMemoryBackend) DeleteServerlessScheduledAction( b.mu.Lock("DeleteServerlessScheduledAction") defer b.mu.Unlock() - sa, ok := b.slScheduledActions[scheduledActionName] + sa, ok := b.slScheduledActions.Get(scheduledActionName) if !ok { return nil, fmt.Errorf( "%w: scheduled action %q not found", @@ -1001,7 +1001,7 @@ func (b *InMemoryBackend) DeleteServerlessScheduledAction( } cp := cloneServerlessScheduledAction(sa) - delete(b.slScheduledActions, scheduledActionName) + b.slScheduledActions.Delete(scheduledActionName) b.slScheduledActionIdx.remove(scheduledActionName) return cp, nil diff --git a/services/redshift/backend_serverless_index.go b/services/redshift/backend_serverless_index.go index d4d71fbaa..ae1bac71a 100644 --- a/services/redshift/backend_serverless_index.go +++ b/services/redshift/backend_serverless_index.go @@ -56,14 +56,14 @@ func (s *sortedStringIndex) rebuildFromKeys(keys []string) { } // rebuildServerlessIndexes reconstructs all serverless sorted indexes from the -// current backing maps. Callers MUST hold b.mu for writing (it is invoked from -// Restore, which swaps every map under the lock). +// current backing store.Tables. Callers MUST hold b.mu for writing (it is +// invoked from Restore, which loads every table under the lock). func (b *InMemoryBackend) rebuildServerlessIndexes() { - b.slNamespaceIdx.rebuildFromKeys(mapKeys(b.slNamespaces)) - b.slWorkgroupIdx.rebuildFromKeys(mapKeys(b.slWorkgroups)) - b.slSnapshotIdx.rebuildFromKeys(mapKeys(b.slSnapshots)) - b.slUsageLimitIdx.rebuildFromKeys(mapKeys(b.slUsageLimits)) - b.slScheduledActionIdx.rebuildFromKeys(mapKeys(b.slScheduledActions)) + b.slNamespaceIdx.rebuildFromKeys(tableKeys(b.slNamespaces, slNamespacesKeyFn)) + b.slWorkgroupIdx.rebuildFromKeys(tableKeys(b.slWorkgroups, slWorkgroupsKeyFn)) + b.slSnapshotIdx.rebuildFromKeys(tableKeys(b.slSnapshots, slSnapshotsKeyFn)) + b.slUsageLimitIdx.rebuildFromKeys(tableKeys(b.slUsageLimits, slUsageLimitsKeyFn)) + b.slScheduledActionIdx.rebuildFromKeys(tableKeys(b.slScheduledActions, slScheduledActionsKeyFn)) } // resetServerlessIndexes clears every serverless sorted index. @@ -74,13 +74,3 @@ func (b *InMemoryBackend) resetServerlessIndexes() { b.slUsageLimitIdx.reset() b.slScheduledActionIdx.reset() } - -// mapKeys returns the keys of m in unspecified order. -func mapKeys[V any](m map[string]V) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - - return keys -} diff --git a/services/redshift/backend_snapshots.go b/services/redshift/backend_snapshots.go index a44d469de..c6d8fd1f2 100644 --- a/services/redshift/backend_snapshots.go +++ b/services/redshift/backend_snapshots.go @@ -17,11 +17,12 @@ func (b *InMemoryBackend) CreateClusterSnapshot(snapshotID, clusterID string) (* b.mu.Lock("CreateClusterSnapshot") defer b.mu.Unlock() - if _, exists := b.clusters[clusterID]; !exists { + srcCluster, exists := b.clusters.Get(clusterID) + if !exists { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } - if _, exists := b.snapshots[snapshotID]; exists { + if b.snapshots.Has(snapshotID) { return nil, fmt.Errorf("%w: snapshot %s already exists", ErrSnapshotAlreadyExists, snapshotID) } @@ -33,12 +34,12 @@ func (b *InMemoryBackend) CreateClusterSnapshot(snapshotID, clusterID string) (* AccountsWithRestoreAccess: []AccountWithRestoreAccess{}, ManualSnapshotRetentionPeriod: -1, SnapshotCreateTime: time.Now(), - NodeType: b.clusters[clusterID].NodeType, - DBName: b.clusters[clusterID].DBName, - MasterUsername: b.clusters[clusterID].MasterUsername, - NumberOfNodes: b.clusters[clusterID].NumberOfNodes, + NodeType: srcCluster.NodeType, + DBName: srcCluster.DBName, + MasterUsername: srcCluster.MasterUsername, + NumberOfNodes: srcCluster.NumberOfNodes, } - b.snapshots[snapshotID] = snap + b.snapshots.Put(snap) return cloneSnapshot(snap), nil } @@ -52,13 +53,13 @@ func (b *InMemoryBackend) DeleteClusterSnapshot(snapshotID string) (*Snapshot, e b.mu.Lock("DeleteClusterSnapshot") defer b.mu.Unlock() - snap, exists := b.snapshots[snapshotID] + snap, exists := b.snapshots.Get(snapshotID) if !exists { return nil, fmt.Errorf("%w: snapshot %s not found", ErrSnapshotNotFound, snapshotID) } cp := cloneSnapshot(snap) - delete(b.snapshots, snapshotID) + b.snapshots.Delete(snapshotID) return cp, nil } @@ -70,7 +71,7 @@ func (b *InMemoryBackend) DescribeClusterSnapshots(snapshotID, clusterID, snapsh defer b.mu.RUnlock() if snapshotID != "" { - snap, exists := b.snapshots[snapshotID] + snap, exists := b.snapshots.Get(snapshotID) if !exists { return nil, fmt.Errorf("%w: snapshot %s not found", ErrSnapshotNotFound, snapshotID) } @@ -78,9 +79,9 @@ func (b *InMemoryBackend) DescribeClusterSnapshots(snapshotID, clusterID, snapsh return []Snapshot{*cloneSnapshot(snap)}, nil } - result := make([]Snapshot, 0, len(b.snapshots)) + result := make([]Snapshot, 0, b.snapshots.Len()) - for _, snap := range b.snapshots { + for _, snap := range b.snapshots.All() { if clusterID != "" && snap.ClusterIdentifier != clusterID { continue } @@ -107,19 +108,19 @@ func (b *InMemoryBackend) CopyClusterSnapshot( b.mu.Lock("CopyClusterSnapshot") defer b.mu.Unlock() - src, exists := b.snapshots[sourceSnapshotID] + src, exists := b.snapshots.Get(sourceSnapshotID) if !exists { return nil, fmt.Errorf("%w: snapshot %s not found", ErrSnapshotNotFound, sourceSnapshotID) } - if _, dstExists := b.snapshots[destinationSnapshotID]; dstExists { + if _, dstExists := b.snapshots.Get(destinationSnapshotID); dstExists { return nil, fmt.Errorf("%w: snapshot %s already exists", ErrSnapshotAlreadyExists, destinationSnapshotID) } cp := cloneSnapshot(src) cp.SnapshotIdentifier = destinationSnapshotID cp.SnapshotType = "manual" - b.snapshots[destinationSnapshotID] = cp + b.snapshots.Put(cp) result := cloneSnapshot(cp) @@ -138,12 +139,12 @@ func (b *InMemoryBackend) RestoreFromClusterSnapshot(clusterID, snapshotID strin b.mu.Lock("RestoreFromClusterSnapshot") defer b.mu.Unlock() - snap, exists := b.snapshots[snapshotID] + snap, exists := b.snapshots.Get(snapshotID) if !exists { return nil, fmt.Errorf("%w: snapshot %s not found", ErrSnapshotNotFound, snapshotID) } - if _, clusterExists := b.clusters[clusterID]; clusterExists { + if _, clusterExists := b.clusters.Get(clusterID); clusterExists { return nil, fmt.Errorf("%w: cluster %s already exists", ErrClusterAlreadyExists, clusterID) } @@ -180,7 +181,7 @@ func (b *InMemoryBackend) RestoreFromClusterSnapshot(clusterID, snapshotID strin NumberOfNodes: numberOfNodes, } - b.clusters[clusterID] = cluster + b.clusters.Put(cluster) if b.dnsRegistrar != nil { b.dnsRegistrar.Register(endpoint) diff --git a/services/redshift/backend_subnet_groups.go b/services/redshift/backend_subnet_groups.go index eb91daec7..7c79b0983 100644 --- a/services/redshift/backend_subnet_groups.go +++ b/services/redshift/backend_subnet_groups.go @@ -31,7 +31,7 @@ func (b *InMemoryBackend) CreateClusterSubnetGroup( b.mu.Lock("CreateClusterSubnetGroup") defer b.mu.Unlock() - if _, exists := b.subnetGroups[name]; exists { + if _, exists := b.subnetGroups.Get(name); exists { return nil, fmt.Errorf("%w: subnet group %s already exists", ErrSubnetGroupAlreadyExists, name) } @@ -47,7 +47,7 @@ func (b *InMemoryBackend) CreateClusterSubnetGroup( SubnetGroupStatus: "Complete", Subnets: subnets, } - b.subnetGroups[name] = sg + b.subnetGroups.Put(sg) return cloneSubnetGroup(sg), nil } @@ -61,11 +61,11 @@ func (b *InMemoryBackend) DeleteClusterSubnetGroup(name string) error { b.mu.Lock("DeleteClusterSubnetGroup") defer b.mu.Unlock() - if _, exists := b.subnetGroups[name]; !exists { + if _, exists := b.subnetGroups.Get(name); !exists { return fmt.Errorf("%w: subnet group %s not found", ErrSubnetGroupNotFound, name) } - delete(b.subnetGroups, name) + b.subnetGroups.Delete(name) return nil } @@ -76,7 +76,7 @@ func (b *InMemoryBackend) DescribeClusterSubnetGroups(name string) ([]ClusterSub defer b.mu.RUnlock() if name != "" { - sg, exists := b.subnetGroups[name] + sg, exists := b.subnetGroups.Get(name) if !exists { return nil, fmt.Errorf("%w: subnet group %s not found", ErrSubnetGroupNotFound, name) } @@ -84,8 +84,8 @@ func (b *InMemoryBackend) DescribeClusterSubnetGroups(name string) ([]ClusterSub return []ClusterSubnetGroup{*cloneSubnetGroup(sg)}, nil } - result := make([]ClusterSubnetGroup, 0, len(b.subnetGroups)) - for _, sg := range b.subnetGroups { + result := make([]ClusterSubnetGroup, 0, b.subnetGroups.Len()) + for _, sg := range b.subnetGroups.All() { result = append(result, *cloneSubnetGroup(sg)) } @@ -104,7 +104,7 @@ func (b *InMemoryBackend) ModifyClusterSubnetGroup( b.mu.Lock("ModifyClusterSubnetGroup") defer b.mu.Unlock() - sg, exists := b.subnetGroups[name] + sg, exists := b.subnetGroups.Get(name) if !exists { return nil, fmt.Errorf("%w: subnet group %s not found", ErrSubnetGroupNotFound, name) } diff --git a/services/redshift/export_test.go b/services/redshift/export_test.go index 1229b63db..be607c0e5 100644 --- a/services/redshift/export_test.go +++ b/services/redshift/export_test.go @@ -7,7 +7,7 @@ func ClusterCount(b *InMemoryBackend) int { b.mu.RLock("ClusterCount") defer b.mu.RUnlock() - return len(b.clusters) + return b.clusters.Len() } // ReservedNodeCount returns the number of reserved nodes in the backend. @@ -15,7 +15,7 @@ func ReservedNodeCount(b *InMemoryBackend) int { b.mu.RLock("ReservedNodeCount") defer b.mu.RUnlock() - return len(b.reservedNodes) + return b.reservedNodes.Len() } // PartnerCount returns the number of partners in the backend. @@ -23,7 +23,7 @@ func PartnerCount(b *InMemoryBackend) int { b.mu.RLock("PartnerCount") defer b.mu.RUnlock() - return len(b.partners) + return b.partners.Len() } // DataShareCount returns the number of data shares in the backend. @@ -31,7 +31,7 @@ func DataShareCount(b *InMemoryBackend) int { b.mu.RLock("DataShareCount") defer b.mu.RUnlock() - return len(b.dataShares) + return b.dataShares.Len() } // SecurityGroupCount returns the number of security groups in the backend. @@ -39,7 +39,7 @@ func SecurityGroupCount(b *InMemoryBackend) int { b.mu.RLock("SecurityGroupCount") defer b.mu.RUnlock() - return len(b.securityGroups) + return b.securityGroups.Len() } // SnapshotCount returns the number of snapshots in the backend. @@ -47,7 +47,7 @@ func SnapshotCount(b *InMemoryBackend) int { b.mu.RLock("SnapshotCount") defer b.mu.RUnlock() - return len(b.snapshots) + return b.snapshots.Len() } // EndpointAuthCount returns the number of endpoint authorizations in the backend. @@ -55,7 +55,7 @@ func EndpointAuthCount(b *InMemoryBackend) int { b.mu.RLock("EndpointAuthCount") defer b.mu.RUnlock() - return len(b.endpointAuths) + return b.endpointAuths.Len() } // ActiveResizeCount returns the number of active resize operations in the backend. @@ -66,12 +66,32 @@ func ActiveResizeCount(b *InMemoryBackend) int { return len(b.activeResizes) } +// LoggingStatusCount returns the number of per-cluster logging statuses held +// by the backend (a raw map -- see store_setup.go for why it isn't a +// store.Table). +func LoggingStatusCount(b *InMemoryBackend) int { + b.mu.RLock("LoggingStatusCount") + defer b.mu.RUnlock() + + return len(b.loggingStatuses) +} + +// SnapshotCopyConfigCount returns the number of cross-region snapshot copy +// configs held by the backend (a raw map -- see store_setup.go for why it +// isn't a store.Table). +func SnapshotCopyConfigCount(b *InMemoryBackend) int { + b.mu.RLock("SnapshotCopyConfigCount") + defer b.mu.RUnlock() + + return len(b.snapshotCopyConfigs) +} + // ParameterGroupCount returns the number of parameter groups in the backend. func ParameterGroupCount(b *InMemoryBackend) int { b.mu.RLock("ParameterGroupCount") defer b.mu.RUnlock() - return len(b.parameterGroups) + return b.parameterGroups.Len() } // SubnetGroupCount returns the number of subnet groups in the backend. @@ -79,7 +99,7 @@ func SubnetGroupCount(b *InMemoryBackend) int { b.mu.RLock("SubnetGroupCount") defer b.mu.RUnlock() - return len(b.subnetGroups) + return b.subnetGroups.Len() } // EventSubscriptionCount returns the number of event subscriptions in the backend. @@ -87,7 +107,7 @@ func EventSubscriptionCount(b *InMemoryBackend) int { b.mu.RLock("EventSubscriptionCount") defer b.mu.RUnlock() - return len(b.eventSubscriptions) + return b.eventSubscriptions.Len() } // HsmClientCertCount returns the number of HSM client certificates in the backend. @@ -95,7 +115,7 @@ func HsmClientCertCount(b *InMemoryBackend) int { b.mu.RLock("HsmClientCertCount") defer b.mu.RUnlock() - return len(b.hsmClientCerts) + return b.hsmClientCerts.Len() } // HsmConfigCount returns the number of HSM configurations in the backend. @@ -103,7 +123,7 @@ func HsmConfigCount(b *InMemoryBackend) int { b.mu.RLock("HsmConfigCount") defer b.mu.RUnlock() - return len(b.hsmConfigs) + return b.hsmConfigs.Len() } // ScheduledActionCount returns the number of scheduled actions in the backend. @@ -111,7 +131,7 @@ func ScheduledActionCount(b *InMemoryBackend) int { b.mu.RLock("ScheduledActionCount") defer b.mu.RUnlock() - return len(b.scheduledActions) + return b.scheduledActions.Len() } // CustomDomainCount returns the number of custom domain associations in the backend. @@ -119,7 +139,7 @@ func CustomDomainCount(b *InMemoryBackend) int { b.mu.RLock("CustomDomainCount") defer b.mu.RUnlock() - return len(b.customDomains) + return b.customDomains.Len() } // EndpointAccessCount returns the number of endpoint accesses in the backend. @@ -127,7 +147,7 @@ func EndpointAccessCount(b *InMemoryBackend) int { b.mu.RLock("EndpointAccessCount") defer b.mu.RUnlock() - return len(b.endpointAccesses) + return b.endpointAccesses.Len() } // IntegrationCount returns the number of integrations in the backend. @@ -135,7 +155,7 @@ func IntegrationCount(b *InMemoryBackend) int { b.mu.RLock("IntegrationCount") defer b.mu.RUnlock() - return len(b.integrations) + return b.integrations.Len() } // IdcApplicationCount returns the number of IDC applications in the backend. @@ -143,7 +163,7 @@ func IdcApplicationCount(b *InMemoryBackend) int { b.mu.RLock("IdcApplicationCount") defer b.mu.RUnlock() - return len(b.idcApplications) + return b.idcApplications.Len() } // HandlerOpsLen returns the number of operations registered in the handler. diff --git a/services/redshift/handler_serverless.go b/services/redshift/handler_serverless.go index e481b6e11..60156f34f 100644 --- a/services/redshift/handler_serverless.go +++ b/services/redshift/handler_serverless.go @@ -78,11 +78,11 @@ func (h *ServerlessHandler) Reset() { h.Backend.mu.Lock("ServerlessHandler.Reset") defer h.Backend.mu.Unlock() - h.Backend.slNamespaces = make(map[string]*Namespace) - h.Backend.slWorkgroups = make(map[string]*Workgroup) - h.Backend.slSnapshots = make(map[string]*ServerlessSnapshot) - h.Backend.slUsageLimits = make(map[string]*ServerlessUsageLimit) - h.Backend.slScheduledActions = make(map[string]*ServerlessScheduledAction) + h.Backend.slNamespaces.Reset() + h.Backend.slWorkgroups.Reset() + h.Backend.slSnapshots.Reset() + h.Backend.slUsageLimits.Reset() + h.Backend.slScheduledActions.Reset() } const ( diff --git a/services/redshift/persistence.go b/services/redshift/persistence.go index 5860f4bb8..668145148 100644 --- a/services/redshift/persistence.go +++ b/services/redshift/persistence.go @@ -3,174 +3,50 @@ package redshift import ( "context" "encoding/json" + "fmt" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" ) +// redshiftSnapshotVersion identifies the shape of backendSnapshot's Tables +// blob (i.e. the set of resources registered on b.registry -- see +// registerAllTables in store_setup.go). It must be bumped whenever a change +// there would make an older snapshot unsafe to decode as the current shape. +// Restore compares this against the persisted value and discards (rather +// than attempts to partially decode) any mismatch -- see Restore below. This +// mirrors the services/ec2 conversion (commit 12e611a4) and the services/sqs +// pilot (commit 0f09d77c). +const redshiftSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the Redshift backend. +// +// Tables holds one JSON-encoded array per registered store.Table, produced by +// [store.Registry.SnapshotAll]. The remaining fields are the handful of +// resource maps that could not be registered (see store_setup.go for why: +// each is keyed externally by clusterID rather than by a field on the value +// itself), plus backend-level scalar configuration. type backendSnapshot struct { - Clusters map[string]*Cluster `json:"clusters"` - ReservedNodes map[string]*ReservedNode `json:"reservedNodes"` - Partners map[string]*Partner `json:"partners"` - DataShares map[string]*DataShare `json:"dataShares"` - SecurityGroups map[string]*ClusterSecurityGroup `json:"securityGroups"` - Snapshots map[string]*Snapshot `json:"snapshots"` - EndpointAuths map[string]*EndpointAuthorization `json:"endpointAuths"` - ActiveResizes map[string]*ResizeProgress `json:"activeResizes"` - ParameterGroups map[string]*ClusterParameterGroup `json:"parameterGroups"` - SubnetGroups map[string]*ClusterSubnetGroup `json:"subnetGroups"` - LoggingStatuses map[string]*LoggingStatus `json:"loggingStatuses"` - EventSubscriptions map[string]*EventSubscription `json:"eventSubscriptions"` - Events map[string]*Event `json:"events"` - HsmClientCerts map[string]*HsmClientCertificate `json:"hsmClientCerts"` - HsmConfigs map[string]*HsmConfiguration `json:"hsmConfigs"` - ScheduledActions map[string]*ScheduledAction `json:"scheduledActions"` - CustomDomains map[string]*CustomDomainAssociation `json:"customDomains"` - EndpointAccesses map[string]*EndpointAccess `json:"endpointAccesses"` - Integrations map[string]*Integration `json:"integrations"` - IdcApplications map[string]*IdcApplication `json:"idcApplications"` - SnapshotCopyGrants map[string]*SnapshotCopyGrant `json:"snapshotCopyGrants"` - SnapshotSchedules map[string]*SnapshotSchedule `json:"snapshotSchedules"` - UsageLimits map[string]*UsageLimit `json:"usageLimits"` - AuthProfiles map[string]*AuthenticationProfile `json:"authProfiles"` - ResourcePolicies map[string]*ResourcePolicy `json:"resourcePolicies"` - TableRestores map[string]*TableRestoreStatus `json:"tableRestores"` - SnapshotCopyConfigs map[string]*SnapshotCopyConfig `json:"snapshotCopyConfigs"` - SlNamespaces map[string]*Namespace `json:"slNamespaces"` - SlWorkgroups map[string]*Workgroup `json:"slWorkgroups"` - SlSnapshots map[string]*ServerlessSnapshot `json:"slSnapshots"` - SlUsageLimits map[string]*ServerlessUsageLimit `json:"slUsageLimits"` - SlScheduledActions map[string]*ServerlessScheduledAction `json:"slScheduledActions"` - AccountID string `json:"accountID"` - Region string `json:"region"` + Tables map[string]json.RawMessage `json:"tables"` + ActiveResizes map[string]*ResizeProgress `json:"activeResizes"` + SnapshotCopyConfigs map[string]*SnapshotCopyConfig `json:"snapshotCopyConfigs"` + LoggingStatuses map[string]*LoggingStatus `json:"loggingStatuses"` + AccountID string `json:"accountID"` + Region string `json:"region"` + Version int `json:"version"` } func (s *backendSnapshot) ensureNonNilMaps() { - s.ensureCoreMaps() - s.ensureExtendedMaps() - s.ensureAuditMaps() - s.ensureServerlessMaps() -} - -func (s *backendSnapshot) ensureCoreMaps() { - if s.Clusters == nil { - s.Clusters = make(map[string]*Cluster) - } - if s.ReservedNodes == nil { - s.ReservedNodes = make(map[string]*ReservedNode) - } - if s.Partners == nil { - s.Partners = make(map[string]*Partner) - } - if s.DataShares == nil { - s.DataShares = make(map[string]*DataShare) - } - if s.SecurityGroups == nil { - s.SecurityGroups = make(map[string]*ClusterSecurityGroup) - } - if s.Snapshots == nil { - s.Snapshots = make(map[string]*Snapshot) - } - if s.EndpointAuths == nil { - s.EndpointAuths = make(map[string]*EndpointAuthorization) - } if s.ActiveResizes == nil { s.ActiveResizes = make(map[string]*ResizeProgress) } -} - -func (s *backendSnapshot) ensureExtendedMaps() { - if s.ParameterGroups == nil { - s.ParameterGroups = make(map[string]*ClusterParameterGroup) - } - if s.SubnetGroups == nil { - s.SubnetGroups = make(map[string]*ClusterSubnetGroup) - } - if s.LoggingStatuses == nil { - s.LoggingStatuses = make(map[string]*LoggingStatus) - } - if s.EventSubscriptions == nil { - s.EventSubscriptions = make(map[string]*EventSubscription) - } - if s.Events == nil { - s.Events = make(map[string]*Event) - } - if s.HsmClientCerts == nil { - s.HsmClientCerts = make(map[string]*HsmClientCertificate) - } - if s.HsmConfigs == nil { - s.HsmConfigs = make(map[string]*HsmConfiguration) - } - if s.ScheduledActions == nil { - s.ScheduledActions = make(map[string]*ScheduledAction) - } - - if s.CustomDomains == nil { - s.CustomDomains = make(map[string]*CustomDomainAssociation) - } - - if s.EndpointAccesses == nil { - s.EndpointAccesses = make(map[string]*EndpointAccess) - } - - if s.Integrations == nil { - s.Integrations = make(map[string]*Integration) - } - - if s.IdcApplications == nil { - s.IdcApplications = make(map[string]*IdcApplication) - } -} - -func (s *backendSnapshot) ensureAuditMaps() { - if s.SnapshotCopyGrants == nil { - s.SnapshotCopyGrants = make(map[string]*SnapshotCopyGrant) - } - - if s.SnapshotSchedules == nil { - s.SnapshotSchedules = make(map[string]*SnapshotSchedule) - } - - if s.UsageLimits == nil { - s.UsageLimits = make(map[string]*UsageLimit) - } - - if s.AuthProfiles == nil { - s.AuthProfiles = make(map[string]*AuthenticationProfile) - } - - if s.ResourcePolicies == nil { - s.ResourcePolicies = make(map[string]*ResourcePolicy) - } - - if s.TableRestores == nil { - s.TableRestores = make(map[string]*TableRestoreStatus) - } if s.SnapshotCopyConfigs == nil { s.SnapshotCopyConfigs = make(map[string]*SnapshotCopyConfig) } -} - -func (s *backendSnapshot) ensureServerlessMaps() { - if s.SlNamespaces == nil { - s.SlNamespaces = make(map[string]*Namespace) - } - - if s.SlWorkgroups == nil { - s.SlWorkgroups = make(map[string]*Workgroup) - } - - if s.SlSnapshots == nil { - s.SlSnapshots = make(map[string]*ServerlessSnapshot) - } - - if s.SlUsageLimits == nil { - s.SlUsageLimits = make(map[string]*ServerlessUsageLimit) - } - if s.SlScheduledActions == nil { - s.SlScheduledActions = make(map[string]*ServerlessScheduledAction) + if s.LoggingStatuses == nil { + s.LoggingStatuses = make(map[string]*LoggingStatus) } } @@ -180,39 +56,23 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "redshift: snapshot table marshal failed", "error", err) + + return nil + } + snap := backendSnapshot{ - Clusters: b.clusters, - ReservedNodes: b.reservedNodes, - Partners: b.partners, - DataShares: b.dataShares, - SecurityGroups: b.securityGroups, - Snapshots: b.snapshots, - HsmClientCerts: b.hsmClientCerts, - HsmConfigs: b.hsmConfigs, - ScheduledActions: b.scheduledActions, - CustomDomains: b.customDomains, - EndpointAccesses: b.endpointAccesses, - Integrations: b.integrations, - IdcApplications: b.idcApplications, - EndpointAuths: b.endpointAuths, + Version: redshiftSnapshotVersion, + Tables: tables, ActiveResizes: b.activeResizes, - ParameterGroups: b.parameterGroups, - SubnetGroups: b.subnetGroups, - LoggingStatuses: b.loggingStatuses, - EventSubscriptions: b.eventSubscriptions, - Events: b.events, - SnapshotCopyGrants: b.snapshotCopyGrants, - SnapshotSchedules: b.snapshotSchedules, - UsageLimits: b.usageLimits, - AuthProfiles: b.authProfiles, - ResourcePolicies: b.resourcePolicies, - TableRestores: b.tableRestores, SnapshotCopyConfigs: b.snapshotCopyConfigs, - SlNamespaces: b.slNamespaces, - SlWorkgroups: b.slWorkgroups, - SlSnapshots: b.slSnapshots, - SlUsageLimits: b.slUsageLimits, - SlScheduledActions: b.slScheduledActions, + LoggingStatuses: b.loggingStatuses, AccountID: b.accountID, Region: b.region, } @@ -241,43 +101,40 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() - b.clusters = snap.Clusters - b.reservedNodes = snap.ReservedNodes - b.partners = snap.Partners - b.dataShares = snap.DataShares - b.securityGroups = snap.SecurityGroups - b.snapshots = snap.Snapshots - b.endpointAuths = snap.EndpointAuths + if snap.Version != redshiftSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors services/ec2 (commit 12e611a4) and + // services/sqs (commit 0f09d77c). + logger.Load(ctx).WarnContext(ctx, + "redshift: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", redshiftSnapshotVersion) + + b.registry.ResetAll() + b.activeResizes = make(map[string]*ResizeProgress) + b.snapshotCopyConfigs = make(map[string]*SnapshotCopyConfig) + b.loggingStatuses = make(map[string]*LoggingStatus) + b.clusterTransitions = make(map[string]*clusterTransition) + b.resetServerlessIndexes() + + return nil + } + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("redshift: restore snapshot tables: %w", err) + } + b.activeResizes = snap.ActiveResizes - b.parameterGroups = snap.ParameterGroups - b.subnetGroups = snap.SubnetGroups - b.loggingStatuses = snap.LoggingStatuses - b.eventSubscriptions = snap.EventSubscriptions - b.events = snap.Events - b.hsmClientCerts = snap.HsmClientCerts - b.hsmConfigs = snap.HsmConfigs - b.scheduledActions = snap.ScheduledActions - b.customDomains = snap.CustomDomains - b.endpointAccesses = snap.EndpointAccesses - b.integrations = snap.Integrations - b.idcApplications = snap.IdcApplications - b.snapshotCopyGrants = snap.SnapshotCopyGrants - b.snapshotSchedules = snap.SnapshotSchedules - b.usageLimits = snap.UsageLimits - b.authProfiles = snap.AuthProfiles - b.resourcePolicies = snap.ResourcePolicies - b.tableRestores = snap.TableRestores b.snapshotCopyConfigs = snap.SnapshotCopyConfigs - b.slNamespaces = snap.SlNamespaces - b.slWorkgroups = snap.SlWorkgroups - b.slSnapshots = snap.SlSnapshots - b.slUsageLimits = snap.SlUsageLimits - b.slScheduledActions = snap.SlScheduledActions + b.loggingStatuses = snap.LoggingStatuses b.accountID = snap.AccountID b.region = snap.Region // In-flight cluster transitions are not persisted; start clean and rebuild - // the serverless sorted indexes from the freshly loaded maps. + // the serverless sorted indexes from the freshly loaded tables. b.clusterTransitions = make(map[string]*clusterTransition) b.rebuildServerlessIndexes() diff --git a/services/redshift/persistence_test.go b/services/redshift/persistence_test.go index 0a84cf1b3..53a539c47 100644 --- a/services/redshift/persistence_test.go +++ b/services/redshift/persistence_test.go @@ -5,6 +5,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/labstack/echo/v5" "github.com/stretchr/testify/assert" @@ -150,3 +151,207 @@ func TestRedshiftHandler_Routing(t *testing.T) { c2 := e.NewContext(req2, httptest.NewRecorder()) assert.Equal(t, "my-cluster", h.ExtractResource(c2)) } + +// TestInMemoryBackend_FullStateRoundTrip is the Phase 3.3 store.Table +// conversion's persistence-safety proof: it seeds one entry in every +// resource collection the backend owns -- every store.Table registered in +// store_setup.go, plus the three resource maps deliberately left raw +// (activeResizes, snapshotCopyConfigs, loggingStatuses) -- snapshots, restores +// into a fresh backend, and asserts every entry survived. Nothing here should +// be dropped; if it is, this test fails. +// +// events is intentionally NOT seeded: no public API populates it (see +// store_setup.go / DescribeEvents), matching pre-conversion behaviour. +func TestInMemoryBackend_FullStateRoundTrip(t *testing.T) { + t.Parallel() + + b := redshift.NewInMemoryBackend("000000000000", "us-east-1") + + _, err := b.CreateCluster("rt-cluster", "ra3.xlplus", "rtdb", "admin") + require.NoError(t, err) + + b.AddReservedNodeInternal(&redshift.ReservedNode{ReservedNodeID: "rn-1"}) + + _, err = b.AddPartner("000000000000", "rt-cluster", "rtdb", "partner-1") + require.NoError(t, err) + + b.AddDataShareInternal(&redshift.DataShare{DataShareArn: "arn:aws:redshift:us-east-1:000000000000:datashare:ds-1"}) + + _, err = b.CreateClusterSecurityGroup("rt-secgroup", "desc") + require.NoError(t, err) + + _, err = b.CreateClusterSnapshot("rt-snapshot", "rt-cluster") + require.NoError(t, err) + + _, err = b.AuthorizeEndpointAccess("rt-cluster", "111111111111", nil) + require.NoError(t, err) + + _, err = b.CreateClusterParameterGroup("rt-paramgroup", "redshift-1.0", "desc") + require.NoError(t, err) + + _, err = b.CreateClusterSubnetGroup("rt-subnetgroup", "desc", "vpc-1", []string{"subnet-1"}) + require.NoError(t, err) + + _, err = b.CreateEventSubscription( + "rt-eventsub", "arn:aws:sns:us-east-1:000000000000:topic", "cluster", "INFO", nil, nil, true, + ) + require.NoError(t, err) + + _, err = b.CreateSnapshotCopyGrant("rt-copygrant", "kms-key-1", nil) + require.NoError(t, err) + + _, err = b.CreateSnapshotSchedule("rt-copyschedule", "desc", []string{"rate(12 hours)"}, nil) + require.NoError(t, err) + + _, err = b.CreateUsageLimit("rt-cluster", "spectrum", "data-scanned", "log", 1000, nil) + require.NoError(t, err) + + _, err = b.CreateAuthenticationProfile("rt-authprofile", `{"AllowDBUserOverride":"1"}`) + require.NoError(t, err) + + _, err = b.PutResourcePolicy( + "arn:aws:redshift:us-east-1:000000000000:cluster:rt-cluster", `{"Version":"2012-10-17"}`, + ) + require.NoError(t, err) + + _, err = b.CreateTableRestoreStatus("rt-cluster", "rt-snapshot", "srcdb", "srctbl", "dstdb", "dsttbl") + require.NoError(t, err) + + _, err = b.CreateHsmClientCertificate("rt-hsmcert", nil) + require.NoError(t, err) + + _, err = b.CreateHsmConfiguration("rt-hsmconfig", "desc", "10.0.0.1", "partition-1", nil) + require.NoError(t, err) + + _, err = b.CreateScheduledAction( + "rt-scheduledaction", "at(2030-01-01T00:00:00)", "arn:aws:iam::000000000000:role/r", "desc", "", + ) + require.NoError(t, err) + + _, err = b.CreateCustomDomainAssociation( + "rt-cluster", "db.example.com", "arn:aws:acm:us-east-1:000000000000:certificate/1", + ) + require.NoError(t, err) + + _, err = b.CreateEndpointAccess("rt-cluster", "rt-endpointaccess", "vpc-1") + require.NoError(t, err) + + _, err = b.CreateIntegration( + "rt-integration", + "arn:aws:redshift:us-east-1:000000000000:cluster:rt-cluster", + "arn:aws:redshift:us-east-1:000000000000:namespace/1", + "", "desc", + ) + require.NoError(t, err) + + _, err = b.CreateIdcApplication( + "rt-idcapp", "arn:aws:sso::000000000000:instance/1", "display", "arn:aws:iam::000000000000:role/r", + ) + require.NoError(t, err) + + _, err = b.CreateNamespace("rt-namespace", "admin", "rtdb", "", nil, nil) + require.NoError(t, err) + + _, err = b.CreateWorkgroup("rt-workgroup", "rt-namespace", 32, nil, nil) + require.NoError(t, err) + + _, err = b.CreateServerlessSnapshot("rt-slsnapshot", "rt-namespace") + require.NoError(t, err) + + _, err = b.CreateServerlessUsageLimit( + "arn:aws:redshift-serverless:us-east-1:000000000000:workgroup/rt-workgroup", + "ComputeCapacity", "daily", "log", 100, + ) + require.NoError(t, err) + + _, err = b.CreateServerlessScheduledAction( + "rt-slscheduledaction", "rt-namespace", "at(2030-01-01T00:00:00)", "", + time.Now(), time.Now().Add(time.Hour), + ) + require.NoError(t, err) + + // The three raw (unconverted) maps. + b.AddActiveResizeInternal("rt-cluster", &redshift.ResizeProgress{Status: "IN_PROGRESS", AllowCancelResize: true}) + + _, err = b.EnableSnapshotCopy("rt-cluster", "us-west-2", "rt-copygrant", 7) + require.NoError(t, err) + + _, err = b.EnableLogging("rt-cluster", "rt-bucket", "prefix/") + require.NoError(t, err) + + // Snapshot and restore into a fresh backend. + data := b.Snapshot(t.Context()) + require.NotNil(t, data) + + fresh := redshift.NewInMemoryBackend("000000000000", "us-east-1") + require.NoError(t, fresh.Restore(t.Context(), data)) + + // Verify every seeded resource survived the round trip. + assert.Equal(t, 1, redshift.ClusterCount(fresh)) + assert.Equal(t, 1, redshift.ReservedNodeCount(fresh)) + assert.Equal(t, 1, redshift.PartnerCount(fresh)) + assert.Equal(t, 1, redshift.DataShareCount(fresh)) + assert.Equal(t, 1, redshift.SecurityGroupCount(fresh)) + assert.Equal(t, 1, redshift.SnapshotCount(fresh)) + assert.Equal(t, 1, redshift.EndpointAuthCount(fresh)) + assert.Equal(t, 1, redshift.ParameterGroupCount(fresh)) + assert.Equal(t, 1, redshift.SubnetGroupCount(fresh)) + assert.Equal(t, 1, redshift.EventSubscriptionCount(fresh)) + assert.Equal(t, 1, redshift.HsmClientCertCount(fresh)) + assert.Equal(t, 1, redshift.HsmConfigCount(fresh)) + assert.Equal(t, 1, redshift.ScheduledActionCount(fresh)) + assert.Equal(t, 1, redshift.CustomDomainCount(fresh)) + assert.Equal(t, 1, redshift.EndpointAccessCount(fresh)) + assert.Equal(t, 1, redshift.IntegrationCount(fresh)) + assert.Equal(t, 1, redshift.IdcApplicationCount(fresh)) + + // The three deliberately-raw maps (see store_setup.go). + assert.Equal(t, 1, redshift.ActiveResizeCount(fresh)) + assert.Equal(t, 1, redshift.SnapshotCopyConfigCount(fresh)) + assert.Equal(t, 1, redshift.LoggingStatusCount(fresh)) + + // Resources without an exported Count helper: verify via the public + // Describe/Get surface instead. + grants, err := fresh.DescribeSnapshotCopyGrants("") + require.NoError(t, err) + assert.Len(t, grants, 1) + + schedules, err := fresh.DescribeSnapshotSchedules("") + require.NoError(t, err) + assert.Len(t, schedules, 1) + + limits, err := fresh.DescribeUsageLimits("", "") + require.NoError(t, err) + assert.Len(t, limits, 1) + + profiles, err := fresh.DescribeAuthenticationProfiles("") + require.NoError(t, err) + assert.Len(t, profiles, 1) + + _, err = fresh.GetResourcePolicy("arn:aws:redshift:us-east-1:000000000000:cluster:rt-cluster") + require.NoError(t, err) + + restores, err := fresh.DescribeTableRestoreStatus("") + require.NoError(t, err) + assert.Len(t, restores, 1) + + _, err = fresh.GetNamespace("rt-namespace") + require.NoError(t, err) + + _, err = fresh.GetWorkgroup("rt-workgroup") + require.NoError(t, err) + + _, err = fresh.GetServerlessSnapshot("rt-slsnapshot") + require.NoError(t, err) + + slLimits, _ := fresh.ListServerlessUsageLimits("", 0, "") + assert.Len(t, slLimits, 1) + + _, err = fresh.GetServerlessScheduledAction("rt-slscheduledaction") + require.NoError(t, err) + + // Disabling snapshot copy only succeeds if snapshotCopyConfigs (raw map) + // survived the round trip with the rt-cluster entry intact. + _, err = fresh.DisableSnapshotCopy("rt-cluster") + require.NoError(t, err) +} diff --git a/services/redshift/store_setup.go b/services/redshift/store_setup.go new file mode 100644 index 000000000..c6f8a7a66 --- /dev/null +++ b/services/redshift/store_setup.go @@ -0,0 +1,206 @@ +package redshift + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend whose key is a pure +// function of the stored value's own fields is registered exactly once, +// here, as a *store.Table[T] on b.registry. See pkgs/store's package doc and +// the services/ec2 conversion (commit 12e611a4) and services/sqs pilot +// (commit 0f09d77c) for the pattern this follows. +// +// The following resource fields are deliberately NOT registered here and +// remain plain maps (see persistence.go for how they are still persisted, +// and backend.go for their declarations): +// - activeResizes: value type ResizeProgress carries no ClusterIdentifier +// field of its own -- it is keyed externally by clusterID (see +// AddActiveResizeInternal, which takes clusterID as a separate +// parameter from the ResizeProgress value). Not a pure function of the +// value's own fields, which store.Table requires. +// - snapshotCopyConfigs: value type SnapshotCopyConfig carries no +// ClusterIdentifier field of its own -- same external-keying issue. +// - loggingStatuses: value type LoggingStatus carries no ClusterIdentifier +// field of its own -- same external-keying issue. +// - clusterTransitions: in-flight lifecycle state, intentionally never +// persisted (see Restore) and also keyed externally by cluster ID. +import "github.com/blackbirdworks/gopherstack/pkgs/store" + +func clustersKeyFn(v *Cluster) string { return v.ClusterIdentifier } + +func reservedNodesKeyFn(v *ReservedNode) string { return v.ReservedNodeID } + +func partnersKeyFn(v *Partner) string { + return partnerKey(v.ClusterIdentifier, v.DatabaseName, v.PartnerName) +} + +func dataSharesKeyFn(v *DataShare) string { return v.DataShareArn } + +func securityGroupsKeyFn(v *ClusterSecurityGroup) string { return v.ClusterSecurityGroupName } + +func snapshotsKeyFn(v *Snapshot) string { return v.SnapshotIdentifier } + +func endpointAuthsKeyFn(v *EndpointAuthorization) string { + return endpointAuthKey(v.ClusterIdentifier, v.Grantee) +} + +func parameterGroupsKeyFn(v *ClusterParameterGroup) string { return v.ParameterGroupName } + +func subnetGroupsKeyFn(v *ClusterSubnetGroup) string { return v.ClusterSubnetGroupName } + +func eventSubscriptionsKeyFn(v *EventSubscription) string { return v.CustSubscriptionID } + +func eventsKeyFn(v *Event) string { return v.EventID } + +func snapshotCopyGrantsKeyFn(v *SnapshotCopyGrant) string { return v.SnapshotCopyGrantName } + +func snapshotSchedulesKeyFn(v *SnapshotSchedule) string { return v.ScheduleIdentifier } + +func usageLimitsKeyFn(v *UsageLimit) string { return v.UsageLimitID } + +func authProfilesKeyFn(v *AuthenticationProfile) string { return v.AuthenticationProfileName } + +func resourcePoliciesKeyFn(v *ResourcePolicy) string { return v.ResourceArn } + +func tableRestoresKeyFn(v *TableRestoreStatus) string { return v.TableRestoreRequestID } + +func hsmClientCertsKeyFn(v *HsmClientCertificate) string { return v.HsmClientCertificateIdentifier } + +func hsmConfigsKeyFn(v *HsmConfiguration) string { return v.HsmConfigurationIdentifier } + +func scheduledActionsKeyFn(v *ScheduledAction) string { return v.ScheduledActionName } + +func customDomainsKeyFn(v *CustomDomainAssociation) string { + return v.ClusterIdentifier + ":" + v.CustomDomainName +} + +func endpointAccessesKeyFn(v *EndpointAccess) string { return v.EndpointName } + +func integrationsKeyFn(v *Integration) string { return v.IntegrationName } + +func idcApplicationsKeyFn(v *IdcApplication) string { return v.IdcApplicationName } + +func slNamespacesKeyFn(v *Namespace) string { return v.NamespaceName } + +func slWorkgroupsKeyFn(v *Workgroup) string { return v.WorkgroupName } + +func slSnapshotsKeyFn(v *ServerlessSnapshot) string { return v.SnapshotName } + +func slUsageLimitsKeyFn(v *ServerlessUsageLimit) string { return v.UsageLimitID } + +func slScheduledActionsKeyFn(v *ServerlessScheduledAction) string { return v.ScheduledActionName } + +// registerAllTables registers every converted resource map on b.registry +// exactly once. It must be called during construction only (immediately +// after b.registry is created), never on every Reset() -- store.Register +// panics on a duplicate name, so runtime resets go through +// registry.ResetAll() instead (see InMemoryBackend.Reset in backend.go). +func registerAllTables(b *InMemoryBackend) { + for _, register := range tableRegistrations { + register(b) + } +} + +// tableRegistrations is the data-driven list registerAllTables walks: one +// closure per resource table, each binding its own store.New/store.Register +// call to the concrete field and value type. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableRegistrations = []func(*InMemoryBackend){ + func(b *InMemoryBackend) { + b.clusters = store.Register(b.registry, "clusters", store.New(clustersKeyFn)) + }, + func(b *InMemoryBackend) { + b.reservedNodes = store.Register(b.registry, "reservedNodes", store.New(reservedNodesKeyFn)) + }, + func(b *InMemoryBackend) { + b.partners = store.Register(b.registry, "partners", store.New(partnersKeyFn)) + }, + func(b *InMemoryBackend) { + b.dataShares = store.Register(b.registry, "dataShares", store.New(dataSharesKeyFn)) + }, + func(b *InMemoryBackend) { + b.securityGroups = store.Register(b.registry, "securityGroups", store.New(securityGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.snapshots = store.Register(b.registry, "snapshots", store.New(snapshotsKeyFn)) + }, + func(b *InMemoryBackend) { + b.endpointAuths = store.Register(b.registry, "endpointAuths", store.New(endpointAuthsKeyFn)) + }, + func(b *InMemoryBackend) { + b.parameterGroups = store.Register(b.registry, "parameterGroups", store.New(parameterGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.subnetGroups = store.Register(b.registry, "subnetGroups", store.New(subnetGroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.eventSubscriptions = store.Register(b.registry, "eventSubscriptions", store.New(eventSubscriptionsKeyFn)) + }, + func(b *InMemoryBackend) { b.events = store.Register(b.registry, "events", store.New(eventsKeyFn)) }, + func(b *InMemoryBackend) { + b.snapshotCopyGrants = store.Register(b.registry, "snapshotCopyGrants", store.New(snapshotCopyGrantsKeyFn)) + }, + func(b *InMemoryBackend) { + b.snapshotSchedules = store.Register(b.registry, "snapshotSchedules", store.New(snapshotSchedulesKeyFn)) + }, + func(b *InMemoryBackend) { + b.usageLimits = store.Register(b.registry, "usageLimits", store.New(usageLimitsKeyFn)) + }, + func(b *InMemoryBackend) { + b.authProfiles = store.Register(b.registry, "authProfiles", store.New(authProfilesKeyFn)) + }, + func(b *InMemoryBackend) { + b.resourcePolicies = store.Register(b.registry, "resourcePolicies", store.New(resourcePoliciesKeyFn)) + }, + func(b *InMemoryBackend) { + b.tableRestores = store.Register(b.registry, "tableRestores", store.New(tableRestoresKeyFn)) + }, + func(b *InMemoryBackend) { + b.hsmClientCerts = store.Register(b.registry, "hsmClientCerts", store.New(hsmClientCertsKeyFn)) + }, + func(b *InMemoryBackend) { + b.hsmConfigs = store.Register(b.registry, "hsmConfigs", store.New(hsmConfigsKeyFn)) + }, + func(b *InMemoryBackend) { + b.scheduledActions = store.Register(b.registry, "scheduledActions", store.New(scheduledActionsKeyFn)) + }, + func(b *InMemoryBackend) { + b.customDomains = store.Register(b.registry, "customDomains", store.New(customDomainsKeyFn)) + }, + func(b *InMemoryBackend) { + b.endpointAccesses = store.Register(b.registry, "endpointAccesses", store.New(endpointAccessesKeyFn)) + }, + func(b *InMemoryBackend) { + b.integrations = store.Register(b.registry, "integrations", store.New(integrationsKeyFn)) + }, + func(b *InMemoryBackend) { + b.idcApplications = store.Register(b.registry, "idcApplications", store.New(idcApplicationsKeyFn)) + }, + func(b *InMemoryBackend) { + b.slNamespaces = store.Register(b.registry, "slNamespaces", store.New(slNamespacesKeyFn)) + }, + func(b *InMemoryBackend) { + b.slWorkgroups = store.Register(b.registry, "slWorkgroups", store.New(slWorkgroupsKeyFn)) + }, + func(b *InMemoryBackend) { + b.slSnapshots = store.Register(b.registry, "slSnapshots", store.New(slSnapshotsKeyFn)) + }, + func(b *InMemoryBackend) { + b.slUsageLimits = store.Register(b.registry, "slUsageLimits", store.New(slUsageLimitsKeyFn)) + }, + func(b *InMemoryBackend) { + b.slScheduledActions = store.Register(b.registry, "slScheduledActions", store.New(slScheduledActionsKeyFn)) + }, +} + +// tableKeys returns the key (per keyFn) of every value currently in t, in +// unspecified order. Used by rebuildServerlessIndexes to seed the sorted +// secondary indexes from a store.Table instead of a bare map. +func tableKeys[V any](t *store.Table[V], keyFn func(*V) string) []string { + all := t.All() + keys := make([]string, 0, len(all)) + + for _, v := range all { + keys = append(keys, keyFn(v)) + } + + return keys +} diff --git a/services/route53/PARITY.md b/services/route53/PARITY.md new file mode 100644 index 000000000..9354ddc05 --- /dev/null +++ b/services/route53/PARITY.md @@ -0,0 +1,165 @@ +--- +service: route53 +sdk_module: aws-sdk-go-v2/service/route53@v1.62.3 +last_audit_commit: 017fc20a +last_audit_date: 2026-07-05 +overall: A # ~1000 LOC genuine fixes this pass (error codes/statuses, tag-family + # existence checks, CallerReference idempotency, routing bug, persistence gap) +ops: + CreateHostedZone: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: CallerReference reuse with different Name/Comment/PrivateZone now returns HostedZoneAlreadyExists (409) instead of silently returning the wrong zone"} + DeleteHostedZone: {wire: ok, errors: ok, state: ok, persist: ok} + GetHostedZone: {wire: ok, errors: ok, state: ok, persist: ok} + ListHostedZones: {wire: ok, errors: ok, state: ok, persist: ok} + ListHostedZonesByName: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateHostedZoneComment: {wire: ok, errors: ok, state: ok, persist: ok} + GetHostedZoneCount: {wire: ok, errors: ok, state: ok, persist: ok} + ChangeResourceRecordSets: {wire: ok, errors: ok, state: ok, persist: ok, note: "CREATE/DELETE/UPSERT, exact-match DELETE validation, all record-type value validators, routing-policy mutual exclusion, batch validated atomically before any mutation applied — see Notes"} + ListResourceRecordSets: {wire: ok, errors: ok, state: ok, persist: ok, note: "Name/Type/SetIdentifier lexicographic sort + pagination cursors"} + CountResourceRecordSets: {wire: ok, errors: ok, state: ok, persist: ok} + GetChange: {wire: ok, errors: ok, state: ok, persist: ok} + CreateHealthCheck: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: CallerReference reuse with a different HealthCheckConfig now returns HealthCheckAlreadyExists (409); fixed: CALCULATED HealthThreshold > len(ChildHealthChecks) now rejected (InvalidInput)"} + GetHealthCheck: {wire: ok, errors: ok, state: ok, persist: ok} + ListHealthChecks: {wire: ok, errors: ok, state: ok, persist: ok} + GetHealthCheckCount: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteHealthCheck: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateHealthCheck: {wire: ok, errors: partial, state: ok, persist: ok, note: "no HealthCheckVersion optimistic-concurrency check, see gaps"} + GetHealthCheckStatus: {wire: ok, errors: ok, state: ok, persist: ok} + GetHealthCheckLastFailureReason: {wire: ok, errors: ok, state: ok, persist: ok} + ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: no longer silently returns empty tags for a nonexistent hosted zone/health check — now validates existence and returns NoSuchHostedZone/NoSuchHealthCheck (404)"} + ListTagsForResources: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed 2 bugs: (1) HTTP route was unreachable (handler checked a bare /2013-04-01/tags path that can never match; real AWS URI is POST /2013-04-01/tags/{ResourceType}), (2) same missing-existence-check bug as ListTagsForResource"} + ChangeTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: the handler discarded ChangeTagsForResource's error return (setTags/removeTags used `_ = ...`), so tagging a nonexistent resource silently 200'd instead of 404ing; also fixed: resource tags (b.tags) were never wired into Snapshot/Restore and were lost across a backend restore"} + CreateKeySigningKey: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: duplicate name in a zone now returns KeySigningKeyAlreadyExists (409) instead of generic InvalidInput"} + ActivateKeySigningKey: {wire: ok, errors: ok, state: ok, persist: ok} + DeactivateKeySigningKey: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteKeySigningKey: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: deleting an ACTIVE KSK returned a fabricated 'KeySigningKeyNotInactive' code that doesn't exist in the AWS API; now returns the real InvalidKeySigningKeyStatus (400)"} + EnableHostedZoneDNSSEC: {wire: ok, errors: ok, state: ok, persist: ok} + DisableHostedZoneDNSSEC: {wire: ok, errors: ok, state: ok, persist: ok} + GetDNSSEC: {wire: ok, errors: ok, state: ok, persist: ok} + AssociateVPCWithHostedZone: {wire: ok, errors: partial, state: ok, persist: ok, note: "duplicate-VPC error code unverified against real AWS, see gaps"} + DisassociateVPCFromHostedZone: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: VPC not associated now returns VPCAssociationNotFound (404) instead of generic InvalidInput; LastVPCAssociation guard already correct"} + ListVPCAssociations: {wire: ok, errors: ok, state: ok, persist: ok} + ListHostedZonesByVPC: {wire: ok, errors: ok, state: ok, persist: ok} + CreateVPCAssociationAuthorization: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteVPCAssociationAuthorization: {wire: ok, errors: ok, state: ok, persist: ok} + ListVPCAssociationAuthorizations: {wire: ok, errors: ok, state: ok, persist: ok} + CountAssociatedVPCs: {wire: ok, errors: ok, state: ok, persist: ok} + CreateCidrCollection: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: duplicate collection name now returns CidrCollectionAlreadyExistsException (400) instead of allowing an unbounded number of same-named collections"} + ChangeCidrCollection: {wire: ok, errors: partial, state: ok, persist: ok, note: "no CollectionVersion optimistic-concurrency check, see gaps"} + DeleteCidrCollection: {wire: ok, errors: partial, state: ok, persist: ok, note: "no in-use check against CidrRoutingConfig references, see gaps"} + ListCidrCollections: {wire: ok, errors: ok, state: ok, persist: ok} + ListCidrLocations: {wire: ok, errors: ok, state: ok, persist: ok, note: "code fix: NoSuchCidrCollection -> NoSuchCidrCollectionException (real AWS shape name has the Exception suffix, confirmed against aws-sdk-go-v2 types/errors.go — unlike every other Route53 NoSuch* error)"} + ListCidrBlocks: {wire: ok, errors: ok, state: ok, persist: ok} + CreateQueryLoggingConfig: {wire: ok, errors: ok, state: ok, persist: ok, note: "status fix: QueryLoggingConfigAlreadyExists 400 -> 409"} + GetQueryLoggingConfig: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteQueryLoggingConfig: {wire: ok, errors: ok, state: ok, persist: ok} + ListQueryLoggingConfigs: {wire: ok, errors: ok, state: ok, persist: ok} + CreateReusableDelegationSet: {wire: ok, errors: partial, state: partial, persist: ok, note: "not linked to hosted zones at all, see gaps; status fix: NoSuchDelegationSet 404 -> 400"} + GetReusableDelegationSet: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteReusableDelegationSet: {wire: ok, errors: partial, state: partial, persist: ok, note: "no DelegationSetInUse check, see gaps"} + ListReusableDelegationSets: {wire: ok, errors: ok, state: ok, persist: ok} + CountZonesByReusableDelegationSet: {wire: ok, errors: ok, state: partial, persist: ok, note: "always returns 0 — see gaps (same root cause as CreateReusableDelegationSet)"} + TestDNSAnswer: {wire: ok, errors: ok, state: ok, persist: n/a, note: "not re-derived line-by-line against AWS routing-policy selection docs this pass, see deferred"} + CreateTrafficPolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "status fix: TrafficPolicyAlreadyExists 400 -> 409"} + CreateTrafficPolicyVersion: {wire: ok, errors: ok, state: ok, persist: ok} + CreateTrafficPolicyInstance: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: allowed unlimited duplicate instances for the same (hostedZoneID, name); now returns TrafficPolicyInstanceAlreadyExists (409)"} + UpdateTrafficPolicyInstance: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateTrafficPolicyComment: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteTrafficPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + GetTrafficPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteTrafficPolicyInstance: {wire: ok, errors: ok, state: ok, persist: ok} + GetTrafficPolicyInstance: {wire: ok, errors: ok, state: ok, persist: ok} + ListTrafficPolicies: {wire: ok, errors: ok, state: ok, persist: ok} + ListTrafficPolicyVersions: {wire: ok, errors: ok, state: ok, persist: ok} + ListTrafficPolicyInstances: {wire: ok, errors: ok, state: ok, persist: ok} + ListTrafficPolicyInstancesByHostedZone: {wire: ok, errors: ok, state: ok, persist: ok} + ListTrafficPolicyInstancesByPolicy: {wire: ok, errors: ok, state: ok, persist: ok} +families: + record_types: {status: ok, note: "A/AAAA/CNAME/MX/TXT/SPF/NS/SOA/PTR/SRV/CAA/DS/NAPTR value-format validators verified against RFC-shaped regexes; HTTPS/SVCB/SSHFP/TLSA intentionally accept any value (no AWS-documented format constraint enforced by the real service either)"} + routing_policies: {status: ok, note: "Weighted(SetIdentifier+Weight 0-255)/Latency(Region)/Failover(PRIMARY|SECONDARY)/Geolocation/Multivalue/Geoproximity(exactly one of AWSRegion|Coordinates|LocalZoneGroup, Bias -99..99, lat/lon range-checked)/CIDR routing all validated for mutual exclusion and SetIdentifier requirement per AWS rules; selection algorithm (selectAnswer) not re-derived line-by-line this pass, see deferred"} + dnssec: {status: ok, note: "EnableHostedZoneDNSSEC requires >=1 ACTIVE KSK (KeySigningKeyWithActiveStatusNotFound), KSK lifecycle (create/activate/deactivate/delete) state machine verified"} + errCodeLookup: {status: ok, note: "every route53 sentinel error's wire code + HTTP status cross-checked this pass against aws-sdk-go-v2/service/route53@v1.62.3 types/errors.go and the botocore api-2.json httpStatusCode field — see fixes in ops table above"} +gaps: + - UpdateHealthCheck has no HealthCheckVersion optimistic-concurrency check (HealthCheckVersionMismatch never returned) (bd: gopherstack-8l0.1) + - ChangeCidrCollection has no CollectionVersion check (CidrCollectionVersionMismatchException never returned); DeleteCidrCollection has no in-use check against CidrRoutingConfig references (CidrCollectionInUseException never returned) (bd: gopherstack-8l0.2) + - Reusable delegation sets are never linked to hosted zones (CreateHostedZone has no DelegationSetId param) — CreateReusableDelegationSet's hostedZoneID param is ignored, DeleteReusableDelegationSet never checks DelegationSetInUse, CountZonesByReusableDelegationSet always returns 0 (bd: gopherstack-8l0.3) + - AssociateVPCWithHostedZone returns generic InvalidInput for a duplicate VPC association; could not confirm the real AWS behavior (error vs. idempotent no-op) with high confidence this pass (bd: gopherstack-8l0.5) +deferred: + - TestDNSAnswer / selectAnswer / collectRoutingCandidates / resolveAlias / multiValueAnswer: routing-policy answer-selection algorithms not re-derived line-by-line against AWS docs this pass (bd: gopherstack-8l0.4) + - SDK-driven integration tests (test/integration/*_parity_test.go) not run for route53 this pass — this pass's fixes are proven by unit/handler tests only, which parity-principles.md notes is not full parity proof (bd: gopherstack-8l0.4) + - CreateKeySigningKey does not validate kmsArn as a well-formed KMS ARN (InvalidKMSArn never returned) + - Alias target cycle/depth handling (rrsValues/resolveAlias `depth` param) not stress-tested against pathological alias chains this pass +leaks: {status: clean, note: "no goroutines, tickers, or background timers anywhere in services/route53 (grep for 'go func|time.After|time.Sleep|Ticker' returns nothing) — all ops are synchronous request/response; Reset()/DeleteHostedZone/DeleteHealthCheck correctly cascade-delete tags/KSKs/VPC-assocs/query-logging-configs so no orphaned map entries accumulate under normal use. b.tags itself was NOT wired into Snapshot/Restore before this pass (fixed) — that was a persistence gap, not a leak, since Reset() already covered it."} +--- + +## Notes + +**Protocol**: REST-XML (path/verb routing, XML request+response bodies), matching +`aws-sdk-go-v2/service/route53`'s `awsRestxml_*` (de)serializers. Namespace +`https://route53.amazonaws.com/doc/2013-04-01/` on every response root element. + +**ChangeResourceRecordSets — the core op, and its traps**: +- Change-batch validation is two-phase: every `Change` in the batch is validated + against the *pre-batch* zone snapshot first (in submission order), and only if + *all* validate does the second phase apply every mutation. This correctly makes + the whole batch atomic (all-or-nothing) and matches AWS's documented behavior for + the canonical example of deleting a CNAME and creating an alias A record for the + same name in one batch (different `Type` values ⇒ different map keys ⇒ no + collision during validation). +- **Trap** (not a bug, verified deliberately): a batch containing a literal + `DELETE` followed by `CREATE` for the *exact same* `(Name, Type, SetIdentifier)` + key will fail with "record set already exists" during the CREATE's validation, + because validation runs against the unmutated zone. This is very unlikely to be + a real-world pattern (UPSERT already covers "replace this record's values"), and + AWS's own documented DELETE+CREATE example always uses two different `Type` + values, so this was deliberately left as-is rather than "fixed" without stronger + evidence of the real batch-processing order for the same-key case. Flagged here + so the next auditor doesn't have to re-derive this. +- DELETE requires an exact match of TTL + resource-record values (or AliasTarget) + against the current record, unless the request omits both (a "bare" delete by + name+type+SetIdentifier) — `deleteValuesMatch` already encodes this correctly. +- Alias vs. non-alias TTL rule: TTL is required (and range-checked to + `2147483647`) only when `AliasTarget == nil`; alias records must omit TTL. Already + correct. + +**Error code auditing method this pass**: every sentinel error in `backend.go`'s +`var (...)` block was cross-checked against two independent AWS sources: (1) +`aws-sdk-go-v2/service/route53@v1.62.3`'s generated `types/errors.go` (gives the +literal `ErrorCode()` wire string per exception type), and (2) the botocore +`api-2.json` model's `"error":{"httpStatusCode":N}` field per shape (gives the +real HTTP status). Six concrete mismatches were found and fixed — see the `ops` +table. The most surprising one: `NoSuchCidrCollectionException`, +`NoSuchCidrLocationException`, and `NoSuchCloudWatchLogsLogGroup` are the *only* +Route 53 "NoSuch*" errors whose wire code carries the `Exception` suffix; every +other `NoSuch*` error (NoSuchHostedZone, NoSuchHealthCheck, NoSuchChange, ...) does +not. Also surprising: `NoSuchDelegationSet` is HTTP 400, not 404, unlike every +other `NoSuch*` error in the service (confirmed twice against both sources before +trusting it). + +**CallerReference idempotency — the real rule**: reusing a `CallerReference` on +`CreateHostedZone`/`CreateHealthCheck` is idempotent (returns the original +resource) *only* when every other input parameter is identical to the original +request. Reusing it with *any* different parameter returns +`HostedZoneAlreadyExists`/`HealthCheckAlreadyExists` (409) — it is not a "last +write wins" or "always return the first" semantic. A prior audit pass had gotten +this backwards and encoded the wrong behavior directly into test assertions +(`same_ref_different_name_still_idempotent`); those tests were corrected this +pass rather than left as false parity proof. + +**ListTagsForResources routing — real AWS request URI**: `POST +/2013-04-01/tags/{ResourceType}` (batch lookup by `ResourceType` + a list of +`ResourceId`s in the XML body) — there is no bare `/2013-04-01/tags` endpoint. +The handler previously checked for the nonexistent bare path (which additionally +could never be reached anyway, since the outer dispatcher requires the +`/2013-04-01/tags/` prefix with a trailing slash), so a real AWS SDK client's +`ListTagsForResources` call was silently misrouted into `ChangeTagsForResource` +and 404'd. Fixed by detecting "no `/` after the ResourceType segment" instead of +comparing against a bare-prefix string. + +**Tag-family disguised-stub trap**: `ChangeTagsForResource`'s backend-level +existence check was always correct — but the handler called it through two +one-line wrappers (`setTags`/`removeTags`) that discarded the returned error +(`_ = h.Backend.ChangeTagsForResource(...)`), so the real validation never +surfaced over the wire. This is the "real-looking op that's actually a disguised +stub" pattern from parity-principles.md: grepping for the backend method alone +would have shown correct-looking code; the bug was purely in the handler +throwing the result away. diff --git a/services/route53/accuracy_batch2_ops_test.go b/services/route53/accuracy_batch2_ops_test.go index 9b777e189..6d97c780d 100644 --- a/services/route53/accuracy_batch2_ops_test.go +++ b/services/route53/accuracy_batch2_ops_test.go @@ -10,13 +10,18 @@ package route53_test // deleting and return ErrHostedZoneNotEmpty. // // 2. CreateHostedZone duplicate CallerReference: always created a new zone. -// AWS returns the existing zone for the same CallerReference (idempotency). -// Fix: scan zones for matching CallerReference before inserting. +// AWS returns the existing zone for the same CallerReference when the rest +// of the request is identical (idempotent retry), but returns +// HostedZoneAlreadyExists (409) when the CallerReference is reused with +// different Name/Comment/PrivateZone. Fix: scan zones for a matching +// CallerReference and compare the other fields before inserting. // // 3. CreateHealthCheck duplicate CallerReference: always created a new health // check. AWS returns the existing health check for the same CallerReference -// (idempotency). Fix: scan health checks for matching CallerReference before -// inserting. +// when the config is identical (idempotent retry), but returns +// HealthCheckAlreadyExists (409) when the CallerReference is reused with a +// different HealthCheckConfig. Fix: scan health checks for a matching +// CallerReference and compare the config before inserting. // // 4. DisassociateVPCFromHostedZone last VPC: allowed removing the last VPC from // a private hosted zone. AWS returns LastVPCAssociation (400) when the @@ -182,11 +187,17 @@ func TestBatch2_CreateHostedZone_DuplicateCallerReference(t *testing.T) { t.Parallel() tests := []struct { - name string - ref string + name string + ref string + name2 string + comment string }{ - {name: "same_ref_returns_existing", ref: "unique-caller-ref-hz-1"}, - {name: "same_ref_different_name_still_idempotent", ref: "unique-caller-ref-hz-2"}, + { + name: "same_ref_same_params_returns_existing", + ref: "unique-caller-ref-hz-1", + name2: "example.com", + comment: "first", + }, } for _, tt := range tests { @@ -198,18 +209,51 @@ func TestBatch2_CreateHostedZone_DuplicateCallerReference(t *testing.T) { first, err := b.CreateHostedZone("example.com", tt.ref, "first", false) require.NoError(t, err) - // Same CallerReference must return the original zone. - second, err := b.CreateHostedZone("other.com", tt.ref, "second", false) + // Same CallerReference *and* identical other parameters is a safe + // retry: AWS returns the original zone. + second, err := b.CreateHostedZone(tt.name2, tt.ref, tt.comment, false) require.NoError(t, err) assert.Equal(t, first.ID, second.ID, - "duplicate CallerReference must return the same zone ID") + "duplicate CallerReference with identical params must return the same zone ID") assert.Equal(t, first.Name, second.Name, "original zone name must be preserved") }) } } +// TestBatch2_CreateHostedZone_DuplicateCallerReference_DifferentParams verifies +// real AWS behavior: reusing a CallerReference with a *different* Name (or +// Comment/PrivateZone) is NOT idempotent — it returns HostedZoneAlreadyExists +// (409), since the CallerReference is now ambiguous between two distinct +// requested zones. A prior version of this test asserted the request was +// "still idempotent" for a different name, which is not how Route 53 behaves. +func TestBatch2_CreateHostedZone_DuplicateCallerReference_DifferentParams(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + ref string + }{ + {name: "same_ref_different_name_rejected", ref: "unique-caller-ref-hz-2"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := route53.NewInMemoryBackend() + + _, err := b.CreateHostedZone("example.com", tt.ref, "first", false) + require.NoError(t, err) + + _, err = b.CreateHostedZone("other.com", tt.ref, "second", false) + require.Error(t, err) + assert.ErrorIs(t, err, route53.ErrHostedZoneAlreadyExists) + }) + } +} + func TestBatch2_CreateHostedZone_UniqueCallerReference_CreatesNew(t *testing.T) { t.Parallel() @@ -284,27 +328,65 @@ func TestBatch2_CreateHealthCheck_DuplicateCallerReference(t *testing.T) { b := route53.NewInMemoryBackend() - first, err := b.CreateHealthCheck(tt.ref, route53.HealthCheckConfig{ + cfg := route53.HealthCheckConfig{ Type: route53.HealthCheckTypeHTTP, Port: 80, - }) + } + + first, err := b.CreateHealthCheck(tt.ref, cfg) require.NoError(t, err) - // Same CallerReference must return the original health check. - second, err := b.CreateHealthCheck(tt.ref, route53.HealthCheckConfig{ - Type: route53.HealthCheckTypeHTTPS, - Port: 443, - }) + // Same CallerReference *and* identical config is a safe retry: + // AWS returns the original health check. + second, err := b.CreateHealthCheck(tt.ref, cfg) require.NoError(t, err) assert.Equal(t, first.ID, second.ID, - "duplicate CallerReference must return the same health check ID") + "duplicate CallerReference with identical config must return the same health check ID") assert.Equal(t, first.Config.Type, second.Config.Type, "original config must be preserved") }) } } +// TestBatch2_CreateHealthCheck_DuplicateCallerReference_DifferentConfig +// verifies real AWS behavior: reusing a CallerReference with a *different* +// HealthCheckConfig returns HealthCheckAlreadyExists (409) rather than +// silently returning (or silently overwriting) the original health check. A +// prior version of this test asserted the request was still idempotent when +// Type/Port differed, which is not how Route 53 behaves. +func TestBatch2_CreateHealthCheck_DuplicateCallerReference_DifferentConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + ref string + }{ + {name: "same_ref_different_config_rejected", ref: "hc-idem-ref-3"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := route53.NewInMemoryBackend() + + _, err := b.CreateHealthCheck(tt.ref, route53.HealthCheckConfig{ + Type: route53.HealthCheckTypeHTTP, + Port: 80, + }) + require.NoError(t, err) + + _, err = b.CreateHealthCheck(tt.ref, route53.HealthCheckConfig{ + Type: route53.HealthCheckTypeHTTPS, + Port: 443, + }) + require.Error(t, err) + assert.ErrorIs(t, err, route53.ErrHealthCheckAlreadyExists) + }) + } +} + func TestBatch2_CreateHealthCheck_UniqueCallerReference_CreatesNew(t *testing.T) { t.Parallel() diff --git a/services/route53/backend.go b/services/route53/backend.go index 0828d4b66..8de651a02 100644 --- a/services/route53/backend.go +++ b/services/route53/backend.go @@ -6,7 +6,9 @@ import ( "errors" "fmt" "net" + "reflect" "regexp" + "slices" "sort" "strconv" "strings" @@ -15,6 +17,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/collections" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" "github.com/blackbirdworks/gopherstack/pkgs/page" + "github.com/blackbirdworks/gopherstack/pkgs/store" svcTags "github.com/blackbirdworks/gopherstack/pkgs/tags" ) @@ -28,12 +31,15 @@ const ( // Errors returned by the backend. var ( - ErrHostedZoneNotFound = errors.New("NoSuchHostedZone") - ErrInvalidInput = errors.New("InvalidInput") - ErrInvalidAction = errors.New("InvalidChangeBatch") - ErrHealthCheckNotFound = errors.New("NoSuchHealthCheck") - ErrKeySigningKeyNotFound = errors.New("NoSuchKeySigningKey") - ErrCidrCollectionNotFound = errors.New("NoSuchCidrCollection") + ErrHostedZoneNotFound = errors.New("NoSuchHostedZone") + ErrInvalidInput = errors.New("InvalidInput") + ErrInvalidAction = errors.New("InvalidChangeBatch") + ErrHealthCheckNotFound = errors.New("NoSuchHealthCheck") + ErrKeySigningKeyNotFound = errors.New("NoSuchKeySigningKey") + // ErrCidrCollectionNotFound wire code intentionally carries the "Exception" + // suffix — that is the literal AWS error code for this shape (unlike most + // Route 53 NoSuch* errors), confirmed against aws-sdk-go-v2 route53 model. + ErrCidrCollectionNotFound = errors.New("NoSuchCidrCollectionException") ErrQueryLoggingConfigNotFound = errors.New("NoSuchQueryLoggingConfig") ErrDelegationSetNotFound = errors.New("NoSuchDelegationSet") ErrTrafficPolicyNotFound = errors.New("NoSuchTrafficPolicy") @@ -57,10 +63,39 @@ var ( ErrInvalidDSRecord = errors.New("invalid DS record value") ErrInvalidSPFRecord = errors.New("invalid SPF record value") ErrTrafficPolicyInUse = errors.New("TrafficPolicyInUse") - ErrKeySigningKeyNotInactive = errors.New("KeySigningKeyNotInactive") - ErrTrafficPolicyAlreadyExists = errors.New("TrafficPolicyAlreadyExists") - ErrHostedZoneNotEmpty = errors.New("HostedZoneNotEmpty") - ErrLastVPCAssociation = errors.New("LastVPCAssociation") + // ErrInvalidKeySigningKeyStatus is returned when a KSK operation is attempted + // while the key is in a status that does not permit it (e.g. deleting an + // ACTIVE key). This is the real AWS wire error code — Route 53 has no + // "KeySigningKeyNotInactive" error. + ErrInvalidKeySigningKeyStatus = errors.New("InvalidKeySigningKeyStatus") + ErrTrafficPolicyAlreadyExists = errors.New("TrafficPolicyAlreadyExists") + ErrHostedZoneNotEmpty = errors.New("HostedZoneNotEmpty") + ErrLastVPCAssociation = errors.New("LastVPCAssociation") + // ErrHostedZoneAlreadyExists is returned when CreateHostedZone reuses a + // CallerReference already associated with a hosted zone that has different + // Name/Comment/PrivateZone values (AWS: HostedZoneAlreadyExists, 409). + ErrHostedZoneAlreadyExists = errors.New("HostedZoneAlreadyExists") + // ErrHealthCheckAlreadyExists is returned when CreateHealthCheck reuses a + // CallerReference already associated with a health check that has a + // different configuration (AWS: HealthCheckAlreadyExists, 409). + ErrHealthCheckAlreadyExists = errors.New("HealthCheckAlreadyExists") + // ErrKeySigningKeyAlreadyExists is returned when CreateKeySigningKey is + // called with a name that already exists in the hosted zone (AWS: + // KeySigningKeyAlreadyExists, 409). + ErrKeySigningKeyAlreadyExists = errors.New("KeySigningKeyAlreadyExists") + // ErrVPCAssociationNotFound is returned by DisassociateVPCFromHostedZone + // when the given VPC is not associated with the hosted zone (AWS: + // VPCAssociationNotFound, 404). + ErrVPCAssociationNotFound = errors.New("VPCAssociationNotFound") + // ErrTrafficPolicyInstanceAlreadyExists is returned when + // CreateTrafficPolicyInstance targets a (hostedZoneID, name) pair that + // already has a traffic policy instance (AWS: + // TrafficPolicyInstanceAlreadyExists, 409). + ErrTrafficPolicyInstanceAlreadyExists = errors.New("TrafficPolicyInstanceAlreadyExists") + // ErrCidrCollectionAlreadyExists is returned when CreateCidrCollection is + // called with a name that is already in use (AWS: + // CidrCollectionAlreadyExistsException, 400). + ErrCidrCollectionAlreadyExists = errors.New("CidrCollectionAlreadyExistsException") ) const ( @@ -391,40 +426,47 @@ type vpcAssociation struct { } // InMemoryBackend stores Route 53 state in memory. +// +// Most resource collections are *store.Table[T], registered once on registry +// at construction time (see store_setup.go); registry.ResetAll() then +// collapses their runtime reset to one call in Reset below. A handful of +// fields are deliberately left as plain maps -- see the doc comment atop +// store_setup.go for the full list and why. type InMemoryBackend struct { - dns DNSRegistrar - zones map[string]*zoneData // key: zone ID - healthChecks map[string]*HealthCheck // key: health check ID - keySigningKeys map[string]*KeySigningKey // key: "hostedZoneId|name" - cidrCollections map[string]*CidrCollection // key: collection ID - queryLoggingConfigs map[string]*QueryLoggingConfig // key: config ID - reusableDelegationSets map[string]*ReusableDelegationSet // key: delegation set ID - trafficPolicies map[string][]*TrafficPolicy // key: policy ID, value: versions - trafficPolicyInstances map[string]*TrafficPolicyInstance // key: instance ID - vpcAssociations map[string][]vpcAssociation // key: zone ID - vpcAssocAuthorizations map[string][]VPCAssociationAuthorization // key: zone ID - changes map[string]*ChangeInfo // key: change ID - tags map[string]*svcTags.Tags - mu *lockmetrics.RWMutex + dns DNSRegistrar + registry *store.Registry + zones *store.Table[zoneData] + healthChecks *store.Table[HealthCheck] + keySigningKeys *store.Table[KeySigningKey] + keySigningKeysByZone *store.Index[KeySigningKey] + cidrCollections *store.Table[CidrCollection] + queryLoggingConfigs *store.Table[QueryLoggingConfig] + queryLoggingConfigsByZone *store.Index[QueryLoggingConfig] + reusableDelegationSets *store.Table[ReusableDelegationSet] + trafficPolicies map[string][]*TrafficPolicy // key: policy ID, value: versions + trafficPolicyInstances *store.Table[TrafficPolicyInstance] + trafficPolicyInstancesByZone *store.Index[TrafficPolicyInstance] + vpcAssociations map[string][]vpcAssociation // key: zone ID + vpcAssocAuthorizations map[string][]VPCAssociationAuthorization // key: zone ID + changes *store.Table[ChangeInfo] + tags map[string]*svcTags.Tags + mu *lockmetrics.RWMutex } // NewInMemoryBackend creates a new InMemoryBackend. func NewInMemoryBackend() *InMemoryBackend { - return &InMemoryBackend{ - zones: make(map[string]*zoneData), - healthChecks: make(map[string]*HealthCheck), - keySigningKeys: make(map[string]*KeySigningKey), - cidrCollections: make(map[string]*CidrCollection), - queryLoggingConfigs: make(map[string]*QueryLoggingConfig), - reusableDelegationSets: make(map[string]*ReusableDelegationSet), + b := &InMemoryBackend{ + registry: store.NewRegistry(), trafficPolicies: make(map[string][]*TrafficPolicy), - trafficPolicyInstances: make(map[string]*TrafficPolicyInstance), vpcAssociations: make(map[string][]vpcAssociation), vpcAssocAuthorizations: make(map[string][]VPCAssociationAuthorization), - changes: make(map[string]*ChangeInfo), tags: make(map[string]*svcTags.Tags), mu: lockmetrics.New("route53"), } + + b.registerTables() + + return b } // SetDNSRegistrar wires a DNS server so A/CNAME records are auto-registered. @@ -488,15 +530,28 @@ func (b *InMemoryBackend) CreateHostedZone( b.mu.Lock("CreateHostedZone") defer b.mu.Unlock() - // CallerReference idempotency: AWS returns the existing zone when the same - // CallerReference is reused, rather than creating a duplicate. - for _, zd := range b.zones { - if zd.zone.CallerReference == callerRef { + // CallerReference idempotency: reusing a CallerReference with the exact + // same Name/Comment/PrivateZone is a safe retry and returns the existing + // zone. Reusing it with any different parameter is rejected — real AWS + // returns HostedZoneAlreadyExists (409) rather than silently returning + // (or silently creating a second zone for) mismatched input. + for _, zd := range b.zones.All() { + if zd.zone.CallerReference != callerRef { + continue + } + + if zd.zone.Name == name && zd.zone.Comment == comment && zd.zone.PrivateZone == private { cp := zd.zone cp.ResourceRecordSetCount = len(zd.records) return &cp, nil } + + return nil, fmt.Errorf( + "%w: a hosted zone already exists for CallerReference %s with different parameters", + ErrHostedZoneAlreadyExists, + callerRef, + ) } id := "Z" + randomZoneID() @@ -513,7 +568,7 @@ func (b *InMemoryBackend) CreateHostedZone( zone: hz, records: make(map[string]*ResourceRecordSet), } - b.zones[id] = zd + b.zones.Put(zd) // Seed the zone with the default NS and SOA records that AWS auto-creates. nsKey := recordSetKey(name, "NS", "") @@ -539,11 +594,11 @@ func (b *InMemoryBackend) CreateHostedZone( // Register a synthetic INSYNC change so that GetChange on the zone-creation // change ID (used by Terraform's waiter) returns INSYNC immediately. syntheticChangeID := "C" + id - b.changes[syntheticChangeID] = &ChangeInfo{ + b.changes.Put(&ChangeInfo{ ID: "/change/" + syntheticChangeID, Status: "INSYNC", SubmittedAt: time.Now(), - } + }) cp := hz @@ -570,7 +625,7 @@ func (b *InMemoryBackend) DeleteHostedZone(zoneID string) error { b.mu.Lock("DeleteHostedZone") defer b.mu.Unlock() - zd, ok := b.zones[zoneID] + zd, ok := b.zones.Get(zoneID) if !ok { return fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } @@ -597,20 +652,19 @@ func (b *InMemoryBackend) DeleteHostedZone(zoneID string) error { // Cascade: delete VPC associations for this zone. delete(b.vpcAssociations, zoneID) - // Cascade: delete query logging configs for this zone. - for id, cfg := range b.queryLoggingConfigs { - if cfg.HostedZoneID == zoneID { - delete(b.queryLoggingConfigs, id) - } + // Cascade: delete query logging configs for this zone. The index's + // backing slice is cloned before the loop because deleting from the + // table mutates the very index groups the slice is a view over. + for _, cfg := range slices.Clone(b.queryLoggingConfigsByZone.Get(zoneID)) { + b.queryLoggingConfigs.Delete(cfg.ID) } - // Cascade: delete key signing keys for this zone. - for k, ksk := range b.keySigningKeys { - if ksk.HostedZoneID == zoneID { - delete(b.keySigningKeys, k) - } + // Cascade: delete key signing keys for this zone. Same clone-before-delete + // reasoning as above. + for _, ksk := range slices.Clone(b.keySigningKeysByZone.Get(zoneID)) { + b.keySigningKeys.Delete(kskKey(ksk.HostedZoneID, ksk.Name)) } - delete(b.zones, zoneID) + b.zones.Delete(zoneID) delete(b.tags, zoneID) return nil @@ -621,7 +675,7 @@ func (b *InMemoryBackend) GetHostedZone(zoneID string) (*HostedZone, error) { b.mu.RLock("GetHostedZone") defer b.mu.RUnlock() - zd, ok := b.zones[zoneID] + zd, ok := b.zones.Get(zoneID) if !ok { return nil, fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } @@ -642,8 +696,9 @@ func (b *InMemoryBackend) ListHostedZones( b.mu.RLock("ListHostedZones") defer b.mu.RUnlock() - result := make([]HostedZone, 0, len(b.zones)) - for _, zd := range b.zones { + all := b.zones.All() + result := make([]HostedZone, 0, len(all)) + for _, zd := range all { cp := zd.zone cp.ResourceRecordSetCount = len(zd.records) result = append(result, cp) @@ -662,8 +717,9 @@ func (b *InMemoryBackend) ListHostedZonesByName( b.mu.RLock("ListHostedZonesByName") defer b.mu.RUnlock() - result := make([]HostedZone, 0, len(b.zones)) - for _, zd := range b.zones { + all := b.zones.All() + result := make([]HostedZone, 0, len(all)) + for _, zd := range all { cp := zd.zone cp.ResourceRecordSetCount = len(zd.records) result = append(result, cp) @@ -1000,6 +1056,13 @@ func validateHealthCheckConfig(cfg HealthCheckConfig) error { ) } + if cfg.Type == HealthCheckTypeCalculated && cfg.HealthThreshold > len(cfg.ChildHealthChecks) { + return fmt.Errorf( + "%w: HealthThreshold (%d) must not exceed the number of ChildHealthChecks (%d)", + ErrInvalidInput, cfg.HealthThreshold, len(cfg.ChildHealthChecks), + ) + } + if cfg.InsufficientDataHealthStatus != "" { switch cfg.InsufficientDataHealthStatus { case defaultHealthStatus, healthUnhealthy, "LastKnownStatus": @@ -1276,7 +1339,7 @@ func (b *InMemoryBackend) ChangeResourceRecordSets( b.mu.Lock("ChangeResourceRecordSets") defer b.mu.Unlock() - zd, ok := b.zones[zoneID] + zd, ok := b.zones.Get(zoneID) if !ok { return "", fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } @@ -1325,7 +1388,7 @@ func (b *InMemoryBackend) ChangeResourceRecordSets( Status: "INSYNC", SubmittedAt: time.Now(), } - b.changes[changeID] = ci + b.changes.Put(ci) return ci.ID, nil } @@ -1335,7 +1398,7 @@ func (b *InMemoryBackend) GetChange(changeID string) (*ChangeInfo, error) { b.mu.RLock("GetChange") defer b.mu.RUnlock() - ci, ok := b.changes[changeID] + ci, ok := b.changes.Get(changeID) if !ok { return nil, fmt.Errorf("%w: change %s not found", ErrChangeNotFound, changeID) } @@ -1366,7 +1429,7 @@ func (b *InMemoryBackend) ListResourceRecordSets( b.mu.RLock("ListResourceRecordSets") defer b.mu.RUnlock() - zd, ok := b.zones[zoneID] + zd, ok := b.zones.Get(zoneID) if !ok { return RRSetPage{}, fmt.Errorf( "%w: hosted zone %s not found", @@ -1461,14 +1524,27 @@ func (b *InMemoryBackend) CreateHealthCheck( b.mu.Lock("CreateHealthCheck") defer b.mu.Unlock() - // CallerReference idempotency: AWS returns the existing health check when the - // same CallerReference is reused, rather than creating a duplicate. - for _, existing := range b.healthChecks { - if existing.CallerReference == callerRef { + // CallerReference idempotency: reusing a CallerReference with the exact + // same HealthCheckConfig is a safe retry and returns the existing health + // check. Reusing it with a different config is rejected — real AWS returns + // HealthCheckAlreadyExists (409) rather than silently returning (or + // silently creating a second check for) mismatched input. + for _, existing := range b.healthChecks.All() { + if existing.CallerReference != callerRef { + continue + } + + if reflect.DeepEqual(existing.Config, cfg) { cp := *existing return &cp, nil } + + return nil, fmt.Errorf( + "%w: a health check already exists for CallerReference %s with a different configuration", + ErrHealthCheckAlreadyExists, + callerRef, + ) } hc := &HealthCheck{ @@ -1479,7 +1555,7 @@ func (b *InMemoryBackend) CreateHealthCheck( CreatedAt: time.Now(), } - b.healthChecks[hc.ID] = hc + b.healthChecks.Put(hc) cp := *hc @@ -1491,7 +1567,7 @@ func (b *InMemoryBackend) GetHealthCheck(id string) (*HealthCheck, error) { b.mu.RLock("GetHealthCheck") defer b.mu.RUnlock() - hc, ok := b.healthChecks[id] + hc, ok := b.healthChecks.Get(id) if !ok { return nil, fmt.Errorf("%w: health check %s not found", ErrHealthCheckNotFound, id) } @@ -1509,8 +1585,9 @@ func (b *InMemoryBackend) ListHealthChecks( b.mu.RLock("ListHealthChecks") defer b.mu.RUnlock() - result := make([]HealthCheck, 0, len(b.healthChecks)) - for _, hc := range b.healthChecks { + all := b.healthChecks.All() + result := make([]HealthCheck, 0, len(all)) + for _, hc := range all { cp := *hc result = append(result, cp) } @@ -1525,11 +1602,11 @@ func (b *InMemoryBackend) DeleteHealthCheck(id string) error { b.mu.Lock("DeleteHealthCheck") defer b.mu.Unlock() - if _, ok := b.healthChecks[id]; !ok { + if !b.healthChecks.Has(id) { return fmt.Errorf("%w: health check %s not found", ErrHealthCheckNotFound, id) } - delete(b.healthChecks, id) + b.healthChecks.Delete(id) delete(b.tags, id) return nil @@ -1543,7 +1620,7 @@ func (b *InMemoryBackend) UpdateHealthCheck( b.mu.Lock("UpdateHealthCheck") defer b.mu.Unlock() - hc, ok := b.healthChecks[id] + hc, ok := b.healthChecks.Get(id) if !ok { return nil, fmt.Errorf("%w: health check %s not found", ErrHealthCheckNotFound, id) } @@ -1560,7 +1637,7 @@ func (b *InMemoryBackend) GetHealthCheckStatus(id string) (string, error) { b.mu.RLock("GetHealthCheckStatus") defer b.mu.RUnlock() - hc, ok := b.healthChecks[id] + hc, ok := b.healthChecks.Get(id) if !ok { return "", fmt.Errorf("%w: health check %s not found", ErrHealthCheckNotFound, id) } @@ -1574,7 +1651,7 @@ func (b *InMemoryBackend) SetHealthCheckStatus(id, status string) error { b.mu.Lock("SetHealthCheckStatus") defer b.mu.Unlock() - hc, ok := b.healthChecks[id] + hc, ok := b.healthChecks.Get(id) if !ok { return fmt.Errorf("%w: health check %s not found", ErrHealthCheckNotFound, id) } @@ -1605,17 +1682,10 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.zones = make(map[string]*zoneData) - b.healthChecks = make(map[string]*HealthCheck) - b.keySigningKeys = make(map[string]*KeySigningKey) - b.cidrCollections = make(map[string]*CidrCollection) - b.queryLoggingConfigs = make(map[string]*QueryLoggingConfig) - b.reusableDelegationSets = make(map[string]*ReusableDelegationSet) + b.registry.ResetAll() b.trafficPolicies = make(map[string][]*TrafficPolicy) - b.trafficPolicyInstances = make(map[string]*TrafficPolicyInstance) b.vpcAssociations = make(map[string][]vpcAssociation) b.vpcAssocAuthorizations = make(map[string][]VPCAssociationAuthorization) - b.changes = make(map[string]*ChangeInfo) b.tags = make(map[string]*svcTags.Tags) } @@ -1637,14 +1707,14 @@ func (b *InMemoryBackend) CreateKeySigningKey( b.mu.Lock("CreateKeySigningKey") defer b.mu.Unlock() - if _, ok := b.zones[hostedZoneID]; !ok { + if _, ok := b.zones.Get(hostedZoneID); !ok { return nil, fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, hostedZoneID) } - if _, exists := b.keySigningKeys[kskKey(hostedZoneID, name)]; exists { + if b.keySigningKeys.Has(kskKey(hostedZoneID, name)) { return nil, fmt.Errorf( "%w: key signing key %s already exists in zone %s", - ErrInvalidInput, + ErrKeySigningKeyAlreadyExists, name, hostedZoneID, ) @@ -1692,7 +1762,7 @@ func (b *InMemoryBackend) CreateKeySigningKey( DSRecord: dsRecord, } - b.keySigningKeys[kskKey(hostedZoneID, name)] = ksk + b.keySigningKeys.Put(ksk) cp := *ksk @@ -1706,7 +1776,7 @@ func (b *InMemoryBackend) ActivateKeySigningKey(hostedZoneID, name string) (*Key key := kskKey(hostedZoneID, name) - ksk, ok := b.keySigningKeys[key] + ksk, ok := b.keySigningKeys.Get(key) if !ok { return nil, fmt.Errorf( "%w: key signing key %s not found in zone %s", @@ -1731,7 +1801,7 @@ func (b *InMemoryBackend) DeactivateKeySigningKey( defer b.mu.Unlock() key := kskKey(hostedZoneID, name) - ksk, ok := b.keySigningKeys[key] + ksk, ok := b.keySigningKeys.Get(key) if !ok { return nil, fmt.Errorf( "%w: key signing key %s not found in zone %s", @@ -1748,13 +1818,14 @@ func (b *InMemoryBackend) DeactivateKeySigningKey( } // DeleteKeySigningKey deletes a key signing key. -// The KSK must be INACTIVE; deleting an ACTIVE KSK returns ErrKeySigningKeyNotInactive. +// The KSK must be INACTIVE; deleting an ACTIVE KSK returns ErrInvalidKeySigningKeyStatus +// (AWS: InvalidKeySigningKeyStatus). func (b *InMemoryBackend) DeleteKeySigningKey(hostedZoneID, name string) error { b.mu.Lock("DeleteKeySigningKey") defer b.mu.Unlock() key := kskKey(hostedZoneID, name) - ksk, ok := b.keySigningKeys[key] + ksk, ok := b.keySigningKeys.Get(key) if !ok { return fmt.Errorf( "%w: key signing key %s not found in zone %s", @@ -1767,12 +1838,12 @@ func (b *InMemoryBackend) DeleteKeySigningKey(hostedZoneID, name string) error { if ksk.Status == kskStatusActive { return fmt.Errorf( "%w: key signing key %s must be INACTIVE before deletion", - ErrKeySigningKeyNotInactive, + ErrInvalidKeySigningKeyStatus, name, ) } - delete(b.keySigningKeys, key) + b.keySigningKeys.Delete(key) return nil } @@ -1783,15 +1854,15 @@ func (b *InMemoryBackend) EnableHostedZoneDNSSEC(zoneID string) error { b.mu.Lock("EnableHostedZoneDNSSEC") defer b.mu.Unlock() - zd, ok := b.zones[zoneID] + zd, ok := b.zones.Get(zoneID) if !ok { return fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } hasActiveKSK := false - for _, ksk := range b.keySigningKeys { - if ksk.HostedZoneID == zoneID && ksk.Status == kskStatusActive { + for _, ksk := range b.keySigningKeysByZone.Get(zoneID) { + if ksk.Status == kskStatusActive { hasActiveKSK = true break @@ -1815,7 +1886,7 @@ func (b *InMemoryBackend) DisableHostedZoneDNSSEC(zoneID string) error { b.mu.Lock("DisableHostedZoneDNSSEC") defer b.mu.Unlock() - zd, ok := b.zones[zoneID] + zd, ok := b.zones.Get(zoneID) if !ok { return fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } @@ -1830,17 +1901,15 @@ func (b *InMemoryBackend) GetDNSSEC(zoneID string) (bool, []KeySigningKey, error b.mu.RLock("GetDNSSEC") defer b.mu.RUnlock() - zd, ok := b.zones[zoneID] + zd, ok := b.zones.Get(zoneID) if !ok { return false, nil, fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } var ksks []KeySigningKey - for _, ksk := range b.keySigningKeys { - if ksk.HostedZoneID == zoneID { - cp := *ksk - ksks = append(ksks, cp) - } + for _, ksk := range b.keySigningKeysByZone.Get(zoneID) { + cp := *ksk + ksks = append(ksks, cp) } sort.Slice(ksks, func(i, j int) bool { return ksks[i].Name < ksks[j].Name }) @@ -1858,7 +1927,7 @@ func (b *InMemoryBackend) AssociateVPCWithHostedZone(zoneID, vpcID, vpcRegion st b.mu.Lock("AssociateVPCWithHostedZone") defer b.mu.Unlock() - zd, ok := b.zones[zoneID] + zd, ok := b.zones.Get(zoneID) if !ok { return fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } @@ -1895,7 +1964,7 @@ func (b *InMemoryBackend) DisassociateVPCFromHostedZone(zoneID, vpcID string) er b.mu.Lock("DisassociateVPCFromHostedZone") defer b.mu.Unlock() - zd, ok := b.zones[zoneID] + zd, ok := b.zones.Get(zoneID) if !ok { return fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } @@ -1916,7 +1985,7 @@ func (b *InMemoryBackend) DisassociateVPCFromHostedZone(zoneID, vpcID string) er if len(newAssocs) == len(assocs) { return fmt.Errorf( "%w: VPC %s is not associated with hosted zone %s", - ErrInvalidInput, + ErrVPCAssociationNotFound, vpcID, zoneID, ) @@ -1941,7 +2010,7 @@ func (b *InMemoryBackend) ListVPCAssociations(zoneID string) ([]vpcAssociation, b.mu.RLock("ListVPCAssociations") defer b.mu.RUnlock() - if _, ok := b.zones[zoneID]; !ok { + if _, ok := b.zones.Get(zoneID); !ok { return nil, fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } @@ -1959,7 +2028,7 @@ func (b *InMemoryBackend) CreateVPCAssociationAuthorization( b.mu.Lock("CreateVPCAssociationAuthorization") defer b.mu.Unlock() - if _, ok := b.zones[zoneID]; !ok { + if _, ok := b.zones.Get(zoneID); !ok { return nil, fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } @@ -1987,7 +2056,7 @@ func (b *InMemoryBackend) DeleteVPCAssociationAuthorization(zoneID, vpcID string b.mu.Lock("DeleteVPCAssociationAuthorization") defer b.mu.Unlock() - if _, ok := b.zones[zoneID]; !ok { + if _, ok := b.zones.Get(zoneID); !ok { return fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } @@ -2025,7 +2094,7 @@ func (b *InMemoryBackend) ListVPCAssociationAuthorizations( b.mu.RLock("ListVPCAssociationAuthorizations") defer b.mu.RUnlock() - if _, ok := b.zones[zoneID]; !ok { + if _, ok := b.zones.Get(zoneID); !ok { return nil, fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } @@ -2045,8 +2114,7 @@ func (b *InMemoryBackend) ListHostedZonesByVPC(vpcID, vpcRegion string) ([]Hoste for zoneID, assocs := range b.vpcAssociations { for _, a := range assocs { if a.VPCID == vpcID && (vpcRegion == "" || a.VPCRegion == vpcRegion) { - zd := b.zones[zoneID] - if zd != nil { + if zd, ok := b.zones.Get(zoneID); ok { cp := zd.zone cp.ResourceRecordSetCount = len(zd.records) result = append(result, cp) @@ -2073,6 +2141,16 @@ func (b *InMemoryBackend) CreateCidrCollection( b.mu.Lock("CreateCidrCollection") defer b.mu.Unlock() + for _, existing := range b.cidrCollections.All() { + if existing.Name == name { + return nil, fmt.Errorf( + "%w: a CIDR collection named %s already exists", + ErrCidrCollectionAlreadyExists, + name, + ) + } + } + id := "Z" + randomZoneID() col := &CidrCollection{ ID: id, @@ -2082,7 +2160,7 @@ func (b *InMemoryBackend) CreateCidrCollection( Locations: make(map[string][]string), } - b.cidrCollections[id] = col + b.cidrCollections.Put(col) cp := *col cp.Locations = copyLocations(col.Locations) @@ -2111,7 +2189,7 @@ func (b *InMemoryBackend) ChangeCidrCollection( b.mu.Lock("ChangeCidrCollection") defer b.mu.Unlock() - col, ok := b.cidrCollections[collectionID] + col, ok := b.cidrCollections.Get(collectionID) if !ok { return nil, fmt.Errorf( "%w: CIDR collection %s not found", @@ -2177,7 +2255,7 @@ func (b *InMemoryBackend) ListCidrLocations(collectionID string) ([]string, erro b.mu.RLock("ListCidrLocations") defer b.mu.RUnlock() - col, ok := b.cidrCollections[collectionID] + col, ok := b.cidrCollections.Get(collectionID) if !ok { return nil, fmt.Errorf( "%w: CIDR collection %s not found", @@ -2196,7 +2274,7 @@ func (b *InMemoryBackend) ListCidrBlocks(collectionID, locationName string) ([]s b.mu.RLock("ListCidrBlocks") defer b.mu.RUnlock() - col, ok := b.cidrCollections[collectionID] + col, ok := b.cidrCollections.Get(collectionID) if !ok { return nil, fmt.Errorf( "%w: CIDR collection %s not found", @@ -2228,17 +2306,15 @@ func (b *InMemoryBackend) CreateQueryLoggingConfig( b.mu.Lock("CreateQueryLoggingConfig") defer b.mu.Unlock() - if _, ok := b.zones[hostedZoneID]; !ok { + if _, ok := b.zones.Get(hostedZoneID); !ok { return nil, fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, hostedZoneID) } - for _, cfg := range b.queryLoggingConfigs { - if cfg.HostedZoneID == hostedZoneID { - return nil, fmt.Errorf( - "%w: a query logging config already exists for hosted zone %s", - ErrQueryLoggingConfigAlreadyExists, hostedZoneID, - ) - } + if len(b.queryLoggingConfigsByZone.Get(hostedZoneID)) > 0 { + return nil, fmt.Errorf( + "%w: a query logging config already exists for hosted zone %s", + ErrQueryLoggingConfigAlreadyExists, hostedZoneID, + ) } id := "Z" + randomZoneID() @@ -2249,7 +2325,7 @@ func (b *InMemoryBackend) CreateQueryLoggingConfig( CreatedAt: time.Now(), } - b.queryLoggingConfigs[id] = cfg + b.queryLoggingConfigs.Put(cfg) cp := *cfg @@ -2261,7 +2337,7 @@ func (b *InMemoryBackend) GetQueryLoggingConfig(id string) (*QueryLoggingConfig, b.mu.RLock("GetQueryLoggingConfig") defer b.mu.RUnlock() - cfg, ok := b.queryLoggingConfigs[id] + cfg, ok := b.queryLoggingConfigs.Get(id) if !ok { return nil, fmt.Errorf( "%w: query logging config %s not found", @@ -2280,7 +2356,7 @@ func (b *InMemoryBackend) DeleteQueryLoggingConfig(id string) error { b.mu.Lock("DeleteQueryLoggingConfig") defer b.mu.Unlock() - if _, ok := b.queryLoggingConfigs[id]; !ok { + if !b.queryLoggingConfigs.Has(id) { return fmt.Errorf( "%w: query logging config %s not found", ErrQueryLoggingConfigNotFound, @@ -2288,7 +2364,7 @@ func (b *InMemoryBackend) DeleteQueryLoggingConfig(id string) error { ) } - delete(b.queryLoggingConfigs, id) + b.queryLoggingConfigs.Delete(id) return nil } @@ -2300,13 +2376,18 @@ func (b *InMemoryBackend) ListQueryLoggingConfigs( b.mu.RLock("ListQueryLoggingConfigs") defer b.mu.RUnlock() - var result []*QueryLoggingConfig + var candidates []*QueryLoggingConfig + if hostedZoneID == "" { + candidates = b.queryLoggingConfigs.All() + } else { + candidates = b.queryLoggingConfigsByZone.Get(hostedZoneID) + } + + result := make([]*QueryLoggingConfig, 0, len(candidates)) - for _, cfg := range b.queryLoggingConfigs { - if hostedZoneID == "" || cfg.HostedZoneID == hostedZoneID { - cp := *cfg - result = append(result, &cp) - } + for _, cfg := range candidates { + cp := *cfg + result = append(result, &cp) } sort.Slice(result, func(i, j int) bool { return result[i].ID < result[j].ID }) @@ -2342,7 +2423,7 @@ func (b *InMemoryBackend) CreateReusableDelegationSet( CreatedAt: time.Now(), } - b.reusableDelegationSets[id] = ds + b.reusableDelegationSets.Put(ds) cp := *ds @@ -2465,7 +2546,7 @@ func (b *InMemoryBackend) CreateTrafficPolicyInstance( b.mu.Lock("CreateTrafficPolicyInstance") defer b.mu.Unlock() - if _, ok := b.zones[hostedZoneID]; !ok { + if _, ok := b.zones.Get(hostedZoneID); !ok { return nil, fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, hostedZoneID) } @@ -2483,11 +2564,28 @@ func (b *InMemoryBackend) CreateTrafficPolicyInstance( } } + normalisedName := normaliseName(name) + + // AWS allows only one traffic policy instance per (hosted zone, name): + // a second CreateTrafficPolicyInstance for the same pair returns + // TrafficPolicyInstanceAlreadyExists (409) rather than creating a + // duplicate or silently overwriting the existing instance. + for _, existing := range b.trafficPolicyInstancesByZone.Get(hostedZoneID) { + if strings.EqualFold(existing.Name, normalisedName) { + return nil, fmt.Errorf( + "%w: a traffic policy instance already exists for name %s in zone %s", + ErrTrafficPolicyInstanceAlreadyExists, + name, + hostedZoneID, + ) + } + } + id := randomTPIID() inst := &TrafficPolicyInstance{ ID: id, HostedZoneID: hostedZoneID, - Name: normaliseName(name), + Name: normalisedName, TrafficPolicyID: tpID, TrafficPolicyVersion: tpVersion, TrafficPolicyType: tpType, @@ -2495,7 +2593,7 @@ func (b *InMemoryBackend) CreateTrafficPolicyInstance( State: tpiStateApplied, } - b.trafficPolicyInstances[id] = inst + b.trafficPolicyInstances.Put(inst) cp := *inst @@ -2506,7 +2604,7 @@ func (b *InMemoryBackend) CreateTrafficPolicyInstance( func (b *InMemoryBackend) AddZoneInternal(hz HostedZone) { b.mu.Lock("AddZoneInternal") defer b.mu.Unlock() - b.zones[hz.ID] = &zoneData{zone: hz, records: make(map[string]*ResourceRecordSet)} + b.zones.Put(&zoneData{zone: hz, records: make(map[string]*ResourceRecordSet)}) } // DeleteTrafficPolicy deletes a specific version of a traffic policy. @@ -2538,7 +2636,7 @@ func (b *InMemoryBackend) DeleteTrafficPolicy(id string, version int32) error { ) } - for _, inst := range b.trafficPolicyInstances { + for _, inst := range b.trafficPolicyInstances.All() { if inst.TrafficPolicyID == id && inst.TrafficPolicyVersion == version { return fmt.Errorf( "%w: traffic policy %s version %d is still in use by instance %s", @@ -2592,7 +2690,7 @@ func (b *InMemoryBackend) DeleteTrafficPolicyInstance(id string) error { b.mu.Lock("DeleteTrafficPolicyInstance") defer b.mu.Unlock() - if _, ok := b.trafficPolicyInstances[id]; !ok { + if !b.trafficPolicyInstances.Has(id) { return fmt.Errorf( "%w: traffic policy instance %s not found", ErrTrafficPolicyInstNotFound, @@ -2600,7 +2698,7 @@ func (b *InMemoryBackend) DeleteTrafficPolicyInstance(id string) error { ) } - delete(b.trafficPolicyInstances, id) + b.trafficPolicyInstances.Delete(id) return nil } @@ -2610,7 +2708,7 @@ func (b *InMemoryBackend) GetTrafficPolicyInstance(id string) (*TrafficPolicyIns b.mu.RLock("GetTrafficPolicyInstance") defer b.mu.RUnlock() - inst, ok := b.trafficPolicyInstances[id] + inst, ok := b.trafficPolicyInstances.Get(id) if !ok { return nil, fmt.Errorf( "%w: traffic policy instance %s not found", @@ -2701,8 +2799,9 @@ func (b *InMemoryBackend) ListTrafficPolicyInstances() ([]*TrafficPolicyInstance b.mu.RLock("ListTrafficPolicyInstances") defer b.mu.RUnlock() - result := make([]*TrafficPolicyInstance, 0, len(b.trafficPolicyInstances)) - for _, inst := range b.trafficPolicyInstances { + all := b.trafficPolicyInstances.All() + result := make([]*TrafficPolicyInstance, 0, len(all)) + for _, inst := range all { cp := *inst result = append(result, &cp) } @@ -2717,11 +2816,11 @@ func (b *InMemoryBackend) DeleteCidrCollection(id string) error { b.mu.Lock("DeleteCidrCollection") defer b.mu.Unlock() - if _, ok := b.cidrCollections[id]; !ok { + if _, ok := b.cidrCollections.Get(id); !ok { return fmt.Errorf("%w: CIDR collection %s not found", ErrCidrCollectionNotFound, id) } - delete(b.cidrCollections, id) + b.cidrCollections.Delete(id) return nil } @@ -2731,8 +2830,9 @@ func (b *InMemoryBackend) ListCidrCollections() ([]*CidrCollection, error) { b.mu.RLock("ListCidrCollections") defer b.mu.RUnlock() - result := make([]*CidrCollection, 0, len(b.cidrCollections)) - for _, col := range b.cidrCollections { + all := b.cidrCollections.All() + result := make([]*CidrCollection, 0, len(all)) + for _, col := range all { cp := *col result = append(result, &cp) } @@ -2747,7 +2847,7 @@ func (b *InMemoryBackend) UpdateHostedZoneComment(zoneID, comment string) (*Host b.mu.Lock("UpdateHostedZoneComment") defer b.mu.Unlock() - zd, ok := b.zones[zoneID] + zd, ok := b.zones.Get(zoneID) if !ok { return nil, fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } @@ -2767,7 +2867,7 @@ func (b *InMemoryBackend) GetReusableDelegationSet(id string) (*ReusableDelegati b.mu.RLock("GetReusableDelegationSet") defer b.mu.RUnlock() - ds, ok := b.reusableDelegationSets[id] + ds, ok := b.reusableDelegationSets.Get(id) if !ok { return nil, fmt.Errorf("%w: delegation set %s not found", ErrDelegationSetNotFound, id) } @@ -2782,11 +2882,11 @@ func (b *InMemoryBackend) DeleteReusableDelegationSet(id string) error { b.mu.Lock("DeleteReusableDelegationSet") defer b.mu.Unlock() - if _, ok := b.reusableDelegationSets[id]; !ok { + if !b.reusableDelegationSets.Has(id) { return fmt.Errorf("%w: delegation set %s not found", ErrDelegationSetNotFound, id) } - delete(b.reusableDelegationSets, id) + b.reusableDelegationSets.Delete(id) return nil } @@ -2796,8 +2896,9 @@ func (b *InMemoryBackend) ListReusableDelegationSets() ([]*ReusableDelegationSet b.mu.RLock("ListReusableDelegationSets") defer b.mu.RUnlock() - result := make([]*ReusableDelegationSet, 0, len(b.reusableDelegationSets)) - for _, ds := range b.reusableDelegationSets { + all := b.reusableDelegationSets.All() + result := make([]*ReusableDelegationSet, 0, len(all)) + for _, ds := range all { cp := *ds result = append(result, &cp) } @@ -2816,7 +2917,7 @@ func (b *InMemoryBackend) UpdateTrafficPolicyInstance( b.mu.Lock("UpdateTrafficPolicyInstance") defer b.mu.Unlock() - inst, ok := b.trafficPolicyInstances[id] + inst, ok := b.trafficPolicyInstances.Get(id) if !ok { return nil, fmt.Errorf( "%w: traffic policy instance %s not found", @@ -2825,6 +2926,9 @@ func (b *InMemoryBackend) UpdateTrafficPolicyInstance( ) } + // inst.HostedZoneID (the only field trafficPolicyInstancesByZone indexes + // on) is never modified here, so the index stays consistent without a + // Put — see trafficPolicyInstanceZoneKeyFn's doc comment. if tpID != "" { inst.TrafficPolicyID = tpID inst.TrafficPolicyVersion = tpVersion @@ -2846,13 +2950,12 @@ func (b *InMemoryBackend) ListTrafficPolicyInstancesByHostedZone( b.mu.RLock("ListTrafficPolicyInstancesByHostedZone") defer b.mu.RUnlock() - var result []*TrafficPolicyInstance + zoneInstances := b.trafficPolicyInstancesByZone.Get(hostedZoneID) + result := make([]*TrafficPolicyInstance, 0, len(zoneInstances)) - for _, inst := range b.trafficPolicyInstances { - if inst.HostedZoneID == hostedZoneID { - cp := *inst - result = append(result, &cp) - } + for _, inst := range zoneInstances { + cp := *inst + result = append(result, &cp) } sort.Slice(result, func(i, j int) bool { return result[i].ID < result[j].ID }) @@ -2870,7 +2973,11 @@ func (b *InMemoryBackend) ListTrafficPolicyInstancesByPolicy( var result []*TrafficPolicyInstance - for _, inst := range b.trafficPolicyInstances { + // No secondary index on TrafficPolicyID: UpdateTrafficPolicyInstance can + // change it in place above, which would make such an index stale (see + // store.Index.AddIndex's doc comment on mutable index keys). A linear + // scan matches the original map-iteration behaviour exactly. + for _, inst := range b.trafficPolicyInstances.All() { if inst.TrafficPolicyID == tpID && (tpVersion == 0 || inst.TrafficPolicyVersion == tpVersion) { cp := *inst @@ -2888,7 +2995,7 @@ func (b *InMemoryBackend) AddHealthCheckInternal(hc HealthCheck) { b.mu.Lock("AddHealthCheckInternal") defer b.mu.Unlock() cp := hc - b.healthChecks[hc.ID] = &cp + b.healthChecks.Put(&cp) } // AddKeySigningKeyInternal adds a KSK directly into the backend for testing. @@ -2896,7 +3003,7 @@ func (b *InMemoryBackend) AddKeySigningKeyInternal(ksk KeySigningKey) { b.mu.Lock("AddKeySigningKeyInternal") defer b.mu.Unlock() cp := ksk - b.keySigningKeys[kskKey(ksk.HostedZoneID, ksk.Name)] = &cp + b.keySigningKeys.Put(&cp) } // AddTrafficPolicyInternal adds a traffic policy directly into the backend for testing. @@ -3065,7 +3172,7 @@ func (b *InMemoryBackend) TestDNSAnswer( b.mu.RLock("TestDNSAnswer") defer b.mu.RUnlock() - zd, ok := b.zones[zoneID] + zd, ok := b.zones.Get(zoneID) if !ok { return nil, fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } @@ -3090,7 +3197,7 @@ func (b *InMemoryBackend) GetHostedZoneCount() int { b.mu.RLock("GetHostedZoneCount") defer b.mu.RUnlock() - return len(b.zones) + return b.zones.Len() } // GetHealthCheckCount returns the total number of health checks. @@ -3098,7 +3205,7 @@ func (b *InMemoryBackend) GetHealthCheckCount() int { b.mu.RLock("GetHealthCheckCount") defer b.mu.RUnlock() - return len(b.healthChecks) + return b.healthChecks.Len() } // CountResourceRecordSets returns the number of resource record sets in the @@ -3107,7 +3214,7 @@ func (b *InMemoryBackend) CountResourceRecordSets(zoneID string) (int, error) { b.mu.RLock("CountResourceRecordSets") defer b.mu.RUnlock() - zd, ok := b.zones[zoneID] + zd, ok := b.zones.Get(zoneID) if !ok { return 0, fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } @@ -3121,7 +3228,7 @@ func (b *InMemoryBackend) CountAssociatedVPCs(zoneID string) (int, error) { b.mu.RLock("CountAssociatedVPCs") defer b.mu.RUnlock() - if _, ok := b.zones[zoneID]; !ok { + if _, ok := b.zones.Get(zoneID); !ok { return 0, fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, zoneID) } @@ -3135,7 +3242,7 @@ func (b *InMemoryBackend) CountZonesByReusableDelegationSet(id string) (int, err b.mu.RLock("CountZonesByReusableDelegationSet") defer b.mu.RUnlock() - if _, ok := b.reusableDelegationSets[id]; !ok { + if !b.reusableDelegationSets.Has(id) { return 0, fmt.Errorf("%w: delegation set %s not found", ErrDelegationSetNotFound, id) } @@ -3144,20 +3251,67 @@ func (b *InMemoryBackend) CountZonesByReusableDelegationSet(id string) (int, err return 0, nil } -func (b *InMemoryBackend) ListTagsForResource(resourceID string) map[string]string { +// tagResourceTypeHealthCheck and tagResourceTypeHostedZone are the wire +// values of the AWS TagResourceType enum used by the tag-family operations +// (ListTagsForResource[s], ChangeTagsForResource). +const ( + tagResourceTypeHealthCheck = "healthcheck" + tagResourceTypeHostedZone = "hostedzone" +) + +// checkTagResourceExists validates that resourceID exists as the given +// resourceType. AWS returns NoSuchHostedZone/NoSuchHealthCheck (404) for the +// tag-family operations when the target resource does not exist; an unknown +// resourceType is InvalidInput (400). +func (b *InMemoryBackend) checkTagResourceExists(resourceType, resourceID string) error { + switch resourceType { + case tagResourceTypeHostedZone: + if _, ok := b.zones.Get(resourceID); !ok { + return fmt.Errorf("%w: hosted zone %s not found", ErrHostedZoneNotFound, resourceID) + } + case tagResourceTypeHealthCheck: + if !b.healthChecks.Has(resourceID) { + return fmt.Errorf("%w: health check %s not found", ErrHealthCheckNotFound, resourceID) + } + default: + return fmt.Errorf("%w: unsupported ResourceType %q", ErrInvalidInput, resourceType) + } + + return nil +} + +// ListTagsForResource returns the tags for a single hosted zone or health check. +func (b *InMemoryBackend) ListTagsForResource(resourceType, resourceID string) (map[string]string, error) { b.mu.RLock("ListTagsForResource") defer b.mu.RUnlock() + + if err := b.checkTagResourceExists(resourceType, resourceID); err != nil { + return nil, err + } + if t, exists := b.tags[resourceID]; exists { - return t.Clone() + return t.Clone(), nil } - return make(map[string]string) + return make(map[string]string), nil } -func (b *InMemoryBackend) ListTagsForResources(resourceIDs []string) map[string]map[string]string { +// ListTagsForResources returns the tags for a batch of same-type resources. +// If any resourceID does not exist, the whole call fails (matching AWS, +// which validates the entire ResourceIds list before returning tags). +func (b *InMemoryBackend) ListTagsForResources( + resourceType string, + resourceIDs []string, +) (map[string]map[string]string, error) { b.mu.RLock("ListTagsForResources") defer b.mu.RUnlock() + for _, id := range resourceIDs { + if err := b.checkTagResourceExists(resourceType, id); err != nil { + return nil, err + } + } + result := make(map[string]map[string]string) for _, id := range resourceIDs { if t, ok := b.tags[id]; ok { @@ -3167,28 +3321,19 @@ func (b *InMemoryBackend) ListTagsForResources(resourceIDs []string) map[string] } } - return result + return result, nil } func (b *InMemoryBackend) ChangeTagsForResource( - resourceID string, + resourceType, resourceID string, addTags map[string]string, removeKeys []string, ) error { b.mu.Lock("ChangeTagsForResource") defer b.mu.Unlock() - // check if the resource exists - // route53 allows tagging hostedzones and healthchecks - var exists bool - if _, okZone := b.zones[resourceID]; okZone { - exists = okZone - } else if _, okHC := b.healthChecks[resourceID]; okHC { - exists = okHC - } - - if !exists { - return fmt.Errorf("%w: %s", ErrHostedZoneNotFound, resourceID) + if err := b.checkTagResourceExists(resourceType, resourceID); err != nil { + return err } if b.tags[resourceID] == nil { diff --git a/services/route53/export_test.go b/services/route53/export_test.go index 771110b0d..4d872d7ba 100644 --- a/services/route53/export_test.go +++ b/services/route53/export_test.go @@ -5,7 +5,7 @@ func ZoneCount(b *InMemoryBackend) int { b.mu.RLock("ZoneCount") defer b.mu.RUnlock() - return len(b.zones) + return b.zones.Len() } // HealthCheckCount returns the number of health checks in the backend for tests. @@ -13,7 +13,7 @@ func HealthCheckCount(b *InMemoryBackend) int { b.mu.RLock("HealthCheckCount") defer b.mu.RUnlock() - return len(b.healthChecks) + return b.healthChecks.Len() } // KeySigningKeyCount returns the number of KSKs in the backend for tests. @@ -21,7 +21,7 @@ func KeySigningKeyCount(b *InMemoryBackend) int { b.mu.RLock("KeySigningKeyCount") defer b.mu.RUnlock() - return len(b.keySigningKeys) + return b.keySigningKeys.Len() } // CidrCollectionCount returns the number of CIDR collections for tests. @@ -29,7 +29,7 @@ func CidrCollectionCount(b *InMemoryBackend) int { b.mu.RLock("CidrCollectionCount") defer b.mu.RUnlock() - return len(b.cidrCollections) + return b.cidrCollections.Len() } // QueryLoggingConfigCount returns the number of query logging configs for tests. @@ -37,7 +37,7 @@ func QueryLoggingConfigCount(b *InMemoryBackend) int { b.mu.RLock("QueryLoggingConfigCount") defer b.mu.RUnlock() - return len(b.queryLoggingConfigs) + return b.queryLoggingConfigs.Len() } // DelegationSetCount returns the number of reusable delegation sets for tests. @@ -45,7 +45,7 @@ func DelegationSetCount(b *InMemoryBackend) int { b.mu.RLock("DelegationSetCount") defer b.mu.RUnlock() - return len(b.reusableDelegationSets) + return b.reusableDelegationSets.Len() } // TrafficPolicyCount returns the number of traffic policies for tests. @@ -61,7 +61,7 @@ func TrafficPolicyInstanceCount(b *InMemoryBackend) int { b.mu.RLock("TrafficPolicyInstanceCount") defer b.mu.RUnlock() - return len(b.trafficPolicyInstances) + return b.trafficPolicyInstances.Len() } // VPCAssociationCount returns the total number of VPC associations across all zones for tests. diff --git a/services/route53/handler.go b/services/route53/handler.go index db599eb33..7fb7d6581 100644 --- a/services/route53/handler.go +++ b/services/route53/handler.go @@ -86,18 +86,6 @@ func (h *Handler) deleteTagsForResource(_ string) { // Let backend handle deletion on resource deletion } -func (h *Handler) setTags(resourceID string, kv map[string]string) { - _ = h.Backend.ChangeTagsForResource(resourceID, kv, nil) -} - -func (h *Handler) removeTags(resourceID string, keys []string) { - _ = h.Backend.ChangeTagsForResource(resourceID, nil, keys) -} - -func (h *Handler) getTags(resourceID string) map[string]string { - return h.Backend.ListTagsForResource(resourceID) -} - // Name returns the service name. func (h *Handler) Name() string { return "Route53" } @@ -761,8 +749,14 @@ func (h *Handler) routeHostedZoneDNSSEC(c *echo.Context, path, method string) (b } func (h *Handler) routeTags(c *echo.Context, path, method string) error { - // POST /2013-04-01/tags (no resource type suffix) → ListTagsForResources - if method == http.MethodPost && path == route53TagsPrefix[:len(route53TagsPrefix)-1] { + // Real Route 53 REST URIs are: + // GET/POST /2013-04-01/tags/{ResourceType}/{ResourceId} → ListTagsForResource / ChangeTagsForResource + // POST /2013-04-01/tags/{ResourceType} → ListTagsForResources (batch) + // i.e. the batch op carries only a ResourceType segment and no ResourceId. + // (This function is only reached for paths with the route53TagsPrefix, so + // rest is always non-empty here.) + rest := strings.TrimPrefix(path, route53TagsPrefix) + if method == http.MethodPost && !strings.Contains(rest, "/") { return h.listTagsForResources(c) } @@ -1420,7 +1414,11 @@ func (h *Handler) listTagsForResource(c *echo.Context, path string) error { resourceID = parts[1] } - tags := h.getTags(resourceID) + tags, err := h.Backend.ListTagsForResource(resourceType, resourceID) + if err != nil { + return handleBackendError(c, err) + } + tagList := make([]r53Tag, 0, len(tags)) for k, v := range tags { tagList = append(tagList, r53Tag{Key: k, Value: v}) @@ -1475,14 +1473,19 @@ func (h *Handler) changeTagsForResource(c *echo.Context) error { path := c.Request().URL.Path rest := strings.TrimPrefix(path, route53TagsPrefix) parts := strings.SplitN(rest, "/", 2) //nolint:mnd // path has two segments: type + id + resourceType := "" resourceID := "" + if len(parts) >= 1 { + resourceType = parts[0] + } + if len(parts) >= 2 { //nolint:mnd // path has two segments: type + id resourceID = parts[1] } - if err := h.applyTagChanges(resourceID, c.Request()); err != nil { - return xmlError(c, http.StatusBadRequest, "InvalidInput", err.Error()) + if err := h.applyTagChanges(resourceType, resourceID, c.Request()); err != nil { + return handleBackendError(c, err) } type changeTagsResp struct { @@ -1498,38 +1501,34 @@ type applyTagChangesInput struct { RemoveTagKeys []string `xml:"RemoveTagKeys>Key"` } -// applyTagChanges reads a ChangeTagsForResource XML body and applies the add/remove operations. -// It returns an error if the body cannot be read or parsed. -func (h *Handler) applyTagChanges(resourceID string, r *http.Request) error { +// applyTagChanges reads a ChangeTagsForResource XML body and applies the add/remove +// operations via a single backend call. Routing through one ChangeTagsForResource call +// (even when the request carries no tag mutations) ensures a nonexistent or +// wrong-resourceType target always surfaces the AWS NoSuchHostedZone/NoSuchHealthCheck +// error instead of silently no-oping. +func (h *Handler) applyTagChanges(resourceType, resourceID string, r *http.Request) error { body, err := httputils.ReadBody(r) if err != nil { - return fmt.Errorf("failed to read request body: %w", err) - } - - if len(body) == 0 { - return nil + return fmt.Errorf("%w: failed to read request body: %s", ErrInvalidInput, err.Error()) } var req applyTagChangesInput - if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { - return fmt.Errorf("failed to parse XML: %w", xmlErr) + if len(body) > 0 { + if xmlErr := xml.Unmarshal(body, &req); xmlErr != nil { + return fmt.Errorf("%w: failed to parse XML: %s", ErrInvalidInput, xmlErr.Error()) + } } + var addTags map[string]string if len(req.AddTags) > 0 { - kv := make(map[string]string, len(req.AddTags)) + addTags = make(map[string]string, len(req.AddTags)) for _, t := range req.AddTags { - kv[t.Key] = t.Value + addTags[t.Key] = t.Value } - - h.setTags(resourceID, kv) } - if len(req.RemoveTagKeys) > 0 { - h.removeTags(resourceID, req.RemoveTagKeys) - } - - return nil + return h.Backend.ChangeTagsForResource(resourceType, resourceID, addTags, req.RemoveTagKeys) } // writeXML marshals v to XML and writes it to the response. @@ -1582,7 +1581,7 @@ func xmlError(c *echo.Context, statusCode int, code, message string) error { // handleBackendError maps backend errors to HTTP responses. // -//nolint:cyclop // one branch per error type; this is an exhaustive mapping function +//nolint:cyclop,funlen // one branch per error type; this is an exhaustive mapping function func handleBackendError(c *echo.Context, err error) error { switch { case errors.Is(err, ErrHostedZoneNotFound): @@ -1592,11 +1591,13 @@ func handleBackendError(c *echo.Context, err error) error { case errors.Is(err, ErrKeySigningKeyNotFound): return xmlError(c, http.StatusNotFound, "NoSuchKeySigningKey", err.Error()) case errors.Is(err, ErrCidrCollectionNotFound): - return xmlError(c, http.StatusNotFound, "NoSuchCidrCollection", err.Error()) + return xmlError(c, http.StatusNotFound, "NoSuchCidrCollectionException", err.Error()) case errors.Is(err, ErrQueryLoggingConfigNotFound): return xmlError(c, http.StatusNotFound, "NoSuchQueryLoggingConfig", err.Error()) case errors.Is(err, ErrDelegationSetNotFound): - return xmlError(c, http.StatusNotFound, "NoSuchDelegationSet", err.Error()) + // AWS: NoSuchDelegationSet has httpStatusCode 400, unlike the other + // NoSuch* Route53 errors which are 404. + return xmlError(c, http.StatusBadRequest, "NoSuchDelegationSet", err.Error()) case errors.Is(err, ErrTrafficPolicyNotFound): return xmlError(c, http.StatusNotFound, "NoSuchTrafficPolicy", err.Error()) case errors.Is(err, ErrTrafficPolicyInstNotFound): @@ -1610,23 +1611,39 @@ func handleBackendError(c *echo.Context, err error) error { case errors.Is(err, ErrNoSuchGeoLocation): return xmlError(c, http.StatusNotFound, "NoSuchGeoLocation", err.Error()) case errors.Is(err, ErrQueryLoggingConfigAlreadyExists): - return xmlError(c, http.StatusBadRequest, "QueryLoggingConfigAlreadyExists", err.Error()) + // AWS: QueryLoggingConfigAlreadyExists has httpStatusCode 409. + return xmlError(c, http.StatusConflict, "QueryLoggingConfigAlreadyExists", err.Error()) case errors.Is(err, ErrPublicZoneVPCAssociation): return xmlError(c, http.StatusBadRequest, "PublicZoneVPCAssociation", err.Error()) case errors.Is(err, ErrVPCAssociationAuthorizationNF): return xmlError(c, http.StatusNotFound, "VPCAssociationAuthorizationNotFound", err.Error()) + case errors.Is(err, ErrVPCAssociationNotFound): + return xmlError(c, http.StatusNotFound, "VPCAssociationNotFound", err.Error()) case errors.Is(err, ErrKeySigningKeyWithActiveStatusNF): return xmlError(c, http.StatusBadRequest, "KeySigningKeyWithActiveStatusNotFound", err.Error()) case errors.Is(err, ErrTrafficPolicyInUse): return xmlError(c, http.StatusBadRequest, "TrafficPolicyInUse", err.Error()) - case errors.Is(err, ErrKeySigningKeyNotInactive): - return xmlError(c, http.StatusBadRequest, "KeySigningKeyNotInactive", err.Error()) + case errors.Is(err, ErrInvalidKeySigningKeyStatus): + return xmlError(c, http.StatusBadRequest, "InvalidKeySigningKeyStatus", err.Error()) case errors.Is(err, ErrTrafficPolicyAlreadyExists): - return xmlError(c, http.StatusBadRequest, "TrafficPolicyAlreadyExists", err.Error()) + // AWS: TrafficPolicyAlreadyExists has httpStatusCode 409. + return xmlError(c, http.StatusConflict, "TrafficPolicyAlreadyExists", err.Error()) case errors.Is(err, ErrHostedZoneNotEmpty): return xmlError(c, http.StatusBadRequest, "HostedZoneNotEmpty", err.Error()) case errors.Is(err, ErrLastVPCAssociation): return xmlError(c, http.StatusBadRequest, "LastVPCAssociation", err.Error()) + case errors.Is(err, ErrHostedZoneAlreadyExists): + return xmlError(c, http.StatusConflict, "HostedZoneAlreadyExists", err.Error()) + case errors.Is(err, ErrHealthCheckAlreadyExists): + return xmlError(c, http.StatusConflict, "HealthCheckAlreadyExists", err.Error()) + case errors.Is(err, ErrKeySigningKeyAlreadyExists): + return xmlError(c, http.StatusConflict, "KeySigningKeyAlreadyExists", err.Error()) + case errors.Is(err, ErrTrafficPolicyInstanceAlreadyExists): + return xmlError(c, http.StatusConflict, "TrafficPolicyInstanceAlreadyExists", err.Error()) + case errors.Is(err, ErrCidrCollectionAlreadyExists): + // AWS: CidrCollectionAlreadyExistsException has httpStatusCode 400 + // (unlike most other *AlreadyExists Route53 errors, which are 409). + return xmlError(c, http.StatusBadRequest, "CidrCollectionAlreadyExistsException", err.Error()) default: return xmlError(c, http.StatusInternalServerError, "InternalError", err.Error()) } diff --git a/services/route53/handler_accuracy_test.go b/services/route53/handler_accuracy_test.go index 53dd468b4..514db36e4 100644 --- a/services/route53/handler_accuracy_test.go +++ b/services/route53/handler_accuracy_test.go @@ -1286,9 +1286,11 @@ func TestCreateTrafficPolicy_NameUniqueness(t *testing.T) { rec := send(t, h, http.MethodPost, "/2013-04-01/trafficpolicy", body) require.Equal(t, http.StatusCreated, rec.Code) - // Second attempt with same name must fail. + // Second attempt with same name must fail. Real Route 53 returns + // TrafficPolicyAlreadyExists with httpStatusCode 409 (confirmed against + // the botocore api-2.json route53 model), not 400. rec = send(t, h, http.MethodPost, "/2013-04-01/trafficpolicy", body) - assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Equal(t, http.StatusConflict, rec.Code) assert.Contains(t, rec.Body.String(), "TrafficPolicyAlreadyExists") } @@ -1369,11 +1371,14 @@ func TestDeleteKeySigningKey_RequiresInactive(t *testing.T) { rec = send(t, h, http.MethodPost, "/2013-04-01/keysigningkey", kskBody) require.Equal(t, http.StatusCreated, rec.Code) - // Delete without deactivating first — must fail. + // Delete without deactivating first — must fail. Real Route 53 has no + // "KeySigningKeyNotInactive" error code; the wire error for this case is + // InvalidKeySigningKeyStatus (confirmed against aws-sdk-go-v2's route53 + // error types and the botocore api-2.json model, both http 400). rec = send(t, h, http.MethodDelete, "/2013-04-01/keysigningkey/"+zoneID+"/active-key", "") assert.Equal(t, http.StatusBadRequest, rec.Code) - assert.Contains(t, rec.Body.String(), "KeySigningKeyNotInactive") + assert.Contains(t, rec.Body.String(), "InvalidKeySigningKeyStatus") // Deactivate then delete — must succeed. rec = send(t, h, http.MethodPost, diff --git a/services/route53/handler_completeness.go b/services/route53/handler_completeness.go index 8d7334243..d4affd97a 100644 --- a/services/route53/handler_completeness.go +++ b/services/route53/handler_completeness.go @@ -1096,7 +1096,10 @@ func (h *Handler) listTagsForResources(c *echo.Context) error { return xmlError(c, http.StatusBadRequest, "InvalidInput", "failed to parse XML: "+err.Error()) } - tagsMap := h.Backend.ListTagsForResources(req.ResourceIDs) + tagsMap, err := h.Backend.ListTagsForResources(req.ResourceType, req.ResourceIDs) + if err != nil { + return handleBackendError(c, err) + } var resourceTagSets []xmlResourceTagSet for _, id := range req.ResourceIDs { diff --git a/services/route53/handler_refinement1_test.go b/services/route53/handler_refinement1_test.go index b6327134c..79f39f805 100644 --- a/services/route53/handler_refinement1_test.go +++ b/services/route53/handler_refinement1_test.go @@ -175,7 +175,11 @@ func TestRefinement1_DuplicateKSK(t *testing.T) { _, err = b.CreateKeySigningKey(hz.ID, "caller-2", tt.kskName, "arn:kms:test", "") if tt.wantErr { require.Error(t, err) - assert.ErrorIs(t, err, route53.ErrInvalidInput) + // Real Route 53 returns KeySigningKeyAlreadyExists (409) for a + // duplicate name within a hosted zone, per the CreateKeySigningKey + // error list in the aws-sdk-go-v2 route53 model — not the generic + // InvalidInput this test previously asserted. + assert.ErrorIs(t, err, route53.ErrKeySigningKeyAlreadyExists) } else { require.NoError(t, err) } diff --git a/services/route53/interfaces.go b/services/route53/interfaces.go index 233669f6a..99993bd05 100644 --- a/services/route53/interfaces.go +++ b/services/route53/interfaces.go @@ -99,10 +99,13 @@ type StorageBackend interface { ListTrafficPolicyInstancesByHostedZone(hostedZoneID string) ([]*TrafficPolicyInstance, error) ListTrafficPolicyInstancesByPolicy(tpID string, tpVersion int32) ([]*TrafficPolicyInstance, error) - // Tags operations - ListTagsForResource(resourceID string) map[string]string - ListTagsForResources(resourceIDs []string) map[string]map[string]string - ChangeTagsForResource(resourceID string, addTags map[string]string, removeKeys []string) error + // Tags operations. resourceType is the AWS TagResourceType wire value + // ("hostedzone" or "healthcheck"); it is used to validate that the + // resource actually exists (and to pick the right NoSuch* error) before + // reading or mutating tags. + ListTagsForResource(resourceType, resourceID string) (map[string]string, error) + ListTagsForResources(resourceType string, resourceIDs []string) (map[string]map[string]string, error) + ChangeTagsForResource(resourceType, resourceID string, addTags map[string]string, removeKeys []string) error // Lifecycle Reset() diff --git a/services/route53/parity_sweep3_test.go b/services/route53/parity_sweep3_test.go new file mode 100644 index 000000000..0178bbdcf --- /dev/null +++ b/services/route53/parity_sweep3_test.go @@ -0,0 +1,530 @@ +package route53_test + +// parity_sweep3_test.go — AWS-accuracy audit for Route53, parity sweep 3 +// (gopherstack-8l0). +// +// Covers concrete deviations from AWS behaviour found in backend.go/handler.go: +// +// 1. Tag-family ops (ListTagsForResource, ListTagsForResources, +// ChangeTagsForResource) never validated that the target resource +// existed. ListTagsForResource[s] silently returned an empty tag set for +// an unknown hosted zone/health check (200 OK), and +// ChangeTagsForResource's existence check was real in the backend but +// the handler discarded its error return, so tagging a nonexistent +// resource silently "succeeded". Real AWS returns NoSuchHostedZone / +// NoSuchHealthCheck (404) for all three ops. Fix: backend now takes a +// resourceType and validates existence; the handler no longer discards +// the backend error. +// +// 2. DisassociateVPCFromHostedZone for a VPC that isn't associated returned +// the generic InvalidInput (400). Real AWS returns VPCAssociationNotFound +// (404). +// +// 3. CreateTrafficPolicyInstance allowed multiple instances for the same +// (hostedZoneID, name) pair. Real AWS returns +// TrafficPolicyInstanceAlreadyExists (409). +// +// 4. Resource tags (b.tags) were never wired into Snapshot/Restore, so tags +// silently vanished across a backend restore even though persistence was +// otherwise enabled for the service. +// +// 5. Wire-code/status mismatches found by diffing every route53 sentinel +// error against the aws-sdk-go-v2 route53 model +// (types/errors.go + api-2.json httpStatusCode): +// NoSuchCidrCollection -> NoSuchCidrCollectionException (404, code fix only) +// NoSuchDelegationSet: 404 -> 400 (status fix only) +// QueryLoggingConfigAlreadyExists: 400 -> 409 +// TrafficPolicyAlreadyExists: 400 -> 409 (covered in handler_accuracy_test.go) +// KeySigningKeyNotInactive (fabricated code, doesn't exist in AWS) -> InvalidKeySigningKeyStatus +// CreateKeySigningKey duplicate name: InvalidInput -> KeySigningKeyAlreadyExists (409) +// +// 6. CreateCidrCollection allowed multiple collections with the same name. +// Real AWS returns CidrCollectionAlreadyExistsException (400). +// +// 7. ListTagsForResources' HTTP route was unreachable: the handler checked +// for a bare "/2013-04-01/tags" POST, but that path fails the earlier +// route53TagsPrefix ("/2013-04-01/tags/") HasPrefix check, and even if it +// didn't, the real AWS request URI is POST /2013-04-01/tags/{ResourceType} +// (no trailing ResourceId) per the botocore route53 model — a path this +// handler previously misrouted to ChangeTagsForResource. +// +// 8. CreateHealthCheck accepted a CALCULATED health check whose +// HealthThreshold exceeded the number of ChildHealthChecks (e.g. +// threshold=5 with only 2 children), which AWS rejects with InvalidInput. + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/route53" +) + +// --------------------------------------------------------------------------- +// Issue #1 — tag-family ops must validate resource existence +// --------------------------------------------------------------------------- + +func TestParitySweep3_ListTagsForResource_NoSuchResource(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + wantBody string + }{ + { + name: "unknown_hostedzone", + path: "/2013-04-01/tags/hostedzone/ZDOESNOTEXIST", + wantBody: "NoSuchHostedZone", + }, + { + name: "unknown_healthcheck", + path: "/2013-04-01/tags/healthcheck/DOESNOTEXIST", + wantBody: "NoSuchHealthCheck", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newHandler(t) + + rec := send(t, h, http.MethodGet, tt.path, "") + assert.Equal(t, http.StatusNotFound, rec.Code, + "ListTagsForResource on a nonexistent resource must 404, not silently return empty tags") + assert.Contains(t, rec.Body.String(), tt.wantBody) + }) + } +} + +func TestParitySweep3_ChangeTagsForResource_NoSuchResource(t *testing.T) { + t.Parallel() + + const body = ` + + envprod +` + + tests := []struct { + name string + path string + wantBody string + }{ + { + name: "unknown_hostedzone", + path: "/2013-04-01/tags/hostedzone/ZDOESNOTEXIST", + wantBody: "NoSuchHostedZone", + }, + { + name: "unknown_healthcheck", + path: "/2013-04-01/tags/healthcheck/DOESNOTEXIST", + wantBody: "NoSuchHealthCheck", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newHandler(t) + + rec := send(t, h, http.MethodPost, tt.path, body) + assert.Equal(t, http.StatusNotFound, rec.Code, + "ChangeTagsForResource on a nonexistent resource must 404, not silently succeed") + assert.Contains(t, rec.Body.String(), tt.wantBody) + + // Verify the tag was genuinely not applied anywhere. + listRec := send(t, h, http.MethodGet, tt.path, "") + assert.Equal(t, http.StatusNotFound, listRec.Code) + }) + } +} + +func TestParitySweep3_ListTagsForResources_NoSuchResource(t *testing.T) { + t.Parallel() + + h := newHandler(t) + + rec := send(t, h, http.MethodPost, "/2013-04-01/hostedzone", createZoneXML) + require.Equal(t, http.StatusCreated, rec.Code) + zoneID := extractZoneID(t, rec.Body.String()) + + body := ` + + hostedzone + + ` + zoneID + ` + ZDOESNOTEXIST + +` + + // Real AWS ListTagsForResources URI is POST /2013-04-01/tags/{ResourceType} + // (a bare "/2013-04-01/tags" request, with no ResourceType segment, is not + // a valid Route53 endpoint at all). + got := send(t, h, http.MethodPost, "/2013-04-01/tags/hostedzone", body) + assert.Equal(t, http.StatusNotFound, got.Code, + "batch tag lookup with any unknown resource ID must 404, not return partial results") + assert.Contains(t, got.Body.String(), "NoSuchHostedZone") +} + +func TestParitySweep3_TagRoundTrip_HealthCheck(t *testing.T) { + t.Parallel() + + h := newHandler(t) + + rec := send(t, h, http.MethodPost, "/2013-04-01/healthcheck", createHealthCheckXML) + require.Equal(t, http.StatusCreated, rec.Code) + hcID := extractHealthCheckID(t, rec.Body.String()) + + tagsPath := "/2013-04-01/tags/healthcheck/" + hcID + + addBody := ` + + ownernet-team +` + + addRec := send(t, h, http.MethodPost, tagsPath, addBody) + require.Equal(t, http.StatusOK, addRec.Code, + "ChangeTagsForResource on a real health check must still succeed") + + listRec := send(t, h, http.MethodGet, tagsPath, "") + assert.Equal(t, http.StatusOK, listRec.Code) + assert.Contains(t, listRec.Body.String(), "net-team") +} + +// --------------------------------------------------------------------------- +// Issue #2 — DisassociateVPCFromHostedZone: not-associated VPC +// --------------------------------------------------------------------------- + +func TestParitySweep3_DisassociateVPC_NotAssociated(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + }{ + {name: "vpc_never_associated_returns_not_found"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := route53.NewInMemoryBackend() + + hz, err := b.CreateHostedZone("example.com", "ref-priv", "", true) + require.NoError(t, err) + + require.NoError(t, b.AssociateVPCWithHostedZone(hz.ID, "vpc-aaa", "us-east-1")) + + // vpc-bbb was never associated, so the disassociate must be + // rejected with VPCAssociationNotFound, not the generic + // InvalidInput this backend previously returned. + err = b.DisassociateVPCFromHostedZone(hz.ID, "vpc-bbb") + require.Error(t, err) + assert.ErrorIs(t, err, route53.ErrVPCAssociationNotFound) + }) + } +} + +// --------------------------------------------------------------------------- +// Issue #3 — CreateTrafficPolicyInstance duplicate (zone, name) +// --------------------------------------------------------------------------- + +func TestParitySweep3_CreateTrafficPolicyInstance_Duplicate(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + }{ + {name: "duplicate_name_in_zone_rejected"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := route53.NewInMemoryBackend() + + hz, err := b.CreateHostedZone("example.com", "ref-tpi", "", false) + require.NoError(t, err) + + tp, err := b.CreateTrafficPolicy( + "tpi-dup-policy", + `{"AWSPolicyFormatVersion":"2015-10-01","RecordType":"A","Endpoints":{},"Rules":{}}`, + "", + ) + require.NoError(t, err) + + _, err = b.CreateTrafficPolicyInstance(hz.ID, "www.example.com", tp.ID, tp.Version, 60) + require.NoError(t, err) + + _, err = b.CreateTrafficPolicyInstance(hz.ID, "www.example.com", tp.ID, tp.Version, 60) + require.Error(t, err) + assert.ErrorIs(t, err, route53.ErrTrafficPolicyInstanceAlreadyExists) + }) + } +} + +// --------------------------------------------------------------------------- +// Issue #4 — tags must survive Snapshot/Restore +// --------------------------------------------------------------------------- + +func TestParitySweep3_TagsPersistAcrossSnapshotRestore(t *testing.T) { + t.Parallel() + + original := route53.NewInMemoryBackend() + + hz, err := original.CreateHostedZone("example.com", "ref-tags-persist", "", false) + require.NoError(t, err) + + require.NoError(t, original.ChangeTagsForResource( + "hostedzone", hz.ID, map[string]string{"env": "prod"}, nil, + )) + + snap := original.Snapshot(t.Context()) + require.NotNil(t, snap) + + fresh := route53.NewInMemoryBackend() + require.NoError(t, fresh.Restore(t.Context(), snap)) + + tags, err := fresh.ListTagsForResource("hostedzone", hz.ID) + require.NoError(t, err) + assert.Equal(t, "prod", tags["env"], + "resource tags must be wired into backendSnapshot and survive a restore") +} + +// --------------------------------------------------------------------------- +// Issue #5 — wire-code / http-status corrections +// --------------------------------------------------------------------------- + +func TestParitySweep3_ErrorWireCodesAndStatuses(t *testing.T) { + t.Parallel() + + tests := []struct { + run func(t *testing.T, h *route53.Handler) + name string + }{ + { + name: "unknown_cidr_collection_uses_exception_suffix", + run: func(t *testing.T, h *route53.Handler) { + t.Helper() + + // GET /cidrcollection/{Id} (no "/cidrblocks" suffix) routes to + // ListCidrLocations, which needs no request body. + rec := send(t, h, http.MethodGet, "/2013-04-01/cidrcollection/DOESNOTEXIST", "") + assert.Equal(t, http.StatusNotFound, rec.Code) + assert.Contains(t, rec.Body.String(), "NoSuchCidrCollectionException") + }, + }, + { + name: "unknown_delegation_set_is_400", + run: func(t *testing.T, h *route53.Handler) { + t.Helper() + + rec := send(t, h, http.MethodDelete, "/2013-04-01/delegationset/N-DOESNOTEXIST", "") + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "NoSuchDelegationSet") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newHandler(t) + tt.run(t, h) + }) + } +} + +func TestParitySweep3_QueryLoggingConfigAlreadyExists_Is409(t *testing.T) { + t.Parallel() + + h := newHandler(t) + + rec := send(t, h, http.MethodPost, "/2013-04-01/hostedzone", createZoneXML) + require.Equal(t, http.StatusCreated, rec.Code) + zoneID := extractZoneID(t, rec.Body.String()) + + body := ` + + ` + zoneID + ` + arn:aws:logs:us-east-1:123456789012:log-group:/r53/test +` + + first := send(t, h, http.MethodPost, "/2013-04-01/queryloggingconfig", body) + require.Equal(t, http.StatusCreated, first.Code) + + second := send(t, h, http.MethodPost, "/2013-04-01/queryloggingconfig", body) + assert.Equal(t, http.StatusConflict, second.Code, + "real AWS QueryLoggingConfigAlreadyExists has httpStatusCode 409, not 400") + assert.Contains(t, second.Body.String(), "QueryLoggingConfigAlreadyExists") +} + +func TestParitySweep3_HostedZoneAlreadyExists_HTTP(t *testing.T) { + t.Parallel() + + h := newHandler(t) + + const body1 = ` + + example.com + shared-ref +` + + const body2 = ` + + different.com + shared-ref +` + + first := send(t, h, http.MethodPost, "/2013-04-01/hostedzone", body1) + require.Equal(t, http.StatusCreated, first.Code) + + second := send(t, h, http.MethodPost, "/2013-04-01/hostedzone", body2) + assert.Equal(t, http.StatusConflict, second.Code) + assert.Contains(t, second.Body.String(), "HostedZoneAlreadyExists") +} + +func TestParitySweep3_HealthCheckAlreadyExists_HTTP(t *testing.T) { + t.Parallel() + + h := newHandler(t) + + const body1 = ` + + shared-hc-ref + HTTP80 +` + + const body2 = ` + + shared-hc-ref + HTTPS443 +` + + first := send(t, h, http.MethodPost, "/2013-04-01/healthcheck", body1) + require.Equal(t, http.StatusCreated, first.Code) + + second := send(t, h, http.MethodPost, "/2013-04-01/healthcheck", body2) + assert.Equal(t, http.StatusConflict, second.Code) + assert.Contains(t, second.Body.String(), "HealthCheckAlreadyExists") +} + +// --------------------------------------------------------------------------- +// Issue #6 — CreateCidrCollection duplicate name +// --------------------------------------------------------------------------- + +func TestParitySweep3_CreateCidrCollection_DuplicateName(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + }{ + {name: "duplicate_collection_name_rejected"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newHandler(t) + + body := ` + + dup-cidrs + cidr-ref-dup +` + + first := send(t, h, http.MethodPost, "/2013-04-01/cidrcollection", body) + require.Equal(t, http.StatusCreated, first.Code) + + second := send(t, h, http.MethodPost, "/2013-04-01/cidrcollection", body) + assert.Equal(t, http.StatusBadRequest, second.Code, + "real AWS CidrCollectionAlreadyExistsException has httpStatusCode 400") + assert.Contains(t, second.Body.String(), "CidrCollectionAlreadyExistsException") + }) + } +} + +// --------------------------------------------------------------------------- +// Issue #7 — ListTagsForResources HTTP routing was unreachable/misrouted +// --------------------------------------------------------------------------- + +func TestParitySweep3_ListTagsForResources_HTTPRoute(t *testing.T) { + t.Parallel() + + h := newHandler(t) + + rec := send(t, h, http.MethodPost, "/2013-04-01/hostedzone", createZoneXML) + require.Equal(t, http.StatusCreated, rec.Code) + zoneID := extractZoneID(t, rec.Body.String()) + + addBody := ` + + envprod +` + addRec := send(t, h, http.MethodPost, "/2013-04-01/tags/hostedzone/"+zoneID, addBody) + require.Equal(t, http.StatusOK, addRec.Code) + + body := ` + + hostedzone + ` + zoneID + ` +` + + // Real AWS ListTagsForResources request URI: POST /2013-04-01/tags/{ResourceType}. + got := send(t, h, http.MethodPost, "/2013-04-01/tags/hostedzone", body) + assert.Equal(t, http.StatusOK, got.Code, + "ListTagsForResources must be reachable at its real AWS request URI") + assert.Contains(t, got.Body.String(), "ListTagsForResourcesResponse") + assert.Contains(t, got.Body.String(), "prod") +} + +// --------------------------------------------------------------------------- +// Issue #8 — CALCULATED health check HealthThreshold vs ChildHealthChecks +// --------------------------------------------------------------------------- + +func TestParitySweep3_CreateHealthCheck_CalculatedThresholdExceedsChildren(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + }{ + {name: "threshold_exceeds_child_count_rejected"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := route53.NewInMemoryBackend() + + child1, err := b.CreateHealthCheck("child-ref-1", route53.HealthCheckConfig{ + Type: route53.HealthCheckTypeHTTP, + Port: 80, + }) + require.NoError(t, err) + + child2, err := b.CreateHealthCheck("child-ref-2", route53.HealthCheckConfig{ + Type: route53.HealthCheckTypeHTTP, + Port: 80, + }) + require.NoError(t, err) + + const thresholdExceedsChildCount = 5 + + _, err = b.CreateHealthCheck("calc-ref-1", route53.HealthCheckConfig{ + Type: route53.HealthCheckTypeCalculated, + ChildHealthChecks: []string{child1.ID, child2.ID}, + HealthThreshold: thresholdExceedsChildCount, + }) + require.Error(t, err) + assert.ErrorIs(t, err, route53.ErrInvalidInput) + }) + } +} diff --git a/services/route53/persistence.go b/services/route53/persistence.go index 7ecad74a0..a2ff297af 100644 --- a/services/route53/persistence.go +++ b/services/route53/persistence.go @@ -3,29 +3,60 @@ package route53 import ( "context" "encoding/json" + "fmt" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" + svcTags "github.com/blackbirdworks/gopherstack/pkgs/tags" ) +// route53SnapshotVersion identifies the shape of [backendSnapshot]. It must be +// bumped whenever a change to zoneDataSnapshot, backendSnapshot, or any table +// registered below would make an older snapshot unsafe to decode as the +// current shape. Restore compares this against the persisted value and +// discards (rather than attempts to partially decode) any mismatch — see +// Restore. +const route53SnapshotVersion = 1 + +// zoneDataSnapshot captures the serialisable fields of a zoneData. It exists +// as a separate DTO (rather than JSON tags directly on zoneData) because +// zoneData's fields are unexported -- encoding/json cannot marshal them at +// all, let alone round-trip them -- making zones the one "dirty" table in +// this backend that needs its own DTO-registry (see store_setup.go's doc +// comment for the clean/dirty split across all converted tables). type zoneDataSnapshot struct { Records map[string]*ResourceRecordSet `json:"records"` Zone HostedZone `json:"zone"` DNSSECEnabled bool `json:"dnssecEnabled,omitempty"` } +// zoneDataSnapshotKey is the [store.Table] key function used for the +// ephemeral DTO table built inside Snapshot/Restore. It mirrors zoneKeyFn so +// the on-disk table is keyed identically to the live b.zones table. +func zoneDataSnapshotKey(zs *zoneDataSnapshot) string { return zs.Zone.ID } + +// backendSnapshot is the top-level on-disk shape for the Route 53 backend. +// +// Tables holds one JSON-encoded array per registered table, produced by +// [store.Registry.SnapshotAll]: "zones" ([]*zoneDataSnapshot, via a throwaway +// DTO registry because zoneData is "dirty"), plus the seven "clean" tables +// registered directly on b.registry ("healthChecks", "keySigningKeys", +// "cidrCollections", "queryLoggingConfigs", "reusableDelegationSets", +// "trafficPolicyInstances", "changes") whose live struct types are already +// JSON-safe. TrafficPolicies, VPCAssociations, VPCAssocAuthorizations, and +// Tags are the fields store_setup.go documents as left as plain maps, so they +// are marshaled the same way they always were, outside the Tables map. +// Version guards against decoding a snapshot from an incompatible (older or +// newer) build of this backend as though it were the current shape; see +// Restore. type backendSnapshot struct { - Zones map[string]*zoneDataSnapshot `json:"zones"` - HealthChecks map[string]*HealthCheck `json:"healthChecks,omitempty"` - KeySigningKeys map[string]*KeySigningKey `json:"keySigningKeys,omitempty"` - CidrCollections map[string]*CidrCollection `json:"cidrCollections,omitempty"` - QueryLoggingConfigs map[string]*QueryLoggingConfig `json:"queryLoggingConfigs,omitempty"` - ReusableDelegationSets map[string]*ReusableDelegationSet `json:"reusableDelegationSets,omitempty"` + Tables map[string]json.RawMessage `json:"tables"` TrafficPolicies map[string][]*TrafficPolicy `json:"trafficPolicies,omitempty"` - TrafficPolicyInstances map[string]*TrafficPolicyInstance `json:"trafficPolicyInstances,omitempty"` VPCAssociations map[string][]vpcAssociation `json:"vpcAssociations,omitempty"` VPCAssocAuthorizations map[string][]VPCAssociationAuthorization `json:"vpcAssocAuthorizations,omitempty"` - Changes map[string]*ChangeInfo `json:"changes,omitempty"` + Tags map[string]*svcTags.Tags `json:"tags,omitempty"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. @@ -34,79 +65,47 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() - snap := backendSnapshot{ - Zones: make(map[string]*zoneDataSnapshot, len(b.zones)), - HealthChecks: make(map[string]*HealthCheck, len(b.healthChecks)), - KeySigningKeys: make(map[string]*KeySigningKey, len(b.keySigningKeys)), - CidrCollections: make(map[string]*CidrCollection, len(b.cidrCollections)), - QueryLoggingConfigs: make(map[string]*QueryLoggingConfig, len(b.queryLoggingConfigs)), - ReusableDelegationSets: make(map[string]*ReusableDelegationSet, len(b.reusableDelegationSets)), - TrafficPolicies: make(map[string][]*TrafficPolicy, len(b.trafficPolicies)), - TrafficPolicyInstances: make(map[string]*TrafficPolicyInstance, len(b.trafficPolicyInstances)), - VPCAssociations: make(map[string][]vpcAssociation, len(b.vpcAssociations)), - VPCAssocAuthorizations: make(map[string][]VPCAssociationAuthorization, len(b.vpcAssocAuthorizations)), - Changes: make(map[string]*ChangeInfo, len(b.changes)), + // The seven "clean" tables can be snapshotted straight off b.registry: + // their live struct types (HealthCheck, KeySigningKey, CidrCollection, + // QueryLoggingConfig, ReusableDelegationSet, TrafficPolicyInstance, + // ChangeInfo) are already JSON-safe, so no DTO mapping is needed. + tables, err := b.registry.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, "route53: snapshot table marshal failed", "error", err) + + return nil } - for id, zd := range b.zones { - snap.Zones[id] = &zoneDataSnapshot{ + // zones is "dirty" (zoneData's fields are all unexported) so it needs a + // throwaway DTO registry purely to reuse store's deterministic, type-erased + // JSON encoding instead of hand-rolling the marshal step. + dtoReg := store.NewRegistry() + zoneDTOs := store.Register(dtoReg, "zones", store.New(zoneDataSnapshotKey)) + + for _, zd := range b.zones.Snapshot() { + zoneDTOs.Put(&zoneDataSnapshot{ Zone: zd.zone, Records: zd.records, DNSSECEnabled: zd.dnssecEnabled, - } - } - - for id, hc := range b.healthChecks { - cp := *hc - snap.HealthChecks[id] = &cp - } - - for k, ksk := range b.keySigningKeys { - cp := *ksk - snap.KeySigningKeys[k] = &cp - } - - for id, col := range b.cidrCollections { - cp := *col - snap.CidrCollections[id] = &cp - } - - for id, cfg := range b.queryLoggingConfigs { - cp := *cfg - snap.QueryLoggingConfigs[id] = &cp + }) } - for id, ds := range b.reusableDelegationSets { - cp := *ds - snap.ReusableDelegationSets[id] = &cp - } - - for id, versions := range b.trafficPolicies { - versionsCopy := make([]*TrafficPolicy, len(versions)) - for i, tp := range versions { - cp := *tp - versionsCopy[i] = &cp - } - - snap.TrafficPolicies[id] = versionsCopy - } - - for id, inst := range b.trafficPolicyInstances { - cp := *inst - snap.TrafficPolicyInstances[id] = &cp - } + zoneTables, err := dtoReg.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, "route53: snapshot zones marshal failed", "error", err) - for id, assocs := range b.vpcAssociations { - snap.VPCAssociations[id] = append([]vpcAssociation(nil), assocs...) + return nil } - for id, auths := range b.vpcAssocAuthorizations { - snap.VPCAssocAuthorizations[id] = append([]VPCAssociationAuthorization(nil), auths...) - } + tables["zones"] = zoneTables["zones"] - for id, ch := range b.changes { - cp := *ch - snap.Changes[id] = &cp + snap := backendSnapshot{ + Version: route53SnapshotVersion, + Tables: tables, + TrafficPolicies: b.trafficPolicies, + VPCAssociations: b.vpcAssociations, + VPCAssocAuthorizations: b.vpcAssocAuthorizations, + Tags: b.tags, } data, err := json.Marshal(snap) @@ -119,165 +118,103 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { return data } -// ensureNonNilMaps initialises any nil maps in the snapshot. -func ensureNonNilMaps(snap *backendSnapshot) { - if snap.Zones == nil { - snap.Zones = make(map[string]*zoneDataSnapshot) - } - - if snap.HealthChecks == nil { - snap.HealthChecks = make(map[string]*HealthCheck) - } - - if snap.KeySigningKeys == nil { - snap.KeySigningKeys = make(map[string]*KeySigningKey) - } - - if snap.CidrCollections == nil { - snap.CidrCollections = make(map[string]*CidrCollection) - } - - if snap.QueryLoggingConfigs == nil { - snap.QueryLoggingConfigs = make(map[string]*QueryLoggingConfig) - } - - if snap.ReusableDelegationSets == nil { - snap.ReusableDelegationSets = make(map[string]*ReusableDelegationSet) - } - - if snap.TrafficPolicies == nil { - snap.TrafficPolicies = make(map[string][]*TrafficPolicy) - } - - if snap.TrafficPolicyInstances == nil { - snap.TrafficPolicyInstances = make(map[string]*TrafficPolicyInstance) - } - - if snap.VPCAssociations == nil { - snap.VPCAssociations = make(map[string][]vpcAssociation) - } - - if snap.VPCAssocAuthorizations == nil { - snap.VPCAssocAuthorizations = make(map[string][]VPCAssociationAuthorization) - } +// Restore loads backend state from a JSON snapshot. +// It implements persistence.Persistable. +// The DNS registrar is not restored — it must be re-wired by the caller after restore. +func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { + var snap backendSnapshot - if snap.Changes == nil { - snap.Changes = make(map[string]*ChangeInfo) + if err := persistence.UnmarshalSnapshot(ctx, "route53", data, &snap); err != nil { + return err } -} - -// restoreSimpleMaps restores the simple (non-zone, non-traffic-policy) maps from a snapshot. -func (b *InMemoryBackend) restoreSimpleMaps(snap *backendSnapshot) { - b.healthChecks = make(map[string]*HealthCheck, len(snap.HealthChecks)) - for id, hc := range snap.HealthChecks { - cp := *hc - b.healthChecks[id] = &cp - } + b.mu.Lock("Restore") + defer b.mu.Unlock() - b.keySigningKeys = make(map[string]*KeySigningKey, len(snap.KeySigningKeys)) + if snap.Version != route53SnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape — that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. + logger.Load(ctx).WarnContext(ctx, + "route53: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", route53SnapshotVersion) + + b.registry.ResetAll() + b.trafficPolicies = make(map[string][]*TrafficPolicy) + b.vpcAssociations = make(map[string][]vpcAssociation) + b.vpcAssocAuthorizations = make(map[string][]VPCAssociationAuthorization) + b.tags = make(map[string]*svcTags.Tags) - for k, ksk := range snap.KeySigningKeys { - cp := *ksk - b.keySigningKeys[k] = &cp + return nil } - b.cidrCollections = make(map[string]*CidrCollection, len(snap.CidrCollections)) - - for id, col := range snap.CidrCollections { - cp := *col - b.cidrCollections[id] = &cp - } + // b.registry.RestoreAll restores the seven "clean" tables directly, but + // must never see "zones": zoneData's fields are all unexported, so + // unmarshaling snap.Tables["zones"] into the live Table[zoneData] would + // silently produce zero-valued garbage (every entry keying to the same + // empty zone ID). zones is restored separately below via its own DTO + // registry, so it is excluded here. + cleanTables := make(map[string]json.RawMessage, len(snap.Tables)) - b.queryLoggingConfigs = make(map[string]*QueryLoggingConfig, len(snap.QueryLoggingConfigs)) + for name, raw := range snap.Tables { + if name == "zones" { + continue + } - for id, cfg := range snap.QueryLoggingConfigs { - cp := *cfg - b.queryLoggingConfigs[id] = &cp + cleanTables[name] = raw } - b.reusableDelegationSets = make(map[string]*ReusableDelegationSet, len(snap.ReusableDelegationSets)) - - for id, ds := range snap.ReusableDelegationSets { - cp := *ds - b.reusableDelegationSets[id] = &cp + if err := b.registry.RestoreAll(cleanTables); err != nil { + return fmt.Errorf("route53: restore snapshot tables: %w", err) } -} -// restoreAssocMaps restores VPC association, authorization, and change maps from a snapshot. -func (b *InMemoryBackend) restoreAssocMaps(snap *backendSnapshot) { - b.vpcAssociations = make(map[string][]vpcAssociation, len(snap.VPCAssociations)) + dtoReg := store.NewRegistry() + zoneDTOs := store.Register(dtoReg, "zones", store.New(zoneDataSnapshotKey)) - for id, assocs := range snap.VPCAssociations { - b.vpcAssociations[id] = append([]vpcAssociation(nil), assocs...) + if err := dtoReg.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("route53: restore snapshot zones: %w", err) } - b.vpcAssocAuthorizations = make(map[string][]VPCAssociationAuthorization, len(snap.VPCAssocAuthorizations)) - - for id, auths := range snap.VPCAssocAuthorizations { - b.vpcAssocAuthorizations[id] = append([]VPCAssociationAuthorization(nil), auths...) - } + liveZones := make([]*zoneData, 0, zoneDTOs.Len()) - b.changes = make(map[string]*ChangeInfo, len(snap.Changes)) + for _, zs := range zoneDTOs.All() { + records := zs.Records + if records == nil { + records = make(map[string]*ResourceRecordSet) + } - for id, ch := range snap.Changes { - cp := *ch - b.changes[id] = &cp + liveZones = append(liveZones, &zoneData{ + zone: zs.Zone, + records: records, + dnssecEnabled: zs.DNSSECEnabled, + }) } -} -// Restore loads backend state from a JSON snapshot. -// It implements persistence.Persistable. -// The DNS registrar is not restored — it must be re-wired by the caller after restore. -func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { - var snap backendSnapshot + b.zones.Restore(liveZones) - if err := persistence.UnmarshalSnapshot(ctx, "route53", data, &snap); err != nil { - return err + b.trafficPolicies = snap.TrafficPolicies + if b.trafficPolicies == nil { + b.trafficPolicies = make(map[string][]*TrafficPolicy) } - ensureNonNilMaps(&snap) - - b.mu.Lock("Restore") - defer b.mu.Unlock() - - b.zones = make(map[string]*zoneData, len(snap.Zones)) - - for id, zds := range snap.Zones { - if zds.Records == nil { - zds.Records = make(map[string]*ResourceRecordSet) - } - - b.zones[id] = &zoneData{ - zone: zds.Zone, - records: zds.Records, - dnssecEnabled: zds.DNSSECEnabled, - } + b.vpcAssociations = snap.VPCAssociations + if b.vpcAssociations == nil { + b.vpcAssociations = make(map[string][]vpcAssociation) } - b.restoreSimpleMaps(&snap) - - b.trafficPolicies = make(map[string][]*TrafficPolicy, len(snap.TrafficPolicies)) - - for id, versions := range snap.TrafficPolicies { - versionsCopy := make([]*TrafficPolicy, len(versions)) - for i, tp := range versions { - cp := *tp - versionsCopy[i] = &cp - } - - b.trafficPolicies[id] = versionsCopy + b.vpcAssocAuthorizations = snap.VPCAssocAuthorizations + if b.vpcAssocAuthorizations == nil { + b.vpcAssocAuthorizations = make(map[string][]VPCAssociationAuthorization) } - b.trafficPolicyInstances = make(map[string]*TrafficPolicyInstance, len(snap.TrafficPolicyInstances)) - - for id, inst := range snap.TrafficPolicyInstances { - cp := *inst - b.trafficPolicyInstances[id] = &cp + b.tags = snap.Tags + if b.tags == nil { + b.tags = make(map[string]*svcTags.Tags) } - b.restoreAssocMaps(&snap) - return nil } diff --git a/services/route53/persistence_test.go b/services/route53/persistence_test.go index 9f8b1fb99..e80246345 100644 --- a/services/route53/persistence_test.go +++ b/services/route53/persistence_test.go @@ -1,6 +1,7 @@ package route53_test import ( + "strings" "testing" "github.com/stretchr/testify/assert" @@ -67,6 +68,197 @@ func TestInMemoryBackend_SnapshotRestore(t *testing.T) { } } +// TestInMemoryBackend_SnapshotRestore_FullState exercises every resource +// family the Phase 3.3 pkgs/store conversion touched -- both the tables +// registered on the backend's registry (zones/records, health checks, key +// signing keys, CIDR collections, query logging configs, reusable delegation +// sets, traffic policy instances, changes) and the maps deliberately left raw +// (traffic policy versions, VPC associations, VPC association +// authorizations, tags) -- in a single Snapshot/Restore round trip, so a +// regression that silently drops any one collection during persistence would +// fail this test even if the collection's own narrower round-trip test above +// still passed. +func TestInMemoryBackend_SnapshotRestore_FullState(t *testing.T) { + t.Parallel() + + b := route53.NewInMemoryBackend() + + zone, err := b.CreateHostedZone("full-state.example.com.", "ref-full-zone", "full state zone", true) + require.NoError(t, err) + + changeID, err := b.ChangeResourceRecordSets(zone.ID, []route53.Change{ + { + Action: route53.ChangeActionCreate, + ResourceRecordSet: route53.ResourceRecordSet{ + Name: "full-state.example.com.", + Type: "A", + TTL: 300, + Records: []route53.ResourceRecord{{Value: "192.0.2.10"}}, + }, + }, + }) + require.NoError(t, err) + + hc, err := b.CreateHealthCheck("ref-full-hc", route53.HealthCheckConfig{ + Type: route53.HealthCheckTypeHTTP, + IPAddress: "192.0.2.20", + Port: 80, + }) + require.NoError(t, err) + + ksk, err := b.CreateKeySigningKey( + zone.ID, "ignored", "full-state-ksk", "arn:aws:kms:us-east-1:123456789012:key/abc", "", + ) + require.NoError(t, err) + + col, err := b.CreateCidrCollection("full-state-collection", "ignored") + require.NoError(t, err) + + _, err = b.ChangeCidrCollection(col.ID, []route53.CidrCollectionChange{ + {LocationName: "loc-1", Action: "PUT", CidrList: []string{"192.0.2.0/24"}}, + }) + require.NoError(t, err) + + qlc, err := b.CreateQueryLoggingConfig(zone.ID, "arn:aws:logs:us-east-1:123456789012:log-group:/route53/full-state") + require.NoError(t, err) + + ds, err := b.CreateReusableDelegationSet("ref-full-ds", "") + require.NoError(t, err) + + tp, err := b.CreateTrafficPolicy("full-state-policy", `{"AWSPolicyFormatVersion":"2015-10-01"}`, "comment") + require.NoError(t, err) + + tpi, err := b.CreateTrafficPolicyInstance(zone.ID, "full-state-tpi.example.com.", tp.ID, tp.Version, 300) + require.NoError(t, err) + + require.NoError(t, b.AssociateVPCWithHostedZone(zone.ID, "vpc-full-state", "us-east-1")) + + auth, err := b.CreateVPCAssociationAuthorization(zone.ID, "vpc-full-auth", "us-west-2") + require.NoError(t, err) + + require.NoError(t, b.ChangeTagsForResource("hostedzone", zone.ID, map[string]string{"env": "test"}, nil)) + require.NoError(t, b.ChangeTagsForResource("healthcheck", hc.ID, map[string]string{"team": "dns"}, nil)) + + bareChangeID := strings.TrimPrefix(changeID, "/change/") + + // Snapshot/Restore into a fresh backend. + snap := b.Snapshot(t.Context()) + require.NotNil(t, snap) + + fresh := route53.NewInMemoryBackend() + require.NoError(t, fresh.Restore(t.Context(), snap)) + + // Every table/map's population count must be identical pre- and post-restore. + assert.Equal(t, route53.ZoneCount(b), route53.ZoneCount(fresh), "zone count") + assert.Equal(t, route53.HealthCheckCount(b), route53.HealthCheckCount(fresh), "health check count") + assert.Equal(t, route53.KeySigningKeyCount(b), route53.KeySigningKeyCount(fresh), "KSK count") + assert.Equal(t, route53.CidrCollectionCount(b), route53.CidrCollectionCount(fresh), "CIDR collection count") + assert.Equal( + t, route53.QueryLoggingConfigCount(b), route53.QueryLoggingConfigCount(fresh), "query logging config count", + ) + assert.Equal(t, route53.DelegationSetCount(b), route53.DelegationSetCount(fresh), "delegation set count") + assert.Equal(t, route53.TrafficPolicyCount(b), route53.TrafficPolicyCount(fresh), "traffic policy count") + assert.Equal( + t, + route53.TrafficPolicyInstanceCount(b), route53.TrafficPolicyInstanceCount(fresh), + "traffic policy instance count", + ) + assert.Equal(t, route53.VPCAssociationCount(b), route53.VPCAssociationCount(fresh), "VPC association count") + + // zones + nested (raw, inline) record sets. + gotZone, err := fresh.GetHostedZone(zone.ID) + require.NoError(t, err) + assert.Equal(t, zone.Name, gotZone.Name) + assert.True(t, gotZone.PrivateZone) + + rrPage, err := fresh.ListResourceRecordSets(zone.ID, "", "", "", 0) + require.NoError(t, err) + + var foundARecord bool + + for _, rr := range rrPage.Records { + if rr.Type == "A" && rr.Name == "full-state.example.com." { + foundARecord = true + + require.Len(t, rr.Records, 1) + assert.Equal(t, "192.0.2.10", rr.Records[0].Value) + } + } + + assert.True(t, foundARecord, "A record should survive restore") + + // health checks. + gotHC, err := fresh.GetHealthCheck(hc.ID) + require.NoError(t, err) + assert.Equal(t, hc.Config.IPAddress, gotHC.Config.IPAddress) + + // key signing keys (verified via the byZone-indexed GetDNSSEC accessor). + _, ksks, err := fresh.GetDNSSEC(zone.ID) + require.NoError(t, err) + require.Len(t, ksks, 1) + assert.Equal(t, ksk.Name, ksks[0].Name) + assert.Equal(t, ksk.KeyManagementServiceArn, ksks[0].KeyManagementServiceArn) + + // CIDR collections (verified via ListCidrLocations/ListCidrBlocks). + locations, err := fresh.ListCidrLocations(col.ID) + require.NoError(t, err) + require.Contains(t, locations, "loc-1") + + blocks, err := fresh.ListCidrBlocks(col.ID, "loc-1") + require.NoError(t, err) + assert.Equal(t, []string{"192.0.2.0/24"}, blocks) + + // query logging configs (verified via the byZone-indexed accessor). + gotQLC, err := fresh.GetQueryLoggingConfig(qlc.ID) + require.NoError(t, err) + assert.Equal(t, qlc.CloudWatchLogsLogGroupArn, gotQLC.CloudWatchLogsLogGroupArn) + + // reusable delegation sets. + gotDS, err := fresh.GetReusableDelegationSet(ds.ID) + require.NoError(t, err) + assert.Equal(t, ds.NameServers, gotDS.NameServers) + + // traffic policies (left as a raw map[string][]*TrafficPolicy). + gotTP, err := fresh.GetTrafficPolicy(tp.ID, tp.Version) + require.NoError(t, err) + assert.Equal(t, tp.Document, gotTP.Document) + + // traffic policy instances (verified via the byZone-indexed accessor). + gotTPI, err := fresh.GetTrafficPolicyInstance(tpi.ID) + require.NoError(t, err) + assert.Equal(t, tpi.Name, gotTPI.Name) + + byZone, err := fresh.ListTrafficPolicyInstancesByHostedZone(zone.ID) + require.NoError(t, err) + require.Len(t, byZone, 1) + assert.Equal(t, tpi.ID, byZone[0].ID) + + // VPC associations and authorizations (raw maps). + assocs, err := fresh.ListVPCAssociations(zone.ID) + require.NoError(t, err) + require.Len(t, assocs, 1) + assert.Equal(t, "vpc-full-state", assocs[0].VPCID) + + auths, err := fresh.ListVPCAssociationAuthorizations(zone.ID) + require.NoError(t, err) + require.Len(t, auths, 1) + assert.Equal(t, auth.VPCID, auths[0].VPCID) + + // changes (keyed via the "/change/" TrimPrefix key function). + gotChange, err := fresh.GetChange(bareChangeID) + require.NoError(t, err) + assert.Equal(t, "INSYNC", gotChange.Status) + + // tags (raw map[string]*tags.Tags, keyed externally by resource ID). + zoneTags, err := fresh.ListTagsForResource("hostedzone", zone.ID) + require.NoError(t, err) + assert.Equal(t, "test", zoneTags["env"]) + + hcTags, err := fresh.ListTagsForResource("healthcheck", hc.ID) + require.NoError(t, err) + assert.Equal(t, "dns", hcTags["team"]) +} + func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { t.Parallel() diff --git a/services/route53/routing.go b/services/route53/routing.go index f9a2eb392..1a65dc37e 100644 --- a/services/route53/routing.go +++ b/services/route53/routing.go @@ -217,7 +217,7 @@ func (b *InMemoryBackend) recordHealthy(rrs *ResourceRecordSet) bool { return true } - hc, ok := b.healthChecks[rrs.HealthCheckID] + hc, ok := b.healthChecks.Get(rrs.HealthCheckID) if !ok { return true } diff --git a/services/route53/store_setup.go b/services/route53/store_setup.go new file mode 100644 index 000000000..abd0dd3a5 --- /dev/null +++ b/services/route53/store_setup.go @@ -0,0 +1,112 @@ +package route53 + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend whose key is a pure +// function of the stored value's own fields is registered exactly once here +// as a *store.Table[T] on b.registry. See pkgs/store's package doc and the +// services/sqs pilot (commit 0f09d77c) for the pattern this follows. +// +// The following resource fields are deliberately NOT registered here and +// remain plain maps: +// - trafficPolicies (map[string][]*TrafficPolicy): the value is a SLICE of +// versions, not a single *TrafficPolicy, so store.Table (which keys one +// *V per entry) does not fit without inventing a synthetic wrapper type. +// - vpcAssociations (map[string][]vpcAssociation) and +// vpcAssocAuthorizations (map[string][]VPCAssociationAuthorization): both +// are slice-valued AND the element type is a plain value (not a pointer), +// so neither the "one *V per key" nor the "key derived from *V" shape +// required by store.Table applies. +// - tags (map[string]*svcTags.Tags): svcTags.Tags carries no identity field +// of its own (it is a thin wrapper over a safemap.Map[string,string]); it +// is keyed externally by resourceID, exactly like EC2's +// instanceIMDSOptions/verifiedAccessGroupPolicies fields. +// +// zoneData.records (the nested per-zone record-set map) is left as a raw map +// field on zoneData, not promoted to its own top-level store.Table: it is +// scoped entirely to its owning zone and never accessed independent of it, +// the same "nested collection stays inline" call the s3 conversion made for +// bucket objects. +import ( + "strings" + + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +// zoneKeyFn is the [store.Table] key function for b.zones: HostedZone.ID is +// set once at CreateHostedZone and never mutated afterward (only Comment is +// updated in place), so it is a stable, pure identity for zoneData. +func zoneKeyFn(v *zoneData) string { return v.zone.ID } + +// healthCheckKeyFn is the [store.Table] key function for b.healthChecks. +func healthCheckKeyFn(v *HealthCheck) string { return v.ID } + +// keySigningKeyKeyFn is the [store.Table] key function for b.keySigningKeys, +// mirroring the composite key kskKey already built from HostedZoneID+Name. +// Both fields are set at creation and never mutated (only Status changes). +func keySigningKeyKeyFn(v *KeySigningKey) string { return kskKey(v.HostedZoneID, v.Name) } + +// keySigningKeyZoneKeyFn is the secondary-index key function grouping key +// signing keys by their owning hosted zone, used for the DeleteHostedZone +// cascade and the DNSSEC status lookups. +func keySigningKeyZoneKeyFn(v *KeySigningKey) string { return v.HostedZoneID } + +// cidrCollectionKeyFn is the [store.Table] key function for b.cidrCollections. +func cidrCollectionKeyFn(v *CidrCollection) string { return v.ID } + +// queryLoggingConfigKeyFn is the [store.Table] key function for +// b.queryLoggingConfigs. +func queryLoggingConfigKeyFn(v *QueryLoggingConfig) string { return v.ID } + +// queryLoggingConfigZoneKeyFn is the secondary-index key function grouping +// query logging configs by their owning hosted zone, used for the +// DeleteHostedZone cascade, the one-config-per-zone existence check in +// CreateQueryLoggingConfig, and the zone filter in ListQueryLoggingConfigs. +func queryLoggingConfigZoneKeyFn(v *QueryLoggingConfig) string { return v.HostedZoneID } + +// delegationSetKeyFn is the [store.Table] key function for +// b.reusableDelegationSets. +func delegationSetKeyFn(v *ReusableDelegationSet) string { return v.ID } + +// trafficPolicyInstanceKeyFn is the [store.Table] key function for +// b.trafficPolicyInstances. +func trafficPolicyInstanceKeyFn(v *TrafficPolicyInstance) string { return v.ID } + +// trafficPolicyInstanceZoneKeyFn is the secondary-index key function grouping +// traffic policy instances by their owning hosted zone. HostedZoneID is set +// at creation and never mutated (unlike TrafficPolicyID, which +// UpdateTrafficPolicyInstance can change in place), so this index never goes +// stale. +func trafficPolicyInstanceZoneKeyFn(v *TrafficPolicyInstance) string { return v.HostedZoneID } + +// changeKeyFn is the [store.Table] key function for b.changes. The stored map +// key has always been the bare change ID (e.g. "C1234..."), while +// ChangeInfo.ID carries the full "/change/C1234..." wire form; TrimPrefix +// recovers the former as a pure function of the latter. +func changeKeyFn(v *ChangeInfo) string { return strings.TrimPrefix(v.ID, "/change/") } + +// registerTables registers every converted resource map on b.registry exactly +// once. It must be called during construction only (immediately after +// b.registry is created), never on every Reset() -- store.Register panics on +// a duplicate name, so runtime resets go through registry.ResetAll() instead +// (see InMemoryBackend.Reset in backend.go). +func (b *InMemoryBackend) registerTables() { + b.zones = store.Register(b.registry, "zones", store.New(zoneKeyFn)) + b.healthChecks = store.Register(b.registry, "healthChecks", store.New(healthCheckKeyFn)) + + b.keySigningKeys = store.Register(b.registry, "keySigningKeys", store.New(keySigningKeyKeyFn)) + b.keySigningKeysByZone = b.keySigningKeys.AddIndex("byZone", keySigningKeyZoneKeyFn) + + b.cidrCollections = store.Register(b.registry, "cidrCollections", store.New(cidrCollectionKeyFn)) + + b.queryLoggingConfigs = store.Register(b.registry, "queryLoggingConfigs", store.New(queryLoggingConfigKeyFn)) + b.queryLoggingConfigsByZone = b.queryLoggingConfigs.AddIndex("byZone", queryLoggingConfigZoneKeyFn) + + b.reusableDelegationSets = store.Register(b.registry, "reusableDelegationSets", store.New(delegationSetKeyFn)) + + b.trafficPolicyInstances = store.Register( + b.registry, "trafficPolicyInstances", store.New(trafficPolicyInstanceKeyFn), + ) + b.trafficPolicyInstancesByZone = b.trafficPolicyInstances.AddIndex("byZone", trafficPolicyInstanceZoneKeyFn) + + b.changes = store.Register(b.registry, "changes", store.New(changeKeyFn)) +} diff --git a/services/s3/PARITY.md b/services/s3/PARITY.md new file mode 100644 index 000000000..9c6783076 --- /dev/null +++ b/services/s3/PARITY.md @@ -0,0 +1,29 @@ +--- +service: s3 +sdk_module: aws-sdk-go-v2/service/s3 # version: see go.mod (backfilled, confirm on next audit) +last_audit_commit: 708d1961 +last_audit_date: 2026-07-04 +overall: B # already mature from prior sweeps; 11 real fixes + op-by-op proof +protocol: REST-XML +families: + multipart: {status: ok, note: part-order InvalidPartOrder, non-last EntityTooSmall, ETag=MD5(concat part-MD5s)-N, SSE sealing} + conditionals: {status: ok, note: If-Match/None-Match/(Un)Modified-Since 412/304 precedence; If-Range; Range 206/416 InvalidRange w/ ActualObjectSize} + versioning: {status: ok, note: delete markers, null-version, suspended vs never-configured, object-lock/legal-hold} + pagination: {status: ok, note: v1 NextMarker only w/ delimiter; v2 KeyCount, ContinuationToken/StartAfter, encoding-type on keys not tokens} + errors: {status: ok, note: full errorTable, no missing-lookup->500; HEAD bodiless} + copy: {status: ok, note: copy-self, directives, version-id, UploadPartCopy range} +ops: + GetObject/HeadObject: {wire: ok, errors: ok, state: ok, persist: ok, note: FIXED response-* override query params (content-type/disposition/expires/cache-control)} + PutBucketAcl: {wire: ok, errors: ok, state: ok, persist: ok, note: FIXED reject object-only canned ACLs; read AccessControlPolicy body} + PutBucketReplication: {wire: ok, errors: ok, state: ok, persist: ok, note: FIXED require versioning=Enabled} + GetObjectAttributes: {wire: ok, errors: ok, state: ok, persist: ok, note: FIXED ObjectSize 0-byte, Last-Modified} +gaps: + - object_lambda dual-lock + no delete-on-bucket-delete (bd: s3 objectLambda follow-up) + - abandoned multipart uploads have no per-upload TTL (only Abort/Complete/Purge) +leaks: {status: clean, note: janitor ctx-parented w/ <-ctx.Done() stop; replication goroutines WaitGroup-drained; Shutdown() cancels} +--- + +## Notes +- **CRITICAL prior bug (fixed here):** SSE-S3/SSE-KMS EncryptionDEK/Nonce were `json:"-"` while ciphertext persisted → every encrypted object undecryptable after snapshot/restore (silent data loss). Multipart SSE same. Now persisted base64; SSE-C key stays request-scoped (`SSECKeyB64` json:"-"). +- Persistence: backendSnapshot covers buckets/tags/uploads/region; per-bucket configs on StoredBucket; bucketIndex rebuilt on restore; legacy-uploads migration present. +- Trap: `x-amz-storage-class` header is OMITTED for STANDARD objects (AWS behavior) — don't re-add it. diff --git a/services/s3/accuracy.go b/services/s3/accuracy.go index 2f8bb9541..579f43fa6 100644 --- a/services/s3/accuracy.go +++ b/services/s3/accuracy.go @@ -62,10 +62,10 @@ type sseInfo struct { // SSECKeyMD5 is the base64-encoded MD5 of the customer-supplied key. SSECKeyMD5 string // SSECKeyB64 is the base64-encoded raw customer key. Kept on the - // request-scoped sseInfo only — not persisted — so the backend can - // encrypt the body on PUT and the GET handler can decrypt when the + // request-scoped sseInfo only — not persisted (json:"-") — so the backend + // can encrypt the body on PUT and the GET handler can decrypt when the // caller re-supplies it. - SSECKeyB64 string + SSECKeyB64 string `json:"-"` } // extractSSEInfo reads SSE-* request headers and validates SSE-C when present. diff --git a/services/s3/backend_memory.go b/services/s3/backend_memory.go index d29ea2a91..77c11180b 100644 --- a/services/s3/backend_memory.go +++ b/services/s3/backend_memory.go @@ -28,6 +28,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/ptrconv" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -112,12 +113,34 @@ func getRegionFromS3Context(ctx context.Context, defaultRegion string) string { } type InMemoryBackend struct { - buckets map[string]map[string]*StoredBucket - bucketIndex map[string]string // name → region for O(1) cross-region lookup - tags map[string][]types.Tag - uploads map[string]map[string]*StoredMultipartUpload // bucket → uploadID → upload - mu *lockmetrics.RWMutex - compressor Compressor + // registry lets Reset/Snapshot/Restore collapse the buckets/uploads + // lifecycle to one call each (registry.ResetAll/SnapshotAll/RestoreAll) + // instead of hand-rolled per-map wiring. See pkgs/store's package doc and + // the services/sqs pilot (commit 0f09d77c) for the pattern this follows. + registry *store.Registry + // buckets is keyed by bucket name (globally unique — see StoredBucket.Region's + // doc comment). This replaces the old region->name->*StoredBucket nesting plus + // the separate bucketIndex name->region map: Table.Get(name) alone now answers + // both "does it exist" and "give me the bucket", and StoredBucket.Region + // carries what bucketIndex used to. + buckets *store.Table[StoredBucket] + // uploads is keyed by UploadID (a random 32-hex-char string — see + // newObjectVersionID — so it is unique across all buckets). uploadsByBucket + // is a secondary index replacing the old bucket->uploadID->*StoredMultipartUpload + // nesting for the "all uploads in bucket X" access pattern (ListMultipartUploads, + // janitor cleanup, DeleteBucket cleanup). A caller-supplied uploadID is only + // valid for the bucket it was issued against — see getUpload, which enforces + // that the same way the old b.uploads[bucketName][uploadID] nesting did. + uploads *store.Table[StoredMultipartUpload] + uploadsByBucket *store.Index[StoredMultipartUpload] + // tags is intentionally left as a plain map (not a store.Table): its key is + // a composite "bucket/key/versionID" string that is not a pure function of + // the stored value (a bare []types.Tag has no identity field of its own) — + // the same reason services/ec2's store_setup.go leaves e.g. + // vpcPeeringOptions/instanceIMDSOptions unconverted. + tags map[string][]types.Tag + mu *lockmetrics.RWMutex + compressor Compressor // serviceCtx is the long-lived context for background work (replication). // Initialised in NewInMemoryBackend so it is always non-nil; overridden by // SetServiceContext when the handler wires in the real service context. @@ -178,17 +201,36 @@ func (b *InMemoryBackend) replicationContext(reqCtx context.Context) context.Con return logger.Save(base, logger.Load(reqCtx)) } +// bucketTableKey is the [store.Table] key function for b.buckets: bucket +// names are globally unique (CreateBucket enforces this), so the name alone +// is the bucket's identity regardless of region. +func bucketTableKey(bucket *StoredBucket) string { return bucket.Name } + +// uploadTableKey is the [store.Table] key function for b.uploads: a +// multipart upload's own UploadID is already its unique identity. +func uploadTableKey(u *StoredMultipartUpload) string { return u.UploadID } + +// uploadBucketIndexKey is the [store.Index] key function grouping uploads by +// their owning bucket, replacing the old bucket->uploadID->*StoredMultipartUpload +// map nesting for bucket-scoped scans (ListMultipartUploads, janitor cleanup). +func uploadBucketIndexKey(u *StoredMultipartUpload) string { return u.Bucket } + func NewInMemoryBackend(compressor Compressor) *InMemoryBackend { ctx, cancel := context.WithCancel(context.Background()) + registry := store.NewRegistry() + uploads := store.Register(registry, "uploads", store.New(uploadTableKey)) + return &InMemoryBackend{ - buckets: make(map[string]map[string]*StoredBucket), - bucketIndex: make(map[string]string), - compressor: compressor, - defaultRegion: defaultRegionName, - mu: lockmetrics.New("s3"), - serviceCtx: ctx, - serviceCancel: cancel, + registry: registry, + buckets: store.Register(registry, "buckets", store.New(bucketTableKey)), + uploads: uploads, + uploadsByBucket: uploads.AddIndex("bucket", uploadBucketIndexKey), + compressor: compressor, + defaultRegion: defaultRegionName, + mu: lockmetrics.New("s3"), + serviceCtx: ctx, + serviceCancel: cancel, } } @@ -220,15 +262,10 @@ func (b *InMemoryBackend) WithCompressionMinBytes(n int) *InMemoryBackend { // getBucket returns the bucket for a given name, returning ErrNoSuchBucket when the // bucket does not exist or is pending async deletion. The caller must hold at least b.mu.RLock. -// bucketIndex provides O(1) name→region resolution, so no region scan is needed. +// b.buckets is keyed by name, so a single Table.Get resolves it in O(1). func (b *InMemoryBackend) getBucket(name string) (*StoredBucket, error) { - region, ok := b.bucketIndex[name] - if !ok { - return nil, ErrNoSuchBucket - } - - bucket := b.buckets[region][name] - if bucket == nil || bucket.DeletePending { + bucket, ok := b.buckets.Get(name) + if !ok || bucket.DeletePending { return nil, ErrNoSuchBucket } @@ -241,7 +278,12 @@ func (b *InMemoryBackend) BucketRegion(name string) string { b.mu.RLock("BucketRegion") defer b.mu.RUnlock() - return b.bucketIndex[name] + bucket, ok := b.buckets.Get(name) + if !ok { + return "" + } + + return bucket.Region } // SetDefaultRegion sets the default region for this backend. @@ -270,22 +312,18 @@ func (b *InMemoryBackend) CreateBucket( bucketName := *input.Bucket - // Use bucketIndex for O(1) global-uniqueness check. Since this is a + // Use b.buckets for O(1) global-uniqueness check. Since this is a // single-tenant mock, a pre-existing bucket is always owned by the // caller → return BucketAlreadyOwnedByYou (not BucketAlreadyExists). - // Pending-delete buckets remain in the index and still block re-creation, + // Pending-delete buckets remain in the table and still block re-creation, // matching real S3 behaviour (you must wait for deletion to complete). - if _, exists := b.bucketIndex[bucketName]; exists { + if b.buckets.Has(bucketName) { return nil, ErrBucketAlreadyOwnedByYou } - // Initialize region map if it doesn't exist - if _, exists := b.buckets[region]; !exists { - b.buckets[region] = make(map[string]*StoredBucket) - } - - b.buckets[region][bucketName] = &StoredBucket{ + b.buckets.Put(&StoredBucket{ Name: bucketName, + Region: region, CreationDate: time.Now().UTC(), Objects: make(map[string]*StoredObject), // Versioning is intentionally not set: new buckets have never had versioning @@ -299,8 +337,7 @@ func (b *InMemoryBackend) CreateBucket( // Detect this at creation time so ListBuckets and ListDirectoryBuckets can // correctly partition general-purpose vs. directory buckets. IsDirectoryBucket: strings.HasSuffix(bucketName, "--x-s3"), - } - b.bucketIndex[bucketName] = region + }) return &s3.CreateBucketOutput{ Location: aws.String("/" + bucketName), @@ -316,18 +353,13 @@ func (b *InMemoryBackend) DeleteBucket( b.mu.Lock("DeleteBucket") defer b.mu.Unlock() - // Use bucketIndex for O(1) lookup. Pending buckets remain in the index - // so that idempotent deletes can be detected without a region scan. - region, ok := b.bucketIndex[bucketName] + // Use b.buckets for O(1) lookup. Pending buckets remain in the table so + // that idempotent deletes can be detected without a region scan. + bucket, ok := b.buckets.Get(bucketName) if !ok { return nil, ErrNoSuchBucket } - bucket := b.buckets[region][bucketName] - if bucket == nil { - return nil, ErrNoSuchBucket - } - // If already pending deletion, return success (idempotent). if bucket.DeletePending { return &s3.DeleteBucketOutput{}, nil @@ -357,19 +389,18 @@ func (b *InMemoryBackend) ListBuckets( _ context.Context, _ *s3.ListBucketsInput, ) (*s3.ListBucketsOutput, error) { - // Snapshot bucket data under lock across all regions, release immediately + // Snapshot bucket data under lock, release immediately. b.mu.RLock("ListBuckets") - buckets := make([]types.Bucket, 0, len(b.buckets)) - for _, regionBuckets := range b.buckets { - for _, bucket := range regionBuckets { - if bucket.DeletePending || bucket.IsDirectoryBucket { - continue - } - buckets = append(buckets, types.Bucket{ - Name: aws.String(bucket.Name), - CreationDate: aws.Time(bucket.CreationDate), - }) + all := b.buckets.All() + buckets := make([]types.Bucket, 0, len(all)) + for _, bucket := range all { + if bucket.DeletePending || bucket.IsDirectoryBucket { + continue } + buckets = append(buckets, types.Bucket{ + Name: aws.String(bucket.Name), + CreationDate: aws.Time(bucket.CreationDate), + }) } b.mu.RUnlock() @@ -395,17 +426,15 @@ func (b *InMemoryBackend) ListDirectoryBuckets(_ context.Context) ([]types.Bucke b.mu.RLock("ListDirectoryBuckets") buckets := make([]types.Bucket, 0) - for _, regionBuckets := range b.buckets { - for _, bucket := range regionBuckets { - if bucket.DeletePending || !bucket.IsDirectoryBucket { - continue - } - - buckets = append(buckets, types.Bucket{ - Name: aws.String(bucket.Name), - CreationDate: aws.Time(bucket.CreationDate), - }) + for _, bucket := range b.buckets.All() { + if bucket.DeletePending || !bucket.IsDirectoryBucket { + continue } + + buckets = append(buckets, types.Bucket{ + Name: aws.String(bucket.Name), + CreationDate: aws.Time(bucket.CreationDate), + }) } b.mu.RUnlock() @@ -421,16 +450,20 @@ func (b *InMemoryBackend) Regions() []string { b.mu.RLock("Regions") defer b.mu.RUnlock() + seen := make(map[string]struct{}) var regions []string - for region, regionBuckets := range b.buckets { - for _, bucket := range regionBuckets { - if !bucket.DeletePending { - regions = append(regions, region) + for _, bucket := range b.buckets.All() { + if bucket.DeletePending { + continue + } - break - } + if _, ok := seen[bucket.Region]; ok { + continue } + + seen[bucket.Region] = struct{}{} + regions = append(regions, bucket.Region) } sort.Strings(regions) @@ -446,21 +479,19 @@ func (b *InMemoryBackend) BucketsByRegion(region string) []types.Bucket { var buckets []types.Bucket - for r, regionBuckets := range b.buckets { - if region != "" && r != region { + for _, bucket := range b.buckets.All() { + if region != "" && bucket.Region != region { continue } - for _, bucket := range regionBuckets { - if bucket.DeletePending { - continue - } - - buckets = append(buckets, types.Bucket{ - Name: aws.String(bucket.Name), - CreationDate: aws.Time(bucket.CreationDate), - }) + if bucket.DeletePending { + continue } + + buckets = append(buckets, types.Bucket{ + Name: aws.String(bucket.Name), + CreationDate: aws.Time(bucket.CreationDate), + }) } sort.Slice(buckets, func(i, j int) bool { @@ -2003,6 +2034,20 @@ func (b *InMemoryBackend) DeleteObjectTagging( // Multipart +// getUpload returns the multipart upload for uploadID, but only if it belongs +// to bucketName — mirroring the old b.uploads[bucketName][uploadID] nested-map +// lookup, which returned nil both when the uploadID was absent and when it +// existed but under a different bucket. The caller must hold at least +// b.mu.RLock. +func (b *InMemoryBackend) getUpload(bucketName, uploadID string) *StoredMultipartUpload { + upload, ok := b.uploads.Get(uploadID) + if !ok || upload.Bucket != bucketName { + return nil + } + + return upload +} + func (b *InMemoryBackend) CreateMultipartUpload( ctx context.Context, input *s3.CreateMultipartUploadInput, @@ -2040,15 +2085,7 @@ func (b *InMemoryBackend) CreateMultipartUpload( } b.mu.Lock("CreateMultipartUpload") - if b.uploads == nil { - b.uploads = make(map[string]map[string]*StoredMultipartUpload) - } - - if b.uploads[bucketName] == nil { - b.uploads[bucketName] = make(map[string]*StoredMultipartUpload) - } - - b.uploads[bucketName][uploadID] = &StoredMultipartUpload{ + b.uploads.Put(&StoredMultipartUpload{ UploadID: uploadID, Bucket: bucketName, Key: key, @@ -2057,7 +2094,7 @@ func (b *InMemoryBackend) CreateMultipartUpload( Tagging: tagging, SSE: sse, mu: lockmetrics.New("s3.upload"), - } + }) b.mu.Unlock() return &s3.CreateMultipartUploadOutput{ @@ -2159,7 +2196,7 @@ func (b *InMemoryBackend) CompleteMultipartUpload( // the upload yet, since assembly may fail (e.g. ErrInvalidPart) and the // caller should still be able to retry or abort. b.mu.RLock(opCompleteMultipartUpload) - upload := b.uploads[bucketName][uploadID] // nil map read returns nil safely + upload := b.getUpload(bucketName, uploadID) b.mu.RUnlock() if upload == nil { @@ -2214,7 +2251,7 @@ func (b *InMemoryBackend) CompleteMultipartUpload( func (b *InMemoryBackend) claimMultipartUpload(bucketName, uploadID string) error { b.mu.Lock("CompleteMultipartUpload.claim") - upload := b.uploads[bucketName][uploadID] // nil map read returns nil safely + upload := b.getUpload(bucketName, uploadID) if upload == nil { b.mu.Unlock() @@ -2227,7 +2264,7 @@ func (b *InMemoryBackend) claimMultipartUpload(bucketName, uploadID string) erro upload.closed = true upload.mu.Unlock() - delete(b.uploads[bucketName], uploadID) + b.uploads.Delete(uploadID) b.mu.Unlock() return nil @@ -2454,7 +2491,7 @@ func (b *InMemoryBackend) AbortMultipartUpload( b.mu.Lock("AbortMultipartUpload") - upload := b.uploads[bucketName][uploadID] + upload := b.getUpload(bucketName, uploadID) if upload == nil { b.mu.Unlock() @@ -2467,7 +2504,7 @@ func (b *InMemoryBackend) AbortMultipartUpload( upload.closed = true upload.mu.Unlock() - delete(b.uploads[bucketName], uploadID) + b.uploads.Delete(uploadID) b.mu.Unlock() return &s3.AbortMultipartUploadOutput{}, nil @@ -2522,7 +2559,7 @@ func (b *InMemoryBackend) ListMultipartUploads( func (b *InMemoryBackend) collectAndSortUploads(bucketName, prefix string) []types.MultipartUpload { var uploads []types.MultipartUpload - for _, u := range b.uploads[bucketName] { + for _, u := range b.uploadsByBucket.Get(bucketName) { if prefix != "" && !strings.HasPrefix(u.Key, prefix) { continue } @@ -2623,7 +2660,7 @@ func (b *InMemoryBackend) ListParts( bucketName := aws.ToString(input.Bucket) b.mu.RLock("ListParts") - upload := b.uploads[bucketName][uploadID] // reading nil map returns nil safely + upload := b.getUpload(bucketName, uploadID) b.mu.RUnlock() if upload == nil { @@ -4166,21 +4203,32 @@ func (b *InMemoryBackend) CreateSession(_ context.Context, bucketName string) (s return sessionXML, nil } +// purgeUploadsForBucketLocked removes every multipart upload belonging to +// bucketName from b.uploads. Caller must hold b.mu. The uploadsByBucket group +// slice is owned by the index (see [store.Index.Get]'s doc), so upload IDs +// are copied out before any Delete call mutates that group. +func (b *InMemoryBackend) purgeUploadsForBucketLocked(bucketName string) { + grouped := b.uploadsByBucket.Get(bucketName) + uploadIDs := make([]string, len(grouped)) + for i, u := range grouped { + uploadIDs[i] = u.UploadID + } + + for _, id := range uploadIDs { + b.uploads.Delete(id) + } +} + // purgeBucketLocked removes a single bucket and its associated data from the // backend. Caller must hold b.mu. -func (b *InMemoryBackend) purgeBucketLocked( - regionBuckets map[string]*StoredBucket, - bucketName string, - bucket *StoredBucket, -) { +func (b *InMemoryBackend) purgeBucketLocked(bucketName string, bucket *StoredBucket) { // Close per-object mutexes to avoid Prometheus metric leaks. for _, obj := range bucket.Objects { obj.mu.Close() } bucket.mu.Close() - delete(regionBuckets, bucketName) - delete(b.bucketIndex, bucketName) + b.buckets.Delete(bucketName) for tagKey := range b.tags { if strings.HasPrefix(tagKey, bucketName+"/") { @@ -4188,7 +4236,7 @@ func (b *InMemoryBackend) purgeBucketLocked( } } - delete(b.uploads, bucketName) + b.purgeUploadsForBucketLocked(bucketName) } // Purge removes all buckets created before the given cutoff time. @@ -4200,15 +4248,13 @@ func (b *InMemoryBackend) Purge(ctx context.Context, cutoff time.Time) { b.mu.Lock("Purge") defer b.mu.Unlock() - for _, regionBuckets := range b.buckets { - for bucketName, bucket := range regionBuckets { - if ctx.Err() != nil { - return - } + for _, bucket := range b.buckets.All() { + if ctx.Err() != nil { + return + } - if bucket.CreationDate.Before(cutoff) { - b.purgeBucketLocked(regionBuckets, bucketName, bucket) - } + if bucket.CreationDate.Before(cutoff) { + b.purgeBucketLocked(bucket.Name, bucket) } } } @@ -4220,19 +4266,15 @@ func (b *InMemoryBackend) Reset() { defer b.mu.Unlock() // Close all bucket and object mutexes to prevent Prometheus metric leaks. - for _, regionBuckets := range b.buckets { - for _, bucket := range regionBuckets { - for _, obj := range bucket.Objects { - obj.mu.Close() - } - bucket.mu.Close() + for _, bucket := range b.buckets.All() { + for _, obj := range bucket.Objects { + obj.mu.Close() } + bucket.mu.Close() } - b.buckets = make(map[string]map[string]*StoredBucket) - b.bucketIndex = make(map[string]string) + b.registry.ResetAll() b.tags = make(map[string][]types.Tag) - b.uploads = make(map[string]map[string]*StoredMultipartUpload) } func (b *InMemoryBackend) GetBucketMetadata( @@ -4240,16 +4282,16 @@ func (b *InMemoryBackend) GetBucketMetadata( bucketName string, ) (string, string, []types.Tag, error) { b.mu.RLock("GetBucketMetadata") - region, ok := b.bucketIndex[bucketName] + bucket, ok := b.buckets.Get(bucketName) if !ok { b.mu.RUnlock() return "", "", nil, ErrNoSuchBucket } - bucket := b.buckets[region][bucketName] b.mu.RUnlock() bucket.mu.RLock("GetBucketMetadata") + region := bucket.Region lcXML := bucket.LifecycleConfig tags := slices.Clone(bucket.Tags) bucket.mu.RUnlock() @@ -4322,14 +4364,10 @@ func (b *InMemoryBackend) storePart( part *StoredPart, ) error { b.mu.RLock("storePart") - bucketUploads, ok := b.uploads[bucketName] - var upload *StoredMultipartUpload - if ok { - upload, ok = bucketUploads[uploadID] - } + upload := b.getUpload(bucketName, uploadID) b.mu.RUnlock() - if !ok { + if upload == nil { return ErrNoSuchUpload } diff --git a/services/s3/bucket_ops.go b/services/s3/bucket_ops.go index 26c5a6457..39b14fdf8 100644 --- a/services/s3/bucket_ops.go +++ b/services/s3/bucket_ops.go @@ -958,15 +958,17 @@ func mapListVersionsOutput( } // validCannedACLs is the complete set of canned ACL strings that AWS S3 accepts -// for PutBucketAcl. +// for PutBucketAcl. This mirrors types.BucketCannedACL (private, public-read, +// public-read-write, authenticated-read) plus log-delivery-write, which is +// bucket-only. The bucket-owner-read / bucket-owner-full-control canned ACLs +// belong to types.ObjectCannedACL only — real S3 rejects them on PutBucketAcl +// with 400 InvalidArgument, so they are deliberately excluded here. var validCannedACLs = map[string]struct{}{ //nolint:gochecknoglobals // package-level lookup table - aclPrivate: {}, - aclPublicRead: {}, - aclPublicReadWrite: {}, - aclAuthenticatedRead: {}, - "bucket-owner-read": {}, - "bucket-owner-full-control": {}, - aclLogDeliveryWrite: {}, + aclPrivate: {}, + aclPublicRead: {}, + aclPublicReadWrite: {}, + aclAuthenticatedRead: {}, + aclLogDeliveryWrite: {}, } func (h *S3Handler) putBucketACL( @@ -977,18 +979,35 @@ func (h *S3Handler) putBucketACL( ) { h.setOperation(ctx, "PutBucketAcl") - acl := r.Header.Get("X-Amz-Acl") - if acl == "" { - acl = "private" - } + // A PutBucketAcl request carries the grant set either as an x-amz-acl canned + // header or as an AccessControlPolicy XML body (mutually exclusive in AWS). + // Mirror the object-ACL path: persist whichever was supplied, and feed both + // the canned value and the body into the Public-Access-Block check so a + // body-only request can't slip a public grant past BlockPublicAcls. + canned := r.Header.Get("X-Amz-Acl") - if _, ok := validCannedACLs[acl]; !ok { - WriteError(ctx, w, r, ErrInvalidArgument) + body, _ := httputils.ReadBody(r) - return + // Only validate the canned value when no explicit body was supplied; an + // AccessControlPolicy body is authoritative and needs no canned name. + if len(body) == 0 { + if canned == "" { + canned = "private" + } + + if _, ok := validCannedACLs[canned]; !ok { + WriteError(ctx, w, r, ErrInvalidArgument) + + return + } + } + + acl := canned + if len(body) > 0 { + acl = string(body) } - if err := h.enforceACLPolicy(ctx, bucketName, acl, ""); err != nil { + if err := h.enforceACLPolicy(ctx, bucketName, canned, string(body)); err != nil { WriteError(ctx, w, r, err) return @@ -1018,6 +1037,17 @@ func (h *S3Handler) getBucketACL( return } + // When PutBucketAcl stored a full AccessControlPolicy XML body, return it + // verbatim (mirrors getObjectACL); otherwise synthesise XML from the canned + // ACL name. + if strings.HasPrefix(strings.TrimSpace(canned), "<") { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(canned)) + + return + } + resp := AccessControlPolicy{ Xmlns: xmlNamespaceS3, Owner: Owner{ @@ -1520,6 +1550,26 @@ func (h *S3Handler) putBucketReplication( return } + // Real S3 requires the source bucket to have versioning enabled before a + // replication configuration can be attached, returning InvalidRequest 400 + // otherwise. GetBucketVersioning surfaces NoSuchBucket for a missing bucket. + verOut, verErr := h.Backend.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{ + Bucket: aws.String(bucket), + }) + if verErr != nil { + WriteError(ctx, w, r, verErr) + + return + } + if verOut.Status != types.BucketVersioningStatusEnabled { + httputils.WriteS3ErrorResponse(ctx, w, r, ErrorResponse{ + Code: errInvalidRequest, + Message: "Versioning must be 'Enabled' on the bucket to apply a replication configuration.", + }, http.StatusBadRequest) + + return + } + if err = h.Backend.PutBucketReplication(ctx, bucket, string(body)); err != nil { WriteError(ctx, w, r, err) diff --git a/services/s3/export_test.go b/services/s3/export_test.go index 7206a95a3..b69f7fbf9 100644 --- a/services/s3/export_test.go +++ b/services/s3/export_test.go @@ -24,7 +24,7 @@ func (b *InMemoryBackend) UploadsForBucket(bucket string) int { b.mu.RLock("UploadsForBucket") defer b.mu.RUnlock() - return len(b.uploads[bucket]) + return len(b.uploadsByBucket.Get(bucket)) } // TagsForBucket returns the number of tag entries for the given bucket. @@ -71,16 +71,11 @@ func PeekStoredBytes(b *InMemoryBackend, bucketName, key string) []byte { b.mu.RLock("PeekStoredBytes") defer b.mu.RUnlock() - region, ok := b.bucketIndex[bucketName] + bucket, ok := b.buckets.Get(bucketName) if !ok { return nil } - bucket := b.buckets[region][bucketName] - if bucket == nil { - return nil - } - bucket.mu.RLock("PeekStoredBytes") defer bucket.mu.RUnlock() @@ -114,16 +109,10 @@ func (h *S3Handler) PendingObjectLambdaRequestsCount() int { // Used in lifecycle transition tests to simulate aged objects. func BackdateObjectForTest(b *InMemoryBackend, bucketName, key string, t time.Time) { b.mu.RLock("BackdateObjectForTest") - region, ok := b.bucketIndex[bucketName] - if !ok { - b.mu.RUnlock() - - return - } - bucket := b.buckets[region][bucketName] + bucket, ok := b.buckets.Get(bucketName) b.mu.RUnlock() - if bucket == nil { + if !ok { return } @@ -150,7 +139,7 @@ func BackdateUploadForTest(b *InMemoryBackend, bucket string, uploadID *string, } b.mu.RLock("BackdateUploadForTest") - upload := b.uploads[bucket][*uploadID] + upload := b.getUpload(bucket, *uploadID) b.mu.RUnlock() if upload == nil { @@ -159,7 +148,7 @@ func BackdateUploadForTest(b *InMemoryBackend, bucket string, uploadID *string, // Mutate directly — Initiated has no per-upload lock. b.mu.Lock("BackdateUploadForTest.write") - if u := b.uploads[bucket][*uploadID]; u != nil { + if u := b.getUpload(bucket, *uploadID); u != nil { u.Initiated = t } b.mu.Unlock() @@ -172,16 +161,10 @@ func StorageClassTransitionsForObject( bucketName, key string, ) []StorageClassTransition { b.mu.RLock("StorageClassTransitionsForObject") - region, ok := b.bucketIndex[bucketName] - if !ok { - b.mu.RUnlock() - - return nil - } - bucket := b.buckets[region][bucketName] + bucket, ok := b.buckets.Get(bucketName) b.mu.RUnlock() - if bucket == nil { + if !ok { return nil } diff --git a/services/s3/extra_backend.go b/services/s3/extra_backend.go index 3552ab202..f9f95cdc2 100644 --- a/services/s3/extra_backend.go +++ b/services/s3/extra_backend.go @@ -126,6 +126,7 @@ func (b *InMemoryBackend) GetBucketRequestPayment( // ObjectAttributes is the projection of object metadata returned by GetObjectAttributes. type ObjectAttributes struct { + LastModified time.Time Checksum map[string]string ETag string StorageClass string @@ -171,6 +172,7 @@ func (b *InMemoryBackend) GetObjectAttributes( ETag: ver.ETag, ObjectSize: ver.Size, StorageClass: ver.StorageClass, + LastModified: ver.LastModified, Checksum: map[string]string{}, } diff --git a/services/s3/handler_extra_test.go b/services/s3/handler_extra_test.go index c499af683..f5dc31b71 100644 --- a/services/s3/handler_extra_test.go +++ b/services/s3/handler_extra_test.go @@ -2202,6 +2202,9 @@ func TestS3BucketReplicationCRUD(t *testing.T) { ) require.NoError(t, err) + // Real S3 requires versioning enabled before replication config. + enableVersioning(t, handler, bucket) + // GetBucketReplication before put → 404 req := httptest.NewRequest(http.MethodGet, "/"+bucket+"?replication", nil) rec := httptest.NewRecorder() @@ -3632,11 +3635,11 @@ func TestHandler_GetObject_ExpirationHeader(t *testing.T) { rec.Header().Get("Accept-Ranges"), "Accept-Ranges should be set on GET", ) - assert.Equal( + // Real S3 omits x-amz-storage-class for STANDARD-class objects. + assert.Empty( t, - "STANDARD", rec.Header().Get("X-Amz-Storage-Class"), - "X-Amz-Storage-Class should be STANDARD", + "X-Amz-Storage-Class should be omitted for STANDARD", ) } @@ -3653,7 +3656,31 @@ func TestHandler_HeadObject_StorageClassAndAcceptRanges(t *testing.T) { require.Equal(t, http.StatusOK, rec.Code) assert.Equal(t, "bytes", rec.Header().Get("Accept-Ranges")) - assert.Equal(t, "STANDARD", rec.Header().Get("X-Amz-Storage-Class")) + // Real S3 omits x-amz-storage-class for STANDARD-class objects. + assert.Empty(t, rec.Header().Get("X-Amz-Storage-Class")) +} + +// TestHandler_GetObject_ResponseHeaderOverrides verifies the AWS response-* +// query parameters override the corresponding response headers on GET (used +// heavily via presigned URLs, e.g. forcing a download filename). +func TestHandler_GetObject_ResponseHeaderOverrides(t *testing.T) { + t.Parallel() + + handler, backend := newTestHandler(t) + mustCreateBucket(t, backend, "bkt") + mustPutObject(t, backend, "bkt", "obj", []byte("data")) + + req := httptest.NewRequest(http.MethodGet, + "/bkt/obj?response-content-type=application/pdf"+ + "&response-content-disposition=attachment%3B%20filename%3D%22r.pdf%22"+ + "&response-cache-control=no-cache", nil) + rec := httptest.NewRecorder() + serveS3Handler(handler, rec, req) + + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "application/pdf", rec.Header().Get("Content-Type")) + assert.Equal(t, `attachment; filename="r.pdf"`, rec.Header().Get("Content-Disposition")) + assert.Equal(t, "no-cache", rec.Header().Get("Cache-Control")) } func TestHandler_GetObject_RangeContentLength(t *testing.T) { @@ -3693,6 +3720,27 @@ func TestHandler_PutBucketACL_InvalidValue(t *testing.T) { assert.Equal(t, http.StatusBadRequest, rec.Code) } +// TestHandler_PutBucketACL_RejectsObjectOnlyCannedACLs verifies that the +// object-only canned ACLs (bucket-owner-read / bucket-owner-full-control) are +// rejected on PutBucketAcl with 400 InvalidArgument, matching real S3 +// (types.BucketCannedACL does not include them). +func TestHandler_PutBucketACL_RejectsObjectOnlyCannedACLs(t *testing.T) { + t.Parallel() + + handler, backend := newTestHandler(t) + mustCreateBucket(t, backend, "bkt") + + for _, acl := range []string{"bucket-owner-read", "bucket-owner-full-control"} { + req := httptest.NewRequest(http.MethodPut, "/bkt?acl", nil) + req.Header.Set("X-Amz-Acl", acl) + rec := httptest.NewRecorder() + serveS3Handler(handler, rec, req) + + assert.Equal(t, http.StatusBadRequest, rec.Code, "canned ACL %q must be rejected", acl) + assert.Contains(t, rec.Body.String(), "InvalidArgument") + } +} + func TestHandler_ListMultipartUploads_MaxUploads(t *testing.T) { t.Parallel() diff --git a/services/s3/handler_stubs.go b/services/s3/handler_stubs.go index 6ad728394..03f8c21f3 100644 --- a/services/s3/handler_stubs.go +++ b/services/s3/handler_stubs.go @@ -214,13 +214,15 @@ func actionIncludesGetObject(action any) bool { } // objectAttributesResult is the populated XML body for GetObjectAttributes. +// ObjectSize carries no omitempty: a legitimate 0-byte object must still emit +// 0 so the SDK populates its *int64 field. type objectAttributesResult struct { XMLName xml.Name `xml:"GetObjectAttributesResult"` Xmlns string `xml:"xmlns,attr"` ETag string `xml:"ETag,omitempty"` Checksum *attrsChecksumElem `xml:"Checksum,omitempty"` StorageClass string `xml:"StorageClass,omitempty"` - ObjectSize int64 `xml:"ObjectSize,omitempty"` + ObjectSize int64 `xml:"ObjectSize"` } type attrsChecksumElem struct { @@ -282,6 +284,12 @@ func (h *S3Handler) handleGetObjectAttributes( w.Header().Set("X-Amz-Version-Id", versionID) } + // AWS returns the object's Last-Modified as an HTTP header (RFC1123), not a + // body element, on GetObjectAttributes. + if !attrs.LastModified.IsZero() { + w.Header().Set("Last-Modified", attrs.LastModified.UTC().Format(http.TimeFormat)) + } + httputils.WriteXML(ctx, w, http.StatusOK, out) } diff --git a/services/s3/janitor.go b/services/s3/janitor.go index 700c6a4c9..098dcd7d1 100644 --- a/services/s3/janitor.go +++ b/services/s3/janitor.go @@ -223,13 +223,12 @@ func (j *Janitor) sweepAndDrain(ctx context.Context) { b := j.Backend b.mu.RLock("S3Janitor") - pending := make([]string, 0, len(b.buckets)) + all := b.buckets.All() + pending := make([]string, 0, len(all)) - for _, regionBuckets := range b.buckets { - for name, bucket := range regionBuckets { - if bucket.DeletePending { - pending = append(pending, name) - } + for _, bucket := range all { + if bucket.DeletePending { + pending = append(pending, bucket.Name) } } b.mu.RUnlock() @@ -305,16 +304,12 @@ func (j *Janitor) cleanupDefaultMultipart(_ context.Context) { now := time.Now().UTC() abortBefore := now.Add(-defaultMultipartMaxAge) - type expiredKey struct{ bucket, uploadID string } - b.mu.RLock("S3Janitor.cleanupDefaultMultipart.scan") - var expired []expiredKey + var expired []string - for bucketName, uploads := range b.uploads { - for uploadID, upload := range uploads { - if upload.Initiated.Before(abortBefore) { - expired = append(expired, expiredKey{bucketName, uploadID}) - } + for _, upload := range b.uploads.All() { + if upload.Initiated.Before(abortBefore) { + expired = append(expired, upload.UploadID) } } b.mu.RUnlock() @@ -324,10 +319,8 @@ func (j *Janitor) cleanupDefaultMultipart(_ context.Context) { } b.mu.Lock("S3Janitor.cleanupDefaultMultipart.delete") - for _, e := range expired { - if uploads, ok := b.uploads[e.bucket]; ok { - delete(uploads, e.uploadID) - } + for _, uploadID := range expired { + b.uploads.Delete(uploadID) } b.mu.Unlock() } @@ -343,12 +336,13 @@ func (j *Janitor) cleanupDefaultMultipart(_ context.Context) { func (j *Janitor) processBucket(ctx context.Context, name string) { b := j.Backend - // Locate the bucket once; it cannot move between regions. + // Locate the bucket once; buckets are keyed by name, so this is a single + // O(1) lookup (bucket names cannot collide, let alone "move regions"). b.mu.RLock("S3Janitor.processBucket") - bucket, foundRegion := findBucketAcrossRegions(b.buckets, name) + bucket, ok := b.buckets.Get(name) b.mu.RUnlock() - if bucket == nil { + if !ok { return } @@ -374,25 +368,11 @@ func (j *Janitor) processBucket(ctx context.Context, name string) { continue } - // Bucket is empty — remove it from the region map and clean up the - // region entry if it has become empty to prevent unbounded map growth. - // Guard the index removal: only delete if it still points at foundRegion - // so a future bucket with the same name keeps its index entry. - // Also purge orphaned uploads and tags to prevent resource leaks. + // Bucket is empty — remove it from the table and purge orphaned + // uploads and tags to prevent resource leaks. b.mu.Lock("S3Janitor.removeBucket") - if regionBuckets, exists := b.buckets[foundRegion]; exists { - delete(regionBuckets, name) - - if len(regionBuckets) == 0 { - delete(b.buckets, foundRegion) - } - } - - if b.bucketIndex[name] == foundRegion { - delete(b.bucketIndex, name) - } - - delete(b.uploads, name) + b.buckets.Delete(name) + b.purgeUploadsForBucketLocked(name) prefix := name + "/" for tagKey := range b.tags { @@ -429,34 +409,33 @@ func (j *Janitor) sweepLifecycle(ctx context.Context) { } var snapshots []bucketSnapshot - for _, regionBuckets := range b.buckets { - for name, bucket := range regionBuckets { - if bucket.DeletePending || bucket.LifecycleConfig == "" { - continue - } - bucket.mu.RLock("S3Janitor.sweepLifecycleLCRead") - lcXML := bucket.LifecycleConfig - bucket.mu.RUnlock() - if lcXML == "" { - continue - } + for _, bucket := range b.buckets.All() { + name := bucket.Name + if bucket.DeletePending || bucket.LifecycleConfig == "" { + continue + } + bucket.mu.RLock("S3Janitor.sweepLifecycleLCRead") + lcXML := bucket.LifecycleConfig + bucket.mu.RUnlock() + if lcXML == "" { + continue + } - // Snapshot tags for this bucket's objects. - tagsByKey := make(map[string][]types.Tag) - pfx := name + "/" - for k, v := range b.tags { - if strings.HasPrefix(k, pfx) { - tagsByKey[k] = slices.Clone(v) - } + // Snapshot tags for this bucket's objects. + tagsByKey := make(map[string][]types.Tag) + pfx := name + "/" + for k, v := range b.tags { + if strings.HasPrefix(k, pfx) { + tagsByKey[k] = slices.Clone(v) } - - snapshots = append(snapshots, bucketSnapshot{ - name: name, - bucket: bucket, - lcXML: lcXML, - tagsByKey: tagsByKey, - }) } + + snapshots = append(snapshots, bucketSnapshot{ + name: name, + bucket: bucket, + lcXML: lcXML, + tagsByKey: tagsByKey, + }) } b.mu.RUnlock() @@ -801,21 +780,6 @@ func tagMatchesFilter(t types.Tag, f lifecycleTag) bool { t.Value != nil && *t.Value == f.Value } -// findBucketAcrossRegions returns the bucket and its region for the given bucket name, -// or nil and an empty string if not found. Must be called with b.mu held. -func findBucketAcrossRegions( - buckets map[string]map[string]*StoredBucket, - name string, -) (*StoredBucket, string) { - for region, regionBuckets := range buckets { - if bkt, exists := regionBuckets[name]; exists { - return bkt, region - } - } - - return nil, "" -} - // GetExpirationHeader calculates the x-amz-expiration header for an object // based on the bucket's lifecycle configuration. func (j *Janitor) GetExpirationHeader( @@ -978,16 +942,20 @@ func (j *Janitor) abortStaleMultipartUploads(bucketName string, abortBefore time b.mu.Lock("S3Janitor.abortStaleMultipartUploads") defer b.mu.Unlock() - bucketUploads, ok := b.uploads[bucketName] - if !ok { - return - } + // Copy matching upload IDs out of the index-owned group slice before + // issuing any Delete, per [store.Index.Get]'s doc. + grouped := b.uploadsByBucket.Get(bucketName) + stale := make([]string, 0, len(grouped)) - for uploadID, upload := range bucketUploads { + for _, upload := range grouped { if upload.Initiated.Before(abortBefore) { - delete(bucketUploads, uploadID) + stale = append(stale, upload.UploadID) } } + + for _, uploadID := range stale { + b.uploads.Delete(uploadID) + } } // applyStorageClassTransitions updates the StorageClass of current (latest) object versions diff --git a/services/s3/multipart_ops.go b/services/s3/multipart_ops.go index 2bf386ccc..6ec8de12b 100644 --- a/services/s3/multipart_ops.go +++ b/services/s3/multipart_ops.go @@ -348,6 +348,7 @@ func (h *S3Handler) listMultipartUploads( Prefix: encodeListKey(encodingType, q.Get("prefix")), Delimiter: encodeListKey(encodingType, q.Get("delimiter")), KeyMarker: encodeListKey(encodingType, q.Get("key-marker")), + UploadIDMarker: q.Get("upload-id-marker"), MaxUploads: int(aws.ToInt32(out.MaxUploads)), IsTruncated: aws.ToBool(out.IsTruncated), NextKeyMarker: encodeListKey(encodingType, aws.ToString(out.NextKeyMarker)), diff --git a/services/s3/object_ops.go b/services/s3/object_ops.go index 016debc8c..0dc26d803 100644 --- a/services/s3/object_ops.go +++ b/services/s3/object_ops.go @@ -281,6 +281,8 @@ func (h *S3Handler) writeHeadObjectResponse( w.Header().Set("Content-Disposition", cd) } + applyResponseOverrideHeaders(w, r) + h.dispatchAccessLog(ctx, r, bucketName, "REST.HEAD.OBJECT", key, http.StatusOK, 0) w.WriteHeader(http.StatusOK) @@ -782,6 +784,33 @@ func (h *S3Handler) setGetObjectResponseHeaders( if r.Header.Get("X-Amz-Checksum-Mode") == "ENABLED" { h.handleChecksumMode(w, ver, details) } + + applyResponseOverrideHeaders(w, r) +} + +// responseOverrideParams maps the AWS GetObject/HeadObject response-override +// query parameters to the HTTP response header each one replaces. These let a +// caller (commonly via a presigned URL) override the object's stored headers, +// e.g. response-content-disposition to force a download filename. +var responseOverrideParams = map[string]string{ //nolint:gochecknoglobals // fixed lookup table + "response-content-type": "Content-Type", + "response-content-language": "Content-Language", + "response-expires": "Expires", + "response-cache-control": "Cache-Control", + "response-content-disposition": "Content-Disposition", + "response-content-encoding": "Content-Encoding", +} + +// applyResponseOverrideHeaders applies any response-* query-parameter overrides +// to the outgoing headers, matching real S3 GetObject/HeadObject behaviour. An +// override always wins over the object's stored header value. +func applyResponseOverrideHeaders(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + for param, header := range responseOverrideParams { + if v := q.Get(param); v != "" { + w.Header().Set(header, v) + } + } } // serveObjectBody handles range requests and writes the object body. @@ -1304,13 +1333,14 @@ func (h *S3Handler) setCommonHeaders(w http.ResponseWriter, out objectCommonDeta w.Header().Set("X-Amz-Version-Id", *out.VersionID) } - // Advertise byte-range support and the object's actual storage class. + // Advertise byte-range support. AWS returns x-amz-storage-class for every + // object EXCEPT those in the STANDARD class, for which the header is omitted + // (see HeadObject/GetObject output docs) — so a blank or STANDARD class + // produces no header here. w.Header().Set("Accept-Ranges", "bytes") - sc := out.StorageClass - if sc == "" { - sc = storageStandard + if sc := out.StorageClass; sc != "" && sc != storageStandard { + w.Header().Set("X-Amz-Storage-Class", sc) } - w.Header().Set("X-Amz-Storage-Class", sc) h.setChecksumHeaders(w, out) } diff --git a/services/s3/parity_batch7_test.go b/services/s3/parity_batch7_test.go index 070aef6f1..f155abd27 100644 --- a/services/s3/parity_batch7_test.go +++ b/services/s3/parity_batch7_test.go @@ -40,6 +40,9 @@ func TestS3BucketReplication_PutObjectReplicates(t *testing.T) { require.Equal(t, http.StatusOK, rec.Code, "create bucket %s", b) } + // Real S3 requires versioning enabled on the source before replication. + enableVersioning(t, handler, src) + cfgXML := fmt.Sprintf(` arn:aws:iam::123456789012:role/repl @@ -89,6 +92,9 @@ func TestS3BucketReplication_PrefixFilter(t *testing.T) { require.Equal(t, http.StatusOK, rec.Code) } + // Real S3 requires versioning enabled on the source before replication. + enableVersioning(t, handler, src) + cfgXML := fmt.Sprintf(` arn:aws:iam::123456789012:role/repl diff --git a/services/s3/parity_c_test.go b/services/s3/parity_c_test.go index 56cf4de78..3b8bfcdd0 100644 --- a/services/s3/parity_c_test.go +++ b/services/s3/parity_c_test.go @@ -64,6 +64,9 @@ func TestParity_PutBucketReplication_RequiresRoleAndRules(t *testing.T) { handler, backend := newTestHandler(t) mustCreateBucket(t, backend, "src-bucket") + // Real S3 requires versioning enabled before a replication config + // can be applied. + enableVersioning(t, handler, "src-bucket") req := httptest.NewRequest( http.MethodPut, @@ -93,6 +96,9 @@ func TestParity_PutBucketReplication_ExistingConfigIsOverwritten(t *testing.T) { handler, backend := newTestHandler(t) mustCreateBucket(t, backend, "rep-bucket") + // Real S3 requires versioning enabled before a replication config can be + // applied. + enableVersioning(t, handler, "rep-bucket") cfg1 := `` + `arn:aws:iam::000000000000:role/R1` + diff --git a/services/s3/persistence.go b/services/s3/persistence.go index 233c261e1..aa1ef0cc7 100644 --- a/services/s3/persistence.go +++ b/services/s3/persistence.go @@ -3,85 +3,50 @@ package s3 import ( "context" "encoding/json" - "maps" + "fmt" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" ) -type backendSnapshot struct { - Buckets map[string]map[string]*StoredBucket `json:"buckets"` - Tags map[string][]types.Tag `json:"tags"` - Uploads map[string]map[string]*StoredMultipartUpload `json:"uploads"` - DefaultRegion string `json:"defaultRegion"` -} - -// UnmarshalJSON implements [json.Unmarshaler] so that backendSnapshot can decode -// both the current nested uploads format introduced in issue #620 and the -// legacy flat format used by older snapshots: +// s3SnapshotVersion identifies the shape of backendSnapshot's Tables blob +// (i.e. the set/shape of resources registered on b.registry -- see +// NewInMemoryBackend in backend_memory.go). It must be bumped whenever a +// change there would make an older snapshot unsafe to decode as the current +// shape. Restore compares this against the persisted value and discards +// (rather than attempts to partially decode) any mismatch -- see Restore +// below. This mirrors services/ec2 (commit 12e611a4) and the services/sqs +// pilot (commit 0f09d77c) that introduced the same pattern. // -// legacy: {"uploads": {"": {uploadID, bucket, key, …}}} -// current: {"uploads": {"": {"": {…}}}} +// Bumping to 1 here (from the previous versionless shape) is a deliberate, +// one-time break: the old {"buckets": {region: {name: ...}}, "uploads": {bucket: +// {uploadID: ...}}} shape (and its legacy flat-uploads variant, see the removed +// migrateUploads) is superseded by the registry's {"tables": {"buckets": [...], +// "uploads": [...]}} shape. Any snapshot written before this change decodes +// with Version == 0, fails the guard below, and is discarded cleanly rather +// than silently misinterpreted -- the same tradeoff services/ec2 and +// services/sqs made for their own Phase 3.x conversions. +const s3SnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the S3 backend. // -// Detection: if a top-level uploads value has a non-empty "uploadID" field it -// is a StoredMultipartUpload (legacy); otherwise it is a bucket-level map. -func (s *backendSnapshot) UnmarshalJSON(data []byte) error { - type snapshotRaw struct { - Buckets map[string]map[string]*StoredBucket `json:"buckets"` - Tags map[string][]types.Tag `json:"tags"` - Uploads map[string]json.RawMessage `json:"uploads"` - DefaultRegion string `json:"defaultRegion"` - } - - var raw snapshotRaw - if err := json.Unmarshal(data, &raw); err != nil { - return err - } - - s.Buckets = raw.Buckets - s.Tags = raw.Tags - s.DefaultRegion = raw.DefaultRegion - s.Uploads = migrateUploads(raw.Uploads) - - return nil -} - -// migrateUploads converts the raw uploads JSON into the nested -// map[bucket]map[uploadID]*StoredMultipartUpload format, transparently -// upgrading the legacy flat map[uploadID]*StoredMultipartUpload shape. -func migrateUploads(raw map[string]json.RawMessage) map[string]map[string]*StoredMultipartUpload { - nested := make(map[string]map[string]*StoredMultipartUpload, len(raw)) - - for topKey, value := range raw { - // Probe: does this value look like a StoredMultipartUpload (legacy flat entry)? - var probe struct { - UploadID string `json:"uploadID"` - } - - if err := json.Unmarshal(value, &probe); err == nil && probe.UploadID != "" { - // Legacy flat format — top key is the upload ID. - var upload StoredMultipartUpload - if unmarshalErr := json.Unmarshal(value, &upload); unmarshalErr == nil { - bkt := upload.Bucket - if nested[bkt] == nil { - nested[bkt] = make(map[string]*StoredMultipartUpload) - } - nested[bkt][upload.UploadID] = &upload - } - - continue - } - - // Current nested format — top key is the bucket name. - var bucketUploads map[string]*StoredMultipartUpload - if err := json.Unmarshal(value, &bucketUploads); err == nil { - nested[topKey] = bucketUploads - } - } - - return nested +// Tables holds one JSON-encoded array per registered table, produced by +// [store.Registry.SnapshotAll] -- currently "buckets" ([]*StoredBucket) and +// "uploads" ([]*StoredMultipartUpload). Both value types serialise directly +// (no DTO layer): StoredBucket's and StoredMultipartUpload's only +// non-serialisable fields (mu, and StoredMultipartUpload.closed) are +// unexported, so encoding/json already skips them -- the same reason +// services/ec2's conversion needed zero DTOs. Restore re-initialises those +// skipped fields via reinitBucketMutexes/reinitUploadMutexes below, exactly as +// the pre-conversion code did. +type backendSnapshot struct { + Tables map[string]json.RawMessage `json:"tables"` + Tags map[string][]types.Tag `json:"tags"` + DefaultRegion string `json:"defaultRegion"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. @@ -90,10 +55,21 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "s3: snapshot table marshal failed", "error", err) + + return nil + } + snap := backendSnapshot{ - Buckets: b.buckets, + Version: s3SnapshotVersion, + Tables: tables, Tags: b.tags, - Uploads: b.uploads, DefaultRegion: b.defaultRegion, } @@ -112,41 +88,48 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() - normalizeSnapshot(&snap) - reinitBucketMutexes(snap.Buckets) - reinitUploadMutexes(snap.Uploads) - - b.buckets = snap.Buckets - b.tags = snap.Tags - b.uploads = snap.Uploads - b.defaultRegion = snap.DefaultRegion - b.bucketIndex = buildBucketIndex(snap.Buckets) + if snap.Version != s3SnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/ec2 and services/sqs pilots. + logger.Load(ctx).WarnContext(ctx, + "s3: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", s3SnapshotVersion) - return nil -} + b.registry.ResetAll() + b.tags = make(map[string][]types.Tag) -// normalizeSnapshot ensures nil maps in the snapshot are replaced with empty maps. -func normalizeSnapshot(snap *backendSnapshot) { - if snap.Buckets == nil { - snap.Buckets = make(map[string]map[string]*StoredBucket) + return nil } - if snap.Tags == nil { - snap.Tags = make(map[string][]types.Tag) + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("s3: restore snapshot tables: %w", err) } - if snap.Uploads == nil { - snap.Uploads = make(map[string]map[string]*StoredMultipartUpload) + reinitBucketMutexes(b.buckets.All()) + reinitUploadMutexes(b.uploads.All()) + + if snap.Tags != nil { + b.tags = snap.Tags + } else { + b.tags = make(map[string][]types.Tag) } + + b.defaultRegion = snap.DefaultRegion + + return nil } // reinitBucketMutexes reinitialises per-bucket and per-object mutexes after -// deserialisation, since [sync.Mutex] values cannot be serialised. -func reinitBucketMutexes(buckets map[string]map[string]*StoredBucket) { - for _, regionBuckets := range buckets { - for _, bucket := range regionBuckets { - reinitSingleBucket(bucket) - } +// deserialisation, since [lockmetrics.RWMutex] values (held via unexported +// pointer fields, and therefore skipped by encoding/json) cannot be +// serialised. +func reinitBucketMutexes(buckets []*StoredBucket) { + for _, bucket := range buckets { + reinitSingleBucket(bucket) } } @@ -184,49 +167,14 @@ func reinitSingleBucket(bucket *StoredBucket) { } // reinitUploadMutexes reinitialises per-upload mutexes after deserialisation. -func reinitUploadMutexes(uploads map[string]map[string]*StoredMultipartUpload) { - for _, bucketUploads := range uploads { - for _, u := range bucketUploads { - if u.mu == nil { - u.mu = lockmetrics.New("s3.upload") - } +func reinitUploadMutexes(uploads []*StoredMultipartUpload) { + for _, u := range uploads { + if u.mu == nil { + u.mu = lockmetrics.New("s3.upload") } } } -// buildBucketIndex constructs the name→region index from the bucket map. -// Active (non-pending) buckets take precedence over pending-delete entries -// to ensure getBucket resolves to the live bucket after a Restore. Pending -// buckets are included only when no active entry exists for that name, so -// that idempotent DeleteBucket calls still work after a Restore. -func buildBucketIndex(buckets map[string]map[string]*StoredBucket) map[string]string { - index := make(map[string]string) - - // Two-pass approach: first register active buckets, then fill in any - // pending-only names. This makes the result deterministic regardless of - // map iteration order. - pendingOnly := make(map[string]string) - - for region, regionBuckets := range buckets { - for bucketName, bucket := range regionBuckets { - if bucket.DeletePending { - // Record as pending-only candidate; active entry wins. - if _, activeExists := index[bucketName]; !activeExists { - pendingOnly[bucketName] = region - } - } else { - index[bucketName] = region - // Remove any pending-only candidate now that active is known. - delete(pendingOnly, bucketName) - } - } - } - - maps.Copy(index, pendingOnly) - - return index -} - // Snapshot implements persistence.Persistable by delegating to the backend. func (h *S3Handler) Snapshot(ctx context.Context) []byte { type snapshotter interface { diff --git a/services/s3/persistence_test.go b/services/s3/persistence_test.go index aa58df9f6..bef53ce22 100644 --- a/services/s3/persistence_test.go +++ b/services/s3/persistence_test.go @@ -1,10 +1,13 @@ package s3_test import ( + "io" + "strings" "testing" "github.com/aws/aws-sdk-go-v2/aws" sdk_s3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -40,6 +43,102 @@ func TestInMemoryBackend_SnapshotRestore(t *testing.T) { assert.Equal(t, id, *out.Buckets[0].Name) }, }, + { + // Exercises every piece of backend state that Snapshot/Restore must + // round-trip: the converted "buckets" and "uploads" store.Table + // entries (including an inline object inside the bucket entry), + // AND the raw-left b.tags map and defaultRegion string that stayed + // plain fields (not store.Tables) — so a future change that drops + // either from Snapshot/Restore fails this test. + name: "full_state_round_trip", + setup: func(b *s3.InMemoryBackend) string { + b.SetDefaultRegion("eu-west-1") + + ctx := t.Context() + bucketName := "full-state-bucket" + + if _, err := b.CreateBucket(ctx, &sdk_s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + }); err != nil { + return "" + } + + if _, err := b.PutObject(ctx, &sdk_s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String("obj.txt"), + Body: strings.NewReader("hello"), + }); err != nil { + return "" + } + + if _, err := b.PutObjectTagging(ctx, &sdk_s3.PutObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String("obj.txt"), + Tagging: &types.Tagging{ + TagSet: []types.Tag{{Key: aws.String("env"), Value: aws.String("prod")}}, + }, + }); err != nil { + return "" + } + + if _, err := b.CreateMultipartUpload(ctx, &sdk_s3.CreateMultipartUploadInput{ + Bucket: aws.String(bucketName), + Key: aws.String("large.bin"), + }); err != nil { + return "" + } + + return bucketName + }, + verify: func(t *testing.T, b *s3.InMemoryBackend, id string) { + t.Helper() + + ctx := t.Context() + + out, err := b.ListBuckets(ctx, &sdk_s3.ListBucketsInput{}) + require.NoError(t, err) + require.Len(t, out.Buckets, 1) + assert.Equal(t, id, aws.ToString(out.Buckets[0].Name)) + + // The object stored inline inside the bucket's store.Table entry. + getOut, err := b.GetObject(ctx, &sdk_s3.GetObjectInput{ + Bucket: aws.String(id), + Key: aws.String("obj.txt"), + }) + require.NoError(t, err) + body, readErr := io.ReadAll(getOut.Body) + require.NoError(t, readErr) + assert.Equal(t, "hello", string(body)) + + // b.tags: a raw (non-store.Table) map — must still round-trip. + tagOut, err := b.GetObjectTagging(ctx, &sdk_s3.GetObjectTaggingInput{ + Bucket: aws.String(id), + Key: aws.String("obj.txt"), + }) + require.NoError(t, err) + require.Len(t, tagOut.TagSet, 1) + assert.Equal(t, "env", aws.ToString(tagOut.TagSet[0].Key)) + assert.Equal(t, "prod", aws.ToString(tagOut.TagSet[0].Value)) + + // The "uploads" store.Table entry, and its bucket-index grouping. + mpOut, err := b.ListMultipartUploads(ctx, &sdk_s3.ListMultipartUploadsInput{ + Bucket: aws.String(id), + }) + require.NoError(t, err) + require.Len(t, mpOut.Uploads, 1) + assert.Equal(t, "large.bin", aws.ToString(mpOut.Uploads[0].Key)) + + // b.defaultRegion: a raw (non-store.Table) string field — a + // freshly created bucket with no explicit region must land in + // the restored default region. + require.NoError(t, err) + _, err = b.CreateBucket(ctx, &sdk_s3.CreateBucketInput{ + Bucket: aws.String("region-check-bucket"), + }) + require.NoError(t, err) + assert.Equal(t, "eu-west-1", b.BucketRegion("region-check-bucket")) + }, + }, { name: "empty_backend_round_trip", setup: func(_ *s3.InMemoryBackend) string { return "" }, @@ -103,52 +202,95 @@ func TestInMemoryBackend_SnapshotRestore(t *testing.T) { } } -// TestInMemoryBackend_RestoreActivePrecedesOverPending verifies that -// buildBucketIndex prefers the active bucket over a pending-delete entry when -// a snapshot (e.g. from an older version) contains both for the same name. -// The snapshot is crafted as raw JSON to exercise the path directly. -func TestInMemoryBackend_RestoreActivePrecedesOverPending(t *testing.T) { +// TestInMemoryBackend_RestoreDiscardsIncompatibleSnapshot verifies that a +// snapshot whose "version" field doesn't match the current s3SnapshotVersion — +// including every snapshot written before the Phase 3.3 pkgs/store conversion +// (which predates the version field entirely, and used the old +// {"buckets": {region: {name: ...}}, "uploads": {bucket: {uploadID: ...}}} +// shape, with a legacy flat-uploads variant on top of that) — is discarded +// cleanly (Restore returns no error, backend ends up empty) rather than +// partially decoded as the current {"tables": {...}} shape. This mirrors the +// services/ec2 and services/sqs Phase 3.x conversions, which made the same +// tradeoff. +func TestInMemoryBackend_RestoreDiscardsIncompatibleSnapshot(t *testing.T) { t.Parallel() - // Craft a snapshot JSON that has: - // us-east-1 / "shared": DeletePending = true (pending-delete) - // us-west-2 / "shared": DeletePending = false (active) - snap := []byte(`{ - "buckets": { - "us-east-1": { - "shared": { - "name": "shared", - "deletePending": true, - "versioning": "Suspended", - "objects": {} - } - }, - "us-west-2": { - "shared": { - "name": "shared", - "deletePending": false, - "versioning": "Suspended", - "objects": {} - } - } + tests := []struct { + name string + snapshot []byte + }{ + { + // Pre-conversion region-nested bucket shape, no version field. + name: "legacy_region_nested_buckets_no_version", + snapshot: []byte(`{ + "buckets": { + "us-east-1": { + "shared": { + "name": "shared", + "deletePending": true, + "versioning": "Suspended", + "objects": {} + } + }, + "us-west-2": { + "shared": { + "name": "shared", + "deletePending": false, + "versioning": "Suspended", + "objects": {} + } + } + }, + "tags": {}, + "uploads": {}, + "defaultRegion": "us-east-1" + }`), + }, + { + // Pre-issue-620 flat uploads shape on top of the pre-conversion + // bucket shape, no version field. + name: "legacy_flat_uploads_no_version", + snapshot: []byte(`{ + "buckets": { + "us-east-1": { + "my-bucket": { + "name": "my-bucket", + "deletePending": false, + "objects": {} + } + } + }, + "tags": {}, + "uploads": { + "1234567890": { + "uploadID": "1234567890", + "bucket": "my-bucket", + "key": "large-file", + "initiated": "2024-01-01T00:00:00Z", + "parts": {} + } + }, + "defaultRegion": "us-east-1" + }`), }, - "tags": {}, - "uploads": {}, - "defaultRegion": "us-east-1" - }`) + { + name: "explicit_future_version", + snapshot: []byte(`{"version": 999, "tables": {}}`), + }, + } - b := s3.NewInMemoryBackend(nil) - require.NoError(t, b.Restore(t.Context(), snap)) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() - // getBucket must resolve to the active (us-west-2) entry. - _, err := b.HeadBucket(t.Context(), &sdk_s3.HeadBucketInput{Bucket: aws.String("shared")}) - require.NoError(t, err, "active bucket must be reachable after restore") + b := s3.NewInMemoryBackend(nil) + require.NoError(t, b.Restore(t.Context(), tc.snapshot)) - // ListBuckets must show exactly one entry (the active bucket). - out, err := b.ListBuckets(t.Context(), &sdk_s3.ListBucketsInput{}) - require.NoError(t, err) - assert.Len(t, out.Buckets, 1) - assert.Equal(t, "shared", aws.ToString(out.Buckets[0].Name)) + out, err := b.ListBuckets(t.Context(), &sdk_s3.ListBucketsInput{}) + require.NoError(t, err) + assert.Empty(t, out.Buckets, "incompatible snapshot must be discarded, not partially decoded") + }) + } } func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { @@ -158,44 +300,3 @@ func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { err := b.Restore(t.Context(), []byte("not-valid-json")) require.Error(t, err) } - -// TestInMemoryBackend_RestoreLegacyFlatUploads verifies that a snapshot using the -// pre-issue-620 flat uploads format (map[uploadID]*StoredMultipartUpload) can be -// restored into the current nested format (map[bucket]map[uploadID]*…). -func TestInMemoryBackend_RestoreLegacyFlatUploads(t *testing.T) { - t.Parallel() - - // Craft a snapshot with the legacy flat uploads format. - snap := []byte(`{ - "buckets": { - "us-east-1": { - "my-bucket": { - "name": "my-bucket", - "deletePending": false, - "objects": {} - } - } - }, - "tags": {}, - "uploads": { - "1234567890": { - "uploadID": "1234567890", - "bucket": "my-bucket", - "key": "large-file", - "initiated": "2024-01-01T00:00:00Z", - "parts": {} - } - }, - "defaultRegion": "us-east-1" - }`) - - b := s3.NewInMemoryBackend(nil) - require.NoError(t, b.Restore(t.Context(), snap)) - - out, err := b.ListMultipartUploads(t.Context(), &sdk_s3.ListMultipartUploadsInput{ - Bucket: aws.String("my-bucket"), - }) - require.NoError(t, err) - require.Len(t, out.Uploads, 1, "legacy upload should be present after restore") - assert.Equal(t, "large-file", aws.ToString(out.Uploads[0].Key)) -} diff --git a/services/s3/post_object.go b/services/s3/post_object.go index 873713ee6..c586e31c8 100644 --- a/services/s3/post_object.go +++ b/services/s3/post_object.go @@ -3,6 +3,7 @@ package s3 import ( "bytes" "context" + "encoding/xml" "io" "mime" "mime/multipart" @@ -249,7 +250,8 @@ func writePostObjectResponse( w.Header().Set("Location", loc) w.Header().Set("Content-Type", "application/xml") w.WriteHeader(http.StatusCreated) - _, _ = w.Write(postObjectResponseXML(bucketName, key, etag, loc, r)) + body := postObjectResponseXML(bucketName, key, etag, loc, r) + _, _ = w.Write(body) //nolint:gosec // XML-escaped by encoding/xml, served as application/xml return } @@ -257,28 +259,32 @@ func writePostObjectResponse( w.WriteHeader(status) } +// postResponseXML is the body S3 returns for a successful +// browser POST upload when success_action_status=201. Marshalling through +// encoding/xml (rather than string concatenation) XML-escapes bucket/key +// values, which may legally contain '&', '<' or '>'. +type postResponseXML struct { + XMLName xml.Name `xml:"PostResponse"` + Location string `xml:"Location"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + ETag string `xml:"ETag,omitempty"` +} + func postObjectResponseXML( bucketName, key string, etag *string, location string, r *http.Request, ) []byte { - var sb strings.Builder - sb.WriteString(``) - sb.WriteString(``) - sb.WriteString(`http://`) - sb.WriteString(r.Host) - sb.WriteString(location) - sb.WriteString(``) - sb.WriteString(``) - sb.WriteString(bucketName) - sb.WriteString(``) - sb.WriteString(``) - sb.WriteString(key) - sb.WriteString(``) - if etag != nil { - sb.WriteString(``) - sb.WriteString(*etag) - sb.WriteString(``) + resp := postResponseXML{ + Location: "http://" + r.Host + location, + Bucket: bucketName, + Key: key, + ETag: aws.ToString(etag), + } + + out, err := xml.Marshal(resp) + if err != nil { + return nil } - sb.WriteString(``) - return []byte(sb.String()) + return append([]byte(xml.Header), out...) } diff --git a/services/s3/presign.go b/services/s3/presign.go index 094a71201..089b6c1c2 100644 --- a/services/s3/presign.go +++ b/services/s3/presign.go @@ -26,6 +26,10 @@ const presignedDateFormat = "20060102T150405Z" // presignedAlgorithm is the only supported pre-signed URL signing algorithm. const presignedAlgorithm = "AWS4-HMAC-SHA256" +// maxPresignExpirySeconds is the maximum lifetime AWS accepts for a presigned +// URL: 7 days expressed in seconds. +const maxPresignExpirySeconds int64 = 604800 + // minPresignCredentialParts is the minimum number of slash-delimited parts in // a valid X-Amz-Credential value: AKID/date/region/service/aws4_request. const minPresignCredentialParts = 5 @@ -125,6 +129,18 @@ func (h *S3Handler) validatePresignedRequest( return false } + // AWS caps presigned-URL lifetimes at 7 days (604800 s); anything larger is + // rejected with 400 AuthorizationQueryParametersError. + if expires > maxPresignExpirySeconds { + httputils.WriteS3ErrorResponse(ctx, w, r, ErrorResponse{ + Code: errAuthQueryParams, + Message: "X-Amz-Expires must be less than a week (in seconds); that is, the given " + + "X-Amz-Expires must be less than 604800 seconds.", + }, http.StatusBadRequest) + + return false + } + expiresAt := signedAt.Add(time.Duration(expires) * time.Second) if time.Now().UTC().After(expiresAt) { diff --git a/services/s3/presign_test.go b/services/s3/presign_test.go index 3fe6da096..f7f211183 100644 --- a/services/s3/presign_test.go +++ b/services/s3/presign_test.go @@ -296,6 +296,21 @@ func TestHandler_PresignedStructuralValidation(t *testing.T) { wantCode: http.StatusBadRequest, wantContain: "AuthorizationQueryParametersError", }, + { + // AWS caps presigned-URL lifetimes at 604800 s (7 days); larger + // values are rejected with 400 AuthorizationQueryParametersError. + name: "X-Amz-Expires over 7 days rejected", + urlFn: func() string { + return fmt.Sprintf( + "/my-bucket/file.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256"+ + "&X-Amz-Credential=test%%2F20240101%%2Fus-east-1%%2Fs3%%2Faws4_request"+ + "&X-Amz-Date=%s&X-Amz-Expires=604801&X-Amz-SignedHeaders=host&X-Amz-Signature=fakesig", + validDate, + ) + }, + wantCode: http.StatusBadRequest, + wantContain: "AuthorizationQueryParametersError", + }, } for _, tt := range tests { diff --git a/services/s3/sse_crypto_test.go b/services/s3/sse_crypto_test.go index 041eca6d3..fb3f72bd4 100644 --- a/services/s3/sse_crypto_test.go +++ b/services/s3/sse_crypto_test.go @@ -54,6 +54,40 @@ func TestSSE_S3_RoundTripEncryptsAtRest(t *testing.T) { require.Equal(t, plaintext, got) } +// TestSSE_S3_SurvivesSnapshotRestore is a regression test for the persistence +// data-loss bug where StoredObjectVersion.EncryptionDEK / EncryptionNonce were +// tagged json:"-": the ciphertext persisted but its key did not, so every +// SSE-S3/SSE-KMS object became permanently undecryptable after a restart. It +// PUTs an SSE-S3 object, snapshots the backend, restores into a fresh backend, +// and asserts the object still decrypts to the original plaintext. +func TestSSE_S3_SurvivesSnapshotRestore(t *testing.T) { + t.Parallel() + + handler, backend := newTestHandler(t) + mustCreateBucket(t, backend, "sse-persist") + + plaintext := []byte("payload that must survive a restart intact") + req := httptest.NewRequest(http.MethodPut, "/sse-persist/k.txt", bytes.NewReader(plaintext)) + req.Header.Set("X-Amz-Server-Side-Encryption", "AES256") + rec := httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + snap := backend.Snapshot(t.Context()) + require.NotNil(t, snap) + + fresh := s3.NewInMemoryBackend(&s3.GzipCompressor{}) + require.NoError(t, fresh.Restore(t.Context(), snap)) + + out, err := fresh.GetObject(context.Background(), &sdk_s3.GetObjectInput{ + Bucket: aws.String("sse-persist"), + Key: aws.String("k.txt"), + }) + require.NoError(t, err, "restored SSE object must remain decryptable") + got, _ := io.ReadAll(out.Body) + require.Equal(t, plaintext, got) +} + // TestSSE_S3_MultipartEncryptsAtRest verifies that a multipart upload created // with SSE-S3 produces a stored version whose Data is ciphertext, and that // CompleteMultipartUpload + GetObject still returns the original concatenation. diff --git a/services/s3/storage_class_test.go b/services/s3/storage_class_test.go index 90a6ed937..1ca5bdd61 100644 --- a/services/s3/storage_class_test.go +++ b/services/s3/storage_class_test.go @@ -47,22 +47,31 @@ func TestStorageClass_PutAndGet(t *testing.T) { serveS3Handler(handler, putRR, putReq) require.Equal(t, http.StatusOK, putRR.Code, "PutObject %s", sc) + // Real S3 omits x-amz-storage-class for STANDARD objects and + // returns it for every other class. + wantHeader := sc + if sc == "STANDARD" { + wantHeader = "" + } + getRR := httptest.NewRecorder() getReq := httptest.NewRequest(http.MethodGet, "/test-bucket-sc/"+key, nil) serveS3Handler(handler, getRR, getReq) require.Equal(t, http.StatusOK, getRR.Code) - assert.Equal(t, sc, getRR.Header().Get("X-Amz-Storage-Class"), "GET storage class") + assert.Equal(t, wantHeader, getRR.Header().Get("X-Amz-Storage-Class"), "GET storage class") headRR := httptest.NewRecorder() headReq := httptest.NewRequest(http.MethodHead, "/test-bucket-sc/"+key, nil) serveS3Handler(handler, headRR, headReq) require.Equal(t, http.StatusOK, headRR.Code) - assert.Equal(t, sc, headRR.Header().Get("X-Amz-Storage-Class"), "HEAD storage class") + assert.Equal(t, wantHeader, headRR.Header().Get("X-Amz-Storage-Class"), "HEAD storage class") }) } } -// TestStorageClass_DefaultIsStandard verifies objects without a storage class header default to STANDARD. +// TestStorageClass_DefaultIsStandard verifies objects without a storage class +// header default to STANDARD. Real S3 signals STANDARD by OMITTING the +// x-amz-storage-class response header, so the header must be absent (empty). func TestStorageClass_DefaultIsStandard(t *testing.T) { t.Parallel() @@ -83,7 +92,7 @@ func TestStorageClass_DefaultIsStandard(t *testing.T) { req = httptest.NewRequest(http.MethodGet, "/sc-default-bucket/obj", nil) serveS3Handler(handler, rr, req) require.Equal(t, http.StatusOK, rr.Code) - assert.Equal(t, "STANDARD", rr.Header().Get("X-Amz-Storage-Class")) + assert.Empty(t, rr.Header().Get("X-Amz-Storage-Class")) } // TestStorageClass_ListObjectsV2 verifies ListObjectsV2 returns actual storage class. diff --git a/services/s3/types.go b/services/s3/types.go index 692f7f2d4..2295d1055 100644 --- a/services/s3/types.go +++ b/services/s3/types.go @@ -13,10 +13,18 @@ import ( const NullVersion = "null" // StoredBucket represents an S3 bucket in memory. +// +// Region is the region the bucket was created in. It is the [store.Table] key +// function's identity companion: buckets are keyed by Name (globally unique — +// CreateBucket enforces this across all regions, mirroring real S3's global +// bucket-namespace), so Region moved here from the old region->name->*StoredBucket +// nesting to make that identity self-contained. It never changes after +// creation (S3 has no "move bucket to another region" operation). type StoredBucket struct { CreationDate time.Time `json:"creationDate"` Objects map[string]*StoredObject `json:"objects,omitempty"` mu *lockmetrics.RWMutex + Region string `json:"region,omitempty"` WebsiteConfig string `json:"websiteConfig,omitempty"` PublicAccessBlockConfig string `json:"publicAccessBlockConfig,omitempty"` LifecycleConfig string `json:"lifecycleConfig,omitempty"` @@ -74,12 +82,16 @@ type StoredObjectVersion struct { // PUT for SSE-S3/SSE-KMS objects. Real S3 wraps this under a KMS CMK and // stores only the wrapped form; for an in-memory mock the storage is // the same address space so we keep the raw key. SSE-C objects don't - // store the key — the customer re-supplies it on GET. - EncryptionDEK []byte `json:"-"` + // store the key — the customer re-supplies it on GET. It MUST persist: + // the ciphertext lives in Data (persisted), so dropping the DEK on a + // snapshot/restore would leave every SSE-S3/SSE-KMS object permanently + // undecryptable ([]byte round-trips as base64 under encoding/json). + EncryptionDEK []byte `json:"encryptionDEK,omitempty"` // EncryptionNonce is the GCM nonce/IV used for this object's ciphertext. // Stored alongside the ciphertext (in StoredObjectVersion.Data) so GET - // can decrypt without re-deriving anything. - EncryptionNonce []byte `json:"-"` + // can decrypt without re-deriving anything. Persisted for the same reason + // as EncryptionDEK. + EncryptionNonce []byte `json:"encryptionNonce,omitempty"` Key string `json:"key"` ETag string `json:"etag"` ContentType string `json:"contentType"` @@ -122,8 +134,12 @@ type StoredMultipartUpload struct { Tagging string `json:"tagging,omitempty"` // SSE captures the encryption headers from CreateMultipartUpload so the // completed object's assembled body can be sealed with the same envelope - // (matching real S3 — SSE is fixed at session-init). - SSE sseInfo `json:"-"` + // (matching real S3 — SSE is fixed at session-init). Persisted so that an + // in-flight upload that survives a snapshot/restore still completes with + // the caller's chosen encryption rather than silently landing unencrypted. + // (The SSE-C customer key inside sseInfo stays request-scoped — see + // sseInfo.SSECKeyB64 — so SSE-C uploads still require the key on Complete.) + SSE sseInfo `json:"sse"` // closed is set to true by AbortMultipartUpload or CompleteMultipartUpload // before the upload is removed from the index, so that concurrent UploadPart // calls that already hold a pointer to this struct can detect the invalidation. diff --git a/services/sagemaker/backend.go b/services/sagemaker/backend.go index c0098f9ac..c97679b51 100644 --- a/services/sagemaker/backend.go +++ b/services/sagemaker/backend.go @@ -14,6 +14,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awserr" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) // regionContextKey is the context key under which the per-request AWS region is stored. @@ -480,59 +481,60 @@ func cloneModelPackage(mp *ModelPackage) *ModelPackage { // are created lazily via the *Store helpers. Callers must hold b.mu while // accessing the inner maps. type InMemoryBackend struct { - models map[string]map[string]*Model - endpointConfigs map[string]map[string]*EndpointConfig - endpoints map[string]map[string]*Endpoint - trainingJobs map[string]map[string]*TrainingJob - notebooks map[string]map[string]*NotebookInstance - hpTuningJobs map[string]map[string]*HyperParameterTuningJob - associations map[string]map[string]*Association - trialComponentAssociations map[string]map[string]*TrialComponentAssociation - actions map[string]map[string]*Action - artifacts map[string]map[string]*Artifact // region -> ArtifactArn -> Artifact - contexts map[string]map[string]*Context // region -> ContextName -> Context - algorithms map[string]map[string]*Algorithm - clusters map[string]map[string]*Cluster - modelPackages map[string]map[string]*ModelPackage - modelPackageGroups map[string]map[string]*ModelPackageGroup - autoMLJobs map[string]map[string]*AutoMLJob - codeRepositories map[string]map[string]*CodeRepository - projects map[string]map[string]*Project - spaces map[string]map[string]*Space - smImages map[string]map[string]*SMImage + models map[string]*store.Table[Model] + endpointConfigs map[string]*store.Table[EndpointConfig] + endpoints map[string]*store.Table[Endpoint] + trainingJobs map[string]*store.Table[TrainingJob] + notebooks map[string]*store.Table[NotebookInstance] + hpTuningJobs map[string]*store.Table[HyperParameterTuningJob] + associations map[string]*store.Table[Association] + trialComponentAssociations map[string]*store.Table[TrialComponentAssociation] + actions map[string]*store.Table[Action] + artifacts map[string]*store.Table[Artifact] // region -> ArtifactArn -> Artifact + contexts map[string]*store.Table[Context] // region -> ContextName -> Context + algorithms map[string]*store.Table[Algorithm] + clusters map[string]*store.Table[Cluster] + modelPackages map[string]*store.Table[ModelPackage] + modelPackageGroups map[string]*store.Table[ModelPackageGroup] + autoMLJobs map[string]*store.Table[AutoMLJob] + codeRepositories map[string]*store.Table[CodeRepository] + projects map[string]*store.Table[Project] + spaces map[string]*store.Table[Space] + smImages map[string]*store.Table[SMImage] imageVersions map[string]map[string]map[int]*ImageVersion // region → imageName → version → ImageVersion imageVersionCounts map[string]map[string]int // region → imageName → latest version number - compilationJobs map[string]map[string]*CompilationJob - monitoringSchedules map[string]map[string]*MonitoringSchedule - workteams map[string]map[string]*Workteam - labelingJobs map[string]map[string]*LabelingJob - dataQualityJobDefs map[string]map[string]*JobDefinition - modelBiasJobDefs map[string]map[string]*JobDefinition - modelQualityJobDefs map[string]map[string]*JobDefinition - modelExplainJobDefs map[string]map[string]*JobDefinition - // monitoringAlerts is region -> scheduleName -> alertName -> alert. - monitoringAlerts map[string]map[string]map[string]*MonitoringAlert + compilationJobs map[string]*store.Table[CompilationJob] + monitoringSchedules map[string]*store.Table[MonitoringSchedule] + workteams map[string]*store.Table[Workteam] + labelingJobs map[string]*store.Table[LabelingJob] + dataQualityJobDefs map[string]*store.Table[JobDefinition] + modelBiasJobDefs map[string]*store.Table[JobDefinition] + modelQualityJobDefs map[string]*store.Table[JobDefinition] + modelExplainJobDefs map[string]*store.Table[JobDefinition] + // monitoringAlerts is region -> store.Table[MonitoringAlert], keyed by + // monitoringAlertKey(scheduleName, alertName). + monitoringAlerts map[string]*store.Table[MonitoringAlert] // monitoringAlertHistory is region -> history entries. monitoringAlertHistory map[string][]*MonitoringAlertHistoryEntry // monitoringExecutions is region -> "scheduleName|processingJobArn" -> execution. - monitoringExecutions map[string]map[string]*MonitoringExecution - humanTaskUis map[string]map[string]*HumanTaskUI - workforces map[string]map[string]*Workforce - flowDefinitions map[string]map[string]*FlowDefinition - appImageConfigs map[string]map[string]*AppImageConfig - inferenceExperiments map[string]map[string]*InferenceExperiment - mlflowTrackingServers map[string]map[string]*MlflowTrackingServer - mlflowApps map[string]map[string]*MlflowApp - modelCards map[string]map[string]*ModelCard - optimizationJobs map[string]map[string]*OptimizationJob - studioLifecycleConfigs map[string]map[string]*StudioLifecycleConfig - partnerApps map[string]map[string]*PartnerApp - trainingPlans map[string]map[string]*TrainingPlan - reservedCapacities map[string]map[string]*ReservedCapacity + monitoringExecutions map[string]*store.Table[MonitoringExecution] + humanTaskUis map[string]*store.Table[HumanTaskUI] + workforces map[string]*store.Table[Workforce] + flowDefinitions map[string]*store.Table[FlowDefinition] + appImageConfigs map[string]*store.Table[AppImageConfig] + inferenceExperiments map[string]*store.Table[InferenceExperiment] + mlflowTrackingServers map[string]*store.Table[MlflowTrackingServer] + mlflowApps map[string]*store.Table[MlflowApp] + modelCards map[string]*store.Table[ModelCard] + optimizationJobs map[string]*store.Table[OptimizationJob] + studioLifecycleConfigs map[string]*store.Table[StudioLifecycleConfig] + partnerApps map[string]*store.Table[PartnerApp] + trainingPlans map[string]*store.Table[TrainingPlan] + reservedCapacities map[string]*store.Table[ReservedCapacity] // trainingPlanExtensionOfferings is region -> extensionOfferingID -> pending extension offer. - trainingPlanExtensionOfferings map[string]map[string]*pendingTrainingPlanExtension + trainingPlanExtensionOfferings map[string]*store.Table[pendingTrainingPlanExtension] // modelCardExportJobs is region -> ModelCardExportJobArn -> job. - modelCardExportJobs map[string]map[string]*ModelCardExportJob + modelCardExportJobs map[string]*store.Table[ModelCardExportJob] modelARNIndex map[string]map[string]string // region → ARN → model name endpointConfigARNIndex map[string]map[string]string // region → ARN → endpoint config name endpointARNIndex map[string]map[string]string // region → ARN → endpoint name @@ -546,44 +548,53 @@ type InMemoryBackend struct { modelPackageARNIndex map[string]map[string]string // region → ARN → model package ARN processingJobARNIndex map[string]map[string]string // region → ARN → job name transformJobARNIndex map[string]map[string]string // region → ARN → job name - domains map[string]map[string]*Domain - userProfiles map[string]map[userProfileKey]*UserProfile - apps map[string]map[appKey]*App - featureGroups map[string]map[string]*FeatureGroup - featureRecords map[string]map[string]*FeatureRecord - featureMetadata map[string]map[string]*FeatureMetadata - pipelines map[string]map[string]*Pipeline - pipelineExecutions map[string]map[string]*PipelineExecution - pipelineExecSteps map[string]map[string]*PipelineExecutionStep - experiments map[string]map[string]*Experiment - trials map[string]map[string]*Trial - trialComponents map[string]map[string]*TrialComponent - notebookLifecycleConfigs map[string]map[string]*NotebookInstanceLifecycleConfig - processingJobs map[string]map[string]*ProcessingJob - transformJobs map[string]map[string]*TransformJob - edgePackagingJobs map[string]map[string]*EdgePackagingJob - edgeDeploymentPlans map[string]map[string]*EdgeDeploymentPlan - inferenceRecommendationsJobs map[string]map[string]*InferenceRecommendationsJob - deviceFleets map[string]map[string]*DeviceFleet - devices map[string]map[deviceKey]*Device - inferenceComponents map[string]map[string]*InferenceComponent - clusterSchedulerConfigs map[string]map[string]*ClusterSchedulerConfig - computeQuotas map[string]map[string]*ComputeQuota - hubs map[string]map[string]*Hub - hubContents map[string]map[hubContentKey]*HubContent + domains map[string]*store.Table[Domain] + userProfiles map[string]*store.Table[UserProfile] + apps map[string]*store.Table[App] + featureGroups map[string]*store.Table[FeatureGroup] + featureRecords map[string]*store.Table[FeatureRecord] + featureMetadata map[string]*store.Table[FeatureMetadata] + pipelines map[string]*store.Table[Pipeline] + pipelineExecutions map[string]*store.Table[PipelineExecution] + pipelineExecSteps map[string]*store.Table[PipelineExecutionStep] + experiments map[string]*store.Table[Experiment] + trials map[string]*store.Table[Trial] + trialComponents map[string]*store.Table[TrialComponent] + notebookLifecycleConfigs map[string]*store.Table[NotebookInstanceLifecycleConfig] + processingJobs map[string]*store.Table[ProcessingJob] + transformJobs map[string]*store.Table[TransformJob] + edgePackagingJobs map[string]*store.Table[EdgePackagingJob] + edgeDeploymentPlans map[string]*store.Table[EdgeDeploymentPlan] + inferenceRecommendationsJobs map[string]*store.Table[InferenceRecommendationsJob] + deviceFleets map[string]*store.Table[DeviceFleet] + devices map[string]*store.Table[Device] + inferenceComponents map[string]*store.Table[InferenceComponent] + clusterSchedulerConfigs map[string]*store.Table[ClusterSchedulerConfig] + computeQuotas map[string]*store.Table[ComputeQuota] + hubs map[string]*store.Table[Hub] + hubContents map[string]*store.Table[HubContent] // pipelineVersions is region -> pipelineName -> versions, ordered oldest-first. pipelineVersions map[string]map[string][]*PipelineVersion // servicecatalogPortfolioEnabled is region -> whether the SageMaker // Service Catalog portfolio has been enabled via // EnableSagemakerServicecatalogPortfolio. Absent/false means Disabled. servicecatalogPortfolioEnabled map[string]bool - lifecycleParent context.Context - lifecycleCtx context.Context - lifecycleCancel context.CancelFunc - mu *lockmetrics.RWMutex - accountID string - region string - wg sync.WaitGroup + // registry lets Reset collapse the per-region store.Table lifecycle for + // every resource collection below to one registry.ResetAll() call, and + // backs Snapshot/Restore via registry.SnapshotAll()/RestoreAll(). Each + // resource field above is itself a map[string]*store.Table[T] keyed by + // region (one store.Table per region, registered lazily on first use of + // that region) rather than a single flat store.Table[T], because these + // resources are natively region-partitioned collections; see the + // xxxStore(r) helpers below for the lazy-create-and-register point. + registry *store.Registry + lifecycleParent context.Context + lifecycleCtx context.Context + lifecycleCancel context.CancelFunc + mu *lockmetrics.RWMutex + accountID string + region string + wg sync.WaitGroup } // NewInMemoryBackend creates a new in-memory SageMaker backend. @@ -595,6 +606,8 @@ func NewInMemoryBackend(accountID, region string) *InMemoryBackend { // lifecycle goroutines (status-transition simulators) are children of svcCtx, so // they are cancelled when the service shuts down rather than leaking. If svcCtx is // nil, [context.Background] is used. +// +//nolint:funlen // must initialise every resource map field; splitting would obscure the invariant func NewInMemoryBackendWithContext( svcCtx context.Context, accountID, region string, @@ -605,51 +618,51 @@ func NewInMemoryBackendWithContext( b := &InMemoryBackend{ lifecycleParent: svcCtx, - models: make(map[string]map[string]*Model), - endpointConfigs: make(map[string]map[string]*EndpointConfig), - endpoints: make(map[string]map[string]*Endpoint), - trainingJobs: make(map[string]map[string]*TrainingJob), - notebooks: make(map[string]map[string]*NotebookInstance), - hpTuningJobs: make(map[string]map[string]*HyperParameterTuningJob), - associations: make(map[string]map[string]*Association), - trialComponentAssociations: make(map[string]map[string]*TrialComponentAssociation), - actions: make(map[string]map[string]*Action), - artifacts: make(map[string]map[string]*Artifact), - contexts: make(map[string]map[string]*Context), - algorithms: make(map[string]map[string]*Algorithm), - clusters: make(map[string]map[string]*Cluster), - modelPackages: make(map[string]map[string]*ModelPackage), - modelPackageGroups: make(map[string]map[string]*ModelPackageGroup), - autoMLJobs: make(map[string]map[string]*AutoMLJob), - codeRepositories: make(map[string]map[string]*CodeRepository), - projects: make(map[string]map[string]*Project), - spaces: make(map[string]map[string]*Space), - smImages: make(map[string]map[string]*SMImage), + models: make(map[string]*store.Table[Model]), + endpointConfigs: make(map[string]*store.Table[EndpointConfig]), + endpoints: make(map[string]*store.Table[Endpoint]), + trainingJobs: make(map[string]*store.Table[TrainingJob]), + notebooks: make(map[string]*store.Table[NotebookInstance]), + hpTuningJobs: make(map[string]*store.Table[HyperParameterTuningJob]), + associations: make(map[string]*store.Table[Association]), + trialComponentAssociations: make(map[string]*store.Table[TrialComponentAssociation]), + actions: make(map[string]*store.Table[Action]), + artifacts: make(map[string]*store.Table[Artifact]), + contexts: make(map[string]*store.Table[Context]), + algorithms: make(map[string]*store.Table[Algorithm]), + clusters: make(map[string]*store.Table[Cluster]), + modelPackages: make(map[string]*store.Table[ModelPackage]), + modelPackageGroups: make(map[string]*store.Table[ModelPackageGroup]), + autoMLJobs: make(map[string]*store.Table[AutoMLJob]), + codeRepositories: make(map[string]*store.Table[CodeRepository]), + projects: make(map[string]*store.Table[Project]), + spaces: make(map[string]*store.Table[Space]), + smImages: make(map[string]*store.Table[SMImage]), imageVersions: make(map[string]map[string]map[int]*ImageVersion), imageVersionCounts: make(map[string]map[string]int), - compilationJobs: make(map[string]map[string]*CompilationJob), - monitoringSchedules: make(map[string]map[string]*MonitoringSchedule), - workteams: make(map[string]map[string]*Workteam), - labelingJobs: make(map[string]map[string]*LabelingJob), - dataQualityJobDefs: make(map[string]map[string]*JobDefinition), - modelBiasJobDefs: make(map[string]map[string]*JobDefinition), - modelQualityJobDefs: make(map[string]map[string]*JobDefinition), - modelExplainJobDefs: make(map[string]map[string]*JobDefinition), - monitoringAlerts: make(map[string]map[string]map[string]*MonitoringAlert), + compilationJobs: make(map[string]*store.Table[CompilationJob]), + monitoringSchedules: make(map[string]*store.Table[MonitoringSchedule]), + workteams: make(map[string]*store.Table[Workteam]), + labelingJobs: make(map[string]*store.Table[LabelingJob]), + dataQualityJobDefs: make(map[string]*store.Table[JobDefinition]), + modelBiasJobDefs: make(map[string]*store.Table[JobDefinition]), + modelQualityJobDefs: make(map[string]*store.Table[JobDefinition]), + modelExplainJobDefs: make(map[string]*store.Table[JobDefinition]), + monitoringAlerts: make(map[string]*store.Table[MonitoringAlert]), monitoringAlertHistory: make(map[string][]*MonitoringAlertHistoryEntry), - monitoringExecutions: make(map[string]map[string]*MonitoringExecution), - humanTaskUis: make(map[string]map[string]*HumanTaskUI), - workforces: make(map[string]map[string]*Workforce), - flowDefinitions: make(map[string]map[string]*FlowDefinition), - appImageConfigs: make(map[string]map[string]*AppImageConfig), - inferenceExperiments: make(map[string]map[string]*InferenceExperiment), - mlflowTrackingServers: make(map[string]map[string]*MlflowTrackingServer), - mlflowApps: make(map[string]map[string]*MlflowApp), - modelCards: make(map[string]map[string]*ModelCard), - optimizationJobs: make(map[string]map[string]*OptimizationJob), - studioLifecycleConfigs: make(map[string]map[string]*StudioLifecycleConfig), - partnerApps: make(map[string]map[string]*PartnerApp), - trainingPlans: make(map[string]map[string]*TrainingPlan), + monitoringExecutions: make(map[string]*store.Table[MonitoringExecution]), + humanTaskUis: make(map[string]*store.Table[HumanTaskUI]), + workforces: make(map[string]*store.Table[Workforce]), + flowDefinitions: make(map[string]*store.Table[FlowDefinition]), + appImageConfigs: make(map[string]*store.Table[AppImageConfig]), + inferenceExperiments: make(map[string]*store.Table[InferenceExperiment]), + mlflowTrackingServers: make(map[string]*store.Table[MlflowTrackingServer]), + mlflowApps: make(map[string]*store.Table[MlflowApp]), + modelCards: make(map[string]*store.Table[ModelCard]), + optimizationJobs: make(map[string]*store.Table[OptimizationJob]), + studioLifecycleConfigs: make(map[string]*store.Table[StudioLifecycleConfig]), + partnerApps: make(map[string]*store.Table[PartnerApp]), + trainingPlans: make(map[string]*store.Table[TrainingPlan]), modelARNIndex: make(map[string]map[string]string), endpointConfigARNIndex: make(map[string]map[string]string), endpointARNIndex: make(map[string]map[string]string), @@ -663,34 +676,35 @@ func NewInMemoryBackendWithContext( modelPackageARNIndex: make(map[string]map[string]string), processingJobARNIndex: make(map[string]map[string]string), transformJobARNIndex: make(map[string]map[string]string), - domains: make(map[string]map[string]*Domain), - userProfiles: make(map[string]map[userProfileKey]*UserProfile), - apps: make(map[string]map[appKey]*App), - featureGroups: make(map[string]map[string]*FeatureGroup), - featureRecords: make(map[string]map[string]*FeatureRecord), - featureMetadata: make(map[string]map[string]*FeatureMetadata), - pipelines: make(map[string]map[string]*Pipeline), - pipelineExecutions: make(map[string]map[string]*PipelineExecution), - pipelineExecSteps: make(map[string]map[string]*PipelineExecutionStep), - experiments: make(map[string]map[string]*Experiment), - trials: make(map[string]map[string]*Trial), - trialComponents: make(map[string]map[string]*TrialComponent), - notebookLifecycleConfigs: make(map[string]map[string]*NotebookInstanceLifecycleConfig), - processingJobs: make(map[string]map[string]*ProcessingJob), - transformJobs: make(map[string]map[string]*TransformJob), - edgePackagingJobs: make(map[string]map[string]*EdgePackagingJob), - edgeDeploymentPlans: make(map[string]map[string]*EdgeDeploymentPlan), - inferenceRecommendationsJobs: make(map[string]map[string]*InferenceRecommendationsJob), - deviceFleets: make(map[string]map[string]*DeviceFleet), - devices: make(map[string]map[deviceKey]*Device), - inferenceComponents: make(map[string]map[string]*InferenceComponent), - clusterSchedulerConfigs: make(map[string]map[string]*ClusterSchedulerConfig), - computeQuotas: make(map[string]map[string]*ComputeQuota), - hubs: make(map[string]map[string]*Hub), - hubContents: make(map[string]map[hubContentKey]*HubContent), + domains: make(map[string]*store.Table[Domain]), + userProfiles: make(map[string]*store.Table[UserProfile]), + apps: make(map[string]*store.Table[App]), + featureGroups: make(map[string]*store.Table[FeatureGroup]), + featureRecords: make(map[string]*store.Table[FeatureRecord]), + featureMetadata: make(map[string]*store.Table[FeatureMetadata]), + pipelines: make(map[string]*store.Table[Pipeline]), + pipelineExecutions: make(map[string]*store.Table[PipelineExecution]), + pipelineExecSteps: make(map[string]*store.Table[PipelineExecutionStep]), + experiments: make(map[string]*store.Table[Experiment]), + trials: make(map[string]*store.Table[Trial]), + trialComponents: make(map[string]*store.Table[TrialComponent]), + notebookLifecycleConfigs: make(map[string]*store.Table[NotebookInstanceLifecycleConfig]), + processingJobs: make(map[string]*store.Table[ProcessingJob]), + transformJobs: make(map[string]*store.Table[TransformJob]), + edgePackagingJobs: make(map[string]*store.Table[EdgePackagingJob]), + edgeDeploymentPlans: make(map[string]*store.Table[EdgeDeploymentPlan]), + inferenceRecommendationsJobs: make(map[string]*store.Table[InferenceRecommendationsJob]), + deviceFleets: make(map[string]*store.Table[DeviceFleet]), + devices: make(map[string]*store.Table[Device]), + inferenceComponents: make(map[string]*store.Table[InferenceComponent]), + clusterSchedulerConfigs: make(map[string]*store.Table[ClusterSchedulerConfig]), + computeQuotas: make(map[string]*store.Table[ComputeQuota]), + hubs: make(map[string]*store.Table[Hub]), + hubContents: make(map[string]*store.Table[HubContent]), accountID: accountID, region: region, mu: lockmetrics.New("sagemaker"), + registry: store.NewRegistry(), } b.initTrainingPlanExtMaps() b.resetLifecycleContext() @@ -709,128 +723,196 @@ func (b *InMemoryBackend) AccountID() string { return b.accountID } // Callers must hold b.mu. // --------------------------------------------------------------------------- -func (b *InMemoryBackend) modelsStore(r string) map[string]*Model { +func (b *InMemoryBackend) modelsStore(r string) *store.Table[Model] { if b.models[r] == nil { - b.models[r] = make(map[string]*Model) + b.models[r] = store.Register(b.registry, "models:"+r, store.New(func(v *Model) string { return v.ModelName })) } return b.models[r] } -func (b *InMemoryBackend) endpointConfigsStore(r string) map[string]*EndpointConfig { +func (b *InMemoryBackend) endpointConfigsStore(r string) *store.Table[EndpointConfig] { if b.endpointConfigs[r] == nil { - b.endpointConfigs[r] = make(map[string]*EndpointConfig) + b.endpointConfigs[r] = store.Register( + b.registry, + "endpointConfigs:"+r, + store.New(func(v *EndpointConfig) string { return v.EndpointConfigName }), + ) } return b.endpointConfigs[r] } -func (b *InMemoryBackend) endpointsStore(r string) map[string]*Endpoint { +func (b *InMemoryBackend) endpointsStore(r string) *store.Table[Endpoint] { if b.endpoints[r] == nil { - b.endpoints[r] = make(map[string]*Endpoint) + b.endpoints[r] = store.Register( + b.registry, + "endpoints:"+r, + store.New(func(v *Endpoint) string { return v.EndpointName }), + ) } return b.endpoints[r] } -func (b *InMemoryBackend) trainingJobsStore(r string) map[string]*TrainingJob { +func (b *InMemoryBackend) trainingJobsStore(r string) *store.Table[TrainingJob] { if b.trainingJobs[r] == nil { - b.trainingJobs[r] = make(map[string]*TrainingJob) + b.trainingJobs[r] = store.Register( + b.registry, + "trainingJobs:"+r, + store.New(func(v *TrainingJob) string { return v.TrainingJobName }), + ) } return b.trainingJobs[r] } -func (b *InMemoryBackend) notebooksStore(r string) map[string]*NotebookInstance { +func (b *InMemoryBackend) notebooksStore(r string) *store.Table[NotebookInstance] { if b.notebooks[r] == nil { - b.notebooks[r] = make(map[string]*NotebookInstance) + b.notebooks[r] = store.Register( + b.registry, + "notebooks:"+r, + store.New(func(v *NotebookInstance) string { return v.NotebookInstanceName }), + ) } return b.notebooks[r] } -func (b *InMemoryBackend) hpTuningJobsStore(r string) map[string]*HyperParameterTuningJob { +func (b *InMemoryBackend) hpTuningJobsStore(r string) *store.Table[HyperParameterTuningJob] { if b.hpTuningJobs[r] == nil { - b.hpTuningJobs[r] = make(map[string]*HyperParameterTuningJob) + b.hpTuningJobs[r] = store.Register( + b.registry, + "hpTuningJobs:"+r, + store.New(func(v *HyperParameterTuningJob) string { return v.HyperParameterTuningJobName }), + ) } return b.hpTuningJobs[r] } -func (b *InMemoryBackend) associationsStore(r string) map[string]*Association { +func (b *InMemoryBackend) associationsStore(r string) *store.Table[Association] { if b.associations[r] == nil { - b.associations[r] = make(map[string]*Association) + b.associations[r] = store.Register( + b.registry, + "associations:"+r, + store.New(func(v *Association) string { return associationKey(v.SourceArn, v.DestinationArn) }), + ) } return b.associations[r] } -func (b *InMemoryBackend) trialComponentAssociationsStore(r string) map[string]*TrialComponentAssociation { +func (b *InMemoryBackend) trialComponentAssociationsStore(r string) *store.Table[TrialComponentAssociation] { if b.trialComponentAssociations[r] == nil { - b.trialComponentAssociations[r] = make(map[string]*TrialComponentAssociation) + b.trialComponentAssociations[r] = store.Register( + b.registry, + "trialComponentAssociations:"+r, + store.New( + func(v *TrialComponentAssociation) string { return trialComponentKey(v.TrialName, v.TrialComponentName) }, + ), + ) } return b.trialComponentAssociations[r] } -func (b *InMemoryBackend) actionsStore(r string) map[string]*Action { +func (b *InMemoryBackend) actionsStore(r string) *store.Table[Action] { if b.actions[r] == nil { - b.actions[r] = make(map[string]*Action) + b.actions[r] = store.Register( + b.registry, + "actions:"+r, + store.New(func(v *Action) string { return v.ActionName }), + ) } return b.actions[r] } -func (b *InMemoryBackend) algorithmsStore(r string) map[string]*Algorithm { +func (b *InMemoryBackend) algorithmsStore(r string) *store.Table[Algorithm] { if b.algorithms[r] == nil { - b.algorithms[r] = make(map[string]*Algorithm) + b.algorithms[r] = store.Register( + b.registry, + "algorithms:"+r, + store.New(func(v *Algorithm) string { return v.AlgorithmName }), + ) } return b.algorithms[r] } -func (b *InMemoryBackend) clustersStore(r string) map[string]*Cluster { +func (b *InMemoryBackend) clustersStore(r string) *store.Table[Cluster] { if b.clusters[r] == nil { - b.clusters[r] = make(map[string]*Cluster) + b.clusters[r] = store.Register( + b.registry, + "clusters:"+r, + store.New(func(v *Cluster) string { return v.ClusterName }), + ) } return b.clusters[r] } -func (b *InMemoryBackend) modelPackagesStore(r string) map[string]*ModelPackage { +func (b *InMemoryBackend) modelPackagesStore(r string) *store.Table[ModelPackage] { if b.modelPackages[r] == nil { - b.modelPackages[r] = make(map[string]*ModelPackage) + b.modelPackages[r] = store.Register( + b.registry, + "modelPackages:"+r, + store.New(func(v *ModelPackage) string { return v.ModelPackageArn }), + ) } return b.modelPackages[r] } -func (b *InMemoryBackend) modelPackageGroupsStore(r string) map[string]*ModelPackageGroup { +func (b *InMemoryBackend) modelPackageGroupsStore(r string) *store.Table[ModelPackageGroup] { if b.modelPackageGroups[r] == nil { - b.modelPackageGroups[r] = make(map[string]*ModelPackageGroup) + b.modelPackageGroups[r] = store.Register( + b.registry, + "modelPackageGroups:"+r, + store.New(func(v *ModelPackageGroup) string { return v.ModelPackageGroupName }), + ) } return b.modelPackageGroups[r] } -func (b *InMemoryBackend) autoMLJobsStore(r string) map[string]*AutoMLJob { +func (b *InMemoryBackend) autoMLJobsStore(r string) *store.Table[AutoMLJob] { if b.autoMLJobs[r] == nil { - b.autoMLJobs[r] = make(map[string]*AutoMLJob) + b.autoMLJobs[r] = store.Register( + b.registry, + "autoMLJobs:"+r, + store.New(func(v *AutoMLJob) string { return v.AutoMLJobName }), + ) } return b.autoMLJobs[r] } -func (b *InMemoryBackend) codeRepositoriesStore(r string) map[string]*CodeRepository { +func (b *InMemoryBackend) codeRepositoriesStore(r string) *store.Table[CodeRepository] { if b.codeRepositories[r] == nil { - b.codeRepositories[r] = make(map[string]*CodeRepository) + b.codeRepositories[r] = store.Register( + b.registry, + "codeRepositories:"+r, + store.New(func(v *CodeRepository) string { return v.CodeRepositoryName }), + ) } return b.codeRepositories[r] } -func (b *InMemoryBackend) projectsStore(r string) map[string]*Project { +func (b *InMemoryBackend) projectsStore(r string) *store.Table[Project] { if b.projects[r] == nil { - b.projects[r] = make(map[string]*Project) + b.projects[r] = store.Register( + b.registry, + "projects:"+r, + store.New(func(v *Project) string { return v.ProjectName }), + ) } return b.projects[r] } -func (b *InMemoryBackend) spacesStore(r string) map[string]*Space { +func (b *InMemoryBackend) spacesStore(r string) *store.Table[Space] { if b.spaces[r] == nil { - b.spaces[r] = make(map[string]*Space) + b.spaces[r] = store.Register(b.registry, "spaces:"+r, store.New(func(v *Space) string { + return spaceKey(v.DomainID, v.SpaceName) + })) } return b.spaces[r] } -func (b *InMemoryBackend) smImagesStore(r string) map[string]*SMImage { +func (b *InMemoryBackend) smImagesStore(r string) *store.Table[SMImage] { if b.smImages[r] == nil { - b.smImages[r] = make(map[string]*SMImage) + b.smImages[r] = store.Register( + b.registry, + "smImages:"+r, + store.New(func(v *SMImage) string { return v.ImageName }), + ) } return b.smImages[r] @@ -849,163 +931,257 @@ func (b *InMemoryBackend) imageVersionCountsStore(r string) map[string]int { return b.imageVersionCounts[r] } -func (b *InMemoryBackend) compilationJobsStore(r string) map[string]*CompilationJob { +func (b *InMemoryBackend) compilationJobsStore(r string) *store.Table[CompilationJob] { if b.compilationJobs[r] == nil { - b.compilationJobs[r] = make(map[string]*CompilationJob) + b.compilationJobs[r] = store.Register( + b.registry, + "compilationJobs:"+r, + store.New(func(v *CompilationJob) string { return v.CompilationJobName }), + ) } return b.compilationJobs[r] } -func (b *InMemoryBackend) monitoringSchedulesStore(r string) map[string]*MonitoringSchedule { +func (b *InMemoryBackend) monitoringSchedulesStore(r string) *store.Table[MonitoringSchedule] { if b.monitoringSchedules[r] == nil { - b.monitoringSchedules[r] = make(map[string]*MonitoringSchedule) + b.monitoringSchedules[r] = store.Register( + b.registry, + "monitoringSchedules:"+r, + store.New(func(v *MonitoringSchedule) string { return v.MonitoringScheduleName }), + ) } return b.monitoringSchedules[r] } -func (b *InMemoryBackend) workteamsStore(r string) map[string]*Workteam { +func (b *InMemoryBackend) workteamsStore(r string) *store.Table[Workteam] { if b.workteams[r] == nil { - b.workteams[r] = make(map[string]*Workteam) + b.workteams[r] = store.Register( + b.registry, + "workteams:"+r, + store.New(func(v *Workteam) string { return v.WorkteamName }), + ) } return b.workteams[r] } -func (b *InMemoryBackend) dataQualityJobDefsStore(r string) map[string]*JobDefinition { +func (b *InMemoryBackend) dataQualityJobDefsStore(r string) *store.Table[JobDefinition] { if b.dataQualityJobDefs[r] == nil { - b.dataQualityJobDefs[r] = make(map[string]*JobDefinition) + b.dataQualityJobDefs[r] = store.Register( + b.registry, + "dataQualityJobDefs:"+r, + store.New(func(v *JobDefinition) string { return v.JobDefinitionName }), + ) } return b.dataQualityJobDefs[r] } -func (b *InMemoryBackend) modelBiasJobDefsStore(r string) map[string]*JobDefinition { +func (b *InMemoryBackend) modelBiasJobDefsStore(r string) *store.Table[JobDefinition] { if b.modelBiasJobDefs[r] == nil { - b.modelBiasJobDefs[r] = make(map[string]*JobDefinition) + b.modelBiasJobDefs[r] = store.Register( + b.registry, + "modelBiasJobDefs:"+r, + store.New(func(v *JobDefinition) string { return v.JobDefinitionName }), + ) } return b.modelBiasJobDefs[r] } -func (b *InMemoryBackend) modelQualityJobDefsStore(r string) map[string]*JobDefinition { +func (b *InMemoryBackend) modelQualityJobDefsStore(r string) *store.Table[JobDefinition] { if b.modelQualityJobDefs[r] == nil { - b.modelQualityJobDefs[r] = make(map[string]*JobDefinition) + b.modelQualityJobDefs[r] = store.Register( + b.registry, + "modelQualityJobDefs:"+r, + store.New(func(v *JobDefinition) string { return v.JobDefinitionName }), + ) } return b.modelQualityJobDefs[r] } -func (b *InMemoryBackend) modelExplainJobDefsStore(r string) map[string]*JobDefinition { +func (b *InMemoryBackend) modelExplainJobDefsStore(r string) *store.Table[JobDefinition] { if b.modelExplainJobDefs[r] == nil { - b.modelExplainJobDefs[r] = make(map[string]*JobDefinition) + b.modelExplainJobDefs[r] = store.Register( + b.registry, + "modelExplainJobDefs:"+r, + store.New(func(v *JobDefinition) string { return v.JobDefinitionName }), + ) } return b.modelExplainJobDefs[r] } -func (b *InMemoryBackend) monitoringExecutionsStore(r string) map[string]*MonitoringExecution { +func (b *InMemoryBackend) monitoringExecutionsStore(r string) *store.Table[MonitoringExecution] { if b.monitoringExecutions[r] == nil { - b.monitoringExecutions[r] = make(map[string]*MonitoringExecution) + b.monitoringExecutions[r] = store.Register( + b.registry, + "monitoringExecutions:"+r, + store.New( + func(v *MonitoringExecution) string { return v.MonitoringScheduleName + "|" + v.ProcessingJobArn }, + ), + ) } return b.monitoringExecutions[r] } -func (b *InMemoryBackend) humanTaskUisStore(r string) map[string]*HumanTaskUI { +func (b *InMemoryBackend) humanTaskUisStore(r string) *store.Table[HumanTaskUI] { if b.humanTaskUis[r] == nil { - b.humanTaskUis[r] = make(map[string]*HumanTaskUI) + b.humanTaskUis[r] = store.Register( + b.registry, + "humanTaskUis:"+r, + store.New(func(v *HumanTaskUI) string { return v.HumanTaskUIName }), + ) } return b.humanTaskUis[r] } -func (b *InMemoryBackend) workforcesStore(r string) map[string]*Workforce { +func (b *InMemoryBackend) workforcesStore(r string) *store.Table[Workforce] { if b.workforces[r] == nil { - b.workforces[r] = make(map[string]*Workforce) + b.workforces[r] = store.Register( + b.registry, + "workforces:"+r, + store.New(func(v *Workforce) string { return v.WorkforceName }), + ) } return b.workforces[r] } -func (b *InMemoryBackend) flowDefinitionsStore(r string) map[string]*FlowDefinition { +func (b *InMemoryBackend) flowDefinitionsStore(r string) *store.Table[FlowDefinition] { if b.flowDefinitions[r] == nil { - b.flowDefinitions[r] = make(map[string]*FlowDefinition) + b.flowDefinitions[r] = store.Register( + b.registry, + "flowDefinitions:"+r, + store.New(func(v *FlowDefinition) string { return v.FlowDefinitionName }), + ) } return b.flowDefinitions[r] } -func (b *InMemoryBackend) appImageConfigsStore(r string) map[string]*AppImageConfig { +func (b *InMemoryBackend) appImageConfigsStore(r string) *store.Table[AppImageConfig] { if b.appImageConfigs[r] == nil { - b.appImageConfigs[r] = make(map[string]*AppImageConfig) + b.appImageConfigs[r] = store.Register( + b.registry, + "appImageConfigs:"+r, + store.New(func(v *AppImageConfig) string { return v.AppImageConfigName }), + ) } return b.appImageConfigs[r] } -func (b *InMemoryBackend) inferenceExperimentsStore(r string) map[string]*InferenceExperiment { +func (b *InMemoryBackend) inferenceExperimentsStore(r string) *store.Table[InferenceExperiment] { if b.inferenceExperiments[r] == nil { - b.inferenceExperiments[r] = make(map[string]*InferenceExperiment) + b.inferenceExperiments[r] = store.Register( + b.registry, + "inferenceExperiments:"+r, + store.New(func(v *InferenceExperiment) string { return v.Name }), + ) } return b.inferenceExperiments[r] } -func (b *InMemoryBackend) mlflowTrackingServersStore(r string) map[string]*MlflowTrackingServer { +func (b *InMemoryBackend) mlflowTrackingServersStore(r string) *store.Table[MlflowTrackingServer] { if b.mlflowTrackingServers[r] == nil { - b.mlflowTrackingServers[r] = make(map[string]*MlflowTrackingServer) + b.mlflowTrackingServers[r] = store.Register( + b.registry, + "mlflowTrackingServers:"+r, + store.New(func(v *MlflowTrackingServer) string { return v.TrackingServerName }), + ) } return b.mlflowTrackingServers[r] } -func (b *InMemoryBackend) mlflowAppsStore(r string) map[string]*MlflowApp { +func (b *InMemoryBackend) mlflowAppsStore(r string) *store.Table[MlflowApp] { if b.mlflowApps[r] == nil { - b.mlflowApps[r] = make(map[string]*MlflowApp) + b.mlflowApps[r] = store.Register( + b.registry, + "mlflowApps:"+r, + store.New(func(v *MlflowApp) string { return v.Arn }), + ) } return b.mlflowApps[r] } -func (b *InMemoryBackend) modelCardsStore(r string) map[string]*ModelCard { +func (b *InMemoryBackend) modelCardsStore(r string) *store.Table[ModelCard] { if b.modelCards[r] == nil { - b.modelCards[r] = make(map[string]*ModelCard) + b.modelCards[r] = store.Register( + b.registry, + "modelCards:"+r, + store.New(func(v *ModelCard) string { return v.ModelCardName }), + ) } return b.modelCards[r] } -func (b *InMemoryBackend) optimizationJobsStore(r string) map[string]*OptimizationJob { +func (b *InMemoryBackend) optimizationJobsStore(r string) *store.Table[OptimizationJob] { if b.optimizationJobs[r] == nil { - b.optimizationJobs[r] = make(map[string]*OptimizationJob) + b.optimizationJobs[r] = store.Register( + b.registry, + "optimizationJobs:"+r, + store.New(func(v *OptimizationJob) string { return v.OptimizationJobName }), + ) } return b.optimizationJobs[r] } -func (b *InMemoryBackend) studioLifecycleConfigsStore(r string) map[string]*StudioLifecycleConfig { +func (b *InMemoryBackend) studioLifecycleConfigsStore(r string) *store.Table[StudioLifecycleConfig] { if b.studioLifecycleConfigs[r] == nil { - b.studioLifecycleConfigs[r] = make(map[string]*StudioLifecycleConfig) + b.studioLifecycleConfigs[r] = store.Register( + b.registry, + "studioLifecycleConfigs:"+r, + store.New(func(v *StudioLifecycleConfig) string { return v.StudioLifecycleConfigName }), + ) } return b.studioLifecycleConfigs[r] } -func (b *InMemoryBackend) partnerAppsStore(r string) map[string]*PartnerApp { +func (b *InMemoryBackend) partnerAppsStore(r string) *store.Table[PartnerApp] { if b.partnerApps[r] == nil { - b.partnerApps[r] = make(map[string]*PartnerApp) + b.partnerApps[r] = store.Register( + b.registry, + "partnerApps:"+r, + store.New(func(v *PartnerApp) string { return v.Arn }), + ) } return b.partnerApps[r] } -func (b *InMemoryBackend) trainingPlansStore(r string) map[string]*TrainingPlan { +func (b *InMemoryBackend) trainingPlansStore(r string) *store.Table[TrainingPlan] { if b.trainingPlans[r] == nil { - b.trainingPlans[r] = make(map[string]*TrainingPlan) + b.trainingPlans[r] = store.Register( + b.registry, + "trainingPlans:"+r, + store.New(func(v *TrainingPlan) string { return v.TrainingPlanName }), + ) } return b.trainingPlans[r] } -func (b *InMemoryBackend) reservedCapacitiesStore(r string) map[string]*ReservedCapacity { +func (b *InMemoryBackend) reservedCapacitiesStore(r string) *store.Table[ReservedCapacity] { if b.reservedCapacities[r] == nil { - b.reservedCapacities[r] = make(map[string]*ReservedCapacity) + b.reservedCapacities[r] = store.Register( + b.registry, + "reservedCapacities:"+r, + store.New(func(v *ReservedCapacity) string { return v.ReservedCapacityArn }), + ) } return b.reservedCapacities[r] } -func (b *InMemoryBackend) trainingPlanExtensionOfferingsStore(r string) map[string]*pendingTrainingPlanExtension { +func (b *InMemoryBackend) trainingPlanExtensionOfferingsStore(r string) *store.Table[pendingTrainingPlanExtension] { if b.trainingPlanExtensionOfferings[r] == nil { - b.trainingPlanExtensionOfferings[r] = make(map[string]*pendingTrainingPlanExtension) + b.trainingPlanExtensionOfferings[r] = store.Register( + b.registry, + "trainingPlanExtensionOfferings:"+r, + store.New(func(v *pendingTrainingPlanExtension) string { return v.ID }), + ) } return b.trainingPlanExtensionOfferings[r] } -func (b *InMemoryBackend) modelCardExportJobsStore(r string) map[string]*ModelCardExportJob { +func (b *InMemoryBackend) modelCardExportJobsStore(r string) *store.Table[ModelCardExportJob] { if b.modelCardExportJobs[r] == nil { - b.modelCardExportJobs[r] = make(map[string]*ModelCardExportJob) + b.modelCardExportJobs[r] = store.Register( + b.registry, + "modelCardExportJobs:"+r, + store.New(func(v *ModelCardExportJob) string { return v.ModelCardExportJobArn }), + ) } return b.modelCardExportJobs[r] @@ -1018,9 +1194,9 @@ func (b *InMemoryBackend) modelCardExportJobsStore(r string) map[string]*ModelCa // ModelCard export job maps, plus the ModelPackageGroup policy / Servicecatalog // portfolio / Pipeline version state introduced in a later de-stubbing round. func (b *InMemoryBackend) initTrainingPlanExtMaps() { - b.reservedCapacities = make(map[string]map[string]*ReservedCapacity) - b.trainingPlanExtensionOfferings = make(map[string]map[string]*pendingTrainingPlanExtension) - b.modelCardExportJobs = make(map[string]map[string]*ModelCardExportJob) + b.reservedCapacities = make(map[string]*store.Table[ReservedCapacity]) + b.trainingPlanExtensionOfferings = make(map[string]*store.Table[pendingTrainingPlanExtension]) + b.modelCardExportJobs = make(map[string]*store.Table[ModelCardExportJob]) b.pipelineVersions = make(map[string]map[string][]*PipelineVersion) b.servicecatalogPortfolioEnabled = make(map[string]bool) } @@ -1108,163 +1284,250 @@ func (b *InMemoryBackend) transformJobARNIndexStore(r string) map[string]string return b.transformJobARNIndex[r] } -func (b *InMemoryBackend) domainsStore(r string) map[string]*Domain { +func (b *InMemoryBackend) domainsStore(r string) *store.Table[Domain] { if b.domains[r] == nil { - b.domains[r] = make(map[string]*Domain) + b.domains[r] = store.Register(b.registry, "domains:"+r, store.New(func(v *Domain) string { return v.DomainID })) } return b.domains[r] } -func (b *InMemoryBackend) userProfilesStore(r string) map[userProfileKey]*UserProfile { +func (b *InMemoryBackend) userProfilesStore(r string) *store.Table[UserProfile] { if b.userProfiles[r] == nil { - b.userProfiles[r] = make(map[userProfileKey]*UserProfile) + b.userProfiles[r] = store.Register(b.registry, "userProfiles:"+r, store.New(func(v *UserProfile) string { + return userProfileKeyString(userProfileKey{DomainID: v.DomainID, UserProfileName: v.UserProfileName}) + })) } return b.userProfiles[r] } -func (b *InMemoryBackend) appsStore(r string) map[appKey]*App { +func (b *InMemoryBackend) appsStore(r string) *store.Table[App] { if b.apps[r] == nil { - b.apps[r] = make(map[appKey]*App) + b.apps[r] = store.Register(b.registry, "apps:"+r, store.New(func(v *App) string { + return appKeyString( + appKey{ + DomainID: v.DomainID, + UserProfileName: v.UserProfileName, + AppType: v.AppType, + AppName: v.AppName, + }, + ) + })) } return b.apps[r] } -func (b *InMemoryBackend) featureGroupsStore(r string) map[string]*FeatureGroup { +func (b *InMemoryBackend) featureGroupsStore(r string) *store.Table[FeatureGroup] { if b.featureGroups[r] == nil { - b.featureGroups[r] = make(map[string]*FeatureGroup) + b.featureGroups[r] = store.Register( + b.registry, + "featureGroups:"+r, + store.New(func(v *FeatureGroup) string { return v.FeatureGroupName }), + ) } return b.featureGroups[r] } -func (b *InMemoryBackend) featureRecordsStore(r string) map[string]*FeatureRecord { +func (b *InMemoryBackend) featureRecordsStore(r string) *store.Table[FeatureRecord] { if b.featureRecords[r] == nil { - b.featureRecords[r] = make(map[string]*FeatureRecord) + b.featureRecords[r] = store.Register( + b.registry, + "featureRecords:"+r, + store.New(func(v *FeatureRecord) string { return v.Key }), + ) } return b.featureRecords[r] } -func (b *InMemoryBackend) featureMetadataStore(r string) map[string]*FeatureMetadata { +func (b *InMemoryBackend) featureMetadataStore(r string) *store.Table[FeatureMetadata] { if b.featureMetadata[r] == nil { - b.featureMetadata[r] = make(map[string]*FeatureMetadata) + b.featureMetadata[r] = store.Register( + b.registry, + "featureMetadata:"+r, + store.New(func(v *FeatureMetadata) string { return featureMetaKey(v.GroupName, v.FeatureName) }), + ) } return b.featureMetadata[r] } -func (b *InMemoryBackend) pipelinesStore(r string) map[string]*Pipeline { +func (b *InMemoryBackend) pipelinesStore(r string) *store.Table[Pipeline] { if b.pipelines[r] == nil { - b.pipelines[r] = make(map[string]*Pipeline) + b.pipelines[r] = store.Register( + b.registry, + "pipelines:"+r, + store.New(func(v *Pipeline) string { return v.PipelineName }), + ) } return b.pipelines[r] } -func (b *InMemoryBackend) pipelineExecutionsStore(r string) map[string]*PipelineExecution { +func (b *InMemoryBackend) pipelineExecutionsStore(r string) *store.Table[PipelineExecution] { if b.pipelineExecutions[r] == nil { - b.pipelineExecutions[r] = make(map[string]*PipelineExecution) + b.pipelineExecutions[r] = store.Register( + b.registry, + "pipelineExecutions:"+r, + store.New(func(v *PipelineExecution) string { return v.PipelineExecutionArn }), + ) } return b.pipelineExecutions[r] } -func (b *InMemoryBackend) pipelineExecStepsStore(r string) map[string]*PipelineExecutionStep { +func (b *InMemoryBackend) pipelineExecStepsStore(r string) *store.Table[PipelineExecutionStep] { if b.pipelineExecSteps[r] == nil { - b.pipelineExecSteps[r] = make(map[string]*PipelineExecutionStep) + b.pipelineExecSteps[r] = store.Register( + b.registry, + "pipelineExecSteps:"+r, + store.New( + func(v *PipelineExecutionStep) string { return pipelineExecutionStepsKey(v.ExecutionArn, v.StepName) }, + ), + ) } return b.pipelineExecSteps[r] } -func (b *InMemoryBackend) experimentsStore(r string) map[string]*Experiment { +func (b *InMemoryBackend) experimentsStore(r string) *store.Table[Experiment] { if b.experiments[r] == nil { - b.experiments[r] = make(map[string]*Experiment) + b.experiments[r] = store.Register( + b.registry, + "experiments:"+r, + store.New(func(v *Experiment) string { return v.ExperimentName }), + ) } return b.experiments[r] } -func (b *InMemoryBackend) trialsStore(r string) map[string]*Trial { +func (b *InMemoryBackend) trialsStore(r string) *store.Table[Trial] { if b.trials[r] == nil { - b.trials[r] = make(map[string]*Trial) + b.trials[r] = store.Register(b.registry, "trials:"+r, store.New(func(v *Trial) string { return v.TrialName })) } return b.trials[r] } -func (b *InMemoryBackend) trialComponentsStore(r string) map[string]*TrialComponent { +func (b *InMemoryBackend) trialComponentsStore(r string) *store.Table[TrialComponent] { if b.trialComponents[r] == nil { - b.trialComponents[r] = make(map[string]*TrialComponent) + b.trialComponents[r] = store.Register( + b.registry, + "trialComponents:"+r, + store.New(func(v *TrialComponent) string { return v.TrialComponentName }), + ) } return b.trialComponents[r] } -func (b *InMemoryBackend) notebookLifecycleConfigsStore(r string) map[string]*NotebookInstanceLifecycleConfig { +func (b *InMemoryBackend) notebookLifecycleConfigsStore(r string) *store.Table[NotebookInstanceLifecycleConfig] { if b.notebookLifecycleConfigs[r] == nil { - b.notebookLifecycleConfigs[r] = make(map[string]*NotebookInstanceLifecycleConfig) + b.notebookLifecycleConfigs[r] = store.Register( + b.registry, + "notebookLifecycleConfigs:"+r, + store.New(func(v *NotebookInstanceLifecycleConfig) string { return v.Name }), + ) } return b.notebookLifecycleConfigs[r] } -func (b *InMemoryBackend) processingJobsStore(r string) map[string]*ProcessingJob { +func (b *InMemoryBackend) processingJobsStore(r string) *store.Table[ProcessingJob] { if b.processingJobs[r] == nil { - b.processingJobs[r] = make(map[string]*ProcessingJob) + b.processingJobs[r] = store.Register( + b.registry, + "processingJobs:"+r, + store.New(func(v *ProcessingJob) string { return v.ProcessingJobName }), + ) } return b.processingJobs[r] } -func (b *InMemoryBackend) transformJobsStore(r string) map[string]*TransformJob { +func (b *InMemoryBackend) transformJobsStore(r string) *store.Table[TransformJob] { if b.transformJobs[r] == nil { - b.transformJobs[r] = make(map[string]*TransformJob) + b.transformJobs[r] = store.Register( + b.registry, + "transformJobs:"+r, + store.New(func(v *TransformJob) string { return v.TransformJobName }), + ) } return b.transformJobs[r] } -func (b *InMemoryBackend) edgePackagingJobsStore(r string) map[string]*EdgePackagingJob { +func (b *InMemoryBackend) edgePackagingJobsStore(r string) *store.Table[EdgePackagingJob] { if b.edgePackagingJobs[r] == nil { - b.edgePackagingJobs[r] = make(map[string]*EdgePackagingJob) + b.edgePackagingJobs[r] = store.Register( + b.registry, + "edgePackagingJobs:"+r, + store.New(func(v *EdgePackagingJob) string { return v.EdgePackagingJobName }), + ) } return b.edgePackagingJobs[r] } -func (b *InMemoryBackend) edgeDeploymentPlansStore(r string) map[string]*EdgeDeploymentPlan { +func (b *InMemoryBackend) edgeDeploymentPlansStore(r string) *store.Table[EdgeDeploymentPlan] { if b.edgeDeploymentPlans[r] == nil { - b.edgeDeploymentPlans[r] = make(map[string]*EdgeDeploymentPlan) + b.edgeDeploymentPlans[r] = store.Register( + b.registry, + "edgeDeploymentPlans:"+r, + store.New(func(v *EdgeDeploymentPlan) string { return v.EdgeDeploymentPlanName }), + ) } return b.edgeDeploymentPlans[r] } -func (b *InMemoryBackend) inferenceRecommendationsJobsStore(r string) map[string]*InferenceRecommendationsJob { +func (b *InMemoryBackend) inferenceRecommendationsJobsStore(r string) *store.Table[InferenceRecommendationsJob] { if b.inferenceRecommendationsJobs[r] == nil { - b.inferenceRecommendationsJobs[r] = make(map[string]*InferenceRecommendationsJob) + b.inferenceRecommendationsJobs[r] = store.Register( + b.registry, + "inferenceRecommendationsJobs:"+r, + store.New(func(v *InferenceRecommendationsJob) string { return v.JobName }), + ) } return b.inferenceRecommendationsJobs[r] } -func (b *InMemoryBackend) deviceFleetsStore(r string) map[string]*DeviceFleet { +func (b *InMemoryBackend) deviceFleetsStore(r string) *store.Table[DeviceFleet] { if b.deviceFleets[r] == nil { - b.deviceFleets[r] = make(map[string]*DeviceFleet) + b.deviceFleets[r] = store.Register( + b.registry, + "deviceFleets:"+r, + store.New(func(v *DeviceFleet) string { return v.DeviceFleetName }), + ) } return b.deviceFleets[r] } -func (b *InMemoryBackend) devicesStore(r string) map[deviceKey]*Device { +func (b *InMemoryBackend) devicesStore(r string) *store.Table[Device] { if b.devices[r] == nil { - b.devices[r] = make(map[deviceKey]*Device) + b.devices[r] = store.Register(b.registry, "devices:"+r, store.New(func(v *Device) string { + return deviceKeyString(deviceKey{fleetName: v.DeviceFleetName, deviceName: v.DeviceName}) + })) } return b.devices[r] } -func (b *InMemoryBackend) inferenceComponentsStore(r string) map[string]*InferenceComponent { +func (b *InMemoryBackend) inferenceComponentsStore(r string) *store.Table[InferenceComponent] { if b.inferenceComponents[r] == nil { - b.inferenceComponents[r] = make(map[string]*InferenceComponent) + b.inferenceComponents[r] = store.Register( + b.registry, + "inferenceComponents:"+r, + store.New(func(v *InferenceComponent) string { return v.InferenceComponentName }), + ) } return b.inferenceComponents[r] } -func (b *InMemoryBackend) clusterSchedulerConfigsStore(r string) map[string]*ClusterSchedulerConfig { +func (b *InMemoryBackend) clusterSchedulerConfigsStore(r string) *store.Table[ClusterSchedulerConfig] { if b.clusterSchedulerConfigs[r] == nil { - b.clusterSchedulerConfigs[r] = make(map[string]*ClusterSchedulerConfig) + b.clusterSchedulerConfigs[r] = store.Register( + b.registry, + "clusterSchedulerConfigs:"+r, + store.New(func(v *ClusterSchedulerConfig) string { return v.ClusterSchedulerConfigName }), + ) } return b.clusterSchedulerConfigs[r] } -func (b *InMemoryBackend) computeQuotasStore(r string) map[string]*ComputeQuota { +func (b *InMemoryBackend) computeQuotasStore(r string) *store.Table[ComputeQuota] { if b.computeQuotas[r] == nil { - b.computeQuotas[r] = make(map[string]*ComputeQuota) + b.computeQuotas[r] = store.Register( + b.registry, + "computeQuotas:"+r, + store.New(func(v *ComputeQuota) string { return v.ComputeQuotaName }), + ) } return b.computeQuotas[r] @@ -1277,51 +1540,57 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - b.models = make(map[string]map[string]*Model) - b.endpointConfigs = make(map[string]map[string]*EndpointConfig) - b.endpoints = make(map[string]map[string]*Endpoint) - b.trainingJobs = make(map[string]map[string]*TrainingJob) - b.notebooks = make(map[string]map[string]*NotebookInstance) - b.hpTuningJobs = make(map[string]map[string]*HyperParameterTuningJob) - b.associations = make(map[string]map[string]*Association) - b.trialComponentAssociations = make(map[string]map[string]*TrialComponentAssociation) - b.actions = make(map[string]map[string]*Action) - b.artifacts = make(map[string]map[string]*Artifact) - b.contexts = make(map[string]map[string]*Context) - b.algorithms = make(map[string]map[string]*Algorithm) - b.clusters = make(map[string]map[string]*Cluster) - b.modelPackages = make(map[string]map[string]*ModelPackage) - b.modelPackageGroups = make(map[string]map[string]*ModelPackageGroup) - b.autoMLJobs = make(map[string]map[string]*AutoMLJob) - b.codeRepositories = make(map[string]map[string]*CodeRepository) - b.projects = make(map[string]map[string]*Project) - b.spaces = make(map[string]map[string]*Space) - b.smImages = make(map[string]map[string]*SMImage) + // Fresh registry: every xxxStore(r) helper below lazily re-registers a new + // per-region store.Table under the same "field:region" name on next use, + // which would panic against the old registry (duplicate name) since the + // old one is never otherwise cleared. + b.registry = store.NewRegistry() + + b.models = make(map[string]*store.Table[Model]) + b.endpointConfigs = make(map[string]*store.Table[EndpointConfig]) + b.endpoints = make(map[string]*store.Table[Endpoint]) + b.trainingJobs = make(map[string]*store.Table[TrainingJob]) + b.notebooks = make(map[string]*store.Table[NotebookInstance]) + b.hpTuningJobs = make(map[string]*store.Table[HyperParameterTuningJob]) + b.associations = make(map[string]*store.Table[Association]) + b.trialComponentAssociations = make(map[string]*store.Table[TrialComponentAssociation]) + b.actions = make(map[string]*store.Table[Action]) + b.artifacts = make(map[string]*store.Table[Artifact]) + b.contexts = make(map[string]*store.Table[Context]) + b.algorithms = make(map[string]*store.Table[Algorithm]) + b.clusters = make(map[string]*store.Table[Cluster]) + b.modelPackages = make(map[string]*store.Table[ModelPackage]) + b.modelPackageGroups = make(map[string]*store.Table[ModelPackageGroup]) + b.autoMLJobs = make(map[string]*store.Table[AutoMLJob]) + b.codeRepositories = make(map[string]*store.Table[CodeRepository]) + b.projects = make(map[string]*store.Table[Project]) + b.spaces = make(map[string]*store.Table[Space]) + b.smImages = make(map[string]*store.Table[SMImage]) b.imageVersions = make(map[string]map[string]map[int]*ImageVersion) b.imageVersionCounts = make(map[string]map[string]int) - b.compilationJobs = make(map[string]map[string]*CompilationJob) - b.monitoringSchedules = make(map[string]map[string]*MonitoringSchedule) - b.workteams = make(map[string]map[string]*Workteam) - b.labelingJobs = make(map[string]map[string]*LabelingJob) - b.dataQualityJobDefs = make(map[string]map[string]*JobDefinition) - b.modelBiasJobDefs = make(map[string]map[string]*JobDefinition) - b.modelQualityJobDefs = make(map[string]map[string]*JobDefinition) - b.modelExplainJobDefs = make(map[string]map[string]*JobDefinition) - b.monitoringAlerts = make(map[string]map[string]map[string]*MonitoringAlert) + b.compilationJobs = make(map[string]*store.Table[CompilationJob]) + b.monitoringSchedules = make(map[string]*store.Table[MonitoringSchedule]) + b.workteams = make(map[string]*store.Table[Workteam]) + b.labelingJobs = make(map[string]*store.Table[LabelingJob]) + b.dataQualityJobDefs = make(map[string]*store.Table[JobDefinition]) + b.modelBiasJobDefs = make(map[string]*store.Table[JobDefinition]) + b.modelQualityJobDefs = make(map[string]*store.Table[JobDefinition]) + b.modelExplainJobDefs = make(map[string]*store.Table[JobDefinition]) + b.monitoringAlerts = make(map[string]*store.Table[MonitoringAlert]) b.monitoringAlertHistory = make(map[string][]*MonitoringAlertHistoryEntry) - b.monitoringExecutions = make(map[string]map[string]*MonitoringExecution) - b.humanTaskUis = make(map[string]map[string]*HumanTaskUI) - b.workforces = make(map[string]map[string]*Workforce) - b.flowDefinitions = make(map[string]map[string]*FlowDefinition) - b.appImageConfigs = make(map[string]map[string]*AppImageConfig) - b.inferenceExperiments = make(map[string]map[string]*InferenceExperiment) - b.mlflowTrackingServers = make(map[string]map[string]*MlflowTrackingServer) - b.mlflowApps = make(map[string]map[string]*MlflowApp) - b.modelCards = make(map[string]map[string]*ModelCard) - b.optimizationJobs = make(map[string]map[string]*OptimizationJob) - b.studioLifecycleConfigs = make(map[string]map[string]*StudioLifecycleConfig) - b.partnerApps = make(map[string]map[string]*PartnerApp) - b.trainingPlans = make(map[string]map[string]*TrainingPlan) + b.monitoringExecutions = make(map[string]*store.Table[MonitoringExecution]) + b.humanTaskUis = make(map[string]*store.Table[HumanTaskUI]) + b.workforces = make(map[string]*store.Table[Workforce]) + b.flowDefinitions = make(map[string]*store.Table[FlowDefinition]) + b.appImageConfigs = make(map[string]*store.Table[AppImageConfig]) + b.inferenceExperiments = make(map[string]*store.Table[InferenceExperiment]) + b.mlflowTrackingServers = make(map[string]*store.Table[MlflowTrackingServer]) + b.mlflowApps = make(map[string]*store.Table[MlflowApp]) + b.modelCards = make(map[string]*store.Table[ModelCard]) + b.optimizationJobs = make(map[string]*store.Table[OptimizationJob]) + b.studioLifecycleConfigs = make(map[string]*store.Table[StudioLifecycleConfig]) + b.partnerApps = make(map[string]*store.Table[PartnerApp]) + b.trainingPlans = make(map[string]*store.Table[TrainingPlan]) b.initTrainingPlanExtMaps() b.modelARNIndex = make(map[string]map[string]string) b.endpointConfigARNIndex = make(map[string]map[string]string) @@ -1336,31 +1605,31 @@ func (b *InMemoryBackend) Reset() { b.modelPackageARNIndex = make(map[string]map[string]string) b.processingJobARNIndex = make(map[string]map[string]string) b.transformJobARNIndex = make(map[string]map[string]string) - b.domains = make(map[string]map[string]*Domain) - b.userProfiles = make(map[string]map[userProfileKey]*UserProfile) - b.apps = make(map[string]map[appKey]*App) - b.featureGroups = make(map[string]map[string]*FeatureGroup) - b.featureRecords = make(map[string]map[string]*FeatureRecord) - b.featureMetadata = make(map[string]map[string]*FeatureMetadata) - b.pipelines = make(map[string]map[string]*Pipeline) - b.pipelineExecutions = make(map[string]map[string]*PipelineExecution) - b.pipelineExecSteps = make(map[string]map[string]*PipelineExecutionStep) - b.experiments = make(map[string]map[string]*Experiment) - b.trials = make(map[string]map[string]*Trial) - b.trialComponents = make(map[string]map[string]*TrialComponent) - b.notebookLifecycleConfigs = make(map[string]map[string]*NotebookInstanceLifecycleConfig) - b.processingJobs = make(map[string]map[string]*ProcessingJob) - b.transformJobs = make(map[string]map[string]*TransformJob) - b.edgePackagingJobs = make(map[string]map[string]*EdgePackagingJob) - b.edgeDeploymentPlans = make(map[string]map[string]*EdgeDeploymentPlan) - b.inferenceRecommendationsJobs = make(map[string]map[string]*InferenceRecommendationsJob) - b.deviceFleets = make(map[string]map[string]*DeviceFleet) - b.devices = make(map[string]map[deviceKey]*Device) - b.inferenceComponents = make(map[string]map[string]*InferenceComponent) - b.clusterSchedulerConfigs = make(map[string]map[string]*ClusterSchedulerConfig) - b.computeQuotas = make(map[string]map[string]*ComputeQuota) - b.hubs = make(map[string]map[string]*Hub) - b.hubContents = make(map[string]map[hubContentKey]*HubContent) + b.domains = make(map[string]*store.Table[Domain]) + b.userProfiles = make(map[string]*store.Table[UserProfile]) + b.apps = make(map[string]*store.Table[App]) + b.featureGroups = make(map[string]*store.Table[FeatureGroup]) + b.featureRecords = make(map[string]*store.Table[FeatureRecord]) + b.featureMetadata = make(map[string]*store.Table[FeatureMetadata]) + b.pipelines = make(map[string]*store.Table[Pipeline]) + b.pipelineExecutions = make(map[string]*store.Table[PipelineExecution]) + b.pipelineExecSteps = make(map[string]*store.Table[PipelineExecutionStep]) + b.experiments = make(map[string]*store.Table[Experiment]) + b.trials = make(map[string]*store.Table[Trial]) + b.trialComponents = make(map[string]*store.Table[TrialComponent]) + b.notebookLifecycleConfigs = make(map[string]*store.Table[NotebookInstanceLifecycleConfig]) + b.processingJobs = make(map[string]*store.Table[ProcessingJob]) + b.transformJobs = make(map[string]*store.Table[TransformJob]) + b.edgePackagingJobs = make(map[string]*store.Table[EdgePackagingJob]) + b.edgeDeploymentPlans = make(map[string]*store.Table[EdgeDeploymentPlan]) + b.inferenceRecommendationsJobs = make(map[string]*store.Table[InferenceRecommendationsJob]) + b.deviceFleets = make(map[string]*store.Table[DeviceFleet]) + b.devices = make(map[string]*store.Table[Device]) + b.inferenceComponents = make(map[string]*store.Table[InferenceComponent]) + b.clusterSchedulerConfigs = make(map[string]*store.Table[ClusterSchedulerConfig]) + b.computeQuotas = make(map[string]*store.Table[ComputeQuota]) + b.hubs = make(map[string]*store.Table[Hub]) + b.hubContents = make(map[string]*store.Table[HubContent]) // Cancel pending goroutines and start fresh lifecycle context. b.resetLifecycleContext() } @@ -1380,7 +1649,7 @@ func (b *InMemoryBackend) CreateModel( region := getRegion(ctx, b.region) models := b.modelsStore(region) - if _, ok := models[name]; ok { + if _, ok := models.Get(name); ok { return nil, fmt.Errorf("%w: model %s already exists", ErrModelAlreadyExists, name) } @@ -1408,7 +1677,7 @@ func (b *InMemoryBackend) CreateModel( CreationTime: time.Now(), Tags: mergeTags(nil, tags), } - models[name] = m + models.Put(m) b.modelARNIndexStore(region)[modelARN] = name return cloneModel(m), nil @@ -1421,7 +1690,7 @@ func (b *InMemoryBackend) DescribeModel(ctx context.Context, name string) (*Mode region := getRegion(ctx, b.region) - m, ok := b.modelsStore(region)[name] + m, ok := b.modelsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: could not find model %q", ErrModelNotFound, name) } @@ -1448,14 +1717,14 @@ func (b *InMemoryBackend) DeleteModel(ctx context.Context, name string) error { region := getRegion(ctx, b.region) models := b.modelsStore(region) - m, ok := models[name] + m, ok := models.Get(name) if !ok { return fmt.Errorf("%w: could not find model %q", ErrModelNotFound, name) } arnIndex := b.modelARNIndexStore(region) delete(arnIndex, m.ModelARN) - delete(models, name) + models.Delete(name) return nil } @@ -1474,7 +1743,7 @@ func (b *InMemoryBackend) SetModelExtras( region := getRegion(ctx, b.region) - m, ok := b.modelsStore(region)[name] + m, ok := b.modelsStore(region).Get(name) if !ok { return fmt.Errorf("%w: could not find model %q", ErrModelNotFound, name) } @@ -1509,7 +1778,7 @@ func (b *InMemoryBackend) CreateEndpointConfig( region := getRegion(ctx, b.region) ecStore := b.endpointConfigsStore(region) - if _, ok := ecStore[name]; ok { + if _, ok := ecStore.Get(name); ok { return nil, fmt.Errorf( "%w: endpoint config %s already exists", ErrEndpointConfigAlreadyExists, @@ -1529,7 +1798,7 @@ func (b *InMemoryBackend) CreateEndpointConfig( CreationTime: time.Now(), Tags: mergeTags(nil, tags), } - ecStore[name] = ec + ecStore.Put(ec) b.endpointConfigARNIndexStore(region)[configARN] = name return cloneEndpointConfig(ec), nil @@ -1542,7 +1811,7 @@ func (b *InMemoryBackend) DescribeEndpointConfig(ctx context.Context, name strin region := getRegion(ctx, b.region) - ec, ok := b.endpointConfigsStore(region)[name] + ec, ok := b.endpointConfigsStore(region).Get(name) if !ok { return nil, fmt.Errorf( "%w: could not find endpoint configuration %q", @@ -1573,7 +1842,7 @@ func (b *InMemoryBackend) DeleteEndpointConfig(ctx context.Context, name string) region := getRegion(ctx, b.region) ecStore := b.endpointConfigsStore(region) - ec, ok := ecStore[name] + ec, ok := ecStore.Get(name) if !ok { return fmt.Errorf( "%w: could not find endpoint configuration %q", @@ -1584,7 +1853,7 @@ func (b *InMemoryBackend) DeleteEndpointConfig(ctx context.Context, name string) arnIndex := b.endpointConfigARNIndexStore(region) delete(arnIndex, ec.EndpointConfigARN) - delete(ecStore, name) + ecStore.Delete(name) return nil } @@ -1607,7 +1876,7 @@ func (b *InMemoryBackend) SetEndpointConfigExtras( region := getRegion(ctx, b.region) - ec, ok := b.endpointConfigsStore(region)[name] + ec, ok := b.endpointConfigsStore(region).Get(name) if !ok { return fmt.Errorf( "%w: could not find endpoint configuration %q", @@ -1656,70 +1925,70 @@ func (b *InMemoryBackend) AddTags(ctx context.Context, resourceARN string, tags region := getRegion(ctx, b.region) if name, ok := b.modelARNIndexStore(region)[resourceARN]; ok { - m := b.modelsStore(region)[name] + m := tableGet(b.modelsStore(region), name) m.Tags = mergeTags(m.Tags, tags) return nil } if name, ok := b.endpointConfigARNIndexStore(region)[resourceARN]; ok { - ec := b.endpointConfigsStore(region)[name] + ec := tableGet(b.endpointConfigsStore(region), name) ec.Tags = mergeTags(ec.Tags, tags) return nil } if name, ok := b.actionARNIndexStore(region)[resourceARN]; ok { - a := b.actionsStore(region)[name] + a := tableGet(b.actionsStore(region), name) a.Tags = mergeTags(a.Tags, tags) return nil } if name, ok := b.algorithmARNIndexStore(region)[resourceARN]; ok { - al := b.algorithmsStore(region)[name] + al := tableGet(b.algorithmsStore(region), name) al.Tags = mergeTags(al.Tags, tags) return nil } if _, ok := b.modelPackageARNIndexStore(region)[resourceARN]; ok { - mp := b.modelPackagesStore(region)[resourceARN] + mp := tableGet(b.modelPackagesStore(region), resourceARN) mp.Tags = mergeTags(mp.Tags, tags) return nil } if name, ok := b.endpointARNIndexStore(region)[resourceARN]; ok { - ep := b.endpointsStore(region)[name] + ep := tableGet(b.endpointsStore(region), name) ep.Tags = mergeTags(ep.Tags, tags) return nil } if name, ok := b.trainingJobARNIndexStore(region)[resourceARN]; ok { - tj := b.trainingJobsStore(region)[name] + tj := tableGet(b.trainingJobsStore(region), name) tj.Tags = mergeTags(tj.Tags, tags) return nil } if name, ok := b.notebookARNIndexStore(region)[resourceARN]; ok { - nb := b.notebooksStore(region)[name] + nb := tableGet(b.notebooksStore(region), name) nb.Tags = mergeTags(nb.Tags, tags) return nil } if name, ok := b.hpTuningJobARNIndexStore(region)[resourceARN]; ok { - j := b.hpTuningJobsStore(region)[name] + j := tableGet(b.hpTuningJobsStore(region), name) j.Tags = mergeTags(j.Tags, tags) return nil } if name, ok := b.processingJobARNIndexStore(region)[resourceARN]; ok { - if pj, found := b.processingJobsStore(region)[name]; found { + if pj, found := b.processingJobsStore(region).Get(name); found { pj.Tags = mergeTags(pj.Tags, tags) return nil @@ -1751,39 +2020,39 @@ func (b *InMemoryBackend) ListTags(ctx context.Context, resourceARN string) (map // Must be called with b.mu held. Returns nil if the resource is not found. func (b *InMemoryBackend) findTagMapLocked(resourceARN string, region string) *map[string]string { if name, ok := b.modelARNIndexStore(region)[resourceARN]; ok { - return &b.modelsStore(region)[name].Tags + return &tableGet(b.modelsStore(region), name).Tags } if name, ok := b.endpointConfigARNIndexStore(region)[resourceARN]; ok { - return &b.endpointConfigsStore(region)[name].Tags + return &tableGet(b.endpointConfigsStore(region), name).Tags } if name, ok := b.actionARNIndexStore(region)[resourceARN]; ok { - return &b.actionsStore(region)[name].Tags + return &tableGet(b.actionsStore(region), name).Tags } if name, ok := b.algorithmARNIndexStore(region)[resourceARN]; ok { - return &b.algorithmsStore(region)[name].Tags + return &tableGet(b.algorithmsStore(region), name).Tags } if _, ok := b.modelPackageARNIndexStore(region)[resourceARN]; ok { - return &b.modelPackagesStore(region)[resourceARN].Tags + return &tableGet(b.modelPackagesStore(region), resourceARN).Tags } if name, ok := b.endpointARNIndexStore(region)[resourceARN]; ok { - return &b.endpointsStore(region)[name].Tags + return &tableGet(b.endpointsStore(region), name).Tags } if name, ok := b.trainingJobARNIndexStore(region)[resourceARN]; ok { - return &b.trainingJobsStore(region)[name].Tags + return &tableGet(b.trainingJobsStore(region), name).Tags } if name, ok := b.notebookARNIndexStore(region)[resourceARN]; ok { - return &b.notebooksStore(region)[name].Tags + return &tableGet(b.notebooksStore(region), name).Tags } if name, ok := b.hpTuningJobARNIndexStore(region)[resourceARN]; ok { - return &b.hpTuningJobsStore(region)[name].Tags + return &tableGet(b.hpTuningJobsStore(region), name).Tags } if tags := b.findTagMapIndexedExtraLocked(resourceARN, region); tags != nil { @@ -1798,19 +2067,19 @@ func (b *InMemoryBackend) findTagMapLocked(resourceARN string, region string) *m // findTagMapLocked to keep it within cyclomatic-complexity limits. func (b *InMemoryBackend) findTagMapIndexedExtraLocked(resourceARN string, region string) *map[string]string { if name, ok := b.processingJobARNIndexStore(region)[resourceARN]; ok { - if pj, found := b.processingJobsStore(region)[name]; found { + if pj, found := b.processingJobsStore(region).Get(name); found { return &pj.Tags } } if name, ok := b.transformJobARNIndexStore(region)[resourceARN]; ok { - if tj, found := b.transformJobsStore(region)[name]; found { + if tj, found := b.transformJobsStore(region).Get(name); found { return &tj.Tags } } if name, ok := b.clusterARNIndexStore(region)[resourceARN]; ok { - if c, found := b.clustersStore(region)[name]; found { + if c, found := b.clustersStore(region).Get(name); found { return &c.Tags } } @@ -1822,32 +2091,32 @@ func (b *InMemoryBackend) findTagMapIndexedExtraLocked(resourceARN string, regio // featureGroups, pipelines, experiments, trials, trialComponents). Separated // to keep findTagMapLocked within cognitive-complexity limits. func (b *InMemoryBackend) findTagMapStatefulLocked(resourceARN string, region string) *map[string]string { - for _, d := range b.domainsStore(region) { + for _, d := range b.domainsStore(region).All() { if d.DomainArn == resourceARN { return &d.Tags } } - for _, fg := range b.featureGroupsStore(region) { + for _, fg := range b.featureGroupsStore(region).All() { if fg.FeatureGroupArn == resourceARN { return &fg.Tags } } - for _, p := range b.pipelinesStore(region) { + for _, p := range b.pipelinesStore(region).All() { if p.PipelineArn == resourceARN { return &p.Tags } } - for _, e := range b.experimentsStore(region) { + for _, e := range b.experimentsStore(region).All() { if e.ExperimentArn == resourceARN { return &e.Tags } } - for _, t := range b.trialsStore(region) { + for _, t := range b.trialsStore(region).All() { if t.TrialArn == resourceARN { return &t.Tags } } - for _, tc := range b.trialComponentsStore(region) { + for _, tc := range b.trialComponentsStore(region).All() { if tc.TrialComponentArn == resourceARN { return &tc.Tags } @@ -1929,7 +2198,7 @@ func (b *InMemoryBackend) AddAssociation( assocStore := b.associationsStore(region) key := associationKey(sourceArn, destinationArn) - if _, ok := assocStore[key]; ok { + if _, ok := assocStore.Get(key); ok { return nil, fmt.Errorf( "%w: association between %s and %s already exists", ErrAssociationAlreadyExists, @@ -1953,7 +2222,7 @@ func (b *InMemoryBackend) AddAssociation( CreationTime: time.Now(), Tags: mergeTags(nil, tags), } - assocStore[key] = a + assocStore.Put(a) return cloneAssociation(a), nil } @@ -1978,7 +2247,7 @@ func (b *InMemoryBackend) AssociateTrialComponent( tcaStore := b.trialComponentAssociationsStore(region) key := trialComponentKey(trialName, trialComponentName) - if _, ok := tcaStore[key]; ok { + if _, ok := tcaStore.Get(key); ok { return nil, fmt.Errorf("%w: trial component %s is already associated with trial %s", ErrAssociationAlreadyExists, trialComponentName, trialName) } @@ -1998,7 +2267,7 @@ func (b *InMemoryBackend) AssociateTrialComponent( TrialComponentArn: componentArn, CreationTime: time.Now(), } - tcaStore[key] = assoc + tcaStore.Put(assoc) return cloneTrialComponentAssociation(assoc), nil } @@ -2007,7 +2276,7 @@ func (b *InMemoryBackend) AssociateTrialComponent( func (b *InMemoryBackend) ensureClusterLocked(ctx context.Context, clusterName string) (*Cluster, error) { region := getRegion(ctx, b.region) - c, ok := b.clustersStore(region)[clusterName] + c, ok := b.clustersStore(region).Get(clusterName) if !ok { return nil, fmt.Errorf("%w: cluster %q not found", ErrClusterNotFound, clusterName) } @@ -2029,7 +2298,7 @@ func (b *InMemoryBackend) AddClusterInternal(ctx context.Context, clusterName st Nodes: make(map[string]*ClusterNode), CreationTime: time.Now(), } - b.clustersStore(region)[clusterName] = c + b.clustersStore(region).Put(c) b.clusterARNIndexStore(region)[clusterARN] = clusterName return cloneCluster(c) @@ -2051,7 +2320,7 @@ func (b *InMemoryBackend) AddActionInternal(ctx context.Context, name, actionTyp LastModifiedTime: now, Tags: make(map[string]string), } - b.actionsStore(region)[name] = a + b.actionsStore(region).Put(a) b.actionARNIndexStore(region)[actionARN] = name return cloneAction(a) @@ -2071,7 +2340,7 @@ func (b *InMemoryBackend) AddAlgorithmInternal(ctx context.Context, name string) CreationTime: time.Now(), Tags: make(map[string]string), } - b.algorithmsStore(region)[name] = al + b.algorithmsStore(region).Put(al) b.algorithmARNIndexStore(region)[algorithmARN] = al.AlgorithmName return cloneAlgorithm(al) @@ -2201,7 +2470,7 @@ func (b *InMemoryBackend) BatchDescribeModelPackage( results := make(map[string]ModelPackageBatchResult, len(modelPackageArns)) for _, arnStr := range modelPackageArns { - mp, ok := mpStore[arnStr] + mp, ok := mpStore.Get(arnStr) if !ok { results[arnStr] = ModelPackageBatchResult{ ErrorCode: "ValidationException", @@ -2306,7 +2575,7 @@ func (b *InMemoryBackend) CreateAction( region := getRegion(ctx, b.region) actionsStore := b.actionsStore(region) - if _, ok := actionsStore[name]; ok { + if _, ok := actionsStore.Get(name); ok { return nil, fmt.Errorf("%w: action %q already exists", ErrActionAlreadyExists, name) } @@ -2325,7 +2594,7 @@ func (b *InMemoryBackend) CreateAction( CreationTime: now, LastModifiedTime: now, } - actionsStore[name] = a + actionsStore.Put(a) b.actionARNIndexStore(region)[actionARN] = name return cloneAction(a), nil @@ -2354,7 +2623,7 @@ func (b *InMemoryBackend) CreateAlgorithm(ctx context.Context, opts CreateAlgori region := getRegion(ctx, b.region) algoStore := b.algorithmsStore(region) - if _, ok := algoStore[opts.AlgorithmName]; ok { + if _, ok := algoStore.Get(opts.AlgorithmName); ok { return nil, fmt.Errorf("%w: algorithm %q already exists", ErrAlgorithmAlreadyExists, opts.AlgorithmName) } @@ -2378,7 +2647,7 @@ func (b *InMemoryBackend) CreateAlgorithm(ctx context.Context, opts CreateAlgori Tags: mergeTags(nil, opts.Tags), CreationTime: time.Now(), } - algoStore[opts.AlgorithmName] = al + algoStore.Put(al) b.algorithmARNIndexStore(region)[algorithmARN] = opts.AlgorithmName return cloneAlgorithm(al), nil @@ -2391,7 +2660,7 @@ func (b *InMemoryBackend) DescribeAlgorithm(ctx context.Context, name string) (* b.mu.RLock("DescribeAlgorithm") defer b.mu.RUnlock() - al, ok := b.algorithmsStore(region)[name] + al, ok := b.algorithmsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: algorithm %q not found", ErrAlgorithmNotFound, name) } @@ -2408,12 +2677,12 @@ func (b *InMemoryBackend) DeleteAlgorithm(ctx context.Context, name string) erro store := b.algorithmsStore(region) - al, ok := store[name] + al, ok := store.Get(name) if !ok { return fmt.Errorf("%w: algorithm %q not found", ErrAlgorithmNotFound, name) } - delete(store, name) + store.Delete(name) delete(b.algorithmARNIndexStore(region), al.AlgorithmArn) return nil @@ -2426,7 +2695,12 @@ func (b *InMemoryBackend) ListAlgorithms(ctx context.Context, nextToken string) b.mu.RLock("ListAlgorithms") defer b.mu.RUnlock() - return sagemakerListKeyPaged(b.algorithmsStore(region), nextToken, cloneAlgorithm) + return sagemakerListKeyPaged( + b.algorithmsStore(region), + nextToken, + cloneAlgorithm, + func(v *Algorithm) string { return v.AlgorithmName }, + ) } // AddModelPackageInternal adds a model package directly for testing. @@ -2435,6 +2709,6 @@ func (b *InMemoryBackend) AddModelPackageInternal(ctx context.Context, mp *Model defer b.mu.Unlock() region := getRegion(ctx, b.region) - b.modelPackagesStore(region)[mp.ModelPackageArn] = mp + b.modelPackagesStore(region).Put(mp) b.modelPackageARNIndexStore(region)[mp.ModelPackageArn] = mp.ModelPackageArn } diff --git a/services/sagemaker/backend_accuracy.go b/services/sagemaker/backend_accuracy.go index 051d88691..b3097c754 100644 --- a/services/sagemaker/backend_accuracy.go +++ b/services/sagemaker/backend_accuracy.go @@ -99,7 +99,7 @@ func (b *InMemoryBackend) CreateNotebookInstanceLifecycleConfig( region := getRegion(ctx, b.region) - if _, ok := b.notebookLifecycleConfigsStore(region)[name]; ok { + if _, ok := b.notebookLifecycleConfigsStore(region).Get(name); ok { return nil, fmt.Errorf( "%w: notebook lifecycle config %s already exists", ErrNotebookLifecycleConfigAlreadyExists, @@ -122,7 +122,7 @@ func (b *InMemoryBackend) CreateNotebookInstanceLifecycleConfig( CreationTime: now, LastModifiedTime: now, } - b.notebookLifecycleConfigsStore(region)[name] = lc + b.notebookLifecycleConfigsStore(region).Put(lc) return cloneNotebookLifecycleConfig(lc), nil } @@ -137,7 +137,7 @@ func (b *InMemoryBackend) DescribeNotebookInstanceLifecycleConfig( region := getRegion(ctx, b.region) - lc, ok := b.notebookLifecycleConfigsStore(region)[name] + lc, ok := b.notebookLifecycleConfigsStore(region).Get(name) if !ok { return nil, fmt.Errorf( "%w: notebook lifecycle config %q not found", @@ -160,7 +160,7 @@ func (b *InMemoryBackend) UpdateNotebookInstanceLifecycleConfig( region := getRegion(ctx, b.region) - lc, ok := b.notebookLifecycleConfigsStore(region)[name] + lc, ok := b.notebookLifecycleConfigsStore(region).Get(name) if !ok { return nil, fmt.Errorf( "%w: notebook lifecycle config %q not found", @@ -188,7 +188,7 @@ func (b *InMemoryBackend) DeleteNotebookInstanceLifecycleConfig(ctx context.Cont region := getRegion(ctx, b.region) store := b.notebookLifecycleConfigsStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf( "%w: notebook lifecycle config %q not found", ErrNotebookLifecycleConfigNotFound, @@ -196,7 +196,7 @@ func (b *InMemoryBackend) DeleteNotebookInstanceLifecycleConfig(ctx context.Cont ) } - delete(store, name) + store.Delete(name) return nil } @@ -231,7 +231,7 @@ func (b *InMemoryBackend) scheduleNotebookTransition( b.mu.Lock("scheduleNotebookTransition.goroutine") defer b.mu.Unlock() - if nb, ok := b.notebooksStore(region)[name]; ok { + if nb, ok := b.notebooksStore(region).Get(name); ok { nb.NotebookInstanceStatus = nextStatus nb.LastModifiedTime = time.Now() } @@ -245,7 +245,7 @@ func (b *InMemoryBackend) StartNotebookInstanceFSM(ctx context.Context, name str region := getRegion(ctx, b.region) - nb, ok := b.notebooksStore(region)[name] + nb, ok := b.notebooksStore(region).Get(name) if !ok { return fmt.Errorf("%w: notebook instance %q not found", ErrNotebookNotFound, name) } @@ -278,7 +278,7 @@ func (b *InMemoryBackend) StopNotebookInstanceFSM(ctx context.Context, name stri region := getRegion(ctx, b.region) - nb, ok := b.notebooksStore(region)[name] + nb, ok := b.notebooksStore(region).Get(name) if !ok { return fmt.Errorf("%w: notebook instance %q not found", ErrNotebookNotFound, name) } @@ -328,7 +328,7 @@ func (b *InMemoryBackend) UpdateNotebookInstanceFull( region := getRegion(ctx, b.region) - nb, ok := b.notebooksStore(region)[name] + nb, ok := b.notebooksStore(region).Get(name) if !ok { return fmt.Errorf("%w: notebook instance %q not found", ErrNotebookNotFound, name) } @@ -512,7 +512,7 @@ func (b *InMemoryBackend) CreateTrainingJobFull(ctx context.Context, opts Traini region := getRegion(ctx, b.region) - if _, ok := b.trainingJobsStore(region)[opts.TrainingJobName]; ok { + if _, ok := b.trainingJobsStore(region).Get(opts.TrainingJobName); ok { return nil, fmt.Errorf( "%w: training job %s already exists", ErrTrainingJobAlreadyExists, @@ -549,7 +549,7 @@ func (b *InMemoryBackend) CreateTrainingJobFull(ctx context.Context, opts Traini {StartTime: now, Status: "Starting", StatusMessage: "Launching requested ML instances"}, }, } - b.trainingJobsStore(region)[opts.TrainingJobName] = tj + b.trainingJobsStore(region).Put(tj) b.trainingJobARNIndexStore(region)[jobARN] = opts.TrainingJobName b.scheduleTrainingCompletion(b.lifecycleCtx, region, opts.TrainingJobName) @@ -565,7 +565,7 @@ func (b *InMemoryBackend) scheduleTrainingCompletion(ctx context.Context, region b.mu.Lock("scheduleTrainingCompletion.goroutine") defer b.mu.Unlock() - tj, ok := b.trainingJobsStore(region)[name] + tj, ok := b.trainingJobsStore(region).Get(name) if !ok { return } @@ -603,7 +603,7 @@ func (b *InMemoryBackend) StopTrainingJobFSM(ctx context.Context, name string) e region := getRegion(ctx, b.region) - tj, ok := b.trainingJobsStore(region)[name] + tj, ok := b.trainingJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: training job %q not found", ErrTrainingJobNotFound, name) } @@ -615,7 +615,7 @@ func (b *InMemoryBackend) StopTrainingJobFSM(ctx context.Context, name string) e b.mu.Lock("StopTrainingJobFSM.goroutine") defer b.mu.Unlock() - if tj2, ok2 := b.trainingJobsStore(region)[name]; ok2 && + if tj2, ok2 := b.trainingJobsStore(region).Get(name); ok2 && tj2.TrainingJobStatus == pipelineStatusStopping { tj2.TrainingJobStatus = pipelineStatusStopped tj2.LastModifiedTime = time.Now() @@ -645,8 +645,8 @@ func (b *InMemoryBackend) ListTrainingJobsFiltered( region := getRegion(ctx, b.region) - list := make([]*TrainingJob, 0, len(b.trainingJobsStore(region))) - for _, tj := range b.trainingJobsStore(region) { + list := make([]*TrainingJob, 0, b.trainingJobsStore(region).Len()) + for _, tj := range b.trainingJobsStore(region).All() { if f.StatusEquals != "" && !strings.EqualFold(tj.TrainingJobStatus, f.StatusEquals) { continue } @@ -706,7 +706,7 @@ func (b *InMemoryBackend) scheduleEndpointTransition( b.mu.Lock("scheduleEndpointTransition.goroutine") defer b.mu.Unlock() - if ep, ok := b.endpointsStore(region)[name]; ok { + if ep, ok := b.endpointsStore(region).Get(name); ok { ep.EndpointStatus = nextStatus ep.LastModifiedTime = time.Now() } @@ -762,7 +762,7 @@ func (b *InMemoryBackend) UpdateEndpointWeightsAndCapacitiesFull( region := getRegion(ctx, b.region) - ep, ok := b.endpointsStore(region)[name] + ep, ok := b.endpointsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: endpoint %q not found", ErrEndpointNotFound, name) } @@ -938,7 +938,7 @@ func (b *InMemoryBackend) CreateProcessingJob(ctx context.Context, opts Processi region := getRegion(ctx, b.region) - if _, ok := b.processingJobsStore(region)[opts.ProcessingJobName]; ok { + if _, ok := b.processingJobsStore(region).Get(opts.ProcessingJobName); ok { return nil, fmt.Errorf( "%w: processing job %s already exists", ErrProcessingJobAlreadyExists, @@ -964,7 +964,7 @@ func (b *InMemoryBackend) CreateProcessingJob(ctx context.Context, opts Processi ProcessingStartTime: &now, Tags: mergeTags(nil, opts.Tags), } - b.processingJobsStore(region)[opts.ProcessingJobName] = pj + b.processingJobsStore(region).Put(pj) b.processingJobARNIndexStore(region)[pjARN] = opts.ProcessingJobName b.scheduleProcessingCompletion(b.lifecycleCtx, region, opts.ProcessingJobName) @@ -980,7 +980,7 @@ func (b *InMemoryBackend) scheduleProcessingCompletion(ctx context.Context, regi b.mu.Lock("scheduleProcessingCompletion.goroutine") defer b.mu.Unlock() - pj, ok := b.processingJobsStore(region)[name] + pj, ok := b.processingJobsStore(region).Get(name) if !ok || pj.ProcessingJobStatus != "InProgress" { return } @@ -998,7 +998,7 @@ func (b *InMemoryBackend) DescribeProcessingJob(ctx context.Context, name string region := getRegion(ctx, b.region) - pj, ok := b.processingJobsStore(region)[name] + pj, ok := b.processingJobsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: processing job %q not found", ErrProcessingJobNotFound, name) } @@ -1013,7 +1013,7 @@ func (b *InMemoryBackend) StopProcessingJob(ctx context.Context, name string) er region := getRegion(ctx, b.region) - pj, ok := b.processingJobsStore(region)[name] + pj, ok := b.processingJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: processing job %q not found", ErrProcessingJobNotFound, name) } @@ -1025,7 +1025,7 @@ func (b *InMemoryBackend) StopProcessingJob(ctx context.Context, name string) er b.mu.Lock("StopProcessingJob.goroutine") defer b.mu.Unlock() - if pj2, ok2 := b.processingJobsStore(region)[name]; ok2 && pj2.ProcessingJobStatus == "Stopping" { + if pj2, ok2 := b.processingJobsStore(region).Get(name); ok2 && pj2.ProcessingJobStatus == "Stopping" { pj2.ProcessingJobStatus = "Stopped" pj2.LastModifiedTime = time.Now() } @@ -1042,7 +1042,7 @@ func (b *InMemoryBackend) DeleteProcessingJob(ctx context.Context, name string) region := getRegion(ctx, b.region) - pj, ok := b.processingJobsStore(region)[name] + pj, ok := b.processingJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: processing job %q not found", ErrProcessingJobNotFound, name) } @@ -1054,7 +1054,7 @@ func (b *InMemoryBackend) DeleteProcessingJob(ctx context.Context, name string) ) } - delete(b.processingJobsStore(region), name) + b.processingJobsStore(region).Delete(name) delete(b.processingJobARNIndexStore(region), pj.ProcessingJobArn) return nil @@ -1071,8 +1071,8 @@ func (b *InMemoryBackend) ListProcessingJobs( region := getRegion(ctx, b.region) - list := make([]*ProcessingJob, 0, len(b.processingJobsStore(region))) - for _, pj := range b.processingJobsStore(region) { + list := make([]*ProcessingJob, 0, b.processingJobsStore(region).Len()) + for _, pj := range b.processingJobsStore(region).All() { if statusEquals != "" && !strings.EqualFold(pj.ProcessingJobStatus, statusEquals) { continue } @@ -1214,7 +1214,7 @@ func (b *InMemoryBackend) CreateNotebookInstanceFull( region := getRegion(ctx, b.region) - if _, ok := b.notebooksStore(region)[opts.Name]; ok { + if _, ok := b.notebooksStore(region).Get(opts.Name); ok { return nil, fmt.Errorf( "%w: notebook instance %s already exists", ErrNotebookAlreadyExists, @@ -1245,7 +1245,7 @@ func (b *InMemoryBackend) CreateNotebookInstanceFull( LastModifiedTime: now, Tags: mergeTags(nil, opts.Tags), } - b.notebooksStore(region)[opts.Name] = nb + b.notebooksStore(region).Put(nb) b.notebookARNIndexStore(region)[nbARN] = opts.Name return cloneNotebook(nb), nil diff --git a/services/sagemaker/backend_accuracy2.go b/services/sagemaker/backend_accuracy2.go index a1043475a..d8d851d8c 100644 --- a/services/sagemaker/backend_accuracy2.go +++ b/services/sagemaker/backend_accuracy2.go @@ -123,7 +123,7 @@ func (b *InMemoryBackend) CreateTransformJob(ctx context.Context, opts Transform b.mu.Lock("CreateTransformJob") defer b.mu.Unlock() - if _, ok := b.transformJobsStore(region)[opts.TransformJobName]; ok { + if _, ok := b.transformJobsStore(region).Get(opts.TransformJobName); ok { return nil, fmt.Errorf( "%w: transform job %s already exists", ErrTransformJobAlreadyExists, @@ -156,7 +156,7 @@ func (b *InMemoryBackend) CreateTransformJob(ctx context.Context, opts Transform CreationTime: now, LastModifiedTime: now, } - b.transformJobsStore(region)[opts.TransformJobName] = tj + b.transformJobsStore(region).Put(tj) b.transformJobARNIndexStore(region)[jobARN] = opts.TransformJobName b.runDelayed(b.lifecycleCtx, transformJobCompletionDelay, func() { @@ -172,7 +172,7 @@ func (b *InMemoryBackend) applyTransformJobCompletion(ctx context.Context, name b.mu.Lock("applyTransformJobCompletion") defer b.mu.Unlock() - tj, ok := b.transformJobsStore(region)[name] + tj, ok := b.transformJobsStore(region).Get(name) if !ok || tj.TransformJobStatus != trainingJobStatusInProgress { return } @@ -190,7 +190,7 @@ func (b *InMemoryBackend) DescribeTransformJob(ctx context.Context, name string) b.mu.RLock("DescribeTransformJob") defer b.mu.RUnlock() - tj, ok := b.transformJobsStore(region)[name] + tj, ok := b.transformJobsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: transform job %q not found", ErrTransformJobNotFound, name) } @@ -205,7 +205,7 @@ func (b *InMemoryBackend) StopTransformJob(ctx context.Context, name string) err b.mu.Lock("StopTransformJob") defer b.mu.Unlock() - tj, ok := b.transformJobsStore(region)[name] + tj, ok := b.transformJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: transform job %q not found", ErrTransformJobNotFound, name) } @@ -224,7 +224,9 @@ func (b *InMemoryBackend) StopTransformJob(ctx context.Context, name string) err b.runDelayed(b.lifecycleCtx, transformJobStoppingDelay, func() { b.mu.Lock("StopTransformJob.goroutine") defer b.mu.Unlock() - if tj2, found := b.transformJobsStore(region)[name]; found && tj2.TransformJobStatus == pipelineStatusStopping { + if tj2, found := b.transformJobsStore(region). + Get(name); found && + tj2.TransformJobStatus == pipelineStatusStopping { tj2.TransformJobStatus = "Stopped" tj2.LastModifiedTime = time.Now() } @@ -250,9 +252,9 @@ func (b *InMemoryBackend) ListTransformJobs( b.mu.RLock("ListTransformJobs") defer b.mu.RUnlock() - list := make([]*TransformJob, 0, len(b.transformJobsStore(region))) + list := make([]*TransformJob, 0, b.transformJobsStore(region).Len()) - for _, tj := range b.transformJobsStore(region) { + for _, tj := range b.transformJobsStore(region).All() { if filter.StatusEquals != "" && tj.TransformJobStatus != filter.StatusEquals { continue } diff --git a/services/sagemaker/backend_accuracy3.go b/services/sagemaker/backend_accuracy3.go index b71914bc7..fc066e93d 100644 --- a/services/sagemaker/backend_accuracy3.go +++ b/services/sagemaker/backend_accuracy3.go @@ -69,7 +69,7 @@ func (b *InMemoryBackend) CreateEdgePackagingJob( return nil, fmt.Errorf("%w: EdgePackagingJobName is required", ErrValidation) } - if _, ok := b.edgePackagingJobsStore(region)[opts.EdgePackagingJobName]; ok { + if _, ok := b.edgePackagingJobsStore(region).Get(opts.EdgePackagingJobName); ok { return nil, fmt.Errorf( "%w: edge packaging job %q already exists", ErrEdgePackagingJobAlreadyExists, @@ -92,7 +92,7 @@ func (b *InMemoryBackend) CreateEdgePackagingJob( CreationTime: now, LastModifiedTime: now, } - b.edgePackagingJobsStore(region)[opts.EdgePackagingJobName] = j + b.edgePackagingJobsStore(region).Put(j) return cloneEdgePackagingJob(j), nil } @@ -104,7 +104,7 @@ func (b *InMemoryBackend) DescribeEdgePackagingJob(ctx context.Context, name str region := getRegion(ctx, b.region) - j, ok := b.edgePackagingJobsStore(region)[name] + j, ok := b.edgePackagingJobsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: edge packaging job %q not found", ErrEdgePackagingJobNotFound, name) } @@ -119,7 +119,7 @@ func (b *InMemoryBackend) StopEdgePackagingJob(ctx context.Context, name string) region := getRegion(ctx, b.region) - j, ok := b.edgePackagingJobsStore(region)[name] + j, ok := b.edgePackagingJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: edge packaging job %q not found", ErrEdgePackagingJobNotFound, name) } @@ -149,16 +149,16 @@ func (b *InMemoryBackend) ListEdgePackagingJobs( var keys []string - for name, j := range b.edgePackagingJobsStore(region) { + for _, j := range b.edgePackagingJobsStore(region).All() { if filter.StatusEquals != "" && j.EdgePackagingJobStatus != filter.StatusEquals { continue } - if filter.NameContains != "" && !strings.Contains(name, filter.NameContains) { + if filter.NameContains != "" && !strings.Contains(j.EdgePackagingJobName, filter.NameContains) { continue } - keys = append(keys, name) + keys = append(keys, j.EdgePackagingJobName) } sort.Strings(keys) @@ -180,7 +180,7 @@ func (b *InMemoryBackend) ListEdgePackagingJobs( out := make([]*EdgePackagingJob, 0, end-start) for _, k := range keys[start:end] { - out = append(out, cloneEdgePackagingJob(store[k])) + out = append(out, cloneEdgePackagingJob(tableGet(store, k))) } next := "" @@ -245,7 +245,7 @@ func (b *InMemoryBackend) CreateInferenceRecommendationsJob( return nil, fmt.Errorf("%w: JobName is required", ErrValidation) } - if _, ok := b.inferenceRecommendationsJobsStore(region)[opts.JobName]; ok { + if _, ok := b.inferenceRecommendationsJobsStore(region).Get(opts.JobName); ok { return nil, fmt.Errorf( "%w: inference recommendations job %q already exists", ErrInferenceRecommendationsJobAlreadyExists, @@ -267,7 +267,7 @@ func (b *InMemoryBackend) CreateInferenceRecommendationsJob( CreationTime: now, LastModifiedTime: now, } - b.inferenceRecommendationsJobsStore(region)[opts.JobName] = j + b.inferenceRecommendationsJobsStore(region).Put(j) return cloneInferenceRecommendationsJob(j), nil } @@ -282,7 +282,7 @@ func (b *InMemoryBackend) DescribeInferenceRecommendationsJob( region := getRegion(ctx, b.region) - j, ok := b.inferenceRecommendationsJobsStore(region)[name] + j, ok := b.inferenceRecommendationsJobsStore(region).Get(name) if !ok { return nil, fmt.Errorf( "%w: inference recommendations job %q not found", @@ -301,7 +301,7 @@ func (b *InMemoryBackend) StopInferenceRecommendationsJob(ctx context.Context, n region := getRegion(ctx, b.region) - j, ok := b.inferenceRecommendationsJobsStore(region)[name] + j, ok := b.inferenceRecommendationsJobsStore(region).Get(name) if !ok { return fmt.Errorf( "%w: inference recommendations job %q not found", @@ -330,6 +330,7 @@ func (b *InMemoryBackend) ListInferenceRecommendationsJobs( b.inferenceRecommendationsJobsStore(region), nextToken, cloneInferenceRecommendationsJob, + func(v *InferenceRecommendationsJob) string { return v.JobName }, ) } @@ -347,9 +348,9 @@ func (b *InMemoryBackend) UpdateUserProfile( region := getRegion(ctx, b.region) - key := userProfileKey{DomainID: domainID, UserProfileName: userProfileName} + key := userProfileKeyString(userProfileKey{DomainID: domainID, UserProfileName: userProfileName}) - up, ok := b.userProfilesStore(region)[key] + up, ok := b.userProfilesStore(region).Get(key) if !ok { return nil, fmt.Errorf( "%w: user profile %q not found in domain %q", @@ -373,7 +374,7 @@ func (b *InMemoryBackend) UpdateSpace(ctx context.Context, domainID, spaceName s key := spaceKey(domainID, spaceName) - s, ok := b.spacesStore(region)[key] + s, ok := b.spacesStore(region).Get(key) if !ok { return nil, fmt.Errorf("%w: space %q not found in domain %q", ErrSpaceNotFound, spaceName, domainID) } @@ -398,7 +399,7 @@ func (b *InMemoryBackend) UpdateModelPackage( arnStr = v } - mp, ok := b.modelPackagesStore(region)[arnStr] + mp, ok := b.modelPackagesStore(region).Get(arnStr) if !ok { return nil, fmt.Errorf("%w: model package %q not found", ErrModelPackageNotFound, nameOrArn) } @@ -420,7 +421,7 @@ func (b *InMemoryBackend) UpdateMlflowTrackingServer( region := getRegion(ctx, b.region) - s, ok := b.mlflowTrackingServersStore(region)[name] + s, ok := b.mlflowTrackingServersStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: MLflow tracking server %q not found", ErrMlflowTrackingServerNotFound, name) } @@ -448,7 +449,12 @@ func (b *InMemoryBackend) ListMlflowTrackingServers( region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.mlflowTrackingServersStore(region), nextToken, cloneMlflowTrackingServer) + return sagemakerListKeyPaged( + b.mlflowTrackingServersStore(region), + nextToken, + cloneMlflowTrackingServer, + func(v *MlflowTrackingServer) string { return v.TrackingServerName }, + ) } // ListModelCards returns all model cards. @@ -458,7 +464,12 @@ func (b *InMemoryBackend) ListModelCards(ctx context.Context, nextToken string) region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.modelCardsStore(region), nextToken, cloneModelCard) + return sagemakerListKeyPaged( + b.modelCardsStore(region), + nextToken, + cloneModelCard, + func(v *ModelCard) string { return v.ModelCardName }, + ) } // ListOptimizationJobs returns all optimization jobs. @@ -468,7 +479,12 @@ func (b *InMemoryBackend) ListOptimizationJobs(ctx context.Context, nextToken st region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.optimizationJobsStore(region), nextToken, cloneOptimizationJob) + return sagemakerListKeyPaged( + b.optimizationJobsStore(region), + nextToken, + cloneOptimizationJob, + func(v *OptimizationJob) string { return v.OptimizationJobName }, + ) } // ListStudioLifecycleConfigs returns all Studio lifecycle configs. @@ -481,7 +497,12 @@ func (b *InMemoryBackend) ListStudioLifecycleConfigs( region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.studioLifecycleConfigsStore(region), nextToken, cloneStudioLifecycleConfig) + return sagemakerListKeyPaged( + b.studioLifecycleConfigsStore(region), + nextToken, + cloneStudioLifecycleConfig, + func(v *StudioLifecycleConfig) string { return v.StudioLifecycleConfigName }, + ) } // ListInferenceExperiments returns all inference experiments. @@ -494,7 +515,12 @@ func (b *InMemoryBackend) ListInferenceExperiments( region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.inferenceExperimentsStore(region), nextToken, cloneInferenceExperiment) + return sagemakerListKeyPaged( + b.inferenceExperimentsStore(region), + nextToken, + cloneInferenceExperiment, + func(v *InferenceExperiment) string { return v.Name }, + ) } // ListFlowDefinitions returns all flow definitions. @@ -504,7 +530,12 @@ func (b *InMemoryBackend) ListFlowDefinitions(ctx context.Context, nextToken str region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.flowDefinitionsStore(region), nextToken, cloneFlowDefinition) + return sagemakerListKeyPaged( + b.flowDefinitionsStore(region), + nextToken, + cloneFlowDefinition, + func(v *FlowDefinition) string { return v.FlowDefinitionName }, + ) } // ListHumanTaskUIs returns all human task UIs. @@ -514,7 +545,12 @@ func (b *InMemoryBackend) ListHumanTaskUIs(ctx context.Context, nextToken string region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.humanTaskUisStore(region), nextToken, cloneHumanTaskUI) + return sagemakerListKeyPaged( + b.humanTaskUisStore(region), + nextToken, + cloneHumanTaskUI, + func(v *HumanTaskUI) string { return v.HumanTaskUIName }, + ) } // ListAppImageConfigs returns all App image configs. @@ -524,7 +560,12 @@ func (b *InMemoryBackend) ListAppImageConfigs(ctx context.Context, nextToken str region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.appImageConfigsStore(region), nextToken, cloneAppImageConfig) + return sagemakerListKeyPaged( + b.appImageConfigsStore(region), + nextToken, + cloneAppImageConfig, + func(v *AppImageConfig) string { return v.AppImageConfigName }, + ) } // ListTrainingJobsForHyperParameterTuningJob returns training jobs for an HP tuning job. @@ -538,7 +579,7 @@ func (b *InMemoryBackend) ListTrainingJobsForHyperParameterTuningJob( region := getRegion(ctx, b.region) - if _, ok := b.hpTuningJobsStore(region)[jobName]; !ok { + if _, ok := b.hpTuningJobsStore(region).Get(jobName); !ok { return nil, "", fmt.Errorf("%w: HP tuning job %q not found", ErrHPTuningJobNotFound, jobName) } diff --git a/services/sagemaker/backend_accuracy4.go b/services/sagemaker/backend_accuracy4.go index 3a4ccea52..bf74843d2 100644 --- a/services/sagemaker/backend_accuracy4.go +++ b/services/sagemaker/backend_accuracy4.go @@ -10,6 +10,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awserr" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) const ( @@ -74,7 +75,7 @@ func (b *InMemoryBackend) CreateDeviceFleet(ctx context.Context, opts CreateDevi return nil, fmt.Errorf("%w: DeviceFleetName is required", ErrValidation) } - if _, ok := b.deviceFleetsStore(region)[opts.DeviceFleetName]; ok { + if _, ok := b.deviceFleetsStore(region).Get(opts.DeviceFleetName); ok { return nil, fmt.Errorf("%w: device fleet %q already exists", ErrDeviceFleetAlreadyExists, opts.DeviceFleetName) } @@ -91,7 +92,7 @@ func (b *InMemoryBackend) CreateDeviceFleet(ctx context.Context, opts CreateDevi CreationTime: now, LastModifiedTime: now, } - b.deviceFleetsStore(region)[opts.DeviceFleetName] = f + b.deviceFleetsStore(region).Put(f) return cloneDeviceFleet(f), nil } @@ -103,7 +104,7 @@ func (b *InMemoryBackend) DescribeDeviceFleet(ctx context.Context, name string) region := getRegion(ctx, b.region) - f, ok := b.deviceFleetsStore(region)[name] + f, ok := b.deviceFleetsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: device fleet %q", ErrDeviceFleetNotFound, name) } @@ -118,7 +119,12 @@ func (b *InMemoryBackend) ListDeviceFleets(ctx context.Context, nextToken string region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.deviceFleetsStore(region), nextToken, cloneDeviceFleet) + return sagemakerListKeyPaged( + b.deviceFleetsStore(region), + nextToken, + cloneDeviceFleet, + func(v *DeviceFleet) string { return v.DeviceFleetName }, + ) } // UpdateDeviceFleet updates a device fleet's description or role ARN. @@ -128,7 +134,7 @@ func (b *InMemoryBackend) UpdateDeviceFleet(ctx context.Context, name, descripti region := getRegion(ctx, b.region) - f, ok := b.deviceFleetsStore(region)[name] + f, ok := b.deviceFleetsStore(region).Get(name) if !ok { return fmt.Errorf("%w: device fleet %q", ErrDeviceFleetNotFound, name) } @@ -154,11 +160,11 @@ func (b *InMemoryBackend) DeleteDeviceFleet(ctx context.Context, name string) er region := getRegion(ctx, b.region) store := b.deviceFleetsStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf("%w: device fleet %q", ErrDeviceFleetNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -176,6 +182,12 @@ type deviceKey struct { deviceName string } +// deviceKeyString flattens a deviceKey to the single delimited string used as +// the store.Table primary key for b.devices. +func deviceKeyString(k deviceKey) string { + return k.fleetName + "|" + k.deviceName +} + // Device represents a SageMaker edge device registered in a fleet. type Device struct { RegistrationTime time.Time `json:"RegistrationTime"` @@ -210,7 +222,7 @@ func (b *InMemoryBackend) RegisterDevices(ctx context.Context, fleetName string, region := getRegion(ctx, b.region) - if _, ok := b.deviceFleetsStore(region)[fleetName]; !ok { + if _, ok := b.deviceFleetsStore(region).Get(fleetName); !ok { return fmt.Errorf("%w: device fleet %q", ErrDeviceFleetNotFound, fleetName) } @@ -221,9 +233,8 @@ func (b *InMemoryBackend) RegisterDevices(ctx context.Context, fleetName string, continue } - k := deviceKey{fleetName: fleetName, deviceName: d.DeviceName} deviceARN := arn.Build("sagemaker", region, b.accountID, "device/"+d.DeviceName) - devicesStore[k] = &Device{ + devicesStore.Put(&Device{ DeviceName: d.DeviceName, DeviceFleetName: fleetName, DeviceArn: deviceARN, @@ -232,7 +243,7 @@ func (b *InMemoryBackend) RegisterDevices(ctx context.Context, fleetName string, Tags: mergeTags(nil, d.Tags), RegistrationTime: now, LastModifiedTime: now, - } + }) } return nil @@ -247,7 +258,7 @@ func (b *InMemoryBackend) DeregisterDevices(ctx context.Context, fleetName strin store := b.devicesStore(region) for _, name := range deviceNames { - delete(store, deviceKey{fleetName: fleetName, deviceName: name}) + store.Delete(deviceKeyString(deviceKey{fleetName: fleetName, deviceName: name})) } return nil @@ -269,7 +280,7 @@ func (b *InMemoryBackend) UpdateDevices(ctx context.Context, fleetName string, d region := getRegion(ctx, b.region) - if _, ok := b.deviceFleetsStore(region)[fleetName]; !ok { + if _, ok := b.deviceFleetsStore(region).Get(fleetName); !ok { return fmt.Errorf("%w: device fleet %q", ErrDeviceFleetNotFound, fleetName) } @@ -277,7 +288,7 @@ func (b *InMemoryBackend) UpdateDevices(ctx context.Context, fleetName string, d now := time.Now() for _, d := range devices { - dev, ok := store[deviceKey{fleetName: fleetName, deviceName: d.DeviceName}] + dev, ok := store.Get(deviceKeyString(deviceKey{fleetName: fleetName, deviceName: d.DeviceName})) if !ok { continue } @@ -303,7 +314,7 @@ func (b *InMemoryBackend) DescribeDevice(ctx context.Context, fleetName, deviceN region := getRegion(ctx, b.region) - d, ok := b.devicesStore(region)[deviceKey{fleetName: fleetName, deviceName: deviceName}] + d, ok := b.devicesStore(region).Get(deviceKeyString(deviceKey{fleetName: fleetName, deviceName: deviceName})) if !ok { return nil, fmt.Errorf("%w: device %q in fleet %q", ErrDeviceNotFound, deviceName, fleetName) } @@ -321,17 +332,17 @@ func (b *InMemoryBackend) ListDevices(ctx context.Context, fleetFilter, nextToke return devicesInFleetPaged(b.devicesStore(region), fleetFilter, nextToken) } -// devicesInFleetPaged filters store by fleetFilter (or all devices if empty), +// devicesInFleetPaged filters tbl by fleetFilter (or all devices if empty), // sorts by "fleetName/deviceName", and paginates the result. Caller must hold // b.mu (read or write). -func devicesInFleetPaged(store map[deviceKey]*Device, fleetFilter, nextToken string) ([]*Device, string) { - keys := make([]string, 0, len(store)) - for k := range store { - if fleetFilter != "" && k.fleetName != fleetFilter { +func devicesInFleetPaged(tbl *store.Table[Device], fleetFilter, nextToken string) ([]*Device, string) { + keys := make([]string, 0, tbl.Len()) + for _, d := range tbl.All() { + if fleetFilter != "" && d.DeviceFleetName != fleetFilter { continue } - keys = append(keys, k.fleetName+"/"+k.deviceName) + keys = append(keys, d.DeviceFleetName+"/"+d.DeviceName) } sort.Strings(keys) @@ -356,7 +367,7 @@ func devicesInFleetPaged(store map[deviceKey]*Device, fleetFilter, nextToken str continue } - if d, ok := store[deviceKey{fleetName: parts[0], deviceName: parts[1]}]; ok { + if d, ok := tbl.Get(deviceKeyString(deviceKey{fleetName: parts[0], deviceName: parts[1]})); ok { out = append(out, cloneDevice(d)) } } @@ -424,7 +435,7 @@ func (b *InMemoryBackend) CreateInferenceComponent( return nil, fmt.Errorf("%w: InferenceComponentName is required", ErrValidation) } - if _, ok := b.inferenceComponentsStore(region)[opts.InferenceComponentName]; ok { + if _, ok := b.inferenceComponentsStore(region).Get(opts.InferenceComponentName); ok { return nil, fmt.Errorf( "%w: inference component %q already exists", ErrInferenceComponentAlreadyExists, @@ -452,7 +463,7 @@ func (b *InMemoryBackend) CreateInferenceComponent( CreationTime: now, LastModifiedTime: now, } - b.inferenceComponentsStore(region)[opts.InferenceComponentName] = c + b.inferenceComponentsStore(region).Put(c) return cloneInferenceComponent(c), nil } @@ -464,7 +475,7 @@ func (b *InMemoryBackend) DescribeInferenceComponent(ctx context.Context, name s region := getRegion(ctx, b.region) - c, ok := b.inferenceComponentsStore(region)[name] + c, ok := b.inferenceComponentsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: inference component %q", ErrInferenceComponentNotFound, name) } @@ -483,13 +494,13 @@ func (b *InMemoryBackend) ListInferenceComponents( region := getRegion(ctx, b.region) store := b.inferenceComponentsStore(region) - keys := make([]string, 0, len(store)) - for k, c := range store { + keys := make([]string, 0, store.Len()) + for _, c := range store.All() { if endpointFilter != "" && c.EndpointName != endpointFilter { continue } - keys = append(keys, k) + keys = append(keys, c.InferenceComponentName) } sort.Strings(keys) @@ -509,7 +520,7 @@ func (b *InMemoryBackend) ListInferenceComponents( out := make([]*InferenceComponent, 0, end-start) for _, k := range keys[start:end] { - out = append(out, cloneInferenceComponent(store[k])) + out = append(out, cloneInferenceComponent(tableGet(store, k))) } next := "" @@ -527,7 +538,7 @@ func (b *InMemoryBackend) UpdateInferenceComponent(ctx context.Context, name, va region := getRegion(ctx, b.region) - c, ok := b.inferenceComponentsStore(region)[name] + c, ok := b.inferenceComponentsStore(region).Get(name) if !ok { return fmt.Errorf("%w: inference component %q", ErrInferenceComponentNotFound, name) } @@ -552,7 +563,7 @@ func (b *InMemoryBackend) UpdateInferenceComponentRuntimeConfig(ctx context.Cont region := getRegion(ctx, b.region) - c, ok := b.inferenceComponentsStore(region)[name] + c, ok := b.inferenceComponentsStore(region).Get(name) if !ok { return fmt.Errorf("%w: inference component %q", ErrInferenceComponentNotFound, name) } @@ -572,11 +583,11 @@ func (b *InMemoryBackend) DeleteInferenceComponent(ctx context.Context, name str region := getRegion(ctx, b.region) store := b.inferenceComponentsStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf("%w: inference component %q", ErrInferenceComponentNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -661,7 +672,7 @@ func (b *InMemoryBackend) DescribeClusterSchedulerConfig( region := getRegion(ctx, b.region) - c, ok := b.clusterSchedulerConfigsStore(region)[name] + c, ok := b.clusterSchedulerConfigsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: cluster scheduler config %q", ErrClusterSchedulerConfigNotFound, name) } @@ -679,7 +690,12 @@ func (b *InMemoryBackend) ListClusterSchedulerConfigs( region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.clusterSchedulerConfigsStore(region), nextToken, cloneClusterSchedulerConfig) + return sagemakerListKeyPaged( + b.clusterSchedulerConfigsStore(region), + nextToken, + cloneClusterSchedulerConfig, + func(v *ClusterSchedulerConfig) string { return v.ClusterSchedulerConfigName }, + ) } // UpdateClusterSchedulerConfig updates a cluster scheduler config's cluster ARN. @@ -689,7 +705,7 @@ func (b *InMemoryBackend) UpdateClusterSchedulerConfig(ctx context.Context, name region := getRegion(ctx, b.region) - c, ok := b.clusterSchedulerConfigsStore(region)[name] + c, ok := b.clusterSchedulerConfigsStore(region).Get(name) if !ok { return fmt.Errorf("%w: cluster scheduler config %q", ErrClusterSchedulerConfigNotFound, name) } @@ -711,11 +727,11 @@ func (b *InMemoryBackend) DeleteClusterSchedulerConfig(ctx context.Context, name region := getRegion(ctx, b.region) store := b.clusterSchedulerConfigsStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf("%w: cluster scheduler config %q", ErrClusterSchedulerConfigNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -793,7 +809,7 @@ func (b *InMemoryBackend) DescribeComputeQuota(ctx context.Context, name string) region := getRegion(ctx, b.region) - q, ok := b.computeQuotasStore(region)[name] + q, ok := b.computeQuotasStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: compute quota %q", ErrComputeQuotaNotFound, name) } @@ -808,7 +824,12 @@ func (b *InMemoryBackend) ListComputeQuotas(ctx context.Context, nextToken strin region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.computeQuotasStore(region), nextToken, cloneComputeQuota) + return sagemakerListKeyPaged( + b.computeQuotasStore(region), + nextToken, + cloneComputeQuota, + func(v *ComputeQuota) string { return v.ComputeQuotaName }, + ) } // UpdateComputeQuota updates a compute quota's cluster ARN. @@ -818,7 +839,7 @@ func (b *InMemoryBackend) UpdateComputeQuota(ctx context.Context, name, clusterA region := getRegion(ctx, b.region) - q, ok := b.computeQuotasStore(region)[name] + q, ok := b.computeQuotasStore(region).Get(name) if !ok { return fmt.Errorf("%w: compute quota %q", ErrComputeQuotaNotFound, name) } @@ -840,11 +861,11 @@ func (b *InMemoryBackend) DeleteComputeQuota(ctx context.Context, name string) e region := getRegion(ctx, b.region) store := b.computeQuotasStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf("%w: compute quota %q", ErrComputeQuotaNotFound, name) } - delete(store, name) + store.Delete(name) return nil } diff --git a/services/sagemaker/backend_automl_search_ext.go b/services/sagemaker/backend_automl_search_ext.go index 0dbb4d7ab..cdec63907 100644 --- a/services/sagemaker/backend_automl_search_ext.go +++ b/services/sagemaker/backend_automl_search_ext.go @@ -115,7 +115,7 @@ func (b *InMemoryBackend) ListCandidatesForAutoMLJob( b.mu.RLock("ListCandidatesForAutoMLJob") defer b.mu.RUnlock() - job, ok := b.autoMLJobsStore(region)[jobName] + job, ok := b.autoMLJobsStore(region).Get(jobName) if !ok { return nil, "", fmt.Errorf("%w: AutoML job %q not found", ErrAutoMLJobNotFound, jobName) } @@ -223,8 +223,8 @@ func matchesSearchExpression(flat map[string]any, filters []SearchFilter, boolOp func (b *InMemoryBackend) searchableResources(region, resource string) []searchResourceItem { switch resource { case resourceTrainingJob: - items := make([]searchResourceItem, 0, len(b.trainingJobsStore(region))) - for _, tj := range b.trainingJobsStore(region) { + items := make([]searchResourceItem, 0, b.trainingJobsStore(region).Len()) + for _, tj := range b.trainingJobsStore(region).All() { items = append(items, searchResourceItem{ raw: tj, flat: toJSONFlatMap(tj), key: tj.TrainingJobName, }) @@ -232,8 +232,8 @@ func (b *InMemoryBackend) searchableResources(region, resource string) []searchR return items case resourcePipeline: - items := make([]searchResourceItem, 0, len(b.pipelinesStore(region))) - for _, p := range b.pipelinesStore(region) { + items := make([]searchResourceItem, 0, b.pipelinesStore(region).Len()) + for _, p := range b.pipelinesStore(region).All() { items = append(items, searchResourceItem{ raw: p, flat: toJSONFlatMap(p), key: p.PipelineName, }) @@ -471,7 +471,7 @@ func (b *InMemoryBackend) GetScalingConfigurationRecommendation( b.mu.RLock("GetScalingConfigurationRecommendation") defer b.mu.RUnlock() - if _, ok := b.inferenceRecommendationsJobsStore(region)[jobName]; !ok { + if _, ok := b.inferenceRecommendationsJobsStore(region).Get(jobName); !ok { return nil, fmt.Errorf( "%w: inference recommendations job %q not found", ErrInferenceRecommendationsJobNotFound, jobName, ) diff --git a/services/sagemaker/backend_batch2.go b/services/sagemaker/backend_batch2.go index ac427efba..c1edfb307 100644 --- a/services/sagemaker/backend_batch2.go +++ b/services/sagemaker/backend_batch2.go @@ -94,7 +94,7 @@ func (b *InMemoryBackend) CreateModelPackageGroup( return nil, fmt.Errorf("%w: ModelPackageGroupName is required", ErrValidation) } - if _, ok := b.modelPackageGroupsStore(region)[name]; ok { + if _, ok := b.modelPackageGroupsStore(region).Get(name); ok { return nil, fmt.Errorf("%w: model package group %q already exists", ErrValidation, name) } @@ -108,7 +108,7 @@ func (b *InMemoryBackend) CreateModelPackageGroup( Tags: mergeTags(nil, tags), CreationTime: time.Now(), } - b.modelPackageGroupsStore(region)[name] = g + b.modelPackageGroupsStore(region).Put(g) return cloneModelPackageGroup(g), nil } @@ -120,7 +120,7 @@ func (b *InMemoryBackend) DescribeModelPackageGroup(ctx context.Context, name st region := getRegion(ctx, b.region) - g, ok := b.modelPackageGroupsStore(region)[name] + g, ok := b.modelPackageGroupsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: model package group %q not found", ErrModelPackageGroupNotFound, name) } @@ -135,12 +135,12 @@ func (b *InMemoryBackend) DeleteModelPackageGroup(ctx context.Context, name stri region := getRegion(ctx, b.region) - if _, ok := b.modelPackageGroupsStore(region)[name]; !ok { + if _, ok := b.modelPackageGroupsStore(region).Get(name); !ok { return fmt.Errorf("%w: model package group %q not found", ErrModelPackageGroupNotFound, name) } // AWS rejects deletion when model packages still exist in the group. - for _, mp := range b.modelPackagesStore(region) { + for _, mp := range b.modelPackagesStore(region).All() { if mp.ModelPackageGroupName == name { return fmt.Errorf("%w: model package group %q has model packages and cannot be deleted", ErrModelPackageGroupHasPackages, name) @@ -148,7 +148,7 @@ func (b *InMemoryBackend) DeleteModelPackageGroup(ctx context.Context, name stri } store := b.modelPackageGroupsStore(region) - delete(store, name) + store.Delete(name) return nil } @@ -160,7 +160,12 @@ func (b *InMemoryBackend) ListModelPackageGroups(ctx context.Context, nextToken region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.modelPackageGroupsStore(region), nextToken, cloneModelPackageGroup) + return sagemakerListKeyPaged( + b.modelPackageGroupsStore(region), + nextToken, + cloneModelPackageGroup, + func(v *ModelPackageGroup) string { return v.ModelPackageGroupName }, + ) } // ErrModelPackageGroupPolicyNotFound is returned when a model package group @@ -175,7 +180,7 @@ func (b *InMemoryBackend) GetModelPackageGroupPolicy(ctx context.Context, name s region := getRegion(ctx, b.region) - g, ok := b.modelPackageGroupsStore(region)[name] + g, ok := b.modelPackageGroupsStore(region).Get(name) if !ok { return "", fmt.Errorf("%w: model package group %q not found", ErrModelPackageGroupNotFound, name) } @@ -201,7 +206,7 @@ func (b *InMemoryBackend) PutModelPackageGroupPolicy( region := getRegion(ctx, b.region) - g, ok := b.modelPackageGroupsStore(region)[name] + g, ok := b.modelPackageGroupsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: model package group %q not found", ErrModelPackageGroupNotFound, name) } @@ -223,7 +228,7 @@ func (b *InMemoryBackend) DeleteModelPackageGroupPolicy(ctx context.Context, nam region := getRegion(ctx, b.region) - g, ok := b.modelPackageGroupsStore(region)[name] + g, ok := b.modelPackageGroupsStore(region).Get(name) if !ok { return fmt.Errorf("%w: model package group %q not found", ErrModelPackageGroupNotFound, name) } @@ -254,7 +259,7 @@ func (b *InMemoryBackend) CreateModelPackage( mpARN := arn.Build("sagemaker", region, b.accountID, "model-package/"+name) - if _, ok := b.modelPackagesStore(region)[mpARN]; ok { + if _, ok := b.modelPackagesStore(region).Get(mpARN); ok { return nil, fmt.Errorf("%w: model package %q already exists", ErrValidation, name) } @@ -267,7 +272,7 @@ func (b *InMemoryBackend) CreateModelPackage( Tags: mergeTags(nil, tags), CreationTime: time.Now(), } - b.modelPackagesStore(region)[mpARN] = mp + b.modelPackagesStore(region).Put(mp) b.modelPackageARNIndexStore(region)[name] = mpARN return cloneModelPackage(mp), nil @@ -281,13 +286,13 @@ func (b *InMemoryBackend) DescribeModelPackage(ctx context.Context, nameOrArn st region := getRegion(ctx, b.region) // Try direct ARN lookup first. - if mp, ok := b.modelPackagesStore(region)[nameOrArn]; ok { + if mp, ok := b.modelPackagesStore(region).Get(nameOrArn); ok { return cloneModelPackage(mp), nil } // Try name → ARN index. if arnStr, ok := b.modelPackageARNIndexStore(region)[nameOrArn]; ok { - if mp, found := b.modelPackagesStore(region)[arnStr]; found { + if mp, found := b.modelPackagesStore(region).Get(arnStr); found { return cloneModelPackage(mp), nil } } @@ -307,15 +312,15 @@ func (b *InMemoryBackend) DeleteModelPackage(ctx context.Context, nameOrArn stri arnStr = v } - if _, ok := b.modelPackagesStore(region)[arnStr]; !ok { + if _, ok := b.modelPackagesStore(region).Get(arnStr); !ok { return fmt.Errorf("%w: model package %q not found", ErrModelPackageNotFound, nameOrArn) } - mp := b.modelPackagesStore(region)[arnStr] + mp := tableGet(b.modelPackagesStore(region), arnStr) arnIdxStore := b.modelPackageARNIndexStore(region) delete(arnIdxStore, mp.ModelPackageName) mpStore := b.modelPackagesStore(region) - delete(mpStore, arnStr) + mpStore.Delete(arnStr) return nil } @@ -331,10 +336,9 @@ func (b *InMemoryBackend) ListModelPackages( region := getRegion(ctx, b.region) var arns []string - for k := range b.modelPackagesStore(region) { - mp := b.modelPackagesStore(region)[k] + for _, mp := range b.modelPackagesStore(region).All() { if groupName == "" || mp.ModelPackageGroupName == groupName { - arns = append(arns, k) + arns = append(arns, mp.ModelPackageArn) } } @@ -355,7 +359,7 @@ func (b *InMemoryBackend) ListModelPackages( out := make([]*ModelPackage, 0, end-start) for _, k := range arns[start:end] { - out = append(out, cloneModelPackage(b.modelPackagesStore(region)[k])) + out = append(out, cloneModelPackage(tableGet(b.modelPackagesStore(region), k))) } next := "" @@ -425,7 +429,7 @@ func (b *InMemoryBackend) CreateAutoMLJob( return nil, fmt.Errorf("%w: AutoMLJobName is required", ErrValidation) } - if _, ok := b.autoMLJobsStore(region)[name]; ok { + if _, ok := b.autoMLJobsStore(region).Get(name); ok { return nil, fmt.Errorf("%w: AutoML job %q already exists", ErrValidation, name) } @@ -439,7 +443,7 @@ func (b *InMemoryBackend) CreateAutoMLJob( Tags: mergeTags(nil, tags), CreationTime: time.Now(), } - b.autoMLJobsStore(region)[name] = j + b.autoMLJobsStore(region).Put(j) return cloneAutoMLJob(j), nil } @@ -451,7 +455,7 @@ func (b *InMemoryBackend) DescribeAutoMLJob(ctx context.Context, name string) (* region := getRegion(ctx, b.region) - j, ok := b.autoMLJobsStore(region)[name] + j, ok := b.autoMLJobsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: AutoML job %q not found", ErrAutoMLJobNotFound, name) } @@ -466,7 +470,7 @@ func (b *InMemoryBackend) StopAutoMLJob(ctx context.Context, name string) error region := getRegion(ctx, b.region) - j, ok := b.autoMLJobsStore(region)[name] + j, ok := b.autoMLJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: AutoML job %q not found", ErrAutoMLJobNotFound, name) } @@ -489,7 +493,12 @@ func (b *InMemoryBackend) ListAutoMLJobs(ctx context.Context, nextToken string) region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.autoMLJobsStore(region), nextToken, cloneAutoMLJob) + return sagemakerListKeyPaged( + b.autoMLJobsStore(region), + nextToken, + cloneAutoMLJob, + func(v *AutoMLJob) string { return v.AutoMLJobName }, + ) } // SetAutoMLJobExtras sets optional configuration fields on an existing AutoML job @@ -505,7 +514,7 @@ func (b *InMemoryBackend) SetAutoMLJobExtras( region := getRegion(ctx, b.region) - j, ok := b.autoMLJobsStore(region)[name] + j, ok := b.autoMLJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: AutoML job %q not found", ErrAutoMLJobNotFound, name) } @@ -561,7 +570,7 @@ func (b *InMemoryBackend) CreateCodeRepository( return nil, fmt.Errorf("%w: CodeRepositoryName is required", ErrValidation) } - if _, ok := b.codeRepositoriesStore(region)[name]; ok { + if _, ok := b.codeRepositoriesStore(region).Get(name); ok { return nil, fmt.Errorf("%w: code repository %q already exists", ErrValidation, name) } @@ -576,7 +585,7 @@ func (b *InMemoryBackend) CreateCodeRepository( CreationTime: now, LastModifiedTime: now, } - b.codeRepositoriesStore(region)[name] = r + b.codeRepositoriesStore(region).Put(r) return cloneCodeRepository(r), nil } @@ -588,7 +597,7 @@ func (b *InMemoryBackend) DescribeCodeRepository(ctx context.Context, name strin region := getRegion(ctx, b.region) - r, ok := b.codeRepositoriesStore(region)[name] + r, ok := b.codeRepositoriesStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: code repository %q not found", ErrCodeRepositoryNotFound, name) } @@ -607,7 +616,7 @@ func (b *InMemoryBackend) UpdateCodeRepository( region := getRegion(ctx, b.region) - r, ok := b.codeRepositoriesStore(region)[name] + r, ok := b.codeRepositoriesStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: code repository %q not found", ErrCodeRepositoryNotFound, name) } @@ -628,12 +637,12 @@ func (b *InMemoryBackend) DeleteCodeRepository(ctx context.Context, name string) region := getRegion(ctx, b.region) - if _, ok := b.codeRepositoriesStore(region)[name]; !ok { + if _, ok := b.codeRepositoriesStore(region).Get(name); !ok { return fmt.Errorf("%w: code repository %q not found", ErrCodeRepositoryNotFound, name) } store := b.codeRepositoriesStore(region) - delete(store, name) + store.Delete(name) return nil } @@ -645,7 +654,12 @@ func (b *InMemoryBackend) ListCodeRepositories(ctx context.Context, nextToken st region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.codeRepositoriesStore(region), nextToken, cloneCodeRepository) + return sagemakerListKeyPaged( + b.codeRepositoriesStore(region), + nextToken, + cloneCodeRepository, + func(v *CodeRepository) string { return v.CodeRepositoryName }, + ) } // --------------------------------------------------------------------------- @@ -685,7 +699,7 @@ func (b *InMemoryBackend) CreateProject( return nil, fmt.Errorf("%w: ProjectName is required", ErrValidation) } - if _, ok := b.projectsStore(region)[name]; ok { + if _, ok := b.projectsStore(region).Get(name); ok { return nil, fmt.Errorf("%w: project %q already exists", ErrValidation, name) } @@ -700,7 +714,7 @@ func (b *InMemoryBackend) CreateProject( Tags: mergeTags(nil, tags), CreationTime: time.Now(), } - b.projectsStore(region)[name] = p + b.projectsStore(region).Put(p) return cloneProject(p), nil } @@ -712,7 +726,7 @@ func (b *InMemoryBackend) DescribeProject(ctx context.Context, name string) (*Pr region := getRegion(ctx, b.region) - p, ok := b.projectsStore(region)[name] + p, ok := b.projectsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: project %q not found", ErrProjectNotFound, name) } @@ -727,12 +741,12 @@ func (b *InMemoryBackend) DeleteProject(ctx context.Context, name string) error region := getRegion(ctx, b.region) - if _, ok := b.projectsStore(region)[name]; !ok { + if _, ok := b.projectsStore(region).Get(name); !ok { return fmt.Errorf("%w: project %q not found", ErrProjectNotFound, name) } store := b.projectsStore(region) - delete(store, name) + store.Delete(name) return nil } @@ -744,7 +758,12 @@ func (b *InMemoryBackend) ListProjects(ctx context.Context, nextToken string) ([ region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.projectsStore(region), nextToken, cloneProject) + return sagemakerListKeyPaged( + b.projectsStore(region), + nextToken, + cloneProject, + func(v *Project) string { return v.ProjectName }, + ) } // --------------------------------------------------------------------------- @@ -794,7 +813,7 @@ func (b *InMemoryBackend) CreateSpace( key := spaceKey(domainID, spaceName) - if _, ok := b.spacesStore(region)[key]; ok { + if _, ok := b.spacesStore(region).Get(key); ok { return nil, fmt.Errorf("%w: space %q already exists in domain %q", ErrValidation, spaceName, domainID) } @@ -810,7 +829,7 @@ func (b *InMemoryBackend) CreateSpace( CreationTime: now, LastModifiedTime: now, } - b.spacesStore(region)[key] = s + b.spacesStore(region).Put(s) return cloneSpace(s), nil } @@ -822,7 +841,7 @@ func (b *InMemoryBackend) DescribeSpace(ctx context.Context, domainID, spaceName region := getRegion(ctx, b.region) - s, ok := b.spacesStore(region)[spaceKey(domainID, spaceName)] + s, ok := b.spacesStore(region).Get(spaceKey(domainID, spaceName)) if !ok { return nil, fmt.Errorf("%w: space %q not found in domain %q", ErrSpaceNotFound, spaceName, domainID) } @@ -839,12 +858,12 @@ func (b *InMemoryBackend) DeleteSpace(ctx context.Context, domainID, spaceName s key := spaceKey(domainID, spaceName) - if _, ok := b.spacesStore(region)[key]; !ok { + if _, ok := b.spacesStore(region).Get(key); !ok { return fmt.Errorf("%w: space %q not found in domain %q", ErrSpaceNotFound, spaceName, domainID) } store := b.spacesStore(region) - delete(store, key) + store.Delete(key) return nil } @@ -857,9 +876,9 @@ func (b *InMemoryBackend) ListSpaces(ctx context.Context, domainID, nextToken st region := getRegion(ctx, b.region) var keys []string - for k, s := range b.spacesStore(region) { + for _, s := range b.spacesStore(region).All() { if domainID == "" || s.DomainID == domainID { - keys = append(keys, k) + keys = append(keys, spaceKey(s.DomainID, s.SpaceName)) } } @@ -880,7 +899,7 @@ func (b *InMemoryBackend) ListSpaces(ctx context.Context, domainID, nextToken st out := make([]*Space, 0, end-start) for _, k := range keys[start:end] { - out = append(out, cloneSpace(b.spacesStore(region)[k])) + out = append(out, cloneSpace(tableGet(b.spacesStore(region), k))) } next := "" @@ -930,7 +949,7 @@ func (b *InMemoryBackend) CreateImage( return nil, fmt.Errorf("%w: ImageName is required", ErrValidation) } - if _, ok := b.smImagesStore(region)[name]; ok { + if _, ok := b.smImagesStore(region).Get(name); ok { return nil, fmt.Errorf("%w: image %q already exists", ErrValidation, name) } @@ -947,7 +966,7 @@ func (b *InMemoryBackend) CreateImage( CreationTime: now, LastModifiedTime: now, } - b.smImagesStore(region)[name] = img + b.smImagesStore(region).Put(img) return cloneSMImage(img), nil } @@ -959,7 +978,7 @@ func (b *InMemoryBackend) DescribeImage(ctx context.Context, name string) (*SMIm region := getRegion(ctx, b.region) - img, ok := b.smImagesStore(region)[name] + img, ok := b.smImagesStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: image %q not found", ErrSMImageNotFound, name) } @@ -989,7 +1008,7 @@ func (b *InMemoryBackend) UpdateImage( region := getRegion(ctx, b.region) - img, ok := b.smImagesStore(region)[name] + img, ok := b.smImagesStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: image %q not found", ErrSMImageNotFound, name) } @@ -1029,7 +1048,7 @@ func (b *InMemoryBackend) DeleteImage(ctx context.Context, name string) error { region := getRegion(ctx, b.region) - if _, ok := b.smImagesStore(region)[name]; !ok { + if _, ok := b.smImagesStore(region).Get(name); !ok { return fmt.Errorf("%w: image %q not found", ErrSMImageNotFound, name) } @@ -1039,7 +1058,7 @@ func (b *InMemoryBackend) DeleteImage(ctx context.Context, name string) error { } store := b.smImagesStore(region) - delete(store, name) + store.Delete(name) return nil } @@ -1051,7 +1070,12 @@ func (b *InMemoryBackend) ListImages(ctx context.Context, nextToken string) ([]* region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.smImagesStore(region), nextToken, cloneSMImage) + return sagemakerListKeyPaged( + b.smImagesStore(region), + nextToken, + cloneSMImage, + func(v *SMImage) string { return v.ImageName }, + ) } // --------------------------------------------------------------------------- @@ -1090,7 +1114,7 @@ func (b *InMemoryBackend) CreateImageVersion(ctx context.Context, imageName stri region := getRegion(ctx, b.region) - img, ok := b.smImagesStore(region)[imageName] + img, ok := b.smImagesStore(region).Get(imageName) if !ok { return nil, fmt.Errorf("%w: image %q not found", ErrSMImageNotFound, imageName) } @@ -1391,7 +1415,7 @@ func (b *InMemoryBackend) CreateCompilationJob( return nil, fmt.Errorf("%w: CompilationJobName is required", ErrValidation) } - if _, ok := b.compilationJobsStore(region)[name]; ok { + if _, ok := b.compilationJobsStore(region).Get(name); ok { return nil, fmt.Errorf("%w: compilation job %q already exists", ErrValidation, name) } @@ -1407,7 +1431,7 @@ func (b *InMemoryBackend) CreateCompilationJob( CreationTime: now, LastModifiedTime: now, } - b.compilationJobsStore(region)[name] = j + b.compilationJobsStore(region).Put(j) return cloneCompilationJob(j), nil } @@ -1419,7 +1443,7 @@ func (b *InMemoryBackend) DescribeCompilationJob(ctx context.Context, name strin region := getRegion(ctx, b.region) - j, ok := b.compilationJobsStore(region)[name] + j, ok := b.compilationJobsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: compilation job %q not found", ErrCompilationJobNotFound, name) } @@ -1434,12 +1458,12 @@ func (b *InMemoryBackend) DeleteCompilationJob(ctx context.Context, name string) region := getRegion(ctx, b.region) - if _, ok := b.compilationJobsStore(region)[name]; !ok { + if _, ok := b.compilationJobsStore(region).Get(name); !ok { return fmt.Errorf("%w: compilation job %q not found", ErrCompilationJobNotFound, name) } store := b.compilationJobsStore(region) - delete(store, name) + store.Delete(name) return nil } @@ -1451,7 +1475,7 @@ func (b *InMemoryBackend) StopCompilationJob(ctx context.Context, name string) e region := getRegion(ctx, b.region) - j, ok := b.compilationJobsStore(region)[name] + j, ok := b.compilationJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: compilation job %q not found", ErrCompilationJobNotFound, name) } @@ -1475,7 +1499,12 @@ func (b *InMemoryBackend) ListCompilationJobs(ctx context.Context, nextToken str region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.compilationJobsStore(region), nextToken, cloneCompilationJob) + return sagemakerListKeyPaged( + b.compilationJobsStore(region), + nextToken, + cloneCompilationJob, + func(v *CompilationJob) string { return v.CompilationJobName }, + ) } // SetCompilationJobExtras sets optional configuration fields on an existing compilation job @@ -1492,7 +1521,7 @@ func (b *InMemoryBackend) SetCompilationJobExtras( region := getRegion(ctx, b.region) - j, ok := b.compilationJobsStore(region)[name] + j, ok := b.compilationJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: compilation job %q not found", ErrCompilationJobNotFound, name) } @@ -1553,7 +1582,7 @@ func (b *InMemoryBackend) CreateMonitoringSchedule( return nil, fmt.Errorf("%w: MonitoringScheduleName is required", ErrValidation) } - if _, ok := b.monitoringSchedulesStore(region)[name]; ok { + if _, ok := b.monitoringSchedulesStore(region).Get(name); ok { return nil, fmt.Errorf("%w: monitoring schedule %q already exists", ErrValidation, name) } @@ -1568,7 +1597,7 @@ func (b *InMemoryBackend) CreateMonitoringSchedule( CreationTime: now, LastModifiedTime: now, } - b.monitoringSchedulesStore(region)[name] = ms + b.monitoringSchedulesStore(region).Put(ms) return cloneMonitoringSchedule(ms), nil } @@ -1580,7 +1609,7 @@ func (b *InMemoryBackend) DescribeMonitoringSchedule(ctx context.Context, name s region := getRegion(ctx, b.region) - ms, ok := b.monitoringSchedulesStore(region)[name] + ms, ok := b.monitoringSchedulesStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: monitoring schedule %q not found", ErrMonitoringScheduleNotFound, name) } @@ -1595,12 +1624,12 @@ func (b *InMemoryBackend) DeleteMonitoringSchedule(ctx context.Context, name str region := getRegion(ctx, b.region) - if _, ok := b.monitoringSchedulesStore(region)[name]; !ok { + if _, ok := b.monitoringSchedulesStore(region).Get(name); !ok { return fmt.Errorf("%w: monitoring schedule %q not found", ErrMonitoringScheduleNotFound, name) } store := b.monitoringSchedulesStore(region) - delete(store, name) + store.Delete(name) return nil } @@ -1612,7 +1641,7 @@ func (b *InMemoryBackend) StopMonitoringSchedule(ctx context.Context, name strin region := getRegion(ctx, b.region) - ms, ok := b.monitoringSchedulesStore(region)[name] + ms, ok := b.monitoringSchedulesStore(region).Get(name) if !ok { return fmt.Errorf("%w: monitoring schedule %q not found", ErrMonitoringScheduleNotFound, name) } @@ -1635,7 +1664,7 @@ func (b *InMemoryBackend) StartMonitoringSchedule(ctx context.Context, name stri region := getRegion(ctx, b.region) - ms, ok := b.monitoringSchedulesStore(region)[name] + ms, ok := b.monitoringSchedulesStore(region).Get(name) if !ok { return fmt.Errorf("%w: monitoring schedule %q not found", ErrMonitoringScheduleNotFound, name) } @@ -1659,7 +1688,7 @@ func (b *InMemoryBackend) UpdateMonitoringSchedule(ctx context.Context, name str region := getRegion(ctx, b.region) - ms, ok := b.monitoringSchedulesStore(region)[name] + ms, ok := b.monitoringSchedulesStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: monitoring schedule %q not found", ErrMonitoringScheduleNotFound, name) } @@ -1679,7 +1708,12 @@ func (b *InMemoryBackend) ListMonitoringSchedules( region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.monitoringSchedulesStore(region), nextToken, cloneMonitoringSchedule) + return sagemakerListKeyPaged( + b.monitoringSchedulesStore(region), + nextToken, + cloneMonitoringSchedule, + func(v *MonitoringSchedule) string { return v.MonitoringScheduleName }, + ) } // --------------------------------------------------------------------------- @@ -1746,14 +1780,14 @@ func (b *InMemoryBackend) CreateWorkteam(ctx context.Context, opts CreateWorktea return nil, fmt.Errorf("%w: WorkteamName is required", ErrValidation) } - if _, ok := b.workteamsStore(region)[opts.Name]; ok { + if _, ok := b.workteamsStore(region).Get(opts.Name); ok { return nil, fmt.Errorf("%w: workteam %q already exists", ErrValidation, opts.Name) } var workforceARN string if opts.WorkforceName != "" { - wf, ok := b.workforcesStore(region)[opts.WorkforceName] + wf, ok := b.workforcesStore(region).Get(opts.WorkforceName) if !ok { return nil, fmt.Errorf( "%w: workforce %q not found", ErrWorkforceNotFound, opts.WorkforceName, @@ -1777,7 +1811,7 @@ func (b *InMemoryBackend) CreateWorkteam(ctx context.Context, opts CreateWorktea CreateDate: now, LastUpdatedDate: now, } - b.workteamsStore(region)[opts.Name] = w + b.workteamsStore(region).Put(w) return cloneWorkteam(w), nil } @@ -1796,7 +1830,7 @@ func (b *InMemoryBackend) UpdateWorkteam(ctx context.Context, opts UpdateWorktea region := getRegion(ctx, b.region) - w, ok := b.workteamsStore(region)[opts.Name] + w, ok := b.workteamsStore(region).Get(opts.Name) if !ok { return nil, fmt.Errorf("%w: workteam %q not found", ErrWorkteamNotFound, opts.Name) } @@ -1821,7 +1855,7 @@ func (b *InMemoryBackend) DescribeWorkteam(ctx context.Context, name string) (*W region := getRegion(ctx, b.region) - w, ok := b.workteamsStore(region)[name] + w, ok := b.workteamsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: workteam %q not found", ErrWorkteamNotFound, name) } @@ -1836,12 +1870,12 @@ func (b *InMemoryBackend) DeleteWorkteam(ctx context.Context, name string) error region := getRegion(ctx, b.region) - if _, ok := b.workteamsStore(region)[name]; !ok { + if _, ok := b.workteamsStore(region).Get(name); !ok { return fmt.Errorf("%w: workteam %q not found", ErrWorkteamNotFound, name) } store := b.workteamsStore(region) - delete(store, name) + store.Delete(name) return nil } @@ -1853,5 +1887,10 @@ func (b *InMemoryBackend) ListWorkteams(ctx context.Context, nextToken string) ( region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.workteamsStore(region), nextToken, cloneWorkteam) + return sagemakerListKeyPaged( + b.workteamsStore(region), + nextToken, + cloneWorkteam, + func(v *Workteam) string { return v.WorkteamName }, + ) } diff --git a/services/sagemaker/backend_batch3.go b/services/sagemaker/backend_batch3.go index 07486535e..a6675c43d 100644 --- a/services/sagemaker/backend_batch3.go +++ b/services/sagemaker/backend_batch3.go @@ -11,6 +11,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awserr" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) // --------------------------------------------------------------------------- @@ -97,7 +98,7 @@ func cloneJobDefinition(j *JobDefinition) *JobDefinition { // per-region map's lazy initialisation stays race-free. func (b *InMemoryBackend) createJobDefinition( ctx context.Context, - storeFn func(string) map[string]*JobDefinition, + storeFn func(string) *store.Table[JobDefinition], defType, name, roleArn, endpointName string, config map[string]json.RawMessage, tags map[string]string, @@ -113,8 +114,8 @@ func (b *InMemoryBackend) createJobDefinition( return nil, fmt.Errorf("%w: %sJobDefinitionName is required", ErrValidation, defType) } - store := storeFn(region) - if _, ok := store[name]; ok { + tbl := storeFn(region) + if _, ok := tbl.Get(name); ok { return nil, fmt.Errorf("%w: %s job definition %q already exists", alreadyExists, defType, name) } @@ -130,14 +131,14 @@ func (b *InMemoryBackend) createJobDefinition( Config: config, CreationTime: time.Now(), } - store[name] = j + tbl.Put(j) return cloneJobDefinition(j), nil } func (b *InMemoryBackend) describeJobDefinition( ctx context.Context, - storeFn func(string) map[string]*JobDefinition, + storeFn func(string) *store.Table[JobDefinition], name string, notFound error, ) (*JobDefinition, error) { @@ -146,7 +147,7 @@ func (b *InMemoryBackend) describeJobDefinition( b.mu.RLock("describeJobDefinition") defer b.mu.RUnlock() - j, ok := storeFn(region)[name] + j, ok := storeFn(region).Get(name) if !ok { return nil, fmt.Errorf("%w: job definition %q not found", notFound, name) } @@ -156,7 +157,7 @@ func (b *InMemoryBackend) describeJobDefinition( func (b *InMemoryBackend) deleteJobDefinition( ctx context.Context, - storeFn func(string) map[string]*JobDefinition, + storeFn func(string) *store.Table[JobDefinition], name string, notFound error, ) error { @@ -165,12 +166,12 @@ func (b *InMemoryBackend) deleteJobDefinition( b.mu.Lock("deleteJobDefinition") defer b.mu.Unlock() - store := storeFn(region) - if _, ok := store[name]; !ok { + tbl := storeFn(region) + if _, ok := tbl.Get(name); !ok { return fmt.Errorf("%w: job definition %q not found", notFound, name) } - delete(store, name) + tbl.Delete(name) return nil } @@ -228,15 +229,15 @@ func sortJobDefinitions(list []*JobDefinition, sortBy, sortOrder string) { // definition types. storeFn is called only while b.mu is held (by the // per-type List* wrapper). func (b *InMemoryBackend) listJobDefinitions( - storeFn func(string) map[string]*JobDefinition, + storeFn func(string) *store.Table[JobDefinition], region, nextToken string, f JobDefinitionFilter, ) ([]*JobDefinition, string) { - store := storeFn(region) + items := storeFn(region).All() - list := make([]*JobDefinition, 0, len(store)) + list := make([]*JobDefinition, 0, len(items)) - for _, j := range store { + for _, j := range items { if matchesJobDefinitionFilter(j, f) { list = append(list, cloneJobDefinition(j)) } @@ -432,21 +433,28 @@ func cloneMonitoringAlert(a *MonitoringAlert) *MonitoringAlert { return &cp } -// monitoringAlertsFor returns (creating if necessary) the alert map for a -// monitoring schedule. Callers must hold b.mu (any lock kind for read-only -// lookups that tolerate a nil result; b.mu.Lock for creation). -func (b *InMemoryBackend) monitoringAlertsFor(region, scheduleName string) map[string]*MonitoringAlert { - perSchedule := b.monitoringAlerts[region] - if perSchedule == nil { - perSchedule = make(map[string]map[string]*MonitoringAlert) - b.monitoringAlerts[region] = perSchedule - } - - if perSchedule[scheduleName] == nil { - perSchedule[scheduleName] = make(map[string]*MonitoringAlert) +// monitoringAlertKey builds the store.Table primary key for a monitoring +// alert: its schedule name and alert name are already unique together within +// a region (an alert belongs to exactly one schedule). +func monitoringAlertKey(scheduleName, alertName string) string { + return scheduleName + "/" + alertName +} + +// monitoringAlertsStore returns (registering if necessary) the per-region +// store.Table of MonitoringAlert, keyed by monitoringAlertKey(scheduleName, +// alertName). Callers must hold b.mu. +func (b *InMemoryBackend) monitoringAlertsStore(r string) *store.Table[MonitoringAlert] { + if b.monitoringAlerts[r] == nil { + b.monitoringAlerts[r] = store.Register( + b.registry, + "monitoringAlerts:"+r, + store.New(func(v *MonitoringAlert) string { + return monitoringAlertKey(v.MonitoringScheduleName, v.MonitoringAlertName) + }), + ) } - return perSchedule[scheduleName] + return b.monitoringAlerts[r] } // UpdateMonitoringAlert updates the datapoints/evaluation-period configuration @@ -462,15 +470,15 @@ func (b *InMemoryBackend) UpdateMonitoringAlert( b.mu.Lock("UpdateMonitoringAlert") defer b.mu.Unlock() - sched, ok := b.monitoringSchedulesStore(region)[scheduleName] + sched, ok := b.monitoringSchedulesStore(region).Get(scheduleName) if !ok { return nil, "", fmt.Errorf("%w: monitoring schedule %q not found", ErrMonitoringScheduleNotFound, scheduleName) } - alerts := b.monitoringAlertsFor(region, scheduleName) + tbl := b.monitoringAlertsStore(region) now := time.Now() - a, ok := alerts[alertName] + a, ok := tbl.Get(monitoringAlertKey(scheduleName, alertName)) if !ok { a = &MonitoringAlert{ MonitoringScheduleName: scheduleName, @@ -478,7 +486,7 @@ func (b *InMemoryBackend) UpdateMonitoringAlert( AlertStatus: "OK", CreationTime: now, } - alerts[alertName] = a + tbl.Put(a) } a.DatapointsToAlert = datapointsToAlert @@ -498,16 +506,22 @@ func (b *InMemoryBackend) ListMonitoringAlerts( b.mu.RLock("ListMonitoringAlerts") defer b.mu.RUnlock() - if _, ok := b.monitoringSchedulesStore(region)[scheduleName]; !ok { + if _, ok := b.monitoringSchedulesStore(region).Get(scheduleName); !ok { return nil, "", fmt.Errorf("%w: monitoring schedule %q not found", ErrMonitoringScheduleNotFound, scheduleName) } - var alerts map[string]*MonitoringAlert - if perSchedule := b.monitoringAlerts[region]; perSchedule != nil { - alerts = perSchedule[scheduleName] + // Rebuild a by-alert-name map scoped to this schedule, matching the + // pre-conversion per-schedule map's exact key shape, so pagination tokens + // (alert names) behave identically. + alerts := make(map[string]*MonitoringAlert) + + for _, a := range b.monitoringAlertsStore(region).All() { + if a.MonitoringScheduleName == scheduleName { + alerts[a.MonitoringAlertName] = a + } } - items, next := sagemakerListKeyPaged(alerts, nextToken, cloneMonitoringAlert) + items, next := sagemakerListKeyPagedMap(alerts, nextToken, cloneMonitoringAlert) return items, next, nil } @@ -717,9 +731,9 @@ func (b *InMemoryBackend) ListMonitoringExecutions( b.mu.RLock("ListMonitoringExecutions") defer b.mu.RUnlock() - list := make([]*MonitoringExecution, 0, len(b.monitoringExecutions[region])) + list := make([]*MonitoringExecution, 0, b.monitoringExecutionsStore(region).Len()) - for _, e := range b.monitoringExecutions[region] { + for _, e := range b.monitoringExecutionsStore(region).All() { if matchesMonitoringExecutionFilter(e, f) { list = append(list, cloneMonitoringExecution(e)) } @@ -775,7 +789,7 @@ func (b *InMemoryBackend) CreateHumanTaskUI( store := b.humanTaskUisStore(region) - if _, ok := store[name]; ok { + if _, ok := store.Get(name); ok { return nil, fmt.Errorf("%w: human task UI %q already exists", ErrValidation, name) } @@ -788,7 +802,7 @@ func (b *InMemoryBackend) CreateHumanTaskUI( Tags: mergeTags(nil, tags), CreationTime: time.Now(), } - store[name] = ui + store.Put(ui) return cloneHumanTaskUI(ui), nil } @@ -800,7 +814,7 @@ func (b *InMemoryBackend) DescribeHumanTaskUI(ctx context.Context, name string) b.mu.RLock("DescribeHumanTaskUI") defer b.mu.RUnlock() - ui, ok := b.humanTaskUisStore(region)[name] + ui, ok := b.humanTaskUisStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: human task UI %q not found", ErrHumanTaskUINotFound, name) } @@ -815,7 +829,7 @@ func (b *InMemoryBackend) HumanTaskUIExistsByARN(ctx context.Context, humanTaskU b.mu.RLock("HumanTaskUIExistsByARN") defer b.mu.RUnlock() - for _, ui := range b.humanTaskUisStore(region) { + for _, ui := range b.humanTaskUisStore(region).All() { if ui.HumanTaskUIArn == humanTaskUIArn { return true } @@ -833,11 +847,11 @@ func (b *InMemoryBackend) DeleteHumanTaskUI(ctx context.Context, name string) er store := b.humanTaskUisStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf("%w: human task UI %q not found", ErrHumanTaskUINotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -951,11 +965,11 @@ func (b *InMemoryBackend) CreateWorkforce(ctx context.Context, opts CreateWorkfo store := b.workforcesStore(region) - if _, ok := store[opts.Name]; ok { + if _, ok := store.Get(opts.Name); ok { return nil, fmt.Errorf("%w: workforce %q already exists", ErrValidation, opts.Name) } - if len(store) > 0 { + if store.Len() > 0 { return nil, fmt.Errorf( "%w: only one workforce is allowed per Amazon Web Services account per Amazon Web Services Region", ErrValidation, @@ -985,7 +999,7 @@ func (b *InMemoryBackend) CreateWorkforce(ctx context.Context, opts CreateWorkfo w.WorkforceVpcConfig.VpcEndpointID = "vpce-" + generateID()[:17] } - store[opts.Name] = w + store.Put(w) return cloneWorkforce(w), nil } @@ -997,7 +1011,7 @@ func (b *InMemoryBackend) DescribeWorkforce(ctx context.Context, name string) (* b.mu.RLock("DescribeWorkforce") defer b.mu.RUnlock() - w, ok := b.workforcesStore(region)[name] + w, ok := b.workforcesStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: workforce %q not found", ErrWorkforceNotFound, name) } @@ -1020,7 +1034,7 @@ func (b *InMemoryBackend) UpdateWorkforce(ctx context.Context, opts UpdateWorkfo b.mu.Lock("UpdateWorkforce") defer b.mu.Unlock() - w, ok := b.workforcesStore(region)[opts.Name] + w, ok := b.workforcesStore(region).Get(opts.Name) if !ok { return nil, fmt.Errorf("%w: workforce %q not found", ErrWorkforceNotFound, opts.Name) } @@ -1052,12 +1066,12 @@ func (b *InMemoryBackend) DeleteWorkforce(ctx context.Context, name string) erro b.mu.Lock("DeleteWorkforce") defer b.mu.Unlock() - w, ok := b.workforcesStore(region)[name] + w, ok := b.workforcesStore(region).Get(name) if !ok { return fmt.Errorf("%w: workforce %q not found", ErrWorkforceNotFound, name) } - for _, wt := range b.workteamsStore(region) { + for _, wt := range b.workteamsStore(region).All() { if wt.WorkforceArn == w.WorkforceArn { return fmt.Errorf( "%w: workforce %q still has associated work teams", ErrWorkteamInUse, name, @@ -1065,7 +1079,7 @@ func (b *InMemoryBackend) DeleteWorkforce(ctx context.Context, name string) erro } } - delete(b.workforcesStore(region), name) + b.workforcesStore(region).Delete(name) return nil } @@ -1078,7 +1092,12 @@ func (b *InMemoryBackend) ListWorkforces(ctx context.Context, nextToken string) b.mu.RLock("ListWorkforces") defer b.mu.RUnlock() - return sagemakerListKeyPaged(b.workforcesStore(region), nextToken, cloneWorkforce) + return sagemakerListKeyPaged( + b.workforcesStore(region), + nextToken, + cloneWorkforce, + func(v *Workforce) string { return v.WorkforceName }, + ) } // --------------------------------------------------------------------------- @@ -1119,7 +1138,7 @@ func (b *InMemoryBackend) CreateFlowDefinition( store := b.flowDefinitionsStore(region) - if _, ok := store[name]; ok { + if _, ok := store.Get(name); ok { return nil, fmt.Errorf("%w: flow definition %q already exists", ErrValidation, name) } @@ -1133,7 +1152,7 @@ func (b *InMemoryBackend) CreateFlowDefinition( Tags: mergeTags(nil, tags), CreationTime: time.Now(), } - store[name] = f + store.Put(f) return cloneFlowDefinition(f), nil } @@ -1145,7 +1164,7 @@ func (b *InMemoryBackend) DescribeFlowDefinition(ctx context.Context, name strin b.mu.RLock("DescribeFlowDefinition") defer b.mu.RUnlock() - f, ok := b.flowDefinitionsStore(region)[name] + f, ok := b.flowDefinitionsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: flow definition %q not found", ErrFlowDefinitionNotFound, name) } @@ -1162,11 +1181,11 @@ func (b *InMemoryBackend) DeleteFlowDefinition(ctx context.Context, name string) store := b.flowDefinitionsStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf("%w: flow definition %q not found", ErrFlowDefinitionNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -1208,7 +1227,7 @@ func (b *InMemoryBackend) CreateAppImageConfig( store := b.appImageConfigsStore(region) - if _, ok := store[name]; ok { + if _, ok := store.Get(name); ok { return nil, fmt.Errorf("%w: app image config %q already exists", ErrValidation, name) } @@ -1222,7 +1241,7 @@ func (b *InMemoryBackend) CreateAppImageConfig( CreationTime: now, LastModifiedTime: now, } - store[name] = a + store.Put(a) return cloneAppImageConfig(a), nil } @@ -1234,7 +1253,7 @@ func (b *InMemoryBackend) DescribeAppImageConfig(ctx context.Context, name strin b.mu.RLock("DescribeAppImageConfig") defer b.mu.RUnlock() - a, ok := b.appImageConfigsStore(region)[name] + a, ok := b.appImageConfigsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: app image config %q not found", ErrAppImageConfigNotFound, name) } @@ -1249,7 +1268,7 @@ func (b *InMemoryBackend) UpdateAppImageConfig(ctx context.Context, name string) b.mu.Lock("UpdateAppImageConfig") defer b.mu.Unlock() - a, ok := b.appImageConfigsStore(region)[name] + a, ok := b.appImageConfigsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: app image config %q not found", ErrAppImageConfigNotFound, name) } @@ -1268,11 +1287,11 @@ func (b *InMemoryBackend) DeleteAppImageConfig(ctx context.Context, name string) store := b.appImageConfigsStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf("%w: app image config %q not found", ErrAppImageConfigNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -1338,7 +1357,7 @@ func (b *InMemoryBackend) DescribeInferenceExperiment(ctx context.Context, name b.mu.RLock("DescribeInferenceExperiment") defer b.mu.RUnlock() - e, ok := b.inferenceExperimentsStore(region)[name] + e, ok := b.inferenceExperimentsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: inference experiment %q not found", ErrInferenceExperimentNotFound, name) } @@ -1353,7 +1372,7 @@ func (b *InMemoryBackend) StopInferenceExperiment(ctx context.Context, name stri b.mu.Lock("StopInferenceExperiment") defer b.mu.Unlock() - e, ok := b.inferenceExperimentsStore(region)[name] + e, ok := b.inferenceExperimentsStore(region).Get(name) if !ok { return fmt.Errorf("%w: inference experiment %q not found", ErrInferenceExperimentNotFound, name) } @@ -1371,7 +1390,7 @@ func (b *InMemoryBackend) StartInferenceExperiment(ctx context.Context, name str b.mu.Lock("StartInferenceExperiment") defer b.mu.Unlock() - e, ok := b.inferenceExperimentsStore(region)[name] + e, ok := b.inferenceExperimentsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: inference experiment %q not found", ErrInferenceExperimentNotFound, name) } @@ -1392,7 +1411,7 @@ func (b *InMemoryBackend) UpdateInferenceExperiment( b.mu.Lock("UpdateInferenceExperiment") defer b.mu.Unlock() - e, ok := b.inferenceExperimentsStore(region)[name] + e, ok := b.inferenceExperimentsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: inference experiment %q not found", ErrInferenceExperimentNotFound, name) } @@ -1415,11 +1434,11 @@ func (b *InMemoryBackend) DeleteInferenceExperiment(ctx context.Context, name st store := b.inferenceExperimentsStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf("%w: inference experiment %q not found", ErrInferenceExperimentNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -1487,7 +1506,7 @@ func (b *InMemoryBackend) DescribeMlflowTrackingServer( b.mu.RLock("DescribeMlflowTrackingServer") defer b.mu.RUnlock() - s, ok := b.mlflowTrackingServersStore(region)[name] + s, ok := b.mlflowTrackingServersStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: MLflow tracking server %q not found", ErrMlflowTrackingServerNotFound, name) } @@ -1504,11 +1523,11 @@ func (b *InMemoryBackend) DeleteMlflowTrackingServer(ctx context.Context, name s store := b.mlflowTrackingServersStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf("%w: MLflow tracking server %q not found", ErrMlflowTrackingServerNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -1520,7 +1539,7 @@ func (b *InMemoryBackend) StartMlflowTrackingServer(ctx context.Context, name st b.mu.Lock("StartMlflowTrackingServer") defer b.mu.Unlock() - s, ok := b.mlflowTrackingServersStore(region)[name] + s, ok := b.mlflowTrackingServersStore(region).Get(name) if !ok { return fmt.Errorf("%w: MLflow tracking server %q not found", ErrMlflowTrackingServerNotFound, name) } @@ -1538,7 +1557,7 @@ func (b *InMemoryBackend) StopMlflowTrackingServer(ctx context.Context, name str b.mu.Lock("StopMlflowTrackingServer") defer b.mu.Unlock() - s, ok := b.mlflowTrackingServersStore(region)[name] + s, ok := b.mlflowTrackingServersStore(region).Get(name) if !ok { return fmt.Errorf("%w: MLflow tracking server %q not found", ErrMlflowTrackingServerNotFound, name) } @@ -1557,7 +1576,7 @@ func (b *InMemoryBackend) CreatePresignedMlflowTrackingServerURL(ctx context.Con b.mu.RLock("CreatePresignedMlflowTrackingServerURL") defer b.mu.RUnlock() - if _, ok := b.mlflowTrackingServersStore(region)[name]; !ok { + if _, ok := b.mlflowTrackingServersStore(region).Get(name); !ok { return "", fmt.Errorf("%w: MLflow tracking server %q not found", ErrMlflowTrackingServerNotFound, name) } @@ -1618,7 +1637,7 @@ func (b *InMemoryBackend) CreateMlflowApp(ctx context.Context, opts CreateMlflow appARN := arn.Build("sagemaker", region, b.accountID, "mlflow-app/"+opts.Name) store := b.mlflowAppsStore(region) - if _, ok := store[appARN]; ok { + if _, ok := store.Get(appARN); ok { return nil, sagemakerDupErr("MLflow App", opts.Name) } @@ -1636,7 +1655,7 @@ func (b *InMemoryBackend) CreateMlflowApp(ctx context.Context, opts CreateMlflow CreationTime: now, LastModifiedTime: now, } - store[appARN] = m + store.Put(m) return cloneMlflowApp(m), nil } @@ -1648,7 +1667,7 @@ func (b *InMemoryBackend) DescribeMlflowApp(ctx context.Context, arnStr string) b.mu.RLock("DescribeMlflowApp") defer b.mu.RUnlock() - m, ok := b.mlflowAppsStore(region)[arnStr] + m, ok := b.mlflowAppsStore(region).Get(arnStr) if !ok { return nil, fmt.Errorf("%w: MLflow App %q not found", ErrMlflowAppNotFound, arnStr) } @@ -1665,11 +1684,11 @@ func (b *InMemoryBackend) DeleteMlflowApp(ctx context.Context, arnStr string) er store := b.mlflowAppsStore(region) - if _, ok := store[arnStr]; !ok { + if _, ok := store.Get(arnStr); !ok { return fmt.Errorf("%w: MLflow App %q not found", ErrMlflowAppNotFound, arnStr) } - delete(store, arnStr) + store.Delete(arnStr) return nil } @@ -1690,7 +1709,7 @@ func (b *InMemoryBackend) UpdateMlflowApp(ctx context.Context, opts UpdateMlflow b.mu.Lock("UpdateMlflowApp") defer b.mu.Unlock() - m, ok := b.mlflowAppsStore(region)[opts.Arn] + m, ok := b.mlflowAppsStore(region).Get(opts.Arn) if !ok { return nil, fmt.Errorf("%w: MLflow App %q not found", ErrMlflowAppNotFound, opts.Arn) } @@ -1723,7 +1742,12 @@ func (b *InMemoryBackend) ListMlflowApps(ctx context.Context, nextToken string) b.mu.RLock("ListMlflowApps") defer b.mu.RUnlock() - return sagemakerListKeyPaged(b.mlflowAppsStore(region), nextToken, cloneMlflowApp) + return sagemakerListKeyPaged( + b.mlflowAppsStore(region), + nextToken, + cloneMlflowApp, + func(v *MlflowApp) string { return v.Arn }, + ) } // CreatePresignedMlflowAppURL returns a one-time presigned URL for accessing @@ -1734,7 +1758,7 @@ func (b *InMemoryBackend) CreatePresignedMlflowAppURL(ctx context.Context, arnSt b.mu.RLock("CreatePresignedMlflowAppURL") defer b.mu.RUnlock() - m, ok := b.mlflowAppsStore(region)[arnStr] + m, ok := b.mlflowAppsStore(region).Get(arnStr) if !ok { return "", fmt.Errorf("%w: MLflow App %q not found", ErrMlflowAppNotFound, arnStr) } @@ -1783,7 +1807,7 @@ func (b *InMemoryBackend) CreateModelCard( store := b.modelCardsStore(region) - if _, ok := store[name]; ok { + if _, ok := store.Get(name); ok { return nil, fmt.Errorf("%w: model card %q already exists", ErrValidation, name) } @@ -1800,7 +1824,7 @@ func (b *InMemoryBackend) CreateModelCard( CreationTime: now, LastModifiedTime: now, } - store[name] = c + store.Put(c) return cloneModelCard(c), nil } @@ -1812,7 +1836,7 @@ func (b *InMemoryBackend) DescribeModelCard(ctx context.Context, name string) (* b.mu.RLock("DescribeModelCard") defer b.mu.RUnlock() - c, ok := b.modelCardsStore(region)[name] + c, ok := b.modelCardsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: model card %q not found", ErrModelCardNotFound, name) } @@ -1827,7 +1851,7 @@ func (b *InMemoryBackend) UpdateModelCard(ctx context.Context, name, content str b.mu.Lock("UpdateModelCard") defer b.mu.Unlock() - c, ok := b.modelCardsStore(region)[name] + c, ok := b.modelCardsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: model card %q not found", ErrModelCardNotFound, name) } @@ -1848,11 +1872,11 @@ func (b *InMemoryBackend) DeleteModelCard(ctx context.Context, name string) erro store := b.modelCardsStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf("%w: model card %q not found", ErrModelCardNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -1896,7 +1920,7 @@ func (b *InMemoryBackend) CreateOptimizationJob( store := b.optimizationJobsStore(region) - if _, ok := store[name]; ok { + if _, ok := store.Get(name); ok { return nil, fmt.Errorf("%w: optimization job %q already exists", ErrValidation, name) } @@ -1912,7 +1936,7 @@ func (b *InMemoryBackend) CreateOptimizationJob( CreationTime: now, LastModifiedTime: now, } - store[name] = j + store.Put(j) return cloneOptimizationJob(j), nil } @@ -1924,7 +1948,7 @@ func (b *InMemoryBackend) DescribeOptimizationJob(ctx context.Context, name stri b.mu.RLock("DescribeOptimizationJob") defer b.mu.RUnlock() - j, ok := b.optimizationJobsStore(region)[name] + j, ok := b.optimizationJobsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: optimization job %q not found", ErrOptimizationJobNotFound, name) } @@ -1941,11 +1965,11 @@ func (b *InMemoryBackend) DeleteOptimizationJob(ctx context.Context, name string store := b.optimizationJobsStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf("%w: optimization job %q not found", ErrOptimizationJobNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -1957,7 +1981,7 @@ func (b *InMemoryBackend) StopOptimizationJob(ctx context.Context, name string) b.mu.Lock("StopOptimizationJob") defer b.mu.Unlock() - j, ok := b.optimizationJobsStore(region)[name] + j, ok := b.optimizationJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: optimization job %q not found", ErrOptimizationJobNotFound, name) } @@ -2006,7 +2030,7 @@ func (b *InMemoryBackend) CreateStudioLifecycleConfig( store := b.studioLifecycleConfigsStore(region) - if _, ok := store[name]; ok { + if _, ok := store.Get(name); ok { return nil, fmt.Errorf("%w: Studio lifecycle config %q already exists", ErrValidation, name) } @@ -2021,7 +2045,7 @@ func (b *InMemoryBackend) CreateStudioLifecycleConfig( CreationTime: now, LastModifiedTime: now, } - store[name] = s + store.Put(s) return cloneStudioLifecycleConfig(s), nil } @@ -2036,7 +2060,7 @@ func (b *InMemoryBackend) DescribeStudioLifecycleConfig( b.mu.RLock("DescribeStudioLifecycleConfig") defer b.mu.RUnlock() - s, ok := b.studioLifecycleConfigsStore(region)[name] + s, ok := b.studioLifecycleConfigsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: Studio lifecycle config %q not found", ErrStudioLifecycleConfigNotFound, name) } @@ -2053,11 +2077,11 @@ func (b *InMemoryBackend) DeleteStudioLifecycleConfig(ctx context.Context, name store := b.studioLifecycleConfigsStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf("%w: Studio lifecycle config %q not found", ErrStudioLifecycleConfigNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -2115,7 +2139,7 @@ func (b *InMemoryBackend) CreatePartnerApp(ctx context.Context, opts CreatePartn store := b.partnerAppsStore(region) - if _, ok := store[appARN]; ok { + if _, ok := store.Get(appARN); ok { return nil, fmt.Errorf("%w: partner app %q already exists", ErrValidation, opts.Name) } @@ -2133,7 +2157,7 @@ func (b *InMemoryBackend) CreatePartnerApp(ctx context.Context, opts CreatePartn CreationTime: now, LastModifiedTime: now, } - store[appARN] = p + store.Put(p) return clonePartnerApp(p), nil } @@ -2145,7 +2169,7 @@ func (b *InMemoryBackend) DescribePartnerApp(ctx context.Context, arnStr string) b.mu.RLock("DescribePartnerApp") defer b.mu.RUnlock() - p, ok := b.partnerAppsStore(region)[arnStr] + p, ok := b.partnerAppsStore(region).Get(arnStr) if !ok { return nil, fmt.Errorf("%w: partner app %q not found", ErrPartnerAppNotFound, arnStr) } @@ -2162,11 +2186,11 @@ func (b *InMemoryBackend) DeletePartnerApp(ctx context.Context, arnStr string) e store := b.partnerAppsStore(region) - if _, ok := store[arnStr]; !ok { + if _, ok := store.Get(arnStr); !ok { return fmt.Errorf("%w: partner app %q not found", ErrPartnerAppNotFound, arnStr) } - delete(store, arnStr) + store.Delete(arnStr) return nil } @@ -2185,7 +2209,7 @@ func (b *InMemoryBackend) UpdatePartnerApp(ctx context.Context, opts UpdatePartn b.mu.Lock("UpdatePartnerApp") defer b.mu.Unlock() - p, ok := b.partnerAppsStore(region)[opts.Arn] + p, ok := b.partnerAppsStore(region).Get(opts.Arn) if !ok { return nil, fmt.Errorf("%w: partner app %q not found", ErrPartnerAppNotFound, opts.Arn) } @@ -2210,7 +2234,12 @@ func (b *InMemoryBackend) ListPartnerApps(ctx context.Context, nextToken string) b.mu.RLock("ListPartnerApps") defer b.mu.RUnlock() - return sagemakerListKeyPaged(b.partnerAppsStore(region), nextToken, clonePartnerApp) + return sagemakerListKeyPaged( + b.partnerAppsStore(region), + nextToken, + clonePartnerApp, + func(v *PartnerApp) string { return v.Arn }, + ) } // CreatePartnerAppPresignedURL returns a one-time presigned URL for accessing @@ -2221,7 +2250,7 @@ func (b *InMemoryBackend) CreatePartnerAppPresignedURL(ctx context.Context, arnS b.mu.RLock("CreatePartnerAppPresignedURL") defer b.mu.RUnlock() - p, ok := b.partnerAppsStore(region)[arnStr] + p, ok := b.partnerAppsStore(region).Get(arnStr) if !ok { return "", fmt.Errorf("%w: partner app %q not found", ErrPartnerAppNotFound, arnStr) } @@ -2324,7 +2353,7 @@ func (b *InMemoryBackend) CreateTrainingPlan( store := b.trainingPlansStore(region) - if _, ok := store[name]; ok { + if _, ok := store.Get(name); ok { return nil, fmt.Errorf("%w: training plan %q already exists", ErrValidation, name) } @@ -2343,7 +2372,7 @@ func (b *InMemoryBackend) CreateTrainingPlan( b.applyOfferingToPlan(region, t, offering, now, spareInstanceCountPerUltraServer) } - store[name] = t + store.Put(t) return cloneTrainingPlan(t), nil } @@ -2384,7 +2413,7 @@ func (b *InMemoryBackend) DescribeTrainingPlan(ctx context.Context, name string) b.mu.RLock("DescribeTrainingPlan") defer b.mu.RUnlock() - t, ok := b.trainingPlansStore(region)[name] + t, ok := b.trainingPlansStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: training plan %q not found", ErrTrainingPlanNotFound, name) } diff --git a/services/sagemaker/backend_cluster.go b/services/sagemaker/backend_cluster.go index efbee46e3..d05cf94f1 100644 --- a/services/sagemaker/backend_cluster.go +++ b/services/sagemaker/backend_cluster.go @@ -19,12 +19,12 @@ import ( func (b *InMemoryBackend) resolveClusterLocked(region, nameOrArn string) (*Cluster, error) { store := b.clustersStore(region) - if c, ok := store[nameOrArn]; ok { + if c, ok := store.Get(nameOrArn); ok { return c, nil } if name, ok := b.clusterARNIndexStore(region)[nameOrArn]; ok { - if c, found := store[name]; found { + if c, found := store.Get(name); found { return c, nil } } @@ -67,7 +67,7 @@ func (b *InMemoryBackend) CreateCluster( region := getRegion(ctx, b.region) store := b.clustersStore(region) - if _, ok := store[name]; ok { + if _, ok := store.Get(name); ok { return nil, fmt.Errorf("%w: cluster %q already exists", ErrClusterAlreadyExists, name) } @@ -98,7 +98,7 @@ func (b *InMemoryBackend) CreateCluster( } } - store[name] = c + store.Put(c) b.clusterARNIndexStore(region)[clusterARN] = name return cloneCluster(c), nil @@ -130,18 +130,15 @@ func (b *InMemoryBackend) ListClusters( region := getRegion(ctx, b.region) all := b.clustersStore(region) - filtered := all - if nameContains != "" { - filtered = make(map[string]*Cluster, len(all)) + filtered := make(map[string]*Cluster, all.Len()) - for k, c := range all { - if strings.Contains(c.ClusterName, nameContains) { - filtered[k] = c - } + for _, c := range all.All() { + if nameContains == "" || strings.Contains(c.ClusterName, nameContains) { + filtered[c.ClusterName] = c } } - return sagemakerListPaged(filtered, nextToken, cloneCluster, + return sagemakerListPagedMap(filtered, nextToken, cloneCluster, func(a, b *Cluster) bool { return a.ClusterName < b.ClusterName }) } @@ -157,7 +154,7 @@ func (b *InMemoryBackend) DeleteCluster(ctx context.Context, nameOrArn string) ( return "", err } - delete(b.clustersStore(region), c.ClusterName) + b.clustersStore(region).Delete(c.ClusterName) delete(b.clusterARNIndexStore(region), c.ClusterArn) return c.ClusterArn, nil @@ -349,7 +346,7 @@ func (b *InMemoryBackend) ListClusterNodes( return nil, "", err } - nodes, next := sagemakerListKeyPaged(c.Nodes, nextToken, func(n *ClusterNode) *ClusterNode { + nodes, next := sagemakerListKeyPagedMap(c.Nodes, nextToken, func(n *ClusterNode) *ClusterNode { cp := *n return &cp diff --git a/services/sagemaker/backend_edge_deployment.go b/services/sagemaker/backend_edge_deployment.go index d28f89507..069adef84 100644 --- a/services/sagemaker/backend_edge_deployment.go +++ b/services/sagemaker/backend_edge_deployment.go @@ -145,12 +145,12 @@ func (b *InMemoryBackend) CreateEdgeDeploymentPlan( return nil, fmt.Errorf("%w: DeviceFleetName is required", ErrValidation) } - if _, ok := b.deviceFleetsStore(region)[opts.DeviceFleetName]; !ok { + if _, ok := b.deviceFleetsStore(region).Get(opts.DeviceFleetName); !ok { return nil, fmt.Errorf("%w: device fleet %q", ErrDeviceFleetNotFound, opts.DeviceFleetName) } store := b.edgeDeploymentPlansStore(region) - if _, ok := store[opts.EdgeDeploymentPlanName]; ok { + if _, ok := store.Get(opts.EdgeDeploymentPlanName); ok { return nil, fmt.Errorf( "%w: edge deployment plan %q already exists", ErrEdgeDeploymentPlanAlreadyExists, @@ -171,7 +171,7 @@ func (b *InMemoryBackend) CreateEdgeDeploymentPlan( CreationTime: now, LastModifiedTime: now, } - store[opts.EdgeDeploymentPlanName] = p + store.Put(p) return cloneEdgeDeploymentPlan(p), nil } @@ -183,7 +183,7 @@ func (b *InMemoryBackend) DescribeEdgeDeploymentPlan(ctx context.Context, name s region := getRegion(ctx, b.region) - p, ok := b.edgeDeploymentPlansStore(region)[name] + p, ok := b.edgeDeploymentPlansStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: edge deployment plan %q", ErrEdgeDeploymentPlanNotFound, name) } @@ -199,11 +199,11 @@ func (b *InMemoryBackend) DeleteEdgeDeploymentPlan(ctx context.Context, name str region := getRegion(ctx, b.region) store := b.edgeDeploymentPlansStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf("%w: edge deployment plan %q", ErrEdgeDeploymentPlanNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -218,7 +218,12 @@ func (b *InMemoryBackend) ListEdgeDeploymentPlans( region := getRegion(ctx, b.region) - return sagemakerListKeyPaged(b.edgeDeploymentPlansStore(region), nextToken, cloneEdgeDeploymentPlan) + return sagemakerListKeyPaged( + b.edgeDeploymentPlansStore(region), + nextToken, + cloneEdgeDeploymentPlan, + func(v *EdgeDeploymentPlan) string { return v.EdgeDeploymentPlanName }, + ) } // CreateEdgeDeploymentStage appends new stages to an existing edge deployment plan. @@ -232,7 +237,7 @@ func (b *InMemoryBackend) CreateEdgeDeploymentStage( region := getRegion(ctx, b.region) - p, ok := b.edgeDeploymentPlansStore(region)[planName] + p, ok := b.edgeDeploymentPlansStore(region).Get(planName) if !ok { return fmt.Errorf("%w: edge deployment plan %q", ErrEdgeDeploymentPlanNotFound, planName) } @@ -250,7 +255,7 @@ func (b *InMemoryBackend) DeleteEdgeDeploymentStage(ctx context.Context, planNam region := getRegion(ctx, b.region) - p, ok := b.edgeDeploymentPlansStore(region)[planName] + p, ok := b.edgeDeploymentPlansStore(region).Get(planName) if !ok { return fmt.Errorf("%w: edge deployment plan %q", ErrEdgeDeploymentPlanNotFound, planName) } @@ -283,7 +288,7 @@ func (b *InMemoryBackend) setEdgeDeploymentStageStatus(ctx context.Context, plan region := getRegion(ctx, b.region) - p, ok := b.edgeDeploymentPlansStore(region)[planName] + p, ok := b.edgeDeploymentPlansStore(region).Get(planName) if !ok { return fmt.Errorf("%w: edge deployment plan %q", ErrEdgeDeploymentPlanNotFound, planName) } @@ -317,15 +322,15 @@ func (b *InMemoryBackend) GetDeviceFleetReport(ctx context.Context, fleetName st region := getRegion(ctx, b.region) - f, ok := b.deviceFleetsStore(region)[fleetName] + f, ok := b.deviceFleetsStore(region).Get(fleetName) if !ok { return nil, 0, fmt.Errorf("%w: device fleet %q", ErrDeviceFleetNotFound, fleetName) } registered := 0 - for k := range b.devicesStore(region) { - if k.fleetName == fleetName { + for _, d := range b.devicesStore(region).All() { + if d.DeviceFleetName == fleetName { registered++ } } @@ -344,7 +349,7 @@ func (b *InMemoryBackend) ListStageDevices( region := getRegion(ctx, b.region) - p, ok := b.edgeDeploymentPlansStore(region)[planName] + p, ok := b.edgeDeploymentPlansStore(region).Get(planName) if !ok { return nil, nil, "", "", fmt.Errorf("%w: edge deployment plan %q", ErrEdgeDeploymentPlanNotFound, planName) } diff --git a/services/sagemaker/backend_feature_store.go b/services/sagemaker/backend_feature_store.go index 43d40b0e7..951e8dfb3 100644 --- a/services/sagemaker/backend_feature_store.go +++ b/services/sagemaker/backend_feature_store.go @@ -14,6 +14,12 @@ import ( type FeatureRecord struct { // Record maps feature name → feature value Record map[string]string `json:"Record"` + // Key is the internal store.Table primary key (featureRecordKey(featureGroupName, + // recordIDValue)), carried on the value itself so the Table's keyFn can derive it + // and Restore can rebuild it after a JSON round-trip. Exported (with a json tag) + // solely so it survives persistence — handler.go never marshals FeatureRecord + // directly to API callers, so this has no effect on the AWS wire shape. + Key string `json:"key"` } // FeatureMetadata stores metadata (description + parameters) for a feature in a group. @@ -21,7 +27,14 @@ type FeatureMetadata struct { Description string `json:"Description,omitempty"` Parameters map[string]string `json:"Parameters,omitempty"` FeatureName string `json:"FeatureName"` - FeatureType string `json:"FeatureType,omitempty"` + // GroupName is the owning feature group's name, carried internally so the + // store.Table's keyFn can derive featureMetaKey(GroupName, FeatureName) and + // Restore can rebuild it after a JSON round-trip. Exported (with a json tag) + // solely so it survives persistence — handler.go never marshals + // FeatureMetadata directly to API callers, so this has no effect on the AWS + // wire shape. + GroupName string `json:"groupName"` + FeatureType string `json:"FeatureType,omitempty"` } // featureRecordKey builds the map key for a feature record. @@ -45,7 +58,7 @@ func (b *InMemoryBackend) PutRecord( region := getRegion(ctx, b.region) - fg, ok := b.featureGroupsStore(region)[featureGroupName] + fg, ok := b.featureGroupsStore(region).Get(featureGroupName) if !ok { return fmt.Errorf( "%w: feature group %q not found", @@ -68,7 +81,7 @@ func (b *InMemoryBackend) PutRecord( cp := make(map[string]string, len(record)) maps.Copy(cp, record) - b.featureRecordsStore(region)[key] = &FeatureRecord{Record: cp} + b.featureRecordsStore(region).Put(&FeatureRecord{Record: cp, Key: key}) return nil } @@ -84,7 +97,7 @@ func (b *InMemoryBackend) GetRecord( region := getRegion(ctx, b.region) - if _, ok := b.featureGroupsStore(region)[featureGroupName]; !ok { + if _, ok := b.featureGroupsStore(region).Get(featureGroupName); !ok { return nil, fmt.Errorf( "%w: feature group %q not found", ErrFeatureGroupNotFound, @@ -94,7 +107,7 @@ func (b *InMemoryBackend) GetRecord( key := featureRecordKey(featureGroupName, recordIDValue) - rec, ok := b.featureRecordsStore(region)[key] + rec, ok := b.featureRecordsStore(region).Get(key) if !ok { return nil, fmt.Errorf( "%w: record %q not found in feature group %q", @@ -130,7 +143,7 @@ func (b *InMemoryBackend) DeleteRecord(ctx context.Context, featureGroupName, re region := getRegion(ctx, b.region) - if _, ok := b.featureGroupsStore(region)[featureGroupName]; !ok { + if _, ok := b.featureGroupsStore(region).Get(featureGroupName); !ok { return fmt.Errorf( "%w: feature group %q not found", ErrFeatureGroupNotFound, @@ -140,7 +153,7 @@ func (b *InMemoryBackend) DeleteRecord(ctx context.Context, featureGroupName, re key := featureRecordKey(featureGroupName, recordIDValue) store := b.featureRecordsStore(region) - delete(store, key) + store.Delete(key) return nil } @@ -176,7 +189,7 @@ func (b *InMemoryBackend) BatchGetRecord( RecordIdentifierValueAsString: ident.RecordIdentifierValueAsString, } - fg, ok := b.featureGroupsStore(region)[ident.FeatureGroupName] + fg, ok := b.featureGroupsStore(region).Get(ident.FeatureGroupName) if !ok { result.ErrorCode = "ResourceNotFoundException" result.ErrorMessage = "feature group " + ident.FeatureGroupName + " not found" @@ -187,7 +200,7 @@ func (b *InMemoryBackend) BatchGetRecord( key := featureRecordKey(fg.FeatureGroupName, ident.RecordIdentifierValueAsString) - rec, ok := b.featureRecordsStore(region)[key] + rec, ok := b.featureRecordsStore(region).Get(key) if !ok { result.ErrorCode = "ResourceNotFoundException" result.ErrorMessage = "record " + ident.RecordIdentifierValueAsString + " not found" @@ -227,7 +240,7 @@ func (b *InMemoryBackend) GetFeatureMetadata( region := getRegion(ctx, b.region) - fg, ok := b.featureGroupsStore(region)[featureGroupName] + fg, ok := b.featureGroupsStore(region).Get(featureGroupName) if !ok { return nil, fmt.Errorf( "%w: feature group %q not found", @@ -249,7 +262,7 @@ func (b *InMemoryBackend) GetFeatureMetadata( key := featureMetaKey(featureGroupName, featureName) - meta, ok := b.featureMetadataStore(region)[key] + meta, ok := b.featureMetadataStore(region).Get(key) if !ok { // Return default metadata if not explicitly set. return &FeatureMetadata{ @@ -276,7 +289,7 @@ func (b *InMemoryBackend) UpdateFeatureMetadata( region := getRegion(ctx, b.region) - if _, ok := b.featureGroupsStore(region)[featureGroupName]; !ok { + if _, ok := b.featureGroupsStore(region).Get(featureGroupName); !ok { return fmt.Errorf( "%w: feature group %q not found", ErrFeatureGroupNotFound, @@ -287,9 +300,9 @@ func (b *InMemoryBackend) UpdateFeatureMetadata( key := featureMetaKey(featureGroupName, featureName) metaStore := b.featureMetadataStore(region) - existing, ok := metaStore[key] + existing, ok := metaStore.Get(key) if !ok { - existing = &FeatureMetadata{FeatureName: featureName} + existing = &FeatureMetadata{FeatureName: featureName, GroupName: featureGroupName} } if description != "" { @@ -303,7 +316,7 @@ func (b *InMemoryBackend) UpdateFeatureMetadata( maps.Copy(existing.Parameters, parameters) } - metaStore[key] = existing + metaStore.Put(existing) return nil } diff --git a/services/sagemaker/backend_hub.go b/services/sagemaker/backend_hub.go index 1a5b396b2..a84e6507b 100644 --- a/services/sagemaker/backend_hub.go +++ b/services/sagemaker/backend_hub.go @@ -11,6 +11,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awserr" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) // --------------------------------------------------------------------------- @@ -75,9 +76,9 @@ type UpdateHubOptions struct { } // hubsStore returns the region-scoped hub map, creating it lazily. -func (b *InMemoryBackend) hubsStore(r string) map[string]*Hub { +func (b *InMemoryBackend) hubsStore(r string) *store.Table[Hub] { if b.hubs[r] == nil { - b.hubs[r] = make(map[string]*Hub) + b.hubs[r] = store.Register(b.registry, "hubs:"+r, store.New(func(v *Hub) string { return v.HubName })) } return b.hubs[r] @@ -85,11 +86,11 @@ func (b *InMemoryBackend) hubsStore(r string) map[string]*Hub { // findHubLocked resolves a hub by name or ARN. Callers must hold b.mu. func (b *InMemoryBackend) findHubLocked(region, idOrArn string) (*Hub, bool) { - if h, ok := b.hubsStore(region)[idOrArn]; ok { + if h, ok := b.hubsStore(region).Get(idOrArn); ok { return h, true } - for _, h := range b.hubsStore(region) { + for _, h := range b.hubsStore(region).All() { if h.HubArn == idOrArn { return h, true } @@ -111,7 +112,7 @@ func (b *InMemoryBackend) CreateHub( region := getRegion(ctx, b.region) store := b.hubsStore(region) - if _, ok := store[name]; ok { + if _, ok := store.Get(name); ok { return nil, fmt.Errorf("%w: hub %q already exists", ErrHubAlreadyExists, name) } @@ -130,7 +131,7 @@ func (b *InMemoryBackend) CreateHub( LastModifiedTime: now, Tags: mergeTags(nil, tags), } - store[name] = h + store.Put(h) return cloneHub(h), nil } @@ -158,15 +159,15 @@ func (b *InMemoryBackend) ListHubs(ctx context.Context, nameContains, nextToken region := getRegion(ctx, b.region) store := b.hubsStore(region) - filtered := make(map[string]*Hub, len(store)) + filtered := make(map[string]*Hub, store.Len()) - for k, v := range store { + for _, v := range store.All() { if nameContains == "" || strings.Contains(v.HubName, nameContains) { - filtered[k] = v + filtered[v.HubName] = v } } - return sagemakerListPaged(filtered, nextToken, cloneHub, + return sagemakerListPagedMap(filtered, nextToken, cloneHub, func(a, bb *Hub) bool { return a.HubName < bb.HubName }) } @@ -211,14 +212,14 @@ func (b *InMemoryBackend) DeleteHub(ctx context.Context, idOrArn string) error { return fmt.Errorf("%w: hub %q not found", ErrHubNotFound, idOrArn) } - for _, hc := range b.hubContentsStore(region) { + for _, hc := range b.hubContentsStore(region).All() { if hc.HubName == h.HubName { return fmt.Errorf("%w: hub %q still has hub content and cannot be deleted", ErrHubNotEmpty, h.HubName) } } - delete(b.hubsStore(region), h.HubName) + b.hubsStore(region).Delete(h.HubName) return nil } @@ -277,9 +278,18 @@ type hubContentKey struct { } // hubContentsStore returns the region-scoped hub content map, creating it lazily. -func (b *InMemoryBackend) hubContentsStore(r string) map[hubContentKey]*HubContent { +func (b *InMemoryBackend) hubContentsStore(r string) *store.Table[HubContent] { if b.hubContents[r] == nil { - b.hubContents[r] = make(map[hubContentKey]*HubContent) + b.hubContents[r] = store.Register(b.registry, "hubContents:"+r, store.New(func(v *HubContent) string { + return hubContentKeyString( + hubContentKey{ + HubName: v.HubName, + HubContentType: v.HubContentType, + HubContentName: v.HubContentName, + HubContentVersion: v.HubContentVersion, + }, + ) + })) } return b.hubContents[r] @@ -327,13 +337,13 @@ func (b *InMemoryBackend) ImportHubContent(ctx context.Context, in ImportHubCont version = defaultHubContentVersion } - key := hubContentKey{ + hcKey := hubContentKey{ HubName: h.HubName, HubContentType: in.HubContentType, HubContentName: in.HubContentName, HubContentVersion: version, } store := b.hubContentsStore(region) - if _, exists := store[key]; exists { + if _, exists := store.Get(hubContentKeyString(hcKey)); exists { return nil, fmt.Errorf( "%w: hub content %q version %q already exists in hub %q", ErrHubContentAlreadyExists, in.HubContentName, version, h.HubName, @@ -345,7 +355,7 @@ func (b *InMemoryBackend) ImportHubContent(ctx context.Context, in ImportHubCont HubName: h.HubName, HubArn: h.HubArn, HubContentName: in.HubContentName, - HubContentArn: hubContentARN(region, b.accountID, key), + HubContentArn: hubContentARN(region, b.accountID, hcKey), HubContentVersion: version, HubContentType: in.HubContentType, DocumentSchemaVersion: in.DocumentSchemaVersion, @@ -360,7 +370,7 @@ func (b *InMemoryBackend) ImportHubContent(ctx context.Context, in ImportHubCont LastModifiedTime: now, Tags: mergeTags(nil, in.Tags), } - store[key] = hc + store.Put(hc) return cloneHubContent(hc), nil } @@ -370,8 +380,8 @@ func (b *InMemoryBackend) ImportHubContent(ctx context.Context, in ImportHubCont func (b *InMemoryBackend) latestHubContentLocked(region, hubName, contentType, contentName string) (*HubContent, bool) { var latest *HubContent - for k, hc := range b.hubContentsStore(region) { - if k.HubName != hubName || k.HubContentType != contentType || k.HubContentName != contentName { + for _, hc := range b.hubContentsStore(region).All() { + if hc.HubName != hubName || hc.HubContentType != contentType || hc.HubContentName != contentName { continue } @@ -404,11 +414,11 @@ func (b *InMemoryBackend) DescribeHubContent( } if version != "" { - key := hubContentKey{ + key := hubContentKeyString(hubContentKey{ HubName: h.HubName, HubContentType: contentType, HubContentName: contentName, HubContentVersion: version, - } - if hc, exists := b.hubContentsStore(region)[key]; exists { + }) + if hc, exists := b.hubContentsStore(region).Get(key); exists { return cloneHubContent(hc), nil } @@ -453,12 +463,12 @@ func (b *InMemoryBackend) UpdateHubContent( return nil, fmt.Errorf("%w: hub %q not found", ErrHubNotFound, hubName) } - key := hubContentKey{ + key := hubContentKeyString(hubContentKey{ HubName: h.HubName, HubContentType: contentType, HubContentName: contentName, HubContentVersion: version, - } + }) - hc, ok := b.hubContentsStore(region)[key] + hc, ok := b.hubContentsStore(region).Get(key) if !ok { return nil, fmt.Errorf( "%w: hub content %q version %q not found in hub %q", ErrHubContentNotFound, contentName, version, h.HubName, @@ -529,21 +539,21 @@ func (b *InMemoryBackend) ListHubContents( latestByName := make(map[string]*HubContent) - for k, hc := range b.hubContentsStore(region) { - if k.HubName != hubName || k.HubContentType != contentType { + for _, hc := range b.hubContentsStore(region).All() { + if hc.HubName != hubName || hc.HubContentType != contentType { continue } - if nameContains != "" && !strings.Contains(k.HubContentName, nameContains) { + if nameContains != "" && !strings.Contains(hc.HubContentName, nameContains) { continue } - if cur, exists := latestByName[k.HubContentName]; !exists || hc.CreationTime.After(cur.CreationTime) { - latestByName[k.HubContentName] = hc + if cur, exists := latestByName[hc.HubContentName]; !exists || hc.CreationTime.After(cur.CreationTime) { + latestByName[hc.HubContentName] = hc } } - return sagemakerListPaged(latestByName, nextToken, cloneHubContent, + return sagemakerListPagedMap(latestByName, nextToken, cloneHubContent, func(a, bb *HubContent) bool { return a.HubContentName < bb.HubContentName }) } @@ -560,15 +570,15 @@ func (b *InMemoryBackend) ListHubContentVersions( byVersion := make(map[string]*HubContent) - for k, hc := range b.hubContentsStore(region) { - if k.HubName != hubName || k.HubContentType != contentType || k.HubContentName != contentName { + for _, hc := range b.hubContentsStore(region).All() { + if hc.HubName != hubName || hc.HubContentType != contentType || hc.HubContentName != contentName { continue } - byVersion[k.HubContentVersion] = hc + byVersion[hc.HubContentVersion] = hc } - return sagemakerListPaged(byVersion, nextToken, cloneHubContent, + return sagemakerListPagedMap(byVersion, nextToken, cloneHubContent, func(a, bb *HubContent) bool { return a.HubContentVersion < bb.HubContentVersion }) } @@ -581,20 +591,20 @@ func (b *InMemoryBackend) DeleteHubContent( defer b.mu.Unlock() region := getRegion(ctx, b.region) - key := hubContentKey{ + key := hubContentKeyString(hubContentKey{ HubName: hubName, HubContentType: contentType, HubContentName: contentName, HubContentVersion: version, - } + }) store := b.hubContentsStore(region) - if _, ok := store[key]; !ok { + if _, ok := store.Get(key); !ok { return fmt.Errorf( "%w: hub content %q version %q not found in hub %q", ErrHubContentNotFound, contentName, version, hubName, ) } - delete(store, key) + store.Delete(key) return nil } @@ -653,13 +663,13 @@ func (b *InMemoryBackend) CreateHubContentReference( version = defaultHubContentVersion } - key := hubContentKey{ + hcKey := hubContentKey{ HubName: h.HubName, HubContentType: hubContentTypeModelRef, HubContentName: name, HubContentVersion: version, } store := b.hubContentsStore(region) - if _, exists := store[key]; exists { + if _, exists := store.Get(hubContentKeyString(hcKey)); exists { return nil, fmt.Errorf( "%w: hub content reference %q already exists in hub %q", ErrHubContentAlreadyExists, name, h.HubName, ) @@ -670,7 +680,7 @@ func (b *InMemoryBackend) CreateHubContentReference( HubName: h.HubName, HubArn: h.HubArn, HubContentName: name, - HubContentArn: hubContentARN(region, b.accountID, key), + HubContentArn: hubContentARN(region, b.accountID, hcKey), HubContentVersion: version, HubContentType: hubContentTypeModelRef, DocumentSchemaVersion: defaultDocumentSchemaVer, @@ -681,7 +691,7 @@ func (b *InMemoryBackend) CreateHubContentReference( LastModifiedTime: now, Tags: mergeTags(nil, tags), } - store[key] = hc + store.Put(hc) return cloneHubContent(hc), nil } @@ -699,9 +709,12 @@ func (b *InMemoryBackend) DeleteHubContentReference( deleted := false - for k := range store { - if k.HubName == hubName && k.HubContentType == contentType && k.HubContentName == contentName { - delete(store, k) + for _, hc := range store.All() { + if hc.HubName == hubName && hc.HubContentType == contentType && hc.HubContentName == contentName { + store.Delete(hubContentKeyString(hubContentKey{ + HubName: hc.HubName, HubContentType: hc.HubContentType, + HubContentName: hc.HubContentName, HubContentVersion: hc.HubContentVersion, + })) deleted = true } @@ -758,12 +771,12 @@ func (b *InMemoryBackend) CreateHubContentPresignedURLs( var hc *HubContent if version != "" { - key := hubContentKey{ + key := hubContentKeyString(hubContentKey{ HubName: h.HubName, HubContentType: contentType, HubContentName: contentName, HubContentVersion: version, - } + }) - found, exists := b.hubContentsStore(region)[key] + found, exists := b.hubContentsStore(region).Get(key) if !exists { return nil, fmt.Errorf( "%w: hub content %q version %q not found in hub %q", diff --git a/services/sagemaker/backend_labeling.go b/services/sagemaker/backend_labeling.go index 3711742f1..68b60644f 100644 --- a/services/sagemaker/backend_labeling.go +++ b/services/sagemaker/backend_labeling.go @@ -10,6 +10,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awserr" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) // --------------------------------------------------------------------------- @@ -206,9 +207,13 @@ type CreateLabelingJobOptions struct { HumanTaskConfig HumanTaskConfig } -func (b *InMemoryBackend) labelingJobsStore(r string) map[string]*LabelingJob { +func (b *InMemoryBackend) labelingJobsStore(r string) *store.Table[LabelingJob] { if b.labelingJobs[r] == nil { - b.labelingJobs[r] = make(map[string]*LabelingJob) + b.labelingJobs[r] = store.Register( + b.registry, + "labelingJobs:"+r, + store.New(func(v *LabelingJob) string { return v.LabelingJobName }), + ) } return b.labelingJobs[r] @@ -229,7 +234,7 @@ func (b *InMemoryBackend) CreateLabelingJob( } store := b.labelingJobsStore(region) - if _, ok := store[opts.LabelingJobName]; ok { + if _, ok := store.Get(opts.LabelingJobName); ok { return nil, fmt.Errorf( "%w: labeling job %q already exists", ErrLabelingJobAlreadyExists, opts.LabelingJobName, ) @@ -255,7 +260,7 @@ func (b *InMemoryBackend) CreateLabelingJob( CreationTime: now, LastModifiedTime: now, } - store[opts.LabelingJobName] = j + store.Put(j) b.scheduleLabelingJobCompletion(b.lifecycleCtx, region, opts.LabelingJobName) @@ -268,7 +273,7 @@ func (b *InMemoryBackend) CreateLabelingJob( func (b *InMemoryBackend) scheduleLabelingJobCompletion(ctx context.Context, region, name string) { b.runDelayed(ctx, labelingJobStopDelay, func() { b.mu.Lock("scheduleLabelingJobCompletion.toInProgress") - j, ok := b.labelingJobsStore(region)[name] + j, ok := b.labelingJobsStore(region).Get(name) if ok && j.LabelingJobStatus == labelingJobStatusInitializing { j.LabelingJobStatus = labelingJobStatusInProgress j.LastModifiedTime = time.Now() @@ -280,7 +285,7 @@ func (b *InMemoryBackend) scheduleLabelingJobCompletion(ctx context.Context, reg b.mu.Lock("scheduleLabelingJobCompletion.toCompleted") defer b.mu.Unlock() - j, ok := b.labelingJobsStore(region)[name] + j, ok := b.labelingJobsStore(region).Get(name) if !ok || j.LabelingJobStatus != labelingJobStatusInProgress { return } @@ -302,7 +307,7 @@ func (b *InMemoryBackend) DescribeLabelingJob(ctx context.Context, name string) region := getRegion(ctx, b.region) - j, ok := b.labelingJobsStore(region)[name] + j, ok := b.labelingJobsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: labeling job %q not found", ErrLabelingJobNotFound, name) } @@ -317,7 +322,7 @@ func (b *InMemoryBackend) StopLabelingJob(ctx context.Context, name string) erro region := getRegion(ctx, b.region) - j, ok := b.labelingJobsStore(region)[name] + j, ok := b.labelingJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: labeling job %q not found", ErrLabelingJobNotFound, name) } @@ -336,7 +341,7 @@ func (b *InMemoryBackend) StopLabelingJob(ctx context.Context, name string) erro b.mu.Lock("StopLabelingJob.goroutine") defer b.mu.Unlock() - if j2, ok2 := b.labelingJobsStore(region)[name]; ok2 && j2.LabelingJobStatus == labelingJobStatusStopping { + if j2, ok2 := b.labelingJobsStore(region).Get(name); ok2 && j2.LabelingJobStatus == labelingJobStatusStopping { j2.LabelingJobStatus = labelingJobStatusStopped j2.LastModifiedTime = time.Now() } @@ -363,9 +368,9 @@ func (b *InMemoryBackend) ListLabelingJobs( region := getRegion(ctx, b.region) - list := make([]*LabelingJob, 0, len(b.labelingJobsStore(region))) + list := make([]*LabelingJob, 0, b.labelingJobsStore(region).Len()) - for _, j := range b.labelingJobsStore(region) { + for _, j := range b.labelingJobsStore(region).All() { if filter.StatusEquals != "" && j.LabelingJobStatus != filter.StatusEquals { continue } @@ -400,9 +405,9 @@ func (b *InMemoryBackend) ListLabelingJobsForWorkteam( region := getRegion(ctx, b.region) - list := make([]*LabelingJob, 0, len(b.labelingJobsStore(region))) + list := make([]*LabelingJob, 0, b.labelingJobsStore(region).Len()) - for _, j := range b.labelingJobsStore(region) { + for _, j := range b.labelingJobsStore(region).All() { if j.HumanTaskConfig.WorkteamArn == workteamArn { list = append(list, cloneLabelingJob(j)) } diff --git a/services/sagemaker/backend_lineage.go b/services/sagemaker/backend_lineage.go index e575ca824..df62a89dc 100644 --- a/services/sagemaker/backend_lineage.go +++ b/services/sagemaker/backend_lineage.go @@ -8,6 +8,7 @@ import ( "time" "github.com/blackbirdworks/gopherstack/pkgs/arn" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) // defaultLineageGroupName is the name of the lineage group SageMaker @@ -92,17 +93,25 @@ type Edge struct { AssociationType string `json:"AssociationType,omitempty"` } -func (b *InMemoryBackend) artifactsStore(r string) map[string]*Artifact { +func (b *InMemoryBackend) artifactsStore(r string) *store.Table[Artifact] { if b.artifacts[r] == nil { - b.artifacts[r] = make(map[string]*Artifact) + b.artifacts[r] = store.Register( + b.registry, + "artifacts:"+r, + store.New(func(v *Artifact) string { return v.ArtifactArn }), + ) } return b.artifacts[r] } -func (b *InMemoryBackend) contextsStore(r string) map[string]*Context { +func (b *InMemoryBackend) contextsStore(r string) *store.Table[Context] { if b.contexts[r] == nil { - b.contexts[r] = make(map[string]*Context) + b.contexts[r] = store.Register( + b.registry, + "contexts:"+r, + store.New(func(v *Context) string { return v.ContextName }), + ) } return b.contexts[r] @@ -148,7 +157,7 @@ func (b *InMemoryBackend) CreateArtifact( } artifactARN := arn.Build("sagemaker", region, b.accountID, "artifact/"+slug) - if _, ok := store[artifactARN]; ok { + if _, ok := store.Get(artifactARN); ok { return nil, fmt.Errorf("%w: artifact %q already exists", ErrArtifactAlreadyExists, artifactARN) } @@ -163,7 +172,7 @@ func (b *InMemoryBackend) CreateArtifact( CreationTime: now, LastModifiedTime: now, } - store[artifactARN] = ar + store.Put(ar) return cloneArtifact(ar), nil } @@ -175,7 +184,7 @@ func (b *InMemoryBackend) DescribeArtifact(ctx context.Context, artifactArn stri region := getRegion(ctx, b.region) - ar, ok := b.artifactsStore(region)[artifactArn] + ar, ok := b.artifactsStore(region).Get(artifactArn) if !ok { return nil, fmt.Errorf("%w: artifact %q not found", ErrArtifactNotFound, artifactArn) } @@ -196,7 +205,7 @@ func (b *InMemoryBackend) UpdateArtifact( region := getRegion(ctx, b.region) store := b.artifactsStore(region) - ar, ok := store[artifactArn] + ar, ok := store.Get(artifactArn) if !ok { return nil, fmt.Errorf("%w: artifact %q not found", ErrArtifactNotFound, artifactArn) } @@ -228,13 +237,13 @@ func (b *InMemoryBackend) DeleteArtifact(ctx context.Context, artifactArn string region := getRegion(ctx, b.region) store := b.artifactsStore(region) - ar, ok := store[artifactArn] + ar, ok := store.Get(artifactArn) if !ok { return nil, fmt.Errorf("%w: artifact %q not found", ErrArtifactNotFound, artifactArn) } cp := cloneArtifact(ar) - delete(store, artifactArn) + store.Delete(artifactArn) return cp, nil } @@ -250,9 +259,9 @@ func (b *InMemoryBackend) ListArtifacts( region := getRegion(ctx, b.region) store := b.artifactsStore(region) - filtered := make(map[string]*Artifact, len(store)) + filtered := make(map[string]*Artifact, store.Len()) - for k, ar := range store { + for _, ar := range store.All() { if artifactType != "" && ar.ArtifactType != artifactType { continue } @@ -261,10 +270,10 @@ func (b *InMemoryBackend) ListArtifacts( continue } - filtered[k] = ar + filtered[ar.ArtifactArn] = ar } - return sagemakerListPaged(filtered, nextToken, cloneArtifact, + return sagemakerListPagedMap(filtered, nextToken, cloneArtifact, func(a, b *Artifact) bool { return a.ArtifactArn < b.ArtifactArn }) } @@ -298,7 +307,7 @@ func (b *InMemoryBackend) CreateContext( region := getRegion(ctx, b.region) store := b.contextsStore(region) - if _, ok := store[name]; ok { + if _, ok := store.Get(name); ok { return nil, fmt.Errorf("%w: context %q already exists", ErrContextAlreadyExists, name) } @@ -315,7 +324,7 @@ func (b *InMemoryBackend) CreateContext( CreationTime: now, LastModifiedTime: now, } - store[name] = c + store.Put(c) b.contextARNIndexStore(region)[contextARN] = name return cloneContext(c), nil @@ -328,7 +337,7 @@ func (b *InMemoryBackend) DescribeContext(ctx context.Context, name string) (*Co region := getRegion(ctx, b.region) - c, ok := b.contextsStore(region)[name] + c, ok := b.contextsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: context %q not found", ErrContextNotFound, name) } @@ -349,7 +358,7 @@ func (b *InMemoryBackend) UpdateContext( region := getRegion(ctx, b.region) store := b.contextsStore(region) - c, ok := store[name] + c, ok := store.Get(name) if !ok { return nil, fmt.Errorf("%w: context %q not found", ErrContextNotFound, name) } @@ -381,13 +390,13 @@ func (b *InMemoryBackend) DeleteContext(ctx context.Context, name string) (*Cont region := getRegion(ctx, b.region) store := b.contextsStore(region) - c, ok := store[name] + c, ok := store.Get(name) if !ok { return nil, fmt.Errorf("%w: context %q not found", ErrContextNotFound, name) } cp := cloneContext(c) - delete(store, name) + store.Delete(name) delete(b.contextARNIndexStore(region), c.ContextArn) return cp, nil @@ -404,9 +413,9 @@ func (b *InMemoryBackend) ListContexts( region := getRegion(ctx, b.region) store := b.contextsStore(region) - filtered := make(map[string]*Context, len(store)) + filtered := make(map[string]*Context, store.Len()) - for k, c := range store { + for _, c := range store.All() { if contextType != "" && c.ContextType != contextType { continue } @@ -415,10 +424,10 @@ func (b *InMemoryBackend) ListContexts( continue } - filtered[k] = c + filtered[c.ContextName] = c } - return sagemakerListPaged(filtered, nextToken, cloneContext, + return sagemakerListPagedMap(filtered, nextToken, cloneContext, func(a, b *Context) bool { return a.ContextName < b.ContextName }) } @@ -433,7 +442,7 @@ func (b *InMemoryBackend) DescribeAction(ctx context.Context, name string) (*Act region := getRegion(ctx, b.region) - a, ok := b.actionsStore(region)[name] + a, ok := b.actionsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: action %q not found", ErrActionNotFound, name) } @@ -454,7 +463,7 @@ func (b *InMemoryBackend) UpdateAction( region := getRegion(ctx, b.region) store := b.actionsStore(region) - a, ok := store[name] + a, ok := store.Get(name) if !ok { return nil, fmt.Errorf("%w: action %q not found", ErrActionNotFound, name) } @@ -490,13 +499,13 @@ func (b *InMemoryBackend) DeleteAction(ctx context.Context, name string) (*Actio region := getRegion(ctx, b.region) store := b.actionsStore(region) - a, ok := store[name] + a, ok := store.Get(name) if !ok { return nil, fmt.Errorf("%w: action %q not found", ErrActionNotFound, name) } cp := cloneAction(a) - delete(store, name) + store.Delete(name) delete(b.actionARNIndexStore(region), a.ActionArn) return cp, nil @@ -513,9 +522,9 @@ func (b *InMemoryBackend) ListActions( region := getRegion(ctx, b.region) store := b.actionsStore(region) - filtered := make(map[string]*Action, len(store)) + filtered := make(map[string]*Action, store.Len()) - for k, a := range store { + for _, a := range store.All() { if actionType != "" && a.ActionType != actionType { continue } @@ -524,10 +533,10 @@ func (b *InMemoryBackend) ListActions( continue } - filtered[k] = a + filtered[a.ActionName] = a } - return sagemakerListPaged(filtered, nextToken, cloneAction, + return sagemakerListPagedMap(filtered, nextToken, cloneAction, func(a, b *Action) bool { return a.ActionName < b.ActionName }) } @@ -544,7 +553,7 @@ func (b *InMemoryBackend) DeleteAssociation(ctx context.Context, sourceArn, dest store := b.associationsStore(region) key := associationKey(sourceArn, destinationArn) - if _, ok := store[key]; !ok { + if _, ok := store.Get(key); !ok { return fmt.Errorf( "%w: association between %s and %s not found", ErrAssociationNotFound, @@ -553,7 +562,7 @@ func (b *InMemoryBackend) DeleteAssociation(ctx context.Context, sourceArn, dest ) } - delete(store, key) + store.Delete(key) return nil } @@ -569,9 +578,9 @@ func (b *InMemoryBackend) ListAssociations( region := getRegion(ctx, b.region) store := b.associationsStore(region) - filtered := make(map[string]*Association, len(store)) + filtered := make(map[string]*Association, store.Len()) - for k, a := range store { + for _, a := range store.All() { if sourceArn != "" && a.SourceArn != sourceArn { continue } @@ -584,10 +593,10 @@ func (b *InMemoryBackend) ListAssociations( continue } - filtered[k] = a + filtered[associationKey(a.SourceArn, a.DestinationArn)] = a } - return sagemakerListPaged(filtered, nextToken, cloneAssociation, + return sagemakerListPagedMap(filtered, nextToken, cloneAssociation, func(a, b *Association) bool { return a.AssociationArn < b.AssociationArn }) } @@ -650,17 +659,17 @@ func (b *InMemoryBackend) lineageEntityLookup( region, entityArn string, ) (string, string, string, bool) { if actionName, found := b.actionARNIndexStore(region)[entityArn]; found { - if a, exists := b.actionsStore(region)[actionName]; exists { + if a, exists := b.actionsStore(region).Get(actionName); exists { return a.ActionName, a.ActionType, "Action", true } } - if ar, found := b.artifactsStore(region)[entityArn]; found { + if ar, found := b.artifactsStore(region).Get(entityArn); found { return ar.ArtifactName, ar.ArtifactType, "Artifact", true } if contextName, found := b.contextARNIndexStore(region)[entityArn]; found { - if c, exists := b.contextsStore(region)[contextName]; exists { + if c, exists := b.contextsStore(region).Get(contextName); exists { return c.ContextName, c.ContextType, "Context", true } } @@ -774,7 +783,7 @@ func (b *InMemoryBackend) buildLineageAdjacency(region string) (map[string][]Edg fwd := make(map[string][]Edge) back := make(map[string][]Edge) - for _, a := range b.associationsStore(region) { + for _, a := range b.associationsStore(region).All() { e := Edge{SourceArn: a.SourceArn, DestinationArn: a.DestinationArn, AssociationType: a.AssociationType} fwd[a.SourceArn] = append(fwd[a.SourceArn], e) back[a.DestinationArn] = append(back[a.DestinationArn], e) diff --git a/services/sagemaker/backend_list_helpers.go b/services/sagemaker/backend_list_helpers.go index e7ce35ac8..27f78e0f6 100644 --- a/services/sagemaker/backend_list_helpers.go +++ b/services/sagemaker/backend_list_helpers.go @@ -9,19 +9,61 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/collections" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) -// sagemakerListPaged paginates a store using index-based tokens. +// tableGet returns the value stored under key in t, or nil if absent. It lets +// a single-value store.Table lookup be substituted inline (including chained +// field access, e.g. tableGet(t, key).Field) for the raw map[string]*V index +// expression it replaces: a missing key yields a nil *V, matching Go's +// zero-value-on-missing-key map semantics, so callers that previously +// tolerated a nil result unchanged. +func tableGet[V any](t *store.Table[V], key string) *V { + v, _ := t.Get(key) + + return v +} + +// sagemakerListPaged paginates a store.Table using index-based tokens. // clone must return a deep copy of its argument. // less defines the sort order. func sagemakerListPaged[T any]( - store map[string]*T, + tbl *store.Table[T], + nextToken string, + clone func(*T) *T, + less func(a, b *T) bool, +) ([]*T, string) { + return sagemakerListPagedSlice(tbl.All(), nextToken, clone, less) +} + +// sagemakerListPagedMap paginates a plain map[string]*T using index-based +// tokens. It preserves the pre-conversion behavior for callers that build a +// local, already-filtered map (e.g. a subset matching a query filter) rather +// than listing a store.Table directly. +func sagemakerListPagedMap[T any]( + m map[string]*T, + nextToken string, + clone func(*T) *T, + less func(a, b *T) bool, +) ([]*T, string) { + items := make([]*T, 0, len(m)) + for _, item := range m { + items = append(items, item) + } + + return sagemakerListPagedSlice(items, nextToken, clone, less) +} + +// sagemakerListPagedSlice is the shared index-based-token pagination core +// used by sagemakerListPaged and sagemakerListPagedMap. +func sagemakerListPagedSlice[T any]( + items []*T, nextToken string, clone func(*T) *T, less func(a, b *T) bool, ) ([]*T, string) { - list := make([]*T, 0, len(store)) - for _, item := range store { + list := make([]*T, 0, len(items)) + for _, item := range items { list = append(list, clone(item)) } @@ -45,14 +87,17 @@ func sagemakerListPaged[T any]( return list[startIdx:end], outToken } -// sagemakerListKeyPaged paginates a store using name-key-based tokens. -// clone must return a deep copy of its argument. -func sagemakerListKeyPaged[T any]( - store map[string]*T, +// sagemakerListKeyPagedMap paginates a plain map[string]*T using name-key-based +// tokens. It preserves the pre-conversion behavior for the small number of +// resource collections not (yet) backed by a store.Table — e.g. a Cluster's +// Nodes, which is itself a value nested inside the Cluster store.Table entry +// rather than its own top-level registered table. +func sagemakerListKeyPagedMap[T any]( + m map[string]*T, nextToken string, clone func(*T) *T, ) ([]*T, string) { - keys := collections.SortedKeys(store) + keys := collections.SortedKeys(m) start := 0 if nextToken != "" { @@ -69,7 +114,7 @@ func sagemakerListKeyPaged[T any]( out := make([]*T, 0, end-start) for _, k := range keys[start:end] { - out = append(out, clone(store[k])) + out = append(out, clone(m[k])) } next := "" @@ -80,13 +125,54 @@ func sagemakerListKeyPaged[T any]( return out, next } +// sagemakerListKeyPaged paginates a store.Table using name-key-based tokens. +// keyFn must reproduce the exact same primary key the table was registered +// with (i.e. its keyFn), so nextToken values remain meaningful key strings. +// clone must return a deep copy of its argument. +func sagemakerListKeyPaged[T any]( + tbl *store.Table[T], + nextToken string, + clone func(*T) *T, + keyFn func(*T) string, +) ([]*T, string) { + // tbl.Snapshot() is already sorted ascending by the table's own primary + // key, i.e. by keyFn(item) — the same order collections.SortedKeys(map) + // produced over the raw map this table replaced. + items := tbl.Snapshot() + + start := 0 + if nextToken != "" { + for i, item := range items { + if keyFn(item) == nextToken { + start = i + + break + } + } + } + + end := min(start+sagemakerDefaultPageSize, len(items)) + + out := make([]*T, 0, end-start) + for _, item := range items[start:end] { + out = append(out, clone(item)) + } + + next := "" + if end < len(items) { + next = keyFn(items[end]) + } + + return out, next +} + // sagemakerCreate handles the common create-resource-by-name pattern: // acquire lock, check for duplicate, build ARN, build item, store, return clone. func sagemakerCreate[T any]( ctx context.Context, b *InMemoryBackend, opName, name, arnResource string, - storeOf func(string) map[string]*T, + storeOf func(string) *store.Table[T], dupErr func(string) error, build func(arnStr string, now time.Time) *T, clone func(*T) *T, @@ -95,8 +181,8 @@ func sagemakerCreate[T any]( b.mu.Lock(opName) defer b.mu.Unlock() - store := storeOf(region) - if _, ok := store[name]; ok { + tbl := storeOf(region) + if _, ok := tbl.Get(name); ok { return nil, dupErr(name) } @@ -104,7 +190,7 @@ func sagemakerCreate[T any]( now := time.Now() item := build(arnStr, now) - store[name] = item + tbl.Put(item) return clone(item), nil } diff --git a/services/sagemaker/backend_misc_destub.go b/services/sagemaker/backend_misc_destub.go index b1d57315d..2de525742 100644 --- a/services/sagemaker/backend_misc_destub.go +++ b/services/sagemaker/backend_misc_destub.go @@ -98,7 +98,7 @@ func (b *InMemoryBackend) DisassociateTrialComponent( ) key := trialComponentKey(trialName, trialComponentName) - delete(b.trialComponentAssociationsStore(region), key) + b.trialComponentAssociationsStore(region).Delete(key) return trialArn, trialComponentArn, nil } @@ -116,11 +116,11 @@ func (b *InMemoryBackend) ListTrialComponents( allowed := b.trialComponentNameFilterLocked(region, experimentName, trialName) - list := make([]*TrialComponent, 0, len(b.trialComponentsStore(region))) + list := make([]*TrialComponent, 0, b.trialComponentsStore(region).Len()) - for name, tc := range b.trialComponentsStore(region) { + for _, tc := range b.trialComponentsStore(region).All() { if allowed != nil { - if _, ok := allowed[name]; !ok { + if _, ok := allowed[tc.TrialComponentName]; !ok { continue } } @@ -148,7 +148,7 @@ func (b *InMemoryBackend) trialComponentNameFilterLocked( if trialName != "" { trialNames[trialName] = true } else { - for _, t := range b.trialsStore(region) { + for _, t := range b.trialsStore(region).All() { if t.ExperimentName == experimentName { trialNames[t.TrialName] = true } @@ -157,7 +157,7 @@ func (b *InMemoryBackend) trialComponentNameFilterLocked( allowed := map[string]bool{} - for _, assoc := range b.trialComponentAssociationsStore(region) { + for _, assoc := range b.trialComponentAssociationsStore(region).All() { if trialNames[assoc.TrialName] { allowed[assoc.TrialComponentName] = true } @@ -184,7 +184,7 @@ func (b *InMemoryBackend) ListImageAliases( region := getRegion(ctx, b.region) - if _, ok := b.smImagesStore(region)[imageName]; !ok { + if _, ok := b.smImagesStore(region).Get(imageName); !ok { return nil, "", fmt.Errorf("%w: image %q not found", ErrSMImageNotFound, imageName) } @@ -246,7 +246,7 @@ func (b *InMemoryBackend) UpdateProject( region := getRegion(ctx, b.region) - p, ok := b.projectsStore(region)[name] + p, ok := b.projectsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: project %q not found", ErrProjectNotFound, name) } diff --git a/services/sagemaker/backend_modelcard_export.go b/services/sagemaker/backend_modelcard_export.go index 566327d19..6183b0cd2 100644 --- a/services/sagemaker/backend_modelcard_export.go +++ b/services/sagemaker/backend_modelcard_export.go @@ -64,7 +64,7 @@ func (b *InMemoryBackend) CreateModelCardExportJob( return nil, fmt.Errorf("%w: OutputConfig.S3OutputPath is required", ErrValidation) } - card, ok := b.modelCardsStore(region)[modelCardName] + card, ok := b.modelCardsStore(region).Get(modelCardName) if !ok { return nil, fmt.Errorf("%w: model card %q not found", ErrModelCardNotFound, modelCardName) } @@ -89,7 +89,7 @@ func (b *InMemoryBackend) CreateModelCardExportJob( LastModifiedAt: now, } - b.modelCardExportJobsStore(region)[jobARN] = j + b.modelCardExportJobsStore(region).Put(j) return cloneModelCardExportJob(j), nil } @@ -104,7 +104,7 @@ func (b *InMemoryBackend) DescribeModelCardExportJob( b.mu.RLock("DescribeModelCardExportJob") defer b.mu.RUnlock() - j, ok := b.modelCardExportJobsStore(region)[jobArn] + j, ok := b.modelCardExportJobsStore(region).Get(jobArn) if !ok { return nil, fmt.Errorf("%w: model card export job %q not found", ErrModelCardExportJobNotFound, jobArn) } @@ -126,7 +126,7 @@ func (b *InMemoryBackend) ListModelCardExportJobs( list := make([]*ModelCardExportJob, 0) - for _, j := range b.modelCardExportJobsStore(region) { + for _, j := range b.modelCardExportJobsStore(region).All() { if modelCardName != "" && j.ModelCardName != modelCardName { continue } diff --git a/services/sagemaker/backend_new_ops.go b/services/sagemaker/backend_new_ops.go index 8ce06a8e7..10759f887 100644 --- a/services/sagemaker/backend_new_ops.go +++ b/services/sagemaker/backend_new_ops.go @@ -198,7 +198,7 @@ func (b *InMemoryBackend) CreateEndpoint( region := getRegion(ctx, b.region) - if _, ok := b.endpointsStore(region)[name]; ok { + if _, ok := b.endpointsStore(region).Get(name); ok { return nil, fmt.Errorf("%w: endpoint %s already exists", ErrEndpointAlreadyExists, name) } @@ -213,7 +213,7 @@ func (b *InMemoryBackend) CreateEndpoint( LastModifiedTime: now, Tags: mergeTags(nil, tags), } - b.endpointsStore(region)[name] = ep + b.endpointsStore(region).Put(ep) b.endpointARNIndexStore(region)[epARN] = name return cloneEndpoint(ep), nil @@ -226,7 +226,7 @@ func (b *InMemoryBackend) DescribeEndpoint(ctx context.Context, name string) (*E region := getRegion(ctx, b.region) - ep, ok := b.endpointsStore(region)[name] + ep, ok := b.endpointsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: endpoint %q not found", ErrEndpointNotFound, name) } @@ -252,7 +252,7 @@ func (b *InMemoryBackend) DeleteEndpoint(ctx context.Context, name string) error region := getRegion(ctx, b.region) - ep, ok := b.endpointsStore(region)[name] + ep, ok := b.endpointsStore(region).Get(name) if !ok { return fmt.Errorf("%w: endpoint %q not found", ErrEndpointNotFound, name) } @@ -260,7 +260,7 @@ func (b *InMemoryBackend) DeleteEndpoint(ctx context.Context, name string) error arnIdx := b.endpointARNIndexStore(region) delete(arnIdx, ep.EndpointArn) endpoints := b.endpointsStore(region) - delete(endpoints, name) + endpoints.Delete(name) return nil } @@ -272,7 +272,7 @@ func (b *InMemoryBackend) UpdateEndpoint(ctx context.Context, name, endpointConf region := getRegion(ctx, b.region) - ep, ok := b.endpointsStore(region)[name] + ep, ok := b.endpointsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: endpoint %q not found", ErrEndpointNotFound, name) } @@ -316,7 +316,7 @@ func (b *InMemoryBackend) DescribeTrainingJob(ctx context.Context, name string) region := getRegion(ctx, b.region) - tj, ok := b.trainingJobsStore(region)[name] + tj, ok := b.trainingJobsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: training job %q not found", ErrTrainingJobNotFound, name) } @@ -342,7 +342,7 @@ func (b *InMemoryBackend) StopTrainingJob(ctx context.Context, name string) erro region := getRegion(ctx, b.region) - tj, ok := b.trainingJobsStore(region)[name] + tj, ok := b.trainingJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: training job %q not found", ErrTrainingJobNotFound, name) } @@ -360,7 +360,7 @@ func (b *InMemoryBackend) DeleteTrainingJob(ctx context.Context, name string) er region := getRegion(ctx, b.region) - tj, ok := b.trainingJobsStore(region)[name] + tj, ok := b.trainingJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: training job %q not found", ErrTrainingJobNotFound, name) } @@ -368,7 +368,7 @@ func (b *InMemoryBackend) DeleteTrainingJob(ctx context.Context, name string) er arnIdx := b.trainingJobARNIndexStore(region) delete(arnIdx, tj.TrainingJobArn) store := b.trainingJobsStore(region) - delete(store, name) + store.Delete(name) return nil } @@ -400,7 +400,7 @@ func (b *InMemoryBackend) CreateNotebookInstance( region := getRegion(ctx, b.region) - if _, ok := b.notebooksStore(region)[name]; ok { + if _, ok := b.notebooksStore(region).Get(name); ok { return nil, fmt.Errorf( "%w: notebook instance %s already exists", ErrNotebookAlreadyExists, @@ -420,7 +420,7 @@ func (b *InMemoryBackend) CreateNotebookInstance( LastModifiedTime: now, Tags: mergeTags(nil, tags), } - b.notebooksStore(region)[name] = nb + b.notebooksStore(region).Put(nb) b.notebookARNIndexStore(region)[nbARN] = name return cloneNotebook(nb), nil @@ -433,7 +433,7 @@ func (b *InMemoryBackend) DescribeNotebookInstance(ctx context.Context, name str region := getRegion(ctx, b.region) - nb, ok := b.notebooksStore(region)[name] + nb, ok := b.notebooksStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: notebook instance %q not found", ErrNotebookNotFound, name) } @@ -462,8 +462,8 @@ func (b *InMemoryBackend) ListNotebookInstances( region := getRegion(ctx, b.region) store := b.notebooksStore(region) - list := make([]*NotebookInstance, 0, len(store)) - for _, nb := range store { + list := make([]*NotebookInstance, 0, store.Len()) + for _, nb := range store.All() { if !matchesNotebookFilter(nb, filter) { continue } @@ -515,7 +515,7 @@ func (b *InMemoryBackend) DeleteNotebookInstance(ctx context.Context, name strin region := getRegion(ctx, b.region) - nb, ok := b.notebooksStore(region)[name] + nb, ok := b.notebooksStore(region).Get(name) if !ok { return fmt.Errorf("%w: notebook instance %q not found", ErrNotebookNotFound, name) } @@ -523,7 +523,7 @@ func (b *InMemoryBackend) DeleteNotebookInstance(ctx context.Context, name strin arnIdx := b.notebookARNIndexStore(region) delete(arnIdx, nb.NotebookInstanceArn) store := b.notebooksStore(region) - delete(store, name) + store.Delete(name) return nil } @@ -535,7 +535,7 @@ func (b *InMemoryBackend) StartNotebookInstance(ctx context.Context, name string region := getRegion(ctx, b.region) - nb, ok := b.notebooksStore(region)[name] + nb, ok := b.notebooksStore(region).Get(name) if !ok { return fmt.Errorf("%w: notebook instance %q not found", ErrNotebookNotFound, name) } @@ -553,7 +553,7 @@ func (b *InMemoryBackend) StopNotebookInstance(ctx context.Context, name string) region := getRegion(ctx, b.region) - nb, ok := b.notebooksStore(region)[name] + nb, ok := b.notebooksStore(region).Get(name) if !ok { return fmt.Errorf("%w: notebook instance %q not found", ErrNotebookNotFound, name) } @@ -571,7 +571,7 @@ func (b *InMemoryBackend) UpdateNotebookInstance(ctx context.Context, name, inst region := getRegion(ctx, b.region) - nb, ok := b.notebooksStore(region)[name] + nb, ok := b.notebooksStore(region).Get(name) if !ok { return fmt.Errorf("%w: notebook instance %q not found", ErrNotebookNotFound, name) } @@ -591,7 +591,7 @@ func (b *InMemoryBackend) CreatePresignedNotebookInstanceURL(ctx context.Context region := getRegion(ctx, b.region) - nb, ok := b.notebooksStore(region)[name] + nb, ok := b.notebooksStore(region).Get(name) if !ok { return "", fmt.Errorf("%w: notebook instance %q not found", ErrNotebookNotFound, name) } @@ -616,7 +616,7 @@ func (b *InMemoryBackend) CreateHyperParameterTuningJob( region := getRegion(ctx, b.region) - if _, ok := b.hpTuningJobsStore(region)[name]; ok { + if _, ok := b.hpTuningJobsStore(region).Get(name); ok { return nil, fmt.Errorf( "%w: HP tuning job %s already exists", ErrHPTuningJobAlreadyExists, @@ -635,7 +635,7 @@ func (b *InMemoryBackend) CreateHyperParameterTuningJob( LastModifiedTime: now, Tags: mergeTags(nil, tags), } - b.hpTuningJobsStore(region)[name] = j + b.hpTuningJobsStore(region).Put(j) b.hpTuningJobARNIndexStore(region)[jobARN] = name return cloneHPTuningJob(j), nil @@ -651,7 +651,7 @@ func (b *InMemoryBackend) DescribeHyperParameterTuningJob( region := getRegion(ctx, b.region) - j, ok := b.hpTuningJobsStore(region)[name] + j, ok := b.hpTuningJobsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: HP tuning job %q not found", ErrHPTuningJobNotFound, name) } @@ -682,7 +682,7 @@ func (b *InMemoryBackend) StopHyperParameterTuningJob(ctx context.Context, name region := getRegion(ctx, b.region) - j, ok := b.hpTuningJobsStore(region)[name] + j, ok := b.hpTuningJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: HP tuning job %q not found", ErrHPTuningJobNotFound, name) } @@ -700,7 +700,7 @@ func (b *InMemoryBackend) DeleteHyperParameterTuningJob(ctx context.Context, nam region := getRegion(ctx, b.region) - j, ok := b.hpTuningJobsStore(region)[name] + j, ok := b.hpTuningJobsStore(region).Get(name) if !ok { return fmt.Errorf("%w: HP tuning job %q not found", ErrHPTuningJobNotFound, name) } @@ -708,7 +708,7 @@ func (b *InMemoryBackend) DeleteHyperParameterTuningJob(ctx context.Context, nam arnIdx := b.hpTuningJobARNIndexStore(region) delete(arnIdx, j.HyperParameterTuningJobArn) store := b.hpTuningJobsStore(region) - delete(store, name) + store.Delete(name) return nil } diff --git a/services/sagemaker/backend_pipeline_ops.go b/services/sagemaker/backend_pipeline_ops.go index 052701f36..addc586d7 100644 --- a/services/sagemaker/backend_pipeline_ops.go +++ b/services/sagemaker/backend_pipeline_ops.go @@ -20,6 +20,13 @@ type PipelineExecutionStep struct { StepType string `json:"StepType"` StepStatus string `json:"StepStatus"` FailureReason string `json:"FailureReason,omitempty"` + // ExecutionArn is the owning pipeline execution's ARN, carried internally so + // the store.Table's keyFn can derive pipelineExecutionStepsKey(ExecutionArn, + // StepName) and Restore can rebuild it after a JSON round-trip. Exported + // (with a json tag) solely so it survives persistence — handler.go never + // marshals PipelineExecutionStep directly to API callers, so this has no + // effect on the AWS wire shape. + ExecutionArn string `json:"executionArn"` } const ( @@ -49,7 +56,7 @@ func (b *InMemoryBackend) RetryPipelineExecution(ctx context.Context, execArn st region := getRegion(ctx, b.region) - pe, ok := b.pipelineExecutionsStore(region)[execArn] + pe, ok := b.pipelineExecutionsStore(region).Get(execArn) if !ok { return nil, fmt.Errorf( "%w: pipeline execution %q not found", @@ -68,14 +75,14 @@ func (b *InMemoryBackend) RetryPipelineExecution(ctx context.Context, execArn st PipelineExecutionStatus: pipelineStatusExecuting, StartTime: now, } - b.pipelineExecutionsStore(region)[newArn] = newExec + b.pipelineExecutionsStore(region).Put(newExec) // Transition to Succeeded after a short delay. b.runDelayed(b.lifecycleCtx, retryTransitionDelay, func() { b.mu.Lock("RetryPipelineExecution.goroutine") defer b.mu.Unlock() - if exec, exists := b.pipelineExecutionsStore(region)[newArn]; exists { + if exec, exists := b.pipelineExecutionsStore(region).Get(newArn); exists { exec.PipelineExecutionStatus = pipelineStatusSucceeded } }) @@ -90,7 +97,7 @@ func (b *InMemoryBackend) StopPipelineExecution(ctx context.Context, execArn str region := getRegion(ctx, b.region) - pe, ok := b.pipelineExecutionsStore(region)[execArn] + pe, ok := b.pipelineExecutionsStore(region).Get(execArn) if !ok { return nil, fmt.Errorf( "%w: pipeline execution %q not found", @@ -107,7 +114,7 @@ func (b *InMemoryBackend) StopPipelineExecution(ctx context.Context, execArn str b.mu.Lock("StopPipelineExecution.goroutine") defer b.mu.Unlock() - if exec, exists := b.pipelineExecutionsStore(region)[execArn]; exists { + if exec, exists := b.pipelineExecutionsStore(region).Get(execArn); exists { exec.PipelineExecutionStatus = pipelineStatusStopped } }) @@ -122,7 +129,7 @@ func (b *InMemoryBackend) SendPipelineExecutionStepSuccess(ctx context.Context, region := getRegion(ctx, b.region) - if _, ok := b.pipelineExecutionsStore(region)[execArn]; !ok { + if _, ok := b.pipelineExecutionsStore(region).Get(execArn); !ok { return fmt.Errorf( "%w: pipeline execution %q not found", ErrPipelineExecutionNotFound, @@ -130,16 +137,16 @@ func (b *InMemoryBackend) SendPipelineExecutionStepSuccess(ctx context.Context, ) } - key := pipelineExecutionStepsKey(execArn, stepName) now := time.Now() - b.pipelineExecStepsStore(region)[key] = &PipelineExecutionStep{ - StartTime: now, - EndTime: now, - StepName: stepName, - StepType: stepTypeCallback, - StepStatus: stepStatusSucceeded, - } + b.pipelineExecStepsStore(region).Put(&PipelineExecutionStep{ + StartTime: now, + EndTime: now, + StepName: stepName, + StepType: stepTypeCallback, + StepStatus: stepStatusSucceeded, + ExecutionArn: execArn, + }) return nil } @@ -153,7 +160,7 @@ func (b *InMemoryBackend) SendPipelineExecutionStepFailure( region := getRegion(ctx, b.region) - if _, ok := b.pipelineExecutionsStore(region)[execArn]; !ok { + if _, ok := b.pipelineExecutionsStore(region).Get(execArn); !ok { return fmt.Errorf( "%w: pipeline execution %q not found", ErrPipelineExecutionNotFound, @@ -161,17 +168,17 @@ func (b *InMemoryBackend) SendPipelineExecutionStepFailure( ) } - key := pipelineExecutionStepsKey(execArn, stepName) now := time.Now() - b.pipelineExecStepsStore(region)[key] = &PipelineExecutionStep{ + b.pipelineExecStepsStore(region).Put(&PipelineExecutionStep{ StartTime: now, EndTime: now, StepName: stepName, StepType: stepTypeCallback, StepStatus: stepStatusFailed, FailureReason: failureReason, - } + ExecutionArn: execArn, + }) return nil } @@ -185,11 +192,10 @@ func (b *InMemoryBackend) ListPipelineExecutionSteps( region := getRegion(ctx, b.region) - prefix := execArn + "|" - list := make([]*PipelineExecutionStep, 0, len(b.pipelineExecStepsStore(region))) + list := make([]*PipelineExecutionStep, 0, b.pipelineExecStepsStore(region).Len()) - for key, step := range b.pipelineExecStepsStore(region) { - if len(key) >= len(prefix) && key[:len(prefix)] == prefix { + for _, step := range b.pipelineExecStepsStore(region).All() { + if step.ExecutionArn == execArn { cp := *step list = append(list, &cp) } diff --git a/services/sagemaker/backend_pipeline_versions.go b/services/sagemaker/backend_pipeline_versions.go index 8e293c3bd..0abcdd8ab 100644 --- a/services/sagemaker/backend_pipeline_versions.go +++ b/services/sagemaker/backend_pipeline_versions.go @@ -81,7 +81,7 @@ func (b *InMemoryBackend) ListPipelineVersions( region := getRegion(ctx, b.region) - if _, ok := b.pipelinesStore(region)[pipelineName]; !ok { + if _, ok := b.pipelinesStore(region).Get(pipelineName); !ok { return nil, "", fmt.Errorf("%w: pipeline %q not found", ErrPipelineNotFound, pipelineName) } @@ -102,7 +102,7 @@ func (b *InMemoryBackend) ListPipelineVersions( // findPipelineByARNLocked returns the pipeline with the given ARN. Callers // must hold b.mu (read or write). func (b *InMemoryBackend) findPipelineByARNLocked(region, pipelineArn string) (*Pipeline, bool) { - for _, p := range b.pipelinesStore(region) { + for _, p := range b.pipelinesStore(region).All() { if p.PipelineArn == pipelineArn { return p, true } @@ -163,7 +163,7 @@ func (b *InMemoryBackend) DescribePipelineDefinitionForExecution( region := getRegion(ctx, b.region) - pe, ok := b.pipelineExecutionsStore(region)[execArn] + pe, ok := b.pipelineExecutionsStore(region).Get(execArn) if !ok { return "", time.Time{}, fmt.Errorf( "%w: pipeline execution %q not found", ErrPipelineExecutionNotFound, execArn, @@ -189,7 +189,7 @@ func (b *InMemoryBackend) UpdatePipelineExecution( region := getRegion(ctx, b.region) - pe, ok := b.pipelineExecutionsStore(region)[execArn] + pe, ok := b.pipelineExecutionsStore(region).Get(execArn) if !ok { return nil, fmt.Errorf( "%w: pipeline execution %q not found", ErrPipelineExecutionNotFound, execArn, diff --git a/services/sagemaker/backend_presigned_session.go b/services/sagemaker/backend_presigned_session.go index bf65b8a64..d8ec14f8d 100644 --- a/services/sagemaker/backend_presigned_session.go +++ b/services/sagemaker/backend_presigned_session.go @@ -21,9 +21,9 @@ func (b *InMemoryBackend) CreatePresignedDomainURL( region := getRegion(ctx, b.region) - d, ok := b.domainsStore(region)[domainID] + d, ok := b.domainsStore(region).Get(domainID) if !ok { - for _, cand := range b.domainsStore(region) { + for _, cand := range b.domainsStore(region).All() { if cand.DomainName == domainID { d = cand ok = true @@ -37,8 +37,8 @@ func (b *InMemoryBackend) CreatePresignedDomainURL( return "", fmt.Errorf("%w: domain %q not found", ErrDomainNotFound, domainID) } - key := userProfileKey{DomainID: d.DomainID, UserProfileName: userProfileName} - if _, profileOK := b.userProfilesStore(region)[key]; !profileOK { + key := userProfileKeyString(userProfileKey{DomainID: d.DomainID, UserProfileName: userProfileName}) + if _, profileOK := b.userProfilesStore(region).Get(key); !profileOK { return "", fmt.Errorf( "%w: user profile %q not found in domain %q", ErrUserProfileNotFound, userProfileName, domainID, ) diff --git a/services/sagemaker/backend_stateful_ops.go b/services/sagemaker/backend_stateful_ops.go index d16aff473..c77102b0d 100644 --- a/services/sagemaker/backend_stateful_ops.go +++ b/services/sagemaker/backend_stateful_ops.go @@ -85,6 +85,12 @@ type userProfileKey struct { UserProfileName string } +// userProfileKeyString flattens a userProfileKey to the single delimited +// string used as the store.Table primary key for b.userProfiles. +func userProfileKeyString(k userProfileKey) string { + return k.DomainID + "|" + k.UserProfileName +} + // UserProfile represents a SageMaker Studio user profile. type UserProfile struct { CreationTime time.Time `json:"CreationTime"` @@ -111,6 +117,12 @@ type appKey struct { AppName string } +// appKeyString flattens an appKey to the single delimited string used as the +// store.Table primary key for b.apps. +func appKeyString(k appKey) string { + return k.DomainID + "|" + k.UserProfileName + "|" + k.AppType + "|" + k.AppName +} + // App represents a SageMaker Studio app. type App struct { CreationTime time.Time `json:"CreationTime"` @@ -321,7 +333,7 @@ func (b *InMemoryBackend) CreateDomain( region := getRegion(ctx, b.region) - for _, d := range b.domainsStore(region) { + for _, d := range b.domainsStore(region).All() { if d.DomainName == name { return nil, fmt.Errorf("%w: domain %s already exists", ErrDomainAlreadyExists, name) } @@ -342,7 +354,7 @@ func (b *InMemoryBackend) CreateDomain( LastModifiedTime: now, Tags: mergeTags(nil, tags), } - b.domainsStore(region)[id] = d + b.domainsStore(region).Put(d) return cloneDomain(d), nil } @@ -354,11 +366,11 @@ func (b *InMemoryBackend) DescribeDomain(ctx context.Context, idOrName string) ( region := getRegion(ctx, b.region) - if d, ok := b.domainsStore(region)[idOrName]; ok { + if d, ok := b.domainsStore(region).Get(idOrName); ok { return cloneDomain(d), nil } - for _, d := range b.domainsStore(region) { + for _, d := range b.domainsStore(region).All() { if d.DomainName == idOrName { return cloneDomain(d), nil } @@ -386,9 +398,9 @@ func (b *InMemoryBackend) DeleteDomain(ctx context.Context, idOrName string) err region := getRegion(ctx, b.region) store := b.domainsStore(region) - for id, d := range store { - if id == idOrName || d.DomainName == idOrName { - delete(store, id) + for _, d := range store.All() { + if d.DomainID == idOrName || d.DomainName == idOrName { + store.Delete(d.DomainID) return nil } @@ -404,7 +416,7 @@ func (b *InMemoryBackend) UpdateDomain(ctx context.Context, idOrName string) (*D region := getRegion(ctx, b.region) - for _, d := range b.domainsStore(region) { + for _, d := range b.domainsStore(region).All() { if d.DomainID == idOrName || d.DomainName == idOrName { d.LastModifiedTime = time.Now() @@ -430,8 +442,8 @@ func (b *InMemoryBackend) CreateUserProfile( region := getRegion(ctx, b.region) - key := userProfileKey{DomainID: domainID, UserProfileName: name} - if _, ok := b.userProfilesStore(region)[key]; ok { + key := userProfileKeyString(userProfileKey{DomainID: domainID, UserProfileName: name}) + if _, ok := b.userProfilesStore(region).Get(key); ok { return nil, fmt.Errorf( "%w: user profile %s in domain %s already exists", ErrUserProfileAlreadyExists, @@ -457,7 +469,7 @@ func (b *InMemoryBackend) CreateUserProfile( LastModifiedTime: now, Tags: mergeTags(nil, tags), } - b.userProfilesStore(region)[key] = up + b.userProfilesStore(region).Put(up) return cloneUserProfile(up), nil } @@ -469,9 +481,9 @@ func (b *InMemoryBackend) DescribeUserProfile(ctx context.Context, domainID, nam region := getRegion(ctx, b.region) - key := userProfileKey{DomainID: domainID, UserProfileName: name} + key := userProfileKeyString(userProfileKey{DomainID: domainID, UserProfileName: name}) - up, ok := b.userProfilesStore(region)[key] + up, ok := b.userProfilesStore(region).Get(key) if !ok { return nil, fmt.Errorf( "%w: user profile %q in domain %q not found", @@ -493,9 +505,9 @@ func (b *InMemoryBackend) ListUserProfiles(ctx context.Context, domainID, nextTo region := getRegion(ctx, b.region) store := b.userProfilesStore(region) - list := make([]*UserProfile, 0, len(store)) + list := make([]*UserProfile, 0, store.Len()) - for _, up := range store { + for _, up := range store.All() { if domainID == "" || up.DomainID == domainID { list = append(list, cloneUserProfile(up)) } @@ -531,8 +543,8 @@ func (b *InMemoryBackend) DeleteUserProfile(ctx context.Context, domainID, name region := getRegion(ctx, b.region) store := b.userProfilesStore(region) - key := userProfileKey{DomainID: domainID, UserProfileName: name} - if _, ok := store[key]; !ok { + key := userProfileKeyString(userProfileKey{DomainID: domainID, UserProfileName: name}) + if _, ok := store.Get(key); !ok { return fmt.Errorf( "%w: user profile %q in domain %q not found", ErrUserProfileNotFound, @@ -541,7 +553,7 @@ func (b *InMemoryBackend) DeleteUserProfile(ctx context.Context, domainID, name ) } - delete(store, key) + store.Delete(key) return nil } @@ -561,13 +573,13 @@ func (b *InMemoryBackend) CreateApp( region := getRegion(ctx, b.region) - key := appKey{ + key := appKeyString(appKey{ DomainID: domainID, UserProfileName: userProfile, AppType: appType, AppName: appName, - } - if _, ok := b.appsStore(region)[key]; ok { + }) + if _, ok := b.appsStore(region).Get(key); ok { return nil, fmt.Errorf("%w: app %s already exists", ErrAppAlreadyExists, appName) } @@ -585,7 +597,7 @@ func (b *InMemoryBackend) CreateApp( CreationTime: now, Tags: mergeTags(nil, tags), } - b.appsStore(region)[key] = a + b.appsStore(region).Put(a) return cloneApp(a), nil } @@ -600,14 +612,14 @@ func (b *InMemoryBackend) DescribeApp( region := getRegion(ctx, b.region) - key := appKey{ + key := appKeyString(appKey{ DomainID: domainID, UserProfileName: userProfile, AppType: appType, AppName: appName, - } + }) - a, ok := b.appsStore(region)[key] + a, ok := b.appsStore(region).Get(key) if !ok { return nil, fmt.Errorf("%w: app %q not found", ErrAppNotFound, appName) } @@ -624,9 +636,9 @@ func (b *InMemoryBackend) ListApps(ctx context.Context, domainID, nextToken stri region := getRegion(ctx, b.region) store := b.appsStore(region) - list := make([]*App, 0, len(store)) + list := make([]*App, 0, store.Len()) - for _, a := range store { + for _, a := range store.All() { if domainID == "" || a.DomainID == domainID { list = append(list, cloneApp(a)) } @@ -659,17 +671,17 @@ func (b *InMemoryBackend) DeleteApp(ctx context.Context, domainID, userProfile, region := getRegion(ctx, b.region) store := b.appsStore(region) - key := appKey{ + key := appKeyString(appKey{ DomainID: domainID, UserProfileName: userProfile, AppType: appType, AppName: appName, - } - if _, ok := store[key]; !ok { + }) + if _, ok := store.Get(key); !ok { return fmt.Errorf("%w: app %q not found", ErrAppNotFound, appName) } - delete(store, key) + store.Delete(key) return nil } @@ -690,7 +702,7 @@ func (b *InMemoryBackend) CreateFeatureGroup( region := getRegion(ctx, b.region) - if _, ok := b.featureGroupsStore(region)[name]; ok { + if _, ok := b.featureGroupsStore(region).Get(name); ok { return nil, fmt.Errorf( "%w: feature group %s already exists", ErrFeatureGroupAlreadyExists, @@ -712,7 +724,7 @@ func (b *InMemoryBackend) CreateFeatureGroup( CreationTime: time.Now(), Tags: mergeTags(nil, tags), } - b.featureGroupsStore(region)[name] = fg + b.featureGroupsStore(region).Put(fg) return cloneFeatureGroup(fg), nil } @@ -724,7 +736,7 @@ func (b *InMemoryBackend) DescribeFeatureGroup(ctx context.Context, name string) region := getRegion(ctx, b.region) - fg, ok := b.featureGroupsStore(region)[name] + fg, ok := b.featureGroupsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: feature group %q not found", ErrFeatureGroupNotFound, name) } @@ -751,11 +763,11 @@ func (b *InMemoryBackend) DeleteFeatureGroup(ctx context.Context, name string) e region := getRegion(ctx, b.region) store := b.featureGroupsStore(region) - if _, ok := store[name]; !ok { + if _, ok := store.Get(name); !ok { return fmt.Errorf("%w: feature group %q not found", ErrFeatureGroupNotFound, name) } - delete(store, name) + store.Delete(name) return nil } @@ -775,7 +787,7 @@ func (b *InMemoryBackend) CreatePipeline( region := getRegion(ctx, b.region) - if _, ok := b.pipelinesStore(region)[name]; ok { + if _, ok := b.pipelinesStore(region).Get(name); ok { return nil, fmt.Errorf("%w: pipeline %s already exists", ErrPipelineAlreadyExists, name) } @@ -792,7 +804,7 @@ func (b *InMemoryBackend) CreatePipeline( LastModifiedTime: now, Tags: mergeTags(nil, tags), } - b.pipelinesStore(region)[name] = p + b.pipelinesStore(region).Put(p) return clonePipeline(p), nil } @@ -804,7 +816,7 @@ func (b *InMemoryBackend) DescribePipeline(ctx context.Context, name string) (*P region := getRegion(ctx, b.region) - p, ok := b.pipelinesStore(region)[name] + p, ok := b.pipelinesStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: pipeline %q not found", ErrPipelineNotFound, name) } @@ -830,7 +842,7 @@ func (b *InMemoryBackend) UpdatePipeline(ctx context.Context, name, definition s region := getRegion(ctx, b.region) - p, ok := b.pipelinesStore(region)[name] + p, ok := b.pipelinesStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: pipeline %q not found", ErrPipelineNotFound, name) } @@ -852,13 +864,13 @@ func (b *InMemoryBackend) DeletePipeline(ctx context.Context, name string) (*Pip region := getRegion(ctx, b.region) store := b.pipelinesStore(region) - p, ok := store[name] + p, ok := store.Get(name) if !ok { return nil, fmt.Errorf("%w: pipeline %q not found", ErrPipelineNotFound, name) } cp := clonePipeline(p) - delete(store, name) + store.Delete(name) delete(b.pipelineVersionsStore(region), name) return cp, nil @@ -871,7 +883,7 @@ func (b *InMemoryBackend) StartPipelineExecution(ctx context.Context, pipelineNa region := getRegion(ctx, b.region) - p, ok := b.pipelinesStore(region)[pipelineName] + p, ok := b.pipelinesStore(region).Get(pipelineName) if !ok { return nil, fmt.Errorf("%w: pipeline %q not found", ErrPipelineNotFound, pipelineName) } @@ -885,7 +897,7 @@ func (b *InMemoryBackend) StartPipelineExecution(ctx context.Context, pipelineNa PipelineExecutionStatus: pipelineStatusSucceeded, StartTime: time.Now(), } - b.pipelineExecutionsStore(region)[execArn] = pe + b.pipelineExecutionsStore(region).Put(pe) return clonePipelineExecution(pe), nil } @@ -897,7 +909,7 @@ func (b *InMemoryBackend) DescribePipelineExecution(ctx context.Context, execArn region := getRegion(ctx, b.region) - pe, ok := b.pipelineExecutionsStore(region)[execArn] + pe, ok := b.pipelineExecutionsStore(region).Get(execArn) if !ok { return nil, fmt.Errorf( "%w: pipeline execution %q not found", @@ -919,12 +931,12 @@ func (b *InMemoryBackend) ListPipelineExecutions( region := getRegion(ctx, b.region) - p, ok := b.pipelinesStore(region)[pipelineName] + p, ok := b.pipelinesStore(region).Get(pipelineName) execStore := b.pipelineExecutionsStore(region) - list := make([]*PipelineExecution, 0, len(execStore)) + list := make([]*PipelineExecution, 0, execStore.Len()) if ok { - for _, pe := range execStore { + for _, pe := range execStore.All() { if pe.PipelineArn == p.PipelineArn { list = append(list, clonePipelineExecution(pe)) } @@ -967,7 +979,7 @@ func (b *InMemoryBackend) CreateExperiment( region := getRegion(ctx, b.region) - if _, ok := b.experimentsStore(region)[name]; ok { + if _, ok := b.experimentsStore(region).Get(name); ok { return nil, fmt.Errorf("%w: experiment %s already exists", ErrExperimentAlreadyExists, name) } @@ -981,7 +993,7 @@ func (b *InMemoryBackend) CreateExperiment( LastModifiedTime: now, Tags: mergeTags(nil, tags), } - b.experimentsStore(region)[name] = e + b.experimentsStore(region).Put(e) return cloneExperiment(e), nil } @@ -993,7 +1005,7 @@ func (b *InMemoryBackend) DescribeExperiment(ctx context.Context, name string) ( region := getRegion(ctx, b.region) - e, ok := b.experimentsStore(region)[name] + e, ok := b.experimentsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: experiment %q not found", ErrExperimentNotFound, name) } @@ -1020,13 +1032,13 @@ func (b *InMemoryBackend) DeleteExperiment(ctx context.Context, name string) (*E region := getRegion(ctx, b.region) store := b.experimentsStore(region) - e, ok := store[name] + e, ok := store.Get(name) if !ok { return nil, fmt.Errorf("%w: experiment %q not found", ErrExperimentNotFound, name) } cp := cloneExperiment(e) - delete(store, name) + store.Delete(name) return cp, nil } @@ -1046,7 +1058,7 @@ func (b *InMemoryBackend) CreateTrial( region := getRegion(ctx, b.region) - if _, ok := b.trialsStore(region)[name]; ok { + if _, ok := b.trialsStore(region).Get(name); ok { return nil, fmt.Errorf("%w: trial %s already exists", ErrTrialAlreadyExists, name) } @@ -1061,7 +1073,7 @@ func (b *InMemoryBackend) CreateTrial( LastModifiedTime: now, Tags: mergeTags(nil, tags), } - b.trialsStore(region)[name] = t + b.trialsStore(region).Put(t) return cloneTrial(t), nil } @@ -1073,7 +1085,7 @@ func (b *InMemoryBackend) DescribeTrial(ctx context.Context, name string) (*Tria region := getRegion(ctx, b.region) - t, ok := b.trialsStore(region)[name] + t, ok := b.trialsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: trial %q not found", ErrTrialNotFound, name) } @@ -1100,13 +1112,13 @@ func (b *InMemoryBackend) DeleteTrial(ctx context.Context, name string) (*Trial, region := getRegion(ctx, b.region) store := b.trialsStore(region) - t, ok := store[name] + t, ok := store.Get(name) if !ok { return nil, fmt.Errorf("%w: trial %q not found", ErrTrialNotFound, name) } cp := cloneTrial(t) - delete(store, name) + store.Delete(name) return cp, nil } @@ -1126,7 +1138,7 @@ func (b *InMemoryBackend) CreateTrialComponent( region := getRegion(ctx, b.region) - if _, ok := b.trialComponentsStore(region)[name]; ok { + if _, ok := b.trialComponentsStore(region).Get(name); ok { return nil, fmt.Errorf( "%w: trial component %s already exists", ErrTrialComponentAlreadyExists, @@ -1144,7 +1156,7 @@ func (b *InMemoryBackend) CreateTrialComponent( LastModifiedTime: now, Tags: mergeTags(nil, tags), } - b.trialComponentsStore(region)[name] = tc + b.trialComponentsStore(region).Put(tc) return cloneTrialComponent(tc), nil } @@ -1156,7 +1168,7 @@ func (b *InMemoryBackend) DescribeTrialComponent(ctx context.Context, name strin region := getRegion(ctx, b.region) - tc, ok := b.trialComponentsStore(region)[name] + tc, ok := b.trialComponentsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: trial component %q not found", ErrTrialComponentNotFound, name) } @@ -1172,13 +1184,13 @@ func (b *InMemoryBackend) DeleteTrialComponent(ctx context.Context, name string) region := getRegion(ctx, b.region) store := b.trialComponentsStore(region) - tc, ok := store[name] + tc, ok := store.Get(name) if !ok { return nil, fmt.Errorf("%w: trial component %q not found", ErrTrialComponentNotFound, name) } cp := cloneTrialComponent(tc) - delete(store, name) + store.Delete(name) return cp, nil } @@ -1198,7 +1210,7 @@ func (b *InMemoryBackend) UpdateFeatureGroup( region := getRegion(ctx, b.region) - fg, ok := b.featureGroupsStore(region)[name] + fg, ok := b.featureGroupsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: feature group %q not found", ErrFeatureGroupNotFound, name) } @@ -1220,7 +1232,7 @@ func (b *InMemoryBackend) UpdateExperiment( region := getRegion(ctx, b.region) - e, ok := b.experimentsStore(region)[name] + e, ok := b.experimentsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: experiment %q not found", ErrExperimentNotFound, name) } @@ -1243,7 +1255,7 @@ func (b *InMemoryBackend) UpdateTrial(ctx context.Context, name, displayName str region := getRegion(ctx, b.region) - t, ok := b.trialsStore(region)[name] + t, ok := b.trialsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: trial %q not found", ErrTrialNotFound, name) } @@ -1276,7 +1288,7 @@ func (b *InMemoryBackend) UpdateTrialComponent( region := getRegion(ctx, b.region) - tc, ok := b.trialComponentsStore(region)[name] + tc, ok := b.trialComponentsStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: trial component %q not found", ErrTrialComponentNotFound, name) } @@ -1328,7 +1340,7 @@ func (b *InMemoryBackend) CreatePipelineFull(ctx context.Context, opts CreatePip region := getRegion(ctx, b.region) - if _, ok := b.pipelinesStore(region)[opts.PipelineName]; ok { + if _, ok := b.pipelinesStore(region).Get(opts.PipelineName); ok { return nil, fmt.Errorf( "%w: pipeline %s already exists", ErrPipelineAlreadyExists, @@ -1352,7 +1364,7 @@ func (b *InMemoryBackend) CreatePipelineFull(ctx context.Context, opts CreatePip LastModifiedTime: now, Tags: mergeTags(nil, opts.Tags), } - b.pipelinesStore(region)[opts.PipelineName] = p + b.pipelinesStore(region).Put(p) b.recordPipelineVersionLocked(region, p) return clonePipeline(p), nil @@ -1369,7 +1381,7 @@ func (b *InMemoryBackend) UpdatePipelineFull( region := getRegion(ctx, b.region) - p, ok := b.pipelinesStore(region)[name] + p, ok := b.pipelinesStore(region).Get(name) if !ok { return nil, fmt.Errorf("%w: pipeline %q not found", ErrPipelineNotFound, name) } @@ -1414,7 +1426,7 @@ func (b *InMemoryBackend) StartPipelineExecutionFull( region := getRegion(ctx, b.region) - p, ok := b.pipelinesStore(region)[opts.PipelineName] + p, ok := b.pipelinesStore(region).Get(opts.PipelineName) if !ok { return nil, fmt.Errorf("%w: pipeline %q not found", ErrPipelineNotFound, opts.PipelineName) } @@ -1436,7 +1448,7 @@ func (b *InMemoryBackend) StartPipelineExecutionFull( ParallelismConfiguration: opts.ParallelismConfiguration, StartTime: time.Now(), } - b.pipelineExecutionsStore(region)[execArn] = pe + b.pipelineExecutionsStore(region).Put(pe) b.recordPipelineExecutionOnLatestVersionLocked(region, opts.PipelineName, execArn) return clonePipelineExecution(pe), nil diff --git a/services/sagemaker/backend_training_plan_ext.go b/services/sagemaker/backend_training_plan_ext.go index e5432a701..456448ce5 100644 --- a/services/sagemaker/backend_training_plan_ext.go +++ b/services/sagemaker/backend_training_plan_ext.go @@ -158,7 +158,7 @@ func (b *InMemoryBackend) createReservedCapacity( rc.ReservedCapacityType = "Instance" } - b.reservedCapacitiesStore(region)[rcArn] = rc + b.reservedCapacitiesStore(region).Put(rc) return rc } @@ -173,7 +173,7 @@ func (b *InMemoryBackend) DescribeReservedCapacity( b.mu.RLock("DescribeReservedCapacity") defer b.mu.RUnlock() - rc, ok := b.reservedCapacitiesStore(region)[reservedCapacityArn] + rc, ok := b.reservedCapacitiesStore(region).Get(reservedCapacityArn) if !ok { return nil, fmt.Errorf("%w: reserved capacity %q not found", ErrReservedCapacityNotFound, reservedCapacityArn) } @@ -192,7 +192,7 @@ func (b *InMemoryBackend) ListUltraServersByReservedCapacity( b.mu.RLock("ListUltraServersByReservedCapacity") defer b.mu.RUnlock() - rc, ok := b.reservedCapacitiesStore(region)[reservedCapacityArn] + rc, ok := b.reservedCapacitiesStore(region).Get(reservedCapacityArn) if !ok { return nil, "", fmt.Errorf( "%w: reserved capacity %q not found", ErrReservedCapacityNotFound, reservedCapacityArn, @@ -364,6 +364,7 @@ type pendingTrainingPlanExtension struct { EndDate time.Time `json:"EndDate"` TrainingPlanArn string `json:"TrainingPlanArn"` CurrencyCode string `json:"CurrencyCode"` + ID string `json:"id"` DurationHours int32 `json:"DurationHours"` } @@ -445,13 +446,14 @@ func (b *InMemoryBackend) generateExtensionOfferings( id := "ext-" + generateID() end := base.Add(time.Duration(d) * time.Hour) - store[id] = &pendingTrainingPlanExtension{ + store.Put(&pendingTrainingPlanExtension{ TrainingPlanArn: plan.TrainingPlanArn, DurationHours: d, StartDate: base, EndDate: end, CurrencyCode: plan.CurrencyCode, - } + ID: id, + }) out = append(out, TrainingPlanExtensionOffering{ TrainingPlanExtensionOfferingID: id, @@ -478,7 +480,7 @@ func (b *InMemoryBackend) ExtendTrainingPlan( offerings := b.trainingPlanExtensionOfferingsStore(region) - pending, ok := offerings[extensionOfferingID] + pending, ok := offerings.Get(extensionOfferingID) if !ok { return nil, fmt.Errorf( "%w: extension offering %q not found", ErrValidation, extensionOfferingID, @@ -503,7 +505,7 @@ func (b *InMemoryBackend) ExtendTrainingPlan( plan.DurationHours += int64(pending.DurationHours) endDate := pending.EndDate plan.EndTime = &endDate - delete(offerings, extensionOfferingID) + offerings.Delete(extensionOfferingID) return append([]*TrainingPlanExtension(nil), plan.Extensions...), nil } @@ -540,7 +542,7 @@ func (b *InMemoryBackend) DescribeTrainingPlanExtensionHistory( // findTrainingPlanByARN scans the region's training plans for a matching ARN. // Called with b.mu held (read or write). func (b *InMemoryBackend) findTrainingPlanByARN(region, trainingPlanArn string) (*TrainingPlan, bool) { - for _, t := range b.trainingPlansStore(region) { + for _, t := range b.trainingPlansStore(region).All() { if t.TrainingPlanArn == trainingPlanArn { return t, true } @@ -573,9 +575,9 @@ func (b *InMemoryBackend) ListTrainingPlans( defer b.mu.RUnlock() store := b.trainingPlansStore(region) - list := make([]*TrainingPlan, 0, len(store)) + list := make([]*TrainingPlan, 0, store.Len()) - for _, t := range store { + for _, t := range store.All() { if params.StatusEquals != "" && t.Status != params.StatusEquals { continue } diff --git a/services/sagemaker/export_test.go b/services/sagemaker/export_test.go index 01fa07af4..cbbe73801 100644 --- a/services/sagemaker/export_test.go +++ b/services/sagemaker/export_test.go @@ -1,9 +1,11 @@ package sagemaker -func sumRegions[T any](m map[string]map[string]*T) int { +import "github.com/blackbirdworks/gopherstack/pkgs/store" + +func sumRegions[T any](m map[string]*store.Table[T]) int { total := 0 - for _, regionMap := range m { - total += len(regionMap) + for _, tbl := range m { + total += tbl.Len() } return total @@ -86,8 +88,7 @@ func SeedMonitoringExecution(b *InMemoryBackend, region string, e *MonitoringExe b.mu.Lock("SeedMonitoringExecution") defer b.mu.Unlock() - store := b.monitoringExecutionsStore(region) - store[e.MonitoringScheduleName+"|"+e.ProcessingJobArn] = e + b.monitoringExecutionsStore(region).Put(e) } // SeedMonitoringAlertHistory inserts an alert-history entry directly into the diff --git a/services/sagemaker/persistence.go b/services/sagemaker/persistence.go index ed320cbe8..1dbd38f94 100644 --- a/services/sagemaker/persistence.go +++ b/services/sagemaker/persistence.go @@ -3,12 +3,33 @@ package sagemaker import ( "context" "encoding/json" + "fmt" + "strings" "time" "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" ) -// persistedCluster is a serialisable version of Cluster that includes Nodes. +// sagemakerSnapshotVersion identifies the shape of [backendSnapshot]. It must +// be bumped whenever a change would make an older snapshot unsafe to decode +// as the current shape. Restore compares this against the persisted value +// and discards (rather than attempts to partially decode) any mismatch — see +// Restore below. This is bumped from the pre-Phase-3.3 format, which had no +// Version field at all and persisted resources as region-nested raw maps +// rather than store.Registry tables, so any snapshot taken before this +// pkgs/store conversion is treated as "missing/incompatible" and discarded +// cleanly rather than misinterpreted. +const sagemakerSnapshotVersion = 1 + +// persistedCluster is a serialisable DTO for Cluster. It exists — mirroring +// the SQS pilot's queueSnapshot — because Cluster is a "dirty" struct for +// persistence purposes: Nodes carries `json:"-"` (hidden from the AWS wire +// shape returned to API callers) but must still be persisted, and +// CreationTime is persisted as an explicit RFC3339 string (this backend's +// pre-conversion snapshot format) rather than store.Table's default +// json.Marshal-of-time.Time encoding. type persistedCluster struct { CreationTime string `json:"CreationTime"` Nodes map[string]*ClusterNode `json:"Nodes"` @@ -20,690 +41,344 @@ type persistedCluster struct { InstanceGroups []ClusterInstanceGroup `json:"InstanceGroups,omitempty"` } -// backendSnapshot holds the serialisable state of InMemoryBackend. -// All resource maps are nested by region (outer key = region). -type backendSnapshot struct { - Models map[string]map[string]*Model `json:"models"` - EndpointConfigs map[string]map[string]*EndpointConfig `json:"endpointConfigs"` - Endpoints map[string]map[string]*Endpoint `json:"endpoints"` - TrainingJobs map[string]map[string]*TrainingJob `json:"trainingJobs"` - Notebooks map[string]map[string]*NotebookInstance `json:"notebooks"` - HPTuningJobs map[string]map[string]*HyperParameterTuningJob `json:"hpTuningJobs"` - Associations map[string]map[string]*Association `json:"associations"` - TrialComponentAssociations map[string]map[string]*TrialComponentAssociation `json:"trialComponentAssociations"` - Actions map[string]map[string]*Action `json:"actions"` - Artifacts map[string]map[string]*Artifact `json:"artifacts"` - Contexts map[string]map[string]*Context `json:"contexts"` - Algorithms map[string]map[string]*Algorithm `json:"algorithms"` - Clusters map[string]map[string]*persistedCluster `json:"clusters"` - ModelPackages map[string]map[string]*ModelPackage `json:"modelPackages"` - Domains map[string]map[string]*Domain `json:"domains"` - // UserProfiles is stored as region → "domainID|profileName" → UserProfile. - UserProfiles map[string]map[string]*UserProfile `json:"userProfiles"` - // Apps is stored as region → "domainID|userProfileName|appType|appName" → App. - Apps map[string]map[string]*App `json:"apps"` - FeatureGroups map[string]map[string]*FeatureGroup `json:"featureGroups"` - Pipelines map[string]map[string]*Pipeline `json:"pipelines"` - PipelineExecutions map[string]map[string]*PipelineExecution `json:"pipelineExecutions"` - PipelineExecSteps map[string]map[string]*PipelineExecutionStep `json:"pipelineExecSteps"` - Experiments map[string]map[string]*Experiment `json:"experiments"` - Trials map[string]map[string]*Trial `json:"trials"` - TrialComponents map[string]map[string]*TrialComponent `json:"trialComponents"` - NotebookLifecycleConfigs map[string]map[string]*NotebookInstanceLifecycleConfig `json:"notebookLifecycleConfigs"` - ProcessingJobs map[string]map[string]*ProcessingJob `json:"processingJobs"` - TransformJobs map[string]map[string]*TransformJob `json:"transformJobs"` - EdgeDeploymentPlans map[string]map[string]*EdgeDeploymentPlan `json:"edgeDeploymentPlans"` - FeatureRecords map[string]map[string]*FeatureRecord `json:"featureRecords"` - FeatureMetadata map[string]map[string]*FeatureMetadata `json:"featureMetadata"` - Hubs map[string]map[string]*Hub `json:"hubs"` - // HubContents is stored as region → "hubName|contentType|contentName|contentVersion" → HubContent. - HubContents map[string]map[string]*HubContent `json:"hubContents"` - // Model Monitor job definitions (Create/Describe/Delete/List*JobDefinitions). - DataQualityJobDefs map[string]map[string]*JobDefinition `json:"dataQualityJobDefs"` - ModelBiasJobDefs map[string]map[string]*JobDefinition `json:"modelBiasJobDefs"` - ModelQualityJobDefs map[string]map[string]*JobDefinition `json:"modelQualityJobDefs"` - ModelExplainJobDefs map[string]map[string]*JobDefinition `json:"modelExplainJobDefs"` - // MonitoringAlerts is stored as region → monitoringScheduleName → alertName → MonitoringAlert. - MonitoringAlerts map[string]map[string]map[string]*MonitoringAlert `json:"monitoringAlerts"` - // MonitoringAlertHistory is stored as region → history entries (across all schedules/alerts). - MonitoringAlertHistory map[string][]*MonitoringAlertHistoryEntry `json:"monitoringAlertHistory"` - // MonitoringExecutions is stored as region → "scheduleName|processingJobArn" → MonitoringExecution. - MonitoringExecutions map[string]map[string]*MonitoringExecution `json:"monitoringExecutions"` - Workteams map[string]map[string]*Workteam `json:"workteams"` - Workforces map[string]map[string]*Workforce `json:"workforces"` - LabelingJobs map[string]map[string]*LabelingJob `json:"labelingJobs"` - MlflowTrackingServers map[string]map[string]*MlflowTrackingServer `json:"mlflowTrackingServers"` - MlflowApps map[string]map[string]*MlflowApp `json:"mlflowApps"` - PartnerApps map[string]map[string]*PartnerApp `json:"partnerApps"` - TrainingPlans map[string]map[string]*TrainingPlan `json:"trainingPlans"` - ReservedCapacities map[string]map[string]*ReservedCapacity `json:"reservedCapacities"` - // TrainingPlanExtensionOfferings is stored as region -> extensionOfferingID -> pending extension offer. - TrainingPlanExtensionOfferings pendingExtensionsByRegion `json:"trainingPlanExtensionOfferings"` - ModelCardExportJobs map[string]map[string]*ModelCardExportJob `json:"modelCardExportJobs"` - ModelPackageGroups map[string]map[string]*ModelPackageGroup `json:"modelPackageGroups"` - // PipelineVersions is stored as region -> pipelineName -> version history. - PipelineVersions map[string]map[string][]*PipelineVersion `json:"pipelineVersions"` - // ServicecatalogPortfolioEnabled is region -> whether the Service Catalog - // portfolio has been enabled. - ServicecatalogPortfolioEnabled map[string]bool `json:"servicecatalogPortfolioEnabled"` - AccountID string `json:"accountID"` - Region string `json:"region"` -} +// toPersistedCluster converts a live Cluster to its persisted DTO shape. +func toPersistedCluster(c *Cluster) *persistedCluster { + pc := &persistedCluster{ + CreationTime: c.CreationTime.Format(time.RFC3339), + ClusterArn: c.ClusterArn, + ClusterName: c.ClusterName, + ClusterStatus: c.ClusterStatus, + NodeRecovery: c.NodeRecovery, + Tags: c.Tags, + InstanceGroups: c.InstanceGroups, + Nodes: make(map[string]*ClusterNode, len(c.Nodes)), + } -// pendingExtensionsByRegion is region -> extensionOfferingID -> pending -// training plan extension offer awaiting an ExtendTrainingPlan call. -type pendingExtensionsByRegion = map[string]map[string]*pendingTrainingPlanExtension - -// snapshotClusters converts map[string]map[string]*Cluster → -// map[string]map[string]*persistedCluster. -func snapshotClusters(b *InMemoryBackend) map[string]map[string]*persistedCluster { - clusters := make(map[string]map[string]*persistedCluster, len(b.clusters)) - - for region, regionClusters := range b.clusters { - clusters[region] = make(map[string]*persistedCluster, len(regionClusters)) - for k, c := range regionClusters { - pc := &persistedCluster{ - CreationTime: c.CreationTime.Format("2006-01-02T15:04:05Z07:00"), - ClusterArn: c.ClusterArn, - ClusterName: c.ClusterName, - ClusterStatus: c.ClusterStatus, - NodeRecovery: c.NodeRecovery, - Tags: c.Tags, - InstanceGroups: c.InstanceGroups, - Nodes: make(map[string]*ClusterNode, len(c.Nodes)), - } - for nk, nv := range c.Nodes { - nodeCopy := *nv - pc.Nodes[nk] = &nodeCopy - } - - clusters[region][k] = pc - } + for nk, nv := range c.Nodes { + nodeCopy := *nv + pc.Nodes[nk] = &nodeCopy } - return clusters + return pc } -// snapshotUserProfiles converts map[string]map[userProfileKey]*UserProfile → -// map[string]map[string]*UserProfile (inner key = "domainID|profileName"). -func snapshotUserProfiles(b *InMemoryBackend) map[string]map[string]*UserProfile { - userProfiles := make(map[string]map[string]*UserProfile, len(b.userProfiles)) - - for region, regionProfiles := range b.userProfiles { - userProfiles[region] = make(map[string]*UserProfile, len(regionProfiles)) - for k, v := range regionProfiles { - cp := *v - userProfiles[region][k.DomainID+"|"+k.UserProfileName] = &cp - } +// fromPersistedCluster rebuilds a live Cluster from its persisted DTO shape. +func fromPersistedCluster(ctx context.Context, pc *persistedCluster) *Cluster { + t, err := time.Parse(time.RFC3339, pc.CreationTime) + if err != nil { + logger.Load(ctx).WarnContext(ctx, + "sagemaker: failed to parse cluster creation time", "cluster", pc.ClusterName, "error", err) } - return userProfiles -} - -// snapshotApps converts map[string]map[appKey]*App → map[string]map[string]*App -// (inner key = "domainID|userProfileName|appType|appName"). -func snapshotApps(b *InMemoryBackend) map[string]map[string]*App { - apps := make(map[string]map[string]*App, len(b.apps)) + c := &Cluster{ + ClusterArn: pc.ClusterArn, + ClusterName: pc.ClusterName, + ClusterStatus: pc.ClusterStatus, + NodeRecovery: pc.NodeRecovery, + Tags: pc.Tags, + InstanceGroups: pc.InstanceGroups, + CreationTime: t, + Nodes: make(map[string]*ClusterNode, len(pc.Nodes)), + } - for region, regionApps := range b.apps { - apps[region] = make(map[string]*App, len(regionApps)) - for k, v := range regionApps { - cp := *v - apps[region][k.DomainID+"|"+k.UserProfileName+"|"+k.AppType+"|"+k.AppName] = &cp - } + for nk, nv := range pc.Nodes { + nodeCopy := *nv + c.Nodes[nk] = &nodeCopy } - return apps + return c } -// snapshotHubContents converts map[string]map[hubContentKey]*HubContent → -// map[string]map[string]*HubContent (inner key = -// "hubName|contentType|contentName|contentVersion"). -func snapshotHubContents(b *InMemoryBackend) map[string]map[string]*HubContent { - hubContents := make(map[string]map[string]*HubContent, len(b.hubContents)) - - for region, regionContents := range b.hubContents { - hubContents[region] = make(map[string]*HubContent, len(regionContents)) - for k, v := range regionContents { - cp := *v - hubContents[region][hubContentKeyString(k)] = &cp - } - } +// clusterTablePrefix names the per-region entries carved out of +// backendSnapshot.Tables for DTO-based Cluster encoding — see Snapshot/Restore. +const clusterTablePrefix = "clusters:" + +// hubContentKeyString serialises a hubContentKey to a single delimited string +// for use as the store.Table primary key for b.hubContents. +func hubContentKeyString(k hubContentKey) string { + return k.HubName + "|" + k.HubContentType + "|" + k.HubContentName + "|" + k.HubContentVersion +} - return hubContents +// backendSnapshot is the top-level on-disk shape for the SageMaker backend. +// +// Tables holds one JSON-encoded array per registered store.Table (produced by +// [store.Registry.SnapshotAll]) — one entry per (resource, region) pair, +// e.g. "models:us-east-1", "endpointConfigs:us-east-1". Clusters are the one +// exception: their table entries are re-encoded through [persistedCluster] +// (see Snapshot/Restore) rather than left as the registry's generic +// json.Marshal-of-Cluster output. +// +// The five fields below are not (yet) backed by store.Table — see the doc +// comments on the corresponding InMemoryBackend fields in backend.go for why +// each one doesn't fit the Table[V] keyed-by-string-derived-from-V model — +// so they are persisted directly here instead of via Tables. +type backendSnapshot struct { + Tables map[string]json.RawMessage `json:"tables"` + ImageVersions map[string]map[string]map[int]*ImageVersion `json:"imageVersions,omitempty"` + ImageVersionCounts map[string]map[string]int `json:"imageVersionCounts,omitempty"` + MonitoringAlertHistory map[string][]*MonitoringAlertHistoryEntry `json:"monitoringAlertHistory,omitempty"` + PipelineVersions map[string]map[string][]*PipelineVersion `json:"pipelineVersions,omitempty"` + SCPortfolioEnabled map[string]bool `json:"servicecatalogPortfolioEnabled,omitempty"` + AccountID string `json:"accountID"` + Region string `json:"region"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. +// It implements persistence.Persistable. func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() - clusters := snapshotClusters(b) - userProfiles := snapshotUserProfiles(b) - apps := snapshotApps(b) - hubContents := snapshotHubContents(b) + tables, err := b.registry.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, "sagemaker: snapshot table marshal failed", "error", err) - snap := backendSnapshot{ - Models: b.models, - EndpointConfigs: b.endpointConfigs, - Endpoints: b.endpoints, - TrainingJobs: b.trainingJobs, - Notebooks: b.notebooks, - HPTuningJobs: b.hpTuningJobs, - Associations: b.associations, - TrialComponentAssociations: b.trialComponentAssociations, - Actions: b.actions, - Artifacts: b.artifacts, - Contexts: b.contexts, - Algorithms: b.algorithms, - Clusters: clusters, - ModelPackages: b.modelPackages, - Domains: b.domains, - UserProfiles: userProfiles, - Apps: apps, - FeatureGroups: b.featureGroups, - Pipelines: b.pipelines, - PipelineExecutions: b.pipelineExecutions, - PipelineExecSteps: b.pipelineExecSteps, - Experiments: b.experiments, - Trials: b.trials, - TrialComponents: b.trialComponents, - NotebookLifecycleConfigs: b.notebookLifecycleConfigs, - ProcessingJobs: b.processingJobs, - TransformJobs: b.transformJobs, - EdgeDeploymentPlans: b.edgeDeploymentPlans, - FeatureRecords: b.featureRecords, - FeatureMetadata: b.featureMetadata, - Hubs: b.hubs, - HubContents: hubContents, - DataQualityJobDefs: b.dataQualityJobDefs, - ModelBiasJobDefs: b.modelBiasJobDefs, - ModelQualityJobDefs: b.modelQualityJobDefs, - ModelExplainJobDefs: b.modelExplainJobDefs, - MonitoringAlerts: b.monitoringAlerts, - MonitoringAlertHistory: b.monitoringAlertHistory, - MonitoringExecutions: b.monitoringExecutions, - Workteams: b.workteams, - Workforces: b.workforces, - LabelingJobs: b.labelingJobs, - MlflowTrackingServers: b.mlflowTrackingServers, - MlflowApps: b.mlflowApps, - PartnerApps: b.partnerApps, - TrainingPlans: b.trainingPlans, - ReservedCapacities: b.reservedCapacities, - TrainingPlanExtensionOfferings: b.trainingPlanExtensionOfferings, - ModelCardExportJobs: b.modelCardExportJobs, - ModelPackageGroups: b.modelPackageGroups, - PipelineVersions: b.pipelineVersions, - ServicecatalogPortfolioEnabled: b.servicecatalogPortfolioEnabled, - AccountID: b.accountID, - Region: b.region, + return nil } - data, err := json.Marshal(snap) - if err != nil { - logger.Load(ctx).WarnContext(ctx, "sagemaker: failed to marshal snapshot", "error", err) + // Overwrite the generic per-region cluster table encoding (which would + // silently drop Nodes, since Cluster.Nodes carries `json:"-"`) with the + // persistedCluster DTO shape. + for region, tbl := range b.clusters { + clusters := tbl.Snapshot() + dtos := make([]*persistedCluster, len(clusters)) - return nil + for i, c := range clusters { + dtos[i] = toPersistedCluster(c) + } + + data, marshalErr := json.Marshal(dtos) + if marshalErr != nil { + logger.Load(ctx).WarnContext(ctx, + "sagemaker: snapshot cluster table marshal failed", "region", region, "error", marshalErr) + + return nil + } + + tables[clusterTablePrefix+region] = data + } + + snap := backendSnapshot{ + Version: sagemakerSnapshotVersion, + Tables: tables, + ImageVersions: b.imageVersions, + ImageVersionCounts: b.imageVersionCounts, + MonitoringAlertHistory: b.monitoringAlertHistory, + PipelineVersions: b.pipelineVersions, + SCPortfolioEnabled: b.servicecatalogPortfolioEnabled, + AccountID: b.accountID, + Region: b.region, } - return data + return persistence.MarshalSnapshot(ctx, "sagemaker", snap) } // Restore loads backend state from a JSON snapshot. +// It implements persistence.Persistable. func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { var snap backendSnapshot - if err := json.Unmarshal(data, &snap); err != nil { + if err := persistence.UnmarshalSnapshot(ctx, "sagemaker", data, &snap); err != nil { return err } - ensureNonNilMaps(&snap) - fixNilTagMaps(&snap) - b.mu.Lock("Restore") defer b.mu.Unlock() - b.restoreFields(ctx, &snap) - b.resetLifecycleContext() // cancel pending goroutines, create fresh context - b.rebuildARNIndexes() + if snap.Version != sagemakerSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape — that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across this Phase 3.3 snapshot-format + // change), not data corruption. + logger.Load(ctx).WarnContext(ctx, + "sagemaker: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", sagemakerSnapshotVersion) + + b.registry.ResetAll() + b.resetNonTableFieldsLocked() + b.resetLifecycleContext() - return nil -} + return nil + } + + // Cluster entries need DTO decoding (see Snapshot): pull them out of the + // raw table map before delegating the rest to registry.RestoreAll, then + // restore each region's store.Table[Cluster] from the decoded DTOs + // afterward. Any region not present in the snapshot is left absent from + // b.clusters here; RestoreAll's "reset tables absent from data" behavior + // (since we've stripped every "clusters:*" key from snap.Tables) clears + // any such region's table if it was already registered. + clusterTables := make(map[string]json.RawMessage) + + for name, raw := range snap.Tables { + if region, ok := strings.CutPrefix(name, clusterTablePrefix); ok { + clusterTables[region] = raw -// restoreFields assigns deserialized maps to backend fields (called with lock held). -func restoreUserProfiles(snap *backendSnapshot) map[string]map[userProfileKey]*UserProfile { - result := make(map[string]map[userProfileKey]*UserProfile, len(snap.UserProfiles)) - for region, regionProfiles := range snap.UserProfiles { - result[region] = make(map[userProfileKey]*UserProfile, len(regionProfiles)) - for _, v := range regionProfiles { - key := userProfileKey{DomainID: v.DomainID, UserProfileName: v.UserProfileName} - cp := *v - result[region][key] = &cp + delete(snap.Tables, name) } } - return result -} + // registry.RestoreAll only visits tables already registered in the + // registry; every resource table is registered lazily per-region on + // first use (see e.g. modelsStore), so a freshly constructed backend — + // the common Restore target — starts with none registered. Without this, + // RestoreAll would silently skip every region the backend hasn't + // happened to touch yet, dropping that data on the floor. + b.ensureTablesRegistered(snap.Tables) -func restoreApps(snap *backendSnapshot) map[string]map[appKey]*App { - result := make(map[string]map[appKey]*App, len(snap.Apps)) - for region, regionApps := range snap.Apps { - result[region] = make(map[appKey]*App, len(regionApps)) - for _, v := range regionApps { - key := appKey{ - DomainID: v.DomainID, - UserProfileName: v.UserProfileName, - AppType: v.AppType, - AppName: v.AppName, - } - cp := *v - result[region][key] = &cp - } + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("sagemaker: restore snapshot tables: %w", err) } - return result -} + for region, raw := range clusterTables { + var dtos []*persistedCluster -// hubContentKeyString serialises a hubContentKey to a single delimited string -// for use as a JSON map key. -func hubContentKeyString(k hubContentKey) string { - return k.HubName + "|" + k.HubContentType + "|" + k.HubContentName + "|" + k.HubContentVersion -} + if err := json.Unmarshal(raw, &dtos); err != nil { + return fmt.Errorf("sagemaker: restore cluster table %q: %w", region, err) + } -func restoreHubContents(snap *backendSnapshot) map[string]map[hubContentKey]*HubContent { - result := make(map[string]map[hubContentKey]*HubContent, len(snap.HubContents)) - for region, regionContents := range snap.HubContents { - result[region] = make(map[hubContentKey]*HubContent, len(regionContents)) - for _, v := range regionContents { - key := hubContentKey{ - HubName: v.HubName, HubContentType: v.HubContentType, - HubContentName: v.HubContentName, HubContentVersion: v.HubContentVersion, - } - cp := *v - result[region][key] = &cp + clusters := make([]*Cluster, len(dtos)) + for i, pc := range dtos { + clusters[i] = fromPersistedCluster(ctx, pc) } + + b.clustersStore(region).Restore(clusters) } - return result -} + b.imageVersions = snap.ImageVersions + b.imageVersionCounts = snap.ImageVersionCounts + b.monitoringAlertHistory = snap.MonitoringAlertHistory + b.pipelineVersions = snap.PipelineVersions + b.servicecatalogPortfolioEnabled = snap.SCPortfolioEnabled -func restoreClusters(ctx context.Context, snap *backendSnapshot) map[string]map[string]*Cluster { - result := make(map[string]map[string]*Cluster, len(snap.Clusters)) - for region, regionClusters := range snap.Clusters { - result[region] = make(map[string]*Cluster, len(regionClusters)) - for k, pc := range regionClusters { - t, err := time.Parse("2006-01-02T15:04:05Z07:00", pc.CreationTime) - if err != nil { - logger.Load(ctx).WarnContext(ctx, - "sagemaker: failed to parse cluster creation time", "cluster", k, "error", err) - } - c := &Cluster{ - ClusterArn: pc.ClusterArn, - ClusterName: pc.ClusterName, - ClusterStatus: pc.ClusterStatus, - NodeRecovery: pc.NodeRecovery, - Tags: ensureSageTagMap(pc.Tags), - InstanceGroups: pc.InstanceGroups, - CreationTime: t, - Nodes: make(map[string]*ClusterNode, len(pc.Nodes)), - } - for nk, nv := range pc.Nodes { - nodeCopy := *nv - c.Nodes[nk] = &nodeCopy - } - result[region][k] = c - } + if b.imageVersions == nil { + b.imageVersions = make(map[string]map[string]map[int]*ImageVersion) } - return result -} + if b.imageVersionCounts == nil { + b.imageVersionCounts = make(map[string]map[string]int) + } + + if b.monitoringAlertHistory == nil { + b.monitoringAlertHistory = make(map[string][]*MonitoringAlertHistoryEntry) + } + + if b.pipelineVersions == nil { + b.pipelineVersions = make(map[string]map[string][]*PipelineVersion) + } + + if b.servicecatalogPortfolioEnabled == nil { + b.servicecatalogPortfolioEnabled = make(map[string]bool) + } -func (b *InMemoryBackend) restoreFields(ctx context.Context, snap *backendSnapshot) { - b.models = snap.Models - b.endpointConfigs = snap.EndpointConfigs - b.endpoints = snap.Endpoints - b.trainingJobs = snap.TrainingJobs - b.notebooks = snap.Notebooks - b.hpTuningJobs = snap.HPTuningJobs - b.associations = snap.Associations - b.trialComponentAssociations = snap.TrialComponentAssociations - b.actions = snap.Actions - b.artifacts = snap.Artifacts - b.contexts = snap.Contexts - b.algorithms = snap.Algorithms - b.modelPackages = snap.ModelPackages - b.domains = snap.Domains - b.featureGroups = snap.FeatureGroups - b.pipelines = snap.Pipelines - b.pipelineExecutions = snap.PipelineExecutions - b.pipelineExecSteps = snap.PipelineExecSteps - b.experiments = snap.Experiments - b.trials = snap.Trials - b.trialComponents = snap.TrialComponents - b.notebookLifecycleConfigs = snap.NotebookLifecycleConfigs - b.processingJobs = snap.ProcessingJobs - b.transformJobs = snap.TransformJobs - b.edgeDeploymentPlans = snap.EdgeDeploymentPlans - b.featureRecords = snap.FeatureRecords - b.featureMetadata = snap.FeatureMetadata - b.hubs = snap.Hubs b.accountID = snap.AccountID b.region = snap.Region - b.userProfiles = restoreUserProfiles(snap) - b.apps = restoreApps(snap) - b.clusters = restoreClusters(ctx, snap) - b.hubContents = restoreHubContents(snap) - b.dataQualityJobDefs = snap.DataQualityJobDefs - b.modelBiasJobDefs = snap.ModelBiasJobDefs - b.modelQualityJobDefs = snap.ModelQualityJobDefs - b.modelExplainJobDefs = snap.ModelExplainJobDefs - b.monitoringAlerts = snap.MonitoringAlerts - b.monitoringAlertHistory = snap.MonitoringAlertHistory - b.monitoringExecutions = snap.MonitoringExecutions - b.workteams = snap.Workteams - b.workforces = snap.Workforces - b.labelingJobs = snap.LabelingJobs - b.mlflowTrackingServers = snap.MlflowTrackingServers - b.mlflowApps = snap.MlflowApps - b.partnerApps = snap.PartnerApps - b.restoreTrainingPlanExtFields(snap) + + b.resetLifecycleContext() + b.rebuildARNIndexes() + + return nil } -// restoreTrainingPlanExtFields assigns the Training Plan / Reserved Capacity -// and ModelCard export job fields from snap. Split out from restoreFields to -// keep that method's statement count low. -func (b *InMemoryBackend) restoreTrainingPlanExtFields(snap *backendSnapshot) { - b.trainingPlans = snap.TrainingPlans - b.reservedCapacities = snap.ReservedCapacities - b.trainingPlanExtensionOfferings = snap.TrainingPlanExtensionOfferings - b.modelCardExportJobs = snap.ModelCardExportJobs - b.modelPackageGroups = snap.ModelPackageGroups - b.pipelineVersions = snap.PipelineVersions - b.servicecatalogPortfolioEnabled = snap.ServicecatalogPortfolioEnabled +// resetNonTableFieldsLocked clears the handful of resource collections not +// backed by store.Table (and therefore not covered by registry.ResetAll), +// plus the ARN indexes (which are pure caches derived from the Table-backed +// resource maps). Callers must hold b.mu. +func (b *InMemoryBackend) resetNonTableFieldsLocked() { + b.imageVersions = make(map[string]map[string]map[int]*ImageVersion) + b.imageVersionCounts = make(map[string]map[string]int) + b.monitoringAlertHistory = make(map[string][]*MonitoringAlertHistoryEntry) + b.pipelineVersions = make(map[string]map[string][]*PipelineVersion) + b.servicecatalogPortfolioEnabled = make(map[string]bool) + b.rebuildARNIndexes() } -func buildARNIndex[V any](src map[string]map[string]V, arnFn func(string, V) string) map[string]map[string]string { +// buildARNIndex reconstructs a region -> ARN -> name index from a region -> +// store.Table[V] resource map. arnFn derives both the resource's own primary +// name/key and its ARN from the stored value (rather than from the table's +// internal key, which [store.Table] does not expose) so the index can be +// rebuilt purely by scanning each table's contents after a restore. +func buildARNIndex[V any]( + src map[string]*store.Table[V], + arnFn func(*V) (name, arn string), +) map[string]map[string]string { idx := make(map[string]map[string]string, len(src)) - for region, regionItems := range src { - regionIdx := make(map[string]string, len(regionItems)) - for name, item := range regionItems { - regionIdx[arnFn(name, item)] = name - } - idx[region] = regionIdx - } - return idx -} + for region, tbl := range src { + regionIdx := make(map[string]string, tbl.Len()) -func fixNestedTagsSage[V any](nested map[string]map[string]V, fix func(V)) { - for _, region := range nested { - for _, item := range region { - fix(item) + for _, item := range tbl.All() { + name, arn := arnFn(item) + regionIdx[arn] = name } - } -} -func ensureSageTagMap(m map[string]string) map[string]string { - if m == nil { - return make(map[string]string) + idx[region] = regionIdx } - return m + return idx } -// rebuildARNIndexes reconstructs all ARN-to-name indexes after a restore (called with lock held). +// rebuildARNIndexes reconstructs all ARN-to-name indexes after a restore +// (called with lock held). These indexes are pure derived caches over the +// Table-backed resource maps (see the "modelARNIndex" family of fields in +// backend.go) — never persisted themselves. func (b *InMemoryBackend) rebuildARNIndexes() { - b.modelARNIndex = buildARNIndex(b.models, func(_ string, m *Model) string { return m.ModelARN }) + b.modelARNIndex = buildARNIndex(b.models, func(m *Model) (string, string) { return m.ModelName, m.ModelARN }) b.endpointConfigARNIndex = buildARNIndex( b.endpointConfigs, - func(_ string, ec *EndpointConfig) string { return ec.EndpointConfigARN }, + func(ec *EndpointConfig) (string, string) { return ec.EndpointConfigName, ec.EndpointConfigARN }, + ) + b.actionARNIndex = buildARNIndex(b.actions, func(a *Action) (string, string) { return a.ActionName, a.ActionArn }) + b.contextARNIndex = buildARNIndex( + b.contexts, + func(c *Context) (string, string) { return c.ContextName, c.ContextArn }, + ) + b.algorithmARNIndex = buildARNIndex( + b.algorithms, + func(al *Algorithm) (string, string) { return al.AlgorithmName, al.AlgorithmArn }, + ) + b.clusterARNIndex = buildARNIndex( + b.clusters, + func(c *Cluster) (string, string) { return c.ClusterName, c.ClusterArn }, + ) + // ModelPackage's own primary store.Table key is already its ARN (see + // modelsStore-family doc in backend.go), so this index degenerates to + // ARN -> ARN — preserved exactly from the pre-conversion behavior. + b.modelPackageARNIndex = buildARNIndex( + b.modelPackages, + func(mp *ModelPackage) (string, string) { return mp.ModelPackageArn, mp.ModelPackageArn }, + ) + b.endpointARNIndex = buildARNIndex( + b.endpoints, + func(ep *Endpoint) (string, string) { return ep.EndpointName, ep.EndpointArn }, ) - b.actionARNIndex = buildARNIndex(b.actions, func(_ string, a *Action) string { return a.ActionArn }) - b.contextARNIndex = buildARNIndex(b.contexts, func(_ string, c *Context) string { return c.ContextArn }) - b.algorithmARNIndex = buildARNIndex(b.algorithms, func(_ string, al *Algorithm) string { return al.AlgorithmArn }) - b.clusterARNIndex = buildARNIndex(b.clusters, func(_ string, c *Cluster) string { return c.ClusterArn }) - b.modelPackageARNIndex = buildARNIndex(b.modelPackages, func(name string, _ *ModelPackage) string { return name }) - b.endpointARNIndex = buildARNIndex(b.endpoints, func(_ string, ep *Endpoint) string { return ep.EndpointArn }) b.trainingJobARNIndex = buildARNIndex( b.trainingJobs, - func(_ string, tj *TrainingJob) string { return tj.TrainingJobArn }, + func(tj *TrainingJob) (string, string) { return tj.TrainingJobName, tj.TrainingJobArn }, ) b.notebookARNIndex = buildARNIndex( b.notebooks, - func(_ string, nb *NotebookInstance) string { return nb.NotebookInstanceArn }, + func(nb *NotebookInstance) (string, string) { return nb.NotebookInstanceName, nb.NotebookInstanceArn }, ) b.hpTuningJobARNIndex = buildARNIndex( b.hpTuningJobs, - func(_ string, j *HyperParameterTuningJob) string { return j.HyperParameterTuningJobArn }, + func(j *HyperParameterTuningJob) (string, string) { + return j.HyperParameterTuningJobName, j.HyperParameterTuningJobArn + }, ) b.processingJobARNIndex = buildARNIndex( b.processingJobs, - func(_ string, pj *ProcessingJob) string { return pj.ProcessingJobArn }, + func(pj *ProcessingJob) (string, string) { return pj.ProcessingJobName, pj.ProcessingJobArn }, ) b.transformJobARNIndex = buildARNIndex( b.transformJobs, - func(_ string, tj *TransformJob) string { return tj.TransformJobArn }, + func(tj *TransformJob) (string, string) { return tj.TransformJobName, tj.TransformJobArn }, ) } -func ensureNonNilMaps(snap *backendSnapshot) { - ensureCoreResourceMaps(snap) - ensureJobMaps(snap) - ensureConfigMaps(snap) - ensureMetadataMaps(snap) - ensureLineageMaps(snap) - ensureHubMaps(snap) - ensureMonitorMaps(snap) - ensureTrainingPlanExtMaps(snap) -} - -// ensureMonitorMaps initialises the Model Monitor job definition and -// alert/execution maps if a snapshot predates their introduction. -func ensureMonitorMaps(snap *backendSnapshot) { - if snap.DataQualityJobDefs == nil { - snap.DataQualityJobDefs = make(map[string]map[string]*JobDefinition) - } - if snap.ModelBiasJobDefs == nil { - snap.ModelBiasJobDefs = make(map[string]map[string]*JobDefinition) - } - if snap.ModelQualityJobDefs == nil { - snap.ModelQualityJobDefs = make(map[string]map[string]*JobDefinition) - } - if snap.ModelExplainJobDefs == nil { - snap.ModelExplainJobDefs = make(map[string]map[string]*JobDefinition) - } - if snap.MonitoringAlerts == nil { - snap.MonitoringAlerts = make(map[string]map[string]map[string]*MonitoringAlert) - } - if snap.MonitoringAlertHistory == nil { - snap.MonitoringAlertHistory = make(map[string][]*MonitoringAlertHistoryEntry) - } - if snap.MonitoringExecutions == nil { - snap.MonitoringExecutions = make(map[string]map[string]*MonitoringExecution) - } - if snap.Workteams == nil { - snap.Workteams = make(map[string]map[string]*Workteam) - } - if snap.Workforces == nil { - snap.Workforces = make(map[string]map[string]*Workforce) - } - if snap.LabelingJobs == nil { - snap.LabelingJobs = make(map[string]map[string]*LabelingJob) - } - if snap.MlflowTrackingServers == nil { - snap.MlflowTrackingServers = make(map[string]map[string]*MlflowTrackingServer) - } - if snap.MlflowApps == nil { - snap.MlflowApps = make(map[string]map[string]*MlflowApp) - } - if snap.PartnerApps == nil { - snap.PartnerApps = make(map[string]map[string]*PartnerApp) - } -} - -// ensureTrainingPlanExtMaps initialises the Training Plan / Reserved -// Capacity and ModelCard export job maps if a snapshot predates their -// introduction. -func ensureTrainingPlanExtMaps(snap *backendSnapshot) { - if snap.TrainingPlans == nil { - snap.TrainingPlans = make(map[string]map[string]*TrainingPlan) - } - - if snap.ReservedCapacities == nil { - snap.ReservedCapacities = make(map[string]map[string]*ReservedCapacity) - } - - if snap.TrainingPlanExtensionOfferings == nil { - snap.TrainingPlanExtensionOfferings = make(map[string]map[string]*pendingTrainingPlanExtension) - } - - if snap.ModelCardExportJobs == nil { - snap.ModelCardExportJobs = make(map[string]map[string]*ModelCardExportJob) - } - - if snap.ModelPackageGroups == nil { - snap.ModelPackageGroups = make(map[string]map[string]*ModelPackageGroup) - } - - if snap.PipelineVersions == nil { - snap.PipelineVersions = make(map[string]map[string][]*PipelineVersion) - } - - if snap.ServicecatalogPortfolioEnabled == nil { - snap.ServicecatalogPortfolioEnabled = make(map[string]bool) - } -} - -func ensureHubMaps(snap *backendSnapshot) { - if snap.Hubs == nil { - snap.Hubs = make(map[string]map[string]*Hub) - } - - if snap.HubContents == nil { - snap.HubContents = make(map[string]map[string]*HubContent) - } -} - -func ensureLineageMaps(snap *backendSnapshot) { - if snap.Artifacts == nil { - snap.Artifacts = make(map[string]map[string]*Artifact) - } - if snap.Contexts == nil { - snap.Contexts = make(map[string]map[string]*Context) - } -} - -func ensureCoreResourceMaps(snap *backendSnapshot) { - if snap.Models == nil { - snap.Models = make(map[string]map[string]*Model) - } - if snap.EndpointConfigs == nil { - snap.EndpointConfigs = make(map[string]map[string]*EndpointConfig) - } - if snap.Endpoints == nil { - snap.Endpoints = make(map[string]map[string]*Endpoint) - } - if snap.Actions == nil { - snap.Actions = make(map[string]map[string]*Action) - } - if snap.Algorithms == nil { - snap.Algorithms = make(map[string]map[string]*Algorithm) - } - if snap.ModelPackages == nil { - snap.ModelPackages = make(map[string]map[string]*ModelPackage) - } -} - -// ensureNestedMap initialises *m to an empty map if it is nil. It is used to -// backfill region-nested resource maps that may be absent from snapshots -// taken before the corresponding resource family was introduced. -func ensureNestedMap[K comparable, V any](m *map[string]map[K]V) { - if *m == nil { - *m = make(map[string]map[K]V) - } -} - -func ensureJobMaps(snap *backendSnapshot) { - ensureNestedMap(&snap.TrainingJobs) - ensureNestedMap(&snap.Notebooks) - ensureNestedMap(&snap.HPTuningJobs) - ensureNestedMap(&snap.ProcessingJobs) - ensureNestedMap(&snap.TransformJobs) - ensureNestedMap(&snap.EdgeDeploymentPlans) - ensureNestedMap(&snap.FeatureRecords) - ensureNestedMap(&snap.FeatureMetadata) -} - -func ensureConfigMaps(snap *backendSnapshot) { - if snap.Domains == nil { - snap.Domains = make(map[string]map[string]*Domain) - } - if snap.UserProfiles == nil { - snap.UserProfiles = make(map[string]map[string]*UserProfile) - } - if snap.Apps == nil { - snap.Apps = make(map[string]map[string]*App) - } - if snap.FeatureGroups == nil { - snap.FeatureGroups = make(map[string]map[string]*FeatureGroup) - } - if snap.NotebookLifecycleConfigs == nil { - snap.NotebookLifecycleConfigs = make(map[string]map[string]*NotebookInstanceLifecycleConfig) - } - if snap.Clusters == nil { - snap.Clusters = make(map[string]map[string]*persistedCluster) - } -} - -func ensureMetadataMaps(snap *backendSnapshot) { - ensureNestedMap(&snap.Pipelines) - ensureNestedMap(&snap.PipelineExecutions) - ensureNestedMap(&snap.PipelineExecSteps) - ensureNestedMap(&snap.Experiments) - ensureNestedMap(&snap.Trials) - ensureNestedMap(&snap.TrialComponents) - ensureNestedMap(&snap.Associations) - ensureNestedMap(&snap.TrialComponentAssociations) -} - -func fixNilTagMaps(snap *backendSnapshot) { - fixNilTagMapsCoreResources(snap) - fixNilTagMapsNewResources(snap) -} - -func fixNilTagMapsCoreResources(snap *backendSnapshot) { - fixNestedTagsSage(snap.Models, func(m *Model) { m.Tags = ensureSageTagMap(m.Tags) }) - fixNestedTagsSage(snap.EndpointConfigs, func(ec *EndpointConfig) { ec.Tags = ensureSageTagMap(ec.Tags) }) - fixNestedTagsSage(snap.Actions, func(a *Action) { a.Tags = ensureSageTagMap(a.Tags) }) - fixNestedTagsSage(snap.Artifacts, func(ar *Artifact) { ar.Tags = ensureSageTagMap(ar.Tags) }) - fixNestedTagsSage(snap.Contexts, func(c *Context) { c.Tags = ensureSageTagMap(c.Tags) }) - fixNestedTagsSage(snap.Algorithms, func(al *Algorithm) { al.Tags = ensureSageTagMap(al.Tags) }) - fixNestedTagsSage( - snap.MlflowTrackingServers, - func(s *MlflowTrackingServer) { s.Tags = ensureSageTagMap(s.Tags) }, - ) - fixNestedTagsSage(snap.MlflowApps, func(m *MlflowApp) { m.Tags = ensureSageTagMap(m.Tags) }) - fixNestedTagsSage(snap.PartnerApps, func(p *PartnerApp) { p.Tags = ensureSageTagMap(p.Tags) }) - fixNestedTagsSage(snap.ModelPackages, func(mp *ModelPackage) { mp.Tags = ensureSageTagMap(mp.Tags) }) -} - -func fixNilTagMapsNewResources(snap *backendSnapshot) { - fixNestedTagsSage(snap.Endpoints, func(ep *Endpoint) { ep.Tags = ensureSageTagMap(ep.Tags) }) - fixNestedTagsSage(snap.TrainingJobs, func(tj *TrainingJob) { tj.Tags = ensureSageTagMap(tj.Tags) }) - fixNestedTagsSage(snap.Notebooks, func(nb *NotebookInstance) { nb.Tags = ensureSageTagMap(nb.Tags) }) - fixNestedTagsSage(snap.HPTuningJobs, func(j *HyperParameterTuningJob) { j.Tags = ensureSageTagMap(j.Tags) }) - fixNestedTagsSage(snap.Hubs, func(h *Hub) { h.Tags = ensureSageTagMap(h.Tags) }) - fixNestedTagsSage(snap.HubContents, func(hc *HubContent) { hc.Tags = ensureSageTagMap(hc.Tags) }) - fixNestedTagsSage(snap.DataQualityJobDefs, func(j *JobDefinition) { j.Tags = ensureSageTagMap(j.Tags) }) - fixNestedTagsSage(snap.ModelBiasJobDefs, func(j *JobDefinition) { j.Tags = ensureSageTagMap(j.Tags) }) - fixNestedTagsSage(snap.ModelQualityJobDefs, func(j *JobDefinition) { j.Tags = ensureSageTagMap(j.Tags) }) - fixNestedTagsSage(snap.ModelExplainJobDefs, func(j *JobDefinition) { j.Tags = ensureSageTagMap(j.Tags) }) - fixNestedTagsSage(snap.EdgeDeploymentPlans, func(p *EdgeDeploymentPlan) { p.Tags = ensureSageTagMap(p.Tags) }) - fixNestedTagsSage(snap.TrainingPlans, func(t *TrainingPlan) { t.Tags = ensureSageTagMap(t.Tags) }) - fixNestedTagsSage(snap.ModelPackageGroups, func(g *ModelPackageGroup) { g.Tags = ensureSageTagMap(g.Tags) }) -} - // Snapshot implements persistence.Persistable by delegating to the backend. func (h *Handler) Snapshot(ctx context.Context) []byte { return h.Backend.Snapshot(ctx) @@ -713,3 +388,105 @@ func (h *Handler) Snapshot(ctx context.Context) []byte { func (h *Handler) Restore(ctx context.Context, data []byte) error { return h.Backend.Restore(ctx, data) } + +// tableRegistrars maps each store.Table-backed resource's field name (the +// part of a "field:region" registry table name before the colon) to a +// function that lazily registers that field's table for a given region — +// i.e. the same xxxStore(region) helper the backend's own operations call. +// +// Restore needs this because store.Registry.RestoreAll only visits tables +// ALREADY registered in the registry; a freshly constructed backend (the +// common Restore target) has none registered until first touched, so +// without this pre-registration step every resource's snapshot data would +// be silently dropped for any region the fresh backend hasn't used yet. +// +//nolint:gochecknoglobals // read-only lookup table initialized once at package load +var tableRegistrars = map[string]func(*InMemoryBackend, string){ + "actions": func(b *InMemoryBackend, r string) { b.actionsStore(r) }, + "algorithms": func(b *InMemoryBackend, r string) { b.algorithmsStore(r) }, + "appImageConfigs": func(b *InMemoryBackend, r string) { b.appImageConfigsStore(r) }, + "apps": func(b *InMemoryBackend, r string) { b.appsStore(r) }, + "artifacts": func(b *InMemoryBackend, r string) { b.artifactsStore(r) }, + "associations": func(b *InMemoryBackend, r string) { b.associationsStore(r) }, + "autoMLJobs": func(b *InMemoryBackend, r string) { b.autoMLJobsStore(r) }, + "clusterSchedulerConfigs": func(b *InMemoryBackend, r string) { b.clusterSchedulerConfigsStore(r) }, + "clusters": func(b *InMemoryBackend, r string) { b.clustersStore(r) }, + "codeRepositories": func(b *InMemoryBackend, r string) { b.codeRepositoriesStore(r) }, + "compilationJobs": func(b *InMemoryBackend, r string) { b.compilationJobsStore(r) }, + "computeQuotas": func(b *InMemoryBackend, r string) { b.computeQuotasStore(r) }, + "contexts": func(b *InMemoryBackend, r string) { b.contextsStore(r) }, + "dataQualityJobDefs": func(b *InMemoryBackend, r string) { b.dataQualityJobDefsStore(r) }, + "deviceFleets": func(b *InMemoryBackend, r string) { b.deviceFleetsStore(r) }, + "devices": func(b *InMemoryBackend, r string) { b.devicesStore(r) }, + "domains": func(b *InMemoryBackend, r string) { b.domainsStore(r) }, + "edgeDeploymentPlans": func(b *InMemoryBackend, r string) { b.edgeDeploymentPlansStore(r) }, + "edgePackagingJobs": func(b *InMemoryBackend, r string) { b.edgePackagingJobsStore(r) }, + "endpointConfigs": func(b *InMemoryBackend, r string) { b.endpointConfigsStore(r) }, + "endpoints": func(b *InMemoryBackend, r string) { b.endpointsStore(r) }, + "experiments": func(b *InMemoryBackend, r string) { b.experimentsStore(r) }, + "featureGroups": func(b *InMemoryBackend, r string) { b.featureGroupsStore(r) }, + "featureMetadata": func(b *InMemoryBackend, r string) { b.featureMetadataStore(r) }, + "featureRecords": func(b *InMemoryBackend, r string) { b.featureRecordsStore(r) }, + "flowDefinitions": func(b *InMemoryBackend, r string) { b.flowDefinitionsStore(r) }, + "hpTuningJobs": func(b *InMemoryBackend, r string) { b.hpTuningJobsStore(r) }, + "hubContents": func(b *InMemoryBackend, r string) { b.hubContentsStore(r) }, + "hubs": func(b *InMemoryBackend, r string) { b.hubsStore(r) }, + "humanTaskUis": func(b *InMemoryBackend, r string) { b.humanTaskUisStore(r) }, + "inferenceComponents": func(b *InMemoryBackend, r string) { b.inferenceComponentsStore(r) }, + "inferenceExperiments": func(b *InMemoryBackend, r string) { b.inferenceExperimentsStore(r) }, + "inferenceRecommendationsJobs": func(b *InMemoryBackend, r string) { b.inferenceRecommendationsJobsStore(r) }, + "labelingJobs": func(b *InMemoryBackend, r string) { b.labelingJobsStore(r) }, + "mlflowApps": func(b *InMemoryBackend, r string) { b.mlflowAppsStore(r) }, + "mlflowTrackingServers": func(b *InMemoryBackend, r string) { b.mlflowTrackingServersStore(r) }, + "modelBiasJobDefs": func(b *InMemoryBackend, r string) { b.modelBiasJobDefsStore(r) }, + "modelCardExportJobs": func(b *InMemoryBackend, r string) { b.modelCardExportJobsStore(r) }, + "modelCards": func(b *InMemoryBackend, r string) { b.modelCardsStore(r) }, + "modelExplainJobDefs": func(b *InMemoryBackend, r string) { b.modelExplainJobDefsStore(r) }, + "modelPackageGroups": func(b *InMemoryBackend, r string) { b.modelPackageGroupsStore(r) }, + "modelPackages": func(b *InMemoryBackend, r string) { b.modelPackagesStore(r) }, + "modelQualityJobDefs": func(b *InMemoryBackend, r string) { b.modelQualityJobDefsStore(r) }, + "models": func(b *InMemoryBackend, r string) { b.modelsStore(r) }, + "monitoringExecutions": func(b *InMemoryBackend, r string) { b.monitoringExecutionsStore(r) }, + "monitoringSchedules": func(b *InMemoryBackend, r string) { b.monitoringSchedulesStore(r) }, + "notebookLifecycleConfigs": func(b *InMemoryBackend, r string) { b.notebookLifecycleConfigsStore(r) }, + "notebooks": func(b *InMemoryBackend, r string) { b.notebooksStore(r) }, + "optimizationJobs": func(b *InMemoryBackend, r string) { b.optimizationJobsStore(r) }, + "partnerApps": func(b *InMemoryBackend, r string) { b.partnerAppsStore(r) }, + "pipelineExecSteps": func(b *InMemoryBackend, r string) { b.pipelineExecStepsStore(r) }, + "pipelineExecutions": func(b *InMemoryBackend, r string) { b.pipelineExecutionsStore(r) }, + "pipelines": func(b *InMemoryBackend, r string) { b.pipelinesStore(r) }, + "processingJobs": func(b *InMemoryBackend, r string) { b.processingJobsStore(r) }, + "projects": func(b *InMemoryBackend, r string) { b.projectsStore(r) }, + "reservedCapacities": func(b *InMemoryBackend, r string) { b.reservedCapacitiesStore(r) }, + "smImages": func(b *InMemoryBackend, r string) { b.smImagesStore(r) }, + "spaces": func(b *InMemoryBackend, r string) { b.spacesStore(r) }, + "studioLifecycleConfigs": func(b *InMemoryBackend, r string) { b.studioLifecycleConfigsStore(r) }, + "trainingJobs": func(b *InMemoryBackend, r string) { b.trainingJobsStore(r) }, + "trainingPlanExtensionOfferings": func(b *InMemoryBackend, r string) { b.trainingPlanExtensionOfferingsStore(r) }, + "trainingPlans": func(b *InMemoryBackend, r string) { b.trainingPlansStore(r) }, + "transformJobs": func(b *InMemoryBackend, r string) { b.transformJobsStore(r) }, + "trialComponentAssociations": func(b *InMemoryBackend, r string) { b.trialComponentAssociationsStore(r) }, + "trialComponents": func(b *InMemoryBackend, r string) { b.trialComponentsStore(r) }, + "trials": func(b *InMemoryBackend, r string) { b.trialsStore(r) }, + "userProfiles": func(b *InMemoryBackend, r string) { b.userProfilesStore(r) }, + "workforces": func(b *InMemoryBackend, r string) { b.workforcesStore(r) }, + "workteams": func(b *InMemoryBackend, r string) { b.workteamsStore(r) }, + "monitoringAlerts": func(b *InMemoryBackend, r string) { b.monitoringAlertsStore(r) }, +} + +// ensureTablesRegistered pre-registers a store.Table for every "field:region" +// key present in raw — see tableRegistrars — so the subsequent +// registry.RestoreAll call actually receives every table the snapshot +// contains, not just whichever ones happen to already be registered. +func (b *InMemoryBackend) ensureTablesRegistered(raw map[string]json.RawMessage) { + for name := range raw { + field, region, cut := strings.Cut(name, ":") + if !cut { + continue + } + + if fn, ok := tableRegistrars[field]; ok { + fn(b, region) + } + } +} diff --git a/services/secretsmanager/PARITY.md b/services/secretsmanager/PARITY.md new file mode 100644 index 000000000..66dd83ffa --- /dev/null +++ b/services/secretsmanager/PARITY.md @@ -0,0 +1,99 @@ +--- +service: secretsmanager +sdk_module: aws-sdk-go-v2/service/secretsmanager@v1.42.5 +last_audit_commit: f093a929 +last_audit_date: 2026-07-05 +overall: A # ~800 LOC of genuine fixes across concurrency, wire-shape, and behavioral gaps +ops: + CreateSecret: {wire: ok, errors: ok, state: ok, persist: ok, note: "added missing ClientRequestToken idempotency contract (matches/mismatches an existing version's content on name collision)"} + GetSecretValue: {wire: ok, errors: ok, state: ok, persist: ok, note: "VersionId+VersionStage resolution correct; access-day clock now uses injectable b.now()"} + PutSecretValue: {wire: ok, errors: ok, state: ok, persist: ok, note: "AWSCURRENT/AWSPREVIOUS rotation on staging labels correct; clock consistency fixed"} + DeleteSecret: {wire: ok, errors: ok, state: ok, persist: ok, note: "force-delete vs 7-30d recovery window, mutual exclusivity with RecoveryWindowInDays, already correct"} + RestoreSecret: {wire: ok, errors: ok, state: ok, persist: ok} + ListSecrets: {wire: fixed, errors: ok, state: ok, persist: ok, note: "IncludeDeleted field name was wrong (real key IncludePlannedDeletion); SortBy was entirely unsupported; NextRotationDate was missing from SecretListEntry. All three fixed. RLock no longer lazily mutates the region map (see leaks)."} + ListSecretVersionIds: {wire: ok, errors: ok, state: ok, persist: ok, note: "RLock no longer lazily mutates the region map (see leaks)"} + DescribeSecret: {wire: ok, errors: ok, state: ok, persist: ok, note: "RLock no longer lazily mutates the region/replication maps (see leaks); OwnerAccountId is a fabricated field not in the real API — deferred, gopherstack-pct"} + UpdateSecret: {wire: ok, errors: ok, state: ok, persist: ok, note: "clock consistency fixed (was time.Now(), now b.now())"} + TagResource: {wire: ok, errors: ok, state: ok, persist: ok} + UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} + RotateSecret: {wire: ok, errors: partial, state: partial, persist: ok, note: "immediate rotation + Lambda 4-step invocation + AWSPENDING->AWSCURRENT promotion correct; RotateImmediately=false doesn't run the testSecret probe (deferred gopherstack-avt); rotation with no rotation function ever configured is accepted rather than rejected (deferred gopherstack-qqq, intentional test-convenience tradeoff)"} + GetRandomPassword: {wire: ok, errors: ok, state: ok, note: "length bounds, exclude-chars, require-each-type, crypto/rand rejection sampling all correct"} + ListAll: {wire: n/a, state: ok, note: "internal dashboard helper, not a wire op"} + BatchGetSecretValue: {wire: ok, errors: ok, state: ok, persist: ok, note: "clock consistency fixed for LastAccessedDate"} + CancelRotateSecret: {wire: ok, errors: ok, state: ok, persist: ok} + GetResourcePolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "RLock no longer lazily mutates the region map (see leaks)"} + PutResourcePolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "BlockPublicPolicy default-true + wildcard-principal detection correct"} + DeleteResourcePolicy: {wire: ok, errors: ok, state: ok, persist: ok} + ReplicateSecretToRegions: {wire: ok, errors: ok, state: ok, persist: ok} + RemoveRegionsFromReplication: {wire: ok, errors: ok, state: ok, persist: ok} + StopReplicationToReplica: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateSecretVersionStage: {wire: ok, errors: fixed, state: fixed, persist: ok, note: "was silently stripping a staging label from wherever it happened to be attached, regardless of RemoveFromVersionId; real API requires RemoveFromVersionId to name the current holder or the call fails. Fixed + regression tests added; one existing test (TestRefinement1_UpdateSecretVersionStageAutoStrip) encoded the old wrong behavior and was corrected."} + ValidateResourcePolicy: {wire: ok, errors: ok, state: ok, note: "RLock no longer lazily mutates the region map (see leaks)"} +families: + version-staging: {status: ok, note: "AWSCURRENT/AWSPENDING/AWSPREVIOUS transitions, auto-demotion of AWSCURRENT->AWSPREVIOUS on PutSecretValue/UpdateSecret/rotation, max 100 versions with unlabeled-oldest-first pruning — all verified against real semantics"} + rotation: {status: partial, note: "Lambda 4-step invocation (createSecret/setSecret/testSecret/finishSecret), rate()/cron() schedule parsing and due-date computation, scheduler goroutine with ctx-bounded lifecycle — all correct. Gaps: RotateImmediately=false test-probe (gopherstack-avt), missing-rotation-function validation (gopherstack-qqq)"} + replication: {status: ok, note: "ReplicateSecretToRegions/RemoveRegionsFromReplication/StopReplicationToReplica + status sync on version change all verified"} + resource-policy: {status: ok, note: "Get/Put/Delete/Validate + BlockPublicPolicy + MalformedPolicyDocumentException/PublicPolicyException verified"} + error-codes: {status: partial, note: "ResourceNotFoundException/ResourceExistsException/InvalidRequestException/InvalidParameterException/MalformedPolicyDocumentException/PublicPolicyException all verified against types/errors.go. LimitExceededException is never used (tag/SecretIdList limits use InvalidParameterException instead) — deferred gopherstack-gvw."} + persistence: {status: ok, note: "Snapshot/Restore round-trips all fields including json:\"-\" internal fields via secretSnapshot; Tags.Close() called on replace to avoid Prometheus registry leaks; rotation scheduler re-armed on restore when RotationEnabled"} + concurrency-locking: {status: fixed, note: "see leaks — RLock-guarded reads were lazily mutating the coarse per-region maps; fixed with non-mutating *StoreRO accessors"} +gaps: + - RotateSecret accepts rotation with no RotationLambdaARN ever configured on the secret or in the request (real AWS requires an existing rotation strategy or managed rotation) — kept as-is because dozens of existing tests rely on the lenient no-Lambda immediate-value-regen behavior as a test convenience, and gopherstack does not model AWS managed rotation at all (bd: gopherstack-qqq) + - RotateSecret with RotateImmediately=false does not invoke the Lambda testSecret probe step or create/remove a transient AWSPENDING version (bd: gopherstack-avt) + - Tag-count (50) and BatchGetSecretValue SecretIdList (20) limit violations return InvalidParameterException instead of the real LimitExceededException (bd: gopherstack-gvw) + - DescribeSecretOutput/SecretListEntry expose a fabricated OwnerAccountId field not present in the real API (harmless — unknown JSON fields are ignored by real deserializers); managed-external-secret fields (ExternalSecretRotationMetadata/RoleArn, OwningService, Type) are entirely unmodeled, so the "owning-service" ListSecrets/BatchGetSecretValue filter always passes rather than matching a tracked owning service (bd: gopherstack-pct) +deferred: + - Managed rotation (AWS-service-owned secrets, e.g. RDS-managed rotation) — out of scope, not modeled at all + - Cross-account resource-policy principal evaluation beyond the wildcard-principal BlockPublicPolicy heuristic +leaks: {status: fixed, note: "Found a real data race: ListSecrets/ListSecretVersionIds/DescribeSecret/GetResourcePolicy/ValidateResourcePolicy held only an RLock (shared reader lock) but called the lazily-creating *Store(region) helper, which does `b.secrets[region] = make(...)` on first touch of a region — a concurrent map write happening under a read lock. Confirmed with a regression test (concurrency_race_test.go) that reproduces the `go test -race` data race pre-fix and passes clean post-fix. Fixed by adding non-mutating *StoreRO(region) accessors for read-only call sites (a nil-map read/range is well-defined in Go, so no mutation is needed there). Rotation scheduler goroutine, janitor, and StopRotationScheduler/Shutdown ctx-cancellation lifecycle were already clean (verified, no changes needed)."} +--- + +## Notes + +- **Protocol**: `application/x-amz-json-1.1` (awsJson1_1), matches `secretsmanager.` + `X-Amz-Target` routing already in place in `handler.go`. +- **Wire field names verified directly against `aws-sdk-go-v2/service/secretsmanager@v1.42.5` + serializers.go/deserializers.go** (not just the Go struct tags), which is how the + `IncludeDeleted` → `IncludePlannedDeletion` and `owned-by-me` → `owning-service` bugs were + caught: both were plausible-looking names that don't exist on the wire. A previous audit + pass invented "owned-by-me" for account-ownership semantics, but the real + `FilterNameStringType` enum has no such value — the real "owning-service" key is about + AWS-service-managed secrets (e.g. RDS-managed rotation), a different concept entirely. + Renamed to the real key while preserving pass-all semantics, since this mock never + tracks AWS-service ownership of secrets. +- **Timestamps** are epoch-seconds JSON numbers (`float64` via `UnixTimeFloat`), matching + `smithytime.ParseEpochSeconds` in the real deserializers — already correct throughout. +- **Clock consistency**: `InMemoryBackend.now` is an injectable clock (`SetNowForTest` in + `export_test.go`) used correctly by `DeleteSecret`/rotation, but `CreateSecret`, + `seedInitialVersion`, `PutSecretValue`, `UpdateSecret`, `GetSecretValue`, and + `BatchGetSecretValue`'s access-day tracking all called `time.Now()` directly, bypassing + the injected clock. Fixed for internal consistency and testability — production behavior + is unchanged since the default `now` is `time.Now`. +- **RemoveFromVersionId semantics** (`UpdateSecretVersionStage`): the real API requires the + caller to name the version that currently holds a staging label before it can be moved + elsewhere — "if the label is attached and you either do not specify [RemoveFromVersionId], + or the version ID does not match, then the operation fails." This mock silently stripped + the label from wherever it was, which is a **looks-wrong-but-was-actually-a-bug** case: an + existing test (`TestRefinement1_UpdateSecretVersionStageAutoStrip`) explicitly asserted the + permissive (wrong) behavior as correct. Fixed the backend and corrected that test rather + than working around it. +- **CreateSecret idempotency**: the real API's `ClientRequestToken` doc text is explicit + about the three-way branch (new version / matching retry ignored / mismatched content + fails) — this was previously entirely unimplemented for the CreateSecret-level name + collision case (it existed for `PutSecretValue`/`UpdateSecret`, just not `CreateSecret`). +- **Region-nested maps**: `InMemoryBackend.secrets`/`resourcePolicies`/`replicationConfigs` + are `map[string]map[string]T]` (outer key = region), lazily created per-region by + `*Store(region)` helpers. Those helpers **must only be called under `b.mu.Lock()`** (write + lock) — read paths must use the new `*StoreRO(region)` accessors instead. This is the kind + of thing that's easy to reintroduce; grep for `RLock(` and confirm no `*Store(` (non-RO) + calls appear before the matching `RUnlock`/`defer`. +- **`GetSupportedOperations`/dispatch-table op-name strings**: several operation names are + used in three or more places (the supported-ops list, the dispatch map key, and a + `lockmetrics` label in `backend.go`) — collapsed the four that tripped `goconst` + (`DescribeSecret`, `GetResourcePolicy`, `ListSecrets`, `ValidateResourcePolicy`) into shared + `opXxx` constants. Not exhaustive for every op name (only fixes what already tripped the + linter); a future full pass could do this for all ~20 ops for consistency. +- **Test files in this package are numerous** (from several prior sweeps — + `batch1_audit_test.go`, `accuracy_audit_test.go`, `parity_a_test.go`, + `parity_deepen_test.go`, `handler_refinement1/2_test.go`, etc.). Before adding a new test, + grep first — there is a good chance similar coverage already exists under a different name. diff --git a/services/secretsmanager/accuracy_audit_test.go b/services/secretsmanager/accuracy_audit_test.go index 86b5af59e..227c31cc0 100644 --- a/services/secretsmanager/accuracy_audit_test.go +++ b/services/secretsmanager/accuracy_audit_test.go @@ -207,12 +207,19 @@ func TestCreateSecret_AddReplicaRegionsHTTP(t *testing.T) { } // --------------------------------------------------------------------------- -// Issue #6 — ListSecrets owned-by-me filter +// Issue #6 — ListSecrets owning-service filter +// +// NOTE: this filter key was previously named "owned-by-me" in this test suite, which +// is not a real AWS FilterNameStringType value (the real key is "owning-service" — +// see aws-sdk-go-v2/service/secretsmanager/types.FilterNameStringType). Renamed to +// match the real wire key; behaviour (always-pass) is preserved because this mock +// never models AWS-service-owned secrets (e.g. RDS-managed rotation secrets), so +// every secret is equally "not owned by a service" and the filter is a no-op here. // --------------------------------------------------------------------------- -// TestListSecrets_OwnedByMeFilterPassesAll verifies that "owned-by-me" always returns all -// secrets in the single-account mock. -func TestListSecrets_OwnedByMeFilterPassesAll(t *testing.T) { +// TestListSecrets_OwningServiceFilterPassesAll verifies that "owning-service" always +// returns all secrets in this mock (no secret is ever AWS-service-owned). +func TestListSecrets_OwningServiceFilterPassesAll(t *testing.T) { t.Parallel() b := sm.NewInMemoryBackend() @@ -223,14 +230,15 @@ func TestListSecrets_OwnedByMeFilterPassesAll(t *testing.T) { } out, err := b.ListSecrets(context.Background(), &sm.ListSecretsInput{ - Filters: []sm.SecretFilter{{Key: "owned-by-me", Values: []string{"true"}}}, + Filters: []sm.SecretFilter{{Key: "owning-service", Values: []string{"rds"}}}, }) require.NoError(t, err) - assert.Len(t, out.SecretList, 3, "owned-by-me must pass all secrets in single-account mock") + assert.Len(t, out.SecretList, 3, "owning-service must pass all secrets in single-account mock") } -// TestListSecrets_OwnedByMeWithOtherFilters verifies owned-by-me can be combined with other filters. -func TestListSecrets_OwnedByMeWithOtherFilters(t *testing.T) { +// TestListSecrets_OwningServiceWithOtherFilters verifies owning-service can be combined +// with other filters (AND semantics across distinct filter keys). +func TestListSecrets_OwningServiceWithOtherFilters(t *testing.T) { t.Parallel() b := sm.NewInMemoryBackend() @@ -247,7 +255,7 @@ func TestListSecrets_OwnedByMeWithOtherFilters(t *testing.T) { out, err := b.ListSecrets(context.Background(), &sm.ListSecretsInput{ Filters: []sm.SecretFilter{ - {Key: "owned-by-me", Values: []string{"true"}}, + {Key: "owning-service", Values: []string{"rds"}}, {Key: "name", Values: []string{"alpha"}}, }, }) @@ -810,8 +818,8 @@ func TestRotationDue_IntervalExpression(t *testing.T) { assert.True(t, sm.RotationDue(rules, after, &base)) } -// TestListSecretsOwnedByMeHTTP verifies the owned-by-me filter via HTTP. -func TestListSecretsOwnedByMeHTTP(t *testing.T) { +// TestListSecretsOwningServiceHTTP verifies the owning-service filter via HTTP. +func TestListSecretsOwningServiceHTTP(t *testing.T) { t.Parallel() b := sm.NewInMemoryBackend() @@ -823,7 +831,7 @@ func TestListSecretsOwnedByMeHTTP(t *testing.T) { require.Equal(t, http.StatusOK, rec.Code) } - filterBody := `{"Filters":[{"Key":"owned-by-me","Values":["true"]}]}` + filterBody := `{"Filters":[{"Key":"owning-service","Values":["rds"]}]}` rec := doR1Request(t, h, "secretsmanager.ListSecrets", filterBody) require.Equal(t, http.StatusOK, rec.Code) diff --git a/services/secretsmanager/backend.go b/services/secretsmanager/backend.go index 32fd6e007..662aa1a94 100644 --- a/services/secretsmanager/backend.go +++ b/services/secretsmanager/backend.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "math" "math/big" "regexp" "slices" @@ -22,11 +23,20 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" "github.com/blackbirdworks/gopherstack/pkgs/arn" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) const ( errResourceNotFoundException = "ResourceNotFoundException" + + // Operation name constants: shared between the handler's dispatch/support tables + // and the backend's lockmetrics labels so the literal appears once per operation + // (avoids goconst duplication across handler.go and backend.go). + opDescribeSecret = "DescribeSecret" + opGetResourcePolicy = "GetResourcePolicy" + opListSecrets = "ListSecrets" + opValidateResourcePolicy = "ValidateResourcePolicy" ) // regionContextKey is the context key under which the per-request AWS region is stored. @@ -119,11 +129,19 @@ const ( ) // InMemoryBackend is a concurrency-safe in-memory Secrets Manager backend. -// InMemoryBackend stores Secrets Manager state. All resource maps are nested by -// region (outer key = region) so that secrets are isolated per region. +// InMemoryBackend stores Secrets Manager state. secrets is isolated per region +// via a region-qualified composite key ("region|name") on a single flat +// store.Table, with a secondary index (secretsByRegion) grouping entries by +// region for per-region scans (ListSecrets, BatchGetSecretValue's filter +// path); see store_setup.go. resourcePolicies and replicationConfigs remain +// nested map[string]map[string]... (outer key = region) since their values +// (a bare string, a bare slice) carry no identity of their own to key a +// store.Table by -- see store_setup.go's doc comment for the full rationale. type InMemoryBackend struct { lambdaInvoker LambdaInvoker - secrets map[string]map[string]*Secret + registry *store.Registry + secrets *store.Table[Secret] + secretsByRegion *store.Index[Secret] resourcePolicies map[string]map[string]string replicationConfigs map[string]map[string][]ReplicationStatusType mu *lockmetrics.RWMutex @@ -160,8 +178,8 @@ func NewInMemoryBackendWithContext(svcCtx context.Context, accountID, region str svcCtx = context.Background() } - return &InMemoryBackend{ - secrets: make(map[string]map[string]*Secret), + b := &InMemoryBackend{ + registry: store.NewRegistry(), resourcePolicies: make(map[string]map[string]string), replicationConfigs: make(map[string]map[string][]ReplicationStatusType), accountID: accountID, @@ -171,19 +189,53 @@ func NewInMemoryBackendWithContext(svcCtx context.Context, accountID, region str schedulerStop: make(chan struct{}), svcCtx: svcCtx, } + + registerAllTables(b) + + return b } -// The *Store helpers return the per-region inner map, lazily creating it. -// Callers must hold b.mu. +// secretGet returns the secret stored under region+name, performing no +// mutation -- safe to call while holding either b.mu.RLock or b.mu.Lock. +// Replaces the pre-Phase-3.3 secretsStoreRO(region)[name] (and, for +// write-path callers, secretsStore(region)[name]) map lookup: store.Table.Get +// never lazily creates anything, so the "safe to call under RLock" guarantee +// the old *StoreRO helpers were carefully split out for is now structural +// rather than a documented caveat callers had to remember to honour. +func (b *InMemoryBackend) secretGet(region, name string) (*Secret, bool) { + return b.secrets.Get(secretTableKey(region, name)) +} -func (b *InMemoryBackend) secretsStore(region string) map[string]*Secret { - if b.secrets[region] == nil { - b.secrets[region] = make(map[string]*Secret) - } +// secretHas reports whether region+name exists, without mutation. +func (b *InMemoryBackend) secretHas(region, name string) bool { + return b.secrets.Has(secretTableKey(region, name)) +} + +// secretPut inserts or replaces s, which must already have its unexported +// region field set (see CreateSecret's literal and AddSecretInternal) so the +// key it is stored under matches secretKeyFn. +func (b *InMemoryBackend) secretPut(s *Secret) { + b.secrets.Put(s) +} - return b.secrets[region] +// secretDelete removes the secret stored under region+name. Every call site +// already knows the secret exists (it was just looked up via secretGet), so +// [store.Table.Delete]'s existed-bool is intentionally discarded here. +func (b *InMemoryBackend) secretDelete(region, name string) { + b.secrets.Delete(secretTableKey(region, name)) } +// secretsInRegion returns every secret registered under region, in +// unspecified order (mirrors the old secrets[region] map's iteration). The +// returned slice is owned by the underlying index -- see [store.Index.Get] -- +// so callers that need to mutate the table while iterating must copy it first. +func (b *InMemoryBackend) secretsInRegion(region string) []*Secret { + return b.secretsByRegion.Get(region) +} + +// The *Store helpers return the per-region inner map, lazily creating it. +// Callers must hold b.mu. + func (b *InMemoryBackend) resourcePoliciesStore(region string) map[string]string { if b.resourcePolicies[region] == nil { b.resourcePolicies[region] = make(map[string]string) @@ -200,6 +252,23 @@ func (b *InMemoryBackend) replicationConfigsStore(region string) map[string][]Re return b.replicationConfigs[region] } +// The *StoreRO helpers return the per-region inner map WITHOUT lazily creating it, +// so they are safe to call while only holding a read lock (b.mu.RLock). A plain Go +// map read/range/lookup on a nil map is well-defined (returns the zero value / no +// iterations), so callers get correct "no entries for this region" behaviour without +// mutating the outer map. Never use these under RLock if the result will be written +// through by the caller — that still requires the write-locked *Store variant above. +// (secrets itself no longer needs a StoreRO variant: store.Table.Get never lazily +// creates anything, so secretGet above is unconditionally RLock-safe.) + +func (b *InMemoryBackend) resourcePoliciesStoreRO(region string) map[string]string { + return b.resourcePolicies[region] +} + +func (b *InMemoryBackend) replicationConfigsStoreRO(region string) map[string][]ReplicationStatusType { + return b.replicationConfigs[region] +} + // resolveSecretID resolves a name or ARN to the internal key (name). func resolveSecretID(secretID string) string { if strings.HasPrefix(secretID, "arn:aws:secretsmanager:") { @@ -325,16 +394,8 @@ func (b *InMemoryBackend) CreateSecret(ctx context.Context, input *CreateSecretI b.mu.Lock("CreateSecret") defer b.mu.Unlock() - secrets := b.secretsStore(region) - if existing, exists := secrets[input.Name]; exists { - if existing.DeletedDate != nil { - return nil, fmt.Errorf( - "%w: a secret with this name is already scheduled for deletion; restore or force-delete it first", - ErrSecretDeleted, - ) - } - - return nil, ErrSecretAlreadyExists + if existing, exists := b.secretGet(region, input.Name); exists { + return b.createSecretNameCollision(region, existing, input) } suffix, err := generateRandomSuffix() @@ -345,6 +406,7 @@ func (b *InMemoryBackend) CreateSecret(ctx context.Context, input *CreateSecretI arn := b.buildARNWithRegion(region, input.Name, suffix) secret := &Secret{ + region: region, ARN: arn, Name: input.Name, Description: input.Description, @@ -352,7 +414,7 @@ func (b *InMemoryBackend) CreateSecret(ctx context.Context, input *CreateSecretI Versions: make(map[string]*SecretVersion), } - createdNow := UnixTimeFloat(time.Now()) + createdNow := UnixTimeFloat(b.now()) secret.CreatedDate = &createdNow if len(input.Tags) > 0 { @@ -363,9 +425,9 @@ func (b *InMemoryBackend) CreateSecret(ctx context.Context, input *CreateSecretI } } - versionID := seedInitialVersion(secret, input) + versionID := seedInitialVersion(secret, input, b.now()) - secrets[input.Name] = secret + b.secretPut(secret) if len(input.AddReplicaRegions) > 0 { replicas := make([]ReplicationStatusType, 0, len(input.AddReplicaRegions)) @@ -390,9 +452,52 @@ func (b *InMemoryBackend) CreateSecret(ctx context.Context, input *CreateSecretI }, nil } +// createSecretNameCollision handles CreateSecret when a secret with input.Name already +// exists: it applies the ClientRequestToken idempotency contract (see +// CreateSecretInput.ClientRequestToken in the real API) — a retried CreateSecret with a +// ClientRequestToken matching an already-created version is ignored (success, no new +// version) when the content matches, and fails when the content differs, since +// CreateSecret cannot modify an existing version. Must be called with b.mu held. +func (b *InMemoryBackend) createSecretNameCollision( + region string, existing *Secret, input *CreateSecretInput, +) (*CreateSecretOutput, error) { + if existing.DeletedDate != nil { + return nil, fmt.Errorf( + "%w: a secret with this name is already scheduled for deletion; restore or force-delete it first", + ErrSecretDeleted, + ) + } + + if input.ClientRequestToken == "" { + return nil, ErrSecretAlreadyExists + } + + v, ok := existing.Versions[input.ClientRequestToken] + if !ok { + return nil, ErrSecretAlreadyExists + } + + if v.SecretString == input.SecretString && string(v.SecretBinary) == string(input.SecretBinary) { + return &CreateSecretOutput{ + ARN: existing.ARN, + Name: existing.Name, + VersionID: v.VersionID, + ReplicationStatus: b.replicationConfigsStore(region)[input.Name], + }, nil + } + + return nil, fmt.Errorf( + "%w: a version with ClientRequestToken %s already exists with different content;"+ + " use PutSecretValue to create a new version", + ErrInvalidParameter, input.ClientRequestToken, + ) +} + // seedInitialVersion creates the initial AWSCURRENT version on a freshly created secret // when the create request carries a value, and returns the version ID (empty if none). -func seedInitialVersion(secret *Secret, input *CreateSecretInput) string { +// nowTime is the backend's (possibly test-injected) clock value, so CreatedDate stays +// consistent with the rest of the secret's timestamps. +func seedInitialVersion(secret *Secret, input *CreateSecretInput, nowTime time.Time) string { if input.SecretString == "" && len(input.SecretBinary) == 0 { return "" } @@ -403,7 +508,7 @@ func seedInitialVersion(secret *Secret, input *CreateSecretInput) string { versionID = uuid.New().String() } - now := UnixTimeFloat(time.Now()) + now := UnixTimeFloat(nowTime) secret.Versions[versionID] = &SecretVersion{ VersionID: versionID, SecretString: input.SecretString, @@ -432,7 +537,7 @@ func (b *InMemoryBackend) GetSecretValue( name := resolveSecretID(input.SecretID) - secret, exists := b.secretsStore(region)[name] + secret, exists := b.secretGet(region, name) if !exists { return nil, ErrSecretNotFound } @@ -460,7 +565,7 @@ func (b *InMemoryBackend) GetSecretValue( } // Track access date (truncated to day granularity as AWS does). - accessDay := UnixTimeFloat(time.Now().UTC().Truncate(hoursPerDay * time.Hour)) + accessDay := UnixTimeFloat(b.now().UTC().Truncate(hoursPerDay * time.Hour)) secret.LastAccessedDate = &accessDay version.LastAccessedDate = &accessDay @@ -530,7 +635,7 @@ func (b *InMemoryBackend) PutSecretValue( name := resolveSecretID(input.SecretID) - secret, exists := b.secretsStore(region)[name] + secret, exists := b.secretGet(region, name) if !exists { return nil, ErrSecretNotFound } @@ -558,7 +663,7 @@ func (b *InMemoryBackend) PutSecretValue( callerWantsCurrentLabel, stagingLabels := b.resolveStagingLabels(secret, input.VersionStages) - now := UnixTimeFloat(time.Now()) + now := UnixTimeFloat(b.now()) version := &SecretVersion{ VersionID: versionID, SecretString: input.SecretString, @@ -701,8 +806,7 @@ func (b *InMemoryBackend) DeleteSecret(ctx context.Context, input *DeleteSecretI name := resolveSecretID(input.SecretID) - secrets := b.secretsStore(region) - secret, exists := secrets[name] + secret, exists := b.secretGet(region, name) if !exists { return nil, ErrSecretNotFound } @@ -724,7 +828,7 @@ func (b *InMemoryBackend) DeleteSecret(ctx context.Context, input *DeleteSecretI secret.Tags.Close() } - delete(secrets, name) + b.secretDelete(region, name) delete(b.resourcePoliciesStore(region), name) delete(b.replicationConfigsStore(region), name) @@ -776,14 +880,14 @@ func (b *InMemoryBackend) ListSecrets(ctx context.Context, input *ListSecretsInp region := getRegion(ctx, b.region) - b.mu.RLock("ListSecrets") + b.mu.RLock(opListSecrets) defer b.mu.RUnlock() - secrets := b.secretsStore(region) - entries := make([]SecretListEntry, 0, len(secrets)) + secretsInRegion := b.secretsInRegion(region) + entries := make([]SecretListEntry, 0, len(secretsInRegion)) - for _, s := range secrets { - if s.DeletedDate != nil && !input.IncludeDeleted { + for _, s := range secretsInRegion { + if s.DeletedDate != nil && !input.IncludePlannedDeletion { continue } @@ -794,13 +898,7 @@ func (b *InMemoryBackend) ListSecrets(ctx context.Context, input *ListSecretsInp entries = append(entries, secretToListEntry(s)) } - sort.Slice(entries, func(i, j int) bool { - if strings.EqualFold(input.SortOrder, "desc") { - return entries[i].Name > entries[j].Name - } - - return entries[i].Name < entries[j].Name - }) + sortSecretListEntries(entries, input.SortBy, input.SortOrder) startIdx := parseToken(input.NextToken) maxResults := int64(defaultMaxResults) @@ -829,6 +927,67 @@ func (b *InMemoryBackend) ListSecrets(ctx context.Context, input *ListSecretsInp }, nil } +// sortSecretListEntries orders entries by the requested SortBy key ("name" (default), +// "created-date", "last-changed-date", "last-accessed-date"), honouring SortOrder +// ("asc" default, or "desc"). Unset date fields sort as the earliest possible value. +// Matches the AWS SortByType enum (ListSecrets request field "SortBy"). +func sortSecretListEntries(entries []SecretListEntry, sortBy, sortOrder string) { + desc := strings.EqualFold(sortOrder, "desc") + + var less func(i, j int) bool + + switch strings.ToLower(strings.TrimSpace(sortBy)) { + case "created-date": + less = func(i, j int) bool { + return float64PtrLess(entries[i].CreatedDate, entries[j].CreatedDate, entries[i].Name, entries[j].Name) + } + case "last-changed-date": + less = func(i, j int) bool { + return float64PtrLess( + entries[i].LastChangedDate, entries[j].LastChangedDate, entries[i].Name, entries[j].Name, + ) + } + case "last-accessed-date": + less = func(i, j int) bool { + return float64PtrLess( + entries[i].LastAccessedDate, entries[j].LastAccessedDate, entries[i].Name, entries[j].Name, + ) + } + default: + less = func(i, j int) bool { return entries[i].Name < entries[j].Name } + } + + sort.Slice(entries, func(i, j int) bool { + if desc { + return less(j, i) + } + + return less(i, j) + }) +} + +// float64PtrLess compares two optional float64 fields, treating a nil pointer as the +// earliest possible value (AWS omits date fields that have never been set, e.g. a +// secret that was never rotated has no LastRotatedDate). Ties are broken by name for +// deterministic, stable ordering. +func float64PtrLess(a, b *float64, nameA, nameB string) bool { + av, bv := ptrFloatOrMin(a), ptrFloatOrMin(b) + if av != bv { + return av < bv + } + + return nameA < nameB +} + +// ptrFloatOrMin dereferences a *float64, returning -math.MaxFloat64 for nil. +func ptrFloatOrMin(f *float64) float64 { + if f == nil { + return -math.MaxFloat64 + } + + return *f +} + // secretMatchesFilters returns true if the secret matches all provided filters. func secretMatchesFilters(s *Secret, filters []SecretFilter) bool { for _, f := range filters { @@ -861,9 +1020,13 @@ func secretMatchesFilter(s *Secret, f SecretFilter) bool { // In a single-region mock every secret belongs to the single region; // the filter always passes (no cross-region replication routing needed). return true - case "owned-by-me": - // In a single-account mock all secrets are owned by the configured account; - // the filter always passes. + case "owning-service": + // Real AWS FilterNameStringType key (this mock previously special-cased a + // fabricated "owned-by-me" key that no real SDK client ever sends). Every + // secret in this mock is user-created, never owned by an AWS-managed + // integration (e.g. RDS-managed rotation secrets), so this behaves like the + // permissive default below and always passes — consistent with the + // single-account/region simplifications used for primary-region above. return true default: return true @@ -928,7 +1091,7 @@ func (b *InMemoryBackend) ListSecretVersionIDs( name := resolveSecretID(input.SecretID) - secret, exists := b.secretsStore(region)[name] + secret, exists := b.secretGet(region, name) if !exists { return nil, ErrSecretNotFound } @@ -997,12 +1160,12 @@ func (b *InMemoryBackend) DescribeSecret( ) (*DescribeSecretOutput, error) { region := getRegion(ctx, b.region) - b.mu.RLock("DescribeSecret") + b.mu.RLock(opDescribeSecret) defer b.mu.RUnlock() name := resolveSecretID(input.SecretID) - secret, exists := b.secretsStore(region)[name] + secret, exists := b.secretGet(region, name) if !exists { return nil, ErrSecretNotFound } @@ -1030,7 +1193,7 @@ func (b *InMemoryBackend) DescribeSecret( LastAccessedDate: secret.LastAccessedDate, VersionIDsToStages: versionIDsToStages, RotationEnabled: secret.RotationEnabled, - ReplicationStatus: b.replicationConfigsStore(region)[name], + ReplicationStatus: b.replicationConfigsStoreRO(region)[name], OwnerAccountID: b.accountID, PrimaryRegion: region, } @@ -1093,7 +1256,7 @@ func (b *InMemoryBackend) UpdateSecret(ctx context.Context, input *UpdateSecretI name := resolveSecretID(input.SecretID) - secret, exists := b.secretsStore(region)[name] + secret, exists := b.secretGet(region, name) if !exists { return nil, ErrSecretNotFound } @@ -1132,7 +1295,7 @@ func (b *InMemoryBackend) UpdateSecret(ctx context.Context, input *UpdateSecretI b.rotateStagingLabels(secret) - now := UnixTimeFloat(time.Now()) + now := UnixTimeFloat(b.now()) version := &SecretVersion{ VersionID: versionID, SecretString: input.SecretString, @@ -1165,7 +1328,7 @@ func (b *InMemoryBackend) RestoreSecret(ctx context.Context, input *RestoreSecre name := resolveSecretID(input.SecretID) - secret, exists := b.secretsStore(region)[name] + secret, exists := b.secretGet(region, name) if !exists { return nil, ErrSecretNotFound } @@ -1194,12 +1357,11 @@ func (b *InMemoryBackend) ListAll() []SecretListEntry { b.mu.RLock("ListAll") defer b.mu.RUnlock() - var entries []SecretListEntry + all := b.secrets.All() + entries := make([]SecretListEntry, 0, len(all)) - for _, regionSecrets := range b.secrets { - for _, s := range regionSecrets { - entries = append(entries, secretToListEntry(s)) - } + for _, s := range all { + entries = append(entries, secretToListEntry(s)) } sort.Slice(entries, func(i, j int) bool { @@ -1237,6 +1399,7 @@ func secretToListEntry(s *Secret) SecretListEntry { LastAccessedDate: s.LastAccessedDate, LastRotatedDate: s.LastRotatedDate, CreatedDate: s.CreatedDate, + NextRotationDate: computeNextRotationDate(s), Tags: s.Tags, SecretVersionsToStages: versionStages, } @@ -1269,7 +1432,7 @@ func (b *InMemoryBackend) TagResource(ctx context.Context, input *TagResourceInp defer b.mu.Unlock() id := resolveSecretID(input.SecretID) - secret, ok := b.secretsStore(region)[id] + secret, ok := b.secretGet(region, id) if !ok { return ErrSecretNotFound } @@ -1315,7 +1478,7 @@ func (b *InMemoryBackend) UntagResource(ctx context.Context, input *UntagResourc defer b.mu.Unlock() id := resolveSecretID(input.SecretID) - secret, ok := b.secretsStore(region)[id] + secret, ok := b.secretGet(region, id) if !ok { return ErrSecretNotFound } @@ -1337,7 +1500,7 @@ func (b *InMemoryBackend) RotateSecret(ctx context.Context, input *RotateSecretI defer b.mu.Unlock() id := resolveSecretID(input.SecretID) - secret, ok := b.secretsStore(region)[id] + secret, ok := b.secretGet(region, id) if !ok { return nil, ErrSecretNotFound } @@ -1460,7 +1623,7 @@ func (b *InMemoryBackend) FinishRotation(ctx context.Context, secretID, versionI defer b.mu.Unlock() id := resolveSecretID(secretID) - secret, ok := b.secretsStore(region)[id] + secret, ok := b.secretGet(region, id) if !ok || secret.DeletedDate != nil { return ErrSecretNotFound @@ -1480,7 +1643,7 @@ func (b *InMemoryBackend) AbortRotation(ctx context.Context, secretID, versionID defer b.mu.Unlock() id := resolveSecretID(secretID) - secret, ok := b.secretsStore(region)[id] + secret, ok := b.secretGet(region, id) if !ok || secret.DeletedDate != nil { return ErrSecretNotFound @@ -1813,19 +1976,17 @@ func (b *InMemoryBackend) TaggedSecrets(_ context.Context) []TaggedSecretInfo { var result []TaggedSecretInfo - for _, regionSecrets := range b.secrets { - for _, secret := range regionSecrets { - if secret.DeletedDate != nil { - continue - } - - var tagMap map[string]string - if secret.Tags != nil { - tagMap = secret.Tags.Clone() - } + for _, secret := range b.secrets.All() { + if secret.DeletedDate != nil { + continue + } - result = append(result, TaggedSecretInfo{ARN: secret.ARN, Tags: tagMap}) + var tagMap map[string]string + if secret.Tags != nil { + tagMap = secret.Tags.Clone() } + + result = append(result, TaggedSecretInfo{ARN: secret.ARN, Tags: tagMap}) } return result @@ -1853,7 +2014,7 @@ func (b *InMemoryBackend) TagSecretByARN(_ context.Context, secretARN string, ne name := resolveSecretID(secretARN) - secret, ok := b.secretsStore(region)[name] + secret, ok := b.secretGet(region, name) if !ok { return fmt.Errorf("%w: %s", ErrSecretNotFound, secretARN) } @@ -1876,7 +2037,7 @@ func (b *InMemoryBackend) UntagSecretByARN(_ context.Context, secretARN string, name := resolveSecretID(secretARN) - secret, ok := b.secretsStore(region)[name] + secret, ok := b.secretGet(region, name) if !ok { return fmt.Errorf("%w: %s", ErrSecretNotFound, secretARN) } @@ -1936,13 +2097,12 @@ func (b *InMemoryBackend) BatchGetSecretValue( // batchGetByIDList populates out with values and errors for each explicit secret ID. // Must be called with write lock held. func (b *InMemoryBackend) batchGetByIDList(region string, ids []string, out *BatchGetSecretValueOutput) { - accessDay := UnixTimeFloat(time.Now().UTC().Truncate(hoursPerDay * time.Hour)) - secrets := b.secretsStore(region) + accessDay := UnixTimeFloat(b.now().UTC().Truncate(hoursPerDay * time.Hour)) for _, id := range ids { name := resolveSecretID(id) - secret, ok := secrets[name] + secret, ok := b.secretGet(region, name) if !ok { out.Errors = append(out.Errors, APIErrorType{ ErrorCode: errResourceNotFoundException, @@ -1987,11 +2147,11 @@ func (b *InMemoryBackend) batchGetByFilter( input *BatchGetSecretValueInput, out *BatchGetSecretValueOutput, ) *BatchGetSecretValueOutput { - secrets := b.secretsStore(region) - allValues := make([]SecretValueEntry, 0, len(secrets)) - accessDay := UnixTimeFloat(time.Now().UTC().Truncate(hoursPerDay * time.Hour)) + secretsInRegion := b.secretsInRegion(region) + allValues := make([]SecretValueEntry, 0, len(secretsInRegion)) + accessDay := UnixTimeFloat(b.now().UTC().Truncate(hoursPerDay * time.Hour)) - for _, secret := range secrets { + for _, secret := range secretsInRegion { if secret.DeletedDate != nil || !batchMatchesFilters(secret, input.Filters) { continue } @@ -2085,7 +2245,7 @@ func (b *InMemoryBackend) CancelRotateSecret( name := resolveSecretID(input.SecretID) - secret, ok := b.secretsStore(region)[name] + secret, ok := b.secretGet(region, name) if !ok { return nil, ErrSecretNotFound } @@ -2127,12 +2287,12 @@ func (b *InMemoryBackend) GetResourcePolicy( ) (*GetResourcePolicyOutput, error) { region := getRegion(ctx, b.region) - b.mu.RLock("GetResourcePolicy") + b.mu.RLock(opGetResourcePolicy) defer b.mu.RUnlock() name := resolveSecretID(input.SecretID) - secret, ok := b.secretsStore(region)[name] + secret, ok := b.secretGet(region, name) if !ok { return nil, ErrSecretNotFound } @@ -2141,7 +2301,7 @@ func (b *InMemoryBackend) GetResourcePolicy( return nil, fmt.Errorf("%w: secret %s is deleted", ErrSecretDeleted, input.SecretID) } - policy := b.resourcePoliciesStore(region)[name] + policy := b.resourcePoliciesStoreRO(region)[name] return &GetResourcePolicyOutput{ ARN: secret.ARN, @@ -2165,7 +2325,7 @@ func (b *InMemoryBackend) PutResourcePolicy( name := resolveSecretID(input.SecretID) - secret, ok := b.secretsStore(region)[name] + secret, ok := b.secretGet(region, name) if !ok { return nil, ErrSecretNotFound } @@ -2210,7 +2370,7 @@ func (b *InMemoryBackend) DeleteResourcePolicy( name := resolveSecretID(input.SecretID) - secret, ok := b.secretsStore(region)[name] + secret, ok := b.secretGet(region, name) if !ok { return nil, ErrSecretNotFound } @@ -2246,7 +2406,7 @@ func (b *InMemoryBackend) ReplicateSecretToRegions( name := resolveSecretID(input.SecretID) - secret, ok := b.secretsStore(region)[name] + secret, ok := b.secretGet(region, name) if !ok { return nil, ErrSecretNotFound } @@ -2306,7 +2466,7 @@ func (b *InMemoryBackend) RemoveRegionsFromReplication( name := resolveSecretID(input.SecretID) - secret, ok := b.secretsStore(region)[name] + secret, ok := b.secretGet(region, name) if !ok { return nil, ErrSecretNotFound } @@ -2351,7 +2511,7 @@ func (b *InMemoryBackend) StopReplicationToReplica( name := resolveSecretID(input.SecretID) - secret, ok := b.secretsStore(region)[name] + secret, ok := b.secretGet(region, name) if !ok { return nil, ErrSecretNotFound } @@ -2380,7 +2540,7 @@ func (b *InMemoryBackend) UpdateSecretVersionStage( name := resolveSecretID(input.SecretID) - secret, ok := b.secretsStore(region)[name] + secret, ok := b.secretGet(region, name) if !ok { return nil, ErrSecretNotFound } @@ -2422,16 +2582,31 @@ func (b *InMemoryBackend) moveStagingLabel(secret *Secret, input *UpdateSecretVe } } - // Strip the label from ALL versions — a staging label belongs to exactly one version. - for _, ver := range secret.Versions { - ver.StagingLabels = removeLabel(ver.StagingLabels, input.VersionStage) - } - targetVer, exists := secret.Versions[input.MoveToVersionID] if !exists { return ErrVersionNotFound } + // Per the real API (UpdateSecretVersionStageInput.RemoveFromVersionId): "If the + // staging label is already attached to a different version of the secret, then you + // must also specify RemoveFromVersionId. ... If the label is attached and you either + // do not specify this parameter, or the version ID does not match, then the + // operation fails." Find who currently holds the label (if anyone) and enforce that. + currentHolderID := versionIDWithLabel(secret, input.VersionStage) + if currentHolderID != "" && currentHolderID != input.MoveToVersionID && + currentHolderID != input.RemoveFromVersionID { + return fmt.Errorf( + "%w: staging label %s is currently attached to version %s;"+ + " RemoveFromVersionId must be set to %s to move it", + ErrInvalidParameter, input.VersionStage, currentHolderID, currentHolderID, + ) + } + + // Strip the label from ALL versions — a staging label belongs to exactly one version. + for _, ver := range secret.Versions { + ver.StagingLabels = removeLabel(ver.StagingLabels, input.VersionStage) + } + targetVer.StagingLabels = append(targetVer.StagingLabels, input.VersionStage) if input.VersionStage == StagingLabelCurrent { @@ -2441,6 +2616,18 @@ func (b *InMemoryBackend) moveStagingLabel(secret *Secret, input *UpdateSecretVe return nil } +// versionIDWithLabel returns the ID of the version currently carrying the given staging +// label, or "" if no version has it. A staging label is attached to at most one version. +func versionIDWithLabel(secret *Secret, label string) string { + for id, ver := range secret.Versions { + if slices.Contains(ver.StagingLabels, label) { + return id + } + } + + return "" +} + // removeLabelFromVersion removes a label from a specific version. func removeLabelFromVersion(secret *Secret, versionID, label string) error { ver, exists := secret.Versions[versionID] @@ -2472,15 +2659,13 @@ func (b *InMemoryBackend) Reset() { b.mu.Lock("Reset") defer b.mu.Unlock() - for _, regionSecrets := range b.secrets { - for _, secret := range regionSecrets { - if secret.Tags != nil { - secret.Tags.Close() - } + for _, secret := range b.secrets.All() { + if secret.Tags != nil { + secret.Tags.Close() } } - b.secrets = make(map[string]map[string]*Secret) + b.registry.ResetAll() b.resourcePolicies = make(map[string]map[string]string) b.replicationConfigs = make(map[string]map[string][]ReplicationStatusType) } @@ -2529,11 +2714,9 @@ func (b *InMemoryBackend) runScheduledRotations(now time.Time) { b.mu.Lock("rotationScheduler") var pending []pendingRotation - for region, regionSecrets := range b.secrets { - for id, secret := range regionSecrets { - if p, ok := b.scheduleRotationLocked(region, id, secret, now); ok { - pending = append(pending, p) - } + for _, secret := range b.secrets.All() { + if p, ok := b.scheduleRotationLocked(secret.region, secret.Name, secret, now); ok { + pending = append(pending, p) } } @@ -2697,8 +2880,8 @@ func (b *InMemoryBackend) Region() string { return b.region } // The secret is placed in the region encoded in its ARN (falling back to the // backend's default region). Must not be called concurrently with other operations. func (b *InMemoryBackend) AddSecretInternal(s *Secret) { - region := regionFromARN(s.ARN, b.region) - b.secretsStore(region)[s.Name] = s + s.region = regionFromARN(s.ARN, b.region) + b.secretPut(s) } // ValidateResourcePolicy validates a resource-based policy document for a secret. @@ -2715,11 +2898,11 @@ func (b *InMemoryBackend) ValidateResourcePolicy( if input.SecretID != "" { region := getRegion(ctx, b.region) - b.mu.RLock("ValidateResourcePolicy") + b.mu.RLock(opValidateResourcePolicy) defer b.mu.RUnlock() name := resolveSecretID(input.SecretID) - if _, ok := b.secretsStore(region)[name]; !ok { + if !b.secretHas(region, name) { return nil, ErrSecretNotFound } } diff --git a/services/secretsmanager/batch1_audit_test.go b/services/secretsmanager/batch1_audit_test.go index 62a81f133..b96cf4cb5 100644 --- a/services/secretsmanager/batch1_audit_test.go +++ b/services/secretsmanager/batch1_audit_test.go @@ -1136,7 +1136,7 @@ func TestAudit_ListSecrets_FilterByTagValue(t *testing.T) { assert.Equal(t, "prod-secret", out.SecretList[0].Name) } -func TestAudit_ListSecrets_IncludeDeleted(t *testing.T) { +func TestAudit_ListSecrets_IncludePlannedDeletion(t *testing.T) { t.Parallel() b := sm.NewInMemoryBackend() @@ -1147,11 +1147,11 @@ func TestAudit_ListSecrets_IncludeDeleted(t *testing.T) { _, err = b.DeleteSecret(context.Background(), &sm.DeleteSecretInput{SecretID: "dead"}) require.NoError(t, err) - out, err := b.ListSecrets(context.Background(), &sm.ListSecretsInput{IncludeDeleted: true}) + out, err := b.ListSecrets(context.Background(), &sm.ListSecretsInput{IncludePlannedDeletion: true}) require.NoError(t, err) assert.Len(t, out.SecretList, 2) - out2, err := b.ListSecrets(context.Background(), &sm.ListSecretsInput{IncludeDeleted: false}) + out2, err := b.ListSecrets(context.Background(), &sm.ListSecretsInput{IncludePlannedDeletion: false}) require.NoError(t, err) assert.Len(t, out2.SecretList, 1) } diff --git a/services/secretsmanager/concurrency_race_test.go b/services/secretsmanager/concurrency_race_test.go new file mode 100644 index 000000000..6802a1635 --- /dev/null +++ b/services/secretsmanager/concurrency_race_test.go @@ -0,0 +1,93 @@ +package secretsmanager //nolint:testpackage // needs access to the unexported region context key. + +import ( + "context" + "fmt" + "sync" + "testing" +) + +// Test_ConcurrentRegionReads_NoDataRace exercises read-only ops (RLock-guarded) against +// many never-before-seen regions concurrently. Before the original fix, ListSecrets, +// ListSecretVersionIDs, DescribeSecret, GetResourcePolicy, and ValidateResourcePolicy all +// called a lazily-creating *Store(region) helper (which wrote b.secrets[region] = +// make(...) on first touch) while holding only an RLock — a concurrent map write during +// a shared read lock, which the Go race detector (and, at higher contention, the runtime +// itself) flags as a data race. The fix introduced non-mutating *StoreRO(region) helpers +// for read paths. Phase 3.3 replaced the region-nested secrets map with a flat +// store.Table[Secret] (see store_setup.go): store.Table.Get never lazily creates +// anything, so secretGet is unconditionally RLock-safe and this race class is now +// structurally impossible for secrets, not just avoided by convention. This test remains +// to guard resourcePolicies/replicationConfigs, which are still region-nested raw maps +// using the original *Store/*StoreRO split. Run with `go test -race` to verify. +func Test_ConcurrentRegionReads_NoDataRace(t *testing.T) { + t.Parallel() + + const goroutinesPerOp = 24 + + cases := []struct { + run func(ctx context.Context, b *InMemoryBackend) + name string + }{ + { + name: opListSecrets, + run: func(ctx context.Context, b *InMemoryBackend) { + _, _ = b.ListSecrets(ctx, &ListSecretsInput{}) + }, + }, + { + name: "ListSecretVersionIDs", + run: func(ctx context.Context, b *InMemoryBackend) { + _, _ = b.ListSecretVersionIDs(ctx, &ListSecretVersionIDsInput{SecretID: "nonexistent"}) + }, + }, + { + name: opDescribeSecret, + run: func(ctx context.Context, b *InMemoryBackend) { + _, _ = b.DescribeSecret(ctx, &DescribeSecretInput{SecretID: "nonexistent"}) + }, + }, + { + name: opGetResourcePolicy, + run: func(ctx context.Context, b *InMemoryBackend) { + _, _ = b.GetResourcePolicy(ctx, &GetResourcePolicyInput{SecretID: "nonexistent"}) + }, + }, + { + name: opValidateResourcePolicy, + run: func(ctx context.Context, b *InMemoryBackend) { + _, _ = b.ValidateResourcePolicy(ctx, &ValidateResourcePolicyInput{ + SecretID: "nonexistent", + ResourcePolicy: `{"Version":"2012-10-17","Statement":[]}`, + }) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + // Fresh, isolated backend per subtest. + b := NewInMemoryBackendWithConfig("000000000000", "us-east-1") + + var wg sync.WaitGroup + + for i := range goroutinesPerOp { + wg.Add(1) + + go func(i int) { + defer wg.Done() + + // Each goroutine targets a distinct, never-before-seen region so the + // lazy per-region map entry is created concurrently by many readers. + region := fmt.Sprintf("race-region-%d", i) + ctx := context.WithValue(context.Background(), regionContextKey{}, region) + tc.run(ctx, b) + }(i) + } + + wg.Wait() + }) + } +} diff --git a/services/secretsmanager/createsecret_idempotency_test.go b/services/secretsmanager/createsecret_idempotency_test.go new file mode 100644 index 000000000..f5dfe1917 --- /dev/null +++ b/services/secretsmanager/createsecret_idempotency_test.go @@ -0,0 +1,96 @@ +package secretsmanager_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + sm "github.com/blackbirdworks/gopherstack/services/secretsmanager" +) + +// Test_CreateSecret_ClientRequestTokenIdempotency verifies the real AWS CreateSecret +// idempotency contract documented on CreateSecretInput.ClientRequestToken +// (aws-sdk-go-v2/service/secretsmanager/api_op_CreateSecret.go): +// +// - If the token isn't already associated with a version, a new version is created. +// - If a version with this token exists and its content matches the request, the +// request is ignored (idempotent retry succeeds without creating anything new). +// - If a version with this token exists but its content differs, the request fails +// (CreateSecret cannot modify an existing version). +// +// This mock previously ignored ClientRequestToken entirely on name collisions, always +// returning ResourceExistsException even for a verbatim retry of a prior successful call. +func Test_CreateSecret_ClientRequestTokenIdempotency(t *testing.T) { + t.Parallel() + + cases := []struct { + wantErrIs error + name string + retryToken string + retrySecretStr string + wantSameVerID bool + }{ + { + name: "same_token_same_content_is_ignored", + retryToken: "req-token-1", + retrySecretStr: "hunter2", + wantSameVerID: true, + }, + { + name: "same_token_different_content_fails", + retryToken: "req-token-1", + retrySecretStr: "different-value", + wantErrIs: sm.ErrInvalidParameter, + }, + { + name: "different_token_fails_as_resource_exists", + retryToken: "req-token-2", + retrySecretStr: "hunter2", + wantErrIs: sm.ErrSecretAlreadyExists, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := sm.NewInMemoryBackend() + secretName := "idempotent-" + tc.name + + first, err := b.CreateSecret(context.Background(), &sm.CreateSecretInput{ + Name: secretName, + SecretString: "hunter2", + ClientRequestToken: "req-token-1", + }) + require.NoError(t, err) + + second, err := b.CreateSecret(context.Background(), &sm.CreateSecretInput{ + Name: secretName, + SecretString: tc.retrySecretStr, + ClientRequestToken: tc.retryToken, + }) + + if tc.wantErrIs != nil { + require.Error(t, err) + assert.ErrorIs(t, err, tc.wantErrIs) + + return + } + + require.NoError(t, err) + if tc.wantSameVerID { + assert.Equal(t, first.VersionID, second.VersionID, "idempotent retry must return the same VersionId") + assert.Equal(t, first.ARN, second.ARN) + } + + // The idempotent retry must not have created a second version. + out, err := b.ListSecretVersionIDs(context.Background(), &sm.ListSecretVersionIDsInput{ + SecretID: secretName, + }) + require.NoError(t, err) + assert.Len(t, out.Versions, 1, "idempotent retry must not create a new version") + }) + } +} diff --git a/services/secretsmanager/export_test.go b/services/secretsmanager/export_test.go index 683e36f08..ead8d4cfe 100644 --- a/services/secretsmanager/export_test.go +++ b/services/secretsmanager/export_test.go @@ -7,12 +7,7 @@ func SecretCount(b *InMemoryBackend) int { b.mu.RLock("SecretCount") defer b.mu.RUnlock() - total := 0 - for _, regionSecrets := range b.secrets { - total += len(regionSecrets) - } - - return total + return b.secrets.Len() } // ResourcePolicyCount returns the total number of resource policies across all regions. diff --git a/services/secretsmanager/handler.go b/services/secretsmanager/handler.go index eeb5c8468..f43bd0aed 100644 --- a/services/secretsmanager/handler.go +++ b/services/secretsmanager/handler.go @@ -151,12 +151,12 @@ func (h *Handler) GetSupportedOperations() []string { "CreateSecret", "DeleteResourcePolicy", "DeleteSecret", - "DescribeSecret", + opDescribeSecret, "GetRandomPassword", - "GetResourcePolicy", + opGetResourcePolicy, "GetSecretValue", "ListSecretVersionIds", - "ListSecrets", + opListSecrets, "PutResourcePolicy", "PutSecretValue", "RemoveRegionsFromReplication", @@ -168,7 +168,7 @@ func (h *Handler) GetSupportedOperations() []string { "UntagResource", "UpdateSecret", "UpdateSecretVersionStage", - "ValidateResourcePolicy", + opValidateResourcePolicy, } } @@ -250,7 +250,7 @@ type smActionFn func(ctx context.Context, region string, body []byte) (any, erro func (h *Handler) smExtendedActions() map[string]smActionFn { return map[string]smActionFn{ - "GetResourcePolicy": func(ctx context.Context, _ string, b []byte) (any, error) { + opGetResourcePolicy: func(ctx context.Context, _ string, b []byte) (any, error) { var input GetResourcePolicyInput if err := json.Unmarshal(b, &input); err != nil { return nil, err @@ -314,7 +314,7 @@ func (h *Handler) smExtendedActions() map[string]smActionFn { return h.Backend.StopReplicationToReplica(ctx, &input) }, - "ValidateResourcePolicy": func(ctx context.Context, _ string, b []byte) (any, error) { + opValidateResourcePolicy: func(ctx context.Context, _ string, b []byte) (any, error) { var input ValidateResourcePolicyInput if err := json.Unmarshal(b, &input); err != nil { return nil, err @@ -360,7 +360,7 @@ func (h *Handler) smCRUDActions() map[string]smActionFn { return h.Backend.DeleteSecret(ctx, &input) }, - "ListSecrets": func(ctx context.Context, _ string, b []byte) (any, error) { + opListSecrets: func(ctx context.Context, _ string, b []byte) (any, error) { var input ListSecretsInput if err := json.Unmarshal(b, &input); err != nil { return nil, err @@ -368,7 +368,7 @@ func (h *Handler) smCRUDActions() map[string]smActionFn { return h.Backend.ListSecrets(ctx, &input) }, - "DescribeSecret": func(ctx context.Context, _ string, b []byte) (any, error) { + opDescribeSecret: func(ctx context.Context, _ string, b []byte) (any, error) { var input DescribeSecretInput if err := json.Unmarshal(b, &input); err != nil { return nil, err diff --git a/services/secretsmanager/handler_refinement1_test.go b/services/secretsmanager/handler_refinement1_test.go index 20993e487..6ce62e23c 100644 --- a/services/secretsmanager/handler_refinement1_test.go +++ b/services/secretsmanager/handler_refinement1_test.go @@ -147,8 +147,21 @@ func TestRefinement1_AddSecretInternal(t *testing.T) { assert.Equal(t, "my-seed", got.Name) } -// TestRefinement1_UpdateSecretVersionStageAutoStrip verifies a staging label is stripped from all -// other versions when moved, not just RemoveFromVersionID. +// TestRefinement1_UpdateSecretVersionStageAutoStrip verifies a staging label is stripped +// from its current holder when moved via a compliant RemoveFromVersionId, and that +// omitting/mismatching RemoveFromVersionId when the label is attached elsewhere is +// rejected. +// +// NOTE: this test previously asserted that UpdateSecretVersionStage would silently move +// AWSPREVIOUS to v2 with NO RemoveFromVersionId at all, even though AWSPREVIOUS was +// already attached to v1 at that point (PutSecretValue auto-rotates the outgoing +// AWSCURRENT to AWSPREVIOUS). That encoded incorrect AWS behavior: per +// UpdateSecretVersionStageInput.RemoveFromVersionId in the real API, "If the staging +// label is already attached to a different version of the secret, then you must also +// specify the RemoveFromVersionId parameter. ... If the label is attached and you +// either do not specify this parameter, or the version ID does not match, then the +// operation fails." Fixed to (a) supply the required RemoveFromVersionId for the +// success path, and (b) add a case proving the rejection when it's omitted. func TestRefinement1_UpdateSecretVersionStageAutoStrip(t *testing.T) { t.Parallel() @@ -167,7 +180,7 @@ func TestRefinement1_UpdateSecretVersionStageAutoStrip(t *testing.T) { v1 = id } - // Create second version. + // Create second version. AWSCURRENT moves to v2; v1 auto-rotates to AWSPREVIOUS. put, err := b.PutSecretValue(context.Background(), &secretsmanager.PutSecretValueInput{ SecretID: "vs-strip", SecretString: "v2", @@ -175,12 +188,24 @@ func TestRefinement1_UpdateSecretVersionStageAutoStrip(t *testing.T) { require.NoError(t, err) v2 := put.VersionID - // Move AWSPREVIOUS label to v2 (stripping from v1 where it may be). + // Moving AWSPREVIOUS to v2 WITHOUT declaring its current holder (v1) must be + // rejected — the caller must acknowledge where the label is coming from. _, err = b.UpdateSecretVersionStage(context.Background(), &secretsmanager.UpdateSecretVersionStageInput{ SecretID: "vs-strip", VersionStage: secretsmanager.StagingLabelPrevious, MoveToVersionID: v2, }) + require.ErrorIs(t, err, secretsmanager.ErrInvalidParameter, + "moving a label away from its current holder without RemoveFromVersionId must fail") + + // With the correct RemoveFromVersionId, the move succeeds and strips the label + // from v1. + _, err = b.UpdateSecretVersionStage(context.Background(), &secretsmanager.UpdateSecretVersionStageInput{ + SecretID: "vs-strip", + VersionStage: secretsmanager.StagingLabelPrevious, + MoveToVersionID: v2, + RemoveFromVersionID: v1, + }) require.NoError(t, err) // Verify v1 no longer has AWSPREVIOUS. diff --git a/services/secretsmanager/handler_test.go b/services/secretsmanager/handler_test.go index 61b0fe6c5..0f1b6bd6c 100644 --- a/services/secretsmanager/handler_test.go +++ b/services/secretsmanager/handler_test.go @@ -276,7 +276,9 @@ func TestSecretsManagerBackendListSecrets(t *testing.T) { _, _ = backend.CreateSecret(context.Background(), &secretsmanager.CreateSecretInput{Name: "deleted"}) _, _ = backend.DeleteSecret(context.Background(), &secretsmanager.DeleteSecretInput{SecretID: "deleted"}) - out, err := backend.ListSecrets(context.Background(), &secretsmanager.ListSecretsInput{IncludeDeleted: true}) + out, err := backend.ListSecrets(context.Background(), &secretsmanager.ListSecretsInput{ + IncludePlannedDeletion: true, + }) require.NoError(t, err) assert.Len(t, out.SecretList, 2) }) diff --git a/services/secretsmanager/janitor.go b/services/secretsmanager/janitor.go index 020de658c..42043f610 100644 --- a/services/secretsmanager/janitor.go +++ b/services/secretsmanager/janitor.go @@ -63,29 +63,27 @@ func (j *Janitor) sweepExpiredSecrets(ctx context.Context) { j.Backend.mu.Lock("sweepExpiredSecrets") purged := 0 - for region, regionSecrets := range j.Backend.secrets { - for name, secret := range regionSecrets { - if secret.DeletedDate == nil { - continue - } - // Use ScheduledDeletionDate if set (reflects the actual RecoveryWindowInDays supplied at - // delete time). Fall back to the default 30-day window for secrets deleted before this - // field was introduced or force-deleted without a recovery window. - var deletionTime float64 - if secret.ScheduledDeletionDate != nil { - deletionTime = *secret.ScheduledDeletionDate - } else { - deletionTime = *secret.DeletedDate + float64(defaultRecoveryWindowDays*secondsPerDay) - } - if nowFloat >= deletionTime { - if secret.Tags != nil { - secret.Tags.Close() - } - delete(regionSecrets, name) - delete(j.Backend.resourcePoliciesStore(region), name) - delete(j.Backend.replicationConfigsStore(region), name) - purged++ + for _, secret := range j.Backend.secrets.All() { + if secret.DeletedDate == nil { + continue + } + // Use ScheduledDeletionDate if set (reflects the actual RecoveryWindowInDays supplied at + // delete time). Fall back to the default 30-day window for secrets deleted before this + // field was introduced or force-deleted without a recovery window. + var deletionTime float64 + if secret.ScheduledDeletionDate != nil { + deletionTime = *secret.ScheduledDeletionDate + } else { + deletionTime = *secret.DeletedDate + float64(defaultRecoveryWindowDays*secondsPerDay) + } + if nowFloat >= deletionTime { + if secret.Tags != nil { + secret.Tags.Close() } + j.Backend.secretDelete(secret.region, secret.Name) + delete(j.Backend.resourcePoliciesStore(secret.region), secret.Name) + delete(j.Backend.replicationConfigsStore(secret.region), secret.Name) + purged++ } } j.Backend.mu.Unlock() diff --git a/services/secretsmanager/listsecrets_wireshape_test.go b/services/secretsmanager/listsecrets_wireshape_test.go new file mode 100644 index 000000000..fbc44f4cf --- /dev/null +++ b/services/secretsmanager/listsecrets_wireshape_test.go @@ -0,0 +1,196 @@ +package secretsmanager_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + sm "github.com/blackbirdworks/gopherstack/services/secretsmanager" +) + +// Test_ListSecrets_SortBy verifies the ListSecrets SortBy field (created-date, +// last-changed-date, last-accessed-date, and the default name ordering), combined +// with SortOrder asc/desc. Matches the real AWS SortByType enum +// (aws-sdk-go-v2/service/secretsmanager/types.SortByType) which this mock previously +// ignored entirely, always ordering by name regardless of the client's request. +func Test_ListSecrets_SortBy(t *testing.T) { + t.Parallel() + + t.Run("created-date", func(t *testing.T) { + t.Parallel() + + b := sm.NewInMemoryBackend() + + // Create secrets out of name order but in a known CreatedDate order: + // zebra (oldest) -> mango -> apple (newest). + times := []time.Time{ + time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + time.Date(2024, 12, 1, 0, 0, 0, 0, time.UTC), + } + names := []string{"zebra", "mango", "apple"} + + for i, name := range names { + ts := times[i] + b.SetNowForTest(func() time.Time { return ts }) + _, err := b.CreateSecret(context.Background(), &sm.CreateSecretInput{Name: name, SecretString: "v"}) + require.NoError(t, err) + } + b.SetNowForTest(time.Now) + + out, err := b.ListSecrets(context.Background(), &sm.ListSecretsInput{SortBy: "created-date"}) + require.NoError(t, err) + require.Len(t, out.SecretList, 3) + gotNames := []string{out.SecretList[0].Name, out.SecretList[1].Name, out.SecretList[2].Name} + assert.Equal(t, []string{"zebra", "mango", "apple"}, gotNames, "ascending created-date order") + + outDesc, err := b.ListSecrets(context.Background(), &sm.ListSecretsInput{ + SortBy: "created-date", SortOrder: "desc", + }) + require.NoError(t, err) + require.Len(t, outDesc.SecretList, 3) + gotNamesDesc := []string{outDesc.SecretList[0].Name, outDesc.SecretList[1].Name, outDesc.SecretList[2].Name} + assert.Equal(t, []string{"apple", "mango", "zebra"}, gotNamesDesc, "descending created-date order") + }) + + t.Run("last-changed-date", func(t *testing.T) { + t.Parallel() + + b := sm.NewInMemoryBackend() + + for _, name := range []string{"c-secret", "a-secret", "b-secret"} { + _, err := b.CreateSecret(context.Background(), &sm.CreateSecretInput{Name: name, SecretString: "v0"}) + require.NoError(t, err) + } + + // Change values out of name order but in a known LastChangedDate order: + // b-secret (oldest change) -> c-secret -> a-secret (most recent change). + changeOrder := []struct { + ts time.Time + name string + }{ + {time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), "b-secret"}, + {time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), "c-secret"}, + {time.Date(2024, 12, 1, 0, 0, 0, 0, time.UTC), "a-secret"}, + } + for _, step := range changeOrder { + ts := step.ts + b.SetNowForTest(func() time.Time { return ts }) + _, err := b.UpdateSecret(context.Background(), &sm.UpdateSecretInput{ + SecretID: step.name, SecretString: "v1", + }) + require.NoError(t, err) + } + b.SetNowForTest(time.Now) + + out, err := b.ListSecrets(context.Background(), &sm.ListSecretsInput{SortBy: "last-changed-date"}) + require.NoError(t, err) + require.Len(t, out.SecretList, 3) + gotNames := []string{out.SecretList[0].Name, out.SecretList[1].Name, out.SecretList[2].Name} + assert.Equal(t, []string{"b-secret", "c-secret", "a-secret"}, gotNames, "ascending last-changed-date order") + }) + + t.Run("last-accessed-date_nil_sorts_first", func(t *testing.T) { + t.Parallel() + + b := sm.NewInMemoryBackend() + + _, err := b.CreateSecret(context.Background(), &sm.CreateSecretInput{Name: "never-read", SecretString: "v"}) + require.NoError(t, err) + _, err = b.CreateSecret(context.Background(), &sm.CreateSecretInput{Name: "was-read", SecretString: "v"}) + require.NoError(t, err) + + _, err = b.GetSecretValue(context.Background(), &sm.GetSecretValueInput{SecretID: "was-read"}) + require.NoError(t, err) + + out, err := b.ListSecrets(context.Background(), &sm.ListSecretsInput{SortBy: "last-accessed-date"}) + require.NoError(t, err) + require.Len(t, out.SecretList, 2) + // A secret that has never been read has no LastAccessedDate, which sorts as the + // earliest possible value — it must come first in ascending order. + assert.Equal(t, "never-read", out.SecretList[0].Name) + assert.Equal(t, "was-read", out.SecretList[1].Name) + }) + + t.Run("default_is_name", func(t *testing.T) { + t.Parallel() + + b := sm.NewInMemoryBackend() + + for _, name := range []string{"zebra", "apple", "mango"} { + _, err := b.CreateSecret(context.Background(), &sm.CreateSecretInput{Name: name, SecretString: "v"}) + require.NoError(t, err) + } + + out, err := b.ListSecrets(context.Background(), &sm.ListSecretsInput{}) + require.NoError(t, err) + require.Len(t, out.SecretList, 3) + gotNames := []string{out.SecretList[0].Name, out.SecretList[1].Name, out.SecretList[2].Name} + assert.Equal(t, []string{"apple", "mango", "zebra"}, gotNames, "default SortBy is name ascending") + }) +} + +// Test_ListSecrets_NextRotationDate verifies that ListSecrets' SecretListEntry carries +// NextRotationDate (matching aws-sdk-go-v2/service/secretsmanager/types.SecretListEntry), +// which this mock previously omitted — only DescribeSecret computed it. +func Test_ListSecrets_NextRotationDate(t *testing.T) { + t.Parallel() + + cases := []struct { + configureFn func(t *testing.T, b *sm.InMemoryBackend, secretName string) + name string + wantHasValue bool + }{ + { + name: "rotation_not_configured_omits_field", + configureFn: func(_ *testing.T, _ *sm.InMemoryBackend, _ string) { + // no rotation configured + }, + wantHasValue: false, + }, + { + name: "rotation_configured_populates_field", + configureFn: func(t *testing.T, b *sm.InMemoryBackend, secretName string) { + t.Helper() + + _, err := b.RotateSecret(context.Background(), &sm.RotateSecretInput{ + SecretID: secretName, + RotationRules: &sm.RotationRulesType{ + AutomaticallyAfterDays: ptr64(30), + }, + }) + require.NoError(t, err) + }, + wantHasValue: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := sm.NewInMemoryBackend() + _, err := b.CreateSecret(context.Background(), &sm.CreateSecretInput{ + Name: "rot-" + tc.name, SecretString: "v", + }) + require.NoError(t, err) + + tc.configureFn(t, b, "rot-"+tc.name) + + out, err := b.ListSecrets(context.Background(), &sm.ListSecretsInput{ + Filters: []sm.SecretFilter{{Key: "name", Values: []string{"rot-" + tc.name}}}, + }) + require.NoError(t, err) + require.Len(t, out.SecretList, 1) + + if tc.wantHasValue { + assert.NotNil(t, out.SecretList[0].NextRotationDate) + } else { + assert.Nil(t, out.SecretList[0].NextRotationDate) + } + }) + } +} diff --git a/services/secretsmanager/models.go b/services/secretsmanager/models.go index 566d53156..396954fe1 100644 --- a/services/secretsmanager/models.go +++ b/services/secretsmanager/models.go @@ -34,6 +34,16 @@ type SecretVersion struct { // Secret represents a stored secret including all versions. type Secret struct { + // region is the AWS region this secret belongs to. It is the outer half of + // the composite key ("region|Name") used by the backend's flat + // store.Table[Secret] (see store_setup.go), which replaces the old + // map[string]map[string]*Secret nesting (outer key = region). Unexported + // so it never appears in Secrets Manager wire responses -- those are + // always built by hand (secretToListEntry, DescribeSecret, etc.), never by + // marshaling Secret directly -- but persistence.go must carry it through + // the secretSnapshot DTO explicitly since json.Marshal never sees + // unexported fields. + region string // Tags is a map of key/value tag pairs. Tags *tags.Tags `json:"Tags,omitempty"` // DeletedDate is set when the secret is deleted; nil means active. @@ -193,6 +203,7 @@ type SecretListEntry struct { LastAccessedDate *float64 `json:"LastAccessedDate,omitempty"` LastRotatedDate *float64 `json:"LastRotatedDate,omitempty"` CreatedDate *float64 `json:"CreatedDate,omitempty"` + NextRotationDate *float64 `json:"NextRotationDate,omitempty"` Tags *tags.Tags `json:"Tags,omitempty"` RotationRules *RotationRulesType `json:"RotationRules,omitempty"` SecretVersionsToStages map[string][]string `json:"SecretVersionsToStages,omitempty"` @@ -206,11 +217,18 @@ type SecretListEntry struct { // ListSecretsInput is the request payload for ListSecrets. type ListSecretsInput struct { - MaxResults *int64 `json:"MaxResults,omitempty"` - NextToken string `json:"NextToken,omitempty"` - SortOrder string `json:"SortOrder,omitempty"` // "asc" or "desc" - Filters []SecretFilter `json:"Filters,omitempty"` - IncludeDeleted bool `json:"IncludeDeleted,omitempty"` + MaxResults *int64 `json:"MaxResults,omitempty"` + NextToken string `json:"NextToken,omitempty"` + SortOrder string `json:"SortOrder,omitempty"` // "asc" or "desc" + // SortBy selects the field secrets are ordered by: "name" (default), + // "created-date", "last-changed-date", or "last-accessed-date". + SortBy string `json:"SortBy,omitempty"` + Filters []SecretFilter `json:"Filters,omitempty"` + // IncludePlannedDeletion specifies whether to include secrets scheduled for + // deletion. By default, secrets scheduled for deletion aren't included. + // This is the real AWS wire field name (previously misnamed IncludeDeleted, + // which real SDK clients never send, silently defeating the filter). + IncludePlannedDeletion bool `json:"IncludePlannedDeletion,omitempty"` } // SecretFilter is a filter criterion for ListSecrets and BatchGetSecretValue. diff --git a/services/secretsmanager/persistence.go b/services/secretsmanager/persistence.go index 59a0466d1..3541c2c6f 100644 --- a/services/secretsmanager/persistence.go +++ b/services/secretsmanager/persistence.go @@ -3,13 +3,29 @@ package secretsmanager import ( "context" "encoding/json" + "fmt" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) -// secretSnapshot captures all fields of Secret including those tagged json:"-". +// secretsManagerSnapshotVersion identifies the shape of [backendSnapshot]. It +// must be bumped whenever a change to secretSnapshot or backendSnapshot +// itself would make an older snapshot unsafe to decode as the current shape +// (e.g. a field's meaning or type changes). Restore compares this against the +// persisted value and discards (rather than attempts to partially decode) any +// mismatch — see Restore below. This is the first version: earlier +// (pre-Phase-3.3) snapshots carried no version field at all, so they also +// fail this check and are discarded rather than misread, which is the safe +// behaviour across any snapshot-format change. +const secretsManagerSnapshotVersion = 1 + +// secretSnapshot captures all fields of Secret including those tagged +// json:"-" and the unexported region field, which is needed to rebuild both +// the live Secret and the store.Table composite key ("region|name", see +// secretTableKey in store_setup.go) on Restore. type secretSnapshot struct { Tags *tags.Tags `json:"tags,omitempty"` DeletedDate *float64 `json:"deletedDate,omitempty"` @@ -20,6 +36,7 @@ type secretSnapshot struct { Versions map[string]*SecretVersion `json:"versions"` ARN string `json:"arn"` Name string `json:"name"` + Region string `json:"region"` Description string `json:"description,omitempty"` KmsKeyID string `json:"kmsKeyID,omitempty"` RotationLambdaARN string `json:"rotationLambdaARN,omitempty"` @@ -28,13 +45,30 @@ type secretSnapshot struct { RotationEnabled bool `json:"rotationEnabled,omitempty"` } -// backendSnapshot mirrors the region-nested backend maps (outer key = region). +// secretSnapshotKeyFn is the [store.Table] key function used for the +// ephemeral DTO table built inside Snapshot/Restore. It mirrors secretKeyFn +// so the on-disk table is keyed identically to the live b.secrets table. +func secretSnapshotKeyFn(ss *secretSnapshot) string { + return secretTableKey(ss.Region, ss.Name) +} + +// backendSnapshot is the top-level on-disk shape for the Secrets Manager +// backend. +// +// Tables holds one JSON-encoded array per registered DTO table, produced by +// [store.Registry.SnapshotAll] — currently just "secrets" ([]*secretSnapshot). +// resourcePolicies and replicationConfigs are still region-nested plain maps +// (see store_setup.go for why they were not converted to store.Table) and are +// persisted directly, as before. Version guards against decoding a snapshot +// from an incompatible (older or newer) build of this backend as though it +// were the current shape; see Restore. type backendSnapshot struct { - Secrets map[string]map[string]*secretSnapshot `json:"secrets"` + Tables map[string]json.RawMessage `json:"tables"` ResourcePolicies map[string]map[string]string `json:"resourcePolicies,omitempty"` ReplicationConfigs map[string]map[string][]ReplicationStatusType `json:"replicationConfigs,omitempty"` AccountID string `json:"accountID"` Region string `json:"region"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. @@ -43,47 +77,58 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() - secrets := make(map[string]map[string]*secretSnapshot, len(b.secrets)) - for region, regionSecrets := range b.secrets { - regionMap := make(map[string]*secretSnapshot, len(regionSecrets)) - for k, s := range regionSecrets { - regionMap[k] = &secretSnapshot{ - ARN: s.ARN, - Name: s.Name, - Description: s.Description, - KmsKeyID: s.KmsKeyID, - RotationLambdaARN: s.RotationLambdaARN, - RotationRules: cloneRotationRules(s.RotationRules), - Tags: s.Tags, - DeletedDate: s.DeletedDate, - LastChangedDate: s.LastChangedDate, - LastRotatedDate: s.LastRotatedDate, - LastAccessedDate: s.LastAccessedDate, - CreatedDate: s.CreatedDate, - Versions: s.Versions, - CurrentVersionID: s.CurrentVersionID, - RotationEnabled: s.RotationEnabled, - } - } - secrets[region] = regionMap + // Build a throwaway DTO registry purely to reuse store's deterministic, + // type-erased JSON encoding (store.Registry.SnapshotAll) instead of + // hand-rolling the marshal step. This is intentionally separate from the + // live b.registry: Secret carries an unexported region field that + // json.Marshal never sees, so a direct snapshot of the live table could + // not round-trip it. + dtoReg := store.NewRegistry() + secretDTOs := store.Register(dtoReg, "secrets", store.New(secretSnapshotKeyFn)) + + for _, s := range b.secrets.Snapshot() { + secretDTOs.Put(&secretSnapshot{ + ARN: s.ARN, + Name: s.Name, + Region: s.region, + Description: s.Description, + KmsKeyID: s.KmsKeyID, + RotationLambdaARN: s.RotationLambdaARN, + RotationRules: cloneRotationRules(s.RotationRules), + Tags: s.Tags, + DeletedDate: s.DeletedDate, + LastChangedDate: s.LastChangedDate, + LastRotatedDate: s.LastRotatedDate, + LastAccessedDate: s.LastAccessedDate, + CreatedDate: s.CreatedDate, + Versions: s.Versions, + CurrentVersionID: s.CurrentVersionID, + RotationEnabled: s.RotationEnabled, + }) + } + + tables, err := dtoReg.SnapshotAll() + if err != nil { + // The DTOs above are plain JSON-friendly structs, so a marshal failure + // here would indicate a programming error rather than bad input data. + // Log and skip the snapshot rather than panic, matching the + // persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, + "secretsmanager: snapshot table marshal failed", "error", err) + + return nil } snap := backendSnapshot{ - Secrets: secrets, + Version: secretsManagerSnapshotVersion, + Tables: tables, ResourcePolicies: b.resourcePolicies, ReplicationConfigs: b.replicationConfigs, AccountID: b.accountID, Region: b.region, } - data, err := json.Marshal(snap) - if err != nil { - logger.Load(ctx).WarnContext(ctx, "secretsmanager: failed to marshal snapshot", "error", err) - - return nil - } - - return data + return persistence.MarshalSnapshot(ctx, "secretsmanager", snap) } // Restore loads backend state from a JSON snapshot. @@ -100,28 +145,47 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { // Close Tags on any secrets that are being replaced to prevent // Prometheus registry leaks. - for _, regionSecrets := range b.secrets { - for _, secret := range regionSecrets { - if secret.Tags != nil { - secret.Tags.Close() - } + for _, secret := range b.secrets.All() { + if secret.Tags != nil { + secret.Tags.Close() } } - if snap.Secrets == nil { - snap.Secrets = make(map[string]map[string]*secretSnapshot) + if snap.Version != secretsManagerSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape — that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. + logger.Load(ctx).WarnContext(ctx, + "secretsmanager: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", secretsManagerSnapshotVersion) + + b.registry.ResetAll() + b.resourcePolicies = make(map[string]map[string]string) + b.replicationConfigs = make(map[string]map[string][]ReplicationStatusType) + b.accountID = snap.AccountID + b.region = snap.Region + + return nil } - b.secrets = make(map[string]map[string]*Secret, len(snap.Secrets)) + dtoReg := store.NewRegistry() + secretDTOs := store.Register(dtoReg, "secrets", store.New(secretSnapshotKeyFn)) - for region, regionSecrets := range snap.Secrets { - regionMap := make(map[string]*Secret, len(regionSecrets)) - for k, ss := range regionSecrets { - regionMap[k] = secretFromSnapshot(ss) - } - b.secrets[region] = regionMap + if err := dtoReg.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("secretsmanager: restore snapshot tables: %w", err) + } + + liveSecrets := make([]*Secret, 0, secretDTOs.Len()) + + for _, ss := range secretDTOs.All() { + liveSecrets = append(liveSecrets, secretFromSnapshot(ss)) } + b.secrets.Restore(liveSecrets) + b.accountID = snap.AccountID b.region = snap.Region @@ -137,27 +201,26 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.replicationConfigs = snap.ReplicationConfigs - b.ensureNonNilMaps() - for _, regionSecrets := range b.secrets { - for _, secret := range regionSecrets { - if secret.RotationRules != nil && secret.RotationEnabled { - b.ensureRotationScheduler() + for _, secret := range b.secrets.All() { + if secret.RotationRules != nil && secret.RotationEnabled { + b.ensureRotationScheduler() - return nil - } + return nil } } return nil } -// secretFromSnapshot rebuilds a live Secret from its persisted representation. +// secretFromSnapshot rebuilds a live Secret (including its region field) from +// its persisted representation. func secretFromSnapshot(ss *secretSnapshot) *Secret { if ss.Versions == nil { ss.Versions = make(map[string]*SecretVersion) } return &Secret{ + region: ss.Region, ARN: ss.ARN, Name: ss.Name, Description: ss.Description, @@ -176,21 +239,6 @@ func secretFromSnapshot(ss *secretSnapshot) *Secret { } } -// ensureNonNilMaps initialises any nil maps to avoid nil-map panics. -func (b *InMemoryBackend) ensureNonNilMaps() { - if b.secrets == nil { - b.secrets = make(map[string]map[string]*Secret) - } - - if b.resourcePolicies == nil { - b.resourcePolicies = make(map[string]map[string]string) - } - - if b.replicationConfigs == nil { - b.replicationConfigs = make(map[string]map[string][]ReplicationStatusType) - } -} - // Snapshot implements persistence.Persistable by delegating to the backend. func (h *Handler) Snapshot(ctx context.Context) []byte { type snapshotter interface { diff --git a/services/secretsmanager/store_conversion_test.go b/services/secretsmanager/store_conversion_test.go new file mode 100644 index 000000000..b2f260045 --- /dev/null +++ b/services/secretsmanager/store_conversion_test.go @@ -0,0 +1,126 @@ +package secretsmanager //nolint:testpackage // needs the unexported regionContextKey to exercise multi-region isolation. + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestFullStateSnapshotRestore is the Phase 3.3 full-state round-trip test: +// it exercises every field the store.Table conversion touches -- secret +// material (SecretString AND SecretBinary across two regions sharing the +// SAME secret name, proving the region-qualified composite key survives a +// round trip), multiple staged versions, tags, rotation configuration, and +// the two raw (non-store.Table) maps resourcePolicies and +// replicationConfigs -- to guard against a silent persistence regression. +func TestFullStateSnapshotRestore(t *testing.T) { + t.Parallel() + + original := NewInMemoryBackendWithConfig("000000000000", "us-east-1") + + ctxEast := smCtxRegion("us-east-1") + ctxWest := smCtxRegion("us-west-2") + + // Two secrets sharing the SAME name in two different regions: only a + // region-qualified key (not a bare name) can keep these distinct across + // a snapshot/restore round trip. + _, err := original.CreateSecret(ctxEast, &CreateSecretInput{ + Name: "shared-name", + Description: "east secret", + SecretString: "east-string-value", + Tags: []Tag{{Key: "env", Value: "east"}}, + }) + require.NoError(t, err) + + _, err = original.CreateSecret(ctxWest, &CreateSecretInput{ + Name: "shared-name", + Description: "west secret", + SecretBinary: []byte("west-binary-value"), + }) + require.NoError(t, err) + + // A second version on the east secret (moves the first version to + // AWSPREVIOUS). + _, err = original.PutSecretValue(ctxEast, &PutSecretValueInput{ + SecretID: "shared-name", + SecretString: "east-string-value-v2", + }) + require.NoError(t, err) + + // A resource policy (raw nested map, left unconverted) on the east secret only. + _, err = original.PutResourcePolicy(ctxEast, &PutResourcePolicyInput{ + SecretID: "shared-name", + ResourcePolicy: `{"Version":"2012-10-17","Statement":[]}`, + }) + require.NoError(t, err) + + // Replication config (raw nested map, left unconverted) on the east secret. + _, err = original.ReplicateSecretToRegions(ctxEast, &ReplicateSecretToRegionsInput{ + SecretID: "shared-name", + AddReplicaRegions: []ReplicaRegion{{Region: "eu-west-1"}}, + }) + require.NoError(t, err) + + // Rotation configuration, to prove RotationRules/RotationEnabled survive. + // RotateImmediately=false configures the schedule without rotating the + // value out from under the assertions below (immediate rotation with no + // LambdaInvoker wired would replace SecretString with a random UUID and + // add a third version -- see RotateSecret/rotateSecretLocked). + afterDays := int64(30) + rotateImmediately := false + _, err = original.RotateSecret(ctxEast, &RotateSecretInput{ + SecretID: "shared-name", + RotationLambdaARN: "arn:aws:lambda:us-east-1:000000000000:function:rotate", + RotationRules: &RotationRulesType{AutomaticallyAfterDays: &afterDays}, + RotateImmediately: &rotateImmediately, + }) + require.NoError(t, err) + + snap := original.Snapshot(context.Background()) + require.NotNil(t, snap) + + fresh := NewInMemoryBackendWithConfig("000000000000", "us-east-1") + require.NoError(t, fresh.Restore(context.Background(), snap)) + + // East secret: current string material, description, rotation, and + // replication config all survive. + eastVal, err := fresh.GetSecretValue(ctxEast, &GetSecretValueInput{SecretID: "shared-name"}) + require.NoError(t, err) + assert.Equal(t, "east-string-value-v2", eastVal.SecretString) + + eastDesc, err := fresh.DescribeSecret(ctxEast, &DescribeSecretInput{SecretID: "shared-name"}) + require.NoError(t, err) + assert.Equal(t, "east secret", eastDesc.Description) + assert.True(t, eastDesc.RotationEnabled) + require.NotNil(t, eastDesc.RotationRules) + require.NotNil(t, eastDesc.RotationRules.AutomaticallyAfterDays) + assert.Equal(t, int64(30), *eastDesc.RotationRules.AutomaticallyAfterDays) + require.Len(t, eastDesc.ReplicationStatus, 1) + assert.Equal(t, "eu-west-1", eastDesc.ReplicationStatus[0].Region) + + eastPolicy, err := fresh.GetResourcePolicy(ctxEast, &GetResourcePolicyInput{SecretID: "shared-name"}) + require.NoError(t, err) + assert.JSONEq(t, `{"Version":"2012-10-17","Statement":[]}`, eastPolicy.ResourcePolicy) + + // West secret: binary material survives, and remains isolated from the + // east secret sharing the same name (region-qualified key proof). + westVal, err := fresh.GetSecretValue(ctxWest, &GetSecretValueInput{SecretID: "shared-name"}) + require.NoError(t, err) + assert.Equal(t, []byte("west-binary-value"), westVal.SecretBinary) + assert.Empty(t, westVal.SecretString) + + westPolicy, err := fresh.GetResourcePolicy(ctxWest, &GetResourcePolicyInput{SecretID: "shared-name"}) + require.NoError(t, err) + assert.Empty(t, westPolicy.ResourcePolicy, "resource policy must stay scoped to the east region") + + // Both versions of the east secret (AWSPREVIOUS holding the original + // string value) survived the round trip too. + versions, err := fresh.ListSecretVersionIDs(ctxEast, &ListSecretVersionIDsInput{SecretID: "shared-name"}) + require.NoError(t, err) + assert.Len(t, versions.Versions, 2) + + // Overall counts: two distinct secrets (same name, different regions). + assert.Equal(t, 2, SecretCount(fresh)) +} diff --git a/services/secretsmanager/store_setup.go b/services/secretsmanager/store_setup.go new file mode 100644 index 000000000..e4a402603 --- /dev/null +++ b/services/secretsmanager/store_setup.go @@ -0,0 +1,56 @@ +package secretsmanager + +// Code in this file supports Phase 3.3 of the datalayer refactor: the +// backend's secrets field -- previously a map[string]map[string]*Secret keyed +// outer-first by region -- is replaced by a single flat *store.Table[Secret]. +// It follows pkgs/store's package doc and the services/ec2 (commit +// 12e611a4, data-driven registration slice) and services/sqs (commit +// 0f09d77c, DTO-registry) conversions, and mirrors services/cloudwatchlogs's +// region-qualified-table pattern (store_setup.go/region_accessors.go) most +// closely, since secrets are likewise nested by region. +// +// resourcePolicies (map[string]map[string]string) and replicationConfigs +// (map[string]map[string][]ReplicationStatusType) are deliberately NOT +// converted: store.Table requires a *V value with its own identity, but +// these hold a bare string and a bare []ReplicationStatusType respectively, +// neither of which carries a key of its own. They remain plain nested maps, +// unchanged by this refactor (see InMemoryBackend's *Store/*StoreRO helpers +// in backend.go, preserved as-is). +// +// # Region-qualified table +// +// Secret gained an unexported `region` field (see models.go) used to build +// the composite store.Table primary key "region|name" (secretTableKey / +// secretKeyFn below). Being unexported, it never appears in the Secrets +// Manager wire responses (those are always built by hand -- +// secretToListEntry, DescribeSecret, etc. -- never by marshaling Secret +// directly), but it also means Secret cannot round-trip through +// Table.Snapshot's own json.Marshal; persistence.go carries it through the +// pre-existing secretSnapshot DTO instead (which already exists for the +// other unexported/json:"-" Secret fields), plus a new Region field. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +// secretTableKey returns the store.Table primary key for a secret. +func secretTableKey(region, name string) string { + return region + "|" + name +} + +// secretKeyFn keys a Secret by its region+Name composite key. +func secretKeyFn(s *Secret) string { return secretTableKey(s.region, s.Name) } + +// secretRegionIndexKeyFn groups every secret belonging to one region +// (mirrors the old secrets[region] map) for per-region operations such as +// ListSecrets and BatchGetSecretValue's filter-based path. +func secretRegionIndexKeyFn(s *Secret) string { return s.region } + +// registerAllTables registers every store.Table on b.registry exactly once. +// It must be called during construction only, immediately after b.registry is +// created -- store.Register panics on a duplicate name, so runtime resets go +// through b.registry.ResetAll() instead (see InMemoryBackend.Reset in +// backend.go), never a second call to this function. +func registerAllTables(b *InMemoryBackend) { + b.secrets = store.Register(b.registry, "secrets", store.New(secretKeyFn)) + b.secretsByRegion = b.secrets.AddIndex("byRegion", secretRegionIndexKeyFn) +} diff --git a/services/secretsmanager/updatesecretversionstage_move_test.go b/services/secretsmanager/updatesecretversionstage_move_test.go new file mode 100644 index 000000000..8d8fcdb2e --- /dev/null +++ b/services/secretsmanager/updatesecretversionstage_move_test.go @@ -0,0 +1,110 @@ +package secretsmanager_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + sm "github.com/blackbirdworks/gopherstack/services/secretsmanager" +) + +// Test_UpdateSecretVersionStage_RemoveFromVersionIDRequiredWhenLabelElsewhere verifies the +// real API contract on UpdateSecretVersionStageInput.RemoveFromVersionId: "If the staging +// label is already attached to a different version of the secret, then you must also +// specify the RemoveFromVersionId parameter... If the label is attached and you either do +// not specify this parameter, or the version ID does not match, then the operation +// fails." This mock previously stripped the label from wherever it happened to be, +// regardless of RemoveFromVersionId — a real client relying on that guard rail (e.g. to +// avoid racing another rotation step) would never see the failure it depends on. +func Test_UpdateSecretVersionStage_RemoveFromVersionIDRequiredWhenLabelElsewhere(t *testing.T) { + t.Parallel() + + const label = "CUSTOM-LABEL" + + setup := func(t *testing.T) (*sm.InMemoryBackend, string, string) { + t.Helper() + + b := sm.NewInMemoryBackend() + out, err := b.CreateSecret(context.Background(), &sm.CreateSecretInput{ + Name: "usvs-move-guard", SecretString: "v1", + }) + require.NoError(t, err) + holderVersion := out.VersionID + + put, err := b.PutSecretValue(context.Background(), &sm.PutSecretValueInput{ + SecretID: "usvs-move-guard", SecretString: "v2", VersionStages: []string{"AWSPENDING"}, + }) + require.NoError(t, err) + otherVersion := put.VersionID + + // Attach the custom label to holderVersion only. + _, err = b.UpdateSecretVersionStage(context.Background(), &sm.UpdateSecretVersionStageInput{ + SecretID: "usvs-move-guard", VersionStage: label, MoveToVersionID: holderVersion, + }) + require.NoError(t, err) + + return b, holderVersion, otherVersion + } + + cases := []struct { + buildInput func(holderVersion, otherVersion string) *sm.UpdateSecretVersionStageInput + name string + wantErr bool + }{ + { + name: "no_remove_from_version_id_rejected", + buildInput: func(_, otherVersion string) *sm.UpdateSecretVersionStageInput { + return &sm.UpdateSecretVersionStageInput{ + SecretID: "usvs-move-guard", VersionStage: label, MoveToVersionID: otherVersion, + } + }, + wantErr: true, + }, + { + name: "mismatched_remove_from_version_id_rejected", + buildInput: func(_, otherVersion string) *sm.UpdateSecretVersionStageInput { + return &sm.UpdateSecretVersionStageInput{ + SecretID: "usvs-move-guard", VersionStage: label, MoveToVersionID: otherVersion, + RemoveFromVersionID: otherVersion, // wrong: label isn't on otherVersion + } + }, + wantErr: true, + }, + { + name: "correct_remove_from_version_id_accepted", + buildInput: func(holderVersion, otherVersion string) *sm.UpdateSecretVersionStageInput { + return &sm.UpdateSecretVersionStageInput{ + SecretID: "usvs-move-guard", VersionStage: label, MoveToVersionID: otherVersion, + RemoveFromVersionID: holderVersion, + } + }, + wantErr: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b, holderVersion, otherVersion := setup(t) + + _, err := b.UpdateSecretVersionStage(context.Background(), tc.buildInput(holderVersion, otherVersion)) + + if tc.wantErr { + require.Error(t, err) + assert.ErrorIs(t, err, sm.ErrInvalidParameter) + + return + } + + require.NoError(t, err) + + desc, err := b.DescribeSecret(context.Background(), &sm.DescribeSecretInput{SecretID: "usvs-move-guard"}) + require.NoError(t, err) + assert.Contains(t, desc.VersionIDsToStages[otherVersion], label) + assert.NotContains(t, desc.VersionIDsToStages[holderVersion], label) + }) + } +} diff --git a/services/ses/PARITY.md b/services/ses/PARITY.md new file mode 100644 index 000000000..f8d94180b --- /dev/null +++ b/services/ses/PARITY.md @@ -0,0 +1,159 @@ +--- +# PARITY MANIFEST SCHEMA — copy to services//PARITY.md, fill, keep updated. +# Purpose: record audit state so the NEXT audit diffs the delta instead of rescanning. +# Re-audit protocol: `git diff ..HEAD -- services//` for local drift, +# AND check the SDK module for ops added since sdk_version. Only audit changed/new surface; +# trust rows marked ok whose files are unchanged since last_audit_commit. +service: ses +sdk_module: aws-sdk-go-v2/service/ses@v1.34.20 # version audited against (query-XML, 2010-12-01) +last_audit_commit: a40e7cc1 # HEAD when this manifest was written +last_audit_date: 2026-07-05 +overall: A # ~370 LOC of genuine production fixes + ~500 LOC of test additions/updates +# Per-op or per-op-family status. Values: ok | partial | gap | deferred. +# wire=response/request shape vs SDK; errors=code+HTTP status; state=real mutate/read; persist=in backendSnapshot. +ops: + PutIdentityPolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "wire fixed this pass — see families.void_result_ops"} + DeleteIdentityPolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "wire fixed this pass"} + SetIdentityDkimEnabled: {wire: ok, errors: ok, state: ok, persist: ok, note: "wire fixed this pass"} + SetIdentityFeedbackForwardingEnabled: {wire: ok, errors: ok, state: ok, persist: ok, note: "wire fixed this pass"} + SetIdentityHeadersInNotificationsEnabled: {wire: ok, errors: ok, state: ok, persist: ok, note: "wire fixed this pass"} + SetIdentityMailFromDomain: {wire: ok, errors: ok, state: ok, persist: ok, note: "wire fixed + BehaviorOnMXFailure now accepted/validated/persisted/returned (was silently dropped)"} + SetIdentityNotificationTopic: {wire: ok, errors: ok, state: ok, persist: ok, note: "wire fixed this pass"} + UpdateReceiptRule: {wire: ok, errors: ok, state: ok, persist: ok, note: "wire fixed this pass"} + ReorderReceiptRuleSet: {wire: ok, errors: ok, state: ok, persist: ok, note: "wire fixed this pass"} + SetReceiptRulePosition: {wire: ok, errors: ok, state: ok, persist: ok, note: "wire fixed this pass"} + PutConfigurationSetDeliveryOptions: {wire: ok, errors: ok, state: ok, persist: ok, note: "wire fixed this pass"} + UpdateConfigurationSetEventDestination: {wire: ok, errors: ok, state: ok, persist: ok, note: "wire fixed this pass"} + UpdateConfigurationSetTrackingOptions: {wire: ok, errors: ok, state: ok, persist: ok, note: "wire fixed this pass"} + VerifyEmailAddress: {wire: ok, errors: ok, state: ok, persist: ok, note: "confirmed zero-member output shape — no Result wrapper, verified against SDK deserializer (body discarded)"} + DeleteVerifiedEmailAddress: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateAccountSendingEnabled: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateCustomVerificationEmailTemplate: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateConfigurationSetReputationMetricsEnabled: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateConfigurationSetSendingEnabled: {wire: ok, errors: ok, state: ok, persist: ok} + SendEmail: {wire: ok, errors: ok, state: ok, persist: ok, note: "added AccountSendingPausedException + 24h-quota + ConfigurationSetDoesNotExist enforcement (all previously unenforced)"} + SendRawEmail: {wire: ok, errors: ok, state: ok, persist: ok, note: "delegates to SendEmail, inherits the same fixes"} + SendTemplatedEmail: {wire: ok, errors: ok, state: ok, persist: ok, note: "same enforcement added as SendEmail"} + SendBulkTemplatedEmail: {wire: ok, errors: ok, state: ok, persist: ok, note: "was silently dropping ConfigurationSetName/ReplyToAddresses/ReturnPath/SourceArn — all real SendBulkTemplatedEmailInput members; now parsed and threaded through"} + SendBounce: {wire: ok, errors: ok, state: ok, persist: ok, note: "was a disguised stub (no field validation, no sender-verification check, deterministic fabricated MessageId); now validates BounceSender + BouncedRecipientInfoList as required (matching SendBounceInput), enforces sender verification, real unique MessageId"} + SendCustomVerificationEmail: {wire: ok, errors: ok, state: ok, persist: ok, note: "was a disguised stub (never checked template existed, never registered/verified the identity, fabricated deterministic MessageId); now validates template + optional ConfigurationSetName, registers+verifies the identity, real unique MessageId"} + VerifyEmailIdentity: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteIdentity: {wire: ok, errors: ok, state: ok, persist: ok} + ListIdentities: {wire: ok, errors: ok, state: ok, persist: ok} + GetIdentityVerificationAttributes: {wire: ok, errors: ok, state: ok, persist: ok, note: "entry/key/value map shape verified against SDK deserializer"} + GetIdentityDkimAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + GetIdentityMailFromDomainAttributes: {wire: ok, errors: ok, state: ok, persist: ok, note: "BehaviorOnMXFailure now actually reflects the persisted value"} + GetIdentityNotificationAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + GetIdentityPolicies: {wire: ok, errors: ok, state: ok, persist: ok} + ListIdentityPolicies: {wire: ok, errors: ok, state: ok, persist: ok} + VerifyDomainIdentity: {wire: ok, errors: ok, state: ok, persist: ok} + VerifyDomainDkim: {wire: ok, errors: ok, state: ok, persist: ok, note: "deterministic tokens per identity, stable across calls"} + ListVerifiedEmailAddresses: {wire: ok, errors: ok, state: ok, persist: ok} + GetAccountSendingEnabled: {wire: ok, errors: ok, state: ok, persist: ok} + GetSendQuota: {wire: ok, errors: ok, state: ok, persist: ok, note: "SentLast24Hours logic factored into sentLast24HoursLocked(), now also used to enforce the quota on send ops"} + GetSendStatistics: {wire: ok, errors: ok, state: partial, persist: ok, note: "DeliveryAttempts accurate; Bounces/Complaints/Rejects always 0 — no bounce/complaint simulation (bd: gopherstack-uve)"} + CreateTemplate: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateTemplate: {wire: ok, errors: ok, state: ok, persist: ok} + GetTemplate: {wire: ok, errors: ok, state: ok, persist: ok} + ListTemplates: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteTemplate: {wire: ok, errors: ok, state: ok, persist: ok} + TestRenderTemplate: {wire: ok, errors: ok, state: ok, persist: ok, note: "{{key}} substitution against real stored template parts"} + CreateConfigurationSet: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteConfigurationSet: {wire: ok, errors: ok, state: ok, persist: ok, note: "cascades event destinations + tracking options"} + ListConfigurationSets: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeConfigurationSet: {wire: ok, errors: ok, state: ok, persist: ok} + CreateConfigurationSetEventDestination: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteConfigurationSetEventDestination: {wire: ok, errors: ok, state: ok, persist: ok} + CreateConfigurationSetTrackingOptions: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteConfigurationSetTrackingOptions: {wire: ok, errors: ok, state: ok, persist: ok} + CreateCustomVerificationEmailTemplate: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteCustomVerificationEmailTemplate: {wire: ok, errors: ok, state: ok, persist: ok} + GetCustomVerificationEmailTemplate: {wire: ok, errors: ok, state: ok, persist: ok} + ListCustomVerificationEmailTemplates: {wire: ok, errors: ok, state: ok, persist: ok} + CreateReceiptRuleSet: {wire: ok, errors: ok, state: ok, persist: ok} + CloneReceiptRuleSet: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteReceiptRuleSet: {wire: ok, errors: ok, state: ok, persist: ok, note: "clears activeRuleSet if it was the active set"} + ListReceiptRuleSets: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeReceiptRuleSet: {wire: ok, errors: ok, state: ok, persist: ok} + SetActiveReceiptRuleSet: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeActiveReceiptRuleSet: {wire: ok, errors: ok, state: ok, persist: ok} + CreateReceiptRule: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeReceiptRule: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteReceiptRule: {wire: ok, errors: ok, state: ok, persist: ok} + CreateReceiptFilter: {wire: ok, errors: ok, state: ok, persist: ok} + ListReceiptFilters: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteReceiptFilter: {wire: ok, errors: ok, state: ok, persist: ok} +families: + void_result_ops: {status: ok, note: "SEVERE FIX — 13 void-result ops (PutIdentityPolicy, DeleteIdentityPolicy, SetIdentityDkimEnabled, SetIdentityFeedbackForwardingEnabled, SetIdentityHeadersInNotificationsEnabled, SetIdentityMailFromDomain, SetIdentityNotificationTopic, UpdateReceiptRule, ReorderReceiptRuleSet, SetReceiptRulePosition, PutConfigurationSetDeliveryOptions, UpdateConfigurationSetEventDestination, UpdateConfigurationSetTrackingOptions) emitted a literal '<*Result>' XML element (a hardcoded xml:\"*Result\" tag on a struct{} field is not a Go/encoding-xml templating mechanism — it marshals as a literal element named '*Result'). Real aws-sdk-go-v2 client deserializers call decoder.GetElement(\"Result\") before parsing the body; the mismatched wrapper caused every real AWS SDK client to fail with a client-side DeserializationError even though the emulator's backend mutation succeeded — i.e. these 19 ops were unusable from a real SDK despite passing this repo's own unit tests. Fixed by replacing the generic emptyResponse.Result struct{} field with a nested emptyResult{XMLName xml.Name} whose name is set per-op at construction (newEmptyResponseWithResult for the 13 ops verified via the SDK deserializer to require a Result wrapper; newEmptyResponse — no wrapper — for the other 6 ops (VerifyEmailAddress, DeleteVerifiedEmailAddress, UpdateAccountSendingEnabled, UpdateCustomVerificationEmailTemplate, UpdateConfigurationSetReputationMetricsEnabled, UpdateConfigurationSetSendingEnabled) whose real output shape has zero members, so the SDK deserializer never looks for a Result element at all — confirmed by reading both deserializers.go code paths line-by-line, not guessing)."} + account_sending_paused: {status: ok, note: "UpdateAccountSendingEnabled(false)/GetAccountSendingEnabled were stored/toggled/persisted but never enforced anywhere — a paused account could still send unboundedly. Added ErrAccountSendingPaused -> AccountSendingPausedException (exact code, verified against aws-sdk-go-v2/service/ses/types/errors.go), enforced in checkSendingAllowedLocked (shared by SendEmail/SendTemplatedEmail/SendRawEmail/SendBulkTemplatedEmail)."} + send_quota_enforcement: {status: ok, note: "GetSendQuota advertised Max24HourSend=200 (the real AWS SES sandbox default) but nothing enforced it — accounts could send arbitrarily many emails despite reporting a 200/24h cap. Added enforcement in checkSendingAllowedLocked using the same sentLast24HoursLocked() counting logic GetSendQuota already used (factored out, not duplicated). MaxSendRate (per-second) remains unenforced — deferred, bd: gopherstack-a6y."} + config_set_existence_validation: {status: ok, note: "ConfigurationSetName was accepted and stored on the Email record by SendEmail/SendTemplatedEmail/SendBulkTemplatedEmail but never validated to exist — real AWS SES returns ConfigurationSetDoesNotExist for an unknown name. Fixed via the same checkSendingAllowedLocked helper; SendBulkTemplatedEmail additionally gained ReplyToAddresses/ReturnPath/SourceArn plumbing it was silently dropping."} + identity_dkim_notification_ops: {status: ok, note: "reviewed GetIdentity{Dkim,MailFromDomain,Notification}Attributes + Set* counterparts; all read/write real per-identity state under the coarse lock; no stubs found beyond the BehaviorOnMXFailure gap (fixed)."} + receipt_rules_filters: {status: ok, note: "reviewed CRUD + ordering (CreateReceiptRule after=, ReorderReceiptRuleSet, SetReceiptRulePosition) — real slice manipulation, deep-copies on read to prevent aliasing, verified index math by tracing each function; no bugs found."} + templates_rendering: {status: ok, note: "{{key}} substitution verified against templated_render_test.go table cases; TemplateData JSON parse errors correctly surfaced as InvalidParameterValue; SendBulkTemplatedEmail default/replacement merge semantics verified (replacement overrides default per-destination)."} + persistence_janitor: {status: ok, note: "Snapshot/Restore cover every map (identities, templates, configSets, receiptRuleSets, receiptFilters, eventDestinations, trackingOptions, customVerifTemplates, policies) with correct deep-copies; Restore re-applies the TTL/maxRetainedEmails bound immediately; janitor uses pkgs/worker ticker + lockmetrics coarse lock; no goroutine or map leaks found."} +gaps: # known divergences NOT fixed — link bd issue ids + - "GetSendStatistics Bounces/Complaints/Rejects always report 0 — no bounce/complaint event simulation exists in this backend (bd: gopherstack-uve)" + - "LimitExceededException never returned — no per-resource count caps modeled (max receipt rules/templates/filters etc.) (bd: gopherstack-ssk)" + - "MailFromDomainNotVerifiedException never triggers — SetIdentityMailFromDomain instantly marks Success, consistent with this service's instant-verify convention everywhere else (VerifyEmailIdentity/VerifyDomainIdentity/VerifyDomainDkim all skip the real Pending window too); deliberately not changed to avoid an inconsistent one-off Pending state (bd: gopherstack-nbp)" + - "MaxSendRate (per-second) advertised via GetSendQuota but not enforced, only the 24h quota is now enforced (bd: gopherstack-a6y)" +deferred: # consciously not audited this pass (scope) — next pass targets + - "services/sesv2/ — separate REST-JSON service, out of scope this pass per task constraints (bd: gopherstack-029)" +leaks: {status: clean, note: "janitor sweep uses pkgs/worker.Group ticker with proper ctx cancellation via WithJanitor/StartWorker/Shutdown; sweepExpiredEmails is O(k) amortized (slice prefix trim, not full rescan); emailsByID map kept in sync on every eviction path (appendEmailLocked cap-eviction, sweepExpiredEmails, Restore pruning); maxRetainedEmails (10000) bounds the emails slice; no unbounded identity/template/config-set/receipt-rule maps found (all are keyed by caller-supplied names with no synthetic churn); no goroutines leaked outside the single janitor ticker."} +--- + +## Notes + +**Protocol**: SES v1, AWS query-XML protocol (`Version=2010-12-01`, `Action=` form-encoded +POST, XML response). A separate `services/sesv2/` exists implementing the modern SESv2 +REST-JSON API — that is a **different wire protocol and separate audit surface**, deliberately +out of scope this pass (see `deferred`, bd: gopherstack-029). + +**The `emptyResponse` XMLName trick is legitimate, not a bug pattern to flag.** Both the outer +`emptyResponse.XMLName xml.Name` (no tag) and the new `emptyResult.XMLName xml.Name` (no tag) +rely on a real `encoding/xml` behavior: when a struct's `XMLName xml.Name` field has **no** +tag (or an empty tag string), `xml.Marshal` uses the runtime value stored in that field as the +element name, rather than requiring a hardcoded tag. This is different from the bug class +called out in `.claude/memories/parity-principles.md` ("a hardcoded `xml:"StubResponse"` +XMLName tag silently overriding every runtime root") — that bug is a **non-empty** literal tag +clobbering the runtime value; an **empty** tag correctly defers to the runtime value. Don't +re-flag `newEmptyResponseWithResult`/`newEmptyResponse` as suspicious; they were specifically +verified against `aws-sdk-go-v2/service/ses/deserializers.go` (see `families.void_result_ops`). + +**Which void-result ops need a `Result` wrapper vs. none at all is NOT a style choice +— it's determined by the real AWS SES service model.** Ops whose output shape has at least a +theoretical member (even if none are ever populated in this emulator) always get a +`Result>` wrapper in the real wire format, and the generated SDK client's +deserializer explicitly calls `decoder.GetElement("Result")`. Ops whose output shape +has **zero members at all** skip body parsing entirely on the client (`io.Copy(io.Discard, +response.Body)`), and real AWS omits the wrapper. The 6 no-wrapper ops found this pass +(`VerifyEmailAddress`, `DeleteVerifiedEmailAddress`, `UpdateAccountSendingEnabled`, +`UpdateCustomVerificationEmailTemplate`, `UpdateConfigurationSetReputationMetricsEnabled`, +`UpdateConfigurationSetSendingEnabled`) were confirmed by reading the corresponding +`awsAwsquery_deserializeOp` function bodies in the SDK, not guessed from symmetry. +If a new void-result op is added, check the SDK deserializer for a `GetElement(...)` call +before assuming a wrapper is needed. + +**Instant-verification is a deliberate, consistent design convention across this entire +service**, not a bug: `VerifyEmailIdentity`, `VerifyDomainIdentity`, `VerifyDomainDkim`, +`SetIdentityMailFromDomain`, and (as of this pass) `SendCustomVerificationEmail` all mark +their target `Success`/`Verified` immediately rather than modeling AWS's real +Pending-until-DNS/click-through window. Don't "fix" any one of these in isolation to add a +Pending state — that would make it *inconsistent* with the rest of the service. If a future +pass wants real Pending-window simulation, it should be applied uniformly across all of +these, as a deliberate feature, not a bug fix. + +**AWS SES sandbox defaults are already correct constants in this codebase**: +`maxSendQuota24Hours = 200` and `maxSendRate = 1` match the real default SES sandbox +quota/rate for a fresh account. This pass didn't invent numbers to enforce — it wired the +*already-correct* advertised quota value into actual enforcement. + +**Test-only `AppendEmailForTest` helper** (`export_test.go`) was added because the new 24h +send-quota enforcement is incompatible with the pre-existing retention/eviction-cap tests +that send `maxRetainedEmails+N` (10000+) emails through `SendEmail` in a loop to exercise +`appendEmailLocked`'s eviction path — a real account would never accumulate 10000 sends +within a 200/day quota. `AppendEmailForTest` calls the same internal `appendEmailLocked` path +`SendEmail` uses (so eviction/O(1)-map-sync is still exercised for real) while bypassing the +business-rule preconditions (verification, quota, account-paused) that are orthogonal to what +those specific tests assert. Don't be alarmed seeing it used for high-volume tests instead of +`SendEmail` — that's intentional now, not a regression. diff --git a/services/ses/backend.go b/services/ses/backend.go index 07ed12212..58b8d5ec0 100644 --- a/services/ses/backend.go +++ b/services/ses/backend.go @@ -113,6 +113,10 @@ var ( ErrCustomVerifTemplateNotFound = errors.New("CustomVerificationEmailTemplateDoesNotExist") ErrCustomVerifTemplateExists = errors.New("CustomVerificationEmailTemplateAlreadyExists") ErrValidation = errors.New("ValidationError") + // ErrAccountSendingPaused is returned by send operations when account-level + // sending has been paused via UpdateAccountSendingEnabled(false), matching + // real AWS SES's AccountSendingPausedException. + ErrAccountSendingPaused = errors.New("AccountSendingPausedException") ) // maxRetainedEmails is the maximum number of sent emails retained in memory. @@ -450,6 +454,37 @@ func (b *InMemoryBackend) isVerifiedLocked(from string) bool { return false } +// checkSendingAllowedLocked validates the account-level, quota, and +// configuration-set preconditions shared by every send operation (SendEmail, +// SendRawEmail via SendEmail, SendTemplatedEmail, SendBulkTemplatedEmail): +// sending must not be paused account-wide, matching real AWS SES's +// AccountSendingPausedException; the simulated 24-hour send quota +// (GetSendQuota's Max24HourSend) must not already be exhausted, matching +// MessageRejected; and a non-empty ConfigurationSetName must reference an +// existing configuration set, matching ConfigurationSetDoesNotExist. +// +// The caller MUST hold b.mu for writing. +func (b *InMemoryBackend) checkSendingAllowedLocked(configurationSetName string) error { + if !b.accountSendingEnabled { + return fmt.Errorf("%w: account-level sending is currently paused", ErrAccountSendingPaused) + } + + if b.sentLast24HoursLocked() >= maxSendQuota24Hours { + return fmt.Errorf( + "%w: 24-hour sending quota of %d messages exceeded", + ErrMessageRejected, maxSendQuota24Hours, + ) + } + + if configurationSetName != "" { + if _, exists := b.configSets[configurationSetName]; !exists { + return fmt.Errorf("%w: %s", ErrConfigSetNotFound, configurationSetName) + } + } + + return nil +} + // appendEmailLocked appends e to the slice and O(1) map, evicting the oldest // entries when the cap is exceeded. // @@ -499,6 +534,10 @@ func (b *InMemoryBackend) SendEmail(in SendEmailInput) (string, error) { b.mu.Lock("SendEmail") defer b.mu.Unlock() + if err := b.checkSendingAllowedLocked(in.ConfigurationSetName); err != nil { + return "", err + } + if !b.isVerifiedLocked(in.From) { return "", fmt.Errorf( "%w: Email address is not verified. The following identities failed the check in region %s: %s", @@ -553,6 +592,10 @@ func (b *InMemoryBackend) SendTemplatedEmail(in SendTemplatedEmailInput) (string b.mu.Lock("SendTemplatedEmail") defer b.mu.Unlock() + if sendErr := b.checkSendingAllowedLocked(in.ConfigurationSetName); sendErr != nil { + return "", sendErr + } + if !b.isVerifiedLocked(in.From) { return "", fmt.Errorf( "%w: Email address is not verified. The following identities failed the check in region %s: %s", @@ -757,12 +800,12 @@ type SendQuota struct { SentLast24Hours float64 } -// GetSendQuota returns simulated quota values. -// SentLast24Hours counts only emails sent within the past 24 hours. -func (b *InMemoryBackend) GetSendQuota() SendQuota { - b.mu.RLock("GetSendQuota") - defer b.mu.RUnlock() - +// sentLast24HoursLocked returns the count of emails sent within the past 24 +// hours. b.emails is append-ordered by increasing Timestamp, so iterating +// backward lets us stop at the first entry older than the cutoff. +// +// The caller MUST hold b.mu for reading or writing. +func (b *InMemoryBackend) sentLast24HoursLocked() int { cutoff := time.Now().UTC().Add(-24 * time.Hour) sent := 0 @@ -774,10 +817,19 @@ func (b *InMemoryBackend) GetSendQuota() SendQuota { sent++ } + return sent +} + +// GetSendQuota returns simulated quota values. +// SentLast24Hours counts only emails sent within the past 24 hours. +func (b *InMemoryBackend) GetSendQuota() SendQuota { + b.mu.RLock("GetSendQuota") + defer b.mu.RUnlock() + return SendQuota{ Max24HourSend: maxSendQuota24Hours, MaxSendRate: maxSendRate, - SentLast24Hours: float64(sent), + SentLast24Hours: float64(b.sentLast24HoursLocked()), } } diff --git a/services/ses/coverage_boost_test.go b/services/ses/coverage_boost_test.go index 2f03bcb65..da7c98553 100644 --- a/services/ses/coverage_boost_test.go +++ b/services/ses/coverage_boost_test.go @@ -312,6 +312,7 @@ func TestHandler_SendBounce_Errors(t *testing.T) { t.Parallel() tests := []struct { + setup func(t *testing.T, h *ses.Handler) name string body string wantContains string @@ -324,10 +325,38 @@ func TestHandler_SendBounce_Errors(t *testing.T) { wantContains: "InvalidParameterValue", }, { - name: "valid_original_message_id", - body: "Action=SendBounce&Version=2010-12-01&OriginalMessageId=msg-123", + // AWS SES models BounceSender as a required SendBounceInput member. + name: "missing_bounce_sender", + body: "Action=SendBounce&Version=2010-12-01&OriginalMessageId=msg-123&" + + "BouncedRecipientInfoList.member.1.Recipient=to@example.com", + wantCode: http.StatusBadRequest, + wantContains: "InvalidParameterValue", + }, + { + // AWS SES models BouncedRecipientInfoList as a required, non-empty member. + name: "missing_recipient_list", + body: "Action=SendBounce&Version=2010-12-01&OriginalMessageId=msg-123&BounceSender=bounce@example.com", + wantCode: http.StatusBadRequest, + wantContains: "InvalidParameterValue", + }, + { + // BounceSender must be a verified identity, matching SendEmail's rule. + name: "unverified_bounce_sender", + body: "Action=SendBounce&Version=2010-12-01&OriginalMessageId=msg-123&BounceSender=bounce@example.com&" + + "BouncedRecipientInfoList.member.1.Recipient=to@example.com", + wantCode: http.StatusBadRequest, + wantContains: "MessageRejected", + }, + { + name: "valid_original_message_id", + body: "Action=SendBounce&Version=2010-12-01&OriginalMessageId=msg-123&BounceSender=bounce@example.com&" + + "BouncedRecipientInfoList.member.1.Recipient=to@example.com", wantCode: http.StatusOK, wantContains: "SendBounceResponse", + setup: func(t *testing.T, h *ses.Handler) { + t.Helper() + require.NoError(t, h.Backend.VerifyEmailIdentity("bounce@example.com")) + }, }, } @@ -336,6 +365,10 @@ func TestHandler_SendBounce_Errors(t *testing.T) { t.Parallel() h := newHandler() + if tt.setup != nil { + tt.setup(t, h) + } + rec := postForm(t, h, tt.body) assert.Equal(t, tt.wantCode, rec.Code) assert.Contains(t, rec.Body.String(), tt.wantContains) @@ -351,6 +384,7 @@ func TestHandler_SendCustomVerificationEmail_Errors(t *testing.T) { t.Parallel() tests := []struct { + setup func(t *testing.T, h *ses.Handler) name string body string wantContains string @@ -368,11 +402,30 @@ func TestHandler_SendCustomVerificationEmail_Errors(t *testing.T) { wantCode: http.StatusBadRequest, wantContains: "InvalidParameterValue", }, + { + // AWS SES requires the custom verification email template to already + // exist (CustomVerificationEmailTemplateDoesNotExist otherwise). + name: "template_not_found", + body: "Action=SendCustomVerificationEmail&Version=2010-12-01&EmailAddress=user@example.com&TemplateName=MyTemplate", //nolint:lll // existing issue. + wantCode: http.StatusBadRequest, + wantContains: "CustomVerificationEmailTemplateDoesNotExist", + }, { name: "valid_request", body: "Action=SendCustomVerificationEmail&Version=2010-12-01&EmailAddress=user@example.com&TemplateName=MyTemplate", //nolint:lll // existing issue. wantCode: http.StatusOK, wantContains: "SendCustomVerificationEmailResponse", + setup: func(t *testing.T, h *ses.Handler) { + t.Helper() + require.NoError(t, h.Backend.CreateCustomVerificationEmailTemplate(ses.CustomVerificationEmailTemplate{ + TemplateName: "MyTemplate", + FromEmailAddress: "noreply@example.com", + TemplateSubject: "Verify your email", + TemplateContent: "

Click here

", + SuccessRedirectionURL: "https://example.com/success", + FailureRedirectionURL: "https://example.com/failure", + })) + }, }, } @@ -381,6 +434,10 @@ func TestHandler_SendCustomVerificationEmail_Errors(t *testing.T) { t.Parallel() h := newHandler() + if tt.setup != nil { + tt.setup(t, h) + } + rec := postForm(t, h, tt.body) assert.Equal(t, tt.wantCode, rec.Code) assert.Contains(t, rec.Body.String(), tt.wantContains) diff --git a/services/ses/export_test.go b/services/ses/export_test.go index e8ef9ad4f..908690d17 100644 --- a/services/ses/export_test.go +++ b/services/ses/export_test.go @@ -1,6 +1,10 @@ package ses -import "time" +import ( + "time" + + "github.com/google/uuid" +) // DefaultJanitorInterval exposes the package default janitor interval for testing. const DefaultJanitorInterval = defaultSESJanitorInterval @@ -212,3 +216,25 @@ func (b *InMemoryBackend) AccountSendingEnabledState() bool { return b.accountSendingEnabled } + +// AppendEmailForTest appends an email directly via the same appendEmailLocked +// path SendEmail uses internally — including the maxRetainedEmails eviction +// behavior — but bypasses SendEmail's business-rule preconditions (sender +// verification, the simulated 24-hour send quota, account-sending-enabled). +// Used by volume/retention tests that need far more than +// maxSendQuota24Hours (200) sends, which real SendEmail now rejects with +// MessageRejected once exhausted. +func (b *InMemoryBackend) AppendEmailForTest(from string, to []string) string { + b.mu.Lock("AppendEmailForTest") + defer b.mu.Unlock() + + msgID := "ses-test-" + uuid.New().String() + b.appendEmailLocked(Email{ + MessageID: msgID, + From: from, + To: to, + Timestamp: time.Now(), + }) + + return msgID +} diff --git a/services/ses/handler.go b/services/ses/handler.go index e6d34fd6e..2ee1191b6 100644 --- a/services/ses/handler.go +++ b/services/ses/handler.go @@ -995,6 +995,8 @@ func sesErrorCode(opErr error) (string, int) { return "ConfigurationSetDoesNotExist", status case errors.Is(opErr, ErrConfigSetExists): return "ConfigurationSetAlreadyExists", status + case errors.Is(opErr, ErrAccountSendingPaused): + return "AccountSendingPausedException", status } return sesNewOpsErrorCode(opErr, status) @@ -2045,7 +2047,7 @@ func (h *Handler) handlePutIdentityPolicy(vals url.Values, reqID string) (any, e return nil, err } - return &emptyResponse{XMLName: xml.Name{Local: "PutIdentityPolicyResponse"}, Xmlns: sesXMLNS, RequestID: reqID}, nil + return newEmptyResponseWithResult("PutIdentityPolicy", reqID), nil } func (h *Handler) handleDeleteIdentityPolicy(vals url.Values, reqID string) (any, error) { @@ -2053,11 +2055,7 @@ func (h *Handler) handleDeleteIdentityPolicy(vals url.Values, reqID string) (any return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "DeleteIdentityPolicyResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponseWithResult("DeleteIdentityPolicy", reqID), nil } func (h *Handler) handleGetIdentityPolicies(vals url.Values, reqID string) (any, error) { @@ -2167,11 +2165,7 @@ func (h *Handler) handleSetIdentityDkimEnabled(vals url.Values, reqID string) (a return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "SetIdentityDkimEnabledResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponseWithResult("SetIdentityDkimEnabled", reqID), nil } func (h *Handler) handleSetIdentityFeedbackForwardingEnabled(vals url.Values, reqID string) (any, error) { @@ -2180,11 +2174,7 @@ func (h *Handler) handleSetIdentityFeedbackForwardingEnabled(vals url.Values, re return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "SetIdentityFeedbackForwardingEnabledResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponseWithResult("SetIdentityFeedbackForwardingEnabled", reqID), nil } func (h *Handler) handleSetIdentityHeadersInNotificationsEnabled(vals url.Values, reqID string) (any, error) { @@ -2197,23 +2187,19 @@ func (h *Handler) handleSetIdentityHeadersInNotificationsEnabled(vals url.Values return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "SetIdentityHeadersInNotificationsEnabledResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponseWithResult("SetIdentityHeadersInNotificationsEnabled", reqID), nil } func (h *Handler) handleSetIdentityMailFromDomain(vals url.Values, reqID string) (any, error) { - if err := h.Backend.SetIdentityMailFromDomain(vals.Get("Identity"), vals.Get("MailFromDomain")); err != nil { + if err := h.Backend.SetIdentityMailFromDomain( + vals.Get("Identity"), + vals.Get("MailFromDomain"), + vals.Get("BehaviorOnMXFailure"), + ); err != nil { return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "SetIdentityMailFromDomainResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponseWithResult("SetIdentityMailFromDomain", reqID), nil } func (h *Handler) handleSetIdentityNotificationTopic(vals url.Values, reqID string) (any, error) { @@ -2225,11 +2211,7 @@ func (h *Handler) handleSetIdentityNotificationTopic(vals url.Values, reqID stri return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "SetIdentityNotificationTopicResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponseWithResult("SetIdentityNotificationTopic", reqID), nil } func (h *Handler) handleVerifyDomainIdentity(vals url.Values, reqID string) (any, error) { @@ -2269,21 +2251,13 @@ func (h *Handler) handleVerifyEmailAddress(vals url.Values, reqID string) (any, return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "VerifyEmailAddressResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponse("VerifyEmailAddress", reqID), nil } func (h *Handler) handleDeleteVerifiedEmailAddress(vals url.Values, reqID string) any { h.Backend.DeleteVerifiedEmailAddress(vals.Get("EmailAddress")) - return &emptyResponse{ - XMLName: xml.Name{Local: "DeleteVerifiedEmailAddressResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - } + return newEmptyResponse("DeleteVerifiedEmailAddress", reqID) } func (h *Handler) handleListVerifiedEmailAddresses(reqID string) any { @@ -2305,15 +2279,13 @@ func (h *Handler) handleUpdateAccountSendingEnabled(vals url.Values, reqID strin enabled := vals.Get("Enabled") == boolTrue h.Backend.UpdateAccountSendingEnabled(enabled) - return &emptyResponse{ - XMLName: xml.Name{Local: "UpdateAccountSendingEnabledResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - } + return newEmptyResponse("UpdateAccountSendingEnabled", reqID) } func (h *Handler) handleSendBounce(vals url.Values, reqID string) (any, error) { - msgID, err := h.Backend.SendBounce(vals.Get("OriginalMessageId")) + recipients := parseBouncedRecipients(vals, "BouncedRecipientInfoList") + + msgID, err := h.Backend.SendBounce(vals.Get("OriginalMessageId"), vals.Get("BounceSender"), recipients) if err != nil { return nil, err } @@ -2325,10 +2297,30 @@ func (h *Handler) handleSendBounce(vals url.Values, reqID string) (any, error) { }, nil } +// parseBouncedRecipients parses ".member.N.Recipient" form values into +// a flat list of bounced recipient email addresses. +func parseBouncedRecipients(vals url.Values, prefix string) []string { + var recipients []string + base := prefix + ".member." + + for i := 1; ; i++ { + v := vals.Get(base + strconv.Itoa(i) + ".Recipient") + if v == "" { + return recipients + } + + recipients = append(recipients, v) + } +} + func (h *Handler) handleSendBulkTemplatedEmail(vals url.Values, reqID string) (any, error) { source := vals.Get("Source") template := vals.Get("Template") defaultTemplateData := vals.Get("DefaultTemplateData") + configSetName := vals.Get("ConfigurationSetName") + returnPath := vals.Get("ReturnPath") + sourceArn := vals.Get("SourceArn") + replyTo := parseSESMemberList(vals, "ReplyToAddresses") // Collect per-destination data. var destinations []BulkEmailDestination @@ -2358,7 +2350,9 @@ func (h *Handler) handleSendBulkTemplatedEmail(vals url.Values, reqID string) (a ErrInvalidParameter, len(destinations), maxBulkDestinations) } - msgIDs, err := h.Backend.SendBulkTemplatedEmail(source, template, defaultTemplateData, destinations) + msgIDs, err := h.Backend.SendBulkTemplatedEmail( + source, template, defaultTemplateData, configSetName, returnPath, sourceArn, replyTo, destinations, + ) if err != nil { return nil, err } @@ -2376,7 +2370,11 @@ func (h *Handler) handleSendBulkTemplatedEmail(vals url.Values, reqID string) (a } func (h *Handler) handleSendCustomVerificationEmail(vals url.Values, reqID string) (any, error) { - msgID, err := h.Backend.SendCustomVerificationEmail(vals.Get("EmailAddress"), vals.Get("TemplateName")) + msgID, err := h.Backend.SendCustomVerificationEmail( + vals.Get("EmailAddress"), + vals.Get("TemplateName"), + vals.Get("ConfigurationSetName"), + ) if err != nil { return nil, err } @@ -2415,11 +2413,7 @@ func (h *Handler) handleUpdateCustomVerificationEmailTemplate(vals url.Values, r return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "UpdateCustomVerificationEmailTemplateResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponse("UpdateCustomVerificationEmailTemplate", reqID), nil } func (h *Handler) handleDescribeReceiptRule(vals url.Values, reqID string) (any, error) { @@ -2453,11 +2447,7 @@ func (h *Handler) handleUpdateReceiptRule(vals url.Values, reqID string) (any, e return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "UpdateReceiptRuleResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponseWithResult("UpdateReceiptRule", reqID), nil } func (h *Handler) handleReorderReceiptRuleSet(vals url.Values, reqID string) (any, error) { @@ -2468,11 +2458,7 @@ func (h *Handler) handleReorderReceiptRuleSet(vals url.Values, reqID string) (an return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "ReorderReceiptRuleSetResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponseWithResult("ReorderReceiptRuleSet", reqID), nil } func (h *Handler) handleSetReceiptRulePosition(vals url.Values, reqID string) (any, error) { @@ -2490,11 +2476,7 @@ func (h *Handler) handleSetReceiptRulePosition(vals url.Values, reqID string) (a return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "SetReceiptRulePositionResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponseWithResult("SetReceiptRulePosition", reqID), nil } func (h *Handler) handleDescribeConfigurationSet(vals url.Values, reqID string) (any, error) { @@ -2550,11 +2532,7 @@ func (h *Handler) handlePutConfigurationSetDeliveryOptions(vals url.Values, reqI return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "PutConfigurationSetDeliveryOptionsResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponseWithResult("PutConfigurationSetDeliveryOptions", reqID), nil } func (h *Handler) handleUpdateConfigurationSetEventDestination(vals url.Values, reqID string) (any, error) { @@ -2569,11 +2547,7 @@ func (h *Handler) handleUpdateConfigurationSetEventDestination(vals url.Values, return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "UpdateConfigurationSetEventDestinationResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponseWithResult("UpdateConfigurationSetEventDestination", reqID), nil } func (h *Handler) handleUpdateConfigurationSetReputationMetricsEnabled(vals url.Values, reqID string) (any, error) { @@ -2584,11 +2558,7 @@ func (h *Handler) handleUpdateConfigurationSetReputationMetricsEnabled(vals url. return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "UpdateConfigurationSetReputationMetricsEnabledResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponse("UpdateConfigurationSetReputationMetricsEnabled", reqID), nil } func (h *Handler) handleUpdateConfigurationSetSendingEnabled(vals url.Values, reqID string) (any, error) { @@ -2597,11 +2567,7 @@ func (h *Handler) handleUpdateConfigurationSetSendingEnabled(vals url.Values, re return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "UpdateConfigurationSetSendingEnabledResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponse("UpdateConfigurationSetSendingEnabled", reqID), nil } func (h *Handler) handleUpdateConfigurationSetTrackingOptions(vals url.Values, reqID string) (any, error) { @@ -2612,21 +2578,52 @@ func (h *Handler) handleUpdateConfigurationSetTrackingOptions(vals url.Values, r return nil, err } - return &emptyResponse{ - XMLName: xml.Name{Local: "UpdateConfigurationSetTrackingOptionsResponse"}, - Xmlns: sesXMLNS, - RequestID: reqID, - }, nil + return newEmptyResponseWithResult("UpdateConfigurationSetTrackingOptions", reqID), nil } // ---- missing ops: XML types ---- +// emptyResult carries the dynamic "Result" wrapper element name for +// void-result operations whose SES wire format wraps an (empty) Result element. +// The XMLName field's tag is intentionally blank so xml.Marshal uses the +// runtime value set by the caller rather than a fixed literal name. +type emptyResult struct { + XMLName xml.Name +} + // emptyResponse is a generic empty-result XML envelope used by no-op operations. +// Result is nil for actions whose SES output shape has zero members, so the +// real wire format omits the Result element entirely (e.g. VerifyEmailAddress). +// Otherwise it carries the per-op "Result" element name: real AWS SDK +// clients call decoder.GetElement("Result") before parsing the body, so +// a missing or misnamed wrapper causes a client-side DeserializationError even +// though the emulator's backend state mutation succeeded. type emptyResponse struct { - XMLName xml.Name `xml:""` - Xmlns string `xml:"xmlns,attr"` - Result struct{} `xml:"*Result"` - RequestID string `xml:"ResponseMetadata>RequestId"` + XMLName xml.Name + Xmlns string `xml:"xmlns,attr"` + Result *emptyResult `xml:",omitempty"` + RequestID string `xml:"ResponseMetadata>RequestId"` +} + +// newEmptyResponseWithResult builds an emptyResponse for an action whose real +// SES output shape wraps an (empty) "Result" element. +func newEmptyResponseWithResult(action, reqID string) *emptyResponse { + return &emptyResponse{ + XMLName: xml.Name{Local: action + "Response"}, + Xmlns: sesXMLNS, + Result: &emptyResult{XMLName: xml.Name{Local: action + "Result"}}, + RequestID: reqID, + } +} + +// newEmptyResponse builds an emptyResponse for an action whose real SES output +// shape has zero members, so the wire format has no Result element at all. +func newEmptyResponse(action, reqID string) *emptyResponse { + return &emptyResponse{ + XMLName: xml.Name{Local: action + "Response"}, + Xmlns: sesXMLNS, + RequestID: reqID, + } } type xmlPolicyEntry struct { diff --git a/services/ses/handler_accuracy_batch1_test.go b/services/ses/handler_accuracy_batch1_test.go index 48362aa86..223e98c09 100644 --- a/services/ses/handler_accuracy_batch1_test.go +++ b/services/ses/handler_accuracy_batch1_test.go @@ -322,8 +322,8 @@ func TestBatch1_SetIdentityMailFromDomain_Clear(t *testing.T) { b := ses.NewInMemoryBackend() require.NoError(t, b.VerifyEmailIdentity("mf@example.com")) - require.NoError(t, b.SetIdentityMailFromDomain("mf@example.com", "mail.example.com")) - require.NoError(t, b.SetIdentityMailFromDomain("mf@example.com", "")) + require.NoError(t, b.SetIdentityMailFromDomain("mf@example.com", "mail.example.com", "")) + require.NoError(t, b.SetIdentityMailFromDomain("mf@example.com", "", "")) attrs := b.GetIdentityMailFromDomainAttributes([]string{"mf@example.com"}) assert.Empty(t, attrs["mf@example.com"].MailFromDomain) @@ -335,7 +335,7 @@ func TestBatch1_GetIdentityMailFromDomainAttributes_Handler(t *testing.T) { h := newHandler() require.NoError(t, h.Backend.VerifyEmailIdentity("mf2@example.com")) - require.NoError(t, h.Backend.SetIdentityMailFromDomain("mf2@example.com", "bounce.example.com")) + require.NoError(t, h.Backend.SetIdentityMailFromDomain("mf2@example.com", "bounce.example.com", "")) rec := postForm(t, h, url.Values{ "Action": {"GetIdentityMailFromDomainAttributes"}, @@ -945,7 +945,7 @@ func TestBatch1_SendBulkTemplatedEmail_PerDestinationData(t *testing.T) { {To: []string{"b@example.com"}, Cc: []string{"cc@example.com"}}, } - ids, err := b.SendBulkTemplatedEmail("s@example.com", "t", "", dests) + ids, err := b.SendBulkTemplatedEmail("s@example.com", "t", "", "", "", "", nil, dests) require.NoError(t, err) assert.Len(t, ids, 2) @@ -969,17 +969,57 @@ func TestBatch1_SendBounce_Handler(t *testing.T) { }) require.NoError(t, err) + // AWS SES requires BounceSender to be a verified identity and + // BouncedRecipientInfoList to contain at least one recipient entry. + require.NoError(t, h.Backend.VerifyEmailIdentity("mailer-daemon@example.com")) + rec := postForm(t, h, url.Values{ - "Action": {"SendBounce"}, - "Version": {"2010-12-01"}, - "OriginalMessageId": {msgID}, - "BounceSender": {"mailer-daemon@example.com"}, - "MessageDsn.ReportingMta": {"dns; example.com"}, + "Action": {"SendBounce"}, + "Version": {"2010-12-01"}, + "OriginalMessageId": {msgID}, + "BounceSender": {"mailer-daemon@example.com"}, + "BouncedRecipientInfoList.member.1.Recipient": {"to@example.com"}, + "MessageDsn.ReportingMta": {"dns; example.com"}, }.Encode()) assert.Equal(t, http.StatusOK, rec.Code) assert.Contains(t, rec.Body.String(), "SendBounceResponse") } +func TestBatch1_SendBounce_Handler_RequiresBounceSenderAndRecipients(t *testing.T) { + t.Parallel() + + h := newHandler() + require.NoError(t, h.Backend.VerifyEmailIdentity("s@example.com")) + msgID, err := h.Backend.SendEmail(ses.SendEmailInput{ + From: "s@example.com", + To: []string{"to@example.com"}, + Subject: "orig", + BodyText: "body", + }) + require.NoError(t, err) + + // Missing BounceSender entirely: AWS SES models it as a required member. + rec := postForm(t, h, url.Values{ + "Action": {"SendBounce"}, + "Version": {"2010-12-01"}, + "OriginalMessageId": {msgID}, + "BouncedRecipientInfoList.member.1.Recipient": {"to@example.com"}, + }.Encode()) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "InvalidParameterValue") + + // BounceSender present but not a verified identity: MessageRejected. + rec = postForm(t, h, url.Values{ + "Action": {"SendBounce"}, + "Version": {"2010-12-01"}, + "OriginalMessageId": {msgID}, + "BounceSender": {"unverified@example.com"}, + "BouncedRecipientInfoList.member.1.Recipient": {"to@example.com"}, + }.Encode()) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "MessageRejected") +} + // ---- SendCustomVerificationEmail ---- func TestBatch1_SendCustomVerificationEmail_Handler(t *testing.T) { @@ -2231,6 +2271,9 @@ func TestBatch1_SendEmail_Tags_ConfigSet(t *testing.T) { b := ses.NewInMemoryBackend() require.NoError(t, b.VerifyEmailIdentity("s@example.com")) + // AWS SES requires ConfigurationSetName to reference an existing configuration + // set (ConfigurationSetDoesNotExist otherwise), so create it up front. + require.NoError(t, b.CreateConfigurationSet("my-cs")) tags := []ses.Tag{{Name: "env", Value: "prod"}, {Name: "team", Value: "backend"}} msgID, err := b.SendEmail(ses.SendEmailInput{ diff --git a/services/ses/handler_accuracy_test.go b/services/ses/handler_accuracy_test.go index d08526aaa..364ca2c11 100644 --- a/services/ses/handler_accuracy_test.go +++ b/services/ses/handler_accuracy_test.go @@ -43,6 +43,9 @@ func TestSendEmail_CcBccReplyTo_Handler(t *testing.T) { h := newHandler() require.NoError(t, h.Backend.VerifyEmailIdentity("s@example.com")) + // AWS SES requires ConfigurationSetName to reference an existing configuration + // set (ConfigurationSetDoesNotExist otherwise), so create it up front. + require.NoError(t, h.Backend.CreateConfigurationSet("my-set")) body := url.Values{ "Action": {"SendEmail"}, @@ -177,6 +180,9 @@ func TestSendEmail_Tags(t *testing.T) { b := ses.NewInMemoryBackend() require.NoError(t, b.VerifyEmailIdentity("s@example.com")) + // AWS SES requires ConfigurationSetName to reference an existing configuration + // set (ConfigurationSetDoesNotExist otherwise), so create it up front. + require.NoError(t, b.CreateConfigurationSet("cs1")) tags := []ses.Tag{{Name: "k1", Value: "v1"}, {Name: "k2", Value: "v2"}} msgID, err := b.SendEmail(ses.SendEmailInput{ @@ -245,7 +251,7 @@ func TestSetIdentityMailFromDomain_Persists(t *testing.T) { b := ses.NewInMemoryBackend() require.NoError(t, b.VerifyEmailIdentity("a@example.com")) - require.NoError(t, b.SetIdentityMailFromDomain("a@example.com", "mail.example.com")) + require.NoError(t, b.SetIdentityMailFromDomain("a@example.com", "mail.example.com", "")) attrs := b.GetIdentityMailFromDomainAttributes([]string{"a@example.com"}) assert.Equal(t, "mail.example.com", attrs["a@example.com"].MailFromDomain) @@ -257,8 +263,8 @@ func TestSetIdentityMailFromDomain_Clear(t *testing.T) { b := ses.NewInMemoryBackend() require.NoError(t, b.VerifyEmailIdentity("a@example.com")) - require.NoError(t, b.SetIdentityMailFromDomain("a@example.com", "mail.example.com")) - require.NoError(t, b.SetIdentityMailFromDomain("a@example.com", "")) + require.NoError(t, b.SetIdentityMailFromDomain("a@example.com", "mail.example.com", "")) + require.NoError(t, b.SetIdentityMailFromDomain("a@example.com", "", "")) attrs := b.GetIdentityMailFromDomainAttributes([]string{"a@example.com"}) assert.Empty(t, attrs["a@example.com"].MailFromDomain) @@ -438,7 +444,7 @@ func TestSendBulkTemplatedEmail_PerDestination(t *testing.T) { {To: []string{"b@example.com"}, Cc: []string{"cc@example.com"}}, } - msgIDs, err := b.SendBulkTemplatedEmail("sender@example.com", "t", "", destinations) + msgIDs, err := b.SendBulkTemplatedEmail("sender@example.com", "t", "", "", "", "", nil, destinations) require.NoError(t, err) assert.Len(t, msgIDs, 2) @@ -506,7 +512,7 @@ func TestIdentityRecord_SnapshotRestore(t *testing.T) { b := ses.NewInMemoryBackend() require.NoError(t, b.VerifyEmailIdentity("a@example.com")) require.NoError(t, b.SetIdentityDkimEnabled("a@example.com", true)) - require.NoError(t, b.SetIdentityMailFromDomain("a@example.com", "mail.example.com")) + require.NoError(t, b.SetIdentityMailFromDomain("a@example.com", "mail.example.com", "")) require.NoError(t, b.SetIdentityNotificationTopic("a@example.com", "Bounce", "arn:topic")) snap := b.Snapshot(t.Context()) diff --git a/services/ses/handler_test.go b/services/ses/handler_test.go index eab2d5185..58d8e903c 100644 --- a/services/ses/handler_test.go +++ b/services/ses/handler_test.go @@ -429,13 +429,11 @@ func TestSESBackend_EmailRetentionLimit(t *testing.T) { b := ses.NewInMemoryBackend() require.NoError(t, b.VerifyEmailIdentity("sender@test.com")) - // Send more emails than the cap. - for i := range ses.MaxRetainedEmails + 100 { - _, err := b.SendEmail(ses.SendEmailInput{ - From: "sender@test.com", To: []string{"to@test.com"}, - Subject: fmt.Sprintf("Subject %d", i), BodyText: "body", - }) - require.NoError(t, err) + // Append more emails than the cap via AppendEmailForTest, which exercises + // the same eviction path as SendEmail without being gated by the + // simulated 200/day send quota that real SendEmail now enforces. + for range ses.MaxRetainedEmails + 100 { + b.AppendEmailForTest("sender@test.com", []string{"to@test.com"}) } assert.Equal(t, ses.MaxRetainedEmails, b.EmailCount()) diff --git a/services/ses/interfaces.go b/services/ses/interfaces.go index d63a1433c..8e5bfaf0a 100644 --- a/services/ses/interfaces.go +++ b/services/ses/interfaces.go @@ -70,7 +70,7 @@ type StorageBackend interface { SetIdentityDkimEnabled(identity string, enabled bool) error SetIdentityFeedbackForwardingEnabled(identity string, enabled bool) error SetIdentityHeadersInNotificationsEnabled(identity, notificationType string, enabled bool) error - SetIdentityMailFromDomain(identity, mailFromDomain string) error + SetIdentityMailFromDomain(identity, mailFromDomain, behaviorOnMXFailure string) error SetIdentityNotificationTopic(identity, notificationType, snsTopic string) error // Domain verification VerifyDomainIdentity(domain string) (string, error) @@ -82,12 +82,13 @@ type StorageBackend interface { UpdateAccountSendingEnabled(enabled bool) GetAccountSendingEnabled() bool // Send ops - SendBounce(originalMsgID string) (string, error) + SendBounce(originalMsgID, bounceSender string, recipients []string) (string, error) SendBulkTemplatedEmail( - source, templateName, defaultTemplateData string, + source, templateName, defaultTemplateData, configurationSetName, returnPath, sourceArn string, + replyTo []string, destinations []BulkEmailDestination, ) ([]string, error) - SendCustomVerificationEmail(email, templateName string) (string, error) + SendCustomVerificationEmail(email, templateName, configurationSetName string) (string, error) TestRenderTemplate(templateName, templateData string) (string, error) Region() string AccountID() string diff --git a/services/ses/missing_ops.go b/services/ses/missing_ops.go index 3572b0a1a..96b1449d3 100644 --- a/services/ses/missing_ops.go +++ b/services/ses/missing_ops.go @@ -9,6 +9,8 @@ import ( "sort" "strings" + "github.com/google/uuid" + "github.com/blackbirdworks/gopherstack/pkgs/collections" ) @@ -304,13 +306,35 @@ func (b *InMemoryBackend) SetIdentityHeadersInNotificationsEnabled( return nil } +// behaviorOnMXFailureUseDefault and behaviorOnMXFailureReject are the two legal +// values of the SetIdentityMailFromDomain BehaviorOnMXFailure parameter, +// matching the AWS SES BehaviorOnMXFailure enum. UseDefaultValue is the +// real-AWS default when the parameter is omitted. +const ( + behaviorOnMXFailureUseDefault = "UseDefaultValue" + behaviorOnMXFailureReject = "RejectMessage" +) + // SetIdentityMailFromDomain persists the custom MAIL FROM domain for an identity. -// An empty mailFromDomain clears the setting. -func (b *InMemoryBackend) SetIdentityMailFromDomain(identity, mailFromDomain string) error { +// An empty mailFromDomain clears the setting (and its BehaviorOnMXFailure). +// behaviorOnMXFailure must be "UseDefaultValue" or "RejectMessage"; an empty +// value defaults to "UseDefaultValue", matching real AWS SES. +func (b *InMemoryBackend) SetIdentityMailFromDomain(identity, mailFromDomain, behaviorOnMXFailure string) error { if strings.TrimSpace(identity) == "" { return fmt.Errorf("%w: Identity is required", ErrInvalidParameter) } + if behaviorOnMXFailure == "" { + behaviorOnMXFailure = behaviorOnMXFailureUseDefault + } + + if behaviorOnMXFailure != behaviorOnMXFailureUseDefault && behaviorOnMXFailure != behaviorOnMXFailureReject { + return fmt.Errorf( + "%w: BehaviorOnMXFailure must be %s or %s", + ErrInvalidParameter, behaviorOnMXFailureUseDefault, behaviorOnMXFailureReject, + ) + } + b.mu.Lock("SetIdentityMailFromDomain") defer b.mu.Unlock() @@ -319,8 +343,10 @@ func (b *InMemoryBackend) SetIdentityMailFromDomain(identity, mailFromDomain str if mailFromDomain == "" { rec.MailFromStatus = "" + rec.BehaviorOnMXFail = "" } else { rec.MailFromStatus = identityStatusSuccess + rec.BehaviorOnMXFail = behaviorOnMXFailure } return nil @@ -443,24 +469,50 @@ func (b *InMemoryBackend) GetAccountSendingEnabled() bool { return b.accountSendingEnabled } -// ---- send operations (stubs) ---- +// ---- send operations ---- -// SendBounce is a no-op stub that returns a synthetic bounce message ID. -func (b *InMemoryBackend) SendBounce(originalMsgID string) (string, error) { +// SendBounce generates and sends a bounce message for a previously received +// email. Real AWS SES models BounceSender and BouncedRecipientInfoList as +// required input members (SendBounceInput), so both must be supplied here; +// BounceSender must additionally be a verified identity (or a verified +// domain), matching the same sender-verification rule enforced by SendEmail. +func (b *InMemoryBackend) SendBounce(originalMsgID, bounceSender string, recipients []string) (string, error) { if strings.TrimSpace(originalMsgID) == "" { return "", fmt.Errorf("%w: OriginalMessageId is required", ErrInvalidParameter) } - return "bounce-" + originalMsgID, nil + if strings.TrimSpace(bounceSender) == "" { + return "", fmt.Errorf("%w: BounceSender is required", ErrInvalidParameter) + } + + if len(recipients) == 0 { + return "", fmt.Errorf("%w: BouncedRecipientInfoList must contain at least one entry", ErrInvalidParameter) + } + + b.mu.Lock("SendBounce") + defer b.mu.Unlock() + + if !b.isVerifiedLocked(bounceSender) { + return "", fmt.Errorf( + "%w: Email address is not verified. The following identities failed the check in region %s: %s", + ErrMessageRejected, strings.ToUpper(b.region), bounceSender, + ) + } + + return "ses-bounce-" + uuid.New().String(), nil } // SendBulkTemplatedEmail sends one email per destination and returns a message // ID for each. Each destination is rendered with the request-level // defaultTemplateData merged with that destination's ReplacementTemplateData, // matching AWS SES SendBulkTemplatedEmail semantics where replacement values -// override defaults on a per-recipient basis. +// override defaults on a per-recipient basis. configurationSetName, replyTo, +// returnPath and sourceArn mirror the corresponding SendBulkTemplatedEmailInput +// members and are threaded through to every generated Email record exactly as +// SendEmail/SendTemplatedEmail do for a single-destination send. func (b *InMemoryBackend) SendBulkTemplatedEmail( - source, templateName, defaultTemplateData string, + source, templateName, defaultTemplateData, configurationSetName, returnPath, sourceArn string, + replyTo []string, destinations []BulkEmailDestination, ) ([]string, error) { if strings.TrimSpace(source) == "" { @@ -478,6 +530,19 @@ func (b *InMemoryBackend) SendBulkTemplatedEmail( return nil, err } + // Validate the configuration set (when supplied) exists up front, matching + // the same ConfigurationSetDoesNotExist precondition enforced by SendEmail + // and SendTemplatedEmail. + if configurationSetName != "" { + b.mu.RLock("SendBulkTemplatedEmail") + _, exists := b.configSets[configurationSetName] + b.mu.RUnlock() + + if !exists { + return nil, fmt.Errorf("%w: %s", ErrConfigSetNotFound, configurationSetName) + } + } + msgIDs := make([]string, 0, len(destinations)) for _, d := range destinations { @@ -495,12 +560,16 @@ func (b *InMemoryBackend) SendBulkTemplatedEmail( } msgID, err := b.SendTemplatedEmail(SendTemplatedEmailInput{ - From: source, - To: d.To, - Cc: d.Cc, - Bcc: d.Bcc, - TemplateName: templateName, - TemplateData: string(mergedJSON), + From: source, + To: d.To, + Cc: d.Cc, + Bcc: d.Bcc, + ReplyTo: replyTo, + TemplateName: templateName, + TemplateData: string(mergedJSON), + ConfigurationSetName: configurationSetName, + ReturnPath: returnPath, + SourceArn: sourceArn, }) if err != nil { return nil, err @@ -512,8 +581,17 @@ func (b *InMemoryBackend) SendBulkTemplatedEmail( return msgIDs, nil } -// SendCustomVerificationEmail is a no-op stub. -func (b *InMemoryBackend) SendCustomVerificationEmail(email, templateName string) (string, error) { +// SendCustomVerificationEmail adds email to the account's identity list and +// attempts to verify it (matching this backend's instant-verification +// convention shared by VerifyEmailIdentity/VerifyEmailAddress) and sends a +// verification email using the named custom template. templateName must +// reference an existing custom verification email template +// (CreateCustomVerificationEmailTemplate), and configurationSetName, if +// supplied, must reference an existing configuration set — both required +// preconditions on the real AWS SES SendCustomVerificationEmailInput. +func (b *InMemoryBackend) SendCustomVerificationEmail( + email, templateName, configurationSetName string, +) (string, error) { if strings.TrimSpace(email) == "" { return "", fmt.Errorf("%w: EmailAddress is required", ErrInvalidParameter) } @@ -522,7 +600,29 @@ func (b *InMemoryBackend) SendCustomVerificationEmail(email, templateName string return "", fmt.Errorf("%w: TemplateName is required", ErrInvalidParameter) } - return "custom-verif-" + email, nil + if _, err := b.GetCustomVerificationEmailTemplate(templateName); err != nil { + return "", err + } + + if configurationSetName != "" { + b.mu.RLock("SendCustomVerificationEmail") + _, exists := b.configSets[configurationSetName] + b.mu.RUnlock() + + if !exists { + return "", fmt.Errorf("%w: %s", ErrConfigSetNotFound, configurationSetName) + } + } + + b.mu.Lock("SendCustomVerificationEmail") + if rec, ok := b.identities[email]; ok { + rec.Verified = true + } else { + b.identities[email] = &IdentityRecord{Verified: true, ForwardingEnabled: true} + } + b.mu.Unlock() + + return "ses-verif-" + uuid.New().String(), nil } // parseTemplateData parses the JSON template-data document into a flat diff --git a/services/ses/new_operations_test.go b/services/ses/new_operations_test.go index 08e3f8602..8efebe70d 100644 --- a/services/ses/new_operations_test.go +++ b/services/ses/new_operations_test.go @@ -39,14 +39,16 @@ func TestSESBackend_EmailsByIDSyncAfterEviction(t *testing.T) { b := ses.NewInMemoryBackend() require.NoError(t, b.VerifyEmailIdentity("x@test.com")) - // Send MaxRetainedEmails+5 so the first 5 are evicted. + // Append MaxRetainedEmails+5 so the first 5 are evicted. This exercises + // the eviction/O(1)-map-sync behavior in appendEmailLocked directly via + // AppendEmailForTest: real SendEmail now enforces the simulated 200/day + // send quota (matching real AWS SES sandbox default), so a real send loop + // of this volume would itself be rejected with MessageRejected long + // before reaching the retention cap. var firstIDs []string for i := range ses.MaxRetainedEmails + 5 { - msgID, err := b.SendEmail(ses.SendEmailInput{ - From: "x@test.com", To: []string{"y@test.com"}, Subject: "s", BodyText: "body", - }) - require.NoError(t, err) + msgID := b.AppendEmailForTest("x@test.com", []string{"y@test.com"}) if i < 5 { firstIDs = append(firstIDs, msgID) @@ -873,12 +875,11 @@ func TestSESPersistence_RestoreCapsToBound(t *testing.T) { original := ses.NewInMemoryBackend() require.NoError(t, original.VerifyEmailIdentity("cap@test.com")) - // Send MaxRetainedEmails+10 emails and snapshot. + // Append MaxRetainedEmails+10 emails and snapshot, via AppendEmailForTest + // so this retention-cap test isn't gated by the simulated 200/day send + // quota that real SendEmail now enforces (see TestSESBackend_EmailsByIDSyncAfterEviction). for range ses.MaxRetainedEmails + 10 { - _, err := original.SendEmail(ses.SendEmailInput{ - From: "cap@test.com", To: []string{"to@test.com"}, Subject: "s", BodyText: "b", - }) - require.NoError(t, err) + original.AppendEmailForTest("cap@test.com", []string{"to@test.com"}) } // The original should already be capped by SendEmail eviction. diff --git a/services/ses/parity_sweep3_test.go b/services/ses/parity_sweep3_test.go new file mode 100644 index 000000000..5c71355fc --- /dev/null +++ b/services/ses/parity_sweep3_test.go @@ -0,0 +1,376 @@ +package ses_test + +import ( + "net/http" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/ses" +) + +// --------------------------------------------------------------------------- +// Void-result operations must wrap an "Result" element (or omit it +// entirely for AWS-empty-output-shape ops), never a literal "*Result" element. +// A real AWS SDK client calls decoder.GetElement("Result") before +// parsing the body, so a wrong wrapper causes a client-side +// DeserializationError even though the emulator's backend state mutated +// correctly — see aws-sdk-go-v2/service/ses deserializers.go. +// --------------------------------------------------------------------------- + +func Test_VoidResultOps_WireShape(t *testing.T) { + t.Parallel() + + cases := []struct { + setup func(t *testing.T, h *ses.Handler) + name string + body string + wantElement string // must be present verbatim + }{ + { + name: "PutIdentityPolicy_has_result_wrapper", + body: url.Values{ + "Action": {"PutIdentityPolicy"}, + "Version": {"2010-12-01"}, + "Identity": {"example.com"}, + "PolicyName": {"p1"}, + "Policy": {"{}"}, + }.Encode(), + wantElement: "", + }, + { + name: "SetIdentityDkimEnabled_has_result_wrapper", + body: url.Values{ + "Action": {"SetIdentityDkimEnabled"}, + "Version": {"2010-12-01"}, + "Identity": {"example.com"}, + "DkimEnabled": {"true"}, + }.Encode(), + wantElement: "", + }, + { + name: "VerifyEmailAddress_has_no_result_wrapper", + body: url.Values{ + "Action": {"VerifyEmailAddress"}, + "Version": {"2010-12-01"}, + "EmailAddress": {"a@example.com"}, + }.Encode(), + // Real SES's VerifyEmailAddress output shape has zero members, so the + // wire format omits any Result element entirely. + wantElement: "" + + "", + }, + { + name: "UpdateAccountSendingEnabled_has_no_result_wrapper", + body: url.Values{ + "Action": {"UpdateAccountSendingEnabled"}, + "Version": {"2010-12-01"}, + "Enabled": {"false"}, + }.Encode(), + wantElement: "" + + "", + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newHandler() + if tt.setup != nil { + tt.setup(t, h) + } + + rec := postForm(t, h, tt.body) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), tt.wantElement) + // The literal broken wrapper must never appear. + assert.NotContains(t, rec.Body.String(), "*Result") + }) + } +} + +// --------------------------------------------------------------------------- +// AccountSendingPausedException: UpdateAccountSendingEnabled(false) must +// actually block subsequent Send* operations, matching real AWS SES. Before +// this fix the flag was stored/persisted but never enforced. +// --------------------------------------------------------------------------- + +func Test_AccountSendingPaused_BlocksSend(t *testing.T) { + t.Parallel() + + b := ses.NewInMemoryBackend() + require.NoError(t, b.VerifyEmailIdentity("s@example.com")) + + b.UpdateAccountSendingEnabled(false) + + _, err := b.SendEmail(ses.SendEmailInput{ + From: "s@example.com", + To: []string{"to@example.com"}, + Subject: "s", + BodyText: "b", + }) + require.ErrorIs(t, err, ses.ErrAccountSendingPaused) + + _, err = b.SendTemplatedEmail(ses.SendTemplatedEmailInput{ + From: "s@example.com", + To: []string{"to@example.com"}, + TemplateName: "does-not-matter", + }) + require.ErrorIs(t, err, ses.ErrAccountSendingPaused) + + // Re-enabling restores sending. + b.UpdateAccountSendingEnabled(true) + + _, err = b.SendEmail(ses.SendEmailInput{ + From: "s@example.com", + To: []string{"to@example.com"}, + Subject: "s", + BodyText: "b", + }) + require.NoError(t, err) +} + +func Test_AccountSendingPaused_Handler_ReturnsExactErrorCode(t *testing.T) { + t.Parallel() + + h := newHandler() + require.NoError(t, h.Backend.VerifyEmailIdentity("s@example.com")) + h.Backend.UpdateAccountSendingEnabled(false) + + rec := postForm(t, h, url.Values{ + "Action": {"SendEmail"}, + "Version": {"2010-12-01"}, + "Source": {"s@example.com"}, + "Destination.ToAddresses.member.1": {"to@example.com"}, + "Message.Subject.Data": {"hi"}, + "Message.Body.Text.Data": {"body"}, + }.Encode()) + + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "AccountSendingPausedException") +} + +// --------------------------------------------------------------------------- +// GetSendQuota advertises Max24HourSend (200, matching the real AWS SES +// sandbox default) but it was never enforced against SendEmail/ +// SendTemplatedEmail: an account could send unboundedly regardless of the +// quota it reported. Real AWS SES rejects sends past the 24-hour quota with +// MessageRejected. +// --------------------------------------------------------------------------- + +func Test_SendEmail_DailyQuota_Exhaustion(t *testing.T) { + t.Parallel() + + b := ses.NewInMemoryBackend() + require.NoError(t, b.VerifyEmailIdentity("s@example.com")) + + quota := b.GetSendQuota() + limit := int(quota.Max24HourSend) + require.Positive(t, limit) + + // Seed exactly the quota via AppendEmailForTest (bypassing SendEmail so the + // seeding itself isn't gated by the very quota being tested). + for range limit { + b.AppendEmailForTest("s@example.com", []string{"to@example.com"}) + } + + assert.InEpsilon(t, float64(limit), b.GetSendQuota().SentLast24Hours, 0.001) + + _, err := b.SendEmail(ses.SendEmailInput{ + From: "s@example.com", + To: []string{"to@example.com"}, + Subject: "s", + BodyText: "b", + }) + require.Error(t, err) + assert.ErrorIs(t, err, ses.ErrMessageRejected) +} + +// --------------------------------------------------------------------------- +// ConfigurationSetName must reference an existing configuration set for +// SendEmail/SendTemplatedEmail/SendBulkTemplatedEmail — real AWS SES returns +// ConfigurationSetDoesNotExist otherwise. Previously the field was stored on +// the Email record but never validated. +// --------------------------------------------------------------------------- + +func Test_SendEmail_UnknownConfigurationSet_Rejected(t *testing.T) { + t.Parallel() + + b := ses.NewInMemoryBackend() + require.NoError(t, b.VerifyEmailIdentity("s@example.com")) + + _, err := b.SendEmail(ses.SendEmailInput{ + From: "s@example.com", + To: []string{"to@example.com"}, + Subject: "s", + BodyText: "b", + ConfigurationSetName: "does-not-exist", + }) + require.Error(t, err) + assert.ErrorIs(t, err, ses.ErrConfigSetNotFound) +} + +func Test_SendBulkTemplatedEmail_UnknownConfigurationSet_Rejected(t *testing.T) { + t.Parallel() + + b := ses.NewInMemoryBackend() + require.NoError(t, b.VerifyEmailIdentity("s@example.com")) + require.NoError(t, b.CreateTemplate(ses.EmailTemplate{TemplateName: "t", SubjectPart: "s", TextPart: "b"})) + + dests := []ses.BulkEmailDestination{{To: []string{"a@example.com"}}} + + _, err := b.SendBulkTemplatedEmail("s@example.com", "t", "", "does-not-exist", "", "", nil, dests) + require.Error(t, err) + assert.ErrorIs(t, err, ses.ErrConfigSetNotFound) +} + +func Test_SendBulkTemplatedEmail_PlumbsMetadataThrough(t *testing.T) { + t.Parallel() + + b := ses.NewInMemoryBackend() + require.NoError(t, b.VerifyEmailIdentity("s@example.com")) + require.NoError(t, b.CreateConfigurationSet("cs1")) + require.NoError(t, b.CreateTemplate(ses.EmailTemplate{TemplateName: "t", SubjectPart: "s", TextPart: "b"})) + + dests := []ses.BulkEmailDestination{{To: []string{"a@example.com"}}} + + ids, err := b.SendBulkTemplatedEmail( + "s@example.com", "t", "", "cs1", "return@example.com", "arn:aws:ses:us-east-1:123:identity/s@example.com", + []string{"reply@example.com"}, dests, + ) + require.NoError(t, err) + require.Len(t, ids, 1) + + email, err := b.GetEmailByID(ids[0]) + require.NoError(t, err) + assert.Equal(t, "cs1", email.ConfigurationSetName) + assert.Equal(t, "return@example.com", email.ReturnPath) + assert.Equal(t, "arn:aws:ses:us-east-1:123:identity/s@example.com", email.SourceArn) + assert.Equal(t, []string{"reply@example.com"}, email.ReplyTo) +} + +// --------------------------------------------------------------------------- +// SendCustomVerificationEmail was a disguised stub: it validated its two +// string params but never checked the template existed, never registered the +// target identity, and returned a fabricated deterministic ID. It must now +// behave like a real AWS SES SendCustomVerificationEmailInput handler. +// --------------------------------------------------------------------------- + +func Test_SendCustomVerificationEmail_RegistersIdentity(t *testing.T) { + t.Parallel() + + b := ses.NewInMemoryBackend() + require.NoError(t, b.CreateCustomVerificationEmailTemplate(ses.CustomVerificationEmailTemplate{ + TemplateName: "cv", + FromEmailAddress: "noreply@example.com", + TemplateSubject: "Verify", + TemplateContent: "

click

", + SuccessRedirectionURL: "https://example.com/ok", + FailureRedirectionURL: "https://example.com/fail", + })) + + // Before the call, the identity is unregistered / NotStarted. + before := b.GetIdentityVerificationAttributes([]string{"user@example.com"}) + require.Equal(t, "NotStarted", before["user@example.com"]) + + msgID, err := b.SendCustomVerificationEmail("user@example.com", "cv", "") + require.NoError(t, err) + assert.NotEmpty(t, msgID) + + after := b.GetIdentityVerificationAttributes([]string{"user@example.com"}) + assert.Equal(t, "Success", after["user@example.com"]) +} + +func Test_SendCustomVerificationEmail_UnknownTemplate_Rejected(t *testing.T) { + t.Parallel() + + b := ses.NewInMemoryBackend() + + _, err := b.SendCustomVerificationEmail("user@example.com", "does-not-exist", "") + require.Error(t, err) + assert.ErrorIs(t, err, ses.ErrCustomVerifTemplateNotFound) +} + +func Test_SendCustomVerificationEmail_UnknownConfigurationSet_Rejected(t *testing.T) { + t.Parallel() + + b := ses.NewInMemoryBackend() + require.NoError(t, b.CreateCustomVerificationEmailTemplate(ses.CustomVerificationEmailTemplate{ + TemplateName: "cv", + FromEmailAddress: "noreply@example.com", + TemplateSubject: "Verify", + TemplateContent: "

click

", + SuccessRedirectionURL: "https://example.com/ok", + FailureRedirectionURL: "https://example.com/fail", + })) + + _, err := b.SendCustomVerificationEmail("user@example.com", "cv", "missing-cs") + require.Error(t, err) + assert.ErrorIs(t, err, ses.ErrConfigSetNotFound) +} + +// --------------------------------------------------------------------------- +// SetIdentityMailFromDomain silently dropped the BehaviorOnMXFailure +// parameter (never plumbed to the backend, so GetIdentityMailFromDomainAttributes +// could never reflect the value a real client set). It must now be validated +// and returned in the attributes response, defaulting to "UseDefaultValue" +// when omitted, matching the AWS SES BehaviorOnMXFailure enum semantics. +// --------------------------------------------------------------------------- + +func Test_SetIdentityMailFromDomain_BehaviorOnMXFailure(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + behavior string + wantPersisted string + wantErr bool + }{ + {name: "default_when_omitted", behavior: "", wantPersisted: "UseDefaultValue"}, + {name: "use_default_value_explicit", behavior: "UseDefaultValue", wantPersisted: "UseDefaultValue"}, + {name: "reject_message", behavior: "RejectMessage", wantPersisted: "RejectMessage"}, + {name: "invalid_value_rejected", behavior: "Bogus", wantErr: true}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := ses.NewInMemoryBackend() + err := b.SetIdentityMailFromDomain("a@example.com", "mail.example.com", tt.behavior) + + if tt.wantErr { + require.Error(t, err) + assert.ErrorIs(t, err, ses.ErrInvalidParameter) + + return + } + + require.NoError(t, err) + + attrs := b.GetIdentityMailFromDomainAttributes([]string{"a@example.com"}) + assert.Equal(t, tt.wantPersisted, attrs["a@example.com"].BehaviorOnMXFailure) + }) + } +} + +func Test_SetIdentityMailFromDomain_Handler_PlumbsBehaviorOnMXFailure(t *testing.T) { + t.Parallel() + + h := newHandler() + + rec := postForm(t, h, url.Values{ + "Action": {"SetIdentityMailFromDomain"}, + "Version": {"2010-12-01"}, + "Identity": {"a@example.com"}, + "MailFromDomain": {"mail.example.com"}, + "BehaviorOnMXFailure": {"RejectMessage"}, + }.Encode()) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + attrs := h.Backend.GetIdentityMailFromDomainAttributes([]string{"a@example.com"}) + assert.Equal(t, "RejectMessage", attrs["a@example.com"].BehaviorOnMXFailure) +} diff --git a/services/ses/templated_render_test.go b/services/ses/templated_render_test.go index be429147d..1cdb528c1 100644 --- a/services/ses/templated_render_test.go +++ b/services/ses/templated_render_test.go @@ -219,6 +219,10 @@ func TestSendBulkTemplatedEmail_DefaultAndReplacementData(t *testing.T) { "sender@example.com", "tmpl", `{"greeting":"Hello","company":"Acme"}`, + "", + "", + "", + nil, dests, ) require.NoError(t, err) @@ -249,7 +253,7 @@ func TestSendBulkTemplatedEmail_MissingTemplate(t *testing.T) { {To: []string{"a@example.com"}}, } - _, err := b.SendBulkTemplatedEmail("sender@example.com", "nope", "", dests) + _, err := b.SendBulkTemplatedEmail("sender@example.com", "nope", "", "", "", "", nil, dests) require.Error(t, err) assert.ErrorIs(t, err, ses.ErrTemplateNotFound, "want TemplateDoesNotExist, got %v", err) } @@ -269,7 +273,7 @@ func TestSendBulkTemplatedEmail_InvalidReplacementData(t *testing.T) { {To: []string{"a@example.com"}, ReplacementTemplateData: `{bad`}, } - _, err := b.SendBulkTemplatedEmail("sender@example.com", "tmpl", "", dests) + _, err := b.SendBulkTemplatedEmail("sender@example.com", "tmpl", "", "", "", "", nil, dests) require.ErrorIs(t, err, ses.ErrInvalidParameter, "got %v", err) assert.Contains(t, err.Error(), "TemplateData", "got %v", err) } diff --git a/services/sns/PARITY.md b/services/sns/PARITY.md new file mode 100644 index 000000000..87ba342b3 --- /dev/null +++ b/services/sns/PARITY.md @@ -0,0 +1,159 @@ +--- +service: sns +sdk_module: aws-sdk-go-v2/service/sns@v1.40.3 +last_audit_commit: 58e50f3a +last_audit_date: 2026-07-05 +overall: B +# Per-op or per-op-family status. Values: ok | partial | gap | deferred. +# wire=response/request shape vs SDK; errors=code+HTTP status; state=real mutate/read; persist=in backendSnapshot. +ops: + CreateTopic: {wire: ok, errors: ok, state: ok, persist: ok, note: "idempotent on existing name; FIFO/CBD/Kms validation correct"} + DeleteTopic: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed this pass: now also drops topicMessageArchive (was leaking + could resurrect stale archive on ARN reuse)"} + ListTopics: {wire: ok, errors: ok, state: ok, persist: ok} + GetTopicAttributes: {wire: ok, errors: ok, state: ok, persist: ok, note: "computed attrs (Owner/TopicArn/EffectiveDeliveryPolicy/SubscriptionsConfirmed|Pending|Deleted) correct"} + SetTopicAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + Subscribe: {wire: ok, errors: ok, state: ok, persist: ok, note: "all 9 protocols; pending-confirmation literal 'pending confirmation'; firehose requires SubscriptionRoleArn; dedup on existing confirmed sub"} + ConfirmSubscription: {wire: ok, errors: ok, state: ok, persist: ok} + Unsubscribe: {wire: ok, errors: ok, state: ok, persist: ok} + ListSubscriptions: {wire: ok, errors: ok, state: ok, persist: ok} + ListSubscriptionsByTopic: {wire: ok, errors: ok, state: ok, persist: ok} + GetSubscriptionAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + SetSubscriptionAttributes: {wire: ok, errors: ok, state: ok, persist: ok, note: "FilterPolicy/FilterPolicyScope/RedrivePolicy(+DLQ existence check)/DeliveryPolicy/ReplayPolicy/RawMessageDelivery/SubscriptionRoleArn"} + Publish: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed this pass: Lambda/Firehose/SQS-emitter now share one signed envelope (buildPublishedEvent) instead of Lambda fabricating a random-UUID signature"} + PublishBatch: {wire: partial->ok, errors: ok, state: ok, persist: ok, note: "fixed this pass: per-entry MessageAttributes field prefix was missing '.MessageAttributes' segment (verified against serializers.go) — every batch entry's attributes were silently dropped, breaking FilterPolicy matching for PublishBatch"} + PublishToTargetArn (TargetArn publish): {wire: ok, errors: ok, state: ok, persist: n/a, note: "EndpointDisabled enforced"} + PublishSMS (PhoneNumber publish): {wire: ok, errors: ok, state: ok, persist: n/a, note: "opt-out + sandbox-unverified enforced"} + CreatePlatformApplication: {wire: ok, errors: ok, state: ok, persist: ok} + GetPlatformApplicationAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + SetPlatformApplicationAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + ListPlatformApplications: {wire: ok, errors: ok, state: ok, persist: ok} + DeletePlatformApplication: {wire: ok, errors: ok, state: ok, persist: ok} + CreatePlatformEndpoint: {wire: ok, errors: ok, state: ok, persist: ok} + GetEndpointAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + SetEndpointAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + ListEndpointsByPlatformApplication: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteEndpoint: {wire: ok, errors: ok, state: ok, persist: ok} + AddPermission: {wire: ok, errors: ok, state: ok, persist: ok, note: "stored on Topic.Permissions, travels with topic snapshot"} + RemovePermission: {wire: ok, errors: ok, state: ok, persist: ok} + GetSMSSandboxAccountStatus/CreateSMSSandboxPhoneNumber/DeleteSMSSandboxPhoneNumber/ListSMSSandboxPhoneNumbers/VerifySMSSandboxPhoneNumber: {wire: ok, errors: ok, state: ok, persist: ok} + CheckIfPhoneNumberIsOptedOut/ListPhoneNumbersOptedOut/OptInPhoneNumber: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed this pass: ErrOptedOut sentinel text was the unrelated copy-pasted string 'KMSOptInRequired'"} + GetSMSAttributes/SetSMSAttributes: {wire: ok, errors: ok, state: ok, persist: ok} + GetDataProtectionPolicy/PutDataProtectionPolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "backed by topic/resource attribute; not deep-audited against a real customer-managed-data-identifier grammar"} + ListOriginationNumbers: {wire: ok, errors: ok, state: ok, persist: ok, note: "AWS has no public create API; empty by default, SeedOriginationNumber for tests"} + TagResource/UntagResource/ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok, note: "pkgs/tags-backed"} +families: + filter_policy_matching: {status: ok, note: "prefix/suffix/equals-ignore-case/anything-but(+nested)/exists/numeric(6 ops)/wildcard/cidr/$or, MessageBody vs MessageAttributes scope, String.Array expansion, 150-condition cap, 256KiB size cap — read in full, no gaps found"} + fifo_topics: {status: ok, note: "MessageGroupId required, ContentBasedDeduplication (SHA-256 body digest) vs explicit MessageDeduplicationId mutually exclusive, 5-min dedup window with bounded+swept map, 20-digit zero-padded monotonic SequenceNumber per topic, PublishBatch per-entry dedup"} + delivery_lambda_firehose_sms_application: {status: ok, note: "fixed this pass: (1) Lambda envelope now carries the real per-publish Timestamp/Signature/SigningCertURL/UnsubscribeURL instead of a fabricated random-UUID signature and empty cert/unsub URLs; (2) Firehose now respects RawMessageDelivery (envelopes as JSON when false, matching AWS default, previously always sent the bare message); DLQ redrive on failure now forwards the same body that was attempted"} + replay_policy_archive: {status: ok, note: "fixed this pass: replay previously only reached HTTP/HTTPS (direct call) and SQS (via the publish emitter) — Lambda/Firehose/SMS/Application subscriptions with a ReplayPolicy silently replayed nothing. Now fans out through the same per-protocol delivery functions Publish uses. NOT investigated: real AWS restricts archive/replay to FIFO topics + SQS/Lambda/Firehose only; this backend allows ArchivePolicy on standard topics and replays to HTTP/email/sms/application too (see gaps)"} + http_https_delivery: {status: ok, note: "RSA-2048 self-signed cert + SignatureVersion=2 SHA256 signing, retry via DeliveryPolicy/EffectiveDeliveryPolicy, DLQ redrive, concurrency-capped worker semaphore, ctx-cancel on shutdown"} + error_codes: {status: ok, note: "NotFound/TopicAlreadyExists/PlatformApplicationAlreadyExists/InvalidParameter/EndpointDisabled/OptedOut/AuthorizationError(permission label) all map to correct AWS code strings; 400 vs 500 split verified in handleBackendError"} +gaps: + - "ArchivePolicy/ReplayPolicy are accepted on any topic and fan out to any protocol (HTTP/email/sms/application); real AWS restricts message archiving/replay to FIFO topics with SQS/Lambda/Firehose subscribers only. Not fixed this pass — no doc access to confirm the exact restriction text/error code, and existing tests exercise HTTP replay deliberately. (bd: gopherstack-bz6)" + - "Subscribe does not validate that an 'sqs' protocol endpoint is a well-formed SQS queue ARN (any string is accepted); AWS rejects malformed endpoint ARNs per-protocol. Low value — SDK-driven callers always pass valid ARNs. (bd: gopherstack-bz6)" + - "SignatureVersion topic/subscription attribute is accepted and stored (isKnownTopicAttribute) but delivery always signs with SHA-256 (AWS 'SignatureVersion 2'); a topic explicitly set to SignatureVersion=1 should sign with SHA-1 instead. Not fixed — no consumer in this codebase verifies signatures, so behavior is unobservable, and getting this wrong is worse than leaving it (bd: gopherstack-bz6)" +deferred: + - "GetDataProtectionPolicy/PutDataProtectionPolicy: not verified against the real data-identifier grammar (e.g. built-in identifiers like 'Name', 'Address'); only checked that the policy round-trips as an opaque JSON string" + - "Cross-service integration (test/integration/*_parity_test.go) was not run this pass — see parity-principles.md note that unit tests are not parity proof; recommend running the SDK-driven integration suite in a follow-up" +leaks: {status: clean, note: "fixed this pass: (1) topicMessageArchive was never persisted (Snapshot/Restore) and was never cleaned up on DeleteTopic (both leak + ARN-reuse resurrection bug); (2) smsDeliveries/emailDeliveries/applicationDeliveries observability buffers had no cap and grew unboundedly under sustained publish traffic without a Drain* call — added appendBounded with maxRecordedDeliveries=100k; (3) notificationSigner.certURL was read/written without synchronization (SetSigningCertBaseURL vs concurrent delivery reads) — added a dedicated RWMutex. HTTP delivery goroutines already had proper ctx-cancel + semaphore + deliveryWg cleanup (unchanged, verified correct)."} +--- + +## Notes + +Freeform notes for the next auditor — AWS-behavior specifics worth remembering, and +"looks-wrong-but-correct" traps. + +### Protocol +SNS is the **AWS query (XML) protocol** (`Version=2010-03-31`, form-encoded request, +XML response with a `ResponseMetadata`/`RequestId` wrapper). PublishBatch entries and +per-entry MessageAttributes use the query-protocol list/map indexing convention: +`Parent.member.N.Field`, and nested maps use `Parent.member.N.MapField.entry.M.Key/Value`. +**Trap**: it is easy to drop a nesting segment when building one of these prefixes by +hand (see the PublishBatch fix this pass) — always cross-check against +`serializers.go` in the vendored SDK (`go list -m -f '{{.Dir}}' .../service/sns` to find +the extracted module, or `go get` into a scratch module) rather than trusting a +hand-derived guess. + +### FilterPolicy matching semantics +- Default `FilterPolicyScope` is `MessageAttributes`; `MessageBody` requires the + published message to parse as a JSON object, otherwise nothing matches. +- A `String.Array` message attribute (JSON array in `StringValue`) expands to one + candidate per element; the condition matches if ANY element matches (OR). +- `$or` is only treated as the OR operator when its array has ≥2 elements, every + element is a JSON object, and no element uses a reserved operator keyword + (`prefix`/`suffix`/`equals-ignore-case`/`anything-but`/`exists`/`numeric`/`wildcard`/ + `cidr`) as a top-level field — otherwise `$or` is an ordinary attribute name. +- `numeric` conditions are `[op, num, op, num, ...]` pairs, ANDed together. +- Non-existent attributes still get a *single* match attempt against `exists:false` + and `anything-but` conditions — do not special-case "attribute missing" as "no match" + without checking those two operators first. +- 150 total attribute conditions across a policy (including `$or` sub-policies), and a + 256 KiB serialized-policy size cap, both enforced at Subscribe/SetSubscriptionAttributes + time (fail fast) not at match time. + +### Per-protocol message envelope shapes +- **HTTP/HTTPS and Firehose** (when `RawMessageDelivery=false`, the AWS default): full + `Notification` JSON envelope — `Type, MessageId, TopicArn, Subject?, Message, Timestamp, + SignatureVersion, Signature, SigningCertURL, UnsubscribeURL`. `RawMessageDelivery=true` + delivers the bare message body only. +- **Lambda**: ALWAYS the full envelope wrapped in `{"Records":[{"EventVersion":"1.0", + "EventSource":"aws:sns","EventSubscriptionArn":...,"Sns":{...}}]}` — Lambda does NOT + support `RawMessageDelivery` (unconditional envelope, unlike HTTP/SQS/Firehose). +- **SQS**: delivered via the `pkgs/events` emitter (`SetPublishEmitter`/`SubscribeToSNS` + in cli.go, out of scope for this service) — SNS's job ends at emitting + `events.SNSPublishedEvent` with the same Timestamp/Signature/SigningCertURL used + everywhere else for this publish. +- **SMS/Application(mobile push)**: no real external sink in this mock; delivery is + recorded (`SMSDelivery`/`ApplicationDelivery`) and drained via `DrainSMSDeliveries`/ + `DrainApplicationDeliveries` for test/dashboard observability. These buffers are now + bounded (`maxRecordedDeliveries`) — do not remove the cap thinking it is dead code. +- **Trap**: as of this pass, every channel for a single `Publish` call shares ONE + `Timestamp`/`Signature`/`SigningCertURL`, built once in `buildPublishedEvent` and + threaded through to the SQS emitter, Lambda, and Firehose. Do not reintroduce a + second, independent `time.Now()`/sign call per channel — that was the bug (Lambda + used to fabricate `uuid.NewString()` as a fake "signature"). + +### Signature / MD5 +- SNS signs the **Notification envelope** (RSA, canonical string of + Message/MessageId/Subject?/Timestamp/TopicArn/Type sorted by field name, + newline-separated) — this is NOT the same as SQS's `MD5OfMessageBody`. **SNS's + `Publish`/`PublishBatch` responses do NOT include any MD5 field** — that's an + SQS-only concept; don't add one here, it would be a fabricated field AWS never + returns. +- This backend always signs with SHA-256 ("SignatureVersion 2") regardless of the + topic's `SignatureVersion` attribute value. Real AWS defaults to SignatureVersion 1 + (SHA-1) and only uses SHA-256 when the topic attribute is explicitly "2". Left + as a known gap (see `gaps:`) rather than guessed at, since nothing in this codebase + verifies the signature and getting the SHA-1 codepath subtly wrong (e.g. real AWS's + exact PKCS1v15 padding/hash combination) is worse than a consistently-SHA-256 mock. + +### Subscription ARN format +- Confirmed subscription ARN: `arn:{partition}:sns:{region}:{account}:{topicName}:{uuid}` + (built via `arn.Build("sns", region, accountID, topicName+":"+uuid.New().String())`). +- **Pending** (unconfirmed) HTTP/HTTPS/email/email-json subscriptions return the + **literal string** `"pending confirmation"` (lowercase, with a space) as the + `SubscriptionArn` in `Subscribe`'s response AND in `ListSubscriptions`/ + `ListSubscriptionsByTopic` — this is NOT a placeholder ARN, it is the exact string + AWS returns. `Subscribe`'s `ReturnSubscriptionArn=true` parameter overrides this and + always returns the real ARN even while pending. +- SQS, Lambda, Firehose, SMS, and Application subscriptions are auto-confirmed + (`PendingConfirmation=false` immediately); only HTTP, HTTPS, email, and email-json + require confirmation. + +### FIFO topics +- `MessageGroupId` is required on every Publish/PublishBatch entry to a `.fifo` topic. +- `ContentBasedDeduplication=true` forbids an explicit `MessageDeduplicationId` (uses + SHA-256 hex of the message body instead); `false` (default) requires one. +- Dedup window is 5 minutes, keyed by `topicArn + "/" + dedupID`; a duplicate within + the window returns a **new** synthesized MessageId/SequenceNumber without + re-publishing or re-delivering — this mirrors real AWS (dedup is silent, not an error). +- `SequenceNumber` is a 20-digit zero-padded, per-topic monotonic counter — not derived + from message content or timestamp. + +### Locking +`InMemoryBackend` uses one coarse `lockmetrics.RWMutex` per the pkgs-catalog rule. +`notificationSigner` additionally has its own small `sync.RWMutex` guarding just its +mutable `certURLValue` field (set once via `SetSigningCertBaseURL` when the mock +server's address becomes known, read on every signed delivery) — this is a +single-field auxiliary lock, not a second backend-resource lock, so it does not +violate the "one coarse lock per backend" rule. diff --git a/services/sns/accuracy_batch2_test.go b/services/sns/accuracy_batch2_test.go index 0037f4c3a..22c878126 100644 --- a/services/sns/accuracy_batch2_test.go +++ b/services/sns/accuracy_batch2_test.go @@ -20,6 +20,7 @@ package sns_test // - Subscribe Firehose requires SubscriptionRoleArn import ( + "encoding/json" "encoding/xml" "fmt" "io" @@ -1090,6 +1091,173 @@ func TestBatch2_ReplayPolicyFutureTimestampReplayesNothing(t *testing.T) { } } +// Test_ReplayPolicyDeliversToAllSubscriptionProtocols verifies that a +// subscription's ReplayPolicy fans archived messages out through the same +// per-protocol delivery path a live Publish uses. Before this fix, replay only +// ever reached HTTP/HTTPS (a direct call in replayMessagesToSubscription) and +// SQS (via the publish emitter); a subscription with a ReplayPolicy on the +// Lambda, Firehose, SMS, or Application protocol silently replayed nothing. +func Test_ReplayPolicyDeliversToAllSubscriptionProtocols(t *testing.T) { + t.Parallel() + + const archivedMessage = "archived-message" + + type caseResult struct { + verify func(t *testing.T) + endpoint string + } + + cases := []struct { + setup func(t *testing.T, b *sns.InMemoryBackend) caseResult + name string + proto string + }{ + { + name: "lambda", + proto: "lambda", + setup: func(t *testing.T, b *sns.InMemoryBackend) caseResult { + t.Helper() + + lambda := &mockLambdaInvoker{} + b.SetLambdaBackend(lambda) + + return caseResult{ + endpoint: "arn:aws:lambda:us-east-1:123456789012:function:replay-fn", + verify: func(t *testing.T) { + t.Helper() + require.Eventually(t, func() bool { return lambda.Count() == 1 }, + 2*time.Second, 10*time.Millisecond, "lambda function was never invoked") + + var envelope map[string]any + require.NoError(t, json.Unmarshal(lambda.Last().Payload, &envelope)) + records, _ := envelope["Records"].([]any) + require.Len(t, records, 1) + record, _ := records[0].(map[string]any) + snsData, _ := record["Sns"].(map[string]any) + assert.Equal(t, archivedMessage, snsData["Message"]) + assert.NotEmpty(t, snsData["Signature"], "replayed Lambda envelope must carry a real signature") + }, + } + }, + }, + { + name: "firehose", + proto: "firehose", + setup: func(t *testing.T, b *sns.InMemoryBackend) caseResult { + t.Helper() + + const streamName = "replay-stream" + firehose := newMockFirehose() + b.SetFirehoseBackend(firehose) + + return caseResult{ + endpoint: "arn:aws:firehose:us-east-1:123456789012:deliverystream/" + streamName, + verify: func(t *testing.T) { + t.Helper() + require.Eventually(t, func() bool { return len(firehose.RecordsFor(streamName)) == 1 }, + 2*time.Second, 10*time.Millisecond, "firehose stream received no record") + + var envelope map[string]any + require.NoError(t, json.Unmarshal(firehose.RecordsFor(streamName)[0], &envelope)) + assert.Equal(t, archivedMessage, envelope["Message"]) + }, + } + }, + }, + { + name: "sms", + proto: "sms", + setup: func(t *testing.T, b *sns.InMemoryBackend) caseResult { + t.Helper() + + return caseResult{ + endpoint: "+15005550006", + verify: func(t *testing.T) { + t.Helper() + + var got []sns.SMSDelivery + require.Eventually(t, func() bool { + if d := b.DrainSMSDeliveries(); len(d) > 0 { + got = d + + return true + } + + return false + }, 2*time.Second, 10*time.Millisecond, "SMS delivery was never recorded") + + require.Len(t, got, 1) + assert.Equal(t, archivedMessage, got[0].Message) + }, + } + }, + }, + { + name: "application", + proto: "application", + setup: func(t *testing.T, b *sns.InMemoryBackend) caseResult { + t.Helper() + + app, err := b.CreatePlatformApplication("replay-app", "GCM", nil) + require.NoError(t, err) + ep, err := b.CreatePlatformEndpoint(app.PlatformApplicationArn, "replay-token", nil) + require.NoError(t, err) + + return caseResult{ + endpoint: ep.EndpointArn, + verify: func(t *testing.T) { + t.Helper() + + var got []sns.ApplicationDelivery + require.Eventually(t, func() bool { + if d := b.DrainApplicationDeliveries(); len(d) > 0 { + got = d + + return true + } + + return false + }, 2*time.Second, 10*time.Millisecond, "application delivery was never recorded") + + require.Len(t, got, 1) + assert.Equal(t, archivedMessage, got[0].Message) + }, + } + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + // Each subtest builds its own isolated backend, topic, and subscription. + b := newB2Backend(t) + tp, err := b.CreateTopic("replay-fanout-"+tc.name, map[string]string{ + "ArchivePolicy": `{"MessageRetentionPeriod":30}`, + }) + require.NoError(t, err) + + // Publish before subscribing so the message lands only in the archive. + pastTime := time.Now().UTC().Add(-time.Hour) + _, err = b.Publish(tp.TopicArn, archivedMessage, "", "", nil) + require.NoError(t, err) + + res := tc.setup(t, b) + + sub, err := b.Subscribe(tp.TopicArn, tc.proto, res.endpoint, "") + require.NoError(t, err) + + replayFrom := pastTime.Format(time.RFC3339) + err = b.SetSubscriptionAttributes(sub.SubscriptionArn, "ReplayPolicy", + fmt.Sprintf(`{"replayFromTimestamp":"%s"}`, replayFrom)) + require.NoError(t, err) + + res.verify(t) + }) + } +} + // --------------------------------------------------------------------------- // Platform application event notifications // --------------------------------------------------------------------------- diff --git a/services/sns/backend.go b/services/sns/backend.go index 36d935e38..fd1bf0846 100644 --- a/services/sns/backend.go +++ b/services/sns/backend.go @@ -34,6 +34,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/config" "github.com/blackbirdworks/gopherstack/pkgs/events" + "github.com/blackbirdworks/gopherstack/pkgs/store" svcTags "github.com/blackbirdworks/gopherstack/pkgs/tags" ) @@ -86,8 +87,13 @@ var ( ErrPermissionLabelExists = errors.New("AuthorizationError") ErrPermissionLabelNotFound = errors.New("AuthorizationError") ErrSandboxPhoneNotVerified = errors.New("InvalidParameter") - ErrOptedOut = errors.New("KMSOptInRequired") - ErrHTTPStatus = errors.New("HTTP status") + // ErrOptedOut maps to the SNS "OptedOut" error code (see errorCode in handler.go). + // The sentinel's own message must describe the actual condition, since %w-wrapping + // embeds it verbatim into the API error message returned to the caller: it + // previously read "KMSOptInRequired", an unrelated KMS error string that leaked + // into every opted-out-SMS error message. + ErrOptedOut = errors.New("OptedOut") + ErrHTTPStatus = errors.New("HTTP status") ) const ( @@ -172,6 +178,14 @@ const ( // When the cap is exceeded, the oldest messages are evicted. maxArchivedMessagesPerTopic = 100_000 + // maxRecordedDeliveries caps each in-memory delivery-observation buffer + // (smsDeliveries, emailDeliveries, applicationDeliveries). Unlike AWS's real + // SMS/email/mobile-push delivery paths, this mock has no external sink for + // those protocols, so it records deliveries for later inspection via the + // Drain* methods. Without a cap, sustained publish traffic to a topic whose + // subscribers never drain these buffers grows them without bound. + maxRecordedDeliveries = 100_000 + // maxTopicNameLen is the maximum length of an SNS topic name. maxTopicNameLen = 256 @@ -394,12 +408,32 @@ type ArchivedMessage struct { // notificationSigner holds the RSA key pair and self-signed certificate used to // sign SNS HTTP/HTTPS notification envelopes per AWS SignatureVersion=2 spec. -// The certificate is served at the URL stored in certURL so subscribers can -// verify signatures without contacting the real AWS endpoint. +// The certificate is served at the URL stored in certURLValue so subscribers can +// verify signatures without contacting the real AWS endpoint. certURLValue is +// guarded by mu because SetSigningCertBaseURL may be called concurrently with +// in-flight deliveries that read the URL (e.g. a late server-address rewire +// racing a publish already in progress). type notificationSigner struct { - privateKey *rsa.PrivateKey - certURL string // URL where certPEM is accessible (configurable for tests) - certPEM []byte // PEM-encoded DER certificate, served at certURL + privateKey *rsa.PrivateKey + certURLValue string + certPEM []byte + mu sync.RWMutex +} + +// certURL returns the URL where certPEM is currently served. +func (s *notificationSigner) certURL() string { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.certURLValue +} + +// setCertURL updates the URL where certPEM is served. +func (s *notificationSigner) setCertURL(u string) { + s.mu.Lock() + defer s.mu.Unlock() + + s.certURLValue = u } // newNotificationSigner generates a fresh RSA-2048 key pair and a self-signed @@ -436,10 +470,10 @@ func newNotificationSigner(region string) *notificationSigner { return ¬ificationSigner{ privateKey: key, certPEM: certPEM, - // certURL is set later via SetSigningCertBaseURL when the server address is known. + // certURLValue is set later via SetSigningCertBaseURL when the server address is known. // The initial value uses the backend region so URLs are correct before // SetSigningCertBaseURL is called (e.g. in non-HTTP test scenarios). - certURL: fmt.Sprintf("https://sns.%s.amazonaws.com/SimpleNotificationService.pem", region), + certURLValue: fmt.Sprintf("https://sns.%s.amazonaws.com/SimpleNotificationService.pem", region), } } @@ -518,22 +552,26 @@ type InMemoryBackend struct { sqsSender SQSSender sqsChecker SQSQueueChecker svcCtx context.Context - topicSubscriptions map[string]map[string]*Subscription httpClient *http.Client - topics map[string]*Topic + registry *store.Registry + topics *store.Table[Topic] topicTags map[string]*svcTags.Tags - platformApplications map[string]*PlatformApplication - smsSandbox map[string]*SandboxPhoneNumber + platformApplications *store.Table[PlatformApplication] + smsSandbox *store.Table[SandboxPhoneNumber] optedOutPhoneNumbers map[string]bool smsAttributes map[string]string // originationNumbers holds origination phone numbers keyed by region. AWS SNS exposes no // public "create origination number" API (numbers are provisioned via Pinpoint / AWS // End User Messaging), so a fresh account legitimately has none. This map reflects real // state when populated via SeedOriginationNumber (used by tests / internal seeding). - originationNumbers map[string][]XMLOriginationPhone - mu *lockmetrics.RWMutex - subscriptions map[string]*Subscription - platformEndpoints map[string]*PlatformEndpoint + originationNumbers map[string][]XMLOriginationPhone + mu *lockmetrics.RWMutex + subscriptions *store.Table[Subscription] + // subscriptionsByTopic is a secondary index over subscriptions keyed by + // TopicArn, replacing the old hand-maintained topicSubscriptions nested + // map. It is kept consistent automatically by subscriptions.Put/Delete. + subscriptionsByTopic *store.Index[Subscription] + platformEndpoints *store.Table[PlatformEndpoint] topicMessageArchive map[string][]*ArchivedMessage // populated when ArchivePolicy is set signer *notificationSigner workerSem chan struct{} @@ -570,15 +608,10 @@ func NewInMemoryBackendWithContext( svcCtx = context.Background() } - return &InMemoryBackend{ - topics: make(map[string]*Topic), - subscriptions: make(map[string]*Subscription), - topicSubscriptions: make(map[string]map[string]*Subscription), + b := &InMemoryBackend{ + registry: store.NewRegistry(), topicTags: make(map[string]*svcTags.Tags), - platformApplications: make(map[string]*PlatformApplication), - platformEndpoints: make(map[string]*PlatformEndpoint), topicMessageArchive: make(map[string][]*ArchivedMessage), - smsSandbox: make(map[string]*SandboxPhoneNumber), optedOutPhoneNumbers: make(map[string]bool), smsAttributes: make(map[string]string), originationNumbers: make(map[string][]XMLOriginationPhone), @@ -591,6 +624,10 @@ func NewInMemoryBackendWithContext( workerSem: make(chan struct{}, snsMaxConcurrentDeliveries), signer: newNotificationSigner(region), } + + registerAllTables(b) + + return b } // SetHTTPDeliveryClient configures the HTTP client used for HTTP/HTTPS subscription delivery. @@ -615,7 +652,7 @@ func (b *InMemoryBackend) SigningCertPEM() []byte { // known so that subscribers can retrieve the verification certificate. // The URL should point to the mock server's /SimpleNotificationService.pem path. func (b *InMemoryBackend) SetSigningCertBaseURL(baseURL string) { - b.signer.certURL = strings.TrimRight(baseURL, "/") + "/SimpleNotificationService.pem" + b.signer.setCertURL(strings.TrimRight(baseURL, "/") + "/SimpleNotificationService.pem") } // SetPublishEmitter registers an event emitter that fires when a message is published. @@ -688,7 +725,7 @@ func (b *InMemoryBackend) CreateTopicInRegion( } topicArn := arn.Build("sns", region, b.accountID, name) - if existing, exists := b.topics[topicArn]; exists { + if existing, exists := b.topics.Get(topicArn); exists { // AWS SNS CreateTopic is idempotent: calling it with an existing name // returns the existing topic ARN rather than an error. return existing, nil @@ -742,8 +779,7 @@ func (b *InMemoryBackend) CreateTopicInRegion( Attributes: attrs, CreationTimestamp: time.Now().UTC(), } - b.topics[topicArn] = topic - b.topicSubscriptions[topicArn] = make(map[string]*Subscription) + b.topics.Put(topic) return topic, nil } @@ -753,11 +789,11 @@ func (b *InMemoryBackend) DeleteTopic(topicArn string) error { b.mu.Lock("DeleteTopic") defer b.mu.Unlock() - if _, exists := b.topics[topicArn]; !exists { + if !b.topics.Has(topicArn) { return ErrTopicNotFound } - delete(b.topics, topicArn) + b.topics.Delete(topicArn) // Close topic tags to prevent resource leak. if t := b.topicTags[topicArn]; t != nil { @@ -765,13 +801,19 @@ func (b *InMemoryBackend) DeleteTopic(topicArn string) error { delete(b.topicTags, topicArn) } - // Remove any orphaned subscriptions for this topic. - for subArn, sub := range b.subscriptions { - if sub.TopicArn == topicArn { - delete(b.subscriptions, subArn) - } + // Remove any orphaned subscriptions for this topic. subscriptionsByTopic.Get + // returns the live index group directly (see store.Index.Get), so the + // slice is copied before deleting to avoid mutating it while iterating. + for _, sub := range append([]*Subscription(nil), b.subscriptionsByTopic.Get(topicArn)...) { + b.subscriptions.Delete(sub.SubscriptionArn) } - delete(b.topicSubscriptions, topicArn) + + // Drop the message archive (ArchivePolicy replay buffer). Without this the + // archive both leaks unboundedly for every created-then-deleted topic ARN and, + // because SNS topic ARNs are deterministic (arn:...:), silently + // resurfaces a deleted topic's old archived messages to ReplayPolicy + // subscribers on a newly created topic that reuses the same name. + delete(b.topicMessageArchive, topicArn) return nil } @@ -809,7 +851,7 @@ func (b *InMemoryBackend) GetTopicAttributes(topicArn string) (map[string]string b.mu.RLock("GetTopicAttributes") defer b.mu.RUnlock() - topic, exists := b.topics[topicArn] + topic, exists := b.topics.Get(topicArn) if !exists { return nil, ErrTopicNotFound } @@ -854,7 +896,7 @@ func (b *InMemoryBackend) GetTopicAttributes(topicArn string) (map[string]string // periodically, so we report 0 for consistency with a fresh mock environment. confirmed, pending := 0, 0 - for _, sub := range b.topicSubscriptions[topicArn] { + for _, sub := range b.subscriptionsByTopic.Get(topicArn) { if sub.PendingConfirmation { pending++ } else { @@ -914,7 +956,7 @@ func (b *InMemoryBackend) SetTopicAttributes(topicArn, attrName, attrValue strin b.mu.Lock("SetTopicAttributes") defer b.mu.Unlock() - topic, exists := b.topics[topicArn] + topic, exists := b.topics.Get(topicArn) if !exists { return ErrTopicNotFound } @@ -1014,14 +1056,14 @@ func (b *InMemoryBackend) Subscribe( b.mu.Lock("Subscribe") defer b.mu.Unlock() - topic, exists := b.topics[topicArn] + topic, exists := b.topics.Get(topicArn) if !exists { return nil, ErrTopicNotFound } // Dedup: return the existing subscription ARN when protocol+endpoint already // has a confirmed subscription on this topic (matches AWS behaviour). - for _, existing := range b.topicSubscriptions[topicArn] { + for _, existing := range b.subscriptionsByTopic.Get(topicArn) { if !existing.PendingConfirmation && existing.Protocol == protocol && existing.Endpoint == endpoint { @@ -1056,8 +1098,7 @@ func (b *InMemoryBackend) Subscribe( CreationTimestamp: time.Now().UTC(), } - b.subscriptions[subArn] = sub - b.indexSubscription(sub) + b.subscriptions.Put(sub) return sub, nil } @@ -1067,14 +1108,10 @@ func (b *InMemoryBackend) Unsubscribe(subscriptionArn string) error { b.mu.Lock("Unsubscribe") defer b.mu.Unlock() - sub, exists := b.subscriptions[subscriptionArn] - if !exists { + if !b.subscriptions.Delete(subscriptionArn) { return ErrSubscriptionNotFound } - delete(b.subscriptions, subscriptionArn) - b.removeIndexedSubscription(sub.TopicArn, subscriptionArn) - return nil } @@ -1091,7 +1128,7 @@ func (b *InMemoryBackend) ConfirmSubscription(topicArn, token string) (*Subscrip defer b.mu.Unlock() // Use topic index for O(topic_subs) instead of O(all_subs). - for _, sub := range b.topicSubscriptions[topicArn] { + for _, sub := range b.subscriptionsByTopic.Get(topicArn) { if sub.PendingConfirmation { sub.PendingConfirmation = false @@ -1109,7 +1146,7 @@ func (b *InMemoryBackend) GetSubscriptionAttributes( b.mu.RLock("GetSubscriptionAttributes") defer b.mu.RUnlock() - sub, exists := b.subscriptions[subscriptionArn] + sub, exists := b.subscriptions.Get(subscriptionArn) if !exists { return nil, ErrSubscriptionNotFound } @@ -1196,7 +1233,7 @@ func (b *InMemoryBackend) SetSubscriptionAttributes( b.mu.Lock("SetSubscriptionAttributes") - sub, exists := b.subscriptions[subscriptionArn] + sub, exists := b.subscriptions.Get(subscriptionArn) if !exists { b.mu.Unlock() @@ -1291,11 +1328,11 @@ func (b *InMemoryBackend) ListSubscriptionsByTopic( b.mu.RLock("ListSubscriptionsByTopic") defer b.mu.RUnlock() - if _, exists := b.topics[topicArn]; !exists { + if !b.topics.Has(topicArn) { return nil, "", ErrTopicNotFound } - topicSubs := b.topicSubscriptions[topicArn] + topicSubs := b.subscriptionsByTopic.Get(topicArn) filtered := make([]Subscription, 0, len(topicSubs)) for _, sub := range topicSubs { filtered = append(filtered, *sub) @@ -1797,7 +1834,18 @@ func (b *InMemoryBackend) collectPublishTargets( ) publishTargets { var out publishTargets - for _, sub := range b.topicSubscriptions[topicArn] { + // The topic is guaranteed to exist here: every caller holds at least + // b.mu.RLock and has already validated topicArn via b.topics.Get/Has + // before calling collectPublishTargets (see Publish). Looked up once + // (rather than per-subscription) since store.Table.Get returns (v, ok) + // and cannot be inlined into the httpDelivery literal below the way the + // old raw map index could. + var topicEffectivePolicy string + if topic, ok := b.topics.Get(topicArn); ok { + topicEffectivePolicy = topic.Attributes["EffectiveDeliveryPolicy"] + } + + for _, sub := range b.subscriptionsByTopic.Get(topicArn) { // Resolve the per-protocol message body for this subscription. // This must happen before filter evaluation when FilterPolicyScope=MessageBody, // because the body itself is the subject of the filter. @@ -1826,7 +1874,7 @@ func (b *InMemoryBackend) collectPublishTargets( rawDelivery: sub.RawMessageDelivery, redrivePolicy: sub.RedrivePolicy, deliveryPolicy: sub.DeliveryPolicy, - topicEffectivePolicy: b.topics[topicArn].Attributes["EffectiveDeliveryPolicy"], + topicEffectivePolicy: topicEffectivePolicy, sqsSender: b.sqsSender, }) } @@ -2090,17 +2138,20 @@ func (b *InMemoryBackend) dispatchHTTPDeliveries(deliveries []httpDelivery, clie } } -// emitPublishedEvent fires an SNSPublishedEvent so other services (e.g. SQS) -// can react. It is a no-op when no emitter has been registered. -func (b *InMemoryBackend) emitPublishedEvent( +// buildPublishedEvent constructs the SNSPublishedEvent broadcast to every +// non-HTTP delivery channel for a single Publish call: the SQS emitter, and +// the Lambda/Firehose/SMS/Application delivery fan-out below. The timestamp, +// RSA signature, and signing certificate URL are computed exactly once here so +// every channel carries an identical, verifiable notification envelope — +// matching real AWS SNS, which signs a message once per publish and reuses +// that signature across all destinations. Previously each delivery function +// received its own bare event with empty Timestamp/Signature/SigningCertURL +// fields, and the Lambda envelope fabricated a random-UUID "signature" instead. +func (b *InMemoryBackend) buildPublishedEvent( topicArn, messageID, message, subject string, attrs map[string]MessageAttribute, subs []events.SNSSubscriptionSnapshot, -) { - if b.emitter == nil { - return - } - +) *events.SNSPublishedEvent { attrSnaps := make(map[string]events.SNSMessageAttributeSnapshot, len(attrs)) for k, v := range attrs { attrSnaps[k] = events.SNSMessageAttributeSnapshot{ @@ -2109,14 +2160,12 @@ func (b *InMemoryBackend) emitPublishedEvent( } } - // Compute a fixed timestamp and RSA-SHA1 signature so SQS envelope - // delivery can include a verifiable Signature field per AWS spec. ts := time.Now().UTC().Format(time.RFC3339) canonical := canonicalNotificationString(messageID, topicArn, subject, message, ts) sig := b.signer.sign(canonical) - certURL := b.signer.certURL + certURL := b.signer.certURL() - _ = b.emitter.Emit(b.svcCtx, &events.SNSPublishedEvent{ + return &events.SNSPublishedEvent{ TopicARN: topicArn, MessageID: messageID, Message: message, @@ -2126,7 +2175,17 @@ func (b *InMemoryBackend) emitPublishedEvent( Timestamp: ts, Signature: sig, SigningCertURL: certURL, - }) + } +} + +// emitPublishedEvent broadcasts ev to the publish emitter (e.g. to SQS). It is +// a no-op when no emitter has been registered. +func (b *InMemoryBackend) emitPublishedEvent(ev *events.SNSPublishedEvent) { + if b.emitter == nil { + return + } + + _ = b.emitter.Emit(b.svcCtx, ev) } // Publish delivers a message to all subscriptions of topicArn. HTTP/HTTPS @@ -2144,7 +2203,7 @@ func (b *InMemoryBackend) Publish( b.mu.RLock("Publish") - topic, exists := b.topics[topicArn] + topic, exists := b.topics.Get(topicArn) if !exists { b.mu.RUnlock() @@ -2188,15 +2247,11 @@ func (b *InMemoryBackend) Publish( b.recordEmailDeliveries(targets.emailDeliveries, messageID, topicArn) - b.emitPublishedEvent(topicArn, messageID, message, subject, attrs, targets.subs) + // Build the shared event once so every channel below carries the same + // verifiable Timestamp/Signature/SigningCertURL (see buildPublishedEvent). + ev := b.buildPublishedEvent(topicArn, messageID, message, subject, attrs, targets.subs) - ev := &events.SNSPublishedEvent{ - TopicARN: topicArn, - MessageID: messageID, - Message: message, - Subject: subject, - Subscriptions: targets.subs, - } + b.emitPublishedEvent(ev) b.deliverToLambdaSubscriptions(ev) b.deliverToFirehoseSubscriptions(ev) b.deliverToSMSSubscriptions(ev) @@ -2216,7 +2271,7 @@ func (b *InMemoryBackend) PublishToTargetArn( b.mu.RLock("PublishToTargetArn") defer b.mu.RUnlock() - ep, exists := b.platformEndpoints[targetArn] + ep, exists := b.platformEndpoints.Get(targetArn) if !exists { return "", ErrEndpointNotFound } @@ -2241,7 +2296,7 @@ func (b *InMemoryBackend) PublishSMS(phoneNumber, message string) (string, error } b.mu.RLock("PublishSMS-check") - sandboxEntry := b.smsSandbox[phoneNumber] + sandboxEntry, _ := b.smsSandbox.Get(phoneNumber) optedOut := b.optedOutPhoneNumbers[phoneNumber] b.mu.RUnlock() @@ -2267,11 +2322,11 @@ func (b *InMemoryBackend) PublishSMS(phoneNumber, message string) (string, error msgID := uuid.New().String() b.mu.Lock("PublishSMS") - b.smsDeliveries = append(b.smsDeliveries, SMSDelivery{ + b.smsDeliveries = appendBounded(b.smsDeliveries, SMSDelivery{ PhoneNumber: phoneNumber, Message: message, MessageID: msgID, - }) + }, maxRecordedDeliveries) b.mu.Unlock() return msgID, nil @@ -2319,7 +2374,7 @@ func (b *InMemoryBackend) recordEmailDeliveries( for i := range deliveries { deliveries[i].MessageID = messageID deliveries[i].TopicARN = topicArn - b.emailDeliveries = append(b.emailDeliveries, deliveries[i]) + b.emailDeliveries = appendBounded(b.emailDeliveries, deliveries[i], maxRecordedDeliveries) } } @@ -2721,27 +2776,6 @@ func matchesConditions(value string, attrExists bool, conditions []json.RawMessa return false } -func (b *InMemoryBackend) indexSubscription(sub *Subscription) { - topicSubs := b.topicSubscriptions[sub.TopicArn] - if topicSubs == nil { - topicSubs = make(map[string]*Subscription) - b.topicSubscriptions[sub.TopicArn] = topicSubs - } - - topicSubs[sub.SubscriptionArn] = sub -} - -func (b *InMemoryBackend) removeIndexedSubscription(topicArn, subscriptionArn string) { - topicSubs := b.topicSubscriptions[topicArn] - if topicSubs == nil { - return - } - - delete(topicSubs, subscriptionArn) - // Preserve the inner map even when empty so the next Subscribe call does not - // need to re-allocate. Only remove it when the topic itself is deleted. -} - // ListAllTopics returns all topics sorted by ARN. func (b *InMemoryBackend) ListAllTopics() []Topic { b.mu.RLock("ListAllTopics") @@ -2763,8 +2797,8 @@ func (b *InMemoryBackend) ListAllPlatformApplications() []PlatformApplication { b.mu.RLock("ListAllPlatformApplications") defer b.mu.RUnlock() - apps := make([]PlatformApplication, 0, len(b.platformApplications)) - for _, app := range b.platformApplications { + apps := make([]PlatformApplication, 0, b.platformApplications.Len()) + for _, app := range b.platformApplications.All() { apps = append(apps, *app) } @@ -2777,8 +2811,8 @@ func (b *InMemoryBackend) ListAllPlatformApplications() []PlatformApplication { // sortedTopics returns all topics sorted by TopicArn. Must be called with at least RLock held. func (b *InMemoryBackend) sortedTopics() []Topic { - topics := make([]Topic, 0, len(b.topics)) - for _, t := range b.topics { + topics := make([]Topic, 0, b.topics.Len()) + for _, t := range b.topics.All() { topics = append(topics, *t) } @@ -2793,8 +2827,8 @@ func (b *InMemoryBackend) sortedTopics() []Topic { // Must be called with at least RLock held. // The region is extracted from the topic ARN (arn:partition:sns:REGION:account:name). func (b *InMemoryBackend) sortedTopicsInRegion(region string) []Topic { - topics := make([]Topic, 0, len(b.topics)) - for _, t := range b.topics { + topics := make([]Topic, 0, b.topics.Len()) + for _, t := range b.topics.All() { if arnRegion(t.TopicArn) == region { topics = append(topics, *t) } @@ -2809,8 +2843,8 @@ func (b *InMemoryBackend) sortedTopicsInRegion(region string) []Topic { // sortedSubscriptions returns subscriptions sorted by SubscriptionArn. Must be called with at least RLock held. func (b *InMemoryBackend) sortedSubscriptions() []Subscription { - subs := make([]Subscription, 0, len(b.subscriptions)) - for _, s := range b.subscriptions { + subs := make([]Subscription, 0, b.subscriptions.Len()) + for _, s := range b.subscriptions.All() { subs = append(subs, *s) } @@ -2899,7 +2933,7 @@ func (b *InMemoryBackend) logDeliveryStatus( err error, ) { b.mu.RLock("logDeliveryStatus") - topic, ok := b.topics[topicARN] + topic, ok := b.topics.Get(topicARN) if !ok { b.mu.RUnlock() @@ -2980,7 +3014,7 @@ func buildHTTPDeliveryPayload(d httpDelivery) string { ) signature := "MOCK-SIGNATURE" if d.signer != nil { - certURL = d.signer.certURL + certURL = d.signer.certURL() canonical := canonicalNotificationString( d.messageID, d.topicARN, d.subject, d.body, timestamp, ) @@ -3111,6 +3145,18 @@ func encodeToken(offset int) string { return base64.StdEncoding.EncodeToString([]byte(strconv.Itoa(offset))) } +// appendBounded appends item to slice, evicting the oldest entries once the +// result exceeds maxLen. Used to bound the smsDeliveries/emailDeliveries/ +// applicationDeliveries observability buffers (see maxRecordedDeliveries). +func appendBounded[T any](slice []T, item T, maxLen int) []T { + slice = append(slice, item) + if len(slice) > maxLen { + slice = slice[len(slice)-maxLen:] + } + + return slice +} + // paginate returns a page of items and the next token, or an empty token when exhausted. func paginate[T any](items []T, offset, size int) ([]T, string) { if offset >= len(items) { @@ -3189,9 +3235,11 @@ func (b *InMemoryBackend) TaggedTopics() []TaggedTopicInfo { b.mu.RLock("TaggedTopics") defer b.mu.RUnlock() - result := make([]TaggedTopicInfo, 0, len(b.topics)) + result := make([]TaggedTopicInfo, 0, b.topics.Len()) + + for _, topic := range b.topics.All() { + topicARN := topic.TopicArn - for topicARN := range b.topics { var tagMap map[string]string if b.topicTags[topicARN] != nil { tagMap = b.topicTags[topicARN].Clone() @@ -3208,7 +3256,7 @@ func (b *InMemoryBackend) TagTopicByARN(topicARN string, newTags map[string]stri b.mu.Lock("TagTopicByARN") defer b.mu.Unlock() - if _, ok := b.topics[topicARN]; !ok { + if !b.topics.Has(topicARN) { return fmt.Errorf("%w: topic %s", ErrTopicNotFound, topicARN) } @@ -3226,7 +3274,7 @@ func (b *InMemoryBackend) UntagTopicByARN(topicARN string, tagKeys []string) err b.mu.Lock("UntagTopicByARN") defer b.mu.Unlock() - if _, ok := b.topics[topicARN]; !ok { + if !b.topics.Has(topicARN) { return fmt.Errorf("%w: topic %s", ErrTopicNotFound, topicARN) } @@ -3278,7 +3326,7 @@ func (b *InMemoryBackend) CreatePlatformApplicationInRegion( appArn := arn.Build("sns", region, b.accountID, "app/"+platform+"/"+name) - if _, exists := b.platformApplications[appArn]; exists { + if b.platformApplications.Has(appArn) { return nil, ErrPlatformApplicationAlreadyExists } @@ -3295,7 +3343,7 @@ func (b *InMemoryBackend) CreatePlatformApplicationInRegion( Attributes: attrs, CreationTimestamp: time.Now().UTC(), } - b.platformApplications[appArn] = app + b.platformApplications.Put(app) return app, nil } @@ -3311,14 +3359,14 @@ func (b *InMemoryBackend) GetPlatformApplicationAttributes( b.mu.RLock("GetPlatformApplicationAttributes") defer b.mu.RUnlock() - app, exists := b.platformApplications[platformApplicationArn] + app, exists := b.platformApplications.Get(platformApplicationArn) if !exists { return nil, ErrPlatformApplicationNotFound } // Count active and disabled endpoints for this application. var activeCount, disabledCount int - for _, ep := range b.platformEndpoints { + for _, ep := range b.platformEndpoints.All() { if ep.PlatformApplicationArn != platformApplicationArn { continue } @@ -3349,7 +3397,7 @@ func (b *InMemoryBackend) SetPlatformApplicationAttributes( b.mu.Lock("SetPlatformApplicationAttributes") defer b.mu.Unlock() - app, exists := b.platformApplications[platformApplicationArn] + app, exists := b.platformApplications.Get(platformApplicationArn) if !exists { return ErrPlatformApplicationNotFound } @@ -3383,19 +3431,26 @@ func (b *InMemoryBackend) DeletePlatformApplication(platformApplicationArn strin b.mu.Lock("DeletePlatformApplication") defer b.mu.Unlock() - if _, exists := b.platformApplications[platformApplicationArn]; !exists { + if !b.platformApplications.Has(platformApplicationArn) { return ErrPlatformApplicationNotFound } - delete(b.platformApplications, platformApplicationArn) + b.platformApplications.Delete(platformApplicationArn) - // Remove all endpoints associated with this platform application. - for endpointArn, ep := range b.platformEndpoints { + // Remove all endpoints associated with this platform application. Collect + // keys first rather than deleting from within Table.Range's own iteration. + var toDelete []string + + for _, ep := range b.platformEndpoints.All() { if ep.PlatformApplicationArn == platformApplicationArn { - delete(b.platformEndpoints, endpointArn) + toDelete = append(toDelete, ep.EndpointArn) } } + for _, endpointArn := range toDelete { + b.platformEndpoints.Delete(endpointArn) + } + return nil } @@ -3410,7 +3465,7 @@ func (b *InMemoryBackend) CreatePlatformEndpoint( ) (*PlatformEndpoint, error) { b.mu.Lock("CreatePlatformEndpoint") - app, exists := b.platformApplications[platformApplicationArn] + app, exists := b.platformApplications.Get(platformApplicationArn) if !exists { b.mu.Unlock() @@ -3419,7 +3474,7 @@ func (b *InMemoryBackend) CreatePlatformEndpoint( // Dedup: return the existing endpoint when the same token is already registered // under this platform application (mirrors AWS CreatePlatformEndpoint behaviour). - for _, ep := range b.platformEndpoints { + for _, ep := range b.platformEndpoints.All() { if ep.PlatformApplicationArn == platformApplicationArn && ep.Attributes["Token"] == token { b.mu.Unlock() @@ -3466,7 +3521,7 @@ func (b *InMemoryBackend) CreatePlatformEndpoint( Attributes: attrs, CreationTimestamp: time.Now().UTC(), } - b.platformEndpoints[endpointArn] = ep + b.platformEndpoints.Put(ep) b.mu.Unlock() @@ -3485,7 +3540,7 @@ func (b *InMemoryBackend) GetEndpointAttributes(endpointArn string) (map[string] b.mu.RLock("GetEndpointAttributes") defer b.mu.RUnlock() - ep, exists := b.platformEndpoints[endpointArn] + ep, exists := b.platformEndpoints.Get(endpointArn) if !exists { return nil, ErrEndpointNotFound } @@ -3505,7 +3560,7 @@ func (b *InMemoryBackend) SetEndpointAttributes( ) error { b.mu.Lock("SetEndpointAttributes") - ep, exists := b.platformEndpoints[endpointArn] + ep, exists := b.platformEndpoints.Get(endpointArn) if !exists { b.mu.Unlock() @@ -3532,7 +3587,7 @@ func (b *InMemoryBackend) ListEndpointsByPlatformApplication( b.mu.RLock("ListEndpointsByPlatformApplication") defer b.mu.RUnlock() - if _, exists := b.platformApplications[platformApplicationArn]; !exists { + if !b.platformApplications.Has(platformApplicationArn) { return nil, "", ErrPlatformApplicationNotFound } @@ -3561,7 +3616,7 @@ func (b *InMemoryBackend) ListEndpointsByPlatformApplication( func (b *InMemoryBackend) DeleteEndpoint(endpointArn string) error { b.mu.Lock("DeleteEndpoint") - ep, exists := b.platformEndpoints[endpointArn] + ep, exists := b.platformEndpoints.Get(endpointArn) if !exists { b.mu.Unlock() @@ -3569,7 +3624,7 @@ func (b *InMemoryBackend) DeleteEndpoint(endpointArn string) error { } platformAppArn := ep.PlatformApplicationArn - delete(b.platformEndpoints, endpointArn) + b.platformEndpoints.Delete(endpointArn) b.mu.Unlock() @@ -3583,8 +3638,8 @@ func (b *InMemoryBackend) DeleteEndpoint(endpointArn string) error { // sortedPlatformApplications returns platform applications sorted by ARN. Must be called with at least RLock held. func (b *InMemoryBackend) sortedPlatformApplications() []PlatformApplication { - apps := make([]PlatformApplication, 0, len(b.platformApplications)) - for _, a := range b.platformApplications { + apps := make([]PlatformApplication, 0, b.platformApplications.Len()) + for _, a := range b.platformApplications.All() { apps = append(apps, *a) } @@ -3597,8 +3652,8 @@ func (b *InMemoryBackend) sortedPlatformApplications() []PlatformApplication { // sortedEndpoints returns platform endpoints sorted by ARN. Must be called with at least RLock held. func (b *InMemoryBackend) sortedEndpoints() []PlatformEndpoint { - eps := make([]PlatformEndpoint, 0, len(b.platformEndpoints)) - for _, ep := range b.platformEndpoints { + eps := make([]PlatformEndpoint, 0, b.platformEndpoints.Len()) + for _, ep := range b.platformEndpoints.All() { eps = append(eps, *ep) } @@ -3616,7 +3671,7 @@ func (b *InMemoryBackend) sortedEndpoints() []PlatformEndpoint { // whether the event topic exists. func (b *InMemoryBackend) fireEndpointEvent(appArn, eventAttr string, payload map[string]string) { b.mu.RLock("fireEndpointEvent") - app, exists := b.platformApplications[appArn] + app, exists := b.platformApplications.Get(appArn) var topicArn string if exists { topicArn = app.Attributes[eventAttr] @@ -3694,7 +3749,7 @@ func (b *InMemoryBackend) replayMessagesToSubscription( sqsSender := b.sqsSender var topicEffectivePolicy string - if topic, ok := b.topics[topicArn]; ok { + if topic, ok := b.topics.Get(topicArn); ok { topicEffectivePolicy = topic.Attributes["EffectiveDeliveryPolicy"] } @@ -3729,23 +3784,29 @@ func (b *InMemoryBackend) replayMessagesToSubscription( deliverHTTPWithMeta(b.svcCtx, d, client, b) } + // Build one shared event for this replayed message and fan it out through + // the same per-protocol delivery functions Publish uses. Previously replay + // only reached HTTP/HTTPS (above) and SQS (via the emitter below); a + // subscription with a ReplayPolicy on Lambda, Firehose, SMS, or Application + // protocol silently received nothing for archived messages. + replayEv := b.buildPublishedEvent( + topicArn, msg.MessageID, msg.Message, msg.Subject, msg.Attributes, + []events.SNSSubscriptionSnapshot{subSnap}, + ) + if emitter != nil { - attrSnaps := make(map[string]events.SNSMessageAttributeSnapshot, len(msg.Attributes)) - for k, v := range msg.Attributes { - attrSnaps[k] = events.SNSMessageAttributeSnapshot{ - DataType: v.DataType, - StringValue: v.StringValue, - } - } + _ = emitter.Emit(b.svcCtx, replayEv) + } - _ = emitter.Emit(b.svcCtx, &events.SNSPublishedEvent{ - TopicARN: topicArn, - MessageID: msg.MessageID, - Message: msg.Message, - Subject: msg.Subject, - Subscriptions: []events.SNSSubscriptionSnapshot{subSnap}, - Attributes: attrSnaps, - }) + switch sub.Protocol { + case protocolLambda: + b.deliverToLambdaSubscriptions(replayEv) + case protocolFirehose: + b.deliverToFirehoseSubscriptions(replayEv) + case protocolSMS: + b.deliverToSMSSubscriptions(replayEv) + case protocolApplication: + b.deliverToApplicationSubscriptions(replayEv) } } } @@ -3861,7 +3922,7 @@ func (b *InMemoryBackend) AddPermission(topicArn, label string, accounts, action b.mu.Lock("AddPermission") defer b.mu.Unlock() - topic, exists := b.topics[topicArn] + topic, exists := b.topics.Get(topicArn) if !exists { return ErrTopicNotFound } @@ -3888,7 +3949,7 @@ func (b *InMemoryBackend) RemovePermission(topicArn, label string) error { b.mu.Lock("RemovePermission") defer b.mu.Unlock() - topic, exists := b.topics[topicArn] + topic, exists := b.topics.Get(topicArn) if !exists { return ErrTopicNotFound } @@ -3935,16 +3996,16 @@ func (b *InMemoryBackend) CreateSMSSandboxPhoneNumber(phoneNumber, languageCode b.mu.Lock("CreateSMSSandboxPhoneNumber") defer b.mu.Unlock() - if _, exists := b.smsSandbox[phoneNumber]; exists { + if b.smsSandbox.Has(phoneNumber) { return ErrSandboxPhoneAlreadyExists } - b.smsSandbox[phoneNumber] = &SandboxPhoneNumber{ + b.smsSandbox.Put(&SandboxPhoneNumber{ PhoneNumber: phoneNumber, LanguageCode: languageCode, Status: "Pending", CreationTimestamp: time.Now().UTC(), - } + }) return nil } @@ -3959,12 +4020,10 @@ func (b *InMemoryBackend) DeleteSMSSandboxPhoneNumber(phoneNumber string) error b.mu.Lock("DeleteSMSSandboxPhoneNumber") defer b.mu.Unlock() - if _, exists := b.smsSandbox[phoneNumber]; !exists { + if !b.smsSandbox.Delete(phoneNumber) { return ErrPhoneNumberNotFound } - delete(b.smsSandbox, phoneNumber) - return nil } @@ -3978,7 +4037,7 @@ func (b *InMemoryBackend) VerifySMSSandboxPhoneNumber(phoneNumber, oneTimePasswo b.mu.Lock("VerifySMSSandboxPhoneNumber") defer b.mu.Unlock() - entry, exists := b.smsSandbox[phoneNumber] + entry, exists := b.smsSandbox.Get(phoneNumber) if !exists { return ErrPhoneNumberNotFound } @@ -4014,8 +4073,8 @@ func (b *InMemoryBackend) ListSMSSandboxPhoneNumbers( // sortedSandboxNumbers returns sandbox phone numbers sorted by phone number. // Must be called with at least RLock held. func (b *InMemoryBackend) sortedSandboxNumbers() []SandboxPhoneNumber { - nums := make([]SandboxPhoneNumber, 0, len(b.smsSandbox)) - for _, n := range b.smsSandbox { + nums := make([]SandboxPhoneNumber, 0, b.smsSandbox.Len()) + for _, n := range b.smsSandbox.All() { nums = append(nums, *n) } @@ -4136,7 +4195,7 @@ func (b *InMemoryBackend) GetDataProtectionPolicy(resourceArn string) (string, e b.mu.RLock("GetDataProtectionPolicy") defer b.mu.RUnlock() - topic, exists := b.topics[resourceArn] + topic, exists := b.topics.Get(resourceArn) if !exists { return "", ErrTopicNotFound } @@ -4154,7 +4213,7 @@ func (b *InMemoryBackend) PutDataProtectionPolicy(resourceArn, policy string) er b.mu.Lock("PutDataProtectionPolicy") defer b.mu.Unlock() - topic, exists := b.topics[resourceArn] + topic, exists := b.topics.Get(resourceArn) if !exists { return ErrTopicNotFound } @@ -4255,22 +4314,36 @@ func (b *InMemoryBackend) Purge(ctx context.Context, cutoff time.Time) { } func (b *InMemoryBackend) purgeTopics(ctx context.Context, cutoff time.Time) { - for arn, topic := range b.topics { + // Collect eligible-for-deletion ARNs first rather than deleting from + // within Table.Range's own iteration. + var expired []string + + b.topics.Range(func(topic *Topic) bool { if ctx.Err() != nil { - return + return false } if topic.CreationTimestamp.Before(cutoff) { - delete(b.topics, arn) - delete(b.topicMessageArchive, arn) + expired = append(expired, topic.TopicArn) + } - if t := b.topicTags[arn]; t != nil { - t.Close() - delete(b.topicTags, arn) - } + return true + }) + + for _, arn := range expired { + b.topics.Delete(arn) + delete(b.topicMessageArchive, arn) + + if t := b.topicTags[arn]; t != nil { + t.Close() + delete(b.topicTags, arn) } } + if ctx.Err() != nil { + return + } + // Evict archived messages whose timestamp predates the cutoff, for topics that remain. for topicArn, archive := range b.topicMessageArchive { if ctx.Err() != nil { @@ -4293,48 +4366,80 @@ func (b *InMemoryBackend) purgeTopics(ctx context.Context, cutoff time.Time) { } func (b *InMemoryBackend) purgeSubscriptions(ctx context.Context, cutoff time.Time) { - for subArn, sub := range b.subscriptions { + var expired []string + + b.subscriptions.Range(func(sub *Subscription) bool { if ctx.Err() != nil { - return + return false } if sub.CreationTimestamp.Before(cutoff) { - delete(b.subscriptions, subArn) - // Also clean up the topic subscription index to prevent stale entries. - b.removeIndexedSubscription(sub.TopicArn, subArn) + expired = append(expired, sub.SubscriptionArn) } + + return true + }) + + for _, subArn := range expired { + // Table.Delete also removes the entry from subscriptionsByTopic, so no + // separate index cleanup (formerly removeIndexedSubscription) is needed. + b.subscriptions.Delete(subArn) } } func (b *InMemoryBackend) purgePlatformApplications(ctx context.Context, cutoff time.Time) { - for arn, app := range b.platformApplications { + var expired []string + + b.platformApplications.Range(func(app *PlatformApplication) bool { if ctx.Err() != nil { - return + return false } if app.CreationTimestamp.Before(cutoff) { - delete(b.platformApplications, arn) + expired = append(expired, app.PlatformApplicationArn) } + + return true + }) + + for _, arn := range expired { + b.platformApplications.Delete(arn) } } func (b *InMemoryBackend) purgePlatformEndpoints(ctx context.Context, cutoff time.Time) { - for arn, ep := range b.platformEndpoints { + var expired []string + + b.platformEndpoints.Range(func(ep *PlatformEndpoint) bool { if ctx.Err() != nil { - return + return false } if ep.CreationTimestamp.Before(cutoff) { - delete(b.platformEndpoints, arn) + expired = append(expired, ep.EndpointArn) } + + return true + }) + + for _, arn := range expired { + b.platformEndpoints.Delete(arn) } } func (b *InMemoryBackend) purgeSMSSandbox(ctx context.Context, cutoff time.Time) { - for phone, entry := range b.smsSandbox { + var expired []string + + b.smsSandbox.Range(func(entry *SandboxPhoneNumber) bool { if ctx.Err() != nil { - return + return false } if entry.CreationTimestamp.Before(cutoff) { - delete(b.smsSandbox, phone) + expired = append(expired, entry.PhoneNumber) } + + return true + }) + + for _, phone := range expired { + b.smsSandbox.Delete(phone) } } @@ -4351,15 +4456,9 @@ func (b *InMemoryBackend) Reset() { } } - b.topics = make(map[string]*Topic) - b.subscriptions = make(map[string]*Subscription) - // topicSubscriptions must be reset alongside subscriptions to avoid stale index entries. - b.topicSubscriptions = make(map[string]map[string]*Subscription) + b.registry.ResetAll() b.topicTags = make(map[string]*svcTags.Tags) - b.platformApplications = make(map[string]*PlatformApplication) - b.platformEndpoints = make(map[string]*PlatformEndpoint) b.topicMessageArchive = make(map[string][]*ArchivedMessage) - b.smsSandbox = make(map[string]*SandboxPhoneNumber) b.optedOutPhoneNumbers = make(map[string]bool) b.smsAttributes = make(map[string]string) b.smsDeliveries = nil diff --git a/services/sns/export_test.go b/services/sns/export_test.go index 25440e91c..d47ecdff6 100644 --- a/services/sns/export_test.go +++ b/services/sns/export_test.go @@ -73,7 +73,7 @@ func WaitDeliveriesForTest(b *InMemoryBackend) { // SigningCertURLForTest exposes the signer's certURL so tests can verify it // reflects the correct region rather than a hardcoded us-east-1. func SigningCertURLForTest(b *InMemoryBackend) string { - return b.signer.certURL + return b.signer.certURL() } // NewFifoDedupForTest creates a fifoDeduplication for white-box unit tests. diff --git a/services/sns/handler.go b/services/sns/handler.go index 7ce317fb4..b6a027a02 100644 --- a/services/sns/handler.go +++ b/services/sns/handler.go @@ -1929,8 +1929,13 @@ func extractBatchEntries(form url.Values) []batchEntry { return entries } - // Extract per-entry MessageAttributes. - prefix := fmt.Sprintf("PublishBatchRequestEntries.member.%d.", i) + // Extract per-entry MessageAttributes. The AWS query-protocol wire shape + // nests each entry's attribute map under its own "MessageAttributes" key + // (PublishBatchRequestEntries.member.N.MessageAttributes.entry.M.Name/Value...), + // matching the same shape a top-level Publish uses under "MessageAttributes.". + // A prior version omitted the "MessageAttributes." segment, so no batch + // entry's message attributes were ever parsed from a real SDK request. + prefix := fmt.Sprintf("PublishBatchRequestEntries.member.%d.MessageAttributes.", i) attrs := extractMessageAttributesWithPrefix(form, prefix) entries = append(entries, batchEntry{ diff --git a/services/sns/handler_test.go b/services/sns/handler_test.go index f41f6e8ad..2b16dad67 100644 --- a/services/sns/handler_test.go +++ b/services/sns/handler_test.go @@ -1498,6 +1498,76 @@ func TestSNSHandler_PublishBatch(t *testing.T) { } } +// Test_PublishBatchEntryMessageAttributesFilterPolicy verifies that a +// PublishBatch entry's MessageAttributes are read from the correct AWS +// query-protocol field name — PublishBatchRequestEntries.member.N.MessageAttributes.entry.M.* +// — and applied to subscription FilterPolicy matching. A prior version parsed +// PublishBatchRequestEntries.member.N.entry.M.* (missing the "MessageAttributes." +// segment), so every batch entry's message attributes were silently dropped and +// no batch publish could ever match an attribute-based FilterPolicy. +func Test_PublishBatchEntryMessageAttributesFilterPolicy(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + colorValue string + wantMatched bool + }{ + {name: "matching attribute delivers", colorValue: "red", wantMatched: true}, + {name: "non-matching attribute is filtered out", colorValue: "blue", wantMatched: false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + // Each subtest builds its own isolated backend, topic, subscriber, and server. + received := make(chan string, 1) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + received <- extractSNSHTTPMessage(string(body)) + w.WriteHeader(http.StatusOK) + })) + defer ts.Close() + + b := sns.NewInMemoryBackend() + h := sns.NewHandler(b) + topicArn := mustCreateTopic(t, b, "batch-attr-topic") + + sub, err := b.Subscribe(topicArn, "http", ts.URL, `{"color":["red"]}`) + require.NoError(t, err) + require.NotEmpty(t, sub.SubscriptionArn) + + rec := snsPost(t, h, url.Values{ + "Action": {"PublishBatch"}, + "Version": {"2010-03-31"}, + "TopicArn": {topicArn}, + "PublishBatchRequestEntries.member.1.Id": {"entry-1"}, + "PublishBatchRequestEntries.member.1.Message": {"batch-message"}, + "PublishBatchRequestEntries.member.1.MessageAttributes.entry.1.Name": {"color"}, + "PublishBatchRequestEntries.member.1.MessageAttributes.entry.1.Value.DataType": {"String"}, + "PublishBatchRequestEntries.member.1.MessageAttributes.entry.1.Value.StringValue": {tc.colorValue}, + }) + require.Equal(t, http.StatusOK, rec.Code) + require.Contains(t, rec.Body.String(), "entry-1") + + sns.WaitDeliveriesForTest(b) + + select { + case msg := <-received: + if !tc.wantMatched { + t.Fatalf("message delivered despite non-matching FilterPolicy: %q", msg) + } + assert.Equal(t, "batch-message", msg) + case <-time.After(300 * time.Millisecond): + if tc.wantMatched { + t.Fatal("message was not delivered despite matching FilterPolicy") + } + } + }) + } +} + func TestSNSHandler_Routing(t *testing.T) { t.Parallel() diff --git a/services/sns/lambda_firehose_delivery.go b/services/sns/lambda_firehose_delivery.go index 9dffa0c4d..f0f1e54a1 100644 --- a/services/sns/lambda_firehose_delivery.go +++ b/services/sns/lambda_firehose_delivery.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "strings" - "time" "github.com/google/uuid" @@ -40,7 +39,24 @@ type snsLambdaMessageAttr struct { Value string `json:"Value"` } +// subscriptionUnsubscribeURL builds the AWS-shape unsubscribe link embedded in +// notification envelopes: https://sns..amazonaws.com/?Action=Unsubscribe&SubscriptionArn=. +// The region is derived from topicARN; it falls back to "us-east-1" for a +// malformed ARN so the URL is always well-formed. +func subscriptionUnsubscribeURL(topicARN, subscriptionARN string) string { + region := arnRegion(topicARN) + if region == "" { + region = "us-east-1" + } + + return "https://sns." + region + ".amazonaws.com/?Action=Unsubscribe&SubscriptionArn=" + subscriptionARN +} + // buildLambdaPayload constructs the SNS → Lambda invocation payload per the AWS spec. +// The Timestamp/Signature/SigningCertURL are the real values computed once per +// Publish call (see buildPublishedEvent) rather than fabricated per-delivery, so a +// Lambda function that verifies the SNS signature (as AWS recommends) succeeds +// against the same envelope other channels (SQS, HTTP/HTTPS) receive. func buildLambdaPayload( ev *events.SNSPublishedEvent, sub events.SNSSubscriptionSnapshot, @@ -60,11 +76,11 @@ func buildLambdaPayload( TopicArn: ev.TopicARN, Subject: ev.Subject, Message: ev.Message, - Timestamp: time.Now().UTC().Format(time.RFC3339), - SignatureVersion: "1", - Signature: uuid.NewString(), - SigningCertURL: "", - UnsubscribeURL: "", + Timestamp: ev.Timestamp, + SignatureVersion: "2", + Signature: ev.Signature, + SigningCertURL: ev.SigningCertURL, + UnsubscribeURL: subscriptionUnsubscribeURL(ev.TopicARN, sub.SubscriptionARN), MessageAttributes: attrs, }, } @@ -77,6 +93,34 @@ func buildLambdaPayload( return payload } +// buildFirehoseEnvelope wraps the published message in the same SNS Notification +// JSON envelope AWS sends to SQS/HTTP subscribers, for delivery to a Firehose +// subscription whose RawMessageDelivery attribute is false (the AWS default). +// Reuses the Timestamp/Signature/SigningCertURL computed once for this publish. +func buildFirehoseEnvelope(ev *events.SNSPublishedEvent, sub events.SNSSubscriptionSnapshot) []byte { + env := snsHTTPNotification{ + Type: messageTypeNotification, + MessageID: ev.MessageID, + TopicArn: ev.TopicARN, + Message: ev.Message, + Timestamp: ev.Timestamp, + SignatureVersion: "2", + Signature: ev.Signature, + SigningCertURL: ev.SigningCertURL, + UnsubscribeURL: subscriptionUnsubscribeURL(ev.TopicARN, sub.SubscriptionARN), + } + if ev.Subject != "" { + env.Subject = ev.Subject + } + + enc, err := json.Marshal(env) + if err != nil { + return []byte(ev.Message) + } + + return enc +} + // deliverToLambdaSubscriptions invokes each Lambda-protocol subscription endpoint. // On invocation failure, the message is forwarded to the subscription DLQ when // a RedrivePolicy is configured and a SQSSender is wired. @@ -90,7 +134,7 @@ func (b *InMemoryBackend) deliverToLambdaSubscriptions(ev *events.SNSPublishedEv b.mu.RLock("lambda-topic-policy") var topicEffectivePolicy string - if topic, ok := b.topics[ev.TopicARN]; ok { + if topic, ok := b.topics.Get(ev.TopicARN); ok { topicEffectivePolicy = topic.Attributes["EffectiveDeliveryPolicy"] } b.mu.RUnlock() @@ -123,8 +167,10 @@ func (b *InMemoryBackend) deliverToLambdaSubscriptions(ev *events.SNSPublishedEv } // deliverToFirehoseSubscriptions puts each Firehose-protocol subscription message as a batch record. -// On delivery failure, the message is forwarded to the subscription DLQ when a RedrivePolicy is -// configured and a SQSSender is wired — matching the HTTP/HTTPS and Lambda paths. +// When the subscription's RawMessageDelivery attribute is false (the AWS default), the record is +// the full SNS Notification JSON envelope, matching real AWS SNS→Firehose delivery; RawMessageDelivery=true +// delivers the bare message body. On delivery failure, the message is forwarded to the subscription DLQ +// when a RedrivePolicy is configured and a SQSSender is wired — matching the HTTP/HTTPS and Lambda paths. func (b *InMemoryBackend) deliverToFirehoseSubscriptions(ev *events.SNSPublishedEvent) { firehose := b.firehoseBackend if firehose == nil { @@ -135,7 +181,7 @@ func (b *InMemoryBackend) deliverToFirehoseSubscriptions(ev *events.SNSPublished b.mu.RLock("firehose-topic-policy") var topicEffectivePolicy string - if topic, ok := b.topics[ev.TopicARN]; ok { + if topic, ok := b.topics.Get(ev.TopicARN); ok { topicEffectivePolicy = topic.Attributes["EffectiveDeliveryPolicy"] } b.mu.RUnlock() @@ -150,10 +196,15 @@ func (b *InMemoryBackend) deliverToFirehoseSubscriptions(ev *events.SNSPublished continue } + record := []byte(ev.Message) + if !sub.RawMessageDelivery { + record = buildFirehoseEnvelope(ev, sub) + } + numRetries := getRetryConfig(topicEffectivePolicy, sub.DeliveryPolicy, protocolFirehose) var err error for i := 0; i <= numRetries; i++ { - _, err = firehose.PutRecordBatch(streamName, [][]byte{[]byte(ev.Message)}) + _, err = firehose.PutRecordBatch(streamName, [][]byte{record}) if err == nil { b.logDeliveryStatus(b.svcCtx, ev.TopicARN, protocolFirehose, sub.Endpoint, "SUCCESS", nil) @@ -164,7 +215,7 @@ func (b *InMemoryBackend) deliverToFirehoseSubscriptions(ev *events.SNSPublished if err != nil { b.logDeliveryStatus(b.svcCtx, ev.TopicARN, protocolFirehose, sub.Endpoint, "FAILURE", err) if sub.RedrivePolicy != "" && sqsSender != nil { - sendLambdaDLQ(b.svcCtx, sqsSender, sub.RedrivePolicy, ev.Message) + sendLambdaDLQ(b.svcCtx, sqsSender, sub.RedrivePolicy, string(record)) } } } @@ -208,7 +259,7 @@ func (b *InMemoryBackend) deliverToApplicationSubscriptions(ev *events.SNSPublis } b.mu.RLock("deliverToApplicationSubscriptions") - ep, exists := b.platformEndpoints[sub.Endpoint] + ep, exists := b.platformEndpoints.Get(sub.Endpoint) enabled := exists && ep.Attributes["Enabled"] != boolFalseStr b.mu.RUnlock() @@ -219,11 +270,11 @@ func (b *InMemoryBackend) deliverToApplicationSubscriptions(ev *events.SNSPublis msgID := uuid.New().String() b.mu.Lock("deliverToApplicationSubscriptions-record") - b.applicationDeliveries = append(b.applicationDeliveries, ApplicationDelivery{ + b.applicationDeliveries = appendBounded(b.applicationDeliveries, ApplicationDelivery{ EndpointARN: sub.Endpoint, Message: ev.Message, MessageID: msgID, - }) + }, maxRecordedDeliveries) b.mu.Unlock() } } diff --git a/services/sns/lambda_firehose_delivery_test.go b/services/sns/lambda_firehose_delivery_test.go index 67cd06c8c..08b2630c1 100644 --- a/services/sns/lambda_firehose_delivery_test.go +++ b/services/sns/lambda_firehose_delivery_test.go @@ -235,7 +235,42 @@ func TestSNS_FirehoseDelivery_PutsRecord(t *testing.T) { records := firehose.RecordsFor(streamName) require.Len(t, records, 1) - assert.Equal(t, "firehose-message", string(records[0])) + + // AWS SNS→Firehose delivery defaults to RawMessageDelivery=false (the + // subscription created above never sets it), so the delivered record is the + // full SNS Notification JSON envelope, not the bare message body. + var envelope map[string]any + require.NoError(t, json.Unmarshal(records[0], &envelope)) + assert.Equal(t, "Notification", envelope["Type"]) + assert.Equal(t, "firehose-message", envelope["Message"]) + assert.Equal(t, topicARN, envelope["TopicArn"]) + assert.NotEmpty(t, envelope["Signature"]) + assert.NotEmpty(t, envelope["SigningCertURL"]) +} + +func TestSNS_FirehoseDelivery_RawMessageDeliveryPutsBareMessage(t *testing.T) { + t.Parallel() + + b := newTestSNSBackend(t) + firehose := newMockFirehose() + b.SetFirehoseBackend(firehose) + + streamName := "raw-delivery-stream" + streamARN := "arn:aws:firehose:us-east-1:123456789012:deliverystream/" + streamName + + topic, err := b.CreateTopic("raw-firehose-topic", nil) + require.NoError(t, err) + + sub, err := b.Subscribe(topic.TopicArn, "firehose", streamARN, "") + require.NoError(t, err) + require.NoError(t, b.SetSubscriptionAttributes(sub.SubscriptionArn, "RawMessageDelivery", "true")) + + _, err = b.Publish(topic.TopicArn, "raw-message", "", "", nil) + require.NoError(t, err) + + records := firehose.RecordsFor(streamName) + require.Len(t, records, 1) + assert.Equal(t, "raw-message", string(records[0])) } func TestSNS_FirehoseDelivery_NoBackendDoesNothing(t *testing.T) { diff --git a/services/sns/persistence.go b/services/sns/persistence.go index 1f8ecd3c2..ac866341b 100644 --- a/services/sns/persistence.go +++ b/services/sns/persistence.go @@ -2,24 +2,50 @@ package sns import ( "context" + "encoding/json" + "fmt" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" svcTags "github.com/blackbirdworks/gopherstack/pkgs/tags" ) +// snsSnapshotVersion identifies the shape of backendSnapshot's Tables blob +// (i.e. the set of resources registered on b.registry -- see registerAllTables +// in store_setup.go). It must be bumped whenever a change there would make an +// older snapshot unsafe to decode as the current shape. Restore compares this +// against the persisted value and discards (rather than attempts to partially +// decode) any mismatch -- see Restore below. This mirrors the services/sqs +// pilot (commit 0f09d77c) and the services/ec2 sweep (commit 12e611a4). +// +// Snapshots written before this field existed decode with Version == 0, which +// never equals snsSnapshotVersion, so they are discarded the same way as any +// other incompatible version rather than partially decoded. +const snsSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the SNS backend. +// +// Tables holds one JSON-encoded array per registered store.Table, produced by +// [store.Registry.SnapshotAll] -- "topics", "subscriptions", +// "platformApplications", "platformEndpoints", and "smsSandbox". The remaining +// fields below are resource maps left un-registered (raw) because their value +// type is not a pure keyed identity or is slice/scalar-valued -- see the +// comment on registerAllTables in store_setup.go for why each one. type backendSnapshot struct { - Topics map[string]*Topic `json:"topics"` - Subscriptions map[string]*Subscription `json:"subscriptions"` + Tables map[string]json.RawMessage `json:"tables"` TopicTags map[string]*svcTags.Tags `json:"topicTags"` - PlatformApplications map[string]*PlatformApplication `json:"platformApplications,omitempty"` - PlatformEndpoints map[string]*PlatformEndpoint `json:"platformEndpoints,omitempty"` - SMSSandbox map[string]*SandboxPhoneNumber `json:"smsSandbox,omitempty"` OptedOutPhoneNumbers map[string]bool `json:"optedOutPhoneNumbers,omitempty"` SMSAttributes map[string]string `json:"smsAttributes,omitempty"` OriginationNumbers map[string][]XMLOriginationPhone `json:"originationNumbers,omitempty"` - SMSSandboxEnabled *bool `json:"smsSandboxEnabled,omitempty"` - AccountID string `json:"accountID"` - Region string `json:"region"` + // TopicMessageArchive holds per-topic ArchivePolicy message archives, keyed by + // topic ARN. Without persisting this, a restart (or snapshot restore) silently + // discards all archived messages, so ReplayPolicy subscriptions set up before + // the restart can no longer replay anything. + TopicMessageArchive map[string][]*ArchivedMessage `json:"topicMessageArchive,omitempty"` + SMSSandboxEnabled *bool `json:"smsSandboxEnabled,omitempty"` + AccountID string `json:"accountID"` + Region string `json:"region"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. @@ -28,17 +54,26 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "sns: snapshot table marshal failed", "error", err) + + return nil + } + sandboxEnabled := b.smsSandboxEnabled snap := backendSnapshot{ - Topics: b.topics, - Subscriptions: b.subscriptions, + Version: snsSnapshotVersion, + Tables: tables, TopicTags: b.topicTags, - PlatformApplications: b.platformApplications, - PlatformEndpoints: b.platformEndpoints, - SMSSandbox: b.smsSandbox, OptedOutPhoneNumbers: b.optedOutPhoneNumbers, SMSAttributes: b.smsAttributes, OriginationNumbers: b.originationNumbers, + TopicMessageArchive: b.topicMessageArchive, AccountID: b.accountID, Region: b.region, SMSSandboxEnabled: &sandboxEnabled, @@ -47,6 +82,31 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { return persistence.MarshalSnapshot(ctx, "sns", snap) } +// fillDefaults replaces any nil map field with an empty, non-nil map so older +// snapshots (written before a field existed) restore cleanly instead of leaving +// the backend with a nil map that panics on first write. +func (snap *backendSnapshot) fillDefaults() { + if snap.TopicTags == nil { + snap.TopicTags = make(map[string]*svcTags.Tags) + } + + if snap.OptedOutPhoneNumbers == nil { + snap.OptedOutPhoneNumbers = make(map[string]bool) + } + + if snap.SMSAttributes == nil { + snap.SMSAttributes = make(map[string]string) + } + + if snap.OriginationNumbers == nil { + snap.OriginationNumbers = make(map[string][]XMLOriginationPhone) + } + + if snap.TopicMessageArchive == nil { + snap.TopicMessageArchive = make(map[string][]*ArchivedMessage) + } +} + // Restore loads backend state from a JSON snapshot. // It implements persistence.Persistable. // The event emitter is not restored — it is re-wired by the CLI after restore. @@ -60,51 +120,33 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() - if snap.Topics == nil { - snap.Topics = make(map[string]*Topic) - } - - if snap.Subscriptions == nil { - snap.Subscriptions = make(map[string]*Subscription) - } - - if snap.TopicTags == nil { - snap.TopicTags = make(map[string]*svcTags.Tags) - } - - if snap.PlatformApplications == nil { - snap.PlatformApplications = make(map[string]*PlatformApplication) - } - - if snap.PlatformEndpoints == nil { - snap.PlatformEndpoints = make(map[string]*PlatformEndpoint) - } + if snap.Version != snsSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/sqs pilot (commit 0f09d77c). + logger.Load(ctx).WarnContext(ctx, + "sns: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", snsSnapshotVersion) - if snap.SMSSandbox == nil { - snap.SMSSandbox = make(map[string]*SandboxPhoneNumber) - } + b.registry.ResetAll() - if snap.OptedOutPhoneNumbers == nil { - snap.OptedOutPhoneNumbers = make(map[string]bool) + return nil } - if snap.SMSAttributes == nil { - snap.SMSAttributes = make(map[string]string) - } + snap.fillDefaults() - if snap.OriginationNumbers == nil { - snap.OriginationNumbers = make(map[string][]XMLOriginationPhone) + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("sns: restore snapshot tables: %w", err) } - b.topics = snap.Topics - b.subscriptions = snap.Subscriptions b.topicTags = snap.TopicTags - b.platformApplications = snap.PlatformApplications - b.platformEndpoints = snap.PlatformEndpoints - b.smsSandbox = snap.SMSSandbox b.optedOutPhoneNumbers = snap.OptedOutPhoneNumbers b.smsAttributes = snap.SMSAttributes b.originationNumbers = snap.OriginationNumbers + b.topicMessageArchive = snap.TopicMessageArchive b.accountID = snap.AccountID b.region = snap.Region if snap.SMSSandboxEnabled != nil { @@ -113,21 +155,15 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.smsSandboxEnabled = true // default for old snapshots that lack this field } - // Rebuild the per-topic subscription index and restore the parsed filter-policy - // cache for each subscription (both are transient and not persisted). - b.topicSubscriptions = make(map[string]map[string]*Subscription, len(b.topics)) - for topicArn := range b.topics { - b.topicSubscriptions[topicArn] = make(map[string]*Subscription) - } - - for _, sub := range b.subscriptions { - if _, ok := b.topicSubscriptions[sub.TopicArn]; !ok { - b.topicSubscriptions[sub.TopicArn] = make(map[string]*Subscription) - } - - b.topicSubscriptions[sub.TopicArn][sub.SubscriptionArn] = sub - // Restore parsed filter policy; ignore errors so a future stricter validation - // upgrade does not break loading older snapshots. + // Restore the parsed filter-policy cache for each subscription -- it is + // transient (unexported field, never marshaled) and not persisted. + // subscriptionsByTopic is rebuilt automatically by b.registry.RestoreAll + // above (store.Table.Restore repopulates every registered secondary index + // from scratch), so no manual topicSubscriptions reconstruction is needed + // here anymore. + for _, sub := range b.subscriptions.All() { + // Ignore errors so a future stricter validation upgrade does not break + // loading older snapshots. sub.parsedFilterPolicy, _ = parseFilterPolicy(sub.FilterPolicy) } diff --git a/services/sns/persistence_test.go b/services/sns/persistence_test.go index a7c6fa6bf..d7403b4e5 100644 --- a/services/sns/persistence_test.go +++ b/services/sns/persistence_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/blackbirdworks/gopherstack/pkgs/tags" "github.com/blackbirdworks/gopherstack/services/sns" ) @@ -74,3 +75,139 @@ func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { err := b.Restore(t.Context(), []byte("not-valid-json")) require.Error(t, err) } + +// TestInMemoryBackend_SnapshotRestore_FullState exercises a full-state +// Snapshot->Restore round trip covering every resource converted to +// pkgs/store (topics, subscriptions, platform applications/endpoints, SMS +// sandbox) plus every raw map left un-registered (topic tags, opted-out phone +// numbers, SMS attributes, origination numbers, and the ArchivePolicy message +// archive). It specifically checks state that is NOT itself JSON-marshaled -- +// the subscriptionsByTopic secondary index and the unexported +// parsedFilterPolicy cache -- by observing their effect on real API behaviour +// (SubscriptionsConfirmed/Pending counts and FilterPolicy-based delivery +// filtering) after restoring into a brand new backend instance. +func TestInMemoryBackend_SnapshotRestore_FullState(t *testing.T) { + t.Parallel() + + original := sns.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + + topic, err := original.CreateTopic("full-state-topic", map[string]string{ + "ArchivePolicy": `{"MessageRetentionSeconds":60}`, + }) + require.NoError(t, err) + + // A confirmed, filtered email subscription: proves both the + // subscriptionsByTopic index and the unexported parsedFilterPolicy cache + // survive the round trip, since neither is itself JSON-serialised. + sub, err := original.Subscribe(topic.TopicArn, "email", "filtered@example.com", `{"kind":["match"]}`) + require.NoError(t, err) + _, err = original.ConfirmSubscription(topic.TopicArn, "token") + require.NoError(t, err) + + // A second, unfiltered SQS subscription on the same topic so the topic + // index has more than one member to rebuild. + _, err = original.Subscribe(topic.TopicArn, "sqs", "arn:aws:sqs:us-east-1:000000000000:q", "") + require.NoError(t, err) + + original.SetTopicTags(topic.TopicArn, tags.FromMap("test", map[string]string{"env": "prod"})) + + app, err := original.CreatePlatformApplication( + "full-state-app", "GCM", map[string]string{"PlatformCredential": "cred"}, + ) + require.NoError(t, err) + + endpoint, err := original.CreatePlatformEndpoint(app.PlatformApplicationArn, "device-token", nil) + require.NoError(t, err) + + require.NoError(t, original.CreateSMSSandboxPhoneNumber("+15550001111", "en")) + + sns.AddOptedOutPhoneNumberForTest(original, "+15550002222") + + require.NoError(t, original.SetSMSAttributes(map[string]string{"DefaultSMSType": "Transactional"})) + + original.SeedOriginationNumber(sns.XMLOriginationPhone{PhoneNumber: "+15550003333", Iso2CountryCode: "US"}) + + // Archive a message so topicMessageArchive (the parity-sweep + // ArchivePolicy/ReplayPolicy buffer) has content to round-trip. + _, err = original.Publish(topic.TopicArn, "archived-message-body", "", "", nil) + require.NoError(t, err) + + snap := original.Snapshot(t.Context()) + require.NotNil(t, snap) + + fresh := sns.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + require.NoError(t, fresh.Restore(t.Context(), snap)) + + // Topic + attributes. + attrs, err := fresh.GetTopicAttributes(topic.TopicArn) + require.NoError(t, err) + assert.Equal(t, `{"MessageRetentionSeconds":60}`, attrs["ArchivePolicy"]) + // Both subscriptions confirmed by this point. + assert.Equal(t, "2", attrs["SubscriptionsConfirmed"]) + assert.Equal(t, "0", attrs["SubscriptionsPending"]) + + // subscriptionsByTopic index rebuilt: ListSubscriptionsByTopic must see both. + subsByTopic, _, err := fresh.ListSubscriptionsByTopic(topic.TopicArn, "") + require.NoError(t, err) + assert.Len(t, subsByTopic, 2) + + // topicMessageArchive (raw map, parity-sweep ArchivePolicy replay buffer). + // Checked here, before the additional Publish calls below add more entries. + archived := fresh.GetArchivedMessages(topic.TopicArn) + require.Len(t, archived, 1) + assert.Equal(t, "archived-message-body", archived[0].Message) + + // parsedFilterPolicy rebuilt: publishing a matching vs non-matching + // message must change what collectPublishTargets records for the email + // subscription, proving the raw FilterPolicy string was re-parsed after + // restore rather than left as an empty/zero parsedFilterPolicy. + subAttrs, err := fresh.GetSubscriptionAttributes(sub.SubscriptionArn) + require.NoError(t, err) + assert.JSONEq(t, `{"kind":["match"]}`, subAttrs["FilterPolicy"]) + + _, err = fresh.Publish(topic.TopicArn, "no-match-body", "", "", map[string]sns.MessageAttribute{ + "kind": {DataType: "String", StringValue: "nomatch"}, + }) + require.NoError(t, err) + assert.Empty(t, fresh.DrainEmailDeliveries(), "filter policy should reject a non-matching attribute") + + _, err = fresh.Publish(topic.TopicArn, "match-body", "", "", map[string]sns.MessageAttribute{ + "kind": {DataType: "String", StringValue: "match"}, + }) + require.NoError(t, err) + assert.Len(t, fresh.DrainEmailDeliveries(), 1, "filter policy should accept a matching attribute") + + // Tags. + assert.Equal(t, map[string]string{"env": "prod"}, fresh.GetTopicTags(topic.TopicArn)) + + // Platform application + endpoint. + appAttrs, err := fresh.GetPlatformApplicationAttributes(app.PlatformApplicationArn) + require.NoError(t, err) + assert.Equal(t, "cred", appAttrs["PlatformCredential"]) + + epAttrs, err := fresh.GetEndpointAttributes(endpoint.EndpointArn) + require.NoError(t, err) + assert.Equal(t, "device-token", epAttrs["Token"]) + + // SMS sandbox. + sandboxNums, _, err := fresh.ListSMSSandboxPhoneNumbers("", 0) + require.NoError(t, err) + require.Len(t, sandboxNums, 1) + assert.Equal(t, "+15550001111", sandboxNums[0].PhoneNumber) + + // Opted-out phone numbers (raw map, left un-registered). + optedOut, err := fresh.CheckIfPhoneNumberIsOptedOut("+15550002222") + require.NoError(t, err) + assert.True(t, optedOut) + + // SMS attributes (raw map, left un-registered). + smsAttrs, err := fresh.GetSMSAttributes(nil) + require.NoError(t, err) + assert.Equal(t, "Transactional", smsAttrs["DefaultSMSType"]) + + // Origination numbers (raw map, left un-registered). + origNums, _, err := fresh.ListOriginationNumbers("", 0) + require.NoError(t, err) + require.Len(t, origNums, 1) + assert.Equal(t, "+15550003333", origNums[0].PhoneNumber) +} diff --git a/services/sns/store_setup.go b/services/sns/store_setup.go new file mode 100644 index 000000000..aa4b947a5 --- /dev/null +++ b/services/sns/store_setup.go @@ -0,0 +1,49 @@ +package sns + +// Code in this file supports Phase 3.3 of the datalayer refactor: every +// map[string]*T resource field on InMemoryBackend that has a value type whose +// identity is a pure function of its own fields is registered exactly once, +// here, as a *store.Table[T] on b.registry. See pkgs/store's package doc and +// the services/sqs pilot (commit 0f09d77c) / services/ec2 sweep (commit +// 12e611a4) for the pattern this follows. +// +// A handful of fields are deliberately NOT registered here and remain plain +// maps -- see the comment block above registerAllTables for the list and why. +import "github.com/blackbirdworks/gopherstack/pkgs/store" + +func topicKeyFn(v *Topic) string { return v.TopicArn } +func subscriptionKeyFn(v *Subscription) string { return v.SubscriptionArn } +func subscriptionTopicKeyFn(v *Subscription) string { return v.TopicArn } +func platformApplicationKeyFn(v *PlatformApplication) string { return v.PlatformApplicationArn } +func platformEndpointKeyFn(v *PlatformEndpoint) string { return v.EndpointArn } +func sandboxPhoneKeyFn(v *SandboxPhoneNumber) string { return v.PhoneNumber } + +// registerAllTables registers every converted resource map on b.registry +// exactly once. It must be called during construction only (immediately after +// b.registry is created), never on every Reset() -- store.Register panics on a +// duplicate name, so runtime resets go through registry.ResetAll() instead +// (see InMemoryBackend.Reset in backend.go). +// +// The following resource fields are deliberately left as plain maps (not +// registered here) because their value type is not a pure keyed identity or +// carries a slice/scalar value store.Table cannot key by: +// - topicTags: value type *tags.Tags carries no identity field of its own; +// keyed externally by topic ARN (same shape as EC2's VerifiedAccessPolicy +// maps left raw in services/ec2/store_setup.go) +// - optedOutPhoneNumbers: value type is bool, which store.Table cannot key +// (no identity field at all) +// - smsAttributes: value type is string, same reason as above +// - originationNumbers: slice-valued (map[string][]XMLOriginationPhone) +// - topicMessageArchive: slice-valued (map[string][]*ArchivedMessage); this +// is the parity-sweep ArchivePolicy/ReplayPolicy replay buffer and must +// keep its own persistence path (see persistence.go) +func registerAllTables(b *InMemoryBackend) { + b.topics = store.Register(b.registry, "topics", store.New(topicKeyFn)) + b.subscriptions = store.Register(b.registry, "subscriptions", store.New(subscriptionKeyFn)) + b.subscriptionsByTopic = b.subscriptions.AddIndex("topic", subscriptionTopicKeyFn) + b.platformApplications = store.Register( + b.registry, "platformApplications", store.New(platformApplicationKeyFn), + ) + b.platformEndpoints = store.Register(b.registry, "platformEndpoints", store.New(platformEndpointKeyFn)) + b.smsSandbox = store.Register(b.registry, "smsSandbox", store.New(sandboxPhoneKeyFn)) +} diff --git a/services/sqs/PARITY.md b/services/sqs/PARITY.md new file mode 100644 index 000000000..37947c7eb --- /dev/null +++ b/services/sqs/PARITY.md @@ -0,0 +1,131 @@ +--- +service: sqs +sdk_module: aws-sdk-go-v2/service/sqs@v1.44.2 +last_audit_commit: 58e50f3a +last_audit_date: 2026-07-05 +overall: A +# Per-op or per-op-family status. Values: ok | partial | gap | deferred. +# wire=response/request shape vs SDK; errors=code+HTTP status; state=real mutate/read; persist=in backendSnapshot. +ops: + CreateQueue: {wire: ok, errors: ok, state: ok, persist: ok, note: "idempotent-on-existing-config check via isConfigurableQueueAttribute; RedrivePolicy applied inline so denyAll/byQueue RedriveAllowPolicy is now enforced at create time too (see families.dlq_redrive)"} + DeleteQueue: {wire: ok, errors: ok, state: ok, persist: ok, note: "closes q.notify (wakes long-pollers with QueueDoesNotExist), closes tag store, cancels involved move tasks"} + ListQueues: {wire: ok, errors: ok, state: ok, persist: n/a, note: "region-scoped, prefix filter, pkgs/page pagination"} + GetQueueUrl: {wire: ok, errors: ok, state: ok, persist: n/a} + GetQueueAttributes: {wire: ok, errors: ok, state: ok, persist: n/a, note: "dynamic attrs (ApproximateNumberOfMessages/NotVisible/Delayed) computed live from q.delayedCount + slice lens under q.mu"} + SetQueueAttributes: {wire: ok, errors: ok, state: ok, persist: ok, note: "FifoQueue immutable; full range validation (VisibilityTimeout/DelaySeconds/MessageRetentionPeriod/MaximumMessageSize/KmsDataKeyReusePeriodSeconds/FIFO enum attrs/RedriveAllowPolicy shape); RedriveAllowPolicy enforcement added this pass"} + SendMessage: {wire: ok, errors: ok, state: ok, persist: ok, note: "MD5OfBody + MD5OfMessageAttributes + MD5OfMessageSystemAttributes all correct AWS byte-packing algorithm; FIFO validated (MessageGroupId required, no per-message DelaySeconds, dedup ID required unless content-based); delay/queue-default resolution correct"} + ReceiveMessage: {wire: ok, errors: partial->ok, state: partial->ok, persist: n/a, note: "fixed this pass: (1) VisibilityTimeout range was validated only in the JSON handler, not centrally — Query protocol silently accepted out-of-range values; (2) re-queued (visibility-expired or explicitly zeroed) FIFO messages were appended to the tail of the pending list instead of reinserted by SequenceNumber, letting a newer same-group message jump ahead — see families.fifo_ordering"} + DeleteMessage: {wire: ok, errors: ok, state: ok, persist: ok, note: "O(1) via inFlightByHandle (#56)"} + ChangeMessageVisibility: {wire: ok, errors: ok, state: partial->ok, persist: ok, note: "0-timeout re-queue had the same FIFO tail-append ordering bug as ReceiveMessage's janitor sweep; both now go through the shared requeueMessage helper"} + SendMessageBatch: {wire: ok, errors: ok, state: ok, persist: ok, note: "per-entry BatchResultErrorEntry, 10-entry cap, BatchRequestTooLong on combined-size overflow, order preserved"} + DeleteMessageBatch: {wire: ok, errors: ok, state: ok, persist: ok, note: "batch-level QueueDoesNotExist, per-entry delegates to DeleteMessage"} + ChangeMessageVisibilityBatch: {wire: ok, errors: ok, state: ok, persist: ok} + PurgeQueue: {wire: ok, errors: ok, state: ok, persist: ok, note: "60s cooldown enforced (PurgeQueueInProgress); FIFO dedup state reset on purge"} + TagQueue/UntagQueue/ListQueueTags: {wire: ok, errors: ok, state: ok, persist: ok, note: "pkgs/tags-backed"} + ListDeadLetterSourceQueues: {wire: ok, errors: ok, state: ok, persist: n/a} + AddPermission/RemovePermission: {wire: ok, errors: ok, state: ok, persist: ok, note: "rebuilds an IAM policy doc into Attributes[Policy], deterministic (sorted labels)"} + StartMessageMoveTask: {wire: ok, errors: ok, state: ok, persist: partial, note: "RUNNING tasks are correctly NOT persisted (goroutine can't resume); default-destination lookup via RedrivePolicy scan; rate-limited via ticker; TOCTOU-safe under b.mu"} + CancelMessageMoveTask: {wire: ok, errors: ok, state: ok, persist: ok} + ListMessageMoveTasks: {wire: ok, errors: ok, state: ok, persist: ok, note: "TaskHandle only populated for RUNNING (AWS behavior); MaxResults default 1 / cap 10; newest-first"} +families: + message_attribute_md5: {status: ok, note: "computeMD5OfMessageAttributes matches the AWS wire algorithm exactly: sorted names, 4-byte-BE-length-prefixed name/dataType/value, 1-byte transport type (1=String|Number, 2=Binary); subset-MD5 on filtered receive re-hashes only when the returned set is a strict subset, reuses the send-time digest otherwise"} + fifo_dedup: {status: ok, note: "explicit MessageDeduplicationId vs ContentBasedDeduplication (SHA-256 of body, NOT MD5) correctly mutually validated; 5-minute window; DeduplicationScope=queue|messageGroup key scoping; bounded map (100k) with oldest-expiry eviction + janitor sweep"} + fifo_ordering: {status: ok, note: "fixed this pass (see ReceiveMessage/ChangeMessageVisibility above): requeueMessage now reinserts by SequenceNumber (fixed-width zero-padded decimal, so lexicographic sort == numeric sort) instead of appending to the tail, preserving strict per-MessageGroupId ordering across visibility resets. Confirmed correct behavior (not a bug): only one message per group may be in-flight at a time — this matches real AWS FIFO semantics, not an over-restriction"} + fifo_throughput_limit: {status: partial, note: "perMessageGroupId (300 TPS/group) enforced via checkFIFOPerGroupRateLimit; perQueue (the AWS default, ~3000/300 TPS) has no limiter at all — see gaps"} + visibility_timeout_and_inflight: {status: ok, note: "12h (43200s) max validated on both ChangeMessageVisibility and now ReceiveMessage (backend-level, was JSON-only before this pass); in-flight caps 120k standard / 20k FIFO -> OverLimit; sweepInFlight/pickVisibleMessages single-pass janitor+receive-path cleanup"} + dlq_redrive: {status: partial->ok, note: "fixed this pass: RedriveAllowPolicy (allowAll/denyAll/byQueue+sourceQueueArns) was shape-validated by validateRedriveAllowPolicy but never enforced — any source queue could point RedrivePolicy at any DLQ regardless of the DLQ's declared permission. checkRedriveAllowPolicy now enforces it in applyRedrivePolicy (shared by CreateQueue/SetQueueAttributes/Restore). maxReceiveCount routing (tryRouteToDLQ), DLQ must be same region + same FIFO-ness, StartMessageMoveTask default-destination-by-RedrivePolicy all verified correct"} + delay_queues: {status: ok, note: "queue-level DelaySeconds + per-message DelaySeconds (message wins), FIFO rejects per-message delay, delayedCount maintained incrementally for O(1) GetQueueAttributes"} + long_polling: {status: ok, note: "broadcast notify-channel-close-and-replace pattern wakes all waiters on send or 0-timeout visibility reset; 1s recheck interval catches janitor-driven requeues without a new SendMessage"} + receive_request_attempt_id: {status: ok, note: "5-minute exactly-once-retry cache keyed by ReceiveRequestAttemptId, pruned alongside dedup IDs"} + error_codes: {status: ok, note: "Query protocol correctly uses legacy codes (AWS.SimpleQueueService.NonExistentQueue, AWS.SimpleQueueService.PurgeQueueInProgress, etc.) vs JSON protocol's com.amazonaws.sqs# namespaced codes; queryErrorDetails/errorDetails share the JSON table for everything except QueueDoesNotExist's legacy override"} + persistence: {status: partial->ok, note: "fixed this pass: (1) hasActivity (janitor skip-idle-queue flag) was never restored, so a restored non-FIFO queue with pending messages was silently invisible to the background janitor until an unrelated SendMessage touched it again; (2) fifoSeqCounter was not persisted, so SequenceNumber could regress/duplicate for a FIFO queue that already had messages sent before a snapshot/restore; (3) lastPurgedAt (PurgeQueue 60s cooldown) was not persisted, resetting the cooldown on every restart"} +gaps: + - "FifoThroughputLimit=perQueue (AWS default) is not rate-limited at all; only perMessageGroupId is. (bd: gopherstack-qgh)" + - "sns_delivery.go's internal SendMessage calls for SNS->SQS fan-out and DLQ redirect never pass a Region, so a subscribed queue in a non-default region is unreachable via SNS delivery. (bd: gopherstack-qgh)" + - "KMS SSE (SqsManagedSseEnabled/KmsMasterKeyId/KmsDataKeyReusePeriodSeconds) are accepted, range/shape-validated, and round-trip through GetQueueAttributes, but no actual encryption is modeled (expected for this class of emulator; would require cross-service KMS integration — out of scope for services/sqs/)." +deferred: + - "SDK-driven integration tests (test/integration/*_parity_test.go) were not run this pass — per parity-principles.md, unit tests are not full parity proof. Recommend a follow-up integration-suite pass." +leaks: {status: clean, note: "fixed this pass: restoreQueueFromSnapshot now seeds hasActivity so the background janitor doesn't silently ignore restored queues forever (previously an unbounded-lifetime leak of retention-expired/DLQ-eligible messages on any queue restored with pending state and no subsequent SendMessage). Verified clean (pre-existing, unchanged): janitor ticker + StartMessageMoveTask goroutines are ctx-scoped and cancelled on Close/DeleteQueue/queue-involved-in-task; dedup maps are bounded (100k) with eviction; receiveAttempts/fifoSendTimes pruned each janitor tick; long-poll goroutines exit via the recheck-interval timer or notify-channel close, no goroutine leak on DeleteQueue mid-poll (input queue lookup re-checked each loop iteration)."} +--- + +## Notes + +### Protocol +SQS implements **both** protocols side by side, dispatched in `handler.go`'s +`Handler()`/`RouteMatcher()`: +- **Query (XML) protocol**: `Content-Type: application/x-www-form-urlencoded` POST with an + `Action=` form field (query.go). Legacy AWS.SimpleQueueService.* error codes. +- **JSON protocol** (`application/x-amz-json-1.1`, `X-Amz-Target: AmazonSQS.`): + handler.go. `com.amazonaws.sqs#`-namespaced error codes (JSON `__type` field). + +**Trap fixed this pass**: every Query-protocol response handler built its XML via +`marshalXML`, which prepended `xml.Header`, and then handed the bytes to echo's +`c.XMLBlob` — which **also** writes `xml.Header` before the blob. Every single +Query-protocol response (success AND error paths) therefore had **two** `` +prologs, which is not well-formed XML (a second XML declaration is a reserved/invalid +processing instruction anywhere but byte offset 0). Go's `encoding/xml` silently +tolerates it (which is why `xml.Unmarshal`-based tests never caught it), but a strict +XML parser in a non-Go AWS SDK could reject the response outright. Fixed by removing +the manual prepend from `marshalXML`, `writeQueryError`, and `buildQueryError` — the +declaration is written exactly once, by `c.XMLBlob`. See +`TestQueryProtocol_SingleXMLDeclaration`. + +### MD5 algorithms (both SQS-specific, do not use general-purpose hashing intuition) +- `MD5OfBody` / `MD5OfMessageAttributes` / `MD5OfMessageSystemAttributes`: plain MD5 of + the body, and MD5 of the AWS wire-packed attribute encoding (sorted attr names, each + encoded as 4-byte-BE-length name + 4-byte-BE-length dataType + 1-byte transport type + (1=String/Number, 2=Binary) + 4-byte-BE-length value). This is a documented AWS + wire-integrity checksum, not a security hash — do not "fix" it to SHA-256. +- FIFO **ContentBasedDeduplication** uses **SHA-256** of the message body, NOT MD5 — + a different algorithm for a different purpose (dedup identity vs wire checksum). Easy + to conflate; `computeSHA256` is deliberately a separate function from + `computeBodyChecksumMD5`. +- `MD5OfMessageAttributes` on `ReceiveMessage` must be recomputed over exactly the + *returned* attribute subset when the caller's `MessageAttributeNames` filters out some + attributes — reusing the send-time digest for a filtered receive would fail SDK-side + checksum verification. `computeMD5OfSubset` handles this using pre-encoded + per-attribute byte caches (`msg.encodedAttrs`) so repeated filtered receives don't + re-sort/re-encode every attribute each time. + +### FIFO ordering / visibility timers (the class of bug this pass focused on) +- Only **one message per MessageGroupId may be in-flight at a time** in this backend — + confirmed this is *correct*, not an over-restriction: real AWS FIFO queues block + further delivery from a group until the earlier message is deleted or its visibility + expires, to guarantee a single consumer sees strict order. Do not "fix" this into + allowing N-in-flight-per-group. +- Sends are **never** blocked by an in-flight predecessor in the same group — only + *receives* are. This means a newer message can be sitting in `q.messages` behind an + older, currently-in-flight message from the same group. When that older message is + returned to the visible pool (via `ChangeMessageVisibility(0)` or automatic + visibility-timeout expiry in the janitor's `sweepInFlight`), it **must** be reinserted + ahead of the newer one, not appended to the tail — `requeueMessage` does a + `sort.Search`-based insert by `SequenceNumber` (a fixed-width zero-padded decimal + string, so lexicographic compare == numeric compare) to restore this invariant. + `q.messages` is otherwise naturally kept in ascending SequenceNumber order because + `SendMessage` only ever appends and `pickVisibleMessages` compacts in place without + reordering. +- `ReceiveMessageInput.VisibilityTimeout` is a plain `int`, not `*int` — AWS's own + `aws-sdk-go-v2` model uses `*int32` specifically to distinguish "caller didn't specify + a value" from "caller explicitly wants 0". This codebase uses a sentinel instead + (`NoVisibilityTimeout = -1`, exported this pass — it was `noVisibilitySet`, + unexported, forcing external test code to hardcode the literal `-1`). **Any direct + Go-level caller of `InMemoryBackend.ReceiveMessage` (tests, or a future cross-service + integration) that leaves `VisibilityTimeout` unset gets an explicit 0-second + visibility timeout, NOT the queue's configured default** — the Go zero value + collides with a legitimate explicit value. The JSON handler (`*int`, nil-checked) and + Query-protocol parser (empty-string-checked) both correctly translate "field absent" + to `NoVisibilityTimeout` before calling the backend; anything bypassing those two + front ends must do the same explicitly. This is documented on the field's/sentinel's + doc comment in `types.go` but is still a live footgun for future Go-level callers — + worth reconsidering a `*int` refactor in a future pass if a real cross-service caller + ever needs this API directly (none exist today; verified via repo-wide grep). + +### Locking +`InMemoryBackend` uses one coarse `lockmetrics.RWMutex` (`b.mu`) guarding the top-level +`queues`/`moveTasks` maps, PLUS a per-`Queue` `sync.Mutex` (`q.mu`) guarding that queue's +own message/dedup/in-flight state (introduced in an earlier pass, tagged `#55` in +comments, to avoid one hot queue blocking all others). The established lock order is +always `b.mu` (RLock to resolve/look up the queue) **then** `q.mu` — never the reverse; +several functions (e.g. `GetQueueAttributes`, `Snapshot`) legitimately nest `q.mu.Lock()` +inside an `b.mu.RLock()` critical section. Do not add a call path that acquires `q.mu` +first and `b.mu` afterward. diff --git a/services/sqs/accuracy1677_test.go b/services/sqs/accuracy1677_test.go index 7027aceef..13f30500a 100644 --- a/services/sqs/accuracy1677_test.go +++ b/services/sqs/accuracy1677_test.go @@ -325,10 +325,15 @@ func TestIssue5_ReceiveRequestAttemptIDDifferentIds(t *testing.T) { require.NoError(t, err) } - // Different attempt IDs are independent. + // Different attempt IDs are independent. VisibilityTimeout must be the + // "unspecified" sentinel (not the Go int zero value) so the first receive + // keeps its message in flight for the queue's default visibility window + // instead of an explicit 0-second timeout that makes it immediately + // redeliverable — see sqs.NoVisibilityTimeout's doc comment. r1, err := b.ReceiveMessage(&sqs.ReceiveMessageInput{ QueueURL: qURL, MaxNumberOfMessages: 1, + VisibilityTimeout: sqs.NoVisibilityTimeout, ReceiveRequestAttemptID: "attempt-1", }) require.NoError(t, err) @@ -336,6 +341,7 @@ func TestIssue5_ReceiveRequestAttemptIDDifferentIds(t *testing.T) { r2, err := b.ReceiveMessage(&sqs.ReceiveMessageInput{ QueueURL: qURL, MaxNumberOfMessages: 1, + VisibilityTimeout: sqs.NoVisibilityTimeout, ReceiveRequestAttemptID: "attempt-2", }) require.NoError(t, err) diff --git a/services/sqs/backend.go b/services/sqs/backend.go index 669747179..753d55f97 100644 --- a/services/sqs/backend.go +++ b/services/sqs/backend.go @@ -7,6 +7,7 @@ import ( "encoding/binary" "encoding/hex" "encoding/json" + "errors" "fmt" "maps" "slices" @@ -20,6 +21,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/collections" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/telemetry" "github.com/blackbirdworks/gopherstack/pkgs/arn" @@ -107,10 +109,13 @@ const sqsMetricUnitCount = "Count" // InMemoryBackend implements StorageBackend using in-memory maps. type InMemoryBackend struct { - metricEmitter MetricEmitter - svcCtx context.Context - queues map[string]*Queue - moveTasks map[string]*moveTaskState + metricEmitter MetricEmitter + svcCtx context.Context + // registry lets Reset collapse the queues/moveTasks lifecycle to one call + // (registry.ResetAll()) instead of hand-rolled re-initialization of each map. + registry *store.Registry + queues *store.Table[Queue] + moveTasks *store.Table[moveTaskState] snsUnsubscribe func() janitorStop chan struct{} mu *lockmetrics.RWMutex @@ -118,6 +123,18 @@ type InMemoryBackend struct { region string } +// queueTableKey is the [store.Table] key function for b.queues, deriving the +// same (region, name) composite key that queueKey builds from raw input. +func queueTableKey(q *Queue) string { + return queueKey(q.Region, q.Name) +} + +// moveTaskTableKey is the [store.Table] key function for b.moveTasks: a move +// task's own taskHandle is already its unique identity. +func moveTaskTableKey(t *moveTaskState) string { + return t.taskHandle +} + // SetMetricEmitter sets the emitter used to forward SQS operation metrics to CloudWatch. func (b *InMemoryBackend) SetMetricEmitter(e MetricEmitter) { b.mu.Lock("SetMetricEmitter") @@ -163,14 +180,16 @@ func NewInMemoryBackendWithContext(svcCtx context.Context, accountID, region str } b := &InMemoryBackend{ - queues: make(map[string]*Queue), - moveTasks: make(map[string]*moveTaskState), + registry: store.NewRegistry(), accountID: accountID, region: region, mu: lockmetrics.New("sqs"), svcCtx: svcCtx, } + b.queues = store.Register(b.registry, "queues", store.New(queueTableKey)) + b.moveTasks = store.Register(b.registry, "moveTasks", store.New(moveTaskTableKey)) + b.startJanitor() return b @@ -195,10 +214,7 @@ func (b *InMemoryBackend) Close() { // already-finished task is a no-op. func (b *InMemoryBackend) cancelAllMoveTasks() { b.mu.Lock("cancelAllMoveTasks") - tasks := make([]*moveTaskState, 0, len(b.moveTasks)) - for _, task := range b.moveTasks { - tasks = append(tasks, task) - } + tasks := b.moveTasks.All() b.mu.Unlock() for _, task := range tasks { @@ -249,11 +265,13 @@ func (b *InMemoryBackend) pruneState(now time.Time) { // This avoids allocating a full-width snapshot when most queues are idle. b.mu.RLock("pruneState.collect") queues := make([]*Queue, 0) - for _, q := range b.queues { + b.queues.Range(func(q *Queue) bool { if q.hasActivity.Load() || q.IsFIFO { queues = append(queues, q) } - } + + return true + }) b.mu.RUnlock() dedupPruned := 0 @@ -289,7 +307,10 @@ func (b *InMemoryBackend) pruneState(now time.Time) { b.mu.Lock("pruneState.tasks") - for taskHandle, task := range b.moveTasks { + // Deleting a Table entry mid-Range is safe for the same reason it is safe + // for a bare map: Go guarantees a concurrently deleted key is not produced + // later in the same range. + b.moveTasks.Range(func(task *moveTaskState) bool { task.mu.Lock() isTerminal := task.status == MoveTaskStatusCompleted || task.status == MoveTaskStatusCancelled || @@ -298,10 +319,12 @@ func (b *InMemoryBackend) pruneState(now time.Time) { task.mu.Unlock() if isTerminal && isExpired { - delete(b.moveTasks, taskHandle) + b.moveTasks.Delete(task.taskHandle) tasksPruned++ } - } + + return true + }) b.mu.Unlock() @@ -341,9 +364,7 @@ func queueKey(region, name string) string { // lookupQueueByName returns the queue stored under (region, name), or false if // no queue exists in that region with that name. func (b *InMemoryBackend) lookupQueueByName(region, name string) (*Queue, bool) { - q, ok := b.queues[queueKey(b.effectiveRegion(region), name)] - - return q, ok + return b.queues.Get(queueKey(b.effectiveRegion(region), name)) } // lookupQueueByURL finds a queue by its URL. @@ -362,9 +383,8 @@ func (b *InMemoryBackend) lookupQueueByName(region, name string) (*Queue, bool) // queue created in us-west-2 if the URL strings happened to match. func (b *InMemoryBackend) lookupQueueByURL(region, queueURL string) (*Queue, bool) { name := queueNameFromInput(queueURL) - q, ok := b.queues[queueKey(b.effectiveRegion(region), name)] - return q, ok + return b.queues.Get(queueKey(b.effectiveRegion(region), name)) } // redrivePolicy represents the JSON structure of an SQS RedrivePolicy attribute. @@ -419,6 +439,15 @@ func applyRedrivePolicy(q *Queue, attrs map[string]string, backend *InMemoryBack } } + if allowErr := checkRedriveAllowPolicy(dlq, q.Attributes[attrQueueArn]); allowErr != nil { + return &InvalidParameterError{ + Message: fmt.Sprintf( + "Value %v for parameter RedrivePolicy is invalid. Reason: %s", + raw, allowErr.Error(), + ), + } + } + q.MaxReceiveCount = int(count) q.dlq = dlq @@ -437,6 +466,69 @@ func applyRedrivePolicy(q *Queue, attrs map[string]string, backend *InMemoryBack return nil } +// errRedriveDeniedAll / errRedriveDeniedByQueue are the reasons surfaced when a +// source queue's RedrivePolicy is rejected by the dead-letter target's +// RedriveAllowPolicy attribute. +var ( + errRedriveDeniedAll = errors.New( + "redrive permission is denied because the dead-letter target queue's RedriveAllowPolicy" + + " has redrivePermission set to denyAll", + ) + errRedriveDeniedByQueue = errors.New( + "redrive permission is denied because this queue's ARN is not listed in the dead-letter" + + " target queue's RedriveAllowPolicy sourceQueueArns", + ) +) + +// checkRedriveAllowPolicy enforces the dead-letter target queue's +// RedriveAllowPolicy attribute against the source queue attempting to point its +// RedrivePolicy at it. AWS lets a DLQ restrict which source queues may redrive +// into it via three redrivePermission values: +// +// - allowAll (default when the attribute is absent/empty): any source queue may use it. +// - denyAll: no source queue may use it. +// - byQueue: only source queues whose ARN appears in sourceQueueArns may use it. +// +// Without this check, RedriveAllowPolicy is accepted and shape-validated by +// validateRedriveAllowPolicy but never actually constrains anything — a +// disguised stub. srcArn is the ARN of the queue whose RedrivePolicy is being +// applied (the would-be source/DLQ-user), dlq is the dead-letter target. +func checkRedriveAllowPolicy(dlq *Queue, srcArn string) error { + raw, ok := dlq.Attributes[attrRedriveAllowPolicy] + if !ok || raw == "" { + return nil + } + + var policy struct { + RedrivePermission string `json:"redrivePermission"` + SourceQueueArns []string `json:"sourceQueueArns"` + } + + // Malformed policies are rejected at SetQueueAttributes time by + // validateRedriveAllowPolicy, so a parse failure here means the DLQ was + // never left with a malformed value; treat it permissively rather than + // blocking on a defensive parse error. + //nolint:nilerr // intentional fail-open on a defensive parse error, see comment above + if json.Unmarshal([]byte(raw), &policy) != nil { + return nil + } + + switch policy.RedrivePermission { + case "", "allowAll": + return nil + case "denyAll": + return errRedriveDeniedAll + case "byQueue": + if slices.Contains(policy.SourceQueueArns, srcArn) { + return nil + } + + return errRedriveDeniedByQueue + default: + return nil + } +} + // computeBodyChecksumMD5 returns the hex-encoded MD5 digest of a message body for the // MD5OfMessageBody / MD5OfMessageAttributes fields SQS returns on SendMessage, // SendMessageBatch, and ReceiveMessage responses. This is NOT a security hash: it is the @@ -614,7 +706,7 @@ func (b *InMemoryBackend) CreateQueue(input *CreateQueueInput) (*CreateQueueOutp // and the caller-supplied configurable attributes are the same (or absent), // return the existing URL. Different configurable attributes yields // QueueNameExists. A name collision in a different region is allowed. - if q, exists := b.queues[queueKey(region, input.QueueName)]; exists { + if q, exists := b.queues.Get(queueKey(region, input.QueueName)); exists { for k, v := range input.Attributes { if !isConfigurableQueueAttribute(k) { continue @@ -658,7 +750,7 @@ func (b *InMemoryBackend) CreateQueue(input *CreateQueueInput) (*CreateQueueOutp return nil, err } - b.queues[queueKey(region, input.QueueName)] = q + b.queues.Put(q) return &CreateQueueOutput{QueueURL: queueURL}, nil } @@ -683,19 +775,12 @@ func (b *InMemoryBackend) DeleteQueue(input *DeleteQueueInput) error { q.Tags.Close() } - delete(b.queues, queueKey(q.Region, q.Name)) + b.queues.Delete(queueKey(q.Region, q.Name)) // Cancel any active move tasks that involve this queue (either as source or // destination) to prevent goroutine leaks. - for _, task := range b.moveTasks { - task.mu.Lock() - isActive := task.status == MoveTaskStatusRunning || task.status == MoveTaskStatusCancelling - involves := task.sourceArn == queueARN || task.destArn == queueARN - task.mu.Unlock() - - if isActive && involves { - task.cancel() - } + for _, task := range b.moveTasks.All() { + b.cancelMoveTaskIfInvolved(task, queueARN) } return nil @@ -708,9 +793,9 @@ func (b *InMemoryBackend) ListQueues(input *ListQueuesInput) (*ListQueuesOutput, scope := b.effectiveRegion(input.Region) - urls := make([]string, 0, len(b.queues)) + urls := make([]string, 0, b.queues.Len()) - for _, q := range b.queues { + for _, q := range b.queues.All() { if q.Region != scope { continue } @@ -1462,6 +1547,21 @@ func validateReceiveInput(input *ReceiveMessageInput) error { return ErrInvalidMaxMessages } + // Validate VisibilityTimeout range here — centrally, in the backend — so + // every caller gets the same AWS-accurate rejection regardless of which + // protocol front-end is in use. Previously only the JSON handler + // (handleReceiveMessage) checked this range; the Query (XML) protocol + // path parsed the parameter and passed it straight through unchecked, so + // an out-of-range VisibilityTimeout sent over the legacy Query API + // silently produced a message that would effectively never become + // visible again instead of the AWS InvalidParameterValue error. + // NoVisibilityTimeout (-1) is the "unspecified, use the queue's default" + // sentinel and is exempt from range checking. + if input.VisibilityTimeout != NoVisibilityTimeout && + (input.VisibilityTimeout < 0 || input.VisibilityTimeout > maxVisibilityTimeoutSeconds) { + return ErrInvalidVisibilityTimeout + } + return nil } @@ -1677,6 +1777,41 @@ func buildBlockedGroups(inflight []*InFlightMessage) map[string]bool { // re-queues visibility-expired entries. Pass 2 sweeps q.messages (including // newly re-queued ones): discards retention-expired, drains to DLQ, picks up // to maxMessages visible messages. maxMessages=0 performs cleanup only. +// requeueMessage returns msg to the pending queue after its visibility timeout +// is reset — either explicitly via ChangeMessageVisibility(0) or implicitly +// via expiry in sweepInFlight. Caller must hold q.mu. +// +// For FIFO queues this must NOT simply append to the end of q.messages: a +// message can be sent to a group while an earlier message from that SAME +// group is in flight (in-flight messages block further delivery but not +// further sends). If the earlier message is later reset/expired and appended +// to the tail, it would land behind the newer same-group message already +// sitting in q.messages, and the next receive would hand out the newer +// message first — violating AWS's strict per-message-group ordering +// guarantee. Reinserting by SequenceNumber restores the correct position. +// q.messages is otherwise kept in ascending SequenceNumber order (SendMessage +// only ever appends in send order, and pickVisibleMessages compacts in place +// without reordering), and SequenceNumber is a fixed-width zero-padded +// decimal string, so lexicographic comparison matches numeric order. +// +// Standard queues have no AWS ordering guarantee, so they take the O(1) +// append path unconditionally. +func requeueMessage(q *Queue, msg *Message) { + if !q.IsFIFO { + q.messages = append(q.messages, msg) + + return + } + + idx := sort.Search(len(q.messages), func(i int) bool { + return q.messages[i].SequenceNumber > msg.SequenceNumber + }) + + q.messages = append(q.messages, nil) + copy(q.messages[idx+1:], q.messages[idx:]) + q.messages[idx] = msg +} + // sweepInFlight processes q.inFlightMessages: discards retention-expired entries and // re-queues visibility-expired entries back onto q.messages. Caller must hold q.mu. func sweepInFlight(q *Queue, cutoff, now time.Time) { @@ -1694,7 +1829,7 @@ func sweepInFlight(q *Queue, cutoff, now time.Time) { if now.After(inf.VisibleAt) { delete(q.inFlightByHandle, inf.ReceiptHandle) if !tryRouteToDLQ(q, inf.Msg, now) { - q.messages = append(q.messages, inf.Msg) + requeueMessage(q, inf.Msg) } changed = true @@ -1922,7 +2057,7 @@ func changeVisibility(q *Queue, receiptHandle string, visibilityTimeout int) err now := time.Now() inf.Msg.VisibleAt = now if !tryRouteToDLQ(q, inf.Msg, now) { - q.messages = append(q.messages, inf.Msg) + requeueMessage(q, inf.Msg) } delete(q.inFlightByHandle, receiptHandle) @@ -2294,7 +2429,7 @@ func (b *InMemoryBackend) ListDeadLetterSourceQueues( var urls []string - for _, q := range b.queues { + for _, q := range b.queues.All() { raw, ok := q.Attributes[attrRedrivePolicy] if !ok || raw == "" { continue @@ -2329,9 +2464,9 @@ func (b *InMemoryBackend) ListAll() []QueueInfo { b.mu.RLock("ListAll") defer b.mu.RUnlock() - result := make([]QueueInfo, 0, len(b.queues)) + result := make([]QueueInfo, 0, b.queues.Len()) - for _, q := range b.queues { + for _, q := range b.queues.All() { result = append(result, QueueInfo{Name: q.Name, URL: q.URL, IsFIFO: q.IsFIFO}) } @@ -2412,9 +2547,9 @@ func (b *InMemoryBackend) TaggedQueues() []TaggedQueueInfo { b.mu.RLock("TaggedQueues") defer b.mu.RUnlock() - result := make([]TaggedQueueInfo, 0, len(b.queues)) + result := make([]TaggedQueueInfo, 0, b.queues.Len()) - for _, q := range b.queues { + for _, q := range b.queues.All() { var tagMap map[string]string if q.Tags != nil { tagMap = q.Tags.Clone() @@ -2435,16 +2570,14 @@ func (b *InMemoryBackend) TagQueueByARN(queueARN string, newTags map[string]stri b.mu.Lock("TagQueueByARN") defer b.mu.Unlock() - for _, q := range b.queues { - if q.Attributes[attrQueueArn] == queueARN { - if q.Tags == nil { - q.Tags = tags.New("sqs.queue." + q.Name + ".tags") - } + if q, ok := b.findQueueByARN(queueARN); ok { + if q.Tags == nil { + q.Tags = tags.New("sqs.queue." + q.Name + ".tags") + } - q.Tags.Merge(newTags) + q.Tags.Merge(newTags) - return nil - } + return nil } return ErrQueueNotFound @@ -2456,19 +2589,29 @@ func (b *InMemoryBackend) UntagQueueByARN(queueARN string, tagKeys []string) err b.mu.Lock("UntagQueueByARN") defer b.mu.Unlock() - for _, q := range b.queues { - if q.Attributes[attrQueueArn] == queueARN { - if q.Tags != nil { - q.Tags.DeleteKeys(tagKeys) - } - - return nil + if q, ok := b.findQueueByARN(queueARN); ok { + if q.Tags != nil { + q.Tags.DeleteKeys(tagKeys) } + + return nil } return ErrQueueNotFound } +// findQueueByARN scans for the queue whose QueueArn attribute equals queueARN. +// Must be called with b.mu held (either read or write). +func (b *InMemoryBackend) findQueueByARN(queueARN string) (*Queue, bool) { + for _, q := range b.queues.All() { + if q.Attributes[attrQueueArn] == queueARN { + return q, true + } + } + + return nil, false +} + // ReceiveMessagesLocal is an internal method used by the ESM poller to pull // messages from a queue without long-polling. It returns up to maxMessages // visible messages, moving them to in-flight state using the queue's default @@ -2481,7 +2624,7 @@ func (b *InMemoryBackend) ReceiveMessagesLocal( QueueURL: queueURL, MaxNumberOfMessages: maxMessages, WaitTimeSeconds: 0, - VisibilityTimeout: noVisibilitySet, + VisibilityTimeout: NoVisibilityTimeout, }) if err != nil { return nil, err @@ -2512,7 +2655,7 @@ func (b *InMemoryBackend) totalMessages() int { defer b.mu.RUnlock() total := 0 - for _, q := range b.queues { + for _, q := range b.queues.All() { total += len(q.messages) + len(q.inFlightMessages) } @@ -2528,7 +2671,7 @@ func (b *InMemoryBackend) Purge(ctx context.Context, cutoff time.Time) { b.mu.Lock("Purge") defer b.mu.Unlock() - for k, q := range b.queues { + for _, q := range b.queues.All() { if ctx.Err() != nil { return } @@ -2544,22 +2687,22 @@ func (b *InMemoryBackend) Purge(ctx context.Context, cutoff time.Time) { } if time.Unix(createdUnix, 0).Before(cutoff) { - b.purgeQueue(k, q) + b.purgeQueue(q) } } } // purgeQueue closes and removes a single queue and cancels any move tasks that involve it. // Caller must hold b.mu. -func (b *InMemoryBackend) purgeQueue(key string, q *Queue) { +func (b *InMemoryBackend) purgeQueue(q *Queue) { close(q.notify) if q.Tags != nil { q.Tags.Close() } - delete(b.queues, key) + b.queues.Delete(queueTableKey(q)) queueARN := q.Attributes[attrQueueArn] - for _, task := range b.moveTasks { + for _, task := range b.moveTasks.All() { b.cancelMoveTaskIfInvolved(task, queueARN) } } @@ -2586,19 +2729,18 @@ func (b *InMemoryBackend) Reset() { defer b.mu.Unlock() // Close all queue tag stores to prevent resource leaks. - for _, q := range b.queues { + for _, q := range b.queues.All() { if q.Tags != nil { q.Tags.Close() } } // Cancel any running move tasks. - for _, task := range b.moveTasks { + for _, task := range b.moveTasks.All() { task.cancel() } - b.queues = make(map[string]*Queue) - b.moveTasks = make(map[string]*moveTaskState) + b.registry.ResetAll() } // iamPolicyDocument represents the IAM resource policy document stored in the @@ -2738,20 +2880,19 @@ func (b *InMemoryBackend) RemovePermission(input *RemovePermissionInput) error { // queueURLFromARNLocked returns the URL and ARN of the queue with the given ARN. // Must be called with b.mu held (either read or write). func (b *InMemoryBackend) queueURLFromARNLocked(queueARN string) (string, bool) { - for _, q := range b.queues { - if q.Attributes[attrQueueArn] == queueARN { - return q.URL, true - } + q, ok := b.findQueueByARN(queueARN) + if !ok { + return "", false } - return "", false + return q.URL, true } // findDefaultMoveDestinationLocked finds the URL of the queue that has a RedrivePolicy // pointing to the DLQ identified by dlqARN. // Must be called with b.mu held (either read or write). func (b *InMemoryBackend) findDefaultMoveDestinationLocked(dlqARN string) (string, bool) { - for _, q := range b.queues { + for _, q := range b.queues.All() { raw, ok := q.Attributes[attrRedrivePolicy] if !ok || raw == "" { continue @@ -2807,17 +2948,19 @@ func (b *InMemoryBackend) StartMessageMoveTask( // Check for existing running task on the same source ARN (AWS realism). // We check task status while holding both b.mu and t.mu to ensure the // status snapshot is consistent with the subsequent task insertion. - for _, t := range b.moveTasks { - if t.sourceArn == input.SourceArn { - t.mu.Lock() - isActive := t.status == MoveTaskStatusRunning || t.status == MoveTaskStatusCancelling - t.mu.Unlock() + for _, t := range b.moveTasks.All() { + if t.sourceArn != input.SourceArn { + continue + } + + t.mu.Lock() + isActive := t.status == MoveTaskStatusRunning || t.status == MoveTaskStatusCancelling + t.mu.Unlock() - if isActive { - b.mu.Unlock() + if isActive { + b.mu.Unlock() - return nil, ErrMoveTaskAlreadyRunning - } + return nil, ErrMoveTaskAlreadyRunning } } @@ -2875,7 +3018,7 @@ func (b *InMemoryBackend) StartMessageMoveTask( totalCount: totalCount, } - b.moveTasks[taskHandle] = state + b.moveTasks.Put(state) b.mu.Unlock() go b.runMoveTask(ctx, state, srcURL, destURL) @@ -3003,7 +3146,7 @@ func (b *InMemoryBackend) CancelMessageMoveTask( input *CancelMessageMoveTaskInput, ) (*CancelMessageMoveTaskOutput, error) { b.mu.RLock("CancelMessageMoveTask") - state, ok := b.moveTasks[input.TaskHandle] + state, ok := b.moveTasks.Get(input.TaskHandle) b.mu.RUnlock() if !ok { @@ -3052,7 +3195,7 @@ func (b *InMemoryBackend) ListMessageMoveTasks( var tasks []*moveTaskState - for _, t := range b.moveTasks { + for _, t := range b.moveTasks.All() { if input.SourceArn == "" || t.sourceArn == input.SourceArn { tasks = append(tasks, t) } diff --git a/services/sqs/backend_test.go b/services/sqs/backend_test.go index 87116f5aa..e60e44539 100644 --- a/services/sqs/backend_test.go +++ b/services/sqs/backend_test.go @@ -707,11 +707,11 @@ func TestReceiveMessageDefaultVisibility(t *testing.T) { _, err := b.SendMessage(&sqs.SendMessageInput{QueueURL: qURL, MessageBody: "hello"}) require.NoError(t, err) - // VisibilityTimeout = noVisibilitySet (-1) uses queue default. + // VisibilityTimeout = sqs.NoVisibilityTimeout uses the queue's default. out, err := b.ReceiveMessage(&sqs.ReceiveMessageInput{ QueueURL: qURL, MaxNumberOfMessages: 1, - VisibilityTimeout: -1, + VisibilityTimeout: sqs.NoVisibilityTimeout, WaitTimeSeconds: 0, }) require.NoError(t, err) @@ -1063,3 +1063,103 @@ func TestSendMessage_MD5OfMessageAttributes(t *testing.T) { require.NoError(t, err) assert.Empty(t, outNoAttrs.MD5OfMessageAttributes) } + +// Test_FIFORequeueOrdering verifies that a FIFO message returned to the +// visible queue (via ChangeMessageVisibility(0) or automatic visibility- +// timeout expiry) is redelivered before a newer message in the SAME +// MessageGroupId that arrived while the earlier one was in flight — AWS's +// strict per-group ordering guarantee. In-flight messages block further +// delivery from their group but do NOT block further sends to that group, so +// it is possible (and exercised here) for a second message to be sitting in +// the queue behind an in-flight message from the same group. Naively +// appending a re-queued message to the tail of the pending list would let the +// newer message jump ahead of it, violating ordering; the backend must +// reinsert by SequenceNumber instead. +func Test_FIFORequeueOrdering(t *testing.T) { + t.Parallel() + + tests := []struct { + requeue func(t *testing.T, b *sqs.InMemoryBackend, queueURL, receiptHandle string) + name string + }{ + { + name: "ChangeMessageVisibilityZero", + requeue: func(t *testing.T, b *sqs.InMemoryBackend, queueURL, receiptHandle string) { + t.Helper() + + err := b.ChangeMessageVisibility(&sqs.ChangeMessageVisibilityInput{ + QueueURL: queueURL, + ReceiptHandle: receiptHandle, + VisibilityTimeout: 0, + }) + require.NoError(t, err) + }, + }, + { + name: "VisibilityTimeoutExpiry", + requeue: func(t *testing.T, b *sqs.InMemoryBackend, _, _ string) { + t.Helper() + + // The message was received with a 1-second visibility timeout; + // simulate the janitor sweeping well past that expiry without a + // real sleep. + b.RunJanitorOnceForTest(time.Now().Add(2 * time.Second)) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := newBackend(t) + qURL := createTestQueue(t, b, "fifo-requeue-order.fifo") + + // msg1 into group G. + send1, err := b.SendMessage(&sqs.SendMessageInput{ + QueueURL: qURL, + MessageBody: "msg1", + MessageGroupID: "G", + MessageDeduplicationID: "dedup1", + }) + require.NoError(t, err) + + // Receive msg1 with a short (1s) visibility timeout so it becomes + // in-flight, blocking group G from further delivery. + recv1, err := b.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: qURL, + MaxNumberOfMessages: 1, + VisibilityTimeout: 1, + }) + require.NoError(t, err) + require.Len(t, recv1.Messages, 1) + require.Equal(t, send1.MessageID, recv1.Messages[0].MessageID) + + // msg2 arrives in the SAME group while msg1 is still in flight — + // sends are never blocked by an in-flight predecessor, only receives. + _, err = b.SendMessage(&sqs.SendMessageInput{ + QueueURL: qURL, + MessageBody: "msg2", + MessageGroupID: "G", + MessageDeduplicationID: "dedup2", + }) + require.NoError(t, err) + + // Return msg1 to the visible queue via whichever mechanism this + // subtest exercises. + tt.requeue(t, b, qURL, recv1.Messages[0].ReceiptHandle) + + // The next receive must hand back msg1 (the earlier SequenceNumber), + // not msg2, even though msg1 was the one just re-queued. + recv2, err := b.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: qURL, + MaxNumberOfMessages: 1, + VisibilityTimeout: sqs.NoVisibilityTimeout, + }) + require.NoError(t, err) + require.Len(t, recv2.Messages, 1) + assert.Equal(t, "msg1", recv2.Messages[0].Body, + "FIFO group ordering violated: newer same-group message delivered before the re-queued older one") + }) + } +} diff --git a/services/sqs/export_test.go b/services/sqs/export_test.go index fd2df7351..bf9d272b6 100644 --- a/services/sqs/export_test.go +++ b/services/sqs/export_test.go @@ -48,19 +48,19 @@ func (b *InMemoryBackend) InjectMoveTaskForTest( b.mu.Lock("InjectMoveTaskForTest") defer b.mu.Unlock() - b.moveTasks[taskHandle] = &moveTaskState{ + b.moveTasks.Put(&moveTaskState{ cancel: func() {}, taskHandle: taskHandle, status: status, startedAt: startedAt, - } + }) } func (b *InMemoryBackend) MoveTaskCountForTest() int { b.mu.RLock("MoveTaskCountForTest") defer b.mu.RUnlock() - return len(b.moveTasks) + return b.moveTasks.Len() } // SetRetentionForTest directly overrides the MessageRetentionPeriod attribute diff --git a/services/sqs/handler.go b/services/sqs/handler.go index 609b43a81..d7cc71745 100644 --- a/services/sqs/handler.go +++ b/services/sqs/handler.go @@ -737,7 +737,7 @@ func (h *Handler) handleReceiveMessage( return nil, ErrUnknownAction } - vt := noVisibilitySet + vt := NoVisibilityTimeout if req.VisibilityTimeout != nil { v := *req.VisibilityTimeout if v < 0 || v > maxVisibilityTimeoutSeconds { diff --git a/services/sqs/persistence.go b/services/sqs/persistence.go index e4bb661bb..a8c815094 100644 --- a/services/sqs/persistence.go +++ b/services/sqs/persistence.go @@ -2,13 +2,30 @@ package sqs import ( "context" + "encoding/json" + "fmt" "time" + "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) -// queueSnapshot captures all serialisable fields of a Queue. +// sqsSnapshotVersion identifies the shape of [backendSnapshot]. It must be +// bumped whenever a change to queueSnapshot, moveTaskSnapshot, or +// backendSnapshot itself would make an older snapshot unsafe to decode as the +// current shape (e.g. a field's meaning or type changes). Restore compares +// this against the persisted value and discards (rather than attempts to +// partially decode) any mismatch — see Restore below. +const sqsSnapshotVersion = 2 + +// queueSnapshot captures all serialisable fields of a Queue. It exists as a +// separate DTO (rather than JSON tags directly on Queue) because Queue also +// carries live, non-serialisable state — an open notify channel, a mutex, a +// self-referential dlq pointer rebuilt post-restore from RedrivePolicy, and +// short-lived caches (fifoSendTimes, receiveAttempts) — that must never be +// part of an on-disk snapshot. type queueSnapshot struct { DeduplicationIDs map[string]time.Time `json:"deduplicationIDs"` Attributes map[string]string `json:"attributes"` @@ -20,8 +37,24 @@ type queueSnapshot struct { Region string `json:"region,omitempty"` Messages []*Message `json:"messages"` InFlightMessages []*InFlightMessage `json:"inFlightMessages"` - MaxReceiveCount int `json:"maxReceiveCount"` - IsFIFO bool `json:"isFIFO"` + // FifoSeqCounter is the last-issued FIFO SequenceNumber counter. Persisting + // it prevents newly sent messages after a restore from reusing (or + // regressing behind) sequence numbers already handed out to consumers + // before the snapshot was taken. + FifoSeqCounter uint64 `json:"fifoSeqCounter,omitempty"` + // LastPurgedAtUnixMilli persists the PurgeQueue cooldown deadline so a + // restore immediately followed by a PurgeQueue call still honours AWS's + // 60-second between-purge cooldown instead of resetting it to zero. + LastPurgedAtUnixMilli int64 `json:"lastPurgedAtUnixMilli,omitempty"` + MaxReceiveCount int `json:"maxReceiveCount"` + IsFIFO bool `json:"isFIFO"` +} + +// queueSnapshotKey is the [store.Table] key function used for the ephemeral +// DTO table built inside Snapshot/Restore. It mirrors queueTableKey so the +// on-disk table is keyed identically to the live b.queues table. +func queueSnapshotKey(qs *queueSnapshot) string { + return queueKey(qs.Region, qs.Name) } // moveTaskSnapshot captures the serialisable state of a completed/cancelled/failed move task. @@ -38,11 +71,24 @@ type moveTaskSnapshot struct { MaxPerSec int32 `json:"maxPerSec,omitempty"` } +// moveTaskSnapshotKey is the [store.Table] key function for the ephemeral +// move-task DTO table, mirroring moveTaskTableKey. +func moveTaskSnapshotKey(ts *moveTaskSnapshot) string { + return ts.TaskHandle +} + +// backendSnapshot is the top-level on-disk shape for the SQS backend. +// +// Tables holds one JSON-encoded array per registered DTO table, produced by +// [store.Registry.SnapshotAll] — currently "queues" ([]*queueSnapshot) and +// "moveTasks" ([]*moveTaskSnapshot). Version guards against decoding a +// snapshot from an incompatible (older or newer) build of this backend as +// though it were the current shape; see Restore. type backendSnapshot struct { - Queues map[string]*queueSnapshot `json:"queues"` - AccountID string `json:"accountID"` - Region string `json:"region"` - MoveTasks []*moveTaskSnapshot `json:"moveTasks,omitempty"` + Tables map[string]json.RawMessage `json:"tables"` + AccountID string `json:"accountID"` + Region string `json:"region"` + Version int `json:"version"` } // Snapshot serialises the backend state to JSON. @@ -51,36 +97,54 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() - queues := make(map[string]*queueSnapshot, len(b.queues)) - for k, q := range b.queues { - queues[k] = &queueSnapshot{ - DeduplicationIDs: q.DeduplicationIDs, - Attributes: q.Attributes, - Tags: q.Tags, - Permissions: q.Permissions, - Messages: q.messages, - InFlightMessages: q.inFlightMessages, - DeduplicationMsgIDs: q.deduplicationMsgIDs, - Name: q.Name, - URL: q.URL, - Region: q.Region, - MaxReceiveCount: q.MaxReceiveCount, - IsFIFO: q.IsFIFO, + // Build a throwaway DTO registry purely to reuse store's deterministic, + // type-erased JSON encoding (store.Registry.SnapshotAll) instead of + // hand-rolling the marshal step. This is intentionally separate from the + // live b.registry: Queue/moveTaskState carry fields (channels, mutexes, + // cancel funcs, a dlq back-pointer) that cannot round-trip through JSON, + // and only terminal move tasks are meant to survive a restart — a direct + // snapshot of the live tables could not express either constraint. + dtoReg := store.NewRegistry() + queueDTOs := store.Register(dtoReg, "queues", store.New(queueSnapshotKey)) + moveDTOs := store.Register(dtoReg, "moveTasks", store.New(moveTaskSnapshotKey)) + + for _, q := range b.queues.Snapshot() { + q.mu.Lock() + var lastPurgedAtMillis int64 + if !q.lastPurgedAt.IsZero() { + lastPurgedAtMillis = q.lastPurgedAt.UnixMilli() } + fifoSeqCounter := q.fifoSeqCounter + q.mu.Unlock() + + queueDTOs.Put(&queueSnapshot{ + DeduplicationIDs: q.DeduplicationIDs, + Attributes: q.Attributes, + Tags: q.Tags, + Permissions: q.Permissions, + Messages: q.messages, + InFlightMessages: q.inFlightMessages, + DeduplicationMsgIDs: q.deduplicationMsgIDs, + Name: q.Name, + URL: q.URL, + Region: q.Region, + MaxReceiveCount: q.MaxReceiveCount, + IsFIFO: q.IsFIFO, + FifoSeqCounter: fifoSeqCounter, + LastPurgedAtUnixMilli: lastPurgedAtMillis, + }) } // Persist terminal move tasks (COMPLETED/CANCELLED/FAILED) so task history // survives restarts. RUNNING tasks are skipped because the goroutine cannot // be resumed, and CANCELLING is a transient state that resolves to CANCELLED. - var moveTasks []*moveTaskSnapshot - for _, t := range b.moveTasks { + b.moveTasks.Range(func(t *moveTaskState) bool { t.mu.Lock() status := t.status - // Skip non-terminal tasks — only snapshot what can be meaningfully restored. if status != MoveTaskStatusCompleted && status != MoveTaskStatusCancelled && status != MoveTaskStatusFailed { t.mu.Unlock() - continue + return true } snap := &moveTaskSnapshot{ @@ -96,12 +160,26 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { } t.mu.Unlock() - moveTasks = append(moveTasks, snap) + moveDTOs.Put(snap) + + return true + }) + + tables, err := dtoReg.SnapshotAll() + if err != nil { + // The DTOs above are plain JSON-friendly structs, so a marshal failure + // here would indicate a programming error rather than bad input data. + // Log and skip the snapshot rather than panic, matching the + // persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, + "sqs: snapshot table marshal failed", "error", err) + + return nil } snap := backendSnapshot{ - Queues: queues, - MoveTasks: moveTasks, + Version: sqsSnapshotVersion, + Tables: tables, AccountID: b.accountID, Region: b.region, } @@ -123,32 +201,54 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() - if snap.Queues == nil { - snap.Queues = make(map[string]*queueSnapshot) + if snap.Version != sqsSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape — that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. + logger.Load(ctx).WarnContext(ctx, + "sqs: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", sqsSnapshotVersion) + + b.registry.ResetAll() + + return nil } - b.queues = make(map[string]*Queue, len(snap.Queues)) + dtoReg := store.NewRegistry() + queueDTOs := store.Register(dtoReg, "queues", store.New(queueSnapshotKey)) + moveDTOs := store.Register(dtoReg, "moveTasks", store.New(moveTaskSnapshotKey)) + + if err := dtoReg.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("sqs: restore snapshot tables: %w", err) + } // Restore region first so we can default per-queue regions when missing. if snap.Region != "" { b.region = snap.Region } - for _, qs := range snap.Queues { + liveQueues := make([]*Queue, 0, queueDTOs.Len()) + + for _, qs := range queueDTOs.All() { region := qs.Region if region == "" { region = b.effectiveRegion("") } - b.queues[queueKey(region, qs.Name)] = restoreQueueFromSnapshot(qs, region) + liveQueues = append(liveQueues, restoreQueueFromSnapshot(qs, region)) } + b.queues.Restore(liveQueues) + // Restore terminal move task history. A no-op cancel function is used because // these tasks are already in a terminal state and their goroutines are gone. - b.moveTasks = make(map[string]*moveTaskState) + liveMoveTasks := make([]*moveTaskState, 0, moveDTOs.Len()) - for _, ts := range snap.MoveTasks { - b.moveTasks[ts.TaskHandle] = &moveTaskState{ + for _, ts := range moveDTOs.All() { + liveMoveTasks = append(liveMoveTasks, &moveTaskState{ cancel: func() {}, taskHandle: ts.TaskHandle, sourceArn: ts.SourceArn, @@ -159,13 +259,15 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { movedCount: ts.MovedCount, totalCount: ts.TotalCount, maxPerSec: ts.MaxPerSec, - } + }) } + b.moveTasks.Restore(liveMoveTasks) + // Re-wire DLQ pointers now that every queue has been reconstructed. We must // do this after the second pass because a queue's RedrivePolicy can target // any other queue regardless of restore iteration order. - for _, q := range b.queues { + for _, q := range b.queues.All() { _ = applyRedrivePolicy(q, q.Attributes, b) } @@ -207,7 +309,12 @@ func restoreQueueFromSnapshot(qs *queueSnapshot, region string) *Queue { } } - return &Queue{ + var lastPurgedAt time.Time + if qs.LastPurgedAtUnixMilli != 0 { + lastPurgedAt = time.UnixMilli(qs.LastPurgedAtUnixMilli) + } + + q := &Queue{ DeduplicationIDs: qs.DeduplicationIDs, Attributes: qs.Attributes, Tags: qs.Tags, @@ -223,7 +330,21 @@ func restoreQueueFromSnapshot(qs *queueSnapshot, region string) *Queue { IsFIFO: qs.IsFIFO, notify: make(chan struct{}), delayedCount: delayedCount, + fifoSeqCounter: qs.FifoSeqCounter, + lastPurgedAt: lastPurgedAt, } + + // The background janitor (pruneState) skips non-FIFO queues whose + // hasActivity flag is false to avoid scanning idle queues (#59 follow-on). + // A restored queue with pending visible/in-flight messages must be marked + // active so retention expiry and visibility-timeout requeue resume without + // waiting for a new SendMessage to flip the flag — otherwise those + // messages are stuck until the next producer write to that specific queue. + if len(qs.Messages) > 0 || len(qs.InFlightMessages) > 0 { + q.hasActivity.Store(true) + } + + return q } // Snapshot implements persistence.Persistable by delegating to the backend. diff --git a/services/sqs/persistence_test.go b/services/sqs/persistence_test.go index 1484815f0..57984a3bb 100644 --- a/services/sqs/persistence_test.go +++ b/services/sqs/persistence_test.go @@ -1,6 +1,8 @@ package sqs_test import ( + "fmt" + "strings" "testing" "time" @@ -83,6 +85,176 @@ func TestInMemoryBackend_SnapshotRestore(t *testing.T) { require.NoError(t, removeErr) }, }, + { + // Regression guard for the pkgs/store conversion: an in-flight + // (received but not yet deleted) message must round-trip through + // Snapshot/Restore still invisible, with its original receipt + // handle still valid — proving the store.Table-backed queues and + // their inline messages/inFlightMessages survive the swap intact. + name: "in_flight_message_round_trip", + setup: func(b *sqs.InMemoryBackend) string { + out, err := b.CreateQueue(&sqs.CreateQueueInput{QueueName: "inflight-queue", Endpoint: "localhost"}) + if err != nil { + return "" + } + + if _, sendErr := b.SendMessage(&sqs.SendMessageInput{ + QueueURL: out.QueueURL, + MessageBody: "in-flight-body", + }); sendErr != nil { + return "" + } + + recvOut, recvErr := b.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: out.QueueURL, + MaxNumberOfMessages: 1, + VisibilityTimeout: 300, + }) + if recvErr != nil || len(recvOut.Messages) != 1 { + return "" + } + + return out.QueueURL + "|" + recvOut.Messages[0].ReceiptHandle + }, + verify: func(t *testing.T, b *sqs.InMemoryBackend, id string) { + t.Helper() + require.NotEmpty(t, id) + + parts := strings.SplitN(id, "|", 2) + require.Len(t, parts, 2) + queueURL, receiptHandle := parts[0], parts[1] + + // The message must still be in flight (invisible) after the round trip. + recvOut, err := b.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: queueURL, + MaxNumberOfMessages: 10, + }) + require.NoError(t, err) + assert.Empty(t, recvOut.Messages, + "in-flight message must remain invisible across a snapshot/restore round trip") + + // Its original receipt handle must still be valid for deletion. + err = b.DeleteMessage(&sqs.DeleteMessageInput{ + QueueURL: queueURL, + ReceiptHandle: receiptHandle, + }) + require.NoError(t, err, "receipt handle issued before the snapshot must remain valid after restore") + }, + }, + { + name: "fifo_sequence_number_persists_across_restore", + setup: func(b *sqs.InMemoryBackend) string { + out, err := b.CreateQueue(&sqs.CreateQueueInput{ + QueueName: "seq-fifo.fifo", + Endpoint: "localhost", + Attributes: map[string]string{ + "FifoQueue": "true", + "ContentBasedDeduplication": "true", + }, + }) + if err != nil { + return "" + } + + // Send several messages, each in its own MessageGroupID, so the FIFO + // sequence counter advances well past its zero value before the + // snapshot is taken. Distinct groups let a single ReceiveMessage + // call in verify() drain all of them at once (this backend only + // allows one in-flight message per group at a time). + const sendCount = 5 + for i := range sendCount { + if _, sendErr := b.SendMessage(&sqs.SendMessageInput{ + QueueURL: out.QueueURL, + MessageBody: fmt.Sprintf("body-%d", i), + MessageGroupID: fmt.Sprintf("group-%d", i), + }); sendErr != nil { + return "" + } + } + + return out.QueueURL + }, + verify: func(t *testing.T, b *sqs.InMemoryBackend, queueURL string) { + t.Helper() + require.NotEmpty(t, queueURL) + + // Drain the 5 pre-restore messages so only the post-restore send's + // SequenceNumber remains to inspect. + drainOut, err := b.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: queueURL, + MaxNumberOfMessages: 10, + }) + require.NoError(t, err) + require.Len(t, drainOut.Messages, 5, "all 5 pre-restore messages should survive the round trip") + + var lastSeq string + for _, m := range drainOut.Messages { + if m.SequenceNumber > lastSeq { + lastSeq = m.SequenceNumber + } + } + + // If the FIFO sequence counter were reset to zero by Restore (the + // bug), this new message's SequenceNumber would regress behind (or + // duplicate) the ones already handed out above, breaking the AWS + // guarantee that SequenceNumber strictly increases per queue. + sendOut, err := b.SendMessage(&sqs.SendMessageInput{ + QueueURL: queueURL, + MessageBody: "post-restore", + MessageGroupID: "group-a", + }) + require.NoError(t, err) + assert.Greater(t, sendOut.SequenceNumber, lastSeq, + "SequenceNumber must keep increasing across a snapshot/restore round trip") + }, + }, + { + name: "restored_queue_janitor_prunes_retention_expired_message", + setup: func(b *sqs.InMemoryBackend) string { + out, err := b.CreateQueue(&sqs.CreateQueueInput{ + QueueName: "restore-janitor-queue", + Endpoint: "localhost", + }) + if err != nil { + return "" + } + + if _, sendErr := b.SendMessage(&sqs.SendMessageInput{ + QueueURL: out.QueueURL, + MessageBody: "expire-me", + }); sendErr != nil { + return "" + } + + // Shorten retention so the message is already expired by the time + // the janitor next runs (simulated below via a future `now`). + b.SetRetentionForTest(out.QueueURL, 1) + + return out.QueueURL + }, + verify: func(t *testing.T, b *sqs.InMemoryBackend, queueURL string) { + t.Helper() + require.NotEmpty(t, queueURL) + + // Regression guard: a freshly restored non-FIFO queue must have its + // hasActivity flag seeded from its message/in-flight counts. + // Previously Restore left it false, so the background janitor's + // pruneState (which skips non-FIFO queues with hasActivity==false) + // silently ignored this queue forever — the retention-expired + // message would never be pruned until an unrelated SendMessage + // happened to flip the flag. Simulate a janitor tick 2 seconds in + // the future (past the 1-second retention window) without a real sleep. + b.RunJanitorOnceForTest(time.Now().Add(2 * time.Second)) + + out, err := b.GetQueueAttributes(&sqs.GetQueueAttributesInput{ + QueueURL: queueURL, + AttributeNames: []string{"ApproximateNumberOfMessages"}, + }) + require.NoError(t, err) + assert.Equal(t, "0", out.Attributes["ApproximateNumberOfMessages"], + "janitor should prune the retention-expired message on a restored queue") + }, + }, { name: "completed_move_task_history_round_trip", setup: func(b *sqs.InMemoryBackend) string { @@ -161,6 +333,28 @@ func TestInMemoryBackend_SnapshotRestore(t *testing.T) { } } +func TestInMemoryBackend_RestoreDiscardsIncompatibleSnapshotVersion(t *testing.T) { + t.Parallel() + + b := sqs.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + t.Cleanup(b.Close) + + _, err := b.CreateQueue(&sqs.CreateQueueInput{QueueName: "pre-existing-queue", Endpoint: "localhost"}) + require.NoError(t, err) + + // Well-formed JSON, but tagged with a version this build does not know how + // to decode (e.g. the pre-pkgs/store on-disk shape, or a future one). + incompatibleSnapshot := []byte(`{"version":1,"queues":{},"accountID":"000000000000","region":"us-east-1"}`) + + err = b.Restore(t.Context(), incompatibleSnapshot) + require.NoError(t, err, + "an incompatible snapshot version must be discarded cleanly, not surfaced as a restore error") + + out, err := b.ListQueues(&sqs.ListQueuesInput{}) + require.NoError(t, err) + assert.Empty(t, out.QueueURLs, "backend must start empty after discarding an incompatible snapshot") +} + func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { t.Parallel() diff --git a/services/sqs/query.go b/services/sqs/query.go index 78ab379de..f6f1befd5 100644 --- a/services/sqs/query.go +++ b/services/sqs/query.go @@ -95,7 +95,11 @@ func writeQueryError(c *echo.Context, code, message string, status int) error { b, _ := xml.Marshal(resp) - return c.XMLBlob(status, append([]byte(xml.Header), b...)) + // c.XMLBlob already writes the XML declaration ("") + // before the blob; prepending xml.Header here as well produced a + // malformed document with two XML prologs (see marshalXML for the full + // explanation of this bug class). + return c.XMLBlob(status, b) } // buildQueryError builds a queryError from a backend error. @@ -114,10 +118,13 @@ func buildQueryError(err error) *queryError { }, } + // Not xml.Header-prefixed here: the caller (handleQueryProtocol) always + // delivers queryError.xml via c.XMLBlob, which itself writes the XML + // declaration. See marshalXML's doc comment for why double-prepending + // produced a malformed (two-prolog) XML document. b, _ := xml.Marshal(resp) - xmlBytes := append([]byte(xml.Header), b...) - return &queryError{xml: xmlBytes, status: status} + return &queryError{xml: b, status: status} } // queryErrorDetails returns the Query-protocol error code, message, and HTTP status for err. @@ -133,14 +140,18 @@ func queryErrorDetails(err error) (string, string, int) { return errorDetails(err) } -// marshalXML marshals v to XML bytes with the XML header prepended. +// marshalXML marshals v to XML bytes. It intentionally does NOT prepend the +// XML declaration (""): every +// caller in this file hands the result to echo's c.XMLBlob, which already +// writes that declaration itself before the blob (see +// echo/v5.Context.XMLBlob). Prepending it here too — as this function +// previously did — produced a response body with TWO XML prologs, which is +// not well-formed XML (a second "" processing instruction is only +// legal at byte offset 0 of a document) and could be rejected outright by +// strict XML parsers in non-Go AWS SDKs even though Go's own encoding/xml +// tolerates it. func marshalXML(v any) ([]byte, error) { - b, err := xml.Marshal(v) - if err != nil { - return nil, err - } - - return append([]byte(xml.Header), b...), nil + return xml.Marshal(v) } // queueURLEndpoint returns the endpoint to use for queue URL construction. @@ -623,7 +634,7 @@ func (h *Handler) querySendMessage(vals url.Values, r *http.Request, region stri func (h *Handler) queryReceiveMessage(vals url.Values, region string) ([]byte, int, *queryError) { maxMsgs, _ := strconv.Atoi(vals.Get("MaxNumberOfMessages")) waitSecs, _ := strconv.Atoi(vals.Get("WaitTimeSeconds")) - vt := noVisibilitySet + vt := NoVisibilityTimeout if vtStr := vals.Get("VisibilityTimeout"); vtStr != "" { if n, err := strconv.Atoi(vtStr); err == nil { diff --git a/services/sqs/query_audit_test.go b/services/sqs/query_audit_test.go index 99422901c..e84ede709 100644 --- a/services/sqs/query_audit_test.go +++ b/services/sqs/query_audit_test.go @@ -172,6 +172,78 @@ func TestQueryProtocol_MessageMoveTasks(t *testing.T) { } } +// TestQueryProtocol_SingleXMLDeclaration verifies that Query-protocol XML +// responses contain exactly one "" declaration. echo's +// c.XMLBlob already writes that declaration before the response body; +// marshalXML/writeQueryError/buildQueryError previously ALSO prepended it, +// producing a body with two XML prologs — not well-formed XML (a second +// "" processing instruction is only legal at byte offset 0). +// Go's encoding/xml happens to tolerate it, which is why this went unnoticed +// by tests that merely xml.Unmarshal the body, but a strict XML parser in a +// non-Go AWS SDK could reject the response outright. +func TestQueryProtocol_SingleXMLDeclaration(t *testing.T) { + t.Parallel() + + tests := []struct { + vals url.Values + name string + }{ + { + name: "success_response", + vals: url.Values{"Action": {"CreateQueue"}, "QueueName": {"xml-decl-queue"}}, + }, + { + name: "error_response", + vals: url.Values{ + "Action": {"GetQueueAttributes"}, + "QueueUrl": {"http://localhost/000000000000/does-not-exist"}, + }, + }, + { + name: "unknown_action_response", + vals: url.Values{"Action": {"TotallyBogusAction"}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := newTestHandler(t) + rec := doQueryRequest(t, h, tt.vals) + + count := strings.Count(rec.Body.String(), "/PARITY.md, fill, keep updated. +# Purpose: record audit state so the NEXT audit diffs the delta instead of rescanning. +# Re-audit protocol: `git diff ..HEAD -- services//` for local drift, +# AND check the SDK module for ops added since sdk_version. Only audit changed/new surface; +# trust rows marked ok whose files are unchanged since last_audit_commit. +service: ssm +sdk_module: aws-sdk-go-v2/service/ssm@v1.69.5 +last_audit_commit: 647d2017 +last_audit_date: 2026-07-05 +overall: B # already-accurate op-by-op for the bulk of the surface (2 prior + # sweeps did the heavy lifting); this pass found and fixed 6 genuine + # bugs concentrated in Parameter Store tier/version/hierarchy rules and + # Document version-selector/wire-shape handling — real but narrower + # than a from-scratch ~1k-LOC sweep. See gaps/Notes for what's proven. +# Per-op or per-op-family status. Values: ok | partial | gap | deferred. +# wire=response/request shape vs SDK; errors=code+HTTP status; state=real mutate/read; persist=in backendSnapshot. +ops: + PutParameter: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED this pass — see Notes: hierarchy-level limit, labeled-oldest-version eviction guard, Intelligent-Tiering auto-upgrade, Policies-require-Advanced-tier"} + GetParameter: {wire: ok, errors: ok, state: ok, persist: ok, note: "selector suffix (:version/:label), SecureString decrypt, ARN population all proven correct"} + GetParameters: {wire: ok, errors: ok, state: ok, persist: ok, note: "unresolvable names/labels/decrypt failures correctly become InvalidParameters entries, not a hard error"} + GetParameterHistory: {wire: ok, errors: ok, state: ok, persist: ok, note: "MaxResults 1-50 default 50 (matches AWS), label backfill via parameterLabelsStore proven correct, pagination via opaque index token"} + DeleteParameter: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteParameters: {wire: ok, errors: ok, state: ok, persist: ok} + GetParametersByPath: {wire: ok, errors: ok, state: ok, persist: ok, note: "MaxResults 1-10 default 10 (matches AWS), recursive/non-recursive prefix matching, ParameterFilters proven correct"} + DescribeParameters: {wire: ok, errors: ok, state: ok, persist: ok, note: "MaxResults 1-50 default 50 (matches AWS)"} + LabelParameterVersion: {wire: ok, errors: ok, state: ok, persist: ok, note: "10-label-per-version cap (appendLabelsWithLimit) and move-label-between-versions semantics proven correct"} + UnlabelParameterVersion: {wire: ok, errors: ok, state: ok, persist: ok} + CreateDocument: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED this pass — DocumentDescription response was leaking the full Content body (see Notes)"} + GetDocument: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED this pass — explicit $DEFAULT selector was conflated with $LATEST (see Notes)"} + UpdateDocument: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED this pass — same Content-leak as CreateDocument; version cap (maxDocumentVersionCap=1000) proven correct"} + DescribeDocument: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED this pass — Content leak, AND the DocumentVersion selector was previously ignored entirely (always described the latest version)"} + DeleteDocument: {wire: ok, errors: ok, state: ok, persist: ok} + ListDocuments: {wire: ok, errors: ok, state: ok, persist: ok} + ListDocumentVersions: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeDocumentPermission: {wire: ok, errors: ok, state: ok, persist: ok} + ModifyDocumentPermission: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateDocumentDefaultVersion: {wire: ok, errors: ok, state: ok, persist: ok, note: "verifies requested version exists in documentVersionsStore before pinning"} + SendCommand: {wire: ok, errors: ok, state: ok, persist: ok} + ListCommands: {wire: ok, errors: ok, state: ok, persist: ok} + ListCommandInvocations: {wire: ok, errors: ok, state: ok, persist: ok} + GetCommandInvocation: {wire: ok, errors: ok, state: ok, persist: ok} + CancelCommand: {wire: ok, errors: ok, state: ok, persist: ok} + PutInventory: {wire: ok, errors: ok, state: ok, persist: ok, note: "merge-by-TypeName semantics proven correct"} + GetInventory: {wire: ok, errors: ok, state: ok, persist: ok} + GetInventorySchema: {wire: ok, errors: ok, state: ok, persist: n/a, note: "static built-in AWS: /Custom: schema catalog, matches real SSM's documented inventory types"} + DeleteInventory: {wire: ok, errors: ok, state: ok, persist: ok, note: "records a real DeletionId job consumed by DescribeInventoryDeletions"} + CreateActivation: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteActivation: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeActivations: {wire: ok, errors: ok, state: ok, persist: ok} + CreateAssociation: {wire: ok, errors: ok, state: ok, persist: ok} + CreateAssociationBatch: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteAssociation: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeAssociation: {wire: ok, errors: ok, state: ok, persist: ok} + CreatePatchBaseline: {wire: ok, errors: ok, state: ok, persist: ok} + DeletePatchBaseline: {wire: ok, errors: ok, state: ok, persist: ok} + GetPatchBaseline: {wire: ok, errors: ok, state: ok, persist: ok} +families: + parameter-store: {status: ok, note: "FIXED this pass (PutParameter): 15-level hierarchy limit (HierarchyLevelLimitExceededException, previously unenforced), labeled-oldest-version eviction guard (ParameterMaxVersionLimitExceeded, previously silently evicted labeled versions and leaked their parameterLabels entries forever), Intelligent-Tiering auto-upgrade-to-Advanced on >4KiB value or Policies attached (previously hard-rejected instead of auto-selecting Advanced, defeating the entire point of Intelligent-Tiering), Policies-require-Advanced-tier (previously any tier accepted policies). Tier value-size limits (4096 Standard / 8192 Advanced), AllowedPattern regex validation, SecureString KMS encrypt/decrypt round-trip via per-instance AES-256 key, parameter selector suffix (:version/:label) parsing were all already correct." + documents: {status: ok, note: "FIXED this pass: CreateDocument/UpdateDocument/DescribeDocument were all returning the internal Document struct (which carries Content) as their metadata-only response — added a DocumentDescription wire type (matches AWS's real DocumentDescription, no Content field) and a Document.toDocumentDescription() converter. Also: GetDocument/DescribeDocument's DocumentVersion selector conflated explicit \"$DEFAULT\" with \"$LATEST\"/omitted, always serving the latest version's content/metadata even when a caller explicitly asked for $DEFAULT after UpdateDocumentDefaultVersion pinned an older version. Left the omitted-DocumentVersion behavior as latest (unchanged) since AWS's own API/CLI reference docs do not state a default and an existing, deliberately-written test (document_test.go TestInMemoryBackend_Snapshot_IncludesDocumentsAndCommands) depends on that behavior — only the unambiguous explicit-$DEFAULT case was fixed. Document version cap (1000) and content-hash-free JSON/YAML round-trip were already correct." + command-execution: {status: ok, note: "no goroutines/timers in command_exec.go or automation_exec.go — command progression is driven synchronously plus the single ctx-cancel-aware janitor sweep (janitor.go), not per-command background workers. Nothing to leak."} + sessions: {status: deferred, note: "StartSession/TerminateSession/ResumeSession not re-audited this pass beyond confirming they route through the janitor's terminated-session sweep (leak_test.go/janitor_test.go already cover this from a prior sweep); no wire/state changes made."} + patch-maintenance-associations-inventory: {status: deferred, note: "spot-checked (Inventory family fully, PutParameter/PutInventory cross-reference) but not re-audited op-by-op this pass; prior sweeps (parity_batch7_test.go, parity_deepdive_test.go, batch2_accuracy_test.go) already cover patch baselines, patch groups/compliance, maintenance window tasks/targets, and state-manager associations in depth and no drift was found in the files backing them." +gaps: # known divergences NOT fixed — link bd issue ids + - "Document version-cap eviction (maxDocumentVersionCap=1000) can, in a very long-lived document, evict the version currently pinned as DefaultVersion, orphaning the $DEFAULT selector (GetDocument/DescribeDocument would then return ErrInvalidDocumentVersion instead of re-pointing or falling back). Needs 1000+ UpdateDocument calls after pinning an old DefaultVersion — rare in practice, not fixed this pass (bd: gopherstack-1hg)" + - "NoChangeNotification parameter policy is stored (Policies field round-trips) but never evaluated/acted on — no EventBridge event is emitted when a parameter goes unchanged past its configured window. ExpirationNotification has the same gap. Only Expiration is enforced (janitor sweep deletes on expiry). Out of scope for this pass — would require EventBridge cross-service wiring (shared-file, not fixed)." +deferred: # consciously not audited this pass (scope) — next pass targets + - Session Manager (StartSession/TerminateSession/ResumeSession) full op-by-op re-verification + - Patch baselines / patch groups / compliance op-by-op re-verification (spot-checked only) + - Maintenance windows (tasks/targets) op-by-op re-verification (spot-checked only) + - State Manager associations op-by-op re-verification (spot-checked only) + - OpsCenter (OpsItem/OpsMetadata) op-by-op re-verification (spot-checked only, errors already wired) +leaks: {status: clean, note: "Janitor (janitor.go) is the only background goroutine, ctx.Done()-aware, single Run() loop shared across all sweeps (parameters/commands/sessions). PutParameter's history cap now also deletes the corresponding parameterLabels[version] entries on eviction (previously left as an unbounded-growth leak: labels attached to since-evicted versions stayed in the map forever with no key ever removed). No new goroutines/tickers/timers introduced this pass."} +--- + +## Notes + +SSM speaks the **json-1.1 protocol** (`AmazonSSM.` `X-Amz-Target`, `application/x-amz-json-1.1` +content type) — confirmed via `handler.go`'s `classifySSMError`/`handleError` using +`service.JSONErrorResponse` with a bare `{"Type":..., "Message":...}` body, not XML. + +### Real bug: Intelligent-Tiering was rejecting the exact case it exists for + +`resolveTier` treated `Intelligent-Tiering` identically to `Standard` for the 4096-byte size +check — a `PutParameter` with `Tier: "Intelligent-Tiering"` and a 5000-byte value returned +`ValidationException` instead of succeeding. This defeats the entire purpose of the tier: per AWS +docs (confirmed via websearch against the GitHub aws-sdk-net issue tracker and the AWS +"Managing tiers" user guide), Intelligent-Tiering auto-promotes to Advanced whenever the request +needs a capability Standard doesn't support — either a value over 4 KiB, or parameter policies +attached — rather than failing. Fixed: `resolveTier` now upgrades `tier` to `"Advanced"` in that +case (and still enforces the 8 KiB Advanced ceiling on top). An explicit `Tier: "Standard"` still +hard-fails on the same conditions, since the caller opted out of auto-selection by naming a +concrete tier. Confirmed via websearch that Policies (Expiration/ExpirationNotification/ +NoChangeNotification) are Advanced-tier-only — Standard rejects them outright (this AWS constraint +was previously entirely unenforced; any tier could carry a Policies string). Three existing tests +in `parity_emr_test.go` (`TestParityEMR_ParameterExpiration_JanitorEvicts`) attached an Expiration +policy without ever setting `Tier`, i.e. exercised Standard+Policies — updated those to +`Tier: "Advanced"` since that combination is what real AWS requires; the janitor-eviction behavior +under test is otherwise untouched. + +### Real bug: labeled parameter versions could be silently evicted + +`PutParameter` caps stored history at 100 versions (`maxHistoryCap`), evicting the oldest entry on +overflow. AWS's actual behavior (confirmed via websearch of the +`ParameterMaxVersionLimitExceeded` exception docs) is that this eviction is refused — and the +whole `PutParameter` call fails with `ParameterMaxVersionLimitExceeded` — when the version about to +be evicted has a label attached, specifically so a labeled ("prod", "release-42", etc.) version is +never silently destroyed out from under a consumer pinned to that label. The emulator previously +evicted unconditionally. Fixed with a pre-mutation check (oldest history entry's +`parameterLabelsStore` entry) that aborts the whole write before any state changes if the oldest +version is labeled. Also closed a companion leak: `parameterLabels[name][version]` entries for +already-evicted versions were never deleted, so a parameter updated thousands of times would +accumulate stale label-map entries forever; eviction now deletes them. + +### Real bug: parameter name hierarchy depth was never validated + +AWS caps a parameter name at 15 "/"-delimited levels (confirmed via the `PutParameter` API +reference's own worked example: `/L1/.../L14/name` is valid, one more level throws +`HierarchyLevelLimitExceededException`). `validateParameterName` checked length, double-slashes, +reserved prefixes, and the name-charset regex, but never counted hierarchy depth. Added +`parameterHierarchyLevels`/`maxParamHierarchyLevels` and the new `ErrHierarchyLevelLimitExceeded` +sentinel, wired into `classifySSMError`. + +### Real bug: DescribeDocument/CreateDocument/UpdateDocument leaked Content in a metadata-only response + +AWS's real `DocumentDescription` structure (returned by all three ops) has **no `Content` field** +— confirmed by grepping `aws-sdk-go-v2/service/ssm/types/types.go` for `DocumentDescription +struct`. Only `GetDocument` returns document content; the metadata ops deliberately omit it (likely +so a `ListDocuments`-adjacent describe call doesn't have to re-transmit a potentially large +document body). This emulator's `CreateDocumentOutput`/`UpdateDocumentOutput`/ +`DescribeDocumentOutput` all embedded the full internal `Document` struct — which does carry +`Content` (no `omitempty`) for `GetDocument`'s own use — so every describe/create/update response +included the entire document body. A conformant SDK client ignores unknown response fields, so this +wasn't client-breaking, but it is a real wire-shape deviation (and a needless +content-in-metadata-response leak) per the audit's wire-shape-accuracy bar. Fixed by introducing a +separate `DocumentDescription` type (mirrors `Document` minus `Content`) and a +`Document.toDocumentDescription()` converter; the three ops now return that type. Covered by a new +JSON-serialization assertion (`Test_DescribeDocument_OmitsContentAndHonorsVersionSelector`) since a +Go zero-value-string field is indistinguishable from an absent field in a struct-level comparison +— only marshaling and checking the wire bytes actually catches this class of bug. + +### Real bug: explicit "$DEFAULT" document version was conflated with "$LATEST" + +`GetDocument` and `DescribeDocument` both special-cased `""`, `"$LATEST"`, and `"$DEFAULT"` +identically, always serving the document's latest content/metadata. But `$DEFAULT` is a distinct, +explicit selector — pinned independently via `UpdateDocumentDefaultVersion` — that can diverge from +`$LATEST` (create v1, `UpdateDocument` to v2, never repoint the default: v1 is still `$DEFAULT`, +v2 is `$LATEST`). A caller explicitly asking for `$DEFAULT` in that state got v2's content instead +of v1's. Fixed via a shared `resolveDocumentVersionSelector(doc, requested)` helper used by both +ops; `DescribeDocument` additionally now looks up the resolved version's own +`DocumentVersion`/`DocumentFormat`/`Status` from `documentVersionsStore` instead of always +reporting the top-level (latest) document's fields. + +**Deliberately NOT changed**: what an *omitted* `DocumentVersion` resolves to. AWS's API reference, +CLI reference, and user guide (all checked via WebFetch this pass) do not state whether omitting +the parameter is equivalent to `$DEFAULT` or `$LATEST` — evidence was genuinely ambiguous — and an +existing test (`document_test.go`'s `TestInMemoryBackend_Snapshot_IncludesDocumentsAndCommands` / +`document_survives_round_trip`) explicitly asserts that an omitted version returns the *latest* +content after an `UpdateDocument`. Changing that risked a real regression on weak secondary +evidence, so omitted-version behavior is left exactly as before (== `$LATEST`); only the +unambiguous explicit-`$DEFAULT` case was fixed. + +### Already-correct traps (do not re-flag) + +- `GetParametersByPath` (`MaxResults` 1-10, default 10) and `DescribeParameters`/ + `GetParameterHistory` (`MaxResults` 1-50, default 50) look asymmetric but are correct — these are + AWS's actual, independently-documented per-op limits, not a copy-paste inconsistency. +- `resolveTier`'s explicit-`Standard`-tier hard-fail on `Policies` is intentional per AWS + (`Standard tier parameters ... can't be configured to use parameter policies`) — do not "fix" it + to silently upgrade the tier the way `Intelligent-Tiering` does; only `Intelligent-Tiering` gets + auto-promotion, `Standard` is an explicit opt-out of that. +- `PutParameter`'s `Intelligent-Tiering` tier is echoed back verbatim in the response (`Tier: + "Intelligent-Tiering"`) when no promotion is needed — it does **not** resolve to the concrete + `"Standard"` tier in the wire response. The `ParameterTier` enum in + `aws-sdk-go-v2/service/ssm/types/enums.go` lists `Intelligent-Tiering` as a first-class value + distinct from `Standard`/`Advanced`, confirming AWS reports what was requested, not the + internally-selected concrete tier, except when a promotion actually occurs. +- `DeleteInventory` succeeding with `removed=0` for a `TypeName` with no stored items is correct, + not a missing not-found check — AWS's `DeleteInventory` operates on a type across the whole + fleet and a zero-item deletion is a valid, successful job (see `DeletionSummary.TotalCount`), not + an error. The unused `ErrInventoryNotFound`/`ErrDocumentVersionNotFound` (duplicate of + `ErrInvalidDocumentVersion`)/`ErrExecutionPreviewNotFound`/`ErrResourcePolicyNotFound` sentinels + declared in `backend_ops.go`/`backend_batch2.go` are dead code from an earlier pass, not evidence + of missing error handling — the operations that would use them either don't need a not-found path + (see DeleteInventory above) or already return a differently-named sentinel with the same string. diff --git a/services/ssm/automation_exec.go b/services/ssm/automation_exec.go index 0a2f94e81..dcdfae87c 100644 --- a/services/ssm/automation_exec.go +++ b/services/ssm/automation_exec.go @@ -71,7 +71,7 @@ func parseAutomationDocSteps(content string) []automationDocStep { // with b.mu held. func (b *InMemoryBackend) buildAutomationSteps(region, docName string) []AutomationStepExec { var content string - if doc, ok := b.documentsStore(region)[docName]; ok { + if doc, ok := b.documentsStore(region).Get(docName); ok { content = doc.Content } diff --git a/services/ssm/backend.go b/services/ssm/backend.go index 30f62821b..659b72059 100644 --- a/services/ssm/backend.go +++ b/services/ssm/backend.go @@ -21,6 +21,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/awsmeta" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) @@ -48,6 +49,8 @@ var ( ErrOpsMetadataNotFound = errors.New("OpsMetadataNotFoundException") ErrPatchBaselineNotFound = errors.New("DoesNotExistException") ErrOpsMetadataAlreadyExists = errors.New("OpsMetadataAlreadyExistsException") + ErrHierarchyLevelLimitExceeded = errors.New("HierarchyLevelLimitExceededException") + ErrParameterMaxVersionLimitExceeded = errors.New("ParameterMaxVersionLimitExceeded") ) const ( @@ -73,6 +76,11 @@ var validParamNameRegex = regexp.MustCompile(`^[a-zA-Z0-9._\-/]+$`) const maxParamNameLength = 2048 +// maxParamHierarchyLevels is the maximum number of "/"-delimited path segments +// allowed in a parameter name. AWS: "A parameter name hierarchy can have a +// maximum of 15 levels" — e.g. /L1/L2/.../L15/name is valid, one more is not. +const maxParamHierarchyLevels = 15 + // validateParameterName returns a ValidationException error when the name is invalid. func validateParameterName(name string) error { if len(name) > maxParamNameLength { @@ -106,9 +114,33 @@ func validateParameterName(name string) error { return fmt.Errorf("%w: parameter name contains invalid characters", ErrValidationException) } + if levels := parameterHierarchyLevels(name); levels > maxParamHierarchyLevels { + return fmt.Errorf( + "%w: parameter name hierarchy has %d levels, maximum is %d", + ErrHierarchyLevelLimitExceeded, levels, maxParamHierarchyLevels, + ) + } + return nil } +// parameterHierarchyLevels counts the "/"-delimited, non-empty segments of a +// parameter name, including the final name segment itself. AWS counts a +// leading slash as the hierarchy root (not a level) — e.g. "/a/b/c" has 3 +// levels, matching how AWS reports HierarchyLevelLimitExceededException. +func parameterHierarchyLevels(name string) int { + segments := strings.Split(name, "/") + + levels := 0 + for _, s := range segments { + if s != "" { + levels++ + } + } + + return levels +} + // aes256KeyLen is the byte length of an AES-256 key. const aes256KeyLen = 32 @@ -181,37 +213,38 @@ type KMSEncryptor interface { type InMemoryBackend struct { kms KMSEncryptor gcm cipher.AEAD // per-instance key; not shared across backends - activations map[string]map[string]Activation - maintenanceWindows map[string]map[string]MaintenanceWindow - maintenanceWindowTargets map[string]map[string]MaintenanceWindowTarget - maintenanceWindowTasks map[string]map[string]MaintenanceWindowTask - sessions map[string]map[string]Session + registry *store.Registry + activations map[string]*store.Table[Activation] + maintenanceWindows map[string]*store.Table[MaintenanceWindow] + maintenanceWindowTargets map[string]*store.Table[MaintenanceWindowTarget] + maintenanceWindowTasks map[string]*store.Table[MaintenanceWindowTask] + sessions map[string]*store.Table[Session] patchGroupToBaseline map[string]map[string]string tags map[string]map[string]*tags.Tags - associations map[string]map[string]Association + associations map[string]*store.Table[Association] documentVersions map[string]map[string][]DocumentVersion documentPermissions map[string]map[string][]string - commands map[string]map[string]Command + commands map[string]*store.Table[Command] commandInvocations map[string]map[string][]CommandInvocation history map[string]map[string][]ParameterHistory - parameters map[string]map[string]Parameter - documents map[string]map[string]Document - opsItems map[string]map[string]OpsItem + parameters map[string]*store.Table[Parameter] + documents map[string]*store.Table[Document] + opsItems map[string]*store.Table[OpsItem] opsItemRelatedItems map[string]map[string][]OpsItemRelatedItem - opsMetadata map[string]map[string]OpsMetadata - patchBaselines map[string]map[string]PatchBaseline + opsMetadata map[string]*store.Table[OpsMetadata] + patchBaselines map[string]*store.Table[PatchBaseline] inventory map[string]map[string][]InventoryItem // key: instanceID compliance map[string]map[string][]ComplianceItem // key: resourceID - resourceDataSyncs map[string]map[string]*ResourceDataSync - parameterLabels map[string]map[string]map[int64][]string // paramName → version → labels (0 = latest) - automationExecutions map[string]map[string]*AutomationExecution // executionID → exec - serviceSettings map[string]map[string]*ServiceSetting // settingID → setting - resourcePolicies map[string]map[string][]*ResourcePolicy // resourceARN → policies - executionPreviews map[string]map[string]*ExecutionPreview // previewID → preview - instancePatchStates map[string]map[string]*InstancePatchState // region → instanceID → state - instancePatches map[string]map[string][]PatchComplianceData // region → instanceID → patches - instanceProperties map[string]map[string]*InstanceProperty // region → instanceID → properties - availablePatches map[string][]Patch // region → patches + resourceDataSyncs map[string]*store.Table[ResourceDataSync] + parameterLabels map[string]map[string]map[int64][]string // paramName → version → labels (0 = latest) + automationExecutions map[string]*store.Table[AutomationExecution] // executionID → exec + serviceSettings map[string]*store.Table[ServiceSetting] // settingID → setting + resourcePolicies map[string]map[string][]*ResourcePolicy // resourceARN → policies + executionPreviews map[string]*store.Table[ExecutionPreview] // previewID → preview + instancePatchStates map[string]*store.Table[InstancePatchState] // region → instanceID → state + instancePatches map[string]map[string][]PatchComplianceData // region → instanceID → patches + instanceProperties map[string]*store.Table[InstanceProperty] // region → instanceID → properties + availablePatches map[string][]Patch // region → patches mu *lockmetrics.RWMutex miscResourceTags map[string]map[string]map[string]string resourceIDToOpsMetadataArn map[string]map[string]string @@ -234,36 +267,37 @@ type InMemoryBackend struct { func NewInMemoryBackend() *InMemoryBackend { b := &InMemoryBackend{ gcm: newInstanceGCM(), - parameters: make(map[string]map[string]Parameter), + registry: store.NewRegistry(), + parameters: make(map[string]*store.Table[Parameter]), history: make(map[string]map[string][]ParameterHistory), tags: make(map[string]map[string]*tags.Tags), - documents: make(map[string]map[string]Document), + documents: make(map[string]*store.Table[Document]), documentVersions: make(map[string]map[string][]DocumentVersion), documentPermissions: make(map[string]map[string][]string), - commands: make(map[string]map[string]Command), + commands: make(map[string]*store.Table[Command]), commandInvocations: make(map[string]map[string][]CommandInvocation), - activations: make(map[string]map[string]Activation), - associations: make(map[string]map[string]Association), - maintenanceWindows: make(map[string]map[string]MaintenanceWindow), - maintenanceWindowTargets: make(map[string]map[string]MaintenanceWindowTarget), - maintenanceWindowTasks: make(map[string]map[string]MaintenanceWindowTask), - sessions: make(map[string]map[string]Session), + activations: make(map[string]*store.Table[Activation]), + associations: make(map[string]*store.Table[Association]), + maintenanceWindows: make(map[string]*store.Table[MaintenanceWindow]), + maintenanceWindowTargets: make(map[string]*store.Table[MaintenanceWindowTarget]), + maintenanceWindowTasks: make(map[string]*store.Table[MaintenanceWindowTask]), + sessions: make(map[string]*store.Table[Session]), patchGroupToBaseline: make(map[string]map[string]string), - opsItems: make(map[string]map[string]OpsItem), + opsItems: make(map[string]*store.Table[OpsItem]), opsItemRelatedItems: make(map[string]map[string][]OpsItemRelatedItem), - opsMetadata: make(map[string]map[string]OpsMetadata), - patchBaselines: make(map[string]map[string]PatchBaseline), + opsMetadata: make(map[string]*store.Table[OpsMetadata]), + patchBaselines: make(map[string]*store.Table[PatchBaseline]), inventory: make(map[string]map[string][]InventoryItem), compliance: make(map[string]map[string][]ComplianceItem), - resourceDataSyncs: make(map[string]map[string]*ResourceDataSync), + resourceDataSyncs: make(map[string]*store.Table[ResourceDataSync]), parameterLabels: make(map[string]map[string]map[int64][]string), - automationExecutions: make(map[string]map[string]*AutomationExecution), - serviceSettings: make(map[string]map[string]*ServiceSetting), + automationExecutions: make(map[string]*store.Table[AutomationExecution]), + serviceSettings: make(map[string]*store.Table[ServiceSetting]), resourcePolicies: make(map[string]map[string][]*ResourcePolicy), - executionPreviews: make(map[string]map[string]*ExecutionPreview), - instancePatchStates: make(map[string]map[string]*InstancePatchState), + executionPreviews: make(map[string]*store.Table[ExecutionPreview]), + instancePatchStates: make(map[string]*store.Table[InstancePatchState]), instancePatches: make(map[string]map[string][]PatchComplianceData), - instanceProperties: make(map[string]map[string]*InstanceProperty), + instanceProperties: make(map[string]*store.Table[InstanceProperty]), availablePatches: make(map[string][]Patch), commandExpirySecs: defaultCommandExpirySecs, mu: lockmetrics.New("ssm"), @@ -332,8 +366,8 @@ func getRegion(ctx context.Context) string { return defaultRegion } -func (b *InMemoryBackend) parametersStore(region string) map[string]Parameter { - return b.parameters[region] +func (b *InMemoryBackend) parametersStore(region string) *store.Table[Parameter] { + return getOrCreateTable(b, b.parameters, "parameters", region, parameterKeyFn) } func (b *InMemoryBackend) historyStore(region string) map[string][]ParameterHistory { @@ -344,8 +378,8 @@ func (b *InMemoryBackend) tagsStore(region string) map[string]*tags.Tags { return b.tags[region] } -func (b *InMemoryBackend) documentsStore(region string) map[string]Document { - return b.documents[region] +func (b *InMemoryBackend) documentsStore(region string) *store.Table[Document] { + return getOrCreateTable(b, b.documents, "documents", region, documentKeyFn) } func (b *InMemoryBackend) documentVersionsStore(region string) map[string][]DocumentVersion { @@ -356,20 +390,20 @@ func (b *InMemoryBackend) documentPermissionsStore(region string) map[string][]s return b.documentPermissions[region] } -func (b *InMemoryBackend) commandsStore(region string) map[string]Command { - return b.commands[region] +func (b *InMemoryBackend) commandsStore(region string) *store.Table[Command] { + return getOrCreateTable(b, b.commands, "commands", region, commandKeyFn) } func (b *InMemoryBackend) commandInvocationsStore(region string) map[string][]CommandInvocation { return b.commandInvocations[region] } -func (b *InMemoryBackend) activationsStore(region string) map[string]Activation { - return b.activations[region] +func (b *InMemoryBackend) activationsStore(region string) *store.Table[Activation] { + return getOrCreateTable(b, b.activations, "activations", region, activationKeyFn) } -func (b *InMemoryBackend) associationsStore(region string) map[string]Association { - return b.associations[region] +func (b *InMemoryBackend) associationsStore(region string) *store.Table[Association] { + return getOrCreateTable(b, b.associations, "associations", region, associationKeyFn) } func (b *InMemoryBackend) associationExecutionsStore( @@ -384,44 +418,48 @@ func (b *InMemoryBackend) associationExecTargetsStore( return b.associationExecTargets[region] } -func (b *InMemoryBackend) maintenanceWindowsStore(region string) map[string]MaintenanceWindow { - return b.maintenanceWindows[region] +func (b *InMemoryBackend) maintenanceWindowsStore(region string) *store.Table[MaintenanceWindow] { + return getOrCreateTable(b, b.maintenanceWindows, "maintenanceWindows", region, maintenanceWindowKeyFn) } func (b *InMemoryBackend) maintenanceWindowTargetsStore( region string, -) map[string]MaintenanceWindowTarget { - return b.maintenanceWindowTargets[region] +) *store.Table[MaintenanceWindowTarget] { + return getOrCreateTable( + b, b.maintenanceWindowTargets, "maintenanceWindowTargets", region, maintenanceWindowTargetKeyFn, + ) } func (b *InMemoryBackend) maintenanceWindowTasksStore( region string, -) map[string]MaintenanceWindowTask { - return b.maintenanceWindowTasks[region] +) *store.Table[MaintenanceWindowTask] { + return getOrCreateTable( + b, b.maintenanceWindowTasks, "maintenanceWindowTasks", region, maintenanceWindowTaskKeyFn, + ) } -func (b *InMemoryBackend) sessionsStore(region string) map[string]Session { - return b.sessions[region] +func (b *InMemoryBackend) sessionsStore(region string) *store.Table[Session] { + return getOrCreateTable(b, b.sessions, "sessions", region, sessionKeyFn) } func (b *InMemoryBackend) patchGroupToBaselineStore(region string) map[string]string { return b.patchGroupToBaseline[region] } -func (b *InMemoryBackend) opsItemsStore(region string) map[string]OpsItem { - return b.opsItems[region] +func (b *InMemoryBackend) opsItemsStore(region string) *store.Table[OpsItem] { + return getOrCreateTable(b, b.opsItems, "opsItems", region, opsItemKeyFn) } func (b *InMemoryBackend) opsItemRelatedItemsStore(region string) map[string][]OpsItemRelatedItem { return b.opsItemRelatedItems[region] } -func (b *InMemoryBackend) opsMetadataStore(region string) map[string]OpsMetadata { - return b.opsMetadata[region] +func (b *InMemoryBackend) opsMetadataStore(region string) *store.Table[OpsMetadata] { + return getOrCreateTable(b, b.opsMetadata, "opsMetadata", region, opsMetadataKeyFn) } -func (b *InMemoryBackend) patchBaselinesStore(region string) map[string]PatchBaseline { - return b.patchBaselines[region] +func (b *InMemoryBackend) patchBaselinesStore(region string) *store.Table[PatchBaseline] { + return getOrCreateTable(b, b.patchBaselines, "patchBaselines", region, patchBaselineKeyFn) } func (b *InMemoryBackend) inventoryStore(region string) map[string][]InventoryItem { @@ -436,28 +474,30 @@ func (b *InMemoryBackend) complianceStore(region string) map[string][]Compliance return b.compliance[region] } -func (b *InMemoryBackend) resourceDataSyncsStore(region string) map[string]*ResourceDataSync { - return b.resourceDataSyncs[region] +func (b *InMemoryBackend) resourceDataSyncsStore(region string) *store.Table[ResourceDataSync] { + return getOrCreateTable(b, b.resourceDataSyncs, "resourceDataSyncs", region, resourceDataSyncKeyFn) } func (b *InMemoryBackend) parameterLabelsStore(region string) map[string]map[int64][]string { return b.parameterLabels[region] } -func (b *InMemoryBackend) automationExecutionsStore(region string) map[string]*AutomationExecution { - return b.automationExecutions[region] +func (b *InMemoryBackend) automationExecutionsStore(region string) *store.Table[AutomationExecution] { + return getOrCreateTable( + b, b.automationExecutions, "automationExecutions", region, automationExecutionKeyFn, + ) } -func (b *InMemoryBackend) serviceSettingsStore(region string) map[string]*ServiceSetting { - return b.serviceSettings[region] +func (b *InMemoryBackend) serviceSettingsStore(region string) *store.Table[ServiceSetting] { + return getOrCreateTable(b, b.serviceSettings, "serviceSettings", region, serviceSettingKeyFn) } func (b *InMemoryBackend) resourcePoliciesStore(region string) map[string][]*ResourcePolicy { return b.resourcePolicies[region] } -func (b *InMemoryBackend) executionPreviewsStore(region string) map[string]*ExecutionPreview { - return b.executionPreviews[region] +func (b *InMemoryBackend) executionPreviewsStore(region string) *store.Table[ExecutionPreview] { + return getOrCreateTable(b, b.executionPreviews, "executionPreviews", region, executionPreviewKeyFn) } func (b *InMemoryBackend) miscResourceTagsStore(region string) map[string]map[string]string { @@ -472,16 +512,16 @@ func (b *InMemoryBackend) opsItemEventsStore(region string) []OpsItemEventSummar return b.opsItemEvents[region] } -func (b *InMemoryBackend) instancePatchStatesStore(region string) map[string]*InstancePatchState { - return b.instancePatchStates[region] +func (b *InMemoryBackend) instancePatchStatesStore(region string) *store.Table[InstancePatchState] { + return getOrCreateTable(b, b.instancePatchStates, "instancePatchStates", region, instancePatchStateKeyFn) } func (b *InMemoryBackend) instancePatchesStore(region string) map[string][]PatchComplianceData { return b.instancePatches[region] } -func (b *InMemoryBackend) instancePropertiesStore(region string) map[string]*InstanceProperty { - return b.instanceProperties[region] +func (b *InMemoryBackend) instancePropertiesStore(region string) *store.Table[InstanceProperty] { + return getOrCreateTable(b, b.instanceProperties, "instanceProperties", region, instancePropertyKeyFn) } // encryptSSMValue encrypts a SecureString value using KMS (when keyID is set @@ -574,32 +614,53 @@ func validateAllowedPattern(pattern, value string) error { return nil } -// resolveTier canonicalises the tier string and enforces per-tier value size limits. -// Returns the resolved tier name or an error. -func resolveTier(tier, value string) (string, error) { +// resolveTier canonicalises the tier string and enforces per-tier value size +// limits, returning the resolved tier name or an error. +// +// AWS auto-upgrades Intelligent-Tiering parameters to Advanced whenever the +// request needs a capability Standard doesn't support — a value over 4 KiB or +// parameter policies attached — rather than rejecting the request. An +// explicit Standard tier still hard-fails on those same conditions, since the +// caller opted out of auto-selection. +func resolveTier(tier, value, policies string) (string, error) { if tier == "" { tier = tierStandard } + needsAdvanced := len(value) > maxStandardValueBytes || policies != "" + switch tier { - case tierStandard, tierIntelligentTiering: + case tierIntelligentTiering: + if needsAdvanced { + tier = tierAdvanced + } + case tierStandard: if len(value) > maxStandardValueBytes { return "", fmt.Errorf( - "%w: parameter value exceeds %d bytes for %s tier", - ErrValidationException, maxStandardValueBytes, tier, + "%w: parameter value exceeds %d bytes for Standard tier", + ErrValidationException, maxStandardValueBytes, ) } - case tierAdvanced: - if len(value) > maxAdvancedValueBytes { + + if policies != "" { return "", fmt.Errorf( - "%w: parameter value exceeds %d bytes for Advanced tier", - ErrValidationException, maxAdvancedValueBytes, + "%w: parameter policies are only supported for Advanced tier parameters", + ErrValidationException, ) } + case tierAdvanced: + // already Advanced; size checked below. default: return "", fmt.Errorf("%w: invalid Tier %q", ErrValidationException, tier) } + if tier == tierAdvanced && len(value) > maxAdvancedValueBytes { + return "", fmt.Errorf( + "%w: parameter value exceeds %d bytes for Advanced tier", + ErrValidationException, maxAdvancedValueBytes, + ) + } + return tier, nil } @@ -637,11 +698,13 @@ func splitParameterSelector(name string) (string, string) { func (b *InMemoryBackend) resolveParameterSelector( region, baseName, selector string, ) (Parameter, error) { - current, exists := b.parametersStore(region)[baseName] + currentPtr, exists := b.parametersStore(region).Get(baseName) if !exists { return Parameter{}, ErrParameterNotFound } + current := *currentPtr + if selector == "" { return current, nil } @@ -752,7 +815,7 @@ func validatePutParameterInput(input *PutParameterInput) (putParameterValidated, return putParameterValidated{}, err } - tier, err := resolveTier(input.Tier, input.Value) + tier, err := resolveTier(input.Tier, input.Value, input.Policies) if err != nil { return putParameterValidated{}, err } @@ -776,18 +839,15 @@ func (b *InMemoryBackend) PutParameter( b.mu.Lock("PutParameter") defer b.mu.Unlock() - if b.parameters[region] == nil { - b.parameters[region] = make(map[string]Parameter) - } params := b.parametersStore(region) - existing, exists := params[input.Name] + existingPtr, exists := params.Get(input.Name) if exists && !input.Overwrite { return nil, ErrParameterAlreadyExists } version := int64(1) if exists { - version = existing.Version + 1 + version = existingPtr.Version + 1 } // Encrypt if SecureString type; use KMS when a KeyID is specified. @@ -814,7 +874,25 @@ func (b *InMemoryBackend) PutParameter( Policies: input.Policies, } - params[input.Name] = param + // AWS retains only the most recent maxHistoryCap versions of a parameter, + // automatically deleting the oldest version when a new one is created. If + // the oldest version has a label attached, AWS refuses to delete it (and + // therefore refuses to create the new version) with + // ParameterMaxVersionLimitExceeded, since that would silently orphan the + // label. Check this before mutating any state. + if existingHist := b.historyStore(region)[input.Name]; len(existingHist) >= maxHistoryCap { + oldest := existingHist[0] + if labels := b.parameterLabelsStore(region)[input.Name][oldest.Version]; len(labels) > 0 { + return nil, fmt.Errorf( + "%w: version %d, the oldest version, can't be deleted because it has a label"+ + " associated with it. Move the label to another version of the parameter,"+ + " and try again", + ErrParameterMaxVersionLimitExceeded, oldest.Version, + ) + } + } + + params.Put(¶m) // Store in history (store encrypted value for SecureString) paramHistory := ParameterHistory{ @@ -838,7 +916,18 @@ func (b *InMemoryBackend) PutParameter( // Cap history to the most recent maxHistoryCap entries to prevent unbounded growth. if len(history[input.Name]) > maxHistoryCap { + evicted := history[input.Name][:len(history[input.Name])-maxHistoryCap] history[input.Name] = history[input.Name][len(history[input.Name])-maxHistoryCap:] + + // Evicted versions can never be labeled here (the pre-check above bars + // evicting a labeled oldest version) but their version-label map + // entries — created lazily on first label — may still exist as empty + // slices; drop them so parameterLabels doesn't grow unboundedly. + if versionLabels := b.parameterLabelsStore(region)[input.Name]; versionLabels != nil { + for _, ev := range evicted { + delete(versionLabels, ev.Version) + } + } } return &PutParameterOutput{Version: version, Tier: tier}, nil @@ -944,11 +1033,11 @@ func (b *InMemoryBackend) DeleteParameter( defer b.mu.Unlock() params := b.parametersStore(region) - if _, exists := params[input.Name]; !exists { + if !params.Has(input.Name) { return nil, ErrParameterNotFound } - delete(params, input.Name) + params.Delete(input.Name) delete(b.historyStore(region), input.Name) tags := b.tagsStore(region) @@ -982,8 +1071,8 @@ func (b *InMemoryBackend) DeleteParameters( } for _, name := range input.Names { - if _, exists := params[name]; exists { - delete(params, name) + if params.Has(name) { + params.Delete(name) delete(history, name) if t, ok := tags[name]; ok { t.Close() @@ -1093,10 +1182,10 @@ func (b *InMemoryBackend) ListAll(ctx context.Context) []Parameter { b.mu.RLock("ListAll") defer b.mu.RUnlock() - paramsMap := b.parametersStore(region) - params := make([]Parameter, 0, len(paramsMap)) - for _, p := range paramsMap { - params = append(params, p) + paramsTable := b.parametersStore(region) + params := make([]Parameter, 0, paramsTable.Len()) + for _, p := range paramsTable.All() { + params = append(params, *p) } sort.Slice(params, func(i, j int) bool { @@ -1131,13 +1220,14 @@ func paramByPathMatchesFilters(param Parameter, filters []ParameterFilter) bool // approach that required maintaining a sorted slice on every PutParameter write // (O(n) insert); the emulator write path is now O(1) and reads are O(n log n). func collectPathParams( - store map[string]Parameter, + paramsTable *store.Table[Parameter], path string, recursive bool, filters []ParameterFilter, ) []Parameter { var matched []Parameter - for name, param := range store { + for _, p := range paramsTable.All() { + name, param := p.Name, *p if !strings.HasPrefix(name, path) { continue } @@ -1167,11 +1257,18 @@ func cleanupEmptyInnerMap[V any](outer map[string]map[string]V, region string) { } } -// cleanupEmptyParamRegion removes the per-region inner maps for parameters, -// history, and tags when the last parameter in a region is deleted. +// cleanupEmptyParamRegion removes the per-region inner maps for history and +// tags when the last parameter in a region is deleted. b.parameters is +// intentionally NOT pruned here: it is now a *store.Table[Parameter] per +// region, registered once by name ("parameters/"+region) on b.registry (see +// store_setup.go's getOrCreateTable); store.Registry has no unregister, so +// deleting the region's entry from b.parameters would make a later write to +// the same region re-register the same name and panic. An empty Table left +// in place is observably identical to an absent one (Len() == 0, no +// entries) — this only affects internal bookkeeping, never a caller-visible +// response. // Caller must hold the write lock. func (b *InMemoryBackend) cleanupEmptyParamRegion(region string) { - cleanupEmptyInnerMap(b.parameters, region) cleanupEmptyInnerMap(b.history, region) cleanupEmptyInnerMap(b.tags, region) } @@ -1270,10 +1367,10 @@ func (b *InMemoryBackend) DescribeParameters( b.mu.RLock("DescribeParameters") defer b.mu.RUnlock() - paramsMap := b.parametersStore(region) - all := make([]ParameterMetadata, 0, len(paramsMap)) + paramsTable := b.parametersStore(region) + all := make([]ParameterMetadata, 0, paramsTable.Len()) - for _, p := range paramsMap { + for _, p := range paramsTable.All() { all = append(all, ParameterMetadata{ Name: p.Name, Type: p.Type, @@ -1424,7 +1521,7 @@ func (b *InMemoryBackend) AddTagsToResource( params := b.parametersStore(region) name := input.ResourceID - if _, ok := params[name]; !ok { + if !params.Has(name) { return ErrParameterNotFound } if b.tags[region] == nil { @@ -1471,7 +1568,7 @@ func (b *InMemoryBackend) RemoveTagsFromResource( params := b.parametersStore(region) name := input.ResourceID - if _, ok := params[name]; !ok { + if !params.Has(name) { return ErrParameterNotFound } tagsStore := b.tagsStore(region) @@ -1508,7 +1605,7 @@ func (b *InMemoryBackend) ListTagsForResource( params := b.parametersStore(region) name := input.ResourceID - if _, ok := params[name]; !ok { + if !params.Has(name) { return nil, ErrParameterNotFound } var tagList []Tag @@ -1565,9 +1662,6 @@ func (b *InMemoryBackend) registerDefaultDocuments(region string) { }, } - if b.documents[region] == nil { - b.documents[region] = make(map[string]Document) - } if b.documentVersions[region] == nil { b.documentVersions[region] = make(map[string][]DocumentVersion) } @@ -1588,7 +1682,7 @@ func (b *InMemoryBackend) registerDefaultDocuments(region string) { LatestVersion: "1", DefaultVersion: "1", } - documents[d.name] = doc + documents.Put(&doc) documentVersions[d.name] = []DocumentVersion{ { Name: d.name, @@ -1614,11 +1708,8 @@ func (b *InMemoryBackend) CreateDocument( b.mu.Lock("CreateDocument") defer b.mu.Unlock() - if b.documents[region] == nil { - b.documents[region] = make(map[string]Document) - } - store := b.documentsStore(region) - if _, exists := store[input.Name]; exists { + documentsTable := b.documentsStore(region) + if documentsTable.Has(input.Name) { return nil, ErrDocumentAlreadyExists } @@ -1650,7 +1741,7 @@ func (b *InMemoryBackend) CreateDocument( Requires: input.Requires, } - store[input.Name] = doc + documentsTable.Put(&doc) if b.documentVersions[region] == nil { b.documentVersions[region] = make(map[string][]DocumentVersion) } @@ -1667,7 +1758,52 @@ func (b *InMemoryBackend) CreateDocument( }, } - return &CreateDocumentOutput{DocumentDescription: doc}, nil + return &CreateDocumentOutput{DocumentDescription: doc.toDocumentDescription()}, nil +} + +// toDocumentDescription converts an internal Document (which carries Content +// for GetDocument's use) to the wire-accurate DocumentDescription shape +// returned by CreateDocument/UpdateDocument/DescribeDocument — real AWS never +// includes Content in these metadata responses. +func (d Document) toDocumentDescription() DocumentDescription { + return DocumentDescription{ + TargetType: d.TargetType, + LatestVersion: d.LatestVersion, + DocumentType: d.DocumentType, + DocumentFormat: d.DocumentFormat, + Status: d.Status, + StatusInformation: d.StatusInformation, + DefaultVersion: d.DefaultVersion, + Name: d.Name, + SchemaVersion: d.SchemaVersion, + Description: d.Description, + DocumentVersion: d.DocumentVersion, + PlatformTypes: d.PlatformTypes, + Attachments: d.Attachments, + Requires: d.Requires, + CreatedDate: d.CreatedDate, + } +} + +// resolveDocumentVersionSelector resolves the "$LATEST"/"$DEFAULT" selectors +// to a concrete version string. An explicit "$DEFAULT" always resolves to +// the document's DefaultVersion (set by UpdateDocumentDefaultVersion), which +// can genuinely differ from LatestVersion — this emulator previously +// conflated the two, always serving the latest content even when $DEFAULT +// was explicitly requested. An omitted DocumentVersion is treated the same +// as this emulator has always treated it (latest), since AWS's own docs +// don't state a default and existing callers depend on that behavior. +func resolveDocumentVersionSelector(doc Document, requested string) string { + switch requested { + case "": + return doc.LatestVersion + case "$DEFAULT": + return doc.DefaultVersion + case "$LATEST": + return doc.LatestVersion + default: + return requested + } } // GetDocument retrieves a document's content. @@ -1679,40 +1815,32 @@ func (b *InMemoryBackend) GetDocument( b.mu.RLock("GetDocument") defer b.mu.RUnlock() - doc, exists := b.documentsStore(region)[input.Name] + docPtr, exists := b.documentsStore(region).Get(input.Name) if !exists { return nil, ErrDocumentNotFound } - content := doc.Content - version := doc.DocumentVersion + doc := *docPtr - if input.DocumentVersion != "" && input.DocumentVersion != "$LATEST" && - input.DocumentVersion != "$DEFAULT" { - versions := b.documentVersionsStore(region)[input.Name] - found := false - for _, v := range versions { - if v.DocumentVersion == input.DocumentVersion { - found = true - version = v.DocumentVersion - content = v.Content + target := resolveDocumentVersionSelector(doc, input.DocumentVersion) - break - } - } - if !found { - return nil, ErrInvalidDocumentVersion + versions := b.documentVersionsStore(region)[input.Name] + for _, v := range versions { + if v.DocumentVersion != target { + continue } + + return &GetDocumentOutput{ + Name: doc.Name, + Content: v.Content, + DocumentType: doc.DocumentType, + DocumentFormat: v.DocumentFormat, + DocumentVersion: v.DocumentVersion, + Status: v.Status, + }, nil } - return &GetDocumentOutput{ - Name: doc.Name, - Content: content, - DocumentType: doc.DocumentType, - DocumentFormat: doc.DocumentFormat, - DocumentVersion: version, - Status: doc.Status, - }, nil + return nil, ErrInvalidDocumentVersion } // documentMatchesFilters returns true when doc satisfies all provided DocumentFilters. @@ -1747,12 +1875,39 @@ func (b *InMemoryBackend) DescribeDocument( b.mu.RLock("DescribeDocument") defer b.mu.RUnlock() - doc, exists := b.documentsStore(region)[input.Name] + docPtr, exists := b.documentsStore(region).Get(input.Name) if !exists { return nil, ErrDocumentNotFound } - return &DescribeDocumentOutput{Document: doc}, nil + doc := *docPtr + + description := doc.toDocumentDescription() + + // Honor a specific/$LATEST/$DEFAULT DocumentVersion selector: the + // per-version fields (DocumentVersion, DocumentFormat, Status) must + // reflect the resolved version, not always the latest. + target := resolveDocumentVersionSelector(doc, input.DocumentVersion) + if target != doc.DocumentVersion { + found := false + + for _, v := range b.documentVersionsStore(region)[input.Name] { + if v.DocumentVersion == target { + description.DocumentVersion = v.DocumentVersion + description.DocumentFormat = v.DocumentFormat + description.Status = v.Status + found = true + + break + } + } + + if !found { + return nil, ErrInvalidDocumentVersion + } + } + + return &DescribeDocumentOutput{Document: description}, nil } // ListDocuments returns a list of document identifiers filtered by key-value criteria. @@ -1769,9 +1924,10 @@ func (b *InMemoryBackend) ListDocuments( allFilters = append(allFilters, input.Filters...) allFilters = append(allFilters, input.DocumentFilters...) - store := b.documentsStore(region) - all := make([]DocumentIdentifier, 0, len(store)) - for _, doc := range store { + docsTable := b.documentsStore(region) + all := make([]DocumentIdentifier, 0, docsTable.Len()) + for _, docPtr := range docsTable.All() { + doc := *docPtr if !documentMatchesFilters(doc, allFilters) { continue } @@ -1824,12 +1980,14 @@ func (b *InMemoryBackend) UpdateDocument( b.mu.Lock("UpdateDocument") defer b.mu.Unlock() - store := b.documentsStore(region) - doc, exists := store[input.Name] + docsTable := b.documentsStore(region) + docPtr, exists := docsTable.Get(input.Name) if !exists { return nil, ErrDocumentNotFound } + doc := *docPtr + // Validate DocumentVersion if provided. if input.DocumentVersion != "" { switch input.DocumentVersion { @@ -1853,7 +2011,7 @@ func (b *InMemoryBackend) UpdateDocument( doc.DocumentVersion = newVer doc.LatestVersion = newVer doc.DocumentFormat = format - store[input.Name] = doc + docsTable.Put(&doc) versionStore := b.documentVersionsStore(region) versionStore[input.Name] = append(versionStore[input.Name], DocumentVersion{ @@ -1871,7 +2029,7 @@ func (b *InMemoryBackend) UpdateDocument( versionStore[input.Name] = vers[len(vers)-maxDocumentVersionCap:] } - return &UpdateDocumentOutput{DocumentDescription: doc}, nil + return &UpdateDocumentOutput{DocumentDescription: doc.toDocumentDescription()}, nil } // DeleteDocument removes a document and all its versions and permissions. @@ -1883,15 +2041,17 @@ func (b *InMemoryBackend) DeleteDocument( b.mu.Lock("DeleteDocument") defer b.mu.Unlock() - if _, exists := b.documentsStore(region)[input.Name]; !exists { + if !b.documentsStore(region).Has(input.Name) { return nil, ErrDocumentNotFound } - delete(b.documentsStore(region), input.Name) + b.documentsStore(region).Delete(input.Name) delete(b.documentVersionsStore(region), input.Name) delete(b.documentPermissionsStore(region), input.Name) - cleanupEmptyInnerMap(b.documents, region) + // b.documents itself is not pruned here — see the comment on + // cleanupEmptyParamRegion above for why a Table-backed region entry must + // never be removed from its outer map once registered. cleanupEmptyInnerMap(b.documentVersions, region) cleanupEmptyInnerMap(b.documentPermissions, region) @@ -1907,7 +2067,7 @@ func (b *InMemoryBackend) DescribeDocumentPermission( b.mu.RLock("DescribeDocumentPermission") defer b.mu.RUnlock() - if _, exists := b.documentsStore(region)[input.Name]; !exists { + if !b.documentsStore(region).Has(input.Name) { return nil, ErrDocumentNotFound } @@ -1931,15 +2091,15 @@ func (b *InMemoryBackend) ModifyDocumentPermission( b.mu.Lock("ModifyDocumentPermission") defer b.mu.Unlock() - if _, exists := b.documentsStore(region)[input.Name]; !exists { + if !b.documentsStore(region).Has(input.Name) { return nil, ErrDocumentNotFound } if b.documentPermissions[region] == nil { b.documentPermissions[region] = make(map[string][]string) } - store := b.documentPermissionsStore(region) - current := store[input.Name] + permStore := b.documentPermissionsStore(region) + current := permStore[input.Name] for _, id := range input.AccountIDsToAdd { if !slices.Contains(current, id) { @@ -1951,7 +2111,7 @@ func (b *InMemoryBackend) ModifyDocumentPermission( current = slices.DeleteFunc(current, func(v string) bool { return v == id }) } - store[input.Name] = current + permStore[input.Name] = current return &ModifyDocumentPermissionOutput{}, nil } @@ -1965,7 +2125,7 @@ func (b *InMemoryBackend) ListDocumentVersions( b.mu.RLock("ListDocumentVersions") defer b.mu.RUnlock() - if _, exists := b.documentsStore(region)[input.Name]; !exists { + if !b.documentsStore(region).Has(input.Name) { return nil, ErrDocumentNotFound } @@ -2012,7 +2172,7 @@ func (b *InMemoryBackend) SendCommand( // documents that exist account-wide without needing to be created first; // only it (of the ones this emulator can act on) is recognised implicitly // here rather than requiring pre-registration like customer documents. - _, exists := b.documentsStore(region)[input.DocumentName] + exists := b.documentsStore(region).Has(input.DocumentName) if !exists && input.DocumentName != docRunPatchBaseline { return nil, ErrDocumentNotFound } @@ -2044,10 +2204,7 @@ func (b *InMemoryBackend) SendCommand( OutputS3Region: input.OutputS3Region, } - if b.commands[region] == nil { - b.commands[region] = make(map[string]Command) - } - b.commandsStore(region)[cmdID] = cmd + b.commandsStore(region).Put(&cmd) stdout, stderr, finalStatus := renderCommandOutput(input.DocumentName, input.Parameters) @@ -2087,13 +2244,15 @@ func (b *InMemoryBackend) SendCommand( if b.commandExecDelaySecs <= 0 { b.completeCommand(region, cmdID) } else { - pending := b.commandsStore(region)[cmdID] + pendingPtr, _ := b.commandsStore(region).Get(cmdID) + pending := *pendingPtr pending.completeAfter = now + b.commandExecDelaySecs - b.commandsStore(region)[cmdID] = pending + b.commandsStore(region).Put(&pending) } // Return a snapshot of the current state. - finalCmd := b.commandsStore(region)[cmdID] + finalCmdPtr, _ := b.commandsStore(region).Get(cmdID) + finalCmd := *finalCmdPtr return &SendCommandOutput{Command: finalCmd}, nil } @@ -2101,16 +2260,17 @@ func (b *InMemoryBackend) SendCommand( // setCommandStatus mutates the command and all its invocations to the given // non-terminal status. Must be called with b.mu held for writing. func (b *InMemoryBackend) setCommandStatus(region, cmdID, status string) { - store := b.commandsStore(region) + cmdTable := b.commandsStore(region) - cmd, ok := store[cmdID] + cmdPtr, ok := cmdTable.Get(cmdID) if !ok { return } + cmd := *cmdPtr cmd.Status = status cmd.StatusDetails = status - store[cmdID] = cmd + cmdTable.Put(&cmd) invStore := b.commandInvocationsStore(region) invs := invStore[cmdID] @@ -2128,13 +2288,15 @@ func (b *InMemoryBackend) setCommandStatus(region, cmdID, status string) { // worst per-invocation status (Failed dominates Success). Must be called with // b.mu held for writing. func (b *InMemoryBackend) completeCommand(region, cmdID string) { - store := b.commandsStore(region) + cmdTable := b.commandsStore(region) - cmd, ok := store[cmdID] + cmdPtr, ok := cmdTable.Get(cmdID) if !ok { return } + cmd := *cmdPtr + invStore := b.commandInvocationsStore(region) invs := invStore[cmdID] @@ -2161,18 +2323,18 @@ func (b *InMemoryBackend) completeCommand(region, cmdID string) { cmd.Status = overall cmd.StatusDetails = overall cmd.completeAfter = 0 - store[cmdID] = cmd + cmdTable.Put(&cmd) } // materializeCommandLocked lazily completes an InProgress command whose exec // delay has elapsed. Must be called with b.mu held for writing. func (b *InMemoryBackend) materializeCommandLocked(region, cmdID string, nowUnix float64) { - cmd, ok := b.commandsStore(region)[cmdID] - if !ok || cmd.Status != commandStatusInProgress { + cmdPtr, ok := b.commandsStore(region).Get(cmdID) + if !ok || cmdPtr.Status != commandStatusInProgress { return } - if cmd.completeAfter == 0 || nowUnix >= cmd.completeAfter { + if cmdPtr.completeAfter == 0 || nowUnix >= cmdPtr.completeAfter { b.completeCommand(region, cmdID) } } @@ -2180,8 +2342,8 @@ func (b *InMemoryBackend) materializeCommandLocked(region, cmdID string, nowUnix // materializeCommandsLocked lazily completes every eligible InProgress command // in the region. Must be called with b.mu held for writing. func (b *InMemoryBackend) materializeCommandsLocked(region string, nowUnix float64) { - for cmdID := range b.commandsStore(region) { - b.materializeCommandLocked(region, cmdID, nowUnix) + for _, cmdPtr := range b.commandsStore(region).All() { + b.materializeCommandLocked(region, cmdPtr.CommandID, nowUnix) } } @@ -2196,13 +2358,13 @@ func (b *InMemoryBackend) ListCommands( b.materializeCommandsLocked(region, UnixTimeFloat(timeNow())) - store := b.commandsStore(region) - all := make([]Command, 0, len(store)) - for _, cmd := range store { - if input.CommandID != "" && cmd.CommandID != input.CommandID { + cmdTable := b.commandsStore(region) + all := make([]Command, 0, cmdTable.Len()) + for _, cmdPtr := range cmdTable.All() { + if input.CommandID != "" && cmdPtr.CommandID != input.CommandID { continue } - all = append(all, cmd) + all = append(all, *cmdPtr) } sort.Slice(all, func(i, j int) bool { return all[i].CommandID < all[j].CommandID }) @@ -2243,7 +2405,7 @@ func (b *InMemoryBackend) GetCommandInvocation( b.mu.Lock("GetCommandInvocation") defer b.mu.Unlock() - if _, exists := b.commandsStore(region)[input.CommandID]; !exists { + if !b.commandsStore(region).Has(input.CommandID) { return nil, ErrCommandNotFound } @@ -2340,39 +2502,65 @@ func (b *InMemoryBackend) Reset() { } } - b.parameters = make(map[string]map[string]Parameter) + // b.registry is rebuilt from scratch (rather than registry.ResetAll'd) + // because Reset also reallocates every *store.Table[V] field below to a + // brand-new map[string]*store.Table[V] -- the old *store.Table[V] + // pointers (and their registry registrations under "/") + // are discarded entirely, so a stale registry would panic on + // re-registration the next time a region is touched (store.Register + // panics on a duplicate name; see store_setup.go's getOrCreateTable). + b.registry = store.NewRegistry() + + b.parameters = make(map[string]*store.Table[Parameter]) b.history = make(map[string]map[string][]ParameterHistory) b.tags = make(map[string]map[string]*tags.Tags) - b.documents = make(map[string]map[string]Document) + b.documents = make(map[string]*store.Table[Document]) b.documentVersions = make(map[string]map[string][]DocumentVersion) b.documentPermissions = make(map[string]map[string][]string) - b.commands = make(map[string]map[string]Command) + b.commands = make(map[string]*store.Table[Command]) b.commandInvocations = make(map[string]map[string][]CommandInvocation) - b.activations = make(map[string]map[string]Activation) - b.associations = make(map[string]map[string]Association) - b.maintenanceWindows = make(map[string]map[string]MaintenanceWindow) - b.maintenanceWindowTargets = make(map[string]map[string]MaintenanceWindowTarget) - b.maintenanceWindowTasks = make(map[string]map[string]MaintenanceWindowTask) - b.sessions = make(map[string]map[string]Session) + b.activations = make(map[string]*store.Table[Activation]) + b.associations = make(map[string]*store.Table[Association]) + b.maintenanceWindows = make(map[string]*store.Table[MaintenanceWindow]) + b.maintenanceWindowTargets = make(map[string]*store.Table[MaintenanceWindowTarget]) + b.maintenanceWindowTasks = make(map[string]*store.Table[MaintenanceWindowTask]) + b.sessions = make(map[string]*store.Table[Session]) b.patchGroupToBaseline = make(map[string]map[string]string) - b.opsItems = make(map[string]map[string]OpsItem) + b.opsItems = make(map[string]*store.Table[OpsItem]) b.opsItemRelatedItems = make(map[string]map[string][]OpsItemRelatedItem) - b.opsMetadata = make(map[string]map[string]OpsMetadata) - b.patchBaselines = make(map[string]map[string]PatchBaseline) + b.opsMetadata = make(map[string]*store.Table[OpsMetadata]) + b.patchBaselines = make(map[string]*store.Table[PatchBaseline]) b.resourceIDToOpsMetadataArn = make(map[string]map[string]string) b.miscResourceTags = make(map[string]map[string]map[string]string) - b.resourceDataSyncs = make(map[string]map[string]*ResourceDataSync) + b.resourceDataSyncs = make(map[string]*store.Table[ResourceDataSync]) b.parameterLabels = make(map[string]map[string]map[int64][]string) - b.automationExecutions = make(map[string]map[string]*AutomationExecution) - b.serviceSettings = make(map[string]map[string]*ServiceSetting) + b.automationExecutions = make(map[string]*store.Table[AutomationExecution]) + b.serviceSettings = make(map[string]*store.Table[ServiceSetting]) b.resourcePolicies = make(map[string]map[string][]*ResourcePolicy) - b.executionPreviews = make(map[string]map[string]*ExecutionPreview) + b.executionPreviews = make(map[string]*store.Table[ExecutionPreview]) b.inventory = make(map[string]map[string][]InventoryItem) b.compliance = make(map[string]map[string][]ComplianceItem) b.associationExecutions = make(map[string]map[string][]AssociationExecution) b.associationExecTargets = make(map[string]map[string][]AssociationExecutionTarget) b.inventoryDeletions = make(map[string][]InventoryDeletion) b.opsItemEvents = nil + + // instancePatchStates/instanceProperties are deliberately NOT reallocated + // above -- Reset() never cleared them even before this conversion (a + // pre-existing gap in Reset's coverage, left as-is to avoid changing + // behavior). Since b.registry was just replaced with an empty one, their + // existing *store.Table[V] values (untouched, from before Reset) must be + // re-registered onto it under their original names, or a later access to + // an already-touched region would try to register that name a second + // time and panic (store.Register panics on a duplicate name). + for region, t := range b.instancePatchStates { + store.Register(b.registry, "instancePatchStates/"+region, t) + } + + for region, t := range b.instanceProperties { + store.Register(b.registry, "instanceProperties/"+region, t) + } + b.registerDefaultDocuments(defaultRegion) } @@ -2446,14 +2634,15 @@ func (b *InMemoryBackend) CancelCommand( b.mu.Lock("CancelCommand") defer b.mu.Unlock() - store := b.commandsStore(region) - cmd, exists := store[input.CommandID] + cmdTable := b.commandsStore(region) + cmdPtr, exists := cmdTable.Get(input.CommandID) if !exists { return nil, ErrCommandNotFound } + cmd := *cmdPtr cmd.Status = commandStatusCancelled - store[input.CommandID] = cmd + cmdTable.Put(&cmd) invStore := b.commandInvocationsStore(region) invs := invStore[input.CommandID] @@ -2516,10 +2705,7 @@ func (b *InMemoryBackend) CreateActivation( CreatedDate: now, } - if b.activations[region] == nil { - b.activations[region] = make(map[string]Activation) - } - b.activationsStore(region)[activationID] = act + b.activationsStore(region).Put(&act) if len(input.Tags) > 0 { if b.miscResourceTags[region] == nil { @@ -2582,7 +2768,7 @@ func (b *InMemoryBackend) CreateAssociation( b.mu.Lock("CreateAssociation") defer b.mu.Unlock() - if _, exists := b.documentsStore(region)[input.Name]; !exists { + if !b.documentsStore(region).Has(input.Name) { return nil, ErrDocumentNotFound } @@ -2601,10 +2787,7 @@ func (b *InMemoryBackend) CreateAssociation( LastUpdateAssociationDate: now, } - if b.associations[region] == nil { - b.associations[region] = make(map[string]Association) - } - b.associationsStore(region)[assocID] = assoc + b.associationsStore(region).Put(&assoc) b.recordAssociationExecutionLocked(region, assoc) return &CreateAssociationOutput{AssociationDescription: assoc}, nil @@ -2626,13 +2809,10 @@ func (b *InMemoryBackend) CreateAssociationBatch( now := UnixTimeFloat(time.Now()) docs := b.documentsStore(region) - if b.associations[region] == nil { - b.associations[region] = make(map[string]Association) - } assocs := b.associationsStore(region) for _, entry := range input.Entries { - if _, exists := docs[entry.Name]; !exists { + if !docs.Has(entry.Name) { output.Failed = append(output.Failed, FailedCreateAssociation{ Entry: entry, Message: ErrDocumentNotFound.Error(), @@ -2655,7 +2835,7 @@ func (b *InMemoryBackend) CreateAssociationBatch( LastUpdateAssociationDate: now, } - assocs[assocID] = assoc + assocs.Put(&assoc) b.recordAssociationExecutionLocked(region, assoc) output.Successful = append(output.Successful, assoc) } @@ -2706,10 +2886,7 @@ func (b *InMemoryBackend) CreateMaintenanceWindow( ModifiedDate: now, } - if b.maintenanceWindows[region] == nil { - b.maintenanceWindows[region] = make(map[string]MaintenanceWindow) - } - b.maintenanceWindowsStore(region)[windowID] = mw + b.maintenanceWindowsStore(region).Put(&mw) if len(input.Tags) > 0 { if b.miscResourceTags[region] == nil { @@ -2763,10 +2940,7 @@ func (b *InMemoryBackend) CreateOpsItem( LastModifiedTime: now, } - if b.opsItems[region] == nil { - b.opsItems[region] = make(map[string]OpsItem) - } - b.opsItemsStore(region)[opsItemID] = item + b.opsItemsStore(region).Put(&item) b.opsItemEvents[region] = append(b.opsItemEvents[region], OpsItemEventSummary{ OpsItemID: opsItemID, @@ -2801,7 +2975,7 @@ func (b *InMemoryBackend) AssociateOpsItemRelatedItem( b.mu.Lock("AssociateOpsItemRelatedItem") defer b.mu.Unlock() - if _, exists := b.opsItemsStore(region)[input.OpsItemID]; !exists { + if !b.opsItemsStore(region).Has(input.OpsItemID) { return nil, ErrOpsItemNotFound } @@ -2859,10 +3033,7 @@ func (b *InMemoryBackend) CreateOpsMetadata( LastModifiedDate: now, } - if b.opsMetadata[region] == nil { - b.opsMetadata[region] = make(map[string]OpsMetadata) - } - b.opsMetadataStore(region)[arn] = meta + b.opsMetadataStore(region).Put(&meta) resToArn[input.ResourceID] = arn return &CreateOpsMetadataOutput{OpsMetadataArn: arn}, nil @@ -2902,10 +3073,7 @@ func (b *InMemoryBackend) CreatePatchBaseline( ModifiedDate: now, } - if b.patchBaselines[region] == nil { - b.patchBaselines[region] = make(map[string]PatchBaseline) - } - b.patchBaselinesStore(region)[baselineID] = bl + b.patchBaselinesStore(region).Put(&bl) if len(input.Tags) > 0 { if b.miscResourceTags[region] == nil { diff --git a/services/ssm/backend_batch2.go b/services/ssm/backend_batch2.go index 1c3292717..61e20bda7 100644 --- a/services/ssm/backend_batch2.go +++ b/services/ssm/backend_batch2.go @@ -51,21 +51,18 @@ func (b *InMemoryBackend) CreateResourceDataSync( syncName = "default-sync" } - if b.resourceDataSyncs[region] == nil { - b.resourceDataSyncs[region] = make(map[string]*ResourceDataSync) - } syncs := b.resourceDataSyncsStore(region) - if _, exists := syncs[syncName]; exists { + if syncs.Has(syncName) { return nil, ErrResourceDataSyncExists } - syncs[syncName] = &ResourceDataSync{ + syncs.Put(&ResourceDataSync{ SyncName: syncName, SyncType: input.SyncType, LastStatus: "InProgress", SyncCreatedTime: time.Now().UTC(), LastSyncTime: time.Now().UTC(), - } + }) return &CreateResourceDataSyncOutput{}, nil } @@ -84,13 +81,11 @@ func (b *InMemoryBackend) DeleteResourceDataSync( } syncs := b.resourceDataSyncsStore(region) - if _, exists := syncs[input.SyncName]; !exists { + if !syncs.Has(input.SyncName) { return nil, fmt.Errorf("%w: %q", ErrResourceDataSyncNotFound, input.SyncName) } - delete(syncs, input.SyncName) - - cleanupEmptyInnerMap(b.resourceDataSyncs, region) + syncs.Delete(input.SyncName) return &DeleteResourceDataSyncOutput{}, nil } @@ -105,8 +100,8 @@ func (b *InMemoryBackend) ListResourceDataSync( defer b.mu.RUnlock() syncs := b.resourceDataSyncsStore(region) - items := make([]ResourceDataSync, 0, len(syncs)) - for _, s := range syncs { + items := make([]ResourceDataSync, 0, syncs.Len()) + for _, s := range syncs.All() { items = append(items, *s) } @@ -130,7 +125,7 @@ func (b *InMemoryBackend) UpdateResourceDataSync( return &UpdateResourceDataSyncOutput{}, nil } - if sync, exists := b.resourceDataSyncsStore(region)[input.SyncName]; exists { + if sync, exists := b.resourceDataSyncsStore(region).Get(input.SyncName); exists { sync.LastSyncTime = time.Now().UTC() } @@ -151,8 +146,8 @@ func (b *InMemoryBackend) DeregisterManagedInstance( // Try to match by ActivationID (InstanceID in the request maps to ActivationID here). activations := b.activationsStore(region) - if _, exists := activations[input.InstanceID]; exists { - delete(activations, input.InstanceID) + if activations.Has(input.InstanceID) { + activations.Delete(input.InstanceID) delete(b.miscResourceTagsStore(region), input.InstanceID) } @@ -169,9 +164,10 @@ func (b *InMemoryBackend) UpdateManagedInstanceRole( defer b.mu.Unlock() activations := b.activationsStore(region) - if act, exists := activations[input.InstanceID]; exists { + if actPtr, exists := activations.Get(input.InstanceID); exists { + act := *actPtr act.IamRole = input.IamRole - activations[input.InstanceID] = act + activations.Put(&act) } return &UpdateManagedInstanceRoleOutput{}, nil @@ -189,10 +185,10 @@ func (b *InMemoryBackend) DescribeSessions( defer b.mu.RUnlock() sessions := b.sessionsStore(region) - list := make([]Session, 0, len(sessions)) - for _, s := range sessions { + list := make([]Session, 0, sessions.Len()) + for _, s := range sessions.All() { if input.State == "" || s.Status == input.State { - list = append(list, s) + list = append(list, *s) } } @@ -215,7 +211,7 @@ func (b *InMemoryBackend) GetConnectionStatus( target := input.Target status := "notConnected" - for _, s := range b.sessionsStore(region) { + for _, s := range b.sessionsStore(region).All() { if s.Target == target && s.Status == sessionStatusConnected { status = connectionStatusConnected @@ -251,13 +247,14 @@ func (b *InMemoryBackend) ResumeSession( defer b.mu.Unlock() sessions := b.sessionsStore(region) - sess, exists := sessions[input.SessionID] + sessPtr, exists := sessions.Get(input.SessionID) if !exists { return &ResumeSessionOutputFull{SessionID: input.SessionID, StreamURL: ""}, nil } + sess := *sessPtr sess.Status = sessionStatusConnected - sessions[input.SessionID] = sess + sessions.Put(&sess) return &ResumeSessionOutputFull{ SessionID: sess.SessionID, @@ -292,7 +289,7 @@ func (b *InMemoryBackend) GetServiceSetting( b.mu.RLock("GetServiceSetting") defer b.mu.RUnlock() - if s, exists := b.serviceSettingsStore(region)[input.SettingID]; exists { + if s, exists := b.serviceSettingsStore(region).Get(input.SettingID); exists { return &GetServiceSettingOutputFull{ServiceSetting: s}, nil } @@ -312,14 +309,11 @@ func (b *InMemoryBackend) UpdateServiceSetting( b.mu.Lock("UpdateServiceSetting") defer b.mu.Unlock() - if b.serviceSettings[region] == nil { - b.serviceSettings[region] = make(map[string]*ServiceSetting) - } - b.serviceSettingsStore(region)[input.SettingID] = &ServiceSetting{ + b.serviceSettingsStore(region).Put(&ServiceSetting{ SettingID: input.SettingID, SettingValue: input.SettingValue, Status: settingStatusCustomized, - } + }) return &UpdateServiceSettingOutput{}, nil } @@ -333,7 +327,7 @@ func (b *InMemoryBackend) ResetServiceSetting( b.mu.Lock("ResetServiceSetting") defer b.mu.Unlock() - delete(b.serviceSettingsStore(region), input.SettingID) + b.serviceSettingsStore(region).Delete(input.SettingID) return &ResetServiceSettingOutputFull{ServiceSetting: &ServiceSetting{ SettingID: input.SettingID, @@ -442,11 +436,13 @@ func (b *InMemoryBackend) LabelParameterVersion( }, nil } - param, exists := b.parametersStore(region)[input.Name] + paramPtr, exists := b.parametersStore(region).Get(input.Name) if !exists { return nil, fmt.Errorf("%w: %q", ErrParameterNotFound, input.Name) } + param := *paramPtr + version := input.ParameterVersion if version == 0 { version = param.Version @@ -520,7 +516,7 @@ func (b *InMemoryBackend) UnlabelParameterVersion( version := input.ParameterVersion if version == 0 { - if param, exists := b.parametersStore(region)[input.Name]; exists { + if param, exists := b.parametersStore(region).Get(input.Name); exists { version = param.Version } } @@ -633,10 +629,7 @@ func (b *InMemoryBackend) StartAutomationExecution( exec.completeAfter = UnixTimeFloat(now) + b.automationExecDelaySecs } - if b.automationExecutions[region] == nil { - b.automationExecutions[region] = make(map[string]*AutomationExecution) - } - b.automationExecutionsStore(region)[execID] = exec + b.automationExecutionsStore(region).Put(exec) return &StartAutomationExecutionOutputFull{AutomationExecutionID: execID}, nil } @@ -650,7 +643,7 @@ func (b *InMemoryBackend) GetAutomationExecution( b.mu.Lock("GetAutomationExecution") defer b.mu.Unlock() - exec, exists := b.automationExecutionsStore(region)[input.AutomationExecutionID] + exec, exists := b.automationExecutionsStore(region).Get(input.AutomationExecutionID) if !exists { return nil, fmt.Errorf( "%w: %q", @@ -677,8 +670,8 @@ func (b *InMemoryBackend) DescribeAutomationExecutions( now := time.Now().UTC() execs := b.automationExecutionsStore(region) - list := make([]AutomationExecution, 0, len(execs)) - for _, exec := range execs { + list := make([]AutomationExecution, 0, execs.Len()) + for _, exec := range execs.All() { materializeAutomationLocked(exec, now) list = append(list, *exec) } @@ -699,7 +692,7 @@ func (b *InMemoryBackend) StopAutomationExecution( b.mu.Lock("StopAutomationExecution") defer b.mu.Unlock() - if exec, exists := b.automationExecutionsStore(region)[input.AutomationExecutionID]; exists { + if exec, exists := b.automationExecutionsStore(region).Get(input.AutomationExecutionID); exists { exec.Status = automationStatusStopped exec.EndTime = UnixTimeFloat(time.Now().UTC()) } @@ -717,7 +710,7 @@ func (b *InMemoryBackend) SendAutomationSignal( b.mu.Lock("SendAutomationSignal") defer b.mu.Unlock() - exec, exists := b.automationExecutionsStore(region)[input.AutomationExecutionID] + exec, exists := b.automationExecutionsStore(region).Get(input.AutomationExecutionID) if !exists { return &SendAutomationSignalOutput{}, nil } @@ -743,7 +736,7 @@ func (b *InMemoryBackend) DescribeAutomationStepExecutions( b.mu.Lock("DescribeAutomationStepExecutions") defer b.mu.Unlock() - exec, exists := b.automationExecutionsStore(region)[input.AutomationExecutionID] + exec, exists := b.automationExecutionsStore(region).Get(input.AutomationExecutionID) if !exists { return &DescribeAutomationStepExecutionsOutputFull{ StepExecutions: []AutomationStepExec{}, @@ -780,10 +773,7 @@ func (b *InMemoryBackend) StartChangeRequestExecution( ExecutionType: "ChangeRequest", Steps: b.buildAutomationSteps(region, input.DocumentName), } - if b.automationExecutions[region] == nil { - b.automationExecutions[region] = make(map[string]*AutomationExecution) - } - b.automationExecutionsStore(region)[execID] = exec + b.automationExecutionsStore(region).Put(exec) return &StartChangeRequestExecutionOutputFull{AutomationExecutionID: execID}, nil } @@ -800,14 +790,11 @@ func (b *InMemoryBackend) StartExecutionPreview( defer b.mu.Unlock() previewID := previewIDPrefix + uuid.NewString() - if b.executionPreviews[region] == nil { - b.executionPreviews[region] = make(map[string]*ExecutionPreview) - } - b.executionPreviewsStore(region)[previewID] = &ExecutionPreview{ + b.executionPreviewsStore(region).Put(&ExecutionPreview{ ExecutionPreviewID: previewID, Status: "Running", DocumentName: input.DocumentName, - } + }) return &StartExecutionPreviewOutputFull{ExecutionPreviewID: previewID}, nil } @@ -821,7 +808,7 @@ func (b *InMemoryBackend) GetExecutionPreview( b.mu.RLock("GetExecutionPreview") defer b.mu.RUnlock() - preview, exists := b.executionPreviewsStore(region)[input.ExecutionPreviewID] + preview, exists := b.executionPreviewsStore(region).Get(input.ExecutionPreviewID) if !exists { return &GetExecutionPreviewOutputFull{ ExecutionPreviewID: input.ExecutionPreviewID, @@ -858,7 +845,7 @@ func (b *InMemoryBackend) GetCalendarState( documents := b.documentsStore(region) for _, name := range input.CalendarNames { - doc, exists := documents[name] + doc, exists := documents.Get(name) if !exists { return nil, fmt.Errorf("%w: calendar document %q not found", ErrDocumentNotFound, name) } @@ -891,7 +878,7 @@ func (b *InMemoryBackend) GetOpsSummary( { ID: "AWS:OpsItem", Data: map[string]OpsSummaryValue{ - "Count": {Count: len(b.opsItemsStore(region)), Unit: "Count"}, + "Count": {Count: b.opsItemsStore(region).Len(), Unit: "Count"}, }, }, }, @@ -908,9 +895,9 @@ func (b *InMemoryBackend) ListOpsMetadata( defer b.mu.RUnlock() opsMetadata := b.opsMetadataStore(region) - list := make([]OpsMetadata, 0, len(opsMetadata)) - for _, m := range opsMetadata { - list = append(list, m) + list := make([]OpsMetadata, 0, opsMetadata.Len()) + for _, m := range opsMetadata.All() { + list = append(list, *m) } sort.Slice(list, func(i, k int) bool { @@ -932,14 +919,15 @@ func (b *InMemoryBackend) UpdateAssociationStatus( defer b.mu.Unlock() associations := b.associationsStore(region) - for id, assoc := range associations { + for _, assocPtr := range associations.All() { + assoc := *assocPtr if assoc.InstanceID == input.InstanceID && assoc.Name == input.Name { if assoc.Overview == nil { assoc.Overview = &AssociationOverview{} } assoc.Overview.Status = input.AssociationStatus.Name - associations[id] = assoc + associations.Put(&assoc) return &UpdateAssociationStatusOutputFull{AssociationDescription: assoc}, nil } @@ -966,9 +954,10 @@ func (b *InMemoryBackend) StartAssociationsOnce( associations := b.associationsStore(region) for _, assocID := range input.AssociationIDs { - if assoc, exists := associations[assocID]; exists { + if assocPtr, exists := associations.Get(assocID); exists { + assoc := *assocPtr assoc.LastUpdateAssociationDate = float64(now.Unix()) - associations[assocID] = assoc + associations.Put(&assoc) // A one-time run produces a fresh, stable execution record. b.recordAssociationExecutionLocked(region, assoc) } @@ -986,13 +975,13 @@ func (b *InMemoryBackend) ListAssociationVersions( b.mu.RLock("ListAssociationVersions") defer b.mu.RUnlock() - assoc, exists := b.associationsStore(region)[input.AssociationID] + assoc, exists := b.associationsStore(region).Get(input.AssociationID) if !exists { return &ListAssociationVersionsOutputFull{AssociationVersions: []Association{}}, nil } return &ListAssociationVersionsOutputFull{ - AssociationVersions: []Association{assoc}, + AssociationVersions: []Association{*assoc}, }, nil } @@ -1085,7 +1074,7 @@ func (b *InMemoryBackend) DescribeAssociationExecutions( b.mu.Lock("DescribeAssociationExecutions") defer b.mu.Unlock() - assoc, exists := b.associationsStore(region)[input.AssociationID] + assocPtr, exists := b.associationsStore(region).Get(input.AssociationID) if !exists { return &DescribeAssociationExecutionsOutputFull{ AssociationExecutions: []AssociationExecution{}, @@ -1094,7 +1083,7 @@ func (b *InMemoryBackend) DescribeAssociationExecutions( execs := b.associationExecutionsStore(region)[input.AssociationID] if len(execs) == 0 { - b.recordAssociationExecutionLocked(region, assoc) + b.recordAssociationExecutionLocked(region, *assocPtr) execs = b.associationExecutionsStore(region)[input.AssociationID] } @@ -1123,13 +1112,15 @@ func (b *InMemoryBackend) DescribeAssociationExecutionTargets( }, nil } - assoc, exists := b.associationsStore(region)[input.AssociationID] + assocPtr, exists := b.associationsStore(region).Get(input.AssociationID) if !exists { return &DescribeAssociationExecutionTargetsOutputFull{ AssociationExecutionTargets: []AssociationExecutionTarget{}, }, nil } + assoc := *assocPtr + execID := input.ExecutionID if execID == "" { execs := b.associationExecutionsStore(region)[input.AssociationID] @@ -1193,7 +1184,7 @@ func (b *InMemoryBackend) DescribeMaintenanceWindowExecutions( }, nil } - win, exists := b.maintenanceWindowsStore(region)[input.WindowID] + win, exists := b.maintenanceWindowsStore(region).Get(input.WindowID) if !exists { return &DescribeMaintenanceWindowExecutionsOutputFull{ WindowExecutions: []MaintenanceWindowExecution{}, @@ -1242,7 +1233,7 @@ func (b *InMemoryBackend) DescribeMaintenanceWindowExecutionTasks( tasks := b.maintenanceWindowTasksStore(region) result := make([]MaintenanceWindowExecutionTask, 0) - for _, task := range tasks { + for _, task := range tasks.All() { if task.WindowID != windowID { continue } @@ -1307,7 +1298,7 @@ func (b *InMemoryBackend) DescribeMaintenanceWindowSchedule( b.mu.RLock("DescribeMaintenanceWindowSchedule") defer b.mu.RUnlock() - win, exists := b.maintenanceWindowsStore(region)[input.WindowID] + win, exists := b.maintenanceWindowsStore(region).Get(input.WindowID) if !exists { return &DescribeMaintenanceWindowScheduleOutputFull{ ScheduledWindowExecutions: []ScheduledWindowExecution{}, @@ -1349,7 +1340,7 @@ func (b *InMemoryBackend) GetMaintenanceWindowExecution( endTime := startTime.Add(time.Hour) if windowID != "" { - win, exists := b.maintenanceWindowsStore(region)[windowID] + win, exists := b.maintenanceWindowsStore(region).Get(windowID) if !exists { return nil, ErrMaintenanceWindowNotFound } @@ -1395,7 +1386,7 @@ func (b *InMemoryBackend) GetMaintenanceWindowExecutionTask( startTime := time.Now().UTC() if windowTaskID != "" { - if task, ok := b.maintenanceWindowTasksStore(region)[windowTaskID]; ok { + if task, ok := b.maintenanceWindowTasksStore(region).Get(windowTaskID); ok { endTime := startTime.Add(time.Minute) return &GetMaintenanceWindowExecutionTaskOutputFull{ @@ -1440,7 +1431,7 @@ func (b *InMemoryBackend) GetMaintenanceWindowExecutionTaskInvocation( endTime := startTime.Add(time.Minute) windowTargetID := "" - for _, target := range b.maintenanceWindowTargetsStore(region) { + for _, target := range b.maintenanceWindowTargetsStore(region).All() { windowID, ok := mwWindowIDFromExec(input.WindowExecutionID) if ok && target.WindowID == windowID { windowTargetID = target.WindowTargetID @@ -1475,8 +1466,8 @@ func (b *InMemoryBackend) ListNodes( defer b.mu.RUnlock() activations := b.activationsStore(region) - nodes := make([]NodeInfo, 0, len(activations)) - for _, act := range activations { + nodes := make([]NodeInfo, 0, activations.Len()) + for _, act := range activations.All() { nodes = append(nodes, NodeInfo{ InstanceID: act.ActivationID, PlatformType: platformTypeLinux, @@ -1503,7 +1494,7 @@ func (b *InMemoryBackend) ListNodesSummary( return &ListNodesSummaryOutputFull{ Summary: []map[string]string{ - {"NodeCount": strconv.Itoa(len(b.activationsStore(region)))}, + {"NodeCount": strconv.Itoa(b.activationsStore(region).Len())}, }, }, nil } @@ -1521,7 +1512,8 @@ func (b *InMemoryBackend) DescribeEffectiveInstanceAssociations( var result []InstanceAssociationInfo - for _, assoc := range b.associationsStore(region) { + for _, assocPtr := range b.associationsStore(region).All() { + assoc := *assocPtr if assoc.InstanceID == input.InstanceID { result = append(result, InstanceAssociationInfo{ AssociationID: assoc.AssociationID, @@ -1550,7 +1542,8 @@ func (b *InMemoryBackend) DescribeInstanceAssociationsStatus( var result []InstanceAssociationStatusInfo - for _, assoc := range b.associationsStore(region) { + for _, assocPtr := range b.associationsStore(region).All() { + assoc := *assocPtr if assoc.InstanceID == input.InstanceID { status := commandStatusSuccess if assoc.Overview != nil { @@ -1585,8 +1578,8 @@ func (b *InMemoryBackend) DescribeInstanceInformation( defer b.mu.RUnlock() activations := b.activationsStore(region) - list := make([]InstanceInformation, 0, len(activations)) - for _, act := range activations { + list := make([]InstanceInformation, 0, activations.Len()) + for _, act := range activations.All() { list = append(list, InstanceInformation{ InstanceID: act.ActivationID, PingStatus: "Online", @@ -1608,16 +1601,16 @@ func (b *InMemoryBackend) DescribeInstancePatchStates( b.mu.RLock("DescribeInstancePatchStates") defer b.mu.RUnlock() - store := b.instancePatchStates[region] + patchStates := b.instancePatchStatesStore(region) states := make([]InstancePatchState, 0) if len(input.InstanceIDs) == 0 { - for _, s := range store { + for _, s := range patchStates.All() { states = append(states, *s) } } else { for _, instanceID := range input.InstanceIDs { - if s, exists := store[instanceID]; exists { + if s, exists := patchStates.Get(instanceID); exists { states = append(states, *s) } } diff --git a/services/ssm/backend_ops.go b/services/ssm/backend_ops.go index bf64b6b40..8bbef3ac3 100644 --- a/services/ssm/backend_ops.go +++ b/services/ssm/backend_ops.go @@ -51,11 +51,13 @@ func (b *InMemoryBackend) UpdateDocumentDefaultVersion( defer b.mu.Unlock() docs := b.documentsStore(region) - doc, exists := docs[input.Name] + docPtr, exists := docs.Get(input.Name) if !exists { return nil, fmt.Errorf("%w: document %q not found", ErrDocumentNotFound, input.Name) } + doc := *docPtr + // Verify the requested version exists in documentVersions. found := false @@ -74,7 +76,7 @@ func (b *InMemoryBackend) UpdateDocumentDefaultVersion( } doc.DefaultVersion = input.DocumentVersion - docs[input.Name] = doc + docs.Put(&doc) // Also mark the version entry. versions := docVersions[input.Name] @@ -108,7 +110,7 @@ func (b *InMemoryBackend) UpdateDocumentMetadata( b.mu.RLock("UpdateDocumentMetadata") defer b.mu.RUnlock() - if _, exists := b.documentsStore(region)[input.Name]; !exists { + if !b.documentsStore(region).Has(input.Name) { return nil, fmt.Errorf("%w: document %q not found", ErrDocumentNotFound, input.Name) } @@ -134,7 +136,7 @@ func (b *InMemoryBackend) ListDocumentMetadataHistory( b.mu.RLock("ListDocumentMetadataHistory") defer b.mu.RUnlock() - doc, exists := b.documentsStore(region)[input.Name] + doc, exists := b.documentsStore(region).Get(input.Name) if !exists { return nil, fmt.Errorf("%w: document %q not found", ErrDocumentNotFound, input.Name) } @@ -769,7 +771,9 @@ func (b *InMemoryBackend) GetDefaultPatchBaseline( if id, ok := b.patchGroupToBaselineStore(region)[key]; ok { os := input.OperatingSystem if os == "" { - os = b.patchBaselinesStore(region)[id].OperatingSystem + if blPtr, foundBl := b.patchBaselinesStore(region).Get(id); foundBl { + os = blPtr.OperatingSystem + } } return &GetDefaultPatchBaselineOutput{ @@ -836,7 +840,7 @@ func (b *InMemoryBackend) RegisterDefaultPatchBaseline( b.mu.Lock("RegisterDefaultPatchBaseline") defer b.mu.Unlock() - if _, exists := b.patchBaselinesStore(region)[input.BaselineID]; !exists { + if !b.patchBaselinesStore(region).Has(input.BaselineID) { return nil, fmt.Errorf( "%w: baseline %q not found", ErrPatchBaselineNotFound, @@ -851,7 +855,7 @@ func (b *InMemoryBackend) RegisterDefaultPatchBaseline( store["default"] = input.BaselineID // Also store per-OS key when the baseline has a known OperatingSystem. - if bl, ok := b.patchBaselinesStore(region)[input.BaselineID]; ok && bl.OperatingSystem != "" { + if bl, ok := b.patchBaselinesStore(region).Get(input.BaselineID); ok && bl.OperatingSystem != "" { store["default-"+bl.OperatingSystem] = input.BaselineID } @@ -867,14 +871,12 @@ func (b *InMemoryBackend) DeletePatchBaseline( b.mu.Lock("DeletePatchBaseline") defer b.mu.Unlock() - store := b.patchBaselinesStore(region) - if _, exists := store[input.BaselineID]; !exists { + patchBaselines := b.patchBaselinesStore(region) + if !patchBaselines.Has(input.BaselineID) { return nil, ErrPatchBaselineNotFound } - delete(store, input.BaselineID) - - cleanupEmptyInnerMap(b.patchBaselines, region) + patchBaselines.Delete(input.BaselineID) return &DeletePatchBaselineOutput{BaselineID: input.BaselineID}, nil } @@ -889,7 +891,7 @@ func (b *InMemoryBackend) DescribePatchGroupState( defer b.mu.RUnlock() out := &DescribePatchGroupStateOutput{} - for _, s := range b.instancePatchStates[region] { + for _, s := range b.instancePatchStatesStore(region).All() { if s.PatchGroup != input.PatchGroup { continue } @@ -923,7 +925,7 @@ func (b *InMemoryBackend) DescribePatchGroups( patchBaselines := b.patchBaselinesStore(region) for group, baselineID := range store { identity := PatchBaselineIdentity{BaselineID: baselineID} - if bl, ok := patchBaselines[baselineID]; ok { + if bl, ok := patchBaselines.Get(baselineID); ok { identity.BaselineName = bl.Name identity.OperatingSystem = bl.OperatingSystem identity.Description = bl.Description @@ -985,7 +987,7 @@ func (b *InMemoryBackend) DescribePatchProperties( seen := map[string]bool{} props := make([]map[string]string, 0) - for _, bl := range b.patchBaselines[region] { + for _, bl := range b.patchBaselinesStore(region).All() { if input.OperatingSystem != "" && bl.OperatingSystem != input.OperatingSystem { continue } @@ -1023,7 +1025,7 @@ func (b *InMemoryBackend) DescribeEffectivePatchesForPatchBaseline( b.mu.RLock("DescribeEffectivePatchesForPatchBaseline") defer b.mu.RUnlock() - baseline, exists := b.patchBaselinesStore(region)[input.BaselineID] + baselinePtr, exists := b.patchBaselinesStore(region).Get(input.BaselineID) if !exists { return nil, fmt.Errorf( "%w: baseline %q not found", @@ -1032,7 +1034,7 @@ func (b *InMemoryBackend) DescribeEffectivePatchesForPatchBaseline( ) } - effective := b.effectivePatchesForBaseline(region, baseline) + effective := b.effectivePatchesForBaseline(region, *baselinePtr) return paginateEffectivePatches(effective, input.NextToken, input.MaxResults), nil } @@ -1159,12 +1161,12 @@ func (b *InMemoryBackend) GetDeployablePatchSnapshotForInstance( product := patchProductAmazonLinux2 baselineID := defaultBaselineID("AMAZON_LINUX_2") - if st, ok := b.instancePatchStatesStore(region)[input.InstanceID]; ok && st != nil { + if st, ok := b.instancePatchStatesStore(region).Get(input.InstanceID); ok && st != nil { if st.BaselineID != "" { baselineID = st.BaselineID } - if bl, found := b.patchBaselinesStore(region)[st.BaselineID]; found && + if bl, found := b.patchBaselinesStore(region).Get(st.BaselineID); found && bl.OperatingSystem != "" { product = bl.OperatingSystem } @@ -1192,14 +1194,12 @@ func (b *InMemoryBackend) DeleteMaintenanceWindow( b.mu.Lock("DeleteMaintenanceWindow") defer b.mu.Unlock() - store := b.maintenanceWindowsStore(region) - if _, exists := store[input.WindowID]; !exists { + mwTable := b.maintenanceWindowsStore(region) + if !mwTable.Has(input.WindowID) { return nil, ErrMaintenanceWindowNotFound } - delete(store, input.WindowID) - - cleanupEmptyInnerMap(b.maintenanceWindows, region) + mwTable.Delete(input.WindowID) return &DeleteMaintenanceWindowOutput{WindowID: input.WindowID}, nil } @@ -1218,12 +1218,12 @@ func (b *InMemoryBackend) GetMaintenanceWindowTask( b.mu.RLock("GetMaintenanceWindowTask") defer b.mu.RUnlock() - task, exists := b.maintenanceWindowTasksStore(region)[input.WindowTaskID] + task, exists := b.maintenanceWindowTasksStore(region).Get(input.WindowTaskID) if !exists || task.WindowID != input.WindowID { return nil, ErrMaintenanceWindowNotFound } - return &GetMaintenanceWindowTaskOutput{MaintenanceWindowTask: task}, nil + return &GetMaintenanceWindowTaskOutput{MaintenanceWindowTask: *task}, nil } // DescribeMaintenanceWindowsForTarget returns windows that have registered targets @@ -1238,7 +1238,7 @@ func (b *InMemoryBackend) DescribeMaintenanceWindowsForTarget( matchedWindowIDs := make(map[string]struct{}) - for _, windowTarget := range b.maintenanceWindowTargetsStore(region) { + for _, windowTarget := range b.maintenanceWindowTargetsStore(region).All() { if input.ResourceType != "" && windowTarget.ResourceType != input.ResourceType { continue } @@ -1252,7 +1252,7 @@ func (b *InMemoryBackend) DescribeMaintenanceWindowsForTarget( maintenanceWindows := b.maintenanceWindowsStore(region) identities := make([]MaintenanceWindowIdentity, 0, len(matchedWindowIDs)) for windowID := range matchedWindowIDs { - if mw, ok := maintenanceWindows[windowID]; ok { + if mw, ok := maintenanceWindows.Get(windowID); ok { identities = append(identities, MaintenanceWindowIdentity{ WindowID: mw.WindowID, Name: mw.Name, @@ -1340,8 +1340,8 @@ func (b *InMemoryBackend) UpdateMaintenanceWindowTarget( defer b.mu.Unlock() store := b.maintenanceWindowTargetsStore(region) - target, exists := store[input.WindowTargetID] - if !exists || target.WindowID != input.WindowID { + targetPtr, exists := store.Get(input.WindowTargetID) + if !exists || targetPtr.WindowID != input.WindowID { // Return a no-op success rather than error to preserve stub compat for // callers that send non-existent IDs (e.g. the simple stub coverage test). return &UpdateMaintenanceWindowTargetOutput{ @@ -1350,6 +1350,8 @@ func (b *InMemoryBackend) UpdateMaintenanceWindowTarget( }, nil } + target := *targetPtr + if input.OwnerInfo != "" { target.OwnerInfo = input.OwnerInfo } @@ -1366,7 +1368,7 @@ func (b *InMemoryBackend) UpdateMaintenanceWindowTarget( target.Targets = input.Targets } - store[input.WindowTargetID] = target + store.Put(&target) return &UpdateMaintenanceWindowTargetOutput{ WindowID: target.WindowID, @@ -1389,14 +1391,16 @@ func (b *InMemoryBackend) UpdateMaintenanceWindowTask( defer b.mu.Unlock() store := b.maintenanceWindowTasksStore(region) - task, exists := store[input.WindowTaskID] - if !exists || task.WindowID != input.WindowID { + taskPtr, exists := store.Get(input.WindowTaskID) + if !exists || taskPtr.WindowID != input.WindowID { return &UpdateMaintenanceWindowTaskOutput{ WindowID: input.WindowID, WindowTaskID: input.WindowTaskID, }, nil } + task := *taskPtr + if input.TaskArn != "" { task.TaskArn = input.TaskArn } @@ -1425,7 +1429,7 @@ func (b *InMemoryBackend) UpdateMaintenanceWindowTask( task.MaxErrors = input.MaxErrors } - store[input.WindowTaskID] = task + store.Put(&task) return &UpdateMaintenanceWindowTaskOutput{ WindowID: task.WindowID, @@ -1454,14 +1458,13 @@ func (b *InMemoryBackend) DeleteOpsItem( defer b.mu.Unlock() opsItems := b.opsItemsStore(region) - if _, exists := opsItems[input.OpsItemID]; !exists { + if !opsItems.Has(input.OpsItemID) { return nil, ErrOpsItemNotFound } - delete(opsItems, input.OpsItemID) + opsItems.Delete(input.OpsItemID) delete(b.opsItemRelatedItemsStore(region), input.OpsItemID) - cleanupEmptyInnerMap(b.opsItems, region) cleanupEmptyInnerMap(b.opsItemRelatedItems, region) return &DeleteOpsItemOutput{}, nil @@ -1630,15 +1633,14 @@ func (b *InMemoryBackend) DeleteOpsMetadata( defer b.mu.Unlock() opsMetadata := b.opsMetadataStore(region) - meta, exists := opsMetadata[input.OpsMetadataArn] + meta, exists := opsMetadata.Get(input.OpsMetadataArn) if !exists { return nil, ErrOpsMetadataNotFound } delete(b.resourceIDToOpsMetadataArnStore(region), meta.ResourceID) - delete(opsMetadata, input.OpsMetadataArn) + opsMetadata.Delete(input.OpsMetadataArn) - cleanupEmptyInnerMap(b.opsMetadata, region) cleanupEmptyInnerMap(b.resourceIDToOpsMetadataArn, region) return &DeleteOpsMetadataOutput{}, nil diff --git a/services/ssm/backend_stubs.go b/services/ssm/backend_stubs.go index 5d7666ecd..be8a9773e 100644 --- a/services/ssm/backend_stubs.go +++ b/services/ssm/backend_stubs.go @@ -957,14 +957,13 @@ func (b *InMemoryBackend) DeleteActivation( defer b.mu.Unlock() activations := b.activationsStore(region) - if _, exists := activations[input.ActivationID]; !exists { + if !activations.Has(input.ActivationID) { return nil, ErrActivationNotFound } - delete(activations, input.ActivationID) + activations.Delete(input.ActivationID) delete(b.miscResourceTagsStore(region), input.ActivationID) - cleanupEmptyInnerMap(b.activations, region) cleanupEmptyInnerMap(b.miscResourceTags, region) return &DeleteActivationOutput{}, nil @@ -980,11 +979,11 @@ func (b *InMemoryBackend) DeleteAssociation( defer b.mu.Unlock() associations := b.associationsStore(region) - if _, exists := associations[input.AssociationID]; !exists { + if !associations.Has(input.AssociationID) { return nil, ErrAssociationNotFound } - delete(associations, input.AssociationID) + associations.Delete(input.AssociationID) // Evict the association's execution records and their per-execution targets // so deleted associations do not leak execution state. @@ -996,7 +995,6 @@ func (b *InMemoryBackend) DeleteAssociation( delete(execs, input.AssociationID) } - cleanupEmptyInnerMap(b.associations, region) cleanupEmptyInnerMap(b.associationExecutions, region) cleanupEmptyInnerMap(b.associationExecTargets, region) @@ -1030,13 +1028,11 @@ func (b *InMemoryBackend) DeregisterTargetFromMaintenanceWindow( defer b.mu.Unlock() targets := b.maintenanceWindowTargetsStore(region) - if _, exists := targets[input.WindowTargetID]; !exists { + if !targets.Has(input.WindowTargetID) { return nil, ErrMaintenanceWindowNotFound } - delete(targets, input.WindowTargetID) - - cleanupEmptyInnerMap(b.maintenanceWindowTargets, region) + targets.Delete(input.WindowTargetID) return &DeregisterTargetFromMaintenanceWindowOutput{ WindowID: input.WindowID, @@ -1054,13 +1050,11 @@ func (b *InMemoryBackend) DeregisterTaskFromMaintenanceWindow( defer b.mu.Unlock() tasks := b.maintenanceWindowTasksStore(region) - if _, exists := tasks[input.WindowTaskID]; !exists { + if !tasks.Has(input.WindowTaskID) { return nil, ErrMaintenanceWindowNotFound } - delete(tasks, input.WindowTaskID) - - cleanupEmptyInnerMap(b.maintenanceWindowTasks, region) + tasks.Delete(input.WindowTaskID) return &DeregisterTaskFromMaintenanceWindowOutput{ WindowID: input.WindowID, @@ -1078,9 +1072,9 @@ func (b *InMemoryBackend) DescribeActivations( defer b.mu.RUnlock() activations := b.activationsStore(region) - list := make([]Activation, 0, len(activations)) - for _, a := range activations { - list = append(list, a) + list := make([]Activation, 0, activations.Len()) + for _, a := range activations.All() { + list = append(list, *a) } return &DescribeActivationsOutput{ActivationList: list}, nil @@ -1095,7 +1089,8 @@ func (b *InMemoryBackend) DescribeAssociation( b.mu.RLock("DescribeAssociation") defer b.mu.RUnlock() - for _, assoc := range b.associationsStore(region) { + for _, assocPtr := range b.associationsStore(region).All() { + assoc := *assocPtr if (input.AssociationID != "" && assoc.AssociationID == input.AssociationID) || (input.Name != "" && assoc.Name == input.Name && (input.InstanceID == "" || assoc.InstanceID == input.InstanceID)) { return &DescribeAssociationOutput{AssociationDescription: assoc}, nil @@ -1132,9 +1127,9 @@ func (b *InMemoryBackend) DescribeInstancePatchStatesForPatchGroup( b.mu.RLock("DescribeInstancePatchStatesForPatchGroup") defer b.mu.RUnlock() - store := b.instancePatchStates[region] + patchStates := b.instancePatchStatesStore(region) states := make([]InstancePatchState, 0) - for _, s := range store { + for _, s := range patchStates.All() { if s.PatchGroup == input.PatchGroup { states = append(states, *s) } @@ -1179,16 +1174,17 @@ func (b *InMemoryBackend) DescribeInstanceProperties( b.mu.RLock("DescribeInstanceProperties") defer b.mu.RUnlock() - store := b.instanceProperties[region] - props := make([]InstanceProperty, 0, len(store)+len(b.activationsStore(region))) - seen := make(map[string]struct{}, len(store)) + instancePropsTable := b.instancePropertiesStore(region) + activationsTable := b.activationsStore(region) + props := make([]InstanceProperty, 0, instancePropsTable.Len()+activationsTable.Len()) + seen := make(map[string]struct{}, instancePropsTable.Len()) - for _, p := range store { + for _, p := range instancePropsTable.All() { props = append(props, *p) seen[p.InstanceID] = struct{}{} } - for _, act := range b.activationsStore(region) { + for _, act := range activationsTable.All() { if _, ok := seen[act.ActivationID]; ok { continue } @@ -1218,9 +1214,9 @@ func (b *InMemoryBackend) DescribeMaintenanceWindowTargets( defer b.mu.RUnlock() var targets []MaintenanceWindowTarget - for _, t := range b.maintenanceWindowTargetsStore(region) { + for _, t := range b.maintenanceWindowTargetsStore(region).All() { if t.WindowID == input.WindowID { - targets = append(targets, t) + targets = append(targets, *t) } } @@ -1241,9 +1237,9 @@ func (b *InMemoryBackend) DescribeMaintenanceWindowTasks( defer b.mu.RUnlock() var tasks []MaintenanceWindowTask - for _, t := range b.maintenanceWindowTasksStore(region) { + for _, t := range b.maintenanceWindowTasksStore(region).All() { if t.WindowID == input.WindowID { - tasks = append(tasks, t) + tasks = append(tasks, *t) } } @@ -1264,8 +1260,8 @@ func (b *InMemoryBackend) DescribeMaintenanceWindows( defer b.mu.RUnlock() mws := b.maintenanceWindowsStore(region) - all := make([]MaintenanceWindowIdentity, 0, len(mws)) - for _, mw := range mws { + all := make([]MaintenanceWindowIdentity, 0, mws.Len()) + for _, mw := range mws.All() { all = append(all, MaintenanceWindowIdentity{ WindowID: mw.WindowID, Name: mw.Name, @@ -1342,8 +1338,9 @@ func (b *InMemoryBackend) DescribeOpsItems( defer b.mu.RUnlock() items := b.opsItemsStore(region) - all := make([]OpsItemSummary, 0, len(items)) - for _, item := range items { + all := make([]OpsItemSummary, 0, items.Len()) + for _, itemPtr := range items.All() { + item := *itemPtr if !opsItemMatchesFilters(item, input.OpsItemFilters) { continue } @@ -1423,8 +1420,9 @@ func (b *InMemoryBackend) DescribePatchBaselines( defer b.mu.RUnlock() baselines := b.patchBaselinesStore(region) - all := make([]PatchBaselineIdentity, 0, len(baselines)) - for _, bl := range baselines { + all := make([]PatchBaselineIdentity, 0, baselines.Len()) + for _, blPtr := range baselines.All() { + bl := *blPtr if !patchBaselineMatchesFilters(bl, input.Filters) { continue } @@ -1475,12 +1473,12 @@ func (b *InMemoryBackend) GetMaintenanceWindow( b.mu.RLock("GetMaintenanceWindow") defer b.mu.RUnlock() - mw, exists := b.maintenanceWindowsStore(region)[input.WindowID] + mw, exists := b.maintenanceWindowsStore(region).Get(input.WindowID) if !exists { return nil, ErrMaintenanceWindowNotFound } - return &GetMaintenanceWindowOutput{MaintenanceWindow: mw}, nil + return &GetMaintenanceWindowOutput{MaintenanceWindow: *mw}, nil } // GetOpsItem retrieves an OpsItem by ID. @@ -1492,12 +1490,12 @@ func (b *InMemoryBackend) GetOpsItem( b.mu.RLock("GetOpsItem") defer b.mu.RUnlock() - item, exists := b.opsItemsStore(region)[input.OpsItemID] + item, exists := b.opsItemsStore(region).Get(input.OpsItemID) if !exists { return nil, ErrOpsItemNotFound } - return &GetOpsItemOutput{OpsItem: item}, nil + return &GetOpsItemOutput{OpsItem: *item}, nil } // GetOpsMetadata retrieves OpsMetadata by ARN. @@ -1509,12 +1507,12 @@ func (b *InMemoryBackend) GetOpsMetadata( b.mu.RLock("GetOpsMetadata") defer b.mu.RUnlock() - meta, exists := b.opsMetadataStore(region)[input.OpsMetadataArn] + meta, exists := b.opsMetadataStore(region).Get(input.OpsMetadataArn) if !exists { return nil, ErrOpsMetadataNotFound } - return &GetOpsMetadataOutput{OpsMetadata: meta}, nil + return &GetOpsMetadataOutput{OpsMetadata: *meta}, nil } // GetPatchBaseline retrieves a patch baseline by ID. @@ -1526,12 +1524,12 @@ func (b *InMemoryBackend) GetPatchBaseline( b.mu.RLock("GetPatchBaseline") defer b.mu.RUnlock() - bl, exists := b.patchBaselinesStore(region)[input.BaselineID] + bl, exists := b.patchBaselinesStore(region).Get(input.BaselineID) if !exists { return nil, ErrPatchBaselineNotFound } - return &GetPatchBaselineOutput{PatchBaseline: bl}, nil + return &GetPatchBaselineOutput{PatchBaseline: *bl}, nil } // ListAssociations lists all stored associations. @@ -1544,9 +1542,9 @@ func (b *InMemoryBackend) ListAssociations( defer b.mu.RUnlock() associations := b.associationsStore(region) - list := make([]Association, 0, len(associations)) - for _, a := range associations { - list = append(list, a) + list := make([]Association, 0, associations.Len()) + for _, a := range associations.All() { + list = append(list, *a) } return &ListAssociationsOutput{Associations: list}, nil @@ -1561,7 +1559,7 @@ func (b *InMemoryBackend) RegisterPatchBaselineForPatchGroup( b.mu.Lock("RegisterPatchBaselineForPatchGroup") defer b.mu.Unlock() - if _, exists := b.patchBaselinesStore(region)[input.BaselineID]; !exists { + if !b.patchBaselinesStore(region).Has(input.BaselineID) { return nil, ErrPatchBaselineNotFound } @@ -1585,7 +1583,7 @@ func (b *InMemoryBackend) RegisterTargetWithMaintenanceWindow( b.mu.Lock("RegisterTargetWithMaintenanceWindow") defer b.mu.Unlock() - if _, exists := b.maintenanceWindowsStore(region)[input.WindowID]; !exists { + if !b.maintenanceWindowsStore(region).Has(input.WindowID) { return nil, ErrMaintenanceWindowNotFound } @@ -1600,10 +1598,7 @@ func (b *InMemoryBackend) RegisterTargetWithMaintenanceWindow( Name: input.Name, } - if b.maintenanceWindowTargets[region] == nil { - b.maintenanceWindowTargets[region] = make(map[string]MaintenanceWindowTarget) - } - b.maintenanceWindowTargetsStore(region)[targetID] = target + b.maintenanceWindowTargetsStore(region).Put(&target) return &RegisterTargetWithMaintenanceWindowOutput{WindowTargetID: targetID}, nil } @@ -1617,7 +1612,7 @@ func (b *InMemoryBackend) RegisterTaskWithMaintenanceWindow( b.mu.Lock("RegisterTaskWithMaintenanceWindow") defer b.mu.Unlock() - if _, exists := b.maintenanceWindowsStore(region)[input.WindowID]; !exists { + if !b.maintenanceWindowsStore(region).Has(input.WindowID) { return nil, ErrMaintenanceWindowNotFound } @@ -1635,10 +1630,7 @@ func (b *InMemoryBackend) RegisterTaskWithMaintenanceWindow( MaxErrors: input.MaxErrors, } - if b.maintenanceWindowTasks[region] == nil { - b.maintenanceWindowTasks[region] = make(map[string]MaintenanceWindowTask) - } - b.maintenanceWindowTasksStore(region)[taskID] = task + b.maintenanceWindowTasksStore(region).Put(&task) return &RegisterTaskWithMaintenanceWindowOutput{WindowTaskID: taskID}, nil } @@ -1675,10 +1667,7 @@ func (b *InMemoryBackend) StartSession( } } - if b.sessions[region] == nil { - b.sessions[region] = make(map[string]Session) - } - b.sessionsStore(region)[sessionID] = sess + b.sessionsStore(region).Put(&sess) return &StartSessionOutput{ SessionID: sessionID, @@ -1697,14 +1686,15 @@ func (b *InMemoryBackend) TerminateSession( defer b.mu.Unlock() sessions := b.sessionsStore(region) - sess, exists := sessions[input.SessionID] + sessPtr, exists := sessions.Get(input.SessionID) if !exists { return &TerminateSessionOutput{SessionID: input.SessionID}, nil } + sess := *sessPtr sess.Status = sessionStatusTerminated sess.EndDate = UnixTimeFloat(timeNow()) - sessions[input.SessionID] = sess + sessions.Put(&sess) // Bound retained terminated (history) sessions so the store cannot grow // without limit under repeated Start/Terminate cycles. @@ -1719,10 +1709,10 @@ func (b *InMemoryBackend) TerminateSession( func (b *InMemoryBackend) evictExcessTerminatedSessionsLocked(region string) { sessions := b.sessionsStore(region) - terminated := make([]Session, 0, len(sessions)) - for _, s := range sessions { + terminated := make([]Session, 0, sessions.Len()) + for _, s := range sessions.All() { if s.Status == sessionStatusTerminated { - terminated = append(terminated, s) + terminated = append(terminated, *s) } } @@ -1744,10 +1734,8 @@ func (b *InMemoryBackend) evictExcessTerminatedSessionsLocked(region string) { }) for _, s := range terminated[:len(terminated)-maxTerminatedSessionsPerRegion] { - delete(sessions, s.SessionID) + sessions.Delete(s.SessionID) } - - cleanupEmptyInnerMap(b.sessions, region) } // UpdateAssociation updates an existing association. @@ -1760,11 +1748,13 @@ func (b *InMemoryBackend) UpdateAssociation( defer b.mu.Unlock() associations := b.associationsStore(region) - assoc, exists := associations[input.AssociationID] + assocPtr, exists := associations.Get(input.AssociationID) if !exists { return nil, ErrAssociationNotFound } + assoc := *assocPtr + if input.AssociationName != "" { assoc.AssociationName = input.AssociationName } @@ -1782,7 +1772,7 @@ func (b *InMemoryBackend) UpdateAssociation( } assoc.LastUpdateAssociationDate = UnixTimeFloat(timeNow()) - associations[input.AssociationID] = assoc + associations.Put(&assoc) return &UpdateAssociationOutput{AssociationDescription: assoc}, nil } @@ -1797,11 +1787,13 @@ func (b *InMemoryBackend) UpdateMaintenanceWindow( defer b.mu.Unlock() windows := b.maintenanceWindowsStore(region) - mw, exists := windows[input.WindowID] + mwPtr, exists := windows.Get(input.WindowID) if !exists { return nil, ErrMaintenanceWindowNotFound } + mw := *mwPtr + if input.Name != "" { mw.Name = input.Name } @@ -1827,7 +1819,7 @@ func (b *InMemoryBackend) UpdateMaintenanceWindow( } mw.ModifiedDate = UnixTimeFloat(timeNow()) - windows[input.WindowID] = mw + windows.Put(&mw) return &UpdateMaintenanceWindowOutput{MaintenanceWindow: mw}, nil } @@ -1842,11 +1834,13 @@ func (b *InMemoryBackend) UpdateOpsItem( defer b.mu.Unlock() items := b.opsItemsStore(region) - item, exists := items[input.OpsItemID] + itemPtr, exists := items.Get(input.OpsItemID) if !exists { return nil, ErrOpsItemNotFound } + item := *itemPtr + if input.Title != "" { item.Title = input.Title } @@ -1876,7 +1870,7 @@ func (b *InMemoryBackend) UpdateOpsItem( } item.LastModifiedTime = UnixTimeFloat(timeNow()) - items[input.OpsItemID] = item + items.Put(&item) // Record an event for the update. b.opsItemEvents[region] = append(b.opsItemEvents[region], OpsItemEventSummary{ @@ -1897,11 +1891,13 @@ func (b *InMemoryBackend) UpdateOpsMetadata( defer b.mu.Unlock() opsMetadata := b.opsMetadataStore(region) - meta, exists := opsMetadata[input.OpsMetadataArn] + metaPtr, exists := opsMetadata.Get(input.OpsMetadataArn) if !exists { return nil, ErrOpsMetadataNotFound } + meta := *metaPtr + if input.Metadata != nil { if meta.Metadata == nil { meta.Metadata = make(map[string]MetadataValue) @@ -1911,7 +1907,7 @@ func (b *InMemoryBackend) UpdateOpsMetadata( } meta.LastModifiedDate = UnixTimeFloat(timeNow()) - opsMetadata[input.OpsMetadataArn] = meta + opsMetadata.Put(&meta) return &UpdateOpsMetadataOutput{OpsMetadataArn: input.OpsMetadataArn}, nil } @@ -1926,11 +1922,13 @@ func (b *InMemoryBackend) UpdatePatchBaseline( defer b.mu.Unlock() baselines := b.patchBaselinesStore(region) - bl, exists := baselines[input.BaselineID] + blPtr, exists := baselines.Get(input.BaselineID) if !exists { return nil, ErrPatchBaselineNotFound } + bl := *blPtr + if input.Name != "" { bl.Name = input.Name } @@ -1952,7 +1950,7 @@ func (b *InMemoryBackend) UpdatePatchBaseline( } bl.ModifiedDate = UnixTimeFloat(timeNow()) - baselines[input.BaselineID] = bl + baselines.Put(&bl) return &UpdatePatchBaselineOutput{PatchBaseline: bl}, nil } diff --git a/services/ssm/export_test.go b/services/ssm/export_test.go index 38233ecc2..7228c0979 100644 --- a/services/ssm/export_test.go +++ b/services/ssm/export_test.go @@ -31,7 +31,7 @@ func (b *InMemoryBackend) CommandCount() int { total := 0 for _, cmds := range b.commands { - total += len(cmds) + total += cmds.Len() } return total @@ -58,9 +58,10 @@ func (b *InMemoryBackend) SetCommandExpiresAfter(cmdID string, expiresAfter floa // Try all regions. for _, cmds := range b.commands { - if cmd, ok := cmds[cmdID]; ok { + if cmdPtr, ok := cmds.Get(cmdID); ok { + cmd := *cmdPtr cmd.ExpiresAfter = expiresAfter - cmds[cmdID] = cmd + cmds.Put(&cmd) return } @@ -118,7 +119,7 @@ func (b *InMemoryBackend) ActivationCount() int { b.mu.RLock("ActivationCount") defer b.mu.RUnlock() - return len(b.activationsStore(b.Region())) + return b.activationsStore(b.Region()).Len() } // AssociationCount returns the number of associations stored in the default region. @@ -126,7 +127,7 @@ func (b *InMemoryBackend) AssociationCount() int { b.mu.RLock("AssociationCount") defer b.mu.RUnlock() - return len(b.associationsStore(b.Region())) + return b.associationsStore(b.Region()).Len() } // MaintenanceWindowCount returns the number of maintenance windows stored in the default region. @@ -134,7 +135,7 @@ func (b *InMemoryBackend) MaintenanceWindowCount() int { b.mu.RLock("MaintenanceWindowCount") defer b.mu.RUnlock() - return len(b.maintenanceWindowsStore(b.Region())) + return b.maintenanceWindowsStore(b.Region()).Len() } // OpsItemCount returns the number of OpsItems stored in the default region. @@ -142,7 +143,7 @@ func (b *InMemoryBackend) OpsItemCount() int { b.mu.RLock("OpsItemCount") defer b.mu.RUnlock() - return len(b.opsItemsStore(b.Region())) + return b.opsItemsStore(b.Region()).Len() } // OpsMetadataCount returns the number of OpsMetadata entries stored in the default region. @@ -150,7 +151,7 @@ func (b *InMemoryBackend) OpsMetadataCount() int { b.mu.RLock("OpsMetadataCount") defer b.mu.RUnlock() - return len(b.opsMetadataStore(b.Region())) + return b.opsMetadataStore(b.Region()).Len() } // PatchBaselineCount returns the number of patch baselines stored in the default region. @@ -158,7 +159,7 @@ func (b *InMemoryBackend) PatchBaselineCount() int { b.mu.RLock("PatchBaselineCount") defer b.mu.RUnlock() - return len(b.patchBaselinesStore(b.Region())) + return b.patchBaselinesStore(b.Region()).Len() } // HandlerOpsLen returns the number of supported operations. @@ -171,10 +172,7 @@ func (b *InMemoryBackend) AddActivationInternal(act Activation) { b.mu.Lock("AddActivationInternal") defer b.mu.Unlock() r := b.Region() - if b.activations[r] == nil { - b.activations[r] = make(map[string]Activation) - } - b.activationsStore(r)[act.ActivationID] = act + b.activationsStore(r).Put(&act) } // AddAssociationInternal seeds an association directly into the backend for testing. @@ -182,10 +180,7 @@ func (b *InMemoryBackend) AddAssociationInternal(assoc Association) { b.mu.Lock("AddAssociationInternal") defer b.mu.Unlock() r := b.Region() - if b.associations[r] == nil { - b.associations[r] = make(map[string]Association) - } - b.associationsStore(r)[assoc.AssociationID] = assoc + b.associationsStore(r).Put(&assoc) } // AddMaintenanceWindowInternal seeds a maintenance window directly into the backend for testing. @@ -193,10 +188,7 @@ func (b *InMemoryBackend) AddMaintenanceWindowInternal(mw MaintenanceWindow) { b.mu.Lock("AddMaintenanceWindowInternal") defer b.mu.Unlock() r := b.Region() - if b.maintenanceWindows[r] == nil { - b.maintenanceWindows[r] = make(map[string]MaintenanceWindow) - } - b.maintenanceWindowsStore(r)[mw.WindowID] = mw + b.maintenanceWindowsStore(r).Put(&mw) } // AddOpsItemInternal seeds an OpsItem directly into the backend for testing. @@ -204,10 +196,7 @@ func (b *InMemoryBackend) AddOpsItemInternal(item OpsItem) { b.mu.Lock("AddOpsItemInternal") defer b.mu.Unlock() r := b.Region() - if b.opsItems[r] == nil { - b.opsItems[r] = make(map[string]OpsItem) - } - b.opsItemsStore(r)[item.OpsItemID] = item + b.opsItemsStore(r).Put(&item) } // AddOpsMetadataInternal seeds OpsMetadata directly into the backend for testing. @@ -215,10 +204,7 @@ func (b *InMemoryBackend) AddOpsMetadataInternal(meta OpsMetadata) { b.mu.Lock("AddOpsMetadataInternal") defer b.mu.Unlock() r := b.Region() - if b.opsMetadata[r] == nil { - b.opsMetadata[r] = make(map[string]OpsMetadata) - } - b.opsMetadataStore(r)[meta.OpsMetadataArn] = meta + b.opsMetadataStore(r).Put(&meta) } // AddPatchBaselineInternal seeds a patch baseline directly into the backend for testing. @@ -226,10 +212,7 @@ func (b *InMemoryBackend) AddPatchBaselineInternal(bl PatchBaseline) { b.mu.Lock("AddPatchBaselineInternal") defer b.mu.Unlock() r := b.Region() - if b.patchBaselines[r] == nil { - b.patchBaselines[r] = make(map[string]PatchBaseline) - } - b.patchBaselinesStore(r)[bl.BaselineID] = bl + b.patchBaselinesStore(r).Put(&bl) } // OpsItemRelatedItemCount returns the total number of related items across all OpsItems. @@ -245,7 +228,12 @@ func (b *InMemoryBackend) GetPatchBaselineInternal(id string) PatchBaseline { b.mu.RLock("GetPatchBaselineInternal") defer b.mu.RUnlock() - return b.patchBaselinesStore(b.Region())[id] + bl, ok := b.patchBaselinesStore(b.Region()).Get(id) + if !ok { + return PatchBaseline{} + } + + return *bl } // ForceInsertParameter injects a parameter directly into the backend, bypassing @@ -254,10 +242,7 @@ func (b *InMemoryBackend) ForceInsertParameter(p Parameter) { b.mu.Lock("ForceInsertParameter") defer b.mu.Unlock() r := b.Region() - if b.parameters[r] == nil { - b.parameters[r] = make(map[string]Parameter) - } - b.parametersStore(r)[p.Name] = p + b.parametersStore(r).Put(&p) } // AddInstancePatchStateInternal seeds an InstancePatchState directly into the backend for testing. @@ -265,10 +250,7 @@ func (b *InMemoryBackend) AddInstancePatchStateInternal(s InstancePatchState) { b.mu.Lock("AddInstancePatchStateInternal") defer b.mu.Unlock() r := b.Region() - if b.instancePatchStates[r] == nil { - b.instancePatchStates[r] = make(map[string]*InstancePatchState) - } - b.instancePatchStatesStore(r)[s.InstanceID] = &s + b.instancePatchStatesStore(r).Put(&s) } // AddInstancePatchesInternal seeds PatchComplianceData for an instance directly into the backend for testing. @@ -287,10 +269,7 @@ func (b *InMemoryBackend) AddInstancePropertyInternal(p InstanceProperty) { b.mu.Lock("AddInstancePropertyInternal") defer b.mu.Unlock() r := b.Region() - if b.instanceProperties[r] == nil { - b.instanceProperties[r] = make(map[string]*InstanceProperty) - } - b.instancePropertiesStore(r)[p.InstanceID] = &p + b.instancePropertiesStore(r).Put(&p) } // AddAvailablePatchInternal seeds a Patch into the available patches catalog for testing. @@ -321,7 +300,7 @@ func (b *InMemoryBackend) ForceCompleteAutomations() { future := timeNow().Add(1_000_000 * time.Hour) for _, execs := range b.automationExecutions { - for _, e := range execs { + for _, e := range execs.All() { materializeAutomationLocked(e, future) } } @@ -332,7 +311,7 @@ func (b *InMemoryBackend) SessionCount() int { b.mu.RLock("SessionCount") defer b.mu.RUnlock() - return len(b.sessionsStore(b.Region())) + return b.sessionsStore(b.Region()).Len() } // AddTerminatedSessionInternal seeds a terminated session with the given end @@ -341,14 +320,11 @@ func (b *InMemoryBackend) AddTerminatedSessionInternal(id string, endDate float6 b.mu.Lock("AddTerminatedSessionInternal") defer b.mu.Unlock() r := b.Region() - if b.sessions[r] == nil { - b.sessions[r] = make(map[string]Session) - } - b.sessionsStore(r)[id] = Session{ + b.sessionsStore(r).Put(&Session{ SessionID: id, Status: sessionStatusTerminated, EndDate: endDate, - } + }) } // AssociationExecutionCount returns the number of stored execution records for diff --git a/services/ssm/handler.go b/services/ssm/handler.go index beb4e9540..3e0764ae8 100644 --- a/services/ssm/handler.go +++ b/services/ssm/handler.go @@ -557,6 +557,10 @@ func classifySSMError(reqErr error) (string, int) { return "InvalidCommandId", statusCode case errors.Is(reqErr, ErrValidationException): return "ValidationException", statusCode + case errors.Is(reqErr, ErrHierarchyLevelLimitExceeded): + return "HierarchyLevelLimitExceededException", statusCode + case errors.Is(reqErr, ErrParameterMaxVersionLimitExceeded): + return "ParameterMaxVersionLimitExceeded", statusCode } return classifySSMErrorExtended(reqErr) diff --git a/services/ssm/janitor.go b/services/ssm/janitor.go index f8db57a32..b93cd4b2b 100644 --- a/services/ssm/janitor.go +++ b/services/ssm/janitor.go @@ -70,21 +70,15 @@ func (j *Janitor) sweepTerminatedSessions(ctx context.Context) { var expired []expiredSession for region, sessions := range b.sessions { - for id, s := range sessions { + for _, s := range sessions.All() { if s.Status == sessionStatusTerminated && s.EndDate > 0 && s.EndDate < cutoff { - expired = append(expired, expiredSession{region: region, id: id}) + expired = append(expired, expiredSession{region: region, id: s.SessionID}) } } } - regions := make(map[string]struct{}, len(expired)) for _, e := range expired { - delete(b.sessions[e.region], e.id) - regions[e.region] = struct{}{} - } - - for region := range regions { - cleanupEmptyInnerMap(b.sessions, region) + b.sessions[e.region].Delete(e.id) } b.mu.Unlock() @@ -114,22 +108,21 @@ func (j *Janitor) sweepExpiredCommands(ctx context.Context) { var expired []expiredCmd for region, commands := range b.commands { - for id, cmd := range commands { + for _, cmd := range commands.All() { if cmd.ExpiresAfter > 0 && cmd.ExpiresAfter < now { - expired = append(expired, expiredCmd{region: region, id: id}) + expired = append(expired, expiredCmd{region: region, id: cmd.CommandID}) } } } regions := make(map[string]struct{}, len(expired)) for _, e := range expired { - delete(b.commands[e.region], e.id) + b.commands[e.region].Delete(e.id) delete(b.commandInvocations[e.region], e.id) regions[e.region] = struct{}{} } for region := range regions { - cleanupEmptyInnerMap(b.commands, region) cleanupEmptyInnerMap(b.commandInvocations, region) } @@ -172,15 +165,15 @@ func (j *Janitor) sweepExpiredParameters(ctx context.Context) { var expired []expiredParam for region, params := range b.parameters { - for name, p := range params { + for _, p := range params.All() { if t, ok := parameterExpiresAt(p.Policies); ok && now.After(t) { - expired = append(expired, expiredParam{region: region, name: name}) + expired = append(expired, expiredParam{region: region, name: p.Name}) } } } for _, e := range expired { - delete(b.parameters[e.region], e.name) + b.parameters[e.region].Delete(e.name) delete(b.history[e.region], e.name) delete(b.parameterLabels[e.region], e.name) } diff --git a/services/ssm/models.go b/services/ssm/models.go index 99b7b3d7d..ad49546e2 100644 --- a/services/ssm/models.go +++ b/services/ssm/models.go @@ -267,6 +267,29 @@ type Document struct { CreatedDate float64 `json:"CreatedDate"` } +// DocumentDescription is the document metadata shape returned by +// CreateDocument, UpdateDocument, and DescribeDocument. Unlike Document (the +// internal storage representation), AWS's real DocumentDescription structure +// deliberately omits Content — only GetDocument returns document content, to +// avoid every metadata call re-transmitting potentially large document bodies. +type DocumentDescription struct { + TargetType string `json:"TargetType,omitempty"` + LatestVersion string `json:"LatestVersion"` + DocumentType string `json:"DocumentType"` + DocumentFormat string `json:"DocumentFormat"` + Status string `json:"Status"` + StatusInformation string `json:"StatusInformation,omitempty"` + DefaultVersion string `json:"DefaultVersion"` + Name string `json:"Name"` + SchemaVersion string `json:"SchemaVersion"` + Description string `json:"Description,omitempty"` + DocumentVersion string `json:"DocumentVersion"` + PlatformTypes []string `json:"PlatformTypes,omitempty"` + Attachments []DocumentAttachment `json:"Attachments,omitempty"` + Requires []DocumentRequires `json:"Requires,omitempty"` + CreatedDate float64 `json:"CreatedDate"` +} + // DocumentVersion represents a specific version of an SSM document. type DocumentVersion struct { Name string `json:"Name"` @@ -315,7 +338,7 @@ type CreateDocumentInput struct { // CreateDocumentOutput is the response payload for CreateDocument. type CreateDocumentOutput struct { - DocumentDescription Document `json:"DocumentDescription"` + DocumentDescription DocumentDescription `json:"DocumentDescription"` } // GetDocumentInput is the request payload for GetDocument. @@ -343,7 +366,7 @@ type DescribeDocumentInput struct { // DescribeDocumentOutput is the response payload for DescribeDocument. type DescribeDocumentOutput struct { - Document Document `json:"Document"` + Document DocumentDescription `json:"Document"` } // ListDocumentsInput is the request payload for ListDocuments. @@ -370,7 +393,7 @@ type UpdateDocumentInput struct { // UpdateDocumentOutput is the response payload for UpdateDocument. type UpdateDocumentOutput struct { - DocumentDescription Document `json:"DocumentDescription"` + DocumentDescription DocumentDescription `json:"DocumentDescription"` } // DeleteDocumentInput is the request payload for DeleteDocument. diff --git a/services/ssm/parity_emr_test.go b/services/ssm/parity_emr_test.go index f1061c112..c12eb8ada 100644 --- a/services/ssm/parity_emr_test.go +++ b/services/ssm/parity_emr_test.go @@ -245,6 +245,7 @@ func TestParityEMR_ParameterExpiration_JanitorEvicts(t *testing.T) { Name: "/expire/past", Value: "gone-soon", Type: "String", + Tier: "Advanced", Policies: policies, }) require.NoError(t, err) @@ -279,6 +280,7 @@ func TestParityEMR_ParameterExpiration_JanitorEvicts(t *testing.T) { Name: "/expire/future", Value: "still-here", Type: "String", + Tier: "Advanced", Policies: policies, }) require.NoError(t, err) @@ -326,6 +328,7 @@ func TestParityEMR_ParameterExpiration_JanitorEvicts(t *testing.T) { Name: "/expire/rfc3339", Value: "gone", Type: "String", + Tier: "Advanced", Policies: policies, }) require.NoError(t, err) diff --git a/services/ssm/parity_sweep3_test.go b/services/ssm/parity_sweep3_test.go new file mode 100644 index 000000000..21b22c263 --- /dev/null +++ b/services/ssm/parity_sweep3_test.go @@ -0,0 +1,404 @@ +package ssm_test + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/ssm" +) + +// Test_PutParameter_HierarchyLevelLimit verifies AWS's documented constraint +// that a parameter name hierarchy can have a maximum of 15 "/"-delimited +// levels (including the final name segment). Exceeding it must fail with +// HierarchyLevelLimitExceededException, matching real SSM PutParameter +// behavior (see API_PutParameter docs: "/L1/.../L14/name" is valid, +// "/L1/.../L15/L16/name" is not). +func Test_PutParameter_HierarchyLevelLimit(t *testing.T) { + t.Parallel() + + fifteenLevels := "/" + strings.Join(makeLevels(14), "/") + "/name" // 14 + name = 15 + sixteenLevels := "/" + strings.Join(makeLevels(15), "/") + "/name" // 15 + name = 16 + + cases := []struct { + name string + paramName string + wantErr bool + }{ + {name: "exactly 15 levels is valid", paramName: fifteenLevels, wantErr: false}, + {name: "16 levels exceeds the limit", paramName: sixteenLevels, wantErr: true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := ssm.NewInMemoryBackend() + ctx := context.Background() + + _, err := b.PutParameter(ctx, &ssm.PutParameterInput{ + Name: tc.paramName, + Type: ssm.StringType, + Value: "v", + }) + + if tc.wantErr { + require.ErrorIs(t, err, ssm.ErrHierarchyLevelLimitExceeded) + } else { + require.NoError(t, err) + } + }) + } +} + +func makeLevels(n int) []string { + levels := make([]string, n) + for i := range levels { + levels[i] = fmt.Sprintf("L%d", i) + } + + return levels +} + +// countParameterHistory paginates through GetParameterHistory (page size 50) +// and returns the total number of history entries for name. +func countParameterHistory(ctx context.Context, t *testing.T, b *ssm.InMemoryBackend, name string) int { + t.Helper() + + total := 0 + nextToken := "" + + for { + out, err := b.GetParameterHistory(ctx, &ssm.GetParameterHistoryInput{ + Name: name, + NextToken: nextToken, + }) + require.NoError(t, err) + + total += len(out.Parameters) + if out.NextToken == "" { + break + } + + nextToken = out.NextToken + } + + return total +} + +// Test_PutParameter_MaxVersionLimit_LabeledOldestBlocksEviction verifies AWS +// behavior: Parameter Store keeps only the most recent MaxHistoryCap (100) +// versions, auto-deleting the oldest on each new PutParameter. But if the +// oldest version carries a label, AWS refuses to delete it and the PutParameter +// call fails with ParameterMaxVersionLimitExceeded instead of silently +// evicting a labeled version. +func Test_PutParameter_MaxVersionLimit_LabeledOldestBlocksEviction(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + labelOldest bool + wantErr bool + }{ + {name: "unlabeled oldest version is evicted normally", labelOldest: false, wantErr: false}, + {name: "labeled oldest version blocks the new write", labelOldest: true, wantErr: true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := ssm.NewInMemoryBackend() + ctx := context.Background() + + const paramName = "/history/param" + + // Fill history to exactly MaxHistoryCap versions. + for range ssm.MaxHistoryCap { + _, err := b.PutParameter(ctx, &ssm.PutParameterInput{ + Name: paramName, + Type: ssm.StringType, + Value: "v", + Overwrite: true, + }) + require.NoError(t, err) + } + + if tc.labelOldest { + _, err := b.LabelParameterVersion(ctx, &ssm.LabelParameterVersionInput{ + Name: paramName, + ParameterVersion: 1, // the oldest surviving version + Labels: []string{"pinned"}, + }) + require.NoError(t, err) + } + + // One more write would push total versions to MaxHistoryCap+1, + // requiring eviction of version 1. + _, err := b.PutParameter(ctx, &ssm.PutParameterInput{ + Name: paramName, + Type: ssm.StringType, + Value: "one-more", + Overwrite: true, + }) + + if tc.wantErr { + require.ErrorIs(t, err, ssm.ErrParameterMaxVersionLimitExceeded) + + // The rejected write must not have mutated state: history stays + // at MaxHistoryCap (paginated at 50/page) and the current + // version is unchanged. + assert.Equal(t, ssm.MaxHistoryCap, countParameterHistory(ctx, t, b, paramName)) + } else { + require.NoError(t, err) + } + }) + } +} + +// Test_PutParameter_IntelligentTiering_AutoUpgradesToAdvanced verifies that, +// per AWS docs, Intelligent-Tiering parameters are auto-promoted to the +// Advanced tier whenever the request needs a capability Standard doesn't +// support (value over 4 KiB, or parameter policies attached) instead of the +// PutParameter call failing. +func Test_PutParameter_IntelligentTiering_AutoUpgradesToAdvanced(t *testing.T) { + t.Parallel() + + smallValue := strings.Repeat("a", 100) + overStandardValue := strings.Repeat("a", 5000) // >4096, <=8192 + + cases := []struct { + name string + tier string + value string + wantTier string + wantErr bool + }{ + { + // AWS echoes back the requested "Intelligent-Tiering" tier as-is + // when no capability forces a promotion — it does not resolve to + // the concrete underlying "Standard" tier in the response. + name: "small value on Intelligent-Tiering reports Intelligent-Tiering", + tier: "Intelligent-Tiering", + value: smallValue, + wantTier: "Intelligent-Tiering", + }, + { + name: "value over 4KiB on Intelligent-Tiering auto-upgrades to Advanced", + tier: "Intelligent-Tiering", + value: overStandardValue, + wantTier: "Advanced", + }, + { + name: "value over 4KiB on explicit Standard tier fails", + tier: "Standard", + value: overStandardValue, + wantErr: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := ssm.NewInMemoryBackend() + ctx := context.Background() + + out, err := b.PutParameter(ctx, &ssm.PutParameterInput{ + Name: "/tier/param", + Type: ssm.StringType, + Value: tc.value, + Tier: tc.tier, + }) + + if tc.wantErr { + require.ErrorIs(t, err, ssm.ErrValidationException) + + return + } + + require.NoError(t, err) + assert.Equal(t, tc.wantTier, out.Tier) + }) + } +} + +// Test_PutParameter_PoliciesRequireAdvancedTier verifies AWS's documented +// constraint that parameter policies (Expiration, ExpirationNotification, +// NoChangeNotification) are only supported on Advanced-tier parameters; +// Standard tier rejects them, and Intelligent-Tiering auto-upgrades to +// Advanced to accommodate them. +func Test_PutParameter_PoliciesRequireAdvancedTier(t *testing.T) { + t.Parallel() + + const policy = `[{"Type":"Expiration","Version":"1.0","Attributes":{"Timestamp":"2099-01-01T00:00:00.000Z"}}]` + + cases := []struct { + name string + tier string + wantTier string + wantErr bool + }{ + {name: "default (Standard) tier rejects policies", tier: "", wantErr: true}, + {name: "explicit Standard tier rejects policies", tier: "Standard", wantErr: true}, + {name: "Advanced tier accepts policies", tier: "Advanced", wantTier: "Advanced"}, + { + name: "Intelligent-Tiering auto-upgrades for policies", + tier: "Intelligent-Tiering", + wantTier: "Advanced", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := ssm.NewInMemoryBackend() + ctx := context.Background() + + out, err := b.PutParameter(ctx, &ssm.PutParameterInput{ + Name: "/policy/param", + Type: ssm.StringType, + Value: "v", + Tier: tc.tier, + Policies: policy, + }) + + if tc.wantErr { + require.ErrorIs(t, err, ssm.ErrValidationException) + + return + } + + require.NoError(t, err) + assert.Equal(t, tc.wantTier, out.Tier) + }) + } +} + +// Test_GetDocument_DefaultVersionSelector verifies that an explicit +// "$DEFAULT" DocumentVersion resolves to the document's DefaultVersion (as +// set by UpdateDocumentDefaultVersion), which can genuinely differ from the +// latest version. Previously $DEFAULT was conflated with $LATEST and always +// served the newest content even when the default had been pinned to an +// older version. +func Test_GetDocument_DefaultVersionSelector(t *testing.T) { + t.Parallel() + + newBackendWithTwoVersions := func(t *testing.T) *ssm.InMemoryBackend { + t.Helper() + + b := ssm.NewInMemoryBackend() + ctx := context.Background() + + _, err := b.CreateDocument(ctx, &ssm.CreateDocumentInput{Name: "Doc", Content: `{"v":1}`}) + require.NoError(t, err) + _, err = b.UpdateDocument(ctx, &ssm.UpdateDocumentInput{Name: "Doc", Content: `{"v":2}`}) + require.NoError(t, err) + + // Pin the default back to version 1, diverging it from LatestVersion (2). + _, err = b.UpdateDocumentDefaultVersion(ctx, &ssm.UpdateDocumentDefaultVersionInput{ + Name: "Doc", DocumentVersion: "1", + }) + require.NoError(t, err) + + return b + } + + cases := []struct { + name string + documentVersion string + wantContent string + }{ + { + name: "explicit $DEFAULT resolves to the pinned default (v1)", + documentVersion: "$DEFAULT", + wantContent: `{"v":1}`, + }, + { + name: "explicit $LATEST resolves to the newest version (v2)", + documentVersion: "$LATEST", + wantContent: `{"v":2}`, + }, + {name: "explicit version 1 resolves to v1 regardless of default", documentVersion: "1", wantContent: `{"v":1}`}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := newBackendWithTwoVersions(t) + ctx := context.Background() + + out, err := b.GetDocument(ctx, &ssm.GetDocumentInput{ + Name: "Doc", DocumentVersion: tc.documentVersion, + }) + require.NoError(t, err) + assert.Equal(t, tc.wantContent, out.Content) + }) + } +} + +// Test_DescribeDocument_OmitsContentAndHonorsVersionSelector verifies two +// wire-shape facts about DescribeDocument that CreateDocument/UpdateDocument +// share: (1) real AWS's DocumentDescription response never includes the +// document Content field — that's GetDocument's job — and (2) an explicit +// DocumentVersion selector changes the per-version metadata (DocumentVersion, +// DocumentFormat, Status) returned, not just the latest version's. +func Test_DescribeDocument_OmitsContentAndHonorsVersionSelector(t *testing.T) { + t.Parallel() + + b := ssm.NewInMemoryBackend() + ctx := context.Background() + + createOut, err := b.CreateDocument(ctx, &ssm.CreateDocumentInput{Name: "Doc", Content: `{"v":1}`}) + require.NoError(t, err) + + updateOut, err := b.UpdateDocument(ctx, &ssm.UpdateDocumentInput{Name: "Doc", Content: `{"v":2}`}) + require.NoError(t, err) + + // The wire-serialized form of DocumentDescription must not carry a + // "Content" field at all — assert this at the JSON level since a Go + // zero-value string would round-trip identically to "field absent" in a + // struct-based comparison. + createJSON, marshalErr := json.Marshal(createOut.DocumentDescription) + require.NoError(t, marshalErr) + assert.NotContains(t, string(createJSON), "Content", + "CreateDocumentOutput.DocumentDescription must not serialize a Content field") + + updateJSON, marshalErr := json.Marshal(updateOut.DocumentDescription) + require.NoError(t, marshalErr) + assert.NotContains(t, string(updateJSON), "Content", + "UpdateDocumentOutput.DocumentDescription must not serialize a Content field") + + cases := []struct { + name string + documentVersion string + wantVersion string + }{ + {name: "no version selector describes the latest version", documentVersion: "", wantVersion: "2"}, + {name: "explicit version 1 describes that older version", documentVersion: "1", wantVersion: "1"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + describeOut, describeErr := b.DescribeDocument(ctx, &ssm.DescribeDocumentInput{ + Name: "Doc", DocumentVersion: tc.documentVersion, + }) + require.NoError(t, describeErr) + assert.Equal(t, tc.wantVersion, describeOut.Document.DocumentVersion) + + describeJSON, jsonErr := json.Marshal(describeOut.Document) + require.NoError(t, jsonErr) + assert.NotContains(t, string(describeJSON), "Content", + "DescribeDocumentOutput.Document must not serialize a Content field") + }) + } +} diff --git a/services/ssm/patch_inventory.go b/services/ssm/patch_inventory.go index 1e4a10544..f5e351960 100644 --- a/services/ssm/patch_inventory.go +++ b/services/ssm/patch_inventory.go @@ -131,9 +131,12 @@ func (b *InMemoryBackend) resolvePatchGroupBaseline(region, patchGroup string) ( return PatchBaseline{}, false } - bl, found := b.patchBaselinesStore(region)[id] + blPtr, found := b.patchBaselinesStore(region).Get(id) + if !found { + return PatchBaseline{}, false + } - return bl, found + return *blPtr, true } // applyPatchBaselineOperation simulates a Patch Manager scan/install run @@ -186,10 +189,7 @@ func (b *InMemoryBackend) applyPatchBaselineOperation( MissingCount: missingCount, } - if b.instancePatchStates[region] == nil { - b.instancePatchStates[region] = make(map[string]*InstancePatchState) - } - b.instancePatchStatesStore(region)[instanceID] = state + b.instancePatchStatesStore(region).Put(state) if b.instancePatches[region] == nil { b.instancePatches[region] = make(map[string][]PatchComplianceData) diff --git a/services/ssm/persistence.go b/services/ssm/persistence.go index 8dff68f11..c9f111f4d 100644 --- a/services/ssm/persistence.go +++ b/services/ssm/persistence.go @@ -3,58 +3,65 @@ package ssm import ( "context" "encoding/json" + "fmt" + "strings" "github.com/blackbirdworks/gopherstack/pkgs/logger" "github.com/blackbirdworks/gopherstack/pkgs/persistence" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) +// ssmSnapshotVersion identifies the shape of backendSnapshot's Tables blob +// (i.e. the set of resource tables registered on b.registry -- see +// store_setup.go). It must be bumped whenever a change there would make an +// older snapshot unsafe to decode as the current shape. Restore compares this +// against the persisted value and discards (rather than attempts to +// partially decode) any mismatch -- see Restore below. This mirrors the +// services/sqs pilot (commit 0f09d77c) and the services/ec2 conversion +// (commit 12e611a4). +const ssmSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the SSM backend. +// +// Tables holds one JSON-encoded, key-sorted array per registered +// *store.Table[V], produced by [github.com/blackbirdworks/gopherstack/pkgs/store.Registry.SnapshotAll]. +// Each table is registered under "/" (see +// store_setup.go's getOrCreateTable) rather than a single flat name per +// resource, because a single ssm InMemoryBackend holds every region's state +// (unlike ec2/sqs, which are one region per backend) -- Restore must +// pre-register every (resource, region) pair found in Tables before calling +// registry.RestoreAll, see below. +// +// Every other field below is a resource collection deliberately left as a +// plain (possibly nested) map rather than a *store.Table -- see +// store_setup.go's package doc for why each one doesn't fit Table's +// func(*V) string keying requirement. type backendSnapshot struct { - Parameters map[string]map[string]Parameter `json:"parameters"` + Tables map[string]json.RawMessage `json:"tables"` History map[string]map[string][]ParameterHistory `json:"history"` Tags map[string]map[string]*tags.Tags `json:"tags"` - Documents map[string]map[string]Document `json:"documents"` DocumentVersions map[string]map[string][]DocumentVersion `json:"document_versions"` DocumentPermissions map[string]map[string][]string `json:"document_permissions"` - Commands map[string]map[string]Command `json:"commands"` CommandInvocations map[string]map[string][]CommandInvocation `json:"command_invocations"` - Activations map[string]map[string]Activation `json:"activations"` - Associations map[string]map[string]Association `json:"associations"` - MaintenanceWindows map[string]map[string]MaintenanceWindow `json:"maintenance_windows"` - MaintenanceWindowTargets map[string]map[string]MaintenanceWindowTarget `json:"maintenance_window_targets"` - MaintenanceWindowTasks map[string]map[string]MaintenanceWindowTask `json:"maintenance_window_tasks"` - Sessions map[string]map[string]Session `json:"sessions"` PatchGroupToBaseline map[string]map[string]string `json:"patch_group_to_baseline"` - OpsItems map[string]map[string]OpsItem `json:"ops_items"` OpsItemRelatedItems map[string]map[string][]OpsItemRelatedItem `json:"ops_item_related_items"` - OpsMetadata map[string]map[string]OpsMetadata `json:"ops_metadata"` - PatchBaselines map[string]map[string]PatchBaseline `json:"patch_baselines"` Inventory map[string]map[string][]InventoryItem `json:"inventory"` Compliance map[string]map[string][]ComplianceItem `json:"compliance"` - ResourceDataSyncs map[string]map[string]*ResourceDataSync `json:"resource_data_syncs"` ParameterLabels map[string]map[string]map[int64][]string `json:"parameter_labels"` - AutomationExecutions map[string]map[string]*AutomationExecution `json:"automation_executions"` - ServiceSettings map[string]map[string]*ServiceSetting `json:"service_settings"` ResourcePolicies map[string]map[string][]*ResourcePolicy `json:"resource_policies"` - ExecutionPreviews map[string]map[string]*ExecutionPreview `json:"execution_previews"` MiscResourceTags map[string]map[string]map[string]string `json:"misc_resource_tags"` ResourceIDToOpsMetadataArn map[string]map[string]string `json:"resource_id_to_ops_metadata_arn"` OpsItemEvents map[string][]OpsItemEventSummary `json:"ops_item_events"` AssociationExecutions map[string]map[string][]AssociationExecution `json:"association_executions"` AssociationExecTargets map[string]map[string][]AssociationExecutionTarget `json:"association_exec_targets"` InventoryDeletions map[string][]InventoryDeletion `json:"inventory_deletions"` - InstancePatchStates map[string]map[string]*InstancePatchState `json:"instance_patch_states"` InstancePatches map[string]map[string][]PatchComplianceData `json:"instance_patches"` - InstanceProperties map[string]map[string]*InstanceProperty `json:"instance_properties"` AvailablePatches map[string][]Patch `json:"available_patches"` + Version int `json:"version"` } // initSnapshotDefaults initializes nil maps in the snapshot for core fields. func initSnapshotDefaults(snap *backendSnapshot) { - if snap.Parameters == nil { - snap.Parameters = make(map[string]map[string]Parameter) - } - if snap.History == nil { snap.History = make(map[string]map[string][]ParameterHistory) } @@ -63,10 +70,6 @@ func initSnapshotDefaults(snap *backendSnapshot) { snap.Tags = make(map[string]map[string]*tags.Tags) } - if snap.Documents == nil { - snap.Documents = make(map[string]map[string]Document) - } - if snap.DocumentVersions == nil { snap.DocumentVersions = make(map[string]map[string][]DocumentVersion) } @@ -75,61 +78,21 @@ func initSnapshotDefaults(snap *backendSnapshot) { snap.DocumentPermissions = make(map[string]map[string][]string) } - if snap.Commands == nil { - snap.Commands = make(map[string]map[string]Command) - } - if snap.CommandInvocations == nil { snap.CommandInvocations = make(map[string]map[string][]CommandInvocation) } } // initSnapshotNewFields initializes nil maps for newer resource types. -func initSnapshotNewFields(snap *backendSnapshot) { //nolint:gocognit,cyclop // existing issue. - if snap.Activations == nil { - snap.Activations = make(map[string]map[string]Activation) - } - - if snap.Associations == nil { - snap.Associations = make(map[string]map[string]Association) - } - - if snap.MaintenanceWindows == nil { - snap.MaintenanceWindows = make(map[string]map[string]MaintenanceWindow) - } - - if snap.MaintenanceWindowTargets == nil { - snap.MaintenanceWindowTargets = make(map[string]map[string]MaintenanceWindowTarget) - } - - if snap.MaintenanceWindowTasks == nil { - snap.MaintenanceWindowTasks = make(map[string]map[string]MaintenanceWindowTask) - } - - if snap.Sessions == nil { - snap.Sessions = make(map[string]map[string]Session) - } - +func initSnapshotNewFields(snap *backendSnapshot) { if snap.PatchGroupToBaseline == nil { snap.PatchGroupToBaseline = make(map[string]map[string]string) } - if snap.OpsItems == nil { - snap.OpsItems = make(map[string]map[string]OpsItem) - } - if snap.OpsItemRelatedItems == nil { snap.OpsItemRelatedItems = make(map[string]map[string][]OpsItemRelatedItem) } - if snap.OpsMetadata == nil { - snap.OpsMetadata = make(map[string]map[string]OpsMetadata) - } - - if snap.PatchBaselines == nil { - snap.PatchBaselines = make(map[string]map[string]PatchBaseline) - } - if snap.Inventory == nil { snap.Inventory = make(map[string]map[string][]InventoryItem) } @@ -138,30 +101,14 @@ func initSnapshotNewFields(snap *backendSnapshot) { //nolint:gocognit,cyclop // snap.Compliance = make(map[string]map[string][]ComplianceItem) } - if snap.ResourceDataSyncs == nil { - snap.ResourceDataSyncs = make(map[string]map[string]*ResourceDataSync) - } - if snap.ParameterLabels == nil { snap.ParameterLabels = make(map[string]map[string]map[int64][]string) } - if snap.AutomationExecutions == nil { - snap.AutomationExecutions = make(map[string]map[string]*AutomationExecution) - } - - if snap.ServiceSettings == nil { - snap.ServiceSettings = make(map[string]map[string]*ServiceSetting) - } - if snap.ResourcePolicies == nil { snap.ResourcePolicies = make(map[string]map[string][]*ResourcePolicy) } - if snap.ExecutionPreviews == nil { - snap.ExecutionPreviews = make(map[string]map[string]*ExecutionPreview) - } - if snap.MiscResourceTags == nil { snap.MiscResourceTags = make(map[string]map[string]map[string]string) } @@ -186,28 +133,19 @@ func initSnapshotNewFields(snap *backendSnapshot) { //nolint:gocognit,cyclop // } // initSnapshotPatchOpsFields initializes nil maps for inventory-deletion -// records and the patch-operation state (available-patches catalogue, -// instance patch states/compliance data, instance properties) written by -// SendCommand's AWS-RunPatchBaseline handling and -// DescribeAvailablePatches/DescribeInstanceProperties. Split out of +// records and the patch-operation state (available-patches catalogue and +// per-instance compliance data) written by SendCommand's +// AWS-RunPatchBaseline handling and DescribeAvailablePatches. Split out of // initSnapshotNewFields to keep it under the function-length limit. func initSnapshotPatchOpsFields(snap *backendSnapshot) { if snap.InventoryDeletions == nil { snap.InventoryDeletions = make(map[string][]InventoryDeletion) } - if snap.InstancePatchStates == nil { - snap.InstancePatchStates = make(map[string]map[string]*InstancePatchState) - } - if snap.InstancePatches == nil { snap.InstancePatches = make(map[string]map[string][]PatchComplianceData) } - if snap.InstanceProperties == nil { - snap.InstanceProperties = make(map[string]map[string]*InstanceProperty) - } - if snap.AvailablePatches == nil { snap.AvailablePatches = make(map[string][]Patch) } @@ -218,43 +156,38 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { b.mu.RLock("Snapshot") defer b.mu.RUnlock() + tables, err := b.registry.SnapshotAll() + if err != nil { + // The registered tables are plain JSON-friendly structs, so a marshal + // failure here would indicate a programming error rather than bad + // input data. Log and skip the snapshot rather than panic, matching + // the persistence.Persistable contract (nil is skipped by the Manager). + logger.Load(ctx).WarnContext(ctx, "ssm: snapshot table marshal failed", "error", err) + + return nil + } + snap := backendSnapshot{ - Parameters: b.parameters, + Version: ssmSnapshotVersion, + Tables: tables, History: b.history, Tags: b.tags, - Documents: b.documents, DocumentVersions: b.documentVersions, DocumentPermissions: b.documentPermissions, - Commands: b.commands, CommandInvocations: b.commandInvocations, - Activations: b.activations, - Associations: b.associations, - MaintenanceWindows: b.maintenanceWindows, - MaintenanceWindowTargets: b.maintenanceWindowTargets, - MaintenanceWindowTasks: b.maintenanceWindowTasks, - Sessions: b.sessions, PatchGroupToBaseline: b.patchGroupToBaseline, - OpsItems: b.opsItems, OpsItemRelatedItems: b.opsItemRelatedItems, - OpsMetadata: b.opsMetadata, - PatchBaselines: b.patchBaselines, Inventory: b.inventory, Compliance: b.compliance, - ResourceDataSyncs: b.resourceDataSyncs, ParameterLabels: b.parameterLabels, - AutomationExecutions: b.automationExecutions, - ServiceSettings: b.serviceSettings, ResourcePolicies: b.resourcePolicies, - ExecutionPreviews: b.executionPreviews, MiscResourceTags: b.miscResourceTags, ResourceIDToOpsMetadataArn: b.resourceIDToOpsMetadataArn, OpsItemEvents: b.opsItemEvents, AssociationExecutions: b.associationExecutions, AssociationExecTargets: b.associationExecTargets, InventoryDeletions: b.inventoryDeletions, - InstancePatchStates: b.instancePatchStates, InstancePatches: b.instancePatches, - InstanceProperties: b.instanceProperties, AvailablePatches: b.availablePatches, } @@ -268,6 +201,28 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { return data } +// preRegisterSnapshotTables ensures every (resource, region) pair present in +// tables is registered on b.registry before registry.RestoreAll is called. +// RestoreAll only restores tables already registered (see +// [github.com/blackbirdworks/gopherstack/pkgs/store.Registry.RestoreAll]); +// a region a fresh backend has never touched would otherwise not yet have a +// table registered under that name, and its persisted data would be +// silently dropped instead of restored. Each table name has the shape +// "/" (see store_setup.go's getOrCreateTable); regions +// never contain "/", so splitting on the first one recovers both parts. +func (b *InMemoryBackend) preRegisterSnapshotTables(tables map[string]json.RawMessage) { + for key := range tables { + name, region, found := strings.Cut(key, "/") + if !found { + continue + } + + if register, ok := tableAccessorsByPrefix[name]; ok { + register(b, region) + } + } +} + // Restore loads backend state from a JSON snapshot. func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { var snap backendSnapshot @@ -279,6 +234,34 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { b.mu.Lock("Restore") defer b.mu.Unlock() + if snap.Version != ssmSnapshotVersion { + // An incompatible (older/newer/absent) snapshot version must never be + // partially decoded as the current shape -- that risks silently + // misinterpreting fields. Discard cleanly and start empty instead of + // erroring, since this is an expected, recoverable condition (e.g. + // upgrading gopherstack across a snapshot-format change), not data + // corruption. Mirrors the services/sqs pilot (commit 0f09d77c) and + // the services/ec2 conversion (commit 12e611a4). + logger.Load(ctx).WarnContext(ctx, + "ssm: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", ssmSnapshotVersion) + + b.registry.ResetAll() + + // Match NewInMemoryBackend's construction-time state: a fresh backend + // is never truly empty of documents, so "starting empty" here means + // starting exactly as fresh as a new backend would, not one that + // lacks even the built-in AWS-* documents every backend is seeded + // with. This preserves pre-existing Restore behavior for legacy + // snapshots taken before this backend supported documents at all + // (they have no "documents" data to discard, only a version + // mismatch), which relied on the default documents always being + // present after Restore. + b.registerDefaultDocuments(defaultRegion) + + return nil + } + initSnapshotDefaults(&snap) initSnapshotNewFields(&snap) @@ -288,46 +271,37 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { } } - b.parameters = snap.Parameters + b.preRegisterSnapshotTables(snap.Tables) + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("ssm: restore snapshot tables: %w", err) + } + b.history = snap.History b.tags = snap.Tags - b.documents = snap.Documents b.documentVersions = snap.DocumentVersions b.documentPermissions = snap.DocumentPermissions - b.commands = snap.Commands b.commandInvocations = snap.CommandInvocations - b.activations = snap.Activations - b.associations = snap.Associations - b.maintenanceWindows = snap.MaintenanceWindows - b.maintenanceWindowTargets = snap.MaintenanceWindowTargets - b.maintenanceWindowTasks = snap.MaintenanceWindowTasks - b.sessions = snap.Sessions b.patchGroupToBaseline = snap.PatchGroupToBaseline - b.opsItems = snap.OpsItems b.opsItemRelatedItems = snap.OpsItemRelatedItems - b.opsMetadata = snap.OpsMetadata - b.patchBaselines = snap.PatchBaselines b.inventory = snap.Inventory b.compliance = snap.Compliance - b.resourceDataSyncs = snap.ResourceDataSyncs b.parameterLabels = snap.ParameterLabels - b.automationExecutions = snap.AutomationExecutions - b.serviceSettings = snap.ServiceSettings b.resourcePolicies = snap.ResourcePolicies - b.executionPreviews = snap.ExecutionPreviews b.miscResourceTags = snap.MiscResourceTags b.resourceIDToOpsMetadataArn = snap.ResourceIDToOpsMetadataArn b.opsItemEvents = snap.OpsItemEvents b.associationExecutions = snap.AssociationExecutions b.associationExecTargets = snap.AssociationExecTargets b.inventoryDeletions = snap.InventoryDeletions - b.restorePatchOpsFields(&snap) + b.instancePatches = snap.InstancePatches + b.availablePatches = snap.AvailablePatches // Re-seed built-in documents if they are absent from the snapshot // (e.g. snapshots taken before document support was added). for _, region := range []string{defaultRegion} { for _, name := range []string{"AWS-RunShellScript", "AWS-RunPowerShellScript"} { - if b.documents[region] == nil || b.documents[region][name].Name == "" { + if !b.documentsStore(region).Has(name) { b.registerDefaultDocuments(region) break @@ -338,17 +312,6 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { return nil } -// restorePatchOpsFields assigns the patch-operation state (available-patches -// catalogue, instance patch states/compliance data, instance properties) from -// a snapshot. Split out of Restore to keep it under the function-length limit. -// Must be called with b.mu held for writing. -func (b *InMemoryBackend) restorePatchOpsFields(snap *backendSnapshot) { - b.instancePatchStates = snap.InstancePatchStates - b.instancePatches = snap.InstancePatches - b.instanceProperties = snap.InstanceProperties - b.availablePatches = snap.AvailablePatches -} - // Snapshot implements persistence.Persistable by delegating to the backend. func (h *Handler) Snapshot(ctx context.Context) []byte { type snapshotter interface { diff --git a/services/ssm/persistence_test.go b/services/ssm/persistence_test.go index 286802870..8c8ba64c2 100644 --- a/services/ssm/persistence_test.go +++ b/services/ssm/persistence_test.go @@ -111,3 +111,185 @@ func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { err := b.Restore(t.Context(), []byte("not-valid-json")) require.Error(t, err) } + +// TestInMemoryBackend_FullStateSnapshotRestore seeds one instance of every +// resource collection converted to *store.Table in Phase 3.3 (see +// store_setup.go), plus a sampling of the resource collections deliberately +// left as raw maps, then round-trips the whole backend through +// Snapshot/Restore and verifies every seeded resource is independently +// readable afterward via its normal (region-scoped) read path. This is the +// persistence-safety regression test the conversion calls for: a table that +// was accidentally left off the registry, or a raw field dropped from +// backendSnapshot, shows up here as a specific missing resource rather than +// only surfacing much later against real traffic. +func TestInMemoryBackend_FullStateSnapshotRestore(t *testing.T) { + t.Parallel() + + ctx := t.Context() + b := ssm.NewInMemoryBackend() + + // --- store.Table-backed resources (18) --- + + _, err := b.PutParameter(ctx, &ssm.PutParameterInput{ + Name: "/full/param", Value: "v", Type: "String", + }) + require.NoError(t, err) + + _, err = b.CreateDocument(ctx, &ssm.CreateDocumentInput{Name: "FullDoc", Content: "{}"}) + require.NoError(t, err) + + sendOut, err := b.SendCommand(ctx, &ssm.SendCommandInput{ + DocumentName: "AWS-RunShellScript", InstanceIDs: []string{"i-full"}, + }) + require.NoError(t, err) + commandID := sendOut.Command.CommandID + + b.AddActivationInternal(ssm.Activation{ActivationID: "act-full", IamRole: "r"}) + b.AddAssociationInternal(ssm.Association{AssociationID: "assoc-full", Name: "FullDoc"}) + b.AddMaintenanceWindowInternal(ssm.MaintenanceWindow{WindowID: "mw-full", Name: "w"}) + b.AddOpsItemInternal(ssm.OpsItem{OpsItemID: "oi-full", Title: "t", Source: "s"}) + b.AddOpsMetadataInternal(ssm.OpsMetadata{OpsMetadataArn: "arn:full", ResourceID: "res-full"}) + b.AddPatchBaselineInternal(ssm.PatchBaseline{BaselineID: "pb-full", Name: "b"}) + b.AddInstancePatchStateInternal(ssm.InstancePatchState{InstanceID: "ips-full"}) + b.AddInstancePropertyInternal(ssm.InstanceProperty{InstanceID: "ip-full"}) + b.AddTerminatedSessionInternal("sess-full", 1000) + + _, err = b.CreateResourceDataSync(ctx, &ssm.CreateResourceDataSyncInput{SyncName: "sync-full"}) + require.NoError(t, err) + + startAutoOut, err := b.StartAutomationExecution(ctx, &ssm.StartAutomationExecutionInput{ + DocumentName: "AWS-RunShellScript", + }) + require.NoError(t, err) + automationExecID := startAutoOut.AutomationExecutionID + + _, err = b.UpdateServiceSetting(ctx, &ssm.UpdateServiceSettingInput{ + SettingID: "/ssm/full-setting", SettingValue: "on", + }) + require.NoError(t, err) + + previewOut, err := b.StartExecutionPreview(ctx, &ssm.StartExecutionPreviewInput{ + DocumentName: "AWS-RunShellScript", + }) + require.NoError(t, err) + previewID := previewOut.ExecutionPreviewID + + // --- a sampling of resources deliberately left as raw maps --- + + err = b.AddTagsToResource(ctx, &ssm.AddTagsToResourceInput{ + ResourceType: "Parameter", ResourceID: "/full/param", + Tags: []ssm.Tag{{Key: "K", Value: "V"}}, + }) + require.NoError(t, err) + + _, err = b.PutInventory(ctx, &ssm.PutInventoryInput{ + InstanceID: "i-full-inv", + Items: []ssm.InventoryItem{{ + TypeName: "AWS:Application", SchemaVersion: "1.0", CaptureTime: "2024-01-01T00:00:00Z", + }}, + }) + require.NoError(t, err) + + // --- round-trip --- + + snap := b.Snapshot(ctx) + require.NotNil(t, snap) + + fresh := ssm.NewInMemoryBackend() + require.NoError(t, fresh.Restore(ctx, snap)) + + // --- verify every seeded resource survived --- + + paramOut, err := fresh.GetParameter(ctx, &ssm.GetParameterInput{Name: "/full/param"}) + require.NoError(t, err) + assert.Equal(t, "v", paramOut.Parameter.Value) + + docOut, err := fresh.GetDocument(ctx, &ssm.GetDocumentInput{Name: "FullDoc"}) + require.NoError(t, err) + assert.Equal(t, "{}", docOut.Content) + + cmdOut, err := fresh.ListCommands(ctx, &ssm.ListCommandsInput{CommandID: commandID}) + require.NoError(t, err) + require.Len(t, cmdOut.Commands, 1) + + actOut, err := fresh.DescribeActivations(ctx, &ssm.DescribeActivationsInput{}) + require.NoError(t, err) + assert.True(t, containsActivation(actOut.ActivationList, "act-full")) + + assocOut, err := fresh.DescribeAssociation(ctx, &ssm.DescribeAssociationInput{AssociationID: "assoc-full"}) + require.NoError(t, err) + assert.Equal(t, "assoc-full", assocOut.AssociationDescription.AssociationID) + + mwOut, err := fresh.GetMaintenanceWindow(ctx, &ssm.GetMaintenanceWindowInput{WindowID: "mw-full"}) + require.NoError(t, err) + assert.Equal(t, "mw-full", mwOut.MaintenanceWindow.WindowID) + + opsItemOut, err := fresh.GetOpsItem(ctx, &ssm.GetOpsItemInput{OpsItemID: "oi-full"}) + require.NoError(t, err) + assert.Equal(t, "oi-full", opsItemOut.OpsItem.OpsItemID) + + opsMetaOut, err := fresh.GetOpsMetadata(ctx, &ssm.GetOpsMetadataInput{OpsMetadataArn: "arn:full"}) + require.NoError(t, err) + assert.Equal(t, "res-full", opsMetaOut.ResourceID) + + blOut, err := fresh.GetPatchBaseline(ctx, &ssm.GetPatchBaselineInput{BaselineID: "pb-full"}) + require.NoError(t, err) + assert.Equal(t, "pb-full", blOut.BaselineID) + + ipsOut, err := fresh.DescribeInstancePatchStates( + ctx, &ssm.DescribeInstancePatchStatesInput{InstanceIDs: []string{"ips-full"}}, + ) + require.NoError(t, err) + require.Len(t, ipsOut.InstancePatchStates, 1) + + syncOut, err := fresh.ListResourceDataSync(ctx, &ssm.ListResourceDataSyncInput{}) + require.NoError(t, err) + assert.True(t, containsSyncName(syncOut.ResourceDataSyncItems, "sync-full")) + + autoOut, err := fresh.GetAutomationExecution( + ctx, &ssm.GetAutomationExecutionInput{AutomationExecutionID: automationExecID}, + ) + require.NoError(t, err) + assert.Equal(t, automationExecID, autoOut.AutomationExecution.AutomationExecutionID) + + settingOut, err := fresh.GetServiceSetting(ctx, &ssm.GetServiceSettingInput{SettingID: "/ssm/full-setting"}) + require.NoError(t, err) + assert.Equal(t, "on", settingOut.ServiceSetting.SettingValue) + + previewOut2, err := fresh.GetExecutionPreview( + ctx, &ssm.GetExecutionPreviewInput{ExecutionPreviewID: previewID}, + ) + require.NoError(t, err) + assert.Equal(t, previewID, previewOut2.ExecutionPreviewID) + + // raw-map resources: tags, inventory + tagsOut, err := fresh.ListTagsForResource( + ctx, &ssm.ListTagsForResourceInput{ResourceType: "Parameter", ResourceID: "/full/param"}, + ) + require.NoError(t, err) + assert.NotEmpty(t, tagsOut.TagList) + + invOut, err := fresh.GetInventory(ctx, &ssm.GetInventoryInput{}) + require.NoError(t, err) + assert.NotEmpty(t, invOut.Entities) +} + +func containsActivation(list []ssm.Activation, id string) bool { + for _, a := range list { + if a.ActivationID == id { + return true + } + } + + return false +} + +func containsSyncName(list []ssm.ResourceDataSync, name string) bool { + for _, s := range list { + if s.SyncName == name { + return true + } + } + + return false +} diff --git a/services/ssm/store_setup.go b/services/ssm/store_setup.go new file mode 100644 index 000000000..3428ec576 --- /dev/null +++ b/services/ssm/store_setup.go @@ -0,0 +1,139 @@ +package ssm + +// Code in this file supports Phase 3.3 of the datalayer refactor: resource +// maps with a pure per-value identity (a Name/ID field of their own) are +// backed by *store.Table[T] instead of a hand-rolled map[string]T / +// map[string]*T. See pkgs/store's package doc and the services/ec2 (commit +// 12e611a4) and services/sqs (commit 0f09d77c) conversions this follows. +// +// # Why this looks different from ec2/sqs +// +// ec2 and sqs each model ONE region per InMemoryBackend instance (a single +// b.Region field), so a flat map[string]*store.Table[T] statically +// registered once at construction time is enough. ssm's InMemoryBackend is +// unusual: it holds EVERY region's state in one instance, via a second map +// layer keyed by region (region -> name -> value). store.Registry has no +// concept of "region" and requires every name registered on it to be known +// and unique up front — it cannot itself express "one table per region, +// regions unknown until first use". +// +// This file bridges the two: each convertible resource keeps its existing +// map[string]*store.Table[T] (region -> Table), created lazily by +// getOrCreateTable on first access to that region — exactly mirroring the +// lazy per-region map creation the hand-rolled code did before conversion +// (e.g. the old `if b.parameters[region] == nil { b.parameters[region] = +// make(map[string]Parameter) }` guards, now folded into the accessor). The +// first time a region's table is created it is ALSO registered on +// b.registry under "/" so that persistence.go can +// still drive Snapshot/Restore through one *store.Registry +// (registry.SnapshotAll/RestoreAll), regardless of how many regions exist — +// see tableAccessorsByPrefix and persistence.go's Restore. +// +// # What is NOT converted here, and why +// +// The following resource fields are deliberately left as plain (possibly +// nested) maps, matching the ec2/sqs precedent of leaving non-pure-key-fn or +// one-to-many collections raw: +// - history, documentVersions, commandInvocations, opsItemRelatedItems, +// inventory, compliance, resourcePolicies, instancePatches, +// associationExecutions, associationExecTargets: one-to-many +// (map[string][]V) — there is no single V to key a Table by; these stay +// slice-valued maps, same as ec2's spotFleetHistory/subnetCIDRAssociations +// pattern. +// - documentPermissions ([]string), patchGroupToBaseline (string), +// parameterLabels (map[int64][]string), miscResourceTags +// (map[string]string), resourceIDToOpsMetadataArn (string): the stored +// value carries no identity field of its own (a bare string/slice/map), +// so there is no pure func(*V) string to write — same rationale as ec2's +// instanceIMDSOptions/verifiedAccessEndpointPolicies exclusions. +// - opsItemEvents, availablePatches, inventoryDeletions: single-level +// region -> []V logs/catalogues, not a keyed collection at all. +// - tags (map[string]*tags.Tags): the map key IS the external resource ID, +// but tags.Tags itself carries no identity field naming that resource, +// and instances require an explicit Close() on Restore (see +// persistence.go) — not a fit for Table's generic JSON round-trip. +import ( + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +func parameterKeyFn(v *Parameter) string { return v.Name } +func documentKeyFn(v *Document) string { return v.Name } +func commandKeyFn(v *Command) string { return v.CommandID } +func activationKeyFn(v *Activation) string { return v.ActivationID } +func associationKeyFn(v *Association) string { return v.AssociationID } +func maintenanceWindowKeyFn(v *MaintenanceWindow) string { return v.WindowID } +func maintenanceWindowTargetKeyFn(v *MaintenanceWindowTarget) string { + return v.WindowTargetID +} +func maintenanceWindowTaskKeyFn(v *MaintenanceWindowTask) string { return v.WindowTaskID } +func sessionKeyFn(v *Session) string { return v.SessionID } +func opsItemKeyFn(v *OpsItem) string { return v.OpsItemID } +func opsMetadataKeyFn(v *OpsMetadata) string { return v.OpsMetadataArn } +func patchBaselineKeyFn(v *PatchBaseline) string { return v.BaselineID } +func resourceDataSyncKeyFn(v *ResourceDataSync) string { return v.SyncName } +func automationExecutionKeyFn(v *AutomationExecution) string { + return v.AutomationExecutionID +} +func serviceSettingKeyFn(v *ServiceSetting) string { return v.SettingID } +func executionPreviewKeyFn(v *ExecutionPreview) string { return v.ExecutionPreviewID } +func instancePatchStateKeyFn(v *InstancePatchState) string { return v.InstanceID } +func instancePropertyKeyFn(v *InstanceProperty) string { return v.InstanceID } + +// getOrCreateTable returns the *store.Table[V] for region from m, the +// backend's region->Table field for one resource type, creating and +// registering it on b.registry (under name+"/"+region) the first time that +// region is touched. m is a Go map (reference type), so inserting into it +// here mutates the same map the caller's field refers to. +func getOrCreateTable[V any]( + b *InMemoryBackend, + m map[string]*store.Table[V], + name, region string, + keyFn func(*V) string, +) *store.Table[V] { + t, ok := m[region] + if !ok { + t = store.Register(b.registry, name+"/"+region, store.New(keyFn)) + m[region] = t + } + + return t +} + +// tableAccessorsByPrefix maps each convertible resource's registry-name +// prefix to a no-op-return accessor call that forces getOrCreateTable to +// register that resource's table for a given region. persistence.go's +// Restore uses this to pre-register every (resource, region) pair present in +// an incoming snapshot's Tables blob BEFORE calling b.registry.RestoreAll — +// RestoreAll only restores tables already registered on b.registry, so a +// region a fresh backend has never seen must be registered first or its +// data would be silently dropped. +// +//nolint:gochecknoglobals // registration table, analogous to errCodeLookup-style lookup tables elsewhere +var tableAccessorsByPrefix = map[string]func(b *InMemoryBackend, region string){ + "parameters": func(b *InMemoryBackend, region string) { b.parametersStore(region) }, + "documents": func(b *InMemoryBackend, region string) { b.documentsStore(region) }, + "commands": func(b *InMemoryBackend, region string) { b.commandsStore(region) }, + "activations": func(b *InMemoryBackend, region string) { b.activationsStore(region) }, + "associations": func(b *InMemoryBackend, region string) { b.associationsStore(region) }, + "maintenanceWindows": func(b *InMemoryBackend, region string) { b.maintenanceWindowsStore(region) }, + "maintenanceWindowTargets": func(b *InMemoryBackend, region string) { + b.maintenanceWindowTargetsStore(region) + }, + "maintenanceWindowTasks": func(b *InMemoryBackend, region string) { + b.maintenanceWindowTasksStore(region) + }, + "sessions": func(b *InMemoryBackend, region string) { b.sessionsStore(region) }, + "opsItems": func(b *InMemoryBackend, region string) { b.opsItemsStore(region) }, + "opsMetadata": func(b *InMemoryBackend, region string) { b.opsMetadataStore(region) }, + "patchBaselines": func(b *InMemoryBackend, region string) { b.patchBaselinesStore(region) }, + "resourceDataSyncs": func(b *InMemoryBackend, region string) { b.resourceDataSyncsStore(region) }, + "automationExecutions": func(b *InMemoryBackend, region string) { + b.automationExecutionsStore(region) + }, + "serviceSettings": func(b *InMemoryBackend, region string) { b.serviceSettingsStore(region) }, + "executionPreviews": func(b *InMemoryBackend, region string) { b.executionPreviewsStore(region) }, + "instancePatchStates": func(b *InMemoryBackend, region string) { + b.instancePatchStatesStore(region) + }, + "instanceProperties": func(b *InMemoryBackend, region string) { b.instancePropertiesStore(region) }, +} diff --git a/services/stepfunctions/PARITY.md b/services/stepfunctions/PARITY.md new file mode 100644 index 000000000..ea842e4b7 --- /dev/null +++ b/services/stepfunctions/PARITY.md @@ -0,0 +1,283 @@ +--- +service: stepfunctions +sdk_module: aws-sdk-go-v2/service/sfn@v1.40.8 +last_audit_commit: e5a9ac69 +last_audit_date: 2026-07-05 +overall: A # ~1150 LOC genuine fixes this pass (see gaps/notes); the rest is op-by-op proof (B) +ops: + CreateStateMachine: {wire: ok, errors: ok, state: ok, persist: ok, note: "STANDARD/EXPRESS, roleArn validation, tags, logging/tracing config; unchanged this pass"} + UpdateStateMachine: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteStateMachine: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeStateMachine: {wire: ok, errors: ok, state: ok, persist: ok} + ListStateMachines: {wire: ok, errors: ok, state: ok, persist: ok, note: "page.Page[T] pagination"} + DescribeStateMachineForExecution: {wire: ok, errors: ok, state: ok, persist: ok} + PublishStateMachineVersion: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeStateMachineVersion: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteStateMachineVersion: {wire: ok, errors: ok, state: ok, persist: ok} + ListStateMachineVersions: {wire: ok, errors: ok, state: ok, persist: ok} + CreateStateMachineAlias: {wire: ok, errors: ok, state: ok, persist: ok, note: "routingConfiguration weighted versions validated"} + UpdateStateMachineAlias: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteStateMachineAlias: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeStateMachineAlias: {wire: ok, errors: ok, state: ok, persist: ok} + ListStateMachineAliases: {wire: ok, errors: ok, state: ok, persist: ok} + TagResource: {wire: ok, errors: ok, state: ok, persist: ok} + UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} + ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok} + ValidateStateMachineDefinition: {wire: ok, errors: ok, state: ok, persist: n/a, note: "JSON/structural validation only, no deep ASL semantic checks (e.g. JitterStrategy enum, ToleratedFailure+INLINE combos) -- see gaps"} + StartExecution: + wire: ok + errors: fixed + state: ok + persist: ok + note: > + FIXED this pass: StartExecution on an EXPRESS state machine was + incorrectly rejected with InvalidExecutionType. AWS supports + asynchronous "Express Workflows" via StartExecution for EITHER type; + only StartSyncExecution is EXPRESS-only. Removed the incorrect check + (backend.go). ClientRequestToken idempotency and EXPRESS's + immediate-name-reuse semantics are not modeled either way (gap, + bd: gopherstack-1sf). + StartSyncExecution: + wire: ok + errors: fixed + state: ok + persist: ok + note: > + FIXED this pass: calling StartSyncExecution on a STANDARD state + machine returned "InvalidExecutionType"; AWS returns + "StateMachineTypeNotSupported". Added ErrStateMachineTypeNotSupported + and rewired backend.go + handler.go's error-code table. + StopExecution: {wire: ok, errors: ok, state: ok, persist: ok, note: "cancels the execution's context via cancelFns; goroutine exits promptly"} + RedriveExecution: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeExecution: {wire: ok, errors: ok, state: ok, persist: ok} + ListExecutions: {wire: ok, errors: ok, state: ok, persist: ok} + GetExecutionHistory: + wire: fixed + errors: ok + state: ok + persist: ok + note: > + FIXED this pass (severe, broad): StateEnteredEventDetails.Input, + StateExitedEventDetails.Output, TaskScheduledEventDetails.Resource, + TaskSucceededEventDetails.Output, and TaskFailedEventDetails.Error/Cause + were ALL parsed into well-shaped Go structs (json tags already + correct) but the historyRecorder methods in backend.go silently + discarded every parameter and only ever wrote {Type, Timestamp} -- + every Task/state history event was an empty shell. Event + types/ordering/IDs/pagination were already correct (that's all prior + tests checked, which is why this went undetected). Added + TaskScheduledEventDetails/TaskSucceededEventDetails/ + TaskFailedEventDetails structs+population; still missing + resourceType/timeoutInSeconds/heartbeatInSeconds/outputDetails and + TaskSubmitted/TaskStarted events (bd: gopherstack-996). + CreateActivity: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteActivity: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeActivity: {wire: ok, errors: ok, state: ok, persist: ok} + ListActivities: {wire: ok, errors: ok, state: ok, persist: ok} + GetActivityTask: {wire: ok, errors: ok, state: ok, persist: ok, note: "long-poll with WaitTimeSeconds; task-token issuance"} + SendTaskSuccess: {wire: ok, errors: ok, state: ok, persist: ok} + SendTaskFailure: {wire: ok, errors: ok, state: ok, persist: ok} + SendTaskHeartbeat: {wire: ok, errors: ok, state: ok, persist: ok, note: "States.HeartbeatTimeout enforced against HeartbeatSeconds"} + DescribeMapRun: {wire: ok, errors: ok, state: ok, persist: ok} + ListMapRuns: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateMapRun: {wire: ok, errors: ok, state: ok, persist: ok, note: "ToleratedFailureCount/Percentage on the MapRun *resource* API were already real; the ASL-definition-level Map state fields were the gap (fixed, see families.asl_map)"} + TestState: {wire: ok, errors: ok, state: ok, persist: n/a} +families: + asl_task: + status: ok + note: > + Resource ARN resolution (Lambda/SQS/SNS/DynamoDB/ECS/Glue/EventBridge/ + Activity), Parameters/ResultSelector/ResultPath/InputPath/OutputPath, + TimeoutSeconds (context.WithTimeout), HeartbeatSeconds, + .waitForTaskToken and .sync/.sync:2 patterns all verified against + aws-sdk-go-v2 sfn behavior and already correct. Retry + (MaxAttempts/IntervalSeconds/BackoffRate default 3/1s/2.0 -- matches + AWS defaults) and Catch (ErrorEquals incl. States.ALL/Timeout/Runtime/ + Permissions/TaskFailed) were already correct for Task. + FIXED this pass: Retry.MaxDelaySeconds and Retry.JitterStrategy + ("FULL"/"NONE", AWS default NONE) were not parsed or applied at all -- + only an internal 24h safety cap existed. Added both fields to + asl.Retrier and applyRetryDelayCapAndJitter(). + FIXED this pass: the Catch error-output object only ever set a single + combined "Error": ": " key; AWS's documented shape is + separate "Error" and "Cause" keys. Also, TaskFailed history events + always recorded errCode= and cause="" (see + GetExecutionHistory finding). Fixed via new + stepFunctionsErrorCause() + reworked checkCatchers()/recordTaskFailed(). + asl_choice: + status: ok + note: > + All comparison families verified: String/Numeric/Timestamp/Boolean + Equals+LessThan+GreaterThan(+Equals) and their *Path variants, + And/Or/Not, IsPresent/IsNull/IsString/IsNumeric/IsBoolean/IsTimestamp, + StringMatches (custom glob matcher handles '*', literal-escaping via + backslash, backtracking for multiple wildcards -- verified against the + AWS doc example "log-*.txt"). No changes needed. + asl_map: + status: fixed + note: > + SEVERE FIX: runMapItem only checked the Go `error` return of the + per-item sub-Executor.Execute() call. But Execute() returns a Fail + state (or an unhandled Task failure inside the iterator) as + `(&ExecutionResult{Error: code, Cause: cause}, nil)` -- a *successful* + Go call with res.Error populated, NOT a Go error. So EVERY failing Map + iteration was silently swallowed: errs[idx] stayed nil, results[idx] + stayed nil, and the Map state always "succeeded" with nil holes in its + output array. Verified runParallelBranch already had the correct + `if res.Error != "" { errs[idx] = &FailError{...} }` check -- + runMapItem was missing the equivalent, asymmetric disguised-stub bug. + Fixed by mirroring runParallelBranch's handling. + ADDED (previously entirely absent): Map-level Retry and Catch (AWS + supports Retry/Catch directly on Map/Parallel, not just Task; a retry + re-runs every item from scratch). Extracted a shared + executeWithStateRetryAndCatch() helper used by both executeMap and + executeParallel. + ADDED (previously entirely absent): ToleratedFailureCount/Percentage + and their *Path variants on the Map state definition (AWS applies + these to Distributed Map; the emulator applies them uniformly since + Map processing mode is not otherwise distinguished -- see + bd: gopherstack-8im). When both a count and percentage threshold are + configured, the Map fails when either is crossed, matching AWS. On + threshold-exceeded, fails with States.ExceedToleratedFailureThreshold; + with no tolerance configured (the common/default case, threshold=0), + the original per-item error is preserved unwrapped so existing + ErrorEquals matching on Catch/Retry keeps working exactly as before. + ItemsPath, MaxConcurrency, ItemBatcher (MaxItemsPerBatch/ + MaxInputBytesPerBatch), ItemReader (S3 CSV/JSON/JSONL via S3Reader), + ItemSelector were already correct and unchanged. + DEFERRED: ResultWriter (S3 write-out for Distributed Map) is not + parsed at all -- results are always returned inline. Implementing this + needs a new S3Writer integration wired from cli.go, outside this + pass's services/stepfunctions/-only scope (bd: gopherstack-8j8). + DEFERRED: ItemProcessor.ProcessorConfig.Mode (INLINE/DISTRIBUTED) is + not parsed, so the emulator can't reject INLINE+ToleratedFailure + combos the way AWS's definition validation does (bd: gopherstack-8im). + asl_parallel: + status: fixed + note: > + ADDED (previously entirely absent): Parallel-level Retry, via the same + executeWithStateRetryAndCatch() helper added for Map (Catch already + existed). Also fixed a latent bug where executeParallel hardcoded + "Parallel" as the stateName passed to checkCatchers/history recording + instead of the actual state's name from the ASL definition (threaded + the real stateName through executeState -> executeParallel). + Branch result aggregation, error propagation (first branch error wins, + after ctx.Err() check), and per-branch FailError reconstruction + (runParallelBranch) were already correct. + asl_wait: + status: ok + note: "Seconds/Timestamp/SecondsPath/TimestampPath all verified; waitForDuration respects ctx cancellation promptly (no goroutine leak). No changes." + asl_pass_succeed_fail: + status: ok + note: "Pass (Result/Parameters), Succeed, Fail (Error/Cause, static only -- no intrinsic in Error/Cause per AWS spec) all verified correct." + asl_intrinsics: + status: ok + note: > + All 18 real AWS intrinsics verified present and correct: States.Format, + StringToJson, JsonToString, Array, ArrayPartition, ArrayContains, + ArrayRange, ArrayGetItem, ArrayLength, ArrayUnique, Base64Encode, + Base64Decode, Hash, JsonMerge (shallow-only, correctly rejects + deep-merge arg per AWS's "third arg must be false" restriction), + MathRandom, MathAdd, StringSplit, UUID. + NOTE (non-AWS extras, informational only): the package also implements + non-standard/invented intrinsics (StringConcat, StringLength, + StringToLower/Upper, StringIndex, ArraySlice, ArrayFlatten, + ArrayReverse, ArraySort, MathSubtract/Multiply/Divide/Mod/Min/Max, + MathMax) that AWS does NOT support. This is permissive (accepts more + than AWS would) rather than a correctness bug for real AWS + definitions, but a definition using these against a real AWS account + would fail at validation time where the emulator accepts it silently. + Not fixed this pass (removing working functionality is net-negative; + flagging for awareness only). + json_1_0_protocol: + status: ok + note: "AWSStepFunctions. X-Amz-Target headers, json content-type, error shapes (__type + message) verified consistent with other json-1.0 services in this codebase. No changes." +gaps: + - "Map Distributed Map ResultWriter (S3 write-out) not implemented -- needs new S3Writer integration wired from cli.go (bd: gopherstack-8j8)" + - "Map ItemProcessor.ProcessorConfig.Mode (INLINE/DISTRIBUTED) not parsed/validated (bd: gopherstack-8im)" + - "Retry.JitterStrategy accepts any string; only \"FULL\" is special-cased, invalid values silently behave as NONE instead of ValidationException at Create/UpdateStateMachine (bd: gopherstack-xtl)" + - "StartExecution has no ClientRequestToken idempotency; EXPRESS's immediate-name-reuse semantics (vs STANDARD's reuse restriction) are not modeled (bd: gopherstack-1sf)" + - "TaskScheduledEventDetails/TaskSucceededEventDetails still omit resourceType/region/parameters/timeoutInSeconds/heartbeatInSeconds/outputDetails.truncated; no TaskSubmitted/TaskStarted history events for .sync/.waitForTaskToken (bd: gopherstack-996)" + - "Non-standard intrinsic functions (StringConcat, ArraySlice, MathSubtract, etc.) are accepted by this emulator but do not exist in real AWS Step Functions -- permissive superset, not a correctness bug against valid AWS definitions, but a definition that only works here would fail on real AWS (no bd filed; informational)" +deferred: + - "State machine CRUD (Create/Update/Delete/Describe/List, versions/aliases, logging/tracing config) -- spot-checked only; last deep audit was PRs #1937/#1742/#2110 (batch1/batch2 audits) and appeared unchanged/correct" + - "Activities (CreateActivity/GetActivityTask/SendTaskSuccess/Failure/Heartbeat) -- spot-checked only, appeared correct" + - "Distributed Map ItemReader S3 CSV/JSON/JSONL decoding -- spot-checked only (unchanged), appeared correct" +leaks: {status: clean, note: "StopExecution/DeleteStateMachine cancel the execution's context via b.cancelFns; Wait/waitForRetry/execSem/semaphore all select on ctx.Done(); Map/Parallel goroutines (wg.Go) all respect ctx cancellation. No new goroutines introduced this pass beyond the existing patterns (executeWithStateRetryAndCatch runs synchronously in the calling goroutine, not a new one)."} +--- + +## Notes + +**The big one this pass**: `runMapItem` (asl/executor.go) checked only the Go +`error` return of the per-iteration sub-executor's `Execute()` call. But +`Execute()` deliberately returns Fail-state/unhandled-Task failures as a +*successful* call — `(&ExecutionResult{Error: code, Cause: cause}, nil)` — so +that the top-level `Execute()` caller can distinguish "the state machine +executed successfully and *produced* a FAILED status" from "the Go call +itself errored" (e.g. bad JSON). `runParallelBranch` already had the correct +`if res.Error != "" { errs[idx] = &FailError{...} }` check for this; only the +Map path was missing it. This meant every Map state with a failing branch +silently succeeded with a `nil` hole in its output array instead of failing +the whole state (and thus never triggered any Catch/Retry, whether or not one +was even defined at the Map level — moot, since Map didn't support state-level +Catch/Retry at all before this pass either). **Trap for the next auditor**: +whenever a state type builds a sub-`Executor` and calls `.Execute()`, always +check `res.Error` in addition to the Go `error` — the two are orthogonal +result channels, and only checking one is what makes this bug so easy to miss +in review (the code "looks complete"). + +**AWS's Catch/Retry belong on Map and Parallel too, not just Task.** Before +this pass, `executeParallel` had ad-hoc Catch handling and `executeMap` had +none at all; neither had Retry. Extracted `executeWithStateRetryAndCatch()` +(shared by both) which re-runs the *entire* state body per attempt — that's +correct AWS semantics (a Map/Parallel retry restarts every branch/item, not +just the failed ones), but is a meaningful behavior difference from Task +Retry (which re-invokes a single resource call). Don't "simplify" this later +by trying to retry only failed items — that would silently diverge from AWS. + +**GetExecutionHistory's Task/State detail objects were empty shells.** +`StateEnteredEventDetails`, `StateExitedEventDetails`, `TaskScheduledEventDetails` +(new), `TaskSucceededEventDetails` (new), and `TaskFailedEventDetails` (new) +all have AWS-correct field shapes and json tags, but the five +`historyRecorder` methods in backend.go took the real values as parameters +and then *threw them away*, writing back only `{Type, Timestamp}`. Every +existing `GetExecutionHistory` test only asserted on event `Type`/ordering/ +pagination — never on the detail payload — which is exactly why this was +never caught. **Trap for the next auditor**: a green test suite for +`GetExecutionHistory` does not mean the event *bodies* are populated; check +that recorder methods actually use their parameters, not just that they +append an event of the right `Type`. + +**AWS's Catch error-output shape is `{"Error": , "Cause": }` +as two separate keys**, not one combined string. The pre-existing code built +`{"Error": err.Error()}` where `err.Error()` on a `*FailError` returns +`": "` — so downstream states reading `$.error.Cause` after a +`ResultPath` would get nothing, and `$.error.Error` would contain a mangled +combined string instead of just the code. Fixed via `stepFunctionsErrorCode`/ +`stepFunctionsErrorCause` (the former already existed for `catchesError` +matching; the latter is new). + +**StartExecution vs StartSyncExecution / STANDARD vs EXPRESS**: AWS allows +*asynchronous* `StartExecution` on EXPRESS state machines ("Asynchronous +Express Workflows", a documented, commonly-used feature) — only +`StartSyncExecution` is restricted to EXPRESS. The emulator had this +backwards in two ways: (1) it rejected `StartExecution` on EXPRESS entirely +(a real functional gap blocking a valid, common integration pattern), and (2) +it used the wrong error code (`InvalidExecutionType`) for the one case that +*is* correctly rejected (`StartSyncExecution` on STANDARD) — AWS's actual +error there is `StateMachineTypeNotSupported`. Fixed both; added +`ErrStateMachineTypeNotSupported` and its `handler.go` error-code mapping. +`ClientRequestToken`/EXPRESS-name-reuse nuances remain unmodeled +(bd: gopherstack-1sf) — lower priority since the core functional gap +(EXPRESS StartExecution being rejected) was the severe part. + +**Retry jitter**: AWS's `Retry.JitterStrategy` default is `"NONE"` (not +`"FULL"` — verify this if you're tempted to "fix" it the other way; it's +counter-intuitive since jitter is usually a sane default elsewhere). +`MaxDelaySeconds` caps the per-attempt delay *before* jitter is applied. +Both were entirely unparsed before this pass (only an internal 24h safety +cap existed, unrelated to the ASL-level `MaxDelaySeconds` field). + +**Protocol**: json-1.0 (`X-Amz-Target: AWSStepFunctions.`), +consistent with the rest of the codebase's json-1.0 services. No wire-format +regressions found in this family. diff --git a/services/stepfunctions/asl/executor.go b/services/stepfunctions/asl/executor.go index 7c26e7d5f..1e653a411 100644 --- a/services/stepfunctions/asl/executor.go +++ b/services/stepfunctions/asl/executor.go @@ -10,6 +10,7 @@ import ( "fmt" "maps" "math" + "math/rand/v2" "runtime" "strconv" "strings" @@ -68,10 +69,19 @@ var ( ) const ( - errCodeStatesPermissions = "States.Permissions" - errCodeStatesRuntime = "States.Runtime" - errCodeStatesTimeout = "States.Timeout" - errCodeStatesTaskFailed = "States.TaskFailed" + errCodeStatesPermissions = "States.Permissions" + errCodeStatesRuntime = "States.Runtime" + errCodeStatesTimeout = "States.Timeout" + errCodeStatesTaskFailed = "States.TaskFailed" + errCodeStatesExceedToleratedFailureThreshold = "States.ExceedToleratedFailureThreshold" +) + +// Sentinel errors for Map state tolerated-failure threshold resolution. +var ( + ErrToleratedFailureCountNotNumber = errors.New("ToleratedFailureCountPath: value is not a number") + ErrToleratedFailurePercentageNotNumber = errors.New( + "ToleratedFailurePercentagePath: value is not a number", + ) ) // LambdaInvoker can invoke a Lambda function. @@ -593,7 +603,7 @@ func (e *Executor) executeState( case "Task": return e.executeTask(ctx, executionARN, stateName, state, input) case "Parallel": - return e.executeParallel(ctx, executionARN, state, input) + return e.executeParallel(ctx, executionARN, stateName, state, input) case "Map": return e.executeMap(ctx, executionARN, stateName, state, input) default: @@ -809,7 +819,7 @@ func (e *Executor) executeTask( return next, out, nil } - e.recordTaskFailed(executionARN, stateName, taskErr.Error()) + e.recordTaskFailed(executionARN, stateName, stepFunctionsErrorCode(taskErr), stepFunctionsErrorCause(taskErr)) return "", nil, &FailError{ErrCode: "TaskFailed", Cause: taskErr.Error()} } @@ -959,6 +969,7 @@ func tryRetry(ctx context.Context, state *State, retryAttempts []int, taskErr er } delay := computeRetryDelay(intervalSeconds, backoffRate, retryAttempts[i]) + delay = applyRetryDelayCapAndJitter(delay, retrier) retryAttempts[i]++ if err := waitForRetry(ctx, delay); err != nil { @@ -992,6 +1003,25 @@ func computeRetryDelay(intervalSeconds int, backoffRate float64, attempts int) t return time.Duration(delaySeconds * float64(time.Second)) } +// applyRetryDelayCapAndJitter applies a Retrier's MaxDelaySeconds cap and +// JitterStrategy to a computed backoff delay. AWS's default JitterStrategy is +// "NONE" (delay used as-is); "FULL" randomizes the delay uniformly in +// [0, delay] to spread out simultaneous retries across concurrent executions. +func applyRetryDelayCapAndJitter(delay time.Duration, retrier *Retrier) time.Duration { + if retrier.MaxDelaySeconds != nil && *retrier.MaxDelaySeconds > 0 { + maxDelay := time.Duration(*retrier.MaxDelaySeconds) * time.Second + if delay > maxDelay { + delay = maxDelay + } + } + + if retrier.JitterStrategy == jitterStrategyFull { + delay = time.Duration(rand.Float64() * float64(delay)) //nolint:gosec // non-cryptographic per ASL spec + } + + return delay +} + // checkCatchers checks Catch clauses and returns (nextState, output, matched). // If a catcher matches, the task failure is recorded in the history. func (e *Executor) checkCatchers( @@ -1002,12 +1032,22 @@ func (e *Executor) checkCatchers( ) (string, any, bool) { for _, catcher := range state.Catch { if catchesError(catcher.ErrorEquals, taskErr) { + errCode := stepFunctionsErrorCode(taskErr) + cause := stepFunctionsErrorCause(taskErr) + + // AWS's error output has separate "Error" and "Cause" fields (the + // error code and a human-readable description), not one combined + // string. See "Handling errors in Step Functions workflows". errorResult := map[string]any{ - "Error": taskErr.Error(), + "Error": errCode, + } + if cause != "" { + errorResult["Cause"] = cause } + out, _ := applyResultPath(catcher.ResultPath, input, errorResult) - e.recordTaskFailed(executionARN, stateName, taskErr.Error()) + e.recordTaskFailed(executionARN, stateName, errCode, cause) return catcher.Next, out, true } @@ -1024,9 +1064,9 @@ func (e *Executor) recordTaskSucceeded(executionARN, stateName string, result an } // recordTaskFailed records a task failure event if a history recorder is configured. -func (e *Executor) recordTaskFailed(executionARN, stateName, errCode string) { +func (e *Executor) recordTaskFailed(executionARN, stateName, errCode, cause string) { if e.history != nil { - e.history.RecordTaskFailed(executionARN, stateName, errCode, "") + e.history.RecordTaskFailed(executionARN, stateName, errCode, cause) } } @@ -1407,39 +1447,82 @@ func parseServiceIntegrationResource(resource string) (string, string) { func (e *Executor) executeParallel( ctx context.Context, - executionARN string, + executionARN, stateName string, state *State, input any, ) (string, any, error) { - results := make([]any, len(state.Branches)) - errs := make([]error, len(state.Branches)) + return e.executeWithStateRetryAndCatch(ctx, executionARN, stateName, state, input, + func(ctx context.Context) (any, error) { + results := make([]any, len(state.Branches)) + errs := make([]error, len(state.Branches)) + + var wg sync.WaitGroup + for i, branch := range state.Branches { + if ctx.Err() != nil { + return nil, ctx.Err() + } - var wg sync.WaitGroup - for i, branch := range state.Branches { - if ctx.Err() != nil { - return "", nil, ctx.Err() - } + wg.Go(func() { e.runParallelBranch(ctx, executionARN, branch, input, results, errs, i) }) + } - wg.Go(func() { e.runParallelBranch(ctx, executionARN, branch, input, results, errs, i) }) - } + wg.Wait() - wg.Wait() + if err := ctx.Err(); err != nil { + return nil, err + } - if err := ctx.Err(); err != nil { - return "", nil, err - } + for _, branchErr := range errs { + if branchErr != nil { + return nil, branchErr + } + } + + return results, nil + }, + ) +} - for _, branchErr := range errs { - if branchErr != nil { - if next, out, matched := e.checkCatchers("", "Parallel", state, input, branchErr); matched { - return next, out, nil +// executeWithStateRetryAndCatch runs attempt (the full body of a Map or +// Parallel state) applying that state's own Retry and Catch clauses, exactly +// as AWS Step Functions supports Retry/Catch directly on Map and Parallel +// states (not just Task). Per AWS semantics, a retry re-executes the ENTIRE +// state body (all branches / all iterations), not just the failed portion. +func (e *Executor) executeWithStateRetryAndCatch( + ctx context.Context, + executionARN, stateName string, + state *State, + input any, + attempt func(ctx context.Context) (any, error), +) (string, any, error) { + retryAttempts := make([]int, len(state.Retry)) + + for { + result, err := attempt(ctx) + if err == nil { + return state.Next, result, nil + } + + if errors.Is(err, context.Canceled) { + return "", nil, err + } + + retried, retryErr := tryRetry(ctx, state, retryAttempts, err) + if retryErr != nil { + if !errors.Is(retryErr, ErrStatesTimeout) { + return "", nil, retryErr } - return "", nil, branchErr + err = ErrStatesTimeout + } else if retried { + continue } - } - return state.Next, results, nil + if next, out, matched := e.checkCatchers(executionARN, stateName, state, input, err); matched { + return next, out, nil + } + + return "", nil, err + } } // runParallelBranch executes a single Parallel state branch in a goroutine. @@ -1477,7 +1560,18 @@ func (e *Executor) runParallelBranch( const maxMapConcurrencyLimit = 40 -// executeMap handles Map state: iterates over an array. +// percentToFractionDivisor converts a 0-100 percentage into a 0-1 fraction. +const percentToFractionDivisor = 100 + +// jitterStrategyFull is the ASL Retry.JitterStrategy value that randomizes +// the computed backoff delay uniformly in [0, delay]. AWS's default, when +// JitterStrategy is omitted, is "NONE" (no randomization). +const jitterStrategyFull = "FULL" + +// executeMap handles Map state: iterates over an array. Like Task and +// Parallel, Map supports Retry/Catch on the Map state itself (in addition to +// per-iteration failures counted against ToleratedFailure*); a retry re-runs +// every item from scratch, matching AWS Map/Distributed-Map semantics. func (e *Executor) executeMap( ctx context.Context, executionARN string, @@ -1485,91 +1579,73 @@ func (e *Executor) executeMap( state *State, input any, ) (string, any, error) { - iterator, err := e.getMapIterator(state) - if err != nil { - return "", nil, err - } - - items, err := e.resolveMapItems(ctx, state, input) - if err != nil { - return "", nil, err + iterator, iterErr := e.getMapIterator(state) + if iterErr != nil { + return "", nil, iterErr } - if len(state.ItemSelector) > 0 { - items, err = applyMapItemSelector(state.ItemSelector, items) - if err != nil { - return "", nil, err - } - } - - // Apply ItemBatcher: wrap items into batches; each batch is one Map iteration. - if state.ItemBatcher != nil { - batched := batchItems(items, state.ItemBatcher) - batchResults := make([]any, len(batched)) - batchErrs := make([]error, len(batched)) - concurrency := resolveMapConcurrency(state.MaxConcurrency, len(batched)) - - var batchMapRunARN string - if e.mapRunNotifier != nil { - batchMapRunARN = e.mapRunNotifier.OnMapRunStart( - executionARN, - stateName, - state.MaxConcurrency, - len(batched), - ) - } + return e.executeWithStateRetryAndCatch(ctx, executionARN, stateName, state, input, + func(ctx context.Context) (any, error) { + items, err := e.resolveMapItems(ctx, state, input) + if err != nil { + return nil, err + } - e.runMapTasks(ctx, executionARN, iterator, batched, batchResults, batchErrs, concurrency) + if len(state.ItemSelector) > 0 { + items, err = applyMapItemSelector(state.ItemSelector, items) + if err != nil { + return nil, err + } + } - batchNext, batchOut, batchErr := e.finalizeMap(ctx, batchResults, batchErrs, state.Next) + // Apply ItemBatcher: wrap items into batches; each batch is one Map iteration. + if state.ItemBatcher != nil { + batched := batchItems(items, state.ItemBatcher) - if e.mapRunNotifier != nil && batchMapRunARN != "" { - batchSucceeded, batchFailed := countMapResults(batchErrs) - batchStatus := "SUCCEEDED" - if batchErr != nil { - batchStatus = "FAILED" + return e.runMapItemsAndFinalize(ctx, executionARN, stateName, iterator, state, input, batched) } - e.mapRunNotifier.OnMapRunEnd( - batchMapRunARN, - batchStatus, - batchSucceeded, - batchFailed, - len(batched), - ) - } - return batchNext, batchOut, batchErr - } + return e.runMapItemsAndFinalize(ctx, executionARN, stateName, iterator, state, input, items) + }, + ) +} +// runMapItemsAndFinalize runs iterator over items (or pre-built batches) at +// the resolved concurrency, notifies the MapRunNotifier (if configured), and +// finalizes the result, applying any ToleratedFailure* threshold. +func (e *Executor) runMapItemsAndFinalize( + ctx context.Context, + executionARN, stateName string, + iterator *StateMachine, + state *State, + mapInput any, + items []any, +) (any, error) { results := make([]any, len(items)) errs := make([]error, len(items)) - concurrency := resolveMapConcurrency(state.MaxConcurrency, len(items)) var mapRunARN string if e.mapRunNotifier != nil { - mapRunARN = e.mapRunNotifier.OnMapRunStart( - executionARN, - stateName, - state.MaxConcurrency, - len(items), - ) + mapRunARN = e.mapRunNotifier.OnMapRunStart(executionARN, stateName, state.MaxConcurrency, len(items)) } e.runMapTasks(ctx, executionARN, iterator, items, results, errs, concurrency) - next, out, finalErr := e.finalizeMap(ctx, results, errs, state.Next) + out, finalErr := e.finalizeMap(ctx, state, mapInput, results, errs) if e.mapRunNotifier != nil && mapRunARN != "" { succeeded, failed := countMapResults(errs) status := "SUCCEEDED" + if finalErr != nil { status = "FAILED" } + e.mapRunNotifier.OnMapRunEnd(mapRunARN, status, succeeded, failed, len(items)) } - return next, out, finalErr + return out, finalErr } // batchItems groups items into batches according to ItemBatcher configuration. @@ -1662,6 +1738,17 @@ func (e *Executor) runMapItem( return } + // A Fail state (or an unhandled Task failure) inside the iterator + // surfaces as a successful Execute() call with res.Error populated + // rather than a Go error — mirror runParallelBranch's handling so Map + // iteration failures propagate to the Map state's own Catch/Retry + // instead of silently producing a nil result for the failed item. + if res.Error != "" { + errs[idx] = &FailError{ErrCode: res.Error, Cause: res.Cause} + + return + } + results[idx] = res.Output } @@ -1887,21 +1974,153 @@ func (e *Executor) spawnMapTask( func (e *Executor) finalizeMap( ctx context.Context, + state *State, + mapInput any, results []any, errs []error, - next string, -) (string, any, error) { +) (any, error) { if ctxErr := ctx.Err(); ctxErr != nil { - return "", nil, ctxErr + return nil, ctxErr } + failedCount := 0 + + var firstErr error + for _, iterErr := range errs { if iterErr != nil { - return "", nil, iterErr + failedCount++ + + if firstErr == nil { + firstErr = iterErr + } + } + } + + if failedCount == 0 { + return results, nil + } + + threshold, err := e.resolveToleratedFailureThreshold(state, mapInput, len(errs)) + if err != nil { + return nil, err + } + + if failedCount <= threshold { + return results, nil + } + + // No tolerance configured (the common case): preserve the original + // per-item error so Catch/Retry on the Map state can match on it. + if threshold == 0 && state.ToleratedFailureCount == nil && + state.ToleratedFailureCountPath == "" && + state.ToleratedFailurePercentage == nil && + state.ToleratedFailurePercentagePath == "" { + return nil, firstErr + } + + return nil, &FailError{ + ErrCode: errCodeStatesExceedToleratedFailureThreshold, + Cause: fmt.Sprintf( + "Map Run failed: %d of %d items failed, exceeding the tolerated failure threshold of %d", + failedCount, len(errs), threshold, + ), + } +} + +// resolveToleratedFailureThreshold computes the maximum number of failed Map +// iterations tolerated before the Map state fails. Returns 0 (no tolerance) +// when neither ToleratedFailureCount(Path) nor ToleratedFailurePercentage(Path) +// is configured. When both are configured, the more restrictive (lower) +// threshold wins, matching AWS "fails when either is crossed" semantics. +func (e *Executor) resolveToleratedFailureThreshold( + state *State, + mapInput any, + totalCount int, +) (int, error) { + threshold := totalCount + hasThreshold := false + + countThreshold, countConfigured, err := e.resolveToleratedFailureCount(state, mapInput) + if err != nil { + return 0, err + } + + if countConfigured { + threshold = countThreshold + hasThreshold = true + } + + pctThreshold, pctConfigured, err := e.resolveToleratedFailurePercentageThreshold(state, mapInput, totalCount) + if err != nil { + return 0, err + } + + if pctConfigured && (!hasThreshold || pctThreshold < threshold) { + threshold = pctThreshold + hasThreshold = true + } + + if !hasThreshold { + return 0, nil + } + + return threshold, nil +} + +// resolveToleratedFailureCount resolves ToleratedFailureCount(Path). The +// second return value reports whether either field was configured. +func (e *Executor) resolveToleratedFailureCount(state *State, mapInput any) (int, bool, error) { + switch { + case state.ToleratedFailureCountPath != "": + val, err := applyPath(state.ToleratedFailureCountPath, mapInput, e.jsonPathCache) + if err != nil { + return 0, false, fmt.Errorf("ToleratedFailureCountPath error: %w", err) + } + + f, ok := toFloat(val) + if !ok { + return 0, false, ErrToleratedFailureCountNotNumber + } + + return int(f), true, nil + case state.ToleratedFailureCount != nil: + return *state.ToleratedFailureCount, true, nil + default: + return 0, false, nil + } +} + +// resolveToleratedFailurePercentageThreshold resolves ToleratedFailurePercentage(Path) +// into an absolute failure-count threshold for totalCount items. The second +// return value reports whether either field was configured. +func (e *Executor) resolveToleratedFailurePercentageThreshold( + state *State, + mapInput any, + totalCount int, +) (int, bool, error) { + var pct float64 + + switch { + case state.ToleratedFailurePercentagePath != "": + val, err := applyPath(state.ToleratedFailurePercentagePath, mapInput, e.jsonPathCache) + if err != nil { + return 0, false, fmt.Errorf("ToleratedFailurePercentagePath error: %w", err) } + + f, ok := toFloat(val) + if !ok { + return 0, false, ErrToleratedFailurePercentageNotNumber + } + + pct = f + case state.ToleratedFailurePercentage != nil: + pct = *state.ToleratedFailurePercentage + default: + return 0, false, nil } - return next, results, nil + return int(math.Floor(float64(totalCount) * pct / percentToFractionDivisor)), true, nil } // resolveMapConcurrency determines the effective concurrency for a Map state. @@ -2660,6 +2879,20 @@ func stepFunctionsErrorCode(err error) string { return err.Error() } +// stepFunctionsErrorCause extracts the human-readable cause for an error, for +// population into a Catch's error output (the "Cause" field) and into +// TaskFailed history events. FailError carries an explicit Cause (e.g. from a +// Fail state or a wrapped service-integration failure); other errors have no +// separate code/cause split, so the full message is used as the cause. +func stepFunctionsErrorCause(err error) string { + var failErr *FailError + if errors.As(err, &failErr) { + return failErr.Cause + } + + return err.Error() +} + // isActivityResource returns true if the resource ARN refers to an activity. func isActivityResource(resource string) bool { return strings.Contains(resource, ":activity:") diff --git a/services/stepfunctions/asl/executor_internal_test.go b/services/stepfunctions/asl/executor_internal_test.go index 36473fa83..6f5eedb8c 100644 --- a/services/stepfunctions/asl/executor_internal_test.go +++ b/services/stepfunctions/asl/executor_internal_test.go @@ -93,6 +93,83 @@ func TestComputeRetryDelay(t *testing.T) { } } +func TestApplyRetryDelayCapAndJitter(t *testing.T) { + t.Parallel() + + intPtr := func(v int) *int { return &v } + + tests := []struct { + retrier *Retrier + name string + delay time.Duration + want time.Duration + wantMax time.Duration + jitter bool + }{ + { + name: "no_max_delay_no_jitter_passthrough", + delay: 10 * time.Second, + retrier: &Retrier{}, + want: 10 * time.Second, + }, + { + name: "max_delay_caps_longer_delay", + delay: 1000 * time.Second, + retrier: &Retrier{MaxDelaySeconds: intPtr(2)}, + want: 2 * time.Second, + }, + { + name: "max_delay_does_not_extend_shorter_delay", + delay: 1 * time.Second, + retrier: &Retrier{MaxDelaySeconds: intPtr(2)}, + want: 1 * time.Second, + }, + { + name: "zero_max_delay_seconds_ignored", + delay: 5 * time.Second, + retrier: &Retrier{MaxDelaySeconds: intPtr(0)}, + want: 5 * time.Second, + }, + { + name: "jitter_strategy_none_is_default_passthrough", + delay: 5 * time.Second, + retrier: &Retrier{JitterStrategy: "NONE"}, + want: 5 * time.Second, + }, + { + name: "jitter_strategy_full_randomizes_within_bound", + delay: 10 * time.Second, + retrier: &Retrier{JitterStrategy: jitterStrategyFull}, + jitter: true, + wantMax: 10 * time.Second, + }, + { + name: "max_delay_and_jitter_combined", + delay: 1000 * time.Second, + retrier: &Retrier{MaxDelaySeconds: intPtr(3), JitterStrategy: jitterStrategyFull}, + jitter: true, + wantMax: 3 * time.Second, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := applyRetryDelayCapAndJitter(tt.delay, tt.retrier) + + if tt.jitter { + assert.GreaterOrEqual(t, got, time.Duration(0)) + assert.LessOrEqual(t, got, tt.wantMax) + + return + } + + assert.Equal(t, tt.want, got) + }) + } +} + func TestJSONPathCacheBounded(t *testing.T) { t.Parallel() diff --git a/services/stepfunctions/asl/executor_test.go b/services/stepfunctions/asl/executor_test.go index 002f3875f..a03c746d4 100644 --- a/services/stepfunctions/asl/executor_test.go +++ b/services/stepfunctions/asl/executor_test.go @@ -772,6 +772,127 @@ func TestExecutor_ParallelState(t *testing.T) { } } +// TestExecutor_ParallelState_RetryAndCatch verifies that Retry and Catch +// clauses defined directly on a Parallel state (not just inside its +// branches) are honored: AWS supports Retry/Catch on Parallel exactly as it +// does on Task, retrying the whole set of branches from scratch. +func TestExecutor_ParallelState_RetryAndCatch(t *testing.T) { + t.Parallel() + + t.Run("retry_reruns_all_branches_until_success", func(t *testing.T) { + t.Parallel() + + var calls atomic.Int64 + + lambda := &mockLambdaFn{fn: func() ([]byte, int, error) { + if calls.Add(1) == 1 { + return nil, 500, errMyError + } + + return []byte(`{}`), 200, nil + }} + + def := `{ + "StartAt": "P", + "States": { + "P": { + "Type": "Parallel", + "End": true, + "Retry": [{"ErrorEquals": ["States.ALL"], "MaxAttempts": 2, "IntervalSeconds": 0}], + "Branches": [ + { + "StartAt": "T", + "States": { + "T": { + "Type": "Task", + "Resource": "arn:aws:lambda:us-east-1:000000000000:function:fn", + "End": true + } + } + } + ] + } + } + }` + + sm, err := asl.Parse(def) + require.NoError(t, err) + exec := asl.NewExecutor(sm, lambda, nil) + result, execErr := exec.Execute(t.Context(), "test", `{}`) + require.NoError(t, execErr) + assert.Empty(t, result.Error) + assert.EqualValues(t, 2, calls.Load()) + + arr, ok := result.Output.([]any) + require.True(t, ok) + assert.Len(t, arr, 1) + }) + + t.Run("catch_routes_to_fallback_on_branch_failure", func(t *testing.T) { + t.Parallel() + + def := `{ + "StartAt": "P", + "States": { + "P": { + "Type": "Parallel", + "End": true, + "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "Fallback"}], + "Branches": [ + { + "StartAt": "Boom", + "States": {"Boom": {"Type": "Fail", "Error": "BranchFailed"}} + } + ] + }, + "Fallback": {"Type": "Pass", "End": true, "Result": "fallback-used"} + } + }` + + result := execute(t, def, `{}`) + assert.Empty(t, result.Error) + assert.Equal(t, "fallback-used", result.Output) + }) +} + +// TestExecutor_Catch_ErrorOutputShape verifies that a Catch's injected error +// object has separate "Error" (error code) and "Cause" (human-readable +// description) fields, matching AWS's documented error-output shape, rather +// than a single field holding the two concatenated together. +func TestExecutor_Catch_ErrorOutputShape(t *testing.T) { + t.Parallel() + + def := `{ + "StartAt": "P", + "States": { + "P": { + "Type": "Parallel", + "End": true, + "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "Fallback", "ResultPath": "$.error"}], + "Branches": [ + { + "StartAt": "Boom", + "States": {"Boom": {"Type": "Fail", "Error": "MyErr", "Cause": "my cause"}} + } + ] + }, + "Fallback": {"Type": "Pass", "End": true} + } + }` + + result := execute(t, def, `{"a": 1}`) + assert.Empty(t, result.Error) + + out, ok := result.Output.(map[string]any) + require.True(t, ok) + assert.InEpsilon(t, 1.0, out["a"], 0) + + errObj, ok := out["error"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "MyErr", errObj["Error"]) + assert.Equal(t, "my cause", errObj["Cause"]) +} + func TestExecutor_MapState(t *testing.T) { t.Parallel() @@ -925,6 +1046,198 @@ func TestExecutor_MapState(t *testing.T) { } } +// mapToleratedFailureDef builds a Map-state definition whose iterator fails +// exactly the items with `"fail": true` (Fail state, Error="Boom"), and +// injects the given ToleratedFailure* clause (raw ASL fragment, may be empty) +// into the Map state. +func mapToleratedFailureDef(toleratedFailureClause string) string { + return `{ + "StartAt": "Map", + "States": { + "Map": { + "Type": "Map", + "End": true, + "ItemsPath": "$.items",` + toleratedFailureClause + ` + "Iterator": { + "StartAt": "Check", + "States": { + "Check": { + "Type": "Choice", + "Choices": [ + {"Variable": "$.fail", "BooleanEquals": true, "Next": "Boom"} + ], + "Default": "OK" + }, + "Boom": {"Type": "Fail", "Error": "Boom", "Cause": "boom"}, + "OK": {"Type": "Pass", "End": true} + } + } + } + } + }` +} + +func TestExecutor_MapState_ToleratedFailure(t *testing.T) { + t.Parallel() + + const fourItemsOneFails = `{"items": [ + {"fail": false}, {"fail": true}, {"fail": false}, {"fail": false} + ], "tolerate": 1}` + const fourItemsTwoFail = `{"items": [ + {"fail": true}, {"fail": true}, {"fail": false}, {"fail": false} + ], "tolerate": 1}` + + tests := []struct { + name string + clause string + input string + wantErrCode string + wantOutputLen int + }{ + { + name: "no_tolerance_configured_propagates_original_error", + clause: "", + input: fourItemsOneFails, + wantErrCode: "Boom", + }, + { + name: "percentage_within_tolerance_succeeds", + clause: `"ToleratedFailurePercentage": 50,`, + input: fourItemsOneFails, + wantOutputLen: 4, + }, + { + name: "percentage_exceeds_tolerance_fails", + clause: `"ToleratedFailurePercentage": 10,`, + input: fourItemsOneFails, + wantErrCode: "States.ExceedToleratedFailureThreshold", + }, + { + name: "count_within_tolerance_succeeds", + clause: `"ToleratedFailureCount": 1,`, + input: fourItemsOneFails, + wantOutputLen: 4, + }, + { + name: "count_exceeds_tolerance_fails", + clause: `"ToleratedFailureCount": 1,`, + input: fourItemsTwoFail, + wantErrCode: "States.ExceedToleratedFailureThreshold", + }, + { + name: "count_path_resolved_from_input_succeeds", + clause: `"ToleratedFailureCountPath": "$.tolerate",`, + input: fourItemsOneFails, + wantOutputLen: 4, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + sm, err := asl.Parse(mapToleratedFailureDef(tt.clause)) + require.NoError(t, err) + + exec := asl.NewExecutor(sm, nil, nil) + result, execErr := exec.Execute(t.Context(), "test", tt.input) + require.NoError(t, execErr) + + if tt.wantErrCode != "" { + assert.Equal(t, tt.wantErrCode, result.Error) + + return + } + + assert.Empty(t, result.Error) + arr, ok := result.Output.([]any) + require.True(t, ok) + assert.Len(t, arr, tt.wantOutputLen) + }) + } +} + +// TestExecutor_MapState_RetryAndCatch verifies that Retry and Catch clauses +// on the Map state itself (as opposed to inside its Iterator) are honored, +// matching AWS's support for Retry/Catch directly on Map/Distributed Map. +func TestExecutor_MapState_RetryAndCatch(t *testing.T) { + t.Parallel() + + t.Run("retry_reruns_all_items_until_success", func(t *testing.T) { + t.Parallel() + + var calls atomic.Int64 + + lambda := &mockLambdaFn{fn: func() ([]byte, int, error) { + if calls.Add(1) == 1 { + return nil, 500, errMyError + } + + return []byte(`{}`), 200, nil + }} + + def := `{ + "StartAt": "M", + "States": { + "M": { + "Type": "Map", + "End": true, + "MaxConcurrency": 1, + "Retry": [{"ErrorEquals": ["States.ALL"], "MaxAttempts": 2, "IntervalSeconds": 0}], + "Iterator": { + "StartAt": "T", + "States": { + "T": { + "Type": "Task", + "Resource": "arn:aws:lambda:us-east-1:000000000000:function:fn", + "End": true + } + } + } + } + } + }` + + sm, err := asl.Parse(def) + require.NoError(t, err) + exec := asl.NewExecutor(sm, lambda, nil) + result, execErr := exec.Execute(t.Context(), "test", `[1]`) + require.NoError(t, execErr) + assert.Empty(t, result.Error) + // The single item fails on the Map's first attempt (1 lambda call), + // then the whole Map is retried and the item succeeds (2nd call). + assert.EqualValues(t, 2, calls.Load()) + + arr, ok := result.Output.([]any) + require.True(t, ok) + assert.Len(t, arr, 1) + }) + + t.Run("catch_routes_to_fallback_on_map_failure", func(t *testing.T) { + t.Parallel() + + def := `{ + "StartAt": "M", + "States": { + "M": { + "Type": "Map", + "End": true, + "Catch": [{"ErrorEquals": ["States.ALL"], "Next": "Fallback"}], + "Iterator": { + "StartAt": "Boom", + "States": {"Boom": {"Type": "Fail", "Error": "ItemFailed"}} + } + }, + "Fallback": {"Type": "Pass", "End": true, "Result": "fallback-used"} + } + }` + + result := execute(t, def, `[1, 2, 3]`) + assert.Empty(t, result.Error) + assert.Equal(t, "fallback-used", result.Output) + }) +} + func TestExecutor_PathTransforms(t *testing.T) { t.Parallel() diff --git a/services/stepfunctions/asl/parser.go b/services/stepfunctions/asl/parser.go index d8f169fe0..556656ad6 100644 --- a/services/stepfunctions/asl/parser.go +++ b/services/stepfunctions/asl/parser.go @@ -47,43 +47,60 @@ type ReaderConfig struct { // State represents a single state in the state machine. type State struct { - Iterator *StateMachine `json:"Iterator,omitempty"` - ItemProcessor *StateMachine `json:"ItemProcessor,omitempty"` - ItemBatcher *ItemBatcher `json:"ItemBatcher,omitempty"` - ItemReader *ItemReader `json:"ItemReader,omitempty"` - ItemSelector json.RawMessage `json:"ItemSelector,omitempty"` - SecondsPath string `json:"SecondsPath,omitempty"` - TimestampPath string `json:"TimestampPath,omitempty"` - ItemsPath string `json:"ItemsPath,omitempty"` - InputPath string `json:"InputPath,omitempty"` - OutputPath string `json:"OutputPath,omitempty"` - ResultPath string `json:"ResultPath,omitempty"` - Type string `json:"Type"` - Error string `json:"Error,omitempty"` - Cause string `json:"Cause,omitempty"` - Comment string `json:"Comment,omitempty"` - Next string `json:"Next,omitempty"` - Default string `json:"Default,omitempty"` - Timestamp string `json:"Timestamp,omitempty"` - Resource string `json:"Resource,omitempty"` - Retry []Retrier `json:"Retry,omitempty"` - Catch []Catcher `json:"Catch,omitempty"` - Choices []ChoiceRule `json:"Choices,omitempty"` - Result json.RawMessage `json:"Result,omitempty"` - Branches []Branch `json:"Branches,omitempty"` - Parameters json.RawMessage `json:"Parameters,omitempty"` - ResultSelector json.RawMessage `json:"ResultSelector,omitempty"` - TimeoutSeconds int `json:"TimeoutSeconds,omitempty"` - HeartbeatSeconds int `json:"HeartbeatSeconds,omitempty"` - Seconds int `json:"Seconds,omitempty"` - MaxConcurrency int `json:"MaxConcurrency,omitempty"` - End bool `json:"End,omitempty"` + Iterator *StateMachine `json:"Iterator,omitempty"` + ItemProcessor *StateMachine `json:"ItemProcessor,omitempty"` + ItemBatcher *ItemBatcher `json:"ItemBatcher,omitempty"` + ItemReader *ItemReader `json:"ItemReader,omitempty"` + ItemSelector json.RawMessage `json:"ItemSelector,omitempty"` + SecondsPath string `json:"SecondsPath,omitempty"` + TimestampPath string `json:"TimestampPath,omitempty"` + ItemsPath string `json:"ItemsPath,omitempty"` + // ToleratedFailureCount/Percentage (and their *Path variants) bound how many + // Map iterations may fail before the Map state itself fails with + // States.ExceedToleratedFailureThreshold. AWS supports these only for + // Distributed Map, but the emulator applies them uniformly since Map + // processing mode is not otherwise distinguished. When both a count and a + // percentage are set, the Map fails when EITHER threshold is crossed. + ToleratedFailureCountPath string `json:"ToleratedFailureCountPath,omitempty"` + ToleratedFailurePercentagePath string `json:"ToleratedFailurePercentagePath,omitempty"` + ToleratedFailureCount *int `json:"ToleratedFailureCount,omitempty"` + ToleratedFailurePercentage *float64 `json:"ToleratedFailurePercentage,omitempty"` + InputPath string `json:"InputPath,omitempty"` + OutputPath string `json:"OutputPath,omitempty"` + ResultPath string `json:"ResultPath,omitempty"` + Type string `json:"Type"` + Error string `json:"Error,omitempty"` + Cause string `json:"Cause,omitempty"` + Comment string `json:"Comment,omitempty"` + Next string `json:"Next,omitempty"` + Default string `json:"Default,omitempty"` + Timestamp string `json:"Timestamp,omitempty"` + Resource string `json:"Resource,omitempty"` + Retry []Retrier `json:"Retry,omitempty"` + Catch []Catcher `json:"Catch,omitempty"` + Choices []ChoiceRule `json:"Choices,omitempty"` + Result json.RawMessage `json:"Result,omitempty"` + Branches []Branch `json:"Branches,omitempty"` + Parameters json.RawMessage `json:"Parameters,omitempty"` + ResultSelector json.RawMessage `json:"ResultSelector,omitempty"` + TimeoutSeconds int `json:"TimeoutSeconds,omitempty"` + HeartbeatSeconds int `json:"HeartbeatSeconds,omitempty"` + Seconds int `json:"Seconds,omitempty"` + MaxConcurrency int `json:"MaxConcurrency,omitempty"` + End bool `json:"End,omitempty"` } // Retrier defines retry behavior for a Task state on error. +// +// JitterStrategy controls whether the computed backoff delay is randomized: +// "FULL" randomizes the delay uniformly between 0 and the computed value; +// "NONE" (the AWS default when omitted) uses the computed delay as-is. +// MaxDelaySeconds, when set, caps the delay between retry attempts. type Retrier struct { IntervalSeconds *int `json:"IntervalSeconds,omitempty"` MaxAttempts *int `json:"MaxAttempts,omitempty"` + MaxDelaySeconds *int `json:"MaxDelaySeconds,omitempty"` + JitterStrategy string `json:"JitterStrategy,omitempty"` ErrorEquals []string `json:"ErrorEquals"` BackoffRate float64 `json:"BackoffRate,omitempty"` } diff --git a/services/stepfunctions/backend.go b/services/stepfunctions/backend.go index 8b7b6e843..ccf4d1aae 100644 --- a/services/stepfunctions/backend.go +++ b/services/stepfunctions/backend.go @@ -33,6 +33,7 @@ var ( ErrExecutionNotRedrivable = errors.New("ExecutionNotRedrivable") ErrInvalidDefinition = errors.New("InvalidDefinition") ErrInvalidExecutionType = errors.New("InvalidExecutionType") + ErrStateMachineTypeNotSupported = errors.New("StateMachineTypeNotSupported") ErrInvalidRoleArn = errors.New("InvalidArn") ErrInvalidName = errors.New("InvalidName") ErrInvalidRoutingConfiguration = errors.New("InvalidRoutingConfiguration") @@ -833,9 +834,11 @@ func (b *InMemoryBackend) StartSyncExecution( if sm.Type != "EXPRESS" { b.mu.RUnlock() + // AWS: StartSyncExecution is only supported for EXPRESS state + // machines and returns StateMachineTypeNotSupported for STANDARD. return nil, fmt.Errorf( - "%w: sync execution requires EXPRESS state machine", - ErrInvalidExecutionType, + "%w: StartSyncExecution requires an EXPRESS state machine", + ErrStateMachineTypeNotSupported, ) } @@ -1008,15 +1011,9 @@ func (b *InMemoryBackend) StartExecution(stateMachineArn, name, input string) (* return nil, fmt.Errorf("%w: %s", ErrStateMachineDoesNotExist, stateMachineArn) } - if sm.Type == "EXPRESS" { - b.mu.Unlock() - - return nil, fmt.Errorf( - "%w: async execution requires STANDARD state machine", - ErrInvalidExecutionType, - ) - } - + // AWS allows StartExecution (asynchronous execution) on EXPRESS state + // machines too -- only StartSyncExecution is restricted to EXPRESS. + // See "Asynchronous Express Workflows" in the AWS Step Functions docs. execArn := b.execARN(stateMachineArn, sm.Name, name) if _, alreadyExists := b.executions[execArn]; alreadyExists { b.mu.Unlock() @@ -1239,45 +1236,76 @@ func (b *InMemoryBackend) appendHistory(execARN string, event *HistoryEvent) { b.history[execARN] = append(events, event) } -func (r *historyRecorder) RecordStateEntered(execARN, stateName, stateType string, _ any) { +func (r *historyRecorder) RecordStateEntered(execARN, stateName, stateType string, input any) { r.backend.appendHistory(execARN, &HistoryEvent{ - Timestamp: float64(time.Now().Unix()), - Type: stateEnteredEventType(stateType), - StateEnteredEventDetails: &StateEnteredEventDetails{Name: stateName}, + Timestamp: float64(time.Now().Unix()), + Type: stateEnteredEventType(stateType), + StateEnteredEventDetails: &StateEnteredEventDetails{ + Name: stateName, + Input: historyValueToJSON(input), + }, }) } -func (r *historyRecorder) RecordStateExited(execARN, stateName, stateType string, _ any) { +func (r *historyRecorder) RecordStateExited(execARN, stateName, stateType string, output any) { r.backend.appendHistory(execARN, &HistoryEvent{ - Timestamp: float64(time.Now().Unix()), - Type: stateExitedEventType(stateType), - StateExitedEventDetails: &StateExitedEventDetails{Name: stateName}, + Timestamp: float64(time.Now().Unix()), + Type: stateExitedEventType(stateType), + StateExitedEventDetails: &StateExitedEventDetails{ + Name: stateName, + Output: historyValueToJSON(output), + }, }) } -func (r *historyRecorder) RecordTaskScheduled(execARN, _ /* stateName */, _ /* resource */ string) { +func (r *historyRecorder) RecordTaskScheduled(execARN, _ /* stateName */, resource string) { r.backend.appendHistory(execARN, &HistoryEvent{ - Timestamp: float64(time.Now().Unix()), - Type: "TaskScheduled", + Timestamp: float64(time.Now().Unix()), + Type: "TaskScheduled", + TaskScheduledEventDetails: &TaskScheduledEventDetails{Resource: resource}, }) } -func (r *historyRecorder) RecordTaskSucceeded(execARN, _ /* stateName */ string, _ any) { +func (r *historyRecorder) RecordTaskSucceeded(execARN, _ /* stateName */ string, output any) { r.backend.appendHistory(execARN, &HistoryEvent{ - Timestamp: float64(time.Now().Unix()), - Type: "TaskSucceeded", + Timestamp: float64(time.Now().Unix()), + Type: "TaskSucceeded", + TaskSucceededEventDetails: &TaskSucceededEventDetails{Output: historyValueToJSON(output)}, }) } func (r *historyRecorder) RecordTaskFailed( - execARN, _ /* stateName */, _ /* errCode */, _ /* cause */ string, + execARN, _ /* stateName */, errCode, cause string, ) { r.backend.appendHistory(execARN, &HistoryEvent{ Timestamp: float64(time.Now().Unix()), Type: "TaskFailed", + TaskFailedEventDetails: &TaskFailedEventDetails{ + Error: errCode, + Cause: cause, + }, }) } +// historyValueToJSON marshals a state input/output value to its JSON string +// representation for execution-history detail fields, matching AWS's +// convention of embedding input/output as a JSON string rather than a nested +// object. Marshal failures (which should not occur for values that already +// round-tripped through the ASL interpreter) fall back to an empty string +// rather than corrupting the history event. +func historyValueToJSON(v any) string { + if v == nil { + return "" + } + + b, err := json.Marshal(v) + if err != nil { + return "" + } + + return string(b) +} + // checkHistoryCapacity returns the current event slice and whether there is // room to append. On the first refusal per execution it logs a warning so that // silent truncation is observable. Caller must hold b.historyMu write lock. diff --git a/services/stepfunctions/backend_test.go b/services/stepfunctions/backend_test.go index 4f746bdeb..baedd557b 100644 --- a/services/stepfunctions/backend_test.go +++ b/services/stepfunctions/backend_test.go @@ -353,12 +353,14 @@ func TestStartExecution(t *testing.T) { wantErr: stepfunctions.ErrExecutionAlreadyExists, }, { - name: "ExpressRequiresStartSyncExecution", - createSM: true, - smType: "EXPRESS", - execName: "exec1", - input: `{"key":"value"}`, - wantErr: stepfunctions.ErrInvalidExecutionType, + // AWS allows asynchronous StartExecution on EXPRESS state + // machines too (only StartSyncExecution is EXPRESS-only). + name: "ExpressAllowsAsyncStartExecution", + createSM: true, + smType: "EXPRESS", + execName: "exec1", + input: `{"key":"value"}`, + wantArnContains: "exec1", }, } @@ -604,6 +606,118 @@ func TestGetExecutionHistory(t *testing.T) { } } +const taskLambdaDefinition = `{ +"StartAt": "T", +"States": { +"T": {"Type": "Task", "Resource": "arn:aws:lambda:us-east-1:000000000000:function:fn", "End": true} +} +}` + +// TestGetExecutionHistory_TaskEventDetails verifies that Task lifecycle +// history events (TaskScheduled/TaskSucceeded/TaskFailed) and state +// entered/exited events carry their AWS-documented detail payloads +// (resource, input, output, error, cause) rather than an empty details +// object. +func TestGetExecutionHistory_TaskEventDetails(t *testing.T) { + t.Parallel() + + t.Run("succeeded_task_populates_resource_and_output", func(t *testing.T) { + t.Parallel() + + b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + b.SetLambdaInvoker(&mockLambdaForBackend{}) + + sm, err := b.CreateStateMachine( + context.Background(), + "hist-task-sm", + taskLambdaDefinition, + "arn:role", + "STANDARD", + ) + require.NoError(t, err) + + exec, err := b.StartExecution(sm.StateMachineArn, "exec-task-ok", `{"in": 1}`) + require.NoError(t, err) + + require.Eventually(t, func() bool { + desc, descErr := b.DescribeExecution(exec.ExecutionArn) + + return descErr == nil && desc.Status != "RUNNING" + }, 5*time.Second, 50*time.Millisecond) + + events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 0, false) + require.NoError(t, err) + + var sawScheduled, sawSucceeded, sawStateEntered bool + + for _, ev := range events { + switch ev.Type { + case "TaskScheduled": + require.NotNil(t, ev.TaskScheduledEventDetails) + assert.Equal( + t, + "arn:aws:lambda:us-east-1:000000000000:function:fn", + ev.TaskScheduledEventDetails.Resource, + ) + sawScheduled = true + case "TaskSucceeded": + require.NotNil(t, ev.TaskSucceededEventDetails) + assert.Contains(t, ev.TaskSucceededEventDetails.Output, "ok") + sawSucceeded = true + case "TaskStateEntered": + require.NotNil(t, ev.StateEnteredEventDetails) + assert.Contains(t, ev.StateEnteredEventDetails.Input, `"in":1`) + sawStateEntered = true + } + } + + assert.True(t, sawScheduled, "expected a TaskScheduled event") + assert.True(t, sawSucceeded, "expected a TaskSucceeded event") + assert.True(t, sawStateEntered, "expected a TaskStateEntered event with populated input") + }) + + t.Run("failed_task_populates_error_and_cause", func(t *testing.T) { + t.Parallel() + + b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + b.SetLambdaInvoker(&mockLambdaForBackend{returnErr: assert.AnError}) + + sm, err := b.CreateStateMachine( + context.Background(), + "hist-task-fail-sm", + taskLambdaDefinition, + "arn:role", + "STANDARD", + ) + require.NoError(t, err) + + exec, err := b.StartExecution(sm.StateMachineArn, "exec-task-fail", `{}`) + require.NoError(t, err) + + require.Eventually(t, func() bool { + desc, descErr := b.DescribeExecution(exec.ExecutionArn) + + return descErr == nil && desc.Status != "RUNNING" + }, 5*time.Second, 50*time.Millisecond) + + events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 0, false) + require.NoError(t, err) + + var sawFailed bool + + for _, ev := range events { + if ev.Type == "TaskFailed" { + require.NotNil(t, ev.TaskFailedEventDetails) + assert.NotEmpty(t, ev.TaskFailedEventDetails.Error) + assert.NotEmpty(t, ev.TaskFailedEventDetails.Cause) + sawFailed = true + } + } + + assert.True(t, sawFailed, "expected a TaskFailed event") + }) +} + func TestStopExecution(t *testing.T) { t.Parallel() diff --git a/services/stepfunctions/batch1_audit_test.go b/services/stepfunctions/batch1_audit_test.go index 5a35d6b3a..d1330ba02 100644 --- a/services/stepfunctions/batch1_audit_test.go +++ b/services/stepfunctions/batch1_audit_test.go @@ -560,7 +560,11 @@ func TestAudit_Execution_StateMachineNotFound(t *testing.T) { // ─── StartExecution / StartSyncExecution type enforcement ──────────────────── -func TestAudit_StartExecution_ExpressMachineReturnsError(t *testing.T) { +// TestAudit_StartExecution_ExpressMachineSucceeds verifies that +// StartExecution (asynchronous execution) is permitted on EXPRESS state +// machines, matching AWS's "Asynchronous Express Workflows" support -- only +// StartSyncExecution is restricted to EXPRESS. +func TestAudit_StartExecution_ExpressMachineSucceeds(t *testing.T) { t.Parallel() b := stepfunctions.NewInMemoryBackend() @@ -573,9 +577,9 @@ func TestAudit_StartExecution_ExpressMachineReturnsError(t *testing.T) { ) require.NoError(t, err) - _, err = b.StartExecution(sm.StateMachineArn, "e1", "{}") - require.Error(t, err) - assert.ErrorIs(t, err, stepfunctions.ErrInvalidExecutionType) + exec, err := b.StartExecution(sm.StateMachineArn, "e1", "{}") + require.NoError(t, err) + assert.Contains(t, exec.ExecutionArn, "e1") } func TestAudit_StartSyncExecution_StandardMachineReturnsError(t *testing.T) { @@ -593,7 +597,7 @@ func TestAudit_StartSyncExecution_StandardMachineReturnsError(t *testing.T) { _, err = b.StartSyncExecution(sm.StateMachineArn, "sync-e1", "{}") require.Error(t, err) - assert.ErrorIs(t, err, stepfunctions.ErrInvalidExecutionType) + assert.ErrorIs(t, err, stepfunctions.ErrStateMachineTypeNotSupported) } func TestAudit_StartSyncExecution_Express_Succeeds(t *testing.T) { diff --git a/services/stepfunctions/handler.go b/services/stepfunctions/handler.go index 38ea5b0a3..4d147b7ec 100644 --- a/services/stepfunctions/handler.go +++ b/services/stepfunctions/handler.go @@ -1386,6 +1386,7 @@ func classifyError(reqErr error) (string, int) { {ErrExecutionNotRedrivable, "ExecutionNotRedrivable", http.StatusBadRequest}, {ErrInvalidDefinition, "InvalidDefinition", http.StatusBadRequest}, {ErrInvalidExecutionType, "InvalidExecutionType", http.StatusBadRequest}, + {ErrStateMachineTypeNotSupported, "StateMachineTypeNotSupported", http.StatusBadRequest}, {ErrInvalidExecutionInput, "InvalidExecutionInput", http.StatusBadRequest}, {ErrInvalidName, "InvalidName", http.StatusBadRequest}, {ErrInvalidRoleArn, "InvalidArn", http.StatusBadRequest}, diff --git a/services/stepfunctions/models.go b/services/stepfunctions/models.go index 635813d1d..11a6c918f 100644 --- a/services/stepfunctions/models.go +++ b/services/stepfunctions/models.go @@ -64,12 +64,15 @@ type Execution struct { // HistoryEvent represents a single event in execution history. type HistoryEvent struct { - StateEnteredEventDetails *StateEnteredEventDetails `json:"stateEnteredEventDetails,omitempty"` - StateExitedEventDetails *StateExitedEventDetails `json:"stateExitedEventDetails,omitempty"` - Type string `json:"type"` // e.g. "ExecutionStarted", "ExecutionSucceeded" - Timestamp float64 `json:"timestamp"` - ID int64 `json:"id"` - PreviousEventID int64 `json:"previousEventId"` + StateEnteredEventDetails *StateEnteredEventDetails `json:"stateEnteredEventDetails,omitempty"` + StateExitedEventDetails *StateExitedEventDetails `json:"stateExitedEventDetails,omitempty"` + TaskScheduledEventDetails *TaskScheduledEventDetails `json:"taskScheduledEventDetails,omitempty"` + TaskSucceededEventDetails *TaskSucceededEventDetails `json:"taskSucceededEventDetails,omitempty"` + TaskFailedEventDetails *TaskFailedEventDetails `json:"taskFailedEventDetails,omitempty"` + Type string `json:"type"` // e.g. "ExecutionStarted", "ExecutionSucceeded" + Timestamp float64 `json:"timestamp"` + ID int64 `json:"id"` + PreviousEventID int64 `json:"previousEventId"` } // StateEnteredEventDetails holds details for state-entered events. @@ -84,6 +87,24 @@ type StateExitedEventDetails struct { Output string `json:"output,omitempty"` } +// TaskScheduledEventDetails holds details for TaskScheduled history events. +type TaskScheduledEventDetails struct { + Resource string `json:"resource,omitempty"` +} + +// TaskSucceededEventDetails holds details for TaskSucceeded history events. +type TaskSucceededEventDetails struct { + Resource string `json:"resource,omitempty"` + Output string `json:"output,omitempty"` +} + +// TaskFailedEventDetails holds details for TaskFailed history events. +type TaskFailedEventDetails struct { + Error string `json:"error,omitempty"` + Cause string `json:"cause,omitempty"` + Resource string `json:"resource,omitempty"` +} + // Activity represents an AWS Step Functions activity resource. type Activity struct { Name string `json:"name"` diff --git a/services/stepfunctions/new_operations_test.go b/services/stepfunctions/new_operations_test.go index 91a8b2078..ae8daa326 100644 --- a/services/stepfunctions/new_operations_test.go +++ b/services/stepfunctions/new_operations_test.go @@ -280,7 +280,7 @@ func TestStartSyncExecution(t *testing.T) { definition: exprPassDef, input: `{}`, wantErr: true, - errIs: stepfunctions.ErrInvalidExecutionType, + errIs: stepfunctions.ErrStateMachineTypeNotSupported, }, { name: "nonexistent_sm", diff --git a/services/sts/PARITY.md b/services/sts/PARITY.md new file mode 100644 index 000000000..632eefd8c --- /dev/null +++ b/services/sts/PARITY.md @@ -0,0 +1,118 @@ +--- +service: sts +sdk_module: aws-sdk-go-v2/service/sts@v1.43.5 # version audited against (pinned in go.mod) +last_audit_commit: 0407b38d # HEAD when this manifest was written +last_audit_date: 2026-07-05 +overall: B # already-accurate op-by-op; four genuine fixes layered on top of a + # service that has clearly been through multiple prior parity sweeps + # (parity_*_test.go, batch2_audit_test.go, refinement1-4_test.go, + # sdk_completeness_test.go all pre-date this pass). +ops: + AssumeRole: {wire: ok, errors: ok, state: ok, persist: ok, note: "trust-policy Principal/Condition/Effect evaluation, ExternalId, MFA absent (n/a for this op), role-chaining 1h cap, transitive tags, PackedPolicySize — all verified correct pre-existing"} + AssumeRoleWithSAML: {wire: ok, errors: ok, state: ok, persist: ok, note: "base64+temporal Conditions window, NameQualifier BASE64(SHA1(issuer;acct;idp)) per AWS spec, PrincipalArn shape — verified correct"} + AssumeRoleWithWebIdentity: {wire: ok, errors: ok, state: ok, persist: ok, note: "JWT exp/nbf/iat self-consistency checks, OIDC lookup, trust-policy federated evaluation — verified correct"} + AssumeRoot: {wire: ok, errors: ok, state: ok, persist: ok, note: "approved TaskPolicyArn allowlist, fixed 900s duration, arn:aws:sts::ACCT:assumed-root — verified correct"} + GetCallerIdentity: {wire: ok, errors: ok, state: ok, persist: ok, note: "session-token mismatch -> InvalidClientTokenId (not AccessDenied), expired session -> ExpiredTokenException — verified correct"} + GetDelegatedAccessToken: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED this pass: TradeInToken was accepted forever with no expiry check (disguised stub); now validates a JWT-shaped token's exp claim and returns ExpiredTradeInTokenException, matching the real exception AWS documents for this op"} + GetFederationToken: {wire: ok, errors: ok, state: ok, persist: ok, note: "federated-user ARN/ID shape, tag/policy-arn/packed-policy validation — verified correct"} + GetSessionToken: {wire: ok, errors: ok, state: ok, persist: ok, note: "MFA serial+code pairing and format validation, 900-129600s range — verified correct"} + GetWebIdentityToken: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED this pass: (1) SigningAlgorithm accepted 9 values (RS256/RS384/RS512/ES256/ES384/ES512/PS256/PS384/PS512) but the real API only documents RS256 and ES384 as valid — narrowed to match; (2) op was wrongly excluded from GetSupportedOperations/SDK-completeness list on the (now-stale) belief it was a gopherstack-only extension — it is a real op in the pinned SDK (api_op_GetWebIdentityToken.go) and is now listed + counted"} + GetAccessKeyInfo: {wire: ok, errors: ok, state: ok, persist: ok, note: "session lookup then well-formed-prefix fallback to backend account ID — verified correct"} + DecodeAuthorizationMessage: {wire: ok, errors: ok, state: ok, persist: n/a, note: "HMAC-signed self-issued messages verified; foreign base64 blobs decoded permissively for emulator usability — verified correct"} +families: + trust-policy-evaluation: {status: ok, note: "Principal (AWS/Federated/Service/wildcard), Action (incl. wildcard glob), Effect Allow/Deny, Condition (StringEquals/StringLike/NotEquals/NotLike + IfExists, case-insensitive keys) all implemented in trustpolicy.go and independently verified against the statements in AssumeRole/WithSAML/WithWebIdentity — no changes needed"} + session-tag-validation: {status: ok, note: "key/value length, charset, aws: reserved prefix, case-insensitive dup detection, MaxTagCount=50, transitive-tag merge on role chaining — verified correct"} + locking: {status: fixed, note: "InMemoryBackend used a raw sync.Mutex, violating pkgs-catalog.md's rule to use lockmetrics.RWMutex for backend maps. Converted mu to *lockmetrics.RWMutex (New(\"sts\")), split read-only accessors (AccountID, getEffectiveMaxDuration, roleDerivedMaxDuration, lookupRoleMeta, validateOIDCProvider, SessionCounts, Snapshot, GetAccessKeyInfo lookup) to RLock, kept map-mutating paths (storeSession, evictExpiredSessionsLocked callers, LookupSession, GetCallerIdentity, ValidateSessionCredential, Reset, Restore, janitor sweep) on Lock, each with an operation-name label for the gopherstack_lock_* metrics."} +gaps: + - "JWTPayloadSizeExceededException, OutboundWebIdentityFederationDisabledException, SessionDurationEscalationException are real error types in aws-sdk-go-v2/service/sts/types but the SDK ships no doc comment describing their trigger thresholds/semantics beyond the type name; the emulator does not currently model any of the three (no JWT-tag-payload size cap, no account-level web-identity-federation-disabled flag, no explicit escalation-attempt detection beyond the existing silent 1h role-chain clamp). Not fixed this pass — no reliable spec to implement against without further doc research. (bd: gopherstack-p05, follow-up)" +deferred: + - "SESSION-POLICY EVALUATION: session Policy/PolicyArns are validated for shape/size (MalformedPolicyDocument, PackedPolicyTooLarge) and PackedPolicySize is computed, but the policy document's *content* is not enforced against subsequent API calls (no IAM policy-evaluation engine wired to session credentials). This mirrors the rest of the emulator's authz model and is out of scope for a service-local sts audit." + - "GetWebIdentityToken Tags payload-size cap (JWTPayloadSizeExceededException) — see gaps above." +leaks: {status: clean, note: "Sessions map is bounded by (a) the background Janitor (sweepExpiredSessions, default 30s tick) when WithJanitor is configured, and (b) an opportunistic sweep on every storeSession once the map reaches sessionEvictThreshold=256, so unbounded growth cannot occur even with the janitor disabled. Janitor.Run selects on ctx.Done() and its worker.Group is Stop()'d on cancellation — no goroutine leak. No unbounded slices/maps found elsewhere in the package."} +--- + +## Notes + +Freeform findings and traps for the next auditor. + +### Wire-format / protocol +STS is the AWS **query (XML)** protocol (`Version=2011-06-15`), not JSON. Every +response envelope in `models.go` is `XMLName` + `xmlns,attr` + `Result` + +`ResponseMetadata` in that field order — **this order is load-bearing for XML +marshaling** (Go's `encoding/xml` serialises struct fields in declaration order, +and `Result` must precede `` on the wire to match AWS). +**Trap for future `fieldalignment`/govet-driven cleanups**: running +`fieldalignment -fix` (or any tool that reorders struct fields for memory +packing) across the whole package will silently swap `Result` and +`ResponseMetadata` in every response struct in `models.go`, breaking wire +compatibility while still compiling and passing `go vet`/`golangci-lint` +(fieldalignment has no concept of XML field order). This was caught during +this sweep (five `TestBatch2_*_ResultBeforeMetadata` tests failed after a +package-wide `fieldalignment -fix` run) and reverted; **never run a +whole-package `fieldalignment -fix` here — scope it to a single non-wire file +(e.g. `backend.go`) or hand-edit the target struct instead.** + +### Credentials / ID shapes (verified correct, no changes) +- Access key IDs: `ASIA` + 16 upper-alnum chars (`generateAccessKeyID`). +- Assumed-role ARN: `arn:aws:sts::ACCOUNT:assumed-role/ROLE_NAME/SESSION` — note + that an IAM path is **stripped**: `role/team/dev/MyRole` → `assumed-role/MyRole/SESSION`, + only the final path segment survives (`buildAssumedRoleArn`/`roleNameFromResource`). + This is a common trap: naively keeping the full path produces a wire-invalid ARN. +- AssumedRoleId: `AROA` + 16-char derived suffix + `:` + session name + (`deriveRoleID`) — deterministic from the role ARN so repeated `AssumeRole` + calls for the same role produce a stable role-ID prefix. +- `Expiration` fields are RFC3339 strings (`time.RFC3339`), which is correct for + the query/XML protocol (unlike JSON-protocol services, which need + `pkgs/awstime.Epoch()` — STS does **not** want epoch numbers here). + +### GetWebIdentityToken / GetDelegatedAccessToken are real ops, not gopherstack extensions +The pinned `aws-sdk-go-v2/service/sts@v1.43.5` ships +`api_op_GetWebIdentityToken.go` and `api_op_GetDelegatedAccessToken.go` — both +are genuine, documented AWS STS actions (apparently added to the API after an +earlier audit pass assumed otherwise). This sweep found and corrected three +places that encoded the stale "GetWebIdentityToken is not real" belief: +`Handler.GetSupportedOperations()`'s doc comment + missing list entry, +`sdk_completeness_test.go`'s `notImplemented` acknowledgement list, and +`TestParity_GetWebIdentityToken_NotInSupportedOps` (inverted to +`TestParity_GetWebIdentityToken_InSupportedOps`). **If a future SDK bump adds +more ops, re-run `TestSDKCompleteness` first** — it is the authoritative +sentinel for this class of drift (it diffs `GetSupportedOperations()` against +the real SDK client's method set via reflection). + +### GetWebIdentityToken SigningAlgorithm — only two values are valid +`aws-sdk-go-v2/service/sts@v1.43.5`'s doc comment for `SigningAlgorithm` is +explicit: *"Valid values are RS256 (RSA with SHA-256) and ES384 (ECDSA using +P-384 curve with SHA-384)."* The emulator previously accepted a 9-value JOSE +allowlist (RS256/RS384/RS512/ES256/ES384/ES512/PS256/PS384/PS512) inherited +from generic JWT-library conventions rather than the STS API's actual +constraint — narrowed to `{RS256, ES384}` this pass. A test +(`TestRefinement2_GetWebIdentityTokenValidSigningAlgorithms`, renamed to +`TestRefinement2_GetWebIdentityTokenSigningAlgorithms`) had asserted the wrong +(permissive) behaviour as correct; it is now a table of accept/reject cases. + +### GetDelegatedAccessToken TradeInToken — was an unvalidated pass-through +`types.ExpiredTradeInTokenException` exists in the SDK ("The trade-in token +provided in the request has expired and can no longer be exchanged for +credentials") and the input doc comment says the token "must be valid and +unexpired at the time of the request" — yet the handler accepted **any** +non-empty string forever. Fixed by adding `validateTradeInTokenExpiry` +(`tokenvalidation.go`), which — mirroring the existing JWT-exp handling already +used for `WebIdentityToken` — checks the `exp` claim only when the token is +JWT-shaped (three non-empty dot-separated segments); opaque test-fixture tokens +remain accepted unchanged (no external issuer keys are available to verify a +signature either way, matching the emulator's existing stance on +WebIdentityToken/SAMLAssertion). New sentinel `ErrExpiredTradeInToken` maps to +`ExpiredTradeInTokenException` / HTTP 400 in `mapErrorToCode`. + +### Locking +`InMemoryBackend.mu` was a raw `sync.Mutex` (violates +`.claude/memories/pkgs-catalog.md`'s "never scatter raw sync.Mutex — use +lockmetrics.RWMutex" rule); every other audited service backend already uses +`lockmetrics.RWMutex`. Converted with per-call-site operation labels; read-only +accessors use `RLock`/`RUnlock`. No behavioural change — `Lock`/`Unlock` stayed +write-exclusive everywhere a map mutation (session store/delete) occurs. + +### Not touched (shared files / out of scope) +`cli.go`'s `stsBk.SetRoleLookup(...)` / `stsBk.SetOIDCLookup(...)` wiring was +read but not modified — the `RoleLookup`/`OIDCLookup` interfaces in +`backend.go` are unchanged (additive-safe; no signature changes were needed). diff --git a/services/sts/backend.go b/services/sts/backend.go index b0b9fb9e7..38a49dd4f 100644 --- a/services/sts/backend.go +++ b/services/sts/backend.go @@ -13,13 +13,13 @@ import ( "regexp" "slices" "strings" - "sync" "sync/atomic" "time" "github.com/google/uuid" "github.com/blackbirdworks/gopherstack/pkgs/arn" + "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" ) var ( @@ -111,6 +111,10 @@ var ( // ErrExpiredToken is returned when a web-identity JWT has expired. ErrExpiredToken = errors.New("token has expired") + // ErrExpiredTradeInToken is returned when GetDelegatedAccessToken's TradeInToken + // has passed its "exp" claim (maps to the real AWS ExpiredTradeInTokenException). + ErrExpiredTradeInToken = errors.New("trade-in token has expired") + // ErrInvalidIdentityToken is returned when a web-identity JWT is structurally invalid or its claims are wrong. ErrInvalidIdentityToken = errors.New("invalid identity token") @@ -241,7 +245,7 @@ func (b *InMemoryBackend) evictExpiredSessionsLocked() { // critical section from storeSession so that session creation (O(1) map insert) // is never blocked by an O(n) sweep. func (b *InMemoryBackend) maybeEvictExpiredSessions() { - b.mu.Lock() + b.mu.Lock("EvictExpiredSessions") if len(b.sessions) >= sessionEvictThreshold { b.evictExpiredSessionsLocked() } @@ -253,7 +257,7 @@ func (b *InMemoryBackend) maybeEvictExpiredSessions() { // eviction of expired sessions is deferred to a separate lock acquisition so // that the 11 credential-issuing operations do not serialize on O(n) sweeps. func (b *InMemoryBackend) storeSession(accessKeyID string, session *SessionInfo) { - b.mu.Lock() + b.mu.Lock("StoreSession") b.sessions[accessKeyID] = session b.totalSessionsCreated.Add(1) b.mu.Unlock() @@ -334,9 +338,12 @@ func validateFederationTokenName(name string) error { } // isValidWebIdentitySigningAlgorithm reports whether the given signing algorithm is supported. +// Per the real AWS STS GetWebIdentityToken API, only RS256 (RSA with SHA-256) and +// ES384 (ECDSA using the P-384 curve with SHA-384) are valid; any other value +// (including other JOSE algorithms such as ES256 or PS256) is rejected. func isValidWebIdentitySigningAlgorithm(alg string) bool { switch alg { - case "RS256", "RS384", "RS512", "ES256", "ES384", "ES512", "PS256", "PS384", "PS512": + case "RS256", "ES384": return true } @@ -386,18 +393,13 @@ type InMemoryBackend struct { roleLookup RoleLookup oidcLookup OIDCLookup sessions map[string]*SessionInfo + mu *lockmetrics.RWMutex accountID string - mu sync.Mutex - - // authMsgSigningKey is a random key used to HMAC-sign encoded authorization messages. - // Only messages signed with this key are accepted by DecodeAuthorizationMessage, - // matching AWS behaviour where only STS-issued encoded messages can be decoded. - authMsgSigningKey [authMsgHMACSize]byte // Operation call counters — incremented atomically. + cntAssumeRoleWithWebIdentity atomic.Int64 cntAssumeRole atomic.Int64 cntAssumeRoleWithSAML atomic.Int64 - cntAssumeRoleWithWebIdentity atomic.Int64 cntAssumeRoot atomic.Int64 cntGetCallerIdentity atomic.Int64 cntGetDelegatedAccessToken atomic.Int64 @@ -409,6 +411,11 @@ type InMemoryBackend struct { // totalSessionsCreated is the lifetime count of sessions issued. totalSessionsCreated atomic.Int64 + + // authMsgSigningKey is a random key used to HMAC-sign encoded authorization messages. + // Only messages signed with this key are accepted by DecodeAuthorizationMessage, + // matching AWS behaviour where only STS-issued encoded messages can be decoded. + authMsgSigningKey [authMsgHMACSize]byte } // NewInMemoryBackend creates a new InMemoryBackend with the default account ID. @@ -427,13 +434,14 @@ func NewInMemoryBackendWithConfig(accountID string) *InMemoryBackend { accountID: accountID, sessions: make(map[string]*SessionInfo), authMsgSigningKey: key, + mu: lockmetrics.New("sts"), } } // SetRoleLookup wires an optional role-lookup implementation (e.g. the IAM backend) // so that AssumeRole can validate ExternalId and enforce MaxSessionDuration. func (b *InMemoryBackend) SetRoleLookup(rl RoleLookup) { - b.mu.Lock() + b.mu.Lock("SetRoleLookup") defer b.mu.Unlock() b.roleLookup = rl @@ -442,7 +450,7 @@ func (b *InMemoryBackend) SetRoleLookup(rl RoleLookup) { // SetOIDCLookup wires an optional OIDC-lookup implementation (e.g. the IAM backend) // so that AssumeRoleWithWebIdentity can validate that the OIDC provider exists. func (b *InMemoryBackend) SetOIDCLookup(ol OIDCLookup) { - b.mu.Lock() + b.mu.Lock("SetOIDCLookup") defer b.mu.Unlock() b.oidcLookup = ol @@ -450,8 +458,8 @@ func (b *InMemoryBackend) SetOIDCLookup(ol OIDCLookup) { // AccountID returns the AWS account ID configured for this backend. func (b *InMemoryBackend) AccountID() string { - b.mu.Lock() - defer b.mu.Unlock() + b.mu.RLock("AccountID") + defer b.mu.RUnlock() return b.accountID } @@ -552,9 +560,9 @@ func (b *InMemoryBackend) AssumeRole(input *AssumeRoleInput) (*AssumeRoleRespons func (b *InMemoryBackend) getEffectiveMaxDuration(roleArn string) int32 { effectiveMax := int32(MaxDurationSeconds) - b.mu.Lock() + b.mu.RLock("GetEffectiveMaxDuration") rl := b.roleLookup - b.mu.Unlock() + b.mu.RUnlock() if rl == nil { return effectiveMax @@ -591,9 +599,9 @@ func (b *InMemoryBackend) validateAndGetMaxDuration(input *AssumeRoleInput) (int // and validates ExternalId. Returns MaxDurationSeconds when no lookup is configured or the // role is not found. func (b *InMemoryBackend) roleDerivedMaxDuration(input *AssumeRoleInput) (int32, error) { - b.mu.Lock() + b.mu.RLock("RoleDerivedMaxDuration") rl := b.roleLookup - b.mu.Unlock() + b.mu.RUnlock() if rl == nil { return int32(MaxDurationSeconds), nil @@ -620,9 +628,9 @@ func (b *InMemoryBackend) roleDerivedMaxDuration(input *AssumeRoleInput) (int32, // returns an error: a missing role or lookup means the emulator falls back to // permissive behaviour for trust evaluation. func (b *InMemoryBackend) lookupRoleMeta(roleArn string) *RoleMeta { - b.mu.Lock() + b.mu.RLock("LookupRoleMeta") rl := b.roleLookup - b.mu.Unlock() + b.mu.RUnlock() if rl == nil { return nil @@ -788,7 +796,7 @@ func (b *InMemoryBackend) LookupSession(accessKeyID, sessionToken string) *Sessi return nil } - b.mu.Lock() + b.mu.Lock("LookupSession") session, ok := b.sessions[accessKeyID] if ok && isSessionExpired(session) { delete(b.sessions, accessKeyID) @@ -819,7 +827,7 @@ func (b *InMemoryBackend) GetCallerIdentity( return b.rootCallerIdentity(), nil } - b.mu.Lock() + b.mu.Lock("GetCallerIdentity") session, ok := b.sessions[accessKeyID] wasExpired := false @@ -895,7 +903,7 @@ func (b *InMemoryBackend) rootCallerIdentity() *GetCallerIdentityResponse { func (b *InMemoryBackend) ValidateSessionCredential( accessKeyID, sessionToken string, ) (*SessionInfo, error) { - b.mu.Lock() + b.mu.Lock("ValidateSessionCredential") session, ok := b.sessions[accessKeyID] if ok && isSessionExpired(session) { @@ -1172,9 +1180,9 @@ func (b *InMemoryBackend) AssumeRoleWithWebIdentity( // corresponds to an existing OIDC provider in the IAM backend. // When no OIDCLookup is configured the check is skipped (permissive mock behaviour). func (b *InMemoryBackend) validateOIDCProvider(token, providerID string) error { - b.mu.Lock() + b.mu.RLock("ValidateOIDCProvider") ol := b.oidcLookup - b.mu.Unlock() + b.mu.RUnlock() if ol == nil { return nil @@ -1519,7 +1527,11 @@ func (b *InMemoryBackend) AssumeRoot(input *AssumeRootInput) (*AssumeRootRespons } // GetDelegatedAccessToken exchanges a trade-in token for temporary AWS credentials. -// In this mock, the TradeInToken is accepted as-is without validation. +// The TradeInToken's cryptographic signature is not verified (the external issuer's +// keys are unavailable to the emulator), but a JWT-shaped token's self-consistent +// "exp" claim is checked so an already-expired token is rejected with +// ErrExpiredTradeInToken (AWS ExpiredTradeInTokenException), matching real STS +// behaviour instead of accepting any non-empty string indefinitely. func (b *InMemoryBackend) GetDelegatedAccessToken( input *GetDelegatedAccessTokenInput, ) (*GetDelegatedAccessTokenResponse, error) { @@ -1529,6 +1541,10 @@ func (b *InMemoryBackend) GetDelegatedAccessToken( return nil, ErrMissingTradeInToken } + if err := validateTradeInTokenExpiry(input.TradeInToken); err != nil { + return nil, err + } + duration := input.DurationSeconds if duration == 0 { duration = DefaultDurationSeconds @@ -2073,7 +2089,7 @@ func (b *InMemoryBackend) VerifyEncodedAuthorizationMessage(encoded string) (str // POST /_gopherstack/reset endpoint for CI pipelines and rapid local development. // Operation counters and totalSessionsCreated are also reset to zero. func (b *InMemoryBackend) Reset() { - b.mu.Lock() + b.mu.Lock("Reset") b.sessions = make(map[string]*SessionInfo) b.mu.Unlock() @@ -2093,8 +2109,8 @@ func (b *InMemoryBackend) Reset() { // SessionCounts returns active and expired session counts at the time of invocation. func (b *InMemoryBackend) SessionCounts() (int, int) { - b.mu.Lock() - defer b.mu.Unlock() + b.mu.RLock("SessionCounts") + defer b.mu.RUnlock() now := time.Now().UTC() active := 0 diff --git a/services/sts/export_test.go b/services/sts/export_test.go index 299f01a93..e01d3a114 100644 --- a/services/sts/export_test.go +++ b/services/sts/export_test.go @@ -60,8 +60,8 @@ const SessionEvictThreshold = sessionEvictThreshold // SessionCount returns the number of sessions currently stored in the backend. // Used in tests to verify janitor eviction. func (b *InMemoryBackend) SessionCount() int { - b.mu.Lock() - defer b.mu.Unlock() + b.mu.RLock("SessionCount") + defer b.mu.RUnlock() return len(b.sessions) } @@ -69,7 +69,7 @@ func (b *InMemoryBackend) SessionCount() int { // SetSessionExpiration overrides the expiration of the session identified by // accessKeyID. Used in tests to fast-forward session expiry without waiting. func (b *InMemoryBackend) SetSessionExpiration(accessKeyID string, exp time.Time) { - b.mu.Lock() + b.mu.Lock("SetSessionExpiration") defer b.mu.Unlock() if s, ok := b.sessions[accessKeyID]; ok { @@ -101,7 +101,7 @@ func (h *Handler) HandlerOpsLen() int { // AddSessionInternal inserts a session directly into the backend for test seeding. func (b *InMemoryBackend) AddSessionInternal(session *SessionInfo) { - b.mu.Lock() + b.mu.Lock("AddSessionInternal") defer b.mu.Unlock() b.sessions[session.AccessKeyID] = session diff --git a/services/sts/handler.go b/services/sts/handler.go index 908b6fb71..206b1db37 100644 --- a/services/sts/handler.go +++ b/services/sts/handler.go @@ -84,8 +84,10 @@ func (h *Handler) Name() string { } // GetSupportedOperations returns the list of supported STS operations. -// Note: GetWebIdentityToken is an internal gopherstack operation and is NOT a real -// AWS STS API action; it is intentionally omitted from this list. +// GetDelegatedAccessToken and GetWebIdentityToken are real actions present in +// aws-sdk-go-v2/service/sts (api_op_GetDelegatedAccessToken.go and +// api_op_GetWebIdentityToken.go); both are included here and are routed by +// dispatch. func (h *Handler) GetSupportedOperations() []string { return []string{ "AssumeRole", @@ -98,6 +100,7 @@ func (h *Handler) GetSupportedOperations() []string { "GetDelegatedAccessToken", "GetFederationToken", "GetSessionToken", + "GetWebIdentityToken", } } @@ -526,9 +529,9 @@ func (h *Handler) dispatchGetAccessKeyInfo(r *http.Request) (*GetAccessKeyInfoRe // Look up the key in the session store. b, ok := h.Backend.(*InMemoryBackend) if ok { - b.mu.Lock() + b.mu.RLock("GetAccessKeyInfo") session, found := b.sessions[accessKeyID] - b.mu.Unlock() + b.mu.RUnlock() if found { return &GetAccessKeyInfoResponse{ @@ -638,6 +641,8 @@ func mapErrorToCode(reqErr error) (string, int) { return "PackedPolicyTooLarge", http.StatusBadRequest case errors.Is(reqErr, ErrExpiredToken), errors.Is(reqErr, ErrSessionExpired): return "ExpiredTokenException", http.StatusBadRequest + case errors.Is(reqErr, ErrExpiredTradeInToken): + return "ExpiredTradeInTokenException", http.StatusBadRequest case errors.Is(reqErr, ErrInvalidIdentityToken), errors.Is(reqErr, ErrInvalidSAMLAssertion): return "InvalidIdentityToken", http.StatusBadRequest case errors.Is(reqErr, ErrInvalidAuthorizationMessage): diff --git a/services/sts/handler_refinement1_test.go b/services/sts/handler_refinement1_test.go index 7bf27424b..e087847a4 100644 --- a/services/sts/handler_refinement1_test.go +++ b/services/sts/handler_refinement1_test.go @@ -52,9 +52,9 @@ func TestRefinement1_HandlerOpsLen(t *testing.T) { b := sts.NewInMemoryBackend() h := sts.NewHandler(b) - // GetWebIdentityToken is an internal gopherstack extension, not a real AWS STS action, - // so it is excluded from GetSupportedOperations (10 real AWS operations remain). - assert.Equal(t, 10, h.HandlerOpsLen()) + // 11 real AWS STS operations are supported, including GetDelegatedAccessToken + // and GetWebIdentityToken (both present in aws-sdk-go-v2/service/sts). + assert.Equal(t, 11, h.HandlerOpsLen()) } // TestRefinement1_SDKOpsSorted verifies GetSupportedOperations is sorted. diff --git a/services/sts/handler_refinement2_test.go b/services/sts/handler_refinement2_test.go index 3bd9f12b6..a9886b610 100644 --- a/services/sts/handler_refinement2_test.go +++ b/services/sts/handler_refinement2_test.go @@ -379,19 +379,29 @@ func TestRefinement2_GetWebIdentityTokenInvalidSigningAlgo(t *testing.T) { assert.ErrorIs(t, err, sts.ErrValidation) } -// TestRefinement2_GetWebIdentityTokenValidSigningAlgorithms verifies RS256/ES256/PS256 all succeed. -func TestRefinement2_GetWebIdentityTokenValidSigningAlgorithms(t *testing.T) { +// TestRefinement2_GetWebIdentityTokenSigningAlgorithms verifies that only the two +// algorithms the real AWS STS GetWebIdentityToken API documents as valid — RS256 +// (RSA with SHA-256) and ES384 (ECDSA P-384 with SHA-384) — are accepted; every +// other JOSE algorithm name (even ones valid for other JWT use cases, such as +// ES256 or PS256) must be rejected with ValidationError. Corrected from a prior +// version of this test that asserted the emulator's old (overly permissive) +// nine-algorithm allowlist, which did not match the real API's documented +// two-algorithm constraint. +func TestRefinement2_GetWebIdentityTokenSigningAlgorithms(t *testing.T) { t.Parallel() tests := []struct { - name string - alg string + name string + alg string + wantErr bool }{ - {name: "RS256", alg: "RS256"}, - {name: "ES256", alg: "ES256"}, - {name: "PS256", alg: "PS256"}, - {name: "RS384", alg: "RS384"}, - {name: "RS512", alg: "RS512"}, + {name: "RS256_valid", alg: "RS256", wantErr: false}, + {name: "ES384_valid", alg: "ES384", wantErr: false}, + {name: "ES256_rejected", alg: "ES256", wantErr: true}, + {name: "PS256_rejected", alg: "PS256", wantErr: true}, + {name: "RS384_rejected", alg: "RS384", wantErr: true}, + {name: "RS512_rejected", alg: "RS512", wantErr: true}, + {name: "HS256_rejected", alg: "HS256", wantErr: true}, } for _, tt := range tests { @@ -404,6 +414,13 @@ func TestRefinement2_GetWebIdentityTokenValidSigningAlgorithms(t *testing.T) { SigningAlgorithm: tt.alg, }) + if tt.wantErr { + require.Error(t, err) + assert.ErrorIs(t, err, sts.ErrValidation) + + return + } + require.NoError(t, err) assert.NotEmpty(t, resp.GetWebIdentityTokenResult.WebIdentityToken) }) diff --git a/services/sts/janitor.go b/services/sts/janitor.go index 2b753fe75..7475d187e 100644 --- a/services/sts/janitor.go +++ b/services/sts/janitor.go @@ -67,7 +67,7 @@ func (j *Janitor) SweepOnce(ctx context.Context) { func (j *Janitor) sweepExpiredSessions(ctx context.Context) { b := j.Backend - b.mu.Lock() + b.mu.Lock("SessionSweep") var expired []string diff --git a/services/sts/new_ops2_test.go b/services/sts/new_ops2_test.go index 80e2a370b..ceec4efe4 100644 --- a/services/sts/new_ops2_test.go +++ b/services/sts/new_ops2_test.go @@ -6,6 +6,7 @@ import ( "net/url" "strings" "testing" + "time" "github.com/labstack/echo/v5" "github.com/stretchr/testify/assert" @@ -496,6 +497,73 @@ func TestGetDelegatedAccessToken_ValidationErrors(t *testing.T) { } } +// TestGetDelegatedAccessToken_TradeInTokenExpiry verifies that a JWT-shaped +// TradeInToken is checked for expiry (AWS ExpiredTradeInTokenException), +// mirroring the real GetDelegatedAccessToken behaviour documented by +// aws-sdk-go-v2/service/sts/types.ExpiredTradeInTokenException: "The trade-in +// token provided in the request has expired". Opaque (non-JWT) tokens remain +// accepted unchanged, matching the emulator's existing permissive handling of +// simple test fixtures that have no verifiable structure. +func TestGetDelegatedAccessToken_TradeInTokenExpiry(t *testing.T) { + t.Parallel() + + tests := []struct { + wantErr error + buildToken func(t *testing.T) string + name string + }{ + { + name: "expired_jwt_shaped_token_rejected", + buildToken: func(t *testing.T) string { + t.Helper() + + return buildJWT(t, map[string]any{ + "sub": "delegate", + "exp": time.Now().Add(-time.Hour).Unix(), + }) + }, + wantErr: sts.ErrExpiredTradeInToken, + }, + { + name: "unexpired_jwt_shaped_token_accepted", + buildToken: func(t *testing.T) string { + t.Helper() + + return buildJWT(t, map[string]any{ + "sub": "delegate", + "exp": time.Now().Add(time.Hour).Unix(), + }) + }, + }, + { + name: "opaque_non_jwt_token_accepted", + buildToken: func(*testing.T) string { + return "opaque-trade-in-token-without-dots" + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := sts.NewInMemoryBackend() + resp, err := b.GetDelegatedAccessToken(&sts.GetDelegatedAccessTokenInput{ + TradeInToken: tt.buildToken(t), + }) + + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + + return + } + + require.NoError(t, err) + assert.NotEmpty(t, resp.GetDelegatedAccessTokenResult.Credentials.AccessKeyID) + }) + } +} + func TestGetDelegatedAccessToken_SessionTrackedForCallerIdentity(t *testing.T) { t.Parallel() diff --git a/services/sts/parity_fixes_test.go b/services/sts/parity_fixes_test.go index ca6289908..2721d2ce4 100644 --- a/services/sts/parity_fixes_test.go +++ b/services/sts/parity_fixes_test.go @@ -179,23 +179,27 @@ func TestParity_AssumeRoleWithSAML_SAMLAssertionViaHandler(t *testing.T) { } } -// ── Parity Fix 3: GetWebIdentityToken not in GetSupportedOperations ────────────── - -func TestParity_GetWebIdentityToken_NotInSupportedOps(t *testing.T) { +// ── Parity Fix 3: GetWebIdentityToken IS a real AWS STS operation ──────────────── +// +// A previous sweep incorrectly believed GetWebIdentityToken was a gopherstack-only +// extension and asserted it must be absent from GetSupportedOperations. The pinned +// aws-sdk-go-v2/service/sts module ships api_op_GetWebIdentityToken.go, confirming +// it is a real, documented AWS STS action — so it belongs in the supported list +// (and is routed by dispatch). This test is corrected to assert presence. + +func TestParity_GetWebIdentityToken_InSupportedOps(t *testing.T) { t.Parallel() b := sts.NewInMemoryBackend() h := sts.NewHandler(b) ops := h.GetSupportedOperations() - for _, op := range ops { - assert.NotEqual( - t, - "GetWebIdentityToken", - op, - "GetWebIdentityToken is not a real AWS STS operation and must not be in GetSupportedOperations", - ) - } + assert.Contains( + t, + ops, + "GetWebIdentityToken", + "GetWebIdentityToken is a real AWS STS operation and must be listed in GetSupportedOperations", + ) } // ── Parity Fix 4: DecodeAuthorizationMessage — only STS-issued messages accepted ── diff --git a/services/sts/persistence.go b/services/sts/persistence.go index 6697f54b7..120a85376 100644 --- a/services/sts/persistence.go +++ b/services/sts/persistence.go @@ -22,8 +22,8 @@ type backendSnapshot struct { // Snapshot serialises the backend state to JSON. // It implements persistence.Persistable. func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { - b.mu.Lock() - defer b.mu.Unlock() + b.mu.RLock("Snapshot") + defer b.mu.RUnlock() sessions := make(map[string]*SessionInfo, len(b.sessions)) for k, v := range b.sessions { @@ -76,7 +76,7 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { now := time.Now().UTC() - b.mu.Lock() + b.mu.Lock("Restore") b.ensureNonNilMaps() b.sessions = make(map[string]*SessionInfo) diff --git a/services/sts/sdk_completeness_test.go b/services/sts/sdk_completeness_test.go index 68b74faa3..7a32172c8 100644 --- a/services/sts/sdk_completeness_test.go +++ b/services/sts/sdk_completeness_test.go @@ -18,7 +18,10 @@ func TestSDKCompleteness(t *testing.T) { backend := sts.NewInMemoryBackend() h := sts.NewHandler(backend) - // GetWebIdentityToken is an internal gopherstack extension, not a real AWS STS API action. - notImplemented := []string{"GetWebIdentityToken"} + // GetWebIdentityToken and GetDelegatedAccessToken are real, routed AWS STS + // operations (present in aws-sdk-go-v2/service/sts as of the pinned SDK + // version) — both are listed in GetSupportedOperations, so there is nothing + // left to acknowledge as not-yet-implemented. + var notImplemented []string sdkcheck.CheckCompleteness(t, &stssdk.Client{}, h.GetSupportedOperations(), notImplemented) } diff --git a/services/sts/tokenvalidation.go b/services/sts/tokenvalidation.go index 43b5e9e9e..39f358fb5 100644 --- a/services/sts/tokenvalidation.go +++ b/services/sts/tokenvalidation.go @@ -89,6 +89,46 @@ func checkJWTWellFormed(token string) error { return nil } +// validateTradeInTokenExpiry checks the self-consistent "exp" claim of a +// JWT-shaped GetDelegatedAccessToken TradeInToken, without verifying its +// cryptographic signature (the external issuer's signing keys are not +// available to the emulator — the same constraint that applies to +// WebIdentityToken and SAMLAssertion validation above). Opaque, non-JWT tokens +// (fixtures without JWT structure) are accepted unchanged, matching the mock's +// existing permissive handling of such test tokens. A JWT-shaped token whose +// exp claim has already passed is rejected with ErrExpiredTradeInToken, which +// the handler maps to the real AWS ExpiredTradeInTokenException error code. +func validateTradeInTokenExpiry(token string) error { + parts := strings.Split(token, ".") + if len(parts) != jwtFullPartCount || slices.Contains(parts, "") { + // Not JWT-shaped: nothing to validate. + return nil + } + + claims := parseJWTPayloadClaims(token) + if claims == nil { + return nil + } + + exp, ok, err := jwtTimeClaim(claims, jwtClaimExp) + if err != nil { + return err + } + + if !ok { + return nil + } + + if time.Now().UTC().After(exp) { + return fmt.Errorf( + "%w: TradeInToken exp claim is %s", + ErrExpiredTradeInToken, exp.Format(time.RFC3339), + ) + } + + return nil +} + // jwtTimeClaim extracts a NumericDate JWT claim (seconds since the Unix epoch) // as a UTC time. The second result reports presence; an error is returned only // when the claim is present but not a parseable number. diff --git a/ui/.oxlintrc.json b/ui/.oxlintrc.json index 706040632..ffe3d235c 100644 --- a/ui/.oxlintrc.json +++ b/ui/.oxlintrc.json @@ -13,6 +13,7 @@ "unicorn/no-empty-file": "off", "unicorn/require-module-specifiers": "off", "unicorn/prefer-add-event-listener": "off", + "unicorn/prefer-number-coercion": "off", "no-alert": "error", "max-lines-per-function": "off", "max-lines": "off", diff --git a/ui/package-lock.json b/ui/package-lock.json index 752ce42f4..918874542 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -8,166 +8,166 @@ "name": "ui", "version": "0.0.1", "dependencies": { - "@aws-sdk/client-accessanalyzer": "3.1070.0", - "@aws-sdk/client-account": "3.1070.0", - "@aws-sdk/client-acm": "3.1070.0", - "@aws-sdk/client-acm-pca": "3.1070.0", - "@aws-sdk/client-amplify": "3.1070.0", - "@aws-sdk/client-api-gateway": "3.1070.0", - "@aws-sdk/client-apigatewaymanagementapi": "3.1070.0", - "@aws-sdk/client-apigatewayv2": "3.1070.0", - "@aws-sdk/client-app-mesh": "3.1070.0", - "@aws-sdk/client-appconfig": "3.1070.0", - "@aws-sdk/client-appfabric": "3.1070.0", - "@aws-sdk/client-application-auto-scaling": "3.1070.0", - "@aws-sdk/client-apprunner": "3.1070.0", - "@aws-sdk/client-appstream": "3.1070.0", - "@aws-sdk/client-appsync": "3.1070.0", - "@aws-sdk/client-athena": "3.1070.0", - "@aws-sdk/client-auto-scaling": "3.1070.0", - "@aws-sdk/client-backup": "3.1070.0", - "@aws-sdk/client-batch": "3.1070.0", - "@aws-sdk/client-bedrock": "3.1070.0", - "@aws-sdk/client-bedrock-runtime": "3.1070.0", - "@aws-sdk/client-cloudcontrol": "3.1070.0", - "@aws-sdk/client-cloudformation": "3.1070.0", - "@aws-sdk/client-cloudfront": "3.1070.0", - "@aws-sdk/client-cloudtrail": "3.1070.0", - "@aws-sdk/client-cloudwatch": "3.1070.0", - "@aws-sdk/client-cloudwatch-logs": "3.1070.0", - "@aws-sdk/client-codeartifact": "3.1070.0", - "@aws-sdk/client-codebuild": "3.1070.0", - "@aws-sdk/client-codecommit": "3.1070.0", - "@aws-sdk/client-codeconnections": "3.1070.0", - "@aws-sdk/client-codedeploy": "3.1070.0", - "@aws-sdk/client-codepipeline": "3.1070.0", - "@aws-sdk/client-codestar-connections": "3.1070.0", - "@aws-sdk/client-cognito-identity": "3.1070.0", - "@aws-sdk/client-cognito-identity-provider": "3.1070.0", - "@aws-sdk/client-comprehend": "3.1070.0", - "@aws-sdk/client-config-service": "3.1070.0", - "@aws-sdk/client-cost-explorer": "3.1070.0", - "@aws-sdk/client-database-migration-service": "3.1070.0", - "@aws-sdk/client-databrew": "3.1070.0", - "@aws-sdk/client-datasync": "3.1070.0", - "@aws-sdk/client-dax": "3.1070.0", - "@aws-sdk/client-detective": "3.1070.0", - "@aws-sdk/client-direct-connect": "3.1070.0", - "@aws-sdk/client-directory-service": "3.1070.0", - "@aws-sdk/client-dlm": "3.1070.0", - "@aws-sdk/client-docdb": "3.1070.0", - "@aws-sdk/client-dynamodb": "3.1070.0", - "@aws-sdk/client-dynamodb-streams": "3.1070.0", - "@aws-sdk/client-ebs": "3.1070.0", - "@aws-sdk/client-ec2": "3.1070.0", - "@aws-sdk/client-ecr": "3.1070.0", - "@aws-sdk/client-ecs": "3.1070.0", - "@aws-sdk/client-efs": "3.1070.0", - "@aws-sdk/client-eks": "3.1070.0", - "@aws-sdk/client-elastic-beanstalk": "3.1070.0", - "@aws-sdk/client-elastic-load-balancing": "3.1070.0", - "@aws-sdk/client-elastic-load-balancing-v2": "3.1070.0", - "@aws-sdk/client-elasticache": "3.1070.0", - "@aws-sdk/client-elasticsearch-service": "3.1070.0", - "@aws-sdk/client-emr": "3.1070.0", - "@aws-sdk/client-emr-serverless": "3.1070.0", - "@aws-sdk/client-eventbridge": "3.1070.0", - "@aws-sdk/client-firehose": "3.1070.0", - "@aws-sdk/client-fis": "3.1070.0", - "@aws-sdk/client-forecast": "3.1070.0", - "@aws-sdk/client-fsx": "3.1070.0", - "@aws-sdk/client-glacier": "3.1070.0", - "@aws-sdk/client-global-accelerator": "3.1070.0", - "@aws-sdk/client-glue": "3.1070.0", - "@aws-sdk/client-grafana": "3.1070.0", - "@aws-sdk/client-guardduty": "3.1070.0", - "@aws-sdk/client-iam": "3.1070.0", - "@aws-sdk/client-identitystore": "3.1070.0", - "@aws-sdk/client-inspector2": "3.1070.0", - "@aws-sdk/client-iot": "3.1070.0", - "@aws-sdk/client-iot-data-plane": "3.1070.0", - "@aws-sdk/client-iot-wireless": "3.1070.0", + "@aws-sdk/client-accessanalyzer": "3.1079.0", + "@aws-sdk/client-account": "3.1079.0", + "@aws-sdk/client-acm": "3.1079.0", + "@aws-sdk/client-acm-pca": "3.1079.0", + "@aws-sdk/client-amplify": "3.1079.0", + "@aws-sdk/client-api-gateway": "3.1079.0", + "@aws-sdk/client-apigatewaymanagementapi": "3.1079.0", + "@aws-sdk/client-apigatewayv2": "3.1079.0", + "@aws-sdk/client-app-mesh": "3.1079.0", + "@aws-sdk/client-appconfig": "3.1079.0", + "@aws-sdk/client-appfabric": "3.1079.0", + "@aws-sdk/client-application-auto-scaling": "3.1079.0", + "@aws-sdk/client-apprunner": "3.1079.0", + "@aws-sdk/client-appstream": "3.1079.0", + "@aws-sdk/client-appsync": "3.1079.0", + "@aws-sdk/client-athena": "3.1079.0", + "@aws-sdk/client-auto-scaling": "3.1079.0", + "@aws-sdk/client-backup": "3.1079.0", + "@aws-sdk/client-batch": "3.1079.0", + "@aws-sdk/client-bedrock": "3.1079.0", + "@aws-sdk/client-bedrock-runtime": "3.1079.0", + "@aws-sdk/client-cloudcontrol": "3.1079.0", + "@aws-sdk/client-cloudformation": "3.1079.0", + "@aws-sdk/client-cloudfront": "3.1079.0", + "@aws-sdk/client-cloudtrail": "3.1079.0", + "@aws-sdk/client-cloudwatch": "3.1079.0", + "@aws-sdk/client-cloudwatch-logs": "3.1079.0", + "@aws-sdk/client-codeartifact": "3.1079.0", + "@aws-sdk/client-codebuild": "3.1079.0", + "@aws-sdk/client-codecommit": "3.1079.0", + "@aws-sdk/client-codeconnections": "3.1079.0", + "@aws-sdk/client-codedeploy": "3.1079.0", + "@aws-sdk/client-codepipeline": "3.1079.0", + "@aws-sdk/client-codestar-connections": "3.1079.0", + "@aws-sdk/client-cognito-identity": "3.1079.0", + "@aws-sdk/client-cognito-identity-provider": "3.1079.0", + "@aws-sdk/client-comprehend": "3.1079.0", + "@aws-sdk/client-config-service": "3.1079.0", + "@aws-sdk/client-cost-explorer": "3.1079.0", + "@aws-sdk/client-database-migration-service": "3.1079.0", + "@aws-sdk/client-databrew": "3.1079.0", + "@aws-sdk/client-datasync": "3.1079.0", + "@aws-sdk/client-dax": "3.1079.0", + "@aws-sdk/client-detective": "3.1079.0", + "@aws-sdk/client-direct-connect": "3.1079.0", + "@aws-sdk/client-directory-service": "3.1079.0", + "@aws-sdk/client-dlm": "3.1079.0", + "@aws-sdk/client-docdb": "3.1079.0", + "@aws-sdk/client-dynamodb": "3.1079.0", + "@aws-sdk/client-dynamodb-streams": "3.1079.0", + "@aws-sdk/client-ebs": "3.1079.0", + "@aws-sdk/client-ec2": "3.1079.0", + "@aws-sdk/client-ecr": "3.1079.0", + "@aws-sdk/client-ecs": "3.1079.0", + "@aws-sdk/client-efs": "3.1079.0", + "@aws-sdk/client-eks": "3.1079.0", + "@aws-sdk/client-elastic-beanstalk": "3.1079.0", + "@aws-sdk/client-elastic-load-balancing": "3.1079.0", + "@aws-sdk/client-elastic-load-balancing-v2": "3.1079.0", + "@aws-sdk/client-elasticache": "3.1079.0", + "@aws-sdk/client-elasticsearch-service": "3.1079.0", + "@aws-sdk/client-emr": "3.1079.0", + "@aws-sdk/client-emr-serverless": "3.1079.0", + "@aws-sdk/client-eventbridge": "3.1079.0", + "@aws-sdk/client-firehose": "3.1079.0", + "@aws-sdk/client-fis": "3.1079.0", + "@aws-sdk/client-forecast": "3.1079.0", + "@aws-sdk/client-fsx": "3.1079.0", + "@aws-sdk/client-glacier": "3.1079.0", + "@aws-sdk/client-global-accelerator": "3.1079.0", + "@aws-sdk/client-glue": "3.1079.0", + "@aws-sdk/client-grafana": "3.1079.0", + "@aws-sdk/client-guardduty": "3.1079.0", + "@aws-sdk/client-iam": "3.1079.0", + "@aws-sdk/client-identitystore": "3.1079.0", + "@aws-sdk/client-inspector2": "3.1079.0", + "@aws-sdk/client-iot": "3.1079.0", + "@aws-sdk/client-iot-data-plane": "3.1079.0", + "@aws-sdk/client-iot-wireless": "3.1079.0", "@aws-sdk/client-iotanalytics": "3.986.0", - "@aws-sdk/client-kafka": "3.1070.0", - "@aws-sdk/client-keyspaces": "3.1070.0", - "@aws-sdk/client-kinesis": "3.1070.0", - "@aws-sdk/client-kinesis-analytics": "3.1070.0", - "@aws-sdk/client-kinesis-analytics-v2": "3.1070.0", - "@aws-sdk/client-kinesis-video": "3.1070.0", - "@aws-sdk/client-kms": "3.1070.0", - "@aws-sdk/client-lakeformation": "3.1070.0", - "@aws-sdk/client-lambda": "3.1070.0", - "@aws-sdk/client-lightsail": "3.1070.0", - "@aws-sdk/client-macie2": "3.1070.0", - "@aws-sdk/client-managedblockchain": "3.1070.0", - "@aws-sdk/client-mediaconvert": "3.1070.0", - "@aws-sdk/client-medialive": "3.1070.0", - "@aws-sdk/client-mediapackage": "3.1070.0", - "@aws-sdk/client-mediastore": "3.1070.0", - "@aws-sdk/client-mediastore-data": "3.1070.0", - "@aws-sdk/client-mediatailor": "3.1070.0", - "@aws-sdk/client-memorydb": "3.1070.0", - "@aws-sdk/client-mgn": "3.1070.0", - "@aws-sdk/client-mq": "3.1070.0", - "@aws-sdk/client-mwaa": "3.1070.0", - "@aws-sdk/client-neptune": "3.1070.0", - "@aws-sdk/client-networkmanager": "3.1070.0", - "@aws-sdk/client-opensearch": "3.1070.0", - "@aws-sdk/client-organizations": "3.1070.0", - "@aws-sdk/client-outposts": "3.1070.0", - "@aws-sdk/client-personalize": "3.1070.0", - "@aws-sdk/client-pinpoint": "3.1070.0", - "@aws-sdk/client-pipes": "3.1070.0", - "@aws-sdk/client-polly": "3.1070.0", - "@aws-sdk/client-quicksight": "3.1070.0", - "@aws-sdk/client-ram": "3.1070.0", - "@aws-sdk/client-rds": "3.1070.0", - "@aws-sdk/client-rds-data": "3.1070.0", - "@aws-sdk/client-redshift": "3.1070.0", - "@aws-sdk/client-redshift-data": "3.1070.0", - "@aws-sdk/client-rekognition": "3.1070.0", - "@aws-sdk/client-resiliencehub": "3.1070.0", - "@aws-sdk/client-resource-groups": "3.1070.0", - "@aws-sdk/client-resource-groups-tagging-api": "3.1070.0", - "@aws-sdk/client-rolesanywhere": "3.1070.0", - "@aws-sdk/client-route-53": "3.1070.0", - "@aws-sdk/client-route53resolver": "3.1070.0", - "@aws-sdk/client-s3": "3.1070.0", - "@aws-sdk/client-s3-control": "3.1070.0", - "@aws-sdk/client-s3tables": "3.1070.0", - "@aws-sdk/client-sagemaker": "3.1070.0", - "@aws-sdk/client-sagemaker-runtime": "3.1070.0", - "@aws-sdk/client-scheduler": "3.1070.0", - "@aws-sdk/client-secrets-manager": "3.1070.0", - "@aws-sdk/client-securityhub": "3.1070.0", - "@aws-sdk/client-serverlessapplicationrepository": "3.1070.0", - "@aws-sdk/client-servicediscovery": "3.1070.0", - "@aws-sdk/client-ses": "3.1070.0", - "@aws-sdk/client-sesv2": "3.1070.0", - "@aws-sdk/client-sfn": "3.1070.0", - "@aws-sdk/client-shield": "3.1070.0", - "@aws-sdk/client-sns": "3.1070.0", - "@aws-sdk/client-sqs": "3.1070.0", - "@aws-sdk/client-ssm": "3.1070.0", - "@aws-sdk/client-sso-admin": "3.1070.0", - "@aws-sdk/client-sts": "3.1070.0", - "@aws-sdk/client-support": "3.1070.0", - "@aws-sdk/client-swf": "3.1070.0", - "@aws-sdk/client-textract": "3.1070.0", - "@aws-sdk/client-timestream-query": "3.1070.0", - "@aws-sdk/client-timestream-write": "3.1070.0", - "@aws-sdk/client-transcribe": "3.1070.0", - "@aws-sdk/client-transfer": "3.1070.0", - "@aws-sdk/client-translate": "3.1070.0", - "@aws-sdk/client-verifiedpermissions": "3.1070.0", - "@aws-sdk/client-wafv2": "3.1070.0", - "@aws-sdk/client-workmail": "3.1070.0", - "@aws-sdk/client-workspaces": "3.1070.0", - "@aws-sdk/client-xray": "3.1070.0", - "@aws-sdk/credential-providers": "3.1070.0", - "@bufbuild/protobuf": "1.10.0", - "@connectrpc/connect": "1.6.1", - "@connectrpc/connect-web": "1.6.1", + "@aws-sdk/client-kafka": "3.1079.0", + "@aws-sdk/client-keyspaces": "3.1079.0", + "@aws-sdk/client-kinesis": "3.1079.0", + "@aws-sdk/client-kinesis-analytics": "3.1079.0", + "@aws-sdk/client-kinesis-analytics-v2": "3.1079.0", + "@aws-sdk/client-kinesis-video": "3.1079.0", + "@aws-sdk/client-kms": "3.1079.0", + "@aws-sdk/client-lakeformation": "3.1079.0", + "@aws-sdk/client-lambda": "3.1079.0", + "@aws-sdk/client-lightsail": "3.1079.0", + "@aws-sdk/client-macie2": "3.1079.0", + "@aws-sdk/client-managedblockchain": "3.1079.0", + "@aws-sdk/client-mediaconvert": "3.1079.0", + "@aws-sdk/client-medialive": "3.1079.0", + "@aws-sdk/client-mediapackage": "3.1079.0", + "@aws-sdk/client-mediastore": "3.1079.0", + "@aws-sdk/client-mediastore-data": "3.1079.0", + "@aws-sdk/client-mediatailor": "3.1079.0", + "@aws-sdk/client-memorydb": "3.1079.0", + "@aws-sdk/client-mgn": "3.1079.0", + "@aws-sdk/client-mq": "3.1079.0", + "@aws-sdk/client-mwaa": "3.1079.0", + "@aws-sdk/client-neptune": "3.1079.0", + "@aws-sdk/client-networkmanager": "3.1079.0", + "@aws-sdk/client-opensearch": "3.1079.0", + "@aws-sdk/client-organizations": "3.1079.0", + "@aws-sdk/client-outposts": "3.1079.0", + "@aws-sdk/client-personalize": "3.1079.0", + "@aws-sdk/client-pinpoint": "3.1079.0", + "@aws-sdk/client-pipes": "3.1079.0", + "@aws-sdk/client-polly": "3.1079.0", + "@aws-sdk/client-quicksight": "3.1079.0", + "@aws-sdk/client-ram": "3.1079.0", + "@aws-sdk/client-rds": "3.1079.0", + "@aws-sdk/client-rds-data": "3.1079.0", + "@aws-sdk/client-redshift": "3.1079.0", + "@aws-sdk/client-redshift-data": "3.1079.0", + "@aws-sdk/client-rekognition": "3.1079.0", + "@aws-sdk/client-resiliencehub": "3.1079.0", + "@aws-sdk/client-resource-groups": "3.1079.0", + "@aws-sdk/client-resource-groups-tagging-api": "3.1079.0", + "@aws-sdk/client-rolesanywhere": "3.1079.0", + "@aws-sdk/client-route-53": "3.1079.0", + "@aws-sdk/client-route53resolver": "3.1079.0", + "@aws-sdk/client-s3": "3.1079.0", + "@aws-sdk/client-s3-control": "3.1079.0", + "@aws-sdk/client-s3tables": "3.1079.0", + "@aws-sdk/client-sagemaker": "3.1079.0", + "@aws-sdk/client-sagemaker-runtime": "3.1079.0", + "@aws-sdk/client-scheduler": "3.1079.0", + "@aws-sdk/client-secrets-manager": "3.1079.0", + "@aws-sdk/client-securityhub": "3.1079.0", + "@aws-sdk/client-serverlessapplicationrepository": "3.1079.0", + "@aws-sdk/client-servicediscovery": "3.1079.0", + "@aws-sdk/client-ses": "3.1079.0", + "@aws-sdk/client-sesv2": "3.1079.0", + "@aws-sdk/client-sfn": "3.1079.0", + "@aws-sdk/client-shield": "3.1079.0", + "@aws-sdk/client-sns": "3.1079.0", + "@aws-sdk/client-sqs": "3.1079.0", + "@aws-sdk/client-ssm": "3.1079.0", + "@aws-sdk/client-sso-admin": "3.1079.0", + "@aws-sdk/client-sts": "3.1079.0", + "@aws-sdk/client-support": "3.1079.0", + "@aws-sdk/client-swf": "3.1079.0", + "@aws-sdk/client-textract": "3.1079.0", + "@aws-sdk/client-timestream-query": "3.1079.0", + "@aws-sdk/client-timestream-write": "3.1079.0", + "@aws-sdk/client-transcribe": "3.1079.0", + "@aws-sdk/client-transfer": "3.1079.0", + "@aws-sdk/client-translate": "3.1079.0", + "@aws-sdk/client-verifiedpermissions": "3.1079.0", + "@aws-sdk/client-wafv2": "3.1079.0", + "@aws-sdk/client-workmail": "3.1079.0", + "@aws-sdk/client-workspaces": "3.1079.0", + "@aws-sdk/client-xray": "3.1079.0", + "@aws-sdk/credential-providers": "3.1079.0", + "@bufbuild/protobuf": "1.10.1", + "@connectrpc/connect": "1.7.0", + "@connectrpc/connect-web": "1.7.0", "bits-ui": "2.18.1", "class-variance-authority": "0.7.1", "clsx": "2.1.1", @@ -177,27 +177,27 @@ }, "devDependencies": { "@sveltejs/adapter-static": "3.0.10", - "@sveltejs/kit": "2.65.2", + "@sveltejs/kit": "2.69.1", "@sveltejs/vite-plugin-svelte": "7.1.2", - "@tailwindcss/vite": "4.3.1", + "@tailwindcss/vite": "4.3.2", "@testing-library/jest-dom": "6.9.1", - "@testing-library/svelte": "5.3.1", + "@testing-library/svelte": "5.4.2", "@vitest/coverage-v8": "4.1.9", "jsdom": "29.1.1", - "oxfmt": "0.55.0", - "oxlint": "1.70.0", - "svelte": "5.56.3", - "svelte-check": "4.6.0", - "tailwindcss": "4.3.1", + "oxfmt": "0.57.0", + "oxlint": "1.72.0", + "svelte": "5.56.4", + "svelte-check": "4.7.1", + "tailwindcss": "4.3.2", "typescript": "6.0.3", - "vite": "8.0.16", + "vite": "8.1.3", "vitest": "4.1.9" } }, "node_modules/@adobe/css-tools": { - "version": "4.4.4", - "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.4.4.tgz", - "integrity": "sha512-Elp+iwUx5rN5+Y8xLt5/GRoG20WGoDCQ/1Fb+1LiGtvwbDavuSk0jhD/eZdckHAuzcDzccnkv+rEjyWfRx18gg==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.5.0.tgz", + "integrity": "sha512-6OzddxPio9UiWTCemp4N8cYLV2ZN1ncRnV1cVGtve7dhPOtRkleRyx32GQCYSwDYgaHU3USMm84tNsvKzRCa1Q==", "dev": true, "license": "MIT" }, @@ -252,83 +252,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@aws-crypto/crc32": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", - "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/util": "^5.2.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-crypto/crc32c": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-5.2.0.tgz", - "integrity": "sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/util": "^5.2.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-crypto/sha1-browser": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-5.2.0.tgz", - "integrity": "sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==", - "license": "Apache-2.0", - "dependencies": { - "@aws-crypto/supports-web-crypto": "^5.2.0", - "@aws-crypto/util": "^5.2.0", - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-locate-window": "^3.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/is-array-buffer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", - "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/util-buffer-from": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", - "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/is-array-buffer": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha1-browser/node_modules/@smithy/util-utf8": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", - "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/util-buffer-from": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/@aws-crypto/sha256-browser": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", @@ -344,31 +267,6 @@ "tslib": "^2.6.2" } }, - "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/is-array-buffer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", - "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-buffer-from": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", - "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/is-array-buffer": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-utf8": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", @@ -416,31 +314,6 @@ "tslib": "^2.6.2" } }, - "node_modules/@aws-crypto/util/node_modules/@smithy/is-array-buffer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", - "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-crypto/util/node_modules/@smithy/util-buffer-from": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", - "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/is-array-buffer": "^2.2.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", @@ -455,29 +328,29 @@ } }, "node_modules/@aws-sdk/body-checksum-browser": { - "version": "3.972.19", - "resolved": "https://registry.npmjs.org/@aws-sdk/body-checksum-browser/-/body-checksum-browser-3.972.19.tgz", - "integrity": "sha512-UNjwofGR4QqWFhcMqb+AvAKZfRIpSemlmHLviW+UD1Y9chL6/4O3gfKeyJJwW8ck1BHL6GCeF8zpvpKxIkgjZw==", + "version": "3.972.22", + "resolved": "https://registry.npmjs.org/@aws-sdk/body-checksum-browser/-/body-checksum-browser-3.972.22.tgz", + "integrity": "sha512-MEZ1upBetC1gLXPCJ5wwIYeWtkitKiwHpOQwpP0DhwyT/FaP58iotUWzhSLWyFZAONUV8Ov9JKks5btlEo+Gcg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/sha256-tree-hash": "^3.972.17", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/sha256-tree-hash": "^3.972.20", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" } }, "node_modules/@aws-sdk/body-checksum-node": { - "version": "3.972.19", - "resolved": "https://registry.npmjs.org/@aws-sdk/body-checksum-node/-/body-checksum-node-3.972.19.tgz", - "integrity": "sha512-YR8HYJlVfv7Tw0c58/d8NdEdeo8ORDTBdid7SRoxMyu7Db/bq8nyY1B8TI5VQDE5P6orzzYz55Gt8aOlVXgnLA==", + "version": "3.972.22", + "resolved": "https://registry.npmjs.org/@aws-sdk/body-checksum-node/-/body-checksum-node-3.972.22.tgz", + "integrity": "sha512-JZKTzRNi8T202jOIGI0jboklbICLJbf1r9oiXOfOgcx7vXmXf7AqgYwlUrj+KAaZg/XYKrx2UMm+EY8WInHtQQ==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/chunked-stream-reader-node": "^3.972.8", - "@aws-sdk/sha256-tree-hash": "^3.972.17", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/sha256-tree-hash": "^3.972.20", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -485,18 +358,15 @@ } }, "node_modules/@aws-sdk/checksums": { - "version": "3.1000.7", - "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.7.tgz", - "integrity": "sha512-qh0fG/RtrFztst4+vn1HZehAvAhr5Jlq/WMP7e5KvvfF16oNVBc9CDNVdxdm19vzOY2x0qiDMFCRjhxQAusGWQ==", + "version": "3.1000.12", + "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.12.tgz", + "integrity": "sha512-RgNDWfhNRIlNEzePIRrYTNi/6q+wwRMMapojn8YVzw4ZcJRa/gxVMtUbeZARR1gmopuv6oIhMbY7J66qIQ0ynw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/crc32": "5.2.0", - "@aws-crypto/crc32c": "5.2.0", - "@aws-crypto/util": "5.2.0", - "@aws-sdk/core": "^3.974.22", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -516,20 +386,18 @@ } }, "node_modules/@aws-sdk/client-accessanalyzer": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-accessanalyzer/-/client-accessanalyzer-3.1070.0.tgz", - "integrity": "sha512-OG+5fsXwAIyqjX4RvRSwbx+k7adApRdGKR2qlq+TmT6glMVCm4UnnWblSnb2u1CGUesyXLlu4nqxxn2Ys+7jIw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-accessanalyzer/-/client-accessanalyzer-3.1079.0.tgz", + "integrity": "sha512-jkRmZa/d8zBN8a0Tx6z4Eb29Ro3tMUTOFDHXe7tElWhEAdIsQwb12DLLiFAsUntKyeCXPTbg7jqFWxKYANcDNw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -537,20 +405,18 @@ } }, "node_modules/@aws-sdk/client-account": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-account/-/client-account-3.1070.0.tgz", - "integrity": "sha512-mOwDiAJh6jQsBsKYze6+IpO/SngM8XuvdkHmPH0PLqpWV/KQpv5y6h7/3+JtmUt8PU7YJ+NDUemK2je4vBvU/g==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-account/-/client-account-3.1079.0.tgz", + "integrity": "sha512-QHgzSLEVNwCciu0X6hI3VmYQkjNxRivKk42OwNYWrYxaCHu0A2KeebTvlLpnfdHw3ti8Hw0ZkSroyxQLHvXNdQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -558,20 +424,18 @@ } }, "node_modules/@aws-sdk/client-acm": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-acm/-/client-acm-3.1070.0.tgz", - "integrity": "sha512-4LhHM3NdNi0+wTGLVWU5nJLOz+8Z4Ben37EOEQZhgoy5xFrNbEkurtGp0chMmXYMgiGMZzijMBq0GJUG5ER7sA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-acm/-/client-acm-3.1079.0.tgz", + "integrity": "sha512-+W0ucfvaoGxopafPPiMCSFfCbsHfERjrWMYVvOO+vppw88aCoFLMPokV+PppGfQKV4/68hZS6zvNuFKET/8a9A==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -579,20 +443,18 @@ } }, "node_modules/@aws-sdk/client-acm-pca": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-acm-pca/-/client-acm-pca-3.1070.0.tgz", - "integrity": "sha512-/3lGNu9aznCeLpQjaqZISWhd4uGXZ3m+X3U1Gq3Ydh1v6alSEcVLnQ4DivuHhSSjhzVPj69FPHw/e0RBkXyRGA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-acm-pca/-/client-acm-pca-3.1079.0.tgz", + "integrity": "sha512-m5GDjPfS6DX+oDiXFjEUdMX3gy/PEBRvSWo1Gv7YsYUDVU3D9/ZShJjoKs5O/i2BWtiv4HjRhXHkWxPXMVUyCA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -600,20 +462,18 @@ } }, "node_modules/@aws-sdk/client-amplify": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-amplify/-/client-amplify-3.1070.0.tgz", - "integrity": "sha512-tqPPxuD3S3e8Q7DJOOjAFBI9sOtej/HFScYkod5Ry4wgkDfRdq30uV9kxtKfuqjFv72k8G47yeoGUOrvJWALFg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-amplify/-/client-amplify-3.1079.0.tgz", + "integrity": "sha512-bw/Sim/0s+i6PaYEPn6tUi/+dBa6Ql/nuTGi1OhyX6sQXiWVjeGgFyZEpzp0hcKgQGmHazd7AMSnuc11apGpeQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -621,21 +481,19 @@ } }, "node_modules/@aws-sdk/client-api-gateway": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-api-gateway/-/client-api-gateway-3.1070.0.tgz", - "integrity": "sha512-NyM2cmRZfiS1czkIFtrPkHS3xBAtmtojhlOG43KtFxiYA1RO23vFOY11Y4O3LyD3Z9eGtPCMPwx7trojHZRJXg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-api-gateway/-/client-api-gateway-3.1079.0.tgz", + "integrity": "sha512-iUAoqoqs1S54fmdhHpZqYffwk0yznuUmb846mW4Vr6utEym+RFNLwnFsFCBV0Jw6MJD5Z0gWytawSJqdfvyVXg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/middleware-sdk-api-gateway": "^3.972.18", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/middleware-sdk-api-gateway": "^3.972.21", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -643,20 +501,18 @@ } }, "node_modules/@aws-sdk/client-apigatewaymanagementapi": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-apigatewaymanagementapi/-/client-apigatewaymanagementapi-3.1070.0.tgz", - "integrity": "sha512-dbJsYrFNbw+WOiR2MCZJ9RELae4K5uNdNiHDNcxpXJHV0EP8cvhPe41OTBtZ35PXYd1jqaZzEwTxoqLhLo9bDA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-apigatewaymanagementapi/-/client-apigatewaymanagementapi-3.1079.0.tgz", + "integrity": "sha512-kWHW1IeVj7TpVlsnLCNLQABME5tVg5Q/S6nooN04PDy+1J/NxxCnq7MRfB8UwZ2qTvMVjsIo2ILLbh7xCv0+CA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -664,20 +520,18 @@ } }, "node_modules/@aws-sdk/client-apigatewayv2": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-apigatewayv2/-/client-apigatewayv2-3.1070.0.tgz", - "integrity": "sha512-DecTHH0ecZs8EiVlJ72aPR8lHTC6hdpMN8h4WA1RkFOnH7TzlTerpCLPhOtb9mKH3Ifb9eTZAg1+G4KAOEdONw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-apigatewayv2/-/client-apigatewayv2-3.1079.0.tgz", + "integrity": "sha512-7TjFagNW6syNSxL3GVnbEUB+4b0wLC3jo1JpoyYt9oafDaHDtgaP/hFPwZDy8BnkOnu3kie/jlpeTiAl25CoPA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -685,20 +539,18 @@ } }, "node_modules/@aws-sdk/client-app-mesh": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-app-mesh/-/client-app-mesh-3.1070.0.tgz", - "integrity": "sha512-trk2YyTR13f8AFj559/cDCGyKwj4RJpNBKtPh578OO3avE/00LM5Kxrbky2THjkGP+mTB64iUl9ZoDV7vq9kEg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-app-mesh/-/client-app-mesh-3.1079.0.tgz", + "integrity": "sha512-NJd1/eWZouTl3gToInc+st1e9hgDj1gAOiG5qoRkbQDv3/zxbeJ6GWFL3ZOJHjv2sy/qdEE+bYTvs3o5Q2iLkQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -706,20 +558,18 @@ } }, "node_modules/@aws-sdk/client-appconfig": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-appconfig/-/client-appconfig-3.1070.0.tgz", - "integrity": "sha512-xKLBl5KqHCAgPzkBE0qOnZD6tN0Rz7Vk9uESvMiUxiPdrPsil4oSRWzlulLIp+k/cLCpJbmhcLDLRP0oO3f/Aw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-appconfig/-/client-appconfig-3.1079.0.tgz", + "integrity": "sha512-9Lodl5S+OIfVZNtrFDwHJ0vNCQ3FcN3fIo7su8VoFsRZiTYPFc4yX8kah0y4/mZvQnzi/RRCDzdUkrgM+cSsLw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -727,20 +577,18 @@ } }, "node_modules/@aws-sdk/client-appfabric": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-appfabric/-/client-appfabric-3.1070.0.tgz", - "integrity": "sha512-jNjJ+0mYzsuxaXP3mqAVjjQvQ0lh61oIVWSJV/wnNVUgvqhpNBcCBoe3CpM5Tlr4Mj1H4OAUCj8ZzQHd6mj3yQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-appfabric/-/client-appfabric-3.1079.0.tgz", + "integrity": "sha512-RVLZ8Mej/3Qt3AFiDf6deUj4d1JURA3Mhku9Ak/BrsP/8Khe1gzHAA87Ng0SxZSGgAtaKXTt/XiwDRlUXncurQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -748,20 +596,18 @@ } }, "node_modules/@aws-sdk/client-application-auto-scaling": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-application-auto-scaling/-/client-application-auto-scaling-3.1070.0.tgz", - "integrity": "sha512-tezk4jkVco/HCLgP2ORmDSwTsgN7LugePLRWLML58O+d9ttS5+8dZPz7cJuMGOzYD/76x1lRh5qYbr8RZWDnyA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-application-auto-scaling/-/client-application-auto-scaling-3.1079.0.tgz", + "integrity": "sha512-1xxlkP1Z+4A+BcbkFn3Bd6oEvfW3L7zmG9NzintHq/uTdG8jexJp+KIC3Sbb3DbDxPGus/gD+xoAzCxPMdQRow==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -769,20 +615,18 @@ } }, "node_modules/@aws-sdk/client-apprunner": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-apprunner/-/client-apprunner-3.1070.0.tgz", - "integrity": "sha512-9Pm2+0/FcIufzDU8iRTWMDgtAEz+JlKnzWkmCuDxd/9z4LaOcaLwKFtcH9lzLD5nrtbz6sWQzmTAVp38umACvA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-apprunner/-/client-apprunner-3.1079.0.tgz", + "integrity": "sha512-UzwgN1PoqYio2tLLzpV/UeMgZLoNKysc+GfZxpxO8YocXky9UXzhnLjjUg2RX7VKz/JhtjMweziVIUnNQpFBIw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -790,20 +634,18 @@ } }, "node_modules/@aws-sdk/client-appstream": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-appstream/-/client-appstream-3.1070.0.tgz", - "integrity": "sha512-ShDkKNr+ksl1+ZJV6lIPfbOWgkEMHwcHK+F6BaITJZaXpsiSP4vU2tQocxvEl3Ma0aWJqq2ftnYbAAhbwdzhvw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-appstream/-/client-appstream-3.1079.0.tgz", + "integrity": "sha512-BhKxQNWbWJbg/ZNZsMcpAI0aUwIlBbGbgXLBVDUPytgq0Cr8vtshzmRa5N1wUIYKkVLbKXA4t6sTFaUACSYasA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -811,20 +653,18 @@ } }, "node_modules/@aws-sdk/client-appsync": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-appsync/-/client-appsync-3.1070.0.tgz", - "integrity": "sha512-6fFWmVD6odgMPskVT3vZ5MYDldNK//98abTXKcodD8V+Q25hlFA9jHpboead9L/gBDyvDYVugnmu1QzdiXvjwA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-appsync/-/client-appsync-3.1079.0.tgz", + "integrity": "sha512-q3j/gl5TIQUjHO1VDVRxFiGS5+SjlxTAbX71iKvSLFoQ3w+Z1MBvK/E3O3zL6pFKeLiq0JBnDkjFP35oDJi5lA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -832,20 +672,18 @@ } }, "node_modules/@aws-sdk/client-athena": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-athena/-/client-athena-3.1070.0.tgz", - "integrity": "sha512-GB+RWa+Sj1zFfM+VGSZuVnxf2jdOp+wA5OjXK2G50EiPoH5YAlVRFRW5QM7lMOUpKJb/prwoNUP4s6bKCbc1Jw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-athena/-/client-athena-3.1079.0.tgz", + "integrity": "sha512-w2bkUa+ZHEP39cVmYpvAms8eUnIrSyEBVOwRU3kyPufDvZfdwdC4S8PKzNlb9/kJ4R23m4QhOitKGjO6HNs8Yw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -853,20 +691,18 @@ } }, "node_modules/@aws-sdk/client-auto-scaling": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-auto-scaling/-/client-auto-scaling-3.1070.0.tgz", - "integrity": "sha512-WX7pFQ7IsdrHE2cSHB6mBl94BaiHojd6+wwXImYsJyPTgWjxbhbOmadOKLxYV0fidFF+5ypccc2RkE7dOlIJGw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-auto-scaling/-/client-auto-scaling-3.1079.0.tgz", + "integrity": "sha512-pHhIILF6ybuOEKE+VIyvJ+1I+WxkpC9lat9qVX/8A02SQHJ9OhEKdoxVG/mJf1CS8U6HG8C/eD3e8uB5NkE5Ug==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -874,20 +710,18 @@ } }, "node_modules/@aws-sdk/client-backup": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-backup/-/client-backup-3.1070.0.tgz", - "integrity": "sha512-WF+07zr4H4c7mUSgzaTr0JGLfmU/Rwu7LFHUYNmWwdh5fJFJ0eUJ7GgLs86oPfJPF8I8U2464iITeAfDFabHiA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-backup/-/client-backup-3.1079.0.tgz", + "integrity": "sha512-nurBN5uD55K5vp/Kd7lJX3S701k65z6dkIFMKingDF0cxZQI+wgKAAMIrW0qeGXOHGxj5tQGL9dNS2wYwklB3g==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -895,20 +729,18 @@ } }, "node_modules/@aws-sdk/client-batch": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-batch/-/client-batch-3.1070.0.tgz", - "integrity": "sha512-HJeMe05bhvjAH+GVhHxGrE0UxyMTZnNLdwl6g1DvGvVDgsuHCs3BaT4c8pXSZ3m3vq91W9ym55/aDsPUO0bzJQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-batch/-/client-batch-3.1079.0.tgz", + "integrity": "sha512-RygniTlHJif4mtmmzDpFGhYJQrLxqDKdWEHfvVL9+YmS+FFiDDAvvVrrPSaLg4i5M4yC80aKGkt8B3gHGuu4wg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -916,21 +748,19 @@ } }, "node_modules/@aws-sdk/client-bedrock": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock/-/client-bedrock-3.1070.0.tgz", - "integrity": "sha512-eNsWd+pnsiEbBUXU6o7SiLMCtCYp0dAjyBxKi1n4Z/VTLSGPoaJiX70UN+MUt0K3MN55LSqr7Iv1l6Q9PS4aDw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock/-/client-bedrock-3.1079.0.tgz", + "integrity": "sha512-hUpq+0bhLeWV6B1AvqjEmV4Yc6W2+7ib5Kidr8KNn/ZHnuK1QsaQrML9tMyVe+9S9l1WQco9xuMziUm2vO2JSA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/token-providers": "3.1070.0", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/token-providers": "3.1079.0", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -938,24 +768,22 @@ } }, "node_modules/@aws-sdk/client-bedrock-runtime": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.1070.0.tgz", - "integrity": "sha512-p6Gw2HhgT7jpFK6JJ4VaFP7CmqWGb7G5ToK3bXI/tGGbyfS0+MTzIRo0+VPeWIdYNV8soIfJr5Irw2CQ3Ynq8A==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.1079.0.tgz", + "integrity": "sha512-+LxXUhnDsQ4Yr+zSQbROKClMfIRA8mLPtZgQukb6omczIqmi0yvDmbLovdLJGnm9GRj83kKSxWV24Ms5aUgJfg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/eventstream-handler-node": "^3.972.22", - "@aws-sdk/middleware-eventstream": "^3.972.18", - "@aws-sdk/middleware-websocket": "^3.972.29", - "@aws-sdk/token-providers": "3.1070.0", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/eventstream-handler-node": "^3.972.25", + "@aws-sdk/middleware-eventstream": "^3.972.21", + "@aws-sdk/middleware-websocket": "^3.972.35", + "@aws-sdk/token-providers": "3.1079.0", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -963,20 +791,18 @@ } }, "node_modules/@aws-sdk/client-cloudcontrol": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudcontrol/-/client-cloudcontrol-3.1070.0.tgz", - "integrity": "sha512-jJmei+N3ij6FYFOEO0pEol6o1D/LXQYdcH2oT7bmZBeu5udftRtsMDebgi2Ok5udVedIcLqsmagCMzCZt1mlLA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudcontrol/-/client-cloudcontrol-3.1079.0.tgz", + "integrity": "sha512-BwjWiguWivUzL5n8XKhxvZeoX43OLqSf6RHKVaQ1qG1ju1/g5rEeuG9+zSq6+GbWw3LGj1Pbp2Tblxmful8jjA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -984,20 +810,18 @@ } }, "node_modules/@aws-sdk/client-cloudformation": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudformation/-/client-cloudformation-3.1070.0.tgz", - "integrity": "sha512-4jVkFb0QE6mZcU01rfMwRQkB6+hwCJmkyvf0YFj7udvdq3vixTGBMG0Jy2L896NWWgBJNgEZwcXnVU6XWPTP0g==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudformation/-/client-cloudformation-3.1079.0.tgz", + "integrity": "sha512-07eZNnT2m9GuLnnIgwUBGVNzwFGoDoVsKvT+3cdCeLLeq+McI87QUz27ixSCOTYM1rOpPznc8v4SNvQ+fja3YA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1005,20 +829,18 @@ } }, "node_modules/@aws-sdk/client-cloudfront": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudfront/-/client-cloudfront-3.1070.0.tgz", - "integrity": "sha512-Bxm3uF3EoWAIMhmB6FosADe+KT5zARqumcGodws4TX+VeyLKW5xyWodD9OIWmbqZkUZW78NY3blYG+YaszrHKw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudfront/-/client-cloudfront-3.1079.0.tgz", + "integrity": "sha512-plOK/g0hvn+rFfzL+1NyhJNQ1RVCfW2Svfzknp0ctU16ji1S9BIK3HL4/8rlbqNgQeH3aiyfurtnYVNMhKHe4Q==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1026,20 +848,18 @@ } }, "node_modules/@aws-sdk/client-cloudtrail": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudtrail/-/client-cloudtrail-3.1070.0.tgz", - "integrity": "sha512-JqaCo5qmORabZBbxoGY2Tp6/Xgnk3vVWBbUIIncHwoZGPGWcpmXsmJKu0aLgLnLO/btcr1aYIQgYLepTuL9g1g==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudtrail/-/client-cloudtrail-3.1079.0.tgz", + "integrity": "sha512-tvkBWdQ85pLHdYMqDlJvYkC4UQ0yLZCmXRr38av5uCTF1LgAm3FLrmsDOyX7HfD3WZV1wpdC/8G9CJqeArzkuA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1047,21 +867,19 @@ } }, "node_modules/@aws-sdk/client-cloudwatch": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch/-/client-cloudwatch-3.1070.0.tgz", - "integrity": "sha512-50JP4215shvkcockuGEMRUbtlCMevPxvJllUnyTjDqiNmVbLsN49Nvbd0p0XZNfC/aIZpevDNVpj4toXrtJSzA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch/-/client-cloudwatch-3.1079.0.tgz", + "integrity": "sha512-/nsSchlpcAC1WsGyvsje2MRy37vn7lBINYpxgc/LQ56GX/hXYLA0jymAVPE8HfckhiHnrii9XDGuzuiI4pIQvw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/middleware-compression": "^4.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/middleware-compression": "^4.5.5", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1069,20 +887,18 @@ } }, "node_modules/@aws-sdk/client-cloudwatch-logs": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch-logs/-/client-cloudwatch-logs-3.1070.0.tgz", - "integrity": "sha512-HhpcIxGLns/idtXqW8E5IyTbJvMfgqDPEDdzPZfZ+6JM78n6BpALVJteFcBaG/zgRvbJOh30Ipq6nu2Z3rUnJA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch-logs/-/client-cloudwatch-logs-3.1079.0.tgz", + "integrity": "sha512-mn+j4nDWi599W90MGljDBKExo6bGH8d6JzNFGPNM0udsWJGWQWNp1teCINEXUKY3uTEslqyPgcxhb62EkdciRA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1090,20 +906,18 @@ } }, "node_modules/@aws-sdk/client-codeartifact": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-codeartifact/-/client-codeartifact-3.1070.0.tgz", - "integrity": "sha512-RkCuSjTlZPwJpc2R2TUh5wrJscUC7lLJU2EQt1Ff+h3ZOaIZp2fHt5hkx3F8FWmIaN7pI9eEosPfWwGvT2oI/w==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-codeartifact/-/client-codeartifact-3.1079.0.tgz", + "integrity": "sha512-zg6VYrpHRe/Rgxyz063YHSv2XPJ4JTkkJoNsk8bz9P/kplRlg5l2NJlp4VY7xzVBLQUw2XQciZNaZBBZVocf3g==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1111,20 +925,18 @@ } }, "node_modules/@aws-sdk/client-codebuild": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-codebuild/-/client-codebuild-3.1070.0.tgz", - "integrity": "sha512-ECiVB/QvnqnPy0/ov+EgdiaalZvlOKxUXpvm5jmFThD/6RtYOBRMfmlx6KvbwSwyhUcPC/2obsTs75c0ttm5NA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-codebuild/-/client-codebuild-3.1079.0.tgz", + "integrity": "sha512-VPn4vVeibj1CArj91QKL1MXqPTs1SoE4ZT4sV+M6Q68MVk4Xcw29iNZpKqsc9SNmXCdQj80kbWwdoG9UT0nfTQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1132,20 +944,18 @@ } }, "node_modules/@aws-sdk/client-codecommit": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-codecommit/-/client-codecommit-3.1070.0.tgz", - "integrity": "sha512-V1mvn2bXkLAOE4SBgHb573mFp/EhbTkZgyGun09haS3bc1Q9aAZhOLi6E7bGK2wF3KkgVpF1TSqvrgnhMRdXeg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-codecommit/-/client-codecommit-3.1079.0.tgz", + "integrity": "sha512-XJULEMOOa1a3dHpDgQ/0mGnw36l5s3owJSK56u4GRl96rMe0yaF1uYlT9jMvmhX5SrwXmnKbzu3zlMOB7AyEmw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1153,20 +963,18 @@ } }, "node_modules/@aws-sdk/client-codeconnections": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-codeconnections/-/client-codeconnections-3.1070.0.tgz", - "integrity": "sha512-JBxsnCbylya9IWfOQxxxrncwKVZ5NgTNzoClDb+VOADNclQgQvyhIEqaL6T7MpObNqT68KNZMKuVFfplgVqQLg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-codeconnections/-/client-codeconnections-3.1079.0.tgz", + "integrity": "sha512-R6FwUUUfbF2UsZizMf8uLTLHT6vLo6eR36H/4WpTGTaE5vhZJC816UdtpwE64HOJg6wk9cmMPf3+qhg6sCvR6A==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1174,20 +982,18 @@ } }, "node_modules/@aws-sdk/client-codedeploy": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-codedeploy/-/client-codedeploy-3.1070.0.tgz", - "integrity": "sha512-jjuH1ZVE7ozScgWy55zRKNTZfy57MLoXbfqdNv1IjssQ9vqfnqCD4TS/mRWw72/LnSRg82xhJgIiiF3C1Xh+6g==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-codedeploy/-/client-codedeploy-3.1079.0.tgz", + "integrity": "sha512-CIuwEqj3R9dhgWi1WVvMZA8wD25lw4MtMI+izPF9w2LFsUfhUOR2J3c3oezi+yhTSdKqtNniSxJhlTK5SDaUYQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1195,20 +1001,18 @@ } }, "node_modules/@aws-sdk/client-codepipeline": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-codepipeline/-/client-codepipeline-3.1070.0.tgz", - "integrity": "sha512-/Wf91IJzKMfg4jwF0xmkV2g+I71VXQE1dN8aKpLvqmA29Tl4fwc6pvxZ9NXHdLWvtusUUIu1LYDguDSM0+F+1w==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-codepipeline/-/client-codepipeline-3.1079.0.tgz", + "integrity": "sha512-WX4ZYusHJu19dEBbWBkoj8v9TrQ+ZUshExFLrx1Xis7wXOaq/u/EI+t0/aZwJkt2OamiEZGG72rgPxLoBcgzrA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1216,20 +1020,18 @@ } }, "node_modules/@aws-sdk/client-codestar-connections": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-codestar-connections/-/client-codestar-connections-3.1070.0.tgz", - "integrity": "sha512-xVemvxNr2O+Yn5+ejNJ/dNbd7N90bKdBeSDL6p62bP3HCitx9S5/4lK7dK1tI4a6BPr3pe9qIJ7aieZRtbkk0Q==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-codestar-connections/-/client-codestar-connections-3.1079.0.tgz", + "integrity": "sha512-/vfTs/Hv8B1tTsBwxFFuiApXZ0Iw3TFfexB4in1uTtrNY/HyZDCKj/ymDB/WIYNLbPagZ7cxrhchJwPIOfWdVw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1237,20 +1039,18 @@ } }, "node_modules/@aws-sdk/client-cognito-identity": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.1070.0.tgz", - "integrity": "sha512-Xca8Ay6hguFCtQ7ZoNwBVDOmcKrOPBn0K3jq8QtBm5wI/YgJrw61ShH8TyaTtGOwdlOcTyAe32Kh6IOlq6kkYA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.1079.0.tgz", + "integrity": "sha512-HIScdAc8q/upCY/f3TPW0pNq1K1LL7tn5fEifKf1K+zs3NRPXLultta96ZwvcZ9Ax503JKKTo9f3xGpR3fpCxQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1258,20 +1058,18 @@ } }, "node_modules/@aws-sdk/client-cognito-identity-provider": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity-provider/-/client-cognito-identity-provider-3.1070.0.tgz", - "integrity": "sha512-Bkh76Sz4/QHvx1uYjMoU1YjeO/KJo1AxD2vT/HwhWbtAGPs2+igEv1rAz/iYWV40GERhk0dO7mRlRjX/1PVJQw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity-provider/-/client-cognito-identity-provider-3.1079.0.tgz", + "integrity": "sha512-EISFwFAkHGPard4hsPxD+dGYoJFoGpWF2woa2meF0Gs6LiZ4RTeFsxDTQ0mGokBxHxt+iZ1bIlmA6zZlSDOdiQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1279,20 +1077,18 @@ } }, "node_modules/@aws-sdk/client-comprehend": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-comprehend/-/client-comprehend-3.1070.0.tgz", - "integrity": "sha512-N6HJOA161kqi0/VPlVeFeef7dhbdZjFKa9veeYd+hkk9Ppz2GXJm+KchTl6OZbDNx1aMuqarBHu0xvpqU0yGkg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-comprehend/-/client-comprehend-3.1079.0.tgz", + "integrity": "sha512-4AgmHun6QhAt9WZFgMbcHfo8nk00AWGl7rBAQs3A1/la80K1Hx1dWMIz8VIfeUTnXS44a0NpnzaOp/dB9rNyKw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1300,20 +1096,18 @@ } }, "node_modules/@aws-sdk/client-config-service": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-config-service/-/client-config-service-3.1070.0.tgz", - "integrity": "sha512-BEN93OXQTpwJNuAZkj3f78ZC6SqeM6VhItwAC2h1CAKcBzywAm3HulOiKZHA77fmS5n5q3Kb2N9Tj1yDlrGaQg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-config-service/-/client-config-service-3.1079.0.tgz", + "integrity": "sha512-eBD8snnUvtjYggUlQnxz6TaQVUNWl2vHvQx8ExAFJqs2+WcHE6AdH54PeEmLob+Zl9oEgdcKRkOF428b3Fsv2w==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1321,20 +1115,18 @@ } }, "node_modules/@aws-sdk/client-cost-explorer": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cost-explorer/-/client-cost-explorer-3.1070.0.tgz", - "integrity": "sha512-DW0z0tbGUbPcjKnjwwg8IEBzBCHZB75+PrLEOXHrd1wp5E+xp1xIqKkluNromQSiSuf+Ex4H7tZRszFUKGuFbQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cost-explorer/-/client-cost-explorer-3.1079.0.tgz", + "integrity": "sha512-pZjJyaLGqGQyTTSg5GMlPPSOQk7pc5DVCQdCjMRZKxRpq5SaIIjkEMJeqX3kdKDIwyBL6dynFRbpArsOUVjl5w==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1342,20 +1134,18 @@ } }, "node_modules/@aws-sdk/client-database-migration-service": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-database-migration-service/-/client-database-migration-service-3.1070.0.tgz", - "integrity": "sha512-qSkWYxbsy7SR72Yu3XC8ucJc81NS8e4lyR88+Vgi07/kBn+7X+QCFePaMaQjN3S/CnlL2VTLHPQsjoOQ0bDReA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-database-migration-service/-/client-database-migration-service-3.1079.0.tgz", + "integrity": "sha512-ty9OA2Ypn4uFJ4KwbkEgoP6+rdMsJsAIbF9+7SI8ZY78okiKStfk1p22khUP1PEvuvxjh3mLS2fV1DwmD2r2Qg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1363,20 +1153,18 @@ } }, "node_modules/@aws-sdk/client-databrew": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-databrew/-/client-databrew-3.1070.0.tgz", - "integrity": "sha512-3YUKuyRjNgERSP3fnzNWG9F1RVTnHKck6qyPguspWDzvOtMQGE3Q0/8YgnyLC5N3cJAhM5aPJyBga4AWOAPVfw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-databrew/-/client-databrew-3.1079.0.tgz", + "integrity": "sha512-i6IC5NcRjgSzwbo2woPISePmnsE92jlzfTEx0AvIJ3XmAIceRihGt65S+go0q1j+CGVwZzE3UTznfLLliDvl8g==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1384,20 +1172,18 @@ } }, "node_modules/@aws-sdk/client-datasync": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-datasync/-/client-datasync-3.1070.0.tgz", - "integrity": "sha512-RvbVuYOz5i9h5vbTiR4RSgGPvhht5r1PZf25ueTQuVG+PoJz7fM0diXlOGpI2U7D/r6enCkhEEJT6G5iJhq0kg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-datasync/-/client-datasync-3.1079.0.tgz", + "integrity": "sha512-wkxHAD4UcK3KEzcX3kUve4ExRSsFUkxy+B0AY6bMYoKhyvGDEXwgUr/0vsrnyemdlzr45XR2SUEZHsvylfOxIA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1405,20 +1191,18 @@ } }, "node_modules/@aws-sdk/client-dax": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-dax/-/client-dax-3.1070.0.tgz", - "integrity": "sha512-4A45fuM0p+4ObMtDnBy6q8t6TjF4x4gHz6vj+9ZvvTLVTmH/RQ0/GugzL0xSuiDiNoaIl0+mZqAl5wibWBFzRA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dax/-/client-dax-3.1079.0.tgz", + "integrity": "sha512-8cdaRlfUDnlLA+6rSCZO61xEaiDlHunxvZxgbD8rCSBUz7+l75duaNGDDWFRTVYCGCZv0ooa5BHx5QlLt7FWqA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1426,20 +1210,18 @@ } }, "node_modules/@aws-sdk/client-detective": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-detective/-/client-detective-3.1070.0.tgz", - "integrity": "sha512-m5oeB7F5h2bTfYkzroHaxYvX7JcG3D7FgkvAuOFrsaadHhmLdxgNMlmEsuecWJFkqANOVGSdkJBU3UQYPEXTRw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-detective/-/client-detective-3.1079.0.tgz", + "integrity": "sha512-JShtNKnS+FP498PTBASBgMEGNxsEJJ6JF+Pf1W2p1Hoz3+RpDWNHW1ij4Sg/Dtp7Yji7e9YM+jg+HWFkRQIzYg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1447,20 +1229,18 @@ } }, "node_modules/@aws-sdk/client-direct-connect": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-direct-connect/-/client-direct-connect-3.1070.0.tgz", - "integrity": "sha512-8NObPzZYihJOE7qIKT43oIpHWfmyW83/KsI9axRF0M7G17Vw1/p4WQHyUHWjvt1W0vnaAKJRT/4qadRGiOb3og==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-direct-connect/-/client-direct-connect-3.1079.0.tgz", + "integrity": "sha512-FQP/9WEdTR8eCpicBHmInDHqA4wbLIrfiUjWImfvi3/+sY/+GVdsovxv+kZPRX/uWyWFUeoAkHEyh4faFu6Hjg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1468,20 +1248,18 @@ } }, "node_modules/@aws-sdk/client-directory-service": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-directory-service/-/client-directory-service-3.1070.0.tgz", - "integrity": "sha512-c/9L5e+OMU5/u0zBEU+Hu3JFl17nCcI9NGIz6rWK1fheStaOX4AY9ZESiOF25Jl93wjQVnw5ILUYFsUeNB8RFg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-directory-service/-/client-directory-service-3.1079.0.tgz", + "integrity": "sha512-x6aj2/Vz+VFRKXlpoMiGdlGM46URoKI2OREHcDgSvP61dtSIYROR8WHXWoUW1BtpJ2gqjhwO1hh6ax64S6pssQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1489,20 +1267,18 @@ } }, "node_modules/@aws-sdk/client-dlm": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-dlm/-/client-dlm-3.1070.0.tgz", - "integrity": "sha512-aJvR74vXTKhK0/b7ZhzTxSH2Dc/lx+A5Z3dW5/3mOtQyoYyE4bzk9t68j0g4cP50/3LGtkWKZkNkpThlbc+01A==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dlm/-/client-dlm-3.1079.0.tgz", + "integrity": "sha512-oBPuQlNyPwh32vAHIAPPasvqgKo50yGM5garoUSGpiCX5ARzH72vbFfsLGRZQBmC8suUijhQduMmVcE9Hx5tsw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1510,21 +1286,19 @@ } }, "node_modules/@aws-sdk/client-docdb": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-docdb/-/client-docdb-3.1070.0.tgz", - "integrity": "sha512-UExDqR5qq/MxDH/9SjUXRxKfnDH02XIBmEHQfhj4xfndAwEbdtmLk0y1Q1NVgn3WSzeHHZ9HqeTP+wzqAr8UlQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-docdb/-/client-docdb-3.1079.0.tgz", + "integrity": "sha512-lEmTfyDnvFmNtZERY6+8lRmm/73sigEwBOxdazm2Lhr6fteuyjUAgrxuUHbxcVrFmVJmPEFsvmhvXZY4chuUig==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/middleware-sdk-rds": "^3.972.35", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/middleware-sdk-rds": "^3.972.41", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1532,22 +1306,20 @@ } }, "node_modules/@aws-sdk/client-dynamodb": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1070.0.tgz", - "integrity": "sha512-H5rA840DhCV0cFb5t5pWcixLXzpG/d8knhkSAh5hrkzJkoPnz2k2MAmSiPy8mzOZysQcTyNq28grRUpZ3aQbWg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1079.0.tgz", + "integrity": "sha512-njnQvv5lqyzX42Af/Z3nuxm6eK9/OPDYkaIah3m2sJoudAKkvvJ5jOTAtw8zGhu5zviT4Sa9giYC1FHkabuAFA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/dynamodb-codec": "^3.973.21", - "@aws-sdk/middleware-endpoint-discovery": "^3.972.19", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/dynamodb-codec": "^3.973.27", + "@aws-sdk/middleware-endpoint-discovery": "^3.972.22", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1555,20 +1327,18 @@ } }, "node_modules/@aws-sdk/client-dynamodb-streams": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb-streams/-/client-dynamodb-streams-3.1070.0.tgz", - "integrity": "sha512-UankWxd8dxBS5I5fIiaWx1Vp0rcksI8vTkVDabHBHDi0Jn/EBNZFpusqG8c8i5lgEn7UgYztksneCBw4uLIGaQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb-streams/-/client-dynamodb-streams-3.1079.0.tgz", + "integrity": "sha512-z6fJjRy7YslzC8vwV6cSvTnqB+dxK40n13WYfLdKyZMLjye3wfKsPTt65yIc5n0p0S+xfU1TsTfwwv3PRRRQDw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1576,20 +1346,18 @@ } }, "node_modules/@aws-sdk/client-ebs": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-ebs/-/client-ebs-3.1070.0.tgz", - "integrity": "sha512-zgXaZqYnMoW1e/l+sRQHGTsEMleOcT5GA4+g/ESU6kFQIHbkwEbYoqGc+Z/4gro2XDe9iyXtiY/vn1PNyqbn3A==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-ebs/-/client-ebs-3.1079.0.tgz", + "integrity": "sha512-NFVIjHo95XpYjX1Y312+isrmesqad34y5+D3PqNjvtPg/qiLnA+ZNKQkfnIefwiYE/83FZlBiECxwGE0Xv+RDw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1597,21 +1365,19 @@ } }, "node_modules/@aws-sdk/client-ec2": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-ec2/-/client-ec2-3.1070.0.tgz", - "integrity": "sha512-mWexBXtlLvV+AfvnfsYJxd9ICy637xIiQBYlsNf++2eQ1wuVLe4YIniq+4aazxPV8EQe7a0BZBFc2M/oaYCipw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-ec2/-/client-ec2-3.1079.0.tgz", + "integrity": "sha512-++opIQshVU9iKeD9ISUGZEWVq+ms8Lc/2G0TJC6aowml4mIlo1LL9Wsb/C0dNulbk8WBIBHSACY6s9dyVrpQ8Q==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/middleware-sdk-ec2": "^3.972.35", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/middleware-sdk-ec2": "^3.972.41", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1619,20 +1385,18 @@ } }, "node_modules/@aws-sdk/client-ecr": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-ecr/-/client-ecr-3.1070.0.tgz", - "integrity": "sha512-aapnXU7CqzDwCR04Z7bf1KcYq1NSWw0cZ9KtOhoOonz+yCPjMlnCYAauV9MY8kBiHjZ6BQiyUy5TQmp/tIJeig==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-ecr/-/client-ecr-3.1079.0.tgz", + "integrity": "sha512-z49jwijHWvKpoXToHDQecq5mGAowYQIFgHaw67AqKaouooFxxcYlBtBmIcJZJ8W9oK+Q46B/puMKdncoAwxGlQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1640,20 +1404,18 @@ } }, "node_modules/@aws-sdk/client-ecs": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-ecs/-/client-ecs-3.1070.0.tgz", - "integrity": "sha512-Uzs/QXPrSs5bMCBRUVnGRShiaTzwhZBSNAjC6aw2fAXNuOF3pZygDA2UM0aonTg0r6SmBGrS+UqGTz4IDFChDw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-ecs/-/client-ecs-3.1079.0.tgz", + "integrity": "sha512-c7u272kLMWLShZzuOsAuGC72aj/GDw/YJfLntlyosAxuuD4Dyu3915XpRsaMGWKG7/mIre1Fke4DGYNU2s6Nhw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1661,20 +1423,18 @@ } }, "node_modules/@aws-sdk/client-efs": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-efs/-/client-efs-3.1070.0.tgz", - "integrity": "sha512-A/yKmE4v/JIAB2mTcJ0vLgA88NAAUg3/0zWP2HpOJ2b9GRjzpU1Sw0WOnfF14x9SYKlpIMMsaAyrJ1a6Rl6VBw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-efs/-/client-efs-3.1079.0.tgz", + "integrity": "sha512-IoO4mtoofhhoqjIR5h+UEZbIwb2OuHEHgdlj+eHh4GhXqzrY+R8J5E6RQUuUtHdKQCxPRPy8I1/QA40WSYFqTA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1682,20 +1442,18 @@ } }, "node_modules/@aws-sdk/client-eks": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-eks/-/client-eks-3.1070.0.tgz", - "integrity": "sha512-pH6h2S/mWMjrpKZwB2jPY8HGbefuo6eJBZu/vFTxQ4J7rc+RTSUePjZCTyv7ZgGL9bAyj0xOTfpZOLH1ihyV6g==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-eks/-/client-eks-3.1079.0.tgz", + "integrity": "sha512-BCEdS7ujC3NQilCNvJbUep+DUqjlokYphMFKrzHeDNdZdcyS2vltqV2vSueADOMCsDwtwXpZpBqZsQZH+HAKfg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1703,20 +1461,18 @@ } }, "node_modules/@aws-sdk/client-elastic-beanstalk": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-elastic-beanstalk/-/client-elastic-beanstalk-3.1070.0.tgz", - "integrity": "sha512-8J7SAAQz/5ZK4rDnJndP1pYGtQ/A6wjJD2s64N5cPPSYYM6qRcm4om154lro2xS3M+NFWFmUHG4RR7bPglerhA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-elastic-beanstalk/-/client-elastic-beanstalk-3.1079.0.tgz", + "integrity": "sha512-7hlXJR46yrPb19EUYvNmJ8YIIgcRs8QNgUngb763dUBcr/TN18gdBOi4WItaaApzDrgrY/9T0mJiHeTwnh/aig==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1724,20 +1480,18 @@ } }, "node_modules/@aws-sdk/client-elastic-load-balancing": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-elastic-load-balancing/-/client-elastic-load-balancing-3.1070.0.tgz", - "integrity": "sha512-6uH4rgppP5iu7u2Nzj7+bHNkVzx5KT2JEa+dT9QGV9Gdl5mg94erOzGQXOishpbheeOUfE+pUN3p7Z99MoVadw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-elastic-load-balancing/-/client-elastic-load-balancing-3.1079.0.tgz", + "integrity": "sha512-piZ663+nQJMLZd2OHMBCTurQ4F9e/TiUSdLJ1Ncj+4Sp0VjS+G6wShvauffemGwcHuCUdJ0fCKbiNiAJWT7+4A==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1745,20 +1499,18 @@ } }, "node_modules/@aws-sdk/client-elastic-load-balancing-v2": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-elastic-load-balancing-v2/-/client-elastic-load-balancing-v2-3.1070.0.tgz", - "integrity": "sha512-6Be/xn7nwiOHavJX+tDtPfh/gy5ngrR0mg51dS4kZZi+/FScKwXHpDdRmIlCl+g+POGbSjVlDoJaYoURByEDqQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-elastic-load-balancing-v2/-/client-elastic-load-balancing-v2-3.1079.0.tgz", + "integrity": "sha512-CEb5RVc68YVyxU+p5wmzsTJBYilUYuqLgp9yCwodPmvrTkBRARX1+4nZHfRYTFk8XgtjS51jj0lznL9lq3RSIw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1766,20 +1518,18 @@ } }, "node_modules/@aws-sdk/client-elasticache": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-elasticache/-/client-elasticache-3.1070.0.tgz", - "integrity": "sha512-cGunOJRVacOIC/9qIgoKwXJuPREqlPz+zTIWXgBbCVPZSZBcWY6OAyvKJt02lS1pne5jQHEBfBk2HdcGOem/vA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-elasticache/-/client-elasticache-3.1079.0.tgz", + "integrity": "sha512-h8rn0Ck83Iv3AwgeEzT/odqkBfJ+jprXD8aNP+v22aqbydFIEUrFmhNYMWrI9tMp1Oml8lpMvi115mx4KYMpKA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1787,20 +1537,18 @@ } }, "node_modules/@aws-sdk/client-elasticsearch-service": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-elasticsearch-service/-/client-elasticsearch-service-3.1070.0.tgz", - "integrity": "sha512-LDa6H/diI0hSB1g0iqseJkHWbiAlesZG4LvEf+MxYOMQBSbxaohoZq9CA+C4b3cTopySDfEMj+ppoAUk6bjArQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-elasticsearch-service/-/client-elasticsearch-service-3.1079.0.tgz", + "integrity": "sha512-yT0Xlv0ZJegz3RsXzp8JtMf6EI0LxqS+nzcAOKPzwYeWPUkaXYaHLIVshbYTyeNO8UBEQ5r8dUQezf9wdMIK9w==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1808,20 +1556,18 @@ } }, "node_modules/@aws-sdk/client-emr": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-emr/-/client-emr-3.1070.0.tgz", - "integrity": "sha512-XoWOn0qKYXqRTMHCOdKYvbQrCDhJwufzUODfnvn6P3PXiCoJov8FsK2pfLnw05X8IH0+lxmDTyrQvy4t/ISleA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-emr/-/client-emr-3.1079.0.tgz", + "integrity": "sha512-sn0BeYzjHbUa6OVUppeaALliku4JMbd4T6bggdepNtWBf9s7m19Ipbbrh/mugrfTVR6IhJ/QHBn+iUek17YWOA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1829,20 +1575,18 @@ } }, "node_modules/@aws-sdk/client-emr-serverless": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-emr-serverless/-/client-emr-serverless-3.1070.0.tgz", - "integrity": "sha512-V1kc0KLUX85QuDPtGTJRoZ1RF5d1C91YKH+m3CV9AQ8ZsVl6em65E6UWIpzyEWHBx2X9Vxv5Htb8Ji169xF1Hw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-emr-serverless/-/client-emr-serverless-3.1079.0.tgz", + "integrity": "sha512-69rnypgGaYAMkf11N+A9MwH/dor639raVc8ZXWmjhkHKHLZJ28QDOsSd5qYqMoVDHmVQl37oxyqCE6FdLjDuZQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1850,21 +1594,19 @@ } }, "node_modules/@aws-sdk/client-eventbridge": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-eventbridge/-/client-eventbridge-3.1070.0.tgz", - "integrity": "sha512-uitoZ6Wb5kn9LNkzakjYny1yw0lwzAnQJ6JAfJnqH2ua6U6FON2ZIJQldx+YffJCAUPjp+tsI6QdfYvn1WZPQw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-eventbridge/-/client-eventbridge-3.1079.0.tgz", + "integrity": "sha512-Q1Zz2as7SJWtj+ernkab4nucS6HO/wgg4Oi1fBD9FnMfEoOLf+Le480hP/cYDxIO6gB6HMLAZwhIXkqCNPsEIg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/signature-v4-multi-region": "^3.996.35", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/signature-v4-multi-region": "^3.996.38", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1872,20 +1614,18 @@ } }, "node_modules/@aws-sdk/client-firehose": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-firehose/-/client-firehose-3.1070.0.tgz", - "integrity": "sha512-0Dmkgjb6LaCp7JPjtbl7QTela7yIjjd91HqoHwNDKUNQq6y705p7BmSy7HatPHoU3MhIBffv3UkOfo2ulciYHA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-firehose/-/client-firehose-3.1079.0.tgz", + "integrity": "sha512-+7ngTn0kIwijgY0Phwkw3X7PnOLDJ1IC6piIqN7r2/AR7hVpEOBgAHfOYx25TIAknIcuP6Pa2CMpJ9JwU/p2fA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1893,20 +1633,18 @@ } }, "node_modules/@aws-sdk/client-fis": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-fis/-/client-fis-3.1070.0.tgz", - "integrity": "sha512-QmPNC+upnxMdinIKNB+ZrjPI/hIBWgcDTPAAhxvmBdXXcdIXCIdAqq8FdvQDe4u0rShpuHYKgwKR2r6oHjc72A==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-fis/-/client-fis-3.1079.0.tgz", + "integrity": "sha512-atpSiBYu0bU6dFrcVWUkWXcti9kVRNoVU8vJznF0HLJ2pgZv9qGHn/6TjLqVzYKugsRvL1TOAM709rGjip/jzg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1914,20 +1652,18 @@ } }, "node_modules/@aws-sdk/client-forecast": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-forecast/-/client-forecast-3.1070.0.tgz", - "integrity": "sha512-goxc5Iu6EQbx4GY43PUy3bVeeE/+wEnri4eB7HhERGBSxipQNWdiNL6EH3paVBUkFxpzS/mq0zg0w/bYsnGB+w==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-forecast/-/client-forecast-3.1079.0.tgz", + "integrity": "sha512-Hv4/xbc/NChO4/2YoI/GjGvUawMO7+ADq3+avi+jKLrhjiJKTqw3ucpPlPtWpX+iCw/QB6oCuS3/zDGtSSQ5kw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1935,20 +1671,18 @@ } }, "node_modules/@aws-sdk/client-fsx": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-fsx/-/client-fsx-3.1070.0.tgz", - "integrity": "sha512-KCxv7croSQI5DsZSmsp3Tbp54eUjgqkLLUKQuqIyiUSeWvGrXXGEAmkFpBSkv6lTz8x6vTixqQ0xEXclo73U1A==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-fsx/-/client-fsx-3.1079.0.tgz", + "integrity": "sha512-gl9uV7WSb/DnPA4+WAJ997zSBBO1EqUfq+3mJCp9vNn5watBFvFTi7khk0vI+2CeYFTpyzDnjMXbmLo7wkoDqw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1956,23 +1690,21 @@ } }, "node_modules/@aws-sdk/client-glacier": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-glacier/-/client-glacier-3.1070.0.tgz", - "integrity": "sha512-w7j8Ocog4oHY/DauWY3yS+/Iy6J3awNBEZOPOijkCdGS7rgHchzgnic31k/GilhuplUtvOcwq7HTcXV3f23VaA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-glacier/-/client-glacier-3.1079.0.tgz", + "integrity": "sha512-sDi31++qNaeT7Ps5Z/0BrxtG56aVD7eGPB7PM+eaRyOT/uFmUSMfQoUM/hSRDvjSjAwsCiSkvpQlXru9dQCC4A==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/body-checksum-browser": "^3.972.19", - "@aws-sdk/body-checksum-node": "^3.972.19", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/middleware-sdk-glacier": "^3.972.18", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/body-checksum-browser": "^3.972.22", + "@aws-sdk/body-checksum-node": "^3.972.22", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/middleware-sdk-glacier": "^3.972.21", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -1980,20 +1712,18 @@ } }, "node_modules/@aws-sdk/client-global-accelerator": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-global-accelerator/-/client-global-accelerator-3.1070.0.tgz", - "integrity": "sha512-wSQSoUAtl0MrrRlkOuNZLZKmkOvHPVJJsSXg5NshJbpwud4IZ0EdTKhMJZUyyWC6zFVeiWdhxmVMtOHFV3Efrw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-global-accelerator/-/client-global-accelerator-3.1079.0.tgz", + "integrity": "sha512-UmozsoX/EwSoSb2wO2W/x368cFHIbl7RSxj7y6Vnu1LHEuXPaCDKHwfpn2r8XMyBeG29TnbhaJ0fXbkAyDPt8A==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2001,20 +1731,18 @@ } }, "node_modules/@aws-sdk/client-glue": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-glue/-/client-glue-3.1070.0.tgz", - "integrity": "sha512-+60FXsEiSDDtGeC2r0bmjOEcmvl96V1xVtYz4x9dI9TUklpXkxdb1w6lMYUHyDjoFhSFXv+bRMZjom76E7e3XA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-glue/-/client-glue-3.1079.0.tgz", + "integrity": "sha512-Ncnm8FOBqBpp/F1MPifa8chpqZEExpSGdgsggEcTHKHWJS8K9bQWwZdIZIz2GH+ZlvdlKygMQeqOZbWuYr6vTg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2022,20 +1750,18 @@ } }, "node_modules/@aws-sdk/client-grafana": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-grafana/-/client-grafana-3.1070.0.tgz", - "integrity": "sha512-zL3nmFp15U98rCSqAvA1YE+t4wsYEg0Y9WFEHftIEFFMiitwjER1bkbl6KV2bZOhU5n2PaksfxhEJvS5/nUd3A==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-grafana/-/client-grafana-3.1079.0.tgz", + "integrity": "sha512-JrFY6UUVwhRxOLjR9gRmqZTCixBEnhTPmgmTnQwqgvtgh9lAP+HgCaOet650k3oL8gzZ6zxr+hT1Q4B9J4RRTw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2043,20 +1769,18 @@ } }, "node_modules/@aws-sdk/client-guardduty": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-guardduty/-/client-guardduty-3.1070.0.tgz", - "integrity": "sha512-aMV3E0kZhi1oR7giD0cMVqJ965bBpEzkyy8JFwyk/+trijhUpBCwMFStLN61BHzklQ7Gw+fDV6YmIyDBHoj1Wg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-guardduty/-/client-guardduty-3.1079.0.tgz", + "integrity": "sha512-bryCQBQ/n/d/6/Wz3msxXUTuSwXhysTmzMHr5okHKBd0KM2HZRNyo3v+Y2w3buETvECopMXPdxEDiDjbsl2bQQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2064,20 +1788,18 @@ } }, "node_modules/@aws-sdk/client-iam": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-iam/-/client-iam-3.1070.0.tgz", - "integrity": "sha512-IC/S11y/e7bxwpgvieFQx4nMn/fsOjNj85n03PQfrQb52X/wC2/Ha25ZD3LOnP1DIWrfDcy3dltKkznbQz1Mfw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-iam/-/client-iam-3.1079.0.tgz", + "integrity": "sha512-BKM1GzF00yeIchUvXbcM3bw72F1JFCgum7s7e95bM+WGeOflpkx1e19OUzABbZnTdmjBf1o7IZ84+qmwoQI8Aw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2085,20 +1807,18 @@ } }, "node_modules/@aws-sdk/client-identitystore": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-identitystore/-/client-identitystore-3.1070.0.tgz", - "integrity": "sha512-IEruMNBIGpYfjwwvTIlg7MIocA+6ZdMMngQ1itM09Of2mE9lEwuecDE7ciajkfbIf8U191wV3Jzyz1RV+jy+7A==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-identitystore/-/client-identitystore-3.1079.0.tgz", + "integrity": "sha512-e/SpLAikU0JEnSVB1rTg4AGmKgtqi5NDwCRpOaQ9z8Jyp5gG+pmEwLSk196Czlyyq5EwMarZyffKLZcnkQNGtw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2106,20 +1826,18 @@ } }, "node_modules/@aws-sdk/client-inspector2": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-inspector2/-/client-inspector2-3.1070.0.tgz", - "integrity": "sha512-bQQ97x4FwlBhqIi5BtU0HnYXcMHevhWItc2XW9UR5hb1E63SIA5uBjoY7QyXCRTsz4kWoXUgLFGwVgyIGTIQnQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-inspector2/-/client-inspector2-3.1079.0.tgz", + "integrity": "sha512-wRB3cqrpMu8P3jpZhVVw//LOzMw5Or+bwVgxJBzUnNEJFq2NjYx6dt7KfqPz9owo7yXCDhllMD3E6ylRJRrwQw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2127,20 +1845,18 @@ } }, "node_modules/@aws-sdk/client-iot": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-iot/-/client-iot-3.1070.0.tgz", - "integrity": "sha512-zNH7a+VzhWvJdSaxM9vofPIwbiWejn0kKyQp535MSti+bVAu0mR7aq6qU7W5or42HxKn+2dgn5Ul8Oe0k3D77g==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-iot/-/client-iot-3.1079.0.tgz", + "integrity": "sha512-7b1nYxqyBRQFrs4wk7V017ZzsqRVKkOusb8IA+oMlHv8Z16rlTKmonJptNmslfTZKSji9A73Ntd9uH/oWIZHBw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2148,20 +1864,18 @@ } }, "node_modules/@aws-sdk/client-iot-data-plane": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-iot-data-plane/-/client-iot-data-plane-3.1070.0.tgz", - "integrity": "sha512-0KkPkewB6KugSurFC4qyQfdEktxksCBtjjB8M1qSz9P0Lmh1UHVidUdwx14yd0KWuS0BMoOllAEF9Vd5zCvukw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-iot-data-plane/-/client-iot-data-plane-3.1079.0.tgz", + "integrity": "sha512-KshONayLXeU4tGu2bGM2cHjmPqzLRObGIemcw2WPjshKhqVmtftzULGWkmZnEhV280Pb25BdEMg+SfzzbcjVWw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2169,20 +1883,18 @@ } }, "node_modules/@aws-sdk/client-iot-wireless": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-iot-wireless/-/client-iot-wireless-3.1070.0.tgz", - "integrity": "sha512-PfCTuL0+XjDJWYVN01z68RXgDmurdBrA03fQbq/l71pwGK+V69/8P4J037hoaEfzk0uLsriLDr9h1gDq1N90mw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-iot-wireless/-/client-iot-wireless-3.1079.0.tgz", + "integrity": "sha512-dns30V5aEHmDFn89Wercz/IrT2jTT0pZXt7P6xiMwzjTHbkhKXq4ofypTvbrlD5ey8u3nRlxzCk3QKrnhHjbzw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2240,37 +1952,19 @@ "node": ">=20.0.0" } }, - "node_modules/@aws-sdk/client-iotanalytics/node_modules/@aws-sdk/util-endpoints": { - "version": "3.986.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.986.0.tgz", - "integrity": "sha512-Mqi79L38qi1gCG3adlVdbNrSxvcm1IPDLiJPA3OBypY5ewxUyWbaA3DD4goG+EwET6LSFgZJcRSIh6KBNpP5pA==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.1", - "@smithy/types": "^4.12.0", - "@smithy/url-parser": "^4.2.8", - "@smithy/util-endpoints": "^3.2.8", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, "node_modules/@aws-sdk/client-kafka": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-kafka/-/client-kafka-3.1070.0.tgz", - "integrity": "sha512-QsWjCDfx3dut8PswcI1BBsRLaCLh2tkeofI3D+x8ak94IH2mpldxxNphK/dVgxuNrtWoYJMWrQlrgkzKkHTrsw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-kafka/-/client-kafka-3.1079.0.tgz", + "integrity": "sha512-jjl2t1GTmgJHMLuKuSpgk6h/9F8nX3nAZQxljxWBRWQq1448AzhgPiVEqx2n3Vp1TcwIOHaHFtH9mEwt9gi/4Q==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2278,20 +1972,18 @@ } }, "node_modules/@aws-sdk/client-keyspaces": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-keyspaces/-/client-keyspaces-3.1070.0.tgz", - "integrity": "sha512-K+uIuDMGc6Cku7uTOS/f641XgDUofdvJ3eOUNImwcbrOwGZY/OjX2FUn2D+MGJdd02Bi/4wqGzxuuNTgNp+VAw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-keyspaces/-/client-keyspaces-3.1079.0.tgz", + "integrity": "sha512-fYmcpT9K2dhJDDHGbvoRRUD4hH3VChkBYR+QBQEwHVR9unLe8AIg3DWDJLjeUzJWlgIRqqz+pBGc5eqS1582Dw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2299,20 +1991,18 @@ } }, "node_modules/@aws-sdk/client-kinesis": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-kinesis/-/client-kinesis-3.1070.0.tgz", - "integrity": "sha512-PoYjA2UX9upzmL37vl6qQ8SyoKT4tT8UWYEMmKv34Dw172eqnL7qs+WZY0JPDOLA2n1AMmzjYcHdd8G2oyP7jQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-kinesis/-/client-kinesis-3.1079.0.tgz", + "integrity": "sha512-K8VoT6sTieXQPa/g6u8KEPgiLO50P7YleGmyBulbDoio4Ptb7ii4vZsyfJFJhomjyLq779dIWgwgcm/qVgujPQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2320,20 +2010,18 @@ } }, "node_modules/@aws-sdk/client-kinesis-analytics": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-kinesis-analytics/-/client-kinesis-analytics-3.1070.0.tgz", - "integrity": "sha512-2KE88YbQ5+EVNbDsFSKXZ34M0qtLHhpHHyglHYFE/dUvwGRT4+aIrHCpib1RM/Ro+VQpdQ7c5qpVWETKYd+NLQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-kinesis-analytics/-/client-kinesis-analytics-3.1079.0.tgz", + "integrity": "sha512-1bEgEmBCVumlci9RRigR/BvfhqU8f/clOlWQb+LbbbjxuoZzZCbER3j40dQSmtE5wABAp3IcJgXoU6jK9i3vyQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2341,20 +2029,18 @@ } }, "node_modules/@aws-sdk/client-kinesis-analytics-v2": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-kinesis-analytics-v2/-/client-kinesis-analytics-v2-3.1070.0.tgz", - "integrity": "sha512-uPTij/+b0TaURAkmlMclwwkTF6yPHAVJ/p8ffX5Tl8rDN5Qrg26Pt7kdwg3j4SP+tvg3jpSy4E11v5vUKTyakg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-kinesis-analytics-v2/-/client-kinesis-analytics-v2-3.1079.0.tgz", + "integrity": "sha512-SKoYGNfYekFLCMp6GfYsBXVjbNHoFoueO3VDXSwI+wlGSWiXHKY7J2LkeSTmVZ62+Mm3KqJL/ITUtIUOR8I5Rg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2362,20 +2048,18 @@ } }, "node_modules/@aws-sdk/client-kinesis-video": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-kinesis-video/-/client-kinesis-video-3.1070.0.tgz", - "integrity": "sha512-+En98FLXMI8POP2gIR8V3dp1GBT4lPVcRxvgVQXAKwWrZ3Yqp6vMmJsYmXo2h/Wt5fBRnnpNOsel0JX/8figPw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-kinesis-video/-/client-kinesis-video-3.1079.0.tgz", + "integrity": "sha512-MQrXqahSjN2Z9Jazdjs+PlHT1qIUFDUaMPkCFt9NRgAlM4wvQfmdIyWng5HQojHF/Gn+3q/OaOvZOiMcz9v/eg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2383,20 +2067,18 @@ } }, "node_modules/@aws-sdk/client-kms": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-kms/-/client-kms-3.1070.0.tgz", - "integrity": "sha512-ELJH11w1IU0V5kqQ/eg2iu4HqaXOJQ/3nC8I7y93+mZW39osINlOHFKgYQNizwZA5KoSxSgVYmLPnnF/i0oKmw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-kms/-/client-kms-3.1079.0.tgz", + "integrity": "sha512-1ptW3hfk437clc43QyWyCBk6D243h2t03zEWJ6KbTki/BTH3FpAWDp8dfElL9k7zisVtVQM8N7s8F5gHH9lIAg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2404,20 +2086,18 @@ } }, "node_modules/@aws-sdk/client-lakeformation": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-lakeformation/-/client-lakeformation-3.1070.0.tgz", - "integrity": "sha512-eA55iFfW9mv+8dczSBxARUrHBy/fWHUUHdSv7QxHMYcJy/VbbbJyPrvrQ8vnSO4wj2bXbJ2BN46Y1eP1w9/85w==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-lakeformation/-/client-lakeformation-3.1079.0.tgz", + "integrity": "sha512-A+88nvvqqakUR8kmu3TxuxWdMr2fAvbXAHd1i8sa1ugzoV5YFzGSXW4MVGGEHZNvhjm2mDmbgzbAQKJb4luRgg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2425,20 +2105,18 @@ } }, "node_modules/@aws-sdk/client-lambda": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-lambda/-/client-lambda-3.1070.0.tgz", - "integrity": "sha512-v9xBUFCtLnhrdf/sWZETvGggTDDMF0dLTQ/D9AmHBXSOjJ9BZ8O3V2Mr0htfu5A58dK8Rkxcw5f7YMuFui35mw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-lambda/-/client-lambda-3.1079.0.tgz", + "integrity": "sha512-+OT6gfX+seDw4fuKnXNlYLvmtessZpnr7DczK507mX48uQlZbHrRObu7zL5PLNdUiNomlNR/mOz9i+TY28GY8Q==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2446,20 +2124,18 @@ } }, "node_modules/@aws-sdk/client-lightsail": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-lightsail/-/client-lightsail-3.1070.0.tgz", - "integrity": "sha512-ph9l03QleZRNckaossFM3uTENEQAkbyEw/VdweKOtpZMDmNk11tOSJH1ksch8C0x7SP43WGcUADx1fmlg5y8lg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-lightsail/-/client-lightsail-3.1079.0.tgz", + "integrity": "sha512-sGfnOTQMHRdxuWU6PJnbcSxkIfgpHUnhPAVNutdJsiSB8/OzrtYcKUp2Mnyr04R8tQaLD+udrWG+a8/VV/C0fQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2467,20 +2143,18 @@ } }, "node_modules/@aws-sdk/client-macie2": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-macie2/-/client-macie2-3.1070.0.tgz", - "integrity": "sha512-e6B1v5o8IjaSLWUjgmYwGfU4wA6d4DK1yKEM0pvlq6EE5UnN8D/gHP4ogca0UxCbwDrEaXE9jYXJU4jDO1ajiw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-macie2/-/client-macie2-3.1079.0.tgz", + "integrity": "sha512-MgQeYsUJXGmB9CcNduWDwbFy6kqhbfAZb2y7uhdg6uY1sbRhhuTCU0KoOoZskLfDg/eTz8rjzmkLgfnsr5UmDQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2488,20 +2162,18 @@ } }, "node_modules/@aws-sdk/client-managedblockchain": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-managedblockchain/-/client-managedblockchain-3.1070.0.tgz", - "integrity": "sha512-Bi8xpG6ds67ML0vyajTabv9BuzM3ya8IIh///qhUlT18tgRU2kMd2YpFQ4VS0jf7VeCkpq+QAHlfRHnnM6YQQw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-managedblockchain/-/client-managedblockchain-3.1079.0.tgz", + "integrity": "sha512-sk+9o8mDPf3k3DOVM/6O6Y9GUiQdJjGwMdmLh1X3OdIIz3Tfd3uD75b+r5zdjgQoL+iTsUEcf50RG5Yas/NFVQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2509,20 +2181,18 @@ } }, "node_modules/@aws-sdk/client-mediaconvert": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-mediaconvert/-/client-mediaconvert-3.1070.0.tgz", - "integrity": "sha512-2AzkhJh+4dmhk/tdfOf4MBLxGve+VrFHVvaluj/RCi1qpxgtMhIIo3ZMzOR0K/DaasWG9HcBM8HrGfDm0WSppg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-mediaconvert/-/client-mediaconvert-3.1079.0.tgz", + "integrity": "sha512-LkZCd/9aKeTc7qJ8W33InCi1bF+CcNOzujFFbd3Z9H1VhtgZowJfnasq2y9AlGCvd7HpCCrEZu5pOFV184Y58w==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2530,20 +2200,18 @@ } }, "node_modules/@aws-sdk/client-medialive": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-medialive/-/client-medialive-3.1070.0.tgz", - "integrity": "sha512-ScCBxzY+IVe7ugfzxDhPQPcLAkRQ/Pa7ZH/BrCPSZ3Z3/+tweT1YwqqRe09BzutkUOMcaUhs8//ZLQj6YRs75g==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-medialive/-/client-medialive-3.1079.0.tgz", + "integrity": "sha512-KTe7wdLUgC/cyBSj1yUgPnGMiOa4Z8FslFVxh/SS7ZgFiVxrnkdARlgvHJdCRuVsQY2ynGpuf3VYRc82hkTosQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2551,20 +2219,18 @@ } }, "node_modules/@aws-sdk/client-mediapackage": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-mediapackage/-/client-mediapackage-3.1070.0.tgz", - "integrity": "sha512-nWY9Hca3BuQG4/9KxMwlh5asLmck1zUcanjxWO4Woi3hjLGU4QdjhleOUIzG3cX1BhLvSS/LtuEuhr3cyvOH1Q==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-mediapackage/-/client-mediapackage-3.1079.0.tgz", + "integrity": "sha512-UPb41vZP2qsyFFOUnCdK4VRRFUh6xLohcYPhOZBdDrXuplnUOQVzoDpa4I+0opMWJF+cUrQL+yVX7PRAqJkmKA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2572,20 +2238,18 @@ } }, "node_modules/@aws-sdk/client-mediastore": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-mediastore/-/client-mediastore-3.1070.0.tgz", - "integrity": "sha512-1YFd00OOpvLAs0G7QcWg9ANlfqcD+AAooAeBixH7Y+mUSfIhnRr8L6n0nrifAsrcbkHjMNpOWX3qyy8qp+swEw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-mediastore/-/client-mediastore-3.1079.0.tgz", + "integrity": "sha512-hY2n7HcZc2VqZVd2eNMB0hpMi1i1fwX8XspfrynvikgCRCrvCgLEBFZS5pYiabuRs3REOqumQy8wo905Y4vsYA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2593,20 +2257,18 @@ } }, "node_modules/@aws-sdk/client-mediastore-data": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-mediastore-data/-/client-mediastore-data-3.1070.0.tgz", - "integrity": "sha512-+ctvCHiCfzZ2PDm/xVIQOfQEVIfgnir2gTPp3vbNTukxrBp9Mjoh8OLeyh1TrtP2jKC5AGv/PMJv44xcCf4zRA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-mediastore-data/-/client-mediastore-data-3.1079.0.tgz", + "integrity": "sha512-8A70Ek3lZJNAhEylNbkCF073rUsAR1Q9LmL4Pc7rfYR2BcpJAiD8x+mimgPnIc0TFe/Wxa/P72zwX05s3V3HNw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2614,20 +2276,18 @@ } }, "node_modules/@aws-sdk/client-mediatailor": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-mediatailor/-/client-mediatailor-3.1070.0.tgz", - "integrity": "sha512-zn0OkYipsL01phW2ypn3JsnBcAGzr3r0MzfVwT9LuPBF0fO6e0O1kiV/JKQf+1msysmjqh/N/nesMcbu4intrQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-mediatailor/-/client-mediatailor-3.1079.0.tgz", + "integrity": "sha512-hT5Dfq/hniM+VI3iJPYxcLP0zJTfTAk7mqvt+HUeFHuHzt5xFo+8urP4CNfaQgQY2Ft4v7gP8k53zDorY2MaNQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2635,20 +2295,18 @@ } }, "node_modules/@aws-sdk/client-memorydb": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-memorydb/-/client-memorydb-3.1070.0.tgz", - "integrity": "sha512-b7Wr8SdtJuL1O5C1vwH2tj6OUygybLnrnxkEX81PGdYugcwim4KGWpO/NYYKP56NGA+JX56StsXhxPFV7w6VxA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-memorydb/-/client-memorydb-3.1079.0.tgz", + "integrity": "sha512-B7LOi68Oh+W8ZdM/IZxq5T4g4CpZ2BfwAPLT1hC8e/0uhUS2cB0sryrco1W4iA2RTTIV73Vb2v+LsCSlI+3wwQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2656,20 +2314,18 @@ } }, "node_modules/@aws-sdk/client-mgn": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-mgn/-/client-mgn-3.1070.0.tgz", - "integrity": "sha512-gzesXB+pPlfjjAlzGGMyZzGM+iqU0IisznDMUDULqEKuI6n3GuBmrBzhj0JIJo3Hi4+RHTOhXMrDhPQ6haosRQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-mgn/-/client-mgn-3.1079.0.tgz", + "integrity": "sha512-zmA0xl2pFPg7o+7Ls7U5WdFCEQy00e0RXgWvlXEjKuwZfCyQXpC+/Fcd+zzXDE5tkZ+cou+2cdxBylnEo6SBTg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2677,20 +2333,18 @@ } }, "node_modules/@aws-sdk/client-mq": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-mq/-/client-mq-3.1070.0.tgz", - "integrity": "sha512-ABFWOwgoBXzomW21xEo/JLypGjBRJ2jFV+r4MmKJtnXw2i82QEN98bYrpDmPkfW5wAMm6LtOD2cq4595VmwOEQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-mq/-/client-mq-3.1079.0.tgz", + "integrity": "sha512-jDiRn0brCihD6yq8fRegc4k6AwAjUeXHXfu74pxclUyMF3djwmTaPCJkrZGuadpA8CDxIsN0guBjXnoZg//zKw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2698,20 +2352,18 @@ } }, "node_modules/@aws-sdk/client-mwaa": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-mwaa/-/client-mwaa-3.1070.0.tgz", - "integrity": "sha512-c77jMOMlusl+K5hM+y6erdQR+CyYZUPoSX4ANN+1j3W8gwwCcaF+DqjKcaqKWeSdf7cli3FO+5F4WSTuqIG4vQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-mwaa/-/client-mwaa-3.1079.0.tgz", + "integrity": "sha512-PfN8vH07bi2qkyHktgKv2nh1oMsqQg3UrYXGE2c3pczHf87de34+cfo91hCCo7wdQkqHu07h3TpCdgsD2H2s6w==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2719,21 +2371,19 @@ } }, "node_modules/@aws-sdk/client-neptune": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-neptune/-/client-neptune-3.1070.0.tgz", - "integrity": "sha512-xltyHUsp1Vk+E/cByugjvUHvmbnkLMB1xsLhls0U+oCv3mIyfCT9noB4NWgyKTWm9rV3O0/P+uuV9s7+kaAj7Q==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-neptune/-/client-neptune-3.1079.0.tgz", + "integrity": "sha512-tVG/5VcNE86P71QTyKl7F0hB1k2obroc2R8A6Fk1HA66YrzbPKOetSi28N7s8cK3kH6mTWQ0/rNQyhV5WO4W+g==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/middleware-sdk-rds": "^3.972.35", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/middleware-sdk-rds": "^3.972.41", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2741,20 +2391,18 @@ } }, "node_modules/@aws-sdk/client-networkmanager": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-networkmanager/-/client-networkmanager-3.1070.0.tgz", - "integrity": "sha512-TfrFF0fjeDgkqc9mx+R/CYZ7u/T73uAu/pvW96z+gzGC4lc2Or6hW8HF+UJyaU7fbkvDmUsK5UeU9uuTJ9ntng==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-networkmanager/-/client-networkmanager-3.1079.0.tgz", + "integrity": "sha512-vhpjq9Gmy5rfUrEZtUUu3bkFt+sPSGD+cnrWpMzw5RP6IxXZB5mvQ8tPqoVTFi107eGXhmAHASUV0C/IGgYNww==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2762,20 +2410,18 @@ } }, "node_modules/@aws-sdk/client-opensearch": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-opensearch/-/client-opensearch-3.1070.0.tgz", - "integrity": "sha512-65ZuBx+GhDknlbxsT18dWvWUACmCwuYe0n+PDv+UaYTWt7fG3lhmpfE19VD6FMHAo2oYF2fejm2F6GJRFNBoqA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-opensearch/-/client-opensearch-3.1079.0.tgz", + "integrity": "sha512-+NLrisczOevuc37MoiTWBIsYMZaXUm5Tz8HlwP3e8IIaIgWSXgMumuO7DxC9lmmdu56IGlt1sxyub2lTvM4uYw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2783,20 +2429,18 @@ } }, "node_modules/@aws-sdk/client-organizations": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-organizations/-/client-organizations-3.1070.0.tgz", - "integrity": "sha512-NO4240ZlJieSk3zwqs1mdRdNRD22mM4baH7sf1ADoOES51TJKE8MWaTFiB9HqLBnx/7m+gCMH4cVIWoz31+ptw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-organizations/-/client-organizations-3.1079.0.tgz", + "integrity": "sha512-wDZVNTj6c/OE3yB1/QQO7pbldJChfM9zg8NO5PYEj1YOnwu2fHhPtfpeh3gmw8ORM41vfqIP2I1Q1PpaE3y1/Q==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2804,20 +2448,18 @@ } }, "node_modules/@aws-sdk/client-outposts": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-outposts/-/client-outposts-3.1070.0.tgz", - "integrity": "sha512-afOlXw7VVUVhKxrXxFY/cUCKrbO8OTmmZA4rIGx1apcthzzZ3Vad3nJUbZKxhZR2RSyyHhM/PQkOicFnKKDdlA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-outposts/-/client-outposts-3.1079.0.tgz", + "integrity": "sha512-Z018pNR19HMRkn0hr6ZdPmrnPmFiaZlEL6o7aj4TLzzeP5ULBPXkgX+sHyfvd69Bxe7o1/nB/IfkTsiON4wLaA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2825,20 +2467,18 @@ } }, "node_modules/@aws-sdk/client-personalize": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-personalize/-/client-personalize-3.1070.0.tgz", - "integrity": "sha512-Ly4yi8JTC23XS+eKltyNpmTnRRmqrnFboygwFymifYAmsSMDSqMVUmW58SylTEIfY8Lb8jaJT1wtxu3FNft8CA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-personalize/-/client-personalize-3.1079.0.tgz", + "integrity": "sha512-q/q6x12UfQ1lJeEszIIUUd5lR2AMfVdRHbRM/9xBkwAilDoKOvy/xEXvVD/sq5nw5dcBu0tIs9aKcVZ6ool6Xg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2846,20 +2486,18 @@ } }, "node_modules/@aws-sdk/client-pinpoint": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-pinpoint/-/client-pinpoint-3.1070.0.tgz", - "integrity": "sha512-AiHk0vL5uTBtKMGDt9CFtns2pQFPrqozP8CSb5uILWxigVI/YhOz2MABPoOzO1kmphns2b6YjW7gUa7Sz7AfZQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-pinpoint/-/client-pinpoint-3.1079.0.tgz", + "integrity": "sha512-jpw26M5tX/m3O5/N9NTH91+wVyE9druiTo6Oi+tUtzUNVB69KmW4l71KRt1u5V0zDQxC/urNyjdTxqivXztLRA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2867,20 +2505,18 @@ } }, "node_modules/@aws-sdk/client-pipes": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-pipes/-/client-pipes-3.1070.0.tgz", - "integrity": "sha512-DKQPOOob2NSw3Zx1uGov4YXvyR6nl2GNoQDmpJhoWtx7MLM5AcsRcaSvbL0PXMsElDK5SplY+QYx6ExwynjuEA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-pipes/-/client-pipes-3.1079.0.tgz", + "integrity": "sha512-J2JwIaGY/sTccrvJT3aonPVkngSaHhS8PHEec83+bkxHkhIMhc3bpC0LcJUxP2/u1HijKCx78zBMKBptUn42gw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2888,22 +2524,20 @@ } }, "node_modules/@aws-sdk/client-polly": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-polly/-/client-polly-3.1070.0.tgz", - "integrity": "sha512-DuPqiI/YwTd8gaxkPnG6bv9G5RPpg8b5kkSwtnNj9uTCfqKxiX/Ud89s2E0X8hflkxwxlunU7JF+rmWfmj0GCw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-polly/-/client-polly-3.1079.0.tgz", + "integrity": "sha512-cgbLq64c2IfPU04EkTEKXxij6y4EBwN8FUvNyYUsmQeFkuRbY3f8gywBR/Z+SPLJBVAZBGfjFL6YSLAJq//CqQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/eventstream-handler-node": "^3.972.22", - "@aws-sdk/middleware-eventstream": "^3.972.18", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/eventstream-handler-node": "^3.972.25", + "@aws-sdk/middleware-eventstream": "^3.972.21", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2911,20 +2545,18 @@ } }, "node_modules/@aws-sdk/client-quicksight": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-quicksight/-/client-quicksight-3.1070.0.tgz", - "integrity": "sha512-IaOHzXdcSZC5wRiaM0FddZFa68FDrcRVK4adi9AvXFTjeAAXYHzBd6kathTIXjFI5N4naGqlV5knqKBG3f7yMw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-quicksight/-/client-quicksight-3.1079.0.tgz", + "integrity": "sha512-bbFpr/dM1yPXh+ti/RLfkgJwQdS8DDCyi9UJsT1iqk998W47zHpYo5PzbWN4Vrr0U+tDUsMReqMVLKGp4DY9QQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2932,20 +2564,18 @@ } }, "node_modules/@aws-sdk/client-ram": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-ram/-/client-ram-3.1070.0.tgz", - "integrity": "sha512-5YCQwwYdSsaWFduaYVCqnjYpgnlX3/aIs6cmpvF8Bo76w4ediSmIWhq8WSjHgrp+8+gJRvei8Imw1i9eqX0V5Q==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-ram/-/client-ram-3.1079.0.tgz", + "integrity": "sha512-OWD3z34lbkRqPOO9MLdKF5Ev511u3YrbI1rO0KxKytJBFsRyhMyYKr+HF9eI3iseizyEkBWgku/7ZjcG5dsSTA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2953,21 +2583,19 @@ } }, "node_modules/@aws-sdk/client-rds": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-rds/-/client-rds-3.1070.0.tgz", - "integrity": "sha512-ifPpp5izdTztL07ZGXKbxHCrLuPOBh6dDa6dYEhAAjrpiilMFDj/93a96aVVafV67rjR5Xd5h8sWGaBMMAWNZA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-rds/-/client-rds-3.1079.0.tgz", + "integrity": "sha512-B+OFy7rVntPeBF6lQNKnP7Xbo/Xy1LOqdzo2xK18VXn06HhBUzeQj6Ke9qKbnR8ka+SyczXCgJiLNgFqKVRyyA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/middleware-sdk-rds": "^3.972.35", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/middleware-sdk-rds": "^3.972.41", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2975,20 +2603,18 @@ } }, "node_modules/@aws-sdk/client-rds-data": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-rds-data/-/client-rds-data-3.1070.0.tgz", - "integrity": "sha512-AAHPInxPoNoiC63dPkzpBZk3Hb0xGbpj0Qimj9vf3rJ/qU3pqsFT1DhJnWessD4uJ9G0TCA1jyJ4TZ03CUZaAA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-rds-data/-/client-rds-data-3.1079.0.tgz", + "integrity": "sha512-j0ioaRcfmKf4eF3tUydplsrrD21YL0nx5IVPwYQFzJQFBWrfk+M+xelG+IEA9fEdstkTk1nC2wDzI9+TJovlYg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -2996,20 +2622,18 @@ } }, "node_modules/@aws-sdk/client-redshift": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-redshift/-/client-redshift-3.1070.0.tgz", - "integrity": "sha512-Gq7LitLPDhWst6bfwUhsiMUbXewaToHd29waz6k9KaQ9Iiu0usEBCJ+GP3qU8M3sRC20fIPDw5gnVfgV5osDZw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-redshift/-/client-redshift-3.1079.0.tgz", + "integrity": "sha512-XOR/Yz3KEqnOPRGbmHNdLOIsjRFqF4WxELcle95tSLAE1+k3DQjPfga0RsfQpjR8HSHviPxzl9FiiFvZAYpqZQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3017,20 +2641,18 @@ } }, "node_modules/@aws-sdk/client-redshift-data": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-redshift-data/-/client-redshift-data-3.1070.0.tgz", - "integrity": "sha512-Kavy6PIfjlNR5T4VZfWdAWmzRtS8s/klYP+qFg7q3PiM/xciPBCjxvIMCEYQihMgvbB/EsM2Ej9UtrtZQGNvBQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-redshift-data/-/client-redshift-data-3.1079.0.tgz", + "integrity": "sha512-r5ZGJAHlghzNEj6+ETVvtY+3LBqeWCuBp/XkOA/a3arE3i9YpWdEslLcE54GICDI+Vvud+IZc2ZqT9FI6w9PQA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3038,20 +2660,18 @@ } }, "node_modules/@aws-sdk/client-rekognition": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-rekognition/-/client-rekognition-3.1070.0.tgz", - "integrity": "sha512-UI54K80Ub5di5Zrq/wk3I8nNbdmmLc3DzctlYCXT1HQDCFd6zcVk63SmnubCWh3XuB/f446gMGOb7uMdoA6SKQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-rekognition/-/client-rekognition-3.1079.0.tgz", + "integrity": "sha512-tm4XAqG7HX+437ErRes9yI8crRZ3iGWX/xNKVi0EdaNnJpnXrev1VY6V8OyWp8qD1kmTC/PXmrr/twr8NoS/AQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3059,20 +2679,18 @@ } }, "node_modules/@aws-sdk/client-resiliencehub": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-resiliencehub/-/client-resiliencehub-3.1070.0.tgz", - "integrity": "sha512-fUUEEutRULHpBeuzbd+DiB9gL+uY93PMv5deaZF6X5duM4GJKJrBFcImInu+3KY0hq2WMORJ3fe0etxSeXMEHQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-resiliencehub/-/client-resiliencehub-3.1079.0.tgz", + "integrity": "sha512-ronbRz5U+7f7G4A7AeDG6Tucrt1pNzTPnGIAR+ZA1zYvmGLM++RzChtpKxYzBCqcEGad6sUv806q6cG1QhA77A==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3080,20 +2698,18 @@ } }, "node_modules/@aws-sdk/client-resource-groups": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-resource-groups/-/client-resource-groups-3.1070.0.tgz", - "integrity": "sha512-XAyHCTa/Zr2Y+2kFlSkqct8/hmOU/BSVG3/J6kLb6K3gY5zpLjW12XYZ/EShtZscLpjvRzprCiJhybJlnNbI0Q==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-resource-groups/-/client-resource-groups-3.1079.0.tgz", + "integrity": "sha512-UhwO8sGh0Bdh0MFh3YeLmD0/ONXz/nT7rPCAe+Q9mjX7K6w6vjT6AtTx18CMd4NuIzOAx9XMIBk3CICTVbtovA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3101,20 +2717,18 @@ } }, "node_modules/@aws-sdk/client-resource-groups-tagging-api": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-resource-groups-tagging-api/-/client-resource-groups-tagging-api-3.1070.0.tgz", - "integrity": "sha512-gjEv0XWl8mia2DIO+GoO7bC0UebYRU+YzcOi+BDBVLA1y2CA5INp2RcIMuOjaJiyqDxUyHCLQyi4UjFQtFrGuQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-resource-groups-tagging-api/-/client-resource-groups-tagging-api-3.1079.0.tgz", + "integrity": "sha512-Bvh6VrnTt8R+AvSc1rbZGxbfI6rQTBhUZ/2vJNO2tygxunenSPspkZUpSUf4o2zurZMvJaRv7VV/Zg2bJ7o+Uw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3122,20 +2736,18 @@ } }, "node_modules/@aws-sdk/client-rolesanywhere": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-rolesanywhere/-/client-rolesanywhere-3.1070.0.tgz", - "integrity": "sha512-OFuFlofZXFavK8qnCfv+0K3qI7wMdLG5U8wWcYOCsjizug+RCREPhJispCT03wRqLjNbPQYo9c6OxIOt6EyNNg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-rolesanywhere/-/client-rolesanywhere-3.1079.0.tgz", + "integrity": "sha512-yktiWBMli1YuY4QwbZrayjbN25S/XavWAp8kGqdwCpAqT6+3rLNHEfYpvXTYYJuCtm4hH5dZFiLB4FG31tfxfQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3143,21 +2755,19 @@ } }, "node_modules/@aws-sdk/client-route-53": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-route-53/-/client-route-53-3.1070.0.tgz", - "integrity": "sha512-8bntU8iFwXGR0PNlEoyUHPwTwsCCKcL08+DHMRLYJPNQqJkyr0VJJXQ+bvLo9Y+cre+0iE1+j7mFsgQZI94kQQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-route-53/-/client-route-53-3.1079.0.tgz", + "integrity": "sha512-sn6z0IFPpYkFrxEQHsl5C254rfoUsjwWK0bj/1Bf1cL+11017c1FQv1xJUucix0H43NY8gZIqT+KyxIITVUEwA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/middleware-sdk-route53": "^3.972.17", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/middleware-sdk-route53": "^3.972.19", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3165,20 +2775,18 @@ } }, "node_modules/@aws-sdk/client-route53resolver": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-route53resolver/-/client-route53resolver-3.1070.0.tgz", - "integrity": "sha512-jqhOUYdl+AwiN+Uvk/XH5zZ1jB9d+DjpKkV1ME0xav8g+gSptSam8vldq5yETXs0paSX44nSv55O/aYx+JzXgg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-route53resolver/-/client-route53resolver-3.1079.0.tgz", + "integrity": "sha512-4G/CqApchv583EQy8ImuMMUwmHB/mgjcI7/oGfrHiPibvUsJBb4GxrNDgdCXBEnRKVD1e3oidk5t9ET1pAdxzw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3186,24 +2794,21 @@ } }, "node_modules/@aws-sdk/client-s3": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1070.0.tgz", - "integrity": "sha512-B/OUiCqGQ4Zr7v9gFFyiuitKN2c0PIgvOlQb5bYg1SM2y0F8a5JQ7FNsjRcl+d2PqYWLHwHx12CvZDyLn4KxIw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1079.0.tgz", + "integrity": "sha512-di9U/7Po7qlVYb2dq58ULsbBAE1pBIk53rux+50LQCvH1X+/l1Ys+BIk/QLBtdaK1nADk0xRNEBbA1QWVnMccw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha1-browser": "5.2.0", - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/middleware-flexible-checksums": "^3.974.31", - "@aws-sdk/middleware-sdk-s3": "^3.972.52", - "@aws-sdk/signature-v4-multi-region": "^3.996.35", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/checksums": "^3.1000.12", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/middleware-sdk-s3": "^3.972.58", + "@aws-sdk/signature-v4-multi-region": "^3.996.38", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3211,22 +2816,20 @@ } }, "node_modules/@aws-sdk/client-s3-control": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3-control/-/client-s3-control-3.1070.0.tgz", - "integrity": "sha512-+2r4ExRgKiFyuPimP35bPQbGAQpJvLYNcDtenKMuulm9ubcBaoy4bvUZdSVmNWiJgntHp30ScioSMIZ9ooBocw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3-control/-/client-s3-control-3.1079.0.tgz", + "integrity": "sha512-ohZEqDwzC7b22z/dfxcI/V/7pwrKob8NpwsDaJ2zLJCbnfN57kPMo8Ix5oJ9dUbr/lMDBnpa6P3LtGDdyIhELg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/middleware-sdk-s3": "^3.972.52", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/middleware-apply-body-checksum": "^4.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/middleware-sdk-s3": "^3.972.58", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/middleware-apply-body-checksum": "^4.5.5", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3234,20 +2837,18 @@ } }, "node_modules/@aws-sdk/client-s3tables": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3tables/-/client-s3tables-3.1070.0.tgz", - "integrity": "sha512-TeYKdZWFsXjC/YWvylfOlNfDF18Yemzi9+x71+ij2gNlJ5MexFe2gKRlH/TsykgwstS6+LufYTKQo0B+WvnH5A==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3tables/-/client-s3tables-3.1079.0.tgz", + "integrity": "sha512-/LP+kP8pisT+WnGHKg0nVYNexZPf8DD8w+cXpV2IzADpR37P8Z9hoG4vaF8kWxxY3Pu4s7jYQ/17acpmq0m24g==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3255,20 +2856,18 @@ } }, "node_modules/@aws-sdk/client-sagemaker": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sagemaker/-/client-sagemaker-3.1070.0.tgz", - "integrity": "sha512-HM6KpJAQv9CF3brC7eW4i/iIJU65wa1cycX57ei6pDY7edZZllwrsCjkmMlXzPC8F8dIrZSuMKFvQPo7/VAazg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sagemaker/-/client-sagemaker-3.1079.0.tgz", + "integrity": "sha512-dx/e6FO5F6r6iG4tLX2xK7BCOVhVu26Xu6VMw/6AZ27Xff02xOFKsrLnhk6giENtPRramRGeNCceadQBCksDwQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3276,20 +2875,18 @@ } }, "node_modules/@aws-sdk/client-sagemaker-runtime": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sagemaker-runtime/-/client-sagemaker-runtime-3.1070.0.tgz", - "integrity": "sha512-JzH7KiO7S8uWKJa6txvcwyWI+u0xtLZs4lUOdAJGDwMJ8gQnCBDmfNjZxlo9DUneHmzXNneJPSKgI2X7Valm6Q==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sagemaker-runtime/-/client-sagemaker-runtime-3.1079.0.tgz", + "integrity": "sha512-MCxz1KMevqs83eJ8gD6QGeXT2LIImo4QWSqwsEwB24ephrWIwAUHp0YmxuLuFxqg4AdfKyKa3xLkGZmzBFd8pA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3297,20 +2894,18 @@ } }, "node_modules/@aws-sdk/client-scheduler": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-scheduler/-/client-scheduler-3.1070.0.tgz", - "integrity": "sha512-KjX6XmF+VDM/Dqw4ZeOpRCDRarEOSqJPNh8v4EWk3HFn7qf0PRuyvFOoVeH2DGC9kmnCpqtKcQf19GHDDV+JzQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-scheduler/-/client-scheduler-3.1079.0.tgz", + "integrity": "sha512-k2cQ6Mt3Tyf5tUWT7M6hvmGdvIm5ihv8EWI25Ghdl9Q6i2tkO13XL12NKfUK2odSN6s1YJ8hPC9FXJQZGA7Eiw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3318,20 +2913,18 @@ } }, "node_modules/@aws-sdk/client-secrets-manager": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-secrets-manager/-/client-secrets-manager-3.1070.0.tgz", - "integrity": "sha512-5t6TmKsESxrrd6iD1tRXaiQxusjboZa846pLZZUwvt4M2xy3FsHUaQoZ50ZPnSr5DNWJubbrHuB+jKn4LGO1eQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-secrets-manager/-/client-secrets-manager-3.1079.0.tgz", + "integrity": "sha512-BnN3yTP28Z+Ac3bsGpVTHPQVpLFFs1tSVJ+QxaWJsvb3curmlGTCHXCGxPSpk+NzzbOfwjE8ZA6NnU/orLr9Gw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3339,20 +2932,18 @@ } }, "node_modules/@aws-sdk/client-securityhub": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-securityhub/-/client-securityhub-3.1070.0.tgz", - "integrity": "sha512-Z5owRuChkXGzsih7fC9KBgdGEw7MvjkSXN04HZRtrRiq99VcI0UQSMqkmskybISe+l4qgrfuxJg3LkoIcar0Zg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-securityhub/-/client-securityhub-3.1079.0.tgz", + "integrity": "sha512-PpXDwAjaRJE/bKaoWDcaR0tWpdpMkahHdBBdSXfAmLnQmegDwwuYBJFf0YAHHzgqbLTsELFQB6E6pmlfhor7tg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3360,20 +2951,18 @@ } }, "node_modules/@aws-sdk/client-serverlessapplicationrepository": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-serverlessapplicationrepository/-/client-serverlessapplicationrepository-3.1070.0.tgz", - "integrity": "sha512-Ets8RCuTIa9g7peC5iYq3UfNgkb5msuZJR2r3muShcjF7SxX1FYEkNvlmrS6q+p2sSPtRU01OKqB0JSnzubYvA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-serverlessapplicationrepository/-/client-serverlessapplicationrepository-3.1079.0.tgz", + "integrity": "sha512-f/i45JACDtRPF1KNy/49FICblhGDdZSvS6T+5qETXQKRzPWtemULmLIjG89vsFzqTFF8NXUE36XyO7/gYYygPg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3381,20 +2970,18 @@ } }, "node_modules/@aws-sdk/client-servicediscovery": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-servicediscovery/-/client-servicediscovery-3.1070.0.tgz", - "integrity": "sha512-0Bc0jmsX0OtSKMqMpR4CzK2YlgPNNwuMUFar14krHdEiHT/rDh3DHDKfQuwMbTqvT2QqeFVrao+C0HwqzsalzQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-servicediscovery/-/client-servicediscovery-3.1079.0.tgz", + "integrity": "sha512-h4f3bHWFed0we41+WRvSxUMLWgTD4Ut2S/JwkI9X9O01YXe7F36yqHm+O+CN5pVaip99b0uCf6QMNElR/pvjQg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3402,20 +2989,18 @@ } }, "node_modules/@aws-sdk/client-ses": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-ses/-/client-ses-3.1070.0.tgz", - "integrity": "sha512-bPfie6sd2pcs4inpRTBnZzNN8ud5aup9ljI72DjOnI1xUzJKbWjS+i1RVcRjgB5zWdWawOhMYauGlrmKNGCUSg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-ses/-/client-ses-3.1079.0.tgz", + "integrity": "sha512-wQOqKTsoaeKuSnFekw8XCs0zaC7cu3TiLCYd7N5S/eiGdGBB20sob6OZhgGlg7C86izmWcy5Z2EGt8+ze0UHtQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3423,21 +3008,19 @@ } }, "node_modules/@aws-sdk/client-sesv2": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sesv2/-/client-sesv2-3.1070.0.tgz", - "integrity": "sha512-fPrvjdqhjbT6VfKO2dCHcEQ2T8gB71Gcyooxr7JlKjDPvOrreJAxJdLw1XzKCPJMDG7y3xzlhskWXHsI9cwy3A==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sesv2/-/client-sesv2-3.1079.0.tgz", + "integrity": "sha512-qxmbzwg7rRNk9zxGO/PeRQXsbSpU6FKSXbWBWMq860Fv1EH+JwDn9RZPv+RzWLqrBWC5VzQxJnwUnRUykyBn2g==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/signature-v4-multi-region": "^3.996.35", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/signature-v4-multi-region": "^3.996.38", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3445,20 +3028,18 @@ } }, "node_modules/@aws-sdk/client-sfn": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sfn/-/client-sfn-3.1070.0.tgz", - "integrity": "sha512-TvqXRE5dPyF+oM1u4o6WzEUA0jO2JOGotWeXq9+iyrDhgFGwxgjtZ+qa+vZ+yxrQK/S6UXcCbD8tWnWbDLuVAA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sfn/-/client-sfn-3.1079.0.tgz", + "integrity": "sha512-1R5Y2FyKhUojQeSjJ6oUsRmifNENthGBIYahLc/H7va5NwEpl0CisCvGGk022F+kyQEzYFdJdu6uy2nPt4LHIg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3466,20 +3047,18 @@ } }, "node_modules/@aws-sdk/client-shield": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-shield/-/client-shield-3.1070.0.tgz", - "integrity": "sha512-wAJShqTW1XIpyAk7r8yPkyLPwfcALMgXOLcFVsLcgR0hNgnyb9DREsmx7A6eIHkwc5JGAxhQlY3Ih+9ZtERmnA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-shield/-/client-shield-3.1079.0.tgz", + "integrity": "sha512-c3ta1qd5PZOI3usCDPLM0e+B50bkkOPr/QC1igJ1K8/+qGhCoWwE4OxCXKx//Q+UVNdBpXuJrBuI4TAdyYe7lA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3487,20 +3066,18 @@ } }, "node_modules/@aws-sdk/client-sns": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sns/-/client-sns-3.1070.0.tgz", - "integrity": "sha512-JtOVpLGuaUhJbewPW08Q3xMTrXhJ+G2nQV4wQkItHm80/WNX0dCEYZSwKek/CfhMHImgzWJkB8s7yPvMxX50LA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sns/-/client-sns-3.1079.0.tgz", + "integrity": "sha512-HCPsOcQPfkesuYxH0IDMjAEN3f9Stcyp4U8XWlCZPJ7V571Xj1gLIyAvXLDDsbSNbzlrHyHHQVfULmOjOjPTFw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3508,21 +3085,19 @@ } }, "node_modules/@aws-sdk/client-sqs": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sqs/-/client-sqs-3.1070.0.tgz", - "integrity": "sha512-3eOY4zpl3ePnsg5eQv7pfLjYqhxXBSwkIga/1ELDO9sdnPGmt7asGAUQNrHg/R9K7JQzK9nDQD38pnmGAyzWLw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sqs/-/client-sqs-3.1079.0.tgz", + "integrity": "sha512-8otd3BCGQGKvAXfsKORdOpDLndDjgEGmw2RmPWmi8lY/B/v1u41ElzcBYnjDWLivwIFVuJfbxrVSJlv5WFm0hA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/middleware-sdk-sqs": "^3.972.31", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/middleware-sdk-sqs": "^3.972.34", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3530,20 +3105,18 @@ } }, "node_modules/@aws-sdk/client-ssm": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-ssm/-/client-ssm-3.1070.0.tgz", - "integrity": "sha512-jSw0/1/PrGurRknme/lpVBE49vXtIAwib04eEBlmDKF2XcDG78/e2wsVi51b9G3z8U0GGI4v1dsojx9XabpJqg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-ssm/-/client-ssm-3.1079.0.tgz", + "integrity": "sha512-5Xwx0E3HmkhynUuXIEa3yL/jVkhm0tRgJloStDggUqPm5kIG/d16Wt4JGEP0ZnLVJ4tT+BdKA7Q3Q9EetDzr/Q==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3551,20 +3124,18 @@ } }, "node_modules/@aws-sdk/client-sso-admin": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-admin/-/client-sso-admin-3.1070.0.tgz", - "integrity": "sha512-a8Z5mqjUUL5wGHYmy9L1YXyNRBWeNEhkTRTD14LSxur9iJyl/urHfeHWcG+EGRrcSjXMypiWqr7ZcXWD33j/dQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-admin/-/client-sso-admin-3.1079.0.tgz", + "integrity": "sha512-hDrdk/6YXEdvhkFX9ZyOSr31/kqDpJF1xKjJzHDBfnASLVna0/MUGmqKqg4J3imIPkq+DfVATFMcAijkdRor4w==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3572,21 +3143,19 @@ } }, "node_modules/@aws-sdk/client-sts": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.1070.0.tgz", - "integrity": "sha512-uX3eD0OxYbaHqbN8aaSzZTrNJUctfCus2NLMVHUSONrzlJdA/w4o5ZAfJhgnTjlfdfRMv1lZd2Wnqm8hlzmwGA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.1079.0.tgz", + "integrity": "sha512-ycZDhMLf805leaBDQk0HnzvNJi293a0h++iG+hZtya5izd4Jyx022tV2owFc1u81c+Esdj6DchF5E9DAXxf8GQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/signature-v4-multi-region": "^3.996.35", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/signature-v4-multi-region": "^3.996.38", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3594,20 +3163,18 @@ } }, "node_modules/@aws-sdk/client-support": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-support/-/client-support-3.1070.0.tgz", - "integrity": "sha512-xWVq+rJJIvbbQnJWpT2UCVxyC4qKtoSFJ5saSZXY/MG5tGW4GbIaxNkvj4LqaWl+Z2sARD1NcvsH3PmpATP4cA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-support/-/client-support-3.1079.0.tgz", + "integrity": "sha512-G0bqbUhQ8Ef8PXG3UUOeJ6q8cAv9cW4p+L3KFaMpdV+XZXnhBun8J8b/RDcvcw0rVtkvgw6cVvqvdm5UEnCCcg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3615,20 +3182,18 @@ } }, "node_modules/@aws-sdk/client-swf": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-swf/-/client-swf-3.1070.0.tgz", - "integrity": "sha512-lKuBjgd1g3h4c+JXlp6hVYq+dHbmhL1rkzG43jCkkFT0TiEQtl161uvz0PEKQhUosEimveg+5KokS1EkLHzzIg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-swf/-/client-swf-3.1079.0.tgz", + "integrity": "sha512-SSn1dsrKYCjEs5huStY2GPdQ1tj0SrdI/NgwMqEVV/ysBKF7vQn8dB4EUxjzcb+RjEfzOG0O3lBTmxRuqlfR5Q==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3636,20 +3201,18 @@ } }, "node_modules/@aws-sdk/client-textract": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-textract/-/client-textract-3.1070.0.tgz", - "integrity": "sha512-5s3OqB83gyXR8MHdjdjydmv7gBIddgAwkBEOozlK0BXs2tQep4Qzkue7EL5sw7N6Vx9mARU9/nmV6s/rhnXSBg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-textract/-/client-textract-3.1079.0.tgz", + "integrity": "sha512-PgeGqEPQU8+H5ZmAZ2Vk9be32W9PjSEIa+uHjzFI9ICr6h3ikicW60eN1D4lRQQEz+woh5gesS7r0yWh3LR2HA==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3657,21 +3220,19 @@ } }, "node_modules/@aws-sdk/client-timestream-query": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-timestream-query/-/client-timestream-query-3.1070.0.tgz", - "integrity": "sha512-N74szKAj4dix7X/HzNwDngR+Gdt6CbdihMYnezCgnqL0zUin7DbdgQ+QA3AubGTEGHwVzVOcqm/rW9MjKl5SWw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-timestream-query/-/client-timestream-query-3.1079.0.tgz", + "integrity": "sha512-6j3Z1nH2RHIW5ZrsAECw9zA1fE3LGcQlxtJ8mSt0FDd+BSMdQJuv1G0DWgYomEN6OQZ81bsFeDDu2p6VAjvRkw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/middleware-endpoint-discovery": "^3.972.19", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/middleware-endpoint-discovery": "^3.972.22", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3679,21 +3240,19 @@ } }, "node_modules/@aws-sdk/client-timestream-write": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-timestream-write/-/client-timestream-write-3.1070.0.tgz", - "integrity": "sha512-sEG27Zk78aizIgYqf3SI4IwfDxnvIj1FIEGt81F5Rywwm22SWJSVrg3VPH+b7Urx/0UeNU+dGOmciruhc4PcLg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-timestream-write/-/client-timestream-write-3.1079.0.tgz", + "integrity": "sha512-MU4C3Y+hiuyakVFuOjDZGq42/SdEPa1Z4Wj3lZxxJI7Gt9YiW24arEzhNSq3bocFF7eKQ2bvlz+dnE/PinHD1g==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/middleware-endpoint-discovery": "^3.972.19", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/middleware-endpoint-discovery": "^3.972.22", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3701,20 +3260,18 @@ } }, "node_modules/@aws-sdk/client-transcribe": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-transcribe/-/client-transcribe-3.1070.0.tgz", - "integrity": "sha512-QQZzm/T3oLSSFo8DRtrpvTgIUPRok27fnDvb5lzP495070cS30MtzRCQa9IW9IgY0d2835Yvt0ug5mA6edYDzA==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-transcribe/-/client-transcribe-3.1079.0.tgz", + "integrity": "sha512-MPXxrYG4TAWWV3V/GC74gvHQubwhXarZPy/iooUmsAQx2CFUuVCGVU+yUd0pjBE7te3MbQ8DM7hlG6n20RC+5A==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3722,20 +3279,18 @@ } }, "node_modules/@aws-sdk/client-transfer": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-transfer/-/client-transfer-3.1070.0.tgz", - "integrity": "sha512-9a0WBWuWvTZe+l8kJix5/VIj9JRVe/4K46qkGyehbIpP2pxLHTX3Tp/2JnZOAiA6freRccXqdaG1BNRLmHadRQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-transfer/-/client-transfer-3.1079.0.tgz", + "integrity": "sha512-e0bX9bXpA0kfykLga23k8hhA6XOviOPjkMBcgZJ0VNn+Wiudcuk/6lnC7GUkzQCwAn6DeMPpDZwEjgXmcDwYOQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3743,20 +3298,18 @@ } }, "node_modules/@aws-sdk/client-translate": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-translate/-/client-translate-3.1070.0.tgz", - "integrity": "sha512-FpFbBwScRBhWC1Cn+mfv2lDPzwp1kFOfirNS5Oa+CKCmrIRt1PTa3iPaSjXofDWpKjMt2l5xIN9EpVkX0Xmmgg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-translate/-/client-translate-3.1079.0.tgz", + "integrity": "sha512-Dfwt00Gc1MyOODDMxB/MSqAmVv6Gi/61NsVUs45AgjPl4pyLz3Awo+BXl1FhiMuUSeqydHeBNKbzpdIlNGbu/A==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3764,20 +3317,18 @@ } }, "node_modules/@aws-sdk/client-verifiedpermissions": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-verifiedpermissions/-/client-verifiedpermissions-3.1070.0.tgz", - "integrity": "sha512-T9vhyKaEEbk/YzLe9nVCQ2MHVvE9jE/pfkCCpwDMCGjvgeO+GkXxkCsbl/ljvvrKB78BleaL0/8SEi4aYw+vLQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-verifiedpermissions/-/client-verifiedpermissions-3.1079.0.tgz", + "integrity": "sha512-X1vkkaV7ZUMVJk49xLro9mWIxkIR1+agYhTLahOYcm1SUzo+d5QdVeY9cpKiPLJq1Rp6p+raLcZTSX/BaPXaJg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3785,20 +3336,18 @@ } }, "node_modules/@aws-sdk/client-wafv2": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-wafv2/-/client-wafv2-3.1070.0.tgz", - "integrity": "sha512-G/yXqHsY/9VfOSB/HTuuCjd635m1g643d668o+aLdu3Id0UgJdpDFA/nzgJwkdqr/63751p8H5jx1EMuyfxsVg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-wafv2/-/client-wafv2-3.1079.0.tgz", + "integrity": "sha512-9p8IqLD64s8a0bKs/aJNEouq8aFFtgwIzPJ9ZeBMQooBE1kNwsA1dO8oRE9jDI39zYPo65sSm5wgm2cgM2fwSQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3806,20 +3355,18 @@ } }, "node_modules/@aws-sdk/client-workmail": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-workmail/-/client-workmail-3.1070.0.tgz", - "integrity": "sha512-vLuTj3todQOu5Yiwf7RGOAwwPC3mFioyOCG87dtaA0D2fZ6H2CKbvJtBd3pTmwjr4yMuQuyI6aS1lWoxae7n0A==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-workmail/-/client-workmail-3.1079.0.tgz", + "integrity": "sha512-G1UY+uBBtBF1mKEnqPG0Q+C1MOjqI9maBoQCDyt1sBOrciZCy1vKBQn8VBSSZIimbSFmZRfWyhcjIbKymrsnPw==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3827,20 +3374,18 @@ } }, "node_modules/@aws-sdk/client-workspaces": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-workspaces/-/client-workspaces-3.1070.0.tgz", - "integrity": "sha512-kU6W9wfp0GNScW3nf2d7qOWRcuuLMW2vQgxMcSAEYcDdbgAwNYIhrZw7phxaH9r15WqEHZiKpx2wdjQB9nOVuQ==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-workspaces/-/client-workspaces-3.1079.0.tgz", + "integrity": "sha512-EbUbQDUGHrFz/pyFcP2VQNDFpMgb51ITXHUuJgLb6dcZ8+dyusovL45VaMFk9KW2t/+Y0fM1haHUxCHnLq8g8g==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3848,20 +3393,18 @@ } }, "node_modules/@aws-sdk/client-xray": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-xray/-/client-xray-3.1070.0.tgz", - "integrity": "sha512-iTZ6OrawnyleD3/XCov2TN8ZftNAhzUh6wFD/ZhlcOWsUUcm9s3hK/u2mfnFw9tndUejUH9HlUyyE5PYcSUiKw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-xray/-/client-xray-3.1079.0.tgz", + "integrity": "sha512-DGvpvNw7x/wjroe1ta2+PpMSDwLeH9U6Q9thjtUNVMR2jrKEd1Z3S6oXqE4vO2xuQsL2dLOi/5bJ0vSWcXQ6gg==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3869,17 +3412,17 @@ } }, "node_modules/@aws-sdk/core": { - "version": "3.974.22", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.974.22.tgz", - "integrity": "sha512-YofH63shc6YRdXjz80BJkpJW+Bkn0Cuu2dn4Rv7s9G2Idt58tgtzQEWxrR2xVljlVfIBeUjPuULnSVYLke3sUQ==", + "version": "3.974.27", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.974.27.tgz", + "integrity": "sha512-WRWEgIq6vx+NU6ot3VrRu4Jovj9MIObitSi6of/GV5THDDPccBhivCRNkWJutMM+m3GvdeI3l/UbGNcoOobxOA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.13", - "@aws-sdk/xml-builder": "^3.972.30", - "@aws/lambda-invoke-store": "^0.2.2", - "@smithy/core": "^3.24.6", - "@smithy/signature-v4": "^5.4.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/types": "^3.973.15", + "@aws-sdk/xml-builder": "^3.972.33", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.29.0", + "@smithy/signature-v4": "^5.6.1", + "@smithy/types": "^4.15.1", "bowser": "^2.11.0", "tslib": "^2.6.2" }, @@ -3888,15 +3431,15 @@ } }, "node_modules/@aws-sdk/credential-provider-cognito-identity": { - "version": "3.972.47", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.972.47.tgz", - "integrity": "sha512-IEW+q7yXgTT6+TFgJlOW+K5FK7kj1pBFW7oc04J/jlitMxq+vOJpmU42j+xaQlhwPx2JR805ryi73aLRWicXjQ==", + "version": "3.972.52", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.972.52.tgz", + "integrity": "sha512-m+akZFJsghShferf2xsMw0Hogl1jNIJl2zUoZBNTFyWvlaOj1aK5sMTzcnw8m1dICvlQ+lC4T1OPGGsmZ+ezXA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/nested-clients": "^3.997.22", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/nested-clients": "^3.997.27", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3904,15 +3447,15 @@ } }, "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.48", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.48.tgz", - "integrity": "sha512-h6FEC95fbexUd6zxm4PdgS82bTcI2PRtUb2ZwMipb/Xr8bPwtf0G8rBo2jp7NA24Mbx2JA8/WingiYpA9RCCyw==", + "version": "3.972.53", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.53.tgz", + "integrity": "sha512-+KDA3uc/HZ1vIneGu5QMQb0gAXDYrm2vOE60+BJ7lS0YinMQ5i2oV4PR1A16XkF6K1IbSwjEHd1hQIIgMsK48w==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.974.22", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3920,17 +3463,17 @@ } }, "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.50", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.50.tgz", - "integrity": "sha512-lJO3OLpjvz5m/RSBQmsG/CEUGsvCy5ruxKwPQaOCqxqCMuyYT2BZwQUTDZVVwqQ9LrZKuK24JSa6r31hL/tvkg==", + "version": "3.972.55", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.55.tgz", + "integrity": "sha512-1gBfkWY3RWeBlCoB9lIJjXMx45/54wxcgfzv6BY9otTmMrZPcNPi1v+MwZxxaCUg441NV3jsr1efnFNCXiW70g==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.974.22", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3938,23 +3481,23 @@ } }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.972.55", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.55.tgz", - "integrity": "sha512-TBoF4buBGYhXjdZAryayY2TrkQj2B2KfE/msG4V53XCt+w0EhEwM2JRjx8p2grJ2C6gtH5++SAwEvGMRdi0yyw==", + "version": "3.972.60", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.60.tgz", + "integrity": "sha512-CV2md+PXvABwRjApWGhQ0wACy9WSFIhnUGrovLcjnjBCd/46TbuivLADtkF8IWNjtCQmQ+2IagSaxqBYqXBNAQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.974.22", - "@aws-sdk/credential-provider-env": "^3.972.48", - "@aws-sdk/credential-provider-http": "^3.972.50", - "@aws-sdk/credential-provider-login": "^3.972.54", - "@aws-sdk/credential-provider-process": "^3.972.48", - "@aws-sdk/credential-provider-sso": "^3.972.54", - "@aws-sdk/credential-provider-web-identity": "^3.972.54", - "@aws-sdk/nested-clients": "^3.997.22", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/credential-provider-imds": "^4.3.7", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-env": "^3.972.53", + "@aws-sdk/credential-provider-http": "^3.972.55", + "@aws-sdk/credential-provider-login": "^3.972.59", + "@aws-sdk/credential-provider-process": "^3.972.53", + "@aws-sdk/credential-provider-sso": "^3.972.59", + "@aws-sdk/credential-provider-web-identity": "^3.972.59", + "@aws-sdk/nested-clients": "^3.997.27", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/credential-provider-imds": "^4.4.5", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3962,16 +3505,16 @@ } }, "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.54", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.54.tgz", - "integrity": "sha512-hBWI3wZTdTGiuMfmPts6AWbAjFfRniOQnqx68tc2cQvRKWawFbN9wkLOVPWM1FAOyowZU73mC6Fi+rHSHNyLFw==", + "version": "3.972.59", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.59.tgz", + "integrity": "sha512-JG4S9yyA1GFzJdJXqLKrUzZbyK+VDp2QIsJD7YOicJHAhqymfHpDJIok2dLnhOdVB0I37RjdC53uOwCMVS00gw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.974.22", - "@aws-sdk/nested-clients": "^3.997.22", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/nested-clients": "^3.997.27", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -3979,21 +3522,21 @@ } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.57", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.57.tgz", - "integrity": "sha512-u6dClpzNdWf1HGWz4wwhdXi1wiOofCLniM9S4BQQGlLAN9TW7VB+ld5V533GdKrYMaFeBGFqKnj0JCYvynLqwQ==", + "version": "3.972.62", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.62.tgz", + "integrity": "sha512-S6Slq3Tx7bvFk5yc34XNADyZYTX2HUXvaFAnowGRQnhjBO8J/mP62Fn7lxvJwjaDyYm/7gh9h6HEHaltRyMFXw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.48", - "@aws-sdk/credential-provider-http": "^3.972.50", - "@aws-sdk/credential-provider-ini": "^3.972.55", - "@aws-sdk/credential-provider-process": "^3.972.48", - "@aws-sdk/credential-provider-sso": "^3.972.54", - "@aws-sdk/credential-provider-web-identity": "^3.972.54", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/credential-provider-imds": "^4.3.7", - "@smithy/types": "^4.14.3", + "@aws-sdk/credential-provider-env": "^3.972.53", + "@aws-sdk/credential-provider-http": "^3.972.55", + "@aws-sdk/credential-provider-ini": "^3.972.60", + "@aws-sdk/credential-provider-process": "^3.972.53", + "@aws-sdk/credential-provider-sso": "^3.972.59", + "@aws-sdk/credential-provider-web-identity": "^3.972.59", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/credential-provider-imds": "^4.4.5", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4001,15 +3544,15 @@ } }, "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.48", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.48.tgz", - "integrity": "sha512-w6VZwojPt12WnEkAUy6Nu4K6sWCbBmR7QX390b0nE6vRvkXbrYr9Lq9VySGkfjiMjpUA87op+J4EgvRmtWIDoQ==", + "version": "3.972.53", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.53.tgz", + "integrity": "sha512-EhfH+MQlqOMCkXIVa8MMObPzAQqwTTtxA7KhEJiyPeuNVA8PLOOUpgK7nBrgaDaGiIDLN/9LpGdaHuDjomeRTw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.974.22", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4017,34 +3560,17 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.972.54", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.54.tgz", - "integrity": "sha512-23uZpIpF2SIFDCa1fcWa202tK4gGeyvX6GIIAjiB8WBsvsVRBMnJ/7dCxHzxf7eZT7GToJg837LDIBnZsl/VUg==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.974.22", - "@aws-sdk/nested-clients": "^3.997.22", - "@aws-sdk/token-providers": "3.1071.0", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-sso/node_modules/@aws-sdk/token-providers": { - "version": "3.1071.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1071.0.tgz", - "integrity": "sha512-4LDW2Qob6LoLFuqYSYZq2AyTE9koSE9+i+n5UZcm10GpmQOK0zRD9L4uYlzItiTKksIWgC/qMFChAi3RvKYtMg==", + "version": "3.972.59", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.59.tgz", + "integrity": "sha512-h8793pOjcImx0SB+VcLONcaQQ57VAvKVuqyewQMRKqqH+CSXsG2dwOeLMUJPMxLdNvL7dXOM0ueTukyNUnu5mA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.974.22", - "@aws-sdk/nested-clients": "^3.997.22", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/nested-clients": "^3.997.27", + "@aws-sdk/token-providers": "3.1079.0", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4052,16 +3578,16 @@ } }, "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.54", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.54.tgz", - "integrity": "sha512-0Iv5QttS6wcATlodYKgvQj6B9Db51rx7NU9fqu0PoLeS4BIgdYMc/QK4smwLwpm5RFrs02V/eLyEFp3FklvlNQ==", + "version": "3.972.59", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.59.tgz", + "integrity": "sha512-VoyO9+vl3XVmpZwn4obskrWIkrA/Jf3lSe1E3ZERlaN9u0D4YZ6+HywC3+L98QOXqZesEfedk67gRER8tK8+8w==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.974.22", - "@aws-sdk/nested-clients": "^3.997.22", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/nested-clients": "^3.997.27", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4069,27 +3595,27 @@ } }, "node_modules/@aws-sdk/credential-providers": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.1070.0.tgz", - "integrity": "sha512-dbRx4iWgJp9mavY7ErFw+I+IAgxrSZn/a9zog9H52R9m8rPiB8zCXO2FLuhVmQek7UJ4/YcB1bmvlJOOvEjWJw==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.1079.0.tgz", + "integrity": "sha512-emoshJjvvyJDjoMlognc1BtdsTDbe/8NQhXM2wIOz/6/vx4lynUYbwhcNdP6rXuT1q0HzugEDkQK9EvbzB94fA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/client-cognito-identity": "3.1070.0", - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/credential-provider-cognito-identity": "^3.972.46", - "@aws-sdk/credential-provider-env": "^3.972.47", - "@aws-sdk/credential-provider-http": "^3.972.49", - "@aws-sdk/credential-provider-ini": "^3.972.54", - "@aws-sdk/credential-provider-login": "^3.972.53", - "@aws-sdk/credential-provider-node": "^3.972.56", - "@aws-sdk/credential-provider-process": "^3.972.47", - "@aws-sdk/credential-provider-sso": "^3.972.53", - "@aws-sdk/credential-provider-web-identity": "^3.972.53", - "@aws-sdk/nested-clients": "^3.997.21", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/credential-provider-imds": "^4.3.7", - "@smithy/types": "^4.14.3", + "@aws-sdk/client-cognito-identity": "3.1079.0", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/credential-provider-cognito-identity": "^3.972.52", + "@aws-sdk/credential-provider-env": "^3.972.53", + "@aws-sdk/credential-provider-http": "^3.972.55", + "@aws-sdk/credential-provider-ini": "^3.972.60", + "@aws-sdk/credential-provider-login": "^3.972.59", + "@aws-sdk/credential-provider-node": "^3.972.62", + "@aws-sdk/credential-provider-process": "^3.972.53", + "@aws-sdk/credential-provider-sso": "^3.972.59", + "@aws-sdk/credential-provider-web-identity": "^3.972.59", + "@aws-sdk/nested-clients": "^3.997.27", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/credential-provider-imds": "^4.4.5", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4097,14 +3623,14 @@ } }, "node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.22", - "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.22.tgz", - "integrity": "sha512-QBs7/nWHsPA0wTEqhU5iPgaDjeZzQHhmwJr6Q0f56rW3Fk8ExJQgXFzEg/l48iDwJVUIMLjPqhw7dNP3cShUxA==", + "version": "3.973.27", + "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.27.tgz", + "integrity": "sha512-eOTdNw3SwpkO/WOBFFY28Us9WcjBxejRw0vRD0eRqp6+aisHnBXiyQhSX2VIPHkCMFThhBRfSftdXBbgh95y8A==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.974.22", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4125,14 +3651,14 @@ } }, "node_modules/@aws-sdk/eventstream-handler-node": { - "version": "3.972.22", - "resolved": "https://registry.npmjs.org/@aws-sdk/eventstream-handler-node/-/eventstream-handler-node-3.972.22.tgz", - "integrity": "sha512-tqPJv0dz4+O0hWGm1a6YekcMZyPhDFs/zH73Von7icaVT5n0Jqvm86typ3jRrG+qoUdPhALOnboRLTmnWQTlYQ==", + "version": "3.972.25", + "resolved": "https://registry.npmjs.org/@aws-sdk/eventstream-handler-node/-/eventstream-handler-node-3.972.25.tgz", + "integrity": "sha512-df7HN1ozwMrB9+59re9PM7tSLxLAcheMWc5u/KyfCPCAWtN/vP7y7RTUZOy48uT1K9MESisVeOPPzF3O1AW01A==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4140,15 +3666,15 @@ } }, "node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.19", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.19.tgz", - "integrity": "sha512-FMgyzUq3Jh+ONRYxryBRNdBd+FUX8PwRl07ccQknNdoms6KCeAEusCkl6whqpDrPQ6OH0ddeSifKyqYSs2DLIw==", + "version": "3.972.22", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.22.tgz", + "integrity": "sha512-GMoLg4XAnkiwevqcOfgz0lOTYmIdM7MJK/BL9EAvsoMFqKIGRpyNIvuSNg5gdFzP57yB+EklMlK0+TwBqj1tCg==", "license": "Apache-2.0", "dependencies": { "@aws-sdk/endpoint-cache": "^3.972.8", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4156,27 +3682,14 @@ } }, "node_modules/@aws-sdk/middleware-eventstream": { - "version": "3.972.18", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-eventstream/-/middleware-eventstream-3.972.18.tgz", - "integrity": "sha512-OHpk8YoZi3yexPq8aFt1vN1IxA2zLKvsIR5GpWYylX/ve6kQmY7wxHNSFy/D3t2apMZ16rs76Co4dJWcDyIk3A==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums": { - "version": "3.974.32", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.974.32.tgz", - "integrity": "sha512-KhuzFMzUbb3oEj43CdPDbEJ/RG/RkErkmXk3J/LE8OPFNvkCn8PYPMpjOLgzAzvxBacsSyytdWf+R50q0alJ4w==", + "version": "3.972.21", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-eventstream/-/middleware-eventstream-3.972.21.tgz", + "integrity": "sha512-HvLgDnxBLaHi9E5K++6Vuk+1+qqn7Pmn8zrlzd+NXH3jBzwujnuzZtAR9WHPkbUGPO92FkoQWj/M1IsdxTlBmQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/checksums": "^3.1000.7", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4184,14 +3697,12 @@ } }, "node_modules/@aws-sdk/middleware-host-header": { - "version": "3.972.10", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.972.10.tgz", - "integrity": "sha512-IJSsIMeVQ8MMCPbuh1AbltkFhLBLXn7aejzfX5YKT/VLDHn++Dcz8886tXckE+wQssyPUhaXrJhdakO2VilRhg==", + "version": "3.972.28", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.972.28.tgz", + "integrity": "sha512-2w4cKugljxKCgBPL5LQJLj+1kUBkWG8HO7C+7GYTS7UNATMijEK+MCEktnCpfA2koBZFcf7Ioe5YhNZ5vh5ruw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.8", - "@smithy/protocol-http": "^5.3.14", - "@smithy/types": "^4.14.1", + "@aws-sdk/core": "^3.974.27", "tslib": "^2.6.2" }, "engines": { @@ -4199,13 +3710,12 @@ } }, "node_modules/@aws-sdk/middleware-logger": { - "version": "3.972.10", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.972.10.tgz", - "integrity": "sha512-OOuGvvz1Dm20SjZo5oEBePFqxt5nf8AwkNDSyUHvD9/bfNASmstcYxFAHUowy4n6Io7mWUZ04JURZwSBvyQanQ==", + "version": "3.972.27", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.972.27.tgz", + "integrity": "sha512-Qia7FsijpSh+LL9H6i9HQ3JZQsN9qdOQjwCAIi4Zi9Xqc62N7c3udBOfliavqn40FKZlgpgV/Js+NpKguyXfiA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.8", - "@smithy/types": "^4.14.1", + "@aws-sdk/core": "^3.974.27", "tslib": "^2.6.2" }, "engines": { @@ -4213,15 +3723,12 @@ } }, "node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.972.11", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.972.11.tgz", - "integrity": "sha512-+zz6f79Kj9V5qFK2P+D8Ehjnw4AhphAlCAsPjUqEcInA9umtSSKMrHbSagEeOIsDNuvVrH98bjRHcyQukTrhaQ==", + "version": "3.972.29", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.972.29.tgz", + "integrity": "sha512-iM0ZSNerIexGGFokLEJqEr/H2kV+WiTENZXHLb391q1ldF+qWDn86MpyH4HKfKwQRzEhInehwX9u9/a1IXAz6g==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.8", - "@aws/lambda-invoke-store": "^0.2.2", - "@smithy/protocol-http": "^5.3.14", - "@smithy/types": "^4.14.1", + "@aws-sdk/core": "^3.974.27", "tslib": "^2.6.2" }, "engines": { @@ -4229,14 +3736,14 @@ } }, "node_modules/@aws-sdk/middleware-sdk-api-gateway": { - "version": "3.972.18", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-api-gateway/-/middleware-sdk-api-gateway-3.972.18.tgz", - "integrity": "sha512-3xZO1L3f+OshQ+ChcyCQtwZ2eeK7V4xqAxZ3cBDVgyEd8HTnIol9t4UNB5YoCaXOtYWduCtyFDBzKT0tSEAjGQ==", + "version": "3.972.21", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-api-gateway/-/middleware-sdk-api-gateway-3.972.21.tgz", + "integrity": "sha512-tlu8JAcbrhh4jnoQXczWQ2QIOaev3ZdVBdU4RRaNBfJmVyuBbMKdh6mdrunEdSFgm2GlJW1lIhE4BHTL0A55dw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4244,16 +3751,16 @@ } }, "node_modules/@aws-sdk/middleware-sdk-ec2": { - "version": "3.972.36", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-ec2/-/middleware-sdk-ec2-3.972.36.tgz", - "integrity": "sha512-ib23mbklPdXeU/7OIAtkCu86nqJoiyJREQQa1IVOJo/AfkEmGyQH3IZPbsetK72qI1ZcUdcxA0zxfWNM1Z1/Rw==", + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-ec2/-/middleware-sdk-ec2-3.972.41.tgz", + "integrity": "sha512-QsRHSajtG9RhcwKNr5WPBq6BDvO1dcNjTbROWaaDz/e5x/ndkOEMbBDsNVQEIvOPYW7q7YwN309orXjyZboe1w==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.974.22", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/signature-v4": "^5.4.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/signature-v4": "^5.6.1", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4261,14 +3768,14 @@ } }, "node_modules/@aws-sdk/middleware-sdk-glacier": { - "version": "3.972.18", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-glacier/-/middleware-sdk-glacier-3.972.18.tgz", - "integrity": "sha512-vmyilZwe4TWgWHpHw+aB7lOBJza6DeJpZUDWo27FGQHpWC/0PjyulZN4l28/tdhP7+LgVoM+tkwM+8oLln+aPQ==", + "version": "3.972.21", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-glacier/-/middleware-sdk-glacier-3.972.21.tgz", + "integrity": "sha512-AvsioxiXVYwAVIP7k9Ob8//CIQrpOd7VnF3Z1SBkfiqZrzfWEIKCa+FGnAUUnFh2DwLEwD2zudZmuy00EMFYoA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4276,16 +3783,16 @@ } }, "node_modules/@aws-sdk/middleware-sdk-rds": { - "version": "3.972.36", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-rds/-/middleware-sdk-rds-3.972.36.tgz", - "integrity": "sha512-NuqAVqPEsEZ2PADr1W6UWoDsb5ukUKoMgHy7bfqGkYCKpBivSgNdlfgzZN56fnitE+EjOqMj4pb7e2C84uvQVA==", + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-rds/-/middleware-sdk-rds-3.972.41.tgz", + "integrity": "sha512-3vgaXrMKcGhiT/eAF/EsDqfm+tstL4kfBZVoH1CYkOEOhRU5Gvmcsnd69JAjWWGN+oheqa7Dtcsn64XSEF4iOg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.974.22", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/signature-v4": "^5.4.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/signature-v4": "^5.6.1", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4293,13 +3800,13 @@ } }, "node_modules/@aws-sdk/middleware-sdk-route53": { - "version": "3.972.17", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-route53/-/middleware-sdk-route53-3.972.17.tgz", - "integrity": "sha512-llqZ4sab/gQGTDRbWjorH+JblWcXfi3QcBrVjyApgPR8uLP0YbgJZI6aNafGww0Nbxlnf08n032qA6/JWfRUhA==", + "version": "3.972.19", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-route53/-/middleware-sdk-route53-3.972.19.tgz", + "integrity": "sha512-yjUYBAmz9/LTcuLWmKvc9MvdA7ZlCb50PwCUpIGv/LFQTTJLL/qRloKGLkQJSdgY8ZB+Wg6b4pD/P0Cblacshg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.13", - "@smithy/types": "^4.14.3", + "@aws-sdk/types": "^3.973.15", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4307,16 +3814,16 @@ } }, "node_modules/@aws-sdk/middleware-sdk-s3": { - "version": "3.972.53", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.53.tgz", - "integrity": "sha512-keWp6Z5cEIJzPwoCf/WRm0ceAeephPDDivhRsK/xXs2ZYXyypJ2/DL9G1IR0bz/s+iZC0EgzmFV4r7rlvLlxQQ==", + "version": "3.972.58", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.58.tgz", + "integrity": "sha512-6uaWRRYJGhOqc9EoTSbLDf9nI/doSAb5vAwGshs5/Hlv5Ce25b246lBkbRd/77fLAi+uMI1a70mJzVyLyCEufQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.974.22", - "@aws-sdk/signature-v4-multi-region": "^3.996.35", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/signature-v4-multi-region": "^3.996.38", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4324,14 +3831,14 @@ } }, "node_modules/@aws-sdk/middleware-sdk-sqs": { - "version": "3.972.31", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sqs/-/middleware-sdk-sqs-3.972.31.tgz", - "integrity": "sha512-56ifsBmK9bLn5EE/t6c0nmjOB1BO8cJDLkA1VOlsN1GR85ROqnaCwVDspqcwsLaBDgPlwyYNedoDIoT3t6Ho1A==", + "version": "3.972.34", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sqs/-/middleware-sdk-sqs-3.972.34.tgz", + "integrity": "sha512-BgCMs873RWtMe8LlNY5hxJjVbZtaHR6nY3BtQZk14drRJb3Iqecp5VADpw3eF5cnBvlZLWblAjCjJFU1mTjoeQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4339,18 +3846,12 @@ } }, "node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.972.38", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.38.tgz", - "integrity": "sha512-iz+B29TXcAZsJpwB+AwG/TTGA5l/VnmMZ2UxtiySOZjI6gCdmviXPwdgzcmuazMy16rXoPY4mYCGe7zdNKfx5A==", + "version": "3.972.57", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.972.57.tgz", + "integrity": "sha512-4gstakrfAVK7CAQ8Rar4Sig7Ztjxdp+Xt/A48VwTxwPVVNs+lAWs1+ZTGF+zlT65qWbnihUsX7uzBdPEllzLSA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.974.8", - "@aws-sdk/types": "^3.973.8", - "@aws-sdk/util-endpoints": "^3.996.8", - "@smithy/core": "^3.23.17", - "@smithy/protocol-http": "^5.3.14", - "@smithy/types": "^4.14.1", - "@smithy/util-retry": "^4.3.6", + "@aws-sdk/core": "^3.974.27", "tslib": "^2.6.2" }, "engines": { @@ -4358,17 +3859,17 @@ } }, "node_modules/@aws-sdk/middleware-websocket": { - "version": "3.972.30", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-websocket/-/middleware-websocket-3.972.30.tgz", - "integrity": "sha512-kH6N4f/Fzi9r/dYap8EQ+Zk4NOz8pl4AtWKhzAoG2C1/4YkIHok9APp/e+75woreWQq264n+LkrJsJVZ0Q+M1Q==", + "version": "3.972.35", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-websocket/-/middleware-websocket-3.972.35.tgz", + "integrity": "sha512-7/ZAlq5o5A9FnRsQEftZvcct9LVZf1YaYmWR3eOzyJAdKU66YpOKy5ahUVvyBvCTLyO4Ej4yWIk8dllWNXPBsw==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.974.22", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/signature-v4": "^5.4.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/signature-v4": "^5.6.1", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4376,20 +3877,18 @@ } }, "node_modules/@aws-sdk/nested-clients": { - "version": "3.997.22", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.22.tgz", - "integrity": "sha512-4IwtcYSxEIVw5hcp8ogq0CMbFNZFw7jJUetpfFUhFFeqsa1K8j2Ihg2hnxLyOp3stMZnXda6VzOmPi1AFZQXcg==", + "version": "3.997.27", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.27.tgz", + "integrity": "sha512-A8PIePF9NIIOJ/4Lg1rl9xm/+QaKkHGetq+Z9wb5B+3Da31YYXRo8n7IDMh5C+HQI5eyEmjrwkGWVdYtnLtbXQ==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/sha256-browser": "5.2.0", - "@aws-crypto/sha256-js": "5.2.0", - "@aws-sdk/core": "^3.974.22", - "@aws-sdk/signature-v4-multi-region": "^3.996.35", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/fetch-http-handler": "^5.4.6", - "@smithy/node-http-handler": "^4.7.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/signature-v4-multi-region": "^3.996.38", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/fetch-http-handler": "^5.6.2", + "@smithy/node-http-handler": "^4.9.2", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4397,15 +3896,12 @@ } }, "node_modules/@aws-sdk/region-config-resolver": { - "version": "3.972.13", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.972.13.tgz", - "integrity": "sha512-CvJ2ZIjK/jVD/lbOpowBVElJyC1YxLTIJ13yM0AEo0t2v7swOzGjSA6lJGH+DwZXQhcjUjoYwc8bVYCX5MDr1A==", + "version": "3.972.31", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.972.31.tgz", + "integrity": "sha512-5qLGtfKWyPK/eSLxIzsrNRfHVYZliXxe5mXYldmVB1Ca5VCpz0TjZhIb9otq1eUER8m2AQ1Jy0EBeR6qC65wvg==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.8", - "@smithy/config-resolver": "^4.4.17", - "@smithy/node-config-provider": "^4.3.14", - "@smithy/types": "^4.14.1", + "@aws-sdk/core": "^3.974.27", "tslib": "^2.6.2" }, "engines": { @@ -4413,13 +3909,13 @@ } }, "node_modules/@aws-sdk/sha256-tree-hash": { - "version": "3.972.17", - "resolved": "https://registry.npmjs.org/@aws-sdk/sha256-tree-hash/-/sha256-tree-hash-3.972.17.tgz", - "integrity": "sha512-EgriHKVinYPV6hm5jepRIABSkGbe2lKVAMDQZTPahTQ6ggtg3pyex86ZXZJu0J9jThEFZEIKP//iaCbShCm8qw==", + "version": "3.972.20", + "resolved": "https://registry.npmjs.org/@aws-sdk/sha256-tree-hash/-/sha256-tree-hash-3.972.20.tgz", + "integrity": "sha512-M6OJ4AaPeozffoo/nwINETU8+FuLhRRSw4JMdCujTEvgQgMB+Rp4gTSqmO3LogvtRz/QMiXaadWSHjWOToD8RQ==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.13", - "@smithy/types": "^4.14.3", + "@aws-sdk/types": "^3.973.15", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4427,14 +3923,14 @@ } }, "node_modules/@aws-sdk/signature-v4-multi-region": { - "version": "3.996.35", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.35.tgz", - "integrity": "sha512-6L/VWs+Wch2stHemCGTmUNqKLMzURxQDK5boNG3Jn3kAOp71meDUuS5sbObpEvFxHDq0uWeSLFDNSYsjNt+Dlg==", + "version": "3.996.38", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.38.tgz", + "integrity": "sha512-C379Sk+MiFZCfWZphKlMyLHKxV22OjoGM5KJjj5IJNJcOCWL4IGIpnEGzv1FQiRwhYXfq55SJMfxlqPE08JJ9g==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.13", - "@smithy/signature-v4": "^5.4.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/types": "^3.973.15", + "@smithy/signature-v4": "^5.6.1", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4442,16 +3938,16 @@ } }, "node_modules/@aws-sdk/token-providers": { - "version": "3.1070.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1070.0.tgz", - "integrity": "sha512-93At+DndjIqzQybznibJX6Jet8jAiFGQkQPnLTKLBoTYZolWE57wzjh4Z4hCqSjh8Q1sBdGFZn4tvgTksfqiRg==", + "version": "3.1079.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1079.0.tgz", + "integrity": "sha512-cbietrLlHPhhmbnMPTuDS4Zj/KNGhY+3vVhn6dwjO6Dqzrwothzg2srtcY34T9mlICsTXn34avDoWLHSntP54A==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.974.21", - "@aws-sdk/nested-clients": "^3.997.21", - "@aws-sdk/types": "^3.973.13", - "@smithy/core": "^3.24.6", - "@smithy/types": "^4.14.3", + "@aws-sdk/core": "^3.974.27", + "@aws-sdk/nested-clients": "^3.997.27", + "@aws-sdk/types": "^3.973.15", + "@smithy/core": "^3.29.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4459,12 +3955,12 @@ } }, "node_modules/@aws-sdk/types": { - "version": "3.973.13", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.13.tgz", - "integrity": "sha512-pEHZqRkAlHfnfAU9tK+WpKv/gBNjGJrHMgA3A0iYRGyswBS2t0pfez+lWlwktb3Bqa0ovh7w/QJTFwp3fDxLNg==", + "version": "3.973.15", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.15.tgz", + "integrity": "sha512-IULn8uBV/SMtmOIANsm4WHXIOtVPBWfOWs3WGL0j/sI+KhaYehvOw0ET+9urnn8MBpiijuU/0JOpuwKOE451PQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.3", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4472,15 +3968,15 @@ } }, "node_modules/@aws-sdk/util-endpoints": { - "version": "3.996.8", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.996.8.tgz", - "integrity": "sha512-oOZHcRDihk5iEe5V25NVWg45b3qEA8OpHWVdU/XQh8Zj4heVPAJqWvMphQnU7LkufmUo10EpvFPZuQMiFLJK3g==", + "version": "3.986.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.986.0.tgz", + "integrity": "sha512-Mqi79L38qi1gCG3adlVdbNrSxvcm1IPDLiJPA3OBypY5ewxUyWbaA3DD4goG+EwET6LSFgZJcRSIh6KBNpP5pA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.8", - "@smithy/types": "^4.14.1", - "@smithy/url-parser": "^4.2.14", - "@smithy/util-endpoints": "^3.4.2", + "@aws-sdk/types": "^3.973.1", + "@smithy/types": "^4.12.0", + "@smithy/url-parser": "^4.2.8", + "@smithy/util-endpoints": "^3.2.8", "tslib": "^2.6.2" }, "engines": { @@ -4488,9 +3984,9 @@ } }, "node_modules/@aws-sdk/util-locate-window": { - "version": "3.965.5", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.965.5.tgz", - "integrity": "sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ==", + "version": "3.965.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.965.8.tgz", + "integrity": "sha512-uUbMs1cBZPafD0ohUj6EwNf0fPZ534NvBxHox4hjX+0Rxq5paSYUem7+hi833pYrzrcnBATKIYpR02MDXT5M9g==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" @@ -4500,50 +3996,35 @@ } }, "node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.972.10", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.972.10.tgz", - "integrity": "sha512-FAzqXvfEssGdSIz8ejatan0bOdx1qefBWKF/gWmVBXIP1HkS7v/wjjaqrAGGKvyihrXTXW00/2/1nTJtxpXz7g==", + "version": "3.972.28", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.972.28.tgz", + "integrity": "sha512-qr0fi8bkpkALaVPQzPinjAlWYR9/gswh4JUXHMoCBKS9YnoHCX+1qHROYGxWRI13BRg+OD4Wpl2NcuM0eFPyUA==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.973.8", - "@smithy/types": "^4.14.1", - "bowser": "^2.11.0", + "@aws-sdk/core": "^3.974.27", "tslib": "^2.6.2" } }, "node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.973.24", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.973.24.tgz", - "integrity": "sha512-ZWwlkjcIp7cEL8ZfTpTAPNkwx25p7xol0xlKoWVVf22+nsjwmLcHYtTPjIV1cSpmB/b6DaK4cb1fSkvCXHgRdw==", + "version": "3.973.43", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.973.43.tgz", + "integrity": "sha512-sM+qbPrMr3kLDSJi6J93On74p9IFNzI2Jx4JyO/8TvYB9sn2e0EbBt2z7pES+NCfKI+ITSqixY8pM+RZdT0Qow==", "license": "Apache-2.0", "dependencies": { - "@aws-sdk/middleware-user-agent": "^3.972.38", - "@aws-sdk/types": "^3.973.8", - "@smithy/node-config-provider": "^4.3.14", - "@smithy/types": "^4.14.1", - "@smithy/util-config-provider": "^4.2.2", + "@aws-sdk/core": "^3.974.27", "tslib": "^2.6.2" }, "engines": { "node": ">=20.0.0" - }, - "peerDependencies": { - "aws-crt": ">=1.0.0" - }, - "peerDependenciesMeta": { - "aws-crt": { - "optional": true - } } }, "node_modules/@aws-sdk/xml-builder": { - "version": "3.972.30", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.30.tgz", - "integrity": "sha512-StElZPEoBquWwNqw1AcfpzEyZqJvFxouG+mpDNYlcH6ZOrqd2CuIryv+8LV8gNHZUOyKyJF3Dq9vxaXEmDR9TQ==", + "version": "3.972.33", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.33.tgz", + "integrity": "sha512-ezbwz9WpuLctm6o7P2t2naDhVVPI5jFGrVefVybhcKGjU57VIyT46pQVO0RI2RYkUdhdj2Z9uSIlAzGZE9NW9A==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.3", - "fast-xml-parser": "5.7.3", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -4551,22 +4032,22 @@ } }, "node_modules/@aws/lambda-invoke-store": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.4.tgz", - "integrity": "sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", "license": "Apache-2.0", "engines": { "node": ">=18.0.0" } }, "node_modules/@babel/code-frame": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", - "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-validator-identifier": "^7.28.5", + "@babel/helper-validator-identifier": "^7.29.7", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" }, @@ -4575,9 +4056,9 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", "dev": true, "license": "MIT", "engines": { @@ -4585,9 +4066,9 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", - "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", "dev": true, "license": "MIT", "engines": { @@ -4595,13 +4076,13 @@ } }, "node_modules/@babel/parser": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.2.tgz", - "integrity": "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", + "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.29.0" + "@babel/types": "^7.29.7" }, "bin": { "parser": "bin/babel-parser.js" @@ -4611,9 +4092,9 @@ } }, "node_modules/@babel/runtime": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", - "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", "dev": true, "license": "MIT", "engines": { @@ -4621,14 +4102,14 @@ } }, "node_modules/@babel/types": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", - "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.28.5" + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -4658,34 +4139,34 @@ } }, "node_modules/@bufbuild/protobuf": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-1.10.0.tgz", - "integrity": "sha512-QDdVFLoN93Zjg36NoQPZfsVH9tZew7wKDKyV5qRdj8ntT4wQCOradQjRaTdwMhWUYsgKsvCINKKm87FdEk96Ag==", + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-1.10.1.tgz", + "integrity": "sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==", "license": "(Apache-2.0 AND BSD-3-Clause)" }, "node_modules/@connectrpc/connect": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@connectrpc/connect/-/connect-1.6.1.tgz", - "integrity": "sha512-KchMDNtU4CDTdkyf0qG7ugJ6qHTOR/aI7XebYn3OTCNagaDYWiZUVKgRgwH79yeMkpNgvEUaXSK7wKjaBK9b/Q==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@connectrpc/connect/-/connect-1.7.0.tgz", + "integrity": "sha512-iNKdJRi69YP3mq6AePRT8F/HrxWCewrhxnLMNm0vpqXAR8biwzRtO6Hjx80C6UvtKJ5sFmffQT7I4Baecz389w==", "license": "Apache-2.0", "peerDependencies": { "@bufbuild/protobuf": "^1.10.0" } }, "node_modules/@connectrpc/connect-web": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@connectrpc/connect-web/-/connect-web-1.6.1.tgz", - "integrity": "sha512-GVfxQOmt3TtgTaKeXLS/EA2IHa3nHxwe2BCHT7X0Q/0hohM+nP5DDnIItGEjGrGdt3LTTqWqE4s70N4h+qIMlQ==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@connectrpc/connect-web/-/connect-web-1.7.0.tgz", + "integrity": "sha512-qyP0YOnUPRWwCc/VfsoydMJvkb7EyUPr2q9sHgBuJzbADjiqck1gKH5V5ZPzPhTLBvmz5UvG+wiZ5sMRQHU1MQ==", "license": "Apache-2.0", "peerDependencies": { "@bufbuild/protobuf": "^1.10.0", - "@connectrpc/connect": "1.6.1" + "@connectrpc/connect": "1.7.0" } }, "node_modules/@csstools/color-helpers": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.0.2.tgz", - "integrity": "sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.1.0.tgz", + "integrity": "sha512-064IFJdjTfUqnjpCVpMOdbr8FLQBhinbZj6yRv2An2E41O/pLEXqfFRWqGq/SxlE5PEUYTlvWsG2r8MswAVvkg==", "dev": true, "funding": [ { @@ -4703,9 +4184,9 @@ } }, "node_modules/@csstools/css-calc": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.2.0.tgz", - "integrity": "sha512-bR9e6o2BDB12jzN/gIbjHa5wLJ4UjD1CB9pM7ehlc0ddk6EBz+yYS1EV2MF55/HUxrHcB/hehAyt5vhsA3hx7w==", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.2.1.tgz", + "integrity": "sha512-DtdHlgXh5ZkA43cwBcAm+huzgJiwx3ZTWVjBs94kwz2xKqSimDA3lBgCjphYgwgVUMWatSM0pDd8TILB1yrVVg==", "dev": true, "funding": [ { @@ -4727,9 +4208,9 @@ } }, "node_modules/@csstools/css-color-parser": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.1.0.tgz", - "integrity": "sha512-U0KhLYmy2GVj6q4T3WaAe6NPuFYCPQoE3b0dRGxejWDgcPp8TP7S5rVdM5ZrFaqu4N67X8YaPBw14dQSYx3IyQ==", + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.1.9.tgz", + "integrity": "sha512-paQcIaOO53Rk5+YrBaBjm/SgrV4INImjo2BT1DtQRYr+XeTRbeAYlS+jxXp9drqvKmtFnWRJKIalDLhZZDu42A==", "dev": true, "funding": [ { @@ -4743,8 +4224,8 @@ ], "license": "MIT", "dependencies": { - "@csstools/color-helpers": "^6.0.2", - "@csstools/css-calc": "^3.2.0" + "@csstools/color-helpers": "^6.1.0", + "@csstools/css-calc": "^3.2.1" }, "engines": { "node": ">=20.19.0" @@ -4778,9 +4259,9 @@ } }, "node_modules/@csstools/css-syntax-patches-for-csstree": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.3.tgz", - "integrity": "sha512-SH60bMfrRCJF3morcdk57WklujF4Jr/EsQUzqkarfHXEFcAR1gg7fS/chAE922Sehgzc1/+Tz5H3Ypa1HiEKrg==", + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.6.tgz", + "integrity": "sha512-TcJCWFbXLPpJYq6z7bfOyjWYJDiDg2/I4gyUC9pqPNqHFRIey0EB0q0L5cSnQDfWJg8Jd6VadakxdIez/3zkqQ==", "dev": true, "funding": [ { @@ -4823,20 +4304,20 @@ } }, "node_modules/@emnapi/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", - "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", + "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", "license": "MIT", "optional": true, "dependencies": { - "@emnapi/wasi-threads": "1.2.1", + "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", "license": "MIT", "optional": true, "dependencies": { @@ -4844,9 +4325,9 @@ } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", - "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", "license": "MIT", "optional": true, "dependencies": { @@ -4854,9 +4335,9 @@ } }, "node_modules/@exodus/bytes": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.0.tgz", - "integrity": "sha512-UY0nlA+feH81UGSHv92sLEPLCeZFjXOuHhrIo0HQydScuQc8s0A7kL/UdgwgDq8g8ilksmuoF35YVTNphV2aBQ==", + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.1.tgz", + "integrity": "sha512-S6mL0yNB/Abt9Ei4tq8gDhcczc4S3+vQ4ra7vxnAf+YHC02srtqxKKZghx2Dq6p0e66THKwR6r8N6P95wEty7Q==", "dev": true, "license": "MIT", "engines": { @@ -4897,9 +4378,9 @@ "license": "MIT" }, "node_modules/@internationalized/date": { - "version": "3.12.1", - "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.12.1.tgz", - "integrity": "sha512-6IedsVWXyq4P9Tj+TxuU8WGWM70hYLl12nbYU8jkikVpa6WXapFazPUcHUMDMoWftIDE2ILDkFFte6W2nFCkRQ==", + "version": "3.12.2", + "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.12.2.tgz", + "integrity": "sha512-FY1Y+H64NDs+HAF6omlnWxm3mEpfgaCSWtL5l551ZZfImA+kGjPFgrnJrGjH6lfmLL0g8Z/mBu1R3kufeCp6Jw==", "license": "Apache-2.0", "peer": true, "dependencies": { @@ -4952,13 +4433,13 @@ } }, "node_modules/@napi-rs/wasm-runtime": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", - "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", + "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", "license": "MIT", "optional": true, "dependencies": { - "@tybys/wasm-util": "^0.10.1" + "@tybys/wasm-util": "^0.10.3" }, "funding": { "type": "github", @@ -4969,22 +4450,10 @@ "@emnapi/runtime": "^1.7.1" } }, - "node_modules/@nodable/entities": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.2.0.tgz", - "integrity": "sha512-9uGyhaQavEUMC8AIddIjau4NsnsXhou+j5sBAGojCM1oxmQpVKTWR/9JxABD6UAv12vpIms55fPZKFQEhG6uBg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/nodable" - } - ], - "license": "MIT" - }, "node_modules/@oxc-project/types": { - "version": "0.133.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.133.0.tgz", - "integrity": "sha512-KzkdCd6Uxqnf6l3HOw1xfatAlUURA0g14cvBYFyJ5SaNOQbOUvBr9PKArcPcrNIeRsBdgcUzOGrhKveVpvOIGA==", + "version": "0.138.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.138.0.tgz", + "integrity": "sha512-1a7ZKmrRTCoN1XMZ4L0PyyqrMnrNlLyPuOkdSX2MZg7IiIGRUyurNhAm73ptDOraoBcIordsIGKNPKUzy3ZmfA==", "devOptional": true, "license": "MIT", "funding": { @@ -4992,9 +4461,9 @@ } }, "node_modules/@oxfmt/binding-android-arm-eabi": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm-eabi/-/binding-android-arm-eabi-0.55.0.tgz", - "integrity": "sha512-+rFDOqQe5LOWgxrAJaZgLRudr6GQm0wGI6gtu7vVkrdLGjNMUSGbAlaCr8j7F2H2Er97vYQCU8WDb30onqMM1g==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm-eabi/-/binding-android-arm-eabi-0.57.0.tgz", + "integrity": "sha512-qVBsEO+KugOsCmUHcO8iqNnqc65p7PCKpCs8M66mPZ+Ri+CWbcpoQOEJBg2OTu03+0qu++NK1jj6IzvQVs0Sig==", "cpu": [ "arm" ], @@ -5009,9 +4478,9 @@ } }, "node_modules/@oxfmt/binding-android-arm64": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm64/-/binding-android-arm64-0.55.0.tgz", - "integrity": "sha512-ctulLq8s3x8Zmvw6+iccB09TIKERAklRSmbJ10gk8mlAn05qZxoyo52dj3Hi9IJcmDSwF54fQaTVh2CbL6PInw==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-android-arm64/-/binding-android-arm64-0.57.0.tgz", + "integrity": "sha512-mp6PibWbao3aizijcheOeHQaYEhcUAt8pwLniYbtLfHxL/psFF0BykAwCj+s3c6qIpa8yN8keZICWrqtZ70w8g==", "cpu": [ "arm64" ], @@ -5026,9 +4495,9 @@ } }, "node_modules/@oxfmt/binding-darwin-arm64": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-arm64/-/binding-darwin-arm64-0.55.0.tgz", - "integrity": "sha512-xDQczLH9pw/RBk1h/GH0qcGMm8hQtmtVHBNLSH3lk1gEIR09hZ4L+mJQl4VqiVAvPK9VG9PYrWWuSQLt7xTbiA==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-arm64/-/binding-darwin-arm64-0.57.0.tgz", + "integrity": "sha512-T+0stuCBqmUVY+aMIvrgXhzGhHO3sD5tNiiEcYqgSdPsnukskQqn2u5qOVD0sv1l7RLdFS5Z/f5Wi9Ktyjr3Eg==", "cpu": [ "arm64" ], @@ -5043,9 +4512,9 @@ } }, "node_modules/@oxfmt/binding-darwin-x64": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-x64/-/binding-darwin-x64-0.55.0.tgz", - "integrity": "sha512-JaNoFCkF2CJdGgpPSMbuO9HVyXyoNGIhMHPvp6NYAjeVKw9XEYc0HcUWJLPQa3Q69WV5wMa9m5jPMJPtbLtcRg==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-darwin-x64/-/binding-darwin-x64-0.57.0.tgz", + "integrity": "sha512-O+3JbqWs/mCI2oi4xfhRO2IVPFJNDDEBV8Odo+ZpmsUOeKJfjXoNH7nDmBEQcDgK7NfjDIyE7kRgYSZcTLDO0A==", "cpu": [ "x64" ], @@ -5060,9 +4529,9 @@ } }, "node_modules/@oxfmt/binding-freebsd-x64": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-freebsd-x64/-/binding-freebsd-x64-0.55.0.tgz", - "integrity": "sha512-DNbszhpg6S2MIzax5azdHFTTBIVkR5xr8yyRZuA4yoDAwOkzIp3tmldgKZM2+VlT+hJIG0xUksA+elISzMEAfA==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-freebsd-x64/-/binding-freebsd-x64-0.57.0.tgz", + "integrity": "sha512-pxwhxVC+JkLX9twOQ/8C/vbuOQcMZyKIDmiRDZfO7yITuVcIdZCiLRqqf4QOxb2+8FWrRXzQpm+1DBKcMpHSSQ==", "cpu": [ "x64" ], @@ -5077,9 +4546,9 @@ } }, "node_modules/@oxfmt/binding-linux-arm-gnueabihf": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.55.0.tgz", - "integrity": "sha512-2snoaoRfFFyGnbOcKUK36rREBYxe/Xgz3uHbiA5zbCB/s6R4DQj4mHqYAaWWhgizCUSDxV8cE9zAZ0XleNpKGw==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.57.0.tgz", + "integrity": "sha512-pxBU4zH2imB/MDBfth2rOMeVxXUMjRQLCazagwLARIFH3hVlxZJBlM4nSnHXaIHJK4/qezoFCIORN6AY8Mra4A==", "cpu": [ "arm" ], @@ -5094,9 +4563,9 @@ } }, "node_modules/@oxfmt/binding-linux-arm-musleabihf": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.55.0.tgz", - "integrity": "sha512-q1aktHF/WRpSK81BX1dE/9vWrS2jGw1Nax2kb4DBLGAewubCLcoNyp4Zl/NSMgbv3vUS46Z33wIQkBVYOP3PYg==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.57.0.tgz", + "integrity": "sha512-JAprOzt8tycYou36ZgEw14DlRHTiN8qdtKANdV3VZIRIvTI/lh/cX13c9pJ/EnDk2GT3FASH7KvCgQ2AufAifQ==", "cpu": [ "arm" ], @@ -5111,13 +4580,16 @@ } }, "node_modules/@oxfmt/binding-linux-arm64-gnu": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.55.0.tgz", - "integrity": "sha512-VD0y36aENezl/3tsclA/4G53Cc7iV+7Uoh7gz4yvcOTaEYBtJpQsE6PKDGTtUtOvGS4kv51ybfXY/nWZejO5IA==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.57.0.tgz", + "integrity": "sha512-ajtjaxSaj9xl4BW7REt+Cef/ttzbAq00Bq4z7JUDZEfgFXdwSjH8K9bF+IcIJzZB9lKqMfQ4eHuSFOvvlvtqOg==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -5128,13 +4600,16 @@ } }, "node_modules/@oxfmt/binding-linux-arm64-musl": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.55.0.tgz", - "integrity": "sha512-r8xlKJFcsRmn0H5jZrdORae6RX9jDBrZVvOoxF+bCQtampQJClv80aZEHsv+NsLsp2KCE5ql79O7DpPVzYWpXA==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.57.0.tgz", + "integrity": "sha512-p4Y/+RYk9Bk5WO+zHSUXAClRmZ2fbJCejMuCAsU2HhyME4jqf6Ftt/mJYEwIah1wGCBDYOB7wEGV1x5bCEZ6hA==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -5145,13 +4620,16 @@ } }, "node_modules/@oxfmt/binding-linux-ppc64-gnu": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.55.0.tgz", - "integrity": "sha512-GRKv/HXHcwIVld/WU61rF0g0R16hl5EJ+ScKdpjevT57lnLnagj/U2YUbXf2mT+2Pg1uCzWC+mvGicPV3CDdLQ==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.57.0.tgz", + "integrity": "sha512-By6tRALAZsno0F4zedmtG+wdMvJiJmJoXM4d3+A9zHE4HRXLqXITwRH8mgrlcXc5yJM2g2W3riRPwTYdgemZLQ==", "cpu": [ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -5162,13 +4640,16 @@ } }, "node_modules/@oxfmt/binding-linux-riscv64-gnu": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.55.0.tgz", - "integrity": "sha512-rdv57enTiPtpSYRMKfAiEbQb0Puw5t9N7isVinDoo5qeLDScro2gznmZqSgSWbVZRzLisTeCTW8Qwgw0bOHv3A==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.57.0.tgz", + "integrity": "sha512-skYeG+RgvyzspqVEBsEprL90OYYZfoVNqB3HcCNR6QDJyXKOzfDRT3zncnHmUaFluIlBHuY23mU1b5WGgR98hA==", "cpu": [ "riscv64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -5179,13 +4660,16 @@ } }, "node_modules/@oxfmt/binding-linux-riscv64-musl": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.55.0.tgz", - "integrity": "sha512-7v1nNrlD43VY6+sYQ6efYyb3lE6QY182304PD/768ZxTjOmFd/3dQa3u/nGBUAXYdGSWOQc5N3PnS0QzUXyEIA==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.57.0.tgz", + "integrity": "sha512-FFgACrZOXAXUh5KQh2mt1CDOVOZmn+QzHP71wM9QobNwyQvoFfyAeefVUltW83g3sm7LTiH3yfFqLLVUpA5ZFQ==", "cpu": [ "riscv64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -5196,13 +4680,16 @@ } }, "node_modules/@oxfmt/binding-linux-s390x-gnu": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.55.0.tgz", - "integrity": "sha512-f4lJLUSPOgScjFl9LiflKCTocyNRwE25JmTMbN4XQdDjoZzEHjqf3wA3VESF1/csg7i8m7+EQLbrZyYDqe10UQ==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.57.0.tgz", + "integrity": "sha512-Nm/BAOfQeFiiKd502mZn/GAVKJwtd0RdCg17G3Wz/WSOIQmDi3+7/SZH4BHn1Ye5KvTVH3ua8WvfwLLycNIuvA==", "cpu": [ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -5213,13 +4700,16 @@ } }, "node_modules/@oxfmt/binding-linux-x64-gnu": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.55.0.tgz", - "integrity": "sha512-MihqiPziJNoWy4MqNSV+jVA1g+07iQDjZiR0vaCaDoPgFEiJpCMsxamktzLV07cEeQsSJ04vQaU4CzCQwIvtDA==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.57.0.tgz", + "integrity": "sha512-BiSy5Ku3mQqyxS6YIqAJgd403wEUWvI7kerfzPxc2l/txZVmZM0pSj7oDM+4bGBExowxOi7o73jEam1W0EDTZg==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -5230,13 +4720,16 @@ } }, "node_modules/@oxfmt/binding-linux-x64-musl": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-musl/-/binding-linux-x64-musl-0.55.0.tgz", - "integrity": "sha512-Yqghym7KYAVjP9MmSrNZiDeerMuoejNjo0r3ox5H3GDKk8eAfl8VyJm9i+pWCLDCTnAbcTUMMN2ZKjUYXH1v3g==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-linux-x64-musl/-/binding-linux-x64-musl-0.57.0.tgz", + "integrity": "sha512-BCRkJiotz5s9afLYD2LuMvzAoDYx9H17E/YbDyu4xK7l4zHDPeny9ErSXL//i/nJyaOwRk08x4b8cgJC00+JDg==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -5247,9 +4740,9 @@ } }, "node_modules/@oxfmt/binding-openharmony-arm64": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-openharmony-arm64/-/binding-openharmony-arm64-0.55.0.tgz", - "integrity": "sha512-s5SDvVVSbyQl1V5UU3Yl12M+XLUQ3rl5SglNqgAA2K4PXUtQhyNSS00wivONPEnNo5W01rCou8WkDNyvI/RGHg==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-openharmony-arm64/-/binding-openharmony-arm64-0.57.0.tgz", + "integrity": "sha512-4Oaxe1qrGgXfpCJ1C/ERJ2iCtV2rN1R79ga9fsfyVHfSQRu/hVW780u2KDqZWFZ/iGTHODJji0JemxqFZ63eIQ==", "cpu": [ "arm64" ], @@ -5264,9 +4757,9 @@ } }, "node_modules/@oxfmt/binding-win32-arm64-msvc": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.55.0.tgz", - "integrity": "sha512-7p9FB5R32tw2KyyNX3wpQrR2WHwEHvMEiBlGXxeTCaRMCVNx3UtFMAUbaQ/pRNWIrEUZmYhJ6tcUH52uPTRYjQ==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.57.0.tgz", + "integrity": "sha512-MYLAsDnhdNsSGheLYhWgbk0vfIrlS84iQYun/y21fX6u0jj8iBtYtbpZMdiqYeuf8U12eVPUjVY2xE2NrCfJ0g==", "cpu": [ "arm64" ], @@ -5281,9 +4774,9 @@ } }, "node_modules/@oxfmt/binding-win32-ia32-msvc": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.55.0.tgz", - "integrity": "sha512-ZYqj3fDnOT1IaVGMP5kpmkQl4F3tQIm2ZyAxvqkJYmI0xgWWak4ss4XYwv3VDfM+TWXeC9K4uQ/wW5jm/5XABA==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.57.0.tgz", + "integrity": "sha512-PBwdzZALJY/jcCx2E6is0yu+cuVXeySTDmwuseD+9j0mHqlRNxwlKgsyRTBed/woPeqfVfuXfWjoq4Cx2Zt3Eg==", "cpu": [ "ia32" ], @@ -5298,9 +4791,9 @@ } }, "node_modules/@oxfmt/binding-win32-x64-msvc": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.55.0.tgz", - "integrity": "sha512-eEYT5tivGnGbPHuOHuQpi6CGLObhh0re/5jcNQHihD2GRYkTM85dyi5a19zjP8Q00t1uqAx+/QGLUGdHeqzWyg==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/@oxfmt/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.57.0.tgz", + "integrity": "sha512-bQJdH9i4RRfw55jm7+8/xS7GzHLLTbHx4huhrrDxQJaJtbSDbsyOnODvP1ftT7EG0KFKAYO2S+q6AcioXODx8w==", "cpu": [ "x64" ], @@ -5315,9 +4808,9 @@ } }, "node_modules/@oxlint/binding-android-arm-eabi": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.70.0.tgz", - "integrity": "sha512-zFh0P4cswmRvw6nkyb89dr18rRanuaCPAsEXsFDoQY8WdaquI8Pt4NWFjaMJg6L23cy5NeN8J9cBnREbWzZhaw==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm-eabi/-/binding-android-arm-eabi-1.72.0.tgz", + "integrity": "sha512-zhCmvn+1Mj3UchAc/90i99S0t7jJUsHmFVSPg4UWrjO8b8eaSGwscgO6QAUtvHBstkjQwBttQNswEnAF1mIQdA==", "cpu": [ "arm" ], @@ -5332,9 +4825,9 @@ } }, "node_modules/@oxlint/binding-android-arm64": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.70.0.tgz", - "integrity": "sha512-qI8o4HZjeGiBrWv+pJv4lH0Yi2Gl/JSp/EumBUApezJprIKa5PS4nU0lQsQngtky8k+SplQIOjv6hwu0SSxeyg==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-android-arm64/-/binding-android-arm64-1.72.0.tgz", + "integrity": "sha512-mtH+aY/ozv1eZoCUC2owjFAtyNBKHpJHygKeEu9zXXnQGW1Q2/qOpvx+I+Lf23+TvTz66F4iiXUbl2cGvoLPCQ==", "cpu": [ "arm64" ], @@ -5349,9 +4842,9 @@ } }, "node_modules/@oxlint/binding-darwin-arm64": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.70.0.tgz", - "integrity": "sha512-8KjgVVHI5F9nVwHCRwwA78Ty7zNKP4Wd9OeN5PSv3iu/F/u1RVXoOCgLhWqust6HmwQG6xc8c+RCyaWENy24+w==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-arm64/-/binding-darwin-arm64-1.72.0.tgz", + "integrity": "sha512-EvnajNPDtfknB3ZieeOOyDTwJn9QXDiwfnF4ZDQqART6RG6hjY4WigQcZdGoK2dkB3e1vrmEzN9aYbQCUkh/gQ==", "cpu": [ "arm64" ], @@ -5366,9 +4859,9 @@ } }, "node_modules/@oxlint/binding-darwin-x64": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.70.0.tgz", - "integrity": "sha512-WVydssv5PSUBXFJTdNBWlmGkbNmvPGaFt/2SUT/EZRB6bq6bEOHmMlbnupZD5jmlEvi9+mZJHi8TCw15lyfSfQ==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-darwin-x64/-/binding-darwin-x64-1.72.0.tgz", + "integrity": "sha512-ZkCdEa/G80A7vEHfeCDz/+L3m33DE73v32mDKhgOIgz8Uwf0DFcK7+uu6qC+7LEhmz5fpOe1osWKyjSNMydFIQ==", "cpu": [ "x64" ], @@ -5383,9 +4876,9 @@ } }, "node_modules/@oxlint/binding-freebsd-x64": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.70.0.tgz", - "integrity": "sha512-hJucmUf8OlinHNb1R7fI4Fw6WsAstOz7i8nmkWQfiHoZXtbufNm+MxiDTIMk1ggh2Ro4vLzgQ+bKvRY54MZoRA==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-freebsd-x64/-/binding-freebsd-x64-1.72.0.tgz", + "integrity": "sha512-NroXv2vh+sxVY1uya/rM5pjhx1hm8BzlYpx9q67QP0Xhw5MH2bf5GJylpvLEC+781p1Xli/317EoV9AlGwViag==", "cpu": [ "x64" ], @@ -5400,9 +4893,9 @@ } }, "node_modules/@oxlint/binding-linux-arm-gnueabihf": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.70.0.tgz", - "integrity": "sha512-1BnS7wbCYDSXwWzJJ+mc3NURoha6m6m6RT5c6vgAY3oz7C3OVXP+S0awo2mRq97arrJkVvO3qRQfyAHL+76xtQ==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.72.0.tgz", + "integrity": "sha512-0NDywYgfj279Ou/BcQuCYSj7NJwBfmWn5qc5uGO/Ny7fUWmXyIpvawqX/8acQlWG6IXelJsJhj+JAy6sjsKj0A==", "cpu": [ "arm" ], @@ -5417,9 +4910,9 @@ } }, "node_modules/@oxlint/binding-linux-arm-musleabihf": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.70.0.tgz", - "integrity": "sha512-yKy/UdbR55+M2yEcuiV5DCNC/gdQAjr/GioUy50QwBzSrKm8ueWADqyRLS9Xk+qjNeCYGg6A8FvUBds56ttfqg==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-1.72.0.tgz", + "integrity": "sha512-4vpXB06h65Ezsy4hRyrGjGrfa1SkVPii09yaajiYhmVpgsFiLD+KNxIx/BNAY+XiO+i1yqp9HHdwqM8VTqa5XQ==", "cpu": [ "arm" ], @@ -5434,13 +4927,16 @@ } }, "node_modules/@oxlint/binding-linux-arm64-gnu": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.70.0.tgz", - "integrity": "sha512-0A5XJ4alvmqFUFP/4oYSyaO+qLto/HrKEWTSaegiVl+HOufFngK2BjYw9x4RbwBt/du5QG6l5q1zeWiJYYG5yg==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.72.0.tgz", + "integrity": "sha512-immaN4g2ZGFiOkKrvRX9LvzZdd2GkQM5wR+UyzYyUuyhUTXGQ4HKUJH18xp4G8OfhCVaVAJfKZxwE1r8+4hhaQ==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -5451,13 +4947,16 @@ } }, "node_modules/@oxlint/binding-linux-arm64-musl": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.70.0.tgz", - "integrity": "sha512-JiylyurlB0CLSedNtx1gzv3FvfWPF1h/2Y3BJszPLNt5XQFlBsH5ke0Jle3iJb3uqu5m2e7A/DwzpuCAHdiU+A==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.72.0.tgz", + "integrity": "sha512-JGHS9Mnr7iWyyLDxgCv1MhzVpAckgptg00F2gnxt/GD7lQ2SW1BRcxHqhSTaSdDpjWRrBkBxMMh4+Hn3aVtExg==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -5468,13 +4967,16 @@ } }, "node_modules/@oxlint/binding-linux-ppc64-gnu": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.70.0.tgz", - "integrity": "sha512-J8VPG7I3/HmgaU4u8pNU2kFx2+0U+vPLS1dXFxXOaR/2TQ0f8AC7DRz0SRGRI1bfphnX2hVYTTtLuhL4nYKL+Q==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.72.0.tgz", + "integrity": "sha512-AOYgBZqxNshrg83P9v0RYv+m8s10Cqkj4/PxXFDhcS3k7FqsIG5+CxErshZCIN7G8iy4Y+VGfAsuEdar8AcbBg==", "cpu": [ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -5485,13 +4987,16 @@ } }, "node_modules/@oxlint/binding-linux-riscv64-gnu": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.70.0.tgz", - "integrity": "sha512-N2+4lV2KLN+oXTIIIwmWDhwkrnvqf5oX7Hw0zPjk+RuIVgiBQSOlJWF7uQoFx2siEYX0ZQ5cfSbEAHm+J3t7Wg==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-1.72.0.tgz", + "integrity": "sha512-QMybPS5ij3/vrKG67mqzHwW++91sYxK/PPUVi6SBtNCEzW4niS52fVBdXbQ6nou0wWbUPEpx8Sl/ZjtgE3clXA==", "cpu": [ "riscv64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -5502,13 +5007,16 @@ } }, "node_modules/@oxlint/binding-linux-riscv64-musl": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.70.0.tgz", - "integrity": "sha512-1e2L7cFCvx9QDzq6NPP+0tABKb5z6nWHyddWTNKprEsjO9xNrAtPowuCGpjNXxkTdsMiZ4jc8YQ5SstZd4XK6g==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-1.72.0.tgz", + "integrity": "sha512-gOc3W7JV0PXRpIL7stUlLe3Wa9Gp0Kdlup87IT3gHDvPKck2xNgMIl/Gs2lldYY2lyXZDC4rWi3hmoLUobkgbQ==", "cpu": [ "riscv64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -5519,13 +5027,16 @@ } }, "node_modules/@oxlint/binding-linux-s390x-gnu": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.70.0.tgz", - "integrity": "sha512-Kwu/l/8GcYibCWA9m9N5pRXMIKVSsL/YbgpLzYkqDhWTiqdRfnNJ/+nqIKRKQiFbHWsdlHEhzMwruJK+qcEruA==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.72.0.tgz", + "integrity": "sha512-rpGxph+FjjHcYI5q6uxB3Az+tnfmEnDbSA8+PK9ZE/VzyUAkvBOMeuY7ZQMhu5mpZH7YQDsTdW6Cx4kV/msc6w==", "cpu": [ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -5536,13 +5047,16 @@ } }, "node_modules/@oxlint/binding-linux-x64-gnu": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.70.0.tgz", - "integrity": "sha512-tap04CsHYOl0nSAQJfPNIuBxqEPB2HnhQqwaOXLg1jnp2XfRo8Fa814dA4QC4zpvTWXCjAAaCY1W5LOORkEQuQ==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.72.0.tgz", + "integrity": "sha512-WND+uhf/Ko13SLqQMWQUgsZuLvYYEvL0ZKgg0tgGYfLqxG7l8Ju123fHDMJyYSDl5E3bUbpFUuii/OvMreFQzw==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -5553,13 +5067,16 @@ } }, "node_modules/@oxlint/binding-linux-x64-musl": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.70.0.tgz", - "integrity": "sha512-hzJa/WgvtJpbBD9rgfy0qe+MjbxOXNUT0bfR1S6EQQzfTtBFA9xg5q8KSwRrQ2QfSS+TaP4j+4mVPQrfNc6UNg==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-linux-x64-musl/-/binding-linux-x64-musl-1.72.0.tgz", + "integrity": "sha512-SrpbrUL70nG9vh6zP4/oKHWgLuHquwsr7MW9XOn0olBVgh10Uqr8qscKhQoBGEn6olK/IUpn5GSKcdQ5AjUhGA==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -5570,9 +5087,9 @@ } }, "node_modules/@oxlint/binding-openharmony-arm64": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.70.0.tgz", - "integrity": "sha512-xbsaNSNzVSnaJACCUYr1HQMyY/Q/Q1LkePmHG3UvZPvGCYGNxrsZp9OmtA6ick8xH47ltRRbRrPCM1YXYcyC+A==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-openharmony-arm64/-/binding-openharmony-arm64-1.72.0.tgz", + "integrity": "sha512-qkrsEn6NmgFKr7U/QnezQMb+q/vzAy0Dd9Y95gQGQTyjzDLN+HRZMuM5u70iyH4nBLCfKBzhjMsYCehKay2jyg==", "cpu": [ "arm64" ], @@ -5587,9 +5104,9 @@ } }, "node_modules/@oxlint/binding-win32-arm64-msvc": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.70.0.tgz", - "integrity": "sha512-icAEsUI7JbW1TMRdEXV83mVAInhRVQYuuAlPpxdGwJ95chNdnCzjloRW8GglT0WvzOEZSio6fnYSk2DJ2Hv7LQ==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.72.0.tgz", + "integrity": "sha512-LWR6ZlFZph+KPjXv8opgZsXRDCdrdQe8VL8Cg9zxCoBS73h6znzZpydVgmdnwj8mB9AuSM5jxEgDJDpQkjboeg==", "cpu": [ "arm64" ], @@ -5604,9 +5121,9 @@ } }, "node_modules/@oxlint/binding-win32-ia32-msvc": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.70.0.tgz", - "integrity": "sha512-FHMSWbVsPVs/f+Jcl04ws4JJ2wUnauyTzlpxWRG/lSO/8GpX08Fo2gQZqdA6CrRFI+zvkxl+N/KwJGWfUwYVZA==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.72.0.tgz", + "integrity": "sha512-yt6HEh7IsHvtjRWtmeZRX134eaXKHq5Gnqlf1xBJdJl1JtdoRUEJw3nAxpZoUDS860cX/foKbztO441anVBtVQ==", "cpu": [ "ia32" ], @@ -5621,9 +5138,9 @@ } }, "node_modules/@oxlint/binding-win32-x64-msvc": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.70.0.tgz", - "integrity": "sha512-ptOlKwCz7n4AKs5VweMqG6DAg677FmKOK+vBkkL9DMNgFATIQ+upqUYBTOEwRQyRAx1ncGlPlXleV2hIcm3z4g==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/@oxlint/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.72.0.tgz", + "integrity": "sha512-b2eKFD2hX7tIwmo/cyH6TDq8vzWRZ2qNHrzoGntUTmq0h3zQh/uX3eTSHCwI8OB/ADQfJCRelLItK8BsxuucDA==", "cpu": [ "x64" ], @@ -5645,9 +5162,9 @@ "license": "MIT" }, "node_modules/@rolldown/binding-android-arm64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.3.tgz", - "integrity": "sha512-454rs7jHngixp/NMxd5srYD57OnzSlZ/eFTETjORQHLwJG1lRtmNOJcBerZlfu4GjKqeq8aCCIQrMdHyhI51Hw==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.4.tgz", + "integrity": "sha512-EZLpf/8y7GXkkra90ML47kzik/GMP3EMcE9bPyHmRfxLC6z9+aW5A8poCsoxjrT5GfEcNAAvWwUHjvP1pUQkfw==", "cpu": [ "arm64" ], @@ -5661,9 +5178,9 @@ } }, "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.3.tgz", - "integrity": "sha512-PcAhP+ynjURNyy8SKGl5DQP94aGuB/7JrXJb/t7P+hanXvQVMWzUvRRhBAcg/lNRadBhoUPqSoP4xw5tR/KBEA==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.4.tgz", + "integrity": "sha512-aUi+HBvmYb7j8krl1+qJgkG8C17fO79gk3c+jPw4S8glRFc1DTija9S3EyaTSQUm5GJXYKDAsugBEhFHH2vYiQ==", "cpu": [ "arm64" ], @@ -5677,9 +5194,9 @@ } }, "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.3.tgz", - "integrity": "sha512-9YpfeUvSE2RS7wysJ81uOZkXJz7f7Q55H2Gvp3VEw/EsahqDtrphrZ0EwDLK5vvKOzaCrBsjF8JmnMLcUt78Gg==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.4.tgz", + "integrity": "sha512-F7hHC3gwY11+vByKPRWqwGbeXWVgKmL+pTGCinaEhdihzBV2aQ0fvZOch9cXYUOKuKKq429HeYXOqQLc7wFCEg==", "cpu": [ "x64" ], @@ -5693,9 +5210,9 @@ } }, "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.3.tgz", - "integrity": "sha512-yB1IlAsSNHncV6SCTL27/MVGR5htvQsoGxIv5KMGXALp+Ll1wYsn+x98M9MW7qa+NdSbvrrY7ANI4wLJ0n1e6g==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.4.tgz", + "integrity": "sha512-sI5yw+7s92SK6odiEhD5lKCBlWcpjHS5qyqpVQbZAJ0fIzEUXrmbl3DH2ybR3PZogulNJF+COLtmA8hUfvkCCQ==", "cpu": [ "x64" ], @@ -5709,9 +5226,9 @@ } }, "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.3.tgz", - "integrity": "sha512-Yi30IVAAfLUCy2MseFjbB1jAMDl1VMCAas5StnYp8da9+CKvMd2H2cbEjWcw5NPaPqzvYkVIaF1nNUG+b7u/sw==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.4.tgz", + "integrity": "sha512-mCi0OKgEieFircrtVYmQAFGszRtMnZ6fpZAXrxanXAu7lqZcsK1E1RAaZNG0uKAnxox3B1f4EyQNnoyMfN1vAA==", "cpu": [ "arm" ], @@ -5725,12 +5242,15 @@ } }, "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.3.tgz", - "integrity": "sha512-jsO7R8To+AdlYgUmN5sHSCZbfhtMBkO0WUx8iORQnPcMMdgr7qM2DQmMwgabs3GhNztdmoKkMKQFHD6DTMCIQw==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.4.tgz", + "integrity": "sha512-B9Ial3Kv5sh0SHnB1g/QWcUQCEvCF6QKGAl4zXypYj65mVI+B4AhFBwPtSN7pDrJeIx8Z7zdy4ntx+wQABom7w==", "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -5741,12 +5261,15 @@ } }, "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.3.tgz", - "integrity": "sha512-VWkUHwWriDciit80wleYwKILoR/KMvxh/IdwS/paX+ZgpuRpCrKLUdadJbc0NpBEiyhpYawsJ73j9aCvOH+f7Q==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.4.tgz", + "integrity": "sha512-lZVym0PuHE1KZ22gmFTC15lAkrg9iTszR617oYRB/iPY1A56ywoJzVKOJBKaot5RiikCObmur6pogpse3gRcng==", "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -5757,12 +5280,15 @@ } }, "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.3.tgz", - "integrity": "sha512-5f1laC0SlIR0yDbFCd8acUhvJIag6N3zC5P7oUPN6wX0aOma+uKJ0wBDH5aq7I1PVI2ttTlhJwzwRIBnLiSGEg==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.4.tgz", + "integrity": "sha512-t2DNiLJWNTbnEHyUzTumldML6ET4/g16467LZoDDJ3tSxGvguL5/NyC2lCsNKuyRycg9XeDQF5SSv+TNOhQEXg==", "cpu": [ "ppc64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -5773,12 +5299,15 @@ } }, "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.3.tgz", - "integrity": "sha512-Iq4ko0r4XsgbrF/LunNgHtAGLRRVE2kXonAXQ/MV0mC6jQpMOhW1SvtZja2EhC/kd05++bP78dsqBeIQyYJ6Yg==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.4.tgz", + "integrity": "sha512-0WIRnL1Uw4BvTZRLQt+PVgo6ZKTJadlC2btP+/EOXv2f/DWbY0rEgl+y834mIVwP1FkTlWVTrGGJXf12lru7EQ==", "cpu": [ "s390x" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -5789,12 +5318,15 @@ } }, "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.3.tgz", - "integrity": "sha512-B8m6tD5+/N5FeNQFbKlLA/2yVq9ycQP1SeedyEYYKWBNR3ZQbkvIUcNnDNM03lO1l5F2roiiFJGgvoLLyZXtSg==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.4.tgz", + "integrity": "sha512-JWtGshGfX+oENAKonoNkqEJX+7hC8yfhi9GUyPX1VX4mdh1y5r+ZiJLR5XzAB0aoP6s/PcILsGjKq8O0mm24bw==", "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -5805,12 +5337,15 @@ } }, "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.3.tgz", - "integrity": "sha512-pSdpdUJHkuCxun9LE7jvgUB9qsRgaiyNNCX7m/AvHTcq67AiT/Yhoxvw5zPfhrM8k/BfP8ce/hMOpthKDpEUow==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.4.tgz", + "integrity": "sha512-rT6yQcxUuXs4CnbofqwHRRV0iem349rLMYpTjkgQGLjrY4ado/eDzwPZPTCgTOlF6Nkp8NEv70yLMTn6qkWxsQ==", "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -5821,9 +5356,9 @@ } }, "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.3.tgz", - "integrity": "sha512-OXXS3RKJgX2uLwM+gYyuH5omcH8fL1LJs96pZGgtetVCahON57+d4SJHzTgZiOjxgGkSnpXpOsWuPDGAKAigEg==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.4.tgz", + "integrity": "sha512-KXMGoboq5cyaCQjDA4GLuRiOwBQ0EyFnJoVViLeZ45/3rFItRODEr+NdsBcVpll40hhNArlm/speWGRvj08LzA==", "cpu": [ "arm64" ], @@ -5837,27 +5372,27 @@ } }, "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.3.tgz", - "integrity": "sha512-JTtb8BWFynicNSoPrehsCzBtOKjZ6jhMiPFEmOiuXg1Fl8dn2KHQob+GuPSGR0dryQa1PQJbzjF3dqO/whhjLg==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.4.tgz", + "integrity": "sha512-5K83rb36oJiY7BCyE9zLZtGcPV4g5wvq+xwdO0XPIwDVZI8cyB/AUjkNXGb92/rnmezEkjMOpgY61rtwjQtFwg==", "cpu": [ "wasm32" ], "license": "MIT", "optional": true, "dependencies": { - "@emnapi/core": "1.10.0", - "@emnapi/runtime": "1.10.0", - "@napi-rs/wasm-runtime": "^1.1.4" + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", + "@napi-rs/wasm-runtime": "^1.1.6" }, "engines": { "node": "^20.19.0 || >=22.12.0" } }, "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.3.tgz", - "integrity": "sha512-gEdFFEN70A/jxb2svrWsN3aDL7OUtmvlOy+6fa2jxG8K0wQ1ZbdeLGnidov6Yu5/733dI5ySfzFlQ/cb0bSz1g==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.4.tgz", + "integrity": "sha512-PnWBtw3TV5KOg69HQQDR0mnQuyCmSGR2pAB4DC1rPF808fgKeTUMj2EOEyKATpgiuxuR5APQmiDO7PDgEjTFSA==", "cpu": [ "arm64" ], @@ -5871,9 +5406,9 @@ } }, "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.3.tgz", - "integrity": "sha512-eXB7CHuaQdqmJcc3koCNtNPmT/bj2gc999kUFgBxG8Ac0NdgXc4rkCHhqrgrhN3zddvvvrgzj1e90SuSfmyIXA==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.4.tgz", + "integrity": "sha512-M1lpniBePobTfsa7Ks9a199e1akxsXn+GYBUKsEzv3YFzOm1HJAMNwKI3qr0Zq+mxwx9gOZoTdP1yXRYsZUocQ==", "cpu": [ "x64" ], @@ -5894,16 +5429,12 @@ "license": "MIT" }, "node_modules/@smithy/config-resolver": { - "version": "4.4.17", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.4.17.tgz", - "integrity": "sha512-TzDZcAnhTyAHbXVxWZo7/tEcrIeFq20IBk8So3OLOetWpR8EwY/yEqBMBFaJMeyEiREDq4NfEl+qO3OAUD+vbQ==", + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-4.6.6.tgz", + "integrity": "sha512-yI8StAYGQKiI+DxT0xkLhHSzxcIFSZTTWi0c4Lk3WLgmbe2QfLpR/Roo81ZF3VYwfwqH4IQVTKsy3FLZiVRaTQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/node-config-provider": "^4.3.14", - "@smithy/types": "^4.14.1", - "@smithy/util-config-provider": "^4.2.2", - "@smithy/util-endpoints": "^3.4.2", - "@smithy/util-middleware": "^4.2.14", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -5911,13 +5442,12 @@ } }, "node_modules/@smithy/core": { - "version": "3.25.1", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.25.1.tgz", - "integrity": "sha512-zpDbpXBCBsxfLtG2GEUyfgvHvSFrw5CwDZSNzL0v52gx/c3oPlPbm+7W7num8xs6vyiUBn+bvYPHcQDOXZynCQ==", + "version": "3.29.1", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.29.1.tgz", + "integrity": "sha512-qoiY4nrk5OCu1+eIR1VB8l5DmON/oKiqrd5zZFAhXJXjJlLWQusKEW/SkBDAtGDcPaz86m9kfcE1lngU0GlM6A==", "license": "Apache-2.0", "dependencies": { - "@aws-crypto/crc32": "5.2.0", - "@smithy/types": "^4.15.0", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -5925,13 +5455,13 @@ } }, "node_modules/@smithy/credential-provider-imds": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.1.tgz", - "integrity": "sha512-TSAF5NHgxEsllbErYWbK8aLnl5L601NGc5VYJlSPsKnf3YlkhdoBN+geGcaU00oiw2OK3QO5LA3QNXiiWhCidQ==", + "version": "4.4.6", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.6.tgz", + "integrity": "sha512-B2WQ/PV/H6Jeg3lrIq6bKUfa6Hy01mtK7CGs6lhjzHA6k4aagldH6T6eEjnzKl4HI0cJnAsxfJ19pgb5PV+CVQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.25.1", - "@smithy/types": "^4.15.0", + "@smithy/core": "^3.29.1", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -5939,13 +5469,13 @@ } }, "node_modules/@smithy/fetch-http-handler": { - "version": "5.5.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.5.1.tgz", - "integrity": "sha512-96JrD1q71anokymx9Iblb+zKmNQYNstlV/25A9ZYIJ2A0rp1r7/GZAIm0bDWSmVvz3DpNOCZuabzsiL+w0UHhw==", + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.3.tgz", + "integrity": "sha512-CwCc/7SMTj45y97MUnDTbTaxvtAsiNNRm81z3abROIuMbMsC2Iy5EKfkkVdsKrz8WExQAAMx1EJapq+9j4fFTQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.25.1", - "@smithy/types": "^4.15.0", + "@smithy/core": "^3.29.1", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -5953,14 +5483,12 @@ } }, "node_modules/@smithy/hash-node": { - "version": "4.2.14", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.2.14.tgz", - "integrity": "sha512-8ZBDY2DD4wr+GGjTpPtiglEsqr0lUP+KHqgZcWczFf6qeZ/YRjMIOoQWVQlmwu7EtxKTd8YXD8lblmYcpBIA1g==", + "version": "4.4.6", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-4.4.6.tgz", + "integrity": "sha512-O6YGQgZGxypohUJDm+aIBNx9ldzOvQOjnnAmHYKAVJcVRMYQLf+g88s/vCUIHJudeXbgEo7xq1vXUGYlBpdBsg==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.1", - "@smithy/util-buffer-from": "^4.2.2", - "@smithy/util-utf8": "^4.2.2", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -5968,12 +5496,12 @@ } }, "node_modules/@smithy/invalid-dependency": { - "version": "4.2.14", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.2.14.tgz", - "integrity": "sha512-c21qJiTSb25xvvOp+H2TNZzPCngrvl5vIPqPB8zQ/DmJF4QWXO19x1dWfMJZ6wZuuWUPPm0gV8C0cU3+ifcWuw==", + "version": "4.4.6", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-4.4.6.tgz", + "integrity": "sha512-M2NNf2yB5pDYQiMyW3q6kD/aQyEfQlVFtFNR+LCnRKS/sg9xqOrOqkIZNP94B2aTwf+pwX3Qgp6foKw2ls9vIg==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.1", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -5981,25 +5509,25 @@ } }, "node_modules/@smithy/is-array-buffer": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-4.2.2.tgz", - "integrity": "sha512-n6rQ4N8Jj4YTQO3YFrlgZuwKodf4zUFs7EJIWH86pSCWBaAtAGBFfCM7Wx6D2bBJ2xqFNxGBSrUWswT3M0VJow==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" }, "engines": { - "node": ">=18.0.0" + "node": ">=14.0.0" } }, "node_modules/@smithy/middleware-apply-body-checksum": { - "version": "4.5.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-apply-body-checksum/-/middleware-apply-body-checksum-4.5.1.tgz", - "integrity": "sha512-uBPMRc7QEArsYCKSUDEopMeVix5fADAmjPcFxLoey812dgcs3nxswqnwMZONpKpIuL/MykkdUbKEsufgGCMl+A==", + "version": "4.5.6", + "resolved": "https://registry.npmjs.org/@smithy/middleware-apply-body-checksum/-/middleware-apply-body-checksum-4.5.6.tgz", + "integrity": "sha512-FcyRrZ1Mw9ueeg9IwFYsBFkuoilIeP5njB3X/ATmyqCTgzTsQ0g/T1cLAy69rkn4tcXVEDZ/he81yINrAPW6Eg==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.25.1", - "@smithy/types": "^4.15.0", + "@smithy/core": "^3.29.1", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -6007,13 +5535,13 @@ } }, "node_modules/@smithy/middleware-compression": { - "version": "4.5.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-compression/-/middleware-compression-4.5.1.tgz", - "integrity": "sha512-UOSkmD6sfprVxOnmPZJ+JWLnHC55dyqdJ+irVMaRbJRu5LAcRFYGzmpLbsex3aR/gQIJKtJaN8FT98V1Qr5Dbg==", + "version": "4.5.6", + "resolved": "https://registry.npmjs.org/@smithy/middleware-compression/-/middleware-compression-4.5.6.tgz", + "integrity": "sha512-BTRqmd5xIL6hFiJzWlQsMPSRmedAbBLEKIiwgvwdDom6tyKty2TWne48pehQPHwjYhCEiqJTlADjqXV36EI3sQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.25.1", - "@smithy/types": "^4.15.0", + "@smithy/core": "^3.29.1", + "@smithy/types": "^4.15.1", "fflate": "0.8.1", "tslib": "^2.6.2" }, @@ -6022,13 +5550,12 @@ } }, "node_modules/@smithy/middleware-content-length": { - "version": "4.2.14", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.2.14.tgz", - "integrity": "sha512-xhHq7fX4/3lv5NHxLUk3OeEvl0xZ+Ek3qIbWaCL4f9JwgDZEclPBElljaZCAItdGPQl/kSM4LPMOpy1MYgprpw==", + "version": "4.4.6", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-4.4.6.tgz", + "integrity": "sha512-kMXyfLgm2iqC3objUiElh1fp0AI33PxiAmPX+C0z7H8v4sfae2I2V9N62MlwO/LVPf+MlXpmfCYCHouetbecGg==", "license": "Apache-2.0", "dependencies": { - "@smithy/protocol-http": "^5.3.14", - "@smithy/types": "^4.14.1", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6036,18 +5563,12 @@ } }, "node_modules/@smithy/middleware-endpoint": { - "version": "4.4.32", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.4.32.tgz", - "integrity": "sha512-ZZkgyjnJppiZbIm6Qbx92pbXYi1uzenIvGhBSCDlc7NwuAkiqSgS75j1czAD25ZLs2FjMjYy1q7gyRVWG6JA0Q==", + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-4.6.6.tgz", + "integrity": "sha512-d7L894W4LE9HL2MkpWi0+KJG6/VCcuoxkquVaXYMMsrqK7PLT83GZsUBYL+ixxpEfwnlSE5hBotsB1A/qfWO9A==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.23.17", - "@smithy/middleware-serde": "^4.2.20", - "@smithy/node-config-provider": "^4.3.14", - "@smithy/shared-ini-file-loader": "^4.4.9", - "@smithy/types": "^4.14.1", - "@smithy/url-parser": "^4.2.14", - "@smithy/util-middleware": "^4.2.14", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6055,20 +5576,12 @@ } }, "node_modules/@smithy/middleware-retry": { - "version": "4.5.7", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.5.7.tgz", - "integrity": "sha512-bRt6ZImqVSeTk39Nm81K20ObIiAZ3WefY7G6+iz/0tZjs4dgRRjvRX2sgsH+zi6iDCRR/aQvQofLKxxz4rPBZg==", + "version": "4.7.6", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-4.7.6.tgz", + "integrity": "sha512-zUF+liYyzSiPcz8vQhGCjUlykg521Y3shrsyzJ37Y3nkpz20KujcVXcOvgZb6e68n7Unl/Pb2RG0jrkx73AE+Q==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.23.17", - "@smithy/node-config-provider": "^4.3.14", - "@smithy/protocol-http": "^5.3.14", - "@smithy/service-error-classification": "^4.3.1", - "@smithy/smithy-client": "^4.12.13", - "@smithy/types": "^4.14.1", - "@smithy/util-middleware": "^4.2.14", - "@smithy/util-retry": "^4.3.6", - "@smithy/uuid": "^1.1.2", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6076,14 +5589,12 @@ } }, "node_modules/@smithy/middleware-serde": { - "version": "4.2.20", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.2.20.tgz", - "integrity": "sha512-Lx9JMO9vArPtiChE3wbEZ5akMIDQpWQtlu90lhACQmNOXcGXRbaDywMHDzuDZ2OkZzP+9wQfZi3YJT9F67zTQQ==", + "version": "4.4.6", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.4.6.tgz", + "integrity": "sha512-h9JBsYQBiltcbe5EJECXoJy+ZJSmnJKhc9UXyoWvEB1BIUkE8inxQ614dpeb+yydm2cq4YXqMXQLQB0QUBE6rA==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.23.17", - "@smithy/protocol-http": "^5.3.14", - "@smithy/types": "^4.14.1", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6091,12 +5602,12 @@ } }, "node_modules/@smithy/middleware-stack": { - "version": "4.2.14", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.2.14.tgz", - "integrity": "sha512-2dvkUKLuFdKsCRmOE4Mn63co0Djtsm+JMh0bYZQupN1pJwMeE8FmQmRLLzzEMN0dnNi7CDCYYH8F0EVwWiPBeA==", + "version": "4.4.6", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-4.4.6.tgz", + "integrity": "sha512-LUO+XAQP381sbiZ0QHOSLKPpS2I0/wXTh44rH/kP4/751Ogscn5JUsAWvUH+rYJWLupmiEsdHZT3Bn4Dvd4J/Q==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.1", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6104,14 +5615,12 @@ } }, "node_modules/@smithy/node-config-provider": { - "version": "4.3.14", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.3.14.tgz", - "integrity": "sha512-S+gFjyo/weSVL0P1b9Ts8C/CwIfNCgUPikk3sl6QVsfE/uUuO+QsF+NsE/JkpvWqqyz1wg7HFdiaZuj5CoBMRg==", + "version": "4.5.6", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-4.5.6.tgz", + "integrity": "sha512-3fFpVGK28z1UhbDIPY6pujmnGwDhcD70NqJ8wJJwdQES5EYBe3mdOojSZdXtL0jfQE9BAbZSBi73WQ6VEOLRgQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/property-provider": "^4.2.14", - "@smithy/shared-ini-file-loader": "^4.4.9", - "@smithy/types": "^4.14.1", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6119,26 +5628,13 @@ } }, "node_modules/@smithy/node-http-handler": { - "version": "4.8.1", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.8.1.tgz", - "integrity": "sha512-emtXvoky671puri18ETf64AFIQUGIEA093F2drXpBgB0OGnBLjcwNR3CA2mYu62IAqNsS56xa5lnTxAgPq7cjw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/core": "^3.25.1", - "@smithy/types": "^4.15.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/property-provider": { - "version": "4.2.14", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-4.2.14.tgz", - "integrity": "sha512-WuM31CgfsnQ/10i7NYr0PyxqknD72Y5uMfUMVSniPjbEPceiTErb4eIqJQ+pdxNEAUEWrewrGjIRjVbVHsxZiQ==", + "version": "4.9.3", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.3.tgz", + "integrity": "sha512-qZTa4gQFUo8RM02rk6q5UVTDLNrQ1oS20LsepBzqq1QBVc/EHJ03OOUADcqMZiXHArW+Y7+OGY0BpdTwZRq/Yg==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.1", + "@smithy/core": "^3.29.1", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -6146,50 +5642,12 @@ } }, "node_modules/@smithy/protocol-http": { - "version": "5.3.14", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.3.14.tgz", - "integrity": "sha512-dN5F8kHx8RNU0r+pCwNmFZyz6ChjMkzShy/zup6MtkRmmix4vZzJdW+di7x//b1LiynIev88FM18ie+wwPcQtQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.14.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/querystring-parser": { - "version": "4.2.14", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-4.2.14.tgz", - "integrity": "sha512-hr+YyqBD23GVvRxGGrcc/oOeNlK3PzT5Fu4dzrDXxzS1LpFiuL2PQQqKPs87M79aW7ziMs+nvB3qdw77SqE7Lw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.14.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/service-error-classification": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-4.3.1.tgz", - "integrity": "sha512-aUQuDGh760ts/8MU+APjIZhlLPKhIIfqyzZaJikLEIMrdxFvxuLYD0WxWzaYWpmLbQlXDe9p7EWM3HsBe0K6Gw==", + "version": "5.5.6", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-5.5.6.tgz", + "integrity": "sha512-ezPS1foOBPU+uv0da16sLF/7fMBXxeo7hFKitn/2gOxHd8HaQ7qK6DT8l0f1dUGoUS1lZC2s7mEkCwJ44QLCJg==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.1" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/shared-ini-file-loader": { - "version": "4.4.9", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-4.4.9.tgz", - "integrity": "sha512-495/V2I15SHgedSJoDPD23JuSfKAp726ZI1V0wtjB07Wh7q/0tri/0e0DLefZCHgxZonrGKt/OCTpAtP1wE1kQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.14.1", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6197,13 +5655,13 @@ } }, "node_modules/@smithy/signature-v4": { - "version": "5.5.1", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.5.1.tgz", - "integrity": "sha512-X9rVls3En0z3NtrmguTmpRM0/NqtWUxBjal6fcAkwtsub+gOdLZ6kD+V7xhUgFMGdG14bHbZ7M5QjaRI1+DatQ==", + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.2.tgz", + "integrity": "sha512-QgHflghMoPxCJ9axiCVh8KZfbC9fuP6vkXXyK//E3cq7nLaSSyyLj0GAoqVWezYeDQmXIZhmlRvLE16jsqDK6g==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.25.1", - "@smithy/types": "^4.15.0", + "@smithy/core": "^3.29.1", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -6211,17 +5669,13 @@ } }, "node_modules/@smithy/smithy-client": { - "version": "4.12.13", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.12.13.tgz", - "integrity": "sha512-y/Pcj1V9+qG98gyu1gvftHB7rDpdh+7kIBIggs55yGm3JdtBV8GT8IFF3a1qxZ79QnaJHX9GXzvBG6tAd+czJA==", + "version": "4.14.6", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-4.14.6.tgz", + "integrity": "sha512-Nr6u0buDP8iprqxAJ83pSFu2RavKHa/OG8n/ReWTYHg1OS3GLoDoa8F+2zoWoYkvwUawTa/dFugJ7NS0CuPcYg==", "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.23.17", - "@smithy/middleware-endpoint": "^4.4.32", - "@smithy/middleware-stack": "^4.2.14", - "@smithy/protocol-http": "^5.3.14", - "@smithy/types": "^4.14.1", - "@smithy/util-stream": "^4.5.25", + "@smithy/core": "^3.29.1", + "@smithy/types": "^4.15.1", "tslib": "^2.6.2" }, "engines": { @@ -6229,9 +5683,9 @@ } }, "node_modules/@smithy/types": { - "version": "4.15.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.15.0.tgz", - "integrity": "sha512-Z5TAOxygoFvybJV3igo5SloFflSokHx2hu1eFA+DxDTcn+FtKxUSui+rbTRG1pAafMA888Z3MVvCWUuvCrTXjg==", + "version": "4.15.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.15.1.tgz", + "integrity": "sha512-x3L0XSACF6UYzKpa9biqiRMgvH5+wnFFew9Tm/grFYqgaupPwx/+ojDPpPJM8dZON3S9tjz5U+PQYsCBd1Mw5Q==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.6.2" @@ -6241,13 +5695,12 @@ } }, "node_modules/@smithy/url-parser": { - "version": "4.2.14", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.2.14.tgz", - "integrity": "sha512-p06BiBigJ8bTA3MgnOfCtDUWnAMY0YfedO/GRpmc7p+wg3KW8vbXy1xwSu5ASy0wV7rRYtlfZOIKH4XqfhjSQQ==", + "version": "4.4.6", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-4.4.6.tgz", + "integrity": "sha512-bR6Te2/feLMCLiyEFqnZpM5AqkXlBGr7A5jHQn+WSVLk0X53NvPq1KF8jh5TJIv56RzDv3oKEW3t0DysRNLZdw==", "license": "Apache-2.0", "dependencies": { - "@smithy/querystring-parser": "^4.2.14", - "@smithy/types": "^4.14.1", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6255,13 +5708,12 @@ } }, "node_modules/@smithy/util-base64": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-4.3.2.tgz", - "integrity": "sha512-XRH6b0H/5A3SgblmMa5ErXQ2XKhfbQB+Fm/oyLZ2O2kCUrwgg55bU0RekmzAhuwOjA9qdN5VU2BprOvGGUkOOQ==", + "version": "4.5.6", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-4.5.6.tgz", + "integrity": "sha512-N3hAANIKhtewRYICJjHscGtqQmAWISV+6Re5kbh8mOMiGKn6CvM252zocDdh9YWCmE/x9S1XAbe+5lzw+zjEZQ==", "license": "Apache-2.0", "dependencies": { - "@smithy/util-buffer-from": "^4.2.2", - "@smithy/util-utf8": "^4.2.2", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6269,11 +5721,12 @@ } }, "node_modules/@smithy/util-body-length-browser": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-4.2.2.tgz", - "integrity": "sha512-JKCrLNOup3OOgmzeaKQwi4ZCTWlYR5H4Gm1r2uTMVBXoemo1UEghk5vtMi1xSu2ymgKVGW631e2fp9/R610ZjQ==", + "version": "4.4.6", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-4.4.6.tgz", + "integrity": "sha512-oCVUok1wVnObl1nvtuj3DrVDtsgFNSmvVFc5VstnXKusqGKycKKPTtQHdlUMCklpB4g8pOp0jUOt7l7aAJlksg==", "license": "Apache-2.0", "dependencies": { + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6281,11 +5734,12 @@ } }, "node_modules/@smithy/util-body-length-node": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-4.2.3.tgz", - "integrity": "sha512-ZkJGvqBzMHVHE7r/hcuCxlTY8pQr1kMtdsVPs7ex4mMU+EAbcXppfo5NmyxMYi2XU49eqaz56j2gsk4dHHPG/g==", + "version": "4.4.6", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-4.4.6.tgz", + "integrity": "sha512-hNk9goWxXF3BGXT7yIyyzG29gYvtqxk55cn7wyZHRsWDWEAA1ZN9E+tLFLpKEkEPVVqprCqLmHAmnmPc/556sg==", "license": "Apache-2.0", "dependencies": { + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6293,39 +5747,25 @@ } }, "node_modules/@smithy/util-buffer-from": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-4.2.2.tgz", - "integrity": "sha512-FDXD7cvUoFWwN6vtQfEta540Y/YBe5JneK3SoZg9bThSoOAC/eGeYEua6RkBgKjGa/sz6Y+DuBZj3+YEY21y4Q==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/is-array-buffer": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-config-provider": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-4.2.2.tgz", - "integrity": "sha512-dWU03V3XUprJwaUIFVv4iOnS1FC9HnMHDfUrlNDSh4315v0cWyaIErP8KiqGVbf5z+JupoVpNM7ZB3jFiTejvQ==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", "license": "Apache-2.0", "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", "tslib": "^2.6.2" }, "engines": { - "node": ">=18.0.0" + "node": ">=14.0.0" } }, "node_modules/@smithy/util-defaults-mode-browser": { - "version": "4.3.49", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.3.49.tgz", - "integrity": "sha512-a5bNrdiONYB/qE2BuKegvUMd/+ZDwdg4vsNuuSzYE8qs2EYAdK9CynL+Rzn29PbPiUqoz/cbpRbcLzD5lEevHw==", + "version": "4.5.6", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-4.5.6.tgz", + "integrity": "sha512-lsApK6dpDK9LY49GMHEsNQV2XpT/Y9WxKy8CmfbN6oLqPMCXlCYVLeIhedtdJmgbi0Mg09ZIjbhYVEg+V/wK0Q==", "license": "Apache-2.0", "dependencies": { - "@smithy/property-provider": "^4.2.14", - "@smithy/smithy-client": "^4.12.13", - "@smithy/types": "^4.14.1", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6333,17 +5773,12 @@ } }, "node_modules/@smithy/util-defaults-mode-node": { - "version": "4.2.54", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.2.54.tgz", - "integrity": "sha512-g1cvrJvOnzeJgEdf7AE4luI7gp6L8weE0y9a9wQUSGtjb8QRHDbCJYuE4Sy0SD9N8RrnNPFsPltAz/OSoBR9Zw==", + "version": "4.4.6", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-4.4.6.tgz", + "integrity": "sha512-lKt5zA8YVx5SXanQ1t1iVZuaCuQcoJxUEh9BEpW64y5qKo3asCua7PiYsKkCv2RctwWZQqeEquB6WMostXBI4w==", "license": "Apache-2.0", "dependencies": { - "@smithy/config-resolver": "^4.4.17", - "@smithy/credential-provider-imds": "^4.2.14", - "@smithy/node-config-provider": "^4.3.14", - "@smithy/property-provider": "^4.2.14", - "@smithy/smithy-client": "^4.12.13", - "@smithy/types": "^4.14.1", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6351,25 +5786,12 @@ } }, "node_modules/@smithy/util-endpoints": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.4.2.tgz", - "integrity": "sha512-a55Tr+3OKld4TTtnT+RhKOQHyPxm3j/xL4OR83WBUhLJaKDS9dnJ7arRMOp3t31dcLhApwG9bgvrRXBHlLdIkg==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/node-config-provider": "^4.3.14", - "@smithy/types": "^4.14.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-hex-encoding": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-4.2.2.tgz", - "integrity": "sha512-Qcz3W5vuHK4sLQdyT93k/rfrUwdJ8/HZ+nMUOyGdpeGA1Wxt65zYwi3oEl9kOM+RswvYq90fzkNDahPS8K0OIg==", + "version": "3.6.6", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-3.6.6.tgz", + "integrity": "sha512-IIgVh4W8OwBP3WL17RROPs7EWkaRdygli4I3vnhEK+zkVvMFagoKN7KbDCitTafQ7GSM3ev0ZawYehN+cFzd+A==", "license": "Apache-2.0", "dependencies": { + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6377,12 +5799,12 @@ } }, "node_modules/@smithy/util-middleware": { - "version": "4.2.14", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.2.14.tgz", - "integrity": "sha512-1Su2vj9RYNDEv/V+2E+jXkkwGsgR7dc4sfHn9Z7ruzQHJIEni9zzw5CauvRXlFJfmgcqYP8fWa0dkh2Q2YaQyw==", + "version": "4.4.6", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-4.4.6.tgz", + "integrity": "sha512-13aBkHIs4auNrqDwGE40rbkF/gXZQ/qNtufQKDHxYjc1juSAzY0ol0Kbv8Md5oaUovnNpTp2cXLQXZn3eAEnlA==", "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.14.1", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6390,32 +5812,12 @@ } }, "node_modules/@smithy/util-retry": { - "version": "4.3.8", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.3.8.tgz", - "integrity": "sha512-LUIxbTBi+OpvXpg91poGA6BdyoleMDLnfXjVDqyi2RvZmTveY5loE/FgYUBCR5LU2BThW2SoZRh8dTIIy38IPw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/service-error-classification": "^4.3.1", - "@smithy/types": "^4.14.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/util-stream": { - "version": "4.5.25", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-4.5.25.tgz", - "integrity": "sha512-/PFpG4k8Ze8Ei+mMKj3oiPICYekthuzePZMgZbCqMiXIHHf4n2aZ4Ps0aSRShycFTGuj/J6XldmC0x0DwednIA==", + "version": "4.5.6", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-4.5.6.tgz", + "integrity": "sha512-o7QZYe6U1HV5c9Oqb/mwf01YL+8NJ1wTWjGNSyA3fuaOaiC0U2UhmAY320dcBZlKOJU8O60QNfr8RWqE+OnU/w==", "license": "Apache-2.0", "dependencies": { - "@smithy/fetch-http-handler": "^5.3.17", - "@smithy/node-http-handler": "^4.6.1", - "@smithy/types": "^4.14.1", - "@smithy/util-base64": "^4.3.2", - "@smithy/util-buffer-from": "^4.2.2", - "@smithy/util-hex-encoding": "^4.2.2", - "@smithy/util-utf8": "^4.2.2", + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6423,24 +5825,12 @@ } }, "node_modules/@smithy/util-utf8": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-4.2.2.tgz", - "integrity": "sha512-75MeYpjdWRe8M5E3AW0O4Cx3UadweS+cwdXjwYGBW5h/gxxnbeZ877sLPX/ZJA9GVTlL/qG0dXP29JWFCD1Ayw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/util-buffer-from": "^4.2.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/uuid": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@smithy/uuid/-/uuid-1.1.2.tgz", - "integrity": "sha512-O/IEdcCUKkubz60tFbGA7ceITTAJsty+lBjNoorP4Z6XRqaFb/OjQjZODophEcuq68nKm6/0r+6/lLQ+XVpk8g==", + "version": "4.4.6", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-4.4.6.tgz", + "integrity": "sha512-gncTZwzB/RTzm29VvK1nZhCHdPjBkR8pNaFtUMbQpkG6vFMjPHe/RsnmMXfrYj8Qs5s6QH5f1Mp9CBXFOHxqlQ==", "license": "Apache-2.0", "dependencies": { + "@smithy/core": "^3.29.1", "tslib": "^2.6.2" }, "engines": { @@ -6474,9 +5864,9 @@ } }, "node_modules/@sveltejs/kit": { - "version": "2.65.2", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.65.2.tgz", - "integrity": "sha512-ZIkyEmxT1gcq50Opn1ZIIx6vc/yt2zNN0rF5hS6op95gqHtNw8QMKDhjJI+RyjMcbvECRw+FzEeAoBe/MOz9AA==", + "version": "2.69.1", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.69.1.tgz", + "integrity": "sha512-+nz8Fx/cElzb2ZPHC+6Ll3y3NEVIe4Na75PeplLlyTmd1dBXAjz2KR14y1ZgNjb2ThfAYzulu+PFy1UE3RCUzA==", "devOptional": true, "license": "MIT", "dependencies": { @@ -6515,20 +5905,10 @@ } } }, - "node_modules/@sveltejs/kit/node_modules/cookie": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.0.2.tgz", - "integrity": "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==", - "devOptional": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, "node_modules/@sveltejs/load-config": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@sveltejs/load-config/-/load-config-0.1.1.tgz", - "integrity": "sha512-BXXm+VOH/9X4N7Dd1iZ2MqA1h7M+9i2noI8QYuLDY8QcN2WHYn7D/VK/+IJNfcAmRw7ACNJ538UT9GXIhnBTiA==", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@sveltejs/load-config/-/load-config-0.2.0.tgz", + "integrity": "sha512-1LgZ/qUqSoq+QorD83lk2hka79Px0wXNW2q5V1nZlxGhQgw1jrsIbVz5YiCeucVLo4XvFLjXukUaQjIiqowkcg==", "dev": true, "license": "MIT", "engines": { @@ -6556,9 +5936,9 @@ } }, "node_modules/@swc/helpers": { - "version": "0.5.21", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.21.tgz", - "integrity": "sha512-jI/VAmtdjB/RnI8GTnokyX7Ug8c+g+ffD6QRLa6XQewtnGyukKkKSk3wLTM3b5cjt1jNh9x0jfVlagdN2gDKQg==", + "version": "0.5.23", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.23.tgz", + "integrity": "sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==", "license": "Apache-2.0", "peer": true, "dependencies": { @@ -6566,9 +5946,9 @@ } }, "node_modules/@tailwindcss/node": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.3.1.tgz", - "integrity": "sha512-6NDaqRoAMSXD1mr/RXu0HBvNE9a2n5tHPsxu9XHLws8o4Twes5rBM2205SUUiJ9goAtadrN6xTGX0UDEwp/N4A==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.3.2.tgz", + "integrity": "sha512-yWP/sqEcBLaD8JuA6zNwxoYKr75qxTioYwlRwekj5Jr/I5GXnoJfjetH/psLUIv74cYTH2lBUEzBkinthoYcBg==", "dev": true, "license": "MIT", "dependencies": { @@ -6578,37 +5958,37 @@ "lightningcss": "1.32.0", "magic-string": "^0.30.21", "source-map-js": "^1.2.1", - "tailwindcss": "4.3.1" + "tailwindcss": "4.3.2" } }, "node_modules/@tailwindcss/oxide": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.3.1.tgz", - "integrity": "sha512-yVPyo8RNkabVr3O2EhHEE0Rewu7YKzc1DhIqfL46LKveFrmu9XbDazNOJY7/GRuvw1h6u3utWnR29H/p5JPlgA==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.3.2.tgz", + "integrity": "sha512-z8ZgnzX8gdNoWLBLqBPoh/sjnxkwvf9ZuWjnO0l0yIzbLa5/9S+eC5QxGZKRobVHIC3/1BoMWjHblqWjcgFgag==", "dev": true, "license": "MIT", "engines": { "node": ">= 20" }, "optionalDependencies": { - "@tailwindcss/oxide-android-arm64": "4.3.1", - "@tailwindcss/oxide-darwin-arm64": "4.3.1", - "@tailwindcss/oxide-darwin-x64": "4.3.1", - "@tailwindcss/oxide-freebsd-x64": "4.3.1", - "@tailwindcss/oxide-linux-arm-gnueabihf": "4.3.1", - "@tailwindcss/oxide-linux-arm64-gnu": "4.3.1", - "@tailwindcss/oxide-linux-arm64-musl": "4.3.1", - "@tailwindcss/oxide-linux-x64-gnu": "4.3.1", - "@tailwindcss/oxide-linux-x64-musl": "4.3.1", - "@tailwindcss/oxide-wasm32-wasi": "4.3.1", - "@tailwindcss/oxide-win32-arm64-msvc": "4.3.1", - "@tailwindcss/oxide-win32-x64-msvc": "4.3.1" + "@tailwindcss/oxide-android-arm64": "4.3.2", + "@tailwindcss/oxide-darwin-arm64": "4.3.2", + "@tailwindcss/oxide-darwin-x64": "4.3.2", + "@tailwindcss/oxide-freebsd-x64": "4.3.2", + "@tailwindcss/oxide-linux-arm-gnueabihf": "4.3.2", + "@tailwindcss/oxide-linux-arm64-gnu": "4.3.2", + "@tailwindcss/oxide-linux-arm64-musl": "4.3.2", + "@tailwindcss/oxide-linux-x64-gnu": "4.3.2", + "@tailwindcss/oxide-linux-x64-musl": "4.3.2", + "@tailwindcss/oxide-wasm32-wasi": "4.3.2", + "@tailwindcss/oxide-win32-arm64-msvc": "4.3.2", + "@tailwindcss/oxide-win32-x64-msvc": "4.3.2" } }, "node_modules/@tailwindcss/oxide-android-arm64": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.3.1.tgz", - "integrity": "sha512-SVlyf61g374l5cHyg8x9kf5xmLcOaxvOTsbsqDnSsDJaKOEFZ7GCvi84VAVGpxojYOs1+3K6M0UjXfqPU8vmOQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.3.2.tgz", + "integrity": "sha512-WHxqIuHpvZ5VtdX6GTl1Ik/Vp2YuN42Et+0CdeaVd/frQ9jAvGmvR8vLT+jk3e8/Q3x8kECB9+R17pgpp2BulA==", "cpu": [ "arm64" ], @@ -6623,9 +6003,9 @@ } }, "node_modules/@tailwindcss/oxide-darwin-arm64": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.3.1.tgz", - "integrity": "sha512-hVnWLwv+e/l7c4WKyVtHVrIPvYdqWHjRB3MDIqARynzFtnQg85kmQEFCbV9Ja0VVx4xXTIiDWY60Y7iz/iNoDA==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.3.2.tgz", + "integrity": "sha512-GZypeUY/IDJW3877KeM+O67vbXr3MBnbtEL4aYhNErv/JWZhye2vGSWWG9tB6iiqR2MqRNkY8IOUy4NdSZV26w==", "cpu": [ "arm64" ], @@ -6640,9 +6020,9 @@ } }, "node_modules/@tailwindcss/oxide-darwin-x64": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.3.1.tgz", - "integrity": "sha512-Cf7abu0WVgbhU7ANgPUnSAvm7nCvMweusHb8FnaHlLfv/Caq4GYaEZg7ZImzzmjx4lIAfuS8q+eLIS7A7IzxIg==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.3.2.tgz", + "integrity": "sha512-UIIzmefR6KO1sDU7MzRqAxC8iBpft/VhkGjTjnhoS6k7Z3rQ9wEgA1ODSiyH/tcSYssulNm4Ci3hOeK1jH7ccQ==", "cpu": [ "x64" ], @@ -6657,9 +6037,9 @@ } }, "node_modules/@tailwindcss/oxide-freebsd-x64": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.3.1.tgz", - "integrity": "sha512-ZZqzX2Y+GXtXXfqSfpJhDm60OoZfvLHLCgm+J7NVqgHHJjG/m9ugZI77RwTsVd4fnBJuCFP6Ae6kTJb71UdS8g==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.3.2.tgz", + "integrity": "sha512-GN+uAmcI6DNspnCDwtOAZrTz6oukJnp337qZvxqCGLd3BHBzJpO0ZbTLRvJNdztOeAmTzewewGIMPb0tk2R4WA==", "cpu": [ "x64" ], @@ -6674,9 +6054,9 @@ } }, "node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.3.1.tgz", - "integrity": "sha512-/Ah/xik0LaMYfv9DZ0S/t4pBlBNYOcqtRwusjgovHkvT8ixueWCLyJjsaF5kQIckjb4IT8Q6K6p/iPmZMixYgg==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.3.2.tgz", + "integrity": "sha512-4ABn7qSbdHRwTiDiuWNegCyb5+2FJ4vKIKc3DmKrvAFw7MU1Lm11dIkTPwUaFdTzc7IsOpDbqBrlh0x6y36U/w==", "cpu": [ "arm" ], @@ -6691,13 +6071,16 @@ } }, "node_modules/@tailwindcss/oxide-linux-arm64-gnu": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.3.1.tgz", - "integrity": "sha512-gqdFoVJlw444GvpnheZLHmvTzSxI/cOUUh2KSNejQjTcYkW062SVD+En0rUgD+QV91bz1XGIGtt1HJd48xUGbQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.3.2.tgz", + "integrity": "sha512-wDgEIGwoM8w8pufh9LVt1PahDgNdKXrLC2qfAnV3vAmococ9RWbxeAw4pxPttd/TsJfwjyLf90Dg1y9y8I6Emw==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -6708,13 +6091,16 @@ } }, "node_modules/@tailwindcss/oxide-linux-arm64-musl": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.3.1.tgz", - "integrity": "sha512-Bwv9KwOvE0VKa86xPFif9b9c3Y1NxOV1P0gLti/IYaWEsQYZXDlxfGEtA8mdDZ7SG3wyNXAWYT5SIn3giL57oA==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.3.2.tgz", + "integrity": "sha512-J5Nuk0uZQIiMTJj3LEx4sAA9tMFUoXQZFv1J6An+QGYe53HKRJuFDi0rpq/tuouCZeAbOBY3kQ6g8qeD4TUjtA==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -6725,13 +6111,16 @@ } }, "node_modules/@tailwindcss/oxide-linux-x64-gnu": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.3.1.tgz", - "integrity": "sha512-Ymi8O8T15HYQdOUWUtTI6ldN0neHP85FC+Qz32xTcZ7iJXtem/x8ITev0o1e9e5rkqj4lONZfTRLvkmin1+tKg==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.3.2.tgz", + "integrity": "sha512-kqCZpSKOBEJO4mz7OqWoofBZeXTAwaVGPj0ErAj7CojmhKpWVWVOnrt9dE8odoIraZq4oj3ausM37kXi+Tow8w==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ @@ -6742,13 +6131,16 @@ } }, "node_modules/@tailwindcss/oxide-linux-x64-musl": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.3.1.tgz", - "integrity": "sha512-M+P/91qJ6uILLw4k2G93GMDRAXj61SMvFQYt39AqvUqYgExXpLL5aepfns7sj4HiAQeolirQF9E0lzRvdf4zPQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.3.2.tgz", + "integrity": "sha512-cixpqbh2toJDmkuCRI68nXA8ZxNmdK9Y+9v5h3MC3ZQKy/0BO8AWzlkWyRM7JAFSGBlfig4YVTPsK6MVgqz1uw==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ @@ -6759,9 +6151,9 @@ } }, "node_modules/@tailwindcss/oxide-wasm32-wasi": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.3.1.tgz", - "integrity": "sha512-zsM8uOeqvVGHsAXsJxsT28ttosFahLJKCLOTUBqRAtKnVgGSRitds9T432QiT8b77Yga7JIBkulIRRlJPtYhRA==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.3.2.tgz", + "integrity": "sha512-4ec2Z/LOmRsAgU23CS4xeJfcJlmRg94A/XrbGRCF1gyU/zdDfRLYDVsS+ynSZCmGNxQ1jQriQOKMQeQxBA3Isw==", "bundleDependencies": [ "@napi-rs/wasm-runtime", "@emnapi/core", @@ -6777,9 +6169,9 @@ "license": "MIT", "optional": true, "dependencies": { - "@emnapi/core": "^1.10.0", - "@emnapi/runtime": "^1.10.0", - "@emnapi/wasi-threads": "^1.2.1", + "@emnapi/core": "^1.11.1", + "@emnapi/runtime": "^1.11.1", + "@emnapi/wasi-threads": "^1.2.2", "@napi-rs/wasm-runtime": "^1.1.4", "@tybys/wasm-util": "^0.10.2", "tslib": "^2.8.1" @@ -6789,9 +6181,9 @@ } }, "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.3.1.tgz", - "integrity": "sha512-aiNvSq9BsVk8V513lDKlrCFAgf8qBMPZTpgEhInL+NwQqs97mYmupVMrPrgBBSL8Pv/0zXu9MrMF9rMun1ZeNg==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.3.2.tgz", + "integrity": "sha512-Zyr/M0+XcYZu3bZrUytc7TXvrk0ftWfl8gN2MwekNDzhqhKRUucMPSeOzM0o0wH5AWOU49BsKRrfKxI2atCPMQ==", "cpu": [ "arm64" ], @@ -6806,9 +6198,9 @@ } }, "node_modules/@tailwindcss/oxide-win32-x64-msvc": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.3.1.tgz", - "integrity": "sha512-xDEyu1rg290472FEGaKHnzyDyh5QH+AlWvsU5hMoMtPpzmKlRI0jaYKCgSHDYtaQWZOYbMaduSyCwFwY4n1HmA==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.3.2.tgz", + "integrity": "sha512-QI9BO7KlNZsp2GuO0jwAAj5jCDABOKXRkCk2XuKTSaNEFSdfzqswYVTtCHBNKHLsqyjFyFkqlDiwkNbTYSssMQ==", "cpu": [ "x64" ], @@ -6823,15 +6215,15 @@ } }, "node_modules/@tailwindcss/vite": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/@tailwindcss/vite/-/vite-4.3.1.tgz", - "integrity": "sha512-hItDHuIIlEV61R+faXu66s1K36aTurO/Qw0e45Vskz57gXl9pWOT6eg3zmcEui6CZXddbN7zd41bwmvag4JGwQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@tailwindcss/vite/-/vite-4.3.2.tgz", + "integrity": "sha512-eHpMeX4JXfVNJDEcsouTeCBubJBTcTLigeaw/NTUW6PB5ATKKXdyonnXgTBX2VuRbjz1hjfz6C5XAhr52ImQXA==", "dev": true, "license": "MIT", "dependencies": { - "@tailwindcss/node": "4.3.1", - "@tailwindcss/oxide": "4.3.1", - "tailwindcss": "4.3.1" + "@tailwindcss/node": "4.3.2", + "@tailwindcss/oxide": "4.3.2", + "tailwindcss": "4.3.2" }, "peerDependencies": { "vite": "^5.2.0 || ^6 || ^7 || ^8" @@ -6895,14 +6287,14 @@ } }, "node_modules/@testing-library/svelte": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/@testing-library/svelte/-/svelte-5.3.1.tgz", - "integrity": "sha512-8Ez7ZOqW5geRf9PF5rkuopODe5RGy3I9XR+kc7zHh26gBiktLaxTfKmhlGaSHYUOTQE7wFsLMN9xCJVCszw47w==", + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/@testing-library/svelte/-/svelte-5.4.2.tgz", + "integrity": "sha512-4o31E4HGo5BU5KwPkulNRocEden+7Tt9JYm9uhln5ajF7DULeyFA46BBWVfKJ8Ms9B3JmOFPTIiVamH7n3KpuQ==", "dev": true, "license": "MIT", "dependencies": { "@testing-library/dom": "9.x.x || 10.x.x", - "@testing-library/svelte-core": "1.0.0" + "@testing-library/svelte-core": "1.1.3" }, "engines": { "node": ">= 10" @@ -6922,9 +6314,9 @@ } }, "node_modules/@testing-library/svelte-core": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@testing-library/svelte-core/-/svelte-core-1.0.0.tgz", - "integrity": "sha512-VkUePoLV6oOYwSUvX6ShA8KLnJqZiYMIbP2JW2t0GLWLkJxKGvuH5qrrZBV/X7cXFnLGuFQEC7RheYiZOW68KQ==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@testing-library/svelte-core/-/svelte-core-1.1.3.tgz", + "integrity": "sha512-KkMAvXeWorxN2Yn0kdC1lfoAItxpoj4uOWzxK5leDrNxonLvS5nwBFvztrroyTszQ0Wf/EU6iLT8JhY5qcn22g==", "dev": true, "license": "MIT", "engines": { @@ -6935,9 +6327,9 @@ } }, "node_modules/@tybys/wasm-util": { - "version": "0.10.2", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", - "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", + "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", "license": "MIT", "optional": true, "dependencies": { @@ -6977,9 +6369,9 @@ "license": "MIT" }, "node_modules/@types/estree": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", - "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", "license": "MIT" }, "node_modules/@types/trusted-types": { @@ -7133,9 +6525,9 @@ } }, "node_modules/acorn": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", - "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", + "version": "8.17.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.17.0.tgz", + "integrity": "sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==", "license": "MIT", "bin": { "acorn": "bin/acorn" @@ -7167,18 +6559,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/anynum": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.0.tgz", - "integrity": "sha512-xjR9/zBVnUOP6ztMIIgShjsxui80nQUQH+5xJnvrYLs+90bF25/KJqaAi8mk+B4RDtX1Nspi6fmp4YTEts8SfA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT" - }, "node_modules/aria-query": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.2.tgz", @@ -7200,9 +6580,9 @@ } }, "node_modules/ast-v8-to-istanbul": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-1.0.0.tgz", - "integrity": "sha512-1fSfIwuDICFA4LKkCzRPO7F0hzFf0B7+Xqrl27ynQaa+Rh0e1Es0v6kWHPott3lU10AyAr7oKHa65OppjLn3Rg==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-1.0.4.tgz", + "integrity": "sha512-0bC0/4bTSrnwdhU3IsZDwEdojvuPrSg59OYZfKsLRtJZ0u8VBx9DebfqqG8bRdCC0I7vjgxmPi41P0lpkhJHtA==", "dev": true, "license": "MIT", "dependencies": { @@ -7321,6 +6701,16 @@ "dev": true, "license": "MIT" }, + "node_modules/cookie": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.0.2.tgz", + "integrity": "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==", + "devOptional": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/css-tree": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", @@ -7433,9 +6823,9 @@ } }, "node_modules/es-module-lexer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.0.0.tgz", - "integrity": "sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.0.tgz", + "integrity": "sha512-KLdwQm2NvGLDkQDCGvmiQrhkd0JbMzXthwQAUgWjQuQdBLFa3eiBP5arXZyA+f8x+x7OXgud6bq2rxjGtHV2tw==", "dev": true, "license": "MIT" }, @@ -7446,9 +6836,9 @@ "license": "MIT" }, "node_modules/esrap": { - "version": "2.2.11", - "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.2.11.tgz", - "integrity": "sha512-gPdx+I+BjYEinNMQaBXFjbaJVyoPMU4ZODg5mE+M4DqVG9VusAVHHjcBX+zqyITlI0DIARwDMMzZwAWj36dRoQ==", + "version": "2.2.13", + "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.2.13.tgz", + "integrity": "sha512-m8jH5hZgJE2RRUK/jjkGPcJEDAV+dYnZYFkosQaPTcE+Yw4xynXHOo6FUdwaWBtdR3b1MMa7wEDTSHeR2VWsGA==", "license": "MIT", "dependencies": { "@jridgewell/sourcemap-codec": "^1.4.15" @@ -7473,52 +6863,15 @@ } }, "node_modules/expect-type": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", - "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", "dev": true, "license": "Apache-2.0", "engines": { "node": ">=12.0.0" } }, - "node_modules/fast-xml-builder": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.2.0.tgz", - "integrity": "sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "path-expression-matcher": "^1.5.0", - "xml-naming": "^0.1.0" - } - }, - "node_modules/fast-xml-parser": { - "version": "5.7.3", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.7.3.tgz", - "integrity": "sha512-C0AaNuC+mscy6vrAQKAc/rMq+zAPHodfHGZu4sGVehvAQt/JLG1O5zEcYcXSY5zSqr4YVgxsB+pHXTq0i7eDlg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "@nodable/entities": "^2.1.0", - "fast-xml-builder": "^1.1.7", - "path-expression-matcher": "^1.5.0", - "strnum": "^2.2.3" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -7870,6 +7223,9 @@ "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -7890,6 +7246,9 @@ "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -7910,6 +7269,9 @@ "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -7930,6 +7292,9 @@ "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "MPL-2.0", "optional": true, "os": [ @@ -7990,9 +7355,9 @@ "license": "MIT" }, "node_modules/lru-cache": { - "version": "11.3.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.5.tgz", - "integrity": "sha512-NxVFwLAnrd9i7KUBxC4DrUhmgjzOs+1Qm50D3oF1/oL+r1NpZ4gA7xvG0/zJ8evR7zIKn4vLf7qTNduWFtCrRw==", + "version": "11.5.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.1.tgz", + "integrity": "sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A==", "dev": true, "license": "BlueOak-1.0.0", "engines": { @@ -8003,6 +7368,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/lucide-svelte/-/lucide-svelte-1.0.1.tgz", "integrity": "sha512-WvzZgk0pqzgda+AErLvgWxHkfg/+GgUwqKMRHvzt0IqyMdmyEDzDCk3Z+Wo/3y753oIgx8u9Q4eUbWkghFa8Jg==", + "deprecated": "Package deprecated. Please use @lucide/svelte instead.", "license": "ISC", "peerDependencies": { "svelte": "^3 || ^4 || ^5.0.0-next.42" @@ -8027,13 +7393,13 @@ } }, "node_modules/magicast": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.2.tgz", - "integrity": "sha512-E3ZJh4J3S9KfwdjZhe2afj6R9lGIN5Pher1pF39UGrXRqq/VDaGVIGN13BjHd2u8B61hArAGOnso7nBOouW3TQ==", + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.3.tgz", + "integrity": "sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/parser": "^7.29.0", + "@babel/parser": "^7.29.3", "@babel/types": "^7.29.0", "source-map-js": "^1.2.1" } @@ -8101,9 +7467,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.12", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", - "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", + "version": "3.3.15", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", + "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", "devOptional": true, "funding": [ { @@ -8126,20 +7492,23 @@ "license": "MIT" }, "node_modules/obug": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", - "integrity": "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.3.tgz", + "integrity": "sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==", "devOptional": true, "funding": [ "https://github.com/sponsors/sxzz", "https://opencollective.com/debug" ], - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } }, "node_modules/oxfmt": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/oxfmt/-/oxfmt-0.55.0.tgz", - "integrity": "sha512-jSj2wCTakwgPMxkfiVZX0jf+nX+Nz6xlyAZjqNE0qXTFdCBPYlP6JAN+ODjmealw7DXBjOzYbdsqwBMAZnPZ6A==", + "version": "0.57.0", + "resolved": "https://registry.npmjs.org/oxfmt/-/oxfmt-0.57.0.tgz", + "integrity": "sha512-ZB7Bi+rGDSqmVIo9jwcLyFgjxXvQhDdU+jx+ZrVy6VRiVXK2+CHc4hO3J4dUQjHe7V0ymHB+MDuv5z+NhK07HA==", "dev": true, "license": "MIT", "dependencies": { @@ -8155,25 +7524,25 @@ "url": "https://github.com/sponsors/Boshen" }, "optionalDependencies": { - "@oxfmt/binding-android-arm-eabi": "0.55.0", - "@oxfmt/binding-android-arm64": "0.55.0", - "@oxfmt/binding-darwin-arm64": "0.55.0", - "@oxfmt/binding-darwin-x64": "0.55.0", - "@oxfmt/binding-freebsd-x64": "0.55.0", - "@oxfmt/binding-linux-arm-gnueabihf": "0.55.0", - "@oxfmt/binding-linux-arm-musleabihf": "0.55.0", - "@oxfmt/binding-linux-arm64-gnu": "0.55.0", - "@oxfmt/binding-linux-arm64-musl": "0.55.0", - "@oxfmt/binding-linux-ppc64-gnu": "0.55.0", - "@oxfmt/binding-linux-riscv64-gnu": "0.55.0", - "@oxfmt/binding-linux-riscv64-musl": "0.55.0", - "@oxfmt/binding-linux-s390x-gnu": "0.55.0", - "@oxfmt/binding-linux-x64-gnu": "0.55.0", - "@oxfmt/binding-linux-x64-musl": "0.55.0", - "@oxfmt/binding-openharmony-arm64": "0.55.0", - "@oxfmt/binding-win32-arm64-msvc": "0.55.0", - "@oxfmt/binding-win32-ia32-msvc": "0.55.0", - "@oxfmt/binding-win32-x64-msvc": "0.55.0" + "@oxfmt/binding-android-arm-eabi": "0.57.0", + "@oxfmt/binding-android-arm64": "0.57.0", + "@oxfmt/binding-darwin-arm64": "0.57.0", + "@oxfmt/binding-darwin-x64": "0.57.0", + "@oxfmt/binding-freebsd-x64": "0.57.0", + "@oxfmt/binding-linux-arm-gnueabihf": "0.57.0", + "@oxfmt/binding-linux-arm-musleabihf": "0.57.0", + "@oxfmt/binding-linux-arm64-gnu": "0.57.0", + "@oxfmt/binding-linux-arm64-musl": "0.57.0", + "@oxfmt/binding-linux-ppc64-gnu": "0.57.0", + "@oxfmt/binding-linux-riscv64-gnu": "0.57.0", + "@oxfmt/binding-linux-riscv64-musl": "0.57.0", + "@oxfmt/binding-linux-s390x-gnu": "0.57.0", + "@oxfmt/binding-linux-x64-gnu": "0.57.0", + "@oxfmt/binding-linux-x64-musl": "0.57.0", + "@oxfmt/binding-openharmony-arm64": "0.57.0", + "@oxfmt/binding-win32-arm64-msvc": "0.57.0", + "@oxfmt/binding-win32-ia32-msvc": "0.57.0", + "@oxfmt/binding-win32-x64-msvc": "0.57.0" }, "peerDependencies": { "svelte": "^5.0.0", @@ -8189,9 +7558,9 @@ } }, "node_modules/oxlint": { - "version": "1.70.0", - "resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.70.0.tgz", - "integrity": "sha512-D6JgHtzkhRwvEC+A0Nw5AEc5bk8x5i1pHzvZIEf/a0C4hOzmAACNGtkDGPyFaxxX3ZVGxCPeig3P3rMM8XU3/g==", + "version": "1.72.0", + "resolved": "https://registry.npmjs.org/oxlint/-/oxlint-1.72.0.tgz", + "integrity": "sha512-1rhdZIP/EvoI91ABIwNU5Q8+bWf8mjrS5UzIOZld4d4bXxJvtlUhlQvaoTogIGin/qdErMOrwaIJvCSIAKTLhA==", "dev": true, "license": "MIT", "bin": { @@ -8204,25 +7573,25 @@ "url": "https://github.com/sponsors/Boshen" }, "optionalDependencies": { - "@oxlint/binding-android-arm-eabi": "1.70.0", - "@oxlint/binding-android-arm64": "1.70.0", - "@oxlint/binding-darwin-arm64": "1.70.0", - "@oxlint/binding-darwin-x64": "1.70.0", - "@oxlint/binding-freebsd-x64": "1.70.0", - "@oxlint/binding-linux-arm-gnueabihf": "1.70.0", - "@oxlint/binding-linux-arm-musleabihf": "1.70.0", - "@oxlint/binding-linux-arm64-gnu": "1.70.0", - "@oxlint/binding-linux-arm64-musl": "1.70.0", - "@oxlint/binding-linux-ppc64-gnu": "1.70.0", - "@oxlint/binding-linux-riscv64-gnu": "1.70.0", - "@oxlint/binding-linux-riscv64-musl": "1.70.0", - "@oxlint/binding-linux-s390x-gnu": "1.70.0", - "@oxlint/binding-linux-x64-gnu": "1.70.0", - "@oxlint/binding-linux-x64-musl": "1.70.0", - "@oxlint/binding-openharmony-arm64": "1.70.0", - "@oxlint/binding-win32-arm64-msvc": "1.70.0", - "@oxlint/binding-win32-ia32-msvc": "1.70.0", - "@oxlint/binding-win32-x64-msvc": "1.70.0" + "@oxlint/binding-android-arm-eabi": "1.72.0", + "@oxlint/binding-android-arm64": "1.72.0", + "@oxlint/binding-darwin-arm64": "1.72.0", + "@oxlint/binding-darwin-x64": "1.72.0", + "@oxlint/binding-freebsd-x64": "1.72.0", + "@oxlint/binding-linux-arm-gnueabihf": "1.72.0", + "@oxlint/binding-linux-arm-musleabihf": "1.72.0", + "@oxlint/binding-linux-arm64-gnu": "1.72.0", + "@oxlint/binding-linux-arm64-musl": "1.72.0", + "@oxlint/binding-linux-ppc64-gnu": "1.72.0", + "@oxlint/binding-linux-riscv64-gnu": "1.72.0", + "@oxlint/binding-linux-riscv64-musl": "1.72.0", + "@oxlint/binding-linux-s390x-gnu": "1.72.0", + "@oxlint/binding-linux-x64-gnu": "1.72.0", + "@oxlint/binding-linux-x64-musl": "1.72.0", + "@oxlint/binding-openharmony-arm64": "1.72.0", + "@oxlint/binding-win32-arm64-msvc": "1.72.0", + "@oxlint/binding-win32-ia32-msvc": "1.72.0", + "@oxlint/binding-win32-x64-msvc": "1.72.0" }, "peerDependencies": { "oxlint-tsgolint": ">=0.22.1", @@ -8250,21 +7619,6 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, - "node_modules/path-expression-matcher": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.5.0.tgz", - "integrity": "sha512-cbrerZV+6rvdQrrD+iGMcZFEiiSrbv9Tfdkvnusy6y0x0GKBXREFg/Y65GhIfm0tnLntThhzCnfKwp1WRjeCyQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -8280,9 +7634,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "devOptional": true, "license": "MIT", "engines": { @@ -8293,9 +7647,9 @@ } }, "node_modules/postcss": { - "version": "8.5.15", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", - "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "version": "8.5.16", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.16.tgz", + "integrity": "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==", "devOptional": true, "funding": [ { @@ -8392,13 +7746,13 @@ } }, "node_modules/rolldown": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.3.tgz", - "integrity": "sha512-i00lAJ2ks1BYr7rjNjKC7BcqAS7nVfiT3QX1SI5aY+AFHblCmaUf9OE9dbdzDvW6dJxbi2ZCZiy9v3CcwOiX3g==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.4.tgz", + "integrity": "sha512-IjZYiLxZwpnhwhdBH2ugdTGVSdhCQUmLxLoqyjiL0JxYjyRst+5a0P3xfrTxJ5F638j4Mvvw5FAX5XE6eHpXbA==", "devOptional": true, "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.133.0", + "@oxc-project/types": "=0.138.0", "@rolldown/pluginutils": "^1.0.0" }, "bin": { @@ -8408,21 +7762,21 @@ "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.0.3", - "@rolldown/binding-darwin-arm64": "1.0.3", - "@rolldown/binding-darwin-x64": "1.0.3", - "@rolldown/binding-freebsd-x64": "1.0.3", - "@rolldown/binding-linux-arm-gnueabihf": "1.0.3", - "@rolldown/binding-linux-arm64-gnu": "1.0.3", - "@rolldown/binding-linux-arm64-musl": "1.0.3", - "@rolldown/binding-linux-ppc64-gnu": "1.0.3", - "@rolldown/binding-linux-s390x-gnu": "1.0.3", - "@rolldown/binding-linux-x64-gnu": "1.0.3", - "@rolldown/binding-linux-x64-musl": "1.0.3", - "@rolldown/binding-openharmony-arm64": "1.0.3", - "@rolldown/binding-wasm32-wasi": "1.0.3", - "@rolldown/binding-win32-arm64-msvc": "1.0.3", - "@rolldown/binding-win32-x64-msvc": "1.0.3" + "@rolldown/binding-android-arm64": "1.1.4", + "@rolldown/binding-darwin-arm64": "1.1.4", + "@rolldown/binding-darwin-x64": "1.1.4", + "@rolldown/binding-freebsd-x64": "1.1.4", + "@rolldown/binding-linux-arm-gnueabihf": "1.1.4", + "@rolldown/binding-linux-arm64-gnu": "1.1.4", + "@rolldown/binding-linux-arm64-musl": "1.1.4", + "@rolldown/binding-linux-ppc64-gnu": "1.1.4", + "@rolldown/binding-linux-s390x-gnu": "1.1.4", + "@rolldown/binding-linux-x64-gnu": "1.1.4", + "@rolldown/binding-linux-x64-musl": "1.1.4", + "@rolldown/binding-openharmony-arm64": "1.1.4", + "@rolldown/binding-wasm32-wasi": "1.1.4", + "@rolldown/binding-win32-arm64-msvc": "1.1.4", + "@rolldown/binding-win32-x64-msvc": "1.1.4" } }, "node_modules/runed": { @@ -8476,9 +7830,9 @@ } }, "node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "dev": true, "license": "ISC", "bin": { @@ -8489,9 +7843,9 @@ } }, "node_modules/set-cookie-parser": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-3.1.0.tgz", - "integrity": "sha512-kjnC1DXBHcxaOaOXBHBeRtltsDG2nUiUni+jP92M9gYdW12rsmx92UsfpH7o5tDRs7I1ZZPSQJQGv3UaRfCiuw==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-3.1.1.tgz", + "integrity": "sha512-vM9SUhjsUYs6UeJUmygc5Ofm5eQGe85riob5ju6XCgFGJI5PLV4nrDAQpQjd+LkFBpAkADn5BQQpZ9EUNkyLuA==", "devOptional": true, "license": "MIT" }, @@ -8554,21 +7908,6 @@ "node": ">=8" } }, - "node_modules/strnum": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.0.tgz", - "integrity": "sha512-sHrVyWWdq28RbhjuJdZsA1SnGRJV6NiXbk6AXBxDOsgAcA+lmpUZCYjOdLBxkXMwis6RRe7dlZt4VlIWFVzkmg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "anynum": "^1.0.0" - } - }, "node_modules/style-to-object": { "version": "1.0.14", "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-1.0.14.tgz", @@ -8592,9 +7931,9 @@ } }, "node_modules/svelte": { - "version": "5.56.3", - "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.56.3.tgz", - "integrity": "sha512-w7JvrM5IFl5cmfbY0TLik9o7mjRUJmRMhOR51tBPu708Gr/MjbGs7VnJnr/B0CaXeI4vtnOh7RKxDr0cwhMdDA==", + "version": "5.56.4", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.56.4.tgz", + "integrity": "sha512-/d0QHehmRuJW8gVz395MTkPcPozxzdjBMBE8oEYGz8O3b9KTMzzQ9ZHJQLuFKOHOPQbU6kx/X4iid/EBBzH7iw==", "license": "MIT", "dependencies": { "@jridgewell/remapping": "^2.3.4", @@ -8608,7 +7947,7 @@ "clsx": "^2.1.1", "devalue": "^5.8.1", "esm-env": "^1.2.1", - "esrap": "^2.2.11", + "esrap": "^2.2.12", "is-reference": "^3.0.3", "locate-character": "^3.0.0", "magic-string": "^0.30.11", @@ -8619,14 +7958,14 @@ } }, "node_modules/svelte-check": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/svelte-check/-/svelte-check-4.6.0.tgz", - "integrity": "sha512-KhVnDFDSid57mmZtHz8gfW8AAGylOZ0vPnOIzVmAL+urzwK8sBYXRss953gD8T0OdgAQ11mdWhE6uadmtOz8TQ==", + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/svelte-check/-/svelte-check-4.7.1.tgz", + "integrity": "sha512-FGUOmAqxXdN/H9Zm8slrqO7SLtFisXRB7rfOsHNJ3MLTD2po/+Stg8XyErkpumPHbuUiYTcqrEIzxpVWKTLqtg==", "dev": true, "license": "MIT", "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", - "@sveltejs/load-config": "0.1.1", + "@sveltejs/load-config": "^0.2.0", "chokidar": "^4.0.1", "fdir": "^6.2.0", "picocolors": "^1.0.0", @@ -8708,9 +8047,9 @@ "license": "MIT" }, "node_modules/tabbable": { - "version": "6.4.0", - "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.4.0.tgz", - "integrity": "sha512-05PUHKSNE8ou2dwIxTngl4EzcnsCDZGJ/iCLtDflR/SHB/ny14rXc+qU5P4mG9JkusiV7EivzY9Mhm55AzAvCg==", + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.5.0.tgz", + "integrity": "sha512-wieBHXygIm7OyQOu5hQlkk62/WyCFYGlWg7L6/ZCUZwx0o398Zkn4pVmMyfYhfMG8kGrj/Krt8eIk6UKC6VzwA==", "license": "MIT" }, "node_modules/tailwind-merge": { @@ -8724,9 +8063,9 @@ } }, "node_modules/tailwindcss": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.3.1.tgz", - "integrity": "sha512-hk+TB1m+K8CYNrP6rjQaq/Y+4Zylwpa87mLYBKCunwnnQ9p+fHb7kmSfGqyEJoxF/O6CDyABWVFEafNSYKll+Q==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.3.2.tgz", + "integrity": "sha512-WtctNNSH8A9jlMIqxzuYumOHU5uGZyRv0Q5svQl+oEPy5w84YpBxdb7MdqyiSPQge5jTJ6zFQLq0PFygdccSBA==", "dev": true, "license": "MIT" }, @@ -8752,9 +8091,9 @@ "license": "MIT" }, "node_modules/tinyexec": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.1.1.tgz", - "integrity": "sha512-VKS/ZaQhhkKFMANmAOhhXVoIfBXblQxGX1myCQ2faQrfmobMftXeJPcZGp0gS07ocvGJWDLZGyOZDadDBqYIJg==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", "dev": true, "license": "MIT", "engines": { @@ -8799,22 +8138,22 @@ } }, "node_modules/tldts": { - "version": "7.0.28", - "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.28.tgz", - "integrity": "sha512-+Zg3vWhRUv8B1maGSTFdev9mjoo8Etn2Ayfs4cnjlD3CsGkxXX4QyW3j2WJ0wdjYcYmy7Lx2RDsZMhgCWafKIw==", + "version": "7.4.6", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.4.6.tgz", + "integrity": "sha512-rbP0Gyx8b3Ae9yO//CU2wbSnQNoQ66m1nJdSbSHmnwKwzkkz/u8mERYU8T2rmlmy+bJvRNn84yNCW8gYqox44Q==", "dev": true, "license": "MIT", "dependencies": { - "tldts-core": "^7.0.28" + "tldts-core": "^7.4.6" }, "bin": { "tldts": "bin/cli.js" } }, "node_modules/tldts-core": { - "version": "7.0.28", - "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.28.tgz", - "integrity": "sha512-7W5Efjhsc3chVdFhqtaU0KtK32J37Zcr9RKtID54nG+tIpcY79CQK/veYPODxtD/LJ4Lue66jvrQzIX2Z2/pUQ==", + "version": "7.4.6", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.4.6.tgz", + "integrity": "sha512-TkQNGJIhlEphpHCjKodMTSe23egUZr/g+flI2qkLgiJ/maAzSgXypSLRTNH3nCmqgayEmtcJBiLcfODSAr1xoA==", "dev": true, "license": "MIT" }, @@ -8885,16 +8224,16 @@ } }, "node_modules/vite": { - "version": "8.0.16", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.16.tgz", - "integrity": "sha512-h9bXPmJichP5fLmVQo3PyaGSDE2n3aPuomeAlVRm0JLmt4rY6zmPKd59HYI4LNW8oTK7tlTsuC7l/m7awx9Jcw==", + "version": "8.1.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.3.tgz", + "integrity": "sha512-Ds+gBRbj0lwRO2Y5hwnUBdxSwlAve9LeRyU4sNnAr0ewW0gWF0n5bgXgUzbgZ49MV9BVUAQUFYVcDUcilUExMA==", "devOptional": true, "license": "MIT", "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.4", - "postcss": "^8.5.15", - "rolldown": "1.0.3", + "postcss": "^8.5.16", + "rolldown": "~1.1.3", "tinyglobby": "^0.2.17" }, "bin": { @@ -8911,7 +8250,7 @@ }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.1.18", + "@vitejs/devtools": "^0.3.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", @@ -9147,21 +8486,6 @@ "node": ">=18" } }, - "node_modules/xml-naming": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.1.0.tgz", - "integrity": "sha512-k8KO9hrMyNk6tUWqUfkTEZbezRRpONVOzUTnc97VnCvyj6Tf9lyUR9EDAIeiVLv56jsMcoXEwjW8Kv5yPY52lw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "engines": { - "node": ">=16.0.0" - } - }, "node_modules/xmlchars": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", diff --git a/ui/package.json b/ui/package.json index c33ded075..b853b3c8e 100644 --- a/ui/package.json +++ b/ui/package.json @@ -19,166 +19,166 @@ "test:coverage": "vitest run --coverage" }, "dependencies": { - "@aws-sdk/client-accessanalyzer": "3.1070.0", - "@aws-sdk/client-account": "3.1070.0", - "@aws-sdk/client-acm": "3.1070.0", - "@aws-sdk/client-acm-pca": "3.1070.0", - "@aws-sdk/client-amplify": "3.1070.0", - "@aws-sdk/client-api-gateway": "3.1070.0", - "@aws-sdk/client-apigatewaymanagementapi": "3.1070.0", - "@aws-sdk/client-apigatewayv2": "3.1070.0", - "@aws-sdk/client-app-mesh": "3.1070.0", - "@aws-sdk/client-appconfig": "3.1070.0", - "@aws-sdk/client-appfabric": "3.1070.0", - "@aws-sdk/client-application-auto-scaling": "3.1070.0", - "@aws-sdk/client-apprunner": "3.1070.0", - "@aws-sdk/client-appstream": "3.1070.0", - "@aws-sdk/client-appsync": "3.1070.0", - "@aws-sdk/client-athena": "3.1070.0", - "@aws-sdk/client-auto-scaling": "3.1070.0", - "@aws-sdk/client-backup": "3.1070.0", - "@aws-sdk/client-batch": "3.1070.0", - "@aws-sdk/client-bedrock": "3.1070.0", - "@aws-sdk/client-bedrock-runtime": "3.1070.0", - "@aws-sdk/client-cloudcontrol": "3.1070.0", - "@aws-sdk/client-cloudformation": "3.1070.0", - "@aws-sdk/client-cloudfront": "3.1070.0", - "@aws-sdk/client-cloudtrail": "3.1070.0", - "@aws-sdk/client-cloudwatch": "3.1070.0", - "@aws-sdk/client-cloudwatch-logs": "3.1070.0", - "@aws-sdk/client-codeartifact": "3.1070.0", - "@aws-sdk/client-codebuild": "3.1070.0", - "@aws-sdk/client-codecommit": "3.1070.0", - "@aws-sdk/client-codeconnections": "3.1070.0", - "@aws-sdk/client-codedeploy": "3.1070.0", - "@aws-sdk/client-codepipeline": "3.1070.0", - "@aws-sdk/client-codestar-connections": "3.1070.0", - "@aws-sdk/client-cognito-identity": "3.1070.0", - "@aws-sdk/client-cognito-identity-provider": "3.1070.0", - "@aws-sdk/client-comprehend": "3.1070.0", - "@aws-sdk/client-config-service": "3.1070.0", - "@aws-sdk/client-cost-explorer": "3.1070.0", - "@aws-sdk/client-database-migration-service": "3.1070.0", - "@aws-sdk/client-databrew": "3.1070.0", - "@aws-sdk/client-datasync": "3.1070.0", - "@aws-sdk/client-dax": "3.1070.0", - "@aws-sdk/client-detective": "3.1070.0", - "@aws-sdk/client-direct-connect": "3.1070.0", - "@aws-sdk/client-directory-service": "3.1070.0", - "@aws-sdk/client-dlm": "3.1070.0", - "@aws-sdk/client-docdb": "3.1070.0", - "@aws-sdk/client-dynamodb": "3.1070.0", - "@aws-sdk/client-dynamodb-streams": "3.1070.0", - "@aws-sdk/client-ebs": "3.1070.0", - "@aws-sdk/client-ec2": "3.1070.0", - "@aws-sdk/client-ecr": "3.1070.0", - "@aws-sdk/client-ecs": "3.1070.0", - "@aws-sdk/client-efs": "3.1070.0", - "@aws-sdk/client-eks": "3.1070.0", - "@aws-sdk/client-elastic-beanstalk": "3.1070.0", - "@aws-sdk/client-elastic-load-balancing": "3.1070.0", - "@aws-sdk/client-elastic-load-balancing-v2": "3.1070.0", - "@aws-sdk/client-elasticache": "3.1070.0", - "@aws-sdk/client-elasticsearch-service": "3.1070.0", - "@aws-sdk/client-emr": "3.1070.0", - "@aws-sdk/client-emr-serverless": "3.1070.0", - "@aws-sdk/client-eventbridge": "3.1070.0", - "@aws-sdk/client-firehose": "3.1070.0", - "@aws-sdk/client-fis": "3.1070.0", - "@aws-sdk/client-forecast": "3.1070.0", - "@aws-sdk/client-fsx": "3.1070.0", - "@aws-sdk/client-glacier": "3.1070.0", - "@aws-sdk/client-global-accelerator": "3.1070.0", - "@aws-sdk/client-glue": "3.1070.0", - "@aws-sdk/client-grafana": "3.1070.0", - "@aws-sdk/client-guardduty": "3.1070.0", - "@aws-sdk/client-iam": "3.1070.0", - "@aws-sdk/client-identitystore": "3.1070.0", - "@aws-sdk/client-inspector2": "3.1070.0", - "@aws-sdk/client-iot": "3.1070.0", - "@aws-sdk/client-iot-data-plane": "3.1070.0", - "@aws-sdk/client-iot-wireless": "3.1070.0", + "@aws-sdk/client-accessanalyzer": "3.1079.0", + "@aws-sdk/client-account": "3.1079.0", + "@aws-sdk/client-acm": "3.1079.0", + "@aws-sdk/client-acm-pca": "3.1079.0", + "@aws-sdk/client-amplify": "3.1079.0", + "@aws-sdk/client-api-gateway": "3.1079.0", + "@aws-sdk/client-apigatewaymanagementapi": "3.1079.0", + "@aws-sdk/client-apigatewayv2": "3.1079.0", + "@aws-sdk/client-app-mesh": "3.1079.0", + "@aws-sdk/client-appconfig": "3.1079.0", + "@aws-sdk/client-appfabric": "3.1079.0", + "@aws-sdk/client-application-auto-scaling": "3.1079.0", + "@aws-sdk/client-apprunner": "3.1079.0", + "@aws-sdk/client-appstream": "3.1079.0", + "@aws-sdk/client-appsync": "3.1079.0", + "@aws-sdk/client-athena": "3.1079.0", + "@aws-sdk/client-auto-scaling": "3.1079.0", + "@aws-sdk/client-backup": "3.1079.0", + "@aws-sdk/client-batch": "3.1079.0", + "@aws-sdk/client-bedrock": "3.1079.0", + "@aws-sdk/client-bedrock-runtime": "3.1079.0", + "@aws-sdk/client-cloudcontrol": "3.1079.0", + "@aws-sdk/client-cloudformation": "3.1079.0", + "@aws-sdk/client-cloudfront": "3.1079.0", + "@aws-sdk/client-cloudtrail": "3.1079.0", + "@aws-sdk/client-cloudwatch": "3.1079.0", + "@aws-sdk/client-cloudwatch-logs": "3.1079.0", + "@aws-sdk/client-codeartifact": "3.1079.0", + "@aws-sdk/client-codebuild": "3.1079.0", + "@aws-sdk/client-codecommit": "3.1079.0", + "@aws-sdk/client-codeconnections": "3.1079.0", + "@aws-sdk/client-codedeploy": "3.1079.0", + "@aws-sdk/client-codepipeline": "3.1079.0", + "@aws-sdk/client-codestar-connections": "3.1079.0", + "@aws-sdk/client-cognito-identity": "3.1079.0", + "@aws-sdk/client-cognito-identity-provider": "3.1079.0", + "@aws-sdk/client-comprehend": "3.1079.0", + "@aws-sdk/client-config-service": "3.1079.0", + "@aws-sdk/client-cost-explorer": "3.1079.0", + "@aws-sdk/client-database-migration-service": "3.1079.0", + "@aws-sdk/client-databrew": "3.1079.0", + "@aws-sdk/client-datasync": "3.1079.0", + "@aws-sdk/client-dax": "3.1079.0", + "@aws-sdk/client-detective": "3.1079.0", + "@aws-sdk/client-direct-connect": "3.1079.0", + "@aws-sdk/client-directory-service": "3.1079.0", + "@aws-sdk/client-dlm": "3.1079.0", + "@aws-sdk/client-docdb": "3.1079.0", + "@aws-sdk/client-dynamodb": "3.1079.0", + "@aws-sdk/client-dynamodb-streams": "3.1079.0", + "@aws-sdk/client-ebs": "3.1079.0", + "@aws-sdk/client-ec2": "3.1079.0", + "@aws-sdk/client-ecr": "3.1079.0", + "@aws-sdk/client-ecs": "3.1079.0", + "@aws-sdk/client-efs": "3.1079.0", + "@aws-sdk/client-eks": "3.1079.0", + "@aws-sdk/client-elastic-beanstalk": "3.1079.0", + "@aws-sdk/client-elastic-load-balancing": "3.1079.0", + "@aws-sdk/client-elastic-load-balancing-v2": "3.1079.0", + "@aws-sdk/client-elasticache": "3.1079.0", + "@aws-sdk/client-elasticsearch-service": "3.1079.0", + "@aws-sdk/client-emr": "3.1079.0", + "@aws-sdk/client-emr-serverless": "3.1079.0", + "@aws-sdk/client-eventbridge": "3.1079.0", + "@aws-sdk/client-firehose": "3.1079.0", + "@aws-sdk/client-fis": "3.1079.0", + "@aws-sdk/client-forecast": "3.1079.0", + "@aws-sdk/client-fsx": "3.1079.0", + "@aws-sdk/client-glacier": "3.1079.0", + "@aws-sdk/client-global-accelerator": "3.1079.0", + "@aws-sdk/client-glue": "3.1079.0", + "@aws-sdk/client-grafana": "3.1079.0", + "@aws-sdk/client-guardduty": "3.1079.0", + "@aws-sdk/client-iam": "3.1079.0", + "@aws-sdk/client-identitystore": "3.1079.0", + "@aws-sdk/client-inspector2": "3.1079.0", + "@aws-sdk/client-iot": "3.1079.0", + "@aws-sdk/client-iot-data-plane": "3.1079.0", + "@aws-sdk/client-iot-wireless": "3.1079.0", "@aws-sdk/client-iotanalytics": "3.986.0", - "@aws-sdk/client-kafka": "3.1070.0", - "@aws-sdk/client-keyspaces": "3.1070.0", - "@aws-sdk/client-kinesis": "3.1070.0", - "@aws-sdk/client-kinesis-analytics": "3.1070.0", - "@aws-sdk/client-kinesis-analytics-v2": "3.1070.0", - "@aws-sdk/client-kinesis-video": "3.1070.0", - "@aws-sdk/client-kms": "3.1070.0", - "@aws-sdk/client-lakeformation": "3.1070.0", - "@aws-sdk/client-lambda": "3.1070.0", - "@aws-sdk/client-lightsail": "3.1070.0", - "@aws-sdk/client-macie2": "3.1070.0", - "@aws-sdk/client-managedblockchain": "3.1070.0", - "@aws-sdk/client-mediaconvert": "3.1070.0", - "@aws-sdk/client-medialive": "3.1070.0", - "@aws-sdk/client-mediapackage": "3.1070.0", - "@aws-sdk/client-mediastore": "3.1070.0", - "@aws-sdk/client-mediastore-data": "3.1070.0", - "@aws-sdk/client-mediatailor": "3.1070.0", - "@aws-sdk/client-memorydb": "3.1070.0", - "@aws-sdk/client-mgn": "3.1070.0", - "@aws-sdk/client-mq": "3.1070.0", - "@aws-sdk/client-mwaa": "3.1070.0", - "@aws-sdk/client-neptune": "3.1070.0", - "@aws-sdk/client-networkmanager": "3.1070.0", - "@aws-sdk/client-opensearch": "3.1070.0", - "@aws-sdk/client-organizations": "3.1070.0", - "@aws-sdk/client-outposts": "3.1070.0", - "@aws-sdk/client-personalize": "3.1070.0", - "@aws-sdk/client-pinpoint": "3.1070.0", - "@aws-sdk/client-pipes": "3.1070.0", - "@aws-sdk/client-polly": "3.1070.0", - "@aws-sdk/client-quicksight": "3.1070.0", - "@aws-sdk/client-ram": "3.1070.0", - "@aws-sdk/client-rds": "3.1070.0", - "@aws-sdk/client-rds-data": "3.1070.0", - "@aws-sdk/client-redshift": "3.1070.0", - "@aws-sdk/client-redshift-data": "3.1070.0", - "@aws-sdk/client-rekognition": "3.1070.0", - "@aws-sdk/client-resiliencehub": "3.1070.0", - "@aws-sdk/client-resource-groups": "3.1070.0", - "@aws-sdk/client-resource-groups-tagging-api": "3.1070.0", - "@aws-sdk/client-rolesanywhere": "3.1070.0", - "@aws-sdk/client-route-53": "3.1070.0", - "@aws-sdk/client-route53resolver": "3.1070.0", - "@aws-sdk/client-s3": "3.1070.0", - "@aws-sdk/client-s3-control": "3.1070.0", - "@aws-sdk/client-s3tables": "3.1070.0", - "@aws-sdk/client-sagemaker": "3.1070.0", - "@aws-sdk/client-sagemaker-runtime": "3.1070.0", - "@aws-sdk/client-scheduler": "3.1070.0", - "@aws-sdk/client-secrets-manager": "3.1070.0", - "@aws-sdk/client-securityhub": "3.1070.0", - "@aws-sdk/client-serverlessapplicationrepository": "3.1070.0", - "@aws-sdk/client-servicediscovery": "3.1070.0", - "@aws-sdk/client-ses": "3.1070.0", - "@aws-sdk/client-sesv2": "3.1070.0", - "@aws-sdk/client-sfn": "3.1070.0", - "@aws-sdk/client-shield": "3.1070.0", - "@aws-sdk/client-sns": "3.1070.0", - "@aws-sdk/client-sqs": "3.1070.0", - "@aws-sdk/client-ssm": "3.1070.0", - "@aws-sdk/client-sso-admin": "3.1070.0", - "@aws-sdk/client-sts": "3.1070.0", - "@aws-sdk/client-support": "3.1070.0", - "@aws-sdk/client-swf": "3.1070.0", - "@aws-sdk/client-textract": "3.1070.0", - "@aws-sdk/client-timestream-query": "3.1070.0", - "@aws-sdk/client-timestream-write": "3.1070.0", - "@aws-sdk/client-transcribe": "3.1070.0", - "@aws-sdk/client-transfer": "3.1070.0", - "@aws-sdk/client-translate": "3.1070.0", - "@aws-sdk/client-verifiedpermissions": "3.1070.0", - "@aws-sdk/client-wafv2": "3.1070.0", - "@aws-sdk/client-workmail": "3.1070.0", - "@aws-sdk/client-workspaces": "3.1070.0", - "@aws-sdk/client-xray": "3.1070.0", - "@aws-sdk/credential-providers": "3.1070.0", - "@bufbuild/protobuf": "1.10.0", - "@connectrpc/connect": "1.6.1", - "@connectrpc/connect-web": "1.6.1", + "@aws-sdk/client-kafka": "3.1079.0", + "@aws-sdk/client-keyspaces": "3.1079.0", + "@aws-sdk/client-kinesis": "3.1079.0", + "@aws-sdk/client-kinesis-analytics": "3.1079.0", + "@aws-sdk/client-kinesis-analytics-v2": "3.1079.0", + "@aws-sdk/client-kinesis-video": "3.1079.0", + "@aws-sdk/client-kms": "3.1079.0", + "@aws-sdk/client-lakeformation": "3.1079.0", + "@aws-sdk/client-lambda": "3.1079.0", + "@aws-sdk/client-lightsail": "3.1079.0", + "@aws-sdk/client-macie2": "3.1079.0", + "@aws-sdk/client-managedblockchain": "3.1079.0", + "@aws-sdk/client-mediaconvert": "3.1079.0", + "@aws-sdk/client-medialive": "3.1079.0", + "@aws-sdk/client-mediapackage": "3.1079.0", + "@aws-sdk/client-mediastore": "3.1079.0", + "@aws-sdk/client-mediastore-data": "3.1079.0", + "@aws-sdk/client-mediatailor": "3.1079.0", + "@aws-sdk/client-memorydb": "3.1079.0", + "@aws-sdk/client-mgn": "3.1079.0", + "@aws-sdk/client-mq": "3.1079.0", + "@aws-sdk/client-mwaa": "3.1079.0", + "@aws-sdk/client-neptune": "3.1079.0", + "@aws-sdk/client-networkmanager": "3.1079.0", + "@aws-sdk/client-opensearch": "3.1079.0", + "@aws-sdk/client-organizations": "3.1079.0", + "@aws-sdk/client-outposts": "3.1079.0", + "@aws-sdk/client-personalize": "3.1079.0", + "@aws-sdk/client-pinpoint": "3.1079.0", + "@aws-sdk/client-pipes": "3.1079.0", + "@aws-sdk/client-polly": "3.1079.0", + "@aws-sdk/client-quicksight": "3.1079.0", + "@aws-sdk/client-ram": "3.1079.0", + "@aws-sdk/client-rds": "3.1079.0", + "@aws-sdk/client-rds-data": "3.1079.0", + "@aws-sdk/client-redshift": "3.1079.0", + "@aws-sdk/client-redshift-data": "3.1079.0", + "@aws-sdk/client-rekognition": "3.1079.0", + "@aws-sdk/client-resiliencehub": "3.1079.0", + "@aws-sdk/client-resource-groups": "3.1079.0", + "@aws-sdk/client-resource-groups-tagging-api": "3.1079.0", + "@aws-sdk/client-rolesanywhere": "3.1079.0", + "@aws-sdk/client-route-53": "3.1079.0", + "@aws-sdk/client-route53resolver": "3.1079.0", + "@aws-sdk/client-s3": "3.1079.0", + "@aws-sdk/client-s3-control": "3.1079.0", + "@aws-sdk/client-s3tables": "3.1079.0", + "@aws-sdk/client-sagemaker": "3.1079.0", + "@aws-sdk/client-sagemaker-runtime": "3.1079.0", + "@aws-sdk/client-scheduler": "3.1079.0", + "@aws-sdk/client-secrets-manager": "3.1079.0", + "@aws-sdk/client-securityhub": "3.1079.0", + "@aws-sdk/client-serverlessapplicationrepository": "3.1079.0", + "@aws-sdk/client-servicediscovery": "3.1079.0", + "@aws-sdk/client-ses": "3.1079.0", + "@aws-sdk/client-sesv2": "3.1079.0", + "@aws-sdk/client-sfn": "3.1079.0", + "@aws-sdk/client-shield": "3.1079.0", + "@aws-sdk/client-sns": "3.1079.0", + "@aws-sdk/client-sqs": "3.1079.0", + "@aws-sdk/client-ssm": "3.1079.0", + "@aws-sdk/client-sso-admin": "3.1079.0", + "@aws-sdk/client-sts": "3.1079.0", + "@aws-sdk/client-support": "3.1079.0", + "@aws-sdk/client-swf": "3.1079.0", + "@aws-sdk/client-textract": "3.1079.0", + "@aws-sdk/client-timestream-query": "3.1079.0", + "@aws-sdk/client-timestream-write": "3.1079.0", + "@aws-sdk/client-transcribe": "3.1079.0", + "@aws-sdk/client-transfer": "3.1079.0", + "@aws-sdk/client-translate": "3.1079.0", + "@aws-sdk/client-verifiedpermissions": "3.1079.0", + "@aws-sdk/client-wafv2": "3.1079.0", + "@aws-sdk/client-workmail": "3.1079.0", + "@aws-sdk/client-workspaces": "3.1079.0", + "@aws-sdk/client-xray": "3.1079.0", + "@aws-sdk/credential-providers": "3.1079.0", + "@bufbuild/protobuf": "1.10.1", + "@connectrpc/connect": "1.7.0", + "@connectrpc/connect-web": "1.7.0", "bits-ui": "2.18.1", "class-variance-authority": "0.7.1", "clsx": "2.1.1", @@ -188,20 +188,20 @@ }, "devDependencies": { "@sveltejs/adapter-static": "3.0.10", - "@sveltejs/kit": "2.65.2", + "@sveltejs/kit": "2.69.1", "@sveltejs/vite-plugin-svelte": "7.1.2", - "@tailwindcss/vite": "4.3.1", + "@tailwindcss/vite": "4.3.2", "@testing-library/jest-dom": "6.9.1", - "@testing-library/svelte": "5.3.1", + "@testing-library/svelte": "5.4.2", "@vitest/coverage-v8": "4.1.9", "jsdom": "29.1.1", - "oxfmt": "0.55.0", - "oxlint": "1.70.0", - "svelte": "5.56.3", - "svelte-check": "4.6.0", - "tailwindcss": "4.3.1", + "oxfmt": "0.57.0", + "oxlint": "1.72.0", + "svelte": "5.56.4", + "svelte-check": "4.7.1", + "tailwindcss": "4.3.2", "typescript": "6.0.3", - "vite": "8.0.16", + "vite": "8.1.3", "vitest": "4.1.9" }, "overrides": {