diff --git a/Cargo.lock b/Cargo.lock index 86dab6abc65..6b150d27b1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -931,7 +931,7 @@ dependencies = [ "sled-agent-types", "sled-agent-types-versions", "sled-hardware-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", ] [[package]] @@ -3082,6 +3082,15 @@ dependencies = [ "ctutils", ] +[[package]] +name = "digest-io" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2de63d600bc7fab91180bc17385f29b342468dc8ef2af09dceba450a293de3da" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -4270,7 +4279,7 @@ dependencies = [ "termios", "tokio", "tokio-tungstenite 0.23.1", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", ] @@ -4407,7 +4416,7 @@ dependencies = [ "serde", "test-strategy", "thiserror 2.0.18", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", ] @@ -5812,7 +5821,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-stream", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "tufaceous-lib", "update-engine", ] @@ -5832,7 +5841,7 @@ dependencies = [ "schemars 0.8.22", "serde", "slog", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "update-engine", "uuid", ] @@ -6005,7 +6014,7 @@ dependencies = [ "slog-error-chain", "test-strategy", "thiserror 2.0.18", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", ] [[package]] @@ -7334,7 +7343,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "trust-quorum-protocol", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", ] @@ -7432,7 +7441,7 @@ dependencies = [ "tokio", "trust-quorum-protocol", "trust-quorum-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "url", "usdt 0.5.0", "uuid", @@ -7496,7 +7505,7 @@ dependencies = [ "serde_json", "slog-error-chain", "trust-quorum-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", ] @@ -7591,7 +7600,7 @@ dependencies = [ "swrite", "thiserror 2.0.18", "tokio", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "typed-rng", "uuid", ] @@ -7725,7 +7734,7 @@ dependencies = [ "tokio-stream", "tokio-util", "trust-quorum-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", ] @@ -7769,7 +7778,7 @@ dependencies = [ "omicron-uuid-kinds", "omicron-workspace-hack", "sled-agent-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", ] [[package]] @@ -7915,8 +7924,8 @@ dependencies = [ "tokio", "toml 0.8.23", "transient-dns-server", - "tufaceous", - "tufaceous-artifact", + "tufaceous 0.1.0", + "tufaceous-artifact 0.1.0", "tufaceous-lib", "typed-rng", "update-common", @@ -7992,7 +8001,7 @@ dependencies = [ "swrite", "sync-ptr", "thiserror 2.0.18", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "typed-rng", "uuid", ] @@ -8217,7 +8226,7 @@ dependencies = [ "tough", "trust-quorum-protocol", "trust-quorum-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "unicode-width 0.1.14", "update-engine", "url", @@ -8257,7 +8266,7 @@ dependencies = [ "strum 0.27.2", "thiserror 2.0.18", "tough", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "url", "uuid", ] @@ -8809,7 +8818,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "toml 0.8.23", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", ] @@ -8965,7 +8974,7 @@ dependencies = [ "tokio-stream", "tokio-tungstenite 0.23.1", "toml 0.8.23", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", "zip 4.6.1", ] @@ -9233,8 +9242,8 @@ dependencies = [ "tokio-util", "tough", "trust-quorum-types", - "tufaceous", - "tufaceous-artifact", + "tufaceous 0.1.0", + "tufaceous-artifact 0.1.0", "tufaceous-lib", "update-common", "update-engine", @@ -9377,7 +9386,7 @@ dependencies = [ "textwrap 0.16.2", "tokio", "trust-quorum-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "unicode-width 0.1.14", "update-engine", "url", @@ -9509,7 +9518,6 @@ dependencies = [ "semver 1.0.28", "serde", "serde_json", - "sha2 0.10.9", "shell-words", "slog", "slog-async", @@ -9517,7 +9525,9 @@ dependencies = [ "tar", "tokio", "toml 0.8.23", - "tufaceous-artifact", + "tufaceous 0.2.0", + "tufaceous-artifact 0.1.0", + "tufaceous-artifact 0.2.0", "tufaceous-lib", "update-common", ] @@ -9556,7 +9566,7 @@ dependencies = [ "slog-error-chain", "tokio", "tokio-util", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "update-common", ] @@ -9692,8 +9702,8 @@ dependencies = [ "trust-quorum", "trust-quorum-protocol", "trust-quorum-types", - "tufaceous-artifact", - "tufaceous-brand-metadata", + "tufaceous-artifact 0.1.0", + "tufaceous-brand-metadata 0.1.0", "usdt 0.5.0", "uuid", "walkdir", @@ -9774,6 +9784,7 @@ dependencies = [ "bitflags 2.11.0", "bstr", "buf-list", + "byte-wrapper", "bytes", "camino", "chacha20 0.10.0", @@ -12144,6 +12155,12 @@ dependencies = [ "bitflags 2.11.0", ] +[[package]] +name = "rawzip" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d9575f44c8cf85bc843ad666dcdf20d05a7753772bef56eb2a5140282b32150" + [[package]] name = "rayon" version = "1.11.0" @@ -12222,8 +12239,8 @@ dependencies = [ "tabled 0.15.0", "tokio", "toml 0.8.23", - "tufaceous", - "tufaceous-artifact", + "tufaceous 0.1.0", + "tufaceous-artifact 0.1.0", "tufaceous-lib", "update-common", "uuid", @@ -12256,7 +12273,7 @@ dependencies = [ "slog", "swrite", "tokio", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", ] [[package]] @@ -12427,7 +12444,7 @@ dependencies = [ "omicron-workspace-hack", "schemars 0.8.22", "serde", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", ] [[package]] @@ -13811,7 +13828,7 @@ dependencies = [ "sled-hardware-types", "slog-error-chain", "trust-quorum-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", "x509-cert", ] @@ -13895,7 +13912,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "trust-quorum-types", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "xshell", "zone", ] @@ -14046,7 +14063,7 @@ dependencies = [ "slog", "slog-error-chain", "thiserror 2.0.18", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", ] [[package]] @@ -14063,7 +14080,7 @@ dependencies = [ "sha2 0.10.9", "sled-agent-types", "sled-storage", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", ] [[package]] @@ -14093,7 +14110,7 @@ dependencies = [ "swrite", "test-strategy", "thiserror 2.0.18", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", ] @@ -14137,7 +14154,7 @@ dependencies = [ "thiserror 2.0.18", "toml 0.8.23", "trust-quorum-types-versions", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", ] @@ -14922,7 +14939,7 @@ dependencies = [ "slog-error-chain", "tokio", "tokio-util", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "uuid", "zip 4.6.1", ] @@ -15132,9 +15149,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.45" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" dependencies = [ "filetime", "libc", @@ -16220,7 +16237,7 @@ dependencies = [ [[package]] name = "tufaceous" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#a7d440f5a111c7e3504e8eb125c105d7baf0deab" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=v1#95395cecb019b851519232468472212efce1e42d" dependencies = [ "anyhow", "camino", @@ -16234,14 +16251,48 @@ dependencies = [ "slog-envlogger", "slog-term", "tokio", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "tufaceous-lib", ] +[[package]] +name = "tufaceous" +version = "0.2.0" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#465d3dd5002c233502337b9350ce144e17d1ef72" +dependencies = [ + "atomicwrites", + "aws-lc-rs", + "bytes", + "camino", + "camino-tempfile", + "chrono", + "ciborium", + "daft", + "digest-io", + "flate2", + "futures-util", + "hex", + "hubtools 0.4.7 (git+https://github.com/oxidecomputer/hubtools.git?rev=2b1ef9b38d75563ea800baa3b17327eec17b1b7a)", + "rats-corim 0.1.0 (git+https://github.com/oxidecomputer/rats-corim.git?rev=f0d5d5168d3d31487a56df32c676b0c6240bcc6b)", + "rawzip", + "semver 1.0.28", + "serde", + "serde_json", + "sha2 0.11.0", + "slog", + "tar", + "thiserror 2.0.18", + "tokio", + "tough", + "tufaceous-artifact 0.2.0", + "tufaceous-brand-metadata 0.2.0", + "url", +] + [[package]] name = "tufaceous-artifact" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#a7d440f5a111c7e3504e8eb125c105d7baf0deab" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=v1#95395cecb019b851519232468472212efce1e42d" dependencies = [ "daft", "hex", @@ -16255,22 +16306,46 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "tufaceous-artifact" +version = "0.2.0" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#465d3dd5002c233502337b9350ce144e17d1ef72" +dependencies = [ + "byte-wrapper", + "daft", + "hubtools 0.4.7 (git+https://github.com/oxidecomputer/hubtools.git?rev=2b1ef9b38d75563ea800baa3b17327eec17b1b7a)", + "serde", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "tufaceous-brand-metadata" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#a7d440f5a111c7e3504e8eb125c105d7baf0deab" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=v1#95395cecb019b851519232468472212efce1e42d" dependencies = [ "semver 1.0.28", "serde", "serde_json", "tar", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", +] + +[[package]] +name = "tufaceous-brand-metadata" +version = "0.2.0" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#465d3dd5002c233502337b9350ce144e17d1ef72" +dependencies = [ + "serde", + "serde_json", + "tar", + "tufaceous-artifact 0.2.0", ] [[package]] name = "tufaceous-lib" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/tufaceous?branch=main#a7d440f5a111c7e3504e8eb125c105d7baf0deab" +source = "git+https://github.com/oxidecomputer/tufaceous?branch=v1#95395cecb019b851519232468472212efce1e42d" dependencies = [ "anyhow", "async-trait", @@ -16303,8 +16378,8 @@ dependencies = [ "tokio", "toml 0.8.23", "tough", - "tufaceous-artifact", - "tufaceous-brand-metadata", + "tufaceous-artifact 0.1.0", + "tufaceous-brand-metadata 0.1.0", "url", "zip 4.6.1", ] @@ -16602,9 +16677,9 @@ dependencies = [ "tokio", "tokio-util", "tough", - "tufaceous", - "tufaceous-artifact", - "tufaceous-brand-metadata", + "tufaceous 0.1.0", + "tufaceous-artifact 0.1.0", + "tufaceous-brand-metadata 0.1.0", "tufaceous-lib", ] @@ -17239,7 +17314,7 @@ dependencies = [ "toml 0.8.23", "toml_edit 0.22.27", "transceiver-controller 0.1.1 (git+https://github.com/oxidecomputer/transceiver-control)", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "tui-tree-widget", "unicode-width 0.1.14", "update-engine", @@ -17275,7 +17350,7 @@ dependencies = [ "tokio", "toml 0.8.23", "transceiver-controller 0.1.1 (git+https://github.com/oxidecomputer/transceiver-control)", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "update-engine", ] @@ -17381,8 +17456,8 @@ dependencies = [ "toml 0.8.23", "tough", "transceiver-controller 0.1.1 (git+https://github.com/oxidecomputer/transceiver-control)", - "tufaceous", - "tufaceous-artifact", + "tufaceous 0.1.0", + "tufaceous-artifact 0.1.0", "tufaceous-lib", "update-common", "update-engine", @@ -17409,7 +17484,7 @@ dependencies = [ "serde", "sled-hardware-types", "slog", - "tufaceous-artifact", + "tufaceous-artifact 0.1.0", "wicket-common", ] diff --git a/Cargo.toml b/Cargo.toml index f8828bf2cc6..ea1b662cabb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -868,10 +868,12 @@ tough = { version = "0.22.0" } transceiver-controller = { git = "https://github.com/oxidecomputer/transceiver-control", features = [ "api-traits" ] } transient-dns-server = { path = "dns-server/transient" } trybuild = "1.0.106" -tufaceous = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" } -tufaceous-artifact = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main", features = ["proptest", "schemars"] } -tufaceous-brand-metadata = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" } -tufaceous-lib = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" } +tufaceous = { git = "https://github.com/oxidecomputer/tufaceous", branch = "v1" } +tufaceous-artifact = { git = "https://github.com/oxidecomputer/tufaceous", branch = "v1", features = ["proptest", "schemars"] } +tufaceous-brand-metadata = { git = "https://github.com/oxidecomputer/tufaceous", branch = "v1" } +tufaceous-lib = { git = "https://github.com/oxidecomputer/tufaceous", branch = "v1" } +tufaceous-v2 = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main", package = "tufaceous" } +tufaceous-artifact-v2 = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main", package = "tufaceous-artifact" } tui-tree-widget = "0.23.1" typed-rng = { path = "typed-rng" } typify = "0.3.0" @@ -1110,7 +1112,8 @@ opt-level = 3 # tufaceous-artifact = { path = "../tufaceous/artifact" } # tufaceous-brand-metadata = { path = "../tufaceous/brand-metadata" } # tufaceous-lib = { path = "../tufaceous/lib" } -# Extra slash here to pretend to be a different source. +# tufaceous-v2 = { path = "../tufaceous/lib", package = "tufaceous" } +# tufaceous-artifact-v2 = { path = "../tufaceous/artifact", package = "tufaceous-artifact" } # [patch."https://github.com/oxidecomputer/typify"] # typify = { path = "../typify/typify" } diff --git a/dev-tools/releng/Cargo.toml b/dev-tools/releng/Cargo.toml index 59f4cdafb7c..93c34a4baef 100644 --- a/dev-tools/releng/Cargo.toml +++ b/dev-tools/releng/Cargo.toml @@ -24,7 +24,6 @@ reqwest.workspace = true semver.workspace = true serde.workspace = true serde_json.workspace = true -sha2.workspace = true shell-words.workspace = true slog.workspace = true slog-async.workspace = true @@ -33,7 +32,9 @@ tar.workspace = true tokio = { workspace = true, features = ["full"] } toml.workspace = true tufaceous-artifact.workspace = true +tufaceous-artifact-v2.workspace = true tufaceous-lib.workspace = true +tufaceous-v2.workspace = true update-common.workspace = true [lints] diff --git a/dev-tools/releng/src/hubris.rs b/dev-tools/releng/src/hubris.rs index bfa1793e6f5..2054e182dbe 100644 --- a/dev-tools/releng/src/hubris.rs +++ b/dev-tools/releng/src/hubris.rs @@ -8,6 +8,7 @@ use std::io::ErrorKind; use anyhow::Context as _; use anyhow::Result; +use anyhow::bail; use anyhow::ensure; use camino::Utf8PathBuf; use fs_err::tokio as fs; @@ -19,6 +20,7 @@ use slog::Logger; use slog::warn; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::KnownArtifactKind; +use tufaceous_artifact_v2::RotSlot; use tufaceous_lib::assemble::DeserializedArtifactData; use tufaceous_lib::assemble::DeserializedArtifactSource; use tufaceous_lib::assemble::DeserializedFileArtifactSource; @@ -53,6 +55,7 @@ pub(crate) async fn fetch_hubris_artifacts( env: Environment, client: reqwest::Client, output_dir: Utf8PathBuf, + editor: crate::tuf_v2::SharedEditor, ) -> Result<()> { let output_dir = output_dir.join(format!("hubris-{}", env.short_name)); let ctx = Context { logger, env, client, output_dir }; @@ -61,6 +64,8 @@ pub(crate) async fn fetch_hubris_artifacts( // This could be parallelized with FuturesUnordered but in practice this // takes less time than OS builds. + // A partial Tufaceous v1 manifest, written to + // $output_dir/hubris-$env/manifest.toml. let mut manifest = DeserializedManifest { system_version: Version::new(0, 0, 0), artifacts: BTreeMap::new(), @@ -74,8 +79,8 @@ pub(crate) async fn fetch_hubris_artifacts( let str = String::from_utf8(data).with_context(|| { format!("hubris artifact manifest {} was not UTF-8", hash) })?; - let hash_manifest: Manifest = - toml::from_str(&str).with_context(|| { + let hash_manifest: PermslipManifest = toml::from_str(&str) + .with_context(|| { format!( "failed to deserialize hubris artifact manifest {}", hash @@ -83,16 +88,25 @@ pub(crate) async fn fetch_hubris_artifacts( })?; for (kind, artifacts) in hash_manifest.artifact { for artifact in artifacts { - let source = match artifact.source { - Source::File(file) => { - let path = ctx.fetch_hash(file.hash, "zip").await?.path; - DeserializedArtifactSource::File { path } - } - Source::CompositeRot { archive_a, archive_b } => { + let source = match (kind, artifact.source) { + ( + PermslipArtifactKind::GimletRot + | PermslipArtifactKind::SwitchRot + | PermslipArtifactKind::PscRot, + PermslipSource::CompositeRot { archive_a, archive_b }, + ) => { let path_a = ctx.fetch_hash(archive_a.hash, "zip").await?.path; + editor + .add_rot_archive(RotSlot::A, path_a.clone()) + .await?; + let path_b = ctx.fetch_hash(archive_b.hash, "zip").await?.path; + editor + .add_rot_archive(RotSlot::B, path_b.clone()) + .await?; + DeserializedArtifactSource::CompositeRot { archive_a: DeserializedFileArtifactSource::File { path: path_a, @@ -102,8 +116,31 @@ pub(crate) async fn fetch_hubris_artifacts( }, } } + ( + PermslipArtifactKind::GimletRotBootloader + | PermslipArtifactKind::SwitchRotBootloader + | PermslipArtifactKind::PscRotBootloader, + PermslipSource::File(file), + ) => { + let path = ctx.fetch_hash(file.hash, "zip").await?.path; + editor.add_rot_bootloader_archive(path.clone()).await?; + DeserializedArtifactSource::File { path } + } + ( + PermslipArtifactKind::GimletSp + | PermslipArtifactKind::SwitchSp + | PermslipArtifactKind::PscSp, + PermslipSource::File(file), + ) => { + let path = ctx.fetch_hash(file.hash, "zip").await?.path; + editor.add_sp_archive(path.clone()).await?; + DeserializedArtifactSource::File { path } + } + (kind @ _, source @ _) => { + bail!("unexpected source {source:?} for kind {kind:?}"); + } }; - manifest.artifacts.entry(kind).or_default().push( + manifest.artifacts.entry(kind.into()).or_default().push( DeserializedArtifactData { name: artifact.name, version: artifact.version, @@ -112,8 +149,11 @@ pub(crate) async fn fetch_hubris_artifacts( ); } } - if let Some(FileSource { hash }) = hash_manifest.measurement_corpus { + if let Some(PermslipFileSource { hash }) = + hash_manifest.measurement_corpus + { let Output { data, path } = ctx.fetch_hash(hash, "corim").await?; + editor.add_measurement_corpus(path.clone()).await?; let corim = rats_corim::Corim::from_bytes(&data)?; manifest .artifacts @@ -283,28 +323,67 @@ impl Context { // tufaceous-lib, except that the source is a hash instead of a file path. This // hash is used to download the artifact from Permission Slip. #[derive(Deserialize)] -struct Manifest { - artifact: HashMap>, +struct PermslipManifest { + artifact: HashMap>, // Add a default for backwards compatibility - measurement_corpus: Option, + measurement_corpus: Option, +} + +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq, Hash)] +#[serde(rename_all = "snake_case")] +enum PermslipArtifactKind { + GimletRot, + SwitchRot, + PscRot, + GimletRotBootloader, + SwitchRotBootloader, + PscRotBootloader, + GimletSp, + SwitchSp, + PscSp, +} + +impl From for KnownArtifactKind { + fn from(value: PermslipArtifactKind) -> Self { + match value { + PermslipArtifactKind::GimletRot => KnownArtifactKind::GimletRot, + PermslipArtifactKind::SwitchRot => KnownArtifactKind::SwitchRot, + PermslipArtifactKind::PscRot => KnownArtifactKind::PscRot, + PermslipArtifactKind::GimletRotBootloader => { + KnownArtifactKind::GimletRotBootloader + } + PermslipArtifactKind::SwitchRotBootloader => { + KnownArtifactKind::SwitchRotBootloader + } + PermslipArtifactKind::PscRotBootloader => { + KnownArtifactKind::PscRotBootloader + } + PermslipArtifactKind::GimletSp => KnownArtifactKind::GimletSp, + PermslipArtifactKind::SwitchSp => KnownArtifactKind::SwitchSp, + PermslipArtifactKind::PscSp => KnownArtifactKind::PscSp, + } + } } #[derive(Deserialize)] -struct Artifact { +struct PermslipArtifact { name: String, version: ArtifactVersion, - source: Source, + source: PermslipSource, } -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] #[serde(tag = "kind", rename_all = "kebab-case")] -enum Source { - File(FileSource), - CompositeRot { archive_a: FileSource, archive_b: FileSource }, +enum PermslipSource { + File(PermslipFileSource), + CompositeRot { + archive_a: PermslipFileSource, + archive_b: PermslipFileSource, + }, } -#[derive(Deserialize)] -struct FileSource { +#[derive(Debug, Deserialize)] +struct PermslipFileSource { #[serde(deserialize_with = "deserialize_hash")] hash: blake3::Hash, } diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index 4f65edb0c05..ff28525f015 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -7,6 +7,7 @@ mod helios; mod hubris; mod job; mod tuf; +mod tuf_v2; use std::collections::BTreeMap; use std::path::PathBuf; @@ -38,6 +39,7 @@ use slog::info; use slog_term::FullFormat; use slog_term::TermDecorator; use tokio::sync::Semaphore; +use tufaceous_artifact_v2::OsVariant; use crate::cmd::Command; use crate::job::Jobs; @@ -247,11 +249,10 @@ async fn main() -> Result<()> { // Read pins.toml. let pins = omicron_pins::Pins::read_from_dir(&metadata.workspace_root)?; - let permits = Arc::new(Semaphore::new( - std::thread::available_parallelism() - .context("couldn't get available parallelism")? - .into(), - )); + let threads = std::thread::available_parallelism() + .context("couldn't get available parallelism")? + .into(); + let permits = Arc::new(Semaphore::new(threads)); let commit = Command::new(&args.git_bin) .args(["rev-parse", "HEAD"]) @@ -448,6 +449,7 @@ async fn main() -> Result<()> { fs::create_dir_all(&args.output_dir).await?; // DEFINE JOBS ============================================================ + let editor_v2 = tuf_v2::SharedEditor::new(version.clone())?; let tempdir = camino_tempfile::tempdir() .context("failed to create temporary directory")?; let mut jobs = Jobs::new(&logger, permits.clone(), &args.output_dir); @@ -650,13 +652,13 @@ async fn main() -> Result<()> { commit.chars().take(7).collect::(), Utc::now().format("%Y-%m-%d %H:%M") ); - + let output_dir = args.output_dir.join(format!("os-{}", target)); let mut image_cmd = Command::new("ptime") .arg("-m") .arg(args.helios_dir.join("helios-build")) .arg("experiment-image") .arg("-o") // output directory for image - .arg(args.output_dir.join(format!("os-{}", target))) + .arg(&output_dir) .arg("-F") // pass extra image builder features .arg(format!("optever={opte_version}")) .arg("-P") // include all files from extra proto area @@ -730,8 +732,9 @@ async fn main() -> Result<()> { .arg(format!("helios-dev=file://{p5p_path}")); } + let image_job_name = format!("{target}-image"); let image_job = jobs - .push_command(format!("{target}-image"), image_cmd) + .push_command(&image_job_name, image_cmd) .after("helios-setup") .after("helios-incorp") .after(format!("{target}-proto")); @@ -739,6 +742,16 @@ async fn main() -> Result<()> { if opte_override.is_some() { image_job.after(format!("{target}-opte-p5p")); } + + let editor = editor_v2.clone(); + jobs.push(format!("tuf-v2-{target}"), async move { + let os_variant = match target { + Target::Host => OsVariant::Host, + Target::Recovery => OsVariant::Recovery, + }; + editor.add_os_image_dir(os_variant, output_dir).await + }) + .after(image_job_name); } // Build the recovery target after we build the host target (and have // finished verifying its binaries). Only one of these will build at a time @@ -762,6 +775,19 @@ async fn main() -> Result<()> { stamp_packages!("tuf-stamp", Target::Host, TUF_PACKAGES) .after("host-stamp") .after("recovery-stamp"); + { + let editor = editor_v2.clone(); + jobs.push("tuf-v2-zones", async move { + for package in TUF_PACKAGES { + let path = crate::WORKSPACE_DIR + .join("out/versioned") + .join(format!("{}.tar.gz", package)); + editor.add_zone_image(path).await?; + } + Ok(()) + }) + .after("tuf-stamp"); + } if args.verify_debug_libraries { // Run `cargo xtask verify-libraries`. This builds *all* binaries in @@ -791,6 +817,7 @@ async fn main() -> Result<()> { env, client.clone(), args.output_dir.clone(), + editor_v2.clone(), ), ); } @@ -803,6 +830,7 @@ async fn main() -> Result<()> { version, manifest, args.extra_manifest, + threads, ), ) .after("tuf-stamp") @@ -811,6 +839,21 @@ async fn main() -> Result<()> { .after("hubris-staging") .after("hubris-production"); + jobs.push( + "tuf-v2-repo", + tuf_v2::build_tuf_repo( + logger.clone(), + args.output_dir.clone(), + editor_v2, + threads, + ), + ) + .after("tuf-v2-host") + .after("tuf-v2-recovery") + .after("tuf-v2-zones") + .after("hubris-staging") + .after("hubris-production"); + // RUN JOBS =============================================================== let start = Instant::now(); jobs.run_all().await?; diff --git a/dev-tools/releng/src/tuf.rs b/dev-tools/releng/src/tuf.rs index 3a9afe98ca3..6c30be83261 100644 --- a/dev-tools/releng/src/tuf.rs +++ b/dev-tools/releng/src/tuf.rs @@ -2,10 +2,13 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use std::error::Error as _; +use std::fmt::Write; use std::sync::Arc; use anyhow::Context; use anyhow::Result; +use anyhow::ensure; use camino::Utf8PathBuf; use chrono::Duration; use chrono::Timelike; @@ -14,10 +17,7 @@ use fs_err::tokio as fs; use fs_err::tokio::File; use omicron_zone_package::config::Config; use semver::Version; -use sha2::Digest; -use sha2::Sha256; use slog::Logger; -use tokio::io::AsyncReadExt; use tufaceous_artifact::ArtifactHash; use tufaceous_artifact::ArtifactVersion; use tufaceous_artifact::KnownArtifactKind; @@ -28,6 +28,7 @@ use tufaceous_lib::assemble::DeserializedArtifactSource; use tufaceous_lib::assemble::DeserializedControlPlaneZoneSource; use tufaceous_lib::assemble::DeserializedManifest; use tufaceous_lib::assemble::OmicronRepoAssembler; +use tufaceous_v2::RepositoryLoader; use update_common::artifacts::{ ArtifactsWithPlan, ControlPlaneZonesMode, VerificationMode, }; @@ -38,7 +39,11 @@ pub(crate) async fn build_tuf_repo( version: Version, package_manifest: Arc, extra_manifest: Option, + threads: usize, ) -> Result<()> { + let repo_path = output_dir.join("repo.zip"); + let sha256_path = output_dir.join("repo.zip.sha256.txt"); + // We currently go about this somewhat strangely; the old release // engineering process produced a Tufaceous manifest, and (the now very many // copies of) the TUF repo download-and-unpack script we use expects to be @@ -145,59 +150,54 @@ pub(crate) async fn build_tuf_repo( keys, expiry, true, - output_dir.join("repo.zip"), + repo_path.clone(), ) .build() .await?; - // Generate the checksum file. - let mut hasher = Sha256::new(); - let mut buf = [0; 8192]; - let mut file = File::open(output_dir.join("repo.zip")).await?; - loop { - let n = file.read(&mut buf).await?; - if n == 0 { - break; + + // Load and check the repository with Tufaceous v2 + let repo = RepositoryLoader::new() + .v1_compatibility(true) + .unsafe_blindly_trust_repo() + .compute_archive_sha256(true) + .load_zip_path(repo_path.clone(), &logger) + .await?; + let sha256 = + *repo.archive_sha256().context("archive sha256 was not calculated")?; + fs::write(sha256_path, format!("{}\n", hex::encode(sha256))).await?; + + repo.verify_targets(threads).await?; + + let mut problems = String::new(); + for problem in repo.check_problems().await? { + write!(&mut problems, "\n- {problem}")?; + let mut source = problem.source(); + while let Some(s) = source { + write!(&mut problems, ": {s}")?; + source = s.source(); } - hasher.update(&buf[..n]); } - fs::write( - output_dir.join("repo.zip.sha256.txt"), - format!("{}\n", hex::encode(&hasher.finalize())), - ) - .await?; + ensure!(problems.is_empty(), "found compatibility problems:{problems}",); // Check that we haven't stepped on any rakes by attempting to generate a // plan from the zip - let zip_bytes = std::fs::File::open(&output_dir.join("repo.zip")) - .context("error opening archive.zip")?; - let repo_hash = ArtifactHash([0u8; 32]); - let _ = ArtifactsWithPlan::from_zip( - zip_bytes, - None, - repo_hash, - ControlPlaneZonesMode::Split, - VerificationMode::BlindlyTrustAnything, - &logger, - ) - .await - .with_context(|| { - "error reading generated TUF repo (split control plane)".to_string() - })?; - - let zip_bytes = std::fs::File::open(&output_dir.join("repo.zip")) - .context("error opening archive.zip")?; - let _ = ArtifactsWithPlan::from_zip( - zip_bytes, - None, - repo_hash, - ControlPlaneZonesMode::Composite, - VerificationMode::BlindlyTrustAnything, - &logger, - ) - .await - .with_context(|| { - "error reading generated TUF repo (composite control plane)".to_string() - })?; + for mode in [ControlPlaneZonesMode::Split, ControlPlaneZonesMode::Composite] + { + let file = File::open(&repo_path).await?.into_std().await; + let repo_hash = ArtifactHash(sha256); + let _ = ArtifactsWithPlan::from_zip( + file, + None, + repo_hash, + mode, + VerificationMode::BlindlyTrustAnything, + &logger, + ) + .await + .with_context(|| { + format!("error reading generated TUF repo ({mode:?} control plane)") + })?; + } Ok(()) } diff --git a/dev-tools/releng/src/tuf_v2.rs b/dev-tools/releng/src/tuf_v2.rs new file mode 100644 index 00000000000..81c66a91fdb --- /dev/null +++ b/dev-tools/releng/src/tuf_v2.rs @@ -0,0 +1,182 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::error::Error as _; +use std::fmt::Write; + +use anyhow::{Context, Result, ensure}; +use camino::Utf8PathBuf; +use chrono::Utc; +use fs_err::tokio as fs; +use semver::Version; +use slog::Logger; +use tokio::sync::{mpsc, oneshot}; +use tufaceous_artifact_v2::{OsVariant, RotSlot}; +use tufaceous_v2::{RepositoryLoader, edit::RepositoryEditor}; + +pub(crate) async fn build_tuf_repo( + logger: Logger, + output_dir: Utf8PathBuf, + editor: SharedEditor, + threads: usize, +) -> Result<()> { + let repo_path = output_dir.join("repo-v2.zip"); + let sha256_path = output_dir.join("repo-v2.zip.sha256.txt"); + + editor + .take_editor() + .await? + .finish() + .await? + .generate_root() + .sign() + .await? + .write_zip_file(&repo_path, Utc::now()) + .await?; + + let repo = RepositoryLoader::new() + .unsafe_blindly_trust_repo() + .compute_archive_sha256(true) + .load_zip_path(repo_path, &logger) + .await?; + let sha256 = + repo.archive_sha256().context("archive sha256 was not calculated")?; + fs::write(sha256_path, format!("{}\n", hex::encode(sha256))).await?; + + repo.verify_targets(threads).await?; + + let mut problems = String::new(); + for problem in repo.check_problems().await? { + write!(&mut problems, "\n- {problem}")?; + let mut source = problem.source(); + while let Some(s) = source { + write!(&mut problems, ": {s}")?; + source = s.source(); + } + } + ensure!(problems.is_empty(), "found compatibility problems:{problems}",); + + Ok(()) +} + +#[derive(Clone)] +pub(crate) struct SharedEditor(mpsc::Sender); + +enum EditorRequest { + AddInput(AddKind, Utf8PathBuf, oneshot::Sender>), + TakeEditor(oneshot::Sender>), +} + +enum AddKind { + MeasurementCorpus, + OsImageDir(OsVariant), + RotArchive(RotSlot), + RotBootloaderArchive, + SpArchive, + ZoneImage, +} + +impl SharedEditor { + pub(crate) fn new(system_version: Version) -> Result { + let mut editor_cell = Some(RepositoryEditor::new(system_version)?); + let (tx, mut rx) = mpsc::channel(1); + tokio::task::spawn(async move { + while let Some(editor) = editor_cell.take() + && let Some(msg) = rx.recv().await + { + match msg { + EditorRequest::AddInput(kind, path, tx) => { + let result = match kind { + AddKind::MeasurementCorpus => { + editor.add_measurement_corpus(path).await + } + AddKind::OsImageDir(os_variant) => { + editor.add_os_image_dir(os_variant, path).await + } + AddKind::RotArchive(rot_slot) => { + editor.add_rot_archive(rot_slot, path).await + } + AddKind::RotBootloaderArchive => { + editor.add_rot_bootloader_archive(path).await + } + AddKind::SpArchive => { + editor.add_sp_archive(path).await + } + AddKind::ZoneImage => { + editor.add_zone_image(path).await + } + }; + match result { + Ok(editor) => { + editor_cell = Some(editor); + tx.send(Ok(())).ok(); + } + Err(err) => { + tx.send(Err(err.into())).ok(); + } + } + } + EditorRequest::TakeEditor(tx) => { + tx.send(editor).ok(); + } + } + } + }); + Ok(Self(tx)) + } + + async fn add(&self, kind: AddKind, path: Utf8PathBuf) -> Result<()> { + let (tx, rx) = oneshot::channel(); + let req = EditorRequest::AddInput(kind, path, tx); + ensure!(self.0.send(req).await.is_ok(), "shared editor task hung up"); + rx.await.context("shared editor task hung up")? + } + + pub(crate) async fn add_measurement_corpus( + &self, + path: Utf8PathBuf, + ) -> Result<()> { + self.add(AddKind::MeasurementCorpus, path).await + } + + pub(crate) async fn add_os_image_dir( + &self, + os_variant: OsVariant, + path: Utf8PathBuf, + ) -> Result<()> { + self.add(AddKind::OsImageDir(os_variant), path).await + } + + pub(crate) async fn add_rot_archive( + &self, + rot_slot: RotSlot, + path: Utf8PathBuf, + ) -> Result<()> { + self.add(AddKind::RotArchive(rot_slot), path).await + } + + pub(crate) async fn add_rot_bootloader_archive( + &self, + path: Utf8PathBuf, + ) -> Result<()> { + self.add(AddKind::RotBootloaderArchive, path).await + } + + pub(crate) async fn add_sp_archive(&self, path: Utf8PathBuf) -> Result<()> { + self.add(AddKind::SpArchive, path).await + } + + pub(crate) async fn add_zone_image(&self, path: Utf8PathBuf) -> Result<()> { + self.add(AddKind::ZoneImage, path).await + } + + pub(crate) async fn take_editor( + &self, + ) -> Result> { + let (tx, rx) = oneshot::channel(); + let req = EditorRequest::TakeEditor(tx); + ensure!(self.0.send(req).await.is_ok(), "shared editor task hung up"); + rx.await.context("shared editor task hung up") + } +} diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 634077d7307..7e03b6525d7 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -31,6 +31,7 @@ bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.11.0", default-features = false, features = ["serde", "std"] } bstr = { version = "1.12.1" } buf-list = { version = "1.1.2", default-features = false, features = ["tokio1"] } +byte-wrapper = { version = "0.1.0", features = ["schemars08", "serde"] } bytes = { version = "1.11.1", features = ["serde"] } camino = { version = "1.2.2", default-features = false, features = ["serde1"] } chrono = { version = "0.4.44", features = ["serde"] } @@ -178,6 +179,7 @@ bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.11.0", default-features = false, features = ["serde", "std"] } bstr = { version = "1.12.1" } buf-list = { version = "1.1.2", default-features = false, features = ["tokio1"] } +byte-wrapper = { version = "0.1.0", features = ["schemars08", "serde"] } bytes = { version = "1.11.1", features = ["serde"] } camino = { version = "1.2.2", default-features = false, features = ["serde1"] } chrono = { version = "0.4.44", features = ["serde"] }