[RFD 621 1/n] releng: generate Tufaceous v2 repos

Cargo.toml

@@ -868,10 +868,12 @@ tough = { version = "0.22.0" }
 transceiver-controller = { git = "https://github.com/oxidecomputer/transceiver-control", features = [ "api-traits" ] }
 transient-dns-server = { path = "dns-server/transient" }
 trybuild = "1.0.106"
-tufaceous = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" }
-tufaceous-artifact = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main", features = ["proptest", "schemars"] }
-tufaceous-brand-metadata = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" }
-tufaceous-lib = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main" }
+tufaceous = { git = "https://github.com/oxidecomputer/tufaceous", branch = "v1" }
+tufaceous-artifact = { git = "https://github.com/oxidecomputer/tufaceous", branch = "v1", features = ["proptest", "schemars"] }
+tufaceous-brand-metadata = { git = "https://github.com/oxidecomputer/tufaceous", branch = "v1" }
+tufaceous-lib = { git = "https://github.com/oxidecomputer/tufaceous", branch = "v1" }
+tufaceous-v2 = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main", package = "tufaceous" }
+tufaceous-artifact-v2 = { git = "https://github.com/oxidecomputer/tufaceous", branch = "main", package = "tufaceous-artifact" }
 tui-tree-widget = "0.23.1"
 typed-rng = { path = "typed-rng" }
 typify = "0.3.0"
@@ -1110,7 +1112,8 @@ opt-level = 3
 # tufaceous-artifact = { path = "../tufaceous/artifact" }
 # tufaceous-brand-metadata = { path = "../tufaceous/brand-metadata" }
 # tufaceous-lib = { path = "../tufaceous/lib" }
-# Extra slash here to pretend to be a different source.
+# tufaceous-v2 = { path = "../tufaceous/lib", package = "tufaceous" }
+# tufaceous-artifact-v2 = { path = "../tufaceous/artifact", package = "tufaceous-artifact" }
 
 # [patch."https://github.com/oxidecomputer/typify"]
 # typify = { path = "../typify/typify" }

dev-tools/releng/Cargo.toml

@@ -24,7 +24,6 @@ reqwest.workspace = true
 semver.workspace = true
 serde.workspace = true
 serde_json.workspace = true
-sha2.workspace = true
 shell-words.workspace = true
 slog.workspace = true
 slog-async.workspace = true
@@ -33,7 +32,9 @@ tar.workspace = true
 tokio = { workspace = true, features = ["full"] }
 toml.workspace = true
 tufaceous-artifact.workspace = true
+tufaceous-artifact-v2.workspace = true
 tufaceous-lib.workspace = true
+tufaceous-v2.workspace = true
 update-common.workspace = true
 
 [lints]

dev-tools/releng/src/hubris.rs

@@ -8,6 +8,7 @@ use std::io::ErrorKind;
 
 use anyhow::Context as _;
 use anyhow::Result;
+use anyhow::bail;
 use anyhow::ensure;
 use camino::Utf8PathBuf;
 use fs_err::tokio as fs;
@@ -19,6 +20,7 @@ use slog::Logger;
 use slog::warn;
 use tufaceous_artifact::ArtifactVersion;
 use tufaceous_artifact::KnownArtifactKind;
+use tufaceous_artifact_v2::RotSlot;
 use tufaceous_lib::assemble::DeserializedArtifactData;
 use tufaceous_lib::assemble::DeserializedArtifactSource;
 use tufaceous_lib::assemble::DeserializedFileArtifactSource;
@@ -53,6 +55,7 @@ pub(crate) async fn fetch_hubris_artifacts(
     env: Environment,
     client: reqwest::Client,
     output_dir: Utf8PathBuf,
+    editor: crate::tuf_v2::SharedEditor,
 ) -> Result<()> {
     let output_dir = output_dir.join(format!("hubris-{}", env.short_name));
     let ctx = Context { logger, env, client, output_dir };
@@ -61,6 +64,8 @@ pub(crate) async fn fetch_hubris_artifacts(
     // This could be parallelized with FuturesUnordered but in practice this
     // takes less time than OS builds.
 
+    // A partial Tufaceous v1 manifest, written to
+    // $output_dir/hubris-$env/manifest.toml.
     let mut manifest = DeserializedManifest {
         system_version: Version::new(0, 0, 0),
         artifacts: BTreeMap::new(),
@@ -74,25 +79,34 @@ pub(crate) async fn fetch_hubris_artifacts(
         let str = String::from_utf8(data).with_context(|| {
             format!("hubris artifact manifest {} was not UTF-8", hash)
         })?;
-        let hash_manifest: Manifest =
-            toml::from_str(&str).with_context(|| {
+        let hash_manifest: PermslipManifest = toml::from_str(&str)
+            .with_context(|| {
                 format!(
                     "failed to deserialize hubris artifact manifest {}",
                     hash
                 )
             })?;
         for (kind, artifacts) in hash_manifest.artifact {
             for artifact in artifacts {
-                let source = match artifact.source {
-                    Source::File(file) => {
-                        let path = ctx.fetch_hash(file.hash, "zip").await?.path;
-                        DeserializedArtifactSource::File { path }
-                    }
-                    Source::CompositeRot { archive_a, archive_b } => {
+                let source = match (kind, artifact.source) {
+                    (
+                        PermslipArtifactKind::GimletRot
+                        | PermslipArtifactKind::SwitchRot
+                        | PermslipArtifactKind::PscRot,
+                        PermslipSource::CompositeRot { archive_a, archive_b },
+                    ) => {
                         let path_a =
                             ctx.fetch_hash(archive_a.hash, "zip").await?.path;
+                        editor
+                            .add_rot_archive(RotSlot::A, path_a.clone())
+                            .await?;
+
                         let path_b =
                             ctx.fetch_hash(archive_b.hash, "zip").await?.path;
+                        editor
+                            .add_rot_archive(RotSlot::B, path_b.clone())
+                            .await?;
+
                         DeserializedArtifactSource::CompositeRot {
                             archive_a: DeserializedFileArtifactSource::File {
                                 path: path_a,
@@ -102,8 +116,31 @@ pub(crate) async fn fetch_hubris_artifacts(
                             },
                         }
                     }
+                    (
+                        PermslipArtifactKind::GimletRotBootloader
+                        | PermslipArtifactKind::SwitchRotBootloader
+                        | PermslipArtifactKind::PscRotBootloader,
+                        PermslipSource::File(file),
+                    ) => {
+                        let path = ctx.fetch_hash(file.hash, "zip").await?.path;
+                        editor.add_rot_bootloader_archive(path.clone()).await?;
+                        DeserializedArtifactSource::File { path }
+                    }
+                    (
+                        PermslipArtifactKind::GimletSp
+                        | PermslipArtifactKind::SwitchSp
+                        | PermslipArtifactKind::PscSp,
+                        PermslipSource::File(file),
+                    ) => {
+                        let path = ctx.fetch_hash(file.hash, "zip").await?.path;
+                        editor.add_sp_archive(path.clone()).await?;
+                        DeserializedArtifactSource::File { path }
+                    }
+                    (kind @ _, source @ _) => {
+                        bail!("unexpected source {source:?} for kind {kind:?}");
+                    }
                 };
-                manifest.artifacts.entry(kind).or_default().push(
+                manifest.artifacts.entry(kind.into()).or_default().push(
                     DeserializedArtifactData {
                         name: artifact.name,
                         version: artifact.version,
@@ -112,8 +149,11 @@ pub(crate) async fn fetch_hubris_artifacts(
                 );
             }
         }
-        if let Some(FileSource { hash }) = hash_manifest.measurement_corpus {
+        if let Some(PermslipFileSource { hash }) =
+            hash_manifest.measurement_corpus
+        {
             let Output { data, path } = ctx.fetch_hash(hash, "corim").await?;
+            editor.add_measurement_corpus(path.clone()).await?;
             let corim = rats_corim::Corim::from_bytes(&data)?;
             manifest
                 .artifacts
@@ -283,28 +323,67 @@ impl Context {
 // tufaceous-lib, except that the source is a hash instead of a file path. This
 // hash is used to download the artifact from Permission Slip.
 #[derive(Deserialize)]
-struct Manifest {
-    artifact: HashMap<KnownArtifactKind, Vec<Artifact>>,
+struct PermslipManifest {
+    artifact: HashMap<PermslipArtifactKind, Vec<PermslipArtifact>>,
     // Add a default for backwards compatibility
-    measurement_corpus: Option<FileSource>,
+    measurement_corpus: Option<PermslipFileSource>,
+}
+
+#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq, Hash)]
+#[serde(rename_all = "snake_case")]
+enum PermslipArtifactKind {
+    GimletRot,
+    SwitchRot,
+    PscRot,
+    GimletRotBootloader,
+    SwitchRotBootloader,
+    PscRotBootloader,
+    GimletSp,
+    SwitchSp,
+    PscSp,
+}
+
+impl From<PermslipArtifactKind> for KnownArtifactKind {
+    fn from(value: PermslipArtifactKind) -> Self {
+        match value {
+            PermslipArtifactKind::GimletRot => KnownArtifactKind::GimletRot,
+            PermslipArtifactKind::SwitchRot => KnownArtifactKind::SwitchRot,
+            PermslipArtifactKind::PscRot => KnownArtifactKind::PscRot,
+            PermslipArtifactKind::GimletRotBootloader => {
+                KnownArtifactKind::GimletRotBootloader
+            }
+            PermslipArtifactKind::SwitchRotBootloader => {
+                KnownArtifactKind::SwitchRotBootloader
+            }
+            PermslipArtifactKind::PscRotBootloader => {
+                KnownArtifactKind::PscRotBootloader
+            }
+            PermslipArtifactKind::GimletSp => KnownArtifactKind::GimletSp,
+            PermslipArtifactKind::SwitchSp => KnownArtifactKind::SwitchSp,
+            PermslipArtifactKind::PscSp => KnownArtifactKind::PscSp,
+        }
+    }
 }
 
 #[derive(Deserialize)]
-struct Artifact {
+struct PermslipArtifact {
     name: String,
     version: ArtifactVersion,
-    source: Source,
+    source: PermslipSource,
 }
 
-#[derive(Deserialize)]
+#[derive(Debug, Deserialize)]
 #[serde(tag = "kind", rename_all = "kebab-case")]
-enum Source {
-    File(FileSource),
-    CompositeRot { archive_a: FileSource, archive_b: FileSource },
+enum PermslipSource {
+    File(PermslipFileSource),
+    CompositeRot {
+        archive_a: PermslipFileSource,
+        archive_b: PermslipFileSource,
+    },
 }
 
-#[derive(Deserialize)]
-struct FileSource {
+#[derive(Debug, Deserialize)]
+struct PermslipFileSource {
     #[serde(deserialize_with = "deserialize_hash")]
     hash: blake3::Hash,
 }

dev-tools/releng/src/main.rs

@@ -7,6 +7,7 @@ mod helios;
 mod hubris;
 mod job;
 mod tuf;
+mod tuf_v2;
 
 use std::collections::BTreeMap;
 use std::path::PathBuf;
@@ -38,6 +39,7 @@ use slog::info;
 use slog_term::FullFormat;
 use slog_term::TermDecorator;
 use tokio::sync::Semaphore;
+use tufaceous_artifact_v2::OsVariant;
 
 use crate::cmd::Command;
 use crate::job::Jobs;
@@ -247,11 +249,10 @@ async fn main() -> Result<()> {
     // Read pins.toml.
     let pins = omicron_pins::Pins::read_from_dir(&metadata.workspace_root)?;
 
-    let permits = Arc::new(Semaphore::new(
-        std::thread::available_parallelism()
-            .context("couldn't get available parallelism")?
-            .into(),
-    ));
+    let threads = std::thread::available_parallelism()
+        .context("couldn't get available parallelism")?
+        .into();
+    let permits = Arc::new(Semaphore::new(threads));
 
     let commit = Command::new(&args.git_bin)
         .args(["rev-parse", "HEAD"])
@@ -448,6 +449,7 @@ async fn main() -> Result<()> {
     fs::create_dir_all(&args.output_dir).await?;
 
     // DEFINE JOBS ============================================================
+    let editor_v2 = tuf_v2::SharedEditor::new(version.clone())?;
     let tempdir = camino_tempfile::tempdir()
         .context("failed to create temporary directory")?;
     let mut jobs = Jobs::new(&logger, permits.clone(), &args.output_dir);
@@ -650,13 +652,13 @@ async fn main() -> Result<()> {
             commit.chars().take(7).collect::<String>(),
             Utc::now().format("%Y-%m-%d %H:%M")
         );
-
+        let output_dir = args.output_dir.join(format!("os-{}", target));
         let mut image_cmd = Command::new("ptime")
             .arg("-m")
             .arg(args.helios_dir.join("helios-build"))
             .arg("experiment-image")
             .arg("-o") // output directory for image
-            .arg(args.output_dir.join(format!("os-{}", target)))
+            .arg(&output_dir)
             .arg("-F") // pass extra image builder features
             .arg(format!("optever={opte_version}"))
             .arg("-P") // include all files from extra proto area
@@ -730,15 +732,26 @@ async fn main() -> Result<()> {
                 .arg(format!("helios-dev=file://{p5p_path}"));
         }
 
+        let image_job_name = format!("{target}-image");
         let image_job = jobs
-            .push_command(format!("{target}-image"), image_cmd)
+            .push_command(&image_job_name, image_cmd)
             .after("helios-setup")
             .after("helios-incorp")
             .after(format!("{target}-proto"));
 
         if opte_override.is_some() {
             image_job.after(format!("{target}-opte-p5p"));
         }
+
+        let editor = editor_v2.clone();
+        jobs.push(format!("tuf-v2-{target}"), async move {
+            let os_variant = match target {
+                Target::Host => OsVariant::Host,
+                Target::Recovery => OsVariant::Recovery,
+            };
+            editor.add_os_image_dir(os_variant, output_dir).await
+        })
+        .after(image_job_name);
     }
     // Build the recovery target after we build the host target (and have
     // finished verifying its binaries). Only one of these will build at a time
@@ -762,6 +775,19 @@ async fn main() -> Result<()> {
     stamp_packages!("tuf-stamp", Target::Host, TUF_PACKAGES)
         .after("host-stamp")
         .after("recovery-stamp");
+    {
+        let editor = editor_v2.clone();
+        jobs.push("tuf-v2-zones", async move {
+            for package in TUF_PACKAGES {
+                let path = crate::WORKSPACE_DIR
+                    .join("out/versioned")
+                    .join(format!("{}.tar.gz", package));
+                editor.add_zone_image(path).await?;
+            }
+            Ok(())
+        })
+        .after("tuf-stamp");
+    }
 
     if args.verify_debug_libraries {
         // Run `cargo xtask verify-libraries`. This builds *all* binaries in
@@ -791,6 +817,7 @@ async fn main() -> Result<()> {
                 env,
                 client.clone(),
                 args.output_dir.clone(),
+                editor_v2.clone(),
             ),
         );
     }
@@ -803,6 +830,7 @@ async fn main() -> Result<()> {
             version,
             manifest,
             args.extra_manifest,
+            threads,
         ),
     )
     .after("tuf-stamp")
@@ -811,6 +839,21 @@ async fn main() -> Result<()> {
     .after("hubris-staging")
     .after("hubris-production");
 
+    jobs.push(
+        "tuf-v2-repo",
+        tuf_v2::build_tuf_repo(
+            logger.clone(),
+            args.output_dir.clone(),
+            editor_v2,
+            threads,
+        ),
+    )
+    .after("tuf-v2-host")
+    .after("tuf-v2-recovery")
+    .after("tuf-v2-zones")
+    .after("hubris-staging")
+    .after("hubris-production");
+
     // RUN JOBS ===============================================================
     let start = Instant::now();
     jobs.run_all().await?;