From 14ddfbf366455cb0eec796f2f3b2f5dbc98fb0d5 Mon Sep 17 00:00:00 2001 From: Amey Pawar Date: Sun, 5 Jul 2026 22:51:19 +0530 Subject: [PATCH] fix!: respect configured zlib compression levels instead of a hard-coded fastest level (#2024) The deflate stream used for pack entries, loose objects and thin-pack completion was hard-coded to the fastest compression level. That stemmed from `miniz_oxide` times where level 1 still compressed reasonably well - with the switch to the `zlib-rs` backend, whose level 1 is tuned for speed at the cost of ratio, pack entries grew by ~50%. Remove the hidden default entirely: `Compress`, `deflate::Write`, `output::Entry::from_data`, `input::Entry::from_data_obj`, `LookupRefDeltaObjectsIter` and `loose::Store` now take an explicit `Compression` level, while option structs default to what `git` uses so behaviour matches it out of the box: packs deflate with the zlib default (level 6, like `pack.compression`), loose objects with the fastest level (like `core.looseCompression`). The `core.compression`, `core.looseCompression` and `pack.compression` configuration keys are now understood and validated, with `-1` mapping to the zlib default just like `git`, and are respected when writing loose objects, creating packs and completing thin packs. The issue's reproduction drops from 29547 to 18724 bytes for a 72kB tree entry, back in line with what `git` produces. --- Cargo.lock | 1 + gitoxide-core/src/pack/create.rs | 12 ++++ gitoxide-core/src/pack/explode.rs | 17 ++++- gitoxide-core/src/pack/index.rs | 1 + gitoxide-core/src/pack/receive.rs | 1 + gix-features/Cargo.toml | 5 ++ gix-features/src/zlib/mod.rs | 34 +++++++++ gix-features/src/zlib/stream/deflate/mod.rs | 31 ++++---- gix-features/src/zlib/stream/deflate/tests.rs | 26 +++++-- gix-odb/src/lib.rs | 2 + gix-odb/src/sink.rs | 10 +-- gix-odb/src/store_impls/dynamic/handle.rs | 1 + gix-odb/src/store_impls/dynamic/init.rs | 8 +++ gix-odb/src/store_impls/dynamic/load_index.rs | 4 +- gix-odb/src/store_impls/loose/mod.rs | 8 +++ gix-odb/src/store_impls/loose/write.rs | 9 +-- gix-odb/tests/odb/store/loose.rs | 70 +++++++++++++++++-- gix-pack/Cargo.toml | 2 +- gix-pack/src/bundle/write/mod.rs | 3 + gix-pack/src/bundle/write/types.rs | 7 ++ gix-pack/src/cache/delta/traverse/resolve.rs | 3 +- gix-pack/src/data/input/entry.rs | 18 +++-- .../data/input/lookup_ref_delta_objects.rs | 9 ++- .../src/data/output/entry/iter_from_counts.rs | 13 +++- gix-pack/src/data/output/entry/mod.rs | 14 +++- gix-pack/tests/pack/bundle.rs | 2 + gix-pack/tests/pack/data/fuzzed.rs | 9 ++- gix-pack/tests/pack/data/input.rs | 27 +++++-- .../pack/data/output/count_and_entries.rs | 68 ++++++++++++++++++ gix-pack/tests/pack/malformed.rs | 3 +- gix/src/config/cache/init.rs | 4 ++ gix/src/config/cache/util.rs | 21 ++++++ gix/src/config/mod.rs | 4 ++ gix/src/config/tree/keys.rs | 46 ++++++++++++ gix/src/config/tree/sections/core.rs | 9 +++ gix/src/config/tree/sections/pack.rs | 5 +- gix/src/open/repository.rs | 1 + .../remote/connection/fetch/receive_pack.rs | 1 + gix/tests/gix/config/tree.rs | 44 ++++++++++++ gix/tests/gix/repository/object.rs | 7 +- 40 files changed, 490 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f485a6febe..d9df6d90021 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1880,6 +1880,7 @@ dependencies = [ "once_cell", "parking_lot", "prodash", + "serde", "thiserror 2.0.18", "walkdir", "zlib-rs", diff --git a/gitoxide-core/src/pack/create.rs b/gitoxide-core/src/pack/create.rs index 8d9a3892b1d..4fa6fd84fc1 100644 --- a/gitoxide-core/src/pack/create.rs +++ b/gitoxide-core/src/pack/create.rs @@ -109,6 +109,17 @@ where type ObjectIdIter = dyn Iterator>> + Send; let repo = gix::discover(repository_path)?.into_sync(); + let pack_compression = { + use gix::config::tree::{Core, Pack}; + let repo = repo.to_thread_local(); + let config = repo.config_snapshot(); + config + .integer(Pack::COMPRESSION) + .or_else(|| config.integer(Core::COMPRESSION)) + .map(|level| Pack::COMPRESSION.try_into_compression(Ok(level))) + .transpose()? + .unwrap_or(gix::features::zlib::Compression::DEFAULT) + }; progress.init(Some(2), progress::steps()); let tips = tips.into_iter(); let make_cancellation_err = || anyhow!("Cancelled by user"); @@ -233,6 +244,7 @@ where allow_thin_pack: thin, chunk_size, version: Default::default(), + compression: pack_compression, }, )) }; diff --git a/gitoxide-core/src/pack/explode.rs b/gitoxide-core/src/pack/explode.rs index d19d2012c2b..56b17fd2220 100644 --- a/gitoxide-core/src/pack/explode.rs +++ b/gitoxide-core/src/pack/explode.rs @@ -146,8 +146,15 @@ impl gix::objs::Write for OutputWriter { impl OutputWriter { fn new(path: Option>, compress: bool, object_hash: gix::hash::Kind) -> Self { match path { - Some(path) => OutputWriter::Loose(loose::Store::at(path.as_ref(), object_hash, None)), - None => OutputWriter::Sink(odb::sink(object_hash).compress(compress)), + Some(path) => OutputWriter::Loose(loose::Store::at( + path.as_ref(), + object_hash, + None, + gix::features::zlib::Compression::BEST_SPEED, + )), + None => OutputWriter::Sink( + odb::sink(object_hash).compress(compress.then_some(gix::features::zlib::Compression::BEST_SPEED)), + ), } } } @@ -217,7 +224,11 @@ pub fn pack_or_pack_index( let object_path = object_path.map(|p| p.as_ref().to_owned()); let out = OutputWriter::new(object_path.clone(), sink_compress, object_hash); let loose_odb = verify - .then(|| object_path.as_ref().map(|path| loose::Store::at(path, object_hash, None))) + .then(|| { + object_path.as_ref().map(|path| { + loose::Store::at(path, object_hash, None, gix::features::zlib::Compression::BEST_SPEED) + }) + }) .flatten(); let mut read_buf = Vec::new(); move |object_kind, buf, index_entry, progress| { diff --git a/gitoxide-core/src/pack/index.rs b/gitoxide-core/src/pack/index.rs index 00f58c97d19..602eddabeb7 100644 --- a/gitoxide-core/src/pack/index.rs +++ b/gitoxide-core/src/pack/index.rs @@ -83,6 +83,7 @@ pub fn from_pack( index_version: pack::index::Version::default(), object_hash: ctx.object_hash, alloc_limit_bytes: None, + compression: gix::features::zlib::Compression::BEST_SPEED, }; let out = ctx.out; let format = ctx.format; diff --git a/gitoxide-core/src/pack/receive.rs b/gitoxide-core/src/pack/receive.rs index 65546e4650e..689bf2cb609 100644 --- a/gitoxide-core/src/pack/receive.rs +++ b/gitoxide-core/src/pack/receive.rs @@ -290,6 +290,7 @@ fn receive_pack_blocking( iteration_mode: pack::data::input::Mode::Verify, object_hash, alloc_limit_bytes: None, + compression: gix::features::zlib::Compression::BEST_SPEED, }; let outcome = pack::Bundle::write_to_directory( &mut input, diff --git a/gix-features/Cargo.toml b/gix-features/Cargo.toml index d47edd1a0b2..d9c3988c0d6 100644 --- a/gix-features/Cargo.toml +++ b/gix-features/Cargo.toml @@ -61,6 +61,9 @@ zlib = ["dep:zlib-rs", "dep:thiserror"] #! ### Other +## Data structures implement `serde::Serialize` and `serde::Deserialize`. +serde = ["dep:serde"] + ## Count cache hits and misses and print that debug information on drop. ## Caches implement this by default, which costs nothing unless this feature is enabled cache-efficiency-debug = [] @@ -111,6 +114,8 @@ bytes = { version = "1.11.1", optional = true } zlib-rs = { version = "0.6.2", optional = true } thiserror = { version = "2.0.18", optional = true } +serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"] } + # Note: once_cell is kept for OnceCell type because std::sync::OnceLock::get_or_try_init() is not yet stable. # Once it's stabilized (tracking issue #109737), we can remove this dependency. once_cell = { version = "1.21.3", optional = true } diff --git a/gix-features/src/zlib/mod.rs b/gix-features/src/zlib/mod.rs index 42489d4c420..0cf1f6dce81 100644 --- a/gix-features/src/zlib/mod.rs +++ b/gix-features/src/zlib/mod.rs @@ -1,5 +1,39 @@ use zlib_rs::InflateError; +/// The compression level to use for zlib-based streams, in the range from 0 (no compression) +/// to 9 (best compression, slowest). +/// +/// Note that `git` maps its configured level of `-1` to the zlib default, which is level 6 +/// and available as [`Compression::DEFAULT`]. +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Compression(i32); + +impl Compression { + /// Do not compress at all, while still producing a valid zlib stream. + pub const NONE: Compression = Compression(0); + /// The fastest compression, with the lowest compression ratio, also known as level 1. + /// + /// This is what `git` uses for loose objects unless configured otherwise with `core.looseCompression`. + pub const BEST_SPEED: Compression = Compression(1); + /// The default compromise between speed and compression ratio, also known as level 6. + /// + /// This is what `git` uses when writing packs unless configured otherwise with `pack.compression`. + pub const DEFAULT: Compression = Compression(6); + /// The best compression ratio at the expense of speed, also known as level 9. + pub const BEST: Compression = Compression(9); + + /// Create a new instance from `level` if it is within the valid range from 0 to 9, inclusive. + pub fn new(level: i32) -> Option { + (0..=9).contains(&level).then_some(Compression(level)) + } + + /// Return the compression level as integer in the range from 0 to 9, inclusive. + pub fn level(&self) -> i32 { + self.0 + } +} + /// A type to hold all state needed for decompressing a ZLIB encoded stream. pub struct Decompress(zlib_rs::Inflate); diff --git a/gix-features/src/zlib/stream/deflate/mod.rs b/gix-features/src/zlib/stream/deflate/mod.rs index 1f7dc4d9a8e..eceafc8d37e 100644 --- a/gix-features/src/zlib/stream/deflate/mod.rs +++ b/gix-features/src/zlib/stream/deflate/mod.rs @@ -1,4 +1,4 @@ -use crate::zlib::Status; +use crate::zlib::{Compression, Status}; use zlib_rs::DeflateError; const BUF_SIZE: usize = 4096 * 8; @@ -8,6 +8,7 @@ const BUF_SIZE: usize = 4096 * 8; /// Be sure to call `flush()` when done to finalize the deflate stream. pub struct Write { compressor: Compress, + compression: Compression, inner: W, buf: [u8; BUF_SIZE], } @@ -18,7 +19,8 @@ where { fn clone(&self) -> Self { Write { - compressor: impls::new_compress(), + compressor: impls::new_compress(self.compression), + compression: self.compression, inner: self.inner.clone(), buf: self.buf, } @@ -28,12 +30,6 @@ where /// Hold all state needed for compressing data. pub struct Compress(zlib_rs::Deflate); -impl Default for Compress { - fn default() -> Self { - Self::new() - } -} - impl Compress { /// The number of bytes that were read from the input. pub fn total_in(&self) -> u64 { @@ -45,9 +41,9 @@ impl Compress { self.0.total_out() } - /// Create a new instance - this allocates so should be done with care. - pub fn new() -> Self { - let config = zlib_rs::DeflateConfig::best_speed(); + /// Create a new instance compressing with `level` - this allocates so should be done with care. + pub fn new(level: Compression) -> Self { + let config = zlib_rs::DeflateConfig::new(level.level()); let header = true; let inner = zlib_rs::Deflate::new(config.level, header, config.window_bits as u8); Self(inner) @@ -145,21 +141,22 @@ pub enum FlushCompress { mod impls { use std::io; - use crate::zlib::Status; use crate::zlib::stream::deflate::{self, Compress, FlushCompress}; + use crate::zlib::{Compression, Status}; - pub(crate) fn new_compress() -> Compress { - Compress::new() + pub(crate) fn new_compress(level: Compression) -> Compress { + Compress::new(level) } impl deflate::Write where W: io::Write, { - /// Create a new instance writing compressed bytes to `inner`. - pub fn new(inner: W) -> deflate::Write { + /// Create a new instance writing bytes compressed with `level` to `inner`. + pub fn new(inner: W, level: Compression) -> deflate::Write { deflate::Write { - compressor: new_compress(), + compressor: new_compress(level), + compression: level, inner, buf: [0; deflate::BUF_SIZE], } diff --git a/gix-features/src/zlib/stream/deflate/tests.rs b/gix-features/src/zlib/stream/deflate/tests.rs index cbd2628bd1e..5f7f52b1b93 100644 --- a/gix-features/src/zlib/stream/deflate/tests.rs +++ b/gix-features/src/zlib/stream/deflate/tests.rs @@ -6,8 +6,8 @@ mod deflate_stream { use bstr::ByteSlice; - use crate::zlib::Decompress; use crate::zlib::stream::deflate; + use crate::zlib::{Compression, Decompress}; /// Provide streaming decompression using the `std::io::Read` trait. /// If `std::io::BufReader` is used, an allocation for the input buffer will be performed. @@ -55,7 +55,7 @@ mod deflate_stream { #[test] fn all_at_once() -> Result<(), Box> { - let mut w = deflate::Write::new(Vec::new()); + let mut w = deflate::Write::new(Vec::new(), Compression::BEST_SPEED); assert_eq!(w.write(b"hello")?, 5); w.flush()?; @@ -65,6 +65,24 @@ mod deflate_stream { assert_deflate_buffer(out, b"hello") } + #[test] + fn higher_levels_compress_better() -> Result<(), Box> { + let data: Vec = (0..128 * 1024).map(|i| (i % 100) as u8).collect(); + let mut sizes = Vec::new(); + for level in [Compression::NONE, Compression::BEST_SPEED, Compression::DEFAULT] { + let mut w = deflate::Write::new(Vec::new(), level); + w.write_all(&data)?; + w.flush()?; + assert_deflate_buffer(w.inner.clone(), &data)?; + sizes.push(w.inner.len()); + } + assert!( + sizes[0] > sizes[1] && sizes[1] > sizes[2], + "each level compresses better than the one before it: {sizes:?}" + ); + Ok(()) + } + fn assert_deflate_buffer(out: Vec, expected: &[u8]) -> Result<(), Box> { let mut actual = Vec::new(); InflateReader::from_read(out.as_slice()).read_to_end(&mut actual)?; @@ -74,7 +92,7 @@ mod deflate_stream { #[test] fn big_file_small_writes() -> Result<(), Box> { - let mut w = deflate::Write::new(Vec::new()); + let mut w = deflate::Write::new(Vec::new(), Compression::BEST_SPEED); let bytes = include_bytes!( "../../../../../gix-odb/tests/fixtures/objects/pack/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack" ); @@ -88,7 +106,7 @@ mod deflate_stream { #[test] fn big_file_a_few_big_writes() -> Result<(), Box> { - let mut w = deflate::Write::new(Vec::new()); + let mut w = deflate::Write::new(Vec::new(), Compression::BEST_SPEED); let bytes = include_bytes!( "../../../../../gix-odb/tests/fixtures/objects/pack/pack-11fdfa9e156ab73caae3b6da867192221f2089c2.pack" ); diff --git a/gix-odb/src/lib.rs b/gix-odb/src/lib.rs index ced2823d4b3..76ab5546da3 100644 --- a/gix-odb/src/lib.rs +++ b/gix-odb/src/lib.rs @@ -170,6 +170,8 @@ pub struct Store { object_hash: gix_hash::Kind, /// The maximum size of a single allocation caused by user-controlled on-disk pack data. alloc_limit_bytes: Option, + /// The compression level to use when writing loose objects. + loose_compression: gix_features::zlib::Compression, } /// Create a new cached handle to the object store with support for additional options. diff --git a/gix-odb/src/sink.rs b/gix-odb/src/sink.rs index ae22efa16bc..67938cc349e 100644 --- a/gix-odb/src/sink.rs +++ b/gix-odb/src/sink.rs @@ -8,13 +8,9 @@ use gix_features::zlib::stream::deflate; use crate::Sink; impl Sink { - /// Enable or disable compression. Compression is disabled by default - pub fn compress(mut self, enable: bool) -> Self { - if enable { - self.compressor = Some(RefCell::new(deflate::Write::new(io::sink()))); - } else { - self.compressor = None; - } + /// Compress with the given level, or disable compression with `None`. Compression is disabled by default. + pub fn compress(mut self, compression: Option) -> Self { + self.compressor = compression.map(|level| RefCell::new(deflate::Write::new(io::sink(), level))); self } } diff --git a/gix-odb/src/store_impls/dynamic/handle.rs b/gix-odb/src/store_impls/dynamic/handle.rs index e0041f133b3..94f0dbfd87b 100644 --- a/gix-odb/src/store_impls/dynamic/handle.rs +++ b/gix-odb/src/store_impls/dynamic/handle.rs @@ -353,6 +353,7 @@ impl TryFrom<&super::Store> for super::Store { use_multi_pack_index: false, alloc_limit_bytes: s.alloc_limit_bytes, current_dir: s.current_dir.clone().into(), + loose_compression: s.loose_compression, }, ) } diff --git a/gix-odb/src/store_impls/dynamic/init.rs b/gix-odb/src/store_impls/dynamic/init.rs index d50d3c52c97..5b59a1e69f2 100644 --- a/gix-odb/src/store_impls/dynamic/init.rs +++ b/gix-odb/src/store_impls/dynamic/init.rs @@ -23,6 +23,11 @@ pub struct Options { /// The current directory of the process at the time of instantiation. /// If unset, it will be retrieved using `gix_fs::current_dir(false)`. pub current_dir: Option, + /// The compression level to use when writing loose objects. + /// + /// Defaults to [`Compression::BEST_SPEED`](gix_features::zlib::Compression::BEST_SPEED), which is + /// also what `git` uses unless configured otherwise with `core.looseCompression` or `core.compression`. + pub loose_compression: gix_features::zlib::Compression, } impl Default for Options { @@ -33,6 +38,7 @@ impl Default for Options { use_multi_pack_index: true, alloc_limit_bytes: None, current_dir: None, + loose_compression: gix_features::zlib::Compression::BEST_SPEED, } } } @@ -83,6 +89,7 @@ impl Store { use_multi_pack_index, alloc_limit_bytes, current_dir, + loose_compression, }: Options, ) -> std::io::Result { let _span = gix_features::trace::detail!("gix_odb::Store::at()"); @@ -138,6 +145,7 @@ impl Store { use_multi_pack_index, object_hash, alloc_limit_bytes, + loose_compression, num_handles_stable: Default::default(), num_handles_unstable: Default::default(), num_disk_state_consolidation: Default::default(), diff --git a/gix-odb/src/store_impls/dynamic/load_index.rs b/gix-odb/src/store_impls/dynamic/load_index.rs index 3b4eb8f0961..d6d42cc1081 100644 --- a/gix-odb/src/store_impls/dynamic/load_index.rs +++ b/gix-odb/src/store_impls/dynamic/load_index.rs @@ -231,7 +231,9 @@ impl super::Store { Arc::new( db_paths .iter() - .map(|path| crate::loose::Store::at(path, self.object_hash, self.alloc_limit_bytes)) + .map(|path| { + crate::loose::Store::at(path, self.object_hash, self.alloc_limit_bytes, self.loose_compression) + }) .collect::>(), ) } else { diff --git a/gix-odb/src/store_impls/loose/mod.rs b/gix-odb/src/store_impls/loose/mod.rs index 0a7f0c1861b..17dbbecce5b 100644 --- a/gix-odb/src/store_impls/loose/mod.rs +++ b/gix-odb/src/store_impls/loose/mod.rs @@ -14,6 +14,8 @@ pub struct Store { pub(crate) object_hash: gix_hash::Kind, /// The maximum size of a single allocation caused by user-controlled loose object data. pub(crate) alloc_limit_bytes: Option, + /// The compression level to use when writing loose objects. + pub(crate) compression: gix_features::zlib::Compression, } /// Initialization @@ -27,15 +29,21 @@ impl Store { /// /// `alloc_limit_bytes` limits allocations caused by loose object bodies declared on disk, useful for untrusted input. /// Use `None` to disable the limit. + /// + /// `compression` is the level to deflate loose objects with when writing them. `git` uses + /// [`Compression::BEST_SPEED`](gix_features::zlib::Compression::BEST_SPEED) unless configured + /// otherwise with `core.looseCompression` or `core.compression`. pub fn at( objects_directory: impl Into, object_hash: gix_hash::Kind, alloc_limit_bytes: Option, + compression: gix_features::zlib::Compression, ) -> Store { Store { path: objects_directory.into(), object_hash, alloc_limit_bytes, + compression, } } diff --git a/gix-odb/src/store_impls/loose/write.rs b/gix-odb/src/store_impls/loose/write.rs index cfc64d024fe..83c84644780 100644 --- a/gix-odb/src/store_impls/loose/write.rs +++ b/gix-odb/src/store_impls/loose/write.rs @@ -170,13 +170,14 @@ impl Store { let perms = std::fs::Permissions::from_mode(0o444); builder.permissions(perms); } - Ok(deflate::Write::new(builder.tempfile_in(&self.path).map_err(|err| { - Error::Io { + Ok(deflate::Write::new( + builder.tempfile_in(&self.path).map_err(|err| Error::Io { source: err.into(), message: "create named temp file in", path: self.path.to_owned(), - } - })?)) + })?, + self.compression, + )) } fn finalize_object( diff --git a/gix-odb/tests/odb/store/loose.rs b/gix-odb/tests/odb/store/loose.rs index 9a63d9ef953..5e310e65fea 100644 --- a/gix-odb/tests/odb/store/loose.rs +++ b/gix-odb/tests/odb/store/loose.rs @@ -8,11 +8,21 @@ use pretty_assertions::assert_eq; use crate::hex_to_id; fn ldb() -> Store { - Store::at(fixture_path("objects"), gix_hash::Kind::Sha1, None) + Store::at( + fixture_path("objects"), + gix_hash::Kind::Sha1, + None, + gix_features::zlib::Compression::BEST_SPEED, + ) } fn limited_ldb(limit: usize) -> Store { - Store::at(fixture_path("objects"), gix_hash::Kind::Sha1, Some(limit)) + Store::at( + fixture_path("objects"), + gix_hash::Kind::Sha1, + Some(limit), + gix_features::zlib::Compression::BEST_SPEED, + ) } pub fn object_ids() -> Vec { @@ -52,10 +62,40 @@ mod write { use crate::store::loose::{locate_oid, object_ids}; + #[test] + fn compression_level_is_respected() -> crate::Result { + use gix_features::zlib::Compression; + let data: Vec = (0..64 * 1024).map(|i| (i % 100) as u8).collect(); + let mut sizes = Vec::new(); + for level in [Compression::NONE, Compression::BEST] { + let dir = gix_testtools::tempfile::tempdir()?; + let db = loose::Store::at(dir.path(), gix_hash::Kind::Sha1, None, level); + let id = db.write_buf(gix_object::Kind::Blob, &data)?; + sizes.push(db.object_path(&id).metadata()?.len()); + + let mut buf = Vec::new(); + assert_eq!( + db.try_find(&id, &mut buf)?.expect("just written").data, + data, + "written objects can be read back regardless of level" + ); + } + assert!( + sizes[0] > sizes[1], + "the best level compresses better than no compression at all: {sizes:?}" + ); + Ok(()) + } + #[test] fn read_and_write() -> crate::Result { let dir = gix_testtools::tempfile::tempdir()?; - let db = loose::Store::at(dir.path(), gix_hash::Kind::Sha1, None); + let db = loose::Store::at( + dir.path(), + gix_hash::Kind::Sha1, + None, + gix_features::zlib::Compression::BEST_SPEED, + ); let mut buf = Vec::new(); let mut buf2 = Vec::new(); @@ -98,6 +138,7 @@ mod write { crate::scripted_fixture_read_only("repo_with_loose_objects.sh")?.join(".git/objects"), hk, None, + gix_features::zlib::Compression::BEST_SPEED, ); let expected_perm = git_store .object_path(&gix_hash::ObjectId::empty_blob(hk)) @@ -105,7 +146,7 @@ mod write { .permissions(); let tmp = gix_testtools::tempfile::TempDir::new()?; - let store = loose::Store::at(tmp.path(), hk, None); + let store = loose::Store::at(tmp.path(), hk, None, gix_features::zlib::Compression::BEST_SPEED); store.write_buf(gix_object::Kind::Blob, &[])?; let actual_perm = store .object_path(&gix_hash::ObjectId::empty_blob(hk)) @@ -123,7 +164,12 @@ mod write { let dir = gix_testtools::tempfile::tempdir()?; fn write_empty_trees(dir: &std::path::Path) { - let db = loose::Store::at(dir, gix_hash::Kind::Sha1, None); + let db = loose::Store::at( + dir, + gix_hash::Kind::Sha1, + None, + gix_features::zlib::Compression::BEST_SPEED, + ); let empty_tree = gix_object::Tree::empty(); for _ in 0..2 { let id = db.write(&empty_tree).expect("works"); @@ -194,7 +240,12 @@ mod lookup_prefix { b"fake", ) .unwrap(); - let store = gix_odb::loose::Store::at(objects_dir.path(), gix_hash::Kind::Sha1, None); + let store = gix_odb::loose::Store::at( + objects_dir.path(), + gix_hash::Kind::Sha1, + None, + gix_features::zlib::Compression::BEST_SPEED, + ); let input_id = hex_to_id("37d4e6c5c48ba0d245164c4e10d5f41140cab980"); let prefix = gix_hash::Prefix::new(&input_id, 4).unwrap(); assert_eq!( @@ -259,7 +310,12 @@ mod find { let base = tmp.path().join("aa"); std::fs::create_dir(&base)?; std::fs::write(base.join("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), [])?; - let db = loose::Store::at(tmp.path(), gix_hash::Kind::Sha1, None); + let db = loose::Store::at( + tmp.path(), + gix_hash::Kind::Sha1, + None, + gix_features::zlib::Compression::BEST_SPEED, + ); let mut buf = Vec::new(); let id = hex_to_id("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); diff --git a/gix-pack/Cargo.toml b/gix-pack/Cargo.toml index f1436c71507..55c5d0d210e 100644 --- a/gix-pack/Cargo.toml +++ b/gix-pack/Cargo.toml @@ -36,7 +36,7 @@ pack-cache-lru-dynamic = ["dep:clru"] ## If set, select algorithms may additionally use a full-object cache which is queried before the pack itself. object-cache-dynamic = ["dep:clru", "dep:gix-hashtable"] ## Data structures implement `serde::Serialize` and `serde::Deserialize`. -serde = ["dep:serde", "gix-object/serde"] +serde = ["dep:serde", "gix-object/serde", "gix-features/serde"] ## Enable parallel algorithms. parallel = ["gix-features/parallel"] ## Make it possible to compile to the `wasm32-unknown-unknown` target. diff --git a/gix-pack/src/bundle/write/mod.rs b/gix-pack/src/bundle/write/mod.rs index 9d01783237c..b4045ce4d63 100644 --- a/gix-pack/src/bundle/write/mod.rs +++ b/gix-pack/src/bundle/write/mod.rs @@ -101,6 +101,7 @@ impl crate::Bundle { object_hash, )?, thin_pack_lookup, + options.compression, ); let pack_version = pack_entries_iter.inner.version(); let pack_entries_iter = data::input::EntriesToBytesIter::new( @@ -209,6 +210,7 @@ impl crate::Bundle { object_hash, )?, thin_pack_lookup, + options.compression, ); let pack_kind = pack_entries_iter.inner.version(); (Box::new(pack_entries_iter), pack_kind) @@ -270,6 +272,7 @@ impl crate::Bundle { index_version: index_kind, object_hash, alloc_limit_bytes, + compression: _, }: Options, data_file: SharedTempFile, mut pack_entries_iter: Box> + 'a>, diff --git a/gix-pack/src/bundle/write/types.rs b/gix-pack/src/bundle/write/types.rs index a53773ff6a1..bd0657abf43 100644 --- a/gix-pack/src/bundle/write/types.rs +++ b/gix-pack/src/bundle/write/types.rs @@ -18,6 +18,12 @@ pub struct Options { /// entries into decoded object and delta result buffers to write the index. /// `Some(0)` rejects all non-empty allocations. pub alloc_limit_bytes: Option, + /// The compression level to use when deflating base objects that are added to the pack to complete a thin pack. + /// + /// Defaults to [`Compression::BEST_SPEED`](gix_features::zlib::Compression::BEST_SPEED) - `git` compresses + /// these with the same level it uses for loose objects, which is `core.looseCompression` or + /// `core.compression`, or level 1 if neither is set. + pub compression: gix_features::zlib::Compression, } impl Default for Options { @@ -29,6 +35,7 @@ impl Default for Options { index_version: Default::default(), object_hash: Default::default(), alloc_limit_bytes: None, + compression: gix_features::zlib::Compression::BEST_SPEED, } } } diff --git a/gix-pack/src/cache/delta/traverse/resolve.rs b/gix-pack/src/cache/delta/traverse/resolve.rs index 416f9a2d9b2..55ea88a6ee2 100644 --- a/gix-pack/src/cache/delta/traverse/resolve.rs +++ b/gix-pack/src/cache/delta/traverse/resolve.rs @@ -620,7 +620,8 @@ mod tests { } fn deflate(bytes: &[u8]) -> Vec { - let mut out = gix_features::zlib::stream::deflate::Write::new(Vec::new()); + let mut out = + gix_features::zlib::stream::deflate::Write::new(Vec::new(), gix_features::zlib::Compression::BEST_SPEED); out.write_all(bytes).expect("writing to deflater succeeds"); out.flush().expect("flushing deflater succeeds"); out.into_inner() diff --git a/gix-pack/src/data/input/entry.rs b/gix-pack/src/data/input/entry.rs index 55fb439afc5..d185338cd56 100644 --- a/gix-pack/src/data/input/entry.rs +++ b/gix-pack/src/data/input/entry.rs @@ -3,12 +3,17 @@ use std::io::Write; use crate::data::{entry::Header, input}; impl input::Entry { - /// Create a new input entry from a given data `obj` set to be placed at the given `pack_offset`. + /// Create a new input entry from a given data `obj` set to be placed at the given `pack_offset`, + /// deflating its data with `compression`. /// /// This method is useful when arbitrary base entries are created - pub fn from_data_obj(obj: &gix_object::Data<'_>, pack_offset: u64) -> Result { + pub fn from_data_obj( + obj: &gix_object::Data<'_>, + pack_offset: u64, + compression: gix_features::zlib::Compression, + ) -> Result { let header = to_header(obj.kind); - let compressed = compress_data(obj)?; + let compressed = compress_data(obj, compression)?; let compressed_size = compressed.len() as u64; let mut entry = input::Entry { header, @@ -50,8 +55,11 @@ fn to_header(kind: gix_object::Kind) -> Header { } } -fn compress_data(obj: &gix_object::Data<'_>) -> Result, input::Error> { - let mut out = gix_features::zlib::stream::deflate::Write::new(Vec::new()); +fn compress_data( + obj: &gix_object::Data<'_>, + compression: gix_features::zlib::Compression, +) -> Result, input::Error> { + let mut out = gix_features::zlib::stream::deflate::Write::new(Vec::new(), compression); if let Err(err) = std::io::copy(&mut &*obj.data, &mut out) { match err.kind() { std::io::ErrorKind::Other => return Err(input::Error::Io(err.into())), diff --git a/gix-pack/src/data/input/lookup_ref_delta_objects.rs b/gix-pack/src/data/input/lookup_ref_delta_objects.rs index bdeb79a97fd..24e6a80e6a5 100644 --- a/gix-pack/src/data/input/lookup_ref_delta_objects.rs +++ b/gix-pack/src/data/input/lookup_ref_delta_objects.rs @@ -7,6 +7,8 @@ pub struct LookupRefDeltaObjectsIter { /// The inner iterator whose entries we will resolve. pub inner: I, lookup: Find, + /// The compression level to use when deflating resolved base objects into entries. + compression: gix_features::zlib::Compression, /// The cached delta to provide next time we are called, it's the delta to go with the base we just resolved in its place. next_delta: Option, /// Fuse to stop iteration after first missing object. @@ -25,11 +27,12 @@ where Find: gix_object::Find, { /// Create a new instance wrapping `iter` and using `lookup` as function to retrieve objects that will serve as bases - /// for ref deltas seen while traversing `iter`. - pub fn new(iter: I, lookup: Find) -> Self { + /// for ref deltas seen while traversing `iter`, deflating them with `compression`. + pub fn new(iter: I, lookup: Find, compression: gix_features::zlib::Compression) -> Self { LookupRefDeltaObjectsIter { inner: iter, lookup, + compression, error: false, inserted_entry_length_at_offset: Vec::new(), inserted_entries_length_in_bytes: 0, @@ -95,7 +98,7 @@ where let base_entry = match self.lookup.try_find(&base_id, &mut self.buf).ok()? { Some(obj) => { let current_pack_offset = entry.pack_offset; - let mut entry = match input::Entry::from_data_obj(&obj, 0) { + let mut entry = match input::Entry::from_data_obj(&obj, 0, self.compression) { Ok(e) => e, Err(err) => return Some(Err(err)), }; diff --git a/gix-pack/src/data/output/entry/iter_from_counts.rs b/gix-pack/src/data/output/entry/iter_from_counts.rs index 9aba8695fae..6d6fa12a4c9 100644 --- a/gix-pack/src/data/output/entry/iter_from_counts.rs +++ b/gix-pack/src/data/output/entry/iter_from_counts.rs @@ -52,6 +52,7 @@ pub(crate) mod function { allow_thin_pack, thread_limit, chunk_size, + compression, }: Options, ) -> impl Iterator), Error>> + parallel::reduce::Finalize> @@ -214,7 +215,7 @@ pub(crate) mod function { None => match db.try_find(&count.id, buf).map_err(Error::Find)? { Some((obj, _location)) => { stats.decoded_and_recompressed_objects += 1; - output::Entry::from_data(count, &obj) + output::Entry::from_data(count, &obj, compression) } None => { stats.missing_objects += 1; @@ -226,7 +227,7 @@ pub(crate) mod function { None => match db.try_find(&count.id, buf).map_err(Error::Find)? { Some((obj, _location)) => { stats.decoded_and_recompressed_objects += 1; - output::Entry::from_data(count, &obj) + output::Entry::from_data(count, &obj, compression) } None => { stats.missing_objects += 1; @@ -385,6 +386,13 @@ mod types { pub chunk_size: usize, /// The pack data version to produce for each entry pub version: crate::data::Version, + /// The compression level to use for objects that are not copied from an existing pack, + /// but deflated from their object data. + /// + /// Defaults to [`Compression::DEFAULT`](gix_features::zlib::Compression::DEFAULT), which is + /// also the default that `git` uses when writing packs, unless configured otherwise + /// with `pack.compression`. + pub compression: gix_features::zlib::Compression, } impl Default for Options { @@ -395,6 +403,7 @@ mod types { allow_thin_pack: false, chunk_size: 10, version: Default::default(), + compression: gix_features::zlib::Compression::DEFAULT, } } } diff --git a/gix-pack/src/data/output/entry/mod.rs b/gix-pack/src/data/output/entry/mod.rs index 2fd57e57e33..169f4b2e443 100644 --- a/gix-pack/src/data/output/entry/mod.rs +++ b/gix-pack/src/data/output/entry/mod.rs @@ -130,14 +130,22 @@ impl output::Entry { }) } - /// Create a new instance from the given `oid` and its corresponding git object data `obj`. - pub fn from_data(count: &output::Count, obj: &gix_object::Data<'_>) -> Result { + /// Create a new instance from the given `oid` and its corresponding git object data `obj`, + /// deflating it with `compression`. + /// + /// Note that `git` compresses pack entries with the level configured with `pack.compression`, + /// whose default is [`Compression::DEFAULT`](gix_features::zlib::Compression::DEFAULT). + pub fn from_data( + count: &output::Count, + obj: &gix_object::Data<'_>, + compression: gix_features::zlib::Compression, + ) -> Result { Ok(output::Entry { id: count.id.to_owned(), kind: Kind::Base(obj.kind), decompressed_size: obj.data.len(), compressed_data: { - let mut out = gix_features::zlib::stream::deflate::Write::new(Vec::new()); + let mut out = gix_features::zlib::stream::deflate::Write::new(Vec::new(), compression); if let Err(err) = std::io::copy(&mut &*obj.data, &mut out) { match err.kind() { std::io::ErrorKind::Other => return Err(Error::ZlibDeflate(err)), diff --git a/gix-pack/tests/pack/bundle.rs b/gix-pack/tests/pack/bundle.rs index 860420eaf5e..00c7b9c435b 100644 --- a/gix-pack/tests/pack/bundle.rs +++ b/gix-pack/tests/pack/bundle.rs @@ -171,6 +171,7 @@ mod write_to_directory { index_version: pack::index::Version::V2, object_hash: gix_hash::Kind::Sha1, alloc_limit_bytes: prevent_allocation, + compression: gix_features::zlib::Compression::BEST_SPEED, }, ) .expect_err("a zero allocation limit rejects the first non-empty decoded object"); @@ -205,6 +206,7 @@ mod write_to_directory { index_version: pack::index::Version::V2, object_hash: gix_hash::Kind::Sha1, alloc_limit_bytes: None, + compression: gix_features::zlib::Compression::BEST_SPEED, }, ) .map_err(Into::into) diff --git a/gix-pack/tests/pack/data/fuzzed.rs b/gix-pack/tests/pack/data/fuzzed.rs index 2396b34cabb..b5e331238e5 100644 --- a/gix-pack/tests/pack/data/fuzzed.rs +++ b/gix-pack/tests/pack/data/fuzzed.rs @@ -58,7 +58,8 @@ fn oversized_pack_entry_header_is_reported_without_panicking() { #[test] fn non_canonical_pack_entry_header_is_accepted() { fn deflate(bytes: &[u8]) -> Vec { - let mut out = gix_features::zlib::stream::deflate::Write::new(Vec::new()); + let mut out = + gix_features::zlib::stream::deflate::Write::new(Vec::new(), gix_features::zlib::Compression::BEST_SPEED); out.write_all(bytes).expect("writing to deflater succeeds"); out.flush().expect("flushing deflater succeeds"); out.into_inner() @@ -136,7 +137,8 @@ fn oversized_declared_object_size_is_reported_without_panicking() { #[test] fn declared_object_size_over_alloc_limit_bytes_is_reported_as_out_of_memory() { fn deflate(bytes: &[u8]) -> Vec { - let mut out = gix_features::zlib::stream::deflate::Write::new(Vec::new()); + let mut out = + gix_features::zlib::stream::deflate::Write::new(Vec::new(), gix_features::zlib::Compression::BEST_SPEED); out.write_all(bytes).expect("writing to deflater succeeds"); out.flush().expect("flushing deflater succeeds"); out.into_inner() @@ -508,7 +510,8 @@ fn overlong_delta_header_size_is_reported_without_panicking() { #[test] fn short_delta_application_is_reported_without_panicking() { fn deflate(bytes: &[u8]) -> Vec { - let mut out = gix_features::zlib::stream::deflate::Write::new(Vec::new()); + let mut out = + gix_features::zlib::stream::deflate::Write::new(Vec::new(), gix_features::zlib::Compression::BEST_SPEED); out.write_all(bytes).expect("writing to deflater succeeds"); out.flush().expect("flushing deflater succeeds"); out.into_inner() diff --git a/gix-pack/tests/pack/data/input.rs b/gix-pack/tests/pack/data/input.rs index 782115392bd..a5ddca2f50a 100644 --- a/gix-pack/tests/pack/data/input.rs +++ b/gix-pack/tests/pack/data/input.rs @@ -35,7 +35,8 @@ mod lookup_ref_delta_objects { object_hash: gix_testtools::object_hash(), data, }; - let mut entry = input::Entry::from_data_obj(&obj, 0).expect("valid object"); + let mut entry = + input::Entry::from_data_obj(&obj, 0, gix_features::zlib::Compression::BEST_SPEED).expect("valid object"); entry.header = header; entry.header_size = header.size(data.len() as u64) as u16; entry @@ -99,8 +100,12 @@ mod lookup_ref_delta_objects { fn only_ref_deltas_are_handled() -> crate::Result { let input = compute_offsets(vec![entry(base(), D_A), entry(delta_ofs(100), D_B)]); let expected = input.clone(); - let actual = LookupRefDeltaObjectsIter::new(into_results_iter(input), gix_object::find::Never) - .collect::, _>>()?; + let actual = LookupRefDeltaObjectsIter::new( + into_results_iter(input), + gix_object::find::Never, + gix_features::zlib::Compression::BEST_SPEED, + ) + .collect::, _>>()?; assert_eq!(actual, expected, "it won't change the input at all"); validate_pack_offsets(&actual); Ok(()) @@ -124,7 +129,7 @@ mod lookup_ref_delta_objects { let input_entries = into_results_iter(input); let actual_size = input_entries.size_hint(); let db = FindData::new(inserted_data, &calls); - let iter = LookupRefDeltaObjectsIter::new(input_entries, &db); + let iter = LookupRefDeltaObjectsIter::new(input_entries, &db, gix_features::zlib::Compression::BEST_SPEED); assert_eq!( iter.size_hint(), (actual_size.0, actual_size.1.map(|s| s * 2)), @@ -182,7 +187,12 @@ mod lookup_ref_delta_objects { let input = vec![entry(delta_ref(gix_hash::Kind::Sha1.null()), D_A), entry(base(), D_B)]; let calls = AtomicUsize::default(); let db = FindData::new(None, &calls); - let mut result = LookupRefDeltaObjectsIter::new(into_results_iter(input), &db).collect::>(); + let mut result = LookupRefDeltaObjectsIter::new( + into_results_iter(input), + &db, + gix_features::zlib::Compression::BEST_SPEED, + ) + .collect::>(); assert_eq!(calls.load(Ordering::Relaxed), 1, "it tries to lookup the object"); assert_eq!(result.len(), 1, "the error stops iteration"); assert!(matches!( @@ -209,7 +219,12 @@ mod lookup_ref_delta_objects { }), Ok(entry(base(), D_B)), ]; - let actual = LookupRefDeltaObjectsIter::new(input.into_iter(), gix_object::find::Never).collect::>(); + let actual = LookupRefDeltaObjectsIter::new( + input.into_iter(), + gix_object::find::Never, + gix_features::zlib::Compression::BEST_SPEED, + ) + .collect::>(); for (actual, expected) in actual.into_iter().zip(expected) { assert_eq!(format!("{actual:?}"), format!("{expected:?}")); } diff --git a/gix-pack/tests/pack/data/output/count_and_entries.rs b/gix-pack/tests/pack/data/output/count_and_entries.rs index 23eca371d20..335279f603c 100644 --- a/gix-pack/tests/pack/data/output/count_and_entries.rs +++ b/gix-pack/tests/pack/data/output/count_and_entries.rs @@ -367,6 +367,74 @@ fn traversals() -> crate::Result { Ok(()) } +/// Reproduces https://github.com/GitoxideLabs/gitoxide/issues/2024: with the default backend's +/// level 1 being much weaker than it used to be, entries have to be compressed with the +/// configured level, defaulting to what `git` uses. +#[test] +fn entry_sizes_depend_on_compression_level() -> crate::Result { + // Deterministic pseudo-random bytes (xorshift64*), so tree content is stable across runs. + struct Rng(u64); + impl Rng { + fn next_byte(&mut self) -> u8 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + (self.0.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 56) as u8 + } + } + let mut rng = Rng(0x2024); + let mut files = (0..1500) + .map(|_| { + use std::fmt::Write; + (0..10).fold(String::new(), |mut buf, _| { + write!(buf, "{:02x}", rng.next_byte()).expect("writing to a string never fails"); + buf + }) + }) + .collect::>(); + files.sort(); + files.dedup(); + + let blob_id = gix_object::compute_hash(gix_hash::Kind::Sha1, gix_object::Kind::Blob, b"xoxo")?; + let tree = gix_object::Tree { + entries: files + .iter() + .map(|name| gix_object::tree::Entry { + mode: gix_object::tree::EntryKind::Blob.into(), + oid: blob_id, + filename: name.as_str().into(), + }) + .collect(), + }; + let mut buf = Vec::new(); + gix_object::WriteTo::write_to(&tree, &mut buf)?; + let tree_id = gix_object::compute_hash(gix_hash::Kind::Sha1, gix_object::Kind::Tree, &buf)?; + + let entry_size = |compression| -> Result { + Ok(output::Entry::from_data( + &output::Count::from_data(tree_id, None), + &gix_object::Data::new(&buf, gix_object::Kind::Tree, gix_hash::Kind::Sha1), + compression, + )? + .compressed_data + .len()) + }; + + use gix_features::zlib::Compression; + let default = entry_size(Compression::DEFAULT)?; + let fastest = entry_size(Compression::BEST_SPEED)?; + let none = entry_size(Compression::NONE)?; + assert!( + default < 25_000, + "the default compression level keeps this ~72KB tree below the issue's regression threshold: {default}" + ); + assert!( + none > fastest && fastest >= default, + "higher levels compress at least as well as lower ones: none={none} fastest={fastest} default={default}" + ); + Ok(()) +} + #[test] #[cfg(all(not(feature = "wasm"), feature = "streaming-input"))] fn empty_pack_is_allowed() { diff --git a/gix-pack/tests/pack/malformed.rs b/gix-pack/tests/pack/malformed.rs index 1a201f06489..cc4f5c457bb 100644 --- a/gix-pack/tests/pack/malformed.rs +++ b/gix-pack/tests/pack/malformed.rs @@ -255,7 +255,8 @@ fn encode_delta_size(mut size: u64) -> Vec { } fn deflate(bytes: &[u8]) -> crate::Result> { - let mut write = gix_features::zlib::stream::deflate::Write::new(Vec::new()); + let mut write = + gix_features::zlib::stream::deflate::Write::new(Vec::new(), gix_features::zlib::Compression::BEST_SPEED); write.write_all(bytes)?; write.flush()?; Ok(write.into_inner()) diff --git a/gix/src/config/cache/init.rs b/gix/src/config/cache/init.rs index 7f793254d58..34a9f8fa73e 100644 --- a/gix/src/config/cache/init.rs +++ b/gix/src/config/cache/init.rs @@ -158,6 +158,7 @@ impl Cache { let object_kind_hint = util::disambiguate_hint(&config, lenient_config)?; let (static_pack_cache_limit_bytes, pack_cache_bytes, object_cache_bytes, alloc_limit_bytes) = util::parse_object_caches(&config, lenient_config, filter_config_section)?; + let loose_compression = util::parse_loose_compression(&config, lenient_config, filter_config_section)?; // NOTE: When adding a new initial cache, consider adjusting `reread_values_and_clear_caches()` as well. Ok(Cache { resolved: config.into(), @@ -169,6 +170,7 @@ impl Cache { pack_cache_bytes, object_cache_bytes, alloc_limit_bytes, + loose_compression, reflog, refs_namespace, is_bare, @@ -248,6 +250,8 @@ impl Cache { self.object_cache_bytes, self.alloc_limit_bytes, ) = util::parse_object_caches(config, self.lenient_config, self.filter_config_section)?; + self.loose_compression = + util::parse_loose_compression(config, self.lenient_config, self.filter_config_section)?; #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] { self.url_scheme = Default::default(); diff --git a/gix/src/config/cache/util.rs b/gix/src/config/cache/util.rs index c1a7f85b2cc..fb619f30fc2 100644 --- a/gix/src/config/cache/util.rs +++ b/gix/src/config/cache/util.rs @@ -100,6 +100,27 @@ pub(crate) fn reflog_or_default( pub(crate) type ObjectCaches = (Option, Option, usize, Option); /// Return `(static_pack_cache_limit, pack_cache_bytes, object_cache_bytes, alloc_limit_bytes)` as parsed from gix-config. +pub(crate) fn parse_loose_compression( + config: &gix_config::File<'static>, + lenient: bool, + mut filter_config_section: fn(&gix_config::file::Metadata) -> bool, +) -> Result { + let level = match config + .integer_filter("core.looseCompression", &mut filter_config_section) + .map(|res| Core::LOOSE_COMPRESSION.try_into_compression(res)) + .transpose() + .with_leniency(lenient)? + { + Some(level) => Some(level), + None => config + .integer_filter("core.compression", &mut filter_config_section) + .map(|res| Core::COMPRESSION.try_into_compression(res)) + .transpose() + .with_leniency(lenient)?, + }; + Ok(level.unwrap_or(gix_features::zlib::Compression::BEST_SPEED)) +} + pub(crate) fn parse_object_caches( config: &gix_config::File<'static>, lenient: bool, diff --git a/gix/src/config/mod.rs b/gix/src/config/mod.rs index 6b2b0e3a1cd..e759f301de2 100644 --- a/gix/src/config/mod.rs +++ b/gix/src/config/mod.rs @@ -83,6 +83,8 @@ pub enum Error { #[error(transparent)] ConfigTypedString(#[from] key::GenericErrorWithValue), #[error(transparent)] + ConfigCompression(#[from] key::GenericError), + #[error(transparent)] RefsNamespace(#[from] refs_namespace::Error), #[error("Cannot handle objects formatted as {:?}", .name)] UnsupportedObjectFormat { name: BString }, @@ -629,6 +631,8 @@ pub(crate) struct Cache { pub(crate) object_cache_bytes: usize, /// The maximum size of a single allocation caused by user-controlled on-disk packed object data. pub(crate) alloc_limit_bytes: Option, + /// The compression level to use when writing loose objects, from `core.looseCompression` or `core.compression`. + pub(crate) loose_compression: gix_features::zlib::Compression, /// The amount of bytes we can hold in our static LRU cache. Otherwise, go with the defaults. pub(crate) static_pack_cache_limit_bytes: Option, /// The config section filter from the options used to initialize this instance. Keep these in sync! diff --git a/gix/src/config/tree/keys.rs b/gix/src/config/tree/keys.rs index db53bf7e96f..5c1258fc887 100644 --- a/gix/src/config/tree/keys.rs +++ b/gix/src/config/tree/keys.rs @@ -177,6 +177,9 @@ pub type Time = Any; /// The `core.(filesRefLockTimeout|packedRefsTimeout)` keys, or any other lock timeout for that matter. pub type LockTimeout = Any; +/// The `core.compression`, `core.looseCompression` and `pack.compression` keys. +pub type Compression = Any; + /// Keys specifying durations in milliseconds. pub type DurationInMilliseconds = Any; @@ -281,6 +284,36 @@ mod lock_timeout { } } +mod compression { + use crate::{ + config, + config::tree::{Section, keys::Compression}, + }; + + impl Compression { + /// Create a new instance. + pub const fn new_compression(name: &'static str, section: &'static dyn Section) -> Self { + Self::new_with_validate(name, section, super::validate::Compression) + } + + /// Convert `value` into a zlib compression level, where `-1` is mapped to the + /// zlib default, just like `git` does. + pub fn try_into_compression( + &'static self, + value: Result, + ) -> Result { + let value = value.map_err(|err| config::key::GenericError::from(self).with_source(err))?; + match value { + -1 => Ok(gix_features::zlib::Compression::DEFAULT), + level => i32::try_from(level) + .ok() + .and_then(gix_features::zlib::Compression::new) + .ok_or_else(|| config::key::GenericError::from(self)), + } + } + } +} + mod refspecs { use crate::config::tree::{ Section, @@ -633,6 +666,19 @@ pub mod validate { } } + /// A zlib compression level. + #[derive(Clone, Copy)] + pub struct Compression; + impl Validate for Compression { + fn validate(&self, value: &BStr) -> Result<(), Box> { + let value = gix_config::Integer::try_from(value)? + .to_decimal() + .ok_or_else(|| format!("integer {value} cannot be represented as integer")); + super::super::Core::COMPRESSION.try_into_compression(Ok(value?))?; + Ok(()) + } + } + /// Durations in milliseconds. #[derive(Clone, Copy)] pub struct DurationInMilliseconds; diff --git a/gix/src/config/tree/sections/core.rs b/gix/src/config/tree/sections/core.rs index 1bf3dcf4ad5..b50b292bdf1 100644 --- a/gix/src/config/tree/sections/core.rs +++ b/gix/src/config/tree/sections/core.rs @@ -12,6 +12,13 @@ impl Core { pub const BIG_FILE_THRESHOLD: keys::UnsignedInteger = keys::UnsignedInteger::new_unsigned_integer("bigFileThreshold", &config::Tree::CORE); /// The `core.checkStat` key. + /// The `core.compression` key. + pub const COMPRESSION: keys::Compression = keys::Compression::new_compression("compression", &config::Tree::CORE) + .with_deviation("git defaults to -1 (zlib default) for packed and 1 for loose objects, gitoxide uses the same values but rejects levels outside of -1..=9 early"); + /// The `core.looseCompression` key. + pub const LOOSE_COMPRESSION: keys::Compression = + keys::Compression::new_compression("looseCompression", &config::Tree::CORE); + /// The `core.checkStat` key. pub const CHECK_STAT: CheckStat = CheckStat::new_with_validate("checkStat", &config::Tree::CORE, validate::CheckStat); /// The `core.deltaBaseCacheLimit` key. @@ -107,6 +114,8 @@ impl Section for Core { &Self::ABBREV, &Self::BARE, &Self::BIG_FILE_THRESHOLD, + &Self::COMPRESSION, + &Self::LOOSE_COMPRESSION, &Self::CHECK_STAT, &Self::DELTA_BASE_CACHE_LIMIT, &Self::DISAMBIGUATE, diff --git a/gix/src/config/tree/sections/pack.rs b/gix/src/config/tree/sections/pack.rs index a17d58e9f60..52ea82b0875 100644 --- a/gix/src/config/tree/sections/pack.rs +++ b/gix/src/config/tree/sections/pack.rs @@ -12,6 +12,9 @@ impl Pack { /// The `pack.indexVersion` key. pub const INDEX_VERSION: IndexVersion = IndexVersion::new_with_validate("indexVersion", &config::Tree::PACK, validate::IndexVersion); + + /// The `pack.compression` key. + pub const COMPRESSION: keys::Compression = keys::Compression::new_compression("compression", &config::Tree::PACK); } /// The `pack.indexVersion` key. @@ -42,7 +45,7 @@ impl Section for Pack { } fn keys(&self) -> &[&dyn Key] { - &[&Self::THREADS, &Self::INDEX_VERSION] + &[&Self::THREADS, &Self::INDEX_VERSION, &Self::COMPRESSION] } } diff --git a/gix/src/open/repository.rs b/gix/src/open/repository.rs index ac2b293111d..a27a4579df0 100644 --- a/gix/src/open/repository.rs +++ b/gix/src/open/repository.rs @@ -494,6 +494,7 @@ impl ThreadSafeRepository { object_hash: config.object_hash, use_multi_pack_index: config.use_multi_pack_index, alloc_limit_bytes: config.alloc_limit_bytes, + loose_compression: config.loose_compression, current_dir: current_dir.to_owned().into(), }, )?), diff --git a/gix/src/remote/connection/fetch/receive_pack.rs b/gix/src/remote/connection/fetch/receive_pack.rs index 143f0578bff..b9b49eb3e51 100644 --- a/gix/src/remote/connection/fetch/receive_pack.rs +++ b/gix/src/remote/connection/fetch/receive_pack.rs @@ -173,6 +173,7 @@ where iteration_mode: gix_pack::data::input::Mode::Verify, object_hash: repo.object_hash(), alloc_limit_bytes: repo.config.alloc_limit_bytes, + compression: repo.config.loose_compression, }; let mut write_pack_bundle = None; diff --git a/gix/tests/gix/config/tree.rs b/gix/tests/gix/config/tree.rs index 67c3dba962f..7e684fa896f 100644 --- a/gix/tests/gix/config/tree.rs +++ b/gix/tests/gix/config/tree.rs @@ -104,6 +104,50 @@ mod keys { } } +mod compression { + use gix::config::tree::Key; + + #[test] + fn validate_and_convert() { + for key in [ + &gix::config::tree::Core::COMPRESSION, + &gix::config::tree::Core::LOOSE_COMPRESSION, + &gix::config::tree::Pack::COMPRESSION, + ] { + for valid in -1..=9 { + assert!(key.validate(valid.to_string().as_str().into()).is_ok()); + } + for invalid in [-2, 10, 100] { + assert!(key.validate(invalid.to_string().as_str().into()).is_err()); + } + } + + assert_eq!( + gix::config::tree::Core::COMPRESSION + .try_into_compression(Ok(-1)) + .expect("git maps -1 to the zlib default"), + gix::features::zlib::Compression::DEFAULT + ); + assert_eq!( + gix::config::tree::Core::COMPRESSION + .try_into_compression(Ok(1)) + .unwrap(), + gix::features::zlib::Compression::BEST_SPEED + ); + assert_eq!( + gix::config::tree::Pack::COMPRESSION + .try_into_compression(Ok(9)) + .unwrap(), + gix::features::zlib::Compression::BEST + ); + assert!( + gix::config::tree::Pack::COMPRESSION + .try_into_compression(Ok(10)) + .is_err() + ); + } +} + mod branch { use gix::config::tree::{Branch, Key, branch}; diff --git a/gix/tests/gix/repository/object.rs b/gix/tests/gix/repository/object.rs index be82c046ed4..c27001821b9 100644 --- a/gix/tests/gix/repository/object.rs +++ b/gix/tests/gix/repository/object.rs @@ -400,7 +400,12 @@ mod write_blob { #[test] fn writes_avoid_io_using_duplicate_check() -> crate::Result { let mut repo = crate::named_repo("make_packed_and_loose.sh")?; - let store = gix::odb::loose::Store::at(repo.git_dir().join("objects"), repo.object_hash(), None); + let store = gix::odb::loose::Store::at( + repo.git_dir().join("objects"), + repo.object_hash(), + None, + gix::features::zlib::Compression::BEST_SPEED, + ); let loose_count = store.iter().count(); assert_eq!(loose_count, 3, "there are some loose objects"); assert_eq!(