fix(worker-executor): bound component cache size and concurrent compilations

golem-debugging-service/config/debug-worker-executor.sample.env

@@ -18,6 +18,7 @@ GOLEM__BLOB_STORAGE__TYPE="LocalFileSystem"
 GOLEM__BLOB_STORAGE__CONFIG__ROOT="../data/blob_storage"
 GOLEM__COMPILED_COMPONENT_SERVICE__TYPE="Enabled"
 GOLEM__COMPONENT_CACHE__MAX_CAPACITY=32
+GOLEM__COMPONENT_CACHE__MAX_CONCURRENT_COMPILATIONS=16
 GOLEM__COMPONENT_CACHE__MAX_METADATA_CAPACITY=16384
 GOLEM__COMPONENT_CACHE__MAX_RESOLVED_COMPONENT_CAPACITY=1024
 GOLEM__COMPONENT_CACHE__TIME_TO_IDLE="12h"
@@ -193,6 +194,7 @@ GOLEM__BLOB_STORAGE__TYPE="LocalFileSystem"
 GOLEM__BLOB_STORAGE__CONFIG__ROOT="../data/blob_storage"
 GOLEM__COMPILED_COMPONENT_SERVICE__TYPE="Enabled"
 GOLEM__COMPONENT_CACHE__MAX_CAPACITY=32
+GOLEM__COMPONENT_CACHE__MAX_CONCURRENT_COMPILATIONS=16
 GOLEM__COMPONENT_CACHE__MAX_METADATA_CAPACITY=16384
 GOLEM__COMPONENT_CACHE__MAX_RESOLVED_COMPONENT_CAPACITY=1024
 GOLEM__COMPONENT_CACHE__TIME_TO_IDLE="12h"

golem-debugging-service/config/debug-worker-executor.toml

@@ -39,6 +39,7 @@ type = "Enabled"
 
 [component_cache]
 max_capacity = 32
+max_concurrent_compilations = 16
 max_metadata_capacity = 16384
 max_resolved_component_capacity = 1024
 time_to_idle = "12h"
@@ -309,6 +310,7 @@ without_time = false
 # 
 # [component_cache]
 # max_capacity = 32
+# max_concurrent_compilations = 16
 # max_metadata_capacity = 16384
 # max_resolved_component_capacity = 1024
 # time_to_idle = "12h"

golem-worker-executor/config/worker-executor.sample.env

@@ -21,6 +21,7 @@ GOLEM__BLOB_STORAGE__TYPE="LocalFileSystem"
 GOLEM__BLOB_STORAGE__CONFIG__ROOT="../data/blob_storage"
 GOLEM__COMPILED_COMPONENT_SERVICE__TYPE="Enabled"
 GOLEM__COMPONENT_CACHE__MAX_CAPACITY=32
+GOLEM__COMPONENT_CACHE__MAX_CONCURRENT_COMPILATIONS=16
 GOLEM__COMPONENT_CACHE__MAX_METADATA_CAPACITY=16384
 GOLEM__COMPONENT_CACHE__MAX_RESOLVED_COMPONENT_CAPACITY=1024
 GOLEM__COMPONENT_CACHE__TIME_TO_IDLE="12h"
@@ -256,6 +257,7 @@ GOLEM__BLOB_STORAGE__CONFIG__RETRIES__MIN_DELAY="100ms"
 GOLEM__BLOB_STORAGE__CONFIG__RETRIES__MULTIPLIER=3.0
 GOLEM__COMPILED_COMPONENT_SERVICE__TYPE="Enabled"
 GOLEM__COMPONENT_CACHE__MAX_CAPACITY=32
+GOLEM__COMPONENT_CACHE__MAX_CONCURRENT_COMPILATIONS=16
 GOLEM__COMPONENT_CACHE__MAX_METADATA_CAPACITY=16384
 GOLEM__COMPONENT_CACHE__MAX_RESOLVED_COMPONENT_CAPACITY=1024
 GOLEM__COMPONENT_CACHE__TIME_TO_IDLE="12h"
@@ -470,6 +472,7 @@ GOLEM__AGENT_WEBHOOKS_SERVICE__USE_HTTPS_FOR_WEBHOOK_URL=true
 GOLEM__BLOB_STORAGE__TYPE="InMemory"
 GOLEM__COMPILED_COMPONENT_SERVICE__TYPE="Enabled"
 GOLEM__COMPONENT_CACHE__MAX_CAPACITY=32
+GOLEM__COMPONENT_CACHE__MAX_CONCURRENT_COMPILATIONS=16
 GOLEM__COMPONENT_CACHE__MAX_METADATA_CAPACITY=16384
 GOLEM__COMPONENT_CACHE__MAX_RESOLVED_COMPONENT_CAPACITY=1024
 GOLEM__COMPONENT_CACHE__TIME_TO_IDLE="12h"

golem-worker-executor/config/worker-executor.toml

@@ -42,6 +42,7 @@ type = "Enabled"
 
 [component_cache]
 max_capacity = 32
+max_concurrent_compilations = 16
 max_metadata_capacity = 16384
 max_resolved_component_capacity = 1024
 time_to_idle = "12h"
@@ -401,6 +402,7 @@ without_time = false
 # 
 # [component_cache]
 # max_capacity = 32
+# max_concurrent_compilations = 16
 # max_metadata_capacity = 16384
 # max_resolved_component_capacity = 1024
 # time_to_idle = "12h"
@@ -733,6 +735,7 @@ without_time = false
 # 
 # [component_cache]
 # max_capacity = 32
+# max_concurrent_compilations = 16
 # max_metadata_capacity = 16384
 # max_resolved_component_capacity = 1024
 # time_to_idle = "12h"

golem-worker-executor/src/services/compilation_limiter.rs

@@ -0,0 +1,148 @@
+// Copyright 2024-2026 Golem Cloud
+//
+// Licensed under the Golem Source License v1.1 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://license.golem.cloud/LICENSE
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::future::Future;
+use std::sync::Arc;
+use tokio::sync::Semaphore;
+
+/// Bounds the number of concurrent local component compile fallback attempts.
+///
+/// Worker admission accounts for guest linear memory and a shared
+/// per-component-revision charge derived from registry metadata. It does not
+/// reserve the transient working set used when the worker-executor in-process
+/// component cache and compiled artifact store both miss. That fallback buffers
+/// the raw component bytes, compiles them with Wasmtime, serializes the compiled
+/// artifact, and writes it to the compiled artifact store.
+///
+/// This limiter caps how many fallback attempts run at the same time; callers
+/// beyond the limit wait for a permit.
+/// A limit of `0` is treated as unlimited: the gate is bypassed entirely.
+#[derive(Clone)]
+pub struct CompilationLimiter {
+    semaphore: Option<Arc<Semaphore>>,
+}
+
+impl CompilationLimiter {
+    pub fn new(max_concurrent: usize) -> Self {
+        let semaphore = if max_concurrent == 0 {
+            None
+        } else {
+            Some(Arc::new(Semaphore::new(max_concurrent)))
+        };
+        Self { semaphore }
+    }
+
+    /// Runs `f` while holding a compilation permit, waiting for one if the
+    /// limiter is at its concurrency limit. With an unlimited limiter the
+    /// future runs immediately with no gating.
+    pub async fn run<F, T>(&self, f: F) -> T
+    where
+        F: Future<Output = T>,
+    {
+        let _permit = match &self.semaphore {
+            Some(semaphore) => Some(
+                semaphore
+                    .clone()
+                    .acquire_owned()
+                    .await
+                    .expect("compilation limiter semaphore closed"),
+            ),
+            None => None,
+        };
+        f.await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use futures::future::join_all;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::time::Duration;
+    use test_r::test;
+
+    #[test]
+    async fn limits_concurrent_compilations_to_the_configured_maximum() {
+        let limit = 4usize;
+        let bursts = 200usize;
+        let limiter = CompilationLimiter::new(limit);
+
+        let current = Arc::new(AtomicUsize::new(0));
+        let observed_max = Arc::new(AtomicUsize::new(0));
+
+        let futs: Vec<_> = (0..bursts)
+            .map(|_| {
+                let limiter = limiter.clone();
+                let current = current.clone();
+                let observed_max = observed_max.clone();
+                async move {
+                    limiter
+                        .run(async {
+                            let now = current.fetch_add(1, Ordering::SeqCst) + 1;
+                            observed_max.fetch_max(now, Ordering::SeqCst);
+                            tokio::time::sleep(Duration::from_millis(5)).await;
+                            current.fetch_sub(1, Ordering::SeqCst);
+                        })
+                        .await;
+                }
+            })
+            .collect();
+
+        tokio::time::timeout(Duration::from_secs(30), join_all(futs))
+            .await
+            .expect("gated compilations timed out");
+
+        let peak = observed_max.load(Ordering::SeqCst);
+        assert!(
+            peak <= limit,
+            "compilation limiter with limit {limit} allowed {peak} concurrent compilations"
+        );
+    }
+
+    #[test]
+    async fn unlimited_limiter_does_not_gate() {
+        let limiter = CompilationLimiter::new(0);
+        let current = Arc::new(AtomicUsize::new(0));
+        let observed_max = Arc::new(AtomicUsize::new(0));
+        let bursts = 50usize;
+
+        let futs: Vec<_> = (0..bursts)
+            .map(|_| {
+                let limiter = limiter.clone();
+                let current = current.clone();
+                let observed_max = observed_max.clone();
+                async move {
+                    limiter
+                        .run(async {
+                            let now = current.fetch_add(1, Ordering::SeqCst) + 1;
+                            observed_max.fetch_max(now, Ordering::SeqCst);
+                            tokio::time::sleep(Duration::from_millis(5)).await;
+                            current.fetch_sub(1, Ordering::SeqCst);
+                        })
+                        .await;
+                }
+            })
+            .collect();
+
+        tokio::time::timeout(Duration::from_secs(30), join_all(futs))
+            .await
+            .expect("ungated compilations timed out");
+
+        let peak = observed_max.load(Ordering::SeqCst);
+        assert!(
+            peak > 1,
+            "unlimited limiter should allow concurrent execution, observed max {peak}"
+        );
+    }
+}