Merge pull request #19 from msk-access/release/0.8.3

Release 0.8.3

Author: rhshah

CHANGELOG.md

@@ -2,6 +2,56 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.8.3] - 2026-04-28
+
+### Fixed
+- **GIL Deadlock in Parallel Modules**: Wrapped all Rayon `par_iter()` calls in
+  `py.allow_threads()` across `mfsd.rs`, `uxm.rs`, `extract_motif.rs`, and
+  `region_mds.rs`. Previously, `pyo3-log` attempted to acquire the GIL from
+  Rayon worker threads while the main thread held it, causing indefinite hangs
+  on multi-threaded runs (observed as 16-hour wall time on IRIS HPC).
+- **mFSD Silent Error Swallowing**: Replaced `Err(_) => continue` in BAM record
+  iterator with a logged error breaker (max 1000 consecutive errors). Previously,
+  corrupt BAM regions could cause infinite silent loops.
+- **FILTER_MAF Substring Match**: Changed `sample_id in tsb` to exact match
+  (`sample_id == tsb`) preventing `T01` from matching `T01-XS1`.
+- **FILTER_MAF Comment Lines**: Stripped `#` comment lines from filtered MAF output
+  in both multi-sample and single-sample modes.
+- **mFSD 0-Variant Guard**: Added early exit at both Python (`wrapper.py`) and Rust
+  layers when MAF has 0 data lines. Produces header-only TSV instead of attempting
+  BAM access.
+- **mFSD GC Correction Fallback**: When reference FASTA is unavailable or GC lookup
+  fails for a region, GC correction is now skipped (weight=1.0) instead of silently
+  using a hardcoded 50% GC.
+
+### Added
+- **mFSD BAM I/O Diagnostics**: Per-variant timing, BAM open/fetch latency logging,
+  record counts, and slow-variant warnings (>30s). All diagnostics use structured
+  `log` macros (visible with `--verbose`).
+- **mFSD Header Constant**: Extracted 46-column TSV header into `MFSD_HEADER` module
+  constant, eliminating duplication between normal output and 0-variant early-exit.
+
+### Changed
+- **Minimum Python**: `requires-python` raised from `>=3.8` to `>=3.10` (Python
+  3.8 and 3.9 are EOL).
+- **Minimum Rust**: Added `rust-version = "1.87"` MSRV to `Cargo.toml`.
+- **Cargo Dependencies**: Updated 81 semver-compatible transitive dependencies
+  (rayon 1.11→1.12, anyhow 1.0.100→1.0.102, flate2 1.1.5→1.1.9, etc.).
+- **Diagnostic Logging**: Converted all `eprintln!` breadcrumbs inside parallel
+  closures to structured `debug!`/`warn!` macros for proper timestamp/level
+  formatting through Python's logging framework.
+
+### Documentation
+- **Developer Guide**: Added "PyO3 + Rayon GIL deadlock" Known Gotcha with correct
+  and incorrect code examples.
+- **Developer Guide**: Updated contributing checklist test count (244→357).
+
+### Tests
+- Added `test_mfsd_zero_variants` — verifies 0-variant input produces header-only TSV.
+- Added `test_mfsd_maf_with_comment_lines` — verifies MAFs with `#` comment headers
+  are parsed correctly.
+- Total: 357 passed, 4 skipped.
+
 ## [0.8.2] - 2026-03-26
 
 ### Fixed

docs/development/developer-guide.md

@@ -354,6 +354,44 @@ This gotcha caused a silent data loss bug in v0.8.0 where `MDS.exon.tsv` and
 
 ---
 
+### PyO3 + Rayon: Always Release the GIL Before `par_iter`
+
+When calling Rayon `par_iter()` from a `#[pyfunction]`, the GIL must be
+released first. Otherwise, `pyo3-log` (our global logger) tries to acquire
+the GIL from worker threads while the main thread holds it — **deadlock**.
+
+```rust
+// ✅ CORRECT: release GIL before parallel work
+fn my_function(py: Python, ...) -> PyResult<...> {
+    let results = py.allow_threads(|| {
+        items.par_iter().map(|item| {
+            info!("Processing {}", item);  // safe — GIL is released
+            process(item)
+        }).collect()
+    });
+    Ok(results)
+}
+
+// ❌ WRONG: GIL held during par_iter
+fn my_function(...) -> PyResult<...> {
+    let results = items.par_iter().map(|item| {
+        info!("Log from worker");  // DEADLOCK — pyo3-log needs GIL
+        process(item)
+    }).collect();
+    Ok(results)
+}
+```
+
+**Affected modules** (all patched in v0.8.3):
+`mfsd.rs`, `uxm.rs`, `extract_motif.rs`, `region_mds.rs`.
+
+**Rule**: Any `#[pyfunction]` using `rayon::par_iter()` MUST accept `py: Python`
+and wrap the parallel block in `py.allow_threads(|| { ... })`.
+
+This gotcha caused 16-hour wall-time hangs on the IRIS HPC cluster in v0.8.2.
+
+---
+
 ### `_core.pyi` — Rust Extension Stub Maintenance
 
 `src/krewlyzer/_core.pyi` is a **type stub** for the compiled Rust/PyO3 extension
@@ -384,7 +422,7 @@ python -m mypy src/krewlyzer/ --ignore-missing-imports --no-error-summary
 - [ ] Code follows existing patterns
 - [ ] Added/updated tests
 - [ ] Updated documentation
-- [ ] Ran `pytest tests/` — 244 pass, 4 skipped
+- [ ] Ran `pytest tests/` — 357 pass, 4 skipped
 - [ ] If Rust functions changed: updated `src/krewlyzer/_core.pyi` stub
 - [ ] Ran Python lint (matches CI lint job):
     ```bash

nextflow/main.nf

@@ -33,7 +33,7 @@ include { KREWLYZER } from './workflows/krewlyzer/main'
 def helpMessage() {
     log.info """
     ================================================================
-     K R E W L Y Z E R   P I P E L I N E  (v${workflow.manifest.version ?: '0.8.2'})
+     K R E W L Y Z E R   P I P E L I N E  (v${workflow.manifest.version ?: '0.8.3'})
     ================================================================
      Usage:
      nextflow run main.nf --samplesheet samples.csv --ref hg19.fa [options]

nextflow/modules/local/krewlyzer/build_pon/main.nf

@@ -10,7 +10,7 @@
 process KREWLYZER_BUILD_PON {
     tag "$assay"
     label 'process_high'
-    container "ghcr.io/msk-access/krewlyzer:0.8.2"
+    container "ghcr.io/msk-access/krewlyzer:0.8.3"
 
     input:
     path sample_list          // Text file with BAM/CRAM/BED.gz paths (one per line)
@@ -60,7 +60,7 @@ process KREWLYZER_BUILD_PON {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        krewlyzer: 0.8.2
+        krewlyzer: 0.8.3
     END_VERSIONS
     """
 }

nextflow/modules/local/krewlyzer/extract/main.nf

@@ -10,7 +10,7 @@
 process KREWLYZER_EXTRACT {
     tag "$meta.id"
     label 'process_medium'
-    container "ghcr.io/msk-access/krewlyzer:0.8.2"
+    container "ghcr.io/msk-access/krewlyzer:0.8.3"
 
     input:
     tuple val(meta), path(bam), path(bai)
@@ -65,7 +65,7 @@ process KREWLYZER_EXTRACT {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        krewlyzer: 0.8.2
+        krewlyzer: 0.8.3
     END_VERSIONS
     """
 }

nextflow/modules/local/krewlyzer/filter_maf/main.nf

@@ -53,9 +53,10 @@ process FILTER_MAF {
         unique_tsbs = set()
 
         for line in infile:
-            outfile.write(line)  # always write — no filtering
+            # Skip comment lines (not needed by downstream mFSD parser)
             if line.startswith('#'):
                 continue
+            outfile.write(line)  # pass through all data rows
             fields = line.strip().split('\\t')
             if tsb_col is None:
                 for i, col in enumerate(fields):
@@ -70,7 +71,7 @@ process FILTER_MAF {
     if len(unique_tsbs) > 1:
         print(f"WARNING: MAF contains {len(unique_tsbs)} unique Tumor_Sample_Barcodes: {unique_tsbs}", file=sys.stderr)
         print(f"WARNING: single_sample_maf=true so no filtering applied — all samples included", file=sys.stderr)
-    if unique_tsbs and not any(sample_id_lower in tsb.lower() for tsb in unique_tsbs):
+    if unique_tsbs and not any(sample_id_lower == tsb.lower() for tsb in unique_tsbs):
         print(f"WARNING: sample '{sample_id}' not found in Tumor_Sample_Barcode values: {unique_tsbs}", file=sys.stderr)
 
     with open('versions.yml', 'w') as vf:
@@ -95,9 +96,8 @@ process FILTER_MAF {
         tsb_col = None
 
         for line in infile:
-            # Handle comment lines (keep them)
+            # Skip comment lines (not needed by downstream mFSD parser)
             if line.startswith('#'):
-                outfile.write(line)
                 continue
 
             fields = line.strip().split('\\t')
@@ -119,9 +119,9 @@ process FILTER_MAF {
 
             total_rows += 1
 
-            # Filter: check if sample_id is a substring of Tumor_Sample_Barcode
+            # Filter: exact case-insensitive match on Tumor_Sample_Barcode
             if tsb_col is not None:
-                if len(fields) > tsb_col and sample_id_lower in fields[tsb_col].lower():
+                if len(fields) > tsb_col and sample_id_lower == fields[tsb_col].lower():
                     outfile.write(line)
                     variant_count += 1
             else:

nextflow/modules/local/krewlyzer/fsc/main.nf

@@ -11,7 +11,7 @@
 process KREWLYZER_FSC {
     tag "$meta.id"
     label 'process_medium'
-    container "ghcr.io/msk-access/krewlyzer:0.8.2"
+    container "ghcr.io/msk-access/krewlyzer:0.8.3"
 
     input:
     tuple val(meta), path(bed)
@@ -64,7 +64,7 @@ process KREWLYZER_FSC {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        krewlyzer: 0.8.2
+        krewlyzer: 0.8.3
     END_VERSIONS
     """
 }

nextflow/modules/local/krewlyzer/fsd/main.nf

@@ -10,7 +10,7 @@
 process KREWLYZER_FSD {
     tag "$meta.id"
     label 'process_medium'
-    container "ghcr.io/msk-access/krewlyzer:0.8.2"
+    container "ghcr.io/msk-access/krewlyzer:0.8.3"
 
     input:
     tuple val(meta), path(bed)
@@ -57,7 +57,7 @@ process KREWLYZER_FSD {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        krewlyzer: 0.8.2
+        krewlyzer: 0.8.3
     END_VERSIONS
     """
 }

nextflow/modules/local/krewlyzer/fsr/main.nf

@@ -10,7 +10,7 @@
 process KREWLYZER_FSR {
     tag "$meta.id"
     label 'process_medium'
-    container "ghcr.io/msk-access/krewlyzer:0.8.2"
+    container "ghcr.io/msk-access/krewlyzer:0.8.3"
 
     input:
     tuple val(meta), path(bed)
@@ -60,7 +60,7 @@ process KREWLYZER_FSR {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        krewlyzer: 0.8.2
+        krewlyzer: 0.8.3
     END_VERSIONS
     """
 }

nextflow/modules/local/krewlyzer/mfsd/main.nf

@@ -13,7 +13,7 @@
 process KREWLYZER_MFSD {
     tag "$meta.id"
     label 'process_medium'
-    container "ghcr.io/msk-access/krewlyzer:0.8.2"
+    container "ghcr.io/msk-access/krewlyzer:0.8.3"
 
     input:
     tuple val(meta), path(bam), path(bai), path(variants)
@@ -81,7 +81,7 @@ process KREWLYZER_MFSD {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        krewlyzer: 0.8.2
+        krewlyzer: 0.8.3
     END_VERSIONS
     """
 }