fix(gemma4): finalize 27B MoE LoRA recipe (released install form, accurate transformers/MoE notes)

Author: aniruddh-alt

configs/recipes/gemma4/README.md

@@ -69,5 +69,5 @@ oumi launch up -c oumi://configs/recipes/gemma4/sft/e4b_lora/gcp_job.yaml --clus
 To launch Gemma 4 27B (MoE) LoRA training on a remote GCP 8x A100 cluster:
 
 ```shell
-oumi launch up -c configs/recipes/gemma4/sft/27b_lora/gcp_job.yaml --cluster gemma4-27b-lora
+oumi launch up -c oumi://configs/recipes/gemma4/sft/27b_lora/gcp_job.yaml --cluster gemma4-27b-lora
 ```

configs/recipes/gemma4/sft/27b_lora/gcp_job.yaml

@@ -1,17 +1,12 @@
 # Job config to LoRA tune Gemma 4 27B (Mixture-of-Experts).
 #
-# NOTE: This recipe installs Oumi in EDITABLE mode from the synced working dir
-# (`-e '.[gpu]'`) and runs the LOCAL config path (no `oumi://` prefix). This is
-# required while the Gemma 4 27B recipe and the `lora_exclude_modules` feature
-# are pre-release and not yet on `main`.
-#
 # Requirements:
 #   - Set up SkyPilot GCP: https://oumi.ai/docs/en/latest/user_guides/launch/launch.html#setup
 #   - apache-2.0 license, no gating: https://huggingface.co/google/gemma-4-26B-A4B-it
 #   - Log into WandB (`wandb login`) or disable `enable_wandb`
 #
 # Usage:
-#   oumi launch up -c configs/recipes/gemma4/sft/27b_lora/gcp_job.yaml --cluster gemma4-27b-lora
+#   oumi launch up -c oumi://configs/recipes/gemma4/sft/27b_lora/gcp_job.yaml --cluster gemma4-27b-lora
 #
 # See Also:
 #   - Documentation: https://oumi.ai/docs/en/latest/user_guides/launch/launch.html
@@ -24,8 +19,7 @@ name: gemma4-27b-lora
 resources:
   cloud: gcp
   accelerators: "A100:8"
-  use_spot: true
-  region: europe-west4  # 40GB A100s available here with abundant spot quota.
+  use_spot: false
   disk_size: 500  # Disk size in GBs
 
 working_dir: .
@@ -39,15 +33,15 @@ envs:
 
 setup: |
   set -e
-  pip install uv && SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OUMI=0.0.0.dev0 uv pip install --system -e '.[gpu]' hf_transfer
+  pip install uv && uv pip install --system oumi[gpu] hf_transfer
 
 run: |
   set -e  # Exit if any command failed.
   source ./configs/examples/misc/sky_init.sh
 
   set -x
   oumi distributed torchrun -m oumi train \
-      -c configs/recipes/gemma4/sft/27b_lora/train.yaml \
+      -c oumi://configs/recipes/gemma4/sft/27b_lora/train.yaml \
       --training.run_name "${OUMI_RUN_NAME}.${SKYPILOT_TASK_ID}"
 
   echo "Node ${SKYPILOT_NODE_RANK} is all done!"

configs/recipes/gemma4/sft/27b_lora/train.yaml

@@ -7,11 +7,12 @@
 #   - LoRA is scoped to the text transformer only. The vision tower and the
 #     multimodal projector are excluded via `lora_exclude_modules` below; they
 #     share projection-like names with the text model but PEFT cannot adapt them.
-#     The standard projection targets suffix-match the expert MLPs; the MoE
-#     router ("gate") is left untouched.
+#     NOTE: on this MoE the standard gate_proj/up_proj/down_proj names do not
+#     match the (fused) expert MLP modules, so LoRA adapts only the attention
+#     projections (~9.3M params). Adapting the experts needs their module names.
 #
 # Requirements:
-#   - transformers >= 5.10.0
+#   - transformers >= 5.5.4
 #   - peft (installed automatically with oumi)
 #   - apache-2.0 license, no gating: https://huggingface.co/google/gemma-4-26B-A4B-it
 #   - Log into WandB (`wandb login`) or disable `enable_wandb`
@@ -44,6 +45,10 @@ training:
   save_final_model: true
   num_train_epochs: 1
   per_device_train_batch_size: 1
+  # NOTE: gradient accumulation inflates reported loss for Gemma 4 (~4x) due
+  # to a missing `accepts_loss_kwargs = False` on Gemma4ForConditionalGeneration.
+  # Fixed on transformers main but not yet released as of 5.5.4. See
+  # huggingface/transformers#40564 (same bug existed in Gemma 3).
   gradient_accumulation_steps: 8
   max_grad_norm: 1.0