diff --git a/ci/Dockerfile b/ci/Dockerfile index ad534b5bd..5655b5416 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -122,6 +122,11 @@ PREP_SPACK_GROUP # Configure Spack: add repos, compilers, externals, and cache locations COPY packages.yaml /tmp/packages.yaml +COPY upgrade_repos.py /tmp/upgrade_repos.py + +ENV GNUPGHOME=/run/gpg +ENV SPACK_GNUPGHOME=/run/gpg + RUN <<'SPACK_CONFIGURE_DEFAULTS' set -euo pipefail @@ -137,15 +142,6 @@ find "$SPACK_REPO_ROOT" -type d -exec chmod g+s {} + . /spack/share/spack/setup-env.sh -# Install settings from our copied packages.yaml -spack config --scope site add -f /tmp/packages.yaml -rm -f /tmp/packages.yaml - -# Recipe repos -spack repo add --scope site $SPACK_REPO_ROOT/phlex-spack-recipes/spack_repo/phlex -spack repo add --scope site $SPACK_REPO_ROOT/fnal_art/spack_repo/fnal_art -spack repo add --scope site $SPACK_REPO_ROOT/spack-packages/repos/spack_repo/builtin - # Other config settings spack config --scope defaults add config:build_stage:/opt/spack-stage spack config --scope defaults add config:source_cache:/opt/spack-cache/downloads @@ -155,16 +151,31 @@ spack config --scope defaults add config:install_tree:padded_length:255 # Find compilers (destination packages.yaml for Spack >=1.0) spack --timestamp compiler find --scope site +# Install settings from our copied packages.yaml +spack config --scope site add -f /tmp/packages.yaml +rm -f /tmp/packages.yaml + # Discover externals # # - Python is excluded because the python provided by the system does # not necessarily include the Python.h header file. We therefore need # Spack to build Python. spack --timestamp external find --exclude python -SPACK_CONFIGURE_DEFAULTS -ENV GNUPGHOME=/run/gpg -ENV SPACK_GNUPGHOME=/run/gpg +# Configure recipe repos +spack repo add --scope site $SPACK_REPO_ROOT/phlex-spack-recipes/spack_repo/phlex +spack repo add --scope site $SPACK_REPO_ROOT/fnal_art/spack_repo/fnal_art +spack repo add --scope site $SPACK_REPO_ROOT/spack-packages/repos/spack_repo/builtin + +spack repo list +chmod +x /tmp/upgrade_repos.py +spack python /tmp/upgrade_repos.py +rm -f /tmp/upgrade_repos.py +spack repo list +spack clean -m +spack audit configs + +SPACK_CONFIGURE_DEFAULTS ######################################################################## # Configure binary mirrors used during installs @@ -214,9 +225,9 @@ set -euo pipefail # Install GCC outside the Phlex development environment. . /spack/share/spack/setup-env.sh -spack --timestamp install --fail-fast -j "$(nproc)" -p 1 \ +spack --timestamp install --fail-fast -j "$(nproc)" \ --no-check-signature \ - gcc@15.2 target=x86_64_v3 \ + gcc@15.3 target=x86_64_v3 \ +binutils+graphite~nvptx+piclibs+strip \ build_type=Release \ languages=c,c++,fortran,lto \ @@ -235,7 +246,7 @@ if [ -d "/build-cache" ]; then fi # Make GCC external to simplify concretization -spack compiler find $(spack location -i gcc@15.2) +spack compiler find $(spack location -i gcc@15) spack clean -dfs SPACK_INSTALL_GCC @@ -277,7 +288,8 @@ set -euo pipefail rm -rf "$PHLEX_SPACK_ENV/.spack-env/view" rm -rf "$PHLEX_SPACK_ENV"/.spack-env/view.old.* -spack --timestamp install --fail-fast -j $(nproc) -p 1 \ + +spack --timestamp install --fail-fast -j $(nproc) \ --no-add --only-concrete --no-check-signature \ cmake lcov ninja py-gcovr py-pytest-cov py-pyyaml @@ -331,7 +343,7 @@ set -euo pipefail rm -rf "$PHLEX_SPACK_ENV/.spack-env/view" rm -rf "$PHLEX_SPACK_ENV"/.spack-env/view.old.* -spack --timestamp install --fail-fast -j $(nproc) -p 1 \ +spack --timestamp install --fail-fast -j $(nproc) \ --no-add --only-concrete --no-check-signature llvm if [ -d "/build-cache" ]; then @@ -363,7 +375,7 @@ set -euo pipefail rm -rf "$PHLEX_SPACK_ENV/.spack-env/view" rm -rf "$PHLEX_SPACK_ENV"/.spack-env/view.old.* -spack --timestamp install --fail-fast -j $(nproc) -p 1 \ +spack --timestamp install --fail-fast -j $(nproc) \ --no-add --only-concrete --no-check-signature \ $(spack find -r --no-groups | sed -Ene 's&^ - &&p' | grep -v phlex) @@ -396,7 +408,7 @@ set -euo pipefail rm -rf "$PHLEX_SPACK_ENV/.spack-env/view" rm -rf "$PHLEX_SPACK_ENV"/.spack-env/view.old.* -spack --timestamp install --fail-fast -j $(nproc) -p 1 \ +spack --timestamp install --fail-fast -j $(nproc) \ --no-add --only-concrete --only dependencies --no-check-signature \ phlex diff --git a/ci/spack.yaml b/ci/spack.yaml index c01ae9a75..0f14dcf89 100644 --- a/ci/spack.yaml +++ b/ci/spack.yaml @@ -8,6 +8,7 @@ spack: specs: - phlex - catch2 + - cetmodules - cmake - lcov - ninja @@ -37,10 +38,11 @@ spack: require: - "~qtgui" - "@4:" + - "target=x86_64_v3" llvm: require: - - "%c,cxx=gcc@15" + - "target=x86_64_v3" phlex: require: diff --git a/ci/upgrade_repos.py b/ci/upgrade_repos.py new file mode 100644 index 000000000..7043a9365 --- /dev/null +++ b/ci/upgrade_repos.py @@ -0,0 +1,116 @@ +#!/usr/bin/env spack python +"""Utility to upgrade Spack repository API versions and fix legacy package imports.""" + +from __future__ import annotations + +import os +import re +from pathlib import Path + +# 1. Leverage Spack internal API to dynamically find all active repositories +import spack.repo +from spack.repo import Repo +from spack.vendor.ruamel.yaml import YAML + +# Mapping old broken imports to modern Spack equivalents +# Mapping old variants to modern canonical Spack equivalents +REPLACEMENTS: dict[str, str] = { + # Fixes the 'llnl.util.filesystem' mistake from the previous run + r"import\s+llnl\.util\.filesystem\s+as\s+filesystem": ( + "import spack.util.filesystem as filesystem" + ), + r"from\s+llnl\.util\.filesystem\s+import": "from spack.util.filesystem import", + # Standard catch-alls for any recipes that haven't been touched yet + r"from\s+spack\.llnl\.util\s+import\s+filesystem": ( + "import spack.util.filesystem as filesystem" + ), + r"from\s+spack\.llnl\.util\.filesystem\s+import": "from spack.util.filesystem import", + r"(import\s+)spack\.llnl\.util\.filesystem": r"\1spack.util.filesystem", +} + + +def upgrade_spack_repo_api(repo_obj: Repo) -> None: + """Upgrade the repo.yaml API setting. + + Args: + repo_obj: Spack repository to migrate. + """ + # repo_obj.root gives the base path of the specific repository + repo_path = Path(repo_obj.root) + yaml_file = repo_path / "repo.yaml" + + if not yaml_file.exists(): + return + + yaml = YAML() + yaml.preserve_quotes = True + + try: + data = yaml.load(yaml_file) + if "repo" in data: + current_api = str(data["repo"].get("api", "v2.0")) + target_api = "v2.2" + current_version = tuple(int(part) for part in current_api.removeprefix("v").split(".")) + target_version = tuple(int(part) for part in target_api.removeprefix("v").split(".")) + if current_version < target_version: + print(f"Upgrading {repo_obj.namespace} from API {current_api} to {target_api}...") + data["repo"]["api"] = target_api + with open(yaml_file, "w") as f: + yaml.dump(data, f) + print(f" [FIXED] {yaml_file}") + except Exception as exc: + raise RuntimeError(f"Error updating {yaml_file}") from exc + + +def clean_package_imports(repo_obj: Repo) -> None: + """Scan and fix package.py files inside the discovered repository. + + Args: + repo_obj: Spack repository whose recipes should be patched. + """ + # repo_obj.root handles varied directory structures seamlessly + packages_path = Path(repo_obj.root) / "packages" + if not packages_path.exists(): + return + + print(f"Scanning recipes in {repo_obj.namespace}: {packages_path}") + count = 0 + + for root, _, files in os.walk(packages_path): + for file in files: + if file == "package.py": + file_path = Path(root) / file + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() + + modified_content = content + for pattern, replacement in REPLACEMENTS.items(): + modified_content = re.sub(pattern, replacement, modified_content) + + if modified_content != content: + with open(file_path, "w", encoding="utf-8") as f: + f.write(modified_content) + print(f" [FIXED] {file_path.relative_to(packages_path)}") + count += 1 + + if count > 0: + print(f"Successfully patched {count} package recipes.\n") + + +# --- Execute Refactoring via Spack Context --- +def main() -> None: + """Main entry point to upgrade Spack repositories and clean package imports.""" + # Spack's repo.path contains a list of all active Repo instances + active_repos = spack.repo.PATH.repos + + for repo in active_repos: + # Skip the core builtin Spack repo to avoid touching core code + if repo.namespace == "builtin": + continue + + upgrade_spack_repo_api(repo) + clean_package_imports(repo) + + +if __name__ == "__main__": + main()