-
Notifications
You must be signed in to change notification settings - Fork 15
CI container improvements inc. standalone Cetmodules #661
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
greenc-FNAL
wants to merge
5
commits into
Framework-R-D:main
Choose a base branch
from
greenc-FNAL:feature/container-tweaks-with-standalone-cetmodules
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit
Hold shift + click to select a range
e250a00
ci: upgrade Spack repos and fix API compatibility issues
greenc-FNAL ac0475e
ci: add type hints and docstrings to upgrade_repos.py
greenc-FNAL e282f41
Update ci/upgrade_repos.py
greenc-FNAL 413cedf
Remove unwanted duplication of cleanup blocks
greenc-FNAL 0746f9e
ci: refine Spack repository upgrade script
greenc-FNAL File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| #!/usr/bin/env spack python | ||
| """Utility to upgrade Spack repository API versions and fix legacy package imports.""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import os | ||
| import re | ||
| from pathlib import Path | ||
|
|
||
| # 1. Leverage Spack internal API to dynamically find all active repositories | ||
| import spack.repo | ||
| from spack.repo import Repo | ||
| from spack.vendor.ruamel.yaml import YAML | ||
|
|
||
| # Mapping old broken imports to modern Spack equivalents | ||
| # Mapping old variants to modern canonical Spack equivalents | ||
| REPLACEMENTS: dict[str, str] = { | ||
| # Fixes the 'llnl.util.filesystem' mistake from the previous run | ||
| r"import\s+llnl\.util\.filesystem\s+as\s+filesystem": ( | ||
| "import spack.util.filesystem as filesystem" | ||
| ), | ||
| r"from\s+llnl\.util\.filesystem\s+import": "from spack.util.filesystem import", | ||
| # Standard catch-alls for any recipes that haven't been touched yet | ||
| r"from\s+spack\.llnl\.util\s+import\s+filesystem": ( | ||
| "import spack.util.filesystem as filesystem" | ||
| ), | ||
| r"from\s+spack\.llnl\.util\.filesystem\s+import": "from spack.util.filesystem import", | ||
| r"import\s+spack\.llnl\.util\.filesystem": "import spack.util.filesystem as filesystem", | ||
| } | ||
|
|
||
|
|
||
| def upgrade_spack_repo_api(repo_obj: Repo) -> None: | ||
| """Upgrades the repo.yaml api setting using Spack's Repo object.""" | ||
|
coderabbitai[bot] marked this conversation as resolved.
Outdated
|
||
| # repo_obj.root gives the base path of the specific repository | ||
| repo_path = Path(repo_obj.root) | ||
| yaml_file = repo_path / "repo.yaml" | ||
|
|
||
| if not yaml_file.exists(): | ||
| return | ||
|
|
||
| yaml = YAML() | ||
| yaml.preserve_quotes = True | ||
|
|
||
| try: | ||
| data = yaml.load(yaml_file) | ||
| if "repo" in data: | ||
| current_api = str(data["repo"].get("api", "v2.0")) | ||
| target_api = "v2.2" | ||
| current_version = tuple( | ||
| int(part) for part in current_api.removeprefix("v").split(".") | ||
| ) | ||
| target_version = tuple(int(part) for part in target_api.removeprefix("v").split(".")) | ||
| if current_version < target_version: | ||
| print( | ||
| f"Upgrading {repo_obj.namespace} from API {current_api} to {target_api}..." | ||
| ) | ||
| data["repo"]["api"] = target_api | ||
| with open(yaml_file, "w") as f: | ||
| yaml.dump(data, f) | ||
| print(f" [FIXED] {yaml_file}") | ||
| except Exception as e: | ||
| print(f"Error updating {yaml_file}: {e}") | ||
|
|
||
|
|
||
| def clean_package_imports(repo_obj: Repo) -> None: | ||
| """Scans and fixes package.py files inside the discovered repository.""" | ||
| # repo_obj.root handles varied directory structures seamlessly | ||
| packages_path = Path(repo_obj.root) / "packages" | ||
| if not packages_path.exists(): | ||
| return | ||
|
|
||
| print(f"Scanning recipes in {repo_obj.namespace}: {packages_path}") | ||
| count = 0 | ||
|
|
||
| for root, _, files in os.walk(packages_path): | ||
| for file in files: | ||
| if file == "package.py": | ||
| file_path = Path(root) / file | ||
| with open(file_path, "r", encoding="utf-8") as f: | ||
| content = f.read() | ||
|
|
||
| modified_content = content | ||
| for pattern, replacement in REPLACEMENTS.items(): | ||
| modified_content = re.sub(pattern, replacement, modified_content) | ||
|
coderabbitai[bot] marked this conversation as resolved.
|
||
|
|
||
| if modified_content != content: | ||
| with open(file_path, "w", encoding="utf-8") as f: | ||
| f.write(modified_content) | ||
| print(f" [FIXED] {file_path.relative_to(packages_path)}") | ||
| count += 1 | ||
|
|
||
| if count > 0: | ||
| print(f"Successfully patched {count} package recipes.\n") | ||
|
|
||
|
|
||
| # --- Execute Refactoring via Spack Context --- | ||
| def main() -> None: | ||
| """Main entry point to upgrade Spack repositories and clean package imports.""" | ||
| # Spack's repo.path contains a list of all active Repo instances | ||
| active_repos = spack.repo.PATH.repos | ||
|
|
||
| for repo in active_repos: | ||
| # Skip the core builtin Spack repo to avoid touching core code | ||
| if repo.namespace == "builtin": | ||
| continue | ||
|
|
||
| upgrade_spack_repo_api(repo) | ||
| clean_package_imports(repo) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.